* [PATCH for-next v3 1/5] RDMA/erdma: Unwind kernel QP initialization failures
2026-09-08 3:37 [PATCH for-next v3 0/5] RDMA/erdma: Support non-contiguous kernel queue buffers Cheng Xu
@ 2026-09-08 3:37 ` Cheng Xu
2026-09-08 3:37 ` [PATCH for-next v3 2/5] RDMA/erdma: Support non-contiguous kernel QP buffers Cheng Xu
` (3 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Cheng Xu @ 2026-09-08 3:37 UTC (permalink / raw)
To: jgg, leon; +Cc: linux-rdma, KaiShen
Allocation failures currently use the normal kernel QP teardown path.
Unwind failures directly so free_kernel_qp() only handles complete QPs.
Signed-off-by: Cheng Xu <chengyou@linux.alibaba.com>
---
drivers/infiniband/hw/erdma/erdma_verbs.c | 58 ++++++++++++-----------
1 file changed, 31 insertions(+), 27 deletions(-)
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c
index 65b1af1e6623..57a35f8b32cf 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.c
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.c
@@ -512,25 +512,14 @@ static void free_kernel_qp(struct erdma_qp *qp)
vfree(qp->kern_qp.swr_tbl);
vfree(qp->kern_qp.rwr_tbl);
- if (qp->kern_qp.sq_buf)
- dma_free_coherent(&dev->pdev->dev,
- qp->attrs.sq_size << SQEBB_SHIFT,
- qp->kern_qp.sq_buf,
- qp->kern_qp.sq_buf_dma_addr);
-
- if (qp->kern_qp.sq_dbrec)
- dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,
- qp->kern_qp.sq_dbrec_dma);
-
- if (qp->kern_qp.rq_buf)
- dma_free_coherent(&dev->pdev->dev,
- qp->attrs.rq_size << RQE_SHIFT,
- qp->kern_qp.rq_buf,
- qp->kern_qp.rq_buf_dma_addr);
-
- if (qp->kern_qp.rq_dbrec)
- dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,
- qp->kern_qp.rq_dbrec_dma);
+ dma_free_coherent(&dev->pdev->dev, qp->attrs.sq_size << SQEBB_SHIFT,
+ qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr);
+ dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,
+ qp->kern_qp.sq_dbrec_dma);
+ dma_free_coherent(&dev->pdev->dev, qp->attrs.rq_size << RQE_SHIFT,
+ qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr);
+ dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,
+ qp->kern_qp.rq_dbrec_dma);
}
static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
@@ -551,36 +540,51 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64));
+ if (!kqp->swr_tbl)
+ return -ENOMEM;
+
kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64));
- if (!kqp->swr_tbl || !kqp->rwr_tbl)
- goto err_out;
+ if (!kqp->rwr_tbl)
+ goto err_free_swr_tbl;
size = qp->attrs.sq_size << SQEBB_SHIFT;
kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
&kqp->sq_buf_dma_addr, GFP_KERNEL);
if (!kqp->sq_buf)
- goto err_out;
+ goto err_free_rwr_tbl;
kqp->sq_dbrec =
dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma);
if (!kqp->sq_dbrec)
- goto err_out;
+ goto err_free_sq_buf;
size = qp->attrs.rq_size << RQE_SHIFT;
kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
&kqp->rq_buf_dma_addr, GFP_KERNEL);
if (!kqp->rq_buf)
- goto err_out;
+ goto err_free_sq_dbrec;
kqp->rq_dbrec =
dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma);
if (!kqp->rq_dbrec)
- goto err_out;
+ goto err_free_rq_buf;
return 0;
-err_out:
- free_kernel_qp(qp);
+err_free_rq_buf:
+ dma_free_coherent(&dev->pdev->dev, size, kqp->rq_buf,
+ kqp->rq_buf_dma_addr);
+err_free_sq_dbrec:
+ dma_pool_free(dev->db_pool, kqp->sq_dbrec, kqp->sq_dbrec_dma);
+err_free_sq_buf:
+ size = qp->attrs.sq_size << SQEBB_SHIFT;
+ dma_free_coherent(&dev->pdev->dev, size, kqp->sq_buf,
+ kqp->sq_buf_dma_addr);
+err_free_rwr_tbl:
+ vfree(kqp->rwr_tbl);
+err_free_swr_tbl:
+ vfree(kqp->swr_tbl);
+
return -ENOMEM;
}
--
2.31.1
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH for-next v3 2/5] RDMA/erdma: Support non-contiguous kernel QP buffers
2026-09-08 3:37 [PATCH for-next v3 0/5] RDMA/erdma: Support non-contiguous kernel queue buffers Cheng Xu
2026-09-08 3:37 ` [PATCH for-next v3 1/5] RDMA/erdma: Unwind kernel QP initialization failures Cheng Xu
@ 2026-09-08 3:37 ` Cheng Xu
2026-09-08 3:37 ` [PATCH for-next v3 3/5] RDMA/erdma: Support non-contiguous kernel CQ buffers Cheng Xu
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Cheng Xu @ 2026-09-08 3:37 UTC (permalink / raw)
To: jgg, leon; +Cc: linux-rdma, KaiShen
A single coherent allocation for kernel QP queues can fail for large
queues when memory is fragmented.
Allocate page-sized coherent buffers and describe them with the existing
MTT. Keep the userspace QP path unchanged.
Signed-off-by: Cheng Xu <chengyou@linux.alibaba.com>
---
drivers/infiniband/hw/erdma/erdma_cq.c | 4 +-
drivers/infiniband/hw/erdma/erdma_qp.c | 38 ++--
drivers/infiniband/hw/erdma/erdma_verbs.c | 208 +++++++++++++---------
drivers/infiniband/hw/erdma/erdma_verbs.h | 40 ++++-
4 files changed, 184 insertions(+), 106 deletions(-)
diff --git a/drivers/infiniband/hw/erdma/erdma_cq.c b/drivers/infiniband/hw/erdma/erdma_cq.c
index 1f456327e63c..4cc0254b0ab5 100644
--- a/drivers/infiniband/hw/erdma/erdma_cq.c
+++ b/drivers/infiniband/hw/erdma/erdma_cq.c
@@ -161,8 +161,8 @@ static int erdma_poll_one_cqe(struct erdma_cq *cq, struct ib_wc *wc)
if (qtype == ERDMA_CQE_QTYPE_SQ) {
id_table = kern_qp->swr_tbl;
depth = qp->attrs.sq_size;
- wqe_hdr = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
- qp->attrs.sq_size, SQEBB_SHIFT);
+ wqe_hdr = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx,
+ qp->attrs.sq_size, SQEBB_SHIFT);
kern_qp->sq_ci =
FIELD_GET(ERDMA_SQE_HDR_WQEBB_CNT_MASK, *wqe_hdr) +
wqe_idx + 1;
diff --git a/drivers/infiniband/hw/erdma/erdma_qp.c b/drivers/infiniband/hw/erdma/erdma_qp.c
index e002343832f7..fc335b674746 100644
--- a/drivers/infiniband/hw/erdma/erdma_qp.c
+++ b/drivers/infiniband/hw/erdma/erdma_qp.c
@@ -259,8 +259,8 @@ static void erdma_reset_qp(struct erdma_qp *qp)
qp->kern_qp.rq_ci = 0;
memset(qp->kern_qp.swr_tbl, 0, qp->attrs.sq_size * sizeof(u64));
memset(qp->kern_qp.rwr_tbl, 0, qp->attrs.rq_size * sizeof(u64));
- memset(qp->kern_qp.sq_buf, 0, qp->attrs.sq_size << SQEBB_SHIFT);
- memset(qp->kern_qp.rq_buf, 0, qp->attrs.rq_size << RQE_SHIFT);
+ erdma_kmem_clear(&qp->kern_qp.sq_mem);
+ erdma_kmem_clear(&qp->kern_qp.rq_mem);
erdma_remove_cqes_of_qp(&qp->scq->ibcq, QP_ID(qp));
if (qp->rcq != qp->scq)
erdma_remove_cqes_of_qp(&qp->rcq->ibcq, QP_ID(qp));
@@ -332,8 +332,8 @@ static int fill_inline_data(struct erdma_qp *qp,
wqe_idx += (sgl_offset >> SQEBB_SHIFT);
sgl_offset &= (SQEBB_SIZE - 1);
- data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, qp->attrs.sq_size,
- SQEBB_SHIFT);
+ data = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx,
+ qp->attrs.sq_size, SQEBB_SHIFT);
while (i < send_wr->num_sge) {
bytes += send_wr->sg_list[i].length;
@@ -356,8 +356,9 @@ static int fill_inline_data(struct erdma_qp *qp,
wqe_idx += (sgl_offset >> SQEBB_SHIFT);
sgl_offset &= (SQEBB_SIZE - 1);
- data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
- qp->attrs.sq_size, SQEBB_SHIFT);
+ data = erdma_kmem_get_entry(&qp->kern_qp.sq_mem,
+ wqe_idx, qp->attrs.sq_size,
+ SQEBB_SHIFT);
if (!remain_size)
break;
}
@@ -385,8 +386,8 @@ static int fill_sgl(struct erdma_qp *qp, const struct ib_send_wr *send_wr,
while (i < send_wr->num_sge) {
wqe_idx += (sgl_offset >> SQEBB_SHIFT);
sgl_offset &= (SQEBB_SIZE - 1);
- sgl = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
- qp->attrs.sq_size, SQEBB_SHIFT);
+ sgl = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx,
+ qp->attrs.sq_size, SQEBB_SHIFT);
bytes += send_wr->sg_list[i].length;
memcpy(sgl + sgl_offset, &send_wr->sg_list[i],
@@ -463,8 +464,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
send_wr->opcode != IB_WR_SEND_WITH_IMM)
return -EINVAL;
- entry = get_queue_entry(qp->kern_qp.sq_buf, idx, qp->attrs.sq_size,
- SQEBB_SHIFT);
+ entry = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx,
+ qp->attrs.sq_size, SQEBB_SHIFT);
/* Clear the SQE header section. */
*entry = 0;
@@ -524,8 +525,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
read_sqe->sink_to_h =
cpu_to_le32(upper_32_bits(send_wr->sg_list[0].addr));
- sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
- qp->attrs.sq_size, SQEBB_SHIFT);
+ sge = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx + 1,
+ qp->attrs.sq_size, SQEBB_SHIFT);
sge->addr = cpu_to_le64(rdma_wr->remote_addr);
sge->key = cpu_to_le32(rdma_wr->rkey);
sge->length = cpu_to_le32(send_wr->sg_list[0].length);
@@ -569,8 +570,9 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
if (mr->mem.mtt_nents <= ERDMA_MAX_INLINE_MTT_ENTRIES) {
attrs |= FIELD_PREP(ERDMA_SQE_MR_MTT_TYPE_MASK, 0);
/* Copy SGLs to SQE content to accelerate */
- memcpy(get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
- qp->attrs.sq_size, SQEBB_SHIFT),
+ memcpy(erdma_kmem_get_entry(&qp->kern_qp.sq_mem,
+ idx + 1, qp->attrs.sq_size,
+ SQEBB_SHIFT),
mr->mem.mtt->buf, MTT_SIZE(mr->mem.mtt_nents));
wqe_size = sizeof(struct erdma_reg_mr_sqe) +
MTT_SIZE(mr->mem.mtt_nents);
@@ -605,8 +607,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
cpu_to_le64(atomic_wr(send_wr)->compare_add);
}
- sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
- qp->attrs.sq_size, SQEBB_SHIFT);
+ sge = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx + 1,
+ qp->attrs.sq_size, SQEBB_SHIFT);
sge->addr = cpu_to_le64(atomic_wr(send_wr)->remote_addr);
sge->key = cpu_to_le32(atomic_wr(send_wr)->rkey);
sge++;
@@ -702,8 +704,8 @@ static int erdma_post_recv_one(struct erdma_qp *qp,
const struct ib_recv_wr *recv_wr)
{
struct erdma_rqe *rqe =
- get_queue_entry(qp->kern_qp.rq_buf, qp->kern_qp.rq_pi,
- qp->attrs.rq_size, RQE_SHIFT);
+ erdma_kmem_get_entry(&qp->kern_qp.rq_mem, qp->kern_qp.rq_pi,
+ qp->attrs.rq_size, RQE_SHIFT);
rqe->qe_idx = cpu_to_le16(qp->kern_qp.rq_pi + 1);
rqe->qpn = cpu_to_le32(QP_ID(qp));
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c
index 57a35f8b32cf..5725336fa1d4 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.c
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.c
@@ -19,6 +19,10 @@
#include "erdma_cm.h"
#include "erdma_verbs.h"
+static int erdma_alloc_kmem(struct erdma_dev *dev, struct erdma_mem *mem,
+ size_t size);
+static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem);
+
static void assemble_qbuf_mtt_for_cmd(struct erdma_mem *mem, u32 *cfg,
u64 *addr0, u64 *addr1)
{
@@ -41,7 +45,7 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)
struct erdma_dev *dev = to_edev(qp->ibqp.device);
struct erdma_pd *pd = to_epd(qp->ibqp.pd);
struct erdma_cmdq_create_qp_req req;
- struct erdma_uqp *user_qp;
+ struct erdma_mem *sq_mem, *rq_mem;
u64 resp0, resp1;
int err;
@@ -63,67 +67,46 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)
ERDMA_QPT_UD);
if (rdma_is_kernel_res(&qp->ibqp.res)) {
- u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT;
-
- req.sq_cqn_mtt_cfg =
- FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
- pgsz_range) |
- FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
- req.rq_cqn_mtt_cfg =
- FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
- pgsz_range) |
- FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
-
- req.sq_mtt_cfg =
- FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) |
- FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) |
- FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
- ERDMA_MR_MTT_0LEVEL);
- req.rq_mtt_cfg = req.sq_mtt_cfg;
-
- req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr;
- req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr;
+ sq_mem = &qp->kern_qp.sq_mem;
+ rq_mem = &qp->kern_qp.rq_mem;
req.sq_dbrec_dma = qp->kern_qp.sq_dbrec_dma;
req.rq_dbrec_dma = qp->kern_qp.rq_dbrec_dma;
} else {
- user_qp = &qp->user_qp;
- req.sq_cqn_mtt_cfg = FIELD_PREP(
- ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
- ilog2(user_qp->sq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
- req.sq_cqn_mtt_cfg |=
- FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
-
- req.rq_cqn_mtt_cfg = FIELD_PREP(
- ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
- ilog2(user_qp->rq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
- req.rq_cqn_mtt_cfg |=
- FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
-
- req.sq_mtt_cfg = user_qp->sq_mem.page_offset;
- req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
- user_qp->sq_mem.mtt_nents);
-
- req.rq_mtt_cfg = user_qp->rq_mem.page_offset;
- req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
- user_qp->rq_mem.mtt_nents);
-
- assemble_qbuf_mtt_for_cmd(&user_qp->sq_mem, &req.sq_mtt_cfg,
- &req.sq_buf_addr, req.sq_mtt_entry);
- assemble_qbuf_mtt_for_cmd(&user_qp->rq_mem, &req.rq_mtt_cfg,
- &req.rq_buf_addr, req.rq_mtt_entry);
-
- req.sq_dbrec_dma = user_qp->sq_dbrec_dma;
- req.rq_dbrec_dma = user_qp->rq_dbrec_dma;
+ sq_mem = &qp->user_qp.sq_mem;
+ rq_mem = &qp->user_qp.rq_mem;
+ req.sq_dbrec_dma = qp->user_qp.sq_dbrec_dma;
+ req.rq_dbrec_dma = qp->user_qp.rq_dbrec_dma;
+ }
- if (uctx->ext_db.enable) {
- req.sq_cqn_mtt_cfg |=
- FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);
- req.db_cfg =
- FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,
- uctx->ext_db.sdb_off) |
- FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,
- uctx->ext_db.rdb_off);
- }
+ req.sq_cqn_mtt_cfg =
+ FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
+ ilog2(sq_mem->page_size) - ERDMA_HW_PAGE_SHIFT);
+ req.sq_cqn_mtt_cfg |=
+ FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
+ req.rq_cqn_mtt_cfg =
+ FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
+ ilog2(rq_mem->page_size) - ERDMA_HW_PAGE_SHIFT);
+ req.rq_cqn_mtt_cfg |=
+ FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
+
+ req.sq_mtt_cfg =
+ sq_mem->page_offset |
+ FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, sq_mem->mtt_nents);
+ req.rq_mtt_cfg =
+ rq_mem->page_offset |
+ FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, rq_mem->mtt_nents);
+
+ assemble_qbuf_mtt_for_cmd(sq_mem, &req.sq_mtt_cfg, &req.sq_buf_addr,
+ req.sq_mtt_entry);
+ assemble_qbuf_mtt_for_cmd(rq_mem, &req.rq_mtt_cfg, &req.rq_buf_addr,
+ req.rq_mtt_entry);
+ if (uctx && uctx->ext_db.enable) {
+ req.sq_cqn_mtt_cfg |=
+ FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);
+ req.db_cfg = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,
+ uctx->ext_db.sdb_off) |
+ FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,
+ uctx->ext_db.rdb_off);
}
err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0, &resp1,
@@ -512,12 +495,10 @@ static void free_kernel_qp(struct erdma_qp *qp)
vfree(qp->kern_qp.swr_tbl);
vfree(qp->kern_qp.rwr_tbl);
- dma_free_coherent(&dev->pdev->dev, qp->attrs.sq_size << SQEBB_SHIFT,
- qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr);
+ put_mtt_entries(dev, &qp->kern_qp.sq_mem);
dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,
qp->kern_qp.sq_dbrec_dma);
- dma_free_coherent(&dev->pdev->dev, qp->attrs.rq_size << RQE_SHIFT,
- qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr);
+ put_mtt_entries(dev, &qp->kern_qp.rq_mem);
dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,
qp->kern_qp.rq_dbrec_dma);
}
@@ -527,6 +508,7 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
{
struct erdma_kqp *kqp = &qp->kern_qp;
int size;
+ int ret = -ENOMEM;
if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
kqp->sig_all = 1;
@@ -548,44 +530,39 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
goto err_free_swr_tbl;
size = qp->attrs.sq_size << SQEBB_SHIFT;
- kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
- &kqp->sq_buf_dma_addr, GFP_KERNEL);
- if (!kqp->sq_buf)
+ ret = erdma_alloc_kmem(dev, &kqp->sq_mem, size);
+ if (ret)
goto err_free_rwr_tbl;
kqp->sq_dbrec =
dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma);
if (!kqp->sq_dbrec)
- goto err_free_sq_buf;
+ goto err_free_sq_mem;
size = qp->attrs.rq_size << RQE_SHIFT;
- kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
- &kqp->rq_buf_dma_addr, GFP_KERNEL);
- if (!kqp->rq_buf)
+ ret = erdma_alloc_kmem(dev, &kqp->rq_mem, size);
+ if (ret)
goto err_free_sq_dbrec;
kqp->rq_dbrec =
dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma);
if (!kqp->rq_dbrec)
- goto err_free_rq_buf;
+ goto err_free_rq_mem;
return 0;
-err_free_rq_buf:
- dma_free_coherent(&dev->pdev->dev, size, kqp->rq_buf,
- kqp->rq_buf_dma_addr);
+err_free_rq_mem:
+ put_mtt_entries(dev, &kqp->rq_mem);
err_free_sq_dbrec:
dma_pool_free(dev->db_pool, kqp->sq_dbrec, kqp->sq_dbrec_dma);
-err_free_sq_buf:
- size = qp->attrs.sq_size << SQEBB_SHIFT;
- dma_free_coherent(&dev->pdev->dev, size, kqp->sq_buf,
- kqp->sq_buf_dma_addr);
+err_free_sq_mem:
+ put_mtt_entries(dev, &kqp->sq_mem);
err_free_rwr_tbl:
vfree(kqp->rwr_tbl);
err_free_swr_tbl:
vfree(kqp->swr_tbl);
- return -ENOMEM;
+ return ret;
}
static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem)
@@ -830,12 +807,79 @@ static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt)
}
}
+static void erdma_free_kmem(struct erdma_dev *dev, struct erdma_mem *mem)
+{
+ struct scatterlist *sgl = mem->kmem.sgl;
+ u32 i;
+
+ for (i = 0; i < mem->page_cnt; i++)
+ dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, sg_virt(&sgl[i]),
+ sg_dma_address(&sgl[i]));
+
+ kfree(sgl);
+}
+
+static int erdma_alloc_kmem(struct erdma_dev *dev, struct erdma_mem *mem,
+ size_t size)
+{
+ struct scatterlist *sgl;
+ struct erdma_mtt *mtt;
+ void *buf;
+ int ret = -ENOMEM;
+ u32 i;
+
+ mem->type = ERDMA_KMEM;
+ mem->page_size = PAGE_SIZE;
+ mem->page_offset = 0;
+ mem->page_cnt = DIV_ROUND_UP(size, PAGE_SIZE);
+ mem->mtt_nents = mem->page_cnt;
+ mem->len = size;
+
+ sgl = kcalloc(mem->page_cnt, sizeof(*sgl), GFP_KERNEL);
+ if (!sgl)
+ return -ENOMEM;
+
+ sg_init_table(sgl, mem->page_cnt);
+
+ for (i = 0; i < mem->page_cnt; i++) {
+ buf = dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE,
+ &sg_dma_address(&sgl[i]), GFP_KERNEL);
+ if (!buf)
+ goto err_free_pages;
+
+ sg_set_buf(&sgl[i], buf, PAGE_SIZE);
+ sg_dma_len(&sgl[i]) = PAGE_SIZE;
+ }
+
+ mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt), true);
+ if (IS_ERR(mtt)) {
+ ret = PTR_ERR(mtt);
+ goto err_free_pages;
+ }
+
+ for (i = 0; i < mem->page_cnt; i++)
+ mtt->buf[i] = sg_dma_address(&sgl[i]);
+ mem->kmem.sgl = sgl;
+ mem->mtt = mtt;
+
+ return 0;
+
+err_free_pages:
+ while (i--)
+ dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, sg_virt(&sgl[i]),
+ sg_dma_address(&sgl[i]));
+ kfree(sgl);
+
+ return ret;
+}
+
static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
u64 start, u64 len, int access, u64 virt,
unsigned long req_page_size, bool force_continuous)
{
int ret = 0;
+ mem->type = ERDMA_UMEM;
mem->umem = ib_umem_get_va(&dev->ibdev, start, len, access);
if (IS_ERR(mem->umem)) {
ret = PTR_ERR(mem->umem);
@@ -871,10 +915,14 @@ static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)
{
- if (mem->mtt)
+ if (mem->mtt) {
erdma_destroy_mtt(dev, mem->mtt);
+ mem->mtt = NULL;
+ }
- if (mem->umem) {
+ if (mem->type == ERDMA_KMEM) {
+ erdma_free_kmem(dev, mem);
+ } else if (mem->umem) {
ib_umem_release(mem->umem);
mem->umem = NULL;
}
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h
index c73cecf92f61..2bc1e1928712 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.h
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.h
@@ -9,6 +9,7 @@
#include <linux/completion.h>
#include <linux/refcount.h>
+#include <linux/scatterlist.h>
#include "erdma.h"
@@ -111,8 +112,21 @@ struct erdma_mtt {
struct erdma_mtt *low_level;
};
+enum erdma_mem_type {
+ ERDMA_UMEM = 0,
+ ERDMA_KMEM = 1,
+};
+
+struct erdma_kmem {
+ struct scatterlist *sgl;
+};
+
struct erdma_mem {
- struct ib_umem *umem;
+ enum erdma_mem_type type;
+ union {
+ struct ib_umem *umem;
+ struct erdma_kmem kmem;
+ };
struct erdma_mtt *mtt;
u32 page_size;
@@ -124,6 +138,23 @@ struct erdma_mem {
u64 len;
};
+static inline void *erdma_kmem_get_entry(struct erdma_mem *mem, u32 idx,
+ u32 depth, u32 shift)
+{
+ u32 offset = (idx & (depth - 1)) << shift;
+
+ return (u8 *)sg_virt(&mem->kmem.sgl[offset >> PAGE_SHIFT]) +
+ offset_in_page(offset);
+}
+
+static inline void erdma_kmem_clear(struct erdma_mem *mem)
+{
+ u32 i;
+
+ for (i = 0; i < mem->page_cnt; i++)
+ memset(sg_virt(&mem->kmem.sgl[i]), 0, PAGE_SIZE);
+}
+
struct erdma_mr {
struct ib_mr ibmr;
struct erdma_mem mem;
@@ -183,11 +214,8 @@ struct erdma_kqp {
void __iomem *hw_sq_db;
void __iomem *hw_rq_db;
- void *sq_buf;
- dma_addr_t sq_buf_dma_addr;
-
- void *rq_buf;
- dma_addr_t rq_buf_dma_addr;
+ struct erdma_mem sq_mem;
+ struct erdma_mem rq_mem;
void *sq_dbrec;
void *rq_dbrec;
--
2.31.1
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH for-next v3 3/5] RDMA/erdma: Support non-contiguous kernel CQ buffers
2026-09-08 3:37 [PATCH for-next v3 0/5] RDMA/erdma: Support non-contiguous kernel queue buffers Cheng Xu
2026-09-08 3:37 ` [PATCH for-next v3 1/5] RDMA/erdma: Unwind kernel QP initialization failures Cheng Xu
2026-09-08 3:37 ` [PATCH for-next v3 2/5] RDMA/erdma: Support non-contiguous kernel QP buffers Cheng Xu
@ 2026-09-08 3:37 ` Cheng Xu
2026-09-08 3:37 ` [PATCH for-next v3 4/5] RDMA/erdma: Unify userspace and kernel queue buffer management Cheng Xu
2026-09-08 3:37 ` [PATCH for-next v3 5/5] RDMA/erdma: Move kernel QP helpers after memory helpers Cheng Xu
4 siblings, 0 replies; 6+ messages in thread
From: Cheng Xu @ 2026-09-08 3:37 UTC (permalink / raw)
To: jgg, leon; +Cc: linux-rdma, KaiShen
A single coherent allocation for a kernel CQ can fail when memory is
fragmented.
Use page-sized coherent buffers and describe them with the existing MTT.
Keep the userspace CQ path and doorbell allocation unchanged.
Signed-off-by: Cheng Xu <chengyou@linux.alibaba.com>
---
drivers/infiniband/hw/erdma/erdma_cq.c | 14 ++---
drivers/infiniband/hw/erdma/erdma_verbs.c | 76 +++++++++--------------
drivers/infiniband/hw/erdma/erdma_verbs.h | 3 +-
3 files changed, 36 insertions(+), 57 deletions(-)
diff --git a/drivers/infiniband/hw/erdma/erdma_cq.c b/drivers/infiniband/hw/erdma/erdma_cq.c
index 4cc0254b0ab5..daec74f0aa6d 100644
--- a/drivers/infiniband/hw/erdma/erdma_cq.c
+++ b/drivers/infiniband/hw/erdma/erdma_cq.c
@@ -8,8 +8,8 @@
static void *get_next_valid_cqe(struct erdma_cq *cq)
{
- __be32 *cqe = get_queue_entry(cq->kern_cq.qbuf, cq->kern_cq.ci,
- cq->depth, CQE_SHIFT);
+ __be32 *cqe = erdma_kmem_get_entry(
+ &cq->kern_cq.qbuf_mem, cq->kern_cq.ci, cq->depth, CQE_SHIFT);
u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
be32_to_cpu(READ_ONCE(*cqe)));
@@ -242,15 +242,15 @@ void erdma_remove_cqes_of_qp(struct ib_cq *ibcq, u32 qpn)
while (ncqe > 0) {
cur_cq_ci = prev_cq_ci + ncqe - 1;
- cqe = get_queue_entry(cq->kern_cq.qbuf, cur_cq_ci, cq->depth,
- CQE_SHIFT);
+ cqe = erdma_kmem_get_entry(&cq->kern_cq.qbuf_mem, cur_cq_ci,
+ cq->depth, CQE_SHIFT);
if (be32_to_cpu(cqe->qpn) == qpn) {
++nqp_cqe;
} else if (nqp_cqe) {
- dst_cqe = get_queue_entry(cq->kern_cq.qbuf,
- cur_cq_ci + nqp_cqe,
- cq->depth, CQE_SHIFT);
+ dst_cqe = erdma_kmem_get_entry(&cq->kern_cq.qbuf_mem,
+ cur_cq_ci + nqp_cqe,
+ cq->depth, CQE_SHIFT);
owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
be32_to_cpu(dst_cqe->hdr));
cqe->hdr = cpu_to_be32(
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c
index 5725336fa1d4..9cf030ce1e2f 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.c
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.c
@@ -177,7 +177,6 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)
struct erdma_dev *dev = to_edev(cq->ibcq.device);
struct erdma_cmdq_create_cq_req req;
struct erdma_mem *mem;
- u32 page_size;
erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
CMDQ_OPCODE_CREATE_CQ);
@@ -187,48 +186,34 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)
req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn);
if (rdma_is_kernel_res(&cq->ibcq.res)) {
- page_size = SZ_32M;
- req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
- ilog2(page_size) - ERDMA_HW_PAGE_SHIFT);
- req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr);
- req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr);
-
- req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) |
- FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
- ERDMA_MR_MTT_0LEVEL);
-
- req.first_page_offset = 0;
+ mem = &cq->kern_cq.qbuf_mem;
req.cq_dbrec_dma = cq->kern_cq.dbrec_dma;
} else {
mem = &cq->user_cq.qbuf_mem;
- req.cfg0 |=
- FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
- ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT);
- if (mem->mtt_nents == 1) {
- req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]);
- req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]);
- req.cfg1 |=
- FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
- ERDMA_MR_MTT_0LEVEL);
- } else {
- req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma);
- req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma);
- req.cfg1 |=
- FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
- ERDMA_MR_MTT_1LEVEL);
- }
- req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK,
- mem->mtt_nents);
-
- req.first_page_offset = mem->page_offset;
req.cq_dbrec_dma = cq->user_cq.dbrec_dma;
+ }
- if (uctx->ext_db.enable) {
- req.cfg1 |= FIELD_PREP(
- ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);
- req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,
- uctx->ext_db.cdb_off);
- }
+ req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
+ ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT);
+ if (mem->mtt_nents == 1) {
+ req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]);
+ req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]);
+ req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
+ ERDMA_MR_MTT_0LEVEL);
+ } else {
+ req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma);
+ req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma);
+ req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
+ ERDMA_MR_MTT_1LEVEL);
+ }
+ req.cfg1 |=
+ FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, mem->mtt_nents);
+ req.first_page_offset = mem->page_offset;
+
+ if (uctx && uctx->ext_db.enable) {
+ req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);
+ req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,
+ uctx->ext_db.cdb_off);
}
return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
@@ -1401,8 +1386,7 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
wait_for_completion(&cq->free);
if (rdma_is_kernel_res(&cq->ibcq.res)) {
- dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
- cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
+ put_mtt_entries(dev, &cq->kern_cq.qbuf_mem);
dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
cq->kern_cq.dbrec_dma);
} else {
@@ -2000,10 +1984,8 @@ static int erdma_init_kernel_cq(struct erdma_cq *cq)
{
struct erdma_dev *dev = to_edev(cq->ibcq.device);
- cq->kern_cq.qbuf =
- dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
- &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL);
- if (!cq->kern_cq.qbuf)
+ if (erdma_alloc_kmem(dev, &cq->kern_cq.qbuf_mem,
+ cq->depth << CQE_SHIFT))
return -ENOMEM;
cq->kern_cq.dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL,
@@ -2018,8 +2000,7 @@ static int erdma_init_kernel_cq(struct erdma_cq *cq)
return 0;
err_out:
- dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
- cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
+ put_mtt_entries(dev, &cq->kern_cq.qbuf_mem);
return -ENOMEM;
}
@@ -2086,8 +2067,7 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
} else {
- dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT,
- cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
+ put_mtt_entries(dev, &cq->kern_cq.qbuf_mem);
dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
cq->kern_cq.dbrec_dma);
}
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h
index 2bc1e1928712..21076c176b46 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.h
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.h
@@ -348,8 +348,7 @@ struct erdma_qp {
};
struct erdma_kcq_info {
- void *qbuf;
- dma_addr_t qbuf_dma_addr;
+ struct erdma_mem qbuf_mem;
u32 ci;
u32 cmdsn;
u32 notify_cnt;
--
2.31.1
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH for-next v3 4/5] RDMA/erdma: Unify userspace and kernel queue buffer management
2026-09-08 3:37 [PATCH for-next v3 0/5] RDMA/erdma: Support non-contiguous kernel queue buffers Cheng Xu
` (2 preceding siblings ...)
2026-09-08 3:37 ` [PATCH for-next v3 3/5] RDMA/erdma: Support non-contiguous kernel CQ buffers Cheng Xu
@ 2026-09-08 3:37 ` Cheng Xu
2026-09-08 3:37 ` [PATCH for-next v3 5/5] RDMA/erdma: Move kernel QP helpers after memory helpers Cheng Xu
4 siblings, 0 replies; 6+ messages in thread
From: Cheng Xu @ 2026-09-08 3:37 UTC (permalink / raw)
To: jgg, leon; +Cc: linux-rdma, KaiShen
Userspace and kernel space queue buffers use separate helpers despite
sharing MTT metadata and lifetime rules. Manage both through
erdma_mem_init() and erdma_mem_uninit(), while keeping backing allocation
and release type-specific.
Signed-off-by: Cheng Xu <chengyou@linux.alibaba.com>
---
drivers/infiniband/hw/erdma/erdma_verbs.c | 271 +++++++++++++---------
drivers/infiniband/hw/erdma/erdma_verbs.h | 17 +-
2 files changed, 172 insertions(+), 116 deletions(-)
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c
index 9cf030ce1e2f..968137c8d478 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.c
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.c
@@ -19,9 +19,9 @@
#include "erdma_cm.h"
#include "erdma_verbs.h"
-static int erdma_alloc_kmem(struct erdma_dev *dev, struct erdma_mem *mem,
- size_t size);
-static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem);
+static int erdma_mem_init(struct erdma_dev *dev, struct erdma_mem *mem,
+ struct erdma_mem_init_attr *attr);
+static void erdma_mem_uninit(struct erdma_dev *dev, struct erdma_mem *mem);
static void assemble_qbuf_mtt_for_cmd(struct erdma_mem *mem, u32 *cfg,
u64 *addr0, u64 *addr1)
@@ -480,10 +480,10 @@ static void free_kernel_qp(struct erdma_qp *qp)
vfree(qp->kern_qp.swr_tbl);
vfree(qp->kern_qp.rwr_tbl);
- put_mtt_entries(dev, &qp->kern_qp.sq_mem);
+ erdma_mem_uninit(dev, &qp->kern_qp.sq_mem);
dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,
qp->kern_qp.sq_dbrec_dma);
- put_mtt_entries(dev, &qp->kern_qp.rq_mem);
+ erdma_mem_uninit(dev, &qp->kern_qp.rq_mem);
dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,
qp->kern_qp.rq_dbrec_dma);
}
@@ -491,6 +491,9 @@ static void free_kernel_qp(struct erdma_qp *qp)
static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
struct ib_qp_init_attr *attrs)
{
+ struct erdma_mem_init_attr attr = {
+ .type = ERDMA_KMEM,
+ };
struct erdma_kqp *kqp = &qp->kern_qp;
int size;
int ret = -ENOMEM;
@@ -515,7 +518,8 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
goto err_free_swr_tbl;
size = qp->attrs.sq_size << SQEBB_SHIFT;
- ret = erdma_alloc_kmem(dev, &kqp->sq_mem, size);
+ attr.len = size;
+ ret = erdma_mem_init(dev, &kqp->sq_mem, &attr);
if (ret)
goto err_free_rwr_tbl;
@@ -525,7 +529,8 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
goto err_free_sq_mem;
size = qp->attrs.rq_size << RQE_SHIFT;
- ret = erdma_alloc_kmem(dev, &kqp->rq_mem, size);
+ attr.len = size;
+ ret = erdma_mem_init(dev, &kqp->rq_mem, &attr);
if (ret)
goto err_free_sq_dbrec;
@@ -537,11 +542,11 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
return 0;
err_free_rq_mem:
- put_mtt_entries(dev, &kqp->rq_mem);
+ erdma_mem_uninit(dev, &kqp->rq_mem);
err_free_sq_dbrec:
dma_pool_free(dev->db_pool, kqp->sq_dbrec, kqp->sq_dbrec_dma);
err_free_sq_mem:
- put_mtt_entries(dev, &kqp->sq_mem);
+ erdma_mem_uninit(dev, &kqp->sq_mem);
err_free_rwr_tbl:
vfree(kqp->rwr_tbl);
err_free_swr_tbl:
@@ -550,17 +555,28 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
return ret;
}
-static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem)
+static void erdma_init_mtt_leaf(struct erdma_mem *mem, struct erdma_mtt *mtt)
{
- struct erdma_mtt *mtt = mem->mtt;
struct ib_block_iter biter;
u32 idx = 0;
+ if (mem->type == ERDMA_UMEM) {
+ rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size)
+ mtt->buf[idx++] = rdma_block_iter_dma_address(&biter);
+ } else {
+ for (; idx < mem->page_cnt; idx++)
+ mtt->buf[idx] = sg_dma_address(&mem->kmem.sgl[idx]);
+ }
+}
+
+static void erdma_init_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem)
+{
+ struct erdma_mtt *mtt = mem->mtt;
+
while (mtt->low_level)
mtt = mtt->low_level;
- rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size)
- mtt->buf[idx++] = rdma_block_iter_dma_address(&biter);
+ erdma_init_mtt_leaf(mem, mtt);
}
static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev,
@@ -792,7 +808,7 @@ static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt)
}
}
-static void erdma_free_kmem(struct erdma_dev *dev, struct erdma_mem *mem)
+static void erdma_kmem_free(struct erdma_dev *dev, struct erdma_mem *mem)
{
struct scatterlist *sgl = mem->kmem.sgl;
u32 i;
@@ -804,29 +820,21 @@ static void erdma_free_kmem(struct erdma_dev *dev, struct erdma_mem *mem)
kfree(sgl);
}
-static int erdma_alloc_kmem(struct erdma_dev *dev, struct erdma_mem *mem,
- size_t size)
+static int erdma_kmem_alloc(struct erdma_dev *dev, struct erdma_mem *mem,
+ u64 size)
{
struct scatterlist *sgl;
- struct erdma_mtt *mtt;
+ u32 page_cnt = DIV_ROUND_UP(size, PAGE_SIZE);
void *buf;
- int ret = -ENOMEM;
u32 i;
- mem->type = ERDMA_KMEM;
- mem->page_size = PAGE_SIZE;
- mem->page_offset = 0;
- mem->page_cnt = DIV_ROUND_UP(size, PAGE_SIZE);
- mem->mtt_nents = mem->page_cnt;
- mem->len = size;
-
- sgl = kcalloc(mem->page_cnt, sizeof(*sgl), GFP_KERNEL);
+ sgl = kcalloc(page_cnt, sizeof(*sgl), GFP_KERNEL);
if (!sgl)
return -ENOMEM;
- sg_init_table(sgl, mem->page_cnt);
+ sg_init_table(sgl, page_cnt);
- for (i = 0; i < mem->page_cnt; i++) {
+ for (i = 0; i < page_cnt; i++) {
buf = dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE,
&sg_dma_address(&sgl[i]), GFP_KERNEL);
if (!buf)
@@ -836,17 +844,8 @@ static int erdma_alloc_kmem(struct erdma_dev *dev, struct erdma_mem *mem,
sg_dma_len(&sgl[i]) = PAGE_SIZE;
}
- mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt), true);
- if (IS_ERR(mtt)) {
- ret = PTR_ERR(mtt);
- goto err_free_pages;
- }
-
- for (i = 0; i < mem->page_cnt; i++)
- mtt->buf[i] = sg_dma_address(&sgl[i]);
mem->kmem.sgl = sgl;
- mem->mtt = mtt;
-
+ mem->mtt_nents = page_cnt;
return 0;
err_free_pages:
@@ -855,62 +854,83 @@ static int erdma_alloc_kmem(struct erdma_dev *dev, struct erdma_mem *mem,
sg_dma_address(&sgl[i]));
kfree(sgl);
- return ret;
+ return -ENOMEM;
}
-static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
- u64 start, u64 len, int access, u64 virt,
- unsigned long req_page_size, bool force_continuous)
+static void erdma_mem_free(struct erdma_dev *dev, struct erdma_mem *mem)
{
- int ret = 0;
+ switch (mem->type) {
+ case ERDMA_UMEM:
+ ib_umem_release(mem->umem);
+ break;
+ case ERDMA_KMEM:
+ erdma_kmem_free(dev, mem);
+ break;
+ default:
+ break;
+ }
+}
- mem->type = ERDMA_UMEM;
- mem->umem = ib_umem_get_va(&dev->ibdev, start, len, access);
- if (IS_ERR(mem->umem)) {
- ret = PTR_ERR(mem->umem);
- mem->umem = NULL;
- return ret;
+static int erdma_mem_init(struct erdma_dev *dev, struct erdma_mem *mem,
+ struct erdma_mem_init_attr *attr)
+{
+ struct erdma_mtt *mtt;
+ int ret;
+
+ mem->type = attr->type;
+
+ switch (mem->type) {
+ case ERDMA_UMEM:
+ mem->umem = ib_umem_get_va(&dev->ibdev, attr->start, attr->len,
+ attr->access);
+ if (IS_ERR(mem->umem)) {
+ ret = PTR_ERR(mem->umem);
+ mem->umem = NULL;
+ return ret;
+ }
+
+ mem->page_size = ib_umem_find_best_pgsz(
+ mem->umem, attr->req_page_size, attr->virt);
+ mem->mtt_nents =
+ ib_umem_num_dma_blocks(mem->umem, mem->page_size);
+ break;
+ case ERDMA_KMEM:
+ ret = erdma_kmem_alloc(dev, mem, attr->len);
+ if (ret)
+ return ret;
+
+ mem->page_size = PAGE_SIZE;
+ break;
+ default:
+ return -EINVAL;
}
- mem->va = virt;
- mem->len = len;
- mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt);
- mem->page_offset = start & (mem->page_size - 1);
- mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size);
+ mem->va = attr->virt;
+ mem->len = attr->len;
+ mem->page_offset = attr->start & (mem->page_size - 1);
mem->page_cnt = mem->mtt_nents;
- mem->mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt),
- force_continuous);
- if (IS_ERR(mem->mtt)) {
- ret = PTR_ERR(mem->mtt);
- goto error_ret;
+ mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt),
+ !(attr->flags & ERDMA_MEM_FLAG_MR_BUF));
+ if (IS_ERR(mtt)) {
+ ret = PTR_ERR(mtt);
+ goto err_free_mem;
}
+ mem->mtt = mtt;
- erdma_fill_bottom_mtt(dev, mem);
+ erdma_init_bottom_mtt(dev, mem);
return 0;
-error_ret:
- if (mem->umem) {
- ib_umem_release(mem->umem);
- mem->umem = NULL;
- }
+err_free_mem:
+ erdma_mem_free(dev, mem);
return ret;
}
-static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)
+static void erdma_mem_uninit(struct erdma_dev *dev, struct erdma_mem *mem)
{
- if (mem->mtt) {
- erdma_destroy_mtt(dev, mem->mtt);
- mem->mtt = NULL;
- }
-
- if (mem->type == ERDMA_KMEM) {
- erdma_free_kmem(dev, mem);
- } else if (mem->umem) {
- ib_umem_release(mem->umem);
- mem->umem = NULL;
- }
+ erdma_destroy_mtt(dev, mem->mtt);
+ erdma_mem_free(dev, mem);
}
static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,
@@ -978,6 +998,9 @@ erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,
static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
u64 va, u32 len, u64 dbrec_va)
{
+ struct erdma_mem_init_attr attr = {
+ .type = ERDMA_UMEM,
+ };
dma_addr_t dbrec_dma;
u32 rq_offset;
int ret;
@@ -986,45 +1009,47 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
qp->attrs.rq_size * RQE_SIZE))
return -EINVAL;
- ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mem, va,
- qp->attrs.sq_size << SQEBB_SHIFT, 0, va,
- (SZ_1M - SZ_4K), true);
+ attr.start = va;
+ attr.virt = va;
+ attr.len = qp->attrs.sq_size << SQEBB_SHIFT;
+ attr.req_page_size = SZ_1M - SZ_4K;
+ ret = erdma_mem_init(qp->dev, &qp->user_qp.sq_mem, &attr);
if (ret)
return ret;
rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);
qp->user_qp.rq_offset = rq_offset;
- ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mem, va + rq_offset,
- qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset,
- (SZ_1M - SZ_4K), true);
+ attr.start = va + rq_offset;
+ attr.virt = va + rq_offset;
+ attr.len = qp->attrs.rq_size << RQE_SHIFT;
+ ret = erdma_mem_init(qp->dev, &qp->user_qp.rq_mem, &attr);
if (ret)
- goto put_sq_mtt;
+ goto uninit_sq_mem;
ret = erdma_map_user_dbrecords(uctx, dbrec_va,
- &qp->user_qp.user_dbr_page,
- &dbrec_dma);
+ &qp->user_qp.user_dbr_page, &dbrec_dma);
if (ret)
- goto put_rq_mtt;
+ goto uninit_rq_mem;
qp->user_qp.sq_dbrec_dma = dbrec_dma;
qp->user_qp.rq_dbrec_dma = dbrec_dma + ERDMA_DB_SIZE;
return 0;
-put_rq_mtt:
- put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
+uninit_rq_mem:
+ erdma_mem_uninit(qp->dev, &qp->user_qp.rq_mem);
-put_sq_mtt:
- put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
+uninit_sq_mem:
+ erdma_mem_uninit(qp->dev, &qp->user_qp.sq_mem);
return ret;
}
static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx)
{
- put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
- put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
+ erdma_mem_uninit(qp->dev, &qp->user_qp.sq_mem);
+ erdma_mem_uninit(qp->dev, &qp->user_qp.rq_mem);
erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page);
}
@@ -1271,8 +1296,11 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
u64 virt, int access, struct ib_dmah *dmah,
struct ib_udata *udata)
{
- struct erdma_mr *mr = NULL;
struct erdma_dev *dev = to_edev(ibpd->device);
+ struct erdma_mem_init_attr attr = {
+ .type = ERDMA_UMEM,
+ };
+ struct erdma_mr *mr = NULL;
u32 stag;
int ret;
@@ -1286,8 +1314,13 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
if (!mr)
return ERR_PTR(-ENOMEM);
- ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt,
- SZ_2G - SZ_4K, false);
+ attr.start = start;
+ attr.virt = virt;
+ attr.len = len;
+ attr.req_page_size = SZ_2G - SZ_4K;
+ attr.access = access;
+ attr.flags = ERDMA_MEM_FLAG_MR_BUF;
+ ret = erdma_mem_init(dev, &mr->mem, &attr);
if (ret)
goto err_out_free;
@@ -1297,8 +1330,6 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
mr->ibmr.lkey = mr->ibmr.rkey = stag;
mr->ibmr.pd = ibpd;
- mr->mem.va = virt;
- mr->mem.len = len;
mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access);
mr->valid = 1;
mr->type = ERDMA_MR_TYPE_NORMAL;
@@ -1314,7 +1345,7 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
mr->ibmr.lkey >> 8);
err_out_put_mtt:
- put_mtt_entries(dev, &mr->mem);
+ erdma_mem_uninit(dev, &mr->mem);
err_out_free:
kfree(mr);
@@ -1351,7 +1382,7 @@ int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8);
- put_mtt_entries(dev, &mr->mem);
+ erdma_mem_uninit(dev, &mr->mem);
kfree(mr);
return 0;
@@ -1386,12 +1417,12 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
wait_for_completion(&cq->free);
if (rdma_is_kernel_res(&cq->ibcq.res)) {
- put_mtt_entries(dev, &cq->kern_cq.qbuf_mem);
+ erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem);
dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
cq->kern_cq.dbrec_dma);
} else {
erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
- put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
+ erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem);
}
return 0;
@@ -1443,8 +1474,8 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
if (rdma_is_kernel_res(&qp->ibqp.res)) {
free_kernel_qp(qp);
} else {
- put_mtt_entries(dev, &qp->user_qp.sq_mem);
- put_mtt_entries(dev, &qp->user_qp.rq_mem);
+ erdma_mem_uninit(dev, &qp->user_qp.sq_mem);
+ erdma_mem_uninit(dev, &qp->user_qp.rq_mem);
erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page);
}
@@ -1962,12 +1993,17 @@ int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
struct erdma_ureq_create_cq *ureq)
{
- int ret;
struct erdma_dev *dev = to_edev(cq->ibcq.device);
+ struct erdma_mem_init_attr attr = {
+ .type = ERDMA_UMEM,
+ };
+ int ret;
- ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va,
- ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K,
- true);
+ attr.start = ureq->qbuf_va;
+ attr.virt = ureq->qbuf_va;
+ attr.len = ureq->qbuf_len;
+ attr.req_page_size = SZ_64M - SZ_4K;
+ ret = erdma_mem_init(dev, &cq->user_cq.qbuf_mem, &attr);
if (ret)
return ret;
@@ -1975,7 +2011,7 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
&cq->user_cq.user_dbr_page,
&cq->user_cq.dbrec_dma);
if (ret)
- put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
+ erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem);
return ret;
}
@@ -1983,10 +2019,15 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
static int erdma_init_kernel_cq(struct erdma_cq *cq)
{
struct erdma_dev *dev = to_edev(cq->ibcq.device);
+ struct erdma_mem_init_attr attr = {
+ .type = ERDMA_KMEM,
+ .len = cq->depth << CQE_SHIFT,
+ };
+ int ret;
- if (erdma_alloc_kmem(dev, &cq->kern_cq.qbuf_mem,
- cq->depth << CQE_SHIFT))
- return -ENOMEM;
+ ret = erdma_mem_init(dev, &cq->kern_cq.qbuf_mem, &attr);
+ if (ret)
+ return ret;
cq->kern_cq.dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL,
&cq->kern_cq.dbrec_dma);
@@ -2000,7 +2041,7 @@ static int erdma_init_kernel_cq(struct erdma_cq *cq)
return 0;
err_out:
- put_mtt_entries(dev, &cq->kern_cq.qbuf_mem);
+ erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem);
return -ENOMEM;
}
@@ -2065,9 +2106,9 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
err_free_res:
if (!rdma_is_kernel_res(&ibcq->res)) {
erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
- put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
+ erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem);
} else {
- put_mtt_entries(dev, &cq->kern_cq.qbuf_mem);
+ erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem);
dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
cq->kern_cq.dbrec_dma);
}
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h
index 21076c176b46..517a82b86b4f 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.h
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.h
@@ -121,19 +121,34 @@ struct erdma_kmem {
struct scatterlist *sgl;
};
+enum erdma_mem_flags {
+ ERDMA_MEM_FLAG_MR_BUF = (1 << 0),
+};
+
+struct erdma_mem_init_attr {
+ enum erdma_mem_type type;
+ u64 start;
+ u64 virt;
+ u64 len;
+ unsigned long req_page_size;
+ int access;
+ u32 flags;
+};
+
struct erdma_mem {
enum erdma_mem_type type;
union {
struct ib_umem *umem;
struct erdma_kmem kmem;
};
- struct erdma_mtt *mtt;
u32 page_size;
u32 page_offset;
u32 page_cnt;
u32 mtt_nents;
+ struct erdma_mtt *mtt;
+
u64 va;
u64 len;
};
--
2.31.1
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH for-next v3 5/5] RDMA/erdma: Move kernel QP helpers after memory helpers
2026-09-08 3:37 [PATCH for-next v3 0/5] RDMA/erdma: Support non-contiguous kernel queue buffers Cheng Xu
` (3 preceding siblings ...)
2026-09-08 3:37 ` [PATCH for-next v3 4/5] RDMA/erdma: Unify userspace and kernel queue buffer management Cheng Xu
@ 2026-09-08 3:37 ` Cheng Xu
4 siblings, 0 replies; 6+ messages in thread
From: Cheng Xu @ 2026-09-08 3:37 UTC (permalink / raw)
To: jgg, leon; +Cc: linux-rdma, KaiShen
Move the kernel QP helpers after the shared memory helpers to remove
forward declarations.
Signed-off-by: Cheng Xu <chengyou@linux.alibaba.com>
---
drivers/infiniband/hw/erdma/erdma_verbs.c | 168 +++++++++++-----------
1 file changed, 82 insertions(+), 86 deletions(-)
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c
index 968137c8d478..dddbab81f411 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.c
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.c
@@ -19,10 +19,6 @@
#include "erdma_cm.h"
#include "erdma_verbs.h"
-static int erdma_mem_init(struct erdma_dev *dev, struct erdma_mem *mem,
- struct erdma_mem_init_attr *attr);
-static void erdma_mem_uninit(struct erdma_dev *dev, struct erdma_mem *mem);
-
static void assemble_qbuf_mtt_for_cmd(struct erdma_mem *mem, u32 *cfg,
u64 *addr0, u64 *addr1)
{
@@ -473,88 +469,6 @@ static int erdma_qp_validate_attr(struct erdma_dev *dev,
return 0;
}
-static void free_kernel_qp(struct erdma_qp *qp)
-{
- struct erdma_dev *dev = qp->dev;
-
- vfree(qp->kern_qp.swr_tbl);
- vfree(qp->kern_qp.rwr_tbl);
-
- erdma_mem_uninit(dev, &qp->kern_qp.sq_mem);
- dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,
- qp->kern_qp.sq_dbrec_dma);
- erdma_mem_uninit(dev, &qp->kern_qp.rq_mem);
- dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,
- qp->kern_qp.rq_dbrec_dma);
-}
-
-static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
- struct ib_qp_init_attr *attrs)
-{
- struct erdma_mem_init_attr attr = {
- .type = ERDMA_KMEM,
- };
- struct erdma_kqp *kqp = &qp->kern_qp;
- int size;
- int ret = -ENOMEM;
-
- if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
- kqp->sig_all = 1;
-
- kqp->sq_pi = 0;
- kqp->sq_ci = 0;
- kqp->rq_pi = 0;
- kqp->rq_ci = 0;
- kqp->hw_sq_db =
- dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);
- kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
-
- kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64));
- if (!kqp->swr_tbl)
- return -ENOMEM;
-
- kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64));
- if (!kqp->rwr_tbl)
- goto err_free_swr_tbl;
-
- size = qp->attrs.sq_size << SQEBB_SHIFT;
- attr.len = size;
- ret = erdma_mem_init(dev, &kqp->sq_mem, &attr);
- if (ret)
- goto err_free_rwr_tbl;
-
- kqp->sq_dbrec =
- dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma);
- if (!kqp->sq_dbrec)
- goto err_free_sq_mem;
-
- size = qp->attrs.rq_size << RQE_SHIFT;
- attr.len = size;
- ret = erdma_mem_init(dev, &kqp->rq_mem, &attr);
- if (ret)
- goto err_free_sq_dbrec;
-
- kqp->rq_dbrec =
- dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma);
- if (!kqp->rq_dbrec)
- goto err_free_rq_mem;
-
- return 0;
-
-err_free_rq_mem:
- erdma_mem_uninit(dev, &kqp->rq_mem);
-err_free_sq_dbrec:
- dma_pool_free(dev->db_pool, kqp->sq_dbrec, kqp->sq_dbrec_dma);
-err_free_sq_mem:
- erdma_mem_uninit(dev, &kqp->sq_mem);
-err_free_rwr_tbl:
- vfree(kqp->rwr_tbl);
-err_free_swr_tbl:
- vfree(kqp->swr_tbl);
-
- return ret;
-}
-
static void erdma_init_mtt_leaf(struct erdma_mem *mem, struct erdma_mtt *mtt)
{
struct ib_block_iter biter;
@@ -933,6 +847,88 @@ static void erdma_mem_uninit(struct erdma_dev *dev, struct erdma_mem *mem)
erdma_mem_free(dev, mem);
}
+static void free_kernel_qp(struct erdma_qp *qp)
+{
+ struct erdma_dev *dev = qp->dev;
+
+ vfree(qp->kern_qp.swr_tbl);
+ vfree(qp->kern_qp.rwr_tbl);
+
+ erdma_mem_uninit(dev, &qp->kern_qp.sq_mem);
+ dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,
+ qp->kern_qp.sq_dbrec_dma);
+ erdma_mem_uninit(dev, &qp->kern_qp.rq_mem);
+ dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,
+ qp->kern_qp.rq_dbrec_dma);
+}
+
+static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
+ struct ib_qp_init_attr *attrs)
+{
+ struct erdma_mem_init_attr attr = {
+ .type = ERDMA_KMEM,
+ };
+ struct erdma_kqp *kqp = &qp->kern_qp;
+ int size;
+ int ret = -ENOMEM;
+
+ if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
+ kqp->sig_all = 1;
+
+ kqp->sq_pi = 0;
+ kqp->sq_ci = 0;
+ kqp->rq_pi = 0;
+ kqp->rq_ci = 0;
+ kqp->hw_sq_db =
+ dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);
+ kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
+
+ kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64));
+ if (!kqp->swr_tbl)
+ return -ENOMEM;
+
+ kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64));
+ if (!kqp->rwr_tbl)
+ goto err_free_swr_tbl;
+
+ size = qp->attrs.sq_size << SQEBB_SHIFT;
+ attr.len = size;
+ ret = erdma_mem_init(dev, &kqp->sq_mem, &attr);
+ if (ret)
+ goto err_free_rwr_tbl;
+
+ kqp->sq_dbrec =
+ dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma);
+ if (!kqp->sq_dbrec)
+ goto err_free_sq_mem;
+
+ size = qp->attrs.rq_size << RQE_SHIFT;
+ attr.len = size;
+ ret = erdma_mem_init(dev, &kqp->rq_mem, &attr);
+ if (ret)
+ goto err_free_sq_dbrec;
+
+ kqp->rq_dbrec =
+ dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma);
+ if (!kqp->rq_dbrec)
+ goto err_free_rq_mem;
+
+ return 0;
+
+err_free_rq_mem:
+ erdma_mem_uninit(dev, &kqp->rq_mem);
+err_free_sq_dbrec:
+ dma_pool_free(dev->db_pool, kqp->sq_dbrec, kqp->sq_dbrec_dma);
+err_free_sq_mem:
+ erdma_mem_uninit(dev, &kqp->sq_mem);
+err_free_rwr_tbl:
+ vfree(kqp->rwr_tbl);
+err_free_swr_tbl:
+ vfree(kqp->swr_tbl);
+
+ return ret;
+}
+
static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,
u64 dbrecords_va,
struct erdma_user_dbrecords_page **dbr_page,
--
2.31.1
^ permalink raw reply related [flat|nested] 6+ messages in thread