* [PATCH for-next 1/4] RDMA/bnxt_re: Update the page sizes supported for MR
2026-08-07 0:51 [PATCH for-next 0/4] DMA/bnxt_re: Add support for larger page sizes for queue memory Selvin Xavier
@ 2026-08-07 0:51 ` Selvin Xavier
2026-08-07 0:51 ` [PATCH for-next 2/4] RDMA/bnxt_re: Prepare infrastructure for larger pages for Queue memory Selvin Xavier
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Selvin Xavier @ 2026-08-07 0:51 UTC (permalink / raw)
To: leon, jgg
Cc: linux-rdma, andrew.gospodarek, kalesh-anakkur.purayil,
saravanan.vajravel, Selvin Xavier
Not all the MR page sizes supported by HW are tested. So reduce
the supported page sizes to 4k, 64k and 2M and 256M only for now.
Signed-off-by: Saravanan Vajravel <saravanan.vajravel@broadcom.com>
Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com>
---
drivers/infiniband/hw/bnxt_re/bnxt_re.h | 1 -
drivers/infiniband/hw/bnxt_re/ib_verbs.c | 4 ++--
drivers/infiniband/hw/bnxt_re/qplib_sp.c | 2 ++
drivers/infiniband/hw/bnxt_re/qplib_sp.h | 1 +
4 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
index a43e678151d3..5d46743f075a 100644
--- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h
+++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
@@ -46,7 +46,6 @@
#define BNXT_RE_DESC "Broadcom NetXtreme-C/E RoCE Driver"
#define BNXT_RE_PAGE_SHIFT_1G (30)
-#define BNXT_RE_PAGE_SIZE_SUPPORTED 0x7FFFF000 /* 4kb - 1G */
#define BNXT_RE_MAX_MR_SIZE_LOW BIT_ULL(BNXT_RE_PAGE_SHIFT_1G)
#define BNXT_RE_MAX_MR_SIZE_HIGH BIT_ULL(39)
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index ccd2702db78b..21caf34a1dc8 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -199,7 +199,7 @@ int bnxt_re_query_device(struct ib_device *ibdev,
addrconf_addr_eui48((u8 *)&ib_attr->sys_image_guid,
rdev->netdev->dev_addr);
ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE;
- ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_SUPPORTED;
+ ib_attr->page_size_cap = dev_attr->mr_page_size_cap;
ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
@@ -4687,7 +4687,7 @@ static struct ib_mr *__bnxt_re_user_reg_mr(struct ib_pd *ib_pd, u64 length, u64
return ERR_PTR(-ENOMEM);
}
- page_size = ib_umem_find_best_pgsz(umem, BNXT_RE_PAGE_SIZE_SUPPORTED, virt_addr);
+ page_size = ib_umem_find_best_pgsz(umem, rdev->dev_attr->mr_page_size_cap, virt_addr);
if (!page_size) {
ibdev_err(&rdev->ibdev, "umem page size unsupported!");
return ERR_PTR(-EINVAL);
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
index ec9eb52a8ebf..71240f7be9e6 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
@@ -177,6 +177,8 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw)
attr->dev_cap_flags = le16_to_cpu(sb->dev_cap_flags);
attr->dev_cap_flags2 = le16_to_cpu(sb->dev_cap_ext_flags_2);
+ attr->mr_page_size_cap = BIT_ULL(28) | BIT_ULL(21) | BIT_ULL(16) | BIT_ULL(12);
+
if (_is_max_srq_ext_supported(attr->dev_cap_flags2))
attr->max_srq += le16_to_cpu(sb->max_srq_ext);
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h
index c4193ae75b54..b16e12d3b7ab 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h
@@ -75,6 +75,7 @@ struct bnxt_qplib_dev_attr {
bool is_atomic;
u16 dev_cap_flags;
u16 dev_cap_flags2;
+ u64 mr_page_size_cap;
u32 max_dpi;
u16 rate_limit_min;
u32 rate_limit_max;
--
2.39.3
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH for-next 2/4] RDMA/bnxt_re: Prepare infrastructure for larger pages for Queue memory
2026-08-07 0:51 [PATCH for-next 0/4] DMA/bnxt_re: Add support for larger page sizes for queue memory Selvin Xavier
2026-08-07 0:51 ` [PATCH for-next 1/4] RDMA/bnxt_re: Update the page sizes supported for MR Selvin Xavier
@ 2026-08-07 0:51 ` Selvin Xavier
2026-08-07 0:51 ` [PATCH for-next 3/4] RDMA/bnxt_re: Add support for larger PBL pages for CQ Selvin Xavier
2026-08-07 0:51 ` [PATCH for-next 4/4] RDMA/bnxt_re: Add support for larger PBL pages for QP Selvin Xavier
3 siblings, 0 replies; 5+ messages in thread
From: Selvin Xavier @ 2026-08-07 0:51 UTC (permalink / raw)
To: leon, jgg
Cc: linux-rdma, andrew.gospodarek, kalesh-anakkur.purayil,
saravanan.vajravel, Selvin Xavier
Add page-shift/FWO helpers, firmware capability detection
(dev_cap_ext_flags_1), and the fwo_offset field so CQ/QP setup
can be done with page sizes more than 4K. This also enables
using PBL pages of size more than 4k. Currently, the PBLs are
defaulted to 4K pages.
Also, update the variable type that holds the max queue entry
per page to accommodate more number of pages when huge pages
are used.
Signed-off-by: Saravanan Vajravel <saravanan.vajravel@broadcom.com>
Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com>
---
drivers/infiniband/hw/bnxt_re/ib_verbs.c | 41 ++++++++++++++++++++---
drivers/infiniband/hw/bnxt_re/qplib_res.c | 21 ++++++++++++
drivers/infiniband/hw/bnxt_re/qplib_res.h | 34 +++++++++++++++++--
drivers/infiniband/hw/bnxt_re/qplib_sp.c | 3 ++
drivers/infiniband/hw/bnxt_re/qplib_sp.h | 2 ++
drivers/infiniband/hw/bnxt_re/roce_hsi.h | 4 ++-
6 files changed, 98 insertions(+), 7 deletions(-)
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index 21caf34a1dc8..430d54f8f736 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -1141,23 +1141,42 @@ static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,
return 0;
}
+static int bnxt_re_get_page_shift(struct ib_umem *umem, u64 va, u64 cmask)
+{
+ unsigned long pgsz;
+
+ pgsz = ib_umem_find_best_pgsz(umem, cmask, va);
+ if (!pgsz)
+ return -EINVAL;
+ return __ffs(pgsz);
+}
+
static int bnxt_re_setup_sginfo(struct bnxt_re_dev *rdev,
struct ib_umem *umem,
struct bnxt_qplib_sg_info *sginfo)
{
+ struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
unsigned long page_size;
+ int page_shift;
if (!umem)
return -EINVAL;
- page_size = ib_umem_find_best_pgsz(umem, SZ_4K, 0);
- if (!page_size || page_size != SZ_4K)
- return -EINVAL;
+ if (bnxt_re_pbl_size_supported(dev_attr->dev_cap_ext_flags_1)) {
+ page_shift = bnxt_re_get_page_shift(umem, umem->address, dev_attr->page_size_cap);
+ if (page_shift < 0)
+ return page_shift;
+ } else {
+ page_shift = __builtin_ctz(SZ_4K);
+ }
+ page_size = BIT(page_shift);
sginfo->umem = umem;
sginfo->npages = ib_umem_num_dma_blocks(umem, page_size);
sginfo->pgsize = page_size;
- sginfo->pgshft = __builtin_ctz(page_size);
+ sginfo->pgshft = page_shift;
+ sginfo->fwo_offset = ib_umem_dma_offset(umem, page_size);
+
return 0;
}
@@ -1224,6 +1243,15 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
if (rc)
goto fail;
+ /*
+ * The FWO field passed in the CREATE_QP command is expressed in 4K
+ * units, so the offset must be a multiple of 4K.
+ */
+ if (qplib_qp->sq.sg_info.fwo_offset & (SZ_4K - 1)) {
+ rc = -EINVAL;
+ goto fail;
+ }
+
if (qp->qplib_qp.srq)
goto done;
@@ -1242,6 +1270,11 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
if (rc)
goto rqfail;
+ if (qplib_qp->rq.sg_info.fwo_offset & (SZ_4K - 1)) {
+ rc = -EINVAL;
+ goto rqfail;
+ }
+
done:
if (dbr_obj)
qplib_qp->dpi = &dbr_obj->dpi;
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c
index 756f8b5f042a..9a980f561661 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c
@@ -174,6 +174,25 @@ void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res,
/* All HWQs are power of 2 in size */
+/*
+ * At PBL_LVL_0 there is a single PBL entry and firmware uses it as the
+ * literal queue base address rather than walking an indirection table,
+ * so it never applies the FWO field for this level. Adjust fwo_offset
+ * directly into that entry's DMA address instead.
+ */
+static void bnxt_qplib_adjust_lvl0_fwo(struct bnxt_qplib_hwq *hwq,
+ struct bnxt_qplib_hwq_attr *hwq_attr)
+{
+ struct bnxt_qplib_pbl *pbl;
+
+ if (!hwq->is_user)
+ return;
+
+ pbl = &hwq->pbl[PBL_LVL_0];
+ pbl->pg_map_arr[0] += hwq_attr->sginfo->fwo_offset;
+ hwq_attr->sginfo->fwo_offset = 0;
+}
+
int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq,
struct bnxt_qplib_hwq_attr *hwq_attr)
{
@@ -220,6 +239,8 @@ int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq,
if (rc)
goto fail;
hwq->level = PBL_LVL_0;
+ if (hwq_attr->type == HWQ_TYPE_QUEUE)
+ bnxt_qplib_adjust_lvl0_fwo(hwq, hwq_attr);
goto done;
}
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h
index b925db796729..ce6823af167e 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h
@@ -166,6 +166,8 @@ struct bnxt_qplib_sg_info {
u32 npages;
u32 pgshft;
u32 pgsize;
+#define BNXT_QPLIB_QP_FWO_SHIFT 12 /* FWO is conveyed to firmware in 4K units */
+ u32 fwo_offset; /* first-word offset into huge page */
bool nopte;
};
@@ -192,7 +194,7 @@ struct bnxt_qplib_hwq {
u32 max_elements;
u32 depth;
u16 element_size; /* Size of each entry */
- u16 qe_ppg; /* queue entry per page */
+ u32 qe_ppg; /* queue entry per page */
u32 prod; /* raw */
u32 cons; /* raw */
@@ -369,7 +371,7 @@ static inline u8 bnxt_qplib_base_pg_size(struct bnxt_qplib_hwq *hwq)
u8 pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K;
struct bnxt_qplib_pbl *pbl;
- pbl = &hwq->pbl[PBL_LVL_0];
+ pbl = &hwq->pbl[hwq->level];
switch (pbl->pg_size) {
case ROCE_PG_SIZE_4K:
pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K;
@@ -639,4 +641,32 @@ static inline bool _is_modify_qp_rate_limit_supported(u16 dev_cap_ext_flags2)
CREQ_QUERY_FUNC_RESP_SB_MODIFY_QP_RATE_LIMIT_SUPPORTED;
}
+/* PBL page size encoding values for HWRM commands */
+#define BNXT_QPLIB_HWRM_PBL_PG_SIZE_PG_4K 0x0UL
+#define BNXT_QPLIB_HWRM_PBL_PG_SIZE_PG_8K 0x1UL
+#define BNXT_QPLIB_HWRM_PBL_PG_SIZE_PG_64K 0x2UL
+#define BNXT_QPLIB_HWRM_PBL_PG_SIZE_PG_2M 0x3UL
+#define BNXT_QPLIB_HWRM_PBL_PG_SIZE_PG_8M 0x4UL
+#define BNXT_QPLIB_HWRM_PBL_PG_SIZE_PG_1G 0x5UL
+
+/**
+ * bnxt_re_pbl_size_supported - check if firmware supports non-4K PBL pages
+ * @dev_cap_ext_flags_1: value of the dev_cap_ext_flags_1 field from QUERY_FUNC response
+ *
+ * Returns non-zero if the firmware can accept PBL pages larger than 4K.
+ */
+static inline int bnxt_re_pbl_size_supported(u8 dev_cap_ext_flags_1)
+{
+ return dev_cap_ext_flags_1 &
+ CREQ_QUERY_FUNC_RESP_SB_PBL_PAGE_SIZE_SUPPORTED;
+}
+
+static inline u8 bnxt_qplib_get_pbl_page_size(struct bnxt_qplib_sg_info *sginfo)
+{
+ /* Only 4K PBL pages are supported for now; extend this when QP/CQ
+ * PBLs start using the larger page sizes sginfo can report.
+ */
+ return BNXT_QPLIB_HWRM_PBL_PG_SIZE_PG_4K;
+}
+
#endif /* __BNXT_QPLIB_RES_H__ */
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
index 71240f7be9e6..5fa78f156df3 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
@@ -176,8 +176,11 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw)
attr->max_sgid = min_t(u32, BNXT_QPLIB_NUM_GIDS_SUPPORTED, 2 * attr->max_sgid);
attr->dev_cap_flags = le16_to_cpu(sb->dev_cap_flags);
attr->dev_cap_flags2 = le16_to_cpu(sb->dev_cap_ext_flags_2);
+ attr->dev_cap_ext_flags_1 = sb->dev_cap_ext_flags_1;
attr->mr_page_size_cap = BIT_ULL(28) | BIT_ULL(21) | BIT_ULL(16) | BIT_ULL(12);
+ /* For Queue memories */
+ attr->page_size_cap = BIT_ULL(21) | BIT_ULL(16) | BIT_ULL(13) | BIT_ULL(12);
if (_is_max_srq_ext_supported(attr->dev_cap_flags2))
attr->max_srq += le16_to_cpu(sb->max_srq_ext);
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h
index b16e12d3b7ab..5d2a5b905405 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h
@@ -79,6 +79,8 @@ struct bnxt_qplib_dev_attr {
u32 max_dpi;
u16 rate_limit_min;
u32 rate_limit_max;
+ u8 dev_cap_ext_flags_1;
+ u64 page_size_cap;
};
struct bnxt_qplib_pd {
diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
index aac338f2afd8..c89fc54f61c3 100644
--- a/drivers/infiniband/hw/bnxt_re/roce_hsi.h
+++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
@@ -2244,7 +2244,9 @@ struct creq_query_func_resp_sb {
__le16 max_inline_data_var_wqe;
__le32 start_qid;
u8 max_msn_table_size;
- u8 reserved8_1;
+ u8 dev_cap_ext_flags_1;
+ #define CREQ_QUERY_FUNC_RESP_SB_PBL_PAGE_SIZE_SUPPORTED 0x1UL
+ #define CREQ_QUERY_FUNC_RESP_SB_WQE_INLINE_OPTIMIZER_SUPPORTED 0x20UL
__le16 dev_cap_ext_flags_2;
#define CREQ_QUERY_FUNC_RESP_SB_OPTIMIZE_MODIFY_QP_SUPPORTED 0x1UL
#define CREQ_QUERY_FUNC_RESP_SB_CHANGE_UDP_SRC_PORT_WQE_SUPPORTED 0x2UL
--
2.39.3
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH for-next 3/4] RDMA/bnxt_re: Add support for larger PBL pages for CQ
2026-08-07 0:51 [PATCH for-next 0/4] DMA/bnxt_re: Add support for larger page sizes for queue memory Selvin Xavier
2026-08-07 0:51 ` [PATCH for-next 1/4] RDMA/bnxt_re: Update the page sizes supported for MR Selvin Xavier
2026-08-07 0:51 ` [PATCH for-next 2/4] RDMA/bnxt_re: Prepare infrastructure for larger pages for Queue memory Selvin Xavier
@ 2026-08-07 0:51 ` Selvin Xavier
2026-08-07 0:51 ` [PATCH for-next 4/4] RDMA/bnxt_re: Add support for larger PBL pages for QP Selvin Xavier
3 siblings, 0 replies; 5+ messages in thread
From: Selvin Xavier @ 2026-08-07 0:51 UTC (permalink / raw)
To: leon, jgg
Cc: linux-rdma, andrew.gospodarek, kalesh-anakkur.purayil,
saravanan.vajravel, Selvin Xavier
Advertise the PBL page size and first-completion offset (FCO)
in CREATE_CQ so the firmware can use huge-page CQ buffers.
Signed-off-by: Saravanan Vajravel <saravanan.vajravel@broadcom.com>
Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com>
---
drivers/infiniband/hw/bnxt_re/ib_verbs.c | 9 +++++++++
drivers/infiniband/hw/bnxt_re/qplib_fp.c | 11 +++++++++--
drivers/infiniband/hw/bnxt_re/qplib_res.h | 1 +
drivers/infiniband/hw/bnxt_re/roce_hsi.h | 13 ++++++++++++-
4 files changed, 31 insertions(+), 3 deletions(-)
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index 430d54f8f736..da4de44cf90d 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -3611,6 +3611,15 @@ int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *att
if (rc)
goto free_umem;
+ /*
+ * The FCO field passed in the CREATE_CQ command is expressed in
+ * 32-byte units, so the offset must be a multiple of 32 bytes.
+ */
+ if (cq->qplib_cq.sg_info.fwo_offset & (BIT(BNXT_QPLIB_CQ_FCO_SHIFT) - 1)) {
+ rc = -EINVAL;
+ goto free_umem;
+ }
+
cq->qplib_cq.dpi = &uctx->dpi;
cq->qplib_cq.max_wqe = entries;
cq->qplib_cq.coalescing = &rdev->cq_coalescing;
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index 5301e4f21ade..0319fe76c634 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -2147,9 +2147,16 @@ int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
pg_sz_lvl |= (cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK);
req.pg_size_lvl = cpu_to_le32(pg_sz_lvl);
req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
+ if (bnxt_re_pbl_size_supported(res->dattr->dev_cap_ext_flags_1)) {
+ req.flags |= cpu_to_le16(CMDQ_CREATE_CQ_FLAGS_PBL_PG_SIZE_VALID);
+ req.pbl_pg_size = bnxt_qplib_get_pbl_page_size(&cq->sg_info);
+ }
req.cq_fco_cnq_id = cpu_to_le32(
- (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
- CMDQ_CREATE_CQ_CNQ_ID_SFT);
+ ((cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
+ CMDQ_CREATE_CQ_CNQ_ID_SFT) |
+ (((cq->sg_info.fwo_offset >> BNXT_QPLIB_CQ_FCO_SHIFT)
+ << CMDQ_CREATE_CQ_CQ_FCO_SFT) &
+ CMDQ_CREATE_CQ_CQ_FCO_MASK));
bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
sizeof(resp), 0);
rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h
index ce6823af167e..d2b3835789f3 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h
@@ -167,6 +167,7 @@ struct bnxt_qplib_sg_info {
u32 pgshft;
u32 pgsize;
#define BNXT_QPLIB_QP_FWO_SHIFT 12 /* FWO is conveyed to firmware in 4K units */
+#define BNXT_QPLIB_CQ_FCO_SHIFT 5 /* FCO is conveyed to firmware in 32-byte units */
u32 fwo_offset; /* first-word offset into huge page */
bool nopte;
};
diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
index c89fc54f61c3..bbcd723cd2b5 100644
--- a/drivers/infiniband/hw/bnxt_re/roce_hsi.h
+++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
@@ -1146,6 +1146,7 @@ struct cmdq_create_cq {
#define CMDQ_CREATE_CQ_FLAGS_STEERING_TAG_VALID 0x2UL
#define CMDQ_CREATE_CQ_FLAGS_INFINITE_CQ_MODE 0x4UL
#define CMDQ_CREATE_CQ_FLAGS_COALESCING_VALID 0x8UL
+ #define CMDQ_CREATE_CQ_FLAGS_PBL_PG_SIZE_VALID 0x10UL
__le16 cookie;
u8 resp_size;
u8 reserved8;
@@ -1178,7 +1179,17 @@ struct cmdq_create_cq {
__le32 cq_size;
__le64 pbl;
__le16 steering_tag;
- u8 reserved48[2];
+ u8 pbl_pg_size;
+ #define CMDQ_CREATE_CQ_PBL_PG_SIZE_MASK 0x7UL
+ #define CMDQ_CREATE_CQ_PBL_PG_SIZE_SFT 0
+ #define CMDQ_CREATE_CQ_PBL_PG_SIZE_PG_4K 0x0UL
+ #define CMDQ_CREATE_CQ_PBL_PG_SIZE_PG_8K 0x1UL
+ #define CMDQ_CREATE_CQ_PBL_PG_SIZE_PG_64K 0x2UL
+ #define CMDQ_CREATE_CQ_PBL_PG_SIZE_PG_2M 0x3UL
+ #define CMDQ_CREATE_CQ_PBL_PG_SIZE_PG_8M 0x4UL
+ #define CMDQ_CREATE_CQ_PBL_PG_SIZE_PG_1G 0x5UL
+ #define CMDQ_CREATE_CQ_PBL_PG_SIZE_LAST CMDQ_CREATE_CQ_PBL_PG_SIZE_PG_1G
+ u8 reserved8_1;
__le32 coalescing;
#define CMDQ_CREATE_CQ_BUF_MAXTIME_MASK 0x1ffUL
#define CMDQ_CREATE_CQ_BUF_MAXTIME_SFT 0
--
2.39.3
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH for-next 4/4] RDMA/bnxt_re: Add support for larger PBL pages for QP
2026-08-07 0:51 [PATCH for-next 0/4] DMA/bnxt_re: Add support for larger page sizes for queue memory Selvin Xavier
` (2 preceding siblings ...)
2026-08-07 0:51 ` [PATCH for-next 3/4] RDMA/bnxt_re: Add support for larger PBL pages for CQ Selvin Xavier
@ 2026-08-07 0:51 ` Selvin Xavier
3 siblings, 0 replies; 5+ messages in thread
From: Selvin Xavier @ 2026-08-07 0:51 UTC (permalink / raw)
To: leon, jgg
Cc: linux-rdma, andrew.gospodarek, kalesh-anakkur.purayil,
saravanan.vajravel, Selvin Xavier
Advertise the PBL page size and first-word offset (FWO) for SQ/RQ
in CREATE_QP so the firmware can use huge-page QP buffers.
Signed-off-by: Saravanan Vajravel <saravanan.vajravel@broadcom.com>
Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com>
---
drivers/infiniband/hw/bnxt_re/qplib_fp.c | 19 +++++++++++++-----
drivers/infiniband/hw/bnxt_re/roce_hsi.h | 25 ++++++++++++++++++++++--
2 files changed, 37 insertions(+), 7 deletions(-)
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index 0319fe76c634..2fb06abda552 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -985,9 +985,14 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
pbl = &sq->hwq.pbl[PBL_LVL_0];
req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
req.sq_pg_size_sq_lvl = sq->hwq.pg_sz_lvl;
+ if (bnxt_re_pbl_size_supported(res->dattr->dev_cap_ext_flags_1)) {
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SQ_PBL_PG_SIZE_VALID;
+ req.sq_pbl_pg_size = bnxt_qplib_get_pbl_page_size(&sq->sg_info);
+ }
req.sq_fwo_sq_sge =
- cpu_to_le16(((sq->max_sge & CMDQ_CREATE_QP_SQ_SGE_MASK) <<
- CMDQ_CREATE_QP_SQ_SGE_SFT) | 0);
+ cpu_to_le16((((sq->sg_info.fwo_offset >> BNXT_QPLIB_QP_FWO_SHIFT) <<
+ CMDQ_CREATE_QP_SQ_FWO_SFT) & CMDQ_CREATE_QP_SQ_FWO_MASK) |
+ (sq->max_sge & CMDQ_CREATE_QP_SQ_SGE_MASK));
req.scq_cid = cpu_to_le32(qp->scq->id);
/* RQ */
@@ -997,12 +1002,16 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
pbl = &rq->hwq.pbl[PBL_LVL_0];
req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
req.rq_pg_size_rq_lvl = rq->hwq.pg_sz_lvl;
+ if (bnxt_re_pbl_size_supported(res->dattr->dev_cap_ext_flags_1)) {
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RQ_PBL_PG_SIZE_VALID;
+ req.rq_pbl_pg_size = bnxt_qplib_get_pbl_page_size(&rq->sg_info);
+ }
nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
6 : rq->max_sge;
req.rq_fwo_rq_sge =
- cpu_to_le16(((nsge &
- CMDQ_CREATE_QP_RQ_SGE_MASK) <<
- CMDQ_CREATE_QP_RQ_SGE_SFT) | 0);
+ cpu_to_le16((((rq->sg_info.fwo_offset >> BNXT_QPLIB_QP_FWO_SHIFT) <<
+ CMDQ_CREATE_QP_RQ_FWO_SFT) & CMDQ_CREATE_QP_RQ_FWO_MASK) |
+ (nsge & CMDQ_CREATE_QP_RQ_SGE_MASK));
} else {
/* SRQ */
qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;
diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
index bbcd723cd2b5..aa85cd379e48 100644
--- a/drivers/infiniband/hw/bnxt_re/roce_hsi.h
+++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
@@ -437,8 +437,10 @@ struct cmdq_create_qp {
#define CMDQ_CREATE_QP_QP_FLAGS_EXPRESS_MODE_ENABLED 0x100UL
#define CMDQ_CREATE_QP_QP_FLAGS_STEERING_TAG_VALID 0x200UL
#define CMDQ_CREATE_QP_QP_FLAGS_RDMA_READ_OR_ATOMICS_USED 0x400UL
+ #define CMDQ_CREATE_QP_QP_FLAGS_RQ_PBL_PG_SIZE_VALID 0x2000UL
+ #define CMDQ_CREATE_QP_QP_FLAGS_SQ_PBL_PG_SIZE_VALID 0x4000UL
#define CMDQ_CREATE_QP_QP_FLAGS_LAST \
- CMDQ_CREATE_QP_QP_FLAGS_RDMA_READ_OR_ATOMICS_USED
+ CMDQ_CREATE_QP_QP_FLAGS_SQ_PBL_PG_SIZE_VALID
u8 type;
#define CMDQ_CREATE_QP_TYPE_RC 0x2UL
#define CMDQ_CREATE_QP_TYPE_UD 0x4UL
@@ -501,7 +503,26 @@ struct cmdq_create_qp {
__le64 orrq_addr;
__le32 request_xid;
__le16 steering_tag;
- __le16 reserved16;
+ u8 sq_pbl_pg_size;
+ #define CMDQ_CREATE_QP_SQ_PBL_PG_SIZE_MASK 0xfUL
+ #define CMDQ_CREATE_QP_SQ_PBL_PG_SIZE_SFT 0
+ #define CMDQ_CREATE_QP_SQ_PBL_PG_SIZE_PG_4K 0x0UL
+ #define CMDQ_CREATE_QP_SQ_PBL_PG_SIZE_PG_8K 0x1UL
+ #define CMDQ_CREATE_QP_SQ_PBL_PG_SIZE_PG_64K 0x2UL
+ #define CMDQ_CREATE_QP_SQ_PBL_PG_SIZE_PG_2M 0x3UL
+ #define CMDQ_CREATE_QP_SQ_PBL_PG_SIZE_PG_8M 0x4UL
+ #define CMDQ_CREATE_QP_SQ_PBL_PG_SIZE_PG_1G 0x5UL
+ #define CMDQ_CREATE_QP_SQ_PBL_PG_SIZE_LAST CMDQ_CREATE_QP_SQ_PBL_PG_SIZE_PG_1G
+ u8 rq_pbl_pg_size;
+ #define CMDQ_CREATE_QP_RQ_PBL_PG_SIZE_MASK 0xfUL
+ #define CMDQ_CREATE_QP_RQ_PBL_PG_SIZE_SFT 0
+ #define CMDQ_CREATE_QP_RQ_PBL_PG_SIZE_PG_4K 0x0UL
+ #define CMDQ_CREATE_QP_RQ_PBL_PG_SIZE_PG_8K 0x1UL
+ #define CMDQ_CREATE_QP_RQ_PBL_PG_SIZE_PG_64K 0x2UL
+ #define CMDQ_CREATE_QP_RQ_PBL_PG_SIZE_PG_2M 0x3UL
+ #define CMDQ_CREATE_QP_RQ_PBL_PG_SIZE_PG_8M 0x4UL
+ #define CMDQ_CREATE_QP_RQ_PBL_PG_SIZE_PG_1G 0x5UL
+ #define CMDQ_CREATE_QP_RQ_PBL_PG_SIZE_LAST CMDQ_CREATE_QP_RQ_PBL_PG_SIZE_PG_1G
};
/* creq_create_qp_resp (size:128b/16B) */
--
2.39.3
^ permalink raw reply related [flat|nested] 5+ messages in thread