* [PATCH 01/15] bnge: Add infrastructure support for RoCE MPC channels
2026-09-04 10:43 [PATCH 00/15] Add BNG_RE control path verbs Siva Reddy Kallam
@ 2026-09-04 10:43 ` Siva Reddy Kallam
2026-09-04 10:43 ` [PATCH 02/15] bnge: Add HSI definitions for 64-bit doorbell and " Siva Reddy Kallam
` (13 subsequent siblings)
14 siblings, 0 replies; 17+ messages in thread
From: Siva Reddy Kallam @ 2026-09-04 10:43 UTC (permalink / raw)
To: leonro, jgg, davem, edumazet, kuba, pabeni, andrew+netdev, horms
Cc: netdev, linux-kernel, linux-rdma, Saswati Das, Siva Reddy Kallam,
Vikas Gupta, Dharmender Garg
From: Saswati Das <saswati.das@broadcom.com>
Add the initial L2 driver plumbing for RoCE MPC (PRIMATE channel)
support:
- Plumb BAR1 through to the RoCE auxiliary device info, alongside the
existing BAR0, so the RoCE driver can reach the doorbell BAR.
This will be used by follow-on patches to issue RoCE commands over the
MPC channels to the firmware.
Signed-off-by: Saswati Das <saswati.das@broadcom.com>
Signed-off-by: Siva Reddy Kallam <siva.kallam@broadcom.com>
Reviewed-by: Vikas Gupta <vikas.gupta@broadcom.com>
Reviewed-by: Dharmender Garg <dharmender.garg@broadcom.com>
---
drivers/net/ethernet/broadcom/bnge/bnge_auxr.c | 1 +
drivers/net/ethernet/broadcom/bnge/bnge_auxr.h | 1 +
2 files changed, 2 insertions(+)
diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_auxr.c b/drivers/net/ethernet/broadcom/bnge/bnge_auxr.c
index 0955b488b6fe..87ae40eab592 100644
--- a/drivers/net/ethernet/broadcom/bnge/bnge_auxr.c
+++ b/drivers/net/ethernet/broadcom/bnge/bnge_auxr.c
@@ -181,6 +181,7 @@ static void bnge_set_auxr_dev_info(struct bnge_auxr_dev *auxr_dev,
auxr_dev->pf_port_id = bd->pf.port_id;
auxr_dev->en_state = bd->state;
auxr_dev->bar0 = bd->bar0;
+ auxr_dev->bar1 = bd->bar1;
}
void bnge_rdma_aux_device_add(struct bnge_dev *bd)
diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_auxr.h b/drivers/net/ethernet/broadcom/bnge/bnge_auxr.h
index 6c5c15ef2b0a..13cb9289f5b8 100644
--- a/drivers/net/ethernet/broadcom/bnge/bnge_auxr.h
+++ b/drivers/net/ethernet/broadcom/bnge/bnge_auxr.h
@@ -46,6 +46,7 @@ struct bnge_auxr_dev {
struct net_device *net;
struct pci_dev *pdev;
void __iomem *bar0;
+ void __iomem *bar1;
struct bnge_msix_info msix_info[BNGE_MAX_ROCE_MSIX];
--
2.43.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 02/15] bnge: Add HSI definitions for 64-bit doorbell and MPC channels
2026-09-04 10:43 [PATCH 00/15] Add BNG_RE control path verbs Siva Reddy Kallam
2026-09-04 10:43 ` [PATCH 01/15] bnge: Add infrastructure support for RoCE MPC channels Siva Reddy Kallam
@ 2026-09-04 10:43 ` Siva Reddy Kallam
2026-09-04 10:43 ` [PATCH 03/15] bnge: reserve TX/completion rings for the RoCE MPC channel Siva Reddy Kallam
` (12 subsequent siblings)
14 siblings, 0 replies; 17+ messages in thread
From: Siva Reddy Kallam @ 2026-09-04 10:43 UTC (permalink / raw)
To: leonro, jgg, davem, edumazet, kuba, pabeni, andrew+netdev, horms
Cc: netdev, linux-kernel, linux-rdma, Saswati Das, Siva Reddy Kallam,
Vikas Gupta, Dharmender Garg
From: Saswati Das <saswati.das@broadcom.com>
Add the Hardware-Software Interface (HSI) structure and macro definitions
required to support 64-bit doorbells and Multi-Path Channel (MPC)
firmware commands/completions for next-generation RoCE:
- Add struct dbc_dbc64: Introduce the 64-bit doorbell layout along with
field masks, shift offsets, and type/path enumerations. This includes
queue types (SQ, RQ, SRQ, CQ, NQ, etc.) and paths (ROCE, L2, ENGINE).
- Add MPC Command Macros: Define the main MPC request types (ICA, RCA,
PFVF, EVENT) and their corresponding subtypes used to manage security
associations (SA enable, key rotation, SA stats/init), QP/AH
modification/batching, and PF/VF commands (QP/AH creation,
modification, and destruction).
- Add MPC Completion Macros & Structs: Add completion error codes for ICA
and introduce the 8-byte completion header structure (struct
mpc_cmpl_hdr) used to parse MPC firmware responses.
These definitions will be used by the driver to interface with the
firmware over the MPC/PRIMATE channels.
Signed-off-by: Saswati Das <saswati.das@broadcom.com>
Signed-off-by: Siva Reddy Kallam <siva.kallam@broadcom.com>
Reviewed-by: Vikas Gupta <vikas.gupta@broadcom.com>
Reviewed-by: Dharmender Garg <dharmender.garg@broadcom.com>
---
include/linux/bnge/hsi.h | 161 +++++++++++++++++++++++++++++++++++++++
1 file changed, 161 insertions(+)
diff --git a/include/linux/bnge/hsi.h b/include/linux/bnge/hsi.h
index 1f7bd96415a5..a3383f2c7381 100644
--- a/include/linux/bnge/hsi.h
+++ b/include/linux/bnge/hsi.h
@@ -12510,6 +12510,43 @@ struct dbc_dbc {
#define DBC_DBC_TYPE_LAST DBC_DBC_TYPE_NULL
};
+/* dbc_dbc64 (size:64b/8B) */
+struct dbc_dbc64 {
+ __le64 dbc;
+ #define DBC_DBC64_INDEX_MASK 0xffffffUL
+ #define DBC_DBC64_INDEX_SFT 0
+ #define DBC_DBC64_EPOCH 0x1000000UL
+ #define DBC_DBC64_TOGGLE_MASK 0x6000000UL
+ #define DBC_DBC64_TOGGLE_SFT 25
+ #define DBC_DBC64_XID_MASK 0xfffff00000000ULL
+ #define DBC_DBC64_XID_SFT 32
+ #define DBC_DBC64_PATH_MASK 0x300000000000000ULL
+ #define DBC_DBC64_PATH_SFT 56
+ #define DBC_DBC64_PATH_ROCE (0x0ULL << 56)
+ #define DBC_DBC64_PATH_L2 (0x1ULL << 56)
+ #define DBC_DBC64_PATH_ENGINE (0x2ULL << 56)
+ #define DBC_DBC64_PATH_LAST DBC_DBC64_PATH_ENGINE
+ #define DBC_DBC64_VALID 0x400000000000000ULL
+ #define DBC_DBC64_DEBUG_TRACE 0x800000000000000ULL
+ #define DBC_DBC64_TYPE_MASK 0xf000000000000000ULL
+ #define DBC_DBC64_TYPE_SFT 60
+ #define DBC_DBC64_TYPE_SQ (0x0ULL << 60)
+ #define DBC_DBC64_TYPE_RQ (0x1ULL << 60)
+ #define DBC_DBC64_TYPE_SRQ (0x2ULL << 60)
+ #define DBC_DBC64_TYPE_SRQ_ARM (0x3ULL << 60)
+ #define DBC_DBC64_TYPE_CQ (0x4ULL << 60)
+ #define DBC_DBC64_TYPE_CQ_ARMSE (0x5ULL << 60)
+ #define DBC_DBC64_TYPE_CQ_ARMALL (0x6ULL << 60)
+ #define DBC_DBC64_TYPE_CQ_ARMENA (0x7ULL << 60)
+ #define DBC_DBC64_TYPE_SRQ_ARMENA (0x8ULL << 60)
+ #define DBC_DBC64_TYPE_CQ_CUTOFF_ACK (0x9ULL << 60)
+ #define DBC_DBC64_TYPE_NQ (0xaULL << 60)
+ #define DBC_DBC64_TYPE_NQ_ARM (0xbULL << 60)
+ #define DBC_DBC64_TYPE_NQ_MASK (0xeULL << 60)
+ #define DBC_DBC64_TYPE_NULL (0xfULL << 60)
+ #define DBC_DBC64_TYPE_LAST DBC_DBC64_TYPE_NULL
+};
+
/* db_push_start (size:64b/8B) */
struct db_push_start {
u64 db;
@@ -12605,4 +12642,128 @@ struct hcomm_status {
#define HCOMM_STATUS_STRUCT_LOC 0x31001F0UL
+/* MPC macros */
+#define MPC_CMD_HDR_REQ_TYPE_ICA 0x1UL
+#define MPC_CMD_HDR_REQ_TYPE_RCA 0x2UL
+#define MPC_CMD_HDR_REQ_TYPE_PFVF 0x3UL
+#define MPC_CMD_HDR_REQ_TYPE_EVENT 0x4UL
+#define MPC_CMD_HDR_REQ_TYPE_LAST MPC_CMD_HDR_REQ_TYPE_EVENT
+
+#define MPC_CMD_HDR_REQ_SUB_TYPE_ICA_SA_ENABLE 0x0UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_ICA_SM_PSP_SA_INIT 0x1UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_ICA_SM_PSP_NEW_RX_ASSOC 0x2UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_ICA_SM_PSP_KEY_ROTATE 0x3UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_ICA_CM_PSP_SA_INIT 0x4UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_ICA_CM_PSP_KEY_ROTATE 0x5UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_ICA_CM_PSP_KEY_SET 0x6UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_ICA_SA_INFO_GET 0x7UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_ICA_SA_STATS_GET 0x8UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_ICA_SA_STATS_CLEAR 0x9UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_ICA_SA_RX_MATCH_RULE_SET 0xaUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_ICA_SM_PSP_SA_SET 0xbUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_RCA_QP_MODIFY 0x0UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_RCA_AH_MODIFY 0x1UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_RCA_QP_MODIFY_CMPL 0x2UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_RCA_AH_MODIFY_CMPL 0x3UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_RCA_QP_MODIFY_BATCH 0x4UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_RCA_AH_MODIFY_BATCH 0x5UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_RCA_QP_MODIFY_BATCH_CMPL 0x6UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_RCA_AH_MODIFY_BATCH_CMPL 0x7UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_RCA_MPC_TEST_CMD 0xc8UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_RCA_MPC_TEST_CMPL 0xc9UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_EVENT_RESP_CMPL 0x0UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_CREATE 0x0UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_CREATE 0x1UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_MODIFY 0x2UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_MODIFY 0x3UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_DESTROY 0x4UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_DESTROY 0x5UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_CREATE_CMPL 0xaUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_CREATE_CMPL 0xbUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_MODIFY_CMPL 0xcUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_MODIFY_CMPL 0xdUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_DESTROY_CMPL 0xeUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_DESTROY_CMPL 0xfUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_CREATE_FWD_REQ 0x14UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_CREATE_FWD_REQ 0x15UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_MODIFY_FWD_REQ 0x16UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_MODIFY_FWD_REQ 0x17UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_DESTROY_FWD_REQ 0x18UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_DESTROY_FWD_REQ 0x19UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_CREATE_FWD_RESP 0x1eUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_CREATE_FWD_RESP 0x1fUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_MODIFY_FWD_RESP 0x20UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_MODIFY_FWD_RESP 0x21UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_DESTROY_FWD_RESP 0x22UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_DESTROY_FWD_RESP 0x23UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_ALL_FWD_RESP_CMPL 0x28UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_CREATE_BATCH 0x32UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_CREATE_BATCH 0x33UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_MODIFY_BATCH 0x34UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_MODIFY_BATCH 0x35UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_DESTROY_BATCH 0x36UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_DESTROY_BATCH 0x37UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_CREATE_BATCH_CMPL 0x3cUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_CREATE_BATCH_CMPL 0x3dUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_MODIFY_BATCH_CMPL 0x3eUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_MODIFY_BATCH_CMPL 0x3fUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_DESTROY_BATCH_CMPL 0x40UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_DESTROY_BATCH_CMPL 0x41UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_CREATE_BATCH_FWD_REQ 0x46UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_CREATE_BATCH_FWD_REQ 0x47UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_MODIFY_BATCH_FWD_REQ 0x48UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_MODIFY_BATCH_FWD_REQ 0x49UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_DESTROY_BATCH_FWD_REQ 0x4aUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_DESTROY_BATCH_FWD_REQ 0x4bUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_CREATE_BATCH_FWD_RESP 0x50UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_CREATE_BATCH_FWD_RESP 0x51UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_MODIFY_BATCH_FWD_RESP 0x52UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_MODIFY_BATCH_FWD_RESP 0x53UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_DESTROY_BATCH_FWD_RESP 0x54UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_DESTROY_BATCH_FWD_RESP 0x55UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_ALL_BATCH_FWD_RESP_CMPL 0x5aUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_DRV_RGTR 0x64UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_DRV_RGTR_CMPL 0x65UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_DRV_UNRGTR 0x66UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_DRV_UNRGTR_CMPL 0x67UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_BUF_RGTR 0x68UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_BUF_RGTR_CMPL 0x69UL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_BUF_UNRGTR 0x6aUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_BUF_UNRGTR_CMPL 0x6bUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_MPC_TEST_CMD 0xcaUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_MPC_TEST_CMPL 0xcbUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_MPC_TEST_FWD_CMD 0xccUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_MPC_TEST_FWD_CMPL 0xcdUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_MPC_TEST_FWD_REQ 0xceUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_MPC_TEST_FWD_RESP 0xcfUL
+#define MPC_CMD_HDR_REQ_SUB_TYPE_LAST MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_MPC_TEST_FWD_RESP
+
+#define MPC_CMPL_HDR_ERROR_CODE_ICA_ERR_NONE 0x0UL
+#define MPC_CMPL_HDR_ERROR_CODE_ICA_ERR_BAD_TARGET_ID 0x1UL
+#define MPC_CMPL_HDR_ERROR_CODE_ICA_ERR_SA_NOT_INIT_ENABLED 0x2UL
+#define MPC_CMPL_HDR_ERROR_CODE_ICA_ERR_HW_CMD_FAILED 0x3UL
+#define MPC_CMPL_HDR_ERROR_CODE_ICA_ERR_BAD_PSP_MODE 0x4UL
+#define MPC_CMPL_HDR_ERROR_CODE_ICA_ERR_BAD_SPI 0x5UL
+#define MPC_CMPL_HDR_ERROR_CODE_ICA_ERR_SPI_THRESHOLD_BREACHED 0x6UL
+#define MPC_CMPL_HDR_ERROR_CODE_ICA_ERR_ACTIVE_QPS_EXIST 0x7UL
+#define MPC_CMPL_HDR_ERROR_CODE_LAST MPC_CMPL_HDR_ERROR_CODE_ICA_ERR_ACTIVE_QPS_EXIST
+
+/* mpc_cmpl_hdr (size:64b/8B) */
+struct mpc_cmpl_hdr {
+ u8 cmpl_type_reserved;
+ #define CMPL_TYPE_RESERVED_CMPL_TYPE_MASK 0x3fUL
+ #define CMPL_TYPE_RESERVED_CMPL_TYPE_SFT 0
+ #define CMPL_TYPE_RESERVED_CMPL_TYPE_MPC_CMP_SHORT 0x1eUL
+ #define CMPL_TYPE_RESERVED_CMPL_TYPE_MPC_CMP_LONG 0x1fUL
+ #define CMPL_TYPE_RESERVED_CMPL_TYPE_LAST CMPL_TYPE_RESERVED_CMPL_TYPE_MPC_CMP_LONG
+ u8 error_code_mp_client;
+ #define ERROR_CODE_MP_CLIENT_ERROR_CODE_MASK 0xfUL
+ #define ERROR_CODE_MP_CLIENT_ERROR_CODE_SFT 0
+ #define ERROR_CODE_MP_CLIENT_MP_CLIENT_MASK 0xf0UL
+ #define ERROR_CODE_MP_CLIENT_MP_CLIENT_SFT 4
+ u8 req_type;
+ u8 req_subtype;
+ __le32 opaque;
+};
+
#endif /* _BNGE_HSI_H_ */
--
2.43.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 03/15] bnge: reserve TX/completion rings for the RoCE MPC channel
2026-09-04 10:43 [PATCH 00/15] Add BNG_RE control path verbs Siva Reddy Kallam
2026-09-04 10:43 ` [PATCH 01/15] bnge: Add infrastructure support for RoCE MPC channels Siva Reddy Kallam
2026-09-04 10:43 ` [PATCH 02/15] bnge: Add HSI definitions for 64-bit doorbell and " Siva Reddy Kallam
@ 2026-09-04 10:43 ` Siva Reddy Kallam
2026-09-04 15:52 ` Jakub Kicinski
2026-09-04 10:43 ` [PATCH 04/15] RDMA/bng_re: Add MPC, XID management, doorbell infrastructure Siva Reddy Kallam
` (11 subsequent siblings)
14 siblings, 1 reply; 17+ messages in thread
From: Siva Reddy Kallam @ 2026-09-04 10:43 UTC (permalink / raw)
To: leonro, jgg, davem, edumazet, kuba, pabeni, andrew+netdev, horms
Cc: netdev, linux-kernel, linux-rdma, Sachin Holla, Siva Reddy Kallam,
Vikas Gupta, Dharmender Garg
From: Sachin Holla <sachin.holla@broadcom.com>
When RoCE is enabled, the bng_re driver allocates its own TX/CQ rings
for the MPC control channel out of the same per-function FW ring pool.
Enhanced bnge_reserve_rings() to account for this extra ring count to
avoid pool overflow while allocating netdev rings.
Signed-off-by: Sachin Holla <sachin.holla@broadcom.com>
Signed-off-by: Siva Reddy Kallam <siva.kallam@broadcom.com>
Reviewed-by: Vikas Gupta <vikas.gupta@broadcom.com>
Reviewed-by: Dharmender Garg <dharmender.garg@broadcom.com>
---
.../net/ethernet/broadcom/bnge/bnge_resc.c | 32 ++++++++++++++++---
1 file changed, 28 insertions(+), 4 deletions(-)
diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_resc.c b/drivers/net/ethernet/broadcom/bnge/bnge_resc.c
index 4711dd4945ff..9dfb22b7c7c7 100644
--- a/drivers/net/ethernet/broadcom/bnge/bnge_resc.c
+++ b/drivers/net/ethernet/broadcom/bnge/bnge_resc.c
@@ -110,9 +110,24 @@ static u16 bnge_nqs_demand(struct bnge_dev *bd)
return bd->nq_nr_rings + bnge_aux_get_msix(bd);
}
+static u16 bnge_tx_rings_demand(struct bnge_dev *bd)
+{
+ u16 tx_rings = bd->tx_nr_rings;
+
+ if (bnge_is_roce_en(bd))
+ tx_rings += 1; /* For MPC TX ring */
+
+ return tx_rings;
+}
+
static u16 bnge_cprs_demand(struct bnge_dev *bd)
{
- return bd->tx_nr_rings + bd->rx_nr_rings;
+ u16 cprs = bd->tx_nr_rings + bd->rx_nr_rings;
+
+ if (bnge_is_roce_en(bd))
+ cprs += 1; /* For MPC CQ ring */
+
+ return cprs;
}
static u16 bnge_get_avail_msix(struct bnge_dev *bd, int num)
@@ -250,7 +265,7 @@ static bool bnge_need_reserve_rings(struct bnge_dev *bd)
u16 nqs = bnge_nqs_demand(bd);
u16 vnic;
- if (hw_resc->resv_tx_rings != bd->tx_nr_rings)
+ if (hw_resc->resv_tx_rings != bnge_tx_rings_demand(bd))
return true;
vnic = bnge_get_total_vnics(bd, rx);
@@ -275,6 +290,7 @@ int bnge_reserve_rings(struct bnge_dev *bd)
u16 nq = bd->nq_nr_rings;
u16 aux_msix = 0;
bool sh = false;
+ u16 tx_demand;
u16 tx_cp;
int rc;
@@ -293,11 +309,12 @@ int bnge_reserve_rings(struct bnge_dev *bd)
hwr.nq = bnge_nqs_demand(bd);
}
- hwr.tx = bd->tx_nr_rings;
+ tx_demand = bnge_tx_rings_demand(bd);
+ hwr.tx = tx_demand;
hwr.rx = bd->rx_nr_rings;
if (bd->flags & BNGE_EN_SHARED_CHNL)
sh = true;
- hwr.cmpl = hwr.rx + hwr.tx;
+ hwr.cmpl = bnge_cprs_demand(bd);
hwr.vnic = bnge_get_total_vnics(bd, hwr.rx);
@@ -327,6 +344,13 @@ int bnge_reserve_rings(struct bnge_dev *bd)
if (hwr.stat > bnge_aux_get_stat_ctxs(bd))
hwr.stat -= bnge_aux_get_stat_ctxs(bd);
hwr.nq = min_t(u16, hwr.nq, hwr.stat);
+ /* Drop MPC TX ring from L2 TX ring count; it is owned by RoCE driver */
+ if (bnge_is_roce_en(bd)) {
+ if (hwr.tx >= tx_demand)
+ hwr.tx -= 1;
+ else
+ return -ENOMEM;
+ }
/* Adjust the rings */
rc = bnge_adjust_rings(bd, &rx_rings, &hwr.tx, hwr.nq, sh);
--
2.43.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [PATCH 03/15] bnge: reserve TX/completion rings for the RoCE MPC channel
2026-09-04 10:43 ` [PATCH 03/15] bnge: reserve TX/completion rings for the RoCE MPC channel Siva Reddy Kallam
@ 2026-09-04 15:52 ` Jakub Kicinski
0 siblings, 0 replies; 17+ messages in thread
From: Jakub Kicinski @ 2026-09-04 15:52 UTC (permalink / raw)
To: Siva Reddy Kallam
Cc: leonro, jgg, davem, edumazet, pabeni, andrew+netdev, horms,
netdev, linux-kernel, linux-rdma, Sachin Holla, Vikas Gupta,
Dharmender Garg
On Fri, 4 Sep 2026 03:43:11 -0700 Siva Reddy Kallam wrote:
> @@ -293,11 +309,12 @@ int bnge_reserve_rings(struct bnge_dev *bd)
> hwr.nq = bnge_nqs_demand(bd);
> }
>
> - hwr.tx = bd->tx_nr_rings;
> + tx_demand = bnge_tx_rings_demand(bd);
> + hwr.tx = tx_demand;
> hwr.rx = bd->rx_nr_rings;
> if (bd->flags & BNGE_EN_SHARED_CHNL)
> sh = true;
> - hwr.cmpl = hwr.rx + hwr.tx;
> + hwr.cmpl = bnge_cprs_demand(bd);
This if nothing else has a solid chance of conflicting with netdev.
Please put the first 3 patches in a pull request so that both netdev
and RDMA can pull that. Once it's pulled post the remaining 13 to RDMA.
^ permalink raw reply [flat|nested] 17+ messages in thread
* [PATCH 04/15] RDMA/bng_re: Add MPC, XID management, doorbell infrastructure
2026-09-04 10:43 [PATCH 00/15] Add BNG_RE control path verbs Siva Reddy Kallam
` (2 preceding siblings ...)
2026-09-04 10:43 ` [PATCH 03/15] bnge: reserve TX/completion rings for the RoCE MPC channel Siva Reddy Kallam
@ 2026-09-04 10:43 ` Siva Reddy Kallam
2026-09-04 10:43 ` [PATCH 05/15] RDMA/bng_re: Add support verbs Siva Reddy Kallam
` (10 subsequent siblings)
14 siblings, 0 replies; 17+ messages in thread
From: Siva Reddy Kallam @ 2026-09-04 10:43 UTC (permalink / raw)
To: leonro, jgg, davem, edumazet, kuba, pabeni, andrew+netdev, horms
Cc: netdev, linux-kernel, linux-rdma, Raqib Jones, Siva Reddy Kallam
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=all, Size: 158435 bytes --]
From: Raqib Jones <raqib.jones@broadcom.com>
This patch has below changes
New MPC channel introduced for AH/QP operations
xid management support for AH/QP
Doorbell infrastructure changes
Signed-off-by: Raqib Jones <raqib.jones@broadcom.com>
Signed-off-by: Siva Reddy Kallam <siva.kallam@broadcom.com>
---
drivers/infiniband/hw/bng_re/Makefile | 13 +-
drivers/infiniband/hw/bng_re/bng_debugfs.c | 308 +++
drivers/infiniband/hw/bng_re/bng_dev.c | 98 +-
drivers/infiniband/hw/bng_re/bng_fw.c | 1 +
drivers/infiniband/hw/bng_re/bng_fw.h | 8 +-
drivers/infiniband/hw/bng_re/bng_mpc.c | 574 +++++
drivers/infiniband/hw/bng_re/bng_re.h | 174 ++
.../infiniband/hw/bng_re/bng_re_mpc_roce.c | 1863 +++++++++++++++++
.../infiniband/hw/bng_re/bng_re_mpc_roce.h | 218 ++
drivers/infiniband/hw/bng_re/bng_res.c | 114 +-
drivers/infiniband/hw/bng_re/bng_res.h | 78 +-
drivers/infiniband/hw/bng_re/bng_roce_hsi.h | 94 +
drivers/infiniband/hw/bng_re/bng_sp.h | 4 +
drivers/infiniband/hw/bng_re/bng_xid.c | 110 +
drivers/infiniband/hw/bng_re/bng_xid.h | 26 +
.../infiniband/hw/bng_re/bng_xid_allocator.c | 870 ++++++++
drivers/infiniband/hw/bng_re/xid_allocator.h | 114 +
17 files changed, 4631 insertions(+), 36 deletions(-)
create mode 100644 drivers/infiniband/hw/bng_re/bng_mpc.c
create mode 100644 drivers/infiniband/hw/bng_re/bng_re_mpc_roce.c
create mode 100644 drivers/infiniband/hw/bng_re/bng_re_mpc_roce.h
create mode 100644 drivers/infiniband/hw/bng_re/bng_xid.c
create mode 100644 drivers/infiniband/hw/bng_re/bng_xid.h
create mode 100644 drivers/infiniband/hw/bng_re/bng_xid_allocator.c
create mode 100644 drivers/infiniband/hw/bng_re/xid_allocator.h
diff --git a/drivers/infiniband/hw/bng_re/Makefile b/drivers/infiniband/hw/bng_re/Makefile
index 17e9d5871d40..ea0c6951728f 100644
--- a/drivers/infiniband/hw/bng_re/Makefile
+++ b/drivers/infiniband/hw/bng_re/Makefile
@@ -3,6 +3,13 @@ ccflags-y := -I $(srctree)/drivers/net/ethernet/broadcom/bnge
obj-$(CONFIG_INFINIBAND_BNG_RE) += bng_re.o
-bng_re-y := bng_dev.o bng_fw.o \
- bng_res.o bng_sp.o \
- bng_debugfs.o
+bng_re-y := \
+ bng_xid_allocator.o \
+ bng_xid.o \
+ bng_mpc.o \
+ bng_re_mpc_roce.o \
+ bng_res.o \
+ bng_debugfs.o \
+ bng_fw.o \
+ bng_dev.o \
+ bng_sp.o
diff --git a/drivers/infiniband/hw/bng_re/bng_debugfs.c b/drivers/infiniband/hw/bng_re/bng_debugfs.c
index 9ec5a8785250..7f1e9bedc4e7 100644
--- a/drivers/infiniband/hw/bng_re/bng_debugfs.c
+++ b/drivers/infiniband/hw/bng_re/bng_debugfs.c
@@ -2,24 +2,332 @@
// Copyright (c) 2025 Broadcom.
#include <linux/debugfs.h>
#include <linux/pci.h>
+#include <linux/seq_file.h>
+#include <linux/uaccess.h>
#include <rdma/ib_verbs.h>
#include "bng_res.h"
+#include "bng_sp.h"
#include "bng_fw.h"
#include "bnge.h"
#include "bnge_auxr.h"
#include "bng_re.h"
#include "bng_debugfs.h"
+#include "bng_re_mpc_roce.h"
static struct dentry *bng_re_debugfs_root;
+/* MPC tuning parameter read/write functions */
+static ssize_t mpc_short_timeout_read(struct file *file, char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct bng_re_dev *rdev = file->private_data;
+ char tmp[16];
+
+ if (!rdev || !rdev->mpc_roce_creq.mpc_tune_params)
+ return -ENODEV;
+
+ snprintf(tmp, sizeof(tmp), "%u\n", rdev->mpc_roce_creq.mpc_tune_params->short_timeout);
+ return simple_read_from_buffer(buf, count, ppos, tmp, strlen(tmp));
+}
+
+static ssize_t mpc_short_timeout_write(struct file *file, const char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct bng_re_dev *rdev = file->private_data;
+ u32 val;
+ int ret;
+
+ if (!rdev || !rdev->mpc_roce_creq.mpc_tune_params)
+ return -ENODEV;
+
+ ret = kstrtou32_from_user(buf, count, 0, &val);
+ if (ret)
+ return ret;
+
+ rdev->mpc_roce_creq.mpc_tune_params->short_timeout = val;
+ return count;
+}
+
+static const struct file_operations mpc_short_timeout_fops = {
+ .owner = THIS_MODULE,
+ .open = simple_open,
+ .read = mpc_short_timeout_read,
+ .write = mpc_short_timeout_write,
+};
+
+static ssize_t mpc_long_timeout_read(struct file *file, char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct bng_re_dev *rdev = file->private_data;
+ char tmp[16];
+
+ if (!rdev || !rdev->mpc_roce_creq.mpc_tune_params)
+ return -ENODEV;
+
+ snprintf(tmp, sizeof(tmp), "%u\n", rdev->mpc_roce_creq.mpc_tune_params->long_timeout);
+ return simple_read_from_buffer(buf, count, ppos, tmp, strlen(tmp));
+}
+
+static ssize_t mpc_long_timeout_write(struct file *file, const char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct bng_re_dev *rdev = file->private_data;
+ u32 val;
+ int ret;
+
+ if (!rdev || !rdev->mpc_roce_creq.mpc_tune_params)
+ return -ENODEV;
+
+ ret = kstrtou32_from_user(buf, count, 0, &val);
+ if (ret)
+ return ret;
+
+ if (!val)
+ return -EINVAL;
+
+ rdev->mpc_roce_creq.mpc_tune_params->long_timeout = val;
+ return count;
+}
+
+static const struct file_operations mpc_long_timeout_fops = {
+ .owner = THIS_MODULE,
+ .open = simple_open,
+ .read = mpc_long_timeout_read,
+ .write = mpc_long_timeout_write,
+};
+
+static ssize_t mpc_max_retries_read(struct file *file, char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct bng_re_dev *rdev = file->private_data;
+ char tmp[16];
+
+ if (!rdev || !rdev->mpc_roce_creq.mpc_tune_params)
+ return -ENODEV;
+
+ snprintf(tmp, sizeof(tmp), "%u\n", rdev->mpc_roce_creq.mpc_tune_params->max_retries);
+ return simple_read_from_buffer(buf, count, ppos, tmp, strlen(tmp));
+}
+
+static ssize_t mpc_max_retries_write(struct file *file, const char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct bng_re_dev *rdev = file->private_data;
+ u16 val;
+ int ret;
+
+ if (!rdev || !rdev->mpc_roce_creq.mpc_tune_params)
+ return -ENODEV;
+
+ ret = kstrtou16_from_user(buf, count, 0, &val);
+ if (ret)
+ return ret;
+
+ rdev->mpc_roce_creq.mpc_tune_params->max_retries = val;
+ return count;
+}
+
+static const struct file_operations mpc_max_retries_fops = {
+ .owner = THIS_MODULE,
+ .open = simple_open,
+ .read = mpc_max_retries_read,
+ .write = mpc_max_retries_write,
+};
+
+static ssize_t mpc_retry_sleep_read(struct file *file, char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct bng_re_dev *rdev = file->private_data;
+ char tmp[16];
+
+ if (!rdev || !rdev->mpc_roce_creq.mpc_tune_params)
+ return -ENODEV;
+
+ snprintf(tmp, sizeof(tmp), "%u\n", rdev->mpc_roce_creq.mpc_tune_params->retry_sleep);
+ return simple_read_from_buffer(buf, count, ppos, tmp, strlen(tmp));
+}
+
+static ssize_t mpc_retry_sleep_write(struct file *file, const char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct bng_re_dev *rdev = file->private_data;
+ u16 val;
+ int ret;
+
+ if (!rdev || !rdev->mpc_roce_creq.mpc_tune_params)
+ return -ENODEV;
+
+ ret = kstrtou16_from_user(buf, count, 0, &val);
+ if (ret)
+ return ret;
+
+ rdev->mpc_roce_creq.mpc_tune_params->retry_sleep = val;
+ return count;
+}
+
+static const struct file_operations mpc_retry_sleep_fops = {
+ .owner = THIS_MODULE,
+ .open = simple_open,
+ .read = mpc_retry_sleep_read,
+ .write = mpc_retry_sleep_write,
+};
+
+static void bng_re_add_mpctune_knobs(struct bng_re_dev *rdev)
+{
+ struct dentry *mpctune_dir;
+
+ if (!rdev || !rdev->dbg_root)
+ return;
+
+ /* Create mpctune directory under debug root directory */
+ mpctune_dir = debugfs_create_dir("mpctune", rdev->dbg_root);
+ if (IS_ERR_OR_NULL(mpctune_dir)) {
+ dev_dbg(rdev_to_dev(rdev), "Unable to create mpctune debugfs dir\n");
+ return;
+ }
+
+ /* Create read-write files for each parameter - pass rdev as private_data */
+ debugfs_create_file("short_timeout", 0600, mpctune_dir,
+ rdev, &mpc_short_timeout_fops);
+ debugfs_create_file("long_timeout", 0600, mpctune_dir,
+ rdev, &mpc_long_timeout_fops);
+ debugfs_create_file("max_retries", 0600, mpctune_dir,
+ rdev, &mpc_max_retries_fops);
+ debugfs_create_file("retry_sleep", 0600, mpctune_dir,
+ rdev, &mpc_retry_sleep_fops);
+}
+
+static int bng_re_mpc_irq_info_show(struct seq_file *s, void *unused)
+{
+ struct bng_re_dev *rdev = s->private;
+ struct bng_mpc_ctx *mpc = rdev->mpc;
+
+ if (!mpc)
+ return -ENODEV;
+
+ seq_puts(s, "MPC IRQ info:\n");
+ seq_printf(s, "\t MPC IRQ Requested : %s\n\t irq_name : %s\n\t msix_vec: 0x%x\n\t NQ Ring ID: 0x%x\n",
+ mpc->requested ? "YES" : "NO",
+ mpc->irq_name,
+ mpc->msix_vec,
+ mpc->ring_id);
+
+ seq_puts(s, "MPC IRQ Stats:\n");
+ seq_printf(s, "\t Num IRQ Started: 0x%x\n\t Num IRQ Stopped : 0x%x\n\t Num IRQ Received : 0x%llx\n",
+ mpc->stats.num_irq_started,
+ mpc->stats.num_irq_stopped,
+ mpc->stats.num_irq_received);
+ seq_printf(s, "\t Num NQ Rearmed : 0x%llx\n\t Num tasklet rescheduled : 0x%llx\n",
+ mpc->stats.num_nq_rearm,
+ mpc->stats.num_tasklet_resched);
+ seq_printf(s, "\t Num CQ pending/has_more_work: 0x%llx\n",
+ mpc->stats.num_cq_pending);
+
+ seq_puts(s, "MPC NQ Info:\n");
+ seq_printf(s, "\t Last Consumer ID: 0x%x\n\t Max elements: 0x%x\n",
+ mpc->hwq.cons,
+ mpc->hwq.max_elements);
+ seq_puts(s, "\n");
+ return 0;
+}
+
+static int bng_re_mpc_irq_info_open(struct inode *inode, struct file *file)
+{
+ struct bng_re_dev *rdev = inode->i_private;
+
+ return single_open(file, bng_re_mpc_irq_info_show, rdev);
+}
+
+static const struct file_operations bng_re_mpc_irq_info_ops = {
+ .owner = THIS_MODULE,
+ .open = bng_re_mpc_irq_info_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+
+/* MPC performance and statistics display (must precede bng_re_debugfs_add_pdev) */
+static int bng_re_mpc_stats_show(struct seq_file *s, void *unused)
+{
+ struct bng_re_dev *rdev = s->private;
+
+ if (!rdev)
+ return -ENODEV;
+
+ seq_puts(s, "=== MPC RoCE Performance Statistics ===\n");
+ bng_re_mpc_roce_perf_debugfs_show(rdev, s, 1);
+
+ seq_puts(s, "\n=== MPC RoCE Diagnostic Counters ===\n");
+ bng_re_mpc_roce_diag_counters_debugfs_show(rdev, s);
+
+ return 0;
+}
+
+static int bng_re_mpc_stats_open(struct inode *inode, struct file *file)
+{
+ struct bng_re_dev *rdev = inode->i_private;
+
+ return single_open(file, bng_re_mpc_stats_show, rdev);
+}
+
+static const struct file_operations bng_re_mpc_stats_ops = {
+ .owner = THIS_MODULE,
+ .open = bng_re_mpc_stats_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+
+static ssize_t mpc_stats_clear_write(struct file *file, const char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct bng_re_dev *rdev = file->private_data;
+
+ if (!rdev)
+ return -ENODEV;
+
+ bng_re_mpc_roce_perf_debugfs_clear(rdev);
+ bng_re_mpc_roce_diag_counters_clear(rdev);
+
+ return count;
+}
+
+static const struct file_operations mpc_stats_clear_fops = {
+ .owner = THIS_MODULE,
+ .open = simple_open,
+ .write = mpc_stats_clear_write,
+};
+
void bng_re_debugfs_add_pdev(struct bng_re_dev *rdev)
{
struct pci_dev *pdev = rdev->aux_dev->pdev;
rdev->dbg_root =
debugfs_create_dir(dev_name(&pdev->dev), bng_re_debugfs_root);
+
+ if (IS_ERR_OR_NULL(rdev->dbg_root)) {
+ dev_dbg(rdev_to_dev(rdev), "Unable to create debugfs %s",
+ dev_name(&pdev->dev));
+ return;
+ }
+
+ /* Add MPC IRQ info debugfs entry */
+ debugfs_create_file("mpc_irq", 0400, rdev->dbg_root,
+ rdev, &bng_re_mpc_irq_info_ops);
+
+ /* Add MPC statistics display */
+ debugfs_create_file("mpc_stats", 0400, rdev->dbg_root,
+ rdev, &bng_re_mpc_stats_ops);
+
+ /* Add MPC statistics clear functionality */
+ debugfs_create_file("mpc_stats_clear", 0200, rdev->dbg_root,
+ rdev, &mpc_stats_clear_fops);
+
+ /* Add MPC tuning knobs for supported chips */
+ if (rdev->mpc_roce_creq.mpc_tune_params)
+ bng_re_add_mpctune_knobs(rdev);
}
void bng_re_debugfs_rem_pdev(struct bng_re_dev *rdev)
diff --git a/drivers/infiniband/hw/bng_re/bng_dev.c b/drivers/infiniband/hw/bng_re/bng_dev.c
index 311c8bc93160..ea6528e80ce3 100644
--- a/drivers/infiniband/hw/bng_re/bng_dev.c
+++ b/drivers/infiniband/hw/bng_re/bng_dev.c
@@ -15,6 +15,8 @@
#include "bng_re.h"
#include "bnge_hwrm.h"
#include "bng_debugfs.h"
+#include "bng_re_mpc_roce.h"
+#include "bng_xid.h"
MODULE_AUTHOR("Siva Reddy Kallam <siva.kallam@broadcom.com>");
MODULE_DESCRIPTION(BNG_RE_DESC);
@@ -41,7 +43,6 @@ static struct bng_re_dev *bng_re_dev_add(struct auxiliary_device *adev,
return rdev;
}
-
static int bng_re_register_netdev(struct bng_re_dev *rdev)
{
struct bnge_auxr_dev *aux_dev;
@@ -54,6 +55,8 @@ static void bng_re_destroy_chip_ctx(struct bng_re_dev *rdev)
{
struct bng_re_chip_ctx *chip_ctx;
+ bng_res_unmap_db_bar(&rdev->bng_res);
+
kfree(rdev->dev_attr);
rdev->dev_attr = NULL;
@@ -65,6 +68,18 @@ static void bng_re_destroy_chip_ctx(struct bng_re_dev *rdev)
kfree(chip_ctx);
}
+static void bng_re_set_db_offset(struct bng_re_dev *rdev)
+{
+ struct bnge_auxr_dev *aux_dev;
+ struct bng_re_res *res;
+
+ res = &rdev->bng_res;
+ aux_dev = rdev->aux_dev;
+
+ res->dpi_tbl.ucreg.offset = aux_dev->l2_db_offset;
+ res->dpi_tbl.wcreg.offset = aux_dev->l2_db_size;
+}
+
static int bng_re_setup_chip_ctx(struct bng_re_dev *rdev)
{
struct bng_re_chip_ctx *chip_ctx;
@@ -87,23 +102,31 @@ static int bng_re_setup_chip_ctx(struct bng_re_dev *rdev)
goto free_chip_ctx;
rdev->bng_res.dattr = rdev->dev_attr;
+ bng_re_set_db_offset(rdev);
+ rc = bng_res_map_db_bar(&rdev->bng_res);
+ if (rc)
+ goto free_attr;
+
return 0;
+free_attr:
+ kfree(rdev->dev_attr);
+ rdev->dev_attr = NULL;
free_chip_ctx:
kfree(rdev->chip_ctx);
rdev->chip_ctx = NULL;
return rc;
}
-static void bng_re_init_hwrm_hdr(struct input *hdr, u16 opcd)
+void bng_re_init_hwrm_hdr(struct input *hdr, u16 opcd)
{
hdr->req_type = cpu_to_le16(opcd);
hdr->cmpl_ring = cpu_to_le16(-1);
hdr->target_id = cpu_to_le16(-1);
}
-static void bng_re_fill_fw_msg(struct bnge_fw_msg *fw_msg, void *msg,
- int msg_len, void *resp, int resp_max_len,
- int timeout)
+void bng_re_fill_fw_msg(struct bnge_fw_msg *fw_msg, void *msg,
+ int msg_len, void *resp, int resp_max_len,
+ int timeout)
{
fw_msg->msg = msg;
fw_msg->msg_len = msg_len;
@@ -112,8 +135,8 @@ static void bng_re_fill_fw_msg(struct bnge_fw_msg *fw_msg, void *msg,
fw_msg->timeout = timeout;
}
-static int bng_re_net_ring_free(struct bng_re_dev *rdev,
- u32 fw_ring_id, int type)
+int bng_re_net_ring_free(struct bng_re_dev *rdev,
+ u32 fw_ring_id, int type)
{
struct bnge_auxr_dev *aux_dev = rdev->aux_dev;
struct hwrm_ring_free_input req = {};
@@ -125,17 +148,17 @@ static int bng_re_net_ring_free(struct bng_re_dev *rdev,
req.ring_type = type;
req.ring_id = cpu_to_le32(fw_ring_id);
bng_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
- sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT);
+ sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT);
rc = bnge_send_msg(aux_dev, &fw_msg);
if (rc)
- ibdev_err(&rdev->ibdev, "Failed to free HW ring:%d :%#x",
- req.ring_id, rc);
+ ibdev_err(&rdev->ibdev, "Failed to free HW ring:%u :%#x",
+ fw_ring_id, rc);
return rc;
}
-static int bng_re_net_ring_alloc(struct bng_re_dev *rdev,
- struct bng_re_ring_attr *ring_attr,
- u16 *fw_ring_id)
+int bng_re_net_ring_alloc(struct bng_re_dev *rdev,
+ struct bng_re_ring_attr *ring_attr,
+ u16 *fw_ring_id)
{
struct bnge_auxr_dev *aux_dev = rdev->aux_dev;
struct hwrm_ring_alloc_input req = {};
@@ -224,8 +247,8 @@ static int bng_re_query_hwrm_version(struct bng_re_dev *rdev)
ver_get_req.hwrm_intf_min = HWRM_VERSION_MINOR;
ver_get_req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
bng_re_fill_fw_msg(&fw_msg, (void *)&ver_get_req, sizeof(ver_get_req),
- (void *)&ver_get_resp, sizeof(ver_get_resp),
- BNGE_DFLT_HWRM_CMD_TIMEOUT);
+ (void *)&ver_get_resp, sizeof(ver_get_resp),
+ BNGE_DFLT_HWRM_CMD_TIMEOUT);
rc = bnge_send_msg(aux_dev, &fw_msg);
if (rc) {
ibdev_err(&rdev->ibdev, "Failed to query HW version, rc = 0x%x",
@@ -251,8 +274,15 @@ static int bng_re_query_hwrm_version(struct bng_re_dev *rdev)
static void bng_re_dev_uninit(struct bng_re_dev *rdev)
{
int rc;
+
+ bng_deinit_mpc(rdev);
+ bng_re_free_xid_tables(&rdev->bng_res);
+
bng_re_debugfs_rem_pdev(rdev);
+ if (test_and_clear_bit(BNG_RE_FLAG_TBLS_ALLOC_INITED, &rdev->flags))
+ bng_res_free_tbls(&rdev->bng_res);
+
if (test_and_clear_bit(BNG_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) {
rc = bng_re_deinit_rcfw(&rdev->rcfw);
if (rc)
@@ -262,7 +292,7 @@ static void bng_re_dev_uninit(struct bng_re_dev *rdev)
bng_re_free_stats_ctx_mem(rdev->bng_res.pdev, &rdev->stats_ctx);
bng_re_disable_rcfw_channel(&rdev->rcfw);
bng_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id,
- RING_ALLOC_REQ_RING_TYPE_NQ);
+ RING_ALLOC_REQ_RING_TYPE_NQ);
bng_re_free_rcfw_channel(&rdev->rcfw);
}
@@ -286,7 +316,7 @@ static int bng_re_dev_init(struct bng_re_dev *rdev)
rc = bng_re_register_netdev(rdev);
if (rc) {
ibdev_err(&rdev->ibdev,
- "Failed to register with netedev: %#x\n", rc);
+ "Failed to register with netedev: %#x\n", rc);
goto reg_netdev_fail;
}
@@ -347,7 +377,7 @@ static int bng_re_dev_init(struct bng_re_dev *rdev)
vid = rdev->nqr->msix_entries[BNG_RE_CREQ_NQ_IDX].vector;
rc = bng_re_enable_fw_channel(&rdev->rcfw,
- vid, db_offt);
+ vid, db_offt);
if (rc) {
ibdev_err(&rdev->ibdev, "Failed to enable RCFW channel: %#x\n",
rc);
@@ -358,6 +388,13 @@ static int bng_re_dev_init(struct bng_re_dev *rdev)
if (rc)
goto disable_rcfw;
+ rc = bng_re_init_xid_tables(rdev);
+ if (rc) {
+ ibdev_err(&rdev->ibdev,
+ "Failed to init XID / IQM tables: %#x\n", rc);
+ goto disable_rcfw;
+ }
+
bng_re_debugfs_add_pdev(rdev);
rc = bng_re_alloc_stats_ctx_mem(rdev->bng_res.pdev, rdev->chip_ctx,
&rdev->stats_ctx);
@@ -370,7 +407,7 @@ static int bng_re_dev_init(struct bng_re_dev *rdev)
rc = bng_re_stats_ctx_alloc(rdev);
if (rc) {
ibdev_err(&rdev->ibdev,
- "Failed to allocate QPLIB context: %#x\n", rc);
+ "Failed to allocate QP context: %#x\n", rc);
goto free_stats_ctx;
}
@@ -382,12 +419,32 @@ static int bng_re_dev_init(struct bng_re_dev *rdev)
}
set_bit(BNG_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags);
+ rc = bng_res_alloc_init_tbls(&rdev->bng_res);
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Failed to allocate tbls: %#x\n", rc);
+ goto free_sctx;
+ }
+ set_bit(BNG_RE_FLAG_TBLS_ALLOC_INITED, &rdev->flags);
+
+ rtnl_lock();
+ rc = bng_alloc_init_mpc(rdev);
+ rtnl_unlock();
+ if (rc) {
+ ibdev_err(&rdev->ibdev,
+ "MPC alloc-init failed rc = %#x\n", rc);
+ goto deinit_mpc;
+ }
return 0;
+deinit_mpc:
+ bng_deinit_mpc(rdev);
+ if (test_and_clear_bit(BNG_RE_FLAG_TBLS_ALLOC_INITED, &rdev->flags))
+ bng_res_free_tbls(&rdev->bng_res);
free_sctx:
bng_re_stats_ctx_free(rdev);
free_stats_ctx:
bng_re_free_stats_ctx_mem(rdev->bng_res.pdev, &rdev->stats_ctx);
disable_rcfw:
+ bng_re_free_xid_tables(&rdev->bng_res);
bng_re_disable_rcfw_channel(&rdev->rcfw);
free_ring:
bng_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id, type);
@@ -434,7 +491,6 @@ static int bng_re_add_device(struct auxiliary_device *adev)
return rc;
}
-
static void bng_re_remove_device(struct bng_re_dev *rdev,
struct auxiliary_device *aux_dev)
{
@@ -442,7 +498,6 @@ static void bng_re_remove_device(struct bng_re_dev *rdev,
ib_dealloc_device(&rdev->ibdev);
}
-
static int bng_re_probe(struct auxiliary_device *adev,
const struct auxiliary_device_id *id)
{
@@ -495,7 +550,6 @@ static int __init bng_re_mod_init(void)
{
int rc;
-
bng_re_register_debugfs();
rc = auxiliary_driver_register(&bng_re_driver);
diff --git a/drivers/infiniband/hw/bng_re/bng_fw.c b/drivers/infiniband/hw/bng_re/bng_fw.c
index ab6a2d2e95b5..a69221368ba8 100644
--- a/drivers/infiniband/hw/bng_re/bng_fw.c
+++ b/drivers/infiniband/hw/bng_re/bng_fw.c
@@ -723,6 +723,7 @@ int bng_re_deinit_rcfw(struct bng_re_rcfw *rcfw)
clear_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags);
return 0;
}
+
static inline bool _is_hw_retx_supported(u16 dev_cap_flags)
{
return dev_cap_flags &
diff --git a/drivers/infiniband/hw/bng_re/bng_fw.h b/drivers/infiniband/hw/bng_re/bng_fw.h
index c89c926ec2fc..f89e4aeb2469 100644
--- a/drivers/infiniband/hw/bng_re/bng_fw.h
+++ b/drivers/infiniband/hw/bng_re/bng_fw.h
@@ -5,6 +5,8 @@
#define __BNG_FW_H__
#include "bng_tlv.h"
+#include "bnge.h"
+#include "bnge_auxr.h"
/* FW DB related */
#define BNG_FW_CMDQ_TRIG_VAL 1
@@ -43,7 +45,6 @@ struct bng_re_crsbe {
u8 data[1024];
};
-
static inline u32 bng_fw_cmdqe_npages(u32 depth)
{
u32 npages;
@@ -58,6 +59,7 @@ static inline u32 bng_fw_cmdqe_page_size(u32 depth)
{
return (bng_fw_cmdqe_npages(depth) * PAGE_SIZE);
}
+
struct bng_re_cmdq_mbox {
struct bng_re_reg_desc reg;
void __iomem *prod;
@@ -208,4 +210,8 @@ int bng_re_rcfw_send_message(struct bng_re_rcfw *rcfw,
int bng_re_init_rcfw(struct bng_re_rcfw *rcfw,
struct bng_re_stats *stats_ctx);
int bng_re_deinit_rcfw(struct bng_re_rcfw *rcfw);
+
+void bng_re_init_hwrm_hdr(struct input *hdr, u16 opcd);
+void bng_re_fill_fw_msg(struct bnge_fw_msg *fw_msg, void *msg, int msg_len,
+ void *resp, int resp_max_len, int timeout);
#endif
diff --git a/drivers/infiniband/hw/bng_re/bng_mpc.c b/drivers/infiniband/hw/bng_re/bng_mpc.c
new file mode 100644
index 000000000000..7bcad0dea632
--- /dev/null
+++ b/drivers/infiniband/hw/bng_re/bng_mpc.c
@@ -0,0 +1,574 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2025 Broadcom.
+
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+#include <linux/dma-mapping.h>
+#include <linux/sched.h>
+#include <linux/pci.h>
+#include <linux/delay.h>
+#include <linux/kthread.h>
+#include <linux/module.h>
+
+#include "bng_roce_hsi.h"
+#include "bng_tlv.h"
+#include "bng_res.h"
+#include "bng_sp.h"
+#include "bng_re.h"
+#include "bng_re_mpc_roce.h"
+#include "bng_fw.h"
+
+struct rdma_user_mmap_entry;
+
+#ifndef DBC_GROUP_SQ
+#define DBC_GROUP_SQ 0
+#define DBC_GROUP_CQ 3
+#endif
+
+static irqreturn_t bng_mpc_irq(int irq, void *dev_instance)
+{
+ struct bng_mpc_ctx *mpc = dev_instance;
+
+ tasklet_schedule(&mpc->mpc_tasklet);
+ dev_dbg(rdev_to_dev(mpc->rdev), "%s: MPC IRQ Received (0x%llx)\n",
+ __func__, mpc->stats.num_irq_received);
+ mpc->stats.num_irq_received++;
+
+ return IRQ_HANDLED;
+}
+
+static void bng_service_mpc(struct tasklet_struct *t)
+{
+ struct bng_mpc_ctx *mpc = from_tasklet(mpc, t, mpc_tasklet);
+
+ struct bng_re_res *res = &mpc->rdev->bng_res;
+ struct bng_re_hwq *nq_hwq = &mpc->hwq;
+ struct bng_re_dev *rdev = mpc->rdev;
+ int budget = 16; /*mpc->budget;*/
+ u32 has_more_nq = 1;
+ struct nq_base *nqe;
+ u32 type;
+
+ /*Process pending MPC completions before servicing NQ*/
+ if (mpc->cq_pending) {
+ mpc->cq_pending = bng_re_mpc_roce_service_cne(rdev, NULL,
+ BNG_MPC_MAX_MESSAGES_PER_POLL);
+ if (mpc->cq_pending) {
+ mpc->stats.num_cq_pending++;
+ mpc->stats.num_tasklet_resched++;
+ tasklet_schedule(&mpc->mpc_tasklet);
+ dev_dbg(rdev_to_dev(mpc->rdev),
+ "%s:%d MPC IRQ Rescheduled due to CQ pending\n",
+ __func__, __LINE__);
+ return;
+ }
+ }
+
+ spin_lock_bh(&nq_hwq->lock);
+ do {
+ nqe = bng_re_get_qe(nq_hwq, nq_hwq->cons, NULL);
+ if (!NQE_CMP_VALID(nqe, mpc->nq_db.dbinfo.flags)) {
+ has_more_nq = 0;
+ break;
+ }
+ /* The valid test of the entry must be done first before
+ * reading any further.
+ */
+ dma_rmb();
+ type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
+ switch (type) {
+ case NQ_CN_TYPE_CQ_NOTIFICATION:
+ mpc->cq_pending =
+ bng_re_mpc_roce_service_cne(rdev, nqe,
+ BNG_MPC_MAX_MESSAGES_PER_POLL);
+ if (mpc->cq_pending)
+ mpc->stats.num_cq_pending++;
+
+ break;
+ default:
+ dev_err(rdev_to_dev(mpc->rdev), "%s: Unsupported NQ type(0x%x)\n",
+ __func__, type);
+ }
+ bng_re_hwq_incr_cons(nq_hwq->max_elements, &nq_hwq->cons,
+ 1, &mpc->nq_db.dbinfo.flags);
+ dev_dbg(rdev_to_dev(mpc->rdev),
+ "%s: MPC IRQ NQ Processed Cons(0x%x) flag(0x%x) type(0x%x) max(0x%x)\n",
+ __func__, nq_hwq->cons, mpc->nq_db.dbinfo.flags,
+ type, nq_hwq->max_elements);
+ } while ((budget--) && (!mpc->cq_pending));
+
+ if (mpc->cq_pending) {
+ /* Update the consumer index only and dont enable arm */
+ bng_re_ring_nq_db(&mpc->nq_db.dbinfo, res->cctx, false);
+ mpc->stats.num_tasklet_resched++;
+ tasklet_schedule(&mpc->mpc_tasklet);
+ dev_dbg(rdev_to_dev(mpc->rdev),
+ "%s:%d MPC IRQ NQ Rescheduled due to CQ pending\n",
+ __func__, __LINE__);
+ } else {
+ if (!has_more_nq) {
+ mpc->stats.num_nq_rearm++;
+ bng_re_ring_nq_db(&mpc->nq_db.dbinfo, res->cctx, true);
+ dev_dbg(rdev_to_dev(mpc->rdev),
+ "%s: MPC IRQ NQ Armed flag(0x%x)\n",
+ __func__, mpc->nq_db.dbinfo.flags);
+ } else if (mpc->requested) {
+ /* Update the consumer index only and dont enable arm */
+ bng_re_ring_nq_db(&mpc->nq_db.dbinfo, res->cctx, false);
+ mpc->stats.num_tasklet_resched++;
+ tasklet_schedule(&mpc->mpc_tasklet);
+ dev_dbg(rdev_to_dev(mpc->rdev),
+ "%s: MPC IRQ NQ Rescheduled due to NQ pending\n",
+ __func__);
+ }
+ }
+ spin_unlock_bh(&nq_hwq->lock);
+}
+
+static int bng_mpc_start_irq(struct bng_mpc_ctx *mpc, int msix_vector,
+ bool need_init)
+{
+ struct bng_re_res *res;
+ int rc;
+
+ res = &mpc->rdev->bng_res;
+
+ if (mpc->requested) {
+ dev_info(rdev_to_dev(mpc->rdev), "%s: MPC IRQ - Already Started\n", __func__);
+ return 0;
+ }
+ mpc->msix_vec = msix_vector;
+ if (need_init)
+ tasklet_setup(&mpc->mpc_tasklet,
+ bng_service_mpc);
+ else
+ tasklet_enable(&mpc->mpc_tasklet);
+
+ mpc->irq_name = kasprintf(GFP_KERNEL, "bng_re-mpc@pci:%s",
+ pci_name(res->pdev));
+ if (!mpc->irq_name) {
+ dev_err(rdev_to_dev(mpc->rdev), "%s: Failed to alloc irq_name\n", __func__);
+ return -ENOMEM;
+ }
+ rc = request_irq(mpc->msix_vec, bng_mpc_irq, 0,
+ mpc->irq_name, mpc);
+ if (rc) {
+ kfree(mpc->irq_name);
+ mpc->irq_name = NULL;
+ tasklet_disable(&mpc->mpc_tasklet);
+ dev_err(rdev_to_dev(mpc->rdev), "%s: request_irq failed rc(0x%x)\n", __func__, rc);
+ return rc;
+ }
+ mpc->requested = true;
+ bng_re_ring_nq_db(&mpc->nq_db.dbinfo, res->cctx, true);
+
+ mpc->stats.num_irq_started++;
+
+ return 0;
+}
+
+void bng_mpc_stop_irq(struct bng_mpc_ctx *mpc, bool kill)
+{
+ struct bng_re_res *res;
+
+ dev_dbg(rdev_to_dev(mpc->rdev), "%s: Start(%d)\n",
+ __func__, mpc->requested);
+
+ if (!mpc->requested)
+ return;
+
+ mpc->requested = false;
+ res = &mpc->rdev->bng_res;
+ /* Mask h/w interrupt */
+ bng_re_ring_nq_db(&mpc->nq_db.dbinfo, res->cctx, false);
+ /* Sync with last running IRQ handler */
+ synchronize_irq(mpc->msix_vec);
+ irq_set_affinity_hint(mpc->msix_vec, NULL);
+ free_irq(mpc->msix_vec, mpc);
+ kfree(mpc->irq_name);
+ mpc->irq_name = NULL;
+
+ /* Cleanup Tasklet */
+ if (kill)
+ tasklet_kill(&mpc->mpc_tasklet);
+ else
+ tasklet_disable(&mpc->mpc_tasklet);
+}
+
+static void bng_map_mpc_nq_db(struct bng_mpc_ctx *mpc, u32 reg_offt)
+{
+ struct bng_re_reg_desc *dbreg;
+ struct bng_re_nq_db *nq_db;
+ struct bng_re_res *res;
+
+ nq_db = &mpc->nq_db;
+ res = &mpc->rdev->bng_res;
+ dbreg = &res->dpi_tbl.ucreg;
+
+ nq_db->reg.bar_id = dbreg->bar_id;
+ nq_db->reg.bar_base = dbreg->bar_base;
+ nq_db->reg.bar_reg = dbreg->bar_reg + reg_offt;
+ nq_db->reg.len = sizeof(u64);
+
+ nq_db->dbinfo.db = nq_db->reg.bar_reg;
+ nq_db->dbinfo.hwq = &mpc->hwq;
+ nq_db->dbinfo.xid = mpc->ring_id;
+ nq_db->dbinfo.seed = mpc->ring_id;
+ nq_db->dbinfo.flags = 0;
+ spin_lock_init(&nq_db->dbinfo.lock);
+ nq_db->dbinfo.res = &mpc->rdev->bng_res;
+}
+
+static int bng_alloc_mpc_nq_mem(struct bng_mpc_ctx *mpc)
+{
+ struct bng_re_hwq_attr hwq_attr = {};
+ struct bng_re_sg_info sginfo = {};
+
+ if (!mpc->hwq.max_elements ||
+ mpc->hwq.max_elements > BNG_NQE_MAX_CNT)
+ mpc->hwq.max_elements = BNG_NQE_MAX_CNT;
+
+ sginfo.pgsize = PAGE_SIZE;
+ sginfo.pgshft = PAGE_SHIFT;
+ hwq_attr.res = &mpc->rdev->bng_res;
+ hwq_attr.sginfo = &sginfo;
+ hwq_attr.depth = mpc->hwq.max_elements;
+ hwq_attr.stride = sizeof(struct nq_base);
+ hwq_attr.type = BNG_HWQ_TYPE_QUEUE;
+ if (bng_re_alloc_init_hwq(&mpc->hwq, &hwq_attr)) {
+ dev_err(&mpc->rdev->bng_res.pdev->dev, "QP: MPC NQ allocation failed");
+ return -ENOMEM;
+ }
+ return 0;
+}
+
+static void bng_free_mpc_nq_mem(struct bng_mpc_ctx *mpc)
+{
+ if (mpc->hwq.max_elements) {
+ bng_re_free_hwq(&mpc->rdev->bng_res, &mpc->hwq);
+ mpc->hwq.max_elements = 0;
+ }
+}
+
+static int bng_setup_mpc_nq(struct bng_mpc_ctx *mpc)
+{
+ struct bng_re_dev *rdev = mpc->rdev;
+ struct bng_re_ring_attr rattr = {};
+ int depth;
+ u32 offt;
+ u16 vec;
+ int rc;
+
+ mutex_init(&rdev->nqr->load_lock);
+
+ depth = BNG_NQE_MAX_CNT;
+ vec = rdev->nqr->msix_entries[BNG_RE_MPC_IRQ_IDX].vector;
+ offt = rdev->nqr->msix_entries[BNG_RE_MPC_IRQ_IDX].db_offset;
+ mpc->hwq.max_elements = depth;
+ rc = bng_alloc_mpc_nq_mem(mpc);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to get mem for MPC NQ %d, rc = 0x%x",
+ BNG_RE_MPC_IRQ_IDX, rc);
+ return rc;
+ }
+
+ rattr.dma_arr = mpc->hwq.pbl[BNG_PBL_LVL_0].pg_map_arr;
+ rattr.pages = mpc->hwq.pbl[mpc->hwq.level].pg_count;
+ rattr.type = RING_ALLOC_REQ_RING_TYPE_NQ;
+ rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;
+ rattr.depth = mpc->hwq.max_elements - 1;
+ rattr.lrid = rdev->nqr->msix_entries[BNG_RE_MPC_IRQ_IDX].ring_idx;
+
+ rc = bng_re_net_ring_alloc(rdev, &rattr, &mpc->ring_id);
+ if (rc) {
+ mpc->ring_id = 0xffff; /* Invalid ring-id */
+ dev_err(rdev_to_dev(rdev),
+ "%s:pages(0x%x) type(0x%x) mode(0x%x)depth(0x%x) lrid(0x%x) ring_id(0x%x)",
+ __func__, rattr.pages, rattr.type, rattr.mode,
+ rattr.depth, rattr.lrid, mpc->ring_id);
+ dev_err(rdev_to_dev(rdev),
+ "Failed to get fw id for MPC NQ %d, rc = 0x%x",
+ BNG_RE_MPC_IRQ_IDX, rc);
+ goto fail_ring;
+ }
+ dev_dbg(rdev_to_dev(rdev),
+ "%s: pages(0x%x) type(0x%x) mode(0x%x) depth(0x%x) lrid(0x%x) ring_id(0x%x)",
+ __func__, rattr.pages, rattr.type, rattr.mode,
+ rattr.depth, rattr.lrid, mpc->ring_id);
+
+ return 0;
+
+fail_ring:
+ bng_free_mpc_nq_mem(mpc);
+ return rc;
+}
+
+static int bng_enable_mpc_nq(struct bng_mpc_ctx *mpc, int msix_vec, int bar_reg_offset)
+{
+ int rc;
+
+ rc = bng_setup_mpc_nq(mpc);
+ if (rc)
+ return rc;
+
+ bng_map_mpc_nq_db(mpc, bar_reg_offset);
+
+ rc = bng_mpc_start_irq(mpc, msix_vec, true);
+ if (rc)
+ return rc;
+
+ return 0;
+}
+
+static int bng_disable_mpc_nq(struct bng_mpc_ctx *mpc)
+{
+ /* Make sure the HW is stopped! */
+ bng_mpc_stop_irq(mpc, true);
+
+ mpc->nq_db.reg.bar_reg = NULL;
+ mpc->nq_db.dbinfo.db = NULL;
+
+ mpc->msix_vec = 0;
+
+ return 0;
+}
+
+static int bng_alloc_mpc_ctx(struct bng_re_dev *rdev)
+{
+ rdev->mpc = kzalloc_obj(*rdev->mpc, GFP_KERNEL);
+ if (!rdev->mpc)
+ return -ENOMEM;
+
+ rdev->mpc->rdev = rdev;
+ return 0;
+}
+
+static void bng_free_mpc_ctx(struct bng_re_dev *rdev)
+{
+ kfree(rdev->mpc);
+ rdev->mpc = NULL;
+}
+
+static int bng_alloc_init_mpc_irq(struct bng_re_dev *rdev)
+{
+ int reg_offset = 0;
+ int msix_vec = 0;
+ int rc = 0;
+
+ rc = bng_alloc_mpc_ctx(rdev);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "%s: MPC Ctx Alloc Failed rc (0x%x)\n", __func__, rc);
+ return rc;
+ }
+
+ msix_vec = rdev->nqr->msix_entries[BNG_RE_MPC_IRQ_IDX].vector;
+ reg_offset = rdev->nqr->msix_entries[BNG_RE_MPC_IRQ_IDX].db_offset;
+
+ rc = bng_enable_mpc_nq(rdev->mpc, msix_vec, reg_offset);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "%s: Enable MPC NQ Failed rc (0x%x)\n", __func__, rc);
+ return rc;
+ }
+
+ return 0;
+}
+
+static void bng_mask_mpc_irq(struct bng_re_dev *rdev)
+{
+ if (!rdev->mpc)
+ return;
+
+ bng_disable_mpc_nq(rdev->mpc);
+}
+
+static void bng_deinit_mpc_irq(struct bng_re_dev *rdev)
+{
+ if (!rdev->mpc)
+ return;
+
+ bng_free_mpc_nq_mem(rdev->mpc);
+ bng_re_net_ring_free(rdev, rdev->mpc->ring_id, RING_ALLOC_REQ_RING_TYPE_NQ);
+ bng_free_mpc_ctx(rdev);
+}
+
+/**
+ * bng_mpc_background_thread_fn - Continuously poll the MPC CQ.
+ * @data: struct bng_re_dev * (RoCE device)
+ * Returns: 0
+ */
+static int bng_mpc_background_thread_fn(void *data)
+{
+ struct bng_re_dev *rdev = (struct bng_re_dev *)data;
+
+ while (!kthread_should_stop()) {
+ usleep_range(BNG_MPC_MIN_USLEEP_POLL, BNG_MPC_MAX_USLEEP_POLL);
+ bng_re_mpc_roce_service_creq(rdev, BNG_MPC_MAX_MESSAGES_PER_POLL);
+ }
+
+ dev_dbg(rdev_to_dev(rdev), "%s: terminating\n", __func__);
+ return 0;
+}
+
+/**
+ * bng_mpc_start_background_poll - Start polling MPC CQ in the background.
+ * @rdev: RoCE device
+ * @new_thread: out param to cache the created thread
+ * Returns: 0 for success
+ */
+int bng_mpc_start_background_poll(struct bng_re_dev *rdev,
+ struct task_struct **new_thread)
+{
+ *new_thread = kthread_create(bng_mpc_background_thread_fn,
+ rdev, "mpc poll background thread");
+ if (IS_ERR(*new_thread)) {
+ dev_err(rdev_to_dev(rdev), "%s: kthread_create failed\n", __func__);
+ return -EPERM;
+ }
+ sched_set_fifo(*new_thread);
+ wake_up_process(*new_thread);
+ return 0;
+}
+
+/**
+ * bng_mpc_stop_background_poll - Stop the MPC background poll thread.
+ * @rdev: RoCE device
+ * @thread: thread to stop (set to NULL on return)
+ */
+void bng_mpc_stop_background_poll(struct bng_re_dev *rdev,
+ struct task_struct **thread)
+{
+ int rc = 0;
+
+ if (*thread) {
+ rc = kthread_stop(*thread);
+ *thread = NULL;
+ }
+
+ if (rc)
+ dev_err(rdev_to_dev(rdev), "%s: kthread_stop returned %d\n",
+ __func__, rc);
+}
+
+static int bng_alloc_init_mpc_creq(struct bng_re_dev *rdev)
+{
+ int rc = 0;
+
+ /* init mpc creq channel */
+ rdev->mpc_roce_creq.bm_lr_index = rdev->nqr->msix_entries[BNG_RE_MPC_IRQ_IDX].ring_idx;
+ rdev->mpc_roce_creq.bm_nq_ring_id = rdev->mpc->ring_id;
+ rdev->mpc_roce_creq.bm_stat_index = rdev->stats_ctx.fw_id;
+
+ dev_info(rdev_to_dev(rdev),
+ "[%s:%p:%d] bm_lr_index:%d, nq_ring_id:%d, stat_index:%d\n",
+ __func__, current, __LINE__,
+ rdev->mpc_roce_creq.bm_lr_index,
+ rdev->mpc_roce_creq.bm_nq_ring_id,
+ rdev->mpc_roce_creq.bm_stat_index);
+
+ rc = bng_re_mpc_roce_alloc_init(rdev);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "[%s:%p:%d] Error alloc init mpc roce creq (err : %d)\n",
+ __func__, current, __LINE__, rc);
+ return rc;
+ }
+ set_bit(BNG_RE_FLAG_MPC_FW_CHANNEL_EN, &rdev->flags);
+
+ return rc;
+}
+
+static int bng_deinit_mpc_creq(struct bng_re_dev *rdev)
+{
+ rdev->mpc_roce_creq.bm_mpc_uninstall_pending = 1;
+ bng_re_check_mpc_pending_empty(rdev);
+ if (test_and_clear_bit(BNG_RE_FLAG_MPC_FW_CHANNEL_EN, &rdev->flags))
+ /*bnge_deinit_and_free_mpc_roce_creq(rdev->en_dev, true);*/
+ bng_re_mpc_roce_deinit_free(rdev);
+ else
+ dev_dbg(rdev_to_dev(rdev),
+ "Did not free mpc roce creq rings");
+
+ return 0;
+}
+
+int bng_alloc_init_mpc(struct bng_re_dev *rdev)
+{
+ struct bng_re_mpc_roce_creq_info *creq = &rdev->mpc_roce_creq;
+ int rc = 0;
+
+ if (creq->mpc_cache && creq->mpc_tune_params && creq->bm_diag_counters)
+ return 0;
+
+ rc = bng_alloc_init_mpc_irq(rdev);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "%s: Init MPC IRQ Failed (0x%x)\n", __func__, rc);
+ return rc;
+ }
+ rc = bng_alloc_init_mpc_creq(rdev);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "%s: Alloc Init MPC CREQ Failed(0x%x)\n", __func__, rc);
+ return rc;
+ }
+ /* to make sure it isn't a ghost setting */
+ rdev->mpc_roce_creq.bm_mpc_uninstall_pending = 0;
+ return rc;
+}
+
+int bng_deinit_mpc(struct bng_re_dev *rdev)
+{
+ int rc = 0;
+
+ /* Mask the MPC IRQ and kill the tasklet before draining/freeing
+ * the CREQ hwqs and cache, so an in-flight completion can't run
+ * against memory that is about to be freed.
+ */
+ bng_mask_mpc_irq(rdev);
+
+ rc = bng_deinit_mpc_creq(rdev);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "%s: DeInit MPC CREQ Failed (0x%x)\n", __func__, rc);
+ return rc;
+ }
+
+ bng_deinit_mpc_irq(rdev);
+ return rc;
+}
+
+int bng_re_alloc_mpc_doorbells(struct bng_re_dev *rdev)
+{
+ /* the hw db recovery functionality is pending */
+ return 0;
+}
+
+void bng_re_unalloc_mpc_doorbells(struct bng_re_dev *rdev)
+{
+ struct bng_re_db_info *tx_db = &rdev->mpc_roce_creq.tx_db_info;
+ struct bng_re_db_info *cq_db = &rdev->mpc_roce_creq.cq_db_info;
+
+ tx_db->dbc = NULL;
+ cq_db->dbc = NULL;
+}
+
+void bng_re_check_mpc_pending_empty(struct bng_re_dev *rdev)
+{
+ struct bng_re_mpc_roce_creq_info *creq = &rdev->mpc_roce_creq;
+ int iters = 0;
+ const int max_iters = (CHECK_MPC_PENDING_MAX_SECS * 1000) / 20;
+
+ while (atomic_read(&creq->pending)) {
+ if (iters >= max_iters) {
+ dev_err(rdev_to_dev(rdev),
+ "%s - MPC pending did not drain within %d seconds\n",
+ __func__,
+ CHECK_MPC_PENDING_MAX_SECS);
+ return;
+ }
+ msleep(20);
+ iters++;
+ }
+ dev_dbg(rdev_to_dev(rdev),
+ "%s - MPC pending completed in %d iterations\n",
+ __func__,
+ iters);
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_re.h b/drivers/infiniband/hw/bng_re/bng_re.h
index dae4862621a7..d6fb205d9f12 100644
--- a/drivers/infiniband/hw/bng_re/bng_re.h
+++ b/drivers/infiniband/hw/bng_re/bng_re.h
@@ -4,7 +4,13 @@
#ifndef __BNG_RE_H__
#define __BNG_RE_H__
+#include "bnge.h"
+#include "bnge_auxr.h"
#include "bng_res.h"
+#include "bng_fw.h"
+#include <rdma/ib_verbs.h>
+
+#define BNG_RE_XID_AVOID_REUSE false
#define BNG_RE_ADEV_NAME "bng_en"
@@ -18,12 +24,165 @@
#define BNG_RE_CREQ_NQ_IDX 0
#define BNGE_INVALID_STATS_CTX_ID -1
+
+/* poll interval while polling for mpc roce creq response */
+#define BNG_MPC_MIN_USLEEP_POLL (5)
+#define BNG_MPC_MAX_USLEEP_POLL (15)
+#define BNG_MPC_MAX_MESSAGES_PER_POLL (32)
+
+/* limit how long we wait for mpc to drain during uninit */
+#define CHECK_MPC_PENDING_MAX_SECS (40)
+
+enum {
+ BNG_RE_AEQ_IDX = 0,
+ BNG_RE_MPC_IRQ_IDX,
+ BNG_RE_MAX_RSVD_IRQ,
+};
+
+/* MPC VF buffer specific information */
+#define BNG_MPC_MAX_REQ_SIZE 512
+#define BNG_MPC_MAX_NUM_REQ 1024
+#define BNG_MPC_MAX_VF_CMD_FWD_PAGES 1
+#define BNG_MPC_VF_BUF_PAGE_SLOTS 4
+
+/* RoCE MPC diagnostics and knobs (mirrors bnge for compatibility) */
+#define BNG_RE_MPC_MAX_LATENCY_SEC_SLAB_INDEX 201
+#define BNG_RE_MPC_MAX_LATENCY_MSEC_SLAB_INDEX 3000
+#define BNG_RE_MPC_MAX_STAT_INDEX 0x3FFFF
+
+#define ROCE_MPC_MAX_STAT_INDEX BNG_RE_MPC_MAX_STAT_INDEX
+#define ROCE_MPC_MAX_LATENCY_SEC_SLAB_INDEX BNG_RE_MPC_MAX_LATENCY_SEC_SLAB_INDEX
+#define ROCE_MPC_MAX_LATENCY_MSEC_SLAB_INDEX BNG_RE_MPC_MAX_LATENCY_MSEC_SLAB_INDEX
+
+struct bng_re_mpc_poll_info {
+ u64 bm_last_poll_work_poll_jiffies_start;
+ u64 bm_last_poll_work_poll_jiffies_end;
+ u64 bm_max_poll_work_poll_jiffies;
+ u32 bm_current_cons_indx;
+ u64 bm_start_time_on_current_cons_indx;
+ u64 bm_curr_time_on_current_cons_indx;
+};
+
+struct bng_re_mpc_diag_counters {
+ atomic_t bm_mpc_verb_try;
+ atomic_t bm_mpc_sent_try;
+ atomic_t bm_mpc_sent_started_ok;
+ atomic_t bm_mpc_sent_started_ebusy;
+ atomic_t bm_mpc_sent_started_misc_err;
+ atomic_t bm_mpc_sent_response_ok;
+ atomic_t bm_mpc_sent_response_err;
+ atomic_t bm_mpc_fatal_ebusy;
+ atomic_t bm_mpc_sent_timeout;
+ atomic_t bm_mpc_misc_err;
+ atomic_t bm_mpc_consecutive_sent_err;
+ atomic_t bm_mpc_stalled_err;
+};
+
+struct bng_re_mpc_tune_params {
+ u32 short_timeout;
+ u32 long_timeout;
+ u16 max_retries;
+ u16 retry_sleep;
+};
+
+struct bng_re_mpc_roce_stats_info {
+ bool bms_stats_enabled;
+ u32 bms_qp_modify_stats_id;
+ u32 bms_ah_modify_stats_id;
+ u32 bms_other_stats_id;
+ u64 *bms_qp_modify_stats;
+ u64 *bms_ah_modify_stats;
+ u64 *bms_other_stats;
+ u32 *bms_lat_slab_sec;
+ u32 *bms_lat_slab_msec;
+};
+
+struct bng_re_mpc_roce_creq_info {
+ struct task_struct *bm_poll_thread;
+ u32 bm_lr_index;
+ u16 bm_nq_ring_id;
+ u32 bm_stat_index;
+ struct bng_re_db_info tx_db_info;
+ struct bng_re_db_info cq_db_info;
+ struct rdma_user_mmap_entry *tx_hdbr_mmap;
+ struct rdma_user_mmap_entry *cq_hdbr_mmap;
+
+ /* Transmit and completion rings (roce-style HWQ) */
+ struct bng_re_hwq tx_hwq;
+ struct bng_re_hwq cq_hwq;
+ u32 tx_ring_id;
+ u16 cq_ring_id;
+
+ /* SW ring for tx: maps prod index to (handle, inline_bds, last_cons) */
+ struct {
+ unsigned long handle;
+ u8 inline_bds;
+ unsigned long last_cons; /* BNG_RE_MPC_INV_HDL when completion received */
+ } *tx_sw_ring;
+ u32 tx_prod;
+ u32 tx_cons;
+ u16 tx_napi_idx; /* for opaque */
+ u32 tx_ring_mask; /* tx_hwq.max_elements - 1 */
+ u32 cq_ring_mask; /* cq_hwq.max_elements - 1 */
+ u32 cq_cp_bit; /* for CMP valid toggle */
+
+ /* lock access to the transmit ring */
+ spinlock_t tx_lock;
+ /* lock access to the mpc context */
+ spinlock_t mpc_ctx_lock;
+ /* lock access to receive buffer processing */
+ spinlock_t bm_lock;
+
+ /* Diagnostics and poll info (from bnge) */
+ struct bng_re_mpc_poll_info *bm_poll_info;
+ struct bng_re_mpc_diag_counters *bm_diag_counters;
+ struct bng_re_mpc_tune_params *mpc_tune_params;
+ struct bng_re_mpc_roce_stats_info *bm_stats_info;
+
+ /* Pending/context tracking (from bnge mpc_roce_info) */
+ atomic_t pending;
+ atomic_t max_pending;
+ u32 avail_buffer;
+ u32 min_avail_buffer;
+ struct kmem_cache *mpc_cache;
+ bool bm_mpc_stall;
+ u8 bm_mpc_uninstall_pending;
+};
+
+struct bng_mpc_stats {
+ u32 num_irq_started;
+ u32 num_irq_stopped;
+ u64 num_tasklet_resched;
+ u64 num_nq_rearm;
+ u64 num_irq_received;
+ u64 num_cq_pending;
+};
+
+struct bng_mpc_db {
+ struct bng_re_reg_desc reg;
+ void __iomem *db;
+ struct bng_re_db_info dbinfo;
+};
+
/* NQ specific structures */
struct bng_re_nq_db {
struct bng_re_reg_desc reg;
struct bng_re_db_info dbinfo;
};
+struct bng_mpc_ctx {
+ struct bng_re_dev *rdev;
+ struct bng_re_hwq hwq;
+ struct bng_re_nq_db nq_db;
+ struct bng_mpc_stats stats;
+ struct tasklet_struct mpc_tasklet;
+ char *irq_name;
+ int msix_vec;
+ u16 ring_id;
+ bool requested; /*irq handler installed */
+ bool cq_pending;
+};
+
struct bng_re_nq {
struct pci_dev *pdev;
struct bng_re_res *res;
@@ -68,6 +227,13 @@ struct bng_re_dev {
unsigned long flags;
#define BNG_RE_FLAG_NETDEV_REGISTERED 0
#define BNG_RE_FLAG_RCFW_CHANNEL_EN 1
+#define BNG_RE_FLAG_TBLS_ALLOC_INITED 2
+#define BNG_RE_FLAG_MPC_FW_CHANNEL_EN 31
+#define BNG_RE_FLAG_MPC_FW_THREAD_EN 32
+#define BNG_RE_FLAG_MPC_DB_PAGE_EN 34
+#define BNG_RE_FLAG_ERR_DEVICE_DETACHED 36
+
+#define BNG_RE_STEERING_TO_HOST 0
struct net_device *netdev;
struct auxiliary_device *adev;
struct bnge_auxr_dev *aux_dev;
@@ -80,6 +246,14 @@ struct bng_re_dev {
struct bng_re_dev_attr *dev_attr;
struct dentry *dbg_root;
struct bng_re_stats stats_ctx;
+ struct bng_re_mpc_roce_creq_info mpc_roce_creq;
+ struct bng_mpc_ctx *mpc;
};
+int bng_re_net_ring_alloc(struct bng_re_dev *rdev,
+ struct bng_re_ring_attr *ring_attr,
+ u16 *fw_ring_id);
+int bng_re_net_ring_free(struct bng_re_dev *rdev,
+ u32 fw_ring_id, int type);
+
#endif
diff --git a/drivers/infiniband/hw/bng_re/bng_re_mpc_roce.c b/drivers/infiniband/hw/bng_re/bng_re_mpc_roce.c
new file mode 100644
index 000000000000..3ecf5f34180a
--- /dev/null
+++ b/drivers/infiniband/hw/bng_re/bng_re_mpc_roce.c
@@ -0,0 +1,1863 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2025 Broadcom.
+
+#include <linux/stddef.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/workqueue.h>
+#include <linux/delay.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/uaccess.h>
+#include <linux/fs.h>
+#include <linux/string.h>
+
+#include "bng_roce_hsi.h"
+#include "bng_re.h"
+#include "bng_res.h"
+#include "bng_fw.h"
+#include "bnge.h"
+#include "bnge_hwrm.h"
+#include "bng_re_mpc_roce.h"
+#include "bnge_auxr.h"
+
+/* Handle unsolicited MPC completion (e.g. RCA QP XID cleanup complete). */
+
+/**
+ * bng_re_mpc_handle_event_cmpl - core handler for unsolicited MPC event completions
+ * @rdev: RoCE device
+ * @cmpl: pointer to the 32-byte completion payload (mpc_event_resp_cmpl)
+ * @cmpl_len: byte length of the payload
+ *
+ * Called directly from bng_re_mpc_roce_service_creq when handle == 0 and the
+ * completion is an EVENT type, bypassing the old ULP callback mechanism.
+ */
+void bng_re_mpc_handle_event_cmpl(struct bng_re_dev *rdev,
+ void *cmpl, u32 cmpl_len)
+{
+ struct mpc_event_resp_cmpl *hdr = (struct mpc_event_resp_cmpl *)cmpl;
+ int event_data;
+ int event;
+ int xid;
+
+ if (!rdev || !cmpl) {
+ dev_warn(rdev_to_dev(rdev),
+ "%s: Invalid driver state or completion pointer: %lu,%lu\n",
+ __func__, (unsigned long)rdev, (unsigned long)cmpl);
+ return;
+ }
+ if (cmpl_len != sizeof(struct mpc_event_resp_cmpl)) {
+ dev_warn(rdev_to_dev(rdev),
+ "unsolicited mpc cmpl too short: %u < %zu\n",
+ cmpl_len, sizeof(struct mpc_event_resp_cmpl));
+ return;
+ }
+ xid = (int)le32_to_cpu(hdr->xid);
+ event = (int)le32_to_cpu(hdr->event_type);
+ event_data = (int)le32_to_cpu(hdr->event_data);
+ /* sanity check the fields look sane */
+ if (!(event == MPC_EVENT_RESP_CMPL_EVENT_TYPE_EVENT_TYPE_QP_EVENT &&
+ event_data == MPC_EVENT_RESP_CMPL_EVENT_DATA_EVENT_TYPE_FASTQP_MD_COMPLETE))
+ dev_warn(rdev_to_dev(rdev),
+ "unexpected event mpc cmpl: xid=%d event=%d data=%d\n",
+ xid, event, event_data);
+ /* process even if event / event_data isn't as expected, as we don't use them rn */
+ /* bng_re_qp_xid_pending_process_unsolicited_cmpl(rdev, xid); */
+}
+
+/**
+ * bng_re_mpc_process_vf_completion - Handle PF/VF MPC completion on the PF path
+ * @rdev: RoCE device
+ * @cmpl: Completion buffer (starts with &struct mpc_cmpl_hdr)
+ * @cmpl_len: Length of @cmpl in bytes
+ *
+ * Dispatches VF-related QP/AH create, modify, and destroy completions without
+ * registering with the bnge ULP. No-op if the device is detached or not registered.
+ */
+
+/**
+ * bng_re_roce_mpc_cmp - Deliver one solicited MPC completion to a command context
+ * @rdev: RoCE device
+ * @handle: Pointer to struct bng_re_roce_cmd_ctx cast to unsigned long
+ * @cmpl: Completion entry (payload and length)
+ *
+ * Copies the completion into the context, issues a write memory barrier, then
+ * either sets cmpl_available for polling mode or completes the wait queue.
+ */
+void bng_re_roce_mpc_cmp(struct bng_re_dev *rdev, unsigned long handle,
+ struct bng_re_cmpl_entry *cmpl)
+{
+ struct bng_re_roce_cmd_ctx *ctx;
+ struct mpc_cmpl_hdr *cmp;
+ u32 len;
+
+ dev_dbg(rdev_to_dev(rdev), "%s: handele:%lx\n", __func__, handle);
+
+ if (cmpl->len != MPC_ROCE_COMPL_MAX_SIZE) {
+ dev_warn(rdev_to_dev(rdev), "%s: Unexpected cmpl_len:%d\n",
+ __func__, cmpl->len);
+ return;
+ }
+ len = cmpl->len;
+ cmp = cmpl->cmpl;
+ ctx = (void *)handle;
+ if (!ctx) {
+ dev_warn(rdev_to_dev(rdev), "%s: ctx null\n",
+ __func__);
+ return;
+ }
+ memcpy(&ctx->roce_cmp, cmp, len);
+ /* make sure memory is updated */
+ smp_wmb();
+ if (ctx->poll_cmp)
+ WRITE_ONCE(ctx->cmpl_available, 1);
+ else
+ complete(&ctx->cmp);
+}
+
+/**
+ * bng_re_mpc_tx_avail - Free TX ring slots for new MPC posts
+ * @creq: RoCE MPC channel state
+ *
+ * Return: Number of unused BD slots in the MPC TX ring (masked ring arithmetic).
+ */
+static u32 bng_re_mpc_tx_avail(struct bng_re_mpc_roce_creq_info *creq)
+{
+ u32 used = READ_ONCE(creq->tx_prod) - READ_ONCE(creq->tx_cons);
+
+ return (creq->tx_ring_mask + 1) - (used & creq->tx_ring_mask);
+}
+
+/**
+ * bng_re_mpc_adv_tx_cons - Advance MPC TX consumer past completed multi-BD posts
+ * @creq: RoCE MPC channel state
+ *
+ * Walks the software TX ring from tx_cons while each slot's last_cons marks
+ * the post as fully completed, so out-of-order completions can be absorbed
+ * safely. Bounded by MAX_ADV_ITERATIONS to avoid infinite loops on corruption.
+ */
+static void bng_re_mpc_adv_tx_cons(struct bng_re_mpc_roce_creq_info *creq)
+{
+ u32 tx_cons, slot, tx_prod, diff;
+ unsigned int iter = 0;
+
+ if (unlikely(!creq || !creq->tx_sw_ring)) {
+ WARN_ONCE(1, "%s: null creq or tx_sw_ring\n", __func__);
+ return;
+ }
+
+ tx_cons = creq->tx_cons;
+ tx_prod = READ_ONCE(creq->tx_prod);
+ /* Use unsigned subtraction (modular u32) so the check remains correct
+ * after tx_prod wraps past 0xFFFFFFFF. A raw tx_cons > tx_prod
+ * comparison is unsafe at rollover: e.g. tx_prod=2, tx_cons=0xFFFFFFFE
+ * is a valid 4-entry-in-flight state but would incorrectly satisfy
+ * tx_cons > tx_prod.
+ */
+ if (unlikely((tx_prod - tx_cons) > (creq->tx_ring_mask + 1))) {
+ WARN_ONCE(1,
+ "%s: tx_cons (%u) > tx_prod (%u): consumer ahead of producer (double-consume, bad chain, or prod reset?)\n",
+ __func__, tx_cons, tx_prod);
+ return;
+ }
+
+ diff = tx_prod - tx_cons;
+ if (diff <= creq->tx_ring_mask && (diff & creq->tx_ring_mask) == 0)
+ return;
+
+ slot = tx_cons & creq->tx_ring_mask;
+ do {
+ tx_prod = READ_ONCE(creq->tx_prod);
+ diff = tx_prod - tx_cons;
+ if (diff <= creq->tx_ring_mask && (diff & creq->tx_ring_mask) == 0)
+ break;
+ {
+ u32 bds = creq->tx_sw_ring[slot].inline_bds;
+ u32 next = tx_cons + bds;
+
+ /* Use unsigned distance (modular u32) so the bound check
+ * stays correct after either counter wraps past 0xFFFFFFFF.
+ * A raw next > tx_prod comparison gives wrong results when
+ * next has wrapped and tx_prod has not, or vice versa.
+ */
+ if (unlikely((tx_prod - next) > (creq->tx_ring_mask + 1))) {
+ WARN_ONCE(1,
+ "%s: would advance tx_cons %u -> %u past tx_prod %u (bds=%u cons_slot=%u)\n",
+ __func__, tx_cons, next, tx_prod, bds, slot);
+ return;
+ }
+ tx_cons = next;
+ }
+ creq->tx_cons = tx_cons;
+ iter++;
+ /*
+ * Re-read producer after advancing: we may have just caught up
+ * (tx_cons == tx_prod) while the physical slot still shows INV for
+ * the next logical post — must exit before the while re-enters.
+ */
+ tx_prod = READ_ONCE(creq->tx_prod);
+ diff = tx_prod - tx_cons;
+ if (diff <= creq->tx_ring_mask && (diff & creq->tx_ring_mask) == 0)
+ break;
+ if (iter >= MAX_ADV_ITERATIONS)
+ break;
+ slot = tx_cons & creq->tx_ring_mask;
+ } while (creq->tx_sw_ring[slot].last_cons == BNG_RE_MPC_INV_HDL);
+
+ if (iter >= MAX_ADV_ITERATIONS) {
+ tx_prod = READ_ONCE(creq->tx_prod);
+ diff = tx_prod - tx_cons;
+ if (diff > creq->tx_ring_mask || (diff & creq->tx_ring_mask) != 0) {
+ u32 prod_slot = tx_prod & creq->tx_ring_mask;
+
+ slot = tx_cons & creq->tx_ring_mask;
+ pr_err("bng_re: %s: exceeded max iterations (%u) tx_cons=%u tx_prod=%u d=%u cons_slot=%u prod_slot=%u last_cons=0x%lx\n",
+ __func__, iter, tx_cons, tx_prod, diff, slot, prod_slot,
+ creq->tx_sw_ring[slot].last_cons);
+ }
+ }
+}
+
+/**
+ * bng_re_mpc_roce_alloc_hwq - Allocate MPC TX and CQ hardware queues
+ * @rdev: RoCE device
+ *
+ * Ensures CQ page layout fits whole LONG completions, then allocates TX and CQ
+ * HWQs via the qpxxx. Sets ring masks and CQ copy-bit metadata.
+ *
+ * Return: 0 on success, %-ENOMEM on allocation failure (partial cleanup on CQ fail).
+ */
+static int bng_re_mpc_roce_alloc_hwq(struct bng_re_dev *rdev)
+{
+ struct bng_re_mpc_roce_creq_info *creq = &rdev->mpc_roce_creq;
+ struct bng_re_res *res = &rdev->bng_res;
+ struct bng_re_hwq_attr hwq_attr = {};
+ struct bng_re_sg_info sginfo = {};
+
+ dev_dbg(rdev_to_dev(rdev), "%s: %p\n", __func__, rdev);
+
+ /*
+ * Each CQ page must hold a whole number of 32-byte LONG completions
+ * (sizeof(mpc_qp_modify_cmpl) == 32 == 2 x BNG_RE_MPC_CQ_STRIDE).
+ * This guarantees that a LONG completion never straddles a page
+ * boundary, making non-contiguous scatter-gather handling unnecessary.
+ */
+ BUILD_BUG_ON(PAGE_SIZE % sizeof(struct mpc_qp_modify_cmpl));
+
+ sginfo.pgsize = PAGE_SIZE;
+ sginfo.pgshft = PAGE_SHIFT;
+ hwq_attr.res = res;
+ hwq_attr.sginfo = &sginfo;
+ hwq_attr.type = BNG_HWQ_TYPE_QUEUE;
+
+ /* TX ring */
+ hwq_attr.depth = BNG_RE_MPC_TX_RING_SIZE;
+ hwq_attr.stride = BNG_RE_MPC_TX_STRIDE;
+ if (bng_re_alloc_init_hwq(&creq->tx_hwq, &hwq_attr)) {
+ dev_err(rdev_to_dev(rdev), "%s: TX HWQ alloc failed\n", __func__);
+ return -ENOMEM;
+ }
+ creq->tx_ring_mask = creq->tx_hwq.max_elements - 1;
+
+ memset(&sginfo, 0, sizeof(sginfo));
+ sginfo.pgsize = PAGE_SIZE;
+ sginfo.pgshft = PAGE_SHIFT;
+ /* CQ ring */
+ hwq_attr.depth = BNG_RE_MPC_CQ_RING_SIZE;
+ hwq_attr.stride = BNG_RE_MPC_CQ_STRIDE;
+ hwq_attr.type = BNG_HWQ_TYPE_QUEUE;
+ if (bng_re_alloc_init_hwq(&creq->cq_hwq, &hwq_attr)) {
+ dev_err(rdev_to_dev(rdev), "%s: CQ HWQ alloc failed\n", __func__);
+ bng_re_free_hwq(res, &creq->tx_hwq);
+ return -ENOMEM;
+ }
+
+ creq->cq_ring_mask = creq->cq_hwq.max_elements - 1;
+ creq->cq_cp_bit = creq->cq_hwq.max_elements;
+
+ return 0;
+}
+
+/**
+ * bng_re_mpc_roce_alloc_init - Initialize RoCE MPC channel software state and HWQs
+ * @rdev: RoCE device
+ *
+ * Allocates poll/diag/tune structures, stats, TX/CQ HWQs, software TX ring,
+ * command context kmem cache, firmware rings, and MPC doorbells. Initializes
+ * spinlocks and DB info for L2 doorbell posting.
+ *
+ * Return: 0 on success, negative errno on failure (all partial allocations rolled back).
+ */
+int bng_re_mpc_roce_alloc_init(struct bng_re_dev *rdev)
+{
+ struct bng_re_mpc_roce_creq_info *creq = &rdev->mpc_roce_creq;
+ struct bnge_auxr_dev *auxr_dev = rdev->aux_dev;
+ struct bng_re_res *res = &rdev->bng_res;
+ int rc;
+
+ res = &rdev->bng_res;
+
+ dev_dbg(rdev_to_dev(rdev), "%s: %p\n", __func__, rdev);
+
+ creq->tx_ring_id = INVALID_HW_RING_ID_32BIT;
+ creq->cq_ring_id = INVALID_HW_RING_ID;
+
+ spin_lock_init(&creq->tx_lock);
+ spin_lock_init(&creq->mpc_ctx_lock);
+ spin_lock_init(&creq->bm_lock);
+
+ creq->bm_poll_info = kzalloc_obj(*creq->bm_poll_info, GFP_KERNEL);
+ if (!creq->bm_poll_info)
+ return -ENOMEM;
+
+ creq->bm_poll_info->bm_current_cons_indx = -1;
+
+ creq->bm_diag_counters = kzalloc_obj(*creq->bm_diag_counters, GFP_KERNEL);
+ if (!creq->bm_diag_counters) {
+ rc = -ENOMEM;
+ goto err_free_poll;
+ }
+
+ creq->mpc_tune_params = kzalloc_obj(*creq->mpc_tune_params, GFP_KERNEL);
+ if (!creq->mpc_tune_params) {
+ rc = -ENOMEM;
+ goto err_free_diag;
+ }
+ creq->mpc_tune_params->short_timeout = BNG_RE_MPC_ROCE_TMO_MSECS_SHORT;
+ creq->mpc_tune_params->long_timeout = BNG_RE_MPC_ROCE_TMO_MSECS_LONG;
+ creq->mpc_tune_params->max_retries = BNG_RE_MPC_ROCE_MAX_RETRIES;
+ creq->mpc_tune_params->retry_sleep = BNG_RE_MPC_ROCE_RETRY_SLEEP;
+
+ rc = bng_re_mpc_roce_alloc_stats(rdev);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "%s: stats_info alloc failed\n", __func__);
+ rc = -ENOMEM;
+ goto err_free_tune;
+ }
+
+ rc = bng_re_mpc_roce_alloc_hwq(rdev);
+ if (rc)
+ goto err_free_stats;
+
+ creq->tx_sw_ring = kcalloc(creq->tx_ring_mask + 1,
+ sizeof(*creq->tx_sw_ring), GFP_KERNEL);
+ if (!creq->tx_sw_ring) {
+ rc = -ENOMEM;
+ goto err_free_hwq;
+ }
+
+ creq->mpc_cache = kmem_cache_create("bng_re_roce_mpc",
+ sizeof(struct bng_re_roce_cmd_ctx), 0,
+ SLAB_HWCACHE_ALIGN, NULL);
+ if (!creq->mpc_cache) {
+ dev_err(rdev_to_dev(rdev), "%s: mpc_cache create failed\n", __func__);
+ rc = -ENOMEM;
+ goto err_free_sw_ring;
+ }
+
+ atomic_set(&creq->pending, 0);
+ atomic_set(&creq->max_pending, 0);
+ creq->avail_buffer = ~0U;
+ creq->min_avail_buffer = ~0U;
+ creq->bm_mpc_stall = false;
+
+ rc = bng_re_mpc_rings_alloc(rdev);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "%s: mpc_rings_alloc failed rc=%d\n",
+ __func__, rc);
+ goto err_free_cache;
+ }
+
+ if (!auxr_dev || !auxr_dev->bar1) {
+ dev_err(rdev_to_dev(rdev), "%s: mpc_rings_alloc failed: bad auxr_dev info: auxr_dev=%p, bar1=%p\n",
+ __func__, auxr_dev, auxr_dev ? auxr_dev->bar1 : 0);
+ goto err_free_cache;
+ }
+
+ creq->tx_db_info.db = auxr_dev->bar1 + auxr_dev->l2_db_offset;
+
+ creq->tx_db_info.hwq = &creq->tx_hwq;
+ creq->tx_db_info.xid = creq->tx_ring_id;
+ creq->tx_db_info.seed = creq->tx_ring_id;
+ creq->tx_db_info.flags = 0;
+ spin_lock_init(&creq->tx_db_info.lock);
+ creq->tx_db_info.max_slot = 1;
+ creq->tx_db_info.res = res;
+ creq->tx_db_info.is_l2 = true;
+ creq->cq_db_info.toggle = 0;
+
+ creq->cq_db_info.db = auxr_dev->bar1 + auxr_dev->l2_db_offset;
+ creq->cq_db_info.hwq = &creq->cq_hwq;
+ creq->cq_db_info.xid = creq->cq_ring_id;
+ creq->cq_db_info.seed = creq->cq_ring_id;
+ creq->cq_db_info.flags = 0;
+ spin_lock_init(&creq->cq_db_info.lock);
+ creq->cq_db_info.max_slot = 1;
+ creq->cq_db_info.res = res;
+ /*
+ * The MPC CQ is an L2 completion ring (RING_ALLOC_REQ_RING_TYPE_L2_CMPL),
+ * so its arm doorbell must carry DBC_DBC64_PATH_L2. Without is_l2 the
+ * CQ_ARMALL below goes out on the RoCE path, the CQ is never armed, and
+ * firmware never raises an MPC NQ notification -> IRQ never fires and
+ * MPC commands time out (-110). Matches thor3.
+ */
+ creq->cq_db_info.is_l2 = true;
+
+ creq->tx_prod = 0;
+ creq->tx_cons = 0;
+ creq->tx_napi_idx = 0;
+
+ rc = bng_re_alloc_mpc_doorbells(rdev);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "[%s:%lx:%d] Error allocating mpc dbs (err : %d)\n",
+ __func__, (unsigned long)current, __LINE__, rc);
+ goto err_free_cache;
+ }
+
+ set_bit(BNG_RE_FLAG_MPC_DB_PAGE_EN, &rdev->flags);
+ bng_re_ring_db(&creq->cq_db_info, DBC_DBC_TYPE_CQ_ARMALL);
+
+ return 0;
+
+err_free_cache:
+ bng_re_mpc_rings_free(rdev);
+ kmem_cache_destroy(creq->mpc_cache);
+ creq->mpc_cache = NULL;
+ kfree(creq->bm_stats_info);
+ creq->bm_stats_info = NULL;
+err_free_sw_ring:
+ kfree(creq->tx_sw_ring);
+ creq->tx_sw_ring = NULL;
+err_free_hwq:
+ bng_re_free_hwq(res, &creq->cq_hwq);
+ bng_re_free_hwq(res, &creq->tx_hwq);
+err_free_stats:
+ bng_re_mpc_roce_free_stats(rdev);
+err_free_tune:
+ kfree(creq->mpc_tune_params);
+ creq->mpc_tune_params = NULL;
+err_free_diag:
+ kfree(creq->bm_diag_counters);
+ creq->bm_diag_counters = NULL;
+err_free_poll:
+ kfree(creq->bm_poll_info);
+ creq->bm_poll_info = NULL;
+
+ return rc;
+}
+
+/**
+ * bng_re_mpc_roce_deinit_free - Tear down RoCE MPC channel and free resources
+ * @rdev: RoCE device
+ *
+ * Frees firmware rings, HWQs, TX software ring, kmem cache, stats, and auxiliary
+ * allocations created by bng_re_mpc_roce_alloc_init().
+ */
+void bng_re_mpc_roce_deinit_free(struct bng_re_dev *rdev)
+{
+ struct bng_re_mpc_roce_creq_info *creq = &rdev->mpc_roce_creq;
+ struct bng_re_res *res = &rdev->bng_res;
+
+ dev_dbg(rdev_to_dev(rdev), "%s: %p\n", __func__, rdev);
+
+ bng_re_mpc_rings_free(rdev);
+ bng_re_free_hwq(res, &creq->cq_hwq);
+ bng_re_free_hwq(res, &creq->tx_hwq);
+ kfree(creq->tx_sw_ring);
+ creq->tx_sw_ring = NULL;
+
+ kmem_cache_destroy(creq->mpc_cache);
+ creq->mpc_cache = NULL;
+
+ bng_re_mpc_roce_free_stats(rdev);
+ kfree(creq->mpc_tune_params);
+ kfree(creq->bm_diag_counters);
+ kfree(creq->bm_poll_info);
+ creq->mpc_tune_params = NULL;
+ creq->bm_diag_counters = NULL;
+ creq->bm_poll_info = NULL;
+}
+
+/**
+ * bng_re_mpc_roce_send - Post one MPC command to the TX ring and ring the doorbell
+ * @rdev: RoCE device
+ * @cmd: MPC command payload
+ * @cmd_len: Length of @cmd in bytes
+ * @handle: Opaque handle stored in the TX software ring (e.g. command context pointer)
+ * @mctx: Optional out; set to the address of the stored handle slot for error cleanup
+ *
+ * Splits the command across header and data BDs, updates HWQ producer with
+ * bng_hwq_incr_prod(), writes a memory barrier, and rings the producer DB.
+ *
+ * Return: 0 on success, %-EBUSY if the ring lacks space, %-EIO on queue indexing failure.
+ */
+static int bng_re_mpc_roce_send(struct bng_re_dev *rdev, void *cmd, uint cmd_len,
+ unsigned long handle, void **mctx)
+{
+ struct bng_re_mpc_roce_creq_info *creq = &rdev->mpc_roce_creq;
+ struct bng_re_hwq *tx_hwq = &creq->tx_hwq;
+ u32 bds, total_bds, free_slots, prod;
+ unsigned long flags;
+ struct tx_bd *txbd;
+
+ dev_dbg(rdev_to_dev(rdev), "[%s:%lx:%d] enter\n",
+ __func__, (unsigned long)current, __LINE__);
+
+ bds = DIV_ROUND_UP(cmd_len, BNG_RE_MPC_TX_STRIDE);
+ total_bds = bds + 1; /* header BD + data BDs */
+
+ spin_lock_irqsave(&creq->tx_lock, flags);
+ free_slots = bng_re_mpc_tx_avail(creq);
+ if (free_slots < total_bds) {
+ spin_unlock_irqrestore(&creq->tx_lock, flags);
+ dev_dbg(rdev_to_dev(rdev), "%s: no space, need %u have %u\n",
+ __func__, total_bds, free_slots);
+ return -EBUSY;
+ }
+
+ prod = creq->tx_prod;
+
+ txbd = bng_re_get_qe(tx_hwq, prod & creq->tx_ring_mask, NULL);
+
+ dev_dbg(rdev_to_dev(rdev), "[1]%s, page:%d, index:%d, prod:%d,len:%d,bds:%d,total_bds:%d\n",
+ __func__,
+ 0,
+ 0,
+ prod,
+ cmd_len,
+ bds,
+ total_bds
+ );
+ if (!txbd) {
+ spin_unlock_irqrestore(&creq->tx_lock, flags);
+ dev_err(rdev_to_dev(rdev), "%s: get_qe failed at prod %u\n",
+ __func__, prod);
+ return -EIO;
+ }
+
+ creq->tx_sw_ring[prod & creq->tx_ring_mask].handle = handle;
+ creq->tx_sw_ring[prod & creq->tx_ring_mask].inline_bds = total_bds;
+ creq->tx_sw_ring[prod & creq->tx_ring_mask].last_cons = 0; /* pending */
+ if (mctx)
+ *(unsigned long **)mctx = &creq->tx_sw_ring[prod & creq->tx_ring_mask].handle;
+
+ txbd->tx_bd_len_flags_type =
+ cpu_to_le32((cmd_len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_MPC_TX_BD |
+ (total_bds << TX_BD_FLAGS_BD_CNT_SHIFT));
+ txbd->tx_bd_opaque = BNG_RE_MPC_SET_TX_OPAQUE(creq->tx_napi_idx, prod,
+ total_bds, creq->tx_ring_mask);
+ prod = ++creq->tx_prod;
+
+ /* Copy payload into following BD slots */
+ while (bds > 0) {
+ u32 chunk = min_t(u32, cmd_len, (uint)BNG_RE_MPC_TX_STRIDE);
+
+ txbd = bng_re_get_qe(tx_hwq, prod & creq->tx_ring_mask, NULL);
+ if (!txbd) {
+ spin_unlock_irqrestore(&creq->tx_lock, flags);
+ dev_err(rdev_to_dev(rdev), "%s: get_qe data failed\n", __func__);
+ return -EIO;
+ }
+ memcpy(txbd, cmd, chunk);
+ cmd += chunk;
+ cmd_len -= chunk;
+ prod = ++creq->tx_prod;
+ bds--;
+ }
+
+ /*
+ * Advance hwq->prod by the total number of BDs posted.
+ * bng_re_hwq_incr_prod keeps hwq->prod within [0, hwq->depth) and
+ * toggles the epoch bit in tx_db_info.flags on wrap-around. Both are
+ * required for bng_re_ring_prod_db to write a correct doorbell.
+ */
+ bng_hwq_incr_prod(&creq->tx_db_info, tx_hwq, total_bds);
+
+ dev_dbg(rdev_to_dev(rdev),
+ "[2]%s, page:%d, index:%d, prod:%d,len:%d,bds:%d,total_bds:%d\n",
+ __func__,
+ 0,
+ 0,
+ prod,
+ cmd_len,
+ bds,
+ total_bds
+ );
+ /* make sure memory is updated */
+ wmb();
+
+ bng_ring_prod_db(&creq->tx_db_info, DBC_DBC_TYPE_SQ);
+
+ dev_info(rdev_to_dev(rdev),
+ "mpc db: db=%p xid=0x%x prod=%u max_slot=%u flags=0x%x is_l2=%d total_bds=%d\n",
+ creq->tx_db_info.db, creq->tx_db_info.xid, tx_hwq->prod,
+ creq->tx_db_info.max_slot, creq->tx_db_info.flags,
+ creq->tx_db_info.is_l2, total_bds);
+
+ spin_unlock_irqrestore(&creq->tx_lock, flags);
+
+ return 0;
+}
+
+/**
+ * bng_re_mpc_dispatch_event_cmpl - Handle unsolicited MPC LONG completions (no TX ctx)
+ * @rdev: RoCE device
+ * @entry: Completion entry (32-byte LONG payload)
+ *
+ * For EVENT/RESP_CMPL, forwards to bng_re_mpc_handle_event_cmpl(). Other
+ * unsolicited types are logged. Mirrors the null-handle path in bnge MPC completion
+ * processing.
+ */
+static void bng_re_mpc_dispatch_event_cmpl(struct bng_re_dev *rdev,
+ struct bng_re_cmpl_entry *entry)
+{
+ struct mpc_cmpl_hdr *hdr = (struct mpc_cmpl_hdr *)entry->cmpl;
+
+ if (hdr->req_type == MPC_CMD_HDR_REQ_TYPE_EVENT &&
+ hdr->req_subtype == MPC_CMD_HDR_REQ_SUB_TYPE_EVENT_RESP_CMPL)
+ bng_re_mpc_handle_event_cmpl(rdev, entry->cmpl, entry->len);
+ else
+ dev_warn(rdev_to_dev(rdev),
+ "%s: null handle, unsolicited mpc cmpl type:%u subtype:%u\n",
+ __func__, hdr->req_type, hdr->req_subtype);
+}
+
+/**
+ * bng_re_mpc_roce_service_creq - Poll the MPC completion ring
+ * @rdev: RoCE device
+ * @budget: Maximum LONG completions to process this call
+ *
+ * Validates CQ epoch and LONG completion layout (two contiguous slots), handles
+ * PF/VF forward completions, unsolicited events, and solicited completions
+ * (opaque TX index, out-of-order-safe tx_cons advance, then bng_re_roce_mpc_cmp()).
+ *
+ * Return: 1 if the budget was fully consumed (more completions may remain), else 0.
+ */
+int bng_re_mpc_roce_service_creq(struct bng_re_dev *rdev, int budget)
+{
+ struct bng_re_mpc_roce_creq_info *creq = &rdev->mpc_roce_creq;
+ struct bng_re_hwq *cq_hwq = &creq->cq_hwq;
+ struct bng_re_cmpl_entry cmpl_entry;
+ struct mpc_cmpl_hdr *hdr = NULL;
+ static u64 last_poll_jiffies;
+ unsigned long handle = 0;
+ struct mpc_cmp *mpcmp2;
+ bool mpc_unsol = false;
+ struct mpc_cmp *mpcmp;
+ struct tx_cmp *txcmp;
+ unsigned long flags;
+ u32 iterations = 0;
+ u8 inline_bds = 0;
+ u32 cons_next;
+ u32 cmp_type;
+ u32 opaque;
+ u32 cons;
+ u16 idx;
+
+ last_poll_jiffies = get_jiffies_64();
+
+ spin_lock_irqsave(&creq->bm_lock, flags);
+
+ cons = cq_hwq->cons;
+
+ while (iterations < budget) {
+ /*
+ * Valid polarity alternates each CQ lap (HW toggles when posting
+ * at index 0 again). bng_re_hwq_incr_cons() XORs the same
+ * phase into cq_db_info.flags when cons wraps — not (cons & cp_bit)
+ * with cp_bit == max_elements and wrapped cons (always 0..max-1).
+ */
+ u32 cq_epoch = creq->cq_db_info.flags &
+ BNG_QP_FLAG_EPOCH_CONS_MASK;
+
+ dma_rmb();
+ txcmp = bng_re_get_qe(cq_hwq, cons, NULL);
+ if (!txcmp) {
+ dev_err(rdev_to_dev(rdev), "%s: get_qe failed cons %u\n",
+ __func__, cons);
+ break;
+ }
+ if (!(!!(txcmp->tx_cmp_errors_v & cpu_to_le32(TX_CMP_V)) ==
+ !cq_epoch))
+ break;
+
+ dma_rmb();
+ cmp_type = le32_to_cpu(((struct mpc_cmp *)txcmp)->mpc_cmp_client_subtype_type) &
+ MPC_CMP_TYPE;
+ if (cmp_type != MPC_CMP_TYPE_MID_PATH_LONG) {
+ dev_err(rdev_to_dev(rdev), "%s: unexpected cmp type 0x%x\n",
+ __func__, cmp_type);
+ bng_re_hwq_incr_cons(cq_hwq->max_elements, &cons,
+ 1, &creq->cq_db_info.flags);
+ iterations++;
+ continue;
+ }
+
+ mpcmp = (struct mpc_cmp *)txcmp;
+ /*
+ * Fetch and validate the second 16-byte slot of the
+ * LONG completion before reading any data from it.
+ * HW may write the two slots non-atomically; returning
+ * without advancing cons lets the caller retry.
+ */
+ cons_next = cons + 1;
+ if (cons_next >= cq_hwq->max_elements)
+ cons_next = 0;
+
+ mpcmp2 = bng_re_get_qe(cq_hwq, cons_next, NULL);
+ if (!mpcmp2) {
+ dev_err(rdev_to_dev(rdev),
+ "%s: get_qe slot2 failed at cons %u\n",
+ __func__, cons_next);
+ break;
+ }
+
+ /* Same HW write as slot 1 — same V polarity / consumer epoch */
+ if (!(!!(mpcmp2->mpc_cmp_v & cpu_to_le32(MPC_CMP_V)) ==
+ !cq_epoch)) {
+ dev_dbg(rdev_to_dev(rdev),
+ "%s: slot2 not valid yet at cons %u\n",
+ __func__, cons_next);
+ break;
+ }
+
+ /* Ensure slot2 valid bit is read before its payload */
+ dma_rmb();
+
+ /*
+ * Safety check: the two slots must be contiguous in
+ * virtual memory (same page). PAGE_SIZE is guaranteed
+ * to be a multiple of sizeof(mpc_qp_modify_cmpl) by the
+ * BUILD_BUG_ON in bng_re_mpc_roce_alloc_hwq, so this
+ * path should never be reached in practice.
+ */
+ if (unlikely((char *)mpcmp2 !=
+ (char *)mpcmp + BNG_RE_MPC_CQ_STRIDE)) {
+ dev_err(rdev_to_dev(rdev),
+ "%s: LONG cmpl sl non-contig [%lx,%lx] cons %u/%u - skip\n",
+ __func__,
+ (unsigned long)mpcmp,
+ (unsigned long)mpcmp2,
+ cons,
+ cons_next);
+ bng_re_hwq_incr_cons(cq_hwq->max_elements,
+ &cons, 2,
+ &creq->cq_db_info.flags);
+ cq_hwq->cons = cons;
+ iterations++;
+ continue;
+ }
+
+ /*
+ * IRQs are already disabled by the outer
+ * bm_lock irqsave; use a plain spin_lock here
+ * so we do not overwrite 'flags' and corrupt
+ * the IRQ state restored when bm_lock is
+ * released.
+ */
+ hdr = (struct mpc_cmpl_hdr *)mpcmp;
+ mpc_unsol =
+ (hdr->req_type == MPC_CMD_HDR_REQ_TYPE_EVENT &&
+ hdr->req_subtype ==
+ MPC_CMD_HDR_REQ_SUB_TYPE_EVENT_RESP_CMPL);
+
+ cmpl_entry.cmpl = mpcmp;
+ cmpl_entry.len = sizeof(*mpcmp) * 2;
+
+ if (mpc_unsol) {
+ /*
+ * Unsolicited EVENT from firmware: no TX ring
+ * slot was consumed, so tx_cons must not change.
+ */
+ bng_re_mpc_dispatch_event_cmpl(rdev, &cmpl_entry);
+ } else {
+ opaque = mpcmp->mpc_cmp_opaque;
+ idx = BNG_RE_MPC_TX_OPAQUE_IDX(opaque);
+ /*
+ * Solicited completion. Mark the slot done and
+ * advance tx_cons using the out-of-order-safe
+ * mechanism (mirrors bnge_adv_mpc_cons_re):
+ *
+ * - Only advance from the current consumer position.
+ * If this completion arrived out of order (i.e.
+ * the slot is not the oldest outstanding entry),
+ * mark last_cons and leave tx_cons unchanged; the
+ * consumer will be swept forward when the older
+ * entry's completion arrives.
+ * - inline_bds is NOT cleared here so the advance
+ * function can use it to step past the slot.
+ */
+ spin_lock(&creq->mpc_ctx_lock);
+ if (idx <= creq->tx_ring_mask) {
+ inline_bds = creq->tx_sw_ring[idx].inline_bds;
+ handle = creq->tx_sw_ring[idx].handle;
+ if (handle) {
+ /*
+ * Acquire a reference on the ctx while
+ * still under the lock so that
+ * bng_re_roce_mpc_xmit cannot free it
+ * between here and the cmp call below.
+ * Zero the slot so a concurrent or
+ * future xmit error-path clear sees an
+ * already-consumed handle.
+ */
+ refcount_inc(&((struct bng_re_roce_cmd_ctx *)
+ handle)->refcnt);
+ creq->tx_sw_ring[idx].handle = 0;
+ }
+ creq->tx_sw_ring[idx].last_cons =
+ BNG_RE_MPC_INV_HDL;
+ }
+ if ((creq->tx_cons & creq->tx_ring_mask) == idx)
+ bng_re_mpc_adv_tx_cons(creq);
+ spin_unlock(&creq->mpc_ctx_lock);
+
+ if (!handle) {
+ dev_warn(rdev_to_dev(rdev),
+ "%s: null ctx at idx %u, opaque: %x %s 0x%x.%x\n",
+ __func__, idx, opaque,
+ "skip processing MPC completion, req_type",
+ hdr->req_type,
+ hdr->req_subtype);
+ } else {
+ struct bng_re_roce_cmd_ctx *cmp_ctx =
+ (struct bng_re_roce_cmd_ctx *)handle;
+
+ bng_re_roce_mpc_cmp(rdev, handle, &cmpl_entry);
+ if (refcount_dec_and_test(&cmp_ctx->refcnt))
+ kmem_cache_free(creq->mpc_cache, cmp_ctx);
+ }
+ }
+
+ /* LONG completion always occupies two ring slots */
+ bng_re_hwq_incr_cons(cq_hwq->max_elements, &cons,
+ 2, &creq->cq_db_info.flags);
+ cq_hwq->cons = cons;
+ iterations++;
+ }
+
+ creq->cq_hwq.cons = cons;
+ spin_unlock_irqrestore(&creq->bm_lock, flags);
+
+ return (iterations >= budget) ? 1 : 0;
+}
+
+int bng_re_mpc_roce_service_cne(struct bng_re_dev *rdev, void *nq_cmp, int budget)
+{
+ struct bng_re_mpc_roce_creq_info *creq = &rdev->mpc_roce_creq;
+ u64 dbr_type = DBC_DBC_TYPE_CQ_ARMALL;
+ struct nq_cn *nqcne = nq_cmp;
+ int has_more_work = 0;
+ u16 type;
+
+ if (nqcne) {
+ type = le16_to_cpu(nqcne->type);
+ if ((type & NQ_CN_TYPE_MASK) == NQ_CN_TYPE_CQ_NOTIFICATION)
+ creq->cq_db_info.toggle = ((type) & NQ_CN_TOGGLE_MASK) >> NQ_CN_TOGGLE_SFT;
+ }
+
+ has_more_work = bng_re_mpc_roce_service_creq(rdev, budget);
+
+ if (has_more_work)
+ dbr_type = DBC_DBC_TYPE_CQ;
+
+ bng_re_ring_db(&creq->cq_db_info, dbr_type);
+
+ return has_more_work;
+}
+
+/**
+ * bng_re_roce_mpc_xmit - Send an MPC command and wait for or poll its completion
+ * @rdev: RoCE device
+ * @cmd: MPC command buffer
+ * @cmd_len: Length of @cmd
+ * @cmpl: Caller buffer for completion payload (copied from internal context)
+ * @cmpl_len: Max bytes to copy into @cmpl
+ * @poll_cmp: If true, busy-poll completion; else use completion / timed wait
+ *
+ * Allocates a per-command context, posts via bng_re_mpc_roce_send() with EBUSY
+ * retries, services the CQ as needed, updates diagnostics and optional stall
+ * detection, and records performance stats.
+ *
+ * Return: 0 on success, negative errno on validation, stall, timeout, or I/O errors.
+ */
+int bng_re_roce_mpc_xmit(struct bng_re_dev *rdev, void *cmd, uint cmd_len,
+ void *cmpl, uint cmpl_len, bool poll_cmp)
+{
+ struct bng_re_mpc_roce_creq_info *creq;
+ unsigned long tx_start_time = jiffies;
+ struct bng_re_mpc_diag_counters *dc;
+ struct bng_re_roce_cmd_ctx *ctx;
+ unsigned long poll_end_time = 0;
+ unsigned long tx_end_time = 0;
+ unsigned int max_pending = 0;
+ unsigned long max_timeout;
+ unsigned long retries = 0;
+ bool sent_started = false;
+ unsigned long tmo_left;
+ unsigned long deadline;
+ unsigned long timeout;
+ unsigned long flags;
+ u16 retry_sleep_ms;
+ void *pctx = NULL;
+ int cnt;
+ int rc;
+
+ dev_dbg(rdev_to_dev(rdev), "%s: enter, poll_cmp:%d\n", __func__, poll_cmp);
+
+ creq = &rdev->mpc_roce_creq;
+ dc = creq->bm_diag_counters;
+
+ if (!cmd || !cmpl) {
+ dev_err(rdev_to_dev(rdev), "%s: NULL cmd or cmpl\n", __func__);
+ return -EINVAL;
+ }
+ if (!creq->mpc_tune_params || !dc || !creq->mpc_cache) {
+ dev_err(rdev_to_dev(rdev), "%s: MPC channel not initialized\n", __func__);
+ return -EINVAL;
+ }
+
+ max_timeout = creq->mpc_tune_params->short_timeout;
+ retry_sleep_ms = creq->mpc_tune_params->retry_sleep;
+
+ atomic_inc(&creq->pending);
+ atomic_inc(&dc->bm_mpc_verb_try);
+
+ if (creq->bm_mpc_stall || test_bit(BNG_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags) ||
+ creq->bm_mpc_uninstall_pending) {
+ rc = -EHOSTUNREACH;
+ atomic_inc(&dc->bm_mpc_stalled_err);
+ dev_warn(rdev_to_dev(rdev), "%s QUIT(1) as %s:%d or %s:%d or %s:%d",
+ __func__,
+ "mpc stall detected",
+ creq->bm_mpc_stall,
+ "HW FATAL Cond(stall)",
+ test_bit(BNG_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags),
+ "driver uninit in progress",
+ creq->bm_mpc_uninstall_pending);
+ goto exit;
+ }
+ /* make sure memory is updated */
+ smp_mb__after_atomic();
+
+ ctx = kmem_cache_zalloc(creq->mpc_cache, GFP_ATOMIC);
+
+ if (!ctx) {
+ rc = -ENOMEM;
+ dev_err(rdev_to_dev(rdev), "%s: ctx alloc failed\n", __func__);
+ goto exit;
+ }
+ refcount_set(&ctx->refcnt, 1);
+ ctx->poll_cmp = poll_cmp;
+ if (!poll_cmp) {
+ init_completion(&ctx->cmp);
+ might_sleep();
+ retries = creq->mpc_tune_params->max_retries;
+ max_timeout = creq->mpc_tune_params->long_timeout;
+ }
+
+ do {
+ atomic_inc(&dc->bm_mpc_sent_try);
+ rc = bng_re_mpc_roce_send(rdev, cmd, cmd_len, (unsigned long)ctx, &pctx);
+ if (rc == -EBUSY) {
+ atomic_inc(&dc->bm_mpc_sent_started_ebusy);
+ } else if (!rc) {
+ atomic_inc(&dc->bm_mpc_sent_started_ok);
+ sent_started = true;
+ } else {
+ atomic_inc(&dc->bm_mpc_sent_started_misc_err);
+ }
+
+ if (rc != -EBUSY) {
+ dev_dbg(rdev_to_dev(rdev), "[%s,%lx]: msg %x.%x BREAK: sleep retry remaining: %ld. outst msgs:%d,rc:%d",
+ __func__,
+ (unsigned long)current,
+ ((u8 *)cmd)[0],
+ ((u8 *)cmd)[1],
+ retries,
+ atomic_read(&creq->pending),
+ rc);
+ break;
+ }
+ /* -EBUSY: one try per (initial + max_retries); last EBUSY exits here */
+ if (!retries) {
+ dev_dbg(rdev_to_dev(rdev), "[%s,%lx]: msg %x.%x BREAK: no retries left. outst msgs:%d,rc:%d",
+ __func__,
+ (unsigned long)current,
+ ((u8 *)cmd)[0],
+ ((u8 *)cmd)[1],
+ atomic_read(&creq->pending),
+ rc);
+ break;
+ }
+ dev_warn(rdev_to_dev(rdev), "[%s,%lx]: msg %x.%x sleep retry remaining: %ld. outstanding msgs:%d",
+ __func__,
+ (unsigned long)current,
+ ((u8 *)cmd)[0],
+ ((u8 *)cmd)[1],
+ retries,
+ atomic_read(&creq->pending));
+ msleep(retry_sleep_ms);
+ retries--;
+ if (creq->bm_mpc_stall || test_bit(BNG_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags) ||
+ creq->bm_mpc_uninstall_pending
+ ) {
+ rc = -EHOSTUNREACH;
+ atomic_inc(&dc->bm_mpc_stalled_err);
+ dev_warn(rdev_to_dev(rdev), "%s QUIT(2) as %s:%d or %s:%d or %s:%d",
+ __func__,
+ "mpc stall detected",
+ creq->bm_mpc_stall,
+ "HW FATAL Cond(stall)",
+ test_bit(BNG_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags),
+ "driver uninit in progress",
+ creq->bm_mpc_uninstall_pending);
+ break;
+ }
+ } while (true);
+
+ tx_end_time = jiffies;
+
+ if (rc)
+ goto xmit_done;
+
+ if (poll_cmp) {
+ deadline = jiffies + msecs_to_jiffies(max_timeout);
+ do {
+ bool avail = READ_ONCE(ctx->cmpl_available);
+ /* make sure completion is available */
+ smp_rmb();
+ /* check for completion availability */
+ if (avail)
+ break;
+
+ udelay(9);
+ bng_re_mpc_roce_service_creq(rdev, 32);
+ if (time_after_eq(jiffies, deadline)) {
+ rc = -ETIMEDOUT;
+ goto xmit_done;
+ }
+ if (creq->bm_mpc_stall ||
+ test_bit(BNG_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags) ||
+ creq->bm_mpc_uninstall_pending
+ ) {
+ rc = -EHOSTUNREACH;
+ atomic_inc(&dc->bm_mpc_stalled_err);
+ dev_warn(rdev_to_dev(rdev), "%s QUIT(3) as %s:%d or %s:%d or %s:%d",
+ __func__,
+ "mpc stall detected",
+ creq->bm_mpc_stall,
+ "HW FATAL Cond(stall)",
+ test_bit(BNG_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags),
+ "driver uninit in progress",
+ creq->bm_mpc_uninstall_pending);
+ goto xmit_done;
+ }
+ } while (true);
+ } else {
+ if (max_timeout < 100) {
+ dev_err(rdev_to_dev(rdev),
+ "%s: long_timeout (%lu ms) < 100; poll_cmp=false wait loop is underspecified\n",
+ __func__, max_timeout);
+ }
+ timeout = max_timeout ?: 1;
+ /*
+ * timeout == max_timeout here, so this is always 1 -- except
+ * when max_timeout is 0, which would otherwise divide by zero.
+ */
+ cnt = max_t(int, 1, (int)(max_timeout / timeout));
+
+ do {
+ tmo_left = wait_for_completion_timeout(&ctx->cmp,
+ msecs_to_jiffies(timeout));
+ if (!tmo_left)
+ break;
+ dev_dbg(rdev_to_dev(rdev), "%s:%lx Wait For Completion: iteration:%d\n",
+ __func__, (unsigned long)current, cnt);
+ cnt--;
+ if (creq->bm_mpc_stall ||
+ test_bit(BNG_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags) ||
+ creq->bm_mpc_uninstall_pending) {
+ rc = -EHOSTUNREACH;
+ atomic_inc(&dc->bm_mpc_stalled_err);
+ dev_warn(rdev_to_dev(rdev), "%s QUIT(4) as %s:%d or %s:%d or %s:%d",
+ __func__,
+ "mpc stall detected",
+ creq->bm_mpc_stall,
+ "HW FATAL Cond(stall)",
+ test_bit(BNG_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags),
+ "driver uninit in progress",
+ creq->bm_mpc_uninstall_pending);
+ goto xmit_done;
+ }
+ } while (cnt);
+ if (!tmo_left) {
+ rc = -ETIMEDOUT;
+ goto xmit_done;
+ }
+ }
+
+ memcpy(cmpl, &ctx->roce_cmp, min(cmpl_len, (uint)sizeof(ctx->roce_cmp)));
+
+xmit_done:
+ if (rc) {
+ int consecutive_errs;
+
+ if (sent_started)
+ atomic_inc(&dc->bm_mpc_sent_response_err);
+
+ dev_warn(rdev_to_dev(rdev), "RoCE MP cmd %08x failed with error:%d\n"
+ "verb try:%d,sent try:%d,sent s.ok:%d,ebusy:%d,resp ok:%d,reps err:%d,%s:%d,%s:%d",
+ *((u32 *)cmd), rc,
+ atomic_read(&dc->bm_mpc_verb_try),
+ atomic_read(&dc->bm_mpc_sent_try),
+ atomic_read(&dc->bm_mpc_sent_started_ok),
+ atomic_read(&dc->bm_mpc_sent_started_ebusy),
+ atomic_read(&dc->bm_mpc_sent_response_ok),
+ atomic_read(&dc->bm_mpc_sent_response_err),
+ "pending",
+ atomic_read(&creq->pending),
+ "stalled",
+ atomic_read(&dc->bm_mpc_stalled_err));
+ spin_lock_irqsave(&creq->mpc_ctx_lock, flags);
+ if (pctx)
+ *(unsigned long **)pctx = 0;
+ spin_unlock_irqrestore(&creq->mpc_ctx_lock, flags);
+ dump_tx_cmpl_ring_info(rdev);
+
+ if (rc == -EBUSY)
+ atomic_inc(&dc->bm_mpc_fatal_ebusy);
+ else if (rc == -ETIMEDOUT)
+ atomic_inc(&dc->bm_mpc_sent_timeout);
+ else
+ atomic_inc(&dc->bm_mpc_misc_err);
+
+ consecutive_errs = atomic_inc_return(&dc->bm_mpc_consecutive_sent_err);
+
+#ifdef MPC_STALL_DETECTION
+ if (consecutive_errs > BNGE_MPC_ROCE_ERR_THRESHOLD) {
+ dev_warn(rdev_to_dev(rdev),
+ "%s:%d consecutive err;MPC ch marked stalled\n",
+ __func__,
+ consecutive_errs);
+ creq->bm_mpc_stall = true;
+ }
+#endif
+ } else {
+ atomic_inc(&dc->bm_mpc_sent_response_ok);
+ atomic_set(&dc->bm_mpc_consecutive_sent_err, 0);
+ }
+ if (refcount_dec_and_test(&ctx->refcnt))
+ kmem_cache_free(creq->mpc_cache, ctx);
+ exit:
+ max_pending = max(atomic_dec_return(&creq->pending),
+ atomic_read(&creq->max_pending));
+ atomic_set(&creq->max_pending, max_pending);
+
+ poll_end_time = jiffies;
+
+ spin_lock_irqsave(&creq->mpc_ctx_lock, flags);
+ /* Track available buffer space */
+ creq->avail_buffer = bng_re_mpc_tx_avail(creq);
+ creq->min_avail_buffer = min(creq->avail_buffer,
+ creq->min_avail_buffer);
+ bng_re_mpc_roce_add_perf_stats(rdev,
+ ((u8 *)cmd)[0],
+ ((u8 *)cmd)[1],
+ (u32)jiffies_to_msecs(poll_end_time - tx_end_time));
+ spin_unlock_irqrestore(&creq->mpc_ctx_lock, flags);
+ bnge_re_print_mpc_msg(rdev, cmd, cmd_len, tx_start_time,
+ tx_end_time, poll_end_time, rc, retries);
+ return rc;
+}
+
+void bnge_re_print_mpc_msg(struct bng_re_dev *rdev,
+ u8 *cmd,
+ uint cmd_len,
+ unsigned long tx_start_time,
+ unsigned long tx_end_time,
+ unsigned long poll_end_time,
+ int rc,
+ int retries_left)
+{
+ unsigned long poll_duration = jiffies_to_msecs(poll_end_time - tx_end_time);
+ unsigned long tx_duration = jiffies_to_msecs(tx_end_time - tx_start_time);
+ struct bng_re_mpc_roce_creq_info *creq = &rdev->mpc_roce_creq;
+ u8 req_subtype = *(cmd + 1);
+ u8 req_type = *cmd;
+
+ dev_dbg(rdev_to_dev(rdev),
+ "MPC_TX mtyp:%x:%x,len:%d,tx st time:%ld,enq dur:%ld,comp dur:%ld,rc %d,outst:%d,max_pend:%d,avail_buf:%u,min_avail_buf:%u,retries_left:%d\n",
+ req_type,
+ req_subtype,
+ cmd_len,
+ tx_start_time,
+ tx_duration,
+ poll_duration,
+ rc,
+ atomic_read(&creq->pending),
+ atomic_read(&creq->max_pending),
+ creq->avail_buffer,
+ creq->min_avail_buffer,
+ retries_left);
+}
+
+/**
+ * bng_re_mpc_rings_alloc - Allocate MPC TX and CQ rings with firmware
+ * @rdev: RoCE device
+ *
+ * Must be called after the MPC TX and CQ HWQs are allocated via
+ * bng_re_alloc_init_hwq(). Issues HWRM_RING_ALLOC for L2 completion then TX
+ * (MPC primate channel), wiring CQ ring id and page tables.
+ *
+ * Return: 0 on success, negative errno from firmware or %-ENODEV if detached.
+ */
+int bng_re_mpc_rings_alloc(struct bng_re_dev *rdev)
+{
+ struct bng_re_mpc_roce_creq_info *creq = &rdev->mpc_roce_creq;
+ struct bng_re_chip_ctx *cctx = rdev->chip_ctx;
+ struct hwrm_ring_alloc_input req = {0};
+ struct hwrm_ring_alloc_output resp;
+ struct bnge_fw_msg fw_msg = {};
+ int cq_pages, tx_pages;
+ int rc;
+
+ dev_dbg(rdev_to_dev(rdev), "%s: %p", __func__, rdev);
+
+ if (test_bit(BNG_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) {
+ dev_err(rdev_to_dev(rdev), "%s: device detached\n", __func__);
+ return -ENODEV;
+ }
+
+ cq_pages = creq->cq_hwq.pbl[creq->cq_hwq.level].pg_count;
+ tx_pages = creq->tx_hwq.pbl[creq->tx_hwq.level].pg_count;
+
+ /* 1. Allocate CQ (completion) ring first */
+ bng_re_init_hwrm_hdr((struct input *)&req, HWRM_RING_ALLOC);
+ req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
+ req.length = cpu_to_le32(creq->cq_ring_mask + 1);
+ req.logical_id = cpu_to_le16(creq->bm_lr_index);
+ req.page_tbl_addr = cpu_to_le64(creq->cq_hwq.pbl[BNG_PBL_LVL_0].pg_map_arr[0]);
+ if (cq_pages > 1) {
+ req.page_size = BNGE_PAGE_SHIFT;
+ req.page_tbl_depth = 1;
+ } else {
+ req.page_size = 4;
+ req.page_tbl_depth = 0;
+ }
+ req.fbo = 0;
+
+ req.enables = cpu_to_le32(RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
+
+ req.nq_ring_id = cpu_to_le16(creq->bm_nq_ring_id);
+ if (cctx->modes.st_tag_supported) {
+ req.steering_tag = cpu_to_le16(BNG_RE_STEERING_TO_HOST);
+ req.enables |= cpu_to_le32(RING_ALLOC_REQ_ENABLES_STEERING_TAG_VALID);
+ }
+ req.flags |=
+ cpu_to_le16(RING_ALLOC_REQ_FLAGS_DISABLE_CQ_OVERFLOW_DETECTION);
+
+ bng_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT);
+ rc = bnge_send_msg(rdev->aux_dev, &fw_msg);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "%s: CQ ring alloc failed rc=%d\n",
+ __func__, rc);
+ return rc;
+ }
+ creq->cq_ring_id = (u16)le32_to_cpu(resp.ring_id);
+ dev_dbg(rdev_to_dev(rdev), "%s: CQ ring_id=0x%x\n", __func__, creq->cq_ring_id);
+
+ /* 2. Allocate TX ring (associated with CQ) */
+ memset(&req, 0, sizeof(req));
+ bng_re_init_hwrm_hdr((void *)&req, HWRM_RING_ALLOC);
+ req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
+ req.length = cpu_to_le32(creq->tx_ring_mask + 1);
+ req.logical_id = cpu_to_le16(creq->bm_lr_index);
+ req.cmpl_ring_id = cpu_to_le16(creq->cq_ring_id);
+ req.stat_ctx_id = cpu_to_le32(creq->bm_stat_index);
+ req.page_tbl_addr = cpu_to_le64(creq->tx_hwq.pbl[BNG_PBL_LVL_0].pg_map_arr[0]);
+ if (tx_pages > 1) {
+ req.page_size = BNGE_PAGE_SHIFT;
+ req.page_tbl_depth = 1;
+ } else {
+ req.page_size = 4;
+ req.page_tbl_depth = 0;
+ }
+ req.fbo = 0;
+ req.enables = cpu_to_le32(RING_ALLOC_REQ_ENABLES_MPC_CHNLS_TYPE);
+ req.mpc_chnls_type = RING_ALLOC_REQ_MPC_CHNLS_TYPE_PRIMATE;
+ if (cctx->modes.st_tag_supported) {
+ req.steering_tag = cpu_to_le16(BNG_RE_STEERING_TO_HOST);
+ req.enables |= cpu_to_le32(RING_ALLOC_REQ_ENABLES_STEERING_TAG_VALID);
+ }
+
+ bng_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT);
+ rc = bnge_send_msg(rdev->aux_dev, &fw_msg);
+
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "%s: TX ring alloc failed rc=%d\n",
+ __func__, rc);
+ bng_re_net_ring_free(rdev, creq->cq_ring_id,
+ RING_FREE_REQ_RING_TYPE_L2_CMPL);
+ creq->cq_ring_id = INVALID_HW_RING_ID;
+ return rc;
+ }
+ creq->tx_ring_id = le32_to_cpu(resp.ring_id);
+ dev_dbg(rdev_to_dev(rdev), "%s: TX ring_id=0x%x\n", __func__, creq->tx_ring_id);
+
+ dev_info(rdev_to_dev(rdev),
+ "mpc rings: cq_ring_id=0x%x tx_ring_id=0x%x lr_index=%u nq_ring_id=%u stat_index=%u\n",
+ creq->cq_ring_id, creq->tx_ring_id, creq->bm_lr_index,
+ creq->bm_nq_ring_id, creq->bm_stat_index);
+
+ return 0;
+}
+
+/**
+ * bng_re_mpc_rings_free - Free MPC TX and CQ rings with firmware
+ * @rdev: RoCE device
+ *
+ * Frees the TX ring first when allocated, then the L2 completion ring.
+ */
+void bng_re_mpc_rings_free(struct bng_re_dev *rdev)
+{
+ struct bng_re_mpc_roce_creq_info *creq = &rdev->mpc_roce_creq;
+
+ dev_dbg(rdev_to_dev(rdev), "%s: %p", __func__, creq);
+
+ if (creq->tx_ring_id != INVALID_HW_RING_ID_32BIT) {
+ bng_re_net_ring_free(rdev, creq->tx_ring_id,
+ RING_FREE_REQ_RING_TYPE_TX);
+ creq->tx_ring_id = INVALID_HW_RING_ID;
+ }
+ if (creq->cq_ring_id != INVALID_HW_RING_ID_32BIT) {
+ bng_re_net_ring_free(rdev, creq->cq_ring_id,
+ RING_FREE_REQ_RING_TYPE_L2_CMPL);
+ creq->cq_ring_id = INVALID_HW_RING_ID;
+ }
+}
+
+/******************************************************
+ * Stats section *
+ *****************************************************/
+
+/**
+ * bng_re_mpc_roce_alloc_stats - Allocate MPC performance statistics structures
+ * @rdev: RoCE device
+ *
+ * Allocates latency histograms and per-verb latency ring buffers under
+ * mpc_roce_creq.bm_stats_info. Enables stats only if every allocation succeeds.
+ *
+ * Return: 0 on success, %-ENOMEM on failure (partial allocations freed).
+ */
+int bng_re_mpc_roce_alloc_stats(struct bng_re_dev *rdev)
+{
+ struct bng_re_mpc_roce_stats_info *p;
+
+ p = vzalloc(sizeof(*p));
+ if (!p)
+ return -ENOMEM;
+
+ p->bms_stats_enabled = false;
+ p->bms_qp_modify_stats = vzalloc(sizeof(u64) * BNG_RE_MPC_MAX_STAT_INDEX);
+ p->bms_ah_modify_stats = vzalloc(sizeof(u64) * BNG_RE_MPC_MAX_STAT_INDEX);
+ p->bms_other_stats = vzalloc(sizeof(u64) * BNG_RE_MPC_MAX_STAT_INDEX);
+ p->bms_lat_slab_sec = vzalloc(sizeof(u32) *
+ BNG_RE_MPC_MAX_LATENCY_SEC_SLAB_INDEX);
+ p->bms_lat_slab_msec = vzalloc(sizeof(u32) *
+ BNG_RE_MPC_MAX_LATENCY_MSEC_SLAB_INDEX);
+ rdev->mpc_roce_creq.bm_stats_info = p;
+
+ if (p->bms_qp_modify_stats &&
+ p->bms_ah_modify_stats &&
+ p->bms_other_stats &&
+ p->bms_lat_slab_sec &&
+ p->bms_lat_slab_msec) {
+ p->bms_stats_enabled = true;
+ return 0;
+ }
+ bng_re_mpc_roce_free_stats(rdev);
+ return -ENOMEM;
+}
+
+/**
+ * bng_re_mpc_roce_free_stats - Free MPC performance statistics
+ * @rdev: RoCE device
+ */
+void bng_re_mpc_roce_free_stats(struct bng_re_dev *rdev)
+{
+ struct bng_re_mpc_roce_stats_info *p = rdev->mpc_roce_creq.bm_stats_info;
+
+ if (!p)
+ return;
+
+ if (p->bms_qp_modify_stats)
+ vfree(p->bms_qp_modify_stats);
+ if (p->bms_ah_modify_stats)
+ vfree(p->bms_ah_modify_stats);
+ if (p->bms_other_stats)
+ vfree(p->bms_other_stats);
+ if (p->bms_lat_slab_sec)
+ vfree(p->bms_lat_slab_sec);
+ if (p->bms_lat_slab_msec)
+ vfree(p->bms_lat_slab_msec);
+
+ p->bms_stats_enabled = false;
+ rdev->mpc_roce_creq.bm_stats_info = NULL;
+}
+
+/**
+ * bng_re_mpc_roce_add_perf_stats - Record one MPC round-trip latency sample
+ * @rdev: RoCE device
+ * @req_type: MPC command header req_type
+ * @req_subtype: MPC command header req_subtype
+ * @latency_msec: Elapsed time from post to completion, in milliseconds
+ *
+ * Updates second- and millisecond-resolution histograms and stores
+ * the latency in the rotating sample array for QP modify, AH modify, or other.
+ */
+void bng_re_mpc_roce_add_perf_stats(struct bng_re_dev *rdev,
+ u8 req_type,
+ u8 req_subtype,
+ u32 latency_msec)
+{
+ struct bng_re_mpc_roce_stats_info *p =
+ rdev->mpc_roce_creq.bm_stats_info;
+ u64 *dest_stats_ptr = NULL;
+ u32 dest_stats_id;
+
+ if (!p || !p->bms_stats_enabled)
+ return;
+
+ if (latency_msec / 1000 < BNG_RE_MPC_MAX_LATENCY_SEC_SLAB_INDEX)
+ p->bms_lat_slab_sec[latency_msec / 1000]++;
+
+ if (latency_msec < BNG_RE_MPC_MAX_LATENCY_MSEC_SLAB_INDEX)
+ p->bms_lat_slab_msec[latency_msec]++;
+
+ if (req_type == MPC_CMD_HDR_REQ_TYPE_RCA &&
+ req_subtype == MPC_CMD_HDR_REQ_SUB_TYPE_RCA_QP_MODIFY) {
+ dest_stats_id = p->bms_qp_modify_stats_id++;
+ dest_stats_id = dest_stats_id % BNG_RE_MPC_MAX_STAT_INDEX;
+ dest_stats_ptr = &p->bms_qp_modify_stats[dest_stats_id];
+ /* stopped here */
+ } else if (((req_type == MPC_CMD_HDR_REQ_TYPE_RCA) ||
+ (req_type == MPC_CMD_HDR_REQ_TYPE_PFVF)) &&
+ (req_subtype == MPC_CMD_HDR_REQ_SUB_TYPE_RCA_AH_MODIFY)) {
+ dest_stats_id = p->bms_ah_modify_stats_id++;
+ dest_stats_id = dest_stats_id % BNG_RE_MPC_MAX_STAT_INDEX;
+ dest_stats_ptr = &p->bms_ah_modify_stats[dest_stats_id];
+ } else {
+ dest_stats_id = p->bms_other_stats_id++;
+ dest_stats_id = dest_stats_id % BNG_RE_MPC_MAX_STAT_INDEX;
+ dest_stats_ptr = &p->bms_other_stats[dest_stats_id];
+ }
+ if (dest_stats_ptr)
+ *dest_stats_ptr = latency_msec;
+}
+
+/**
+ * bng_re_mpc_roce_diag_counters_debugfs_show - Print MPC diagnostic counters to seq_file
+ * @rdev: RoCE device
+ * @s: debugfs seq_file output
+ *
+ * Return: 0 on success, %-ENOMEM if diagnostic counters are not allocated.
+ */
+int bng_re_mpc_roce_diag_counters_debugfs_show(struct bng_re_dev *rdev,
+ struct seq_file *s)
+{
+ struct bng_re_mpc_roce_creq_info *creq = &rdev->mpc_roce_creq;
+ struct bng_re_mpc_diag_counters *mpc_diag_counters =
+ creq->bm_diag_counters;
+
+ if (!mpc_diag_counters)
+ return -ENOMEM;
+
+ seq_printf(s,
+ "%s:\n%s: %d\n%s: %d\n%s: %d\n%s: %d\n%s: %d\n%s: %d\n",
+ "RoCE MP cmd stats",
+ "verb try",
+ atomic_read(&mpc_diag_counters->bm_mpc_verb_try),
+ "sent try",
+ atomic_read(&mpc_diag_counters->bm_mpc_sent_try),
+ "sent s.ok",
+ atomic_read(&mpc_diag_counters->bm_mpc_sent_started_ok),
+ "ebusy",
+ atomic_read(&mpc_diag_counters->bm_mpc_sent_started_ebusy),
+ "sent s.misc err",
+ atomic_read(&mpc_diag_counters->bm_mpc_sent_started_misc_err),
+ "resp ok",
+ atomic_read(&mpc_diag_counters->bm_mpc_sent_response_ok));
+ seq_printf(s,
+ "%s: %d\n%s: %d\n%s: %d\n%s: %u\n%s: %u\n",
+ "resp err",
+ atomic_read(&mpc_diag_counters->bm_mpc_sent_response_err),
+ "pending",
+ atomic_read(&creq->pending),
+ "max_pending",
+ atomic_read(&creq->max_pending),
+ "avail_buf",
+ creq->avail_buffer,
+ "min_avail_buf",
+ creq->min_avail_buffer);
+ seq_puts(s, ".... MP cmd stats:\n");
+ seq_printf(s, "fatal ebusy: %d\nsent timeout: %d\nmisc err: %d\n",
+ atomic_read(&mpc_diag_counters->bm_mpc_fatal_ebusy),
+ atomic_read(&mpc_diag_counters->bm_mpc_sent_timeout),
+ atomic_read(&mpc_diag_counters->bm_mpc_misc_err));
+ seq_printf(s, "consecutive errs: %d\nchannel stalled: %d\nstalled errs:%d\n",
+ atomic_read(&mpc_diag_counters->bm_mpc_consecutive_sent_err),
+ creq->bm_mpc_stall,
+ atomic_read(&mpc_diag_counters->bm_mpc_stalled_err));
+
+ return 0;
+}
+
+/**
+ * bng_re_mpc_roce_perf_debugfs_show - Print MPC latency / perf stats to seq_file
+ * @rdev: RoCE device
+ * @s: debugfs seq_file output
+ * @level: Verbosity (0 = slabs in seconds only; 1 adds msec slabs; 3 adds per-slot
+ * totals; 10+ dumps each index line for QP/AH/other arrays)
+ *
+ * Return: 0 on success, %-ENOMEM if stats are disabled or missing.
+ */
+int bng_re_mpc_roce_perf_debugfs_show(struct bng_re_dev *rdev,
+ struct seq_file *s,
+ u8 level)
+{
+ struct bng_re_mpc_roce_stats_info *p = rdev->mpc_roce_creq.bm_stats_info;
+ u64 qp_modify_total_msec = 0;
+ u64 ah_modify_total_msec = 0;
+ u64 other_total_msec = 0;
+ int qp_modify_total = 0;
+ int ah_modify_total = 0;
+ bool add_entry = false;
+ int other_total = 0;
+ int i = 0;
+
+ seq_puts(s, "==\n");
+
+ seq_printf(s, "bng_re mpc roce perf stats enabled:%s\n",
+ (p && p->bms_stats_enabled) ? "Enabled" : "Disabled");
+
+ if (!(p && p->bms_stats_enabled))
+ return -ENOMEM;
+
+ for (i = 0; i < BNG_RE_MPC_MAX_LATENCY_SEC_SLAB_INDEX; i++) {
+ if (p->bms_lat_slab_sec[i])
+ seq_printf(s, "\tlatency_slab [%d - %d] sec = %d\n",
+ i, i + 1, p->bms_lat_slab_sec[i]);
+ }
+ if (level < 1)
+ goto exit;
+
+ seq_puts(s, "==\n");
+ for (i = 0; i < BNG_RE_MPC_MAX_LATENCY_MSEC_SLAB_INDEX; i++) {
+ if (p->bms_lat_slab_msec[i])
+ seq_printf(s, "\tlatency_slab [%d - %d] msec = %d\n",
+ i, i + 1, p->bms_lat_slab_msec[i]);
+ }
+
+ if (level < 3)
+ goto exit;
+
+ seq_puts(s, "==\n");
+ for (i = 0; i < BNG_RE_MPC_MAX_STAT_INDEX; i++) {
+ if (p->bms_qp_modify_stats[i] > 0) {
+ qp_modify_total++;
+ qp_modify_total_msec += p->bms_qp_modify_stats[i];
+ add_entry = true;
+ }
+ if (p->bms_ah_modify_stats[i] > 0) {
+ ah_modify_total++;
+ ah_modify_total_msec += p->bms_ah_modify_stats[i];
+ add_entry = true;
+ }
+ if (p->bms_other_stats[i] > 0) {
+ other_total++;
+ other_total_msec += p->bms_other_stats[i];
+ add_entry = true;
+ }
+
+ if (level >= 10) {
+ if (add_entry)
+ seq_printf(s, "<qp_modify> %lld <ah_modify> %lld <other> %lld\n",
+ p->bms_qp_modify_stats[i],
+ p->bms_ah_modify_stats[i],
+ p->bms_other_stats[i]);
+ }
+
+ add_entry = false;
+ }
+
+ seq_puts(s, "==\n");
+ seq_printf(s, "Total qp_modify %d in msec %lld\n",
+ qp_modify_total, qp_modify_total_msec);
+ seq_printf(s, "Total ah_modify %d in msec %lld\n",
+ ah_modify_total, ah_modify_total_msec);
+ seq_printf(s, "Total other %d in msec %lld\n",
+ other_total, other_total_msec);
+exit:
+ seq_puts(s, "\n");
+
+ return 0;
+}
+
+/**
+ * bng_re_mpc_roce_perf_debugfs_clear - Zero MPC perf histograms and sample arrays
+ * @rdev: RoCE device
+ */
+void bng_re_mpc_roce_perf_debugfs_clear(struct bng_re_dev *rdev)
+{
+ struct bng_re_mpc_roce_stats_info *p = rdev->mpc_roce_creq.bm_stats_info;
+ int i;
+
+ if (!(p && p->bms_stats_enabled))
+ return;
+
+ for (i = 0; i < BNG_RE_MPC_MAX_LATENCY_MSEC_SLAB_INDEX; i++)
+ p->bms_lat_slab_msec[i] = 0;
+
+ for (i = 0; i < BNG_RE_MPC_MAX_LATENCY_SEC_SLAB_INDEX; i++)
+ p->bms_lat_slab_sec[i] = 0;
+
+ for (i = 0; i < BNG_RE_MPC_MAX_STAT_INDEX; i++) {
+ p->bms_qp_modify_stats[i] = 0;
+ p->bms_ah_modify_stats[i] = 0;
+ p->bms_other_stats[i] = 0;
+ }
+}
+
+/**
+ * bng_re_mpc_roce_diag_counters_clear - Reset MPC xmit/diagnostic atomics to zero
+ * @rdev: RoCE device
+ */
+void bng_re_mpc_roce_diag_counters_clear(struct bng_re_dev *rdev)
+{
+ struct bng_re_mpc_diag_counters *mpc_diag_counters =
+ rdev->mpc_roce_creq.bm_diag_counters;
+
+ if (!mpc_diag_counters)
+ return;
+
+ atomic_set(&mpc_diag_counters->bm_mpc_verb_try, 0);
+ atomic_set(&mpc_diag_counters->bm_mpc_sent_try, 0);
+ atomic_set(&mpc_diag_counters->bm_mpc_sent_started_ok, 0);
+ atomic_set(&mpc_diag_counters->bm_mpc_sent_started_ebusy, 0);
+ atomic_set(&mpc_diag_counters->bm_mpc_sent_started_misc_err, 0);
+ atomic_set(&mpc_diag_counters->bm_mpc_sent_response_ok, 0);
+ atomic_set(&mpc_diag_counters->bm_mpc_sent_response_err, 0);
+ atomic_set(&mpc_diag_counters->bm_mpc_fatal_ebusy, 0);
+ atomic_set(&mpc_diag_counters->bm_mpc_sent_timeout, 0);
+ atomic_set(&mpc_diag_counters->bm_mpc_misc_err, 0);
+ atomic_set(&mpc_diag_counters->bm_mpc_consecutive_sent_err, 0);
+ atomic_set(&mpc_diag_counters->bm_mpc_stalled_err, 0);
+}
+
+/**
+ * bng_re_snapdump_mpc_stats - Collect MPC performance and latency stats for coredump.
+ * @rdev: RoCE device
+ * @buf: Dump buffer
+ * @buf_len: Buffer length
+ * Returns: Number of bytes written
+ */
+u32 bng_re_snapdump_mpc_stats(struct bng_re_dev *rdev, void *buf, u32 buf_len)
+{
+ struct bng_re_mpc_roce_creq_info *creq = &rdev->mpc_roce_creq;
+ struct bng_re_mpc_roce_stats_info *p = creq->bm_stats_info;
+ struct bng_re_mpc_diag_counters *mpc_diag =
+ creq->bm_diag_counters;
+ u32 len = 0;
+ int i;
+
+ dev_dbg(rdev_to_dev(rdev), "%s: buf_len:%d\n", __func__, buf_len);
+
+ /* Collect MPC perf stats if enabled */
+ if (p && p->bms_stats_enabled) {
+ len += snprintf(buf + len, buf_len - len, "==\n");
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len,
+ "bnge mpc roce perf stats enabled:%s\n",
+ p->bms_stats_enabled ? "Enabled" : "Disabled");
+ if (len >= buf_len)
+ return len;
+
+ for (i = 0; i < BNG_RE_MPC_MAX_LATENCY_SEC_SLAB_INDEX; i++) {
+ if (p->bms_lat_slab_sec[i]) {
+ len += snprintf(buf + len, buf_len - len,
+ "\tlatency_slab [%d - %d] sec = %d\n",
+ i, i + 1, p->bms_lat_slab_sec[i]);
+ if (len >= buf_len)
+ return len;
+ }
+ }
+
+ len += snprintf(buf + len, buf_len - len, "==\n");
+ if (len >= buf_len)
+ return len;
+
+ for (i = 0; i < BNG_RE_MPC_MAX_LATENCY_MSEC_SLAB_INDEX; i++) {
+ if (p->bms_lat_slab_msec[i]) {
+ len += snprintf(buf + len, buf_len - len,
+ "\tlatency_slab [%d - %d] msec = %d\n",
+ i, i + 1, p->bms_lat_slab_msec[i]);
+ if (len >= buf_len)
+ return len;
+ }
+ }
+ }
+
+ /* Collect MPC diagnostic counters */
+ if (mpc_diag && creq->mpc_cache) {
+ len += snprintf(buf + len, buf_len - len, "RoCE MP cmd stats:\n");
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, "verb try: %d\n",
+ atomic_read(&mpc_diag->bm_mpc_verb_try));
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, "sent try: %d\n",
+ atomic_read(&mpc_diag->bm_mpc_sent_try));
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, "sent s.ok: %d\n",
+ atomic_read(&mpc_diag->bm_mpc_sent_started_ok));
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, "pending: %d\n",
+ atomic_read(&creq->pending));
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, "max_pending: %d\n",
+ atomic_read(&creq->max_pending));
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, "avail_buf: %u\n",
+ creq->avail_buffer);
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, "min_avail_buf: %u\n",
+ creq->min_avail_buffer);
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, "ebusy: %d\n",
+ atomic_read(&mpc_diag->bm_mpc_sent_started_ebusy));
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, "sent s.misc err: %d\n",
+ atomic_read(&mpc_diag->bm_mpc_sent_started_misc_err));
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, "resp ok: %d\n",
+ atomic_read(&mpc_diag->bm_mpc_sent_response_ok));
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, "resp err: %d\n",
+ atomic_read(&mpc_diag->bm_mpc_sent_response_err));
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, ".... MP cmd stats:\n");
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, "fatal ebusy: %d\n",
+ atomic_read(&mpc_diag->bm_mpc_fatal_ebusy));
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, "sent timeout: %d\n",
+ atomic_read(&mpc_diag->bm_mpc_sent_timeout));
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, "misc err: %d\n",
+ atomic_read(&mpc_diag->bm_mpc_misc_err));
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, "consecutive sent err: %d\n",
+ atomic_read(&mpc_diag->bm_mpc_consecutive_sent_err));
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, "channel stalled: %d\n",
+ creq->bm_mpc_stall);
+ if (len >= buf_len)
+ return len;
+
+ len += snprintf(buf + len, buf_len - len, "stalled err: %d\n",
+ atomic_read(&mpc_diag->bm_mpc_stalled_err));
+ if (len >= buf_len)
+ return len;
+ }
+
+ return len;
+}
+
+/**
+ * dump_tx_cmpl_ring_info - Log TX/CQ producer-consumer state and recent CQ slots
+ * @rdev: RoCE device
+ *
+ * Takes tx_lock for ring indices and bm_lock while dumping poll timing and a
+ * short hex decode window around the last stalled consumer index.
+ */
+void dump_tx_cmpl_ring_info(struct bng_re_dev *rdev)
+{
+ struct bng_re_mpc_roce_creq_info *creq = &rdev->mpc_roce_creq;
+ struct bng_re_mpc_poll_info *poll_info = creq->bm_poll_info;
+ u64 curr_jiffies = get_jiffies_64();
+ u32 start_raw_index = 0;
+ unsigned long flags;
+
+ spin_lock_irqsave(&creq->tx_lock, flags);
+ dev_warn(rdev_to_dev(rdev),
+ "[%s.%lx] curr ms:%d,tx ring prod,cons,cq_cons:%d,%d,%d,tx_ring_size:%d,cmpl_ring_size:%d\n",
+ __func__,
+ (unsigned long)current,
+ jiffies_to_msecs(curr_jiffies),
+ READ_ONCE(creq->tx_prod),
+ READ_ONCE(creq->tx_cons),
+ READ_ONCE(creq->cq_hwq.cons),
+ creq->tx_hwq.max_elements,
+ creq->cq_hwq.max_elements);
+ spin_unlock_irqrestore(&creq->tx_lock, flags);
+
+ spin_lock_irqsave(&creq->bm_lock, flags);
+ dev_warn(rdev_to_dev(rdev), "[%s.%lx] %s:%d,%d;%s:%d,%s:%d,%s:%d,%d",
+ __func__,
+ (unsigned long)current,
+ "last_poll_st/end",
+ jiffies_to_msecs(poll_info->bm_last_poll_work_poll_jiffies_start),
+ jiffies_to_msecs(poll_info->bm_last_poll_work_poll_jiffies_end),
+ "max_ms_b_polls",
+ jiffies_to_msecs(poll_info->bm_max_poll_work_poll_jiffies),
+ "curr cons",
+ poll_info->bm_current_cons_indx,
+ "ms on curr cons[st/diff]",
+ jiffies_to_msecs(poll_info->bm_start_time_on_current_cons_indx),
+ jiffies_to_msecs(poll_info->bm_curr_time_on_current_cons_indx));
+ if (poll_info->bm_current_cons_indx >= 5)
+ start_raw_index = poll_info->bm_current_cons_indx - 5;
+
+ spin_unlock_irqrestore(&creq->bm_lock, flags);
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_re_mpc_roce.h b/drivers/infiniband/hw/bng_re/bng_re_mpc_roce.h
new file mode 100644
index 000000000000..152471462048
--- /dev/null
+++ b/drivers/infiniband/hw/bng_re/bng_re_mpc_roce.h
@@ -0,0 +1,218 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2025 Broadcom. */
+
+#ifndef __BNG_RE_MPC_ROCE_H__
+#define __BNG_RE_MPC_ROCE_H__
+
+#include <linux/stddef.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/workqueue.h>
+#include <linux/delay.h>
+#include <linux/refcount.h>
+
+#include "bng_roce_hsi.h"
+#include "bng_re.h"
+#include "bng_res.h"
+
+/* RoCE MPC diagnostics and knobs (mirrors bnge for compatibility) */
+#define BNG_RE_MPC_MAX_LATENCY_SEC_SLAB_INDEX 201
+#define BNG_RE_MPC_MAX_LATENCY_MSEC_SLAB_INDEX 3000
+#define BNG_RE_MPC_MAX_STAT_INDEX 0x3FFFF
+
+#define BNG_RE_MPC_ROCE_TMO_MSECS_SHORT 8000 /* for nonblocking context */
+#ifdef BNGE_FPGA
+#define BNG_RE_MPC_ROCE_TMO_MSECS_LONG 100000 /* for normal context */
+#define BNG_RE_MPC_ROCE_RETRY_SLEEP 2000 /* ms */
+#else
+#define BNG_RE_MPC_ROCE_TMO_MSECS_LONG 40000 /* for normal context */
+#define BNG_RE_MPC_ROCE_RETRY_SLEEP 200 /* ms */
+#endif
+#define BNG_RE_MPC_ROCE_MAX_RETRIES 32
+#define BNG_RE_MPC_ROCE_ERR_THRESHOLD 10
+
+/* bng_re_mpc_poll_info, bng_re_mpc_diag_counters, bng_re_mpc_tune_params,
+ * bng_re_mpc_roce_stats_info are defined in bng_re.h — include that instead.
+ */
+#define MPC_ROCE_COMPL_MAX_SIZE 32
+
+/*
+ * Single work item for all PF→VF forwarded MPC completions.
+ * bng_re_mpc_process_vf_completion() dispatches on req_subtype internally,
+ * so no per-operation struct is needed.
+ */
+struct bng_re_vf_pfvf_cmpl_work {
+ struct work_struct work;
+ struct bng_re_dev *rdev;
+ u8 roce_cmp[MPC_ROCE_COMPL_MAX_SIZE];
+};
+
+struct bng_re_cmpl_entry {
+ void *cmpl;
+ u32 len;
+};
+
+struct mpc_cmp {
+ __le32 mpc_cmp_client_subtype_type;
+ #define MPC_CMP_TYPE (0x3f << 0)
+ #define MPC_CMP_TYPE_MID_PATH_SHORT 0x1e
+ #define MPC_CMP_TYPE_MID_PATH_LONG 0x1f
+ #define MPC_CMP_SUBTYPE 0xf00
+ #define MPC_CMP_SUBTYPE_SFT 8
+ #define MPC_CMP_SUBTYPE_SOLICITED (0x0 << 8)
+ #define MPC_CMP_SUBTYPE_ERR (0x1 << 8)
+ #define MPC_CMP_SUBTYPE_RESYNC (0x2 << 8)
+ #define MPC_CMP_CLIENT (0xf << 12)
+ #define MPC_CMP_CLIENT_SFT 12
+ #define MPC_CMP_CLIENT_TCE (0x0 << 12)
+ #define MPC_CMP_CLIENT_RCE (0x1 << 12)
+ #define MPC_CMP_CLIENT_TE_CFA (0x2 << 12)
+ #define MPC_CMP_CLIENT_RE_CFA (0x3 << 12)
+ #define MPC_CMP_CLIENT_PRIMATE (0x4 << 12)
+ u32 mpc_cmp_opaque;
+ __le32 mpc_cmp_v;
+ #define MPC_CMP_V BIT(0)
+ __le32 mpc_cmp_filler;
+};
+
+struct bng_re_roce_cmd_ctx {
+ struct completion cmp;
+ u8 roce_cmp[MPC_ROCE_COMPL_MAX_SIZE];
+ bool poll_cmp;
+ bool cmpl_available;
+ /*
+ * Shared ownership between bng_re_roce_mpc_xmit (always holds 1 ref)
+ * and bng_re_mpc_roce_service_creq (acquires +1 while processing a
+ * completion for this ctx). Whoever decrements to zero frees the
+ * object. This prevents the use-after-free where a late completion
+ * writes into a ctx that xmit has already freed on a timeout/error
+ * path.
+ */
+ refcount_t refcnt;
+};
+
+/* Same message types as bnge */
+#define BNG_RE_MPC_TX_RING_SIZE 512
+#define BNG_RE_MPC_CQ_RING_SIZE (BNG_RE_MPC_TX_RING_SIZE * 4)
+#define BNG_RE_MPC_TX_STRIDE 16 /* sizeof(struct tx_bd) */
+#define BNG_RE_MPC_CQ_STRIDE 16 /* sizeof(struct tx_cmp) */
+#define BNG_RE_MPC_INV_HDL (-1UL)
+
+#define MAX_ADV_ITERATIONS (1000)
+
+#ifndef TX_OPAQUE_IDX_MASK
+#define TX_OPAQUE_IDX_MASK 0x0000ffff
+#endif
+#ifndef TX_OPAQUE_BDS_SHIFT
+#define TX_OPAQUE_BDS_SHIFT 16
+#endif
+#ifndef TX_OPAQUE_RING_SHIFT
+#define TX_OPAQUE_RING_SHIFT 24
+#endif
+#ifndef TX_BD_TYPE_MPC_TX_BD
+#define TX_BD_TYPE_MPC_TX_BD (0x08 << 0)
+#endif
+#ifndef BNGE_DFLT_HWRM_CMD_TIMEOUT
+#define BNGE_DFLT_HWRM_CMD_TIMEOUT 500
+#endif
+
+#define BNG_RE_MPC_SET_TX_OPAQUE(tx_napi_idx, idx, bds, tx_ring_mask) \
+ (((tx_napi_idx) << TX_OPAQUE_RING_SHIFT) | \
+ ((bds) << TX_OPAQUE_BDS_SHIFT) | ((idx) & (tx_ring_mask)))
+
+#define BNG_RE_MPC_TX_OPAQUE_IDX(opq) ((opq) & TX_OPAQUE_IDX_MASK)
+
+static inline void bng_re_roce_mpc_set_stall(struct bng_re_dev *rdev)
+{
+ dev_warn(rdev_to_dev(rdev), "%s: MPC ch marked stalled\n", __func__);
+ rdev->mpc_roce_creq.bm_mpc_stall = true;
+}
+
+struct bng_re_mpc_roce_creq_info *bng_re_mpc_roce_tx_ring(struct bng_re_dev *rdev);
+
+void bng_re_ulp_vf_qp_req(struct bng_re_dev *rdev, void *cmpl, u32 cmpl_len);
+void bng_re_mpc_process_vf_completion(struct bng_re_dev *rdev, void *cmpl, u32 cmpl_len);
+
+void bng_re_mpc_roce_set_stall(struct bng_re_dev *rdev);
+
+int bng_re_mpc_roce_alloc_init(struct bng_re_dev *rdev);
+
+void bng_re_mpc_roce_deinit_free(struct bng_re_dev *rdev);
+
+int bng_re_mpc_roce_service_creq(struct bng_re_dev *rdev, int budget);
+
+int bng_re_mpc_roce_service_cne(struct bng_re_dev *rdev, void *nq_cmp, int budget);
+
+int bng_re_mpc_roce_xmit(struct bng_re_dev *rdev, void *cmd, uint cmd_len,
+ void *cmpl, uint cmpl_len, bool poll_cmp);
+int bng_re_mpc_roce_diag_counters_debugfs_show(struct bng_re_dev *rdev,
+ struct seq_file *s);
+int bng_re_mpc_roce_perf_debugfs_show(struct bng_re_dev *rdev,
+ struct seq_file *s, u8 level);
+void bng_re_mpc_roce_perf_debugfs_clear(struct bng_re_dev *rdev);
+
+void bng_re_mpc_roce_diag_counters_clear(struct bng_re_dev *rdev);
+
+int bng_re_mpc_rings_alloc(struct bng_re_dev *rdev);
+void bng_re_mpc_rings_free(struct bng_re_dev *rdev);
+
+int bng_re_mpc_roce_alloc_stats(struct bng_re_dev *rdev);
+void bng_re_mpc_roce_free_stats(struct bng_re_dev *rdev);
+
+void bng_re_mpc_roce_add_perf_stats(struct bng_re_dev *rdev,
+ u8 req_type,
+ u8 req_subtype,
+ u32 latency_msec);
+
+int bng_re_mpc_roce_diag_counters_debugfs_show(struct bng_re_dev *rdev,
+ struct seq_file *s);
+
+void bnge_re_print_mpc_msg(struct bng_re_dev *rdev,
+ u8 *cmd,
+ uint cmd_len,
+ unsigned long tx_start_time,
+ unsigned long tx_end_time,
+ unsigned long poll_end_time,
+ int rc,
+ int retries_left);
+
+void dump_tx_cmpl_ring_info(struct bng_re_dev *rdev);
+
+void bng_re_mpc_handle_event_cmpl(struct bng_re_dev *rdev,
+ void *cmpl, u32 cmpl_len);
+
+void bng_re_mpc_rings_free(struct bng_re_dev *rdev);
+u32 bng_re_snapdump_mpc_stats(struct bng_re_dev *rdev, void *buf, u32 buf_len);
+void bng_re_mpc_process_vf_completion(struct bng_re_dev *rdev, void *cmpl, u32 cmpl_len);
+
+void bng_re_ulp_vf_qp_req(struct bng_re_dev *rdev, void *cmpl, u32 cmpl_len);
+
+void bng_re_roce_mpc_cmp(struct bng_re_dev *rdev, unsigned long handle,
+ struct bng_re_cmpl_entry *cmpl);
+int bng_re_roce_mpc_xmit(struct bng_re_dev *rdev, void *cmd, uint cmd_len,
+ void *cmpl, uint cmpl_len, bool poll_cmp);
+int bng_re_roce_mpc_diag_counters_debugfs_show(struct bng_re_dev *rdev,
+ struct seq_file *s);
+int bng_re_roce_mpc_perf_debugfs_show(struct bng_re_dev *rdev,
+ struct seq_file *s, u8 level);
+void bng_re_roce_mpc_perf_debugfs_clear(struct bng_re_dev *rdev);
+void bng_re_roce_mpc_diag_counters_clear(struct bng_re_dev *rdev);
+
+int bng_mpc_start_background_poll(struct bng_re_dev *rdev,
+ struct task_struct **new_thread);
+void bng_mpc_stop_background_poll(struct bng_re_dev *rdev,
+ struct task_struct **thread);
+
+int bng_alloc_init_mpc(struct bng_re_dev *rdev);
+int bng_deinit_mpc(struct bng_re_dev *rdev);
+void bng_mpc_stop_irq(struct bng_mpc_ctx *mpc, bool kill);
+
+int bng_re_alloc_mpc_doorbells(struct bng_re_dev *rdev);
+void bng_re_unalloc_mpc_doorbells(struct bng_re_dev *rdev);
+void bng_re_check_mpc_pending_empty(struct bng_re_dev *rdev);
+
+void bng_re_vf_pfvf_cmpl_wq_task(struct work_struct *work);
+
+#endif
diff --git a/drivers/infiniband/hw/bng_re/bng_res.c b/drivers/infiniband/hw/bng_re/bng_res.c
index f6e3528e7f4c..be6d5099aa35 100644
--- a/drivers/infiniband/hw/bng_re/bng_res.c
+++ b/drivers/infiniband/hw/bng_re/bng_res.c
@@ -8,6 +8,7 @@
#include <linux/bnge/hsi.h>
#include "bng_res.h"
#include "bng_roce_hsi.h"
+#include "bng_sp.h"
/* Stats */
void bng_re_free_stats_ctx_mem(struct pci_dev *pdev,
@@ -50,7 +51,7 @@ static void bng_free_pbl(struct bng_re_res *res, struct bng_re_pbl *pbl)
pbl->pg_map_arr[i]);
else
dev_warn(&pdev->dev,
- "PBL free pg_arr[%d] empty?!\n", i);
+ "PBL free pg_arr[%d] empty?!\n", i);
pbl->pg_arr[i] = NULL;
}
@@ -90,9 +91,9 @@ static int bng_alloc_pbl(struct bng_re_res *res,
for (i = 0; i < pages; i++) {
pbl->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
- pbl->pg_size,
- &pbl->pg_map_arr[i],
- GFP_KERNEL);
+ pbl->pg_size,
+ &pbl->pg_map_arr[i],
+ GFP_KERNEL);
if (!pbl->pg_arr[i])
goto fail;
pbl->pg_count++;
@@ -192,7 +193,7 @@ int bng_re_alloc_init_hwq(struct bng_re_hwq *hwq,
/* Alloc or init PTEs */
rc = bng_alloc_pbl(res, &hwq->pbl[BNG_PBL_LVL_2],
- hwq_attr->sginfo);
+ hwq_attr->sginfo);
if (rc)
goto fail;
hwq->level = BNG_PBL_LVL_2;
@@ -231,7 +232,7 @@ int bng_re_alloc_init_hwq(struct bng_re_hwq *hwq,
goto fail;
/* Alloc or init PTEs */
rc = bng_alloc_pbl(res, &hwq->pbl[BNG_PBL_LVL_1],
- hwq_attr->sginfo);
+ hwq_attr->sginfo);
if (rc)
goto fail;
hwq->level = BNG_PBL_LVL_1;
@@ -277,3 +278,104 @@ int bng_re_alloc_init_hwq(struct bng_re_hwq *hwq,
bng_re_free_hwq(res, hwq);
return -ENOMEM;
}
+
+void bng_res_unmap_db_bar(struct bng_re_res *res)
+{
+ struct bng_re_reg_desc *reg;
+
+ reg = &res->dpi_tbl.ucreg;
+ if (reg->bar_reg)
+ pci_iounmap(res->pdev, reg->bar_reg);
+ reg->bar_reg = NULL;
+ reg->bar_base = 0;
+ reg->len = 0;
+ reg->bar_id = 0;
+}
+
+int bng_res_map_db_bar(struct bng_re_res *res)
+{
+ struct bng_re_reg_desc *ucreg;
+ struct bng_re_reg_desc *wcreg;
+
+ wcreg = &res->dpi_tbl.wcreg;
+ wcreg->bar_id = RCFW_DBR_PCI_BAR_REGION;
+ wcreg->bar_base = pci_resource_start(res->pdev, wcreg->bar_id);
+
+ ucreg = &res->dpi_tbl.ucreg;
+ ucreg->bar_id = RCFW_DBR_PCI_BAR_REGION;
+ ucreg->bar_base = pci_resource_start(res->pdev, ucreg->bar_id);
+ ucreg->len = ucreg->offset + PAGE_SIZE;
+ if (!ucreg->len) {
+ dev_err(&res->pdev->dev, "invalid dbr length %d",
+ (int)ucreg->len);
+ return -EINVAL;
+ }
+ ucreg->bar_reg = ioremap(ucreg->bar_base, ucreg->len);
+ if (!ucreg->bar_reg) {
+ dev_err(&res->pdev->dev, "privileged dpi map failed!");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void bng_res_free_dpi_tbl(struct bng_re_dpi_tbl *dpit)
+{
+ kfree(dpit->tbl);
+ kfree(dpit->app_tbl);
+ dpit->tbl = NULL;
+ dpit->app_tbl = NULL;
+ dpit->max = 0;
+}
+
+static int bng_res_alloc_dpi_tbl(struct bng_re_res *res,
+ struct bng_re_dev_attr *dev_attr)
+{
+ struct bng_re_dpi_tbl *dpit;
+ struct bng_re_reg_desc *reg;
+ unsigned long bar_len;
+ u32 bytes;
+
+ dpit = &res->dpi_tbl;
+ reg = &dpit->wcreg;
+
+ bar_len = pci_resource_len(res->pdev, reg->bar_id);
+
+ dpit->max = (bar_len - reg->offset) / PAGE_SIZE;
+ if (dev_attr->max_dpi)
+ dpit->max = min_t(u32, dpit->max, dev_attr->max_dpi);
+
+ dpit->app_tbl = kcalloc(dpit->max, sizeof(void *), GFP_KERNEL);
+ if (!dpit->app_tbl)
+ return -ENOMEM;
+
+ bytes = (dpit->max + 7) >> 3;
+ dpit->tbl = kmalloc(bytes, GFP_KERNEL);
+ if (!dpit->tbl) {
+ kfree(dpit->app_tbl);
+ dpit->app_tbl = NULL;
+ dev_err(&res->pdev->dev,
+ "DPI tbl allocation failed for size = %d", bytes);
+ return -ENOMEM;
+ }
+
+ memset((u8 *)dpit->tbl, 0xFF, bytes);
+ dpit->priv_db = dpit->ucreg.bar_reg + dpit->ucreg.offset;
+ return 0;
+}
+
+void bng_res_free_tbls(struct bng_re_res *res)
+{
+ bng_res_free_dpi_tbl(&res->dpi_tbl);
+}
+
+int bng_res_alloc_init_tbls(struct bng_re_res *res)
+{
+ int rc;
+
+ rc = bng_res_alloc_dpi_tbl(res, res->dattr);
+ if (rc)
+ dev_err(&res->pdev->dev, "DPI tbl alloc failed\n");
+
+ return rc;
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_res.h b/drivers/infiniband/hw/bng_re/bng_res.h
index 2c4e9191ad1c..9745609190d9 100644
--- a/drivers/infiniband/hw/bng_re/bng_res.h
+++ b/drivers/infiniband/hw/bng_re/bng_res.h
@@ -5,6 +5,10 @@
#define __BNG_RES_H__
#include "bng_roce_hsi.h"
+#include "xid_allocator.h"
+#include <linux/bnge/hsi.h>
+
+#define BNG_NQE_MAX_CNT (128 * 1024)
#define BNG_ROCE_FW_MAX_TIMEOUT 60
@@ -14,10 +18,10 @@
#define PTR_IDX(x) ((x) & PTR_MAX_IDX_PER_PG)
#define HWQ_CMP(idx, hwq) ((idx) & ((hwq)->max_elements - 1))
-#define HWQ_FREE_SLOTS(hwq) (hwq->max_elements - \
- ((HWQ_CMP(hwq->prod, hwq)\
- - HWQ_CMP(hwq->cons, hwq))\
- & (hwq->max_elements - 1)))
+#define HWQ_FREE_SLOTS(hwq) ((hwq)->max_elements - \
+ ((HWQ_CMP((hwq)->prod, hwq)\
+ - HWQ_CMP((hwq)->cons, hwq))\
+ & ((hwq)->max_elements - 1)))
#define MAX_PBL_LVL_0_PGS 1
#define MAX_PBL_LVL_1_PGS 512
@@ -32,6 +36,8 @@
#define BNG_MAX_TQM_ALLOC_REQ 48
+#define RCFW_DBR_PCI_BAR_REGION 2
+
struct bng_re_reg_desc {
u8 bar_id;
resource_size_t bar_base;
@@ -43,11 +49,16 @@ struct bng_re_reg_desc {
struct bng_re_db_info {
void __iomem *db;
void __iomem *priv_db;
+ void *dbc; /* HDBR registration handle (optional) */
struct bng_re_hwq *hwq;
u32 xid;
+ u32 seed;
u32 max_slot;
u32 flags;
u8 toggle;
+ spinlock_t lock; /* protects concurrent doorbell ring updates */
+ struct bng_re_res *res;
+ bool is_l2;
};
enum bng_re_db_info_flags_mask {
@@ -57,16 +68,29 @@ enum bng_re_db_info_flags_mask {
BNG_RE_FLAG_EPOCH_PROD_MASK = 0x2UL,
};
+#define BNG_QP_FLAG_EPOCH_CONS_MASK BNG_RE_FLAG_EPOCH_CONS_MASK
+
enum bng_re_db_epoch_flag_shift {
BNG_RE_DB_EPOCH_CONS_SHIFT = BNG_RE_DBR_EPOCH_SHIFT,
BNG_RE_DB_EPOCH_PROD_SHIFT = (BNG_RE_DBR_EPOCH_SHIFT - 1),
};
+struct bng_re_drv_modes {
+ u8 wqe_mode;
+ bool db_push;
+ bool dbr_pacing;
+ u32 toggle_bits;
+ u8 roce_mirror;
+ u8 dbr_primary_pf;
+ bool st_tag_supported;
+};
+
struct bng_re_chip_ctx {
u16 chip_num;
u16 hw_stats_size;
u64 hwrm_intf_ver;
u16 hwrm_cmd_max_timeout;
+ struct bng_re_drv_modes modes;
};
struct bng_re_pbl {
@@ -132,10 +156,23 @@ struct bng_re_stats {
u32 fw_id;
};
+/* DPI table */
+struct bng_re_dpi_tbl {
+ void **app_tbl;
+ unsigned long *tbl;
+ u16 max;
+ struct bng_re_reg_desc ucreg; /* Hold entire DB bar. */
+ struct bng_re_reg_desc wcreg;
+ void __iomem *priv_db;
+};
+
struct bng_re_res {
struct pci_dev *pdev;
struct bng_re_chip_ctx *cctx;
struct bng_re_dev_attr *dattr;
+ struct bng_re_dpi_tbl dpi_tbl;
+ struct xid_manager *qp_xids;
+ struct xid_manager *ah_xids;
};
static inline void *bng_re_get_qe(struct bng_re_hwq *hwq,
@@ -195,6 +232,34 @@ static inline void bng_re_hwq_incr_cons(u32 max_elements, u32 *cons, u32 cnt,
}
}
+static inline void bng_hwq_incr_prod(struct bng_re_db_info *dbinfo,
+ struct bng_re_hwq *hwq, u32 cnt)
+{
+ /* move prod and update toggle/epoch if wrap around */
+ hwq->prod += cnt;
+ if (hwq->prod >= hwq->depth) {
+ hwq->prod %= hwq->depth;
+ dbinfo->flags ^= 1UL << BNG_RE_FLAG_EPOCH_PROD_SHIFT;
+ }
+}
+
+static inline void bng_ring_prod_db(struct bng_re_db_info *info, u32 type)
+{
+ u64 key = 0;
+ u32 indx;
+
+ indx = (((info->hwq->prod / info->max_slot) & DBC_DBC_INDEX_MASK) |
+ ((info->flags & BNG_RE_FLAG_EPOCH_PROD_MASK) <<
+ BNG_RE_DB_EPOCH_PROD_SHIFT));
+
+ key = BNG_RE_INIT_DBHDR(info->xid, type, indx, 0);
+
+ if (info->is_l2)
+ key |= DBC_DBC64_PATH_L2;
+
+ writeq(key, info->db);
+}
+
static inline bool _is_max_srq_ext_supported(u16 dev_cap_ext_flags_2)
{
return !!(dev_cap_ext_flags_2 & CREQ_QUERY_FUNC_RESP_SB_MAX_SRQ_EXTENDED);
@@ -212,4 +277,9 @@ void bng_re_free_stats_ctx_mem(struct pci_dev *pdev,
int bng_re_alloc_stats_ctx_mem(struct pci_dev *pdev,
struct bng_re_chip_ctx *cctx,
struct bng_re_stats *stats);
+
+void bng_res_unmap_db_bar(struct bng_re_res *res);
+int bng_res_map_db_bar(struct bng_re_res *res);
+void bng_res_free_tbls(struct bng_re_res *res);
+int bng_res_alloc_init_tbls(struct bng_re_res *res);
#endif
diff --git a/drivers/infiniband/hw/bng_re/bng_roce_hsi.h b/drivers/infiniband/hw/bng_re/bng_roce_hsi.h
index 5ebd7ba90b7b..6c9d45464ef6 100644
--- a/drivers/infiniband/hw/bng_re/bng_roce_hsi.h
+++ b/drivers/infiniband/hw/bng_re/bng_roce_hsi.h
@@ -6447,4 +6447,98 @@ struct ptu_pde {
#define PTU_PDE_PAGE_SFT 12
};
+/* mpc_qp_modify_cmpl (size:256b/32B) */
+struct mpc_qp_modify_cmpl {
+ u8 cmpl_type_reserved;
+ #define MPC_QP_MODIFY_CMPL_CMPL_TYPE_RESERVED_CMPL_TYPE_MASK 0x3fUL
+ #define MPC_QP_MODIFY_CMPL_CMPL_TYPE_RESERVED_CMPL_TYPE_SFT 0
+ #define MPC_QP_MODIFY_CMPL_CMPL_TYPE_RESERVED_CMPL_TYPE_MPC_CMP_SHORT 0x1eUL
+ #define MPC_QP_MODIFY_CMPL_CMPL_TYPE_RESERVED_CMPL_TYPE_MPC_CMP_LONG 0x1fUL
+ #define MPC_QP_MODIFY_CMPL_CMPL_TYPE_RESERVED_CMPL_TYPE_LAST \
+ MPC_QP_MODIFY_CMPL_CMPL_TYPE_RESERVED_CMPL_TYPE_MPC_CMP_LONG
+ u8 error_code_mp_client;
+ #define MPC_QP_MODIFY_CMPL_ERROR_CODE_MP_CLIENT_ERROR_CODE_MASK 0xfUL
+ #define MPC_QP_MODIFY_CMPL_ERROR_CODE_MP_CLIENT_ERROR_CODE_SFT 0
+ #define MPC_QP_MODIFY_CMPL_ERROR_CODE_MP_CLIENT_MP_CLIENT_MASK 0xf0UL
+ #define MPC_QP_MODIFY_CMPL_ERROR_CODE_MP_CLIENT_MP_CLIENT_SFT 4
+ u8 req_type;
+ u8 req_subtype;
+ __le32 opaque;
+ u8 v;
+ #define MPC_QP_MODIFY_CMPL_V 0x1UL
+ u8 reserved_0[7];
+ __le32 xid;
+ __le16 error_code;
+ #define MPC_QP_MODIFY_CMPL_ERROR_CODE_SUCCESS 0x0UL
+ #define MPC_QP_MODIFY_CMPL_ERROR_CODE_FAIL 0x1UL
+ #define MPC_QP_MODIFY_CMPL_ERROR_CODE_RESOURCES 0x2UL
+ #define MPC_QP_MODIFY_CMPL_ERROR_CODE_INVALID_CMD 0x3UL
+ #define MPC_QP_MODIFY_CMPL_ERROR_CODE_NOT_IMPLEMENTED 0x4UL
+ #define MPC_QP_MODIFY_CMPL_ERROR_CODE_INVALID_PARAMETER 0x5UL
+ #define MPC_QP_MODIFY_CMPL_ERROR_CODE_HARDWARE_ERROR 0x6UL
+ #define MPC_QP_MODIFY_CMPL_ERROR_CODE_INTERNAL_ERROR 0x7UL
+ #define MPC_QP_MODIFY_CMPL_ERROR_CODE_LAST \
+ MPC_QP_MODIFY_CMPL_ERROR_CODE_INTERNAL_ERROR
+ u8 pingpong_push_state_index_enabled;
+ #define MPC_QP_MODIFY_CMPL_PINGPONG_PUSH_ENABLED 0x1UL
+ #define MPC_QP_MODIFY_CMPL_PINGPONG_PUSH_INDEX_MASK 0xeUL
+ #define MPC_QP_MODIFY_CMPL_PINGPONG_PUSH_INDEX_SFT 1
+ #define MPC_QP_MODIFY_CMPL_PINGPONG_PUSH_STATE 0x10UL
+ u8 flags;
+ #define MPC_QP_MODIFY_CMPL_FLAGS_FLAGS_MASK 0xffUL
+ #define MPC_QP_MODIFY_CMPL_FLAGS_FLAGS_SFT 0
+ #define MPC_QP_MODIFY_CMPL_FLAGS_FLAGS_QP_MD_COMPLETE 0x0UL
+ #define MPC_QP_MODIFY_CMPL_FLAGS_FLAGS_QP_MD_PENDING 0x1UL
+ #define MPC_QP_MODIFY_CMPL_FLAGS_FLAGS_LAST \
+ MPC_QP_MODIFY_CMPL_FLAGS_FLAGS_QP_MD_PENDING
+ u8 v2;
+ #define MPC_QP_MODIFY_CMPL_V2 0x1UL
+ u8 reserved_3[3];
+ __le32 lag_src_mac;
+};
+
+/* mpc_event_resp_cmpl (size:256b/32B) */
+struct mpc_event_resp_cmpl {
+ u8 cmpl_type_reserved;
+ #define MPC_EVENT_RESP_CMPL_CMPL_TYPE_RESERVED_CMPL_TYPE_MASK 0x3fUL
+ #define MPC_EVENT_RESP_CMPL_CMPL_TYPE_RESERVED_CMPL_TYPE_SFT 0
+ #define MPC_EVENT_RESP_CMPL_CMPL_TYPE_RESERVED_CMPL_TYPE_MPC_CMP_SHORT 0x1eUL
+ #define MPC_EVENT_RESP_CMPL_CMPL_TYPE_RESERVED_CMPL_TYPE_MPC_CMP_LONG 0x1fUL
+ #define MPC_EVENT_RESP_CMPL_CMPL_TYPE_RESERVED_CMPL_TYPE_LAST \
+ MPC_EVENT_RESP_CMPL_CMPL_TYPE_RESERVED_CMPL_TYPE_MPC_CMP_LONG
+ u8 error_code_mp_client;
+ #define MPC_EVENT_RESP_CMPL_ERROR_CODE_MP_CLIENT_ERROR_CODE_MASK 0xfUL
+ #define MPC_EVENT_RESP_CMPL_ERROR_CODE_MP_CLIENT_ERROR_CODE_SFT 0
+ #define MPC_EVENT_RESP_CMPL_ERROR_CODE_MP_CLIENT_MP_CLIENT_MASK 0xf0UL
+ #define MPC_EVENT_RESP_CMPL_ERROR_CODE_MP_CLIENT_MP_CLIENT_SFT 4
+ u8 req_type;
+ u8 req_subtype;
+ __le32 opaque;
+ u8 v;
+ #define MPC_EVENT_RESP_CMPL_V 0x1UL
+ u8 reserved_0[1];
+ __le16 error_code;
+ #define MPC_EVENT_RESP_CMPL_ERROR_CODE_SUCCESS 0x0UL
+ #define MPC_EVENT_RESP_CMPL_ERROR_CODE_FAIL 0x1UL
+ #define MPC_EVENT_RESP_CMPL_ERROR_CODE_LAST MPC_EVENT_RESP_CMPL_ERROR_CODE_FAIL
+ __le32 xid;
+ __le32 event_type;
+ #define MPC_EVENT_RESP_CMPL_EVENT_TYPE_EVENT_TYPE_MASK 0xffffffffUL
+ #define MPC_EVENT_RESP_CMPL_EVENT_TYPE_EVENT_TYPE_SFT 0
+ #define MPC_EVENT_RESP_CMPL_EVENT_TYPE_EVENT_TYPE_QP_EVENT 0x1UL
+ #define MPC_EVENT_RESP_CMPL_EVENT_TYPE_EVENT_TYPE_LAST \
+ MPC_EVENT_RESP_CMPL_EVENT_TYPE_EVENT_TYPE_QP_EVENT
+ __le32 event_data;
+ #define MPC_EVENT_RESP_CMPL_EVENT_DATA_EVENT_TYPE_MASK 0xffffffffUL
+ #define MPC_EVENT_RESP_CMPL_EVENT_DATA_EVENT_TYPE_SFT 0
+ #define MPC_EVENT_RESP_CMPL_EVENT_DATA_EVENT_TYPE_FASTQP_MD_COMPLETE 0x0UL
+ #define MPC_EVENT_RESP_CMPL_EVENT_DATA_EVENT_TYPE_FASTQP_MD_PENDING 0x1UL
+ #define MPC_EVENT_RESP_CMPL_EVENT_DATA_EVENT_TYPE_LAST \
+ MPC_EVENT_RESP_CMPL_EVENT_DATA_EVENT_TYPE_FASTQP_MD_PENDING
+ u8 v2;
+ #define MPC_EVENT_RESP_CMPL_V2 0x1UL
+ u8 reserved_1[3];
+ __le32 resp_data;
+};
+
#endif /* _BNG_RE_HSI_H_ */
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.h b/drivers/infiniband/hw/bng_re/bng_sp.h
index e15190515ed1..91faa5ac7464 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.h
+++ b/drivers/infiniband/hw/bng_re/bng_sp.h
@@ -9,6 +9,10 @@
#define BNG_VAR_MAX_WQE 4352
#define BNG_VAR_MAX_SGE 13
+#define NQE_CMP_VALID(hdr, pass) \
+ (!!(le32_to_cpu((hdr)->info63_v[0]) & NQ_BASE_V) == \
+ !((pass) & BNG_RE_FLAG_EPOCH_CONS_MASK))
+
struct bng_re_dev_attr {
#define FW_VER_ARR_LEN 4
u8 fw_ver[FW_VER_ARR_LEN];
diff --git a/drivers/infiniband/hw/bng_re/bng_xid.c b/drivers/infiniband/hw/bng_re/bng_xid.c
new file mode 100644
index 000000000000..6070183d9120
--- /dev/null
+++ b/drivers/infiniband/hw/bng_re/bng_xid.c
@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2025 Broadcom.
+
+#include <linux/bitops.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+
+#include "bng_re.h"
+#include "bng_res.h"
+#include "bng_sp.h"
+#include "bng_xid.h"
+#include "bng_fw.h"
+
+int bng_re_qp_alloc_xid_and_iqm(struct bng_re_res *res, u8 qp_type,
+ int num_xids, u32 max_wqe, int vf_id,
+ void *iqm_res, int *xid)
+{
+ dev_dbg(&res->pdev->dev,
+ "XID+IQM: enter qp_type=%u num_xids=%d max_wqe=%u vf_id=%d qp_xids=%p\n",
+ qp_type, num_xids, max_wqe, vf_id, res->qp_xids);
+
+ if (qp_type == CMDQ_CREATE_QP_TYPE_GSI) {
+ *xid = 1;
+ dev_dbg(&res->pdev->dev, "XID+IQM: GSI fixed xid=%d\n", *xid);
+ return 0;
+ }
+ if (qp_type == CMDQ_CREATE_QP_TYPE_RC) {
+ /* IQM not implemented yet */
+ return -EOPNOTSUPP;
+ }
+ if (!res->qp_xids) {
+ dev_dbg(&res->pdev->dev,
+ "XID+IQM: no driver qp_xids pool -> ENODEV\n");
+ return -ENODEV;
+ }
+ *xid = bng_re_xm_alloc_range(res->qp_xids, num_xids, vf_id);
+ if (*xid < 0) {
+ dev_dbg(&res->pdev->dev,
+ "XID+IQM: xm_alloc_range failed ret=%d num_xids=%d vf_id=%d\n",
+ *xid, num_xids, vf_id);
+ {
+ struct xid_public_stats st;
+
+ bng_re_xm_get_stats(res->qp_xids, &st);
+ dev_warn(&res->pdev->dev,
+ "QP xid allocation failed, curr_active=%d max_active=%d alloc_ids=%llu\n",
+ bng_re_xm_curr_active_get(res->qp_xids),
+ bng_re_xm_max_active_get(res->qp_xids),
+ st.xp_current_alloc_ids);
+ }
+ return -ENOMEM;
+ }
+ dev_dbg(&res->pdev->dev, "XID+IQM: xm_alloc ok xid=%d\n", *xid);
+ return 0;
+}
+
+void bng_re_qp_free_xid_and_iqm(struct bng_re_res *res, u8 qp_type, int xid,
+ void *iqm_res, bool free_qp_xid)
+{
+ if (qp_type == CMDQ_CREATE_QP_TYPE_GSI)
+ return;
+
+ if (free_qp_xid && res->qp_xids)
+ bng_re_xm_free_range(res->qp_xids, xid, true);
+}
+
+int bng_re_init_xid_tables(struct bng_re_dev *rdev)
+{
+ struct bng_re_res *res = &rdev->bng_res;
+ int qp_start = 2;
+ int qp_num_ids;
+ int qp_max;
+ int rc;
+
+ qp_max = (int)rdev->dev_attr->max_qp;
+ if (qp_max < 1)
+ return -EINVAL;
+ qp_num_ids = qp_max + 1024;
+
+ res->qp_xids = bng_re_xm_init(qp_start, qp_num_ids, qp_max, BNG_RE_XID_AVOID_REUSE);
+ if (!res->qp_xids)
+ return -ENOMEM;
+
+ res->ah_xids = bng_re_xm_init(0, (int)rdev->dev_attr->max_ah,
+ (int)rdev->dev_attr->max_ah,
+ BNG_RE_XID_AVOID_REUSE);
+ if (!res->ah_xids) {
+ rc = -ENOMEM;
+ goto free_qp_xm;
+ }
+
+ return 0;
+
+free_qp_xm:
+ bng_re_xm_destroy(res->qp_xids);
+ res->qp_xids = NULL;
+ return rc;
+}
+
+void bng_re_free_xid_tables(struct bng_re_res *res)
+{
+ if (res->ah_xids) {
+ bng_re_xm_destroy(res->ah_xids);
+ res->ah_xids = NULL;
+ }
+ if (res->qp_xids) {
+ bng_re_xm_destroy(res->qp_xids);
+ res->qp_xids = NULL;
+ }
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_xid.h b/drivers/infiniband/hw/bng_re/bng_xid.h
new file mode 100644
index 000000000000..f76babb546cf
--- /dev/null
+++ b/drivers/infiniband/hw/bng_re/bng_xid.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2025 Broadcom. */
+
+#ifndef __BNG_XID_H__
+#define __BNG_XID_H__
+
+#include <linux/kernel.h>
+
+#include "bng_roce_hsi.h"
+#include "xid_allocator.h"
+
+struct bng_re_dev;
+struct bng_re_res;
+
+#define ROUND_UP_SIZE_BNG(n, sz) round_up(n, sz)
+
+int bng_re_qp_alloc_xid_and_iqm(struct bng_re_res *res, u8 qp_type,
+ int num_xids, u32 max_wqe, int vf_id,
+ void *iqm_res, int *xid);
+void bng_re_qp_free_xid_and_iqm(struct bng_re_res *res, u8 qp_type, int xid,
+ void *iqm_res, bool free_qp_xid);
+
+int bng_re_init_xid_tables(struct bng_re_dev *rdev);
+void bng_re_free_xid_tables(struct bng_re_res *res);
+
+#endif /* __BNG_XID_H__ */
diff --git a/drivers/infiniband/hw/bng_re/bng_xid_allocator.c b/drivers/infiniband/hw/bng_re/bng_xid_allocator.c
new file mode 100644
index 000000000000..41dfb83a0a92
--- /dev/null
+++ b/drivers/infiniband/hw/bng_re/bng_xid_allocator.c
@@ -0,0 +1,870 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2025 Broadcom.
+
+#include "xid_allocator.h"
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/stddef.h>
+/* #include <string.h> - required for standalone compile */
+
+#define DEBUG_XID (g_debug_xid)
+
+int g_debug_xid;
+
+static void bng_re_xm_consolidate_ranges(struct xid_manager *xm,
+ struct xid_node_info *prev,
+ struct rb_root *tree
+);
+
+static int bng_re_xm_trans_range_by_pointer(struct xid_manager *xm, struct xid_node_info *nd);
+
+/**
+ * bng_re_xm_rb_insert - common fn for inserting range into rb tree
+ *
+ * @param root - root of rb tree to insert into
+ * @param data - range to insert
+ * @param fn - function for compare
+ * @param return - true for success, false for an issue (duplicate key)
+ */
+static int bng_re_xm_rb_insert(struct rb_root *root, struct xid_node_info *data,
+ rb_compare_fn_t fn)
+{
+ struct rb_node **new = &root->rb_node, *parent = NULL;
+
+ /* Figure out where to put new node */
+ while (*new) {
+ struct xid_node_info *this = container_of(*new, struct xid_node_info, node);
+ int result = fn(data, this);
+
+ parent = *new;
+ if (result < 0)
+ new = &((*new)->rb_left);
+ else if (result > 0)
+ new = &((*new)->rb_right);
+ else
+ return false;
+ }
+
+ /* Add new node and rebalance tree. */
+ rb_link_node(&data->node, parent, new);
+ rb_insert_color(&data->node, root);
+
+ return true;
+}
+
+static int bng_re_xi_free_compare(void *left, void *right)
+{
+ struct xid_node_info *lp = (struct xid_node_info *)left;
+ struct xid_node_info *rp = (struct xid_node_info *)right;
+
+ if (lp->xi_num_ids != rp->xi_num_ids)
+ return lp->xi_num_ids - rp->xi_num_ids;
+
+ return lp->xi_starting_id - rp->xi_starting_id;
+}
+
+static int bng_re_xi_alloced_compare(void *left, void *right)
+{
+ struct xid_node_info *lp = (struct xid_node_info *)left;
+ struct xid_node_info *rp = (struct xid_node_info *)right;
+
+ return lp->xi_starting_id - rp->xi_starting_id;
+}
+
+/**************************************************
+ * util fns for the id based tree *
+ **************************************************/
+
+/**
+ * bng_re_xm_insert_by_id - insert info into xid based tree. Info is a range
+ *of available ids
+ *
+ * @param root - root of rb tree to insert into
+ * @param data - range to insert
+ * @param return - 0 for success, -1 for error
+ */
+static int bng_re_xm_insert_by_id(struct rb_root *root,
+ struct xid_node_info *data)
+{
+ return bng_re_xm_rb_insert(root, data, bng_re_xi_alloced_compare);
+}
+
+/**
+ * bng_re_xm_search_by_id - search xid based tree.
+ *
+ * @param root - root of rb tree to search
+ * @param xid - xid to search
+ * @param return - range which was found; or NULL
+ */
+static struct xid_node_info *bng_re_xm_search_by_id(struct rb_root *root, int xid)
+{
+ struct rb_node *node = root->rb_node;
+
+ while (node) {
+ struct xid_node_info *data = container_of(node, struct xid_node_info, node);
+ int result;
+
+ result = xid - data->xi_starting_id;
+
+ if (result < 0)
+ node = node->rb_left;
+ else if (result > 0)
+ node = node->rb_right;
+ else
+ return data;
+ }
+ return NULL;
+}
+
+/**************************************************
+ * util fns for the free range based tree *
+ **************************************************/
+
+/**
+ * bng_re_xm_insert_by_free_range - insert info into range based tree.
+ * Info is a range of ids. Order by
+ *range size.
+ *
+ * @param root - root of rb tree to insert into
+ * @param data - range to insert
+ * @return - 0 for success, -1 for error
+ */
+static int bng_re_xm_insert_by_free_range(struct rb_root *root,
+ struct xid_node_info *data)
+{
+ return bng_re_xm_rb_insert(root, data, bng_re_xi_free_compare);
+}
+
+/**
+ * bng_re_xm_search_best_free - look for best range for requested number of
+ * ids.
+ *
+ * @param root - root of rb tree to insert into
+ * @param ids_required - number of ids needed
+ * @return - best matching range
+ */
+static struct xid_node_info *bng_re_xm_search_best_free(struct rb_root *root,
+ int ids_required)
+{
+ struct rb_node *node = root->rb_node;
+ struct xid_node_info *best = NULL;
+
+ while (node) {
+ struct xid_node_info *data = container_of(node, struct xid_node_info, node);
+
+ if (data->xi_num_ids == ids_required) {
+ best = data;
+ break;
+ } else if (data->xi_num_ids > ids_required) {
+ best = data;
+ node = node->rb_left;
+ } else {
+ node = node->rb_right;
+ }
+ }
+ return best;
+}
+
+/**
+ * bng_re_xm_reset_pending_ranges -
+ *
+ * @param root - root of rb tree to insert into
+ */
+static int bng_re_xm_reset_pending_ranges(struct xid_manager *xm)
+{
+ struct rb_root *root = &xm->xm_pending_tree;
+ struct rb_node *node = root->rb_node;
+ struct xid_node_info *data = NULL;
+ int num_resets = 0;
+ /* pending => free tree */
+ while (node) {
+ data = container_of(node, struct xid_node_info, node);
+ num_resets++;
+ bng_re_xm_trans_range_by_pointer(xm, data);
+ node = root->rb_node;
+ }
+ return num_resets;
+}
+
+/**
+ * bng_re_xm_search_best_free_wrapper - look for best range for requested number of
+ *ids.
+ *
+ * @param root - root of rb tree to insert into
+ * @param ids_required - number of ids needed
+ * @param best range or NULL
+ */
+static struct xid_node_info *bng_re_xm_search_best_free_wrapper(struct xid_manager *xm,
+ int ids_required)
+{
+ struct xid_node_info *best = NULL;
+ int num_resets;
+
+ best = bng_re_xm_search_best_free(&xm->xm_free_tree, ids_required);
+ if (!best) {
+ num_resets = bng_re_xm_reset_pending_ranges(xm);
+ if (num_resets > 0)
+ best = bng_re_xm_search_best_free(&xm->xm_free_tree, ids_required);
+ }
+ return best;
+}
+
+/* for all trees - common removal function */
+static void bng_re_xm_del_from_tree(struct rb_root *root,
+ struct xid_node_info *data)
+{
+ if (data)
+ rb_erase(&data->node, root);
+}
+
+/**************************************************
+ * util fns for the xid_node_info *
+ **************************************************/
+
+static struct xid_node_info *bng_re_xm_tree_info_alloc(void)
+{
+ struct xid_node_info *xi = (struct xid_node_info *)CALLOC(1, sizeof(struct xid_node_info));
+ return xi;
+}
+
+static void bng_re_xm_tree_info_del(struct xid_node_info *xi)
+{
+ memset(xi, 0xa5, sizeof(*xi));
+ FREE(xi);
+}
+
+/**
+ * bng_re_xm_init - initialize xid manager structure
+ *
+ * @param start_id - starting id
+ * @param max_ids - size of id pool
+ * @param max_active_ids - cap on active IDs (caller passes e.g. max_ids for AH/IQM)
+ * @param avoid_reuse - whether or not to avoid immediate xid reuse
+ *
+ * @return - xid manager, or NULL on failure (e.g. max_active_ids > max_ids)
+ */
+struct xid_manager *bng_re_xm_init(int start_id, int max_ids, int max_active_ids,
+ bool avoid_reuse)
+{
+ struct xid_manager *p = CALLOC(1, sizeof(struct xid_manager));
+ struct xid_node_info *xi = NULL;
+
+ if (!p)
+ return NULL;
+ if (max_active_ids > max_ids)
+ goto err;
+ spin_lock_init(&p->xm_lock);
+ p->xm_free_tree = RB_ROOT;
+ p->xm_allocated_tree = RB_ROOT;
+ p->xm_pending_tree = RB_ROOT;
+ p->xm_num_ids = max_ids;
+ p->xm_starting_id = start_id;
+ p->xm_max_active_ids = max_active_ids;
+ p->xm_curr_active_ids = 0;
+ /* Add initial range to the free tree */
+ xi = bng_re_xm_tree_info_alloc();
+ xi->xi_starting_id = start_id;
+ xi->xi_num_ids = max_ids;
+ xi->xi_state = X_FREED;
+ bng_re_xm_insert_by_free_range(&p->xm_free_tree, xi);
+ p->xm_head = xi;
+ p->xm_avoid_reuse = avoid_reuse;
+ /* Initialize current counters - start with all IDs in free state */
+ p->xm_stats.xs_current_free_ids = max_ids;
+ p->xm_stats.xs_current_alloc_ids = 0;
+ p->xm_stats.xs_current_pending_free_ids = 0;
+ p->xm_stats.xs_max_alloc_ids = 0;
+ return p;
+err:
+ FREE(p);
+ return NULL;
+}
+
+static struct xid_node_info *bng_re_xm_find_best_free_range(struct xid_manager *xm,
+ int num_ids)
+{
+ struct xid_node_info *best = NULL;
+
+ best = bng_re_xm_search_best_free_wrapper(xm, num_ids);
+
+ return best;
+}
+
+/**
+ * bng_re_xm_alloc_range - Allocate a range from the free pool
+ *
+ * @param xm -
+ * @param num_ids - number of ids required
+ * @param tag - tag for the allocation
+ *
+ * @return - starting id for the range, or negative on failure
+ */
+int bng_re_xm_alloc_range(struct xid_manager *xm, int num_ids, int tag)
+{
+ struct xid_node_info *xi_alloced;
+ struct xid_node_info *xi_free;
+ int starting_id = -1;
+ unsigned long flags;
+
+ spin_lock_irqsave(&xm->xm_lock, flags);
+ if (xm->xm_curr_active_ids >= xm->xm_max_active_ids) {
+ xm->xm_stats.xs_alloc_err_active_limit_ctr++;
+ goto exit;
+ }
+ xi_free = bng_re_xm_find_best_free_range(xm, num_ids);
+
+ if (!xi_free) {
+ /* couldnt find that joint */
+ xm->xm_stats.xs_alloc_err_ctr++;
+ goto exit;
+ }
+
+ /*remove it from the tree because; after we change the num ids
+ * it has to be reinserted into the optimal position
+ */
+ bng_re_xm_del_from_tree(&xm->xm_free_tree, xi_free);
+
+ if (num_ids == xi_free->xi_num_ids) {
+ /* an exact fit. Just move it to the allocated tree */
+ xi_free->xi_state = X_ALLOCED;
+ bng_re_xm_insert_by_id(&xm->xm_allocated_tree, xi_free);
+ starting_id = xi_free->xi_starting_id;
+ xi_free->xi_tag = tag;
+ goto count_exit;
+ }
+ /* not exact match. need a range split */
+ xi_alloced = bng_re_xm_tree_info_alloc();
+ *xi_alloced = *xi_free;
+ xi_alloced->xi_next = xi_free;
+ xi_alloced->xi_num_ids = num_ids;
+ xi_alloced->xi_state = X_ALLOCED;
+ xi_alloced->xi_tag = tag;
+ if (xi_free->xi_prev)
+ xi_free->xi_prev->xi_next = xi_alloced;
+ /* We copied over the other fields already : starting id, prev ptr */
+ bng_re_xm_insert_by_id(&xm->xm_allocated_tree, xi_alloced);
+
+ /* adjust the range of the free entries */
+ xi_free->xi_prev = xi_alloced;
+ xi_free->xi_starting_id += num_ids;
+ xi_free->xi_num_ids -= num_ids;
+ /* Now reinsert into the free tree */
+ bng_re_xm_insert_by_free_range(&xm->xm_free_tree, xi_free);
+ if (xm->xm_head == xi_free) {
+ /*newly alloced entry may be the first one */
+ xm->xm_head = xi_alloced;
+ }
+ starting_id = xi_alloced->xi_starting_id;
+count_exit:
+ /* Update current counters: move IDs to allocated tree */
+ xm->xm_stats.xs_current_free_ids -= num_ids;
+ xm->xm_stats.xs_current_alloc_ids += num_ids;
+ xm->xm_curr_active_ids++;
+ /* Update maximum allocated IDs if we reached a new high */
+ if (xm->xm_stats.xs_current_alloc_ids > xm->xm_stats.xs_max_alloc_ids)
+ xm->xm_stats.xs_max_alloc_ids = xm->xm_stats.xs_current_alloc_ids;
+ xm->xm_stats.xs_alloc_op_ctr++;
+ xm->xm_stats.xs_alloc_count_ctr += num_ids;
+exit:
+ spin_unlock_irqrestore(&xm->xm_lock, flags);
+ return starting_id;
+}
+
+/**
+ * bng_re_xm_free_range - find a range by id and free it, optionally
+ * decrementing the active-id count.
+ *
+ * @param xm - xid manager
+ * @param id - id of range to free, or BNG_RE_XID_DECR_ONLY for decr only
+ * @param decr_active - if true, decrement curr_active_ids; for normal id
+ * also remove from tree. For BNG_RE_XID_DECR_ONLY must be true.
+ * @return - 0 for success, negative on error
+ */
+int bng_re_xm_free_range(struct xid_manager *xm, int id, bool decr_active)
+{
+ struct xid_node_info *xi_alloced;
+ unsigned long flags;
+ int rc = -1;
+ int num_ids;
+
+ spin_lock_irqsave(&xm->xm_lock, flags);
+
+ /* dump_stack(); */
+ if (id == BNG_RE_XID_DECR_ONLY) {
+ if (!decr_active) {
+ /* spec: log error */
+ XID_PRINT("xid free: BNG_RE_XID_DECR_ONLY with decr_active false\n");
+ goto exit;
+ }
+ xm->xm_curr_active_ids--;
+ xm->xm_stats.xs_free_decr_only++;
+ rc = 0;
+ goto exit;
+ }
+
+ xi_alloced = bng_re_xm_search_by_id(&xm->xm_allocated_tree, id);
+ if (!xi_alloced) {
+ xm->xm_stats.xs_free_err_ctr++;
+ goto exit;
+ }
+ num_ids = xi_alloced->xi_num_ids;
+ bng_re_xm_trans_range_by_pointer(xm, xi_alloced);
+ xm->xm_stats.xs_free_op_ctr++;
+ xm->xm_stats.xs_free_count_ctr += num_ids;
+ if (decr_active) {
+ xm->xm_curr_active_ids--;
+ xm->xm_stats.xs_free_decr_and_remove++;
+ } else {
+ xm->xm_stats.xs_free_remove_only++;
+ }
+ rc = 0;
+exit:
+ spin_unlock_irqrestore(&xm->xm_lock, flags);
+ return rc;
+}
+
+#ifdef STANDALONE_BUILD
+struct xid_node_info *bng_re_xm_find_allocated(struct xid_manager *xm, int id)
+{
+ struct xid_node_info *n;
+
+ n = bng_re_xm_search_by_id(&xm->xm_allocated_tree, id);
+ return n;
+}
+#endif
+
+bool bng_re_xm_is_id_allocated(struct xid_manager *xm, int id)
+{
+ struct xid_node_info *n = NULL;
+
+ n = bng_re_xm_search_by_id(&xm->xm_allocated_tree, id);
+
+ return n ? true : false;
+}
+
+/**
+ * bng_re_xm_trans_range_by_pointer - trans range to next tree
+ *
+ * @param xm -
+ * @param struct xid_node_info * - node to free
+ * @param new_state = new state to go to
+ * @return - 0 for success
+ */
+static int bng_re_xm_trans_range_by_pointer(struct xid_manager *xm, struct xid_node_info *nd)
+{
+ struct rb_root *curr_tree;
+ struct rb_root *new_tree;
+ int target_state;
+
+ if (xm->xm_avoid_reuse) {
+ if (nd->xi_state == X_ALLOCED) {
+ curr_tree = &xm->xm_allocated_tree;
+ new_tree = &xm->xm_pending_tree;
+ target_state = X_FREE_PENDING;
+ } else if (nd->xi_state == X_FREE_PENDING) {
+ curr_tree = &xm->xm_pending_tree;
+ new_tree = &xm->xm_free_tree;
+ target_state = X_FREED;
+ } else {
+ return -1;
+ }
+ } else {
+ if (nd->xi_state == X_ALLOCED) {
+ curr_tree = &xm->xm_allocated_tree;
+ new_tree = &xm->xm_free_tree;
+ target_state = X_FREED;
+ } else {
+ return -1;
+ }
+ }
+
+ bng_re_xm_del_from_tree(curr_tree, nd);
+
+ /* Update current counters based on state transition */
+ if (nd->xi_state == X_ALLOCED && target_state == X_FREE_PENDING) {
+ /* ALLOCED -> PENDING: move IDs from allocated to pending */
+ xm->xm_stats.xs_current_alloc_ids -= nd->xi_num_ids;
+ xm->xm_stats.xs_current_pending_free_ids += nd->xi_num_ids;
+ } else if (nd->xi_state == X_FREE_PENDING && target_state == X_FREED) {
+ /* PENDING -> FREE: move IDs from pending to free */
+ xm->xm_stats.xs_current_pending_free_ids -= nd->xi_num_ids;
+ xm->xm_stats.xs_current_free_ids += nd->xi_num_ids;
+ } else if (nd->xi_state == X_ALLOCED && target_state == X_FREED) {
+ /* ALLOCED -> FREE (direct): move IDs from allocated to free */
+ xm->xm_stats.xs_current_alloc_ids -= nd->xi_num_ids;
+ xm->xm_stats.xs_current_free_ids += nd->xi_num_ids;
+ }
+
+ nd->xi_state = target_state;
+ bng_re_xm_insert_by_free_range(new_tree, nd);
+
+ if (nd->xi_next && nd->xi_next->xi_state == target_state)
+ bng_re_xm_consolidate_ranges(xm, nd, new_tree);
+ if (nd->xi_prev && nd->xi_prev->xi_state == target_state)
+ bng_re_xm_consolidate_ranges(xm, nd->xi_prev, new_tree);
+ return 0;
+}
+
+/**
+ * bng_re_xm_consolidate_ranges - combine 2 ranges.
+ *
+ * @param xm - xid manager
+ * @param prev - previous range; which will include
+ * the next block after this call
+ */
+static void bng_re_xm_consolidate_ranges(struct xid_manager *xm,
+ struct xid_node_info *prev,
+ struct rb_root *root)
+{
+ struct xid_node_info *next;
+
+ next = prev->xi_next;
+
+ bng_re_xm_del_from_tree(root, prev);
+ bng_re_xm_del_from_tree(root, next);
+
+ prev->xi_num_ids += next->xi_num_ids;
+ prev->xi_next = next->xi_next;
+ if (next->xi_next)
+ next->xi_next->xi_prev = prev;
+
+ /* note we only consolidate in free or free pending trees */
+ bng_re_xm_insert_by_free_range(root, prev);
+ bng_re_xm_tree_info_del(next);
+}
+
+static char *bng_re_map_xi_state_to_string(int state)
+{
+ switch (state) {
+ case X_FREED:
+ return("FREED");
+ case X_FREE_PENDING:
+ return ("FREE_PENDING");
+ case X_ALLOCED:
+ return("ALLOCED");
+ }
+ return("unknown");
+}
+
+static bool bng_re_xm_print_one_range(void *p)
+{
+ struct xid_node_info *info = (struct xid_node_info *)p;
+ char *state = bng_re_map_xi_state_to_string(info->xi_state);
+
+ if (info && state) {
+ XID_PRINT(" info:[%p](start:%d,num_ids:%d,xi_state:%s,nxt:%p,prev:%p,tag:%d\n ",
+ info, info->xi_starting_id, info->xi_num_ids,
+ state, info->xi_next, info->xi_prev, info->xi_tag);
+ }
+ return false;
+}
+
+/* FIXME: move to xm manager */
+struct xid_node_info *g_searched_node;
+int g_searched_id;
+
+#ifdef STANDALONE_BUILD
+/**
+ * bng_re_xm_search_free_node_for_id - Find a node with a specific id
+ *
+ * @param p - node whose id we're searching for
+ */
+bool bng_re_xm_search_free_node_for_id(void *p)
+{
+ struct xid_node_info *info = (struct xid_node_info *)p;
+
+ if (info->xi_starting_id == g_searched_id) {
+ g_searched_node = info;
+ return true;
+ }
+ return false;
+}
+#endif
+
+/**
+ * bng_re_xm_rb_in_order_tree - traverse rb tree in order and execute a
+ *function
+ *
+ * @param root - root of rb tree to insert into
+ * @param fn - range function to execute
+ */
+static void bng_re_xm_rb_in_order_tree(struct rb_root *root,
+ rb_print_fn_t fn)
+{
+ struct rb_node *next_nd;
+ struct rb_node *nd;
+ bool quit = false;
+
+ nd = rb_first(root);
+
+ while (nd) {
+ next_nd = rb_next(nd);
+ quit = fn(nd);
+ if (quit)
+ break;
+ nd = next_nd;
+ }
+}
+
+/**
+ * bng_re_xm_print_all_ranges - debug fn to print range for one xid manager
+ *
+ * @param xm - xid manager
+ */
+static void bng_re_xm_print_all_ranges(struct xid_manager *xm)
+{
+ /* first have to find 0 elem */
+ struct xid_node_info *p;
+
+ p = xm->xm_head;
+ while (p) {
+ bng_re_xm_print_one_range(p);
+ p = p->xi_next;
+ }
+}
+
+static void bng_re_xm_print_free_tree(struct xid_manager *xm)
+{
+ bng_re_xm_rb_in_order_tree(&xm->xm_free_tree,
+ bng_re_xm_print_one_range);
+}
+
+static void bng_re_xm_print_allocated_tree(struct xid_manager *xm)
+{
+ bng_re_xm_rb_in_order_tree(&xm->xm_allocated_tree,
+ bng_re_xm_print_one_range);
+}
+
+static void bng_re_xm_print_pending_tree(struct xid_manager *xm)
+{
+ bng_re_xm_rb_in_order_tree(&xm->xm_pending_tree,
+ bng_re_xm_print_one_range);
+}
+
+void bng_re_xm_print_all_info(struct xid_manager *xm)
+{
+ struct xid_public_stats stats;
+
+ XID_PRINT(" ====================\n");
+ XID_PRINT(" FREE tree\n");
+ bng_re_xm_print_free_tree(xm);
+ XID_PRINT(" ====================\n");
+ XID_PRINT(" ALLOC tree\n");
+ bng_re_xm_print_allocated_tree(xm);
+ XID_PRINT(" ====================\n");
+ XID_PRINT(" FREE pending tree\n");
+ bng_re_xm_print_pending_tree(xm);
+ XID_PRINT(" ====================\n");
+ XID_PRINT(" MEM list\n");
+ bng_re_xm_print_all_ranges(xm);
+ XID_PRINT(" ====================\n");
+ bng_re_xm_get_stats(xm, &stats);
+ XID_PRINT(" Stats:alloc_op:%lu,allced_ids:%lu,allc_errs:%lu,allc_err_active_limit:%lu\n",
+ stats.xp_alloc_op_ctr,
+ stats.xp_alloc_count_ctr,
+ stats.xp_alloc_err_ctr,
+ stats.xp_alloc_err_active_limit_ctr);
+ XID_PRINT(" Stats: free_op:%lu, freed_ids:%lu, free_errs:%lu\n",
+ stats.xp_free_op_ctr,
+ stats.xp_free_count_ctr,
+ stats.xp_free_err_ctr);
+ XID_PRINT(" Gauge: starting id:%lu, num_ids:%lu, largest block:%lu\n",
+ stats.xp_starting_id_ge,
+ stats.xp_num_ids_ge,
+ stats.xp_largest_free_block_ge);
+ XID_PRINT(" Current: alloc_ids:%lu, pending_ids:%lu, free_ids:%lu\n",
+ stats.xp_current_alloc_ids,
+ stats.xp_current_pending_free_ids,
+ stats.xp_current_free_ids);
+ XID_PRINT(" Peak: max_alloc_ids:%lu (%.1f%% of capacity)\n",
+ stats.xp_max_alloc_ids,
+ stats.xp_num_ids_ge > 0 ?
+ (stats.xp_max_alloc_ids * 100.0) / stats.xp_num_ids_ge : 0.0);
+}
+
+/**
+ * bng_re_xm_free_node_matching_tag- optionally free a node if it matches
+ * a tag. Handle repercussions of node free.
+ *
+ * @param xm - xid manager
+ * @param tag - tag id to free
+ * @param nd - node to (maybe) free
+ *
+ * @return next node
+ */
+static struct xid_node_info *bng_re_xm_free_node_matching_tag(struct xid_manager *xm,
+ int tag,
+ struct xid_node_info *nd)
+{
+ struct xid_node_info *next;
+
+ next = nd->xi_next;
+ if (nd && nd->xi_tag == tag && nd->xi_state == X_ALLOCED) {
+ /* before we free anything- have to calculate the next. */
+ if (next) {
+ /* if the next node is not allocated,
+ * then it may be consolidated and freed. we need to
+ * go to the one after.
+ */
+ if (xm->xm_avoid_reuse) {
+ if (next->xi_state == X_FREE_PENDING)
+ next = next->xi_next;
+ } else {
+ if (next->xi_state == X_FREED)
+ next = next->xi_next;
+ }
+ }
+ bng_re_xm_trans_range_by_pointer(xm, nd);
+ }
+ return next;
+}
+
+/**
+ * bng_re_xm_free_tag - free all entries which match a tag
+ * We do this by finding the first allocated node in alloced tree;
+ * then walking the linked list of ranges to find matches.
+ *
+ * @param xm - xid manager
+ * @param tag - tag id to free
+ */
+void bng_re_xm_free_tag(struct xid_manager *xm, int tag)
+{
+ struct xid_node_info *nd;
+ unsigned long flags;
+
+ spin_lock_irqsave(&xm->xm_lock, flags);
+ nd = (struct xid_node_info *)rb_first(&xm->xm_allocated_tree);
+ while (nd)
+ nd = bng_re_xm_free_node_matching_tag(xm, tag, nd);
+
+ spin_unlock_irqrestore(&xm->xm_lock, flags);
+}
+
+static int bng_re_xm_get_largest_free_block(struct xid_manager *xm)
+{
+ struct xid_node_info *nd;
+ int num_ids = 0;
+
+ nd = (struct xid_node_info *)rb_last(&xm->xm_free_tree);
+ if (nd)
+ num_ids = nd->xi_num_ids;
+ return num_ids;
+}
+
+/**
+ * bng_re_xm_get_stats - get the stats of this xid manager for external consumption
+ *
+ * @param xm - xid manager
+ */
+void bng_re_xm_get_stats(struct xid_manager *xm, struct xid_public_stats *stats)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&xm->xm_lock, flags);
+
+ memset(stats, 0, sizeof(*stats));
+
+ stats->xp_alloc_op_ctr = xm->xm_stats.xs_alloc_op_ctr;
+ stats->xp_alloc_count_ctr = xm->xm_stats.xs_alloc_count_ctr;
+ stats->xp_alloc_err_ctr = xm->xm_stats.xs_alloc_err_ctr;
+ stats->xp_alloc_err_active_limit_ctr = xm->xm_stats.xs_alloc_err_active_limit_ctr;
+ stats->xp_free_op_ctr = xm->xm_stats.xs_free_op_ctr;
+ stats->xp_free_count_ctr = xm->xm_stats.xs_free_count_ctr;
+ stats->xp_free_err_ctr = xm->xm_stats.xs_free_err_ctr;
+ stats->xp_starting_id_ge = xm->xm_starting_id;
+ stats->xp_num_ids_ge = xm->xm_num_ids;
+ stats->xp_largest_free_block_ge = bng_re_xm_get_largest_free_block(xm);
+ stats->xp_current_alloc_ids = xm->xm_stats.xs_current_alloc_ids;
+ stats->xp_current_pending_free_ids = xm->xm_stats.xs_current_pending_free_ids;
+ stats->xp_current_free_ids = xm->xm_stats.xs_current_free_ids;
+ stats->xp_max_alloc_ids = xm->xm_stats.xs_max_alloc_ids;
+ stats->xp_max_active_ids = (u64)xm->xm_max_active_ids;
+ stats->xp_curr_active_ids = (u64)xm->xm_curr_active_ids;
+ stats->xp_free_decr_and_remove = xm->xm_stats.xs_free_decr_and_remove;
+ stats->xp_free_decr_only = xm->xm_stats.xs_free_decr_only;
+ stats->xp_free_remove_only = xm->xm_stats.xs_free_remove_only;
+ stats->xp_current_pending_entries = (u64)(xm->xm_stats.xs_current_alloc_ids -
+ xm->xm_curr_active_ids);
+ if (stats->xp_current_pending_entries > stats->xp_current_alloc_ids)
+ stats->xp_current_pending_entries = 0;
+ spin_unlock_irqrestore(&xm->xm_lock, flags);
+}
+
+void bng_re_xm_curr_active_set(struct xid_manager *xm, int val)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&xm->xm_lock, flags);
+ xm->xm_curr_active_ids = val;
+ spin_unlock_irqrestore(&xm->xm_lock, flags);
+}
+
+void bng_re_xm_curr_active_inc(struct xid_manager *xm)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&xm->xm_lock, flags);
+ xm->xm_curr_active_ids++;
+ spin_unlock_irqrestore(&xm->xm_lock, flags);
+}
+
+void bng_re_xm_curr_active_dec(struct xid_manager *xm)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&xm->xm_lock, flags);
+ xm->xm_curr_active_ids--;
+ spin_unlock_irqrestore(&xm->xm_lock, flags);
+}
+
+int bng_re_xm_curr_active_get(struct xid_manager *xm)
+{
+ unsigned long flags;
+ int v;
+
+ spin_lock_irqsave(&xm->xm_lock, flags);
+ v = xm->xm_curr_active_ids;
+ spin_unlock_irqrestore(&xm->xm_lock, flags);
+ return v;
+}
+
+int bng_re_xm_max_active_get(struct xid_manager *xm)
+{
+ unsigned long flags;
+ int v;
+
+ spin_lock_irqsave(&xm->xm_lock, flags);
+ v = xm->xm_max_active_ids;
+ spin_unlock_irqrestore(&xm->xm_lock, flags);
+ return v;
+}
+
+static void bng_re_xm_destroy_tree(struct rb_root *root)
+{
+ struct xid_node_info *kn;
+ struct rb_node *n;
+
+ while ((n = rb_first(root))) {
+ kn = rb_entry(n, struct xid_node_info, node);
+ rb_erase(n, root);
+ bng_re_xm_tree_info_del(kn);
+ }
+}
+
+void bng_re_xm_destroy(struct xid_manager *xm)
+{
+ unsigned long flags;
+
+ if (!xm)
+ return;
+ spin_lock_irqsave(&xm->xm_lock, flags);
+ bng_re_xm_destroy_tree(&xm->xm_free_tree);
+ bng_re_xm_destroy_tree(&xm->xm_allocated_tree);
+ bng_re_xm_destroy_tree(&xm->xm_pending_tree);
+ xm->xm_head = NULL;
+ spin_unlock_irqrestore(&xm->xm_lock, flags);
+ kfree(xm);
+}
+
+/* temp */
diff --git a/drivers/infiniband/hw/bng_re/xid_allocator.h b/drivers/infiniband/hw/bng_re/xid_allocator.h
new file mode 100644
index 000000000000..db9c77f63986
--- /dev/null
+++ b/drivers/infiniband/hw/bng_re/xid_allocator.h
@@ -0,0 +1,114 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2026 Broadcom */
+
+#ifndef XID_ALLOCATOR_H
+#define XID_ALLOCATOR_H
+
+#include <linux/rbtree.h>
+#include <linux/spinlock.h>
+#include <linux/slab.h>
+
+#define CALLOC(NMEMB, SIZE) kcalloc(NMEMB, SIZE, GFP_ATOMIC)
+#define FREE(PTR) kfree(PTR)
+#define XID_PRINT(...)
+#define ASSERT(...) WARN_ON(__VA_ARGS__)
+
+typedef bool (*rb_print_fn_t)(void *);
+typedef int (*rb_compare_fn_t)(void *, void *);
+
+enum xid_range_state {
+ X_FREED = 1,
+ X_FREE_PENDING = 2,
+ X_ALLOCED = 3
+};
+
+/*Info block for an XID range */
+struct xid_node_info {
+ struct rb_node node;
+ int xi_starting_id;
+ int xi_num_ids;
+ enum xid_range_state xi_state;
+ /*
+ * Note the linked list maintains the ranges in order
+ * of id ; whether they are in allocated tree or free tree
+ */
+ struct xid_node_info *xi_next;
+ struct xid_node_info *xi_prev;
+ int xi_tag; /* tag can identify VFs or other distinctions */
+};
+
+/* stats for an xid manager */
+struct xid_private_stats {
+ u64 xs_alloc_op_ctr; /* how many alloc ops were done */
+ u64 xs_alloc_count_ctr; /* how many ids were alloced */
+ u64 xs_alloc_err_ctr; /* how many errs during alloc */
+ u64 xs_alloc_err_active_limit_ctr; /* alloc failures due to curr_active >= max_active */
+ u64 xs_free_op_ctr; /* how many free ops were done */
+ u64 xs_free_count_ctr; /* how many ids were freed */
+ u64 xs_free_err_ctr; /* errs during free */
+ u64 xs_current_alloc_ids; /* current number of allocated IDs */
+ u64 xs_current_pending_free_ids; /* current number of pending IDs */
+ u64 xs_current_free_ids; /* current number of free IDs */
+ u64 xs_max_alloc_ids; /* maximum number of allocated IDs reached */
+ /* Deferred QP XID: free operations by semantics */
+ u64 xs_free_decr_and_remove;
+ u64 xs_free_decr_only;
+ u64 xs_free_remove_only;
+};
+
+struct xid_public_stats {
+ u64 xp_alloc_op_ctr; /* how many alloc ops were done */
+ u64 xp_alloc_count_ctr; /* how many ids were alloced */
+ u64 xp_alloc_err_ctr; /* how many errs during alloc */
+ u64 xp_alloc_err_active_limit_ctr; /* alloc errs due to curr_active >= max_active */
+ u64 xp_free_op_ctr; /* how many free ops were done */
+ u64 xp_free_count_ctr; /* how many ids were freed */
+ u64 xp_free_err_ctr; /* errs during free */
+ u64 xp_starting_id_ge; /* start id */
+ u64 xp_num_ids_ge; /* num ids */
+ u64 xp_largest_free_block_ge; /* largest free block */
+ u64 xp_current_alloc_ids; /* current number of allocated IDs */
+ u64 xp_current_pending_free_ids; /* current number of pending IDs */
+ u64 xp_current_free_ids; /* current number of free IDs */
+ u64 xp_max_alloc_ids; /* maximum number of allocated IDs reached */
+ u64 xp_max_active_ids; /* cap on active IDs */
+ u64 xp_curr_active_ids; /* current active IDs */
+ u64 xp_free_decr_and_remove;
+ u64 xp_free_decr_only;
+ u64 xp_free_remove_only;
+ u64 xp_current_pending_entries; /* current_alloc_ids - curr_active_ids */
+};
+
+/* Special XID value for "decrement active count only" (no tree removal). */
+
+struct xid_manager {
+ struct rb_root xm_free_tree;
+ struct rb_root xm_allocated_tree;
+ struct rb_root xm_pending_tree;
+ struct xid_private_stats xm_stats;
+ int xm_starting_id;
+ int xm_num_ids;
+ spinlock_t xm_lock; /* lock this instance of xm */
+ struct xid_node_info *xm_head;
+ bool xm_avoid_reuse;
+ int xm_max_active_ids;
+ int xm_curr_active_ids;
+};
+
+/* public api */
+struct xid_manager *bng_re_xm_init(int start_id, int max_ids, int max_active_ids,
+ bool avoid_reuse);
+int bng_re_xm_alloc_range(struct xid_manager *xm, int num_ids, int tag);
+int bng_re_xm_free_range(struct xid_manager *xm, int id, bool decr_active);
+void bng_re_xm_print_all_info(struct xid_manager *xm);
+void bng_re_xm_free_tag(struct xid_manager *xm, int tag);
+void bng_re_xm_get_stats(struct xid_manager *xm, struct xid_public_stats *stats);
+bool bng_re_xm_is_id_allocated(struct xid_manager *xm, int id);
+void bng_re_xm_curr_active_set(struct xid_manager *xm, int val);
+void bng_re_xm_curr_active_inc(struct xid_manager *xm);
+void bng_re_xm_curr_active_dec(struct xid_manager *xm);
+int bng_re_xm_curr_active_get(struct xid_manager *xm);
+int bng_re_xm_max_active_get(struct xid_manager *xm);
+void bng_re_xm_destroy(struct xid_manager *xm);
+#define BNG_RE_XID_DECR_ONLY (-1)
+#endif
--
2.43.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 05/15] RDMA/bng_re: Add support verbs
2026-09-04 10:43 [PATCH 00/15] Add BNG_RE control path verbs Siva Reddy Kallam
` (3 preceding siblings ...)
2026-09-04 10:43 ` [PATCH 04/15] RDMA/bng_re: Add MPC, XID management, doorbell infrastructure Siva Reddy Kallam
@ 2026-09-04 10:43 ` Siva Reddy Kallam
2026-09-04 10:43 ` [PATCH 06/15] RDMA/bng_re: Add ucontext/mmap verbs Siva Reddy Kallam
` (9 subsequent siblings)
14 siblings, 0 replies; 17+ messages in thread
From: Siva Reddy Kallam @ 2026-09-04 10:43 UTC (permalink / raw)
To: leonro, jgg, davem, edumazet, kuba, pabeni, andrew+netdev, horms
Cc: netdev, linux-kernel, linux-rdma, Usman Ansari, Siva Reddy Kallam
From: Usman Ansari <usman.ansari@broadcom.com>
This patch adds below verbs.
- bng_re_query_device
- bng_re_modify_device
- bng_re_query_fw_str
- bng_re_query_port
- bng_re_get_link_layer
- bng_re_get_port_immutable
- bng_re_process_mad
- bng_re_query_pkey
Signed-off-by: Usman Ansari <usman.ansari@broadcom.com>
Signed-off-by: Siva Reddy Kallam <siva.kallam@broadcom.com>
---
drivers/infiniband/hw/bng_re/Makefile | 3 +-
drivers/infiniband/hw/bng_re/bng_dev.c | 16 ++
drivers/infiniband/hw/bng_re/bng_re.h | 11 ++
drivers/infiniband/hw/bng_re/bng_sp.h | 1 +
drivers/infiniband/hw/bng_re/bng_verbs.c | 200 +++++++++++++++++++++++
drivers/infiniband/hw/bng_re/bng_verbs.h | 30 ++++
include/uapi/rdma/bng_re-abi.h | 12 ++
include/uapi/rdma/ib_user_ioctl_verbs.h | 1 +
8 files changed, 273 insertions(+), 1 deletion(-)
create mode 100644 drivers/infiniband/hw/bng_re/bng_verbs.c
create mode 100644 drivers/infiniband/hw/bng_re/bng_verbs.h
create mode 100644 include/uapi/rdma/bng_re-abi.h
diff --git a/drivers/infiniband/hw/bng_re/Makefile b/drivers/infiniband/hw/bng_re/Makefile
index ea0c6951728f..1b954a35993b 100644
--- a/drivers/infiniband/hw/bng_re/Makefile
+++ b/drivers/infiniband/hw/bng_re/Makefile
@@ -12,4 +12,5 @@ bng_re-y := \
bng_debugfs.o \
bng_fw.o \
bng_dev.o \
- bng_sp.o
+ bng_sp.o \
+ bng_verbs.o
diff --git a/drivers/infiniband/hw/bng_re/bng_dev.c b/drivers/infiniband/hw/bng_re/bng_dev.c
index ea6528e80ce3..05665c82b4e1 100644
--- a/drivers/infiniband/hw/bng_re/bng_dev.c
+++ b/drivers/infiniband/hw/bng_re/bng_dev.c
@@ -6,6 +6,7 @@
#include <linux/auxiliary_bus.h>
#include <rdma/ib_verbs.h>
+#include <rdma/bng_re-abi.h>
#include "bng_res.h"
#include "bng_sp.h"
@@ -17,11 +18,26 @@
#include "bng_debugfs.h"
#include "bng_re_mpc_roce.h"
#include "bng_xid.h"
+#include "bng_verbs.h"
MODULE_AUTHOR("Siva Reddy Kallam <siva.kallam@broadcom.com>");
MODULE_DESCRIPTION(BNG_RE_DESC);
MODULE_LICENSE("Dual BSD/GPL");
+static const struct ib_device_ops bng_re_dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_BNG_RE,
+ .uverbs_abi_ver = BNG_RE_ABI_VERSION,
+ .query_device = bng_re_query_device,
+ .modify_device = bng_re_modify_device,
+ .get_dev_fw_str = bng_re_query_fw_str,
+ .query_port = bng_re_query_port,
+ .get_link_layer = bng_re_get_link_layer,
+ .get_port_immutable = bng_re_get_port_immutable,
+ .process_mad = bng_re_process_mad,
+ .query_pkey = bng_re_query_pkey,
+};
+
static struct bng_re_dev *bng_re_dev_add(struct auxiliary_device *adev,
struct bnge_auxr_dev *aux_dev)
{
diff --git a/drivers/infiniband/hw/bng_re/bng_re.h b/drivers/infiniband/hw/bng_re/bng_re.h
index d6fb205d9f12..4fa5db15625f 100644
--- a/drivers/infiniband/hw/bng_re/bng_re.h
+++ b/drivers/infiniband/hw/bng_re/bng_re.h
@@ -21,6 +21,14 @@
#define BNG_RE_MIN_MSIX 2
#define BNG_RE_MAX_MSIX BNGE_MAX_ROCE_MSIX
+#define BNG_RE_PAGE_SHIFT_1G (30)
+#define BNG_RE_MAX_MR_SIZE_LOW BIT_ULL(BNG_RE_PAGE_SHIFT_1G)
+#define BNG_RE_MAX_MR_SIZE_HIGH BIT_ULL(39)
+#define BNG_RE_MAX_MR_SIZE BNG_RE_MAX_MR_SIZE_HIGH
+
+#define BNG_RE_DEFAULT_ACK_DELAY 16
+#define BNG_RE_PAGE_SIZE_SUPPORTED 0x7FFFF000 /* 4kb - 1G */
+
#define BNG_RE_CREQ_NQ_IDX 0
#define BNGE_INVALID_STATS_CTX_ID -1
@@ -250,6 +258,9 @@ struct bng_re_dev {
struct bng_mpc_ctx *mpc;
};
+#define to_bng_re_dev(ptr, member) \
+ container_of((ptr), struct bng_re_dev, member)
+
int bng_re_net_ring_alloc(struct bng_re_dev *rdev,
struct bng_re_ring_attr *ring_attr,
u16 *fw_ring_id);
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.h b/drivers/infiniband/hw/bng_re/bng_sp.h
index 91faa5ac7464..4dac43540f54 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.h
+++ b/drivers/infiniband/hw/bng_re/bng_sp.h
@@ -24,6 +24,7 @@ struct bng_re_dev_attr {
u32 max_qp_rd_atom;
u32 max_qp_init_rd_atom;
u32 max_qp_wqes;
+ u32 max_sq_wqes;
u32 max_qp_sges;
u32 max_cq;
u32 max_cq_wqes;
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.c b/drivers/infiniband/hw/bng_re/bng_verbs.c
new file mode 100644
index 000000000000..25996133c83d
--- /dev/null
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.c
@@ -0,0 +1,200 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2026 Broadcom.
+
+#include <linux/interrupt.h>
+#include <linux/types.h>
+#include <linux/pci.h>
+#include <linux/io.h>
+#include <net/addrconf.h>
+#include <rdma/ib_cache.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_umem.h>
+#include <rdma/ib_pack.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_mad.h>
+#include <rdma/uverbs_ioctl.h>
+#include <rdma/bng_re-abi.h>
+
+#include "bng_res.h"
+#include "bng_fw.h"
+#include "bnge.h"
+#include "bnge_auxr.h"
+#include "bng_re.h"
+#include "bng_verbs.h"
+
+int bng_re_query_device(struct ib_device *ibdev,
+ struct ib_device_attr *ib_attr,
+ struct ib_udata *udata)
+{
+ struct bng_re_dev *rdev = to_bng_re_dev(ibdev, ibdev);
+ struct bng_re_dev_attr *dev_attr = rdev->dev_attr;
+
+ memset(ib_attr, 0, sizeof(*ib_attr));
+ memcpy(&ib_attr->fw_ver, dev_attr->fw_ver,
+ min(sizeof(dev_attr->fw_ver),
+ sizeof(ib_attr->fw_ver)));
+ addrconf_addr_eui48((u8 *)&ib_attr->sys_image_guid,
+ rdev->netdev->dev_addr);
+ ib_attr->max_mr_size = BNG_RE_MAX_MR_SIZE;
+ ib_attr->page_size_cap = BNG_RE_PAGE_SIZE_SUPPORTED;
+ ib_attr->vendor_id = rdev->aux_dev->pdev->vendor;
+ ib_attr->vendor_part_id = rdev->aux_dev->pdev->device;
+ ib_attr->hw_ver = rdev->aux_dev->pdev->revision;
+ ib_attr->max_qp = dev_attr->max_qp;
+ ib_attr->max_qp_wr = dev_attr->max_sq_wqes;
+ ib_attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD
+ | IB_DEVICE_RC_RNR_NAK_GEN
+ | IB_DEVICE_SHUTDOWN_PORT
+ | IB_DEVICE_SYS_IMAGE_GUID
+ | IB_DEVICE_RESIZE_MAX_WR
+ | IB_DEVICE_PORT_ACTIVE_EVENT
+ | IB_DEVICE_N_NOTIFY_CQ
+ | IB_DEVICE_MEM_WINDOW
+ | IB_DEVICE_MEM_WINDOW_TYPE_2B
+ | IB_DEVICE_MEM_MGT_EXTENSIONS;
+ ib_attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
+ ib_attr->max_send_sge = dev_attr->max_qp_sges;
+ ib_attr->max_recv_sge = dev_attr->max_qp_sges;
+ ib_attr->max_sge_rd = dev_attr->max_qp_sges;
+ ib_attr->max_cq = dev_attr->max_cq;
+ ib_attr->max_cqe = dev_attr->max_cq_wqes;
+ ib_attr->max_mr = dev_attr->max_mr;
+ ib_attr->max_pd = dev_attr->max_pd;
+ ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
+ ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom;
+ ib_attr->atomic_cap = IB_ATOMIC_NONE;
+ ib_attr->masked_atomic_cap = IB_ATOMIC_NONE;
+ if (dev_attr->is_atomic) {
+ ib_attr->atomic_cap = IB_ATOMIC_GLOB;
+ ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB;
+ }
+ ib_attr->max_ee_rd_atom = 0;
+ ib_attr->max_res_rd_atom = 0;
+ ib_attr->max_ee_init_rd_atom = 0;
+ ib_attr->max_ee = 0;
+ ib_attr->max_rdd = 0;
+ ib_attr->max_mw = dev_attr->max_mw;
+ ib_attr->max_raw_ipv6_qp = 0;
+ ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp;
+ ib_attr->max_mcast_grp = 0;
+ ib_attr->max_mcast_qp_attach = 0;
+ ib_attr->max_total_mcast_qp_attach = 0;
+ ib_attr->max_ah = dev_attr->max_ah;
+ ib_attr->max_srq = dev_attr->max_srq;
+ ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
+ ib_attr->max_srq_sge = dev_attr->max_srq_sges;
+ ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS;
+ ib_attr->max_pkeys = 1;
+ ib_attr->local_ca_ack_delay = BNG_RE_DEFAULT_ACK_DELAY;
+
+ return 0;
+}
+
+int bng_re_modify_device(struct ib_device *ibdev,
+ int device_modify_mask,
+ struct ib_device_modify *device_modify)
+{
+ ibdev_dbg(ibdev, "Modify device with mask 0x%x", device_modify_mask);
+
+ if (device_modify_mask & ~IB_DEVICE_MODIFY_NODE_DESC)
+ return -EOPNOTSUPP;
+
+ if (!(device_modify_mask & IB_DEVICE_MODIFY_NODE_DESC))
+ return 0;
+
+ memcpy(ibdev->node_desc, device_modify->node_desc, IB_DEVICE_NODE_DESC_MAX);
+
+ return 0;
+}
+
+void bng_re_query_fw_str(struct ib_device *ibdev, char *str)
+{
+ struct bng_re_dev *rdev = to_bng_re_dev(ibdev, ibdev);
+
+ sprintf(str, "%d.%d.%d.%d", rdev->dev_attr->fw_ver[0],
+ rdev->dev_attr->fw_ver[1], rdev->dev_attr->fw_ver[2],
+ rdev->dev_attr->fw_ver[3]);
+}
+
+int bng_re_query_port(struct ib_device *ibdev, u32 port_num,
+ struct ib_port_attr *port_attr)
+{
+ struct bng_re_dev *rdev = to_bng_re_dev(ibdev, ibdev);
+ struct bng_re_dev_attr *dev_attr = rdev->dev_attr;
+ int rc;
+
+ memset(port_attr, 0, sizeof(*port_attr));
+
+ if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) {
+ port_attr->state = IB_PORT_ACTIVE;
+ port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
+ } else {
+ port_attr->state = IB_PORT_DOWN;
+ port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
+ }
+ port_attr->max_mtu = IB_MTU_4096;
+ port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu);
+ port_attr->gid_tbl_len = dev_attr->max_sgid;
+ port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
+ IB_PORT_DEVICE_MGMT_SUP |
+ IB_PORT_VENDOR_CLASS_SUP;
+ port_attr->ip_gids = true;
+
+ port_attr->max_msg_sz = (u32)BNG_RE_MAX_MR_SIZE_LOW;
+ port_attr->bad_pkey_cntr = 0;
+ port_attr->qkey_viol_cntr = 0;
+ port_attr->pkey_tbl_len = dev_attr->max_pkey;
+ port_attr->lid = 0;
+ port_attr->sm_lid = 0;
+ port_attr->lmc = 0;
+ port_attr->max_vl_num = 4;
+ port_attr->sm_sl = 0;
+ port_attr->subnet_timeout = 0;
+ port_attr->init_type_reply = 0;
+ rc = ib_get_eth_speed(&rdev->ibdev, port_num, &port_attr->active_speed,
+ &port_attr->active_width);
+
+ return rc;
+}
+
+enum rdma_link_layer bng_re_get_link_layer(struct ib_device *ibdev, u32 port_num)
+{
+ return IB_LINK_LAYER_ETHERNET;
+}
+
+int bng_re_get_port_immutable(struct ib_device *ibdev, u32 port_num,
+ struct ib_port_immutable *immutable)
+{
+ struct ib_port_attr port_attr;
+
+ if (bng_re_query_port(ibdev, port_num, &port_attr))
+ return -EINVAL;
+
+ immutable->pkey_tbl_len = port_attr.pkey_tbl_len;
+ immutable->gid_tbl_len = port_attr.gid_tbl_len;
+ immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
+ immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
+ immutable->max_mad_size = IB_MGMT_MAD_SIZE;
+
+ return 0;
+}
+
+int bng_re_query_pkey(struct ib_device *ibdev, u32 port_num,
+ u16 index, u16 *pkey)
+{
+ if (index > 0)
+ return -EINVAL;
+
+ *pkey = IB_DEFAULT_PKEY_FULL;
+
+ return 0;
+}
+
+int bng_re_process_mad(struct ib_device *ibdev, int process_mad_flags,
+ u32 port_num, const struct ib_wc *in_wc,
+ const struct ib_grh *in_grh,
+ const struct ib_mad *in_mad, struct ib_mad *out_mad,
+ size_t *out_mad_size, u16 *out_mad_pkey_index)
+{
+ return IB_MAD_RESULT_FAILURE;
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.h b/drivers/infiniband/hw/bng_re/bng_verbs.h
new file mode 100644
index 000000000000..5875ec565ea4
--- /dev/null
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+// Copyright (c) 2026 Broadcom.
+
+#ifndef __BNG_RE_VERBS_H__
+#define __BNG_RE_VERBS_H__
+
+#include <linux/refcount.h>
+#include <rdma/ib_verbs.h>
+
+#include "bng_sp.h"
+#include "bng_re.h"
+
+int bng_re_query_device(struct ib_device *ibdev, struct ib_device_attr *ib_attr,
+ struct ib_udata *udata);
+int bng_re_modify_device(struct ib_device *ibdev, int device_modify_mask,
+ struct ib_device_modify *device_modify);
+void bng_re_query_fw_str(struct ib_device *ibdev, char *str);
+int bng_re_query_port(struct ib_device *ibdev, u32 port_num,
+ struct ib_port_attr *port_attr);
+int bng_re_get_port_immutable(struct ib_device *ibdev, u32 port_num,
+ struct ib_port_immutable *immutable);
+int bng_re_query_pkey(struct ib_device *ibdev, u32 port_num, u16 index, u16 *pkey);
+enum rdma_link_layer bng_re_get_link_layer(struct ib_device *ibdev, u32 port_num);
+int bng_re_process_mad(struct ib_device *ibdev, int mad_flags,
+ u32 port_num, const struct ib_wc *in_wc,
+ const struct ib_grh *in_grh,
+ const struct ib_mad *in_mad, struct ib_mad *out_mad,
+ size_t *out_mad_size, u16 *out_mad_pkey_index);
+#endif /* __BNG_RE_VERBS_H__ */
+
diff --git a/include/uapi/rdma/bng_re-abi.h b/include/uapi/rdma/bng_re-abi.h
new file mode 100644
index 000000000000..cea056dc8610
--- /dev/null
+++ b/include/uapi/rdma/bng_re-abi.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-2-Clause) */
+/* Copyright (c) 2025 Broadcom */
+
+#ifndef __BNG_RE_UVERBS_ABI_H__
+#define __BNG_RE_UVERBS_ABI_H__
+
+#include <linux/types.h>
+#include <rdma/ib_user_ioctl_cmds.h>
+
+#define BNG_RE_ABI_VERSION 1
+
+#endif /* __BNG_RE_UVERBS_ABI_H__*/
diff --git a/include/uapi/rdma/ib_user_ioctl_verbs.h b/include/uapi/rdma/ib_user_ioctl_verbs.h
index 21f86cc7bb1f..d8ffeb6fbb4a 100644
--- a/include/uapi/rdma/ib_user_ioctl_verbs.h
+++ b/include/uapi/rdma/ib_user_ioctl_verbs.h
@@ -257,6 +257,7 @@ enum rdma_driver_id {
RDMA_DRIVER_ERDMA,
RDMA_DRIVER_MANA,
RDMA_DRIVER_IONIC,
+ RDMA_DRIVER_BNG_RE,
};
enum ib_uverbs_gid_type {
--
2.43.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 06/15] RDMA/bng_re: Add ucontext/mmap verbs
2026-09-04 10:43 [PATCH 00/15] Add BNG_RE control path verbs Siva Reddy Kallam
` (4 preceding siblings ...)
2026-09-04 10:43 ` [PATCH 05/15] RDMA/bng_re: Add support verbs Siva Reddy Kallam
@ 2026-09-04 10:43 ` Siva Reddy Kallam
2026-09-04 10:43 ` [PATCH 07/15] RDMA/bng_re: Add GID verbs Siva Reddy Kallam
` (8 subsequent siblings)
14 siblings, 0 replies; 17+ messages in thread
From: Siva Reddy Kallam @ 2026-09-04 10:43 UTC (permalink / raw)
To: leonro, jgg, davem, edumazet, kuba, pabeni, andrew+netdev, horms
Cc: netdev, linux-kernel, linux-rdma, Siva Reddy Kallam
This patch adds below verbs.
- bng_re_alloc_ucontext
- bng_re_dealloc_ucontext
- bng_re_disassociate_ucontext
- bng_re_mmap
- bng_re_mmap_free
Signed-off-by: Siva Reddy Kallam <siva.kallam@broadcom.com>
---
drivers/infiniband/hw/bng_re/bng_dev.c | 9 +
drivers/infiniband/hw/bng_re/bng_re.h | 32 ++++
drivers/infiniband/hw/bng_re/bng_res.c | 94 ++++++++++
drivers/infiniband/hw/bng_re/bng_res.h | 25 +++
drivers/infiniband/hw/bng_re/bng_verbs.c | 219 +++++++++++++++++++++++
drivers/infiniband/hw/bng_re/bng_verbs.h | 22 +++
include/uapi/rdma/bng_re-abi.h | 31 ++++
7 files changed, 432 insertions(+)
diff --git a/drivers/infiniband/hw/bng_re/bng_dev.c b/drivers/infiniband/hw/bng_re/bng_dev.c
index 05665c82b4e1..401c7a0ddb85 100644
--- a/drivers/infiniband/hw/bng_re/bng_dev.c
+++ b/drivers/infiniband/hw/bng_re/bng_dev.c
@@ -36,6 +36,12 @@ static const struct ib_device_ops bng_re_dev_ops = {
.get_port_immutable = bng_re_get_port_immutable,
.process_mad = bng_re_process_mad,
.query_pkey = bng_re_query_pkey,
+ .alloc_ucontext = bng_re_alloc_ucontext,
+ .dealloc_ucontext = bng_re_dealloc_ucontext,
+ .disassociate_ucontext = bng_re_disassociate_ucontext,
+ .mmap = bng_re_mmap,
+ .mmap_free = bng_re_mmap_free,
+ INIT_RDMA_OBJ_SIZE(ib_ucontext, bng_re_ucontext, ib_uctx),
};
static struct bng_re_dev *bng_re_dev_add(struct auxiliary_device *adev,
@@ -284,6 +290,9 @@ static int bng_re_query_hwrm_version(struct bng_re_dev *rdev)
if (!cctx->hwrm_cmd_max_timeout)
cctx->hwrm_cmd_max_timeout = BNG_ROCE_FW_MAX_TIMEOUT;
+ cctx->chip_rev = ver_get_resp.chip_rev;
+ cctx->chip_metal = ver_get_resp.chip_metal;
+
return 0;
}
diff --git a/drivers/infiniband/hw/bng_re/bng_re.h b/drivers/infiniband/hw/bng_re/bng_re.h
index 4fa5db15625f..19b1e69dfb1e 100644
--- a/drivers/infiniband/hw/bng_re/bng_re.h
+++ b/drivers/infiniband/hw/bng_re/bng_re.h
@@ -172,6 +172,38 @@ struct bng_mpc_db {
struct bng_re_db_info dbinfo;
};
+enum bng_re_mmap_flag {
+ BNG_RE_MMAP_WC_DB,
+ BNG_RE_MMAP_DBR_PAGE,
+ BNG_RE_MMAP_DB_RECOVERY_PAGE,
+ BNG_RE_MMAP_DBR_PACING_BAR,
+ BNG_RE_MMAP_UC_DB,
+ BNG_RE_MMAP_TOGGLE_PAGE,
+ BNG_RE_MMAP_HDBR_BASE,
+};
+
+static inline char *bng_re_mmap_str(enum bng_re_mmap_flag mmap_flag)
+{
+ switch (mmap_flag) {
+ case BNG_RE_MMAP_WC_DB:
+ return "BNG_RE_MMAP_WC_DB";
+ case BNG_RE_MMAP_DBR_PAGE:
+ return "BNG_RE_MMAP_DBR_PAGE";
+ case BNG_RE_MMAP_DB_RECOVERY_PAGE:
+ return "BNG_RE_MMAP_DB_RECOVERY_PAGE";
+ case BNG_RE_MMAP_DBR_PACING_BAR:
+ return "BNG_RE_MMAP_DBR_PACING_BAR";
+ case BNG_RE_MMAP_UC_DB:
+ return "BNG_RE_MMAP_UC_DB";
+ case BNG_RE_MMAP_HDBR_BASE:
+ return "BNG_RE_MMAP_HDBR_BASE";
+ case BNG_RE_MMAP_TOGGLE_PAGE:
+ return "BNG_RE_MMAP_TOGGLE_PAGE";
+ default:
+ return "Unknown";
+ }
+}
+
/* NQ specific structures */
struct bng_re_nq_db {
struct bng_re_reg_desc reg;
diff --git a/drivers/infiniband/hw/bng_re/bng_res.c b/drivers/infiniband/hw/bng_re/bng_res.c
index be6d5099aa35..fe261354c4cb 100644
--- a/drivers/infiniband/hw/bng_re/bng_res.c
+++ b/drivers/infiniband/hw/bng_re/bng_res.c
@@ -360,6 +360,7 @@ static int bng_res_alloc_dpi_tbl(struct bng_re_res *res,
}
memset((u8 *)dpit->tbl, 0xFF, bytes);
+ mutex_init(&res->dpi_tbl_lock);
dpit->priv_db = dpit->ucreg.bar_reg + dpit->ucreg.offset;
return 0;
}
@@ -379,3 +380,96 @@ int bng_res_alloc_init_tbls(struct bng_re_res *res)
return rc;
}
+
+int bng_re_alloc_dpi(struct bng_re_res *res,
+ struct bng_re_dpi *dpi,
+ void *app, enum bng_re_dpi_type type)
+{
+ struct bng_re_dpi_tbl *dpit = &res->dpi_tbl;
+ struct bng_re_reg_desc *reg;
+ u32 bit_num;
+ u64 umaddr;
+ int rc = 0;
+
+ reg = &dpit->wcreg;
+ mutex_lock(&res->dpi_tbl_lock);
+ if (!dpit->tbl) {
+ rc = -ENOMEM;
+ goto unlock;
+ }
+ bit_num = find_first_bit(dpit->tbl, dpit->max);
+ if (bit_num >= dpit->max) {
+ rc = -ENOMEM;
+ goto unlock;
+ }
+ clear_bit(bit_num, dpit->tbl);
+ dpit->app_tbl[bit_num] = app;
+ dpi->bit = bit_num;
+ dpi->dpi = bit_num + (reg->offset - dpit->ucreg.offset) / PAGE_SIZE;
+ umaddr = reg->bar_base + reg->offset + bit_num * PAGE_SIZE;
+ dpi->umdbr = umaddr;
+ switch (type) {
+ case BNG_RE_DPI_TYPE_KERNEL:
+ dpi->umdbr = dpit->ucreg.bar_base +
+ dpit->ucreg.offset + bit_num * PAGE_SIZE;
+ dpi->dbr = dpit->priv_db;
+ dpi->dpi = dpi->bit;
+ break;
+ case BNG_RE_DPI_TYPE_WC:
+ dpi->dbr = ioremap_wc(umaddr, PAGE_SIZE);
+ break;
+ default:
+ dpi->dbr = ioremap(umaddr, PAGE_SIZE);
+ break;
+ }
+
+ if (!dpi->dbr) {
+ dev_err(&res->pdev->dev, "DB remap failed, type = %d\n",
+ type);
+ rc = -ENOMEM;
+ set_bit(bit_num, dpit->tbl);
+ dpit->app_tbl[bit_num] = NULL;
+ goto unlock;
+ }
+ dpi->type = type;
+unlock:
+ mutex_unlock(&res->dpi_tbl_lock);
+ return rc;
+}
+
+int bng_re_dealloc_dpi(struct bng_re_res *res,
+ struct bng_re_dpi *dpi)
+{
+ struct bng_re_dpi_tbl *dpit = &res->dpi_tbl;
+ int rc = 0;
+
+ mutex_lock(&res->dpi_tbl_lock);
+ if (dpi->bit >= dpit->max) {
+ dev_warn(&res->pdev->dev,
+ "Invalid DPI? dpi = %d, bit = %d\n",
+ dpi->dpi, dpi->bit);
+ rc = -EINVAL;
+ goto unlock;
+ }
+ if (dpi->dpi && dpi->type != BNG_RE_DPI_TYPE_KERNEL)
+ pci_iounmap(res->pdev, dpi->dbr);
+ if (test_and_set_bit(dpi->bit, dpit->tbl)) {
+ dev_warn(&res->pdev->dev,
+ "Freeing an unused DPI? dpi = %d, bit = %d\n",
+ dpi->dpi, dpi->bit);
+ dev_warn(&res->pdev->dev,
+ "umdbr = %lld, dbr = %p, type = %d\n",
+ dpi->umdbr, dpi->dbr, dpi->type);
+ dev_warn(&res->pdev->dev,
+ "dpit max = %d, priv_db = %p\n",
+ dpit->max, dpit->priv_db);
+ rc = -EINVAL;
+ goto unlock;
+ }
+ if (dpit->app_tbl)
+ dpit->app_tbl[dpi->bit] = NULL;
+ memset(dpi, 0, sizeof(*dpi));
+unlock:
+ mutex_unlock(&res->dpi_tbl_lock);
+ return rc;
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_res.h b/drivers/infiniband/hw/bng_re/bng_res.h
index 9745609190d9..11e453d4db36 100644
--- a/drivers/infiniband/hw/bng_re/bng_res.h
+++ b/drivers/infiniband/hw/bng_re/bng_res.h
@@ -91,6 +91,8 @@ struct bng_re_chip_ctx {
u64 hwrm_intf_ver;
u16 hwrm_cmd_max_timeout;
struct bng_re_drv_modes modes;
+ u8 chip_rev;
+ u8 chip_metal;
};
struct bng_re_pbl {
@@ -156,6 +158,22 @@ struct bng_re_stats {
u32 fw_id;
};
+/* DPI types */
+enum bng_re_dpi_type {
+ BNG_RE_DPI_TYPE_KERNEL = 0,
+ BNG_RE_DPI_TYPE_UC = 1,
+ BNG_RE_DPI_TYPE_WC = 2
+};
+
+/* DPI structure */
+struct bng_re_dpi {
+ u32 dpi;
+ u32 bit;
+ u64 umdbr;
+ void __iomem *dbr;
+ enum bng_re_dpi_type type;
+};
+
/* DPI table */
struct bng_re_dpi_tbl {
void **app_tbl;
@@ -171,6 +189,8 @@ struct bng_re_res {
struct bng_re_chip_ctx *cctx;
struct bng_re_dev_attr *dattr;
struct bng_re_dpi_tbl dpi_tbl;
+ /* Serialize access to DPI table */
+ struct mutex dpi_tbl_lock;
struct xid_manager *qp_xids;
struct xid_manager *ah_xids;
};
@@ -282,4 +302,9 @@ void bng_res_unmap_db_bar(struct bng_re_res *res);
int bng_res_map_db_bar(struct bng_re_res *res);
void bng_res_free_tbls(struct bng_re_res *res);
int bng_res_alloc_init_tbls(struct bng_re_res *res);
+int bng_re_alloc_dpi(struct bng_re_res *res,
+ struct bng_re_dpi *dpi,
+ void *app, enum bng_re_dpi_type type);
+int bng_re_dealloc_dpi(struct bng_re_res *res,
+ struct bng_re_dpi *dpi);
#endif
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.c b/drivers/infiniband/hw/bng_re/bng_verbs.c
index 25996133c83d..ef2b5bd65e54 100644
--- a/drivers/infiniband/hw/bng_re/bng_verbs.c
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.c
@@ -198,3 +198,222 @@ int bng_re_process_mad(struct ib_device *ibdev, int process_mad_flags,
{
return IB_MAD_RESULT_FAILURE;
}
+
+static struct bng_re_user_mmap_entry*
+bng_re_mmap_entry_insert(struct bng_re_ucontext *uctx, u64 cpu_addr,
+ dma_addr_t dma_addr, u8 user_mmap_hint, u64 *offset)
+{
+ struct bng_re_user_mmap_entry *entry;
+ int ret;
+
+ entry = kzalloc_obj(*entry, GFP_KERNEL);
+ if (!entry)
+ return NULL;
+
+ entry->cpu_addr = cpu_addr;
+ entry->dma_addr = dma_addr;
+ entry->uctx = uctx;
+ entry->mmap_flag = user_mmap_hint;
+
+ switch (user_mmap_hint) {
+ case BNG_RE_MMAP_UC_DB:
+ ret = rdma_user_mmap_entry_insert_range(&uctx->ib_uctx,
+ &entry->rdma_entry,
+ PAGE_SIZE, 0xF, U32_MAX);
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+
+ if (ret) {
+ kfree(entry);
+ return NULL;
+ }
+
+ dev_dbg(rdev_to_dev(uctx->rdev),
+ "%s %d uctx->ib_uctx 0x%lx cpu 0x%lx dma 0x%lx mmap hint %s offset 0x%lx\n",
+ __func__, __LINE__, (unsigned long)&uctx->ib_uctx,
+ (unsigned long)entry->cpu_addr, (unsigned long)entry->dma_addr,
+ bng_re_mmap_str(user_mmap_hint),
+ (unsigned long)rdma_user_mmap_get_offset(&entry->rdma_entry));
+
+ if (offset)
+ *offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
+
+ return entry;
+}
+
+static int bng_re_get_user_dpi(struct bng_re_dev *rdev,
+ struct bng_re_ucontext *cntx)
+{
+ int ret = 0;
+ u8 type;
+
+ type = BNG_RE_DPI_TYPE_UC;
+ ret = bng_re_alloc_dpi(&rdev->bng_res, &cntx->dpi, cntx, type);
+ if (ret)
+ dev_err(rdev_to_dev(rdev), "Alloc doorbell page failed!");
+
+ return ret;
+}
+
+int bng_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)
+{
+ struct ib_device *ibdev = ctx->device;
+ struct bng_re_ucontext *uctx =
+ container_of(ctx, struct bng_re_ucontext, ib_uctx);
+ struct bng_re_dev *rdev = to_bng_re_dev(ibdev, ibdev);
+ struct bng_re_dev_attr *dev_attr = rdev->dev_attr;
+ struct bng_re_user_mmap_entry *is_entry;
+ struct bng_re_uctx_resp resp = {};
+ struct bng_re_uctx_req ureq = {};
+ u32 chip_met_rev_num = 0;
+ int rc;
+
+ ibdev_dbg(ibdev, "ABI version requested %u", ibdev->ops.uverbs_abi_ver);
+
+ if (ibdev->ops.uverbs_abi_ver != BNG_RE_ABI_VERSION) {
+ ibdev_dbg(ibdev, " is different from the device %d ",
+ BNG_RE_ABI_VERSION);
+ return -EPERM;
+ }
+
+ uctx->rdev = rdev;
+
+ rc = bng_re_get_user_dpi(rdev, uctx);
+ if (rc) {
+ ibdev_dbg(ibdev, "user DPI alloc failed");
+ return rc;
+ }
+
+ resp.dpi = uctx->dpi.dpi;
+ is_entry = bng_re_mmap_entry_insert(uctx,
+ (u64)uctx->dpi.umdbr, 0,
+ BNG_RE_MMAP_UC_DB,
+ &resp.uc_db_mmap_key);
+ if (!is_entry) {
+ rc = -ENOMEM;
+ goto err_entry_insert;
+ }
+ uctx->uc_db_mmap = &is_entry->rdma_entry;
+
+ resp.comp_mask = BNG_RE_UCNTX_CMASK_HAVE_CCTX;
+ chip_met_rev_num = rdev->chip_ctx->chip_num;
+ chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_rev & 0xFF) <<
+ BNG_RE_CHIP_REV_SFT;
+ chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_metal & 0xFF) <<
+ BNG_RE_CHIP_MET_SFT;
+ resp.chip_id = chip_met_rev_num;
+ resp.dev_id = rdev->aux_dev->pdev->devfn;
+ resp.max_qp = rdev->dev_attr->max_qp;
+ resp.cqe_sz = sizeof(struct cq_base);
+ resp.max_cqd = dev_attr->max_cq_wqes;
+
+ if (udata->inlen >= sizeof(ureq)) {
+ rc = ib_copy_from_udata(&ureq, udata, min(udata->inlen, sizeof(ureq)));
+ if (rc)
+ goto copy_udata_fail;
+ if (ureq.comp_mask & BNG_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT)
+ resp.comp_mask |= BNG_RE_UCNTX_CMASK_POW2_DISABLED;
+
+ if (ureq.comp_mask & BNG_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE)
+ resp.comp_mask |= BNG_RE_UCNTX_CMASK_RSVD_WQE_DISABLED;
+ }
+
+ uctx->cmask = (uint64_t)resp.comp_mask;
+
+ rc = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp)));
+ if (rc) {
+ ibdev_err(ibdev, "Failed to copy user context");
+ rc = -EFAULT;
+ goto copy_udata_fail;
+ }
+
+ return 0;
+
+copy_udata_fail:
+ rdma_user_mmap_entry_remove(uctx->uc_db_mmap);
+ uctx->uc_db_mmap = NULL;
+err_entry_insert:
+ bng_re_dealloc_dpi(&rdev->bng_res, &uctx->dpi);
+ return rc;
+}
+
+void bng_re_disassociate_ucontext(struct ib_ucontext *ibcontext)
+{
+}
+
+void bng_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)
+{
+ struct bng_re_ucontext *uctx = container_of(ib_uctx,
+ struct bng_re_ucontext,
+ ib_uctx);
+ struct bng_re_dev *rdev = uctx->rdev;
+ int rc;
+
+ rdma_user_mmap_entry_remove(uctx->uc_db_mmap);
+ uctx->uc_db_mmap = NULL;
+ rc = bng_re_dealloc_dpi(&rdev->bng_res, &uctx->dpi);
+ if (rc)
+ dev_err(rdev_to_dev(rdev), "Deallocate DPI failed");
+ uctx->dpi.dbr = NULL;
+}
+
+int bng_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
+{
+ struct bng_re_user_mmap_entry *bng_entry;
+ struct rdma_user_mmap_entry *rdma_entry;
+ int ret = 0;
+ u64 pfn;
+
+ if (vma->vm_end - vma->vm_start != PAGE_SIZE)
+ return -EINVAL;
+
+ rdma_entry = rdma_user_mmap_entry_get(ib_uctx, vma);
+ if (!rdma_entry)
+ return -EINVAL;
+
+ bng_entry = container_of(rdma_entry, struct bng_re_user_mmap_entry,
+ rdma_entry);
+
+ switch (bng_entry->mmap_flag) {
+ case BNG_RE_MMAP_WC_DB:
+ pfn = bng_entry->cpu_addr >> PAGE_SHIFT;
+ ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE,
+ pgprot_writecombine(vma->vm_page_prot),
+ rdma_entry);
+ break;
+ case BNG_RE_MMAP_UC_DB:
+ pfn = bng_entry->cpu_addr >> PAGE_SHIFT;
+ ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE,
+ pgprot_noncached(vma->vm_page_prot),
+ rdma_entry);
+ break;
+ case BNG_RE_MMAP_DBR_PAGE:
+ case BNG_RE_MMAP_TOGGLE_PAGE:
+ /* Driver doesn't expect write access for user space */
+ if (vma->vm_flags & VM_WRITE)
+ ret = -EFAULT;
+ else
+ ret = vm_insert_page(vma, vma->vm_start,
+ virt_to_page((void *)bng_entry->cpu_addr));
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+
+ rdma_user_mmap_entry_put(rdma_entry);
+ return ret;
+}
+
+void bng_re_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
+{
+ struct bng_re_user_mmap_entry *bng_entry;
+
+ bng_entry = container_of(rdma_entry, struct bng_re_user_mmap_entry,
+ rdma_entry);
+
+ kfree(bng_entry);
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.h b/drivers/infiniband/hw/bng_re/bng_verbs.h
index 5875ec565ea4..565571b2f1fe 100644
--- a/drivers/infiniband/hw/bng_re/bng_verbs.h
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.h
@@ -10,6 +10,22 @@
#include "bng_sp.h"
#include "bng_re.h"
+struct bng_re_ucontext {
+ struct ib_ucontext ib_uctx;
+ struct bng_re_dev *rdev;
+ struct bng_re_dpi dpi;
+ struct rdma_user_mmap_entry *uc_db_mmap;
+ u64 cmask;
+};
+
+struct bng_re_user_mmap_entry {
+ struct rdma_user_mmap_entry rdma_entry;
+ struct bng_re_ucontext *uctx;
+ dma_addr_t dma_addr;
+ u64 cpu_addr;
+ u8 mmap_flag;
+};
+
int bng_re_query_device(struct ib_device *ibdev, struct ib_device_attr *ib_attr,
struct ib_udata *udata);
int bng_re_modify_device(struct ib_device *ibdev, int device_modify_mask,
@@ -26,5 +42,11 @@ int bng_re_process_mad(struct ib_device *ibdev, int mad_flags,
const struct ib_grh *in_grh,
const struct ib_mad *in_mad, struct ib_mad *out_mad,
size_t *out_mad_size, u16 *out_mad_pkey_index);
+int bng_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata);
+void bng_re_dealloc_ucontext(struct ib_ucontext *context);
+void bng_re_disassociate_ucontext(struct ib_ucontext *ibcontext);
+int bng_re_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
+void bng_re_mmap_free(struct rdma_user_mmap_entry *rdma_entry);
+
#endif /* __BNG_RE_VERBS_H__ */
diff --git a/include/uapi/rdma/bng_re-abi.h b/include/uapi/rdma/bng_re-abi.h
index cea056dc8610..14d56a88def4 100644
--- a/include/uapi/rdma/bng_re-abi.h
+++ b/include/uapi/rdma/bng_re-abi.h
@@ -9,4 +9,35 @@
#define BNG_RE_ABI_VERSION 1
+#define BNG_RE_CHIP_NUM_SFT 0x00
+#define BNG_RE_CHIP_REV_SFT 0x10
+#define BNG_RE_CHIP_MET_SFT 0x18
+
+enum {
+ BNG_RE_UCNTX_CMASK_HAVE_CCTX = 0x1ULL,
+ BNG_RE_UCNTX_CMASK_POW2_DISABLED = 0x02ULL,
+ BNG_RE_UCNTX_CMASK_RSVD_WQE_DISABLED = 0x04ULL,
+ BNG_RE_UCNTX_CMASK_INTERNAL_QUEUE_MEMORY = 0x08ULL,
+};
+
+enum {
+ BNG_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT = 0x01,
+ BNG_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE = 0x02,
+};
+
+struct bng_re_uctx_req {
+ __aligned_u64 comp_mask;
+};
+
+struct bng_re_uctx_resp {
+ __u32 dev_id;
+ __u32 max_qp;
+ __u32 cqe_sz;
+ __u32 max_cqd;
+ __aligned_u64 comp_mask;
+ __u32 chip_id;
+ __u32 dpi;
+ __u64 uc_db_mmap_key;
+};
+
#endif /* __BNG_RE_UVERBS_ABI_H__*/
--
2.43.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 07/15] RDMA/bng_re: Add GID verbs
2026-09-04 10:43 [PATCH 00/15] Add BNG_RE control path verbs Siva Reddy Kallam
` (5 preceding siblings ...)
2026-09-04 10:43 ` [PATCH 06/15] RDMA/bng_re: Add ucontext/mmap verbs Siva Reddy Kallam
@ 2026-09-04 10:43 ` Siva Reddy Kallam
2026-09-04 10:43 ` [PATCH 08/15] RDMA/bng_re: Add PD verbs Siva Reddy Kallam
` (7 subsequent siblings)
14 siblings, 0 replies; 17+ messages in thread
From: Siva Reddy Kallam @ 2026-09-04 10:43 UTC (permalink / raw)
To: leonro, jgg, davem, edumazet, kuba, pabeni, andrew+netdev, horms
Cc: netdev, linux-kernel, linux-rdma, Sachin Holla, Siva Reddy Kallam
From: Sachin Holla <sachin.holla@broadcom.com>
This patch adds below verbs.
- bng_re_query_gid
- bng_re_add_gid
- bng_re_del_gid
Signed-off-by: Sachin Holla <sachin.holla@broadcom.com>
Signed-off-by: Siva Reddy Kallam <siva.kallam@broadcom.com>
---
drivers/infiniband/hw/bng_re/bng_dev.c | 10 ++
drivers/infiniband/hw/bng_re/bng_re.h | 2 +
drivers/infiniband/hw/bng_re/bng_res.c | 13 +-
drivers/infiniband/hw/bng_re/bng_res.h | 16 ++
drivers/infiniband/hw/bng_re/bng_sp.c | 206 +++++++++++++++++++++++
drivers/infiniband/hw/bng_re/bng_sp.h | 16 ++
drivers/infiniband/hw/bng_re/bng_verbs.c | 109 ++++++++++++
drivers/infiniband/hw/bng_re/bng_verbs.h | 10 +-
8 files changed, 380 insertions(+), 2 deletions(-)
diff --git a/drivers/infiniband/hw/bng_re/bng_dev.c b/drivers/infiniband/hw/bng_re/bng_dev.c
index 401c7a0ddb85..9ce8e1bb420a 100644
--- a/drivers/infiniband/hw/bng_re/bng_dev.c
+++ b/drivers/infiniband/hw/bng_re/bng_dev.c
@@ -41,6 +41,9 @@ static const struct ib_device_ops bng_re_dev_ops = {
.disassociate_ucontext = bng_re_disassociate_ucontext,
.mmap = bng_re_mmap,
.mmap_free = bng_re_mmap_free,
+ .query_gid = bng_re_query_gid,
+ .add_gid = bng_re_add_gid,
+ .del_gid = bng_re_del_gid,
INIT_RDMA_OBJ_SIZE(ib_ucontext, bng_re_ucontext, ib_uctx),
};
@@ -296,6 +299,11 @@ static int bng_re_query_hwrm_version(struct bng_re_dev *rdev)
return 0;
}
+static void bng_re_free_res(struct bng_re_dev *rdev)
+{
+ bng_sp_cleanup_sgid_tbl(&rdev->bng_res.sgid_tbl);
+}
+
static void bng_re_dev_uninit(struct bng_re_dev *rdev)
{
int rc;
@@ -305,6 +313,8 @@ static void bng_re_dev_uninit(struct bng_re_dev *rdev)
bng_re_debugfs_rem_pdev(rdev);
+ bng_re_free_res(rdev);
+
if (test_and_clear_bit(BNG_RE_FLAG_TBLS_ALLOC_INITED, &rdev->flags))
bng_res_free_tbls(&rdev->bng_res);
diff --git a/drivers/infiniband/hw/bng_re/bng_re.h b/drivers/infiniband/hw/bng_re/bng_re.h
index 19b1e69dfb1e..0c0e362164d7 100644
--- a/drivers/infiniband/hw/bng_re/bng_re.h
+++ b/drivers/infiniband/hw/bng_re/bng_re.h
@@ -62,6 +62,8 @@ enum {
#define ROCE_MPC_MAX_LATENCY_SEC_SLAB_INDEX BNG_RE_MPC_MAX_LATENCY_SEC_SLAB_INDEX
#define ROCE_MPC_MAX_LATENCY_MSEC_SLAB_INDEX BNG_RE_MPC_MAX_LATENCY_MSEC_SLAB_INDEX
+typedef u32 PORT_NUM;
+
struct bng_re_mpc_poll_info {
u64 bm_last_poll_work_poll_jiffies_start;
u64 bm_last_poll_work_poll_jiffies_end;
diff --git a/drivers/infiniband/hw/bng_re/bng_res.c b/drivers/infiniband/hw/bng_re/bng_res.c
index fe261354c4cb..a169ed2bb16b 100644
--- a/drivers/infiniband/hw/bng_re/bng_res.c
+++ b/drivers/infiniband/hw/bng_re/bng_res.c
@@ -368,6 +368,7 @@ static int bng_res_alloc_dpi_tbl(struct bng_re_res *res,
void bng_res_free_tbls(struct bng_re_res *res)
{
bng_res_free_dpi_tbl(&res->dpi_tbl);
+ bng_sp_free_sgid_tbl(&res->sgid_tbl);
}
int bng_res_alloc_init_tbls(struct bng_re_res *res)
@@ -376,7 +377,17 @@ int bng_res_alloc_init_tbls(struct bng_re_res *res)
rc = bng_res_alloc_dpi_tbl(res, res->dattr);
if (rc)
- dev_err(&res->pdev->dev, "DPI tbl alloc failed\n");
+ return rc;
+
+ rc = bng_sp_alloc_sgid_tbl(&res->sgid_tbl,
+ res->dattr->max_sgid);
+ if (rc)
+ goto free_dpi_tbl;
+
+ return 0;
+
+free_dpi_tbl:
+ bng_res_free_dpi_tbl(&res->dpi_tbl);
return rc;
}
diff --git a/drivers/infiniband/hw/bng_re/bng_res.h b/drivers/infiniband/hw/bng_re/bng_res.h
index 11e453d4db36..58a6012998c1 100644
--- a/drivers/infiniband/hw/bng_re/bng_res.h
+++ b/drivers/infiniband/hw/bng_re/bng_res.h
@@ -38,6 +38,18 @@
#define RCFW_DBR_PCI_BAR_REGION 2
+struct bng_re_gid {
+ u8 data[16];
+};
+
+struct bng_re_sgid_tbl {
+ u16 active;
+ u16 max; /* number of entries */
+ struct bng_re_gid_info *tbl; /* array gid entries */
+ u16 *hw_id; /* array of hw gid indices */
+ void *ctx; /* array of IB context pointers */
+};
+
struct bng_re_reg_desc {
u8 bar_id;
resource_size_t bar_base;
@@ -193,8 +205,12 @@ struct bng_re_res {
struct mutex dpi_tbl_lock;
struct xid_manager *qp_xids;
struct xid_manager *ah_xids;
+ struct bng_re_sgid_tbl sgid_tbl;
+ bool prio;
};
+#define to_bng_re_res(ptr, member) container_of(ptr, struct bng_re_res, member)
+
static inline void *bng_re_get_qe(struct bng_re_hwq *hwq,
u32 indx, u64 *pg)
{
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.c b/drivers/infiniband/hw/bng_re/bng_sp.c
index 83099e05328d..8401d1a7ff28 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.c
+++ b/drivers/infiniband/hw/bng_re/bng_sp.c
@@ -7,6 +7,15 @@
#include "bng_fw.h"
#include "bng_sp.h"
#include "bng_tlv.h"
+#include "bng_re.h"
+
+const struct bng_re_gid bng_re_gid_zero = {{0,}};
+
+static inline bool is_zero_gid(struct bng_re_gid *gid)
+{
+ return !memcmp(gid, &bng_re_gid_zero, sizeof(bng_re_gid_zero));
+}
+
static bool bng_re_is_atomic_cap(struct bng_re_rcfw *rcfw)
{
@@ -129,3 +138,200 @@ int bng_re_get_dev_attr(struct bng_re_rcfw *rcfw)
sbuf.sb, sbuf.dma_addr);
return rc;
}
+
+int bng_sp_alloc_sgid_tbl(struct bng_re_sgid_tbl *sgid_tbl, u16 size)
+{
+ u16 i;
+
+ sgid_tbl->tbl = kcalloc(size, sizeof(*sgid_tbl->tbl), GFP_KERNEL);
+ if (!sgid_tbl->tbl)
+ return -ENOMEM;
+
+ sgid_tbl->hw_id = kcalloc(size, sizeof(u16), GFP_KERNEL);
+ if (!sgid_tbl->hw_id)
+ goto free_tbl;
+
+ sgid_tbl->ctx = kcalloc(size, sizeof(void *), GFP_KERNEL);
+ if (!sgid_tbl->ctx)
+ goto free_hwid;
+
+ for (i = 0; i < size; i++)
+ sgid_tbl->tbl[i].vlan_id = 0xFFFF;
+ sgid_tbl->max = size;
+ memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
+
+ return 0;
+
+free_hwid:
+ kfree(sgid_tbl->hw_id);
+ sgid_tbl->hw_id = NULL;
+free_tbl:
+ kfree(sgid_tbl->tbl);
+ sgid_tbl->tbl = NULL;
+ return -ENOMEM;
+}
+
+void bng_sp_free_sgid_tbl(struct bng_re_sgid_tbl *sgid_tbl)
+{
+ kfree(sgid_tbl->tbl);
+ kfree(sgid_tbl->hw_id);
+ kfree(sgid_tbl->ctx);
+ memset(sgid_tbl, 0, sizeof(*sgid_tbl));
+}
+
+void bng_sp_cleanup_sgid_tbl(struct bng_re_sgid_tbl *sgid_tbl)
+{
+ void **ctx_tbl = sgid_tbl->ctx;
+ u16 i;
+
+ for (i = 0; i < sgid_tbl->max; i++) {
+ if (!is_zero_gid(&sgid_tbl->tbl[i].gid))
+ bng_sp_del_sgid(sgid_tbl, i, true);
+ kfree(ctx_tbl[i]);
+ }
+
+ memset(sgid_tbl->tbl, 0, sizeof(*sgid_tbl->tbl) * sgid_tbl->max);
+ memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
+ memset(sgid_tbl->ctx, 0, sizeof(void *) * sgid_tbl->max);
+ sgid_tbl->active = 0;
+}
+
+int bng_sp_add_sgid(struct bng_re_sgid_tbl *sgid_tbl,
+ struct bng_re_gid *gid, const u8 *smac, u16 vlan_id,
+ bool update, u32 *index, bool is_ugid, u32 stats_ctx_id)
+{
+ struct bng_re_res *res = to_bng_re_res(sgid_tbl, sgid_tbl);
+ struct bng_re_dev *rdev = to_bng_re_dev(res, bng_res);
+ int i, free_idx;
+
+ if (sgid_tbl->active == sgid_tbl->max) {
+ dev_err(&res->pdev->dev, "SGID table is full");
+ return -ENOMEM;
+ }
+
+ free_idx = sgid_tbl->max;
+ for (i = 0; i < sgid_tbl->max; i++) {
+ if (!memcmp(&sgid_tbl->tbl[i].gid, gid, sizeof(*gid)) &&
+ sgid_tbl->tbl[i].vlan_id == vlan_id) {
+ dev_dbg(&res->pdev->dev,
+ "SGID entry already exist in entry %d!",
+ i);
+ *index = i;
+ return -EALREADY;
+ }
+ if (free_idx == sgid_tbl->max &&
+ is_zero_gid(&sgid_tbl->tbl[i].gid))
+ free_idx = i;
+ }
+
+ if (free_idx == sgid_tbl->max) {
+ dev_dbg(&res->pdev->dev,
+ "SGID table is FULL but count is not MAX!");
+ return -ENOMEM;
+ }
+
+ if (update) {
+ struct creq_add_gid_resp resp = {};
+ struct bng_re_cmdqmsg msg = {};
+ struct cmdq_add_gid req = {};
+ int rc;
+
+ bng_re_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_ADD_GID, sizeof(req));
+
+ req.host_gid_index = cpu_to_le16(free_idx);
+ req.gid[0] = cpu_to_be32(((u32 *)gid->data)[3]);
+ req.gid[1] = cpu_to_be32(((u32 *)gid->data)[2]);
+ req.gid[2] = cpu_to_be32(((u32 *)gid->data)[1]);
+ req.gid[3] = cpu_to_be32(((u32 *)gid->data)[0]);
+
+ if (vlan_id != 0xFFFF || res->prio) {
+ if (vlan_id != 0xFFFF)
+ req.vlan = cpu_to_le16(vlan_id &
+ CMDQ_ADD_GID_VLAN_VLAN_ID_MASK);
+ req.vlan |=
+ cpu_to_le16(CMDQ_ADD_GID_VLAN_TPID_TPID_8100 |
+ CMDQ_ADD_GID_VLAN_VLAN_EN);
+ }
+
+ /* MAC in network format */
+ req.src_mac[0] = cpu_to_be16(((u16 *)smac)[0]);
+ req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]);
+ req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]);
+
+ if (is_ugid)
+ req.stats_ctx = cpu_to_le16(CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID |
+ (u16)stats_ctx_id);
+
+ bng_re_fill_cmdqmsg(&msg, &req, &resp, NULL,
+ sizeof(req), sizeof(resp), 0);
+ rc = bng_re_rcfw_send_message(&rdev->rcfw, &msg);
+ if (rc)
+ return rc;
+ sgid_tbl->hw_id[free_idx] = le32_to_cpu(resp.xid);
+ }
+
+ /* Add GID to the sgid_tbl */
+ memcpy(&sgid_tbl->tbl[free_idx].gid, gid, sizeof(*gid));
+ sgid_tbl->tbl[free_idx].vlan_id = vlan_id;
+ sgid_tbl->active++;
+
+ dev_dbg(&res->pdev->dev,
+ "SGID added hw_id[0x%x] = 0x%x active = 0x%x",
+ free_idx, sgid_tbl->hw_id[free_idx], sgid_tbl->active);
+
+ *index = free_idx;
+ return 0;
+}
+
+int bng_sp_del_sgid(struct bng_re_sgid_tbl *sgid_tbl, u32 index, bool update)
+{
+ struct bng_re_res *res = to_bng_re_res(sgid_tbl, sgid_tbl);
+ u16 hw_id;
+
+ if (!sgid_tbl->active) {
+ dev_err(&res->pdev->dev, "SGID table has no active entries");
+ return -EINVAL;
+ }
+ if (index >= sgid_tbl->max) {
+ dev_err(&res->pdev->dev, "Invalid index: %d", index);
+ return -EINVAL;
+ }
+
+ /* Remove GID from the SGID table */
+ hw_id = sgid_tbl->hw_id[index];
+ if (update) {
+ struct bng_re_dev *rdev = to_bng_re_dev(res, bng_res);
+ struct creq_delete_gid_resp resp = {};
+ struct cmdq_delete_gid req = {};
+ struct bng_re_cmdqmsg msg = {};
+ int rc;
+
+ if (hw_id == 0xFFFF) {
+ dev_err(&res->pdev->dev,
+ "GID entry contains an invalid HW id\n");
+ return -EINVAL;
+ }
+
+ bng_re_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_DELETE_GID, sizeof(req));
+ req.gid_index = cpu_to_le16(hw_id);
+ bng_re_fill_cmdqmsg(&msg, &req, &resp, NULL,
+ sizeof(req), sizeof(resp), 0);
+ rc = bng_re_rcfw_send_message(&rdev->rcfw, &msg);
+ if (rc)
+ return rc;
+ }
+
+ memcpy(&sgid_tbl->tbl[index].gid, &bng_re_gid_zero,
+ sizeof(bng_re_gid_zero));
+ sgid_tbl->tbl[index].vlan_id = 0xFFFF;
+ sgid_tbl->hw_id[index] = 0xFFFF;
+ sgid_tbl->active--;
+
+ dev_dbg(&res->pdev->dev,
+ "SGID deleted hw_id[0x%x] = 0x%x active = 0x%x\n",
+ index, hw_id, sgid_tbl->active);
+ return 0;
+}
+
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.h b/drivers/infiniband/hw/bng_re/bng_sp.h
index 4dac43540f54..05c2749419d8 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.h
+++ b/drivers/infiniband/hw/bng_re/bng_sp.h
@@ -48,5 +48,21 @@ struct bng_re_dev_attr {
u32 max_dpi;
};
+struct bng_re_gid_info {
+ struct bng_re_gid gid;
+ u16 vlan_id;
+};
+
int bng_re_get_dev_attr(struct bng_re_rcfw *rcfw);
+int bng_sp_alloc_sgid_tbl(struct bng_re_sgid_tbl *sgid_tbl, u16 size);
+
+void bng_sp_free_sgid_tbl(struct bng_re_sgid_tbl *sgid_tbl);
+
+void bng_sp_cleanup_sgid_tbl(struct bng_re_sgid_tbl *sgid_tbl);
+
+int bng_sp_add_sgid(struct bng_re_sgid_tbl *sgid_tbl,
+ struct bng_re_gid *gid, const u8 *smac, u16 vlan_id,
+ bool update, u32 *index, bool is_ugid, u32 stats_ctx_id);
+
+int bng_sp_del_sgid(struct bng_re_sgid_tbl *sgid_tbl, u32 index, bool update);
#endif
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.c b/drivers/infiniband/hw/bng_re/bng_verbs.c
index ef2b5bd65e54..59c7d95830f4 100644
--- a/drivers/infiniband/hw/bng_re/bng_verbs.c
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.c
@@ -417,3 +417,112 @@ void bng_re_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
kfree(bng_entry);
}
+
+static int bng_get_sgid(struct bng_re_res *res,
+ struct bng_re_sgid_tbl *sgid_tbl, int index,
+ struct bng_re_gid *gid)
+{
+ if (index > sgid_tbl->max) {
+ dev_err(&res->pdev->dev,
+ "Index %d exceeded SGID table max (%d)",
+ index, sgid_tbl->max);
+ return -EINVAL;
+ }
+ memcpy(gid, &sgid_tbl->tbl[index].gid, sizeof(*gid));
+ return 0;
+}
+
+int bng_re_query_gid(struct ib_device *ibdev, PORT_NUM port_num,
+ int index, union ib_gid *gid)
+{
+ struct bng_re_dev *rdev = to_bng_re_dev(ibdev, ibdev);
+ int rc = 0;
+
+ memset(gid, 0, sizeof(*gid));
+ rc = bng_get_sgid(&rdev->bng_res,
+ &rdev->bng_res.sgid_tbl, index,
+ (struct bng_re_gid *)gid);
+
+ return rc;
+}
+
+int bng_re_add_gid(const struct ib_gid_attr *attr, void **context)
+{
+ struct bng_re_dev *rdev = to_bng_re_dev(attr->device, ibdev);
+ struct bng_re_sgid_tbl *sgid_tbl = &rdev->bng_res.sgid_tbl;
+ struct bng_re_gid *gid = (struct bng_re_gid *)&attr->gid;
+ struct bng_re_gid_ctx *ctx, **ctx_tbl;
+ u8 mac[ETH_ALEN] = {0};
+ u16 vlan_id = 0xFFFF;
+ u32 tbl_idx = 0;
+ int rc;
+
+ rc = rdma_read_gid_l2_fields(attr, &vlan_id, mac);
+ if (rc)
+ return rc;
+
+ rc = bng_sp_add_sgid(sgid_tbl, gid, mac, vlan_id,
+ true, &tbl_idx, false, 0);
+ if (rc == -EALREADY) {
+ ctx_tbl = sgid_tbl->ctx;
+ ctx_tbl[tbl_idx]->refcnt++;
+ *context = ctx_tbl[tbl_idx];
+ return 0;
+ }
+
+ if (rc < 0) {
+ dev_err(rdev_to_dev(rdev), "Failed to add GID: rc=%#x", rc);
+ return rc;
+ }
+
+ ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
+ if (!ctx)
+ return -ENOMEM;
+
+ ctx_tbl = sgid_tbl->ctx;
+ ctx->idx = tbl_idx;
+ ctx->refcnt = 1;
+ ctx_tbl[tbl_idx] = ctx;
+ *context = ctx;
+
+ return 0;
+}
+
+int bng_re_del_gid(const struct ib_gid_attr *attr, void **context)
+{
+ struct bng_re_dev *rdev = to_bng_re_dev(attr->device, ibdev);
+ struct bng_re_sgid_tbl *sgid_tbl = &rdev->bng_res.sgid_tbl;
+ struct bng_re_gid_ctx *ctx, **ctx_tbl;
+ int rc = 0;
+
+ ctx_tbl = sgid_tbl->ctx;
+ ctx = *context;
+ if (!ctx) {
+ dev_err(rdev_to_dev(rdev), "GID entry has no ctx");
+ return -EINVAL;
+ }
+ if (!sgid_tbl->active) {
+ dev_err(rdev_to_dev(rdev), "SGID table has no active entries");
+ return -EINVAL;
+ }
+ if (ctx->idx >= sgid_tbl->max || ctx != ctx_tbl[ctx->idx]) {
+ dev_err(rdev_to_dev(rdev), "GID entry is invalid ctx");
+ return -EINVAL;
+ }
+
+ /* Delete the entry from the hardware */
+ ctx->refcnt--;
+ if (!ctx->refcnt) {
+ rc = bng_sp_del_sgid(sgid_tbl, ctx->idx, true);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to remove GID: %#x", rc);
+ } else {
+ ctx_tbl[ctx->idx] = NULL;
+ kfree(ctx);
+ }
+ }
+
+ return rc;
+}
+
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.h b/drivers/infiniband/hw/bng_re/bng_verbs.h
index 565571b2f1fe..eba56cf42672 100644
--- a/drivers/infiniband/hw/bng_re/bng_verbs.h
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.h
@@ -26,6 +26,11 @@ struct bng_re_user_mmap_entry {
u8 mmap_flag;
};
+struct bng_re_gid_ctx {
+ u32 idx;
+ u32 refcnt;
+};
+
int bng_re_query_device(struct ib_device *ibdev, struct ib_device_attr *ib_attr,
struct ib_udata *udata);
int bng_re_modify_device(struct ib_device *ibdev, int device_modify_mask,
@@ -47,6 +52,9 @@ void bng_re_dealloc_ucontext(struct ib_ucontext *context);
void bng_re_disassociate_ucontext(struct ib_ucontext *ibcontext);
int bng_re_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
void bng_re_mmap_free(struct rdma_user_mmap_entry *rdma_entry);
-
+int bng_re_query_gid(struct ib_device *ibdev, u32 port_num, int index,
+ union ib_gid *gid);
+int bng_re_add_gid(const struct ib_gid_attr *attr, void **context);
+int bng_re_del_gid(const struct ib_gid_attr *attr, void **context);
#endif /* __BNG_RE_VERBS_H__ */
--
2.43.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 08/15] RDMA/bng_re: Add PD verbs
2026-09-04 10:43 [PATCH 00/15] Add BNG_RE control path verbs Siva Reddy Kallam
` (6 preceding siblings ...)
2026-09-04 10:43 ` [PATCH 07/15] RDMA/bng_re: Add GID verbs Siva Reddy Kallam
@ 2026-09-04 10:43 ` Siva Reddy Kallam
2026-09-04 10:43 ` [PATCH 09/15] RDMA/bng_re: Add MR verbs Siva Reddy Kallam
` (6 subsequent siblings)
14 siblings, 0 replies; 17+ messages in thread
From: Siva Reddy Kallam @ 2026-09-04 10:43 UTC (permalink / raw)
To: leonro, jgg, davem, edumazet, kuba, pabeni, andrew+netdev, horms
Cc: netdev, linux-kernel, linux-rdma, Suresh Rupanagudi,
Siva Reddy Kallam
From: Suresh Rupanagudi <suresh.rupanagudi@broadcom.com>
This patch adds below verbs.
- bng_re_alloc_pd
- bng_re_dealloc_pd
Signed-off-by: Suresh Rupanagudi <suresh.rupanagudi@broadcom.com>
Signed-off-by: Siva Reddy Kallam <siva.kallam@broadcom.com>
---
drivers/infiniband/hw/bng_re/bng_dev.c | 3 ++
drivers/infiniband/hw/bng_re/bng_res.c | 38 +++++++++++++++++++++++-
drivers/infiniband/hw/bng_re/bng_res.h | 9 ++++++
drivers/infiniband/hw/bng_re/bng_sp.c | 36 ++++++++++++++++++++++
drivers/infiniband/hw/bng_re/bng_sp.h | 11 +++++++
drivers/infiniband/hw/bng_re/bng_verbs.c | 34 +++++++++++++++++++++
drivers/infiniband/hw/bng_re/bng_verbs.h | 8 +++++
7 files changed, 138 insertions(+), 1 deletion(-)
diff --git a/drivers/infiniband/hw/bng_re/bng_dev.c b/drivers/infiniband/hw/bng_re/bng_dev.c
index 9ce8e1bb420a..b8a50c8cd299 100644
--- a/drivers/infiniband/hw/bng_re/bng_dev.c
+++ b/drivers/infiniband/hw/bng_re/bng_dev.c
@@ -44,6 +44,9 @@ static const struct ib_device_ops bng_re_dev_ops = {
.query_gid = bng_re_query_gid,
.add_gid = bng_re_add_gid,
.del_gid = bng_re_del_gid,
+ .alloc_pd = bng_re_alloc_pd,
+ .dealloc_pd = bng_re_dealloc_pd,
+ INIT_RDMA_OBJ_SIZE(ib_pd, bng_re_pd, ib_pd),
INIT_RDMA_OBJ_SIZE(ib_ucontext, bng_re_ucontext, ib_uctx),
};
diff --git a/drivers/infiniband/hw/bng_re/bng_res.c b/drivers/infiniband/hw/bng_re/bng_res.c
index a169ed2bb16b..8bf6e8efa182 100644
--- a/drivers/infiniband/hw/bng_re/bng_res.c
+++ b/drivers/infiniband/hw/bng_re/bng_res.c
@@ -365,8 +365,38 @@ static int bng_res_alloc_dpi_tbl(struct bng_re_res *res,
return 0;
}
+static void bng_res_free_pd_tbl(struct bng_re_pd_tbl *pdt)
+{
+ kfree(pdt->tbl);
+ pdt->tbl = NULL;
+ pdt->max = 0;
+}
+
+static int bng_res_alloc_pd_tbl(struct bng_re_res *res,
+ struct bng_re_dev_attr *dev_attr)
+{
+ struct bng_re_pd_tbl *pdt = &res->pd_tbl;
+ u32 max = dev_attr->max_pd;
+ u32 bytes;
+
+ max++; /* one extra slot, reserved so it's never allocated */
+ bytes = BITS_TO_LONGS(max) * sizeof(unsigned long);
+
+ pdt->tbl = kmalloc(bytes, GFP_KERNEL);
+ if (!pdt->tbl)
+ return -ENOMEM;
+
+ pdt->max = max;
+ memset(pdt->tbl, 0xFF, bytes); /* mark every pd id as available */
+ clear_bit(pdt->max - 1, pdt->tbl); /* reserve the last pd id */
+ mutex_init(&res->pd_tbl_lock);
+
+ return 0;
+}
+
void bng_res_free_tbls(struct bng_re_res *res)
{
+ bng_res_free_pd_tbl(&res->pd_tbl);
bng_res_free_dpi_tbl(&res->dpi_tbl);
bng_sp_free_sgid_tbl(&res->sgid_tbl);
}
@@ -375,10 +405,14 @@ int bng_res_alloc_init_tbls(struct bng_re_res *res)
{
int rc;
- rc = bng_res_alloc_dpi_tbl(res, res->dattr);
+ rc = bng_res_alloc_pd_tbl(res, res->dattr);
if (rc)
return rc;
+ rc = bng_res_alloc_dpi_tbl(res, res->dattr);
+ if (rc)
+ goto free_pd_tbl;
+
rc = bng_sp_alloc_sgid_tbl(&res->sgid_tbl,
res->dattr->max_sgid);
if (rc)
@@ -388,6 +422,8 @@ int bng_res_alloc_init_tbls(struct bng_re_res *res)
free_dpi_tbl:
bng_res_free_dpi_tbl(&res->dpi_tbl);
+free_pd_tbl:
+ bng_res_free_pd_tbl(&res->pd_tbl);
return rc;
}
diff --git a/drivers/infiniband/hw/bng_re/bng_res.h b/drivers/infiniband/hw/bng_re/bng_res.h
index 58a6012998c1..72afe51c6bfa 100644
--- a/drivers/infiniband/hw/bng_re/bng_res.h
+++ b/drivers/infiniband/hw/bng_re/bng_res.h
@@ -196,6 +196,12 @@ struct bng_re_dpi_tbl {
void __iomem *priv_db;
};
+/* PD table */
+struct bng_re_pd_tbl {
+ unsigned long *tbl;
+ u32 max;
+};
+
struct bng_re_res {
struct pci_dev *pdev;
struct bng_re_chip_ctx *cctx;
@@ -207,6 +213,9 @@ struct bng_re_res {
struct xid_manager *ah_xids;
struct bng_re_sgid_tbl sgid_tbl;
bool prio;
+ struct bng_re_pd_tbl pd_tbl;
+ /* Serialize access to PD table */
+ struct mutex pd_tbl_lock;
};
#define to_bng_re_res(ptr, member) container_of(ptr, struct bng_re_res, member)
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.c b/drivers/infiniband/hw/bng_re/bng_sp.c
index 8401d1a7ff28..97768a25c0e4 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.c
+++ b/drivers/infiniband/hw/bng_re/bng_sp.c
@@ -335,3 +335,39 @@ int bng_sp_del_sgid(struct bng_re_sgid_tbl *sgid_tbl, u32 index, bool update)
return 0;
}
+/* PD allocation/de-allocation functions from Gerrit */
+int bng_sp_alloc_pd(struct bng_re_res *res, struct bng_sp_pd *pd)
+{
+ struct bng_re_pd_tbl *pdt = &res->pd_tbl;
+ u32 bit_num;
+
+ mutex_lock(&res->pd_tbl_lock);
+ bit_num = find_first_bit(pdt->tbl, pdt->max);
+ if (bit_num >= pdt->max - 1) {
+ mutex_unlock(&res->pd_tbl_lock);
+ return -ENOMEM;
+ }
+
+ /* Found unused PD - mark it as allocated */
+ clear_bit(bit_num, pdt->tbl);
+ pd->id = bit_num;
+
+ mutex_unlock(&res->pd_tbl_lock);
+ return 0;
+}
+
+int bng_sp_dealloc_pd(struct bng_re_res *res,
+ struct bng_re_pd_tbl *pdt,
+ struct bng_sp_pd *pd)
+{
+ mutex_lock(&res->pd_tbl_lock);
+ if (pd->id >= pdt->max - 1 || test_and_set_bit(pd->id, pdt->tbl)) {
+ dev_warn(&res->pdev->dev, "Freeing an unused PD? pdn = %d\n",
+ pd->id);
+ mutex_unlock(&res->pd_tbl_lock);
+ return -EINVAL;
+ }
+ pd->id = pdt->max - 1;
+ mutex_unlock(&res->pd_tbl_lock);
+ return 0;
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.h b/drivers/infiniband/hw/bng_re/bng_sp.h
index 05c2749419d8..136703942e1f 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.h
+++ b/drivers/infiniband/hw/bng_re/bng_sp.h
@@ -53,6 +53,10 @@ struct bng_re_gid_info {
u16 vlan_id;
};
+struct bng_sp_pd {
+ u32 id;
+};
+
int bng_re_get_dev_attr(struct bng_re_rcfw *rcfw);
int bng_sp_alloc_sgid_tbl(struct bng_re_sgid_tbl *sgid_tbl, u16 size);
@@ -65,4 +69,11 @@ int bng_sp_add_sgid(struct bng_re_sgid_tbl *sgid_tbl,
bool update, u32 *index, bool is_ugid, u32 stats_ctx_id);
int bng_sp_del_sgid(struct bng_re_sgid_tbl *sgid_tbl, u32 index, bool update);
+
+int bng_sp_alloc_pd(struct bng_re_res *res, struct bng_sp_pd *pd);
+
+int bng_sp_dealloc_pd(struct bng_re_res *res,
+ struct bng_re_pd_tbl *pdt,
+ struct bng_sp_pd *pd);
+
#endif
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.c b/drivers/infiniband/hw/bng_re/bng_verbs.c
index 59c7d95830f4..140a972a0316 100644
--- a/drivers/infiniband/hw/bng_re/bng_verbs.c
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.c
@@ -526,3 +526,37 @@ int bng_re_del_gid(const struct ib_gid_attr *attr, void **context)
return rc;
}
+int bng_re_alloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
+{
+ struct bng_re_pd *pd = container_of(ib_pd, struct bng_re_pd, ib_pd);
+ struct ib_device *ibdev = ib_pd->device;
+ struct bng_re_dev *rdev = container_of(ibdev, struct bng_re_dev, ibdev);
+ int rc;
+
+ pd->rdev = rdev;
+
+ rc = bng_sp_alloc_pd(&rdev->bng_res, &pd->sp_pd);
+ if (rc) {
+ ibdev_err(&rdev->ibdev,
+ "Failed to allocate HW PD\n");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+int bng_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
+{
+ struct bng_re_pd *pd = container_of(ib_pd, struct bng_re_pd, ib_pd);
+ struct bng_re_dev *rdev = pd->rdev;
+ int rc;
+
+ rc = bng_sp_dealloc_pd(&rdev->bng_res, &rdev->bng_res.pd_tbl,
+ &pd->sp_pd);
+ if (rc) {
+ ibdev_err(&rdev->ibdev,
+ "Dealloc PD failed: pdid=%u, rc=%d\n",
+ pd->sp_pd.id, rc);
+ }
+ return 0;
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.h b/drivers/infiniband/hw/bng_re/bng_verbs.h
index eba56cf42672..ea017f70142e 100644
--- a/drivers/infiniband/hw/bng_re/bng_verbs.h
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.h
@@ -31,6 +31,12 @@ struct bng_re_gid_ctx {
u32 refcnt;
};
+struct bng_re_pd {
+ struct ib_pd ib_pd;
+ struct bng_re_dev *rdev;
+ struct bng_sp_pd sp_pd;
+};
+
int bng_re_query_device(struct ib_device *ibdev, struct ib_device_attr *ib_attr,
struct ib_udata *udata);
int bng_re_modify_device(struct ib_device *ibdev, int device_modify_mask,
@@ -56,5 +62,7 @@ int bng_re_query_gid(struct ib_device *ibdev, u32 port_num, int index,
union ib_gid *gid);
int bng_re_add_gid(const struct ib_gid_attr *attr, void **context);
int bng_re_del_gid(const struct ib_gid_attr *attr, void **context);
+int bng_re_alloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata);
+int bng_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata);
#endif /* __BNG_RE_VERBS_H__ */
--
2.43.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 09/15] RDMA/bng_re: Add MR verbs
2026-09-04 10:43 [PATCH 00/15] Add BNG_RE control path verbs Siva Reddy Kallam
` (7 preceding siblings ...)
2026-09-04 10:43 ` [PATCH 08/15] RDMA/bng_re: Add PD verbs Siva Reddy Kallam
@ 2026-09-04 10:43 ` Siva Reddy Kallam
2026-09-04 10:43 ` [PATCH 10/15] RDMA/bng_re: Add CQ verbs Siva Reddy Kallam
` (5 subsequent siblings)
14 siblings, 0 replies; 17+ messages in thread
From: Siva Reddy Kallam @ 2026-09-04 10:43 UTC (permalink / raw)
To: leonro, jgg, davem, edumazet, kuba, pabeni, andrew+netdev, horms
Cc: netdev, linux-kernel, linux-rdma, Suresh Rupanagudi,
Siva Reddy Kallam
From: Suresh Rupanagudi <suresh.rupanagudi@broadcom.com>
This patch adds below verbs.
- bng_re_get_dma_mr
- bng_re_alloc_mr
- bng_re_map_mr_sg
- bng_re_reg_user_mr
- bng_re_reg_user_mr_dmabuf
- bng_re_dereg_mr
Signed-off-by: Suresh Rupanagudi <suresh.rupanagudi@broadcom.com>
Signed-off-by: Siva Reddy Kallam <siva.kallam@broadcom.com>
---
drivers/infiniband/hw/bng_re/bng_dev.c | 8 +
drivers/infiniband/hw/bng_re/bng_res.h | 15 +-
drivers/infiniband/hw/bng_re/bng_sp.c | 210 ++++++++++++++
drivers/infiniband/hw/bng_re/bng_sp.h | 52 ++++
drivers/infiniband/hw/bng_re/bng_verbs.c | 333 +++++++++++++++++++++++
drivers/infiniband/hw/bng_re/bng_verbs.h | 27 ++
6 files changed, 640 insertions(+), 5 deletions(-)
diff --git a/drivers/infiniband/hw/bng_re/bng_dev.c b/drivers/infiniband/hw/bng_re/bng_dev.c
index b8a50c8cd299..44c278a93778 100644
--- a/drivers/infiniband/hw/bng_re/bng_dev.c
+++ b/drivers/infiniband/hw/bng_re/bng_dev.c
@@ -46,6 +46,12 @@ static const struct ib_device_ops bng_re_dev_ops = {
.del_gid = bng_re_del_gid,
.alloc_pd = bng_re_alloc_pd,
.dealloc_pd = bng_re_dealloc_pd,
+ .get_dma_mr = bng_re_get_dma_mr,
+ .alloc_mr = bng_re_alloc_mr,
+ .map_mr_sg = bng_re_map_mr_sg,
+ .reg_user_mr = bng_re_reg_user_mr,
+ .reg_user_mr_dmabuf = bng_re_reg_user_mr_dmabuf,
+ .dereg_mr = bng_re_dereg_mr,
INIT_RDMA_OBJ_SIZE(ib_pd, bng_re_pd, ib_pd),
INIT_RDMA_OBJ_SIZE(ib_ucontext, bng_re_ucontext, ib_uctx),
};
@@ -91,6 +97,7 @@ static void bng_re_destroy_chip_ctx(struct bng_re_dev *rdev)
chip_ctx = rdev->chip_ctx;
rdev->chip_ctx = NULL;
rdev->rcfw.res = NULL;
+ rdev->bng_res.rcfw = NULL;
rdev->bng_res.cctx = NULL;
rdev->bng_res.pdev = NULL;
kfree(chip_ctx);
@@ -117,6 +124,7 @@ static int bng_re_setup_chip_ctx(struct bng_re_dev *rdev)
aux_dev = rdev->aux_dev;
rdev->bng_res.pdev = aux_dev->pdev;
rdev->rcfw.res = &rdev->bng_res;
+ rdev->bng_res.rcfw = &rdev->rcfw;
chip_ctx = kzalloc_obj(*chip_ctx);
if (!chip_ctx)
return -ENOMEM;
diff --git a/drivers/infiniband/hw/bng_re/bng_res.h b/drivers/infiniband/hw/bng_re/bng_res.h
index 72afe51c6bfa..cb0f302fc06a 100644
--- a/drivers/infiniband/hw/bng_re/bng_res.h
+++ b/drivers/infiniband/hw/bng_re/bng_res.h
@@ -38,6 +38,8 @@
#define RCFW_DBR_PCI_BAR_REGION 2
+#define BNG_RE_FR_PMR 0x80000000
+
struct bng_re_gid {
u8 data[16];
};
@@ -123,14 +125,16 @@ enum bng_re_pbl_lvl {
enum bng_re_hwq_type {
BNG_HWQ_TYPE_CTX,
- BNG_HWQ_TYPE_QUEUE
+ BNG_HWQ_TYPE_QUEUE,
+ BNG_HWQ_TYPE_MR
};
struct bng_re_sg_info {
- u32 npages;
- u32 pgshft;
- u32 pgsize;
- bool nopte;
+ u32 npages;
+ u32 pgshft;
+ u32 pgsize;
+ bool nopte;
+ struct ib_umem *umem;
};
struct bng_re_hwq_attr {
@@ -216,6 +220,7 @@ struct bng_re_res {
struct bng_re_pd_tbl pd_tbl;
/* Serialize access to PD table */
struct mutex pd_tbl_lock;
+ struct bng_re_rcfw *rcfw;
};
#define to_bng_re_res(ptr, member) container_of(ptr, struct bng_re_res, member)
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.c b/drivers/infiniband/hw/bng_re/bng_sp.c
index 97768a25c0e4..cd33ff32845a 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.c
+++ b/drivers/infiniband/hw/bng_re/bng_sp.c
@@ -371,3 +371,213 @@ int bng_sp_dealloc_pd(struct bng_re_res *res,
mutex_unlock(&res->pd_tbl_lock);
return 0;
}
+
+/* MR allocation/de-allocation functions */
+int bng_sp_alloc_mrw(struct bng_re_res *res, struct bng_re_mrw *mrw)
+{
+ struct bng_re_rcfw *rcfw = res->rcfw;
+ struct creq_allocate_mrw_resp resp = {};
+ struct bng_re_cmdqmsg msg = {};
+ struct cmdq_allocate_mrw req = {};
+ int rc;
+
+ bng_re_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_ALLOCATE_MRW,
+ sizeof(req));
+
+ req.pd_id = cpu_to_le32(mrw->pd->id);
+ req.mrw_flags = mrw->type;
+
+ if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR &&
+ mrw->access_flags & BNG_RE_FR_PMR) ||
+ mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A ||
+ mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)
+ req.access = CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY;
+ req.mrw_handle = cpu_to_le64(mrw);
+
+ bng_re_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bng_re_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ return rc;
+
+ if (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 ||
+ mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A ||
+ mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)
+ mrw->rkey = le32_to_cpu(resp.xid);
+ else
+ mrw->lkey = le32_to_cpu(resp.xid);
+
+ return 0;
+}
+
+int bng_sp_alloc_fast_reg_page_list(struct bng_re_res *res,
+ struct bng_re_frpl *frpl,
+ int max_pg_ptrs)
+{
+ struct bng_re_hwq_attr hwq_attr = {};
+ struct bng_re_sg_info sginfo = {};
+ struct bng_re_hwq *hwq;
+ int pg_ptrs, pages, rc;
+
+ pg_ptrs = roundup_pow_of_two(max_pg_ptrs);
+ pages = pg_ptrs >> MAX_PBL_LVL_1_PGS_SHIFT;
+ if (!pages)
+ pages++;
+
+ if (pages > MAX_PBL_LVL_1_PGS)
+ return -ENOMEM;
+
+ sginfo.pgsize = PAGE_SIZE;
+ sginfo.nopte = true;
+
+ hwq_attr.res = res;
+ hwq_attr.depth = pg_ptrs;
+ hwq_attr.stride = PAGE_SIZE;
+ hwq_attr.sginfo = &sginfo;
+ hwq_attr.type = BNG_HWQ_TYPE_CTX;
+
+ hwq = &frpl->hwq;
+ rc = bng_re_alloc_init_hwq(hwq, &hwq_attr);
+ if (!rc)
+ frpl->max_pg_ptrs = pg_ptrs;
+
+ return rc;
+}
+
+int bng_sp_free_mrw(struct bng_re_res *res,
+ struct bng_re_mrw *mrw)
+{
+ struct creq_deallocate_key_resp resp = {};
+ struct cmdq_deallocate_key req = {};
+ struct bng_re_rcfw *rcfw = res->rcfw;
+ struct bng_re_cmdqmsg msg = {};
+ struct bng_re_hwq *hwq;
+ int rc;
+
+ if (mrw->lkey == BNG_RE_RSVD_LKEY)
+ return 0;
+
+ bng_re_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_DEALLOCATE_KEY,
+ sizeof(req));
+
+ req.mrw_flags = mrw->type;
+
+ if (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 ||
+ mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A ||
+ mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)
+ req.key = cpu_to_le32(mrw->rkey);
+ else
+ req.key = cpu_to_le32(mrw->lkey);
+
+ bng_re_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bng_re_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ return rc;
+
+ hwq = &mrw->hwq;
+ if (hwq->max_elements)
+ bng_re_free_hwq(res, hwq);
+
+ return 0;
+}
+
+int bng_sp_free_fast_reg_page_list(struct bng_re_res *res,
+ struct bng_re_frpl *frpl)
+{
+ struct bng_re_hwq *hwq;
+
+ hwq = &frpl->hwq;
+ bng_re_free_hwq(res, hwq);
+ return 0;
+}
+
+int bng_sp_reg_mr(struct bng_re_res *res, struct bng_re_mrw *mr,
+ struct ib_umem *umem, int num_pbls, u32 buf_pg_size,
+ bool unified_mr)
+{
+ struct bng_re_rcfw *rcfw = res->rcfw;
+ struct bng_re_hwq_attr hwq_attr = {};
+ struct bng_re_sg_info sginfo = {};
+ struct creq_register_mr_resp resp = {};
+ struct bng_re_cmdqmsg msg = {};
+ struct cmdq_register_mr req = {};
+ int pages, rc;
+ u32 pg_size;
+ u16 level;
+
+ if (num_pbls) {
+ pages = roundup_pow_of_two(num_pbls);
+ /* Allocate memory for the non-leaf pages to store buf ptrs.
+ * Non-leaf pages always uses system PAGE_SIZE
+ */
+ /* Free the hwq if it already exist, must be a rereg */
+ if (mr->hwq.max_elements)
+ bng_re_free_hwq(res, &mr->hwq);
+ hwq_attr.res = res;
+ hwq_attr.depth = pages;
+ hwq_attr.stride = sizeof(dma_addr_t);
+ hwq_attr.type = BNG_HWQ_TYPE_MR;
+ hwq_attr.sginfo = &sginfo;
+ hwq_attr.sginfo->umem = umem;
+ hwq_attr.sginfo->npages = pages;
+ hwq_attr.sginfo->pgsize = buf_pg_size;
+ hwq_attr.sginfo->pgshft = ilog2(buf_pg_size);
+ rc = bng_re_alloc_init_hwq(&mr->hwq, &hwq_attr);
+ if (rc) {
+ dev_err(&res->pdev->dev,
+ "SP: Reg MR memory allocation failed\n");
+ return -ENOMEM;
+ }
+ }
+
+ bng_re_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_REGISTER_MR,
+ sizeof(req));
+
+ /* Configure the request */
+ if (mr->hwq.level == BNG_PBL_LVL_MAX) {
+ /* No PBL provided, just use system PAGE_SIZE */
+ level = 0;
+ req.pbl = 0;
+ pg_size = PAGE_SIZE;
+ } else {
+ level = mr->hwq.level;
+ req.pbl = cpu_to_le64(mr->hwq.pbl[BNG_PBL_LVL_0].pg_map_arr[0]);
+ }
+ pg_size = buf_pg_size ? buf_pg_size : PAGE_SIZE;
+ req.log2_pg_size_lvl = (level << CMDQ_REGISTER_MR_LVL_SFT) |
+ ((ilog2(pg_size) <<
+ CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT) &
+ CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK);
+ req.log2_pbl_pg_size = cpu_to_le16(((ilog2(PAGE_SIZE) <<
+ CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_SFT) &
+ CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_MASK));
+ req.access = (mr->access_flags & BNG_RE_MR_ACCESS_MASK);
+ req.va = cpu_to_le64(mr->va);
+ req.key = cpu_to_le32(mr->lkey);
+ if (unified_mr)
+ req.key = cpu_to_le32(mr->pd->id);
+ req.flags = cpu_to_le16(mr->flags);
+ req.mr_size = cpu_to_le64(mr->total_size);
+
+ bng_re_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bng_re_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ goto free_hwq;
+
+ if (unified_mr) {
+ mr->lkey = le32_to_cpu(resp.xid);
+ mr->rkey = mr->lkey;
+ }
+
+ return 0;
+
+free_hwq:
+ if (mr->hwq.max_elements)
+ bng_re_free_hwq(res, &mr->hwq);
+ return rc;
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.h b/drivers/infiniband/hw/bng_re/bng_sp.h
index 136703942e1f..6dd3ff911548 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.h
+++ b/drivers/infiniband/hw/bng_re/bng_sp.h
@@ -4,6 +4,8 @@
#ifndef __BNG_SP_H__
#define __BNG_SP_H__
+#include <rdma/ib_umem.h>
+
#include "bng_fw.h"
#define BNG_VAR_MAX_WQE 4352
@@ -57,6 +59,45 @@ struct bng_sp_pd {
u32 id;
};
+/* Access flags */
+#define BNG_RE_ACCESS_LOCAL_WRITE BIT(0)
+#define BNG_RE_ACCESS_REMOTE_READ BIT(1)
+#define BNG_RE_ACCESS_REMOTE_WRITE BIT(2)
+#define BNG_RE_ACCESS_REMOTE_ATOMIC BIT(3)
+#define BNG_RE_ACCESS_MW_BIND BIT(4)
+#define BNG_RE_ACCESS_ZERO_BASED BIT(5)
+#define BNG_RE_ACCESS_ON_DEMAND BIT(6)
+
+struct bng_re_mrw {
+ struct bng_sp_pd *pd;
+ int type;
+ u32 access_flags;
+#define BNG_RE_MR_ACCESS_MASK 0xFF
+ u32 lkey;
+ u32 rkey;
+#define BNG_RE_RSVD_LKEY 0xFFFFFFFF
+ u64 va;
+ u64 total_size;
+ u32 npages;
+ u16 flags;
+ u64 mr_handle;
+ struct bng_re_hwq hwq;
+ u16 ctx_size_sb;
+};
+
+struct bng_re_frpl {
+ int max_pg_ptrs;
+ struct bng_re_hwq hwq;
+};
+
+struct bng_sp_mrinfo {
+ struct bng_re_mrw *mrw;
+ struct bng_re_sg_info sg;
+ u64 *ptes;
+ bool is_dma;
+ bool request_relax_order;
+};
+
int bng_re_get_dev_attr(struct bng_re_rcfw *rcfw);
int bng_sp_alloc_sgid_tbl(struct bng_re_sgid_tbl *sgid_tbl, u16 size);
@@ -76,4 +117,15 @@ int bng_sp_dealloc_pd(struct bng_re_res *res,
struct bng_re_pd_tbl *pdt,
struct bng_sp_pd *pd);
+int bng_sp_alloc_mrw(struct bng_re_res *res, struct bng_re_mrw *mrw);
+int bng_sp_alloc_fast_reg_page_list(struct bng_re_res *res,
+ struct bng_re_frpl *frpl,
+ int max_pg_ptrs);
+int bng_sp_free_fast_reg_page_list(struct bng_re_res *res,
+ struct bng_re_frpl *frpl);
+int bng_sp_free_mrw(struct bng_re_res *res, struct bng_re_mrw *mrw);
+int bng_sp_reg_mr(struct bng_re_res *res, struct bng_re_mrw *mr,
+ struct ib_umem *umem, int num_pbls, u32 buf_pg_size,
+ bool unified_mr);
+
#endif
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.c b/drivers/infiniband/hw/bng_re/bng_verbs.c
index 140a972a0316..fe227d20513a 100644
--- a/drivers/infiniband/hw/bng_re/bng_verbs.c
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.c
@@ -560,3 +560,336 @@ int bng_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
}
return 0;
}
+
+/* Helper functions for MR operations */
+static inline bool _is_relaxed_ordering_supported(u16 dev_cap_ext_flags2)
+{
+ return dev_cap_ext_flags2 & CREQ_QUERY_FUNC_RESP_SB_MEMORY_REGION_RO_SUPPORTED;
+}
+
+static void bng_re_check_and_set_relaxed_ordering(struct bng_re_dev *rdev,
+ struct bng_re_mrw *mr)
+{
+ struct bng_re_dev_attr *dev_attr = rdev->dev_attr;
+
+ if (_is_relaxed_ordering_supported(dev_attr->dev_cap_flags2) &&
+ pcie_relaxed_ordering_enabled(rdev->aux_dev->pdev))
+ mr->flags |= CMDQ_REGISTER_MR_FLAGS_ENABLE_RO;
+}
+
+static int __from_ib_access_flags(int iflags)
+{
+ int qflags = 0;
+
+ if (iflags & IB_ACCESS_LOCAL_WRITE)
+ qflags |= BNG_RE_ACCESS_LOCAL_WRITE;
+ if (iflags & IB_ACCESS_REMOTE_READ)
+ qflags |= BNG_RE_ACCESS_REMOTE_READ;
+ if (iflags & IB_ACCESS_REMOTE_WRITE)
+ qflags |= BNG_RE_ACCESS_REMOTE_WRITE;
+ if (iflags & IB_ACCESS_REMOTE_ATOMIC)
+ qflags |= BNG_RE_ACCESS_REMOTE_ATOMIC;
+ if (iflags & IB_ACCESS_MW_BIND)
+ qflags |= BNG_RE_ACCESS_MW_BIND;
+ if (iflags & IB_ZERO_BASED)
+ qflags |= BNG_RE_ACCESS_ZERO_BASED;
+ if (iflags & IB_ACCESS_ON_DEMAND)
+ qflags |= BNG_RE_ACCESS_ON_DEMAND;
+
+ return qflags;
+}
+
+static inline bool _is_alloc_mr_unified(struct bng_re_dev_attr *dattr)
+{
+ return dattr->dev_cap_flags &
+ CREQ_QUERY_FUNC_RESP_SB_MR_REGISTER_ALLOC;
+}
+
+struct ib_mr *bng_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
+{
+ struct bng_re_pd *pd = container_of(ib_pd, struct bng_re_pd, ib_pd);
+ struct bng_re_dev *rdev = pd->rdev;
+ struct bng_re_mr *mr;
+ struct bng_re_res *sp_res;
+ int rc;
+
+ mr = kzalloc_obj(*mr, GFP_KERNEL);
+ if (!mr)
+ return ERR_PTR(-ENOMEM);
+
+ mr->rdev = rdev;
+ mr->sp_mr.pd = &pd->sp_pd;
+ mr->sp_mr.access_flags = __from_ib_access_flags(mr_access_flags);
+ mr->sp_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
+
+ if (mr_access_flags & IB_ACCESS_RELAXED_ORDERING)
+ bng_re_check_and_set_relaxed_ordering(rdev, &mr->sp_mr);
+
+ sp_res = &rdev->bng_res;
+ rc = bng_sp_alloc_mrw(sp_res, &mr->sp_mr);
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Allocate DMA MR failed: %d\n", rc);
+ goto free_mr;
+ }
+
+ mr->sp_mr.hwq.level = BNG_PBL_LVL_MAX;
+ /* Infinite length for DMA MR */
+ mr->sp_mr.total_size = -1;
+
+ /* Register the MR */
+ rc = bng_sp_reg_mr(sp_res, &mr->sp_mr, NULL, 0, PAGE_SIZE, false);
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Register DMA MR failed: %d\n", rc);
+ goto free_mrw;
+ }
+
+ /* Set the keys */
+ mr->ib_mr.lkey = mr->sp_mr.lkey;
+ if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |
+ IB_ACCESS_REMOTE_ATOMIC))
+ mr->ib_mr.rkey = mr->ib_mr.lkey;
+
+ return &mr->ib_mr;
+
+free_mrw:
+ bng_sp_free_mrw(sp_res, &mr->sp_mr);
+free_mr:
+ kfree(mr);
+ return ERR_PTR(rc);
+}
+
+struct ib_mr *bng_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
+ u32 max_num_sg)
+{
+ struct bng_re_pd *pd = container_of(ib_pd, struct bng_re_pd, ib_pd);
+ struct bng_re_dev *rdev = pd->rdev;
+ struct bng_re_res *sp_res;
+ struct bng_re_mr *mr = NULL;
+ int rc;
+
+ if (type != IB_MR_TYPE_MEM_REG) {
+ ibdev_dbg(&rdev->ibdev, "MR type 0x%x not supported\n", type);
+ return ERR_PTR(-EINVAL);
+ }
+
+ if (max_num_sg > MAX_PBL_LVL_1_PGS) {
+ ibdev_dbg(&rdev->ibdev, "Max SG %u exceeds limit %d\n",
+ max_num_sg, MAX_PBL_LVL_1_PGS);
+ return ERR_PTR(-EINVAL);
+ }
+
+ mr = kzalloc_obj(*mr, GFP_KERNEL);
+ if (!mr)
+ return ERR_PTR(-ENOMEM);
+
+ mr->rdev = rdev;
+ mr->sp_mr.pd = &pd->sp_pd;
+ mr->sp_mr.access_flags = BNG_RE_FR_PMR;
+ mr->sp_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
+
+ sp_res = &rdev->bng_res;
+ rc = bng_sp_alloc_mrw(sp_res, &mr->sp_mr);
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Allocate MR failed: %d\n", rc);
+ goto free_mr;
+ }
+
+ mr->ib_mr.lkey = mr->sp_mr.lkey;
+ mr->ib_mr.rkey = mr->ib_mr.lkey;
+
+ rc = bng_sp_alloc_fast_reg_page_list(sp_res,
+ &mr->sp_frpl, max_num_sg);
+ if (rc) {
+ ibdev_err(&rdev->ibdev,
+ "Failed to allocate HW FR page list: %d\n", rc);
+ goto free_mrw;
+ }
+
+ mr->pages = kcalloc(mr->sp_frpl.max_pg_ptrs, sizeof(*mr->pages),
+ GFP_KERNEL);
+ if (!mr->pages) {
+ rc = -ENOMEM;
+ goto free_frpl;
+ }
+
+ return &mr->ib_mr;
+
+free_frpl:
+ bng_sp_free_fast_reg_page_list(sp_res, &mr->sp_frpl);
+free_mrw:
+ bng_sp_free_mrw(sp_res, &mr->sp_mr);
+free_mr:
+ kfree(mr);
+ return ERR_PTR(rc);
+}
+
+static int bng_re_set_page(struct ib_mr *ib_mr, u64 addr)
+{
+ struct bng_re_mr *mr = container_of(ib_mr, struct bng_re_mr, ib_mr);
+
+ if (unlikely(mr->npages == mr->sp_frpl.max_pg_ptrs))
+ return -ENOMEM;
+
+ mr->pages[mr->npages++] = addr;
+ return 0;
+}
+
+int bng_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
+ unsigned int *sg_offset)
+{
+ struct bng_re_mr *mr = container_of(ib_mr, struct bng_re_mr, ib_mr);
+
+ mr->npages = 0;
+ return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bng_re_set_page);
+}
+
+static struct ib_mr *__bng_re_user_reg_mr(struct ib_pd *ib_pd, u64 length,
+ u64 virt_addr, int mr_access_flags,
+ struct ib_umem *umem)
+{
+ struct bng_re_pd *pd = container_of(ib_pd, struct bng_re_pd, ib_pd);
+ struct bng_re_dev *rdev = pd->rdev;
+ unsigned long page_size;
+ struct bng_re_mr *mr;
+ int umem_pgs, rc;
+
+ if (length > BNG_RE_MAX_MR_SIZE) {
+ ibdev_err(&rdev->ibdev, "MR Size: %llu > Max supported:%llu\n",
+ (unsigned long long)length,
+ (unsigned long long)BNG_RE_MAX_MR_SIZE);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ page_size = ib_umem_find_best_pgsz(umem, BNG_RE_PAGE_SIZE_SUPPORTED, virt_addr);
+ if (!page_size) {
+ ibdev_err(&rdev->ibdev, "umem page size unsupported!\n");
+ return ERR_PTR(-EINVAL);
+ }
+
+ mr = kzalloc_obj(*mr, GFP_KERNEL);
+ if (!mr)
+ return ERR_PTR(-ENOMEM);
+
+ mr->rdev = rdev;
+ mr->sp_mr.pd = &pd->sp_pd;
+ mr->sp_mr.access_flags = __from_ib_access_flags(mr_access_flags);
+ mr->sp_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR;
+
+ if (!_is_alloc_mr_unified(rdev->dev_attr)) {
+ rc = bng_sp_alloc_mrw(&rdev->bng_res, &mr->sp_mr);
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Failed to allocate MR rc = %d\n", rc);
+ rc = -EIO;
+ goto free_mr;
+ }
+ /* The fixed portion of the rkey is the same as the lkey */
+ mr->ib_mr.rkey = mr->sp_mr.rkey;
+ } else {
+ mr->sp_mr.flags = CMDQ_REGISTER_MR_FLAGS_ALLOC_MR;
+ }
+ mr->ib_umem = umem;
+ mr->sp_mr.va = virt_addr;
+ mr->sp_mr.total_size = length;
+
+ if (mr_access_flags & IB_ACCESS_RELAXED_ORDERING)
+ bng_re_check_and_set_relaxed_ordering(rdev, &mr->sp_mr);
+
+ umem_pgs = ib_umem_num_dma_blocks(umem, page_size);
+ rc = bng_sp_reg_mr(&rdev->bng_res, &mr->sp_mr, umem,
+ umem_pgs, page_size,
+ _is_alloc_mr_unified(rdev->dev_attr));
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Failed to register user MR - rc = %d\n", rc);
+ rc = -EIO;
+ goto free_mrw;
+ }
+
+ mr->ib_mr.lkey = mr->sp_mr.lkey;
+ mr->ib_mr.rkey = mr->sp_mr.lkey;
+
+ return &mr->ib_mr;
+
+free_mrw:
+ bng_sp_free_mrw(&rdev->bng_res, &mr->sp_mr);
+free_mr:
+ kfree(mr);
+ return ERR_PTR(rc);
+}
+
+struct ib_mr *bng_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
+ u64 virt_addr, int mr_access_flags,
+ struct ib_dmah *dmah,
+ struct ib_udata *udata)
+{
+ struct bng_re_pd *pd = container_of(ib_pd, struct bng_re_pd, ib_pd);
+ struct bng_re_dev *rdev = pd->rdev;
+ struct ib_umem *umem;
+ struct ib_mr *ib_mr;
+
+ umem = ib_umem_get_va(&rdev->ibdev, start, length, mr_access_flags);
+ if (IS_ERR(umem))
+ return ERR_CAST(umem);
+
+ ib_mr = __bng_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem);
+ if (IS_ERR(ib_mr))
+ ib_umem_release(umem);
+ return ib_mr;
+}
+
+struct ib_mr *bng_re_reg_user_mr_dmabuf(struct ib_pd *ib_pd, u64 start,
+ u64 length, u64 virt_addr, int fd,
+ int mr_access_flags,
+ struct ib_dmah *dmah,
+ struct uverbs_attr_bundle *attrs)
+{
+ struct bng_re_pd *pd = container_of(ib_pd, struct bng_re_pd, ib_pd);
+ struct bng_re_dev *rdev = pd->rdev;
+ struct ib_umem_dmabuf *umem_dmabuf;
+ struct ib_umem *umem;
+ struct ib_mr *ib_mr;
+
+ ibdev_dbg(&rdev->ibdev, "Register user DMA-BUF MR\n");
+
+ umem_dmabuf = ib_umem_dmabuf_get_pinned(&rdev->ibdev, start, length,
+ fd, mr_access_flags);
+ if (IS_ERR(umem_dmabuf))
+ return ERR_CAST(umem_dmabuf);
+
+ umem = &umem_dmabuf->umem;
+
+ ib_mr = __bng_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem);
+ if (IS_ERR(ib_mr)) {
+ ib_umem_release(umem);
+ return ib_mr;
+ }
+
+ return ib_mr;
+}
+
+int bng_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
+{
+ struct bng_re_mr *mr = container_of(ib_mr, struct bng_re_mr, ib_mr);
+ struct bng_re_dev *rdev = mr->rdev;
+ struct bng_re_res *sp_res;
+ int rc;
+
+ sp_res = &rdev->bng_res;
+
+ rc = bng_sp_free_mrw(sp_res, &mr->sp_mr);
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Dereg MR failed: %d\n", rc);
+ return rc;
+ }
+
+ /* Free page list if it was allocated for fast registration */
+ if (mr->pages) {
+ bng_sp_free_fast_reg_page_list(sp_res, &mr->sp_frpl);
+ kfree(mr->pages);
+ }
+
+ ib_umem_release(mr->ib_umem);
+
+ kfree(mr);
+
+ return 0;
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.h b/drivers/infiniband/hw/bng_re/bng_verbs.h
index ea017f70142e..0ef7a7535e87 100644
--- a/drivers/infiniband/hw/bng_re/bng_verbs.h
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.h
@@ -37,6 +37,16 @@ struct bng_re_pd {
struct bng_sp_pd sp_pd;
};
+struct bng_re_mr {
+ struct bng_re_dev *rdev;
+ struct ib_mr ib_mr;
+ struct ib_umem *ib_umem;
+ struct bng_re_mrw sp_mr;
+ u32 npages;
+ u64 *pages;
+ struct bng_re_frpl sp_frpl;
+};
+
int bng_re_query_device(struct ib_device *ibdev, struct ib_device_attr *ib_attr,
struct ib_udata *udata);
int bng_re_modify_device(struct ib_device *ibdev, int device_modify_mask,
@@ -64,5 +74,22 @@ int bng_re_add_gid(const struct ib_gid_attr *attr, void **context);
int bng_re_del_gid(const struct ib_gid_attr *attr, void **context);
int bng_re_alloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata);
int bng_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata);
+
+struct ib_mr *bng_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
+ u32 max_num_sg);
+int bng_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
+ unsigned int *sg_offset);
+int bng_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);
+struct ib_mr *bng_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags);
+struct ib_mr *bng_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
+ u64 virt_addr, int mr_access_flags,
+ struct ib_dmah *dmah,
+ struct ib_udata *udata);
+struct ib_mr *bng_re_reg_user_mr_dmabuf(struct ib_pd *ib_pd, u64 start,
+ u64 length, u64 virt_addr,
+ int fd, int mr_access_flags,
+ struct ib_dmah *dmah,
+ struct uverbs_attr_bundle *attrs);
+
#endif /* __BNG_RE_VERBS_H__ */
--
2.43.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 10/15] RDMA/bng_re: Add CQ verbs
2026-09-04 10:43 [PATCH 00/15] Add BNG_RE control path verbs Siva Reddy Kallam
` (8 preceding siblings ...)
2026-09-04 10:43 ` [PATCH 09/15] RDMA/bng_re: Add MR verbs Siva Reddy Kallam
@ 2026-09-04 10:43 ` Siva Reddy Kallam
2026-09-04 10:43 ` [PATCH 11/15] RDMA/bng_re: Add SRQ verbs Siva Reddy Kallam
` (4 subsequent siblings)
14 siblings, 0 replies; 17+ messages in thread
From: Siva Reddy Kallam @ 2026-09-04 10:43 UTC (permalink / raw)
To: leonro, jgg, davem, edumazet, kuba, pabeni, andrew+netdev, horms
Cc: netdev, linux-kernel, linux-rdma, Siva Reddy Kallam
This patch adds below verbs.
- bng_re_create_cq
- bng_re_create_user_cq
- bng_re_resize_cq
- bng_re_req_notify_cq
- bng_re_destroy_cq
Signed-off-by: Siva Reddy Kallam <siva.kallam@broadcom.com>
---
drivers/infiniband/hw/bng_re/Makefile | 1 +
drivers/infiniband/hw/bng_re/bng_dev.c | 17 ++
drivers/infiniband/hw/bng_re/bng_fp.c | 38 +++
drivers/infiniband/hw/bng_re/bng_fp.h | 8 +
drivers/infiniband/hw/bng_re/bng_re.h | 6 +
drivers/infiniband/hw/bng_re/bng_res.h | 73 ++++++
drivers/infiniband/hw/bng_re/bng_sp.c | 235 ++++++++++++++++++
drivers/infiniband/hw/bng_re/bng_sp.h | 73 ++++++
drivers/infiniband/hw/bng_re/bng_verbs.c | 303 +++++++++++++++++++++++
drivers/infiniband/hw/bng_re/bng_verbs.h | 32 +++
include/uapi/rdma/bng_re-abi.h | 21 ++
11 files changed, 807 insertions(+)
create mode 100644 drivers/infiniband/hw/bng_re/bng_fp.c
create mode 100644 drivers/infiniband/hw/bng_re/bng_fp.h
diff --git a/drivers/infiniband/hw/bng_re/Makefile b/drivers/infiniband/hw/bng_re/Makefile
index 1b954a35993b..5e650a35b185 100644
--- a/drivers/infiniband/hw/bng_re/Makefile
+++ b/drivers/infiniband/hw/bng_re/Makefile
@@ -13,4 +13,5 @@ bng_re-y := \
bng_fw.o \
bng_dev.o \
bng_sp.o \
+ bng_fp.o \
bng_verbs.o
diff --git a/drivers/infiniband/hw/bng_re/bng_dev.c b/drivers/infiniband/hw/bng_re/bng_dev.c
index 44c278a93778..9c35c56ae583 100644
--- a/drivers/infiniband/hw/bng_re/bng_dev.c
+++ b/drivers/infiniband/hw/bng_re/bng_dev.c
@@ -52,6 +52,12 @@ static const struct ib_device_ops bng_re_dev_ops = {
.reg_user_mr = bng_re_reg_user_mr,
.reg_user_mr_dmabuf = bng_re_reg_user_mr_dmabuf,
.dereg_mr = bng_re_dereg_mr,
+ .create_cq = bng_re_create_cq,
+ .create_user_cq = bng_re_create_user_cq,
+ .resize_user_cq = bng_re_resize_cq,
+ .req_notify_cq = bng_re_req_notify_cq,
+ .destroy_cq = bng_re_destroy_cq,
+ INIT_RDMA_OBJ_SIZE(ib_cq, bng_re_cq, ib_cq),
INIT_RDMA_OBJ_SIZE(ib_pd, bng_re_pd, ib_pd),
INIT_RDMA_OBJ_SIZE(ib_ucontext, bng_re_ucontext, ib_uctx),
};
@@ -319,6 +325,10 @@ static void bng_re_dev_uninit(struct bng_re_dev *rdev)
{
int rc;
+ if (rdev->bng_res.dpi_tbl.max)
+ bng_re_dealloc_dpi(&rdev->bng_res,
+ &rdev->dpi_privileged);
+
bng_deinit_mpc(rdev);
bng_re_free_xid_tables(&rdev->bng_res);
@@ -480,6 +490,13 @@ static int bng_re_dev_init(struct bng_re_dev *rdev)
"MPC alloc-init failed rc = %#x\n", rc);
goto deinit_mpc;
}
+
+ rc = bng_re_alloc_dpi(&rdev->bng_res,
+ &rdev->dpi_privileged,
+ rdev, BNG_RE_DPI_TYPE_KERNEL);
+ if (rc)
+ goto deinit_mpc;
+
return 0;
deinit_mpc:
bng_deinit_mpc(rdev);
diff --git a/drivers/infiniband/hw/bng_re/bng_fp.c b/drivers/infiniband/hw/bng_re/bng_fp.c
new file mode 100644
index 000000000000..287729a83402
--- /dev/null
+++ b/drivers/infiniband/hw/bng_re/bng_fp.c
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2026 Broadcom.
+//
+#include <linux/interrupt.h>
+#include <linux/types.h>
+#include <linux/io.h>
+
+#include "bng_res.h"
+#include "bng_sp.h"
+#include "bng_fp.h"
+
+/* CQE and NQE validation macros */
+#define CQE_CMP_VALID(hdr, pass) \
+ (!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \
+ !((pass) & BNG_RE_FLAG_EPOCH_CONS_MASK))
+
+bool bng_fp_is_cq_empty(struct bng_sp_cq *cq)
+{
+ struct cq_base *hw_cqe;
+ bool rc = true;
+
+ hw_cqe = bng_re_get_qe(&cq->hwq, cq->hwq.cons, NULL);
+
+ /* Check for Valid bit. If the CQE is valid, return false */
+ rc = !CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags);
+ return rc;
+}
+
+void bng_fp_req_notify_cq(struct bng_sp_cq *cq, u32 arm_type)
+{
+ cq->dbinfo.toggle = cq->toggle;
+
+ if (arm_type)
+ bng_re_ring_db(&cq->dbinfo, arm_type);
+
+ /* Using cq->arm_state variable to track whether to issue cq handler */
+ atomic_set(&cq->arm_state, 1);
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_fp.h b/drivers/infiniband/hw/bng_re/bng_fp.h
new file mode 100644
index 000000000000..4604b4f44816
--- /dev/null
+++ b/drivers/infiniband/hw/bng_re/bng_fp.h
@@ -0,0 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2026 Broadcom. */
+
+#ifndef __BNG_FP_H__
+#define __BNG_FP_H__
+bool bng_fp_is_cq_empty(struct bng_sp_cq *cq);
+void bng_fp_req_notify_cq(struct bng_sp_cq *cq, u32 arm_type);
+#endif /* __BNG_FP_H__ */
diff --git a/drivers/infiniband/hw/bng_re/bng_re.h b/drivers/infiniband/hw/bng_re/bng_re.h
index 0c0e362164d7..47b4037e124e 100644
--- a/drivers/infiniband/hw/bng_re/bng_re.h
+++ b/drivers/infiniband/hw/bng_re/bng_re.h
@@ -8,6 +8,7 @@
#include "bnge_auxr.h"
#include "bng_res.h"
#include "bng_fw.h"
+#include "bng_sp.h"
#include <rdma/ib_verbs.h>
#define BNG_RE_XID_AVOID_REUSE false
@@ -62,6 +63,8 @@ enum {
#define ROCE_MPC_MAX_LATENCY_SEC_SLAB_INDEX BNG_RE_MPC_MAX_LATENCY_SEC_SLAB_INDEX
#define ROCE_MPC_MAX_LATENCY_MSEC_SLAB_INDEX BNG_RE_MPC_MAX_LATENCY_MSEC_SLAB_INDEX
+#define MAX_CQ_HASH_BITS (16)
+
typedef u32 PORT_NUM;
struct bng_re_mpc_poll_info {
@@ -290,6 +293,9 @@ struct bng_re_dev {
struct bng_re_stats stats_ctx;
struct bng_re_mpc_roce_creq_info mpc_roce_creq;
struct bng_mpc_ctx *mpc;
+ struct bng_re_dpi dpi_privileged;
+ struct bng_sp_cq_coal_param cq_coalescing;
+ DECLARE_HASHTABLE(cq_hash, MAX_CQ_HASH_BITS);
};
#define to_bng_re_dev(ptr, member) \
diff --git a/drivers/infiniband/hw/bng_re/bng_res.h b/drivers/infiniband/hw/bng_re/bng_res.h
index cb0f302fc06a..0f832f9bae20 100644
--- a/drivers/infiniband/hw/bng_re/bng_res.h
+++ b/drivers/infiniband/hw/bng_re/bng_res.h
@@ -40,6 +40,11 @@
#define BNG_RE_FR_PMR 0x80000000
+enum bng_re_toggle_modes {
+ BNG_RE_CQ_TOGGLE_BIT = 0x1,
+ BNG_RE_SRQ_TOGGLE_BIT = 0x2,
+};
+
struct bng_re_gid {
u8 data[16];
};
@@ -242,6 +247,18 @@ static inline void *bng_re_get_qe(struct bng_re_hwq *hwq,
(type) | BNG_RE_DBR_VALID) << 32) | (indx) | \
(((u32)(toggle)) << (BNG_RE_DBR_TOGGLE_SHIFT)))
+static inline void bng_re_armen_db(struct bng_re_db_info *info, u32 type)
+{
+ u64 key = 0;
+ u8 toggle = 0;
+
+ if (type == DBC_DBC_TYPE_CQ_ARMENA || type == DBC_DBC_TYPE_SRQ_ARMENA)
+ toggle = info->toggle;
+ /* Index always at 0 */
+ key = BNG_RE_INIT_DBHDR(info->xid, type, 0, toggle);
+ writeq(key, info->priv_db);
+}
+
static inline void bng_re_ring_db(struct bng_re_db_info *info,
u32 type)
{
@@ -315,6 +332,59 @@ static inline bool _is_max_srq_ext_supported(u16 dev_cap_ext_flags_2)
return !!(dev_cap_ext_flags_2 & CREQ_QUERY_FUNC_RESP_SB_MAX_SRQ_EXTENDED);
}
+static inline bool _is_cq_coalescing_supported(u16 dev_cap_ext_flags2)
+{
+ return dev_cap_ext_flags2 & CREQ_QUERY_FUNC_RESP_SB_CQ_COALESCING_SUPPORTED;
+}
+
+#define ROCE_PG_SIZE_4K (4 * 1024)
+#define ROCE_PG_SIZE_8K (8 * 1024)
+#define ROCE_PG_SIZE_64K (64 * 1024)
+#define ROCE_PG_SIZE_2M (2 * 1024 * 1024)
+#define ROCE_PG_SIZE_8M (8 * 1024 * 1024)
+#define ROCE_PG_SIZE_1G (1024 * 1024 * 1024)
+
+enum bng_re_hwrm_pg_size {
+ BNG_RE_HWRM_PG_SIZE_4K = 0,
+ BNG_RE_HWRM_PG_SIZE_8K = 1,
+ BNG_RE_HWRM_PG_SIZE_64K = 2,
+ BNG_RE_HWRM_PG_SIZE_2M = 3,
+ BNG_RE_HWRM_PG_SIZE_8M = 4,
+ BNG_RE_HWRM_PG_SIZE_1G = 5,
+};
+
+static inline u8 bng_re_base_pg_size(struct bng_re_hwq *hwq)
+{
+ u8 pg_size = BNG_RE_HWRM_PG_SIZE_4K;
+ struct bng_re_pbl *pbl;
+
+ pbl = &hwq->pbl[BNG_PBL_LVL_0];
+ switch (pbl->pg_size) {
+ case ROCE_PG_SIZE_4K:
+ pg_size = BNG_RE_HWRM_PG_SIZE_4K;
+ break;
+ case ROCE_PG_SIZE_8K:
+ pg_size = BNG_RE_HWRM_PG_SIZE_8K;
+ break;
+ case ROCE_PG_SIZE_64K:
+ pg_size = BNG_RE_HWRM_PG_SIZE_64K;
+ break;
+ case ROCE_PG_SIZE_2M:
+ pg_size = BNG_RE_HWRM_PG_SIZE_2M;
+ break;
+ case ROCE_PG_SIZE_8M:
+ pg_size = BNG_RE_HWRM_PG_SIZE_8M;
+ break;
+ case ROCE_PG_SIZE_1G:
+ pg_size = BNG_RE_HWRM_PG_SIZE_1G;
+ break;
+ default:
+ break;
+ }
+
+ return pg_size;
+}
+
void bng_re_free_hwq(struct bng_re_res *res,
struct bng_re_hwq *hwq);
@@ -337,4 +407,7 @@ int bng_re_alloc_dpi(struct bng_re_res *res,
void *app, enum bng_re_dpi_type type);
int bng_re_dealloc_dpi(struct bng_re_res *res,
struct bng_re_dpi *dpi);
+void bng_re_alloc_kernel_dpi(struct bng_re_res *res,
+ struct bng_re_dpi *dpi);
+void bng_re_dealloc_kernel_dpi(struct bng_re_dpi *dpi);
#endif
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.c b/drivers/infiniband/hw/bng_re/bng_sp.c
index cd33ff32845a..04488db6d016 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.c
+++ b/drivers/infiniband/hw/bng_re/bng_sp.c
@@ -581,3 +581,238 @@ int bng_sp_reg_mr(struct bng_re_res *res, struct bng_re_mrw *mr,
bng_re_free_hwq(res, &mr->hwq);
return rc;
}
+
+int bng_sp_create_cq(struct bng_re_res *res, struct bng_sp_cq *cq)
+{
+ struct bng_re_rcfw *rcfw = res->rcfw;
+ struct bng_re_hwq_attr hwq_attr = {};
+ struct creq_create_cq_resp resp = {};
+ struct bng_re_cmdqmsg msg = {};
+ struct cmdq_create_cq req = {};
+ struct bng_re_pbl *pbl;
+ u32 coalescing = 0;
+ u32 pg_sz_lvl;
+ int rc;
+
+ if (!cq->dpi) {
+ dev_err(&rcfw->pdev->dev,
+ "FP: CREATE_CQ failed due to NULL DPI\n");
+ return -EINVAL;
+ }
+
+ cq->dbinfo.flags = 0;
+ hwq_attr.res = res;
+ hwq_attr.depth = cq->max_wqe;
+ hwq_attr.stride = sizeof(struct cq_base);
+ hwq_attr.type = BNG_HWQ_TYPE_QUEUE;
+ hwq_attr.sginfo = &cq->sg_info;
+ rc = bng_re_alloc_init_hwq(&cq->hwq, &hwq_attr);
+ if (rc)
+ return rc;
+
+ bng_re_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_CREATE_CQ,
+ sizeof(req));
+
+ req.dpi = cpu_to_le32(cq->dpi->dpi);
+ req.cq_handle = cpu_to_le64(cq->cq_handle);
+ req.cq_size = cpu_to_le32(cq->max_wqe);
+
+ if (_is_cq_coalescing_supported(res->dattr->dev_cap_flags2) &&
+ cq->coalescing.enable) {
+ req.flags |= cpu_to_le16(CMDQ_CREATE_CQ_FLAGS_COALESCING_VALID);
+ coalescing |= ((cq->coalescing.buf_maxtime <<
+ CMDQ_CREATE_CQ_BUF_MAXTIME_SFT) &
+ CMDQ_CREATE_CQ_BUF_MAXTIME_MASK);
+ coalescing |= ((cq->coalescing.normal_maxbuf <<
+ CMDQ_CREATE_CQ_NORMAL_MAXBUF_SFT) &
+ CMDQ_CREATE_CQ_NORMAL_MAXBUF_MASK);
+ coalescing |= ((cq->coalescing.during_maxbuf <<
+ CMDQ_CREATE_CQ_DURING_MAXBUF_SFT) &
+ CMDQ_CREATE_CQ_DURING_MAXBUF_MASK);
+ if (cq->coalescing.en_ring_idle_mode)
+ coalescing |= CMDQ_CREATE_CQ_ENABLE_RING_IDLE_MODE;
+ else
+ coalescing &= ~CMDQ_CREATE_CQ_ENABLE_RING_IDLE_MODE;
+ req.coalescing = cpu_to_le32(coalescing);
+ }
+
+ pbl = &cq->hwq.pbl[BNG_PBL_LVL_0];
+ pg_sz_lvl = (bng_re_base_pg_size(&cq->hwq) <<
+ CMDQ_CREATE_CQ_PG_SIZE_SFT);
+ pg_sz_lvl |= (cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK);
+ req.pg_size_lvl = cpu_to_le32(pg_sz_lvl);
+ req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
+ req.cq_fco_cnq_id = cpu_to_le32(
+ (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
+ CMDQ_CREATE_CQ_CNQ_ID_SFT);
+ bng_re_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bng_re_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ goto fail;
+
+ cq->id = le32_to_cpu(resp.xid);
+ cq->period = BNG_QUEUE_START_PERIOD;
+ init_waitqueue_head(&cq->waitq);
+ INIT_LIST_HEAD(&cq->sqf_head);
+ INIT_LIST_HEAD(&cq->rqf_head);
+ spin_lock_init(&cq->compl_lock);
+ spin_lock_init(&cq->flush_lock);
+
+ cq->dbinfo.hwq = &cq->hwq;
+ cq->dbinfo.xid = cq->id;
+ cq->dbinfo.db = cq->dpi->dbr;
+ cq->dbinfo.priv_db = res->dpi_tbl.priv_db;
+ cq->dbinfo.flags = 0;
+ cq->dbinfo.toggle = 0;
+ cq->dbinfo.seed = 0;
+ spin_lock_init(&cq->dbinfo.lock);
+ cq->dbinfo.res = res;
+ cq->dbinfo.is_l2 = false;
+
+ bng_re_armen_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMENA);
+
+ return 0;
+
+fail:
+ bng_re_free_hwq(res, &cq->hwq);
+ return rc;
+}
+
+void bng_sp_resize_cq_complete(struct bng_re_res *res, struct bng_sp_cq *cq)
+{
+ bng_re_free_hwq(res, &cq->hwq);
+ memcpy(&cq->hwq, &cq->resize_hwq, sizeof(cq->hwq));
+ cq->dbinfo.flags &= ~(1UL << BNG_RE_FLAG_EPOCH_CONS_SHIFT);
+}
+
+int bng_sp_resize_cq(struct bng_re_res *res, struct bng_sp_cq *cq,
+ int new_cqes)
+{
+ struct bng_re_hwq_attr hwq_attr = {};
+ struct bng_re_rcfw *rcfw = res->rcfw;
+ struct creq_resize_cq_resp resp = {};
+ struct bng_re_cmdqmsg msg = {};
+ struct cmdq_resize_cq req = {};
+ struct bng_re_pbl *pbl;
+ u32 pg_sz, lvl, new_sz;
+ int rc;
+
+ bng_re_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_RESIZE_CQ,
+ sizeof(req));
+ hwq_attr.sginfo = &cq->sg_info;
+ hwq_attr.res = res;
+ hwq_attr.depth = new_cqes;
+ hwq_attr.stride = sizeof(struct cq_base);
+ hwq_attr.type = BNG_HWQ_TYPE_QUEUE;
+ rc = bng_re_alloc_init_hwq(&cq->resize_hwq, &hwq_attr);
+ if (rc)
+ return rc;
+
+ req.cq_cid = cpu_to_le32(cq->id);
+ pbl = &cq->resize_hwq.pbl[BNG_PBL_LVL_0];
+ pg_sz = bng_re_base_pg_size(&cq->resize_hwq);
+ lvl = (cq->resize_hwq.level << CMDQ_RESIZE_CQ_LVL_SFT) &
+ CMDQ_RESIZE_CQ_LVL_MASK;
+ new_sz = (new_cqes << CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT) &
+ CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK;
+ req.new_cq_size_pg_size_lvl = cpu_to_le32(new_sz | pg_sz | lvl);
+ req.new_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
+
+ bng_re_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bng_re_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ bng_re_free_hwq(res, &cq->resize_hwq);
+ return rc;
+}
+
+static void clean_nq(struct bng_re_nq *nq, struct bng_sp_cq *cq)
+{
+ struct bng_re_hwq *hwq = &nq->hwq;
+ struct nq_base *nqe, **nq_ptr;
+ int budget = nq->budget;
+ uintptr_t q_handle;
+ u16 type;
+
+ spin_lock_bh(&hwq->lock);
+ /* Service the NQ until empty */
+ while (budget--) {
+ nq_ptr = (struct nq_base **)hwq->pbl_ptr;
+ nqe = &nq_ptr[NQE_PG(hwq->cons)][NQE_IDX(hwq->cons)];
+ if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags))
+ break;
+
+ /*
+ * The valid test of the entry must be done first before
+ * reading any further.
+ */
+ dma_rmb();
+
+ type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
+ switch (type) {
+ case NQ_BASE_TYPE_CQ_NOTIFICATION:
+ {
+ struct nq_cn *nqcne = (struct nq_cn *)nqe;
+
+ q_handle = le32_to_cpu(nqcne->cq_handle_low);
+ q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high)
+ << 32;
+ if ((unsigned long)cq == q_handle) {
+ nqcne->cq_handle_low = 0;
+ nqcne->cq_handle_high = 0;
+ cq->cnq_events++;
+ }
+ break;
+ }
+ default:
+ break;
+ }
+ bng_re_hwq_incr_cons(hwq->max_elements, &hwq->cons,
+ 1, &nq->nq_db.dbinfo.flags);
+ }
+ spin_unlock_bh(&hwq->lock);
+}
+
+/* Wait for receiving all NQEs for this CQ and clean the NQEs associated with
+ * this CQ.
+ */
+static void __wait_for_all_nqes(struct bng_sp_cq *cq, u16 cnq_events)
+{
+ u32 retry_cnt = 100;
+
+ while (retry_cnt--) {
+ if (cnq_events == cq->cnq_events)
+ return;
+ usleep_range(50, 100);
+ clean_nq(cq->nq, cq);
+ }
+}
+
+int bng_sp_destroy_cq(struct bng_re_res *res, struct bng_sp_cq *cq)
+{
+ struct bng_re_rcfw *rcfw = res->rcfw;
+ struct creq_destroy_cq_resp resp = {};
+ struct bng_re_cmdqmsg msg = {};
+ struct cmdq_destroy_cq req = {};
+ u16 total_cnq_events;
+ int rc;
+
+ bng_re_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_DESTROY_CQ,
+ sizeof(req));
+
+ req.cq_cid = cpu_to_le32(cq->id);
+ bng_re_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bng_re_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ return rc;
+ total_cnq_events = le16_to_cpu(resp.total_cnq_events);
+ __wait_for_all_nqes(cq, total_cnq_events);
+ bng_re_free_hwq(res, &cq->hwq);
+ return 0;
+}
+
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.h b/drivers/infiniband/hw/bng_re/bng_sp.h
index 6dd3ff911548..d05151123f98 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.h
+++ b/drivers/infiniband/hw/bng_re/bng_sp.h
@@ -15,6 +15,12 @@
(!!(le32_to_cpu((hdr)->info63_v[0]) & NQ_BASE_V) == \
!((pass) & BNG_RE_FLAG_EPOCH_CONS_MASK))
+#define BNG_RE_MAX_NQE_ENTRY_SIZE sizeof(struct nq_base)
+#define NQE_MAX_IDX_PER_PG (NQE_CNT_PER_PG - 1)
+#define NQE_CNT_PER_PG (PAGE_SIZE / BNG_RE_MAX_NQE_ENTRY_SIZE)
+#define NQE_PG(x) (((x) & ~NQE_MAX_IDX_PER_PG) / NQE_CNT_PER_PG)
+#define NQE_IDX(x) ((x) & NQE_MAX_IDX_PER_PG)
+
struct bng_re_dev_attr {
#define FW_VER_ARR_LEN 4
u8 fw_ver[FW_VER_ARR_LEN];
@@ -98,6 +104,68 @@ struct bng_sp_mrinfo {
bool request_relax_order;
};
+struct bng_sp_cq_coal_param {
+ u16 buf_maxtime;
+ u8 normal_maxbuf;
+ u8 during_maxbuf;
+ u8 en_ring_idle_mode;
+ u8 enable;
+};
+
+struct bng_sp_cqe {
+ u8 status;
+ u8 type;
+ u8 opcode;
+ u32 length;
+ u16 cfa_meta;
+ u64 wr_id;
+ union {
+ u32 immdata;
+ u32 invrkey;
+ };
+ u64 qp_handle;
+ u64 mr_handle;
+ u16 flags;
+ u8 smac[6];
+ u32 src_qp;
+ u16 raweth_qp1_flags;
+ u16 raweth_qp1_errors;
+ u16 raweth_qp1_cfa_code;
+ u32 raweth_qp1_flags2;
+ u32 raweth_qp1_metadata;
+ u8 raweth_qp1_payload_offset;
+};
+
+#define BNG_QUEUE_START_PERIOD 0x01
+struct bng_sp_cq {
+ struct bng_re_dpi *dpi;
+ struct bng_re_db_info dbinfo;
+ u32 max_wqe;
+ u32 id;
+ u16 count;
+ u16 period;
+ struct bng_re_hwq hwq;
+ struct bng_re_hwq resize_hwq;
+ u32 cnq_hw_ring_id;
+ struct bng_re_nq *nq;
+ bool resize_in_progress;
+ struct bng_re_sg_info sg_info;
+ u64 cq_handle;
+ u8 toggle;
+ struct bng_re_chip_ctx *cctx;
+
+#define CQ_RESIZE_WAIT_TIME_MS 500
+ unsigned long flags;
+#define CQ_FLAGS_RESIZE_IN_PROG 1
+ wait_queue_head_t waitq;
+ struct bng_sp_cq_coal_param coalescing;
+ struct list_head sqf_head, rqf_head;
+ atomic_t arm_state;
+ spinlock_t compl_lock;
+ spinlock_t flush_lock;
+ u16 cnq_events;
+};
+
int bng_re_get_dev_attr(struct bng_re_rcfw *rcfw);
int bng_sp_alloc_sgid_tbl(struct bng_re_sgid_tbl *sgid_tbl, u16 size);
@@ -127,5 +195,10 @@ int bng_sp_free_mrw(struct bng_re_res *res, struct bng_re_mrw *mrw);
int bng_sp_reg_mr(struct bng_re_res *res, struct bng_re_mrw *mr,
struct ib_umem *umem, int num_pbls, u32 buf_pg_size,
bool unified_mr);
+int bng_sp_create_cq(struct bng_re_res *res, struct bng_sp_cq *cq);
+void bng_sp_resize_cq_complete(struct bng_re_res *res, struct bng_sp_cq *cq);
+int bng_sp_resize_cq(struct bng_re_res *res, struct bng_sp_cq *cq,
+ int new_cqes);
+int bng_sp_destroy_cq(struct bng_re_res *res, struct bng_sp_cq *cq);
#endif
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.c b/drivers/infiniband/hw/bng_re/bng_verbs.c
index fe227d20513a..b08ac20d788d 100644
--- a/drivers/infiniband/hw/bng_re/bng_verbs.c
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.c
@@ -21,6 +21,7 @@
#include "bnge_auxr.h"
#include "bng_re.h"
#include "bng_verbs.h"
+#include "bng_fp.h"
int bng_re_query_device(struct ib_device *ibdev,
struct ib_device_attr *ib_attr,
@@ -893,3 +894,305 @@ int bng_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
return 0;
}
+
+static struct bng_re_nq *bng_re_get_nq(struct bng_re_dev *rdev)
+{
+ int min, indx;
+
+ mutex_lock(&rdev->nqr->load_lock);
+ for (indx = 0, min = 0; indx < (rdev->nqr->num_msix - 1); indx++) {
+ if (rdev->nqr->nq[min].load > rdev->nqr->nq[indx].load)
+ min = indx;
+ }
+ rdev->nqr->nq[min].load++;
+ mutex_unlock(&rdev->nqr->load_lock);
+
+ return &rdev->nqr->nq[min];
+}
+
+int bng_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct uverbs_attr_bundle *attrs)
+{
+ struct bng_re_cq *cq = container_of(ibcq, struct bng_re_cq, ib_cq);
+ struct bng_re_dev *rdev = to_bng_re_dev(ibcq->device, ibdev);
+ struct ib_udata *udata = &attrs->driver_udata;
+ struct bng_re_ucontext *uctx =
+ rdma_udata_to_drv_context(udata, struct bng_re_ucontext, ib_uctx);
+ struct bng_re_dev_attr *dev_attr = rdev->dev_attr;
+ struct bng_re_chip_ctx *cctx;
+ struct bng_re_cq_resp resp = {};
+ struct bng_re_cq_req req;
+ int rc, entries;
+
+ if (attr->flags)
+ return -EOPNOTSUPP;
+
+ /* Check for valid CQ entries */
+ if (attr->cqe > dev_attr->max_cq_wqes) {
+ ibdev_err(&rdev->ibdev,
+ "Failed to create CQ, cqe:%d max_cq_wqes:%d",
+ attr->cqe, dev_attr->max_cq_wqes);
+ return -EINVAL;
+ }
+
+ cq->rdev = rdev;
+ cctx = rdev->chip_ctx;
+ cq->sp_cq.cq_handle = (u64)(unsigned long)(&cq->sp_cq);
+
+ entries = bng_re_init_depth(attr->cqe + 1, uctx);
+
+ rc = ib_copy_from_udata(&req, udata, sizeof(req));
+ if (rc)
+ return rc;
+
+ cq->umem = ib_umem_get_va(&rdev->ibdev, req.cq_va,
+ entries * sizeof(struct cq_base),
+ IB_ACCESS_LOCAL_WRITE);
+ if (IS_ERR(cq->umem))
+ return PTR_ERR(cq->umem);
+
+ cq->sp_cq.sg_info.umem = cq->umem;
+ cq->sp_cq.dpi = &uctx->dpi;
+ cq->sp_cq.max_wqe = entries;
+ cq->sp_cq.coalescing = rdev->cq_coalescing;
+ cq->sp_cq.nq = bng_re_get_nq(rdev);
+ cq->sp_cq.cnq_hw_ring_id = cq->sp_cq.nq->ring_id;
+ rc = bng_sp_create_cq(&rdev->bng_res, &cq->sp_cq);
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Failed to create HW CQ");
+ goto udata_cq_fail;
+ }
+
+ cq->ib_cq.cqe = entries;
+ cq->cq_period = cq->sp_cq.period;
+
+ spin_lock_init(&cq->cq_lock);
+
+ if (cctx->modes.toggle_bits & BNG_RE_CQ_TOGGLE_BIT) {
+ /* Allocate a page */
+ cq->uctx_cq_page = (void *)get_zeroed_page(GFP_KERNEL);
+ if (!cq->uctx_cq_page) {
+ rc = -ENOMEM;
+ goto uctx_cq_page_fail;
+ }
+ resp.comp_mask |= BNG_RE_CQ_TOGGLE_PAGE_SUPPORT;
+ }
+ resp.cqid = cq->sp_cq.id;
+ resp.tail = cq->sp_cq.hwq.cons;
+ resp.phase = cq->sp_cq.period;
+ resp.rsvd = 0;
+ rc = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Failed to copy CQ udata");
+ goto destroy_cq;
+ }
+
+ return 0;
+
+destroy_cq:
+ if (cq->uctx_cq_page)
+ free_page((unsigned long)cq->uctx_cq_page);
+uctx_cq_page_fail:
+ bng_sp_destroy_cq(&rdev->bng_res, &cq->sp_cq);
+ ib_umem_release(cq->umem);
+udata_cq_fail:
+ kfree(cq->cql);
+ return rc;
+}
+
+int bng_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct uverbs_attr_bundle *attrs)
+{
+ struct bng_re_cq *cq = container_of(ibcq, struct bng_re_cq, ib_cq);
+ struct bng_re_dev *rdev = to_bng_re_dev(ibcq->device, ibdev);
+ struct bng_re_dev_attr *dev_attr = rdev->dev_attr;
+ int rc, entries;
+
+ if (attr->flags)
+ return -EOPNOTSUPP;
+
+ /* Check for valid CQ entries */
+ if (attr->cqe > dev_attr->max_cq_wqes) {
+ ibdev_err(&rdev->ibdev,
+ "Failed to create CQ, cqe:%d max_cq_wqes:%d",
+ attr->cqe, dev_attr->max_cq_wqes);
+ return -EINVAL;
+ }
+
+ cq->rdev = rdev;
+ cq->sp_cq.cq_handle = (u64)(unsigned long)(&cq->sp_cq);
+
+ cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL);
+ cq->cql = kcalloc(cq->max_cql, sizeof(struct bng_sp_cqe),
+ GFP_KERNEL);
+ if (!cq->cql)
+ return -ENOMEM;
+
+ cq->sp_cq.sg_info.pgsize = SZ_4K;
+ cq->sp_cq.dpi = &rdev->dpi_privileged;
+ cq->sp_cq.max_wqe = cq->max_cql;
+ cq->sp_cq.coalescing = rdev->cq_coalescing;
+ cq->sp_cq.nq = bng_re_get_nq(rdev);
+ cq->sp_cq.cnq_hw_ring_id = cq->sp_cq.nq->ring_id;
+
+ rc = bng_sp_create_cq(&rdev->bng_res, &cq->sp_cq);
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Failed to create HW CQ");
+ goto create_cq_fail;
+ }
+
+ cq->ib_cq.cqe = cq->max_cql;
+ cq->cq_period = cq->sp_cq.period;
+
+ spin_lock_init(&cq->cq_lock);
+
+ return 0;
+
+create_cq_fail:
+ kfree(cq->cql);
+ return rc;
+}
+
+int bng_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe, struct ib_udata *udata)
+{
+ struct bng_re_sg_info sg_info = {};
+ struct bng_re_dpi *orig_dpi = NULL;
+ struct bng_re_dev_attr *dev_attr;
+ struct bng_re_ucontext *uctx = NULL;
+ struct bng_re_resize_cq_req req;
+ struct bng_re_dev *rdev;
+ struct bng_re_cq *cq;
+ int rc, entries;
+
+ cq = container_of(ibcq, struct bng_re_cq, ib_cq);
+ rdev = cq->rdev;
+ dev_attr = rdev->dev_attr;
+ if (!ibcq->uobject) {
+ ibdev_err(&rdev->ibdev, "Kernel CQ Resize not supported");
+ return -EOPNOTSUPP;
+ }
+
+ if (cq->resize_umem) {
+ ibdev_err(&rdev->ibdev, "Resize CQ %#x failed - Busy",
+ cq->sp_cq.id);
+ return -EBUSY;
+ }
+
+ /* Check the requested cq depth out of supported depth */
+ if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
+ ibdev_err(&rdev->ibdev, "Resize CQ %#x failed - out of range cqe %d",
+ cq->sp_cq.id, cqe);
+ return -EINVAL;
+ }
+
+ uctx = rdma_udata_to_drv_context(udata, struct bng_re_ucontext, ib_uctx);
+ entries = bng_re_init_depth(cqe + 1, uctx);
+ if (entries > dev_attr->max_cq_wqes + 1)
+ entries = dev_attr->max_cq_wqes + 1;
+
+ /* uverbs consumer */
+ if (ib_copy_from_udata(&req, udata, sizeof(req))) {
+ rc = -EFAULT;
+ goto fail;
+ }
+
+ cq->resize_umem = ib_umem_get_va(&rdev->ibdev, req.cq_va,
+ entries * sizeof(struct cq_base),
+ IB_ACCESS_LOCAL_WRITE);
+ if (IS_ERR(cq->resize_umem)) {
+ rc = PTR_ERR(cq->resize_umem);
+ ibdev_err(&rdev->ibdev, "%s: ib_umem_get_va failed! rc = %pe\n",
+ __func__, cq->resize_umem);
+ cq->resize_umem = NULL;
+ goto fail;
+ }
+ cq->resize_cqe = entries;
+ memcpy(&sg_info, &cq->sp_cq.sg_info, sizeof(sg_info));
+ orig_dpi = cq->sp_cq.dpi;
+
+ cq->sp_cq.sg_info.umem = cq->resize_umem;
+ cq->sp_cq.sg_info.pgsize = PAGE_SIZE;
+ cq->sp_cq.sg_info.pgshft = PAGE_SHIFT;
+ cq->sp_cq.dpi = &uctx->dpi;
+
+ rc = bng_sp_resize_cq(&rdev->bng_res, &cq->sp_cq, entries);
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Resize HW CQ %#x failed!",
+ cq->sp_cq.id);
+ goto fail;
+ }
+
+ cq->ib_cq.cqe = cq->resize_cqe;
+
+ return 0;
+
+fail:
+ if (cq->resize_umem) {
+ ib_umem_release(cq->resize_umem);
+ cq->resize_umem = NULL;
+ cq->resize_cqe = 0;
+ memcpy(&cq->sp_cq.sg_info, &sg_info, sizeof(sg_info));
+ cq->sp_cq.dpi = orig_dpi;
+ }
+ return rc;
+}
+
+int bng_re_req_notify_cq(struct ib_cq *ib_cq,
+ enum ib_cq_notify_flags ib_cqn_flags)
+{
+ struct bng_re_cq *cq = container_of(ib_cq, struct bng_re_cq, ib_cq);
+ int type = 0, rc = 0;
+ unsigned long flags;
+
+ spin_lock_irqsave(&cq->cq_lock, flags);
+ /* Trigger on the very next completion */
+ if (ib_cqn_flags & IB_CQ_NEXT_COMP)
+ type = DBC_DBC_TYPE_CQ_ARMALL;
+ /* Trigger on the next solicited completion */
+ else if (ib_cqn_flags & IB_CQ_SOLICITED)
+ type = DBC_DBC_TYPE_CQ_ARMSE;
+
+ /* Poll to see if there are missed events */
+ if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) &&
+ !(bng_fp_is_cq_empty(&cq->sp_cq))) {
+ rc = 1;
+ goto mis_events;
+ }
+ bng_fp_req_notify_cq(&cq->sp_cq, type);
+
+mis_events:
+ spin_unlock_irqrestore(&cq->cq_lock, flags);
+ return rc;
+}
+
+static void bng_re_put_nq(struct bng_re_dev *rdev, struct bng_re_nq *nq)
+{
+ mutex_lock(&rdev->nqr->load_lock);
+ nq->load--;
+ mutex_unlock(&rdev->nqr->load_lock);
+}
+
+int bng_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
+{
+ struct bng_re_chip_ctx *cctx;
+ struct bng_re_nq *nq;
+ struct bng_re_dev *rdev;
+ struct bng_re_cq *cq;
+
+ cq = container_of(ib_cq, struct bng_re_cq, ib_cq);
+ rdev = cq->rdev;
+ nq = cq->sp_cq.nq;
+ cctx = rdev->chip_ctx;
+
+ if (cctx->modes.toggle_bits & BNG_RE_CQ_TOGGLE_BIT) {
+ free_page((unsigned long)cq->uctx_cq_page);
+ hash_del(&cq->hash_entry);
+ }
+ bng_sp_destroy_cq(&rdev->bng_res, &cq->sp_cq);
+
+ bng_re_put_nq(rdev, nq);
+ ib_umem_release(cq->umem);
+
+ kfree(cq->cql);
+ return 0;
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.h b/drivers/infiniband/hw/bng_re/bng_verbs.h
index 0ef7a7535e87..839967c73f08 100644
--- a/drivers/infiniband/hw/bng_re/bng_verbs.h
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.h
@@ -47,6 +47,30 @@ struct bng_re_mr {
struct bng_re_frpl sp_frpl;
};
+struct bng_re_cq {
+ struct ib_cq ib_cq;
+ struct bng_re_dev *rdev;
+ spinlock_t cq_lock;
+ u16 cq_count;
+ u16 cq_period;
+ struct bng_sp_cq sp_cq;
+ struct bng_sp_cqe *cql;
+ #define MAX_CQL_PER_POLL 1024
+ u32 max_cql;
+ struct ib_umem *umem;
+ struct ib_umem *resize_umem;
+ int resize_cqe;
+ void *uctx_cq_page;
+ struct hlist_node hash_entry;
+};
+
+static inline u32 bng_re_init_depth(u32 ent, struct bng_re_ucontext *uctx)
+{
+ /* roundup_pow_of_two(0) is undefined (UBSAN: shift by 64 on 64-bit). */
+ return uctx ? (uctx->cmask & BNG_RE_UCNTX_CMASK_POW2_DISABLED) ?
+ ent : roundup_pow_of_two(ent) : ent;
+}
+
int bng_re_query_device(struct ib_device *ibdev, struct ib_device_attr *ib_attr,
struct ib_udata *udata);
int bng_re_modify_device(struct ib_device *ibdev, int device_modify_mask,
@@ -90,6 +114,14 @@ struct ib_mr *bng_re_reg_user_mr_dmabuf(struct ib_pd *ib_pd, u64 start,
int fd, int mr_access_flags,
struct ib_dmah *dmah,
struct uverbs_attr_bundle *attrs);
+int bng_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct uverbs_attr_bundle *attrs);
+int bng_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct uverbs_attr_bundle *attrs);
+int bng_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe, struct ib_udata *udata);
+int bng_re_req_notify_cq(struct ib_cq *ib_cq,
+ enum ib_cq_notify_flags ib_cqn_flags);
+int bng_re_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
#endif /* __BNG_RE_VERBS_H__ */
diff --git a/include/uapi/rdma/bng_re-abi.h b/include/uapi/rdma/bng_re-abi.h
index 14d56a88def4..2a88f36bcacd 100644
--- a/include/uapi/rdma/bng_re-abi.h
+++ b/include/uapi/rdma/bng_re-abi.h
@@ -40,4 +40,25 @@ struct bng_re_uctx_resp {
__u64 uc_db_mmap_key;
};
+struct bng_re_cq_req {
+ __aligned_u64 cq_va;
+ __aligned_u64 cq_handle;
+};
+
+enum bng_re_cq_mask {
+ BNG_RE_CQ_TOGGLE_PAGE_SUPPORT = 0x1,
+};
+
+struct bng_re_cq_resp {
+ __u32 cqid;
+ __u32 tail;
+ __u32 phase;
+ __u32 rsvd;
+ __aligned_u64 comp_mask;
+};
+
+struct bng_re_resize_cq_req {
+ __aligned_u64 cq_va;
+};
+
#endif /* __BNG_RE_UVERBS_ABI_H__*/
--
2.43.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 11/15] RDMA/bng_re: Add SRQ verbs
2026-09-04 10:43 [PATCH 00/15] Add BNG_RE control path verbs Siva Reddy Kallam
` (9 preceding siblings ...)
2026-09-04 10:43 ` [PATCH 10/15] RDMA/bng_re: Add CQ verbs Siva Reddy Kallam
@ 2026-09-04 10:43 ` Siva Reddy Kallam
2026-09-04 10:43 ` [PATCH 12/15] RDMA/bng_re: Add Stats verbs Siva Reddy Kallam
` (3 subsequent siblings)
14 siblings, 0 replies; 17+ messages in thread
From: Siva Reddy Kallam @ 2026-09-04 10:43 UTC (permalink / raw)
To: leonro, jgg, davem, edumazet, kuba, pabeni, andrew+netdev, horms
Cc: netdev, linux-kernel, linux-rdma, Suresh Rupanagudi,
Siva Reddy Kallam
From: Suresh Rupanagudi <suresh.rupanagudi@broadcom.com>
This patch adds below verbs.
- bng_re_create_srq
- bng_re_modify_srq
- bng_re_query_srq
- bng_re_destroy_srq
Signed-off-by: Suresh Rupanagudi <suresh.rupanagudi@broadcom.com>
Signed-off-by: Siva Reddy Kallam <siva.kallam@broadcom.com>
---
drivers/infiniband/hw/bng_re/bng_dev.c | 5 +
drivers/infiniband/hw/bng_re/bng_re.h | 1 +
drivers/infiniband/hw/bng_re/bng_res.c | 1 +
drivers/infiniband/hw/bng_re/bng_res.h | 9 +
drivers/infiniband/hw/bng_re/bng_sp.c | 144 +++++++++++++++
drivers/infiniband/hw/bng_re/bng_sp.h | 28 +++
drivers/infiniband/hw/bng_re/bng_verbs.c | 216 +++++++++++++++++++++++
drivers/infiniband/hw/bng_re/bng_verbs.h | 25 +++
include/uapi/rdma/bng_re-abi.h | 14 ++
9 files changed, 443 insertions(+)
diff --git a/drivers/infiniband/hw/bng_re/bng_dev.c b/drivers/infiniband/hw/bng_re/bng_dev.c
index 9c35c56ae583..a42e24691cad 100644
--- a/drivers/infiniband/hw/bng_re/bng_dev.c
+++ b/drivers/infiniband/hw/bng_re/bng_dev.c
@@ -57,6 +57,11 @@ static const struct ib_device_ops bng_re_dev_ops = {
.resize_user_cq = bng_re_resize_cq,
.req_notify_cq = bng_re_req_notify_cq,
.destroy_cq = bng_re_destroy_cq,
+ .create_srq = bng_re_create_srq,
+ .modify_srq = bng_re_modify_srq,
+ .query_srq = bng_re_query_srq,
+ .destroy_srq = bng_re_destroy_srq,
+ INIT_RDMA_OBJ_SIZE(ib_srq, bng_re_srq, ib_srq),
INIT_RDMA_OBJ_SIZE(ib_cq, bng_re_cq, ib_cq),
INIT_RDMA_OBJ_SIZE(ib_pd, bng_re_pd, ib_pd),
INIT_RDMA_OBJ_SIZE(ib_ucontext, bng_re_ucontext, ib_uctx),
diff --git a/drivers/infiniband/hw/bng_re/bng_re.h b/drivers/infiniband/hw/bng_re/bng_re.h
index 47b4037e124e..70851f1180d0 100644
--- a/drivers/infiniband/hw/bng_re/bng_re.h
+++ b/drivers/infiniband/hw/bng_re/bng_re.h
@@ -296,6 +296,7 @@ struct bng_re_dev {
struct bng_re_dpi dpi_privileged;
struct bng_sp_cq_coal_param cq_coalescing;
DECLARE_HASHTABLE(cq_hash, MAX_CQ_HASH_BITS);
+ DECLARE_HASHTABLE(srq_hash, 8); /* SRQ hash table */
};
#define to_bng_re_dev(ptr, member) \
diff --git a/drivers/infiniband/hw/bng_re/bng_res.c b/drivers/infiniband/hw/bng_re/bng_res.c
index 8bf6e8efa182..024d49f68b09 100644
--- a/drivers/infiniband/hw/bng_re/bng_res.c
+++ b/drivers/infiniband/hw/bng_re/bng_res.c
@@ -265,6 +265,7 @@ int bng_re_alloc_init_hwq(struct bng_re_hwq *hwq,
hwq->max_elements = hwq->depth;
hwq->element_size = stride;
hwq->qe_ppg = pg_size / stride;
+ hwq->is_user = !!hwq_attr->sginfo->umem;
/* For direct access to the elements */
lvl = hwq->level;
if (hwq_attr->sginfo->nopte && hwq->level)
diff --git a/drivers/infiniband/hw/bng_re/bng_res.h b/drivers/infiniband/hw/bng_re/bng_res.h
index 0f832f9bae20..00429b4a7d30 100644
--- a/drivers/infiniband/hw/bng_re/bng_res.h
+++ b/drivers/infiniband/hw/bng_re/bng_res.h
@@ -170,6 +170,7 @@ struct bng_re_hwq {
u32 cons;
/* queue entry per page */
u16 qe_ppg;
+ u8 is_user;
};
struct bng_re_stats {
@@ -288,6 +289,14 @@ static inline void bng_re_ring_nq_db(struct bng_re_db_info *info,
bng_re_ring_db(info, type);
}
+static inline void bng_re_srq_arm_db(struct bng_re_db_info *info, u32 th)
+{
+ u64 key = 0;
+
+ key = BNG_RE_INIT_DBHDR(info->xid, DBC_DBC_TYPE_SRQ_ARM, th, info->toggle);
+ writeq(key, info->priv_db);
+}
+
static inline void bng_re_hwq_incr_cons(u32 max_elements, u32 *cons, u32 cnt,
u32 *dbinfo_flags)
{
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.c b/drivers/infiniband/hw/bng_re/bng_sp.c
index 04488db6d016..78e2bb9ad859 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.c
+++ b/drivers/infiniband/hw/bng_re/bng_sp.c
@@ -816,3 +816,147 @@ int bng_sp_destroy_cq(struct bng_re_res *res, struct bng_sp_cq *cq)
return 0;
}
+int bng_sp_create_srq(struct bng_re_res *res, struct bng_sp_srq *srq)
+{
+ struct bng_re_rcfw *rcfw = res->rcfw;
+ struct bng_re_hwq_attr hwq_attr = {};
+ struct creq_create_srq_resp resp = {};
+ struct bng_re_cmdqmsg msg = {};
+ struct cmdq_create_srq req = {};
+ struct bng_re_pbl *pbl;
+ u16 pg_sz_lvl;
+ int rc, idx;
+
+ hwq_attr.res = res;
+ hwq_attr.sginfo = &srq->sginfo;
+ hwq_attr.depth = srq->max_wqe;
+ hwq_attr.stride = srq->wqe_size;
+ hwq_attr.type = BNG_HWQ_TYPE_QUEUE;
+ rc = bng_re_alloc_init_hwq(&srq->hwq, &hwq_attr);
+ if (rc)
+ return rc;
+
+ srq->dbinfo.flags = 0;
+ bng_re_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_CREATE_SRQ,
+ sizeof(req));
+
+ /* Configure the request */
+ req.dpi = cpu_to_le32(srq->dpi->dpi);
+ req.srq_handle = cpu_to_le64((uintptr_t)srq);
+
+ req.srq_size = cpu_to_le16((u16)srq->hwq.max_elements);
+ pbl = &srq->hwq.pbl[BNG_PBL_LVL_0];
+ pg_sz_lvl = ((u16)bng_re_base_pg_size(&srq->hwq) <<
+ CMDQ_CREATE_SRQ_PG_SIZE_SFT);
+ pg_sz_lvl |= (srq->hwq.level & CMDQ_CREATE_SRQ_LVL_MASK) <<
+ CMDQ_CREATE_SRQ_LVL_SFT;
+ req.pg_size_lvl = cpu_to_le16(pg_sz_lvl);
+ req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
+ req.pd_id = cpu_to_le32(srq->pd->id);
+ req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id);
+
+ bng_re_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0);
+ rc = bng_re_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ goto fail;
+
+ srq->id = le32_to_cpu(resp.xid);
+ srq->start_idx = 0;
+ srq->last_idx = srq->hwq.max_elements - 1;
+
+ /* Allocate shadow queue for kernel SRQs */
+ if (!srq->hwq.is_user) {
+ srq->swq = kcalloc(srq->hwq.max_elements, sizeof(*srq->swq),
+ GFP_KERNEL);
+ if (!srq->swq) {
+ rc = -ENOMEM;
+ goto fail_swq;
+ }
+ for (idx = 0; idx < srq->hwq.max_elements; idx++)
+ srq->swq[idx].next_idx = idx + 1;
+ srq->swq[srq->last_idx].next_idx = -1;
+ }
+
+ srq->dbinfo.hwq = &srq->hwq;
+ srq->dbinfo.xid = srq->id;
+ srq->dbinfo.db = srq->dpi->dbr;
+ srq->dbinfo.max_slot = 1;
+ srq->dbinfo.priv_db = res->dpi_tbl.priv_db;
+
+ /* Arm the SRQ */
+ bng_re_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA);
+
+ return 0;
+
+fail_swq:
+ bng_sp_destroy_srq(res, srq);
+ return rc;
+
+fail:
+ bng_re_free_hwq(res, &srq->hwq);
+ kfree(srq->swq);
+ srq->swq = NULL;
+
+ return rc;
+}
+
+int bng_sp_query_srq(struct bng_re_res *res, struct bng_sp_srq *srq)
+{
+ struct bng_re_rcfw *rcfw = res->rcfw;
+ struct creq_query_srq_resp resp = {};
+ struct bng_re_cmdqmsg msg = {};
+ struct bng_re_rcfw_sbuf sbuf;
+ struct creq_query_srq_resp_sb *sb;
+ struct cmdq_query_srq req = {};
+ int rc;
+
+ bng_re_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_QUERY_SRQ,
+ sizeof(req));
+
+ /* Configure the request */
+ sbuf.size = ALIGN(sizeof(*sb), BNG_FW_CMDQE_UNITS);
+ sbuf.sb = dma_alloc_coherent(&rcfw->pdev->dev, sbuf.size,
+ &sbuf.dma_addr, GFP_KERNEL);
+ if (!sbuf.sb)
+ return -ENOMEM;
+ req.resp_size = sbuf.size / BNG_FW_CMDQE_UNITS;
+ req.srq_cid = cpu_to_le32(srq->id);
+ sb = sbuf.sb;
+
+ bng_re_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
+ sizeof(resp), 0);
+ rc = bng_re_rcfw_send_message(rcfw, &msg);
+ if (!rc)
+ srq->threshold = le16_to_cpu(sb->srq_limit);
+ dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
+ sbuf.sb, sbuf.dma_addr);
+
+ return rc;
+}
+
+int bng_sp_destroy_srq(struct bng_re_res *res, struct bng_sp_srq *srq)
+{
+ struct bng_re_rcfw *rcfw = res->rcfw;
+ struct creq_destroy_srq_resp resp = {};
+ struct bng_re_cmdqmsg msg = {};
+ struct cmdq_destroy_srq req = {};
+ int rc;
+
+ bng_re_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_DESTROY_SRQ,
+ sizeof(req));
+
+ /* Configure the request */
+ req.srq_cid = cpu_to_le32(srq->id);
+
+ bng_re_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0);
+ rc = bng_re_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ return rc;
+ bng_re_free_hwq(res, &srq->hwq);
+ kfree(srq->swq);
+
+ return 0;
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.h b/drivers/infiniband/hw/bng_re/bng_sp.h
index d05151123f98..da4dceba4f46 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.h
+++ b/drivers/infiniband/hw/bng_re/bng_sp.h
@@ -166,6 +166,31 @@ struct bng_sp_cq {
u16 cnq_events;
};
+/* SRQ shadow work queue entry */
+struct bng_sp_swq {
+ u64 wr_id;
+ int next_idx;
+};
+
+/* SRQ SP layer structure */
+struct bng_sp_srq {
+ struct bng_sp_pd *pd;
+ struct bng_re_dpi *dpi;
+ struct bng_re_hwq hwq;
+ struct bng_re_db_info dbinfo;
+ struct bng_re_sg_info sginfo;
+ struct bng_sp_swq *swq;
+ u32 start_idx;
+ u32 last_idx;
+ u64 srq_handle;
+ u32 id;
+ u16 wqe_size;
+ u32 max_wqe;
+ u32 max_sge;
+ u32 threshold;
+ u16 eventq_hw_ring_id;
+};
+
int bng_re_get_dev_attr(struct bng_re_rcfw *rcfw);
int bng_sp_alloc_sgid_tbl(struct bng_re_sgid_tbl *sgid_tbl, u16 size);
@@ -200,5 +225,8 @@ void bng_sp_resize_cq_complete(struct bng_re_res *res, struct bng_sp_cq *cq);
int bng_sp_resize_cq(struct bng_re_res *res, struct bng_sp_cq *cq,
int new_cqes);
int bng_sp_destroy_cq(struct bng_re_res *res, struct bng_sp_cq *cq);
+int bng_sp_create_srq(struct bng_re_res *res, struct bng_sp_srq *srq);
+int bng_sp_query_srq(struct bng_re_res *res, struct bng_sp_srq *srq);
+int bng_sp_destroy_srq(struct bng_re_res *res, struct bng_sp_srq *srq);
#endif
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.c b/drivers/infiniband/hw/bng_re/bng_verbs.c
index b08ac20d788d..e48caefa64ee 100644
--- a/drivers/infiniband/hw/bng_re/bng_verbs.c
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.c
@@ -1196,3 +1196,219 @@ int bng_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
kfree(cq->cql);
return 0;
}
+
+/* SRQ helper function */
+static int bng_re_init_user_srq(struct bng_re_dev *rdev,
+ struct bng_re_pd *pd,
+ struct bng_re_srq *srq,
+ struct ib_udata *udata)
+{
+ struct bng_re_ucontext *cntx =
+ rdma_udata_to_drv_context(udata, struct bng_re_ucontext,
+ ib_uctx);
+ struct bng_sp_srq *sp_srq = &srq->sp_srq;
+ struct bng_re_srq_req ureq;
+ struct ib_umem *umem;
+ int bytes = 0;
+
+ if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
+ return -EFAULT;
+
+ bytes = (sp_srq->max_wqe * sp_srq->wqe_size);
+ bytes = PAGE_ALIGN(bytes);
+ umem = ib_umem_get_va(&rdev->ibdev, ureq.srqva, bytes,
+ IB_ACCESS_LOCAL_WRITE);
+ if (IS_ERR(umem))
+ return PTR_ERR(umem);
+
+ srq->umem = umem;
+ sp_srq->sginfo.umem = umem;
+ sp_srq->sginfo.pgsize = PAGE_SIZE;
+ sp_srq->sginfo.pgshft = PAGE_SHIFT;
+ sp_srq->srq_handle = ureq.srq_handle;
+ sp_srq->dpi = &cntx->dpi;
+
+ return 0;
+}
+
+/* SRQ operations */
+int bng_re_create_srq(struct ib_srq *ib_srq,
+ struct ib_srq_init_attr *srq_init_attr,
+ struct ib_udata *udata)
+{
+ struct bng_re_dev_attr *dev_attr;
+ struct bng_re_ucontext *uctx;
+ struct bng_re_dev *rdev;
+ struct bng_re_srq *srq;
+ struct bng_re_pd *pd;
+ struct ib_pd *ib_pd;
+ int rc, entries;
+
+ ib_pd = ib_srq->pd;
+ pd = container_of(ib_pd, struct bng_re_pd, ib_pd);
+ rdev = pd->rdev;
+ dev_attr = rdev->dev_attr;
+ srq = container_of(ib_srq, struct bng_re_srq, ib_srq);
+
+ if (srq_init_attr->attr.max_wr > dev_attr->max_srq_wqes) {
+ ibdev_err(&rdev->ibdev, "Create SRQ failed - max exceeded\n");
+ rc = -EINVAL;
+ goto exit;
+ }
+
+ if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
+ rc = -EOPNOTSUPP;
+ goto exit;
+ }
+
+ uctx = rdma_udata_to_drv_context(udata, struct bng_re_ucontext, ib_uctx);
+ srq->rdev = rdev;
+ srq->sp_srq.pd = &pd->sp_pd;
+ srq->sp_srq.dpi = &rdev->dpi_privileged;
+ /* Allocate 1 more than what's provided so posting max doesn't
+ * mean empty
+ */
+ entries = bng_re_init_depth(srq_init_attr->attr.max_wr + 1, uctx);
+ if (entries > dev_attr->max_srq_wqes + 1)
+ entries = dev_attr->max_srq_wqes + 1;
+ srq->sp_srq.max_wqe = entries;
+
+ srq->sp_srq.max_sge = srq_init_attr->attr.max_sge;
+ /* 128 byte wqe size for SRQ. So use max sges */
+ srq->sp_srq.wqe_size = bng_re_get_rwqe_size(dev_attr->max_srq_sges);
+ srq->sp_srq.threshold = srq_init_attr->attr.srq_limit;
+ srq->srq_limit = srq_init_attr->attr.srq_limit;
+ srq->sp_srq.eventq_hw_ring_id = rdev->nqr->nq[0].ring_id;
+ srq->sp_srq.sginfo.pgsize = PAGE_SIZE;
+ srq->sp_srq.sginfo.pgshft = PAGE_SHIFT;
+
+ if (udata) {
+ rc = bng_re_init_user_srq(rdev, pd, srq, udata);
+ if (rc)
+ goto fail;
+ }
+
+ rc = bng_sp_create_srq(&rdev->bng_res, &srq->sp_srq);
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Create HW SRQ failed!\n");
+ goto fail;
+ }
+
+ if (udata) {
+ struct bng_re_srq_resp resp = {};
+
+ resp.srqid = srq->sp_srq.id;
+ if (rdev->chip_ctx->modes.toggle_bits & BNG_RE_SRQ_TOGGLE_BIT) {
+ srq->uctx_srq_page = (void *)get_zeroed_page(GFP_KERNEL);
+ if (!srq->uctx_srq_page) {
+ rc = -ENOMEM;
+ bng_sp_destroy_srq(&rdev->bng_res, &srq->sp_srq);
+ goto fail;
+ }
+ hash_add(rdev->srq_hash, &srq->hash_entry, srq->sp_srq.id);
+ resp.comp_mask |= BNG_RE_SRQ_TOGGLE_PAGE_SUPPORT;
+ }
+ rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "SRQ copy to udata failed!\n");
+ if (srq->uctx_srq_page) {
+ free_page((unsigned long)srq->uctx_srq_page);
+ hash_del(&srq->hash_entry);
+ }
+ bng_sp_destroy_srq(&rdev->bng_res,
+ &srq->sp_srq);
+ goto fail;
+ }
+ }
+ spin_lock_init(&srq->lock);
+
+ return 0;
+
+fail:
+ ib_umem_release(srq->umem);
+exit:
+ return rc;
+}
+
+int bng_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
+ enum ib_srq_attr_mask srq_attr_mask,
+ struct ib_udata *udata)
+{
+ struct bng_re_srq *srq = container_of(ib_srq, struct bng_re_srq, ib_srq);
+ struct bng_re_dev *rdev = srq->rdev;
+
+ switch (srq_attr_mask) {
+ case IB_SRQ_MAX_WR:
+ /* SRQ resize is not supported */
+ return -EINVAL;
+ case IB_SRQ_LIMIT:
+ {
+ unsigned long flags;
+
+ /* Change the SRQ threshold */
+ if (srq_attr->srq_limit > srq->sp_srq.max_wqe)
+ return -EINVAL;
+
+ spin_lock_irqsave(&srq->lock, flags);
+ srq->sp_srq.threshold = srq_attr->srq_limit;
+ bng_re_srq_arm_db(&srq->sp_srq.dbinfo, srq->sp_srq.threshold);
+
+ /* On success, update the shadow */
+ srq->srq_limit = srq_attr->srq_limit;
+ spin_unlock_irqrestore(&srq->lock, flags);
+ return 0;
+ }
+ default:
+ ibdev_err(&rdev->ibdev,
+ "Unsupported srq_attr_mask 0x%x\n", srq_attr_mask);
+ return -EINVAL;
+ }
+}
+
+int bng_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr)
+{
+ struct bng_re_srq *srq = container_of(ib_srq, struct bng_re_srq, ib_srq);
+ struct bng_re_dev *rdev = srq->rdev;
+ struct bng_sp_srq sp_srq = {};
+ int rc;
+
+ /* Get live SRQ attr */
+ sp_srq.id = srq->sp_srq.id;
+ rc = bng_sp_query_srq(&rdev->bng_res, &sp_srq);
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Query HW SRQ failed!\n");
+ return rc;
+ }
+
+ srq_attr->max_wr = srq->sp_srq.max_wqe;
+ srq_attr->max_sge = srq->sp_srq.max_sge;
+ srq_attr->srq_limit = sp_srq.threshold;
+
+ return 0;
+}
+
+int bng_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
+{
+ struct bng_re_dev *rdev;
+ struct bng_re_srq *srq;
+ int rc;
+
+ srq = container_of(ib_srq, struct bng_re_srq, ib_srq);
+ rdev = srq->rdev;
+
+ rc = bng_sp_destroy_srq(&rdev->bng_res, &srq->sp_srq);
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Destroy SRQ failed: %d\n", rc);
+ return rc;
+ }
+
+ if (srq->uctx_srq_page) {
+ free_page((unsigned long)srq->uctx_srq_page);
+ srq->uctx_srq_page = NULL;
+ hash_del(&srq->hash_entry);
+ }
+
+ ib_umem_release(srq->umem);
+
+ return 0;
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.h b/drivers/infiniband/hw/bng_re/bng_verbs.h
index 839967c73f08..dd9fac0f33c0 100644
--- a/drivers/infiniband/hw/bng_re/bng_verbs.h
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.h
@@ -64,6 +64,17 @@ struct bng_re_cq {
struct hlist_node hash_entry;
};
+struct bng_re_srq {
+ struct ib_srq ib_srq;
+ struct bng_re_dev *rdev;
+ u32 srq_limit;
+ struct bng_sp_srq sp_srq;
+ struct ib_umem *umem;
+ spinlock_t lock; /* protect srq */
+ void *uctx_srq_page;
+ struct hlist_node hash_entry;
+};
+
static inline u32 bng_re_init_depth(u32 ent, struct bng_re_ucontext *uctx)
{
/* roundup_pow_of_two(0) is undefined (UBSAN: shift by 64 on 64-bit). */
@@ -71,6 +82,12 @@ static inline u32 bng_re_init_depth(u32 ent, struct bng_re_ucontext *uctx)
ent : roundup_pow_of_two(ent) : ent;
}
+static inline u16 bng_re_get_rwqe_size(int nsge)
+{
+ /* RQ WQE header + SGEs */
+ return 16 + (nsge * 16); /* header=16, each sge=16 bytes */
+}
+
int bng_re_query_device(struct ib_device *ibdev, struct ib_device_attr *ib_attr,
struct ib_udata *udata);
int bng_re_modify_device(struct ib_device *ibdev, int device_modify_mask,
@@ -122,6 +139,14 @@ int bng_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe, struct ib_udata *udat
int bng_re_req_notify_cq(struct ib_cq *ib_cq,
enum ib_cq_notify_flags ib_cqn_flags);
int bng_re_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
+int bng_re_create_srq(struct ib_srq *ib_srq,
+ struct ib_srq_init_attr *srq_init_attr,
+ struct ib_udata *udata);
+int bng_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
+ enum ib_srq_attr_mask srq_attr_mask,
+ struct ib_udata *udata);
+int bng_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr);
+int bng_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata);
#endif /* __BNG_RE_VERBS_H__ */
diff --git a/include/uapi/rdma/bng_re-abi.h b/include/uapi/rdma/bng_re-abi.h
index 2a88f36bcacd..49931b7c1344 100644
--- a/include/uapi/rdma/bng_re-abi.h
+++ b/include/uapi/rdma/bng_re-abi.h
@@ -61,4 +61,18 @@ struct bng_re_resize_cq_req {
__aligned_u64 cq_va;
};
+struct bng_re_srq_req {
+ __aligned_u64 srqva;
+ __aligned_u64 srq_handle;
+};
+
+enum bng_re_srq_mask {
+ BNG_RE_SRQ_TOGGLE_PAGE_SUPPORT = 0x1,
+};
+
+struct bng_re_srq_resp {
+ __u32 srqid;
+ __aligned_u64 comp_mask;
+};
+
#endif /* __BNG_RE_UVERBS_ABI_H__*/
--
2.43.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 12/15] RDMA/bng_re: Add Stats verbs
2026-09-04 10:43 [PATCH 00/15] Add BNG_RE control path verbs Siva Reddy Kallam
` (10 preceding siblings ...)
2026-09-04 10:43 ` [PATCH 11/15] RDMA/bng_re: Add SRQ verbs Siva Reddy Kallam
@ 2026-09-04 10:43 ` Siva Reddy Kallam
2026-09-04 10:43 ` [PATCH 13/15] RDMA/bng_re: Add AH verbs Siva Reddy Kallam
` (2 subsequent siblings)
14 siblings, 0 replies; 17+ messages in thread
From: Siva Reddy Kallam @ 2026-09-04 10:43 UTC (permalink / raw)
To: leonro, jgg, davem, edumazet, kuba, pabeni, andrew+netdev, horms
Cc: netdev, linux-kernel, linux-rdma, Usman Ansari, Siva Reddy Kallam
From: Usman Ansari <usman.ansari@broadcom.com>
This patch adds below verbs.
- bng_re_alloc_hw_port_stats
- bng_re_ib_get_hw_stats
Signed-off-by: Usman Ansari <usman.ansari@broadcom.com>
Signed-off-by: Siva Reddy Kallam <siva.kallam@broadcom.com>
---
drivers/infiniband/hw/bng_re/Makefile | 1 +
drivers/infiniband/hw/bng_re/bng_dev.c | 3 +
drivers/infiniband/hw/bng_re/bng_fw.h | 12 +
.../infiniband/hw/bng_re/bng_hw_counters.c | 304 ++++++++++++++++++
.../infiniband/hw/bng_re/bng_hw_counters.h | 118 +++++++
drivers/infiniband/hw/bng_re/bng_re.h | 4 +
drivers/infiniband/hw/bng_re/bng_res.h | 21 ++
drivers/infiniband/hw/bng_re/bng_sp.c | 144 +++++++++
drivers/infiniband/hw/bng_re/bng_sp.h | 84 +++++
9 files changed, 691 insertions(+)
create mode 100644 drivers/infiniband/hw/bng_re/bng_hw_counters.c
create mode 100644 drivers/infiniband/hw/bng_re/bng_hw_counters.h
diff --git a/drivers/infiniband/hw/bng_re/Makefile b/drivers/infiniband/hw/bng_re/Makefile
index 5e650a35b185..a9a851cc2fa6 100644
--- a/drivers/infiniband/hw/bng_re/Makefile
+++ b/drivers/infiniband/hw/bng_re/Makefile
@@ -14,4 +14,5 @@ bng_re-y := \
bng_dev.o \
bng_sp.o \
bng_fp.o \
+ bng_hw_counters.o \
bng_verbs.o
diff --git a/drivers/infiniband/hw/bng_re/bng_dev.c b/drivers/infiniband/hw/bng_re/bng_dev.c
index a42e24691cad..978b7efab03e 100644
--- a/drivers/infiniband/hw/bng_re/bng_dev.c
+++ b/drivers/infiniband/hw/bng_re/bng_dev.c
@@ -19,6 +19,7 @@
#include "bng_re_mpc_roce.h"
#include "bng_xid.h"
#include "bng_verbs.h"
+#include "bng_hw_counters.h"
MODULE_AUTHOR("Siva Reddy Kallam <siva.kallam@broadcom.com>");
MODULE_DESCRIPTION(BNG_RE_DESC);
@@ -61,6 +62,8 @@ static const struct ib_device_ops bng_re_dev_ops = {
.modify_srq = bng_re_modify_srq,
.query_srq = bng_re_query_srq,
.destroy_srq = bng_re_destroy_srq,
+ .alloc_hw_port_stats = bng_re_alloc_hw_port_stats,
+ .get_hw_stats = bng_re_ib_get_hw_stats,
INIT_RDMA_OBJ_SIZE(ib_srq, bng_re_srq, ib_srq),
INIT_RDMA_OBJ_SIZE(ib_cq, bng_re_cq, ib_cq),
INIT_RDMA_OBJ_SIZE(ib_pd, bng_re_pd, ib_pd),
diff --git a/drivers/infiniband/hw/bng_re/bng_fw.h b/drivers/infiniband/hw/bng_re/bng_fw.h
index f89e4aeb2469..011dc18592d0 100644
--- a/drivers/infiniband/hw/bng_re/bng_fw.h
+++ b/drivers/infiniband/hw/bng_re/bng_fw.h
@@ -28,11 +28,15 @@
!((pass) & BNG_RE_FLAG_EPOCH_CONS_MASK))
#define BNG_FW_CREQ_ENTRY_POLL_BUDGET 0x100
+#define BNG_RE_OOS_COUNT_MASK 0xFFFFFFFF
+
/* CMDQ */
struct bng_fw_cmdqe {
u8 data[16];
};
+#define BNG_RE_CMDQE_UNITS sizeof(struct bng_fw_cmdqe)
+
#define BNG_FW_CMDQE_MAX_CNT 8192
#define BNG_FW_CMDQE_UNITS sizeof(struct bng_fw_cmdqe)
#define BNG_FW_CMDQE_BYTES(depth) ((depth) * BNG_FW_CMDQE_UNITS)
@@ -40,6 +44,12 @@ struct bng_fw_cmdqe {
#define BNG_FW_MAX_COOKIE_VALUE (BNG_FW_CMDQE_MAX_CNT - 1)
#define BNG_FW_CMD_IS_BLOCKING 0x8000
+struct bng_rcfw_sbuf {
+ void *sb;
+ dma_addr_t dma_addr;
+ u32 size;
+};
+
/* Crsq buf is 1024-Byte */
struct bng_re_crsbe {
u8 data[1024];
@@ -128,6 +138,8 @@ struct bng_re_rcfw {
/* cached from chip cctx for quick reference in slow path */
u16 max_timeout;
atomic_t rcfw_intr_enabled;
+ u64 oos_prev;
+ u32 init_oos_stats;
};
struct bng_re_cmdqmsg {
diff --git a/drivers/infiniband/hw/bng_re/bng_hw_counters.c b/drivers/infiniband/hw/bng_re/bng_hw_counters.c
new file mode 100644
index 000000000000..cb9a00fed621
--- /dev/null
+++ b/drivers/infiniband/hw/bng_re/bng_hw_counters.c
@@ -0,0 +1,304 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2026 Broadcom.
+
+#include <linux/types.h>
+#include <linux/pci.h>
+#include <rdma/ib_mad.h>
+#include <rdma/ib_pma.h>
+#include <rdma/bng_re-abi.h>
+
+#include "bng_res.h"
+#include "bng_sp.h"
+#include "bng_fw.h"
+#include "bnge.h"
+#include "bnge_auxr.h"
+#include "bng_fp.h"
+#include "bng_re.h"
+#include "bng_hw_counters.h"
+
+static const struct rdma_stat_desc bng_re_stat_descs[] = {
+ [BNG_RE_RX_PKTS].name = "rx_pkts",
+ [BNG_RE_RX_BYTES].name = "rx_bytes",
+ [BNG_RE_TX_PKTS].name = "tx_pkts",
+ [BNG_RE_TX_BYTES].name = "tx_bytes",
+ [BNG_RE_RECOVERABLE_ERRORS].name = "recoverable_errors",
+ [BNG_RE_TX_ERRORS].name = "tx_roce_errors",
+ [BNG_RE_TX_DISCARDS].name = "tx_roce_discards",
+ [BNG_RE_RX_ERRORS].name = "rx_roce_errors",
+ [BNG_RE_RX_DISCARDS].name = "rx_roce_discards",
+ [BNG_RE_TO_RETRANSMITS].name = "local_ack_timeout_err",
+ [BNG_RE_SEQ_ERR_NAKS_RCVD].name = "packet_seq_err",
+ [BNG_RE_MAX_RETRY_EXCEEDED].name = "max_retry_exceeded",
+ [BNG_RE_RNR_NAKS_RCVD].name = "rnr_nak_retry_err",
+ [BNG_RE_MISSING_RESP].name = "implied_nak_seq_err",
+ [BNG_RE_UNRECOVERABLE_ERR].name = "unrecoverable_err",
+ [BNG_RE_BAD_RESP_ERR].name = "bad_resp_err",
+ [BNG_RE_LOCAL_QP_OP_ERR].name = "local_qp_op_err",
+ [BNG_RE_LOCAL_PROTECTION_ERR].name = "local_protection_err",
+ [BNG_RE_MEM_MGMT_OP_ERR].name = "mem_mgmt_op_err",
+ [BNG_RE_REMOTE_INVALID_REQ_ERR].name = "req_remote_invalid_request",
+ [BNG_RE_REMOTE_ACCESS_ERR].name = "req_remote_access_errors",
+ [BNG_RE_REMOTE_OP_ERR].name = "remote_op_err",
+ [BNG_RE_DUP_REQ].name = "duplicate_request",
+ [BNG_RE_RES_EXCEED_MAX].name = "res_exceed_max",
+ [BNG_RE_RES_LENGTH_MISMATCH].name = "resp_local_length_error",
+ [BNG_RE_RES_EXCEEDS_WQE].name = "res_exceeds_wqe",
+ [BNG_RE_RES_OPCODE_ERR].name = "res_opcode_err",
+ [BNG_RE_RES_RX_INVALID_RKEY].name = "res_rx_invalid_rkey",
+ [BNG_RE_RES_RX_DOMAIN_ERR].name = "res_rx_domain_err",
+ [BNG_RE_RES_RX_NO_PERM].name = "res_rx_no_perm",
+ [BNG_RE_RES_RX_RANGE_ERR].name = "res_rx_range_err",
+ [BNG_RE_RES_TX_INVALID_RKEY].name = "res_tx_invalid_rkey",
+ [BNG_RE_RES_TX_DOMAIN_ERR].name = "res_tx_domain_err",
+ [BNG_RE_RES_TX_NO_PERM].name = "res_tx_no_perm",
+ [BNG_RE_RES_TX_RANGE_ERR].name = "res_tx_range_err",
+ [BNG_RE_RES_IRRQ_OFLOW].name = "res_irrq_oflow",
+ [BNG_RE_RES_UNSUP_OPCODE].name = "res_unsup_opcode",
+ [BNG_RE_RES_UNALIGNED_ATOMIC].name = "res_unaligned_atomic",
+ [BNG_RE_RES_REM_INV_ERR].name = "res_rem_inv_err",
+ [BNG_RE_RES_MEM_ERROR].name = "res_mem_err",
+ [BNG_RE_RES_SRQ_ERR].name = "res_srq_err",
+ [BNG_RE_RES_CMP_ERR].name = "res_cmp_err",
+ [BNG_RE_RES_INVALID_DUP_RKEY].name = "res_invalid_dup_rkey",
+ [BNG_RE_RES_WQE_FORMAT_ERR].name = "res_wqe_format_err",
+ [BNG_RE_RES_CQ_LOAD_ERR].name = "res_cq_load_err",
+ [BNG_RE_RES_SRQ_LOAD_ERR].name = "res_srq_load_err",
+ [BNG_RE_RES_TX_PCI_ERR].name = "res_tx_pci_err",
+ [BNG_RE_RES_RX_PCI_ERR].name = "res_rx_pci_err",
+ [BNG_RE_OUT_OF_SEQ_ERR].name = "out_of_sequence",
+ [BNG_RE_TX_ATOMIC_REQ].name = "tx_atomic_req",
+ [BNG_RE_TX_READ_REQ].name = "tx_read_req",
+ [BNG_RE_TX_READ_RES].name = "tx_read_resp",
+ [BNG_RE_TX_WRITE_REQ].name = "tx_write_req",
+ [BNG_RE_TX_SEND_REQ].name = "tx_send_req",
+ [BNG_RE_TX_ROCE_PKTS].name = "tx_roce_only_pkts",
+ [BNG_RE_TX_ROCE_BYTES].name = "tx_roce_only_bytes",
+ [BNG_RE_RX_ATOMIC_REQ].name = "rx_atomic_requests",
+ [BNG_RE_RX_READ_REQ].name = "rx_read_requests",
+ [BNG_RE_RX_READ_RESP].name = "rx_read_resp",
+ [BNG_RE_RX_WRITE_REQ].name = "rx_write_requests",
+ [BNG_RE_RX_SEND_REQ].name = "rx_send_req",
+ [BNG_RE_RX_ROCE_PKTS].name = "rx_roce_only_pkts",
+ [BNG_RE_RX_ROCE_BYTES].name = "rx_roce_only_bytes",
+ [BNG_RE_RX_ROCE_GOOD_PKTS].name = "rx_roce_good_pkts",
+ [BNG_RE_RX_ROCE_GOOD_BYTES].name = "rx_roce_good_bytes",
+ [BNG_RE_OOB].name = "out_of_buffer",
+ [BNG_RE_TX_CNP].name = "np_cnp_pkts",
+ [BNG_RE_RX_CNP].name = "rp_cnp_handled",
+ [BNG_RE_RX_ECN].name = "np_ecn_marked_roce_packets",
+ [BNG_RE_REQ_CQE_ERROR].name = "req_cqe_error",
+ [BNG_RE_RESP_CQE_ERROR].name = "resp_cqe_error",
+ [BNG_RE_RESP_REMOTE_ACCESS_ERRS].name = "resp_remote_access_errors",
+};
+
+struct rdma_hw_stats *bng_re_alloc_hw_port_stats(struct ib_device *ibdev,
+ u32 port_num)
+{
+ int num_counters = 0;
+
+ num_counters = BNG_RE_NUM_EXT_COUNTERS;
+
+ return rdma_alloc_hw_stats_struct(bng_re_stat_descs, num_counters,
+ RDMA_HW_STATS_DEFAULT_LIFESPAN);
+}
+
+static void bng_re_copy_err_stats(struct bng_re_dev *rdev,
+ struct rdma_hw_stats *stats,
+ struct bng_roce_stats *err_s)
+{
+ stats->value[BNG_RE_TO_RETRANSMITS] =
+ err_s->to_retransmits;
+ stats->value[BNG_RE_SEQ_ERR_NAKS_RCVD] =
+ err_s->seq_err_naks_rcvd;
+ stats->value[BNG_RE_MAX_RETRY_EXCEEDED] =
+ err_s->max_retry_exceeded;
+ stats->value[BNG_RE_RNR_NAKS_RCVD] =
+ err_s->rnr_naks_rcvd;
+ stats->value[BNG_RE_MISSING_RESP] =
+ err_s->missing_resp;
+ stats->value[BNG_RE_UNRECOVERABLE_ERR] =
+ err_s->unrecoverable_err;
+ stats->value[BNG_RE_BAD_RESP_ERR] =
+ err_s->bad_resp_err;
+ stats->value[BNG_RE_LOCAL_QP_OP_ERR] =
+ err_s->local_qp_op_err;
+ stats->value[BNG_RE_LOCAL_PROTECTION_ERR] =
+ err_s->local_protection_err;
+ stats->value[BNG_RE_MEM_MGMT_OP_ERR] =
+ err_s->mem_mgmt_op_err;
+ stats->value[BNG_RE_REMOTE_INVALID_REQ_ERR] =
+ err_s->remote_invalid_req_err;
+ stats->value[BNG_RE_REMOTE_ACCESS_ERR] =
+ err_s->remote_access_err;
+ stats->value[BNG_RE_REMOTE_OP_ERR] =
+ err_s->remote_op_err;
+ stats->value[BNG_RE_DUP_REQ] =
+ err_s->dup_req;
+ stats->value[BNG_RE_RES_EXCEED_MAX] =
+ err_s->res_exceed_max;
+ stats->value[BNG_RE_RES_LENGTH_MISMATCH] =
+ err_s->res_length_mismatch;
+ stats->value[BNG_RE_RES_EXCEEDS_WQE] =
+ err_s->res_exceeds_wqe;
+ stats->value[BNG_RE_RES_OPCODE_ERR] =
+ err_s->res_opcode_err;
+ stats->value[BNG_RE_RES_RX_INVALID_RKEY] =
+ err_s->res_rx_invalid_rkey;
+ stats->value[BNG_RE_RES_RX_DOMAIN_ERR] =
+ err_s->res_rx_domain_err;
+ stats->value[BNG_RE_RES_RX_NO_PERM] =
+ err_s->res_rx_no_perm;
+ stats->value[BNG_RE_RES_RX_RANGE_ERR] =
+ err_s->res_rx_range_err;
+ stats->value[BNG_RE_RES_TX_INVALID_RKEY] =
+ err_s->res_tx_invalid_rkey;
+ stats->value[BNG_RE_RES_TX_DOMAIN_ERR] =
+ err_s->res_tx_domain_err;
+ stats->value[BNG_RE_RES_TX_NO_PERM] =
+ err_s->res_tx_no_perm;
+ stats->value[BNG_RE_RES_TX_RANGE_ERR] =
+ err_s->res_tx_range_err;
+ stats->value[BNG_RE_RES_IRRQ_OFLOW] =
+ err_s->res_irrq_oflow;
+ stats->value[BNG_RE_RES_UNSUP_OPCODE] =
+ err_s->res_unsup_opcode;
+ stats->value[BNG_RE_RES_UNALIGNED_ATOMIC] =
+ err_s->res_unaligned_atomic;
+ stats->value[BNG_RE_RES_REM_INV_ERR] =
+ err_s->res_rem_inv_err;
+ stats->value[BNG_RE_RES_MEM_ERROR] =
+ err_s->res_mem_error;
+ stats->value[BNG_RE_RES_SRQ_ERR] =
+ err_s->res_srq_err;
+ stats->value[BNG_RE_RES_CMP_ERR] =
+ err_s->res_cmp_err;
+ stats->value[BNG_RE_RES_INVALID_DUP_RKEY] =
+ err_s->res_invalid_dup_rkey;
+ stats->value[BNG_RE_RES_WQE_FORMAT_ERR] =
+ err_s->res_wqe_format_err;
+ stats->value[BNG_RE_RES_CQ_LOAD_ERR] =
+ err_s->res_cq_load_err;
+ stats->value[BNG_RE_RES_SRQ_LOAD_ERR] =
+ err_s->res_srq_load_err;
+ stats->value[BNG_RE_RES_TX_PCI_ERR] =
+ err_s->res_tx_pci_err;
+ stats->value[BNG_RE_RES_RX_PCI_ERR] =
+ err_s->res_rx_pci_err;
+ stats->value[BNG_RE_OUT_OF_SEQ_ERR] =
+ err_s->res_oos_drop_count;
+ stats->value[BNG_RE_REQ_CQE_ERROR] =
+ err_s->bad_resp_err +
+ err_s->local_qp_op_err +
+ err_s->local_protection_err +
+ err_s->mem_mgmt_op_err +
+ err_s->remote_invalid_req_err +
+ err_s->remote_access_err +
+ err_s->remote_op_err;
+ stats->value[BNG_RE_RESP_CQE_ERROR] =
+ err_s->res_cmp_err +
+ err_s->res_cq_load_err;
+ stats->value[BNG_RE_RESP_REMOTE_ACCESS_ERRS] =
+ err_s->res_rx_no_perm +
+ err_s->res_tx_no_perm;
+}
+
+static void bng_re_copy_ext_stats(struct bng_re_dev *rdev,
+ struct rdma_hw_stats *stats,
+ struct bng_re_ext_stat *s)
+{
+ stats->value[BNG_RE_TX_ATOMIC_REQ] = s->tx_atomic_req;
+ stats->value[BNG_RE_TX_READ_REQ] = s->tx_read_req;
+ stats->value[BNG_RE_TX_READ_RES] = s->tx_read_res;
+ stats->value[BNG_RE_TX_WRITE_REQ] = s->tx_write_req;
+ stats->value[BNG_RE_TX_SEND_REQ] = s->tx_send_req;
+ stats->value[BNG_RE_TX_ROCE_PKTS] = s->tx_roce_pkts;
+ stats->value[BNG_RE_TX_ROCE_BYTES] = s->tx_roce_bytes;
+ stats->value[BNG_RE_RX_ATOMIC_REQ] = s->rx_atomic_req;
+ stats->value[BNG_RE_RX_READ_REQ] = s->rx_read_req;
+ stats->value[BNG_RE_RX_READ_RESP] = s->rx_read_res;
+ stats->value[BNG_RE_RX_WRITE_REQ] = s->rx_write_req;
+ stats->value[BNG_RE_RX_SEND_REQ] = s->rx_send_req;
+ stats->value[BNG_RE_RX_ROCE_PKTS] = s->rx_roce_pkts;
+ stats->value[BNG_RE_RX_ROCE_BYTES] = s->rx_roce_bytes;
+ stats->value[BNG_RE_RX_ROCE_GOOD_PKTS] = s->rx_roce_good_pkts;
+ stats->value[BNG_RE_RX_ROCE_GOOD_BYTES] = s->rx_roce_good_bytes;
+ stats->value[BNG_RE_OOB] = s->rx_out_of_buffer;
+ stats->value[BNG_RE_TX_CNP] = s->tx_cnp;
+ stats->value[BNG_RE_RX_CNP] = s->rx_cnp;
+ stats->value[BNG_RE_RX_ECN] = s->rx_ecn_marked;
+ stats->value[BNG_RE_OUT_OF_SEQ_ERR] = s->rx_out_of_sequence;
+}
+
+static int bng_re_get_ext_stat(struct bng_re_dev *rdev,
+ struct rdma_hw_stats *stats)
+{
+ struct bng_re_ext_stat *estat = &rdev->stats.rstat.ext_stat;
+ u32 fid;
+ int rc;
+
+ fid = PCI_FUNC(rdev->aux_dev->pdev->devfn);
+ rc = bng_re_qext_stat(&rdev->rcfw, fid, estat);
+ if (rc)
+ goto done;
+ bng_re_copy_ext_stats(rdev, stats, estat);
+
+done:
+ return rc;
+}
+
+int bng_re_ib_get_hw_stats(struct ib_device *ibdev,
+ struct rdma_hw_stats *stats,
+ u32 port, int index)
+{
+ struct bng_re_dev *rdev = to_bng_re_dev(ibdev, ibdev);
+ struct bng_roce_stats *err_s = NULL;
+ struct ctx_hw_stats *hw_stats = NULL;
+ int rc = 0;
+
+ hw_stats = rdev->ctx.stats.dma;
+ if (!port || !stats)
+ return -EINVAL;
+
+ if (hw_stats) {
+ stats->value[BNG_RE_RECOVERABLE_ERRORS] =
+ le64_to_cpu(hw_stats->tx_bcast_pkts);
+ stats->value[BNG_RE_TX_DISCARDS] =
+ le64_to_cpu(hw_stats->tx_discard_pkts);
+ stats->value[BNG_RE_TX_ERRORS] =
+ le64_to_cpu(hw_stats->tx_error_pkts);
+ stats->value[BNG_RE_RX_ERRORS] =
+ le64_to_cpu(hw_stats->rx_error_pkts);
+ stats->value[BNG_RE_RX_DISCARDS] =
+ le64_to_cpu(hw_stats->rx_discard_pkts);
+ stats->value[BNG_RE_RX_PKTS] =
+ le64_to_cpu(hw_stats->rx_ucast_pkts);
+ stats->value[BNG_RE_RX_BYTES] =
+ le64_to_cpu(hw_stats->rx_ucast_bytes);
+ stats->value[BNG_RE_TX_PKTS] =
+ le64_to_cpu(hw_stats->tx_ucast_pkts);
+ stats->value[BNG_RE_TX_BYTES] =
+ le64_to_cpu(hw_stats->tx_ucast_bytes);
+ }
+ err_s = &rdev->stats.rstat.errs;
+ if (test_bit(BNG_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags)) {
+ rc = bng_re_get_roce_stats(&rdev->rcfw, err_s);
+ if (rc) {
+ clear_bit(BNG_RE_FLAG_ISSUE_ROCE_STATS,
+ &rdev->flags);
+ goto done;
+ }
+ bng_re_copy_err_stats(rdev, stats, err_s);
+ if (bng_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags,
+ rdev->bng_res.is_vf)) {
+ rc = bng_re_get_ext_stat(rdev, stats);
+ if (rc) {
+ clear_bit(BNG_RE_FLAG_ISSUE_ROCE_STATS,
+ &rdev->flags);
+ goto done;
+ }
+ }
+ }
+
+done:
+ return BNG_RE_NUM_EXT_COUNTERS;
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_hw_counters.h b/drivers/infiniband/hw/bng_re/bng_hw_counters.h
new file mode 100644
index 000000000000..792dbb655f7e
--- /dev/null
+++ b/drivers/infiniband/hw/bng_re/bng_hw_counters.h
@@ -0,0 +1,118 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+// Copyright (c) 2026 Broadcom
+//
+#ifndef __HW_COUNTERS_H__
+#define __HW_COUNTERS_H__
+
+struct bng_roce_stats;
+struct bng_re_ext_stat;
+
+enum bng_re_hw_enum_stats {
+ BNG_RE_RX_PKTS,
+ BNG_RE_RX_BYTES,
+ BNG_RE_TX_PKTS,
+ BNG_RE_TX_BYTES,
+ BNG_RE_RECOVERABLE_ERRORS,
+ BNG_RE_TX_ERRORS,
+ BNG_RE_TX_DISCARDS,
+ BNG_RE_RX_ERRORS,
+ BNG_RE_RX_DISCARDS,
+ BNG_RE_TO_RETRANSMITS,
+ BNG_RE_SEQ_ERR_NAKS_RCVD,
+ BNG_RE_MAX_RETRY_EXCEEDED,
+ BNG_RE_RNR_NAKS_RCVD,
+ BNG_RE_MISSING_RESP,
+ BNG_RE_UNRECOVERABLE_ERR,
+ BNG_RE_BAD_RESP_ERR,
+ BNG_RE_LOCAL_QP_OP_ERR,
+ BNG_RE_LOCAL_PROTECTION_ERR,
+ BNG_RE_MEM_MGMT_OP_ERR,
+ BNG_RE_REMOTE_INVALID_REQ_ERR,
+ BNG_RE_REMOTE_ACCESS_ERR,
+ BNG_RE_REMOTE_OP_ERR,
+ BNG_RE_DUP_REQ,
+ BNG_RE_RES_EXCEED_MAX,
+ BNG_RE_RES_LENGTH_MISMATCH,
+ BNG_RE_RES_EXCEEDS_WQE,
+ BNG_RE_RES_OPCODE_ERR,
+ BNG_RE_RES_RX_INVALID_RKEY,
+ BNG_RE_RES_RX_DOMAIN_ERR,
+ BNG_RE_RES_RX_NO_PERM,
+ BNG_RE_RES_RX_RANGE_ERR,
+ BNG_RE_RES_TX_INVALID_RKEY,
+ BNG_RE_RES_TX_DOMAIN_ERR,
+ BNG_RE_RES_TX_NO_PERM,
+ BNG_RE_RES_TX_RANGE_ERR,
+ BNG_RE_RES_IRRQ_OFLOW,
+ BNG_RE_RES_UNSUP_OPCODE,
+ BNG_RE_RES_UNALIGNED_ATOMIC,
+ BNG_RE_RES_REM_INV_ERR,
+ BNG_RE_RES_MEM_ERROR,
+ BNG_RE_RES_SRQ_ERR,
+ BNG_RE_RES_CMP_ERR,
+ BNG_RE_RES_INVALID_DUP_RKEY,
+ BNG_RE_RES_WQE_FORMAT_ERR,
+ BNG_RE_RES_CQ_LOAD_ERR,
+ BNG_RE_RES_SRQ_LOAD_ERR,
+ BNG_RE_RES_TX_PCI_ERR,
+ BNG_RE_RES_RX_PCI_ERR,
+ BNG_RE_OUT_OF_SEQ_ERR,
+ BNG_RE_TX_ATOMIC_REQ,
+ BNG_RE_TX_READ_REQ,
+ BNG_RE_TX_READ_RES,
+ BNG_RE_TX_WRITE_REQ,
+ BNG_RE_TX_SEND_REQ,
+ BNG_RE_TX_ROCE_PKTS,
+ BNG_RE_TX_ROCE_BYTES,
+ BNG_RE_RX_ATOMIC_REQ,
+ BNG_RE_RX_READ_REQ,
+ BNG_RE_RX_READ_RESP,
+ BNG_RE_RX_WRITE_REQ,
+ BNG_RE_RX_SEND_REQ,
+ BNG_RE_RX_ROCE_PKTS,
+ BNG_RE_RX_ROCE_BYTES,
+ BNG_RE_RX_ROCE_GOOD_PKTS,
+ BNG_RE_RX_ROCE_GOOD_BYTES,
+ BNG_RE_OOB,
+ BNG_RE_TX_CNP,
+ BNG_RE_RX_CNP,
+ BNG_RE_RX_ECN,
+ BNG_RE_REQ_CQE_ERROR,
+ BNG_RE_RESP_CQE_ERROR,
+ BNG_RE_RESP_REMOTE_ACCESS_ERRS,
+ BNG_RE_NUM_EXT_COUNTERS
+};
+
+struct bng_re_res_cntrs {
+ atomic_t qp_count;
+ atomic_t rc_qp_count;
+ atomic_t ud_qp_count;
+ atomic_t cq_count;
+ atomic_t srq_count;
+ atomic_t mr_count;
+ atomic_t mr_dmabuf_count;
+ atomic_t peer_mem_mr_count;
+ atomic_t mw_count;
+ atomic_t ah_count;
+ atomic_t ah_hw_count;
+ atomic_t pd_count;
+ atomic_t resize_count;
+ atomic_t max_qp_count;
+ atomic_t max_rc_qp_count;
+ atomic_t max_ud_qp_count;
+ atomic_t max_cq_count;
+ atomic_t max_srq_count;
+ atomic_t max_mr_count;
+ atomic_t max_mr_dmabuf_count;
+ atomic_t max_peer_mem_mr_count;
+ atomic_t max_mw_count;
+ atomic_t max_ah_count;
+ atomic_t max_ah_hw_count;
+ atomic_t max_pd_count;
+};
+
+struct rdma_hw_stats *bng_re_alloc_hw_port_stats(struct ib_device *ibdev,
+ u32 port_num);
+int bng_re_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
+ u32 port, int index);
+#endif /* __HW_COUNTERS_H__ */
diff --git a/drivers/infiniband/hw/bng_re/bng_re.h b/drivers/infiniband/hw/bng_re/bng_re.h
index 70851f1180d0..414768585e67 100644
--- a/drivers/infiniband/hw/bng_re/bng_re.h
+++ b/drivers/infiniband/hw/bng_re/bng_re.h
@@ -9,6 +9,7 @@
#include "bng_res.h"
#include "bng_fw.h"
#include "bng_sp.h"
+#include "bng_hw_counters.h"
#include <rdma/ib_verbs.h>
#define BNG_RE_XID_AVOID_REUSE false
@@ -273,6 +274,7 @@ struct bng_re_dev {
#define BNG_RE_FLAG_NETDEV_REGISTERED 0
#define BNG_RE_FLAG_RCFW_CHANNEL_EN 1
#define BNG_RE_FLAG_TBLS_ALLOC_INITED 2
+#define BNG_RE_FLAG_ISSUE_ROCE_STATS 29
#define BNG_RE_FLAG_MPC_FW_CHANNEL_EN 31
#define BNG_RE_FLAG_MPC_FW_THREAD_EN 32
#define BNG_RE_FLAG_MPC_DB_PAGE_EN 34
@@ -297,6 +299,8 @@ struct bng_re_dev {
struct bng_sp_cq_coal_param cq_coalescing;
DECLARE_HASHTABLE(cq_hash, MAX_CQ_HASH_BITS);
DECLARE_HASHTABLE(srq_hash, 8); /* SRQ hash table */
+ struct bng_re_hw_stats stats;
+ struct bng_re_ctx ctx;
};
#define to_bng_re_dev(ptr, member) \
diff --git a/drivers/infiniband/hw/bng_re/bng_res.h b/drivers/infiniband/hw/bng_re/bng_res.h
index 00429b4a7d30..5a6db1786886 100644
--- a/drivers/infiniband/hw/bng_re/bng_res.h
+++ b/drivers/infiniband/hw/bng_re/bng_res.h
@@ -227,6 +227,15 @@ struct bng_re_res {
/* Serialize access to PD table */
struct mutex pd_tbl_lock;
struct bng_re_rcfw *rcfw;
+ bool is_vf;
+};
+
+struct bng_re_ctx {
+ u32 qpc_count;
+ u32 mrw_count;
+ u32 srqc_count;
+ u32 cq_count;
+ struct bng_re_stats stats;
};
#define to_bng_re_res(ptr, member) container_of(ptr, struct bng_re_res, member)
@@ -394,6 +403,18 @@ static inline u8 bng_re_base_pg_size(struct bng_re_hwq *hwq)
return pg_size;
}
+static inline bool _is_ext_stats_supported(u16 dev_cap_flags)
+{
+ return dev_cap_flags & CREQ_QUERY_FUNC_RESP_SB_EXT_STATS;
+}
+
+static inline int bng_ext_stats_supported(struct bng_re_chip_ctx *ctx,
+ u16 flags, bool virtfn)
+{
+ return (_is_ext_stats_supported(flags) && ((virtfn) || (!virtfn)));
+}
+
+
void bng_re_free_hwq(struct bng_re_res *res,
struct bng_re_hwq *hwq);
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.c b/drivers/infiniband/hw/bng_re/bng_sp.c
index 78e2bb9ad859..26435fe4e892 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.c
+++ b/drivers/infiniband/hw/bng_re/bng_sp.c
@@ -960,3 +960,147 @@ int bng_sp_destroy_srq(struct bng_re_res *res, struct bng_sp_srq *srq)
return 0;
}
+
+int bng_re_get_roce_stats(struct bng_re_rcfw *rcfw, struct bng_roce_stats *stats)
+{
+ struct creq_query_roce_stats_resp resp = {};
+ struct creq_query_roce_stats_resp_sb *sb;
+ struct cmdq_query_roce_stats req = {};
+ struct bng_re_cmdqmsg msg = {};
+ struct bng_rcfw_sbuf sbuf;
+ int rc;
+
+ bng_re_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_QUERY_ROCE_STATS, sizeof(req));
+
+ sbuf.size = ALIGN(sizeof(*sb), BNG_RE_CMDQE_UNITS);
+ sbuf.sb = dma_alloc_coherent(&rcfw->pdev->dev, sbuf.size,
+ &sbuf.dma_addr, GFP_KERNEL);
+ if (!sbuf.sb)
+ return -ENOMEM;
+ sb = sbuf.sb;
+ req.resp_size = sbuf.size / BNG_RE_CMDQE_UNITS;
+ bng_re_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
+ sizeof(resp), 0);
+ rc = bng_re_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ goto bail;
+ stats->to_retransmits = le64_to_cpu(sb->to_retransmits);
+ stats->seq_err_naks_rcvd = le64_to_cpu(sb->seq_err_naks_rcvd);
+ stats->max_retry_exceeded = le64_to_cpu(sb->max_retry_exceeded);
+ stats->rnr_naks_rcvd = le64_to_cpu(sb->rnr_naks_rcvd);
+ stats->missing_resp = le64_to_cpu(sb->missing_resp);
+ stats->unrecoverable_err = le64_to_cpu(sb->unrecoverable_err);
+ stats->bad_resp_err = le64_to_cpu(sb->bad_resp_err);
+ stats->local_qp_op_err = le64_to_cpu(sb->local_qp_op_err);
+ stats->local_protection_err = le64_to_cpu(sb->local_protection_err);
+ stats->mem_mgmt_op_err = le64_to_cpu(sb->mem_mgmt_op_err);
+ stats->remote_invalid_req_err = le64_to_cpu(sb->remote_invalid_req_err);
+ stats->remote_access_err = le64_to_cpu(sb->remote_access_err);
+ stats->remote_op_err = le64_to_cpu(sb->remote_op_err);
+ stats->dup_req = le64_to_cpu(sb->dup_req);
+ stats->res_exceed_max = le64_to_cpu(sb->res_exceed_max);
+ stats->res_length_mismatch = le64_to_cpu(sb->res_length_mismatch);
+ stats->res_exceeds_wqe = le64_to_cpu(sb->res_exceeds_wqe);
+ stats->res_opcode_err = le64_to_cpu(sb->res_opcode_err);
+ stats->res_rx_invalid_rkey = le64_to_cpu(sb->res_rx_invalid_rkey);
+ stats->res_rx_domain_err = le64_to_cpu(sb->res_rx_domain_err);
+ stats->res_rx_no_perm = le64_to_cpu(sb->res_rx_no_perm);
+ stats->res_rx_range_err = le64_to_cpu(sb->res_rx_range_err);
+ stats->res_tx_invalid_rkey = le64_to_cpu(sb->res_tx_invalid_rkey);
+ stats->res_tx_domain_err = le64_to_cpu(sb->res_tx_domain_err);
+ stats->res_tx_no_perm = le64_to_cpu(sb->res_tx_no_perm);
+ stats->res_tx_range_err = le64_to_cpu(sb->res_tx_range_err);
+ stats->res_irrq_oflow = le64_to_cpu(sb->res_irrq_oflow);
+ stats->res_unsup_opcode = le64_to_cpu(sb->res_unsup_opcode);
+ stats->res_unaligned_atomic = le64_to_cpu(sb->res_unaligned_atomic);
+ stats->res_rem_inv_err = le64_to_cpu(sb->res_rem_inv_err);
+ stats->res_mem_error = le64_to_cpu(sb->res_mem_error);
+ stats->res_srq_err = le64_to_cpu(sb->res_srq_err);
+ stats->res_cmp_err = le64_to_cpu(sb->res_cmp_err);
+ stats->res_invalid_dup_rkey = le64_to_cpu(sb->res_invalid_dup_rkey);
+ stats->res_wqe_format_err = le64_to_cpu(sb->res_wqe_format_err);
+ stats->res_cq_load_err = le64_to_cpu(sb->res_cq_load_err);
+ stats->res_srq_load_err = le64_to_cpu(sb->res_srq_load_err);
+ stats->res_tx_pci_err = le64_to_cpu(sb->res_tx_pci_err);
+ stats->res_rx_pci_err = le64_to_cpu(sb->res_rx_pci_err);
+ if (!rcfw->init_oos_stats) {
+ rcfw->oos_prev = le64_to_cpu(sb->res_oos_drop_count);
+ rcfw->init_oos_stats = 1;
+ } else {
+ stats->res_oos_drop_count +=
+ (le64_to_cpu(sb->res_oos_drop_count) -
+ rcfw->oos_prev) & BNG_RE_OOS_COUNT_MASK;
+ rcfw->oos_prev = le64_to_cpu(sb->res_oos_drop_count);
+ }
+
+bail:
+ dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
+ sbuf.sb, sbuf.dma_addr);
+ return rc;
+}
+
+int bng_re_qext_stat(struct bng_re_rcfw *rcfw, u32 fid,
+ struct bng_re_ext_stat *estat)
+{
+ struct creq_query_roce_stats_ext_resp resp = {};
+ struct creq_query_roce_stats_ext_resp_sb *sb;
+ struct cmdq_query_roce_stats_ext req = {};
+ struct bng_re_cmdqmsg msg = {};
+ struct bng_rcfw_sbuf sbuf;
+ int rc;
+
+ sbuf.size = ALIGN(sizeof(*sb), BNG_RE_CMDQE_UNITS);
+ sbuf.sb = dma_alloc_coherent(&rcfw->pdev->dev, sbuf.size,
+ &sbuf.dma_addr, GFP_KERNEL);
+ if (!sbuf.sb)
+ return -ENOMEM;
+
+ sb = sbuf.sb;
+ bng_re_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_QUERY_ROCE_STATS,
+ sizeof(req));
+
+ req.resp_size = sbuf.size / BNG_RE_CMDQE_UNITS;
+ req.resp_addr = cpu_to_le64(sbuf.dma_addr);
+ if (rcfw->res->is_vf)
+ req.function_id =
+ cpu_to_le32(CMDQ_QUERY_ROCE_STATS_EXT_VF_VALID |
+ (fid << CMDQ_QUERY_ROCE_STATS_EXT_VF_NUM_SFT));
+ else
+ req.function_id = cpu_to_le32(fid);
+ req.flags = cpu_to_le16(CMDQ_QUERY_ROCE_STATS_EXT_FLAGS_FUNCTION_ID);
+
+ bng_re_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
+ sizeof(resp), 0);
+ rc = bng_re_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ goto bail;
+
+ estat->tx_atomic_req = le64_to_cpu(sb->tx_atomic_req_pkts);
+ estat->tx_read_req = le64_to_cpu(sb->tx_read_req_pkts);
+ estat->tx_read_res = le64_to_cpu(sb->tx_read_res_pkts);
+ estat->tx_write_req = le64_to_cpu(sb->tx_write_req_pkts);
+ estat->tx_send_req = le64_to_cpu(sb->tx_send_req_pkts);
+ estat->tx_roce_pkts = le64_to_cpu(sb->tx_roce_pkts);
+ estat->tx_roce_bytes = le64_to_cpu(sb->tx_roce_bytes);
+ estat->rx_atomic_req = le64_to_cpu(sb->rx_atomic_req_pkts);
+ estat->rx_read_req = le64_to_cpu(sb->rx_read_req_pkts);
+ estat->rx_read_res = le64_to_cpu(sb->rx_read_res_pkts);
+ estat->rx_write_req = le64_to_cpu(sb->rx_write_req_pkts);
+ estat->rx_send_req = le64_to_cpu(sb->rx_send_req_pkts);
+ estat->rx_roce_pkts = le64_to_cpu(sb->rx_roce_pkts);
+ estat->rx_roce_bytes = le64_to_cpu(sb->rx_roce_bytes);
+ estat->rx_roce_good_pkts = le64_to_cpu(sb->rx_roce_good_pkts);
+ estat->rx_roce_good_bytes = le64_to_cpu(sb->rx_roce_good_bytes);
+ estat->rx_out_of_buffer = le64_to_cpu(sb->rx_out_of_buffer_pkts);
+ estat->rx_out_of_sequence = le64_to_cpu(sb->rx_out_of_sequence_pkts);
+ estat->tx_cnp = le64_to_cpu(sb->tx_cnp_pkts);
+ estat->rx_cnp = le64_to_cpu(sb->rx_cnp_pkts);
+ estat->rx_ecn_marked = le64_to_cpu(sb->rx_ecn_marked_pkts);
+
+bail:
+ dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
+ sbuf.sb, sbuf.dma_addr);
+ return rc;
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.h b/drivers/infiniband/hw/bng_re/bng_sp.h
index da4dceba4f46..d69d97e1940b 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.h
+++ b/drivers/infiniband/hw/bng_re/bng_sp.h
@@ -191,6 +191,86 @@ struct bng_sp_srq {
u16 eventq_hw_ring_id;
};
+struct bng_roce_stats {
+ u64 to_retransmits;
+ u64 seq_err_naks_rcvd;
+ u64 max_retry_exceeded;
+ u64 rnr_naks_rcvd;
+ u64 missing_resp;
+ u64 unrecoverable_err;
+ u64 bad_resp_err;
+ u64 local_qp_op_err;
+ u64 local_protection_err;
+ u64 mem_mgmt_op_err;
+ u64 remote_invalid_req_err;
+ u64 remote_access_err;
+ u64 remote_op_err;
+ u64 dup_req;
+ u64 res_exceed_max;
+ u64 res_length_mismatch;
+ u64 res_exceeds_wqe;
+ u64 res_opcode_err;
+ u64 res_rx_invalid_rkey;
+ u64 res_rx_domain_err;
+ u64 res_rx_no_perm;
+ u64 res_rx_range_err;
+ u64 res_tx_invalid_rkey;
+ u64 res_tx_domain_err;
+ u64 res_tx_no_perm;
+ u64 res_tx_range_err;
+ u64 res_irrq_oflow;
+ u64 res_unsup_opcode;
+ u64 res_unaligned_atomic;
+ u64 res_rem_inv_err;
+ u64 res_mem_error;
+ u64 res_srq_err;
+ u64 res_cmp_err;
+ u64 res_invalid_dup_rkey;
+ u64 res_wqe_format_err;
+ u64 res_cq_load_err;
+ u64 res_srq_load_err;
+ u64 res_tx_pci_err;
+ u64 res_rx_pci_err;
+ u64 res_oos_drop_count;
+ u64 active_qp_count_p0;
+ u64 active_qp_count_p1;
+ u64 active_qp_count_p2;
+ u64 active_qp_count_p3;
+};
+
+struct bng_re_ext_stat {
+ u64 tx_atomic_req;
+ u64 tx_read_req;
+ u64 tx_read_res;
+ u64 tx_write_req;
+ u64 tx_send_req;
+ u64 tx_roce_pkts;
+ u64 tx_roce_bytes;
+ u64 rx_atomic_req;
+ u64 rx_read_req;
+ u64 rx_read_res;
+ u64 rx_write_req;
+ u64 rx_send_req;
+ u64 rx_roce_pkts;
+ u64 rx_roce_bytes;
+ u64 rx_roce_good_pkts;
+ u64 rx_roce_good_bytes;
+ u64 rx_out_of_buffer;
+ u64 rx_out_of_sequence;
+ u64 tx_cnp;
+ u64 rx_cnp;
+ u64 rx_ecn_marked;
+};
+
+struct bng_re_rstat {
+ struct bng_roce_stats errs;
+ struct bng_re_ext_stat ext_stat;
+};
+
+struct bng_re_hw_stats {
+ struct bng_re_rstat rstat;
+};
+
int bng_re_get_dev_attr(struct bng_re_rcfw *rcfw);
int bng_sp_alloc_sgid_tbl(struct bng_re_sgid_tbl *sgid_tbl, u16 size);
@@ -228,5 +308,9 @@ int bng_sp_destroy_cq(struct bng_re_res *res, struct bng_sp_cq *cq);
int bng_sp_create_srq(struct bng_re_res *res, struct bng_sp_srq *srq);
int bng_sp_query_srq(struct bng_re_res *res, struct bng_sp_srq *srq);
int bng_sp_destroy_srq(struct bng_re_res *res, struct bng_sp_srq *srq);
+int bng_re_qext_stat(struct bng_re_rcfw *rcfw, u32 fid,
+ struct bng_re_ext_stat *estat);
+int bng_re_get_roce_stats(struct bng_re_rcfw *rcfw,
+ struct bng_roce_stats *stats);
#endif
--
2.43.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 13/15] RDMA/bng_re: Add AH verbs
2026-09-04 10:43 [PATCH 00/15] Add BNG_RE control path verbs Siva Reddy Kallam
` (11 preceding siblings ...)
2026-09-04 10:43 ` [PATCH 12/15] RDMA/bng_re: Add Stats verbs Siva Reddy Kallam
@ 2026-09-04 10:43 ` Siva Reddy Kallam
2026-09-04 10:43 ` [PATCH 14/15] RDMA/bng_re: Add QP verbs Siva Reddy Kallam
2026-09-04 10:43 ` [PATCH 15/15] RDMA/bng_re: Register with ib-core Siva Reddy Kallam
14 siblings, 0 replies; 17+ messages in thread
From: Siva Reddy Kallam @ 2026-09-04 10:43 UTC (permalink / raw)
To: leonro, jgg, davem, edumazet, kuba, pabeni, andrew+netdev, horms
Cc: netdev, linux-kernel, linux-rdma, Mohammed Faraaz,
Siva Reddy Kallam
From: Mohammed Faraaz <mohammed.faraaz@broadcom.com>
This patch adds below verbs.
- bng_re_create_ah
- bng_re_destroy_ah
- bng_re_query_ah
Signed-off-by: Mohammed Faraaz <mohammed.faraaz@broadcom.com>
Signed-off-by: Siva Reddy Kallam <siva.kallam@broadcom.com>
---
drivers/infiniband/hw/bng_re/bng_dev.c | 16 ++
drivers/infiniband/hw/bng_re/bng_re.h | 2 +
drivers/infiniband/hw/bng_re/bng_roce_hsi.h | 65 ++++++
drivers/infiniband/hw/bng_re/bng_sp.c | 120 +++++++++++
drivers/infiniband/hw/bng_re/bng_sp.h | 19 ++
drivers/infiniband/hw/bng_re/bng_verbs.c | 215 ++++++++++++++++++++
drivers/infiniband/hw/bng_re/bng_verbs.h | 29 +++
include/uapi/rdma/bng_re-abi.h | 5 +
8 files changed, 471 insertions(+)
diff --git a/drivers/infiniband/hw/bng_re/bng_dev.c b/drivers/infiniband/hw/bng_re/bng_dev.c
index 978b7efab03e..3ebdd777eca3 100644
--- a/drivers/infiniband/hw/bng_re/bng_dev.c
+++ b/drivers/infiniband/hw/bng_re/bng_dev.c
@@ -64,6 +64,11 @@ static const struct ib_device_ops bng_re_dev_ops = {
.destroy_srq = bng_re_destroy_srq,
.alloc_hw_port_stats = bng_re_alloc_hw_port_stats,
.get_hw_stats = bng_re_ib_get_hw_stats,
+ .create_ah = bng_re_create_ah,
+ .create_user_ah = bng_re_create_ah,
+ .destroy_ah = bng_re_destroy_ah,
+ .query_ah = bng_re_query_ah,
+ INIT_RDMA_OBJ_SIZE(ib_ah, bng_re_ah, ib_ah),
INIT_RDMA_OBJ_SIZE(ib_srq, bng_re_srq, ib_srq),
INIT_RDMA_OBJ_SIZE(ib_cq, bng_re_cq, ib_cq),
INIT_RDMA_OBJ_SIZE(ib_pd, bng_re_pd, ib_pd),
@@ -333,6 +338,11 @@ static void bng_re_dev_uninit(struct bng_re_dev *rdev)
{
int rc;
+ if (rdev->dest_ah_wq) {
+ destroy_workqueue(rdev->dest_ah_wq);
+ rdev->dest_ah_wq = NULL;
+ }
+
if (rdev->bng_res.dpi_tbl.max)
bng_re_dealloc_dpi(&rdev->bng_res,
&rdev->dpi_privileged);
@@ -504,6 +514,12 @@ static int bng_re_dev_init(struct bng_re_dev *rdev)
rdev, BNG_RE_DPI_TYPE_KERNEL);
if (rc)
goto deinit_mpc;
+ rdev->dest_ah_wq = alloc_ordered_workqueue("bng_re_dest_ah", 0);
+ if (!rdev->dest_ah_wq) {
+ rc = -ENOMEM;
+ goto deinit_mpc;
+ }
+ refcount_set(&rdev->pos_destah_cnt, 0);
return 0;
deinit_mpc:
diff --git a/drivers/infiniband/hw/bng_re/bng_re.h b/drivers/infiniband/hw/bng_re/bng_re.h
index 414768585e67..217383a2b252 100644
--- a/drivers/infiniband/hw/bng_re/bng_re.h
+++ b/drivers/infiniband/hw/bng_re/bng_re.h
@@ -301,6 +301,8 @@ struct bng_re_dev {
DECLARE_HASHTABLE(srq_hash, 8); /* SRQ hash table */
struct bng_re_hw_stats stats;
struct bng_re_ctx ctx;
+ struct workqueue_struct *dest_ah_wq;
+ refcount_t pos_destah_cnt;
};
#define to_bng_re_dev(ptr, member) \
diff --git a/drivers/infiniband/hw/bng_re/bng_roce_hsi.h b/drivers/infiniband/hw/bng_re/bng_roce_hsi.h
index 6c9d45464ef6..3001b9d4d056 100644
--- a/drivers/infiniband/hw/bng_re/bng_roce_hsi.h
+++ b/drivers/infiniband/hw/bng_re/bng_roce_hsi.h
@@ -6541,4 +6541,69 @@ struct mpc_event_resp_cmpl {
__le32 resp_data;
};
+#define AH_MODIFY_DATA_AH_MODIFY_FLAGS_AH_MODIFY_FLAGS_AH_CREATE 0x1UL
+#define AH_MODIFY_DATA_AH_MODIFY_FLAGS_AH_MODIFY_FLAGS_AH_DESTROY 0x2UL
+#define AH_MODIFY_DATA_AH_MODIFY_FLAGS_AH_MODIFY_FLAGS_XID_FIRST_USE 0x4UL
+
+struct ah_modify_data {
+ __le32 ah_modify_flags;
+ __le32 reserved_0;
+ u8 type;
+ u8 unused_1[3];
+ __le32 pd_id;
+ u8 hop_limit;
+ u8 traffic_class;
+ __le16 sgid_index;
+ __le32 dest_vlan_id_flow_label;
+ __le64 ah_handle;
+ __le32 dgid[4];
+ __le16 dest_mac[3];
+ u8 enable_cc;
+ u8 unused_2[1];
+ __le32 xid;
+ __le32 reserved_1;
+};
+
+struct mpc_ah_modify_cmd {
+ u8 req_type;
+ u8 req_subtype;
+ __le16 cookie;
+ __le16 target_id;
+ u8 resp_size;
+ u8 reserved;
+ __le64 resp_addr;
+ __le32 ah_modify_flags;
+ __le32 reserved_0;
+ u8 type;
+ u8 unused_1[3];
+ __le32 pd_id;
+ u8 hop_limit;
+ u8 traffic_class;
+ __le16 sgid_index;
+ __le32 dest_vlan_id_flow_label;
+ __le64 ah_handle;
+ __le32 dgid[4];
+ __le16 dest_mac[3];
+ u8 enable_cc;
+ u8 unused_2[1];
+ __le32 xid;
+ __le32 reserved_1;
+};
+
+struct mpc_ah_modify_cmpl {
+ u8 cmpl_type_reserved;
+ u8 error_code_mp_client;
+ u8 req_type;
+ u8 req_subtype;
+ __le32 opaque;
+ u8 v;
+ u8 reserved_0[7];
+ __le32 xid;
+ __le16 error_code;
+ u8 reserved_1[2];
+ u8 v2;
+ u8 reserved_2[3];
+ __le32 reserved_3;
+};
+
#endif /* _BNG_RE_HSI_H_ */
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.c b/drivers/infiniband/hw/bng_re/bng_sp.c
index 26435fe4e892..b44af599e675 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.c
+++ b/drivers/infiniband/hw/bng_re/bng_sp.c
@@ -8,6 +8,8 @@
#include "bng_sp.h"
#include "bng_tlv.h"
#include "bng_re.h"
+#include "bng_re_mpc_roce.h"
+#include "bng_fp.h"
const struct bng_re_gid bng_re_gid_zero = {{0,}};
@@ -1104,3 +1106,121 @@ int bng_re_qext_stat(struct bng_re_rcfw *rcfw, u32 fid,
sbuf.sb, sbuf.dma_addr);
return rc;
}
+
+static int bng_sp_fill_mpc_ah_modify_parms(struct bng_re_res *res,
+ struct ah_modify_data *data,
+ struct bng_sp_ah *ah,
+ bool for_destroy)
+{
+ u32 flags = 0;
+
+ if (for_destroy) {
+ flags |= AH_MODIFY_DATA_AH_MODIFY_FLAGS_AH_MODIFY_FLAGS_AH_DESTROY;
+ } else {
+ flags |= AH_MODIFY_DATA_AH_MODIFY_FLAGS_AH_MODIFY_FLAGS_AH_CREATE;
+ flags |= AH_MODIFY_DATA_AH_MODIFY_FLAGS_AH_MODIFY_FLAGS_XID_FIRST_USE;
+ }
+ data->ah_modify_flags = cpu_to_le32(flags);
+ data->ah_handle = cpu_to_le64((u64)ah);
+ data->xid = cpu_to_le32(ah->id);
+
+ if (!for_destroy) {
+ if (ah->pd)
+ data->pd_id = cpu_to_le32(ah->pd->id);
+ data->hop_limit = ah->hop_limit;
+ data->traffic_class = ah->traffic_class;
+ data->sgid_index = cpu_to_le16(ah->sgid_index);
+ data->dest_vlan_id_flow_label = cpu_to_le32(ah->flow_label | (ah->vlan_id << 16));
+ memcpy(data->dgid, ah->dgid.data, sizeof(data->dgid));
+ memcpy(data->dest_mac, ah->dmac, sizeof(ah->dmac));
+ }
+
+ dev_dbg(&res->pdev->dev, "MPC: AH modify params - AH ID=%d, flags=0x%x, for_destroy=%s\n",
+ ah->id, flags, for_destroy ? "true" : "false");
+
+ return 0;
+}
+
+int bng_sp_fill_and_send_mpc_ah_modify(struct bng_re_res *res,
+ struct bng_sp_ah **ahs,
+ int num_ahs, bool for_destroy, bool block)
+{
+ struct bng_re_dev *rdev = to_bng_re_dev(res, bng_res);
+ struct mpc_ah_modify_cmpl resp;
+ struct mpc_ah_modify_cmd cmd;
+ struct ah_modify_data *data;
+ struct bng_sp_ah *ah;
+ int rc = 0;
+
+ /* for 1st iteration, don't allow batch */
+ if (num_ahs != 1)
+ return -EINVAL;
+
+ ah = *ahs;
+
+ if (!for_destroy) {
+ if (res->is_vf)
+ return -EOPNOTSUPP;
+
+ if (res->ah_xids) {
+ int xid;
+
+ xid = bng_re_xm_alloc_range(res->ah_xids, 1, 0);
+ if (xid < 0) {
+ dev_err(&res->pdev->dev,
+ "%s - error allocating ah xid\n", __func__);
+ return -ENOMEM;
+ }
+ ah->id = xid;
+ } else {
+ ah->id = 1;
+ }
+ }
+
+ memset(&cmd, 0, sizeof(cmd));
+ memset(&resp, 0, sizeof(resp));
+ data = (struct ah_modify_data *)&cmd.ah_modify_flags;
+
+ bng_sp_fill_mpc_ah_modify_parms(res, data, ah, for_destroy);
+
+ cmd.resp_size = sizeof(resp) / BNG_RE_MPC_CQ_STRIDE;
+
+ /* Set MPC command type based on VF vs PF context */
+ if (res->is_vf) {
+ cmd.req_type = MPC_CMD_HDR_REQ_TYPE_PFVF;
+ cmd.req_subtype = MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_AH_MODIFY;
+ dev_dbg(&res->pdev->dev, "MPC: [%s] VF path\n", __func__);
+ } else {
+ cmd.req_type = MPC_CMD_HDR_REQ_TYPE_RCA;
+ cmd.req_subtype = MPC_CMD_HDR_REQ_SUB_TYPE_RCA_AH_MODIFY;
+ dev_dbg(&res->pdev->dev, "MPC: [%s] PF path\n", __func__);
+ }
+
+ dev_dbg(&res->pdev->dev, "MPC: %s - sending to mpc: AH ID=%d, block=%s\n",
+ __func__, ah->id, block ? "true" : "false");
+
+ if (!rdev) {
+ dev_err(&res->pdev->dev, "%s: no rdev for MPC xmit\n", __func__);
+ return -ENODEV;
+ }
+
+ rc = bng_re_roce_mpc_xmit(rdev, &cmd, sizeof(cmd), &resp, sizeof(resp), block);
+
+ if (rc < 0) {
+ dev_err(&res->pdev->dev, "%s - error sending mpc:%d,%d",
+ __func__, ah->id, rc);
+
+ /* Cleanup on error */
+ if (!res->is_vf && res->ah_xids)
+ bng_re_xm_free_range(res->ah_xids, ah->id, true);
+ return rc;
+ }
+
+ dev_info(&res->pdev->dev, "MPC: AH %s completed for AH ID=%d\n",
+ for_destroy ? "destroy" : "create", ah->id);
+
+ if (for_destroy && !res->is_vf && res->ah_xids)
+ bng_re_xm_free_range(res->ah_xids, ah->id, true);
+
+ return rc;
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.h b/drivers/infiniband/hw/bng_re/bng_sp.h
index d69d97e1940b..a6f688c0af3d 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.h
+++ b/drivers/infiniband/hw/bng_re/bng_sp.h
@@ -271,6 +271,22 @@ struct bng_re_hw_stats {
struct bng_re_rstat rstat;
};
+struct bng_sp_ah {
+ struct bng_re_gid dgid;
+ struct bng_sp_pd *pd;
+ u32 id;
+ u8 sgid_index;
+ /* For Query AH if the hw table and SW table are different */
+ u8 host_sgid_index;
+ u8 traffic_class;
+ u32 flow_label;
+ u8 hop_limit;
+ u8 sl;
+ u8 dmac[6];
+ u16 vlan_id;
+ u8 nw_type;
+};
+
int bng_re_get_dev_attr(struct bng_re_rcfw *rcfw);
int bng_sp_alloc_sgid_tbl(struct bng_re_sgid_tbl *sgid_tbl, u16 size);
@@ -312,5 +328,8 @@ int bng_re_qext_stat(struct bng_re_rcfw *rcfw, u32 fid,
struct bng_re_ext_stat *estat);
int bng_re_get_roce_stats(struct bng_re_rcfw *rcfw,
struct bng_roce_stats *stats);
+int bng_sp_fill_and_send_mpc_ah_modify(struct bng_re_res *res,
+ struct bng_sp_ah **ahs,
+ int num_ahs, bool for_destroy, bool block);
#endif
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.c b/drivers/infiniband/hw/bng_re/bng_verbs.c
index e48caefa64ee..c40f8686b6f6 100644
--- a/drivers/infiniband/hw/bng_re/bng_verbs.c
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.c
@@ -1412,3 +1412,218 @@ int bng_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
return 0;
}
+
+void bng_re_posted_destroy_ah(struct work_struct *work)
+{
+ struct bng_destroy_ah *ah = container_of(work, struct bng_destroy_ah, dwork.work);
+ struct bng_re_dev *rdev = ah->rdev;
+ struct bng_sp_ah *ahs[1];
+ u32 ah_ref_cnt = 0;
+ int rc = 0;
+
+ ahs[0] = &ah->sp_ah;
+
+ ah_ref_cnt = refcount_read(ah->ah_ref_cnt);
+ if (rdev->dest_ah_wq && ah_ref_cnt && ah->retry) {
+ ah->retry--;
+ queue_delayed_work(rdev->dest_ah_wq, &ah->dwork, BNG_RE_AH_DEST_DELAY);
+ return;
+ }
+
+ if (!ah->retry)
+ dev_dbg(rdev_to_dev(rdev), "Non-zero AH reference %d for AH id %d\n",
+ ah_ref_cnt, ah->sp_ah.id);
+
+ dev_dbg(&rdev->ibdev.dev,
+ "MPC: %s - sending AH modify(destroy) for: res:%p, ah:%p\n",
+ __func__, &rdev->bng_res, &ah->sp_ah);
+ rc = bng_sp_fill_and_send_mpc_ah_modify(&rdev->bng_res,
+ ahs, 1, true, false);
+ if (rc)
+ dev_err_ratelimited(&rdev->ibdev.dev,
+ "%s failed to destroy AH id = %d rc = %d",
+ __func__, ah->sp_ah.id, rc);
+ if (!ah_ref_cnt) {
+ kfree(ah->ah_ref_cnt);
+ ah->ah_ref_cnt = NULL;
+ }
+ kfree(ah);
+ refcount_dec(&rdev->pos_destah_cnt);
+}
+
+static u8 bng_re_stack_to_dev_nw_type(enum rdma_network_type ntype)
+{
+ u8 nw_type;
+
+ switch (ntype) {
+ case RDMA_NETWORK_IPV4:
+ nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4;
+ break;
+ case RDMA_NETWORK_IPV6:
+ nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6;
+ break;
+ default:
+ nw_type = CMDQ_CREATE_AH_TYPE_V1;
+ break;
+ }
+ return nw_type;
+}
+
+int bng_re_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
+ struct ib_udata *udata)
+{
+ struct ib_pd *ibpd = ibah->pd;
+ struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
+ const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
+ struct bng_re_dev *rdev = to_bng_re_dev(ibpd->device, ibdev);
+ struct bng_re_ah *ah = to_bng_re_ah(ibah);
+ const struct ib_gid_attr *sgid_attr;
+ struct bng_re_gid_ctx *ctx;
+ enum rdma_network_type nw_type;
+ int rc;
+
+ if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) {
+ ibdev_err(&rdev->ibdev, "Failed to create AH: GRH not set");
+ return -EINVAL;
+ }
+
+ ah->rdev = rdev;
+ if (ibpd) {
+ struct bng_re_pd *pd = container_of(ibpd, struct bng_re_pd, ib_pd);
+
+ ah->sp_ah.pd = &pd->sp_pd;
+ }
+
+ /* Supply the configuration for the HW */
+ memcpy(ah->sp_ah.dgid.data, grh->dgid.raw, sizeof(union ib_gid));
+ sgid_attr = grh->sgid_attr;
+ /* Get the HW context of the GID. The reference
+ * of GID table entry is already taken by the caller.
+ */
+ ctx = rdma_read_gid_hw_context(sgid_attr);
+ if (!ctx) {
+ ibdev_err(&rdev->ibdev, "Failed to read GID HW context");
+ return -EINVAL;
+ }
+ ah->sp_ah.sgid_index = ctx->idx;
+ ah->sp_ah.host_sgid_index = grh->sgid_index;
+ ah->sp_ah.flow_label = grh->flow_label;
+ ah->sp_ah.hop_limit = grh->hop_limit;
+ ah->sp_ah.traffic_class = grh->traffic_class;
+ ah->sp_ah.sl = rdma_ah_get_sl(ah_attr);
+
+ /* Get network type from GID attribute */
+ nw_type = rdma_gid_attr_network_type(sgid_attr);
+ ah->sp_ah.nw_type = bng_re_stack_to_dev_nw_type(nw_type);
+
+ memcpy(ah->sp_ah.dmac, ah_attr->roce.dmac, ETH_ALEN);
+
+ struct bng_sp_ah *ahp[1] = { &ah->sp_ah };
+
+ dev_dbg(&rdev->ibdev.dev, "MPC: %s - sending AH modify(create) for: res:%p, ah:%p\n",
+ __func__, &rdev->bng_res, &ah->sp_ah);
+ rc = bng_sp_fill_and_send_mpc_ah_modify(&rdev->bng_res,
+ ahp, 1, false, false);
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Failed to create HW AH");
+ return rc;
+ }
+
+ /* Allocate and initialize reference count for deferred destruction */
+ ah->ref_cnt = kzalloc_obj(*ah->ref_cnt, GFP_KERNEL);
+ if (!ah->ref_cnt) {
+ rc = -ENOMEM;
+ goto fail_destroy;
+ }
+ refcount_set(ah->ref_cnt, 1);
+
+ if (udata) {
+ struct bng_re_ah_resp resp = {};
+
+ resp.ah_id = ah->sp_ah.id;
+ rc = ib_copy_to_udata(udata, &resp,
+ min(udata->outlen, sizeof(resp)));
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Failed to copy AH response");
+ goto fail_ref;
+ }
+ }
+
+ ibdev_dbg(&rdev->ibdev, "AH created with ID: %d", ah->sp_ah.id);
+ return 0;
+
+fail_ref:
+ kfree(ah->ref_cnt);
+ ah->ref_cnt = NULL;
+fail_destroy:
+ struct bng_sp_ah *ahs[1] = { &ah->sp_ah };
+
+ bng_sp_fill_and_send_mpc_ah_modify(&rdev->bng_res,
+ ahs, 1, true, true);
+ return rc;
+}
+
+int bng_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)
+{
+ struct bng_re_ah *ah = to_bng_re_ah(ib_ah);
+ struct bng_re_dev *rdev = ah->rdev;
+ struct bng_destroy_ah *pos_ah;
+ struct bng_sp_ah *ahs[1];
+ bool zero_ref = false;
+ bool block = true;
+ int rc = 0;
+
+ block = !!(flags & RDMA_DESTROY_AH_SLEEPABLE);
+ ahs[0] = &ah->sp_ah;
+
+ zero_ref = refcount_dec_and_test(ah->ref_cnt);
+ if (rdev->dest_ah_wq && (block || !zero_ref)) {
+ pos_ah = kzalloc_obj(*pos_ah, GFP_ATOMIC);
+ if (!pos_ah) {
+ rc = bng_sp_fill_and_send_mpc_ah_modify(&rdev->bng_res,
+ ahs, 1, true, block);
+ if (rc)
+ dev_err_ratelimited(&rdev->ibdev.dev,
+ "%s id = %d dest_ah failed rc = %d",
+ __func__, ah->sp_ah.id, rc);
+ if (zero_ref)
+ kfree(ah->ref_cnt);
+ } else {
+ INIT_DELAYED_WORK(&pos_ah->dwork, bng_re_posted_destroy_ah);
+ pos_ah->rdev = rdev;
+ pos_ah->ah_ref_cnt = ah->ref_cnt;
+ pos_ah->retry = BNG_RE_AH_DEST_RETRY;
+ memcpy(&pos_ah->sp_ah, &ah->sp_ah, sizeof(pos_ah->sp_ah));
+ refcount_inc(&rdev->pos_destah_cnt);
+ queue_delayed_work(rdev->dest_ah_wq, &pos_ah->dwork, 0);
+ /* We queued it, do not free ah->ref_cnt yet! */
+ }
+ } else {
+ rc = bng_sp_fill_and_send_mpc_ah_modify(&rdev->bng_res,
+ ahs, 1, true, block);
+ if (rc)
+ dev_err_ratelimited(&rdev->ibdev.dev,
+ "%s id = %d dest_ah failed rc = %d",
+ __func__, ah->sp_ah.id, rc);
+ if (zero_ref)
+ kfree(ah->ref_cnt);
+ }
+ return 0;
+}
+
+int bng_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
+{
+ struct bng_re_ah *ah = to_bng_re_ah(ib_ah);
+
+ ah_attr->type = ib_ah->type;
+ ah_attr->ah_flags = IB_AH_GRH;
+ rdma_ah_set_sl(ah_attr, ah->sp_ah.sl);
+ memcpy(ah_attr->roce.dmac, ah->sp_ah.dmac, ETH_ALEN);
+ rdma_ah_set_grh(ah_attr, NULL, 0,
+ ah->sp_ah.host_sgid_index,
+ 0, ah->sp_ah.traffic_class);
+ rdma_ah_set_dgid_raw(ah_attr, ah->sp_ah.dgid.data);
+ rdma_ah_set_port_num(ah_attr, 1);
+ rdma_ah_set_static_rate(ah_attr, 0);
+ return 0;
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.h b/drivers/infiniband/hw/bng_re/bng_verbs.h
index dd9fac0f33c0..c5643ee70885 100644
--- a/drivers/infiniband/hw/bng_re/bng_verbs.h
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.h
@@ -75,6 +75,30 @@ struct bng_re_srq {
struct hlist_node hash_entry;
};
+/* Deferred AH destruction constants */
+#define BNG_RE_AH_DEST_DELAY msecs_to_jiffies(5000)
+#define BNG_RE_AH_DEST_RETRY 10
+
+struct bng_destroy_ah {
+ struct delayed_work dwork;
+ struct bng_re_dev *rdev;
+ struct bng_sp_ah sp_ah;
+ refcount_t *ah_ref_cnt;
+ u8 retry;
+};
+
+struct bng_re_ah {
+ struct ib_ah ib_ah;
+ struct bng_re_dev *rdev;
+ struct bng_sp_ah sp_ah;
+ refcount_t *ref_cnt;
+};
+
+static inline struct bng_re_ah *to_bng_re_ah(struct ib_ah *ibah)
+{
+ return container_of(ibah, struct bng_re_ah, ib_ah);
+}
+
static inline u32 bng_re_init_depth(u32 ent, struct bng_re_ucontext *uctx)
{
/* roundup_pow_of_two(0) is undefined (UBSAN: shift by 64 on 64-bit). */
@@ -147,6 +171,11 @@ int bng_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
struct ib_udata *udata);
int bng_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr);
int bng_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata);
+int bng_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,
+ struct ib_udata *udata);
+int bng_re_destroy_ah(struct ib_ah *ib_ah, u32 flags);
+int bng_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr);
+void bng_re_posted_destroy_ah(struct work_struct *work);
#endif /* __BNG_RE_VERBS_H__ */
diff --git a/include/uapi/rdma/bng_re-abi.h b/include/uapi/rdma/bng_re-abi.h
index 49931b7c1344..36bc902600be 100644
--- a/include/uapi/rdma/bng_re-abi.h
+++ b/include/uapi/rdma/bng_re-abi.h
@@ -75,4 +75,9 @@ struct bng_re_srq_resp {
__aligned_u64 comp_mask;
};
+struct bng_re_ah_resp {
+ __u32 ah_id;
+ __u64 comp_mask;
+} __attribute__((packed));
+
#endif /* __BNG_RE_UVERBS_ABI_H__*/
--
2.43.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 14/15] RDMA/bng_re: Add QP verbs
2026-09-04 10:43 [PATCH 00/15] Add BNG_RE control path verbs Siva Reddy Kallam
` (12 preceding siblings ...)
2026-09-04 10:43 ` [PATCH 13/15] RDMA/bng_re: Add AH verbs Siva Reddy Kallam
@ 2026-09-04 10:43 ` Siva Reddy Kallam
2026-09-04 10:43 ` [PATCH 15/15] RDMA/bng_re: Register with ib-core Siva Reddy Kallam
14 siblings, 0 replies; 17+ messages in thread
From: Siva Reddy Kallam @ 2026-09-04 10:43 UTC (permalink / raw)
To: leonro, jgg, davem, edumazet, kuba, pabeni, andrew+netdev, horms
Cc: netdev, linux-kernel, linux-rdma, Usman S. Ansari,
Siva Reddy Kallam
From: "Usman S. Ansari" <usman.ansari@broadcom.com>
This patch adds below verbs.
-bng_re_query_qp
-bng_re_create_qp
-bng_re_modify_qp
-bng_re_destroy_qp
Signed-off-by: Usman S. Ansari <usman.ansari@broadcom.com>
Signed-off-by: Siva Reddy Kallam <siva.kallam@broadcom.com>
---
drivers/infiniband/hw/bng_re/bng_dev.c | 5 +
drivers/infiniband/hw/bng_re/bng_fw.c | 5 -
drivers/infiniband/hw/bng_re/bng_fw.h | 7 +
drivers/infiniband/hw/bng_re/bng_re.h | 43 +
.../infiniband/hw/bng_re/bng_re_mpc_roce.c | 2 -
drivers/infiniband/hw/bng_re/bng_res.c | 45 +-
drivers/infiniband/hw/bng_re/bng_res.h | 75 +
drivers/infiniband/hw/bng_re/bng_roce_hsi.h | 158 ++
drivers/infiniband/hw/bng_re/bng_sp.c | 870 +++++++++
drivers/infiniband/hw/bng_re/bng_sp.h | 172 +-
drivers/infiniband/hw/bng_re/bng_verbs.c | 1582 +++++++++++++++++
drivers/infiniband/hw/bng_re/bng_verbs.h | 198 +++
include/uapi/rdma/bng_re-abi.h | 35 +
13 files changed, 3188 insertions(+), 9 deletions(-)
diff --git a/drivers/infiniband/hw/bng_re/bng_dev.c b/drivers/infiniband/hw/bng_re/bng_dev.c
index 3ebdd777eca3..6e9df737c751 100644
--- a/drivers/infiniband/hw/bng_re/bng_dev.c
+++ b/drivers/infiniband/hw/bng_re/bng_dev.c
@@ -62,12 +62,17 @@ static const struct ib_device_ops bng_re_dev_ops = {
.modify_srq = bng_re_modify_srq,
.query_srq = bng_re_query_srq,
.destroy_srq = bng_re_destroy_srq,
+ .query_qp = bng_re_query_qp,
+ .create_qp = bng_re_create_qp,
+ .modify_qp = bng_re_modify_qp,
+ .destroy_qp = bng_re_destroy_qp,
.alloc_hw_port_stats = bng_re_alloc_hw_port_stats,
.get_hw_stats = bng_re_ib_get_hw_stats,
.create_ah = bng_re_create_ah,
.create_user_ah = bng_re_create_ah,
.destroy_ah = bng_re_destroy_ah,
.query_ah = bng_re_query_ah,
+ INIT_RDMA_OBJ_SIZE(ib_qp, bng_re_qp, ib_qp),
INIT_RDMA_OBJ_SIZE(ib_ah, bng_re_ah, ib_ah),
INIT_RDMA_OBJ_SIZE(ib_srq, bng_re_srq, ib_srq),
INIT_RDMA_OBJ_SIZE(ib_cq, bng_re_cq, ib_cq),
diff --git a/drivers/infiniband/hw/bng_re/bng_fw.c b/drivers/infiniband/hw/bng_re/bng_fw.c
index a69221368ba8..a6b43866666c 100644
--- a/drivers/infiniband/hw/bng_re/bng_fw.c
+++ b/drivers/infiniband/hw/bng_re/bng_fw.c
@@ -732,11 +732,6 @@ static inline bool _is_hw_retx_supported(u16 dev_cap_flags)
}
#define BNG_RE_HW_RETX(a) _is_hw_retx_supported((a))
-static inline bool _is_optimize_modify_qp_supported(u16 dev_cap_ext_flags2)
-{
- return dev_cap_ext_flags2 &
- CREQ_QUERY_FUNC_RESP_SB_OPTIMIZE_MODIFY_QP_SUPPORTED;
-}
int bng_re_init_rcfw(struct bng_re_rcfw *rcfw,
struct bng_re_stats *stats_ctx)
diff --git a/drivers/infiniband/hw/bng_re/bng_fw.h b/drivers/infiniband/hw/bng_re/bng_fw.h
index 011dc18592d0..c2d9fc94c674 100644
--- a/drivers/infiniband/hw/bng_re/bng_fw.h
+++ b/drivers/infiniband/hw/bng_re/bng_fw.h
@@ -125,6 +125,11 @@ struct bng_re_rcfw_sbuf {
u32 size;
};
+struct bng_re_qp_node {
+ u32 qp_id;
+ void *qp_handle;
+};
+
/* RoCE FW Communication Channels */
struct bng_re_rcfw {
struct pci_dev *pdev;
@@ -140,6 +145,8 @@ struct bng_re_rcfw {
atomic_t rcfw_intr_enabled;
u64 oos_prev;
u32 init_oos_stats;
+ int qp_tbl_size;
+ struct bng_re_qp_node *qp_tbl;
};
struct bng_re_cmdqmsg {
diff --git a/drivers/infiniband/hw/bng_re/bng_re.h b/drivers/infiniband/hw/bng_re/bng_re.h
index 217383a2b252..ef81e09f262b 100644
--- a/drivers/infiniband/hw/bng_re/bng_re.h
+++ b/drivers/infiniband/hw/bng_re/bng_re.h
@@ -244,6 +244,7 @@ struct bng_re_nq {
u32 load;
struct workqueue_struct *cqn_wq;
+ int (*cqn_handler)(struct bng_re_nq *nq, void *handle);
};
struct bng_re_nq_record {
@@ -268,6 +269,37 @@ struct bng_re_ring_attr {
u8 mode;
};
+struct bng_re_gsi_context {
+ struct bng_re_qp *gsi_qp;
+ u8 gsi_qp_mode;
+};
+
+struct bng_re_dscp2pri {
+ u8 dscp;
+ u8 mask;
+ u8 pri;
+};
+
+struct bng_re_tc_rec {
+ u8 cos_id_roce;
+ u8 tc_roce;
+ u8 cos_id_cnp;
+ u8 tc_cnp;
+ u8 tc_def;
+ u8 cos_id_def;
+ u8 max_tc;
+ u8 roce_prio;
+ u8 cnp_prio;
+ u8 roce_dscp;
+ u8 cnp_dscp;
+ u8 prio_valid;
+ u8 dscp_valid;
+ bool ecn_enabled;
+ bool serv_type_enabled;
+ u64 cnp_dscp_bv;
+ u64 roce_dscp_bv;
+};
+
struct bng_re_dev {
struct ib_device ibdev;
unsigned long flags;
@@ -303,6 +335,17 @@ struct bng_re_dev {
struct bng_re_ctx ctx;
struct workqueue_struct *dest_ah_wq;
refcount_t pos_destah_cnt;
+ struct bng_re_gsi_context gsi_ctx;
+ struct mutex qp_lock; /* protect qp list */
+ struct list_head qp_list;
+ u8 d2p_count;
+ struct bng_re_dscp2pri *d2p;
+ u8 p2cos[IEEE_8021QAZ_MAX_TCS];
+ u8 lossless_q_count;
+ u8 *lossless_qid;
+ union ib_gid ugid;
+ u32 min_tx_depth;
+ struct bng_re_tc_rec tc_rec[2];
};
#define to_bng_re_dev(ptr, member) \
diff --git a/drivers/infiniband/hw/bng_re/bng_re_mpc_roce.c b/drivers/infiniband/hw/bng_re/bng_re_mpc_roce.c
index 3ecf5f34180a..91cfdcdc2ff4 100644
--- a/drivers/infiniband/hw/bng_re/bng_re_mpc_roce.c
+++ b/drivers/infiniband/hw/bng_re/bng_re_mpc_roce.c
@@ -63,8 +63,6 @@ void bng_re_mpc_handle_event_cmpl(struct bng_re_dev *rdev,
dev_warn(rdev_to_dev(rdev),
"unexpected event mpc cmpl: xid=%d event=%d data=%d\n",
xid, event, event_data);
- /* process even if event / event_data isn't as expected, as we don't use them rn */
- /* bng_re_qp_xid_pending_process_unsolicited_cmpl(rdev, xid); */
}
/**
diff --git a/drivers/infiniband/hw/bng_re/bng_res.c b/drivers/infiniband/hw/bng_re/bng_res.c
index 024d49f68b09..b40c13cbc3a1 100644
--- a/drivers/infiniband/hw/bng_re/bng_res.c
+++ b/drivers/infiniband/hw/bng_re/bng_res.c
@@ -10,6 +10,11 @@
#include "bng_roce_hsi.h"
#include "bng_sp.h"
+bool bng_re_init_fw_state_rtr_enabled(struct bng_re_chip_ctx *chip_ctx)
+{
+ return chip_ctx->modes.init_fw_state_rtr;
+}
+
/* Stats */
void bng_re_free_stats_ctx_mem(struct pci_dev *pdev,
struct bng_re_stats *stats)
@@ -395,8 +400,40 @@ static int bng_res_alloc_pd_tbl(struct bng_re_res *res,
return 0;
}
+static int bng_res_alloc_reftbl(struct bng_re_reftbl *tbl, u32 max)
+{
+ tbl->max = max;
+ tbl->rec = vzalloc(sizeof(*tbl->rec) * max);
+ if (!tbl->rec)
+ return -ENOMEM;
+ spin_lock_init(&tbl->lock);
+ return 0;
+}
+
+static void bng_res_free_reftbls(struct bng_re_res *res)
+{
+ struct bng_re_reftbl *tbl;
+
+ tbl = &res->reftbl.qpref;
+ vfree(tbl->rec);
+}
+
+static int bng_res_alloc_reftbls(struct bng_re_res *res, struct bng_re_dev_attr *dattr)
+{
+ struct bng_re_reftbl *tbl;
+ int rc;
+
+ tbl = &res->reftbl.qpref;
+ rc = bng_res_alloc_reftbl(tbl, BNG_RE_MAX_QPC_COUNT);
+ if (rc)
+ return rc;
+
+ return 0;
+}
+
void bng_res_free_tbls(struct bng_re_res *res)
{
+ bng_res_free_reftbls(res);
bng_res_free_pd_tbl(&res->pd_tbl);
bng_res_free_dpi_tbl(&res->dpi_tbl);
bng_sp_free_sgid_tbl(&res->sgid_tbl);
@@ -406,10 +443,14 @@ int bng_res_alloc_init_tbls(struct bng_re_res *res)
{
int rc;
- rc = bng_res_alloc_pd_tbl(res, res->dattr);
+ rc = bng_res_alloc_reftbls(res, res->dattr);
if (rc)
return rc;
+ rc = bng_res_alloc_pd_tbl(res, res->dattr);
+ if (rc)
+ goto free_reftbl;
+
rc = bng_res_alloc_dpi_tbl(res, res->dattr);
if (rc)
goto free_pd_tbl;
@@ -425,6 +466,8 @@ int bng_res_alloc_init_tbls(struct bng_re_res *res)
bng_res_free_dpi_tbl(&res->dpi_tbl);
free_pd_tbl:
bng_res_free_pd_tbl(&res->pd_tbl);
+free_reftbl:
+ bng_res_free_reftbls(res);
return rc;
}
diff --git a/drivers/infiniband/hw/bng_re/bng_res.h b/drivers/infiniband/hw/bng_re/bng_res.h
index 5a6db1786886..d5a6db80de59 100644
--- a/drivers/infiniband/hw/bng_re/bng_res.h
+++ b/drivers/infiniband/hw/bng_re/bng_res.h
@@ -4,6 +4,7 @@
#ifndef __BNG_RES_H__
#define __BNG_RES_H__
+#include <linux/bnge/hsi.h>
#include "bng_roce_hsi.h"
#include "xid_allocator.h"
#include <linux/bnge/hsi.h>
@@ -39,6 +40,7 @@
#define RCFW_DBR_PCI_BAR_REGION 2
#define BNG_RE_FR_PMR 0x80000000
+#define BNG_RE_MAX_QPC_COUNT (64 * 1024)
enum bng_re_toggle_modes {
BNG_RE_CQ_TOGGLE_BIT = 0x1,
@@ -102,6 +104,9 @@ struct bng_re_drv_modes {
u8 roce_mirror;
u8 dbr_primary_pf;
bool st_tag_supported;
+ u8 driver_alloc_xid_supported;
+ u8 init_fw_state_rtr;
+ u8 te_bypass;
};
struct bng_re_chip_ctx {
@@ -212,9 +217,30 @@ struct bng_re_pd_tbl {
u32 max;
};
+/* Reference record structure with MPC support */
+struct bng_re_refrec {
+ void *handle;
+ u32 xid;
+ u8 initial_mpc_sent; /* indicates xid has been sent at least 1x to fw */
+};
+
+/* Reference table structure */
+struct bng_re_reftbl {
+ struct bng_re_refrec *rec;
+ u32 max;
+ spinlock_t lock; /* reftbl lock */
+};
+
+/* Collection of reference tables */
+struct bng_re_reftbls {
+ struct bng_re_reftbl qpref;
+};
+
struct bng_re_res {
struct pci_dev *pdev;
struct bng_re_chip_ctx *cctx;
+ struct net_device *netdev;
+ struct bng_re_dev *rdev;
struct bng_re_dev_attr *dattr;
struct bng_re_dpi_tbl dpi_tbl;
/* Serialize access to DPI table */
@@ -222,6 +248,7 @@ struct bng_re_res {
struct xid_manager *qp_xids;
struct xid_manager *ah_xids;
struct bng_re_sgid_tbl sgid_tbl;
+ struct bng_re_reftbls reftbl;
bool prio;
struct bng_re_pd_tbl pd_tbl;
/* Serialize access to PD table */
@@ -238,6 +265,28 @@ struct bng_re_ctx {
struct bng_re_stats stats;
};
+struct bng_re_q {
+ struct bng_re_hwq hwq;
+ struct bng_re_swq *swq;
+ struct bng_re_db_info dbinfo;
+ struct bng_re_sg_info sg_info;
+ u32 max_wqe;
+ u32 max_sw_wqe;
+ u16 wqe_size;
+ u16 q_full_delta;
+ u16 max_sge;
+ u32 psn;
+ bool condition;
+ bool single;
+ bool send_phantom;
+ u32 phantom_wqe_cnt;
+ u32 phantom_cqe_cnt;
+ u32 next_cq_cons;
+ bool flushed;
+ u32 swq_start;
+ u32 swq_last;
+};
+
#define to_bng_re_res(ptr, member) container_of(ptr, struct bng_re_res, member)
static inline void *bng_re_get_qe(struct bng_re_hwq *hwq,
@@ -414,6 +463,31 @@ static inline int bng_ext_stats_supported(struct bng_re_chip_ctx *ctx,
return (_is_ext_stats_supported(flags) && ((virtfn) || (!virtfn)));
}
+static inline bool _is_optimize_modify_qp_supported(u16 dev_cap_ext_flags2)
+{
+ return dev_cap_ext_flags2 &
+ CREQ_QUERY_FUNC_RESP_SB_OPTIMIZE_MODIFY_QP_SUPPORTED;
+}
+
+static inline bool _is_min_rnr_in_rtr_rts_mandatory(u16 dev_cap_ext_flags2)
+{
+ return !!(dev_cap_ext_flags2 &
+ CREQ_QUERY_FUNC_RESP_SB_MIN_RNR_RTR_RTS_OPT_SUPPORTED);
+}
+
+#define GET_REFTBL_INDEX(id, tbl) ((id) % (((tbl)->max) - 1))
+static inline u32 map_qp_id_to_reftbl_indx(u32 qid, struct bng_re_reftbl *tbl)
+{
+ return (qid == 1) ? tbl->max : GET_REFTBL_INDEX(qid, tbl);
+}
+
+static inline bool _is_change_udp_src_port_wqe_supported(u16 flags)
+{
+ return !!(flags &
+ CREQ_QUERY_FUNC_RESP_SB_CHANGE_UDP_SRC_PORT_WQE_SUPPORTED);
+}
+
+#define BNG_RE_UDP_SP_WQE(a) _is_change_udp_src_port_wqe_supported((a))
void bng_re_free_hwq(struct bng_re_res *res,
struct bng_re_hwq *hwq);
@@ -440,4 +514,5 @@ int bng_re_dealloc_dpi(struct bng_re_res *res,
void bng_re_alloc_kernel_dpi(struct bng_re_res *res,
struct bng_re_dpi *dpi);
void bng_re_dealloc_kernel_dpi(struct bng_re_dpi *dpi);
+bool bng_re_init_fw_state_rtr_enabled(struct bng_re_chip_ctx *chip_ctx);
#endif
diff --git a/drivers/infiniband/hw/bng_re/bng_roce_hsi.h b/drivers/infiniband/hw/bng_re/bng_roce_hsi.h
index 3001b9d4d056..efef9de95361 100644
--- a/drivers/infiniband/hw/bng_re/bng_roce_hsi.h
+++ b/drivers/infiniband/hw/bng_re/bng_roce_hsi.h
@@ -6606,4 +6606,162 @@ struct mpc_ah_modify_cmpl {
__le32 reserved_3;
};
+/* mpc_qp_modify_cmd (size:1792b/224B) */
+struct mpc_qp_modify_cmd {
+ u8 req_type;
+ u8 req_subtype;
+ __le16 cookie;
+ __le16 target_id;
+ u8 resp_size;
+ u8 reserved;
+ __le64 resp_addr;
+ __le32 qp_cid;
+ u8 qp_modify_flags;
+ #define MPC_QP_MODIFY_CMD_QP_MODIFY_DATA_QP_MODIFY_FLAGS_QP_MODIFY_FLAGS_MASK \
+ 0xffUL
+ #define MPC_QP_MODIFY_CMD_QP_MODIFY_DATA_QP_MODIFY_FLAGS_QP_MODIFY_FLAGS_SFT \
+ 0
+ #define MPC_QP_MODIFY_CMD_QP_MODIFY_DATA_QP_MODIFY_FLAGS_QP_MODIFY_FLAGS_QP_FIRST_MODIFY \
+ 0x1UL
+ #define MPC_QP_MODIFY_CMD_QP_MODIFY_DATA_QP_MODIFY_FLAGS_QP_MODIFY_FLAGS_QP_FREE \
+ 0x2UL
+ #define MPC_QP_MODIFY_CMD_QP_MODIFY_DATA_QP_MODIFY_FLAGS_QP_MODIFY_FLAGS_XID_FIRST_USE \
+ 0x4UL
+ #define MPC_QP_MODIFY_CMD_QP_MODIFY_DATA_QP_MODIFY_FLAGS_QP_MODIFY_FLAGS_LAST \
+ MPC_QP_MODIFY_CMD_QP_MODIFY_DATA_QP_MODIFY_FLAGS_QP_MODIFY_FLAGS_XID_FIRST_USE
+ u8 qp_type;
+ __le16 schq_id;
+ __le32 qp_flags;
+ u8 sq_pg_size_sq_lvl;
+ u8 rq_pg_size_rq_lvl;
+ __le16 sq_fwo_sq_sge;
+ __le16 rq_fwo_rq_sge;
+ __le16 sq_max_num_wqes;
+ __le16 flags;
+ u8 type;
+ u8 unused_2;
+ __le32 modify_mask;
+ u8 network_type_en_sqd_async_notify_new_state;
+ u8 access;
+ __le16 pkey;
+ __le32 qkey;
+ __le32 flow_label;
+ __le32 dgid[4];
+ __le16 sgid_index;
+ u8 hop_limit;
+ u8 traffic_class;
+ u8 tos_dscp_tos_ecn;
+ u8 path_mtu_pingpong_push_enable;
+ u8 timeout;
+ u8 retry_cnt;
+ u8 rnr_retry;
+ u8 min_rnr_timer;
+ u8 max_dest_rd_atomic;
+ u8 max_rd_atomic;
+ __le32 rq_psn;
+ __le32 sq_psn;
+ __le32 sq_size;
+ __le32 rq_size;
+ __le16 sq_sge;
+ __le16 rq_sge;
+ __le32 max_inline_data;
+ __le32 dest_qp_id;
+ __le32 pingpong_push_dpi;
+ u8 unused_3[4];
+ __le16 enable_cc;
+ __le16 dest_mac[3];
+ __le16 src_mac[3];
+ __le16 vlan_pcp_vlan_dei_vlan_id;
+ __le32 ext_modify_mask;
+ __le32 ext_stats_ctx_id;
+ __le64 qp_handle;
+ __le32 dpi;
+ __le32 scq_cid;
+ __le32 rcq_cid;
+ __le32 srq_cid;
+ __le32 pd_id;
+ __le32 request_xid;
+ __le64 sq_pbl;
+ __le64 rq_pbl;
+ __le32 msn_iqp;
+ __le32 irrq_iqp;
+ __le32 orrq_iqp;
+ __le32 msn_size;
+ __le32 irrq_size;
+ __le32 orrq_size;
+ __le16 steering_tag;
+ __le16 eroce;
+ __le16 rq_prod_idx;
+ u8 reserved_1[2];
+};
+
+struct qp_modify_data {
+ __le32 qp_cid;
+ u8 qp_modify_flags;
+ u8 qp_type;
+ __le16 schq_id;
+ __le32 qp_flags;
+ u8 sq_pg_size_sq_lvl;
+ u8 rq_pg_size_rq_lvl;
+ __le16 sq_fwo_sq_sge;
+ __le16 rq_fwo_rq_sge;
+ __le16 sq_max_num_wqes;
+ __le16 flags;
+ u8 type;
+ u8 unused_2;
+ __le32 modify_mask;
+ u8 network_type_en_sqd_async_notify_new_state;
+ u8 access;
+ __le16 pkey;
+ __le32 qkey;
+ __le32 flow_label;
+ __le32 dgid[4];
+ __le16 sgid_index;
+ u8 hop_limit;
+ u8 traffic_class;
+ u8 tos_dscp_tos_ecn;
+ u8 path_mtu_pingpong_push_enable;
+ u8 timeout;
+ u8 retry_cnt;
+ u8 rnr_retry;
+ u8 min_rnr_timer;
+ u8 max_dest_rd_atomic;
+ u8 max_rd_atomic;
+ __le32 rq_psn;
+ __le32 sq_psn;
+ __le32 sq_size;
+ __le32 rq_size;
+ __le16 sq_sge;
+ __le16 rq_sge;
+ __le32 max_inline_data;
+ __le32 dest_qp_id;
+ __le32 pingpong_push_dpi;
+ u8 unused_3[4];
+ __le16 enable_cc;
+ __le16 dest_mac[3];
+ __le16 src_mac[3];
+ __le16 vlan_pcp_vlan_dei_vlan_id;
+ __le32 ext_modify_mask;
+ __le32 ext_stats_ctx_id;
+ __le64 qp_handle;
+ __le32 dpi;
+ __le32 scq_cid;
+ __le32 rcq_cid;
+ __le32 srq_cid;
+ __le32 pd_id;
+ __le32 request_xid;
+ __le64 sq_pbl;
+ __le64 rq_pbl;
+ __le32 msn_iqp;
+ __le32 irrq_iqp;
+ __le32 orrq_iqp;
+ __le32 msn_size;
+ __le32 irrq_size;
+ __le32 orrq_size;
+ __le16 steering_tag;
+ __le16 eroce;
+ __le16 rq_prod_idx;
+ u8 reserved_1[2];
+};
+
#endif /* _BNG_RE_HSI_H_ */
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.c b/drivers/infiniband/hw/bng_re/bng_sp.c
index b44af599e675..7e93a1be9bad 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.c
+++ b/drivers/infiniband/hw/bng_re/bng_sp.c
@@ -10,6 +10,11 @@
#include "bng_re.h"
#include "bng_re_mpc_roce.h"
#include "bng_fp.h"
+#include "bng_verbs.h"
+
+#define CQE_CMP_VALID(hdr, pass) \
+ (!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \
+ !((pass) & BNG_RE_FLAG_EPOCH_CONS_MASK))
const struct bng_re_gid bng_re_gid_zero = {{0,}};
@@ -1224,3 +1229,868 @@ int bng_sp_fill_and_send_mpc_ah_modify(struct bng_re_res *res,
return rc;
}
+
+void bng_re_flush_cqn_wq(struct bng_sp_qp *qp)
+{
+ /*
+ * Partial create (e.g. attr without send_cq/recv_cq) or teardown ordering
+ * can leave scq/rcq or NQ workqueues unset; flush must not dereference them.
+ */
+ if (qp->scq && qp->scq->nq && qp->scq->nq->cqn_wq)
+ flush_workqueue(qp->scq->nq->cqn_wq);
+ if (qp->rcq && qp->scq != qp->rcq && qp->rcq->nq && qp->rcq->nq->cqn_wq)
+ flush_workqueue(qp->rcq->nq->cqn_wq);
+}
+
+static void bng_re_free_qp_hdr_buf(struct bng_re_res *res,
+ struct bng_sp_qp *qp)
+{
+ struct bng_re_q *rq = &qp->rq;
+ struct bng_re_q *sq = &qp->sq;
+
+ if (qp->rq_hdr_buf)
+ dma_free_coherent(&res->pdev->dev,
+ rq->max_wqe * qp->rq_hdr_buf_size,
+ qp->rq_hdr_buf, qp->rq_hdr_buf_map);
+ if (qp->sq_hdr_buf)
+ dma_free_coherent(&res->pdev->dev,
+ sq->max_wqe * qp->sq_hdr_buf_size,
+ qp->sq_hdr_buf, qp->sq_hdr_buf_map);
+ qp->rq_hdr_buf = NULL;
+ qp->sq_hdr_buf = NULL;
+ qp->rq_hdr_buf_map = 0;
+ qp->sq_hdr_buf_map = 0;
+ qp->sq_hdr_buf_size = 0;
+ qp->rq_hdr_buf_size = 0;
+}
+
+void bng_re_free_qp_res(struct bng_re_res *res,
+ struct bng_sp_qp *qp)
+{
+ bng_re_free_qp_hdr_buf(res, qp);
+ bng_re_free_hwq(res, &qp->sq.hwq);
+ kvfree(qp->sq.swq);
+
+ bng_re_free_hwq(res, &qp->rq.hwq);
+ kvfree(qp->rq.swq);
+
+ if (qp->irrq.max_elements)
+ bng_re_free_hwq(res, &qp->irrq);
+ if (qp->orrq.max_elements)
+ bng_re_free_hwq(res, &qp->orrq);
+}
+
+static void bng_re_acquire_cq_flush_locks(struct bng_sp_qp *qp,
+ unsigned long *flags)
+ __acquires(&qp->scq->flush_lock) __acquires(&qp->rcq->flush_lock)
+{
+ spin_lock_irqsave(&qp->scq->flush_lock, *flags);
+ if (qp->scq == qp->rcq)
+ __acquire(&qp->rcq->flush_lock);
+ else
+ spin_lock(&qp->rcq->flush_lock);
+}
+
+static void bng_re_cancel_phantom_processing(struct bng_sp_qp *qp)
+{
+ qp->sq.condition = false;
+ qp->sq.send_phantom = false;
+ qp->sq.single = false;
+}
+
+static void __bng_re_add_flush_qp(struct bng_sp_qp *qp)
+{
+ struct bng_sp_cq *scq, *rcq;
+
+ scq = qp->scq;
+ rcq = qp->rcq;
+
+ if (!qp->sq.flushed) {
+ dev_dbg(&scq->hwq.pdev->dev,
+ "SP: Adding to SQ Flush list = %p\n", qp);
+ bng_re_cancel_phantom_processing(qp);
+ list_add_tail(&qp->sq_flush, &scq->sqf_head);
+ qp->sq.flushed = true;
+ }
+ if (!qp->srq) {
+ if (!qp->rq.flushed) {
+ dev_dbg(&rcq->hwq.pdev->dev,
+ "SP: Adding to RQ Flush list = %p\n", qp);
+ list_add_tail(&qp->rq_flush, &rcq->rqf_head);
+ qp->rq.flushed = true;
+ }
+ }
+}
+
+static void bng_re_release_cq_flush_locks(struct bng_sp_qp *qp,
+ unsigned long *flags)
+ __releases(&qp->scq->flush_lock) __releases(&qp->rcq->flush_lock)
+{
+ if (qp->scq == qp->rcq)
+ __release(&qp->rcq->flush_lock);
+ else
+ spin_unlock(&qp->rcq->flush_lock);
+ spin_unlock_irqrestore(&qp->scq->flush_lock, *flags);
+}
+
+void bng_re_add_flush_qp(struct bng_sp_qp *qp)
+{
+ unsigned long flags;
+
+ bng_re_acquire_cq_flush_locks(qp, &flags);
+ __bng_re_add_flush_qp(qp);
+ bng_re_release_cq_flush_locks(qp, &flags);
+}
+
+static void __bng_re_del_flush_qp(struct bng_sp_qp *qp)
+{
+ if (qp->sq.flushed) {
+ qp->sq.flushed = false;
+ list_del(&qp->sq_flush);
+ }
+ if (!qp->srq) {
+ if (qp->rq.flushed) {
+ qp->rq.flushed = false;
+ list_del(&qp->rq_flush);
+ }
+ }
+}
+
+static void __clean_cq(struct bng_sp_cq *cq, u64 qp)
+{
+ struct bng_re_hwq *cq_hwq = &cq->hwq;
+ u32 peek_flags, peek_cons;
+ struct cq_base *hw_cqe;
+ int i;
+
+ peek_flags = cq->dbinfo.flags;
+ peek_cons = cq_hwq->cons;
+ for (i = 0; i < cq_hwq->max_elements; i++) {
+ hw_cqe = bng_re_get_qe(cq_hwq, peek_cons, NULL);
+ if (!CQE_CMP_VALID(hw_cqe, peek_flags))
+ continue;
+ /*
+ * The valid test of the entry must be done first before
+ * reading any further.
+ */
+ dma_rmb();
+ switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
+ case CQ_BASE_CQE_TYPE_REQ:
+ case CQ_BASE_CQE_TYPE_TERMINAL:
+ {
+ struct cq_req *cqe = (struct cq_req *)hw_cqe;
+
+ if (qp == le64_to_cpu(cqe->qp_handle))
+ cqe->qp_handle = 0;
+ break;
+ }
+ case CQ_BASE_CQE_TYPE_RES_RC:
+ case CQ_BASE_CQE_TYPE_RES_UD:
+ case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
+ {
+ struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe;
+
+ if (qp == le64_to_cpu(cqe->qp_handle))
+ cqe->qp_handle = 0;
+ break;
+ }
+ default:
+ break;
+ }
+ bng_re_hwq_incr_cons(cq_hwq->max_elements, &peek_cons,
+ 1, &peek_flags);
+ }
+}
+
+void bng_re_clean_qp(struct bng_sp_qp *qp)
+{
+ unsigned long flags;
+
+ bng_re_acquire_cq_flush_locks(qp, &flags);
+ __clean_cq(qp->scq, (u64)(unsigned long)qp);
+ qp->sq.hwq.prod = 0;
+ qp->sq.hwq.cons = 0;
+ __clean_cq(qp->rcq, (u64)(unsigned long)qp);
+ qp->rq.hwq.prod = 0;
+ qp->rq.hwq.cons = 0;
+
+ __bng_re_del_flush_qp(qp);
+ bng_re_release_cq_flush_locks(qp, &flags);
+}
+
+int bng_re_alloc_init_swq(struct bng_re_q *que)
+{
+ int indx;
+
+ que->swq = kvcalloc(que->max_sw_wqe, sizeof(*que->swq), GFP_KERNEL);
+ if (!que->swq)
+ return -ENOMEM;
+
+ que->swq_start = 0;
+ que->swq_last = que->max_sw_wqe - 1;
+ for (indx = 0; indx < que->max_sw_wqe; indx++)
+ que->swq[indx].next_idx = indx + 1;
+ que->swq[que->swq_last].next_idx = 0;
+ que->swq_last = 0;
+
+ return 0;
+}
+
+static bool bng_sp_get_initial_created_flag(struct bng_re_res *res,
+ struct bng_sp_qp *qp)
+{
+ struct bng_re_reftbl *tbl;
+ unsigned long flag;
+ bool val = false;
+ u32 qp_idx;
+
+ tbl = &res->reftbl.qpref;
+ qp_idx = map_qp_id_to_reftbl_indx(qp->id, tbl);
+ spin_lock_irqsave(&tbl->lock, flag);
+ /* Make sure we have the right table index */
+ if (tbl->rec[qp_idx].xid == qp->id &&
+ tbl->rec[qp_idx].handle == qp)
+ val = tbl->rec[qp_idx].initial_mpc_sent;
+ spin_unlock_irqrestore(&tbl->lock, flag);
+ return val;
+}
+
+static bool is_optimized_state_transition(struct bng_sp_qp *qp)
+{
+ if ((qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_INIT &&
+ qp->state == CMDQ_MODIFY_QP_NEW_STATE_RTR) ||
+ (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_RTR &&
+ qp->state == CMDQ_MODIFY_QP_NEW_STATE_RTS))
+ return true;
+
+ return false;
+}
+
+static void bng_set_mandatory_attributes(struct bng_re_res *res,
+ struct bng_sp_qp *qp,
+ struct cmdq_modify_qp *req)
+{
+ u32 mandatory_flags = 0;
+
+ if (qp->type == CMDQ_MODIFY_QP_QP_TYPE_RC)
+ mandatory_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
+
+ if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_INIT &&
+ qp->state == CMDQ_MODIFY_QP_NEW_STATE_RTR) {
+ if (qp->type == CMDQ_MODIFY_QP_QP_TYPE_RC && qp->srq)
+ req->flags = CMDQ_MODIFY_QP_FLAGS_SRQ_USED;
+ mandatory_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
+ }
+
+ if (_is_min_rnr_in_rtr_rts_mandatory(res->dattr->dev_cap_ext_flags2) &&
+ (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_RTR &&
+ qp->state == CMDQ_MODIFY_QP_NEW_STATE_RTS)) {
+ if (qp->type == CMDQ_MODIFY_QP_QP_TYPE_RC)
+ mandatory_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
+ }
+
+ if (qp->type == CMDQ_MODIFY_QP_QP_TYPE_UD || qp->type == CMDQ_MODIFY_QP_QP_TYPE_GSI)
+ mandatory_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
+
+ qp->modify_flags |= mandatory_flags;
+ req->qp_type = qp->type;
+}
+
+static int bng_sp_get_iqm_xrrq_size(bool xrrq, struct bng_re_res *res, struct bng_re_qp *qp)
+{
+ struct bng_sp_qp *sp = &qp->sp_qp;
+ int xrrq_overflow = 0;
+ int dattr_xrrq_size;
+ int iqm_xrrq_size;
+ int qp_xrrq_size;
+ int xrrq_size;
+
+ iqm_xrrq_size = xrrq ? qp->iqm_res.irrq_size : qp->iqm_res.orrq_size;
+ qp_xrrq_size = xrrq ? sp->max_dest_rd_atomic : sp->max_rd_atomic;
+ qp_xrrq_size = max_t(int, 4, qp_xrrq_size);
+ dattr_xrrq_size = xrrq ? res->dattr->max_qp_init_rd_atom : res->dattr->max_qp_rd_atom;
+ /* To avoid irrq overflow, irrq needs to be greater than orrq */
+ xrrq_overflow = xrrq ? BNG_RE_IQM_IRRQ_OVERFLOW_COMP : 0;
+ qp_xrrq_size = qp_xrrq_size + xrrq_overflow;
+
+ if (!iqm_xrrq_size) {
+ /* Allocate IQM resource for both orrq and irrq (applicable only for PF) */
+ /* Will be used once IQM resource allocation is moved to INIT to RTR */
+ }
+
+ if (xrrq)
+ iqm_xrrq_size = IRRQ_CACHELINE_TO_ENTRIES(iqm_xrrq_size);
+ else
+ iqm_xrrq_size = ORRQ_CACHELINE_TO_ENTRIES(iqm_xrrq_size);
+
+ /* irrq/orrq size will be minimum of device attribute & per qp rd_atomic size */
+ xrrq_size = min_t(int, iqm_xrrq_size, qp_xrrq_size);
+
+ return xrrq_size;
+}
+#define QP_FIRST_MODIFY \
+ MPC_QP_MODIFY_CMD_QP_MODIFY_DATA_QP_MODIFY_FLAGS_QP_MODIFY_FLAGS_QP_FIRST_MODIFY
+#define QP_MODIFY_XID_FIRST_USE \
+ MPC_QP_MODIFY_CMD_QP_MODIFY_DATA_QP_MODIFY_FLAGS_QP_MODIFY_FLAGS_XID_FIRST_USE
+static int bng_sp_fill_mpc_qp_create_parms(struct bng_re_res *res,
+ struct qp_modify_data *d,
+ struct bng_re_qp *qp)
+{
+ struct bng_sp_qp *sp = &qp->sp_qp;
+ struct bng_re_q *sq = &sp->sq;
+ struct bng_re_q *rq = &sp->rq;
+ struct bng_re_pbl *pbl;
+ u32 qp_flags = 0;
+ u8 pg_sz_lvl = 0;
+ u16 nsge;
+
+ d->qp_cid = cpu_to_le32(sp->id);
+ if (!test_bit(BNG_SP_QP_FLAG_MPC_INITIAL_CREATE_SENT, &sp->flags)) {
+ u8 qmf = QP_FIRST_MODIFY;
+
+ if (!bng_sp_get_initial_created_flag(res, sp))
+ qmf |= QP_MODIFY_XID_FIRST_USE;
+
+ d->qp_modify_flags |= qmf;
+ }
+
+ if (sp->type != CMDQ_CREATE_QP_TYPE_GSI &&
+ _is_ext_stats_supported(sp->dev_cap_ext_flags)) {
+ d->ext_stats_ctx_id = cpu_to_le32(sp->roce_stat_ext_xid);
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_EXT_STATS_CTX_VALID;
+ }
+
+ d->qp_type = sp->type;
+ d->type = sp->type;
+ d->schq_id = 0;
+ d->dpi = cpu_to_le32(sp->dpi->dpi);
+ d->qp_handle = cpu_to_le64(sp->qp_handle);
+ d->sq_size = cpu_to_le32(sq->max_sw_wqe);
+ d->scq_cid = cpu_to_le32(sp->scq->id);
+ pbl = &sq->hwq.pbl[BNG_PBL_LVL_0];
+ d->sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
+
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
+ if (sp->sig_type)
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED;
+ if (sp->type == CMDQ_CREATE_QP_TYPE_RC)
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RDMA_READ_OR_ATOMICS_USED;
+ if (sp->type == CMDQ_CREATE_QP_TYPE_GSI || sp->type == CMDQ_CREATE_QP_TYPE_UD)
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESPONDER_UD_CQE_WITH_CFA;
+ if (res->dattr &&
+ bng_ext_stats_supported(res->cctx, res->dattr->dev_cap_flags, res->is_vf))
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_EXT_STATS_ENABLED;
+ if (res->cctx->modes.te_bypass)
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_OPTIMIZED_TRANSMIT_ENABLED;
+
+ pg_sz_lvl = (bng_re_base_pg_size(&sq->hwq) << CMDQ_CREATE_QP_SQ_PG_SIZE_SFT);
+ pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) << CMDQ_CREATE_QP_SQ_LVL_SFT;
+ d->sq_pg_size_sq_lvl = pg_sz_lvl;
+ d->sq_fwo_sq_sge =
+ cpu_to_le16(((0 << CMDQ_CREATE_QP_SQ_FWO_SFT) &
+ CMDQ_CREATE_QP_SQ_FWO_MASK) |
+ (sq->max_sge & CMDQ_CREATE_QP_SQ_SGE_MASK));
+
+ d->sq_max_num_wqes = min_t(u16, sq->max_wqe - 1, SQ_MAX_NUM_WQES_DEFAULT);
+
+ if (!sp->srq && rq->max_wqe) {
+ pbl = &rq->hwq.pbl[BNG_PBL_LVL_0];
+ d->rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
+ pg_sz_lvl = (bng_re_base_pg_size(&rq->hwq) <<
+ CMDQ_CREATE_QP_RQ_PG_SIZE_SFT);
+ pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK)
+ << CMDQ_CREATE_QP_RQ_LVL_SFT;
+ d->rq_pg_size_rq_lvl = pg_sz_lvl;
+ nsge = rq->max_sge;
+ d->rq_fwo_rq_sge =
+ cpu_to_le16(((0 << CMDQ_CREATE_QP_RQ_FWO_SFT) &
+ CMDQ_CREATE_QP_RQ_FWO_MASK) |
+ (nsge & CMDQ_CREATE_QP_RQ_SGE_MASK));
+
+ d->rq_size = cpu_to_le32(rq->max_wqe);
+ d->rq_prod_idx = cpu_to_le16((u16)(rq->hwq.prod & 0xffffU));
+ } else {
+ d->rq_size = 0;
+ d->rq_prod_idx = 0;
+ }
+
+ if (sp->srq) {
+ d->qp_flags = cpu_to_le32(le32_to_cpu(d->qp_flags) |
+ CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED);
+ d->srq_cid = cpu_to_le32(sp->srq->id);
+ }
+
+ d->rcq_cid = cpu_to_le32(sp->rcq->id);
+
+ if (sp->type == CMDQ_CREATE_QP_TYPE_RC &&
+ qp->iqm_res.irrq_addr != BNG_RE_IQM_INVALID_IDX) {
+ d->msn_iqp = cpu_to_le32(qp->iqm_res.msn_addr);
+ d->irrq_iqp = cpu_to_le32(qp->iqm_res.irrq_addr);
+ d->orrq_iqp = cpu_to_le32(qp->iqm_res.orrq_addr);
+ d->msn_size = cpu_to_le32(qp->iqm_res.msn_size);
+ d->irrq_size = cpu_to_le32(qp->iqm_res.irrq_size);
+ d->orrq_size = cpu_to_le32(qp->iqm_res.orrq_size);
+ }
+
+ d->qp_flags = cpu_to_le32(qp_flags);
+ d->pd_id = cpu_to_le32(sp->pd->id);
+
+ return 0;
+}
+
+static void __filter_modify_flags(struct bng_sp_qp *qp)
+{
+ switch (qp->cur_qp_state) {
+ case CMDQ_MODIFY_QP_NEW_STATE_RESET:
+ switch (qp->state) {
+ case CMDQ_MODIFY_QP_NEW_STATE_INIT:
+ break;
+ default:
+ break;
+ }
+ break;
+ case CMDQ_MODIFY_QP_NEW_STATE_INIT:
+ switch (qp->state) {
+ case CMDQ_MODIFY_QP_NEW_STATE_RTR:
+ /* INIT->RTR, configure the path_mtu to the default
+ * 2048 if not being requested
+ */
+ if (qp->type != CMDQ_CREATE_QP_TYPE_GSI &&
+ qp->type != CMDQ_CREATE_QP_TYPE_UD &&
+ !(qp->modify_flags &
+ CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) {
+ qp->modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
+ qp->path_mtu = CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
+ }
+
+ /* Bono FW requires the max_dest_rd_atomic to be >= 1 */
+ if (qp->max_dest_rd_atomic < 1)
+ qp->max_dest_rd_atomic = 1;
+
+ /* TODO: Bono FW 0.0.12.0+ does not allow SRC_MAC modification */
+ qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC;
+ /* Bono FW 20.6.5 requires SGID_INDEX to be configured */
+ if (!(qp->modify_flags & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) {
+ qp->modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX;
+ qp->ah.sgid_index = 0;
+ }
+ break;
+ default:
+ break;
+ }
+ break;
+ case CMDQ_MODIFY_QP_NEW_STATE_RTR:
+ switch (qp->state) {
+ case CMDQ_MODIFY_QP_NEW_STATE_RTS:
+ /* Bono FW requires the max_rd_atomic to be >= 1 */
+ if (qp->max_rd_atomic < 1)
+ qp->max_rd_atomic = 1;
+
+ /* TODO: Bono FW 0.0.12.0+ does not allow PKEY_INDEX,
+ * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT,
+ * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN,
+ * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID
+ * modification
+ */
+
+ qp->modify_flags &=
+ ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY |
+ CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
+ CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
+ CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
+ CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
+ CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
+ CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
+ CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU |
+ CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN |
+ CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER |
+ CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC |
+ CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID);
+ break;
+ default:
+ break;
+ }
+ break;
+ case CMDQ_MODIFY_QP_NEW_STATE_RTS:
+ break;
+ case CMDQ_MODIFY_QP_NEW_STATE_SQD:
+ break;
+ case CMDQ_MODIFY_QP_NEW_STATE_SQE:
+ break;
+ case CMDQ_MODIFY_QP_NEW_STATE_ERR:
+ break;
+ default:
+ break;
+ }
+}
+
+static int bng_sp_fill_mpc_qp_modify_parms(struct bng_re_res *res,
+ struct qp_modify_data *d,
+ struct bng_re_qp *qp)
+{
+ struct cmdq_modify_qp temp_req = {};
+ struct bng_sp_qp *sp = &qp->sp_qp;
+ u32 m = (u32)sp->modify_flags;
+ u32 temp32[4];
+
+ /* Check if QP free flag is required */
+ if (test_bit(BNG_SP_QP_FLAG_MPC_QP_FREE, &sp->flags))
+ m |= MPC_QP_MODIFY_CMD_QP_MODIFY_DATA_QP_MODIFY_FLAGS_QP_MODIFY_FLAGS_QP_FREE;
+
+ /* Filter out the qp_attr_mask based on the state->new transition */
+ __filter_modify_flags(sp);
+ m = (u32)sp->modify_flags;
+
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) {
+ /* Set mandatory attributes for INIT -> RTR and RTR -> RTS
+ * transition
+ */
+ if (_is_optimize_modify_qp_supported(res->dattr->dev_cap_ext_flags2) &&
+ is_optimized_state_transition(sp)) {
+ bng_set_mandatory_attributes(res, sp, &temp_req);
+ temp_req.flags =
+ CMDQ_MODIFY_QP_FLAGS_SRQ_USED;
+ d->flags = temp_req.flags;
+ d->qp_type = temp_req.qp_type;
+ }
+ }
+
+ if (bng_re_init_fw_state_rtr_enabled(res->cctx))
+ d->rq_prod_idx = cpu_to_le16(sp->req_buffer_count);
+
+ if (sp->udcc_exclude)
+ d->flags |= CMDQ_MODIFY_QP_FLAGS_EXCLUDE_QP_UDCC;
+
+ d->modify_mask = cpu_to_le32(m);
+ d->qp_cid = cpu_to_le32(sp->id);
+
+ if (d->modify_mask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) {
+ d->network_type_en_sqd_async_notify_new_state =
+ (sp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) |
+ (sp->en_sqd_async_notify
+ ? CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0);
+ }
+ d->network_type_en_sqd_async_notify_new_state |= sp->nw_type;
+
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS)
+ d->access = sp->access;
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY)
+ d->pkey = cpu_to_le16(IB_DEFAULT_PKEY_FULL);
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY)
+ d->qkey = cpu_to_le32(sp->qkey);
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL)
+ d->flow_label = cpu_to_le32(sp->ah.flow_label);
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) {
+ memcpy(temp32, sp->ah.dgid.data, sizeof(struct bng_re_gid));
+ d->dgid[0] = cpu_to_le32(temp32[0]);
+ d->dgid[1] = cpu_to_le32(temp32[1]);
+ d->dgid[2] = cpu_to_le32(temp32[2]);
+ d->dgid[3] = cpu_to_le32(temp32[3]);
+ }
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX) {
+ if (sp->is_roce_mirror_qp)
+ d->sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[sp->ugid_index]);
+ else
+ d->sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[sp->ah.sgid_index]);
+ }
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT)
+ d->hop_limit = sp->ah.hop_limit;
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS)
+ d->traffic_class = sp->ah.traffic_class;
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC)
+ memcpy(d->dest_mac, sp->ah.dmac, 6);
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)
+ d->path_mtu_pingpong_push_enable = sp->path_mtu;
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT)
+ d->timeout = sp->timeout;
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT)
+ d->retry_cnt = sp->retry_cnt;
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY)
+ d->rnr_retry = sp->rnr_retry;
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER)
+ d->min_rnr_timer = sp->min_rnr_timer;
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN)
+ d->rq_psn = cpu_to_le32(sp->rq.psn);
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN)
+ d->sq_psn = cpu_to_le32(sp->sq.psn);
+
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC)
+ d->max_rd_atomic = bng_sp_get_iqm_xrrq_size(0, res, qp);
+
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC)
+ d->max_dest_rd_atomic = bng_sp_get_iqm_xrrq_size(1, res, qp);
+
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE)
+ d->sq_size = cpu_to_le32(bng_re_set_sq_size(&sp->sq));
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE) {
+ if (!sp->srq && sp->rq.max_wqe)
+ d->rq_size = cpu_to_le32(sp->rq.max_wqe);
+ }
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE)
+ d->sq_sge = cpu_to_le16(sp->sq.max_sge);
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE && !sp->srq)
+ d->rq_sge = cpu_to_le16(sp->rq.max_sge);
+
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA)
+ d->max_inline_data = cpu_to_le32(sp->max_inline_data);
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID)
+ d->dest_qp_id = cpu_to_le32(sp->dest_qpn);
+
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC)
+ d->enable_cc = cpu_to_le16(CMDQ_MODIFY_QP_ENABLE_CC);
+
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN)
+ d->tos_dscp_tos_ecn =
+ ((sp->tos_ecn << CMDQ_MODIFY_QP_TOS_ECN_SFT) &
+ CMDQ_MODIFY_QP_TOS_ECN_MASK);
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP)
+ d->tos_dscp_tos_ecn |=
+ ((sp->tos_dscp << CMDQ_MODIFY_QP_TOS_DSCP_SFT) &
+ CMDQ_MODIFY_QP_TOS_DSCP_MASK);
+ if (m & CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID) {
+ d->vlan_pcp_vlan_dei_vlan_id =
+ ((res->sgid_tbl.tbl[sp->ah.sgid_index].vlan_id
+ << CMDQ_MODIFY_QP_VLAN_ID_SFT) &
+ CMDQ_MODIFY_QP_VLAN_ID_MASK);
+ d->vlan_pcp_vlan_dei_vlan_id |=
+ ((sp->ah.sl << CMDQ_MODIFY_QP_VLAN_PCP_SFT) &
+ CMDQ_MODIFY_QP_VLAN_PCP_MASK);
+ d->vlan_pcp_vlan_dei_vlan_id =
+ cpu_to_le16(d->vlan_pcp_vlan_dei_vlan_id);
+ }
+
+ m = sp->ext_modify_flags;
+ d->ext_modify_mask = cpu_to_le32(sp->ext_modify_flags);
+ if (m & CMDQ_MODIFY_QP_EXT_MODIFY_MASK_EXT_STATS_CTX)
+ d->ext_stats_ctx_id = cpu_to_le32(sp->roce_stat_ext_xid);
+
+ return 0;
+}
+
+static int bng_sp_fill_mpc_qp_modify_and_create_parms(struct bng_re_res *res,
+ struct qp_modify_data *d,
+ struct bng_re_qp *qp)
+{
+ if (!test_bit(BNG_SP_QP_FLAG_MPC_INITIAL_CREATE_SENT, &qp->sp_qp.flags) &&
+ !test_bit(BNG_SP_QP_FLAG_MPC_QP_FREE, &qp->sp_qp.flags))
+ bng_sp_fill_mpc_qp_create_parms(res, d, qp);
+
+ bng_sp_fill_mpc_qp_modify_parms(res, d, qp);
+ return 0;
+}
+
+static void bng_sp_set_initial_created_flag(struct bng_re_res *res,
+ struct bng_sp_qp *qp)
+{
+ u32 qp_idx;
+ unsigned long flag;
+ struct bng_re_reftbl *tbl;
+
+ tbl = &res->reftbl.qpref;
+ qp_idx = map_qp_id_to_reftbl_indx(qp->id, tbl);
+ spin_lock_irqsave(&tbl->lock, flag);
+ /* Make sure we have the right table index */
+ if (tbl->rec[qp_idx].xid == qp->id &&
+ tbl->rec[qp_idx].handle == qp &&
+ !tbl->rec[qp_idx].initial_mpc_sent)
+ tbl->rec[qp_idx].initial_mpc_sent = true;
+ spin_unlock_irqrestore(&tbl->lock, flag);
+}
+
+int bng_sp_fill_and_send_mpc_qp_modify(struct bng_re_res *res,
+ struct bng_re_qp **qps,
+ int num_qps,
+ bool *xid_cleanup_handled)
+{
+ struct mpc_qp_modify_cmpl resp;
+ struct mpc_qp_modify_cmd cmd;
+ struct bng_re_qp *qp = *qps;
+ struct qp_modify_data *d;
+ u32 cpu_modify_mask;
+ int rc = 0;
+
+ if (xid_cleanup_handled)
+ *xid_cleanup_handled = false;
+
+ if (num_qps != 1)
+ return -EINVAL;
+
+ if (bng_re_init_fw_state_rtr_enabled(res->cctx)) {
+ if (!test_bit(BNG_SP_QP_FLAG_MPC_QP_FREE, &qp->sp_qp.flags)) {
+ if ((qp->sp_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET ||
+ qp->sp_qp.state == CMDQ_MODIFY_QP_NEW_STATE_INIT) &&
+ (qp->sp_qp.cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_RESET ||
+ qp->sp_qp.cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_INIT)) {
+ dev_warn(&res->pdev->dev,
+ "MPC-skip modify qp- id: %d State: %d->%d\n",
+ qp->sp_qp.id, qp->sp_qp.cur_qp_state,
+ qp->sp_qp.state);
+ qp->sp_qp.cur_qp_state = qp->sp_qp.state;
+ return 0;
+ }
+ }
+ }
+
+ memset(&cmd, 0, sizeof(cmd));
+ memset(&resp, 0, sizeof(resp));
+
+ d = (struct qp_modify_data *)&cmd.qp_cid;
+ bng_sp_fill_mpc_qp_modify_and_create_parms(res, d, qp);
+
+ cmd.resp_size = sizeof(resp) / BNG_RE_MPC_CQ_STRIDE;
+ if (res->is_vf) {
+ cmd.req_type = MPC_CMD_HDR_REQ_TYPE_PFVF;
+ cmd.req_subtype = MPC_CMD_HDR_REQ_SUB_TYPE_PFVF_QP_MODIFY;
+ } else {
+ cmd.req_type = MPC_CMD_HDR_REQ_TYPE_RCA;
+ cmd.req_subtype = MPC_CMD_HDR_REQ_SUB_TYPE_RCA_QP_MODIFY;
+ }
+
+ dev_err(&res->pdev->dev, "MPC: ROCE QP modify id:%d %d->%d\n",
+ qp->sp_qp.id, qp->sp_qp.cur_qp_state, qp->sp_qp.state);
+
+ cpu_modify_mask = le32_to_cpu(d->modify_mask);
+
+ if ((MPC_QP_MODIFY_CMD_QP_MODIFY_DATA_QP_MODIFY_FLAGS_QP_MODIFY_FLAGS_QP_FIRST_MODIFY &
+ cmd.qp_modify_flags) &&
+ !(cpu_modify_mask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE)) {
+ dev_err(&res->pdev->dev,
+ "%s: bad state trans. cqpst:%d,qpst:%d,xid:%d,mask:%x",
+ __func__,
+ qp->sp_qp.cur_qp_state,
+ qp->sp_qp.state,
+ qp->sp_qp.id,
+ cpu_modify_mask);
+ return -EINVAL;
+ }
+
+ if (!res->rdev) {
+ dev_err(&res->pdev->dev, "%s: no rdev for MPC xmit\n", __func__);
+ return -ENODEV;
+ }
+
+ rc = bng_re_roce_mpc_xmit(res->rdev, &cmd, sizeof(cmd),
+ &resp, sizeof(resp), false);
+
+ if (rc) {
+ dev_err(&res->pdev->dev, "MPC: QP modify failed for QP %d, rc=%d\n",
+ qp->sp_qp.id, rc);
+ return rc;
+ }
+
+ if (le16_to_cpu(resp.error_code) != MPC_QP_MODIFY_CMPL_ERROR_CODE_SUCCESS) {
+ dev_err(&res->pdev->dev,
+ "MPC: QP modify cmpl error=%u qp=%u\n",
+ le16_to_cpu(resp.error_code), qp->sp_qp.id);
+ return -EIO;
+ }
+
+ if (qp->sp_qp.id != 1 &&
+ le32_to_cpu(resp.xid) != qp->sp_qp.id) {
+ dev_err(&res->pdev->dev,
+ "MPC: ROCE QP modify xid mismatch cmd:%d resp:%u\n",
+ qp->sp_qp.id, le32_to_cpu(resp.xid));
+ return -EIO;
+ }
+
+ if (qp->sp_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RTR)
+ qp->sp_qp.lag_src_mac = be32_to_cpu(resp.lag_src_mac);
+
+ if (!test_bit(BNG_SP_QP_FLAG_MPC_INITIAL_CREATE_SENT, &qp->sp_qp.flags)) {
+ set_bit(BNG_SP_QP_FLAG_MPC_INITIAL_CREATE_SENT, &qp->sp_qp.flags);
+
+ /* INITIAL_CREATE_SENT is reset when the QP state is modified to RST
+ * QP_DESTROY has QP_FREE flag to release some stats resource
+ * as part of QP destroy in FW and can be sent in RST state
+ */
+
+ if (!test_bit(BNG_SP_QP_FLAG_MPC_ALLOW_QP_DESTROY, &qp->sp_qp.flags)) {
+ set_bit(BNG_SP_QP_FLAG_MPC_ALLOW_QP_DESTROY, &qp->sp_qp.flags);
+ dev_dbg(&res->pdev->dev, "MPC: QP %d marked as initial create sent\n",
+ qp->sp_qp.id);
+ }
+ }
+
+ qp->sp_qp.cur_qp_state = qp->sp_qp.state;
+
+ /* If the QP is moved back to INIT->RTR then send create params again */
+ if (qp->sp_qp.cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
+ clear_bit(BNG_SP_QP_FLAG_MPC_INITIAL_CREATE_SENT, &qp->sp_qp.flags);
+ dev_dbg(&res->pdev->dev, "MPC: QP[%d] CLEAR initial create sent flag\n",
+ qp->sp_qp.id);
+ }
+
+ bng_sp_set_initial_created_flag(res, &qp->sp_qp);
+
+ return 0;
+}
+
+int bng_sp_mpc_modify_qp(struct bng_re_res *res, struct bng_sp_qp *sp_qp)
+{
+ struct bng_re_qp *qp = container_of(sp_qp, struct bng_re_qp, sp_qp);
+
+ return bng_sp_fill_and_send_mpc_qp_modify(res, &qp, 1, NULL);
+}
+
+int bng_sp_query_qp(struct bng_re_res *res, struct bng_sp_qp *qp)
+{
+ struct bng_re_rcfw *rcfw = res->rcfw;
+ struct creq_query_qp_resp resp = {};
+ struct bng_re_cmdqmsg msg = {};
+ struct bng_re_rcfw_sbuf sbuf;
+ struct creq_query_qp_resp_sb *sb;
+ struct cmdq_query_qp req = {};
+ int rc;
+
+ bng_re_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_QUERY_QP,
+ sizeof(req));
+
+ sbuf.size = ALIGN(sizeof(*sb), BNG_FW_CMDQE_UNITS);
+ sbuf.sb = dma_alloc_coherent(&rcfw->pdev->dev, sbuf.size,
+ &sbuf.dma_addr, GFP_KERNEL);
+ if (!sbuf.sb)
+ return -ENOMEM;
+ req.resp_size = sbuf.size / BNG_FW_CMDQE_UNITS;
+ req.qp_cid = cpu_to_le32(qp->id);
+ sb = sbuf.sb;
+
+ bng_re_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
+ sizeof(resp), 0);
+ rc = bng_re_rcfw_send_message(rcfw, &msg);
+ if (!rc) {
+ qp->state = sb->en_sqd_async_notify_state & CREQ_QUERY_QP_RESP_SB_STATE_MASK;
+ qp->cur_qp_state = qp->state; /* Assume same for now */
+ qp->en_sqd_async_notify =
+ (sb->en_sqd_async_notify_state &
+ CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY) ? 1 : 0;
+ qp->access = sb->access;
+ qp->pkey_index = le16_to_cpu(sb->pkey);
+ qp->qkey = le32_to_cpu(sb->qkey);
+ qp->ah.host_sgid_index = le16_to_cpu(sb->sgid_index);
+ qp->udp_sport = le16_to_cpu(sb->udp_src_port);
+ qp->ah.hop_limit = sb->hop_limit;
+ qp->ah.traffic_class = sb->traffic_class;
+ qp->ah.sl = 0;
+ qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id)
+ & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK)
+ >> CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT;
+ qp->timeout = sb->timeout;
+ qp->retry_cnt = sb->retry_cnt;
+ qp->rnr_retry = sb->rnr_retry;
+ qp->min_rnr_timer = sb->min_rnr_timer;
+ qp->port_id = le16_to_cpu(sb->port_id);
+ qp->rq.psn = le32_to_cpu(sb->rq_psn) & 0xffffff;
+ qp->max_rd_atomic = sb->max_rd_atomic;
+ qp->sq.psn = le32_to_cpu(sb->sq_psn) & 0xffffff;
+ qp->max_dest_rd_atomic = sb->max_dest_rd_atomic;
+ qp->dest_qpn = le32_to_cpu(sb->dest_qp_id);
+ memcpy(qp->ah.dgid.data, sb->dgid, sizeof(qp->ah.dgid.data));
+ }
+ dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
+ sbuf.sb, sbuf.dma_addr);
+
+ return rc;
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_sp.h b/drivers/infiniband/hw/bng_re/bng_sp.h
index a6f688c0af3d..60f1cdef145e 100644
--- a/drivers/infiniband/hw/bng_re/bng_sp.h
+++ b/drivers/infiniband/hw/bng_re/bng_sp.h
@@ -21,6 +21,35 @@
#define NQE_PG(x) (((x) & ~NQE_MAX_IDX_PER_PG) / NQE_CNT_PER_PG)
#define NQE_IDX(x) ((x) & NQE_MAX_IDX_PER_PG)
+#define BNG_RE_MAX_SQSZ 0xffffu
+
+#define BNG_VAR_MAX_SLOT_ALIGN 256
+#define IRD_LIMIT_TO_IRRQ_SLOTS(x) (2 * (x) + 2)
+#define ORD_LIMIT_TO_ORRQ_SLOTS(x) ((x) + 1)
+#define SQ_MAX_NUM_WQES_DEFAULT 16
+#define IB_DEFAULT_PKEY_FULL 0xFFFF
+
+#define BNG_RE_IQM_INVALID_IDX (0xFFFFFFFFU)
+#define BNG_RE_IQM_ROUND_UP_BYTE_SIZE 128
+
+#define BNG_RE_MAX_ORRQE_ENTRY_SIZE sizeof(struct xrrq_orrq)
+#define BNG_RE_MAX_IRRQE_ENTRY_SIZE sizeof(struct xrrq_irrq)
+#define BNG_RE_IQM_IRRQ_OVERFLOW_COMP \
+ ((1) * (BNG_RE_IQM_ROUND_UP_BYTE_SIZE / BNG_RE_MAX_IRRQE_ENTRY_SIZE))
+#define IRRQ_CACHELINE_TO_ENTRIES(x) ((x) * (BNG_RE_IQM_ROUND_UP_BYTE_SIZE / \
+ BNG_RE_MAX_IRRQE_ENTRY_SIZE))
+#define ORRQ_CACHELINE_TO_ENTRIES(x) ((x) * (BNG_RE_IQM_ROUND_UP_BYTE_SIZE / \
+ BNG_RE_MAX_ORRQE_ENTRY_SIZE))
+
+struct bng_re_qp;
+
+/* MPC slowpath software bits in sp_qp.flags */
+enum {
+ BNG_SP_QP_FLAG_MPC_INITIAL_CREATE_SENT = 0,
+ BNG_SP_QP_FLAG_MPC_QP_FREE = 1,
+ BNG_SP_QP_FLAG_MPC_ALLOW_QP_DESTROY = 2,
+};
+
struct bng_re_dev_attr {
#define FW_VER_ARR_LEN 4
u8 fw_ver[FW_VER_ARR_LEN];
@@ -33,6 +62,7 @@ struct bng_re_dev_attr {
u32 max_qp_init_rd_atom;
u32 max_qp_wqes;
u32 max_sq_wqes;
+ u32 max_rq_wqes;
u32 max_qp_sges;
u32 max_cq;
u32 max_cq_wqes;
@@ -53,6 +83,7 @@ struct bng_re_dev_attr {
bool is_atomic;
u16 dev_cap_flags;
u16 dev_cap_flags2;
+ u16 dev_cap_ext_flags2;
u32 max_dpi;
};
@@ -287,6 +318,135 @@ struct bng_sp_ah {
u8 nw_type;
};
+struct bng_sp_qp {
+ struct bng_sp_pd *pd;
+ struct bng_re_dpi *dpi;
+ struct bng_re_chip_ctx *cctx;
+ u64 qp_handle;
+ u32 id;
+ u8 is_user;
+ u8 type;
+ u8 sig_type;
+ u8 wqe_mode;
+ u8 state;
+ u8 cur_qp_state;
+ u16 dev_cap_ext_flags2;
+ u64 modify_flags;
+ unsigned long flags;
+ u32 max_inline_data;
+ u32 mtu;
+ u8 path_mtu;
+ bool en_sqd_async_notify;
+ u16 pkey_index;
+ u32 qkey;
+ u32 dest_qp_id;
+ u8 access;
+ u8 timeout;
+ u8 retry_cnt;
+ u8 rnr_retry;
+ u64 wqe_cnt;
+ u32 min_rnr_timer;
+ u32 max_rd_atomic;
+ u32 max_dest_rd_atomic;
+ u32 dest_qpn;
+ u32 ext_modify_flags;
+ u32 roce_stat_ext_xid;
+ u8 smac[6];
+ u16 vlan_id;
+ u16 port_id;
+ u16 udp_sport;
+ u8 nw_type;
+ bool is_roce_mirror_qp;
+ u8 tos_ecn;
+ u8 tos_dscp;
+ struct bng_sp_ah ah;
+
+ struct bng_re_q sq;
+ struct bng_re_q rq;
+ struct bng_sp_srq *srq;
+ struct bng_sp_cq *scq;
+ struct bng_sp_cq *rcq;
+ struct bng_re_hwq irrq;
+ struct bng_re_hwq orrq;
+ int sq_hdr_buf_size;
+ int rq_hdr_buf_size;
+ void *sq_hdr_buf;
+ dma_addr_t sq_hdr_buf_map;
+ void *rq_hdr_buf;
+ dma_addr_t rq_hdr_buf_map;
+ struct list_head sq_flush;
+ struct list_head rq_flush;
+ /* 4-byte scrambled MAC received from FW on RTR transition */
+ u32 lag_src_mac;
+ u32 msn;
+ /* indicates buffers avail for rq; at transition to rtr */
+ u32 req_buffer_count;
+ u8 rtr_transition_pending;
+ u8 dev_cap_ext_flags;
+ bool udcc_exclude;
+ u32 ugid_index;
+#define INVALID_VF_IDX 0xFFFFFFFF
+ u32 vf_idx;
+};
+
+/* SWQ */
+struct bng_re_swq {
+ u64 wr_id;
+ int next_idx;
+ u8 type;
+ u8 flags;
+ u32 slot_idx;
+ u8 slots;
+};
+
+static inline u32 bng_re_get_depth(struct bng_re_q *que, bool is_sq)
+{
+ u32 slots;
+
+ slots = (que->wqe_size * que->max_wqe) / sizeof(struct sq_sge);
+ if (is_sq)
+ slots = ALIGN(slots, BNG_VAR_MAX_SLOT_ALIGN);
+ return slots;
+}
+
+static inline u32 bng_re_set_sq_size(struct bng_re_q *que)
+{
+ return bng_re_get_depth(que, true);
+}
+
+static inline u32 bng_re_set_sq_max_slot(u8 wqe_mode)
+{
+ return 1;
+}
+
+static inline u32 bng_re_set_rq_max_slot(u32 wqe_size)
+{
+ return (wqe_size / sizeof(struct sq_sge));
+}
+
+static inline const char *__to_qp_state_str(u8 state)
+{
+ switch (state) {
+ case CMDQ_MODIFY_QP_NEW_STATE_RESET:
+ return "RESET";
+ case CMDQ_MODIFY_QP_NEW_STATE_INIT:
+ return "INIT";
+ case CMDQ_MODIFY_QP_NEW_STATE_RTR:
+ return "RTR";
+ case CMDQ_MODIFY_QP_NEW_STATE_RTS:
+ return "RTS";
+ case CMDQ_MODIFY_QP_NEW_STATE_SQD:
+ return "SQD";
+ case CMDQ_MODIFY_QP_NEW_STATE_SQE:
+ return "SQE";
+ case CMDQ_MODIFY_QP_NEW_STATE_ERR:
+ return "ERR";
+ default:
+ return "NotSupp";
+ }
+}
+
+int bng_re_alloc_init_swq(struct bng_re_q *que);
int bng_re_get_dev_attr(struct bng_re_rcfw *rcfw);
int bng_sp_alloc_sgid_tbl(struct bng_re_sgid_tbl *sgid_tbl, u16 size);
@@ -322,6 +482,7 @@ int bng_sp_resize_cq(struct bng_re_res *res, struct bng_sp_cq *cq,
int new_cqes);
int bng_sp_destroy_cq(struct bng_re_res *res, struct bng_sp_cq *cq);
int bng_sp_create_srq(struct bng_re_res *res, struct bng_sp_srq *srq);
+int bng_sp_query_qp(struct bng_re_res *res, struct bng_sp_qp *qp);
int bng_sp_query_srq(struct bng_re_res *res, struct bng_sp_srq *srq);
int bng_sp_destroy_srq(struct bng_re_res *res, struct bng_sp_srq *srq);
int bng_re_qext_stat(struct bng_re_rcfw *rcfw, u32 fid,
@@ -331,5 +492,14 @@ int bng_re_get_roce_stats(struct bng_re_rcfw *rcfw,
int bng_sp_fill_and_send_mpc_ah_modify(struct bng_re_res *res,
struct bng_sp_ah **ahs,
int num_ahs, bool for_destroy, bool block);
-
+void bng_re_flush_cqn_wq(struct bng_sp_qp *qp);
+void bng_re_free_qp_res(struct bng_re_res *res, struct bng_sp_qp *qp);
+void bng_re_clean_qp(struct bng_sp_qp *qp);
+int bng_sp_fill_and_send_mpc_qp_modify(struct bng_re_res *res,
+ struct bng_re_qp **qps,
+ int num_qps,
+ bool *xid_cleanup_handled);
+void bng_re_add_flush_qp(struct bng_sp_qp *qp);
+int bng_sp_mpc_create_qp(struct bng_re_res *res, struct bng_sp_qp *qp);
+int bng_sp_mpc_modify_qp(struct bng_re_res *res, struct bng_sp_qp *sp_qp);
#endif
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.c b/drivers/infiniband/hw/bng_re/bng_verbs.c
index c40f8686b6f6..e91fd111d2f1 100644
--- a/drivers/infiniband/hw/bng_re/bng_verbs.c
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.c
@@ -5,6 +5,7 @@
#include <linux/types.h>
#include <linux/pci.h>
#include <linux/io.h>
+#include <linux/crc16.h>
#include <net/addrconf.h>
#include <rdma/ib_cache.h>
#include <rdma/ib_verbs.h>
@@ -22,6 +23,7 @@
#include "bng_re.h"
#include "bng_verbs.h"
#include "bng_fp.h"
+#include "bng_xid.h"
int bng_re_query_device(struct ib_device *ibdev,
struct ib_device_attr *ib_attr,
@@ -578,6 +580,21 @@ static void bng_re_check_and_set_relaxed_ordering(struct bng_re_dev *rdev,
mr->flags |= CMDQ_REGISTER_MR_FLAGS_ENABLE_RO;
}
+static int __qp_access_flags_to_ib(u32 cctx, u8 access)
+{
+ int qflags = 0;
+
+ if (access & CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE)
+ qflags |= IB_ACCESS_LOCAL_WRITE;
+ if (access & CMDQ_MODIFY_QP_ACCESS_REMOTE_READ)
+ qflags |= IB_ACCESS_REMOTE_READ;
+ if (access & CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE)
+ qflags |= IB_ACCESS_REMOTE_WRITE;
+ if (access & CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC)
+ qflags |= IB_ACCESS_REMOTE_ATOMIC;
+ return qflags;
+}
+
static int __from_ib_access_flags(int iflags)
{
int qflags = 0;
@@ -1627,3 +1644,1568 @@ int bng_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
rdma_ah_set_static_rate(ah_attr, 0);
return 0;
}
+
+/* Helper functions for QP operations */
+static bool bng_re_init_qp_wqe_mode(struct bng_re_dev *rdev)
+{
+ return rdev->chip_ctx->modes.wqe_mode;
+}
+
+static u8 __from_ib_qp_type(enum ib_qp_type type)
+{
+ switch (type) {
+ case IB_QPT_GSI:
+ return CMDQ_CREATE_QP1_TYPE_GSI;
+ case IB_QPT_RC:
+ return CMDQ_CREATE_QP_TYPE_RC;
+ case IB_QPT_UD:
+ return CMDQ_CREATE_QP_TYPE_UD;
+ case IB_QPT_RAW_PACKET:
+ return CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE;
+ default:
+ return IB_QPT_MAX;
+ }
+}
+
+static bool bng_re_test_qp_limits(struct bng_re_dev *rdev,
+ struct ib_qp_init_attr *init_attr,
+ struct bng_re_dev_attr *dev_attr)
+{
+ bool rc = true;
+
+ int ilsize = ALIGN(init_attr->cap.max_inline_data, sizeof(struct sq_sge));
+
+ if (init_attr->cap.max_send_wr < 1) {
+ ibdev_err(&rdev->ibdev, "Create QP failed - max_send_wr must be >= 1");
+ rc = false;
+ }
+ if (!init_attr->srq && init_attr->cap.max_recv_wr < 1) {
+ ibdev_err(&rdev->ibdev, "Create QP failed - max_recv_wr must be >= 1 when not using SRQ");
+ rc = false;
+ }
+ if (init_attr->cap.max_send_wr > dev_attr->max_sq_wqes ||
+ init_attr->cap.max_recv_wr > dev_attr->max_rq_wqes ||
+ init_attr->cap.max_send_sge > dev_attr->max_qp_sges ||
+ init_attr->cap.max_recv_sge > dev_attr->max_qp_sges ||
+ ilsize > dev_attr->max_inline_data) {
+ ibdev_err(&rdev->ibdev,
+ "Create QP failed - max exceeded! 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x",
+ init_attr->cap.max_send_wr, dev_attr->max_sq_wqes,
+ init_attr->cap.max_recv_wr, dev_attr->max_rq_wqes,
+ init_attr->cap.max_send_sge, dev_attr->max_qp_sges,
+ init_attr->cap.max_recv_sge, dev_attr->max_qp_sges,
+ ilsize,
+ dev_attr->max_inline_data);
+ rc = false;
+ }
+ return rc;
+}
+
+static int bng_re_init_qp_type(struct bng_re_dev *rdev,
+ struct ib_qp_init_attr *init_attr)
+{
+ int qptype;
+
+ qptype = __from_ib_qp_type(init_attr->qp_type);
+ if (qptype == IB_QPT_MAX) {
+ ibdev_err(&rdev->ibdev, "QP type 0x%x not supported", qptype);
+ qptype = -EOPNOTSUPP;
+ goto out;
+ }
+
+ if (init_attr->qp_type == IB_QPT_GSI) {
+ qptype = CMDQ_CREATE_QP_TYPE_GSI;
+ rdev->gsi_ctx.gsi_qp_mode = BNG_RE_GSI_MODE_UD;
+ }
+out:
+ return qptype;
+}
+
+static int bng_re_init_rq_attr(struct bng_re_qp *qp,
+ struct ib_qp_init_attr *init_attr,
+ struct bng_re_ucontext *uctx)
+{
+ struct bng_re_dev_attr *dev_attr;
+ struct bng_sp_qp *sp_qp;
+ struct bng_re_dev *rdev;
+ struct bng_re_q *rq;
+ int entries;
+
+ rdev = qp->rdev;
+ sp_qp = &qp->sp_qp;
+ rq = &sp_qp->rq;
+ dev_attr = rdev->dev_attr;
+
+ if (init_attr->srq) {
+ struct bng_re_srq *srq;
+
+ srq = container_of(init_attr->srq, struct bng_re_srq, ib_srq);
+ if (!srq) {
+ dev_err(rdev_to_dev(rdev), "SRQ not found");
+ return -EINVAL;
+ }
+ sp_qp->srq = (struct bng_sp_srq *)&srq->sp_srq;
+ rq->max_wqe = 0;
+ } else {
+ rq->max_sge = init_attr->cap.max_recv_sge;
+ if (rq->max_sge > dev_attr->max_qp_sges)
+ rq->max_sge = dev_attr->max_qp_sges;
+ init_attr->cap.max_recv_sge = rq->max_sge;
+ rq->wqe_size = bng_re_get_rwqe_size(rq->max_sge);
+ /* Allocate 1 more than what's provided so posting max doesn't
+ * mean empty.
+ */
+
+ entries = init_attr->cap.max_recv_wr + 1;
+ entries = bng_re_init_depth(entries, uctx);
+ if (init_attr->qp_type == IB_QPT_GSI)
+ entries++;
+ rq->max_wqe = min_t(u32, entries, dev_attr->max_rq_wqes);
+ rq->max_sw_wqe = rq->max_wqe;
+ rq->q_full_delta = 0;
+ rq->sg_info.pgsize = PAGE_SIZE;
+ rq->sg_info.pgshft = PAGE_SHIFT;
+ }
+
+ return 0;
+}
+
+static u16 bng_re_get_swqe_size(int ilsize, int nsge, int align)
+{
+ u16 wqe_size, calc_ils;
+
+ wqe_size = __get_swqe_size(nsge);
+ if (ilsize) {
+ calc_ils = sizeof(struct sq_atomic_hdr) + ilsize;
+ wqe_size = max_t(int, calc_ils, wqe_size);
+ }
+ wqe_size = ALIGN(wqe_size, align);
+ return wqe_size;
+}
+
+static int bng_re_setup_swqe_size(struct bng_re_qp *qp,
+ u32 *max_inline_data)
+{
+ struct bng_re_dev_attr *dev_attr;
+ struct bng_sp_qp *sp_qp;
+ struct bng_re_dev *rdev;
+ struct bng_re_q *sq;
+ int align, ilsize;
+
+ rdev = qp->rdev;
+ sp_qp = &qp->sp_qp;
+ sq = &sp_qp->sq;
+ dev_attr = rdev->dev_attr;
+
+ align = sizeof(struct sq_sge);
+ ilsize = ALIGN(*max_inline_data, align);
+
+ sq->wqe_size = bng_re_get_swqe_size(ilsize, sq->max_sge, align);
+ if (sq->wqe_size > ALIGN(__get_swqe_size(dev_attr->max_qp_sges), align))
+ return -EINVAL;
+
+ if (*max_inline_data) {
+ sp_qp->max_inline_data = sq->wqe_size -
+ ALIGN(sizeof(struct sq_atomic_hdr), align);
+ *max_inline_data = sp_qp->max_inline_data;
+ }
+
+ return 0;
+}
+
+static int bng_re_init_sq_attr(struct bng_re_qp *qp,
+ struct ib_qp_init_attr *init_attr,
+ struct bng_re_ucontext *uctx)
+{
+ struct bng_re_dev_attr *dev_attr;
+ struct bng_sp_qp *sp_qp;
+ struct bng_re_dev *rdev;
+ struct bng_re_q *sq;
+ int diff = 0;
+ int entries;
+ int rc;
+
+ rdev = qp->rdev;
+ sp_qp = &qp->sp_qp;
+ sq = &sp_qp->sq;
+ dev_attr = rdev->dev_attr;
+
+ entries = init_attr->cap.max_send_wr;
+
+ if (sq->max_sge > dev_attr->max_qp_sges) {
+ sq->max_sge = dev_attr->max_qp_sges;
+ init_attr->cap.max_send_sge = sq->max_sge;
+ }
+
+ rc = bng_re_setup_swqe_size(qp, &init_attr->cap.max_inline_data);
+ if (rc)
+ return rc;
+
+ /*
+ * Change the SQ depth if user has requested minimum using
+ * configfs. Only supported for kernel consumers. Setting
+ * min_tx_depth to 4096 to handle iser SQ full condition
+ * in most of the newer OS distros
+ */
+
+ entries = init_attr->cap.max_send_wr;
+
+ if (!uctx && rdev->min_tx_depth && init_attr->qp_type != IB_QPT_GSI) {
+ /*
+ * If users specify any value greater than 1 use min_tx_depth
+ * provided by user for comparison. Else, compare it with the
+ * BNG_RE_MIN_KERNEL_QP_TX_DEPTH and adjust it accordingly.
+ */
+ if (rdev->min_tx_depth > 1 && entries < rdev->min_tx_depth)
+ entries = rdev->min_tx_depth;
+ else if (entries < BNG_RE_MIN_KERNEL_QP_TX_DEPTH)
+ entries = BNG_RE_MIN_KERNEL_QP_TX_DEPTH;
+ }
+
+ diff = bng_re_get_diff(uctx, rdev->chip_ctx);
+ entries = bng_re_init_depth(entries + diff + 1, uctx);
+ sq->max_wqe = min_t(u32, entries, dev_attr->max_sq_wqes + diff + 1);
+ sq->q_full_delta = diff + 1;
+ /*
+ * Reserving one slot for Phantom WQE. Application can
+ * post one extra entry in this case. But allowing this to avoid
+ * unexpected Queue full condition
+ */
+ sp_qp->sq.q_full_delta -= 1;
+ sp_qp->sq.sg_info.pgsize = PAGE_SIZE;
+ sp_qp->sq.sg_info.pgshft = PAGE_SHIFT;
+
+ return 0;
+}
+
+static int bng_re_init_user_qp(struct bng_re_dev *rdev, struct bng_re_pd *pd,
+ struct bng_re_qp *qp, struct ib_udata *udata)
+{
+ struct bng_re_qp_req ureq = {};
+ struct bng_re_sg_info *sginfo;
+ struct bng_re_ucontext *cntx;
+ struct ib_ucontext *context;
+ struct bng_sp_qp *sp_qp;
+ struct ib_umem *umem;
+ int rc, bytes = 0;
+
+ sp_qp = &qp->sp_qp;
+ context = pd->ib_pd.uobject->context;
+ cntx = to_bng_re(context, struct bng_re_ucontext, ib_uctx);
+ sginfo = &sp_qp->sq.sg_info;
+
+ if (udata) {
+ if (udata->inlen < sizeof(ureq))
+ dev_warn_once(rdev_to_dev(rdev),
+ "Update the library ulen %d klen %d",
+ (unsigned int)udata->inlen,
+ (unsigned int)sizeof(ureq));
+ rc = ib_copy_from_udata(&ureq, udata, min(udata->inlen, sizeof(ureq)));
+ if (rc)
+ return rc;
+ }
+
+ bytes = (sp_qp->sq.max_wqe * sp_qp->sq.wqe_size);
+ bytes = PAGE_ALIGN(bytes);
+
+ umem = ib_umem_get_va(&rdev->ibdev, ureq.qpsva, bytes,
+ IB_ACCESS_LOCAL_WRITE);
+ if (IS_ERR(umem)) {
+ dev_err(rdev_to_dev(rdev), "%s: ib_umem_get_va failed with %ld\n",
+ __func__, PTR_ERR(umem));
+ return PTR_ERR(umem);
+ }
+
+ qp->sumem = umem;
+ sginfo->npages = ib_umem_num_dma_blocks(umem, PAGE_SIZE);
+ sginfo->umem = umem;
+ sp_qp->qp_handle = ureq.qp_handle;
+
+ if (!qp->sp_qp.srq) {
+ sginfo = &sp_qp->rq.sg_info;
+ bytes = (sp_qp->rq.max_wqe * sp_qp->rq.wqe_size);
+ bytes = PAGE_ALIGN(bytes);
+ umem = ib_umem_get_va(&rdev->ibdev, ureq.qprva, bytes,
+ IB_ACCESS_LOCAL_WRITE);
+ if (IS_ERR(umem)) {
+ dev_err(rdev_to_dev(rdev),
+ "%s: ib_umem_get_va failed ret =%ld\n",
+ __func__, PTR_ERR(umem));
+ goto rq_fail;
+ }
+ qp->rumem = umem;
+ sp_qp->rq.sg_info.umem = umem;
+ sginfo->npages = ib_umem_num_dma_blocks(umem, PAGE_SIZE);
+ sginfo->umem = umem;
+ }
+
+ sp_qp->dpi = &cntx->dpi;
+ sp_qp->is_user = true;
+
+ return 0;
+rq_fail:
+ ib_umem_release(qp->sumem);
+ qp->sumem = NULL;
+ sp_qp->sq.sg_info.umem = NULL;
+
+ return PTR_ERR(umem);
+}
+
+static int bng_re_init_qp_attr(struct bng_re_qp *qp, struct bng_re_pd *pd,
+ struct ib_qp_init_attr *init_attr, struct ib_udata *udata)
+{
+ struct bng_re_ucontext *cntx = NULL;
+ struct bng_re_dev_attr *dev_attr;
+ struct ib_ucontext *context;
+ struct bng_sp_qp *sp_qp;
+ struct bng_re_dev *rdev;
+ struct bng_re_cq *cq;
+ int rc = 0, qptype;
+
+ rdev = qp->rdev;
+ sp_qp = &qp->sp_qp;
+ dev_attr = rdev->dev_attr;
+
+ sp_qp->vf_idx = INVALID_VF_IDX;
+
+ if (udata) {
+ context = pd->ib_pd.uobject->context;
+ cntx = to_bng_re(context, struct bng_re_ucontext, ib_uctx);
+ }
+
+ sp_qp->is_user = udata ? true : false;
+ sp_qp->pd = &pd->sp_pd;
+ sp_qp->qp_handle = (u64)sp_qp;
+ sp_qp->max_inline_data = init_attr->cap.max_inline_data;
+ sp_qp->sig_type = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
+ qptype = bng_re_init_qp_type(rdev, init_attr);
+ if (qptype < 0) {
+ rc = qptype;
+ goto out;
+ }
+ sp_qp->type = (u8)qptype;
+ sp_qp->wqe_mode = bng_re_init_qp_wqe_mode(rdev);
+ ether_addr_copy(sp_qp->smac, rdev->netdev->dev_addr);
+
+ if (init_attr->qp_type == IB_QPT_RC) {
+ sp_qp->max_rd_atomic = dev_attr->max_qp_rd_atom;
+ sp_qp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
+ }
+ sp_qp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
+ sp_qp->dpi = &rdev->dpi_privileged;
+ if (init_attr->create_flags) {
+ ibdev_dbg(&rdev->ibdev,
+ "QP create flags 0x%x not supported",
+ init_attr->create_flags);
+ return -EOPNOTSUPP;
+ }
+
+ /* Setup CQs */
+ if (init_attr->send_cq) {
+ cq = to_bng_re(init_attr->send_cq, struct bng_re_cq, ib_cq);
+ if (!cq) {
+ dev_err(rdev_to_dev(rdev), "Send CQ not found");
+ rc = -EINVAL;
+ goto out;
+ }
+ sp_qp->scq = &cq->sp_cq;
+ qp->scq = cq;
+ }
+
+ if (init_attr->recv_cq) {
+ cq = to_bng_re(init_attr->recv_cq, struct bng_re_cq, ib_cq);
+ if (!cq) {
+ dev_err(rdev_to_dev(rdev), "Send CQ not found");
+ rc = -EINVAL;
+ goto out;
+ }
+ sp_qp->rcq = &cq->sp_cq;
+ qp->rcq = cq;
+ }
+
+ if ((init_attr->qp_type == IB_QPT_RC || init_attr->qp_type == IB_QPT_UC) &&
+ (!sp_qp->scq || !sp_qp->rcq)) {
+ ibdev_err(&rdev->ibdev,
+ "Create QP: RC/UC requires both send_cq and recv_cq\n");
+ return -EINVAL;
+ }
+
+ /* Setup RQ/SRQ */
+ rc = bng_re_init_rq_attr(qp, init_attr, cntx);
+ if (rc)
+ goto out;
+
+ /* Setup SQ */
+ rc = bng_re_init_sq_attr(qp, init_attr, cntx);
+ if (rc)
+ goto out;
+
+ if (udata) /* This will update DPI and qp_handle */
+ rc = bng_re_init_user_qp(rdev, pd, qp, udata);
+out:
+ return rc;
+}
+
+/*
+ * bng_re_create_qp_stage1
+ *
+ * Return: 0 on success, negative errno on failure (all partial allocations unwound).
+ */
+static int bng_re_create_qp_stage1(struct bng_re_res *res, struct bng_re_qp *qp)
+{
+ struct bng_re_rcfw *rcfw;
+ struct bng_re_hwq_attr hwq_attr = {};
+ struct bng_sp_qp *fp = &qp->sp_qp;
+ struct bng_re_q *sq = &fp->sq;
+ struct bng_re_q *rq = &fp->rq;
+ u32 tbl_indx;
+ struct bng_re_reftbl *tbl;
+ u32 sqsz;
+ int vf_id = 0;
+ int xid_out = 0;
+ int rc;
+ u32 qp_idx;
+ unsigned long flag;
+
+ if (!res || !res->pdev || !res->rcfw)
+ return -EINVAL;
+ rcfw = res->rcfw;
+ if (!rcfw->qp_tbl || rcfw->qp_tbl_size < 3) {
+ dev_err(&res->pdev->dev,
+ "Stage1: RCFW qp_tbl missing or too small (size=%d)\n",
+ rcfw->qp_tbl_size);
+ return -ENODEV;
+ }
+ if (!fp->dpi || !fp->dpi->dbr) {
+ dev_err(&res->pdev->dev, "Stage1: QP DPI / doorbell not initialized\n");
+ return -EINVAL;
+ }
+
+ dev_dbg(&res->pdev->dev,
+ "MPC: Stage1 enter type=%u srq=%d wqe_mode=%u rq_max_wqe=%u qp_tbl_sz=%d dpi=%u\n",
+ fp->type, fp->srq ? 1 : 0, fp->wqe_mode,
+ fp->srq ? 0U : rq->max_wqe, rcfw->qp_tbl_size,
+ fp->dpi ? fp->dpi->dpi : 0U);
+
+ fp->cctx = res->cctx;
+ fp->flags = 0;
+ sq->dbinfo.flags = 0;
+
+ hwq_attr.res = res;
+ hwq_attr.sginfo = &sq->sg_info;
+ hwq_attr.stride = sizeof(struct sq_sge);
+ hwq_attr.depth = bng_re_get_depth(sq, true);
+ /* Thor3: no SQ aux ring for retransmission metadata */
+ hwq_attr.aux_stride = 0;
+ hwq_attr.aux_depth = 0;
+ hwq_attr.type = BNG_HWQ_TYPE_QUEUE;
+
+ rc = bng_re_alloc_init_hwq(&sq->hwq, &hwq_attr);
+ if (rc) {
+ dev_dbg(&res->pdev->dev, "MPC: Stage1 SQ hwq alloc failed rc=%d\n", rc);
+ goto exit;
+ }
+
+ sqsz = bng_re_set_sq_size(sq);
+ sq->max_sw_wqe = sqsz;
+ if (!sqsz || sqsz > BNG_RE_MAX_SQSZ) {
+ dev_err(&res->pdev->dev,
+ "bng_re: Stage1 SQ size %u invalid (max %u)\n",
+ sqsz, BNG_RE_MAX_SQSZ);
+ rc = -EINVAL;
+ goto fail_sq;
+ }
+
+ if (!fp->srq) {
+ rq->dbinfo.flags = 0;
+ hwq_attr.res = res;
+ hwq_attr.sginfo = &rq->sg_info;
+ hwq_attr.stride = sizeof(struct sq_sge);
+ hwq_attr.depth = bng_re_get_depth(rq, false);
+ hwq_attr.aux_stride = 0;
+ hwq_attr.aux_depth = 0;
+ hwq_attr.type = BNG_HWQ_TYPE_QUEUE;
+ dev_dbg(&res->pdev->dev,
+ "MPC: Stage1 RQ hwq depth=%u max_wqe=%u wqe_sz=%u\n",
+ hwq_attr.depth, rq->max_wqe, rq->wqe_size);
+ rc = bng_re_alloc_init_hwq(&rq->hwq, &hwq_attr);
+ if (rc) {
+ dev_dbg(&res->pdev->dev,
+ "MPC: Stage1 RQ hwq alloc failed rc=%d\n", rc);
+ goto fail_sq;
+ }
+ dev_dbg(&res->pdev->dev,
+ "MPC: Stage1 RQ hwq ok max_elems=%u is_user=%d\n",
+ rq->hwq.max_elements, rq->hwq.is_user);
+ } else {
+ dev_dbg(&res->pdev->dev, "MPC: Stage1 skip RQ hwq (SRQ)\n");
+ }
+
+ if (!sq->hwq.is_user) {
+ dev_dbg(&res->pdev->dev,
+ "MPC: Stage1 kernel SWQ path max_sw_wqe=%u\n", sq->max_sw_wqe);
+ rc = bng_re_alloc_init_swq(sq);
+ if (rc) {
+ dev_dbg(&res->pdev->dev,
+ "MPC: Stage1 SQ swq alloc failed rc=%d\n", rc);
+ goto swq_sq;
+ }
+ if (!fp->srq) {
+ rc = bng_re_alloc_init_swq(rq);
+ if (rc) {
+ dev_dbg(&res->pdev->dev,
+ "MPC: Stage1 RQ swq alloc failed rc=%d\n", rc);
+ goto swq_rq;
+ }
+ }
+ } else {
+ dev_dbg(&res->pdev->dev, "MPC: Stage1 skip kernel SWQ (user SQ)\n");
+ }
+
+ rc = bng_re_qp_alloc_xid_and_iqm(res, fp->type, 1, sq->max_sw_wqe,
+ vf_id, &qp->iqm_res, &xid_out);
+ if (!rc)
+ fp->id = (u32)xid_out;
+ if (rc) {
+ dev_dbg(&res->pdev->dev,
+ "MPC: Stage1 XID/VF path failed rc=%d (no fp->id assign)\n", rc);
+ goto swq_rq;
+ }
+
+ fp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
+ INIT_LIST_HEAD(&fp->sq_flush);
+ INIT_LIST_HEAD(&fp->rq_flush);
+
+ sq->dbinfo.hwq = &sq->hwq;
+ sq->dbinfo.xid = fp->id;
+ sq->dbinfo.db = fp->dpi->dbr;
+ sq->dbinfo.max_slot = bng_re_set_sq_max_slot(fp->wqe_mode);
+ sq->dbinfo.flags = 0;
+ spin_lock_init(&sq->dbinfo.lock);
+ sq->dbinfo.seed = fp->id;
+ sq->dbinfo.res = res;
+ sq->dbinfo.is_l2 = false;
+
+ if (rq->max_wqe) {
+ rq->dbinfo.hwq = &rq->hwq;
+ rq->dbinfo.xid = fp->id;
+ rq->dbinfo.db = fp->dpi->dbr;
+ rq->dbinfo.max_slot = bng_re_set_rq_max_slot(rq->wqe_size);
+ rq->dbinfo.flags = 0;
+ spin_lock_init(&rq->dbinfo.lock);
+ rq->dbinfo.seed = fp->id;
+ rq->dbinfo.res = res;
+ rq->dbinfo.is_l2 = false;
+ }
+
+ tbl = &res->reftbl.qpref;
+ qp_idx = map_qp_id_to_reftbl_indx(fp->id, tbl);
+ spin_lock_irqsave(&tbl->lock, flag);
+ tbl->rec[qp_idx].xid = fp->id;
+ tbl->rec[qp_idx].handle = fp;
+ spin_unlock_irqrestore(&tbl->lock, flag);
+
+ if (fp->type == CMDQ_CREATE_QP_TYPE_RC) {
+ dev_dbg(&res->pdev->dev,
+ "MPC: Stage1 ok id=%u tbl=%u IQM irrq=%#x sz=%u orrq=%#x msn=%#x\n",
+ fp->id, tbl_indx, qp->iqm_res.irrq_addr, qp->iqm_res.irrq_size,
+ qp->iqm_res.orrq_addr, qp->iqm_res.msn_addr);
+ } else {
+ dev_dbg(&res->pdev->dev,
+ "MPC: Stage1 ok id=%u tbl=%u type=%u (no RC IQM dump)\n",
+ fp->id, tbl_indx, fp->type);
+ }
+
+ return 0;
+
+swq_rq:
+ kvfree(sq->swq);
+ sq->swq = NULL;
+ if (!fp->srq) {
+ kvfree(rq->swq);
+ rq->swq = NULL;
+ bng_re_free_hwq(res, &rq->hwq);
+ }
+ goto fail_sq;
+swq_sq:
+ if (!fp->srq)
+ bng_re_free_hwq(res, &rq->hwq);
+fail_sq:
+ bng_re_free_hwq(res, &sq->hwq);
+exit:
+ if (rc)
+ dev_dbg(&res->pdev->dev, "MPC: Stage1 fail exit rc=%d\n", rc);
+ return rc;
+}
+
+static enum ib_qp_state bng_re_cmdq_new_state_to_ib(u8 state)
+{
+ switch (state) {
+ case CMDQ_MODIFY_QP_NEW_STATE_RESET:
+ return IB_QPS_RESET;
+ case CMDQ_MODIFY_QP_NEW_STATE_INIT:
+ return IB_QPS_INIT;
+ case CMDQ_MODIFY_QP_NEW_STATE_RTR:
+ return IB_QPS_RTR;
+ case CMDQ_MODIFY_QP_NEW_STATE_RTS:
+ return IB_QPS_RTS;
+ case CMDQ_MODIFY_QP_NEW_STATE_SQD:
+ return IB_QPS_SQD;
+ case CMDQ_MODIFY_QP_NEW_STATE_SQE:
+ return IB_QPS_SQE;
+ case CMDQ_MODIFY_QP_NEW_STATE_ERR:
+ default:
+ return IB_QPS_ERR;
+ }
+}
+
+int bng_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
+ int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
+{
+ struct bng_re_qp *qp = to_bng_re(ib_qp, struct bng_re_qp, ib_qp);
+ struct bng_re_dev *rdev = qp->rdev;
+ struct bng_sp_qp *sp_qp;
+ bool is_alloced = false;
+ int rc = 0;
+
+ if (rdev->bng_res.qp_xids &&
+ (!test_bit(BNG_SP_QP_FLAG_MPC_INITIAL_CREATE_SENT, &qp->sp_qp.flags))) {
+ sp_qp = &qp->sp_qp;
+ } else {
+ sp_qp = kcalloc(1, sizeof(*sp_qp), GFP_KERNEL);
+ if (!sp_qp)
+ return -ENOMEM;
+ is_alloced = true;
+ sp_qp->id = qp->sp_qp.id;
+ sp_qp->ah.host_sgid_index = qp->sp_qp.ah.host_sgid_index;
+
+ rc = bng_sp_query_qp(&rdev->bng_res, sp_qp);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Query HW QP (0x%x) failed! rc = %d",
+ sp_qp->id, rc);
+ goto free_mem;
+ }
+ }
+ qp_attr->qp_state = bng_re_cmdq_new_state_to_ib(sp_qp->state);
+ qp_attr->cur_qp_state = bng_re_cmdq_new_state_to_ib(sp_qp->cur_qp_state);
+ qp_attr->en_sqd_async_notify = sp_qp->en_sqd_async_notify ? 1 : 0;
+ qp_attr->qp_access_flags = __qp_access_flags_to_ib(0,
+ sp_qp->access);
+ qp_attr->pkey_index = sp_qp->pkey_index;
+ qp_attr->qkey = sp_qp->qkey;
+ qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
+ memcpy(qp_attr->ah_attr.grh.dgid.raw, sp_qp->ah.dgid.data,
+ sizeof(sp_qp->ah.dgid.data));
+ qp_attr->ah_attr.grh.flow_label = sp_qp->udp_sport;
+ qp_attr->ah_attr.grh.sgid_index = sp_qp->ah.host_sgid_index;
+ qp_attr->ah_attr.grh.hop_limit = sp_qp->ah.hop_limit;
+ qp_attr->ah_attr.grh.traffic_class = sp_qp->ah.traffic_class;
+ qp_attr->ah_attr.sl = sp_qp->ah.sl;
+ qp_attr->path_mtu = __to_ib_mtu(sp_qp->path_mtu);
+ qp_attr->timeout = sp_qp->timeout;
+ qp_attr->retry_cnt = sp_qp->retry_cnt;
+ qp_attr->rnr_retry = sp_qp->rnr_retry;
+ qp_attr->min_rnr_timer = sp_qp->min_rnr_timer;
+ qp_attr->port_num = __to_ib_port_num(sp_qp->port_id);
+ qp_attr->rq_psn = sp_qp->rq.psn;
+ qp_attr->max_rd_atomic = sp_qp->max_rd_atomic;
+ qp_attr->sq_psn = sp_qp->sq.psn;
+ qp_attr->max_dest_rd_atomic = sp_qp->max_dest_rd_atomic;
+ qp_init_attr->sq_sig_type = sp_qp->sig_type ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
+ qp_attr->dest_qp_num = sp_qp->dest_qpn;
+
+ qp_attr->cap.max_send_wr = qp->sp_qp.sq.max_wqe;
+ qp_attr->cap.max_send_sge = qp->sp_qp.sq.max_sge;
+ qp_attr->cap.max_recv_wr = qp->sp_qp.rq.max_wqe;
+ qp_attr->cap.max_recv_sge = qp->sp_qp.rq.max_sge;
+ qp_attr->cap.max_inline_data = sp_qp->max_inline_data;
+ qp_init_attr->cap = qp_attr->cap;
+
+free_mem:
+ if (is_alloced)
+ kfree(sp_qp);
+ return rc;
+}
+
+int bng_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
+ struct ib_udata *udata)
+{
+ struct bng_re_dev_attr *dev_attr;
+ struct bng_re_qp_resp resp;
+ struct bng_re_dev *rdev;
+ struct bng_re_pd *pd;
+ struct bng_re_qp *qp;
+ struct ib_pd *ib_pd;
+ int rc;
+
+ ib_pd = ib_qp->pd;
+ pd = container_of(ib_pd, struct bng_re_pd, ib_pd);
+ rdev = pd->rdev;
+ dev_attr = rdev->dev_attr;
+ qp = container_of(ib_qp, struct bng_re_qp, ib_qp);
+
+ rc = bng_re_test_qp_limits(rdev, qp_init_attr, dev_attr);
+ if (!rc) {
+ rc = -EINVAL;
+ goto fail;
+ }
+
+ qp->rdev = rdev;
+ rc = bng_re_init_qp_attr(qp, pd, qp_init_attr, udata);
+ if (rc)
+ goto fail;
+
+ rc = bng_re_create_qp_stage1(&rdev->bng_res, qp);
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Failed to create HW QP (stage1 / XID prep)");
+ goto free_umem;
+ }
+ if (bng_re_init_fw_state_rtr_enabled(rdev->chip_ctx))
+ qp->sp_qp.rtr_transition_pending = true;
+
+ if (udata) {
+ resp.qpid = qp->sp_qp.id;
+ resp.rsvd = 0;
+ if (udata->outlen < sizeof(resp)) {
+ ibdev_err(&rdev->ibdev,
+ "create_qp: udata outlen %zu < sizeof(bng_re_qp_resp) %zu\n",
+ udata->outlen, sizeof(resp));
+ rc = -EINVAL;
+ goto free_umem;
+ }
+ rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
+ if (rc) {
+ ibdev_err(&rdev->ibdev,
+ "create_qp: ib_copy_to_udata failed rc=%d qpid=%u outlen=%zu\n",
+ rc, resp.qpid, udata->outlen);
+ goto free_umem;
+ }
+ }
+
+ qp->ib_qp.qp_num = qp->sp_qp.id;
+ if (qp_init_attr->qp_type == IB_QPT_GSI)
+ rdev->gsi_ctx.gsi_qp = qp;
+ spin_lock_init(&qp->sq_lock);
+ spin_lock_init(&qp->rq_lock);
+ INIT_LIST_HEAD(&qp->list);
+ mutex_lock(&rdev->qp_lock);
+ list_add_tail(&qp->list, &rdev->qp_list);
+ mutex_unlock(&rdev->qp_lock);
+
+ return 0;
+free_umem:
+ bng_re_free_qp_res(&rdev->bng_res, &qp->sp_qp);
+ ib_umem_release(qp->rumem);
+ ib_umem_release(qp->sumem);
+fail:
+ return rc;
+}
+
+static unsigned long bng_re_lock_cqs(struct bng_re_qp *qp)
+ __acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&qp->scq->cq_lock, flags);
+ if (qp->rcq != qp->scq)
+ spin_lock(&qp->rcq->cq_lock);
+ else
+ __acquire(&qp->rcq->cq_lock);
+
+ return flags;
+}
+
+static void bng_re_unlock_cqs(struct bng_re_qp *qp, unsigned long flags)
+ __releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock)
+{
+ if (qp->rcq != qp->scq)
+ spin_unlock(&qp->rcq->cq_lock);
+ else
+ __release(&qp->rcq->cq_lock);
+ spin_unlock_irqrestore(&qp->scq->cq_lock, flags);
+}
+
+static u8 __from_ib_qp_state(enum ib_qp_state state)
+{
+ switch (state) {
+ case IB_QPS_RESET:
+ return CMDQ_MODIFY_QP_NEW_STATE_RESET;
+ case IB_QPS_INIT:
+ return CMDQ_MODIFY_QP_NEW_STATE_INIT;
+ case IB_QPS_RTR:
+ return CMDQ_MODIFY_QP_NEW_STATE_RTR;
+ case IB_QPS_RTS:
+ return CMDQ_MODIFY_QP_NEW_STATE_RTS;
+ case IB_QPS_SQD:
+ return CMDQ_MODIFY_QP_NEW_STATE_SQD;
+ case IB_QPS_SQE:
+ return CMDQ_MODIFY_QP_NEW_STATE_SQE;
+ case IB_QPS_ERR:
+ default:
+ return CMDQ_MODIFY_QP_NEW_STATE_ERR;
+ }
+}
+
+static int bng_re_copy_to_udata(struct bng_re_dev *rdev, void *data,
+ int len, struct ib_udata *udata)
+{
+ int rc;
+
+ rc = ib_copy_to_udata(udata, data, len);
+ if (rc)
+ dev_err(rdev_to_dev(rdev),
+ "ucontext copy failed from %ps rc %d",
+ __builtin_return_address(0), rc);
+
+ return rc;
+}
+
+static u8 __qp_access_flags_from_ib(struct bng_re_chip_ctx *cctx, int iflags)
+{
+ u8 qflags = 0;
+
+ if (iflags & IB_ACCESS_LOCAL_WRITE)
+ qflags |= CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE;
+ if (iflags & IB_ACCESS_REMOTE_WRITE)
+ qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE;
+ if (iflags & IB_ACCESS_REMOTE_READ)
+ qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ;
+ if (iflags & IB_ACCESS_REMOTE_ATOMIC)
+ qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC;
+
+ return qflags;
+}
+
+static void bng_re_update_qp_addr(struct bng_re_dev *rdev, struct bng_re_qp *qp)
+{
+ /* User-space can extract ip address with sgid_index. */
+ if (ipv6_addr_v4mapped((struct in6_addr *)&qp->sp_qp.ah.dgid)) {
+ qp->qp_info_entry.s_ip.ipv4_addr = ipv4_from_gid(qp->qp_info_entry.sgid.raw);
+ qp->qp_info_entry.d_ip.ipv4_addr = ipv4_from_gid(qp->sp_qp.ah.dgid.data);
+ } else {
+ memcpy(&qp->qp_info_entry.s_ip.ipv6_addr, qp->qp_info_entry.sgid.raw,
+ sizeof(qp->qp_info_entry.s_ip.ipv6_addr));
+ memcpy(&qp->qp_info_entry.d_ip.ipv6_addr, qp->sp_qp.ah.dgid.data,
+ sizeof(qp->qp_info_entry.d_ip.ipv6_addr));
+ }
+}
+
+static u16 get_source_port(struct bng_re_dev *rdev, struct bng_re_qp *qp)
+{
+ u8 ip_off, data[48], smac[ETH_ALEN];
+ u16 crc = 0, buf_len = 0, i;
+ u8 addr_len;
+ u32 qpn;
+
+ if (qp->sp_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6) {
+ addr_len = 16;
+ ip_off = 0;
+ } else {
+ addr_len = 4;
+ ip_off = 12;
+ }
+
+ memcpy(smac, qp->sp_qp.smac, ETH_ALEN);
+
+ memset(data, 0, 48);
+ memcpy(data, qp->sp_qp.ah.dmac, ETH_ALEN);
+ buf_len += ETH_ALEN;
+
+ memcpy(data + buf_len, smac, ETH_ALEN);
+ buf_len += ETH_ALEN;
+
+ memcpy(data + buf_len, qp->sp_qp.ah.dgid.data + ip_off, addr_len);
+ buf_len += addr_len;
+
+ memcpy(data + buf_len, qp->qp_info_entry.sgid.raw + ip_off, addr_len);
+ buf_len += addr_len;
+
+ qpn = htonl(qp->sp_qp.dest_qpn);
+ memcpy(data + buf_len, (u8 *)&qpn + 1, 3);
+ buf_len += 3;
+
+ for (i = 0; i < buf_len; i++)
+ crc = crc16(crc, (data + i), 1);
+
+ crc |= 0xc000;
+
+ return crc;
+}
+
+static void bng_re_update_qp_info(struct bng_re_dev *rdev, struct bng_re_qp *qp)
+{
+ u16 type;
+
+ type = __from_hw_to_ib_qp_type(qp->sp_qp.type);
+ bng_re_update_qp_addr(rdev, qp);
+
+ if ((type == IB_QPT_RC ||
+ (!BNG_RE_UDP_SP_WQE(qp->sp_qp.dev_cap_ext_flags2))) &&
+ (qp->sp_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 ||
+ qp->sp_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6)) {
+ qp->qp_info_entry.s_port = get_source_port(rdev, qp);
+ }
+ qp->qp_info_entry.d_port = BNG_RE_QP_DEST_PORT;
+}
+
+static int bng_re_update_ah_dscp_sl(struct bng_re_qp *qp, struct ib_qp_attr *qp_attr)
+{
+ struct bng_re_dev *rdev = qp->rdev;
+ bool dscp_valid = false;
+ bool pcp_valid = false;
+ u8 cos, dscp_pri = 0;
+ u8 dscp;
+ u8 i, j;
+ u8 pri;
+
+ /*
+ * The traffic class passed by the applications
+ * contains both dscp and ecn values. DSCP is
+ * upper six bits of the traffic class.
+ */
+
+ dscp = qp_attr->ah_attr.grh.traffic_class >> 2;
+ pri = rdma_ah_get_sl(&qp_attr->ah_attr);
+ qp->sp_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
+
+ if (!dscp && !pri)
+ goto default_queue;
+
+ if (dscp) {
+ for (i = 0; i < rdev->d2p_count; i++) {
+ if (rdev->d2p[i].dscp == dscp) {
+ dscp_pri = rdev->d2p[i].pri;
+ cos = rdev->p2cos[dscp_pri];
+ for (j = 0; j < rdev->lossless_q_count; j++) {
+ if (cos == rdev->lossless_qid[j]) {
+ dscp_valid = true;
+ break;
+ }
+ }
+ }
+ }
+
+ if (dscp_valid) {
+ qp->sp_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS;
+ qp->sp_qp.ah.traffic_class = dscp;
+ qp->sp_qp.ah.sl = dscp_pri;
+ return 0;
+ }
+ }
+ if (pri) {
+ cos = rdev->p2cos[pri];
+ for (j = 0; j < rdev->lossless_q_count; j++) {
+ if (cos == rdev->lossless_qid[j]) {
+ pcp_valid = true;
+ break;
+ }
+ }
+ if (pcp_valid) {
+ qp->sp_qp.ah.sl = qp_attr->ah_attr.sl;
+ qp->sp_qp.ah.traffic_class = 0;
+ return 0;
+ }
+ }
+ if (!dscp_valid && !pcp_valid) {
+ dev_warn_ratelimited(rdev_to_dev(qp->rdev),
+ "Given DSCP %d and/or SL %d not mapping to lossless queue",
+ dscp, pri);
+ dev_warn_ratelimited(rdev_to_dev(qp->rdev),
+ "Changing to default roce traffic class DSCP %d and SL %d",
+ rdev->tc_rec[0].roce_dscp, rdev->tc_rec[0].roce_prio);
+ }
+
+default_queue:
+ qp->sp_qp.ah.traffic_class = rdev->tc_rec[0].roce_dscp;
+ qp->sp_qp.ah.sl = rdev->tc_rec[0].roce_prio;
+ return 0;
+}
+
+static void bng_re_handle_cqn(struct bng_sp_cq *cq)
+{
+ struct bng_re_nq *nq;
+
+ if (!(cq && cq->nq))
+ return;
+
+ nq = cq->nq;
+ spin_lock_bh(&cq->compl_lock);
+ if (nq->cqn_handler) {
+ dev_dbg(&nq->res->pdev->dev, "%s:Trigger cq = %p event nq = %p\n",
+ __func__, cq, nq);
+ nq->cqn_handler(nq, cq);
+ }
+ spin_unlock_bh(&cq->compl_lock);
+}
+
+static void bng_re_manage_flush_qp(struct bng_re_qp *qp)
+{
+ struct bng_re_q *rq, *sq;
+ struct bng_re_dev *rdev;
+ unsigned long flags;
+
+ if (qp->sumem)
+ return;
+
+ rdev = qp->rdev;
+
+ if (qp->sp_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
+ rq = &qp->sp_qp.rq;
+ sq = &qp->sp_qp.sq;
+
+ dev_dbg(rdev_to_dev(rdev),
+ "Move QP = %p to flush list\n", qp);
+ flags = bng_re_lock_cqs(qp);
+ bng_re_add_flush_qp(&qp->sp_qp);
+ bng_re_unlock_cqs(qp, flags);
+
+ if (sq->hwq.prod != sq->hwq.cons)
+ bng_re_handle_cqn(&qp->scq->sp_cq);
+
+ if (qp->rcq && qp->rcq != qp->scq &&
+ rq->hwq.prod != rq->hwq.cons)
+ bng_re_handle_cqn(&qp->rcq->sp_cq);
+ }
+
+ if (qp->sp_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
+ dev_dbg(rdev_to_dev(rdev),
+ "Move QP = %p out of flush list\n", qp);
+ flags = bng_re_lock_cqs(qp);
+ bng_re_clean_qp(&qp->sp_qp);
+ bng_re_unlock_cqs(qp, flags);
+ }
+}
+
+static void bng_post_recv_db(struct bng_sp_qp *qp)
+{
+ struct bng_re_q *rq = &qp->rq;
+
+ if (unlikely(qp->cur_qp_state != CMDQ_MODIFY_QP_NEW_STATE_INIT))
+ bng_ring_prod_db(&rq->dbinfo, DBC_DBC_TYPE_RQ);
+}
+
+static bool bng_re_is_qp_valid(struct bng_re_res *res, struct bng_sp_qp *qp)
+{
+ bool is_valid = false;
+
+ if (qp) {
+ if (!res->is_vf) {
+ /*Chk if the QP ID is valid*/
+ is_valid = bng_re_xm_is_id_allocated(res->qp_xids, qp->id);
+ } else {
+ /*revisit for vf*/
+ is_valid = true;
+ }
+ }
+ return is_valid;
+}
+
+int bng_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
+ int qp_attr_mask, struct ib_udata *udata)
+{
+ struct bng_re_qp *qp = container_of(ib_qp, struct bng_re_qp, ib_qp);
+ enum ib_qp_state curr_qp_state, new_qp_state;
+ struct bng_re_modify_qp_ex_resp resp = {};
+ struct bng_re_modify_qp_ex_req ureq = {};
+ struct bng_re_dev *rdev = qp->rdev;
+ struct bng_re_dev_attr *dev_attr;
+ union ib_gid *gid_ptr = NULL;
+ int rc, entries, status;
+ bool update_fw = true;
+ unsigned long flags;
+ u8 nw_type;
+
+ dev_attr = rdev->dev_attr;
+
+ if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
+ return -EOPNOTSUPP;
+
+ if (!bng_re_init_fw_state_rtr_enabled(rdev->chip_ctx))
+ qp->sp_qp.modify_flags = 0;
+
+ qp->sp_qp.udcc_exclude = true;
+
+ if (udata && ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
+ dev_err(rdev_to_dev(rdev), "qp %#x udata copy failed",
+ qp->sp_qp.id);
+ return -EINVAL;
+ }
+
+ if (qp_attr_mask & IB_QP_STATE) {
+ curr_qp_state = bng_re_cmdq_new_state_to_ib(qp->sp_qp.cur_qp_state);
+ new_qp_state = qp_attr->qp_state;
+
+ if (bng_re_init_fw_state_rtr_enabled(rdev->chip_ctx)) {
+ /* don't update fw; as we want to hold
+ * the parameters until we move to rtr.
+ *
+ * Also- only reset the modify flags if we've at least
+ * sent the initial create; as we're accumulating flags until then
+ */
+ if (qp->sp_qp.rtr_transition_pending && new_qp_state != IB_QPS_RTR)
+ update_fw = false;
+ else if (test_bit(BNG_SP_QP_FLAG_MPC_INITIAL_CREATE_SENT, &qp->sp_qp.flags))
+ qp->sp_qp.modify_flags = 0;
+ }
+
+ if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state,
+ ib_qp->qp_type, qp_attr_mask)) {
+ ibdev_err(&rdev->ibdev,
+ "Invalid attribute mask: %#x specified ",
+ qp_attr_mask);
+ ibdev_err(&rdev->ibdev,
+ "for qpn: %#x type: %#x",
+ ib_qp->qp_num, ib_qp->qp_type);
+ ibdev_err(&rdev->ibdev,
+ "curr_qp_state=0x%x, new_qp_state=0x%x\n",
+ curr_qp_state, new_qp_state);
+ return -EINVAL;
+ }
+
+ if (!bng_re_init_fw_state_rtr_enabled(rdev->chip_ctx)) {
+ if (curr_qp_state == IB_QPS_RESET && new_qp_state == IB_QPS_RESET) {
+ if (!test_bit(BNG_SP_QP_FLAG_MPC_INITIAL_CREATE_SENT,
+ &qp->sp_qp.flags)) {
+ dev_warn(rdev_to_dev(rdev),
+ "%s -%s:%d,%s:0x%x,%s:0x%x,%s:0x%x,%s:0x%x,%s:0x%x\n",
+ "FirstModify Not Sent: NoStateChange",
+ "is_valid:",
+ bng_re_is_qp_valid(&rdev->bng_res, &qp->sp_qp),
+ "attribute mask:",
+ qp_attr_mask,
+ "qpn:",
+ ib_qp->qp_num,
+ "type:",
+ ib_qp->qp_type,
+ "current_qp_st",
+ curr_qp_state,
+ "new_qp_st",
+ new_qp_state);
+ return 0;
+ }
+ }
+ }
+
+ qp->sp_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
+ qp->sp_qp.state = __from_ib_qp_state(qp_attr->qp_state);
+
+ /* MTU settings allowed only during INIT -> RTR */
+ if (qp_attr->qp_state == IB_QPS_RTR &&
+ qp->sp_qp.type != CMDQ_CREATE_QP_TYPE_GSI &&
+ qp->sp_qp.type != CMDQ_CREATE_QP_TYPE_UD) {
+ rc = bng_re_init_qpmtu(qp, rdev->netdev->mtu, qp_attr_mask, qp_attr);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "qp %#x invalid mtu %d\n",
+ qp->sp_qp.id, ib_mtu_enum_to_int(qp_attr->path_mtu));
+ return rc;
+ }
+ }
+ /*
+ * if we're moving from init => rtr, then for non user and with posted
+ * rx buffers and not srq, set eq buffer coount
+ */
+ if (bng_re_init_fw_state_rtr_enabled(rdev->chip_ctx)) {
+ u32 indx;
+ struct bng_re_db_info *info;
+
+ if (!qp->sp_qp.is_user && !qp->sp_qp.srq &&
+ new_qp_state == IB_QPS_RTR && qp->sp_qp.rq.hwq.prod) {
+ spin_lock_irqsave(&qp->rq_lock, flags);
+
+ info = &qp->sp_qp.rq.dbinfo;
+ indx = (((info->hwq->prod / info->max_slot) & DBC_DBC_INDEX_MASK) |
+ ((info->flags & BNG_RE_FLAG_EPOCH_PROD_MASK) <<
+ BNG_RE_DB_EPOCH_PROD_SHIFT));
+
+ qp->sp_qp.req_buffer_count = indx;
+ spin_unlock_irqrestore(&qp->rq_lock, flags);
+ }
+ }
+ } else {
+ curr_qp_state = __to_ib_qp_state(qp->sp_qp.cur_qp_state);
+ new_qp_state = qp_attr->qp_state;
+
+ if (bng_re_init_fw_state_rtr_enabled(rdev->chip_ctx)) {
+ /* don't update fw; as we want to hold
+ * the parameters until we move to rtr.
+ * Also- only reset the modify flags if we've at least
+ * sent the initial create; as we're accumulating flags until then
+ */
+ if (qp->sp_qp.rtr_transition_pending && new_qp_state != IB_QPS_RTR)
+ update_fw = false;
+ else if (test_bit(BNG_SP_QP_FLAG_MPC_INITIAL_CREATE_SENT, &qp->sp_qp.flags))
+ qp->sp_qp.modify_flags = 0;
+ } else {
+ /* Fix me: we can't check new qp state if the
+ * state modify bit is not set.
+ */
+ /* Check First modify is sent */
+ if ((!test_bit(BNG_SP_QP_FLAG_MPC_INITIAL_CREATE_SENT, &qp->sp_qp.flags))) {
+ dev_warn(rdev_to_dev(rdev),
+ "%s -%s:%d,%s:0x%x,%s:0x%x,%s:0x%x,%s:0x%x,%s:0x%x\n",
+ "FirstModify Not Sent: NoStateChange",
+ "is_valid:",
+ bng_re_is_qp_valid(&rdev->bng_res, &qp->sp_qp),
+ "attribute mask:",
+ qp_attr_mask,
+ "qpn:",
+ ib_qp->qp_num,
+ "type:",
+ ib_qp->qp_type,
+ "current_qp_st",
+ curr_qp_state,
+ "new_qp_st",
+ new_qp_state);
+ return 0;
+ }
+ }
+ }
+
+ if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
+ qp->sp_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY;
+ qp->sp_qp.en_sqd_async_notify = true;
+ }
+ if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
+ qp->sp_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
+ qp->sp_qp.access =
+ __qp_access_flags_from_ib(qp->sp_qp.cctx,
+ qp_attr->qp_access_flags);
+ /* LOCAL_WRITE access must be set to allow RC receive */
+ qp->sp_qp.access |= CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE;
+ }
+ if (qp_attr_mask & IB_QP_PKEY_INDEX) {
+ qp->sp_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
+ qp->sp_qp.pkey_index = qp_attr->pkey_index;
+ }
+ if (qp_attr_mask & IB_QP_QKEY) {
+ qp->sp_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
+ qp->sp_qp.qkey = qp_attr->qkey;
+ }
+ if (qp_attr_mask & IB_QP_AV) {
+ const struct ib_global_route *grh =
+ rdma_ah_read_grh(&qp_attr->ah_attr);
+ const struct ib_gid_attr *sgid_attr;
+ struct bng_re_gid_ctx *ctx;
+
+ qp->sp_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
+ CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
+ CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
+ CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
+ CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
+ CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
+ CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
+ memcpy(qp->sp_qp.ah.dgid.data, qp_attr->ah_attr.grh.dgid.raw,
+ sizeof(qp->sp_qp.ah.dgid.data));
+
+ qp->sp_qp.ah.flow_label = grh->flow_label;
+
+ sgid_attr = qp_attr->ah_attr.grh.sgid_attr;
+
+ /* Get the HW context of the GID. The reference
+ * of GID table entry is already taken by the caller.
+ */
+ ctx = rdma_read_gid_hw_context(sgid_attr);
+ qp->sp_qp.ah.sgid_index = ctx->idx;
+ qp->sp_qp.ah.host_sgid_index = grh->sgid_index;
+ qp->sp_qp.ah.hop_limit = grh->hop_limit;
+ qp->sp_qp.ah.traffic_class = grh->traffic_class >> 2;
+
+ status = bng_re_update_ah_dscp_sl(qp, qp_attr);
+ if (status) {
+ dev_err(rdev_to_dev(rdev), "%s: qp %d: error updating ah_dscp",
+ __func__, qp->sp_qp.id);
+ return status;
+ }
+
+ ether_addr_copy(qp->sp_qp.ah.dmac,
+ qp_attr->ah_attr.roce.dmac);
+
+ gid_ptr = (union ib_gid *)&sgid_attr->gid;
+
+ if (!sgid_attr->ndev) {
+ dev_err(rdev_to_dev(rdev), "%s: qp %d: sgid_attr has no netdev",
+ __func__, qp->sp_qp.id);
+ return -ENODEV;
+ }
+
+ memcpy(qp->sp_qp.smac, sgid_attr->ndev->dev_addr, ETH_ALEN);
+
+ nw_type = rdma_gid_attr_network_type(sgid_attr);
+ switch (nw_type) {
+ case RDMA_NETWORK_IPV4:
+ qp->sp_qp.nw_type =
+ CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4;
+ break;
+ case RDMA_NETWORK_IPV6:
+ qp->sp_qp.nw_type =
+ CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6;
+ break;
+ default:
+ qp->sp_qp.nw_type =
+ CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1;
+ break;
+ }
+ memcpy(&qp->qp_info_entry.sgid, gid_ptr, sizeof(qp->qp_info_entry.sgid));
+ qp->sp_qp.udcc_exclude = true;
+ }
+
+ if (qp_attr_mask & IB_QP_TIMEOUT) {
+ qp->sp_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT;
+ qp->sp_qp.timeout = qp_attr->timeout;
+ }
+ if (qp_attr_mask & IB_QP_RETRY_CNT) {
+ qp->sp_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT;
+ qp->sp_qp.retry_cnt = qp_attr->retry_cnt;
+ }
+ if (qp_attr_mask & IB_QP_RNR_RETRY) {
+ qp->sp_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY;
+ qp->sp_qp.rnr_retry = qp_attr->rnr_retry;
+ }
+ if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) {
+ qp->sp_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
+ qp->sp_qp.min_rnr_timer = qp_attr->min_rnr_timer;
+ }
+ if (qp_attr_mask & IB_QP_RQ_PSN) {
+ qp->sp_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN;
+ qp->sp_qp.rq.psn = qp_attr->rq_psn;
+ }
+ if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
+ qp->sp_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC;
+ /* Cap the max_rd_atomic to device max */
+ qp->sp_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic,
+ dev_attr->max_qp_rd_atom);
+ }
+ if (qp_attr_mask & IB_QP_SQ_PSN) {
+ qp->sp_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
+ qp->sp_qp.sq.psn = qp_attr->sq_psn;
+ }
+ if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
+ if (qp_attr->max_dest_rd_atomic >
+ dev_attr->max_qp_init_rd_atom) {
+ ibdev_err(&rdev->ibdev,
+ "max_dest_rd_atomic requested%d is > dev_max%d",
+ qp_attr->max_dest_rd_atomic,
+ dev_attr->max_qp_init_rd_atom);
+ return -EINVAL;
+ }
+
+ qp->sp_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC;
+ qp->sp_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
+ }
+ if (qp_attr_mask & IB_QP_CAP) {
+ struct bng_re_ucontext *uctx =
+ rdma_udata_to_drv_context(udata, struct bng_re_ucontext, ib_uctx);
+
+ qp->sp_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
+ CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
+ CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE |
+ CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE |
+ CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA;
+ if (qp_attr->cap.max_send_wr >= dev_attr->max_sq_wqes ||
+ qp_attr->cap.max_recv_wr >= dev_attr->max_rq_wqes ||
+ qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges ||
+ qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges ||
+ qp_attr->cap.max_inline_data >=
+ dev_attr->max_inline_data) {
+ ibdev_err(&rdev->ibdev,
+ "Create QP failed - max exceeded");
+ return -EINVAL;
+ }
+ entries = bng_re_init_depth(qp_attr->cap.max_send_wr, uctx);
+
+ if (entries > dev_attr->max_sq_wqes)
+ entries = dev_attr->max_sq_wqes;
+ entries = min_t(u32, entries, dev_attr->max_sq_wqes);
+ qp->sp_qp.sq.max_wqe = entries;
+ qp->sp_qp.sq.q_full_delta = qp->sp_qp.sq.max_wqe - qp_attr->cap.max_send_wr;
+
+ /*
+ * Reserving one slot for Phantom WQE. Some application can
+ * post one extra entry in this case. Allowing this to avoid
+ * unexpected Queue full condition
+ */
+ qp->sp_qp.sq.q_full_delta -= 1;
+ qp->sp_qp.sq.max_sge = qp_attr->cap.max_send_sge;
+ if (qp->sp_qp.rq.max_wqe) {
+ entries = bng_re_init_depth(qp_attr->cap.max_recv_wr, uctx);
+ if (entries > dev_attr->max_rq_wqes)
+ entries = dev_attr->max_rq_wqes;
+ qp->sp_qp.rq.max_wqe = entries;
+ qp->sp_qp.rq.q_full_delta = qp->sp_qp.rq.max_wqe - qp_attr->cap.max_recv_wr;
+ qp->sp_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
+ } else {
+ /* SRQ was used prior, just ignore the RQ caps */
+ }
+ }
+
+ if (qp_attr_mask & IB_QP_DEST_QPN) {
+ qp->sp_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID;
+ qp->sp_qp.dest_qpn = qp_attr->dest_qp_num;
+ }
+
+ if (udata && (ureq.comp_mask & BNG_RE_COMP_MASK_REQ_BUFF_CNT)) {
+ qp->sp_qp.req_buffer_count = ureq.req_buffer_count;
+ } else {
+ dev_warn(rdev_to_dev(rdev),
+ "QP %d : no req_buffer_count set in user data",
+ qp->sp_qp.id);
+ }
+
+ if (update_fw) {
+ rc = bng_sp_mpc_modify_qp(&rdev->bng_res, &qp->sp_qp);
+ } else {
+ /* Transition the state ourselves if we're skipping the hw update */
+ if (qp_attr_mask & IB_QP_STATE)
+ qp->sp_qp.cur_qp_state = __from_ib_qp_state(new_qp_state);
+ }
+
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Failed to modify HW QP");
+ return rc;
+ }
+
+ if (qp_attr_mask & IB_QP_STATE) {
+ /*
+ * When the QP moves to INIT to RTR in the modify_qp
+ * call, firmware has a workaround to update the context
+ * field with CDUDMA read/write. During the QP INIT
+ * state in the post_receive driver needs to ensure
+ * no doorbell is rung to work this properly. And once
+ * QP moves to RTR ring the doorbell if the producer
+ * index is present.
+ */
+ if (qp_attr->qp_state == IB_QPS_RTR && !qp->sp_qp.srq) {
+ if (qp->sp_qp.rq.hwq.prod) {
+ spin_lock_irqsave(&qp->rq_lock, flags);
+ bng_post_recv_db(&qp->sp_qp);
+ spin_unlock_irqrestore(&qp->rq_lock, flags);
+ }
+ }
+
+ if (bng_re_init_fw_state_rtr_enabled(rdev->chip_ctx)) {
+ /* Turn off rtr pending after we've transitioned to rtr; or,
+ * ; when back to reset.
+ */
+ if (qp_attr->qp_state == IB_QPS_RTR)
+ qp->sp_qp.rtr_transition_pending = false;
+ else if (qp_attr->qp_state == IB_QPS_RESET)
+ qp->sp_qp.rtr_transition_pending = true;
+ }
+
+ bng_re_manage_flush_qp(qp);
+ }
+
+ /*
+ * Update info when qp_info_info
+ */
+ bng_re_update_qp_info(rdev, qp);
+
+ if (udata) {
+ rc = bng_re_copy_to_udata(rdev, &resp,
+ min(udata->outlen, sizeof(resp)),
+ udata);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "%s: qp:%d error copying user data:%d\n",
+ __func__, qp->sp_qp.id, rc);
+ return rc;
+ }
+ }
+
+ return rc;
+}
+
+/**
+ * qp_destroy_qp_mpc - destroy a QP w/mpc / rca
+ *
+ * @res: resource info
+ * @qp: qp to destroy
+ *
+ * Return: 0 for success, anything else; error
+ */
+static int qp_destroy_qp_mpc(struct bng_re_res *res, struct bng_re_qp *qp)
+{
+ int rc = 0;
+ bool xid_cleanup_handled = false;
+
+ /* Reset modify flags during QP destroy and set only the required mask */
+ qp->sp_qp.modify_flags = 0;
+ qp->sp_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
+
+ /* Set state to RESET to take QP back to first state */
+ qp->sp_qp.state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
+
+ if (test_bit(BNG_SP_QP_FLAG_MPC_ALLOW_QP_DESTROY, &qp->sp_qp.flags)) {
+ xid_cleanup_handled = false;
+ /* Set the QP Free flag for actual QP destroy */
+ set_bit(BNG_SP_QP_FLAG_MPC_QP_FREE, &qp->sp_qp.flags);
+ rc = bng_sp_fill_and_send_mpc_qp_modify(res, &qp, 1, &xid_cleanup_handled);
+ } else {
+ dev_warn(&res->pdev->dev,
+ "MPC-skip destroy qp- not yet created to FW. id:%d.State:%d,%d\n",
+ qp->sp_qp.id,
+ qp->sp_qp.cur_qp_state,
+ qp->sp_qp.state);
+ }
+
+ if (!xid_cleanup_handled && qp->sp_qp.id != 1 && !res->is_vf &&
+ res->qp_xids) {
+ dev_warn(&res->pdev->dev,
+ "MPC- pathological xid cleanup. id:%d.State:%d,%d, destroy_flag set:%d\n",
+ qp->sp_qp.id,
+ qp->sp_qp.cur_qp_state,
+ qp->sp_qp.state,
+ test_bit(BNG_SP_QP_FLAG_MPC_ALLOW_QP_DESTROY, &qp->sp_qp.flags));
+
+ bng_re_qp_free_xid_and_iqm(res, qp->sp_qp.type, (int)qp->sp_qp.id,
+ &qp->iqm_res, true);
+ }
+
+ return rc;
+}
+
+int bng_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
+{
+ struct bng_re_qp *qp = container_of(ib_qp, struct bng_re_qp, ib_qp);
+ struct bng_sp_qp *sp_qp = &qp->sp_qp;
+ struct bng_re_dev *rdev = qp->rdev;
+ unsigned int flags;
+ int rc;
+
+ bng_re_flush_cqn_wq(&qp->sp_qp);
+
+ rc = qp_destroy_qp_mpc(&rdev->bng_res, qp);
+ if (rc)
+ ibdev_err(&rdev->ibdev, "Failed to destroy HW QP");
+
+ if (rdma_is_kernel_res(&qp->ib_qp.res)) {
+ flags = bng_re_lock_cqs(qp);
+ bng_re_clean_qp(&qp->sp_qp);
+ bng_re_unlock_cqs(qp, flags);
+ }
+
+ bng_re_free_qp_res(&rdev->bng_res, &qp->sp_qp);
+
+ mutex_lock(&rdev->qp_lock);
+ list_del(&qp->list);
+ mutex_unlock(&rdev->qp_lock);
+
+ ib_umem_release(qp->rumem);
+ ib_umem_release(qp->sumem);
+
+ /* Same as bng_re_flush_cqn_wq: tolerate partial QP / missing CQ-NQ links. */
+ if (sp_qp->scq && sp_qp->scq->nq && sp_qp->scq->nq->cqn_wq)
+ flush_workqueue(sp_qp->scq->nq->cqn_wq);
+ if (sp_qp->rcq && sp_qp->scq != sp_qp->rcq &&
+ sp_qp->rcq->nq && sp_qp->rcq->nq->cqn_wq)
+ flush_workqueue(sp_qp->rcq->nq->cqn_wq);
+
+ return 0;
+}
diff --git a/drivers/infiniband/hw/bng_re/bng_verbs.h b/drivers/infiniband/hw/bng_re/bng_verbs.h
index c5643ee70885..f89db8d33ded 100644
--- a/drivers/infiniband/hw/bng_re/bng_verbs.h
+++ b/drivers/infiniband/hw/bng_re/bng_verbs.h
@@ -6,10 +6,26 @@
#include <linux/refcount.h>
#include <rdma/ib_verbs.h>
+#include <rdma/ib_addr.h>
+#include <rdma/bng_re-abi.h>
#include "bng_sp.h"
#include "bng_re.h"
+#define BNG_RE_RESERVED_QP_WRS 128
+#define BNG_RE_XID_AVOID_REUSE false
+#define BNG_RE_MIN_KERNEL_QP_TX_DEPTH 4096
+
+/* GSI QP mode enum */
+enum bng_re_gsi_mode {
+ BNG_RE_GSI_MODE_INVALID = 0,
+ BNG_RE_GSI_MODE_ALL = 1,
+ BNG_RE_GSI_MODE_ROCE_V1,
+ BNG_RE_GSI_MODE_ROCE_V2_IPV4,
+ BNG_RE_GSI_MODE_ROCE_V2_IPV6,
+ BNG_RE_GSI_MODE_UD
+};
+
struct bng_re_ucontext {
struct ib_ucontext ib_uctx;
struct bng_re_dev *rdev;
@@ -94,6 +110,48 @@ struct bng_re_ah {
refcount_t *ref_cnt;
};
+union ip_addr {
+ u32 ipv4_addr;
+ u8 ipv6_addr[16];
+};
+
+struct bng_re_iqm_res {
+ u32 irrq_addr;
+ u32 irrq_size;
+ u32 orrq_addr;
+ u32 orrq_size;
+ u32 msn_addr;
+ u32 msn_size;
+};
+
+struct bng_re_qp_info_entry {
+ union ib_gid sgid;
+ union ib_gid dgid;
+ union ip_addr s_ip;
+ union ip_addr d_ip;
+ u16 s_port;
+#define BNG_RE_QP_DEST_PORT 4791
+ u16 d_port;
+ u32 rate_limit;
+};
+
+struct bng_re_qp {
+ struct ib_qp ib_qp;
+ struct list_head list;
+ struct bng_re_dev *rdev;
+ spinlock_t sq_lock;
+ spinlock_t rq_lock;
+ struct bng_sp_qp sp_qp;
+ struct bng_re_iqm_res iqm_res;
+ struct ib_umem *sumem;
+ struct ib_umem *rumem;
+ u32 send_psn;
+ struct bng_re_cq *scq;
+ struct bng_re_cq *rcq;
+ struct dentry *dentry;
+ struct bng_re_qp_info_entry qp_info_entry;
+};
+
static inline struct bng_re_ah *to_bng_re_ah(struct ib_ah *ibah)
{
return container_of(ibah, struct bng_re_ah, ib_ah);
@@ -112,6 +170,139 @@ static inline u16 bng_re_get_rwqe_size(int nsge)
return 16 + (nsge * 16); /* header=16, each sge=16 bytes */
}
+static inline u16 __get_swqe_size(int nsge)
+{
+ u16 wqe_size;
+
+ wqe_size = sizeof(struct sq_atomic_hdr) + nsge * sizeof(struct sq_sge);
+ return wqe_size;
+}
+
+static inline u32 bng_re_get_diff(struct bng_re_ucontext *uctx,
+ struct bng_re_chip_ctx *cctx)
+{
+ if (!uctx)
+ return 0;
+ else if (uctx->cmask & BNG_RE_UCNTX_CMASK_RSVD_WQE_DISABLED)
+ return 0;
+
+ /* old lib */
+ return BNG_RE_RESERVED_QP_WRS;
+}
+
+static inline enum ib_qp_state __to_ib_qp_state(u8 state)
+{
+ switch (state) {
+ case CMDQ_MODIFY_QP_NEW_STATE_RESET:
+ return IB_QPS_RESET;
+ case CMDQ_MODIFY_QP_NEW_STATE_INIT:
+ return IB_QPS_INIT;
+ case CMDQ_MODIFY_QP_NEW_STATE_RTR:
+ return IB_QPS_RTR;
+ case CMDQ_MODIFY_QP_NEW_STATE_RTS:
+ return IB_QPS_RTS;
+ case CMDQ_MODIFY_QP_NEW_STATE_SQD:
+ return IB_QPS_SQD;
+ case CMDQ_MODIFY_QP_NEW_STATE_SQE:
+ return IB_QPS_SQE;
+ case CMDQ_MODIFY_QP_NEW_STATE_ERR:
+ default:
+ return IB_QPS_ERR;
+ }
+}
+
+static inline enum ib_mtu __to_ib_mtu(u32 mtu)
+{
+ switch (mtu) {
+ case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
+ return IB_MTU_256;
+ case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
+ return IB_MTU_512;
+ case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
+ return IB_MTU_1024;
+ case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
+ return IB_MTU_2048;
+ case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
+ return IB_MTU_4096;
+ default:
+ return IB_MTU_1024;
+ }
+}
+
+static inline u8 __to_ib_port_num(u16 port_id)
+{
+ /* Typically Broadcom driver has a 1-based port number mapped to port_id 0 */
+ return (u8)(port_id + 1);
+}
+
+static u32 __from_ib_mtu(enum ib_mtu mtu)
+{
+ switch (mtu) {
+ case IB_MTU_256:
+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
+ case IB_MTU_512:
+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
+ case IB_MTU_1024:
+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
+ case IB_MTU_2048:
+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
+ case IB_MTU_4096:
+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
+ default:
+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
+ }
+}
+
+static inline enum ib_qp_type __from_hw_to_ib_qp_type(u8 type)
+{
+ switch (type) {
+ case CMDQ_CREATE_QP1_TYPE_GSI:
+ case CMDQ_CREATE_QP_TYPE_GSI:
+ return IB_QPT_GSI;
+ case CMDQ_CREATE_QP_TYPE_RC:
+ return IB_QPT_RC;
+ case CMDQ_CREATE_QP_TYPE_UD:
+ return IB_QPT_UD;
+ case CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE:
+ return IB_QPT_RAW_ETHERTYPE;
+ default:
+ return IB_QPT_MAX;
+ }
+}
+
+/* Extract IPv4 from last 4 bytes of IPv4-mapped GID (bytes 12-15, host order). */
+static inline u32 ipv4_from_gid(u8 *gid)
+{
+ return (u32)(gid[15] << 24 | gid[14] << 16 | gid[13] << 8 | gid[12]);
+}
+
+static inline int bng_re_init_qpmtu(struct bng_re_qp *qp, int mtu,
+ int mask, struct ib_qp_attr *qp_attr)
+{
+ int qpmtu, qpmtu_int;
+ int ifmtu, ifmtu_int;
+
+ ifmtu = iboe_get_mtu(mtu);
+ ifmtu_int = ib_mtu_enum_to_int(ifmtu);
+ qpmtu = ifmtu;
+ qpmtu_int = ifmtu_int;
+ if (mask & IB_QP_PATH_MTU) {
+ qpmtu = qp_attr->path_mtu;
+ qpmtu_int = ib_mtu_enum_to_int(qpmtu);
+ if (qpmtu_int > ifmtu_int)
+ return -EINVAL;
+ }
+ qp->sp_qp.path_mtu = __from_ib_mtu(qpmtu);
+ qp->sp_qp.mtu = qpmtu_int;
+ qp->sp_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
+
+ return 0;
+}
+
+#define to_bng_re(ptr, type, member) \
+ container_of(ptr, type, member)
+
int bng_re_query_device(struct ib_device *ibdev, struct ib_device_attr *ib_attr,
struct ib_udata *udata);
int bng_re_modify_device(struct ib_device *ibdev, int device_modify_mask,
@@ -176,6 +367,13 @@ int bng_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,
int bng_re_destroy_ah(struct ib_ah *ib_ah, u32 flags);
int bng_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr);
void bng_re_posted_destroy_ah(struct work_struct *work);
+int bng_re_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
+ int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
+int bng_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
+ struct ib_udata *udata);
+int bng_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
+ int qp_attr_mask, struct ib_udata *udata);
+int bng_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata);
#endif /* __BNG_RE_VERBS_H__ */
diff --git a/include/uapi/rdma/bng_re-abi.h b/include/uapi/rdma/bng_re-abi.h
index 36bc902600be..7cee818f06d1 100644
--- a/include/uapi/rdma/bng_re-abi.h
+++ b/include/uapi/rdma/bng_re-abi.h
@@ -25,6 +25,14 @@ enum {
BNG_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE = 0x02,
};
+enum {
+ BNG_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK = 0x1,
+ BNG_RE_COMP_MASK_MQP_EX_PPP_REQ_EN = 0x1,
+ BNG_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK = 0x2,
+ BNG_RE_COMP_MASK_MQP_EX_OOO_DP_EN_MASK = 0x20,
+ BNG_RE_COMP_MASK_REQ_BUFF_CNT = 0x400UL,
+};
+
struct bng_re_uctx_req {
__aligned_u64 comp_mask;
};
@@ -80,4 +88,31 @@ struct bng_re_ah_resp {
__u64 comp_mask;
} __attribute__((packed));
+struct bng_re_qp_req {
+ __aligned_u64 qpsva;
+ __aligned_u64 qprva;
+ __aligned_u64 qp_handle;
+ __aligned_u64 comp_mask;
+};
+
+struct bng_re_qp_resp {
+ __u32 qpid;
+ __u32 rsvd;
+};
+
+struct bng_re_modify_qp_ex_req {
+ __aligned_u64 comp_mask;
+ __u64 path_ctx_id;
+ __u32 dpi;
+ __u8 no_grp;
+ __u8 rsvd[3];
+ __u32 req_buffer_count;
+} __packed;
+
+struct bng_re_modify_qp_ex_resp {
+ __aligned_u64 comp_mask;
+ __u32 ppp_st_idx;
+ __u32 path_mtu;
+} __packed;
+
#endif /* __BNG_RE_UVERBS_ABI_H__*/
--
2.43.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 15/15] RDMA/bng_re: Register with ib-core
2026-09-04 10:43 [PATCH 00/15] Add BNG_RE control path verbs Siva Reddy Kallam
` (13 preceding siblings ...)
2026-09-04 10:43 ` [PATCH 14/15] RDMA/bng_re: Add QP verbs Siva Reddy Kallam
@ 2026-09-04 10:43 ` Siva Reddy Kallam
14 siblings, 0 replies; 17+ messages in thread
From: Siva Reddy Kallam @ 2026-09-04 10:43 UTC (permalink / raw)
To: leonro, jgg, davem, edumazet, kuba, pabeni, andrew+netdev, horms
Cc: netdev, linux-kernel, linux-rdma, Siva Reddy Kallam
This patch registers with ib-core
Signed-off-by: Siva Reddy Kallam <siva.kallam@broadcom.com>
---
drivers/infiniband/hw/bng_re/bng_dev.c | 79 ++++++++++++++++++++++++++
1 file changed, 79 insertions(+)
diff --git a/drivers/infiniband/hw/bng_re/bng_dev.c b/drivers/infiniband/hw/bng_re/bng_dev.c
index 6e9df737c751..2633bb289f09 100644
--- a/drivers/infiniband/hw/bng_re/bng_dev.c
+++ b/drivers/infiniband/hw/bng_re/bng_dev.c
@@ -5,6 +5,8 @@
#include <linux/pci.h>
#include <linux/auxiliary_bus.h>
+#include <net/addrconf.h>
+
#include <rdma/ib_verbs.h>
#include <rdma/bng_re-abi.h>
@@ -553,6 +555,73 @@ static int bng_re_dev_init(struct bng_re_dev *rdev)
return rc;
}
+static void bng_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp,
+ u8 port_num, enum ib_event_type event)
+{
+ struct ib_event ib_event;
+
+ ib_event.device = ibdev;
+ if (qp) {
+ ib_event.element.qp = qp;
+ ib_event.event = event;
+ if (qp->event_handler)
+ qp->event_handler(&ib_event, qp->qp_context);
+
+ } else {
+ ib_event.element.port_num = port_num;
+ ib_event.event = event;
+ ib_dispatch_event(&ib_event);
+ }
+}
+
+static int bng_re_register_ib(struct bng_re_dev *rdev)
+{
+ struct ib_device *ibdev = &rdev->ibdev;
+ int ret;
+
+ /* ib device init */
+ ibdev->node_type = RDMA_NODE_IB_CA;
+ strscpy(ibdev->node_desc, BNG_RE_DESC " HCA");
+ ibdev->phys_port_cnt = 1;
+
+ addrconf_addr_eui48((u8 *)&ibdev->node_guid, rdev->netdev->dev_addr);
+
+ ibdev->num_comp_vectors = rdev->nqr->num_msix - BNG_RE_MAX_RSVD_IRQ;
+ ibdev->dev.parent = &rdev->aux_dev->pdev->dev;
+ ibdev->local_dma_lkey = BNG_RE_RSVD_LKEY;
+
+ ib_set_device_ops(ibdev, &bng_re_dev_ops);
+ ret = ib_device_set_netdev(&rdev->ibdev, rdev->netdev, 1);
+ if (ret)
+ return ret;
+
+ dma_set_max_seg_size(&rdev->aux_dev->pdev->dev, UINT_MAX);
+ ibdev->uverbs_cmd_mask |= BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ);
+ return ib_register_device(ibdev, "bng_re%d", &rdev->aux_dev->pdev->dev);
+}
+
+static int bng_re_ib_init(struct bng_re_dev *rdev)
+{
+ int rc;
+ u32 event;
+
+ /* Register ib dev */
+ rc = bng_re_register_ib(rdev);
+ if (rc) {
+ pr_err("Failed to register with IB: %#x\n", rc);
+ return rc;
+ }
+ dev_info(rdev_to_dev(rdev), "Device registered with IB successfully");
+ set_bit(BNG_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags);
+
+ event = netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev) ?
+ IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
+
+ bng_re_dispatch_event(&rdev->ibdev, NULL, 1, event);
+
+ return rc;
+}
+
static int bng_re_add_device(struct auxiliary_device *adev)
{
struct bnge_auxr_priv *auxr_priv =
@@ -575,8 +644,17 @@ static int bng_re_add_device(struct auxiliary_device *adev)
if (rc)
goto re_dev_dealloc;
+ rc = bng_re_ib_init(rdev);
+ if (rc) {
+ pr_err("Failed to register with IB: %s",
+ auxr_priv->aux_dev.name);
+ goto re_dev_uninit;
+ }
+
return 0;
+re_dev_uninit:
+ bng_re_dev_uninit(rdev);
re_dev_dealloc:
ib_dealloc_device(&rdev->ibdev);
exit:
@@ -586,6 +664,7 @@ static int bng_re_add_device(struct auxiliary_device *adev)
static void bng_re_remove_device(struct bng_re_dev *rdev,
struct auxiliary_device *aux_dev)
{
+ ib_unregister_device(&rdev->ibdev);
bng_re_dev_uninit(rdev);
ib_dealloc_device(&rdev->ibdev);
}
--
2.43.5
^ permalink raw reply related [flat|nested] 17+ messages in thread