* [PATCH rdma-next v5 01/10] RDMA/mana_ib: Allocate and map fast-registration MRs
2026-10-17 6:44 [PATCH rdma-next v5 00/10] RDMA/mana_ib: Add kernel RC and fast registration support Konstantin Taranov
@ 2026-10-17 6:44 ` Konstantin Taranov
2026-10-17 6:49 ` sashiko-bot
2026-10-17 6:44 ` [PATCH rdma-next v5 02/10] RDMA/mana: Create and destroy kernel RC QPs Konstantin Taranov
` (8 subsequent siblings)
9 siblings, 1 reply; 21+ messages in thread
From: Konstantin Taranov @ 2026-10-17 6:44 UTC (permalink / raw)
To: kotaranov, snsanghvi, longli, jgg, leon; +Cc: linux-rdma, linux-kernel
From: Konstantin Taranov <kotaranov@microsoft.com>
Implement alloc_mr and map_mr_sg for memory-registration MRs.
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
---
v5:
- Reject max_num_sg above MANA_MAX_FMR_PAGES
v4:
- Use an explicit 64-byte FMR MST alignment
drivers/infiniband/hw/mana/device.c | 2 +
drivers/infiniband/hw/mana/main.c | 3 +-
drivers/infiniband/hw/mana/mana_ib.h | 48 +++++++++++++++
drivers/infiniband/hw/mana/mr.c | 87 +++++++++++++++++++++++++++-
include/net/mana/gdma.h | 15 +++++
5 files changed, 153 insertions(+), 2 deletions(-)
diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c
index 62c7803ca6c6..cdaada7f6e8a 100644
--- a/drivers/infiniband/hw/mana/device.c
+++ b/drivers/infiniband/hw/mana/device.c
@@ -19,6 +19,7 @@ static const struct ib_device_ops mana_ib_dev_ops = {
.add_gid = mana_ib_gd_add_gid,
.alloc_mw = mana_ib_alloc_mw,
+ .alloc_mr = mana_ib_alloc_mr,
.alloc_pd = mana_ib_alloc_pd,
.alloc_ucontext = mana_ib_alloc_ucontext,
.create_ah = mana_ib_create_ah,
@@ -40,6 +41,7 @@ static const struct ib_device_ops mana_ib_dev_ops = {
.get_dma_mr = mana_ib_get_dma_mr,
.get_link_layer = mana_ib_get_link_layer,
.get_port_immutable = mana_ib_get_port_immutable,
+ .map_mr_sg = mana_ib_map_mr_sg,
.mmap = mana_ib_mmap,
.modify_qp = mana_ib_modify_qp,
.modify_wq = mana_ib_modify_wq,
diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c
index 672bd3f650dd..ceb4ec58023a 100644
--- a/drivers/infiniband/hw/mana/main.c
+++ b/drivers/infiniband/hw/mana/main.c
@@ -133,7 +133,7 @@ int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
sizeof(resp));
if (!udata)
- flags |= GDMA_PD_FLAG_ALLOW_GPA_MR;
+ flags |= GDMA_PD_FLAG_ALLOW_GPA_MR | GDMA_PD_FLAG_ALLOW_FMR_MR;
req.flags = flags;
err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
@@ -647,6 +647,7 @@ int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
props->max_ah = INT_MAX;
props->max_pkeys = 1;
props->local_ca_ack_delay = MANA_CA_ACK_DELAY;
+ props->max_fast_reg_page_list_len = MANA_MAX_FMR_PAGES;
if (!mana_ib_is_rnic(dev))
props->raw_packet_caps = IB_RAW_PACKET_CAP_IP_CSUM;
diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index 74edb6e707a1..5ca6e2acf974 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -39,6 +39,8 @@
*/
#define MANA_IB_MAX_MR 0xFFFFFFu
+#define MANA_MAX_FMR_PAGES 504
+
/*
* The CA timeout is approx. 260ms (4us * 2^(DELAY))
*/
@@ -51,6 +53,8 @@
#define MANA_GSI_QPN (1)
+#define MANA_PTE_VALID_MASK 0x1ULL
+
struct mana_ib_adapter_caps {
u32 max_sq_id;
u32 max_rq_id;
@@ -150,10 +154,46 @@ struct mana_ib_mw {
mana_handle_t mw_handle;
};
+#define MANA_FMR_MST_ALIGNMENT 64
+
+struct mana_fmr_mst_entry {
+ u32 reserved1;
+ u32 reserved2 : 8;
+ u32 page_size : 5;
+ u32 reserved3 : 3;
+ u32 access_right : 8;
+ u32 addr_type : 2;
+ u32 reserved4 : 6;
+ u32 reserved5[2];
+ u64 start_va;
+ u64 length;
+ u32 reserved6[8];
+}; /* HW DATA */
+
+static inline u32 mana_ib_fmr_header_offset(u32 num_ptes)
+{
+ /* The header follows the PTEs with the required MST alignment. */
+ return ALIGN(sizeof(u64) * num_ptes, MANA_FMR_MST_ALIGNMENT);
+}
+
+static inline u32 mana_ib_fmr_buf_sz(u32 num_ptes)
+{
+ return mana_ib_fmr_header_offset(num_ptes) + sizeof(struct mana_fmr_mst_entry);
+}
+
+struct mana_ib_fmr {
+ u64 *ptes;
+ dma_addr_t dma_handle;
+ size_t size;
+ u32 max_ptes;
+ u32 pte_count;
+};
+
struct mana_ib_mr {
struct ib_mr ibmr;
struct ib_umem *umem;
mana_handle_t mr_handle;
+ struct mana_ib_fmr fmr;
};
struct mana_ib_dm {
@@ -840,8 +880,16 @@ int mana_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,
int mana_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl);
+enum gdma_mr_access_flags
+mana_ib_verbs_to_gdma_access_flags(int access_flags);
+
struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags);
+struct ib_mr *mana_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
+ u32 max_num_sg);
+int mana_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
+ unsigned int *sg_offset);
+
struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
u64 iova, int access_flags,
struct ib_dmah *dmah,
diff --git a/drivers/infiniband/hw/mana/mr.c b/drivers/infiniband/hw/mana/mr.c
index 1233685b4b89..2be252778e17 100644
--- a/drivers/infiniband/hw/mana/mr.c
+++ b/drivers/infiniband/hw/mana/mr.c
@@ -10,7 +10,7 @@
#define VALID_DMA_MR_FLAGS (IB_ACCESS_LOCAL_WRITE)
-static enum gdma_mr_access_flags
+enum gdma_mr_access_flags
mana_ib_verbs_to_gdma_access_flags(int access_flags)
{
enum gdma_mr_access_flags flags = GDMA_ACCESS_FLAG_LOCAL_READ;
@@ -55,6 +55,11 @@ static int mana_ib_gd_create_mr(struct mana_ib_dev *dev, struct mana_ib_mr *mr,
req.gva.virtual_address = mr_params->gva.virtual_address;
req.gva.access_flags = mr_params->gva.access_flags;
break;
+ case GDMA_MR_TYPE_FMR:
+ /* page_size is deprecated field and must be 0. */
+ req.fmr.page_size = 0;
+ req.fmr.rsvd_pte_count = mr_params->fmr.rsvd_pte_count;
+ break;
case GDMA_MR_TYPE_ZBVA:
req.zbva.dma_region_handle = mr_params->zbva.dma_region_handle;
req.zbva.access_flags = mr_params->zbva.access_flags;
@@ -294,6 +299,80 @@ err_free:
return ERR_PTR(err);
}
+struct ib_mr *mana_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
+ u32 max_num_sg)
+{
+ struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
+ struct gdma_create_mr_params mr_params = {};
+ struct gdma_context *gc;
+ struct mana_ib_dev *dev;
+ struct mana_ib_mr *mr;
+ int err;
+
+ dev = container_of(ibpd->device, struct mana_ib_dev, ib_dev);
+ gc = mdev_to_gc(dev);
+
+ if (mr_type != IB_MR_TYPE_MEM_REG)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ if (max_num_sg > MANA_MAX_FMR_PAGES)
+ return ERR_PTR(-EINVAL);
+
+ mr = kzalloc_obj(*mr);
+ if (!mr)
+ return ERR_PTR(-ENOMEM);
+
+ mr->fmr.max_ptes = max_num_sg;
+ mr->fmr.size = mana_ib_fmr_buf_sz(max_num_sg);
+ mr->fmr.ptes = dma_alloc_coherent(gc->dev, mr->fmr.size, &mr->fmr.dma_handle,
+ GFP_KERNEL);
+ if (!mr->fmr.ptes) {
+ err = -ENOMEM;
+ goto err_free;
+ }
+
+ mr_params.pd_handle = pd->pd_handle;
+ mr_params.mr_type = GDMA_MR_TYPE_FMR;
+ mr_params.fmr.rsvd_pte_count = max_num_sg;
+
+ err = mana_ib_gd_create_mr(dev, mr, &mr_params);
+ if (err)
+ goto err_dma;
+
+ return &mr->ibmr;
+
+err_dma:
+ dma_free_coherent(gc->dev, mr->fmr.size, mr->fmr.ptes,
+ mr->fmr.dma_handle);
+
+err_free:
+ kfree(mr);
+ return ERR_PTR(err);
+}
+
+static int mana_ib_set_page(struct ib_mr *ibmr, u64 addr)
+{
+ struct mana_ib_mr *mr = container_of(ibmr, struct mana_ib_mr, ibmr);
+
+ if (mr->fmr.pte_count >= mr->fmr.max_ptes)
+ return -ENOMEM;
+
+ mr->fmr.ptes[mr->fmr.pte_count] = addr | MANA_PTE_VALID_MASK;
+ mr->fmr.pte_count++;
+
+ return 0;
+}
+
+int mana_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
+ unsigned int *sg_offset)
+{
+ struct mana_ib_mr *mr = container_of(ibmr, struct mana_ib_mr, ibmr);
+
+ mr->fmr.pte_count = 0;
+
+ return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, mana_ib_set_page);
+}
+
static int mana_ib_gd_create_mw(struct mana_ib_dev *dev, struct mana_ib_pd *pd, struct ib_mw *ibmw)
{
struct mana_ib_mw *mw = container_of(ibmw, struct mana_ib_mw, ibmw);
@@ -352,6 +431,7 @@ int mana_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
{
struct mana_ib_mr *mr = container_of(ibmr, struct mana_ib_mr, ibmr);
struct ib_device *ibdev = ibmr->device;
+ struct gdma_context *gc;
struct mana_ib_dev *dev;
int err;
@@ -360,11 +440,16 @@ int mana_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
return err;
dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
+ gc = mdev_to_gc(dev);
err = mana_ib_gd_destroy_mr(dev, mr->mr_handle);
if (err)
return err;
+ if (mr->fmr.ptes)
+ dma_free_coherent(gc->dev, mr->fmr.size, mr->fmr.ptes,
+ mr->fmr.dma_handle);
+
if (mr->umem)
ib_umem_release(mr->umem);
diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h
index 2a28fcaa637d..332f394b82e1 100644
--- a/include/net/mana/gdma.h
+++ b/include/net/mana/gdma.h
@@ -865,6 +865,8 @@ enum gdma_mr_access_flags {
GDMA_ACCESS_FLAG_REMOTE_WRITE = BIT_ULL(3),
GDMA_ACCESS_FLAG_REMOTE_ATOMIC = BIT_ULL(4),
GDMA_ACCESS_FLAG_BIND_MW = BIT_ULL(5),
+ GDMA_ACCESS_FLAG_INVALIDATE = BIT_ULL(6),
+ GDMA_ACCESS_FLAG_FMR = BIT_ULL(7),
};
/* GDMA_CREATE_DMA_REGION */
@@ -917,6 +919,7 @@ struct gdma_destroy_dma_region_req {
enum gdma_pd_flags {
GDMA_PD_FLAG_ALLOW_GPA_MR = BIT(0),
+ GDMA_PD_FLAG_ALLOW_FMR_MR = BIT(1),
GDMA_PD_FLAG_SHORT_PDN = BIT(2),
};
@@ -953,6 +956,11 @@ enum gdma_mr_type {
* address that is set up in the MST
*/
GDMA_MR_TYPE_GVA = 2,
+ /* Fast Memory - MRs are initially put in the free state,
+ * and the specified number of PTEs are reserved for future fast
+ * memory registrations
+ */
+ GDMA_MR_TYPE_FMR = 3,
/* Guest zero-based address MRs */
GDMA_MR_TYPE_ZBVA = 4,
/* Device address MRs */
@@ -982,6 +990,9 @@ struct gdma_create_mr_params {
u64 length;
enum gdma_mr_access_flags access_flags;
} da;
+ struct {
+ u32 rsvd_pte_count;
+ } fmr;
};
};
@@ -1006,6 +1017,10 @@ struct gdma_create_mr_request {
u64 offset;
enum gdma_mr_access_flags access_flags;
} __packed da;
+ struct {
+ u32 page_size;
+ u32 rsvd_pte_count;
+ } __packed fmr;
} __packed;
u32 reserved_2;
union {
--
2.43.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH rdma-next v5 02/10] RDMA/mana: Create and destroy kernel RC QPs
2026-10-17 6:44 [PATCH rdma-next v5 00/10] RDMA/mana_ib: Add kernel RC and fast registration support Konstantin Taranov
2026-10-17 6:44 ` [PATCH rdma-next v5 01/10] RDMA/mana_ib: Allocate and map fast-registration MRs Konstantin Taranov
@ 2026-10-17 6:44 ` Konstantin Taranov
2026-10-17 6:52 ` sashiko-bot
2026-10-17 6:44 ` [PATCH rdma-next v5 03/10] net/mana: Extend GDMA encoding for new RDMA WQEs Konstantin Taranov
` (7 subsequent siblings)
9 siblings, 1 reply; 21+ messages in thread
From: Konstantin Taranov @ 2026-10-17 6:44 UTC (permalink / raw)
To: kotaranov, snsanghvi, longli, jgg, leon; +Cc: linux-rdma, linux-kernel
From: Konstantin Taranov <kotaranov@microsoft.com>
Implement kernel RC QP creation and destruction. This feature requires
extended-WQE and power-of-two-SQ support from the HW.
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
---
v5:
- Validate kernel QP queue sizes before allocating queues.
- Bound SGE counts and enforce the TX/RX WQE size limits.
v4:
- Reject RNIC firmware send-SGE counts below 2.
- Subtract the reserved SGE for extra-large OOB only for RNIC mana_ib.
drivers/infiniband/hw/mana/main.c | 10 +-
drivers/infiniband/hw/mana/mana_ib.h | 6 +
drivers/infiniband/hw/mana/qp.c | 206 +++++++++++++++++++++++++--
3 files changed, 211 insertions(+), 11 deletions(-)
diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c
index ceb4ec58023a..f54905764f59 100644
--- a/drivers/infiniband/hw/mana/main.c
+++ b/drivers/infiniband/hw/mana/main.c
@@ -733,6 +733,13 @@ int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev)
if (err)
return err;
+ /* Hardware supports at least one data SGE plus one reserved for extra-large OOB. */
+ if (resp.max_send_sge_count < 2) {
+ ibdev_err(&dev->ib_dev, "Invalid max_send_sge_count %u, expected at least 2\n",
+ resp.max_send_sge_count);
+ return -EINVAL;
+ }
+
caps->max_sq_id = resp.max_sq_id;
caps->max_rq_id = resp.max_rq_id;
caps->max_cq_id = resp.max_cq_id;
@@ -748,7 +755,8 @@ int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev)
resp.max_requester_sq_size / GDMA_MAX_SQE_SIZE,
resp.max_requester_rq_size / GDMA_MAX_RQE_SIZE);
caps->max_inline_data_size = resp.max_inline_data_size;
- caps->max_send_sge_count = resp.max_send_sge_count;
+ /* Account for the SGE reserved for extra-large OOB. */
+ caps->max_send_sge_count = resp.max_send_sge_count - 1;
caps->max_recv_sge_count = resp.max_recv_sge_count;
caps->feature_flags = resp.feature_flags;
diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index 5ca6e2acf974..6f5df64e838a 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -267,6 +267,10 @@ struct mana_ib_qp {
u32 sq_psn;
bool sq_sig_all;
+ /* Serializes receive WR posting. */
+ spinlock_t rq_lock;
+ /* Serializes send and memory-management WR posting. */
+ spinlock_t sq_lock;
/* Serializes QP modification and error-list transitions. */
struct mutex modify_lock;
@@ -274,6 +278,7 @@ struct mana_ib_qp {
struct list_head recv_err_node;
struct shadow_queue shadow_rq;
struct shadow_queue shadow_sq;
+ struct shadow_queue shadow_mmq;
refcount_t refcount;
struct completion free;
@@ -315,6 +320,7 @@ enum mana_ib_adapter_features {
MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5),
MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6),
MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT = BIT(7),
+ MANA_IB_FEATURE_EXTENDED_WQE_FORMAT = BIT(9),
MANA_IB_FEATURE_RC_QP_SQ_POW2_SUPPORT = BIT(14),
MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT = BIT(15),
};
diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
index ae2ebbcc759a..387586c45aa0 100644
--- a/drivers/infiniband/hw/mana/qp.c
+++ b/drivers/infiniband/hw/mana/qp.c
@@ -427,27 +427,88 @@ static u32 mana_ib_fixed_wqe_size(u32 sge, u32 oob_size)
return roundup_pow_of_two(mana_ib_wqe_size(sge, oob_size));
}
-static u32 mana_ib_queue_size(struct ib_qp_init_attr *attr, u32 queue_type)
+static u32 mana_ib_queue_size(struct mana_ib_dev *mdev, struct ib_qp_init_attr *attr,
+ u32 queue_type)
{
- u32 queue_size;
+ u32 send_wr = max(attr->cap.max_send_wr, 1U);
+ u32 recv_wr = max(attr->cap.max_recv_wr, 1U);
+ u32 max_wqe_size = MAX_TX_WQE_SIZE;
+ u32 wr, wqe_size, read_sge;
+ u64 queue_size;
+
+ if (attr->cap.max_send_sge > MAX_TX_WQE_SIZE / sizeof(struct gdma_sge) ||
+ attr->cap.max_recv_sge > MAX_RX_WQE_SIZE / sizeof(struct gdma_sge))
+ return 0;
switch (attr->qp_type) {
case IB_QPT_UD:
case IB_QPT_GSI:
- if (queue_type == MANA_UD_SEND_QUEUE)
- queue_size = attr->cap.max_send_wr *
- mana_ib_wqe_size(attr->cap.max_send_sge, INLINE_OOB_LARGE_SIZE);
- else
- queue_size = attr->cap.max_recv_wr *
- mana_ib_wqe_size(attr->cap.max_recv_sge, INLINE_OOB_SMALL_SIZE);
+ if (queue_type == MANA_UD_SEND_QUEUE) {
+ wr = send_wr;
+ wqe_size = mana_ib_wqe_size(attr->cap.max_send_sge, INLINE_OOB_LARGE_SIZE);
+ } else {
+ wr = recv_wr;
+ max_wqe_size = MAX_RX_WQE_SIZE;
+ wqe_size = mana_ib_wqe_size(attr->cap.max_recv_sge, INLINE_OOB_SMALL_SIZE);
+ }
+ break;
+ case IB_QPT_RC:
+ switch (queue_type) {
+ case MANA_RC_SEND_QUEUE_REQUESTER:
+ wr = send_wr;
+ wqe_size = mana_ib_fixed_wqe_size(attr->cap.max_send_sge,
+ INLINE_OOB_EXTRA_LARGE_SIZE);
+ break;
+ case MANA_RC_SEND_QUEUE_MMQ:
+ wr = send_wr;
+ wqe_size = mana_ib_wqe_size(1U, INLINE_OOB_EXTRA_LARGE_SIZE);
+ break;
+ case MANA_RC_SEND_QUEUE_RESPONDER:
+ return MANA_PAGE_SIZE;
+ case MANA_RC_RECV_QUEUE_REQUESTER:
+ wr = send_wr;
+ max_wqe_size = MAX_RX_WQE_SIZE;
+ read_sge = min(attr->cap.max_send_sge, mdev->ib_dev.attrs.max_sge_rd);
+ wqe_size = mana_ib_wqe_size(read_sge, INLINE_OOB_SMALL_SIZE);
+ break;
+ case MANA_RC_RECV_QUEUE_RESPONDER:
+ wr = recv_wr;
+ max_wqe_size = MAX_RX_WQE_SIZE;
+ wqe_size = mana_ib_wqe_size(attr->cap.max_recv_sge, INLINE_OOB_SMALL_SIZE);
+ break;
+ default:
+ return 0;
+ }
break;
default:
return 0;
}
+ if (!wqe_size || wqe_size > max_wqe_size)
+ return 0;
+
+ queue_size = (u64)wr * wqe_size;
+ if (queue_size > U32_MAX / 2 + 1)
+ return 0;
+
return MANA_PAGE_ALIGN(roundup_pow_of_two(queue_size));
}
+static int mana_ib_check_kernel_qp_sizes(struct mana_ib_dev *mdev,
+ struct ib_qp_init_attr *attr, u32 num_queues)
+{
+ u32 i;
+
+ for (i = 0; i < num_queues; i++) {
+ if (!mana_ib_queue_size(mdev, attr, i)) {
+ ibdev_dbg(&mdev->ib_dev, "Invalid kernel QP queue %u size\n", i);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
static enum gdma_queue_type mana_ib_queue_type(struct ib_qp_init_attr *attr, u32 queue_type)
{
enum gdma_queue_type type;
@@ -460,6 +521,21 @@ static enum gdma_queue_type mana_ib_queue_type(struct ib_qp_init_attr *attr, u32
else
type = GDMA_RQ;
break;
+ case IB_QPT_RC:
+ switch (queue_type) {
+ case MANA_RC_SEND_QUEUE_REQUESTER:
+ case MANA_RC_SEND_QUEUE_RESPONDER:
+ case MANA_RC_SEND_QUEUE_MMQ:
+ type = GDMA_SQ;
+ break;
+ case MANA_RC_RECV_QUEUE_REQUESTER:
+ case MANA_RC_RECV_QUEUE_RESPONDER:
+ type = GDMA_RQ;
+ break;
+ default:
+ type = GDMA_INVALID_QUEUE;
+ }
+ break;
default:
type = GDMA_INVALID_QUEUE;
}
@@ -725,8 +801,12 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
if (udata)
return -EOPNOTSUPP;
+ err = mana_ib_check_kernel_qp_sizes(mdev, attr, MANA_UD_QUEUE_TYPE_MAX);
+ if (err)
+ return err;
+
for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i) {
- queue_size = mana_ib_queue_size(attr, i);
+ queue_size = mana_ib_queue_size(mdev, attr, i);
err = mana_ib_create_kernel_queue(mdev, queue_size, mana_ib_queue_type(attr, i),
&qp->ud_qp.queues[i]);
if (err) {
@@ -779,12 +859,111 @@ destroy_queues:
return err;
}
+static int mana_init_kernel_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
+ struct ib_qp_init_attr *attr, u64 *flags)
+{
+ u32 wqe_size;
+ int err;
+
+ err = create_shadow_queue(&qp->shadow_rq, attr->cap.max_recv_wr,
+ sizeof(struct shadow_wqe_header));
+ if (err)
+ return err;
+
+ err = create_shadow_queue(&qp->shadow_sq, attr->cap.max_send_wr,
+ sizeof(struct shadow_wqe_header));
+ if (err)
+ goto destroy_rq;
+
+ err = create_shadow_queue(&qp->shadow_mmq, attr->cap.max_send_wr,
+ sizeof(struct shadow_wqe_header));
+ if (err)
+ goto destroy_sq;
+
+ wqe_size = mana_ib_fixed_wqe_size(attr->cap.max_send_sge,
+ INLINE_OOB_EXTRA_LARGE_SIZE);
+ *flags |= MANA_RC_FLAG_FIXED_SIZE_WQE;
+ if (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT)
+ *flags |= MANA_RC_FLAG_MSN_IN_WQE;
+ qp->rc_qp.wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE;
+
+ return 0;
+
+destroy_sq:
+ destroy_shadow_queue(&qp->shadow_sq);
+destroy_rq:
+ destroy_shadow_queue(&qp->shadow_rq);
+ return err;
+}
+
+static int mana_ib_create_rc_qp_kernel(struct ib_qp *ibqp, struct ib_pd *ibpd,
+ struct ib_qp_init_attr *attr)
+{
+ struct mana_ib_dev *mdev = container_of(ibpd->device, struct mana_ib_dev, ib_dev);
+ struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+ u32 doorbell = mdev->gdma_dev->doorbell;
+ u32 queue_size;
+ u64 flags = 0;
+ int i, err;
+
+ if (!(mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_EXTENDED_WQE_FORMAT))
+ return -EOPNOTSUPP;
+ if (!(mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_RC_QP_SQ_POW2_SUPPORT))
+ return -EOPNOTSUPP;
+
+ err = mana_ib_check_kernel_qp_sizes(mdev, attr, MANA_RC_QUEUE_TYPE_MAX);
+ if (err)
+ return err;
+
+ for (i = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
+ queue_size = mana_ib_queue_size(mdev, attr, i);
+ err = mana_ib_create_kernel_queue(mdev, queue_size,
+ mana_ib_queue_type(attr, i),
+ &qp->rc_qp.queues[i]);
+ if (err)
+ goto destroy_queues;
+ }
+
+ err = mana_init_kernel_rc_qp(mdev, qp, attr, &flags);
+ if (err)
+ goto destroy_queues;
+
+ err = mana_ib_gd_create_rc_qp(mdev, qp, attr, doorbell, flags);
+ if (err)
+ goto deinit_queues;
+
+ qp->ibqp.qp_num = qp->rc_qp.queues[MANA_RC_RECV_QUEUE_RESPONDER].id;
+ qp->port = attr->port_num;
+
+ for (i = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i)
+ qp->rc_qp.queues[i].kmem->id = qp->rc_qp.queues[i].id;
+
+ err = mana_table_store_qp(mdev, qp);
+ if (err)
+ goto destroy_qp;
+
+ return 0;
+
+destroy_qp:
+ mana_ib_gd_destroy_rnic_qp(mdev, qp);
+deinit_queues:
+ destroy_shadow_queue(&qp->shadow_rq);
+ destroy_shadow_queue(&qp->shadow_sq);
+ destroy_shadow_queue(&qp->shadow_mmq);
+destroy_queues:
+ while (i-- > 0)
+ mana_ib_destroy_queue(mdev, &qp->rc_qp.queues[i]);
+ return err;
+}
+
int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
struct ib_udata *udata)
{
struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
qp->sq_sig_all = attr->sq_sig_type == IB_SIGNAL_ALL_WR;
+ spin_lock_init(&qp->rq_lock);
+ spin_lock_init(&qp->sq_lock);
mutex_init(&qp->modify_lock);
INIT_LIST_HEAD(&qp->send_err_node);
INIT_LIST_HEAD(&qp->recv_err_node);
@@ -798,7 +977,10 @@ int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
return mana_ib_create_qp_raw(ibqp, ibqp->pd, attr, udata);
case IB_QPT_RC:
- return mana_ib_create_rc_qp(ibqp, ibqp->pd, attr, udata);
+ if (udata)
+ return mana_ib_create_rc_qp(ibqp, ibqp->pd, attr, udata);
+ else
+ return mana_ib_create_rc_qp_kernel(ibqp, ibqp->pd, attr);
case IB_QPT_UC:
return mana_ib_create_uc_qp(ibqp, ibqp->pd, attr, udata);
case IB_QPT_UD:
@@ -1030,7 +1212,11 @@ static int mana_ib_destroy_rc_qp(struct mana_ib_qp *qp, struct ib_udata *udata)
return err;
mana_table_remove_qp(mdev, qp);
+ mana_remove_qp_from_cqs(qp, false);
+ destroy_shadow_queue(&qp->shadow_rq);
+ destroy_shadow_queue(&qp->shadow_sq);
+ destroy_shadow_queue(&qp->shadow_mmq);
/* Ignore return code as there is not much we can do about it.
* The error message is printed inside.
*/
--
2.43.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH rdma-next v5 03/10] net/mana: Extend GDMA encoding for new RDMA WQEs
2026-10-17 6:44 [PATCH rdma-next v5 00/10] RDMA/mana_ib: Add kernel RC and fast registration support Konstantin Taranov
2026-10-17 6:44 ` [PATCH rdma-next v5 01/10] RDMA/mana_ib: Allocate and map fast-registration MRs Konstantin Taranov
2026-10-17 6:44 ` [PATCH rdma-next v5 02/10] RDMA/mana: Create and destroy kernel RC QPs Konstantin Taranov
@ 2026-10-17 6:44 ` Konstantin Taranov
2026-10-17 6:54 ` sashiko-bot
2026-10-17 6:44 ` [PATCH rdma-next v5 04/10] RDMA/mana_ib: Maintain kernel RC QP state Konstantin Taranov
` (6 subsequent siblings)
9 siblings, 1 reply; 21+ messages in thread
From: Konstantin Taranov @ 2026-10-17 6:44 UTC (permalink / raw)
To: kotaranov, snsanghvi, longli, jgg, leon; +Cc: linux-rdma, linux-kernel
From: Konstantin Taranov <kotaranov@microsoft.com>
Extend mana_gd_post_work_request() to support extra-large OOB data and
fixed-size WQEs. These formats are supported only for power-of-two WQE
sizes.
For fixed-size WQEs, use num_pad_sge to specify the padding size. Encode
the WQE header fields required by RC QPs based on the WR flags.
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
---
v2:
- Reject fixed WQEs that cannot fit the header, aligned OOB and SGEs.
.../net/ethernet/microsoft/mana/gdma_main.c | 55 ++++++++++++++++---
include/net/mana/gdma.h | 21 +++++--
2 files changed, 63 insertions(+), 13 deletions(-)
diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c
index 05ba7055405d..2717889e626e 100644
--- a/drivers/net/ethernet/microsoft/mana/gdma_main.c
+++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c
@@ -1699,8 +1699,27 @@ static u32 mana_gd_write_client_oob(const struct gdma_wqe_request *wqe_req,
header->num_sge = wqe_req->num_sge;
header->inline_oob_size_div4 = client_oob_size / sizeof(u32);
+ if (client_oob_size == INLINE_OOB_EXTRA_LARGE_SIZE) {
+ /* Extra large oob is encoded as 7 */
+ header->inline_oob_size_div4 = 7;
+ /* Extra large oob has 8 byte padding */
+ client_oob_size += 8;
+ }
+
+ if (wqe_req->wqe_size_in_bu) {
+ header->num_sge = (wqe_req->wqe_size_in_bu * GDMA_WQE_BU_SIZE
+ - sizeof(struct gdma_wqe) - client_oob_size) / sizeof(struct gdma_sge);
+ header->num_pad_sge = header->num_sge - wqe_req->num_sge;
+ }
+
+ header->check_rsn = !!(wqe_req->flags & GDMA_WR_CHECK_RSN);
+ header->check_sn = !!(wqe_req->flags & GDMA_WR_CHECK_SN);
+ header->fence = !!(wqe_req->flags & GDMA_WR_FENCE);
+ header->fence_extended = !!(wqe_req->flags & GDMA_WR_FENCE_EXTENDED);
+ header->generation1 = !!(wqe_req->flags & GDMA_WR_GENERATION_1);
+
if (oob_in_sgl) {
- WARN_ON_ONCE(wqe_req->num_sge < 2);
+ WARN_ON_ONCE(wqe_req->num_sge < 1);
header->client_oob_in_sgl = 1;
@@ -1714,8 +1733,10 @@ static u32 mana_gd_write_client_oob(const struct gdma_wqe_request *wqe_req,
/* The size of gdma_wqe + client_oob_size must be less than or equal
* to one Basic Unit (i.e. 32 bytes), so the pointer can't go beyond
* the queue memory buffer boundary.
+ * The exception is when the WQE size is fixed and a power of 2, as
+ * such a wqe cannot go beyond the queue memory buffer boundary.
*/
- ptr = wqe_ptr + sizeof(header);
+ ptr = wqe_ptr + sizeof(struct gdma_wqe);
if (wqe_req->inline_oob_data && wqe_req->inline_oob_size > 0) {
memcpy(ptr, wqe_req->inline_oob_data, wqe_req->inline_oob_size);
@@ -1724,8 +1745,8 @@ static u32 mana_gd_write_client_oob(const struct gdma_wqe_request *wqe_req,
memset(ptr + wqe_req->inline_oob_size, 0,
client_oob_size - wqe_req->inline_oob_size);
}
-
- return sizeof(header) + client_oob_size;
+ /* The hw requires that (gdma_wqe + client_oob_size) is sge size aligned */
+ return ALIGN(sizeof(struct gdma_wqe) + client_oob_size, sizeof(struct gdma_sge));
}
static void mana_gd_write_sgl(struct gdma_queue *wq, u32 sgl_offset,
@@ -1784,7 +1805,8 @@ int mana_gd_post_work_request(struct gdma_queue *wq,
return -EINVAL;
if (wq->type == GDMA_RQ) {
- if (client_oob_size != 0)
+ if (client_oob_size != 0 &&
+ client_oob_size != INLINE_OOB_SMALL_SIZE)
return -EINVAL;
client_oob_size = INLINE_OOB_SMALL_SIZE;
@@ -1792,15 +1814,32 @@ int mana_gd_post_work_request(struct gdma_queue *wq,
max_wqe_size = GDMA_MAX_RQE_SIZE;
} else {
if (client_oob_size != INLINE_OOB_SMALL_SIZE &&
- client_oob_size != INLINE_OOB_LARGE_SIZE)
+ client_oob_size != INLINE_OOB_LARGE_SIZE &&
+ client_oob_size != INLINE_OOB_EXTRA_LARGE_SIZE)
return -EINVAL;
max_wqe_size = GDMA_MAX_SQE_SIZE;
}
sgl_data_size = sizeof(struct gdma_sge) * wqe_req->num_sge;
- wqe_size = ALIGN(sizeof(struct gdma_wqe) + client_oob_size +
- sgl_data_size, GDMA_WQE_BU_SIZE);
+ if (wqe_req->wqe_size_in_bu) {
+ wqe_size = wqe_req->wqe_size_in_bu * GDMA_WQE_BU_SIZE;
+ oob_len = ALIGN(sizeof(struct gdma_wqe) + client_oob_size,
+ sizeof(struct gdma_sge));
+ if (wqe_size < oob_len ||
+ wqe_req->num_sge > (wqe_size - oob_len) / sizeof(struct gdma_sge))
+ return -EINVAL;
+ } else {
+ wqe_size = ALIGN(sizeof(struct gdma_wqe) + client_oob_size +
+ sgl_data_size, GDMA_WQE_BU_SIZE);
+ }
+ /* Ensure the WQE size is a power of 2 when a fixed wqe size or extra large OOB are used.
+ * Otherwise, the WQE may cross the queue memory buffer boundary, which is not allowed.
+ */
+ if (wqe_req->wqe_size_in_bu || client_oob_size == INLINE_OOB_EXTRA_LARGE_SIZE) {
+ if (!is_power_of_2(wqe_size))
+ return -EINVAL;
+ }
if (wqe_size > max_wqe_size)
return -EINVAL;
diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h
index 332f394b82e1..508da28591ae 100644
--- a/include/net/mana/gdma.h
+++ b/include/net/mana/gdma.h
@@ -56,6 +56,11 @@ enum gdma_work_request_flags {
GDMA_WR_OOB_IN_SGL = BIT(0),
GDMA_WR_PAD_BY_SGE0 = BIT(1),
GDMA_WR_IB_SGL = BIT(2),
+ GDMA_WR_CHECK_SN = BIT(3),
+ GDMA_WR_CHECK_RSN = BIT(4),
+ GDMA_WR_FENCE = BIT(5),
+ GDMA_WR_FENCE_EXTENDED = BIT(6),
+ GDMA_WR_GENERATION_1 = BIT(7),
};
enum gdma_eqe_type {
@@ -219,8 +224,8 @@ struct gdma_wqe_request {
struct ib_sge *ib_sgl;
};
u32 num_sge;
-
- u32 inline_oob_size;
+ u16 wqe_size_in_bu;
+ u16 inline_oob_size;
const void *inline_oob_data;
u32 flags;
@@ -538,7 +543,9 @@ void mana_gd_ring_dim(struct gdma_queue *cq, u32 mod_usec, bool mod_usec_vld,
u32 mod_comps, bool mod_comps_vld);
struct gdma_wqe {
- u32 reserved :24;
+ u32 num_pad_sge :5;
+ u32 generation1 :1;
+ u32 reserved :18;
u32 last_vbytes :8;
union {
@@ -548,9 +555,13 @@ struct gdma_wqe {
u32 num_sge :8;
u32 inline_oob_size_div4:3;
u32 client_oob_in_sgl :1;
- u32 reserved1 :4;
+ u32 check_rsn :1;
+ u32 fence :1;
+ u32 fence_extended :1;
+ u32 reserved1 :1;
u32 client_data_unit :14;
- u32 reserved2 :2;
+ u32 check_sn :1;
+ u32 reserved2 :1;
};
};
}; /* HW DATA */
--
2.43.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH rdma-next v5 04/10] RDMA/mana_ib: Maintain kernel RC QP state
2026-10-17 6:44 [PATCH rdma-next v5 00/10] RDMA/mana_ib: Add kernel RC and fast registration support Konstantin Taranov
` (2 preceding siblings ...)
2026-10-17 6:44 ` [PATCH rdma-next v5 03/10] net/mana: Extend GDMA encoding for new RDMA WQEs Konstantin Taranov
@ 2026-10-17 6:44 ` Konstantin Taranov
2026-10-17 6:48 ` sashiko-bot
2026-10-17 6:44 ` [PATCH rdma-next v5 05/10] RDMA/mana_ib: Post receive WRs on kernel RC QPs Konstantin Taranov
` (5 subsequent siblings)
9 siblings, 1 reply; 21+ messages in thread
From: Konstantin Taranov @ 2026-10-17 6:44 UTC (permalink / raw)
To: kotaranov, snsanghvi, longli, jgg, leon; +Cc: linux-rdma, linux-kernel
From: Konstantin Taranov <kotaranov@microsoft.com>
Update mana_ib_modify_qp_state() to support RC QPs. Initialize the
packet, message, and frame counters for use by work requests.
Arm the requester receive queue at the initial PSN so that a CQE is
generated when the specified PSN is acknowledged, indicating send
completions.
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
---
v2:
- Preserve IB_QPT_UC handling in mana_ib_modify_qp().
drivers/infiniband/hw/mana/mana_ib.h | 5 +++++
drivers/infiniband/hw/mana/qp.c | 22 ++++++++++++++++++++--
include/net/mana/gdma.h | 2 ++
3 files changed, 27 insertions(+), 2 deletions(-)
diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index 6f5df64e838a..1094429af801 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -265,7 +265,12 @@ struct mana_ib_qp {
/* The port on the IB device, starting with 1 */
u32 port;
u32 sq_psn;
+ u32 sq_ssn;
+ u32 sq_fsn;
+ u16 sq_rsn;
+ bool pending_mmq_fence;
bool sq_sig_all;
+ enum ib_mtu mtu;
/* Serializes receive WR posting. */
spinlock_t rq_lock;
diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
index 387586c45aa0..5d6f45b9f53d 100644
--- a/drivers/infiniband/hw/mana/qp.c
+++ b/drivers/infiniband/hw/mana/qp.c
@@ -779,8 +779,10 @@ static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp, bool reset)
spin_lock_irqsave(&send_cq->cq_lock, flags);
list_del_init(&qp->send_err_node);
/* Keep shadow reset serialized with hardware polling and SW flushing. */
- if (reset)
+ if (reset) {
reset_shadow_queue(&qp->shadow_sq);
+ reset_shadow_queue(&qp->shadow_mmq);
+ }
spin_unlock_irqrestore(&send_cq->cq_lock, flags);
spin_lock_irqsave(&recv_cq->cq_lock, flags);
@@ -1071,11 +1073,16 @@ static int mana_ib_gd_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
static void mana_ib_modify_qp_state(struct ib_qp *ibqp, struct ib_qp_attr *attr,
int attr_mask, struct ib_udata *udata)
{
+ struct mana_ib_dev *mdev = container_of(ibqp->device, struct mana_ib_dev, ib_dev);
struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+ struct gdma_queue *rq;
if (udata)
return;
+ if (attr_mask & IB_QP_PATH_MTU)
+ qp->mtu = attr->path_mtu;
+
if (attr_mask & IB_QP_STATE) {
switch (attr->qp_state) {
case IB_QPS_RESET:
@@ -1086,8 +1093,19 @@ static void mana_ib_modify_qp_state(struct ib_qp *ibqp, struct ib_qp_attr *attr,
}
}
- if (attr_mask & IB_QP_SQ_PSN)
+ if (attr_mask & IB_QP_SQ_PSN) {
+ qp->sq_ssn = 1;
+ qp->sq_rsn = 1;
+ qp->sq_fsn = 1;
qp->sq_psn = attr->sq_psn;
+ qp->pending_mmq_fence = false;
+
+ if (qp->ibqp.qp_type == IB_QPT_RC) {
+ rq = qp->rc_qp.queues[MANA_RC_RECV_QUEUE_REQUESTER].kmem;
+ mana_gd_wq_ring_doorbell_ext(mdev_to_gc(mdev), rq, attr->sq_psn,
+ SET_ARM_BIT, MANA_PSN_CLIENT_OFFSET);
+ }
+ }
}
int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h
index 508da28591ae..fd0ec8fc44d5 100644
--- a/include/net/mana/gdma.h
+++ b/include/net/mana/gdma.h
@@ -605,6 +605,8 @@ struct gdma_cqe {
#define GDMA_EQE_OWNER_BITS 3
+#define MANA_PSN_CLIENT_OFFSET 0x8
+
union gdma_eqe_info {
u32 as_uint32;
--
2.43.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH rdma-next v5 05/10] RDMA/mana_ib: Post receive WRs on kernel RC QPs
2026-10-17 6:44 [PATCH rdma-next v5 00/10] RDMA/mana_ib: Add kernel RC and fast registration support Konstantin Taranov
` (3 preceding siblings ...)
2026-10-17 6:44 ` [PATCH rdma-next v5 04/10] RDMA/mana_ib: Maintain kernel RC QP state Konstantin Taranov
@ 2026-10-17 6:44 ` Konstantin Taranov
2026-10-17 6:48 ` sashiko-bot
2026-10-17 6:44 ` [PATCH rdma-next v5 06/10] RDMA/mana_ib: Post send and memory-management WRs on " Konstantin Taranov
` (4 subsequent siblings)
9 siblings, 1 reply; 21+ messages in thread
From: Konstantin Taranov @ 2026-10-17 6:44 UTC (permalink / raw)
To: kotaranov, snsanghvi, longli, jgg, leon; +Cc: linux-rdma, linux-kernel
From: Konstantin Taranov <kotaranov@microsoft.com>
Implement receive request posting for RC QPs. When posting a batch of
WRs, ring the doorbell periodically to prevent the u8 counter from
overflowing.
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
---
drivers/infiniband/hw/mana/wr.c | 37 +++++++++++++++++++++++++++++++++
1 file changed, 37 insertions(+)
diff --git a/drivers/infiniband/hw/mana/wr.c b/drivers/infiniband/hw/mana/wr.c
index 250d673f58f2..5d42af91fce4 100644
--- a/drivers/infiniband/hw/mana/wr.c
+++ b/drivers/infiniband/hw/mana/wr.c
@@ -70,6 +70,41 @@ static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *
return err;
}
+static int mana_ib_post_recv_rc(struct mana_ib_qp *qp, const struct ib_recv_wr *wr,
+ const struct ib_recv_wr **bad_wr)
+{
+ struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
+ struct mana_ib_queue *ib_rq = mana_qp_get_rq(qp);
+ unsigned long flags;
+ u8 wqe_cnt = 0;
+ int err = 0;
+
+ spin_lock_irqsave(&qp->rq_lock, flags);
+
+ for (; wr; wr = wr->next) {
+ err = mana_ib_post_rq(qp, wr);
+ if (unlikely(err)) {
+ *bad_wr = wr;
+ break;
+ }
+ wqe_cnt++;
+ if (wqe_cnt == 0xFF) {
+ mana_gd_wq_ring_doorbell_ext(mdev_to_gc(mdev), ib_rq->kmem,
+ ib_rq->kmem->head * GDMA_WQE_BU_SIZE,
+ wqe_cnt, 0);
+ wqe_cnt = 0;
+ }
+ }
+
+ if (wqe_cnt)
+ mana_gd_wq_ring_doorbell_ext(mdev_to_gc(mdev), ib_rq->kmem,
+ ib_rq->kmem->head * GDMA_WQE_BU_SIZE,
+ wqe_cnt, 0);
+
+ spin_unlock_irqrestore(&qp->rq_lock, flags);
+ return err;
+}
+
int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
const struct ib_recv_wr **bad_wr)
{
@@ -79,6 +114,8 @@ int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
case IB_QPT_UD:
case IB_QPT_GSI:
return mana_ib_post_recv_ud(qp, wr, bad_wr);
+ case IB_QPT_RC:
+ return mana_ib_post_recv_rc(qp, wr, bad_wr);
default:
/* Unsupported QP type */
*bad_wr = wr;
--
2.43.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH rdma-next v5 06/10] RDMA/mana_ib: Post send and memory-management WRs on RC QPs
2026-10-17 6:44 [PATCH rdma-next v5 00/10] RDMA/mana_ib: Add kernel RC and fast registration support Konstantin Taranov
` (4 preceding siblings ...)
2026-10-17 6:44 ` [PATCH rdma-next v5 05/10] RDMA/mana_ib: Post receive WRs on kernel RC QPs Konstantin Taranov
@ 2026-10-17 6:44 ` Konstantin Taranov
2026-10-17 7:00 ` sashiko-bot
2026-10-17 6:44 ` [PATCH rdma-next v5 07/10] RDMA/mana_ib: Poll RC completions using PSN and FSN progress Konstantin Taranov
` (3 subsequent siblings)
9 siblings, 1 reply; 21+ messages in thread
From: Konstantin Taranov @ 2026-10-17 6:44 UTC (permalink / raw)
To: kotaranov, snsanghvi, longli, jgg, leon; +Cc: linux-rdma, linux-kernel
From: Konstantin Taranov <kotaranov@microsoft.com>
Implement send work request posting for RC QPs. Post communication WRs
to the send queue and memory WRs to the memory management queue.
Advertise the memory management extensions on adapters that support
extended WQEs.
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
---
v5:
- Preserve fence flags on unsignaled WQEs.
- Check message length for WQEs to prevent overflows.
- Reject read WRs that use more sges than supported.
drivers/infiniband/hw/mana/main.c | 2 +
drivers/infiniband/hw/mana/mana_ib.h | 1 +
drivers/infiniband/hw/mana/wr.c | 355 +++++++++++++++++++++++++++
3 files changed, 358 insertions(+)
diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c
index f54905764f59..775b3cca8ec4 100644
--- a/drivers/infiniband/hw/mana/main.c
+++ b/drivers/infiniband/hw/mana/main.c
@@ -632,6 +632,8 @@ int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
props->max_qp = dev->adapter_caps.max_qp_count;
props->max_qp_wr = dev->adapter_caps.max_qp_wr;
props->device_cap_flags = IB_DEVICE_RC_RNR_NAK_GEN;
+ if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_EXTENDED_WQE_FORMAT)
+ props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
props->max_send_sge = dev->adapter_caps.max_send_sge_count;
props->max_recv_sge = dev->adapter_caps.max_recv_sge_count;
props->max_sge_rd = dev->adapter_caps.max_recv_sge_count;
diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index 1094429af801..f7cc9abeb071 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -325,6 +325,7 @@ enum mana_ib_adapter_features {
MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5),
MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6),
MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT = BIT(7),
+ MANA_IB_FEATURE_FMR_IN_OOB_SUPPORT = BIT(8),
MANA_IB_FEATURE_EXTENDED_WQE_FORMAT = BIT(9),
MANA_IB_FEATURE_RC_QP_SQ_POW2_SUPPORT = BIT(14),
MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT = BIT(15),
diff --git a/drivers/infiniband/hw/mana/wr.c b/drivers/infiniband/hw/mana/wr.c
index 5d42af91fce4..d5dec02aa70a 100644
--- a/drivers/infiniband/hw/mana/wr.c
+++ b/drivers/infiniband/hw/mana/wr.c
@@ -7,6 +7,71 @@
#define MAX_WR_SGL_NUM (2)
+static inline u32 mana_count_packets(u32 msg_size, enum ib_mtu mtu)
+{
+ return (msg_size - !!msg_size) >> (mtu + 7);
+}
+
+static inline bool wr_is_mmq(enum ib_wr_opcode opcode)
+{
+ switch (opcode) {
+ case IB_WR_LOCAL_INV:
+ case IB_WR_REG_MR:
+ case IB_WR_REG_MR_INTEGRITY:
+ case IB_WR_BIND_MW:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static inline u8 convert_wr_to_wc(enum ib_wr_opcode opcode)
+{
+ switch (opcode) {
+ case IB_WR_SEND_WITH_IMM:
+ case IB_WR_SEND_WITH_INV:
+ case IB_WR_SEND:
+ return IB_WC_SEND;
+ case IB_WR_RDMA_WRITE_WITH_IMM:
+ case IB_WR_RDMA_WRITE:
+ return IB_WC_RDMA_WRITE;
+ case IB_WR_RDMA_READ:
+ return IB_WC_RDMA_READ;
+ case IB_WR_LOCAL_INV:
+ return IB_WC_LOCAL_INV;
+ case IB_WR_BIND_MW:
+ return IB_WC_BIND_MW;
+ case IB_WR_REG_MR:
+ return IB_WC_REG_MR;
+ default:
+ return 0;
+ }
+}
+
+static inline u8 convert_wr_to_hw_opcode(enum ib_wr_opcode opcode)
+{
+ switch (opcode) {
+ case IB_WR_RDMA_WRITE:
+ return WQE_TYPE_WRITE;
+ case IB_WR_RDMA_WRITE_WITH_IMM:
+ return WQE_TYPE_WRITE_IMM;
+ case IB_WR_SEND:
+ return WQE_TYPE_RC_SEND;
+ case IB_WR_SEND_WITH_IMM:
+ return WQE_TYPE_RC_SEND_IMM;
+ case IB_WR_SEND_WITH_INV:
+ return WQE_TYPE_RC_SEND_INV;
+ case IB_WR_LOCAL_INV:
+ return WQE_TYPE_LOCAL_INV;
+ case IB_WR_RDMA_READ:
+ return WQE_TYPE_READ;
+ case IB_WR_REG_MR:
+ return WQE_TYPE_REG_MR;
+ default:
+ return 0;
+ }
+}
+
static int mana_ib_post_rq(struct mana_ib_qp *qp, const struct ib_recv_wr *wr)
{
struct ib_sge mana_ib_dummy_sge = {.addr = 1, .length = 0, .lkey = 0};
@@ -224,6 +289,294 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_send_wr *
return err;
}
+static int mana_ib_post_sq(struct mana_ib_qp *qp, const struct ib_send_wr *wr)
+{
+ struct gdma_queue *sq = qp->rc_qp.queues[MANA_RC_SEND_QUEUE_REQUESTER].kmem;
+ struct ib_sge mana_ib_dummy_sge = {.addr = 1, .length = 0, .lkey = 0};
+ struct gdma_posted_wqe_info wqe_info = {0};
+ struct gdma_wqe_request wqe_req = {0};
+ struct rdma_send_oob send_oob = {0};
+ struct shadow_wqe_header *shadow_wqe;
+ bool signaled = qp->sq_sig_all || (wr->send_flags & IB_SEND_SIGNALED);
+ u32 msg_sz = 0;
+ int err, i;
+
+ if (shadow_queue_full(&qp->shadow_sq))
+ return -ENOMEM;
+
+ if (wr->num_sge > MAX_TX_WQE_SGL_ENTRIES)
+ return -EINVAL;
+
+ if (wr->opcode == IB_WR_RDMA_READ &&
+ wr->num_sge > qp->ibqp.max_read_sge)
+ return -EINVAL;
+
+ for (i = 0; i < wr->num_sge; i++) {
+ if (wr->sg_list[i].length > U32_MAX - msg_sz)
+ return -EINVAL;
+ msg_sz += wr->sg_list[i].length;
+ }
+
+ shadow_wqe = shadow_queue_producer_entry(&qp->shadow_sq);
+ memset(shadow_wqe, 0, sizeof(*shadow_wqe));
+
+ wqe_req.num_sge = wr->num_sge;
+ wqe_req.ib_sgl = wr->sg_list;
+ wqe_req.flags = GDMA_WR_IB_SGL;
+
+ if (wr->num_sge == 0) {
+ wqe_req.ib_sgl = &mana_ib_dummy_sge;
+ wqe_req.num_sge = 1;
+ }
+
+ if (wr->opcode == IB_WR_RDMA_READ) {
+ struct gdma_queue *rq = qp->rc_qp.queues[MANA_RC_RECV_QUEUE_REQUESTER].kmem;
+ struct rdma_recv_oob recv_oob = {0};
+
+ recv_oob.psn_start = qp->sq_psn;
+ recv_oob.msn = qp->sq_ssn;
+
+ wqe_req.inline_oob_size = INLINE_OOB_SMALL_SIZE;
+ wqe_req.inline_oob_data = &recv_oob;
+ wqe_req.flags |= GDMA_WR_CHECK_SN;
+
+ err = mana_gd_post_work_request(rq, &wqe_req, &wqe_info);
+ if (err)
+ return err;
+
+ /* For reads we store the WQE size of the receive WQE */
+ shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu;
+
+ /* SQ WQE for read uses dummy SGE */
+ wqe_req.num_sge = 1;
+ wqe_req.ib_sgl = &mana_ib_dummy_sge;
+ wqe_req.flags = GDMA_WR_IB_SGL;
+ }
+
+ wqe_req.inline_oob_size = INLINE_OOB_EXTRA_LARGE_SIZE;
+ wqe_req.inline_oob_data = &send_oob;
+ wqe_req.client_data_unit = ib_mtu_enum_to_int(qp->mtu);
+ wqe_req.flags |= GDMA_WR_GENERATION_1;
+ wqe_req.wqe_size_in_bu = qp->rc_qp.wqe_size_in_bu;
+
+ send_oob.wqe_type = convert_wr_to_hw_opcode(wr->opcode);
+ send_oob.signaled = signaled;
+ send_oob.solicited = !!(wr->send_flags & IB_SEND_SOLICITED);
+ send_oob.psn = qp->sq_psn;
+ send_oob.ssn_or_rqpn = qp->sq_ssn;
+
+ if (qp->pending_mmq_fence) {
+ /* set local fence */
+ send_oob.fence = 1;
+ send_oob.fsn = qp->sq_fsn;
+ wqe_req.flags |= GDMA_WR_FENCE;
+ }
+
+ if (wr->send_flags & IB_SEND_FENCE) {
+ /* add remote fence */
+ if (qp->pending_mmq_fence)
+ wqe_req.flags |= GDMA_WR_FENCE_EXTENDED;
+ else
+ wqe_req.flags |= GDMA_WR_FENCE;
+ }
+
+ switch (wr->opcode) {
+ case IB_WR_SEND_WITH_INV:
+ send_oob.rc_send.invalidate_key = wr->ex.invalidate_rkey;
+ wqe_req.flags |= GDMA_WR_CHECK_SN;
+ break;
+ case IB_WR_SEND_WITH_IMM:
+ send_oob.rc_send.immediate = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
+ fallthrough;
+ case IB_WR_SEND:
+ wqe_req.flags |= GDMA_WR_CHECK_SN;
+ break;
+ case IB_WR_RDMA_WRITE_WITH_IMM:
+ send_oob.immediate_ext = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
+ fallthrough;
+ case IB_WR_RDMA_WRITE:
+ case IB_WR_RDMA_READ:
+ send_oob.rdma.address_hi = upper_32_bits(rdma_wr(wr)->remote_addr);
+ send_oob.rdma.address_low = lower_32_bits(rdma_wr(wr)->remote_addr);
+ send_oob.rdma.rkey = rdma_wr(wr)->rkey;
+ send_oob.rdma.dma_len = msg_sz;
+ if (wr->opcode == IB_WR_RDMA_READ) {
+ send_oob.read.rsn = qp->sq_rsn;
+ wqe_req.flags |= GDMA_WR_CHECK_RSN;
+ } else {
+ wqe_req.flags |= GDMA_WR_CHECK_SN;
+ }
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ err = mana_gd_post_work_request(sq, &wqe_req, &wqe_info);
+ if (err)
+ return err;
+
+ shadow_wqe->wr_id = wr->wr_id;
+ shadow_wqe->send_opcode = convert_wr_to_wc(wr->opcode);
+ if (!signaled)
+ shadow_wqe->flags |= MANA_WQ_NO_SIGNAL_WC;
+ if ((wr->send_flags & IB_SEND_FENCE) || qp->pending_mmq_fence)
+ shadow_wqe->flags |= MANA_WQ_FENCE_WC;
+ shadow_wqe->wqe_offset_or_psn = qp->sq_psn + mana_count_packets(msg_sz, qp->mtu);
+ shadow_wqe->fsn = qp->sq_fsn - 1;
+ shadow_queue_advance_producer(&qp->shadow_sq);
+
+ qp->pending_mmq_fence = false;
+ qp->sq_psn = shadow_wqe->wqe_offset_or_psn + 1;
+ qp->sq_ssn++;
+ if (wr->opcode == IB_WR_RDMA_READ)
+ qp->sq_rsn++;
+
+ return 0;
+}
+
+static u8 mana_ib_fmr_access_flags(int ib_access_flags)
+{
+ u8 access = mana_ib_verbs_to_gdma_access_flags(ib_access_flags);
+
+ return (access | GDMA_ACCESS_FLAG_INVALIDATE | GDMA_ACCESS_FLAG_FMR);
+}
+
+static void mana_ib_build_fmr_payload(const struct ib_reg_wr *wr, struct ib_sge *sge)
+{
+ struct mana_ib_mr *mr = container_of(wr->mr, struct mana_ib_mr, ibmr);
+ u8 access = mana_ib_fmr_access_flags(wr->access);
+ u8 *mst_base = (u8 *)mr->fmr.ptes;
+ struct mana_fmr_mst_entry *mst;
+
+ mst = (struct mana_fmr_mst_entry *)(mst_base +
+ mana_ib_fmr_header_offset(mr->fmr.pte_count));
+ memset(mst, 0, sizeof(struct mana_fmr_mst_entry));
+ mst->access_right = access;
+ mst->addr_type = GDMA_MR_TYPE_GVA;
+ mst->start_va = wr->mr->iova;
+ mst->length = wr->mr->length;
+ mst->page_size = order_base_2(wr->mr->page_size) - MANA_PAGE_SHIFT;
+
+ sge->addr = mr->fmr.dma_handle;
+ sge->lkey = mr->ibmr.pd->local_dma_lkey;
+ sge->length = mana_ib_fmr_buf_sz(mr->fmr.pte_count);
+}
+
+static int mana_ib_post_mmq(struct mana_ib_qp *qp, const struct ib_send_wr *wr)
+{
+ struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
+ struct gdma_queue *queue = qp->rc_qp.queues[MANA_RC_SEND_QUEUE_MMQ].kmem;
+ struct ib_sge sge = {.addr = 1, .length = 0, .lkey = 0};
+ struct gdma_posted_wqe_info wqe_info = {0};
+ struct gdma_wqe_request wqe_req = {0};
+ struct rdma_send_oob send_oob = {0};
+ struct shadow_wqe_header *shadow_wqe;
+ bool signaled = qp->sq_sig_all || (wr->send_flags & IB_SEND_SIGNALED);
+ int ret;
+
+ if (shadow_queue_full(&qp->shadow_mmq))
+ return -ENOMEM;
+
+ shadow_wqe = shadow_queue_producer_entry(&qp->shadow_mmq);
+ memset(shadow_wqe, 0, sizeof(*shadow_wqe));
+
+ wqe_req.num_sge = 1;
+ wqe_req.ib_sgl = &sge;
+ wqe_req.flags = GDMA_WR_IB_SGL;
+
+ wqe_req.inline_oob_size = INLINE_OOB_EXTRA_LARGE_SIZE;
+ wqe_req.inline_oob_data = &send_oob;
+ wqe_req.client_data_unit = 0;
+ wqe_req.flags |= GDMA_WR_GENERATION_1;
+
+ send_oob.wqe_type = convert_wr_to_hw_opcode(wr->opcode);
+ send_oob.psn = qp->sq_fsn;
+ send_oob.ssn_or_rqpn = qp->sq_ssn;
+ send_oob.fsn = qp->sq_psn;
+ send_oob.signaled = signaled;
+
+ if (wr->send_flags & IB_SEND_FENCE)
+ wqe_req.flags |= GDMA_WR_FENCE | GDMA_WR_FENCE_EXTENDED;
+
+ switch (wr->opcode) {
+ case IB_WR_LOCAL_INV:
+ send_oob.mm.mkey = wr->ex.invalidate_rkey;
+ break;
+ case IB_WR_REG_MR:
+ send_oob.mm.mkey = reg_wr(wr)->key;
+ mana_ib_build_fmr_payload(reg_wr(wr), &sge);
+ if (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_FMR_IN_OOB_SUPPORT)
+ wqe_req.flags |= GDMA_WR_OOB_IN_SGL;
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ ret = mana_gd_post_work_request(queue, &wqe_req, &wqe_info);
+ if (ret)
+ return ret;
+
+ shadow_wqe->wr_id = wr->wr_id;
+ shadow_wqe->send_opcode = convert_wr_to_wc(wr->opcode);
+ if (!signaled)
+ shadow_wqe->flags |= MANA_WQ_NO_SIGNAL_WC;
+ if (wr->send_flags & IB_SEND_FENCE)
+ shadow_wqe->flags |= MANA_WQ_FENCE_WC;
+ shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu;
+ shadow_wqe->wqe_offset_or_psn = qp->sq_psn - 1;
+ shadow_wqe->fsn = qp->sq_fsn;
+
+ shadow_queue_advance_producer(&qp->shadow_mmq);
+
+ qp->sq_fsn++;
+ qp->pending_mmq_fence = true;
+
+ return 0;
+}
+
+static int mana_ib_post_send_rc(struct mana_ib_qp *qp, const struct ib_send_wr *wr,
+ const struct ib_send_wr **bad_wr)
+{
+ struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
+ bool ring_mmq = false, ring_sq = false, ring_rq = false;
+ unsigned long flags;
+ int err = 0;
+
+ spin_lock_irqsave(&qp->sq_lock, flags);
+
+ for (; wr; wr = wr->next) {
+ if (wr_is_mmq(wr->opcode)) {
+ err = mana_ib_post_mmq(qp, wr);
+ if (unlikely(err)) {
+ *bad_wr = wr;
+ break;
+ }
+ ring_mmq = true;
+ } else {
+ err = mana_ib_post_sq(qp, wr);
+ if (unlikely(err)) {
+ *bad_wr = wr;
+ break;
+ }
+ if (wr->opcode == IB_WR_RDMA_READ)
+ ring_rq = true;
+ ring_sq = true;
+ }
+ }
+ if (ring_rq)
+ mana_gd_wq_ring_doorbell(mdev_to_gc(mdev),
+ qp->rc_qp.queues[MANA_RC_RECV_QUEUE_REQUESTER].kmem);
+ if (ring_mmq)
+ mana_gd_wq_ring_doorbell(mdev_to_gc(mdev),
+ qp->rc_qp.queues[MANA_RC_SEND_QUEUE_MMQ].kmem);
+ if (ring_sq)
+ mana_gd_wq_ring_doorbell(mdev_to_gc(mdev),
+ qp->rc_qp.queues[MANA_RC_SEND_QUEUE_REQUESTER].kmem);
+
+ spin_unlock_irqrestore(&qp->sq_lock, flags);
+ return err;
+}
+
int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
const struct ib_send_wr **bad_wr)
{
@@ -233,6 +586,8 @@ int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
case IB_QPT_UD:
case IB_QPT_GSI:
return mana_ib_post_send_ud(qp, wr, bad_wr);
+ case IB_QPT_RC:
+ return mana_ib_post_send_rc(qp, wr, bad_wr);
default:
/* Unsupported QP type */
*bad_wr = wr;
--
2.43.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH rdma-next v5 07/10] RDMA/mana_ib: Poll RC completions using PSN and FSN progress
2026-10-17 6:44 [PATCH rdma-next v5 00/10] RDMA/mana_ib: Add kernel RC and fast registration support Konstantin Taranov
` (5 preceding siblings ...)
2026-10-17 6:44 ` [PATCH rdma-next v5 06/10] RDMA/mana_ib: Post send and memory-management WRs on " Konstantin Taranov
@ 2026-10-17 6:44 ` Konstantin Taranov
2026-10-17 6:56 ` sashiko-bot
2026-10-17 6:44 ` [PATCH rdma-next v5 08/10] RDMA/mana_ib: Flush and notify CQs when kernel QPs enter ERR Konstantin Taranov
` (2 subsequent siblings)
9 siblings, 1 reply; 21+ messages in thread
From: Konstantin Taranov @ 2026-10-17 6:44 UTC (permalink / raw)
To: kotaranov, snsanghvi, longli, jgg, leon; +Cc: linux-rdma, linux-kernel
From: Konstantin Taranov <kotaranov@microsoft.com>
Decode RC receive completions, armed-PSN notifications, and MMQ
completions. Retire requester and MMQ shadow entries using their
sequence numbers, honor cross-queue fence dependencies, and suppress
successful WCs for unsignaled work requests.
On an ARMED completion, rearm the next signal PSN to generate the next
send CQE.
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
---
v4:
- Assign wqe_size in shadow wqes for RC SEND/WRITE wrs
- Report a missed-event hint for a cached CQE for arming CQ
v3:
- Mask arm_psn to 24 bits before ringing the requester doorbell.
drivers/infiniband/hw/mana/cq.c | 155 ++++++++++++++++++++--
drivers/infiniband/hw/mana/mana_ib.h | 31 +++++
drivers/infiniband/hw/mana/shadow_queue.h | 22 +++
drivers/infiniband/hw/mana/wr.c | 3 +
4 files changed, 200 insertions(+), 11 deletions(-)
diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c
index 3b8044b7dc8f..a4d886b9ecc6 100644
--- a/drivers/infiniband/hw/mana/cq.c
+++ b/drivers/infiniband/hw/mana/cq.c
@@ -5,6 +5,12 @@
#include "mana_ib.h"
+#define PSN_SIGN_BIT 0x800000
+#define PSN_GE(PSN1, PSN2) ((((PSN1) - (PSN2)) & PSN_SIGN_BIT) == 0)
+#define PSN_GT(PSN1, PSN2) PSN_GE(PSN1, (PSN2) + 1)
+#define PSN_LE(PSN1, PSN2) PSN_GE(PSN2, PSN1)
+#define PSN_LT(PSN1, PSN2) PSN_GT(PSN2, PSN1)
+
static enum ib_wc_status vendor_error_to_wc_error(uint32_t vendor_error)
{
switch (vendor_error) {
@@ -277,15 +283,18 @@ int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
struct gdma_queue *gdma_cq = cq->queue.kmem;
unsigned long irq_flags;
+ bool missed = false;
if (!gdma_cq)
return -EINVAL;
spin_lock_irqsave(&cq->cq_lock, irq_flags);
mana_ib_cq_doorbell(cq, SET_ARM_BIT);
+ if (flags & IB_CQ_REPORT_MISSED_EVENTS)
+ missed = cq->has_pending_cqe;
spin_unlock_irqrestore(&cq->cq_lock, irq_flags);
- return 0;
+ return missed;
}
struct mana_cq_poll {
@@ -311,10 +320,10 @@ static struct ib_wc *mana_fill_wc(struct mana_ib_qp *qp,
return wc;
}
-static void mana_complete_send(struct mana_ib_qp *qp,
+static void mana_complete_send(struct mana_ib_qp *qp, bool mmq,
struct mana_cq_poll *poll, u32 vendor_error)
{
- struct shadow_queue *shadow = &qp->shadow_sq;
+ struct shadow_queue *shadow = mmq ? &qp->shadow_mmq : &qp->shadow_sq;
struct shadow_wqe_header *wqe = shadow_queue_get_next_to_consume(shadow);
struct gdma_queue *queue;
@@ -324,8 +333,19 @@ static void mana_complete_send(struct mana_ib_qp *qp,
if (vendor_error || !(wqe->flags & MANA_WQ_NO_SIGNAL_WC))
mana_fill_wc(qp, poll, wqe, wqe->send_opcode, vendor_error);
- queue = mana_qp_get_sq(qp)->kmem;
- queue->tail += wqe->wqe_size_in_bu;
+ if (mmq) {
+ queue = qp->rc_qp.queues[MANA_RC_SEND_QUEUE_MMQ].kmem;
+ queue->tail += wqe->wqe_size_in_bu;
+ } else {
+ queue = mana_qp_get_sq(qp)->kmem;
+ if (wqe->send_opcode == IB_WC_RDMA_READ) {
+ qp->rc_qp.queues[MANA_RC_RECV_QUEUE_REQUESTER].kmem->tail +=
+ wqe->wqe_size_in_bu;
+ queue->tail += qp->rc_qp.wqe_size_in_bu;
+ } else {
+ queue->tail += wqe->wqe_size_in_bu;
+ }
+ }
shadow_queue_advance_consumer(shadow);
}
@@ -354,7 +374,7 @@ static void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct mana_rdma_cqe *rdma_c
return;
for (; to_complete; to_complete--)
- mana_complete_send(qp, poll, VENDOR_ERR_OK);
+ mana_complete_send(qp, false, poll, VENDOR_ERR_OK);
}
static void handle_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe,
@@ -382,6 +402,21 @@ static void handle_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe,
wc->src_qp = rdma_cqe->ud_recv.src_qpn;
wc->wc_flags |= IB_WC_GRH;
break;
+ case CQE_TYPE_RC_WRITE_IMM:
+ wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
+ fallthrough;
+ case CQE_TYPE_RC_SEND_IMM:
+ wc->wc_flags |= IB_WC_WITH_IMM;
+ wc->ex.imm_data = cpu_to_be32(rdma_cqe->rc_recv.imm_data);
+ fallthrough;
+ case CQE_TYPE_RC_SEND:
+ wc->byte_len = rdma_cqe->rc_recv.msg_len;
+ break;
+ case CQE_TYPE_RC_SEND_INV:
+ wc->wc_flags |= IB_WC_WITH_INVALIDATE;
+ wc->byte_len = rdma_cqe->rc_recv.msg_len;
+ wc->ex.invalidate_rkey = rdma_cqe->rc_recv.imm_data;
+ break;
default:
break;
}
@@ -390,17 +425,105 @@ static void handle_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe,
shadow_queue_advance_consumer(&qp->shadow_rq);
}
+/*
+ * A fenced WQE can acknowledge the other send queue. Retire those dependencies
+ * first, without losing the fence when the caller's WC array fills up.
+ */
+static bool mana_advance_send(struct mana_ib_qp *qp, bool mmq, u32 seq,
+ struct mana_cq_poll *poll)
+{
+ struct shadow_queue *shadow = mmq ? &qp->shadow_mmq : &qp->shadow_sq;
+ struct shadow_queue *other = mmq ? &qp->shadow_sq : &qp->shadow_mmq;
+ struct shadow_wqe_header *shadow_wqe;
+
+ while ((shadow_wqe = shadow_queue_get_next_to_consume(shadow))) {
+ if (PSN_LT(seq, mmq ? shadow_wqe->fsn : shadow_wqe->wqe_offset_or_psn))
+ break;
+ if (poll->produced == poll->budget)
+ return false;
+ if (shadow_wqe->flags & MANA_WQ_FENCE_WC) {
+ u32 dependency = mmq ? shadow_wqe->wqe_offset_or_psn : shadow_wqe->fsn;
+ struct shadow_wqe_header *other_wqe;
+
+ while ((other_wqe = shadow_queue_get_next_to_consume(other))) {
+ if (PSN_LT(dependency, mmq ? other_wqe->wqe_offset_or_psn :
+ other_wqe->fsn))
+ break;
+ mana_complete_send(qp, !mmq, poll, VENDOR_ERR_OK);
+ if (poll->produced == poll->budget)
+ return false;
+ }
+ }
+
+ mana_complete_send(qp, mmq, poll, VENDOR_ERR_OK);
+ }
+
+ return true;
+}
+
+static bool handle_mmq_cqe(struct mana_ib_qp *qp, u32 fsn,
+ struct mana_cq_poll *poll)
+{
+ return mana_advance_send(qp, true, fsn, poll);
+}
+
+static bool handle_psn_cqe(struct mana_ib_qp *qp, u32 psn,
+ struct mana_cq_poll *poll)
+{
+ struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
+ struct shadow_wqe_header *shadow_wqe;
+ struct gdma_queue *recv_q;
+ bool consumed;
+ u32 arm_psn;
+
+ if (qp->ibqp.qp_type != IB_QPT_RC)
+ return true;
+
+ recv_q = qp->rc_qp.queues[MANA_RC_RECV_QUEUE_REQUESTER].kmem;
+
+ consumed = mana_advance_send(qp, false, psn, poll);
+ if (consumed) {
+ shadow_wqe = shadow_queue_get_next_to_signal(&qp->shadow_sq);
+ arm_psn = shadow_wqe ? shadow_wqe->wqe_offset_or_psn : psn + 1;
+ arm_psn &= MANA_WQE_OFFSET_MASK;
+ mana_gd_wq_ring_doorbell_ext(mdev_to_gc(mdev), recv_q, arm_psn,
+ SET_ARM_BIT, MANA_PSN_CLIENT_OFFSET);
+ }
+
+ return consumed;
+}
+
+static bool next_shadow_wqe_is_sq(struct shadow_wqe_header *shadow_wqe_sq,
+ struct shadow_wqe_header *shadow_wqe_mq)
+{
+ if (!shadow_wqe_mq)
+ return true;
+
+ if (!shadow_wqe_sq)
+ return false;
+
+ return (PSN_LE(shadow_wqe_sq->wqe_offset_or_psn, shadow_wqe_mq->wqe_offset_or_psn) &&
+ PSN_LE(shadow_wqe_sq->fsn, shadow_wqe_mq->fsn));
+}
+
static bool mana_handle_cqe(struct mana_ib_cq *cq, struct mana_ib_dev *mdev,
struct mana_cq_poll *poll)
{
struct gdma_comp *cqe = &cq->pending_cqe;
struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, cqe->is_sq);
+ bool consumed = true;
if (!qp)
return true;
switch (rdma_cqe->cqe_type) {
+ case CQE_TYPE_ARMED_CMPL:
+ consumed = handle_psn_cqe(qp, rdma_cqe->rc_armed_completion.psn, poll);
+ break;
+ case CQE_TYPE_LWR:
+ consumed = handle_mmq_cqe(qp, rdma_cqe->rc_mm.fsn, poll);
+ break;
case CQE_TYPE_UD_SEND:
if (cqe->is_sq) {
handle_ud_sq_cqe(qp, rdma_cqe, poll);
@@ -408,6 +531,10 @@ static bool mana_handle_cqe(struct mana_ib_cq *cq, struct mana_ib_dev *mdev,
}
fallthrough;
case CQE_TYPE_UD_SEND_IMM:
+ case CQE_TYPE_RC_SEND:
+ case CQE_TYPE_RC_SEND_IMM:
+ case CQE_TYPE_RC_SEND_INV:
+ case CQE_TYPE_RC_WRITE_IMM:
handle_rq_cqe(qp, cqe, poll);
break;
default:
@@ -416,21 +543,27 @@ static bool mana_handle_cqe(struct mana_ib_cq *cq, struct mana_ib_dev *mdev,
break;
}
mana_put_qp_ref(qp);
- return true;
+ return consumed;
}
static void mana_flush_completions(struct mana_ib_cq *cq, struct mana_cq_poll *poll)
{
- struct shadow_wqe_header *wqe;
+ struct shadow_wqe_header *sq_wqe, *mmq_wqe, *wqe;
struct mana_ib_qp *qp;
+ bool next_is_sq;
if (poll->produced >= poll->budget)
return;
list_for_each_entry(qp, &cq->send_err_qp_list, send_err_node) {
- while (poll->produced < poll->budget &&
- shadow_queue_get_next_to_consume(&qp->shadow_sq))
- mana_complete_send(qp, poll, VENDOR_ERR_SW_FLUSHED);
+ while (poll->produced < poll->budget) {
+ sq_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq);
+ mmq_wqe = shadow_queue_get_next_to_consume(&qp->shadow_mmq);
+ if (!sq_wqe && !mmq_wqe)
+ break;
+ next_is_sq = next_shadow_wqe_is_sq(sq_wqe, mmq_wqe);
+ mana_complete_send(qp, !next_is_sq, poll, VENDOR_ERR_SW_FLUSHED);
+ }
if (poll->produced == poll->budget)
return;
}
diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index f7cc9abeb071..d64d3a9c992d 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -689,6 +689,12 @@ enum mana_ib_error_code {
enum mana_ib_cqe_type {
CQE_TYPE_UD_SEND = 1,
CQE_TYPE_UD_SEND_IMM = 2,
+ CQE_TYPE_RC_SEND = 3,
+ CQE_TYPE_RC_SEND_IMM = 4,
+ CQE_TYPE_RC_SEND_INV = 5,
+ CQE_TYPE_RC_WRITE_IMM = 6,
+ CQE_TYPE_ARMED_CMPL = 7,
+ CQE_TYPE_LWR = 8,
}; /* HW DATA */
struct mana_rdma_cqe {
@@ -712,6 +718,31 @@ struct mana_rdma_cqe {
u32 imm_data;
u32 rx_wqe_offset;
} ud_recv;
+ struct {
+ u32 cqe_type : 8;
+ u32 reserved1 : 24;
+ u32 msg_len;
+ u32 psn : 24;
+ u32 reserved2 : 8;
+ u32 imm_data;
+ u32 rx_wqe_offset;
+ } rc_recv;
+ struct {
+ u32 cqe_type : 8;
+ u32 reserved1 : 24;
+ u32 msn : 24;
+ u32 syndrome : 8;
+ u32 psn : 24;
+ u32 reserved2 : 8;
+ } rc_armed_completion;
+ struct {
+ u32 cqe_type : 8;
+ u32 reserved1 : 24;
+ u32 psn : 24;
+ u32 reserved2 : 8;
+ u32 fsn : 24;
+ u32 reserved3 : 8;
+ } rc_mm;
};
}; /* HW DATA */
diff --git a/drivers/infiniband/hw/mana/shadow_queue.h b/drivers/infiniband/hw/mana/shadow_queue.h
index b5d6bb28bd45..9abf390cf3a3 100644
--- a/drivers/infiniband/hw/mana/shadow_queue.h
+++ b/drivers/infiniband/hw/mana/shadow_queue.h
@@ -26,6 +26,8 @@ struct shadow_queue {
u64 prod_idx;
/* Unmasked consumer index, Incremented on cq polling */
u64 cons_idx;
+ /* Unmasked index of next-to-signal shadow WQE */
+ u64 next_to_signal_idx;
/* queue size in wqes */
u32 length;
/* distance between elements in bytes */
@@ -50,6 +52,7 @@ static inline void reset_shadow_queue(struct shadow_queue *queue)
{
queue->prod_idx = 0;
queue->cons_idx = 0;
+ queue->next_to_signal_idx = 0;
}
static inline void destroy_shadow_queue(struct shadow_queue *queue)
@@ -105,4 +108,23 @@ static inline void shadow_queue_advance_consumer(struct shadow_queue *queue)
smp_store_release(&queue->cons_idx, queue->cons_idx + 1);
}
+static inline struct shadow_wqe_header *
+shadow_queue_get_next_to_signal(struct shadow_queue *queue)
+{
+ struct shadow_wqe_header *wqe = NULL;
+ /* Inspect only entries that have been fully published by posting. */
+ u64 prod_idx = smp_load_acquire(&queue->prod_idx);
+
+ queue->next_to_signal_idx = max(queue->next_to_signal_idx,
+ queue->cons_idx);
+ while (queue->next_to_signal_idx < prod_idx) {
+ wqe = shadow_queue_get_element(queue, queue->next_to_signal_idx);
+ queue->next_to_signal_idx++;
+ if (!(wqe->flags & MANA_WQ_NO_SIGNAL_WC))
+ return wqe;
+ }
+
+ return NULL;
+}
+
#endif
diff --git a/drivers/infiniband/hw/mana/wr.c b/drivers/infiniband/hw/mana/wr.c
index d5dec02aa70a..8cec2e621fa7 100644
--- a/drivers/infiniband/hw/mana/wr.c
+++ b/drivers/infiniband/hw/mana/wr.c
@@ -415,6 +415,9 @@ static int mana_ib_post_sq(struct mana_ib_qp *qp, const struct ib_send_wr *wr)
if (err)
return err;
+ if (wr->opcode != IB_WR_RDMA_READ)
+ shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu;
+
shadow_wqe->wr_id = wr->wr_id;
shadow_wqe->send_opcode = convert_wr_to_wc(wr->opcode);
if (!signaled)
--
2.43.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH rdma-next v5 08/10] RDMA/mana_ib: Flush and notify CQs when kernel QPs enter ERR
2026-10-17 6:44 [PATCH rdma-next v5 00/10] RDMA/mana_ib: Add kernel RC and fast registration support Konstantin Taranov
` (6 preceding siblings ...)
2026-10-17 6:44 ` [PATCH rdma-next v5 07/10] RDMA/mana_ib: Poll RC completions using PSN and FSN progress Konstantin Taranov
@ 2026-10-17 6:44 ` Konstantin Taranov
2026-10-17 7:12 ` sashiko-bot
2026-10-17 6:44 ` [PATCH rdma-next v5 09/10] RDMA/mana_ib: Handle error CQEs for RC QPs Konstantin Taranov
2026-10-17 6:44 ` [PATCH rdma-next v5 10/10] RDMA/mana_ib: Drain kernel receive and send queues Konstantin Taranov
9 siblings, 1 reply; 21+ messages in thread
From: Konstantin Taranov @ 2026-10-17 6:44 UTC (permalink / raw)
To: kotaranov, snsanghvi, longli, jgg, leon; +Cc: linux-rdma, linux-kernel
From: Konstantin Taranov <kotaranov@microsoft.com>
Complete the RC software flush path by adding a kernel QP to the CQ
error lists when it transitions to the ERR state.
Invoke the CQ handlers after the transition to flush pending work
requests.
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
---
v5:
- Remove the unused QP state field.
v3:
- Skip kernel state update handling for userspace QPs.
- Warn and refuse to add userspace QPs to the CQ error lists.
- Add mana_remove_qp_from_cqs() for UC
v2:
- Skip software CQ notifications for IB_POLL_DIRECT.
drivers/infiniband/hw/mana/cq.c | 2 +-
drivers/infiniband/hw/mana/qp.c | 53 ++++++++++++++++++++++++++++++---
2 files changed, 50 insertions(+), 5 deletions(-)
diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c
index a4d886b9ecc6..90a43c7feb31 100644
--- a/drivers/infiniband/hw/mana/cq.c
+++ b/drivers/infiniband/hw/mana/cq.c
@@ -589,7 +589,7 @@ static void mana_drain_gsi_sq(struct mana_ib_qp *qp)
list_add_tail(&qp->send_err_node, &cq->send_err_qp_list);
spin_unlock_irqrestore(&cq->cq_lock, flags);
- if (cq->ibcq.comp_handler)
+ if (cq->ibcq.poll_ctx != IB_POLL_DIRECT && cq->ibcq.comp_handler)
cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
}
diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
index 5d6f45b9f53d..18242e1a6360 100644
--- a/drivers/infiniband/hw/mana/qp.c
+++ b/drivers/infiniband/hw/mana/qp.c
@@ -770,6 +770,27 @@ destroy_queues:
return err;
}
+static void mana_add_qp_to_error_cqs(struct mana_ib_qp *qp)
+{
+ struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
+ struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq);
+ unsigned long flags;
+
+ if (WARN_ON_ONCE(qp->ibqp.uobject))
+ return;
+
+ /* Keep ERR QPs linked until reset/destroy, including later drain WRs. */
+ spin_lock_irqsave(&send_cq->cq_lock, flags);
+ if (list_empty(&qp->send_err_node))
+ list_add_tail(&qp->send_err_node, &send_cq->send_err_qp_list);
+ spin_unlock_irqrestore(&send_cq->cq_lock, flags);
+
+ spin_lock_irqsave(&recv_cq->cq_lock, flags);
+ if (list_empty(&qp->recv_err_node))
+ list_add_tail(&qp->recv_err_node, &recv_cq->recv_err_qp_list);
+ spin_unlock_irqrestore(&recv_cq->cq_lock, flags);
+}
+
static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp, bool reset)
{
struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
@@ -1070,15 +1091,16 @@ static int mana_ib_gd_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
return 0;
}
-static void mana_ib_modify_qp_state(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+static bool mana_ib_modify_qp_state(struct ib_qp *ibqp, struct ib_qp_attr *attr,
int attr_mask, struct ib_udata *udata)
{
struct mana_ib_dev *mdev = container_of(ibqp->device, struct mana_ib_dev, ib_dev);
struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
struct gdma_queue *rq;
+ bool notify = false;
- if (udata)
- return;
+ if (udata || ibqp->uobject)
+ return false;
if (attr_mask & IB_QP_PATH_MTU)
qp->mtu = attr->path_mtu;
@@ -1091,6 +1113,10 @@ static void mana_ib_modify_qp_state(struct ib_qp *ibqp, struct ib_qp_attr *attr,
default:
break;
}
+ if (attr->qp_state == IB_QPS_ERR) {
+ mana_add_qp_to_error_cqs(qp);
+ notify = true;
+ }
}
if (attr_mask & IB_QP_SQ_PSN) {
@@ -1106,12 +1132,27 @@ static void mana_ib_modify_qp_state(struct ib_qp *ibqp, struct ib_qp_attr *attr,
SET_ARM_BIT, MANA_PSN_CLIENT_OFFSET);
}
}
+
+ return notify;
+}
+
+static void mana_ib_notify_error_cqs(struct mana_ib_qp *qp)
+{
+ struct ib_cq *send_cq = qp->ibqp.send_cq;
+ struct ib_cq *recv_cq = qp->ibqp.recv_cq;
+
+ if (send_cq->poll_ctx != IB_POLL_DIRECT && send_cq->comp_handler)
+ send_cq->comp_handler(send_cq, send_cq->cq_context);
+ if (recv_cq != send_cq && recv_cq->poll_ctx != IB_POLL_DIRECT &&
+ recv_cq->comp_handler)
+ recv_cq->comp_handler(recv_cq, recv_cq->cq_context);
}
int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
int attr_mask, struct ib_udata *udata)
{
struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+ bool notify = false;
int ret;
mutex_lock(&qp->modify_lock);
@@ -1130,11 +1171,14 @@ int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
goto out_unlock;
}
- mana_ib_modify_qp_state(ibqp, attr, attr_mask, udata);
+ notify = mana_ib_modify_qp_state(ibqp, attr, attr_mask, udata);
out_unlock:
mutex_unlock(&qp->modify_lock);
+ if (notify)
+ mana_ib_notify_error_cqs(qp);
+
return ret;
}
@@ -1256,6 +1300,7 @@ static int mana_ib_destroy_uc_qp(struct mana_ib_qp *qp, struct ib_udata *udata)
return err;
mana_table_remove_qp(mdev, qp);
+ mana_remove_qp_from_cqs(qp, false);
/* Ignore return code as there is not much we can do about it.
* The error message is printed inside.
*/
--
2.43.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH rdma-next v5 08/10] RDMA/mana_ib: Flush and notify CQs when kernel QPs enter ERR
2026-10-17 6:44 ` [PATCH rdma-next v5 08/10] RDMA/mana_ib: Flush and notify CQs when kernel QPs enter ERR Konstantin Taranov
@ 2026-10-17 7:12 ` sashiko-bot
0 siblings, 0 replies; 21+ messages in thread
From: sashiko-bot @ 2026-10-17 7:12 UTC (permalink / raw)
To: Konstantin Taranov; +Cc: linux-rdma
Thank you for your contribution! Sashiko AI review found 3 potential issue(s) to consider:
- [High] RDMA/mana_ib: missing CQ notification for WRs posted after IB_QPS_ERR
- [High] RDMA/mana_ib: lost software flush notification in mana_ib_arm_cq()
- [Medium] RDMA/mana_ib: missing IB_POLL_DIRECT check in mana_ib_post_send_ud()
--
commit 254e0a6be9165a866ac991ad943bce590f88dfb0
Author: Konstantin Taranov <kotaranov@microsoft.com>
RDMA/mana_ib: Flush and notify CQs when kernel QPs enter ERR
This commit links kernel QPs onto the CQ error lists when transitioning to
IB_QPS_ERR and invokes the CQ completion handlers so pending work requests
can be software-flushed.
> diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c
> index cd77e305c38f9..080f09f3c634e 100644
> --- a/drivers/infiniband/hw/mana/cq.c
> +++ b/drivers/infiniband/hw/mana/cq.c
> @@ -589,7 +589,7 @@ static void mana_drain_gsi_sq(struct mana_ib_qp *qp)
> list_add_tail(&qp->send_err_node, &cq->send_err_qp_list);
> spin_unlock_irqrestore(&cq->cq_lock, flags);
>
> - if (cq->ibcq.comp_handler)
> + if (cq->ibcq.poll_ctx != IB_POLL_DIRECT && cq->ibcq.comp_handler)
> cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
> }
[Severity: Medium]
Should mana_ib_post_send_ud() in drivers/infiniband/hw/mana/wr.c also check
for IB_POLL_DIRECT before invoking comp_handler?
With this commit, mana_ib_modify_qp_state() calls mana_add_qp_to_error_cqs()
for any kernel QP entering IB_QPS_ERR, including kernel UD QPs, which links
qp->send_err_node onto send_cq->send_err_qp_list.
When mana_ib_post_send_ud() subsequently posts a WR on that QP, it checks
send_err_node and calls comp_handler without checking poll_ctx:
drivers/infiniband/hw/mana/wr.c:mana_ib_post_send_ud() {
...
spin_lock_irqsave(&cq->cq_lock, flags);
flush_send = !list_empty(&qp->send_err_node);
spin_unlock_irqrestore(&cq->cq_lock, flags);
if (flush_send && cq->ibcq.comp_handler)
cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
}
If the kernel UD QP uses a send CQ created with IB_POLL_DIRECT, will calling
comp_handler invoke ib_cq_completion_direct() and trigger its WARN_ONCE()?
> diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
> index cd2bc39fce6ec..e8c479c96cbf7 100644
> --- a/drivers/infiniband/hw/mana/qp.c
> +++ b/drivers/infiniband/hw/mana/qp.c
> @@ -771,6 +771,27 @@ static int mana_ib_create_uc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
> return err;
> }
>
> +static void mana_add_qp_to_error_cqs(struct mana_ib_qp *qp)
> +{
> + struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
> + struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq);
> + unsigned long flags;
> +
> + if (WARN_ON_ONCE(qp->ibqp.uobject))
> + return;
> +
> + /* Keep ERR QPs linked until reset/destroy, including later drain WRs. */
> + spin_lock_irqsave(&send_cq->cq_lock, flags);
> + if (list_empty(&qp->send_err_node))
> + list_add_tail(&qp->send_err_node, &send_cq->send_err_qp_list);
> + spin_unlock_irqrestore(&send_cq->cq_lock, flags);
> +
> + spin_lock_irqsave(&recv_cq->cq_lock, flags);
> + if (list_empty(&qp->recv_err_node))
> + list_add_tail(&qp->recv_err_node, &recv_cq->recv_err_qp_list);
> + spin_unlock_irqrestore(&recv_cq->cq_lock, flags);
> +}
[Severity: High]
Can work requests posted concurrently with or after the transition to
IB_QPS_ERR be left unpolled in the shadow queues?
Once the QP is in IB_QPS_ERR in hardware, MANA hardware ignores WQ doorbells
for that QP and generates no hardware CQEs for subsequently posted WRs.
Unlike mana_ib_post_send_ud(), which checks !list_empty(&qp->send_err_node)
under cq->cq_lock and invokes cq->ibcq.comp_handler after ringing the
doorbell, mana_ib_post_send_rc(), mana_ib_post_recv_rc(), and
mana_ib_post_recv_ud() in drivers/infiniband/hw/mana/wr.c only queue the WR
to shadow_sq, shadow_mmq, or shadow_rq and ring the doorbell without checking
send_err_node or recv_err_node or invoking comp_handler:
drivers/infiniband/hw/mana/wr.c:mana_ib_post_send_rc() {
...
if (ring_sq)
mana_gd_wq_ring_doorbell(mdev_to_gc(mdev),
qp->rc_qp.queues[MANA_RC_SEND_QUEUE_REQUESTER].kmem);
spin_unlock_irqrestore(&qp->sq_lock, flags);
return err;
}
At this commit, __ib_drain_sq() and __ib_drain_rq() call
ib_modify_qp(IB_QPS_ERR) and then post the drain WR via ib_post_send() or
ib_post_recv() without calling comp_handler again. If the CQ poll triggered
by mana_ib_notify_error_cqs() runs and finishes mana_ib_poll_cq() before the
drain WR is posted to shadow_sq or shadow_rq, wait_for_completion() hangs.
Even with the later commits in this series ("RDMA/mana_ib: Drain kernel
receive and send queues", which adds custom mana_ib_drain_sq() and
mana_ib_drain_rq() callbacks that call cq->comp_handler after posting the
drain WR, and "RDMA/mana_ib: Handle error CQEs for RC QPs", which adds
mana_ib_qp_error_work), any WR posted concurrently with or after
mana_ib_modify_qp(IB_QPS_ERR) outside mana_ib_drain_sq() or
mana_ib_drain_rq() — for example when mana_ib_qp_error_work asynchronously
moves the QP to IB_QPS_ERR while a ULP is posting WRs, or when a ULP moves
the QP to IB_QPS_ERR and posts a WR directly — will be queued to the shadow
queue after mana_ib_notify_error_cqs() without re-triggering CQ polling.
Should mana_ib_post_send_rc(), mana_ib_post_recv_rc(), and
mana_ib_post_recv_ud() also check whether the QP is on the CQ error list and
notify the CQ, similar to mana_ib_post_send_ud()?
[ ... ]
> @@ -1107,12 +1133,27 @@ static void mana_ib_modify_qp_state(struct ib_qp *ibqp, struct ib_qp_attr *attr,
> SET_ARM_BIT, MANA_PSN_CLIENT_OFFSET);
> }
> }
> +
> + return notify;
> +}
> +
> +static void mana_ib_notify_error_cqs(struct mana_ib_qp *qp)
> +{
> + struct ib_cq *send_cq = qp->ibqp.send_cq;
> + struct ib_cq *recv_cq = qp->ibqp.recv_cq;
> +
> + if (send_cq->poll_ctx != IB_POLL_DIRECT && send_cq->comp_handler)
> + send_cq->comp_handler(send_cq, send_cq->cq_context);
> + if (recv_cq != send_cq && recv_cq->poll_ctx != IB_POLL_DIRECT &&
> + recv_cq->comp_handler)
> + recv_cq->comp_handler(recv_cq, recv_cq->cq_context);
> }
[Severity: High]
Can the software flush notification in mana_ib_notify_error_cqs() be lost on
IB_POLL_SOFTIRQ CQs if ib_poll_handler() is concurrently finishing a poll?
For IB_POLL_SOFTIRQ CQs, irq_poll_softirq() keeps IRQ_POLL_F_SCHED set in
cq->iop.state throughout __ib_process_cq() until ib_poll_handler() calls
irq_poll_complete(&cq->iop). Consider the following sequence:
CPU 0 CPU 1
ib_poll_handler()
__ib_process_cq()
mana_ib_poll_cq()
mana_flush_completions()
spin_unlock_irqrestore(&cq->cq_lock)
/* executing wc->wr_cqe->done() */
mana_ib_modify_qp()
mana_add_qp_to_error_cqs()
mana_ib_notify_error_cqs()
ib_cq_completion_softirq()
irq_poll_sched(&cq->iop)
/* IRQ_POLL_F_SCHED still set,
* returns without scheduling */
irq_poll_complete(&cq->iop)
ib_req_notify_cq(cq,
IB_CQ_NEXT_COMP |
IB_CQ_REPORT_MISSED_EVENTS)
mana_ib_arm_cq()
In mana_ib_arm_cq() in drivers/infiniband/hw/mana/cq.c,
IB_CQ_REPORT_MISSED_EVENTS only checks cq->has_pending_cqe:
drivers/infiniband/hw/mana/cq.c:mana_ib_arm_cq() {
...
spin_lock_irqsave(&cq->cq_lock, irq_flags);
mana_ib_cq_doorbell(cq, SET_ARM_BIT);
if (flags & IB_CQ_REPORT_MISSED_EVENTS)
missed = cq->has_pending_cqe;
spin_unlock_irqrestore(&cq->cq_lock, irq_flags);
return missed ? 1 : 0;
}
Because mana_ib_arm_cq() does not check whether send_err_qp_list or
recv_err_qp_list has unconsumed WQEs in shadow_sq, shadow_mmq, or shadow_rq,
and software-flushed WRs do not generate hardware CQ interrupts, won't
mana_ib_arm_cq() return 0 and leave the error QP's pending WRs (or a drain WR
notified via comp_handler during that window) unpolled?
[ ... ]
>
>base-commit: 978361dd2ccff357d4fe6aac6ce057d1bfae8312
([978361dd2ccf: RDMA/mana_ib: Implement software flushing of RC QPs])
([254e0a6be916: RDMA/mana_ib: Flush and notify CQs when kernel QPs enter ERR])
([daeb2bf3de14: RDMA/mana_ib: Drain kernel receive and send queues])
([33d06de2231e: RDMA/mana_ib: Handle error CQEs for RC QPs])
([a966dc60ab42: RDMA/mana_ib: Report port events for RC])
([acdb8e2b2de9: RDMA/mana_ib: Implement modify_qp for UC])
([763b5797538e: RDMA/mana_ib: Set the correct syndrome properly in CQE])
([32de7988de0a: RDMA/mana_ib: Configure interrupt coalescing timer for RC])
([bc57232325f5: RDMA/mana_ib: Set RC transport capabilities])
([737b4ca9ed86: RDMA/mana_ib: Enable the RC support])
--
2.51.0
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261017064413.995985-1-kotaranov@linux.microsoft.com?part=8
^ permalink raw reply [flat|nested] 21+ messages in thread
* [PATCH rdma-next v5 09/10] RDMA/mana_ib: Handle error CQEs for RC QPs
2026-10-17 6:44 [PATCH rdma-next v5 00/10] RDMA/mana_ib: Add kernel RC and fast registration support Konstantin Taranov
` (7 preceding siblings ...)
2026-10-17 6:44 ` [PATCH rdma-next v5 08/10] RDMA/mana_ib: Flush and notify CQs when kernel QPs enter ERR Konstantin Taranov
@ 2026-10-17 6:44 ` Konstantin Taranov
2026-10-17 7:23 ` sashiko-bot
2026-10-17 6:44 ` [PATCH rdma-next v5 10/10] RDMA/mana_ib: Drain kernel receive and send queues Konstantin Taranov
9 siblings, 1 reply; 21+ messages in thread
From: Konstantin Taranov @ 2026-10-17 6:44 UTC (permalink / raw)
To: kotaranov, snsanghvi, longli, jgg, leon; +Cc: linux-rdma, linux-kernel
From: Konstantin Taranov <kotaranov@microsoft.com>
Decode hardware error CQEs and process them on the corresponding work
queue. Complete the failing shadow entry with the vendor status from
the CQE.
Defer transitioning RC QPs to the ERR state so that the other work
queues can be flushed.
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
---
v2:
- Flush error_work after draining QP references during teardown.
drivers/infiniband/hw/mana/cq.c | 91 ++++++++++++++++++++++++++++
drivers/infiniband/hw/mana/mana_ib.h | 41 ++++++++++++-
drivers/infiniband/hw/mana/qp.c | 18 ++++++
3 files changed, 149 insertions(+), 1 deletion(-)
diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c
index 90a43c7feb31..63c92a028785 100644
--- a/drivers/infiniband/hw/mana/cq.c
+++ b/drivers/infiniband/hw/mana/cq.c
@@ -18,6 +18,7 @@ static enum ib_wc_status vendor_error_to_wc_error(uint32_t vendor_error)
return IB_WC_SUCCESS;
case VENDOR_ERR_RX_PKT_LEN:
case VENDOR_ERR_RX_MSG_LEN_OVFL:
+ case VENDOR_ERR_RX_READRESP_LEN_MISMATCH:
return IB_WC_LOC_LEN_ERR;
case VENDOR_ERR_TX_GDMA_CORRUPTED_WQE:
case VENDOR_ERR_TX_PCIE_WQE:
@@ -37,12 +38,30 @@ static enum ib_wc_status vendor_error_to_wc_error(uint32_t vendor_error)
case VENDOR_ERR_TX_ATB_WQE_ACCESS_VIOLATION:
case VENDOR_ERR_TX_ATB_WQE_ADDR_RANGE:
case VENDOR_ERR_TX_ATB_WQE_CONFIG_ERR:
+ case VENDOR_ERR_TX_RDMA_ATB_CMD_MISS:
+ case VENDOR_ERR_TX_RDMA_ATB_CMD_IDX_ERROR:
+ case VENDOR_ERR_TX_RDMA_ATB_CMD_TAG_MISMATCH_ERROR:
+ case VENDOR_ERR_TX_RDMA_ATB_CMD_PDID_MISMATCH_ERROR:
+ case VENDOR_ERR_TX_RDMA_ATB_CMD_AR_ERROR:
+ case VENDOR_ERR_TX_RDMA_ATB_CMD_PT_OVF:
+ case VENDOR_ERR_TX_RDMA_ATB_CMD_PT_LENGHT_MISMATCH:
+ case VENDOR_ERR_TX_RDMA_ATB_CMD_ILLEGAL_CMD:
case VENDOR_ERR_RX_ATB_SGE_ADDR_RANGE:
case VENDOR_ERR_RX_ATB_SGE_MISSCONFIG:
return IB_WC_LOC_PROT_ERR;
case VENDOR_ERR_RX_ATB_SGE_ADDR_RIGHT:
case VENDOR_ERR_RX_GFID:
return IB_WC_LOC_ACCESS_ERR;
+ case VENDOR_ERR_RX_OP_REQ:
+ return IB_WC_REM_INV_REQ_ERR;
+ case VENDOR_ERR_RX_ATB_RKEY_MISCONFIG_ERR:
+ case VENDOR_ERR_RX_ATB_RKEY_ADDR_RIGHT:
+ case VENDOR_ERR_RX_ATB_RKEY_ADDR_RANGE:
+ case VENDOR_ERR_RX_REMOTE_ACCESS_NAK:
+ return IB_WC_REM_ACCESS_ERR;
+ case VENDOR_ERR_RX_INVALID_REQ_NAK:
+ case VENDOR_ERR_RX_REMOTE_OP_ERR_NAK:
+ return IB_WC_REM_OP_ERR;
case VENDOR_ERR_RX_MISBEHAVING_CLIENT:
case VENDOR_ERR_RX_CLIENT_ID:
case VENDOR_ERR_RX_PCIE:
@@ -58,6 +77,10 @@ static enum ib_wc_status vendor_error_to_wc_error(uint32_t vendor_error)
case VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE:
case VENDOR_ERR_SW_FLUSHED:
return IB_WC_WR_FLUSH_ERR;
+ case VENDOR_ERR_TX_RETRY_LIMIT_EXCEEDED:
+ return IB_WC_RETRY_EXC_ERR;
+ case VENDOR_ERR_RX_RNR_NAK:
+ return IB_WC_RNR_RETRY_EXC_ERR;
default:
return IB_WC_GENERAL_ERR;
}
@@ -493,6 +516,71 @@ static bool handle_psn_cqe(struct mana_ib_qp *qp, u32 psn,
return consumed;
}
+static bool mana_cqe_is_send(struct mana_ib_qp *qp, struct gdma_comp *cqe)
+{
+ u32 qtype = cqe->wq_num & MANA_QID_SUBTYPE_MASK;
+
+ if (qp->ibqp.qp_type != IB_QPT_RC)
+ return cqe->is_sq;
+
+ return cqe->is_sq ? (qtype == MANA_QID_SUBTYPE_SREQ ||
+ qtype == MANA_QID_SUBTYPE_MMQ) :
+ qtype == MANA_QID_SUBTYPE_RREQ;
+}
+
+static void mana_ib_schedule_qp_error(struct mana_ib_qp *qp)
+{
+ /* CQ polling cannot sleep; teardown drains this reference before freeing. */
+ refcount_inc(&qp->refcount);
+ if (!schedule_work(&qp->error_work))
+ mana_put_qp_ref(qp);
+}
+
+static bool handle_error_cqe(struct mana_ib_cq *cq, struct mana_ib_qp *qp,
+ struct gdma_comp *cqe, struct mana_cq_poll *poll)
+{
+ bool mmq = cqe->is_sq && (cqe->wq_num & MANA_QID_SUBTYPE_MASK) == MANA_QID_SUBTYPE_MMQ;
+ struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
+ bool send = mana_cqe_is_send(qp, cqe);
+ u32 vendor_err = rdma_cqe->error.vendor_error;
+ struct shadow_wqe_header *shadow_wqe;
+ struct shadow_queue *shadow_q;
+
+ if (send && qp->ibqp.qp_type == IB_QPT_RC) {
+ /* The error PSN/FSN belongs to the failing WQE, not a success. */
+ if (!mana_advance_send(qp, mmq, rdma_cqe->error.psn - 1, poll))
+ return false;
+ if (poll->produced == poll->budget)
+ return false;
+ }
+
+ if (!send)
+ shadow_q = &qp->shadow_rq;
+ else if (mmq)
+ shadow_q = &qp->shadow_mmq;
+ else
+ shadow_q = &qp->shadow_sq;
+
+ shadow_wqe = shadow_queue_get_next_to_consume(shadow_q);
+ if (shadow_wqe) {
+ mana_fill_wc(qp, poll, shadow_wqe,
+ send ? shadow_wqe->send_opcode : IB_WC_RECV, vendor_err);
+ shadow_queue_advance_consumer(shadow_q);
+ }
+
+ if (send) {
+ if (list_empty(&qp->send_err_node))
+ list_add_tail(&qp->send_err_node, &cq->send_err_qp_list);
+ } else {
+ if (list_empty(&qp->recv_err_node))
+ list_add_tail(&qp->recv_err_node, &cq->recv_err_qp_list);
+ }
+ if (qp->ibqp.qp_type != IB_QPT_GSI)
+ mana_ib_schedule_qp_error(qp);
+
+ return true;
+}
+
static bool next_shadow_wqe_is_sq(struct shadow_wqe_header *shadow_wqe_sq,
struct shadow_wqe_header *shadow_wqe_mq)
{
@@ -524,6 +612,9 @@ static bool mana_handle_cqe(struct mana_ib_cq *cq, struct mana_ib_dev *mdev,
case CQE_TYPE_LWR:
consumed = handle_mmq_cqe(qp, rdma_cqe->rc_mm.fsn, poll);
break;
+ case CQE_TYPE_ERROR:
+ consumed = handle_error_cqe(cq, qp, cqe, poll);
+ break;
case CQE_TYPE_UD_SEND:
if (cqe->is_sq) {
handle_ud_sq_cqe(qp, rdma_cqe, poll);
diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index d64d3a9c992d..3a6a4ad8e641 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -12,6 +12,7 @@
#include <rdma/mana-abi.h>
#include <rdma/uverbs_ioctl.h>
#include <linux/dmapool.h>
+#include <linux/workqueue.h>
#include <net/mana/mana.h>
#include "shadow_queue.h"
@@ -33,6 +34,11 @@
/* Queue ID encodes type in the lower 2 bits */
#define MANA_QID_SUBTYPE_MASK 0x3
+#define MANA_QID_SUBTYPE_SREQ 0x0
+#define MANA_QID_SUBTYPE_MMQ 0x2
+#define MANA_QID_SUBTYPE_RRESP 0x0
+#define MANA_QID_SUBTYPE_RREQ 0x1
+
/*
* The hardware limit of number of MRs is greater than maximum number of MRs
* that can possibly represent in 24 bits
@@ -276,8 +282,9 @@ struct mana_ib_qp {
spinlock_t rq_lock;
/* Serializes send and memory-management WR posting. */
spinlock_t sq_lock;
- /* Serializes QP modification and error-list transitions. */
+ /* Serializes QP modification and deferred error work. */
struct mutex modify_lock;
+ struct work_struct error_work;
struct list_head send_err_node;
struct list_head recv_err_node;
@@ -646,20 +653,30 @@ struct rdma_recv_oob {
enum mana_ib_error_code {
VENDOR_ERR_OK = 0x0,
+ VENDOR_ERR_RX_OP_REQ = 0x03,
VENDOR_ERR_RX_PKT_LEN = 0x05,
+ VENDOR_ERR_RX_ATB_RKEY_MISCONFIG_ERR = 0x43,
+ VENDOR_ERR_RX_ATB_RKEY_ADDR_RIGHT = 0x83,
+ VENDOR_ERR_RX_ATB_RKEY_ADDR_RANGE = 0xc3,
VENDOR_ERR_RX_MSG_LEN_OVFL = 0x102,
VENDOR_ERR_RX_MISBEHAVING_CLIENT = 0x108,
VENDOR_ERR_RX_MALFORMED_WQE = 0x109,
VENDOR_ERR_RX_CLIENT_ID = 0x10a,
VENDOR_ERR_RX_GFID = 0x10b,
+ VENDOR_ERR_RX_READRESP_LEN_MISMATCH = 0x10f,
VENDOR_ERR_RX_PCIE = 0x10c,
VENDOR_ERR_RX_NO_AVAIL_WQE = 0x111,
VENDOR_ERR_RX_ATB_SGE_MISSCONFIG = 0x143,
VENDOR_ERR_RX_ATB_WQE_MISCONFIG = 0x145,
+ VENDOR_ERR_RX_INVALID_REQ_NAK = 0x161,
+ VENDOR_ERR_RX_REMOTE_ACCESS_NAK = 0x162,
+ VENDOR_ERR_RX_REMOTE_OP_ERR_NAK = 0x163,
+ VENDOR_ERR_RX_RNR_NAK = 0x164,
VENDOR_ERR_RX_ATB_SGE_ADDR_RIGHT = 0x183,
VENDOR_ERR_RX_ATB_WQE_ADDR_RIGHT = 0x185,
VENDOR_ERR_RX_ATB_SGE_ADDR_RANGE = 0x1c3,
VENDOR_ERR_RX_ATB_WQE_ADDR_RANGE = 0x1c5,
+ VENDOR_ERR_TX_RETRY_LIMIT_EXCEEDED = 0x1c6,
VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE = 0x1c7,
VENDOR_ERR_TX_GDMA_CORRUPTED_WQE = 0x201,
VENDOR_ERR_TX_ATB_WQE_ACCESS_VIOLATION = 0x202,
@@ -681,6 +698,14 @@ enum mana_ib_error_code {
VENDOR_ERR_TX_RDMA_WQE_LEN_ERR = 0x216,
VENDOR_ERR_TX_RDMA_MTU_ERR = 0x217,
VENDOR_ERR_TX_RDMA_VFID_MISMATCH = 0x218,
+ VENDOR_ERR_TX_RDMA_ATB_CMD_MISS = 0x220,
+ VENDOR_ERR_TX_RDMA_ATB_CMD_IDX_ERROR = 0x221,
+ VENDOR_ERR_TX_RDMA_ATB_CMD_TAG_MISMATCH_ERROR = 0x222,
+ VENDOR_ERR_TX_RDMA_ATB_CMD_PDID_MISMATCH_ERROR = 0x223,
+ VENDOR_ERR_TX_RDMA_ATB_CMD_AR_ERROR = 0x224,
+ VENDOR_ERR_TX_RDMA_ATB_CMD_PT_OVF = 0x225,
+ VENDOR_ERR_TX_RDMA_ATB_CMD_PT_LENGHT_MISMATCH = 0x226,
+ VENDOR_ERR_TX_RDMA_ATB_CMD_ILLEGAL_CMD = 0x227,
VENDOR_ERR_HW_MAX = 0x3ff,
/* SW vendor errors */
VENDOR_ERR_SW_FLUSHED = 0xfff,
@@ -695,6 +720,7 @@ enum mana_ib_cqe_type {
CQE_TYPE_RC_WRITE_IMM = 6,
CQE_TYPE_ARMED_CMPL = 7,
CQE_TYPE_LWR = 8,
+ CQE_TYPE_ERROR = 34,
}; /* HW DATA */
struct mana_rdma_cqe {
@@ -727,6 +753,19 @@ struct mana_rdma_cqe {
u32 imm_data;
u32 rx_wqe_offset;
} rc_recv;
+ struct {
+ u32 cqe_type : 8;
+ u32 vendor_error : 10;
+ u32 reserved1 : 14;
+ u32 msn : 24;
+ u32 syndrome : 8;
+ u32 psn : 24;
+ u32 opcode : 8;
+ u32 rsp_msn : 24;
+ u32 reserved2 : 8;
+ u32 rsp_psn : 24;
+ u32 reserved3 : 8;
+ } error;
struct {
u32 cqe_type : 8;
u32 reserved1 : 24;
diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
index 18242e1a6360..519a161b994d 100644
--- a/drivers/infiniband/hw/mana/qp.c
+++ b/drivers/infiniband/hw/mana/qp.c
@@ -5,6 +5,8 @@
#include "mana_ib.h"
+static void mana_ib_qp_error_work(struct work_struct *work);
+
static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,
struct net_device *ndev,
mana_handle_t default_rxobj,
@@ -546,6 +548,7 @@ static void mana_table_drain_qp_ref(struct mana_ib_qp *qp)
{
mana_put_qp_ref(qp);
wait_for_completion(&qp->free);
+ flush_work(&qp->error_work);
}
static int mana_table_store_qp_qid(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
@@ -988,6 +991,7 @@ int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
spin_lock_init(&qp->rq_lock);
spin_lock_init(&qp->sq_lock);
mutex_init(&qp->modify_lock);
+ INIT_WORK(&qp->error_work, mana_ib_qp_error_work);
INIT_LIST_HEAD(&qp->send_err_node);
INIT_LIST_HEAD(&qp->recv_err_node);
@@ -1182,6 +1186,20 @@ out_unlock:
return ret;
}
+static void mana_ib_qp_error_work(struct work_struct *work)
+{
+ struct mana_ib_qp *qp = container_of(work, struct mana_ib_qp, error_work);
+ struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
+ int ret;
+
+ ret = mana_ib_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL);
+ if (ret)
+ ibdev_err(qp->ibqp.device, "Failed to move QP %u to ERR: %d\n",
+ qp->ibqp.qp_num, ret);
+
+ mana_put_qp_ref(qp);
+}
+
static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp,
struct ib_rwq_ind_table *ind_tbl,
struct ib_udata *udata)
--
2.43.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH rdma-next v5 09/10] RDMA/mana_ib: Handle error CQEs for RC QPs
2026-10-17 6:44 ` [PATCH rdma-next v5 09/10] RDMA/mana_ib: Handle error CQEs for RC QPs Konstantin Taranov
@ 2026-10-17 7:23 ` sashiko-bot
0 siblings, 0 replies; 21+ messages in thread
From: sashiko-bot @ 2026-10-17 7:23 UTC (permalink / raw)
To: Konstantin Taranov; +Cc: linux-rdma
Thank you for your contribution! Sashiko AI review found 5 potential issue(s) to consider:
- [High] RDMA/mana_ib: handle_error_cqe() fails to retire fenced WQE dependencies
- [High] RDMA/mana_ib: mana_ib_modify_qp() does not cancel error_work on QP reset
- [High] RDMA/mana_ib: handle_error_cqe() adds active GSI QPs to CQ error lists
- [High] RDMA/mana_ib: unchecked shadow queue advancement in handle_error_cqe()
- [High] RDMA/mana_ib: premature WR software flush in handle_error_cqe() before HW ERR
--
commit 75bf7e7dda34f9cbdab6ae2d0e0c7423140a1078
Author: Konstantin Taranov <kotaranov@microsoft.com>
RDMA/mana_ib: Handle error CQEs for RC QPs
This commit adds hardware error CQE decoding for RC QPs, completing the
failing shadow entry with the CQE vendor status and deferring the QP
transition to the error state via a workqueue so other work queues can be
flushed.
> diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c
> index 080f09f3c634e..a66fe0cc92b4a 100644
> --- a/drivers/infiniband/hw/mana/cq.c
> +++ b/drivers/infiniband/hw/mana/cq.c
[ ... ]
> @@ -493,6 +516,71 @@ static bool handle_psn_cqe(struct mana_ib_qp *qp, u32 psn,
> return consumed;
> }
>
> +static bool mana_cqe_is_send(struct mana_ib_qp *qp, struct gdma_comp *cqe)
> +{
> + u32 qtype = cqe->wq_num & MANA_QID_SUBTYPE_MASK;
> +
> + if (qp->ibqp.qp_type != IB_QPT_RC)
> + return cqe->is_sq;
> +
> + return cqe->is_sq ? (qtype == MANA_QID_SUBTYPE_SREQ ||
> + qtype == MANA_QID_SUBTYPE_MMQ) :
> + qtype == MANA_QID_SUBTYPE_RREQ;
> +}
> +
> +static void mana_ib_schedule_qp_error(struct mana_ib_qp *qp)
> +{
> + /* CQ polling cannot sleep; teardown drains this reference before freeing. */
> + refcount_inc(&qp->refcount);
> + if (!schedule_work(&qp->error_work))
> + mana_put_qp_ref(qp);
> +}
> +
> +static bool handle_error_cqe(struct mana_ib_cq *cq, struct mana_ib_qp *qp,
> + struct gdma_comp *cqe, struct mana_cq_poll *poll)
> +{
> + bool mmq = cqe->is_sq && (cqe->wq_num & MANA_QID_SUBTYPE_MASK) == MANA_QID_SUBTYPE_MMQ;
> + struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
> + bool send = mana_cqe_is_send(qp, cqe);
> + u32 vendor_err = rdma_cqe->error.vendor_error;
> + struct shadow_wqe_header *shadow_wqe;
> + struct shadow_queue *shadow_q;
> +
> + if (send && qp->ibqp.qp_type == IB_QPT_RC) {
> + /* The error PSN/FSN belongs to the failing WQE, not a success. */
> + if (!mana_advance_send(qp, mmq, rdma_cqe->error.psn - 1, poll))
> + return false;
> + if (poll->produced == poll->budget)
> + return false;
> + }
> +
> + if (!send)
> + shadow_q = &qp->shadow_rq;
> + else if (mmq)
> + shadow_q = &qp->shadow_mmq;
> + else
> + shadow_q = &qp->shadow_sq;
> +
> + shadow_wqe = shadow_queue_get_next_to_consume(shadow_q);
> + if (shadow_wqe) {
> + mana_fill_wc(qp, poll, shadow_wqe,
> + send ? shadow_wqe->send_opcode : IB_WC_RECV, vendor_err);
> + shadow_queue_advance_consumer(shadow_q);
> + }
[Severity: High]
When a fenced send WQE fails with CQE_TYPE_ERROR, does this code fail to
retire the older fenced dependencies on the companion shadow queue before
completing the failing WQE?
In mana_advance_send(), the loop checks PSN_LT() before checking
MANA_WQ_FENCE_WC:
drivers/infiniband/hw/mana/cq.c:mana_advance_send()
while ((shadow_wqe = shadow_queue_get_next_to_consume(shadow))) {
if (PSN_LT(seq, mmq ? shadow_wqe->fsn : shadow_wqe->wqe_offset_or_psn))
break;
if (poll->produced == poll->budget)
return false;
if (shadow_wqe->flags & MANA_WQ_FENCE_WC) {
...
}
mana_complete_send(qp, mmq, poll, VENDOR_ERR_OK);
}
Because handle_error_cqe() passes rdma_cqe->error.psn - 1 as seq,
mana_advance_send() breaks out of the loop before reaching the failing WQE's
MANA_WQ_FENCE_WC check. handle_error_cqe() then directly consumes the failing
shadow_wqe from shadow_q (also skipping the gdma_queue tail update performed
by mana_complete_send()) and adds the QP to cq->send_err_qp_list.
When mana_flush_completions() subsequently runs at the end of
mana_ib_poll_cq(), any older unsignaled WRs on the companion shadow queue
(such as IB_WR_REG_MR or IB_WR_LOCAL_INV posted before the fence that already
succeeded in hardware) are still unconsumed and get flushed via
mana_complete_send(..., VENDOR_ERR_SW_FLUSHED), emitting spurious
IB_WC_WR_FLUSH_ERR completions out of order after the dependent failing WQE.
---
[Severity: High]
Can calling mana_advance_send(qp, mmq, rdma_cqe->error.psn - 1, poll) and
unconditionally popping the head of shadow_q in handle_error_cqe() falsely
complete unacknowledged WRs with IB_WC_SUCCESS or consume mismatched shadow
WQEs?
In mana_ib_post_sq(), all non-MMQ send WRs (IB_WR_SEND, IB_WR_RDMA_WRITE,
IB_WR_RDMA_READ) are recorded in qp->shadow_sq and posted to SREQ, whereas
only IB_WR_RDMA_READ posts a WQE to RREQ:
drivers/infiniband/hw/mana/wr.c:mana_ib_post_sq()
if (wr->opcode == IB_WR_RDMA_READ) {
struct gdma_queue *rq = qp->rc_qp.queues[MANA_RC_RECV_QUEUE_REQUESTER].kmem;
struct rdma_recv_oob recv_oob = {0};
recv_oob.psn_start = qp->sq_psn;
recv_oob.msn = qp->sq_ssn;
...
err = mana_gd_post_work_request(rq, &wqe_req, &wqe_info);
Because mana_cqe_is_send() maps both SREQ and RREQ to send = true (and
mmq = false), an error CQE on RREQ (such as VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE
when the QP transitions to IB_QPS_ERR, or an RREQ WQE error) carries the
IB_WR_RDMA_READ PSN, skipping over any preceding unacknowledged IB_WR_SEND or
IB_WR_RDMA_WRITE entries in qp->shadow_sq. Calling mana_advance_send() with
rdma_cqe->error.psn - 1 retires those earlier unacknowledged or unexecuted
shadow_sq entries (and any fenced shadow_mmq entries) with VENDOR_ERR_OK
(IB_WC_SUCCESS).
Additionally, shadow_queue_get_next_to_consume(shadow_q) does not verify that
shadow_wqe matches rdma_cqe->error.psn. If both SREQ and RREQ emit error CQEs
for the same IB_WR_RDMA_READ, or if a disable-generated error CQE is polled
after the queue was already advanced or flushed, won't handle_error_cqe()
consume the next unrelated WQE from shadow_q with vendor_err?
> +
> + if (send) {
> + if (list_empty(&qp->send_err_node))
> + list_add_tail(&qp->send_err_node, &cq->send_err_qp_list);
> + } else {
> + if (list_empty(&qp->recv_err_node))
> + list_add_tail(&qp->recv_err_node, &cq->recv_err_qp_list);
> + }
> + if (qp->ibqp.qp_type != IB_QPT_GSI)
> + mana_ib_schedule_qp_error(qp);
> +
> + return true;
> +}
[Severity: High]
What happens when handle_error_cqe() processes an error CQE on a GSI QP?
mana_ib_schedule_qp_error(qp) is skipped when qp->ibqp.qp_type == IB_QPT_GSI
so the GSI QP remains in IB_QPS_RTS, but qp->send_err_node or
qp->recv_err_node is still added unconditionally to cq->send_err_qp_list or
cq->recv_err_qp_list.
Won't mana_flush_completions() then immediately flush all other posted GSI
WRs with IB_WC_WR_FLUSH_ERR while hardware still owns their DMA buffers in
RTS, and won't mana_ib_post_send_ud() / mana_ib_post_recv_ud() continue
flushing every future WR posted to that GSI queue for the remainder of the
QP's lifetime?
---
[Severity: High]
For RC QPs, does adding the QP to cq->send_err_qp_list or
cq->recv_err_qp_list here cause mana_flush_completions() to software-flush
in-flight WRs before qp->error_work has transitioned the hardware queues to
IB_QPS_ERR?
In mana_ib_poll_cq(), mana_flush_completions() runs before returning:
drivers/infiniband/hw/mana/cq.c:mana_flush_completions()
list_for_each_entry(qp, &cq->send_err_qp_list, send_err_node) {
while (poll->produced < poll->budget) {
sq_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq);
mmq_wqe = shadow_queue_get_next_to_consume(&qp->shadow_mmq);
if (!sq_wqe && !mmq_wqe)
break;
next_is_sq = next_shadow_wqe_is_sq(sq_wqe, mmq_wqe);
mana_complete_send(qp, !next_is_sq, poll, VENDOR_ERR_SW_FLUSHED);
}
if (poll->produced == poll->budget)
return;
}
Because cq->send_err_qp_list flushes both shadow_sq and shadow_mmq across
SREQ, RREQ, and MMQ, an error CQE on one hardware queue will immediately
return IB_WC_WR_FLUSH_ERR for in-flight WRs on the other hardware queues
while they are still active in RTS (before mana_ib_qp_error_work() runs
mana_ib_gd_modify_qp() to stop them and calls mana_add_qp_to_error_cqs()).
Could this allow ULPs to unmap or free DMA buffers while hardware is still
accessing them?
> diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
> index e8c479c96cbf7..04755bec015ed 100644
> --- a/drivers/infiniband/hw/mana/qp.c
> +++ b/drivers/infiniband/hw/mana/qp.c
[ ... ]
> @@ -1183,6 +1187,20 @@ int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
> return ret;
> }
>
> +static void mana_ib_qp_error_work(struct work_struct *work)
> +{
> + struct mana_ib_qp *qp = container_of(work, struct mana_ib_qp, error_work);
> + struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
> + int ret;
> +
> + ret = mana_ib_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL);
> + if (ret)
> + ibdev_err(qp->ibqp.device, "Failed to move QP %u to ERR: %d\n",
> + qp->ibqp.qp_num, ret);
> +
> + mana_put_qp_ref(qp);
> +}
[Severity: High]
If a consumer resets a QP to IB_QPS_RESET via mana_ib_modify_qp() after
polling error/flush completions (or after a VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE
CQE schedules qp->error_work), can a pending mana_ib_qp_error_work() run
afterward and force the reset or re-initialized QP back into IB_QPS_ERR?
In mana_ib_modify_qp_state(), transitioning to IB_QPS_RESET removes the QP
from the CQs and resets the shadow queues:
drivers/infiniband/hw/mana/qp.c:mana_ib_modify_qp_state()
if (attr_mask & IB_QP_STATE) {
switch (attr->qp_state) {
case IB_QPS_RESET:
mana_remove_qp_from_cqs(qp, true);
break;
default:
break;
}
if (attr->qp_state == IB_QPS_ERR) {
mana_add_qp_to_error_cqs(qp);
notify = true;
}
}
However, qp->error_work is not cancelled on IB_QPS_RESET, and
mana_ib_qp_error_work() unconditionally calls mana_ib_modify_qp() with
IB_QPS_ERR when it executes.
>
> static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp,
> struct ib_rwq_ind_table *ind_tbl,
[ ... ]
完美结束，确保最后有一空行。
Let's make sure I don't output the Chinese note at the end! Let me check the exact output.commit 75bf7e7dda34f9cbdab6ae2d0e0c7423140a1078
Author: Konstantin Taranov <kotaranov@microsoft.com>
RDMA/mana_ib: Handle error CQEs for RC QPs
This commit adds hardware error CQE decoding for RC QPs, completing the
failing shadow entry with the CQE vendor status and deferring the QP
transition to the error state via a workqueue so other work queues can be
flushed.
> diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c
> index 080f09f3c634e..a66fe0cc92b4a 100644
> --- a/drivers/infiniband/hw/mana/cq.c
> +++ b/drivers/infiniband/hw/mana/cq.c
[ ... ]
> @@ -493,6 +516,71 @@ static bool handle_psn_cqe(struct mana_ib_qp *qp, u32 psn,
> return consumed;
> }
>
> +static bool mana_cqe_is_send(struct mana_ib_qp *qp, struct gdma_comp *cqe)
> +{
> + u32 qtype = cqe->wq_num & MANA_QID_SUBTYPE_MASK;
> +
> + if (qp->ibqp.qp_type != IB_QPT_RC)
> + return cqe->is_sq;
> +
> + return cqe->is_sq ? (qtype == MANA_QID_SUBTYPE_SREQ ||
> + qtype == MANA_QID_SUBTYPE_MMQ) :
> + qtype == MANA_QID_SUBTYPE_RREQ;
> +}
> +
> +static void mana_ib_schedule_qp_error(struct mana_ib_qp *qp)
> +{
> + /* CQ polling cannot sleep; teardown drains this reference before freeing. */
> + refcount_inc(&qp->refcount);
> + if (!schedule_work(&qp->error_work))
> + mana_put_qp_ref(qp);
> +}
> +
> +static bool handle_error_cqe(struct mana_ib_cq *cq, struct mana_ib_qp *qp,
> + struct gdma_comp *cqe, struct mana_cq_poll *poll)
> +{
> + bool mmq = cqe->is_sq && (cqe->wq_num & MANA_QID_SUBTYPE_MASK) == MANA_QID_SUBTYPE_MMQ;
> + struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
> + bool send = mana_cqe_is_send(qp, cqe);
> + u32 vendor_err = rdma_cqe->error.vendor_error;
> + struct shadow_wqe_header *shadow_wqe;
> + struct shadow_queue *shadow_q;
> +
> + if (send && qp->ibqp.qp_type == IB_QPT_RC) {
> + /* The error PSN/FSN belongs to the failing WQE, not a success. */
> + if (!mana_advance_send(qp, mmq, rdma_cqe->error.psn - 1, poll))
> + return false;
> + if (poll->produced == poll->budget)
> + return false;
> + }
> +
> + if (!send)
> + shadow_q = &qp->shadow_rq;
> + else if (mmq)
> + shadow_q = &qp->shadow_mmq;
> + else
> + shadow_q = &qp->shadow_sq;
> +
> + shadow_wqe = shadow_queue_get_next_to_consume(shadow_q);
> + if (shadow_wqe) {
> + mana_fill_wc(qp, poll, shadow_wqe,
> + send ? shadow_wqe->send_opcode : IB_WC_RECV, vendor_err);
> + shadow_queue_advance_consumer(shadow_q);
> + }
[Severity: High]
When a fenced send WQE fails with CQE_TYPE_ERROR, does this code fail to
retire the older fenced dependencies on the companion shadow queue before
completing the failing WQE?
In mana_advance_send(), the loop checks PSN_LT() before checking
MANA_WQ_FENCE_WC:
drivers/infiniband/hw/mana/cq.c:mana_advance_send()
while ((shadow_wqe = shadow_queue_get_next_to_consume(shadow))) {
if (PSN_LT(seq, mmq ? shadow_wqe->fsn : shadow_wqe->wqe_offset_or_psn))
break;
if (poll->produced == poll->budget)
return false;
if (shadow_wqe->flags & MANA_WQ_FENCE_WC) {
...
}
mana_complete_send(qp, mmq, poll, VENDOR_ERR_OK);
}
Because handle_error_cqe() passes rdma_cqe->error.psn - 1 as seq,
mana_advance_send() breaks out of the loop before reaching the failing WQE's
MANA_WQ_FENCE_WC check. handle_error_cqe() then directly consumes the failing
shadow_wqe from shadow_q (also skipping the gdma_queue tail update performed
by mana_complete_send()) and adds the QP to cq->send_err_qp_list.
When mana_flush_completions() subsequently runs at the end of
mana_ib_poll_cq(), any older unsignaled WRs on the companion shadow queue
(such as IB_WR_REG_MR or IB_WR_LOCAL_INV posted before the fence that already
succeeded in hardware) are still unconsumed and get flushed via
mana_complete_send(..., VENDOR_ERR_SW_FLUSHED), emitting spurious
IB_WC_WR_FLUSH_ERR completions out of order after the dependent failing WQE.
---
[Severity: High]
Can calling mana_advance_send(qp, mmq, rdma_cqe->error.psn - 1, poll) and
unconditionally popping the head of shadow_q in handle_error_cqe() falsely
complete unacknowledged WRs with IB_WC_SUCCESS or consume mismatched shadow
WQEs?
In mana_ib_post_sq(), all non-MMQ send WRs (IB_WR_SEND, IB_WR_RDMA_WRITE,
IB_WR_RDMA_READ) are recorded in qp->shadow_sq and posted to SREQ, whereas
only IB_WR_RDMA_READ posts a WQE to RREQ:
drivers/infiniband/hw/mana/wr.c:mana_ib_post_sq()
if (wr->opcode == IB_WR_RDMA_READ) {
struct gdma_queue *rq = qp->rc_qp.queues[MANA_RC_RECV_QUEUE_REQUESTER].kmem;
struct rdma_recv_oob recv_oob = {0};
recv_oob.psn_start = qp->sq_psn;
recv_oob.msn = qp->sq_ssn;
...
err = mana_gd_post_work_request(rq, &wqe_req, &wqe_info);
Because mana_cqe_is_send() maps both SREQ and RREQ to send = true (and
mmq = false), an error CQE on RREQ (such as VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE
when the QP transitions to IB_QPS_ERR, or an RREQ WQE error) carries the
IB_WR_RDMA_READ PSN, skipping over any preceding unacknowledged IB_WR_SEND or
IB_WR_RDMA_WRITE entries in qp->shadow_sq. Calling mana_advance_send() with
rdma_cqe->error.psn - 1 retires those earlier unacknowledged or unexecuted
shadow_sq entries (and any fenced shadow_mmq entries) with VENDOR_ERR_OK
(IB_WC_SUCCESS).
Additionally, shadow_queue_get_next_to_consume(shadow_q) does not verify that
shadow_wqe matches rdma_cqe->error.psn. If both SREQ and RREQ emit error CQEs
for the same IB_WR_RDMA_READ, or if a disable-generated error CQE is polled
after the queue was already advanced or flushed, won't handle_error_cqe()
consume the next unrelated WQE from shadow_q with vendor_err?
> +
> + if (send) {
> + if (list_empty(&qp->send_err_node))
> + list_add_tail(&qp->send_err_node, &cq->send_err_qp_list);
> + } else {
> + if (list_empty(&qp->recv_err_node))
> + list_add_tail(&qp->recv_err_node, &cq->recv_err_qp_list);
> + }
> + if (qp->ibqp.qp_type != IB_QPT_GSI)
> + mana_ib_schedule_qp_error(qp);
> +
> + return true;
> +}
[Severity: High]
What happens when handle_error_cqe() processes an error CQE on a GSI QP?
mana_ib_schedule_qp_error(qp) is skipped when qp->ibqp.qp_type == IB_QPT_GSI
so the GSI QP remains in IB_QPS_RTS, but qp->send_err_node or
qp->recv_err_node is still added unconditionally to cq->send_err_qp_list or
cq->recv_err_qp_list.
Won't mana_flush_completions() then immediately flush all other posted GSI
WRs with IB_WC_WR_FLUSH_ERR while hardware still owns their DMA buffers in
RTS, and won't mana_ib_post_send_ud() / mana_ib_post_recv_ud() continue
flushing every future WR posted to that GSI queue for the remainder of the
QP's lifetime?
---
[Severity: High]
For RC QPs, does adding the QP to cq->send_err_qp_list or
cq->recv_err_qp_list here cause mana_flush_completions() to software-flush
in-flight WRs before qp->error_work has transitioned the hardware queues to
IB_QPS_ERR?
In mana_ib_poll_cq(), mana_flush_completions() runs before returning:
drivers/infiniband/hw/mana/cq.c:mana_flush_completions()
list_for_each_entry(qp, &cq->send_err_qp_list, send_err_node) {
while (poll->produced < poll->budget) {
sq_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq);
mmq_wqe = shadow_queue_get_next_to_consume(&qp->shadow_mmq);
if (!sq_wqe && !mmq_wqe)
break;
next_is_sq = next_shadow_wqe_is_sq(sq_wqe, mmq_wqe);
mana_complete_send(qp, !next_is_sq, poll, VENDOR_ERR_SW_FLUSHED);
}
if (poll->produced == poll->budget)
return;
}
Because cq->send_err_qp_list flushes both shadow_sq and shadow_mmq across
SREQ, RREQ, and MMQ, an error CQE on one hardware queue will immediately
return IB_WC_WR_FLUSH_ERR for in-flight WRs on the other hardware queues
while they are still active in RTS (before mana_ib_qp_error_work() runs
mana_ib_gd_modify_qp() to stop them and calls mana_add_qp_to_error_cqs()).
Could this allow ULPs to unmap or free DMA buffers while hardware is still
accessing them?
> diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
> index e8c479c96cbf7..04755bec015ed 100644
> --- a/drivers/infiniband/hw/mana/qp.c
> +++ b/drivers/infiniband/hw/mana/qp.c
[ ... ]
> @@ -1183,6 +1187,20 @@ int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
> return ret;
> }
>
> +static void mana_ib_qp_error_work(struct work_struct *work)
> +{
> + struct mana_ib_qp *qp = container_of(work, struct mana_ib_qp, error_work);
> + struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
> + int ret;
> +
> + ret = mana_ib_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL);
> + if (ret)
> + ibdev_err(qp->ibqp.device, "Failed to move QP %u to ERR: %d\n",
> + qp->ibqp.qp_num, ret);
> +
> + mana_put_qp_ref(qp);
> +}
[Severity: High]
If a consumer resets a QP to IB_QPS_RESET via mana_ib_modify_qp() after
polling error/flush completions (or after a VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE
CQE schedules qp->error_work), can a pending mana_ib_qp_error_work() run
afterward and force the reset or re-initialized QP back into IB_QPS_ERR?
In mana_ib_modify_qp_state(), transitioning to IB_QPS_RESET removes the QP
from the CQs and resets the shadow queues:
drivers/infiniband/hw/mana/qp.c:mana_ib_modify_qp_state()
if (attr_mask & IB_QP_STATE) {
switch (attr->qp_state) {
case IB_QPS_RESET:
mana_remove_qp_from_cqs(qp, true);
break;
default:
break;
}
if (attr->qp_state == IB_QPS_ERR) {
mana_add_qp_to_error_cqs(qp);
notify = true;
}
}
However, qp->error_work is not cancelled on IB_QPS_RESET, and
mana_ib_qp_error_work() unconditionally calls mana_ib_modify_qp() with
IB_QPS_ERR when it executes.
>
> static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp,
> struct ib_rwq_ind_table *ind_tbl,
[ ... ]
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261017064413.995985-1-kotaranov@linux.microsoft.com?part=9
^ permalink raw reply [flat|nested] 21+ messages in thread
* [PATCH rdma-next v5 10/10] RDMA/mana_ib: Drain kernel receive and send queues
2026-10-17 6:44 [PATCH rdma-next v5 00/10] RDMA/mana_ib: Add kernel RC and fast registration support Konstantin Taranov
` (8 preceding siblings ...)
2026-10-17 6:44 ` [PATCH rdma-next v5 09/10] RDMA/mana_ib: Handle error CQEs for RC QPs Konstantin Taranov
@ 2026-10-17 6:44 ` Konstantin Taranov
2026-10-17 7:01 ` sashiko-bot
9 siblings, 1 reply; 21+ messages in thread
From: Konstantin Taranov @ 2026-10-17 6:44 UTC (permalink / raw)
To: kotaranov, snsanghvi, longli, jgg, leon; +Cc: linux-rdma, linux-kernel
From: Konstantin Taranov <kotaranov@microsoft.com>
Implement the core drain callbacks by moving the QP to ERR, posting a
completion-bearing marker, and waiting until that marker is flushed.
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
---
v2:
- Limit the drain callbacks to RC QPs.
- Skip completion callbacks for IB_POLL_DIRECT and retain direct polling.
drivers/infiniband/hw/mana/device.c | 2 +
drivers/infiniband/hw/mana/mana_ib.h | 2 +
drivers/infiniband/hw/mana/qp.c | 103 +++++++++++++++++++++++++++
3 files changed, 107 insertions(+)
diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c
index cdaada7f6e8a..07c7fe678ff7 100644
--- a/drivers/infiniband/hw/mana/device.c
+++ b/drivers/infiniband/hw/mana/device.c
@@ -38,6 +38,8 @@ static const struct ib_device_ops mana_ib_dev_ops = {
.destroy_rwq_ind_table = mana_ib_destroy_rwq_ind_table,
.destroy_wq = mana_ib_destroy_wq,
.disassociate_ucontext = mana_ib_disassociate_ucontext,
+ .drain_rq = mana_ib_drain_rq,
+ .drain_sq = mana_ib_drain_sq,
.get_dma_mr = mana_ib_get_dma_mr,
.get_link_layer = mana_ib_get_link_layer,
.get_port_immutable = mana_ib_get_port_immutable,
diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index 3a6a4ad8e641..d810080d2baf 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -1060,6 +1060,8 @@ int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
const struct ib_send_wr **bad_wr);
+void mana_ib_drain_sq(struct ib_qp *ibqp);
+void mana_ib_drain_rq(struct ib_qp *ibqp);
void mana_drain_gsi_sqs(struct mana_ib_dev *mdev);
int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
index 519a161b994d..0dc7deb81921 100644
--- a/drivers/infiniband/hw/mana/qp.c
+++ b/drivers/infiniband/hw/mana/qp.c
@@ -1380,3 +1380,106 @@ int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
return -ENOENT;
}
+
+struct mana_ib_drain_cqe {
+ struct ib_cqe cqe;
+ struct completion done;
+};
+
+static void mana_ib_drain_qp_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+ struct mana_ib_drain_cqe *cqe = container_of(wc->wr_cqe,
+ struct mana_ib_drain_cqe,
+ cqe);
+ complete(&cqe->done);
+}
+
+void mana_ib_drain_sq(struct ib_qp *ibqp)
+{
+ struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
+ struct mana_ib_drain_cqe sdrain;
+ struct ib_rdma_wr swr = {
+ .wr = {
+ .next = NULL,
+ .wr_cqe = &sdrain.cqe,
+ .opcode = IB_WR_SEND,
+ },
+ };
+ const struct ib_send_wr *bad_swr;
+ struct ib_cq *cq = ibqp->send_cq;
+ int ret;
+
+ if (ibqp->qp_type != IB_QPT_RC) {
+ ibdev_err(ibqp->device, "SQ drain is only supported for RC QPs (type %u)\n",
+ ibqp->qp_type);
+ return;
+ }
+
+ ret = ib_modify_qp(ibqp, &attr, IB_QP_STATE);
+ if (ret) {
+ ibdev_err(ibqp->device, "Failed to drain SQ: modify QP returned %d\n", ret);
+ return;
+ }
+
+ sdrain.cqe.done = mana_ib_drain_qp_done;
+ init_completion(&sdrain.done);
+
+ ret = ib_post_send(ibqp, &swr.wr, &bad_swr);
+ if (ret) {
+ ibdev_err(ibqp->device, "Failed to post SQ drain WR: %d\n", ret);
+ return;
+ }
+
+ if (cq->poll_ctx != IB_POLL_DIRECT && cq->comp_handler)
+ cq->comp_handler(cq, cq->cq_context);
+
+ if (cq->poll_ctx == IB_POLL_DIRECT) {
+ while (!wait_for_completion_timeout(&sdrain.done, HZ / 10))
+ ib_process_cq_direct(cq, -1);
+ } else {
+ wait_for_completion(&sdrain.done);
+ }
+}
+
+void mana_ib_drain_rq(struct ib_qp *ibqp)
+{
+ struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
+ struct mana_ib_drain_cqe rdrain;
+ struct ib_recv_wr rwr = {
+ .next = NULL,
+ .wr_cqe = &rdrain.cqe,
+ };
+ struct ib_cq *cq = ibqp->recv_cq;
+ int ret;
+
+ if (ibqp->qp_type != IB_QPT_RC) {
+ ibdev_err(ibqp->device, "RQ drain is only supported for RC QPs (type %u)\n",
+ ibqp->qp_type);
+ return;
+ }
+
+ ret = ib_modify_qp(ibqp, &attr, IB_QP_STATE);
+ if (ret) {
+ ibdev_err(ibqp->device, "Failed to drain RQ: modify QP returned %d\n", ret);
+ return;
+ }
+
+ rdrain.cqe.done = mana_ib_drain_qp_done;
+ init_completion(&rdrain.done);
+
+ ret = ib_post_recv(ibqp, &rwr, NULL);
+ if (ret) {
+ ibdev_err(ibqp->device, "Failed to post RQ drain WR: %d\n", ret);
+ return;
+ }
+
+ if (cq->poll_ctx != IB_POLL_DIRECT && cq->comp_handler)
+ cq->comp_handler(cq, cq->cq_context);
+
+ if (cq->poll_ctx == IB_POLL_DIRECT) {
+ while (!wait_for_completion_timeout(&rdrain.done, HZ / 10))
+ ib_process_cq_direct(cq, -1);
+ } else {
+ wait_for_completion(&rdrain.done);
+ }
+}
--
2.43.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH rdma-next v5 10/10] RDMA/mana_ib: Drain kernel receive and send queues
2026-10-17 6:44 ` [PATCH rdma-next v5 10/10] RDMA/mana_ib: Drain kernel receive and send queues Konstantin Taranov
@ 2026-10-17 7:01 ` sashiko-bot
0 siblings, 0 replies; 21+ messages in thread
From: sashiko-bot @ 2026-10-17 7:01 UTC (permalink / raw)
To: Konstantin Taranov; +Cc: linux-rdma
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [High] RDMA/mana_ib: lost poll wakeup in mana_ib_drain_sq() and mana_ib_drain_rq()
--
commit cdad9731bfb8c6291bf5161e04f6798a87eaf888
Author: Konstantin Taranov <kotaranov@microsoft.com>
RDMA/mana_ib: Drain kernel receive and send queues
This commit implements the core drain_rq and drain_sq device operations for
mana_ib RC QPs by transitioning the QP to IB_QPS_ERR, posting a marker WR,
and waiting for its completion.
> diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
> index 04755bec015ed..8557453d39194 100644
> --- a/drivers/infiniband/hw/mana/qp.c
> +++ b/drivers/infiniband/hw/mana/qp.c
> @@ -1381,3 +1381,106 @@ int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
>
> return -ENOENT;
> }
> +
> +struct mana_ib_drain_cqe {
> + struct ib_cqe cqe;
> + struct completion done;
> +};
> +
> +static void mana_ib_drain_qp_done(struct ib_cq *cq, struct ib_wc *wc)
> +{
> + struct mana_ib_drain_cqe *cqe = container_of(wc->wr_cqe,
> + struct mana_ib_drain_cqe,
> + cqe);
> + complete(&cqe->done);
> +}
> +
> +void mana_ib_drain_sq(struct ib_qp *ibqp)
> +{
> + struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
> + struct mana_ib_drain_cqe sdrain;
> + struct ib_rdma_wr swr = {
> + .wr = {
> + .next = NULL,
> + .wr_cqe = &sdrain.cqe,
> + .opcode = IB_WR_SEND,
> + },
> + };
> + const struct ib_send_wr *bad_swr;
> + struct ib_cq *cq = ibqp->send_cq;
> + int ret;
> +
> + if (ibqp->qp_type != IB_QPT_RC) {
> + ibdev_err(ibqp->device, "SQ drain is only supported for RC QPs (type %u)\n",
> + ibqp->qp_type);
> + return;
> + }
> +
> + ret = ib_modify_qp(ibqp, &attr, IB_QP_STATE);
> + if (ret) {
> + ibdev_err(ibqp->device, "Failed to drain SQ: modify QP returned %d\n", ret);
> + return;
> + }
> +
> + sdrain.cqe.done = mana_ib_drain_qp_done;
> + init_completion(&sdrain.done);
> +
> + ret = ib_post_send(ibqp, &swr.wr, &bad_swr);
> + if (ret) {
> + ibdev_err(ibqp->device, "Failed to post SQ drain WR: %d\n", ret);
> + return;
> + }
> +
> + if (cq->poll_ctx != IB_POLL_DIRECT && cq->comp_handler)
> + cq->comp_handler(cq, cq->cq_context);
> +
> + if (cq->poll_ctx == IB_POLL_DIRECT) {
> + while (!wait_for_completion_timeout(&sdrain.done, HZ / 10))
> + ib_process_cq_direct(cq, -1);
> + } else {
> + wait_for_completion(&sdrain.done);
> + }
> +}
> +
> +void mana_ib_drain_rq(struct ib_qp *ibqp)
> +{
> + struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
> + struct mana_ib_drain_cqe rdrain;
> + struct ib_recv_wr rwr = {
> + .next = NULL,
> + .wr_cqe = &rdrain.cqe,
> + };
> + struct ib_cq *cq = ibqp->recv_cq;
> + int ret;
> +
> + if (ibqp->qp_type != IB_QPT_RC) {
> + ibdev_err(ibqp->device, "RQ drain is only supported for RC QPs (type %u)\n",
> + ibqp->qp_type);
> + return;
> + }
> +
> + ret = ib_modify_qp(ibqp, &attr, IB_QP_STATE);
> + if (ret) {
> + ibdev_err(ibqp->device, "Failed to drain RQ: modify QP returned %d\n", ret);
> + return;
> + }
> +
> + rdrain.cqe.done = mana_ib_drain_qp_done;
> + init_completion(&rdrain.done);
> +
> + ret = ib_post_recv(ibqp, &rwr, NULL);
> + if (ret) {
> + ibdev_err(ibqp->device, "Failed to post RQ drain WR: %d\n", ret);
> + return;
> + }
> +
> + if (cq->poll_ctx != IB_POLL_DIRECT && cq->comp_handler)
> + cq->comp_handler(cq, cq->cq_context);
> +
> + if (cq->poll_ctx == IB_POLL_DIRECT) {
> + while (!wait_for_completion_timeout(&rdrain.done, HZ / 10))
> + ib_process_cq_direct(cq, -1);
> + } else {
> + wait_for_completion(&rdrain.done);
> + }
> +}
[Severity: High]
Can mana_ib_drain_sq() and mana_ib_drain_rq() lose a wakeup and hang in
wait_for_completion() when cq->poll_ctx is IB_POLL_SOFTIRQ?
In mana_ib_drain_sq() and mana_ib_drain_rq(), ib_modify_qp() transitions
the QP to IB_QPS_ERR before the drain WR is posted. Inside
mana_ib_modify_qp(), mana_add_qp_to_error_cqs() adds the QP to the CQ error
lists and mana_ib_notify_error_cqs() immediately calls cq->comp_handler():
drivers/infiniband/hw/mana/qp.c:mana_ib_notify_error_cqs() {
if (send_cq->poll_ctx != IB_POLL_DIRECT && send_cq->comp_handler)
send_cq->comp_handler(send_cq, send_cq->cq_context);
if (recv_cq != send_cq && recv_cq->poll_ctx != IB_POLL_DIRECT &&
recv_cq->comp_handler)
recv_cq->comp_handler(recv_cq, recv_cq->cq_context);
}
For IB_POLL_SOFTIRQ, cq->comp_handler is ib_cq_completion_softirq(), which
calls irq_poll_sched(&cq->iop), setting IRQ_POLL_F_SCHED and scheduling
ib_poll_handler():
drivers/infiniband/core/cq.c:ib_poll_handler() {
completed = __ib_process_cq(cq, budget, cq->wc, IB_POLL_BATCH);
if (completed < budget) {
irq_poll_complete(&cq->iop);
if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) {
trace_cq_reschedule(cq);
irq_poll_sched(&cq->iop);
}
}
...
}
In __ib_process_cq(), mana_ib_poll_cq() takes cq->cq_lock, polls hardware
CQEs, calls mana_flush_completions() for currently queued shadow WQEs, and
releases cq->cq_lock. __ib_process_cq() then invokes wc->wr_cqe->done() for
those flushed entries while IRQ_POLL_F_SCHED is still set.
Also, during ib_drain_qp() when send_cq == recv_cq, mana_flush_completions()
flushes both send_err_qp_list (including sdrain) and recv_err_qp_list in a
single mana_ib_poll_cq() call; __ib_process_cq() invokes sdrain.cqe.done()
on the first WC while still iterating through the remaining flushed RQ WCs
with IRQ_POLL_F_SCHED set, waking the caller to immediately enter
mana_ib_drain_rq().
If mana_ib_drain_sq() or mana_ib_drain_rq() posts its drain WR into
shadow_sq or shadow_rq after mana_ib_poll_cq() drops cq->cq_lock and then
calls cq->comp_handler() -> irq_poll_sched(&cq->iop) before
ib_poll_handler() reaches irq_poll_complete(&cq->iop), irq_poll_sched() sees
IRQ_POLL_F_SCHED is already set and returns as a no-op.
When ib_poll_handler() then calls irq_poll_complete(&cq->iop) followed by
ib_req_notify_cq(cq, IB_POLL_FLAGS) -> mana_ib_arm_cq(), mana_ib_arm_cq()
only checks cq->has_pending_cqe:
drivers/infiniband/hw/mana/cq.c:mana_ib_arm_cq() {
spin_lock_irqsave(&cq->cq_lock, irq_flags);
mana_ib_cq_doorbell(cq, SET_ARM_BIT);
if (flags & IB_CQ_REPORT_MISSED_EVENTS)
missed = cq->has_pending_cqe;
spin_unlock_irqrestore(&cq->cq_lock, irq_flags);
return missed;
}
Because mana_ib_arm_cq() does not check for unconsumed software-flushed
shadow WQEs on send_err_qp_list or recv_err_qp_list and the QP in
IB_QPS_ERR will not produce hardware CQEs for the drain WR,
mana_ib_arm_cq() returns 0 and cq->iop is never rescheduled.
Should this synchronize with the softirq poll context using
irq_poll_disable(&cq->iop) and irq_poll_enable(&cq->iop) when invoking the
completion handler (similar to mlx4, mlx5, and hns), or report pending
error QP shadow WQEs in mana_ib_arm_cq()?
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261017064413.995985-1-kotaranov@linux.microsoft.com?part=10
^ permalink raw reply [flat|nested] 21+ messages in thread