* [PATCH for-next 1/2] RDMA/efa: Use device ABI MR permissions instead of verbs flags
2026-08-06 7:57 [PATCH for-next 0/2] RDMA/efa: Add support for MR relaxed ordering Yonatan Nachum
@ 2026-08-06 7:57 ` Yonatan Nachum
2026-08-06 7:57 ` [PATCH for-next 2/2] RDMA/efa: Pass relaxed ordering flag to device Yonatan Nachum
1 sibling, 0 replies; 3+ messages in thread
From: Yonatan Nachum @ 2026-08-06 7:57 UTC (permalink / raw)
To: jgg, leon, linux-rdma
Cc: mrgolin, sleybo, matua, gal.pressman, danamala, Chen Brasch,
Yonatan Nachum
From: Dana Malachi <danamala@amazon.com>
Set MR permissions explicitly using device interface definitions rather
than copying raw verbs access flags. This is needed for the next
commit, to allow access flag bits that are not in the permissions field.
Reviewed-by: Chen Brasch <cbrasch@amazon.com>
Reviewed-by: Michael Margolin <mrgolin@amazon.com>
Signed-off-by: Dana Malachi <danamala@amazon.com>
Signed-off-by: Yonatan Nachum <ynachum@amazon.com>
---
drivers/infiniband/hw/efa/efa_com_cmd.c | 7 ++++++-
drivers/infiniband/hw/efa/efa_com_cmd.h | 7 +++++--
drivers/infiniband/hw/efa/efa_verbs.c | 4 +++-
3 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c
index 1b00f16b8ea8..aac0e936601a 100644
--- a/drivers/infiniband/hw/efa/efa_com_cmd.c
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.c
@@ -237,7 +237,12 @@ int efa_com_register_mr(struct efa_com_dev *edev,
EFA_SET(&mr_cmd.flags, EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT,
params->page_shift);
mr_cmd.iova = params->iova;
- mr_cmd.permissions = params->permissions;
+ if (params->permissions.local_write)
+ EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE, 1);
+ if (params->permissions.remote_write)
+ EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE, 1);
+ if (params->permissions.remote_read)
+ EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_REMOTE_READ_ENABLE, 1);
if (params->inline_pbl) {
memcpy(mr_cmd.pbl.inline_pbl_array,
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h
index bca722f021ea..7b0399811cfd 100644
--- a/drivers/infiniband/hw/efa/efa_com_cmd.h
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.h
@@ -204,8 +204,11 @@ struct efa_com_reg_mr_params {
* address mapping
*/
u8 page_shift;
- /* see permissions field of struct efa_admin_reg_mr_cmd */
- u8 permissions;
+ struct {
+ u8 local_write : 1;
+ u8 remote_write : 1;
+ u8 remote_read : 1;
+ } permissions;
u8 inline_pbl;
u8 indirect;
};
diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c
index 2d28d68efe77..635687275350 100644
--- a/drivers/infiniband/hw/efa/efa_verbs.c
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -1676,7 +1676,9 @@ static int efa_register_mr(struct ib_pd *ibpd, struct efa_mr *mr, u64 start,
params.pd = to_epd(ibpd)->pdn;
params.iova = virt_addr;
params.mr_length_in_bytes = length;
- params.permissions = access_flags;
+ params.permissions.local_write = !!(access_flags & IB_ACCESS_LOCAL_WRITE);
+ params.permissions.remote_write = !!(access_flags & IB_ACCESS_REMOTE_WRITE);
+ params.permissions.remote_read = !!(access_flags & IB_ACCESS_REMOTE_READ);
pg_sz = ib_umem_find_best_pgsz(mr->umem,
dev->dev_attr.page_size_cap,
--
2.50.1
^ permalink raw reply related [flat|nested] 3+ messages in thread* [PATCH for-next 2/2] RDMA/efa: Pass relaxed ordering flag to device
2026-08-06 7:57 [PATCH for-next 0/2] RDMA/efa: Add support for MR relaxed ordering Yonatan Nachum
2026-08-06 7:57 ` [PATCH for-next 1/2] RDMA/efa: Use device ABI MR permissions instead of verbs flags Yonatan Nachum
@ 2026-08-06 7:57 ` Yonatan Nachum
1 sibling, 0 replies; 3+ messages in thread
From: Yonatan Nachum @ 2026-08-06 7:57 UTC (permalink / raw)
To: jgg, leon, linux-rdma
Cc: mrgolin, sleybo, matua, gal.pressman, danamala, Chen Brasch,
Yonatan Nachum
From: Dana Malachi <danamala@amazon.com>
Accept the relaxed ordering access flag during memory region
registration and forward it to the device firmware via the admin command
path.
Reviewed-by: Chen Brasch <cbrasch@amazon.com>
Reviewed-by: Michael Margolin <mrgolin@amazon.com>
Signed-off-by: Dana Malachi <danamala@amazon.com>
Signed-off-by: Yonatan Nachum <ynachum@amazon.com>
---
drivers/infiniband/hw/efa/efa_admin_cmds_defs.h | 10 +++++++---
drivers/infiniband/hw/efa/efa_com_cmd.c | 2 ++
drivers/infiniband/hw/efa/efa_com_cmd.h | 1 +
drivers/infiniband/hw/efa/efa_verbs.c | 7 +++++++
include/uapi/rdma/efa-abi.h | 1 +
5 files changed, 18 insertions(+), 3 deletions(-)
diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
index ab830764e3b4..eec660a32061 100644
--- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
+++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
@@ -358,7 +358,8 @@ struct efa_admin_reg_mr_cmd {
* 5:0 : phys_page_size_shift - page size is (1 <<
* phys_page_size_shift). Page size is used for
* building the Virtual to Physical address mapping
- * 6 : reserved - MBZ
+ * 6 : relaxed_ordering_enable - Enable PCIe relaxed
+ * ordering for this memory region
* 7 : mem_addr_phy_mode_en - Enable bit for physical
* memory registration (no translation), can be used
* only by privileged clients. If set, PBL must
@@ -697,7 +698,9 @@ struct efa_admin_feature_device_attr_desc {
* write with imm. receive is supported
* 5 : event_counters - If set, event counters are
* supported
- * 9:6 : reserved1 - MBZ
+ * 8:6 : reserved1 - MBZ
+ * 9 : mr_relaxed_ordering - If set, MR PCIe relaxed
+ * ordering is supported
* 10 : sq_64_bit_req_id - If set, SQ can use 64-bit
* work request ids
* 31:11 : reserved2 - MBZ
@@ -1203,6 +1206,7 @@ struct efa_admin_modify_event_counter_resp {
/* reg_mr_cmd */
#define EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK GENMASK(5, 0)
+#define EFA_ADMIN_REG_MR_CMD_RELAXED_ORDERING_ENABLE_MASK BIT(6)
#define EFA_ADMIN_REG_MR_CMD_MEM_ADDR_PHY_MODE_EN_MASK BIT(7)
#define EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE_MASK BIT(0)
#define EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE_MASK BIT(1)
@@ -1231,7 +1235,7 @@ struct efa_admin_modify_event_counter_resp {
#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_WRITE_MASK BIT(3)
#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_UNSOLICITED_WRITE_RECV_MASK BIT(4)
#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_EVENT_COUNTERS_MASK BIT(5)
-#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_SQ_64_BIT_REQ_ID_SHIFT 10
+#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_MR_RELAXED_ORDERING_MASK BIT(9)
#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_SQ_64_BIT_REQ_ID_MASK BIT(10)
/* feature_queue_attr_desc_2 */
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c
index aac0e936601a..07bfcb093df8 100644
--- a/drivers/infiniband/hw/efa/efa_com_cmd.c
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.c
@@ -236,6 +236,8 @@ int efa_com_register_mr(struct efa_com_dev *edev,
mr_cmd.mr_length = params->mr_length_in_bytes;
EFA_SET(&mr_cmd.flags, EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT,
params->page_shift);
+ if (params->relaxed_ordering)
+ EFA_SET(&mr_cmd.flags, EFA_ADMIN_REG_MR_CMD_RELAXED_ORDERING_ENABLE, 1);
mr_cmd.iova = params->iova;
if (params->permissions.local_write)
EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE, 1);
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h
index 7b0399811cfd..524407f36d43 100644
--- a/drivers/infiniband/hw/efa/efa_com_cmd.h
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.h
@@ -209,6 +209,7 @@ struct efa_com_reg_mr_params {
u8 remote_write : 1;
u8 remote_read : 1;
} permissions;
+ u8 relaxed_ordering;
u8 inline_pbl;
u8 indirect;
};
diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c
index 635687275350..a91d02158a6a 100644
--- a/drivers/infiniband/hw/efa/efa_verbs.c
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -272,6 +272,9 @@ int efa_query_device(struct ib_device *ibdev,
if (EFA_DEV_CAP(dev, EVENT_COUNTERS))
resp.device_caps |= EFA_QUERY_DEVICE_CAPS_COMP_CNTR;
+ if (EFA_DEV_CAP(dev, MR_RELAXED_ORDERING))
+ resp.device_caps |= EFA_QUERY_DEVICE_CAPS_MR_RELAXED_ORDERING;
+
if (EFA_DEV_CAP(dev, SQ_64_BIT_REQ_ID))
resp.device_caps |= EFA_QUERY_DEVICE_CAPS_SQ_64_BIT_REQ_ID;
@@ -1647,6 +1650,9 @@ static struct efa_mr *efa_alloc_mr(struct ib_pd *ibpd, int access_flags,
(EFA_DEV_CAP(dev, RDMA_READ) ? IB_ACCESS_REMOTE_READ : 0) |
(EFA_DEV_CAP(dev, RDMA_WRITE) ? IB_ACCESS_REMOTE_WRITE : 0);
+ if (EFA_DEV_CAP(dev, MR_RELAXED_ORDERING))
+ supp_access_flags |= IB_ACCESS_RELAXED_ORDERING;
+
access_flags &= ~IB_ACCESS_OPTIONAL;
if (access_flags & ~supp_access_flags) {
ibdev_dbg(&dev->ibdev,
@@ -1679,6 +1685,7 @@ static int efa_register_mr(struct ib_pd *ibpd, struct efa_mr *mr, u64 start,
params.permissions.local_write = !!(access_flags & IB_ACCESS_LOCAL_WRITE);
params.permissions.remote_write = !!(access_flags & IB_ACCESS_REMOTE_WRITE);
params.permissions.remote_read = !!(access_flags & IB_ACCESS_REMOTE_READ);
+ params.relaxed_ordering = !!(access_flags & IB_ACCESS_RELAXED_ORDERING);
pg_sz = ib_umem_find_best_pgsz(mr->umem,
dev->dev_attr.page_size_cap,
diff --git a/include/uapi/rdma/efa-abi.h b/include/uapi/rdma/efa-abi.h
index 2094b4bcc5cf..675aca45e697 100644
--- a/include/uapi/rdma/efa-abi.h
+++ b/include/uapi/rdma/efa-abi.h
@@ -137,6 +137,7 @@ enum {
EFA_QUERY_DEVICE_CAPS_CQ_WITH_EXT_MEM = 1 << 7,
EFA_QUERY_DEVICE_CAPS_COMP_CNTR = 1 << 8,
EFA_QUERY_DEVICE_CAPS_SQ_64_BIT_REQ_ID = 1 << 9,
+ EFA_QUERY_DEVICE_CAPS_MR_RELAXED_ORDERING = 1 << 10,
};
struct efa_ibv_ex_query_device_resp {
--
2.50.1
^ permalink raw reply related [flat|nested] 3+ messages in thread