* [PATCH rdma-core 01/12] Update kernel headers
2020-04-20 14:06 [PATCH rdma-core 00/12] Add Enhanced Connection Established (ECE) APIs Leon Romanovsky
@ 2020-04-20 14:06 ` Leon Romanovsky
2020-04-20 14:06 ` [PATCH rdma-core 02/12] libibverbs: Add interfaces to configure and use ECE Leon Romanovsky
` (10 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Leon Romanovsky @ 2020-04-20 14:06 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe; +Cc: Leon Romanovsky, linux-rdma
From: Leon Romanovsky <leonro@mellanox.com>
To commit ?? ("RDMA/mlx5: Verify that QP is created with RQ or SQ")
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
| 6 ++++++
| 15 ++++++++++++++-
2 files changed, 20 insertions(+), 1 deletion(-)
--git a/kernel-headers/rdma/mlx5_user_ioctl_cmds.h b/kernel-headers/rdma/mlx5_user_ioctl_cmds.h
index 24f3388c..8e316ef8 100644
--- a/kernel-headers/rdma/mlx5_user_ioctl_cmds.h
+++ b/kernel-headers/rdma/mlx5_user_ioctl_cmds.h
@@ -241,6 +241,11 @@ enum mlx5_ib_flow_type {
MLX5_IB_FLOW_TYPE_MC_DEFAULT,
};
+enum mlx5_ib_create_flow_flags {
+ MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS = 1 << 0,
+ MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP = 1 << 1,
+};
+
enum mlx5_ib_create_flow_attrs {
MLX5_IB_ATTR_CREATE_FLOW_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE,
@@ -251,6 +256,7 @@ enum mlx5_ib_create_flow_attrs {
MLX5_IB_ATTR_CREATE_FLOW_TAG,
MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX,
MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
+ MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
};
enum mlx5_ib_destoy_flow_attrs {
--git a/kernel-headers/rdma/rdma_user_cm.h b/kernel-headers/rdma/rdma_user_cm.h
index e42940a2..e545f2de 100644
--- a/kernel-headers/rdma/rdma_user_cm.h
+++ b/kernel-headers/rdma/rdma_user_cm.h
@@ -78,6 +78,10 @@ enum rdma_ucm_port_space {
RDMA_PS_UDP = 0x0111,
};
+enum rdma_ucm_reject_reason {
+ RDMA_USER_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED = 35
+};
+
/*
* command ABI structures.
*/
@@ -206,10 +210,16 @@ struct rdma_ucm_ud_param {
__u8 reserved[7];
};
+struct rdma_ucm_ece {
+ __u32 vendor_id;
+ __u32 attr_mod;
+};
+
struct rdma_ucm_connect {
struct rdma_ucm_conn_param conn_param;
__u32 id;
__u32 reserved;
+ struct rdma_ucm_ece ece;
};
struct rdma_ucm_listen {
@@ -222,12 +232,14 @@ struct rdma_ucm_accept {
struct rdma_ucm_conn_param conn_param;
__u32 id;
__u32 reserved;
+ struct rdma_ucm_ece ece;
};
struct rdma_ucm_reject {
__u32 id;
__u8 private_data_len;
- __u8 reserved[3];
+ __u8 reason; /* enum rdma_ucm_reject_reason */
+ __u8 reserved[2];
__u8 private_data[RDMA_MAX_PRIVATE_DATA];
};
@@ -287,6 +299,7 @@ struct rdma_ucm_event_resp {
struct rdma_ucm_ud_param ud;
} param;
__u32 reserved;
+ struct rdma_ucm_ece ece;
};
/* Option levels */
--
2.25.2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH rdma-core 02/12] libibverbs: Add interfaces to configure and use ECE
2020-04-20 14:06 [PATCH rdma-core 00/12] Add Enhanced Connection Established (ECE) APIs Leon Romanovsky
2020-04-20 14:06 ` [PATCH rdma-core 01/12] Update kernel headers Leon Romanovsky
@ 2020-04-20 14:06 ` Leon Romanovsky
2020-04-20 14:06 ` [PATCH rdma-core 03/12] libibverbs: Document ECE API Leon Romanovsky
` (9 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Leon Romanovsky @ 2020-04-20 14:06 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe; +Cc: Leon Romanovsky, linux-rdma
From: Leon Romanovsky <leonro@mellanox.com>
ECE parameters are vendor specific information per-QP,
provide a way to set, query and commit ECE data.
* ibv_set_ece() - overwrite default ECE options and instruct
libibverbs to use ECE data while enabling QP.
* ibv_query_ece() - get ECE options.
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
CMakeLists.txt | 2 +-
debian/libibverbs1.symbols | 5 ++++-
libibverbs/CMakeLists.txt | 2 +-
libibverbs/driver.h | 2 ++
libibverbs/dummy_ops.c | 14 ++++++++++++++
libibverbs/libibverbs.map.in | 6 ++++++
libibverbs/verbs.c | 15 +++++++++++++++
libibverbs/verbs.h | 18 ++++++++++++++++++
8 files changed, 61 insertions(+), 3 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 53843767..afbf4f31 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -72,7 +72,7 @@ set(PACKAGE_VERSION "30.0")
# When this is changed the values in these files need changing too:
# debian/control
# debian/libibverbs1.symbols
-set(IBVERBS_PABI_VERSION "25")
+set(IBVERBS_PABI_VERSION "26")
set(IBVERBS_PROVIDER_SUFFIX "-rdmav${IBVERBS_PABI_VERSION}.so")
#-------------------------
diff --git a/debian/libibverbs1.symbols b/debian/libibverbs1.symbols
index ec40b29b..0a6b3e70 100644
--- a/debian/libibverbs1.symbols
+++ b/debian/libibverbs1.symbols
@@ -6,7 +6,8 @@ libibverbs.so.1 libibverbs1 #MINVER#
IBVERBS_1.6@IBVERBS_1.6 24
IBVERBS_1.7@IBVERBS_1.7 25
IBVERBS_1.8@IBVERBS_1.8 28
- (symver)IBVERBS_PRIVATE_25 25
+ IBVERBS_1.9@IBVERBS_1.9 28
+ (symver)IBVERBS_PRIVATE_26 26
ibv_ack_async_event@IBVERBS_1.0 1.1.6
ibv_ack_async_event@IBVERBS_1.1 1.1.6
ibv_ack_cq_events@IBVERBS_1.0 1.1.6
@@ -76,6 +77,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
ibv_qp_to_qp_ex@IBVERBS_1.6 24
ibv_query_device@IBVERBS_1.0 1.1.6
ibv_query_device@IBVERBS_1.1 1.1.6
+ ibv_query_ece@IBVERBS_1.9 28
ibv_query_gid@IBVERBS_1.0 1.1.6
ibv_query_gid@IBVERBS_1.1 1.1.6
ibv_query_pkey@IBVERBS_1.0 1.1.6
@@ -98,6 +100,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
ibv_resize_cq@IBVERBS_1.0 1.1.6
ibv_resize_cq@IBVERBS_1.1 1.1.6
ibv_resolve_eth_l2_from_gid@IBVERBS_1.1 1.2.0
+ ibv_set_ece@IBVERBS_1.9 28
ibv_wc_status_str@IBVERBS_1.1 1.1.6
mbps_to_ibv_rate@IBVERBS_1.1 1.1.8
mult_to_ibv_rate@IBVERBS_1.0 1.1.6
diff --git a/libibverbs/CMakeLists.txt b/libibverbs/CMakeLists.txt
index 43285489..a10bf103 100644
--- a/libibverbs/CMakeLists.txt
+++ b/libibverbs/CMakeLists.txt
@@ -21,7 +21,7 @@ configure_file("libibverbs.map.in"
rdma_library(ibverbs "${CMAKE_CURRENT_BINARY_DIR}/libibverbs.map"
# See Documentation/versioning.md
- 1 1.8.${PACKAGE_VERSION}
+ 1 1.9.${PACKAGE_VERSION}
all_providers.c
cmd.c
cmd_ah.c
diff --git a/libibverbs/driver.h b/libibverbs/driver.h
index a0e6f89a..f14ce537 100644
--- a/libibverbs/driver.h
+++ b/libibverbs/driver.h
@@ -348,6 +348,7 @@ struct verbs_context_ops {
const struct ibv_query_device_ex_input *input,
struct ibv_device_attr_ex *attr,
size_t attr_size);
+ int (*query_ece)(struct ibv_qp *qp, struct ibv_ece *ece);
int (*query_port)(struct ibv_context *context, uint8_t port_num,
struct ibv_port_attr *port_attr);
int (*query_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
@@ -368,6 +369,7 @@ struct verbs_context_ops {
int (*rereg_mr)(struct verbs_mr *vmr, int flags, struct ibv_pd *pd,
void *addr, size_t length, int access);
int (*resize_cq)(struct ibv_cq *cq, int cqe);
+ int (*set_ece)(struct ibv_qp *qp, struct ibv_ece *ece);
};
static inline struct verbs_device *
diff --git a/libibverbs/dummy_ops.c b/libibverbs/dummy_ops.c
index 32fec71f..b5f48be8 100644
--- a/libibverbs/dummy_ops.c
+++ b/libibverbs/dummy_ops.c
@@ -450,6 +450,16 @@ static int resize_cq(struct ibv_cq *cq, int cqe)
return EOPNOTSUPP;
}
+static int set_ece(struct ibv_qp *qp, struct ibv_ece *ece)
+{
+ return EOPNOTSUPP;
+}
+
+static int query_ece(struct ibv_qp *qp, struct ibv_ece *ece)
+{
+ return EOPNOTSUPP;
+}
+
/*
* Ops in verbs_dummy_ops simply return an EOPNOTSUPP error code when called, or
* do nothing. They are placed in the ops structures if the provider does not
@@ -519,6 +529,7 @@ const struct verbs_context_ops verbs_dummy_ops = {
post_srq_recv,
query_device,
query_device_ex,
+ query_ece,
query_port,
query_qp,
query_rt_values,
@@ -529,6 +540,7 @@ const struct verbs_context_ops verbs_dummy_ops = {
req_notify_cq,
rereg_mr,
resize_cq,
+ set_ece,
};
/*
@@ -635,6 +647,7 @@ void verbs_set_ops(struct verbs_context *vctx,
SET_OP(ctx, post_srq_recv);
SET_PRIV_OP(ctx, query_device);
SET_OP(vctx, query_device_ex);
+ SET_PRIV_OP_IC(vctx, query_ece);
SET_PRIV_OP_IC(ctx, query_port);
SET_PRIV_OP(ctx, query_qp);
SET_OP(vctx, query_rt_values);
@@ -645,6 +658,7 @@ void verbs_set_ops(struct verbs_context *vctx,
SET_OP(ctx, req_notify_cq);
SET_PRIV_OP(ctx, rereg_mr);
SET_PRIV_OP(ctx, resize_cq);
+ SET_PRIV_OP_IC(vctx, set_ece);
#undef SET_OP
#undef SET_OP2
diff --git a/libibverbs/libibverbs.map.in b/libibverbs/libibverbs.map.in
index 5280cfe6..52ce0c80 100644
--- a/libibverbs/libibverbs.map.in
+++ b/libibverbs/libibverbs.map.in
@@ -126,6 +126,12 @@ IBVERBS_1.8 {
ibv_reg_mr_iova2;
} IBVERBS_1.7;
+IBVERBS_1.9 {
+ global:
+ ibv_query_ece;
+ ibv_set_ece;
+} IBVERBS_1.8;
+
/* If any symbols in this stanza change ABI then the entire staza gets a new symbol
version. See the top level CMakeLists.txt for this setting. */
diff --git a/libibverbs/verbs.c b/libibverbs/verbs.c
index 629f24c2..35d471e4 100644
--- a/libibverbs/verbs.c
+++ b/libibverbs/verbs.c
@@ -1080,3 +1080,18 @@ free_resources:
return ret;
}
+
+int ibv_set_ece(struct ibv_qp *qp, struct ibv_ece *ece)
+{
+ if (!ece->vendor_id || ece->comp_mask) {
+ errno = EINVAL;
+ return -1;
+ }
+
+ return get_ops(qp->context)->set_ece(qp, ece);
+}
+
+int ibv_query_ece(struct ibv_qp *qp, struct ibv_ece *ece)
+{
+ return get_ops(qp->context)->query_ece(qp, ece);
+}
diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h
index 288985d5..0968c0aa 100644
--- a/libibverbs/verbs.h
+++ b/libibverbs/verbs.h
@@ -1401,6 +1401,21 @@ static inline void ibv_wr_abort(struct ibv_qp_ex *qp)
qp->wr_abort(qp);
}
+struct ibv_ece {
+ /*
+ * Unique identifier of the provider vendor on the network.
+ * The providers will set IEEE OUI here to distinguish
+ * itself in non-homogenius network.
+ */
+ uint32_t vendor_id;
+ /*
+ * Provider specific attributes which are supported or
+ * needed to be enabled by ECE users.
+ */
+ uint32_t options;
+ uint32_t comp_mask;
+};
+
struct ibv_comp_channel {
struct ibv_context *context;
int fd;
@@ -3342,6 +3357,9 @@ static inline uint16_t ibv_flow_label_to_udp_sport(uint32_t fl)
return (uint16_t)(fl_low | IB_ROCE_UDP_ENCAP_VALID_PORT_MIN);
}
+int ibv_set_ece(struct ibv_qp *qp, struct ibv_ece *ece);
+int ibv_query_ece(struct ibv_qp *qp, struct ibv_ece *ece);
+
#ifdef __cplusplus
}
#endif
--
2.25.2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH rdma-core 03/12] libibverbs: Document ECE API
2020-04-20 14:06 [PATCH rdma-core 00/12] Add Enhanced Connection Established (ECE) APIs Leon Romanovsky
2020-04-20 14:06 ` [PATCH rdma-core 01/12] Update kernel headers Leon Romanovsky
2020-04-20 14:06 ` [PATCH rdma-core 02/12] libibverbs: Add interfaces to configure and use ECE Leon Romanovsky
@ 2020-04-20 14:06 ` Leon Romanovsky
2020-04-20 14:06 ` [PATCH rdma-core 04/12] debian: Install all available librdmacm man pages Leon Romanovsky
` (8 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Leon Romanovsky @ 2020-04-20 14:06 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe; +Cc: Leon Romanovsky, linux-rdma
From: Leon Romanovsky <leonro@mellanox.com>
Add manual pages for libibverbs part of ECE.
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
libibverbs/man/CMakeLists.txt | 2 +
libibverbs/man/ibv_query_ece.3.md | 56 ++++++++++++++++++++++++++++
libibverbs/man/ibv_set_ece.3.md | 61 +++++++++++++++++++++++++++++++
3 files changed, 119 insertions(+)
create mode 100644 libibverbs/man/ibv_query_ece.3.md
create mode 100644 libibverbs/man/ibv_set_ece.3.md
diff --git a/libibverbs/man/CMakeLists.txt b/libibverbs/man/CMakeLists.txt
index e1d5edf8..c0ff7a25 100644
--- a/libibverbs/man/CMakeLists.txt
+++ b/libibverbs/man/CMakeLists.txt
@@ -51,6 +51,7 @@ rdma_man_pages(
ibv_post_srq_recv.3
ibv_query_device.3
ibv_query_device_ex.3
+ ibv_query_ece.3.md
ibv_query_gid.3.md
ibv_query_pkey.3.md
ibv_query_port.3
@@ -65,6 +66,7 @@ rdma_man_pages(
ibv_req_notify_cq.3.md
ibv_rereg_mr.3.md
ibv_resize_cq.3.md
+ ibv_set_ece.3.md
ibv_srq_pingpong.1
ibv_uc_pingpong.1
ibv_ud_pingpong.1
diff --git a/libibverbs/man/ibv_query_ece.3.md b/libibverbs/man/ibv_query_ece.3.md
new file mode 100644
index 00000000..b9eee699
--- /dev/null
+++ b/libibverbs/man/ibv_query_ece.3.md
@@ -0,0 +1,56 @@
+---
+date: 2020-01-22
+footer: libibverbs
+header: "Libibverbs Programmer's Manual"
+layout: page
+license: 'Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md'
+section: 3
+title: IBV_QUERY_ECE
+---
+
+# NAME
+
+ibv_query_ece - query ECE options.
+
+# SYNOPSIS
+
+```c
+#include <infiniband/verbs.h>
+
+int ibv_query_ece(struct ibv_qp *qp, struct ibv_ece *ece);
+```
+
+# DESCRIPTION
+
+**ibv_query_ece()** query ECE options.
+
+Return to the user current ECE state for the QP.
+
+# ARGUMENTS
+*qp*
+: The queue pair (QP) associated with the ECE options.
+
+*ece*
+: The ECE values.
+
+# RETURN VALUE
+
+**ibv_query_ece()** returns 0 when the call was successful, or the negative
+ value with errno which indicates the failure reason.
+
+*EOPNOTSUPP*
+: libibverbs or provider driver doesn't support the ibv_set_ece() verb.
+
+*EINVAL*
+: In one of the following:
+ o The QP is invalid.
+ o The ECE options are invalid.
+
+# SEE ALSO
+
+**ibv_set_ece**(3),
+
+# AUTHOR
+
+Leon Romanovsky <leonro@mellanox.com>
+
diff --git a/libibverbs/man/ibv_set_ece.3.md b/libibverbs/man/ibv_set_ece.3.md
new file mode 100644
index 00000000..52d5f8f8
--- /dev/null
+++ b/libibverbs/man/ibv_set_ece.3.md
@@ -0,0 +1,61 @@
+---
+date: 2020-01-22
+footer: libibverbs
+header: "Libibverbs Programmer's Manual"
+layout: page
+license: 'Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md'
+section: 3
+title: IBV_COMMIT_ECE
+---
+
+# NAME
+
+ibv_set_ece - set ECE options and use them for QP configuration stage.
+
+# SYNOPSIS
+
+```c
+#include <infiniband/verbs.h>
+
+int ibv_set_ece(struct ibv_qp *qp, struct ibv_ece *ece);
+```
+
+# DESCRIPTION
+
+**ibv_set_ece()** set ECE options and use them for QP configuration stage.
+
+The desired ECE options will be used during various modify QP stages
+based on supported options in relevant QP state.
+
+# ARGUMENTS
+*qp*
+: The queue pair (QP) associated with the ECE options.
+
+*ece*
+: The requested ECE values. This is IN/OUT field, the accepted options
+ will be returned in this field.
+
+# RETURN VALUE
+
+**ibv_set_ece()** returns 0 when the call was successful, or the negative
+ value with errno which indicates the failure reason.
+
+*EOPNOTSUPP*
+: libibverbs or provider driver doesn't support the ibv_set_ece() verb.
+
+*ECONNREFUSED*
+: Connection refused, provider doesn't want those options.
+
+*EINVAL*
+: In one of the following:
+ o The QP is invalid.
+ o The ECE options are invalid.
+
+# SEE ALSO
+
+**ibv_query_ece**(3),
+
+# AUTHOR
+
+Leon Romanovsky <leonro@mellanox.com>
+
--
2.25.2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH rdma-core 04/12] debian: Install all available librdmacm man pages
2020-04-20 14:06 [PATCH rdma-core 00/12] Add Enhanced Connection Established (ECE) APIs Leon Romanovsky
` (2 preceding siblings ...)
2020-04-20 14:06 ` [PATCH rdma-core 03/12] libibverbs: Document ECE API Leon Romanovsky
@ 2020-04-20 14:06 ` Leon Romanovsky
2020-04-20 14:06 ` [PATCH rdma-core 05/12] librdmacm: Provide interface to use ECE for external QPs Leon Romanovsky
` (7 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Leon Romanovsky @ 2020-04-20 14:06 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe; +Cc: Leon Romanovsky, linux-rdma
From: Leon Romanovsky <leonro@mellanox.com>
There is no need to be explicit for every man page in librdmacm,
use simple '*' to install all rdma_*.3 man pages in one shot.
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
debian/librdmacm-dev.install | 53 +-----------------------------------
1 file changed, 1 insertion(+), 52 deletions(-)
diff --git a/debian/librdmacm-dev.install b/debian/librdmacm-dev.install
index e12c300a..68835cb0 100644
--- a/debian/librdmacm-dev.install
+++ b/debian/librdmacm-dev.install
@@ -6,57 +6,6 @@ usr/include/rdma/rsocket.h
usr/lib/*/librdmacm*.so
usr/lib/*/librdmacm.a
usr/lib/*/pkgconfig/librdmacm.pc
-usr/share/man/man3/rdma_accept.3
-usr/share/man/man3/rdma_ack_cm_event.3
-usr/share/man/man3/rdma_bind_addr.3
-usr/share/man/man3/rdma_connect.3
-usr/share/man/man3/rdma_create_ep.3
-usr/share/man/man3/rdma_create_event_channel.3
-usr/share/man/man3/rdma_create_id.3
-usr/share/man/man3/rdma_create_qp.3
-usr/share/man/man3/rdma_create_srq.3
-usr/share/man/man3/rdma_dereg_mr.3
-usr/share/man/man3/rdma_destroy_ep.3
-usr/share/man/man3/rdma_destroy_event_channel.3
-usr/share/man/man3/rdma_destroy_id.3
-usr/share/man/man3/rdma_destroy_qp.3
-usr/share/man/man3/rdma_destroy_srq.3
-usr/share/man/man3/rdma_disconnect.3
-usr/share/man/man3/rdma_establish.3
-usr/share/man/man3/rdma_event_str.3
-usr/share/man/man3/rdma_free_devices.3
-usr/share/man/man3/rdma_get_cm_event.3
-usr/share/man/man3/rdma_get_devices.3
-usr/share/man/man3/rdma_get_dst_port.3
-usr/share/man/man3/rdma_get_local_addr.3
-usr/share/man/man3/rdma_get_peer_addr.3
-usr/share/man/man3/rdma_get_recv_comp.3
-usr/share/man/man3/rdma_get_request.3
-usr/share/man/man3/rdma_get_send_comp.3
-usr/share/man/man3/rdma_get_src_port.3
-usr/share/man/man3/rdma_getaddrinfo.3
-usr/share/man/man3/rdma_init_qp_attr.3
-usr/share/man/man3/rdma_join_multicast.3
-usr/share/man/man3/rdma_join_multicast_ex.3
-usr/share/man/man3/rdma_leave_multicast.3
-usr/share/man/man3/rdma_listen.3
-usr/share/man/man3/rdma_migrate_id.3
-usr/share/man/man3/rdma_notify.3
-usr/share/man/man3/rdma_post_read.3
-usr/share/man/man3/rdma_post_readv.3
-usr/share/man/man3/rdma_post_recv.3
-usr/share/man/man3/rdma_post_recvv.3
-usr/share/man/man3/rdma_post_send.3
-usr/share/man/man3/rdma_post_sendv.3
-usr/share/man/man3/rdma_post_ud_send.3
-usr/share/man/man3/rdma_post_write.3
-usr/share/man/man3/rdma_post_writev.3
-usr/share/man/man3/rdma_reg_msgs.3
-usr/share/man/man3/rdma_reg_read.3
-usr/share/man/man3/rdma_reg_write.3
-usr/share/man/man3/rdma_reject.3
-usr/share/man/man3/rdma_resolve_addr.3
-usr/share/man/man3/rdma_resolve_route.3
-usr/share/man/man3/rdma_set_option.3
+usr/share/man/man3/rdma_*.3
usr/share/man/man7/rdma_cm.7
usr/share/man/man7/rsocket.7
--
2.25.2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH rdma-core 05/12] librdmacm: Provide interface to use ECE for external QPs
2020-04-20 14:06 [PATCH rdma-core 00/12] Add Enhanced Connection Established (ECE) APIs Leon Romanovsky
` (3 preceding siblings ...)
2020-04-20 14:06 ` [PATCH rdma-core 04/12] debian: Install all available librdmacm man pages Leon Romanovsky
@ 2020-04-20 14:06 ` Leon Romanovsky
2020-04-20 14:06 ` [PATCH rdma-core 06/12] librdmacm: Connect rdma_connect to the ECE Leon Romanovsky
` (6 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Leon Romanovsky @ 2020-04-20 14:06 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe; +Cc: Leon Romanovsky, linux-rdma
From: Leon Romanovsky <leonro@mellanox.com>
Add the following calls to allow use of ECE for external QPs.
Those QPs are not managed by librdmacm and can be any type
and not strictly ibv_qp.
* rdma_set_local_ece() - provide to the librdmacm the desired
ECE options to be used in REQ/REP handshake.
* rdma_get_remote_ece() - get ECE options received from the peer,
so users will be able to accept/reject/mask supported ECE options.
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
debian/librdmacm1.symbols | 3 +++
librdmacm/CMakeLists.txt | 2 +-
librdmacm/cma.c | 51 +++++++++++++++++++++++++++++++++++++++
librdmacm/librdmacm.map | 6 +++++
librdmacm/rdma_cma.h | 16 ++++++++++++
5 files changed, 77 insertions(+), 1 deletion(-)
diff --git a/debian/librdmacm1.symbols b/debian/librdmacm1.symbols
index 996122f3..cc4d7a6c 100644
--- a/debian/librdmacm1.symbols
+++ b/debian/librdmacm1.symbols
@@ -3,6 +3,7 @@ librdmacm.so.1 librdmacm1 #MINVER#
RDMACM_1.0@RDMACM_1.0 1.0.15
RDMACM_1.1@RDMACM_1.1 16
RDMACM_1.2@RDMACM_1.2 23
+ RDMACM_1.3@RDMACM_1.3 28
raccept@RDMACM_1.0 1.0.16
rbind@RDMACM_1.0 1.0.16
rclose@RDMACM_1.0 1.0.16
@@ -31,6 +32,7 @@ librdmacm.so.1 librdmacm1 #MINVER#
rdma_get_cm_event@RDMACM_1.0 1.0.15
rdma_get_devices@RDMACM_1.0 1.0.15
rdma_get_dst_port@RDMACM_1.0 1.0.19
+ rdma_get_remote_ece@RDMACM_1.3 28
rdma_get_request@RDMACM_1.0 1.0.15
rdma_get_src_port@RDMACM_1.0 1.0.19
rdma_getaddrinfo@RDMACM_1.0 1.0.15
@@ -44,6 +46,7 @@ librdmacm.so.1 librdmacm1 #MINVER#
rdma_reject@RDMACM_1.0 1.0.15
rdma_resolve_addr@RDMACM_1.0 1.0.15
rdma_resolve_route@RDMACM_1.0 1.0.15
+ rdma_set_local_ece@RDMACM_1.3 28
rdma_set_option@RDMACM_1.0 1.0.15
rfcntl@RDMACM_1.0 1.0.16
rgetpeername@RDMACM_1.0 1.0.16
diff --git a/librdmacm/CMakeLists.txt b/librdmacm/CMakeLists.txt
index f0767cff..b01fef4f 100644
--- a/librdmacm/CMakeLists.txt
+++ b/librdmacm/CMakeLists.txt
@@ -11,7 +11,7 @@ publish_headers(infiniband
rdma_library(rdmacm librdmacm.map
# See Documentation/versioning.md
- 1 1.2.${PACKAGE_VERSION}
+ 1 1.3.${PACKAGE_VERSION}
acm.c
addrinfo.c
cma.c
diff --git a/librdmacm/cma.c b/librdmacm/cma.c
index 9855d0a8..a12ed1e0 100644
--- a/librdmacm/cma.c
+++ b/librdmacm/cma.c
@@ -106,6 +106,8 @@ struct cma_id_private {
struct ibv_qp_init_attr *qp_init_attr;
uint8_t initiator_depth;
uint8_t responder_resources;
+ struct ibv_ece local_ece;
+ struct ibv_ece remote_ece;
};
struct cma_multicast {
@@ -1382,6 +1384,24 @@ void rdma_destroy_srq(struct rdma_cm_id *id)
ucma_destroy_cqs(id);
}
+static int init_ece(struct rdma_cm_id *id, struct ibv_qp *qp)
+{
+ struct cma_id_private *id_priv =
+ container_of(id, struct cma_id_private, id);
+ struct ibv_ece ece = {};
+ int ret;
+
+ ret = ibv_query_ece(qp, &ece);
+ if (ret)
+ return 0;
+
+ id_priv->local_ece.vendor_id = ece.vendor_id;
+ id_priv->local_ece.options = ece.options;
+
+ return 0;
+}
+
+
int rdma_create_qp_ex(struct rdma_cm_id *id,
struct ibv_qp_init_attr_ex *attr)
{
@@ -1429,6 +1449,9 @@ int rdma_create_qp_ex(struct rdma_cm_id *id,
goto err1;
}
+ ret = init_ece(id, qp);
+ if (ret)
+ goto err2;
if (ucma_is_ud_qp(id->qp_type))
ret = ucma_init_ud_qp(id_priv, qp);
else
@@ -2561,3 +2584,31 @@ __be16 rdma_get_dst_port(struct rdma_cm_id *id)
return ucma_get_port(&id->route.addr.dst_addr);
}
+int rdma_set_local_ece(struct rdma_cm_id *id, struct ibv_ece *ece)
+{
+ struct cma_id_private *id_priv;
+
+ if (!id || id->qp || !ece || !ece->vendor_id || ece->comp_mask)
+ return ERR(EINVAL);
+
+ id_priv = container_of(id, struct cma_id_private, id);
+ id_priv->local_ece.vendor_id = ece->vendor_id;
+ id_priv->local_ece.options = ece->options;
+
+ return 0;
+}
+
+int rdma_get_remote_ece(struct rdma_cm_id *id, struct ibv_ece *ece)
+{
+ struct cma_id_private *id_priv;
+
+ if (id->qp || !ece)
+ return ERR(EINVAL);
+
+ id_priv = container_of(id, struct cma_id_private, id);
+ ece->vendor_id = id_priv->remote_ece.vendor_id;
+ ece->options = id_priv->remote_ece.options;
+ ece->comp_mask = 0;
+
+ return 0;
+}
diff --git a/librdmacm/librdmacm.map b/librdmacm/librdmacm.map
index 7f55e844..f29a23b4 100644
--- a/librdmacm/librdmacm.map
+++ b/librdmacm/librdmacm.map
@@ -82,3 +82,9 @@ RDMACM_1.2 {
rdma_establish;
rdma_init_qp_attr;
} RDMACM_1.1;
+
+RDMACM_1.3 {
+ global:
+ rdma_get_remote_ece;
+ rdma_set_local_ece;
+} RDMACM_1.2;
diff --git a/librdmacm/rdma_cma.h b/librdmacm/rdma_cma.h
index 19050332..c42a28f7 100644
--- a/librdmacm/rdma_cma.h
+++ b/librdmacm/rdma_cma.h
@@ -753,6 +753,22 @@ void rdma_freeaddrinfo(struct rdma_addrinfo *res);
*/
int rdma_init_qp_attr(struct rdma_cm_id *id, struct ibv_qp_attr *qp_attr,
int *qp_attr_mask);
+
+/**
+ * rdma_set_local_ece - Set local ECE options to be used for REQ/REP
+ * communication. In use to implement ECE handshake in external QP.
+ * @id: Communication identifier to establish connection
+ * @ece: ECE parameters
+ */
+int rdma_set_local_ece(struct rdma_cm_id *id, struct ibv_ece *ece);
+
+/**
+ * rdma_get_remote_ece - Provide remote ECE parameters as received
+ * in REQ/REP events. In use to implement ECE handshake in external QP.
+ * @id: Communication identifier to establish connection
+ * @ece: ECE parameters
+ */
+int rdma_get_remote_ece(struct rdma_cm_id *id, struct ibv_ece *ece);
#ifdef __cplusplus
}
#endif
--
2.25.2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH rdma-core 06/12] librdmacm: Connect rdma_connect to the ECE
2020-04-20 14:06 [PATCH rdma-core 00/12] Add Enhanced Connection Established (ECE) APIs Leon Romanovsky
` (4 preceding siblings ...)
2020-04-20 14:06 ` [PATCH rdma-core 05/12] librdmacm: Provide interface to use ECE for external QPs Leon Romanovsky
@ 2020-04-20 14:06 ` Leon Romanovsky
2020-04-20 14:06 ` [PATCH rdma-core 07/12] librdmacm: Return ECE results through rdma_accept Leon Romanovsky
` (5 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Leon Romanovsky @ 2020-04-20 14:06 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe; +Cc: Leon Romanovsky, linux-rdma
From: Leon Romanovsky <leonro@mellanox.com>
The ECE options are handled to the kernel at the time
of connection request, e.g. rdma_connect().
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
librdmacm/cma.c | 9 +++++++++
librdmacm/rdma_cma_abi.h | 6 ++++++
2 files changed, 15 insertions(+)
diff --git a/librdmacm/cma.c b/librdmacm/cma.c
index a12ed1e0..91735225 100644
--- a/librdmacm/cma.c
+++ b/librdmacm/cma.c
@@ -1547,6 +1547,13 @@ static void ucma_copy_conn_param_to_kern(struct cma_id_private *id_priv,
}
}
+static void ucma_copy_ece_param_to_kern_req(struct cma_id_private *id_priv,
+ struct ucma_abi_ece *dst)
+{
+ dst->vendor_id = id_priv->local_ece.vendor_id;
+ dst->attr_mod = id_priv->local_ece.options;
+}
+
int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
{
uint32_t qp_num = conn_param ? conn_param->qp_num : 0;
@@ -1579,6 +1586,8 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
ucma_copy_conn_param_to_kern(id_priv, &cmd.conn_param, conn_param,
qp_num, srq);
+ ucma_copy_ece_param_to_kern_req(id_priv, &cmd.ece);
+
ret = write(id->channel->fd, &cmd, sizeof cmd);
if (ret != sizeof cmd)
return (ret >= 0) ? ERR(ENODATA) : -1;
diff --git a/librdmacm/rdma_cma_abi.h b/librdmacm/rdma_cma_abi.h
index ab4adb00..6451862e 100644
--- a/librdmacm/rdma_cma_abi.h
+++ b/librdmacm/rdma_cma_abi.h
@@ -223,6 +223,11 @@ struct ucma_abi_ud_param {
__u8 reserved2[4]; /* Round to 8-byte boundary to support 32/64 */
};
+struct ucma_abi_ece {
+ __u32 vendor_id;
+ __u32 attr_mod;
+};
+
struct ucma_abi_connect {
__u32 cmd;
__u16 in;
@@ -230,6 +235,7 @@ struct ucma_abi_connect {
struct ucma_abi_conn_param conn_param;
__u32 id;
__u32 reserved;
+ struct ucma_abi_ece ece;
};
struct ucma_abi_listen {
--
2.25.2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH rdma-core 07/12] librdmacm: Return ECE results through rdma_accept
2020-04-20 14:06 [PATCH rdma-core 00/12] Add Enhanced Connection Established (ECE) APIs Leon Romanovsky
` (5 preceding siblings ...)
2020-04-20 14:06 ` [PATCH rdma-core 06/12] librdmacm: Connect rdma_connect to the ECE Leon Romanovsky
@ 2020-04-20 14:06 ` Leon Romanovsky
2020-04-20 14:06 ` [PATCH rdma-core 08/12] librdmacm: Add an option to reject ECE request Leon Romanovsky
` (4 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Leon Romanovsky @ 2020-04-20 14:06 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe; +Cc: Leon Romanovsky, linux-rdma
From: Leon Romanovsky <leonro@mellanox.com>
Passive side (server) returns its answer through REP message,
which is constructed as a result of rdma_accept().
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
librdmacm/cma.c | 9 +++++++++
librdmacm/rdma_cma_abi.h | 1 +
2 files changed, 10 insertions(+)
diff --git a/librdmacm/cma.c b/librdmacm/cma.c
index 91735225..3c356eca 100644
--- a/librdmacm/cma.c
+++ b/librdmacm/cma.c
@@ -1668,6 +1668,14 @@ err:
return ret;
}
+static void ucma_copy_ece_param_to_kern_rep(struct cma_id_private *id_priv,
+ struct ucma_abi_ece *dst)
+{
+ /* Return result with same ID as received. */
+ dst->vendor_id = id_priv->remote_ece.vendor_id;
+ dst->attr_mod = id_priv->local_ece.options;
+}
+
int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
{
uint32_t qp_num = id->qp ? id->qp->qp_num : conn_param->qp_num;
@@ -1709,6 +1717,7 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
cmd.uid = (uintptr_t) id_priv;
ucma_copy_conn_param_to_kern(id_priv, &cmd.conn_param, conn_param,
qp_num, srq);
+ ucma_copy_ece_param_to_kern_rep(id_priv, &cmd.ece);
ret = write(id->channel->fd, &cmd, sizeof cmd);
if (ret != sizeof cmd) {
diff --git a/librdmacm/rdma_cma_abi.h b/librdmacm/rdma_cma_abi.h
index 6451862e..4639941b 100644
--- a/librdmacm/rdma_cma_abi.h
+++ b/librdmacm/rdma_cma_abi.h
@@ -254,6 +254,7 @@ struct ucma_abi_accept {
struct ucma_abi_conn_param conn_param;
__u32 id;
__u32 reserved;
+ struct ucma_abi_ece ece;
};
struct ucma_abi_reject {
--
2.25.2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH rdma-core 08/12] librdmacm: Add an option to reject ECE request
2020-04-20 14:06 [PATCH rdma-core 00/12] Add Enhanced Connection Established (ECE) APIs Leon Romanovsky
` (6 preceding siblings ...)
2020-04-20 14:06 ` [PATCH rdma-core 07/12] librdmacm: Return ECE results through rdma_accept Leon Romanovsky
@ 2020-04-20 14:06 ` Leon Romanovsky
2020-04-20 14:06 ` [PATCH rdma-core 09/12] librdmacm: Implement ECE handshake logic Leon Romanovsky
` (3 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Leon Romanovsky @ 2020-04-20 14:06 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe; +Cc: Leon Romanovsky, linux-rdma
From: Leon Romanovsky <leonro@mellanox.com>
IBTA has specific rejected reason for users who doesn't
want proposed ECE options in request messages. Provide special
version (rdma_reject_ece) to mark such rejects.
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
debian/librdmacm1.symbols | 1 +
librdmacm/cma.c | 19 +++++++++++++++++--
librdmacm/librdmacm.map | 1 +
librdmacm/rdma_cma.h | 8 ++++++++
librdmacm/rdma_cma_abi.h | 7 ++++++-
5 files changed, 33 insertions(+), 3 deletions(-)
diff --git a/debian/librdmacm1.symbols b/debian/librdmacm1.symbols
index cc4d7a6c..ce5ffd20 100644
--- a/debian/librdmacm1.symbols
+++ b/debian/librdmacm1.symbols
@@ -44,6 +44,7 @@ librdmacm.so.1 librdmacm1 #MINVER#
rdma_migrate_id@RDMACM_1.0 1.0.15
rdma_notify@RDMACM_1.0 1.0.15
rdma_reject@RDMACM_1.0 1.0.15
+ rdma_reject_ece@RDMACM_1.3 28
rdma_resolve_addr@RDMACM_1.0 1.0.15
rdma_resolve_route@RDMACM_1.0 1.0.15
rdma_set_local_ece@RDMACM_1.3 28
diff --git a/librdmacm/cma.c b/librdmacm/cma.c
index 3c356eca..fc6e6a89 100644
--- a/librdmacm/cma.c
+++ b/librdmacm/cma.c
@@ -1731,8 +1731,9 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
return ucma_complete(id);
}
-int rdma_reject(struct rdma_cm_id *id, const void *private_data,
- uint8_t private_data_len)
+static int reject_with_reason(struct rdma_cm_id *id, const void *private_data,
+ uint8_t private_data_len,
+ enum ucm_abi_reject_reason reason)
{
struct ucma_abi_reject cmd;
struct cma_id_private *id_priv;
@@ -1746,6 +1747,7 @@ int rdma_reject(struct rdma_cm_id *id, const void *private_data,
memcpy(cmd.private_data, private_data, private_data_len);
cmd.private_data_len = private_data_len;
}
+ cmd.reason = reason;
ret = write(id->channel->fd, &cmd, sizeof cmd);
if (ret != sizeof cmd)
@@ -1754,6 +1756,19 @@ int rdma_reject(struct rdma_cm_id *id, const void *private_data,
return 0;
}
+int rdma_reject(struct rdma_cm_id *id, const void *private_data,
+ uint8_t private_data_len)
+{
+ return reject_with_reason(id, private_data, private_data_len, 0);
+}
+
+int rdma_reject_ece(struct rdma_cm_id *id, const void *private_data,
+ uint8_t private_data_len)
+{
+ return reject_with_reason(id, private_data, private_data_len,
+ RDMA_USER_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED);
+}
+
int rdma_notify(struct rdma_cm_id *id, enum ibv_event_type event)
{
struct ucma_abi_notify cmd;
diff --git a/librdmacm/librdmacm.map b/librdmacm/librdmacm.map
index f29a23b4..d162ef09 100644
--- a/librdmacm/librdmacm.map
+++ b/librdmacm/librdmacm.map
@@ -86,5 +86,6 @@ RDMACM_1.2 {
RDMACM_1.3 {
global:
rdma_get_remote_ece;
+ rdma_reject_ece;
rdma_set_local_ece;
} RDMACM_1.2;
diff --git a/librdmacm/rdma_cma.h b/librdmacm/rdma_cma.h
index c42a28f7..e1f4e236 100644
--- a/librdmacm/rdma_cma.h
+++ b/librdmacm/rdma_cma.h
@@ -524,6 +524,14 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param);
int rdma_reject(struct rdma_cm_id *id, const void *private_data,
uint8_t private_data_len);
+/**
+ * rdma_reject_ece - Called to reject a connection request with ECE
+ * rejected reason.
+ * The same as rdma_reject()
+ */
+int rdma_reject_ece(struct rdma_cm_id *id, const void *private_data,
+ uint8_t private_data_len);
+
/**
* rdma_notify - Notifies the librdmacm of an asynchronous event.
* @id: RDMA identifier.
diff --git a/librdmacm/rdma_cma_abi.h b/librdmacm/rdma_cma_abi.h
index 4639941b..911863cc 100644
--- a/librdmacm/rdma_cma_abi.h
+++ b/librdmacm/rdma_cma_abi.h
@@ -73,6 +73,10 @@ enum {
UCMA_CMD_JOIN_MCAST
};
+enum ucm_abi_reject_reason {
+ RDMA_USER_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED = 35
+};
+
struct ucma_abi_cmd_hdr {
__u32 cmd;
__u16 in;
@@ -263,7 +267,8 @@ struct ucma_abi_reject {
__u16 out;
__u32 id;
__u8 private_data_len;
- __u8 reserved[3];
+ __u8 reason; /* enum ucm_abi_reject_reason */
+ __u8 reserved[2];
__u8 private_data[RDMA_MAX_PRIVATE_DATA];
};
--
2.25.2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH rdma-core 09/12] librdmacm: Implement ECE handshake logic
2020-04-20 14:06 [PATCH rdma-core 00/12] Add Enhanced Connection Established (ECE) APIs Leon Romanovsky
` (7 preceding siblings ...)
2020-04-20 14:06 ` [PATCH rdma-core 08/12] librdmacm: Add an option to reject ECE request Leon Romanovsky
@ 2020-04-20 14:06 ` Leon Romanovsky
2020-04-20 14:06 ` [PATCH rdma-core 10/12] librdmacm: Document ECE API Leon Romanovsky
` (2 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Leon Romanovsky @ 2020-04-20 14:06 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe; +Cc: Leon Romanovsky, linux-rdma
From: Leon Romanovsky <leonro@mellanox.com>
Perform ECE handshake through REQ/REP messages. The REQ message
initiated by active side towards passive side and carries the
ECE data. The passive side returns its decision in REP message.
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
librdmacm/cma.c | 46 ++++++++++++++++++++++++++++++++++------
librdmacm/rdma_cma_abi.h | 1 +
2 files changed, 40 insertions(+), 7 deletions(-)
diff --git a/librdmacm/cma.c b/librdmacm/cma.c
index fc6e6a89..db562496 100644
--- a/librdmacm/cma.c
+++ b/librdmacm/cma.c
@@ -1398,10 +1398,20 @@ static int init_ece(struct rdma_cm_id *id, struct ibv_qp *qp)
id_priv->local_ece.vendor_id = ece.vendor_id;
id_priv->local_ece.options = ece.options;
+ if (!id_priv->remote_ece.vendor_id)
+ return 0;
+
+ /* This QP was created due to REQ event */
+ ece.vendor_id = id_priv->remote_ece.vendor_id;
+ ece.options = id_priv->remote_ece.options;
+ ret = ibv_set_ece(qp, &ece);
+ if (ret)
+ return (errno == ECONNREFUSED) ? errno : 0;
+
+ id_priv->local_ece.options = ece.options;
return 0;
}
-
int rdma_create_qp_ex(struct rdma_cm_id *id,
struct ibv_qp_init_attr_ex *attr)
{
@@ -2070,8 +2080,8 @@ static int ucma_query_req_info(struct rdma_cm_id *id)
return 0;
}
-static int ucma_process_conn_req(struct cma_event *evt,
- uint32_t handle)
+static int ucma_process_conn_req(struct cma_event *evt, uint32_t handle,
+ struct ucma_abi_ece *ece)
{
struct cma_id_private *id_priv;
int ret;
@@ -2091,6 +2101,8 @@ static int ucma_process_conn_req(struct cma_event *evt,
ucma_insert_id(id_priv);
id_priv->initiator_depth = evt->event.param.conn.initiator_depth;
id_priv->responder_resources = evt->event.param.conn.responder_resources;
+ id_priv->remote_ece.vendor_id = ece->vendor_id;
+ id_priv->remote_ece.options = ece->attr_mod;
if (evt->id_priv->sync) {
ret = rdma_migrate_id(&id_priv->id, NULL);
@@ -2139,6 +2151,26 @@ err:
return ret;
}
+static int ucma_process_conn_resp_ece(struct cma_id_private *id_priv,
+ struct ucma_abi_ece *ece)
+{
+ struct ibv_ece ibv_ece = { .vendor_id = ece->vendor_id,
+ .options = ece->attr_mod };
+ int ret;
+
+ ret = ibv_set_ece(id_priv->id.qp, &ibv_ece);
+ if (ret && errno == EOPNOTSUPP)
+ goto out;
+
+ if (ret && errno == ECONNREFUSED)
+ /* libibverbs provider asked to reject connection */
+ return -1;
+
+ id_priv->local_ece.options = ibv_ece.options;
+out:
+ return ucma_process_conn_resp(id_priv);
+}
+
static int ucma_process_join(struct cma_event *evt)
{
evt->mc->mgid = evt->event.param.ud.ah_attr.grh.dgid;
@@ -2275,7 +2307,7 @@ retry:
else
ucma_copy_conn_event(evt, &resp.param.conn);
- ret = ucma_process_conn_req(evt, resp.id);
+ ret = ucma_process_conn_req(evt, resp.id, &resp.ece);
if (ret)
goto retry;
break;
@@ -2284,8 +2316,8 @@ retry:
if (!evt->id_priv->id.qp) {
evt->event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
} else {
- evt->event.status =
- ucma_process_conn_resp(evt->id_priv);
+ evt->event.status = ucma_process_conn_resp_ece(
+ evt->id_priv, &resp.ece);
if (!evt->event.status)
evt->event.event = RDMA_CM_EVENT_ESTABLISHED;
else {
@@ -2635,7 +2667,7 @@ int rdma_get_remote_ece(struct rdma_cm_id *id, struct ibv_ece *ece)
{
struct cma_id_private *id_priv;
- if (id->qp || !ece)
+ if (!id || id->qp || !ece)
return ERR(EINVAL);
id_priv = container_of(id, struct cma_id_private, id);
diff --git a/librdmacm/rdma_cma_abi.h b/librdmacm/rdma_cma_abi.h
index 911863cc..f24417dd 100644
--- a/librdmacm/rdma_cma_abi.h
+++ b/librdmacm/rdma_cma_abi.h
@@ -334,6 +334,7 @@ struct ucma_abi_event_resp {
struct ucma_abi_conn_param conn;
struct ucma_abi_ud_param ud;
} param;
+ struct ucma_abi_ece ece;
};
struct ucma_abi_set_option {
--
2.25.2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH rdma-core 10/12] librdmacm: Document ECE API
2020-04-20 14:06 [PATCH rdma-core 00/12] Add Enhanced Connection Established (ECE) APIs Leon Romanovsky
` (8 preceding siblings ...)
2020-04-20 14:06 ` [PATCH rdma-core 09/12] librdmacm: Implement ECE handshake logic Leon Romanovsky
@ 2020-04-20 14:06 ` Leon Romanovsky
2020-04-20 14:06 ` [PATCH rdma-core 11/12] pyverbs: Add support for ECE Leon Romanovsky
2020-04-20 14:06 ` [PATCH rdma-core 12/12] tests: Add test for rdmacm ECE mechanism Leon Romanovsky
11 siblings, 0 replies; 13+ messages in thread
From: Leon Romanovsky @ 2020-04-20 14:06 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe; +Cc: Leon Romanovsky, linux-rdma
From: Leon Romanovsky <leonro@mellanox.com>
Add manual pages for librdmacm part of ECE.
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
librdmacm/man/CMakeLists.txt | 2 +
librdmacm/man/rdma_cm.7 | 14 +++++-
librdmacm/man/rdma_get_remote_ece.3.md | 61 +++++++++++++++++++++++++
librdmacm/man/rdma_set_local_ece.3.md | 62 ++++++++++++++++++++++++++
4 files changed, 138 insertions(+), 1 deletion(-)
create mode 100644 librdmacm/man/rdma_get_remote_ece.3.md
create mode 100644 librdmacm/man/rdma_set_local_ece.3.md
diff --git a/librdmacm/man/CMakeLists.txt b/librdmacm/man/CMakeLists.txt
index 2d1efbff..6159c3e3 100644
--- a/librdmacm/man/CMakeLists.txt
+++ b/librdmacm/man/CMakeLists.txt
@@ -29,6 +29,7 @@ rdma_man_pages(
rdma_get_local_addr.3
rdma_get_peer_addr.3
rdma_get_recv_comp.3
+ rdma_get_remote_ece.3.md
rdma_get_request.3
rdma_get_send_comp.3
rdma_get_src_port.3
@@ -56,6 +57,7 @@ rdma_man_pages(
rdma_resolve_addr.3
rdma_resolve_route.3
rdma_server.1
+ rdma_set_local_ece.3.md
rdma_set_option.3
rdma_xclient.1
rdma_xserver.1
diff --git a/librdmacm/man/rdma_cm.7 b/librdmacm/man/rdma_cm.7
index 8e5ad99e..122c96f0 100644
--- a/librdmacm/man/rdma_cm.7
+++ b/librdmacm/man/rdma_cm.7
@@ -26,6 +26,10 @@ parameter in specific calls. If an event channel is provided, an rdma_cm identi
will report its event data (results of connecting, for example), on that channel.
If a channel is not provided, then all rdma_cm operations for the selected
rdma_cm identifier will block until they complete.
+.P
+The RDMA CM gives an option to different libibverbs providers to advertise and
+use various specific to that provider QP configuration options. This functionality
+is called ECE (enhanced connection establishment).
.SH "RDMA VERBS"
The rdma_cm supports the full range of verbs available through the libibverbs
library and interfaces. However, it also provides wrapper functions for some
@@ -111,6 +115,8 @@ destroy the QP
release the rdma_cm_id
.IP rdma_destroy_event_channel
release the event channel
+.IP rdma_set_local_ece
+set desired ECE options
.P
An almost identical process is used to setup unreliable datagram (UD)
communication between nodes. No actual connection is formed between QPs
@@ -157,6 +163,10 @@ release the connected rdma_cm_id
release the listening rdma_cm_id
.IP rdma_destroy_event_channel
release the event channel
+.IP rdma_get_remote_ece
+get ECe options sent by the client
+.IP rdma_set_local_ece
+set desired ECE options
.SH "RETURN CODES"
.IP "= 0"
success
@@ -198,6 +208,7 @@ rdma_get_dst_port(3),
rdma_get_local_addr(3),
rdma_get_peer_addr(3),
rdma_get_recv_comp(3),
+rdma_get_remote_ece(3),
rdma_get_request(3),
rdma_get_send_comp(3),
rdma_get_src_port(3),
@@ -221,7 +232,8 @@ rdma_reg_write(3),
rdma_reject(3),
rdma_resolve_addr(3),
rdma_resolve_route(3),
-rdma_set_option(3)
+rdma_get_remote_ece(3),
+rdma_set_option(3),
mckey(1),
rdma_client(1),
rdma_server(1),
diff --git a/librdmacm/man/rdma_get_remote_ece.3.md b/librdmacm/man/rdma_get_remote_ece.3.md
new file mode 100644
index 00000000..1db1f8ee
--- /dev/null
+++ b/librdmacm/man/rdma_get_remote_ece.3.md
@@ -0,0 +1,61 @@
+---
+date: 2020-02-02
+footer: librdmacm
+header: "Librdmacm Programmer's Manual"
+layout: page
+license: 'Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md'
+section: 3
+title: RDMA_GET_REMOTE_ECE
+---
+
+# NAME
+
+rdma_get_remote_ece - Get remote ECE paraemters as received from the peer.
+
+# SYNOPSIS
+
+```c
+#include <rdma/rdma_cma.h>
+
+int rdma_get_remote_ece(struct rdma_cm_id *id, struct ibv_ece *ece);
+```
+# DESCRIPTION
+
+**rdma_get_remote_ece()** get ECE parameters as were received from the communication peer.
+
+This function is suppose to be used by the users of external QPs. The call needs
+to be performed before replying to the peer and needed to allow for the passive
+side to know ECE options of other side.
+
+Being used by external QP and RDMA_CM doesn't manage that QP, the peer needs
+to call to libibverbs API by itself.
+
+Usual flow for the passive side will be:
+
+ * ibv_create_qp() <- create data QP.
+ * ece = rdma_get_remote_ece() <- get ECE options from remote peer
+ * ibv_set_ece(ece) <- set local ECE options with data received from the peer.
+ * ibv_modify_qp() <- enable data QP.
+ * rdma_set_local_ece(ece) <- set desired ECE options after respective
+ libibverbs provider masked unsupported options.
+ * rdma_accept()/rdma_establish()/rdma_reject_ece()
+
+# ARGUMENTS
+
+*id
+: RDMA communication identifier.
+
+*ece
+: ECE struct to be filled.
+
+# RETURN VALUE
+
+**rdma_get_remote_ece()** returns 0 on success, or -1 on error. If an error occurs, errno will be set to indicate the failure reason.
+
+# SEE ALSO
+
+**rdma_cm**(7), rdma_set_local_ece(3)
+
+# AUTHOR
+
+Leon Romanovsky <leonro@mellanox.com>
diff --git a/librdmacm/man/rdma_set_local_ece.3.md b/librdmacm/man/rdma_set_local_ece.3.md
new file mode 100644
index 00000000..253e60df
--- /dev/null
+++ b/librdmacm/man/rdma_set_local_ece.3.md
@@ -0,0 +1,62 @@
+---
+date: 2020-02-02
+footer: librdmacm
+header: "Librdmacm Programmer's Manual"
+layout: page
+license: 'Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md'
+section: 3
+title: RDMA_SET_LOCAL_ECE
+---
+
+# NAME
+
+rdma_set_local_ece - Set local ECE paraemters to be used for REQ/REP communication.
+
+# SYNOPSIS
+
+```c
+#include <rdma/rdma_cma.h>
+
+int rdma_set_local_ece(struct rdma_cm_id *id, struct ibv_ece *ece);
+```
+# DESCRIPTION
+
+**rdma_set_local_ece()** set local ECE parameters.
+
+This function is suppose to be used by the users of external QPs. The call needs
+to be performed before replying to the peer and needed to configure RDMA_CM with
+desired ECE options.
+
+Being used by external QP and RDMA_CM doesn't manage that QP, the peer needs
+to call to libibverbs API by itself.
+
+Usual flow for the passive side will be:
+
+ * ibv_create_qp() <- create data QP.
+ * ece = ibv_get_ece() <- get ECE from libibvers provider.
+ * rdma_set_local_ece(ece) <- set desired ECE options.
+ * rdma_connect() <- send connection request
+ * ece = rdma_get_remote_ece() <- get ECE options from remote peer
+ * ibv_set_ece(ece) <- set local ECE options with data received from the peer.
+ * ibv_modify_qp() <- enable data QP.
+ * rdma_accept()/rdma_establish()/rdma_reject_ece()
+
+# ARGUMENTS
+
+*id*
+: RDMA communication identifier.
+
+*ece
+: ECE parameters.
+
+# RETURN VALUE
+
+**rdma_set_local_ece()** returns 0 on success, or -1 on error. If an error occurs, errno will be set to indicate the failure reason.
+
+# SEE ALSO
+
+**rdma_cm**(7), rdma_get_remote_ece(3)
+
+# AUTHOR
+
+Leon Romanovsky <leonro@mellanox.com>
--
2.25.2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH rdma-core 11/12] pyverbs: Add support for ECE
2020-04-20 14:06 [PATCH rdma-core 00/12] Add Enhanced Connection Established (ECE) APIs Leon Romanovsky
` (9 preceding siblings ...)
2020-04-20 14:06 ` [PATCH rdma-core 10/12] librdmacm: Document ECE API Leon Romanovsky
@ 2020-04-20 14:06 ` Leon Romanovsky
2020-04-20 14:06 ` [PATCH rdma-core 12/12] tests: Add test for rdmacm ECE mechanism Leon Romanovsky
11 siblings, 0 replies; 13+ messages in thread
From: Leon Romanovsky @ 2020-04-20 14:06 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe; +Cc: Ido Kalir, Leon Romanovsky, linux-rdma
From: Ido Kalir <idok@mellanox.com>
ECE (enhanced connection establishment) is a mechanism that gives an
option to different libibverbs providers to advertise and use various
provider-specific QP configuration options.
Add those verbs:
ibv_query_ece - get QPs ece.
ibv_set_ece - set the QPs ece.
rdma_set_local_ece - set the local CMs ece.
rdma_get_remote_ece - get the remote CM ece from the connection request.
Signed-off-by: Maxim Chicherin <maximc@mellanox.com>
Signed-off-by: Ido Kalir <idok@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
pyverbs/cmid.pyx | 23 ++++++++++++++++++++-
pyverbs/libibverbs.pxd | 7 +++++++
pyverbs/librdmacm.pxd | 2 ++
pyverbs/qp.pxd | 3 +++
pyverbs/qp.pyx | 45 ++++++++++++++++++++++++++++++++++++++++--
5 files changed, 77 insertions(+), 3 deletions(-)
diff --git a/pyverbs/cmid.pyx b/pyverbs/cmid.pyx
index 66d73268..2505ec70 100755
--- a/pyverbs/cmid.pyx
+++ b/pyverbs/cmid.pyx
@@ -1,7 +1,7 @@
from libc.string cimport memset
from pyverbs.pyverbs_error import PyverbsUserError
-from pyverbs.qp cimport QPInitAttr, QPAttr
+from pyverbs.qp cimport QPInitAttr, QPAttr, ECE
from pyverbs.base import PyverbsRDMAErrno
cimport pyverbs.libibverbs_enums as e
cimport pyverbs.librdmacm_enums as ce
@@ -424,6 +424,27 @@ cdef class CMID(PyverbsCM):
if ret != 0:
raise PyverbsRDMAErrno('Failed to Complete an active connection request')
+ def set_local_ece(self, ECE ece):
+ """
+ Set local ECE paraemters to be used for REQ/REP communication.
+ :param ece: ECE object with the requested configuration
+ :return: None
+ """
+ rc = cm.rdma_set_local_ece(self.id, &ece.ece)
+ if rc != 0:
+ raise PyverbsRDMAErrno('Failed to set local ECE')
+
+ def get_remote_ece(self):
+ """
+ Get ECE parameters as were received from the communication peer.
+ :return: ECE object with the ece configuration
+ """
+ ece = ECE()
+ rc = cm.rdma_get_remote_ece(self.id, &ece.ece)
+ if rc != 0:
+ raise PyverbsRDMAErrno('Failed to get remote ECE')
+ return ece
+
def create_qp(self, QPInitAttr qp_init not None):
"""
Create a QP, which is associated with CMID.
diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd
index 6ffa303c..52f51f07 100755
--- a/pyverbs/libibverbs.pxd
+++ b/pyverbs/libibverbs.pxd
@@ -475,6 +475,11 @@ cdef extern from 'infiniband/verbs.h':
uint64_t wr_id
unsigned int wr_flags
+ cdef struct ibv_ece:
+ uint32_t vendor_id
+ uint32_t options
+ uint32_t comp_mask
+
ibv_device **ibv_get_device_list(int *n)
void ibv_free_device_list(ibv_device **list)
ibv_context *ibv_open_device(ibv_device *device)
@@ -599,3 +604,5 @@ cdef extern from 'infiniband/verbs.h':
cdef extern from 'infiniband/driver.h':
int ibv_query_gid_type(ibv_context *context, uint8_t port_num,
unsigned int index, ibv_gid_type *type)
+ int ibv_set_ece(ibv_qp *qp, ibv_ece *ece)
+ int ibv_query_ece(ibv_qp *qp, ibv_ece *ece)
diff --git a/pyverbs/librdmacm.pxd b/pyverbs/librdmacm.pxd
index 03c0cddc..ff579205 100755
--- a/pyverbs/librdmacm.pxd
+++ b/pyverbs/librdmacm.pxd
@@ -97,6 +97,8 @@ cdef extern from '<rdma/rdma_cma.h>':
int rdma_create_id(rdma_event_channel *channel, rdma_cm_id **id,
void *context, rdma_port_space ps)
int rdma_destroy_id(rdma_cm_id *id)
+ int rdma_get_remote_ece(rdma_cm_id *id, ibv_ece *ece)
+ int rdma_set_local_ece(rdma_cm_id *id, ibv_ece *ece)
int rdma_get_request(rdma_cm_id *listen, rdma_cm_id **id)
int rdma_bind_addr(rdma_cm_id *id, sockaddr *addr)
int rdma_resolve_addr(rdma_cm_id *id, sockaddr *src_addr,
diff --git a/pyverbs/qp.pxd b/pyverbs/qp.pxd
index 209a2438..1294560a 100644
--- a/pyverbs/qp.pxd
+++ b/pyverbs/qp.pxd
@@ -43,3 +43,6 @@ cdef class DataBuffer(PyverbsCM):
cdef class QPEx(QP):
cdef v.ibv_qp_ex *qp_ex
+
+cdef class ECE(PyverbsCM):
+ cdef v.ibv_ece ece
diff --git a/pyverbs/qp.pyx b/pyverbs/qp.pyx
index 95ef554c..fd851443 100755
--- a/pyverbs/qp.pyx
+++ b/pyverbs/qp.pyx
@@ -4,9 +4,8 @@
from libc.stdlib cimport malloc, free
from libc.string cimport memcpy
+from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError, PyverbsRDMAError
from pyverbs.utils import gid_str, qp_type_to_str, qp_state_to_str, mtu_to_str
-from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError, \
- PyverbsRDMAError
from pyverbs.utils import access_flags_to_str, mig_state_to_str
from pyverbs.base import PyverbsRDMAErrno
from pyverbs.wr cimport RecvWR, SendWR, SGE
@@ -871,6 +870,27 @@ cdef class QPAttr(PyverbsObject):
print_format.format('Rate limit', self.attr.rate_limit)
+cdef class ECE(PyverbsCM):
+ def __init__(self, vendor_id=0, options=0, comp_mask=0):
+ """
+ :param vendor_id: Unique identifier of the provider vendor.
+ :param options: Provider specific attributes which are supported or
+ needed to be enabled by ECE users.
+ :param comp_mask: A bitmask specifying which ECE options should be
+ valid.
+ """
+ super().__init__()
+ self.ece.vendor_id = vendor_id
+ self.ece.options = options
+ self.ece.comp_mask = comp_mask
+
+ def __str__(self):
+ print_format = '{:22}: {:<20}\n'
+ print_format.format('Vendor ID', self.ece.vendor_id) +\
+ print_format.format('Options', self.ece.options) +\
+ print_format.format('Comp Mask', self.ece.comp_mask)
+
+
cdef class QP(PyverbsCM):
def __init__(self, object creator not None, object init_attr not None,
QPAttr qp_attr=None):
@@ -1133,6 +1153,27 @@ cdef class QP(PyverbsCM):
memcpy(&bad_wr.send_wr, my_bad_wr, sizeof(bad_wr.send_wr))
raise PyverbsRDMAError('Failed to post send', rc)
+ def set_ece(self, ECE ece):
+ """
+ Set ECE options and use them for QP configuration stage
+ :param ece: The requested ECE values.
+ :return: None
+ """
+ rc = v.ibv_set_ece(self.qp, &ece.ece)
+ if rc != 0:
+ raise PyverbsRDMAError('Failed to set ECE', rc)
+
+ def query_ece(self):
+ """
+ Query QPs ECE options
+ :return: ECE object with this QP ece configuration.
+ """
+ ece = ECE()
+ rc = v.ibv_query_ece(self.qp, &ece.ece)
+ if rc != 0:
+ raise PyverbsRDMAError('Failed to query ECE', rc)
+ return ece
+
@property
def qp_type(self):
return self.qp.qp_type
--
2.25.2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH rdma-core 12/12] tests: Add test for rdmacm ECE mechanism
2020-04-20 14:06 [PATCH rdma-core 00/12] Add Enhanced Connection Established (ECE) APIs Leon Romanovsky
` (10 preceding siblings ...)
2020-04-20 14:06 ` [PATCH rdma-core 11/12] pyverbs: Add support for ECE Leon Romanovsky
@ 2020-04-20 14:06 ` Leon Romanovsky
11 siblings, 0 replies; 13+ messages in thread
From: Leon Romanovsky @ 2020-04-20 14:06 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe; +Cc: Ido Kalir, Leon Romanovsky, linux-rdma
From: Ido Kalir <idok@mellanox.com>
Add test case that use the ECE mechanism in RDMA CM connection
establishment.
Signed-off-by: Ido Kalir <idok@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
tests/rdmacm_utils.py | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
diff --git a/tests/rdmacm_utils.py b/tests/rdmacm_utils.py
index c71bab18..d8d22f65 100755
--- a/tests/rdmacm_utils.py
+++ b/tests/rdmacm_utils.py
@@ -4,10 +4,12 @@
Provide some useful helper function for pyverbs rdmacm' tests.
"""
from tests.utils import validate, poll_cq, get_send_element, get_recv_wr
-from pyverbs.pyverbs_error import PyverbsError
+from pyverbs.pyverbs_error import PyverbsError, PyverbsRDMAError
from tests.base import CMResources
from pyverbs.cmid import CMEvent
import pyverbs.cm_enums as ce
+import unittest
+import errno
import os
events_dict = {ce.RDMA_CM_EVENT_ADDR_ERROR: 'Resolve Address Error',
@@ -117,6 +119,13 @@ def event_handler(agr_obj):
agr_obj.create_qp()
param = agr_obj.create_conn_param()
if agr_obj.with_ext_qp:
+ try:
+ ece = agr_obj.qp.query_ece()
+ agr_obj.cmid.set_local_ece(ece)
+ except PyverbsRDMAError as ex:
+ if ex.error_code == errno.EOPNOTSUPP:
+ pass
+ raise ex
param.qpn = agr_obj.qp.qp_num
agr_obj.cmid.connect(param)
elif cm_event.event_type == ce.RDMA_CM_EVENT_CONNECT_REQUEST:
@@ -124,6 +133,13 @@ def event_handler(agr_obj):
param = agr_obj.create_conn_param()
agr_obj.create_qp()
if agr_obj.with_ext_qp:
+ try:
+ ece = agr_obj.child_id.get_remote_ece()
+ agr_obj.qp.set_ece(ece)
+ except PyverbsRDMAError as ex:
+ if ex.error_code == errno.EOPNOTSUPP:
+ pass
+ raise ex
agr_obj.modify_ext_qp_to_rts()
param.qpn = agr_obj.qp.qp_num
agr_obj.child_id.accept(param)
--
2.25.2
^ permalink raw reply related [flat|nested] 13+ messages in thread