[PATCH libibverbs V3 3/3] Add ibv_query_port_ex support

Or Gerlitz
2014-05-08T06:51:23Z

From: Matan Barak <matanb-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>

This patch adds extended support for ibv_query_port.

This allows to request fields that aren't availible by the current
ibv_query_port API and avoid fetching from vendor library fields that
the user doesn't need, which gives more room for optimizations.

Signed-off-by: Matan Barak <matanb-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
Signed-off-by: Or Gerlitz <ogerlitz-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
---
include/infiniband/verbs.h | 105 +++++++++++++++++++++++++++++++++++++++++++-
src/device.c | 2 +
src/verbs.c | 7 ++-
3 files changed, 111 insertions(+), 3 deletions(-)

diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 92d6a60..abf0058 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -492,6 +492,69 @@ struct ibv_ah_attr_ex {
uint16_t vid;
};

+enum {
+ IBV_QUERY_PORT_EX_STATE = 1 << 0,
+ IBV_QUERY_PORT_EX_MAX_MTU = 1 << 1,
+ IBV_QUERY_PORT_EX_ACTIVE_MTU = 1 << 2,
+ IBV_QUERY_PORT_EX_GID_TBL_LEN = 1 << 3,
+ IBV_QUERY_PORT_EX_CAP_FLAGS = 1 << 4,
+ IBV_QUERY_PORT_EX_MAX_MSG_SZ = 1 << 5,
+ IBV_QUERY_PORT_EX_BAD_PKEY_CNTR = 1 << 6,
+ IBV_QUERY_PORT_EX_QKEY_VIOL_CNTR = 1 << 7,
+ IBV_QUERY_PORT_EX_PKEY_TBL_LEN = 1 << 8,
+ IBV_QUERY_PORT_EX_LID = 1 << 9,
+ IBV_QUERY_PORT_EX_SM_LID = 1 << 10,
+ IBV_QUERY_PORT_EX_LMC = 1 << 11,
+ IBV_QUERY_PORT_EX_MAX_VL_NUM = 1 << 12,
+ IBV_QUERY_PORT_EX_SM_SL = 1 << 13,
+ IBV_QUERY_PORT_EX_SUBNET_TIMEOUT = 1 << 14,
+ IBV_QUERY_PORT_EX_INIT_TYPE_REPLY = 1 << 15,
+ IBV_QUERY_PORT_EX_ACTIVE_WIDTH = 1 << 16,
+ IBV_QUERY_PORT_EX_ACTIVE_SPEED = 1 << 17,
+ IBV_QUERY_PORT_EX_PHYS_STATE = 1 << 18,
+ IBV_QUERY_PORT_EX_LINK_LAYER = 1 << 19,
+ /* mask of the fields that exists in the standard query_port_command */
+ IBV_QUERY_PORT_EX_STD_MASK = (1 << 20) - 1,
+ /* mask of all supported fields */
+ IBV_QUERY_PORT_EX_MASK = IBV_QUERY_PORT_EX_STD_MASK,
+};
+
+enum ibv_query_port_ex_attr_mask {
+ IBV_QUERY_PORT_EX_ATTR_MASK1 = 1 << 0,
+ IBV_QUERY_PORT_EX_ATTR_MASKS = (1 << 1) - 1
+};
+
+struct ibv_port_attr_ex {
+ union {
+ struct {
+ enum ibv_port_state state;
+ enum ibv_mtu max_mtu;
+ enum ibv_mtu active_mtu;
+ int gid_tbl_len;
+ uint32_t port_cap_flags;
+ uint32_t max_msg_sz;
+ uint32_t bad_pkey_cntr;
+ uint32_t qkey_viol_cntr;
+ uint16_t pkey_tbl_len;
+ uint16_t lid;
+ uint16_t sm_lid;
+ uint8_t lmc;
+ uint8_t max_vl_num;
+ uint8_t sm_sl;
+ uint8_t subnet_timeout;
+ uint8_t init_type_reply;
+ uint8_t active_width;
+ uint8_t active_speed;
+ uint8_t phys_state;
+ uint8_t link_layer;
+ uint8_t reserved;
+ };
+ struct ibv_port_attr port_attr;
+ };
+ uint32_t comp_mask;
+ uint32_t mask1;
+};
+

enum ibv_srq_attr_mask {
IBV_SRQ_MAX_WR = 1 << 0,
@@ -999,11 +1062,16 @@ enum verbs_context_mask {
VERBS_CONTEXT_CREATE_FLOW = 1 << 3,
VERBS_CONTEXT_DESTROY_FLOW = 1 << 4,
VERBS_CONTEXT_CREATE_AH = 1 << 5,
- VERBS_CONTEXT_RESERVED = 1 << 6
+ VERBS_CONTEXT_QUERY_PORT = 1 << 6,
+ VERBS_CONTEXT_RESERVED = 1 << 7
};

struct verbs_context {
/* "grows up" - new fields go here */
+ int (*drv_query_port_ex)(struct ibv_context *context, uint8_t port_num,
+ struct ibv_port_attr_ex *port_attr);
+ int (*lib_query_port_ex)(struct ibv_context *context, uint8_t port_num,
+ struct ibv_port_attr_ex *port_attr);
struct ibv_ah * (*drv_ibv_create_ah_ex)(struct ibv_pd *pd,
struct ibv_ah_attr_ex *attr);
int (*drv_ibv_destroy_flow) (struct ibv_flow *flow);
@@ -1140,6 +1208,41 @@ static inline int ___ibv_query_port(struct ibv_context *context,
#define ibv_query_port(context, port_num, port_attr) \
___ibv_query_port(context, port_num, port_attr)

+static inline int ibv_query_port_ex(struct ibv_context *context,
+ uint8_t port_num,
+ struct ibv_port_attr_ex *port_attr)
+{
+ struct verbs_context *vctx;
+
+ if (0 == port_attr->comp_mask)
+ return ibv_query_port(context, port_num,
+ &port_attr->port_attr);
+
+ /* Check that only valid flags were given */
+ if ((!port_attr->comp_mask & IBV_QUERY_PORT_EX_ATTR_MASK1) ||
+ (port_attr->comp_mask & ~IBV_QUERY_PORT_EX_ATTR_MASKS) ||
+ (port_attr->mask1 & ~IBV_QUERY_PORT_EX_MASK)) {
+ errno = EINVAL;
+ return -errno;
+ }
+
+ vctx = verbs_get_ctx_op(context, lib_query_port_ex);
+
+ if (!vctx) {
+ /* Fallback to legacy mode */
+ if (port_attr->comp_mask == IBV_QUERY_PORT_EX_ATTR_MASK1 &&
+ !(port_attr->mask1 & ~IBV_QUERY_PORT_EX_STD_MASK))
+ return ibv_query_port(context, port_num,
+ &port_attr->port_attr);
+
+ /* Unsupported field was requested */
+ errno = ENOSYS;
+ return -errno;
+ }
+
+ return vctx->lib_query_port_ex(context, port_num, port_attr);
+}
+
/**
* ibv_query_gid - Get a GID table entry
*/
diff --git a/src/device.c b/src/device.c
index 9642ead..eb3f2cd 100644
--- a/src/device.c
+++ b/src/device.c
@@ -173,6 +173,8 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device)
context_ex->drv_ibv_create_flow;
context_ex->lib_ibv_destroy_flow =
context_ex->drv_ibv_destroy_flow;
+ context_ex->lib_query_port_ex =
+ context_ex->drv_query_port_ex;
}

context->device = device;
diff --git a/src/verbs.c b/src/verbs.c
index e7343a9..f8245b0 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -550,7 +550,7 @@ struct ibv_ah *__ibv_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
int err;
struct ibv_ah *ah = NULL;
#ifndef NRESOLVE_NEIGH
- struct ibv_port_attr port_attr;
+ struct ibv_port_attr_ex port_attr;
int dst_family;
int src_family;
int oif;
@@ -570,7 +570,10 @@ struct ibv_ah *__ibv_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
goto return_ah;
}

- err = ibv_query_port(pd->context, attr->port_num, &port_attr);
+ port_attr.comp_mask = IBV_QUERY_PORT_EX_ATTR_MASK1;
+ port_attr.mask1 = IBV_QUERY_PORT_EX_LINK_LAYER |
+ IBV_QUERY_PORT_EX_CAP_FLAGS;
+ err = ibv_query_port_ex(pd->context, attr->port_num, &port_attr);

if (err) {
fprintf(stderr, PFX "ibv_create_ah failed to query port.\n");
--
1.7.1