[PATCH] Add new verb: uv_query_port_max_datagram()

[Jeff Squyres <jsquyres-FYB4Gu1CFyUAvxtiuMwx3w@public.gmane.org>]

Per lengthy discussion on the linux-rdma list, add a new verb to get
max datagram size (in bytes) since the methods for retrieving MTU
values are limited to a finite enum set, and are difficult to change
for backwards compatibility reasons.

Also add corresponding command: uv_cmd_query_port_max_datagram().
Since this is a new verb, there was no need to add a _V2 enum for the
command macro, which required adding a UB_INIT_CMD_RESP() macro.

Bumped the ABI version to 7 (the new verb will return -ENOSYS if
abi_verb is < 7).

Note that the name for this verb was chosen with the following
rationale:

* After discussion with Roland, use the prefix "uv" instead of "ibv",
  since this verb is generic to both Ethernet, InfiniBand, and
  whatever other transports are underneath.
* "query" was used (vs. "get") because it invokes a command (vs. a
  struct lookup)

If the community likes this approach, I'll send the corresponding
kernel patch.

Signed-off-by: Jeff Squyres <jsquyres-FYB4Gu1CFyUAvxtiuMwx3w@public.gmane.org>
---
 Makefile.am                      |  3 +-
 examples/devinfo.c               |  7 +++++
 include/infiniband/driver.h      |  4 +++
 include/infiniband/kern-abi.h    | 19 +++++++++++--
 include/infiniband/verbs.h       |  7 +++++
 man/uv_query_port_max_datagram.3 | 60 ++++++++++++++++++++++++++++++++++++++++
 src/cmd.c                        | 25 +++++++++++++++++
 src/ibverbs.h                    |  8 ++++++
 src/libibverbs.map               |  2 ++
 src/verbs.c                      | 10 +++++++
 10 files changed, 142 insertions(+), 3 deletions(-)
 create mode 100644 man/uv_query_port_max_datagram.3

diff --git a/Makefile.am b/Makefile.am
index 40e83be..51fe5d5 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -54,7 +54,8 @@ man_MANS = man/ibv_asyncwatch.1 man/ibv_devices.1 man/ibv_devinfo.1	\
     man/ibv_post_srq_recv.3 man/ibv_query_device.3 man/ibv_query_gid.3	\
     man/ibv_query_pkey.3 man/ibv_query_port.3 man/ibv_query_qp.3	\
     man/ibv_query_srq.3 man/ibv_rate_to_mult.3 man/ibv_reg_mr.3		\
-    man/ibv_req_notify_cq.3 man/ibv_resize_cq.3 man/ibv_rate_to_mbps.3
+    man/ibv_req_notify_cq.3 man/ibv_resize_cq.3 man/ibv_rate_to_mbps.3	\
+    man/uv_query_port_max_datagram.3
 
 DEBIAN = debian/changelog debian/compat debian/control debian/copyright \
     debian/ibverbs-utils.install debian/libibverbs1.install \
diff --git a/examples/devinfo.c b/examples/devinfo.c
index ff078e4..f51620b 100644
--- a/examples/devinfo.c
+++ b/examples/devinfo.c
@@ -209,6 +209,7 @@ static int print_hca_cap(struct ibv_device *ib_dev, uint8_t ib_port)
 	struct ibv_port_attr port_attr;
 	int rc = 0;
 	uint8_t port;
+	uint32_t max_datagram;
 	char buf[256];
 
 	ctx = ibv_open_device(ib_dev);
@@ -298,6 +299,11 @@ static int print_hca_cap(struct ibv_device *ib_dev, uint8_t ib_port)
 			fprintf(stderr, "Failed to query port %u props\n", port);
 			goto cleanup;
 		}
+		rc = uv_query_port_max_datagram(ctx, port, &max_datagram);
+		if (rc) {
+			fprintf(stderr, "Failed to query port %u max datagram size\n", port);
+			goto cleanup;
+		}
 		printf("\t\tport:\t%d\n", port);
 		printf("\t\t\tstate:\t\t\t%s (%d)\n",
 		       port_state_str(port_attr.state), port_attr.state);
@@ -305,6 +311,7 @@ static int print_hca_cap(struct ibv_device *ib_dev, uint8_t ib_port)
 		       mtu_str(port_attr.max_mtu), port_attr.max_mtu);
 		printf("\t\t\tactive_mtu:\t\t%s (%d)\n",
 		       mtu_str(port_attr.active_mtu), port_attr.active_mtu);
+		printf("\t\t\tmax_datagram_size:\t%u\n", max_datagram);
 		printf("\t\t\tsm_lid:\t\t\t%d\n", port_attr.sm_lid);
 		printf("\t\t\tport_lid:\t\t%d\n", port_attr.lid);
 		printf("\t\t\tport_lmc:\t\t0x%02x\n", port_attr.lmc);
diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index 9a81416..6e1236c 100644
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -67,6 +67,10 @@ int ibv_cmd_query_device(struct ibv_context *context,
 int ibv_cmd_query_port(struct ibv_context *context, uint8_t port_num,
 		       struct ibv_port_attr *port_attr,
 		       struct ibv_query_port *cmd, size_t cmd_size);
+int uv_cmd_query_port_max_datagram(struct ibv_context *context, uint8_t port_num,
+				   uint32_t *max_datagram,
+				   struct uv_query_port_max_datagram *cmd,
+				   size_t cmd_size);
 int ibv_cmd_query_gid(struct ibv_context *context, uint8_t port_num,
 		      int index, union ibv_gid *gid);
 int ibv_cmd_query_pkey(struct ibv_context *context, uint8_t port_num,
diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index 619ea7e..951108e 100644
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -46,7 +46,7 @@
  * The minimum and maximum kernel ABI that we can handle.
  */
 #define IB_USER_VERBS_MIN_ABI_VERSION	1
-#define IB_USER_VERBS_MAX_ABI_VERSION	6
+#define IB_USER_VERBS_MAX_ABI_VERSION	7
 
 enum {
 	IB_USER_VERBS_CMD_GET_CONTEXT,
@@ -85,7 +85,8 @@ enum {
 	IB_USER_VERBS_CMD_MODIFY_SRQ,
 	IB_USER_VERBS_CMD_QUERY_SRQ,
 	IB_USER_VERBS_CMD_DESTROY_SRQ,
-	IB_USER_VERBS_CMD_POST_SRQ_RECV
+	IB_USER_VERBS_CMD_POST_SRQ_RECV,
+	USER_VERBS_CMD_QUERY_PORT_MAX_DATAGRAM
 };
 
 /*
@@ -227,6 +228,20 @@ struct ibv_query_port_resp {
 	__u8  reserved[2];
 };
 
+struct uv_query_port_max_datagram {
+	__u32 command;
+	__u16 in_words;
+	__u16 out_words;
+	__u64 response;
+	__u8  port_num;
+	__u8  reserved[7];
+	__u64 driver_data[0];
+};
+
+struct uv_query_port_max_datagram_resp {
+	__u32 max_datagram;
+};
+
 struct ibv_alloc_pd {
 	__u32 command;
 	__u16 in_words;
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 4b1ab57..a1d6c3d 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -712,6 +712,7 @@ struct ibv_context_ops {
 	int			(*detach_mcast)(struct ibv_qp *qp, const union ibv_gid *gid,
 						uint16_t lid);
 	void			(*async_event)(struct ibv_async_event *event);
+	int			(*query_port_max_datagram)(struct ibv_context *context, uint8_t port_num, uint32_t *max_datagram);
 };
 
 struct ibv_context {
@@ -813,6 +814,12 @@ static inline int ___ibv_query_port(struct ibv_context *context,
 	___ibv_query_port(context, port_num, port_attr)
 
 /**
+ * uv_query_port_max_datagram - Get port's max datagram size
+ */
+int uv_query_port_max_datagram(struct ibv_context *context,
+			uint8_t port_num, uint32_t *max_datagram);
+
+/**
  * ibv_query_gid - Get a GID table entry
  */
 int ibv_query_gid(struct ibv_context *context, uint8_t port_num,
diff --git a/man/uv_query_port_max_datagram.3 b/man/uv_query_port_max_datagram.3
new file mode 100644
index 0000000..a258d54
--- /dev/null
+++ b/man/uv_query_port_max_datagram.3
@@ -0,0 +1,60 @@
+.\" -*- nroff -*-
+.\"
+.TH UV_QUERY_PORT_MAX_DATAGRAM 3 2013-08-12 libibverbs "Libibverbs Programmer's Manual"
+.SH "NAME"
+uv_query_port_max_datagram \- query an RDMA port's max datagram size
+.SH "SYNOPSIS"
+.nf
+.B #include <infiniband/verbs.h>
+.sp
+.BI "int uv_query_port_max_datagram(struct ibv_context " "*context" ","
+.BI "                                    uint8_t " "port_num" "," 
+.BI "                                    uin32_t " "*max_datagram" ");"
+.fi
+.SH "DESCRIPTION"
+.B uv_query_port_max_datagram()
+returns the max datagram size of port
+.I port_num
+for device context
+.I context
+through the pointer
+.I port_attr\fR.
+.fi
+.PP
+This function was added primarily to support non-InfiniBand transports
+that can have MTU values outside the small set of enums that can be
+returned by
+.BR ibv_query_port() .
+In order to be backwards compatible, 
+.B ibv_query_port() 
+and its associated data structures were left unchanged, and the
+.B uv_query_port_max_datagram()
+verb was created specifically to allow returning arbitrary MTU /
+datagram sizes.
+.PP
+This verb should be used to find the active MTU size (instead of the
+values returned from
+.BR ibv_query_port() )
+for all devices that support it.
+.PP
+The "uv" prefix refers to "user verbs" because the "ibv" prefix would
+imply that the function is specific to InfiniBand devices (which would
+be ironic, since this verb was created to return arbitrary MTU / max
+datagram sizes, but InfiniBand devices are restricted to a small set
+of MTU values).
+.fi
+.SH "RETURN VALUE"
+.B uv_query_port_max_datagram()
+returns 0 upon success, and the max datagram size (in bytes) is returned
+in
+.IR max_datagram .
+Otherwise, the negative value of errno is returned on failure (which
+indicates the failure reason), and the value of
+.I max_datagram
+is undefined.  -ENOSYS will be returned if the kernel verbs call does
+not support this verb.
+.SH "SEE ALSO"
+.BR ibv_query_port (3),
+.SH "AUTHORS"
+.TP
+Jeff Squyres <jsquyres-FYB4Gu1CFyUAvxtiuMwx3w@public.gmane.org>
diff --git a/src/cmd.c b/src/cmd.c
index 9789092..fdafbb4 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -203,6 +203,31 @@ int ibv_cmd_query_port(struct ibv_context *context, uint8_t port_num,
 	return 0;
 }
 
+int uv_cmd_query_port_max_datagram(struct ibv_context *context, uint8_t port_num,
+				   uint32_t *max_datagram,
+				   struct uv_query_port_max_datagram *cmd,
+				   size_t cmd_size)
+{
+	struct uv_query_port_max_datagram_resp resp;
+
+	if (abi_ver < 7)
+		return -ENOSYS;
+
+	UV_INIT_CMD_RESP(cmd, cmd_size, QUERY_PORT_MAX_DATAGRAM,
+			&resp, sizeof resp);
+	cmd->port_num = port_num;
+	memset(cmd->reserved, 0, sizeof cmd->reserved);
+
+	if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
+		return errno;
+
+	(void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
+
+	*max_datagram = resp.max_datagram;
+
+	return 0;
+}
+
 int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd,
 		     struct ibv_alloc_pd *cmd, size_t cmd_size,
 		     struct ibv_alloc_pd_resp *resp, size_t resp_size)
diff --git a/src/ibverbs.h b/src/ibverbs.h
index fa6cd41..26cd1d3 100644
--- a/src/ibverbs.h
+++ b/src/ibverbs.h
@@ -102,4 +102,12 @@ HIDDEN int ibverbs_init(struct ibv_device ***list);
 		(cmd)->response  = (uintptr_t) (out);			\
 	} while (0)
 
+#define UV_INIT_CMD_RESP(cmd, size, opcode, out, outsize)		\
+	do {								\
+		(cmd)->command = USER_VERBS_CMD_##opcode;		\
+		(cmd)->in_words  = (size) / 4;				\
+		(cmd)->out_words = (outsize) / 4;			\
+		(cmd)->response  = (uintptr_t) (out);			\
+	} while (0)
+
 #endif /* IB_VERBS_H */
diff --git a/src/libibverbs.map b/src/libibverbs.map
index 7e722f4..e855c25 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -99,4 +99,6 @@ IBVERBS_1.1 {
 
 		ibv_rate_to_mbps;
 		mbps_to_ibv_rate;
+
+		uv_query_port_max_datagram;
 } IBVERBS_1.0;
diff --git a/src/verbs.c b/src/verbs.c
index a6aae70..a86262d 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -138,6 +138,16 @@ int __ibv_query_port(struct ibv_context *context, uint8_t port_num,
 }
 default_symver(__ibv_query_port, ibv_query_port);
 
+int __uv_query_port_max_datagram(struct ibv_context *context, uint8_t port_num,
+				uint32_t *max_datagram)
+{
+	if (context->ops.query_port_max_datagram)
+		return context->ops.query_port_max_datagram(context, port_num,
+							max_datagram);
+	return 0;
+}
+default_symver(__uv_query_port_max_datagram, uv_query_port_max_datagram);
+
 int __ibv_query_gid(struct ibv_context *context, uint8_t port_num,
 		    int index, union ibv_gid *gid)
 {
-- 
1.8.2.1

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html