* [PATCH v3 iproute2-next 1/8] rdma: include rdma-core <rdma/rdma_cma.h>
2018-03-26 20:59 [PATCH v3 iproute2-next 0/8] cm_id, cq, mr, and pd resource tracking Steve Wise
@ 2018-03-26 20:57 ` Steve Wise
2018-03-27 5:46 ` Leon Romanovsky
2018-03-26 20:57 ` [PATCH v3 iproute2-next 2/8] rdma: Makefile change to find <rdma/rdma_cma.h> Steve Wise
` (6 subsequent siblings)
7 siblings, 1 reply; 11+ messages in thread
From: Steve Wise @ 2018-03-26 20:57 UTC (permalink / raw)
To: dsahern; +Cc: leon, stephen, netdev, linux-rdma
This avoids requiring rdma-core be installed on systems.
Signed-off-by: Steve Wise <swise@opengridcomputing.com>
---
rdma/include/rdma/rdma_cma.h | 728 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 728 insertions(+)
create mode 100644 rdma/include/rdma/rdma_cma.h
diff --git a/rdma/include/rdma/rdma_cma.h b/rdma/include/rdma/rdma_cma.h
new file mode 100644
index 0000000..456aa4e
--- /dev/null
+++ b/rdma/include/rdma/rdma_cma.h
@@ -0,0 +1,728 @@
+/*
+ * Copyright (c) 2005 Voltaire Inc. All rights reserved.
+ * Copyright (c) 2005-2014 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if !defined(_RDMA_CMA_H)
+#define _RDMA_CMA_H
+
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <infiniband/verbs.h>
+#include <infiniband/sa.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Upon receiving a device removal event, users must destroy the associated
+ * RDMA identifier and release all resources allocated with the device.
+ */
+enum rdma_cm_event_type {
+ RDMA_CM_EVENT_ADDR_RESOLVED,
+ RDMA_CM_EVENT_ADDR_ERROR,
+ RDMA_CM_EVENT_ROUTE_RESOLVED,
+ RDMA_CM_EVENT_ROUTE_ERROR,
+ RDMA_CM_EVENT_CONNECT_REQUEST,
+ RDMA_CM_EVENT_CONNECT_RESPONSE,
+ RDMA_CM_EVENT_CONNECT_ERROR,
+ RDMA_CM_EVENT_UNREACHABLE,
+ RDMA_CM_EVENT_REJECTED,
+ RDMA_CM_EVENT_ESTABLISHED,
+ RDMA_CM_EVENT_DISCONNECTED,
+ RDMA_CM_EVENT_DEVICE_REMOVAL,
+ RDMA_CM_EVENT_MULTICAST_JOIN,
+ RDMA_CM_EVENT_MULTICAST_ERROR,
+ RDMA_CM_EVENT_ADDR_CHANGE,
+ RDMA_CM_EVENT_TIMEWAIT_EXIT
+};
+
+enum rdma_port_space {
+ RDMA_PS_IPOIB = 0x0002,
+ RDMA_PS_TCP = 0x0106,
+ RDMA_PS_UDP = 0x0111,
+ RDMA_PS_IB = 0x013F,
+};
+
+#define RDMA_IB_IP_PS_MASK 0xFFFFFFFFFFFF0000ULL
+#define RDMA_IB_IP_PORT_MASK 0x000000000000FFFFULL
+#define RDMA_IB_IP_PS_TCP 0x0000000001060000ULL
+#define RDMA_IB_IP_PS_UDP 0x0000000001110000ULL
+#define RDMA_IB_PS_IB 0x00000000013F0000ULL
+
+/*
+ * Global qkey value for UDP QPs and multicast groups created via the
+ * RDMA CM.
+ */
+#define RDMA_UDP_QKEY 0x01234567
+
+struct rdma_ib_addr {
+ union ibv_gid sgid;
+ union ibv_gid dgid;
+ __be16 pkey;
+};
+
+struct rdma_addr {
+ union {
+ struct sockaddr src_addr;
+ struct sockaddr_in src_sin;
+ struct sockaddr_in6 src_sin6;
+ struct sockaddr_storage src_storage;
+ };
+ union {
+ struct sockaddr dst_addr;
+ struct sockaddr_in dst_sin;
+ struct sockaddr_in6 dst_sin6;
+ struct sockaddr_storage dst_storage;
+ };
+ union {
+ struct rdma_ib_addr ibaddr;
+ } addr;
+};
+
+struct rdma_route {
+ struct rdma_addr addr;
+ struct ibv_sa_path_rec *path_rec;
+ int num_paths;
+};
+
+struct rdma_event_channel {
+ int fd;
+};
+
+struct rdma_cm_id {
+ struct ibv_context *verbs;
+ struct rdma_event_channel *channel;
+ void *context;
+ struct ibv_qp *qp;
+ struct rdma_route route;
+ enum rdma_port_space ps;
+ uint8_t port_num;
+ struct rdma_cm_event *event;
+ struct ibv_comp_channel *send_cq_channel;
+ struct ibv_cq *send_cq;
+ struct ibv_comp_channel *recv_cq_channel;
+ struct ibv_cq *recv_cq;
+ struct ibv_srq *srq;
+ struct ibv_pd *pd;
+ enum ibv_qp_type qp_type;
+};
+
+enum {
+ RDMA_MAX_RESP_RES = 0xFF,
+ RDMA_MAX_INIT_DEPTH = 0xFF
+};
+
+struct rdma_conn_param {
+ const void *private_data;
+ uint8_t private_data_len;
+ uint8_t responder_resources;
+ uint8_t initiator_depth;
+ uint8_t flow_control;
+ uint8_t retry_count; /* ignored when accepting */
+ uint8_t rnr_retry_count;
+ /* Fields below ignored if a QP is created on the rdma_cm_id. */
+ uint8_t srq;
+ uint32_t qp_num;
+};
+
+struct rdma_ud_param {
+ const void *private_data;
+ uint8_t private_data_len;
+ struct ibv_ah_attr ah_attr;
+ uint32_t qp_num;
+ uint32_t qkey;
+};
+
+struct rdma_cm_event {
+ struct rdma_cm_id *id;
+ struct rdma_cm_id *listen_id;
+ enum rdma_cm_event_type event;
+ int status;
+ union {
+ struct rdma_conn_param conn;
+ struct rdma_ud_param ud;
+ } param;
+};
+
+#define RAI_PASSIVE 0x00000001
+#define RAI_NUMERICHOST 0x00000002
+#define RAI_NOROUTE 0x00000004
+#define RAI_FAMILY 0x00000008
+
+struct rdma_addrinfo {
+ int ai_flags;
+ int ai_family;
+ int ai_qp_type;
+ int ai_port_space;
+ socklen_t ai_src_len;
+ socklen_t ai_dst_len;
+ struct sockaddr *ai_src_addr;
+ struct sockaddr *ai_dst_addr;
+ char *ai_src_canonname;
+ char *ai_dst_canonname;
+ size_t ai_route_len;
+ void *ai_route;
+ size_t ai_connect_len;
+ void *ai_connect;
+ struct rdma_addrinfo *ai_next;
+};
+
+/* Multicast join compatibility mask attributes */
+enum rdma_cm_join_mc_attr_mask {
+ RDMA_CM_JOIN_MC_ATTR_ADDRESS = 1 << 0,
+ RDMA_CM_JOIN_MC_ATTR_JOIN_FLAGS = 1 << 1,
+ RDMA_CM_JOIN_MC_ATTR_RESERVED = 1 << 2,
+};
+
+/* Multicast join flags */
+enum rdma_cm_mc_join_flags {
+ RDMA_MC_JOIN_FLAG_FULLMEMBER,
+ RDMA_MC_JOIN_FLAG_SENDONLY_FULLMEMBER,
+ RDMA_MC_JOIN_FLAG_RESERVED,
+};
+
+struct rdma_cm_join_mc_attr_ex {
+ /* Bitwise OR between "rdma_cm_join_mc_attr_mask" enum */
+ uint32_t comp_mask;
+ /* Use a flag from "rdma_cm_mc_join_flags" enum */
+ uint32_t join_flags;
+ /* Multicast address identifying the group to join */
+ struct sockaddr *addr;
+};
+
+/**
+ * rdma_create_event_channel - Open a channel used to report communication events.
+ * Description:
+ * Asynchronous events are reported to users through event channels. Each
+ * event channel maps to a file descriptor.
+ * Notes:
+ * All created event channels must be destroyed by calling
+ * rdma_destroy_event_channel. Users should call rdma_get_cm_event to
+ * retrieve events on an event channel.
+ * See also:
+ * rdma_get_cm_event, rdma_destroy_event_channel
+ */
+struct rdma_event_channel *rdma_create_event_channel(void);
+
+/**
+ * rdma_destroy_event_channel - Close an event communication channel.
+ * @channel: The communication channel to destroy.
+ * Description:
+ * Release all resources associated with an event channel and closes the
+ * associated file descriptor.
+ * Notes:
+ * All rdma_cm_id's associated with the event channel must be destroyed,
+ * and all returned events must be acked before calling this function.
+ * See also:
+ * rdma_create_event_channel, rdma_get_cm_event, rdma_ack_cm_event
+ */
+void rdma_destroy_event_channel(struct rdma_event_channel *channel);
+
+/**
+ * rdma_create_id - Allocate a communication identifier.
+ * @channel: The communication channel that events associated with the
+ * allocated rdma_cm_id will be reported on.
+ * @id: A reference where the allocated communication identifier will be
+ * returned.
+ * @context: User specified context associated with the rdma_cm_id.
+ * @ps: RDMA port space.
+ * Description:
+ * Creates an identifier that is used to track communication information.
+ * Notes:
+ * Rdma_cm_id's are conceptually equivalent to a socket for RDMA
+ * communication. The difference is that RDMA communication requires
+ * explicitly binding to a specified RDMA device before communication
+ * can occur, and most operations are asynchronous in nature. Communication
+ * events on an rdma_cm_id are reported through the associated event
+ * channel. Users must release the rdma_cm_id by calling rdma_destroy_id.
+ * See also:
+ * rdma_create_event_channel, rdma_destroy_id, rdma_get_devices,
+ * rdma_bind_addr, rdma_resolve_addr, rdma_connect, rdma_listen,
+ */
+int rdma_create_id(struct rdma_event_channel *channel,
+ struct rdma_cm_id **id, void *context,
+ enum rdma_port_space ps);
+
+/**
+ * rdma_create_ep - Allocate a communication identifier and qp.
+ * @id: A reference where the allocated communication identifier will be
+ * returned.
+ * @res: Result from rdma_getaddrinfo, which specifies the source and
+ * destination addresses, plus optional routing and connection information.
+ * @pd: Optional protection domain. This parameter is ignored if qp_init_attr
+ * is NULL.
+ * @qp_init_attr: Optional attributes for a QP created on the rdma_cm_id.
+ * Description:
+ * Create an identifier and option QP used for communication.
+ * Notes:
+ * If qp_init_attr is provided, then a queue pair will be allocated and
+ * associated with the rdma_cm_id. If a pd is provided, the QP will be
+ * created on that PD. Otherwise, the QP will be allocated on a default
+ * PD.
+ * The rdma_cm_id will be set to use synchronous operations (connect,
+ * listen, and get_request). To convert to asynchronous operation, the
+ * rdma_cm_id should be migrated to a user allocated event channel.
+ * See also:
+ * rdma_create_id, rdma_create_qp, rdma_migrate_id, rdma_connect,
+ * rdma_listen
+ */
+int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
+ struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr);
+
+/**
+ * rdma_destroy_ep - Deallocates a communication identifier and qp.
+ * @id: The communication identifier to destroy.
+ * Description:
+ * Destroys the specified rdma_cm_id and any associated QP created
+ * on that id.
+ * See also:
+ * rdma_create_ep
+ */
+void rdma_destroy_ep(struct rdma_cm_id *id);
+
+/**
+ * rdma_destroy_id - Release a communication identifier.
+ * @id: The communication identifier to destroy.
+ * Description:
+ * Destroys the specified rdma_cm_id and cancels any outstanding
+ * asynchronous operation.
+ * Notes:
+ * Users must free any associated QP with the rdma_cm_id before
+ * calling this routine and ack an related events.
+ * See also:
+ * rdma_create_id, rdma_destroy_qp, rdma_ack_cm_event
+ */
+int rdma_destroy_id(struct rdma_cm_id *id);
+
+/**
+ * rdma_bind_addr - Bind an RDMA identifier to a source address.
+ * @id: RDMA identifier.
+ * @addr: Local address information. Wildcard values are permitted.
+ * Description:
+ * Associates a source address with an rdma_cm_id. The address may be
+ * wildcarded. If binding to a specific local address, the rdma_cm_id
+ * will also be bound to a local RDMA device.
+ * Notes:
+ * Typically, this routine is called before calling rdma_listen to bind
+ * to a specific port number, but it may also be called on the active side
+ * of a connection before calling rdma_resolve_addr to bind to a specific
+ * address.
+ * See also:
+ * rdma_create_id, rdma_listen, rdma_resolve_addr, rdma_create_qp
+ */
+int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr);
+
+/**
+ * rdma_resolve_addr - Resolve destination and optional source addresses.
+ * @id: RDMA identifier.
+ * @src_addr: Source address information. This parameter may be NULL.
+ * @dst_addr: Destination address information.
+ * @timeout_ms: Time to wait for resolution to complete.
+ * Description:
+ * Resolve destination and optional source addresses from IP addresses
+ * to an RDMA address. If successful, the specified rdma_cm_id will
+ * be bound to a local device.
+ * Notes:
+ * This call is used to map a given destination IP address to a usable RDMA
+ * address. If a source address is given, the rdma_cm_id is bound to that
+ * address, the same as if rdma_bind_addr were called. If no source
+ * address is given, and the rdma_cm_id has not yet been bound to a device,
+ * then the rdma_cm_id will be bound to a source address based on the
+ * local routing tables. After this call, the rdma_cm_id will be bound to
+ * an RDMA device. This call is typically made from the active side of a
+ * connection before calling rdma_resolve_route and rdma_connect.
+ * See also:
+ * rdma_create_id, rdma_resolve_route, rdma_connect, rdma_create_qp,
+ * rdma_get_cm_event, rdma_bind_addr
+ */
+int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
+ struct sockaddr *dst_addr, int timeout_ms);
+
+/**
+ * rdma_resolve_route - Resolve the route information needed to establish a connection.
+ * @id: RDMA identifier.
+ * @timeout_ms: Time to wait for resolution to complete.
+ * Description:
+ * Resolves an RDMA route to the destination address in order to establish
+ * a connection. The destination address must have already been resolved
+ * by calling rdma_resolve_addr.
+ * Notes:
+ * This is called on the client side of a connection after calling
+ * rdma_resolve_addr, but before calling rdma_connect.
+ * See also:
+ * rdma_resolve_addr, rdma_connect, rdma_get_cm_event
+ */
+int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms);
+
+/**
+ * rdma_create_qp - Allocate a QP.
+ * @id: RDMA identifier.
+ * @pd: Optional protection domain for the QP.
+ * @qp_init_attr: initial QP attributes.
+ * Description:
+ * Allocate a QP associated with the specified rdma_cm_id and transition it
+ * for sending and receiving.
+ * Notes:
+ * The rdma_cm_id must be bound to a local RDMA device before calling this
+ * function, and the protection domain must be for that same device.
+ * QPs allocated to an rdma_cm_id are automatically transitioned by the
+ * librdmacm through their states. After being allocated, the QP will be
+ * ready to handle posting of receives. If the QP is unconnected, it will
+ * be ready to post sends.
+ * If pd is NULL, then the QP will be allocated using a default protection
+ * domain associated with the underlying RDMA device.
+ * See also:
+ * rdma_bind_addr, rdma_resolve_addr, rdma_destroy_qp, ibv_create_qp,
+ * ibv_modify_qp
+ */
+int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd,
+ struct ibv_qp_init_attr *qp_init_attr);
+int rdma_create_qp_ex(struct rdma_cm_id *id,
+ struct ibv_qp_init_attr_ex *qp_init_attr);
+
+/**
+ * rdma_destroy_qp - Deallocate a QP.
+ * @id: RDMA identifier.
+ * Description:
+ * Destroy a QP allocated on the rdma_cm_id.
+ * Notes:
+ * Users must destroy any QP associated with an rdma_cm_id before
+ * destroying the ID.
+ * See also:
+ * rdma_create_qp, rdma_destroy_id, ibv_destroy_qp
+ */
+void rdma_destroy_qp(struct rdma_cm_id *id);
+
+/**
+ * rdma_connect - Initiate an active connection request.
+ * @id: RDMA identifier.
+ * @conn_param: optional connection parameters.
+ * Description:
+ * For a connected rdma_cm_id, this call initiates a connection request
+ * to a remote destination. For an unconnected rdma_cm_id, it initiates
+ * a lookup of the remote QP providing the datagram service.
+ * Notes:
+ * Users must have resolved a route to the destination address
+ * by having called rdma_resolve_route before calling this routine.
+ * A user may override the default connection parameters and exchange
+ * private data as part of the connection by using the conn_param parameter.
+ * See also:
+ * rdma_resolve_route, rdma_disconnect, rdma_listen, rdma_get_cm_event
+ */
+int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param);
+
+/**
+ * rdma_listen - Listen for incoming connection requests.
+ * @id: RDMA identifier.
+ * @backlog: backlog of incoming connection requests.
+ * Description:
+ * Initiates a listen for incoming connection requests or datagram service
+ * lookup. The listen will be restricted to the locally bound source
+ * address.
+ * Notes:
+ * Users must have bound the rdma_cm_id to a local address by calling
+ * rdma_bind_addr before calling this routine. If the rdma_cm_id is
+ * bound to a specific IP address, the listen will be restricted to that
+ * address and the associated RDMA device. If the rdma_cm_id is bound
+ * to an RDMA port number only, the listen will occur across all RDMA
+ * devices.
+ * See also:
+ * rdma_bind_addr, rdma_connect, rdma_accept, rdma_reject, rdma_get_cm_event
+ */
+int rdma_listen(struct rdma_cm_id *id, int backlog);
+
+/**
+ * rdma_get_request
+ */
+int rdma_get_request(struct rdma_cm_id *listen, struct rdma_cm_id **id);
+
+/**
+ * rdma_accept - Called to accept a connection request.
+ * @id: Connection identifier associated with the request.
+ * @conn_param: Optional information needed to establish the connection.
+ * Description:
+ * Called from the listening side to accept a connection or datagram
+ * service lookup request.
+ * Notes:
+ * Unlike the socket accept routine, rdma_accept is not called on a
+ * listening rdma_cm_id. Instead, after calling rdma_listen, the user
+ * waits for a connection request event to occur. Connection request
+ * events give the user a newly created rdma_cm_id, similar to a new
+ * socket, but the rdma_cm_id is bound to a specific RDMA device.
+ * rdma_accept is called on the new rdma_cm_id.
+ * A user may override the default connection parameters and exchange
+ * private data as part of the connection by using the conn_param parameter.
+ * See also:
+ * rdma_listen, rdma_reject, rdma_get_cm_event
+ */
+int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param);
+
+/**
+ * rdma_reject - Called to reject a connection request.
+ * @id: Connection identifier associated with the request.
+ * @private_data: Optional private data to send with the reject message.
+ * @private_data_len: Size of the private_data to send, in bytes.
+ * Description:
+ * Called from the listening side to reject a connection or datagram
+ * service lookup request.
+ * Notes:
+ * After receiving a connection request event, a user may call rdma_reject
+ * to reject the request. If the underlying RDMA transport supports
+ * private data in the reject message, the specified data will be passed to
+ * the remote side.
+ * See also:
+ * rdma_listen, rdma_accept, rdma_get_cm_event
+ */
+int rdma_reject(struct rdma_cm_id *id, const void *private_data,
+ uint8_t private_data_len);
+
+/**
+ * rdma_notify - Notifies the librdmacm of an asynchronous event.
+ * @id: RDMA identifier.
+ * @event: Asynchronous event.
+ * Description:
+ * Used to notify the librdmacm of asynchronous events that have occurred
+ * on a QP associated with the rdma_cm_id.
+ * Notes:
+ * Asynchronous events that occur on a QP are reported through the user's
+ * device event handler. This routine is used to notify the librdmacm of
+ * communication events. In most cases, use of this routine is not
+ * necessary, however if connection establishment is done out of band
+ * (such as done through Infiniband), it's possible to receive data on a
+ * QP that is not yet considered connected. This routine forces the
+ * connection into an established state in this case in order to handle
+ * the rare situation where the connection never forms on its own.
+ * Events that should be reported to the CM are: IB_EVENT_COMM_EST.
+ * See also:
+ * rdma_connect, rdma_accept, rdma_listen
+ */
+int rdma_notify(struct rdma_cm_id *id, enum ibv_event_type event);
+
+/**
+ * rdma_disconnect - This function disconnects a connection.
+ * @id: RDMA identifier.
+ * Description:
+ * Disconnects a connection and transitions any associated QP to the
+ * error state.
+ * See also:
+ * rdma_connect, rdma_listen, rdma_accept
+ */
+int rdma_disconnect(struct rdma_cm_id *id);
+
+/**
+ * rdma_join_multicast - Joins a multicast group.
+ * @id: Communication identifier associated with the request.
+ * @addr: Multicast address identifying the group to join.
+ * @context: User-defined context associated with the join request.
+ * Description:
+ * Joins a multicast group and attaches an associated QP to the group.
+ * Notes:
+ * Before joining a multicast group, the rdma_cm_id must be bound to
+ * an RDMA device by calling rdma_bind_addr or rdma_resolve_addr. Use of
+ * rdma_resolve_addr requires the local routing tables to resolve the
+ * multicast address to an RDMA device. The user must call
+ * rdma_leave_multicast to leave the multicast group and release any
+ * multicast resources. The context is returned to the user through
+ * the private_data field in the rdma_cm_event.
+ * See also:
+ * rdma_leave_multicast, rdma_bind_addr, rdma_resolve_addr, rdma_create_qp
+ */
+int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
+ void *context);
+
+/**
+ * rdma_leave_multicast - Leaves a multicast group.
+ * @id: Communication identifier associated with the request.
+ * @addr: Multicast address identifying the group to leave.
+ * Description:
+ * Leaves a multicast group and detaches an associated QP from the group.
+ * Notes:
+ * Calling this function before a group has been fully joined results in
+ * canceling the join operation. Users should be aware that messages
+ * received from the multicast group may stilled be queued for
+ * completion processing immediately after leaving a multicast group.
+ * Destroying an rdma_cm_id will automatically leave all multicast groups.
+ * See also:
+ * rdma_join_multicast, rdma_destroy_qp
+ */
+int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr);
+
+/**
+ * rdma_multicast_ex - Joins a multicast group with options.
+ * @id: Communication identifier associated with the request.
+ * @mc_join_attr: Extensive struct containing multicast join parameters.
+ * @context: User-defined context associated with the join request.
+ * Description:
+ * Joins a multicast group with options. Currently supporting MC join flags.
+ * The QP will be attached based on the given join flag.
+ * Join message will be sent according to the join flag.
+ * Notes:
+ * Before joining a multicast group, the rdma_cm_id must be bound to
+ * an RDMA device by calling rdma_bind_addr or rdma_resolve_addr. Use of
+ * rdma_resolve_addr requires the local routing tables to resolve the
+ * multicast address to an RDMA device. The user must call
+ * rdma_leave_multicast to leave the multicast group and release any
+ * multicast resources. The context is returned to the user through
+ * the private_data field in the rdma_cm_event.
+ * See also:
+ * rdma_leave_multicast, rdma_bind_addr, rdma_resolve_addr, rdma_create_qp
+ */
+int rdma_join_multicast_ex(struct rdma_cm_id *id,
+ struct rdma_cm_join_mc_attr_ex *mc_join_attr,
+ void *context);
+
+/**
+ * rdma_get_cm_event - Retrieves the next pending communication event.
+ * @channel: Event channel to check for events.
+ * @event: Allocated information about the next communication event.
+ * Description:
+ * Retrieves a communication event. If no events are pending, by default,
+ * the call will block until an event is received.
+ * Notes:
+ * The default synchronous behavior of this routine can be changed by
+ * modifying the file descriptor associated with the given channel. All
+ * events that are reported must be acknowledged by calling rdma_ack_cm_event.
+ * Destruction of an rdma_cm_id will block until related events have been
+ * acknowledged.
+ * See also:
+ * rdma_ack_cm_event, rdma_create_event_channel, rdma_event_str
+ */
+int rdma_get_cm_event(struct rdma_event_channel *channel,
+ struct rdma_cm_event **event);
+
+/**
+ * rdma_ack_cm_event - Free a communication event.
+ * @event: Event to be released.
+ * Description:
+ * All events which are allocated by rdma_get_cm_event must be released,
+ * there should be a one-to-one correspondence between successful gets
+ * and acks.
+ * See also:
+ * rdma_get_cm_event, rdma_destroy_id
+ */
+int rdma_ack_cm_event(struct rdma_cm_event *event);
+
+__be16 rdma_get_src_port(struct rdma_cm_id *id);
+__be16 rdma_get_dst_port(struct rdma_cm_id *id);
+
+static inline struct sockaddr *rdma_get_local_addr(struct rdma_cm_id *id)
+{
+ return &id->route.addr.src_addr;
+}
+
+static inline struct sockaddr *rdma_get_peer_addr(struct rdma_cm_id *id)
+{
+ return &id->route.addr.dst_addr;
+}
+
+/**
+ * rdma_get_devices - Get list of RDMA devices currently available.
+ * @num_devices: If non-NULL, set to the number of devices returned.
+ * Description:
+ * Return a NULL-terminated array of opened RDMA devices. Callers can use
+ * this routine to allocate resources on specific RDMA devices that will be
+ * shared across multiple rdma_cm_id's.
+ * Notes:
+ * The returned array must be released by calling rdma_free_devices. Devices
+ * remain opened while the librdmacm is loaded.
+ * See also:
+ * rdma_free_devices
+ */
+struct ibv_context **rdma_get_devices(int *num_devices);
+
+/**
+ * rdma_free_devices - Frees the list of devices returned by rdma_get_devices.
+ * @list: List of devices returned from rdma_get_devices.
+ * Description:
+ * Frees the device array returned by rdma_get_devices.
+ * See also:
+ * rdma_get_devices
+ */
+void rdma_free_devices(struct ibv_context **list);
+
+/**
+ * rdma_event_str - Returns a string representation of an rdma cm event.
+ * @event: Asynchronous event.
+ * Description:
+ * Returns a string representation of an asynchronous event.
+ * See also:
+ * rdma_get_cm_event
+ */
+const char *rdma_event_str(enum rdma_cm_event_type event);
+
+/* Option levels */
+enum {
+ RDMA_OPTION_ID = 0,
+ RDMA_OPTION_IB = 1
+};
+
+/* Option details */
+enum {
+ RDMA_OPTION_ID_TOS = 0, /* uint8_t: RFC 2474 */
+ RDMA_OPTION_ID_REUSEADDR = 1, /* int: ~SO_REUSEADDR */
+ RDMA_OPTION_ID_AFONLY = 2, /* int: ~IPV6_V6ONLY */
+ RDMA_OPTION_IB_PATH = 1 /* struct ibv_path_data[] */
+};
+
+/**
+ * rdma_set_option - Set options for an rdma_cm_id.
+ * @id: Communication identifier to set option for.
+ * @level: Protocol level of the option to set.
+ * @optname: Name of the option to set.
+ * @optval: Reference to the option data.
+ * @optlen: The size of the %optval buffer.
+ */
+int rdma_set_option(struct rdma_cm_id *id, int level, int optname,
+ void *optval, size_t optlen);
+
+/**
+ * rdma_migrate_id - Move an rdma_cm_id to a new event channel.
+ * @id: Communication identifier to migrate.
+ * @channel: New event channel for rdma_cm_id events.
+ */
+int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel);
+
+/**
+ * rdma_getaddrinfo - RDMA address and route resolution service.
+ */
+int rdma_getaddrinfo(const char *node, const char *service,
+ const struct rdma_addrinfo *hints,
+ struct rdma_addrinfo **res);
+
+void rdma_freeaddrinfo(struct rdma_addrinfo *res);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RDMA_CMA_H */
--
1.8.3.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* Re: [PATCH v3 iproute2-next 1/8] rdma: include rdma-core <rdma/rdma_cma.h>
2018-03-26 20:57 ` [PATCH v3 iproute2-next 1/8] rdma: include rdma-core <rdma/rdma_cma.h> Steve Wise
@ 2018-03-27 5:46 ` Leon Romanovsky
2018-03-27 13:23 ` David Ahern
0 siblings, 1 reply; 11+ messages in thread
From: Leon Romanovsky @ 2018-03-27 5:46 UTC (permalink / raw)
To: Steve Wise; +Cc: dsahern, stephen, netdev, linux-rdma
[-- Attachment #1: Type: text/plain, Size: 535 bytes --]
On Mon, Mar 26, 2018 at 01:57:32PM -0700, Steve Wise wrote:
> This avoids requiring rdma-core be installed on systems.
>
> Signed-off-by: Steve Wise <swise@opengridcomputing.com>
> ---
> rdma/include/rdma/rdma_cma.h | 728 +++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 728 insertions(+)
> create mode 100644 rdma/include/rdma/rdma_cma.h
>
Steve,
Sorry for not spotting it before, you actually need only 3 enums for the
cm_id_ps_to_str() from rdma_cma.h.
Simply copy/paste that enum into cm_id_ps_to_str().
Thanks
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH v3 iproute2-next 1/8] rdma: include rdma-core <rdma/rdma_cma.h>
2018-03-27 5:46 ` Leon Romanovsky
@ 2018-03-27 13:23 ` David Ahern
0 siblings, 0 replies; 11+ messages in thread
From: David Ahern @ 2018-03-27 13:23 UTC (permalink / raw)
To: Leon Romanovsky, Steve Wise; +Cc: stephen, netdev, linux-rdma
On 3/26/18 11:46 PM, Leon Romanovsky wrote:
> On Mon, Mar 26, 2018 at 01:57:32PM -0700, Steve Wise wrote:
>> This avoids requiring rdma-core be installed on systems.
>>
>> Signed-off-by: Steve Wise <swise@opengridcomputing.com>
>> ---
>> rdma/include/rdma/rdma_cma.h | 728 +++++++++++++++++++++++++++++++++++++++++++
>> 1 file changed, 728 insertions(+)
>> create mode 100644 rdma/include/rdma/rdma_cma.h
>>
>
> Steve,
>
> Sorry for not spotting it before, you actually need only 3 enums for the
> cm_id_ps_to_str() from rdma_cma.h.
>
> Simply copy/paste that enum into cm_id_ps_to_str().
>
> Thanks
>
I think this is Jason's point: if that enum is part of the UAPI why
isn't it part of a uapi header file?
We definitely do not want that entire header file brought in to iproute2
^ permalink raw reply [flat|nested] 11+ messages in thread
* [PATCH v3 iproute2-next 2/8] rdma: Makefile change to find <rdma/rdma_cma.h>
2018-03-26 20:59 [PATCH v3 iproute2-next 0/8] cm_id, cq, mr, and pd resource tracking Steve Wise
2018-03-26 20:57 ` [PATCH v3 iproute2-next 1/8] rdma: include rdma-core <rdma/rdma_cma.h> Steve Wise
@ 2018-03-26 20:57 ` Steve Wise
2018-03-26 20:57 ` [PATCH v3 iproute2-next 3/8] rdma: update rdma_netlink.h Steve Wise
` (5 subsequent siblings)
7 siblings, 0 replies; 11+ messages in thread
From: Steve Wise @ 2018-03-26 20:57 UTC (permalink / raw)
To: dsahern; +Cc: leon, stephen, netdev, linux-rdma
Signed-off-by: Steve Wise <swise@opengridcomputing.com>
---
rdma/Makefile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/rdma/Makefile b/rdma/Makefile
index 819fcbe..2cb46b6 100644
--- a/rdma/Makefile
+++ b/rdma/Makefile
@@ -4,7 +4,7 @@ include ../config.mk
TARGETS :=
ifeq ($(HAVE_MNL),y)
-CFLAGS += -I./include/uapi/
+CFLAGS += -I./include/uapi/ -I./include/
RDMA_OBJ = rdma.o utils.o dev.o link.o res.o
--
1.8.3.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v3 iproute2-next 3/8] rdma: update rdma_netlink.h
2018-03-26 20:59 [PATCH v3 iproute2-next 0/8] cm_id, cq, mr, and pd resource tracking Steve Wise
2018-03-26 20:57 ` [PATCH v3 iproute2-next 1/8] rdma: include rdma-core <rdma/rdma_cma.h> Steve Wise
2018-03-26 20:57 ` [PATCH v3 iproute2-next 2/8] rdma: Makefile change to find <rdma/rdma_cma.h> Steve Wise
@ 2018-03-26 20:57 ` Steve Wise
2018-03-26 20:57 ` [PATCH v3 iproute2-next 4/8] rdma: initialize the rd struct Steve Wise
` (4 subsequent siblings)
7 siblings, 0 replies; 11+ messages in thread
From: Steve Wise @ 2018-03-26 20:57 UTC (permalink / raw)
To: dsahern; +Cc: leon, stephen, netdev, linux-rdma
From: Steve Wise <swise@opengridcomputing.com>
Pull in the latest rdma_netlink.h which has support for
the rdma nldev resource tracking objects being added
with this patch series.
Signed-off-by: Steve Wise <swise@opengridcomputing.com>
---
rdma/include/uapi/rdma/rdma_netlink.h | 38 +++++++++++++++++++++++++++++++++++
1 file changed, 38 insertions(+)
diff --git a/rdma/include/uapi/rdma/rdma_netlink.h b/rdma/include/uapi/rdma/rdma_netlink.h
index dbac3b8..9446a72 100644
--- a/rdma/include/uapi/rdma/rdma_netlink.h
+++ b/rdma/include/uapi/rdma/rdma_netlink.h
@@ -238,6 +238,14 @@ enum rdma_nldev_command {
RDMA_NLDEV_CMD_RES_QP_GET, /* can dump */
+ RDMA_NLDEV_CMD_RES_CM_ID_GET, /* can dump */
+
+ RDMA_NLDEV_CMD_RES_CQ_GET, /* can dump */
+
+ RDMA_NLDEV_CMD_RES_MR_GET, /* can dump */
+
+ RDMA_NLDEV_CMD_RES_PD_GET, /* can dump */
+
RDMA_NLDEV_NUM_OPS
};
@@ -350,6 +358,36 @@ enum rdma_nldev_attr {
*/
RDMA_NLDEV_ATTR_RES_KERN_NAME, /* string */
+ RDMA_NLDEV_ATTR_RES_CM_ID, /* nested table */
+ RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY, /* nested table */
+ /*
+ * rdma_cm_id port space.
+ */
+ RDMA_NLDEV_ATTR_RES_PS, /* u32 */
+ /*
+ * Source and destination socket addresses
+ */
+ RDMA_NLDEV_ATTR_RES_SRC_ADDR, /* __kernel_sockaddr_storage */
+ RDMA_NLDEV_ATTR_RES_DST_ADDR, /* __kernel_sockaddr_storage */
+
+ RDMA_NLDEV_ATTR_RES_CQ, /* nested table */
+ RDMA_NLDEV_ATTR_RES_CQ_ENTRY, /* nested table */
+ RDMA_NLDEV_ATTR_RES_CQE, /* u32 */
+ RDMA_NLDEV_ATTR_RES_USECNT, /* u64 */
+ RDMA_NLDEV_ATTR_RES_POLL_CTX, /* u8 */
+
+ RDMA_NLDEV_ATTR_RES_MR, /* nested table */
+ RDMA_NLDEV_ATTR_RES_MR_ENTRY, /* nested table */
+ RDMA_NLDEV_ATTR_RES_RKEY, /* u32 */
+ RDMA_NLDEV_ATTR_RES_LKEY, /* u32 */
+ RDMA_NLDEV_ATTR_RES_IOVA, /* u64 */
+ RDMA_NLDEV_ATTR_RES_MRLEN, /* u64 */
+
+ RDMA_NLDEV_ATTR_RES_PD, /* nested table */
+ RDMA_NLDEV_ATTR_RES_PD_ENTRY, /* nested table */
+ RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY, /* u32 */
+ RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY, /* u32 */
+
RDMA_NLDEV_ATTR_MAX
};
#endif /* _RDMA_NETLINK_H */
--
1.8.3.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v3 iproute2-next 4/8] rdma: initialize the rd struct
2018-03-26 20:59 [PATCH v3 iproute2-next 0/8] cm_id, cq, mr, and pd resource tracking Steve Wise
` (2 preceding siblings ...)
2018-03-26 20:57 ` [PATCH v3 iproute2-next 3/8] rdma: update rdma_netlink.h Steve Wise
@ 2018-03-26 20:57 ` Steve Wise
2018-03-26 20:57 ` [PATCH v3 iproute2-next 5/8] rdma: Add CM_ID resource tracking information Steve Wise
` (3 subsequent siblings)
7 siblings, 0 replies; 11+ messages in thread
From: Steve Wise @ 2018-03-26 20:57 UTC (permalink / raw)
To: dsahern; +Cc: leon, stephen, netdev, linux-rdma
Initialize the rd struct so port_idx is 0 unless set otherwise.
Otherwise, strict_port queries end up passing an uninitialized PORT
nlattr.
Signed-off-by: Steve Wise <swise@opengridcomputing.com>
Reviewed-by: Leon Romanovsky <leonro@mellanox.com>
---
rdma/rdma.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/rdma/rdma.c b/rdma/rdma.c
index ab2c960..b43e538 100644
--- a/rdma/rdma.c
+++ b/rdma/rdma.c
@@ -135,7 +135,7 @@ int main(int argc, char **argv)
bool json_output = false;
bool force = false;
char *filename;
- struct rd rd;
+ struct rd rd = {};
int opt;
int err;
--
1.8.3.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v3 iproute2-next 5/8] rdma: Add CM_ID resource tracking information
2018-03-26 20:59 [PATCH v3 iproute2-next 0/8] cm_id, cq, mr, and pd resource tracking Steve Wise
` (3 preceding siblings ...)
2018-03-26 20:57 ` [PATCH v3 iproute2-next 4/8] rdma: initialize the rd struct Steve Wise
@ 2018-03-26 20:57 ` Steve Wise
2018-03-26 20:57 ` [PATCH v3 iproute2-next 6/8] rdma: Add CQ " Steve Wise
` (2 subsequent siblings)
7 siblings, 0 replies; 11+ messages in thread
From: Steve Wise @ 2018-03-26 20:57 UTC (permalink / raw)
To: dsahern; +Cc: leon, stephen, netdev, linux-rdma
Sample output:
# rdma resource
2: cxgb4_0: pd 5 cq 2 qp 2 cm_id 3 mr 7
3: mlx4_0: pd 7 cq 3 qp 3 cm_id 3 mr 7
# rdma resource show cm_id
link cxgb4_0/- lqpn 0 qp-type RC state LISTEN ps TCP pid 30485 comm rping src-addr 0.0.0.0:7174
link cxgb4_0/2 lqpn 1048 qp-type RC state CONNECT ps TCP pid 30503 comm rping src-addr 172.16.2.1:7174 dst-addr 172.16.2.1:38246
link cxgb4_0/2 lqpn 1040 qp-type RC state CONNECT ps TCP pid 30498 comm rping src-addr 172.16.2.1:38246 dst-addr 172.16.2.1:7174
link mlx4_0/- lqpn 0 qp-type RC state LISTEN ps TCP pid 30485 comm rping src-addr 0.0.0.0:7174
link mlx4_0/1 lqpn 539 qp-type RC state CONNECT ps TCP pid 30494 comm rping src-addr 172.16.99.1:7174 dst-addr 172.16.99.1:43670
link mlx4_0/1 lqpn 538 qp-type RC state CONNECT ps TCP pid 30492 comm rping src-addr 172.16.99.1:43670 dst-addr 172.16.99.1:7174
# rdma resource show cm_id dst-port 7174
link cxgb4_0/2 lqpn 1040 qp-type RC state CONNECT ps TCP pid 30498 comm rping src-addr 172.16.2.1:38246 dst-addr 172.16.2.1:7174
link mlx4_0/1 lqpn 538 qp-type RC state CONNECT ps TCP pid 30492 comm rping src-addr 172.16.99.1:43670 dst-addr 172.16.99.1:7174
Signed-off-by: Steve Wise <swise@opengridcomputing.com>
Reviewed-by: Leon Romanovsky <leonro@mellanox.com>
---
rdma/rdma.h | 2 +
rdma/res.c | 262 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
rdma/utils.c | 5 ++
3 files changed, 268 insertions(+), 1 deletion(-)
diff --git a/rdma/rdma.h b/rdma/rdma.h
index 5809f70..e55205b 100644
--- a/rdma/rdma.h
+++ b/rdma/rdma.h
@@ -18,10 +18,12 @@
#include <libmnl/libmnl.h>
#include <rdma/rdma_netlink.h>
#include <time.h>
+#include <net/if_arp.h>
#include "list.h"
#include "utils.h"
#include "json_writer.h"
+#include <rdma/rdma_cma.h>
#define pr_err(args...) fprintf(stderr, ##args)
#define pr_out(args...) fprintf(stdout, ##args)
diff --git a/rdma/res.c b/rdma/res.c
index 62f5c54..5506cf3 100644
--- a/rdma/res.c
+++ b/rdma/res.c
@@ -16,9 +16,11 @@ static int res_help(struct rd *rd)
{
pr_out("Usage: %s resource\n", rd->filename);
pr_out(" resource show [DEV]\n");
- pr_out(" resource show [qp]\n");
+ pr_out(" resource show [qp|cm_id]\n");
pr_out(" resource show qp link [DEV/PORT]\n");
pr_out(" resource show qp link [DEV/PORT] [FILTER-NAME FILTER-VALUE]\n");
+ pr_out(" resource show cm_id link [DEV/PORT]\n");
+ pr_out(" resource show cm_id link [DEV/PORT] [FILTER-NAME FILTER-VALUE]\n");
return 0;
}
@@ -433,6 +435,245 @@ static int res_qp_parse_cb(const struct nlmsghdr *nlh, void *data)
return MNL_CB_OK;
}
+static void print_qp_type(struct rd *rd, uint32_t val)
+{
+ if (rd->json_output)
+ jsonw_string_field(rd->jw, "qp-type",
+ qp_types_to_str(val));
+ else
+ pr_out("qp-type %s ", qp_types_to_str(val));
+}
+
+static const char *cm_id_state_to_str(uint8_t idx)
+{
+ static const char * const cm_id_states_str[] = {
+ "IDLE", "ADDR_QUERY", "ADDR_RESOLVED", "ROUTE_QUERY",
+ "ROUTE_RESOLVED", "CONNECT", "DISCONNECT", "ADDR_BOUND",
+ "LISTEN", "DEVICE_REMOVAL", "DESTROYING" };
+
+ if (idx < ARRAY_SIZE(cm_id_states_str))
+ return cm_id_states_str[idx];
+ return "UNKNOWN";
+}
+
+static const char *cm_id_ps_to_str(uint32_t ps)
+{
+ switch (ps) {
+ case RDMA_PS_IPOIB:
+ return "IPoIB";
+ case RDMA_PS_IB:
+ return "IPoIB";
+ case RDMA_PS_TCP:
+ return "TCP";
+ case RDMA_PS_UDP:
+ return "UDP";
+ default:
+ return "---";
+ }
+}
+
+static void print_cm_id_state(struct rd *rd, uint8_t state)
+{
+ if (rd->json_output) {
+ jsonw_string_field(rd->jw, "state", cm_id_state_to_str(state));
+ return;
+ }
+ pr_out("state %s ", cm_id_state_to_str(state));
+}
+
+static void print_ps(struct rd *rd, uint32_t ps)
+{
+ if (rd->json_output) {
+ jsonw_string_field(rd->jw, "ps", cm_id_ps_to_str(ps));
+ return;
+ }
+ pr_out("ps %s ", cm_id_ps_to_str(ps));
+}
+
+static void print_ipaddr(struct rd *rd, const char *key, char *addrstr,
+ uint16_t port)
+{
+ if (rd->json_output) {
+ int name_size = INET6_ADDRSTRLEN+strlen(":65535");
+ char json_name[name_size];
+
+ snprintf(json_name, name_size, "%s:%u", addrstr, port);
+ jsonw_string_field(rd->jw, key, json_name);
+ return;
+ }
+ pr_out("%s %s:%u ", key, addrstr, port);
+}
+
+static int ss_ntop(struct nlattr *nla_line, char *addr_str, uint16_t *port)
+{
+ struct __kernel_sockaddr_storage *addr;
+
+ addr = (struct __kernel_sockaddr_storage *)
+ mnl_attr_get_payload(nla_line);
+ switch (addr->ss_family) {
+ case AF_INET: {
+ struct sockaddr_in *sin = (struct sockaddr_in *)addr;
+
+ if (!inet_ntop(AF_INET, (const void *)&sin->sin_addr, addr_str,
+ INET6_ADDRSTRLEN))
+ return -EINVAL;
+ *port = ntohs(sin->sin_port);
+ break;
+ }
+ case AF_INET6: {
+ struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
+
+ if (!inet_ntop(AF_INET6, (const void *)&sin6->sin6_addr,
+ addr_str, INET6_ADDRSTRLEN))
+ return -EINVAL;
+ *port = ntohs(sin6->sin6_port);
+ break;
+ }
+ default:
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static int res_cm_id_parse_cb(const struct nlmsghdr *nlh, void *data)
+{
+ struct nlattr *tb[RDMA_NLDEV_ATTR_MAX] = {};
+ struct nlattr *nla_table, *nla_entry;
+ struct rd *rd = data;
+ const char *name;
+ int idx;
+
+ mnl_attr_parse(nlh, 0, rd_attr_cb, tb);
+ if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX] ||
+ !tb[RDMA_NLDEV_ATTR_DEV_NAME] ||
+ !tb[RDMA_NLDEV_ATTR_RES_CM_ID])
+ return MNL_CB_ERROR;
+
+ name = mnl_attr_get_str(tb[RDMA_NLDEV_ATTR_DEV_NAME]);
+ idx = mnl_attr_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
+ nla_table = tb[RDMA_NLDEV_ATTR_RES_CM_ID];
+ mnl_attr_for_each_nested(nla_entry, nla_table) {
+ struct nlattr *nla_line[RDMA_NLDEV_ATTR_MAX] = {};
+ char src_addr_str[INET6_ADDRSTRLEN];
+ char dst_addr_str[INET6_ADDRSTRLEN];
+ uint16_t src_port, dst_port;
+ uint32_t port = 0, pid = 0;
+ uint8_t type = 0, state;
+ uint32_t lqpn = 0, ps;
+ char *comm = NULL;
+ int err;
+
+ err = mnl_attr_parse_nested(nla_entry, rd_attr_cb, nla_line);
+ if (err != MNL_CB_OK)
+ return -EINVAL;
+
+ if (!nla_line[RDMA_NLDEV_ATTR_RES_STATE] ||
+ !nla_line[RDMA_NLDEV_ATTR_RES_PS] ||
+ (!nla_line[RDMA_NLDEV_ATTR_RES_PID] &&
+ !nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME])) {
+ return MNL_CB_ERROR;
+ }
+
+ if (nla_line[RDMA_NLDEV_ATTR_PORT_INDEX])
+ port = mnl_attr_get_u32(
+ nla_line[RDMA_NLDEV_ATTR_PORT_INDEX]);
+
+ if (port && port != rd->port_idx)
+ continue;
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_LQPN]) {
+ lqpn = mnl_attr_get_u32(
+ nla_line[RDMA_NLDEV_ATTR_RES_LQPN]);
+ if (rd_check_is_filtered(rd, "lqpn", lqpn))
+ continue;
+ }
+ if (nla_line[RDMA_NLDEV_ATTR_RES_TYPE]) {
+ type = mnl_attr_get_u8(
+ nla_line[RDMA_NLDEV_ATTR_RES_TYPE]);
+ if (rd_check_is_string_filtered(rd, "qp-type",
+ qp_types_to_str(type)))
+ continue;
+ }
+
+ ps = mnl_attr_get_u32(nla_line[RDMA_NLDEV_ATTR_RES_PS]);
+ if (rd_check_is_string_filtered(rd, "ps", cm_id_ps_to_str(ps)))
+ continue;
+
+ state = mnl_attr_get_u8(nla_line[RDMA_NLDEV_ATTR_RES_STATE]);
+ if (rd_check_is_string_filtered(rd, "state",
+ cm_id_state_to_str(state)))
+ continue;
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_SRC_ADDR]) {
+ if (ss_ntop(nla_line[RDMA_NLDEV_ATTR_RES_SRC_ADDR],
+ src_addr_str, &src_port))
+ continue;
+ if (rd_check_is_string_filtered(rd, "src-addr",
+ src_addr_str))
+ continue;
+ }
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_DST_ADDR]) {
+ if (ss_ntop(nla_line[RDMA_NLDEV_ATTR_RES_DST_ADDR],
+ dst_addr_str, &dst_port))
+ continue;
+ if (rd_check_is_string_filtered(rd, "dst-addr",
+ dst_addr_str))
+ continue;
+ }
+
+ if (rd_check_is_filtered(rd, "src-port", src_port))
+ continue;
+
+ if (rd_check_is_filtered(rd, "dst-port", dst_port))
+ continue;
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_PID]) {
+ pid = mnl_attr_get_u32(
+ nla_line[RDMA_NLDEV_ATTR_RES_PID]);
+ comm = get_task_name(pid);
+ }
+
+ if (rd_check_is_filtered(rd, "pid", pid)) {
+ free(comm);
+ continue;
+ }
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME]) {
+ /* discard const from mnl_attr_get_str */
+ comm = (char *)mnl_attr_get_str(
+ nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME]);
+ }
+
+ if (rd->json_output)
+ jsonw_start_array(rd->jw);
+
+ print_link(rd, idx, name, port, nla_line);
+ if (nla_line[RDMA_NLDEV_ATTR_RES_LQPN])
+ print_lqpn(rd, lqpn);
+ if (nla_line[RDMA_NLDEV_ATTR_RES_TYPE])
+ print_qp_type(rd, type);
+ print_cm_id_state(rd, state);
+ print_ps(rd, ps);
+ print_pid(rd, pid);
+ print_comm(rd, comm, nla_line);
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_SRC_ADDR])
+ print_ipaddr(rd, "src-addr", src_addr_str, src_port);
+ if (nla_line[RDMA_NLDEV_ATTR_RES_DST_ADDR])
+ print_ipaddr(rd, "dst-addr", dst_addr_str, dst_port);
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_PID])
+ free(comm);
+
+ if (rd->json_output)
+ jsonw_end_array(rd->jw);
+ else
+ pr_out("\n");
+ }
+ return MNL_CB_OK;
+}
+
RES_FUNC(res_no_args, RDMA_NLDEV_CMD_RES_GET, NULL, true);
static const struct
@@ -457,11 +698,30 @@ filters qp_valid_filters[MAX_NUMBER_OF_FILTERS] = {{ .name = "link",
RES_FUNC(res_qp, RDMA_NLDEV_CMD_RES_QP_GET, qp_valid_filters, false);
+static const
+struct filters cm_id_valid_filters[MAX_NUMBER_OF_FILTERS] = {
+ { .name = "link", .is_number = false },
+ { .name = "lqpn", .is_number = true },
+ { .name = "qp-type", .is_number = false },
+ { .name = "state", .is_number = false },
+ { .name = "ps", .is_number = false },
+ { .name = "dev-type", .is_number = false },
+ { .name = "transport-type", .is_number = false },
+ { .name = "pid", .is_number = true },
+ { .name = "src-addr", .is_number = false },
+ { .name = "src-port", .is_number = true },
+ { .name = "dst-addr", .is_number = false },
+ { .name = "dst-port", .is_number = true }
+};
+
+RES_FUNC(res_cm_id, RDMA_NLDEV_CMD_RES_CM_ID_GET, cm_id_valid_filters, false);
+
static int res_show(struct rd *rd)
{
const struct rd_cmd cmds[] = {
{ NULL, res_no_args },
{ "qp", res_qp },
+ { "cm_id", res_cm_id },
{ 0 }
};
diff --git a/rdma/utils.c b/rdma/utils.c
index f946016..ec81737 100644
--- a/rdma/utils.c
+++ b/rdma/utils.c
@@ -375,6 +375,11 @@ static const enum mnl_attr_data_type nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
[RDMA_NLDEV_ATTR_RES_STATE] = MNL_TYPE_U8,
[RDMA_NLDEV_ATTR_RES_PID] = MNL_TYPE_U32,
[RDMA_NLDEV_ATTR_RES_KERN_NAME] = MNL_TYPE_NUL_STRING,
+ [RDMA_NLDEV_ATTR_RES_CM_ID] = MNL_TYPE_NESTED,
+ [RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY] = MNL_TYPE_NESTED,
+ [RDMA_NLDEV_ATTR_RES_PS] = MNL_TYPE_U32,
+ [RDMA_NLDEV_ATTR_RES_SRC_ADDR] = MNL_TYPE_UNSPEC,
+ [RDMA_NLDEV_ATTR_RES_DST_ADDR] = MNL_TYPE_UNSPEC,
};
int rd_attr_cb(const struct nlattr *attr, void *data)
--
1.8.3.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v3 iproute2-next 6/8] rdma: Add CQ resource tracking information
2018-03-26 20:59 [PATCH v3 iproute2-next 0/8] cm_id, cq, mr, and pd resource tracking Steve Wise
` (4 preceding siblings ...)
2018-03-26 20:57 ` [PATCH v3 iproute2-next 5/8] rdma: Add CM_ID resource tracking information Steve Wise
@ 2018-03-26 20:57 ` Steve Wise
2018-03-26 20:57 ` [PATCH v3 iproute2-next 7/8] rdma: Add MR " Steve Wise
2018-03-26 20:57 ` [PATCH v3 iproute2-next 8/8] rdma: Add PD " Steve Wise
7 siblings, 0 replies; 11+ messages in thread
From: Steve Wise @ 2018-03-26 20:57 UTC (permalink / raw)
To: dsahern; +Cc: leon, stephen, netdev, linux-rdma
Sample output:
# rdma resource show cq
link cxgb4_0/- cqe 46 users 2 pid 30503 comm rping
link cxgb4_0/- cqe 46 users 2 pid 30498 comm rping
link mlx4_0/- cqe 63 users 2 pid 30494 comm rping
link mlx4_0/- cqe 63 users 2 pid 30489 comm rping
link mlx4_0/- cqe 1023 users 2 poll_ctx WORKQUEUE pid 0 comm [ib_core]
# rdma resource show cq pid 30489
link mlx4_0/- cqe 63 users 2 pid 30489 comm rping
Signed-off-by: Steve Wise <swise@opengridcomputing.com>
Reviewed-by: Leon Romanovsky <leonro@mellanox.com>
---
rdma/res.c | 139 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
rdma/utils.c | 5 +++
2 files changed, 144 insertions(+)
diff --git a/rdma/res.c b/rdma/res.c
index 5506cf3..f1a90d3 100644
--- a/rdma/res.c
+++ b/rdma/res.c
@@ -21,6 +21,8 @@ static int res_help(struct rd *rd)
pr_out(" resource show qp link [DEV/PORT] [FILTER-NAME FILTER-VALUE]\n");
pr_out(" resource show cm_id link [DEV/PORT]\n");
pr_out(" resource show cm_id link [DEV/PORT] [FILTER-NAME FILTER-VALUE]\n");
+ pr_out(" resource show cq link [DEV/PORT]\n");
+ pr_out(" resource show cq link [DEV/PORT] [FILTER-NAME FILTER-VALUE]\n");
return 0;
}
@@ -674,6 +676,132 @@ static int res_cm_id_parse_cb(const struct nlmsghdr *nlh, void *data)
return MNL_CB_OK;
}
+static void print_cqe(struct rd *rd, uint32_t val)
+{
+ if (rd->json_output)
+ jsonw_uint_field(rd->jw, "cqe", val);
+ else
+ pr_out("cqe %u ", val);
+}
+
+static void print_users(struct rd *rd, uint64_t val)
+{
+ if (rd->json_output)
+ jsonw_uint_field(rd->jw, "users", val);
+ else
+ pr_out("users %" PRIu64 " ", val);
+}
+
+static const char *poll_ctx_to_str(uint8_t idx)
+{
+ static const char * const cm_id_states_str[] = {
+ "DIRECT", "SOFTIRQ", "WORKQUEUE"};
+
+ if (idx < ARRAY_SIZE(cm_id_states_str))
+ return cm_id_states_str[idx];
+ return "UNKNOWN";
+}
+
+static void print_poll_ctx(struct rd *rd, uint8_t poll_ctx)
+{
+ if (rd->json_output) {
+ jsonw_string_field(rd->jw, "poll-ctx",
+ poll_ctx_to_str(poll_ctx));
+ return;
+ }
+ pr_out("poll-ctx %s ", poll_ctx_to_str(poll_ctx));
+}
+
+static int res_cq_parse_cb(const struct nlmsghdr *nlh, void *data)
+{
+ struct nlattr *tb[RDMA_NLDEV_ATTR_MAX] = {};
+ struct nlattr *nla_table, *nla_entry;
+ struct rd *rd = data;
+ const char *name;
+ uint32_t idx;
+
+ mnl_attr_parse(nlh, 0, rd_attr_cb, tb);
+ if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX] ||
+ !tb[RDMA_NLDEV_ATTR_DEV_NAME] ||
+ !tb[RDMA_NLDEV_ATTR_RES_CQ])
+ return MNL_CB_ERROR;
+
+ name = mnl_attr_get_str(tb[RDMA_NLDEV_ATTR_DEV_NAME]);
+ idx = mnl_attr_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
+ nla_table = tb[RDMA_NLDEV_ATTR_RES_CQ];
+
+ mnl_attr_for_each_nested(nla_entry, nla_table) {
+ struct nlattr *nla_line[RDMA_NLDEV_ATTR_MAX] = {};
+ char *comm = NULL;
+ uint32_t pid = 0;
+ uint8_t poll_ctx = 0;
+ uint64_t users;
+ uint32_t cqe;
+ int err;
+
+ err = mnl_attr_parse_nested(nla_entry, rd_attr_cb, nla_line);
+ if (err != MNL_CB_OK)
+ return MNL_CB_ERROR;
+
+ if (!nla_line[RDMA_NLDEV_ATTR_RES_CQE] ||
+ !nla_line[RDMA_NLDEV_ATTR_RES_USECNT] ||
+ (!nla_line[RDMA_NLDEV_ATTR_RES_PID] &&
+ !nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME])) {
+ return MNL_CB_ERROR;
+ }
+
+ cqe = mnl_attr_get_u32(nla_line[RDMA_NLDEV_ATTR_RES_CQE]);
+
+ users = mnl_attr_get_u64(nla_line[RDMA_NLDEV_ATTR_RES_USECNT]);
+ if (rd_check_is_filtered(rd, "users", users))
+ continue;
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_POLL_CTX]) {
+ poll_ctx = mnl_attr_get_u8(
+ nla_line[RDMA_NLDEV_ATTR_RES_POLL_CTX]);
+ if (rd_check_is_string_filtered(rd, "poll-ctx",
+ poll_ctx_to_str(poll_ctx)))
+ continue;
+ }
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_PID]) {
+ pid = mnl_attr_get_u32(
+ nla_line[RDMA_NLDEV_ATTR_RES_PID]);
+ comm = get_task_name(pid);
+ }
+
+ if (rd_check_is_filtered(rd, "pid", pid)) {
+ free(comm);
+ continue;
+ }
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME])
+ /* discard const from mnl_attr_get_str */
+ comm = (char *)mnl_attr_get_str(
+ nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME]);
+
+ if (rd->json_output)
+ jsonw_start_array(rd->jw);
+
+ print_link(rd, idx, name, 0, nla_line);
+ print_cqe(rd, cqe);
+ print_users(rd, users);
+ if (nla_line[RDMA_NLDEV_ATTR_RES_POLL_CTX])
+ print_poll_ctx(rd, poll_ctx);
+ print_pid(rd, pid);
+ print_comm(rd, comm, nla_line);
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_PID])
+ free(comm);
+
+ if (rd->json_output)
+ jsonw_end_array(rd->jw);
+ else
+ pr_out("\n");
+ }
+ return MNL_CB_OK;
+}
+
RES_FUNC(res_no_args, RDMA_NLDEV_CMD_RES_GET, NULL, true);
static const struct
@@ -716,12 +844,23 @@ struct filters cm_id_valid_filters[MAX_NUMBER_OF_FILTERS] = {
RES_FUNC(res_cm_id, RDMA_NLDEV_CMD_RES_CM_ID_GET, cm_id_valid_filters, false);
+static const
+struct filters cq_valid_filters[MAX_NUMBER_OF_FILTERS] = {
+ { .name = "link", .is_number = false },
+ { .name = "users", .is_number = true },
+ { .name = "poll-ctx", .is_number = false },
+ { .name = "pid", .is_number = true }
+};
+
+RES_FUNC(res_cq, RDMA_NLDEV_CMD_RES_CQ_GET, cq_valid_filters, true);
+
static int res_show(struct rd *rd)
{
const struct rd_cmd cmds[] = {
{ NULL, res_no_args },
{ "qp", res_qp },
{ "cm_id", res_cm_id },
+ { "cq", res_cq },
{ 0 }
};
diff --git a/rdma/utils.c b/rdma/utils.c
index ec81737..5e79b62 100644
--- a/rdma/utils.c
+++ b/rdma/utils.c
@@ -380,6 +380,11 @@ static const enum mnl_attr_data_type nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
[RDMA_NLDEV_ATTR_RES_PS] = MNL_TYPE_U32,
[RDMA_NLDEV_ATTR_RES_SRC_ADDR] = MNL_TYPE_UNSPEC,
[RDMA_NLDEV_ATTR_RES_DST_ADDR] = MNL_TYPE_UNSPEC,
+ [RDMA_NLDEV_ATTR_RES_CQ] = MNL_TYPE_NESTED,
+ [RDMA_NLDEV_ATTR_RES_CQ_ENTRY] = MNL_TYPE_NESTED,
+ [RDMA_NLDEV_ATTR_RES_CQE] = MNL_TYPE_U32,
+ [RDMA_NLDEV_ATTR_RES_USECNT] = MNL_TYPE_U64,
+ [RDMA_NLDEV_ATTR_RES_POLL_CTX] = MNL_TYPE_U8,
};
int rd_attr_cb(const struct nlattr *attr, void *data)
--
1.8.3.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v3 iproute2-next 7/8] rdma: Add MR resource tracking information
2018-03-26 20:59 [PATCH v3 iproute2-next 0/8] cm_id, cq, mr, and pd resource tracking Steve Wise
` (5 preceding siblings ...)
2018-03-26 20:57 ` [PATCH v3 iproute2-next 6/8] rdma: Add CQ " Steve Wise
@ 2018-03-26 20:57 ` Steve Wise
2018-03-26 20:57 ` [PATCH v3 iproute2-next 8/8] rdma: Add PD " Steve Wise
7 siblings, 0 replies; 11+ messages in thread
From: Steve Wise @ 2018-03-26 20:57 UTC (permalink / raw)
To: dsahern; +Cc: leon, stephen, netdev, linux-rdma
Sample output:
Without CAP_NET_ADMIN:
$ rdma resource show mr mrlen 65536
link mlx4_0/- mrlen 65536 pid 0 comm [nvme_rdma]
link cxgb4_0/- mrlen 65536 pid 0 comm [nvme_rdma]
With CAP_NET_ADMIN:
# rdma resource show mr mrlen 65536
link mlx4_0/- rkey 0x12702 lkey 0x12702 iova 0x85724a000 mrlen 65536 pid 0 comm [nvme_rdma]
link cxgb4_0/- rkey 0x68fe4e9 lkey 0x68fe4e9 iova 0x835b91000 mrlen 65536 pid 0 comm [nvme_rdma]
Signed-off-by: Steve Wise <swise@opengridcomputing.com>
Reviewed-by: Leon Romanovsky <leonro@mellanox.com>
---
include/json_writer.h | 2 +
lib/json_writer.c | 11 +++++
rdma/res.c | 127 ++++++++++++++++++++++++++++++++++++++++++++++++++
rdma/utils.c | 6 +++
4 files changed, 146 insertions(+)
diff --git a/include/json_writer.h b/include/json_writer.h
index 45459fa..4b4dec2 100644
--- a/include/json_writer.h
+++ b/include/json_writer.h
@@ -35,6 +35,7 @@ void jsonw_bool(json_writer_t *self, bool value);
void jsonw_float(json_writer_t *self, double number);
void jsonw_float_fmt(json_writer_t *self, const char *fmt, double num);
void jsonw_uint(json_writer_t *self, uint64_t number);
+void jsonw_xint(json_writer_t *self, uint64_t number);
void jsonw_hu(json_writer_t *self, unsigned short number);
void jsonw_int(json_writer_t *self, int64_t number);
void jsonw_null(json_writer_t *self);
@@ -45,6 +46,7 @@ void jsonw_string_field(json_writer_t *self, const char *prop, const char *val);
void jsonw_bool_field(json_writer_t *self, const char *prop, bool value);
void jsonw_float_field(json_writer_t *self, const char *prop, double num);
void jsonw_uint_field(json_writer_t *self, const char *prop, uint64_t num);
+void jsonw_xint_field(json_writer_t *self, const char *prop, uint64_t num);
void jsonw_hu_field(json_writer_t *self, const char *prop, unsigned short num);
void jsonw_int_field(json_writer_t *self, const char *prop, int64_t num);
void jsonw_null_field(json_writer_t *self, const char *prop);
diff --git a/lib/json_writer.c b/lib/json_writer.c
index 68401ae..0ad0421 100644
--- a/lib/json_writer.c
+++ b/lib/json_writer.c
@@ -225,6 +225,11 @@ void jsonw_uint(json_writer_t *self, uint64_t num)
jsonw_printf(self, "%"PRIu64, num);
}
+void jsonw_xint(json_writer_t *self, uint64_t num)
+{
+ jsonw_printf(self, "%"PRIx64, num);
+}
+
void jsonw_lluint(json_writer_t *self, unsigned long long int num)
{
jsonw_printf(self, "%llu", num);
@@ -269,6 +274,12 @@ void jsonw_uint_field(json_writer_t *self, const char *prop, uint64_t num)
jsonw_uint(self, num);
}
+void jsonw_xint_field(json_writer_t *self, const char *prop, uint64_t num)
+{
+ jsonw_name(self, prop);
+ jsonw_xint(self, num);
+}
+
void jsonw_hu_field(json_writer_t *self, const char *prop, unsigned short num)
{
jsonw_name(self, prop);
diff --git a/rdma/res.c b/rdma/res.c
index f1a90d3..aa61e73 100644
--- a/rdma/res.c
+++ b/rdma/res.c
@@ -802,6 +802,121 @@ static int res_cq_parse_cb(const struct nlmsghdr *nlh, void *data)
return MNL_CB_OK;
}
+static void print_key(struct rd *rd, const char *name, uint32_t val)
+{
+ if (rd->json_output)
+ jsonw_xint_field(rd->jw, name, val);
+ else
+ pr_out("%s 0x%x ", name, val);
+}
+
+static void print_iova(struct rd *rd, uint64_t val)
+{
+ if (rd->json_output)
+ jsonw_xint_field(rd->jw, "iova", val);
+ else
+ pr_out("iova 0x%" PRIx64 " ", val);
+}
+
+static void print_mrlen(struct rd *rd, uint64_t val)
+{
+ if (rd->json_output)
+ jsonw_uint_field(rd->jw, "mrlen", val);
+ else
+ pr_out("mrlen %" PRIu64 " ", val);
+}
+
+static int res_mr_parse_cb(const struct nlmsghdr *nlh, void *data)
+{
+ struct nlattr *tb[RDMA_NLDEV_ATTR_MAX] = {};
+ struct nlattr *nla_table, *nla_entry;
+ struct rd *rd = data;
+ const char *name;
+ uint32_t idx;
+
+ mnl_attr_parse(nlh, 0, rd_attr_cb, tb);
+ if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX] ||
+ !tb[RDMA_NLDEV_ATTR_DEV_NAME] ||
+ !tb[RDMA_NLDEV_ATTR_RES_MR])
+ return MNL_CB_ERROR;
+
+ name = mnl_attr_get_str(tb[RDMA_NLDEV_ATTR_DEV_NAME]);
+ idx = mnl_attr_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
+ nla_table = tb[RDMA_NLDEV_ATTR_RES_MR];
+
+ mnl_attr_for_each_nested(nla_entry, nla_table) {
+ struct nlattr *nla_line[RDMA_NLDEV_ATTR_MAX] = {};
+ uint32_t rkey = 0, lkey = 0;
+ uint64_t iova = 0, mrlen;
+ char *comm = NULL;
+ uint32_t pid = 0;
+ int err;
+
+ err = mnl_attr_parse_nested(nla_entry, rd_attr_cb, nla_line);
+ if (err != MNL_CB_OK)
+ return MNL_CB_ERROR;
+
+ if (!nla_line[RDMA_NLDEV_ATTR_RES_MRLEN] ||
+ (!nla_line[RDMA_NLDEV_ATTR_RES_PID] &&
+ !nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME])) {
+ return MNL_CB_ERROR;
+ }
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_RKEY])
+ rkey = mnl_attr_get_u32(
+ nla_line[RDMA_NLDEV_ATTR_RES_RKEY]);
+ if (nla_line[RDMA_NLDEV_ATTR_RES_LKEY])
+ lkey = mnl_attr_get_u32(
+ nla_line[RDMA_NLDEV_ATTR_RES_LKEY]);
+ if (nla_line[RDMA_NLDEV_ATTR_RES_IOVA])
+ iova = mnl_attr_get_u64(
+ nla_line[RDMA_NLDEV_ATTR_RES_IOVA]);
+
+ mrlen = mnl_attr_get_u64(nla_line[RDMA_NLDEV_ATTR_RES_MRLEN]);
+ if (rd_check_is_filtered(rd, "mrlen", mrlen))
+ continue;
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_PID]) {
+ pid = mnl_attr_get_u32(
+ nla_line[RDMA_NLDEV_ATTR_RES_PID]);
+ comm = get_task_name(pid);
+ }
+
+ if (rd_check_is_filtered(rd, "pid", pid)) {
+ free(comm);
+ continue;
+ }
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME])
+ /* discard const from mnl_attr_get_str */
+ comm = (char *)mnl_attr_get_str(
+ nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME]);
+
+ if (rd->json_output)
+ jsonw_start_array(rd->jw);
+
+ print_link(rd, idx, name, 0, nla_line);
+ if (nla_line[RDMA_NLDEV_ATTR_RES_RKEY])
+ print_key(rd, "rkey", rkey);
+ if (nla_line[RDMA_NLDEV_ATTR_RES_LKEY])
+ print_key(rd, "lkey", lkey);
+ if (nla_line[RDMA_NLDEV_ATTR_RES_IOVA])
+ print_iova(rd, iova);
+ print_mrlen(rd, mrlen);
+ print_pid(rd, pid);
+ print_comm(rd, comm, nla_line);
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_PID])
+ free(comm);
+
+ if (rd->json_output)
+ jsonw_end_array(rd->jw);
+ else
+ pr_out("\n");
+ }
+ return MNL_CB_OK;
+}
+
RES_FUNC(res_no_args, RDMA_NLDEV_CMD_RES_GET, NULL, true);
static const struct
@@ -854,6 +969,17 @@ struct filters cq_valid_filters[MAX_NUMBER_OF_FILTERS] = {
RES_FUNC(res_cq, RDMA_NLDEV_CMD_RES_CQ_GET, cq_valid_filters, true);
+static const
+struct filters mr_valid_filters[MAX_NUMBER_OF_FILTERS] = {
+ { .name = "link", .is_number = false },
+ { .name = "rkey", .is_number = true },
+ { .name = "lkey", .is_number = true },
+ { .name = "mrlen", .is_number = true },
+ { .name = "pid", .is_number = true }
+};
+
+RES_FUNC(res_mr, RDMA_NLDEV_CMD_RES_MR_GET, mr_valid_filters, true);
+
static int res_show(struct rd *rd)
{
const struct rd_cmd cmds[] = {
@@ -861,6 +987,7 @@ static int res_show(struct rd *rd)
{ "qp", res_qp },
{ "cm_id", res_cm_id },
{ "cq", res_cq },
+ { "mr", res_mr },
{ 0 }
};
diff --git a/rdma/utils.c b/rdma/utils.c
index 5e79b62..a2e08e9 100644
--- a/rdma/utils.c
+++ b/rdma/utils.c
@@ -385,6 +385,12 @@ static const enum mnl_attr_data_type nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
[RDMA_NLDEV_ATTR_RES_CQE] = MNL_TYPE_U32,
[RDMA_NLDEV_ATTR_RES_USECNT] = MNL_TYPE_U64,
[RDMA_NLDEV_ATTR_RES_POLL_CTX] = MNL_TYPE_U8,
+ [RDMA_NLDEV_ATTR_RES_MR] = MNL_TYPE_NESTED,
+ [RDMA_NLDEV_ATTR_RES_MR_ENTRY] = MNL_TYPE_NESTED,
+ [RDMA_NLDEV_ATTR_RES_RKEY] = MNL_TYPE_U32,
+ [RDMA_NLDEV_ATTR_RES_LKEY] = MNL_TYPE_U32,
+ [RDMA_NLDEV_ATTR_RES_IOVA] = MNL_TYPE_U64,
+ [RDMA_NLDEV_ATTR_RES_MRLEN] = MNL_TYPE_U64,
};
int rd_attr_cb(const struct nlattr *attr, void *data)
--
1.8.3.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v3 iproute2-next 8/8] rdma: Add PD resource tracking information
2018-03-26 20:59 [PATCH v3 iproute2-next 0/8] cm_id, cq, mr, and pd resource tracking Steve Wise
` (6 preceding siblings ...)
2018-03-26 20:57 ` [PATCH v3 iproute2-next 7/8] rdma: Add MR " Steve Wise
@ 2018-03-26 20:57 ` Steve Wise
7 siblings, 0 replies; 11+ messages in thread
From: Steve Wise @ 2018-03-26 20:57 UTC (permalink / raw)
To: dsahern; +Cc: leon, stephen, netdev, linux-rdma
Sample output:
Without CAP_NET_ADMIN capability:
link mlx4_0/- users 0 pid 0 comm [ib_srpt]
link mlx4_0/- users 0 pid 0 comm [ib_srp]
link mlx4_0/- users 1 pid 0 comm [ib_core]
link cxgb4_0/- users 0 pid 0 comm [ib_srp]
With CAP_NET_ADMIN capability:
link mlx4_0/- local_dma_lkey 0x8000 users 0 pid 0 comm [ib_srpt]
link mlx4_0/- local_dma_lkey 0x8000 users 0 pid 0 comm [ib_srp]
link mlx4_0/- local_dma_lkey 0x8000 users 1 pid 0 comm [ib_core]
link cxgb4_0/- local_dma_lkey 0x0 users 0 pid 0 comm [ib_srp]
Signed-off-by: Steve Wise <swise@opengridcomputing.com>
Reviewed-by: Leon Romanovsky <leonro@mellanox.com>
---
rdma/res.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 95 insertions(+)
diff --git a/rdma/res.c b/rdma/res.c
index aa61e73..6177e46 100644
--- a/rdma/res.c
+++ b/rdma/res.c
@@ -917,6 +917,91 @@ static int res_mr_parse_cb(const struct nlmsghdr *nlh, void *data)
return MNL_CB_OK;
}
+static int res_pd_parse_cb(const struct nlmsghdr *nlh, void *data)
+{
+ struct nlattr *tb[RDMA_NLDEV_ATTR_MAX] = {};
+ struct nlattr *nla_table, *nla_entry;
+ struct rd *rd = data;
+ const char *name;
+ uint32_t idx;
+
+ mnl_attr_parse(nlh, 0, rd_attr_cb, tb);
+ if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX] ||
+ !tb[RDMA_NLDEV_ATTR_DEV_NAME] ||
+ !tb[RDMA_NLDEV_ATTR_RES_PD])
+ return MNL_CB_ERROR;
+
+ name = mnl_attr_get_str(tb[RDMA_NLDEV_ATTR_DEV_NAME]);
+ idx = mnl_attr_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
+ nla_table = tb[RDMA_NLDEV_ATTR_RES_PD];
+
+ mnl_attr_for_each_nested(nla_entry, nla_table) {
+ uint32_t local_dma_lkey = 0, unsafe_global_rkey = 0;
+ struct nlattr *nla_line[RDMA_NLDEV_ATTR_MAX] = {};
+ char *comm = NULL;
+ uint32_t pid = 0;
+ uint64_t users;
+ int err;
+
+ err = mnl_attr_parse_nested(nla_entry, rd_attr_cb, nla_line);
+ if (err != MNL_CB_OK)
+ return MNL_CB_ERROR;
+
+ if (!nla_line[RDMA_NLDEV_ATTR_RES_USECNT] ||
+ (!nla_line[RDMA_NLDEV_ATTR_RES_PID] &&
+ !nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME])) {
+ return MNL_CB_ERROR;
+ }
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY])
+ local_dma_lkey = mnl_attr_get_u32(
+ nla_line[RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY]);
+
+ users = mnl_attr_get_u64(nla_line[RDMA_NLDEV_ATTR_RES_USECNT]);
+ if (rd_check_is_filtered(rd, "users", users))
+ continue;
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY])
+ unsafe_global_rkey = mnl_attr_get_u32(
+ nla_line[RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY]);
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_PID]) {
+ pid = mnl_attr_get_u32(
+ nla_line[RDMA_NLDEV_ATTR_RES_PID]);
+ comm = get_task_name(pid);
+ }
+
+ if (rd_check_is_filtered(rd, "pid", pid))
+ continue;
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME])
+ /* discard const from mnl_attr_get_str */
+ comm = (char *)mnl_attr_get_str(
+ nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME]);
+
+ if (rd->json_output)
+ jsonw_start_array(rd->jw);
+
+ print_link(rd, idx, name, 0, nla_line);
+ if (nla_line[RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY])
+ print_key(rd, "local_dma_lkey", local_dma_lkey);
+ print_users(rd, users);
+ if (nla_line[RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY])
+ print_key(rd, "unsafe_global_rkey", unsafe_global_rkey);
+ print_pid(rd, pid);
+ print_comm(rd, comm, nla_line);
+
+ if (nla_line[RDMA_NLDEV_ATTR_RES_PID])
+ free(comm);
+
+ if (rd->json_output)
+ jsonw_end_array(rd->jw);
+ else
+ pr_out("\n");
+ }
+ return MNL_CB_OK;
+}
+
RES_FUNC(res_no_args, RDMA_NLDEV_CMD_RES_GET, NULL, true);
static const struct
@@ -980,6 +1065,15 @@ struct filters mr_valid_filters[MAX_NUMBER_OF_FILTERS] = {
RES_FUNC(res_mr, RDMA_NLDEV_CMD_RES_MR_GET, mr_valid_filters, true);
+static const
+struct filters pd_valid_filters[MAX_NUMBER_OF_FILTERS] = {
+ { .name = "link", .is_number = false },
+ { .name = "users", .is_number = true },
+ { .name = "pid", .is_number = true }
+};
+
+RES_FUNC(res_pd, RDMA_NLDEV_CMD_RES_PD_GET, pd_valid_filters, true);
+
static int res_show(struct rd *rd)
{
const struct rd_cmd cmds[] = {
@@ -988,6 +1082,7 @@ static int res_show(struct rd *rd)
{ "cm_id", res_cm_id },
{ "cq", res_cq },
{ "mr", res_mr },
+ { "pd", res_pd },
{ 0 }
};
--
1.8.3.1
^ permalink raw reply related [flat|nested] 11+ messages in thread