* [PATCH 36/37] librdmacm: check if kernel supports AF_IB
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Add check during initialization to determine if the kernel
supports AF_IB.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/cma.c | 27 +++++++++++++++++++++++++++
1 files changed, 27 insertions(+), 0 deletions(-)
diff --git a/src/cma.c b/src/cma.c
index faf4264..66a7643 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -195,6 +195,32 @@ static int check_abi_version(void)
return 0;
}
+/*
+ * This function is called holding the mutex lock
+ * cma_dev_cnt must be set before calling this function to
+ * ensure that the lock is not acquired recursively.
+ */
+static void ucma_set_af_ib_support(void)
+{
+ struct rdma_cm_id *id;
+ struct sockaddr_ib sib;
+ int ret;
+
+ ret = rdma_create_id(NULL, &id, NULL, RDMA_PS_IB);
+ if (ret)
+ return;
+
+ memset(&sib, 0, sizeof sib);
+ sib.sib_family = AF_IB;
+ sib.sib_sid = htonll(RDMA_IB_IP_PS_TCP);
+ sib.sib_sid_mask = htonll(RDMA_IB_IP_PS_MASK);
+ af_ib_support = 1;
+ ret = rdma_bind_addr(id, (struct sockaddr *) &sib);
+ af_ib_support = !ret;
+
+ rdma_destroy_id(id);
+}
+
int ucma_init(void)
{
struct ibv_device **dev_list = NULL;
@@ -263,6 +289,7 @@ int ucma_init(void)
if (ib)
ucma_ib_init();
cma_dev_cnt = dev_cnt;
+ ucma_set_af_ib_support();
pthread_mutex_unlock(&mut);
ibv_free_device_list(dev_list);
return 0;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 37/37] librdmacm: use IB ACM to resolve IB path
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Starting with 2.6.33, the kernel supports the ability to
manually specify the path record that a connection should
use. Allow the librdmacm to contact the IB ACM to acquire
path record data, even if rdma_getaddrinfo is not used and
the kernel does not support AF_IB.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/acm.c | 2 +-
src/cma.c | 37 +++++++++++++++++++++++++++++++++++--
src/cma.h | 2 ++
3 files changed, 38 insertions(+), 3 deletions(-)
diff --git a/src/acm.c b/src/acm.c
index 8975b97..ca338bc 100644
--- a/src/acm.c
+++ b/src/acm.c
@@ -205,7 +205,7 @@ static void ucma_ib_save_resp(struct rdma_addrinfo *rai, struct acm_resolve_msg
rai->ai_route = path_data;
rai->ai_route_len = len;
- if (af_ib_support) {
+ if (af_ib_support && !(rai->ai_flags & RAI_ROUTEONLY)) {
ucma_ib_format_connect(rai);
if (ucma_ib_convert_addr(rai, pri_path) &&
rai->ai_connect) {
diff --git a/src/cma.c b/src/cma.c
index 66a7643..258a324 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -858,15 +858,47 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
return ucma_complete(id_priv);
}
+static int ucma_set_ib_route(struct rdma_cm_id *id)
+{
+ struct rdma_addrinfo rai;
+ struct sockaddr_in6 src, dst;
+ int size, ret;
+
+ memset(&rai, 0, sizeof rai);
+ rai.ai_flags = RAI_ROUTEONLY;
+ rai.ai_family = id->route.addr.src_addr.sa_family;
+ size = ucma_addrlen((struct sockaddr *) &id->route.addr.src_addr);
+
+ memcpy(&src, &id->route.addr.src_addr, size);
+ memcpy(&dst, &id->route.addr.dst_addr, size);
+ rai.ai_src_addr = (struct sockaddr *) &src;
+ rai.ai_dst_addr = (struct sockaddr *) &dst;
+
+ ucma_ib_resolve(&rai);
+ if (!rai.ai_route_len)
+ return ERR(ENODATA);
+
+ ret = rdma_set_option(id, RDMA_OPTION_IB, RDMA_OPTION_IB_PATH,
+ rai.ai_route, rai.ai_route_len);
+ free(rai.ai_route);
+ return ret;
+}
+
int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
{
struct ucma_abi_resolve_route *cmd;
struct cma_id_private *id_priv;
void *msg;
int ret, size;
-
- CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_RESOLVE_ROUTE, size);
+
id_priv = container_of(id, struct cma_id_private, id);
+ if (id->verbs->device->transport_type == IBV_TRANSPORT_IB) {
+ ret = ucma_set_ib_route(id);
+ if (!ret)
+ goto out;
+ }
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_RESOLVE_ROUTE, size);
cmd->id = id_priv->handle;
cmd->timeout_ms = timeout_ms;
@@ -874,6 +906,7 @@ int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
if (ret != size)
return (ret >= 0) ? ERR(ENODATA) : -1;
+out:
return ucma_complete(id_priv);
}
diff --git a/src/cma.h b/src/cma.h
index 1e0c571..d25cdea 100644
--- a/src/cma.h
+++ b/src/cma.h
@@ -87,6 +87,8 @@ static inline void *zalloc(size_t size)
int ucma_init();
extern int af_ib_support;
+#define RAI_ROUTEONLY 0x01000000
+
#ifdef USE_IB_ACM
void ucma_ib_init();
void ucma_ib_cleanup();
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 34/37] librdmacm: update man pages
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Update man pages to reflect changes to the APIs.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
man/rdma_cm.7 | 56 +++++++++++++++++++++++++++++++++++++++++------
man/rdma_create_ep.3 | 57 ++++++++++++++++++++++++++++++++++++++++++++++++
man/rdma_create_id.3 | 14 ++++++++----
man/rdma_create_qp.3 | 15 +++++++++++--
man/rdma_get_request.3 | 31 ++++++++++++++++++++++++++
man/rdma_migrate_id.3 | 6 ++++-
6 files changed, 165 insertions(+), 14 deletions(-)
diff --git a/man/rdma_cm.7 b/man/rdma_cm.7
index fd04959..ff5d489 100644
--- a/man/rdma_cm.7
+++ b/man/rdma_cm.7
@@ -8,17 +8,59 @@ Used to establish communication over RDMA transports.
.SH "NOTES"
The RDMA CM is a communication manager used to setup reliable, connected
and unreliable datagram data transfers. It provides an RDMA transport
-neutral interface for establishing connections. The API is based on sockets,
-but adapted for queue pair (QP) based semantics: communication must be
-over a specific RDMA device, and data transfers are message based.
+neutral interface for establishing connections. The API concepts are
+is based on sockets, but adapted for queue pair (QP) based semantics:
+communication must be over a specific RDMA device, and data transfers
+are message based.
.P
-The RDMA CM only provides the communication management (connection setup /
-teardown) portion of an RDMA API. It works in conjunction with the verbs
+The RDMA CM can control both the QP and communication management (connection setup /
+teardown) portions of an RDMA API, or only the communication management
+piece. It works in conjunction with the verbs
API defined by the libibverbs library. The libibverbs library provides the
-interfaces needed to send and receive data.
+underlying interfaces needed to send and receive data.
+.P
+The RDMA CM can operate asynchronously or synchronously. The mode of
+operation is controlled by the user through the use of the rdma_cm event channel
+parameter in specific calls. If an event channel is provided, an rdma_cm identifier
+will report its event data (results of connecting, for example), on that channel.
+If a channel is not provided, then all rdma_cm operations for the selected
+rdma_cm identifier are will block until they complete.
+.SH "RDMA VERBS"
+The rdma_cm supports the full range of verbs available through the libibverbs
+library and interfaces. However, it also provides wrapper functions for some
+of the more commonly used verbs funcationality. The full set of abstracted
+verb calls are:
+.P rdma_reg_msgs - register an array of buffers for sending and receiving
+.P rdma_reg_read - registers a buffer for RDMA read operations
+.P rdma_reg_write - registers a buffer for RDMA write operations
+.P rdma_dereg_mr - deregisters a memory region
+.P
+.P rdma_post_recv - post a buffer to receive a message
+.P rdma_post_send - post a buffer to send a message
+.P rdma_post_read - post an RDMA to read data into a buffer
+.P rdma_post_write - post an RDMA to send data from a buffer
+.P
+.P rdma_post_recvv - post a vector of buffers to receive a message
+.P rdma_post_sendv - post a vector of buffers to send a message
+.P rdma_post_readv - post a vector of buffers to receive an RDMA read
+.P rdma_post_writev - post a vector of buffers to send an RDMA write
+.P
+.P rdma_post_ud_send - post a buffer to send a message on a UD QP
+.P
+.P rdma_get_send_comp - get completion status for a send or RDMA operation
+.P rdma_get_recv_comp - get information about a completed receive
.SH "CLIENT OPERATION"
This section provides a general overview of the basic operation for the active,
-or client, side of communication. A general connection flow would be:
+or client, side of communication. This flow assume asynchronous operation with
+low level call details shown. For
+synchronous operation, calls to rdma_create_event_channel, rdma_get_cm_event,
+rdma_ack_cm_event, and rdma_destroy_event_channel
+would be eliminated. Abstracted calls, such as rdma_create_ep encapsulate
+serveral of these calls under a single API.
+Users may also refer to the example applications for
+code samples. A general connection flow would be:
+.IP rdma_getaddrinfo
+retrieve address information of the destination
.IP rdma_create_event_channel
create channel to receive events
.IP rdma_create_id
diff --git a/man/rdma_create_ep.3 b/man/rdma_create_ep.3
new file mode 100644
index 0000000..ae07113
--- /dev/null
+++ b/man/rdma_create_ep.3
@@ -0,0 +1,57 @@
+.TH "RDMA_CREATE_EP" 3 "2007-08-06" "librdmacm" "Librdmacm Programmer's Manual" librdmacm
+.SH NAME
+rdma_create_ep \- Allocate a communication identifier and optional QP.
+.SH SYNOPSIS
+.B "#include <rdma/rdma_cma.h>"
+.P
+.B "int" rdma_create_ep
+.BI "(struct rdma_cm_id **" id ","
+.BI "struct rdma_addrinfo *" res ","
+.BI "struct ibv_pd *" pd ","
+.BI "struct ibv_qp_init_attr *" qp_init_attr ");"
+.SH ARGUMENTS
+.IP "id" 12
+A reference where the allocated communication identifier will be
+returned.
+.IP "res" 12
+Address information associated with the rdma_cm_id returned from
+rdma_getaddrinfo.
+.IP "pd" 12
+Optional protection domain if a QP is associated with the rdma_cm_id.
+.IP "qp_init_attr" 12
+Optional initial QP attributes.
+.SH "DESCRIPTION"
+Creates an identifier that is used to track communication information.
+.SH "NOTES"
+After resolving address information using rdma_getaddrinfo, a user
+may use this call to allocate an rdma_cm_id based on the results.
+.P
+If the rdma_cm_id will be used on the active side of a connection,
+meaning that res->ai_flag is not RAI_PASSIVE set, rdma_create_ep
+will automatically create a QP on the rdma_cm_id if qp_init_attr is
+not NULL. The QP will be associated with the specified protection
+domain, if provided, or a default protection domain if not. Users
+should see rdma_create_qp for details on the use of the pd and
+qp_init_attr parameters. After calling rdma_create_ep, the returned
+rdma_cm_id may be connected by calling rdma_connect. The active side
+calls rdma_resolve_addr and rdma_resolve_route are not necessary.
+.P
+If the rdma_cm_id will be used on the passive side of a connection,
+indicated by having res->ai_flag RAI_PASSIVE set, this call will save
+the provided pd and qp_init_attr parameters. When a new connection
+request is retrieved by calling rdma_get_request, the rdma_cm_id
+associated with the new connection will automatically be associated
+with a QP using the pd and qp_init_attr parameters. After calling
+rdma_create_ep, the returned rdma_cm_id may be placed into a listening
+state by immediately calling rdma_listen. The passive side call
+rdma_bind_addr is not necessary. Connection requests may then be
+retrieved by calling rdma_get_request.
+.P
+The newly created rdma_cm_id will be set to use synchronous operation.
+Users that wish asynchronous operation must migrate the rdma_cm_id
+to a user created event channel using rdma_migrate_id.
+.P
+Users must release the created rdma_cm_id by calling rdma_destroy_ep.
+.SH "SEE ALSO"
+rdma_cm(7), rdma_getaddrinfo(3), rdma_create_event_channel(3),
+rdma_connect(3), rdma_listen(3), rdma_destroy_ep(3), rdma_migrate_id(3)
diff --git a/man/rdma_create_id.3 b/man/rdma_create_id.3
index eb29f3c..a7926d4 100644
--- a/man/rdma_create_id.3
+++ b/man/rdma_create_id.3
@@ -12,7 +12,7 @@ rdma_create_id \- Allocate a communication identifier.
.SH ARGUMENTS
.IP "channel" 12
The communication channel that events associated with the
-allocated rdma_cm_id will be reported on.
+allocated rdma_cm_id will be reported on. This may be NULL.
.IP "id" 12
A reference where the allocated communication identifier will be
returned.
@@ -26,9 +26,15 @@ Creates an identifier that is used to track communication information.
Rdma_cm_id's are conceptually equivalent to a socket for RDMA
communication. The difference is that RDMA communication requires
explicitly binding to a specified RDMA device before communication
-can occur, and most operations are asynchronous in nature. Communication
-events on an rdma_cm_id are reported through the associated event
-channel. Users must release the rdma_cm_id by calling rdma_destroy_id.
+can occur, and most operations are asynchronous in nature. Asynchronous
+communication events on an rdma_cm_id are reported through the associated
+event channel. If the channel paramet is NULL, the rdma_cm_id will
+be placed into synchronous operation. While operating synchronously,
+calls that result in an event will block until the operation completes.
+The event will be returned to the user through the rdma_cm_id structure,
+and be available for access until another rdma_cm call is made.
+.P
+Users must release the rdma_cm_id by calling rdma_destroy_id.
.SH "PORT SPACE"
Details of the services provided by the different port spaces are outlined
below.
diff --git a/man/rdma_create_qp.3 b/man/rdma_create_qp.3
index 0931499..ce1d862 100644
--- a/man/rdma_create_qp.3
+++ b/man/rdma_create_qp.3
@@ -12,9 +12,9 @@ rdma_create_qp \- Allocate a QP.
.IP "id" 12
RDMA identifier.
.IP "pd" 12
-protection domain for the QP.
+Optional protection domain for the QP.
.IP "qp_init_attr" 12
-initial QP attributes.
+Initial QP attributes.
.SH "DESCRIPTION"
Allocate a QP associated with the specified rdma_cm_id and transition it
for sending and receiving.
@@ -25,6 +25,17 @@ QPs allocated to an rdma_cm_id are automatically transitioned by the
librdmacm through their states. After being allocated, the QP will be
ready to handle posting of receives. If the QP is unconnected, it will
be ready to post sends.
+.P
+If a protection domain is not given - pd parameter is NULL - then
+the rdma_cm_id will be created using a default protection domain. One
+default protection domain is allocated per RDMA device.
+.P
+The initial QP attributes are specified by the qp_init_attr parameter. The
+send_cq and recv_cq fields in the ibv_qp_init_attr are optional. If
+a send or receive completion queue is not specified, then a CQ will be
+allocated by the rdma_cm for the QP, along with corresponding completion
+channels. Completion channels and CQ data created by the rdma_cm are
+exposed to the user through the rdma_cm_id structure.
.SH "SEE ALSO"
rdma_bind_addr(3), rdma_resolve_addr(3), rdma_destroy_qp(3), ibv_create_qp(3),
ibv_modify_qp(3)
diff --git a/man/rdma_get_request.3 b/man/rdma_get_request.3
new file mode 100644
index 0000000..cd48918
--- /dev/null
+++ b/man/rdma_get_request.3
@@ -0,0 +1,31 @@
+.TH "RDMA_GET_REQUEST" 3 "2007-10-31" "librdmacm" "Librdmacm Programmer's Manual" librdmacm
+.SH NAME
+rdma_get_request \- Retrieves the next pending connection request event.
+.SH SYNOPSIS
+.B "#include <rdma/rdma_cma.h>"
+.P
+.B "int" rdma_get_request
+.BI "(struct rdma_cm_id *" listen ","
+.BI "struct rdma_cm_id **" id ");"
+.SH ARGUMENTS
+.IP "listen" 12
+Listening rdma_cm_id.
+.IP "id" 12
+rdma_cm_id associated with the new connection.
+.SH "DESCRIPTION"
+Retrieves a connection request event. If no requests are pending,
+the call will block until an event is received.
+.SH "NOTES"
+This call may only be used on listening rdma_cm_id's operating
+synchronously. On success, a new rdma_cm_id representing the
+connection request will be returned to the user. The new rdma_cm_id
+will reference event information associated with the request until
+the user calls rdma_reject, rdma_accept, or rdma_destroy_id on the
+newly created identifier. For a description of the event data,
+see rdma_get_cm_event.
+.P
+If QP attributes are associated with the listening endpoint, the
+returned rdma_cm_id will also reference an allocated QP.
+.SH "SEE ALSO"
+rdma_get_cm_event(3), rdma_accept(3), rdma_reject(3),
+rdma_connect(3), rdma_listen(3), rdma_destroy_id(3)
diff --git a/man/rdma_migrate_id.3 b/man/rdma_migrate_id.3
index ffbad45..64448f6 100644
--- a/man/rdma_migrate_id.3
+++ b/man/rdma_migrate_id.3
@@ -12,7 +12,7 @@ rdma_migrate_id \- Move a communication identifer to a different event channel.
An existing communication identifier to migrate.
.IP "channel" 12
The communication channel that events associated with the
-allocated rdma_cm_id will be reported on.
+allocated rdma_cm_id will be reported on. May be NULL.
.SH "DESCRIPTION"
Migrates a communication identifier to a different event channel.
.SH "NOTES"
@@ -22,6 +22,10 @@ to the new channel. Users should not poll for events on the
rdma_cm_id's current event channel or invoke other routines on the
rdma_cm_id while migrating between channels. This call will block while
there are any unacknowledged events on the current event channel.
+.P
+If the channel parameter is NULL, the specified rdma_cm_id will be
+placed into synchronous operation mode. All calls on the id
+will block until the operation completes.
.SH "SEE ALSO"
rdma_cm(7), rdma_create_event_channel(3), rdma_create_id(3),
rdma_get_cm_event(3)
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 27/37] librdmacm: add support for IB ACM service
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Allow the librdmacm to contact a service via sockets to obtain
address mapping and path record data. The use of the service
is controlled through a build option (with-ib_acm). If the
library fails to contact the service, it falls back to using
the kernel services to resolve address and routing data.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Once IB ACM is proven, the build option can be removed.
Makefile.am | 2 -
configure.in | 14 +++++
src/acm.c | 160 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
src/addrinfo.c | 3 +
src/cma.c | 9 ++-
src/cma.h | 13 ++++-
6 files changed, 197 insertions(+), 4 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index be53c78..8d86045 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -12,7 +12,7 @@ else
librdmacm_version_script =
endif
-src_librdmacm_la_SOURCES = src/cma.c src/addrinfo.c
+src_librdmacm_la_SOURCES = src/cma.c src/addrinfo.c src/acm.c
src_librdmacm_la_LDFLAGS = -version-info 1 -export-dynamic \
$(librdmacm_version_script)
src_librdmacm_la_DEPENDENCIES = $(srcdir)/src/librdmacm.map
diff --git a/configure.in b/configure.in
index 1122966..3db4247 100644
--- a/configure.in
+++ b/configure.in
@@ -21,6 +21,15 @@ if test "$with_valgrind" != "" && test "$with_valgrind" != "no"; then
fi
fi
+AC_ARG_WITH([ib_acm],
+ AC_HELP_STRING([--with-ib_acm],
+ [Use IB ACM for route resolution - default NO]))
+
+if test "$with_ib_acm" != "" && test "$with_ib_acm" != "no"; then
+ AC_DEFINE([USE_IB_ACM], 1,
+ [Define to 1 to use IB ACM for endpoint resolution])
+fi
+
AC_ARG_ENABLE(libcheck, [ --disable-libcheck do not test for presence of ib libraries],
[ if test "$enableval" = "no"; then
disable_libcheck=yes
@@ -51,6 +60,11 @@ AC_CHECK_HEADER(valgrind/memcheck.h, [],
AC_MSG_ERROR([valgrind requested but <valgrind/memcheck.h> not found.]))
fi
+if test "$with_ib_acm" != "" && test "$with_ib_acm" != "no"; then
+AC_CHECK_HEADER(infiniband/acm.h, [],
+ AC_MSG_ERROR([IB ACM requested but <infiniband/acm.h> not found.]))
+fi
+
fi
AC_CACHE_CHECK(whether ld accepts --version-script, ac_cv_version_script,
diff --git a/src/acm.c b/src/acm.c
new file mode 100644
index 0000000..34fdf3c
--- /dev/null
+++ b/src/acm.c
@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <unistd.h>
+
+#include "cma.h"
+#include <rdma/rdma_cma.h>
+#include <infiniband/ib.h>
+#include <infiniband/sa.h>
+
+#ifdef USE_IB_ACM
+#include <infiniband/acm.h>
+
+static pthread_mutex_t acm_lock = PTHREAD_MUTEX_INITIALIZER;
+static int sock;
+static short server_port = 6125;
+
+void ucma_ib_init(void)
+{
+ struct sockaddr_in addr;
+ int ret;
+
+ sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+ if (sock < 0)
+ return;
+
+ memset(&addr, 0, sizeof addr);
+ addr.sin_family = AF_INET;
+ addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+ addr.sin_port = htons(server_port);
+ ret = connect(sock, (struct sockaddr *) &addr, sizeof(addr));
+ if (ret)
+ goto err;
+
+ return;
+
+err:
+ close(sock);
+ sock = 0;
+}
+
+void ucma_ib_cleanup(void)
+{
+ if (sock > 0) {
+ shutdown(sock, SHUT_RDWR);
+ close(sock);
+ }
+}
+
+static void ucma_ib_save_resp(struct rdma_addrinfo *rai, struct acm_resolve_msg *msg)
+{
+ struct ib_path_data *path_data = NULL;
+ int len, i, cnt;
+
+ len = msg->hdr.length - ACM_MSG_HDR_LENGTH;
+ cnt = len / sizeof(struct acm_ep_addr_data);
+ path_data = malloc(len);
+ if (!path_data)
+ return;
+
+ memcpy(path_data, msg->data, len);
+ for (i = 0; i < cnt; i++) {
+ if (msg->data[i].type != ACM_EP_INFO_PATH)
+ goto err;
+ path_data[i].reserved = 0;
+ }
+
+ rai->ai_route = path_data;
+ rai->ai_route_len = len;
+ return;
+err:
+ free(path_data);
+}
+
+void ucma_ib_resolve(struct rdma_addrinfo *rai)
+{
+ struct acm_msg msg;
+ struct acm_resolve_msg *resolve_msg = (struct acm_resolve_msg *) &msg;
+ struct acm_ep_addr_data *src_data, *dst_data;
+ int ret;
+
+ if (sock <= 0)
+ return;
+
+ memset(&msg, 0, sizeof msg);
+ msg.hdr.version = ACM_VERSION;
+ msg.hdr.opcode = ACM_OP_RESOLVE;
+ msg.hdr.length = ACM_MSG_HDR_LENGTH + (2 * ACM_MSG_EP_LENGTH);
+
+ src_data = &resolve_msg->data[0];
+ dst_data = &resolve_msg->data[1];
+
+ src_data->flags = ACM_EP_FLAG_SOURCE;
+ dst_data->flags = ACM_EP_FLAG_DEST;
+ if (rai->ai_family == AF_INET) {
+ src_data->type = dst_data->type = ACM_EP_INFO_ADDRESS_IP;
+ memcpy(src_data->info.addr,
+ &((struct sockaddr_in *) rai->ai_src_addr)->sin_addr, 4);
+ memcpy(dst_data->info.addr,
+ &((struct sockaddr_in *) rai->ai_dst_addr)->sin_addr, 4);
+ } else {
+ src_data->type = dst_data->type = ACM_EP_INFO_ADDRESS_IP6;
+ memcpy(src_data->info.addr,
+ &((struct sockaddr_in6 *) rai->ai_src_addr)->sin6_addr, 16);
+ memcpy(dst_data->info.addr,
+ &((struct sockaddr_in6 *) rai->ai_dst_addr)->sin6_addr, 16);
+ }
+
+ pthread_mutex_lock(&acm_lock);
+ ret = send(sock, (char *) &msg, msg.hdr.length, 0);
+ if (ret != msg.hdr.length) {
+ pthread_mutex_unlock(&acm_lock);
+ return;
+ }
+
+ ret = recv(sock, (char *) &msg, sizeof msg, 0);
+ pthread_mutex_unlock(&acm_lock);
+ if (ret < ACM_MSG_HDR_LENGTH || ret != msg.hdr.length || msg.hdr.status)
+ return;
+
+ ucma_ib_save_resp(rai, resolve_msg);
+}
+
+#endif /* USE_IB_ACM */
diff --git a/src/addrinfo.c b/src/addrinfo.c
index dfaf9d5..dad9b82 100644
--- a/src/addrinfo.c
+++ b/src/addrinfo.c
@@ -219,6 +219,9 @@ int rdma_getaddrinfo(char *node, char *service,
}
}
+ if (!(rai->ai_flags & RAI_PASSIVE))
+ ucma_ib_resolve(rai);
+
freeaddrinfo(ai);
*res = rai;
return 0;
diff --git a/src/cma.c b/src/cma.c
index 1cc4f1f..6ac949a 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -149,6 +149,8 @@ int af_ib_support;
static void ucma_cleanup(void)
{
+ ucma_ib_cleanup();
+
if (cma_dev_cnt) {
while (cma_dev_cnt--) {
ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd);
@@ -196,7 +198,7 @@ int ucma_init(void)
struct ibv_device **dev_list = NULL;
struct cma_device *cma_dev;
struct ibv_device_attr attr;
- int i, ret, dev_cnt;
+ int i, ret, dev_cnt, ib;
/* Quick check without lock to see if we're already initialized */
if (cma_dev_cnt)
@@ -225,7 +227,7 @@ int ucma_init(void)
goto err2;
}
- for (i = 0; dev_list[i];) {
+ for (i = 0, ib = 0; dev_list[i];) {
cma_dev = &cma_dev_array[i];
cma_dev->guid = ibv_get_device_guid(dev_list[i]);
@@ -253,8 +255,11 @@ int ucma_init(void)
cma_dev->port_cnt = attr.phys_port_cnt;
cma_dev->max_initiator_depth = (uint8_t) attr.max_qp_init_rd_atom;
cma_dev->max_responder_resources = (uint8_t) attr.max_qp_rd_atom;
+ ib += (cma_dev->verbs->device->transport_type == IBV_TRANSPORT_IB);
}
+ if (ib)
+ ucma_ib_init();
cma_dev_cnt = dev_cnt;
pthread_mutex_unlock(&mut);
ibv_free_device_list(dev_list);
diff --git a/src/cma.h b/src/cma.h
index 06ca38c..62785de 100644
--- a/src/cma.h
+++ b/src/cma.h
@@ -44,6 +44,8 @@
#include <byteswap.h>
#include <string.h>
+#include <rdma/rdma_cma.h>
+
#ifdef INCLUDE_VALGRIND
# include <valgrind/memcheck.h>
# ifndef VALGRIND_MAKE_MEM_DEFINED
@@ -84,5 +86,14 @@ static inline void *zalloc(size_t size)
int ucma_init();
-#endif /* CMA_H */
+#ifdef USE_IB_ACM
+void ucma_ib_init();
+void ucma_ib_cleanup();
+void ucma_ib_resolve(struct rdma_addrinfo *rai);
+#else
+#define ucma_ib_init()
+#define ucma_ib_cleanup()
+#define ucma_ib_resolve(x)
+#endif
+#endif /* CMA_H */
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 32/37] librdmacm/rdma_client: add new client sample
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Provide a very simple client application that shows the
minimal coding needed to establish a connection and exchange
messages with a server. The client makes use of the new
rdma_getaddrinfo and rdma_create_ep calls, plus rdma verbs
abstractions.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Makefile.am | 5 +-
examples/rdma_client.c | 134 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 138 insertions(+), 1 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index 8aef24a..6071599 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -17,7 +17,8 @@ src_librdmacm_la_LDFLAGS = -version-info 1 -export-dynamic \
$(librdmacm_version_script)
src_librdmacm_la_DEPENDENCIES = $(srcdir)/src/librdmacm.map
-bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy examples/mckey
+bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy examples/mckey \
+ examples/rdma_client
examples_ucmatose_SOURCES = examples/cmatose.c
examples_ucmatose_LDADD = $(top_builddir)/src/librdmacm.la
examples_rping_SOURCES = examples/rping.c
@@ -26,6 +27,8 @@ examples_udaddy_SOURCES = examples/udaddy.c
examples_udaddy_LDADD = $(top_builddir)/src/librdmacm.la
examples_mckey_SOURCES = examples/mckey.c
examples_mckey_LDADD = $(top_builddir)/src/librdmacm.la
+examples_rdma_client_SOURCES = examples/rdma_client.c
+examples_rdma_client_LDADD = $(top_builddir)/src/librdmacm.la
librdmacmincludedir = $(includedir)/rdma $(includedir)/infiniband
diff --git a/examples/rdma_client.c b/examples/rdma_client.c
new file mode 100644
index 0000000..7a59d97
--- /dev/null
+++ b/examples/rdma_client.c
@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under the OpenIB.org BSD license
+ * below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <getopt.h>
+#include <rdma/rdma_cma.h>
+#include <rdma/rdma_verbs.h>
+
+static char *server = "127.0.0.1";
+static char *port = "7471";
+
+struct rdma_cm_id *id;
+struct ibv_mr *mr;
+uint8_t send_msg[16];
+uint8_t recv_msg[16];
+
+static int run(void)
+{
+ struct rdma_addrinfo hints, *res;
+ struct ibv_qp_init_attr attr;
+ struct ibv_wc wc;
+ int ret;
+
+ memset(&hints, 0, sizeof hints);
+ hints.ai_port_space = RDMA_PS_TCP;
+ ret = rdma_getaddrinfo(server, port, &hints, &res);
+ if (ret) {
+ printf("rdma_getaddrinfo %d\n", errno);
+ return ret;
+ }
+
+ memset(&attr, 0, sizeof attr);
+ attr.cap.max_send_wr = attr.cap.max_recv_wr = 1;
+ attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
+ attr.cap.max_inline_data = 16;
+ attr.qp_context = id;
+ attr.sq_sig_all = 1;
+ ret = rdma_create_ep(&id, res, NULL, &attr);
+ rdma_freeaddrinfo(res);
+ if (ret) {
+ printf("rdma_create_ep %d\n", errno);
+ return ret;
+ }
+
+ mr = rdma_reg_msgs(id, recv_msg, 16);
+ if (!mr) {
+ printf("rdma_reg_msgs %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);
+ if (ret) {
+ printf("rdma_post_recv %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_connect(id, NULL);
+ if (ret) {
+ printf("rdma_connect %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_post_send(id, NULL, send_msg, 16, NULL, IBV_SEND_INLINE);
+ if (ret) {
+ printf("rdma_post_send %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_get_recv_comp(id, &wc);
+ if (ret <= 0) {
+ printf("rdma_get_recv_comp %d\n", ret);
+ return ret;
+ }
+
+ rdma_disconnect(id);
+ rdma_dereg_mr(mr);
+ rdma_destroy_ep(id);
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ int op, ret;
+
+ while ((op = getopt(argc, argv, "s:p:")) != -1) {
+ switch (op) {
+ case 's':
+ server = optarg;
+ break;
+ case 'p':
+ port = optarg;
+ break;
+ default:
+ printf("usage: %s\n", argv[0]);
+ printf("\t[-s server_address]\n");
+ printf("\t[-p port_number]\n");
+ exit(1);
+ }
+ }
+
+ printf("rdma_client: start\n");
+ ret = run();
+ printf("rdma_client: end %d\n", ret);
+ return ret;
+}
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 35/37] librdmacm/mckey: use AF_IB for unmapped multicast addresses
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
If the user joins an unmapped multicast address, use AF_IB,
rather than AF_INET6, to communicate that information with the
kernel.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
This requires AF_IB support in the kernel.
examples/mckey.c | 21 ++++++++++++++++++---
1 files changed, 18 insertions(+), 3 deletions(-)
diff --git a/examples/mckey.c b/examples/mckey.c
index ddc3495..a6b5c4d 100644
--- a/examples/mckey.c
+++ b/examples/mckey.c
@@ -46,6 +46,7 @@
#include <getopt.h>
#include <rdma/rdma_cma.h>
+#include <infiniband/ib.h>
struct cmatest_node {
int id;
@@ -67,9 +68,9 @@ struct cmatest {
int conn_index;
int connects_left;
- struct sockaddr_in6 dst_in;
+ struct sockaddr_storage dst_in;
struct sockaddr *dst_addr;
- struct sockaddr_in6 src_in;
+ struct sockaddr_storage src_in;
struct sockaddr *src_addr;
};
@@ -460,6 +461,20 @@ static int get_addr(char *dst, struct sockaddr *addr)
return ret;
}
+static int get_dst_addr(char *dst, struct sockaddr *addr)
+{
+ struct sockaddr_ib *sib;
+
+ if (!unmapped_addr)
+ return get_addr(dst, addr);
+
+ sib = (struct sockaddr_ib *) addr;
+ memset(sib, 0, sizeof *sib);
+ sib->sib_family = AF_IB;
+ inet_pton(AF_INET6, dst, &sib->sib_addr);
+ return 0;
+}
+
static int run(void)
{
int i, ret;
@@ -471,7 +486,7 @@ static int run(void)
return ret;
}
- ret = get_addr(dst_addr, (struct sockaddr *) &test.dst_in);
+ ret = get_dst_addr(dst_addr, (struct sockaddr *) &test.dst_in);
if (ret)
return ret;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 28/37] librdmacm: define RDMA_PS_IB
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
AF_IB uses the IB port space.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 12 +++++++++---
1 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 289b728..8f72201 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -67,11 +67,17 @@ enum rdma_cm_event_type {
};
enum rdma_port_space {
- RDMA_PS_IPOIB= 0x0002,
- RDMA_PS_TCP = 0x0106,
- RDMA_PS_UDP = 0x0111,
+ RDMA_PS_IPOIB = 0x0002,
+ RDMA_PS_IB = 0x0003,
+ RDMA_PS_TCP = 0x0106,
+ RDMA_PS_UDP = 0x0111,
};
+#define RDMA_IB_IP_PS_MASK 0xFFFFFFFFFFFF0000ULL
+#define RDMA_IB_IP_PORT_MASK 0x000000000000FFFFULL
+#define RDMA_IB_IP_PS_TCP 0x0000000001060000ULL
+#define RDMA_IB_IP_PS_UDP 0x0000000001110000ULL
+
/*
* Global qkey value for UDP QPs and multicast groups created via the
* RDMA CM.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 31/37] librdmacm: provide abstracted verb calls
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Provide abstractions to the verb calls to simplify the user
interface for more casual verbs consumers. Users still have
access to the full range of verbs functionality by calling
verbs directly.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Makefile.am | 5 -
include/rdma/rdma_verbs.h | 287 +++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 290 insertions(+), 2 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index 8d86045..8aef24a 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -31,7 +31,8 @@ librdmacmincludedir = $(includedir)/rdma $(includedir)/infiniband
librdmacminclude_HEADERS = include/rdma/rdma_cma_abi.h \
include/rdma/rdma_cma.h \
- include/infiniband/ib.h
+ include/infiniband/ib.h \
+ include/rdma/rdma_verbs.h
man_MANS = \
man/rdma_accept.3 \
@@ -69,7 +70,7 @@ man_MANS = \
man/rdma_cm.7
EXTRA_DIST = include/rdma/rdma_cma_abi.h include/rdma/rdma_cma.h \
- include/infiniband/ib.h \
+ include/infiniband/ib.h include/rdma/rdma_verbs.h \
src/cma.h src/librdmacm.map librdmacm.spec.in $(man_MANS)
dist-hook: librdmacm.spec
diff --git a/include/rdma/rdma_verbs.h b/include/rdma/rdma_verbs.h
new file mode 100644
index 0000000..05964c1
--- /dev/null
+++ b/include/rdma/rdma_verbs.h
@@ -0,0 +1,287 @@
+/*
+ * Copyright (c) 2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if !defined(RDMA_VERBS_H)
+#define RDMA_VERBS_H
+
+#include <assert.h>
+#include <infiniband/verbs.h>
+#include <rdma/rdma_cma.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Memory registration helpers.
+ */
+static inline struct ibv_mr *
+rdma_reg_msgs(struct rdma_cm_id *id, void *addr, size_t length)
+{
+ return ibv_reg_mr(id->qp->pd, addr, length, IBV_ACCESS_LOCAL_WRITE);
+}
+
+static inline struct ibv_mr *
+rdma_reg_read(struct rdma_cm_id *id, void *addr, size_t length)
+{
+ return ibv_reg_mr(id->qp->pd, addr, length, IBV_ACCESS_LOCAL_WRITE|
+ IBV_ACCESS_REMOTE_READ);
+}
+
+static inline struct ibv_mr *
+rdma_reg_write(struct rdma_cm_id *id, void *addr, size_t length)
+{
+ return ibv_reg_mr(id->qp->pd, addr, length, IBV_ACCESS_LOCAL_WRITE |
+ IBV_ACCESS_REMOTE_WRITE);
+}
+
+static inline int
+rdma_dereg_mr(struct ibv_mr *mr)
+{
+ return ibv_dereg_mr(mr);
+}
+
+
+/*
+ * Vectored send, receive, and RDMA operations.
+ * Support multiple scatter-gather entries.
+ */
+static inline int
+rdma_post_recvv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
+ int nsge)
+{
+ struct ibv_recv_wr wr, *bad;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = sgl;
+ wr.num_sge = nsge;
+
+ return ibv_post_recv(id->qp, &wr, &bad);
+}
+
+static inline int
+rdma_post_sendv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
+ int nsge, int flags)
+{
+ struct ibv_send_wr wr, *bad;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = sgl;
+ wr.num_sge = nsge;
+ wr.opcode = IBV_WR_SEND;
+ wr.send_flags = flags;
+
+ return ibv_post_send(id->qp, &wr, &bad);
+}
+
+static inline int
+rdma_post_readv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
+ int nsge, int flags, uint64_t remote_addr, uint32_t rkey)
+{
+ struct ibv_send_wr wr, *bad;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = sgl;
+ wr.num_sge = nsge;
+ wr.opcode = IBV_WR_RDMA_READ;
+ wr.send_flags = flags;
+ wr.wr.rdma.remote_addr = remote_addr;
+ wr.wr.rdma.rkey = rkey;
+
+ return ibv_post_send(id->qp, &wr, &bad);
+}
+
+static inline int
+rdma_post_writev(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
+ int nsge, int flags, uint64_t remote_addr, uint32_t rkey)
+{
+ struct ibv_send_wr wr, *bad;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = sgl;
+ wr.num_sge = nsge;
+ wr.opcode = IBV_WR_RDMA_WRITE;
+ wr.send_flags = flags;
+ wr.wr.rdma.remote_addr = remote_addr;
+ wr.wr.rdma.rkey = rkey;
+
+ return ibv_post_send(id->qp, &wr, &bad);
+}
+
+/*
+ * Simple send, receive, and RDMA calls.
+ */
+static inline int
+rdma_post_recv(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr)
+{
+ struct ibv_sge sge;
+
+ assert((addr >= mr->addr) && ((addr + length) <= (mr->addr + mr->length)));
+ sge.addr = (uint64_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr->lkey;
+
+ return rdma_post_recvv(id, context, &sge, 1);
+}
+
+static inline int
+rdma_post_send(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr, int flags)
+{
+ struct ibv_sge sge;
+
+ sge.addr = (uint64_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr ? mr->lkey : 0;
+
+ return rdma_post_sendv(id, context, &sge, 1, flags);
+}
+
+static inline int
+rdma_post_read(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr, int flags,
+ uint64_t remote_addr, uint32_t rkey)
+{
+ struct ibv_sge sge;
+
+ sge.addr = (uint64_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr->lkey;
+
+ return rdma_post_readv(id, context, &sge, 1, flags, remote_addr, rkey);
+}
+
+static inline int
+rdma_post_write(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr, int flags,
+ uint64_t remote_addr, uint32_t rkey)
+{
+ struct ibv_sge sge;
+
+ sge.addr = (uint64_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr ? mr->lkey : 0;
+
+ return rdma_post_writev(id, context, &sge, 1, flags, remote_addr, rkey);
+}
+
+static inline int
+rdma_post_ud_send(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr, int flags,
+ struct ibv_ah *ah, uint32_t remote_qpn)
+{
+ struct ibv_send_wr wr, *bad;
+ struct ibv_sge sge;
+
+ sge.addr = (uint64_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr ? mr->lkey : 0;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = &sge;
+ wr.num_sge = 1;
+ wr.opcode = IBV_WR_SEND;
+ wr.send_flags = flags;
+ wr.wr.ud.ah = ah;
+ wr.wr.ud.remote_qpn = remote_qpn;
+ wr.wr.ud.remote_qkey = RDMA_UDP_QKEY;
+
+ return ibv_post_send(id->qp, &wr, &bad);
+}
+
+static inline int
+rdma_get_send_comp(struct rdma_cm_id *id, struct ibv_wc *wc)
+{
+ struct ibv_cq *cq;
+ void *context;
+ int ret;
+
+ ret = ibv_poll_cq(id->send_cq, 1, wc);
+ if (ret)
+ return ret;
+
+ ret = ibv_req_notify_cq(id->send_cq, 0);
+ if (ret)
+ return ret;
+
+ ret = ibv_poll_cq(id->send_cq, 1, wc);
+ if (ret)
+ return ret;
+
+ ret = ibv_get_cq_event(id->send_cq_channel, &cq, &context);
+ if (ret)
+ return ret;
+
+ assert(cq == id->send_cq && context == id);
+ ibv_ack_cq_events(id->send_cq, 1);
+ return ibv_poll_cq(id->send_cq, 1, wc);
+}
+
+static inline int
+rdma_get_recv_comp(struct rdma_cm_id *id, struct ibv_wc *wc)
+{
+ struct ibv_cq *cq;
+ void *context;
+ int ret;
+
+ ret = ibv_poll_cq(id->recv_cq, 1, wc);
+ if (ret)
+ return ret;
+
+ ret = ibv_req_notify_cq(id->recv_cq, 0);
+ if (ret)
+ return ret;
+
+ ret = ibv_poll_cq(id->recv_cq, 1, wc);
+ if (ret)
+ return ret;
+
+ ret = ibv_get_cq_event(id->recv_cq_channel, &cq, &context);
+ if (ret)
+ return ret;
+
+ assert(cq == id->recv_cq && context == id);
+ ibv_ack_cq_events(id->recv_cq, 1);
+ return ibv_poll_cq(id->recv_cq, 1, wc);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RDMA_CMA_H */
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 29/37] librdmacm: use sockaddr_ib addressing if IB ACM is in use
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
If IB ACM route resolution succeeds, provide the user with
AF_IB addresses, rather than AF_INET or AF_INET6 addressing.
AF_IB identifies the local and remote devices directly,
eliminating the need to perform address resolution a second
time via rdma_resolve_addr.
AF_IB addresses are returned using sockaddr_ib as part of
rdma_getaddrinfo.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/acm.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
src/cma.h | 1 +
2 files changed, 66 insertions(+), 0 deletions(-)
diff --git a/src/acm.c b/src/acm.c
index 34fdf3c..5dd8899 100644
--- a/src/acm.c
+++ b/src/acm.c
@@ -83,9 +83,69 @@ void ucma_ib_cleanup(void)
}
}
+static void ucma_set_sid(enum rdma_port_space ps, struct sockaddr *addr,
+ struct sockaddr_ib *sib)
+{
+ uint16_t port;
+
+ if (addr->sa_family == AF_INET)
+ port = ((struct sockaddr_in *) addr)->sin_port;
+ else
+ port = ((struct sockaddr_in6 *) addr)->sin6_port;
+
+ sib->sib_sid = htonll(((uint64_t) ps << 16) + ntohs(port));
+ if (port)
+ sib->sib_sid_mask = ~0ULL;
+ else
+ sib->sib_sid_mask = htonll(RDMA_IB_IP_PS_MASK);
+}
+
+static void ucma_ib_convert_addr(struct rdma_addrinfo *rai,
+ struct ib_path_record *path)
+{
+ struct sockaddr_ib *src, *dst;
+
+ if (!path)
+ return;
+
+ src = zalloc(sizeof *src);
+ if (!src)
+ return;
+
+ dst = zalloc(sizeof *dst);
+ if (!dst) {
+ free(src);
+ return;
+ }
+
+ src->sib_family = AF_IB;
+ src->sib_pkey = path->pkey;
+ src->sib_flowinfo = htonl(ntohl(path->flowlabel_hoplimit) >> 8);
+ memcpy(&src->sib_addr, &path->sgid, 16);
+ ucma_set_sid(rai->ai_port_space, rai->ai_src_addr, src);
+
+ dst->sib_family = AF_IB;
+ dst->sib_pkey = path->pkey;
+ dst->sib_flowinfo = htonl(ntohl(path->flowlabel_hoplimit) >> 8);
+ memcpy(&dst->sib_addr, &path->dgid, 16);
+ ucma_set_sid(rai->ai_port_space, rai->ai_dst_addr, dst);
+
+ free(rai->ai_src_addr);
+ rai->ai_src_addr = (struct sockaddr *) src;
+ rai->ai_src_len = sizeof(*src);
+
+ free(rai->ai_dst_addr);
+ rai->ai_dst_addr = (struct sockaddr *) dst;
+ rai->ai_dst_len = sizeof(*dst);
+
+ rai->ai_family = AF_IB;
+ rai->ai_port_space = RDMA_PS_IB;
+}
+
static void ucma_ib_save_resp(struct rdma_addrinfo *rai, struct acm_resolve_msg *msg)
{
struct ib_path_data *path_data = NULL;
+ struct ib_path_record *pri_path;
int len, i, cnt;
len = msg->hdr.length - ACM_MSG_HDR_LENGTH;
@@ -98,11 +158,16 @@ static void ucma_ib_save_resp(struct rdma_addrinfo *rai, struct acm_resolve_msg
for (i = 0; i < cnt; i++) {
if (msg->data[i].type != ACM_EP_INFO_PATH)
goto err;
+ if (path_data[i].flags |
+ (IB_PATH_FLAG_PRIMARY | IB_PATH_FLAG_OUTBOUND))
+ pri_path = &path_data[i].path;
path_data[i].reserved = 0;
}
rai->ai_route = path_data;
rai->ai_route_len = len;
+ if (af_ib_support)
+ ucma_ib_convert_addr(rai, pri_path);
return;
err:
free(path_data);
diff --git a/src/cma.h b/src/cma.h
index 62785de..1e0c571 100644
--- a/src/cma.h
+++ b/src/cma.h
@@ -85,6 +85,7 @@ static inline void *zalloc(size_t size)
}
int ucma_init();
+extern int af_ib_support;
#ifdef USE_IB_ACM
void ucma_ib_init();
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 26/37] librdmacm: set src_addr in rdma_getaddrinfo
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
RDMA requires the user to allocate hardware resources before
establishing a connection. To support this, the user must know
the source address that the connection will use to reach the
remote endpoint. Modify rdma_getaddrinfo to determine an
appropriate source address based on the specified destination,
when a source address is not given.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/addrinfo.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 60 insertions(+), 0 deletions(-)
diff --git a/src/addrinfo.c b/src/addrinfo.c
index 15ae071..dfaf9d5 100644
--- a/src/addrinfo.c
+++ b/src/addrinfo.c
@@ -39,6 +39,7 @@
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
+#include <unistd.h>
#include "cma.h"
#include <rdma/rdma_cma.h>
@@ -129,6 +130,48 @@ static int ucma_convert_to_rai(struct rdma_addrinfo *rai, struct addrinfo *ai)
return 0;
}
+static int ucma_resolve_src(struct rdma_addrinfo *rai)
+{
+ struct sockaddr *addr;
+ socklen_t len;
+ int ret, s;
+
+ s = socket(rai->ai_family, SOCK_DGRAM, IPPROTO_UDP);
+ if (s < 0)
+ return s;
+
+ ret = connect(s, rai->ai_dst_addr, rai->ai_dst_len);
+ if (ret)
+ goto err1;
+
+ addr = zalloc(rai->ai_dst_len);
+ if (!addr) {
+ ret = ERR(ENOMEM);
+ goto err1;
+ }
+
+ len = rai->ai_dst_len;
+ ret = getsockname(s, addr, &len);
+ if (ret)
+ goto err2;
+
+ if (addr->sa_family == AF_INET)
+ ((struct sockaddr_in *) addr)->sin_port = 0;
+ else
+ ((struct sockaddr_in6 *) addr)->sin6_port = 0;
+ rai->ai_src_addr = addr;
+ rai->ai_src_len = len;
+
+ close(s);
+ return 0;
+
+err2:
+ free(addr);
+err1:
+ close(s);
+ return ret;
+}
+
int rdma_getaddrinfo(char *node, char *service,
struct rdma_addrinfo *hints,
struct rdma_addrinfo **res)
@@ -159,6 +202,23 @@ int rdma_getaddrinfo(char *node, char *service,
if (ret)
goto err2;
+ if (!rai->ai_src_len) {
+ if (hints && hints->ai_src_len) {
+ rai->ai_src_addr = zalloc(hints->ai_src_len);
+ if (!rai->ai_src_addr) {
+ ret = ERR(ENOMEM);
+ goto err2;
+ }
+ memcpy(rai->ai_src_addr, hints->ai_src_addr,
+ hints->ai_src_len);
+ rai->ai_src_len = hints->ai_src_len;
+ } else {
+ ret = ucma_resolve_src(rai);
+ if (ret)
+ goto err2;
+ }
+ }
+
freeaddrinfo(ai);
*res = rai;
return 0;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 25/37] librdmacm: auto create QPs in rdma_get_request
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
If the user passed initial QP attributes into rdma_create_ep,
allocate a QP for the user as part of rdma_get_request.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/cma.c | 6 ++++++
1 files changed, 6 insertions(+), 0 deletions(-)
diff --git a/src/cma.c b/src/cma.c
index c59f47b..1cc4f1f 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -1287,6 +1287,12 @@ int rdma_get_request(struct rdma_cm_id *listen, struct rdma_cm_id **id)
goto err;
}
+ if (id_priv->qp_init_attr) {
+ ret = rdma_create_qp(event->id, id_priv->pd, id_priv->qp_init_attr);
+ if (ret)
+ goto err;
+ }
+
*id = event->id;
(*id)->event = event;
return 0;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 30/37] librdmacm: format IB CM private data RDMA CM header
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/acm.c | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++-------
src/cma.c | 32 +++++++++++++++++++++++++++++---
2 files changed, 81 insertions(+), 10 deletions(-)
diff --git a/src/acm.c b/src/acm.c
index 5dd8899..8975b97 100644
--- a/src/acm.c
+++ b/src/acm.c
@@ -51,6 +51,18 @@ static pthread_mutex_t acm_lock = PTHREAD_MUTEX_INITIALIZER;
static int sock;
static short server_port = 6125;
+struct ib_connect_hdr {
+ uint8_t cma_version;
+ uint8_t ip_version; /* IP version: 7:4 */
+ uint16_t port;
+ uint32_t src_addr[4];
+ uint32_t dst_addr[4];
+#define cma_src_ip4 src_addr[3]
+#define cma_src_ip6 src_addr[0]
+#define cma_dst_ip4 dst_addr[3]
+#define cma_dst_ip6 dst_addr[0]
+};
+
void ucma_ib_init(void)
{
struct sockaddr_in addr;
@@ -100,22 +112,22 @@ static void ucma_set_sid(enum rdma_port_space ps, struct sockaddr *addr,
sib->sib_sid_mask = htonll(RDMA_IB_IP_PS_MASK);
}
-static void ucma_ib_convert_addr(struct rdma_addrinfo *rai,
- struct ib_path_record *path)
+static int ucma_ib_convert_addr(struct rdma_addrinfo *rai,
+ struct ib_path_record *path)
{
struct sockaddr_ib *src, *dst;
if (!path)
- return;
+ return ERR(ENODATA);
src = zalloc(sizeof *src);
if (!src)
- return;
+ return ERR(ENOMEM);
dst = zalloc(sizeof *dst);
if (!dst) {
free(src);
- return;
+ return ERR(ENOMEM);
}
src->sib_family = AF_IB;
@@ -140,6 +152,33 @@ static void ucma_ib_convert_addr(struct rdma_addrinfo *rai,
rai->ai_family = AF_IB;
rai->ai_port_space = RDMA_PS_IB;
+ return 0;
+}
+
+static void ucma_ib_format_connect(struct rdma_addrinfo *rai)
+{
+ struct ib_connect_hdr *hdr;
+
+ hdr = zalloc(sizeof *hdr);
+ if (!hdr)
+ return;
+
+ if (rai->ai_family == AF_INET) {
+ hdr->ip_version = 4 << 4;
+ memcpy(&hdr->cma_src_ip4,
+ &((struct sockaddr_in *) rai->ai_src_addr)->sin_addr, 4);
+ memcpy(&hdr->cma_dst_ip4,
+ &((struct sockaddr_in *) rai->ai_dst_addr)->sin_addr, 4);
+ } else {
+ hdr->ip_version = 6 << 4;
+ memcpy(&hdr->cma_src_ip6,
+ &((struct sockaddr_in6 *) rai->ai_src_addr)->sin6_addr, 16);
+ memcpy(&hdr->cma_dst_ip6,
+ &((struct sockaddr_in6 *) rai->ai_dst_addr)->sin6_addr, 16);
+ }
+
+ rai->ai_connect = hdr;
+ rai->ai_connect_len = sizeof(*hdr);
}
static void ucma_ib_save_resp(struct rdma_addrinfo *rai, struct acm_resolve_msg *msg)
@@ -166,8 +205,14 @@ static void ucma_ib_save_resp(struct rdma_addrinfo *rai, struct acm_resolve_msg
rai->ai_route = path_data;
rai->ai_route_len = len;
- if (af_ib_support)
- ucma_ib_convert_addr(rai, pri_path);
+ if (af_ib_support) {
+ ucma_ib_format_connect(rai);
+ if (ucma_ib_convert_addr(rai, pri_path) &&
+ rai->ai_connect) {
+ free(rai->ai_connect);
+ rai->ai_connect_len = 0;
+ }
+ }
return;
err:
free(path_data);
diff --git a/src/cma.c b/src/cma.c
index 6ac949a..faf4264 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -106,6 +106,8 @@ struct cma_device {
struct cma_id_private {
struct rdma_cm_id id;
struct cma_device *cma_dev;
+ void *connect;
+ size_t connect_len;
int events_completed;
int connect_error;
int sync;
@@ -363,6 +365,8 @@ static void ucma_free_id(struct cma_id_private *id_priv)
if (id_priv->sync)
rdma_destroy_event_channel(id_priv->id.channel);
+ if (id_priv->connect)
+ free(id_priv->connect);
free(id_priv);
}
@@ -1186,15 +1190,20 @@ static void ucma_copy_conn_param_to_kern(struct cma_id_private *id_priv,
dst->initiator_depth = id_priv->initiator_depth;
dst->valid = 1;
+ if (id_priv->connect_len) {
+ memcpy(dst->private_data, id_priv->connect, id_priv->connect_len);
+ dst->private_data_len = id_priv->connect_len;
+ }
+
if (src) {
dst->flow_control = src->flow_control;
dst->retry_count = src->retry_count;
dst->rnr_retry_count = src->rnr_retry_count;
if (src->private_data && src->private_data_len) {
- memcpy(dst->private_data, src->private_data,
- src->private_data_len);
- dst->private_data_len = src->private_data_len;
+ memcpy(dst->private_data + dst->private_data_len,
+ src->private_data, src->private_data_len);
+ dst->private_data_len += src->private_data_len;
}
} else {
dst->retry_count = 7;
@@ -1238,6 +1247,11 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
if (ret != size)
return (ret >= 0) ? ERR(ENODATA) : -1;
+ if (id_priv->connect) {
+ free(id_priv->connect);
+ id_priv->connect_len = 0;
+ }
+
return ucma_complete(id_priv);
}
@@ -2038,6 +2052,7 @@ int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
{
struct rdma_cm_id *cm_id;
+ struct cma_id_private *id_priv;
int ret;
ret = rdma_create_id2(NULL, &cm_id, NULL, res->ai_port_space, res->ai_qp_type);
@@ -2075,6 +2090,17 @@ int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
if (ret)
goto err;
+ if (res->ai_connect_len) {
+ id_priv = container_of(cm_id, struct cma_id_private, id);
+ id_priv->connect = malloc(res->ai_connect_len);
+ if (!id_priv->connect) {
+ ret = ERR(ENOMEM);
+ goto err;
+ }
+ memcpy(id_priv->connect, res->ai_connect, res->ai_connect_len);
+ id_priv->connect_len = res->ai_connect_len;
+ }
+
out:
*id = cm_id;
return 0;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 33/37] librdmacm/rdma_server: add new sample server application
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Provide a simple server application to demonstrate the minimal
amount of coding needed to accept a connection request from
a client and exchange messages.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Makefile.am | 4 +
examples/rdma_server.c | 149 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 152 insertions(+), 1 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index 6071599..8701002 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -18,7 +18,7 @@ src_librdmacm_la_LDFLAGS = -version-info 1 -export-dynamic \
src_librdmacm_la_DEPENDENCIES = $(srcdir)/src/librdmacm.map
bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy examples/mckey \
- examples/rdma_client
+ examples/rdma_client examples/rdma_server
examples_ucmatose_SOURCES = examples/cmatose.c
examples_ucmatose_LDADD = $(top_builddir)/src/librdmacm.la
examples_rping_SOURCES = examples/rping.c
@@ -29,6 +29,8 @@ examples_mckey_SOURCES = examples/mckey.c
examples_mckey_LDADD = $(top_builddir)/src/librdmacm.la
examples_rdma_client_SOURCES = examples/rdma_client.c
examples_rdma_client_LDADD = $(top_builddir)/src/librdmacm.la
+examples_rdma_server_SOURCES = examples/rdma_server.c
+examples_rdma_server_LDADD = $(top_builddir)/src/librdmacm.la
librdmacmincludedir = $(includedir)/rdma $(includedir)/infiniband
diff --git a/examples/rdma_server.c b/examples/rdma_server.c
new file mode 100644
index 0000000..2831d0c
--- /dev/null
+++ b/examples/rdma_server.c
@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2005-2009 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under the OpenIB.org BSD license
+ * below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <getopt.h>
+#include <netdb.h>
+#include <rdma/rdma_cma.h>
+#include <rdma/rdma_verbs.h>
+
+static char *port = "7471";
+
+struct rdma_cm_id *listen_id, *id;
+struct ibv_mr *mr;
+uint8_t send_msg[16];
+uint8_t recv_msg[16];
+
+static int run(void)
+{
+ struct rdma_addrinfo hints, *res;
+ struct ibv_qp_init_attr attr;
+ struct ibv_wc wc;
+ int ret;
+
+ memset(&hints, 0, sizeof hints);
+ hints.ai_flags = RAI_PASSIVE;
+ hints.ai_port_space = RDMA_PS_TCP;
+ ret = rdma_getaddrinfo(NULL, port, &hints, &res);
+ if (ret) {
+ printf("rdma_getaddrinfo %d\n", errno);
+ return ret;
+ }
+
+ memset(&attr, 0, sizeof attr);
+ attr.cap.max_send_wr = attr.cap.max_recv_wr = 1;
+ attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
+ attr.cap.max_inline_data = 16;
+ attr.sq_sig_all = 1;
+ ret = rdma_create_ep(&listen_id, res, NULL, &attr);
+ rdma_freeaddrinfo(res);
+ if (ret) {
+ printf("rdma_create_ep %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_listen(listen_id, 0);
+ if (ret) {
+ printf("rdma_listen %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_get_request(listen_id, &id);
+ if (ret) {
+ printf("rdma_get_request %d\n", errno);
+ return ret;
+ }
+
+ mr = rdma_reg_msgs(id, recv_msg, 16);
+ if (!mr) {
+ printf("rdma_reg_msgs %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);
+ if (ret) {
+ printf("rdma_post_recv %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_accept(id, NULL);
+ if (ret) {
+ printf("rdma_connect %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_get_recv_comp(id, &wc);
+ if (ret <= 0) {
+ printf("rdma_get_recv_comp %d\n", ret);
+ return ret;
+ }
+
+ ret = rdma_post_send(id, NULL, send_msg, 16, NULL, IBV_SEND_INLINE);
+ if (ret) {
+ printf("rdma_post_send %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_get_send_comp(id, &wc);
+ if (ret <= 0) {
+ printf("rdma_get_send_comp %d\n", ret);
+ return ret;
+ }
+
+ rdma_disconnect(id);
+ rdma_dereg_mr(mr);
+ rdma_destroy_ep(id);
+ rdma_destroy_ep(listen_id);
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ int op, ret;
+
+ while ((op = getopt(argc, argv, "p:")) != -1) {
+ switch (op) {
+ case 'p':
+ port = optarg;
+ break;
+ default:
+ printf("usage: %s\n", argv[0]);
+ printf("\t[-p port_number]\n");
+ exit(1);
+ }
+ }
+
+ printf("rdma_server: start\n");
+ ret = run();
+ printf("rdma_server: end %d\n", ret);
+ return ret;
+}
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 23/37] librdmacm: add rdma_destoy_ep
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 11 +++++++++++
src/cma.c | 8 ++++++++
src/librdmacm.map | 1 +
3 files changed, 20 insertions(+), 0 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 74762a2..289b728 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -258,6 +258,17 @@ int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr);
/**
+ * rdma_destroy_ep - Deallocates a communication identifier and qp.
+ * @id: The communication identifer to destroy.
+ * Description:
+ * Destroys the specified rdma_cm_id and any associated QP created
+ * on that id.
+ * See also:
+ * rdma_create_ep
+ */
+void rdma_destroy_ep(struct rdma_cm_id *id);
+
+/**
* rdma_destroy_id - Release a communication identifier.
* @id: The communication identifier to destroy.
* Description:
diff --git a/src/cma.c b/src/cma.c
index b911b3f..3b80542 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -1142,6 +1142,7 @@ void rdma_destroy_qp(struct rdma_cm_id *id)
{
ibv_destroy_qp(id->qp);
ucma_destroy_cqs(id);
+ id->qp = NULL;
}
static int ucma_valid_param(struct cma_id_private *id_priv,
@@ -2033,3 +2034,10 @@ err:
rdma_destroy_id(cm_id);
return ret;
}
+
+void rdma_destroy_ep(struct rdma_cm_id *id)
+{
+ if (id->qp)
+ rdma_destroy_qp(id);
+ rdma_destroy_id(id);
+}
diff --git a/src/librdmacm.map b/src/librdmacm.map
index 85d5b3c..19b193a 100644
--- a/src/librdmacm.map
+++ b/src/librdmacm.map
@@ -32,5 +32,6 @@ RDMACM_1.0 {
rdma_freeaddrinfo;
rdma_get_request;
rdma_create_ep;
+ rdma_destroy_ep;
local: *;
};
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 22/37] librdmacm: add new call to create id
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Provide a simple call to create an rdma_id, with optional QP. The
id is created using synchronous operation, using the output of
rdma_getaddrinfo.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 26 ++++++++++++++++++++++++++
src/cma.c | 43 +++++++++++++++++++++++++++++++++++++++++++
src/librdmacm.map | 1 +
3 files changed, 70 insertions(+), 0 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 391c35c..74762a2 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -232,6 +232,32 @@ int rdma_create_id(struct rdma_event_channel *channel,
enum rdma_port_space ps);
/**
+ * rdma_create_ep - Allocate a communication identifier and qp.
+ * @id: A reference where the allocated communication identifier will be
+ * returned.
+ * @res: Result from rdma_getaddrinfo, which specifies the source and
+ * destination addresses, plus optional routing and connection information.
+ * @pd: Optional protection domain. This parameter is ignored if qp_init_attr
+ * is NULL.
+ * @qp_init_attr: Optional attributes for a QP created on the rdma_cm_id.
+ * Description:
+ * Create an identifier and option QP used for communication.
+ * Notes:
+ * If qp_init_attr is provided, then a queue pair will be allocated and
+ * associated with the rdma_cm_id. If a pd is provided, the QP will be
+ * created on that PD. Otherwise, the QP will be allocated on a default
+ * PD.
+ * The rdma_cm_id will be set to use synchronous operations (connect,
+ * listen, and get_request). To convert to synchronous operation, the
+ * rdma_cm_id should be migrated to a user allocated event channel.
+ * See also:
+ * rdma_create_id, rdma_create_qp, rdma_migrate_id, rdma_connect,
+ * rdma_listen
+ */
+int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
+ struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr);
+
+/**
* rdma_destroy_id - Release a communication identifier.
* @id: The communication identifier to destroy.
* Description:
diff --git a/src/cma.c b/src/cma.c
index e31fb8a..b911b3f 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -49,6 +49,7 @@
#include <endian.h>
#include <byteswap.h>
#include <stddef.h>
+#include <netdb.h>
#include "cma.h"
#include <infiniband/driver.h>
@@ -1990,3 +1991,45 @@ int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel)
return 0;
}
+
+int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
+ struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
+{
+ struct rdma_cm_id *cm_id;
+ int ret;
+
+ ret = rdma_create_id2(NULL, &cm_id, NULL, res->ai_port_space, res->ai_qp_type);
+ if (ret)
+ return ret;
+
+ if (af_ib_support)
+ ret = rdma_resolve_addr2(cm_id, res->ai_src_addr, res->ai_src_len,
+ res->ai_dst_addr, res->ai_dst_len, 2000);
+ else
+ ret = rdma_resolve_addr(cm_id, res->ai_src_addr, res->ai_dst_addr, 2000);
+ if (ret)
+ goto err;
+
+ if (res->ai_route_len) {
+ ret = rdma_set_option(cm_id, RDMA_OPTION_IB, RDMA_OPTION_IB_PATH,
+ res->ai_route, res->ai_route_len);
+ if (!ret)
+ ret = ucma_complete(container_of(cm_id, struct cma_id_private, id));
+ } else {
+ ret = rdma_resolve_route(cm_id, 2000);
+ }
+ if (ret)
+ goto err;
+
+ qp_init_attr->qp_type = res->ai_qp_type;
+ ret = rdma_create_qp(cm_id, pd, qp_init_attr);
+ if (ret)
+ goto err;
+
+ *id = cm_id;
+ return 0;
+
+err:
+ rdma_destroy_id(cm_id);
+ return ret;
+}
diff --git a/src/librdmacm.map b/src/librdmacm.map
index f6af452..85d5b3c 100644
--- a/src/librdmacm.map
+++ b/src/librdmacm.map
@@ -31,5 +31,6 @@ RDMACM_1.0 {
rdma_getaddrinfo;
rdma_freeaddrinfo;
rdma_get_request;
+ rdma_create_ep;
local: *;
};
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 20/37] librdmacm: define options to set IB route directly
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Add the definitions needed to set IB path record data using set_option.
This will be used by rdma_create_ep().
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 6 ++++--
1 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 89013a0..391c35c 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -588,12 +588,14 @@ const char *rdma_event_str(enum rdma_cm_event_type event);
/* Option levels */
enum {
- RDMA_OPTION_ID = 0
+ RDMA_OPTION_ID = 0,
+ RDMA_OPTION_IB = 1
};
/* Option details */
enum {
- RDMA_OPTION_ID_TOS = 0 /* uint8_t: RFC 2474 */
+ RDMA_OPTION_ID_TOS = 0, /* uint8_t: RFC 2474 */
+ RDMA_OPTION_IB_PATH = 1 /* struct ib_path_data[] */
};
/**
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 16/37] librdmacm: allow conn_param to be optional
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
If conn_param is not provided to rdma_connect or rdma_accept,
then default values are used to establish the connection.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 8 ++++++--
src/cma.c | 34 +++++++++++++++++++++++-----------
2 files changed, 29 insertions(+), 13 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index f50b4dd..88b3796 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -325,7 +325,7 @@ void rdma_destroy_qp(struct rdma_cm_id *id);
/**
* rdma_connect - Initiate an active connection request.
* @id: RDMA identifier.
- * @conn_param: connection parameters.
+ * @conn_param: optional connection parameters.
* Description:
* For a connected rdma_cm_id, this call initiates a connection request
* to a remote destination. For an unconnected rdma_cm_id, it initiates
@@ -333,6 +333,8 @@ void rdma_destroy_qp(struct rdma_cm_id *id);
* Notes:
* Users must have resolved a route to the destination address
* by having called rdma_resolve_route before calling this routine.
+ * A user may override the default connection parameters and exchange
+ * private data as part of the connection by using the conn_param parameter.
* See also:
* rdma_resolve_route, rdma_disconnect, rdma_listen, rdma_get_cm_event
*/
@@ -361,7 +363,7 @@ int rdma_listen(struct rdma_cm_id *id, int backlog);
/**
* rdma_accept - Called to accept a connection request.
* @id: Connection identifier associated with the request.
- * @conn_param: Information needed to establish the connection.
+ * @conn_param: Optional information needed to establish the connection.
* Description:
* Called from the listening side to accept a connection or datagram
* service lookup request.
@@ -372,6 +374,8 @@ int rdma_listen(struct rdma_cm_id *id, int backlog);
* events give the user a newly created rdma_cm_id, similar to a new
* socket, but the rdma_cm_id is bound to a specific RDMA device.
* rdma_accept is called on the new rdma_cm_id.
+ * A user may override the default connection parameters and exchange
+ * private data as part of the connection by using the conn_param parameter.
* See also:
* rdma_listen, rdma_reject, rdma_get_cm_event
*/
diff --git a/src/cma.c b/src/cma.c
index b8d57a5..6ef4b96 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -1133,6 +1133,12 @@ static int ucma_valid_param(struct cma_id_private *id_priv,
if (id_priv->id.ps != RDMA_PS_TCP)
return 0;
+ if (!id_priv->id.qp && !param)
+ return ERR(EINVAL);
+
+ if (!param)
+ return 0;
+
if ((param->responder_resources != RDMA_MAX_RESP_RES) &&
(param->responder_resources > id_priv->cma_dev->max_responder_resources))
return ERR(EINVAL);
@@ -1153,15 +1159,21 @@ static void ucma_copy_conn_param_to_kern(struct cma_id_private *id_priv,
dst->srq = srq;
dst->responder_resources = id_priv->responder_resources;
dst->initiator_depth = id_priv->initiator_depth;
- dst->flow_control = src->flow_control;
- dst->retry_count = src->retry_count;
- dst->rnr_retry_count = src->rnr_retry_count;
dst->valid = 1;
- if (src->private_data && src->private_data_len) {
- memcpy(dst->private_data, src->private_data,
- src->private_data_len);
- dst->private_data_len = src->private_data_len;
+ if (src) {
+ dst->flow_control = src->flow_control;
+ dst->retry_count = src->retry_count;
+ dst->rnr_retry_count = src->rnr_retry_count;
+
+ if (src->private_data && src->private_data_len) {
+ memcpy(dst->private_data, src->private_data,
+ src->private_data_len);
+ dst->private_data_len = src->private_data_len;
+ }
+ } else {
+ dst->retry_count = 7;
+ dst->rnr_retry_count = 7;
}
}
@@ -1177,11 +1189,11 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
if (ret)
return ret;
- if (conn_param->initiator_depth != RDMA_MAX_INIT_DEPTH)
+ if (conn_param && conn_param->initiator_depth != RDMA_MAX_INIT_DEPTH)
id_priv->initiator_depth = conn_param->initiator_depth;
else
id_priv->initiator_depth = id_priv->cma_dev->max_initiator_depth;
- if (conn_param->responder_resources != RDMA_MAX_RESP_RES)
+ if (conn_param && conn_param->responder_resources != RDMA_MAX_RESP_RES)
id_priv->responder_resources = conn_param->responder_resources;
else
id_priv->responder_resources = id_priv->cma_dev->max_responder_resources;
@@ -1238,13 +1250,13 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
if (ret)
return ret;
- if (conn_param->initiator_depth == RDMA_MAX_INIT_DEPTH) {
+ if (!conn_param || conn_param->initiator_depth == RDMA_MAX_INIT_DEPTH) {
id_priv->initiator_depth = min(id_priv->initiator_depth,
id_priv->cma_dev->max_initiator_depth);
} else {
id_priv->initiator_depth = conn_param->initiator_depth;
}
- if (conn_param->responder_resources == RDMA_MAX_RESP_RES) {
+ if (!conn_param || conn_param->responder_resources == RDMA_MAX_RESP_RES) {
id_priv->responder_resources = min(id_priv->responder_resources,
id_priv->cma_dev->max_responder_resources);
} else {
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 24/37] librdmacm: add support for passive side in rdma_create_ep
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Allow calling rdma_create_ep for a listening rdma_cm_id.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/cma.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++-----------
1 files changed, 57 insertions(+), 12 deletions(-)
diff --git a/src/cma.c b/src/cma.c
index 3b80542..c59f47b 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -104,17 +104,19 @@ struct cma_device {
};
struct cma_id_private {
- struct rdma_cm_id id;
- struct cma_device *cma_dev;
- int events_completed;
- int connect_error;
- int sync;
- pthread_cond_t cond;
- pthread_mutex_t mut;
- uint32_t handle;
- struct cma_multicast *mc_list;
- uint8_t initiator_depth;
- uint8_t responder_resources;
+ struct rdma_cm_id id;
+ struct cma_device *cma_dev;
+ int events_completed;
+ int connect_error;
+ int sync;
+ pthread_cond_t cond;
+ pthread_mutex_t mut;
+ uint32_t handle;
+ struct cma_multicast *mc_list;
+ struct ibv_pd *pd;
+ struct ibv_qp_init_attr *qp_init_attr;
+ uint8_t initiator_depth;
+ uint8_t responder_resources;
};
struct cma_multicast {
@@ -1993,6 +1995,34 @@ int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel)
return 0;
}
+static int ucma_passive_ep(struct rdma_cm_id *id, struct rdma_addrinfo *res,
+ struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
+{
+ struct cma_id_private *id_priv;
+ int ret;
+
+ if (af_ib_support)
+ ret = rdma_bind_addr2(id, res->ai_src_addr, res->ai_src_len);
+ else
+ ret = rdma_bind_addr(id, res->ai_src_addr);
+ if (ret)
+ return ret;
+
+ id_priv = container_of(id, struct cma_id_private, id);
+ id_priv->pd = pd;
+
+ if (qp_init_attr) {
+ id_priv->qp_init_attr = malloc(sizeof *qp_init_attr);
+ if (!id_priv->qp_init_attr)
+ return ERR(ENOMEM);
+
+ *id_priv->qp_init_attr = *qp_init_attr;
+ id_priv->qp_init_attr->qp_type = res->ai_qp_type;
+ }
+
+ return 0;
+}
+
int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
{
@@ -2003,6 +2033,13 @@ int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
if (ret)
return ret;
+ if (res->ai_flags & RAI_PASSIVE) {
+ ret = ucma_passive_ep(cm_id, res, pd, qp_init_attr);
+ if (ret)
+ goto err;
+ goto out;
+ }
+
if (af_ib_support)
ret = rdma_resolve_addr2(cm_id, res->ai_src_addr, res->ai_src_len,
res->ai_dst_addr, res->ai_dst_len, 2000);
@@ -2027,17 +2064,25 @@ int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
if (ret)
goto err;
+out:
*id = cm_id;
return 0;
err:
- rdma_destroy_id(cm_id);
+ rdma_destroy_ep(cm_id);
return ret;
}
void rdma_destroy_ep(struct rdma_cm_id *id)
{
+ struct cma_id_private *id_priv;
+
if (id->qp)
rdma_destroy_qp(id);
+
+ id_priv = container_of(id, struct cma_id_private, id);
+ if (id_priv->qp_init_attr)
+ free(id_priv->qp_init_attr);
+
rdma_destroy_id(id);
}
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 19/37] librdmacm: add rdma_get_request
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
To simplify passive side operation and better support synchronous
operations, add rdma_get_request(). This function is called on the
listening side to retrieve a connection request event.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Ideally, this call would have been rdma_accept, to match with the socket
accept call, but it was already taken.
include/rdma/rdma_cma.h | 5 +++++
src/cma.c | 38 ++++++++++++++++++++++++++++++++++++++
src/librdmacm.map | 1 +
3 files changed, 44 insertions(+), 0 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 1db559e..89013a0 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -381,6 +381,11 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param);
int rdma_listen(struct rdma_cm_id *id, int backlog);
/**
+ * rdma_get_request
+ */
+int rdma_get_request(struct rdma_cm_id *listen, struct rdma_cm_id **id);
+
+/**
* rdma_accept - Called to accept a connection request.
* @id: Connection identifier associated with the request.
* @conn_param: Optional information needed to establish the connection.
diff --git a/src/cma.c b/src/cma.c
index 8aa7b05..9de33d4 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -1242,6 +1242,44 @@ int rdma_listen(struct rdma_cm_id *id, int backlog)
return ucma_query_route(id);
}
+int rdma_get_request(struct rdma_cm_id *listen, struct rdma_cm_id **id)
+{
+ struct cma_id_private *id_priv;
+ struct rdma_cm_event *event;
+ int ret;
+
+ id_priv = container_of(listen, struct cma_id_private, id);
+ if (!id_priv->sync)
+ return ERR(EINVAL);
+
+ if (listen->event) {
+ rdma_ack_cm_event(listen->event);
+ listen->event = NULL;
+ }
+
+ ret = rdma_get_cm_event(listen->channel, &event);
+ if (ret)
+ return ret;
+
+ if (event->status) {
+ ret = event->status;
+ goto err;
+ }
+
+ if (event->event != RDMA_CM_EVENT_CONNECT_REQUEST) {
+ ret = ERR(EINVAL);
+ goto err;
+ }
+
+ *id = event->id;
+ (*id)->event = event;
+ return 0;
+
+err:
+ listen->event = event;
+ return ret;
+}
+
int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
{
struct ucma_abi_accept *cmd;
diff --git a/src/librdmacm.map b/src/librdmacm.map
index 1f07102..f6af452 100644
--- a/src/librdmacm.map
+++ b/src/librdmacm.map
@@ -30,5 +30,6 @@ RDMACM_1.0 {
rdma_migrate_id;
rdma_getaddrinfo;
rdma_freeaddrinfo;
+ rdma_get_request;
local: *;
};
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 11/37] librdmacm: add zalloc call
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/cma.c | 6 ++----
src/cma.h | 10 ++++++++++
2 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/src/cma.c b/src/cma.c
index a85448b..4025aeb 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -342,11 +342,10 @@ static struct cma_id_private *ucma_alloc_id(struct rdma_event_channel *channel,
{
struct cma_id_private *id_priv;
- id_priv = malloc(sizeof *id_priv);
+ id_priv = zalloc(sizeof *id_priv);
if (!id_priv)
return NULL;
- memset(id_priv, 0, sizeof *id_priv);
id_priv->id.context = context;
id_priv->id.ps = ps;
id_priv->id.channel = channel;
@@ -1228,11 +1227,10 @@ static int rdma_join_multicast2(struct rdma_cm_id *id, struct sockaddr *addr,
int ret, size;
id_priv = container_of(id, struct cma_id_private, id);
- mc = malloc(sizeof *mc);
+ mc = zalloc(sizeof *mc);
if (!mc)
return ERR(ENOMEM);
- memset(mc, 0, sizeof *mc);
mc->context = context;
mc->id_priv = id_priv;
memcpy(&mc->addr, addr, addrlen);
diff --git a/src/cma.h b/src/cma.h
index 1c0ab8b..fcfb1f7 100644
--- a/src/cma.h
+++ b/src/cma.h
@@ -42,6 +42,7 @@
#include <errno.h>
#include <endian.h>
#include <byteswap.h>
+#include <string.h>
#ifdef INCLUDE_VALGRIND
# include <valgrind/memcheck.h>
@@ -70,5 +71,14 @@ static inline int ERR(int err)
return -1;
}
+static inline void *zalloc(size_t size)
+{
+ void *buf;
+
+ if ((buf = malloc(size)))
+ memset(buf, 0, size);
+ return buf;
+}
+
#endif /* CMA_H */
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 3/37] librdmacm: add ability to query IB path records
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
The current query_route command only supports 2 path records.
Add support for query_path, which is capable of supporting
multiple paths.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma_abi.h | 10 +++++
src/cma.c | 82 +++++++++++++++++++++++++++++++++++++++++++
2 files changed, 91 insertions(+), 1 deletions(-)
diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index 5c736fb..6c83fe8 100644
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -35,6 +35,7 @@
#include <infiniband/kern-abi.h>
#include <infiniband/sa-kern-abi.h>
+#include <infiniband/sa.h>
/*
* This file must be kept in sync with the kernel's version of rdma_user_cm.h
@@ -114,7 +115,8 @@ struct ucma_abi_resolve_route {
};
enum {
- UCMA_QUERY_ADDR
+ UCMA_QUERY_ADDR,
+ UCMA_QUERY_PATH
};
struct ucma_abi_query {
@@ -144,6 +146,12 @@ struct ucma_abi_query_addr_resp {
struct sockaddr_storage dst_addr;
};
+struct ucma_abi_query_path_resp {
+ __u32 num_paths;
+ __u32 reserved;
+ struct ib_path_data path_data[0];
+};
+
struct ucma_abi_conn_param {
__u32 qp_num;
__u32 reserved;
diff --git a/src/cma.c b/src/cma.c
index 2aef594..c3c6b73 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -506,6 +506,88 @@ static int ucma_query_addr(struct rdma_cm_id *id)
return 0;
}
+static void ucma_convert_path(struct ib_path_data *path_data,
+ struct ibv_sa_path_rec *sa_path)
+{
+ uint32_t fl_hop;
+
+ sa_path->dgid = path_data->path.dgid;
+ sa_path->sgid = path_data->path.sgid;
+ sa_path->dlid = path_data->path.dlid;
+ sa_path->slid = path_data->path.slid;
+ sa_path->raw_traffic = 0;
+
+ fl_hop = ntohl(path_data->path.flowlabel_hoplimit);
+ sa_path->flow_label = htonl(fl_hop >> 8);
+ sa_path->hop_limit = (uint8_t) fl_hop;
+
+ sa_path->traffic_class = path_data->path.tclass;
+ sa_path->reversible = path_data->path.reversible_numpath >> 7;
+ sa_path->numb_path = 1;
+ sa_path->pkey = path_data->path.pkey;
+ sa_path->sl = ntohs(path_data->path.qosclass_sl) & 0xF;
+ sa_path->mtu_selector = 1;
+ sa_path->mtu = path_data->path.mtu & 0x1F;
+ sa_path->rate_selector = 1;
+ sa_path->rate = path_data->path.rate & 0x1F;
+ sa_path->packet_life_time_selector = 1;
+ sa_path->packet_life_time = path_data->path.packetlifetime & 0x1F;
+
+ sa_path->preference = (uint8_t) path_data->flags;
+}
+
+static int ucma_query_path(struct rdma_cm_id *id)
+{
+ struct ucma_abi_query_path_resp *resp;
+ struct ucma_abi_query *cmd;
+ struct ucma_abi_cmd_hdr *hdr;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size, i;
+
+ size = sizeof(*hdr) + sizeof(*cmd);
+ msg = alloca(size);
+ if (!msg)
+ return ERR(ENOMEM);
+
+ hdr = msg;
+ cmd = msg + sizeof(*hdr);
+
+ hdr->cmd = UCMA_CMD_QUERY;
+ hdr->in = sizeof(*cmd);
+ hdr->out = sizeof(*resp) + sizeof(struct ib_path_data) * 6;
+
+ memset(cmd, 0, sizeof(*cmd));
+
+ resp = alloca(hdr->out);
+ if (!resp)
+ return ERR(ENOMEM);
+
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->response = (uintptr_t) resp;
+ cmd->id = id_priv->handle;
+ cmd->option = UCMA_QUERY_PATH;
+
+ ret = write(id->channel->fd, msg, size);
+ if (ret != size)
+ return (ret >= 0) ? ERR(ENODATA) : -1;
+
+ VALGRIND_MAKE_MEM_DEFINED(resp, hdr->out);
+
+ if (resp->num_paths) {
+ id->route.path_rec = malloc(sizeof(*id->route.path_rec) *
+ resp->num_paths);
+ if (!id->route.path_rec)
+ return ERR(ENOMEM);
+
+ id->route.num_paths = resp->num_paths;
+ for (i = 0; i < resp->num_paths; i++)
+ ucma_convert_path(&resp->path_data[i], &id->route.path_rec[i]);
+ }
+
+ return 0;
+}
+
static int ucma_query_route(struct rdma_cm_id *id)
{
struct ucma_abi_query_route_resp *resp;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 18/37] librdmacm: add rdma_getaddrinfo
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Provide a call similar to getaddrinfo for RDMA devices and
connections. rdma_get_addrinfo is modeled after getaddrinfo, with
the following modifications:
A source address is returned as part of the call to allow the
user to allocate the necessary resources for connections.
Optional routing information may be returned to support
Infiniband fabrics. IB routing information includes necessary
path record data.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Makefile.am | 2
include/rdma/rdma_cma.h | 31 +++++++
src/addrinfo.c | 201 +++++++++++++++++++++++++++++++++++++++++++++++
src/librdmacm.map | 2
4 files changed, 234 insertions(+), 2 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index c9be437..be53c78 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -12,7 +12,7 @@ else
librdmacm_version_script =
endif
-src_librdmacm_la_SOURCES = src/cma.c
+src_librdmacm_la_SOURCES = src/cma.c src/addrinfo.c
src_librdmacm_la_LDFLAGS = -version-info 1 -export-dynamic \
$(librdmacm_version_script)
src_librdmacm_la_DEPENDENCIES = $(srcdir)/src/librdmacm.map
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 88b3796..1db559e 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2005 Voltaire Inc. All rights reserved.
- * Copyright (c) 2005-2007 Intel Corporation. All rights reserved.
+ * Copyright (c) 2005-2010 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -158,6 +158,26 @@ struct rdma_cm_event {
} param;
};
+#define RAI_PASSIVE 0x00000001
+
+struct rdma_addrinfo {
+ int ai_flags;
+ int ai_family;
+ int ai_qp_type;
+ int ai_port_space;
+ socklen_t ai_src_len;
+ socklen_t ai_dst_len;
+ struct sockaddr *ai_src_addr;
+ struct sockaddr *ai_dst_addr;
+ char *ai_src_canonname;
+ char *ai_dst_canonname;
+ size_t ai_route_len;
+ void *ai_route;
+ size_t ai_connect_len;
+ void *ai_connect;
+ struct rdma_addrinfo *ai_next;
+};
+
/**
* rdma_create_event_channel - Open a channel used to report communication events.
* Description:
@@ -589,6 +609,15 @@ int rdma_set_option(struct rdma_cm_id *id, int level, int optname,
*/
int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel);
+/**
+ * rdma_getaddrinfo - RDMA address and route resolution service.
+ */
+int rdma_getaddrinfo(char *node, char *service,
+ struct rdma_addrinfo *hints,
+ struct rdma_addrinfo **res);
+
+void rdma_freeaddrinfo(struct rdma_addrinfo *res);
+
#ifdef __cplusplus
}
#endif
diff --git a/src/addrinfo.c b/src/addrinfo.c
new file mode 100644
index 0000000..15ae071
--- /dev/null
+++ b/src/addrinfo.c
@@ -0,0 +1,201 @@
+/*
+ * Copyright (c) 2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id: cm.c 3453 2005-09-15 21:43:21Z sean.hefty $
+ */
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+
+#include "cma.h"
+#include <rdma/rdma_cma.h>
+#include <infiniband/ib.h>
+
+static void ucma_convert_to_ai(struct addrinfo *ai, struct rdma_addrinfo *rai)
+{
+ memset(ai, 0, sizeof *ai);
+ ai->ai_flags = rai->ai_flags;
+ ai->ai_family = rai->ai_family;
+
+ switch (rai->ai_qp_type) {
+ case IBV_QPT_RC:
+ ai->ai_socktype = SOCK_STREAM;
+ break;
+ case IBV_QPT_UD:
+ ai->ai_socktype = SOCK_DGRAM;
+ break;
+ }
+
+ switch (rai->ai_port_space) {
+ case RDMA_PS_TCP:
+ ai->ai_protocol = IPPROTO_TCP;
+ break;
+ case RDMA_PS_IPOIB:
+ case RDMA_PS_UDP:
+ ai->ai_protocol = IPPROTO_UDP;
+ break;
+ }
+
+ if (rai->ai_flags & RAI_PASSIVE) {
+ ai->ai_addrlen = rai->ai_src_len;
+ ai->ai_addr = rai->ai_src_addr;
+ } else {
+ ai->ai_addrlen = rai->ai_dst_len;
+ ai->ai_addr = rai->ai_dst_addr;
+ }
+ ai->ai_canonname = rai->ai_dst_canonname;
+ ai->ai_next = NULL;
+}
+
+static int ucma_convert_to_rai(struct rdma_addrinfo *rai, struct addrinfo *ai)
+{
+ struct sockaddr *addr;
+ char *canonname;
+
+ memset(rai, 0, sizeof *rai);
+ rai->ai_flags = ai->ai_flags;
+ rai->ai_family = ai->ai_family;
+
+ switch (ai->ai_socktype) {
+ case SOCK_STREAM:
+ rai->ai_qp_type = IBV_QPT_RC;
+ break;
+ case SOCK_DGRAM:
+ rai->ai_qp_type = IBV_QPT_UD;
+ break;
+ }
+
+ switch (ai->ai_protocol) {
+ case IPPROTO_TCP:
+ rai->ai_port_space = RDMA_PS_TCP;
+ break;
+ case IPPROTO_UDP:
+ rai->ai_port_space = RDMA_PS_UDP;
+ break;
+ }
+
+ addr = malloc(ai->ai_addrlen);
+ if (!addr)
+ return ERR(ENOMEM);
+
+ canonname = ai->ai_canonname ? malloc(strlen(ai->ai_canonname) + 1) : NULL;
+ if (canonname)
+ strcpy(canonname, ai->ai_canonname);
+
+ memcpy(addr, ai->ai_addr, ai->ai_addrlen);
+ if (ai->ai_flags & RAI_PASSIVE) {
+ rai->ai_src_addr = addr;
+ rai->ai_src_len = ai->ai_addrlen;
+ rai->ai_src_canonname = canonname;
+ } else {
+ rai->ai_dst_addr = addr;
+ rai->ai_dst_len = ai->ai_addrlen;
+ rai->ai_dst_canonname = canonname;
+ }
+
+ return 0;
+}
+
+int rdma_getaddrinfo(char *node, char *service,
+ struct rdma_addrinfo *hints,
+ struct rdma_addrinfo **res)
+{
+ struct rdma_addrinfo *rai;
+ struct addrinfo ai_hints;
+ struct addrinfo *ai;
+ int ret;
+
+ ret = ucma_init();
+ if (ret)
+ return ret;
+
+ if (hints)
+ ucma_convert_to_ai(&ai_hints, hints);
+
+ ret = getaddrinfo(node, service, &ai_hints, &ai);
+ if (ret)
+ return ret;
+
+ rai = malloc(sizeof(*rai));
+ if (!rai) {
+ ret = ERR(ENOMEM);
+ goto err1;
+ }
+
+ ret = ucma_convert_to_rai(rai, ai);
+ if (ret)
+ goto err2;
+
+ freeaddrinfo(ai);
+ *res = rai;
+ return 0;
+
+err2:
+ rdma_freeaddrinfo(rai);
+err1:
+ freeaddrinfo(ai);
+ return ret;
+}
+
+void rdma_freeaddrinfo(struct rdma_addrinfo *res)
+{
+ struct rdma_addrinfo *rai;
+
+ while (res) {
+ rai = res;
+ res = res->ai_next;
+
+ if (rai->ai_connect)
+ free(rai->ai_connect);
+
+ if (rai->ai_route)
+ free(rai->ai_route);
+
+ if (rai->ai_src_canonname)
+ free(rai->ai_src_canonname);
+
+ if (rai->ai_dst_canonname)
+ free(rai->ai_dst_canonname);
+
+ if (rai->ai_src_addr)
+ free(rai->ai_src_addr);
+
+ if (rai->ai_dst_addr)
+ free(rai->ai_dst_addr);
+
+ free(rai);
+ }
+}
diff --git a/src/librdmacm.map b/src/librdmacm.map
index cb94efe..1f07102 100644
--- a/src/librdmacm.map
+++ b/src/librdmacm.map
@@ -28,5 +28,7 @@ RDMACM_1.0 {
rdma_get_local_addr;
rdma_get_peer_addr;
rdma_migrate_id;
+ rdma_getaddrinfo;
+ rdma_freeaddrinfo;
local: *;
};
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 14/37] librdmacm: make CQs optional for rdma_create_qp
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Allow the user to specify NULL for the send and receive CQs when
creating a QP through rdma_create_qp. The librdmacm will automatically
create CQs for the user, along with completion channel.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 4 +++
src/cma.c | 74 ++++++++++++++++++++++++++++++++++++++++++++---
2 files changed, 74 insertions(+), 4 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index ccf6cd4..d8cbb91 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -115,6 +115,10 @@ struct rdma_cm_id {
enum rdma_port_space ps;
uint8_t port_num;
struct rdma_cm_event *event;
+ struct ibv_comp_channel *send_cq_channel;
+ struct ibv_cq *send_cq;
+ struct ibv_comp_channel *recv_cq_channel;
+ struct ibv_cq *recv_cq;
};
struct rdma_conn_param {
diff --git a/src/cma.c b/src/cma.c
index 0587ab3..805aca3 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -1025,6 +1025,63 @@ static int ucma_init_ud_qp(struct cma_id_private *id_priv, struct ibv_qp *qp)
return ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_SQ_PSN);
}
+static void ucma_destroy_cqs(struct rdma_cm_id *id)
+{
+ if (id->recv_cq)
+ ibv_destroy_cq(id->recv_cq);
+
+ if (id->recv_cq_channel)
+ ibv_destroy_comp_channel(id->recv_cq_channel);
+
+ if (id->send_cq)
+ ibv_destroy_cq(id->send_cq);
+
+ if (id->send_cq_channel)
+ ibv_destroy_comp_channel(id->send_cq_channel);
+}
+
+static int ucma_create_cqs(struct rdma_cm_id *id, struct ibv_qp_init_attr *attr)
+{
+ int ret;
+
+ if (!attr->recv_cq) {
+ id->recv_cq_channel = ibv_create_comp_channel(id->verbs);
+ if (!id->recv_cq_channel) {
+ ret = ERR(ENOMEM);
+ goto err;
+ }
+
+ id->recv_cq = ibv_create_cq(id->verbs, attr->cap.max_recv_wr,
+ id, id->recv_cq_channel, 0);
+ if (!id->recv_cq) {
+ ret = ERR(ENOMEM);
+ goto err;
+ }
+ attr->recv_cq = id->recv_cq;
+ }
+
+ if (!attr->send_cq) {
+ id->send_cq_channel = ibv_create_comp_channel(id->verbs);
+ if (!id->send_cq_channel) {
+ ret = ERR(ENOMEM);
+ goto err;
+ }
+
+ id->send_cq = ibv_create_cq(id->verbs, attr->cap.max_send_wr,
+ id, id->send_cq_channel, 0);
+ if (!id->send_cq) {
+ ret = ERR(ENOMEM);
+ goto err;
+ }
+ attr->send_cq = id->send_cq;
+ }
+
+ return 0;
+err:
+ ucma_destroy_cqs(id);
+ return ret;
+}
+
int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd,
struct ibv_qp_init_attr *qp_init_attr)
{
@@ -1038,27 +1095,36 @@ int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd,
else if (id->verbs != pd->context)
return ERR(EINVAL);
+ ret = ucma_create_cqs(id, qp_init_attr);
+ if (ret)
+ return ret;
+
qp = ibv_create_qp(pd, qp_init_attr);
- if (!qp)
- return ERR(ENOMEM);
+ if (!qp) {
+ ret = ERR(ENOMEM);
+ goto err1;
+ }
if (ucma_is_ud_ps(id->ps))
ret = ucma_init_ud_qp(id_priv, qp);
else
ret = ucma_init_conn_qp(id_priv, qp);
if (ret)
- goto err;
+ goto err2;
id->qp = qp;
return 0;
-err:
+err2:
ibv_destroy_qp(qp);
+err1:
+ ucma_destroy_cqs(id);
return ret;
}
void rdma_destroy_qp(struct rdma_cm_id *id)
{
ibv_destroy_qp(id->qp);
+ ucma_destroy_cqs(id);
}
static int ucma_valid_param(struct cma_id_private *id_priv,
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 2/37] librdmacm: support querying AF_IB addresses
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
The current query route command returns path record data and address
information. The latter is restricted to sizeof(sockaddr_in6). In
order to support AF_IB, modify the library to use the new query addr
command, which supports larger address sizes and avoids querying for
path records data when none are available.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma_abi.h | 22 +++++++++++++++++++---
src/cma.c | 35 ++++++++++++++++++++++++++++++++++-
2 files changed, 53 insertions(+), 4 deletions(-)
diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index e51a372..5c736fb 100644
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -64,7 +64,8 @@ enum {
UCMA_CMD_NOTIFY,
UCMA_CMD_JOIN_IP_MCAST,
UCMA_CMD_LEAVE_MCAST,
- UCMA_CMD_MIGRATE_ID
+ UCMA_CMD_MIGRATE_ID,
+ UCMA_CMD_QUERY
};
struct ucma_abi_cmd_hdr {
@@ -112,10 +113,14 @@ struct ucma_abi_resolve_route {
__u32 timeout_ms;
};
-struct ucma_abi_query_route {
+enum {
+ UCMA_QUERY_ADDR
+};
+
+struct ucma_abi_query {
__u64 response;
__u32 id;
- __u32 reserved;
+ __u32 option;
};
struct ucma_abi_query_route_resp {
@@ -128,6 +133,17 @@ struct ucma_abi_query_route_resp {
__u8 reserved[3];
};
+struct ucma_abi_query_addr_resp {
+ __u64 node_guid;
+ __u8 port_num;
+ __u8 reserved;
+ __u16 pkey;
+ __u16 src_size;
+ __u16 dst_size;
+ struct sockaddr_storage src_addr;
+ struct sockaddr_storage dst_addr;
+};
+
struct ucma_abi_conn_param {
__u32 qp_num;
__u32 reserved;
diff --git a/src/cma.c b/src/cma.c
index b5f71d0..2aef594 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -473,10 +473,43 @@ static int ucma_addrlen(struct sockaddr *addr)
}
}
+static int ucma_query_addr(struct rdma_cm_id *id)
+{
+ struct ucma_abi_query_addr_resp *resp;
+ struct ucma_abi_query *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_QUERY, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ cmd->option = UCMA_QUERY_ADDR;
+
+ ret = write(id->channel->fd, msg, size);
+ if (ret != size)
+ return (ret >= 0) ? ERR(ENODATA) : -1;
+
+ VALGRIND_MAKE_MEM_DEFINED(resp, sizeof *resp);
+
+ memcpy(&id->route.addr.src_addr, &resp->src_addr, resp->src_size);
+ memcpy(&id->route.addr.dst_addr, &resp->dst_addr, resp->dst_size);
+
+ if (!id_priv->cma_dev && resp->node_guid) {
+ ret = ucma_get_device(id_priv, resp->node_guid);
+ if (ret)
+ return ret;
+ id->port_num = resp->port_num;
+ id->route.addr.addr.ibaddr.pkey = resp->pkey;
+ }
+
+ return 0;
+}
+
static int ucma_query_route(struct rdma_cm_id *id)
{
struct ucma_abi_query_route_resp *resp;
- struct ucma_abi_query_route *cmd;
+ struct ucma_abi_query *cmd;
struct cma_id_private *id_priv;
void *msg;
int ret, size, i;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 17/37] librdmacm: expose ucma_init to other internal modules
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Remove static property from ucma_init and expose its
definition in cma.h. The address resolution module will
need access to this function.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/cma.c | 14 +++++++++-----
src/cma.h | 2 ++
2 files changed, 11 insertions(+), 5 deletions(-)
diff --git a/src/cma.c b/src/cma.c
index 6ef4b96..8aa7b05 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -188,13 +188,17 @@ static int check_abi_version(void)
return 0;
}
-static int ucma_init(void)
+int ucma_init(void)
{
struct ibv_device **dev_list = NULL;
struct cma_device *cma_dev;
struct ibv_device_attr attr;
int i, ret, dev_cnt;
+ /* Quick check without lock to see if we're already initialized */
+ if (cma_dev_cnt)
+ return 0;
+
pthread_mutex_lock(&mut);
if (cma_dev_cnt) {
pthread_mutex_unlock(&mut);
@@ -271,7 +275,7 @@ struct ibv_context **rdma_get_devices(int *num_devices)
struct ibv_context **devs = NULL;
int i;
- if (!cma_dev_cnt && ucma_init())
+ if (ucma_init())
goto out;
devs = malloc(sizeof *devs * (cma_dev_cnt + 1));
@@ -301,7 +305,7 @@ struct rdma_event_channel *rdma_create_event_channel(void)
{
struct rdma_event_channel *channel;
- if (!cma_dev_cnt && ucma_init())
+ if (ucma_init())
return NULL;
channel = malloc(sizeof *channel);
@@ -396,7 +400,7 @@ int rdma_create_id(struct rdma_event_channel *channel,
void *msg;
int ret, size;
- ret = cma_dev_cnt ? 0 : ucma_init();
+ ret = ucma_init();
if (ret)
return ret;
@@ -1712,7 +1716,7 @@ int rdma_get_cm_event(struct rdma_event_channel *channel,
void *msg;
int ret, size;
- ret = cma_dev_cnt ? 0 : ucma_init();
+ ret = ucma_init();
if (ret)
return ret;
diff --git a/src/cma.h b/src/cma.h
index 92e771e..06ca38c 100644
--- a/src/cma.h
+++ b/src/cma.h
@@ -82,5 +82,7 @@ static inline void *zalloc(size_t size)
return buf;
}
+int ucma_init();
+
#endif /* CMA_H */
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox