* [PATCH 13/37] librdmacm: allow pd parameter to be optional
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Allow the user to create a QP using rdma_create_qp without
specifying a PD. If a PD is not given, a default PD will be
used instead. This simplifies the user interface.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 4 +++-
src/cma.c | 24 +++++++++++++++++++-----
2 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 83418c3..ccf6cd4 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -279,7 +279,7 @@ int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms);
/**
* rdma_create_qp - Allocate a QP.
* @id: RDMA identifier.
- * @pd: protection domain for the QP.
+ * @pd: Optional protection domain for the QP.
* @qp_init_attr: initial QP attributes.
* Description:
* Allocate a QP associated with the specified rdma_cm_id and transition it
@@ -291,6 +291,8 @@ int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms);
* librdmacm through their states. After being allocated, the QP will be
* ready to handle posting of receives. If the QP is unconnected, it will
* be ready to post sends.
+ * If pd is NULL, then the QP will be allocated using a default protection
+ * domain associated with the underlying RDMA device.
* See also:
* rdma_bind_addr, rdma_resolve_addr, rdma_destroy_qp, ibv_create_qp,
* ibv_modify_qp
diff --git a/src/cma.c b/src/cma.c
index c7a3a7b..0587ab3 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -95,6 +95,7 @@ do { \
struct cma_device {
struct ibv_context *verbs;
+ struct ibv_pd *pd;
uint64_t guid;
int port_cnt;
uint8_t max_initiator_depth;
@@ -144,9 +145,11 @@ int af_ib_support;
static void ucma_cleanup(void)
{
if (cma_dev_cnt) {
- while (cma_dev_cnt)
- ibv_close_device(cma_dev_array[--cma_dev_cnt].verbs);
-
+ while (cma_dev_cnt--) {
+ ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd);
+ ibv_close_device(cma_dev_array[cma_dev_cnt].verbs);
+ }
+
free(cma_dev_array);
cma_dev_cnt = 0;
}
@@ -224,6 +227,13 @@ static int ucma_init(void)
goto err3;
}
+ cma_dev->pd = ibv_alloc_pd(cma_dev->verbs);
+ if (!cma_dev->pd) {
+ ibv_close_device(cma_dev->verbs);
+ ret = ERR(ENOMEM);
+ goto err3;
+ }
+
i++;
ret = ibv_query_device(cma_dev->verbs, &attr);
if (ret) {
@@ -242,8 +252,10 @@ static int ucma_init(void)
return 0;
err3:
- while (i--)
+ while (i--) {
+ ibv_dealloc_pd(cma_dev_array[i].pd);
ibv_close_device(cma_dev_array[i].verbs);
+ }
free(cma_dev_array);
err2:
ibv_free_device_list(dev_list);
@@ -1021,7 +1033,9 @@ int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd,
int ret;
id_priv = container_of(id, struct cma_id_private, id);
- if (id->verbs != pd->context)
+ if (!pd)
+ pd = id_priv->cma_dev->pd;
+ else if (id->verbs != pd->context)
return ERR(EINVAL);
qp = ibv_create_qp(pd, qp_init_attr);
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 21/37] librdmacm: specify qp_type when creating id
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
To support AF_IB / PS_IB, we need to specify the qp type when
creating the rdma_cm_id. The kernel requires this in order
to select the correct type of operation to perform (e.g. SIDR
versus REQ).
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma_abi.h | 3 ++-
src/cma.c | 18 +++++++++++++++---
2 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index c3981e6..bd4ca0f 100644
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -82,7 +82,8 @@ struct ucma_abi_create_id {
__u64 uid;
__u64 response;
__u16 ps;
- __u8 reserved[6];
+ __u8 qp_type;
+ __u8 reserved[5];
};
struct ucma_abi_create_id_resp {
diff --git a/src/cma.c b/src/cma.c
index 9de33d4..e31fb8a 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -390,9 +390,9 @@ err: ucma_free_id(id_priv);
return NULL;
}
-int rdma_create_id(struct rdma_event_channel *channel,
- struct rdma_cm_id **id, void *context,
- enum rdma_port_space ps)
+static int rdma_create_id2(struct rdma_event_channel *channel,
+ struct rdma_cm_id **id, void *context,
+ enum rdma_port_space ps, enum ibv_qp_type qp_type)
{
struct ucma_abi_create_id_resp *resp;
struct ucma_abi_create_id *cmd;
@@ -411,6 +411,7 @@ int rdma_create_id(struct rdma_event_channel *channel,
CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_CREATE_ID, size);
cmd->uid = (uintptr_t) id_priv;
cmd->ps = ps;
+ cmd->qp_type = qp_type;
ret = write(id_priv->id.channel->fd, msg, size);
if (ret != size)
@@ -426,6 +427,17 @@ err: ucma_free_id(id_priv);
return ret;
}
+int rdma_create_id(struct rdma_event_channel *channel,
+ struct rdma_cm_id **id, void *context,
+ enum rdma_port_space ps)
+{
+ enum ibv_qp_type qp_type;
+
+ qp_type = (ps == RDMA_PS_IPOIB || ps == RDMA_PS_UDP) ?
+ IBV_QPT_UD : IBV_QPT_RC;
+ return rdma_create_id2(channel, id, context, ps, qp_type);
+}
+
static int ucma_destroy_kern_id(int fd, uint32_t handle)
{
struct ucma_abi_destroy_id_resp *resp;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 8/37] librdmacm: add support for PF_IB to resolve_addr
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Allow user to specify PF_IB addresses to rdma_resolve_addr.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma_abi.h | 13 ++++++++++++-
src/cma.c | 44 +++++++++++++++++++++++++++++++++++++------
2 files changed, 50 insertions(+), 7 deletions(-)
diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index 8add397..4a7a55d 100644
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -67,7 +67,8 @@ enum {
UCMA_CMD_LEAVE_MCAST,
UCMA_CMD_MIGRATE_ID,
UCMA_CMD_QUERY,
- UCMA_CMD_BIND
+ UCMA_CMD_BIND,
+ UCMA_CMD_RESOLVE_ADDR
};
struct ucma_abi_cmd_hdr {
@@ -117,6 +118,16 @@ struct ucma_abi_resolve_ip {
__u32 timeout_ms;
};
+struct ucma_abi_resolve_addr {
+ __u32 id;
+ __u32 timeout_ms;
+ __u16 src_size;
+ __u16 dst_size;
+ __u32 reserved;
+ struct sockaddr_storage src_addr;
+ struct sockaddr_storage dst_addr;
+};
+
struct ucma_abi_resolve_route {
__u32 id;
__u32 timeout_ms;
diff --git a/src/cma.c b/src/cma.c
index be61333..e22e1b4 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -721,31 +721,63 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
return ucma_query_route(id);
}
+static int rdma_resolve_addr2(struct rdma_cm_id *id, struct sockaddr *src_addr,
+ socklen_t src_len, struct sockaddr *dst_addr,
+ socklen_t dst_len, int timeout_ms)
+{
+ struct ucma_abi_resolve_addr *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_RESOLVE_ADDR, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ if ((cmd->src_size = src_len))
+ memcpy(&cmd->src_addr, src_addr, src_len);
+ memcpy(&cmd->dst_addr, dst_addr, dst_len);
+ cmd->dst_size = dst_len;
+ cmd->timeout_ms = timeout_ms;
+ cmd->reserved = 0;
+
+ ret = write(id->channel->fd, msg, size);
+ if (ret != size)
+ return (ret >= 0) ? ERR(ENODATA) : -1;
+
+ memcpy(&id->route.addr.dst_addr, dst_addr, dst_len);
+ return 0;
+}
+
int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
struct sockaddr *dst_addr, int timeout_ms)
{
struct ucma_abi_resolve_ip *cmd;
struct cma_id_private *id_priv;
void *msg;
- int ret, size, daddrlen;
+ int ret, size, dst_len, src_len;
- daddrlen = ucma_addrlen(dst_addr);
- if (!daddrlen)
+ dst_len = ucma_addrlen(dst_addr);
+ if (!dst_len)
return ERR(EINVAL);
+ src_len = ucma_addrlen(src_addr);
+ if (af_ib_support)
+ return rdma_resolve_addr2(id, src_addr, src_len, dst_addr,
+ dst_len, timeout_ms);
+
CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_RESOLVE_IP, size);
id_priv = container_of(id, struct cma_id_private, id);
cmd->id = id_priv->handle;
if (src_addr)
- memcpy(&cmd->src_addr, src_addr, ucma_addrlen(src_addr));
- memcpy(&cmd->dst_addr, dst_addr, daddrlen);
+ memcpy(&cmd->src_addr, src_addr, src_len);
+ memcpy(&cmd->dst_addr, dst_addr, dst_len);
cmd->timeout_ms = timeout_ms;
ret = write(id->channel->fd, msg, size);
if (ret != size)
return (ret >= 0) ? ERR(ENODATA) : -1;
- memcpy(&id->route.addr.dst_addr, dst_addr, daddrlen);
+ memcpy(&id->route.addr.dst_addr, dst_addr, dst_len);
return 0;
}
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 12/37] librdmacm: support synchronous rdma_cm_id's
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Allow the user to specify NULL as the rdma_event_channel in
order to indicate that the rdma_cm_id should process all requests
synchronously.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 1 +
src/cma.c | 93 +++++++++++++++++++++++++++++++++++++++--------
2 files changed, 78 insertions(+), 16 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index a071a9b..83418c3 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -114,6 +114,7 @@ struct rdma_cm_id {
struct rdma_route route;
enum rdma_port_space ps;
uint8_t port_num;
+ struct rdma_cm_event *event;
};
struct rdma_conn_param {
diff --git a/src/cma.c b/src/cma.c
index 4025aeb..c7a3a7b 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -106,6 +106,7 @@ struct cma_id_private {
struct cma_device *cma_dev;
int events_completed;
int connect_error;
+ int sync;
pthread_cond_t cond;
pthread_mutex_t mut;
uint32_t handle;
@@ -333,6 +334,9 @@ static void ucma_free_id(struct cma_id_private *id_priv)
pthread_mutex_destroy(&id_priv->mut);
if (id_priv->id.route.path_rec)
free(id_priv->id.route.path_rec);
+
+ if (id_priv->sync)
+ rdma_destroy_event_channel(id_priv->id.channel);
free(id_priv);
}
@@ -348,7 +352,16 @@ static struct cma_id_private *ucma_alloc_id(struct rdma_event_channel *channel,
id_priv->id.context = context;
id_priv->id.ps = ps;
- id_priv->id.channel = channel;
+
+ if (!channel) {
+ id_priv->id.channel = rdma_create_event_channel();
+ if (!id_priv->id.channel)
+ goto err;
+ id_priv->sync = 1;
+ } else {
+ id_priv->id.channel = channel;
+ }
+
pthread_mutex_init(&id_priv->mut, NULL);
if (pthread_cond_init(&id_priv->cond, NULL))
goto err;
@@ -381,7 +394,7 @@ int rdma_create_id(struct rdma_event_channel *channel,
cmd->uid = (uintptr_t) id_priv;
cmd->ps = ps;
- ret = write(channel->fd, msg, size);
+ ret = write(id_priv->id.channel->fd, msg, size);
if (ret != size)
goto err;
@@ -424,6 +437,9 @@ int rdma_destroy_id(struct rdma_cm_id *id)
if (ret < 0)
return ret;
+ if (id_priv->id.event)
+ rdma_ack_cm_event(id_priv->id.event);
+
pthread_mutex_lock(&id_priv->mut);
while (id_priv->events_completed < ret)
pthread_cond_wait(&id_priv->cond, &id_priv->mut);
@@ -694,6 +710,25 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
return ucma_query_route(id);
}
+static int ucma_complete(struct cma_id_private *id_priv)
+{
+ int ret;
+
+ if (!id_priv->sync)
+ return 0;
+
+ if (id_priv->id.event) {
+ rdma_ack_cm_event(id_priv->id.event);
+ id_priv->id.event = NULL;
+ }
+
+ ret = rdma_get_cm_event(id_priv->id.channel, &id_priv->id.event);
+ if (ret)
+ return ret;
+
+ return id_priv->id.event->status;
+}
+
static int rdma_resolve_addr2(struct rdma_cm_id *id, struct sockaddr *src_addr,
socklen_t src_len, struct sockaddr *dst_addr,
socklen_t dst_len, int timeout_ms)
@@ -718,7 +753,7 @@ static int rdma_resolve_addr2(struct rdma_cm_id *id, struct sockaddr *src_addr,
return (ret >= 0) ? ERR(ENODATA) : -1;
memcpy(&id->route.addr.dst_addr, dst_addr, dst_len);
- return 0;
+ return ucma_complete(id_priv);
}
int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
@@ -751,7 +786,7 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
return (ret >= 0) ? ERR(ENODATA) : -1;
memcpy(&id->route.addr.dst_addr, dst_addr, dst_len);
- return 0;
+ return ucma_complete(id_priv);
}
int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
@@ -770,7 +805,7 @@ int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
if (ret != size)
return (ret >= 0) ? ERR(ENODATA) : -1;
- return 0;
+ return ucma_complete(id_priv);
}
static int ucma_is_ud_ps(enum rdma_port_space ps)
@@ -1074,7 +1109,7 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
if (ret != size)
return (ret >= 0) ? ERR(ENODATA) : -1;
- return 0;
+ return ucma_complete(id_priv);
}
int rdma_listen(struct rdma_cm_id *id, int backlog)
@@ -1139,7 +1174,7 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
return (ret >= 0) ? ERR(ENODATA) : -1;
}
- return 0;
+ return ucma_complete(id_priv);
}
int rdma_reject(struct rdma_cm_id *id, const void *private_data,
@@ -1214,7 +1249,7 @@ int rdma_disconnect(struct rdma_cm_id *id)
if (ret != size)
return (ret >= 0) ? ERR(ENODATA) : -1;
- return 0;
+ return ucma_complete(id_priv);
}
static int rdma_join_multicast2(struct rdma_cm_id *id, struct sockaddr *addr,
@@ -1271,7 +1306,8 @@ static int rdma_join_multicast2(struct rdma_cm_id *id, struct sockaddr *addr,
VALGRIND_MAKE_MEM_DEFINED(resp, sizeof *resp);
mc->handle = resp->id;
- return 0;
+ return ucma_complete(id_priv);
+
err2:
pthread_mutex_lock(&id_priv->mut);
for (pos = &id_priv->mc_list; *pos != mc; pos = &(*pos)->next)
@@ -1443,21 +1479,28 @@ static int ucma_process_conn_req(struct cma_event *evt,
if (!id_priv) {
ucma_destroy_kern_id(evt->id_priv->id.channel->fd, handle);
ret = ERR(ENOMEM);
- goto err;
+ goto err1;
}
evt->event.listen_id = &evt->id_priv->id;
evt->event.id = &id_priv->id;
id_priv->handle = handle;
- ret = ucma_query_req_info(&id_priv->id);
- if (ret) {
- rdma_destroy_id(&id_priv->id);
- goto err;
+ if (evt->id_priv->sync) {
+ ret = rdma_migrate_id(&id_priv->id, NULL);
+ if (ret)
+ goto err2;
}
+ ret = ucma_query_req_info(&id_priv->id);
+ if (ret)
+ goto err2;
+
return 0;
-err:
+
+err2:
+ rdma_destroy_id(&id_priv->id);
+err1:
ucma_complete_event(evt->id_priv);
return ret;
}
@@ -1728,9 +1771,18 @@ int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel)
struct ucma_abi_migrate_id *cmd;
struct cma_id_private *id_priv;
void *msg;
- int ret, size;
+ int ret, size, sync;
id_priv = container_of(id, struct cma_id_private, id);
+ if (id_priv->sync && !channel)
+ return ERR(EINVAL);
+
+ if ((sync = (channel == NULL))) {
+ channel = rdma_create_event_channel();
+ if (!channel)
+ return ERR(ENOMEM);
+ }
+
CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_MIGRATE_ID, size);
cmd->id = id_priv->handle;
cmd->fd = id->channel->fd;
@@ -1741,6 +1793,14 @@ int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel)
VALGRIND_MAKE_MEM_DEFINED(resp, sizeof *resp);
+ if (id_priv->sync) {
+ if (id->event) {
+ rdma_ack_cm_event(id->event);
+ id->event = NULL;
+ }
+ rdma_destroy_event_channel(id->channel);
+ }
+
/*
* Eventually if we want to support migrating channels while events are
* being processed on the current channel, we need to block here while
@@ -1749,6 +1809,7 @@ int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel)
* channel after this call returns.
*/
pthread_mutex_lock(&id_priv->mut);
+ id_priv->sync = sync;
id->channel = channel;
while (id_priv->events_completed < resp->events_reported)
pthread_cond_wait(&id_priv->cond, &id_priv->mut);
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 10/37] librdmacm: move common definitions to internal header file
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Makefile.am | 2 +-
src/cma.c | 28 +---------------------
src/cma.h | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 76 insertions(+), 28 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index 2898ad9..c9be437 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -70,7 +70,7 @@ man_MANS = \
EXTRA_DIST = include/rdma/rdma_cma_abi.h include/rdma/rdma_cma.h \
include/infiniband/ib.h \
- src/librdmacm.map librdmacm.spec.in $(man_MANS)
+ src/cma.h src/librdmacm.map librdmacm.spec.in $(man_MANS)
dist-hook: librdmacm.spec
cp librdmacm.spec $(distdir)
diff --git a/src/cma.c b/src/cma.c
index c83d9d2..a85448b 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -50,39 +50,13 @@
#include <byteswap.h>
#include <stddef.h>
+#include "cma.h"
#include <infiniband/driver.h>
#include <infiniband/marshall.h>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_cma_abi.h>
#include <infiniband/ib.h>
-#ifdef INCLUDE_VALGRIND
-# include <valgrind/memcheck.h>
-# ifndef VALGRIND_MAKE_MEM_DEFINED
-# warning "Valgrind requested, but VALGRIND_MAKE_MEM_DEFINED undefined"
-# endif
-#endif
-
-#ifndef VALGRIND_MAKE_MEM_DEFINED
-# define VALGRIND_MAKE_MEM_DEFINED(addr,len)
-#endif
-
-#define PFX "librdmacm: "
-
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-static inline uint64_t htonll(uint64_t x) { return bswap_64(x); }
-static inline uint64_t ntohll(uint64_t x) { return bswap_64(x); }
-#else
-static inline uint64_t htonll(uint64_t x) { return x; }
-static inline uint64_t ntohll(uint64_t x) { return x; }
-#endif
-
-static inline int ERR(int err)
-{
- errno = err;
- return -1;
-}
-
#define CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, type, size) \
do { \
struct ucma_abi_cmd_hdr *hdr; \
diff --git a/src/cma.h b/src/cma.h
new file mode 100644
index 0000000..1c0ab8b
--- /dev/null
+++ b/src/cma.h
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2005-2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#if !defined(CMA_H)
+#define CMA_H
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <stdlib.h>
+#include <errno.h>
+#include <endian.h>
+#include <byteswap.h>
+
+#ifdef INCLUDE_VALGRIND
+# include <valgrind/memcheck.h>
+# ifndef VALGRIND_MAKE_MEM_DEFINED
+# warning "Valgrind requested, but VALGRIND_MAKE_MEM_DEFINED undefined"
+# endif
+#endif
+
+#ifndef VALGRIND_MAKE_MEM_DEFINED
+# define VALGRIND_MAKE_MEM_DEFINED(addr,len)
+#endif
+
+#define PFX "librdmacm: "
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+static inline uint64_t htonll(uint64_t x) { return bswap_64(x); }
+static inline uint64_t ntohll(uint64_t x) { return bswap_64(x); }
+#else
+static inline uint64_t htonll(uint64_t x) { return x; }
+static inline uint64_t ntohll(uint64_t x) { return x; }
+#endif
+
+static inline int ERR(int err)
+{
+ errno = err;
+ return -1;
+}
+
+#endif /* CMA_H */
+
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 5/37] librdmacm: replace query_route call with separate queries
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
To support other address families and multiple path records,
replace the query_route call with specific query calls to obtain
only the desired information.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/cma.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++--------
1 files changed, 60 insertions(+), 9 deletions(-)
diff --git a/src/cma.c b/src/cma.c
index 2a70d20..c57d166 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -161,6 +161,7 @@ static struct cma_device *cma_dev_array;
static int cma_dev_cnt;
static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
static int abi_ver = RDMA_USER_CM_MAX_ABI_VERSION;
+int af_ib_support;
#define container_of(ptr, type, field) \
((type *) ((void *)ptr - offsetof(type, field)))
@@ -627,7 +628,7 @@ static int ucma_query_route(struct rdma_cm_id *id)
struct cma_id_private *id_priv;
void *msg;
int ret, size, i;
-
+
CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_QUERY_ROUTE, size);
id_priv = container_of(id, struct cma_id_private, id);
cmd->id = id_priv->handle;
@@ -1060,7 +1061,10 @@ int rdma_listen(struct rdma_cm_id *id, int backlog)
if (ret != size)
return (ret >= 0) ? ERR(ENODATA) : -1;
- return ucma_query_route(id);
+ if (af_ib_support)
+ return ucma_query_addr(id);
+ else
+ return ucma_query_route(id);
}
int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
@@ -1326,6 +1330,57 @@ int rdma_ack_cm_event(struct rdma_cm_event *event)
return 0;
}
+static void ucma_process_addr_resolved(struct cma_event *evt)
+{
+ if (af_ib_support) {
+ evt->event.status = ucma_query_addr(&evt->id_priv->id);
+ if (!evt->event.status &&
+ evt->id_priv->id.verbs->device->transport_type == IBV_TRANSPORT_IB)
+ evt->event.status = ucma_query_gid(&evt->id_priv->id);
+ } else {
+ evt->event.status = ucma_query_route(&evt->id_priv->id);
+ }
+
+ if (evt->event.status)
+ evt->event.event = RDMA_CM_EVENT_ADDR_ERROR;
+}
+
+static void ucma_process_route_resolved(struct cma_event *evt)
+{
+ if (evt->id_priv->id.verbs->device->transport_type != IBV_TRANSPORT_IB)
+ return;
+
+ if (af_ib_support)
+ evt->event.status = ucma_query_path(&evt->id_priv->id);
+ else
+ evt->event.status = ucma_query_route(&evt->id_priv->id);
+
+ if (evt->event.status)
+ evt->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
+}
+
+static int ucma_query_req_info(struct rdma_cm_id *id)
+{
+ int ret;
+
+ if (!af_ib_support)
+ return ucma_query_route(id);
+
+ ret = ucma_query_addr(id);
+ if (ret)
+ return ret;
+
+ ret = ucma_query_gid(id);
+ if (ret)
+ return ret;
+
+ ret = ucma_query_path(id);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
static int ucma_process_conn_req(struct cma_event *evt,
uint32_t handle)
{
@@ -1344,7 +1399,7 @@ static int ucma_process_conn_req(struct cma_event *evt,
evt->event.id = &id_priv->id;
id_priv->handle = handle;
- ret = ucma_query_route(&id_priv->id);
+ ret = ucma_query_req_info(&id_priv->id);
if (ret) {
rdma_destroy_id(&id_priv->id);
goto err;
@@ -1473,14 +1528,10 @@ retry:
switch (resp->event) {
case RDMA_CM_EVENT_ADDR_RESOLVED:
- evt->event.status = ucma_query_route(&evt->id_priv->id);
- if (evt->event.status)
- evt->event.event = RDMA_CM_EVENT_ADDR_ERROR;
+ ucma_process_addr_resolved(evt);
break;
case RDMA_CM_EVENT_ROUTE_RESOLVED:
- evt->event.status = ucma_query_route(&evt->id_priv->id);
- if (evt->event.status)
- evt->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
+ ucma_process_route_resolved(evt);
break;
case RDMA_CM_EVENT_CONNECT_REQUEST:
evt->id_priv = (void *) (uintptr_t) resp->uid;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 7/37] librdmacm: add support to bind using AF_IB
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Allow rdma_bind_addr to accept AF_IB addresses.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma_abi.h | 10 +++++++++-
src/cma.c | 25 +++++++++++++++++++++++++
2 files changed, 34 insertions(+), 1 deletions(-)
diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index 0ef9564..8add397 100644
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -66,7 +66,8 @@ enum {
UCMA_CMD_JOIN_IP_MCAST,
UCMA_CMD_LEAVE_MCAST,
UCMA_CMD_MIGRATE_ID,
- UCMA_CMD_QUERY
+ UCMA_CMD_QUERY,
+ UCMA_CMD_BIND
};
struct ucma_abi_cmd_hdr {
@@ -102,6 +103,13 @@ struct ucma_abi_bind_ip {
__u32 id;
};
+struct ucma_abi_bind {
+ __u32 id;
+ __u16 addr_size;
+ __u16 reserved;
+ struct sockaddr_storage addr;
+};
+
struct ucma_abi_resolve_ip {
struct sockaddr_in6 src_addr;
struct sockaddr_in6 dst_addr;
diff --git a/src/cma.c b/src/cma.c
index e85ef2d..be61333 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -673,6 +673,28 @@ static int ucma_query_route(struct rdma_cm_id *id)
return 0;
}
+static int rdma_bind_addr2(struct rdma_cm_id *id, struct sockaddr *addr,
+ socklen_t addrlen)
+{
+ struct ucma_abi_bind *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_BIND, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ cmd->addr_size = addrlen;
+ cmd->reserved = 0;
+ memcpy(&cmd->addr, addr, addrlen);
+
+ ret = write(id->channel->fd, msg, size);
+ if (ret != size)
+ return (ret >= 0) ? ERR(ENODATA) : -1;
+
+ return ucma_query_addr(id);
+}
+
int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
{
struct ucma_abi_bind_ip *cmd;
@@ -684,6 +706,9 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
if (!addrlen)
return ERR(EINVAL);
+ if (af_ib_support)
+ return rdma_bind_addr2(id, addr, addrlen);
+
CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_BIND_IP, size);
id_priv = container_of(id, struct cma_id_private, id);
cmd->id = id_priv->handle;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 1/37] librdmacm: name changes to indicate only IP addresses supported
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Several commands to the kernel RDMA CM only support IP addresses
because of limitations in the structure definition. Update
the library to match the name changes in the kernel and indicate
that only IP addresses can be used with the current commands.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma_abi.h | 12 ++++++------
src/cma.c | 12 ++++++------
2 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index 1a3a9c2..e51a372 100644
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -48,8 +48,8 @@
enum {
UCMA_CMD_CREATE_ID,
UCMA_CMD_DESTROY_ID,
- UCMA_CMD_BIND_ADDR,
- UCMA_CMD_RESOLVE_ADDR,
+ UCMA_CMD_BIND_IP,
+ UCMA_CMD_RESOLVE_IP,
UCMA_CMD_RESOLVE_ROUTE,
UCMA_CMD_QUERY_ROUTE,
UCMA_CMD_CONNECT,
@@ -62,7 +62,7 @@ enum {
UCMA_CMD_GET_OPTION,
UCMA_CMD_SET_OPTION,
UCMA_CMD_NOTIFY,
- UCMA_CMD_JOIN_MCAST,
+ UCMA_CMD_JOIN_IP_MCAST,
UCMA_CMD_LEAVE_MCAST,
UCMA_CMD_MIGRATE_ID
};
@@ -94,13 +94,13 @@ struct ucma_abi_destroy_id_resp {
__u32 events_reported;
};
-struct ucma_abi_bind_addr {
+struct ucma_abi_bind_ip {
__u64 response;
struct sockaddr_in6 addr;
__u32 id;
};
-struct ucma_abi_resolve_addr {
+struct ucma_abi_resolve_ip {
struct sockaddr_in6 src_addr;
struct sockaddr_in6 dst_addr;
__u32 id;
@@ -192,7 +192,7 @@ struct ucma_abi_notify {
__u32 event;
};
-struct ucma_abi_join_mcast {
+struct ucma_abi_join_ip_mcast {
__u64 response; /* ucma_abi_create_id_resp */
__u64 uid;
struct sockaddr_in6 addr;
diff --git a/src/cma.c b/src/cma.c
index 59e89dd..b5f71d0 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -525,7 +525,7 @@ static int ucma_query_route(struct rdma_cm_id *id)
int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
{
- struct ucma_abi_bind_addr *cmd;
+ struct ucma_abi_bind_ip *cmd;
struct cma_id_private *id_priv;
void *msg;
int ret, size, addrlen;
@@ -534,7 +534,7 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
if (!addrlen)
return ERR(EINVAL);
- CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_BIND_ADDR, size);
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_BIND_IP, size);
id_priv = container_of(id, struct cma_id_private, id);
cmd->id = id_priv->handle;
memcpy(&cmd->addr, addr, addrlen);
@@ -549,7 +549,7 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
struct sockaddr *dst_addr, int timeout_ms)
{
- struct ucma_abi_resolve_addr *cmd;
+ struct ucma_abi_resolve_ip *cmd;
struct cma_id_private *id_priv;
void *msg;
int ret, size, daddrlen;
@@ -558,7 +558,7 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
if (!daddrlen)
return ERR(EINVAL);
- CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_RESOLVE_ADDR, size);
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_RESOLVE_IP, size);
id_priv = container_of(id, struct cma_id_private, id);
cmd->id = id_priv->handle;
if (src_addr)
@@ -1037,7 +1037,7 @@ int rdma_disconnect(struct rdma_cm_id *id)
int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
void *context)
{
- struct ucma_abi_join_mcast *cmd;
+ struct ucma_abi_join_ip_mcast *cmd;
struct ucma_abi_create_id_resp *resp;
struct cma_id_private *id_priv;
struct cma_multicast *mc, **pos;
@@ -1067,7 +1067,7 @@ int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
id_priv->mc_list = mc;
pthread_mutex_unlock(&id_priv->mut);
- CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_JOIN_MCAST, size);
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_JOIN_IP_MCAST, size);
cmd->id = id_priv->handle;
memcpy(&cmd->addr, addr, addrlen);
cmd->uid = (uintptr_t) mc;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 6/37] librdmacm: add support for sockaddr_ib length
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Enhance ucma_addrlen to return the correct address size
for AF_IB.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/cma.c | 2 ++
1 files changed, 2 insertions(+), 0 deletions(-)
diff --git a/src/cma.c b/src/cma.c
index c57d166..e85ef2d 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -470,6 +470,8 @@ static int ucma_addrlen(struct sockaddr *addr)
return sizeof(struct sockaddr_in);
case PF_INET6:
return sizeof(struct sockaddr_in6);
+ case PF_IB:
+ return sizeof(struct sockaddr_ib);
default:
return 0;
}
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 4/37] librdmacm: add support to query GIDs
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Support query GID ABI to obtain GID information separately from
path record data and sa_family addressing.
This patch also adds the definition for sockaddr_ib for userspace.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Makefile.am | 6 ++-
include/infiniband/ib.h | 97 +++++++++++++++++++++++++++++++++++++++++++
include/rdma/rdma_cma.h | 4 +-
include/rdma/rdma_cma_abi.h | 3 +
src/cma.c | 32 ++++++++++++++
5 files changed, 137 insertions(+), 5 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index 290cbc3..2898ad9 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -27,10 +27,11 @@ examples_udaddy_LDADD = $(top_builddir)/src/librdmacm.la
examples_mckey_SOURCES = examples/mckey.c
examples_mckey_LDADD = $(top_builddir)/src/librdmacm.la
-librdmacmincludedir = $(includedir)/rdma
+librdmacmincludedir = $(includedir)/rdma $(includedir)/infiniband
librdmacminclude_HEADERS = include/rdma/rdma_cma_abi.h \
- include/rdma/rdma_cma.h
+ include/rdma/rdma_cma.h \
+ include/infiniband/ib.h
man_MANS = \
man/rdma_accept.3 \
@@ -68,6 +69,7 @@ man_MANS = \
man/rdma_cm.7
EXTRA_DIST = include/rdma/rdma_cma_abi.h include/rdma/rdma_cma.h \
+ include/infiniband/ib.h \
src/librdmacm.map librdmacm.spec.in $(man_MANS)
dist-hook: librdmacm.spec
diff --git a/include/infiniband/ib.h b/include/infiniband/ib.h
new file mode 100644
index 0000000..3a97322
--- /dev/null
+++ b/include/infiniband/ib.h
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if !defined(_RDMA_IB_H)
+#define _RDMA_IB_H
+
+#include <linux/types.h>
+#include <string.h>
+
+#ifndef AF_IB
+#define AF_IB 27
+#endif
+#ifndef PF_IB
+#define PF_IB AF_IB
+#endif
+
+struct ib_addr {
+ union {
+ __u8 uib_addr8[16];
+ __be16 uib_addr16[8];
+ __be32 uib_addr32[4];
+ __be64 uib_addr64[2];
+ } ib_u;
+#define sib_addr8 ib_u.uib_addr8
+#define sib_addr16 ib_u.uib_addr16
+#define sib_addr32 ib_u.uib_addr32
+#define sib_addr64 ib_u.uib_addr64
+#define sib_raw ib_u.uib_addr8
+#define sib_subnet_prefix ib_u.uib_addr64[0]
+#define sib_interface_id ib_u.uib_addr64[1]
+};
+
+static inline int ib_addr_any(const struct ib_addr *a)
+{
+ return ((a->sib_addr64[0] | a->sib_addr64[1]) == 0);
+}
+
+static inline int ib_addr_loopback(const struct ib_addr *a)
+{
+ return ((a->sib_addr32[0] | a->sib_addr32[1] |
+ a->sib_addr32[2] | (a->sib_addr32[3] ^ htonl(1))) == 0);
+}
+
+static inline void ib_addr_set(struct ib_addr *addr,
+ __be32 w1, __be32 w2, __be32 w3, __be32 w4)
+{
+ addr->sib_addr32[0] = w1;
+ addr->sib_addr32[1] = w2;
+ addr->sib_addr32[2] = w3;
+ addr->sib_addr32[3] = w4;
+}
+
+static inline int ib_addr_cmp(const struct ib_addr *a1, const struct ib_addr *a2)
+{
+ return memcmp(a1, a2, sizeof(struct ib_addr));
+}
+
+struct sockaddr_ib {
+ unsigned short int sib_family; /* AF_IB */
+ __be16 sib_pkey;
+ __be32 sib_flowinfo;
+ struct ib_addr sib_addr;
+ __be64 sib_sid;
+ __be64 sib_sid_mask;
+ __u64 sib_scope_id;
+};
+
+#endif /* _RDMA_IB_H */
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 534489d..a071a9b 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -78,7 +78,7 @@ enum rdma_port_space {
*/
#define RDMA_UDP_QKEY 0x01234567
-struct ib_addr {
+struct rdma_ib_addr {
union ibv_gid sgid;
union ibv_gid dgid;
uint16_t pkey;
@@ -92,7 +92,7 @@ struct rdma_addr {
uint8_t dst_pad[sizeof(struct sockaddr_storage) -
sizeof(struct sockaddr)];
union {
- struct ib_addr ibaddr;
+ struct rdma_ib_addr ibaddr;
} addr;
};
diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index 6c83fe8..0ef9564 100644
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -116,7 +116,8 @@ struct ucma_abi_resolve_route {
enum {
UCMA_QUERY_ADDR,
- UCMA_QUERY_PATH
+ UCMA_QUERY_PATH,
+ UCMA_QUERY_GID
};
struct ucma_abi_query {
diff --git a/src/cma.c b/src/cma.c
index c3c6b73..2a70d20 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -54,6 +54,7 @@
#include <infiniband/marshall.h>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_cma_abi.h>
+#include <infiniband/ib.h>
#ifdef INCLUDE_VALGRIND
# include <valgrind/memcheck.h>
@@ -506,6 +507,37 @@ static int ucma_query_addr(struct rdma_cm_id *id)
return 0;
}
+static int ucma_query_gid(struct rdma_cm_id *id)
+{
+ struct ucma_abi_query_addr_resp *resp;
+ struct ucma_abi_query *cmd;
+ struct cma_id_private *id_priv;
+ struct sockaddr_ib *sib;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_QUERY, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ cmd->option = UCMA_QUERY_GID;
+
+ ret = write(id->channel->fd, msg, size);
+ if (ret != size)
+ return (ret >= 0) ? ERR(ENODATA) : -1;
+
+ VALGRIND_MAKE_MEM_DEFINED(resp, sizeof *resp);
+
+ sib = (struct sockaddr_ib *) &resp->src_addr;
+ memcpy(id->route.addr.addr.ibaddr.sgid.raw, sib->sib_addr.sib_raw,
+ sizeof id->route.addr.addr.ibaddr.sgid);
+
+ sib = (struct sockaddr_ib *) &resp->dst_addr;
+ memcpy(id->route.addr.addr.ibaddr.dgid.raw, sib->sib_addr.sib_raw,
+ sizeof id->route.addr.addr.ibaddr.dgid);
+
+ return 0;
+}
+
static void ucma_convert_path(struct ib_path_data *path_data,
struct ibv_sa_path_rec *sa_path)
{
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 15/37] librdmacm: allow user to specify max RDMA resources
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Allow the user to indicate that the library should select the
maximum RDMA read values available that should be used when
establishing a connection. The library selects the maximum
based on local hardware limitations and connection request
data.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 5 +++
src/cma.c | 83 +++++++++++++++++++++++++++++++----------------
src/cma.h | 2 +
3 files changed, 62 insertions(+), 28 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index d8cbb91..f50b4dd 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -121,6 +121,11 @@ struct rdma_cm_id {
struct ibv_cq *recv_cq;
};
+enum {
+ RDMA_MAX_RESP_RES = 0xFF,
+ RDMA_MAX_INIT_DEPTH = 0xFF
+};
+
struct rdma_conn_param {
const void *private_data;
uint8_t private_data_len;
diff --git a/src/cma.c b/src/cma.c
index 805aca3..b8d57a5 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -112,6 +112,8 @@ struct cma_id_private {
pthread_mutex_t mut;
uint32_t handle;
struct cma_multicast *mc_list;
+ uint8_t initiator_depth;
+ uint8_t responder_resources;
};
struct cma_multicast {
@@ -850,8 +852,7 @@ static int rdma_init_qp_attr(struct rdma_cm_id *id, struct ibv_qp_attr *qp_attr,
return 0;
}
-static int ucma_modify_qp_rtr(struct rdma_cm_id *id,
- struct rdma_conn_param *conn_param)
+static int ucma_modify_qp_rtr(struct rdma_cm_id *id, uint8_t resp_res)
{
struct ibv_qp_attr qp_attr;
int qp_attr_mask, ret;
@@ -874,13 +875,12 @@ static int ucma_modify_qp_rtr(struct rdma_cm_id *id,
if (ret)
return ret;
- if (conn_param)
- qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
+ if (resp_res != RDMA_MAX_RESP_RES)
+ qp_attr.max_dest_rd_atomic = resp_res;
return ibv_modify_qp(id->qp, &qp_attr, qp_attr_mask);
}
-static int ucma_modify_qp_rts(struct rdma_cm_id *id,
- struct rdma_conn_param *conn_param)
+static int ucma_modify_qp_rts(struct rdma_cm_id *id, uint8_t init_depth)
{
struct ibv_qp_attr qp_attr;
int qp_attr_mask, ret;
@@ -890,8 +890,8 @@ static int ucma_modify_qp_rts(struct rdma_cm_id *id,
if (ret)
return ret;
- if (conn_param)
- qp_attr.max_rd_atomic = conn_param->initiator_depth;
+ if (init_depth != RDMA_MAX_INIT_DEPTH)
+ qp_attr.max_rd_atomic = init_depth;
return ibv_modify_qp(id->qp, &qp_attr, qp_attr_mask);
}
@@ -1128,28 +1128,31 @@ void rdma_destroy_qp(struct rdma_cm_id *id)
}
static int ucma_valid_param(struct cma_id_private *id_priv,
- struct rdma_conn_param *conn_param)
+ struct rdma_conn_param *param)
{
if (id_priv->id.ps != RDMA_PS_TCP)
return 0;
- if ((conn_param->responder_resources >
- id_priv->cma_dev->max_responder_resources) ||
- (conn_param->initiator_depth >
- id_priv->cma_dev->max_initiator_depth))
+ if ((param->responder_resources != RDMA_MAX_RESP_RES) &&
+ (param->responder_resources > id_priv->cma_dev->max_responder_resources))
+ return ERR(EINVAL);
+
+ if ((param->initiator_depth != RDMA_MAX_INIT_DEPTH) &&
+ (param->initiator_depth > id_priv->cma_dev->max_initiator_depth))
return ERR(EINVAL);
return 0;
}
-static void ucma_copy_conn_param_to_kern(struct ucma_abi_conn_param *dst,
+static void ucma_copy_conn_param_to_kern(struct cma_id_private *id_priv,
+ struct ucma_abi_conn_param *dst,
struct rdma_conn_param *src,
uint32_t qp_num, uint8_t srq)
{
dst->qp_num = qp_num;
dst->srq = srq;
- dst->responder_resources = src->responder_resources;
- dst->initiator_depth = src->initiator_depth;
+ dst->responder_resources = id_priv->responder_resources;
+ dst->initiator_depth = id_priv->initiator_depth;
dst->flow_control = src->flow_control;
dst->retry_count = src->retry_count;
dst->rnr_retry_count = src->rnr_retry_count;
@@ -1174,15 +1177,24 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
if (ret)
return ret;
+ if (conn_param->initiator_depth != RDMA_MAX_INIT_DEPTH)
+ id_priv->initiator_depth = conn_param->initiator_depth;
+ else
+ id_priv->initiator_depth = id_priv->cma_dev->max_initiator_depth;
+ if (conn_param->responder_resources != RDMA_MAX_RESP_RES)
+ id_priv->responder_resources = conn_param->responder_resources;
+ else
+ id_priv->responder_resources = id_priv->cma_dev->max_responder_resources;
+
CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_CONNECT, size);
cmd->id = id_priv->handle;
if (id->qp)
- ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
- id->qp->qp_num,
+ ucma_copy_conn_param_to_kern(id_priv, &cmd->conn_param,
+ conn_param, id->qp->qp_num,
(id->qp->srq != NULL));
else
- ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
- conn_param->qp_num,
+ ucma_copy_conn_param_to_kern(id_priv, &cmd->conn_param,
+ conn_param, conn_param->qp_num,
conn_param->srq);
ret = write(id->channel->fd, msg, size);
@@ -1226,12 +1238,25 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
if (ret)
return ret;
+ if (conn_param->initiator_depth == RDMA_MAX_INIT_DEPTH) {
+ id_priv->initiator_depth = min(id_priv->initiator_depth,
+ id_priv->cma_dev->max_initiator_depth);
+ } else {
+ id_priv->initiator_depth = conn_param->initiator_depth;
+ }
+ if (conn_param->responder_resources == RDMA_MAX_RESP_RES) {
+ id_priv->responder_resources = min(id_priv->responder_resources,
+ id_priv->cma_dev->max_responder_resources);
+ } else {
+ id_priv->responder_resources = conn_param->responder_resources;
+ }
+
if (!ucma_is_ud_ps(id->ps)) {
- ret = ucma_modify_qp_rtr(id, conn_param);
+ ret = ucma_modify_qp_rtr(id, id_priv->responder_resources);
if (ret)
return ret;
- ret = ucma_modify_qp_rts(id, conn_param);
+ ret = ucma_modify_qp_rts(id, id_priv->initiator_depth);
if (ret)
return ret;
}
@@ -1240,12 +1265,12 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
cmd->id = id_priv->handle;
cmd->uid = (uintptr_t) id_priv;
if (id->qp)
- ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
- id->qp->qp_num,
+ ucma_copy_conn_param_to_kern(id_priv, &cmd->conn_param,
+ conn_param, id->qp->qp_num,
(id->qp->srq != NULL));
else
- ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
- conn_param->qp_num,
+ ucma_copy_conn_param_to_kern(id_priv, &cmd->conn_param,
+ conn_param, conn_param->qp_num,
conn_param->srq);
ret = write(id->channel->fd, msg, size);
@@ -1565,6 +1590,8 @@ static int ucma_process_conn_req(struct cma_event *evt,
evt->event.listen_id = &evt->id_priv->id;
evt->event.id = &id_priv->id;
id_priv->handle = handle;
+ id_priv->initiator_depth = evt->event.param.conn.initiator_depth;
+ id_priv->responder_resources = evt->event.param.conn.responder_resources;
if (evt->id_priv->sync) {
ret = rdma_migrate_id(&id_priv->id, NULL);
@@ -1591,11 +1618,11 @@ static int ucma_process_conn_resp(struct cma_id_private *id_priv)
void *msg;
int ret, size;
- ret = ucma_modify_qp_rtr(&id_priv->id, NULL);
+ ret = ucma_modify_qp_rtr(&id_priv->id, RDMA_MAX_RESP_RES);
if (ret)
goto err;
- ret = ucma_modify_qp_rts(&id_priv->id, NULL);
+ ret = ucma_modify_qp_rts(&id_priv->id, RDMA_MAX_INIT_DEPTH);
if (ret)
goto err;
diff --git a/src/cma.h b/src/cma.h
index fcfb1f7..92e771e 100644
--- a/src/cma.h
+++ b/src/cma.h
@@ -65,6 +65,8 @@ static inline uint64_t htonll(uint64_t x) { return x; }
static inline uint64_t ntohll(uint64_t x) { return x; }
#endif
+#define min(a, b) (a < b ? a : b)
+
static inline int ERR(int err)
{
errno = err;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 17/37] librdmacm: expose ucma_init to other internal modules
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Remove static property from ucma_init and expose its
definition in cma.h. The address resolution module will
need access to this function.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/cma.c | 14 +++++++++-----
src/cma.h | 2 ++
2 files changed, 11 insertions(+), 5 deletions(-)
diff --git a/src/cma.c b/src/cma.c
index 6ef4b96..8aa7b05 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -188,13 +188,17 @@ static int check_abi_version(void)
return 0;
}
-static int ucma_init(void)
+int ucma_init(void)
{
struct ibv_device **dev_list = NULL;
struct cma_device *cma_dev;
struct ibv_device_attr attr;
int i, ret, dev_cnt;
+ /* Quick check without lock to see if we're already initialized */
+ if (cma_dev_cnt)
+ return 0;
+
pthread_mutex_lock(&mut);
if (cma_dev_cnt) {
pthread_mutex_unlock(&mut);
@@ -271,7 +275,7 @@ struct ibv_context **rdma_get_devices(int *num_devices)
struct ibv_context **devs = NULL;
int i;
- if (!cma_dev_cnt && ucma_init())
+ if (ucma_init())
goto out;
devs = malloc(sizeof *devs * (cma_dev_cnt + 1));
@@ -301,7 +305,7 @@ struct rdma_event_channel *rdma_create_event_channel(void)
{
struct rdma_event_channel *channel;
- if (!cma_dev_cnt && ucma_init())
+ if (ucma_init())
return NULL;
channel = malloc(sizeof *channel);
@@ -396,7 +400,7 @@ int rdma_create_id(struct rdma_event_channel *channel,
void *msg;
int ret, size;
- ret = cma_dev_cnt ? 0 : ucma_init();
+ ret = ucma_init();
if (ret)
return ret;
@@ -1712,7 +1716,7 @@ int rdma_get_cm_event(struct rdma_event_channel *channel,
void *msg;
int ret, size;
- ret = cma_dev_cnt ? 0 : ucma_init();
+ ret = ucma_init();
if (ret)
return ret;
diff --git a/src/cma.h b/src/cma.h
index 92e771e..06ca38c 100644
--- a/src/cma.h
+++ b/src/cma.h
@@ -82,5 +82,7 @@ static inline void *zalloc(size_t size)
return buf;
}
+int ucma_init();
+
#endif /* CMA_H */
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 2/37] librdmacm: support querying AF_IB addresses
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
The current query route command returns path record data and address
information. The latter is restricted to sizeof(sockaddr_in6). In
order to support AF_IB, modify the library to use the new query addr
command, which supports larger address sizes and avoids querying for
path records data when none are available.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma_abi.h | 22 +++++++++++++++++++---
src/cma.c | 35 ++++++++++++++++++++++++++++++++++-
2 files changed, 53 insertions(+), 4 deletions(-)
diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index e51a372..5c736fb 100644
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -64,7 +64,8 @@ enum {
UCMA_CMD_NOTIFY,
UCMA_CMD_JOIN_IP_MCAST,
UCMA_CMD_LEAVE_MCAST,
- UCMA_CMD_MIGRATE_ID
+ UCMA_CMD_MIGRATE_ID,
+ UCMA_CMD_QUERY
};
struct ucma_abi_cmd_hdr {
@@ -112,10 +113,14 @@ struct ucma_abi_resolve_route {
__u32 timeout_ms;
};
-struct ucma_abi_query_route {
+enum {
+ UCMA_QUERY_ADDR
+};
+
+struct ucma_abi_query {
__u64 response;
__u32 id;
- __u32 reserved;
+ __u32 option;
};
struct ucma_abi_query_route_resp {
@@ -128,6 +133,17 @@ struct ucma_abi_query_route_resp {
__u8 reserved[3];
};
+struct ucma_abi_query_addr_resp {
+ __u64 node_guid;
+ __u8 port_num;
+ __u8 reserved;
+ __u16 pkey;
+ __u16 src_size;
+ __u16 dst_size;
+ struct sockaddr_storage src_addr;
+ struct sockaddr_storage dst_addr;
+};
+
struct ucma_abi_conn_param {
__u32 qp_num;
__u32 reserved;
diff --git a/src/cma.c b/src/cma.c
index b5f71d0..2aef594 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -473,10 +473,43 @@ static int ucma_addrlen(struct sockaddr *addr)
}
}
+static int ucma_query_addr(struct rdma_cm_id *id)
+{
+ struct ucma_abi_query_addr_resp *resp;
+ struct ucma_abi_query *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_QUERY, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ cmd->option = UCMA_QUERY_ADDR;
+
+ ret = write(id->channel->fd, msg, size);
+ if (ret != size)
+ return (ret >= 0) ? ERR(ENODATA) : -1;
+
+ VALGRIND_MAKE_MEM_DEFINED(resp, sizeof *resp);
+
+ memcpy(&id->route.addr.src_addr, &resp->src_addr, resp->src_size);
+ memcpy(&id->route.addr.dst_addr, &resp->dst_addr, resp->dst_size);
+
+ if (!id_priv->cma_dev && resp->node_guid) {
+ ret = ucma_get_device(id_priv, resp->node_guid);
+ if (ret)
+ return ret;
+ id->port_num = resp->port_num;
+ id->route.addr.addr.ibaddr.pkey = resp->pkey;
+ }
+
+ return 0;
+}
+
static int ucma_query_route(struct rdma_cm_id *id)
{
struct ucma_abi_query_route_resp *resp;
- struct ucma_abi_query_route *cmd;
+ struct ucma_abi_query *cmd;
struct cma_id_private *id_priv;
void *msg;
int ret, size, i;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 14/37] librdmacm: make CQs optional for rdma_create_qp
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Allow the user to specify NULL for the send and receive CQs when
creating a QP through rdma_create_qp. The librdmacm will automatically
create CQs for the user, along with completion channel.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 4 +++
src/cma.c | 74 ++++++++++++++++++++++++++++++++++++++++++++---
2 files changed, 74 insertions(+), 4 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index ccf6cd4..d8cbb91 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -115,6 +115,10 @@ struct rdma_cm_id {
enum rdma_port_space ps;
uint8_t port_num;
struct rdma_cm_event *event;
+ struct ibv_comp_channel *send_cq_channel;
+ struct ibv_cq *send_cq;
+ struct ibv_comp_channel *recv_cq_channel;
+ struct ibv_cq *recv_cq;
};
struct rdma_conn_param {
diff --git a/src/cma.c b/src/cma.c
index 0587ab3..805aca3 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -1025,6 +1025,63 @@ static int ucma_init_ud_qp(struct cma_id_private *id_priv, struct ibv_qp *qp)
return ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_SQ_PSN);
}
+static void ucma_destroy_cqs(struct rdma_cm_id *id)
+{
+ if (id->recv_cq)
+ ibv_destroy_cq(id->recv_cq);
+
+ if (id->recv_cq_channel)
+ ibv_destroy_comp_channel(id->recv_cq_channel);
+
+ if (id->send_cq)
+ ibv_destroy_cq(id->send_cq);
+
+ if (id->send_cq_channel)
+ ibv_destroy_comp_channel(id->send_cq_channel);
+}
+
+static int ucma_create_cqs(struct rdma_cm_id *id, struct ibv_qp_init_attr *attr)
+{
+ int ret;
+
+ if (!attr->recv_cq) {
+ id->recv_cq_channel = ibv_create_comp_channel(id->verbs);
+ if (!id->recv_cq_channel) {
+ ret = ERR(ENOMEM);
+ goto err;
+ }
+
+ id->recv_cq = ibv_create_cq(id->verbs, attr->cap.max_recv_wr,
+ id, id->recv_cq_channel, 0);
+ if (!id->recv_cq) {
+ ret = ERR(ENOMEM);
+ goto err;
+ }
+ attr->recv_cq = id->recv_cq;
+ }
+
+ if (!attr->send_cq) {
+ id->send_cq_channel = ibv_create_comp_channel(id->verbs);
+ if (!id->send_cq_channel) {
+ ret = ERR(ENOMEM);
+ goto err;
+ }
+
+ id->send_cq = ibv_create_cq(id->verbs, attr->cap.max_send_wr,
+ id, id->send_cq_channel, 0);
+ if (!id->send_cq) {
+ ret = ERR(ENOMEM);
+ goto err;
+ }
+ attr->send_cq = id->send_cq;
+ }
+
+ return 0;
+err:
+ ucma_destroy_cqs(id);
+ return ret;
+}
+
int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd,
struct ibv_qp_init_attr *qp_init_attr)
{
@@ -1038,27 +1095,36 @@ int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd,
else if (id->verbs != pd->context)
return ERR(EINVAL);
+ ret = ucma_create_cqs(id, qp_init_attr);
+ if (ret)
+ return ret;
+
qp = ibv_create_qp(pd, qp_init_attr);
- if (!qp)
- return ERR(ENOMEM);
+ if (!qp) {
+ ret = ERR(ENOMEM);
+ goto err1;
+ }
if (ucma_is_ud_ps(id->ps))
ret = ucma_init_ud_qp(id_priv, qp);
else
ret = ucma_init_conn_qp(id_priv, qp);
if (ret)
- goto err;
+ goto err2;
id->qp = qp;
return 0;
-err:
+err2:
ibv_destroy_qp(qp);
+err1:
+ ucma_destroy_cqs(id);
return ret;
}
void rdma_destroy_qp(struct rdma_cm_id *id)
{
ibv_destroy_qp(id->qp);
+ ucma_destroy_cqs(id);
}
static int ucma_valid_param(struct cma_id_private *id_priv,
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 18/37] librdmacm: add rdma_getaddrinfo
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Provide a call similar to getaddrinfo for RDMA devices and
connections. rdma_get_addrinfo is modeled after getaddrinfo, with
the following modifications:
A source address is returned as part of the call to allow the
user to allocate the necessary resources for connections.
Optional routing information may be returned to support
Infiniband fabrics. IB routing information includes necessary
path record data.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Makefile.am | 2
include/rdma/rdma_cma.h | 31 +++++++
src/addrinfo.c | 201 +++++++++++++++++++++++++++++++++++++++++++++++
src/librdmacm.map | 2
4 files changed, 234 insertions(+), 2 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index c9be437..be53c78 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -12,7 +12,7 @@ else
librdmacm_version_script =
endif
-src_librdmacm_la_SOURCES = src/cma.c
+src_librdmacm_la_SOURCES = src/cma.c src/addrinfo.c
src_librdmacm_la_LDFLAGS = -version-info 1 -export-dynamic \
$(librdmacm_version_script)
src_librdmacm_la_DEPENDENCIES = $(srcdir)/src/librdmacm.map
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 88b3796..1db559e 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2005 Voltaire Inc. All rights reserved.
- * Copyright (c) 2005-2007 Intel Corporation. All rights reserved.
+ * Copyright (c) 2005-2010 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -158,6 +158,26 @@ struct rdma_cm_event {
} param;
};
+#define RAI_PASSIVE 0x00000001
+
+struct rdma_addrinfo {
+ int ai_flags;
+ int ai_family;
+ int ai_qp_type;
+ int ai_port_space;
+ socklen_t ai_src_len;
+ socklen_t ai_dst_len;
+ struct sockaddr *ai_src_addr;
+ struct sockaddr *ai_dst_addr;
+ char *ai_src_canonname;
+ char *ai_dst_canonname;
+ size_t ai_route_len;
+ void *ai_route;
+ size_t ai_connect_len;
+ void *ai_connect;
+ struct rdma_addrinfo *ai_next;
+};
+
/**
* rdma_create_event_channel - Open a channel used to report communication events.
* Description:
@@ -589,6 +609,15 @@ int rdma_set_option(struct rdma_cm_id *id, int level, int optname,
*/
int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel);
+/**
+ * rdma_getaddrinfo - RDMA address and route resolution service.
+ */
+int rdma_getaddrinfo(char *node, char *service,
+ struct rdma_addrinfo *hints,
+ struct rdma_addrinfo **res);
+
+void rdma_freeaddrinfo(struct rdma_addrinfo *res);
+
#ifdef __cplusplus
}
#endif
diff --git a/src/addrinfo.c b/src/addrinfo.c
new file mode 100644
index 0000000..15ae071
--- /dev/null
+++ b/src/addrinfo.c
@@ -0,0 +1,201 @@
+/*
+ * Copyright (c) 2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id: cm.c 3453 2005-09-15 21:43:21Z sean.hefty $
+ */
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+
+#include "cma.h"
+#include <rdma/rdma_cma.h>
+#include <infiniband/ib.h>
+
+static void ucma_convert_to_ai(struct addrinfo *ai, struct rdma_addrinfo *rai)
+{
+ memset(ai, 0, sizeof *ai);
+ ai->ai_flags = rai->ai_flags;
+ ai->ai_family = rai->ai_family;
+
+ switch (rai->ai_qp_type) {
+ case IBV_QPT_RC:
+ ai->ai_socktype = SOCK_STREAM;
+ break;
+ case IBV_QPT_UD:
+ ai->ai_socktype = SOCK_DGRAM;
+ break;
+ }
+
+ switch (rai->ai_port_space) {
+ case RDMA_PS_TCP:
+ ai->ai_protocol = IPPROTO_TCP;
+ break;
+ case RDMA_PS_IPOIB:
+ case RDMA_PS_UDP:
+ ai->ai_protocol = IPPROTO_UDP;
+ break;
+ }
+
+ if (rai->ai_flags & RAI_PASSIVE) {
+ ai->ai_addrlen = rai->ai_src_len;
+ ai->ai_addr = rai->ai_src_addr;
+ } else {
+ ai->ai_addrlen = rai->ai_dst_len;
+ ai->ai_addr = rai->ai_dst_addr;
+ }
+ ai->ai_canonname = rai->ai_dst_canonname;
+ ai->ai_next = NULL;
+}
+
+static int ucma_convert_to_rai(struct rdma_addrinfo *rai, struct addrinfo *ai)
+{
+ struct sockaddr *addr;
+ char *canonname;
+
+ memset(rai, 0, sizeof *rai);
+ rai->ai_flags = ai->ai_flags;
+ rai->ai_family = ai->ai_family;
+
+ switch (ai->ai_socktype) {
+ case SOCK_STREAM:
+ rai->ai_qp_type = IBV_QPT_RC;
+ break;
+ case SOCK_DGRAM:
+ rai->ai_qp_type = IBV_QPT_UD;
+ break;
+ }
+
+ switch (ai->ai_protocol) {
+ case IPPROTO_TCP:
+ rai->ai_port_space = RDMA_PS_TCP;
+ break;
+ case IPPROTO_UDP:
+ rai->ai_port_space = RDMA_PS_UDP;
+ break;
+ }
+
+ addr = malloc(ai->ai_addrlen);
+ if (!addr)
+ return ERR(ENOMEM);
+
+ canonname = ai->ai_canonname ? malloc(strlen(ai->ai_canonname) + 1) : NULL;
+ if (canonname)
+ strcpy(canonname, ai->ai_canonname);
+
+ memcpy(addr, ai->ai_addr, ai->ai_addrlen);
+ if (ai->ai_flags & RAI_PASSIVE) {
+ rai->ai_src_addr = addr;
+ rai->ai_src_len = ai->ai_addrlen;
+ rai->ai_src_canonname = canonname;
+ } else {
+ rai->ai_dst_addr = addr;
+ rai->ai_dst_len = ai->ai_addrlen;
+ rai->ai_dst_canonname = canonname;
+ }
+
+ return 0;
+}
+
+int rdma_getaddrinfo(char *node, char *service,
+ struct rdma_addrinfo *hints,
+ struct rdma_addrinfo **res)
+{
+ struct rdma_addrinfo *rai;
+ struct addrinfo ai_hints;
+ struct addrinfo *ai;
+ int ret;
+
+ ret = ucma_init();
+ if (ret)
+ return ret;
+
+ if (hints)
+ ucma_convert_to_ai(&ai_hints, hints);
+
+ ret = getaddrinfo(node, service, &ai_hints, &ai);
+ if (ret)
+ return ret;
+
+ rai = malloc(sizeof(*rai));
+ if (!rai) {
+ ret = ERR(ENOMEM);
+ goto err1;
+ }
+
+ ret = ucma_convert_to_rai(rai, ai);
+ if (ret)
+ goto err2;
+
+ freeaddrinfo(ai);
+ *res = rai;
+ return 0;
+
+err2:
+ rdma_freeaddrinfo(rai);
+err1:
+ freeaddrinfo(ai);
+ return ret;
+}
+
+void rdma_freeaddrinfo(struct rdma_addrinfo *res)
+{
+ struct rdma_addrinfo *rai;
+
+ while (res) {
+ rai = res;
+ res = res->ai_next;
+
+ if (rai->ai_connect)
+ free(rai->ai_connect);
+
+ if (rai->ai_route)
+ free(rai->ai_route);
+
+ if (rai->ai_src_canonname)
+ free(rai->ai_src_canonname);
+
+ if (rai->ai_dst_canonname)
+ free(rai->ai_dst_canonname);
+
+ if (rai->ai_src_addr)
+ free(rai->ai_src_addr);
+
+ if (rai->ai_dst_addr)
+ free(rai->ai_dst_addr);
+
+ free(rai);
+ }
+}
diff --git a/src/librdmacm.map b/src/librdmacm.map
index cb94efe..1f07102 100644
--- a/src/librdmacm.map
+++ b/src/librdmacm.map
@@ -28,5 +28,7 @@ RDMACM_1.0 {
rdma_get_local_addr;
rdma_get_peer_addr;
rdma_migrate_id;
+ rdma_getaddrinfo;
+ rdma_freeaddrinfo;
local: *;
};
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 3/37] librdmacm: add ability to query IB path records
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
The current query_route command only supports 2 path records.
Add support for query_path, which is capable of supporting
multiple paths.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma_abi.h | 10 +++++
src/cma.c | 82 +++++++++++++++++++++++++++++++++++++++++++
2 files changed, 91 insertions(+), 1 deletions(-)
diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index 5c736fb..6c83fe8 100644
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -35,6 +35,7 @@
#include <infiniband/kern-abi.h>
#include <infiniband/sa-kern-abi.h>
+#include <infiniband/sa.h>
/*
* This file must be kept in sync with the kernel's version of rdma_user_cm.h
@@ -114,7 +115,8 @@ struct ucma_abi_resolve_route {
};
enum {
- UCMA_QUERY_ADDR
+ UCMA_QUERY_ADDR,
+ UCMA_QUERY_PATH
};
struct ucma_abi_query {
@@ -144,6 +146,12 @@ struct ucma_abi_query_addr_resp {
struct sockaddr_storage dst_addr;
};
+struct ucma_abi_query_path_resp {
+ __u32 num_paths;
+ __u32 reserved;
+ struct ib_path_data path_data[0];
+};
+
struct ucma_abi_conn_param {
__u32 qp_num;
__u32 reserved;
diff --git a/src/cma.c b/src/cma.c
index 2aef594..c3c6b73 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -506,6 +506,88 @@ static int ucma_query_addr(struct rdma_cm_id *id)
return 0;
}
+static void ucma_convert_path(struct ib_path_data *path_data,
+ struct ibv_sa_path_rec *sa_path)
+{
+ uint32_t fl_hop;
+
+ sa_path->dgid = path_data->path.dgid;
+ sa_path->sgid = path_data->path.sgid;
+ sa_path->dlid = path_data->path.dlid;
+ sa_path->slid = path_data->path.slid;
+ sa_path->raw_traffic = 0;
+
+ fl_hop = ntohl(path_data->path.flowlabel_hoplimit);
+ sa_path->flow_label = htonl(fl_hop >> 8);
+ sa_path->hop_limit = (uint8_t) fl_hop;
+
+ sa_path->traffic_class = path_data->path.tclass;
+ sa_path->reversible = path_data->path.reversible_numpath >> 7;
+ sa_path->numb_path = 1;
+ sa_path->pkey = path_data->path.pkey;
+ sa_path->sl = ntohs(path_data->path.qosclass_sl) & 0xF;
+ sa_path->mtu_selector = 1;
+ sa_path->mtu = path_data->path.mtu & 0x1F;
+ sa_path->rate_selector = 1;
+ sa_path->rate = path_data->path.rate & 0x1F;
+ sa_path->packet_life_time_selector = 1;
+ sa_path->packet_life_time = path_data->path.packetlifetime & 0x1F;
+
+ sa_path->preference = (uint8_t) path_data->flags;
+}
+
+static int ucma_query_path(struct rdma_cm_id *id)
+{
+ struct ucma_abi_query_path_resp *resp;
+ struct ucma_abi_query *cmd;
+ struct ucma_abi_cmd_hdr *hdr;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size, i;
+
+ size = sizeof(*hdr) + sizeof(*cmd);
+ msg = alloca(size);
+ if (!msg)
+ return ERR(ENOMEM);
+
+ hdr = msg;
+ cmd = msg + sizeof(*hdr);
+
+ hdr->cmd = UCMA_CMD_QUERY;
+ hdr->in = sizeof(*cmd);
+ hdr->out = sizeof(*resp) + sizeof(struct ib_path_data) * 6;
+
+ memset(cmd, 0, sizeof(*cmd));
+
+ resp = alloca(hdr->out);
+ if (!resp)
+ return ERR(ENOMEM);
+
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->response = (uintptr_t) resp;
+ cmd->id = id_priv->handle;
+ cmd->option = UCMA_QUERY_PATH;
+
+ ret = write(id->channel->fd, msg, size);
+ if (ret != size)
+ return (ret >= 0) ? ERR(ENODATA) : -1;
+
+ VALGRIND_MAKE_MEM_DEFINED(resp, hdr->out);
+
+ if (resp->num_paths) {
+ id->route.path_rec = malloc(sizeof(*id->route.path_rec) *
+ resp->num_paths);
+ if (!id->route.path_rec)
+ return ERR(ENOMEM);
+
+ id->route.num_paths = resp->num_paths;
+ for (i = 0; i < resp->num_paths; i++)
+ ucma_convert_path(&resp->path_data[i], &id->route.path_rec[i]);
+ }
+
+ return 0;
+}
+
static int ucma_query_route(struct rdma_cm_id *id)
{
struct ucma_abi_query_route_resp *resp;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 11/37] librdmacm: add zalloc call
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/cma.c | 6 ++----
src/cma.h | 10 ++++++++++
2 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/src/cma.c b/src/cma.c
index a85448b..4025aeb 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -342,11 +342,10 @@ static struct cma_id_private *ucma_alloc_id(struct rdma_event_channel *channel,
{
struct cma_id_private *id_priv;
- id_priv = malloc(sizeof *id_priv);
+ id_priv = zalloc(sizeof *id_priv);
if (!id_priv)
return NULL;
- memset(id_priv, 0, sizeof *id_priv);
id_priv->id.context = context;
id_priv->id.ps = ps;
id_priv->id.channel = channel;
@@ -1228,11 +1227,10 @@ static int rdma_join_multicast2(struct rdma_cm_id *id, struct sockaddr *addr,
int ret, size;
id_priv = container_of(id, struct cma_id_private, id);
- mc = malloc(sizeof *mc);
+ mc = zalloc(sizeof *mc);
if (!mc)
return ERR(ENOMEM);
- memset(mc, 0, sizeof *mc);
mc->context = context;
mc->id_priv = id_priv;
memcpy(&mc->addr, addr, addrlen);
diff --git a/src/cma.h b/src/cma.h
index 1c0ab8b..fcfb1f7 100644
--- a/src/cma.h
+++ b/src/cma.h
@@ -42,6 +42,7 @@
#include <errno.h>
#include <endian.h>
#include <byteswap.h>
+#include <string.h>
#ifdef INCLUDE_VALGRIND
# include <valgrind/memcheck.h>
@@ -70,5 +71,14 @@ static inline int ERR(int err)
return -1;
}
+static inline void *zalloc(size_t size)
+{
+ void *buf;
+
+ if ((buf = malloc(size)))
+ memset(buf, 0, size);
+ return buf;
+}
+
#endif /* CMA_H */
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 19/37] librdmacm: add rdma_get_request
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
To simplify passive side operation and better support synchronous
operations, add rdma_get_request(). This function is called on the
listening side to retrieve a connection request event.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Ideally, this call would have been rdma_accept, to match with the socket
accept call, but it was already taken.
include/rdma/rdma_cma.h | 5 +++++
src/cma.c | 38 ++++++++++++++++++++++++++++++++++++++
src/librdmacm.map | 1 +
3 files changed, 44 insertions(+), 0 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 1db559e..89013a0 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -381,6 +381,11 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param);
int rdma_listen(struct rdma_cm_id *id, int backlog);
/**
+ * rdma_get_request
+ */
+int rdma_get_request(struct rdma_cm_id *listen, struct rdma_cm_id **id);
+
+/**
* rdma_accept - Called to accept a connection request.
* @id: Connection identifier associated with the request.
* @conn_param: Optional information needed to establish the connection.
diff --git a/src/cma.c b/src/cma.c
index 8aa7b05..9de33d4 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -1242,6 +1242,44 @@ int rdma_listen(struct rdma_cm_id *id, int backlog)
return ucma_query_route(id);
}
+int rdma_get_request(struct rdma_cm_id *listen, struct rdma_cm_id **id)
+{
+ struct cma_id_private *id_priv;
+ struct rdma_cm_event *event;
+ int ret;
+
+ id_priv = container_of(listen, struct cma_id_private, id);
+ if (!id_priv->sync)
+ return ERR(EINVAL);
+
+ if (listen->event) {
+ rdma_ack_cm_event(listen->event);
+ listen->event = NULL;
+ }
+
+ ret = rdma_get_cm_event(listen->channel, &event);
+ if (ret)
+ return ret;
+
+ if (event->status) {
+ ret = event->status;
+ goto err;
+ }
+
+ if (event->event != RDMA_CM_EVENT_CONNECT_REQUEST) {
+ ret = ERR(EINVAL);
+ goto err;
+ }
+
+ *id = event->id;
+ (*id)->event = event;
+ return 0;
+
+err:
+ listen->event = event;
+ return ret;
+}
+
int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
{
struct ucma_abi_accept *cmd;
diff --git a/src/librdmacm.map b/src/librdmacm.map
index 1f07102..f6af452 100644
--- a/src/librdmacm.map
+++ b/src/librdmacm.map
@@ -30,5 +30,6 @@ RDMACM_1.0 {
rdma_migrate_id;
rdma_getaddrinfo;
rdma_freeaddrinfo;
+ rdma_get_request;
local: *;
};
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 24/37] librdmacm: add support for passive side in rdma_create_ep
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Allow calling rdma_create_ep for a listening rdma_cm_id.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/cma.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++-----------
1 files changed, 57 insertions(+), 12 deletions(-)
diff --git a/src/cma.c b/src/cma.c
index 3b80542..c59f47b 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -104,17 +104,19 @@ struct cma_device {
};
struct cma_id_private {
- struct rdma_cm_id id;
- struct cma_device *cma_dev;
- int events_completed;
- int connect_error;
- int sync;
- pthread_cond_t cond;
- pthread_mutex_t mut;
- uint32_t handle;
- struct cma_multicast *mc_list;
- uint8_t initiator_depth;
- uint8_t responder_resources;
+ struct rdma_cm_id id;
+ struct cma_device *cma_dev;
+ int events_completed;
+ int connect_error;
+ int sync;
+ pthread_cond_t cond;
+ pthread_mutex_t mut;
+ uint32_t handle;
+ struct cma_multicast *mc_list;
+ struct ibv_pd *pd;
+ struct ibv_qp_init_attr *qp_init_attr;
+ uint8_t initiator_depth;
+ uint8_t responder_resources;
};
struct cma_multicast {
@@ -1993,6 +1995,34 @@ int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel)
return 0;
}
+static int ucma_passive_ep(struct rdma_cm_id *id, struct rdma_addrinfo *res,
+ struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
+{
+ struct cma_id_private *id_priv;
+ int ret;
+
+ if (af_ib_support)
+ ret = rdma_bind_addr2(id, res->ai_src_addr, res->ai_src_len);
+ else
+ ret = rdma_bind_addr(id, res->ai_src_addr);
+ if (ret)
+ return ret;
+
+ id_priv = container_of(id, struct cma_id_private, id);
+ id_priv->pd = pd;
+
+ if (qp_init_attr) {
+ id_priv->qp_init_attr = malloc(sizeof *qp_init_attr);
+ if (!id_priv->qp_init_attr)
+ return ERR(ENOMEM);
+
+ *id_priv->qp_init_attr = *qp_init_attr;
+ id_priv->qp_init_attr->qp_type = res->ai_qp_type;
+ }
+
+ return 0;
+}
+
int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
{
@@ -2003,6 +2033,13 @@ int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
if (ret)
return ret;
+ if (res->ai_flags & RAI_PASSIVE) {
+ ret = ucma_passive_ep(cm_id, res, pd, qp_init_attr);
+ if (ret)
+ goto err;
+ goto out;
+ }
+
if (af_ib_support)
ret = rdma_resolve_addr2(cm_id, res->ai_src_addr, res->ai_src_len,
res->ai_dst_addr, res->ai_dst_len, 2000);
@@ -2027,17 +2064,25 @@ int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
if (ret)
goto err;
+out:
*id = cm_id;
return 0;
err:
- rdma_destroy_id(cm_id);
+ rdma_destroy_ep(cm_id);
return ret;
}
void rdma_destroy_ep(struct rdma_cm_id *id)
{
+ struct cma_id_private *id_priv;
+
if (id->qp)
rdma_destroy_qp(id);
+
+ id_priv = container_of(id, struct cma_id_private, id);
+ if (id_priv->qp_init_attr)
+ free(id_priv->qp_init_attr);
+
rdma_destroy_id(id);
}
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 16/37] librdmacm: allow conn_param to be optional
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
If conn_param is not provided to rdma_connect or rdma_accept,
then default values are used to establish the connection.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 8 ++++++--
src/cma.c | 34 +++++++++++++++++++++++-----------
2 files changed, 29 insertions(+), 13 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index f50b4dd..88b3796 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -325,7 +325,7 @@ void rdma_destroy_qp(struct rdma_cm_id *id);
/**
* rdma_connect - Initiate an active connection request.
* @id: RDMA identifier.
- * @conn_param: connection parameters.
+ * @conn_param: optional connection parameters.
* Description:
* For a connected rdma_cm_id, this call initiates a connection request
* to a remote destination. For an unconnected rdma_cm_id, it initiates
@@ -333,6 +333,8 @@ void rdma_destroy_qp(struct rdma_cm_id *id);
* Notes:
* Users must have resolved a route to the destination address
* by having called rdma_resolve_route before calling this routine.
+ * A user may override the default connection parameters and exchange
+ * private data as part of the connection by using the conn_param parameter.
* See also:
* rdma_resolve_route, rdma_disconnect, rdma_listen, rdma_get_cm_event
*/
@@ -361,7 +363,7 @@ int rdma_listen(struct rdma_cm_id *id, int backlog);
/**
* rdma_accept - Called to accept a connection request.
* @id: Connection identifier associated with the request.
- * @conn_param: Information needed to establish the connection.
+ * @conn_param: Optional information needed to establish the connection.
* Description:
* Called from the listening side to accept a connection or datagram
* service lookup request.
@@ -372,6 +374,8 @@ int rdma_listen(struct rdma_cm_id *id, int backlog);
* events give the user a newly created rdma_cm_id, similar to a new
* socket, but the rdma_cm_id is bound to a specific RDMA device.
* rdma_accept is called on the new rdma_cm_id.
+ * A user may override the default connection parameters and exchange
+ * private data as part of the connection by using the conn_param parameter.
* See also:
* rdma_listen, rdma_reject, rdma_get_cm_event
*/
diff --git a/src/cma.c b/src/cma.c
index b8d57a5..6ef4b96 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -1133,6 +1133,12 @@ static int ucma_valid_param(struct cma_id_private *id_priv,
if (id_priv->id.ps != RDMA_PS_TCP)
return 0;
+ if (!id_priv->id.qp && !param)
+ return ERR(EINVAL);
+
+ if (!param)
+ return 0;
+
if ((param->responder_resources != RDMA_MAX_RESP_RES) &&
(param->responder_resources > id_priv->cma_dev->max_responder_resources))
return ERR(EINVAL);
@@ -1153,15 +1159,21 @@ static void ucma_copy_conn_param_to_kern(struct cma_id_private *id_priv,
dst->srq = srq;
dst->responder_resources = id_priv->responder_resources;
dst->initiator_depth = id_priv->initiator_depth;
- dst->flow_control = src->flow_control;
- dst->retry_count = src->retry_count;
- dst->rnr_retry_count = src->rnr_retry_count;
dst->valid = 1;
- if (src->private_data && src->private_data_len) {
- memcpy(dst->private_data, src->private_data,
- src->private_data_len);
- dst->private_data_len = src->private_data_len;
+ if (src) {
+ dst->flow_control = src->flow_control;
+ dst->retry_count = src->retry_count;
+ dst->rnr_retry_count = src->rnr_retry_count;
+
+ if (src->private_data && src->private_data_len) {
+ memcpy(dst->private_data, src->private_data,
+ src->private_data_len);
+ dst->private_data_len = src->private_data_len;
+ }
+ } else {
+ dst->retry_count = 7;
+ dst->rnr_retry_count = 7;
}
}
@@ -1177,11 +1189,11 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
if (ret)
return ret;
- if (conn_param->initiator_depth != RDMA_MAX_INIT_DEPTH)
+ if (conn_param && conn_param->initiator_depth != RDMA_MAX_INIT_DEPTH)
id_priv->initiator_depth = conn_param->initiator_depth;
else
id_priv->initiator_depth = id_priv->cma_dev->max_initiator_depth;
- if (conn_param->responder_resources != RDMA_MAX_RESP_RES)
+ if (conn_param && conn_param->responder_resources != RDMA_MAX_RESP_RES)
id_priv->responder_resources = conn_param->responder_resources;
else
id_priv->responder_resources = id_priv->cma_dev->max_responder_resources;
@@ -1238,13 +1250,13 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
if (ret)
return ret;
- if (conn_param->initiator_depth == RDMA_MAX_INIT_DEPTH) {
+ if (!conn_param || conn_param->initiator_depth == RDMA_MAX_INIT_DEPTH) {
id_priv->initiator_depth = min(id_priv->initiator_depth,
id_priv->cma_dev->max_initiator_depth);
} else {
id_priv->initiator_depth = conn_param->initiator_depth;
}
- if (conn_param->responder_resources == RDMA_MAX_RESP_RES) {
+ if (!conn_param || conn_param->responder_resources == RDMA_MAX_RESP_RES) {
id_priv->responder_resources = min(id_priv->responder_resources,
id_priv->cma_dev->max_responder_resources);
} else {
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 20/37] librdmacm: define options to set IB route directly
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Add the definitions needed to set IB path record data using set_option.
This will be used by rdma_create_ep().
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 6 ++++--
1 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 89013a0..391c35c 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -588,12 +588,14 @@ const char *rdma_event_str(enum rdma_cm_event_type event);
/* Option levels */
enum {
- RDMA_OPTION_ID = 0
+ RDMA_OPTION_ID = 0,
+ RDMA_OPTION_IB = 1
};
/* Option details */
enum {
- RDMA_OPTION_ID_TOS = 0 /* uint8_t: RFC 2474 */
+ RDMA_OPTION_ID_TOS = 0, /* uint8_t: RFC 2474 */
+ RDMA_OPTION_IB_PATH = 1 /* struct ib_path_data[] */
};
/**
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 22/37] librdmacm: add new call to create id
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Provide a simple call to create an rdma_id, with optional QP. The
id is created using synchronous operation, using the output of
rdma_getaddrinfo.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 26 ++++++++++++++++++++++++++
src/cma.c | 43 +++++++++++++++++++++++++++++++++++++++++++
src/librdmacm.map | 1 +
3 files changed, 70 insertions(+), 0 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 391c35c..74762a2 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -232,6 +232,32 @@ int rdma_create_id(struct rdma_event_channel *channel,
enum rdma_port_space ps);
/**
+ * rdma_create_ep - Allocate a communication identifier and qp.
+ * @id: A reference where the allocated communication identifier will be
+ * returned.
+ * @res: Result from rdma_getaddrinfo, which specifies the source and
+ * destination addresses, plus optional routing and connection information.
+ * @pd: Optional protection domain. This parameter is ignored if qp_init_attr
+ * is NULL.
+ * @qp_init_attr: Optional attributes for a QP created on the rdma_cm_id.
+ * Description:
+ * Create an identifier and option QP used for communication.
+ * Notes:
+ * If qp_init_attr is provided, then a queue pair will be allocated and
+ * associated with the rdma_cm_id. If a pd is provided, the QP will be
+ * created on that PD. Otherwise, the QP will be allocated on a default
+ * PD.
+ * The rdma_cm_id will be set to use synchronous operations (connect,
+ * listen, and get_request). To convert to synchronous operation, the
+ * rdma_cm_id should be migrated to a user allocated event channel.
+ * See also:
+ * rdma_create_id, rdma_create_qp, rdma_migrate_id, rdma_connect,
+ * rdma_listen
+ */
+int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
+ struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr);
+
+/**
* rdma_destroy_id - Release a communication identifier.
* @id: The communication identifier to destroy.
* Description:
diff --git a/src/cma.c b/src/cma.c
index e31fb8a..b911b3f 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -49,6 +49,7 @@
#include <endian.h>
#include <byteswap.h>
#include <stddef.h>
+#include <netdb.h>
#include "cma.h"
#include <infiniband/driver.h>
@@ -1990,3 +1991,45 @@ int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel)
return 0;
}
+
+int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
+ struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
+{
+ struct rdma_cm_id *cm_id;
+ int ret;
+
+ ret = rdma_create_id2(NULL, &cm_id, NULL, res->ai_port_space, res->ai_qp_type);
+ if (ret)
+ return ret;
+
+ if (af_ib_support)
+ ret = rdma_resolve_addr2(cm_id, res->ai_src_addr, res->ai_src_len,
+ res->ai_dst_addr, res->ai_dst_len, 2000);
+ else
+ ret = rdma_resolve_addr(cm_id, res->ai_src_addr, res->ai_dst_addr, 2000);
+ if (ret)
+ goto err;
+
+ if (res->ai_route_len) {
+ ret = rdma_set_option(cm_id, RDMA_OPTION_IB, RDMA_OPTION_IB_PATH,
+ res->ai_route, res->ai_route_len);
+ if (!ret)
+ ret = ucma_complete(container_of(cm_id, struct cma_id_private, id));
+ } else {
+ ret = rdma_resolve_route(cm_id, 2000);
+ }
+ if (ret)
+ goto err;
+
+ qp_init_attr->qp_type = res->ai_qp_type;
+ ret = rdma_create_qp(cm_id, pd, qp_init_attr);
+ if (ret)
+ goto err;
+
+ *id = cm_id;
+ return 0;
+
+err:
+ rdma_destroy_id(cm_id);
+ return ret;
+}
diff --git a/src/librdmacm.map b/src/librdmacm.map
index f6af452..85d5b3c 100644
--- a/src/librdmacm.map
+++ b/src/librdmacm.map
@@ -31,5 +31,6 @@ RDMACM_1.0 {
rdma_getaddrinfo;
rdma_freeaddrinfo;
rdma_get_request;
+ rdma_create_ep;
local: *;
};
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 23/37] librdmacm: add rdma_destoy_ep
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 11 +++++++++++
src/cma.c | 8 ++++++++
src/librdmacm.map | 1 +
3 files changed, 20 insertions(+), 0 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 74762a2..289b728 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -258,6 +258,17 @@ int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr);
/**
+ * rdma_destroy_ep - Deallocates a communication identifier and qp.
+ * @id: The communication identifer to destroy.
+ * Description:
+ * Destroys the specified rdma_cm_id and any associated QP created
+ * on that id.
+ * See also:
+ * rdma_create_ep
+ */
+void rdma_destroy_ep(struct rdma_cm_id *id);
+
+/**
* rdma_destroy_id - Release a communication identifier.
* @id: The communication identifier to destroy.
* Description:
diff --git a/src/cma.c b/src/cma.c
index b911b3f..3b80542 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -1142,6 +1142,7 @@ void rdma_destroy_qp(struct rdma_cm_id *id)
{
ibv_destroy_qp(id->qp);
ucma_destroy_cqs(id);
+ id->qp = NULL;
}
static int ucma_valid_param(struct cma_id_private *id_priv,
@@ -2033,3 +2034,10 @@ err:
rdma_destroy_id(cm_id);
return ret;
}
+
+void rdma_destroy_ep(struct rdma_cm_id *id)
+{
+ if (id->qp)
+ rdma_destroy_qp(id);
+ rdma_destroy_id(id);
+}
diff --git a/src/librdmacm.map b/src/librdmacm.map
index 85d5b3c..19b193a 100644
--- a/src/librdmacm.map
+++ b/src/librdmacm.map
@@ -32,5 +32,6 @@ RDMACM_1.0 {
rdma_freeaddrinfo;
rdma_get_request;
rdma_create_ep;
+ rdma_destroy_ep;
local: *;
};
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 33/37] librdmacm/rdma_server: add new sample server application
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Provide a simple server application to demonstrate the minimal
amount of coding needed to accept a connection request from
a client and exchange messages.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Makefile.am | 4 +
examples/rdma_server.c | 149 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 152 insertions(+), 1 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index 6071599..8701002 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -18,7 +18,7 @@ src_librdmacm_la_LDFLAGS = -version-info 1 -export-dynamic \
src_librdmacm_la_DEPENDENCIES = $(srcdir)/src/librdmacm.map
bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy examples/mckey \
- examples/rdma_client
+ examples/rdma_client examples/rdma_server
examples_ucmatose_SOURCES = examples/cmatose.c
examples_ucmatose_LDADD = $(top_builddir)/src/librdmacm.la
examples_rping_SOURCES = examples/rping.c
@@ -29,6 +29,8 @@ examples_mckey_SOURCES = examples/mckey.c
examples_mckey_LDADD = $(top_builddir)/src/librdmacm.la
examples_rdma_client_SOURCES = examples/rdma_client.c
examples_rdma_client_LDADD = $(top_builddir)/src/librdmacm.la
+examples_rdma_server_SOURCES = examples/rdma_server.c
+examples_rdma_server_LDADD = $(top_builddir)/src/librdmacm.la
librdmacmincludedir = $(includedir)/rdma $(includedir)/infiniband
diff --git a/examples/rdma_server.c b/examples/rdma_server.c
new file mode 100644
index 0000000..2831d0c
--- /dev/null
+++ b/examples/rdma_server.c
@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2005-2009 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under the OpenIB.org BSD license
+ * below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <getopt.h>
+#include <netdb.h>
+#include <rdma/rdma_cma.h>
+#include <rdma/rdma_verbs.h>
+
+static char *port = "7471";
+
+struct rdma_cm_id *listen_id, *id;
+struct ibv_mr *mr;
+uint8_t send_msg[16];
+uint8_t recv_msg[16];
+
+static int run(void)
+{
+ struct rdma_addrinfo hints, *res;
+ struct ibv_qp_init_attr attr;
+ struct ibv_wc wc;
+ int ret;
+
+ memset(&hints, 0, sizeof hints);
+ hints.ai_flags = RAI_PASSIVE;
+ hints.ai_port_space = RDMA_PS_TCP;
+ ret = rdma_getaddrinfo(NULL, port, &hints, &res);
+ if (ret) {
+ printf("rdma_getaddrinfo %d\n", errno);
+ return ret;
+ }
+
+ memset(&attr, 0, sizeof attr);
+ attr.cap.max_send_wr = attr.cap.max_recv_wr = 1;
+ attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
+ attr.cap.max_inline_data = 16;
+ attr.sq_sig_all = 1;
+ ret = rdma_create_ep(&listen_id, res, NULL, &attr);
+ rdma_freeaddrinfo(res);
+ if (ret) {
+ printf("rdma_create_ep %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_listen(listen_id, 0);
+ if (ret) {
+ printf("rdma_listen %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_get_request(listen_id, &id);
+ if (ret) {
+ printf("rdma_get_request %d\n", errno);
+ return ret;
+ }
+
+ mr = rdma_reg_msgs(id, recv_msg, 16);
+ if (!mr) {
+ printf("rdma_reg_msgs %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);
+ if (ret) {
+ printf("rdma_post_recv %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_accept(id, NULL);
+ if (ret) {
+ printf("rdma_connect %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_get_recv_comp(id, &wc);
+ if (ret <= 0) {
+ printf("rdma_get_recv_comp %d\n", ret);
+ return ret;
+ }
+
+ ret = rdma_post_send(id, NULL, send_msg, 16, NULL, IBV_SEND_INLINE);
+ if (ret) {
+ printf("rdma_post_send %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_get_send_comp(id, &wc);
+ if (ret <= 0) {
+ printf("rdma_get_send_comp %d\n", ret);
+ return ret;
+ }
+
+ rdma_disconnect(id);
+ rdma_dereg_mr(mr);
+ rdma_destroy_ep(id);
+ rdma_destroy_ep(listen_id);
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ int op, ret;
+
+ while ((op = getopt(argc, argv, "p:")) != -1) {
+ switch (op) {
+ case 'p':
+ port = optarg;
+ break;
+ default:
+ printf("usage: %s\n", argv[0]);
+ printf("\t[-p port_number]\n");
+ exit(1);
+ }
+ }
+
+ printf("rdma_server: start\n");
+ ret = run();
+ printf("rdma_server: end %d\n", ret);
+ return ret;
+}
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 30/37] librdmacm: format IB CM private data RDMA CM header
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/acm.c | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++-------
src/cma.c | 32 +++++++++++++++++++++++++++++---
2 files changed, 81 insertions(+), 10 deletions(-)
diff --git a/src/acm.c b/src/acm.c
index 5dd8899..8975b97 100644
--- a/src/acm.c
+++ b/src/acm.c
@@ -51,6 +51,18 @@ static pthread_mutex_t acm_lock = PTHREAD_MUTEX_INITIALIZER;
static int sock;
static short server_port = 6125;
+struct ib_connect_hdr {
+ uint8_t cma_version;
+ uint8_t ip_version; /* IP version: 7:4 */
+ uint16_t port;
+ uint32_t src_addr[4];
+ uint32_t dst_addr[4];
+#define cma_src_ip4 src_addr[3]
+#define cma_src_ip6 src_addr[0]
+#define cma_dst_ip4 dst_addr[3]
+#define cma_dst_ip6 dst_addr[0]
+};
+
void ucma_ib_init(void)
{
struct sockaddr_in addr;
@@ -100,22 +112,22 @@ static void ucma_set_sid(enum rdma_port_space ps, struct sockaddr *addr,
sib->sib_sid_mask = htonll(RDMA_IB_IP_PS_MASK);
}
-static void ucma_ib_convert_addr(struct rdma_addrinfo *rai,
- struct ib_path_record *path)
+static int ucma_ib_convert_addr(struct rdma_addrinfo *rai,
+ struct ib_path_record *path)
{
struct sockaddr_ib *src, *dst;
if (!path)
- return;
+ return ERR(ENODATA);
src = zalloc(sizeof *src);
if (!src)
- return;
+ return ERR(ENOMEM);
dst = zalloc(sizeof *dst);
if (!dst) {
free(src);
- return;
+ return ERR(ENOMEM);
}
src->sib_family = AF_IB;
@@ -140,6 +152,33 @@ static void ucma_ib_convert_addr(struct rdma_addrinfo *rai,
rai->ai_family = AF_IB;
rai->ai_port_space = RDMA_PS_IB;
+ return 0;
+}
+
+static void ucma_ib_format_connect(struct rdma_addrinfo *rai)
+{
+ struct ib_connect_hdr *hdr;
+
+ hdr = zalloc(sizeof *hdr);
+ if (!hdr)
+ return;
+
+ if (rai->ai_family == AF_INET) {
+ hdr->ip_version = 4 << 4;
+ memcpy(&hdr->cma_src_ip4,
+ &((struct sockaddr_in *) rai->ai_src_addr)->sin_addr, 4);
+ memcpy(&hdr->cma_dst_ip4,
+ &((struct sockaddr_in *) rai->ai_dst_addr)->sin_addr, 4);
+ } else {
+ hdr->ip_version = 6 << 4;
+ memcpy(&hdr->cma_src_ip6,
+ &((struct sockaddr_in6 *) rai->ai_src_addr)->sin6_addr, 16);
+ memcpy(&hdr->cma_dst_ip6,
+ &((struct sockaddr_in6 *) rai->ai_dst_addr)->sin6_addr, 16);
+ }
+
+ rai->ai_connect = hdr;
+ rai->ai_connect_len = sizeof(*hdr);
}
static void ucma_ib_save_resp(struct rdma_addrinfo *rai, struct acm_resolve_msg *msg)
@@ -166,8 +205,14 @@ static void ucma_ib_save_resp(struct rdma_addrinfo *rai, struct acm_resolve_msg
rai->ai_route = path_data;
rai->ai_route_len = len;
- if (af_ib_support)
- ucma_ib_convert_addr(rai, pri_path);
+ if (af_ib_support) {
+ ucma_ib_format_connect(rai);
+ if (ucma_ib_convert_addr(rai, pri_path) &&
+ rai->ai_connect) {
+ free(rai->ai_connect);
+ rai->ai_connect_len = 0;
+ }
+ }
return;
err:
free(path_data);
diff --git a/src/cma.c b/src/cma.c
index 6ac949a..faf4264 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -106,6 +106,8 @@ struct cma_device {
struct cma_id_private {
struct rdma_cm_id id;
struct cma_device *cma_dev;
+ void *connect;
+ size_t connect_len;
int events_completed;
int connect_error;
int sync;
@@ -363,6 +365,8 @@ static void ucma_free_id(struct cma_id_private *id_priv)
if (id_priv->sync)
rdma_destroy_event_channel(id_priv->id.channel);
+ if (id_priv->connect)
+ free(id_priv->connect);
free(id_priv);
}
@@ -1186,15 +1190,20 @@ static void ucma_copy_conn_param_to_kern(struct cma_id_private *id_priv,
dst->initiator_depth = id_priv->initiator_depth;
dst->valid = 1;
+ if (id_priv->connect_len) {
+ memcpy(dst->private_data, id_priv->connect, id_priv->connect_len);
+ dst->private_data_len = id_priv->connect_len;
+ }
+
if (src) {
dst->flow_control = src->flow_control;
dst->retry_count = src->retry_count;
dst->rnr_retry_count = src->rnr_retry_count;
if (src->private_data && src->private_data_len) {
- memcpy(dst->private_data, src->private_data,
- src->private_data_len);
- dst->private_data_len = src->private_data_len;
+ memcpy(dst->private_data + dst->private_data_len,
+ src->private_data, src->private_data_len);
+ dst->private_data_len += src->private_data_len;
}
} else {
dst->retry_count = 7;
@@ -1238,6 +1247,11 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
if (ret != size)
return (ret >= 0) ? ERR(ENODATA) : -1;
+ if (id_priv->connect) {
+ free(id_priv->connect);
+ id_priv->connect_len = 0;
+ }
+
return ucma_complete(id_priv);
}
@@ -2038,6 +2052,7 @@ int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
{
struct rdma_cm_id *cm_id;
+ struct cma_id_private *id_priv;
int ret;
ret = rdma_create_id2(NULL, &cm_id, NULL, res->ai_port_space, res->ai_qp_type);
@@ -2075,6 +2090,17 @@ int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
if (ret)
goto err;
+ if (res->ai_connect_len) {
+ id_priv = container_of(cm_id, struct cma_id_private, id);
+ id_priv->connect = malloc(res->ai_connect_len);
+ if (!id_priv->connect) {
+ ret = ERR(ENOMEM);
+ goto err;
+ }
+ memcpy(id_priv->connect, res->ai_connect, res->ai_connect_len);
+ id_priv->connect_len = res->ai_connect_len;
+ }
+
out:
*id = cm_id;
return 0;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox