* [PATCH 9/26 v2] rdma/cm: set qkey for port space IB
From: Sean Hefty @ 2010-04-07 0:49 UTC (permalink / raw)
To: Hefty, Sean, linux-rdma-u79uwXL29TY76Z2rM5mHXA
In-Reply-To: <0B94CA7CD279474EAAF856BCEC45BDED-Zpru7NauK7drdx17CPfAsdBPR1lH4CV8@public.gmane.org>
Allow the user to specify the qkey when using port space IB. The
qkey is added to struct rdma_ucm_conn_param in place of a reserved
field, but for backwards compatibility, is only accessed if the
associated rdma_cm_id is using the IB port space.
The qkey associated with an id is now set when sending or receiving the
SIDR REP. If the qkey is read before being set, to modify the QP
state early for example, it reverts to the default value.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Changes from v1:
Allow the user to specify the QKey, rather than automatically using
the hard-coded RDMA_QKEY_UDP value.
drivers/infiniband/core/cma.c | 33 ++++++++++++++++++++-------------
drivers/infiniband/core/ucma.c | 11 ++++++++---
include/rdma/rdma_cm.h | 1 +
include/rdma/rdma_user_cm.h | 2 +-
4 files changed, 30 insertions(+), 17 deletions(-)
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index ccf47c6..f8ca424 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -293,16 +293,25 @@ static void cma_detach_from_dev(struct rdma_id_private *id_priv)
id_priv->cma_dev = NULL;
}
-static int cma_set_qkey(struct rdma_id_private *id_priv)
+static int cma_set_qkey(struct rdma_id_private *id_priv, u32 qkey)
{
struct ib_sa_mcmember_rec rec;
int ret = 0;
- if (id_priv->qkey)
+ if (id_priv->qkey) {
+ if (qkey && id_priv->qkey != qkey)
+ return -EINVAL;
return 0;
+ }
+
+ if (qkey) {
+ id_priv->qkey = qkey;
+ return 0;
+ }
switch (id_priv->id.ps) {
case RDMA_PS_UDP:
+ case RDMA_PS_IB:
id_priv->qkey = RDMA_UDP_QKEY;
break;
case RDMA_PS_IPOIB:
@@ -565,7 +574,7 @@ static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
*qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
if (id_priv->id.qp_type == IB_QPT_UD) {
- ret = cma_set_qkey(id_priv);
+ ret = cma_set_qkey(id_priv, 0);
if (ret)
return ret;
@@ -2302,15 +2311,10 @@ static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
event.status = ib_event->param.sidr_rep_rcvd.status;
break;
}
- ret = cma_set_qkey(id_priv);
+ ret = cma_set_qkey(id_priv, rep->qkey);
if (ret) {
event.event = RDMA_CM_EVENT_ADDR_ERROR;
- event.status = -EINVAL;
- break;
- }
- if (id_priv->qkey != rep->qkey) {
- event.event = RDMA_CM_EVENT_UNREACHABLE;
- event.status = -EINVAL;
+ event.status = ret;
break;
}
ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
@@ -2583,7 +2587,7 @@ static int cma_accept_iw(struct rdma_id_private *id_priv,
}
static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
- enum ib_cm_sidr_status status,
+ enum ib_cm_sidr_status status, u32 qkey,
const void *private_data, int private_data_len)
{
struct ib_cm_sidr_rep_param rep;
@@ -2592,7 +2596,7 @@ static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
memset(&rep, 0, sizeof rep);
rep.status = status;
if (status == IB_SIDR_SUCCESS) {
- ret = cma_set_qkey(id_priv);
+ ret = cma_set_qkey(id_priv, qkey);
if (ret)
return ret;
rep.qp_num = id_priv->qp_num;
@@ -2622,6 +2626,7 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
case RDMA_TRANSPORT_IB:
if (id->qp_type == IB_QPT_UD)
ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
+ conn_param->qkey,
conn_param->private_data,
conn_param->private_data_len);
else if (conn_param)
@@ -2682,7 +2687,7 @@ int rdma_reject(struct rdma_cm_id *id, const void *private_data,
switch (rdma_node_get_transport(id->device->node_type)) {
case RDMA_TRANSPORT_IB:
if (id->qp_type == IB_QPT_UD)
- ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
+ ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 0,
private_data, private_data_len);
else
ret = ib_send_cm_rej(id_priv->cm_id.ib,
@@ -2743,6 +2748,8 @@ static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
cma_disable_callback(id_priv, CMA_ADDR_RESOLVED))
return 0;
+ if (!status)
+ status = cma_set_qkey(id_priv, be32_to_cpu(multicast->rec.qkey));
mutex_lock(&id_priv->qp_mutex);
if (!status && id_priv->id.qp)
status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c
index 723fe83..07afd38 100644
--- a/drivers/infiniband/core/ucma.c
+++ b/drivers/infiniband/core/ucma.c
@@ -655,7 +655,8 @@ out:
return ret;
}
-static void ucma_copy_conn_param(struct rdma_conn_param *dst,
+static void ucma_copy_conn_param(struct rdma_cm_id *id,
+ struct rdma_conn_param *dst,
struct rdma_ucm_conn_param *src)
{
dst->private_data = src->private_data;
@@ -667,6 +668,10 @@ static void ucma_copy_conn_param(struct rdma_conn_param *dst,
dst->rnr_retry_count = src->rnr_retry_count;
dst->srq = src->srq;
dst->qp_num = src->qp_num;
+ if (id->ps == RDMA_PS_IB)
+ dst->qkey = src->qkey;
+ else
+ dst->qkey = 0;
}
static ssize_t ucma_connect(struct ucma_file *file, const char __user *inbuf,
@@ -687,7 +692,7 @@ static ssize_t ucma_connect(struct ucma_file *file, const char __user *inbuf,
if (IS_ERR(ctx))
return PTR_ERR(ctx);
- ucma_copy_conn_param(&conn_param, &cmd.conn_param);
+ ucma_copy_conn_param(ctx->cm_id, &conn_param, &cmd.conn_param);
ret = rdma_connect(ctx->cm_id, &conn_param);
ucma_put_ctx(ctx);
return ret;
@@ -731,7 +736,7 @@ static ssize_t ucma_accept(struct ucma_file *file, const char __user *inbuf,
if (cmd.conn_param.valid) {
ctx->uid = cmd.uid;
- ucma_copy_conn_param(&conn_param, &cmd.conn_param);
+ ucma_copy_conn_param(ctx->cm_id, &conn_param, &cmd.conn_param);
ret = rdma_accept(ctx->cm_id, &conn_param);
} else
ret = rdma_accept(ctx->cm_id, NULL);
diff --git a/include/rdma/rdma_cm.h b/include/rdma/rdma_cm.h
index 68e73d1..363e7a3 100644
--- a/include/rdma/rdma_cm.h
+++ b/include/rdma/rdma_cm.h
@@ -99,6 +99,7 @@ struct rdma_conn_param {
/* Fields below ignored if a QP is created on the rdma_cm_id. */
u8 srq;
u32 qp_num;
+ u32 qkey;
};
struct rdma_ud_param {
diff --git a/include/rdma/rdma_user_cm.h b/include/rdma/rdma_user_cm.h
index d88c23c..fa6693c 100644
--- a/include/rdma/rdma_user_cm.h
+++ b/include/rdma/rdma_user_cm.h
@@ -131,7 +131,7 @@ struct rdma_ucm_query_route_resp {
struct rdma_ucm_conn_param {
__u32 qp_num;
- __u32 reserved;
+ __u32 qkey;
__u8 private_data[RDMA_MAX_PRIVATE_DATA];
__u8 private_data_len;
__u8 srq;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* RE: [PATCH 18/26] rdma/ucm: support querying for AF_IB addresses
From: Sean Hefty @ 2010-04-07 0:49 UTC (permalink / raw)
To: Hefty, Sean, linux-rdma-u79uwXL29TY76Z2rM5mHXA
In-Reply-To: <90625C05F167432FB27BE6E40893FFF1-Zpru7NauK7drdx17CPfAsdBPR1lH4CV8@public.gmane.org>
The sockaddr structure for AF_IB is larger than sockaddr_in6.
The rdma cm user space ABI uses the latter to exchange address
information between user space and the kernel.
To support querying for larger addresses, define a new query
command that exchanges data using sockaddr_storage, rather
than sockaddr_in6. Unlike the existing query_route command,
the new command only returns address information. Route
(i.e. path record) data is separated.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Changes from v1:
Updated to resolve conflicts resulting from changes to patch 9
titled: 'rdma/cm: set qkey for port space IB'
drivers/infiniband/core/ucma.c | 76 +++++++++++++++++++++++++++++++++++++++-
include/rdma/rdma_user_cm.h | 22 ++++++++++--
2 files changed, 93 insertions(+), 5 deletions(-)
diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c
index 07afd38..685e7fa 100644
--- a/drivers/infiniband/core/ucma.c
+++ b/drivers/infiniband/core/ucma.c
@@ -44,6 +44,7 @@
#include <rdma/ib_marshall.h>
#include <rdma/rdma_cm.h>
#include <rdma/rdma_cm_ib.h>
+#include <rdma/ib_addr.h>
MODULE_AUTHOR("Sean Hefty");
MODULE_DESCRIPTION("RDMA Userspace Connection Manager Access");
@@ -608,7 +609,7 @@ static ssize_t ucma_query_route(struct ucma_file *file,
const char __user *inbuf,
int in_len, int out_len)
{
- struct rdma_ucm_query_route cmd;
+ struct rdma_ucm_query cmd;
struct rdma_ucm_query_route_resp resp;
struct ucma_context *ctx;
struct sockaddr *addr;
@@ -655,6 +656,76 @@ out:
return ret;
}
+static void ucma_query_device_addr(struct rdma_cm_id *cm_id,
+ struct rdma_ucm_query_addr_resp *resp)
+{
+ if (!cm_id->device)
+ return;
+
+ resp->node_guid = (__force __u64) cm_id->device->node_guid;
+ resp->port_num = cm_id->port_num;
+ resp->pkey = (__force __u16) cpu_to_be16(
+ ib_addr_get_pkey(&cm_id->route.addr.dev_addr));
+}
+
+static ssize_t ucma_query_addr(struct ucma_context *ctx,
+ void __user *response, int out_len)
+{
+ struct rdma_ucm_query_addr_resp resp;
+ struct sockaddr *addr;
+ int ret = 0;
+
+ if (out_len < sizeof(resp))
+ return -ENOSPC;
+
+ memset(&resp, 0, sizeof resp);
+
+ addr = (struct sockaddr *) &ctx->cm_id->route.addr.src_addr;
+ resp.src_size = rdma_addr_size(addr);
+ memcpy(&resp.src_addr, addr, resp.src_size);
+
+ addr = (struct sockaddr *) &ctx->cm_id->route.addr.dst_addr;
+ resp.dst_size = rdma_addr_size(addr);
+ memcpy(&resp.dst_addr, addr, resp.dst_size);
+
+ ucma_query_device_addr(ctx->cm_id, &resp);
+
+ if (copy_to_user(response, &resp, sizeof(resp)))
+ ret = -EFAULT;
+
+ return ret;
+}
+
+static ssize_t ucma_query(struct ucma_file *file,
+ const char __user *inbuf,
+ int in_len, int out_len)
+{
+ struct rdma_ucm_query cmd;
+ struct ucma_context *ctx;
+ void __user *response;
+ int ret;
+
+ if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+ return -EFAULT;
+
+ response = (void __user *)(unsigned long) cmd.response;
+ ctx = ucma_get_ctx(file, cmd.id);
+ if (IS_ERR(ctx))
+ return PTR_ERR(ctx);
+
+ switch (cmd.option) {
+ case RDMA_USER_CM_QUERY_ADDR:
+ ret = ucma_query_addr(ctx, response, out_len);
+ break;
+ default:
+ ret = -ENOSYS;
+ break;
+ }
+
+ ucma_put_ctx(ctx);
+ return ret;
+}
+
static void ucma_copy_conn_param(struct rdma_cm_id *id,
struct rdma_conn_param *dst,
struct rdma_ucm_conn_param *src)
@@ -1178,7 +1249,8 @@ static ssize_t (*ucma_cmd_table[])(struct ucma_file *file,
[RDMA_USER_CM_CMD_NOTIFY] = ucma_notify,
[RDMA_USER_CM_CMD_JOIN_MCAST] = ucma_join_multicast,
[RDMA_USER_CM_CMD_LEAVE_MCAST] = ucma_leave_multicast,
- [RDMA_USER_CM_CMD_MIGRATE_ID] = ucma_migrate_id
+ [RDMA_USER_CM_CMD_MIGRATE_ID] = ucma_migrate_id,
+ [RDMA_USER_CM_CMD_QUERY] = ucma_query
};
static ssize_t ucma_write(struct file *filp, const char __user *buf,
diff --git a/include/rdma/rdma_user_cm.h b/include/rdma/rdma_user_cm.h
index fa6693c..1bf2da8 100644
--- a/include/rdma/rdma_user_cm.h
+++ b/include/rdma/rdma_user_cm.h
@@ -61,7 +61,8 @@ enum {
RDMA_USER_CM_CMD_NOTIFY,
RDMA_USER_CM_CMD_JOIN_MCAST,
RDMA_USER_CM_CMD_LEAVE_MCAST,
- RDMA_USER_CM_CMD_MIGRATE_ID
+ RDMA_USER_CM_CMD_MIGRATE_ID,
+ RDMA_USER_CM_CMD_QUERY
};
/*
@@ -113,10 +114,14 @@ struct rdma_ucm_resolve_route {
__u32 timeout_ms;
};
-struct rdma_ucm_query_route {
+enum {
+ RDMA_USER_CM_QUERY_ADDR
+};
+
+struct rdma_ucm_query {
__u64 response;
__u32 id;
- __u32 reserved;
+ __u32 option;
};
struct rdma_ucm_query_route_resp {
@@ -129,6 +134,17 @@ struct rdma_ucm_query_route_resp {
__u8 reserved[3];
};
+struct rdma_ucm_query_addr_resp {
+ __u64 node_guid;
+ __u8 port_num;
+ __u8 reserved;
+ __u16 pkey;
+ __u16 src_size;
+ __u16 dst_size;
+ struct sockaddr_storage src_addr;
+ struct sockaddr_storage dst_addr;
+};
+
struct rdma_ucm_conn_param {
__u32 qp_num;
__u32 qkey;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 26/26 v2] rdma/ucm: allow user space to specify AF_IB when joining multicast
From: Sean Hefty @ 2010-04-07 0:49 UTC (permalink / raw)
To: Hefty, Sean, linux-rdma-u79uwXL29TY76Z2rM5mHXA
In-Reply-To: <9D903E76C4EA4CF48BF2548F2BC8D2EA-Zpru7NauK7drdx17CPfAsdBPR1lH4CV8@public.gmane.org>
Allow user space applications to join multicast groups using MGIDs
directly. MGIDs may be passed using AF_IB addresses. Since the
current multicast join command only supports addresses as large as
sockaddr_in6, define a new structure for joining addresses specified
using sockaddr_ib.
Since port space IB allows the user to specify the qkey when
resolving a remote UD QP address, when joining the multicast
group use the qkey value, if one has been assigned.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Changes from v1:
Restrict PS IB to using AF_IB multicast addresses.
Use the qkey stored with the rdma_cm_id if it has been set.
Note: specifying transport specific multicast parameters, such
as the qkey, are outside of this series. This patch simply ensures
that the qkey for a multicast join is in sync with whatever value
the user specified for unicast operation. Future support for transport
specific parameters can either be accomplished by extending
struct rdma_ucm_join_mcast or through a set_option command.
drivers/infiniband/core/cma.c | 12 +++++++--
drivers/infiniband/core/ucma.c | 55 ++++++++++++++++++++++++++++++++--------
include/rdma/rdma_user_cm.h | 12 ++++++++-
3 files changed, 65 insertions(+), 14 deletions(-)
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index a642325..a42dd94 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -2934,6 +2934,8 @@ static void cma_set_mgid(struct rdma_id_private *id_priv,
0xFF10A01B)) {
/* IPv6 address is an SA assigned MGID. */
memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
+ } else if (addr->sa_family == AF_IB) {
+ memcpy(mgid, &((struct sockaddr_ib *) addr)->sib_addr, sizeof *mgid);
} else if ((addr->sa_family == AF_INET6)) {
ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map);
if (id_priv->id.ps == RDMA_PS_UDP)
@@ -2961,9 +2963,12 @@ static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
if (ret)
return ret;
+ ret = cma_set_qkey(id_priv, 0);
+ if (ret)
+ return ret;
+
cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
- if (id_priv->id.ps == RDMA_PS_UDP)
- rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
+ rec.qkey = cpu_to_be32(id_priv->qkey);
rdma_addr_get_sgid(dev_addr, &rec.port_gid);
rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
rec.join_state = 1;
@@ -3000,6 +3005,9 @@ int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
!cma_comp(id_priv, CMA_ADDR_RESOLVED))
return -EINVAL;
+ if (id_priv->id.ps == RDMA_PS_IB && addr->sa_family != AF_IB)
+ return -EINVAL;
+
mc = kmalloc(sizeof *mc, GFP_KERNEL);
if (!mc)
return -ENOMEM;
diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c
index da85dee..ab0e2f9 100644
--- a/drivers/infiniband/core/ucma.c
+++ b/drivers/infiniband/core/ucma.c
@@ -1166,23 +1166,23 @@ static ssize_t ucma_notify(struct ucma_file *file, const char __user *inbuf,
return ret;
}
-static ssize_t ucma_join_ip_multicast(struct ucma_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
+static ssize_t ucma_process_join(struct ucma_file *file,
+ struct rdma_ucm_join_mcast *cmd, int out_len)
{
- struct rdma_ucm_join_ip_mcast cmd;
struct rdma_ucm_create_id_resp resp;
struct ucma_context *ctx;
struct ucma_multicast *mc;
+ struct sockaddr *addr;
int ret;
if (out_len < sizeof(resp))
return -ENOSPC;
- if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
- return -EFAULT;
+ addr = (struct sockaddr *) &cmd->addr;
+ if (cmd->reserved || !cmd->addr_size || (cmd->addr_size != rdma_addr_size(addr)))
+ return -EINVAL;
- ctx = ucma_get_ctx(file, cmd.id);
+ ctx = ucma_get_ctx(file, cmd->id);
if (IS_ERR(ctx))
return PTR_ERR(ctx);
@@ -1193,14 +1193,14 @@ static ssize_t ucma_join_ip_multicast(struct ucma_file *file,
goto err1;
}
- mc->uid = cmd.uid;
- memcpy(&mc->addr, &cmd.addr, sizeof cmd.addr);
+ mc->uid = cmd->uid;
+ memcpy(&mc->addr, addr, cmd->addr_size);
ret = rdma_join_multicast(ctx->cm_id, (struct sockaddr *) &mc->addr, mc);
if (ret)
goto err2;
resp.id = mc->id;
- if (copy_to_user((void __user *)(unsigned long)cmd.response,
+ if (copy_to_user((void __user *)(unsigned long) cmd->response,
&resp, sizeof(resp))) {
ret = -EFAULT;
goto err3;
@@ -1225,6 +1225,38 @@ err1:
return ret;
}
+static ssize_t ucma_join_ip_multicast(struct ucma_file *file,
+ const char __user *inbuf,
+ int in_len, int out_len)
+{
+ struct rdma_ucm_join_ip_mcast cmd;
+ struct rdma_ucm_join_mcast join_cmd;
+
+ if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+ return -EFAULT;
+
+ join_cmd.response = cmd.response;
+ join_cmd.uid = cmd.uid;
+ join_cmd.id = cmd.id;
+ join_cmd.addr_size = rdma_addr_size((struct sockaddr *) &cmd.addr);
+ join_cmd.reserved = 0;
+ memcpy(&join_cmd.addr, &cmd.addr, join_cmd.addr_size);
+
+ return ucma_process_join(file, &join_cmd, out_len);
+}
+
+static ssize_t ucma_join_multicast(struct ucma_file *file,
+ const char __user *inbuf,
+ int in_len, int out_len)
+{
+ struct rdma_ucm_join_mcast cmd;
+
+ if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+ return -EFAULT;
+
+ return ucma_process_join(file, &cmd, out_len);
+}
+
static ssize_t ucma_leave_multicast(struct ucma_file *file,
const char __user *inbuf,
int in_len, int out_len)
@@ -1388,7 +1420,8 @@ static ssize_t (*ucma_cmd_table[])(struct ucma_file *file,
[RDMA_USER_CM_CMD_MIGRATE_ID] = ucma_migrate_id,
[RDMA_USER_CM_CMD_QUERY] = ucma_query,
[RDMA_USER_CM_CMD_BIND] = ucma_bind,
- [RDMA_USER_CM_CMD_RESOLVE_ADDR] = ucma_resolve_addr
+ [RDMA_USER_CM_CMD_RESOLVE_ADDR] = ucma_resolve_addr,
+ [RDMA_USER_CM_CMD_JOIN_MCAST] = ucma_join_multicast
};
static ssize_t ucma_write(struct file *filp, const char __user *buf,
diff --git a/include/rdma/rdma_user_cm.h b/include/rdma/rdma_user_cm.h
index 8b2e9fb..a35c2da 100644
--- a/include/rdma/rdma_user_cm.h
+++ b/include/rdma/rdma_user_cm.h
@@ -64,7 +64,8 @@ enum {
RDMA_USER_CM_CMD_MIGRATE_ID,
RDMA_USER_CM_CMD_QUERY,
RDMA_USER_CM_CMD_BIND,
- RDMA_USER_CM_CMD_RESOLVE_ADDR
+ RDMA_USER_CM_CMD_RESOLVE_ADDR,
+ RDMA_USER_CM_CMD_JOIN_MCAST
};
/*
@@ -242,6 +243,15 @@ struct rdma_ucm_join_ip_mcast {
__u32 id;
};
+struct rdma_ucm_join_mcast {
+ __u64 response; /* rdma_ucma_create_id_resp */
+ __u64 uid;
+ __u32 id;
+ __u16 addr_size;
+ __u16 reserved;
+ struct sockaddr_storage addr;
+};
+
struct rdma_ucm_get_event {
__u64 response;
};
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* Re: [PATCH v2 26/51] IB/qib: Add qib_mr.c
From: Roland Dreier @ 2010-04-07 4:15 UTC (permalink / raw)
To: Ralph Campbell; +Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA
In-Reply-To: <20091203190521.29507.6261.stgit-/vjeY7uYZjrPXfVEPVhPGq6RkeBMCJyt@public.gmane.org>
> +struct ib_fmr *qib_alloc_fmr(struct ib_pd *pd, int mr_access_flags,
> + struct ib_fmr_attr *fmr_attr)
Instead of the crufty old FMR API, I think it would be preferable to
implement the real IB spec fast registration through a work queue
stuff. I understand that it might take some effort to do that but if we
end up with our newest driver stuck with FMRs, we'll never be able to
move away from that.
--
Roland Dreier <rolandd-FYB4Gu1CFyUAvxtiuMwx3w@public.gmane.org> || For corporate legal information go to:
http://www.cisco.com/web/about/doing_business/legal/cri/index.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* [PATCH v2] opensm/osmeventplugin: added couple of events to monitor SM
From: Yevgeny Kliteynik @ 2010-04-07 9:20 UTC (permalink / raw)
To: Sasha Khapyorsky; +Cc: Linux RDMA
In-Reply-To: <4B587AFA.9020102-LDSdmyG8hGV8YrgS2mwiifqBs+8SCbDb@public.gmane.org>
Hi Sasha,
I've added a couple of new events that allow event
plug-in to see what SM is doing, when it is sweeping
and when it updates dump files:
OSM_EVENT_ID_L_SWEEP_STARTED,
OSM_EVENT_ID_L_SWEEP_DONE,
OSM_EVENT_ID_H_SWEEP_STARTED,
OSM_EVENT_ID_H_SWEEP_DONE,
OSM_EVENT_ID_REROUTE_DONE,
OSM_EVENT_ID_ENTERING_STANDBY,
OSM_EVENT_ID_SM_PORT_DOWN,
OSM_EVENT_ID_SA_DB_DUMPED
The last event is reported when SA DB was actually dumped.
I'm thinking of similar optimization for guid2lid file - it
doesn't have to be dumped at the end of each heavy sweep,
as many heavy sweeps don't really happen because of nodes
appearing/disappearing.
Signed-off-by: Yevgeny Kliteynik <kliteyn-LDSdmyG8hGV8YrgS2mwiifqBs+8SCbDb@public.gmane.org>
---
Changes from V1:
- added reporting OSM_EVENT_ID_H_SWEEP_DONE event
- rebased to latest master
opensm/include/opensm/osm_event_plugin.h | 10 +++++++++-
opensm/opensm/osm_state_mgr.c | 22 +++++++++++++++++++++-
opensm/osmeventplugin/src/osmeventplugin.c | 24 ++++++++++++++++++++++++
3 files changed, 54 insertions(+), 2 deletions(-)
diff --git a/opensm/include/opensm/osm_event_plugin.h b/opensm/include/opensm/osm_event_plugin.h
index 33d1920..f5a57d7 100644
--- a/opensm/include/opensm/osm_event_plugin.h
+++ b/opensm/include/opensm/osm_event_plugin.h
@@ -72,7 +72,15 @@ typedef enum {
OSM_EVENT_ID_PORT_SELECT,
OSM_EVENT_ID_TRAP,
OSM_EVENT_ID_SUBNET_UP,
- OSM_EVENT_ID_MAX
+ OSM_EVENT_ID_MAX,
+ OSM_EVENT_ID_L_SWEEP_STARTED,
+ OSM_EVENT_ID_L_SWEEP_DONE,
+ OSM_EVENT_ID_H_SWEEP_STARTED,
+ OSM_EVENT_ID_H_SWEEP_DONE,
+ OSM_EVENT_ID_REROUTE_DONE,
+ OSM_EVENT_ID_ENTERING_STANDBY,
+ OSM_EVENT_ID_SM_PORT_DOWN,
+ OSM_EVENT_ID_SA_DB_DUMPED
} osm_epi_event_id_t;
typedef struct osm_epi_port_id {
diff --git a/opensm/opensm/osm_state_mgr.c b/opensm/opensm/osm_state_mgr.c
index e43463f..d5dff14 100644
--- a/opensm/opensm/osm_state_mgr.c
+++ b/opensm/opensm/osm_state_mgr.c
@@ -1076,6 +1076,9 @@ static void do_sweep(osm_sm_t * sm)
sm->p_subn->sm_state != IB_SMINFO_STATE_DISCOVERING)
return;
+ osm_opensm_report_event(sm->p_subn->p_osm,
+ OSM_EVENT_ID_L_SWEEP_STARTED, NULL);
+
if (sm->p_subn->coming_out_of_standby)
/*
* Need to force re-write of sm_base_lid to all ports
@@ -1111,6 +1114,8 @@ static void do_sweep(osm_sm_t * sm)
osm_sa_db_file_dump(sm->p_subn->p_osm);
OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE,
"LIGHT SWEEP COMPLETE");
+ osm_opensm_report_event(sm->p_subn->p_osm,
+ OSM_EVENT_ID_L_SWEEP_DONE, NULL);
return;
}
}
@@ -1151,6 +1156,8 @@ static void do_sweep(osm_sm_t * sm)
if (!sm->p_subn->subnet_initialization_error) {
OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE,
"REROUTE COMPLETE");
+ osm_opensm_report_event(sm->p_subn->p_osm,
+ OSM_EVENT_ID_REROUTE_DONE, NULL);
return;
}
}
@@ -1158,6 +1165,9 @@ static void do_sweep(osm_sm_t * sm)
/* go to heavy sweep */
repeat_discovery:
+ osm_opensm_report_event(sm->p_subn->p_osm,
+ OSM_EVENT_ID_H_SWEEP_STARTED, NULL);
+
/* First of all - unset all flags */
sm->p_subn->force_heavy_sweep = FALSE;
sm->p_subn->force_reroute = FALSE;
@@ -1185,6 +1195,8 @@ repeat_discovery:
/* Move to DISCOVERING state */
osm_sm_state_mgr_process(sm, OSM_SM_SIGNAL_DISCOVER);
+ osm_opensm_report_event(sm->p_subn->p_osm,
+ OSM_EVENT_ID_SM_PORT_DOWN, NULL);
return;
}
@@ -1205,6 +1217,8 @@ repeat_discovery:
"ENTERING STANDBY STATE");
/* notify master SM about us */
osm_send_trap144(sm, 0);
+ osm_opensm_report_event(sm->p_subn->p_osm,
+ OSM_EVENT_ID_ENTERING_STANDBY, NULL);
return;
}
@@ -1212,6 +1226,9 @@ repeat_discovery:
if (sm->p_subn->force_heavy_sweep)
goto repeat_discovery;
+ osm_opensm_report_event(sm->p_subn->p_osm,
+ OSM_EVENT_ID_H_SWEEP_DONE, NULL);
+
OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE, "HEAVY SWEEP COMPLETE");
/* If we are MASTER - get the highest remote_sm, and
@@ -1375,7 +1392,10 @@ repeat_discovery:
if (osm_log_is_active(sm->p_log, OSM_LOG_VERBOSE) ||
sm->p_subn->opt.sa_db_dump)
- osm_sa_db_file_dump(sm->p_subn->p_osm);
+ if (!osm_sa_db_file_dump(sm->p_subn->p_osm))
+ osm_opensm_report_event(sm->p_subn->p_osm,
+ OSM_EVENT_ID_SA_DB_DUMPED, NULL);
+
}
/*
diff --git a/opensm/osmeventplugin/src/osmeventplugin.c b/opensm/osmeventplugin/src/osmeventplugin.c
index b4d9ce9..5029be2 100644
--- a/opensm/osmeventplugin/src/osmeventplugin.c
+++ b/opensm/osmeventplugin/src/osmeventplugin.c
@@ -176,6 +176,30 @@ static void report(void *_log, osm_epi_event_id_t event_id, void *event_data)
case OSM_EVENT_ID_SUBNET_UP:
fprintf(log->log_file, "Subnet up reported\n");
break;
+ case OSM_EVENT_ID_L_SWEEP_STARTED:
+ fprintf(log->log_file, "Light sweep started\n");
+ break;
+ case OSM_EVENT_ID_L_SWEEP_DONE:
+ fprintf(log->log_file, "Light sweep completed\n");
+ break;
+ case OSM_EVENT_ID_H_SWEEP_STARTED:
+ fprintf(log->log_file, "Heavy sweep started\n");
+ break;
+ case OSM_EVENT_ID_H_SWEEP_DONE:
+ fprintf(log->log_file, "Heavy sweep completed\n");
+ break;
+ case OSM_EVENT_ID_REROUTE_DONE:
+ fprintf(log->log_file, "Re-route completed\n");
+ break;
+ case OSM_EVENT_ID_ENTERING_STANDBY:
+ fprintf(log->log_file, "Entering stand-by state\n");
+ break;
+ case OSM_EVENT_ID_SM_PORT_DOWN:
+ fprintf(log->log_file, "SM port is down\n");
+ break;
+ case OSM_EVENT_ID_SA_DB_DUMPED:
+ fprintf(log->log_file, "SA DB dump file updated\n");
+ break;
case OSM_EVENT_ID_MAX:
default:
osm_log(log->osmlog, OSM_LOG_ERROR,
--
1.5.1.4
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [patch] infiniband: checking the wrong variable
From: Dan Carpenter @ 2010-04-07 9:39 UTC (permalink / raw)
To: Roland Dreier
Cc: Sean Hefty, Hal Rosenstock, Tejun Heo, Christoph Lameter,
Jack Morgenstein, linux-rdma-u79uwXL29TY76Z2rM5mHXA,
kernel-janitors-u79uwXL29TY76Z2rM5mHXA
The intent here was to check the "mfrpl->mapped_page_list" allocation.
We checked "mfrpl->ibfrpl.page_list" earlier.
Signed-off-by: Dan Carpenter <error27-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org>
diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c
index 56147b2..1d27b9a 100644
--- a/drivers/infiniband/hw/mlx4/mr.c
+++ b/drivers/infiniband/hw/mlx4/mr.c
@@ -240,7 +240,7 @@ struct ib_fast_reg_page_list *mlx4_ib_alloc_fast_reg_page_list(struct ib_device
mfrpl->mapped_page_list = dma_alloc_coherent(&dev->dev->pdev->dev,
size, &mfrpl->map,
GFP_KERNEL);
- if (!mfrpl->ibfrpl.page_list)
+ if (!mfrpl->mapped_page_list)
goto err_free;
WARN_ON(mfrpl->map & 0x3f);
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH] [RFC] ummunotify: Userspace support for MMU notifications
From: Eric B Munson @ 2010-04-07 12:30 UTC (permalink / raw)
To: linux-kernel-u79uwXL29TY76Z2rM5mHXA
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA, rolandd-FYB4Gu1CFyUAvxtiuMwx3w,
peterz-wEGCiKHe2LqWVfeAwA7xHQ, pavel-+ZI9xUNit7I,
mingo-X9Un+BFzKDI, Eric B Munson
I am resubmitting this to try and restart the discussion about how
this should be implemented properly. As the dicsussion was left
I see two possible solutions, one is a new class of perf event that
would be setup to prioritize catching every event over resource
consumption that would monitor MMU events filtered by registered
address ranges. The other option is the one presented below, a
character device that uses ioctl and read to register address ranges
and return MMU events. I want to try and pick the best solution
so I can move forward with it.
From: Roland Dreier <rolandd <at> cisco.com>
As discussed in <http://article.gmane.org/gmane.linux.drivers.openib/61925>
and follow-up messages, libraries using RDMA would like to track
precisely when application code changes memory mapping via free(),
munmap(), etc. Current pure-userspace solutions using malloc hooks
and other tricks are not robust, and the feeling among experts is that
the issue is unfixable without kernel help.
We solve this not by implementing the full API proposed in the email
linked above but rather with a simpler and more generic interface,
which may be useful in other contexts. Specifically, we implement a
new character device driver, ummunotify, that creates a /dev/ummunotify
node. A userspace process can open this node read-only and use the fd
as follows:
1. ioctl() to register/unregister an address range to watch in the
kernel (cf struct ummunotify_register_ioctl in <linux/ummunotify.h>).
2. read() to retrieve events generated when a mapping in a watched
address range is invalidated (cf struct ummunotify_event in
<linux/ummunotify.h>). select()/poll()/epoll() and SIGIO are
handled for this IO.
3. mmap() one page at offset 0 to map a kernel page that contains a
generation counter that is incremented each time an event is
generated. This allows userspace to have a fast path that checks
that no events have occurred without a system call.
Thanks to Jason Gunthorpe <jgunthorpe <at> obsidianresearch.com> for
suggestions on the interface design. Also thanks to Jeff Squyres
<jsquyres <at> cisco.com> for prototyping support for this in Open MPI, which
helped find several bugs during development.
Signed-off-by: Roland Dreier <rolandd <at> cisco.com>
Signed-off-by: Eric B Munson <ebmunson-r/Jw6+rmf7HQT0dZR+AlfA@public.gmane.org>
---
Changes since v3:
- Fixed replaced [get|put] user with copy_[from|to]_user to fix x86
builds
---
Documentation/Makefile | 3 +-
drivers/char/Kconfig | 12 +
drivers/char/Makefile | 1 +
drivers/char/ummunotify.c | 567 +++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 582 insertions(+), 1 deletions(-)
create mode 100644 drivers/char/ummunotify.c
diff --git a/Documentation/Makefile b/Documentation/Makefile
index 6fc7ea1..27ba76a 100644
--- a/Documentation/Makefile
+++ b/Documentation/Makefile
@@ -1,3 +1,4 @@
obj-m := DocBook/ accounting/ auxdisplay/ connector/ \
filesystems/ filesystems/configfs/ ia64/ laptops/ networking/ \
- pcmcia/ spi/ timers/ video4linux/ vm/ watchdog/src/
+ pcmcia/ spi/ timers/ video4linux/ vm/ ummunotify/ \
+ watchdog/src/
diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig
index 3141dd3..cf26019 100644
--- a/drivers/char/Kconfig
+++ b/drivers/char/Kconfig
@@ -1111,6 +1111,18 @@ config DEVPORT
depends on ISA || PCI
default y
+config UMMUNOTIFY
+ tristate "Userspace MMU notifications"
+ select MMU_NOTIFIER
+ help
+ The ummunotify (userspace MMU notification) driver creates a
+ character device that can be used by userspace libraries to
+ get notifications when an application's memory mapping
+ changed. This is used, for example, by RDMA libraries to
+ improve the reliability of memory registration caching, since
+ the kernel's MMU notifications can be used to know precisely
+ when to shoot down a cached registration.
+
source "drivers/s390/char/Kconfig"
endmenu
diff --git a/drivers/char/Makefile b/drivers/char/Makefile
index f957edf..521e5de 100644
--- a/drivers/char/Makefile
+++ b/drivers/char/Makefile
@@ -97,6 +97,7 @@ obj-$(CONFIG_NSC_GPIO) += nsc_gpio.o
obj-$(CONFIG_CS5535_GPIO) += cs5535_gpio.o
obj-$(CONFIG_GPIO_TB0219) += tb0219.o
obj-$(CONFIG_TELCLOCK) += tlclk.o
+obj-$(CONFIG_UMMUNOTIFY) += ummunotify.o
obj-$(CONFIG_MWAVE) += mwave/
obj-$(CONFIG_AGP) += agp/
diff --git a/drivers/char/ummunotify.c b/drivers/char/ummunotify.c
new file mode 100644
index 0000000..c14df3f
--- /dev/null
+++ b/drivers/char/ummunotify.c
@@ -0,0 +1,567 @@
+/*
+ * Copyright (c) 2009 Cisco Systems. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version
+ * 2 as published by the Free Software Foundation.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/fs.h>
+#include <linux/init.h>
+#include <linux/list.h>
+#include <linux/miscdevice.h>
+#include <linux/mm.h>
+#include <linux/mmu_notifier.h>
+#include <linux/module.h>
+#include <linux/poll.h>
+#include <linux/rbtree.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/uaccess.h>
+#include <linux/ummunotify.h>
+
+#include <asm/cacheflush.h>
+
+MODULE_AUTHOR("Roland Dreier");
+MODULE_DESCRIPTION("Userspace MMU notifiers");
+MODULE_LICENSE("GPL v2");
+
+/*
+ * Information about an address range userspace has asked us to watch.
+ *
+ * user_cookie: Opaque cookie given to us when userspace registers the
+ * address range.
+ *
+ * start, end: Address range; start is inclusive, end is exclusive.
+ *
+ * hint_start, hint_end: If a single MMU notification event
+ * invalidates the address range, we hold the actual range of
+ * addresses that were invalidated (and set UMMUNOTIFY_FLAG_HINT).
+ * If another event hits this range before userspace reads the
+ * event, we give up and don't try to keep track of which subsets
+ * got invalidated.
+ *
+ * flags: Holds the INVALID flag for ranges that are on the invalid
+ * list and/or the HINT flag for ranges where the hint range holds
+ * good information.
+ *
+ * node: Used to put the range into an rbtree we use to be able to
+ * scan address ranges in order.
+ *
+ * list: Used to put the range on the invalid list when an MMU
+ * notification event hits the range.
+ */
+enum {
+ UMMUNOTIFY_FLAG_INVALID = 1,
+ UMMUNOTIFY_FLAG_HINT = 2,
+};
+
+struct ummunotify_reg {
+ u64 user_cookie;
+ unsigned long start;
+ unsigned long end;
+ unsigned long hint_start;
+ unsigned long hint_end;
+ unsigned long flags;
+ struct rb_node node;
+ struct list_head list;
+};
+
+/*
+ * Context attached to each file that userspace opens.
+ *
+ * mmu_notifier: MMU notifier registered for this context.
+ *
+ * mm: mm_struct for process that created the context; we use this to
+ * hold a reference to the mm to make sure it doesn't go away until
+ * we're done with it.
+ *
+ * reg_tree: RB tree of address ranges being watched, sorted by start
+ * address.
+ *
+ * invalid_list: List of address ranges that have been invalidated by
+ * MMU notification events; as userspace reads events, the address
+ * range corresponding to the event is removed from the list.
+ *
+ * counter: Page that can be mapped read-only by userspace, which
+ * holds a generation count that is incremented each time an event
+ * occurs.
+ *
+ * lock: Spinlock used to protect all context.
+ *
+ * read_wait: Wait queue used to wait for data to become available in
+ * blocking read()s.
+ *
+ * async_queue: Used to implement fasync().
+ *
+ * need_empty: Set when userspace reads an invalidation event, so that
+ * read() knows it must generate an "empty" event when userspace
+ * drains the invalid_list.
+ *
+ * used: Set after userspace does anything with the file, so that the
+ * "exchange flags" ioctl() knows it's too late to change anything.
+ */
+struct ummunotify_file {
+ struct mmu_notifier mmu_notifier;
+ struct mm_struct *mm;
+ struct rb_root reg_tree;
+ struct list_head invalid_list;
+ u64 *counter;
+ spinlock_t lock;
+ wait_queue_head_t read_wait;
+ struct fasync_struct *async_queue;
+ int need_empty;
+ int used;
+};
+
+static void ummunotify_handle_notify(struct mmu_notifier *mn,
+ unsigned long start, unsigned long end)
+{
+ struct ummunotify_file *priv =
+ container_of(mn, struct ummunotify_file, mmu_notifier);
+ struct rb_node *n;
+ struct ummunotify_reg *reg;
+ unsigned long flags;
+ int hit = 0;
+
+ spin_lock_irqsave(&priv->lock, flags);
+
+ for (n = rb_first(&priv->reg_tree); n; n = rb_next(n)) {
+ reg = rb_entry(n, struct ummunotify_reg, node);
+
+ /*
+ * Ranges overlap if they're not disjoint; and they're
+ * disjoint if the end of one is before the start of
+ * the other one. So if both disjointness comparisons
+ * fail then the ranges overlap.
+ *
+ * Since we keep the tree of regions we're watching
+ * sorted by start address, we can end this loop as
+ * soon as we hit a region that starts past the end of
+ * the range for the event we're handling.
+ */
+ if (reg->start >= end)
+ break;
+
+ /*
+ * Just go to the next region if the start of the
+ * range is after the end of the region -- there
+ * might still be more overlapping ranges that have a
+ * greater start.
+ */
+ if (start >= reg->end)
+ continue;
+
+ hit = 1;
+
+ if (test_and_set_bit(UMMUNOTIFY_FLAG_INVALID, ®->flags)) {
+ /* Already on invalid list */
+ clear_bit(UMMUNOTIFY_FLAG_HINT, ®->flags);
+ } else {
+ list_add_tail(®->list, &priv->invalid_list);
+ set_bit(UMMUNOTIFY_FLAG_HINT, ®->flags);
+ reg->hint_start = start;
+ reg->hint_end = end;
+ }
+ }
+
+ if (hit) {
+ ++(*priv->counter);
+ flush_dcache_page(virt_to_page(priv->counter));
+ wake_up_interruptible(&priv->read_wait);
+ kill_fasync(&priv->async_queue, SIGIO, POLL_IN);
+ }
+
+ spin_unlock_irqrestore(&priv->lock, flags);
+}
+
+static void ummunotify_invalidate_page(struct mmu_notifier *mn,
+ struct mm_struct *mm,
+ unsigned long addr)
+{
+ ummunotify_handle_notify(mn, addr, addr + PAGE_SIZE);
+}
+
+static void ummunotify_invalidate_range_start(struct mmu_notifier *mn,
+ struct mm_struct *mm,
+ unsigned long start,
+ unsigned long end)
+{
+ ummunotify_handle_notify(mn, start, end);
+}
+
+static const struct mmu_notifier_ops ummunotify_mmu_notifier_ops = {
+ .invalidate_page = ummunotify_invalidate_page,
+ .invalidate_range_start = ummunotify_invalidate_range_start,
+};
+
+static int ummunotify_open(struct inode *inode, struct file *filp)
+{
+ struct ummunotify_file *priv;
+ int ret;
+
+ if (filp->f_mode & FMODE_WRITE)
+ return -EINVAL;
+
+ priv = kmalloc(sizeof *priv, GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ priv->counter = (void *) get_zeroed_page(GFP_KERNEL);
+ if (!priv->counter) {
+ ret = -ENOMEM;
+ goto err;
+ }
+
+ priv->reg_tree = RB_ROOT;
+ INIT_LIST_HEAD(&priv->invalid_list);
+ spin_lock_init(&priv->lock);
+ init_waitqueue_head(&priv->read_wait);
+ priv->async_queue = NULL;
+ priv->need_empty = 0;
+ priv->used = 0;
+
+ priv->mmu_notifier.ops = &ummunotify_mmu_notifier_ops;
+ /*
+ * Register notifier last, since notifications can occur as
+ * soon as we register....
+ */
+ ret = mmu_notifier_register(&priv->mmu_notifier, current->mm);
+ if (ret)
+ goto err_page;
+
+ priv->mm = current->mm;
+ atomic_inc(&priv->mm->mm_count);
+
+ filp->private_data = priv;
+
+ return 0;
+
+err_page:
+ free_page((unsigned long) priv->counter);
+
+err:
+ kfree(priv);
+ return ret;
+}
+
+static int ummunotify_close(struct inode *inode, struct file *filp)
+{
+ struct ummunotify_file *priv = filp->private_data;
+ struct rb_node *n;
+ struct ummunotify_reg *reg;
+
+ mmu_notifier_unregister(&priv->mmu_notifier, priv->mm);
+ mmdrop(priv->mm);
+ free_page((unsigned long) priv->counter);
+
+ for (n = rb_first(&priv->reg_tree); n; n = rb_next(n)) {
+ reg = rb_entry(n, struct ummunotify_reg, node);
+ kfree(reg);
+ }
+
+ kfree(priv);
+
+ return 0;
+}
+
+static bool ummunotify_readable(struct ummunotify_file *priv)
+{
+ return priv->need_empty || !list_empty(&priv->invalid_list);
+}
+
+static ssize_t ummunotify_read(struct file *filp, char __user *buf,
+ size_t count, loff_t *pos)
+{
+ struct ummunotify_file *priv = filp->private_data;
+ struct ummunotify_reg *reg;
+ ssize_t ret;
+ struct ummunotify_event *events;
+ int max;
+ int n;
+
+ priv->used = 1;
+
+ events = (void *) get_zeroed_page(GFP_KERNEL);
+ if (!events) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ spin_lock_irq(&priv->lock);
+
+ while (!ummunotify_readable(priv)) {
+ spin_unlock_irq(&priv->lock);
+
+ if (filp->f_flags & O_NONBLOCK) {
+ ret = -EAGAIN;
+ goto out;
+ }
+
+ if (wait_event_interruptible(priv->read_wait,
+ ummunotify_readable(priv))) {
+ ret = -ERESTARTSYS;
+ goto out;
+ }
+
+ spin_lock_irq(&priv->lock);
+ }
+
+ max = min_t(size_t, PAGE_SIZE, count) / sizeof *events;
+
+ for (n = 0; n < max; ++n) {
+ if (list_empty(&priv->invalid_list)) {
+ events[n].type = UMMUNOTIFY_EVENT_TYPE_LAST;
+ events[n].user_cookie_counter = *priv->counter;
+ ++n;
+ priv->need_empty = 0;
+ break;
+ }
+
+ reg = list_first_entry(&priv->invalid_list,
+ struct ummunotify_reg, list);
+
+ events[n].type = UMMUNOTIFY_EVENT_TYPE_INVAL;
+ if (test_bit(UMMUNOTIFY_FLAG_HINT, ®->flags)) {
+ events[n].flags = UMMUNOTIFY_EVENT_FLAG_HINT;
+ events[n].hint_start = max(reg->start, reg->hint_start);
+ events[n].hint_end = min(reg->end, reg->hint_end);
+ } else {
+ events[n].hint_start = reg->start;
+ events[n].hint_end = reg->end;
+ }
+ events[n].user_cookie_counter = reg->user_cookie;
+
+ list_del(®->list);
+ reg->flags = 0;
+ priv->need_empty = 1;
+ }
+
+ spin_unlock_irq(&priv->lock);
+
+ if (copy_to_user(buf, events, n * sizeof *events))
+ ret = -EFAULT;
+ else
+ ret = n * sizeof *events;
+
+out:
+ free_page((unsigned long) events);
+ return ret;
+}
+
+static unsigned int ummunotify_poll(struct file *filp,
+ struct poll_table_struct *wait)
+{
+ struct ummunotify_file *priv = filp->private_data;
+
+ poll_wait(filp, &priv->read_wait, wait);
+
+ return ummunotify_readable(priv) ? (POLLIN | POLLRDNORM) : 0;
+}
+
+static long ummunotify_exchange_features(struct ummunotify_file *priv,
+ __u32 __user *arg)
+{
+ u32 feature_mask;
+
+ if (priv->used)
+ return -EINVAL;
+
+ priv->used = 1;
+
+ if (copy_from_user(&feature_mask, arg, sizeof(feature_mask)))
+ return -EFAULT;
+
+ /* No extensions defined at present. */
+ feature_mask = 0;
+
+ if (copy_to_user(arg, &feature_mask, sizeof(feature_mask)))
+ return -EFAULT;
+
+ return 0;
+}
+
+static long ummunotify_register_region(struct ummunotify_file *priv,
+ void __user *arg)
+{
+ struct ummunotify_register_ioctl parm;
+ struct ummunotify_reg *reg, *treg;
+ struct rb_node **n = &priv->reg_tree.rb_node;
+ struct rb_node *pn;
+ int ret = 0;
+
+ if (copy_from_user(&parm, arg, sizeof parm))
+ return -EFAULT;
+
+ priv->used = 1;
+
+ reg = kmalloc(sizeof *reg, GFP_KERNEL);
+ if (!reg)
+ return -ENOMEM;
+
+ reg->user_cookie = parm.user_cookie;
+ reg->start = parm.start;
+ reg->end = parm.end;
+ reg->flags = 0;
+
+ spin_lock_irq(&priv->lock);
+
+ for (pn = rb_first(&priv->reg_tree); pn; pn = rb_next(pn)) {
+ treg = rb_entry(pn, struct ummunotify_reg, node);
+
+ if (treg->user_cookie == parm.user_cookie) {
+ kfree(reg);
+ ret = -EINVAL;
+ goto out;
+ }
+ }
+
+ pn = NULL;
+ while (*n) {
+ pn = *n;
+ treg = rb_entry(pn, struct ummunotify_reg, node);
+
+ if (reg->start <= treg->start)
+ n = &pn->rb_left;
+ else
+ n = &pn->rb_right;
+ }
+
+ rb_link_node(®->node, pn, n);
+ rb_insert_color(®->node, &priv->reg_tree);
+
+out:
+ spin_unlock_irq(&priv->lock);
+
+ return ret;
+}
+
+static long ummunotify_unregister_region(struct ummunotify_file *priv,
+ __u64 __user *arg)
+{
+ u64 user_cookie;
+ struct rb_node *n;
+ struct ummunotify_reg *reg;
+ int ret = -EINVAL;
+
+ if (copy_from_user(&user_cookie, arg, sizeof(user_cookie)))
+ return -EFAULT;
+
+ spin_lock_irq(&priv->lock);
+
+ for (n = rb_first(&priv->reg_tree); n; n = rb_next(n)) {
+ reg = rb_entry(n, struct ummunotify_reg, node);
+
+ if (reg->user_cookie == user_cookie) {
+ rb_erase(n, &priv->reg_tree);
+ if (test_bit(UMMUNOTIFY_FLAG_INVALID, ®->flags))
+ list_del(®->list);
+ kfree(reg);
+ ret = 0;
+ break;
+ }
+ }
+
+ spin_unlock_irq(&priv->lock);
+
+ return ret;
+}
+
+static long ummunotify_ioctl(struct file *filp, unsigned int cmd,
+ unsigned long arg)
+{
+ struct ummunotify_file *priv = filp->private_data;
+ void __user *argp = (void __user *) arg;
+
+ switch (cmd) {
+ case UMMUNOTIFY_EXCHANGE_FEATURES:
+ return ummunotify_exchange_features(priv, argp);
+ case UMMUNOTIFY_REGISTER_REGION:
+ return ummunotify_register_region(priv, argp);
+ case UMMUNOTIFY_UNREGISTER_REGION:
+ return ummunotify_unregister_region(priv, argp);
+ default:
+ return -ENOIOCTLCMD;
+ }
+}
+
+static int ummunotify_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
+{
+ struct ummunotify_file *priv = vma->vm_private_data;
+
+ if (vmf->pgoff != 0)
+ return VM_FAULT_SIGBUS;
+
+ vmf->page = virt_to_page(priv->counter);
+ get_page(vmf->page);
+
+ return 0;
+
+}
+
+static struct vm_operations_struct ummunotify_vm_ops = {
+ .fault = ummunotify_fault,
+};
+
+static int ummunotify_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+ struct ummunotify_file *priv = filp->private_data;
+
+ if (vma->vm_end - vma->vm_start != PAGE_SIZE || vma->vm_pgoff != 0)
+ return -EINVAL;
+
+ vma->vm_ops = &ummunotify_vm_ops;
+ vma->vm_private_data = priv;
+
+ return 0;
+}
+
+static int ummunotify_fasync(int fd, struct file *filp, int on)
+{
+ struct ummunotify_file *priv = filp->private_data;
+
+ return fasync_helper(fd, filp, on, &priv->async_queue);
+}
+
+static const struct file_operations ummunotify_fops = {
+ .owner = THIS_MODULE,
+ .open = ummunotify_open,
+ .release = ummunotify_close,
+ .read = ummunotify_read,
+ .poll = ummunotify_poll,
+ .unlocked_ioctl = ummunotify_ioctl,
+#ifdef CONFIG_COMPAT
+ .compat_ioctl = ummunotify_ioctl,
+#endif
+ .mmap = ummunotify_mmap,
+ .fasync = ummunotify_fasync,
+};
+
+static struct miscdevice ummunotify_misc = {
+ .minor = MISC_DYNAMIC_MINOR,
+ .name = "ummunotify",
+ .fops = &ummunotify_fops,
+};
+
+static int __init ummunotify_init(void)
+{
+ return misc_register(&ummunotify_misc);
+}
+
+static void __exit ummunotify_cleanup(void)
+{
+ misc_deregister(&ummunotify_misc);
+}
+
+module_init(ummunotify_init);
+module_exit(ummunotify_cleanup);
--
1.6.3.3
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [infiniband-diags] [0/3] support --diff and --diffcheck in ibnetdiscover
From: Al Chu @ 2010-04-07 17:05 UTC (permalink / raw)
To: sashak-smomgflXvOZWk0Htik3J/w@public.gmane.org
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
Hey Sasha,
The following sets of patches implement a --diff and --diffcheck options
in ibnetdiscover to let users diff an ibnetdiscover state to a previous
ibnetdiscover state. The goal of this option is to help system
administrators isolate/determine changes in the network quickly compared
to a previous state. Here's an example:
# > ./ibnetdiscover --diff=orig.cache
vendid=0x8f1
devid=0x5a30
sysimgguid=0x8f10400411f57
switchguid=0x8f10400411f56(8f10400411f56)
Switch 24 "S-0008f10400411f56" # "ISR9024D Voltaire" base port 0 lid 11 lmc 0
< [14] "H-0002c90200219ef0"[1](2c90200219ef1) # "wopr0" lid 64 4xDDR
< [19] "H-0002c9030000ff7c"[1](2c9030000ff7d) # "wopr9" lid 48 4xDDR
> [20] "H-0002c9030000ff7c"[1](2c9030000ff7d) # "wopr9" lid 4 4xDDR
< vendid=0x2c9
< devid=0x6282
< sysimgguid=0x2c90200219ef3
< caguid=0x2c90200219ef0
< Ca 2 "H-0002c90200219ef0" # "wopr0"
< [1](2c90200219ef1) "S-0008f10400411f56"[14] # lid 64 lmc 2 "ISR9024D Voltaire" lid 11 4xDDR
In this particular example, port 14 on the switch (which is connected to
node 'wopr0') was up before but is now down (and the associated CA is
noted too). In addition, 'wopr9' is connected to port 20 instead of
port 19 on the switch.
By default --diff checks switches, cas, routers, and port connections.
The --diffcheck option allows the user to specify which diff options
they want done, and also adds other diff checks for lids and/or node
descriptions. More diff checks could be added later as needed. For
example, the following only checks for differences of lids on switches.
# > ./ibnetdiscover --diff=orig.cache --diffcheck=sw,lid
vendid=0x8f1
devid=0x5a30
sysimgguid=0x8f10400411f57
switchguid=0x8f10400411f56(8f10400411f56)
< Switch 24 "S-0008f10400411f56" # "ISR9024D Voltaire" base port 0 lid 11 lmc 0
> Switch 24 "S-0008f10400411f56" # "ISR9024D Voltaire" base port 0 lid 3 lmc 0
< [13] "H-0002c90200219e64"[1](2c90200219e65) # "wopri" lid 4 4xDDR
> [13] "H-0002c90200219e64"[1](2c90200219e65) # "wopri" lid 1 4xDDR
Others on the list may wonder how this is different than just using the
normal 'diff' tool. The differences I can think of are:
1) This checks differences in the network, not text. This is
particularly important when lids, lmc, etc. are changed. Otherwise
there are many differences in a normal diff output that aren't
necessary.
2) This provides the appropriate "context" in the diff output, showing
the appropriate system ids to allow a system administrator to identify
ports on what switch have changed. Under normal diff output, you may
not get that appropriate context of information. The system
administrator can of course use options like --context in diff, but the
goal is to make the diff output clear and concise, not outputting
unnecessary junk.
3) As parallelization has been added into ibnetdisocver/libibnetdiscover
this becomes more critical as output in ibnetdiscover/libibnetdiscover
can be re-ordered. So a normal diff suddenly is non-functional.
There's probably other minor advantages. Even if minor output tweaks
happen to ibnetdiscover in the future, this can still work against old
cache files.
Al
--
Albert Chu
chu11-i2BcT+NCU+M@public.gmane.org
Computer Scientist
High Performance Systems Division
Lawrence Livermore National Laboratory
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* [infiniband-diags] [1/3] support --diff in ibnetdiscover
From: Al Chu @ 2010-04-07 17:05 UTC (permalink / raw)
To: sashak-smomgflXvOZWk0Htik3J/w@public.gmane.org
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
[-- Attachment #1: Type: text/plain, Size: 219 bytes --]
Hi Sasha,
This patch adds the default --diff support in ibnetdiscover.
Al
--
Albert Chu
chu11-i2BcT+NCU+M@public.gmane.org
Computer Scientist
High Performance Systems Division
Lawrence Livermore National Laboratory
[-- Attachment #2: 0001-support-diff-in-ibnetdiscover.patch --]
[-- Type: message/rfc822, Size: 16838 bytes --]
From: Albert Chu <chu11-i2BcT+NCU+M@public.gmane.org>
Subject: [PATCH] support --diff in ibnetdiscover
Date: Wed, 31 Mar 2010 11:10:34 -0700
Message-ID: <1270659341.26381.27.camel-RLKWKRZIcZkVVsCFsIUZTRy+HRzXvqW9@public.gmane.org>
Signed-off-by: Albert Chu <chu11-i2BcT+NCU+M@public.gmane.org>
---
infiniband-diags/man/ibnetdiscover.8 | 7 +
infiniband-diags/src/ibnetdiscover.c | 246 +++++++++++++++++++++++++++++----
2 files changed, 223 insertions(+), 30 deletions(-)
diff --git a/infiniband-diags/man/ibnetdiscover.8 b/infiniband-diags/man/ibnetdiscover.8
index 082a8e4..975b999 100644
--- a/infiniband-diags/man/ibnetdiscover.8
+++ b/infiniband-diags/man/ibnetdiscover.8
@@ -57,6 +57,13 @@ Load and use the cached ibnetdiscover data stored in the specified
filename. May be useful for outputting and learning about other
fabrics or a previous state of a fabric.
.TP
+\fB\-\-diff\fR <filename>
+Load cached ibnetdiscover data and do a diff comparison to the current
+network or another cache. A special diff output for ibnetdiscover
+output will be displayed showing differences between the old and current
+fabric. By default, the following are compared for differences: switches,
+channel adapters, routers, and port connections.
+.TP
\fB\-p\fR, \fB\-\-ports\fR
Obtain a ports report which is a
list of connected ports with relevant information (like LID, portnum,
diff --git a/infiniband-diags/src/ibnetdiscover.c b/infiniband-diags/src/ibnetdiscover.c
index 651bafd..4da09ce 100644
--- a/infiniband-diags/src/ibnetdiscover.c
+++ b/infiniband-diags/src/ibnetdiscover.c
@@ -57,6 +57,16 @@
#define LIST_SWITCH_NODE (1 << IB_NODE_SWITCH)
#define LIST_ROUTER_NODE (1 << IB_NODE_ROUTER)
+#define DIFF_FLAG_SWITCH 0x00000001
+#define DIFF_FLAG_CA 0x00000002
+#define DIFF_FLAG_ROUTER 0x00000004
+#define DIFF_FLAG_PORT_CONNECTION 0x00000008
+
+#define DIFF_FLAG_DEFAULT (DIFF_FLAG_SWITCH \
+ | DIFF_FLAG_CA \
+ | DIFF_FLAG_ROUTER \
+ | DIFF_FLAG_PORT_CONNECTION)
+
struct ibmad_port *srcport;
static FILE *f;
@@ -65,6 +75,7 @@ static char *node_name_map_file = NULL;
static nn_map_t *node_name_map = NULL;
static char *cache_file = NULL;
static char *load_cache_file = NULL;
+static char *diff_cache_file = NULL;
static int report_max_hops = 0;
@@ -183,16 +194,20 @@ void list_nodes(ibnd_fabric_t * fabric, int list)
ibnd_iter_nodes_type(fabric, list_node, IB_NODE_ROUTER, NULL);
}
-void out_ids(ibnd_node_t * node, int group, char *chname)
+void out_ids(ibnd_node_t * node, int group, char *chname, char *out_prefix)
{
uint64_t sysimgguid =
mad_get_field64(node->info, 0, IB_NODE_SYSTEM_GUID_F);
- fprintf(f, "\nvendid=0x%x\ndevid=0x%x\n",
- mad_get_field(node->info, 0, IB_NODE_VENDORID_F),
+ fprintf(f, "\n%svendid=0x%x\n",
+ out_prefix ? out_prefix : "",
+ mad_get_field(node->info, 0, IB_NODE_VENDORID_F));
+ fprintf(f, "%sdevid=0x%x\n",
+ out_prefix ? out_prefix : "",
mad_get_field(node->info, 0, IB_NODE_DEVID_F));
if (sysimgguid)
- fprintf(f, "sysimgguid=0x%" PRIx64, sysimgguid);
+ fprintf(f, "%ssysimgguid=0x%" PRIx64,
+ out_prefix ? out_prefix : "", sysimgguid);
if (group && node->chassis && node->chassis->chassisnum) {
fprintf(f, "\t\t# Chassis %d", node->chassis->chassisnum);
if (chname)
@@ -217,14 +232,15 @@ uint64_t out_chassis(ibnd_fabric_t * fabric, unsigned char chassisnum)
return guid;
}
-void out_switch(ibnd_node_t * node, int group, char *chname)
+void out_switch(ibnd_node_t * node, int group, char *chname, char *out_prefix)
{
char *str;
char str2[256];
char *nodename = NULL;
- out_ids(node, group, chname);
- fprintf(f, "switchguid=0x%" PRIx64, node->guid);
+ out_ids(node, group, chname, out_prefix);
+ fprintf(f, "%sswitchguid=0x%" PRIx64,
+ out_prefix ? out_prefix : "", node->guid);
fprintf(f, "(%" PRIx64 ")",
mad_get_field64(node->info, 0, IB_NODE_PORT_GUID_F));
if (group) {
@@ -239,7 +255,8 @@ void out_switch(ibnd_node_t * node, int group, char *chname)
nodename = remap_node_name(node_name_map, node->guid, node->nodedesc);
- fprintf(f, "\nSwitch\t%d %s\t\t# \"%s\" %s port 0 lid %d lmc %d\n",
+ fprintf(f, "\n%sSwitch\t%d %s\t\t# \"%s\" %s port 0 lid %d lmc %d\n",
+ out_prefix ? out_prefix : "",
node->numports, node_name(node), nodename,
node->smaenhsp0 ? "enhanced" : "base",
node->smalid, node->smalmc);
@@ -247,12 +264,12 @@ void out_switch(ibnd_node_t * node, int group, char *chname)
free(nodename);
}
-void out_ca(ibnd_node_t * node, int group, char *chname)
+void out_ca(ibnd_node_t * node, int group, char *chname, char *out_prefix)
{
char *node_type;
char *node_type2;
- out_ids(node, group, chname);
+ out_ids(node, group, chname, out_prefix);
switch (node->type) {
case IB_NODE_CA:
node_type = "ca";
@@ -268,8 +285,10 @@ void out_ca(ibnd_node_t * node, int group, char *chname)
break;
}
- fprintf(f, "%sguid=0x%" PRIx64 "\n", node_type, node->guid);
- fprintf(f, "%s\t%d %s\t\t# \"%s\"",
+ fprintf(f, "%s%sguid=0x%" PRIx64 "\n",
+ out_prefix ? out_prefix : "", node_type, node->guid);
+ fprintf(f, "%s%s\t%d %s\t\t# \"%s\"",
+ out_prefix ? out_prefix : "",
node_type2, node->numports, node_name(node),
clean_nodedesc(node->nodedesc));
if (group && ibnd_is_xsigo_hca(node->guid))
@@ -291,7 +310,7 @@ static char *out_ext_port(ibnd_port_t * port, int group)
return (NULL);
}
-void out_switch_port(ibnd_port_t * port, int group)
+void out_switch_port(ibnd_port_t * port, int group, char *out_prefix)
{
char *ext_port_str = NULL;
char *rem_nodename = NULL;
@@ -302,7 +321,7 @@ void out_switch_port(ibnd_port_t * port, int group)
DEBUG("port %p:%d remoteport %p\n", port, port->portnum,
port->remoteport);
- fprintf(f, "[%d]", port->portnum);
+ fprintf(f, "%s[%d]", out_prefix ? out_prefix : "", port->portnum);
ext_port_str = out_ext_port(port, group);
if (ext_port_str)
@@ -334,7 +353,7 @@ void out_switch_port(ibnd_port_t * port, int group)
free(rem_nodename);
}
-void out_ca_port(ibnd_port_t * port, int group)
+void out_ca_port(ibnd_port_t * port, int group, char *out_prefix)
{
char *str = NULL;
char *rem_nodename = NULL;
@@ -343,7 +362,7 @@ void out_ca_port(ibnd_port_t * port, int group)
uint32_t ispeed = mad_get_field(port->info, 0,
IB_PORT_LINK_SPEED_ACTIVE_F);
- fprintf(f, "[%d]", port->portnum);
+ fprintf(f, "%s[%d]", out_prefix ? out_prefix : "", port->portnum);
if (port->node->type != IB_NODE_SWITCH)
fprintf(f, "(%" PRIx64 ") ", port->guid);
fprintf(f, "\t%s[%d]",
@@ -386,11 +405,11 @@ static void switch_iter_func(ibnd_node_t * node, void *iter_user_data)
&& node->chassis->chassisnum)
return;
- out_switch(node, data->group, NULL);
+ out_switch(node, data->group, NULL, NULL);
for (p = 1; p <= node->numports; p++) {
port = node->ports[p];
if (port && port->remoteport)
- out_switch_port(port, data->group);
+ out_switch_port(port, data->group, NULL);
}
}
@@ -404,12 +423,12 @@ static void ca_iter_func(ibnd_node_t * node, void *iter_user_data)
/* Now, skip chassis based CAs */
if (data->group && node->chassis && node->chassis->chassisnum)
return;
- out_ca(node, data->group, NULL);
+ out_ca(node, data->group, NULL, NULL);
for (p = 1; p <= node->numports; p++) {
port = node->ports[p];
if (port && port->remoteport)
- out_ca_port(port, data->group);
+ out_ca_port(port, data->group, NULL);
}
}
@@ -423,11 +442,11 @@ static void router_iter_func(ibnd_node_t * node, void *iter_user_data)
/* Now, skip chassis based RTs */
if (data->group && node->chassis && node->chassis->chassisnum)
return;
- out_ca(node, data->group, NULL);
+ out_ca(node, data->group, NULL, NULL);
for (p = 1; p <= node->numports; p++) {
port = node->ports[p];
if (port && port->remoteport)
- out_ca_port(port, data->group);
+ out_ca_port(port, data->group, NULL);
}
}
@@ -482,7 +501,7 @@ int dump_topology(int group, ibnd_fabric_t * fabric)
for (n = 1; n <= SPINES_MAX_NUM; n++) {
if (ch->spinenode[n]) {
out_switch(ch->spinenode[n], group,
- chname);
+ chname, NULL);
for (p = 1;
p <= ch->spinenode[n]->numports;
p++) {
@@ -490,7 +509,8 @@ int dump_topology(int group, ibnd_fabric_t * fabric)
ch->spinenode[n]->ports[p];
if (port && port->remoteport)
out_switch_port(port,
- group);
+ group,
+ NULL);
}
}
}
@@ -498,7 +518,7 @@ int dump_topology(int group, ibnd_fabric_t * fabric)
for (n = 1; n <= LINES_MAX_NUM; n++) {
if (ch->linenode[n]) {
out_switch(ch->linenode[n], group,
- chname);
+ chname, NULL);
for (p = 1;
p <= ch->linenode[n]->numports;
p++) {
@@ -506,7 +526,8 @@ int dump_topology(int group, ibnd_fabric_t * fabric)
ch->linenode[n]->ports[p];
if (port && port->remoteport)
out_switch_port(port,
- group);
+ group,
+ NULL);
}
}
}
@@ -515,12 +536,13 @@ int dump_topology(int group, ibnd_fabric_t * fabric)
for (node = ch->nodes; node;
node = node->next_chassis_node) {
if (node->type == IB_NODE_SWITCH) {
- out_switch(node, group, chname);
+ out_switch(node, group, chname, NULL);
for (p = 1; p <= node->numports; p++) {
port = node->ports[p];
if (port && port->remoteport)
out_switch_port(port,
- group);
+ group,
+ NULL);
}
}
@@ -530,12 +552,13 @@ int dump_topology(int group, ibnd_fabric_t * fabric)
for (node = ch->nodes; node;
node = node->next_chassis_node) {
if (node->type == IB_NODE_CA) {
- out_ca(node, group, chname);
+ out_ca(node, group, chname, NULL);
for (p = 1; p <= node->numports; p++) {
port = node->ports[p];
if (port && port->remoteport)
out_ca_port(port,
- group);
+ group,
+ NULL);
}
}
}
@@ -610,6 +633,155 @@ void dump_ports_report(ibnd_node_t * node, void *user_data)
}
}
+struct iter_diff_data {
+ uint32_t diff_flags;
+ ibnd_fabric_t * fabric1;
+ ibnd_fabric_t * fabric2;
+ char *fabric1_prefix;
+ char *fabric2_prefix;
+ void (*out_header)(ibnd_node_t *, int, char *, char *);
+ void (*out_port)(ibnd_port_t *, int, char *);
+};
+
+static void diff_iter_out_header(ibnd_node_t * node,
+ struct iter_diff_data *data,
+ int *out_header_flag)
+{
+ if (!(*out_header_flag)) {
+ (*data->out_header)(node, 0, NULL, NULL);
+ (*out_header_flag)++;
+ }
+}
+
+static void diff_iter_func(ibnd_node_t * fabric1_node, void *iter_user_data)
+{
+ struct iter_diff_data *data = (struct iter_diff_data *)iter_user_data;
+ ibnd_node_t *fabric2_node;
+ ibnd_port_t *fabric1_port;
+ int p;
+
+ DEBUG("DEBUG: fabric1_node %p\n", fabric1_node);
+
+ fabric2_node = ibnd_find_node_guid (data->fabric2, fabric1_node->guid);
+
+ if (!fabric2_node) {
+ (*data->out_header)(fabric1_node, 0, NULL, data->fabric1_prefix);
+ for (p = 1; p <= fabric1_node->numports; p++) {
+ fabric1_port = fabric1_node->ports[p];
+ if (fabric1_port && fabric1_port->remoteport)
+ (*data->out_port)(fabric1_port, 0, data->fabric1_prefix);
+ }
+ }
+ else if (data->diff_flags & DIFF_FLAG_PORT_CONNECTION) {
+ ibnd_port_t *fabric2_port;
+ int out_header_flag = 0;
+
+ if (fabric1_node->numports != fabric2_node->numports) {
+ diff_iter_out_header(fabric1_node, data, &out_header_flag);
+ fprintf(f, "%snumports = %d\n",
+ data->fabric1_prefix, fabric1_node->numports);
+ fprintf(f, "%snumports = %d\n",
+ data->fabric2_prefix, fabric2_node->numports);
+ return;
+ }
+
+ for (p = 1; p <= fabric1_node->numports; p++) {
+ fabric1_port = fabric1_node->ports[p];
+ fabric2_port = fabric2_node->ports[p];
+ if ((fabric1_port && !fabric2_port)
+ || ((fabric1_port && fabric2_port)
+ && (fabric1_port->remoteport && !fabric2_port->remoteport))) {
+ diff_iter_out_header(fabric1_node, data, &out_header_flag);
+ (*data->out_port)(fabric1_port, 0, data->fabric1_prefix);
+ }
+ else if ((!fabric1_port && fabric2_port)
+ || ((fabric1_port && fabric2_port)
+ && (!fabric1_port->remoteport && fabric2_port->remoteport))) {
+ diff_iter_out_header(fabric1_node, data, &out_header_flag);
+ (*data->out_port)(fabric2_port, 0, data->fabric2_prefix);
+ }
+ else if ((fabric1_port && fabric2_port)
+ && ((fabric1_port->guid != fabric2_port->guid)
+ || ((fabric1_port->remoteport && fabric2_port->remoteport)
+ && (fabric1_port->remoteport->guid != fabric2_port->remoteport->guid)))) {
+ diff_iter_out_header(fabric1_node, data, &out_header_flag);
+ (*data->out_port)(fabric1_port, 0, data->fabric1_prefix);
+ (*data->out_port)(fabric2_port, 0, data->fabric2_prefix);
+ }
+ }
+ }
+}
+
+static int diff_common(ibnd_fabric_t * orig_fabric,
+ ibnd_fabric_t * new_fabric,
+ int node_type,
+ uint32_t diff_flags,
+ void (*out_header)(ibnd_node_t *, int, char *, char *),
+ void (*out_port)(ibnd_port_t *, int, char *))
+{
+ struct iter_diff_data iter_diff_data;
+
+ iter_diff_data.diff_flags = diff_flags;
+ iter_diff_data.fabric1 = orig_fabric;
+ iter_diff_data.fabric2 = new_fabric;
+ iter_diff_data.fabric1_prefix = "< ";
+ iter_diff_data.fabric2_prefix = "> ";
+ iter_diff_data.out_header = out_header;
+ iter_diff_data.out_port = out_port;
+ ibnd_iter_nodes_type(orig_fabric, diff_iter_func,
+ node_type, &iter_diff_data);
+
+ /* Do opposite diff to find existence of node types
+ * in new_fabric but not in orig_fabric.
+ *
+ * In this diff, we don't need to check port connections,
+ * since it has already been done before.
+ */
+ iter_diff_data.diff_flags = diff_flags & ~DIFF_FLAG_PORT_CONNECTION;
+ iter_diff_data.fabric1 = new_fabric;
+ iter_diff_data.fabric2 = orig_fabric;
+ iter_diff_data.fabric1_prefix = "> ";
+ iter_diff_data.fabric2_prefix = "< ";
+ iter_diff_data.out_header = out_header;
+ iter_diff_data.out_port = out_port;
+ ibnd_iter_nodes_type(new_fabric, diff_iter_func,
+ node_type, &iter_diff_data);
+
+ return 0;
+}
+
+int diff(ibnd_fabric_t * orig_fabric, ibnd_fabric_t * new_fabric)
+{
+ uint32_t diff_flags = DIFF_FLAG_DEFAULT;
+
+ if (diff_flags & DIFF_FLAG_SWITCH)
+ diff_common(orig_fabric,
+ new_fabric,
+ IB_NODE_SWITCH,
+ diff_flags,
+ out_switch,
+ out_switch_port);
+
+ if (diff_flags & DIFF_FLAG_CA)
+ diff_common(orig_fabric,
+ new_fabric,
+ IB_NODE_CA,
+ diff_flags,
+ out_ca,
+ out_ca_port);
+
+ if (diff_flags & DIFF_FLAG_ROUTER)
+ diff_common(orig_fabric,
+ new_fabric,
+ IB_NODE_ROUTER,
+ diff_flags,
+ out_ca,
+ out_ca_port);
+
+
+ return 0;
+}
+
static int list, group, ports_report;
static int process_opt(void *context, int ch, char *optarg)
@@ -624,6 +796,9 @@ static int process_opt(void *context, int ch, char *optarg)
case 3:
load_cache_file = strdup(optarg);
break;
+ case 4:
+ diff_cache_file = strdup(optarg);
+ break;
case 's':
ibnd_show_progress(1);
break;
@@ -658,6 +833,7 @@ static int process_opt(void *context, int ch, char *optarg)
int main(int argc, char **argv)
{
ibnd_fabric_t *fabric = NULL;
+ ibnd_fabric_t *diff_fabric = NULL;
struct ibmad_port *ibmad_port;
int mgmt_classes[2] = { IB_SMI_CLASS, IB_SMI_DIRECT_CLASS };
@@ -674,6 +850,8 @@ int main(int argc, char **argv)
"filename to cache ibnetdiscover data to"},
{"load-cache", 3, 1, "<file>",
"filename of ibnetdiscover cache to load"},
+ {"diff", 4, 1, "<file>",
+ "filename of ibnetdiscover cache to diff"},
{"ports", 'p', 0, NULL, "obtain a ports report"},
{"max_hops", 'm', 0, NULL,
"report max hops discovered by the library"},
@@ -704,6 +882,11 @@ int main(int argc, char **argv)
node_name_map = open_node_name_map(node_name_map_file);
+ if (diff_cache_file) {
+ if ((diff_fabric = ibnd_load_fabric(diff_cache_file, 0)) == NULL)
+ IBERROR("loading cached fabric for diff failed\n");
+ }
+
if (load_cache_file) {
if ((fabric = ibnd_load_fabric(load_cache_file, 0)) == NULL)
IBERROR("loading cached fabric failed\n");
@@ -717,6 +900,8 @@ int main(int argc, char **argv)
ibnd_iter_nodes(fabric, dump_ports_report, NULL);
else if (list)
list_nodes(fabric, list);
+ else if (diff_fabric)
+ diff(diff_fabric, fabric);
else
dump_topology(group, fabric);
@@ -725,6 +910,7 @@ int main(int argc, char **argv)
IBERROR("caching ibnetdiscover data failed\n");
ibnd_destroy_fabric(fabric);
+ ibnd_destroy_fabric(diff_fabric);
close_node_name_map(node_name_map);
mad_rpc_close_port(ibmad_port);
exit(0);
--
1.5.4.5
^ permalink raw reply related
* [infiniband-diags] [2/3] support --diffcheck in ibnetdiscover
From: Al Chu @ 2010-04-07 17:06 UTC (permalink / raw)
To: sashak-smomgflXvOZWk0Htik3J/w@public.gmane.org
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
[-- Attachment #1: Type: text/plain, Size: 295 bytes --]
Hi Sasha,
This patch adds basic --diffcheck support in ibnetdiscover, allowing
configuration of the diff checks done in the default --diff option.
Al
--
Albert Chu
chu11-i2BcT+NCU+M@public.gmane.org
Computer Scientist
High Performance Systems Division
Lawrence Livermore National Laboratory
[-- Attachment #2: 0002-support-diffcheck-in-ibnetdiscover.patch --]
[-- Type: message/rfc822, Size: 5251 bytes --]
From: Albert Chu <chu11-i2BcT+NCU+M@public.gmane.org>
Subject: [PATCH] support --diffcheck in ibnetdiscover
Date: Wed, 31 Mar 2010 11:13:37 -0700
Message-ID: <1270659341.26381.29.camel-RLKWKRZIcZkVVsCFsIUZTRy+HRzXvqW9@public.gmane.org>
Signed-off-by: Albert Chu <chu11-i2BcT+NCU+M@public.gmane.org>
---
infiniband-diags/man/ibnetdiscover.8 | 8 +++++
infiniband-diags/src/ibnetdiscover.c | 50 +++++++++++++++++++++++++---------
2 files changed, 45 insertions(+), 13 deletions(-)
diff --git a/infiniband-diags/man/ibnetdiscover.8 b/infiniband-diags/man/ibnetdiscover.8
index 975b999..e122736 100644
--- a/infiniband-diags/man/ibnetdiscover.8
+++ b/infiniband-diags/man/ibnetdiscover.8
@@ -64,6 +64,14 @@ output will be displayed showing differences between the old and current
fabric. By default, the following are compared for differences: switches,
channel adapters, routers, and port connections.
.TP
+\fB\-\-diffcheck\fR <key(s)>
+Specify what diff checks should be done in the \fB\-\-diff\fR option above.
+Comma separate multiple diff check key(s). The available diff checks
+are: \fIsw\fR = switches, \fIca\fR = channel adapters, \fIrouter\fR = routers,
+\fIport\fR = port connections descriptions. Note that \fIport\fR is
+checked only for the node types that are specified (e.g. \fIsw\fR,
+\fIca\fR, \fIrouter\fR).
+.TP
\fB\-p\fR, \fB\-\-ports\fR
Obtain a ports report which is a
list of connected ports with relevant information (like LID, portnum,
diff --git a/infiniband-diags/src/ibnetdiscover.c b/infiniband-diags/src/ibnetdiscover.c
index 4da09ce..4435ade 100644
--- a/infiniband-diags/src/ibnetdiscover.c
+++ b/infiniband-diags/src/ibnetdiscover.c
@@ -57,10 +57,10 @@
#define LIST_SWITCH_NODE (1 << IB_NODE_SWITCH)
#define LIST_ROUTER_NODE (1 << IB_NODE_ROUTER)
-#define DIFF_FLAG_SWITCH 0x00000001
-#define DIFF_FLAG_CA 0x00000002
-#define DIFF_FLAG_ROUTER 0x00000004
-#define DIFF_FLAG_PORT_CONNECTION 0x00000008
+#define DIFF_FLAG_SWITCH 0x00000001
+#define DIFF_FLAG_CA 0x00000002
+#define DIFF_FLAG_ROUTER 0x00000004
+#define DIFF_FLAG_PORT_CONNECTION 0x00000008
#define DIFF_FLAG_DEFAULT (DIFF_FLAG_SWITCH \
| DIFF_FLAG_CA \
@@ -76,6 +76,7 @@ static nn_map_t *node_name_map = NULL;
static char *cache_file = NULL;
static char *load_cache_file = NULL;
static char *diff_cache_file = NULL;
+static uint32_t diffcheck_flags = DIFF_FLAG_DEFAULT;
static int report_max_hops = 0;
@@ -735,7 +736,9 @@ static int diff_common(ibnd_fabric_t * orig_fabric,
* in new_fabric but not in orig_fabric.
*
* In this diff, we don't need to check port connections,
- * since it has already been done before.
+ * lids, or node descriptions since it has already been
+ * done (i.e. checks are only done when guid exists on both
+ * orig and new).
*/
iter_diff_data.diff_flags = diff_flags & ~DIFF_FLAG_PORT_CONNECTION;
iter_diff_data.fabric1 = new_fabric;
@@ -752,29 +755,27 @@ static int diff_common(ibnd_fabric_t * orig_fabric,
int diff(ibnd_fabric_t * orig_fabric, ibnd_fabric_t * new_fabric)
{
- uint32_t diff_flags = DIFF_FLAG_DEFAULT;
-
- if (diff_flags & DIFF_FLAG_SWITCH)
+ if (diffcheck_flags & DIFF_FLAG_SWITCH)
diff_common(orig_fabric,
new_fabric,
IB_NODE_SWITCH,
- diff_flags,
+ diffcheck_flags,
out_switch,
out_switch_port);
- if (diff_flags & DIFF_FLAG_CA)
+ if (diffcheck_flags & DIFF_FLAG_CA)
diff_common(orig_fabric,
new_fabric,
IB_NODE_CA,
- diff_flags,
+ diffcheck_flags,
out_ca,
out_ca_port);
- if (diff_flags & DIFF_FLAG_ROUTER)
+ if (diffcheck_flags & DIFF_FLAG_ROUTER)
diff_common(orig_fabric,
new_fabric,
IB_NODE_ROUTER,
- diff_flags,
+ diffcheck_flags,
out_ca,
out_ca_port);
@@ -786,6 +787,8 @@ static int list, group, ports_report;
static int process_opt(void *context, int ch, char *optarg)
{
+ char *p;
+
switch (ch) {
case 1:
node_name_map_file = strdup(optarg);
@@ -799,6 +802,25 @@ static int process_opt(void *context, int ch, char *optarg)
case 4:
diff_cache_file = strdup(optarg);
break;
+ case 5:
+ diffcheck_flags = 0;
+ p = strtok(optarg, ",");
+ while (p) {
+ if (!strcasecmp(p, "sw"))
+ diffcheck_flags |= DIFF_FLAG_SWITCH;
+ else if (!strcasecmp(p, "ca"))
+ diffcheck_flags |= DIFF_FLAG_CA;
+ else if (!strcasecmp(p, "router"))
+ diffcheck_flags |= DIFF_FLAG_ROUTER;
+ else if (!strcasecmp(p, "port"))
+ diffcheck_flags |= DIFF_FLAG_PORT_CONNECTION;
+ else {
+ fprintf(stderr, "invalid diff check key: %s\n", p);
+ return -1;
+ }
+ p = strtok(NULL, ",");
+ }
+ break;
case 's':
ibnd_show_progress(1);
break;
@@ -852,6 +874,8 @@ int main(int argc, char **argv)
"filename of ibnetdiscover cache to load"},
{"diff", 4, 1, "<file>",
"filename of ibnetdiscover cache to diff"},
+ {"diffcheck", 5, 1, "<key(s)>",
+ "specify checks to execute for --diff"},
{"ports", 'p', 0, NULL, "obtain a ports report"},
{"max_hops", 'm', 0, NULL,
"report max hops discovered by the library"},
--
1.5.4.5
^ permalink raw reply related
* [infiniband-diags] [3/3] support lid and nodedesc diffchecks in ibnetdiscover
From: Al Chu @ 2010-04-07 17:06 UTC (permalink / raw)
To: sashak-smomgflXvOZWk0Htik3J/w@public.gmane.org
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
[-- Attachment #1: Type: text/plain, Size: 246 bytes --]
Hi Sasha,
This patch adds lid and node description diff options for --diffcheck in
ibnetdiscover.
Al
--
Albert Chu
chu11-i2BcT+NCU+M@public.gmane.org
Computer Scientist
High Performance Systems Division
Lawrence Livermore National Laboratory
[-- Attachment #2: 0003-support-lid-and-nodedesc-diffchecks-in-ibnetdiscover.patch --]
[-- Type: message/rfc822, Size: 15545 bytes --]
From: Albert Chu <chu11-i2BcT+NCU+M@public.gmane.org>
Subject: [PATCH] support lid and nodedesc diffchecks in ibnetdiscover
Date: Wed, 31 Mar 2010 11:18:43 -0700
Message-ID: <1270659341.26381.31.camel-RLKWKRZIcZkVVsCFsIUZTRy+HRzXvqW9@public.gmane.org>
Signed-off-by: Albert Chu <chu11-i2BcT+NCU+M@public.gmane.org>
---
infiniband-diags/man/ibnetdiscover.8 | 3 +-
infiniband-diags/src/ibnetdiscover.c | 211 ++++++++++++++++++++++++----------
2 files changed, 154 insertions(+), 60 deletions(-)
diff --git a/infiniband-diags/man/ibnetdiscover.8 b/infiniband-diags/man/ibnetdiscover.8
index e122736..76cfbc8 100644
--- a/infiniband-diags/man/ibnetdiscover.8
+++ b/infiniband-diags/man/ibnetdiscover.8
@@ -68,7 +68,8 @@ channel adapters, routers, and port connections.
Specify what diff checks should be done in the \fB\-\-diff\fR option above.
Comma separate multiple diff check key(s). The available diff checks
are: \fIsw\fR = switches, \fIca\fR = channel adapters, \fIrouter\fR = routers,
-\fIport\fR = port connections descriptions. Note that \fIport\fR is
+\fIport\fR = port connections, \fIlid\fR = lids, \fInodedesc\fR = node
+descriptions. Note that \fIport\fR, \fIlid\fR, and \fInodedesc\fR are
checked only for the node types that are specified (e.g. \fIsw\fR,
\fIca\fR, \fIrouter\fR).
.TP
diff --git a/infiniband-diags/src/ibnetdiscover.c b/infiniband-diags/src/ibnetdiscover.c
index 4435ade..770c589 100644
--- a/infiniband-diags/src/ibnetdiscover.c
+++ b/infiniband-diags/src/ibnetdiscover.c
@@ -61,6 +61,8 @@
#define DIFF_FLAG_CA 0x00000002
#define DIFF_FLAG_ROUTER 0x00000004
#define DIFF_FLAG_PORT_CONNECTION 0x00000008
+#define DIFF_FLAG_LID 0x00000010
+#define DIFF_FLAG_NODE_DESCRIPTION 0x00000020
#define DIFF_FLAG_DEFAULT (DIFF_FLAG_SWITCH \
| DIFF_FLAG_CA \
@@ -233,15 +235,29 @@ uint64_t out_chassis(ibnd_fabric_t * fabric, unsigned char chassisnum)
return guid;
}
-void out_switch(ibnd_node_t * node, int group, char *chname, char *out_prefix)
+void out_switch_detail(ibnd_node_t * node, char *sw_prefix)
+{
+ char *nodename = NULL;
+
+ nodename = remap_node_name(node_name_map, node->guid, node->nodedesc);
+
+ fprintf(f, "%sSwitch\t%d %s\t\t# \"%s\" %s port 0 lid %d lmc %d",
+ sw_prefix ? sw_prefix : "",
+ node->numports, node_name(node), nodename,
+ node->smaenhsp0 ? "enhanced" : "base",
+ node->smalid, node->smalmc);
+
+ free(nodename);
+}
+
+void out_switch(ibnd_node_t * node, int group, char *chname, char *id_prefix, char *sw_prefix)
{
char *str;
char str2[256];
- char *nodename = NULL;
- out_ids(node, group, chname, out_prefix);
+ out_ids(node, group, chname, id_prefix);
fprintf(f, "%sswitchguid=0x%" PRIx64,
- out_prefix ? out_prefix : "", node->guid);
+ id_prefix ? id_prefix : "", node->guid);
fprintf(f, "(%" PRIx64 ")",
mad_get_field64(node->info, 0, IB_NODE_PORT_GUID_F));
if (group) {
@@ -253,45 +269,54 @@ void out_switch(ibnd_node_t * node, int group, char *chname, char *out_prefix)
if (str)
fprintf(f, "%s", str);
}
+ fprintf(f, "\n");
- nodename = remap_node_name(node_name_map, node->guid, node->nodedesc);
+ out_switch_detail(node, sw_prefix);
+ fprintf(f, "\n");
+}
- fprintf(f, "\n%sSwitch\t%d %s\t\t# \"%s\" %s port 0 lid %d lmc %d\n",
- out_prefix ? out_prefix : "",
- node->numports, node_name(node), nodename,
- node->smaenhsp0 ? "enhanced" : "base",
- node->smalid, node->smalmc);
+void out_ca_detail(ibnd_node_t * node, char *ca_prefix)
+{
+ char *node_type;
- free(nodename);
+ switch (node->type) {
+ case IB_NODE_CA:
+ node_type = "Ca";
+ break;
+ case IB_NODE_ROUTER:
+ node_type = "Rt";
+ break;
+ default:
+ node_type = "???";
+ break;
+ }
+
+ fprintf(f, "%s%s\t%d %s\t\t# \"%s\"",
+ ca_prefix ? ca_prefix : "",
+ node_type, node->numports, node_name(node),
+ clean_nodedesc(node->nodedesc));
}
-void out_ca(ibnd_node_t * node, int group, char *chname, char *out_prefix)
+void out_ca(ibnd_node_t * node, int group, char *chname, char *id_prefix, char *ca_prefix)
{
char *node_type;
- char *node_type2;
- out_ids(node, group, chname, out_prefix);
+ out_ids(node, group, chname, id_prefix);
switch (node->type) {
case IB_NODE_CA:
node_type = "ca";
- node_type2 = "Ca";
break;
case IB_NODE_ROUTER:
node_type = "rt";
- node_type2 = "Rt";
break;
default:
node_type = "???";
- node_type2 = "???";
break;
}
fprintf(f, "%s%sguid=0x%" PRIx64 "\n",
- out_prefix ? out_prefix : "", node_type, node->guid);
- fprintf(f, "%s%s\t%d %s\t\t# \"%s\"",
- out_prefix ? out_prefix : "",
- node_type2, node->numports, node_name(node),
- clean_nodedesc(node->nodedesc));
+ id_prefix ? id_prefix : "", node_type, node->guid);
+ out_ca_detail(node, ca_prefix);
if (group && ibnd_is_xsigo_hca(node->guid))
fprintf(f, " (scp)");
fprintf(f, "\n");
@@ -406,7 +431,7 @@ static void switch_iter_func(ibnd_node_t * node, void *iter_user_data)
&& node->chassis->chassisnum)
return;
- out_switch(node, data->group, NULL, NULL);
+ out_switch(node, data->group, NULL, NULL, NULL);
for (p = 1; p <= node->numports; p++) {
port = node->ports[p];
if (port && port->remoteport)
@@ -424,7 +449,7 @@ static void ca_iter_func(ibnd_node_t * node, void *iter_user_data)
/* Now, skip chassis based CAs */
if (data->group && node->chassis && node->chassis->chassisnum)
return;
- out_ca(node, data->group, NULL, NULL);
+ out_ca(node, data->group, NULL, NULL, NULL);
for (p = 1; p <= node->numports; p++) {
port = node->ports[p];
@@ -443,7 +468,7 @@ static void router_iter_func(ibnd_node_t * node, void *iter_user_data)
/* Now, skip chassis based RTs */
if (data->group && node->chassis && node->chassis->chassisnum)
return;
- out_ca(node, data->group, NULL, NULL);
+ out_ca(node, data->group, NULL, NULL, NULL);
for (p = 1; p <= node->numports; p++) {
port = node->ports[p];
if (port && port->remoteport)
@@ -502,7 +527,7 @@ int dump_topology(int group, ibnd_fabric_t * fabric)
for (n = 1; n <= SPINES_MAX_NUM; n++) {
if (ch->spinenode[n]) {
out_switch(ch->spinenode[n], group,
- chname, NULL);
+ chname, NULL, NULL);
for (p = 1;
p <= ch->spinenode[n]->numports;
p++) {
@@ -519,7 +544,7 @@ int dump_topology(int group, ibnd_fabric_t * fabric)
for (n = 1; n <= LINES_MAX_NUM; n++) {
if (ch->linenode[n]) {
out_switch(ch->linenode[n], group,
- chname, NULL);
+ chname, NULL, NULL);
for (p = 1;
p <= ch->linenode[n]->numports;
p++) {
@@ -537,7 +562,8 @@ int dump_topology(int group, ibnd_fabric_t * fabric)
for (node = ch->nodes; node;
node = node->next_chassis_node) {
if (node->type == IB_NODE_SWITCH) {
- out_switch(node, group, chname, NULL);
+ out_switch(node, group, chname, NULL,
+ NULL);
for (p = 1; p <= node->numports; p++) {
port = node->ports[p];
if (port && port->remoteport)
@@ -553,7 +579,8 @@ int dump_topology(int group, ibnd_fabric_t * fabric)
for (node = ch->nodes; node;
node = node->next_chassis_node) {
if (node->type == IB_NODE_CA) {
- out_ca(node, group, chname, NULL);
+ out_ca(node, group, chname, NULL,
+ NULL);
for (p = 1; p <= node->numports; p++) {
port = node->ports[p];
if (port && port->remoteport)
@@ -640,7 +667,8 @@ struct iter_diff_data {
ibnd_fabric_t * fabric2;
char *fabric1_prefix;
char *fabric2_prefix;
- void (*out_header)(ibnd_node_t *, int, char *, char *);
+ void (*out_header)(ibnd_node_t *, int, char *, char *, char *);
+ void (*out_header_detail)(ibnd_node_t *, char *);
void (*out_port)(ibnd_port_t *, int, char *);
};
@@ -649,11 +677,63 @@ static void diff_iter_out_header(ibnd_node_t * node,
int *out_header_flag)
{
if (!(*out_header_flag)) {
- (*data->out_header)(node, 0, NULL, NULL);
+ (*data->out_header)(node, 0, NULL, NULL, NULL);
(*out_header_flag)++;
}
}
+static void diff_ports(ibnd_node_t * fabric1_node,
+ ibnd_node_t * fabric2_node,
+ int *out_header_flag,
+ struct iter_diff_data *data)
+{
+ ibnd_port_t *fabric1_port;
+ ibnd_port_t *fabric2_port;
+ int p;
+
+ for (p = 1; p <= fabric1_node->numports; p++) {
+ int fabric1_out = 0, fabric2_out = 0;
+
+ fabric1_port = fabric1_node->ports[p];
+ fabric2_port = fabric2_node->ports[p];
+
+ if (data->diff_flags & DIFF_FLAG_PORT_CONNECTION) {
+ if ((fabric1_port && !fabric2_port)
+ || ((fabric1_port && fabric2_port)
+ && (fabric1_port->remoteport && !fabric2_port->remoteport)))
+ fabric1_out++;
+ else if ((!fabric1_port && fabric2_port)
+ || ((fabric1_port && fabric2_port)
+ && (!fabric1_port->remoteport && fabric2_port->remoteport)))
+ fabric2_out++;
+ else if ((fabric1_port && fabric2_port)
+ && ((fabric1_port->guid != fabric2_port->guid)
+ || ((fabric1_port->remoteport && fabric2_port->remoteport)
+ && (fabric1_port->remoteport->guid != fabric2_port->remoteport->guid)))) {
+ fabric1_out++;
+ fabric2_out++;
+ }
+ }
+
+ if (data->diff_flags & DIFF_FLAG_LID) {
+ if ((fabric1_port && fabric2_port)
+ && (fabric1_port->base_lid != fabric2_port->base_lid)) {
+ fabric1_out++;
+ fabric2_out++;
+ }
+ }
+
+ if (fabric1_out) {
+ diff_iter_out_header(fabric1_node, data, out_header_flag);
+ (*data->out_port)(fabric1_port, 0, data->fabric1_prefix);
+ }
+ if (fabric2_out) {
+ diff_iter_out_header(fabric1_node, data, out_header_flag);
+ (*data->out_port)(fabric2_port, 0, data->fabric2_prefix);
+ }
+ }
+}
+
static void diff_iter_func(ibnd_node_t * fabric1_node, void *iter_user_data)
{
struct iter_diff_data *data = (struct iter_diff_data *)iter_user_data;
@@ -666,16 +746,35 @@ static void diff_iter_func(ibnd_node_t * fabric1_node, void *iter_user_data)
fabric2_node = ibnd_find_node_guid (data->fabric2, fabric1_node->guid);
if (!fabric2_node) {
- (*data->out_header)(fabric1_node, 0, NULL, data->fabric1_prefix);
+ (*data->out_header)(fabric1_node, 0, NULL, data->fabric1_prefix,
+ data->fabric1_prefix);
for (p = 1; p <= fabric1_node->numports; p++) {
fabric1_port = fabric1_node->ports[p];
if (fabric1_port && fabric1_port->remoteport)
(*data->out_port)(fabric1_port, 0, data->fabric1_prefix);
}
}
- else if (data->diff_flags & DIFF_FLAG_PORT_CONNECTION) {
- ibnd_port_t *fabric2_port;
+ else if (data->diff_flags & DIFF_FLAG_PORT_CONNECTION
+ || data->diff_flags & DIFF_FLAG_LID
+ || data->diff_flags & DIFF_FLAG_NODE_DESCRIPTION) {
int out_header_flag = 0;
+ int out_header_detail_diff = 0;
+
+ if (data->diff_flags & DIFF_FLAG_LID
+ && fabric1_node->smalid != fabric2_node->smalid)
+ out_header_detail_diff++;
+
+ if (data->diff_flags & DIFF_FLAG_NODE_DESCRIPTION
+ && memcmp(fabric1_node->nodedesc, fabric2_node->nodedesc, IB_SMP_DATA_SIZE))
+ out_header_detail_diff++;
+
+ if (out_header_detail_diff) {
+ (*data->out_header)(fabric1_node, 0, NULL, NULL,
+ data->fabric1_prefix);
+ (*data->out_header_detail)(fabric2_node, data->fabric2_prefix);
+ fprintf(f, "\n");
+ out_header_flag++;
+ }
if (fabric1_node->numports != fabric2_node->numports) {
diff_iter_out_header(fabric1_node, data, &out_header_flag);
@@ -686,30 +785,12 @@ static void diff_iter_func(ibnd_node_t * fabric1_node, void *iter_user_data)
return;
}
- for (p = 1; p <= fabric1_node->numports; p++) {
- fabric1_port = fabric1_node->ports[p];
- fabric2_port = fabric2_node->ports[p];
- if ((fabric1_port && !fabric2_port)
- || ((fabric1_port && fabric2_port)
- && (fabric1_port->remoteport && !fabric2_port->remoteport))) {
- diff_iter_out_header(fabric1_node, data, &out_header_flag);
- (*data->out_port)(fabric1_port, 0, data->fabric1_prefix);
- }
- else if ((!fabric1_port && fabric2_port)
- || ((fabric1_port && fabric2_port)
- && (!fabric1_port->remoteport && fabric2_port->remoteport))) {
- diff_iter_out_header(fabric1_node, data, &out_header_flag);
- (*data->out_port)(fabric2_port, 0, data->fabric2_prefix);
- }
- else if ((fabric1_port && fabric2_port)
- && ((fabric1_port->guid != fabric2_port->guid)
- || ((fabric1_port->remoteport && fabric2_port->remoteport)
- && (fabric1_port->remoteport->guid != fabric2_port->remoteport->guid)))) {
- diff_iter_out_header(fabric1_node, data, &out_header_flag);
- (*data->out_port)(fabric1_port, 0, data->fabric1_prefix);
- (*data->out_port)(fabric2_port, 0, data->fabric2_prefix);
- }
- }
+ if (data->diff_flags & DIFF_FLAG_PORT_CONNECTION
+ || data->diff_flags & DIFF_FLAG_LID)
+ diff_ports(fabric1_node,
+ fabric2_node,
+ &out_header_flag,
+ data);
}
}
@@ -717,7 +798,8 @@ static int diff_common(ibnd_fabric_t * orig_fabric,
ibnd_fabric_t * new_fabric,
int node_type,
uint32_t diff_flags,
- void (*out_header)(ibnd_node_t *, int, char *, char *),
+ void (*out_header)(ibnd_node_t *, int, char *, char *, char*),
+ void (*out_header_detail)(ibnd_node_t *, char *),
void (*out_port)(ibnd_port_t *, int, char *))
{
struct iter_diff_data iter_diff_data;
@@ -728,6 +810,7 @@ static int diff_common(ibnd_fabric_t * orig_fabric,
iter_diff_data.fabric1_prefix = "< ";
iter_diff_data.fabric2_prefix = "> ";
iter_diff_data.out_header = out_header;
+ iter_diff_data.out_header_detail = out_header_detail;
iter_diff_data.out_port = out_port;
ibnd_iter_nodes_type(orig_fabric, diff_iter_func,
node_type, &iter_diff_data);
@@ -741,11 +824,14 @@ static int diff_common(ibnd_fabric_t * orig_fabric,
* orig and new).
*/
iter_diff_data.diff_flags = diff_flags & ~DIFF_FLAG_PORT_CONNECTION;
+ iter_diff_data.diff_flags &= ~DIFF_FLAG_LID;
+ iter_diff_data.diff_flags &= ~DIFF_FLAG_NODE_DESCRIPTION;
iter_diff_data.fabric1 = new_fabric;
iter_diff_data.fabric2 = orig_fabric;
iter_diff_data.fabric1_prefix = "> ";
iter_diff_data.fabric2_prefix = "< ";
iter_diff_data.out_header = out_header;
+ iter_diff_data.out_header_detail = out_header_detail;
iter_diff_data.out_port = out_port;
ibnd_iter_nodes_type(new_fabric, diff_iter_func,
node_type, &iter_diff_data);
@@ -761,6 +847,7 @@ int diff(ibnd_fabric_t * orig_fabric, ibnd_fabric_t * new_fabric)
IB_NODE_SWITCH,
diffcheck_flags,
out_switch,
+ out_switch_detail,
out_switch_port);
if (diffcheck_flags & DIFF_FLAG_CA)
@@ -769,6 +856,7 @@ int diff(ibnd_fabric_t * orig_fabric, ibnd_fabric_t * new_fabric)
IB_NODE_CA,
diffcheck_flags,
out_ca,
+ out_ca_detail,
out_ca_port);
if (diffcheck_flags & DIFF_FLAG_ROUTER)
@@ -777,6 +865,7 @@ int diff(ibnd_fabric_t * orig_fabric, ibnd_fabric_t * new_fabric)
IB_NODE_ROUTER,
diffcheck_flags,
out_ca,
+ out_ca_detail,
out_ca_port);
@@ -814,6 +903,10 @@ static int process_opt(void *context, int ch, char *optarg)
diffcheck_flags |= DIFF_FLAG_ROUTER;
else if (!strcasecmp(p, "port"))
diffcheck_flags |= DIFF_FLAG_PORT_CONNECTION;
+ else if (!strcasecmp(p, "lid"))
+ diffcheck_flags |= DIFF_FLAG_LID;
+ else if (!strcasecmp(p, "nodedesc"))
+ diffcheck_flags |= DIFF_FLAG_NODE_DESCRIPTION;
else {
fprintf(stderr, "invalid diff check key: %s\n", p);
return -1;
--
1.5.4.5
^ permalink raw reply related
* Re: Fork safe clarification
From: Matthew Small @ 2010-04-07 17:09 UTC (permalink / raw)
To: linux-rdma, rdreier-FYB4Gu1CFyUAvxtiuMwx3w,
or.gerlitz-Re5JQEeQqe8AvxtiuMwx3w
In-Reply-To: <s2pbb8be6ca1004010709hf5d0fe3cr2a76f28eaab97373-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
I know many of the people here are very busy and these might be
questions that are not about your work at hand, but these intricacies
of libibverbs are hard to figure out without an intimate knowledge of
the drivers (from the user side). I would really appreciate anyone
who would take the time to respond.
On Thu, Apr 1, 2010 at 10:09 AM, Matthew Small <matthewtsmall-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org> wrote:
> I am trying to understand the behavior of the libibverbs after it has been
> set into fork safe mode via a successful call to ibv_fork_init() or setting
> the environmental variable IBV_FORK_SAFE. For my purposes I would like to
> know the following :
>
> Are PDs, QPs and CQs created before a fork shared by the parent and child
> after fork() has returned (ie. both can submit WRs, poll CQ, etc.)?
>
>
> What about MRs registered before the fork? Even though the child doesn't
> have access to the parent's memory, can he sill submit WRs on a QP with an
> MR created before the fork?
>
>
> What if the MR pages in the above scenario are accessible in both parent and
> child (shared memory)? Are there complications with registering shared
> memory?
>
>
> In general, are pointers returned by libibverbs pointer to user/process
> address space (as ibv_mr pointers must be) or kernel space (eg. if an
> unrelated process had another process's QP pointer, lkey, and a virtual
> address could it post (almost certainly unsafely) a WR to the other
> process's QP?
>
>
> Sorry if the questions seem progressively more goofy and thanks in advance
> for any clarification.
>
> -Matt
>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* [PATCH 36/37] librdmacm: check if kernel supports AF_IB
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Add check during initialization to determine if the kernel
supports AF_IB.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/cma.c | 27 +++++++++++++++++++++++++++
1 files changed, 27 insertions(+), 0 deletions(-)
diff --git a/src/cma.c b/src/cma.c
index faf4264..66a7643 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -195,6 +195,32 @@ static int check_abi_version(void)
return 0;
}
+/*
+ * This function is called holding the mutex lock
+ * cma_dev_cnt must be set before calling this function to
+ * ensure that the lock is not acquired recursively.
+ */
+static void ucma_set_af_ib_support(void)
+{
+ struct rdma_cm_id *id;
+ struct sockaddr_ib sib;
+ int ret;
+
+ ret = rdma_create_id(NULL, &id, NULL, RDMA_PS_IB);
+ if (ret)
+ return;
+
+ memset(&sib, 0, sizeof sib);
+ sib.sib_family = AF_IB;
+ sib.sib_sid = htonll(RDMA_IB_IP_PS_TCP);
+ sib.sib_sid_mask = htonll(RDMA_IB_IP_PS_MASK);
+ af_ib_support = 1;
+ ret = rdma_bind_addr(id, (struct sockaddr *) &sib);
+ af_ib_support = !ret;
+
+ rdma_destroy_id(id);
+}
+
int ucma_init(void)
{
struct ibv_device **dev_list = NULL;
@@ -263,6 +289,7 @@ int ucma_init(void)
if (ib)
ucma_ib_init();
cma_dev_cnt = dev_cnt;
+ ucma_set_af_ib_support();
pthread_mutex_unlock(&mut);
ibv_free_device_list(dev_list);
return 0;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 37/37] librdmacm: use IB ACM to resolve IB path
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Starting with 2.6.33, the kernel supports the ability to
manually specify the path record that a connection should
use. Allow the librdmacm to contact the IB ACM to acquire
path record data, even if rdma_getaddrinfo is not used and
the kernel does not support AF_IB.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/acm.c | 2 +-
src/cma.c | 37 +++++++++++++++++++++++++++++++++++--
src/cma.h | 2 ++
3 files changed, 38 insertions(+), 3 deletions(-)
diff --git a/src/acm.c b/src/acm.c
index 8975b97..ca338bc 100644
--- a/src/acm.c
+++ b/src/acm.c
@@ -205,7 +205,7 @@ static void ucma_ib_save_resp(struct rdma_addrinfo *rai, struct acm_resolve_msg
rai->ai_route = path_data;
rai->ai_route_len = len;
- if (af_ib_support) {
+ if (af_ib_support && !(rai->ai_flags & RAI_ROUTEONLY)) {
ucma_ib_format_connect(rai);
if (ucma_ib_convert_addr(rai, pri_path) &&
rai->ai_connect) {
diff --git a/src/cma.c b/src/cma.c
index 66a7643..258a324 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -858,15 +858,47 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
return ucma_complete(id_priv);
}
+static int ucma_set_ib_route(struct rdma_cm_id *id)
+{
+ struct rdma_addrinfo rai;
+ struct sockaddr_in6 src, dst;
+ int size, ret;
+
+ memset(&rai, 0, sizeof rai);
+ rai.ai_flags = RAI_ROUTEONLY;
+ rai.ai_family = id->route.addr.src_addr.sa_family;
+ size = ucma_addrlen((struct sockaddr *) &id->route.addr.src_addr);
+
+ memcpy(&src, &id->route.addr.src_addr, size);
+ memcpy(&dst, &id->route.addr.dst_addr, size);
+ rai.ai_src_addr = (struct sockaddr *) &src;
+ rai.ai_dst_addr = (struct sockaddr *) &dst;
+
+ ucma_ib_resolve(&rai);
+ if (!rai.ai_route_len)
+ return ERR(ENODATA);
+
+ ret = rdma_set_option(id, RDMA_OPTION_IB, RDMA_OPTION_IB_PATH,
+ rai.ai_route, rai.ai_route_len);
+ free(rai.ai_route);
+ return ret;
+}
+
int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
{
struct ucma_abi_resolve_route *cmd;
struct cma_id_private *id_priv;
void *msg;
int ret, size;
-
- CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_RESOLVE_ROUTE, size);
+
id_priv = container_of(id, struct cma_id_private, id);
+ if (id->verbs->device->transport_type == IBV_TRANSPORT_IB) {
+ ret = ucma_set_ib_route(id);
+ if (!ret)
+ goto out;
+ }
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_RESOLVE_ROUTE, size);
cmd->id = id_priv->handle;
cmd->timeout_ms = timeout_ms;
@@ -874,6 +906,7 @@ int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
if (ret != size)
return (ret >= 0) ? ERR(ENODATA) : -1;
+out:
return ucma_complete(id_priv);
}
diff --git a/src/cma.h b/src/cma.h
index 1e0c571..d25cdea 100644
--- a/src/cma.h
+++ b/src/cma.h
@@ -87,6 +87,8 @@ static inline void *zalloc(size_t size)
int ucma_init();
extern int af_ib_support;
+#define RAI_ROUTEONLY 0x01000000
+
#ifdef USE_IB_ACM
void ucma_ib_init();
void ucma_ib_cleanup();
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 34/37] librdmacm: update man pages
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Update man pages to reflect changes to the APIs.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
man/rdma_cm.7 | 56 +++++++++++++++++++++++++++++++++++++++++------
man/rdma_create_ep.3 | 57 ++++++++++++++++++++++++++++++++++++++++++++++++
man/rdma_create_id.3 | 14 ++++++++----
man/rdma_create_qp.3 | 15 +++++++++++--
man/rdma_get_request.3 | 31 ++++++++++++++++++++++++++
man/rdma_migrate_id.3 | 6 ++++-
6 files changed, 165 insertions(+), 14 deletions(-)
diff --git a/man/rdma_cm.7 b/man/rdma_cm.7
index fd04959..ff5d489 100644
--- a/man/rdma_cm.7
+++ b/man/rdma_cm.7
@@ -8,17 +8,59 @@ Used to establish communication over RDMA transports.
.SH "NOTES"
The RDMA CM is a communication manager used to setup reliable, connected
and unreliable datagram data transfers. It provides an RDMA transport
-neutral interface for establishing connections. The API is based on sockets,
-but adapted for queue pair (QP) based semantics: communication must be
-over a specific RDMA device, and data transfers are message based.
+neutral interface for establishing connections. The API concepts are
+is based on sockets, but adapted for queue pair (QP) based semantics:
+communication must be over a specific RDMA device, and data transfers
+are message based.
.P
-The RDMA CM only provides the communication management (connection setup /
-teardown) portion of an RDMA API. It works in conjunction with the verbs
+The RDMA CM can control both the QP and communication management (connection setup /
+teardown) portions of an RDMA API, or only the communication management
+piece. It works in conjunction with the verbs
API defined by the libibverbs library. The libibverbs library provides the
-interfaces needed to send and receive data.
+underlying interfaces needed to send and receive data.
+.P
+The RDMA CM can operate asynchronously or synchronously. The mode of
+operation is controlled by the user through the use of the rdma_cm event channel
+parameter in specific calls. If an event channel is provided, an rdma_cm identifier
+will report its event data (results of connecting, for example), on that channel.
+If a channel is not provided, then all rdma_cm operations for the selected
+rdma_cm identifier are will block until they complete.
+.SH "RDMA VERBS"
+The rdma_cm supports the full range of verbs available through the libibverbs
+library and interfaces. However, it also provides wrapper functions for some
+of the more commonly used verbs funcationality. The full set of abstracted
+verb calls are:
+.P rdma_reg_msgs - register an array of buffers for sending and receiving
+.P rdma_reg_read - registers a buffer for RDMA read operations
+.P rdma_reg_write - registers a buffer for RDMA write operations
+.P rdma_dereg_mr - deregisters a memory region
+.P
+.P rdma_post_recv - post a buffer to receive a message
+.P rdma_post_send - post a buffer to send a message
+.P rdma_post_read - post an RDMA to read data into a buffer
+.P rdma_post_write - post an RDMA to send data from a buffer
+.P
+.P rdma_post_recvv - post a vector of buffers to receive a message
+.P rdma_post_sendv - post a vector of buffers to send a message
+.P rdma_post_readv - post a vector of buffers to receive an RDMA read
+.P rdma_post_writev - post a vector of buffers to send an RDMA write
+.P
+.P rdma_post_ud_send - post a buffer to send a message on a UD QP
+.P
+.P rdma_get_send_comp - get completion status for a send or RDMA operation
+.P rdma_get_recv_comp - get information about a completed receive
.SH "CLIENT OPERATION"
This section provides a general overview of the basic operation for the active,
-or client, side of communication. A general connection flow would be:
+or client, side of communication. This flow assume asynchronous operation with
+low level call details shown. For
+synchronous operation, calls to rdma_create_event_channel, rdma_get_cm_event,
+rdma_ack_cm_event, and rdma_destroy_event_channel
+would be eliminated. Abstracted calls, such as rdma_create_ep encapsulate
+serveral of these calls under a single API.
+Users may also refer to the example applications for
+code samples. A general connection flow would be:
+.IP rdma_getaddrinfo
+retrieve address information of the destination
.IP rdma_create_event_channel
create channel to receive events
.IP rdma_create_id
diff --git a/man/rdma_create_ep.3 b/man/rdma_create_ep.3
new file mode 100644
index 0000000..ae07113
--- /dev/null
+++ b/man/rdma_create_ep.3
@@ -0,0 +1,57 @@
+.TH "RDMA_CREATE_EP" 3 "2007-08-06" "librdmacm" "Librdmacm Programmer's Manual" librdmacm
+.SH NAME
+rdma_create_ep \- Allocate a communication identifier and optional QP.
+.SH SYNOPSIS
+.B "#include <rdma/rdma_cma.h>"
+.P
+.B "int" rdma_create_ep
+.BI "(struct rdma_cm_id **" id ","
+.BI "struct rdma_addrinfo *" res ","
+.BI "struct ibv_pd *" pd ","
+.BI "struct ibv_qp_init_attr *" qp_init_attr ");"
+.SH ARGUMENTS
+.IP "id" 12
+A reference where the allocated communication identifier will be
+returned.
+.IP "res" 12
+Address information associated with the rdma_cm_id returned from
+rdma_getaddrinfo.
+.IP "pd" 12
+Optional protection domain if a QP is associated with the rdma_cm_id.
+.IP "qp_init_attr" 12
+Optional initial QP attributes.
+.SH "DESCRIPTION"
+Creates an identifier that is used to track communication information.
+.SH "NOTES"
+After resolving address information using rdma_getaddrinfo, a user
+may use this call to allocate an rdma_cm_id based on the results.
+.P
+If the rdma_cm_id will be used on the active side of a connection,
+meaning that res->ai_flag is not RAI_PASSIVE set, rdma_create_ep
+will automatically create a QP on the rdma_cm_id if qp_init_attr is
+not NULL. The QP will be associated with the specified protection
+domain, if provided, or a default protection domain if not. Users
+should see rdma_create_qp for details on the use of the pd and
+qp_init_attr parameters. After calling rdma_create_ep, the returned
+rdma_cm_id may be connected by calling rdma_connect. The active side
+calls rdma_resolve_addr and rdma_resolve_route are not necessary.
+.P
+If the rdma_cm_id will be used on the passive side of a connection,
+indicated by having res->ai_flag RAI_PASSIVE set, this call will save
+the provided pd and qp_init_attr parameters. When a new connection
+request is retrieved by calling rdma_get_request, the rdma_cm_id
+associated with the new connection will automatically be associated
+with a QP using the pd and qp_init_attr parameters. After calling
+rdma_create_ep, the returned rdma_cm_id may be placed into a listening
+state by immediately calling rdma_listen. The passive side call
+rdma_bind_addr is not necessary. Connection requests may then be
+retrieved by calling rdma_get_request.
+.P
+The newly created rdma_cm_id will be set to use synchronous operation.
+Users that wish asynchronous operation must migrate the rdma_cm_id
+to a user created event channel using rdma_migrate_id.
+.P
+Users must release the created rdma_cm_id by calling rdma_destroy_ep.
+.SH "SEE ALSO"
+rdma_cm(7), rdma_getaddrinfo(3), rdma_create_event_channel(3),
+rdma_connect(3), rdma_listen(3), rdma_destroy_ep(3), rdma_migrate_id(3)
diff --git a/man/rdma_create_id.3 b/man/rdma_create_id.3
index eb29f3c..a7926d4 100644
--- a/man/rdma_create_id.3
+++ b/man/rdma_create_id.3
@@ -12,7 +12,7 @@ rdma_create_id \- Allocate a communication identifier.
.SH ARGUMENTS
.IP "channel" 12
The communication channel that events associated with the
-allocated rdma_cm_id will be reported on.
+allocated rdma_cm_id will be reported on. This may be NULL.
.IP "id" 12
A reference where the allocated communication identifier will be
returned.
@@ -26,9 +26,15 @@ Creates an identifier that is used to track communication information.
Rdma_cm_id's are conceptually equivalent to a socket for RDMA
communication. The difference is that RDMA communication requires
explicitly binding to a specified RDMA device before communication
-can occur, and most operations are asynchronous in nature. Communication
-events on an rdma_cm_id are reported through the associated event
-channel. Users must release the rdma_cm_id by calling rdma_destroy_id.
+can occur, and most operations are asynchronous in nature. Asynchronous
+communication events on an rdma_cm_id are reported through the associated
+event channel. If the channel paramet is NULL, the rdma_cm_id will
+be placed into synchronous operation. While operating synchronously,
+calls that result in an event will block until the operation completes.
+The event will be returned to the user through the rdma_cm_id structure,
+and be available for access until another rdma_cm call is made.
+.P
+Users must release the rdma_cm_id by calling rdma_destroy_id.
.SH "PORT SPACE"
Details of the services provided by the different port spaces are outlined
below.
diff --git a/man/rdma_create_qp.3 b/man/rdma_create_qp.3
index 0931499..ce1d862 100644
--- a/man/rdma_create_qp.3
+++ b/man/rdma_create_qp.3
@@ -12,9 +12,9 @@ rdma_create_qp \- Allocate a QP.
.IP "id" 12
RDMA identifier.
.IP "pd" 12
-protection domain for the QP.
+Optional protection domain for the QP.
.IP "qp_init_attr" 12
-initial QP attributes.
+Initial QP attributes.
.SH "DESCRIPTION"
Allocate a QP associated with the specified rdma_cm_id and transition it
for sending and receiving.
@@ -25,6 +25,17 @@ QPs allocated to an rdma_cm_id are automatically transitioned by the
librdmacm through their states. After being allocated, the QP will be
ready to handle posting of receives. If the QP is unconnected, it will
be ready to post sends.
+.P
+If a protection domain is not given - pd parameter is NULL - then
+the rdma_cm_id will be created using a default protection domain. One
+default protection domain is allocated per RDMA device.
+.P
+The initial QP attributes are specified by the qp_init_attr parameter. The
+send_cq and recv_cq fields in the ibv_qp_init_attr are optional. If
+a send or receive completion queue is not specified, then a CQ will be
+allocated by the rdma_cm for the QP, along with corresponding completion
+channels. Completion channels and CQ data created by the rdma_cm are
+exposed to the user through the rdma_cm_id structure.
.SH "SEE ALSO"
rdma_bind_addr(3), rdma_resolve_addr(3), rdma_destroy_qp(3), ibv_create_qp(3),
ibv_modify_qp(3)
diff --git a/man/rdma_get_request.3 b/man/rdma_get_request.3
new file mode 100644
index 0000000..cd48918
--- /dev/null
+++ b/man/rdma_get_request.3
@@ -0,0 +1,31 @@
+.TH "RDMA_GET_REQUEST" 3 "2007-10-31" "librdmacm" "Librdmacm Programmer's Manual" librdmacm
+.SH NAME
+rdma_get_request \- Retrieves the next pending connection request event.
+.SH SYNOPSIS
+.B "#include <rdma/rdma_cma.h>"
+.P
+.B "int" rdma_get_request
+.BI "(struct rdma_cm_id *" listen ","
+.BI "struct rdma_cm_id **" id ");"
+.SH ARGUMENTS
+.IP "listen" 12
+Listening rdma_cm_id.
+.IP "id" 12
+rdma_cm_id associated with the new connection.
+.SH "DESCRIPTION"
+Retrieves a connection request event. If no requests are pending,
+the call will block until an event is received.
+.SH "NOTES"
+This call may only be used on listening rdma_cm_id's operating
+synchronously. On success, a new rdma_cm_id representing the
+connection request will be returned to the user. The new rdma_cm_id
+will reference event information associated with the request until
+the user calls rdma_reject, rdma_accept, or rdma_destroy_id on the
+newly created identifier. For a description of the event data,
+see rdma_get_cm_event.
+.P
+If QP attributes are associated with the listening endpoint, the
+returned rdma_cm_id will also reference an allocated QP.
+.SH "SEE ALSO"
+rdma_get_cm_event(3), rdma_accept(3), rdma_reject(3),
+rdma_connect(3), rdma_listen(3), rdma_destroy_id(3)
diff --git a/man/rdma_migrate_id.3 b/man/rdma_migrate_id.3
index ffbad45..64448f6 100644
--- a/man/rdma_migrate_id.3
+++ b/man/rdma_migrate_id.3
@@ -12,7 +12,7 @@ rdma_migrate_id \- Move a communication identifer to a different event channel.
An existing communication identifier to migrate.
.IP "channel" 12
The communication channel that events associated with the
-allocated rdma_cm_id will be reported on.
+allocated rdma_cm_id will be reported on. May be NULL.
.SH "DESCRIPTION"
Migrates a communication identifier to a different event channel.
.SH "NOTES"
@@ -22,6 +22,10 @@ to the new channel. Users should not poll for events on the
rdma_cm_id's current event channel or invoke other routines on the
rdma_cm_id while migrating between channels. This call will block while
there are any unacknowledged events on the current event channel.
+.P
+If the channel parameter is NULL, the specified rdma_cm_id will be
+placed into synchronous operation mode. All calls on the id
+will block until the operation completes.
.SH "SEE ALSO"
rdma_cm(7), rdma_create_event_channel(3), rdma_create_id(3),
rdma_get_cm_event(3)
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 27/37] librdmacm: add support for IB ACM service
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Allow the librdmacm to contact a service via sockets to obtain
address mapping and path record data. The use of the service
is controlled through a build option (with-ib_acm). If the
library fails to contact the service, it falls back to using
the kernel services to resolve address and routing data.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Once IB ACM is proven, the build option can be removed.
Makefile.am | 2 -
configure.in | 14 +++++
src/acm.c | 160 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
src/addrinfo.c | 3 +
src/cma.c | 9 ++-
src/cma.h | 13 ++++-
6 files changed, 197 insertions(+), 4 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index be53c78..8d86045 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -12,7 +12,7 @@ else
librdmacm_version_script =
endif
-src_librdmacm_la_SOURCES = src/cma.c src/addrinfo.c
+src_librdmacm_la_SOURCES = src/cma.c src/addrinfo.c src/acm.c
src_librdmacm_la_LDFLAGS = -version-info 1 -export-dynamic \
$(librdmacm_version_script)
src_librdmacm_la_DEPENDENCIES = $(srcdir)/src/librdmacm.map
diff --git a/configure.in b/configure.in
index 1122966..3db4247 100644
--- a/configure.in
+++ b/configure.in
@@ -21,6 +21,15 @@ if test "$with_valgrind" != "" && test "$with_valgrind" != "no"; then
fi
fi
+AC_ARG_WITH([ib_acm],
+ AC_HELP_STRING([--with-ib_acm],
+ [Use IB ACM for route resolution - default NO]))
+
+if test "$with_ib_acm" != "" && test "$with_ib_acm" != "no"; then
+ AC_DEFINE([USE_IB_ACM], 1,
+ [Define to 1 to use IB ACM for endpoint resolution])
+fi
+
AC_ARG_ENABLE(libcheck, [ --disable-libcheck do not test for presence of ib libraries],
[ if test "$enableval" = "no"; then
disable_libcheck=yes
@@ -51,6 +60,11 @@ AC_CHECK_HEADER(valgrind/memcheck.h, [],
AC_MSG_ERROR([valgrind requested but <valgrind/memcheck.h> not found.]))
fi
+if test "$with_ib_acm" != "" && test "$with_ib_acm" != "no"; then
+AC_CHECK_HEADER(infiniband/acm.h, [],
+ AC_MSG_ERROR([IB ACM requested but <infiniband/acm.h> not found.]))
+fi
+
fi
AC_CACHE_CHECK(whether ld accepts --version-script, ac_cv_version_script,
diff --git a/src/acm.c b/src/acm.c
new file mode 100644
index 0000000..34fdf3c
--- /dev/null
+++ b/src/acm.c
@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <unistd.h>
+
+#include "cma.h"
+#include <rdma/rdma_cma.h>
+#include <infiniband/ib.h>
+#include <infiniband/sa.h>
+
+#ifdef USE_IB_ACM
+#include <infiniband/acm.h>
+
+static pthread_mutex_t acm_lock = PTHREAD_MUTEX_INITIALIZER;
+static int sock;
+static short server_port = 6125;
+
+void ucma_ib_init(void)
+{
+ struct sockaddr_in addr;
+ int ret;
+
+ sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+ if (sock < 0)
+ return;
+
+ memset(&addr, 0, sizeof addr);
+ addr.sin_family = AF_INET;
+ addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+ addr.sin_port = htons(server_port);
+ ret = connect(sock, (struct sockaddr *) &addr, sizeof(addr));
+ if (ret)
+ goto err;
+
+ return;
+
+err:
+ close(sock);
+ sock = 0;
+}
+
+void ucma_ib_cleanup(void)
+{
+ if (sock > 0) {
+ shutdown(sock, SHUT_RDWR);
+ close(sock);
+ }
+}
+
+static void ucma_ib_save_resp(struct rdma_addrinfo *rai, struct acm_resolve_msg *msg)
+{
+ struct ib_path_data *path_data = NULL;
+ int len, i, cnt;
+
+ len = msg->hdr.length - ACM_MSG_HDR_LENGTH;
+ cnt = len / sizeof(struct acm_ep_addr_data);
+ path_data = malloc(len);
+ if (!path_data)
+ return;
+
+ memcpy(path_data, msg->data, len);
+ for (i = 0; i < cnt; i++) {
+ if (msg->data[i].type != ACM_EP_INFO_PATH)
+ goto err;
+ path_data[i].reserved = 0;
+ }
+
+ rai->ai_route = path_data;
+ rai->ai_route_len = len;
+ return;
+err:
+ free(path_data);
+}
+
+void ucma_ib_resolve(struct rdma_addrinfo *rai)
+{
+ struct acm_msg msg;
+ struct acm_resolve_msg *resolve_msg = (struct acm_resolve_msg *) &msg;
+ struct acm_ep_addr_data *src_data, *dst_data;
+ int ret;
+
+ if (sock <= 0)
+ return;
+
+ memset(&msg, 0, sizeof msg);
+ msg.hdr.version = ACM_VERSION;
+ msg.hdr.opcode = ACM_OP_RESOLVE;
+ msg.hdr.length = ACM_MSG_HDR_LENGTH + (2 * ACM_MSG_EP_LENGTH);
+
+ src_data = &resolve_msg->data[0];
+ dst_data = &resolve_msg->data[1];
+
+ src_data->flags = ACM_EP_FLAG_SOURCE;
+ dst_data->flags = ACM_EP_FLAG_DEST;
+ if (rai->ai_family == AF_INET) {
+ src_data->type = dst_data->type = ACM_EP_INFO_ADDRESS_IP;
+ memcpy(src_data->info.addr,
+ &((struct sockaddr_in *) rai->ai_src_addr)->sin_addr, 4);
+ memcpy(dst_data->info.addr,
+ &((struct sockaddr_in *) rai->ai_dst_addr)->sin_addr, 4);
+ } else {
+ src_data->type = dst_data->type = ACM_EP_INFO_ADDRESS_IP6;
+ memcpy(src_data->info.addr,
+ &((struct sockaddr_in6 *) rai->ai_src_addr)->sin6_addr, 16);
+ memcpy(dst_data->info.addr,
+ &((struct sockaddr_in6 *) rai->ai_dst_addr)->sin6_addr, 16);
+ }
+
+ pthread_mutex_lock(&acm_lock);
+ ret = send(sock, (char *) &msg, msg.hdr.length, 0);
+ if (ret != msg.hdr.length) {
+ pthread_mutex_unlock(&acm_lock);
+ return;
+ }
+
+ ret = recv(sock, (char *) &msg, sizeof msg, 0);
+ pthread_mutex_unlock(&acm_lock);
+ if (ret < ACM_MSG_HDR_LENGTH || ret != msg.hdr.length || msg.hdr.status)
+ return;
+
+ ucma_ib_save_resp(rai, resolve_msg);
+}
+
+#endif /* USE_IB_ACM */
diff --git a/src/addrinfo.c b/src/addrinfo.c
index dfaf9d5..dad9b82 100644
--- a/src/addrinfo.c
+++ b/src/addrinfo.c
@@ -219,6 +219,9 @@ int rdma_getaddrinfo(char *node, char *service,
}
}
+ if (!(rai->ai_flags & RAI_PASSIVE))
+ ucma_ib_resolve(rai);
+
freeaddrinfo(ai);
*res = rai;
return 0;
diff --git a/src/cma.c b/src/cma.c
index 1cc4f1f..6ac949a 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -149,6 +149,8 @@ int af_ib_support;
static void ucma_cleanup(void)
{
+ ucma_ib_cleanup();
+
if (cma_dev_cnt) {
while (cma_dev_cnt--) {
ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd);
@@ -196,7 +198,7 @@ int ucma_init(void)
struct ibv_device **dev_list = NULL;
struct cma_device *cma_dev;
struct ibv_device_attr attr;
- int i, ret, dev_cnt;
+ int i, ret, dev_cnt, ib;
/* Quick check without lock to see if we're already initialized */
if (cma_dev_cnt)
@@ -225,7 +227,7 @@ int ucma_init(void)
goto err2;
}
- for (i = 0; dev_list[i];) {
+ for (i = 0, ib = 0; dev_list[i];) {
cma_dev = &cma_dev_array[i];
cma_dev->guid = ibv_get_device_guid(dev_list[i]);
@@ -253,8 +255,11 @@ int ucma_init(void)
cma_dev->port_cnt = attr.phys_port_cnt;
cma_dev->max_initiator_depth = (uint8_t) attr.max_qp_init_rd_atom;
cma_dev->max_responder_resources = (uint8_t) attr.max_qp_rd_atom;
+ ib += (cma_dev->verbs->device->transport_type == IBV_TRANSPORT_IB);
}
+ if (ib)
+ ucma_ib_init();
cma_dev_cnt = dev_cnt;
pthread_mutex_unlock(&mut);
ibv_free_device_list(dev_list);
diff --git a/src/cma.h b/src/cma.h
index 06ca38c..62785de 100644
--- a/src/cma.h
+++ b/src/cma.h
@@ -44,6 +44,8 @@
#include <byteswap.h>
#include <string.h>
+#include <rdma/rdma_cma.h>
+
#ifdef INCLUDE_VALGRIND
# include <valgrind/memcheck.h>
# ifndef VALGRIND_MAKE_MEM_DEFINED
@@ -84,5 +86,14 @@ static inline void *zalloc(size_t size)
int ucma_init();
-#endif /* CMA_H */
+#ifdef USE_IB_ACM
+void ucma_ib_init();
+void ucma_ib_cleanup();
+void ucma_ib_resolve(struct rdma_addrinfo *rai);
+#else
+#define ucma_ib_init()
+#define ucma_ib_cleanup()
+#define ucma_ib_resolve(x)
+#endif
+#endif /* CMA_H */
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 32/37] librdmacm/rdma_client: add new client sample
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Provide a very simple client application that shows the
minimal coding needed to establish a connection and exchange
messages with a server. The client makes use of the new
rdma_getaddrinfo and rdma_create_ep calls, plus rdma verbs
abstractions.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Makefile.am | 5 +-
examples/rdma_client.c | 134 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 138 insertions(+), 1 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index 8aef24a..6071599 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -17,7 +17,8 @@ src_librdmacm_la_LDFLAGS = -version-info 1 -export-dynamic \
$(librdmacm_version_script)
src_librdmacm_la_DEPENDENCIES = $(srcdir)/src/librdmacm.map
-bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy examples/mckey
+bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy examples/mckey \
+ examples/rdma_client
examples_ucmatose_SOURCES = examples/cmatose.c
examples_ucmatose_LDADD = $(top_builddir)/src/librdmacm.la
examples_rping_SOURCES = examples/rping.c
@@ -26,6 +27,8 @@ examples_udaddy_SOURCES = examples/udaddy.c
examples_udaddy_LDADD = $(top_builddir)/src/librdmacm.la
examples_mckey_SOURCES = examples/mckey.c
examples_mckey_LDADD = $(top_builddir)/src/librdmacm.la
+examples_rdma_client_SOURCES = examples/rdma_client.c
+examples_rdma_client_LDADD = $(top_builddir)/src/librdmacm.la
librdmacmincludedir = $(includedir)/rdma $(includedir)/infiniband
diff --git a/examples/rdma_client.c b/examples/rdma_client.c
new file mode 100644
index 0000000..7a59d97
--- /dev/null
+++ b/examples/rdma_client.c
@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under the OpenIB.org BSD license
+ * below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <getopt.h>
+#include <rdma/rdma_cma.h>
+#include <rdma/rdma_verbs.h>
+
+static char *server = "127.0.0.1";
+static char *port = "7471";
+
+struct rdma_cm_id *id;
+struct ibv_mr *mr;
+uint8_t send_msg[16];
+uint8_t recv_msg[16];
+
+static int run(void)
+{
+ struct rdma_addrinfo hints, *res;
+ struct ibv_qp_init_attr attr;
+ struct ibv_wc wc;
+ int ret;
+
+ memset(&hints, 0, sizeof hints);
+ hints.ai_port_space = RDMA_PS_TCP;
+ ret = rdma_getaddrinfo(server, port, &hints, &res);
+ if (ret) {
+ printf("rdma_getaddrinfo %d\n", errno);
+ return ret;
+ }
+
+ memset(&attr, 0, sizeof attr);
+ attr.cap.max_send_wr = attr.cap.max_recv_wr = 1;
+ attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
+ attr.cap.max_inline_data = 16;
+ attr.qp_context = id;
+ attr.sq_sig_all = 1;
+ ret = rdma_create_ep(&id, res, NULL, &attr);
+ rdma_freeaddrinfo(res);
+ if (ret) {
+ printf("rdma_create_ep %d\n", errno);
+ return ret;
+ }
+
+ mr = rdma_reg_msgs(id, recv_msg, 16);
+ if (!mr) {
+ printf("rdma_reg_msgs %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);
+ if (ret) {
+ printf("rdma_post_recv %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_connect(id, NULL);
+ if (ret) {
+ printf("rdma_connect %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_post_send(id, NULL, send_msg, 16, NULL, IBV_SEND_INLINE);
+ if (ret) {
+ printf("rdma_post_send %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_get_recv_comp(id, &wc);
+ if (ret <= 0) {
+ printf("rdma_get_recv_comp %d\n", ret);
+ return ret;
+ }
+
+ rdma_disconnect(id);
+ rdma_dereg_mr(mr);
+ rdma_destroy_ep(id);
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ int op, ret;
+
+ while ((op = getopt(argc, argv, "s:p:")) != -1) {
+ switch (op) {
+ case 's':
+ server = optarg;
+ break;
+ case 'p':
+ port = optarg;
+ break;
+ default:
+ printf("usage: %s\n", argv[0]);
+ printf("\t[-s server_address]\n");
+ printf("\t[-p port_number]\n");
+ exit(1);
+ }
+ }
+
+ printf("rdma_client: start\n");
+ ret = run();
+ printf("rdma_client: end %d\n", ret);
+ return ret;
+}
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 35/37] librdmacm/mckey: use AF_IB for unmapped multicast addresses
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
If the user joins an unmapped multicast address, use AF_IB,
rather than AF_INET6, to communicate that information with the
kernel.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
This requires AF_IB support in the kernel.
examples/mckey.c | 21 ++++++++++++++++++---
1 files changed, 18 insertions(+), 3 deletions(-)
diff --git a/examples/mckey.c b/examples/mckey.c
index ddc3495..a6b5c4d 100644
--- a/examples/mckey.c
+++ b/examples/mckey.c
@@ -46,6 +46,7 @@
#include <getopt.h>
#include <rdma/rdma_cma.h>
+#include <infiniband/ib.h>
struct cmatest_node {
int id;
@@ -67,9 +68,9 @@ struct cmatest {
int conn_index;
int connects_left;
- struct sockaddr_in6 dst_in;
+ struct sockaddr_storage dst_in;
struct sockaddr *dst_addr;
- struct sockaddr_in6 src_in;
+ struct sockaddr_storage src_in;
struct sockaddr *src_addr;
};
@@ -460,6 +461,20 @@ static int get_addr(char *dst, struct sockaddr *addr)
return ret;
}
+static int get_dst_addr(char *dst, struct sockaddr *addr)
+{
+ struct sockaddr_ib *sib;
+
+ if (!unmapped_addr)
+ return get_addr(dst, addr);
+
+ sib = (struct sockaddr_ib *) addr;
+ memset(sib, 0, sizeof *sib);
+ sib->sib_family = AF_IB;
+ inet_pton(AF_INET6, dst, &sib->sib_addr);
+ return 0;
+}
+
static int run(void)
{
int i, ret;
@@ -471,7 +486,7 @@ static int run(void)
return ret;
}
- ret = get_addr(dst_addr, (struct sockaddr *) &test.dst_in);
+ ret = get_dst_addr(dst_addr, (struct sockaddr *) &test.dst_in);
if (ret)
return ret;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 28/37] librdmacm: define RDMA_PS_IB
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
AF_IB uses the IB port space.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 12 +++++++++---
1 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 289b728..8f72201 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -67,11 +67,17 @@ enum rdma_cm_event_type {
};
enum rdma_port_space {
- RDMA_PS_IPOIB= 0x0002,
- RDMA_PS_TCP = 0x0106,
- RDMA_PS_UDP = 0x0111,
+ RDMA_PS_IPOIB = 0x0002,
+ RDMA_PS_IB = 0x0003,
+ RDMA_PS_TCP = 0x0106,
+ RDMA_PS_UDP = 0x0111,
};
+#define RDMA_IB_IP_PS_MASK 0xFFFFFFFFFFFF0000ULL
+#define RDMA_IB_IP_PORT_MASK 0x000000000000FFFFULL
+#define RDMA_IB_IP_PS_TCP 0x0000000001060000ULL
+#define RDMA_IB_IP_PS_UDP 0x0000000001110000ULL
+
/*
* Global qkey value for UDP QPs and multicast groups created via the
* RDMA CM.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 31/37] librdmacm: provide abstracted verb calls
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Provide abstractions to the verb calls to simplify the user
interface for more casual verbs consumers. Users still have
access to the full range of verbs functionality by calling
verbs directly.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Makefile.am | 5 -
include/rdma/rdma_verbs.h | 287 +++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 290 insertions(+), 2 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index 8d86045..8aef24a 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -31,7 +31,8 @@ librdmacmincludedir = $(includedir)/rdma $(includedir)/infiniband
librdmacminclude_HEADERS = include/rdma/rdma_cma_abi.h \
include/rdma/rdma_cma.h \
- include/infiniband/ib.h
+ include/infiniband/ib.h \
+ include/rdma/rdma_verbs.h
man_MANS = \
man/rdma_accept.3 \
@@ -69,7 +70,7 @@ man_MANS = \
man/rdma_cm.7
EXTRA_DIST = include/rdma/rdma_cma_abi.h include/rdma/rdma_cma.h \
- include/infiniband/ib.h \
+ include/infiniband/ib.h include/rdma/rdma_verbs.h \
src/cma.h src/librdmacm.map librdmacm.spec.in $(man_MANS)
dist-hook: librdmacm.spec
diff --git a/include/rdma/rdma_verbs.h b/include/rdma/rdma_verbs.h
new file mode 100644
index 0000000..05964c1
--- /dev/null
+++ b/include/rdma/rdma_verbs.h
@@ -0,0 +1,287 @@
+/*
+ * Copyright (c) 2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if !defined(RDMA_VERBS_H)
+#define RDMA_VERBS_H
+
+#include <assert.h>
+#include <infiniband/verbs.h>
+#include <rdma/rdma_cma.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Memory registration helpers.
+ */
+static inline struct ibv_mr *
+rdma_reg_msgs(struct rdma_cm_id *id, void *addr, size_t length)
+{
+ return ibv_reg_mr(id->qp->pd, addr, length, IBV_ACCESS_LOCAL_WRITE);
+}
+
+static inline struct ibv_mr *
+rdma_reg_read(struct rdma_cm_id *id, void *addr, size_t length)
+{
+ return ibv_reg_mr(id->qp->pd, addr, length, IBV_ACCESS_LOCAL_WRITE|
+ IBV_ACCESS_REMOTE_READ);
+}
+
+static inline struct ibv_mr *
+rdma_reg_write(struct rdma_cm_id *id, void *addr, size_t length)
+{
+ return ibv_reg_mr(id->qp->pd, addr, length, IBV_ACCESS_LOCAL_WRITE |
+ IBV_ACCESS_REMOTE_WRITE);
+}
+
+static inline int
+rdma_dereg_mr(struct ibv_mr *mr)
+{
+ return ibv_dereg_mr(mr);
+}
+
+
+/*
+ * Vectored send, receive, and RDMA operations.
+ * Support multiple scatter-gather entries.
+ */
+static inline int
+rdma_post_recvv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
+ int nsge)
+{
+ struct ibv_recv_wr wr, *bad;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = sgl;
+ wr.num_sge = nsge;
+
+ return ibv_post_recv(id->qp, &wr, &bad);
+}
+
+static inline int
+rdma_post_sendv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
+ int nsge, int flags)
+{
+ struct ibv_send_wr wr, *bad;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = sgl;
+ wr.num_sge = nsge;
+ wr.opcode = IBV_WR_SEND;
+ wr.send_flags = flags;
+
+ return ibv_post_send(id->qp, &wr, &bad);
+}
+
+static inline int
+rdma_post_readv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
+ int nsge, int flags, uint64_t remote_addr, uint32_t rkey)
+{
+ struct ibv_send_wr wr, *bad;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = sgl;
+ wr.num_sge = nsge;
+ wr.opcode = IBV_WR_RDMA_READ;
+ wr.send_flags = flags;
+ wr.wr.rdma.remote_addr = remote_addr;
+ wr.wr.rdma.rkey = rkey;
+
+ return ibv_post_send(id->qp, &wr, &bad);
+}
+
+static inline int
+rdma_post_writev(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
+ int nsge, int flags, uint64_t remote_addr, uint32_t rkey)
+{
+ struct ibv_send_wr wr, *bad;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = sgl;
+ wr.num_sge = nsge;
+ wr.opcode = IBV_WR_RDMA_WRITE;
+ wr.send_flags = flags;
+ wr.wr.rdma.remote_addr = remote_addr;
+ wr.wr.rdma.rkey = rkey;
+
+ return ibv_post_send(id->qp, &wr, &bad);
+}
+
+/*
+ * Simple send, receive, and RDMA calls.
+ */
+static inline int
+rdma_post_recv(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr)
+{
+ struct ibv_sge sge;
+
+ assert((addr >= mr->addr) && ((addr + length) <= (mr->addr + mr->length)));
+ sge.addr = (uint64_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr->lkey;
+
+ return rdma_post_recvv(id, context, &sge, 1);
+}
+
+static inline int
+rdma_post_send(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr, int flags)
+{
+ struct ibv_sge sge;
+
+ sge.addr = (uint64_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr ? mr->lkey : 0;
+
+ return rdma_post_sendv(id, context, &sge, 1, flags);
+}
+
+static inline int
+rdma_post_read(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr, int flags,
+ uint64_t remote_addr, uint32_t rkey)
+{
+ struct ibv_sge sge;
+
+ sge.addr = (uint64_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr->lkey;
+
+ return rdma_post_readv(id, context, &sge, 1, flags, remote_addr, rkey);
+}
+
+static inline int
+rdma_post_write(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr, int flags,
+ uint64_t remote_addr, uint32_t rkey)
+{
+ struct ibv_sge sge;
+
+ sge.addr = (uint64_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr ? mr->lkey : 0;
+
+ return rdma_post_writev(id, context, &sge, 1, flags, remote_addr, rkey);
+}
+
+static inline int
+rdma_post_ud_send(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr, int flags,
+ struct ibv_ah *ah, uint32_t remote_qpn)
+{
+ struct ibv_send_wr wr, *bad;
+ struct ibv_sge sge;
+
+ sge.addr = (uint64_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr ? mr->lkey : 0;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = &sge;
+ wr.num_sge = 1;
+ wr.opcode = IBV_WR_SEND;
+ wr.send_flags = flags;
+ wr.wr.ud.ah = ah;
+ wr.wr.ud.remote_qpn = remote_qpn;
+ wr.wr.ud.remote_qkey = RDMA_UDP_QKEY;
+
+ return ibv_post_send(id->qp, &wr, &bad);
+}
+
+static inline int
+rdma_get_send_comp(struct rdma_cm_id *id, struct ibv_wc *wc)
+{
+ struct ibv_cq *cq;
+ void *context;
+ int ret;
+
+ ret = ibv_poll_cq(id->send_cq, 1, wc);
+ if (ret)
+ return ret;
+
+ ret = ibv_req_notify_cq(id->send_cq, 0);
+ if (ret)
+ return ret;
+
+ ret = ibv_poll_cq(id->send_cq, 1, wc);
+ if (ret)
+ return ret;
+
+ ret = ibv_get_cq_event(id->send_cq_channel, &cq, &context);
+ if (ret)
+ return ret;
+
+ assert(cq == id->send_cq && context == id);
+ ibv_ack_cq_events(id->send_cq, 1);
+ return ibv_poll_cq(id->send_cq, 1, wc);
+}
+
+static inline int
+rdma_get_recv_comp(struct rdma_cm_id *id, struct ibv_wc *wc)
+{
+ struct ibv_cq *cq;
+ void *context;
+ int ret;
+
+ ret = ibv_poll_cq(id->recv_cq, 1, wc);
+ if (ret)
+ return ret;
+
+ ret = ibv_req_notify_cq(id->recv_cq, 0);
+ if (ret)
+ return ret;
+
+ ret = ibv_poll_cq(id->recv_cq, 1, wc);
+ if (ret)
+ return ret;
+
+ ret = ibv_get_cq_event(id->recv_cq_channel, &cq, &context);
+ if (ret)
+ return ret;
+
+ assert(cq == id->recv_cq && context == id);
+ ibv_ack_cq_events(id->recv_cq, 1);
+ return ibv_poll_cq(id->recv_cq, 1, wc);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RDMA_CMA_H */
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 29/37] librdmacm: use sockaddr_ib addressing if IB ACM is in use
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
If IB ACM route resolution succeeds, provide the user with
AF_IB addresses, rather than AF_INET or AF_INET6 addressing.
AF_IB identifies the local and remote devices directly,
eliminating the need to perform address resolution a second
time via rdma_resolve_addr.
AF_IB addresses are returned using sockaddr_ib as part of
rdma_getaddrinfo.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/acm.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
src/cma.h | 1 +
2 files changed, 66 insertions(+), 0 deletions(-)
diff --git a/src/acm.c b/src/acm.c
index 34fdf3c..5dd8899 100644
--- a/src/acm.c
+++ b/src/acm.c
@@ -83,9 +83,69 @@ void ucma_ib_cleanup(void)
}
}
+static void ucma_set_sid(enum rdma_port_space ps, struct sockaddr *addr,
+ struct sockaddr_ib *sib)
+{
+ uint16_t port;
+
+ if (addr->sa_family == AF_INET)
+ port = ((struct sockaddr_in *) addr)->sin_port;
+ else
+ port = ((struct sockaddr_in6 *) addr)->sin6_port;
+
+ sib->sib_sid = htonll(((uint64_t) ps << 16) + ntohs(port));
+ if (port)
+ sib->sib_sid_mask = ~0ULL;
+ else
+ sib->sib_sid_mask = htonll(RDMA_IB_IP_PS_MASK);
+}
+
+static void ucma_ib_convert_addr(struct rdma_addrinfo *rai,
+ struct ib_path_record *path)
+{
+ struct sockaddr_ib *src, *dst;
+
+ if (!path)
+ return;
+
+ src = zalloc(sizeof *src);
+ if (!src)
+ return;
+
+ dst = zalloc(sizeof *dst);
+ if (!dst) {
+ free(src);
+ return;
+ }
+
+ src->sib_family = AF_IB;
+ src->sib_pkey = path->pkey;
+ src->sib_flowinfo = htonl(ntohl(path->flowlabel_hoplimit) >> 8);
+ memcpy(&src->sib_addr, &path->sgid, 16);
+ ucma_set_sid(rai->ai_port_space, rai->ai_src_addr, src);
+
+ dst->sib_family = AF_IB;
+ dst->sib_pkey = path->pkey;
+ dst->sib_flowinfo = htonl(ntohl(path->flowlabel_hoplimit) >> 8);
+ memcpy(&dst->sib_addr, &path->dgid, 16);
+ ucma_set_sid(rai->ai_port_space, rai->ai_dst_addr, dst);
+
+ free(rai->ai_src_addr);
+ rai->ai_src_addr = (struct sockaddr *) src;
+ rai->ai_src_len = sizeof(*src);
+
+ free(rai->ai_dst_addr);
+ rai->ai_dst_addr = (struct sockaddr *) dst;
+ rai->ai_dst_len = sizeof(*dst);
+
+ rai->ai_family = AF_IB;
+ rai->ai_port_space = RDMA_PS_IB;
+}
+
static void ucma_ib_save_resp(struct rdma_addrinfo *rai, struct acm_resolve_msg *msg)
{
struct ib_path_data *path_data = NULL;
+ struct ib_path_record *pri_path;
int len, i, cnt;
len = msg->hdr.length - ACM_MSG_HDR_LENGTH;
@@ -98,11 +158,16 @@ static void ucma_ib_save_resp(struct rdma_addrinfo *rai, struct acm_resolve_msg
for (i = 0; i < cnt; i++) {
if (msg->data[i].type != ACM_EP_INFO_PATH)
goto err;
+ if (path_data[i].flags |
+ (IB_PATH_FLAG_PRIMARY | IB_PATH_FLAG_OUTBOUND))
+ pri_path = &path_data[i].path;
path_data[i].reserved = 0;
}
rai->ai_route = path_data;
rai->ai_route_len = len;
+ if (af_ib_support)
+ ucma_ib_convert_addr(rai, pri_path);
return;
err:
free(path_data);
diff --git a/src/cma.h b/src/cma.h
index 62785de..1e0c571 100644
--- a/src/cma.h
+++ b/src/cma.h
@@ -85,6 +85,7 @@ static inline void *zalloc(size_t size)
}
int ucma_init();
+extern int af_ib_support;
#ifdef USE_IB_ACM
void ucma_ib_init();
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 26/37] librdmacm: set src_addr in rdma_getaddrinfo
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
RDMA requires the user to allocate hardware resources before
establishing a connection. To support this, the user must know
the source address that the connection will use to reach the
remote endpoint. Modify rdma_getaddrinfo to determine an
appropriate source address based on the specified destination,
when a source address is not given.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/addrinfo.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 60 insertions(+), 0 deletions(-)
diff --git a/src/addrinfo.c b/src/addrinfo.c
index 15ae071..dfaf9d5 100644
--- a/src/addrinfo.c
+++ b/src/addrinfo.c
@@ -39,6 +39,7 @@
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
+#include <unistd.h>
#include "cma.h"
#include <rdma/rdma_cma.h>
@@ -129,6 +130,48 @@ static int ucma_convert_to_rai(struct rdma_addrinfo *rai, struct addrinfo *ai)
return 0;
}
+static int ucma_resolve_src(struct rdma_addrinfo *rai)
+{
+ struct sockaddr *addr;
+ socklen_t len;
+ int ret, s;
+
+ s = socket(rai->ai_family, SOCK_DGRAM, IPPROTO_UDP);
+ if (s < 0)
+ return s;
+
+ ret = connect(s, rai->ai_dst_addr, rai->ai_dst_len);
+ if (ret)
+ goto err1;
+
+ addr = zalloc(rai->ai_dst_len);
+ if (!addr) {
+ ret = ERR(ENOMEM);
+ goto err1;
+ }
+
+ len = rai->ai_dst_len;
+ ret = getsockname(s, addr, &len);
+ if (ret)
+ goto err2;
+
+ if (addr->sa_family == AF_INET)
+ ((struct sockaddr_in *) addr)->sin_port = 0;
+ else
+ ((struct sockaddr_in6 *) addr)->sin6_port = 0;
+ rai->ai_src_addr = addr;
+ rai->ai_src_len = len;
+
+ close(s);
+ return 0;
+
+err2:
+ free(addr);
+err1:
+ close(s);
+ return ret;
+}
+
int rdma_getaddrinfo(char *node, char *service,
struct rdma_addrinfo *hints,
struct rdma_addrinfo **res)
@@ -159,6 +202,23 @@ int rdma_getaddrinfo(char *node, char *service,
if (ret)
goto err2;
+ if (!rai->ai_src_len) {
+ if (hints && hints->ai_src_len) {
+ rai->ai_src_addr = zalloc(hints->ai_src_len);
+ if (!rai->ai_src_addr) {
+ ret = ERR(ENOMEM);
+ goto err2;
+ }
+ memcpy(rai->ai_src_addr, hints->ai_src_addr,
+ hints->ai_src_len);
+ rai->ai_src_len = hints->ai_src_len;
+ } else {
+ ret = ucma_resolve_src(rai);
+ if (ret)
+ goto err2;
+ }
+ }
+
freeaddrinfo(ai);
*res = rai;
return 0;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 25/37] librdmacm: auto create QPs in rdma_get_request
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
If the user passed initial QP attributes into rdma_create_ep,
allocate a QP for the user as part of rdma_get_request.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/cma.c | 6 ++++++
1 files changed, 6 insertions(+), 0 deletions(-)
diff --git a/src/cma.c b/src/cma.c
index c59f47b..1cc4f1f 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -1287,6 +1287,12 @@ int rdma_get_request(struct rdma_cm_id *listen, struct rdma_cm_id **id)
goto err;
}
+ if (id_priv->qp_init_attr) {
+ ret = rdma_create_qp(event->id, id_priv->pd, id_priv->qp_init_attr);
+ if (ret)
+ goto err;
+ }
+
*id = event->id;
(*id)->event = event;
return 0;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 30/37] librdmacm: format IB CM private data RDMA CM header
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/acm.c | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++-------
src/cma.c | 32 +++++++++++++++++++++++++++++---
2 files changed, 81 insertions(+), 10 deletions(-)
diff --git a/src/acm.c b/src/acm.c
index 5dd8899..8975b97 100644
--- a/src/acm.c
+++ b/src/acm.c
@@ -51,6 +51,18 @@ static pthread_mutex_t acm_lock = PTHREAD_MUTEX_INITIALIZER;
static int sock;
static short server_port = 6125;
+struct ib_connect_hdr {
+ uint8_t cma_version;
+ uint8_t ip_version; /* IP version: 7:4 */
+ uint16_t port;
+ uint32_t src_addr[4];
+ uint32_t dst_addr[4];
+#define cma_src_ip4 src_addr[3]
+#define cma_src_ip6 src_addr[0]
+#define cma_dst_ip4 dst_addr[3]
+#define cma_dst_ip6 dst_addr[0]
+};
+
void ucma_ib_init(void)
{
struct sockaddr_in addr;
@@ -100,22 +112,22 @@ static void ucma_set_sid(enum rdma_port_space ps, struct sockaddr *addr,
sib->sib_sid_mask = htonll(RDMA_IB_IP_PS_MASK);
}
-static void ucma_ib_convert_addr(struct rdma_addrinfo *rai,
- struct ib_path_record *path)
+static int ucma_ib_convert_addr(struct rdma_addrinfo *rai,
+ struct ib_path_record *path)
{
struct sockaddr_ib *src, *dst;
if (!path)
- return;
+ return ERR(ENODATA);
src = zalloc(sizeof *src);
if (!src)
- return;
+ return ERR(ENOMEM);
dst = zalloc(sizeof *dst);
if (!dst) {
free(src);
- return;
+ return ERR(ENOMEM);
}
src->sib_family = AF_IB;
@@ -140,6 +152,33 @@ static void ucma_ib_convert_addr(struct rdma_addrinfo *rai,
rai->ai_family = AF_IB;
rai->ai_port_space = RDMA_PS_IB;
+ return 0;
+}
+
+static void ucma_ib_format_connect(struct rdma_addrinfo *rai)
+{
+ struct ib_connect_hdr *hdr;
+
+ hdr = zalloc(sizeof *hdr);
+ if (!hdr)
+ return;
+
+ if (rai->ai_family == AF_INET) {
+ hdr->ip_version = 4 << 4;
+ memcpy(&hdr->cma_src_ip4,
+ &((struct sockaddr_in *) rai->ai_src_addr)->sin_addr, 4);
+ memcpy(&hdr->cma_dst_ip4,
+ &((struct sockaddr_in *) rai->ai_dst_addr)->sin_addr, 4);
+ } else {
+ hdr->ip_version = 6 << 4;
+ memcpy(&hdr->cma_src_ip6,
+ &((struct sockaddr_in6 *) rai->ai_src_addr)->sin6_addr, 16);
+ memcpy(&hdr->cma_dst_ip6,
+ &((struct sockaddr_in6 *) rai->ai_dst_addr)->sin6_addr, 16);
+ }
+
+ rai->ai_connect = hdr;
+ rai->ai_connect_len = sizeof(*hdr);
}
static void ucma_ib_save_resp(struct rdma_addrinfo *rai, struct acm_resolve_msg *msg)
@@ -166,8 +205,14 @@ static void ucma_ib_save_resp(struct rdma_addrinfo *rai, struct acm_resolve_msg
rai->ai_route = path_data;
rai->ai_route_len = len;
- if (af_ib_support)
- ucma_ib_convert_addr(rai, pri_path);
+ if (af_ib_support) {
+ ucma_ib_format_connect(rai);
+ if (ucma_ib_convert_addr(rai, pri_path) &&
+ rai->ai_connect) {
+ free(rai->ai_connect);
+ rai->ai_connect_len = 0;
+ }
+ }
return;
err:
free(path_data);
diff --git a/src/cma.c b/src/cma.c
index 6ac949a..faf4264 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -106,6 +106,8 @@ struct cma_device {
struct cma_id_private {
struct rdma_cm_id id;
struct cma_device *cma_dev;
+ void *connect;
+ size_t connect_len;
int events_completed;
int connect_error;
int sync;
@@ -363,6 +365,8 @@ static void ucma_free_id(struct cma_id_private *id_priv)
if (id_priv->sync)
rdma_destroy_event_channel(id_priv->id.channel);
+ if (id_priv->connect)
+ free(id_priv->connect);
free(id_priv);
}
@@ -1186,15 +1190,20 @@ static void ucma_copy_conn_param_to_kern(struct cma_id_private *id_priv,
dst->initiator_depth = id_priv->initiator_depth;
dst->valid = 1;
+ if (id_priv->connect_len) {
+ memcpy(dst->private_data, id_priv->connect, id_priv->connect_len);
+ dst->private_data_len = id_priv->connect_len;
+ }
+
if (src) {
dst->flow_control = src->flow_control;
dst->retry_count = src->retry_count;
dst->rnr_retry_count = src->rnr_retry_count;
if (src->private_data && src->private_data_len) {
- memcpy(dst->private_data, src->private_data,
- src->private_data_len);
- dst->private_data_len = src->private_data_len;
+ memcpy(dst->private_data + dst->private_data_len,
+ src->private_data, src->private_data_len);
+ dst->private_data_len += src->private_data_len;
}
} else {
dst->retry_count = 7;
@@ -1238,6 +1247,11 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
if (ret != size)
return (ret >= 0) ? ERR(ENODATA) : -1;
+ if (id_priv->connect) {
+ free(id_priv->connect);
+ id_priv->connect_len = 0;
+ }
+
return ucma_complete(id_priv);
}
@@ -2038,6 +2052,7 @@ int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
{
struct rdma_cm_id *cm_id;
+ struct cma_id_private *id_priv;
int ret;
ret = rdma_create_id2(NULL, &cm_id, NULL, res->ai_port_space, res->ai_qp_type);
@@ -2075,6 +2090,17 @@ int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
if (ret)
goto err;
+ if (res->ai_connect_len) {
+ id_priv = container_of(cm_id, struct cma_id_private, id);
+ id_priv->connect = malloc(res->ai_connect_len);
+ if (!id_priv->connect) {
+ ret = ERR(ENOMEM);
+ goto err;
+ }
+ memcpy(id_priv->connect, res->ai_connect, res->ai_connect_len);
+ id_priv->connect_len = res->ai_connect_len;
+ }
+
out:
*id = cm_id;
return 0;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 33/37] librdmacm/rdma_server: add new sample server application
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Provide a simple server application to demonstrate the minimal
amount of coding needed to accept a connection request from
a client and exchange messages.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Makefile.am | 4 +
examples/rdma_server.c | 149 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 152 insertions(+), 1 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index 6071599..8701002 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -18,7 +18,7 @@ src_librdmacm_la_LDFLAGS = -version-info 1 -export-dynamic \
src_librdmacm_la_DEPENDENCIES = $(srcdir)/src/librdmacm.map
bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy examples/mckey \
- examples/rdma_client
+ examples/rdma_client examples/rdma_server
examples_ucmatose_SOURCES = examples/cmatose.c
examples_ucmatose_LDADD = $(top_builddir)/src/librdmacm.la
examples_rping_SOURCES = examples/rping.c
@@ -29,6 +29,8 @@ examples_mckey_SOURCES = examples/mckey.c
examples_mckey_LDADD = $(top_builddir)/src/librdmacm.la
examples_rdma_client_SOURCES = examples/rdma_client.c
examples_rdma_client_LDADD = $(top_builddir)/src/librdmacm.la
+examples_rdma_server_SOURCES = examples/rdma_server.c
+examples_rdma_server_LDADD = $(top_builddir)/src/librdmacm.la
librdmacmincludedir = $(includedir)/rdma $(includedir)/infiniband
diff --git a/examples/rdma_server.c b/examples/rdma_server.c
new file mode 100644
index 0000000..2831d0c
--- /dev/null
+++ b/examples/rdma_server.c
@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2005-2009 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under the OpenIB.org BSD license
+ * below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <getopt.h>
+#include <netdb.h>
+#include <rdma/rdma_cma.h>
+#include <rdma/rdma_verbs.h>
+
+static char *port = "7471";
+
+struct rdma_cm_id *listen_id, *id;
+struct ibv_mr *mr;
+uint8_t send_msg[16];
+uint8_t recv_msg[16];
+
+static int run(void)
+{
+ struct rdma_addrinfo hints, *res;
+ struct ibv_qp_init_attr attr;
+ struct ibv_wc wc;
+ int ret;
+
+ memset(&hints, 0, sizeof hints);
+ hints.ai_flags = RAI_PASSIVE;
+ hints.ai_port_space = RDMA_PS_TCP;
+ ret = rdma_getaddrinfo(NULL, port, &hints, &res);
+ if (ret) {
+ printf("rdma_getaddrinfo %d\n", errno);
+ return ret;
+ }
+
+ memset(&attr, 0, sizeof attr);
+ attr.cap.max_send_wr = attr.cap.max_recv_wr = 1;
+ attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
+ attr.cap.max_inline_data = 16;
+ attr.sq_sig_all = 1;
+ ret = rdma_create_ep(&listen_id, res, NULL, &attr);
+ rdma_freeaddrinfo(res);
+ if (ret) {
+ printf("rdma_create_ep %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_listen(listen_id, 0);
+ if (ret) {
+ printf("rdma_listen %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_get_request(listen_id, &id);
+ if (ret) {
+ printf("rdma_get_request %d\n", errno);
+ return ret;
+ }
+
+ mr = rdma_reg_msgs(id, recv_msg, 16);
+ if (!mr) {
+ printf("rdma_reg_msgs %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);
+ if (ret) {
+ printf("rdma_post_recv %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_accept(id, NULL);
+ if (ret) {
+ printf("rdma_connect %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_get_recv_comp(id, &wc);
+ if (ret <= 0) {
+ printf("rdma_get_recv_comp %d\n", ret);
+ return ret;
+ }
+
+ ret = rdma_post_send(id, NULL, send_msg, 16, NULL, IBV_SEND_INLINE);
+ if (ret) {
+ printf("rdma_post_send %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_get_send_comp(id, &wc);
+ if (ret <= 0) {
+ printf("rdma_get_send_comp %d\n", ret);
+ return ret;
+ }
+
+ rdma_disconnect(id);
+ rdma_dereg_mr(mr);
+ rdma_destroy_ep(id);
+ rdma_destroy_ep(listen_id);
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ int op, ret;
+
+ while ((op = getopt(argc, argv, "p:")) != -1) {
+ switch (op) {
+ case 'p':
+ port = optarg;
+ break;
+ default:
+ printf("usage: %s\n", argv[0]);
+ printf("\t[-p port_number]\n");
+ exit(1);
+ }
+ }
+
+ printf("rdma_server: start\n");
+ ret = run();
+ printf("rdma_server: end %d\n", ret);
+ return ret;
+}
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox