* One Million Pounds has been Awarded to in you in our BT PROMO.Send your
From: British Telecom @ 2010-05-20 6:28 UTC (permalink / raw)
To: info-AyqxTnQTQtw
Names...
Country...
Occupation...
Tel...
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* [PATCH/RFC] opensm: toggle sweeping V2
From: Arthur Kepner @ 2010-05-19 23:57 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Cc: sashak-smomgflXvOZWk0Htik3J/w, Dale.R.Talcott-NSQ8wuThN14
One of our customers recently merged some new systems into a
large, existing cluster. They requested a mechanism to prevent
opensm from sweeping while the new equipment was being added to
the IB fabric, and then resume sweeping once they felt confident
that the newly added (sub)fabric was correctly cabled, and fully
functional. They used something similar to the following patch.
Comments?
Signed-off-by: Arthur Kepner <akepner-sJ/iWh9BUns@public.gmane.org>
---
include/opensm/osm_subnet.h | 6 ++++++
opensm/osm_console.c | 32 ++++++++++++++++++++++++++++++++
opensm/osm_state_mgr.c | 8 +++++++-
opensm/osm_subnet.c | 1 +
opensm/osm_trap_rcv.c | 35 +++++++++++++++++++++--------------
5 files changed, 67 insertions(+), 15 deletions(-)
diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h
index d79ed8f..2a1db99 100644
--- a/opensm/include/opensm/osm_subnet.h
+++ b/opensm/include/opensm/osm_subnet.h
@@ -532,6 +532,7 @@ typedef struct osm_subn {
boolean_t in_sweep_hop_0;
boolean_t first_time_master_sweep;
boolean_t coming_out_of_standby;
+ boolean_t sweeping_enabled;
unsigned need_update;
cl_fmap_t mgrp_mgid_tbl;
void *mboxes[IB_LID_MCAST_END_HO - IB_LID_MCAST_START_HO + 1];
@@ -651,6 +652,11 @@ typedef struct osm_subn {
* The flag is set true if the SM state was standby and now
* changed to MASTER it is reset at the end of the sweep.
*
+* sweeping_enabled
+* FALSE - sweeping is administratively disabled, all
+* sweeping is inhibited, TRUE - sweeping is done
+* normally
+*
* need_update
* This flag should be on during first non-master heavy
* (including pre-master discovery stage)
diff --git a/opensm/opensm/osm_console.c b/opensm/opensm/osm_console.c
index 968486e..bc7bea3 100644
--- a/opensm/opensm/osm_console.c
+++ b/opensm/opensm/osm_console.c
@@ -150,6 +150,16 @@ static void help_reroute(FILE * out, int detail)
}
}
+static void help_sweep(FILE * out, int detail)
+{
+ fprintf(out, "sweep [on|off]\n");
+ if (detail) {
+ fprintf(out, "enable or disable sweeping\n");
+ fprintf(out, " [on] sweep normally\n");
+ fprintf(out, " [off] inhibit all sweeping\n");
+ }
+}
+
static void help_status(FILE * out, int detail)
{
fprintf(out, "status [loop]\n");
@@ -427,11 +437,15 @@ static void print_status(osm_opensm_t * p_osm, FILE * out)
p_osm->stats.sa_mads_ignored);
fprintf(out, "\n Subnet flags\n"
" ------------\n"
+ " Sweeping enabled : %d\n"
+ " Sweep interval (seconds) : %d\n"
" Ignore existing lfts : %d\n"
" Subnet Init errors : %d\n"
" In sweep hop 0 : %d\n"
" First time master sweep : %d\n"
" Coming out of standby : %d\n",
+ p_osm->subn.sweeping_enabled,
+ p_osm->subn.opt.sweep_interval,
p_osm->subn.ignore_existing_lfts,
p_osm->subn.subnet_initialization_error,
p_osm->subn.in_sweep_hop_0,
@@ -495,6 +509,23 @@ static void reroute_parse(char **p_last, osm_opensm_t * p_osm, FILE * out)
osm_opensm_sweep(p_osm);
}
+static void sweep_parse(char **p_last, osm_opensm_t * p_osm, FILE * out)
+{
+ char *p_cmd;
+
+ p_cmd = next_token(p_last);
+ if (!p_cmd ||
+ (strcmp(p_cmd, "on") != 0 && strcmp(p_cmd, "off") != 0)) {
+ fprintf(out, "Invalid sweep command\n");
+ help_sweep(out, 1);
+ } else {
+ if (strcmp(p_cmd, "on") == 0)
+ p_osm->subn.sweeping_enabled = TRUE;
+ else
+ p_osm->subn.sweeping_enabled = FALSE;
+ }
+}
+
static void logflush_parse(char **p_last, osm_opensm_t * p_osm, FILE * out)
{
fflush(p_osm->log.out_port);
@@ -1332,6 +1363,7 @@ static const struct command console_cmds[] = {
{"priority", &help_priority, &priority_parse},
{"resweep", &help_resweep, &resweep_parse},
{"reroute", &help_reroute, &reroute_parse},
+ {"sweep", &help_sweep, &sweep_parse},
{"status", &help_status, &status_parse},
{"logflush", &help_logflush, &logflush_parse},
{"querylid", &help_querylid, &querylid_parse},
diff --git a/opensm/opensm/osm_state_mgr.c b/opensm/opensm/osm_state_mgr.c
index e43463f..81c8f54 100644
--- a/opensm/opensm/osm_state_mgr.c
+++ b/opensm/opensm/osm_state_mgr.c
@@ -1415,7 +1415,13 @@ void osm_state_mgr_process(IN osm_sm_t * sm, IN osm_signal_t signal)
switch (signal) {
case OSM_SIGNAL_SWEEP:
- do_sweep(sm);
+ if (!sm->p_subn->sweeping_enabled) {
+ OSM_LOG(sm->p_log, OSM_LOG_DEBUG, "sweeping disabled - "
+ "ignoring signal %s in state %s\n",
+ osm_get_sm_signal_str(signal),
+ osm_get_sm_mgr_state_str(sm->p_subn->sm_state));
+ } else
+ do_sweep(sm);
break;
case OSM_SIGNAL_IDLE_TIME_PROCESS_REQUEST:
do_process_mgrp_queue(sm);
diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c
index ac8cb37..ba2c812 100644
--- a/opensm/opensm/osm_subnet.c
+++ b/opensm/opensm/osm_subnet.c
@@ -531,6 +531,7 @@ ib_api_status_t osm_subn_init(IN osm_subn_t * p_subn, IN osm_opensm_t * p_osm,
/* we assume master by default - so we only need to set it true if STANDBY */
p_subn->coming_out_of_standby = FALSE;
+ p_subn->sweeping_enabled = TRUE;
return IB_SUCCESS;
}
diff --git a/opensm/opensm/osm_trap_rcv.c b/opensm/opensm/osm_trap_rcv.c
index bf13239..ba366a9 100644
--- a/opensm/opensm/osm_trap_rcv.c
+++ b/opensm/opensm/osm_trap_rcv.c
@@ -515,23 +515,30 @@ static void trap_rcv_process_request(IN osm_sm_t * sm,
check_sweep:
/* do a sweep if we received a trap */
if (sm->p_subn->opt.sweep_on_trap) {
- /* if this is trap number 128 or run_heavy_sweep is TRUE -
- update the force_heavy_sweep flag of the subnet.
- Sweep also on traps 144 - these traps signal a change of
- certain port capabilities.
- TODO: In the future this can be changed to just getting
- PortInfo on this port instead of sweeping the entire subnet. */
- if (ib_notice_is_generic(p_ntci) &&
- (cl_ntoh16(p_ntci->g_or_v.generic.trap_num) == 128 ||
- cl_ntoh16(p_ntci->g_or_v.generic.trap_num) == 144 ||
- run_heavy_sweep)) {
- OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
- "Forcing heavy sweep. Received trap:%u\n",
+ if (!sm->p_subn->sweeping_enabled) {
+ OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
+ "sweeping disabled - ignoring trap %u\n",
cl_ntoh16(p_ntci->g_or_v.generic.trap_num));
+ } else {
+ /* if this is trap number 128 or run_heavy_sweep is
+ TRUE - update the force_heavy_sweep flag of the
+ subnet. Sweep also on traps 144 - these traps
+ signal a change of certain port capabilities.
+ TODO: In the future this can be changed to just
+ getting PortInfo on this port instead of sweeping
+ the entire subnet. */
+ if (ib_notice_is_generic(p_ntci) &&
+ (cl_ntoh16(p_ntci->g_or_v.generic.trap_num) == 128 ||
+ cl_ntoh16(p_ntci->g_or_v.generic.trap_num) == 144 ||
+ run_heavy_sweep)) {
+ OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
+ "Forcing heavy sweep. Received trap:%u\n",
+ cl_ntoh16(p_ntci->g_or_v.generic.trap_num));
- sm->p_subn->force_heavy_sweep = TRUE;
+ sm->p_subn->force_heavy_sweep = TRUE;
+ }
+ osm_sm_signal(sm, OSM_SIGNAL_SWEEP);
}
- osm_sm_signal(sm, OSM_SIGNAL_SWEEP);
}
/* If we reached here due to trap 129/130/131 - do not need to do
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH] dapl-2.0: dapltest: server info devicename is not large enough for dapl_name storage
From: Davis, Arlin R @ 2010-05-19 23:31 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
Server info device name is a 80 char array but the dapl device name
that is copied is 256 bytes. Increase started_server.devicename definition.
Chalk one up for windows SDK OACR (auto code review).
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
test/dapltest/include/dapl_server_info.h | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/test/dapltest/include/dapl_server_info.h b/test/dapltest/include/dapl_server_info.h
index de038c5..898f9cc 100644
--- a/test/dapltest/include/dapl_server_info.h
+++ b/test/dapltest/include/dapl_server_info.h
@@ -37,7 +37,7 @@
struct started_server
{
- char devicename[80];
+ char devicename[256];
struct started_server *next;
};
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH] dapl-2.0: cma, scm: add new provider entries for Mellanox RDMA over Ethernet device
From: Davis, Arlin R @ 2010-05-19 23:31 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
Add options for netdev eth2/eth3 for cma and for device mlx4_0 port 1/2 for scm.
ofa-v2-cma-roe-eth2 u2.0 nonthreadsafe default libdaplofa.so.2 dapl.2.0 "eth2 0" ""
ofa-v2-cma-roe-eth3 u2.0 nonthreadsafe default libdaplofa.so.2 dapl.2.0 "eth3 0" ""
ofa-v2-scm-roe-mlx4_0-1 u2.0 nonthreadsafe default libdaploscm.so.2 dapl.2.0 "mlx4_0 1" ""
ofa-v2-scm-roe-mlx4_0-2 u2.0 nonthreadsafe default libdaploscm.so.2 dapl.2.0 "mlx4_0 2" ""
Signed-off-by: Arlin Davis <arlin.r.davis-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
diff --git a/Makefile.am b/Makefile.am
index bf82853..bef95ec 100755
--- a/Makefile.am
+++ b/Makefile.am
@@ -562,7 +562,11 @@ install-exec-hook:
echo ofa-v2-mlx4_0-1u u2.0 nonthreadsafe default libdaploucm.so.2 dapl.2.0 '"mlx4_0 1" ""' >> $(DESTDIR)$(sysconfdir)/dat.conf; \
echo ofa-v2-mlx4_0-2u u2.0 nonthreadsafe default libdaploucm.so.2 dapl.2.0 '"mlx4_0 2" ""' >> $(DESTDIR)$(sysconfdir)/dat.conf; \
echo ofa-v2-mthca0-1u u2.0 nonthreadsafe default libdaploucm.so.2 dapl.2.0 '"mthca0 1" ""' >> $(DESTDIR)$(sysconfdir)/dat.conf; \
- echo ofa-v2-mthca0-2u u2.0 nonthreadsafe default libdaploucm.so.2 dapl.2.0 '"mthca0 2" ""' >> $(DESTDIR)$(sysconfdir)/dat.conf;
+ echo ofa-v2-mthca0-2u u2.0 nonthreadsafe default libdaploucm.so.2 dapl.2.0 '"mthca0 2" ""' >> $(DESTDIR)$(sysconfdir)/dat.conf; \
+ echo ofa-v2-cma-roe-eth2 u2.0 nonthreadsafe default libdaplofa.so.2 dapl.2.0 '"eth2 0" ""' >> $(DESTDIR)$(sysconfdir)/dat.conf; \
+ echo ofa-v2-cma-roe-eth3 u2.0 nonthreadsafe default libdaplofa.so.2 dapl.2.0 '"eth3 0" ""' >> $(DESTDIR)$(sysconfdir)/dat.conf; \
+ echo ofa-v2-scm-roe-mlx4_0-1 u2.0 nonthreadsafe default libdaploscm.so.2 dapl.2.0 '"mlx4_0 1" ""' >> $(DESTDIR)$(sysconfdir)/dat.conf; \
+ echo ofa-v2-scm-roe-mlx4_0-2 u2.0 nonthreadsafe default libdaploscm.so.2 dapl.2.0 '"mlx4_0 2" ""' >> $(DESTDIR)$(sysconfdir)/dat.conf;
uninstall-hook:
if test -e $(DESTDIR)$(sysconfdir)/dat.conf; then \
diff --git a/dapl.spec.in b/dapl.spec.in
index 0f2c380..45187db 100644
--- a/dapl.spec.in
+++ b/dapl.spec.in
@@ -110,6 +110,10 @@ echo ofa-v2-mlx4_0-1u u2.0 nonthreadsafe default libdaploucm.so.2 dapl.2.0 '"mlx
echo ofa-v2-mlx4_0-2u u2.0 nonthreadsafe default libdaploucm.so.2 dapl.2.0 '"mlx4_0 2" ""' >> %{_sysconfdir}/dat.conf
echo ofa-v2-mthca0-1u u2.0 nonthreadsafe default libdaploucm.so.2 dapl.2.0 '"mthca0 1" ""' >> %{_sysconfdir}/dat.conf
echo ofa-v2-mthca0-2u u2.0 nonthreadsafe default libdaploucm.so.2 dapl.2.0 '"mthca0 2" ""' >> %{_sysconfdir}/dat.conf
+echo ofa-v2-cma-roe-eth2 u2.0 nonthreadsafe default libdaplofa.so.2 dapl.2.0 '"eth2 0" ""' >> %{_sysconfdir}/dat.conf
+echo ofa-v2-cma-roe-eth3 u2.0 nonthreadsafe default libdaplofa.so.2 dapl.2.0 '"eth3 0" ""' >> %{_sysconfdir}/dat.conf
+echo ofa-v2-scm-roe-mlx4_0-1 u2.0 nonthreadsafe default libdaploscm.so.2 dapl.2.0 '"mlx4_0 1" ""' >> %{_sysconfdir}/dat.conf
+echo ofa-v2-scm-roe-mlx4_0-2 u2.0 nonthreadsafe default libdaploscm.so.2 dapl.2.0 '"mlx4_0 2" ""' >> %{_sysconfdir}/dat.conf
%postun
/sbin/ldconfig
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH] dapl-2.0: windows: comp_channel.cpp is included by util.c in the openib_common.
From: Davis, Arlin R @ 2010-05-19 23:27 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
Remove it from device.c in individual providers to avoid
duplicate definitions.
Line endings were corrected to linux format from windows as part of
the change.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
dapl/openib_cma/device.c | 75 ++++++++++++++++++++++-----------------------
dapl/openib_scm/device.c | 1 -
dapl/openib_ucm/device.c | 5 +--
3 files changed, 39 insertions(+), 42 deletions(-)
diff --git a/dapl/openib_cma/device.c b/dapl/openib_cma/device.c
index 99b8c55..e4ff22e 100644
--- a/dapl/openib_cma/device.c
+++ b/dapl/openib_cma/device.c
@@ -54,7 +54,6 @@ DAPL_OS_LOCK g_hca_lock;
struct dapl_llist_entry *g_hca_list;
#if defined(_WIN64) || defined(_WIN32)
-#include "..\..\..\..\..\etc\user\comp_channel.cpp"
#include <rdma\winverbs.h>
static COMP_SET ufds;
@@ -144,43 +143,43 @@ static int dapls_thread_signal(void)
}
#endif
-/* Get IP address using network name, address, or device name */
-static int getipaddr(char *name, char *addr, int len)
-{
- struct addrinfo *res;
-
- /* assume netdev for first attempt, then network and address type */
- if (getipaddr_netdev(name, addr, len)) {
- if (getaddrinfo(name, NULL, NULL, &res)) {
- dapl_log(DAPL_DBG_TYPE_ERR,
- " open_hca: getaddr_netdev ERROR:"
- " %s. Is %s configured?\n",
- strerror(errno), name);
- return 1;
- } else {
- if (len >= res->ai_addrlen)
- memcpy(addr, res->ai_addr, res->ai_addrlen);
- else {
- freeaddrinfo(res);
- return 1;
- }
- freeaddrinfo(res);
- }
- }
-
- dapl_dbg_log(
- DAPL_DBG_TYPE_UTIL,
- " getipaddr: family %d port %d addr %d.%d.%d.%d\n",
- ((struct sockaddr_in *)addr)->sin_family,
- ((struct sockaddr_in *)addr)->sin_port,
- ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 0 & 0xff,
- ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 8 & 0xff,
- ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 16 & 0xff,
- ((struct sockaddr_in *)addr)->sin_addr.
- s_addr >> 24 & 0xff);
-
- return 0;
-}
+/* Get IP address using network name, address, or device name */
+static int getipaddr(char *name, char *addr, int len)
+{
+ struct addrinfo *res;
+
+ /* assume netdev for first attempt, then network and address type */
+ if (getipaddr_netdev(name, addr, len)) {
+ if (getaddrinfo(name, NULL, NULL, &res)) {
+ dapl_log(DAPL_DBG_TYPE_ERR,
+ " open_hca: getaddr_netdev ERROR:"
+ " %s. Is %s configured?\n",
+ strerror(errno), name);
+ return 1;
+ } else {
+ if (len >= res->ai_addrlen)
+ memcpy(addr, res->ai_addr, res->ai_addrlen);
+ else {
+ freeaddrinfo(res);
+ return 1;
+ }
+ freeaddrinfo(res);
+ }
+ }
+
+ dapl_dbg_log(
+ DAPL_DBG_TYPE_UTIL,
+ " getipaddr: family %d port %d addr %d.%d.%d.%d\n",
+ ((struct sockaddr_in *)addr)->sin_family,
+ ((struct sockaddr_in *)addr)->sin_port,
+ ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 0 & 0xff,
+ ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 8 & 0xff,
+ ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 16 & 0xff,
+ ((struct sockaddr_in *)addr)->sin_addr.
+ s_addr >> 24 & 0xff);
+
+ return 0;
+}
/*
* dapls_ib_init, dapls_ib_release
diff --git a/dapl/openib_scm/device.c b/dapl/openib_scm/device.c
index a5b0742..4c50f03 100644
--- a/dapl/openib_scm/device.c
+++ b/dapl/openib_scm/device.c
@@ -67,7 +67,6 @@ DAT_RETURN dapli_ib_thread_init(void);
void dapli_ib_thread_destroy(void);
#if defined(_WIN64) || defined(_WIN32)
-#include "..\..\..\..\..\etc\user\comp_channel.cpp"
#include <rdma\winverbs.h>
static COMP_SET ufds;
diff --git a/dapl/openib_ucm/device.c b/dapl/openib_ucm/device.c
index 1f324b3..1959c76 100644
--- a/dapl/openib_ucm/device.c
+++ b/dapl/openib_ucm/device.c
@@ -37,17 +37,16 @@ static void ucm_service_destroy(IN DAPL_HCA *hca);
static int ucm_service_create(IN DAPL_HCA *hca);
#if defined (_WIN32)
-#include "..\..\..\..\..\etc\user\comp_channel.cpp"
#include <rdma\winverbs.h>
static int32_t create_os_signal(IN DAPL_HCA * hca_ptr)
{
- return CompSetInit(&hca_ptr->ib_trans.signal.set);
+ return CompSetInit(&hca_ptr->ib_trans.signal.set);
}
static void destroy_os_signal(IN DAPL_HCA * hca_ptr)
{
- CompSetCleanup(&hca_ptr->ib_trans.signal.set);
+ CompSetCleanup(&hca_ptr->ib_trans.signal.set);
}
static int dapls_config_verbs(struct ibv_context *verbs)
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH] dapl-2.0: windows: need to include linux directory to pick up _errno.h
From: Davis, Arlin R @ 2010-05-19 23:27 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
dapl/openib_cma/SOURCES | 2 +-
dapl/openib_scm/SOURCES | 2 +-
dapl/openib_ucm/SOURCES | 3 ++-
3 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/dapl/openib_cma/SOURCES b/dapl/openib_cma/SOURCES
index d6b97a2..0c3764b 100644
--- a/dapl/openib_cma/SOURCES
+++ b/dapl/openib_cma/SOURCES
@@ -25,7 +25,7 @@ SOURCES = \
cm.c
INCLUDES = ..\include;..\openib_common;..\common;windows;..\..\dat\include;\
- ..\..\dat\udat\windows;..\udapl\windows;\
+ ..\..\dat\udat\windows;..\udapl\windows;..\..\..\..\inc\user\linux;\
..\..\..\..\inc;..\..\..\..\inc\user;..\..\..\libibverbs\include;\
..\..\..\librdmacm\include
diff --git a/dapl/openib_scm/SOURCES b/dapl/openib_scm/SOURCES
index 6e4ad30..2129e27 100644
--- a/dapl/openib_scm/SOURCES
+++ b/dapl/openib_scm/SOURCES
@@ -25,7 +25,7 @@ SOURCES = \
cm.c
INCLUDES = ..\include;..\openib_common\;..\common;windows;..\..\dat\include;\
- ..\..\dat\udat\windows;..\udapl\windows;\
+ ..\..\dat\udat\windows;..\udapl\windows;..\..\..\..\inc\user\linux;\
..\..\..\..\inc;..\..\..\..\inc\user;..\..\..\libibverbs\include
DAPL_OPTS = -DEXPORT_DAPL_SYMBOLS -DDAT_EXTENSIONS -DSOCK_CM -DOPENIB -DCQ_WAIT_OBJECT
diff --git a/dapl/openib_ucm/SOURCES b/dapl/openib_ucm/SOURCES
index 7eecf48..a2b5dce 100644
--- a/dapl/openib_ucm/SOURCES
+++ b/dapl/openib_ucm/SOURCES
@@ -21,7 +21,8 @@ SOURCES = udapl.rc ..\dapl_common_src.c ..\dapl_udapl_src.c ..\openib_common.c \
INCLUDES = ..\include;..\openib_common\;..\common;windows;..\..\dat\include;\
..\..\dat\udat\windows;..\udapl\windows;\
- ..\..\..\..\inc;..\..\..\..\inc\user;..\..\..\libibverbs\include
+ ..\..\..\..\inc;..\..\..\..\inc\user;..\..\..\libibverbs\include;\
+ ..\..\..\..\inc\user\linux;
DAPL_OPTS = -DEXPORT_DAPL_SYMBOLS -DDAT_EXTENSIONS -DOPENIB -DCQ_WAIT_OBJECT
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* Re: [PATCH] RDMA/ucma: Copy iWARP route information.
From: Steve Wise @ 2010-05-19 22:09 UTC (permalink / raw)
To: Roland Dreier; +Cc: Sean Hefty, linux-rdma-u79uwXL29TY76Z2rM5mHXA
In-Reply-To: <adaeih7eegq.fsf-BjVyx320WGW9gfZ95n9DRSW4+XlvGpQz@public.gmane.org>
Roland Dreier wrote:
> > Roland/Sean, is this ok for 2.6.35?
>
> I guess it's fine. What does it give us by itself though?
>
>
Piece of mind. :)
IMO it is a bug fix.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: [PATCH] RDMA/ucma: Copy iWARP route information.
From: Roland Dreier @ 2010-05-19 22:03 UTC (permalink / raw)
To: Steve Wise; +Cc: Sean Hefty, linux-rdma-u79uwXL29TY76Z2rM5mHXA
In-Reply-To: <4BF45F1A.4010707-7bPotxP6k4+P2YhJcF5u+vpXobYPEAuW@public.gmane.org>
> Roland/Sean, is this ok for 2.6.35?
I guess it's fine. What does it give us by itself though?
- R.
--
Roland Dreier <rolandd-FYB4Gu1CFyUAvxtiuMwx3w@public.gmane.org> || For corporate legal information go to:
http://www.cisco.com/web/about/doing_business/legal/cri/index.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: [PATCH] RDMA/ucma: Copy iWARP route information.
From: Steve Wise @ 2010-05-19 21:58 UTC (permalink / raw)
To: Roland Dreier, Sean Hefty; +Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA
In-Reply-To: <20100510154656.22415.47740.stgit-T4OLL4TyM9aNDNWfRnPdfg@public.gmane.org>
Roland/Sean, is this ok for 2.6.35?
Steve Wise wrote:
> For iWARP rdma_cm ids, the "route" information is the L2 src and
> next hop addresses.
>
> Signed-off-by: Steve Wise <swise-7bPotxP6k4+P2YhJcF5u+vpXobYPEAuW@public.gmane.org>
> ---
>
> drivers/infiniband/core/ucma.c | 13 +++++++++++++
> 1 files changed, 13 insertions(+), 0 deletions(-)
>
> diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c
> index ac7edc2..0498383 100644
> --- a/drivers/infiniband/core/ucma.c
> +++ b/drivers/infiniband/core/ucma.c
> @@ -583,6 +583,16 @@ static void ucma_copy_ib_route(struct rdma_ucm_query_route_resp *resp,
> }
> }
>
> +static void ucma_copy_iw_route(struct rdma_ucm_query_route_resp *resp,
> + struct rdma_route *route)
> +{
> + struct rdma_dev_addr *dev_addr;
> +
> + dev_addr = &route->addr.dev_addr;
> + rdma_addr_get_dgid(dev_addr, (union ib_gid *) &resp->ib_route[0].dgid);
> + rdma_addr_get_sgid(dev_addr, (union ib_gid *) &resp->ib_route[0].sgid);
> +}
> +
> static ssize_t ucma_query_route(struct ucma_file *file,
> const char __user *inbuf,
> int in_len, int out_len)
> @@ -621,6 +631,9 @@ static ssize_t ucma_query_route(struct ucma_file *file,
> case RDMA_TRANSPORT_IB:
> ucma_copy_ib_route(&resp, &ctx->cm_id->route);
> break;
> + case RDMA_TRANSPORT_IWARP:
> + ucma_copy_iw_route(&resp, &ctx->cm_id->route);
> + break;
> default:
> break;
> }
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* RE: [PATCH v2] libibverbs: add path record definitions to sa.h
From: Sean Hefty @ 2010-05-19 21:31 UTC (permalink / raw)
To: 'Steve Wise', Walukiewicz, Miroslaw; +Cc: Roland Dreier, linux-rdma
In-Reply-To: <4BF4565B.4040208-7bPotxP6k4+P2YhJcF5u+vpXobYPEAuW@public.gmane.org>
>Also, There is a rdmacm kernel change to pass up iwarp L2 addresses once
>a cm_id has resolved the addresses. I posted it earlier and I think
>Sean is going to integrate it for 2.6.36.
I believe that the patch you posted earlier is sufficient for 2.6.35. I just
need to update my patch set for AF_IB support, which were written assuming that
iWarp devices did not return L2 addresses.
- Sean
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: [PATCH v2] libibverbs: add path record definitions to sa.h
From: Steve Wise @ 2010-05-19 21:21 UTC (permalink / raw)
To: Walukiewicz, Miroslaw; +Cc: Roland Dreier, Hefty, Sean, linux-rdma
In-Reply-To: <4BF40B09.6070809-7bPotxP6k4+P2YhJcF5u+vpXobYPEAuW@public.gmane.org>
Steve Wise wrote:
> Walukiewicz, Miroslaw wrote:
>> Hello Steve,
>> Do you plan some changes in the core code related to RAW_QPT?
>>
>
>
> The only changes I see needed to the kernel core is the mcast change
> you already proposed to allow mcast attach/detach for RAW_ETY qps...
>
Also, There is a rdmacm kernel change to pass up iwarp L2 addresses once
a cm_id has resolved the addresses. I posted it earlier and I think
Sean is going to integrate it for 2.6.36.
Steve.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* [PATCH 9/9] dapl-2.0: scm: check for hca object before signaling thread
From: Davis, Arlin R @ 2010-05-19 18:25 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
There may not be an hca object attached to cm object
when freeing during cleanup.
Signed-off-by: Arlin Davis <arlin.r.davis-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
dapl/openib_scm/cm.c | 3 ++-
1 files changed, 2 insertions(+), 1 deletions(-)
diff --git a/dapl/openib_scm/cm.c b/dapl/openib_scm/cm.c
index 975ffd5..ce0d961 100644
--- a/dapl/openib_scm/cm.c
+++ b/dapl/openib_scm/cm.c
@@ -322,7 +322,8 @@ static int dapl_select(struct dapl_fd_set *set)
static void dapli_cm_thread_signal(dp_ib_cm_handle_t cm_ptr)
{
- send(cm_ptr->hca->ib_trans.scm[1], "w", sizeof "w", 0);
+ if (cm_ptr->hca)
+ send(cm_ptr->hca->ib_trans.scm[1], "w", sizeof "w", 0);
}
static void dapli_cm_free(dp_ib_cm_handle_t cm_ptr)
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 8/9] dapl-2.0: scm, cma: fini code can be called multiple times and hang via fork
From: Davis, Arlin R @ 2010-05-19 18:25 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
The providers should protect against forked child exits and
not cleanup until the parent init actually exits. Otherwise,
the child will hang trying to cleanup dapl thread. Modify to
check process id for proper init to fini cleanup and limit
cleanup to parent only.
Signed-off-by: Arlin Davis <arlin.r.davis-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
dapl/openib_cma/device.c | 8 ++++++--
dapl/openib_scm/device.c | 7 +++++++
2 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/dapl/openib_cma/device.c b/dapl/openib_cma/device.c
index c9fc8c3..99b8c55 100644
--- a/dapl/openib_cma/device.c
+++ b/dapl/openib_cma/device.c
@@ -197,9 +197,10 @@ static int getipaddr(char *name, char *addr, int len)
* 0 success, -1 error
*
*/
+DAT_UINT32 g_parent = 0;
int32_t dapls_ib_init(void)
{
- dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " dapl_ib_init: \n");
+ g_parent = dapl_os_getpid();
/* initialize hca_list lock */
dapl_os_lock_init(&g_hca_lock);
@@ -215,7 +216,10 @@ int32_t dapls_ib_init(void)
int32_t dapls_ib_release(void)
{
- dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " dapl_ib_release: \n");
+ /* only parent will cleanup */
+ if (dapl_os_getpid() != g_parent)
+ return 0;
+
dapli_ib_thread_destroy();
if (g_cm_events != NULL)
rdma_destroy_event_channel(g_cm_events);
diff --git a/dapl/openib_scm/device.c b/dapl/openib_scm/device.c
index 03d38a6..a5b0742 100644
--- a/dapl/openib_scm/device.c
+++ b/dapl/openib_scm/device.c
@@ -216,8 +216,11 @@ static void destroy_cr_pipe(IN DAPL_HCA * hca_ptr)
* 0 success, -1 error
*
*/
+DAT_UINT32 g_parent = 0;
int32_t dapls_ib_init(void)
{
+ g_parent = dapl_os_getpid();
+
/* initialize hca_list */
dapl_os_lock_init(&g_hca_lock);
dapl_llist_init_head(&g_hca_list);
@@ -230,6 +233,10 @@ int32_t dapls_ib_init(void)
int32_t dapls_ib_release(void)
{
+ /* only parent init will cleanup */
+ if (dapl_os_getpid() != g_parent)
+ return 0;
+
dapli_ib_thread_destroy();
dapls_os_release();
return 0;
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 7/9] dapl-2.0: scm: add option to use other network devices with environment variable DAPL_SCM_NETDEV
From: Davis, Arlin R @ 2010-05-19 18:25 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
New environment variable can be used to set the netdev
for sockets to use instead of the default network device
returned using gethostname.
Signed-off-by: Arlin Davis <arlin.r.davis-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
dapl/openib_cma/device.c | 152 +++++++++--------------------------
dapl/openib_common/dapl_ib_common.h | 3 +-
dapl/openib_common/util.c | 101 +++++++++++++++++++++++-
dapl/openib_scm/device.c | 2 +-
4 files changed, 141 insertions(+), 117 deletions(-)
diff --git a/dapl/openib_cma/device.c b/dapl/openib_cma/device.c
index 12593cf..c9fc8c3 100644
--- a/dapl/openib_cma/device.c
+++ b/dapl/openib_cma/device.c
@@ -59,46 +59,6 @@ struct dapl_llist_entry *g_hca_list;
static COMP_SET ufds;
-static int getipaddr_netdev(char *name, char *addr, int addr_len)
-{
- IWVProvider *prov;
- WV_DEVICE_ADDRESS devaddr;
- struct addrinfo *res, *ai;
- HRESULT hr;
- int index;
-
- if (strncmp(name, "rdma_dev", 8)) {
- return EINVAL;
- }
-
- index = atoi(name + 8);
-
- hr = WvGetObject(&IID_IWVProvider, (LPVOID *) &prov);
- if (FAILED(hr)) {
- return hr;
- }
-
- hr = getaddrinfo("..localmachine", NULL, NULL, &res);
- if (hr) {
- goto release;
- }
-
- for (ai = res; ai; ai = ai->ai_next) {
- hr = prov->lpVtbl->TranslateAddress(prov, ai->ai_addr, &devaddr);
- if (SUCCEEDED(hr) && (ai->ai_addrlen <= addr_len) && (index-- == 0)) {
- memcpy(addr, ai->ai_addr, ai->ai_addrlen);
- goto free;
- }
- }
- hr = ENODEV;
-
-free:
- freeaddrinfo(res);
-release:
- prov->lpVtbl->Release(prov);
- return hr;
-}
-
static int dapls_os_init(void)
{
return CompSetInit(&ufds);
@@ -133,6 +93,7 @@ static int dapls_thread_signal(void)
return 0;
}
#else // _WIN64 || WIN32
+
int g_ib_pipe[2];
static int dapls_os_init(void)
@@ -146,43 +107,6 @@ static void dapls_os_release(void)
/* close pipe? */
}
-/* Get IP address using network device name */
-static int getipaddr_netdev(char *name, char *addr, int addr_len)
-{
- struct ifreq ifr;
- int skfd, ret, len;
-
- /* Fill in the structure */
- snprintf(ifr.ifr_name, IFNAMSIZ, "%s", name);
- ifr.ifr_hwaddr.sa_family = ARPHRD_INFINIBAND;
-
- /* Create a socket fd */
- skfd = socket(PF_INET, SOCK_STREAM, 0);
- ret = ioctl(skfd, SIOCGIFADDR, &ifr);
- if (ret)
- goto bail;
-
- switch (ifr.ifr_addr.sa_family) {
-#ifdef AF_INET6
- case AF_INET6:
- len = sizeof(struct sockaddr_in6);
- break;
-#endif
- case AF_INET:
- default:
- len = sizeof(struct sockaddr);
- break;
- }
-
- if (len <= addr_len)
- memcpy(addr, &ifr.ifr_addr, len);
- else
- ret = EINVAL;
-
- bail:
- close(skfd);
- return ret;
-}
static int dapls_config_fd(int fd)
{
@@ -220,43 +144,43 @@ static int dapls_thread_signal(void)
}
#endif
-/* Get IP address using network name, address, or device name */
-static int getipaddr(char *name, char *addr, int len)
-{
- struct addrinfo *res;
-
- /* assume netdev for first attempt, then network and address type */
- if (getipaddr_netdev(name, addr, len)) {
- if (getaddrinfo(name, NULL, NULL, &res)) {
- dapl_log(DAPL_DBG_TYPE_ERR,
- " open_hca: getaddr_netdev ERROR:"
- " %s. Is %s configured?\n",
- strerror(errno), name);
- return 1;
- } else {
- if (len >= res->ai_addrlen)
- memcpy(addr, res->ai_addr, res->ai_addrlen);
- else {
- freeaddrinfo(res);
- return 1;
- }
- freeaddrinfo(res);
- }
- }
-
- dapl_dbg_log(
- DAPL_DBG_TYPE_UTIL,
- " getipaddr: family %d port %d addr %d.%d.%d.%d\n",
- ((struct sockaddr_in *)addr)->sin_family,
- ((struct sockaddr_in *)addr)->sin_port,
- ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 0 & 0xff,
- ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 8 & 0xff,
- ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 16 & 0xff,
- ((struct sockaddr_in *)addr)->sin_addr.
- s_addr >> 24 & 0xff);
-
- return 0;
-}
+/* Get IP address using network name, address, or device name */
+static int getipaddr(char *name, char *addr, int len)
+{
+ struct addrinfo *res;
+
+ /* assume netdev for first attempt, then network and address type */
+ if (getipaddr_netdev(name, addr, len)) {
+ if (getaddrinfo(name, NULL, NULL, &res)) {
+ dapl_log(DAPL_DBG_TYPE_ERR,
+ " open_hca: getaddr_netdev ERROR:"
+ " %s. Is %s configured?\n",
+ strerror(errno), name);
+ return 1;
+ } else {
+ if (len >= res->ai_addrlen)
+ memcpy(addr, res->ai_addr, res->ai_addrlen);
+ else {
+ freeaddrinfo(res);
+ return 1;
+ }
+ freeaddrinfo(res);
+ }
+ }
+
+ dapl_dbg_log(
+ DAPL_DBG_TYPE_UTIL,
+ " getipaddr: family %d port %d addr %d.%d.%d.%d\n",
+ ((struct sockaddr_in *)addr)->sin_family,
+ ((struct sockaddr_in *)addr)->sin_port,
+ ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 0 & 0xff,
+ ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 8 & 0xff,
+ ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 16 & 0xff,
+ ((struct sockaddr_in *)addr)->sin_addr.
+ s_addr >> 24 & 0xff);
+
+ return 0;
+}
/*
* dapls_ib_init, dapls_ib_release
diff --git a/dapl/openib_common/dapl_ib_common.h b/dapl/openib_common/dapl_ib_common.h
index cc416d2..d3cf2e0 100644
--- a/dapl/openib_common/dapl_ib_common.h
+++ b/dapl/openib_common/dapl_ib_common.h
@@ -299,7 +299,8 @@ int32_t dapls_ib_release(void);
/* util.c */
enum ibv_mtu dapl_ib_mtu(int mtu);
char *dapl_ib_mtu_str(enum ibv_mtu mtu);
-DAT_RETURN getlocalipaddr(DAT_SOCK_ADDR *addr, int addr_len);
+int getipaddr_netdev(char *name, char *addr, int addr_len);
+DAT_RETURN getlocalipaddr(char *addr, int addr_len);
/* qp.c */
DAT_RETURN dapls_modify_qp_ud(IN DAPL_HCA *hca, IN ib_qp_handle_t qp);
diff --git a/dapl/openib_common/util.c b/dapl/openib_common/util.c
index 704d85a..dd1454e 100644
--- a/dapl/openib_common/util.c
+++ b/dapl/openib_common/util.c
@@ -31,6 +31,91 @@
int g_dapl_loopback_connection = 0;
+#if defined(_WIN64) || defined(_WIN32)
+#include "..\..\..\..\..\etc\user\comp_channel.cpp"
+#include <rdma\winverbs.h>
+
+int getipaddr_netdev(char *name, char *addr, int addr_len)
+{
+ IWVProvider *prov;
+ WV_DEVICE_ADDRESS devaddr;
+ struct addrinfo *res, *ai;
+ HRESULT hr;
+ int index;
+
+ if (strncmp(name, "rdma_dev", 8)) {
+ return EINVAL;
+ }
+
+ index = atoi(name + 8);
+
+ hr = WvGetObject(&IID_IWVProvider, (LPVOID *) &prov);
+ if (FAILED(hr)) {
+ return hr;
+ }
+
+ hr = getaddrinfo("..localmachine", NULL, NULL, &res);
+ if (hr) {
+ goto release;
+ }
+
+ for (ai = res; ai; ai = ai->ai_next) {
+ hr = prov->lpVtbl->TranslateAddress(prov, ai->ai_addr, &devaddr);
+ if (SUCCEEDED(hr) && (ai->ai_addrlen <= addr_len) && (index-- == 0)) {
+ memcpy(addr, ai->ai_addr, ai->ai_addrlen);
+ goto free;
+ }
+ }
+ hr = ENODEV;
+
+free:
+ freeaddrinfo(res);
+release:
+ prov->lpVtbl->Release(prov);
+ return hr;
+}
+
+#else // _WIN64 || WIN32
+
+/* Get IP address using network device name */
+int getipaddr_netdev(char *name, char *addr, int addr_len)
+{
+ struct ifreq ifr;
+ int skfd, ret, len;
+
+ /* Fill in the structure */
+ snprintf(ifr.ifr_name, IFNAMSIZ, "%s", name);
+ ifr.ifr_hwaddr.sa_family = ARPHRD_INFINIBAND;
+
+ /* Create a socket fd */
+ skfd = socket(PF_INET, SOCK_STREAM, 0);
+ ret = ioctl(skfd, SIOCGIFADDR, &ifr);
+ if (ret)
+ goto bail;
+
+ switch (ifr.ifr_addr.sa_family) {
+#ifdef AF_INET6
+ case AF_INET6:
+ len = sizeof(struct sockaddr_in6);
+ break;
+#endif
+ case AF_INET:
+ default:
+ len = sizeof(struct sockaddr);
+ break;
+ }
+
+ if (len <= addr_len)
+ memcpy(addr, &ifr.ifr_addr, len);
+ else
+ ret = EINVAL;
+
+ bail:
+ close(skfd);
+ return ret;
+}
+#endif
+
enum ibv_mtu dapl_ib_mtu(int mtu)
{
switch (mtu) {
@@ -67,12 +152,26 @@ char *dapl_ib_mtu_str(enum ibv_mtu mtu)
}
}
-DAT_RETURN getlocalipaddr(DAT_SOCK_ADDR * addr, int addr_len)
+DAT_RETURN getlocalipaddr(char *addr, int addr_len)
{
struct sockaddr_in *sin;
struct addrinfo *res, hint, *ai;
int ret;
char hostname[256];
+ char *netdev = getenv("DAPL_SCM_NETDEV");
+
+ /* use provided netdev instead of default hostname */
+ if (netdev != NULL) {
+ ret = getipaddr_netdev(netdev, addr, addr_len);
+ if (ret) {
+ dapl_log(DAPL_DBG_TYPE_ERR,
+ " getlocalipaddr: DAPL_SCM_NETDEV provided %s"
+ " but not configured on system? ERR = %s\n",
+ netdev, strerror(ret));
+ return dapl_convert_errno(ret, "getlocalipaddr");
+ } else
+ return DAT_SUCCESS;
+ }
if (addr_len < sizeof(*sin)) {
return DAT_INTERNAL_ERROR;
diff --git a/dapl/openib_scm/device.c b/dapl/openib_scm/device.c
index eb87a85..03d38a6 100644
--- a/dapl/openib_scm/device.c
+++ b/dapl/openib_scm/device.c
@@ -263,7 +263,7 @@ DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name, IN DAPL_HCA * hca_ptr)
" open_hca: %s - %p\n", hca_name, hca_ptr);
/* get the IP address of the device */
- dat_status = getlocalipaddr((DAT_SOCK_ADDR *) &hca_ptr->hca_address,
+ dat_status = getlocalipaddr((char *)&hca_ptr->hca_address,
sizeof(DAT_SOCK_ADDR6));
if (dat_status != DAT_SUCCESS)
return dat_status;
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 6/9] dapl-2.0: scm: cr_thread occasionally segv's when disconnecting all-to-all MPI static connections
From: Davis, Arlin R @ 2010-05-19 18:25 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
Note: no valid calltrace for segv on cr_thread because
of state changing in switch statement from another
thread, jumped unknown location.
Program received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x41a65940 (LWP 1328)]
0x00002b2e7d9d5134 in ?? ()
Add cm object locking on all state change/checking. When
freeing CM object wakeup cr_thread to process
state change to CM_FREE.
Signed-off-by: Arlin Davis <arlin.r.davis-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
dapl/openib_scm/cm.c | 39 ++++++++++++++++++++++++++++++++-------
1 files changed, 32 insertions(+), 7 deletions(-)
diff --git a/dapl/openib_scm/cm.c b/dapl/openib_scm/cm.c
index 4c8d4a1..975ffd5 100644
--- a/dapl/openib_scm/cm.c
+++ b/dapl/openib_scm/cm.c
@@ -436,6 +436,7 @@ void dapls_cm_free(dp_ib_cm_handle_t cm_ptr)
dapl_os_lock(&cm_ptr->lock);
cm_ptr->state = DCM_FREE;
while (cm_ptr->ref_count != 1) {
+ dapli_cm_thread_signal(cm_ptr);
dapl_os_unlock(&cm_ptr->lock);
dapl_os_sleep_usec(10000);
dapl_os_lock(&cm_ptr->lock);
@@ -524,7 +525,9 @@ static void dapli_socket_connected(dp_ib_cm_handle_t cm_ptr, int err)
goto bail;
}
+ dapl_os_lock(&cm_ptr->lock);
cm_ptr->state = DCM_REP_PENDING;
+ dapl_os_unlock(&cm_ptr->lock);
/* send qp info and pdata to remote peer */
exp = sizeof(ib_cm_msg_t) - DCM_MAX_PDATA_SIZE;
@@ -836,7 +839,10 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr)
dapl_dbg_log(DAPL_DBG_TYPE_EP, " connect_rtu: send RTU\n");
/* complete handshake after final QP state change, Just ver+op */
+ dapl_os_lock(&cm_ptr->lock);
cm_ptr->state = DCM_CONNECTED;
+ dapl_os_unlock(&cm_ptr->lock);
+
cm_ptr->msg.op = ntohs(DCM_RTU);
if (send(cm_ptr->socket, (char *)&cm_ptr->msg, 4, 0) == -1) {
int err = dapl_socket_errno();
@@ -914,7 +920,10 @@ bail:
goto ud_bail;
#endif
/* close socket, and post error event */
+ dapl_os_lock(&cm_ptr->lock);
cm_ptr->state = DCM_REJECTED;
+ dapl_os_unlock(&cm_ptr->lock);
+
dapl_evd_connection_callback(NULL, event, cm_ptr->msg.p_data,
DCM_MAX_PDATA_SIZE, ep_ptr);
dapli_cm_free(cm_ptr);
@@ -1093,8 +1102,9 @@ static void dapli_socket_accept_data(ib_cm_srvc_handle_t acm_ptr)
}
p_data = acm_ptr->msg.p_data;
}
-
+ dapl_os_lock(&acm_ptr->lock);
acm_ptr->state = DCM_ACCEPTING_DATA;
+ dapl_os_unlock(&acm_ptr->lock);
dapl_dbg_log(DAPL_DBG_TYPE_CM,
" ACCEPT: DST %s %x lid=0x%x, qpn=0x%x, psz=%d\n",
@@ -1235,7 +1245,9 @@ dapli_socket_accept_usr(DAPL_EP * ep_ptr,
dapl_os_memcpy(local.resv, cm_ptr->msg.resv, 4);
#endif
cm_ptr->hca = ia_ptr->hca_ptr;
+ dapl_os_lock(&cm_ptr->lock);
cm_ptr->state = DCM_ACCEPTED;
+ dapl_os_unlock(&cm_ptr->lock);
/* Link CM to EP, already queued on work thread */
dapl_ep_link_cm(ep_ptr, cm_ptr);
@@ -1305,7 +1317,9 @@ static void dapli_socket_accept_rtu(dp_ib_cm_handle_t cm_ptr)
}
/* save state and reference to EP, queue for disc event */
+ dapl_os_lock(&cm_ptr->lock);
cm_ptr->state = DCM_CONNECTED;
+ dapl_os_unlock(&cm_ptr->lock);
/* final data exchange if remote QP state is good to go */
dapl_dbg_log(DAPL_DBG_TYPE_EP, " PASSIVE: connected!\n");
@@ -1368,7 +1382,10 @@ bail:
if (cm_ptr->msg.saddr.ib.qp_type == IBV_QPT_UD)
goto ud_bail;
#endif
+ dapl_os_lock(&cm_ptr->lock);
cm_ptr->state = DCM_REJECTED;
+ dapl_os_unlock(&cm_ptr->lock);
+
dapls_cr_callback(cm_ptr, event, NULL, 0, cm_ptr->sp);
dapli_cm_free(cm_ptr);
}
@@ -1759,47 +1776,55 @@ void cr_thread(void *arg)
cr->socket);
/* data on listen, qp exchange, and on disc req */
+ dapl_os_lock(&cr->lock);
if ((ret == DAPL_FD_READ) ||
(cr->state != DCM_CONN_PENDING && ret == DAPL_FD_ERROR)) {
if (cr->socket != DAPL_INVALID_SOCKET) {
switch (cr->state) {
case DCM_LISTEN:
+ dapl_os_unlock(&cr->lock);
dapli_socket_accept(cr);
- break;
+ break;
case DCM_ACCEPTING:
+ dapl_os_unlock(&cr->lock);
dapli_socket_accept_data(cr);
break;
case DCM_ACCEPTED:
+ dapl_os_unlock(&cr->lock);
dapli_socket_accept_rtu(cr);
break;
case DCM_REP_PENDING:
+ dapl_os_unlock(&cr->lock);
dapli_socket_connect_rtu(cr);
break;
case DCM_CONNECTED:
+ dapl_os_unlock(&cr->lock);
dapli_socket_disconnect(cr);
break;
default:
+ dapl_os_unlock(&cr->lock);
break;
}
- }
+ } else
+ dapl_os_unlock(&cr->lock);
+
/* ASYNC connections, writable, readable, error; check status */
} else if (ret == DAPL_FD_WRITE ||
(cr->state == DCM_CONN_PENDING &&
ret == DAPL_FD_ERROR)) {
-
- if (ret == DAPL_FD_ERROR)
- dapl_log(DAPL_DBG_TYPE_ERR, " CONN_PENDING - FD_ERROR\n");
opt = 0;
opt_len = sizeof(opt);
ret = getsockopt(cr->socket, SOL_SOCKET,
SO_ERROR, (char *)&opt,
&opt_len);
+ dapl_os_unlock(&cr->lock);
if (!ret && !opt)
dapli_socket_connected(cr, opt);
else
dapli_socket_connected(cr, opt ? opt : dapl_socket_errno());
- }
+ } else
+ dapl_os_unlock(&cr->lock);
dapls_cm_release(cr); /* release ref */
dapl_os_lock(&hca_ptr->ib_trans.lock);
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 5/9] dapl-2.0: scm: SOCKOPT ERR Connection timed out on large clusters
From: Davis, Arlin R @ 2010-05-19 18:25 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
Large scale all to all connections on +1500 cores
the listen backlog is reached and SYN's are dropped
which causes the connect to timeout. Retry connect
on timeout errors.
Signed-off-by: Arlin Davis <arlin.r.davis-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
dapl/openib_scm/cm.c | 24 ++++++++++++++++++++++--
1 files changed, 22 insertions(+), 2 deletions(-)
diff --git a/dapl/openib_scm/cm.c b/dapl/openib_scm/cm.c
index 7465190..4c8d4a1 100644
--- a/dapl/openib_scm/cm.c
+++ b/dapl/openib_scm/cm.c
@@ -60,6 +60,12 @@
#include "dapl_ep_util.h"
#include "dapl_osd.h"
+/* forward declarations */
+static DAT_RETURN
+dapli_socket_connect(DAPL_EP * ep_ptr,
+ DAT_IA_ADDRESS_PTR r_addr,
+ DAT_CONN_QUAL r_qual, DAT_COUNT p_size, DAT_PVOID p_data);
+
#ifdef DAPL_DBG
/* Check for EP linking to IA and proper connect state */
void dapli_ep_check(DAPL_EP *ep)
@@ -494,13 +500,27 @@ static void dapli_socket_connected(dp_ib_cm_handle_t cm_ptr, int err)
if (err) {
dapl_log(DAPL_DBG_TYPE_ERR,
- " CONN_PENDING: %s ERR %s -> %s %d\n",
+ " CONN_PENDING: %s ERR %s -> %s %d - %s\n",
err == -1 ? "POLL" : "SOCKOPT",
err == -1 ? strerror(dapl_socket_errno()) : strerror(err),
inet_ntoa(((struct sockaddr_in *)
&cm_ptr->addr)->sin_addr),
ntohs(((struct sockaddr_in *)
- &cm_ptr->addr)->sin_port));
+ &cm_ptr->addr)->sin_port),
+ err == ETIMEDOUT ? "RETRYING...":"ABORTING");
+
+ /* retry a timeout */
+ if (err == ETIMEDOUT) {
+ closesocket(cm_ptr->socket);
+ cm_ptr->socket = DAPL_INVALID_SOCKET;
+ dapli_socket_connect(cm_ptr->ep, (DAT_IA_ADDRESS_PTR)&cm_ptr->addr,
+ ntohs(((struct sockaddr_in *)&cm_ptr->addr)->sin_port) - 1000,
+ ntohs(cm_ptr->msg.p_size), &cm_ptr->msg.p_data);
+ dapl_ep_unlink_cm(cm_ptr->ep, cm_ptr);
+ dapli_cm_free(cm_ptr);
+ return;
+ }
+
goto bail;
}
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 4/9] dapl-2.0: ucm: UD mode, active side cm object released to soon, the RTU could be lost.
From: Davis, Arlin R @ 2010-05-19 18:25 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
Will see following message with DAPL_DBG_TYPE set for Errors & Warnings (0x3):
ucm_recv: NO MATCH op REP 0x120 65487 i0x60005e c0x60005e < 0xd2 19824 0x60006a
The cm object was released on the active side after the connection
was established, RTU sent. This is a problem if the RTU is lost
and the remote side retries the REPLY. The RTU is never resent.
Keep the cm object until the EP is destroyed.
Signed-off-by: Arlin Davis <arlin.r.davis-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
dapl/openib_ucm/cm.c | 8 --------
1 files changed, 0 insertions(+), 8 deletions(-)
diff --git a/dapl/openib_ucm/cm.c b/dapl/openib_ucm/cm.c
index c82147e..94af988 100644
--- a/dapl/openib_ucm/cm.c
+++ b/dapl/openib_ucm/cm.c
@@ -1143,10 +1143,6 @@ ud_bail:
(DAT_COUNT)ntohs(cm->msg.p_size),
(DAT_PVOID *)cm->msg.p_data,
(DAT_PVOID *)&xevent);
-
- /* release cm_ptr, EP refs will prevent destroy */
- dapli_cm_free(cm);
-
} else
#endif
{
@@ -1310,10 +1306,6 @@ static void ucm_accept_rtu(dp_ib_cm_handle_t cm, ib_cm_msg_t *msg)
(DAT_COUNT)ntohs(cm->msg.p_size),
(DAT_PVOID *)cm->msg.p_data,
(DAT_PVOID *)&xevent);
-
- /* done with CM object, EP ref will hold object for pdata */
- dapli_cm_free(cm);
-
} else {
#endif
dapls_cr_callback(cm, IB_CME_CONNECTED, NULL, 0, cm->sp);
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 3/9] dapl-2.0: cma, ucm: cleanup issues with dat_ep_free on a connected EP without disconnecting.
From: Davis, Arlin R @ 2010-05-19 18:25 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
During EP free, disconnecting with ABRUPT close flag, the disconnect should wait
for the DISC event to fire to allow the CM to be properly destroyed upon return.
The cma must also release the lock when calling the blocking rdma_destroy_id given
the callback thread could attempt to acquire the lock for reference counting.
Signed-off-by: Arlin Davis <arlin.r.davis-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
dapl/openib_cma/cm.c | 56 ++++++++++++++++++++++++++++++++++++-------------
dapl/openib_ucm/cm.c | 15 ++++++++++++-
2 files changed, 55 insertions(+), 16 deletions(-)
diff --git a/dapl/openib_cma/cm.c b/dapl/openib_cma/cm.c
index 1e846aa..503df96 100644
--- a/dapl/openib_cma/cm.c
+++ b/dapl/openib_cma/cm.c
@@ -209,18 +209,13 @@ void dapls_cm_acquire(dp_ib_cm_handle_t conn)
void dapls_cm_release(dp_ib_cm_handle_t conn)
{
dapl_os_lock(&conn->lock);
- conn->ref_count--;
- if (conn->ref_count) {
- dapl_os_unlock(&conn->lock);
- return;
- }
- if (conn->cm_id) {
- if (conn->cm_id->qp)
- rdma_destroy_qp(conn->cm_id);
- rdma_destroy_id(conn->cm_id);
- }
- dapl_os_unlock(&conn->lock);
- dapli_cm_dealloc(conn);
+ conn->ref_count--;
+ if (conn->ref_count) {
+ dapl_os_unlock(&conn->lock);
+ return;
+ }
+ dapl_os_unlock(&conn->lock);
+ dapli_cm_dealloc(conn);
}
/* BLOCKING: called from dapl_ep_free, EP link will be last ref */
@@ -235,10 +230,14 @@ void dapls_cm_free(dp_ib_cm_handle_t conn)
/* Destroy cm_id, wait until EP is last ref */
dapl_os_lock(&conn->lock);
if (conn->cm_id) {
- if (conn->cm_id->qp)
- rdma_destroy_qp(conn->cm_id);
- rdma_destroy_id(conn->cm_id);
+ struct rdma_cm_id *cm_id = conn->cm_id;
+
+ if (cm_id->qp)
+ rdma_destroy_qp(cm_id);
conn->cm_id = NULL;
+ dapl_os_unlock(&conn->lock);
+ rdma_destroy_id(cm_id); /* blocking, event processing */
+ dapl_os_lock(&conn->lock);
}
/* EP linking is last reference */
@@ -640,6 +639,17 @@ dapls_ib_disconnect(IN DAPL_EP * ep_ptr, IN DAT_CLOSE_FLAGS close_flags)
/* no graceful half-pipe disconnect option */
rdma_disconnect(conn->cm_id);
+ /* ABRUPT close, wait for callback and DISCONNECTED state */
+ if (close_flags == DAT_CLOSE_ABRUPT_FLAG) {
+ dapl_os_lock(&ep_ptr->header.lock);
+ while (ep_ptr->param.ep_state != DAT_EP_STATE_DISCONNECTED) {
+ dapl_os_unlock(&ep_ptr->header.lock);
+ dapl_os_sleep_usec(10000);
+ dapl_os_lock(&ep_ptr->header.lock);
+ }
+ dapl_os_unlock(&ep_ptr->header.lock);
+ }
+
/*
* DAT event notification occurs from the callback
* Note: will fire even if DREQ goes unanswered on timeout
@@ -759,6 +769,7 @@ dapls_ib_setup_conn_listener(IN DAPL_IA * ia_ptr,
return DAT_SUCCESS;
bail:
+ rdma_destroy_id(conn->cm_id);
dapls_cm_release(conn);
return dat_status;
}
@@ -791,8 +802,13 @@ dapls_ib_remove_conn_listener(IN DAPL_IA * ia_ptr, IN DAPL_SP * sp_ptr)
if (conn != IB_INVALID_HANDLE) {
sp_ptr->cm_srvc_handle = NULL;
+ if (conn->cm_id) {
+ rdma_destroy_id(conn->cm_id);
+ conn->cm_id = NULL;
+ }
dapls_cm_release(conn);
}
+
return DAT_SUCCESS;
}
@@ -869,6 +885,7 @@ dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle,
dapl_ep_unlink_cm(ep_ptr, ep_conn);
ep_conn->cm_id->qp = NULL;
ep_conn->ep = NULL;
+ rdma_destroy_id(ep_conn->cm_id);
dapls_cm_release(ep_conn);
/* add new CM to EP linking, qp_handle unchanged */
@@ -912,6 +929,7 @@ bail:
rdma_reject(cr_conn->cm_id, NULL, 0);
/* no EP linking, ok to destroy */
+ rdma_destroy_id(cr_conn->cm_id);
dapls_cm_release(cr_conn);
return dat_status;
}
@@ -974,6 +992,7 @@ dapls_ib_reject_connection(IN dp_ib_cm_handle_t cm_handle,
cm_handle->p_data, offset + private_data_size);
/* no EP linking, ok to destroy */
+ rdma_destroy_id(cm_handle->cm_id);
dapls_cm_release(cm_handle);
return dapl_convert_errno(ret, "reject");
}
@@ -1067,6 +1086,13 @@ void dapli_cma_event_cb(void)
dapls_cm_acquire(conn);
+ /* destroying cm_id, consumer thread blocking waiting for ACK */
+ if (conn->cm_id == NULL) {
+ dapls_cm_release(conn);
+ rdma_ack_cm_event(event);
+ return;
+ }
+
dapl_dbg_log(DAPL_DBG_TYPE_CM,
" cm_event: EVENT=%d ID=%p LID=%p CTX=%p\n",
event->event, event->id, event->listen_id, conn);
diff --git a/dapl/openib_ucm/cm.c b/dapl/openib_ucm/cm.c
index 5d5e7d2..c82147e 100644
--- a/dapl/openib_ucm/cm.c
+++ b/dapl/openib_ucm/cm.c
@@ -1586,7 +1586,20 @@ dapls_ib_disconnect(IN DAPL_EP *ep_ptr, IN DAT_CLOSE_FLAGS close_flags)
}
dapl_os_unlock(&ep_ptr->header.lock);
- return (dapli_cm_disconnect(cm_ptr));
+ dapli_cm_disconnect(cm_ptr);
+
+ /* ABRUPT close, wait for callback and DISCONNECTED state */
+ if (close_flags == DAT_CLOSE_ABRUPT_FLAG) {
+ dapl_os_lock(&ep_ptr->header.lock);
+ while (ep_ptr->param.ep_state != DAT_EP_STATE_DISCONNECTED) {
+ dapl_os_unlock(&ep_ptr->header.lock);
+ dapl_os_sleep_usec(10000);
+ dapl_os_lock(&ep_ptr->header.lock);
+ }
+ dapl_os_unlock(&ep_ptr->header.lock);
+ }
+
+ return DAT_SUCCESS;
}
/*
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 2/9] dapl-2.0: ucm: increase default UCM retry count for connect reply to 15
From: Davis, Arlin R @ 2010-05-19 18:25 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
on large clusters UCM is timing out with retries at 10.
Signed-off-by: Arlin Davis <arlin.r.davis-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
dapl/openib_common/dapl_ib_common.h | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/dapl/openib_common/dapl_ib_common.h b/dapl/openib_common/dapl_ib_common.h
index 3e32fab..cc416d2 100644
--- a/dapl/openib_common/dapl_ib_common.h
+++ b/dapl/openib_common/dapl_ib_common.h
@@ -166,7 +166,7 @@ typedef uint16_t ib_hca_port_t;
#define DCM_TCLASS 0
/* DAPL uCM timers, default queue sizes */
-#define DCM_RETRY_CNT 10
+#define DCM_RETRY_CNT 15
#define DCM_REP_TIME 800 /* reply timeout in m_secs */
#define DCM_RTU_TIME 400 /* rtu timeout in m_secs */
#define DCM_QP_SIZE 500 /* uCM tx, rx qp size */
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 1/9] dapl-2.0: scm: remove modify QP to ERR state during disconnect on UD type QP
From: Davis, Arlin R @ 2010-05-19 18:23 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
Patch set of bug fixes as a result of scale-out testing on 128 nodes/1538 cores.
1/9 scm: remove modify QP to ERR state during disconnect on UD type QP
2/9 ucm: increase default UCM retry count for connect reply to 15
3/9 cma, ucm: cleanup issues with dat_ep_free on a connected EP without disconnecting.
4/9 ucm: UD mode, active side cm object released to soon, the RTU could be lost.
5/9 scm: SOCKOPT ERR Connection timed out on large clusters
6/9 scm: cr_thread occasionally segv's when disconnecting all-to-all MPI static connections
7/9 scm: add option to use other network devices with environment variable DAPL_SCM_NETDEV
8/9 scm, cma: fini code can be called multiple times and hang via fork
9/9 scm: check for hca object before signaling thread
The disconnect on a UD type QP should not modify QP to error
since this is a shared QP. The disconnect should be treated
as a NOP on the UD type QP and only be transitioned during
the QP destroy (dat_ep_free).
Signed-off-by: Arlin Davis <arlin.r.davis-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
dapl/openib_scm/cm.c | 6 +++---
1 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/dapl/openib_scm/cm.c b/dapl/openib_scm/cm.c
index afd0d93..7465190 100644
--- a/dapl/openib_scm/cm.c
+++ b/dapl/openib_scm/cm.c
@@ -458,13 +458,13 @@ DAT_RETURN dapli_socket_disconnect(dp_ib_cm_handle_t cm_ptr)
dapl_os_unlock(&cm_ptr->lock);
/* send disc date, close socket, schedule destroy */
- dapl_os_lock(&cm_ptr->ep->header.lock);
- dapls_modify_qp_state(cm_ptr->ep->qp_handle, IBV_QPS_ERR, 0,0,0);
- dapl_os_unlock(&cm_ptr->ep->header.lock);
send(cm_ptr->socket, (char *)&disc_data, sizeof(disc_data), 0);
/* disconnect events for RC's only */
if (cm_ptr->ep->param.ep_attr.service_type == DAT_SERVICE_TYPE_RC) {
+ dapl_os_lock(&cm_ptr->ep->header.lock);
+ dapls_modify_qp_state(cm_ptr->ep->qp_handle, IBV_QPS_ERR, 0,0,0);
+ dapl_os_unlock(&cm_ptr->ep->header.lock);
if (cm_ptr->ep->cr_ptr) {
dapls_cr_callback(cm_ptr,
IB_CME_DISCONNECTED,
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 12/12] dapl-2.0: common: EP links to EVD, PZ incorrectly released before provider CM objects freed.
From: Davis, Arlin R @ 2010-05-19 18:11 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
unlink/clear references after ALL CM objects linked to EP are freed.
Otherwise, event processing via CM objects could reference the handles
still linked to EP. After CM objects are freed (blocking) these handles
linked to EP are guaranteed not to refereence from underlying provider.
Signed-off-by: Arlin Davis <arlin.r.davis-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
dapl/common/dapl_ep_free.c | 28 ++++++++++++++--------------
1 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/dapl/common/dapl_ep_free.c b/dapl/common/dapl_ep_free.c
index 8708e6f..3bfc541 100644
--- a/dapl/common/dapl_ep_free.c
+++ b/dapl/common/dapl_ep_free.c
@@ -110,6 +110,20 @@ DAT_RETURN DAT_API dapl_ep_free(IN DAT_EP_HANDLE ep_handle)
*/
(void)dapl_ep_disconnect(ep_ptr, DAT_CLOSE_ABRUPT_FLAG);
+ /* Free all CM objects */
+ cm_ptr = (dapl_llist_is_empty(&ep_ptr->cm_list_head)
+ ? NULL : dapl_llist_peek_head(&ep_ptr->cm_list_head));
+ while (cm_ptr != NULL) {
+ dapl_log(DAPL_DBG_TYPE_EP,
+ "dapl_ep_free: Free CM: EP=%p CM=%p\n",
+ ep_ptr, cm_ptr);
+
+ next_cm_ptr = dapl_llist_next_entry(&ep_ptr->cm_list_head,
+ &cm_ptr->list_entry);
+ dapls_cm_free(cm_ptr); /* blocking call */
+ cm_ptr = next_cm_ptr;
+ }
+
/*
* Do verification of parameters and the state change atomically.
*/
@@ -188,20 +202,6 @@ DAT_RETURN DAT_API dapl_ep_free(IN DAT_EP_HANDLE ep_handle)
}
}
- /* Free all CM objects */
- cm_ptr = (dapl_llist_is_empty(&ep_ptr->cm_list_head)
- ? NULL : dapl_llist_peek_head(&ep_ptr->cm_list_head));
- while (cm_ptr != NULL) {
- dapl_log(DAPL_DBG_TYPE_EP,
- "dapl_ep_free: Free CM: EP=%p CM=%p\n",
- ep_ptr, cm_ptr);
-
- next_cm_ptr = dapl_llist_next_entry(&ep_ptr->cm_list_head,
- &cm_ptr->list_entry);
- dapls_cm_free(cm_ptr); /* blocking call */
- cm_ptr = next_cm_ptr;
- }
-
/* Free the resource */
dapl_ep_dealloc(ep_ptr);
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 11/12] dapl-2.0: ucm: set timer during RTU_PENDING state change
From: Davis, Arlin R @ 2010-05-19 18:11 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
The timer thread may pick up an unitialized timer
value and timeout before the reply was sent.
Signed-off-by: Arlin Davis <arlin.r.davis-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
dapl/openib_ucm/cm.c | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)
diff --git a/dapl/openib_ucm/cm.c b/dapl/openib_ucm/cm.c
index 8fed8f6..5d5e7d2 100644
--- a/dapl/openib_ucm/cm.c
+++ b/dapl/openib_ucm/cm.c
@@ -1486,6 +1486,7 @@ dapli_accept_usr(DAPL_EP *ep, DAPL_CR *cr, DAT_COUNT p_size, DAT_PVOID p_data)
cm->hca = ia->hca_ptr;
dapl_os_lock(&cm->lock);
+ dapl_os_get_time(&cm->timer); /* RTU expected */
cm->state = DCM_RTU_PENDING;
dapl_os_unlock(&cm->lock);
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 10/12] dapl-2.0: ucm: fix issues with new EP to CM linking changes
From: Davis, Arlin R @ 2010-05-19 18:11 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
Add EP locking around QP modify
Remove release during disconnect event processing
Add check in cm_free to check state and schedule thread if necessary.
Add some additional debugging
Add processing in disconnect_clean for conn_req timeout
Remove extra CR's
Signed-off-by: Arlin Davis <arlin.r.davis-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
dapl/openib_ucm/cm.c | 107 ++++++++++++++++++++++++++++++++++++-------------
1 files changed, 78 insertions(+), 29 deletions(-)
diff --git a/dapl/openib_ucm/cm.c b/dapl/openib_ucm/cm.c
index 85c8b4b..8fed8f6 100644
--- a/dapl/openib_ucm/cm.c
+++ b/dapl/openib_ucm/cm.c
@@ -392,13 +392,13 @@ static void ucm_process_recv(ib_hca_transport_t *tp,
cm->msg.op = htons(DCM_DREP);
ucm_send(&cm->hca->ib_trans, &cm->msg, NULL, 0);
- }
- /* UD reply retried ok to ignore, any other print warning */
- if (ntohs(msg->op) != DCM_REP) {
+ } else if (ntohs(msg->op) != DCM_DREP){
+ /* DREP ok to ignore, any other print warning */
dapl_log(DAPL_DBG_TYPE_WARN,
- " ucm_recv: UNKNOWN operation"
- " <- op %d, %s spsp %d sqpn %d\n",
- ntohs(msg->op), dapl_cm_state_str(cm->state),
+ " ucm_recv: UNEXPECTED MSG on cm %p"
+ " <- op %s, st %s spsp %d sqpn %d\n",
+ cm, dapl_cm_op_str(ntohs(msg->op)),
+ dapl_cm_state_str(cm->state),
ntohs(msg->sport), ntohl(msg->sqpn));
}
dapl_os_unlock(&cm->lock);
@@ -635,11 +635,11 @@ void dapls_cm_acquire(dp_ib_cm_handle_t cm)
void dapls_cm_release(dp_ib_cm_handle_t cm)
{
dapl_os_lock(&cm->lock);
- cm->ref_count--;
- if (cm->ref_count) {
- dapl_os_unlock(&cm->lock);
- return;
- }
+ cm->ref_count--;
+ if (cm->ref_count) {
+ dapl_os_unlock(&cm->lock);
+ return;
+ }
/* client, release local conn id port */
if (!cm->sp && cm->msg.sport)
ucm_free_port(&cm->hca->ib_trans, ntohs(cm->msg.sport));
@@ -652,9 +652,9 @@ void dapls_cm_release(dp_ib_cm_handle_t cm)
if (cm->ah) {
ibv_destroy_ah(cm->ah);
cm->ah = NULL;
- }
- dapl_os_unlock(&cm->lock);
- dapli_cm_dealloc(cm);
+ }
+ dapl_os_unlock(&cm->lock);
+ dapli_cm_dealloc(cm);
}
dp_ib_cm_handle_t dapls_ib_cm_create(DAPL_EP *ep)
@@ -710,6 +710,11 @@ bail:
/* schedule destruction of CM object */
void dapli_cm_free(dp_ib_cm_handle_t cm)
{
+ dapl_log(DAPL_DBG_TYPE_CM,
+ " dapli_cm_free: cm %p %s ep %p refs=%d\n",
+ cm, dapl_cm_state_str(cm->state),
+ cm->ep, cm->ref_count);
+
dapl_os_lock(&cm->lock);
cm->state = DCM_FREE;
dapls_thread_signal(&cm->hca->ib_trans.signal);
@@ -720,15 +725,18 @@ void dapli_cm_free(dp_ib_cm_handle_t cm)
void dapls_cm_free(dp_ib_cm_handle_t cm)
{
dapl_log(DAPL_DBG_TYPE_CM,
- " cm_free: cm %p %s ep %p refs=%d\n",
+ " dapl_cm_free: cm %p %s ep %p refs=%d\n",
cm, dapl_cm_state_str(cm->state),
cm->ep, cm->ref_count);
/* free from internal workq, wait until EP is last ref */
dapl_os_lock(&cm->lock);
- cm->state = DCM_FREE;
+ if (cm->state != DCM_FREE)
+ cm->state = DCM_FREE;
+
while (cm->ref_count != 1) {
dapl_os_unlock(&cm->lock);
+ dapls_thread_signal(&cm->hca->ib_trans.signal);
dapl_os_sleep_usec(10000);
dapl_os_lock(&cm->lock);
}
@@ -804,8 +812,6 @@ static void ucm_disconnect_final(dp_ib_cm_handle_t cm)
else
dapl_evd_connection_callback(cm, IB_CME_DISCONNECTED, NULL, 0, cm->ep);
- /* free local resources, EP ref will prevent destory until dat_ep_free */
- dapls_cm_release(cm);
}
/*
@@ -815,6 +821,7 @@ static void ucm_disconnect_final(dp_ib_cm_handle_t cm)
DAT_RETURN dapli_cm_disconnect(dp_ib_cm_handle_t cm)
{
int finalize = 1;
+ int wakeup = 0;
dapl_os_lock(&cm->lock);
switch (cm->state) {
@@ -826,8 +833,8 @@ DAT_RETURN dapli_cm_disconnect(dp_ib_cm_handle_t cm)
/* send DREQ, event after DREP or DREQ timeout */
cm->state = DCM_DISC_PENDING;
cm->msg.op = htons(DCM_DREQ);
- finalize = 0; /* wait for DREP, wakeup timer thread */
- dapls_thread_signal(&cm->hca->ib_trans.signal);
+ finalize = 0; /* wait for DREP, wakeup timer after DREQ sent */
+ wakeup = 1;
break;
case DCM_DISC_PENDING:
/* DREQ timeout, resend until retries exhausted */
@@ -850,10 +857,29 @@ DAT_RETURN dapli_cm_disconnect(dp_ib_cm_handle_t cm)
if (cm->ep->qp_handle->qp_type != IBV_QPT_UD)
dapls_modify_qp_state(cm->ep->qp_handle, IBV_QPS_ERR,0,0,0);
- /* DREQ received, send DREP and schedule event */
+ /* DREQ received, send DREP and schedule event, finalize */
cm->msg.op = htons(DCM_DREP);
break;
+ case DCM_DISCONNECTED:
+ dapl_os_unlock(&cm->lock);
+ return DAT_SUCCESS;
default:
+ dapl_log(DAPL_DBG_TYPE_WARN,
+ " disconnect UNKNOWN state: ep %p cm %p %s %s"
+ " %x %x %x %s %x %x %x r_pid %x,%d\n",
+ cm->ep, cm,
+ cm->msg.saddr.ib.qp_type == IBV_QPT_RC ? "RC" : "UD",
+ dapl_cm_state_str(cm->state),
+ ntohs(cm->msg.saddr.ib.lid),
+ ntohs(cm->msg.sport),
+ ntohl(cm->msg.saddr.ib.qpn),
+ cm->sp ? "<-" : "->",
+ ntohs(cm->msg.daddr.ib.lid),
+ ntohs(cm->msg.dport),
+ ntohl(cm->msg.daddr.ib.qpn),
+ ntohs(cm->msg.op) == DCM_REQ ? 0 : ntohl(*(DAT_UINT32*)cm->msg.resv),
+ ntohs(cm->msg.op) == DCM_REQ ? 0 : ntohl(*(DAT_UINT32*)cm->msg.resv));
+
dapl_os_unlock(&cm->lock);
return DAT_SUCCESS;
}
@@ -861,6 +887,9 @@ DAT_RETURN dapli_cm_disconnect(dp_ib_cm_handle_t cm)
dapl_os_get_time(&cm->timer); /* reply expected */
ucm_send(&cm->hca->ib_trans, &cm->msg, NULL, 0);
dapl_os_unlock(&cm->lock);
+
+ if (wakeup)
+ dapls_thread_signal(&cm->hca->ib_trans.signal);
if (finalize)
ucm_disconnect_final(cm);
@@ -1302,6 +1331,9 @@ static int ucm_reply(dp_ib_cm_handle_t cm)
{
dapl_os_lock(&cm->lock);
if (cm->state != DCM_RTU_PENDING) {
+ dapl_log(DAPL_DBG_TYPE_ERR,
+ " CM_REPLY: wrong state %s",
+ dapl_cm_state_str(cm->state));
dapl_os_unlock(&cm->lock);
return -1;
}
@@ -1375,6 +1407,7 @@ dapli_accept_usr(DAPL_EP *ep, DAPL_CR *cr, DAT_COUNT p_size, DAT_PVOID p_data)
dapl_os_unlock(&cm->lock);
return DAT_INVALID_STATE;
}
+ dapl_os_unlock(&cm->lock);
dapl_dbg_log(DAPL_DBG_TYPE_CM,
" ACCEPT_USR: remote lid=%x"
@@ -1401,6 +1434,7 @@ dapli_accept_usr(DAPL_EP *ep, DAPL_CR *cr, DAT_COUNT p_size, DAT_PVOID p_data)
#endif
/* modify QP to RTR and then to RTS with remote info already read */
+ dapl_os_lock(&ep->header.lock);
if (dapls_modify_qp_state(ep->qp_handle,
IBV_QPS_RTR,
cm->msg.daddr.ib.qpn,
@@ -1410,7 +1444,7 @@ dapli_accept_usr(DAPL_EP *ep, DAPL_CR *cr, DAT_COUNT p_size, DAT_PVOID p_data)
" ACCEPT_USR: QPS_RTR ERR %s -> lid %x qpn %x\n",
strerror(errno), ntohs(cm->msg.daddr.ib.lid),
ntohl(cm->msg.daddr.ib.qpn));
- dapl_os_unlock(&cm->lock);
+ dapl_os_unlock(&ep->header.lock);
goto bail;
}
if (dapls_modify_qp_state(ep->qp_handle,
@@ -1422,9 +1456,10 @@ dapli_accept_usr(DAPL_EP *ep, DAPL_CR *cr, DAT_COUNT p_size, DAT_PVOID p_data)
" ACCEPT_USR: QPS_RTS ERR %s -> lid %x qpn %x\n",
strerror(errno), ntohs(cm->msg.daddr.ib.lid),
ntohl(cm->msg.daddr.ib.qpn));
- dapl_os_unlock(&cm->lock);
+ dapl_os_unlock(&ep->header.lock);
goto bail;
}
+ dapl_os_unlock(&ep->header.lock);
/* save remote address information */
dapl_os_memcpy(&ep->remote_ia_address,
@@ -1449,8 +1484,9 @@ dapli_accept_usr(DAPL_EP *ep, DAPL_CR *cr, DAT_COUNT p_size, DAT_PVOID p_data)
dapl_ep_link_cm(ep, cm);
cm->ep = ep;
cm->hca = ia->hca_ptr;
+
+ dapl_os_lock(&cm->lock);
cm->state = DCM_RTU_PENDING;
- dapl_os_get_time(&cm->timer); /* RTU expected */
dapl_os_unlock(&cm->lock);
if (ucm_reply(cm)) {
@@ -1540,14 +1576,15 @@ dapls_ib_disconnect(IN DAPL_EP *ep_ptr, IN DAT_CLOSE_FLAGS close_flags)
{
dp_ib_cm_handle_t cm_ptr = dapl_get_cm_from_ep(ep_ptr);
+ dapl_os_lock(&ep_ptr->header.lock);
if (ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECTED ||
- ep_ptr->param.ep_attr.service_type != DAT_SERVICE_TYPE_RC) {
+ ep_ptr->param.ep_attr.service_type != DAT_SERVICE_TYPE_RC ||
+ cm_ptr == NULL) {
+ dapl_os_unlock(&ep_ptr->header.lock);
return DAT_SUCCESS;
}
+ dapl_os_unlock(&ep_ptr->header.lock);
- /* RC. Transition to error state to flush queue */
- dapls_modify_qp_state(ep_ptr->qp_handle, IBV_QPS_ERR, 0, 0, 0);
-
return (dapli_cm_disconnect(cm_ptr));
}
@@ -1575,7 +1612,19 @@ dapls_ib_disconnect_clean(IN DAPL_EP *ep,
IN DAT_BOOLEAN active,
IN const ib_cm_events_t ib_cm_event)
{
- /* nothing to cleanup */
+ if (ib_cm_event == IB_CME_TIMEOUT) {
+ dp_ib_cm_handle_t cm_ptr;
+
+ if ((cm_ptr = dapl_get_cm_from_ep(ep)) == NULL)
+ return;
+
+ dapl_log(DAPL_DBG_TYPE_WARN,
+ "dapls_ib_disc_clean: CONN_TIMEOUT ep %p cm %p %s\n",
+ ep, cm_ptr, dapl_cm_state_str(cm_ptr->state));
+
+ /* schedule release of socket and local resources */
+ dapli_cm_free(cm_ptr);
+ }
}
/*
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 09/12] dapl-2.0: scm: add EP locking and cm checking to socket cm disconnect
From: Davis, Arlin R @ 2010-05-19 18:11 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
Signed-off-by: Arlin Davis <arlin.r.davis-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
dapl/openib_scm/cm.c | 7 +++++--
1 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/dapl/openib_scm/cm.c b/dapl/openib_scm/cm.c
index b6ffbe9..afd0d93 100644
--- a/dapl/openib_scm/cm.c
+++ b/dapl/openib_scm/cm.c
@@ -1410,11 +1410,14 @@ dapls_ib_disconnect(IN DAPL_EP * ep_ptr, IN DAT_CLOSE_FLAGS close_flags)
{
dp_ib_cm_handle_t cm_ptr = dapl_get_cm_from_ep(ep_ptr);
+ dapl_os_lock(&ep_ptr->header.lock);
if (ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECTED ||
- ep_ptr->param.ep_attr.service_type != DAT_SERVICE_TYPE_RC) {
+ ep_ptr->param.ep_attr.service_type != DAT_SERVICE_TYPE_RC ||
+ cm_ptr == NULL) {
+ dapl_os_unlock(&ep_ptr->header.lock);
return DAT_SUCCESS;
}
-
+ dapl_os_unlock(&ep_ptr->header.lock);
return (dapli_socket_disconnect(cm_ptr));
}
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 08/12] dapl-2.0: scm: new cm_ep linking broke UD mode over socket cm
From: Davis, Arlin R @ 2010-05-19 18:11 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, ofw_list
Add EP locking around modify_qp for EP state.
Add new dapli_ep_check for debugging EP
Cleanup extra CR's
Change socket errno to dapl_socket_errno() abstraction
Signed-off-by: Arlin Davis <arlin.r.davis-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
dapl/openib_scm/cm.c | 177 ++++++++++++++++++++++++++++++++++++--------------
1 files changed, 128 insertions(+), 49 deletions(-)
diff --git a/dapl/openib_scm/cm.c b/dapl/openib_scm/cm.c
index 6958b67..b6ffbe9 100644
--- a/dapl/openib_scm/cm.c
+++ b/dapl/openib_scm/cm.c
@@ -60,6 +60,48 @@
#include "dapl_ep_util.h"
#include "dapl_osd.h"
+#ifdef DAPL_DBG
+/* Check for EP linking to IA and proper connect state */
+void dapli_ep_check(DAPL_EP *ep)
+{
+ DAPL_IA *ia_ptr = ep->header.owner_ia;
+ DAPL_EP *ep_ptr, *next_ep_ptr;
+ int found = 0;
+
+ dapl_os_lock(&ia_ptr->header.lock);
+ ep_ptr = (dapl_llist_is_empty (&ia_ptr->ep_list_head)
+ ? NULL : dapl_llist_peek_head (&ia_ptr->ep_list_head));
+
+ while (ep_ptr != NULL) {
+ next_ep_ptr =
+ dapl_llist_next_entry(&ia_ptr->ep_list_head,
+ &ep_ptr->header.ia_list_entry);
+ if (ep == ep_ptr) {
+ found++;
+ if ((ep->cr_ptr && ep->param.ep_state
+ != DAT_EP_STATE_COMPLETION_PENDING) ||
+ (!ep->cr_ptr && ep->param.ep_state
+ != DAT_EP_STATE_ACTIVE_CONNECTION_PENDING))
+ goto err;
+ else
+ goto match;
+ }
+ ep_ptr = next_ep_ptr;
+ }
+err:
+ dapl_log(DAPL_DBG_TYPE_ERR,
+ " dapli_ep_check ERR: %s %s ep=%p state=%d magic=0x%x\n",
+ ep->cr_ptr ? "PASSIVE":"ACTIVE",
+ found ? "WRONG_STATE":"NOT_FOUND" ,
+ ep, ep->param.ep_state, ep->header.magic);
+match:
+ dapl_os_unlock(&ia_ptr->header.lock);
+ return;
+}
+#else
+#define dapli_ep_check(ep)
+#endif
+
#if defined(_WIN32) || defined(_WIN64)
enum DAPL_FD_EVENTS {
DAPL_FD_READ = 0x1,
@@ -311,13 +353,13 @@ void dapls_cm_acquire(dp_ib_cm_handle_t cm_ptr)
void dapls_cm_release(dp_ib_cm_handle_t cm_ptr)
{
dapl_os_lock(&cm_ptr->lock);
- cm_ptr->ref_count--;
- if (cm_ptr->ref_count) {
- dapl_os_unlock(&cm_ptr->lock);
- return;
- }
- dapl_os_unlock(&cm_ptr->lock);
- dapli_cm_dealloc(cm_ptr);
+ cm_ptr->ref_count--;
+ if (cm_ptr->ref_count) {
+ dapl_os_unlock(&cm_ptr->lock);
+ return;
+ }
+ dapl_os_unlock(&cm_ptr->lock);
+ dapli_cm_dealloc(cm_ptr);
}
static dp_ib_cm_handle_t dapli_cm_alloc(DAPL_EP *ep_ptr)
@@ -416,7 +458,9 @@ DAT_RETURN dapli_socket_disconnect(dp_ib_cm_handle_t cm_ptr)
dapl_os_unlock(&cm_ptr->lock);
/* send disc date, close socket, schedule destroy */
+ dapl_os_lock(&cm_ptr->ep->header.lock);
dapls_modify_qp_state(cm_ptr->ep->qp_handle, IBV_QPS_ERR, 0,0,0);
+ dapl_os_unlock(&cm_ptr->ep->header.lock);
send(cm_ptr->socket, (char *)&disc_data, sizeof(disc_data), 0);
/* disconnect events for RC's only */
@@ -452,7 +496,7 @@ static void dapli_socket_connected(dp_ib_cm_handle_t cm_ptr, int err)
dapl_log(DAPL_DBG_TYPE_ERR,
" CONN_PENDING: %s ERR %s -> %s %d\n",
err == -1 ? "POLL" : "SOCKOPT",
- err == -1 ? strerror(errno) : strerror(err),
+ err == -1 ? strerror(dapl_socket_errno()) : strerror(err),
inet_ntoa(((struct sockaddr_in *)
&cm_ptr->addr)->sin_addr),
ntohs(((struct sockaddr_in *)
@@ -475,9 +519,10 @@ static void dapli_socket_connected(dp_ib_cm_handle_t cm_ptr, int err)
}
if (len != (exp + ntohs(cm_ptr->msg.p_size))) {
+ int err = dapl_socket_errno();
dapl_log(DAPL_DBG_TYPE_ERR,
- " CONN_PENDING len ERR %s, wcnt=%d(%d) -> %s\n",
- strerror(errno), len,
+ " CONN_PENDING len ERR 0x%x %s, wcnt=%d(%d) -> %s\n",
+ err, strerror(err), len,
exp + ntohs(cm_ptr->msg.p_size),
inet_ntoa(((struct sockaddr_in *)
ep_ptr->param.
@@ -530,16 +575,19 @@ dapli_socket_connect(DAPL_EP * ep_ptr,
/* create, connect, sockopt, and exchange QP information */
if ((cm_ptr->socket =
socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) == DAPL_INVALID_SOCKET) {
+ int err = dapl_socket_errno();
dapl_log(DAPL_DBG_TYPE_ERR,
- " connect: socket create ERR %s\n", strerror(errno));
+ " connect: socket create ERR 0x%x %s\n",
+ err, strerror(err));
goto bail;
}
ret = dapl_config_socket(cm_ptr->socket);
if (ret < 0) {
dapl_log(DAPL_DBG_TYPE_ERR,
- " connect: config socket %d ERR %d %s\n",
- cm_ptr->socket, ret, strerror(dapl_socket_errno()));
+ " connect: config socket %d RET %d ERR 0x%x %s\n",
+ cm_ptr->socket, ret,
+ dapl_socket_errno(), strerror(dapl_socket_errno()));
dat_ret = DAT_INTERNAL_ERROR;
goto bail;
}
@@ -556,6 +604,10 @@ dapli_socket_connect(DAPL_EP * ep_ptr,
ret = dapl_connect_socket(cm_ptr->socket, (struct sockaddr *)&cm_ptr->addr,
sizeof(cm_ptr->addr));
if (ret && ret != EAGAIN) {
+ dapl_log(DAPL_DBG_TYPE_ERR,
+ " connect: dapl_connect_socket RET %d ERR 0x%x %s\n",
+ ret, dapl_socket_errno(),
+ strerror(dapl_socket_errno()));
dat_ret = DAT_INVALID_ADDRESS;
goto bail;
}
@@ -572,9 +624,10 @@ dapli_socket_connect(DAPL_EP * ep_ptr,
/* get local address information from socket */
sl = sizeof(cm_ptr->msg.daddr.so);
if (getsockname(cm_ptr->socket, (struct sockaddr *)&cm_ptr->msg.daddr.so, &sl)) {
+ int err = dapl_socket_errno();
dapl_log(DAPL_DBG_TYPE_ERR,
- " connect getsockname ERROR: %s -> %s r_qual %d\n",
- strerror(errno),
+ " connect getsockname ERROR: 0x%x %s -> %s r_qual %d\n",
+ err, strerror(err),
inet_ntoa(((struct sockaddr_in *)r_addr)->sin_addr),
(unsigned int)r_qual);;
}
@@ -604,8 +657,7 @@ dapli_socket_connect(DAPL_EP * ep_ptr,
return DAT_SUCCESS;
bail:
dapl_log(DAPL_DBG_TYPE_ERR,
- " connect ERROR: %s -> %s r_qual %d\n",
- strerror(errno),
+ " connect ERROR: -> %s r_qual %d\n",
inet_ntoa(((struct sockaddr_in *)r_addr)->sin_addr),
(unsigned int)r_qual);
@@ -629,9 +681,10 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr)
len = recv(cm_ptr->socket, (char *)&cm_ptr->msg, exp, 0);
if (len != exp || ntohs(cm_ptr->msg.ver) != DCM_VER) {
+ int err = dapl_socket_errno();
dapl_log(DAPL_DBG_TYPE_WARN,
- " CONN_RTU read: sk %d ERR %s, rcnt=%d, v=%d -> %s PORT L-%x R-%x PID L-%x R-%x\n",
- cm_ptr->socket, strerror(errno), len, ntohs(cm_ptr->msg.ver),
+ " CONN_RTU read: sk %d ERR 0x%x, rcnt=%d, v=%d -> %s PORT L-%x R-%x PID L-%x R-%x\n",
+ cm_ptr->socket, err, len, ntohs(cm_ptr->msg.ver),
inet_ntoa(((struct sockaddr_in *)&cm_ptr->addr)->sin_addr),
ntohs(((struct sockaddr_in *)&cm_ptr->msg.daddr.so)->sin_port),
ntohs(((struct sockaddr_in *)&cm_ptr->addr)->sin_port),
@@ -639,7 +692,7 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr)
ntohs(*(uint16_t*)&cm_ptr->msg.resv[2]));
/* Retry; corner case where server tcp stack resets under load */
- if (dapl_socket_errno() == ECONNRESET) {
+ if (err == ECONNRESET) {
closesocket(cm_ptr->socket);
cm_ptr->socket = DAPL_INVALID_SOCKET;
dapli_socket_connect(cm_ptr->ep, (DAT_IA_ADDRESS_PTR)&cm_ptr->addr,
@@ -692,9 +745,10 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr)
if (exp) {
len = recv(cm_ptr->socket, cm_ptr->msg.p_data, exp, 0);
if (len != exp) {
+ int err = dapl_socket_errno();
dapl_log(DAPL_DBG_TYPE_ERR,
- " CONN_RTU read pdata: ERR %s, rcnt=%d -> %s\n",
- strerror(errno), len,
+ " CONN_RTU read pdata: ERR 0x%x %s, rcnt=%d -> %s\n",
+ err, strerror(err), len,
inet_ntoa(((struct sockaddr_in *)
ep_ptr->param.
remote_ia_address_ptr)->sin_addr));
@@ -721,6 +775,7 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr)
}
/* modify QP to RTR and then to RTS with remote info */
+ dapl_os_lock(&ep_ptr->header.lock);
if (dapls_modify_qp_state(ep_ptr->qp_handle,
IBV_QPS_RTR,
cm_ptr->msg.saddr.ib.qpn,
@@ -736,6 +791,7 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr)
&cm_ptr->msg.daddr.so)->sin_addr),
ntohs(((struct sockaddr_in *)
&cm_ptr->msg.daddr.so)->sin_port));
+ dapl_os_unlock(&ep_ptr->header.lock);
goto bail;
}
if (dapls_modify_qp_state(ep_ptr->qp_handle,
@@ -753,16 +809,20 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr)
&cm_ptr->msg.daddr.so)->sin_addr),
ntohs(((struct sockaddr_in *)
&cm_ptr->msg.daddr.so)->sin_port));
+ dapl_os_unlock(&ep_ptr->header.lock);
goto bail;
}
+ dapl_os_unlock(&ep_ptr->header.lock);
dapl_dbg_log(DAPL_DBG_TYPE_EP, " connect_rtu: send RTU\n");
/* complete handshake after final QP state change, Just ver+op */
cm_ptr->state = DCM_CONNECTED;
cm_ptr->msg.op = ntohs(DCM_RTU);
if (send(cm_ptr->socket, (char *)&cm_ptr->msg, 4, 0) == -1) {
+ int err = dapl_socket_errno();
dapl_log(DAPL_DBG_TYPE_ERR,
- " CONN_RTU: write error = %s\n", strerror(errno));
+ " CONN_RTU: write ERR = 0x%x %s\n",
+ err, strerror(err));
goto bail;
}
/* post the event with private data */
@@ -821,6 +881,7 @@ ud_bail:
} else
#endif
{
+ dapli_ep_check(cm_ptr->ep);
dapl_evd_connection_callback(cm_ptr, event, cm_ptr->msg.p_data,
DCM_MAX_PDATA_SIZE, ep_ptr);
}
@@ -848,7 +909,7 @@ dapli_socket_listen(DAPL_IA * ia_ptr, DAT_CONN_QUAL serviceID, DAPL_SP * sp_ptr)
struct sockaddr_in addr;
ib_cm_srvc_handle_t cm_ptr = NULL;
DAT_RETURN dat_status = DAT_SUCCESS;
- int opt = 1;
+ int opt = 1;
dapl_dbg_log(DAPL_DBG_TYPE_CM,
" setup listen(ia_ptr %p ServiceID %d sp_ptr %p)\n",
@@ -864,23 +925,26 @@ dapli_socket_listen(DAPL_IA * ia_ptr, DAT_CONN_QUAL serviceID, DAPL_SP * sp_ptr)
/* bind, listen, set sockopt, accept, exchange data */
if ((cm_ptr->socket =
socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) == DAPL_INVALID_SOCKET) {
- dapl_log(DAPL_DBG_TYPE_ERR, " ERR: listen socket create: %s\n",
- strerror(errno));
+ int err = dapl_socket_errno();
+ dapl_log(DAPL_DBG_TYPE_ERR,
+ " listen: socket create: ERR 0x%x %s\n",
+ err, strerror(err));
dat_status = DAT_INSUFFICIENT_RESOURCES;
goto bail;
}
- setsockopt(cm_ptr->socket, SOL_SOCKET, SO_REUSEADDR, (char*)&opt, sizeof(opt));
+ setsockopt(cm_ptr->socket, SOL_SOCKET, SO_REUSEADDR, (char*)&opt, sizeof(opt));
addr.sin_port = htons(serviceID + 1000);
addr.sin_family = AF_INET;
addr.sin_addr = ((struct sockaddr_in *) &ia_ptr->hca_ptr->hca_address)->sin_addr;
if ((bind(cm_ptr->socket, (struct sockaddr *)&addr, sizeof(addr)) < 0)
|| (listen(cm_ptr->socket, 128) < 0)) {
+ int err = dapl_socket_errno();
dapl_log(DAPL_DBG_TYPE_CM,
- " listen: ERROR %s on port %d\n",
- strerror(errno), serviceID + 1000);
- if (dapl_socket_errno() == EADDRINUSE)
+ " listen: ERROR 0x%x %s on port %d\n",
+ err, strerror(err), serviceID + 1000);
+ if (err == EADDRINUSE)
dat_status = DAT_CONN_QUAL_IN_USE;
else
dat_status = DAT_CONN_QUAL_UNAVAILABLE;
@@ -933,9 +997,10 @@ static void dapli_socket_accept(ib_cm_srvc_handle_t cm_ptr)
&acm_ptr->msg.daddr.so,
(socklen_t *) &len);
if (acm_ptr->socket == DAPL_INVALID_SOCKET) {
+ int err = dapl_socket_errno();
dapl_log(DAPL_DBG_TYPE_ERR,
- " ACCEPT: ERR %s on FD %d l_cr %p\n",
- strerror(errno), cm_ptr->socket, cm_ptr);
+ " ACCEPT: ERR 0x%x %s on FD %d l_cr %p\n",
+ err, strerror(err), cm_ptr->socket, cm_ptr);
dapls_cm_release(acm_ptr);
return;
}
@@ -948,11 +1013,14 @@ static void dapli_socket_accept(ib_cm_srvc_handle_t cm_ptr)
/* no delay for small packets */
ret = setsockopt(acm_ptr->socket, IPPROTO_TCP, TCP_NODELAY,
(char *)&opt, sizeof(opt));
- if (ret)
+ if (ret) {
+ int err = dapl_socket_errno();
dapl_log(DAPL_DBG_TYPE_ERR,
- " ACCEPT: NODELAY setsockopt: 0x%x 0x%x %s\n",
- ret, dapl_socket_errno(), strerror(dapl_socket_errno()));
-
+ " ACCEPT: NODELAY setsockopt:"
+ " RET %d ERR 0x%x %s\n",
+ ret, err, strerror(err));
+ }
+
/* get local address information from socket */
sl = sizeof(acm_ptr->addr);
getsockname(acm_ptr->socket, (struct sockaddr *)&acm_ptr->addr, &sl);
@@ -975,9 +1043,10 @@ static void dapli_socket_accept_data(ib_cm_srvc_handle_t acm_ptr)
/* read in DST QP info, IA address. check for private data */
len = recv(acm_ptr->socket, (char *)&acm_ptr->msg, exp, 0);
if (len != exp || ntohs(acm_ptr->msg.ver) != DCM_VER) {
+ int err = dapl_socket_errno();
dapl_log(DAPL_DBG_TYPE_ERR,
- " ACCEPT read: ERR %s, rcnt=%d, ver=%d\n",
- strerror(errno), len, ntohs(acm_ptr->msg.ver));
+ " ACCEPT read: ERR 0x%x %s, rcnt=%d, ver=%d\n",
+ err, strerror(err), len, ntohs(acm_ptr->msg.ver));
goto bail;
}
@@ -996,9 +1065,10 @@ static void dapli_socket_accept_data(ib_cm_srvc_handle_t acm_ptr)
if (exp) {
len = recv(acm_ptr->socket, acm_ptr->msg.p_data, exp, 0);
if (len != exp) {
+ int err = dapl_socket_errno();
dapl_log(DAPL_DBG_TYPE_ERR,
- " accept read pdata: ERR %s, rcnt=%d\n",
- strerror(errno), len);
+ " accept read pdata: ERR 0x%x %s, rcnt=%d\n",
+ err, strerror(err), len);
goto bail;
}
p_data = acm_ptr->msg.p_data;
@@ -1092,6 +1162,7 @@ dapli_socket_accept_usr(DAPL_EP * ep_ptr,
#endif
/* modify QP to RTR and then to RTS with remote info already read */
+ dapl_os_lock(&ep_ptr->header.lock);
if (dapls_modify_qp_state(ep_ptr->qp_handle,
IBV_QPS_RTR,
cm_ptr->msg.saddr.ib.qpn,
@@ -1102,6 +1173,7 @@ dapli_socket_accept_usr(DAPL_EP * ep_ptr,
strerror(errno),
inet_ntoa(((struct sockaddr_in *)
&cm_ptr->msg.daddr.so)->sin_addr));
+ dapl_os_unlock(&ep_ptr->header.lock);
goto bail;
}
if (dapls_modify_qp_state(ep_ptr->qp_handle,
@@ -1114,8 +1186,10 @@ dapli_socket_accept_usr(DAPL_EP * ep_ptr,
strerror(errno),
inet_ntoa(((struct sockaddr_in *)
&cm_ptr->msg.daddr.so)->sin_addr));
+ dapl_os_unlock(&ep_ptr->header.lock);
goto bail;
}
+ dapl_os_unlock(&ep_ptr->header.lock);
/* save remote address information */
dapl_os_memcpy(&ep_ptr->remote_ia_address,
@@ -1143,6 +1217,10 @@ dapli_socket_accept_usr(DAPL_EP * ep_ptr,
cm_ptr->hca = ia_ptr->hca_ptr;
cm_ptr->state = DCM_ACCEPTED;
+ /* Link CM to EP, already queued on work thread */
+ dapl_ep_link_cm(ep_ptr, cm_ptr);
+ cm_ptr->ep = ep_ptr;
+
local.p_size = htons(p_size);
iov[0].iov_base = (void *)&local;
iov[0].iov_len = exp;
@@ -1155,11 +1233,14 @@ dapli_socket_accept_usr(DAPL_EP * ep_ptr,
len = writev(cm_ptr->socket, iov, 1);
if (len != (p_size + exp)) {
+ int err = dapl_socket_errno();
dapl_log(DAPL_DBG_TYPE_ERR,
- " ACCEPT_USR: ERR %s, wcnt=%d -> %s\n",
- strerror(errno), len,
+ " ACCEPT_USR: ERR 0x%x %s, wcnt=%d -> %s\n",
+ err, strerror(err), len,
inet_ntoa(((struct sockaddr_in *)
&cm_ptr->msg.daddr.so)->sin_addr));
+ dapl_ep_unlink_cm(ep_ptr, cm_ptr);
+ cm_ptr->ep = NULL;
goto bail;
}
@@ -1176,9 +1257,6 @@ dapli_socket_accept_usr(DAPL_EP * ep_ptr,
dapl_dbg_log(DAPL_DBG_TYPE_EP, " PASSIVE: accepted!\n");
- /* Link CM to EP, already queued on work thread */
- dapl_ep_link_cm(ep_ptr, cm_ptr);
- cm_ptr->ep = ep_ptr;
return DAT_SUCCESS;
bail:
/* schedule cleanup from workq */
@@ -1260,6 +1338,7 @@ ud_bail:
} else
#endif
{
+ dapli_ep_check(cm_ptr->ep);
dapls_cr_callback(cm_ptr, event, NULL, 0, cm_ptr->sp);
}
return;
@@ -1336,9 +1415,6 @@ dapls_ib_disconnect(IN DAPL_EP * ep_ptr, IN DAT_CLOSE_FLAGS close_flags)
return DAT_SUCCESS;
}
- /* RC. Transition to error state to flush queue */
- dapls_modify_qp_state(ep_ptr->qp_handle, IBV_QPS_ERR, 0, 0, 0);
-
return (dapli_socket_disconnect(cm_ptr));
}
@@ -1367,7 +1443,10 @@ dapls_ib_disconnect_clean(IN DAPL_EP * ep_ptr,
IN const ib_cm_events_t ib_cm_event)
{
if (ib_cm_event == IB_CME_TIMEOUT) {
- dp_ib_cm_handle_t cm_ptr = dapl_get_cm_from_ep(ep_ptr);
+ dp_ib_cm_handle_t cm_ptr;
+
+ if ((cm_ptr = dapl_get_cm_from_ep(ep_ptr)) == NULL)
+ return;
dapl_log(DAPL_DBG_TYPE_WARN,
"dapls_ib_disc_clean: CONN_TIMEOUT ep %p cm %p %s\n",
--
1.5.2.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox