* [PATCHv5 1/2][RESEND] opensm/PerfMgr: Better redirection support
@ 2010-06-17 17:28 Hal Rosenstock
[not found] ` <20100617172849.GA17245-Wuw85uim5zDR7s880joybQ@public.gmane.org>
0 siblings, 1 reply; 3+ messages in thread
From: Hal Rosenstock @ 2010-06-17 17:28 UTC (permalink / raw)
To: sashak-smomgflXvOZWk0Htik3J/w; +Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Handle PKey and QPN redirection information
GID redirection handling remains
Signed-off-by: Hal Rosenstock <hal.rosenstock-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org>
---
Changes since v4:
Fixed some trailing whitespace problems
Changes since v3:
Rebased
Changes since v2:
Use OpenSM DB rather than vendor layer for local port number and PKeys
Change most log levels from ERROR to VERBOSE
Redirection info validity now determined by single flag
validate_redir_pkey returns pkey index or -1 rather than boolean
Removed redir_ prefixes
Changes since v1:
Added include of osm_helper.h to osm_perfmgr.c
diff --git a/opensm/include/opensm/osm_perfmgr.h b/opensm/include/opensm/osm_perfmgr.h
index c26c141..34925e8 100644
--- a/opensm/include/opensm/osm_perfmgr.h
+++ b/opensm/include/opensm/osm_perfmgr.h
@@ -1,7 +1,7 @@
/*
* Copyright (c) 2007 The Regents of the University of California.
* Copyright (c) 2007-2009 Voltaire, Inc. All rights reserved.
- * Copyright (c) 2009 HNR Consulting. All rights reserved.
+ * Copyright (c) 2009,2010 HNR Consulting. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -90,11 +90,17 @@ typedef enum {
PERFMGR_SWEEP_SUSPENDED
} osm_perfmgr_sweep_state_t;
-/* Redirection information */
-typedef struct redir {
- ib_net16_t redir_lid;
- ib_net32_t redir_qp;
-} redir_t;
+typedef struct monitored_port {
+ uint16_t pkey_ix;
+ ib_net16_t orig_lid;
+ boolean_t redirection;
+ boolean_t valid;
+ /* Redirection fields from ClassPortInfo */
+ ib_gid_t gid;
+ ib_net16_t lid;
+ ib_net16_t pkey;
+ ib_net32_t qp;
+} monitored_port_t;
/* Node to store information about nodes being monitored */
typedef struct monitored_node {
@@ -104,7 +110,7 @@ typedef struct monitored_node {
boolean_t esp0;
char *name;
uint32_t num_ports;
- redir_t redir_port[1]; /* redirection on a per port basis */
+ monitored_port_t port[1];
} monitored_node_t;
struct osm_opensm;
@@ -134,6 +140,8 @@ typedef struct osm_perfmgr {
uint32_t max_outstanding_queries;
cl_qmap_t monitored_map; /* map the nodes being tracked */
monitored_node_t *remove_list;
+ ib_net64_t port_guid;
+ int16_t local_port;
} osm_perfmgr_t;
/*
* FIELDS
diff --git a/opensm/opensm/osm_perfmgr.c b/opensm/opensm/osm_perfmgr.c
index 398b463..fccf9d6 100644
--- a/opensm/opensm/osm_perfmgr.c
+++ b/opensm/opensm/osm_perfmgr.c
@@ -1,7 +1,7 @@
/*
* Copyright (c) 2007 The Regents of the University of California.
* Copyright (c) 2007-2009 Voltaire, Inc. All rights reserved.
- * Copyright (c) 2009 HNR Consulting. All rights reserved.
+ * Copyright (c) 2009,2010 HNR Consulting. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -64,6 +64,7 @@
#include <opensm/osm_log.h>
#include <opensm/osm_node.h>
#include <opensm/osm_opensm.h>
+#include <opensm/osm_helper.h>
#define PERFMGR_INITIAL_TID_VALUE 0xcafe
@@ -194,6 +195,7 @@ static void perfmgr_mad_send_err_callback(void *bind_context,
uint8_t port = context->perfmgr_context.port;
cl_map_item_t *p_node;
monitored_node_t *p_mon_node;
+ ib_net16_t orig_lid;
OSM_LOG_ENTER(pm->log);
@@ -225,9 +227,11 @@ static void perfmgr_mad_send_err_callback(void *bind_context,
p_mon_node->num_ports);
goto Exit;
}
- /* Clear redirection info */
- p_mon_node->redir_port[port].redir_lid = 0;
- p_mon_node->redir_port[port].redir_qp = 0;
+ /* Clear redirection info for this port except orig_lid */
+ orig_lid = p_mon_node->port[port].orig_lid;
+ memset(&p_mon_node->port[port], 0, sizeof(monitored_port_t));
+ p_mon_node->port[port].orig_lid = orig_lid;
+ p_mon_node->port[port].valid = TRUE;
cl_plock_release(&pm->osm->lock);
}
@@ -256,7 +260,7 @@ ib_api_status_t osm_perfmgr_bind(osm_perfmgr_t * pm, ib_net64_t port_guid)
goto Exit;
}
- bind_info.port_guid = port_guid;
+ bind_info.port_guid = pm->port_guid = port_guid;
bind_info.mad_class = IB_MCLASS_PERF;
bind_info.class_version = 1;
bind_info.is_responder = FALSE;
@@ -309,24 +313,14 @@ static ib_net32_t get_qp(monitored_node_t * mon_node, uint8_t port)
ib_net32_t qp = IB_QP1;
if (mon_node && mon_node->num_ports && port < mon_node->num_ports &&
- mon_node->redir_port[port].redir_lid &&
- mon_node->redir_port[port].redir_qp)
- qp = mon_node->redir_port[port].redir_qp;
+ mon_node->port[port].redirection && mon_node->port[port].qp)
+ qp = mon_node->port[port].qp;
return qp;
}
-/**********************************************************************
- * Given a node, a port, and an optional monitored node,
- * return the appropriate lid to query that port
- **********************************************************************/
-static ib_net16_t get_lid(osm_node_t * p_node, uint8_t port,
- monitored_node_t * mon_node)
+static ib_net16_t get_base_lid(osm_node_t * p_node, uint8_t port)
{
- if (mon_node && mon_node->num_ports && port < mon_node->num_ports &&
- mon_node->redir_port[port].redir_lid)
- return mon_node->redir_port[port].redir_lid;
-
switch (p_node->node_info.node_type) {
case IB_NODE_TYPE_CA:
case IB_NODE_TYPE_ROUTER:
@@ -339,12 +333,26 @@ static ib_net16_t get_lid(osm_node_t * p_node, uint8_t port,
}
/**********************************************************************
+ * Given a node, a port, and an optional monitored node,
+ * return the lid appropriate to query that port
+ **********************************************************************/
+static ib_net16_t get_lid(osm_node_t * p_node, uint8_t port,
+ monitored_node_t * mon_node)
+{
+ if (mon_node && mon_node->num_ports && port < mon_node->num_ports &&
+ mon_node->port[port].lid)
+ return mon_node->port[port].lid;
+
+ return get_base_lid(p_node, port);
+}
+
+/**********************************************************************
* Form and send the Port Counters MAD for a single port.
**********************************************************************/
static ib_api_status_t perfmgr_send_pc_mad(osm_perfmgr_t * perfmgr,
ib_net16_t dest_lid,
- ib_net32_t dest_qp, uint8_t port,
- uint8_t mad_method,
+ ib_net32_t dest_qp, uint16_t pkey_ix,
+ uint8_t port, uint8_t mad_method,
osm_madw_context_t * p_context)
{
ib_api_status_t status = IB_SUCCESS;
@@ -383,8 +391,7 @@ static ib_api_status_t perfmgr_send_pc_mad(osm_perfmgr_t * perfmgr,
p_madw->mad_addr.addr_type.gsi.remote_qp = dest_qp;
p_madw->mad_addr.addr_type.gsi.remote_qkey =
cl_hton32(IB_QP1_WELL_KNOWN_Q_KEY);
- /* FIXME what about other partitions */
- p_madw->mad_addr.addr_type.gsi.pkey_ix = 0;
+ p_madw->mad_addr.addr_type.gsi.pkey_ix = pkey_ix;
p_madw->mad_addr.addr_type.gsi.service_level = 0;
p_madw->mad_addr.addr_type.gsi.global_route = FALSE;
p_madw->resp_expected = TRUE;
@@ -420,6 +427,7 @@ static void collect_guids(cl_map_item_t * p_map_item, void *context)
osm_perfmgr_t *pm = (osm_perfmgr_t *) context;
monitored_node_t *mon_node = NULL;
uint32_t num_ports;
+ int port;
OSM_LOG_ENTER(pm->log);
@@ -428,7 +436,7 @@ static void collect_guids(cl_map_item_t * p_map_item, void *context)
/* if not already in map add it */
num_ports = osm_node_get_num_physp(node);
mon_node = malloc(sizeof(*mon_node) +
- sizeof(redir_t) * num_ports);
+ sizeof(monitored_port_t) * num_ports);
if (!mon_node) {
OSM_LOG(pm->log, OSM_LOG_ERROR, "PerfMgr: ERR 4C06: "
"malloc failed: not handling node %s"
@@ -437,7 +445,7 @@ static void collect_guids(cl_map_item_t * p_map_item, void *context)
goto Exit;
}
memset(mon_node, 0,
- sizeof(*mon_node) + sizeof(redir_t) * num_ports);
+ sizeof(*mon_node) + sizeof(monitored_port_t) * num_ports);
mon_node->guid = node_guid;
mon_node->name = strdup(node->print_desc);
mon_node->num_ports = num_ports;
@@ -445,6 +453,11 @@ static void collect_guids(cl_map_item_t * p_map_item, void *context)
mon_node->esp0 = (node->sw &&
ib_switch_info_is_enhanced_port0(&node->sw->
switch_info));
+ for (port = mon_node->esp0 ? 0 : 1; port < num_ports; port++) {
+ mon_node->port[port].orig_lid = get_base_lid(node, port);
+ mon_node->port[port].valid = TRUE;
+ }
+
cl_qmap_insert(&pm->monitored_map, node_guid,
(cl_map_item_t *) mon_node);
}
@@ -501,6 +514,9 @@ static void perfmgr_query_counters(cl_map_item_t * p_map_item, void *context)
if (!osm_node_get_physp_ptr(node, port))
continue;
+ if (!mon_node->port[port].valid)
+ continue;
+
lid = get_lid(node, port, mon_node);
if (lid == 0) {
OSM_LOG(pm->log, OSM_LOG_DEBUG, "WARN: node 0x%" PRIx64
@@ -521,8 +537,10 @@ static void perfmgr_query_counters(cl_map_item_t * p_map_item, void *context)
OSM_LOG(pm->log, OSM_LOG_VERBOSE, "Getting stats for node 0x%"
PRIx64 " port %d (lid %u) (%s)\n", node_guid, port,
cl_ntoh16(lid), node->print_desc);
- status = perfmgr_send_pc_mad(pm, lid, remote_qp, port,
- IB_MAD_METHOD_GET, &mad_context);
+ status = perfmgr_send_pc_mad(pm, lid, remote_qp,
+ mon_node->port[port].pkey_ix,
+ port, IB_MAD_METHOD_GET,
+ &mad_context);
if (status != IB_SUCCESS)
OSM_LOG(pm->log, OSM_LOG_ERROR, "ERR 4C09: "
"Failed to issue port counter query for node 0x%"
@@ -769,6 +787,24 @@ void osm_perfmgr_process(osm_perfmgr_t * pm)
pm->subn->sm_state == IB_SMINFO_STATE_NOTACTIVE)
perfmgr_discovery(pm->subn->p_osm);
+ /* if redirection enabled, determine local port */
+ if (pm->subn->opt.perfmgr_redir && pm->local_port == -1) {
+ osm_node_t *p_node;
+ osm_port_t *p_port;
+
+ CL_PLOCK_ACQUIRE(pm->sm->p_lock);
+ p_port = osm_get_port_by_guid(pm->subn, pm->port_guid);
+ if (p_port) {
+ p_node = p_port->p_node;
+ CL_ASSERT(p_node);
+ pm->local_port =
+ ib_node_info_get_local_port_num(&p_node->node_info);
+ } else
+ OSM_LOG(pm->log, OSM_LOG_ERROR,
+ "ERR 4C87: No PerfMgr port object\n");
+ CL_PLOCK_RELEASE(pm->sm->p_lock);
+ }
+
#if ENABLE_OSM_PERF_MGR_PROFILE
gettimeofday(&before, NULL);
#endif
@@ -932,8 +968,8 @@ static int counter_overflow_32(ib_net32_t val)
* MAD to the port.
**********************************************************************/
static void perfmgr_check_overflow(osm_perfmgr_t * pm,
- monitored_node_t * mon_node, uint8_t port,
- ib_port_counters_t * pc)
+ monitored_node_t * mon_node, int16_t pkey_ix,
+ uint8_t port, ib_port_counters_t * pc)
{
osm_madw_context_t mad_context;
ib_api_status_t status;
@@ -960,6 +996,9 @@ static void perfmgr_check_overflow(osm_perfmgr_t * pm,
osm_node_t *p_node = NULL;
ib_net16_t lid = 0;
+ if (!mon_node->port[port].valid)
+ goto Exit;
+
osm_log(pm->log, OSM_LOG_VERBOSE,
"PerfMgr: Counter overflow: %s (0x%" PRIx64
") port %d; clearing counters\n",
@@ -984,8 +1023,9 @@ static void perfmgr_check_overflow(osm_perfmgr_t * pm,
mad_context.perfmgr_context.port = port;
mad_context.perfmgr_context.mad_method = IB_MAD_METHOD_SET;
/* clear port counters */
- status = perfmgr_send_pc_mad(pm, lid, remote_qp, port,
- IB_MAD_METHOD_SET, &mad_context);
+ status = perfmgr_send_pc_mad(pm, lid, remote_qp, pkey_ix,
+ port, IB_MAD_METHOD_SET,
+ &mad_context);
if (status != IB_SUCCESS)
OSM_LOG(pm->log, OSM_LOG_ERROR, "PerfMgr: ERR 4C11: "
"Failed to send clear counters MAD for %s (0x%"
@@ -1043,6 +1083,64 @@ static void perfmgr_log_events(osm_perfmgr_t * pm,
time_diff, mon_node->name, mon_node->guid, port);
}
+static int16_t validate_redir_pkey(osm_perfmgr_t *pm, ib_net16_t pkey)
+{
+ int16_t pkey_ix = -1;
+ osm_port_t *p_port;
+ osm_pkey_tbl_t *p_pkey_tbl;
+ ib_net16_t *p_orig_pkey;
+ uint16_t block;
+ uint8_t index;
+
+ OSM_LOG_ENTER(pm->log);
+
+ CL_PLOCK_ACQUIRE(pm->sm->p_lock);
+ p_port = osm_get_port_by_guid(pm->subn, pm->port_guid);
+ if (!p_port) {
+ CL_PLOCK_RELEASE(pm->sm->p_lock);
+ OSM_LOG(pm->log, OSM_LOG_ERROR,
+ "ERR 4C1E: No PerfMgr port object\n");
+ goto Exit;
+ }
+ if (p_port->p_physp && osm_physp_is_valid(p_port->p_physp)) {
+ p_pkey_tbl = &p_port->p_physp->pkeys;
+ if (!p_pkey_tbl) {
+ CL_PLOCK_RELEASE(pm->sm->p_lock);
+ OSM_LOG(pm->log, OSM_LOG_VERBOSE,
+ "No PKey table found for PerfMgr port\n");
+ goto Exit;
+ }
+ p_orig_pkey = cl_map_get(&p_pkey_tbl->keys,
+ ib_pkey_get_base(pkey));
+ if (!p_orig_pkey) {
+ CL_PLOCK_RELEASE(pm->sm->p_lock);
+ OSM_LOG(pm->log, OSM_LOG_VERBOSE,
+ "PKey 0x%x not found for PerfMgr port\n",
+ cl_ntoh16(pkey));
+ goto Exit;
+ }
+ if (osm_pkey_tbl_get_block_and_idx(p_pkey_tbl, p_orig_pkey,
+ &block, &index) == IB_SUCCESS) {
+ CL_PLOCK_RELEASE(pm->sm->p_lock);
+ pkey_ix = block * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + index;
+ } else {
+ CL_PLOCK_RELEASE(pm->sm->p_lock);
+ OSM_LOG(pm->log, OSM_LOG_ERROR,
+ "ERR 0x4C1F: Failed to obtain P_Key 0x%04x "
+ "block and index for PerfMgr port\n",
+ cl_ntoh16(pkey));
+ }
+ } else {
+ CL_PLOCK_RELEASE(pm->sm->p_lock);
+ OSM_LOG(pm->log, OSM_LOG_ERROR,
+ "ERR 4C20: Local PerfMgt port physp invalid\n");
+ }
+
+Exit:
+ OSM_LOG_EXIT(pm->log);
+ return pkey_ix;
+}
+
/**********************************************************************
* The dispatcher uses a thread pool which will call this function when
* there is a thread available to process the mad received on the wire.
@@ -1061,6 +1159,8 @@ static void pc_recv_process(void *context, void *data)
perfmgr_db_data_cnt_reading_t data_reading;
cl_map_item_t *p_node;
monitored_node_t *p_mon_node;
+ int16_t pkey_ix = 0;
+ boolean_t valid = TRUE;
OSM_LOG_ENTER(pm->log);
@@ -1084,7 +1184,8 @@ static void pc_recv_process(void *context, void *data)
p_mad->attr_id == IB_MAD_ATTR_CLASS_PORT_INFO);
/* Response could also be redirection (IBM eHCA PMA does this) */
- if (p_mad->attr_id == IB_MAD_ATTR_CLASS_PORT_INFO) {
+ if (p_mad->status & IB_MAD_STATUS_REDIRECT &&
+ p_mad->attr_id == IB_MAD_ATTR_CLASS_PORT_INFO) {
char gid_str[INET6_ADDRSTRLEN];
ib_class_port_info_t *cpi =
(ib_class_port_info_t *) &
@@ -1097,17 +1198,46 @@ static void pc_recv_process(void *context, void *data)
inet_ntop(AF_INET6, cpi->redir_gid.raw, gid_str,
sizeof gid_str), cl_ntoh32(cpi->redir_qp));
- /* LID or GID redirection ? */
- /* For GID redirection, need to get PathRecord from SA */
+ if (!pm->subn->opt.perfmgr_redir) {
+ OSM_LOG(pm->log, OSM_LOG_VERBOSE,
+ "Redirection requested but disabled\n");
+ valid = FALSE;
+ }
+
+ /* valid redirection ? */
if (cpi->redir_lid == 0) {
+ if (!ib_gid_is_notzero(&cpi->redir_gid)) {
+ OSM_LOG(pm->log, OSM_LOG_VERBOSE,
+ "Invalid redirection "
+ "(both redirect LID and GID are zero)\n");
+ valid = FALSE;
+ }
+ }
+ if (cpi->redir_qp == 0) {
+ OSM_LOG(pm->log, OSM_LOG_VERBOSE, "Invalid RedirectQP\n");
+ valid = FALSE;
+ }
+ if (cpi->redir_pkey == 0) {
+ OSM_LOG(pm->log, OSM_LOG_VERBOSE, "Invalid RedirectP_Key\n");
+ valid = FALSE;
+ }
+ if (cpi->redir_qkey != IB_QP1_WELL_KNOWN_Q_KEY) {
+ OSM_LOG(pm->log, OSM_LOG_VERBOSE, "Invalid RedirectQ_Key\n");
+ valid = FALSE;
+ }
+
+ pkey_ix = validate_redir_pkey(pm, cpi->redir_pkey);
+ if (pkey_ix == -1) {
OSM_LOG(pm->log, OSM_LOG_VERBOSE,
- "GID redirection not currently implemented!\n");
- goto Exit;
+ "Index for Pkey 0x%x not found\n",
+ cl_ntoh16(cpi->redir_pkey));
+ valid = FALSE;
}
- if (!pm->subn->opt.perfmgr_redir) {
- OSM_LOG(pm->log, OSM_LOG_ERROR, "ERR 4C16: "
- "redirection requested but disabled\n");
+ if (cpi->redir_lid == 0) {
+ /* GID redirection: get PathRecord information */
+ OSM_LOG(pm->log, OSM_LOG_VERBOSE,
+ "GID redirection not currently supported\n");
goto Exit;
}
@@ -1122,13 +1252,24 @@ static void pc_recv_process(void *context, void *data)
p_mon_node->num_ports);
goto Exit;
}
- p_mon_node->redir_port[port].redir_lid = cpi->redir_lid;
- p_mon_node->redir_port[port].redir_qp = cpi->redir_qp;
+ p_mon_node->port[port].redirection = TRUE;
+ p_mon_node->port[port].valid = valid;
+ memcpy(&p_mon_node->port[port].gid, &cpi->redir_gid,
+ sizeof(ib_gid_t));
+ p_mon_node->port[port].lid = cpi->redir_lid;
+ p_mon_node->port[port].qp = cpi->redir_qp;
+ p_mon_node->port[port].pkey = cpi->redir_pkey;
+ if (pkey_ix != -1)
+ p_mon_node->port[port].pkey_ix = pkey_ix;
cl_plock_release(&pm->osm->lock);
+ if (!valid)
+ goto Exit;
+
/* Finally, reissue the query to the redirected location */
status = perfmgr_send_pc_mad(pm, cpi->redir_lid, cpi->redir_qp,
- port, mad_context->perfmgr_context.
+ pkey_ix, port,
+ mad_context->perfmgr_context.
mad_method, mad_context);
if (status != IB_SUCCESS)
OSM_LOG(pm->log, OSM_LOG_ERROR, "ERR 4C14: "
@@ -1163,7 +1304,7 @@ static void pc_recv_process(void *context, void *data)
perfmgr_db_clear_prev_dc(pm->db, node_guid, port);
}
- perfmgr_check_overflow(pm, p_mon_node, port, wire_read);
+ perfmgr_check_overflow(pm, p_mon_node, pkey_ix, port, wire_read);
#if ENABLE_OSM_PERF_MGR_PROFILE
do {
@@ -1208,6 +1349,7 @@ ib_api_status_t osm_perfmgr_init(osm_perfmgr_t * pm, osm_opensm_t * osm,
pm->sweep_time_s = p_opt->perfmgr_sweep_time_s;
pm->max_outstanding_queries = p_opt->perfmgr_max_outstanding_queries;
pm->osm = osm;
+ pm->local_port = -1;
status = cl_timer_init(&pm->sweep_timer, perfmgr_sweep, pm);
if (status != IB_SUCCESS)
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related [flat|nested] 3+ messages in thread[parent not found: <20100617172849.GA17245-Wuw85uim5zDR7s880joybQ@public.gmane.org>]
* Re: [PATCHv5 1/2][RESEND] opensm/PerfMgr: Better redirection support [not found] ` <20100617172849.GA17245-Wuw85uim5zDR7s880joybQ@public.gmane.org> @ 2010-07-14 1:28 ` Ira Weiny 2011-03-13 15:54 ` Alex Netes 1 sibling, 0 replies; 3+ messages in thread From: Ira Weiny @ 2010-07-14 1:28 UTC (permalink / raw) To: Hal Rosenstock Cc: sashak-smomgflXvOZWk0Htik3J/w@public.gmane.org, linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org Sasha, What is the status of this patch and the follow on patch: "opensm/osm_console.c: Add dump and clear redir perfmgr command support" Thanks, Ira On Thu, 17 Jun 2010 10:28:49 -0700 Hal Rosenstock <hnrose-Wuw85uim5zDR7s880joybQ@public.gmane.org> wrote: > > Handle PKey and QPN redirection information > GID redirection handling remains > > Signed-off-by: Hal Rosenstock <hal.rosenstock-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org> > --- > Changes since v4: > Fixed some trailing whitespace problems > > Changes since v3: > Rebased > > Changes since v2: > Use OpenSM DB rather than vendor layer for local port number and PKeys > Change most log levels from ERROR to VERBOSE > Redirection info validity now determined by single flag > validate_redir_pkey returns pkey index or -1 rather than boolean > Removed redir_ prefixes > > Changes since v1: > Added include of osm_helper.h to osm_perfmgr.c > > diff --git a/opensm/include/opensm/osm_perfmgr.h b/opensm/include/opensm/osm_perfmgr.h > index c26c141..34925e8 100644 > --- a/opensm/include/opensm/osm_perfmgr.h > +++ b/opensm/include/opensm/osm_perfmgr.h > @@ -1,7 +1,7 @@ > /* > * Copyright (c) 2007 The Regents of the University of California. > * Copyright (c) 2007-2009 Voltaire, Inc. All rights reserved. > - * Copyright (c) 2009 HNR Consulting. All rights reserved. > + * Copyright (c) 2009,2010 HNR Consulting. All rights reserved. > * > * This software is available to you under a choice of one of two > * licenses. You may choose to be licensed under the terms of the GNU > @@ -90,11 +90,17 @@ typedef enum { > PERFMGR_SWEEP_SUSPENDED > } osm_perfmgr_sweep_state_t; > > -/* Redirection information */ > -typedef struct redir { > - ib_net16_t redir_lid; > - ib_net32_t redir_qp; > -} redir_t; > +typedef struct monitored_port { > + uint16_t pkey_ix; > + ib_net16_t orig_lid; > + boolean_t redirection; > + boolean_t valid; > + /* Redirection fields from ClassPortInfo */ > + ib_gid_t gid; > + ib_net16_t lid; > + ib_net16_t pkey; > + ib_net32_t qp; > +} monitored_port_t; > > /* Node to store information about nodes being monitored */ > typedef struct monitored_node { > @@ -104,7 +110,7 @@ typedef struct monitored_node { > boolean_t esp0; > char *name; > uint32_t num_ports; > - redir_t redir_port[1]; /* redirection on a per port basis */ > + monitored_port_t port[1]; > } monitored_node_t; > > struct osm_opensm; > @@ -134,6 +140,8 @@ typedef struct osm_perfmgr { > uint32_t max_outstanding_queries; > cl_qmap_t monitored_map; /* map the nodes being tracked */ > monitored_node_t *remove_list; > + ib_net64_t port_guid; > + int16_t local_port; > } osm_perfmgr_t; > /* > * FIELDS > diff --git a/opensm/opensm/osm_perfmgr.c b/opensm/opensm/osm_perfmgr.c > index 398b463..fccf9d6 100644 > --- a/opensm/opensm/osm_perfmgr.c > +++ b/opensm/opensm/osm_perfmgr.c > @@ -1,7 +1,7 @@ > /* > * Copyright (c) 2007 The Regents of the University of California. > * Copyright (c) 2007-2009 Voltaire, Inc. All rights reserved. > - * Copyright (c) 2009 HNR Consulting. All rights reserved. > + * Copyright (c) 2009,2010 HNR Consulting. All rights reserved. > * > * This software is available to you under a choice of one of two > * licenses. You may choose to be licensed under the terms of the GNU > @@ -64,6 +64,7 @@ > #include <opensm/osm_log.h> > #include <opensm/osm_node.h> > #include <opensm/osm_opensm.h> > +#include <opensm/osm_helper.h> > > #define PERFMGR_INITIAL_TID_VALUE 0xcafe > > @@ -194,6 +195,7 @@ static void perfmgr_mad_send_err_callback(void *bind_context, > uint8_t port = context->perfmgr_context.port; > cl_map_item_t *p_node; > monitored_node_t *p_mon_node; > + ib_net16_t orig_lid; > > OSM_LOG_ENTER(pm->log); > > @@ -225,9 +227,11 @@ static void perfmgr_mad_send_err_callback(void *bind_context, > p_mon_node->num_ports); > goto Exit; > } > - /* Clear redirection info */ > - p_mon_node->redir_port[port].redir_lid = 0; > - p_mon_node->redir_port[port].redir_qp = 0; > + /* Clear redirection info for this port except orig_lid */ > + orig_lid = p_mon_node->port[port].orig_lid; > + memset(&p_mon_node->port[port], 0, sizeof(monitored_port_t)); > + p_mon_node->port[port].orig_lid = orig_lid; > + p_mon_node->port[port].valid = TRUE; > cl_plock_release(&pm->osm->lock); > } > > @@ -256,7 +260,7 @@ ib_api_status_t osm_perfmgr_bind(osm_perfmgr_t * pm, ib_net64_t port_guid) > goto Exit; > } > > - bind_info.port_guid = port_guid; > + bind_info.port_guid = pm->port_guid = port_guid; > bind_info.mad_class = IB_MCLASS_PERF; > bind_info.class_version = 1; > bind_info.is_responder = FALSE; > @@ -309,24 +313,14 @@ static ib_net32_t get_qp(monitored_node_t * mon_node, uint8_t port) > ib_net32_t qp = IB_QP1; > > if (mon_node && mon_node->num_ports && port < mon_node->num_ports && > - mon_node->redir_port[port].redir_lid && > - mon_node->redir_port[port].redir_qp) > - qp = mon_node->redir_port[port].redir_qp; > + mon_node->port[port].redirection && mon_node->port[port].qp) > + qp = mon_node->port[port].qp; > > return qp; > } > > -/********************************************************************** > - * Given a node, a port, and an optional monitored node, > - * return the appropriate lid to query that port > - **********************************************************************/ > -static ib_net16_t get_lid(osm_node_t * p_node, uint8_t port, > - monitored_node_t * mon_node) > +static ib_net16_t get_base_lid(osm_node_t * p_node, uint8_t port) > { > - if (mon_node && mon_node->num_ports && port < mon_node->num_ports && > - mon_node->redir_port[port].redir_lid) > - return mon_node->redir_port[port].redir_lid; > - > switch (p_node->node_info.node_type) { > case IB_NODE_TYPE_CA: > case IB_NODE_TYPE_ROUTER: > @@ -339,12 +333,26 @@ static ib_net16_t get_lid(osm_node_t * p_node, uint8_t port, > } > > /********************************************************************** > + * Given a node, a port, and an optional monitored node, > + * return the lid appropriate to query that port > + **********************************************************************/ > +static ib_net16_t get_lid(osm_node_t * p_node, uint8_t port, > + monitored_node_t * mon_node) > +{ > + if (mon_node && mon_node->num_ports && port < mon_node->num_ports && > + mon_node->port[port].lid) > + return mon_node->port[port].lid; > + > + return get_base_lid(p_node, port); > +} > + > +/********************************************************************** > * Form and send the Port Counters MAD for a single port. > **********************************************************************/ > static ib_api_status_t perfmgr_send_pc_mad(osm_perfmgr_t * perfmgr, > ib_net16_t dest_lid, > - ib_net32_t dest_qp, uint8_t port, > - uint8_t mad_method, > + ib_net32_t dest_qp, uint16_t pkey_ix, > + uint8_t port, uint8_t mad_method, > osm_madw_context_t * p_context) > { > ib_api_status_t status = IB_SUCCESS; > @@ -383,8 +391,7 @@ static ib_api_status_t perfmgr_send_pc_mad(osm_perfmgr_t * perfmgr, > p_madw->mad_addr.addr_type.gsi.remote_qp = dest_qp; > p_madw->mad_addr.addr_type.gsi.remote_qkey = > cl_hton32(IB_QP1_WELL_KNOWN_Q_KEY); > - /* FIXME what about other partitions */ > - p_madw->mad_addr.addr_type.gsi.pkey_ix = 0; > + p_madw->mad_addr.addr_type.gsi.pkey_ix = pkey_ix; > p_madw->mad_addr.addr_type.gsi.service_level = 0; > p_madw->mad_addr.addr_type.gsi.global_route = FALSE; > p_madw->resp_expected = TRUE; > @@ -420,6 +427,7 @@ static void collect_guids(cl_map_item_t * p_map_item, void *context) > osm_perfmgr_t *pm = (osm_perfmgr_t *) context; > monitored_node_t *mon_node = NULL; > uint32_t num_ports; > + int port; > > OSM_LOG_ENTER(pm->log); > > @@ -428,7 +436,7 @@ static void collect_guids(cl_map_item_t * p_map_item, void *context) > /* if not already in map add it */ > num_ports = osm_node_get_num_physp(node); > mon_node = malloc(sizeof(*mon_node) + > - sizeof(redir_t) * num_ports); > + sizeof(monitored_port_t) * num_ports); > if (!mon_node) { > OSM_LOG(pm->log, OSM_LOG_ERROR, "PerfMgr: ERR 4C06: " > "malloc failed: not handling node %s" > @@ -437,7 +445,7 @@ static void collect_guids(cl_map_item_t * p_map_item, void *context) > goto Exit; > } > memset(mon_node, 0, > - sizeof(*mon_node) + sizeof(redir_t) * num_ports); > + sizeof(*mon_node) + sizeof(monitored_port_t) * num_ports); > mon_node->guid = node_guid; > mon_node->name = strdup(node->print_desc); > mon_node->num_ports = num_ports; > @@ -445,6 +453,11 @@ static void collect_guids(cl_map_item_t * p_map_item, void *context) > mon_node->esp0 = (node->sw && > ib_switch_info_is_enhanced_port0(&node->sw-> > switch_info)); > + for (port = mon_node->esp0 ? 0 : 1; port < num_ports; port++) { > + mon_node->port[port].orig_lid = get_base_lid(node, port); > + mon_node->port[port].valid = TRUE; > + } > + > cl_qmap_insert(&pm->monitored_map, node_guid, > (cl_map_item_t *) mon_node); > } > @@ -501,6 +514,9 @@ static void perfmgr_query_counters(cl_map_item_t * p_map_item, void *context) > if (!osm_node_get_physp_ptr(node, port)) > continue; > > + if (!mon_node->port[port].valid) > + continue; > + > lid = get_lid(node, port, mon_node); > if (lid == 0) { > OSM_LOG(pm->log, OSM_LOG_DEBUG, "WARN: node 0x%" PRIx64 > @@ -521,8 +537,10 @@ static void perfmgr_query_counters(cl_map_item_t * p_map_item, void *context) > OSM_LOG(pm->log, OSM_LOG_VERBOSE, "Getting stats for node 0x%" > PRIx64 " port %d (lid %u) (%s)\n", node_guid, port, > cl_ntoh16(lid), node->print_desc); > - status = perfmgr_send_pc_mad(pm, lid, remote_qp, port, > - IB_MAD_METHOD_GET, &mad_context); > + status = perfmgr_send_pc_mad(pm, lid, remote_qp, > + mon_node->port[port].pkey_ix, > + port, IB_MAD_METHOD_GET, > + &mad_context); > if (status != IB_SUCCESS) > OSM_LOG(pm->log, OSM_LOG_ERROR, "ERR 4C09: " > "Failed to issue port counter query for node 0x%" > @@ -769,6 +787,24 @@ void osm_perfmgr_process(osm_perfmgr_t * pm) > pm->subn->sm_state == IB_SMINFO_STATE_NOTACTIVE) > perfmgr_discovery(pm->subn->p_osm); > > + /* if redirection enabled, determine local port */ > + if (pm->subn->opt.perfmgr_redir && pm->local_port == -1) { > + osm_node_t *p_node; > + osm_port_t *p_port; > + > + CL_PLOCK_ACQUIRE(pm->sm->p_lock); > + p_port = osm_get_port_by_guid(pm->subn, pm->port_guid); > + if (p_port) { > + p_node = p_port->p_node; > + CL_ASSERT(p_node); > + pm->local_port = > + ib_node_info_get_local_port_num(&p_node->node_info); > + } else > + OSM_LOG(pm->log, OSM_LOG_ERROR, > + "ERR 4C87: No PerfMgr port object\n"); > + CL_PLOCK_RELEASE(pm->sm->p_lock); > + } > + > #if ENABLE_OSM_PERF_MGR_PROFILE > gettimeofday(&before, NULL); > #endif > @@ -932,8 +968,8 @@ static int counter_overflow_32(ib_net32_t val) > * MAD to the port. > **********************************************************************/ > static void perfmgr_check_overflow(osm_perfmgr_t * pm, > - monitored_node_t * mon_node, uint8_t port, > - ib_port_counters_t * pc) > + monitored_node_t * mon_node, int16_t pkey_ix, > + uint8_t port, ib_port_counters_t * pc) > { > osm_madw_context_t mad_context; > ib_api_status_t status; > @@ -960,6 +996,9 @@ static void perfmgr_check_overflow(osm_perfmgr_t * pm, > osm_node_t *p_node = NULL; > ib_net16_t lid = 0; > > + if (!mon_node->port[port].valid) > + goto Exit; > + > osm_log(pm->log, OSM_LOG_VERBOSE, > "PerfMgr: Counter overflow: %s (0x%" PRIx64 > ") port %d; clearing counters\n", > @@ -984,8 +1023,9 @@ static void perfmgr_check_overflow(osm_perfmgr_t * pm, > mad_context.perfmgr_context.port = port; > mad_context.perfmgr_context.mad_method = IB_MAD_METHOD_SET; > /* clear port counters */ > - status = perfmgr_send_pc_mad(pm, lid, remote_qp, port, > - IB_MAD_METHOD_SET, &mad_context); > + status = perfmgr_send_pc_mad(pm, lid, remote_qp, pkey_ix, > + port, IB_MAD_METHOD_SET, > + &mad_context); > if (status != IB_SUCCESS) > OSM_LOG(pm->log, OSM_LOG_ERROR, "PerfMgr: ERR 4C11: " > "Failed to send clear counters MAD for %s (0x%" > @@ -1043,6 +1083,64 @@ static void perfmgr_log_events(osm_perfmgr_t * pm, > time_diff, mon_node->name, mon_node->guid, port); > } > > +static int16_t validate_redir_pkey(osm_perfmgr_t *pm, ib_net16_t pkey) > +{ > + int16_t pkey_ix = -1; > + osm_port_t *p_port; > + osm_pkey_tbl_t *p_pkey_tbl; > + ib_net16_t *p_orig_pkey; > + uint16_t block; > + uint8_t index; > + > + OSM_LOG_ENTER(pm->log); > + > + CL_PLOCK_ACQUIRE(pm->sm->p_lock); > + p_port = osm_get_port_by_guid(pm->subn, pm->port_guid); > + if (!p_port) { > + CL_PLOCK_RELEASE(pm->sm->p_lock); > + OSM_LOG(pm->log, OSM_LOG_ERROR, > + "ERR 4C1E: No PerfMgr port object\n"); > + goto Exit; > + } > + if (p_port->p_physp && osm_physp_is_valid(p_port->p_physp)) { > + p_pkey_tbl = &p_port->p_physp->pkeys; > + if (!p_pkey_tbl) { > + CL_PLOCK_RELEASE(pm->sm->p_lock); > + OSM_LOG(pm->log, OSM_LOG_VERBOSE, > + "No PKey table found for PerfMgr port\n"); > + goto Exit; > + } > + p_orig_pkey = cl_map_get(&p_pkey_tbl->keys, > + ib_pkey_get_base(pkey)); > + if (!p_orig_pkey) { > + CL_PLOCK_RELEASE(pm->sm->p_lock); > + OSM_LOG(pm->log, OSM_LOG_VERBOSE, > + "PKey 0x%x not found for PerfMgr port\n", > + cl_ntoh16(pkey)); > + goto Exit; > + } > + if (osm_pkey_tbl_get_block_and_idx(p_pkey_tbl, p_orig_pkey, > + &block, &index) == IB_SUCCESS) { > + CL_PLOCK_RELEASE(pm->sm->p_lock); > + pkey_ix = block * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + index; > + } else { > + CL_PLOCK_RELEASE(pm->sm->p_lock); > + OSM_LOG(pm->log, OSM_LOG_ERROR, > + "ERR 0x4C1F: Failed to obtain P_Key 0x%04x " > + "block and index for PerfMgr port\n", > + cl_ntoh16(pkey)); > + } > + } else { > + CL_PLOCK_RELEASE(pm->sm->p_lock); > + OSM_LOG(pm->log, OSM_LOG_ERROR, > + "ERR 4C20: Local PerfMgt port physp invalid\n"); > + } > + > +Exit: > + OSM_LOG_EXIT(pm->log); > + return pkey_ix; > +} > + > /********************************************************************** > * The dispatcher uses a thread pool which will call this function when > * there is a thread available to process the mad received on the wire. > @@ -1061,6 +1159,8 @@ static void pc_recv_process(void *context, void *data) > perfmgr_db_data_cnt_reading_t data_reading; > cl_map_item_t *p_node; > monitored_node_t *p_mon_node; > + int16_t pkey_ix = 0; > + boolean_t valid = TRUE; > > OSM_LOG_ENTER(pm->log); > > @@ -1084,7 +1184,8 @@ static void pc_recv_process(void *context, void *data) > p_mad->attr_id == IB_MAD_ATTR_CLASS_PORT_INFO); > > /* Response could also be redirection (IBM eHCA PMA does this) */ > - if (p_mad->attr_id == IB_MAD_ATTR_CLASS_PORT_INFO) { > + if (p_mad->status & IB_MAD_STATUS_REDIRECT && > + p_mad->attr_id == IB_MAD_ATTR_CLASS_PORT_INFO) { > char gid_str[INET6_ADDRSTRLEN]; > ib_class_port_info_t *cpi = > (ib_class_port_info_t *) & > @@ -1097,17 +1198,46 @@ static void pc_recv_process(void *context, void *data) > inet_ntop(AF_INET6, cpi->redir_gid.raw, gid_str, > sizeof gid_str), cl_ntoh32(cpi->redir_qp)); > > - /* LID or GID redirection ? */ > - /* For GID redirection, need to get PathRecord from SA */ > + if (!pm->subn->opt.perfmgr_redir) { > + OSM_LOG(pm->log, OSM_LOG_VERBOSE, > + "Redirection requested but disabled\n"); > + valid = FALSE; > + } > + > + /* valid redirection ? */ > if (cpi->redir_lid == 0) { > + if (!ib_gid_is_notzero(&cpi->redir_gid)) { > + OSM_LOG(pm->log, OSM_LOG_VERBOSE, > + "Invalid redirection " > + "(both redirect LID and GID are zero)\n"); > + valid = FALSE; > + } > + } > + if (cpi->redir_qp == 0) { > + OSM_LOG(pm->log, OSM_LOG_VERBOSE, "Invalid RedirectQP\n"); > + valid = FALSE; > + } > + if (cpi->redir_pkey == 0) { > + OSM_LOG(pm->log, OSM_LOG_VERBOSE, "Invalid RedirectP_Key\n"); > + valid = FALSE; > + } > + if (cpi->redir_qkey != IB_QP1_WELL_KNOWN_Q_KEY) { > + OSM_LOG(pm->log, OSM_LOG_VERBOSE, "Invalid RedirectQ_Key\n"); > + valid = FALSE; > + } > + > + pkey_ix = validate_redir_pkey(pm, cpi->redir_pkey); > + if (pkey_ix == -1) { > OSM_LOG(pm->log, OSM_LOG_VERBOSE, > - "GID redirection not currently implemented!\n"); > - goto Exit; > + "Index for Pkey 0x%x not found\n", > + cl_ntoh16(cpi->redir_pkey)); > + valid = FALSE; > } > > - if (!pm->subn->opt.perfmgr_redir) { > - OSM_LOG(pm->log, OSM_LOG_ERROR, "ERR 4C16: " > - "redirection requested but disabled\n"); > + if (cpi->redir_lid == 0) { > + /* GID redirection: get PathRecord information */ > + OSM_LOG(pm->log, OSM_LOG_VERBOSE, > + "GID redirection not currently supported\n"); > goto Exit; > } > > @@ -1122,13 +1252,24 @@ static void pc_recv_process(void *context, void *data) > p_mon_node->num_ports); > goto Exit; > } > - p_mon_node->redir_port[port].redir_lid = cpi->redir_lid; > - p_mon_node->redir_port[port].redir_qp = cpi->redir_qp; > + p_mon_node->port[port].redirection = TRUE; > + p_mon_node->port[port].valid = valid; > + memcpy(&p_mon_node->port[port].gid, &cpi->redir_gid, > + sizeof(ib_gid_t)); > + p_mon_node->port[port].lid = cpi->redir_lid; > + p_mon_node->port[port].qp = cpi->redir_qp; > + p_mon_node->port[port].pkey = cpi->redir_pkey; > + if (pkey_ix != -1) > + p_mon_node->port[port].pkey_ix = pkey_ix; > cl_plock_release(&pm->osm->lock); > > + if (!valid) > + goto Exit; > + > /* Finally, reissue the query to the redirected location */ > status = perfmgr_send_pc_mad(pm, cpi->redir_lid, cpi->redir_qp, > - port, mad_context->perfmgr_context. > + pkey_ix, port, > + mad_context->perfmgr_context. > mad_method, mad_context); > if (status != IB_SUCCESS) > OSM_LOG(pm->log, OSM_LOG_ERROR, "ERR 4C14: " > @@ -1163,7 +1304,7 @@ static void pc_recv_process(void *context, void *data) > perfmgr_db_clear_prev_dc(pm->db, node_guid, port); > } > > - perfmgr_check_overflow(pm, p_mon_node, port, wire_read); > + perfmgr_check_overflow(pm, p_mon_node, pkey_ix, port, wire_read); > > #if ENABLE_OSM_PERF_MGR_PROFILE > do { > @@ -1208,6 +1349,7 @@ ib_api_status_t osm_perfmgr_init(osm_perfmgr_t * pm, osm_opensm_t * osm, > pm->sweep_time_s = p_opt->perfmgr_sweep_time_s; > pm->max_outstanding_queries = p_opt->perfmgr_max_outstanding_queries; > pm->osm = osm; > + pm->local_port = -1; > > status = cl_timer_init(&pm->sweep_timer, perfmgr_sweep, pm); > if (status != IB_SUCCESS) > -- > To unsubscribe from this list: send the line "unsubscribe linux-rdma" in > the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org > More majordomo info at http://*vger.kernel.org/majordomo-info.html > -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org More majordomo info at http://vger.kernel.org/majordomo-info.html ^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [PATCHv5 1/2][RESEND] opensm/PerfMgr: Better redirection support [not found] ` <20100617172849.GA17245-Wuw85uim5zDR7s880joybQ@public.gmane.org> 2010-07-14 1:28 ` Ira Weiny @ 2011-03-13 15:54 ` Alex Netes 1 sibling, 0 replies; 3+ messages in thread From: Alex Netes @ 2011-03-13 15:54 UTC (permalink / raw) To: Hal Rosenstock Cc: Sasha Khapyorsky, linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org On 20:28 Thu 17 Jun , Hal Rosenstock wrote: > > Handle PKey and QPN redirection information > GID redirection handling remains > > Signed-off-by: Hal Rosenstock <hal.rosenstock-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org> > --- > Changes since v4: > Fixed some trailing whitespace problems > > Changes since v3: > Rebased > > Changes since v2: > Use OpenSM DB rather than vendor layer for local port number and PKeys > Change most log levels from ERROR to VERBOSE > Redirection info validity now determined by single flag > validate_redir_pkey returns pkey index or -1 rather than boolean > Removed redir_ prefixes > > Changes since v1: > Added include of osm_helper.h to osm_perfmgr.c > Applied. Thanks. -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org More majordomo info at http://vger.kernel.org/majordomo-info.html ^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2011-03-13 15:54 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2010-06-17 17:28 [PATCHv5 1/2][RESEND] opensm/PerfMgr: Better redirection support Hal Rosenstock
[not found] ` <20100617172849.GA17245-Wuw85uim5zDR7s880joybQ@public.gmane.org>
2010-07-14 1:28 ` Ira Weiny
2011-03-13 15:54 ` Alex Netes
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox