[PATCH] opensm: add force_link_speed_file config option.

[PATCH] opensm: add force_link_speed_file config option.

[Ira Weiny]

From: Ira Weiny <weiny2-i2BcT+NCU+M@public.gmane.org>
Date: Wed, 14 Oct 2009 16:09:25 -0700
Subject: [PATCH] opensm: add force_link_speed_file config option.

	From the config file help:

	# Optional file used to override individual port GUIDs and speeds to be forced.
	# Port GUIDs which are _not_ listed in this file will be set to "force_link_speed"
	# Speeds specified in this file may be higher or lower than the global
	# setting (force_link_speed) above
	# File Format:
	# # comment
	# <port_guid> [<setting>] [# comment]
	# ...
	# if the optional <setting> is not specified the default (15) will be used

This option along with the force_link_speed option can be used to override the
PortInfo:LinkSpeedSupported option for hardware which is not operating
properly.  Previously one could only force a singe speed across a fabric.  With
this option one can set speeds on an individual node basis.

Signed-off-by: Ira Weiny <weiny2-i2BcT+NCU+M@public.gmane.org>
---
 opensm/include/opensm/osm_base.h   |   10 ++++
 opensm/include/opensm/osm_sm.h     |    1 +
 opensm/include/opensm/osm_subnet.h |    6 ++
 opensm/opensm/osm_link_mgr.c       |   22 +++++++-
 opensm/opensm/osm_sm.c             |    2 +
 opensm/opensm/osm_subnet.c         |   99 +++++++++++++++++++++++++++++++++++-
 6 files changed, 135 insertions(+), 5 deletions(-)

diff --git a/opensm/include/opensm/osm_base.h b/opensm/include/opensm/osm_base.h
index 06223ce..5457bc6 100644
--- a/opensm/include/opensm/osm_base.h
+++ b/opensm/include/opensm/osm_base.h
@@ -146,6 +146,16 @@ BEGIN_C_DECLS
 * SYNOPSIS
 */
 #define OSM_DEFAULT_MAX_OP_VLS 5
+/****s* OpenSM: Base/OSM_DEFAULT_FORCE_LINK_SPEED
+* NAME
+*	OSM_DEFAULT_FORCE_LINK_SPEED
+*
+* DESCRIPTION
+*	Default force link speed value
+*
+* SYNOPSIS
+*/
+#define OSM_DEFAULT_FORCE_LINK_SPEED 15
 /********/
 /****s* OpenSM: Base/OSM_DEFAULT_SL
 * NAME
diff --git a/opensm/include/opensm/osm_sm.h b/opensm/include/opensm/osm_sm.h
index e653d49..7086ca6 100644
--- a/opensm/include/opensm/osm_sm.h
+++ b/opensm/include/opensm/osm_sm.h
@@ -143,6 +143,7 @@ typedef struct osm_sm {
 	cl_disp_reg_handle_t slvl_disp_h;
 	cl_disp_reg_handle_t vla_disp_h;
 	cl_disp_reg_handle_t pkey_disp_h;
+	cl_map_t force_link_speed_except_guids;
 } osm_sm_t;
 /*
 * FIELDS
diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h
index 9488225..5353298 100644
--- a/opensm/include/opensm/osm_subnet.h
+++ b/opensm/include/opensm/osm_subnet.h
@@ -154,6 +154,7 @@ typedef struct osm_subn_opt {
 	boolean_t lmc_esp0;
 	uint8_t max_op_vls;
 	uint8_t force_link_speed;
+	char *force_link_speed_file;
 	boolean_t reassign_lids;
 	boolean_t ignore_other_sm;
 	boolean_t single_thread;
@@ -469,6 +470,11 @@ typedef struct osm_subn_opt {
 *	Subnet object
 *********/
 
+typedef struct force_speed_exception {
+	uint64_t port_guid;
+	uint8_t val;
+} force_speed_exception_t;
+
 /****s* OpenSM: Subnet/osm_subn_t
 * NAME
 *	osm_subn_t
diff --git a/opensm/opensm/osm_link_mgr.c b/opensm/opensm/osm_link_mgr.c
index c9bdfee..21dcc50 100644
--- a/opensm/opensm/osm_link_mgr.c
+++ b/opensm/opensm/osm_link_mgr.c
@@ -106,6 +106,8 @@ static int link_mgr_set_physp_pi(osm_sm_t * sm, IN osm_physp_t * p_physp,
 	boolean_t esp0 = FALSE, send_set = FALSE;
 	osm_physp_t *p_remote_physp;
 	int ret = 0;
+	uint8_t force_link_speed;
+	force_speed_exception_t *except;
 
 	OSM_LOG_ENTER(sm->p_log);
 
@@ -326,12 +328,26 @@ static int link_mgr_set_physp_pi(osm_sm_t * sm, IN osm_physp_t * p_physp,
 			   sizeof(p_pi->link_width_enabled)))
 			send_set = TRUE;
 
-		if (sm->p_subn->opt.force_link_speed &&
-		    (sm->p_subn->opt.force_link_speed != 15 ||
+		/* use the force link speed set in the config */
+		force_link_speed = sm->p_subn->opt.force_link_speed;
+		/* unless this port guid is in our "except"ion file */
+		if ((except = cl_map_get(
+				&sm->force_link_speed_except_guids,
+				cl_ntoh64(p_physp->port_guid)))
+					!= NULL) {
+			force_link_speed = except->val;
+			OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
+				"Skipping force_link_speed on GUID 0x%016"
+				PRIx64 " (%d)\n",
+			       cl_ntoh64(p_physp->port_guid),
+			       force_link_speed);
+		}
+
+		if (force_link_speed &&
+		     (force_link_speed != OSM_DEFAULT_FORCE_LINK_SPEED ||
 		     ib_port_info_get_link_speed_enabled(p_pi) !=
 		     ib_port_info_get_link_speed_sup(p_pi))) {
 			ib_port_info_set_link_speed_enabled(p_pi,
-							    sm->p_subn->opt.
 							    force_link_speed);
 			if (memcmp(&p_pi->link_speed, &p_old_pi->link_speed,
 				   sizeof(p_pi->link_speed)))
diff --git a/opensm/opensm/osm_sm.c b/opensm/opensm/osm_sm.c
index f780124..e5cf2b4 100644
--- a/opensm/opensm/osm_sm.c
+++ b/opensm/opensm/osm_sm.c
@@ -168,6 +168,8 @@ void osm_sm_construct(IN osm_sm_t * p_sm)
 	osm_sm_mad_ctrl_construct(&p_sm->mad_ctrl);
 	osm_lid_mgr_construct(&p_sm->lid_mgr);
 	osm_ucast_mgr_construct(&p_sm->ucast_mgr);
+	cl_map_construct(&p_sm->force_link_speed_except_guids);
+	cl_map_init(&p_sm->force_link_speed_except_guids, 10);
 }
 
 /**********************************************************************
diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c
index 647950e..6aa338f 100644
--- a/opensm/opensm/osm_subnet.c
+++ b/opensm/opensm/osm_subnet.c
@@ -46,6 +46,7 @@
 #  include <config.h>
 #endif				/* HAVE_CONFIG_H */
 
+#include <sys/stat.h>
 #include <string.h>
 #include <stdio.h>
 #include <stdarg.h>
@@ -305,6 +306,7 @@ static const opt_rec_t opt_tbl[] = {
 	{ "lmc_esp0", OPT_OFFSET(lmc_esp0), opts_parse_boolean, NULL, 1 },
 	{ "max_op_vls", OPT_OFFSET(max_op_vls), opts_parse_uint8, NULL, 1 },
 	{ "force_link_speed", OPT_OFFSET(force_link_speed), opts_parse_uint8, NULL, 1 },
+	{ "force_link_speed_file", OPT_OFFSET(force_link_speed_file), opts_parse_charp, NULL, 1 },
 	{ "reassign_lids", OPT_OFFSET(reassign_lids), opts_parse_boolean, NULL, 1 },
 	{ "ignore_other_sm", OPT_OFFSET(ignore_other_sm), opts_parse_boolean, NULL, 1 },
 	{ "single_thread", OPT_OFFSET(single_thread), opts_parse_boolean, NULL, 0 },
@@ -700,7 +702,8 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * p_opt)
 	p_opt->lmc = OSM_DEFAULT_LMC;
 	p_opt->lmc_esp0 = FALSE;
 	p_opt->max_op_vls = OSM_DEFAULT_MAX_OP_VLS;
-	p_opt->force_link_speed = 15;
+	p_opt->force_link_speed = OSM_DEFAULT_FORCE_LINK_SPEED;
+	p_opt->force_link_speed_file = NULL;
 	p_opt->reassign_lids = FALSE;
 	p_opt->ignore_other_sm = FALSE;
 	p_opt->single_thread = FALSE;
@@ -927,6 +930,73 @@ static ib_api_status_t osm_parse_prefix_routes_file(IN osm_subn_t * p_subn)
 
 /**********************************************************************
  **********************************************************************/
+void osm_parse_force_link_speed_except_file(IN osm_opensm_t * const p_osm,
+					    IN char *file)
+{
+	FILE *fp;
+	char line[1024];
+	uint64_t guid = 0;
+	force_speed_exception_t *except = NULL;
+	cl_map_t *map = &p_osm->sm.force_link_speed_except_guids;
+	cl_map_iterator_t head;
+
+	CL_ASSERT(cl_is_map_inited(map));
+
+	/* clear previous entries */
+	head = cl_map_head(map);
+	while (head != cl_map_end(map)) {
+		except = (force_speed_exception_t *)cl_map_obj(head);
+		cl_map_remove_item(map, head);
+		free(except);
+		head = cl_map_head(map);
+	}
+
+	if (file) {
+		fp = fopen(file, "r");
+		if (!fp) {
+			osm_log(&p_osm->log, OSM_LOG_ERROR,
+				"failed to open force_link_speed_file "
+				"\'%s\'\n",
+				file);
+			return;
+		}
+
+		while (fgets(line, sizeof(line) - 1, fp) != NULL) {
+			char *p;
+			char *guid_str, *force_str;
+			unsigned long force_val;
+
+			p = strchr(line, '#');
+			if (p)
+				*p = '\0';
+
+			guid_str = strtok_r(line, " \t\n", &p);
+			if (guid_str &&
+			   ((guid = strtoull(guid_str, NULL, 0)) != 0)) {
+				except = calloc(1, sizeof *except);
+
+				force_str = strtok_r(NULL, " \t\n", &p);
+				if (force_str) {
+					force_val = strtoul(force_str, NULL, 0);
+				} else
+					force_val = OSM_DEFAULT_FORCE_LINK_SPEED;
+
+				except->port_guid = guid;
+				except->val = (uint8_t)force_val;
+				OSM_LOG(&p_osm->log, OSM_LOG_DEBUG,
+					"Setting exception to "
+					"force_link_speed on GUID 0x%016"
+					PRIx64 " (%d)\n", except->port_guid, except->val);
+				cl_map_insert(map, guid, (void *)except);
+			}
+		}
+
+		fclose(fp);
+	}
+}
+
+/**********************************************************************
+ **********************************************************************/
 static void subn_verify_max_vls(unsigned *max_vls, const char *prefix, unsigned dflt)
 {
 	if (!*max_vls || *max_vls > 15) {
@@ -1097,6 +1167,15 @@ int osm_subn_verify_config(IN osm_subn_opt_t * p_opts)
 		p_opts->max_wire_smps = OSM_DEFAULT_SMP_MAX_ON_WIRE;
 	}
 
+	if (p_opts->force_link_speed_file) {
+		struct stat statbuf;
+		if (stat(p_opts->force_link_speed_file, &statbuf))
+			log_report(" Invalid Cached Option "
+				   "Value:force_link_speed_file = %s"
+				   "; file not found\n",
+				   p_opts->force_link_speed_file);
+	}
+
 	if (strcmp(p_opts->console, OSM_DISABLE_CONSOLE)
 	    && strcmp(p_opts->console, OSM_LOCAL_CONSOLE)
 #ifdef ENABLE_OSM_CONSOLE_SOCKET
@@ -1265,6 +1344,9 @@ int osm_subn_rescan_conf_files(IN osm_subn_t * p_subn)
 
 	osm_parse_prefix_routes_file(p_subn);
 
+	osm_parse_force_link_speed_except_file(p_subn->p_osm,
+					       p_opts->force_link_speed_file);
+
 	return 0;
 }
 
@@ -1331,8 +1413,18 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t * p_opts)
 		"#    5: 2.5 or 10.0 Gbps\n"
 		"#    7: 2.5 or 5.0 or 10.0 Gbps\n"
 		"#    2,4,6,8-14 Reserved\n"
-		"#    Default 15: set to PortInfo:LinkSpeedSupported\n"
+		"#    Default %d: set to PortInfo:LinkSpeedSupported\n"
 		"force_link_speed %u\n\n"
+		"# Optional file used to override individual port GUIDs and speeds to be forced.\n"
+		"# Port GUIDs which are _not_ listed in this file will be set to \"force_link_speed\"\n"
+		"# Speeds specified in this file may be higher or lower than the global setting\n"
+		"# (force_link_speed) above\n"
+		"# File Format:\n"
+		"# # comment\n"
+		"# <port_guid> [<setting>] [# comment]\n"
+		"# ...\n"
+		"# if the optional <setting> is not specified the default (%d) will be used\n"
+		"force_link_speed_file %s\n\n"
 		"# The subnet_timeout code that will be set for all the ports\n"
 		"# The actual timeout is 4.096usec * 2^<subnet_timeout>\n"
 		"subnet_timeout %u\n\n"
@@ -1355,7 +1447,10 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t * p_opts)
 		p_opts->head_of_queue_lifetime,
 		p_opts->leaf_head_of_queue_lifetime,
 		p_opts->max_op_vls,
+		OSM_DEFAULT_FORCE_LINK_SPEED,
 		p_opts->force_link_speed,
+		OSM_DEFAULT_FORCE_LINK_SPEED,
+		p_opts->force_link_speed_file,
 		p_opts->subnet_timeout,
 		p_opts->local_phy_errors_threshold,
 		p_opts->overrun_errors_threshold);
-- 
1.5.4.5

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] opensm: add force_link_speed_file config option.

[Ira Weiny]

Sasha,

Hal caught that I am not freeing the force_link_speed_except_guids on shutdown.  V2 coming shortly.

Ira

On Thu, 15 Oct 2009 13:26:50 -0700
Ira Weiny <weiny2-i2BcT+NCU+M@public.gmane.org> wrote:

> 
> From: Ira Weiny <weiny2-i2BcT+NCU+M@public.gmane.org>
> Date: Wed, 14 Oct 2009 16:09:25 -0700
> Subject: [PATCH] opensm: add force_link_speed_file config option.
> 
> 	From the config file help:
> 
> 	# Optional file used to override individual port GUIDs and speeds to be forced.
> 	# Port GUIDs which are _not_ listed in this file will be set to "force_link_speed"
> 	# Speeds specified in this file may be higher or lower than the global
> 	# setting (force_link_speed) above
> 	# File Format:
> 	# # comment
> 	# <port_guid> [<setting>] [# comment]
> 	# ...
> 	# if the optional <setting> is not specified the default (15) will be used
> 
> This option along with the force_link_speed option can be used to override the
> PortInfo:LinkSpeedSupported option for hardware which is not operating
> properly.  Previously one could only force a singe speed across a fabric.  With
> this option one can set speeds on an individual node basis.
> 
> Signed-off-by: Ira Weiny <weiny2-i2BcT+NCU+M@public.gmane.org>
> ---
>  opensm/include/opensm/osm_base.h   |   10 ++++
>  opensm/include/opensm/osm_sm.h     |    1 +
>  opensm/include/opensm/osm_subnet.h |    6 ++
>  opensm/opensm/osm_link_mgr.c       |   22 +++++++-
>  opensm/opensm/osm_sm.c             |    2 +
>  opensm/opensm/osm_subnet.c         |   99 +++++++++++++++++++++++++++++++++++-
>  6 files changed, 135 insertions(+), 5 deletions(-)
> 
> diff --git a/opensm/include/opensm/osm_base.h b/opensm/include/opensm/osm_base.h
> index 06223ce..5457bc6 100644
> --- a/opensm/include/opensm/osm_base.h
> +++ b/opensm/include/opensm/osm_base.h
> @@ -146,6 +146,16 @@ BEGIN_C_DECLS
>  * SYNOPSIS
>  */
>  #define OSM_DEFAULT_MAX_OP_VLS 5
> +/****s* OpenSM: Base/OSM_DEFAULT_FORCE_LINK_SPEED
> +* NAME
> +*	OSM_DEFAULT_FORCE_LINK_SPEED
> +*
> +* DESCRIPTION
> +*	Default force link speed value
> +*
> +* SYNOPSIS
> +*/
> +#define OSM_DEFAULT_FORCE_LINK_SPEED 15
>  /********/
>  /****s* OpenSM: Base/OSM_DEFAULT_SL
>  * NAME
> diff --git a/opensm/include/opensm/osm_sm.h b/opensm/include/opensm/osm_sm.h
> index e653d49..7086ca6 100644
> --- a/opensm/include/opensm/osm_sm.h
> +++ b/opensm/include/opensm/osm_sm.h
> @@ -143,6 +143,7 @@ typedef struct osm_sm {
>  	cl_disp_reg_handle_t slvl_disp_h;
>  	cl_disp_reg_handle_t vla_disp_h;
>  	cl_disp_reg_handle_t pkey_disp_h;
> +	cl_map_t force_link_speed_except_guids;
>  } osm_sm_t;
>  /*
>  * FIELDS
> diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h
> index 9488225..5353298 100644
> --- a/opensm/include/opensm/osm_subnet.h
> +++ b/opensm/include/opensm/osm_subnet.h
> @@ -154,6 +154,7 @@ typedef struct osm_subn_opt {
>  	boolean_t lmc_esp0;
>  	uint8_t max_op_vls;
>  	uint8_t force_link_speed;
> +	char *force_link_speed_file;
>  	boolean_t reassign_lids;
>  	boolean_t ignore_other_sm;
>  	boolean_t single_thread;
> @@ -469,6 +470,11 @@ typedef struct osm_subn_opt {
>  *	Subnet object
>  *********/
>  
> +typedef struct force_speed_exception {
> +	uint64_t port_guid;
> +	uint8_t val;
> +} force_speed_exception_t;
> +
>  /****s* OpenSM: Subnet/osm_subn_t
>  * NAME
>  *	osm_subn_t
> diff --git a/opensm/opensm/osm_link_mgr.c b/opensm/opensm/osm_link_mgr.c
> index c9bdfee..21dcc50 100644
> --- a/opensm/opensm/osm_link_mgr.c
> +++ b/opensm/opensm/osm_link_mgr.c
> @@ -106,6 +106,8 @@ static int link_mgr_set_physp_pi(osm_sm_t * sm, IN osm_physp_t * p_physp,
>  	boolean_t esp0 = FALSE, send_set = FALSE;
>  	osm_physp_t *p_remote_physp;
>  	int ret = 0;
> +	uint8_t force_link_speed;
> +	force_speed_exception_t *except;
>  
>  	OSM_LOG_ENTER(sm->p_log);
>  
> @@ -326,12 +328,26 @@ static int link_mgr_set_physp_pi(osm_sm_t * sm, IN osm_physp_t * p_physp,
>  			   sizeof(p_pi->link_width_enabled)))
>  			send_set = TRUE;
>  
> -		if (sm->p_subn->opt.force_link_speed &&
> -		    (sm->p_subn->opt.force_link_speed != 15 ||
> +		/* use the force link speed set in the config */
> +		force_link_speed = sm->p_subn->opt.force_link_speed;
> +		/* unless this port guid is in our "except"ion file */
> +		if ((except = cl_map_get(
> +				&sm->force_link_speed_except_guids,
> +				cl_ntoh64(p_physp->port_guid)))
> +					!= NULL) {
> +			force_link_speed = except->val;
> +			OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
> +				"Skipping force_link_speed on GUID 0x%016"
> +				PRIx64 " (%d)\n",
> +			       cl_ntoh64(p_physp->port_guid),
> +			       force_link_speed);
> +		}
> +
> +		if (force_link_speed &&
> +		     (force_link_speed != OSM_DEFAULT_FORCE_LINK_SPEED ||
>  		     ib_port_info_get_link_speed_enabled(p_pi) !=
>  		     ib_port_info_get_link_speed_sup(p_pi))) {
>  			ib_port_info_set_link_speed_enabled(p_pi,
> -							    sm->p_subn->opt.
>  							    force_link_speed);
>  			if (memcmp(&p_pi->link_speed, &p_old_pi->link_speed,
>  				   sizeof(p_pi->link_speed)))
> diff --git a/opensm/opensm/osm_sm.c b/opensm/opensm/osm_sm.c
> index f780124..e5cf2b4 100644
> --- a/opensm/opensm/osm_sm.c
> +++ b/opensm/opensm/osm_sm.c
> @@ -168,6 +168,8 @@ void osm_sm_construct(IN osm_sm_t * p_sm)
>  	osm_sm_mad_ctrl_construct(&p_sm->mad_ctrl);
>  	osm_lid_mgr_construct(&p_sm->lid_mgr);
>  	osm_ucast_mgr_construct(&p_sm->ucast_mgr);
> +	cl_map_construct(&p_sm->force_link_speed_except_guids);
> +	cl_map_init(&p_sm->force_link_speed_except_guids, 10);
>  }
>  
>  /**********************************************************************
> diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c
> index 647950e..6aa338f 100644
> --- a/opensm/opensm/osm_subnet.c
> +++ b/opensm/opensm/osm_subnet.c
> @@ -46,6 +46,7 @@
>  #  include <config.h>
>  #endif				/* HAVE_CONFIG_H */
>  
> +#include <sys/stat.h>
>  #include <string.h>
>  #include <stdio.h>
>  #include <stdarg.h>
> @@ -305,6 +306,7 @@ static const opt_rec_t opt_tbl[] = {
>  	{ "lmc_esp0", OPT_OFFSET(lmc_esp0), opts_parse_boolean, NULL, 1 },
>  	{ "max_op_vls", OPT_OFFSET(max_op_vls), opts_parse_uint8, NULL, 1 },
>  	{ "force_link_speed", OPT_OFFSET(force_link_speed), opts_parse_uint8, NULL, 1 },
> +	{ "force_link_speed_file", OPT_OFFSET(force_link_speed_file), opts_parse_charp, NULL, 1 },
>  	{ "reassign_lids", OPT_OFFSET(reassign_lids), opts_parse_boolean, NULL, 1 },
>  	{ "ignore_other_sm", OPT_OFFSET(ignore_other_sm), opts_parse_boolean, NULL, 1 },
>  	{ "single_thread", OPT_OFFSET(single_thread), opts_parse_boolean, NULL, 0 },
> @@ -700,7 +702,8 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * p_opt)
>  	p_opt->lmc = OSM_DEFAULT_LMC;
>  	p_opt->lmc_esp0 = FALSE;
>  	p_opt->max_op_vls = OSM_DEFAULT_MAX_OP_VLS;
> -	p_opt->force_link_speed = 15;
> +	p_opt->force_link_speed = OSM_DEFAULT_FORCE_LINK_SPEED;
> +	p_opt->force_link_speed_file = NULL;
>  	p_opt->reassign_lids = FALSE;
>  	p_opt->ignore_other_sm = FALSE;
>  	p_opt->single_thread = FALSE;
> @@ -927,6 +930,73 @@ static ib_api_status_t osm_parse_prefix_routes_file(IN osm_subn_t * p_subn)
>  
>  /**********************************************************************
>   **********************************************************************/
> +void osm_parse_force_link_speed_except_file(IN osm_opensm_t * const p_osm,
> +					    IN char *file)
> +{
> +	FILE *fp;
> +	char line[1024];
> +	uint64_t guid = 0;
> +	force_speed_exception_t *except = NULL;
> +	cl_map_t *map = &p_osm->sm.force_link_speed_except_guids;
> +	cl_map_iterator_t head;
> +
> +	CL_ASSERT(cl_is_map_inited(map));
> +
> +	/* clear previous entries */
> +	head = cl_map_head(map);
> +	while (head != cl_map_end(map)) {
> +		except = (force_speed_exception_t *)cl_map_obj(head);
> +		cl_map_remove_item(map, head);
> +		free(except);
> +		head = cl_map_head(map);
> +	}
> +
> +	if (file) {
> +		fp = fopen(file, "r");
> +		if (!fp) {
> +			osm_log(&p_osm->log, OSM_LOG_ERROR,
> +				"failed to open force_link_speed_file "
> +				"\'%s\'\n",
> +				file);
> +			return;
> +		}
> +
> +		while (fgets(line, sizeof(line) - 1, fp) != NULL) {
> +			char *p;
> +			char *guid_str, *force_str;
> +			unsigned long force_val;
> +
> +			p = strchr(line, '#');
> +			if (p)
> +				*p = '\0';
> +
> +			guid_str = strtok_r(line, " \t\n", &p);
> +			if (guid_str &&
> +			   ((guid = strtoull(guid_str, NULL, 0)) != 0)) {
> +				except = calloc(1, sizeof *except);
> +
> +				force_str = strtok_r(NULL, " \t\n", &p);
> +				if (force_str) {
> +					force_val = strtoul(force_str, NULL, 0);
> +				} else
> +					force_val = OSM_DEFAULT_FORCE_LINK_SPEED;
> +
> +				except->port_guid = guid;
> +				except->val = (uint8_t)force_val;
> +				OSM_LOG(&p_osm->log, OSM_LOG_DEBUG,
> +					"Setting exception to "
> +					"force_link_speed on GUID 0x%016"
> +					PRIx64 " (%d)\n", except->port_guid, except->val);
> +				cl_map_insert(map, guid, (void *)except);
> +			}
> +		}
> +
> +		fclose(fp);
> +	}
> +}
> +
> +/**********************************************************************
> + **********************************************************************/
>  static void subn_verify_max_vls(unsigned *max_vls, const char *prefix, unsigned dflt)
>  {
>  	if (!*max_vls || *max_vls > 15) {
> @@ -1097,6 +1167,15 @@ int osm_subn_verify_config(IN osm_subn_opt_t * p_opts)
>  		p_opts->max_wire_smps = OSM_DEFAULT_SMP_MAX_ON_WIRE;
>  	}
>  
> +	if (p_opts->force_link_speed_file) {
> +		struct stat statbuf;
> +		if (stat(p_opts->force_link_speed_file, &statbuf))
> +			log_report(" Invalid Cached Option "
> +				   "Value:force_link_speed_file = %s"
> +				   "; file not found\n",
> +				   p_opts->force_link_speed_file);
> +	}
> +
>  	if (strcmp(p_opts->console, OSM_DISABLE_CONSOLE)
>  	    && strcmp(p_opts->console, OSM_LOCAL_CONSOLE)
>  #ifdef ENABLE_OSM_CONSOLE_SOCKET
> @@ -1265,6 +1344,9 @@ int osm_subn_rescan_conf_files(IN osm_subn_t * p_subn)
>  
>  	osm_parse_prefix_routes_file(p_subn);
>  
> +	osm_parse_force_link_speed_except_file(p_subn->p_osm,
> +					       p_opts->force_link_speed_file);
> +
>  	return 0;
>  }
>  
> @@ -1331,8 +1413,18 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t * p_opts)
>  		"#    5: 2.5 or 10.0 Gbps\n"
>  		"#    7: 2.5 or 5.0 or 10.0 Gbps\n"
>  		"#    2,4,6,8-14 Reserved\n"
> -		"#    Default 15: set to PortInfo:LinkSpeedSupported\n"
> +		"#    Default %d: set to PortInfo:LinkSpeedSupported\n"
>  		"force_link_speed %u\n\n"
> +		"# Optional file used to override individual port GUIDs and speeds to be forced.\n"
> +		"# Port GUIDs which are _not_ listed in this file will be set to \"force_link_speed\"\n"
> +		"# Speeds specified in this file may be higher or lower than the global setting\n"
> +		"# (force_link_speed) above\n"
> +		"# File Format:\n"
> +		"# # comment\n"
> +		"# <port_guid> [<setting>] [# comment]\n"
> +		"# ...\n"
> +		"# if the optional <setting> is not specified the default (%d) will be used\n"
> +		"force_link_speed_file %s\n\n"
>  		"# The subnet_timeout code that will be set for all the ports\n"
>  		"# The actual timeout is 4.096usec * 2^<subnet_timeout>\n"
>  		"subnet_timeout %u\n\n"
> @@ -1355,7 +1447,10 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t * p_opts)
>  		p_opts->head_of_queue_lifetime,
>  		p_opts->leaf_head_of_queue_lifetime,
>  		p_opts->max_op_vls,
> +		OSM_DEFAULT_FORCE_LINK_SPEED,
>  		p_opts->force_link_speed,
> +		OSM_DEFAULT_FORCE_LINK_SPEED,
> +		p_opts->force_link_speed_file,
>  		p_opts->subnet_timeout,
>  		p_opts->local_phy_errors_threshold,
>  		p_opts->overrun_errors_threshold);
> -- 
> 1.5.4.5
> 


-- 
Ira Weiny
Math Programmer/Computer Scientist
Lawrence Livermore National Lab
925-423-8008
weiny2-i2BcT+NCU+M@public.gmane.org
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] opensm: add force_link_speed_file config option.

[Sasha Khapyorsky]

Hi Ira,

On 13:26 Thu 15 Oct     , Ira Weiny wrote:
> 
> From: Ira Weiny <weiny2-i2BcT+NCU+M@public.gmane.org>
> Date: Wed, 14 Oct 2009 16:09:25 -0700
> Subject: [PATCH] opensm: add force_link_speed_file config option.
> 
> 	From the config file help:
> 
> 	# Optional file used to override individual port GUIDs and speeds to be forced.

And what about width and other potentially tunable port parameters?

I would suggest instead of adding config file per option to develop more
generic config file(s) framework where various per port (or in even more
generic case per port group) PortInfo parameters could be preconfigured.
BTW it could include some existing stuff like sl2vl, vlarb, etc..

Some other comments are below.

> 	# Port GUIDs which are _not_ listed in this file will be set to "force_link_speed"
> 	# Speeds specified in this file may be higher or lower than the global
> 	# setting (force_link_speed) above
> 	# File Format:
> 	# # comment
> 	# <port_guid> [<setting>] [# comment]
> 	# ...
> 	# if the optional <setting> is not specified the default (15) will be used
> 
> This option along with the force_link_speed option can be used to override the
> PortInfo:LinkSpeedSupported option for hardware which is not operating
> properly.  Previously one could only force a singe speed across a fabric.  With
> this option one can set speeds on an individual node basis.
> 
> Signed-off-by: Ira Weiny <weiny2-i2BcT+NCU+M@public.gmane.org>
> ---
>  opensm/include/opensm/osm_base.h   |   10 ++++
>  opensm/include/opensm/osm_sm.h     |    1 +
>  opensm/include/opensm/osm_subnet.h |    6 ++
>  opensm/opensm/osm_link_mgr.c       |   22 +++++++-
>  opensm/opensm/osm_sm.c             |    2 +
>  opensm/opensm/osm_subnet.c         |   99 +++++++++++++++++++++++++++++++++++-
>  6 files changed, 135 insertions(+), 5 deletions(-)
> 
> diff --git a/opensm/include/opensm/osm_base.h b/opensm/include/opensm/osm_base.h
> index 06223ce..5457bc6 100644
> --- a/opensm/include/opensm/osm_base.h
> +++ b/opensm/include/opensm/osm_base.h
> @@ -146,6 +146,16 @@ BEGIN_C_DECLS
>  * SYNOPSIS
>  */
>  #define OSM_DEFAULT_MAX_OP_VLS 5
> +/****s* OpenSM: Base/OSM_DEFAULT_FORCE_LINK_SPEED
> +* NAME
> +*	OSM_DEFAULT_FORCE_LINK_SPEED
> +*
> +* DESCRIPTION
> +*	Default force link speed value
> +*
> +* SYNOPSIS
> +*/
> +#define OSM_DEFAULT_FORCE_LINK_SPEED 15
>  /********/
>  /****s* OpenSM: Base/OSM_DEFAULT_SL
>  * NAME
> diff --git a/opensm/include/opensm/osm_sm.h b/opensm/include/opensm/osm_sm.h
> index e653d49..7086ca6 100644
> --- a/opensm/include/opensm/osm_sm.h
> +++ b/opensm/include/opensm/osm_sm.h
> @@ -143,6 +143,7 @@ typedef struct osm_sm {
>  	cl_disp_reg_handle_t slvl_disp_h;
>  	cl_disp_reg_handle_t vla_disp_h;
>  	cl_disp_reg_handle_t pkey_disp_h;
> +	cl_map_t force_link_speed_except_guids;
>  } osm_sm_t;
>  /*
>  * FIELDS
> diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h
> index 9488225..5353298 100644
> --- a/opensm/include/opensm/osm_subnet.h
> +++ b/opensm/include/opensm/osm_subnet.h
> @@ -154,6 +154,7 @@ typedef struct osm_subn_opt {
>  	boolean_t lmc_esp0;
>  	uint8_t max_op_vls;
>  	uint8_t force_link_speed;
> +	char *force_link_speed_file;
>  	boolean_t reassign_lids;
>  	boolean_t ignore_other_sm;
>  	boolean_t single_thread;
> @@ -469,6 +470,11 @@ typedef struct osm_subn_opt {
>  *	Subnet object
>  *********/
>  
> +typedef struct force_speed_exception {
> +	uint64_t port_guid;
> +	uint8_t val;
> +} force_speed_exception_t;
> +
>  /****s* OpenSM: Subnet/osm_subn_t
>  * NAME
>  *	osm_subn_t
> diff --git a/opensm/opensm/osm_link_mgr.c b/opensm/opensm/osm_link_mgr.c
> index c9bdfee..21dcc50 100644
> --- a/opensm/opensm/osm_link_mgr.c
> +++ b/opensm/opensm/osm_link_mgr.c
> @@ -106,6 +106,8 @@ static int link_mgr_set_physp_pi(osm_sm_t * sm, IN osm_physp_t * p_physp,
>  	boolean_t esp0 = FALSE, send_set = FALSE;
>  	osm_physp_t *p_remote_physp;
>  	int ret = 0;
> +	uint8_t force_link_speed;
> +	force_speed_exception_t *except;

I would suggest to avoid word "exception" - it is not actually exception
but rather desired configuration, - when not requested default value is
used.

>  
>  	OSM_LOG_ENTER(sm->p_log);
>  
> @@ -326,12 +328,26 @@ static int link_mgr_set_physp_pi(osm_sm_t * sm, IN osm_physp_t * p_physp,
>  			   sizeof(p_pi->link_width_enabled)))
>  			send_set = TRUE;
>  
> -		if (sm->p_subn->opt.force_link_speed &&
> -		    (sm->p_subn->opt.force_link_speed != 15 ||
> +		/* use the force link speed set in the config */
> +		force_link_speed = sm->p_subn->opt.force_link_speed;
> +		/* unless this port guid is in our "except"ion file */
> +		if ((except = cl_map_get(
> +				&sm->force_link_speed_except_guids,
> +				cl_ntoh64(p_physp->port_guid)))
> +					!= NULL) {
> +			force_link_speed = except->val;
> +			OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
> +				"Skipping force_link_speed on GUID 0x%016"
> +				PRIx64 " (%d)\n",
> +			       cl_ntoh64(p_physp->port_guid),
> +			       force_link_speed);
> +		}
> +
> +		if (force_link_speed &&
> +		     (force_link_speed != OSM_DEFAULT_FORCE_LINK_SPEED ||
>  		     ib_port_info_get_link_speed_enabled(p_pi) !=
>  		     ib_port_info_get_link_speed_sup(p_pi))) {
>  			ib_port_info_set_link_speed_enabled(p_pi,
> -							    sm->p_subn->opt.
>  							    force_link_speed);
>  			if (memcmp(&p_pi->link_speed, &p_old_pi->link_speed,
>  				   sizeof(p_pi->link_speed)))
> diff --git a/opensm/opensm/osm_sm.c b/opensm/opensm/osm_sm.c
> index f780124..e5cf2b4 100644
> --- a/opensm/opensm/osm_sm.c
> +++ b/opensm/opensm/osm_sm.c
> @@ -168,6 +168,8 @@ void osm_sm_construct(IN osm_sm_t * p_sm)
>  	osm_sm_mad_ctrl_construct(&p_sm->mad_ctrl);
>  	osm_lid_mgr_construct(&p_sm->lid_mgr);
>  	osm_ucast_mgr_construct(&p_sm->ucast_mgr);
> +	cl_map_construct(&p_sm->force_link_speed_except_guids);
> +	cl_map_init(&p_sm->force_link_speed_except_guids, 10);
>  }
>  
>  /**********************************************************************
> diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c
> index 647950e..6aa338f 100644
> --- a/opensm/opensm/osm_subnet.c
> +++ b/opensm/opensm/osm_subnet.c
> @@ -46,6 +46,7 @@
>  #  include <config.h>
>  #endif				/* HAVE_CONFIG_H */
>  
> +#include <sys/stat.h>
>  #include <string.h>
>  #include <stdio.h>
>  #include <stdarg.h>
> @@ -305,6 +306,7 @@ static const opt_rec_t opt_tbl[] = {
>  	{ "lmc_esp0", OPT_OFFSET(lmc_esp0), opts_parse_boolean, NULL, 1 },
>  	{ "max_op_vls", OPT_OFFSET(max_op_vls), opts_parse_uint8, NULL, 1 },
>  	{ "force_link_speed", OPT_OFFSET(force_link_speed), opts_parse_uint8, NULL, 1 },
> +	{ "force_link_speed_file", OPT_OFFSET(force_link_speed_file), opts_parse_charp, NULL, 1 },
>  	{ "reassign_lids", OPT_OFFSET(reassign_lids), opts_parse_boolean, NULL, 1 },
>  	{ "ignore_other_sm", OPT_OFFSET(ignore_other_sm), opts_parse_boolean, NULL, 1 },
>  	{ "single_thread", OPT_OFFSET(single_thread), opts_parse_boolean, NULL, 0 },
> @@ -700,7 +702,8 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * p_opt)
>  	p_opt->lmc = OSM_DEFAULT_LMC;
>  	p_opt->lmc_esp0 = FALSE;
>  	p_opt->max_op_vls = OSM_DEFAULT_MAX_OP_VLS;
> -	p_opt->force_link_speed = 15;
> +	p_opt->force_link_speed = OSM_DEFAULT_FORCE_LINK_SPEED;
> +	p_opt->force_link_speed_file = NULL;
>  	p_opt->reassign_lids = FALSE;
>  	p_opt->ignore_other_sm = FALSE;
>  	p_opt->single_thread = FALSE;
> @@ -927,6 +930,73 @@ static ib_api_status_t osm_parse_prefix_routes_file(IN osm_subn_t * p_subn)
>  
>  /**********************************************************************
>   **********************************************************************/
> +void osm_parse_force_link_speed_except_file(IN osm_opensm_t * const p_osm,
> +					    IN char *file)

We have generic guid file parser helper already - parse_node_map().

Also instead of keeping yet another port map you can just add desired
configuration parameters to existing port (osm_physp) structure.

> +{
> +	FILE *fp;
> +	char line[1024];
> +	uint64_t guid = 0;
> +	force_speed_exception_t *except = NULL;
> +	cl_map_t *map = &p_osm->sm.force_link_speed_except_guids;
> +	cl_map_iterator_t head;
> +
> +	CL_ASSERT(cl_is_map_inited(map));
> +
> +	/* clear previous entries */
> +	head = cl_map_head(map);
> +	while (head != cl_map_end(map)) {
> +		except = (force_speed_exception_t *)cl_map_obj(head);
> +		cl_map_remove_item(map, head);
> +		free(except);
> +		head = cl_map_head(map);
> +	}
> +
> +	if (file) {
> +		fp = fopen(file, "r");
> +		if (!fp) {
> +			osm_log(&p_osm->log, OSM_LOG_ERROR,
> +				"failed to open force_link_speed_file "
> +				"\'%s\'\n",
> +				file);
> +			return;
> +		}
> +
> +		while (fgets(line, sizeof(line) - 1, fp) != NULL) {
> +			char *p;
> +			char *guid_str, *force_str;
> +			unsigned long force_val;
> +
> +			p = strchr(line, '#');
> +			if (p)
> +				*p = '\0';
> +
> +			guid_str = strtok_r(line, " \t\n", &p);
> +			if (guid_str &&
> +			   ((guid = strtoull(guid_str, NULL, 0)) != 0)) {
> +				except = calloc(1, sizeof *except);
> +
> +				force_str = strtok_r(NULL, " \t\n", &p);
> +				if (force_str) {
> +					force_val = strtoul(force_str, NULL, 0);
> +				} else
> +					force_val = OSM_DEFAULT_FORCE_LINK_SPEED;
> +
> +				except->port_guid = guid;
> +				except->val = (uint8_t)force_val;
> +				OSM_LOG(&p_osm->log, OSM_LOG_DEBUG,
> +					"Setting exception to "
> +					"force_link_speed on GUID 0x%016"
> +					PRIx64 " (%d)\n", except->port_guid, except->val);
> +				cl_map_insert(map, guid, (void *)except);
> +			}
> +		}
> +
> +		fclose(fp);
> +	}
> +}
> +
> +/**********************************************************************
> + **********************************************************************/
>  static void subn_verify_max_vls(unsigned *max_vls, const char *prefix, unsigned dflt)
>  {
>  	if (!*max_vls || *max_vls > 15) {
> @@ -1097,6 +1167,15 @@ int osm_subn_verify_config(IN osm_subn_opt_t * p_opts)
>  		p_opts->max_wire_smps = OSM_DEFAULT_SMP_MAX_ON_WIRE;
>  	}
>  
> +	if (p_opts->force_link_speed_file) {
> +		struct stat statbuf;
> +		if (stat(p_opts->force_link_speed_file, &statbuf))
> +			log_report(" Invalid Cached Option "
> +				   "Value:force_link_speed_file = %s"
> +				   "; file not found\n",
> +				   p_opts->force_link_speed_file);
> +	}
> +
>  	if (strcmp(p_opts->console, OSM_DISABLE_CONSOLE)
>  	    && strcmp(p_opts->console, OSM_LOCAL_CONSOLE)
>  #ifdef ENABLE_OSM_CONSOLE_SOCKET
> @@ -1265,6 +1344,9 @@ int osm_subn_rescan_conf_files(IN osm_subn_t * p_subn)
>  
>  	osm_parse_prefix_routes_file(p_subn);
>  
> +	osm_parse_force_link_speed_except_file(p_subn->p_osm,
> +					       p_opts->force_link_speed_file);
> +
>  	return 0;
>  }
>  
> @@ -1331,8 +1413,18 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t * p_opts)
>  		"#    5: 2.5 or 10.0 Gbps\n"
>  		"#    7: 2.5 or 5.0 or 10.0 Gbps\n"
>  		"#    2,4,6,8-14 Reserved\n"
> -		"#    Default 15: set to PortInfo:LinkSpeedSupported\n"
> +		"#    Default %d: set to PortInfo:LinkSpeedSupported\n"
>  		"force_link_speed %u\n\n"
> +		"# Optional file used to override individual port GUIDs and speeds to be forced.\n"
> +		"# Port GUIDs which are _not_ listed in this file will be set to \"force_link_speed\"\n"
> +		"# Speeds specified in this file may be higher or lower than the global setting\n"
> +		"# (force_link_speed) above\n"
> +		"# File Format:\n"
> +		"# # comment\n"
> +		"# <port_guid> [<setting>] [# comment]\n"
> +		"# ...\n"
> +		"# if the optional <setting> is not specified the default (%d) will be used\n"
> +		"force_link_speed_file %s\n\n"
>  		"# The subnet_timeout code that will be set for all the ports\n"
>  		"# The actual timeout is 4.096usec * 2^<subnet_timeout>\n"
>  		"subnet_timeout %u\n\n"
> @@ -1355,7 +1447,10 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t * p_opts)
>  		p_opts->head_of_queue_lifetime,
>  		p_opts->leaf_head_of_queue_lifetime,
>  		p_opts->max_op_vls,
> +		OSM_DEFAULT_FORCE_LINK_SPEED,
>  		p_opts->force_link_speed,
> +		OSM_DEFAULT_FORCE_LINK_SPEED,

Shouldn't here be p_opts->force_link_speed?

> +		p_opts->force_link_speed_file,
>  		p_opts->subnet_timeout,
>  		p_opts->local_phy_errors_threshold,
>  		p_opts->overrun_errors_threshold);
> -- 
> 1.5.4.5
>

Sasha
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] opensm: add force_link_speed_file config option.

[Ira Weiny]

On Thu, 15 Oct 2009 23:26:01 +0200
Sasha Khapyorsky <sashak-smomgflXvOZWk0Htik3J/w@public.gmane.org> wrote:

> Hi Ira,
> 
> On 13:26 Thu 15 Oct     , Ira Weiny wrote:
> > 
> > From: Ira Weiny <weiny2-i2BcT+NCU+M@public.gmane.org>
> > Date: Wed, 14 Oct 2009 16:09:25 -0700
> > Subject: [PATCH] opensm: add force_link_speed_file config option.
> > 
> > 	From the config file help:
> > 
> > 	# Optional file used to override individual port GUIDs and speeds to be forced.
> 
> And what about width and other potentially tunable port parameters?

I don't think width is "tunable" via the spec.  The force_link_speed option
was a non-standard work around for potentially bad hardware.  I was trying to
avoid going down the non-standard path too much.

> 
> I would suggest instead of adding config file per option to develop more
> generic config file(s) framework where various per port (or in even more
> generic case per port group) PortInfo parameters could be preconfigured.
> BTW it could include some existing stuff like sl2vl, vlarb, etc..

Ok I could see some of these parameters being included on a per port basis.  I
will make the file generic but I don't have time right now to include every
option we could think of.  What type of file format were you thinking?
Something like:

<begin>
# Port Option Config File

# compute node with profiling turned off forced to max of DDR speed
0x0002c9030004ea79 force_link_speed=3,cn=true,port_profile=false

# root guid running at link speed supported
0x0002c9020023c288 force_link_speed=15,root=true

<end>

Combine port_prof_ignore_file, root_guid_file, cn_guid_file, io_guid_file,
ids_guid_file, guid_routing_order_file, and add in a sl2vl and vlarb setting
to this file?  This is a bit of work.

> 
> Some other comments are below.
> 

[snip]

> > diff --git a/opensm/opensm/osm_link_mgr.c b/opensm/opensm/osm_link_mgr.c
> > index c9bdfee..21dcc50 100644
> > --- a/opensm/opensm/osm_link_mgr.c
> > +++ b/opensm/opensm/osm_link_mgr.c
> > @@ -106,6 +106,8 @@ static int link_mgr_set_physp_pi(osm_sm_t * sm, IN osm_physp_t * p_physp,
> >  	boolean_t esp0 = FALSE, send_set = FALSE;
> >  	osm_physp_t *p_remote_physp;
> >  	int ret = 0;
> > +	uint8_t force_link_speed;
> > +	force_speed_exception_t *except;
> 
> I would suggest to avoid word "exception" - it is not actually exception
> but rather desired configuration, - when not requested default value is
> used.

Yea, I thought the same thing but it is an "exception" to the
"force_link_speed" option.  GUID's which are not listed in this map/file will
default to the "force_link_speed" setting.  Therefore, GUID's in this file are
an exception...

However, I can see the confusion, I will change the name.

> > @@ -927,6 +930,73 @@ static ib_api_status_t osm_parse_prefix_routes_file(IN osm_subn_t * p_subn)
> >  
> >  /**********************************************************************
> >   **********************************************************************/
> > +void osm_parse_force_link_speed_except_file(IN osm_opensm_t * const p_osm,
> > +					    IN char *file)
> 
> We have generic guid file parser helper already - parse_node_map().

Ok, I will use this.

>
> Also instead of keeping yet another port map you can just add desired
> configuration parameters to existing port (osm_physp) structure.

I don't think so, this is read at startup.  There will not be any osm_physp
structures at that time...

Ah, ok, I see how this is done with other things now...  The file is parsed
during runtime when you need the option value...  Don't you think that might
be slow?  Especially if we combine all these port options into one big config
file?  If we had settings for root guids, ignore profiling, routing order,
link speed overrides, io nodes, cn nodes, etc, this is going to be a lot of
file open, parsing from the start of a file, and file closes.

[snip]

> >  
> > @@ -1331,8 +1413,18 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t * p_opts)
> >  		"#    5: 2.5 or 10.0 Gbps\n"
> >  		"#    7: 2.5 or 5.0 or 10.0 Gbps\n"
> >  		"#    2,4,6,8-14 Reserved\n"
> > -		"#    Default 15: set to PortInfo:LinkSpeedSupported\n"
> > +		"#    Default %d: set to PortInfo:LinkSpeedSupported\n"
> >  		"force_link_speed %u\n\n"
> > +		"# Optional file used to override individual port GUIDs and speeds to be forced.\n"
> > +		"# Port GUIDs which are _not_ listed in this file will be set to \"force_link_speed\"\n"
> > +		"# Speeds specified in this file may be higher or lower than the global setting\n"
> > +		"# (force_link_speed) above\n"
> > +		"# File Format:\n"
> > +		"# # comment\n"
> > +		"# <port_guid> [<setting>] [# comment]\n"
> > +		"# ...\n"
> > +		"# if the optional <setting> is not specified the default (%d) will be used\n"
> > +		"force_link_speed_file %s\n\n"
> >  		"# The subnet_timeout code that will be set for all the ports\n"
> >  		"# The actual timeout is 4.096usec * 2^<subnet_timeout>\n"
> >  		"subnet_timeout %u\n\n"
> > @@ -1355,7 +1447,10 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t * p_opts)
> >  		p_opts->head_of_queue_lifetime,
> >  		p_opts->leaf_head_of_queue_lifetime,
> >  		p_opts->max_op_vls,
> > +		OSM_DEFAULT_FORCE_LINK_SPEED,
> >  		p_opts->force_link_speed,
> > +		OSM_DEFAULT_FORCE_LINK_SPEED,
> 
> Shouldn't here be p_opts->force_link_speed?

No, one could leave out "<setting>" and the default will be set.

0x0005ad0000092106    # set to 15: LinkSpeedSupported
0x0002c9020040fec8  3 # set to  3: 2.5 or 5.0 Gbps

This way a user only has to list the GUIDs of the ports they want running at
full supported speed.  This maps well with our current usage where most of a
cluster is forced to some speed but a small part (maybe 1 switch) can run at
full supported speed.  This is where the "exception" comes into the verbiage
above.  ;-)

Anyway, if we go with a generic port config this does not matter.  Let me know
what you were thinking for a file format.  Does the format I mentioned in the
beginning work for you?

Ira

[snip]

-- 
Ira Weiny
Math Programmer/Computer Scientist
Lawrence Livermore National Lab
925-423-8008
weiny2-i2BcT+NCU+M@public.gmane.org
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] opensm: add force_link_speed_file config option.

[Dave Olson]

On Thu, 15 Oct 2009, Ira Weiny wrote:
| > And what about width and other potentially tunable port parameters?
| 
| I don't think width is "tunable" via the spec.  The force_link_speed option
| was a non-standard work around for potentially bad hardware.  I was trying to
| avoid going down the non-standard path too much.

There's infiniband hardware that can't do 1X, also.  Yes, it's
non-compliant, but it exists.

4X-only mode can be set (and reported correctly) by the tools as an extension
in the same manner as the speed extension, so it may as well be handled
the same way, if you are going to fix the one.

Dave Olson
dave.olson-h88ZbnxC6KDQT0dZR+AlfA@public.gmane.org
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] opensm: add force_link_speed_file config option.

[Sasha Khapyorsky]

On 17:04 Thu 15 Oct     , Ira Weiny wrote:
> 
> Ok I could see some of these parameters being included on a per port basis.  I
> will make the file generic but I don't have time right now to include every
> option we could think of.  What type of file format were you thinking?
> Something like:
> 
> <begin>
> # Port Option Config File
> 
> # compute node with profiling turned off forced to max of DDR speed
> 0x0002c9030004ea79 force_link_speed=3,cn=true,port_profile=false
> 
> # root guid running at link speed supported
> 0x0002c9020023c288 force_link_speed=15,root=true
> 
> <end>

Such format example may work. But I wouldn't mix port parameters (PortInfo
and friends) with other things like routing. So right now it could be
something like:

<guid> LinkSpeed=3, LinkWidth=1, sl2vl="..." ..., etc.

> This is a bit of work.

To make this reasonably simple it can be fine to start with implementing
just LinkSpeed.

> > Also instead of keeping yet another port map you can just add desired
> > configuration parameters to existing port (osm_physp) structure.
> 
> I don't think so, this is read at startup.  There will not be any osm_physp
> structures at that time...

You may want to update the file content without restarting OpenSM. If so
it would be reasonable to read the file at sweep time after subnet
discovery.

> Ah, ok, I see how this is done with other things now...  The file is parsed
> during runtime when you need the option value...  Don't you think that might
> be slow?  Especially if we combine all these port options into one big config
> file?  If we had settings for root guids, ignore profiling, routing order,
> link speed overrides, io nodes, cn nodes, etc, this is going to be a lot of
> file open, parsing from the start of a file, and file closes.

Comparing to other sweep activities it is not a big overhead and this
let us to keep configuration dynamically.

> [snip]
> 
> > >  
> > > @@ -1331,8 +1413,18 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t * p_opts)
> > >  		"#    5: 2.5 or 10.0 Gbps\n"
> > >  		"#    7: 2.5 or 5.0 or 10.0 Gbps\n"
> > >  		"#    2,4,6,8-14 Reserved\n"
> > > -		"#    Default 15: set to PortInfo:LinkSpeedSupported\n"
> > > +		"#    Default %d: set to PortInfo:LinkSpeedSupported\n"
> > >  		"force_link_speed %u\n\n"
> > > +		"# Optional file used to override individual port GUIDs and speeds to be forced.\n"
> > > +		"# Port GUIDs which are _not_ listed in this file will be set to \"force_link_speed\"\n"
> > > +		"# Speeds specified in this file may be higher or lower than the global setting\n"
> > > +		"# (force_link_speed) above\n"
> > > +		"# File Format:\n"
> > > +		"# # comment\n"
> > > +		"# <port_guid> [<setting>] [# comment]\n"
> > > +		"# ...\n"
> > > +		"# if the optional <setting> is not specified the default (%d) will be used\n"
> > > +		"force_link_speed_file %s\n\n"
> > >  		"# The subnet_timeout code that will be set for all the ports\n"
> > >  		"# The actual timeout is 4.096usec * 2^<subnet_timeout>\n"
> > >  		"subnet_timeout %u\n\n"
> > > @@ -1355,7 +1447,10 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t * p_opts)
> > >  		p_opts->head_of_queue_lifetime,
> > >  		p_opts->leaf_head_of_queue_lifetime,
> > >  		p_opts->max_op_vls,
> > > +		OSM_DEFAULT_FORCE_LINK_SPEED,
> > >  		p_opts->force_link_speed,
> > > +		OSM_DEFAULT_FORCE_LINK_SPEED,
> > 
> > Shouldn't here be p_opts->force_link_speed?
> 
> No, one could leave out "<setting>" and the default will be set.
> 
> 0x0005ad0000092106    # set to 15: LinkSpeedSupported

And when force_link_speed is reconfigured globally in config file and
!= 15?

> 0x0002c9020040fec8  3 # set to  3: 2.5 or 5.0 Gbps
> 
> This way a user only has to list the GUIDs of the ports they want running at
> full supported speed.  This maps well with our current usage where most of a
> cluster is forced to some speed but a small part (maybe 1 switch) can run at
> full supported speed.  This is where the "exception" comes into the verbiage
> above.  ;-)
> 
> Anyway, if we go with a generic port config this does not matter.  Let me know
> what you were thinking for a file format.  Does the format I mentioned in the
> beginning work for you?

Yes, looks fine to me. But maybe we can RFC file format separately.

Sasha
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html