Re: [PATCH v3] opensm: support routing engine update

["Eli Dorfman (Voltaire)" <dorfman.eli-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org>]

Slava Strebkov wrote:
> HI!
> That was my misunderstanding - upon heavy sweep SM will try to load
> routing engines as defined in the SM.conf file (ftree & updn - in that
> order).
> So ftree will be loaded when switch comes back from reboot.
> 
> Slava
> 
> -----Original Message-----
> From: linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> [mailto:linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org] On Behalf Of Yevgeny Kliteynik
> Sent: Monday, December 07, 2009 10:22 AM
> To: Slava Strebkov
> Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> Subject: Re: [PATCH v3] opensm: support routing engine update
> 
> Slava,
> 
> Slava Strebkov wrote:
>> Hi Yevgeny,
>> In that case SM will use updn and will not come back to ftree
>> automatically.
> 
> I think that this is a bad thing.
> 
> I wouldn't want *temporary* change of fabric to cause
> *permanent* change of SM mode of operation. Such changes
> do happen, and I'd prefer SM to continue functioning
> in accordance to the user's configuration once the
> fabric is settled again.
> 
> I do see the cases were the change that you propose is
> beneficial - if fabric topology doesn't fits the chosen
> routing, SM will waste time on retrying the wrong routing
> at every heavy sweep, but this happens due to suboptimal
> SM configuration and not as a result of some event that
> user has no control of.

For every heavy sweep the SM will try to configure the routing engines as specified in the conf file.
So when a switch goes up and ftree configuration is valid, the SM will configure ftree instead of updn.

Eli

> 
> -- Yevgeny
> 
> 
>  
>> Slava
>>
>> -----Original Message-----
>> From: linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
>> [mailto:linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org] On Behalf Of Yevgeny
> Kliteynik
>> Sent: Sunday, December 06, 2009 6:03 PM
>> To: Slava Strebkov
>> Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
>> Subject: Re: [PATCH v3] opensm: support routing engine update
>>
>> Slava,
>>
>> Slava Strebkov wrote:
>>> setup routing engine when in use and delete when failed.
>>> setup routing engine before use.
>>> delete resources when routing algorithm fails.
>>> this will save allocation for routing algorithms that are not used.
>> Suppose a user runs SM with ftree & updn routings (in that order),
>> and SM manages to route the fabric with ftree. At some point 
>> some switch reboots and causes ftree to fail and SM routes the
>> fabric with updn.
>> Does this mean that ftree will be removed from the list, and
>> when the switch comes back, SM won't try ftree any more?
>>
>> -- Yevgeny
>>  
>>> Signed-off-by: Slava Strebkov <slavas-smomgflXvOZWk0Htik3J/w@public.gmane.org>
>>> ---
>>>  opensm/include/opensm/osm_opensm.h |    5 +++
>>>  opensm/opensm/osm_opensm.c         |   57
>> +++++++++++++++++++++++++++++++-----
>>>  opensm/opensm/osm_subnet.c         |    7 ++++-
>>>  opensm/opensm/osm_ucast_mgr.c      |   28 +++++++++++++++++
>>>  4 files changed, 88 insertions(+), 9 deletions(-)
>>>
>>> diff --git a/opensm/include/opensm/osm_opensm.h
>> b/opensm/include/opensm/osm_opensm.h
>>> index c121be4..ca0fddb 100644
>>> --- a/opensm/include/opensm/osm_opensm.h
>>> +++ b/opensm/include/opensm/osm_opensm.h
>>> @@ -109,6 +109,7 @@ typedef enum _osm_routing_engine_type {
>>>  } osm_routing_engine_type_t;
>>>  /***********/
>>>  
>>> +struct osm_opensm;
>>>  /****s* OpenSM: OpenSM/osm_routing_engine
>>>  * NAME
>>>  *	struct osm_routing_engine
>>> @@ -122,6 +123,8 @@ typedef enum _osm_routing_engine_type {
>>>  struct osm_routing_engine {
>>>  	const char *name;
>>>  	void *context;
>>> +	int initialized;
>>> +	int (*setup) (struct osm_routing_engine *re, struct osm_opensm
>> *p_osm);
>>>  	int (*build_lid_matrices) (void *context);
>>>  	int (*ucast_build_fwd_tables) (void *context);
>>>  	void (*ucast_dump_tables) (void *context);
>>> @@ -183,6 +186,7 @@ typedef struct osm_opensm {
>>>  	cl_dispatcher_t disp;
>>>  	cl_plock_t lock;
>>>  	struct osm_routing_engine *routing_engine_list;
>>> +	struct osm_routing_engine *last_routing_engine;
>>>  	osm_routing_engine_type_t routing_engine_used;
>>>  	osm_stats_t stats;
>>>  	osm_console_t console;
>>> @@ -522,6 +526,7 @@ extern volatile unsigned int osm_exit_flag;
>>>  * DESCRIPTION
>>>  *  Set to one to cause all threads to leave
>>>  *********/
>>> +void osm_update_routing_engines(osm_opensm_t *osm, const char
>> *engine_names);
>>>  
>>>  END_C_DECLS
>>>  #endif				/* _OSM_OPENSM_H_ */
>>> diff --git a/opensm/opensm/osm_opensm.c b/opensm/opensm/osm_opensm.c
>>> index 50d1349..f90584d 100644
>>> --- a/opensm/opensm/osm_opensm.c
>>> +++ b/opensm/opensm/osm_opensm.c
>>> @@ -169,14 +169,7 @@ static void setup_routing_engine(osm_opensm_t
>> *osm, const char *name)
>>>  			memset(re, 0, sizeof(struct
>> osm_routing_engine));
>>>  
>>>  			re->name = m->name;
>>> -			if (m->setup(re, osm)) {
>>> -				OSM_LOG(&osm->log, OSM_LOG_VERBOSE,
>>> -					"setup of routing"
>>> -					" engine \'%s\' failed\n",
>> name);
>>> -				return;
>>> -			}
>>> -			OSM_LOG(&osm->log, OSM_LOG_DEBUG,
>>> -				"\'%s\' routing engine set up\n",
>> re->name);
>>> +			re->setup = m->setup;
>>>  			append_routing_engine(osm, re);
>>>  			return;
>>>  		}
>>> @@ -236,6 +229,54 @@ static void destroy_routing_engines(osm_opensm_t
>> *osm)
>>>  			r->delete(r->context);
>>>  		free(r);
>>>  	}
>>> +    osm->routing_engine_list = NULL;
>>> +}
>>> +
>>> +static void update_routing_engine(
>>> +       struct osm_routing_engine *cur,
>>> +       struct osm_routing_engine *last)
>>> +{
>>> +	struct osm_routing_engine *next = cur->next;
>>> +	if (!last)
>>> +		return; /* no last routing engine */
>>> +	memcpy(cur, last, sizeof(*cur));
>>> +	/* restore next */
>>> +	cur->next = next;
>>> +}
>>> +
>>> +void osm_update_routing_engines(osm_opensm_t *osm, const char
>> *engine_names)
>>> +{
>>> +	struct osm_routing_engine *r, *l;
>>> +	/* find used routing engine and save as last */
>>> +	l = r = osm->routing_engine_list;
>>> +	if (r && osm->routing_engine_used ==
>> osm_routing_engine_type(r->name)) {
>>> +		osm->last_routing_engine = r;
>>> +		osm->routing_engine_list = r->next;
>>> +	}
>>> +	else while ((r = r->next)) {
>>> +		if (osm->routing_engine_used ==
>>> +			osm_routing_engine_type(r->name)) {
>>> +				osm->last_routing_engine = r;
>>> +				l->next = r->next;
>>> +				break;
>>> +		}
>>> +		l = r;
>>> +	}
>>> +	/* cleanup prev routing engine list and replace with current
>> list */
>>> +	destroy_routing_engines(osm);
>>> +	setup_routing_engines(osm, engine_names);
>>> +	/* check if last routing engine exist in new list and update
>> callbacks */
>>> +	r = osm->routing_engine_list;
>>> +	while (r) {
>>> +		if (osm->routing_engine_used ==
>>> +			osm_routing_engine_type(r->name)) {
>>> +				update_routing_engine(r,
>> osm->last_routing_engine);
>>> +				free(osm->last_routing_engine);
>>> +				osm->last_routing_engine = NULL;
>>> +				break;
>>> +		}
>>> +	r = r->next;
>>> +	}
>>>  }
>>>  
>>>
> /**********************************************************************
>>> diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c
>>> index 8d63a75..742ae64 100644
>>> --- a/opensm/opensm/osm_subnet.c
>>> +++ b/opensm/opensm/osm_subnet.c
>>> @@ -152,6 +152,11 @@ static void opts_setup_sm_priority(osm_subn_t
>> *p_subn, void *p_val)
>>>  	osm_set_sm_priority(p_sm, sm_priority);
>>>  }
>>>  
>>> +static void opts_setup_routing_engine(osm_subn_t *p_subn, void
>> *p_val)
>>> +{
>>> +	osm_update_routing_engines(p_subn->p_osm, p_val);
>>> +}
>>> +
>>>  static void opts_parse_net64(IN osm_subn_t *p_subn, IN char *p_key,
>>>  			     IN char *p_val_str, void *p_v1, void *p_v2,
>>>  			     void (*pfn)(osm_subn_t *, void *))
>>> @@ -324,7 +329,7 @@ static const opt_rec_t opt_tbl[] = {
>>>  	{ "hop_weights_file", OPT_OFFSET(hop_weights_file),
>> opts_parse_charp, NULL, 0 },
>>>  	{ "port_profile_switch_nodes",
>> OPT_OFFSET(port_profile_switch_nodes), opts_parse_boolean, NULL, 1 },
>>>  	{ "sweep_on_trap", OPT_OFFSET(sweep_on_trap),
>> opts_parse_boolean, NULL, 1 },
>>> -	{ "routing_engine", OPT_OFFSET(routing_engine_names),
>> opts_parse_charp, NULL, 0 },
>>> +	{ "routing_engine", OPT_OFFSET(routing_engine_names),
>> opts_parse_charp, opts_setup_routing_engine, 1 },
>>>  	{ "connect_roots", OPT_OFFSET(connect_roots),
>> opts_parse_boolean, NULL, 1 },
>>>  	{ "use_ucast_cache", OPT_OFFSET(use_ucast_cache),
>> opts_parse_boolean, NULL, 1 },
>>>  	{ "log_file", OPT_OFFSET(log_file), opts_parse_charp, NULL, 0 },
>>> diff --git a/opensm/opensm/osm_ucast_mgr.c
>> b/opensm/opensm/osm_ucast_mgr.c
>>> index 39d825c..d6294ac 100644
>>> --- a/opensm/opensm/osm_ucast_mgr.c
>>> +++ b/opensm/opensm/osm_ucast_mgr.c
>>> @@ -998,8 +998,23 @@ int osm_ucast_mgr_process(IN osm_ucast_mgr_t *
>> p_mgr)
>>>  
>>>  	p_osm->routing_engine_used = OSM_ROUTING_ENGINE_TYPE_NONE;
>>>  	while (p_routing_eng) {
>>> +		if (!p_routing_eng->initialized &&
>>> +			p_routing_eng->setup(p_routing_eng, p_osm)) {
>>> +			OSM_LOG(p_mgr->p_log, OSM_LOG_ERROR,
>>> +				"ERR 3A0F: setup of routing engine
>> \'%s\' failed\n",
>>> +					p_routing_eng->name);
>>> +					p_routing_eng =
>> p_routing_eng->next;
>>> +					continue;
>>> +		}
>>> +		OSM_LOG(p_mgr->p_log, OSM_LOG_INFO,
>>> +			"\'%s\' routing engine set up\n",
>> p_routing_eng->name);
>>> +		p_routing_eng->initialized = 1;
>>>  		if (!ucast_mgr_route(p_routing_eng, p_osm))
>>>  			break;
>>> +		/* delete unused routing engine */
>>> +		if (p_routing_eng->delete)
>>> +			p_routing_eng->delete(p_routing_eng->context);
>>> +		p_routing_eng->initialized = 0;
>>>  		p_routing_eng = p_routing_eng->next;
>>>  	}
>>>  
>>> @@ -1011,6 +1026,19 @@ int osm_ucast_mgr_process(IN osm_ucast_mgr_t *
>> p_mgr)
>>>  		p_osm->routing_engine_used =
>> OSM_ROUTING_ENGINE_TYPE_MINHOP;
>>>  	}
>>>  
>>> +	/* if for some reason different routing engine is used */
>>> +	/* cleanup last unused routing engine */
>>> +	p_routing_eng = p_osm->last_routing_engine;
>>> +	if (p_routing_eng) {
>>> +			if (p_routing_eng->initialized &&
>>> +					p_routing_eng->delete &&
>>> +					p_osm->routing_engine_used !=
>>> +
>> osm_routing_engine_type(p_routing_eng->name))
>>> +
>> p_routing_eng->delete(p_routing_eng->context);
>>> +			free(p_routing_eng);
>>> +			p_osm->last_routing_engine = NULL;
>>> +	}
>>> +
>>>  	OSM_LOG(p_mgr->p_log, OSM_LOG_INFO,
>>>  		"%s tables configured on all switches\n",
>>>
>> osm_routing_engine_type_str(p_osm->routing_engine_used));
>>
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-rdma"
> in
>> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>>
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html