[SET 2][PATCH 2/8][bonding] Propagating master's settings to slaves

[SET 2][PATCH 2/8][bonding] Propagating master's settings to slaves

[Shmulik Hen]

2 - Change monitoring function use the new functionality.

diff -Nuarp linux-2.4.22-rc1/drivers/net/bonding/bond_main.c linux-2.4.22-rc1-devel/drivers/net/bonding/bond_main.c
--- linux-2.4.22-rc1/drivers/net/bonding/bond_main.c	Fri Aug  8 14:03:16 2003
+++ linux-2.4.22-rc1-devel/drivers/net/bonding/bond_main.c	Fri Aug  8 14:03:17 2003
@@ -2207,8 +2207,9 @@ out:
 static void bond_mii_monitor(struct net_device *master)
 {
 	bonding_t *bond = (struct bonding *) master->priv;
-	slave_t *slave, *bestslave, *oldcurrent;
+	slave_t *slave, *oldcurrent;
 	int slave_died = 0;
+	int do_failover = 0;
 
 	read_lock(&bond->lock);
 
@@ -2218,7 +2219,6 @@ static void bond_mii_monitor(struct net_
 	 * program could monitor the link itself if needed.
 	 */
 
-	bestslave = NULL;
 	slave = (slave_t *)bond;
 
 	read_lock(&bond->ptrlock);
@@ -2226,8 +2226,6 @@ static void bond_mii_monitor(struct net_
 	read_unlock(&bond->ptrlock);
 
 	while ((slave = slave->prev) != (slave_t *)bond) {
-		/* use updelay+1 to match an UP slave even when updelay is 0 */
-		int mindelay = updelay + 1;
 		struct net_device *dev = slave->dev;
 		int link_state;
 		u16 old_speed = slave->speed;
@@ -2238,14 +2236,7 @@ static void bond_mii_monitor(struct net_
 		switch (slave->link) {
 		case BOND_LINK_UP:	/* the link was up */
 			if (link_state == BMSR_LSTATUS) {
-				/* link stays up, tell that this one
-				   is immediately available */
-				if (IS_UP(dev) && (mindelay > -2)) {
-					/* -2 is the best case :
-					   this slave was already up */
-					mindelay = -2;
-					bestslave = slave;
-				}
+				/* link stays up, nothing more to do */
 				break;
 			}
 			else { /* link going down */
@@ -2285,6 +2276,7 @@ static void bond_mii_monitor(struct net_
 					    (bond_mode == BOND_MODE_8023AD)) {
 						bond_set_slave_inactive_flags(slave);
 					}
+
 					printk(KERN_INFO
 						"%s: link status definitely down "
 						"for interface %s, disabling it",
@@ -2301,12 +2293,10 @@ static void bond_mii_monitor(struct net_
 						bond_alb_handle_link_change(bond, slave, BOND_LINK_DOWN);
 					}
 
-					write_lock(&bond->ptrlock);
-					if (slave == bond->current_slave) {
-						/* find a new interface and be verbose */
-						reselect_active_interface(bond);
+					if (slave == oldcurrent) {
+						do_failover = 1;
 					}
-					write_unlock(&bond->ptrlock);
+
 					slave_died = 1;
 				} else {
 					slave->delay--;
@@ -2321,13 +2311,6 @@ static void bond_mii_monitor(struct net_
 					master->name,
 					(downdelay - slave->delay) * miimon,
 					dev->name);
-
-				if (IS_UP(dev) && (mindelay > -1)) {
-					/* -1 is a good case : this slave went
-					   down only for a short time */
-					mindelay = -1;
-					bestslave = slave;
-				}
 			}
 			break;
 		case BOND_LINK_DOWN:	/* the link was down */
@@ -2397,26 +2380,12 @@ static void bond_mii_monitor(struct net_
 						bond_alb_handle_link_change(bond, slave, BOND_LINK_UP);
 					}
 
-					write_lock(&bond->ptrlock);
-					if ( (bond->primary_slave != NULL)
-					  && (slave == bond->primary_slave) )
-						reselect_active_interface(bond); 
-					write_unlock(&bond->ptrlock);
-				}
-				else
+					if ((oldcurrent == NULL) ||
+					    (slave == bond->primary_slave)) {
+						do_failover = 1;
+					}
+				} else {
 					slave->delay--;
-
-				/* we'll also look for the mostly eligible slave */
-				if (bond->primary_slave == NULL)  {
-				    if (IS_UP(dev) && (slave->delay < mindelay)) {
-					mindelay = slave->delay;
-					bestslave = slave;
-				    } 
-				} else if ( (IS_UP(bond->primary_slave->dev))  || 
-				          ( (!IS_UP(bond->primary_slave->dev))  && 
-				          (IS_UP(dev) && (slave->delay < mindelay)) ) ) {
-					mindelay = slave->delay;
-					bestslave = slave;
 				}
 			}
 			break;
@@ -2435,26 +2404,17 @@ static void bond_mii_monitor(struct net_
 
 	} /* end of while */
 
-	/* 
-	 * if there's no active interface and we discovered that one
-	 * of the slaves could be activated earlier, so we do it.
-	 */
-	read_lock(&bond->ptrlock);
-	oldcurrent = bond->current_slave;
-	read_unlock(&bond->ptrlock);
+	if (do_failover) {
+		write_lock(&bond->ptrlock);
 
-	/* no active interface at the moment or need to bring up the primary */
-	if (oldcurrent == NULL)  { /* no active interface at the moment */
-		if (bestslave != NULL) { /* last chance to find one ? */
-			write_lock(&bond->ptrlock);
-			change_active_interface(bond, bestslave);
-			write_unlock(&bond->ptrlock);
-		} else if (slave_died) {
-			/* print this message only once a slave has just died */
+		reselect_active_interface(bond);
+		if (oldcurrent && !bond->current_slave) {
 			printk(KERN_INFO
 				"%s: now running without any active interface !\n",
 				master->name);
 		}
+
+		write_unlock(&bond->ptrlock);
 	}
 
 	read_unlock(&bond->lock);
@@ -2472,9 +2432,10 @@ static void bond_mii_monitor(struct net_
 static void loadbalance_arp_monitor(struct net_device *master)
 {
 	bonding_t *bond;
-	slave_t *slave;
+	slave_t *slave, *oldcurrent;
 	int the_delta_in_ticks =  arp_interval * HZ / 1000;
 	int next_timer = jiffies + (arp_interval * HZ / 1000);
+	int do_failover = 0;
 
 	bond = (struct bonding *) master->priv; 
 	if (master->priv == NULL) {
@@ -2498,6 +2459,10 @@ static void loadbalance_arp_monitor(stru
 
 	read_lock(&bond->lock);
 
+	read_lock(&bond->ptrlock);
+	oldcurrent = bond->current_slave;
+	read_unlock(&bond->ptrlock);
+
 	/* see if any of the previous devices are up now (i.e. they have
 	 * xmt and rcv traffic). the current_slave does not come into
 	 * the picture unless it is null. also, slave->jiffies is not needed
@@ -2524,21 +2489,19 @@ static void loadbalance_arp_monitor(stru
 				 * current_slave being null after enslaving
 				 * is closed.
 				 */
-				write_lock(&bond->ptrlock);
-				if (bond->current_slave == NULL) {
+				if (oldcurrent == NULL) {
 					printk(KERN_INFO
 						"%s: link status definitely up "
 						"for interface %s, ",
 						master->name,
 						slave->dev->name);
-					reselect_active_interface(bond); 
+					do_failover = 1;
 				} else {
 					printk(KERN_INFO
 						"%s: interface %s is now up\n",
 						master->name,
 						slave->dev->name);
 				}
-				write_unlock(&bond->ptrlock);
 			} 
 		} else {
 			/* slave->link == BOND_LINK_UP */
@@ -2561,11 +2524,9 @@ static void loadbalance_arp_monitor(stru
 				       master->name,
 				       slave->dev->name);
 
-				write_lock(&bond->ptrlock);
-				if (slave == bond->current_slave) {
-					reselect_active_interface(bond);
+				if (slave == oldcurrent) {
+					do_failover = 1;
 				}
-				write_unlock(&bond->ptrlock);
 			}
 		} 
 
@@ -2579,6 +2540,19 @@ static void loadbalance_arp_monitor(stru
 		if (IS_UP(slave->dev)) {
 			arp_send_all(slave);
 		}
+	}
+
+	if (do_failover) {
+		write_lock(&bond->ptrlock);
+
+		reselect_active_interface(bond);
+		if (oldcurrent && !bond->current_slave) {
+			printk(KERN_INFO
+				"%s: now running without any active interface !\n",
+				master->name);
+		}
+
+		write_unlock(&bond->ptrlock);
 	}
 
 	read_unlock(&bond->lock);

-- 
| Shmulik Hen   Advanced Network Services  |
| Israel Design Center, Jerusalem          |
| LAN Access Division, Platform Networking |
| Intel Communications Group, Intel corp.  |

Re: [SET 2][PATCH 2/8][bonding] Propagating master's settings to slaves

[jamal]

Shmulik,

Some of this bonding stuff is pretty scary. Lotsa policies in the 
kernel and communication seems to be centred around /proc.
Shouldnt policies on failover be really driven from user space?
Also shouldnt communication be using something like netlink?

cheers,
jamal

On Fri, 2003-08-08 at 10:44, Shmulik Hen wrote:
> 2 - Change monitoring function use the new functionality.
> 

RE: [SET 2][PATCH 2/8][bonding] Propagating master's settings to slaves

[Hen, Shmulik]

> -----Original Message-----
> From: jamal [mailto:hadi@cyberus.ca]
> Sent: Saturday, August 09, 2003 1:01 AM
> To: Hen, Shmulik
> Cc: bonding-devel@lists.sourceforge.net; netdev@oss.sgi.com
> Subject: Re: [SET 2][PATCH 2/8][bonding] Propagating master's settings
> to slaves
> 
> Shmulik,
> 
> Some of this bonding stuff is pretty scary. Lotsa policies in the 
> kernel and communication seems to be centred around /proc.
> Shouldnt policies on failover be really driven from user space?
> Also shouldnt communication be using something like netlink?
> 
> cheers,
> jamal
> 
> On Fri, 2003-08-08 at 10:44, Shmulik Hen wrote:
> > 2 - Change monitoring function use the new functionality.
> > 
> 

Not sure I fully understood the concerns above, but I'll try
to explain what the change was all about.

By monitoring, I meant the 3 timer function running in bonding
to monitor link changes and act once a link fail/recovery is
detected. The old code used to do all the activity related to
changing the current active slave separately in each timer
function and it seemed redundant since it was basically the
same thing repeated 3 times. Instead, we thought it would be
best if we put that into 3 new functions - reselect_active,
find_best_slave and change_active that does all the actual stuff
of swapping an old current with the new one.

The change we did in /proc was to reduce the amount of data
extarcted each time the proc entry is polled. Instead of dumping
all the data of all the bond devices that exist, each bond returns
just data that is relevant to itself.

In the lonf term, the drive is to move any *smart* code done in
the config application into the driver itself and be left with
the smallest, most compact application as possible. This is the
trend we've seen in the VLAN config app, and the bridge module.
All the "brain" is in the kernel module and very little should be
done in the application.


	Shmulik.

RE: [SET 2][PATCH 2/8][bonding] Propagating master's settings to slaves

[jamal]

On Sat, 2003-08-09 at 06:29, Hen, Shmulik wrote:

> > 
> 
> Not sure I fully understood the concerns above, but I'll try
> to explain what the change was all about.
> 

I think it wasnt the one specific change rather a few posted that i
spent a minute or two staring at. And you confirm my suspicion below.

[..]

> 
> In the lonf term, the drive is to move any *smart* code done in
> the config application into the driver itself and be left with
> the smallest, most compact application as possible. This is the
> trend we've seen in the VLAN config app, and the bridge module.
> All the "brain" is in the kernel module and very little should be
> done in the application.

I am not very familiar with the bonding code although i think you guys
have been doing very good work since you got involved.
In any case the approach you state above is wrong. Actually Stephen
Hemminger and I discussed this for bridging. Post 2.6 he is going to
remove a lot of the bridge policy (or "brain" as you call it) out of the
kernel. Netlink for kernel<->userspace not /proc. I think we should head
towards that direction so we can have more sophisticated management.

Thoughts?

cheers,
jamal

Re: [SET 2][PATCH 2/8][bonding] Propagating master's settings to slaves

[Shmulik Hen]

On Monday 11 August 2003 05:51 am, you wrote:
> On Sat, 2003-08-09 at 06:29, Hen, Shmulik wrote:
> > Not sure I fully understood the concerns above, but I'll try
> > to explain what the change was all about.
>
> I think it wasnt the one specific change rather a few posted that i
> spent a minute or two staring at. And you confirm my suspicion
> below.

I probably didn't make myself clear - by "understood" I wanted to say 
I probably didn't get the *meaning* of the whole sentence , and not 
"I don't under stand why you are concerned".
(English is not my native tongue :) ).

> I am not very familiar with the bonding code although i think you
> guys have been doing very good work since you got involved.
> In any case the approach you state above is wrong. Actually Stephen
> Hemminger and I discussed this for bridging. Post 2.6 he is going
> to remove a lot of the bridge policy (or "brain" as you call it)
> out of the kernel. Netlink for kernel<->userspace not /proc. I
> think we should head towards that direction so we can have more
> sophisticated management.

I, on the other hand, am not familiar with the bridging code and I 
don't know what it actually does internally, I just noticed that 
regarding config operations, most of the code is done at the kernel 
level as response to ioctl commands.

I'll try to clarify how that relates to bonding. The ifenslave utility 
has very little "brain" as it is, and all it knows how to do 
currently is enslave/release slave devices and change the current 
active slave. It also has some ability to extract status info from 
the bond and present it nicely for a user.

The "brain" I was referring to in the bonding module itself has to do 
with timer functions monitoring link status or Tx/Rx activity of the 
slaves, and once a faulty slave is detected, switch to use another 
one instead according to the teaming mode. There are no large scale 
decision making nor major CPU consuming computations that are part of 
the continuous operation of the module that is basically handle Rx/Tx 
on slaves.

The bonding module doesn't need to access any special info that is 
normally available to user space apps. What it does need is very 
short response time and accessibility to kernel internal resources 
like net devices info to make it a high availability intermediate 
driver.

Trying to move that from the kernel module into the config application 
seems to be a very hard task to implement since we'll have to find a 
way to make the application constantly aware to the specifics like 
current topology, slave-to-bond affiliation, updated status of each 
slave, etc., etc. It would also mean that the driver will have to 
wait for the application to tell it what to do each time it needs a 
decision, and by that we'll surely suffer some performance hit and 
probably get low availability or temporary loss of communications.

Going back to the first problem, discussions on the bonding 
development list pointed that it might be better if we moved the 
configuration-time decisions making to the driver, so the application 
wouldn't have to deal with situations like:
1) get the master's MTU settings, master's teaming mode, communication 
   version, backwards compatibility issues, etc.
2) figure if need to set MTU to slave according to all that,
3) try to set that on the new slave being added,
4) if not successfull, decide if may enslave anyway or,
5) maybe undo all previous settings already done to the slave 
   (needs a way to retrieve old values)
6) decide if should go on or fail any further operations
7) repeat the above for all other settings

On the other hand, what we want to get to is something more like:
1) tell bonding to add slave X to bond Y,
2) watch for error returns,
3) print a nice message according to the type of the error.

While the driver, already aware of all possible relevant data, makes 
all decisions, performs settings, handles compatibility issues, 
checks for failures at each stage, handles any undo steps, and return 
success/error values accordingly.

>
> Thoughts?

Mostly explanations :)

Is there anywhere I can see what you refereed to as discussions with 
Stephen Hemminger ? I would really like to know how and what could 
also be applied to bonding.


	Regards,
	Shmulik.

Re: [SET 2][PATCH 2/8][bonding] Propagating master's settings to slaves

[jamal]

On Mon, 2003-08-11 at 06:08, Shmulik Hen wrote:
> On Monday 11 August 2003 05:51 am, you wrote:
> > On Sat, 2003-08-09 at 06:29, Hen, Shmulik wrote:
> > > Not sure I fully understood the concerns above, but I'll try
> > > to explain what the change was all about.
> >
> > I think it wasnt the one specific change rather a few posted that i
> > spent a minute or two staring at. And you confirm my suspicion
> > below.
> 
> I probably didn't make myself clear - by "understood" I wanted to say 
> I probably didn't get the *meaning* of the whole sentence , and not 
> "I don't under stand why you are concerned".
> (English is not my native tongue :) ).
> 
> > I am not very familiar with the bonding code although i think you
> > guys have been doing very good work since you got involved.
> > In any case the approach you state above is wrong. Actually Stephen
> > Hemminger and I discussed this for bridging. Post 2.6 he is going
> > to remove a lot of the bridge policy (or "brain" as you call it)
> > out of the kernel. Netlink for kernel<->userspace not /proc. I
> > think we should head towards that direction so we can have more
> > sophisticated management.
> 
> I, on the other hand, am not familiar with the bridging code and I 
> don't know what it actually does internally, I just noticed that 
> regarding config operations, most of the code is done at the kernel 
> level as response to ioctl commands.
> 

Theres two main components to it: a control protocol and a forwarding
path. The control protocol known as STP tells the forwarding path how to
behave. Essentially, STP carries the policy implemented by the
forwarding path. This is the same breakdown to say routing protocols
like OSPF and regular forwarding path. At the moment STP sits in the
kernel. STP is really the "brains". 

> I'll try to clarify how that relates to bonding. The ifenslave utility 
> has very little "brain" as it is, and all it knows how to do 
> currently is enslave/release slave devices and change the current 
> active slave. It also has some ability to extract status info from 
> the bond and present it nicely for a user.
> 
> The "brain" I was referring to in the bonding module itself has to do 
> with timer functions monitoring link status or Tx/Rx activity of the 
> slaves, and once a faulty slave is detected, switch to use another 
> one instead according to the teaming mode. 
> There are no large scale 
> decision making nor major CPU consuming computations that are part of 
> the continuous operation of the module that is basically handle Rx/Tx 
> on slaves.
> 
> The bonding module doesn't need to access any special info that is 
> normally available to user space apps. What it does need is very 
> short response time and accessibility to kernel internal resources 
> like net devices info to make it a high availability intermediate 
> driver.
> 
> Trying to move that from the kernel module into the config application 
> seems to be a very hard task to implement since we'll have to find a 
> way to make the application constantly aware to the specifics like 
> current topology, slave-to-bond affiliation, updated status of each 
> slave, etc., etc. It would also mean that the driver will have to 
> wait for the application to tell it what to do each time it needs a 
> decision, and by that we'll surely suffer some performance hit and 
> probably get low availability or temporary loss of communications.
> 

Not at all. If you let some app control this i am sure whoever writes
the app has vested interest in getting fast failovers etc.

> Going back to the first problem, discussions on the bonding 
> development list pointed that it might be better if we moved the 
> configuration-time decisions making to the driver, so the application 
> wouldn't have to deal with situations like:
> 1) get the master's MTU settings, master's teaming mode, communication 
>    version, backwards compatibility issues, etc.
> 2) figure if need to set MTU to slave according to all that,
> 3) try to set that on the new slave being added,
> 4) if not successfull, decide if may enslave anyway or,
> 5) maybe undo all previous settings already done to the slave 
>    (needs a way to retrieve old values)
> 6) decide if should go on or fail any further operations
> 7) repeat the above for all other settings
> 
> On the other hand, what we want to get to is something more like:
> 1) tell bonding to add slave X to bond Y,
> 2) watch for error returns,
> 3) print a nice message according to the type of the error.
> 

Dont you think that anything thats "rich" like you list above should
stay out of the kernel? In any case, if you have a controlling app, you
could do more interesting things; example add or delete routes, firewall
rules, qos policies etc which all have very strong correlation with
availability - these are examples btw, not an exhaustive list. If all
you are satisfied with is link management alone, then by all means 
hardcoding behavior into the kernel is fine. I dont think it is
sufficient.

> While the driver, already aware of all possible relevant data, makes 
> all decisions, performs settings, handles compatibility issues, 
> checks for failures at each stage, handles any undo steps, and return 
> success/error values accordingly.
> 

Driver - actually bonding - should have minimal failover policy built in
for the lazy; example what i used to know about bonding - failover to
the next link, maybe send a grat arp etc. If I want more than basic,
then send me netlink events to user space and let me control how it
goes. Maybe i dont want to go to the second link but rather the 4th
link.

> >
> > Thoughts?
> 
> Mostly explanations :)
> 
> Is there anywhere I can see what you refereed to as discussions with 
> Stephen Hemminger ? I would really like to know how and what could 
> also be applied to bonding.
> 

Basically what i described at the top. Move any "richness" to user
space.

cheers,
jamal   

Re: [Bonding-devel] Re: [SET 2][PATCH 2/8][bonding] Propagating master's settings toslaves

[Jay Vosburgh]

>The answer is, like life, it's a balance.
[...]
>This is why I push for a "bonding-utils" package from Jay.... because of 
>the general rule above:  put it into userspace, where possible.

	Hmm.  My impression from our prior discussions was that your
interest in moving ifenslave out of the kernel source and into its own
package was more of a source code management concern rather than
moving functionality from the kernel into user space (because
ifenslave is in user space to begin with).

	Anyway, for most of the core bonding failover logic, I don't
see how a user space daemon implementation can perform equivalently to
a kernel-only implementation.  I could be wrong (I haven't done any
testing) but for the core "eth0 is dead, enable eth1" type stuff, it
seems to me that in-kernel beats "user space yakking with kernel" for
reliability and speed, particularly on heavily loaded systems.

	Now, that said, I can see a use for a user space monitoring /
control program, for the "strategic" problems (as opposed to the
"tactical" problems, like the previous paragraph).  If we want to,
e.g., monitor bandwidth usage and add or remove links from the
aggregation, that is (a) not as time critical, and (b) somewhat
fuzzier in definition.  Such a user space program could also interface
with various system management or HA thingies and report status for
its activities as well as the activities that bonding performs
independent of it.

	One thought I've had (which dovetails somewhat with an earlier
comment from Laurent) is a tcpdump/bpf-style "policy engine" blob in
the kernel, which is programmed from user space with enough brains to
handle the "tactical" level problems (the "strategic" problems might
be more than such a blob could handle, and if its easy enough to yak
with user space for those problems, it may not be necessary).  I
haven't done much more than think about this, though; it may very well
be overkill for the basic stuff.

	-J

---
	-Jay Vosburgh, IBM Linux Technology Center, fubar@us.ibm.com

Re: [SET 2][PATCH 2/8][bonding] Propagating master's settings to slaves

[Shmulik Hen]

May I remind you all that the original discussion was only about  
stuff that has to do with configuration time. There was no mention of 
any run time code. ifenslave only does three simple things - add a 
slave, remove a slave and set the current active slave, that's all. 

The drive was to try and make ifenslave slimmer regarding those three 
operations only in the way that any setting of the slave will be done 
by the kernel module instead of the configuration application. There 
is no real "brain" there anyway.

We had some experience with creating an configuration application that 
was incredibly smart and was always aware of what was going on in the 
driver and could make all possible decisions before even attempting 
to access the driver so it could fail the operation without  
"bothering" the driver. It's gigantic. It's extremely hard to install 
and configure. It's even harder to maintain. And all it was meant to 
do is configuration. Imagine what would happen if it was also 
supposed to handle run time issues.

I am not aware of anything like moving kernel code into applications. 
Was that something that was discussed in OLS ? Where can I find some 
more info about this trend ?


	Shmulik.

Re: [SET 2][PATCH 2/8][bonding] Propagating master's settings to slaves

[jamal]

Shmulik, the only discussion as far as i know is the one that happened
in this thread. I have not seen any discussion before. 

Folks, I really didnt mean to start such a long thread ;->

cheers,
jamal

On Mon, 2003-08-11 at 19:15, Shmulik Hen wrote:
> May I remind you all that the original discussion was only about  
> stuff that has to do with configuration time. There was no mention of 
> any run time code. ifenslave only does three simple things - add a 
> slave, remove a slave and set the current active slave, that's all. 
> 
> The drive was to try and make ifenslave slimmer regarding those three 
> operations only in the way that any setting of the slave will be done 
> by the kernel module instead of the configuration application. There 
> is no real "brain" there anyway.
> 
> We had some experience with creating an configuration application that 
> was incredibly smart and was always aware of what was going on in the 
> driver and could make all possible decisions before even attempting 
> to access the driver so it could fail the operation without  
> "bothering" the driver. It's gigantic. It's extremely hard to install 
> and configure. It's even harder to maintain. And all it was meant to 
> do is configuration. Imagine what would happen if it was also 
> supposed to handle run time issues.
> 
> I am not aware of anything like moving kernel code into applications. 
> Was that something that was discussed in OLS ? Where can I find some 
> more info about this trend ?
> 
> 
> 	Shmulik.
> 
>