hidden interface (ARP) 2.4.20

hidden interface (ARP) 2.4.20

[Bingner Sam J Contractor PACAF CSS/SCHE]

[-- Attachment #1: Type: text/plain, Size: 149 bytes --]

Attached is a patch that seems to work for the hidden flag in 2.4.20... for
anybody else who needs this functionality

	Sam Bingner
	PACAF CSS/SCHE


[-- Attachment #2: hidden-2.4.20.diff --]
[-- Type: application/octet-stream, Size: 6802 bytes --]

diff -u -r linux-2.4.20/Documentation/filesystems/proc.txt linux-hidden/Documentation/filesystems/proc.txt
--- linux-2.4.20/Documentation/filesystems/proc.txt	Thu Nov 28 13:53:08 2002
+++ linux-hidden/Documentation/filesystems/proc.txt	Thu Dec  5 08:57:03 2002
@@ -1573,6 +1573,16 @@
 
 Determines whether to send ICMP redirects to other hosts.
 
+hidden
+------
+
+Hide addresses attached to this device from another devices.
+Such addresses will never be selected by source address autoselection
+mechanism, host does not answer broadcast ARP requests for them,
+does not announce it as source address of ARP requests, but they
+are still reachable via IP. This flag is activated only if it is
+enabled both in specific device section and in "all" section.
+
 Routing settings
 ----------------
 
diff -u -r linux-2.4.20/Documentation/networking/ip-sysctl.txt linux-hidden/Documentation/networking/ip-sysctl.txt
--- linux-2.4.20/Documentation/networking/ip-sysctl.txt	Fri Aug  2 14:39:42 2002
+++ linux-hidden/Documentation/networking/ip-sysctl.txt	Thu Dec  5 08:57:03 2002
@@ -445,6 +445,14 @@
 Alpha 1/1024s. See the HZ define in /usr/include/asm/param.h for the exact
 value on your system. 
 
+hidden - BOOLEAN
+	Hide addresses attached to this device from another devices.
+	Such addresses will never be selected by source address autoselection
+	mechanism, host does not answer broadcast ARP requests for them,
+	does not announce it as source address of ARP requests, but they
+	are still reachable via IP. This flag is activated only if it is
+	enabled both in specific device section and in "all" section.
+
 Alexey Kuznetsov.
 kuznet@ms2.inr.ac.ru
 
diff -u -r linux-2.4.20/include/linux/inetdevice.h linux-hidden/include/linux/inetdevice.h
--- linux-2.4.20/include/linux/inetdevice.h	Fri Aug  2 14:39:45 2002
+++ linux-hidden/include/linux/inetdevice.h	Thu Dec  5 08:57:03 2002
@@ -17,6 +17,7 @@
 	int	forwarding;
 	int	mc_forwarding;
 	int	tag;
+	int	hidden;
 	int     arp_filter;
 	int	medium_id;
 	void	*sysctl;
@@ -45,6 +46,7 @@
 
 #define IN_DEV_LOG_MARTIANS(in_dev)	(ipv4_devconf.log_martians || (in_dev)->cnf.log_martians)
 #define IN_DEV_PROXY_ARP(in_dev)	(ipv4_devconf.proxy_arp || (in_dev)->cnf.proxy_arp)
+#define IN_DEV_HIDDEN(in_dev)		((in_dev)->cnf.hidden && ipv4_devconf.hidden)
 #define IN_DEV_SHARED_MEDIA(in_dev)	(ipv4_devconf.shared_media || (in_dev)->cnf.shared_media)
 #define IN_DEV_TX_REDIRECTS(in_dev)	(ipv4_devconf.send_redirects || (in_dev)->cnf.send_redirects)
 #define IN_DEV_SEC_REDIRECTS(in_dev)	(ipv4_devconf.secure_redirects || (in_dev)->cnf.secure_redirects)
diff -u -r linux-2.4.20/include/linux/sysctl.h linux-hidden/include/linux/sysctl.h
--- linux-2.4.20/include/linux/sysctl.h	Thu Nov 28 13:53:15 2002
+++ linux-hidden/include/linux/sysctl.h	Thu Dec  5 08:57:03 2002
@@ -339,6 +339,7 @@
 	NET_IPV4_CONF_TAG=12,
 	NET_IPV4_CONF_ARPFILTER=13,
 	NET_IPV4_CONF_MEDIUM_ID=14,
+	NET_IPV4_CONF_HIDDEN=15,
 };
 
 /* /proc/sys/net/ipv6 */
diff -u -r linux-2.4.20/net/ipv4/arp.c linux-hidden/net/ipv4/arp.c
--- linux-2.4.20/net/ipv4/arp.c	Thu Nov 28 13:53:15 2002
+++ linux-hidden/net/ipv4/arp.c	Thu Dec  5 08:57:42 2002
@@ -66,6 +66,8 @@
  *		Alexey Kuznetsov:	new arp state machine;
  *					now it is in net/core/neighbour.c.
  *		Krzysztof Halasa:	Added Frame Relay ARP support.
+ *		Julian Anastasov:	"hidden" flag: hide the
+ *					interface and don't reply for it
  */
 
 #include <linux/types.h>
@@ -317,12 +319,23 @@
 static void arp_solicit(struct neighbour *neigh, struct sk_buff *skb)
 {
 	u32 saddr;
+	int from_skb;
+	struct in_device *in_dev2 = NULL;
+	struct net_device *dev2 = NULL;
 	u8  *dst_ha = NULL;
 	struct net_device *dev = neigh->dev;
 	u32 target = *(u32*)neigh->primary_key;
 	int probes = atomic_read(&neigh->probes);
 
-	if (skb && inet_addr_type(skb->nh.iph->saddr) == RTN_LOCAL)
+	from_skb = (skb &&
+		(dev2 = ip_dev_find(skb->nh.iph->saddr)) != NULL &&
+		(in_dev2 = in_dev_get(dev2)) != NULL &&
+		!IN_DEV_HIDDEN(in_dev2));
+	if (dev2) {
+		if (in_dev2) in_dev_put(in_dev2);
+		dev_put(dev2);
+	}
+	if (from_skb)
 		saddr = skb->nh.iph->saddr;
 	else
 		saddr = inet_select_addr(dev, target, RT_SCOPE_LINK);
@@ -754,9 +767,22 @@
 
 	/* Special case: IPv4 duplicate address detection packet (RFC2131) */
 	if (sip == 0) {
-		if (arp->ar_op == htons(ARPOP_REQUEST) &&
-		    inet_addr_type(tip) == RTN_LOCAL)
+ 		int reply;
+ 		struct net_device *dev2 = NULL;
+ 		struct in_device *in_dev2 = NULL;
+ 
+ 		reply =
+ 		    (arp->ar_op == htons(ARPOP_REQUEST) &&
+ 		    (dev2 = ip_dev_find(tip)) != NULL &&
+ 		    (dev2 == dev ||
+ 		    ((in_dev2 = in_dev_get(dev2)) != NULL &&
+ 		    !IN_DEV_HIDDEN(in_dev2))));
+ 		if (dev2) {
+ 		    if (in_dev2) in_dev_put(in_dev2);
+ 		    dev_put(dev2);
+ 		    if (reply)
 			arp_send(ARPOP_REPLY,ETH_P_ARP,tip,dev,tip,sha,dev->dev_addr,dev->dev_addr);
+ 		}
 		goto out;
 	}
 
@@ -770,6 +796,21 @@
 			n = neigh_event_ns(&arp_tbl, sha, &sip, dev);
 			if (n) {
 				int dont_send = 0;
+				if (ipv4_devconf.hidden &&
+				    skb->pkt_type != PACKET_HOST) {
+					struct net_device *dev2 = NULL;
+					struct in_device *in_dev2 = NULL;
+
+					dont_send |=
+					  ((dev2 = ip_dev_find(tip)) != NULL &&
+					  dev2 != dev &&
+					  (in_dev2=in_dev_get(dev2)) != NULL &&
+					  IN_DEV_HIDDEN(in_dev2));
+					if (dev2) {
+					    if (in_dev2) in_dev_put(in_dev2);
+					    dev_put(dev2);
+					}
+				}
 				if (IN_DEV_ARPFILTER(in_dev))
 					dont_send |= arp_filter(sip,tip,dev); 
 				if (!dont_send)
diff -u -r linux-2.4.20/net/ipv4/devinet.c linux-hidden/net/ipv4/devinet.c
--- linux-2.4.20/net/ipv4/devinet.c	Fri Aug  2 14:39:46 2002
+++ linux-hidden/net/ipv4/devinet.c	Thu Dec  5 08:57:03 2002
@@ -756,7 +756,8 @@
 
 		read_lock(&in_dev->lock);
 		for_primary_ifa(in_dev) {
-			if (ifa->ifa_scope != RT_SCOPE_LINK &&
+			if (!IN_DEV_HIDDEN(in_dev) &&
+			    ifa->ifa_scope != RT_SCOPE_LINK &&
 			    ifa->ifa_scope <= scope) {
 				read_unlock(&in_dev->lock);
 				read_unlock(&inetdev_lock);
@@ -1032,7 +1033,7 @@
 static struct devinet_sysctl_table
 {
 	struct ctl_table_header *sysctl_header;
-	ctl_table devinet_vars[15];
+	ctl_table devinet_vars[16];
 	ctl_table devinet_dev[2];
 	ctl_table devinet_conf_dir[2];
 	ctl_table devinet_proto_dir[2];
@@ -1078,6 +1079,9 @@
 	{NET_IPV4_CONF_TAG, "tag",
 	 &ipv4_devconf.tag, sizeof(int), 0644, NULL,
 	 &proc_dointvec},
+	{NET_IPV4_CONF_HIDDEN, "hidden",
+	 &ipv4_devconf.hidden, sizeof(int), 0644, NULL,
+	 &proc_dointvec},
 	{NET_IPV4_CONF_ARPFILTER, "arp_filter",
 	 &ipv4_devconf.arp_filter, sizeof(int), 0644, NULL,
 	 &proc_dointvec},

Re: hidden interface (ARP) 2.4.20

[David S. Miller]

On Thu, 2002-12-05 at 12:53, Bingner Sam J Contractor PACAF CSS/SCHE
wrote:
> Attached is a patch that seems to work for the hidden flag in 2.4.20... for
> anybody else who needs this functionality

Use the ARP filter netfilter module or the routing based solutions
instead, please.

Re: hidden interface (ARP) 2.4.20

[Phil Oester]

So we should enable netfilter for all x-hundred webservers we have?  Or play games with routing tables?

Why was something which:

a) works
b) was present in 2.2.xx kernels
c) is trivial to include and doesn't seem to 'hurt' anything

ripped from 2.4 kernels?

What some people fail to grasp is that _many_ people in the real world are using the hidden flag in load balancing scenarios for its simplicity.  Removing it (without any particularly valid reason that anyone is aware of) doesn't make much sense.

-Phil

p.s. flame away, Dave

On Thu, Dec 05, 2002 at 01:42:10PM -0800, David S. Miller wrote:
> On Thu, 2002-12-05 at 12:53, Bingner Sam J Contractor PACAF CSS/SCHE
> wrote:
> > Attached is a patch that seems to work for the hidden flag in 2.4.20... for
> > anybody else who needs this functionality
> 
> Use the ARP filter netfilter module or the routing based solutions
> instead, please.

Re: hidden interface (ARP) 2.4.20

[Roberto Nibali]

Hello,

First I would like to ask people not to post such patches to lkml but 
rather to the LVS list, because this affects only LVS so far and we 
cover all kernel versions pretty much up to date. Julian just needs to 
do the s/__constant_htons/htons/ fixes and upload the changes to his site ;)

The inclusion of the hidden feature has been discussed almost to death 
on netdev (where these questions should have gone in the first place) 
and it was decided against inclusion of this patch for various reasons.

Phil Oester wrote:
> So we should enable netfilter for all x-hundred webservers we have?  Or play games with routing tables?

Yes. What is the problem? You need to setup the x-hundred webservers 
anyway, 2 routing entry lines certainly won't hurt. Yes, I understand 
that if you're in process of upgrading your webservers from 2.2.x to 
2.4.x this is a bit of an additional pain. There are also other 
solutions to this arp problem, but please address this on the LVS 
mailinglist.

> Why was something which:
> 
> a) works
> b) was present in 2.2.xx kernels
> c) is trivial to include and doesn't seem to 'hurt' anything
> 
> ripped from 2.4 kernels?

http://marc.theaimsgroup.com/?t=95743539800002&r=1&w=2

> What some people fail to grasp is that _many_ people in the real world are using 
 > the hidden flag in load balancing scenarios for its simplicity.
 > Removing it (without any particularly valid reason that anyone is
 > aware of) doesn't make much sense.

Depends if it was a hack before that shouldn't have been there in the 
first place. In an evolutionary process things get optimized ... as has 
happened with the network stack code.

> -Phil
> 
> p.s. flame away, Dave

Search the LVS and the netdev archives for constructive discussions 
about it. No need to flame anyone. But hey, if people keep coming up 
with this, DaveM and Alexey might get weak and put it back in 2.5.x :)

Best regards,
Roberto Nibali, ratz
-- 
echo '[q]sa[ln0=aln256%Pln256/snlbx]sb3135071790101768542287578439snlbxq'|dc

Re: hidden interface (ARP) 2.4.20

[Phil Oester]

On Thu, Dec 05, 2002 at 11:50:45PM +0100, Roberto Nibali wrote:
> First I would like to ask people not to post such patches to lkml but 
> rather to the LVS list, because this affects only LVS so far and we 

*snip*

Eh?  Last I checked, there were other loadbalancing solutions out there.  Some
even use hardware (like ours).  IOW - LVS isn't the only use for the hidden flag.

This flag is useful for many folks, and is a generic (not LVS specific) kernel feature
which many of us would like to see added back to the kernel.

Granted, further discussion on the matter is an exercise in futility, as the current
net maintainer has already stated his disdain for it.  So...we'll go on patching
our kernels ad infinitum.

-Phil

Re: hidden interface (ARP) 2.4.20

[Roberto Nibali]

Hi,

[trimmed list drastically because I get a lot of bounce emails back]

> Eh?  Last I checked, there were other loadbalancing solutions out there.  Some
> even use hardware (like ours).  IOW - LVS isn't the only use for the hidden flag.

Oops, right. I forgot the HW LBs that do triangulation. I wonder 
however, why one wants to use a HW LB and not configure it to work in 
NAT mode.

> Granted, further discussion on the matter is an exercise in futility, as the current
> net maintainer has already stated his disdain for it.  So...we'll go on patching
> our kernels ad infinitum.

As mentioned before, you don't necessarily need to patch your kernels, 
there are other possibilities to overcome the arp problem. You could (if 
not already done so) consult the LVS howto where solutions are certainly 
applicable also to non-LVS LBs.

Regards and sorry for my wrong assumptions,
Roberto Nibali, ratz
-- 
echo '[q]sa[ln0=aln256%Pln256/snlbx]sb3135071790101768542287578439snlbxq'|dc

Re: hidden interface (ARP) 2.4.20

[Willy Tarreau]

On Fri, Dec 06, 2002 at 12:59:38AM +0100, Roberto Nibali wrote:
<snip>
> Oops, right. I forgot the HW LBs that do triangulation. I wonder 
> however, why one wants to use a HW LB and not configure it to work in 
> NAT mode.

Because when you have to deal with thousands of session per second, NAT is
really a pain in the ass. When you have to consider security, NAT is a pain
too because it makes end to end tracking much more difficult when you deal with
multiple proxy levels. And last but not least, there are protocols that don't
like NAT. Simply think about a farm of FTP servers. It's really easy to
load-balance internet clients on it with redirection (call it as you like) using
a hash algorithm. NAT would be more difficult.

I personnaly suggested and used both NAT and redirection at a big customer's
site. NAT was there only to be compatible with broken systems that would never
handle redirection right, in the event we would have to use them. But at the
moment it's already the first source of architectural problems.

Cheers,
Willy

Re: hidden interface (ARP) 2.4.20

[Stephan von Krawczynski]

On Fri, 6 Dec 2002 07:01:35 +0100
Willy Tarreau <willy@w.ods.org> wrote:

> On Fri, Dec 06, 2002 at 12:59:38AM +0100, Roberto Nibali wrote:
> <snip>
> > Oops, right. I forgot the HW LBs that do triangulation. I wonder 
> > however, why one wants to use a HW LB and not configure it to work in 
> > NAT mode.
> 
> Because when you have to deal with thousands of session per second, NAT is
> really a pain in the ass. When you have to consider security, NAT is a pain
> too because it makes end to end tracking much more difficult when you deal
> with multiple proxy levels.

Oh, a poor soul who experienced my everyday life ... ;-)
netfilter-NAT may be a real nice choice for your-cool-server-at-home. Talking
about many thousand NATted sessions you may as well flush it through the
toilet. sorry for the open words.
In complete contrary I have _never_ experienced problems with the hidden patch
(after correct setup of the boxes). And for another reason: it is plain simple.

-- 
Regards,
Stephan

Re: hidden interface (ARP) 2.4.20

[Roberto Nibali]

Hello,

[Maybe we should discuss this in private, it doesn't have a lot to do 
with kernel development anymore.]

> Because when you have to deal with thousands of session per second, NAT is
> really a pain in the ass. When you have to consider security, NAT is a pain

Not with a HW LB, and with a SW LB (LVS-NAT) you can very well sustain 
20000 NAT'd load balanced connections with 5 minutes of stickyness 
(persistency) with 1GB RAM and a PIII Tualatin with 512 kb L2 cache. I'm 
not sure if you meant this when mentioning pain.

> too because it makes end to end tracking much more difficult when you deal with
> multiple proxy levels. And last but not least, there are protocols that don't

Security mustn't rely on the LB with such a high volume traffic service. 
You need a carefully designed firewall framework. At least in the setups 
I've been dealing with LBs (a couple dozen) this was the case. Load 
balancing a service with multiple proxies doing NAT certainly imposes an 
additional problem when doing proper end-to-end healthchecking, but is 
nothing that couldn't be solved or would be extremely difficult.

> like NAT. Simply think about a farm of FTP servers. It's really easy to
> load-balance internet clients on it with redirection (call it as you like) using
> a hash algorithm. NAT would be more difficult.

I agree completely with this one. Don't get me wrong, I also most of the 
time deploy a LB using the redirection method.

> I personnaly suggested and used both NAT and redirection at a big customer's
> site. NAT was there only to be compatible with broken systems that would never
> handle redirection right, in the event we would have to use them. But at the

HP/UX < 11i is such an example of horribly broken system. It can still 
be solved with direct routing (redirection, triangulation) but you need 
additional NICs.

> moment it's already the first source of architectural problems.

Best regards,
Roberto Nibali, ratz
-- 
echo '[q]sa[ln0=aln256%Pln256/snlbx]sb3135071790101768542287578439snlbxq'|dc

Re: hidden interface (ARP) 2.4.20

[Stephan von Krawczynski]

On Sun, 08 Dec 2002 00:30:23 +0100
Roberto Nibali <ratz@drugphish.ch> wrote:

> Hello,
> 
> [Maybe we should discuss this in private, it doesn't have a lot to do 
> with kernel development anymore.]

To be honest: I think this _is_ indeed a kernel development issue. We are
somehow talking of a performance lack that can be overcome by a simple patch
(call it hack) and some brain.

> > Because when you have to deal with thousands of session per second, NAT is
> > really a pain in the ass. When you have to consider security, NAT is a pain
> 
> Not with a HW LB, and with a SW LB (LVS-NAT) you can very well sustain 
> 20000 NAT'd load balanced connections with 5 minutes of stickyness 
> (persistency) with 1GB RAM and a PIII Tualatin with 512 kb L2 cache. I'm 
> not sure if you meant this when mentioning pain.

I guess he probably meant a _bit_ more. I may add some zeros to your 20000 to
give you a glimpse of a _standard_ load we are talking about. And you can
easily do this with the hardware you mentioned _not_ using NAT (of course ;-).

I guess it would really be a great help if someone did tests like Cons'
"overall performance" ones for network performance explicitly. Like e.g.
performance for various packet-sizes of all available protocol types, possibly
including NAT connections. We have no comparable figures at hand right now, I
guess.

-- 
Regards,
Stephan

Re: hidden interface (ARP) 2.4.20

[Willy Tarreau]

On Sun, Dec 08, 2002 at 05:03:36PM +0100, Stephan von Krawczynski wrote:
> > Not with a HW LB, and with a SW LB (LVS-NAT) you can very well sustain 
> > 20000 NAT'd load balanced connections with 5 minutes of stickyness 
> > (persistency) with 1GB RAM and a PIII Tualatin with 512 kb L2 cache. I'm 
> > not sure if you meant this when mentioning pain.
> 
> I guess he probably meant a _bit_ more. I may add some zeros to your 20000 to
> give you a glimpse of a _standard_ load we are talking about. And you can
> easily do this with the hardware you mentioned _not_ using NAT (of course ;-).

You're right, we have been discussing this privately and agreed we were both
talking about higher numbers ; Robert seems to have a good experience of very
high traffic ;-)

> I guess it would really be a great help if someone did tests like Cons'
> "overall performance" ones for network performance explicitly. Like e.g.
> performance for various packet-sizes of all available protocol types, possibly
> including NAT connections. We have no comparable figures at hand right now, I
> guess.

Why not ?
I've often been doing this to check the reliability of the network layer of
kernels that I distribute. I often use Tux for this, because it can easily
sustain 10k hits/s during months. But Tux is not in mainstream kernel, we have
to use other tools. Since I'm working on a task scheduler, I may soon have the
base to rewrite my injecter and a fake server to do these tests on mainstream
kernels. I think that several tools already exist for this. You can take a look
at the C10K project to find links. I don't have the URL in mind, google is your
friend.

Cheers,
Willy

Re: hidden interface (ARP) 2.4.20

[Stephan von Krawczynski]

On Sun, 8 Dec 2002 18:01:35 +0100
Willy Tarreau <willy@w.ods.org> wrote:

> > I guess it would really be a great help if someone did tests like Cons'
> > "overall performance" ones for network performance explicitly. Like e.g.
> > performance for various packet-sizes of all available protocol types,
> > possibly including NAT connections. We have no comparable figures at hand
> > right now, I guess.
> 
> Why not ?
> I've often been doing this to check the reliability of the network layer of
> kernels that I distribute. I often use Tux for this, because it can easily
> sustain 10k hits/s during months.

This is unfortunately not sufficient, not even close to. If you really want to
have a good idea what is going on you should as well check out what is happening
with packet sizes a lot smaller than 1500 (normal mtu). Check data rate an
packet loss with packet sizes around 80 bytes or so to get an idea what bothers
us :-)

-- 
Regards,
Stephan

Re: hidden interface (ARP) 2.4.20

[Gilad Ben-Yossef]

On Mon, 2002-12-09 at 13:08, Stephan von Krawczynski wrote:

> This is unfortunately not sufficient, not even close to. If you really want to
> have a good idea what is going on you should as well check out what is happening
> with packet sizes a lot smaller than 1500 (normal mtu). Check data rate an
> packet loss with packet sizes around 80 bytes or so to get an idea what bothers
> us :-)

VoIP? :-)

Have you tried those tests with the NAPI network drivers patch? my
experience has been that interrupt live lock is what's hitting you
really hard in the scenarios you described and the NAPI network drivers
patch id supposed to fix that. 


Gilad



-- 
 Gilad Ben-Yossef <gilad@benyossef.com> 
 http://benyossef.com 
 "Denial really is a river in Eygept."

Re: hidden interface (ARP) 2.4.20

[Roberto Nibali]

Hello,

>>Why not ?
>>I've often been doing this to check the reliability of the network layer of
>>kernels that I distribute. I often use Tux for this, because it can easily
>>sustain 10k hits/s during months.
>  
> This is unfortunately not sufficient, not even close to. If you really want to
> have a good idea what is going on you should as well check out what is happening
> with packet sizes a lot smaller than 1500 (normal mtu). Check data rate an
> packet loss with packet sizes around 80 bytes or so to get an idea what bothers
> us :-)

But this doesn't have anything to do with the hidden patch! It can be 
multiple things:

o missing TCP segment offload support
o inefficient zerocopy DMA support
o IRQ routing problems
o wrong QoS settings

Could you please be more specific on what exactly you're trying to 
achieve? Do you want to load balance an application whose average 
package size is 80 bytes? How many sustained connections per seconds do 
you have?

Best regards,
Roberto Nibali, ratz
-- 
echo '[q]sa[ln0=aln256%Pln256/snlbx]sb3135071790101768542287578439snlbxq'|dc

Re: hidden interface (ARP) 2.4.20 / network performance

[Stephan von Krawczynski]

On Tue, 10 Dec 2002 11:40:18 +0100
Roberto Nibali <ratz@drugphish.ch> wrote:

> > This is unfortunately not sufficient, not even close to. If you really want
> > to have a good idea what is going on you should as well check out what is
> > happening with packet sizes a lot smaller than 1500 (normal mtu). Check
> > data rate an packet loss with packet sizes around 80 bytes or so to get an
> > idea what bothers us :-)
> 
> But this doesn't have anything to do with the hidden patch! It can be 
> multiple things:
> 
> o missing TCP segment offload support
> o inefficient zerocopy DMA support

cannot comment these.

> o IRQ routing problems

no.

> o wrong QoS settings

no.

> Could you please be more specific on what exactly you're trying to 
> achieve? Do you want to load balance an application whose average 
> package size is 80 bytes? How many sustained connections per seconds do 
> you have?

Well, what I am trying to say is this: my experience is that under load with
small sized packets even standard routing/packet forwarding becomes lossy. If I
put NAT and other nice netfilter features on top of such a situation things get
a lot worse (obviously) - no comparison to building the "application" (e.g.
cluster) with routing and hidden-patch (mainly because of its pure simplicity I
guess).
Don't get me wrong: I am pretty content with the hidden-patch and my setup
without NAT. But I wanted to point to the direction of possible further routing
performance improvement in 2.4.X tree. Is it correct that I can expect higher
data-rates (concerning small packets) if using higher HZ ?
Someone selling E3 cards told me he cannot manage loads like these (small
packet stuff) with a stock kernel, and that you _at least_ have to increase HZ
to get acceptable throughput results.

-- 
Regards,
Stephan

Re: hidden interface (ARP) 2.4.20 / network performance

[Roberto Nibali]

>>Could you please be more specific on what exactly you're trying to 
>>achieve? Do you want to load balance an application whose average 
>>package size is 80 bytes? How many sustained connections per seconds do 
>>you have?
>  
> Well, what I am trying to say is this: my experience is that under load with
> small sized packets even standard routing/packet forwarding becomes lossy. If I

That's a known fact, however I experienced Linux to perform the best in 
such worst case situations among several other Unices I tested, and NAPI 
certainly has brought some improvements in this area.

> put NAT and other nice netfilter features on top of such a situation things get
> a lot worse (obviously) - no comparison to building the "application" (e.g.
> cluster) with routing and hidden-patch (mainly because of its pure simplicity I
> guess).

I'm afraid but I don't understand what you mean with the second part of 
your statement.

> Don't get me wrong: I am pretty content with the hidden-patch and my setup
> without NAT. But I wanted to point to the direction of possible further routing
> performance improvement in 2.4.X tree. Is it correct that I can expect higher

Huh? Routing performance improvements? Routing is almost possible at 
wire speed. Some 60us delay per packet maybe (in case of load balancing 
decisions) but what do you want to improve?

I agree with you that netfilter NAT performance should and possibly can 
be impoved. And people are working on proof-of-concept improvements of 
NAPT in the Linux kernel including the netfilter team. But again, for me 
the hidden patch (http://www.ssi.bg/~ja/hidden-2.4.20pre10-1.diff) as it 
can be found does nothing to improve your situation.

> data-rates (concerning small packets) if using higher HZ ?

I doubt this would help much but I haven't tested it and I do not see 
all consequences on the routing, the routing cache and the FIB policy of 
modifying HZ. I couldn't comment on that.

> Someone selling E3 cards told me he cannot manage loads like these (small
> packet stuff) with a stock kernel, and that you _at least_ have to increase HZ
> to get acceptable throughput results.

Now that certainly is interesting, does he have any nice numbers to back 
this up? I'd be very interested. Also I've cc'd Jamal (I hope he will 
forgive me for that) who's working in this field since a couple of years 
now. Maybe he can comment on the HZ changes.

Best regards,
Roberto Nibali, ratz
-- 
echo '[q]sa[ln0=aln256%Pln256/snlbx]sb3135071790101768542287578439snlbxq'|dc

Re: hidden interface (ARP) 2.4.20 / network performance

[Willy TARREAU]

On Tue, Dec 10, 2002 at 02:09:12PM +0100, Stephan von Krawczynski wrote:
 
> Well, what I am trying to say is this: my experience is that under load with
> small sized packets even standard routing/packet forwarding becomes lossy.

This is more often dependant on hardware itself (NICs, chipsets). When your NIC
doesn't support scatter/gather, mitigated interrupts and other wonderful features,
and it receives 148600 pkts/second, it generates as many interrupts. Many chipsets
completely die under such a load. I can tell you that I wasn't proud of hanging my
Dual Athlon 1800+ with its 64/66 PCI slots and so from a single Celeron 800 on
100 Mbps copper !

> If I put NAT and other nice netfilter features on top of such a situation things
> get a lot worse (obviously) - no comparison to building the "application" (e.g.
> cluster) with routing and hidden-patch (mainly because of its pure simplicity I
> guess).

don't even need that to kill a system. Only a cheap NIC, a responding MAC address
and that's all. Of course routing make it worse and NAT even more. And BTW, when I
get 10 to 12 kHits/s with Tux on a 100 Mbps network, you'll notice that it only
happens on empty files. This is about 1 kB per hit, from a wire point of vue.
Count the ACKs, the data (tcp headers), and global overhead, and you're not far
from wire-speed on very small packets.

> Don't get me wrong: I am pretty content with the hidden-patch and my setup
> without NAT. But I wanted to point to the direction of possible further routing
> performance improvement in 2.4.X tree. Is it correct that I can expect higher
> data-rates (concerning small packets) if using higher HZ ?

don't know. perhaps forwarding packets between input and output involves queues
that are processed alternatively at HZ rate, but that seems strange to me.

> Someone selling E3 cards told me he cannot manage loads like these (small
> packet stuff) with a stock kernel, and that you _at least_ have to increase HZ
> to get acceptable throughput results.

E3 is only 45 Mbps (or I'm mistaken) ? Tweaking such parameters for such medium
rates doesn't seem the most appropriate to me. Perhaps his driver has some problems.

Cheers,
Willy

Re: hidden interface (ARP) 2.4.20

[Bill Davidsen]

On Fri, 6 Dec 2002, Roberto Nibali wrote:

> As mentioned before, you don't necessarily need to patch your kernels, 
> there are other possibilities to overcome the arp problem. You could (if 
> not already done so) consult the LVS howto where solutions are certainly 
> applicable also to non-LVS LBs.

In spite of vast resistance from some developers, it is highly desirable
on some systems to force a packet with a given source IP out an interface
which has that IP assigned. With this capability it is possible to make a
single machine with multiple NICs behave like two machines with individual
NICs. This also solves ARP issues, although it's a much more general
thing.

Think development, think debug, think UML virtual machines, or a machine
with multiple boundary routers which are addressed to separate NICs. The
usual proxy_arp fix doesn't really address these cases. If I have multiple
default routes, the router on which the packet arrived is likely to be on
the route with the fewest hops, randomly using another route doesn't help.

Source routing takes too much overhead for lots of connections, and as I
recall is limited to 256 rules. I'm not sure the hidden interface patch
really does this, although I just looked quickly.

Patches like the hidden interface will continue as long as there are
useful things people want to do with Linux and can't. It's one of many in
the networking area. I don't expect them to be adopted in the main kernel,
but as long as they're easier than making multiple configs, particularly
at runtime, they will be around.

-- 
bill davidsen <davidsen@tmr.com>
  CTO, TMR Associates, Inc
Doing interesting things with little computers since 1979.

Re: hidden interface (ARP) 2.4.20

[Roberto Nibali]

Hi,

Maybe I should first note that I am not against the hidden patch at all, 
I just accepted that it won't go in. I use it on an almost daily basis.

> In spite of vast resistance from some developers, it is highly desirable
> on some systems to force a packet with a given source IP out an interface
> which has that IP assigned. With this capability it is possible to make a

Yes, source address selection based on different rules and routing 
tables. What does it have to do with the hidden patch?

> single machine with multiple NICs behave like two machines with individual
> NICs. This also solves ARP issues, although it's a much more general
> thing.

Yes.

> Think development, think debug, think UML virtual machines, or a machine
> with multiple boundary routers which are addressed to separate NICs. The

Ok, I've quite a few boxes running that way.

> usual proxy_arp fix doesn't really address these cases. If I have multiple
> default routes, the router on which the packet arrived is likely to be on
> the route with the fewest hops, randomly using another route doesn't help.

??? Depends how you use those multiple default routes. If you do nexthop 
routing you do sort of RR balancing on preferred routes. If you do 
source address selection routing based on rules you have fixed default 
routes which will not match because of the fewest hops but because of 
the rule. I am a bit confused as to what you're trying to tell me.

> Source routing takes too much overhead for lots of connections, and as I

Either we have a different view of source routing or I have to ask you 
why you think there is too much overhead with source routing.

> recall is limited to 256 rules. I'm not sure the hidden interface patch
> really does this, although I just looked quickly.

The hidden patch doesn't do source routing and the limit of available 
source routes is 254 but not because of the rules (you can have 2**16 
rule entries) but because of the amount of routing tables which is 256 
[0..255] minus local table minus main table which equals to 254 tables.

> Patches like the hidden interface will continue as long as there are
> useful things people want to do with Linux and can't. It's one of many in

Yes, that's the nice thing about patches, isn't it :).

> the networking area. I don't expect them to be adopted in the main kernel,
> but as long as they're easier than making multiple configs, particularly
> at runtime, they will be around.

Yes, definitely. And I think noone has said anything against that.

Best regards,
Roberto Nibali, ratz
-- 
echo '[q]sa[ln0=aln256%Pln256/snlbx]sb3135071790101768542287578439snlbxq'|dc

Re: hidden interface (ARP) 2.4.20

[Bill Davidsen]

On Tue, 10 Dec 2002, Roberto Nibali wrote:

> Maybe I should first note that I am not against the hidden patch at all, 
> I just accepted that it won't go in. I use it on an almost daily basis.
> 
> > In spite of vast resistance from some developers, it is highly desirable
> > on some systems to force a packet with a given source IP out an interface
> > which has that IP assigned. With this capability it is possible to make a
> 
> Yes, source address selection based on different rules and routing 
> tables. What does it have to do with the hidden patch?

  I see it as an alternative solution to the problem the hidden patch is
addressing. Perhaps a more general one, although the method of causing
such routing might not be source routing in the "ip" sense.

> > usual proxy_arp fix doesn't really address these cases. If I have multiple
> > default routes, the router on which the packet arrived is likely to be on
> > the route with the fewest hops, randomly using another route doesn't help.
> 
> ??? Depends how you use those multiple default routes. If you do nexthop 
> routing you do sort of RR balancing on preferred routes. If you do 
> source address selection routing based on rules you have fixed default 
> routes which will not match because of the fewest hops but because of 
> the rule. I am a bit confused as to what you're trying to tell me.

I have in mid multiple ISPs for redundancy, perhaps a pair of OC12s or
similar. Sites would be reachable from either, but fewer hops to one or
the other. When the client connects, it avoids asymmetric routing to reply
on the same router.

> > Source routing takes too much overhead for lots of connections, and as I
> 
> Either we have a different view of source routing or I have to ask you 
> why you think there is too much overhead with source routing.

More rules, more overhead, having to set up a rule per IP (which can be
dynamic) takes overhead.

> > recall is limited to 256 rules. I'm not sure the hidden interface patch
> > really does this, although I just looked quickly.
> 
> The hidden patch doesn't do source routing and the limit of available 
> source routes is 254 but not because of the rules (you can have 2**16 
> rule entries) but because of the amount of routing tables which is 256 
> [0..255] minus local table minus main table which equals to 254 tables.

I actually meant that the patch didn't do this in another way, but you
have noted that the number of routing tables is limited. That may or may
not be a limitation depending on complexity. In any case a single
use-configured-interface patch avoids having tables.

> > Patches like the hidden interface will continue as long as there are
> > useful things people want to do with Linux and can't. It's one of many in
> 
> Yes, that's the nice thing about patches, isn't it :).
> 
> > the networking area. I don't expect them to be adopted in the main kernel,
> > but as long as they're easier than making multiple configs, particularly
> > at runtime, they will be around.
> 
> Yes, definitely. And I think noone has said anything against that.

I thought this thread had a "please don't post patches like that we don't
want it in the kernel" early on in the thread, but I've expired the
message and lack time to dig archives.

-- 
bill davidsen <davidsen@tmr.com>
  CTO, TMR Associates, Inc
Doing interesting things with little computers since 1979.

Re: hidden interface (ARP) 2.4.20

[Roberto Nibali]

>>Yes, source address selection based on different rules and routing 
>>tables. What does it have to do with the hidden patch?
>  
>   I see it as an alternative solution to the problem the hidden patch is
> addressing. Perhaps a more general one, although the method of causing
> such routing might not be source routing in the "ip" sense.

Ohh, now I see where you're coming from. You mean the additional 
blackhole routes you need to add on every box that need to mimic the 
'non-arp parlance' or the 'do not choose this address for reply', right?

>>??? Depends how you use those multiple default routes. If you do nexthop 
>>routing you do sort of RR balancing on preferred routes. If you do 
>>source address selection routing based on rules you have fixed default 
>>routes which will not match because of the fewest hops but because of 
>>the rule. I am a bit confused as to what you're trying to tell me.
>  
> I have in mid multiple ISPs for redundancy, perhaps a pair of OC12s or
> similar. Sites would be reachable from either, but fewer hops to one or
> the other. When the client connects, it avoids asymmetric routing to reply
> on the same router.

I understand everything but the last sentence. You have a couple of 
redundant ISP links which can all act as a router to the Internet, the 
only difference is that if you go over some of them you need less hops. 
Now in order to avoid asymmetric routing you need the hidden patch? I 
apologise for being so narrow minded but I still don't get it.

>>>Source routing takes too much overhead for lots of connections, and as I
>>
>>Either we have a different view of source routing or I have to ask you 
>>why you think there is too much overhead with source routing.
>  
> More rules, more overhead, having to set up a rule per IP (which can be
> dynamic) takes overhead.

Only if you change your rules once every 1000 packets maybe but other 
than that I doubt there is a significant overhead to the hidden patch. I 
would denote the overhead as being something in the range of O(log N), 
with N being the amount of routes. The way I understand the source 
address selection algorithm efficiency for routing decision is that you 
look up the fast routing cache and if there is no hit you try to find 
the preferred route by walking the tree-like structure of rules and 
their according routes. Of course you have a worst case bounce table 
walking if every rule matches but no route in the according table can be 
selected (this would be a pretty stupid setup to begin with). This would 
mean a complete walk through all routing tables until you have a 
preferred match. In this case it is 0(N) but after that it is in the 
routing cache and therefore O(1) again :).

Please anyone correct me if I'm wrong.

>>>recall is limited to 256 rules. I'm not sure the hidden interface patch
>>>really does this, although I just looked quickly.
>>
>>The hidden patch doesn't do source routing and the limit of available 
>>source routes is 254 but not because of the rules (you can have 2**16 
>>rule entries) but because of the amount of routing tables which is 256 
>>[0..255] minus local table minus main table which equals to 254 tables.
>  
> I actually meant that the patch didn't do this in another way, but you
> have noted that the number of routing tables is limited. That may or may
> not be a limitation depending on complexity. In any case a single
> use-configured-interface patch avoids having tables.

That is something I certainly agree with you.

>>>the networking area. I don't expect them to be adopted in the main kernel,
>>>but as long as they're easier than making multiple configs, particularly
>>>at runtime, they will be around.
>>
>>Yes, definitely. And I think noone has said anything against that.
>  
> I thought this thread had a "please don't post patches like that we don't
> want it in the kernel" early on in the thread, but I've expired the
> message and lack time to dig archives.

You're right. After rereading my email I think I owe the original poster 
my apology for those rather harsh words. He even cc'd Julian who is the 
author and maintainer of those patches.

Best regards,
Roberto Nibali, ratz
-- 
echo '[q]sa[ln0=aln256%Pln256/snlbx]sb3135071790101768542287578439snlbxq'|dc

Re: hidden interface (ARP) 2.4.20

[Bill Davidsen]

On Tue, 10 Dec 2002, Roberto Nibali wrote:

> > I have in mid multiple ISPs for redundancy, perhaps a pair of OC12s or
> > similar. Sites would be reachable from either, but fewer hops to one or
> > the other. When the client connects, it avoids asymmetric routing to reply
> > on the same router.
> 
> I understand everything but the last sentence. You have a couple of 
> redundant ISP links which can all act as a router to the Internet, the 
> only difference is that if you go over some of them you need less hops. 
> Now in order to avoid asymmetric routing you need the hidden patch? I 
> apologise for being so narrow minded but I still don't get it.

Don't. You are right about this one, a client originated connection will
have an ARP entry and route back by the original route. Connections
originated on the dual-homed system might put a packet out on either NIC,
from any IP, that's a different issue, and the whole hidden interface
patch really doesn't address it.

I was mixing things from two problems I've seen, sorry for any confusion.

-- 
bill davidsen <davidsen@tmr.com>
  CTO, TMR Associates, Inc
Doing interesting things with little computers since 1979.

Re: hidden interface (ARP) 2.4.20

[Bernd Eckenfels]

In article <Pine.LNX.3.96.1021211110329.18520C-100000@gatekeeper.tmr.com> you wrote:
> Don't. You are right about this one, a client originated connection will
> have an ARP entry and route back by the original route.

Most likely it will not have an ARP entry, since there is only the ARP entry
for the router. Not-directly connected hosts are not listed in the
neighbours cache, but yes, you will have an routing cache entry (netstat
-C).


Greetings
Bernd
-- 
eckes privat - http://www.eckes.org/
Project Freefire - http://www.freefire.org/

Re: hidden interface (ARP) 2.4.20

[Martin Josefsson]

On Thu, 2002-12-05 at 22:42, David S. Miller wrote:
> On Thu, 2002-12-05 at 12:53, Bingner Sam J Contractor PACAF CSS/SCHE
> wrote:
> > Attached is a patch that seems to work for the hidden flag in 2.4.20... for
> > anybody else who needs this functionality
> 
> Use the ARP filter netfilter module or the routing based solutions
> instead, please.

How does one use the arpfilter module? Any pointers to userspace
utilities? 

-- 
/Martin

Never argue with an idiot. They drag you down to their level, then beat
you with experience.

RE: hidden interface (ARP) 2.4.20

[Bingner Sam J Contractor PACAF CSS/SCHE]

to risk getting jumped on again, I still don't see why an address that is
-=ASSIGNED TO AN INTERFACE=- should be responded to on a completely
different interface... if we wanted the ip address to be assigned to the
system, there should be a pseudo interface that will work on any of the
interfaces attached.  Why assign an address to an interface if it would work
just the same if you assigned it to the loopback adapter?  Why would you
assign an address to the loopback adapter if you wanted it to be accessed
from the world?  

Anyways, just wasting my breath by expressing my point of view... have fun

Also, if anybody has a link to something that explains how to do this using
an alternate method, or usage for arp_filter... I'd appreciate it if you
could email me... I've been searching for like 2 hours and I havn't found
anything useful.

	Sam Bingner

-----Original Message-----
From: Roberto Nibali [mailto:ratz@drugphish.ch]
Sent: Thursday, December 05, 2002 12:51 PM
To: Phil Oester
Cc: David S. Miller; Bingner Sam J Contractor PACAF CSS/SCHE;
'ja@ssi.bg'; 'linux-kernel@vger.kernel.org'
Subject: Re: hidden interface (ARP) 2.4.20


Hello,

First I would like to ask people not to post such patches to lkml but 
rather to the LVS list, because this affects only LVS so far and we 
cover all kernel versions pretty much up to date. Julian just needs to 
do the s/__constant_htons/htons/ fixes and upload the changes to his site ;)

The inclusion of the hidden feature has been discussed almost to death 
on netdev (where these questions should have gone in the first place) 
and it was decided against inclusion of this patch for various reasons.

Phil Oester wrote:
> So we should enable netfilter for all x-hundred webservers we have?  Or
play games with routing tables?

Yes. What is the problem? You need to setup the x-hundred webservers 
anyway, 2 routing entry lines certainly won't hurt. Yes, I understand 
that if you're in process of upgrading your webservers from 2.2.x to 
2.4.x this is a bit of an additional pain. There are also other 
solutions to this arp problem, but please address this on the LVS 
mailinglist.

> Why was something which:
> 
> a) works
> b) was present in 2.2.xx kernels
> c) is trivial to include and doesn't seem to 'hurt' anything
> 
> ripped from 2.4 kernels?

http://marc.theaimsgroup.com/?t=95743539800002&r=1&w=2

> What some people fail to grasp is that _many_ people in the real world are
using 
 > the hidden flag in load balancing scenarios for its simplicity.
 > Removing it (without any particularly valid reason that anyone is
 > aware of) doesn't make much sense.

Depends if it was a hack before that shouldn't have been there in the 
first place. In an evolutionary process things get optimized ... as has 
happened with the network stack code.

> -Phil
> 
> p.s. flame away, Dave

Search the LVS and the netdev archives for constructive discussions 
about it. No need to flame anyone. But hey, if people keep coming up 
with this, DaveM and Alexey might get weak and put it back in 2.5.x :)

Best regards,
Roberto Nibali, ratz
-- 
echo '[q]sa[ln0=aln256%Pln256/snlbx]sb3135071790101768542287578439snlbxq'|dc