[PATCH net-next-2.6] gro: __napi_gro_receive() optimizations

[PATCH net-next-2.6] gro: __napi_gro_receive() optimizations

[Eric Dumazet]

compare_ether_header() can have a special implementation on 64 bit
arches if CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS is defined

__napi_gro_receive() can avoid a conditional branch to perform device
match.

__napi_gro_receive() can be used from vlan_gro_common() instead of being
duplicated.

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
---
 include/linux/etherdevice.h |   10 +++++++++-
 include/linux/netdevice.h   |    2 ++
 net/8021q/vlan_core.c       |   14 ++------------
 net/core/dev.c              |   12 +++++++-----
 4 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/include/linux/etherdevice.h b/include/linux/etherdevice.h
index 2308fbb..02144fd 100644
--- a/include/linux/etherdevice.h
+++ b/include/linux/etherdevice.h
@@ -237,13 +237,21 @@ static inline bool is_etherdev_addr(const struct net_device *dev,
  * entry points.
  */
 
-static inline int compare_ether_header(const void *a, const void *b)
+static inline unsigned long compare_ether_header(const void *a, const void *b)
 {
+#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64
+	unsigned long fold;
+
+	fold = *(unsigned long *)a ^ *(unsigned long *)b;
+	fold |= *(unsigned long *)(a + 6) ^ *(unsigned long *)(b + 6);
+	return fold;
+#else
 	u32 *a32 = (u32 *)((u8 *)a + 2);
 	u32 *b32 = (u32 *)((u8 *)b + 2);
 
 	return (*(u16 *)a ^ *(u16 *)b) | (a32[0] ^ b32[0]) |
 	       (a32[1] ^ b32[1]) | (a32[2] ^ b32[2]);
+#endif
 }
 
 #endif	/* _LINUX_ETHERDEVICE_H */
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 59962db..629773e 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -1695,6 +1695,8 @@ extern gro_result_t	dev_gro_receive(struct napi_struct *napi,
 extern gro_result_t	napi_skb_finish(gro_result_t ret, struct sk_buff *skb);
 extern gro_result_t	napi_gro_receive(struct napi_struct *napi,
 					 struct sk_buff *skb);
+extern gro_result_t   __napi_gro_receive(struct napi_struct *napi,
+					 struct sk_buff *skb);
 extern void		napi_reuse_skb(struct napi_struct *napi,
 				       struct sk_buff *skb);
 extern struct sk_buff *	napi_get_frags(struct napi_struct *napi);
diff --git a/net/8021q/vlan_core.c b/net/8021q/vlan_core.c
index 07eeb5b..947650a 100644
--- a/net/8021q/vlan_core.c
+++ b/net/8021q/vlan_core.c
@@ -102,19 +102,9 @@ vlan_gro_common(struct napi_struct *napi, struct vlan_group *grp,
 	if (vlan_dev)
 		skb->dev = vlan_dev;
 	else if (vlan_id)
-		goto drop;
-
-	for (p = napi->gro_list; p; p = p->next) {
-		NAPI_GRO_CB(p)->same_flow =
-			p->dev == skb->dev && !compare_ether_header(
-				skb_mac_header(p), skb_gro_mac_header(skb));
-		NAPI_GRO_CB(p)->flush = 0;
-	}
-
-	return dev_gro_receive(napi, skb);
+		return GRO_DROP;
 
-drop:
-	return GRO_DROP;
+	return __napi_gro_receive(napi, skb);
 }
 
 gro_result_t vlan_gro_receive(struct napi_struct *napi, struct vlan_group *grp,
diff --git a/net/core/dev.c b/net/core/dev.c
index 859e30f..61e9c96 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3169,21 +3169,23 @@ normal:
 }
 EXPORT_SYMBOL(dev_gro_receive);
 
-static gro_result_t
-__napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
+gro_result_t __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
 {
 	struct sk_buff *p;
 
 	for (p = napi->gro_list; p; p = p->next) {
-		NAPI_GRO_CB(p)->same_flow =
-			(p->dev == skb->dev) &&
-			!compare_ether_header(skb_mac_header(p),
+		unsigned long diffs;
+
+		diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
+		diffs |= compare_ether_header(skb_mac_header(p),
 					      skb_gro_mac_header(skb));
+		NAPI_GRO_CB(p)->same_flow = !diffs;
 		NAPI_GRO_CB(p)->flush = 0;
 	}
 
 	return dev_gro_receive(napi, skb);
 }
+EXPORT_SYMBOL(__napi_gro_receive);
 
 gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
 {

Re: [PATCH net-next-2.6] gro: __napi_gro_receive() optimizations

[Stephen Hemminger]

On Wed, 25 Aug 2010 22:33:51 +0200
Eric Dumazet <eric.dumazet@gmail.com> wrote:

> compare_ether_header() can have a special implementation on 64 bit
> arches if CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS is defined
> 
> __napi_gro_receive() can avoid a conditional branch to perform device
> match.
> 
> __napi_gro_receive() can be used from vlan_gro_common() instead of being
> duplicated.
> 
> Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>

Please, 2 patches. not mixing unrelated bits in one patch.

Re: [PATCH net-next-2.6] gro: __napi_gro_receive() optimizations

[Eric Dumazet]

Le mercredi 25 août 2010 à 13:45 -0700, Stephen Hemminger a écrit :
> On Wed, 25 Aug 2010 22:33:51 +0200
> Eric Dumazet <eric.dumazet@gmail.com> wrote:
> 
> > compare_ether_header() can have a special implementation on 64 bit
> > arches if CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS is defined
> > 
> > __napi_gro_receive() can avoid a conditional branch to perform device
> > match.
> > 
> > __napi_gro_receive() can be used from vlan_gro_common() instead of being
> > duplicated.
> > 
> > Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
> 
> Please, 2 patches. not mixing unrelated bits in one patch.


Well, all this stuff was added in a single commit, 

commit aa4b9f533ed5a22952e038b9fac2447ccc682124
Author: Herbert Xu <herbert@gondor.apana.org.au>
Date:   Sun Feb 8 18:00:37 2009 +0000

    gro: Optimise Ethernet header comparison


But yes, I guess I can cook three different patches, if you feel its
better this way...

Re: [PATCH net-next-2.6] gro: __napi_gro_receive() optimizations

[David Miller]

From: Eric Dumazet <eric.dumazet@gmail.com>
Date: Wed, 25 Aug 2010 22:33:51 +0200

> @@ -102,19 +102,9 @@ vlan_gro_common(struct napi_struct *napi, struct vlan_group *grp,
>  	if (vlan_dev)
>  		skb->dev = vlan_dev;
>  	else if (vlan_id)
> -		goto drop;
> -
> -	for (p = napi->gro_list; p; p = p->next) {
> -		NAPI_GRO_CB(p)->same_flow =
> -			p->dev == skb->dev && !compare_ether_header(
> -				skb_mac_header(p), skb_gro_mac_header(skb));
> -		NAPI_GRO_CB(p)->flush = 0;
> -	}
> -
> -	return dev_gro_receive(napi, skb);
> +		return GRO_DROP;
>  
> -drop:
> -	return GRO_DROP;
> +	return __napi_gro_receive(napi, skb);

I was looking at this the other day and considering something
similar but I didn't do it because this now makes the call chain
deeper.

And that can make a performance difference.

I don't want to add this hunk unless some GRO perf regression tests
are done.

Thanks.

Re: [PATCH net-next-2.6] gro: __napi_gro_receive() optimizations

[Eric Dumazet]

Le mercredi 25 août 2010 à 13:57 -0700, David Miller a écrit :
> From: Eric Dumazet <eric.dumazet@gmail.com>
> Date: Wed, 25 Aug 2010 22:33:51 +0200
> 
> > @@ -102,19 +102,9 @@ vlan_gro_common(struct napi_struct *napi, struct vlan_group *grp,
> >  	if (vlan_dev)
> >  		skb->dev = vlan_dev;
> >  	else if (vlan_id)
> > -		goto drop;
> > -
> > -	for (p = napi->gro_list; p; p = p->next) {
> > -		NAPI_GRO_CB(p)->same_flow =
> > -			p->dev == skb->dev && !compare_ether_header(
> > -				skb_mac_header(p), skb_gro_mac_header(skb));
> > -		NAPI_GRO_CB(p)->flush = 0;
> > -	}
> > -
> > -	return dev_gro_receive(napi, skb);
> > +		return GRO_DROP;
> >  
> > -drop:
> > -	return GRO_DROP;
> > +	return __napi_gro_receive(napi, skb);
> 
> I was looking at this the other day and considering something
> similar but I didn't do it because this now makes the call chain
> deeper.
> 
> And that can make a performance difference.
> 
> I don't want to add this hunk unless some GRO perf regression tests
> are done.
> 

Yes, we can inline it, this will speedup the non vlan case as well :)

Thanks !

[PATCH net-next-2.6 v2] gro: __napi_gro_receive() optimizations

compare_ether_header() can have a special implementation on 64 bit
arches if CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS is defined

__napi_gro_receive() can avoid a conditional branch to perform device
match.

__napi_gro_receive() can be used from vlan_gro_common() instead of being
duplicated.

As David requested, make it an inline function so that no extra level is
added in call chain.

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
---
 include/linux/etherdevice.h |   27 ++++++++++++++++++++++++++-
 net/8021q/vlan_core.c       |   15 ++-------------
 net/core/dev.c              |   16 ----------------
 3 files changed, 28 insertions(+), 30 deletions(-)

diff --git a/include/linux/etherdevice.h b/include/linux/etherdevice.h
index 2308fbb..9c58d68 100644
--- a/include/linux/etherdevice.h
+++ b/include/linux/etherdevice.h
@@ -237,13 +237,38 @@ static inline bool is_etherdev_addr(const struct net_device *dev,
  * entry points.
  */
 
-static inline int compare_ether_header(const void *a, const void *b)
+static inline unsigned long compare_ether_header(const void *a, const void *b)
 {
+#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64
+	unsigned long fold;
+
+	fold = *(unsigned long *)a ^ *(unsigned long *)b;
+	fold |= *(unsigned long *)(a + 6) ^ *(unsigned long *)(b + 6);
+	return fold;
+#else
 	u32 *a32 = (u32 *)((u8 *)a + 2);
 	u32 *b32 = (u32 *)((u8 *)b + 2);
 
 	return (*(u16 *)a ^ *(u16 *)b) | (a32[0] ^ b32[0]) |
 	       (a32[1] ^ b32[1]) | (a32[2] ^ b32[2]);
+#endif
+}
+
+static inline gro_result_t __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
+{
+	struct sk_buff *p;
+
+	for (p = napi->gro_list; p; p = p->next) {
+		unsigned long diffs;
+
+		diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
+		diffs |= compare_ether_header(skb_mac_header(p),
+					      skb_gro_mac_header(skb));
+		NAPI_GRO_CB(p)->same_flow = !diffs;
+		NAPI_GRO_CB(p)->flush = 0;
+	}
+
+	return dev_gro_receive(napi, skb);
 }
 
 #endif	/* _LINUX_ETHERDEVICE_H */
diff --git a/net/8021q/vlan_core.c b/net/8021q/vlan_core.c
index 07eeb5b..ce7b4b1 100644
--- a/net/8021q/vlan_core.c
+++ b/net/8021q/vlan_core.c
@@ -87,7 +87,6 @@ static gro_result_t
 vlan_gro_common(struct napi_struct *napi, struct vlan_group *grp,
 		unsigned int vlan_tci, struct sk_buff *skb)
 {
-	struct sk_buff *p;
 	struct net_device *vlan_dev;
 	u16 vlan_id;
 
@@ -102,19 +101,9 @@ vlan_gro_common(struct napi_struct *napi, struct vlan_group *grp,
 	if (vlan_dev)
 		skb->dev = vlan_dev;
 	else if (vlan_id)
-		goto drop;
-
-	for (p = napi->gro_list; p; p = p->next) {
-		NAPI_GRO_CB(p)->same_flow =
-			p->dev == skb->dev && !compare_ether_header(
-				skb_mac_header(p), skb_gro_mac_header(skb));
-		NAPI_GRO_CB(p)->flush = 0;
-	}
-
-	return dev_gro_receive(napi, skb);
+		return GRO_DROP;
 
-drop:
-	return GRO_DROP;
+	return __napi_gro_receive(napi, skb);
 }
 
 gro_result_t vlan_gro_receive(struct napi_struct *napi, struct vlan_group *grp,
diff --git a/net/core/dev.c b/net/core/dev.c
index 859e30f..195f9c7 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3169,22 +3169,6 @@ normal:
 }
 EXPORT_SYMBOL(dev_gro_receive);
 
-static gro_result_t
-__napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
-{
-	struct sk_buff *p;
-
-	for (p = napi->gro_list; p; p = p->next) {
-		NAPI_GRO_CB(p)->same_flow =
-			(p->dev == skb->dev) &&
-			!compare_ether_header(skb_mac_header(p),
-					      skb_gro_mac_header(skb));
-		NAPI_GRO_CB(p)->flush = 0;
-	}
-
-	return dev_gro_receive(napi, skb);
-}
-
 gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
 {
 	switch (ret) {

Re: [PATCH net-next-2.6] gro: __napi_gro_receive() optimizations

[David Miller]

From: Eric Dumazet <eric.dumazet@gmail.com>
Date: Wed, 25 Aug 2010 23:15:11 +0200

> Yes, we can inline it, this will speedup the non vlan case as well :)

Agreed.

But I believe you can find a better place to put __napi_gro_receive()
than etherdevice.h, even though yes it does have a hard dependency on
that thing. :-)

Re: [PATCH net-next-2.6] gro: __napi_gro_receive() optimizations

[Eric Dumazet]

Le jeudi 26 août 2010 à 00:51 -0700, David Miller a écrit :
> From: Eric Dumazet <eric.dumazet@gmail.com>
> Date: Wed, 25 Aug 2010 23:15:11 +0200
> 
> > Yes, we can inline it, this will speedup the non vlan case as well :)
> 
> Agreed.
> 
> But I believe you can find a better place to put __napi_gro_receive()
> than etherdevice.h, even though yes it does have a hard dependency on
> that thing. :-)
> 

I tried but failed :(

Or move __napi_gro_receive() in a new header,
because etherdevice.h needs to include netdevice.h

(is_etherdev_addr(), dev_hw_addr_random())

[PATCH net-next-2.6 v3] gro: __napi_gro_receive() optimizations

[Eric Dumazet]

compare_ether_header() can have a special implementation on 64 bit
arches if CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS is defined.

__napi_gro_receive() and vlan_gro_common() can avoid a conditional
branch to perform device match.

On x86_64, __napi_gro_receive() has now 38 instructions instead of 53

As gcc-4.4.3 still choose to not inline it, add inline keyword to this
performance critical function.

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
CC: Herbert Xu <herbert@gondor.apana.org.au>
---
 include/linux/etherdevice.h |   10 +++++++++-
 net/8021q/vlan_core.c       |    9 ++++++---
 net/core/dev.c              |   10 ++++++----
 3 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/include/linux/etherdevice.h b/include/linux/etherdevice.h
index 2308fbb..02144fd 100644
--- a/include/linux/etherdevice.h
+++ b/include/linux/etherdevice.h
@@ -237,13 +237,21 @@ static inline bool is_etherdev_addr(const struct net_device *dev,
  * entry points.
  */
 
-static inline int compare_ether_header(const void *a, const void *b)
+static inline unsigned long compare_ether_header(const void *a, const void *b)
 {
+#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64
+	unsigned long fold;
+
+	fold = *(unsigned long *)a ^ *(unsigned long *)b;
+	fold |= *(unsigned long *)(a + 6) ^ *(unsigned long *)(b + 6);
+	return fold;
+#else
 	u32 *a32 = (u32 *)((u8 *)a + 2);
 	u32 *b32 = (u32 *)((u8 *)b + 2);
 
 	return (*(u16 *)a ^ *(u16 *)b) | (a32[0] ^ b32[0]) |
 	       (a32[1] ^ b32[1]) | (a32[2] ^ b32[2]);
+#endif
 }
 
 #endif	/* _LINUX_ETHERDEVICE_H */
diff --git a/net/8021q/vlan_core.c b/net/8021q/vlan_core.c
index 07eeb5b..3438c01 100644
--- a/net/8021q/vlan_core.c
+++ b/net/8021q/vlan_core.c
@@ -105,9 +105,12 @@ vlan_gro_common(struct napi_struct *napi, struct vlan_group *grp,
 		goto drop;
 
 	for (p = napi->gro_list; p; p = p->next) {
-		NAPI_GRO_CB(p)->same_flow =
-			p->dev == skb->dev && !compare_ether_header(
-				skb_mac_header(p), skb_gro_mac_header(skb));
+		unsigned long diffs;
+
+		diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
+		diffs |= compare_ether_header(skb_mac_header(p),
+					      skb_gro_mac_header(skb));
+		NAPI_GRO_CB(p)->same_flow = !diffs;
 		NAPI_GRO_CB(p)->flush = 0;
 	}
 
diff --git a/net/core/dev.c b/net/core/dev.c
index 859e30f..63bd20a 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3169,16 +3169,18 @@ normal:
 }
 EXPORT_SYMBOL(dev_gro_receive);
 
-static gro_result_t
+static inline gro_result_t
 __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
 {
 	struct sk_buff *p;
 
 	for (p = napi->gro_list; p; p = p->next) {
-		NAPI_GRO_CB(p)->same_flow =
-			(p->dev == skb->dev) &&
-			!compare_ether_header(skb_mac_header(p),
+		unsigned long diffs;
+
+		diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
+		diffs |= compare_ether_header(skb_mac_header(p),
 					      skb_gro_mac_header(skb));
+		NAPI_GRO_CB(p)->same_flow = !diffs;
 		NAPI_GRO_CB(p)->flush = 0;
 	}
 

Re: [PATCH net-next-2.6 v3] gro: __napi_gro_receive() optimizations

[David Miller]

From: Eric Dumazet <eric.dumazet@gmail.com>
Date: Fri, 27 Aug 2010 06:35:13 +0200

> +	fold |= *(unsigned long *)(a + 6) ^ *(unsigned long *)(b + 6);

Shouldn't these be "+ 8" instead of "+ 6"?

Re: [PATCH net-next-2.6 v3] gro: __napi_gro_receive() optimizations

[Herbert Xu]

On Thu, Aug 26, 2010 at 09:38:16PM -0700, David Miller wrote:
> From: Eric Dumazet <eric.dumazet@gmail.com>
> Date: Fri, 27 Aug 2010 06:35:13 +0200
> 
> > +	fold |= *(unsigned long *)(a + 6) ^ *(unsigned long *)(b + 6);
> 
> Shouldn't these be "+ 8" instead of "+ 6"?

I think +6 is right since we're trying to compare 14 bytes worth
of data.

Thanks,
-- 
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

Re: [PATCH net-next-2.6 v3] gro: __napi_gro_receive() optimizations

[David Miller]

From: Herbert Xu <herbert@gondor.apana.org.au>
Date: Thu, 26 Aug 2010 21:42:01 -0700

> On Thu, Aug 26, 2010 at 09:38:16PM -0700, David Miller wrote:
>> From: Eric Dumazet <eric.dumazet@gmail.com>
>> Date: Fri, 27 Aug 2010 06:35:13 +0200
>> 
>> > +	fold |= *(unsigned long *)(a + 6) ^ *(unsigned long *)(b + 6);
>> 
>> Shouldn't these be "+ 8" instead of "+ 6"?
> 
> I think +6 is right since we're trying to compare 14 bytes worth
> of data.

Indeed, thanks.  I need to borrow some brain cells :)

Re: [PATCH net-next-2.6 v3] gro: __napi_gro_receive() optimizations

[Eric Dumazet]

Le jeudi 26 août 2010 à 21:38 -0700, David Miller a écrit :
> From: Eric Dumazet <eric.dumazet@gmail.com>
> Date: Fri, 27 Aug 2010 06:35:13 +0200
> 
> > +	fold |= *(unsigned long *)(a + 6) ^ *(unsigned long *)(b + 6);
> 
> Shouldn't these be "+ 8" instead of "+ 6"?

Not really, we want to compare 14 bytes, not 16

So we have (sizeof(long) + sizeof(long)) - intersection

I guess I should comment this :)

Re: [PATCH net-next-2.6 v3] gro: __napi_gro_receive() optimizations

[David Miller]

From: Eric Dumazet <eric.dumazet@gmail.com>
Date: Fri, 27 Aug 2010 06:43:44 +0200

> Le jeudi 26 août 2010 à 21:38 -0700, David Miller a écrit :
>> From: Eric Dumazet <eric.dumazet@gmail.com>
>> Date: Fri, 27 Aug 2010 06:35:13 +0200
>> 
>> > +	fold |= *(unsigned long *)(a + 6) ^ *(unsigned long *)(b + 6);
>> 
>> Shouldn't these be "+ 8" instead of "+ 6"?
> 
> Not really, we want to compare 14 bytes, not 16
> 
> So we have (sizeof(long) + sizeof(long)) - intersection

I see.

> I guess I should comment this :)

Yes, that would help feeble minds like mine :-)  Repost with a comment
and I'll apply it, thanks!

[PATCH net-next-2.6 v4] gro: __napi_gro_receive() optimizations

[Eric Dumazet]

compare_ether_header() can have a special implementation on 64 bit
arches if CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS is defined.

__napi_gro_receive() and vlan_gro_common() can avoid a conditional
branch to perform device match.

On x86_64, __napi_gro_receive() has now 38 instructions instead of 53

As gcc-4.4.3 still choose to not inline it, add inline keyword to this
performance critical function.

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
CC: Herbert Xu <herbert@gondor.apana.org.au>
---
v4 : added a nice comment in compare_ether_header()

 include/linux/etherdevice.h |   18 +++++++++++++++++-
 net/8021q/vlan_core.c       |    9 ++++++---
 net/core/dev.c              |   10 ++++++----
 3 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/include/linux/etherdevice.h b/include/linux/etherdevice.h
index 2308fbb..fb6aa60 100644
--- a/include/linux/etherdevice.h
+++ b/include/linux/etherdevice.h
@@ -237,13 +237,29 @@ static inline bool is_etherdev_addr(const struct net_device *dev,
  * entry points.
  */
 
-static inline int compare_ether_header(const void *a, const void *b)
+static inline unsigned long compare_ether_header(const void *a, const void *b)
 {
+#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64
+	unsigned long fold;
+
+	/*
+	 * We want to compare 14 bytes:
+	 *  [a0 ... a13] ^ [b0 ... b13]
+	 * Use two long XOR, ORed together, with an overlap of two bytes.
+	 *  [a0  a1  a2  a3  a4  a5  a6  a7 ] ^ [b0  b1  b2  b3  b4  b5  b6  b7 ] |
+	 *  [a6  a7  a8  a9  a10 a11 a12 a13] ^ [b6  b7  b8  b9  b10 b11 b12 b13]
+	 * This means the [a6 a7] ^ [b6 b7] part is done two times.
+	*/
+	fold = *(unsigned long *)a ^ *(unsigned long *)b;
+	fold |= *(unsigned long *)(a + 6) ^ *(unsigned long *)(b + 6);
+	return fold;
+#else
 	u32 *a32 = (u32 *)((u8 *)a + 2);
 	u32 *b32 = (u32 *)((u8 *)b + 2);
 
 	return (*(u16 *)a ^ *(u16 *)b) | (a32[0] ^ b32[0]) |
 	       (a32[1] ^ b32[1]) | (a32[2] ^ b32[2]);
+#endif
 }
 
 #endif	/* _LINUX_ETHERDEVICE_H */
diff --git a/net/8021q/vlan_core.c b/net/8021q/vlan_core.c
index 07eeb5b..3438c01 100644
--- a/net/8021q/vlan_core.c
+++ b/net/8021q/vlan_core.c
@@ -105,9 +105,12 @@ vlan_gro_common(struct napi_struct *napi, struct vlan_group *grp,
 		goto drop;
 
 	for (p = napi->gro_list; p; p = p->next) {
-		NAPI_GRO_CB(p)->same_flow =
-			p->dev == skb->dev && !compare_ether_header(
-				skb_mac_header(p), skb_gro_mac_header(skb));
+		unsigned long diffs;
+
+		diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
+		diffs |= compare_ether_header(skb_mac_header(p),
+					      skb_gro_mac_header(skb));
+		NAPI_GRO_CB(p)->same_flow = !diffs;
 		NAPI_GRO_CB(p)->flush = 0;
 	}
 
diff --git a/net/core/dev.c b/net/core/dev.c
index 859e30f..63bd20a 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3169,16 +3169,18 @@ normal:
 }
 EXPORT_SYMBOL(dev_gro_receive);
 
-static gro_result_t
+static inline gro_result_t
 __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
 {
 	struct sk_buff *p;
 
 	for (p = napi->gro_list; p; p = p->next) {
-		NAPI_GRO_CB(p)->same_flow =
-			(p->dev == skb->dev) &&
-			!compare_ether_header(skb_mac_header(p),
+		unsigned long diffs;
+
+		diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
+		diffs |= compare_ether_header(skb_mac_header(p),
 					      skb_gro_mac_header(skb));
+		NAPI_GRO_CB(p)->same_flow = !diffs;
 		NAPI_GRO_CB(p)->flush = 0;
 	}
 

Re: [PATCH net-next-2.6 v4] gro: __napi_gro_receive() optimizations

[David Miller]

From: Eric Dumazet <eric.dumazet@gmail.com>
Date: Fri, 27 Aug 2010 07:01:34 +0200

> compare_ether_header() can have a special implementation on 64 bit
> arches if CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS is defined.
> 
> __napi_gro_receive() and vlan_gro_common() can avoid a conditional
> branch to perform device match.
> 
> On x86_64, __napi_gro_receive() has now 38 instructions instead of 53
> 
> As gcc-4.4.3 still choose to not inline it, add inline keyword to this
> performance critical function.
> 
> Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
> CC: Herbert Xu <herbert@gondor.apana.org.au>

Applied.