[PATCH] [NET 2.6.26]: Add per-connection option to set max TSO frame size

[PATCH] [NET 2.6.26]: Add per-connection option to set max TSO frame size

[PJ Waskiewicz]

Update: Respun for net-2.6.26 tree.

Update: changed max_gso_frame_size and sk_gso_max_size from signed to
unsigned - thanks Stephen!

This patch adds the ability for device drivers to control the size of the
TSO frames being sent to them, per TCP connection.  By setting the
netdevice's max_gso_frame_size value, the socket layer will set the GSO
frame size based on that value.  This will propogate into the TCP layer,
and send TSO's of that size to the hardware.

This can be desirable to help tune the bursty nature of TSO on a
per-adapter basis, where one may have 1 GbE and 10 GbE devices coexisting
in a system, one running multiqueue and the other not, etc.

This can also be desirable for devices that cannot support full 64 KB
TSO's, but still want to benefit from some level of segmentation
offloading.

Signed-off-by: Peter P Waskiewicz Jr <peter.p.waskiewicz.jr@intel.com>
---

 include/linux/netdevice.h |    6 ++++++
 include/net/sock.h        |    2 ++
 net/core/dev.c            |    1 +
 net/core/sock.c           |    6 ++++--
 net/ipv4/tcp_output.c     |    4 ++--
 5 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index a2f0032..3bf825b 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -620,6 +620,7 @@ struct net_device
 
 	/* Partially transmitted GSO packet. */
 	struct sk_buff		*gso_skb;
+	u16			max_gso_frame_size;
 
 /*
  * One part is mostly used on xmit path (device)
@@ -1475,6 +1476,11 @@ static inline int netif_needs_gso(struct net_device *dev, struct sk_buff *skb)
 		unlikely(skb->ip_summed != CHECKSUM_PARTIAL));
 }
 
+static inline void netif_set_max_gso_size(struct net_device *dev, u16 size)
+{
+       dev->max_gso_frame_size = size;
+}
+
 /* On bonding slaves other than the currently active slave, suppress
  * duplicates except for 802.3ad ETH_P_SLOW, alb non-mcast/bcast, and
  * ARP on active-backup slaves with arp_validate enabled.
diff --git a/include/net/sock.h b/include/net/sock.h
index 8a7889b..2b07af0 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -151,6 +151,7 @@ struct sock_common {
   *	@sk_no_check: %SO_NO_CHECK setting, wether or not checkup packets
   *	@sk_route_caps: route capabilities (e.g. %NETIF_F_TSO)
   *	@sk_gso_type: GSO type (e.g. %SKB_GSO_TCPV4)
+  *	@sk_gso_max_size: Maximum GSO segment size to build
   *	@sk_lingertime: %SO_LINGER l_linger setting
   *	@sk_backlog: always used with the per-socket spinlock held
   *	@sk_callback_lock: used with the callbacks in the end of this struct
@@ -236,6 +237,7 @@ struct sock {
 	gfp_t			sk_allocation;
 	int			sk_route_caps;
 	int			sk_gso_type;
+	__u16			sk_gso_max_size;
 	int			sk_rcvlowat;
 	unsigned long 		sk_flags;
 	unsigned long	        sk_lingertime;
diff --git a/net/core/dev.c b/net/core/dev.c
index 908f07c..689a678 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4021,6 +4021,7 @@ struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
 	}
 
 	dev->egress_subqueue_count = queue_count;
+	dev->max_gso_frame_size = 65536;
 
 	dev->get_stats = internal_stats;
 	netpoll_netdev_init(dev);
diff --git a/net/core/sock.c b/net/core/sock.c
index 09cb3a7..49dacd5 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1076,10 +1076,12 @@ void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
 	if (sk->sk_route_caps & NETIF_F_GSO)
 		sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
 	if (sk_can_gso(sk)) {
-		if (dst->header_len)
+		if (dst->header_len) {
 			sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
-		else
+		} else {
 			sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
+			sk->sk_gso_max_size = dst->dev->max_gso_frame_size;
+		}
 	}
 }
 EXPORT_SYMBOL_GPL(sk_setup_caps);
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index ed750f9..8cd128d 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -998,7 +998,7 @@ unsigned int tcp_current_mss(struct sock *sk, int large_allowed)
 	xmit_size_goal = mss_now;
 
 	if (doing_tso) {
-		xmit_size_goal = (65535 -
+		xmit_size_goal = ((sk->sk_gso_max_size - 1) -
 				  inet_csk(sk)->icsk_af_ops->net_header_len -
 				  inet_csk(sk)->icsk_ext_hdr_len -
 				  tp->tcp_header_len);
@@ -1274,7 +1274,7 @@ static int tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb)
 	limit = min(send_win, cong_win);
 
 	/* If a full-sized TSO skb can be sent, do it. */
-	if (limit >= 65536)
+	if (limit >= sk->sk_gso_max_size)
 		goto send_now;
 
 	if (sysctl_tcp_tso_win_divisor) {

Re: [PATCH] [NET 2.6.26]: Add per-connection option to set max TSO frame size

[Jarek Poplawski]

On 27-02-2008 18:26, PJ Waskiewicz wrote:
...
> diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
> index a2f0032..3bf825b 100644
> --- a/include/linux/netdevice.h
> +++ b/include/linux/netdevice.h
> @@ -620,6 +620,7 @@ struct net_device
>  
>  	/* Partially transmitted GSO packet. */
>  	struct sk_buff		*gso_skb;
> +	u16			max_gso_frame_size;
...
> +static inline void netif_set_max_gso_size(struct net_device *dev, u16 size)
> +	__u16			sk_gso_max_size;
>  	int			sk_rcvlowat;
>  	unsigned long 		sk_flags;
>  	unsigned long	        sk_lingertime;
> diff --git a/net/core/dev.c b/net/core/dev.c
> index 908f07c..689a678 100644
> --- a/net/core/dev.c
> +++ b/net/core/dev.c
> @@ -4021,6 +4021,7 @@ struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
>  	}
>  
>  	dev->egress_subqueue_count = queue_count;
> +	dev->max_gso_frame_size = 65536;

Does it fit into u16?

Isn't it "nicer" with some constant (#define MAX_GSO_FRAME_SIZE ...)?

Why not more consistent names, e.g.:
dev->gso_max_size;
sk_gso_max_size;
netif_set_gso_max_size()
(GSO_MAX_SIZE)
or
dev->max_gso_frame_size;
sk_max_gso_frame_size;
netif_set_max_gso_frame_size()
(MAX_GSO_FRAME_SIZE)
etc.?

Regards,
Jarek P.

Re: [PATCH] [NET 2.6.26]: Add per-connection option to set max TSO frame size

[Jarek Poplawski]

On Thu, Feb 28, 2008 at 08:07:46AM +0000, Jarek Poplawski wrote:
> On 27-02-2008 18:26, PJ Waskiewicz wrote:
> ...
> > diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
> > index a2f0032..3bf825b 100644
> > --- a/include/linux/netdevice.h
> > +++ b/include/linux/netdevice.h
> > @@ -620,6 +620,7 @@ struct net_device
> >  
> >  	/* Partially transmitted GSO packet. */
> >  	struct sk_buff		*gso_skb;
> > +	u16			max_gso_frame_size;

...And since it looks like not very often used - maybe it could be
placed a bit lower?

Jarek P.

RE: [PATCH] [NET 2.6.26]: Add per-connection option to set max TSOframe size

[Waskiewicz Jr, Peter P]

> ...And since it looks like not very often used - maybe it 
> could be placed a bit lower?
> 
> Jarek P.

Yes, I can move it.  I just grouped it with the other GSO/TSO elements
in the struct.  I'll take a closer look and see if I can place it
elsewhere for better efficiency.  I'll send change along with the other
changes from your other feedback.

Thanks,

-PJ Waskiewicz

RE: [PATCH] [NET 2.6.26]: Add per-connection option to set max TSOframe size

[Waskiewicz Jr, Peter P]

> >  	dev->egress_subqueue_count = queue_count;
> > +	dev->max_gso_frame_size = 65536;
> 
> Does it fit into u16?

No it doesn't.  I originally had this as an int, and borked it when
switching it to unsigned.  I'll fix this asap.

> Isn't it "nicer" with some constant (#define MAX_GSO_FRAME_SIZE ...)?

Yes it is.  I'll add this.

> Why not more consistent names, e.g.:
> dev->gso_max_size;
> sk_gso_max_size;
> netif_set_gso_max_size()
> (GSO_MAX_SIZE)
> or
> dev->max_gso_frame_size;
> sk_max_gso_frame_size;
> netif_set_max_gso_frame_size()
> (MAX_GSO_FRAME_SIZE)
> etc.?

Very good point.  I'll change this and resend today.  Thanks a bunch for
the feedback!

Cheers,

-PJ Waskiewicz

[PATCH] [NET 2.6.26]: Add per-connection option to set max TSO frame size

[PJ Waskiewicz]

Update: My mailer ate one of Jarek's feedback mails...  Fixed the parameter
in netif_set_gso_max_size() to be u32, not u16.  Fixed the whitespace issue
due to a patch import botch.  Changed the types from u32 to unsigned int to
be more consistent with other variables in the area.  Also brought the patch
up to the latest net-2.6.26 tree.

Update: Made gso_max_size container 32 bits, not 16.  Moved the
location of gso_max_size within netdev to be less hotpath.  Made
more consistent names between the sock and netdev layers, and added a
define for the max GSO size.

Update: Respun for net-2.6.26 tree.

Update: changed max_gso_frame_size and sk_gso_max_size from signed to
unsigned - thanks Stephen!

This patch adds the ability for device drivers to control the size of the
TSO frames being sent to them, per TCP connection.  By setting the
netdevice's gso_max_size value, the socket layer will set the GSO
frame size based on that value.  This will propogate into the TCP layer,
and send TSO's of that size to the hardware.

This can be desirable to help tune the bursty nature of TSO on a
per-adapter basis, where one may have 1 GbE and 10 GbE devices coexisting
in a system, one running multiqueue and the other not, etc.

This can also be desirable for devices that cannot support full 64 KB
TSO's, but still want to benefit from some level of segmentation
offloading.

Signed-off-by: Peter P Waskiewicz Jr <peter.p.waskiewicz.jr@intel.com>
---

 include/linux/netdevice.h |   10 ++++++++++
 include/net/sock.h        |    2 ++
 net/core/dev.c            |    1 +
 net/core/sock.c           |    6 ++++--
 net/ipv4/tcp_output.c     |    4 ++--
 5 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index a2f0032..ced61f8 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -724,6 +724,10 @@ struct net_device
 	/* rtnetlink link ops */
 	const struct rtnl_link_ops *rtnl_link_ops;
 
+	/* for setting kernel sock attribute on TCP connection setup */
+#define GSO_MAX_SIZE		65536
+	unsigned int		gso_max_size;
+
 	/* The TX queue control structures */
 	unsigned int			egress_subqueue_count;
 	struct net_device_subqueue	egress_subqueue[1];
@@ -1475,6 +1479,12 @@ static inline int netif_needs_gso(struct net_device *dev, struct sk_buff *skb)
 		unlikely(skb->ip_summed != CHECKSUM_PARTIAL));
 }
 
+static inline void netif_set_gso_max_size(struct net_device *dev,
+					  unsigned int size)
+{
+	dev->gso_max_size = size;
+}
+
 /* On bonding slaves other than the currently active slave, suppress
  * duplicates except for 802.3ad ETH_P_SLOW, alb non-mcast/bcast, and
  * ARP on active-backup slaves with arp_validate enabled.
diff --git a/include/net/sock.h b/include/net/sock.h
index 39112e7..8358fff 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -151,6 +151,7 @@ struct sock_common {
   *	@sk_no_check: %SO_NO_CHECK setting, wether or not checkup packets
   *	@sk_route_caps: route capabilities (e.g. %NETIF_F_TSO)
   *	@sk_gso_type: GSO type (e.g. %SKB_GSO_TCPV4)
+  *	@sk_gso_max_size: Maximum GSO segment size to build
   *	@sk_lingertime: %SO_LINGER l_linger setting
   *	@sk_backlog: always used with the per-socket spinlock held
   *	@sk_callback_lock: used with the callbacks in the end of this struct
@@ -237,6 +238,7 @@ struct sock {
 	gfp_t			sk_allocation;
 	int			sk_route_caps;
 	int			sk_gso_type;
+	unsigned int		sk_gso_max_size;
 	int			sk_rcvlowat;
 	unsigned long 		sk_flags;
 	unsigned long	        sk_lingertime;
diff --git a/net/core/dev.c b/net/core/dev.c
index fcdf03c..f973e38 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4021,6 +4021,7 @@ struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
 	}
 
 	dev->egress_subqueue_count = queue_count;
+	dev->gso_max_size = GSO_MAX_SIZE;
 
 	dev->get_stats = internal_stats;
 	netpoll_netdev_init(dev);
diff --git a/net/core/sock.c b/net/core/sock.c
index 0ca0697..3629c77 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1095,10 +1095,12 @@ void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
 	if (sk->sk_route_caps & NETIF_F_GSO)
 		sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
 	if (sk_can_gso(sk)) {
-		if (dst->header_len)
+		if (dst->header_len) {
 			sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
-		else
+		} else {
 			sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
+			sk->sk_gso_max_size = dst->dev->gso_max_size;
+		}
 	}
 }
 EXPORT_SYMBOL_GPL(sk_setup_caps);
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index cbfef8b..2c9cdb1 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -998,7 +998,7 @@ unsigned int tcp_current_mss(struct sock *sk, int large_allowed)
 	xmit_size_goal = mss_now;
 
 	if (doing_tso) {
-		xmit_size_goal = (65535 -
+		xmit_size_goal = ((sk->sk_gso_max_size - 1) -
 				  inet_csk(sk)->icsk_af_ops->net_header_len -
 				  inet_csk(sk)->icsk_ext_hdr_len -
 				  tp->tcp_header_len);
@@ -1274,7 +1274,7 @@ static int tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb)
 	limit = min(send_win, cong_win);
 
 	/* If a full-sized TSO skb can be sent, do it. */
-	if (limit >= 65536)
+	if (limit >= sk->sk_gso_max_size)
 		goto send_now;
 
 	if (sysctl_tcp_tso_win_divisor) {

Re: [PATCH] [NET 2.6.26]: Add per-connection option to set max TSO frame size

[David Miller]

From: PJ Waskiewicz <peter.p.waskiewicz.jr@intel.com>
Date: Wed, 05 Mar 2008 09:35:10 -0800

> This patch adds the ability for device drivers to control the size of the
> TSO frames being sent to them, per TCP connection.  By setting the
> netdevice's gso_max_size value, the socket layer will set the GSO
> frame size based on that value.  This will propogate into the TCP layer,
> and send TSO's of that size to the hardware.

Applied to net-2.6.26, thanks.

Sorry I didn't get to this sooner as I've been away in Japan most of
this month and have been only applying the most important fixes and
the most trivial feature patches.

[PATCH] [NET 2.6.26]: Add per-connection option to set max TSO frame size

[PJ Waskiewicz]

Update: Made gso_max_size container 32 bits, not 16.  Moved the
location of gso_max_size within netdev to be less hotpath.  Made
more consistent names between the sock and netdev layers, and added a
define for the max GSO size.
 
Update: Respun for net-2.6.26 tree.

Update: changed max_gso_frame_size and sk_gso_max_size from signed to
unsigned - thanks Stephen!

This patch adds the ability for device drivers to control the size of the
TSO frames being sent to them, per TCP connection.  By setting the
netdevice's gso_max_size value, the socket layer will set the GSO
frame size based on that value.  This will propogate into the TCP layer,
and send TSO's of that size to the hardware.

This can be desirable to help tune the bursty nature of TSO on a
per-adapter basis, where one may have 1 GbE and 10 GbE devices coexisting
in a system, one running multiqueue and the other not, etc.

This can also be desirable for devices that cannot support full 64 KB
TSO's, but still want to benefit from some level of segmentation
offloading.

Signed-off-by: Peter P Waskiewicz Jr <peter.p.waskiewicz.jr@intel.com>
---

 include/linux/netdevice.h |    9 +++++++++
 include/net/sock.h        |    2 ++
 net/core/dev.c            |    1 +
 net/core/sock.c           |    6 ++++--
 net/ipv4/tcp_output.c     |    4 ++--
 5 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index a2f0032..b460fd3 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -724,6 +724,10 @@ struct net_device
 	/* rtnetlink link ops */
 	const struct rtnl_link_ops *rtnl_link_ops;
 
+	/* for setting kernel sock attribute on TCP connection setup */
+#define GSO_MAX_SIZE		65536
+	u32			gso_max_size;
+
 	/* The TX queue control structures */
 	unsigned int			egress_subqueue_count;
 	struct net_device_subqueue	egress_subqueue[1];
@@ -1475,6 +1479,11 @@ static inline int netif_needs_gso(struct net_device *dev, struct sk_buff *skb)
 		unlikely(skb->ip_summed != CHECKSUM_PARTIAL));
 }
 
+static inline void netif_set_gso_max_size(struct net_device *dev, u16 size)
+{
+       dev->gso_max_size = size;
+}
+
 /* On bonding slaves other than the currently active slave, suppress
  * duplicates except for 802.3ad ETH_P_SLOW, alb non-mcast/bcast, and
  * ARP on active-backup slaves with arp_validate enabled.
diff --git a/include/net/sock.h b/include/net/sock.h
index 8a7889b..1c52822 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -151,6 +151,7 @@ struct sock_common {
   *	@sk_no_check: %SO_NO_CHECK setting, wether or not checkup packets
   *	@sk_route_caps: route capabilities (e.g. %NETIF_F_TSO)
   *	@sk_gso_type: GSO type (e.g. %SKB_GSO_TCPV4)
+  *	@sk_gso_max_size: Maximum GSO segment size to build
   *	@sk_lingertime: %SO_LINGER l_linger setting
   *	@sk_backlog: always used with the per-socket spinlock held
   *	@sk_callback_lock: used with the callbacks in the end of this struct
@@ -236,6 +237,7 @@ struct sock {
 	gfp_t			sk_allocation;
 	int			sk_route_caps;
 	int			sk_gso_type;
+	__u32			sk_gso_max_size;
 	int			sk_rcvlowat;
 	unsigned long 		sk_flags;
 	unsigned long	        sk_lingertime;
diff --git a/net/core/dev.c b/net/core/dev.c
index 908f07c..53821db 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4021,6 +4021,7 @@ struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
 	}
 
 	dev->egress_subqueue_count = queue_count;
+	dev->gso_max_size = GSO_MAX_SIZE;
 
 	dev->get_stats = internal_stats;
 	netpoll_netdev_init(dev);
diff --git a/net/core/sock.c b/net/core/sock.c
index 09cb3a7..147e098 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1076,10 +1076,12 @@ void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
 	if (sk->sk_route_caps & NETIF_F_GSO)
 		sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
 	if (sk_can_gso(sk)) {
-		if (dst->header_len)
+		if (dst->header_len) {
 			sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
-		else
+		} else {
 			sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
+			sk->sk_gso_max_size = dst->dev->gso_max_size;
+		}
 	}
 }
 EXPORT_SYMBOL_GPL(sk_setup_caps);
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index ed750f9..8cd128d 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -998,7 +998,7 @@ unsigned int tcp_current_mss(struct sock *sk, int large_allowed)
 	xmit_size_goal = mss_now;
 
 	if (doing_tso) {
-		xmit_size_goal = (65535 -
+		xmit_size_goal = ((sk->sk_gso_max_size - 1) -
 				  inet_csk(sk)->icsk_af_ops->net_header_len -
 				  inet_csk(sk)->icsk_ext_hdr_len -
 				  tp->tcp_header_len);
@@ -1274,7 +1274,7 @@ static int tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb)
 	limit = min(send_win, cong_win);
 
 	/* If a full-sized TSO skb can be sent, do it. */
-	if (limit >= 65536)
+	if (limit >= sk->sk_gso_max_size)
 		goto send_now;
 
 	if (sysctl_tcp_tso_win_divisor) {

Re: [PATCH] [NET 2.6.26]: Add per-connection option to set max TSO frame size

[Jarek Poplawski]

On Thu, Feb 28, 2008 at 10:31:50AM -0800, PJ Waskiewicz wrote:
...

Looks ...almost OK to me, but:

> diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
> index a2f0032..b460fd3 100644
> --- a/include/linux/netdevice.h
> +++ b/include/linux/netdevice.h
> @@ -724,6 +724,10 @@ struct net_device
>  	/* rtnetlink link ops */
>  	const struct rtnl_link_ops *rtnl_link_ops;
>  
> +	/* for setting kernel sock attribute on TCP connection setup */
> +#define GSO_MAX_SIZE		65536
> +	u32			gso_max_size;
> +

Now probably unsigned int should "suffice" too.

>  	/* The TX queue control structures */
>  	unsigned int			egress_subqueue_count;
>  	struct net_device_subqueue	egress_subqueue[1];
> @@ -1475,6 +1479,11 @@ static inline int netif_needs_gso(struct net_device *dev, struct sk_buff *skb)
>  		unlikely(skb->ip_summed != CHECKSUM_PARTIAL));
>  }
>  
> +static inline void netif_set_gso_max_size(struct net_device *dev, u16 size)

u16?

> +{
> +       dev->gso_max_size = size;

^^^^^^^^^^ a tab needed here.
...
> diff --git a/include/net/sock.h b/include/net/sock.h
> index 8a7889b..1c52822 100644
> --- a/include/net/sock.h
> +++ b/include/net/sock.h
...
> @@ -236,6 +237,7 @@ struct sock {
>  	gfp_t			sk_allocation;
>  	int			sk_route_caps;
>  	int			sk_gso_type;
> +	__u32			sk_gso_max_size;

unsigned int? But, actually I wonder if sk_gso_type and sk_gso_max_size
can't be two shorts yet?

Regards,
Jarek P.

Re: [PATCH] [NET 2.6.26]: Add per-connection option to set max TSO frame size

[Jarek Poplawski]

PJ Waskiewicz wrote, On 02/28/2008 07:31 PM:
...
> This can be desirable to help tune the bursty nature of TSO on a
> per-adapter basis, where one may have 1 GbE and 10 GbE devices coexisting
> in a system, one running multiqueue and the other not, etc.
> 
> This can also be desirable for devices that cannot support full 64 KB
> TSO's, but still want to benefit from some level of segmentation
> offloading.
...
> +++ b/include/linux/netdevice.h
...
> +	/* for setting kernel sock attribute on TCP connection setup */
> +#define GSO_MAX_SIZE		65536
> +	u32			gso_max_size;

One more, maybe foolish, thrifty idea: do we really need such a
precision here? I don't know those devices, but maybe it would be
enough (for the beginning at least?) to use a few divisors e.g.:

gso_max_size = 65536 >> GSO_MAX_SIZE_SHIFT

Then it seems 3 or 4 bits should be enough for this, and BTW, it looks
like dev->features has 16 bits for various GSO flags, with "a lot" of
free space yet - unless I missed something?

Jarek P.

Re: [PATCH] [NET 2.6.26]: Add per-connection option to set max TSO frame size

[Jarek Poplawski]

On Fri, Feb 29, 2008 at 09:46:50PM +0100, Jarek Poplawski wrote:
...
> gso_max_size = 65536 >> GSO_MAX_SIZE_SHIFT
> 
> Then it seems 3 or 4 bits should be enough for this, and BTW, it looks

...Hm... 2 or 3 bits! 

Jarek P.

[PATCH] [NET 2.6.26]: Add per-connection option to set max TSO frame size

[PJ Waskiewicz]

Update: changed max_gso_frame_size and sk_gso_max_size from signed to
unsigned - thanks Stephen!

This patch adds the ability for device drivers to control the size of the
TSO frames being sent to them, per TCP connection.  By setting the
netdevice's max_gso_frame_size value, the socket layer will set the GSO
frame size based on that value.  This will propogate into the TCP layer,
and send TSO's of that size to the hardware.

This can be desirable to help tune the bursty nature of TSO on a
per-adapter basis, where one may have 1 GbE and 10 GbE devices coexisting
in a system, one running multiqueue and the other not, etc.

This can also be desirable for devices that cannot support full 64 KB
TSO's, but still want to benefit from some level of segmentation
offloading.

Signed-off-by: Peter P Waskiewicz Jr <peter.p.waskiewicz.jr@intel.com>
---

 include/linux/netdevice.h |    6 ++++++
 include/net/sock.h        |    2 ++
 net/core/dev.c            |    1 +
 net/core/sock.c           |    6 ++++--
 net/ipv4/tcp_output.c     |    4 ++--
 5 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 047d432..853caca 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -616,6 +616,7 @@ struct net_device
 
 	/* Partially transmitted GSO packet. */
 	struct sk_buff		*gso_skb;
+	u16			max_gso_frame_size;
 
 	/* ingress path synchronizer */
 	spinlock_t		ingress_lock;
@@ -1475,6 +1476,11 @@ static inline int netif_needs_gso(struct net_device *dev, struct sk_buff *skb)
 		unlikely(skb->ip_summed != CHECKSUM_PARTIAL));
 }
 
+static inline void netif_set_max_gso_size(struct net_device *dev, u16 size)
+{
+	dev->max_gso_frame_size = size;
+}
+
 /* On bonding slaves other than the currently active slave, suppress
  * duplicates except for 802.3ad ETH_P_SLOW, alb non-mcast/bcast, and
  * ARP on active-backup slaves with arp_validate enabled.
diff --git a/include/net/sock.h b/include/net/sock.h
index 8a7889b..2b07af0 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -151,6 +151,7 @@ struct sock_common {
   *	@sk_no_check: %SO_NO_CHECK setting, wether or not checkup packets
   *	@sk_route_caps: route capabilities (e.g. %NETIF_F_TSO)
   *	@sk_gso_type: GSO type (e.g. %SKB_GSO_TCPV4)
+  *	@sk_gso_max_size: Maximum GSO segment size to build
   *	@sk_lingertime: %SO_LINGER l_linger setting
   *	@sk_backlog: always used with the per-socket spinlock held
   *	@sk_callback_lock: used with the callbacks in the end of this struct
@@ -236,6 +237,7 @@ struct sock {
 	gfp_t			sk_allocation;
 	int			sk_route_caps;
 	int			sk_gso_type;
+	__u16			sk_gso_max_size;
 	int			sk_rcvlowat;
 	unsigned long 		sk_flags;
 	unsigned long	        sk_lingertime;
diff --git a/net/core/dev.c b/net/core/dev.c
index 9549417..f635b29 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4022,6 +4022,7 @@ struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
 	}
 
 	dev->egress_subqueue_count = queue_count;
+	dev->max_gso_frame_size = 65536;
 
 	dev->get_stats = internal_stats;
 	netpoll_netdev_init(dev);
diff --git a/net/core/sock.c b/net/core/sock.c
index 433715f..a8b0ae5 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1076,10 +1076,12 @@ void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
 	if (sk->sk_route_caps & NETIF_F_GSO)
 		sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
 	if (sk_can_gso(sk)) {
-		if (dst->header_len)
+		if (dst->header_len) {
 			sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
-		else
+		} else {
 			sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
+			sk->sk_gso_max_size = dst->dev->max_gso_frame_size;
+		}
 	}
 }
 EXPORT_SYMBOL_GPL(sk_setup_caps);
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index ed750f9..8cd128d 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -998,7 +998,7 @@ unsigned int tcp_current_mss(struct sock *sk, int large_allowed)
 	xmit_size_goal = mss_now;
 
 	if (doing_tso) {
-		xmit_size_goal = (65535 -
+		xmit_size_goal = ((sk->sk_gso_max_size - 1) -
 				  inet_csk(sk)->icsk_af_ops->net_header_len -
 				  inet_csk(sk)->icsk_ext_hdr_len -
 				  tp->tcp_header_len);
@@ -1274,7 +1274,7 @@ static int tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb)
 	limit = min(send_win, cong_win);
 
 	/* If a full-sized TSO skb can be sent, do it. */
-	if (limit >= 65536)
+	if (limit >= sk->sk_gso_max_size)
 		goto send_now;
 
 	if (sysctl_tcp_tso_win_divisor) {