Re: TCP transmit performance regression

Eric Dumazet
2012-07-06T05:16:10Z

On Fri, 2012-07-06 at 06:58 +0200, Eric Dumazet wrote:
> On Fri, 2012-07-06 at 08:45 +0800, Ming Lei wrote:
>
> > Unfortunately, the patch still hasn't any improvement on the transmit
> > performance of beagle-xm.
>
> Ah yes, I need to change usbnet as well to be able to fully recycle the
> big skbs allocated in turbo mode.
>
> Right now they are constantly allocated/freed and this sucks if SLAB
> wants to check poison bytes in debug mode.

In the mean time, you also can use the following patch I have to polish,
but this should give you a nice boost, since the big skb skb->head wont
be checked by SLAB debug :

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 5b21522..d31efa2 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -296,9 +296,18 @@ EXPORT_SYMBOL(build_skb);
struct netdev_alloc_cache {
struct page *page;
unsigned int offset;
+ unsigned int pagecnt_bias;
};
static DEFINE_PER_CPU(struct netdev_alloc_cache, netdev_alloc_cache);

+#if PAGE_SIZE > 32768
+#define MAX_NETDEV_FRAGSIZE PAGE_SIZE
+#else
+#define MAX_NETDEV_FRAGSIZE 32768
+#endif
+
+#define NETDEV_PAGECNT_BIAS (MAX_NETDEV_FRAGSIZE / \
+ SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
/**
* netdev_alloc_frag - allocate a page fragment
* @fragsz: fragment size
@@ -316,18 +325,25 @@ void *netdev_alloc_frag(unsigned int fragsz)
nc = &__get_cpu_var(netdev_alloc_cache);
if (unlikely(!nc->page)) {
refill:
- nc->page = alloc_page(GFP_ATOMIC | __GFP_COLD);
+ nc->page = alloc_pages(GFP_ATOMIC | __GFP_COLD | __GFP_COMP,
+ get_order(MAX_NETDEV_FRAGSIZE));
+ if (unlikely(!nc->page))
+ goto end;
+recycle:
+ atomic_set(&nc->page->_count, NETDEV_PAGECNT_BIAS);
+ nc->pagecnt_bias = NETDEV_PAGECNT_BIAS;
nc->offset = 0;
}
- if (likely(nc->page)) {
- if (nc->offset + fragsz > PAGE_SIZE) {
- put_page(nc->page);
- goto refill;
- }
- data = page_address(nc->page) + nc->offset;
- nc->offset += fragsz;
- get_page(nc->page);
+ if (nc->offset + fragsz > MAX_NETDEV_FRAGSIZE) {
+ if (!atomic_sub_return(nc->pagecnt_bias,
+ &nc->page->_count))
+ goto recycle;
+ goto refill;
}
+ data = page_address(nc->page) + nc->offset;
+ nc->offset += fragsz;
+ nc->pagecnt_bias--; /* avoid get_page()/get_page() false sharing */
+end:
local_irq_restore(flags);
return data;
}
@@ -353,7 +369,7 @@ struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
unsigned int fragsz = SKB_DATA_ALIGN(length + NET_SKB_PAD) +
SKB_DATA_ALIGN(sizeof(struct skb_shared_info));

- if (fragsz <= PAGE_SIZE && !(gfp_mask & __GFP_WAIT)) {
+ if (fragsz <= MAX_NETDEV_FRAGSIZE && !(gfp_mask & __GFP_WAIT)) {
void *data = netdev_alloc_frag(fragsz);

if (likely(data)) {