diff for duplicates of <20100713102003.2835.88018.sendpatchset@danny.redhat> diff --git a/a/1.txt b/N1/1.txt index 3b309a7..e4844c7 100644 --- a/a/1.txt +++ b/N1/1.txt @@ -464,9 +464,3 @@ index 34432b4..9e36dc2 100644 if (pos < offset) { -- 1.7.1.1 - --- -To unsubscribe, send a message with 'unsubscribe linux-mm' in -the body to majordomo@kvack.org. For more info on Linux MM, -see: http://www.linux-mm.org/ . -Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a> diff --git a/a/content_digest b/N1/content_digest index 63b0b09..8034201 100644 --- a/a/content_digest +++ b/N1/content_digest @@ -481,12 +481,6 @@ " \n" " \t\t\tif (pos < offset) {\n" "-- \n" - "1.7.1.1\n" - "\n" - "--\n" - "To unsubscribe, send a message with 'unsubscribe linux-mm' in\n" - "the body to majordomo@kvack.org. For more info on Linux MM,\n" - "see: http://www.linux-mm.org/ .\n" - "Don't email: <a href=mailto:\"dont@kvack.org\"> email@kvack.org </a>" + 1.7.1.1 -d42aa7c7229253b0869ec391cb59d0b2650da413c6256afcec8d0048c18af4b9 +f6a9f1c953831bbd94a12a76437ff1244ff09e515c6fd243865050f844f65c64
diff --git a/a/1.txt b/N2/1.txt index 3b309a7..8b13789 100644 --- a/a/1.txt +++ b/N2/1.txt @@ -1,472 +1 @@ ->From 3e824860934af5aa0150608314693c5e0e3608b6 Mon Sep 17 00:00:00 2001 -From: Xiaotian Feng <dfeng@redhat.com> -Date: Tue, 13 Jul 2010 11:30:27 +0800 -Subject: [PATCH 17/30] netvm: hook skb allocation to reserves -Change the skb allocation api to indicate RX usage and use this to fall back to -the reserve when needed. SKBs allocated from the reserve are tagged in -skb->emergency. - -Teach all other skb ops about emergency skbs and the reserve accounting. - -Use the (new) packet split API to allocate and track fragment pages from the -emergency reserve. Do this using an atomic counter in page->index. This is -needed because the fragments have a different sharing semantic than that -indicated by skb_shinfo()->dataref. - -Note that the decision to distinguish between regular and emergency SKBs allows -the accounting overhead to be limited to the later kind. - -Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> -Signed-off-by: Suresh Jayaraman <sjayaraman@suse.de> -Signed-off-by: Xiaotian Feng <dfeng@redhat.com> ---- - include/linux/mm_types.h | 1 + - include/linux/skbuff.h | 27 +++++++-- - net/core/skbuff.c | 137 +++++++++++++++++++++++++++++++++++++--------- - 3 files changed, 133 insertions(+), 32 deletions(-) - -diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h -index a95a202..73e0526 100644 ---- a/include/linux/mm_types.h -+++ b/include/linux/mm_types.h -@@ -72,6 +72,7 @@ struct page { - pgoff_t index; /* Our offset within mapping. */ - void *freelist; /* SLUB: freelist req. slab lock */ - int reserve; /* page_alloc: page is a reserve page */ -+ atomic_t frag_count; /* skb fragment use count */ - }; - struct list_head lru; /* Pageout list, eg. active_list - * protected by zone->lru_lock ! -diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h -index 988a4dc..4ac45ad 100644 ---- a/include/linux/skbuff.h -+++ b/include/linux/skbuff.h -@@ -385,9 +385,12 @@ struct sk_buff { - #else - __u8 deliver_no_wcard:1; - #endif -+#ifdef CONFIG_NETVM -+ __u8 emergency:1; -+#endif - kmemcheck_bitfield_end(flags2); - -- /* 0/14 bit hole */ -+ /* 0/13/14 bit hole */ - - #ifdef CONFIG_NET_DMA - dma_cookie_t dma_cookie; -@@ -429,6 +432,18 @@ struct sk_buff { - #define SKB_DST_NOREF 1UL - #define SKB_DST_PTRMASK ~(SKB_DST_NOREF) - -+#define SKB_ALLOC_FCLONE 0x01 -+#define SKB_ALLOC_RX 0x02 -+ -+static inline bool skb_emergency(const struct sk_buff *skb) -+{ -+#ifdef CONFIG_NETVM -+ return unlikely(skb->emergency); -+#else -+ return false; -+#endif -+} -+ - /** - * skb_dst - returns skb dst_entry - * @skb: buffer -@@ -491,7 +506,7 @@ extern void kfree_skb(struct sk_buff *skb); - extern void consume_skb(struct sk_buff *skb); - extern void __kfree_skb(struct sk_buff *skb); - extern struct sk_buff *__alloc_skb(unsigned int size, -- gfp_t priority, int fclone, int node); -+ gfp_t priority, int flags, int node); - static inline struct sk_buff *alloc_skb(unsigned int size, - gfp_t priority) - { -@@ -501,7 +516,7 @@ static inline struct sk_buff *alloc_skb(unsigned int size, - static inline struct sk_buff *alloc_skb_fclone(unsigned int size, - gfp_t priority) - { -- return __alloc_skb(size, priority, 1, -1); -+ return __alloc_skb(size, priority, SKB_ALLOC_FCLONE, -1); - } - - extern bool skb_recycle_check(struct sk_buff *skb, int skb_size); -@@ -1516,7 +1531,8 @@ static inline void __skb_queue_purge(struct sk_buff_head *list) - static inline struct sk_buff *__dev_alloc_skb(unsigned int length, - gfp_t gfp_mask) - { -- struct sk_buff *skb = alloc_skb(length + NET_SKB_PAD, gfp_mask); -+ struct sk_buff *skb = -+ __alloc_skb(length + NET_SKB_PAD, gfp_mask, SKB_ALLOC_RX, -1); - if (likely(skb)) - skb_reserve(skb, NET_SKB_PAD); - return skb; -@@ -1557,6 +1573,7 @@ static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev, - } - - extern struct page *__netdev_alloc_page(struct net_device *dev, gfp_t gfp_mask); -+extern void __netdev_free_page(struct net_device *dev, struct page *page); - - /** - * netdev_alloc_page - allocate a page for ps-rx on a specific device -@@ -1573,7 +1590,7 @@ static inline struct page *netdev_alloc_page(struct net_device *dev) - - static inline void netdev_free_page(struct net_device *dev, struct page *page) - { -- __free_page(page); -+ __netdev_free_page(dev, page); - } - - /** -diff --git a/net/core/skbuff.c b/net/core/skbuff.c -index 34432b4..9e36dc2 100644 ---- a/net/core/skbuff.c -+++ b/net/core/skbuff.c -@@ -168,14 +168,21 @@ static void skb_under_panic(struct sk_buff *skb, int sz, void *here) - * %GFP_ATOMIC. - */ - struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, -- int fclone, int node) -+ int flags, int node) - { - struct kmem_cache *cache; - struct skb_shared_info *shinfo; - struct sk_buff *skb; - u8 *data; -+ int emergency = 0; -+ int memalloc = sk_memalloc_socks(); - -- cache = fclone ? skbuff_fclone_cache : skbuff_head_cache; -+ size = SKB_DATA_ALIGN(size); -+ cache = (flags & SKB_ALLOC_FCLONE) -+ ? skbuff_fclone_cache : skbuff_head_cache; -+ -+ if (memalloc && (flags & SKB_ALLOC_RX)) -+ gfp_mask |= __GFP_MEMALLOC; - - /* Get the HEAD */ - skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node); -@@ -183,9 +190,8 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, - goto out; - prefetchw(skb); - -- size = SKB_DATA_ALIGN(size); -- data = kmalloc_node_track_caller(size + sizeof(struct skb_shared_info), -- gfp_mask, node); -+ data = kmalloc_reserve(size + sizeof(struct skb_shared_info), -+ gfp_mask, node, &net_skb_reserve, &emergency); - if (!data) - goto nodata; - prefetchw(data + size); -@@ -196,6 +202,9 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, - * the tail pointer in struct sk_buff! - */ - memset(skb, 0, offsetof(struct sk_buff, tail)); -+#ifdef CONFIG_NETVM -+ skb->emergency = emergency; -+#endif - skb->truesize = size + sizeof(struct sk_buff); - atomic_set(&skb->users, 1); - skb->head = data; -@@ -213,7 +222,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, - memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); - atomic_set(&shinfo->dataref, 1); - -- if (fclone) { -+ if (flags & SKB_ALLOC_FCLONE) { - struct sk_buff *child = skb + 1; - atomic_t *fclone_ref = (atomic_t *) (child + 1); - -@@ -223,6 +232,9 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, - atomic_set(fclone_ref, 1); - - child->fclone = SKB_FCLONE_UNAVAILABLE; -+#ifdef CONFIG_NETVM -+ child->emergency = skb->emergency; -+#endif - } - out: - return skb; -@@ -252,7 +264,7 @@ struct sk_buff *__netdev_alloc_skb(struct net_device *dev, - int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1; - struct sk_buff *skb; - -- skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, 0, node); -+ skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, SKB_ALLOC_RX, node); - if (likely(skb)) { - skb_reserve(skb, NET_SKB_PAD); - skb->dev = dev; -@@ -266,11 +278,19 @@ struct page *__netdev_alloc_page(struct net_device *dev, gfp_t gfp_mask) - int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1; - struct page *page; - -- page = alloc_pages_node(node, gfp_mask, 0); -+ page = alloc_pages_reserve(node, gfp_mask | __GFP_MEMALLOC, 0, -+ &net_skb_reserve, NULL); -+ - return page; - } - EXPORT_SYMBOL(__netdev_alloc_page); - -+void __netdev_free_page(struct net_device *dev, struct page *page) -+{ -+ free_pages_reserve(page, 0, &net_skb_reserve, page->reserve); -+} -+EXPORT_SYMBOL(__netdev_free_page); -+ - void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, - int size) - { -@@ -278,6 +298,27 @@ void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, - skb->len += size; - skb->data_len += size; - skb->truesize += size; -+ -+#ifdef CONFIG_NETVM -+ /* -+ * In the rare case that skb_emergency() != page->reserved we'll -+ * skew the accounting slightly, but since its only a 'small' constant -+ * shift its ok. -+ */ -+ if (skb_emergency(skb)) { -+ /* -+ * We need to track fragment pages so that we properly -+ * release their reserve in skb_put_page(). -+ */ -+ atomic_set(&page->frag_count, 1); -+ } else if (unlikely(page->reserve)) { -+ /* -+ * Release the reserve now, because normal skbs don't -+ * do the emergency accounting. -+ */ -+ mem_reserve_pages_charge(&net_skb_reserve, -1); -+ } -+#endif - } - EXPORT_SYMBOL(skb_add_rx_frag); - -@@ -329,21 +370,38 @@ static void skb_clone_fraglist(struct sk_buff *skb) - skb_get(list); - } - -+static void skb_get_page(struct sk_buff *skb, struct page *page) -+{ -+ get_page(page); -+ if (skb_emergency(skb)) -+ atomic_inc(&page->frag_count); -+} -+ -+static void skb_put_page(struct sk_buff *skb, struct page *page) -+{ -+ if (skb_emergency(skb) && atomic_dec_and_test(&page->frag_count)) -+ mem_reserve_pages_charge(&net_skb_reserve, -1); -+ put_page(page); -+} -+ - static void skb_release_data(struct sk_buff *skb) - { - if (!skb->cloned || - !atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1, - &skb_shinfo(skb)->dataref)) { -+ - if (skb_shinfo(skb)->nr_frags) { - int i; -- for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) -- put_page(skb_shinfo(skb)->frags[i].page); -+ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { -+ skb_put_page(skb, -+ skb_shinfo(skb)->frags[i].page); -+ } - } - - if (skb_has_frags(skb)) - skb_drop_fraglist(skb); - -- kfree(skb->head); -+ kfree_reserve(skb->head, &net_skb_reserve, skb_emergency(skb)); - } - } - -@@ -536,6 +594,9 @@ static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) - #if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE) - new->ipvs_property = old->ipvs_property; - #endif -+#ifdef CONFIG_NETVM -+ new->emergency = old->emergency; -+#endif - new->protocol = old->protocol; - new->mark = old->mark; - new->skb_iif = old->skb_iif; -@@ -630,6 +691,9 @@ struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) - n->fclone = SKB_FCLONE_CLONE; - atomic_inc(fclone_ref); - } else { -+ if (skb_emergency(skb)) -+ gfp_mask |= __GFP_MEMALLOC; -+ - n = kmem_cache_alloc(skbuff_head_cache, gfp_mask); - if (!n) - return NULL; -@@ -666,6 +730,14 @@ static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old) - skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type; - } - -+static inline int skb_alloc_rx_flag(const struct sk_buff *skb) -+{ -+ if (skb_emergency(skb)) -+ return SKB_ALLOC_RX; -+ -+ return 0; -+} -+ - /** - * skb_copy - create private copy of an sk_buff - * @skb: buffer to copy -@@ -686,15 +758,17 @@ static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old) - struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) - { - int headerlen = skb->data - skb->head; -+ int size; - /* - * Allocate the copy buffer - */ - struct sk_buff *n; - #ifdef NET_SKBUFF_DATA_USES_OFFSET -- n = alloc_skb(skb->end + skb->data_len, gfp_mask); -+ size = skb->end + skb->data_len; - #else -- n = alloc_skb(skb->end - skb->head + skb->data_len, gfp_mask); -+ size = skb->end - skb->head + skb->data_len; - #endif -+ n = __alloc_skb(size, gfp_mask, skb_alloc_rx_flag(skb), -1); - if (!n) - return NULL; - -@@ -729,12 +803,14 @@ struct sk_buff *pskb_copy(struct sk_buff *skb, gfp_t gfp_mask) - /* - * Allocate the copy buffer - */ -+ int size; - struct sk_buff *n; - #ifdef NET_SKBUFF_DATA_USES_OFFSET -- n = alloc_skb(skb->end, gfp_mask); -+ size = skb->end; - #else -- n = alloc_skb(skb->end - skb->head, gfp_mask); -+ size = skb->end - skb->head; - #endif -+ n = __alloc_skb(size, gfp_mask, skb_alloc_rx_flag(skb), -1); - if (!n) - goto out; - -@@ -753,8 +829,9 @@ struct sk_buff *pskb_copy(struct sk_buff *skb, gfp_t gfp_mask) - int i; - - for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { -- skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; -- get_page(skb_shinfo(n)->frags[i].page); -+ skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; -+ skb_shinfo(n)->frags[i] = *frag; -+ skb_get_page(n, frag->page); - } - skb_shinfo(n)->nr_frags = i; - } -@@ -805,7 +882,11 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, - - size = SKB_DATA_ALIGN(size); - -- data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask); -+ if (skb_emergency(skb)) -+ gfp_mask |= __GFP_MEMALLOC; -+ -+ data = kmalloc_reserve(size + sizeof(struct skb_shared_info), -+ gfp_mask, -1, &net_skb_reserve, NULL); - if (!data) - goto nodata; - -@@ -820,7 +901,7 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, - sizeof(struct skb_shared_info)); - - for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) -- get_page(skb_shinfo(skb)->frags[i].page); -+ skb_get_page(skb, skb_shinfo(skb)->frags[i].page); - - if (skb_has_frags(skb)) - skb_clone_fraglist(skb); -@@ -901,8 +982,8 @@ struct sk_buff *skb_copy_expand(const struct sk_buff *skb, - /* - * Allocate the copy buffer - */ -- struct sk_buff *n = alloc_skb(newheadroom + skb->len + newtailroom, -- gfp_mask); -+ struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom, -+ gfp_mask, skb_alloc_rx_flag(skb), -1); - int oldheadroom = skb_headroom(skb); - int head_copy_len, head_copy_off; - int off; -@@ -1094,7 +1175,7 @@ drop_pages: - skb_shinfo(skb)->nr_frags = i; - - for (; i < nfrags; i++) -- put_page(skb_shinfo(skb)->frags[i].page); -+ skb_put_page(skb, skb_shinfo(skb)->frags[i].page); - - if (skb_has_frags(skb)) - skb_drop_fraglist(skb); -@@ -1263,7 +1344,7 @@ pull_pages: - k = 0; - for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { - if (skb_shinfo(skb)->frags[i].size <= eat) { -- put_page(skb_shinfo(skb)->frags[i].page); -+ skb_put_page(skb, skb_shinfo(skb)->frags[i].page); - eat -= skb_shinfo(skb)->frags[i].size; - } else { - skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; -@@ -2045,6 +2126,7 @@ static inline void skb_split_no_header(struct sk_buff *skb, - skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i]; - - if (pos < len) { -+ struct page *page = skb_shinfo(skb)->frags[i].page; - /* Split frag. - * We have two variants in this case: - * 1. Move all the frag to the second -@@ -2053,7 +2135,7 @@ static inline void skb_split_no_header(struct sk_buff *skb, - * where splitting is expensive. - * 2. Split is accurately. We make this. - */ -- get_page(skb_shinfo(skb)->frags[i].page); -+ skb_get_page(skb1, page); - skb_shinfo(skb1)->frags[0].page_offset += len - pos; - skb_shinfo(skb1)->frags[0].size -= len - pos; - skb_shinfo(skb)->frags[i].size = len - pos; -@@ -2552,8 +2634,9 @@ struct sk_buff *skb_segment(struct sk_buff *skb, int features) - skb_release_head_state(nskb); - __skb_push(nskb, doffset); - } else { -- nskb = alloc_skb(hsize + doffset + headroom, -- GFP_ATOMIC); -+ nskb = __alloc_skb(hsize + doffset + headroom, -+ GFP_ATOMIC, skb_alloc_rx_flag(skb), -+ -1); - - if (unlikely(!nskb)) - goto err; -@@ -2595,7 +2678,7 @@ struct sk_buff *skb_segment(struct sk_buff *skb, int features) - - while (pos < offset + len && i < nfrags) { - *frag = skb_shinfo(skb)->frags[i]; -- get_page(frag->page); -+ skb_get_page(nskb, frag->page); - size = frag->size; - - if (pos < offset) { --- -1.7.1.1 - --- -To unsubscribe, send a message with 'unsubscribe linux-mm' in -the body to majordomo@kvack.org. For more info on Linux MM, -see: http://www.linux-mm.org/ . -Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a> diff --git a/a/content_digest b/N2/content_digest index 63b0b09..f236a7e 100644 --- a/a/content_digest +++ b/N2/content_digest @@ -16,477 +16,5 @@ " davem@davemloft.net\0" "\00:1\0" "b\0" - ">From 3e824860934af5aa0150608314693c5e0e3608b6 Mon Sep 17 00:00:00 2001\n" - "From: Xiaotian Feng <dfeng@redhat.com>\n" - "Date: Tue, 13 Jul 2010 11:30:27 +0800\n" - "Subject: [PATCH 17/30] netvm: hook skb allocation to reserves\n" - "\n" - "Change the skb allocation api to indicate RX usage and use this to fall back to\n" - "the reserve when needed. SKBs allocated from the reserve are tagged in\n" - "skb->emergency.\n" - "\n" - "Teach all other skb ops about emergency skbs and the reserve accounting.\n" - "\n" - "Use the (new) packet split API to allocate and track fragment pages from the\n" - "emergency reserve. Do this using an atomic counter in page->index. This is\n" - "needed because the fragments have a different sharing semantic than that\n" - "indicated by skb_shinfo()->dataref.\n" - "\n" - "Note that the decision to distinguish between regular and emergency SKBs allows\n" - "the accounting overhead to be limited to the later kind.\n" - "\n" - "Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>\n" - "Signed-off-by: Suresh Jayaraman <sjayaraman@suse.de>\n" - "Signed-off-by: Xiaotian Feng <dfeng@redhat.com>\n" - "---\n" - " include/linux/mm_types.h | 1 +\n" - " include/linux/skbuff.h | 27 +++++++--\n" - " net/core/skbuff.c | 137 +++++++++++++++++++++++++++++++++++++---------\n" - " 3 files changed, 133 insertions(+), 32 deletions(-)\n" - "\n" - "diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h\n" - "index a95a202..73e0526 100644\n" - "--- a/include/linux/mm_types.h\n" - "+++ b/include/linux/mm_types.h\n" - "@@ -72,6 +72,7 @@ struct page {\n" - " \t\tpgoff_t index;\t\t/* Our offset within mapping. */\n" - " \t\tvoid *freelist;\t\t/* SLUB: freelist req. slab lock */\n" - " \t\tint reserve;\t\t/* page_alloc: page is a reserve page */\n" - "+\t\tatomic_t frag_count;\t/* skb fragment use count */\n" - " \t};\n" - " \tstruct list_head lru;\t\t/* Pageout list, eg. active_list\n" - " \t\t\t\t\t * protected by zone->lru_lock !\n" - "diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h\n" - "index 988a4dc..4ac45ad 100644\n" - "--- a/include/linux/skbuff.h\n" - "+++ b/include/linux/skbuff.h\n" - "@@ -385,9 +385,12 @@ struct sk_buff {\n" - " #else\n" - " \t__u8\t\t\tdeliver_no_wcard:1;\n" - " #endif\n" - "+#ifdef CONFIG_NETVM\n" - "+\t__u8 emergency:1;\n" - "+#endif\n" - " \tkmemcheck_bitfield_end(flags2);\n" - " \n" - "-\t/* 0/14 bit hole */\n" - "+\t/* 0/13/14 bit hole */\n" - " \n" - " #ifdef CONFIG_NET_DMA\n" - " \tdma_cookie_t\t\tdma_cookie;\n" - "@@ -429,6 +432,18 @@ struct sk_buff {\n" - " #define SKB_DST_NOREF\t1UL\n" - " #define SKB_DST_PTRMASK\t~(SKB_DST_NOREF)\n" - " \n" - "+#define SKB_ALLOC_FCLONE 0x01\n" - "+#define SKB_ALLOC_RX 0x02\n" - "+\n" - "+static inline bool skb_emergency(const struct sk_buff *skb)\n" - "+{\n" - "+#ifdef CONFIG_NETVM\n" - "+\treturn unlikely(skb->emergency);\n" - "+#else\n" - "+\treturn false;\n" - "+#endif\n" - "+}\n" - "+\n" - " /**\n" - " * skb_dst - returns skb dst_entry\n" - " * @skb: buffer\n" - "@@ -491,7 +506,7 @@ extern void kfree_skb(struct sk_buff *skb);\n" - " extern void consume_skb(struct sk_buff *skb);\n" - " extern void\t __kfree_skb(struct sk_buff *skb);\n" - " extern struct sk_buff *__alloc_skb(unsigned int size,\n" - "-\t\t\t\t gfp_t priority, int fclone, int node);\n" - "+\t\t\t\t gfp_t priority, int flags, int node);\n" - " static inline struct sk_buff *alloc_skb(unsigned int size,\n" - " \t\t\t\t\tgfp_t priority)\n" - " {\n" - "@@ -501,7 +516,7 @@ static inline struct sk_buff *alloc_skb(unsigned int size,\n" - " static inline struct sk_buff *alloc_skb_fclone(unsigned int size,\n" - " \t\t\t\t\t gfp_t priority)\n" - " {\n" - "-\treturn __alloc_skb(size, priority, 1, -1);\n" - "+\treturn __alloc_skb(size, priority, SKB_ALLOC_FCLONE, -1);\n" - " }\n" - " \n" - " extern bool skb_recycle_check(struct sk_buff *skb, int skb_size);\n" - "@@ -1516,7 +1531,8 @@ static inline void __skb_queue_purge(struct sk_buff_head *list)\n" - " static inline struct sk_buff *__dev_alloc_skb(unsigned int length,\n" - " \t\t\t\t\t gfp_t gfp_mask)\n" - " {\n" - "-\tstruct sk_buff *skb = alloc_skb(length + NET_SKB_PAD, gfp_mask);\n" - "+\tstruct sk_buff *skb =\n" - "+\t\t__alloc_skb(length + NET_SKB_PAD, gfp_mask, SKB_ALLOC_RX, -1);\n" - " \tif (likely(skb))\n" - " \t\tskb_reserve(skb, NET_SKB_PAD);\n" - " \treturn skb;\n" - "@@ -1557,6 +1573,7 @@ static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev,\n" - " }\n" - " \n" - " extern struct page *__netdev_alloc_page(struct net_device *dev, gfp_t gfp_mask);\n" - "+extern void __netdev_free_page(struct net_device *dev, struct page *page);\n" - " \n" - " /**\n" - " *\tnetdev_alloc_page - allocate a page for ps-rx on a specific device\n" - "@@ -1573,7 +1590,7 @@ static inline struct page *netdev_alloc_page(struct net_device *dev)\n" - " \n" - " static inline void netdev_free_page(struct net_device *dev, struct page *page)\n" - " {\n" - "-\t__free_page(page);\n" - "+\t__netdev_free_page(dev, page);\n" - " }\n" - " \n" - " /**\n" - "diff --git a/net/core/skbuff.c b/net/core/skbuff.c\n" - "index 34432b4..9e36dc2 100644\n" - "--- a/net/core/skbuff.c\n" - "+++ b/net/core/skbuff.c\n" - "@@ -168,14 +168,21 @@ static void skb_under_panic(struct sk_buff *skb, int sz, void *here)\n" - " *\t%GFP_ATOMIC.\n" - " */\n" - " struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,\n" - "-\t\t\t int fclone, int node)\n" - "+\t\t\t int flags, int node)\n" - " {\n" - " \tstruct kmem_cache *cache;\n" - " \tstruct skb_shared_info *shinfo;\n" - " \tstruct sk_buff *skb;\n" - " \tu8 *data;\n" - "+\tint emergency = 0;\n" - "+\tint memalloc = sk_memalloc_socks();\n" - " \n" - "-\tcache = fclone ? skbuff_fclone_cache : skbuff_head_cache;\n" - "+\tsize = SKB_DATA_ALIGN(size);\n" - "+\tcache = (flags & SKB_ALLOC_FCLONE)\n" - "+\t\t? skbuff_fclone_cache : skbuff_head_cache;\n" - "+\n" - "+\tif (memalloc && (flags & SKB_ALLOC_RX))\n" - "+\t\tgfp_mask |= __GFP_MEMALLOC;\n" - " \n" - " \t/* Get the HEAD */\n" - " \tskb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);\n" - "@@ -183,9 +190,8 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,\n" - " \t\tgoto out;\n" - " \tprefetchw(skb);\n" - " \n" - "-\tsize = SKB_DATA_ALIGN(size);\n" - "-\tdata = kmalloc_node_track_caller(size + sizeof(struct skb_shared_info),\n" - "-\t\t\tgfp_mask, node);\n" - "+\tdata = kmalloc_reserve(size + sizeof(struct skb_shared_info),\n" - "+\t\t\tgfp_mask, node, &net_skb_reserve, &emergency);\n" - " \tif (!data)\n" - " \t\tgoto nodata;\n" - " \tprefetchw(data + size);\n" - "@@ -196,6 +202,9 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,\n" - " \t * the tail pointer in struct sk_buff!\n" - " \t */\n" - " \tmemset(skb, 0, offsetof(struct sk_buff, tail));\n" - "+#ifdef CONFIG_NETVM\n" - "+\tskb->emergency = emergency;\n" - "+#endif\n" - " \tskb->truesize = size + sizeof(struct sk_buff);\n" - " \tatomic_set(&skb->users, 1);\n" - " \tskb->head = data;\n" - "@@ -213,7 +222,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,\n" - " \tmemset(shinfo, 0, offsetof(struct skb_shared_info, dataref));\n" - " \tatomic_set(&shinfo->dataref, 1);\n" - " \n" - "-\tif (fclone) {\n" - "+\tif (flags & SKB_ALLOC_FCLONE) {\n" - " \t\tstruct sk_buff *child = skb + 1;\n" - " \t\tatomic_t *fclone_ref = (atomic_t *) (child + 1);\n" - " \n" - "@@ -223,6 +232,9 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,\n" - " \t\tatomic_set(fclone_ref, 1);\n" - " \n" - " \t\tchild->fclone = SKB_FCLONE_UNAVAILABLE;\n" - "+#ifdef CONFIG_NETVM\n" - "+\t\tchild->emergency = skb->emergency;\n" - "+#endif\n" - " \t}\n" - " out:\n" - " \treturn skb;\n" - "@@ -252,7 +264,7 @@ struct sk_buff *__netdev_alloc_skb(struct net_device *dev,\n" - " \tint node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1;\n" - " \tstruct sk_buff *skb;\n" - " \n" - "-\tskb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, 0, node);\n" - "+\tskb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, SKB_ALLOC_RX, node);\n" - " \tif (likely(skb)) {\n" - " \t\tskb_reserve(skb, NET_SKB_PAD);\n" - " \t\tskb->dev = dev;\n" - "@@ -266,11 +278,19 @@ struct page *__netdev_alloc_page(struct net_device *dev, gfp_t gfp_mask)\n" - " \tint node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1;\n" - " \tstruct page *page;\n" - " \n" - "-\tpage = alloc_pages_node(node, gfp_mask, 0);\n" - "+\tpage = alloc_pages_reserve(node, gfp_mask | __GFP_MEMALLOC, 0,\n" - "+\t\t\t&net_skb_reserve, NULL);\n" - "+\n" - " \treturn page;\n" - " }\n" - " EXPORT_SYMBOL(__netdev_alloc_page);\n" - " \n" - "+void __netdev_free_page(struct net_device *dev, struct page *page)\n" - "+{\n" - "+\tfree_pages_reserve(page, 0, &net_skb_reserve, page->reserve);\n" - "+}\n" - "+EXPORT_SYMBOL(__netdev_free_page);\n" - "+\n" - " void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,\n" - " \t\tint size)\n" - " {\n" - "@@ -278,6 +298,27 @@ void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,\n" - " \tskb->len += size;\n" - " \tskb->data_len += size;\n" - " \tskb->truesize += size;\n" - "+\n" - "+#ifdef CONFIG_NETVM\n" - "+\t/*\n" - "+\t * In the rare case that skb_emergency() != page->reserved we'll\n" - "+\t * skew the accounting slightly, but since its only a 'small' constant\n" - "+\t * shift its ok.\n" - "+\t */\n" - "+\tif (skb_emergency(skb)) {\n" - "+\t\t/*\n" - "+\t\t * We need to track fragment pages so that we properly\n" - "+\t\t * release their reserve in skb_put_page().\n" - "+\t\t */\n" - "+\t\tatomic_set(&page->frag_count, 1);\n" - "+\t} else if (unlikely(page->reserve)) {\n" - "+\t\t/*\n" - "+\t\t * Release the reserve now, because normal skbs don't\n" - "+\t\t * do the emergency accounting.\n" - "+\t\t */\n" - "+\t\tmem_reserve_pages_charge(&net_skb_reserve, -1);\n" - "+\t}\n" - "+#endif\n" - " }\n" - " EXPORT_SYMBOL(skb_add_rx_frag);\n" - " \n" - "@@ -329,21 +370,38 @@ static void skb_clone_fraglist(struct sk_buff *skb)\n" - " \t\tskb_get(list);\n" - " }\n" - " \n" - "+static void skb_get_page(struct sk_buff *skb, struct page *page)\n" - "+{\n" - "+\tget_page(page);\n" - "+\tif (skb_emergency(skb))\n" - "+\t\tatomic_inc(&page->frag_count);\n" - "+}\n" - "+\n" - "+static void skb_put_page(struct sk_buff *skb, struct page *page)\n" - "+{\n" - "+\tif (skb_emergency(skb) && atomic_dec_and_test(&page->frag_count))\n" - "+\t\tmem_reserve_pages_charge(&net_skb_reserve, -1);\n" - "+\tput_page(page);\n" - "+}\n" - "+\n" - " static void skb_release_data(struct sk_buff *skb)\n" - " {\n" - " \tif (!skb->cloned ||\n" - " \t !atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,\n" - " \t\t\t &skb_shinfo(skb)->dataref)) {\n" - "+\n" - " \t\tif (skb_shinfo(skb)->nr_frags) {\n" - " \t\t\tint i;\n" - "-\t\t\tfor (i = 0; i < skb_shinfo(skb)->nr_frags; i++)\n" - "-\t\t\t\tput_page(skb_shinfo(skb)->frags[i].page);\n" - "+\t\t\tfor (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {\n" - "+\t\t\t\tskb_put_page(skb,\n" - "+\t\t\t\t\t skb_shinfo(skb)->frags[i].page);\n" - "+\t\t\t}\n" - " \t\t}\n" - " \n" - " \t\tif (skb_has_frags(skb))\n" - " \t\t\tskb_drop_fraglist(skb);\n" - " \n" - "-\t\tkfree(skb->head);\n" - "+\t\tkfree_reserve(skb->head, &net_skb_reserve, skb_emergency(skb));\n" - " \t}\n" - " }\n" - " \n" - "@@ -536,6 +594,9 @@ static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)\n" - " #if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)\n" - " \tnew->ipvs_property\t= old->ipvs_property;\n" - " #endif\n" - "+#ifdef CONFIG_NETVM\n" - "+\tnew->emergency\t\t= old->emergency;\n" - "+#endif\n" - " \tnew->protocol\t\t= old->protocol;\n" - " \tnew->mark\t\t= old->mark;\n" - " \tnew->skb_iif\t\t= old->skb_iif;\n" - "@@ -630,6 +691,9 @@ struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)\n" - " \t\tn->fclone = SKB_FCLONE_CLONE;\n" - " \t\tatomic_inc(fclone_ref);\n" - " \t} else {\n" - "+\t\tif (skb_emergency(skb))\n" - "+\t\t\tgfp_mask |= __GFP_MEMALLOC;\n" - "+\n" - " \t\tn = kmem_cache_alloc(skbuff_head_cache, gfp_mask);\n" - " \t\tif (!n)\n" - " \t\t\treturn NULL;\n" - "@@ -666,6 +730,14 @@ static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)\n" - " \tskb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;\n" - " }\n" - " \n" - "+static inline int skb_alloc_rx_flag(const struct sk_buff *skb)\n" - "+{\n" - "+\tif (skb_emergency(skb))\n" - "+\t\treturn SKB_ALLOC_RX;\n" - "+\n" - "+\treturn 0;\n" - "+}\n" - "+\n" - " /**\n" - " *\tskb_copy\t-\tcreate private copy of an sk_buff\n" - " *\t@skb: buffer to copy\n" - "@@ -686,15 +758,17 @@ static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)\n" - " struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)\n" - " {\n" - " \tint headerlen = skb->data - skb->head;\n" - "+\tint size;\n" - " \t/*\n" - " \t *\tAllocate the copy buffer\n" - " \t */\n" - " \tstruct sk_buff *n;\n" - " #ifdef NET_SKBUFF_DATA_USES_OFFSET\n" - "-\tn = alloc_skb(skb->end + skb->data_len, gfp_mask);\n" - "+\tsize = skb->end + skb->data_len;\n" - " #else\n" - "-\tn = alloc_skb(skb->end - skb->head + skb->data_len, gfp_mask);\n" - "+\tsize = skb->end - skb->head + skb->data_len;\n" - " #endif\n" - "+\tn = __alloc_skb(size, gfp_mask, skb_alloc_rx_flag(skb), -1);\n" - " \tif (!n)\n" - " \t\treturn NULL;\n" - " \n" - "@@ -729,12 +803,14 @@ struct sk_buff *pskb_copy(struct sk_buff *skb, gfp_t gfp_mask)\n" - " \t/*\n" - " \t *\tAllocate the copy buffer\n" - " \t */\n" - "+\tint size;\n" - " \tstruct sk_buff *n;\n" - " #ifdef NET_SKBUFF_DATA_USES_OFFSET\n" - "-\tn = alloc_skb(skb->end, gfp_mask);\n" - "+\tsize = skb->end;\n" - " #else\n" - "-\tn = alloc_skb(skb->end - skb->head, gfp_mask);\n" - "+\tsize = skb->end - skb->head;\n" - " #endif\n" - "+\tn = __alloc_skb(size, gfp_mask, skb_alloc_rx_flag(skb), -1);\n" - " \tif (!n)\n" - " \t\tgoto out;\n" - " \n" - "@@ -753,8 +829,9 @@ struct sk_buff *pskb_copy(struct sk_buff *skb, gfp_t gfp_mask)\n" - " \t\tint i;\n" - " \n" - " \t\tfor (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {\n" - "-\t\t\tskb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];\n" - "-\t\t\tget_page(skb_shinfo(n)->frags[i].page);\n" - "+\t\t\tskb_frag_t *frag = &skb_shinfo(skb)->frags[i];\n" - "+\t\t\tskb_shinfo(n)->frags[i] = *frag;\n" - "+\t\t\tskb_get_page(n, frag->page);\n" - " \t\t}\n" - " \t\tskb_shinfo(n)->nr_frags = i;\n" - " \t}\n" - "@@ -805,7 +882,11 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,\n" - " \n" - " \tsize = SKB_DATA_ALIGN(size);\n" - " \n" - "-\tdata = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask);\n" - "+\tif (skb_emergency(skb))\n" - "+\t\tgfp_mask |= __GFP_MEMALLOC;\n" - "+\n" - "+\tdata = kmalloc_reserve(size + sizeof(struct skb_shared_info),\n" - "+\t\t\tgfp_mask, -1, &net_skb_reserve, NULL);\n" - " \tif (!data)\n" - " \t\tgoto nodata;\n" - " \n" - "@@ -820,7 +901,7 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,\n" - " \t sizeof(struct skb_shared_info));\n" - " \n" - " \tfor (i = 0; i < skb_shinfo(skb)->nr_frags; i++)\n" - "-\t\tget_page(skb_shinfo(skb)->frags[i].page);\n" - "+\t\tskb_get_page(skb, skb_shinfo(skb)->frags[i].page);\n" - " \n" - " \tif (skb_has_frags(skb))\n" - " \t\tskb_clone_fraglist(skb);\n" - "@@ -901,8 +982,8 @@ struct sk_buff *skb_copy_expand(const struct sk_buff *skb,\n" - " \t/*\n" - " \t *\tAllocate the copy buffer\n" - " \t */\n" - "-\tstruct sk_buff *n = alloc_skb(newheadroom + skb->len + newtailroom,\n" - "-\t\t\t\t gfp_mask);\n" - "+\tstruct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,\n" - "+\t\t\t\t\tgfp_mask, skb_alloc_rx_flag(skb), -1);\n" - " \tint oldheadroom = skb_headroom(skb);\n" - " \tint head_copy_len, head_copy_off;\n" - " \tint off;\n" - "@@ -1094,7 +1175,7 @@ drop_pages:\n" - " \t\tskb_shinfo(skb)->nr_frags = i;\n" - " \n" - " \t\tfor (; i < nfrags; i++)\n" - "-\t\t\tput_page(skb_shinfo(skb)->frags[i].page);\n" - "+\t\t\tskb_put_page(skb, skb_shinfo(skb)->frags[i].page);\n" - " \n" - " \t\tif (skb_has_frags(skb))\n" - " \t\t\tskb_drop_fraglist(skb);\n" - "@@ -1263,7 +1344,7 @@ pull_pages:\n" - " \tk = 0;\n" - " \tfor (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {\n" - " \t\tif (skb_shinfo(skb)->frags[i].size <= eat) {\n" - "-\t\t\tput_page(skb_shinfo(skb)->frags[i].page);\n" - "+\t\t\tskb_put_page(skb, skb_shinfo(skb)->frags[i].page);\n" - " \t\t\teat -= skb_shinfo(skb)->frags[i].size;\n" - " \t\t} else {\n" - " \t\t\tskb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];\n" - "@@ -2045,6 +2126,7 @@ static inline void skb_split_no_header(struct sk_buff *skb,\n" - " \t\t\tskb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];\n" - " \n" - " \t\t\tif (pos < len) {\n" - "+\t\t\t\tstruct page *page = skb_shinfo(skb)->frags[i].page;\n" - " \t\t\t\t/* Split frag.\n" - " \t\t\t\t * We have two variants in this case:\n" - " \t\t\t\t * 1. Move all the frag to the second\n" - "@@ -2053,7 +2135,7 @@ static inline void skb_split_no_header(struct sk_buff *skb,\n" - " \t\t\t\t * where splitting is expensive.\n" - " \t\t\t\t * 2. Split is accurately. We make this.\n" - " \t\t\t\t */\n" - "-\t\t\t\tget_page(skb_shinfo(skb)->frags[i].page);\n" - "+\t\t\t\tskb_get_page(skb1, page);\n" - " \t\t\t\tskb_shinfo(skb1)->frags[0].page_offset += len - pos;\n" - " \t\t\t\tskb_shinfo(skb1)->frags[0].size -= len - pos;\n" - " \t\t\t\tskb_shinfo(skb)->frags[i].size\t= len - pos;\n" - "@@ -2552,8 +2634,9 @@ struct sk_buff *skb_segment(struct sk_buff *skb, int features)\n" - " \t\t\tskb_release_head_state(nskb);\n" - " \t\t\t__skb_push(nskb, doffset);\n" - " \t\t} else {\n" - "-\t\t\tnskb = alloc_skb(hsize + doffset + headroom,\n" - "-\t\t\t\t\t GFP_ATOMIC);\n" - "+\t\t\tnskb = __alloc_skb(hsize + doffset + headroom,\n" - "+\t\t\t\t\t GFP_ATOMIC, skb_alloc_rx_flag(skb),\n" - "+\t\t\t\t\t -1);\n" - " \n" - " \t\t\tif (unlikely(!nskb))\n" - " \t\t\t\tgoto err;\n" - "@@ -2595,7 +2678,7 @@ struct sk_buff *skb_segment(struct sk_buff *skb, int features)\n" - " \n" - " \t\twhile (pos < offset + len && i < nfrags) {\n" - " \t\t\t*frag = skb_shinfo(skb)->frags[i];\n" - "-\t\t\tget_page(frag->page);\n" - "+\t\t\tskb_get_page(nskb, frag->page);\n" - " \t\t\tsize = frag->size;\n" - " \n" - " \t\t\tif (pos < offset) {\n" - "-- \n" - "1.7.1.1\n" - "\n" - "--\n" - "To unsubscribe, send a message with 'unsubscribe linux-mm' in\n" - "the body to majordomo@kvack.org. For more info on Linux MM,\n" - "see: http://www.linux-mm.org/ .\n" - "Don't email: <a href=mailto:\"dont@kvack.org\"> email@kvack.org </a>" -d42aa7c7229253b0869ec391cb59d0b2650da413c6256afcec8d0048c18af4b9 +de9417c1dcb1d32c83e61eba2ad87a428ee5dc62f435adcb3557dda338883e6a
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.