From: Hyeonggon Yoo <42.hyeyoo@gmail.com>
To: Vlastimil Babka <vbabka@suse.cz>
Cc: David Rientjes <rientjes@google.com>,
Christoph Lameter <cl@linux.com>,
Pekka Enberg <penberg@kernel.org>,
Joonsoo Kim <iamjoonsoo.kim@lge.com>,
Andrew Morton <akpm@linux-foundation.org>,
Roman Gushchin <roman.gushchin@linux.dev>,
Andrey Ryabinin <ryabinin.a.a@gmail.com>,
Alexander Potapenko <glider@google.com>,
Andrey Konovalov <andreyknvl@gmail.com>,
Dmitry Vyukov <dvyukov@google.com>,
Vincenzo Frascino <vincenzo.frascino@arm.com>,
Marco Elver <elver@google.com>,
Johannes Weiner <hannes@cmpxchg.org>,
Michal Hocko <mhocko@kernel.org>,
Shakeel Butt <shakeelb@google.com>,
Muchun Song <muchun.song@linux.dev>,
Kees Cook <keescook@chromium.org>,
linux-mm@kvack.org, linux-kernel@vger.kernel.org,
kasan-dev@googlegroups.com, cgroups@vger.kernel.org,
linux-hardening@vger.kernel.org
Subject: Re: [PATCH v2 18/21] mm/slab: move kmalloc() functions from slab_common.c to slub.c
Date: Thu, 7 Dec 2023 10:30:57 +0900 [thread overview]
Message-ID: <ZXEgUVrUuIHlgsec@localhost.localdomain> (raw)
In-Reply-To: <20231120-slab-remove-slab-v2-18-9c9c70177183@suse.cz>
On Mon, Nov 20, 2023 at 07:34:29PM +0100, Vlastimil Babka wrote:
> This will eliminate a call between compilation units through
> __kmem_cache_alloc_node() and allow better inlining of the allocation
> fast path.
>
> Reviewed-by: Kees Cook <keescook@chromium.org>
> Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
> ---
> mm/slab.h | 3 --
> mm/slab_common.c | 119 ----------------------------------------------------
> mm/slub.c | 126 +++++++++++++++++++++++++++++++++++++++++++++++++++----
> 3 files changed, 118 insertions(+), 130 deletions(-)
>
> diff --git a/mm/slab.h b/mm/slab.h
> index 7d7cc7af614e..54deeb0428c6 100644
> --- a/mm/slab.h
> +++ b/mm/slab.h
> @@ -416,9 +416,6 @@ kmalloc_slab(size_t size, gfp_t flags, unsigned long caller)
> return kmalloc_caches[kmalloc_type(flags, caller)][index];
> }
>
> -void *__kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags,
> - int node, size_t orig_size,
> - unsigned long caller);
> gfp_t kmalloc_fix_flags(gfp_t flags);
>
> /* Functions provided by the slab allocators */
> diff --git a/mm/slab_common.c b/mm/slab_common.c
> index 31ade17a7ad9..238293b1dbe1 100644
> --- a/mm/slab_common.c
> +++ b/mm/slab_common.c
> @@ -936,50 +936,6 @@ void __init create_kmalloc_caches(slab_flags_t flags)
> slab_state = UP;
> }
>
> -static void *__kmalloc_large_node(size_t size, gfp_t flags, int node);
> -static __always_inline
> -void *__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
> -{
> - struct kmem_cache *s;
> - void *ret;
> -
> - if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
> - ret = __kmalloc_large_node(size, flags, node);
> - trace_kmalloc(caller, ret, size,
> - PAGE_SIZE << get_order(size), flags, node);
> - return ret;
> - }
> -
> - if (unlikely(!size))
> - return ZERO_SIZE_PTR;
> -
> - s = kmalloc_slab(size, flags, caller);
> -
> - ret = __kmem_cache_alloc_node(s, flags, node, size, caller);
> - ret = kasan_kmalloc(s, ret, size, flags);
> - trace_kmalloc(caller, ret, size, s->size, flags, node);
> - return ret;
> -}
> -
> -void *__kmalloc_node(size_t size, gfp_t flags, int node)
> -{
> - return __do_kmalloc_node(size, flags, node, _RET_IP_);
> -}
> -EXPORT_SYMBOL(__kmalloc_node);
> -
> -void *__kmalloc(size_t size, gfp_t flags)
> -{
> - return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_);
> -}
> -EXPORT_SYMBOL(__kmalloc);
> -
> -void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
> - int node, unsigned long caller)
> -{
> - return __do_kmalloc_node(size, flags, node, caller);
> -}
> -EXPORT_SYMBOL(__kmalloc_node_track_caller);
> -
> /**
> * __ksize -- Report full size of underlying allocation
> * @object: pointer to the object
> @@ -1016,30 +972,6 @@ size_t __ksize(const void *object)
> return slab_ksize(folio_slab(folio)->slab_cache);
> }
>
> -void *kmalloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
> -{
> - void *ret = __kmem_cache_alloc_node(s, gfpflags, NUMA_NO_NODE,
> - size, _RET_IP_);
> -
> - trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, NUMA_NO_NODE);
> -
> - ret = kasan_kmalloc(s, ret, size, gfpflags);
> - return ret;
> -}
> -EXPORT_SYMBOL(kmalloc_trace);
> -
> -void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,
> - int node, size_t size)
> -{
> - void *ret = __kmem_cache_alloc_node(s, gfpflags, node, size, _RET_IP_);
> -
> - trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, node);
> -
> - ret = kasan_kmalloc(s, ret, size, gfpflags);
> - return ret;
> -}
> -EXPORT_SYMBOL(kmalloc_node_trace);
> -
> gfp_t kmalloc_fix_flags(gfp_t flags)
> {
> gfp_t invalid_mask = flags & GFP_SLAB_BUG_MASK;
> @@ -1052,57 +984,6 @@ gfp_t kmalloc_fix_flags(gfp_t flags)
> return flags;
> }
>
> -/*
> - * To avoid unnecessary overhead, we pass through large allocation requests
> - * directly to the page allocator. We use __GFP_COMP, because we will need to
> - * know the allocation order to free the pages properly in kfree.
> - */
> -
> -static void *__kmalloc_large_node(size_t size, gfp_t flags, int node)
> -{
> - struct page *page;
> - void *ptr = NULL;
> - unsigned int order = get_order(size);
> -
> - if (unlikely(flags & GFP_SLAB_BUG_MASK))
> - flags = kmalloc_fix_flags(flags);
> -
> - flags |= __GFP_COMP;
> - page = alloc_pages_node(node, flags, order);
> - if (page) {
> - ptr = page_address(page);
> - mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
> - PAGE_SIZE << order);
> - }
> -
> - ptr = kasan_kmalloc_large(ptr, size, flags);
> - /* As ptr might get tagged, call kmemleak hook after KASAN. */
> - kmemleak_alloc(ptr, size, 1, flags);
> - kmsan_kmalloc_large(ptr, size, flags);
> -
> - return ptr;
> -}
> -
> -void *kmalloc_large(size_t size, gfp_t flags)
> -{
> - void *ret = __kmalloc_large_node(size, flags, NUMA_NO_NODE);
> -
> - trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size),
> - flags, NUMA_NO_NODE);
> - return ret;
> -}
> -EXPORT_SYMBOL(kmalloc_large);
> -
> -void *kmalloc_large_node(size_t size, gfp_t flags, int node)
> -{
> - void *ret = __kmalloc_large_node(size, flags, node);
> -
> - trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size),
> - flags, node);
> - return ret;
> -}
> -EXPORT_SYMBOL(kmalloc_large_node);
> -
> #ifdef CONFIG_SLAB_FREELIST_RANDOM
> /* Randomize a generic freelist */
> static void freelist_randomize(unsigned int *list,
> diff --git a/mm/slub.c b/mm/slub.c
> index 2baa9e94d9df..d6bc15929d22 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -3851,14 +3851,6 @@ void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru,
> }
> EXPORT_SYMBOL(kmem_cache_alloc_lru);
>
> -void *__kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags,
> - int node, size_t orig_size,
> - unsigned long caller)
> -{
> - return slab_alloc_node(s, NULL, gfpflags, node,
> - caller, orig_size);
> -}
> -
> /**
> * kmem_cache_alloc_node - Allocate an object on the specified node
> * @s: The cache to allocate from.
> @@ -3882,6 +3874,124 @@ void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
> }
> EXPORT_SYMBOL(kmem_cache_alloc_node);
>
> +/*
> + * To avoid unnecessary overhead, we pass through large allocation requests
> + * directly to the page allocator. We use __GFP_COMP, because we will need to
> + * know the allocation order to free the pages properly in kfree.
> + */
> +static void *__kmalloc_large_node(size_t size, gfp_t flags, int node)
> +{
> + struct page *page;
> + void *ptr = NULL;
> + unsigned int order = get_order(size);
> +
> + if (unlikely(flags & GFP_SLAB_BUG_MASK))
> + flags = kmalloc_fix_flags(flags);
> +
> + flags |= __GFP_COMP;
> + page = alloc_pages_node(node, flags, order);
> + if (page) {
> + ptr = page_address(page);
> + mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
> + PAGE_SIZE << order);
> + }
> +
> + ptr = kasan_kmalloc_large(ptr, size, flags);
> + /* As ptr might get tagged, call kmemleak hook after KASAN. */
> + kmemleak_alloc(ptr, size, 1, flags);
> + kmsan_kmalloc_large(ptr, size, flags);
> +
> + return ptr;
> +}
> +
> +void *kmalloc_large(size_t size, gfp_t flags)
> +{
> + void *ret = __kmalloc_large_node(size, flags, NUMA_NO_NODE);
> +
> + trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size),
> + flags, NUMA_NO_NODE);
> + return ret;
> +}
> +EXPORT_SYMBOL(kmalloc_large);
> +
> +void *kmalloc_large_node(size_t size, gfp_t flags, int node)
> +{
> + void *ret = __kmalloc_large_node(size, flags, node);
> +
> + trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size),
> + flags, node);
> + return ret;
> +}
> +EXPORT_SYMBOL(kmalloc_large_node);
> +
> +static __always_inline
> +void *__do_kmalloc_node(size_t size, gfp_t flags, int node,
> + unsigned long caller)
> +{
> + struct kmem_cache *s;
> + void *ret;
> +
> + if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
> + ret = __kmalloc_large_node(size, flags, node);
> + trace_kmalloc(caller, ret, size,
> + PAGE_SIZE << get_order(size), flags, node);
> + return ret;
> + }
> +
> + if (unlikely(!size))
> + return ZERO_SIZE_PTR;
> +
> + s = kmalloc_slab(size, flags, caller);
> +
> + ret = slab_alloc_node(s, NULL, flags, node, caller, size);
> + ret = kasan_kmalloc(s, ret, size, flags);
> + trace_kmalloc(caller, ret, size, s->size, flags, node);
> + return ret;
> +}
> +
> +void *__kmalloc_node(size_t size, gfp_t flags, int node)
> +{
> + return __do_kmalloc_node(size, flags, node, _RET_IP_);
> +}
> +EXPORT_SYMBOL(__kmalloc_node);
> +
> +void *__kmalloc(size_t size, gfp_t flags)
> +{
> + return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_);
> +}
> +EXPORT_SYMBOL(__kmalloc);
> +
> +void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
> + int node, unsigned long caller)
> +{
> + return __do_kmalloc_node(size, flags, node, caller);
> +}
> +EXPORT_SYMBOL(__kmalloc_node_track_caller);
> +
> +void *kmalloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
> +{
> + void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE,
> + _RET_IP_, size);
> +
> + trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, NUMA_NO_NODE);
> +
> + ret = kasan_kmalloc(s, ret, size, gfpflags);
> + return ret;
> +}
> +EXPORT_SYMBOL(kmalloc_trace);
> +
> +void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,
> + int node, size_t size)
> +{
> + void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size);
> +
> + trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, node);
> +
> + ret = kasan_kmalloc(s, ret, size, gfpflags);
> + return ret;
> +}
> +EXPORT_SYMBOL(kmalloc_node_trace);
> +
> static noinline void free_to_partial_list(
> struct kmem_cache *s, struct slab *slab,
> void *head, void *tail, int bulk_cnt,
>
> --
Looks good to me,
Reviewed-by: Hyeonggon Yoo <42.hyeyoo@gmail.com>
> 2.42.1
>
>
next prev parent reply other threads:[~2023-12-07 1:31 UTC|newest]
Thread overview: 56+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-11-20 18:34 [PATCH v2 00/21] remove the SLAB allocator Vlastimil Babka
2023-11-20 18:34 ` [PATCH v2 01/21] mm/slab, docs: switch mm-api docs generation from slab.c to slub.c Vlastimil Babka
2023-11-24 0:46 ` David Rientjes
2023-12-05 3:53 ` Hyeonggon Yoo
2023-11-20 18:34 ` [PATCH v2 02/21] mm/slab: remove CONFIG_SLAB from all Kconfig and Makefile Vlastimil Babka
2023-12-05 4:15 ` Hyeonggon Yoo
2023-12-05 10:14 ` Vlastimil Babka
2023-12-06 0:08 ` Hyeonggon Yoo
2023-11-20 18:34 ` [PATCH v2 03/21] KASAN: remove code paths guarded by CONFIG_SLAB Vlastimil Babka
2023-11-21 8:23 ` Hyeonggon Yoo
2023-11-21 16:47 ` Andrey Konovalov
2023-12-05 4:26 ` Hyeonggon Yoo
2023-12-05 4:48 ` Hyeonggon Yoo
2023-12-05 10:16 ` Vlastimil Babka
2023-11-20 18:34 ` [PATCH v2 04/21] KFENCE: cleanup kfence_guarded_alloc() after CONFIG_SLAB removal Vlastimil Babka
2023-12-06 8:01 ` Hyeonggon Yoo
2023-11-20 18:34 ` [PATCH v2 05/21] mm/memcontrol: remove CONFIG_SLAB #ifdef guards Vlastimil Babka
2023-12-06 8:12 ` Hyeonggon Yoo
2023-11-20 18:34 ` [PATCH v2 06/21] cpu/hotplug: remove CPUHP_SLAB_PREPARE hooks Vlastimil Babka
2023-12-01 11:28 ` Thomas Gleixner
2023-12-06 8:28 ` Hyeonggon Yoo
2023-11-20 18:34 ` [PATCH v2 07/21] mm/slab: remove CONFIG_SLAB code from slab common code Vlastimil Babka
2023-12-06 9:05 ` Hyeonggon Yoo
2023-11-20 18:34 ` [PATCH v2 08/21] mm/mempool/dmapool: remove CONFIG_DEBUG_SLAB ifdefs Vlastimil Babka
2023-12-06 9:10 ` Hyeonggon Yoo
2023-11-20 18:34 ` [PATCH v2 09/21] mm/slab: remove mm/slab.c and slab_def.h Vlastimil Babka
2023-11-22 20:07 ` Christoph Lameter
2023-12-06 9:31 ` Hyeonggon Yoo
2023-12-06 9:37 ` Vlastimil Babka
2023-11-20 18:34 ` [PATCH v2 10/21] mm/slab: move struct kmem_cache_cpu declaration to slub.c Vlastimil Babka
2023-12-06 9:35 ` Hyeonggon Yoo
2023-11-20 18:34 ` [PATCH v2 11/21] mm/slab: move the rest of slub_def.h to mm/slab.h Vlastimil Babka
2023-12-06 9:45 ` Hyeonggon Yoo
2023-11-20 18:34 ` [PATCH v2 12/21] mm/slab: consolidate includes in the internal mm/slab.h Vlastimil Babka
2023-12-07 0:30 ` Hyeonggon Yoo
2023-11-20 18:34 ` [PATCH v2 13/21] mm/slab: move pre/post-alloc hooks from slab.h to slub.c Vlastimil Babka
2023-12-07 0:43 ` Hyeonggon Yoo
2023-11-20 18:34 ` [PATCH v2 14/21] mm/slab: move memcg related functions " Vlastimil Babka
2023-12-07 0:59 ` Hyeonggon Yoo
2023-11-20 18:34 ` [PATCH v2 15/21] mm/slab: move struct kmem_cache_node " Vlastimil Babka
2023-12-07 1:11 ` Hyeonggon Yoo
2023-11-20 18:34 ` [PATCH v2 16/21] mm/slab: move kfree() from slab_common.c " Vlastimil Babka
2023-12-05 4:38 ` Hyeonggon Yoo
2023-11-20 18:34 ` [PATCH v2 17/21] mm/slab: move kmalloc_slab() to mm/slab.h Vlastimil Babka
2023-12-07 1:28 ` Hyeonggon Yoo
2023-11-20 18:34 ` [PATCH v2 18/21] mm/slab: move kmalloc() functions from slab_common.c to slub.c Vlastimil Babka
2023-12-07 1:30 ` Hyeonggon Yoo [this message]
2023-11-20 18:34 ` [PATCH v2 19/21] mm/slub: remove slab_alloc() and __kmem_cache_alloc_lru() wrappers Vlastimil Babka
2023-12-07 1:35 ` Hyeonggon Yoo
2023-11-20 18:34 ` [PATCH v2 20/21] mm/slub: optimize alloc fastpath code layout Vlastimil Babka
2023-12-07 2:32 ` Hyeonggon Yoo
2023-11-20 18:34 ` [PATCH v2 21/21] mm/slub: optimize free fast path " Vlastimil Babka
2023-12-07 2:40 ` Hyeonggon Yoo
2023-11-24 0:45 ` [PATCH v2 00/21] remove the SLAB allocator David Rientjes
2023-11-24 9:26 ` Vlastimil Babka
2023-12-07 2:45 ` Hyeonggon Yoo
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=ZXEgUVrUuIHlgsec@localhost.localdomain \
--to=42.hyeyoo@gmail.com \
--cc=akpm@linux-foundation.org \
--cc=andreyknvl@gmail.com \
--cc=cgroups@vger.kernel.org \
--cc=cl@linux.com \
--cc=dvyukov@google.com \
--cc=elver@google.com \
--cc=glider@google.com \
--cc=hannes@cmpxchg.org \
--cc=iamjoonsoo.kim@lge.com \
--cc=kasan-dev@googlegroups.com \
--cc=keescook@chromium.org \
--cc=linux-hardening@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=mhocko@kernel.org \
--cc=muchun.song@linux.dev \
--cc=penberg@kernel.org \
--cc=rientjes@google.com \
--cc=roman.gushchin@linux.dev \
--cc=ryabinin.a.a@gmail.com \
--cc=shakeelb@google.com \
--cc=vbabka@suse.cz \
--cc=vincenzo.frascino@arm.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.