From: Dmitry Vyukov <dvyukov@google.com>
To: Marco Elver <elver@google.com>
Cc: Alexander Potapenko <glider@google.com>,
Andrew Morton <akpm@linux-foundation.org>,
Catalin Marinas <catalin.marinas@arm.com>,
Christoph Lameter <cl@linux.com>,
David Rientjes <rientjes@google.com>,
Joonsoo Kim <iamjoonsoo.kim@lge.com>,
Mark Rutland <mark.rutland@arm.com>,
Pekka Enberg <penberg@kernel.org>,
"H. Peter Anvin" <hpa@zytor.com>,
"Paul E. McKenney" <paulmck@kernel.org>,
Andrey Konovalov <andreyknvl@google.com>,
Andrey Ryabinin <aryabinin@virtuozzo.com>,
Andy Lutomirski <luto@kernel.org>,
Borislav Petkov <bp@alien8.de>,
Dave Hansen <dave.hansen@linux.intel.com>,
Eric Dumazet <edumazet@google.com>,
Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
Ingo Molnar <mingo@redhat.com>, Jann Horn <jannh@google.com>,
Jonathan Corbet <corbet@lwn.net>,
Kees Cook <keescook@chromium.org>,
Peter Zijlstra <peterz@infradead.org>, Qian Cai <cai@lca.pw>,
Thomas Gleixner <tglx@linutronix.de>,
Will Deacon <will@kernel.org>,
"the arch/x86 maintainers" <x86@kernel.org>,
"open list:DOCUMENTATION" <linux-doc@vger.kernel.org>,
LKML <linux-kernel@vger.kernel.org>,
kasan-dev <kasan-dev@googlegroups.com>,
Linux ARM <linux-arm-kernel@lists.infradead.org>,
Linux-MM <linux-mm@kvack.org>
Subject: Re: [PATCH RFC 04/10] mm, kfence: insert KFENCE hooks for SLAB
Date: Fri, 11 Sep 2020 09:17:14 +0200 [thread overview]
Message-ID: <CACT4Y+aXNmQzp6J+mP+ELj8kUHmRPkibc1--KtV9a3ud_X8miw@mail.gmail.com> (raw)
In-Reply-To: <20200907134055.2878499-5-elver@google.com>
On Mon, Sep 7, 2020 at 3:41 PM Marco Elver <elver@google.com> wrote:
>
> From: Alexander Potapenko <glider@google.com>
>
> Inserts KFENCE hooks into the SLAB allocator.
>
> We note the addition of the 'orig_size' argument to slab_alloc*()
> functions, to be able to pass the originally requested size to KFENCE.
> When KFENCE is disabled, there is no additional overhead, since these
> functions are __always_inline.
>
> Co-developed-by: Marco Elver <elver@google.com>
> Signed-off-by: Marco Elver <elver@google.com>
> Signed-off-by: Alexander Potapenko <glider@google.com>
> ---
> mm/slab.c | 46 ++++++++++++++++++++++++++++++++++------------
> mm/slab_common.c | 6 +++++-
> 2 files changed, 39 insertions(+), 13 deletions(-)
>
> diff --git a/mm/slab.c b/mm/slab.c
> index 3160dff6fd76..30aba06ae02b 100644
> --- a/mm/slab.c
> +++ b/mm/slab.c
> @@ -100,6 +100,7 @@
> #include <linux/seq_file.h>
> #include <linux/notifier.h>
> #include <linux/kallsyms.h>
> +#include <linux/kfence.h>
> #include <linux/cpu.h>
> #include <linux/sysctl.h>
> #include <linux/module.h>
> @@ -3206,7 +3207,7 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
> }
>
> static __always_inline void *
> -slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
> +slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, size_t orig_size,
> unsigned long caller)
> {
> unsigned long save_flags;
> @@ -3219,6 +3220,10 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
> if (unlikely(!cachep))
> return NULL;
>
> + ptr = kfence_alloc(cachep, orig_size, flags);
> + if (unlikely(ptr))
> + goto out_hooks;
> +
> cache_alloc_debugcheck_before(cachep, flags);
> local_irq_save(save_flags);
>
> @@ -3251,6 +3256,7 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
> if (unlikely(slab_want_init_on_alloc(flags, cachep)) && ptr)
> memset(ptr, 0, cachep->object_size);
>
> +out_hooks:
> slab_post_alloc_hook(cachep, objcg, flags, 1, &ptr);
> return ptr;
> }
> @@ -3288,7 +3294,7 @@ __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
> #endif /* CONFIG_NUMA */
>
> static __always_inline void *
> -slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
> +slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned long caller)
> {
> unsigned long save_flags;
> void *objp;
> @@ -3299,6 +3305,10 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
> if (unlikely(!cachep))
> return NULL;
>
> + objp = kfence_alloc(cachep, orig_size, flags);
> + if (unlikely(objp))
> + goto leave;
> +
> cache_alloc_debugcheck_before(cachep, flags);
> local_irq_save(save_flags);
> objp = __do_cache_alloc(cachep, flags);
> @@ -3309,6 +3319,7 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
> if (unlikely(slab_want_init_on_alloc(flags, cachep)) && objp)
> memset(objp, 0, cachep->object_size);
>
> +leave:
> slab_post_alloc_hook(cachep, objcg, flags, 1, &objp);
> return objp;
> }
> @@ -3414,6 +3425,11 @@ static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
> static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp,
> unsigned long caller)
> {
> + if (kfence_free(objp)) {
> + kmemleak_free_recursive(objp, cachep->flags);
> + return;
> + }
> +
> /* Put the object into the quarantine, don't touch it for now. */
> if (kasan_slab_free(cachep, objp, _RET_IP_))
> return;
> @@ -3479,7 +3495,7 @@ void ___cache_free(struct kmem_cache *cachep, void *objp,
> */
> void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
> {
> - void *ret = slab_alloc(cachep, flags, _RET_IP_);
> + void *ret = slab_alloc(cachep, flags, cachep->object_size, _RET_IP_);
It's kinda minor, but since we are talking about malloc fast path:
will passing 0 instead of cachep->object_size (here and everywhere
else) and then using cachep->object_size on the slow path if 0 is
passed as size improve codegen?
> trace_kmem_cache_alloc(_RET_IP_, ret,
> cachep->object_size, cachep->size, flags);
> @@ -3512,7 +3528,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
>
> local_irq_disable();
> for (i = 0; i < size; i++) {
> - void *objp = __do_cache_alloc(s, flags);
> + void *objp = kfence_alloc(s, s->object_size, flags) ?: __do_cache_alloc(s, flags);
>
> if (unlikely(!objp))
> goto error;
> @@ -3545,7 +3561,7 @@ kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)
> {
> void *ret;
>
> - ret = slab_alloc(cachep, flags, _RET_IP_);
> + ret = slab_alloc(cachep, flags, size, _RET_IP_);
>
> ret = kasan_kmalloc(cachep, ret, size, flags);
> trace_kmalloc(_RET_IP_, ret,
> @@ -3571,7 +3587,7 @@ EXPORT_SYMBOL(kmem_cache_alloc_trace);
> */
> void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
> {
> - void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
> + void *ret = slab_alloc_node(cachep, flags, nodeid, cachep->object_size, _RET_IP_);
>
> trace_kmem_cache_alloc_node(_RET_IP_, ret,
> cachep->object_size, cachep->size,
> @@ -3589,7 +3605,7 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
> {
> void *ret;
>
> - ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
> + ret = slab_alloc_node(cachep, flags, nodeid, size, _RET_IP_);
>
> ret = kasan_kmalloc(cachep, ret, size, flags);
> trace_kmalloc_node(_RET_IP_, ret,
> @@ -3650,7 +3666,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
> cachep = kmalloc_slab(size, flags);
> if (unlikely(ZERO_OR_NULL_PTR(cachep)))
> return cachep;
> - ret = slab_alloc(cachep, flags, caller);
> + ret = slab_alloc(cachep, flags, size, caller);
>
> ret = kasan_kmalloc(cachep, ret, size, flags);
> trace_kmalloc(caller, ret,
> @@ -4138,18 +4154,24 @@ void __check_heap_object(const void *ptr, unsigned long n, struct page *page,
> bool to_user)
> {
> struct kmem_cache *cachep;
> - unsigned int objnr;
> + unsigned int objnr = 0;
> unsigned long offset;
> + bool is_kfence = is_kfence_address(ptr);
>
> ptr = kasan_reset_tag(ptr);
>
> /* Find and validate object. */
> cachep = page->slab_cache;
> - objnr = obj_to_index(cachep, page, (void *)ptr);
> - BUG_ON(objnr >= cachep->num);
> + if (!is_kfence) {
> + objnr = obj_to_index(cachep, page, (void *)ptr);
> + BUG_ON(objnr >= cachep->num);
> + }
>
> /* Find offset within object. */
> - offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep);
> + if (is_kfence_address(ptr))
> + offset = ptr - kfence_object_start(ptr);
> + else
> + offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep);
>
> /* Allow address range falling entirely within usercopy region. */
> if (offset >= cachep->useroffset &&
> diff --git a/mm/slab_common.c b/mm/slab_common.c
> index f9ccd5dc13f3..6e35e273681a 100644
> --- a/mm/slab_common.c
> +++ b/mm/slab_common.c
> @@ -12,6 +12,7 @@
> #include <linux/memory.h>
> #include <linux/cache.h>
> #include <linux/compiler.h>
> +#include <linux/kfence.h>
> #include <linux/module.h>
> #include <linux/cpu.h>
> #include <linux/uaccess.h>
> @@ -448,6 +449,9 @@ static int shutdown_cache(struct kmem_cache *s)
> /* free asan quarantined objects */
> kasan_cache_shutdown(s);
>
> + if (!kfence_shutdown_cache(s))
> + return -EBUSY;
> +
> if (__kmem_cache_shutdown(s) != 0)
> return -EBUSY;
>
> @@ -1171,7 +1175,7 @@ size_t ksize(const void *objp)
> if (unlikely(ZERO_OR_NULL_PTR(objp)) || !__kasan_check_read(objp, 1))
> return 0;
>
> - size = __ksize(objp);
> + size = kfence_ksize(objp) ?: __ksize(objp);
> /*
> * We assume that ksize callers could use whole allocated area,
> * so we need to unpoison this area.
> --
> 2.28.0.526.ge36021eeef-goog
>
next prev parent reply other threads:[~2020-09-11 7:17 UTC|newest]
Thread overview: 55+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-09-07 13:40 [PATCH RFC 00/10] KFENCE: A low-overhead sampling-based memory safety error detector Marco Elver
2020-09-07 13:40 ` [PATCH RFC 01/10] mm: add Kernel Electric-Fence infrastructure Marco Elver
2020-09-07 15:41 ` Jonathan Cameron
2020-09-07 16:38 ` Marco Elver
2020-09-10 14:57 ` Dmitry Vyukov
2020-09-10 15:06 ` Marco Elver
2020-09-10 15:48 ` Dmitry Vyukov
2020-09-10 16:22 ` Marco Elver
2020-09-10 15:42 ` Dmitry Vyukov
2020-09-10 16:19 ` Alexander Potapenko
2020-09-10 17:11 ` Dmitry Vyukov
2020-09-10 17:41 ` Marco Elver
2020-09-10 20:25 ` Paul E. McKenney
2020-09-15 13:57 ` SeongJae Park
2020-09-15 14:14 ` Marco Elver
2020-09-15 14:26 ` SeongJae Park
2020-09-07 13:40 ` [PATCH RFC 02/10] x86, kfence: enable KFENCE for x86 Marco Elver
2020-09-07 13:40 ` [PATCH RFC 03/10] arm64, kfence: enable KFENCE for ARM64 Marco Elver
2020-09-09 15:13 ` Marco Elver
2020-09-07 13:40 ` [PATCH RFC 04/10] mm, kfence: insert KFENCE hooks for SLAB Marco Elver
2020-09-11 7:17 ` Dmitry Vyukov [this message]
2020-09-11 12:24 ` Marco Elver
2020-09-11 13:03 ` Dmitry Vyukov
2020-09-07 13:40 ` [PATCH RFC 05/10] mm, kfence: insert KFENCE hooks for SLUB Marco Elver
2020-09-07 13:40 ` [PATCH RFC 06/10] kfence, kasan: make KFENCE compatible with KASAN Marco Elver
2020-09-11 7:04 ` Dmitry Vyukov
2020-09-11 13:00 ` Marco Elver
2020-09-07 13:40 ` [PATCH RFC 07/10] kfence, kmemleak: make KFENCE compatible with KMEMLEAK Marco Elver
2020-09-08 11:53 ` Catalin Marinas
2020-09-08 12:29 ` Alexander Potapenko
2020-09-07 13:40 ` [PATCH RFC 08/10] kfence, lockdep: make KFENCE compatible with lockdep Marco Elver
2020-09-07 13:40 ` [PATCH RFC 09/10] kfence, Documentation: add KFENCE documentation Marco Elver
2020-09-07 15:33 ` Andrey Konovalov
2020-09-07 16:33 ` Marco Elver
2020-09-07 17:55 ` Andrey Konovalov
2020-09-07 18:16 ` Marco Elver
2020-09-08 15:54 ` Dave Hansen
2020-09-08 16:14 ` Marco Elver
2020-09-11 7:14 ` Dmitry Vyukov
2020-09-11 7:46 ` Marco Elver
2020-09-07 13:40 ` [PATCH RFC 10/10] kfence: add test suite Marco Elver
2020-09-08 11:48 ` [PATCH RFC 00/10] KFENCE: A low-overhead sampling-based memory safety error detector Vlastimil Babka
2020-09-08 12:16 ` Alexander Potapenko
2020-09-08 14:40 ` Vlastimil Babka
2020-09-08 15:21 ` Marco Elver
2020-09-08 14:52 ` Dave Hansen
2020-09-08 15:31 ` Marco Elver
2020-09-08 15:36 ` Vlastimil Babka
2020-09-08 15:56 ` Marco Elver
2020-09-11 7:35 ` Dmitry Vyukov
2020-09-11 12:03 ` Marco Elver
2020-09-11 13:09 ` Dmitry Vyukov
2020-09-11 13:33 ` Marco Elver
2020-09-11 16:33 ` Marco Elver
2020-09-08 15:37 ` Dave Hansen
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=CACT4Y+aXNmQzp6J+mP+ELj8kUHmRPkibc1--KtV9a3ud_X8miw@mail.gmail.com \
--to=dvyukov@google.com \
--cc=akpm@linux-foundation.org \
--cc=andreyknvl@google.com \
--cc=aryabinin@virtuozzo.com \
--cc=bp@alien8.de \
--cc=cai@lca.pw \
--cc=catalin.marinas@arm.com \
--cc=cl@linux.com \
--cc=corbet@lwn.net \
--cc=dave.hansen@linux.intel.com \
--cc=edumazet@google.com \
--cc=elver@google.com \
--cc=glider@google.com \
--cc=gregkh@linuxfoundation.org \
--cc=hpa@zytor.com \
--cc=iamjoonsoo.kim@lge.com \
--cc=jannh@google.com \
--cc=kasan-dev@googlegroups.com \
--cc=keescook@chromium.org \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-doc@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=luto@kernel.org \
--cc=mark.rutland@arm.com \
--cc=mingo@redhat.com \
--cc=paulmck@kernel.org \
--cc=penberg@kernel.org \
--cc=peterz@infradead.org \
--cc=rientjes@google.com \
--cc=tglx@linutronix.de \
--cc=will@kernel.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).