From mboxrd@z Thu Jan 1 00:00:00 1970 Content-Type: multipart/mixed; boundary="===============4349067265344414832==" MIME-Version: 1.0 From: Aaron Tomlin To: lkp@lists.01.org Subject: [RFC PATCH v3 04/13] module: Move latched RB-tree support to a separate file Date: Fri, 28 Jan 2022 12:50:37 +0000 Message-ID: <20220128125046.497173-5-atomlin@redhat.com> In-Reply-To: <20220128125046.497173-1-atomlin@redhat.com> List-Id: --===============4349067265344414832== Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable No functional change. This patch migrates module latched RB-tree support (e.g. see __module_address()) from core module code into kernel/module/tree_lookup.c. Signed-off-by: Aaron Tomlin --- include/linux/module.h | 37 +++++++++- kernel/module/Makefile | 1 + kernel/module/main.c | 134 ------------------------------------ kernel/module/tree_lookup.c | 108 +++++++++++++++++++++++++++++ 4 files changed, 144 insertions(+), 136 deletions(-) create mode 100644 kernel/module/tree_lookup.c diff --git a/include/linux/module.h b/include/linux/module.h index 8d49f12a7601..377bdb9952cf 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -340,11 +340,44 @@ struct module_layout { }; = #ifdef CONFIG_MODULES_TREE_LOOKUP +struct mod_tree_root { + struct latch_tree_root root; + unsigned long addr_min; + unsigned long addr_max; +} mod_tree __cacheline_aligned =3D { + .addr_min =3D -1UL, +}; + +#define module_addr_min mod_tree.addr_min +#define module_addr_max mod_tree.addr_max + +extern void mod_tree_insert(struct module *mod); +extern void mod_tree_remove_init(struct module *mod); +extern void mod_tree_remove(struct module *mod); +extern struct module *mod_find(unsigned long addr); /* Only touch one cacheline for common rbtree-for-core-layout case. */ #define __module_layout_align ____cacheline_aligned -#else +#else /* !CONFIG_MODULES_TREE_LOOKUP */ + +static unsigned long module_addr_min =3D -1UL, module_addr_max; + +static void mod_tree_insert(struct module *mod) { } +static void mod_tree_remove_init(struct module *mod) { } +static void mod_tree_remove(struct module *mod) { } +static struct module *mod_find(unsigned long addr) +{ + struct module *mod; + + list_for_each_entry_rcu(mod, &modules, list, + lockdep_is_held(&module_mutex)) { + if (within_module(addr, mod)) + return mod; + } + + return NULL; +} #define __module_layout_align -#endif +#endif /* CONFIG_MODULES_TREE_LOOKUP */ = struct mod_kallsyms { Elf_Sym *symtab; diff --git a/kernel/module/Makefile b/kernel/module/Makefile index ba3ebdb7055b..5990d2e4566d 100644 --- a/kernel/module/Makefile +++ b/kernel/module/Makefile @@ -8,3 +8,4 @@ obj-$(CONFIG_MODULE_DECOMPRESS) +=3D decompress.o obj-$(CONFIG_MODULE_SIG) +=3D signing.o obj-$(CONFIG_MODULE_SIG_FORMAT) +=3D signature.o obj-$(CONFIG_LIVEPATCH) +=3D livepatch.o +obj-$(CONFIG_MODULES_TREE_LOOKUP) +=3D tree_lookup.o diff --git a/kernel/module/main.c b/kernel/module/main.c index c91c7e57bca7..5c7c3201b5b4 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -89,140 +89,6 @@ static void do_free_init(struct work_struct *w); static DECLARE_WORK(init_free_wq, do_free_init); static LLIST_HEAD(init_free_list); = -#ifdef CONFIG_MODULES_TREE_LOOKUP - -/* - * Use a latched RB-tree for __module_address(); this allows us to use - * RCU-sched lookups of the address from any context. - * - * This is conditional on PERF_EVENTS || TRACING because those can really = hit - * __module_address() hard by doing a lot of stack unwinding; potentially = from - * NMI context. - */ - -static __always_inline unsigned long __mod_tree_val(struct latch_tree_node= *n) -{ - struct module_layout *layout =3D container_of(n, struct module_layout, mt= n.node); - - return (unsigned long)layout->base; -} - -static __always_inline unsigned long __mod_tree_size(struct latch_tree_nod= e *n) -{ - struct module_layout *layout =3D container_of(n, struct module_layout, mt= n.node); - - return (unsigned long)layout->size; -} - -static __always_inline bool -mod_tree_less(struct latch_tree_node *a, struct latch_tree_node *b) -{ - return __mod_tree_val(a) < __mod_tree_val(b); -} - -static __always_inline int -mod_tree_comp(void *key, struct latch_tree_node *n) -{ - unsigned long val =3D (unsigned long)key; - unsigned long start, end; - - start =3D __mod_tree_val(n); - if (val < start) - return -1; - - end =3D start + __mod_tree_size(n); - if (val >=3D end) - return 1; - - return 0; -} - -static const struct latch_tree_ops mod_tree_ops =3D { - .less =3D mod_tree_less, - .comp =3D mod_tree_comp, -}; - -static struct mod_tree_root { - struct latch_tree_root root; - unsigned long addr_min; - unsigned long addr_max; -} mod_tree __cacheline_aligned =3D { - .addr_min =3D -1UL, -}; - -#define module_addr_min mod_tree.addr_min -#define module_addr_max mod_tree.addr_max - -static noinline void __mod_tree_insert(struct mod_tree_node *node) -{ - latch_tree_insert(&node->node, &mod_tree.root, &mod_tree_ops); -} - -static void __mod_tree_remove(struct mod_tree_node *node) -{ - latch_tree_erase(&node->node, &mod_tree.root, &mod_tree_ops); -} - -/* - * These modifications: insert, remove_init and remove; are serialized by = the - * module_mutex. - */ -static void mod_tree_insert(struct module *mod) -{ - mod->core_layout.mtn.mod =3D mod; - mod->init_layout.mtn.mod =3D mod; - - __mod_tree_insert(&mod->core_layout.mtn); - if (mod->init_layout.size) - __mod_tree_insert(&mod->init_layout.mtn); -} - -static void mod_tree_remove_init(struct module *mod) -{ - if (mod->init_layout.size) - __mod_tree_remove(&mod->init_layout.mtn); -} - -static void mod_tree_remove(struct module *mod) -{ - __mod_tree_remove(&mod->core_layout.mtn); - mod_tree_remove_init(mod); -} - -static struct module *mod_find(unsigned long addr) -{ - struct latch_tree_node *ltn; - - ltn =3D latch_tree_find((void *)addr, &mod_tree.root, &mod_tree_ops); - if (!ltn) - return NULL; - - return container_of(ltn, struct mod_tree_node, node)->mod; -} - -#else /* MODULES_TREE_LOOKUP */ - -static unsigned long module_addr_min =3D -1UL, module_addr_max =3D 0; - -static void mod_tree_insert(struct module *mod) { } -static void mod_tree_remove_init(struct module *mod) { } -static void mod_tree_remove(struct module *mod) { } - -static struct module *mod_find(unsigned long addr) -{ - struct module *mod; - - list_for_each_entry_rcu(mod, &modules, list, - lockdep_is_held(&module_mutex)) { - if (within_module(addr, mod)) - return mod; - } - - return NULL; -} - -#endif /* MODULES_TREE_LOOKUP */ - /* * Bounds of module text, for speeding up __module_address. * Protected by module_mutex. diff --git a/kernel/module/tree_lookup.c b/kernel/module/tree_lookup.c new file mode 100644 index 000000000000..5f6d6de2dc60 --- /dev/null +++ b/kernel/module/tree_lookup.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Modules tree lookup + * + * Copyright (C) 2015 Peter Zijlstra + * Copyright (C) 2015 Rusty Russell + */ + +#include +#include + +/* + * Use a latched RB-tree for __module_address(); this allows us to use + * RCU-sched lookups of the address from any context. + * + * This is conditional on PERF_EVENTS || TRACING because those can really = hit + * __module_address() hard by doing a lot of stack unwinding; potentially = from + * NMI context. + */ + +__always_inline unsigned long __mod_tree_val(struct latch_tree_node *n) +{ + struct module_layout *layout =3D container_of(n, struct module_layout, mt= n.node); + + return (unsigned long)layout->base; +} + +__always_inline unsigned long __mod_tree_size(struct latch_tree_node *n) +{ + struct module_layout *layout =3D container_of(n, struct module_layout, mt= n.node); + + return (unsigned long)layout->size; +} + +__always_inline bool +mod_tree_less(struct latch_tree_node *a, struct latch_tree_node *b) +{ + return __mod_tree_val(a) < __mod_tree_val(b); +} + +__always_inline int +mod_tree_comp(void *key, struct latch_tree_node *n) +{ + unsigned long val =3D (unsigned long)key; + unsigned long start, end; + + start =3D __mod_tree_val(n); + if (val < start) + return -1; + + end =3D start + __mod_tree_size(n); + if (val >=3D end) + return 1; + + return 0; +} + +const struct latch_tree_ops mod_tree_ops =3D { + .less =3D mod_tree_less, + .comp =3D mod_tree_comp, +}; + +noinline void __mod_tree_insert(struct mod_tree_node *node) +{ + latch_tree_insert(&node->node, &mod_tree.root, &mod_tree_ops); +} + +void __mod_tree_remove(struct mod_tree_node *node) +{ + latch_tree_erase(&node->node, &mod_tree.root, &mod_tree_ops); +} + +/* + * These modifications: insert, remove_init and remove; are serialized by = the + * module_mutex. + */ +void mod_tree_insert(struct module *mod) +{ + mod->core_layout.mtn.mod =3D mod; + mod->init_layout.mtn.mod =3D mod; + + __mod_tree_insert(&mod->core_layout.mtn); + if (mod->init_layout.size) + __mod_tree_insert(&mod->init_layout.mtn); +} + +void mod_tree_remove_init(struct module *mod) +{ + if (mod->init_layout.size) + __mod_tree_remove(&mod->init_layout.mtn); +} + +void mod_tree_remove(struct module *mod) +{ + __mod_tree_remove(&mod->core_layout.mtn); + mod_tree_remove_init(mod); +} + +struct module *mod_find(unsigned long addr) +{ + struct latch_tree_node *ltn; + + ltn =3D latch_tree_find((void *)addr, &mod_tree.root, &mod_tree_ops); + if (!ltn) + return NULL; + + return container_of(ltn, struct mod_tree_node, node)->mod; +} -- = 2.34.1 --===============4349067265344414832==--