[patch 24/38] module: Add layout for callthunks tracking

Thomas Gleixner
2022-07-16T23:19:02Z

From: Peter Zijlstra <peterz@infradead.org>

Various things will need to be able to tell if a specific address is a
callthunk or not (ORC, BPF, static_call). In order to answer this
question in the face of modules it is necessary to (quickly) find the
module associated with a specific (callthunk) address.

Extend the __module_address() infrastructure with knowledge of the
(per module) callthunk range.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
include/linux/module.h | 21 +++++++++++++++++++--
kernel/module/internal.h | 8 ++++++++
kernel/module/main.c | 6 ++++++
kernel/module/tree_lookup.c | 17 ++++++++++++++++-
4 files changed, 49 insertions(+), 3 deletions(-)

--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -424,6 +424,9 @@ struct module {
/* Core layout: rbtree is accessed frequently, so keep together. */
struct module_layout core_layout __module_layout_align;
struct module_layout init_layout;
+#ifdef CONFIG_CALL_THUNKS
+ struct module_layout thunk_layout;
+#endif
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
struct module_layout data_layout;
#endif
@@ -590,9 +593,23 @@ static inline bool within_module_init(un
addr < (unsigned long)mod->init_layout.base + mod->init_layout.size;
}

-static inline bool within_module(unsigned long addr, const struct module *mod)
+static inline bool within_module_thunk(unsigned long addr,
+ const struct module *mod)
+{
+#ifdef CONFIG_CALL_THUNKS
+ return (unsigned long)mod->thunk_layout.base <= addr &&
+ addr < (unsigned long)mod->thunk_layout.base + mod->thunk_layout.size;
+#else
+ return false;
+#endif
+}
+
+static inline bool within_module(unsigned long addr,
+ const struct module *mod)
{
- return within_module_init(addr, mod) || within_module_core(addr, mod);
+ return within_module_core(addr, mod) ||
+ within_module_thunk(addr, mod) ||
+ within_module_init(addr, mod);
}

/* Search for module by name: must be in a RCU-sched critical section. */
--- a/kernel/module/internal.h
+++ b/kernel/module/internal.h
@@ -219,6 +219,14 @@ static inline struct module *mod_find(un
}
#endif /* CONFIG_MODULES_TREE_LOOKUP */

+#if defined(CONFIG_MODULES_TREE_LOOKUP) && defined(CONFIG_CALL_THUNKS)
+void mod_tree_insert_thunk(struct module *mod);
+void mod_tree_remove_thunk(struct module *mod);
+#else
+static inline void mod_tree_insert_thunk(struct module *mod) { }
+static inline void mod_tree_remove_thunk(struct module *mod) { }
+#endif
+
void module_enable_ro(const struct module *mod, bool after_init);
void module_enable_nx(const struct module *mod);
void module_enable_x(const struct module *mod);
--- a/kernel/module/main.c
+++ b/kernel/module/main.c
@@ -1154,6 +1154,7 @@ static void free_module(struct module *m
*/
mutex_lock(&module_mutex);
mod->state = MODULE_STATE_UNFORMED;
+ mod_tree_remove_thunk(mod);
mutex_unlock(&module_mutex);

/* Remove dynamic debug info */
@@ -2770,6 +2771,10 @@ static int load_module(struct load_info
if (err < 0)
goto free_modinfo;

+ mutex_lock(&module_mutex);
+ mod_tree_insert_thunk(mod);
+ mutex_unlock(&module_mutex);
+
flush_module_icache(mod);

/* Setup CFI for the module. */
@@ -2859,6 +2864,7 @@ static int load_module(struct load_info
mutex_lock(&module_mutex);
/* Unlink carefully: kallsyms could be walking list. */
list_del_rcu(&mod->list);
+ mod_tree_remove_thunk(mod);
mod_tree_remove(mod);
wake_up_all(&module_wq);
/* Wait for RCU-sched synchronizing before releasing mod->list. */
--- a/kernel/module/tree_lookup.c
+++ b/kernel/module/tree_lookup.c
@@ -66,11 +66,26 @@ static noinline void __mod_tree_insert(s
latch_tree_insert(&node->node, &tree->root, &mod_tree_ops);
}

-static void __mod_tree_remove(struct mod_tree_node *node, struct mod_tree_root *tree)
+static noinline void __mod_tree_remove(struct mod_tree_node *node, struct mod_tree_root *tree)
{
latch_tree_erase(&node->node, &tree->root, &mod_tree_ops);
}

+#ifdef CONFIG_CALL_THUNKS
+void mod_tree_insert_thunk(struct module *mod)
+{
+ mod->thunk_layout.mtn.mod = mod;
+ if (mod->thunk_layout.size)
+ __mod_tree_insert(&mod->thunk_layout.mtn, &mod_tree);
+}
+
+void mod_tree_remove_thunk(struct module *mod)
+{
+ if (mod->thunk_layout.size)
+ __mod_tree_remove(&mod->thunk_layout.mtn, &mod_tree);
+}
+#endif
+
/*
* These modifications: insert, remove_init and remove; are serialized by the
* module_mutex.