* Re: [PATCH 3/4] mm: convert compaction to zone lock wrappers
From: Cheatham, Benjamin @ 2026-02-20 19:10 UTC (permalink / raw)
To: Dmitry Ilvokhin
Cc: linux-kernel, linux-mm, linux-trace-kernel, linux-cxl,
kernel-team, Andrew Morton, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Steven Rostedt,
Masami Hiramatsu, Mathieu Desnoyers, Brendan Jackman,
Johannes Weiner, Zi Yan, Oscar Salvador, Qi Zheng, Shakeel Butt,
Axel Rasmussen, Yuanchu Xie, Wei Xu
In-Reply-To: <3462b7fd26123c69ccdd121a894da14bbfafdd9d.1770821420.git.d@ilvokhin.com>
On 2/11/2026 9:22 AM, Dmitry Ilvokhin wrote:
> Compaction uses compact_lock_irqsave(), which currently operates
> on a raw spinlock_t pointer so that it can be used for both
> zone->lock and lru_lock. Since zone lock operations are now wrapped,
> compact_lock_irqsave() can no longer operate directly on a spinlock_t
> when the lock belongs to a zone.
>
> Introduce struct compact_lock to abstract the underlying lock type. The
> structure carries a lock type enum and a union holding either a zone
> pointer or a raw spinlock_t pointer, and dispatches to the appropriate
> lock/unlock helper.
>
> No functional change intended.
>
> Signed-off-by: Dmitry Ilvokhin <d@ilvokhin.com>
> ---
> mm/compaction.c | 108 +++++++++++++++++++++++++++++++++++++++---------
> 1 file changed, 89 insertions(+), 19 deletions(-)
>
> diff --git a/mm/compaction.c b/mm/compaction.c
> index 1e8f8eca318c..1b000d2b95b2 100644
> --- a/mm/compaction.c
> +++ b/mm/compaction.c
> @@ -24,6 +24,7 @@
> #include <linux/page_owner.h>
> #include <linux/psi.h>
> #include <linux/cpuset.h>
> +#include <linux/zone_lock.h>
> #include "internal.h"
>
> #ifdef CONFIG_COMPACTION
> @@ -493,6 +494,65 @@ static bool test_and_set_skip(struct compact_control *cc, struct page *page)
> }
> #endif /* CONFIG_COMPACTION */
>
> +enum compact_lock_type {
> + COMPACT_LOCK_ZONE,
> + COMPACT_LOCK_RAW_SPINLOCK,
> +};
> +
> +struct compact_lock {
> + enum compact_lock_type type;
> + union {
> + struct zone *zone;
> + spinlock_t *lock; /* Reference to lru lock */
> + };
> +};
> +
> +static bool compact_do_zone_trylock_irqsave(struct zone *zone,
> + unsigned long *flags)
> +{
> + return zone_trylock_irqsave(zone, *flags);
> +}
> +
> +static bool compact_do_raw_trylock_irqsave(spinlock_t *lock,
> + unsigned long *flags)
> +{
> + return spin_trylock_irqsave(lock, *flags);
> +}
> +
> +static bool compact_do_trylock_irqsave(struct compact_lock lock,
> + unsigned long *flags)
> +{
> + if (lock.type == COMPACT_LOCK_ZONE)
> + return compact_do_zone_trylock_irqsave(lock.zone, flags);
> +
> + return compact_do_raw_trylock_irqsave(lock.lock, flags);
> +}
Nit: You could remove the helpers above and just do the calls directly in this function, though
it would remove the parity with the compact helpers. compact_do_lock_irqsave() helpers can stay
since they have the __acquires() annotations.
> +
> +static void compact_do_zone_lock_irqsave(struct zone *zone,
> + unsigned long *flags)
> +__acquires(zone->lock)
> +{
> + zone_lock_irqsave(zone, *flags);
> +}
> +
> +static void compact_do_raw_lock_irqsave(spinlock_t *lock,
> + unsigned long *flags)
> +__acquires(lock)
> +{
> + spin_lock_irqsave(lock, *flags);
> +}
> +
> +static void compact_do_lock_irqsave(struct compact_lock lock,
> + unsigned long *flags)
> +{
> + if (lock.type == COMPACT_LOCK_ZONE) {
> + compact_do_zone_lock_irqsave(lock.zone, flags);
> + return;
> + }
> +
> + return compact_do_raw_lock_irqsave(lock.lock, flags);
You don't need the return statement here (and you shouldn't be returning a value at all).
It may be cleaner to just do an if-else statement here instead.
> +}
> +
> /*
> * Compaction requires the taking of some coarse locks that are potentially
> * very heavily contended. For async compaction, trylock and record if the
> @@ -502,19 +562,19 @@ static bool test_and_set_skip(struct compact_control *cc, struct page *page)
> *
> * Always returns true which makes it easier to track lock state in callers.
> */
> -static bool compact_lock_irqsave(spinlock_t *lock, unsigned long *flags,
> - struct compact_control *cc)
> - __acquires(lock)
> +static bool compact_lock_irqsave(struct compact_lock lock,
> + unsigned long *flags,
> + struct compact_control *cc)
> {
> /* Track if the lock is contended in async mode */
> if (cc->mode == MIGRATE_ASYNC && !cc->contended) {
> - if (spin_trylock_irqsave(lock, *flags))
> + if (compact_do_trylock_irqsave(lock, flags))
> return true;
>
> cc->contended = true;
> }
>
> - spin_lock_irqsave(lock, *flags);
> + compact_do_lock_irqsave(lock, flags);
> return true;
> }
>
> @@ -530,11 +590,13 @@ static bool compact_lock_irqsave(spinlock_t *lock, unsigned long *flags,
> * Returns true if compaction should abort due to fatal signal pending.
> * Returns false when compaction can continue.
> */
> -static bool compact_unlock_should_abort(spinlock_t *lock,
> - unsigned long flags, bool *locked, struct compact_control *cc)
> +static bool compact_unlock_should_abort(struct zone *zone,
> + unsigned long flags,
> + bool *locked,
> + struct compact_control *cc)
> {
> if (*locked) {
> - spin_unlock_irqrestore(lock, flags);
> + zone_unlock_irqrestore(zone, flags);
I would move this (and other wrapper changes below that don't use compact_*) to the last patch. I understand you
didn't change it due to location but I would argue it isn't really relevant to what's being added in this patch
and fits better in the last.
> *locked = false;
> }
>
> @@ -582,9 +644,8 @@ static unsigned long isolate_freepages_block(struct compact_control *cc,
> * contention, to give chance to IRQs. Abort if fatal signal
> * pending.
> */
> - if (!(blockpfn % COMPACT_CLUSTER_MAX)
> - && compact_unlock_should_abort(&cc->zone->lock, flags,
> - &locked, cc))
> + if (!(blockpfn % COMPACT_CLUSTER_MAX) &&
> + compact_unlock_should_abort(cc->zone, flags, &locked, cc))
> break;
>
> nr_scanned++;
> @@ -613,8 +674,12 @@ static unsigned long isolate_freepages_block(struct compact_control *cc,
>
> /* If we already hold the lock, we can skip some rechecking. */
> if (!locked) {
> - locked = compact_lock_irqsave(&cc->zone->lock,
> - &flags, cc);
> + struct compact_lock zol = {
> + .type = COMPACT_LOCK_ZONE,
> + .zone = cc->zone,
> + };
> +
> + locked = compact_lock_irqsave(zol, &flags, cc);
>
> /* Recheck this is a buddy page under lock */
> if (!PageBuddy(page))
> @@ -649,7 +714,7 @@ static unsigned long isolate_freepages_block(struct compact_control *cc,
> }
>
> if (locked)
> - spin_unlock_irqrestore(&cc->zone->lock, flags);
> + zone_unlock_irqrestore(cc->zone, flags);
>
> /*
> * Be careful to not go outside of the pageblock.
> @@ -1157,10 +1222,15 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
>
> /* If we already hold the lock, we can skip some rechecking */
> if (lruvec != locked) {
> + struct compact_lock zol = {
> + .type = COMPACT_LOCK_RAW_SPINLOCK,
> + .lock = &lruvec->lru_lock,
> + };
> +
> if (locked)
> unlock_page_lruvec_irqrestore(locked, flags);
>
> - compact_lock_irqsave(&lruvec->lru_lock, &flags, cc);
> + compact_lock_irqsave(zol, &flags, cc);
> locked = lruvec;
>
> lruvec_memcg_debug(lruvec, folio);
> @@ -1555,7 +1625,7 @@ static void fast_isolate_freepages(struct compact_control *cc)
> if (!area->nr_free)
> continue;
>
> - spin_lock_irqsave(&cc->zone->lock, flags);
> + zone_lock_irqsave(cc->zone, flags);
> freelist = &area->free_list[MIGRATE_MOVABLE];
> list_for_each_entry_reverse(freepage, freelist, buddy_list) {
> unsigned long pfn;
> @@ -1614,7 +1684,7 @@ static void fast_isolate_freepages(struct compact_control *cc)
> }
> }
>
> - spin_unlock_irqrestore(&cc->zone->lock, flags);
> + zone_unlock_irqrestore(cc->zone, flags);
>
> /* Skip fast search if enough freepages isolated */
> if (cc->nr_freepages >= cc->nr_migratepages)
> @@ -1988,7 +2058,7 @@ static unsigned long fast_find_migrateblock(struct compact_control *cc)
> if (!area->nr_free)
> continue;
>
> - spin_lock_irqsave(&cc->zone->lock, flags);
> + zone_lock_irqsave(cc->zone, flags);
> freelist = &area->free_list[MIGRATE_MOVABLE];
> list_for_each_entry(freepage, freelist, buddy_list) {
> unsigned long free_pfn;
> @@ -2021,7 +2091,7 @@ static unsigned long fast_find_migrateblock(struct compact_control *cc)
> break;
> }
> }
> - spin_unlock_irqrestore(&cc->zone->lock, flags);
> + zone_unlock_irqrestore(cc->zone, flags);
> }
>
> cc->total_migrate_scanned += nr_scanned;
^ permalink raw reply
* Re: [PATCH bpf-next 04/17] bpf: Add struct bpf_tramp_node object
From: kernel test robot @ 2026-02-20 19:52 UTC (permalink / raw)
To: Jiri Olsa, Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: llvm, oe-kbuild-all, bpf, linux-trace-kernel, Martin KaFai Lau,
Eduard Zingerman, Song Liu, Yonghong Song, Menglong Dong,
Steven Rostedt
In-Reply-To: <20260220100649.628307-5-jolsa@kernel.org>
Hi Jiri,
kernel test robot noticed the following build errors:
[auto build test ERROR on bpf-next/master]
url: https://github.com/intel-lab-lkp/linux/commits/Jiri-Olsa/ftrace-Add-ftrace_hash_count-function/20260220-181324
base: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master
patch link: https://lore.kernel.org/r/20260220100649.628307-5-jolsa%40kernel.org
patch subject: [PATCH bpf-next 04/17] bpf: Add struct bpf_tramp_node object
config: riscv-allyesconfig (https://download.01.org/0day-ci/archive/20260221/202602210330.ukNZdClO-lkp@intel.com/config)
compiler: clang version 16.0.6 (https://github.com/llvm/llvm-project 7cbf1a2591520c2491aa35339f227775f4d3adf6)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260221/202602210330.ukNZdClO-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202602210330.ukNZdClO-lkp@intel.com/
All errors (new ones prefixed by >>):
arch/riscv/net/bpf_jit_comp64.c:944:9: error: no member named 'cookie' in 'struct bpf_tramp_link'
if (l->cookie)
~ ^
arch/riscv/net/bpf_jit_comp64.c:945:67: error: no member named 'cookie' in 'struct bpf_tramp_link'
emit_store_stack_imm64(RV_REG_T1, -run_ctx_off + cookie_off, l->cookie, ctx);
~ ^
arch/riscv/net/bpf_jit_comp64.c:999:30: warning: declaration of 'struct bpf_tramp_links' will not be visible outside of this function [-Wvisibility]
static int invoke_bpf(struct bpf_tramp_links *tl, int args_off, int retval_off,
^
arch/riscv/net/bpf_jit_comp64.c:1005:20: error: incomplete definition of type 'struct bpf_tramp_links'
for (i = 0; i < tl->nr_links; i++) {
~~^
arch/riscv/net/bpf_jit_comp64.c:999:30: note: forward declaration of 'struct bpf_tramp_links'
static int invoke_bpf(struct bpf_tramp_links *tl, int args_off, int retval_off,
^
arch/riscv/net/bpf_jit_comp64.c:1008:39: error: incomplete definition of type 'struct bpf_tramp_links'
if (bpf_prog_calls_session_cookie(tl->links[i])) {
~~^
arch/riscv/net/bpf_jit_comp64.c:999:30: note: forward declaration of 'struct bpf_tramp_links'
static int invoke_bpf(struct bpf_tramp_links *tl, int args_off, int retval_off,
^
arch/riscv/net/bpf_jit_comp64.c:1014:27: error: incomplete definition of type 'struct bpf_tramp_links'
err = invoke_bpf_prog(tl->links[i], args_off, retval_off, run_ctx_off,
~~^
arch/riscv/net/bpf_jit_comp64.c:999:30: note: forward declaration of 'struct bpf_tramp_links'
static int invoke_bpf(struct bpf_tramp_links *tl, int args_off, int retval_off,
^
arch/riscv/net/bpf_jit_comp64.c:1024:14: warning: declaration of 'struct bpf_tramp_links' will not be visible outside of this function [-Wvisibility]
struct bpf_tramp_links *tlinks,
^
arch/riscv/net/bpf_jit_comp64.c:1033:42: error: subscript of pointer to incomplete type 'struct bpf_tramp_links'
struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
~~~~~~^
arch/riscv/net/bpf_jit_comp64.c:1024:14: note: forward declaration of 'struct bpf_tramp_links'
struct bpf_tramp_links *tlinks,
^
arch/riscv/net/bpf_jit_comp64.c:1034:41: error: subscript of pointer to incomplete type 'struct bpf_tramp_links'
struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
~~~~~~^
arch/riscv/net/bpf_jit_comp64.c:1024:14: note: forward declaration of 'struct bpf_tramp_links'
struct bpf_tramp_links *tlinks,
^
arch/riscv/net/bpf_jit_comp64.c:1035:44: error: subscript of pointer to incomplete type 'struct bpf_tramp_links'
struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
~~~~~~^
arch/riscv/net/bpf_jit_comp64.c:1024:14: note: forward declaration of 'struct bpf_tramp_links'
struct bpf_tramp_links *tlinks,
^
>> arch/riscv/net/bpf_jit_comp64.c:1118:39: error: incompatible pointer types passing 'struct bpf_tramp_links *' to parameter of type 'struct bpf_tramp_nodes *' [-Werror,-Wincompatible-pointer-types]
cookie_cnt = bpf_fsession_cookie_cnt(tlinks);
^~~~~~
include/linux/bpf.h:2207:67: note: passing argument to parameter 'nodes' here
static inline int bpf_fsession_cookie_cnt(struct bpf_tramp_nodes *nodes)
^
arch/riscv/net/bpf_jit_comp64.c:1175:23: error: incompatible pointer types passing 'struct bpf_tramp_links *' to parameter of type 'struct bpf_tramp_nodes *' [-Werror,-Wincompatible-pointer-types]
if (bpf_fsession_cnt(tlinks)) {
^~~~~~
include/linux/bpf.h:2189:60: note: passing argument to parameter 'nodes' here
static inline int bpf_fsession_cnt(struct bpf_tramp_nodes *nodes)
^
arch/riscv/net/bpf_jit_comp64.c:1190:12: error: incomplete definition of type 'struct bpf_tramp_links'
if (fentry->nr_links) {
~~~~~~^
arch/riscv/net/bpf_jit_comp64.c:1024:14: note: forward declaration of 'struct bpf_tramp_links'
struct bpf_tramp_links *tlinks,
^
arch/riscv/net/bpf_jit_comp64.c:1191:20: error: incompatible pointer types passing 'struct bpf_tramp_links *' to parameter of type 'struct bpf_tramp_links *' [-Werror,-Wincompatible-pointer-types]
ret = invoke_bpf(fentry, args_off, retval_off, run_ctx_off, func_meta_off,
^~~~~~
arch/riscv/net/bpf_jit_comp64.c:999:47: note: passing argument to parameter 'tl' here
static int invoke_bpf(struct bpf_tramp_links *tl, int args_off, int retval_off,
^
arch/riscv/net/bpf_jit_comp64.c:1197:14: error: incomplete definition of type 'struct bpf_tramp_links'
if (fmod_ret->nr_links) {
~~~~~~~~^
arch/riscv/net/bpf_jit_comp64.c:1024:14: note: forward declaration of 'struct bpf_tramp_links'
struct bpf_tramp_links *tlinks,
^
arch/riscv/net/bpf_jit_comp64.c:1198:34: error: incomplete definition of type 'struct bpf_tramp_links'
branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL);
~~~~~~~~^
include/linux/slab.h:1154:48: note: expanded from macro 'kcalloc'
#define kcalloc(n, size, flags) kmalloc_array(n, size, (flags) | __GFP_ZERO)
^
include/linux/slab.h:1115:63: note: expanded from macro 'kmalloc_array'
#define kmalloc_array(...) alloc_hooks(kmalloc_array_noprof(__VA_ARGS__))
^~~~~~~~~~~
include/linux/alloc_tag.h:265:31: note: expanded from macro 'alloc_hooks'
alloc_hooks_tag(&_alloc_tag, _do_alloc); \
^~~~~~~~~
include/linux/alloc_tag.h:251:9: note: expanded from macro 'alloc_hooks_tag'
typeof(_do_alloc) _res; \
^~~~~~~~~
arch/riscv/net/bpf_jit_comp64.c:1024:14: note: forward declaration of 'struct bpf_tramp_links'
struct bpf_tramp_links *tlinks,
^
arch/riscv/net/bpf_jit_comp64.c:1198:34: error: incomplete definition of type 'struct bpf_tramp_links'
branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL);
~~~~~~~~^
include/linux/slab.h:1154:48: note: expanded from macro 'kcalloc'
#define kcalloc(n, size, flags) kmalloc_array(n, size, (flags) | __GFP_ZERO)
^
include/linux/slab.h:1115:63: note: expanded from macro 'kmalloc_array'
#define kmalloc_array(...) alloc_hooks(kmalloc_array_noprof(__VA_ARGS__))
^~~~~~~~~~~
include/linux/alloc_tag.h:265:31: note: expanded from macro 'alloc_hooks'
alloc_hooks_tag(&_alloc_tag, _do_alloc); \
^~~~~~~~~
include/linux/alloc_tag.h:255:10: note: expanded from macro 'alloc_hooks_tag'
_res = _do_alloc; \
^~~~~~~~~
arch/riscv/net/bpf_jit_comp64.c:1024:14: note: forward declaration of 'struct bpf_tramp_links'
struct bpf_tramp_links *tlinks,
^
arch/riscv/net/bpf_jit_comp64.c:1198:34: error: incomplete definition of type 'struct bpf_tramp_links'
branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL);
~~~~~~~~^
include/linux/slab.h:1154:48: note: expanded from macro 'kcalloc'
#define kcalloc(n, size, flags) kmalloc_array(n, size, (flags) | __GFP_ZERO)
^
include/linux/slab.h:1115:63: note: expanded from macro 'kmalloc_array'
#define kmalloc_array(...) alloc_hooks(kmalloc_array_noprof(__VA_ARGS__))
^~~~~~~~~~~
include/linux/alloc_tag.h:265:31: note: expanded from macro 'alloc_hooks'
alloc_hooks_tag(&_alloc_tag, _do_alloc); \
^~~~~~~~~
include/linux/alloc_tag.h:258:10: note: expanded from macro 'alloc_hooks_tag'
_res = _do_alloc; \
^~~~~~~~~
arch/riscv/net/bpf_jit_comp64.c:1024:14: note: forward declaration of 'struct bpf_tramp_links'
struct bpf_tramp_links *tlinks,
^
arch/riscv/net/bpf_jit_comp64.c:1204:27: error: incomplete definition of type 'struct bpf_tramp_links'
for (i = 0; i < fmod_ret->nr_links; i++) {
~~~~~~~~^
arch/riscv/net/bpf_jit_comp64.c:1024:14: note: forward declaration of 'struct bpf_tramp_links'
struct bpf_tramp_links *tlinks,
^
arch/riscv/net/bpf_jit_comp64.c:1205:34: error: incomplete definition of type 'struct bpf_tramp_links'
ret = invoke_bpf_prog(fmod_ret->links[i], args_off, retval_off,
~~~~~~~~^
arch/riscv/net/bpf_jit_comp64.c:1024:14: note: forward declaration of 'struct bpf_tramp_links'
struct bpf_tramp_links *tlinks,
^
arch/riscv/net/bpf_jit_comp64.c:1233:40: error: incomplete definition of type 'struct bpf_tramp_links'
for (i = 0; ctx->insns && i < fmod_ret->nr_links; i++) {
~~~~~~~~^
arch/riscv/net/bpf_jit_comp64.c:1024:14: note: forward declaration of 'struct bpf_tramp_links'
struct bpf_tramp_links *tlinks,
vim +1118 arch/riscv/net/bpf_jit_comp64.c
35b3515be0ecb9 Menglong Dong 2026-02-08 1021
49b5e77ae3e214 Pu Lehui 2023-02-15 1022 static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
49b5e77ae3e214 Pu Lehui 2023-02-15 1023 const struct btf_func_model *m,
49b5e77ae3e214 Pu Lehui 2023-02-15 1024 struct bpf_tramp_links *tlinks,
49b5e77ae3e214 Pu Lehui 2023-02-15 1025 void *func_addr, u32 flags,
49b5e77ae3e214 Pu Lehui 2023-02-15 1026 struct rv_jit_context *ctx)
49b5e77ae3e214 Pu Lehui 2023-02-15 1027 {
49b5e77ae3e214 Pu Lehui 2023-02-15 1028 int i, ret, offset;
49b5e77ae3e214 Pu Lehui 2023-02-15 1029 int *branches_off = NULL;
6801b0aef79db4 Pu Lehui 2024-07-02 1030 int stack_size = 0, nr_arg_slots = 0;
35b3515be0ecb9 Menglong Dong 2026-02-08 1031 int retval_off, args_off, func_meta_off, ip_off, run_ctx_off, sreg_off, stk_arg_off;
35b3515be0ecb9 Menglong Dong 2026-02-08 1032 int cookie_off, cookie_cnt;
49b5e77ae3e214 Pu Lehui 2023-02-15 1033 struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
49b5e77ae3e214 Pu Lehui 2023-02-15 1034 struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
49b5e77ae3e214 Pu Lehui 2023-02-15 1035 struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
1732ebc4a26181 Pu Lehui 2024-01-23 1036 bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT;
49b5e77ae3e214 Pu Lehui 2023-02-15 1037 void *orig_call = func_addr;
49b5e77ae3e214 Pu Lehui 2023-02-15 1038 bool save_ret;
35b3515be0ecb9 Menglong Dong 2026-02-08 1039 u64 func_meta;
49b5e77ae3e214 Pu Lehui 2023-02-15 1040 u32 insn;
49b5e77ae3e214 Pu Lehui 2023-02-15 1041
25ad10658dc106 Pu Lehui 2023-07-21 1042 /* Two types of generated trampoline stack layout:
25ad10658dc106 Pu Lehui 2023-07-21 1043 *
25ad10658dc106 Pu Lehui 2023-07-21 1044 * 1. trampoline called from function entry
25ad10658dc106 Pu Lehui 2023-07-21 1045 * --------------------------------------
25ad10658dc106 Pu Lehui 2023-07-21 1046 * FP + 8 [ RA to parent func ] return address to parent
25ad10658dc106 Pu Lehui 2023-07-21 1047 * function
25ad10658dc106 Pu Lehui 2023-07-21 1048 * FP + 0 [ FP of parent func ] frame pointer of parent
25ad10658dc106 Pu Lehui 2023-07-21 1049 * function
25ad10658dc106 Pu Lehui 2023-07-21 1050 * FP - 8 [ T0 to traced func ] return address of traced
25ad10658dc106 Pu Lehui 2023-07-21 1051 * function
25ad10658dc106 Pu Lehui 2023-07-21 1052 * FP - 16 [ FP of traced func ] frame pointer of traced
25ad10658dc106 Pu Lehui 2023-07-21 1053 * function
25ad10658dc106 Pu Lehui 2023-07-21 1054 * --------------------------------------
49b5e77ae3e214 Pu Lehui 2023-02-15 1055 *
25ad10658dc106 Pu Lehui 2023-07-21 1056 * 2. trampoline called directly
25ad10658dc106 Pu Lehui 2023-07-21 1057 * --------------------------------------
25ad10658dc106 Pu Lehui 2023-07-21 1058 * FP - 8 [ RA to caller func ] return address to caller
49b5e77ae3e214 Pu Lehui 2023-02-15 1059 * function
25ad10658dc106 Pu Lehui 2023-07-21 1060 * FP - 16 [ FP of caller func ] frame pointer of caller
49b5e77ae3e214 Pu Lehui 2023-02-15 1061 * function
25ad10658dc106 Pu Lehui 2023-07-21 1062 * --------------------------------------
49b5e77ae3e214 Pu Lehui 2023-02-15 1063 *
49b5e77ae3e214 Pu Lehui 2023-02-15 1064 * FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or
49b5e77ae3e214 Pu Lehui 2023-02-15 1065 * BPF_TRAMP_F_RET_FENTRY_RET
49b5e77ae3e214 Pu Lehui 2023-02-15 1066 * [ argN ]
49b5e77ae3e214 Pu Lehui 2023-02-15 1067 * [ ... ]
49b5e77ae3e214 Pu Lehui 2023-02-15 1068 * FP - args_off [ arg1 ]
49b5e77ae3e214 Pu Lehui 2023-02-15 1069 *
35b3515be0ecb9 Menglong Dong 2026-02-08 1070 * FP - func_meta_off [ regs count, etc ]
49b5e77ae3e214 Pu Lehui 2023-02-15 1071 *
49b5e77ae3e214 Pu Lehui 2023-02-15 1072 * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG
49b5e77ae3e214 Pu Lehui 2023-02-15 1073 *
35b3515be0ecb9 Menglong Dong 2026-02-08 1074 * [ stack cookie N ]
35b3515be0ecb9 Menglong Dong 2026-02-08 1075 * [ ... ]
35b3515be0ecb9 Menglong Dong 2026-02-08 1076 * FP - cookie_off [ stack cookie 1 ]
35b3515be0ecb9 Menglong Dong 2026-02-08 1077 *
49b5e77ae3e214 Pu Lehui 2023-02-15 1078 * FP - run_ctx_off [ bpf_tramp_run_ctx ]
49b5e77ae3e214 Pu Lehui 2023-02-15 1079 *
49b5e77ae3e214 Pu Lehui 2023-02-15 1080 * FP - sreg_off [ callee saved reg ]
49b5e77ae3e214 Pu Lehui 2023-02-15 1081 *
49b5e77ae3e214 Pu Lehui 2023-02-15 1082 * [ pads ] pads for 16 bytes alignment
6801b0aef79db4 Pu Lehui 2024-07-02 1083 *
6801b0aef79db4 Pu Lehui 2024-07-02 1084 * [ stack_argN ]
6801b0aef79db4 Pu Lehui 2024-07-02 1085 * [ ... ]
6801b0aef79db4 Pu Lehui 2024-07-02 1086 * FP - stk_arg_off [ stack_arg1 ] BPF_TRAMP_F_CALL_ORIG
49b5e77ae3e214 Pu Lehui 2023-02-15 1087 */
49b5e77ae3e214 Pu Lehui 2023-02-15 1088
49b5e77ae3e214 Pu Lehui 2023-02-15 1089 if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY))
49b5e77ae3e214 Pu Lehui 2023-02-15 1090 return -ENOTSUPP;
49b5e77ae3e214 Pu Lehui 2023-02-15 1091
6801b0aef79db4 Pu Lehui 2024-07-02 1092 if (m->nr_args > MAX_BPF_FUNC_ARGS)
49b5e77ae3e214 Pu Lehui 2023-02-15 1093 return -ENOTSUPP;
49b5e77ae3e214 Pu Lehui 2023-02-15 1094
6801b0aef79db4 Pu Lehui 2024-07-02 1095 for (i = 0; i < m->nr_args; i++)
6801b0aef79db4 Pu Lehui 2024-07-02 1096 nr_arg_slots += round_up(m->arg_size[i], 8) / 8;
6801b0aef79db4 Pu Lehui 2024-07-02 1097
25ad10658dc106 Pu Lehui 2023-07-21 1098 /* room of trampoline frame to store return address and frame pointer */
25ad10658dc106 Pu Lehui 2023-07-21 1099 stack_size += 16;
49b5e77ae3e214 Pu Lehui 2023-02-15 1100
49b5e77ae3e214 Pu Lehui 2023-02-15 1101 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
d0bf7cd5df1846 Chenghao Duan 2025-09-22 1102 if (save_ret)
7112cd26e606c7 Björn Töpel 2023-10-04 1103 stack_size += 16; /* Save both A5 (BPF R0) and A0 */
49b5e77ae3e214 Pu Lehui 2023-02-15 1104 retval_off = stack_size;
49b5e77ae3e214 Pu Lehui 2023-02-15 1105
6801b0aef79db4 Pu Lehui 2024-07-02 1106 stack_size += nr_arg_slots * 8;
49b5e77ae3e214 Pu Lehui 2023-02-15 1107 args_off = stack_size;
49b5e77ae3e214 Pu Lehui 2023-02-15 1108
35b3515be0ecb9 Menglong Dong 2026-02-08 1109 /* function metadata, such as regs count */
49b5e77ae3e214 Pu Lehui 2023-02-15 1110 stack_size += 8;
35b3515be0ecb9 Menglong Dong 2026-02-08 1111 func_meta_off = stack_size;
49b5e77ae3e214 Pu Lehui 2023-02-15 1112
49b5e77ae3e214 Pu Lehui 2023-02-15 1113 if (flags & BPF_TRAMP_F_IP_ARG) {
49b5e77ae3e214 Pu Lehui 2023-02-15 1114 stack_size += 8;
49b5e77ae3e214 Pu Lehui 2023-02-15 1115 ip_off = stack_size;
49b5e77ae3e214 Pu Lehui 2023-02-15 1116 }
49b5e77ae3e214 Pu Lehui 2023-02-15 1117
35b3515be0ecb9 Menglong Dong 2026-02-08 @1118 cookie_cnt = bpf_fsession_cookie_cnt(tlinks);
35b3515be0ecb9 Menglong Dong 2026-02-08 1119 /* room for session cookies */
35b3515be0ecb9 Menglong Dong 2026-02-08 1120 stack_size += cookie_cnt * 8;
35b3515be0ecb9 Menglong Dong 2026-02-08 1121 cookie_off = stack_size;
35b3515be0ecb9 Menglong Dong 2026-02-08 1122
49b5e77ae3e214 Pu Lehui 2023-02-15 1123 stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8);
49b5e77ae3e214 Pu Lehui 2023-02-15 1124 run_ctx_off = stack_size;
49b5e77ae3e214 Pu Lehui 2023-02-15 1125
49b5e77ae3e214 Pu Lehui 2023-02-15 1126 stack_size += 8;
49b5e77ae3e214 Pu Lehui 2023-02-15 1127 sreg_off = stack_size;
49b5e77ae3e214 Pu Lehui 2023-02-15 1128
a5912c37faf723 Puranjay Mohan 2024-07-08 1129 if ((flags & BPF_TRAMP_F_CALL_ORIG) && (nr_arg_slots - RV_MAX_REG_ARGS > 0))
6801b0aef79db4 Pu Lehui 2024-07-02 1130 stack_size += (nr_arg_slots - RV_MAX_REG_ARGS) * 8;
6801b0aef79db4 Pu Lehui 2024-07-02 1131
e944fc8152744a Xiao Wang 2024-05-23 1132 stack_size = round_up(stack_size, STACK_ALIGN);
49b5e77ae3e214 Pu Lehui 2023-02-15 1133
6801b0aef79db4 Pu Lehui 2024-07-02 1134 /* room for args on stack must be at the top of stack */
6801b0aef79db4 Pu Lehui 2024-07-02 1135 stk_arg_off = stack_size;
6801b0aef79db4 Pu Lehui 2024-07-02 1136
1732ebc4a26181 Pu Lehui 2024-01-23 1137 if (!is_struct_ops) {
25ad10658dc106 Pu Lehui 2023-07-21 1138 /* For the trampoline called from function entry,
25ad10658dc106 Pu Lehui 2023-07-21 1139 * the frame of traced function and the frame of
25ad10658dc106 Pu Lehui 2023-07-21 1140 * trampoline need to be considered.
25ad10658dc106 Pu Lehui 2023-07-21 1141 */
25ad10658dc106 Pu Lehui 2023-07-21 1142 emit_addi(RV_REG_SP, RV_REG_SP, -16, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1143 emit_sd(RV_REG_SP, 8, RV_REG_RA, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1144 emit_sd(RV_REG_SP, 0, RV_REG_FP, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1145 emit_addi(RV_REG_FP, RV_REG_SP, 16, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1146
25ad10658dc106 Pu Lehui 2023-07-21 1147 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1148 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_T0, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1149 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1150 emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1151 } else {
e63985ecd22681 Puranjay Mohan 2024-03-03 1152 /* emit kcfi hash */
e63985ecd22681 Puranjay Mohan 2024-03-03 1153 emit_kcfi(cfi_get_func_hash(func_addr), ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1154 /* For the trampoline called directly, just handle
25ad10658dc106 Pu Lehui 2023-07-21 1155 * the frame of trampoline.
25ad10658dc106 Pu Lehui 2023-07-21 1156 */
25ad10658dc106 Pu Lehui 2023-07-21 1157 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1158 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_RA, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1159 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1160 emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1161 }
49b5e77ae3e214 Pu Lehui 2023-02-15 1162
49b5e77ae3e214 Pu Lehui 2023-02-15 1163 /* callee saved register S1 to pass start time */
49b5e77ae3e214 Pu Lehui 2023-02-15 1164 emit_sd(RV_REG_FP, -sreg_off, RV_REG_S1, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1165
49b5e77ae3e214 Pu Lehui 2023-02-15 1166 /* store ip address of the traced function */
93fd420d71beed Menglong Dong 2026-02-08 1167 if (flags & BPF_TRAMP_F_IP_ARG)
93fd420d71beed Menglong Dong 2026-02-08 1168 emit_store_stack_imm64(RV_REG_T1, -ip_off, (u64)func_addr, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1169
35b3515be0ecb9 Menglong Dong 2026-02-08 1170 func_meta = nr_arg_slots;
35b3515be0ecb9 Menglong Dong 2026-02-08 1171 emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1172
6801b0aef79db4 Pu Lehui 2024-07-02 1173 store_args(nr_arg_slots, args_off, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1174
35b3515be0ecb9 Menglong Dong 2026-02-08 1175 if (bpf_fsession_cnt(tlinks)) {
35b3515be0ecb9 Menglong Dong 2026-02-08 1176 /* clear all session cookies' value */
35b3515be0ecb9 Menglong Dong 2026-02-08 1177 for (i = 0; i < cookie_cnt; i++)
35b3515be0ecb9 Menglong Dong 2026-02-08 1178 emit_sd(RV_REG_FP, -cookie_off + 8 * i, RV_REG_ZERO, ctx);
35b3515be0ecb9 Menglong Dong 2026-02-08 1179 /* clear return value to make sure fentry always get 0 */
35b3515be0ecb9 Menglong Dong 2026-02-08 1180 emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx);
35b3515be0ecb9 Menglong Dong 2026-02-08 1181 }
35b3515be0ecb9 Menglong Dong 2026-02-08 1182
49b5e77ae3e214 Pu Lehui 2023-02-15 1183 if (flags & BPF_TRAMP_F_CALL_ORIG) {
9f1e16fb1fc982 Pu Lehui 2024-06-22 1184 emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1185 ret = emit_call((const u64)__bpf_tramp_enter, true, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1186 if (ret)
49b5e77ae3e214 Pu Lehui 2023-02-15 1187 return ret;
49b5e77ae3e214 Pu Lehui 2023-02-15 1188 }
49b5e77ae3e214 Pu Lehui 2023-02-15 1189
35b3515be0ecb9 Menglong Dong 2026-02-08 1190 if (fentry->nr_links) {
35b3515be0ecb9 Menglong Dong 2026-02-08 1191 ret = invoke_bpf(fentry, args_off, retval_off, run_ctx_off, func_meta_off,
35b3515be0ecb9 Menglong Dong 2026-02-08 1192 flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1193 if (ret)
49b5e77ae3e214 Pu Lehui 2023-02-15 1194 return ret;
49b5e77ae3e214 Pu Lehui 2023-02-15 1195 }
49b5e77ae3e214 Pu Lehui 2023-02-15 1196
49b5e77ae3e214 Pu Lehui 2023-02-15 1197 if (fmod_ret->nr_links) {
49b5e77ae3e214 Pu Lehui 2023-02-15 1198 branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL);
49b5e77ae3e214 Pu Lehui 2023-02-15 1199 if (!branches_off)
49b5e77ae3e214 Pu Lehui 2023-02-15 1200 return -ENOMEM;
49b5e77ae3e214 Pu Lehui 2023-02-15 1201
49b5e77ae3e214 Pu Lehui 2023-02-15 1202 /* cleanup to avoid garbage return value confusion */
49b5e77ae3e214 Pu Lehui 2023-02-15 1203 emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1204 for (i = 0; i < fmod_ret->nr_links; i++) {
49b5e77ae3e214 Pu Lehui 2023-02-15 1205 ret = invoke_bpf_prog(fmod_ret->links[i], args_off, retval_off,
49b5e77ae3e214 Pu Lehui 2023-02-15 1206 run_ctx_off, true, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1207 if (ret)
49b5e77ae3e214 Pu Lehui 2023-02-15 1208 goto out;
49b5e77ae3e214 Pu Lehui 2023-02-15 1209 emit_ld(RV_REG_T1, -retval_off, RV_REG_FP, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1210 branches_off[i] = ctx->ninsns;
49b5e77ae3e214 Pu Lehui 2023-02-15 1211 /* nop reserved for conditional jump */
49b5e77ae3e214 Pu Lehui 2023-02-15 1212 emit(rv_nop(), ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1213 }
49b5e77ae3e214 Pu Lehui 2023-02-15 1214 }
49b5e77ae3e214 Pu Lehui 2023-02-15 1215
49b5e77ae3e214 Pu Lehui 2023-02-15 1216 if (flags & BPF_TRAMP_F_CALL_ORIG) {
8f3e00af8e52c0 Menglong Dong 2025-12-19 1217 /* skip to actual body of traced function */
8f3e00af8e52c0 Menglong Dong 2025-12-19 1218 orig_call += RV_FENTRY_NINSNS * 4;
6801b0aef79db4 Pu Lehui 2024-07-02 1219 restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx);
6801b0aef79db4 Pu Lehui 2024-07-02 1220 restore_stack_args(nr_arg_slots - RV_MAX_REG_ARGS, args_off, stk_arg_off, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1221 ret = emit_call((const u64)orig_call, true, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1222 if (ret)
49b5e77ae3e214 Pu Lehui 2023-02-15 1223 goto out;
49b5e77ae3e214 Pu Lehui 2023-02-15 1224 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx);
7112cd26e606c7 Björn Töpel 2023-10-04 1225 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx);
2382a405c581ae Pu Lehui 2024-06-22 1226 im->ip_after_call = ctx->ro_insns + ctx->ninsns;
49b5e77ae3e214 Pu Lehui 2023-02-15 1227 /* 2 nops reserved for auipc+jalr pair */
49b5e77ae3e214 Pu Lehui 2023-02-15 1228 emit(rv_nop(), ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1229 emit(rv_nop(), ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1230 }
49b5e77ae3e214 Pu Lehui 2023-02-15 1231
49b5e77ae3e214 Pu Lehui 2023-02-15 1232 /* update branches saved in invoke_bpf_mod_ret with bnez */
49b5e77ae3e214 Pu Lehui 2023-02-15 1233 for (i = 0; ctx->insns && i < fmod_ret->nr_links; i++) {
49b5e77ae3e214 Pu Lehui 2023-02-15 1234 offset = ninsns_rvoff(ctx->ninsns - branches_off[i]);
49b5e77ae3e214 Pu Lehui 2023-02-15 1235 insn = rv_bne(RV_REG_T1, RV_REG_ZERO, offset >> 1);
49b5e77ae3e214 Pu Lehui 2023-02-15 1236 *(u32 *)(ctx->insns + branches_off[i]) = insn;
49b5e77ae3e214 Pu Lehui 2023-02-15 1237 }
49b5e77ae3e214 Pu Lehui 2023-02-15 1238
35b3515be0ecb9 Menglong Dong 2026-02-08 1239 /* set "is_return" flag for fsession */
35b3515be0ecb9 Menglong Dong 2026-02-08 1240 func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT);
35b3515be0ecb9 Menglong Dong 2026-02-08 1241 if (bpf_fsession_cnt(tlinks))
35b3515be0ecb9 Menglong Dong 2026-02-08 1242 emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx);
35b3515be0ecb9 Menglong Dong 2026-02-08 1243
35b3515be0ecb9 Menglong Dong 2026-02-08 1244 if (fexit->nr_links) {
35b3515be0ecb9 Menglong Dong 2026-02-08 1245 ret = invoke_bpf(fexit, args_off, retval_off, run_ctx_off, func_meta_off,
35b3515be0ecb9 Menglong Dong 2026-02-08 1246 false, func_meta, cookie_off, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1247 if (ret)
49b5e77ae3e214 Pu Lehui 2023-02-15 1248 goto out;
49b5e77ae3e214 Pu Lehui 2023-02-15 1249 }
49b5e77ae3e214 Pu Lehui 2023-02-15 1250
49b5e77ae3e214 Pu Lehui 2023-02-15 1251 if (flags & BPF_TRAMP_F_CALL_ORIG) {
2382a405c581ae Pu Lehui 2024-06-22 1252 im->ip_epilogue = ctx->ro_insns + ctx->ninsns;
9f1e16fb1fc982 Pu Lehui 2024-06-22 1253 emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1254 ret = emit_call((const u64)__bpf_tramp_exit, true, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1255 if (ret)
49b5e77ae3e214 Pu Lehui 2023-02-15 1256 goto out;
49b5e77ae3e214 Pu Lehui 2023-02-15 1257 }
49b5e77ae3e214 Pu Lehui 2023-02-15 1258
49b5e77ae3e214 Pu Lehui 2023-02-15 1259 if (flags & BPF_TRAMP_F_RESTORE_REGS)
6801b0aef79db4 Pu Lehui 2024-07-02 1260 restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1261
7112cd26e606c7 Björn Töpel 2023-10-04 1262 if (save_ret) {
7112cd26e606c7 Björn Töpel 2023-10-04 1263 emit_ld(regmap[BPF_REG_0], -(retval_off - 8), RV_REG_FP, ctx);
fd2e08128944a7 Hengqi Chen 2025-09-08 1264 if (is_struct_ops) {
fd2e08128944a7 Hengqi Chen 2025-09-08 1265 ret = sign_extend(RV_REG_A0, regmap[BPF_REG_0], m->ret_size,
fd2e08128944a7 Hengqi Chen 2025-09-08 1266 m->ret_flags & BTF_FMODEL_SIGNED_ARG, ctx);
fd2e08128944a7 Hengqi Chen 2025-09-08 1267 if (ret)
fd2e08128944a7 Hengqi Chen 2025-09-08 1268 goto out;
fd2e08128944a7 Hengqi Chen 2025-09-08 1269 } else {
fd2e08128944a7 Hengqi Chen 2025-09-08 1270 emit_ld(RV_REG_A0, -retval_off, RV_REG_FP, ctx);
fd2e08128944a7 Hengqi Chen 2025-09-08 1271 }
7112cd26e606c7 Björn Töpel 2023-10-04 1272 }
49b5e77ae3e214 Pu Lehui 2023-02-15 1273
49b5e77ae3e214 Pu Lehui 2023-02-15 1274 emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1275
1732ebc4a26181 Pu Lehui 2024-01-23 1276 if (!is_struct_ops) {
25ad10658dc106 Pu Lehui 2023-07-21 1277 /* trampoline called from function entry */
25ad10658dc106 Pu Lehui 2023-07-21 1278 emit_ld(RV_REG_T0, stack_size - 8, RV_REG_SP, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1279 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1280 emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1281
25ad10658dc106 Pu Lehui 2023-07-21 1282 emit_ld(RV_REG_RA, 8, RV_REG_SP, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1283 emit_ld(RV_REG_FP, 0, RV_REG_SP, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1284 emit_addi(RV_REG_SP, RV_REG_SP, 16, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1285
49b5e77ae3e214 Pu Lehui 2023-02-15 1286 if (flags & BPF_TRAMP_F_SKIP_FRAME)
25ad10658dc106 Pu Lehui 2023-07-21 1287 /* return to parent function */
25ad10658dc106 Pu Lehui 2023-07-21 1288 emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1289 else
25ad10658dc106 Pu Lehui 2023-07-21 1290 /* return to traced function */
25ad10658dc106 Pu Lehui 2023-07-21 1291 emit_jalr(RV_REG_ZERO, RV_REG_T0, 0, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1292 } else {
25ad10658dc106 Pu Lehui 2023-07-21 1293 /* trampoline called directly */
25ad10658dc106 Pu Lehui 2023-07-21 1294 emit_ld(RV_REG_RA, stack_size - 8, RV_REG_SP, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1295 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1296 emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 1297
49b5e77ae3e214 Pu Lehui 2023-02-15 1298 emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx);
25ad10658dc106 Pu Lehui 2023-07-21 1299 }
49b5e77ae3e214 Pu Lehui 2023-02-15 1300
49b5e77ae3e214 Pu Lehui 2023-02-15 1301 ret = ctx->ninsns;
49b5e77ae3e214 Pu Lehui 2023-02-15 1302 out:
49b5e77ae3e214 Pu Lehui 2023-02-15 1303 kfree(branches_off);
49b5e77ae3e214 Pu Lehui 2023-02-15 1304 return ret;
49b5e77ae3e214 Pu Lehui 2023-02-15 1305 }
49b5e77ae3e214 Pu Lehui 2023-02-15 1306
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply
* Re: [PATCH v2 2/4] ring-buffer: Flush and stop persistent ring buffer on panic
From: Steven Rostedt @ 2026-02-20 19:53 UTC (permalink / raw)
To: Masami Hiramatsu (Google)
Cc: Mathieu Desnoyers, linux-kernel, linux-trace-kernel
In-Reply-To: <177140966801.1537493.7352910810257662003.stgit@mhiramat.tok.corp.google.com>
On Wed, 18 Feb 2026 19:14:28 +0900
"Masami Hiramatsu (Google)" <mhiramat@kernel.org> wrote:
> From: Masami Hiramatsu (Google) <mhiramat@kernel.org>
>
> On a real hardware, since panic and reboot the machine will not
> flush hardware cache to the persistent ring buffer, the events
> written right before the panic can be lost. Moreover, since
> there will be an inconsistency between the commit counter (which
> is written atomically via local_set()) and the data, validation
> will fail and all data in the persistent ring buffer will be lost.
Here's a bit of a fix up on the text:
On real hardware, panic and machine reboot may not flush hardware cache
to memory. This means the persistent ring buffer, which relies on a
coherent state of memory, may not have its events written to the buffer
and they may be lost. Moreover, there may be inconsistency with the
counters which are used for validation of the integrity of the
persistent ring buffer which may cause all data to be discarded.
>
> To avoid this issue, this will stop recording on the ring buffer
> and flush cache at the reserved memory on panic.
To avoid this issue, stop recording of the ring buffer on panic and
flush the cache of the ring buffer's memory.
-- Steve
^ permalink raw reply
* Re: [PATCH v2 3/4] ring-buffer: Skip invalid sub-buffers when validating persistent ring buffer
From: Steven Rostedt @ 2026-02-20 19:56 UTC (permalink / raw)
To: Masami Hiramatsu (Google)
Cc: Mathieu Desnoyers, linux-kernel, linux-trace-kernel
In-Reply-To: <177140967567.1537493.7265236420134382381.stgit@mhiramat.tok.corp.google.com>
On Wed, 18 Feb 2026 19:14:35 +0900
"Masami Hiramatsu (Google)" <mhiramat@kernel.org> wrote:
> From: Masami Hiramatsu (Google) <mhiramat@kernel.org>
>
> Skip invalid sub-buffers when validating the persistent ring buffer
> instead of invalidate all ring buffers.
instead of discarding the entire ring buffer.
>
> If the cache data in memory fails to be synchronized during a reboot,
> the persistent ring buffer may become partially corrupted, but other
> sub-buffers may still contain readable event data, allowing usersto
> recover data from the corrupted ring buffer.
... contain readable event data. Only discard the
subbuffers that are found to be corrupted.
>
> Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
> ---
> kernel/trace/ring_buffer.c | 22 ++++++++++++----------
> 1 file changed, 12 insertions(+), 10 deletions(-)
>
> diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
> index d2b69221a94c..0ae2a5ad8c3e 100644
> --- a/kernel/trace/ring_buffer.c
> +++ b/kernel/trace/ring_buffer.c
> @@ -2045,17 +2045,19 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer)
> if (ret < 0) {
> pr_info("Ring buffer meta [%d] invalid buffer page\n",
> cpu_buffer->cpu);
> - goto invalid;
> - }
> -
> - /* If the buffer has content, update pages_touched */
> - if (ret)
> - local_inc(&cpu_buffer->pages_touched);
> -
> - entries += ret;
> - entry_bytes += local_read(&head_page->page->commit);
> - local_set(&cpu_buffer->head_page->entries, ret);
> + /* Instead of invalidate whole ring buffer, just clear this subbuffer. */
> + local_set(&head_page->entries, 0);
> + local_set(&head_page->page->commit, 0);
> + /* TODO: commit an event to mark this is broken. */
Here's how to fix the TODO:
local_set(&head_page->page->commit, RB_MISSED_EVENTS);
-- Steve
> + } else {
> + /* If the buffer has content, update pages_touched */
> + if (ret)
> + local_inc(&cpu_buffer->pages_touched);
>
> + entries += ret;
> + entry_bytes += local_read(&head_page->page->commit);
> + local_set(&cpu_buffer->head_page->entries, ret);
> + }
> if (head_page == cpu_buffer->commit_page)
> break;
> }
^ permalink raw reply
* Re: [PATCH bpf-next 02/17] bpf: Use mutex lock pool for bpf trampolines
From: Alexei Starovoitov @ 2026-02-20 19:58 UTC (permalink / raw)
To: Jiri Olsa
Cc: Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, bpf,
linux-trace-kernel, Martin KaFai Lau, Eduard Zingerman, Song Liu,
Yonghong Song, Menglong Dong, Steven Rostedt
In-Reply-To: <20260220100649.628307-3-jolsa@kernel.org>
On Fri, Feb 20, 2026 at 2:07 AM Jiri Olsa <jolsa@kernel.org> wrote:
>
> Adding mutex lock pool that replaces bpf trampolines mutex.
>
> For tracing_multi link coming in following changes we need to lock all
> the involved trampolines during the attachment. This could mean thousands
> of mutex locks, which is not convenient.
>
> As suggested by Andrii we can replace bpf trampolines mutex with mutex
> pool, where each trampoline is hash-ed to one of the locks from the pool.
>
> It's better to lock all the pool mutexes (64 at the moment) than
> thousands of them.
>
> Removing the mutex_is_locked in bpf_trampoline_put, because we removed
> the mutex from bpf_trampoline.
>
> Suggested-by: Andrii Nakryiko <andrii@kernel.org>
> Signed-off-by: Jiri Olsa <jolsa@kernel.org>
> ---
> include/linux/bpf.h | 2 --
> kernel/bpf/trampoline.c | 74 +++++++++++++++++++++++++++++++----------
> 2 files changed, 56 insertions(+), 20 deletions(-)
>
> diff --git a/include/linux/bpf.h b/include/linux/bpf.h
> index cd9b96434904..46bf3d86bdb2 100644
> --- a/include/linux/bpf.h
> +++ b/include/linux/bpf.h
> @@ -1335,8 +1335,6 @@ struct bpf_trampoline {
> /* hlist for trampoline_ip_table */
> struct hlist_node hlist_ip;
> struct ftrace_ops *fops;
> - /* serializes access to fields of this trampoline */
> - struct mutex mutex;
> refcount_t refcnt;
> u32 flags;
> u64 key;
> diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
> index 952cd7932461..05dc0358654d 100644
> --- a/kernel/bpf/trampoline.c
> +++ b/kernel/bpf/trampoline.c
> @@ -30,6 +30,45 @@ static struct hlist_head trampoline_ip_table[TRAMPOLINE_TABLE_SIZE];
> /* serializes access to trampoline tables */
> static DEFINE_MUTEX(trampoline_mutex);
>
> +#define TRAMPOLINE_LOCKS_BITS 6
> +#define TRAMPOLINE_LOCKS_TABLE_SIZE (1 << TRAMPOLINE_LOCKS_BITS)
> +
> +static struct {
> + struct mutex mutex;
> + struct lock_class_key key;
> +} *trampoline_locks;
> +
> +static struct mutex *trampoline_locks_lookup(struct bpf_trampoline *tr)
select_trampoline_lock() ?
> +{
> + return &trampoline_locks[hash_64((u64) tr, TRAMPOLINE_LOCKS_BITS)].mutex;
> +}
> +
> +static void trampoline_lock(struct bpf_trampoline *tr)
> +{
> + mutex_lock(trampoline_locks_lookup(tr));
> +}
> +
> +static void trampoline_unlock(struct bpf_trampoline *tr)
> +{
> + mutex_unlock(trampoline_locks_lookup(tr));
> +}
> +
> +static int __init trampoline_locks_init(void)
> +{
> + int i;
> +
> + trampoline_locks = kmalloc_array(TRAMPOLINE_LOCKS_TABLE_SIZE,
> + sizeof(trampoline_locks[0]), GFP_KERNEL);
why bother with memory allocation? This is just 64 mutexes.
> + if (!trampoline_locks)
> + return -ENOMEM;
> +
> + for (i = 0; i < TRAMPOLINE_LOCKS_TABLE_SIZE; i++) {
> + lockdep_register_key(&trampoline_locks[i].key);
why special key?
^ permalink raw reply
* Re: [PATCH v2 4/4] ring-buffer: Record invalid buffer event
From: Steven Rostedt @ 2026-02-20 19:59 UTC (permalink / raw)
To: Masami Hiramatsu (Google)
Cc: Mathieu Desnoyers, linux-kernel, linux-trace-kernel
In-Reply-To: <177140968338.1537493.3298484599734677165.stgit@mhiramat.tok.corp.google.com>
On Wed, 18 Feb 2026 19:14:43 +0900
"Masami Hiramatsu (Google)" <mhiramat@kernel.org> wrote:
> From: Masami Hiramatsu (Google) <mhiramat@kernel.org>
>
> Record an invalid buffer event on the invalidated sub buffer
> so that user can notice how much data is skipped.
>
> Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
As I showed in patch 3, just mark it as having missed events. We could add
a pr_warn() that says the buffer was corrupted, but we don't need a
"invalid" event.
-- Steve
^ permalink raw reply
* Re: [PATCH bpf-next 04/17] bpf: Add struct bpf_tramp_node object
From: kernel test robot @ 2026-02-20 21:05 UTC (permalink / raw)
To: Jiri Olsa, Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: oe-kbuild-all, bpf, linux-trace-kernel, Martin KaFai Lau,
Eduard Zingerman, Song Liu, Yonghong Song, Menglong Dong,
Steven Rostedt
In-Reply-To: <20260220100649.628307-5-jolsa@kernel.org>
Hi Jiri,
kernel test robot noticed the following build errors:
[auto build test ERROR on bpf-next/master]
url: https://github.com/intel-lab-lkp/linux/commits/Jiri-Olsa/ftrace-Add-ftrace_hash_count-function/20260220-181324
base: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master
patch link: https://lore.kernel.org/r/20260220100649.628307-5-jolsa%40kernel.org
patch subject: [PATCH bpf-next 04/17] bpf: Add struct bpf_tramp_node object
config: riscv-allnoconfig-bpf (https://download.01.org/0day-ci/archive/20260220/202602202212.yC5wLunx-lkp@intel.com/config)
compiler: riscv64-linux-gnu-gcc (Debian 14.2.0-19) 14.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260220/202602202212.yC5wLunx-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202602202212.yC5wLunx-lkp@intel.com/
All errors (new ones prefixed by >>):
arch/riscv/net/bpf_jit_comp64.c: In function 'invoke_bpf_prog':
>> arch/riscv/net/bpf_jit_comp64.c:944:14: error: 'struct bpf_tramp_link' has no member named 'cookie'
944 | if (l->cookie)
| ^~
arch/riscv/net/bpf_jit_comp64.c:945:79: error: 'struct bpf_tramp_link' has no member named 'cookie'
945 | emit_store_stack_imm64(RV_REG_T1, -run_ctx_off + cookie_off, l->cookie, ctx);
| ^~
arch/riscv/net/bpf_jit_comp64.c: At top level:
arch/riscv/net/bpf_jit_comp64.c:999:30: warning: 'struct bpf_tramp_links' declared inside parameter list will not be visible outside of this definition or declaration
999 | static int invoke_bpf(struct bpf_tramp_links *tl, int args_off, int retval_off,
| ^~~~~~~~~~~~~~~
arch/riscv/net/bpf_jit_comp64.c: In function 'invoke_bpf':
>> arch/riscv/net/bpf_jit_comp64.c:1005:27: error: invalid use of undefined type 'struct bpf_tramp_links'
1005 | for (i = 0; i < tl->nr_links; i++) {
| ^~
arch/riscv/net/bpf_jit_comp64.c:1008:53: error: invalid use of undefined type 'struct bpf_tramp_links'
1008 | if (bpf_prog_calls_session_cookie(tl->links[i])) {
| ^~
arch/riscv/net/bpf_jit_comp64.c:1014:41: error: invalid use of undefined type 'struct bpf_tramp_links'
1014 | err = invoke_bpf_prog(tl->links[i], args_off, retval_off, run_ctx_off,
| ^~
arch/riscv/net/bpf_jit_comp64.c: At top level:
arch/riscv/net/bpf_jit_comp64.c:1024:49: warning: 'struct bpf_tramp_links' declared inside parameter list will not be visible outside of this definition or declaration
1024 | struct bpf_tramp_links *tlinks,
| ^~~~~~~~~~~~~~~
arch/riscv/net/bpf_jit_comp64.c: In function '__arch_prepare_bpf_trampoline':
arch/riscv/net/bpf_jit_comp64.c:1033:49: error: invalid use of undefined type 'struct bpf_tramp_links'
1033 | struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
| ^
arch/riscv/net/bpf_jit_comp64.c:1034:48: error: invalid use of undefined type 'struct bpf_tramp_links'
1034 | struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
| ^
arch/riscv/net/bpf_jit_comp64.c:1035:51: error: invalid use of undefined type 'struct bpf_tramp_links'
1035 | struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
| ^
arch/riscv/net/bpf_jit_comp64.c:1118:46: error: passing argument 1 of 'bpf_fsession_cookie_cnt' from incompatible pointer type [-Wincompatible-pointer-types]
1118 | cookie_cnt = bpf_fsession_cookie_cnt(tlinks);
| ^~~~~~
| |
| struct bpf_tramp_links *
In file included from arch/riscv/net/bpf_jit_comp64.c:9:
./include/linux/bpf.h:2207:67: note: expected 'struct bpf_tramp_nodes *' but argument is of type 'struct bpf_tramp_links *'
2207 | static inline int bpf_fsession_cookie_cnt(struct bpf_tramp_nodes *nodes)
| ~~~~~~~~~~~~~~~~~~~~~~~~^~~~~
arch/riscv/net/bpf_jit_comp64.c:1175:30: error: passing argument 1 of 'bpf_fsession_cnt' from incompatible pointer type [-Wincompatible-pointer-types]
1175 | if (bpf_fsession_cnt(tlinks)) {
| ^~~~~~
| |
| struct bpf_tramp_links *
./include/linux/bpf.h:2189:60: note: expected 'struct bpf_tramp_nodes *' but argument is of type 'struct bpf_tramp_links *'
2189 | static inline int bpf_fsession_cnt(struct bpf_tramp_nodes *nodes)
| ~~~~~~~~~~~~~~~~~~~~~~~~^~~~~
arch/riscv/net/bpf_jit_comp64.c:1190:19: error: invalid use of undefined type 'struct bpf_tramp_links'
1190 | if (fentry->nr_links) {
| ^~
arch/riscv/net/bpf_jit_comp64.c:1191:34: error: passing argument 1 of 'invoke_bpf' from incompatible pointer type [-Wincompatible-pointer-types]
1191 | ret = invoke_bpf(fentry, args_off, retval_off, run_ctx_off, func_meta_off,
| ^~~~~~
| |
| struct bpf_tramp_links *
arch/riscv/net/bpf_jit_comp64.c:999:47: note: expected 'struct bpf_tramp_links *' but argument is of type 'struct bpf_tramp_links *'
999 | static int invoke_bpf(struct bpf_tramp_links *tl, int args_off, int retval_off,
| ~~~~~~~~~~~~~~~~~~~~~~~~^~
arch/riscv/net/bpf_jit_comp64.c:1197:21: error: invalid use of undefined type 'struct bpf_tramp_links'
1197 | if (fmod_ret->nr_links) {
| ^~
In file included from ./include/linux/workqueue.h:9,
from ./include/linux/bpf.h:11:
arch/riscv/net/bpf_jit_comp64.c:1198:48: error: invalid use of undefined type 'struct bpf_tramp_links'
1198 | branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL);
| ^~
./include/linux/alloc_tag.h:251:16: note: in definition of macro 'alloc_hooks_tag'
251 | typeof(_do_alloc) _res; \
| ^~~~~~~~~
./include/linux/slab.h:1115:49: note: in expansion of macro 'alloc_hooks'
1115 | #define kmalloc_array(...) alloc_hooks(kmalloc_array_noprof(__VA_ARGS__))
| ^~~~~~~~~~~
./include/linux/slab.h:1154:41: note: in expansion of macro 'kmalloc_array'
1154 | #define kcalloc(n, size, flags) kmalloc_array(n, size, (flags) | __GFP_ZERO)
| ^~~~~~~~~~~~~
arch/riscv/net/bpf_jit_comp64.c:1198:32: note: in expansion of macro 'kcalloc'
1198 | branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL);
| ^~~~~~~
arch/riscv/net/bpf_jit_comp64.c:1198:48: error: invalid use of undefined type 'struct bpf_tramp_links'
1198 | branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL);
| ^~
./include/linux/alloc_tag.h:255:24: note: in definition of macro 'alloc_hooks_tag'
255 | _res = _do_alloc; \
| ^~~~~~~~~
./include/linux/slab.h:1115:49: note: in expansion of macro 'alloc_hooks'
1115 | #define kmalloc_array(...) alloc_hooks(kmalloc_array_noprof(__VA_ARGS__))
| ^~~~~~~~~~~
./include/linux/slab.h:1154:41: note: in expansion of macro 'kmalloc_array'
1154 | #define kcalloc(n, size, flags) kmalloc_array(n, size, (flags) | __GFP_ZERO)
| ^~~~~~~~~~~~~
arch/riscv/net/bpf_jit_comp64.c:1198:32: note: in expansion of macro 'kcalloc'
1198 | branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL);
| ^~~~~~~
arch/riscv/net/bpf_jit_comp64.c:1198:48: error: invalid use of undefined type 'struct bpf_tramp_links'
1198 | branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL);
| ^~
./include/linux/alloc_tag.h:258:24: note: in definition of macro 'alloc_hooks_tag'
258 | _res = _do_alloc; \
| ^~~~~~~~~
./include/linux/slab.h:1115:49: note: in expansion of macro 'alloc_hooks'
1115 | #define kmalloc_array(...) alloc_hooks(kmalloc_array_noprof(__VA_ARGS__))
| ^~~~~~~~~~~
./include/linux/slab.h:1154:41: note: in expansion of macro 'kmalloc_array'
1154 | #define kcalloc(n, size, flags) kmalloc_array(n, size, (flags) | __GFP_ZERO)
| ^~~~~~~~~~~~~
arch/riscv/net/bpf_jit_comp64.c:1198:32: note: in expansion of macro 'kcalloc'
1198 | branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL);
| ^~~~~~~
vim +944 arch/riscv/net/bpf_jit_comp64.c
93fd420d71beed Menglong Dong 2026-02-08 936
49b5e77ae3e214 Pu Lehui 2023-02-15 937 static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_off,
49b5e77ae3e214 Pu Lehui 2023-02-15 938 int run_ctx_off, bool save_ret, struct rv_jit_context *ctx)
49b5e77ae3e214 Pu Lehui 2023-02-15 939 {
49b5e77ae3e214 Pu Lehui 2023-02-15 940 int ret, branch_off;
49b5e77ae3e214 Pu Lehui 2023-02-15 941 struct bpf_prog *p = l->link.prog;
49b5e77ae3e214 Pu Lehui 2023-02-15 942 int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
49b5e77ae3e214 Pu Lehui 2023-02-15 943
93fd420d71beed Menglong Dong 2026-02-08 @944 if (l->cookie)
93fd420d71beed Menglong Dong 2026-02-08 945 emit_store_stack_imm64(RV_REG_T1, -run_ctx_off + cookie_off, l->cookie, ctx);
93fd420d71beed Menglong Dong 2026-02-08 946 else
49b5e77ae3e214 Pu Lehui 2023-02-15 947 emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_ZERO, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 948
49b5e77ae3e214 Pu Lehui 2023-02-15 949 /* arg1: prog */
49b5e77ae3e214 Pu Lehui 2023-02-15 950 emit_imm(RV_REG_A0, (const s64)p, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 951 /* arg2: &run_ctx */
49b5e77ae3e214 Pu Lehui 2023-02-15 952 emit_addi(RV_REG_A1, RV_REG_FP, -run_ctx_off, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 953 ret = emit_call((const u64)bpf_trampoline_enter(p), true, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 954 if (ret)
49b5e77ae3e214 Pu Lehui 2023-02-15 955 return ret;
49b5e77ae3e214 Pu Lehui 2023-02-15 956
10541b374aa05c Xu Kuohai 2024-04-16 957 /* store prog start time */
10541b374aa05c Xu Kuohai 2024-04-16 958 emit_mv(RV_REG_S1, RV_REG_A0, ctx);
10541b374aa05c Xu Kuohai 2024-04-16 959
49b5e77ae3e214 Pu Lehui 2023-02-15 960 /* if (__bpf_prog_enter(prog) == 0)
49b5e77ae3e214 Pu Lehui 2023-02-15 961 * goto skip_exec_of_prog;
49b5e77ae3e214 Pu Lehui 2023-02-15 962 */
49b5e77ae3e214 Pu Lehui 2023-02-15 963 branch_off = ctx->ninsns;
49b5e77ae3e214 Pu Lehui 2023-02-15 964 /* nop reserved for conditional jump */
49b5e77ae3e214 Pu Lehui 2023-02-15 965 emit(rv_nop(), ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 966
49b5e77ae3e214 Pu Lehui 2023-02-15 967 /* arg1: &args_off */
49b5e77ae3e214 Pu Lehui 2023-02-15 968 emit_addi(RV_REG_A0, RV_REG_FP, -args_off, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 969 if (!p->jited)
49b5e77ae3e214 Pu Lehui 2023-02-15 970 /* arg2: progs[i]->insnsi for interpreter */
49b5e77ae3e214 Pu Lehui 2023-02-15 971 emit_imm(RV_REG_A1, (const s64)p->insnsi, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 972 ret = emit_call((const u64)p->bpf_func, true, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 973 if (ret)
49b5e77ae3e214 Pu Lehui 2023-02-15 974 return ret;
49b5e77ae3e214 Pu Lehui 2023-02-15 975
7112cd26e606c7 Björn Töpel 2023-10-04 976 if (save_ret) {
7112cd26e606c7 Björn Töpel 2023-10-04 977 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx);
7112cd26e606c7 Björn Töpel 2023-10-04 978 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx);
7112cd26e606c7 Björn Töpel 2023-10-04 979 }
49b5e77ae3e214 Pu Lehui 2023-02-15 980
49b5e77ae3e214 Pu Lehui 2023-02-15 981 /* update branch with beqz */
49b5e77ae3e214 Pu Lehui 2023-02-15 982 if (ctx->insns) {
49b5e77ae3e214 Pu Lehui 2023-02-15 983 int offset = ninsns_rvoff(ctx->ninsns - branch_off);
49b5e77ae3e214 Pu Lehui 2023-02-15 984 u32 insn = rv_beq(RV_REG_A0, RV_REG_ZERO, offset >> 1);
49b5e77ae3e214 Pu Lehui 2023-02-15 985 *(u32 *)(ctx->insns + branch_off) = insn;
49b5e77ae3e214 Pu Lehui 2023-02-15 986 }
49b5e77ae3e214 Pu Lehui 2023-02-15 987
49b5e77ae3e214 Pu Lehui 2023-02-15 988 /* arg1: prog */
49b5e77ae3e214 Pu Lehui 2023-02-15 989 emit_imm(RV_REG_A0, (const s64)p, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 990 /* arg2: prog start time */
49b5e77ae3e214 Pu Lehui 2023-02-15 991 emit_mv(RV_REG_A1, RV_REG_S1, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 992 /* arg3: &run_ctx */
49b5e77ae3e214 Pu Lehui 2023-02-15 993 emit_addi(RV_REG_A2, RV_REG_FP, -run_ctx_off, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 994 ret = emit_call((const u64)bpf_trampoline_exit(p), true, ctx);
49b5e77ae3e214 Pu Lehui 2023-02-15 995
49b5e77ae3e214 Pu Lehui 2023-02-15 996 return ret;
49b5e77ae3e214 Pu Lehui 2023-02-15 997 }
49b5e77ae3e214 Pu Lehui 2023-02-15 998
35b3515be0ecb9 Menglong Dong 2026-02-08 999 static int invoke_bpf(struct bpf_tramp_links *tl, int args_off, int retval_off,
35b3515be0ecb9 Menglong Dong 2026-02-08 1000 int run_ctx_off, int func_meta_off, bool save_ret, u64 func_meta,
35b3515be0ecb9 Menglong Dong 2026-02-08 1001 int cookie_off, struct rv_jit_context *ctx)
35b3515be0ecb9 Menglong Dong 2026-02-08 1002 {
35b3515be0ecb9 Menglong Dong 2026-02-08 1003 int i, cur_cookie = (cookie_off - args_off) / 8;
35b3515be0ecb9 Menglong Dong 2026-02-08 1004
35b3515be0ecb9 Menglong Dong 2026-02-08 @1005 for (i = 0; i < tl->nr_links; i++) {
35b3515be0ecb9 Menglong Dong 2026-02-08 1006 int err;
35b3515be0ecb9 Menglong Dong 2026-02-08 1007
35b3515be0ecb9 Menglong Dong 2026-02-08 1008 if (bpf_prog_calls_session_cookie(tl->links[i])) {
35b3515be0ecb9 Menglong Dong 2026-02-08 1009 u64 meta = func_meta | ((u64)cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT);
35b3515be0ecb9 Menglong Dong 2026-02-08 1010
35b3515be0ecb9 Menglong Dong 2026-02-08 1011 emit_store_stack_imm64(RV_REG_T1, -func_meta_off, meta, ctx);
35b3515be0ecb9 Menglong Dong 2026-02-08 1012 cur_cookie--;
35b3515be0ecb9 Menglong Dong 2026-02-08 1013 }
35b3515be0ecb9 Menglong Dong 2026-02-08 1014 err = invoke_bpf_prog(tl->links[i], args_off, retval_off, run_ctx_off,
35b3515be0ecb9 Menglong Dong 2026-02-08 1015 save_ret, ctx);
35b3515be0ecb9 Menglong Dong 2026-02-08 1016 if (err)
35b3515be0ecb9 Menglong Dong 2026-02-08 1017 return err;
35b3515be0ecb9 Menglong Dong 2026-02-08 1018 }
35b3515be0ecb9 Menglong Dong 2026-02-08 1019 return 0;
35b3515be0ecb9 Menglong Dong 2026-02-08 1020 }
35b3515be0ecb9 Menglong Dong 2026-02-08 1021
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply
* Re: [PATCH 0/4] mm: zone lock tracepoint instrumentation
From: Shakeel Butt @ 2026-02-20 22:36 UTC (permalink / raw)
To: Cheatham, Benjamin
Cc: Dmitry Ilvokhin, linux-kernel, linux-mm, linux-trace-kernel,
linux-cxl, kernel-team, Andrew Morton, David Hildenbrand,
Lorenzo Stoakes, Liam R. Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Steven Rostedt,
Masami Hiramatsu, Mathieu Desnoyers, Brendan Jackman,
Johannes Weiner, Zi Yan, Oscar Salvador, Qi Zheng, Axel Rasmussen,
Yuanchu Xie, Wei Xu
In-Reply-To: <06b2a2b6-d5c8-4522-8e22-10616f887846@amd.com>
On Fri, Feb 20, 2026 at 01:09:59PM -0600, Cheatham, Benjamin wrote:
> On 2/11/2026 9:22 AM, Dmitry Ilvokhin wrote:
> > Zone lock contention can significantly impact allocation and
> > reclaim latency, as it is a central synchronization point in
> > the page allocator and reclaim paths. Improved visibility into
> > its behavior is therefore important for diagnosing performance
> > issues in memory-intensive workloads.
> >
> > On some production workloads at Meta, we have observed noticeable
> > zone lock contention. Deeper analysis of lock holders and waiters
> > is currently difficult with existing instrumentation.
> >
> > While generic lock contention_begin/contention_end tracepoints
> > cover the slow path, they do not provide sufficient visibility
> > into lock hold times. In particular, the lack of a release-side
> > event makes it difficult to identify long lock holders and
> > correlate them with waiters. As a result, distinguishing between
> > short bursts of contention and pathological long hold times
> > requires additional instrumentation.
> >
> > This patch series adds dedicated tracepoint instrumentation to
> > zone lock, following the existing mmap_lock tracing model.
> >
> > The goal is to enable detailed holder/waiter analysis and lock
> > hold time measurements without affecting the fast path when
> > tracing is disabled.
> >
> > The series is structured as follows:
> >
> > 1. Introduce zone lock wrappers.
> > 2. Mechanically convert zone lock users to the wrappers.
> > 3. Convert compaction to use the wrappers (requires minor
> > restructuring of compact_lock_irqsave()).
> > 4. Add zone lock tracepoints.
>
> I think you can improve the flow of this series if reorder as follows:
> 1. Introduce zone lock wrappers
> 4. Add zone lock tracepoints
> 2. Mechanically convert zone lock users to the wrappers
> 3. Convert compaction to use the wrappers...
>
> and possibly squash 1 & 4 (though that might be too big of a patch). It's better to introduce the
> wrappers and their tracepoints together before the reviewer (i.e. me) forgets what was added in
> patch 1 by the time they get to patch 4.
I don't think this suggestion will make anything better. This just seems like a
different taste. If I make a suggestion, I would request to squash (1) and (2)
i.e. patch containing wrappers and their use together but that is just my taste
and would be a nit. The series ordering is good as is.
^ permalink raw reply
* Re: [PATCH bpf-next 04/17] bpf: Add struct bpf_tramp_node object
From: kernel test robot @ 2026-02-21 3:00 UTC (permalink / raw)
To: Jiri Olsa, Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: oe-kbuild-all, bpf, linux-trace-kernel, Martin KaFai Lau,
Eduard Zingerman, Song Liu, Yonghong Song, Menglong Dong,
Steven Rostedt
In-Reply-To: <20260220100649.628307-5-jolsa@kernel.org>
Hi Jiri,
kernel test robot noticed the following build errors:
[auto build test ERROR on bpf-next/master]
url: https://github.com/intel-lab-lkp/linux/commits/Jiri-Olsa/ftrace-Add-ftrace_hash_count-function/20260220-181324
base: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master
patch link: https://lore.kernel.org/r/20260220100649.628307-5-jolsa%40kernel.org
patch subject: [PATCH bpf-next 04/17] bpf: Add struct bpf_tramp_node object
config: riscv-randconfig-001-20260221 (https://download.01.org/0day-ci/archive/20260221/202602211023.EiuS4wkF-lkp@intel.com/config)
compiler: riscv64-linux-gcc (GCC) 8.5.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260221/202602211023.EiuS4wkF-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202602211023.EiuS4wkF-lkp@intel.com/
All errors (new ones prefixed by >>):
arch/riscv/net/bpf_jit_comp64.c: In function 'invoke_bpf_prog':
>> arch/riscv/net/bpf_jit_comp64.c:944:7: error: 'struct bpf_tramp_link' has no member named 'cookie'
if (l->cookie)
^~
arch/riscv/net/bpf_jit_comp64.c:945:65: error: 'struct bpf_tramp_link' has no member named 'cookie'
emit_store_stack_imm64(RV_REG_T1, -run_ctx_off + cookie_off, l->cookie, ctx);
^~
arch/riscv/net/bpf_jit_comp64.c: At top level:
arch/riscv/net/bpf_jit_comp64.c:999:30: warning: 'struct bpf_tramp_links' declared inside parameter list will not be visible outside of this definition or declaration
static int invoke_bpf(struct bpf_tramp_links *tl, int args_off, int retval_off,
^~~~~~~~~~~~~~~
arch/riscv/net/bpf_jit_comp64.c: In function 'invoke_bpf':
>> arch/riscv/net/bpf_jit_comp64.c:1005:20: error: dereferencing pointer to incomplete type 'struct bpf_tramp_links'
for (i = 0; i < tl->nr_links; i++) {
^~
arch/riscv/net/bpf_jit_comp64.c: At top level:
arch/riscv/net/bpf_jit_comp64.c:1024:14: warning: 'struct bpf_tramp_links' declared inside parameter list will not be visible outside of this definition or declaration
struct bpf_tramp_links *tlinks,
^~~~~~~~~~~~~~~
arch/riscv/net/bpf_jit_comp64.c: In function '__arch_prepare_bpf_trampoline':
>> arch/riscv/net/bpf_jit_comp64.c:1033:42: error: invalid use of undefined type 'struct bpf_tramp_links'
struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
^
arch/riscv/net/bpf_jit_comp64.c:1033:42: error: dereferencing pointer to incomplete type 'struct bpf_tramp_links'
arch/riscv/net/bpf_jit_comp64.c:1034:41: error: invalid use of undefined type 'struct bpf_tramp_links'
struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
^
arch/riscv/net/bpf_jit_comp64.c:1035:44: error: invalid use of undefined type 'struct bpf_tramp_links'
struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
^
arch/riscv/net/bpf_jit_comp64.c:1118:39: error: passing argument 1 of 'bpf_fsession_cookie_cnt' from incompatible pointer type [-Werror=incompatible-pointer-types]
cookie_cnt = bpf_fsession_cookie_cnt(tlinks);
^~~~~~
In file included from arch/riscv/net/bpf_jit_comp64.c:9:
include/linux/bpf.h:2207:67: note: expected 'struct bpf_tramp_nodes *' but argument is of type 'struct bpf_tramp_links *'
static inline int bpf_fsession_cookie_cnt(struct bpf_tramp_nodes *nodes)
~~~~~~~~~~~~~~~~~~~~~~~~^~~~~
arch/riscv/net/bpf_jit_comp64.c:1175:23: error: passing argument 1 of 'bpf_fsession_cnt' from incompatible pointer type [-Werror=incompatible-pointer-types]
if (bpf_fsession_cnt(tlinks)) {
^~~~~~
In file included from arch/riscv/net/bpf_jit_comp64.c:9:
include/linux/bpf.h:2189:60: note: expected 'struct bpf_tramp_nodes *' but argument is of type 'struct bpf_tramp_links *'
static inline int bpf_fsession_cnt(struct bpf_tramp_nodes *nodes)
~~~~~~~~~~~~~~~~~~~~~~~~^~~~~
arch/riscv/net/bpf_jit_comp64.c:1191:20: error: passing argument 1 of 'invoke_bpf' from incompatible pointer type [-Werror=incompatible-pointer-types]
ret = invoke_bpf(fentry, args_off, retval_off, run_ctx_off, func_meta_off,
^~~~~~
arch/riscv/net/bpf_jit_comp64.c:999:47: note: expected 'struct bpf_tramp_links *' but argument is of type 'struct bpf_tramp_links *'
static int invoke_bpf(struct bpf_tramp_links *tl, int args_off, int retval_off,
~~~~~~~~~~~~~~~~~~~~~~~~^~
arch/riscv/net/bpf_jit_comp64.c:1198:16: warning: assignment to 'int *' from 'int' makes pointer from integer without a cast [-Wint-conversion]
branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL);
^
arch/riscv/net/bpf_jit_comp64.c:1241:23: error: passing argument 1 of 'bpf_fsession_cnt' from incompatible pointer type [-Werror=incompatible-pointer-types]
if (bpf_fsession_cnt(tlinks))
^~~~~~
In file included from arch/riscv/net/bpf_jit_comp64.c:9:
include/linux/bpf.h:2189:60: note: expected 'struct bpf_tramp_nodes *' but argument is of type 'struct bpf_tramp_links *'
static inline int bpf_fsession_cnt(struct bpf_tramp_nodes *nodes)
~~~~~~~~~~~~~~~~~~~~~~~~^~~~~
arch/riscv/net/bpf_jit_comp64.c:1245:20: error: passing argument 1 of 'invoke_bpf' from incompatible pointer type [-Werror=incompatible-pointer-types]
ret = invoke_bpf(fexit, args_off, retval_off, run_ctx_off, func_meta_off,
^~~~~
arch/riscv/net/bpf_jit_comp64.c:999:47: note: expected 'struct bpf_tramp_links *' but argument is of type 'struct bpf_tramp_links *'
static int invoke_bpf(struct bpf_tramp_links *tl, int args_off, int retval_off,
~~~~~~~~~~~~~~~~~~~~~~~~^~
arch/riscv/net/bpf_jit_comp64.c: At top level:
arch/riscv/net/bpf_jit_comp64.c:1308:16: warning: 'struct bpf_tramp_links' declared inside parameter list will not be visible outside of this definition or declaration
struct bpf_tramp_links *tlinks, void *func_addr)
^~~~~~~~~~~~~~~
arch/riscv/net/bpf_jit_comp64.c:1307:5: error: conflicting types for 'arch_bpf_trampoline_size'
int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
^~~~~~~~~~~~~~~~~~~~~~~~
In file included from arch/riscv/net/bpf_jit_comp64.c:9:
include/linux/bpf.h:1271:5: note: previous declaration of 'arch_bpf_trampoline_size' was here
int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
^~~~~~~~~~~~~~~~~~~~~~~~
arch/riscv/net/bpf_jit_comp64.c: In function 'arch_bpf_trampoline_size':
arch/riscv/net/bpf_jit_comp64.c:1317:46: error: passing argument 3 of '__arch_prepare_bpf_trampoline' from incompatible pointer type [-Werror=incompatible-pointer-types]
ret = __arch_prepare_bpf_trampoline(&im, m, tlinks, func_addr, flags, &ctx);
^~~~~~
arch/riscv/net/bpf_jit_comp64.c:1024:31: note: expected 'struct bpf_tramp_links *' but argument is of type 'struct bpf_tramp_links *'
struct bpf_tramp_links *tlinks,
~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~
arch/riscv/net/bpf_jit_comp64.c: At top level:
arch/riscv/net/bpf_jit_comp64.c:1334:23: warning: 'struct bpf_tramp_links' declared inside parameter list will not be visible outside of this definition or declaration
u32 flags, struct bpf_tramp_links *tlinks,
^~~~~~~~~~~~~~~
arch/riscv/net/bpf_jit_comp64.c:1332:5: error: conflicting types for 'arch_prepare_bpf_trampoline'
int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image,
^~~~~~~~~~~~~~~~~~~~~~~~~~~
In file included from arch/riscv/net/bpf_jit_comp64.c:9:
include/linux/bpf.h:1264:5: note: previous declaration of 'arch_prepare_bpf_trampoline' was here
int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
^~~~~~~~~~~~~~~~~~~~~~~~~~~
arch/riscv/net/bpf_jit_comp64.c: In function 'arch_prepare_bpf_trampoline':
arch/riscv/net/bpf_jit_comp64.c:1349:45: error: passing argument 3 of '__arch_prepare_bpf_trampoline' from incompatible pointer type [-Werror=incompatible-pointer-types]
ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx);
^~~~~~
arch/riscv/net/bpf_jit_comp64.c:1024:31: note: expected 'struct bpf_tramp_links *' but argument is of type 'struct bpf_tramp_links *'
struct bpf_tramp_links *tlinks,
~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~
cc1: some warnings being treated as errors
vim +944 arch/riscv/net/bpf_jit_comp64.c
93fd420d71beed5 Menglong Dong 2026-02-08 936
49b5e77ae3e214a Pu Lehui 2023-02-15 937 static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_off,
49b5e77ae3e214a Pu Lehui 2023-02-15 938 int run_ctx_off, bool save_ret, struct rv_jit_context *ctx)
49b5e77ae3e214a Pu Lehui 2023-02-15 939 {
49b5e77ae3e214a Pu Lehui 2023-02-15 940 int ret, branch_off;
49b5e77ae3e214a Pu Lehui 2023-02-15 941 struct bpf_prog *p = l->link.prog;
49b5e77ae3e214a Pu Lehui 2023-02-15 942 int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
49b5e77ae3e214a Pu Lehui 2023-02-15 943
93fd420d71beed5 Menglong Dong 2026-02-08 @944 if (l->cookie)
93fd420d71beed5 Menglong Dong 2026-02-08 945 emit_store_stack_imm64(RV_REG_T1, -run_ctx_off + cookie_off, l->cookie, ctx);
93fd420d71beed5 Menglong Dong 2026-02-08 946 else
49b5e77ae3e214a Pu Lehui 2023-02-15 947 emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_ZERO, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 948
49b5e77ae3e214a Pu Lehui 2023-02-15 949 /* arg1: prog */
49b5e77ae3e214a Pu Lehui 2023-02-15 950 emit_imm(RV_REG_A0, (const s64)p, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 951 /* arg2: &run_ctx */
49b5e77ae3e214a Pu Lehui 2023-02-15 952 emit_addi(RV_REG_A1, RV_REG_FP, -run_ctx_off, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 953 ret = emit_call((const u64)bpf_trampoline_enter(p), true, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 954 if (ret)
49b5e77ae3e214a Pu Lehui 2023-02-15 955 return ret;
49b5e77ae3e214a Pu Lehui 2023-02-15 956
10541b374aa05c8 Xu Kuohai 2024-04-16 957 /* store prog start time */
10541b374aa05c8 Xu Kuohai 2024-04-16 958 emit_mv(RV_REG_S1, RV_REG_A0, ctx);
10541b374aa05c8 Xu Kuohai 2024-04-16 959
49b5e77ae3e214a Pu Lehui 2023-02-15 960 /* if (__bpf_prog_enter(prog) == 0)
49b5e77ae3e214a Pu Lehui 2023-02-15 961 * goto skip_exec_of_prog;
49b5e77ae3e214a Pu Lehui 2023-02-15 962 */
49b5e77ae3e214a Pu Lehui 2023-02-15 963 branch_off = ctx->ninsns;
49b5e77ae3e214a Pu Lehui 2023-02-15 964 /* nop reserved for conditional jump */
49b5e77ae3e214a Pu Lehui 2023-02-15 965 emit(rv_nop(), ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 966
49b5e77ae3e214a Pu Lehui 2023-02-15 967 /* arg1: &args_off */
49b5e77ae3e214a Pu Lehui 2023-02-15 968 emit_addi(RV_REG_A0, RV_REG_FP, -args_off, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 969 if (!p->jited)
49b5e77ae3e214a Pu Lehui 2023-02-15 970 /* arg2: progs[i]->insnsi for interpreter */
49b5e77ae3e214a Pu Lehui 2023-02-15 971 emit_imm(RV_REG_A1, (const s64)p->insnsi, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 972 ret = emit_call((const u64)p->bpf_func, true, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 973 if (ret)
49b5e77ae3e214a Pu Lehui 2023-02-15 974 return ret;
49b5e77ae3e214a Pu Lehui 2023-02-15 975
7112cd26e606c7b Björn Töpel 2023-10-04 976 if (save_ret) {
7112cd26e606c7b Björn Töpel 2023-10-04 977 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx);
7112cd26e606c7b Björn Töpel 2023-10-04 978 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx);
7112cd26e606c7b Björn Töpel 2023-10-04 979 }
49b5e77ae3e214a Pu Lehui 2023-02-15 980
49b5e77ae3e214a Pu Lehui 2023-02-15 981 /* update branch with beqz */
49b5e77ae3e214a Pu Lehui 2023-02-15 982 if (ctx->insns) {
49b5e77ae3e214a Pu Lehui 2023-02-15 983 int offset = ninsns_rvoff(ctx->ninsns - branch_off);
49b5e77ae3e214a Pu Lehui 2023-02-15 984 u32 insn = rv_beq(RV_REG_A0, RV_REG_ZERO, offset >> 1);
49b5e77ae3e214a Pu Lehui 2023-02-15 985 *(u32 *)(ctx->insns + branch_off) = insn;
49b5e77ae3e214a Pu Lehui 2023-02-15 986 }
49b5e77ae3e214a Pu Lehui 2023-02-15 987
49b5e77ae3e214a Pu Lehui 2023-02-15 988 /* arg1: prog */
49b5e77ae3e214a Pu Lehui 2023-02-15 989 emit_imm(RV_REG_A0, (const s64)p, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 990 /* arg2: prog start time */
49b5e77ae3e214a Pu Lehui 2023-02-15 991 emit_mv(RV_REG_A1, RV_REG_S1, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 992 /* arg3: &run_ctx */
49b5e77ae3e214a Pu Lehui 2023-02-15 993 emit_addi(RV_REG_A2, RV_REG_FP, -run_ctx_off, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 994 ret = emit_call((const u64)bpf_trampoline_exit(p), true, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 995
49b5e77ae3e214a Pu Lehui 2023-02-15 996 return ret;
49b5e77ae3e214a Pu Lehui 2023-02-15 997 }
49b5e77ae3e214a Pu Lehui 2023-02-15 998
35b3515be0ecb9d Menglong Dong 2026-02-08 999 static int invoke_bpf(struct bpf_tramp_links *tl, int args_off, int retval_off,
35b3515be0ecb9d Menglong Dong 2026-02-08 1000 int run_ctx_off, int func_meta_off, bool save_ret, u64 func_meta,
35b3515be0ecb9d Menglong Dong 2026-02-08 1001 int cookie_off, struct rv_jit_context *ctx)
35b3515be0ecb9d Menglong Dong 2026-02-08 1002 {
35b3515be0ecb9d Menglong Dong 2026-02-08 1003 int i, cur_cookie = (cookie_off - args_off) / 8;
35b3515be0ecb9d Menglong Dong 2026-02-08 1004
35b3515be0ecb9d Menglong Dong 2026-02-08 @1005 for (i = 0; i < tl->nr_links; i++) {
35b3515be0ecb9d Menglong Dong 2026-02-08 1006 int err;
35b3515be0ecb9d Menglong Dong 2026-02-08 1007
35b3515be0ecb9d Menglong Dong 2026-02-08 1008 if (bpf_prog_calls_session_cookie(tl->links[i])) {
35b3515be0ecb9d Menglong Dong 2026-02-08 1009 u64 meta = func_meta | ((u64)cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT);
35b3515be0ecb9d Menglong Dong 2026-02-08 1010
35b3515be0ecb9d Menglong Dong 2026-02-08 1011 emit_store_stack_imm64(RV_REG_T1, -func_meta_off, meta, ctx);
35b3515be0ecb9d Menglong Dong 2026-02-08 1012 cur_cookie--;
35b3515be0ecb9d Menglong Dong 2026-02-08 1013 }
35b3515be0ecb9d Menglong Dong 2026-02-08 1014 err = invoke_bpf_prog(tl->links[i], args_off, retval_off, run_ctx_off,
35b3515be0ecb9d Menglong Dong 2026-02-08 1015 save_ret, ctx);
35b3515be0ecb9d Menglong Dong 2026-02-08 1016 if (err)
35b3515be0ecb9d Menglong Dong 2026-02-08 1017 return err;
35b3515be0ecb9d Menglong Dong 2026-02-08 1018 }
35b3515be0ecb9d Menglong Dong 2026-02-08 1019 return 0;
35b3515be0ecb9d Menglong Dong 2026-02-08 1020 }
35b3515be0ecb9d Menglong Dong 2026-02-08 1021
49b5e77ae3e214a Pu Lehui 2023-02-15 1022 static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
49b5e77ae3e214a Pu Lehui 2023-02-15 1023 const struct btf_func_model *m,
49b5e77ae3e214a Pu Lehui 2023-02-15 1024 struct bpf_tramp_links *tlinks,
49b5e77ae3e214a Pu Lehui 2023-02-15 1025 void *func_addr, u32 flags,
49b5e77ae3e214a Pu Lehui 2023-02-15 1026 struct rv_jit_context *ctx)
49b5e77ae3e214a Pu Lehui 2023-02-15 1027 {
49b5e77ae3e214a Pu Lehui 2023-02-15 1028 int i, ret, offset;
49b5e77ae3e214a Pu Lehui 2023-02-15 1029 int *branches_off = NULL;
6801b0aef79db47 Pu Lehui 2024-07-02 1030 int stack_size = 0, nr_arg_slots = 0;
35b3515be0ecb9d Menglong Dong 2026-02-08 1031 int retval_off, args_off, func_meta_off, ip_off, run_ctx_off, sreg_off, stk_arg_off;
35b3515be0ecb9d Menglong Dong 2026-02-08 1032 int cookie_off, cookie_cnt;
49b5e77ae3e214a Pu Lehui 2023-02-15 @1033 struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
49b5e77ae3e214a Pu Lehui 2023-02-15 1034 struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
49b5e77ae3e214a Pu Lehui 2023-02-15 1035 struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
1732ebc4a26181c Pu Lehui 2024-01-23 1036 bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT;
49b5e77ae3e214a Pu Lehui 2023-02-15 1037 void *orig_call = func_addr;
49b5e77ae3e214a Pu Lehui 2023-02-15 1038 bool save_ret;
35b3515be0ecb9d Menglong Dong 2026-02-08 1039 u64 func_meta;
49b5e77ae3e214a Pu Lehui 2023-02-15 1040 u32 insn;
49b5e77ae3e214a Pu Lehui 2023-02-15 1041
25ad10658dc1068 Pu Lehui 2023-07-21 1042 /* Two types of generated trampoline stack layout:
25ad10658dc1068 Pu Lehui 2023-07-21 1043 *
25ad10658dc1068 Pu Lehui 2023-07-21 1044 * 1. trampoline called from function entry
25ad10658dc1068 Pu Lehui 2023-07-21 1045 * --------------------------------------
25ad10658dc1068 Pu Lehui 2023-07-21 1046 * FP + 8 [ RA to parent func ] return address to parent
25ad10658dc1068 Pu Lehui 2023-07-21 1047 * function
25ad10658dc1068 Pu Lehui 2023-07-21 1048 * FP + 0 [ FP of parent func ] frame pointer of parent
25ad10658dc1068 Pu Lehui 2023-07-21 1049 * function
25ad10658dc1068 Pu Lehui 2023-07-21 1050 * FP - 8 [ T0 to traced func ] return address of traced
25ad10658dc1068 Pu Lehui 2023-07-21 1051 * function
25ad10658dc1068 Pu Lehui 2023-07-21 1052 * FP - 16 [ FP of traced func ] frame pointer of traced
25ad10658dc1068 Pu Lehui 2023-07-21 1053 * function
25ad10658dc1068 Pu Lehui 2023-07-21 1054 * --------------------------------------
49b5e77ae3e214a Pu Lehui 2023-02-15 1055 *
25ad10658dc1068 Pu Lehui 2023-07-21 1056 * 2. trampoline called directly
25ad10658dc1068 Pu Lehui 2023-07-21 1057 * --------------------------------------
25ad10658dc1068 Pu Lehui 2023-07-21 1058 * FP - 8 [ RA to caller func ] return address to caller
49b5e77ae3e214a Pu Lehui 2023-02-15 1059 * function
25ad10658dc1068 Pu Lehui 2023-07-21 1060 * FP - 16 [ FP of caller func ] frame pointer of caller
49b5e77ae3e214a Pu Lehui 2023-02-15 1061 * function
25ad10658dc1068 Pu Lehui 2023-07-21 1062 * --------------------------------------
49b5e77ae3e214a Pu Lehui 2023-02-15 1063 *
49b5e77ae3e214a Pu Lehui 2023-02-15 1064 * FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or
49b5e77ae3e214a Pu Lehui 2023-02-15 1065 * BPF_TRAMP_F_RET_FENTRY_RET
49b5e77ae3e214a Pu Lehui 2023-02-15 1066 * [ argN ]
49b5e77ae3e214a Pu Lehui 2023-02-15 1067 * [ ... ]
49b5e77ae3e214a Pu Lehui 2023-02-15 1068 * FP - args_off [ arg1 ]
49b5e77ae3e214a Pu Lehui 2023-02-15 1069 *
35b3515be0ecb9d Menglong Dong 2026-02-08 1070 * FP - func_meta_off [ regs count, etc ]
49b5e77ae3e214a Pu Lehui 2023-02-15 1071 *
49b5e77ae3e214a Pu Lehui 2023-02-15 1072 * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG
49b5e77ae3e214a Pu Lehui 2023-02-15 1073 *
35b3515be0ecb9d Menglong Dong 2026-02-08 1074 * [ stack cookie N ]
35b3515be0ecb9d Menglong Dong 2026-02-08 1075 * [ ... ]
35b3515be0ecb9d Menglong Dong 2026-02-08 1076 * FP - cookie_off [ stack cookie 1 ]
35b3515be0ecb9d Menglong Dong 2026-02-08 1077 *
49b5e77ae3e214a Pu Lehui 2023-02-15 1078 * FP - run_ctx_off [ bpf_tramp_run_ctx ]
49b5e77ae3e214a Pu Lehui 2023-02-15 1079 *
49b5e77ae3e214a Pu Lehui 2023-02-15 1080 * FP - sreg_off [ callee saved reg ]
49b5e77ae3e214a Pu Lehui 2023-02-15 1081 *
49b5e77ae3e214a Pu Lehui 2023-02-15 1082 * [ pads ] pads for 16 bytes alignment
6801b0aef79db47 Pu Lehui 2024-07-02 1083 *
6801b0aef79db47 Pu Lehui 2024-07-02 1084 * [ stack_argN ]
6801b0aef79db47 Pu Lehui 2024-07-02 1085 * [ ... ]
6801b0aef79db47 Pu Lehui 2024-07-02 1086 * FP - stk_arg_off [ stack_arg1 ] BPF_TRAMP_F_CALL_ORIG
49b5e77ae3e214a Pu Lehui 2023-02-15 1087 */
49b5e77ae3e214a Pu Lehui 2023-02-15 1088
49b5e77ae3e214a Pu Lehui 2023-02-15 1089 if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY))
49b5e77ae3e214a Pu Lehui 2023-02-15 1090 return -ENOTSUPP;
49b5e77ae3e214a Pu Lehui 2023-02-15 1091
6801b0aef79db47 Pu Lehui 2024-07-02 1092 if (m->nr_args > MAX_BPF_FUNC_ARGS)
49b5e77ae3e214a Pu Lehui 2023-02-15 1093 return -ENOTSUPP;
49b5e77ae3e214a Pu Lehui 2023-02-15 1094
6801b0aef79db47 Pu Lehui 2024-07-02 1095 for (i = 0; i < m->nr_args; i++)
6801b0aef79db47 Pu Lehui 2024-07-02 1096 nr_arg_slots += round_up(m->arg_size[i], 8) / 8;
6801b0aef79db47 Pu Lehui 2024-07-02 1097
25ad10658dc1068 Pu Lehui 2023-07-21 1098 /* room of trampoline frame to store return address and frame pointer */
25ad10658dc1068 Pu Lehui 2023-07-21 1099 stack_size += 16;
49b5e77ae3e214a Pu Lehui 2023-02-15 1100
49b5e77ae3e214a Pu Lehui 2023-02-15 1101 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
d0bf7cd5df18466 Chenghao Duan 2025-09-22 1102 if (save_ret)
7112cd26e606c7b Björn Töpel 2023-10-04 1103 stack_size += 16; /* Save both A5 (BPF R0) and A0 */
49b5e77ae3e214a Pu Lehui 2023-02-15 1104 retval_off = stack_size;
49b5e77ae3e214a Pu Lehui 2023-02-15 1105
6801b0aef79db47 Pu Lehui 2024-07-02 1106 stack_size += nr_arg_slots * 8;
49b5e77ae3e214a Pu Lehui 2023-02-15 1107 args_off = stack_size;
49b5e77ae3e214a Pu Lehui 2023-02-15 1108
35b3515be0ecb9d Menglong Dong 2026-02-08 1109 /* function metadata, such as regs count */
49b5e77ae3e214a Pu Lehui 2023-02-15 1110 stack_size += 8;
35b3515be0ecb9d Menglong Dong 2026-02-08 1111 func_meta_off = stack_size;
49b5e77ae3e214a Pu Lehui 2023-02-15 1112
49b5e77ae3e214a Pu Lehui 2023-02-15 1113 if (flags & BPF_TRAMP_F_IP_ARG) {
49b5e77ae3e214a Pu Lehui 2023-02-15 1114 stack_size += 8;
49b5e77ae3e214a Pu Lehui 2023-02-15 1115 ip_off = stack_size;
49b5e77ae3e214a Pu Lehui 2023-02-15 1116 }
49b5e77ae3e214a Pu Lehui 2023-02-15 1117
35b3515be0ecb9d Menglong Dong 2026-02-08 1118 cookie_cnt = bpf_fsession_cookie_cnt(tlinks);
35b3515be0ecb9d Menglong Dong 2026-02-08 1119 /* room for session cookies */
35b3515be0ecb9d Menglong Dong 2026-02-08 1120 stack_size += cookie_cnt * 8;
35b3515be0ecb9d Menglong Dong 2026-02-08 1121 cookie_off = stack_size;
35b3515be0ecb9d Menglong Dong 2026-02-08 1122
49b5e77ae3e214a Pu Lehui 2023-02-15 1123 stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8);
49b5e77ae3e214a Pu Lehui 2023-02-15 1124 run_ctx_off = stack_size;
49b5e77ae3e214a Pu Lehui 2023-02-15 1125
49b5e77ae3e214a Pu Lehui 2023-02-15 1126 stack_size += 8;
49b5e77ae3e214a Pu Lehui 2023-02-15 1127 sreg_off = stack_size;
49b5e77ae3e214a Pu Lehui 2023-02-15 1128
a5912c37faf723c Puranjay Mohan 2024-07-08 1129 if ((flags & BPF_TRAMP_F_CALL_ORIG) && (nr_arg_slots - RV_MAX_REG_ARGS > 0))
6801b0aef79db47 Pu Lehui 2024-07-02 1130 stack_size += (nr_arg_slots - RV_MAX_REG_ARGS) * 8;
6801b0aef79db47 Pu Lehui 2024-07-02 1131
e944fc8152744a4 Xiao Wang 2024-05-23 1132 stack_size = round_up(stack_size, STACK_ALIGN);
49b5e77ae3e214a Pu Lehui 2023-02-15 1133
6801b0aef79db47 Pu Lehui 2024-07-02 1134 /* room for args on stack must be at the top of stack */
6801b0aef79db47 Pu Lehui 2024-07-02 1135 stk_arg_off = stack_size;
6801b0aef79db47 Pu Lehui 2024-07-02 1136
1732ebc4a26181c Pu Lehui 2024-01-23 1137 if (!is_struct_ops) {
25ad10658dc1068 Pu Lehui 2023-07-21 1138 /* For the trampoline called from function entry,
25ad10658dc1068 Pu Lehui 2023-07-21 1139 * the frame of traced function and the frame of
25ad10658dc1068 Pu Lehui 2023-07-21 1140 * trampoline need to be considered.
25ad10658dc1068 Pu Lehui 2023-07-21 1141 */
25ad10658dc1068 Pu Lehui 2023-07-21 1142 emit_addi(RV_REG_SP, RV_REG_SP, -16, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1143 emit_sd(RV_REG_SP, 8, RV_REG_RA, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1144 emit_sd(RV_REG_SP, 0, RV_REG_FP, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1145 emit_addi(RV_REG_FP, RV_REG_SP, 16, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1146
25ad10658dc1068 Pu Lehui 2023-07-21 1147 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1148 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_T0, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1149 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1150 emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1151 } else {
e63985ecd22681c Puranjay Mohan 2024-03-03 1152 /* emit kcfi hash */
e63985ecd22681c Puranjay Mohan 2024-03-03 1153 emit_kcfi(cfi_get_func_hash(func_addr), ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1154 /* For the trampoline called directly, just handle
25ad10658dc1068 Pu Lehui 2023-07-21 1155 * the frame of trampoline.
25ad10658dc1068 Pu Lehui 2023-07-21 1156 */
25ad10658dc1068 Pu Lehui 2023-07-21 1157 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1158 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_RA, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1159 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1160 emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1161 }
49b5e77ae3e214a Pu Lehui 2023-02-15 1162
49b5e77ae3e214a Pu Lehui 2023-02-15 1163 /* callee saved register S1 to pass start time */
49b5e77ae3e214a Pu Lehui 2023-02-15 1164 emit_sd(RV_REG_FP, -sreg_off, RV_REG_S1, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1165
49b5e77ae3e214a Pu Lehui 2023-02-15 1166 /* store ip address of the traced function */
93fd420d71beed5 Menglong Dong 2026-02-08 1167 if (flags & BPF_TRAMP_F_IP_ARG)
93fd420d71beed5 Menglong Dong 2026-02-08 1168 emit_store_stack_imm64(RV_REG_T1, -ip_off, (u64)func_addr, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1169
35b3515be0ecb9d Menglong Dong 2026-02-08 1170 func_meta = nr_arg_slots;
35b3515be0ecb9d Menglong Dong 2026-02-08 1171 emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1172
6801b0aef79db47 Pu Lehui 2024-07-02 1173 store_args(nr_arg_slots, args_off, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1174
35b3515be0ecb9d Menglong Dong 2026-02-08 1175 if (bpf_fsession_cnt(tlinks)) {
35b3515be0ecb9d Menglong Dong 2026-02-08 1176 /* clear all session cookies' value */
35b3515be0ecb9d Menglong Dong 2026-02-08 1177 for (i = 0; i < cookie_cnt; i++)
35b3515be0ecb9d Menglong Dong 2026-02-08 1178 emit_sd(RV_REG_FP, -cookie_off + 8 * i, RV_REG_ZERO, ctx);
35b3515be0ecb9d Menglong Dong 2026-02-08 1179 /* clear return value to make sure fentry always get 0 */
35b3515be0ecb9d Menglong Dong 2026-02-08 1180 emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx);
35b3515be0ecb9d Menglong Dong 2026-02-08 1181 }
35b3515be0ecb9d Menglong Dong 2026-02-08 1182
49b5e77ae3e214a Pu Lehui 2023-02-15 1183 if (flags & BPF_TRAMP_F_CALL_ORIG) {
9f1e16fb1fc9826 Pu Lehui 2024-06-22 1184 emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1185 ret = emit_call((const u64)__bpf_tramp_enter, true, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1186 if (ret)
49b5e77ae3e214a Pu Lehui 2023-02-15 1187 return ret;
49b5e77ae3e214a Pu Lehui 2023-02-15 1188 }
49b5e77ae3e214a Pu Lehui 2023-02-15 1189
35b3515be0ecb9d Menglong Dong 2026-02-08 1190 if (fentry->nr_links) {
35b3515be0ecb9d Menglong Dong 2026-02-08 1191 ret = invoke_bpf(fentry, args_off, retval_off, run_ctx_off, func_meta_off,
35b3515be0ecb9d Menglong Dong 2026-02-08 1192 flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1193 if (ret)
49b5e77ae3e214a Pu Lehui 2023-02-15 1194 return ret;
49b5e77ae3e214a Pu Lehui 2023-02-15 1195 }
49b5e77ae3e214a Pu Lehui 2023-02-15 1196
49b5e77ae3e214a Pu Lehui 2023-02-15 1197 if (fmod_ret->nr_links) {
49b5e77ae3e214a Pu Lehui 2023-02-15 1198 branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL);
49b5e77ae3e214a Pu Lehui 2023-02-15 1199 if (!branches_off)
49b5e77ae3e214a Pu Lehui 2023-02-15 1200 return -ENOMEM;
49b5e77ae3e214a Pu Lehui 2023-02-15 1201
49b5e77ae3e214a Pu Lehui 2023-02-15 1202 /* cleanup to avoid garbage return value confusion */
49b5e77ae3e214a Pu Lehui 2023-02-15 1203 emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1204 for (i = 0; i < fmod_ret->nr_links; i++) {
49b5e77ae3e214a Pu Lehui 2023-02-15 1205 ret = invoke_bpf_prog(fmod_ret->links[i], args_off, retval_off,
49b5e77ae3e214a Pu Lehui 2023-02-15 1206 run_ctx_off, true, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1207 if (ret)
49b5e77ae3e214a Pu Lehui 2023-02-15 1208 goto out;
49b5e77ae3e214a Pu Lehui 2023-02-15 1209 emit_ld(RV_REG_T1, -retval_off, RV_REG_FP, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1210 branches_off[i] = ctx->ninsns;
49b5e77ae3e214a Pu Lehui 2023-02-15 1211 /* nop reserved for conditional jump */
49b5e77ae3e214a Pu Lehui 2023-02-15 1212 emit(rv_nop(), ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1213 }
49b5e77ae3e214a Pu Lehui 2023-02-15 1214 }
49b5e77ae3e214a Pu Lehui 2023-02-15 1215
49b5e77ae3e214a Pu Lehui 2023-02-15 1216 if (flags & BPF_TRAMP_F_CALL_ORIG) {
8f3e00af8e52c0d Menglong Dong 2025-12-19 1217 /* skip to actual body of traced function */
8f3e00af8e52c0d Menglong Dong 2025-12-19 1218 orig_call += RV_FENTRY_NINSNS * 4;
6801b0aef79db47 Pu Lehui 2024-07-02 1219 restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx);
6801b0aef79db47 Pu Lehui 2024-07-02 1220 restore_stack_args(nr_arg_slots - RV_MAX_REG_ARGS, args_off, stk_arg_off, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1221 ret = emit_call((const u64)orig_call, true, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1222 if (ret)
49b5e77ae3e214a Pu Lehui 2023-02-15 1223 goto out;
49b5e77ae3e214a Pu Lehui 2023-02-15 1224 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx);
7112cd26e606c7b Björn Töpel 2023-10-04 1225 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx);
2382a405c581ae8 Pu Lehui 2024-06-22 1226 im->ip_after_call = ctx->ro_insns + ctx->ninsns;
49b5e77ae3e214a Pu Lehui 2023-02-15 1227 /* 2 nops reserved for auipc+jalr pair */
49b5e77ae3e214a Pu Lehui 2023-02-15 1228 emit(rv_nop(), ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1229 emit(rv_nop(), ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1230 }
49b5e77ae3e214a Pu Lehui 2023-02-15 1231
49b5e77ae3e214a Pu Lehui 2023-02-15 1232 /* update branches saved in invoke_bpf_mod_ret with bnez */
49b5e77ae3e214a Pu Lehui 2023-02-15 1233 for (i = 0; ctx->insns && i < fmod_ret->nr_links; i++) {
49b5e77ae3e214a Pu Lehui 2023-02-15 1234 offset = ninsns_rvoff(ctx->ninsns - branches_off[i]);
49b5e77ae3e214a Pu Lehui 2023-02-15 1235 insn = rv_bne(RV_REG_T1, RV_REG_ZERO, offset >> 1);
49b5e77ae3e214a Pu Lehui 2023-02-15 1236 *(u32 *)(ctx->insns + branches_off[i]) = insn;
49b5e77ae3e214a Pu Lehui 2023-02-15 1237 }
49b5e77ae3e214a Pu Lehui 2023-02-15 1238
35b3515be0ecb9d Menglong Dong 2026-02-08 1239 /* set "is_return" flag for fsession */
35b3515be0ecb9d Menglong Dong 2026-02-08 1240 func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT);
35b3515be0ecb9d Menglong Dong 2026-02-08 1241 if (bpf_fsession_cnt(tlinks))
35b3515be0ecb9d Menglong Dong 2026-02-08 1242 emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx);
35b3515be0ecb9d Menglong Dong 2026-02-08 1243
35b3515be0ecb9d Menglong Dong 2026-02-08 1244 if (fexit->nr_links) {
35b3515be0ecb9d Menglong Dong 2026-02-08 1245 ret = invoke_bpf(fexit, args_off, retval_off, run_ctx_off, func_meta_off,
35b3515be0ecb9d Menglong Dong 2026-02-08 1246 false, func_meta, cookie_off, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1247 if (ret)
49b5e77ae3e214a Pu Lehui 2023-02-15 1248 goto out;
49b5e77ae3e214a Pu Lehui 2023-02-15 1249 }
49b5e77ae3e214a Pu Lehui 2023-02-15 1250
49b5e77ae3e214a Pu Lehui 2023-02-15 1251 if (flags & BPF_TRAMP_F_CALL_ORIG) {
2382a405c581ae8 Pu Lehui 2024-06-22 1252 im->ip_epilogue = ctx->ro_insns + ctx->ninsns;
9f1e16fb1fc9826 Pu Lehui 2024-06-22 1253 emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1254 ret = emit_call((const u64)__bpf_tramp_exit, true, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1255 if (ret)
49b5e77ae3e214a Pu Lehui 2023-02-15 1256 goto out;
49b5e77ae3e214a Pu Lehui 2023-02-15 1257 }
49b5e77ae3e214a Pu Lehui 2023-02-15 1258
49b5e77ae3e214a Pu Lehui 2023-02-15 1259 if (flags & BPF_TRAMP_F_RESTORE_REGS)
6801b0aef79db47 Pu Lehui 2024-07-02 1260 restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1261
7112cd26e606c7b Björn Töpel 2023-10-04 1262 if (save_ret) {
7112cd26e606c7b Björn Töpel 2023-10-04 1263 emit_ld(regmap[BPF_REG_0], -(retval_off - 8), RV_REG_FP, ctx);
fd2e08128944a76 Hengqi Chen 2025-09-08 1264 if (is_struct_ops) {
fd2e08128944a76 Hengqi Chen 2025-09-08 1265 ret = sign_extend(RV_REG_A0, regmap[BPF_REG_0], m->ret_size,
fd2e08128944a76 Hengqi Chen 2025-09-08 1266 m->ret_flags & BTF_FMODEL_SIGNED_ARG, ctx);
fd2e08128944a76 Hengqi Chen 2025-09-08 1267 if (ret)
fd2e08128944a76 Hengqi Chen 2025-09-08 1268 goto out;
fd2e08128944a76 Hengqi Chen 2025-09-08 1269 } else {
fd2e08128944a76 Hengqi Chen 2025-09-08 1270 emit_ld(RV_REG_A0, -retval_off, RV_REG_FP, ctx);
fd2e08128944a76 Hengqi Chen 2025-09-08 1271 }
7112cd26e606c7b Björn Töpel 2023-10-04 1272 }
49b5e77ae3e214a Pu Lehui 2023-02-15 1273
49b5e77ae3e214a Pu Lehui 2023-02-15 1274 emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1275
1732ebc4a26181c Pu Lehui 2024-01-23 1276 if (!is_struct_ops) {
25ad10658dc1068 Pu Lehui 2023-07-21 1277 /* trampoline called from function entry */
25ad10658dc1068 Pu Lehui 2023-07-21 1278 emit_ld(RV_REG_T0, stack_size - 8, RV_REG_SP, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1279 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1280 emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1281
25ad10658dc1068 Pu Lehui 2023-07-21 1282 emit_ld(RV_REG_RA, 8, RV_REG_SP, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1283 emit_ld(RV_REG_FP, 0, RV_REG_SP, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1284 emit_addi(RV_REG_SP, RV_REG_SP, 16, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1285
49b5e77ae3e214a Pu Lehui 2023-02-15 1286 if (flags & BPF_TRAMP_F_SKIP_FRAME)
25ad10658dc1068 Pu Lehui 2023-07-21 1287 /* return to parent function */
25ad10658dc1068 Pu Lehui 2023-07-21 1288 emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1289 else
25ad10658dc1068 Pu Lehui 2023-07-21 1290 /* return to traced function */
25ad10658dc1068 Pu Lehui 2023-07-21 1291 emit_jalr(RV_REG_ZERO, RV_REG_T0, 0, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1292 } else {
25ad10658dc1068 Pu Lehui 2023-07-21 1293 /* trampoline called directly */
25ad10658dc1068 Pu Lehui 2023-07-21 1294 emit_ld(RV_REG_RA, stack_size - 8, RV_REG_SP, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1295 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1296 emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx);
49b5e77ae3e214a Pu Lehui 2023-02-15 1297
49b5e77ae3e214a Pu Lehui 2023-02-15 1298 emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx);
25ad10658dc1068 Pu Lehui 2023-07-21 1299 }
49b5e77ae3e214a Pu Lehui 2023-02-15 1300
49b5e77ae3e214a Pu Lehui 2023-02-15 1301 ret = ctx->ninsns;
49b5e77ae3e214a Pu Lehui 2023-02-15 1302 out:
49b5e77ae3e214a Pu Lehui 2023-02-15 1303 kfree(branches_off);
49b5e77ae3e214a Pu Lehui 2023-02-15 1304 return ret;
49b5e77ae3e214a Pu Lehui 2023-02-15 1305 }
49b5e77ae3e214a Pu Lehui 2023-02-15 1306
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply
* [PATCH 0/2] fgraph: fixes for thresh_return return handler
From: hu.shengming @ 2026-02-21 3:16 UTC (permalink / raw)
To: rostedt, mhiramat
Cc: mathieu.desnoyers, linux-kernel, linux-trace-kernel, zhang.run,
yang.tao172, yang.yang29, hu.shengming
[-- Attachment #1.1.1: Type: text/plain, Size: 729 bytes --]
From: Shengming Hu <hu.shengming@zte.com.cn>
Hi,
This series fixes two issues in trace_graph_thresh_return(), the function
graph return handler used when tracing_thresh is enabled:
PATCH1: Clear the per-task TRACE_GRAPH_NOTRACE state like
trace_graph_return() does.
PATCH2: Avoid double no-sleep-time adjustment by emitting
the return event directly when the threshold is met.
Patch details are in the individual commit messages.
Shengming Hu (2):
[PATCH 1/2] fgraph: fix thresh_return clear per-task notrace
[PATCH 2/2] fgraph: fix thresh_return nosleeptime double-adjust
kernel/trace/trace_functions_graph.c | 19 +++++++++++++------
1 file changed, 13 insertions(+), 6 deletions(-)
--
2.25.1
[-- Attachment #1.1.2: Type: text/html , Size: 1619 bytes --]
^ permalink raw reply
* [PATCH 1/2] fgraph: fix thresh_return clear per-task notrace
From: hu.shengming @ 2026-02-21 3:30 UTC (permalink / raw)
To: hu.shengming
Cc: rostedt, mhiramat, mathieu.desnoyers, linux-kernel,
linux-trace-kernel, zhang.run, yang.tao172, yang.yang29
In-Reply-To: <202602211116123227p4Je6QELxr2byqvy9kTE@zte.com.cn>
From: Shengming Hu <hu.shengming@zte.com.cn>
When tracing_thresh is enabled, function graph tracing uses
trace_graph_thresh_return() as the return handler. Unlike
trace_graph_return(), it did not clear the per-task
TRACE_GRAPH_NOTRACE flag set by the entry handler for
set_graph_notrace addresses. This could leave the task
permanently in "notrace" state and effectively disable
function graph tracing for that task.
Mirror trace_graph_return()'s per-task notrace handling by
clearing TRACE_GRAPH_NOTRACE and returning early when set.
Fixes: b84214890a9bc ("function_graph: Move graph notrace bit to
shadow stack global var")
Signed-off-by: Shengming Hu <hu.shengming@zte.com.cn>
---
kernel/trace/trace_functions_graph.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 1de6f1573..cbe43680c 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -400,14 +400,15 @@ static void trace_graph_thresh_return(struct ftrace_graph_ret *trace,
struct fgraph_ops *gops,
struct ftrace_regs *fregs)
{
+ unsigned long *task_var = fgraph_get_task_var(gops);
struct fgraph_times *ftimes;
struct trace_array *tr;
int size;
ftrace_graph_addr_finish(gops, trace);
- if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) {
- trace_recursion_clear(TRACE_GRAPH_NOTRACE_BIT);
+ if (*task_var & TRACE_GRAPH_NOTRACE) {
+ *task_var &= ~TRACE_GRAPH_NOTRACE;
return;
}
--
2.25.1
^ permalink raw reply related
* [PATCH 2/2] fgraph: fix thresh_return nosleeptime double-adjust
From: hu.shengming @ 2026-02-21 3:33 UTC (permalink / raw)
To: hu.shengming
Cc: rostedt, mhiramat, mathieu.desnoyers, linux-kernel,
linux-trace-kernel, zhang.run, yang.tao172, yang.yang29
In-Reply-To: <202602211116123227p4Je6QELxr2byqvy9kTE@zte.com.cn>
From: Shengming Hu <hu.shengming@zte.com.cn>
trace_graph_thresh_return() called handle_nosleeptime() and then
delegated to trace_graph_return(), which calls
handle_nosleeptime() again. When sleep-time accounting is
disabled this double-adjusts calltime and can produce bogus
durations (including underflow).
Fix this by computing rettime once, applying
handle_nosleeptime() only once, using the adjusted calltime
for threshold comparison, and writing the return event
directly via __trace_graph_return() when the threshold is met.
Fixes: 3c9880f3ab52b ("ftrace: Use a running sleeptime instead of
saving on shadow stack")
Signed-off-by: Shengming Hu <hu.shengming@zte.com.cn>
---
kernel/trace/trace_functions_graph.c | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index cbe43680c..b9c81fbd9 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -403,8 +403,12 @@ static void trace_graph_thresh_return(struct ftrace_graph_ret *trace,
unsigned long *task_var = fgraph_get_task_var(gops);
struct fgraph_times *ftimes;
struct trace_array *tr;
+ unsigned int trace_ctx;
+ u64 calltime, rettime;
int size;
+ rettime = trace_clock_local();
+
ftrace_graph_addr_finish(gops, trace);
if (*task_var & TRACE_GRAPH_NOTRACE) {
@@ -419,11 +423,13 @@ static void trace_graph_thresh_return(struct ftrace_graph_ret *trace,
tr = gops->private;
handle_nosleeptime(tr, trace, ftimes, size);
- if (tracing_thresh &&
- (trace_clock_local() - ftimes->calltime < tracing_thresh))
+ calltime = ftimes->calltime;
+
+ if (tracing_thresh && (rettime - calltime < tracing_thresh))
return;
- else
- trace_graph_return(trace, gops, fregs);
+
+ trace_ctx = tracing_gen_ctx();
+ __trace_graph_return(tr, trace, trace_ctx, calltime, rettime);
}
static struct fgraph_ops funcgraph_ops = {
--
2.25.1
^ permalink raw reply related
* [syzbot] [trace?] [bpf?] KASAN: slab-use-after-free Read in bpf_trace_run2 (3)
From: syzbot @ 2026-02-21 4:46 UTC (permalink / raw)
To: andrii, ast, bpf, daniel, eddyz87, haoluo, john.fastabend, jolsa,
kpsingh, linux-kernel, linux-trace-kernel, martin.lau,
mathieu.desnoyers, mattbobrowski, mhiramat, rostedt, sdf, song,
syzkaller-bugs, yonghong.song
Hello,
syzbot found the following issue on:
HEAD commit: 8bf22c33e7a1 Merge tag 'net-7.0-rc1' of git://git.kernel.o..
git tree: upstream
console output: https://syzkaller.appspot.com/x/log.txt?x=17c8c73a580000
kernel config: https://syzkaller.appspot.com/x/.config?x=65722f41f7edc17e
dashboard link: https://syzkaller.appspot.com/bug?extid=59701a78e84b0bccfe1b
compiler: Debian clang version 21.1.8 (++20251221033036+2078da43e25a-1~exp1~20251221153213.50), Debian LLD 21.1.8
syz repro: https://syzkaller.appspot.com/x/repro.syz?x=141f7ffa580000
Downloadable assets:
disk image (non-bootable): https://storage.googleapis.com/syzbot-assets/d900f083ada3/non_bootable_disk-8bf22c33.raw.xz
vmlinux: https://storage.googleapis.com/syzbot-assets/ee3f5bf71c12/vmlinux-8bf22c33.xz
kernel image: https://storage.googleapis.com/syzbot-assets/ef5b82c2d846/bzImage-8bf22c33.xz
IMPORTANT: if you fix the issue, please add the following tag to the commit:
Reported-by: syzbot+59701a78e84b0bccfe1b@syzkaller.appspotmail.com
==================================================================
BUG: KASAN: slab-use-after-free in __bpf_trace_run kernel/trace/bpf_trace.c:2075 [inline]
BUG: KASAN: slab-use-after-free in bpf_trace_run2+0xb1/0x840 kernel/trace/bpf_trace.c:2129
Read of size 8 at addr ffff888053465b18 by task syz-executor/5307
CPU: 0 UID: 0 PID: 5307 Comm: syz-executor Not tainted syzkaller #0 PREEMPT(full)
Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.16.3-debian-1.16.3-2 04/01/2014
Call Trace:
<TASK>
dump_stack_lvl+0xe8/0x150 lib/dump_stack.c:120
print_address_description mm/kasan/report.c:378 [inline]
print_report+0xba/0x230 mm/kasan/report.c:482
kasan_report+0x117/0x150 mm/kasan/report.c:595
__bpf_trace_run kernel/trace/bpf_trace.c:2075 [inline]
bpf_trace_run2+0xb1/0x840 kernel/trace/bpf_trace.c:2129
__traceiter_kfree+0x2e/0x50 include/trace/events/kmem.h:97
__do_trace_kfree include/trace/events/kmem.h:97 [inline]
trace_kfree include/trace/events/kmem.h:97 [inline]
kfree+0x5b2/0x630 mm/slub.c:6428
futex_hash_free+0x65/0xb0 kernel/futex/core.c:1736
__mmput+0x38d/0x430 kernel/fork.c:1185
exit_mm+0x168/0x220 kernel/exit.c:581
do_exit+0x62e/0x2320 kernel/exit.c:959
do_group_exit+0x21b/0x2d0 kernel/exit.c:1112
get_signal+0x1284/0x1330 kernel/signal.c:3034
arch_do_signal_or_restart+0xbc/0x830 arch/x86/kernel/signal.c:337
__exit_to_user_mode_loop kernel/entry/common.c:64 [inline]
exit_to_user_mode_loop+0x86/0x480 kernel/entry/common.c:98
__exit_to_user_mode_prepare include/linux/irq-entry-common.h:226 [inline]
syscall_exit_to_user_mode_prepare include/linux/irq-entry-common.h:256 [inline]
syscall_exit_to_user_mode include/linux/entry-common.h:325 [inline]
do_syscall_64+0x32d/0xf80 arch/x86/entry/syscall_64.c:100
entry_SYSCALL_64_after_hwframe+0x77/0x7f
RIP: 0033:0x7fc4bbd57817
Code: Unable to access opcode bytes at 0x7fc4bbd577ed.
RSP: 002b:00007ffe1f8e8350 EFLAGS: 00000202 ORIG_RAX: 0000000000000001
RAX: 00000000000000b0 RBX: 0000555576290500 RCX: 00007fc4bbd57817
RDX: 00000000000000b0 RSI: 00007fc4b863ff50 RDI: 0000000000000003
RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000000 R11: 0000000000000202 R12: 00000000000000b0
R13: 00005555762a51c0 R14: 00007ffe1f8e88f0 R15: 00007fc4b863ff50
</TASK>
Allocated by task 6102:
kasan_save_stack mm/kasan/common.c:57 [inline]
kasan_save_track+0x3e/0x80 mm/kasan/common.c:78
poison_kmalloc_redzone mm/kasan/common.c:398 [inline]
__kasan_kmalloc+0x93/0xb0 mm/kasan/common.c:415
kasan_kmalloc include/linux/kasan.h:263 [inline]
__kmalloc_cache_noprof+0x31c/0x660 mm/slub.c:5339
kmalloc_noprof include/linux/slab.h:962 [inline]
kzalloc_noprof include/linux/slab.h:1204 [inline]
bpf_raw_tp_link_attach+0x278/0x700 kernel/bpf/syscall.c:4264
bpf_raw_tracepoint_open+0x1b2/0x220 kernel/bpf/syscall.c:4312
__sys_bpf+0x846/0x950 kernel/bpf/syscall.c:6271
__do_sys_bpf kernel/bpf/syscall.c:6342 [inline]
__se_sys_bpf kernel/bpf/syscall.c:6340 [inline]
__x64_sys_bpf+0x7c/0x90 kernel/bpf/syscall.c:6340
do_syscall_x64 arch/x86/entry/syscall_64.c:63 [inline]
do_syscall_64+0x14d/0xf80 arch/x86/entry/syscall_64.c:94
entry_SYSCALL_64_after_hwframe+0x77/0x7f
Freed by task 1100:
kasan_save_stack mm/kasan/common.c:57 [inline]
kasan_save_track+0x3e/0x80 mm/kasan/common.c:78
kasan_save_free_info+0x46/0x50 mm/kasan/generic.c:584
poison_slab_object mm/kasan/common.c:253 [inline]
__kasan_slab_free+0x5c/0x80 mm/kasan/common.c:285
kasan_slab_free include/linux/kasan.h:235 [inline]
slab_free_hook mm/slub.c:2687 [inline]
slab_free mm/slub.c:6124 [inline]
kfree+0x1c1/0x630 mm/slub.c:6442
rcu_do_batch kernel/rcu/tree.c:2617 [inline]
rcu_core+0x7cd/0x1070 kernel/rcu/tree.c:2869
handle_softirqs+0x22a/0x870 kernel/softirq.c:622
do_softirq+0x76/0xd0 kernel/softirq.c:523
__local_bh_enable_ip+0xf8/0x130 kernel/softirq.c:450
spin_unlock_bh include/linux/spinlock.h:395 [inline]
batadv_iv_ogm_queue_add+0x73e/0xd30 net/batman-adv/bat_iv_ogm.c:668
batadv_iv_ogm_schedule_buff net/batman-adv/bat_iv_ogm.c:841 [inline]
batadv_iv_ogm_schedule+0x874/0xf50 net/batman-adv/bat_iv_ogm.c:873
batadv_iv_send_outstanding_bat_ogm_packet+0x6c8/0x7e0 net/batman-adv/bat_iv_ogm.c:1709
process_one_work kernel/workqueue.c:3275 [inline]
process_scheduled_works+0xb02/0x1830 kernel/workqueue.c:3358
worker_thread+0xa50/0xfc0 kernel/workqueue.c:3439
kthread+0x388/0x470 kernel/kthread.c:467
ret_from_fork+0x51e/0xb90 arch/x86/kernel/process.c:158
ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
Last potentially related work creation:
kasan_save_stack+0x3e/0x60 mm/kasan/common.c:57
kasan_record_aux_stack+0xbd/0xd0 mm/kasan/generic.c:556
__call_rcu_common kernel/rcu/tree.c:3131 [inline]
call_rcu+0xee/0x890 kernel/rcu/tree.c:3251
bpf_link_put_direct kernel/bpf/syscall.c:3323 [inline]
bpf_link_release+0x6b/0x80 kernel/bpf/syscall.c:3330
__fput+0x44f/0xa70 fs/file_table.c:469
task_work_run+0x1d9/0x270 kernel/task_work.c:233
resume_user_mode_work include/linux/resume_user_mode.h:50 [inline]
__exit_to_user_mode_loop kernel/entry/common.c:67 [inline]
exit_to_user_mode_loop+0xed/0x480 kernel/entry/common.c:98
__exit_to_user_mode_prepare include/linux/irq-entry-common.h:226 [inline]
syscall_exit_to_user_mode_prepare include/linux/irq-entry-common.h:256 [inline]
syscall_exit_to_user_mode include/linux/entry-common.h:325 [inline]
do_syscall_64+0x32d/0xf80 arch/x86/entry/syscall_64.c:100
entry_SYSCALL_64_after_hwframe+0x77/0x7f
The buggy address belongs to the object at ffff888053465b00
which belongs to the cache kmalloc-192 of size 192
The buggy address is located 24 bytes inside of
freed 192-byte region [ffff888053465b00, ffff888053465bc0)
The buggy address belongs to the physical page:
page: refcount:0 mapcount:0 mapping:0000000000000000 index:0x0 pfn:0x53465
flags: 0x4fff00000000000(node=1|zone=1|lastcpupid=0x7ff)
page_type: f5(slab)
raw: 04fff00000000000 ffff88801a8413c0 dead000000000100 dead000000000122
raw: 0000000000000000 0000000000100010 00000000f5000000 0000000000000000
page dumped because: kasan: bad access detected
page_owner tracks the page as allocated
page last allocated via order 0, migratetype Unmovable, gfp_mask 0xd2cc0(GFP_KERNEL|__GFP_NOWARN|__GFP_NORETRY|__GFP_COMP|__GFP_NOMEMALLOC), pid 5436, tgid 5436 (syz-executor), ts 104947827691, free_ts 103695571452
set_page_owner include/linux/page_owner.h:32 [inline]
post_alloc_hook+0x231/0x280 mm/page_alloc.c:1889
prep_new_page mm/page_alloc.c:1897 [inline]
get_page_from_freelist+0x24dc/0x2580 mm/page_alloc.c:3962
__alloc_frozen_pages_noprof+0x18d/0x380 mm/page_alloc.c:5250
alloc_slab_page mm/slub.c:3255 [inline]
allocate_slab+0x77/0x660 mm/slub.c:3444
new_slab mm/slub.c:3502 [inline]
refill_objects+0x331/0x3c0 mm/slub.c:7134
refill_sheaf mm/slub.c:2804 [inline]
__pcs_replace_empty_main+0x2b9/0x620 mm/slub.c:4578
alloc_from_pcs mm/slub.c:4681 [inline]
slab_alloc_node mm/slub.c:4815 [inline]
__do_kmalloc_node mm/slub.c:5218 [inline]
__kmalloc_node_track_caller_noprof+0x572/0x7b0 mm/slub.c:5327
kmemdup_noprof+0x2b/0x70 mm/util.c:138
kmemdup_noprof include/linux/fortify-string.h:763 [inline]
neigh_parms_alloc+0x87/0x510 net/core/neighbour.c:1772
ipv6_add_dev+0x40d/0x13a0 net/ipv6/addrconf.c:403
addrconf_notify+0x771/0x1050 net/ipv6/addrconf.c:3655
notifier_call_chain+0x1be/0x400 kernel/notifier.c:85
call_netdevice_notifiers_extack net/core/dev.c:2287 [inline]
call_netdevice_notifiers net/core/dev.c:2301 [inline]
register_netdevice+0x173a/0x1cf0 net/core/dev.c:11451
ip6gre_newlink_common+0x403/0x5b0 net/ipv6/ip6_gre.c:1965
ip6gre_newlink+0x228/0x350 net/ipv6/ip6_gre.c:1998
rtnl_newlink_create+0x329/0xb70 net/core/rtnetlink.c:3840
page last free pid 5378 tgid 5378 stack trace:
reset_page_owner include/linux/page_owner.h:25 [inline]
__free_pages_prepare mm/page_alloc.c:1433 [inline]
__free_frozen_pages+0xc2b/0xdb0 mm/page_alloc.c:2978
vfree+0x25a/0x400 mm/vmalloc.c:3479
kcov_put kernel/kcov.c:442 [inline]
kcov_close+0x28/0x50 kernel/kcov.c:543
__fput+0x44f/0xa70 fs/file_table.c:469
task_work_run+0x1d9/0x270 kernel/task_work.c:233
exit_task_work include/linux/task_work.h:40 [inline]
do_exit+0x69b/0x2320 kernel/exit.c:971
do_group_exit+0x21b/0x2d0 kernel/exit.c:1112
get_signal+0x1284/0x1330 kernel/signal.c:3034
arch_do_signal_or_restart+0xbc/0x830 arch/x86/kernel/signal.c:337
__exit_to_user_mode_loop kernel/entry/common.c:64 [inline]
exit_to_user_mode_loop+0x86/0x480 kernel/entry/common.c:98
__exit_to_user_mode_prepare include/linux/irq-entry-common.h:226 [inline]
syscall_exit_to_user_mode_prepare include/linux/irq-entry-common.h:256 [inline]
syscall_exit_to_user_mode include/linux/entry-common.h:325 [inline]
do_syscall_64+0x32d/0xf80 arch/x86/entry/syscall_64.c:100
entry_SYSCALL_64_after_hwframe+0x77/0x7f
Memory state around the buggy address:
ffff888053465a00: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
ffff888053465a80: 00 00 00 fc fc fc fc fc fc fc fc fc fc fc fc fc
>ffff888053465b00: fa fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb
^
ffff888053465b80: fb fb fb fb fb fb fb fb fc fc fc fc fc fc fc fc
ffff888053465c00: fa fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb
==================================================================
---
This report is generated by a bot. It may contain errors.
See https://goo.gl/tpsmEJ for more information about syzbot.
syzbot engineers can be reached at syzkaller@googlegroups.com.
syzbot will keep track of this issue. See:
https://goo.gl/tpsmEJ#status for how to communicate with syzbot.
If the report is already addressed, let syzbot know by replying with:
#syz fix: exact-commit-title
If you want syzbot to run the reproducer, reply with:
#syz test: git://repo/address.git branch-or-commit-hash
If you attach or paste a git patch, syzbot will apply it before testing.
If you want to overwrite report's subsystems, reply with:
#syz set subsystems: new-subsystem
(See the list of subsystem names on the web dashboard)
If the report is a duplicate of another one, reply with:
#syz dup: exact-subject-of-another-report
If you want to undo deduplication, reply with:
#syz undup
^ permalink raw reply
* Re: [PATCH v4 10/12] riscv: Implement HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS
From: Conor Dooley @ 2026-02-21 12:15 UTC (permalink / raw)
To: Andy Chiu
Cc: linux-riscv, alexghiti, palmer, Puranjay Mohan,
Björn Töpel, linux-kernel, linux-trace-kernel,
Alexandre Ghiti, Mark Rutland, paul.walmsley, greentime.hu,
nick.hu, nylon.chen, eric.lin, vicent.chen, zong.li,
yongxuan.wang, samuel.holland, olivia.chu, c2232430, arnd,
Sami Tolvanen, Kees Cook, Nathan Chancellor, llvm, pjw
In-Reply-To: <20250407180838.42877-10-andybnac@gmail.com>
[-- Attachment #1: Type: text/plain, Size: 7719 bytes --]
Hey,
On Tue, Apr 08, 2025 at 02:08:34AM +0800, Andy Chiu wrote:
> From: Puranjay Mohan <puranjay12@gmail.com>
>
> This patch enables support for DYNAMIC_FTRACE_WITH_CALL_OPS on RISC-V.
> This allows each ftrace callsite to provide an ftrace_ops to the common
> ftrace trampoline, allowing each callsite to invoke distinct tracer
> functions without the need to fall back to list processing or to
> allocate custom trampolines for each callsite. This significantly speeds
> up cases where multiple distinct trace functions are used and callsites
> are mostly traced by a single tracer.
>
> The idea and most of the implementation is taken from the ARM64's
> implementation of the same feature. The idea is to place a pointer to
> the ftrace_ops as a literal at a fixed offset from the function entry
> point, which can be recovered by the common ftrace trampoline.
>
> We use -fpatchable-function-entry to reserve 8 bytes above the function
> entry by emitting 2 4 byte or 4 2 byte nops depending on the presence of
> CONFIG_RISCV_ISA_C. These 8 bytes are patched at runtime with a pointer
> to the associated ftrace_ops for that callsite. Functions are aligned to
> 8 bytes to make sure that the accesses to this literal are atomic.
>
> This approach allows for directly invoking ftrace_ops::func even for
> ftrace_ops which are dynamically-allocated (or part of a module),
> without going via ftrace_ops_list_func.
>
> We've benchamrked this with the ftrace_ops sample module on Spacemit K1
> Jupiter:
>
> Without this patch:
>
> baseline (Linux rivos 6.14.0-09584-g7d06015d936c #3 SMP Sat Mar 29
> +-----------------------+-----------------+----------------------------+
> | Number of tracers | Total time (ns) | Per-call average time |
> |-----------------------+-----------------+----------------------------|
> | Relevant | Irrelevant | 100000 calls | Total (ns) | Overhead (ns) |
> |----------+------------+-----------------+------------+---------------|
> | 0 | 0 | 1357958 | 13 | - |
> | 0 | 1 | 1302375 | 13 | - |
> | 0 | 2 | 1302375 | 13 | - |
> | 0 | 10 | 1379084 | 13 | - |
> | 0 | 100 | 1302458 | 13 | - |
> | 0 | 200 | 1302333 | 13 | - |
> |----------+------------+-----------------+------------+---------------|
> | 1 | 0 | 13677833 | 136 | 123 |
> | 1 | 1 | 18500916 | 185 | 172 |
> | 1 | 2 | 22856459 | 228 | 215 |
> | 1 | 10 | 58824709 | 588 | 575 |
> | 1 | 100 | 505141584 | 5051 | 5038 |
> | 1 | 200 | 1580473126 | 15804 | 15791 |
> |----------+------------+-----------------+------------+---------------|
> | 1 | 0 | 13561000 | 135 | 122 |
> | 2 | 0 | 19707292 | 197 | 184 |
> | 10 | 0 | 67774750 | 677 | 664 |
> | 100 | 0 | 714123125 | 7141 | 7128 |
> | 200 | 0 | 1918065668 | 19180 | 19167 |
> +----------+------------+-----------------+------------+---------------+
>
> Note: per-call overhead is estimated relative to the baseline case with
> 0 relevant tracers and 0 irrelevant tracers.
>
> With this patch:
>
> v4-rc4 (Linux rivos 6.14.0-09598-gd75747611c93 #4 SMP Sat Mar 29
> +-----------------------+-----------------+----------------------------+
> | Number of tracers | Total time (ns) | Per-call average time |
> |-----------------------+-----------------+----------------------------|
> | Relevant | Irrelevant | 100000 calls | Total (ns) | Overhead (ns) |
> |----------+------------+-----------------+------------+---------------|
> | 0 | 0 | 1459917 | 14 | - |
> | 0 | 1 | 1408000 | 14 | - |
> | 0 | 2 | 1383792 | 13 | - |
> | 0 | 10 | 1430709 | 14 | - |
> | 0 | 100 | 1383791 | 13 | - |
> | 0 | 200 | 1383750 | 13 | - |
> |----------+------------+-----------------+------------+---------------|
> | 1 | 0 | 5238041 | 52 | 38 |
> | 1 | 1 | 5228542 | 52 | 38 |
> | 1 | 2 | 5325917 | 53 | 40 |
> | 1 | 10 | 5299667 | 52 | 38 |
> | 1 | 100 | 5245250 | 52 | 39 |
> | 1 | 200 | 5238459 | 52 | 39 |
> |----------+------------+-----------------+------------+---------------|
> | 1 | 0 | 5239083 | 52 | 38 |
> | 2 | 0 | 19449417 | 194 | 181 |
> | 10 | 0 | 67718584 | 677 | 663 |
> | 100 | 0 | 709840708 | 7098 | 7085 |
> | 200 | 0 | 2203580626 | 22035 | 22022 |
> +----------+------------+-----------------+------------+---------------+
>
> Note: per-call overhead is estimated relative to the baseline case with
> 0 relevant tracers and 0 irrelevant tracers.
>
> As can be seen from the above:
>
> a) Whenever there is a single relevant tracer function associated with a
> tracee, the overhead of invoking the tracer is constant, and does not
> scale with the number of tracers which are *not* associated with that
> tracee.
>
> b) The overhead for a single relevant tracer has dropped to ~1/3 of the
> overhead prior to this series (from 122ns to 38ns). This is largely
> due to permitting calls to dynamically-allocated ftrace_ops without
> going through ftrace_ops_list_func.
>
> Signed-off-by: Puranjay Mohan <puranjay12@gmail.com>
>
> [update kconfig, asm, refactor]
>
> Signed-off-by: Andy Chiu <andybnac@gmail.com>
> Tested-by: Björn Töpel <bjorn@rivosinc.com>
I bisected a boot failure to this commit [c217157bcd1df ("riscv:
Implement HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS")] yesterday, that appears
to be affecting all LLVM versions that I currently have installed. From
some initial testing of Kconfig options, it looks like the issue is
CFI_CLANG related because when I disable CFI_CLANG things work once
more. Since this option depends on !CFI_CLANG, but is def_bool y, I
modified Kconfig to force disable it at all times and tested
!DYNAMIC_FTRACE_WITH_CALL_OPS && !CFG_CLANG, which did boot.
I dunno anything about what's going on in this patch, but so little in
it relates to having DYNAMIC_FTRACE_WITH_CALL_OPS, that I was able to
figure out that the problem is -fpatchable-function-entry=8,4
FWIW, if anyone checks out this commit directly, you'll need to
cherry-pick commit e9d86b8e17e72 ("scripts: Do not strip .rela.dyn
section"), as the base of the branch that c217157bcd1df is on is
v6.15-rc3, which is in itself broken in turn by the issue fixed by
e9d86b8e17e72. Probably not someone anyone will do, but made for an
awful time trying to figure out what commit was at fault!
Cheers,
Conor.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 228 bytes --]
^ permalink raw reply
* [syzbot] [net?] [trace?] WARNING: refcount bug in call_timer_fn (4)
From: syzbot @ 2026-02-21 22:24 UTC (permalink / raw)
To: linux-kernel, linux-trace-kernel, netdev, peterz, syzkaller-bugs,
tglx
Hello,
syzbot found the following issue on:
HEAD commit: 8b690556d8fe Merge tag 'for-linus' of git://git.kernel.org..
git tree: upstream
console output: https://syzkaller.appspot.com/x/log.txt?x=11715658580000
kernel config: https://syzkaller.appspot.com/x/.config?x=38a0c4cddc846161
dashboard link: https://syzkaller.appspot.com/bug?extid=07dcf509f4c013e25dc5
compiler: Debian clang version 20.1.8 (++20250708063551+0c9f909b7976-1~exp1~20250708183702.136), Debian LLD 20.1.8
Unfortunately, I don't have any reproducer for this issue yet.
Downloadable assets:
disk image: https://storage.googleapis.com/syzbot-assets/3ca2379f6d75/disk-8b690556.raw.xz
vmlinux: https://storage.googleapis.com/syzbot-assets/ce7e9a97f8b4/vmlinux-8b690556.xz
kernel image: https://storage.googleapis.com/syzbot-assets/0936d2c0b069/bzImage-8b690556.xz
IMPORTANT: if you fix the issue, please add the following tag to the commit:
Reported-by: syzbot+07dcf509f4c013e25dc5@syzkaller.appspotmail.com
------------[ cut here ]------------
refcount_t: underflow; use-after-free.
WARNING: CPU: 1 PID: 29 at lib/refcount.c:28 refcount_warn_saturate+0x11a/0x1d0 lib/refcount.c:28
Modules linked in:
CPU: 1 UID: 0 PID: 29 Comm: ktimers/1 Not tainted syzkaller #0 PREEMPT_{RT,(full)}
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 10/25/2025
RIP: 0010:refcount_warn_saturate+0x11a/0x1d0 lib/refcount.c:28
Code: c0 30 3d 8b e8 c7 91 09 fd 90 0f 0b 90 90 eb d7 e8 ab 4a 45 fd c6 05 10 1a 47 0a 01 90 48 c7 c7 20 31 3d 8b e8 a7 91 09 fd 90 <0f> 0b 90 90 eb b7 e8 8b 4a 45 fd c6 05 ed 19 47 0a 01 90 48 c7 c7
RSP: 0018:ffffc90000a3f888 EFLAGS: 00010246
RAX: 74da3aaac3757600 RBX: 0000000000000003 RCX: ffff88801be99e00
RDX: 0000000000000100 RSI: 0000000000000000 RDI: 0000000000000100
RBP: ffffc90000a3f990 R08: 0000000000000000 R09: 0000000000000100
R10: dffffc0000000000 R11: ffffed101712487b R12: 1ffff92000147f18
R13: ffff88805d5311b8 R14: ffff88805d531020 R15: 0000000000000001
FS: 0000000000000000(0000) GS:ffff888126ef7000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000200000404030 CR3: 000000000d3a6000 CR4: 00000000003526f0
Call Trace:
<TASK>
call_timer_fn+0x17e/0x5f0 kernel/time/timer.c:1747
expire_timers kernel/time/timer.c:1798 [inline]
__run_timers kernel/time/timer.c:2372 [inline]
__run_timer_base+0x648/0x970 kernel/time/timer.c:2384
run_timer_base kernel/time/timer.c:2393 [inline]
run_timer_softirq+0xb7/0x180 kernel/time/timer.c:2403
handle_softirqs+0x22f/0x710 kernel/softirq.c:622
__do_softirq kernel/softirq.c:656 [inline]
run_ktimerd+0xcf/0x190 kernel/softirq.c:1138
smpboot_thread_fn+0x542/0xa60 kernel/smpboot.c:160
kthread+0x711/0x8a0 kernel/kthread.c:463
ret_from_fork+0x4bc/0x870 arch/x86/kernel/process.c:158
ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
</TASK>
---
This report is generated by a bot. It may contain errors.
See https://goo.gl/tpsmEJ for more information about syzbot.
syzbot engineers can be reached at syzkaller@googlegroups.com.
syzbot will keep track of this issue. See:
https://goo.gl/tpsmEJ#status for how to communicate with syzbot.
If the report is already addressed, let syzbot know by replying with:
#syz fix: exact-commit-title
If you want to overwrite report's subsystems, reply with:
#syz set subsystems: new-subsystem
(See the list of subsystem names on the web dashboard)
If the report is a duplicate of another one, reply with:
#syz dup: exact-subject-of-another-report
If you want to undo deduplication, reply with:
#syz undup
^ permalink raw reply
* [LSF/MM/BPF TOPIC][RFC PATCH v4 00/27] Private Memory Nodes (w/ Compressed RAM)
From: Gregory Price @ 2026-02-22 8:48 UTC (permalink / raw)
To: lsf-pc
Cc: linux-kernel, linux-cxl, cgroups, linux-mm, linux-trace-kernel,
damon, kernel-team, gregkh, rafael, dakr, dave, jonathan.cameron,
dave.jiang, alison.schofield, vishal.l.verma, ira.weiny,
dan.j.williams, longman, akpm, david, lorenzo.stoakes,
Liam.Howlett, vbabka, rppt, surenb, mhocko, osalvador, ziy,
matthew.brost, joshua.hahnjy, rakie.kim, byungchul, gourry,
ying.huang, apopple, axelrasmussen, yuanchu, weixugc, yury.norov,
linux, mhiramat, mathieu.desnoyers, tj, hannes, mkoutny, jackmanb,
sj, baolin.wang, npache, ryan.roberts, dev.jain, baohua,
lance.yang, muchun.song, xu.xin16, chengming.zhou, jannh,
linmiaohe, nao.horiguchi, pfalcato, rientjes, shakeel.butt, riel,
harry.yoo, cl, roman.gushchin, chrisl, kasong, shikemeng, nphamcs,
bhe, zhengqi.arch, terry.bowman
Topic type: MM
Presenter: Gregory Price <gourry@gourry.net>
This series introduces N_MEMORY_PRIVATE, a NUMA node state for memory
managed by the buddy allocator but excluded from normal allocations.
I present it with an end-to-end Compressed RAM service (mm/cram.c)
that would otherwise not be possible (or would be considerably more
difficult, be device-specific, and add to the ZONE_DEVICE boondoggle).
TL;DR
===
N_MEMORY_PRIVATE is all about isolating NUMA nodes and then punching
explicit holes in that isolation to do useful things we couldn't do
before without re-implementing entire portions of mm/ in a driver.
/* This is my memory. There are many like it, but this one is mine. */
rc = add_private_memory_driver_managed(nid, start, size, name, flags,
online_type, private_context);
page = alloc_pages_node(nid, __GFP_PRIVATE, 0);
/* Ok but I want to do something useful with it */
static const struct node_private_ops ops = {
.migrate_to = my_migrate_to,
.folio_migrate = my_folio_migrate,
.flags = NP_OPS_MIGRATION | NP_OPS_MEMPOLICY,
};
node_private_set_ops(nid, &ops);
/* And now I can use mempolicy with my memory */
buf = mmap(...);
mbind(buf, len, mode, private_node, ...);
buf[0] = 0xdeadbeef; /* Faults onto private node */
/* And to be clear, no one else gets my memory */
buf2 = malloc(4096); /* Standard allocation */
buf2[0] = 0xdeadbeef; /* Can never land on private node */
/* But i can choose to migrate it to the private node */
move_pages(0, 1, &buf, &private_node, NULL, ...);
/* And more fun things like this */
Patchwork
===
A fully working branch based on cxl/next can be found here:
https://github.com/gourryinverse/linux/tree/private_compression
A QEMU device which can inject high/low interrupts can be found here:
https://github.com/gourryinverse/qemu/tree/compressed_cxl_clean
The additional patches on these branches are CXL and DAX driver
housecleaning only tangentially relevant to this RFC, so i've
omitted them for the sake of trying to keep it somewhat clean
here. Those patches should (hopefully) be going upstream anyway.
Patches 1-22: Core Private Node Infrastructure
Patch 1: Introduce N_MEMORY_PRIVATE scaffolding
Patch 2: Introduce __GFP_PRIVATE
Patch 3: Apply allocation isolation mechanisms
Patch 4: Add N_MEMORY nodes to private fallback lists
Patches 5-9: Filter operations not yet supported
Patch 10: free_folio callback
Patch 11: split_folio callback
Patches 12-20: mm/ service opt-ins:
Migration, Mempolicy, Demotion, Write Protect,
Reclaim, OOM, NUMA Balancing, Compaction,
LongTerm Pinning
Patch 21: memory_failure callback
Patch 22: Memory hotplug plumbing for private nodes
Patch 23: mm/cram -- Compressed RAM Management
Patches 24-27: CXL Driver examples
Sysram Regions with Private node support
Basic Driver Example: (MIGRATION | MEMPOLICY)
Compression Driver Example (Generic)
Background
===
Today, drivers that want mm-like services on non-general-purpose
memory either use ZONE_DEVICE (self-managed memory) or hotplug into
N_MEMORY and accept the risk of uncontrolled allocation.
Neither option provides what we really want - the ability to:
1) selectively participate in mm/ subsystems, while
2) isolating that memory from general purpose use.
Some device-attached memory cannot be managed as fully general-purpose
system RAM. CXL devices with inline compression, for example, may
corrupt data or crash the machine if the compression ratio drops
below a threshold -- we simply run out of physical memory.
This is a hard problem to solve: how does an operating system deal
with a device that basically lies about how much capacity it has?
(We'll discuss that in the CRAM section)
Core Proposal: N_MEMORY_PRIVATE
===
Introduce N_MEMORY_PRIVATE, a NUMA node state for memory managed by
the buddy allocator, but excluded from normal allocation paths.
Private nodes:
- Are filtered from zonelist fallback: all existing callers to
get_page_from_freelist cannot reach these nodes through any
normal fallback mechanism.
- Filter allocation requests on __GFP_PRIVATE
numa_zone_allowed() excludes them otherwise.
Applies to systems with and without cpusets.
GFP_PRIVATE is (__GFP_PRIVATE | __GFP_THISNODE).
Services use it when they need to allocate specifically from
a private node (e.g., CRAM allocating a destination folio).
No existing allocator path sets __GFP_PRIVATE, so private nodes
are unreachable by default.
- Use standard struct page / folio. No ZONE_DEVICE, no pgmap,
no struct page metadata limitations.
- Use a node-scoped metadata structure to accomplish filtering
and callback support.
- May participate in the buddy allocator, reclaim, compaction,
and LRU like normal memory, gated by an opt-in set of flags.
The key abstraction is node_private_ops: a per-node callback table
registered by a driver or service.
Each callback is individually gated by an NP_OPS_* capability flag.
A driver opts in only to the mm/ operations it needs.
It is similar to ZONE_DEVICE's pgmap at a node granularity.
In fact...
Re-use of ZONE_DEVICE Hooks
===
The callback insertion points deliberately mirror existing ZONE_DEVICE
hooks to minimize the surface area of the mechanism.
I believe this could subsume most DEVICE_COHERENT users, and greatly
simplify the device-managed memory development process (no more
per-driver allocator and migration code).
(Also it's just "So Fresh, So Clean").
The base set of callbacks introduced include:
free_folio - mirrors ZONE_DEVICE's
free_zone_device_page() hook in
__folio_put() / folios_put_refs()
folio_split - mirrors ZONE_DEVICE's
called when a huge page is split up
migrate_to - demote_folio_list() custom demotion (same
site as ZONE_DEVICE demotion rejection)
folio_migrate - called when private node folio is moved to
another location (e.g. compaction)
handle_fault - mirrors the ZONE_DEVICE fault dispatch in
handle_pte_fault() (do_wp_page path)
reclaim_policy - called by reclaim to let a driver own the
boost lifecycle (driver can driver node reclaim)
memory_failure - parallels memory_failure_dev_pagemap(),
but for online pages that enter the normal
hwpoison path
At skip sites (mlock, madvise, KSM, user migration), a unified
folio_is_private_managed() predicate covers both ZONE_DEVICE and
N_MEMORY_PRIVATE folios, consolidating existing zone_device checks
with private node checks rather than adding new ones.
static inline bool folio_is_private_managed(struct folio *folio)
{
return folio_is_zone_device(folio) ||
folio_is_private_node(folio);
}
Most integration points become a one-line swap:
- if (folio_is_zone_device(folio))
+ if (unlikely(folio_is_private_managed(folio)))
Where a one-line integration is insufficient, the integration is
kept as clean as possible with zone_device, rather than simply
adding more call-sites on top of it:
static inline bool folio_managed_handle_fault(struct folio *folio,
struct vm_fault *vmf, vm_fault_t *ret)
{
/* Zone device pages use swap entries; handled in do_swap_page */
if (folio_is_zone_device(folio))
return false;
if (folio_is_private_node(folio)) {
const struct node_private_ops *ops = folio_node_private_ops(folio);
if (ops && ops->handle_fault) {
*ret = ops->handle_fault(vmf);
return true;
}
}
return false;
}
Flag-gated behavior (NP_OPS_*) controls:
===
We use OPS flags to denote what mm/ services we want to allow on our
private node. I've plumbed these through so far:
NP_OPS_MIGRATION - Node supports migration
NP_OPS_MEMPOLICY - Node supports mempolicy actions
NP_OPS_DEMOTION - Node appears in demotion target lists
NP_OPS_PROTECT_WRITE - Node memory is read-only (wrprotect)
NP_OPS_RECLAIM - Node supports reclaim
NP_OPS_NUMA_BALANCING - Node supports numa balancing
NP_OPS_COMPACTION - Node supports compaction
NP_OPS_LONGTERM_PIN - Node supports longterm pinning
NP_OPS_OOM_ELIGIBLE - (MIGRATION | DEMOTION), node is reachable
as normal system ram storage, so it should
be considered in OOM pressure calculations.
I wasn't quite sure how to classify ksm, khugepaged, madvise, and
mlock - so i have omitted those for now.
Most hooks are straightforward.
Including a node as a demotion-eligible target was as simple as:
static void establish_demotion_targets(void)
{
..... snip .....
/*
* Include private nodes that have opted in to demotion
* via NP_OPS_DEMOTION. A node might have custom migrate
*/
all_memory = node_states[N_MEMORY];
for_each_node_state(node, N_MEMORY_PRIVATE) {
if (node_private_has_flag(node, NP_OPS_DEMOTION))
node_set(node, all_memory);
}
..... snip .....
}
The Migration and Mempolicy support are the two most complex pieces,
and most useful things are built on top of Migration (meaning the
remaining implementations are usually simple).
Private Node Hotplug Lifecycle
===
Registration follows a strict order enforced by
add_private_memory_driver_managed():
1. Driver calls add_private_memory_driver_managed(nid, start,
size, resource_name, mhp_flags, online_type, &np).
2. node_private_register(nid, &np) stores the driver's
node_private in pgdat and sets pgdat->private. N_MEMORY and
N_MEMORY_PRIVATE are mutually exclusive -- registration fails
with -EBUSY if the node already has N_MEMORY set.
Only one driver may register per private node.
3. Memory is hotplugged via __add_memory_driver_managed().
When online_pages() runs, it checks pgdat->private and sets
N_MEMORY_PRIVATE instead of N_MEMORY.
Zonelist construction gives private nodes a self-only NOFALLBACK
list and an N_MEMORY fallback list (so kernel/slab allocations on
behalf of private node work can fall back to DRAM).
4. kswapd and kcompactd are NOT started for private nodes. The
owning service is responsible for driving reclaim if needed
(e.g., CRAM uses watermark_boost to wake kswapd on demand).
Teardown is the reverse:
1. Driver calls offline_and_remove_private_memory(nid, start,
size).
2. offline_pages() offlines the memory. When the last block is
offlined, N_MEMORY_PRIVATE is cleared automatically.
3. node_private_unregister() clears pgdat->node_private and
drops the refcount. It refuses to unregister (-EBUSY) if
N_MEMORY_PRIVATE is still set (other memory ranges remain).
The driver is responsible for ensuring memory is hot-unpluggable
before teardown. The service must ensure all memory is cleaned
up before hot-unplug - or the service must support migration (so
memory_hotplug.c can evacuate the memory itself).
In the CRAM example, the service supports migration, so memory
hot-unplug can remove memory without any special infrastructure.
Application: Compressed RAM (mm/cram)
===
Compressed RAM has a serious design issue: Its capacity a lie.
A compression device reports more capacity than it physically has.
If workloads write faster than the OS can reclaim from the device,
we run out of real backing store and corrupt data or crash.
I call this problem: "Trying to Out Run A Bear"
I.e. This is only stable as long as we stay ahead of the pressure.
We don't want to design a system where stability depends on outrunning
a bear - I am slow and do not know where to acquire bear spray.
Fun fact: Grizzly bears have a top-speed of 56-64 km/h.
Unfun Fact: Humans typically top out at ~24 km/h.
This MVP takes a conservative position:
all compressed memory is mapped read-only.
- Folios reach the private node only via reclaim (demotion)
- migrate_to implements custom demotion with backpressure.
- fixup_migration_pte write-protects PTEs on arrival.
- wrprotect hooks prevent silent upgrades
- handle_fault promotes folios back to DRAM on write.
- free_folio scrubs stale data before buddy free.
Because pages are read-only, writes can never cause runaway
compression ratio loss behind the allocator's back. Every write
goes through handle_fault, which promotes the folio to DRAM first.
The device only ever sees net compression (demotion in) and explicit
decompression (promotion out via fault or reclaim), and has a much
wider timeframe to respond to poor compression scenarios.
That means there's no bear to out run. The bears are safely asleep in
their bear den, and even if they show up we have a bear-proof cage.
The backpressure system is our bear-proof cage: the driver reports real
device utilization (generalized via watermark_boost on the private
node's zone), and CRAM throttles demotion when capacity is tight.
If compression ratios are bad, we stop demoting pages and start
evicting pages aggressively.
The service as designed is ~350 functional lines of code because it
re-uses mm/ services:
- Existing reclaim/vmscan code handles demotion.
- Existing migration code handles migration to/from.
- Existing page fault handling dispatches faults.
The driver contains all the CXL nastiness core developers don't want
anything to do with - No vendor logic touches mm/ internals.
Future CRAM : Loosening the read-only constraint
===
The read-only model is safe but conservative. For workloads where
compressed pages are occasionally written, the promotion fault adds
latency. A future optimization could allow a tunable fraction of
compressed pages to be mapped writable, accepting some risk of
write-driven decompression in exchange for lower overhead.
The private node ops make this straightforward:
- Adjust fixup_migration_pte to selectively skip
write-protection.
- Use the backpressure system to either revoke writable mappings,
deny additional demotions, or evict when device pressure rises.
This comes at a mild memory overhead: 32MB of DRAM per 1TB of CRAM.
(1 bit per 4KB page).
This is not proposed here, but it should be somewhat trivial.
Discussion Topics
===
0. Obviously I've included the set as an RFC, please rip it apart.
1. Is N_MEMORY_PRIVATE the right isolation abstraction, or should
this extend ZONE_DEVICE? Prior feedback pushed away from new
ZONE logic, but this will likely be debated further.
My comments on this:
ZONE_DEVICE requires re-implementing every service you want to
provide to your device memory, including basic allocation.
Private nodes use real struct pages with no metadata
limitations, participate in the buddy allocator, and get NUMA
topology for free.
2. Can this subsume ZONE_DEVICE COHERENT users? The architecture
was designed with this in mind, but it is only a thought experiment.
3. Is a dedicated mm/ service (cram) the right place for compressed
memory management, or should this be purely driver-side until
more devices exist?
I wrote it this way because I forsee more "innovation" in the
compressed RAM space given current... uh... "Market Conditions".
I don't see CRAM being CXL-specific, though the only solutions I've
seen have been CXL. Nothing is stopping someone from soldering such
memory directly to a PCB.
5. Where is your hardware-backed data that shows this works?
I should have some by conference time.
Thanks for reading
Gregory (Gourry)
Gregory Price (27):
numa: introduce N_MEMORY_PRIVATE node state
mm,cpuset: gate allocations from N_MEMORY_PRIVATE behind __GFP_PRIVATE
mm/page_alloc: add numa_zone_allowed() and wire it up
mm/page_alloc: Add private node handling to build_zonelists
mm: introduce folio_is_private_managed() unified predicate
mm/mlock: skip mlock for managed-memory folios
mm/madvise: skip madvise for managed-memory folios
mm/ksm: skip KSM for managed-memory folios
mm/khugepaged: skip private node folios when trying to collapse.
mm/swap: add free_folio callback for folio release cleanup
mm/huge_memory.c: add private node folio split notification callback
mm/migrate: NP_OPS_MIGRATION - support private node user migration
mm/mempolicy: NP_OPS_MEMPOLICY - support private node mempolicy
mm/memory-tiers: NP_OPS_DEMOTION - support private node demotion
mm/mprotect: NP_OPS_PROTECT_WRITE - gate PTE/PMD write-upgrades
mm: NP_OPS_RECLAIM - private node reclaim participation
mm/oom: NP_OPS_OOM_ELIGIBLE - private node OOM participation
mm/memory: NP_OPS_NUMA_BALANCING - private node NUMA balancing
mm/compaction: NP_OPS_COMPACTION - private node compaction support
mm/gup: NP_OPS_LONGTERM_PIN - private node longterm pin support
mm/memory-failure: add memory_failure callback to node_private_ops
mm/memory_hotplug: add add_private_memory_driver_managed()
mm/cram: add compressed ram memory management subsystem
cxl/core: Add cxl_sysram region type
cxl/core: Add private node support to cxl_sysram
cxl: add cxl_mempolicy sample PCI driver
cxl: add cxl_compression PCI driver
drivers/base/node.c | 250 +++-
drivers/cxl/Kconfig | 2 +
drivers/cxl/Makefile | 2 +
drivers/cxl/core/Makefile | 1 +
drivers/cxl/core/core.h | 4 +
drivers/cxl/core/port.c | 2 +
drivers/cxl/core/region_sysram.c | 381 ++++++
drivers/cxl/cxl.h | 53 +
drivers/cxl/type3_drivers/Kconfig | 3 +
drivers/cxl/type3_drivers/Makefile | 3 +
.../cxl/type3_drivers/cxl_compression/Kconfig | 20 +
.../type3_drivers/cxl_compression/Makefile | 4 +
.../cxl_compression/compression.c | 1025 +++++++++++++++++
.../cxl/type3_drivers/cxl_mempolicy/Kconfig | 16 +
.../cxl/type3_drivers/cxl_mempolicy/Makefile | 4 +
.../type3_drivers/cxl_mempolicy/mempolicy.c | 297 +++++
include/linux/cpuset.h | 9 -
include/linux/cram.h | 66 ++
include/linux/gfp_types.h | 15 +-
include/linux/memory-tiers.h | 9 +
include/linux/memory_hotplug.h | 11 +
include/linux/migrate.h | 17 +-
include/linux/mm.h | 22 +
include/linux/mmzone.h | 16 +
include/linux/node_private.h | 532 +++++++++
include/linux/nodemask.h | 1 +
include/trace/events/mmflags.h | 4 +-
include/uapi/linux/mempolicy.h | 1 +
kernel/cgroup/cpuset.c | 49 +-
mm/Kconfig | 10 +
mm/Makefile | 1 +
mm/compaction.c | 32 +-
mm/cram.c | 508 ++++++++
mm/damon/paddr.c | 3 +
mm/huge_memory.c | 23 +-
mm/hugetlb.c | 2 +-
mm/internal.h | 226 +++-
mm/khugepaged.c | 7 +-
mm/ksm.c | 9 +-
mm/madvise.c | 5 +-
mm/memory-failure.c | 15 +
mm/memory-tiers.c | 46 +-
mm/memory.c | 26 +
mm/memory_hotplug.c | 122 +-
mm/mempolicy.c | 69 +-
mm/migrate.c | 63 +-
mm/mlock.c | 5 +-
mm/mprotect.c | 4 +-
mm/oom_kill.c | 52 +-
mm/page_alloc.c | 79 +-
mm/rmap.c | 4 +-
mm/slub.c | 3 +-
mm/swap.c | 21 +-
mm/vmscan.c | 55 +-
54 files changed, 4057 insertions(+), 152 deletions(-)
create mode 100644 drivers/cxl/core/region_sysram.c
create mode 100644 drivers/cxl/type3_drivers/Kconfig
create mode 100644 drivers/cxl/type3_drivers/Makefile
create mode 100644 drivers/cxl/type3_drivers/cxl_compression/Kconfig
create mode 100644 drivers/cxl/type3_drivers/cxl_compression/Makefile
create mode 100644 drivers/cxl/type3_drivers/cxl_compression/compression.c
create mode 100644 drivers/cxl/type3_drivers/cxl_mempolicy/Kconfig
create mode 100644 drivers/cxl/type3_drivers/cxl_mempolicy/Makefile
create mode 100644 drivers/cxl/type3_drivers/cxl_mempolicy/mempolicy.c
create mode 100644 include/linux/cram.h
create mode 100644 include/linux/node_private.h
create mode 100644 mm/cram.c
--
2.53.0
^ permalink raw reply
* [RFC PATCH v4 01/27] numa: introduce N_MEMORY_PRIVATE node state
From: Gregory Price @ 2026-02-22 8:48 UTC (permalink / raw)
To: lsf-pc
Cc: linux-kernel, linux-cxl, cgroups, linux-mm, linux-trace-kernel,
damon, kernel-team, gregkh, rafael, dakr, dave, jonathan.cameron,
dave.jiang, alison.schofield, vishal.l.verma, ira.weiny,
dan.j.williams, longman, akpm, david, lorenzo.stoakes,
Liam.Howlett, vbabka, rppt, surenb, mhocko, osalvador, ziy,
matthew.brost, joshua.hahnjy, rakie.kim, byungchul, gourry,
ying.huang, apopple, axelrasmussen, yuanchu, weixugc, yury.norov,
linux, mhiramat, mathieu.desnoyers, tj, hannes, mkoutny, jackmanb,
sj, baolin.wang, npache, ryan.roberts, dev.jain, baohua,
lance.yang, muchun.song, xu.xin16, chengming.zhou, jannh,
linmiaohe, nao.horiguchi, pfalcato, rientjes, shakeel.butt, riel,
harry.yoo, cl, roman.gushchin, chrisl, kasong, shikemeng, nphamcs,
bhe, zhengqi.arch, terry.bowman
In-Reply-To: <20260222084842.1824063-1-gourry@gourry.net>
N_MEMORY nodes are intended to contain general System RAM. Today, some
device drivers hotplug their memory (marked Specific Purpose or Reserved)
to get access to mm/ services, but don't intend it for general consumption.
Create N_MEMORY_PRIVATE for memory nodes whose memory is not intended for
general consumption. This state is mutually exclusive with N_MEMORY.
Add the node_private infrastructure for N_MEMORY_PRIVATE nodes:
- struct node_private: Per-node container stored in NODE_DATA(nid),
holding driver callbacks (ops), owner, and refcount.
- struct node_private_ops: Initial structure with void *reserved
placeholder and flags field. Callbacks will be added by subsequent
commits as each consumer is wired up.
- folio_is_private_node() / page_is_private_node(): check if a
folio/page resides on a private node.
- folio_node_private_ops() / node_private_flags(): retrieve the ops
vtable or flags for a folio's node.
- Registration API: node_private_register()/unregister() for drivers
to register callbacks for private nodes. Only one driver callback
can be registered per node - attempting to register different ops
returns -EBUSY.
- sysfs attribute exposing N_MEMORY_PRIVATE node state.
Zonelist construction changes for private nodes are deferred to a
subsequent commit.
Signed-off-by: Gregory Price <gourry@gourry.net>
---
drivers/base/node.c | 197 ++++++++++++++++++++++++++++++++
include/linux/mmzone.h | 4 +
include/linux/node_private.h | 210 +++++++++++++++++++++++++++++++++++
include/linux/nodemask.h | 1 +
4 files changed, 412 insertions(+)
create mode 100644 include/linux/node_private.h
diff --git a/drivers/base/node.c b/drivers/base/node.c
index 00cf4532f121..646dc48a23b5 100644
--- a/drivers/base/node.c
+++ b/drivers/base/node.c
@@ -22,6 +22,7 @@
#include <linux/swap.h>
#include <linux/slab.h>
#include <linux/memblock.h>
+#include <linux/node_private.h>
static const struct bus_type node_subsys = {
.name = "node",
@@ -861,6 +862,198 @@ void register_memory_blocks_under_node_hotplug(int nid, unsigned long start_pfn,
(void *)&nid, register_mem_block_under_node_hotplug);
return;
}
+
+static DEFINE_MUTEX(node_private_lock);
+static bool node_private_initialized;
+
+/**
+ * node_private_register - Register a private node
+ * @nid: Node identifier
+ * @np: The node_private structure (driver-allocated, driver-owned)
+ *
+ * Register a driver for a private node. Only one driver can register
+ * per node. If another driver has already registered (with different np),
+ * -EBUSY is returned. Re-registration with the same np is allowed.
+ *
+ * The driver owns the node_private memory and must ensure it remains valid
+ * until refcount reaches 0 after node_private_unregister().
+ *
+ * Returns 0 on success, negative errno on failure.
+ */
+int node_private_register(int nid, struct node_private *np)
+{
+ struct node_private *existing;
+ pg_data_t *pgdat;
+ int ret = 0;
+
+ if (!np || !node_possible(nid))
+ return -EINVAL;
+
+ if (!node_private_initialized)
+ return -ENODEV;
+
+ mutex_lock(&node_private_lock);
+ mem_hotplug_begin();
+
+ /* N_MEMORY_PRIVATE and N_MEMORY are mutually exclusive */
+ if (node_state(nid, N_MEMORY)) {
+ ret = -EBUSY;
+ goto out;
+ }
+
+ pgdat = NODE_DATA(nid);
+ existing = rcu_dereference_protected(pgdat->node_private,
+ lockdep_is_held(&node_private_lock));
+
+ /* Only one source my register this node */
+ if (existing) {
+ if (existing != np) {
+ ret = -EBUSY;
+ goto out;
+ }
+ goto out;
+ }
+
+ refcount_set(&np->refcount, 1);
+ init_completion(&np->released);
+
+ rcu_assign_pointer(pgdat->node_private, np);
+ pgdat->private = true;
+
+out:
+ mem_hotplug_done();
+ mutex_unlock(&node_private_lock);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(node_private_register);
+
+/**
+ * node_private_set_ops - Set service callbacks on a registered private node
+ * @nid: Node identifier
+ * @ops: Service callbacks and flags (driver-owned, must outlive registration)
+ *
+ * Validates flag dependencies and sets the ops on the node's node_private.
+ * The node must already be registered via node_private_register().
+ *
+ * Returns 0 on success, -EINVAL for invalid flag combinations,
+ * -ENODEV if no node_private is registered on @nid.
+ */
+int node_private_set_ops(int nid, const struct node_private_ops *ops)
+{
+ struct node_private *np;
+ int ret = 0;
+
+ if (!ops)
+ return -EINVAL;
+
+ if (!node_possible(nid))
+ return -EINVAL;
+
+ mutex_lock(&node_private_lock);
+ np = rcu_dereference_protected(NODE_DATA(nid)->node_private,
+ lockdep_is_held(&node_private_lock));
+ if (!np)
+ ret = -ENODEV;
+ else
+ np->ops = ops;
+ mutex_unlock(&node_private_lock);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(node_private_set_ops);
+
+/**
+ * node_private_clear_ops - Clear service callbacks from a private node
+ * @nid: Node identifier
+ * @ops: Expected ops pointer (must match current ops)
+ *
+ * Clears the ops only if @ops matches the currently registered ops,
+ * preventing one service from accidentally clearing another's callbacks.
+ *
+ * Returns 0 on success, -ENODEV if no node_private is registered,
+ * -EINVAL if @ops does not match.
+ */
+int node_private_clear_ops(int nid, const struct node_private_ops *ops)
+{
+ struct node_private *np;
+ int ret = 0;
+
+ if (!node_possible(nid))
+ return -EINVAL;
+
+ mutex_lock(&node_private_lock);
+ np = rcu_dereference_protected(NODE_DATA(nid)->node_private,
+ lockdep_is_held(&node_private_lock));
+ if (!np)
+ ret = -ENODEV;
+ else if (np->ops != ops)
+ ret = -EINVAL;
+ else
+ np->ops = NULL;
+ mutex_unlock(&node_private_lock);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(node_private_clear_ops);
+
+/**
+ * node_private_unregister - Unregister a private node
+ * @nid: Node identifier
+ *
+ * Unregister the driver from a private node. Only succeeds if all memory
+ * has been offlined and the node is no longer N_MEMORY_PRIVATE.
+ * When successful, drops the refcount to 0 indicating the driver can
+ * free its context.
+ *
+ * N_MEMORY_PRIVATE state is cleared by offline_pages() when the last
+ * memory is offlined, not by this function.
+ *
+ * Return: 0 if unregistered, -EBUSY if N_MEMORY_PRIVATE is still set
+ * (other memory blocks remain on this node).
+ */
+int node_private_unregister(int nid)
+{
+ struct node_private *np;
+ pg_data_t *pgdat;
+
+ if (!node_possible(nid))
+ return 0;
+
+ mutex_lock(&node_private_lock);
+ mem_hotplug_begin();
+
+ pgdat = NODE_DATA(nid);
+ np = rcu_dereference_protected(pgdat->node_private,
+ lockdep_is_held(&node_private_lock));
+ if (!np) {
+ mem_hotplug_done();
+ mutex_unlock(&node_private_lock);
+ return 0;
+ }
+
+ /*
+ * Only unregister if all memory is offline and N_MEMORY_PRIVATE is
+ * cleared. N_MEMORY_PRIVATE is cleared by offline_pages() when the
+ * last memory block is offlined.
+ */
+ if (node_state(nid, N_MEMORY_PRIVATE)) {
+ mem_hotplug_done();
+ mutex_unlock(&node_private_lock);
+ return -EBUSY;
+ }
+
+ rcu_assign_pointer(pgdat->node_private, NULL);
+ pgdat->private = false;
+
+ mem_hotplug_done();
+ mutex_unlock(&node_private_lock);
+
+ synchronize_rcu();
+
+ if (!refcount_dec_and_test(&np->refcount))
+ wait_for_completion(&np->released);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(node_private_unregister);
+
#endif /* CONFIG_MEMORY_HOTPLUG */
/**
@@ -959,6 +1152,7 @@ static struct node_attr node_state_attr[] = {
[N_HIGH_MEMORY] = _NODE_ATTR(has_high_memory, N_HIGH_MEMORY),
#endif
[N_MEMORY] = _NODE_ATTR(has_memory, N_MEMORY),
+ [N_MEMORY_PRIVATE] = _NODE_ATTR(has_private_memory, N_MEMORY_PRIVATE),
[N_CPU] = _NODE_ATTR(has_cpu, N_CPU),
[N_GENERIC_INITIATOR] = _NODE_ATTR(has_generic_initiator,
N_GENERIC_INITIATOR),
@@ -972,6 +1166,7 @@ static struct attribute *node_state_attrs[] = {
&node_state_attr[N_HIGH_MEMORY].attr.attr,
#endif
&node_state_attr[N_MEMORY].attr.attr,
+ &node_state_attr[N_MEMORY_PRIVATE].attr.attr,
&node_state_attr[N_CPU].attr.attr,
&node_state_attr[N_GENERIC_INITIATOR].attr.attr,
NULL
@@ -1007,5 +1202,7 @@ void __init node_dev_init(void)
panic("%s() failed to add node: %d\n", __func__, ret);
}
+ node_private_initialized = true;
+
register_memory_blocks_under_nodes();
}
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index b01cb1e49896..992eb1c5a2c6 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -25,6 +25,8 @@
#include <linux/zswap.h>
#include <asm/page.h>
+struct node_private;
+
/* Free memory management - zoned buddy allocator. */
#ifndef CONFIG_ARCH_FORCE_MAX_ORDER
#define MAX_PAGE_ORDER 10
@@ -1514,6 +1516,8 @@ typedef struct pglist_data {
atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS];
#ifdef CONFIG_NUMA
struct memory_tier __rcu *memtier;
+ struct node_private __rcu *node_private;
+ bool private;
#endif
#ifdef CONFIG_MEMORY_FAILURE
struct memory_failure_stats mf_stats;
diff --git a/include/linux/node_private.h b/include/linux/node_private.h
new file mode 100644
index 000000000000..6a70ec39d569
--- /dev/null
+++ b/include/linux/node_private.h
@@ -0,0 +1,210 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_NODE_PRIVATE_H
+#define _LINUX_NODE_PRIVATE_H
+
+#include <linux/completion.h>
+#include <linux/mm.h>
+#include <linux/nodemask.h>
+#include <linux/rcupdate.h>
+#include <linux/refcount.h>
+
+struct page;
+struct vm_area_struct;
+struct vm_fault;
+
+/**
+ * struct node_private_ops - Callbacks for private node services
+ *
+ * Services register these callbacks to intercept MM operations that affect
+ * their private nodes.
+ *
+ * Flag bits control which MM subsystems may operate on folios on this node.
+ *
+ * The pgdat->node_private pointer is RCU-protected. Callbacks fall into
+ * three categories based on their calling context:
+ *
+ * Folio-referenced callbacks (RCU released before callback):
+ * The caller holds a reference to a folio on the private node, which
+ * pins the node's memory online and prevents node_private teardown.
+ *
+ * Refcounted callbacks (RCU released before callback):
+ * The caller has no folio on the private node (e.g., folios are on a
+ * source node being migrated TO this node). A temporary refcount is
+ * taken on node_private under rcu_read_lock to keep the structure (and
+ * the service module) alive across the callback. node_private_unregister
+ * waits for all temporary references to drain before returning.
+ *
+ * Non-folio callbacks (rcu_read_lock held during callback):
+ * No folio reference exists, so rcu_read_lock is held across the
+ * callback to prevent node_private from being freed.
+ * These callbacks MUST NOT sleep.
+ *
+ * @flags: Operation exclusion flags (NP_OPS_* constants).
+ *
+ */
+struct node_private_ops {
+ unsigned long flags;
+};
+
+/**
+ * struct node_private - Per-node container for N_MEMORY_PRIVATE nodes
+ *
+ * This structure is allocated by the driver and passed to node_private_register().
+ * The driver owns the memory and must ensure it remains valid until after
+ * node_private_unregister() returns with the reference count dropped to 0.
+ *
+ * @owner: Opaque driver identifier
+ * @refcount: Reference count (1 = registered; temporary refs for non-folio
+ * callbacks that may sleep; 0 = fully released)
+ * @released: Signaled when refcount drops to 0; unregister waits on this
+ * @ops: Service callbacks and exclusion flags (NULL until service registers)
+ */
+struct node_private {
+ void *owner;
+ refcount_t refcount;
+ struct completion released;
+ const struct node_private_ops *ops;
+};
+
+#ifdef CONFIG_NUMA
+
+#include <linux/mmzone.h>
+
+/**
+ * folio_is_private_node - Check if folio is on an N_MEMORY_PRIVATE node
+ * @folio: The folio to check
+ *
+ * Returns true if the folio resides on a private node.
+ */
+static inline bool folio_is_private_node(struct folio *folio)
+{
+ return node_state(folio_nid(folio), N_MEMORY_PRIVATE);
+}
+
+/**
+ * page_is_private_node - Check if page is on an N_MEMORY_PRIVATE node
+ * @page: The page to check
+ *
+ * Returns true if the page resides on a private node.
+ */
+static inline bool page_is_private_node(struct page *page)
+{
+ return node_state(page_to_nid(page), N_MEMORY_PRIVATE);
+}
+
+static inline const struct node_private_ops *
+folio_node_private_ops(struct folio *folio)
+{
+ const struct node_private_ops *ops;
+ struct node_private *np;
+
+ rcu_read_lock();
+ np = rcu_dereference(NODE_DATA(folio_nid(folio))->node_private);
+ ops = np ? np->ops : NULL;
+ rcu_read_unlock();
+
+ return ops;
+}
+
+static inline unsigned long node_private_flags(int nid)
+{
+ struct node_private *np;
+ unsigned long flags;
+
+ rcu_read_lock();
+ np = rcu_dereference(NODE_DATA(nid)->node_private);
+ flags = (np && np->ops) ? np->ops->flags : 0;
+ rcu_read_unlock();
+
+ return flags;
+}
+
+static inline bool folio_private_flags(struct folio *f, unsigned long flag)
+{
+ return node_private_flags(folio_nid(f)) & flag;
+}
+
+static inline bool node_private_has_flag(int nid, unsigned long flag)
+{
+ return node_private_flags(nid) & flag;
+}
+
+static inline bool zone_private_flags(struct zone *z, unsigned long flag)
+{
+ return node_private_flags(zone_to_nid(z)) & flag;
+}
+
+#else /* !CONFIG_NUMA */
+
+static inline bool folio_is_private_node(struct folio *folio)
+{
+ return false;
+}
+
+static inline bool page_is_private_node(struct page *page)
+{
+ return false;
+}
+
+static inline const struct node_private_ops *
+folio_node_private_ops(struct folio *folio)
+{
+ return NULL;
+}
+
+static inline unsigned long node_private_flags(int nid)
+{
+ return 0;
+}
+
+static inline bool folio_private_flags(struct folio *f, unsigned long flag)
+{
+ return false;
+}
+
+static inline bool node_private_has_flag(int nid, unsigned long flag)
+{
+ return false;
+}
+
+static inline bool zone_private_flags(struct zone *z, unsigned long flag)
+{
+ return false;
+}
+
+#endif /* CONFIG_NUMA */
+
+#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
+
+int node_private_register(int nid, struct node_private *np);
+int node_private_unregister(int nid);
+int node_private_set_ops(int nid, const struct node_private_ops *ops);
+int node_private_clear_ops(int nid, const struct node_private_ops *ops);
+
+#else /* !CONFIG_NUMA || !CONFIG_MEMORY_HOTPLUG */
+
+static inline int node_private_register(int nid, struct node_private *np)
+{
+ return -ENODEV;
+}
+
+static inline int node_private_unregister(int nid)
+{
+ return 0;
+}
+
+static inline int node_private_set_ops(int nid,
+ const struct node_private_ops *ops)
+{
+ return -ENODEV;
+}
+
+static inline int node_private_clear_ops(int nid,
+ const struct node_private_ops *ops)
+{
+ return -ENODEV;
+}
+
+#endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */
+
+#endif /* _LINUX_NODE_PRIVATE_H */
diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h
index bd38648c998d..c9bcfd5a9a06 100644
--- a/include/linux/nodemask.h
+++ b/include/linux/nodemask.h
@@ -391,6 +391,7 @@ enum node_states {
N_HIGH_MEMORY = N_NORMAL_MEMORY,
#endif
N_MEMORY, /* The node has memory(regular, high, movable) */
+ N_MEMORY_PRIVATE, /* The node's memory is private */
N_CPU, /* The node has one or more cpus */
N_GENERIC_INITIATOR, /* The node has one or more Generic Initiators */
NR_NODE_STATES
--
2.53.0
^ permalink raw reply related
* [RFC PATCH v4 02/27] mm,cpuset: gate allocations from N_MEMORY_PRIVATE behind __GFP_PRIVATE
From: Gregory Price @ 2026-02-22 8:48 UTC (permalink / raw)
To: lsf-pc
Cc: linux-kernel, linux-cxl, cgroups, linux-mm, linux-trace-kernel,
damon, kernel-team, gregkh, rafael, dakr, dave, jonathan.cameron,
dave.jiang, alison.schofield, vishal.l.verma, ira.weiny,
dan.j.williams, longman, akpm, david, lorenzo.stoakes,
Liam.Howlett, vbabka, rppt, surenb, mhocko, osalvador, ziy,
matthew.brost, joshua.hahnjy, rakie.kim, byungchul, gourry,
ying.huang, apopple, axelrasmussen, yuanchu, weixugc, yury.norov,
linux, mhiramat, mathieu.desnoyers, tj, hannes, mkoutny, jackmanb,
sj, baolin.wang, npache, ryan.roberts, dev.jain, baohua,
lance.yang, muchun.song, xu.xin16, chengming.zhou, jannh,
linmiaohe, nao.horiguchi, pfalcato, rientjes, shakeel.butt, riel,
harry.yoo, cl, roman.gushchin, chrisl, kasong, shikemeng, nphamcs,
bhe, zhengqi.arch, terry.bowman
In-Reply-To: <20260222084842.1824063-1-gourry@gourry.net>
N_MEMORY_PRIVATE nodes hold device-managed memory that should not be
used for general allocations. Without a gating mechanism, any allocation
could land on a private node if it appears in the task's mems_allowed.
Introduce __GFP_PRIVATE that explicitly opts in to allocation from
N_MEMORY_PRIVATE nodes.
Add the GFP_PRIVATE compound mask (__GFP_PRIVATE | __GFP_THISNODE)
for callers that explicitly target private nodes to help prevent
fallback allocations from DRAM.
Update cpuset_current_node_allowed() to filter out N_MEMORY_PRIVATE
nodes unless __GFP_PRIVATE is set.
In interrupt context, only N_MEMORY nodes are valid.
Update cpuset_handle_hotplug() to include N_MEMORY_PRIVATE nodes in
the effective mems set, allowing cgroup-level control over private
node access.
Signed-off-by: Gregory Price <gourry@gourry.net>
---
include/linux/gfp_types.h | 15 +++++++++++++--
include/trace/events/mmflags.h | 4 ++--
kernel/cgroup/cpuset.c | 32 ++++++++++++++++++++++++++++----
3 files changed, 43 insertions(+), 8 deletions(-)
diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h
index 3de43b12209e..ac375f9a0fc2 100644
--- a/include/linux/gfp_types.h
+++ b/include/linux/gfp_types.h
@@ -33,7 +33,7 @@ enum {
___GFP_IO_BIT,
___GFP_FS_BIT,
___GFP_ZERO_BIT,
- ___GFP_UNUSED_BIT, /* 0x200u unused */
+ ___GFP_PRIVATE_BIT,
___GFP_DIRECT_RECLAIM_BIT,
___GFP_KSWAPD_RECLAIM_BIT,
___GFP_WRITE_BIT,
@@ -69,7 +69,7 @@ enum {
#define ___GFP_IO BIT(___GFP_IO_BIT)
#define ___GFP_FS BIT(___GFP_FS_BIT)
#define ___GFP_ZERO BIT(___GFP_ZERO_BIT)
-/* 0x200u unused */
+#define ___GFP_PRIVATE BIT(___GFP_PRIVATE_BIT)
#define ___GFP_DIRECT_RECLAIM BIT(___GFP_DIRECT_RECLAIM_BIT)
#define ___GFP_KSWAPD_RECLAIM BIT(___GFP_KSWAPD_RECLAIM_BIT)
#define ___GFP_WRITE BIT(___GFP_WRITE_BIT)
@@ -139,6 +139,11 @@ enum {
* %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg.
*
* %__GFP_NO_OBJ_EXT causes slab allocation to have no object extension.
+ *
+ * %__GFP_PRIVATE allows allocation from N_MEMORY_PRIVATE nodes (e.g., compressed
+ * memory, accelerator memory). Without this flag, allocations are restricted
+ * to N_MEMORY nodes only. Used by migration/demotion paths when explicitly
+ * targeting private nodes.
*/
#define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE)
#define __GFP_WRITE ((__force gfp_t)___GFP_WRITE)
@@ -146,6 +151,7 @@ enum {
#define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE)
#define __GFP_ACCOUNT ((__force gfp_t)___GFP_ACCOUNT)
#define __GFP_NO_OBJ_EXT ((__force gfp_t)___GFP_NO_OBJ_EXT)
+#define __GFP_PRIVATE ((__force gfp_t)___GFP_PRIVATE)
/**
* DOC: Watermark modifiers
@@ -367,6 +373,10 @@ enum {
* available and will not wake kswapd/kcompactd on failure. The _LIGHT
* version does not attempt reclaim/compaction at all and is by default used
* in page fault path, while the non-light is used by khugepaged.
+ *
+ * %GFP_PRIVATE adds %__GFP_THISNODE by default to prevent any fallback
+ * allocations to other nodes, given that the caller was already attempting
+ * to access driver-managed memory explicitly.
*/
#define GFP_ATOMIC (__GFP_HIGH|__GFP_KSWAPD_RECLAIM)
#define GFP_KERNEL (__GFP_RECLAIM | __GFP_IO | __GFP_FS)
@@ -382,5 +392,6 @@ enum {
#define GFP_TRANSHUGE_LIGHT ((GFP_HIGHUSER_MOVABLE | __GFP_COMP | \
__GFP_NOMEMALLOC | __GFP_NOWARN) & ~__GFP_RECLAIM)
#define GFP_TRANSHUGE (GFP_TRANSHUGE_LIGHT | __GFP_DIRECT_RECLAIM)
+#define GFP_PRIVATE (__GFP_PRIVATE | __GFP_THISNODE)
#endif /* __LINUX_GFP_TYPES_H */
diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h
index a6e5a44c9b42..f042cd848451 100644
--- a/include/trace/events/mmflags.h
+++ b/include/trace/events/mmflags.h
@@ -37,7 +37,8 @@
TRACE_GFP_EM(HARDWALL) \
TRACE_GFP_EM(THISNODE) \
TRACE_GFP_EM(ACCOUNT) \
- TRACE_GFP_EM(ZEROTAGS)
+ TRACE_GFP_EM(ZEROTAGS) \
+ TRACE_GFP_EM(PRIVATE)
#ifdef CONFIG_KASAN_HW_TAGS
# define TRACE_GFP_FLAGS_KASAN \
@@ -73,7 +74,6 @@
TRACE_GFP_FLAGS
/* Just in case these are ever used */
-TRACE_DEFINE_ENUM(___GFP_UNUSED_BIT);
TRACE_DEFINE_ENUM(___GFP_LAST_BIT);
#define gfpflag_string(flag) {(__force unsigned long)flag, #flag}
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index 473aa9261e16..1a597f0c7c6c 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -444,21 +444,32 @@ static void guarantee_active_cpus(struct task_struct *tsk,
}
/*
- * Return in *pmask the portion of a cpusets's mems_allowed that
+ * Return in *pmask the portion of a cpuset's mems_allowed that
* are online, with memory. If none are online with memory, walk
* up the cpuset hierarchy until we find one that does have some
* online mems. The top cpuset always has some mems online.
*
* One way or another, we guarantee to return some non-empty subset
- * of node_states[N_MEMORY].
+ * of node_states[N_MEMORY]. N_MEMORY_PRIVATE nodes from the
+ * original cpuset are preserved, but only N_MEMORY nodes are
+ * pulled from ancestors.
*
* Call with callback_lock or cpuset_mutex held.
*/
static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask)
{
+ struct cpuset *orig_cs = cs;
+ int nid;
+
while (!nodes_intersects(cs->effective_mems, node_states[N_MEMORY]))
cs = parent_cs(cs);
+
nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY]);
+
+ for_each_node_state(nid, N_MEMORY_PRIVATE) {
+ if (node_isset(nid, orig_cs->effective_mems))
+ node_set(nid, *pmask);
+ }
}
/**
@@ -4075,7 +4086,9 @@ static void cpuset_handle_hotplug(void)
/* fetch the available cpus/mems and find out which changed how */
cpumask_copy(&new_cpus, cpu_active_mask);
- new_mems = node_states[N_MEMORY];
+
+ /* Include N_MEMORY_PRIVATE so cpuset controls access the same way */
+ nodes_or(new_mems, node_states[N_MEMORY], node_states[N_MEMORY_PRIVATE]);
/*
* If subpartitions_cpus is populated, it is likely that the check
@@ -4488,10 +4501,21 @@ bool cpuset_node_allowed(struct cgroup *cgroup, int nid)
* __alloc_pages() will include all nodes. If the slab allocator
* is passed an offline node, it will fall back to the local node.
* See kmem_cache_alloc_node().
+ *
+ *
+ * Private nodes aren't eligible for these allocations, so skip them.
+ * guarantee_online_mems guaranttes at least one N_MEMORY node is set.
*/
static int cpuset_spread_node(int *rotor)
{
- return *rotor = next_node_in(*rotor, current->mems_allowed);
+ int node;
+
+ do {
+ node = next_node_in(*rotor, current->mems_allowed);
+ *rotor = node;
+ } while (node_state(node, N_MEMORY_PRIVATE));
+
+ return node;
}
/**
--
2.53.0
^ permalink raw reply related
* [RFC PATCH v4 03/27] mm/page_alloc: add numa_zone_allowed() and wire it up
From: Gregory Price @ 2026-02-22 8:48 UTC (permalink / raw)
To: lsf-pc
Cc: linux-kernel, linux-cxl, cgroups, linux-mm, linux-trace-kernel,
damon, kernel-team, gregkh, rafael, dakr, dave, jonathan.cameron,
dave.jiang, alison.schofield, vishal.l.verma, ira.weiny,
dan.j.williams, longman, akpm, david, lorenzo.stoakes,
Liam.Howlett, vbabka, rppt, surenb, mhocko, osalvador, ziy,
matthew.brost, joshua.hahnjy, rakie.kim, byungchul, gourry,
ying.huang, apopple, axelrasmussen, yuanchu, weixugc, yury.norov,
linux, mhiramat, mathieu.desnoyers, tj, hannes, mkoutny, jackmanb,
sj, baolin.wang, npache, ryan.roberts, dev.jain, baohua,
lance.yang, muchun.song, xu.xin16, chengming.zhou, jannh,
linmiaohe, nao.horiguchi, pfalcato, rientjes, shakeel.butt, riel,
harry.yoo, cl, roman.gushchin, chrisl, kasong, shikemeng, nphamcs,
bhe, zhengqi.arch, terry.bowman
In-Reply-To: <20260222084842.1824063-1-gourry@gourry.net>
Various locations in mm/ open-code cpuset filtering with:
cpusets_enabled() && ALLOC_CPUSET && !__cpuset_zone_allowed()
This pattern does not account for N_MEMORY_PRIVATE nodes on systems
without cpusets, so private-node zones can leak into allocation
paths that should only see general-purpose memory.
Add numa_zone_allowed() which consolidates zone filtering. It checks
cpuset membership when cpusets are enabled, and otherwise gates
N_MEMORY_PRIVATE zones behind __GFP_PRIVATE globally.
Replace the open-coded patterns in mm/ with the new helper.
Signed-off-by: Gregory Price <gourry@gourry.net>
---
mm/compaction.c | 6 ++----
mm/hugetlb.c | 2 +-
mm/internal.h | 7 +++++++
mm/page_alloc.c | 31 ++++++++++++++++++++-----------
mm/slub.c | 3 ++-
5 files changed, 32 insertions(+), 17 deletions(-)
diff --git a/mm/compaction.c b/mm/compaction.c
index 1e8f8eca318c..6a65145b03d8 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -2829,10 +2829,8 @@ enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
ac->highest_zoneidx, ac->nodemask) {
enum compact_result status;
- if (cpusets_enabled() &&
- (alloc_flags & ALLOC_CPUSET) &&
- !__cpuset_zone_allowed(zone, gfp_mask))
- continue;
+ if (!numa_zone_alloc_allowed(alloc_flags, zone, gfp_mask))
+ continue;
if (prio > MIN_COMPACT_PRIORITY
&& compaction_deferred(zone, order)) {
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 51273baec9e5..f2b914ab5910 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1353,7 +1353,7 @@ static struct folio *dequeue_hugetlb_folio_nodemask(struct hstate *h, gfp_t gfp_
for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) {
struct folio *folio;
- if (!cpuset_zone_allowed(zone, gfp_mask))
+ if (!numa_zone_alloc_allowed(ALLOC_CPUSET, zone, gfp_mask))
continue;
/*
* no need to ask again on the same node. Pool is node rather than
diff --git a/mm/internal.h b/mm/internal.h
index 23ee14790227..97023748e6a9 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1206,6 +1206,8 @@ extern int node_reclaim_mode;
extern int node_reclaim(struct pglist_data *, gfp_t, unsigned int);
extern int find_next_best_node(int node, nodemask_t *used_node_mask);
+extern bool numa_zone_alloc_allowed(int alloc_flags, struct zone *zone,
+ gfp_t gfp_mask);
#else
#define node_reclaim_mode 0
@@ -1218,6 +1220,11 @@ static inline int find_next_best_node(int node, nodemask_t *used_node_mask)
{
return NUMA_NO_NODE;
}
+static inline bool numa_zone_alloc_allowed(int alloc_flags, struct zone *zone,
+ gfp_t gfp_mask)
+{
+ return true;
+}
#endif
static inline bool node_reclaim_enabled(void)
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 2facee0805da..47f2619d3840 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -3690,6 +3690,21 @@ static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=
node_reclaim_distance;
}
+
+/* Returns true if allocation from this zone is permitted */
+bool numa_zone_alloc_allowed(int alloc_flags, struct zone *zone, gfp_t gfp_mask)
+{
+ /* Gate N_MEMORY_PRIVATE zones behind __GFP_PRIVATE */
+ if (!(gfp_mask & __GFP_PRIVATE) &&
+ node_state(zone_to_nid(zone), N_MEMORY_PRIVATE))
+ return false;
+
+ /* If cpusets is being used, check mems_allowed */
+ if (cpusets_enabled() && (alloc_flags & ALLOC_CPUSET))
+ return cpuset_zone_allowed(zone, gfp_mask);
+
+ return true;
+}
#else /* CONFIG_NUMA */
static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
{
@@ -3781,10 +3796,8 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
struct page *page;
unsigned long mark;
- if (cpusets_enabled() &&
- (alloc_flags & ALLOC_CPUSET) &&
- !__cpuset_zone_allowed(zone, gfp_mask))
- continue;
+ if (!numa_zone_alloc_allowed(alloc_flags, zone, gfp_mask))
+ continue;
/*
* When allocating a page cache page for writing, we
* want to get it from a node that is within its dirty
@@ -4585,10 +4598,8 @@ should_reclaim_retry(gfp_t gfp_mask, unsigned order,
unsigned long min_wmark = min_wmark_pages(zone);
bool wmark;
- if (cpusets_enabled() &&
- (alloc_flags & ALLOC_CPUSET) &&
- !__cpuset_zone_allowed(zone, gfp_mask))
- continue;
+ if (!numa_zone_alloc_allowed(alloc_flags, zone, gfp_mask))
+ continue;
available = reclaimable = zone_reclaimable_pages(zone);
available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
@@ -5084,10 +5095,8 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,
for_next_zone_zonelist_nodemask(zone, z, ac.highest_zoneidx, ac.nodemask) {
unsigned long mark;
- if (cpusets_enabled() && (alloc_flags & ALLOC_CPUSET) &&
- !__cpuset_zone_allowed(zone, gfp)) {
+ if (!numa_zone_alloc_allowed(alloc_flags, zone, gfp))
continue;
- }
if (nr_online_nodes > 1 && zone != zonelist_zone(ac.preferred_zoneref) &&
zone_to_nid(zone) != zonelist_node_idx(ac.preferred_zoneref)) {
diff --git a/mm/slub.c b/mm/slub.c
index 861592ac5425..e4bd6ede81d1 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3595,7 +3595,8 @@ static struct slab *get_any_partial(struct kmem_cache *s,
n = get_node(s, zone_to_nid(zone));
- if (n && cpuset_zone_allowed(zone, pc->flags) &&
+ if (n && numa_zone_alloc_allowed(ALLOC_CPUSET, zone,
+ pc->flags) &&
n->nr_partial > s->min_partial) {
slab = get_partial_node(s, n, pc);
if (slab) {
--
2.53.0
^ permalink raw reply related
* [RFC PATCH v4 04/27] mm/page_alloc: Add private node handling to build_zonelists
From: Gregory Price @ 2026-02-22 8:48 UTC (permalink / raw)
To: lsf-pc
Cc: linux-kernel, linux-cxl, cgroups, linux-mm, linux-trace-kernel,
damon, kernel-team, gregkh, rafael, dakr, dave, jonathan.cameron,
dave.jiang, alison.schofield, vishal.l.verma, ira.weiny,
dan.j.williams, longman, akpm, david, lorenzo.stoakes,
Liam.Howlett, vbabka, rppt, surenb, mhocko, osalvador, ziy,
matthew.brost, joshua.hahnjy, rakie.kim, byungchul, gourry,
ying.huang, apopple, axelrasmussen, yuanchu, weixugc, yury.norov,
linux, mhiramat, mathieu.desnoyers, tj, hannes, mkoutny, jackmanb,
sj, baolin.wang, npache, ryan.roberts, dev.jain, baohua,
lance.yang, muchun.song, xu.xin16, chengming.zhou, jannh,
linmiaohe, nao.horiguchi, pfalcato, rientjes, shakeel.butt, riel,
harry.yoo, cl, roman.gushchin, chrisl, kasong, shikemeng, nphamcs,
bhe, zhengqi.arch, terry.bowman
In-Reply-To: <20260222084842.1824063-1-gourry@gourry.net>
N_MEMORY fallback lists should not include N_MEMORY_PRIVATE nodes, at
worst this would allow allocation from them in some scenarios, and at
best it causes iterations over nodes that aren't eligible.
Private node primary fallback lists do include N_MEMORY nodes so
kernel/slab allocations made on behalf of the private node can
fall back to DRAM when __GFP_PRIVATE is not set.
The nofallback list contains only the node's own zones, restricting
__GFP_THISNODE allocations to the private node.
Signed-off-by: Gregory Price <gourry@gourry.net>
---
mm/page_alloc.c | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 47f2619d3840..5a1b35421d78 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -5683,6 +5683,26 @@ static void build_zonelists(pg_data_t *pgdat)
local_node = pgdat->node_id;
prev_node = local_node;
+ /*
+ * Private nodes need N_MEMORY nodes as fallback for kernel allocations
+ * (e.g., slab objects allocated on behalf of this node).
+ */
+ if (node_state(local_node, N_MEMORY_PRIVATE)) {
+ node_order[nr_nodes++] = local_node;
+ node_set(local_node, used_mask);
+
+ while ((node = find_next_best_node(local_node, &used_mask)) >= 0)
+ node_order[nr_nodes++] = node;
+
+ build_zonelists_in_node_order(pgdat, node_order, nr_nodes);
+ build_thisnode_zonelists(pgdat);
+ pr_info("Fallback order for Node %d (private):", local_node);
+ for (node = 0; node < nr_nodes; node++)
+ pr_cont(" %d", node_order[node]);
+ pr_cont("\n");
+ return;
+ }
+
memset(node_order, 0, sizeof(node_order));
while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
/*
--
2.53.0
^ permalink raw reply related
* [RFC PATCH v4 05/27] mm: introduce folio_is_private_managed() unified predicate
From: Gregory Price @ 2026-02-22 8:48 UTC (permalink / raw)
To: lsf-pc
Cc: linux-kernel, linux-cxl, cgroups, linux-mm, linux-trace-kernel,
damon, kernel-team, gregkh, rafael, dakr, dave, jonathan.cameron,
dave.jiang, alison.schofield, vishal.l.verma, ira.weiny,
dan.j.williams, longman, akpm, david, lorenzo.stoakes,
Liam.Howlett, vbabka, rppt, surenb, mhocko, osalvador, ziy,
matthew.brost, joshua.hahnjy, rakie.kim, byungchul, gourry,
ying.huang, apopple, axelrasmussen, yuanchu, weixugc, yury.norov,
linux, mhiramat, mathieu.desnoyers, tj, hannes, mkoutny, jackmanb,
sj, baolin.wang, npache, ryan.roberts, dev.jain, baohua,
lance.yang, muchun.song, xu.xin16, chengming.zhou, jannh,
linmiaohe, nao.horiguchi, pfalcato, rientjes, shakeel.butt, riel,
harry.yoo, cl, roman.gushchin, chrisl, kasong, shikemeng, nphamcs,
bhe, zhengqi.arch, terry.bowman
In-Reply-To: <20260222084842.1824063-1-gourry@gourry.net>
Multiple mm/ subsystems already skip operations for ZONE_DEVICE folios,
and N_MEMORY_PRIVATE folios share the checkpoints for ZONE_DEVICE pages.
Add folio_is_private_managed() as a unified predicate that returns true
for folios on N_MEMORY_PRIVATE nodes or in ZONE_DEVICE.
This predicate replaces folio_is_zone_device at skip sites where both
folio types should be excluded from an MM operation.
At some locations, explicit zone_device vs private_node checks are more
appropriate when the operations between the two fundamentally differ.
The !CONFIG_NUMA stubs fall through to folio_is_zone_device() only,
preserving existing behavior when NUMA is disabled.
Signed-off-by: Gregory Price <gourry@gourry.net>
---
include/linux/node_private.h | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/include/linux/node_private.h b/include/linux/node_private.h
index 6a70ec39d569..7687a4cf990c 100644
--- a/include/linux/node_private.h
+++ b/include/linux/node_private.h
@@ -92,6 +92,16 @@ static inline bool page_is_private_node(struct page *page)
return node_state(page_to_nid(page), N_MEMORY_PRIVATE);
}
+static inline bool folio_is_private_managed(struct folio *folio)
+{
+ return folio_is_zone_device(folio) || folio_is_private_node(folio);
+}
+
+static inline bool page_is_private_managed(struct page *page)
+{
+ return folio_is_private_managed(page_folio(page));
+}
+
static inline const struct node_private_ops *
folio_node_private_ops(struct folio *folio)
{
@@ -146,6 +156,16 @@ static inline bool page_is_private_node(struct page *page)
return false;
}
+static inline bool folio_is_private_managed(struct folio *folio)
+{
+ return folio_is_zone_device(folio);
+}
+
+static inline bool page_is_private_managed(struct page *page)
+{
+ return folio_is_private_managed(page_folio(page));
+}
+
static inline const struct node_private_ops *
folio_node_private_ops(struct folio *folio)
{
--
2.53.0
^ permalink raw reply related
* [RFC PATCH v4 06/27] mm/mlock: skip mlock for managed-memory folios
From: Gregory Price @ 2026-02-22 8:48 UTC (permalink / raw)
To: lsf-pc
Cc: linux-kernel, linux-cxl, cgroups, linux-mm, linux-trace-kernel,
damon, kernel-team, gregkh, rafael, dakr, dave, jonathan.cameron,
dave.jiang, alison.schofield, vishal.l.verma, ira.weiny,
dan.j.williams, longman, akpm, david, lorenzo.stoakes,
Liam.Howlett, vbabka, rppt, surenb, mhocko, osalvador, ziy,
matthew.brost, joshua.hahnjy, rakie.kim, byungchul, gourry,
ying.huang, apopple, axelrasmussen, yuanchu, weixugc, yury.norov,
linux, mhiramat, mathieu.desnoyers, tj, hannes, mkoutny, jackmanb,
sj, baolin.wang, npache, ryan.roberts, dev.jain, baohua,
lance.yang, muchun.song, xu.xin16, chengming.zhou, jannh,
linmiaohe, nao.horiguchi, pfalcato, rientjes, shakeel.butt, riel,
harry.yoo, cl, roman.gushchin, chrisl, kasong, shikemeng, nphamcs,
bhe, zhengqi.arch, terry.bowman
In-Reply-To: <20260222084842.1824063-1-gourry@gourry.net>
Private node folios are managed by device drivers and should not be
mlocked. The existing folio_is_zone_device check is already correctly
placed to handle this - simply extend it for private nodes.
Signed-off-by: Gregory Price <gourry@gourry.net>
---
mm/mlock.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/mm/mlock.c b/mm/mlock.c
index 2f699c3497a5..c56159253e45 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -25,6 +25,7 @@
#include <linux/memcontrol.h>
#include <linux/mm_inline.h>
#include <linux/secretmem.h>
+#include <linux/node_private.h>
#include "internal.h"
@@ -366,7 +367,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
if (is_huge_zero_pmd(*pmd))
goto out;
folio = pmd_folio(*pmd);
- if (folio_is_zone_device(folio))
+ if (unlikely(folio_is_private_managed(folio)))
goto out;
if (vma->vm_flags & VM_LOCKED)
mlock_folio(folio);
@@ -386,7 +387,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
if (!pte_present(ptent))
continue;
folio = vm_normal_folio(vma, addr, ptent);
- if (!folio || folio_is_zone_device(folio))
+ if (!folio || unlikely(folio_is_private_managed(folio)))
continue;
step = folio_mlock_step(folio, pte, addr, end);
--
2.53.0
^ permalink raw reply related
* [RFC PATCH v4 07/27] mm/madvise: skip madvise for managed-memory folios
From: Gregory Price @ 2026-02-22 8:48 UTC (permalink / raw)
To: lsf-pc
Cc: linux-kernel, linux-cxl, cgroups, linux-mm, linux-trace-kernel,
damon, kernel-team, gregkh, rafael, dakr, dave, jonathan.cameron,
dave.jiang, alison.schofield, vishal.l.verma, ira.weiny,
dan.j.williams, longman, akpm, david, lorenzo.stoakes,
Liam.Howlett, vbabka, rppt, surenb, mhocko, osalvador, ziy,
matthew.brost, joshua.hahnjy, rakie.kim, byungchul, gourry,
ying.huang, apopple, axelrasmussen, yuanchu, weixugc, yury.norov,
linux, mhiramat, mathieu.desnoyers, tj, hannes, mkoutny, jackmanb,
sj, baolin.wang, npache, ryan.roberts, dev.jain, baohua,
lance.yang, muchun.song, xu.xin16, chengming.zhou, jannh,
linmiaohe, nao.horiguchi, pfalcato, rientjes, shakeel.butt, riel,
harry.yoo, cl, roman.gushchin, chrisl, kasong, shikemeng, nphamcs,
bhe, zhengqi.arch, terry.bowman
In-Reply-To: <20260222084842.1824063-1-gourry@gourry.net>
Private node folios are managed by device drivers and should not be
subjectto madvise cold/pageout/free operations that would interfere
with the driver's memory management.
Extend the existing zone_device check to cover private nodes.
Signed-off-by: Gregory Price <gourry@gourry.net>
---
mm/madvise.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/mm/madvise.c b/mm/madvise.c
index b617b1be0f53..3aac105e840b 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -32,6 +32,7 @@
#include <linux/leafops.h>
#include <linux/shmem_fs.h>
#include <linux/mmu_notifier.h>
+#include <linux/node_private.h>
#include <asm/tlb.h>
@@ -475,7 +476,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
continue;
folio = vm_normal_folio(vma, addr, ptent);
- if (!folio || folio_is_zone_device(folio))
+ if (!folio || unlikely(folio_is_private_managed(folio)))
continue;
/*
@@ -704,7 +705,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
}
folio = vm_normal_folio(vma, addr, ptent);
- if (!folio || folio_is_zone_device(folio))
+ if (!folio || unlikely(folio_is_private_managed(folio)))
continue;
/*
--
2.53.0
^ permalink raw reply related
* [RFC PATCH v4 08/27] mm/ksm: skip KSM for managed-memory folios
From: Gregory Price @ 2026-02-22 8:48 UTC (permalink / raw)
To: lsf-pc
Cc: linux-kernel, linux-cxl, cgroups, linux-mm, linux-trace-kernel,
damon, kernel-team, gregkh, rafael, dakr, dave, jonathan.cameron,
dave.jiang, alison.schofield, vishal.l.verma, ira.weiny,
dan.j.williams, longman, akpm, david, lorenzo.stoakes,
Liam.Howlett, vbabka, rppt, surenb, mhocko, osalvador, ziy,
matthew.brost, joshua.hahnjy, rakie.kim, byungchul, gourry,
ying.huang, apopple, axelrasmussen, yuanchu, weixugc, yury.norov,
linux, mhiramat, mathieu.desnoyers, tj, hannes, mkoutny, jackmanb,
sj, baolin.wang, npache, ryan.roberts, dev.jain, baohua,
lance.yang, muchun.song, xu.xin16, chengming.zhou, jannh,
linmiaohe, nao.horiguchi, pfalcato, rientjes, shakeel.butt, riel,
harry.yoo, cl, roman.gushchin, chrisl, kasong, shikemeng, nphamcs,
bhe, zhengqi.arch, terry.bowman
In-Reply-To: <20260222084842.1824063-1-gourry@gourry.net>
Private node folios should not participate in KSM merging by default.
The driver manages the memory lifecycle and KSM's page sharing can
interfere with driver operations.
Extend the existing zone_device checks in get_mergeable_page and
ksm_next_page_pmd_entry to cover private node folios as well.
Signed-off-by: Gregory Price <gourry@gourry.net>
---
mm/ksm.c | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/mm/ksm.c b/mm/ksm.c
index 2d89a7c8b4eb..c48e95a6fff9 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -40,6 +40,7 @@
#include <linux/oom.h>
#include <linux/numa.h>
#include <linux/pagewalk.h>
+#include <linux/node_private.h>
#include <asm/tlbflush.h>
#include "internal.h"
@@ -808,7 +809,7 @@ static struct page *get_mergeable_page(struct ksm_rmap_item *rmap_item)
folio = folio_walk_start(&fw, vma, addr, 0);
if (folio) {
- if (!folio_is_zone_device(folio) &&
+ if (!folio_is_private_managed(folio) &&
folio_test_anon(folio)) {
folio_get(folio);
page = fw.page;
@@ -2521,7 +2522,8 @@ static int ksm_next_page_pmd_entry(pmd_t *pmdp, unsigned long addr, unsigned lon
goto not_found_unlock;
folio = page_folio(page);
- if (folio_is_zone_device(folio) || !folio_test_anon(folio))
+ if (unlikely(folio_is_private_managed(folio)) ||
+ !folio_test_anon(folio))
goto not_found_unlock;
page += ((addr & (PMD_SIZE - 1)) >> PAGE_SHIFT);
@@ -2545,7 +2547,8 @@ static int ksm_next_page_pmd_entry(pmd_t *pmdp, unsigned long addr, unsigned lon
continue;
folio = page_folio(page);
- if (folio_is_zone_device(folio) || !folio_test_anon(folio))
+ if (unlikely(folio_is_private_managed(folio)) ||
+ !folio_test_anon(folio))
continue;
goto found_unlock;
}
--
2.53.0
^ permalink raw reply related
* [RFC PATCH v4 09/27] mm/khugepaged: skip private node folios when trying to collapse.
From: Gregory Price @ 2026-02-22 8:48 UTC (permalink / raw)
To: lsf-pc
Cc: linux-kernel, linux-cxl, cgroups, linux-mm, linux-trace-kernel,
damon, kernel-team, gregkh, rafael, dakr, dave, jonathan.cameron,
dave.jiang, alison.schofield, vishal.l.verma, ira.weiny,
dan.j.williams, longman, akpm, david, lorenzo.stoakes,
Liam.Howlett, vbabka, rppt, surenb, mhocko, osalvador, ziy,
matthew.brost, joshua.hahnjy, rakie.kim, byungchul, gourry,
ying.huang, apopple, axelrasmussen, yuanchu, weixugc, yury.norov,
linux, mhiramat, mathieu.desnoyers, tj, hannes, mkoutny, jackmanb,
sj, baolin.wang, npache, ryan.roberts, dev.jain, baohua,
lance.yang, muchun.song, xu.xin16, chengming.zhou, jannh,
linmiaohe, nao.horiguchi, pfalcato, rientjes, shakeel.butt, riel,
harry.yoo, cl, roman.gushchin, chrisl, kasong, shikemeng, nphamcs,
bhe, zhengqi.arch, terry.bowman
In-Reply-To: <20260222084842.1824063-1-gourry@gourry.net>
A collapse operation allocates a new large folio and migrates the
smaller folios into it. This is an issue for private nodes:
1. The private node service may not support migration
2. Collapse may promotes pages from the private node to a local node,
which may result in an LRU inversion that defeats memory tiering.
Handle this just like zone_device for now.
It may be possible to support this later for some private node services
that report explicit support for collapse (and migration).
Signed-off-by: Gregory Price <gourry@gourry.net>
---
mm/khugepaged.c | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 97d1b2824386..36f6bc5da53c 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -21,6 +21,7 @@
#include <linux/shmem_fs.h>
#include <linux/dax.h>
#include <linux/ksm.h>
+#include <linux/node_private.h>
#include <linux/pgalloc.h>
#include <asm/tlb.h>
@@ -571,7 +572,7 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
goto out;
}
page = vm_normal_page(vma, addr, pteval);
- if (unlikely(!page) || unlikely(is_zone_device_page(page))) {
+ if (unlikely(!page) || unlikely(page_is_private_managed(page))) {
result = SCAN_PAGE_NULL;
goto out;
}
@@ -1323,7 +1324,7 @@ static int hpage_collapse_scan_pmd(struct mm_struct *mm,
}
page = vm_normal_page(vma, addr, pteval);
- if (unlikely(!page) || unlikely(is_zone_device_page(page))) {
+ if (unlikely(!page) || unlikely(page_is_private_managed(page))) {
result = SCAN_PAGE_NULL;
goto out_unmap;
}
@@ -1575,7 +1576,7 @@ int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr,
}
page = vm_normal_page(vma, addr, ptent);
- if (WARN_ON_ONCE(page && is_zone_device_page(page)))
+ if (WARN_ON_ONCE(page && page_is_private_managed(page)))
page = NULL;
/*
* Note that uprobe, debugger, or MAP_PRIVATE may change the
--
2.53.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox