From: Dylan Hatch <dylanbhatch@google.com>
To: Catalin Marinas <catalin.marinas@arm.com>,
Will Deacon <will@kernel.org>,
Mark Rutland <mark.rutland@arm.com>
Cc: Herbert Xu <herbert@gondor.apana.org.au>,
"David S . Miller" <davem@davemloft.net>,
Nathan Chancellor <nathan@kernel.org>,
Nicolas Schier <nsc@kernel.org>,
Nick Desaulniers <ndesaulniers@google.com>,
Bill Wendling <morbo@google.com>,
Justin Stitt <justinstitt@google.com>,
Weinan Liu <wnliu@google.com>, Jens Remus <jremus@linux.ibm.com>,
linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org, linux-kbuild@vger.kernel.org,
linux-crypto@vger.kernel.org, llvm@lists.linux.dev,
Josh Poimboeuf <jpoimboe@kernel.org>,
Steven Rostedt <rostedt@goodmis.org>,
Ard Biesheuvel <ardb@kernel.org>,
Indu Bhagat <ibhagatgnu@gmail.com>,
Prasanna Kumar T S M <ptsm@linux.microsoft.com>,
live-patching@vger.kernel.org,
Dylan Hatch <dylanbhatch@google.com>
Subject: [RFC PATCH 08/12] arm64, unwind: add in-kernel .lr_unwind lookup and validation
Date: Tue, 17 Nov 2026 04:35:00 +0000 [thread overview]
Message-ID: <20261117043504.1021063-9-dylanbhatch@google.com> (raw)
In-Reply-To: <20261117043504.1021063-1-dylanbhatch@google.com>
Add runtime lookup and optional boot-time validation for the vmlinux
.lr_unwind_fde and .lr_unwind_ranges tables.
lr_unwind_find() binary searches the sorted .lr_unwind_fde table with
monotonic bounds checking, validates start IPs against arm64 kernel text
sections (.text, .init.text, .exit.text, and .rodata.text), and scans
the inline or out-of-line instruction ranges to determine the exception
boundary unwind mode (LR_UNWIND_LIVE, LR_UNWIND_FP, or LR_UNWIND_SP) and
any SP-relative frame-record offset.
Suggested-by: Jens Remus <jremus@linux.ibm.com>
Signed-off-by: Dylan Hatch <dylanbhatch@google.com>
---
arch/arm64/Kconfig | 9 +
arch/arm64/include/asm/lr_unwind.h | 31 ++++
arch/arm64/include/asm/sections.h | 3 +
arch/arm64/kernel/Makefile | 5 +
arch/arm64/kernel/lr_unwind.c | 274 +++++++++++++++++++++++++++++
arch/arm64/kernel/setup.c | 3 +
arch/arm64/kernel/vmlinux.lds.S | 2 +
7 files changed, 327 insertions(+)
create mode 100644 arch/arm64/kernel/lr_unwind.c
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index aa09c7ea61be3..1503b8a0b7021 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -2517,6 +2517,15 @@ config ARM64_LR_UNWIND
live in x30, saved in the frame record at [x29], or saved in the
frame record at an SP-relative offset).
+config ARM64_LR_UNWIND_VALIDATION
+ bool "Validate .lr_unwind sections at boot and module load"
+ depends on ARM64_LR_UNWIND
+ help
+ Validate every FDE and range entry in .lr_unwind_fde and
+ .lr_unwind_ranges when initializing vmlinux and module unwind
+ tables. This is a debugging option that adds boot and module load
+ overhead and should normally be disabled in production.
+
config ARM64_CONTPTE
bool "Contiguous PTE mappings for user memory" if EXPERT
depends on TRANSPARENT_HUGEPAGE
diff --git a/arch/arm64/include/asm/lr_unwind.h b/arch/arm64/include/asm/lr_unwind.h
index 81c743994c4d5..bfb6abed05f76 100644
--- a/arch/arm64/include/asm/lr_unwind.h
+++ b/arch/arm64/include/asm/lr_unwind.h
@@ -108,4 +108,35 @@ static __always_inline void lr_fde_unpack_inline(u32 desc, lr_range_t ranges[4])
#define LR_OOL_IDX(d) (((d) >> 6) & LR_OOL_IDX_MAX)
#define LR_OOL_COUNT(d) ((d) & LR_OOL_COUNT_MAX)
+#ifdef __KERNEL__
+
+#include <linux/errno.h>
+#include <linux/init.h>
+
+struct lr_unwind_section {
+ const struct lr_fde *fdes;
+ unsigned int num_fdes;
+ const lr_range_t *ranges;
+ unsigned int num_ranges;
+};
+
+#ifdef CONFIG_ARM64_LR_UNWIND
+
+void __init init_lr_unwind_table(void);
+int lr_unwind_find(unsigned long ip, enum lr_unwind_mode *mode,
+ unsigned int *sp_off);
+
+#else /* !CONFIG_ARM64_LR_UNWIND */
+
+static inline void __init init_lr_unwind_table(void) {}
+static inline int lr_unwind_find(unsigned long ip, enum lr_unwind_mode *mode,
+ unsigned int *sp_off)
+{
+ return -EINVAL;
+}
+
+#endif /* CONFIG_ARM64_LR_UNWIND */
+
+#endif /* __KERNEL__ */
+
#endif /* __ASM_LR_UNWIND_H */
diff --git a/arch/arm64/include/asm/sections.h b/arch/arm64/include/asm/sections.h
index 51b0d594239eb..2f950aa28225f 100644
--- a/arch/arm64/include/asm/sections.h
+++ b/arch/arm64/include/asm/sections.h
@@ -23,6 +23,9 @@ extern char __irqentry_text_start[], __irqentry_text_end[];
extern char __mmuoff_data_start[], __mmuoff_data_end[];
extern char __entry_tramp_text_start[], __entry_tramp_text_end[];
extern char __relocate_new_kernel_start[], __relocate_new_kernel_end[];
+extern char _srodatatext[], _erodatatext[];
+extern char __start_lr_unwind_fde[], __stop_lr_unwind_fde[];
+extern char __start_lr_unwind_ranges[], __stop_lr_unwind_ranges[];
static inline size_t entry_tramp_text_size(void)
{
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index fc5d1a0c38b3c..f07a0cead3ee3 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -18,12 +18,16 @@ CFLAGS_syscall.o += -fno-stack-protector
# can significantly impact performance. Avoid instrumenting the stack trace
# collection code to minimize this impact.
KASAN_SANITIZE_stacktrace.o := n
+KASAN_SANITIZE_lr_unwind.o := n
+KCSAN_SANITIZE_lr_unwind.o := n
+UBSAN_SANITIZE_lr_unwind.o := n
# It's not safe to invoke KCOV when portions of the kernel environment aren't
# available or are out-of-sync with HW state. Since `noinstr` doesn't always
# inhibit KCOV instrumentation, disable it for the entire compilation unit.
KCOV_INSTRUMENT_entry-common.o := n
KCOV_INSTRUMENT_idle.o := n
+KCOV_INSTRUMENT_lr_unwind.o := n
# Softirq stack switching can run outside interrupt context.
KCOV_INSTRUMENT_irq.o := n
@@ -71,6 +75,7 @@ obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o
obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o
obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o
+obj-$(CONFIG_ARM64_LR_UNWIND) += lr_unwind.o
obj-$(CONFIG_ARM64_MPAM) += mpam.o
obj-$(CONFIG_ARM64_MTE) += mte.o
obj-y += vdso-wrap.o
diff --git a/arch/arm64/kernel/lr_unwind.c b/arch/arm64/kernel/lr_unwind.c
new file mode 100644
index 0000000000000..53e2c78a8bdaf
--- /dev/null
+++ b/arch/arm64/kernel/lr_unwind.c
@@ -0,0 +1,274 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Kernel .lr_unwind lookup and validation
+ */
+
+#define pr_fmt(fmt) "lr_unwind: " fmt
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/kprobes.h>
+#include <linux/limits.h>
+#include <linux/printk.h>
+
+#include <asm/lr_unwind.h>
+#include <asm/sections.h>
+
+static bool lr_unwind_init __ro_after_init;
+static struct lr_unwind_section kernel_lr_sec __ro_after_init;
+
+static const char *objname(const struct lr_unwind_section *sec)
+{
+ return "vmlinux";
+}
+
+#define dbg_sec(sec, fmt, ...) \
+ pr_debug("(%s): " fmt, objname(sec), ##__VA_ARGS__)
+
+static __always_inline unsigned long lr_fde_start_addr(const struct lr_fde *fde)
+{
+ return (unsigned long)&fde->start_offset + fde->start_offset;
+}
+
+static __always_inline bool is_kernel_ip_valid(unsigned long ip)
+{
+ if (ip >= (unsigned long)_stext && ip < (unsigned long)_etext)
+ return true;
+
+ if (system_state < SYSTEM_FREEING_INITMEM &&
+ ip >= (unsigned long)_sinittext && ip < (unsigned long)_einittext)
+ return true;
+
+ /* .exit.text is retained in vmlinux on arm64. */
+ if (ip >= (unsigned long)__exittext_begin &&
+ ip < (unsigned long)__exittext_end)
+ return true;
+
+ /*
+ * .rodata.text is never executed from the kernel mapping, but it still
+ * has unwind data.
+ */
+ if (ip >= (unsigned long)_srodatatext &&
+ ip < (unsigned long)_erodatatext)
+ return true;
+
+ return false;
+}
+
+static __always_inline bool is_sec_ip_valid(const struct lr_unwind_section *sec,
+ unsigned long ip)
+{
+ return is_kernel_ip_valid(ip);
+}
+
+static __always_inline int __find_fde(const struct lr_unwind_section *sec,
+ unsigned long ip,
+ const struct lr_fde **fde_out)
+{
+ unsigned long func_addr_low = 0, func_addr_high = ULONG_MAX;
+ unsigned int low = 0, high = sec->num_fdes;
+ const struct lr_fde *found = NULL;
+
+ while (low < high) {
+ unsigned int mid = low + ((high - low) / 2);
+ const struct lr_fde *fde = &sec->fdes[mid];
+ unsigned long func_addr = lr_fde_start_addr(fde);
+
+ if (ip >= func_addr) {
+ if (func_addr < func_addr_low)
+ return -EFAULT;
+
+ func_addr_low = func_addr;
+ found = fde;
+ low = mid + 1;
+ } else {
+ if (func_addr > func_addr_high)
+ return -EFAULT;
+
+ func_addr_high = func_addr;
+ high = mid;
+ }
+ }
+
+ if (!found || !is_sec_ip_valid(sec, lr_fde_start_addr(found)))
+ return -EINVAL;
+
+ *fde_out = found;
+ return 0;
+}
+
+static __always_inline int __lr_unwind_find(const struct lr_unwind_section *sec,
+ unsigned long ip,
+ enum lr_unwind_mode *mode,
+ unsigned int *sp_off)
+{
+ lr_range_t inline_ranges[4];
+ const lr_range_t *ranges;
+ const struct lr_fde *fde;
+ unsigned long insn_off;
+ u32 desc, cnt;
+ int ret;
+
+ if (ip & 0x3)
+ return -EINVAL;
+
+ ret = __find_fde(sec, ip, &fde);
+ if (ret)
+ return ret;
+
+ insn_off = (ip - lr_fde_start_addr(fde)) >> 2;
+ desc = fde->range_desc;
+
+ if (likely(desc & LR_FDE_INLINE)) {
+ lr_fde_unpack_inline(desc, inline_ranges);
+ ranges = inline_ranges;
+ cnt = 4;
+ } else {
+ u32 idx = LR_OOL_IDX(desc);
+
+ cnt = LR_OOL_COUNT(desc);
+ if (!cnt || idx + cnt > sec->num_ranges)
+ return -EFAULT;
+ ranges = &sec->ranges[idx];
+ }
+
+ for (u32 i = 0; i < cnt; i++) {
+ u32 len = LR_RANGE_LEN(ranges[i]);
+
+ if (insn_off < len) {
+ enum lr_unwind_mode m = LR_RANGE_MODE(ranges[i]);
+
+ if (unlikely(m > LR_UNWIND_SP))
+ return -EFAULT;
+ *mode = m;
+ *sp_off = LR_FDE_SP_OFF_BYTES(desc);
+ return 0;
+ }
+ insn_off -= len;
+ }
+
+ /* Make sure IP is not in an unannotated gap past this FDE */
+ return -EINVAL;
+}
+
+noinstr int lr_unwind_find(unsigned long ip, enum lr_unwind_mode *mode,
+ unsigned int *sp_off)
+{
+ if (!mode || !sp_off)
+ return -EINVAL;
+
+ if (is_kernel_ip_valid(ip)) {
+ if (!lr_unwind_init)
+ return -EINVAL;
+
+ return __lr_unwind_find(&kernel_lr_sec, ip, mode, sp_off);
+ }
+
+ return -EINVAL;
+}
+NOKPROBE_SYMBOL(lr_unwind_find);
+
+#ifdef CONFIG_ARM64_LR_UNWIND_VALIDATION
+
+static int lr_unwind_validate_section(const struct lr_unwind_section *sec)
+{
+ unsigned long prev_end = 0;
+
+ for (unsigned int i = 0; i < sec->num_fdes; i++) {
+ const struct lr_fde *fde = &sec->fdes[i];
+ unsigned long ip = lr_fde_start_addr(fde);
+ u32 desc = fde->range_desc;
+ u32 total_insns = 0;
+
+ if ((ip & 0x3) || !is_sec_ip_valid(sec, ip)) {
+ dbg_sec(sec, "FDE %u invalid start IP 0x%lx\n", i, ip);
+ return -EINVAL;
+ }
+
+ if (ip < prev_end) {
+ dbg_sec(sec, "FDE %u unsorted or overlapping\n", i);
+ return -EFAULT;
+ }
+
+ if (desc & LR_FDE_INLINE) {
+ total_insns = LR_INLINE_R0(desc) +
+ LR_INLINE_R1(desc) +
+ LR_INLINE_R2(desc) +
+ LR_INLINE_R3(desc);
+ } else {
+ u32 idx = LR_OOL_IDX(desc);
+ u32 cnt = LR_OOL_COUNT(desc);
+
+ if (!cnt || idx + cnt > sec->num_ranges) {
+ dbg_sec(sec, "FDE %u OOL range out of bounds\n", i);
+ return -EFAULT;
+ }
+ for (u32 j = 0; j < cnt; j++) {
+ lr_range_t r = sec->ranges[idx + j];
+ u32 len = LR_RANGE_LEN(r);
+
+ if (!len || LR_RANGE_MODE(r) > LR_UNWIND_SP) {
+ dbg_sec(sec, "FDE %u invalid OOL entry\n", i);
+ return -EINVAL;
+ }
+ total_insns += len;
+ }
+ }
+
+ if (!total_insns) {
+ dbg_sec(sec, "FDE %u has empty range\n", i);
+ return -EINVAL;
+ }
+
+ prev_end = ip + ((unsigned long)total_insns << 2);
+ if (!is_sec_ip_valid(sec, prev_end - 4)) {
+ dbg_sec(sec, "FDE %u end IP out of bounds\n", i);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+#else /* !CONFIG_ARM64_LR_UNWIND_VALIDATION */
+
+static int lr_unwind_validate_section(const struct lr_unwind_section *sec)
+{
+ return 0;
+}
+
+#endif /* CONFIG_ARM64_LR_UNWIND_VALIDATION */
+
+static void dbg_print_section(const struct lr_unwind_section *sec)
+{
+ dbg_sec(sec, "fdes:0x%lx (%u) ranges:0x%lx (%u)\n",
+ (unsigned long)sec->fdes, sec->num_fdes,
+ (unsigned long)sec->ranges, sec->num_ranges);
+}
+
+void __init init_lr_unwind_table(void)
+{
+ size_t fde_bytes = (unsigned long)__stop_lr_unwind_fde -
+ (unsigned long)__start_lr_unwind_fde;
+ size_t range_bytes = (unsigned long)__stop_lr_unwind_ranges -
+ (unsigned long)__start_lr_unwind_ranges;
+
+ if (!fde_bytes || (fde_bytes % sizeof(struct lr_fde)) ||
+ (range_bytes % sizeof(lr_range_t))) {
+ pr_warn("invalid vmlinux .lr_unwind section sizes\n");
+ return;
+ }
+
+ kernel_lr_sec.fdes = (const struct lr_fde *)__start_lr_unwind_fde;
+ kernel_lr_sec.num_fdes = fde_bytes / sizeof(struct lr_fde);
+ kernel_lr_sec.ranges = (const lr_range_t *)__start_lr_unwind_ranges;
+ kernel_lr_sec.num_ranges = range_bytes / sizeof(lr_range_t);
+
+ if (lr_unwind_validate_section(&kernel_lr_sec)) {
+ pr_warn("invalid vmlinux .lr_unwind section\n");
+ return;
+ }
+
+ lr_unwind_init = true;
+ dbg_print_section(&kernel_lr_sec);
+}
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index a707deba6ae70..a19563e382ebd 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -43,6 +43,7 @@
#include <asm/cpufeature.h>
#include <asm/cpu_ops.h>
#include <asm/kasan.h>
+#include <asm/lr_unwind.h>
#include <asm/numa.h>
#include <asm/scs.h>
#include <asm/sections.h>
@@ -373,6 +374,8 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p)
"This indicates a broken bootloader or old kernel\n",
boot_args[1], boot_args[2], boot_args[3]);
}
+
+ init_lr_unwind_table();
}
static inline bool cpu_can_disable(unsigned int cpu)
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index b8f2963d10030..dc6c4a041f691 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -256,12 +256,14 @@ SECTIONS
/* code sections that are never executed via the kernel mapping */
.rodata.text : {
+ _srodatatext = .;
TRAMP_TEXT
HIBERNATE_TEXT
KEXEC_TEXT
IDMAP_TEXT
. = ALIGN(PAGE_SIZE);
}
+ _erodatatext = .;
idmap_pg_dir = .;
. += PAGE_SIZE;
--
2.56.0.679.gc527c65306-goog
next prev parent reply other threads:[~2026-11-17 4:35 UTC|newest]
Thread overview: 18+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-17 4:34 [RFC PATCH 00/12] arm64, unwind: add .lr_unwind unwinder for reliable stacktrace Dylan Hatch
2026-11-17 4:34 ` [RFC PATCH 01/12] arm64, unwind: build with .eh_frame and add gen_lr_unwind skeleton Dylan Hatch
2026-11-18 4:36 ` sashiko-bot
2026-11-17 4:34 ` [RFC PATCH 02/12] arm64: factor .eh_frame parser out of patch-scs.c into <asm/eh_frame.h> Dylan Hatch
2026-11-18 4:36 ` sashiko-bot
2026-11-17 4:34 ` [RFC PATCH 03/12] arm64: eh_frame: track CFA and x29/x30 register state in CFI parser Dylan Hatch
2026-11-17 4:34 ` [RFC PATCH 04/12] arm64, unwind: parse .eh_frame into .lr_unwind entries in gen_lr_unwind Dylan Hatch
2026-11-17 4:34 ` [RFC PATCH 05/12] scripts/gen_lr_unwind: emit .lr_unwind_fde and .lr_unwind_ranges Dylan Hatch
2026-11-17 4:34 ` [RFC PATCH 06/12] scripts/gen_lr_unwind: add --dump option to inspect .lr_unwind tables Dylan Hatch
2026-11-18 4:36 ` sashiko-bot
2026-11-17 4:34 ` [RFC PATCH 07/12] scripts/sorttable: sort .lr_unwind_fde by start_offset Dylan Hatch
2026-11-17 4:35 ` Dylan Hatch [this message]
2026-11-17 4:35 ` [RFC PATCH 09/12] arm64/module, unwind: add .lr_unwind support for modules Dylan Hatch
2026-11-17 4:35 ` [RFC PATCH 10/12] arm64: entry: add unwind info for call_on_irq_stack() Dylan Hatch
2026-11-17 4:35 ` [RFC PATCH 11/12] arm64, crypto/lib: annotate leaf functions with CFI info Dylan Hatch
2026-11-18 5:07 ` Eric Biggers
2026-11-17 4:35 ` [RFC PATCH 12/12] unwind: arm64: use .lr_unwind to unwind interrupt frames Dylan Hatch
2026-11-18 4:36 ` sashiko-bot
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261117043504.1021063-9-dylanbhatch@google.com \
--to=dylanbhatch@google.com \
--cc=ardb@kernel.org \
--cc=catalin.marinas@arm.com \
--cc=davem@davemloft.net \
--cc=herbert@gondor.apana.org.au \
--cc=ibhagatgnu@gmail.com \
--cc=jpoimboe@kernel.org \
--cc=jremus@linux.ibm.com \
--cc=justinstitt@google.com \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-crypto@vger.kernel.org \
--cc=linux-kbuild@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=live-patching@vger.kernel.org \
--cc=llvm@lists.linux.dev \
--cc=mark.rutland@arm.com \
--cc=morbo@google.com \
--cc=nathan@kernel.org \
--cc=ndesaulniers@google.com \
--cc=nsc@kernel.org \
--cc=ptsm@linux.microsoft.com \
--cc=rostedt@goodmis.org \
--cc=will@kernel.org \
--cc=wnliu@google.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.