Linux cryptographic layer development
 help / color / mirror / Atom feed
From: Dylan Hatch <dylanbhatch@google.com>
To: Catalin Marinas <catalin.marinas@arm.com>,
	Will Deacon <will@kernel.org>,
	 Mark Rutland <mark.rutland@arm.com>
Cc: Herbert Xu <herbert@gondor.apana.org.au>,
	"David S . Miller" <davem@davemloft.net>,
	 Nathan Chancellor <nathan@kernel.org>,
	Nicolas Schier <nsc@kernel.org>,
	 Nick Desaulniers <ndesaulniers@google.com>,
	Bill Wendling <morbo@google.com>,
	 Justin Stitt <justinstitt@google.com>,
	Weinan Liu <wnliu@google.com>,  Jens Remus <jremus@linux.ibm.com>,
	linux-arm-kernel@lists.infradead.org,
	 linux-kernel@vger.kernel.org, linux-kbuild@vger.kernel.org,
	 linux-crypto@vger.kernel.org, llvm@lists.linux.dev,
	 Josh Poimboeuf <jpoimboe@kernel.org>,
	Steven Rostedt <rostedt@goodmis.org>,
	 Ard Biesheuvel <ardb@kernel.org>,
	Indu Bhagat <ibhagatgnu@gmail.com>,
	 Prasanna Kumar T S M <ptsm@linux.microsoft.com>,
	live-patching@vger.kernel.org,
	 Dylan Hatch <dylanbhatch@google.com>
Subject: [RFC PATCH 08/12] arm64, unwind: add in-kernel .lr_unwind lookup and validation
Date: Tue, 17 Nov 2026 04:35:00 +0000	[thread overview]
Message-ID: <20261117043504.1021063-9-dylanbhatch@google.com> (raw)
In-Reply-To: <20261117043504.1021063-1-dylanbhatch@google.com>

Add runtime lookup and optional boot-time validation for the vmlinux
.lr_unwind_fde and .lr_unwind_ranges tables.

lr_unwind_find() binary searches the sorted .lr_unwind_fde table with
monotonic bounds checking, validates start IPs against arm64 kernel text
sections (.text, .init.text, .exit.text, and .rodata.text), and scans
the inline or out-of-line instruction ranges to determine the exception
boundary unwind mode (LR_UNWIND_LIVE, LR_UNWIND_FP, or LR_UNWIND_SP) and
any SP-relative frame-record offset.

Suggested-by: Jens Remus <jremus@linux.ibm.com>
Signed-off-by: Dylan Hatch <dylanbhatch@google.com>
---
 arch/arm64/Kconfig                 |   9 +
 arch/arm64/include/asm/lr_unwind.h |  31 ++++
 arch/arm64/include/asm/sections.h  |   3 +
 arch/arm64/kernel/Makefile         |   5 +
 arch/arm64/kernel/lr_unwind.c      | 274 +++++++++++++++++++++++++++++
 arch/arm64/kernel/setup.c          |   3 +
 arch/arm64/kernel/vmlinux.lds.S    |   2 +
 7 files changed, 327 insertions(+)
 create mode 100644 arch/arm64/kernel/lr_unwind.c

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index aa09c7ea61be3..1503b8a0b7021 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -2517,6 +2517,15 @@ config ARM64_LR_UNWIND
 	  live in x30, saved in the frame record at [x29], or saved in the
 	  frame record at an SP-relative offset).
 
+config ARM64_LR_UNWIND_VALIDATION
+	bool "Validate .lr_unwind sections at boot and module load"
+	depends on ARM64_LR_UNWIND
+	help
+	  Validate every FDE and range entry in .lr_unwind_fde and
+	  .lr_unwind_ranges when initializing vmlinux and module unwind
+	  tables. This is a debugging option that adds boot and module load
+	  overhead and should normally be disabled in production.
+
 config ARM64_CONTPTE
 	bool "Contiguous PTE mappings for user memory" if EXPERT
 	depends on TRANSPARENT_HUGEPAGE
diff --git a/arch/arm64/include/asm/lr_unwind.h b/arch/arm64/include/asm/lr_unwind.h
index 81c743994c4d5..bfb6abed05f76 100644
--- a/arch/arm64/include/asm/lr_unwind.h
+++ b/arch/arm64/include/asm/lr_unwind.h
@@ -108,4 +108,35 @@ static __always_inline void lr_fde_unpack_inline(u32 desc, lr_range_t ranges[4])
 #define LR_OOL_IDX(d)		(((d) >> 6) & LR_OOL_IDX_MAX)
 #define LR_OOL_COUNT(d)		((d) & LR_OOL_COUNT_MAX)
 
+#ifdef __KERNEL__
+
+#include <linux/errno.h>
+#include <linux/init.h>
+
+struct lr_unwind_section {
+	const struct lr_fde	*fdes;
+	unsigned int		num_fdes;
+	const lr_range_t	*ranges;
+	unsigned int		num_ranges;
+};
+
+#ifdef CONFIG_ARM64_LR_UNWIND
+
+void __init init_lr_unwind_table(void);
+int lr_unwind_find(unsigned long ip, enum lr_unwind_mode *mode,
+		   unsigned int *sp_off);
+
+#else /* !CONFIG_ARM64_LR_UNWIND */
+
+static inline void __init init_lr_unwind_table(void) {}
+static inline int lr_unwind_find(unsigned long ip, enum lr_unwind_mode *mode,
+				 unsigned int *sp_off)
+{
+	return -EINVAL;
+}
+
+#endif /* CONFIG_ARM64_LR_UNWIND */
+
+#endif /* __KERNEL__ */
+
 #endif /* __ASM_LR_UNWIND_H */
diff --git a/arch/arm64/include/asm/sections.h b/arch/arm64/include/asm/sections.h
index 51b0d594239eb..2f950aa28225f 100644
--- a/arch/arm64/include/asm/sections.h
+++ b/arch/arm64/include/asm/sections.h
@@ -23,6 +23,9 @@ extern char __irqentry_text_start[], __irqentry_text_end[];
 extern char __mmuoff_data_start[], __mmuoff_data_end[];
 extern char __entry_tramp_text_start[], __entry_tramp_text_end[];
 extern char __relocate_new_kernel_start[], __relocate_new_kernel_end[];
+extern char _srodatatext[], _erodatatext[];
+extern char __start_lr_unwind_fde[], __stop_lr_unwind_fde[];
+extern char __start_lr_unwind_ranges[], __stop_lr_unwind_ranges[];
 
 static inline size_t entry_tramp_text_size(void)
 {
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index fc5d1a0c38b3c..f07a0cead3ee3 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -18,12 +18,16 @@ CFLAGS_syscall.o	+= -fno-stack-protector
 # can significantly impact performance. Avoid instrumenting the stack trace
 # collection code to minimize this impact.
 KASAN_SANITIZE_stacktrace.o := n
+KASAN_SANITIZE_lr_unwind.o := n
+KCSAN_SANITIZE_lr_unwind.o := n
+UBSAN_SANITIZE_lr_unwind.o := n
 
 # It's not safe to invoke KCOV when portions of the kernel environment aren't
 # available or are out-of-sync with HW state. Since `noinstr` doesn't always
 # inhibit KCOV instrumentation, disable it for the entire compilation unit.
 KCOV_INSTRUMENT_entry-common.o := n
 KCOV_INSTRUMENT_idle.o := n
+KCOV_INSTRUMENT_lr_unwind.o := n
 
 # Softirq stack switching can run outside interrupt context.
 KCOV_INSTRUMENT_irq.o := n
@@ -71,6 +75,7 @@ obj-$(CONFIG_CRASH_DUMP)		+= crash_dump.o
 obj-$(CONFIG_VMCORE_INFO)		+= vmcore_info.o
 obj-$(CONFIG_ARM_SDE_INTERFACE)		+= sdei.o
 obj-$(CONFIG_ARM64_PTR_AUTH)		+= pointer_auth.o
+obj-$(CONFIG_ARM64_LR_UNWIND)		+= lr_unwind.o
 obj-$(CONFIG_ARM64_MPAM)		+= mpam.o
 obj-$(CONFIG_ARM64_MTE)			+= mte.o
 obj-y					+= vdso-wrap.o
diff --git a/arch/arm64/kernel/lr_unwind.c b/arch/arm64/kernel/lr_unwind.c
new file mode 100644
index 0000000000000..53e2c78a8bdaf
--- /dev/null
+++ b/arch/arm64/kernel/lr_unwind.c
@@ -0,0 +1,274 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Kernel .lr_unwind lookup and validation
+ */
+
+#define pr_fmt(fmt)	"lr_unwind: " fmt
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/kprobes.h>
+#include <linux/limits.h>
+#include <linux/printk.h>
+
+#include <asm/lr_unwind.h>
+#include <asm/sections.h>
+
+static bool lr_unwind_init __ro_after_init;
+static struct lr_unwind_section kernel_lr_sec __ro_after_init;
+
+static const char *objname(const struct lr_unwind_section *sec)
+{
+	return "vmlinux";
+}
+
+#define dbg_sec(sec, fmt, ...) \
+	pr_debug("(%s): " fmt, objname(sec), ##__VA_ARGS__)
+
+static __always_inline unsigned long lr_fde_start_addr(const struct lr_fde *fde)
+{
+	return (unsigned long)&fde->start_offset + fde->start_offset;
+}
+
+static __always_inline bool is_kernel_ip_valid(unsigned long ip)
+{
+	if (ip >= (unsigned long)_stext && ip < (unsigned long)_etext)
+		return true;
+
+	if (system_state < SYSTEM_FREEING_INITMEM &&
+	    ip >= (unsigned long)_sinittext && ip < (unsigned long)_einittext)
+		return true;
+
+	/* .exit.text is retained in vmlinux on arm64. */
+	if (ip >= (unsigned long)__exittext_begin &&
+	    ip < (unsigned long)__exittext_end)
+		return true;
+
+	/*
+	 * .rodata.text is never executed from the kernel mapping, but it still
+	 * has unwind data.
+	 */
+	if (ip >= (unsigned long)_srodatatext &&
+	    ip < (unsigned long)_erodatatext)
+		return true;
+
+	return false;
+}
+
+static __always_inline bool is_sec_ip_valid(const struct lr_unwind_section *sec,
+					    unsigned long ip)
+{
+	return is_kernel_ip_valid(ip);
+}
+
+static __always_inline int __find_fde(const struct lr_unwind_section *sec,
+				      unsigned long ip,
+				      const struct lr_fde **fde_out)
+{
+	unsigned long func_addr_low = 0, func_addr_high = ULONG_MAX;
+	unsigned int low = 0, high = sec->num_fdes;
+	const struct lr_fde *found = NULL;
+
+	while (low < high) {
+		unsigned int mid = low + ((high - low) / 2);
+		const struct lr_fde *fde = &sec->fdes[mid];
+		unsigned long func_addr = lr_fde_start_addr(fde);
+
+		if (ip >= func_addr) {
+			if (func_addr < func_addr_low)
+				return -EFAULT;
+
+			func_addr_low = func_addr;
+			found = fde;
+			low = mid + 1;
+		} else {
+			if (func_addr > func_addr_high)
+				return -EFAULT;
+
+			func_addr_high = func_addr;
+			high = mid;
+		}
+	}
+
+	if (!found || !is_sec_ip_valid(sec, lr_fde_start_addr(found)))
+		return -EINVAL;
+
+	*fde_out = found;
+	return 0;
+}
+
+static __always_inline int __lr_unwind_find(const struct lr_unwind_section *sec,
+					    unsigned long ip,
+					    enum lr_unwind_mode *mode,
+					    unsigned int *sp_off)
+{
+	lr_range_t inline_ranges[4];
+	const lr_range_t *ranges;
+	const struct lr_fde *fde;
+	unsigned long insn_off;
+	u32 desc, cnt;
+	int ret;
+
+	if (ip & 0x3)
+		return -EINVAL;
+
+	ret = __find_fde(sec, ip, &fde);
+	if (ret)
+		return ret;
+
+	insn_off = (ip - lr_fde_start_addr(fde)) >> 2;
+	desc = fde->range_desc;
+
+	if (likely(desc & LR_FDE_INLINE)) {
+		lr_fde_unpack_inline(desc, inline_ranges);
+		ranges = inline_ranges;
+		cnt = 4;
+	} else {
+		u32 idx = LR_OOL_IDX(desc);
+
+		cnt = LR_OOL_COUNT(desc);
+		if (!cnt || idx + cnt > sec->num_ranges)
+			return -EFAULT;
+		ranges = &sec->ranges[idx];
+	}
+
+	for (u32 i = 0; i < cnt; i++) {
+		u32 len = LR_RANGE_LEN(ranges[i]);
+
+		if (insn_off < len) {
+			enum lr_unwind_mode m = LR_RANGE_MODE(ranges[i]);
+
+			if (unlikely(m > LR_UNWIND_SP))
+				return -EFAULT;
+			*mode = m;
+			*sp_off = LR_FDE_SP_OFF_BYTES(desc);
+			return 0;
+		}
+		insn_off -= len;
+	}
+
+	/* Make sure IP is not in an unannotated gap past this FDE */
+	return -EINVAL;
+}
+
+noinstr int lr_unwind_find(unsigned long ip, enum lr_unwind_mode *mode,
+			   unsigned int *sp_off)
+{
+	if (!mode || !sp_off)
+		return -EINVAL;
+
+	if (is_kernel_ip_valid(ip)) {
+		if (!lr_unwind_init)
+			return -EINVAL;
+
+		return __lr_unwind_find(&kernel_lr_sec, ip, mode, sp_off);
+	}
+
+	return -EINVAL;
+}
+NOKPROBE_SYMBOL(lr_unwind_find);
+
+#ifdef CONFIG_ARM64_LR_UNWIND_VALIDATION
+
+static int lr_unwind_validate_section(const struct lr_unwind_section *sec)
+{
+	unsigned long prev_end = 0;
+
+	for (unsigned int i = 0; i < sec->num_fdes; i++) {
+		const struct lr_fde *fde = &sec->fdes[i];
+		unsigned long ip = lr_fde_start_addr(fde);
+		u32 desc = fde->range_desc;
+		u32 total_insns = 0;
+
+		if ((ip & 0x3) || !is_sec_ip_valid(sec, ip)) {
+			dbg_sec(sec, "FDE %u invalid start IP 0x%lx\n", i, ip);
+			return -EINVAL;
+		}
+
+		if (ip < prev_end) {
+			dbg_sec(sec, "FDE %u unsorted or overlapping\n", i);
+			return -EFAULT;
+		}
+
+		if (desc & LR_FDE_INLINE) {
+			total_insns = LR_INLINE_R0(desc) +
+				      LR_INLINE_R1(desc) +
+				      LR_INLINE_R2(desc) +
+				      LR_INLINE_R3(desc);
+		} else {
+			u32 idx = LR_OOL_IDX(desc);
+			u32 cnt = LR_OOL_COUNT(desc);
+
+			if (!cnt || idx + cnt > sec->num_ranges) {
+				dbg_sec(sec, "FDE %u OOL range out of bounds\n", i);
+				return -EFAULT;
+			}
+			for (u32 j = 0; j < cnt; j++) {
+				lr_range_t r = sec->ranges[idx + j];
+				u32 len = LR_RANGE_LEN(r);
+
+				if (!len || LR_RANGE_MODE(r) > LR_UNWIND_SP) {
+					dbg_sec(sec, "FDE %u invalid OOL entry\n", i);
+					return -EINVAL;
+				}
+				total_insns += len;
+			}
+		}
+
+		if (!total_insns) {
+			dbg_sec(sec, "FDE %u has empty range\n", i);
+			return -EINVAL;
+		}
+
+		prev_end = ip + ((unsigned long)total_insns << 2);
+		if (!is_sec_ip_valid(sec, prev_end - 4)) {
+			dbg_sec(sec, "FDE %u end IP out of bounds\n", i);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+#else /* !CONFIG_ARM64_LR_UNWIND_VALIDATION */
+
+static int lr_unwind_validate_section(const struct lr_unwind_section *sec)
+{
+	return 0;
+}
+
+#endif /* CONFIG_ARM64_LR_UNWIND_VALIDATION */
+
+static void dbg_print_section(const struct lr_unwind_section *sec)
+{
+	dbg_sec(sec, "fdes:0x%lx (%u) ranges:0x%lx (%u)\n",
+		(unsigned long)sec->fdes, sec->num_fdes,
+		(unsigned long)sec->ranges, sec->num_ranges);
+}
+
+void __init init_lr_unwind_table(void)
+{
+	size_t fde_bytes = (unsigned long)__stop_lr_unwind_fde -
+			   (unsigned long)__start_lr_unwind_fde;
+	size_t range_bytes = (unsigned long)__stop_lr_unwind_ranges -
+			     (unsigned long)__start_lr_unwind_ranges;
+
+	if (!fde_bytes || (fde_bytes % sizeof(struct lr_fde)) ||
+	    (range_bytes % sizeof(lr_range_t))) {
+		pr_warn("invalid vmlinux .lr_unwind section sizes\n");
+		return;
+	}
+
+	kernel_lr_sec.fdes = (const struct lr_fde *)__start_lr_unwind_fde;
+	kernel_lr_sec.num_fdes = fde_bytes / sizeof(struct lr_fde);
+	kernel_lr_sec.ranges = (const lr_range_t *)__start_lr_unwind_ranges;
+	kernel_lr_sec.num_ranges = range_bytes / sizeof(lr_range_t);
+
+	if (lr_unwind_validate_section(&kernel_lr_sec)) {
+		pr_warn("invalid vmlinux .lr_unwind section\n");
+		return;
+	}
+
+	lr_unwind_init = true;
+	dbg_print_section(&kernel_lr_sec);
+}
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index a707deba6ae70..a19563e382ebd 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -43,6 +43,7 @@
 #include <asm/cpufeature.h>
 #include <asm/cpu_ops.h>
 #include <asm/kasan.h>
+#include <asm/lr_unwind.h>
 #include <asm/numa.h>
 #include <asm/scs.h>
 #include <asm/sections.h>
@@ -373,6 +374,8 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p)
 			"This indicates a broken bootloader or old kernel\n",
 			boot_args[1], boot_args[2], boot_args[3]);
 	}
+
+	init_lr_unwind_table();
 }
 
 static inline bool cpu_can_disable(unsigned int cpu)
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index b8f2963d10030..dc6c4a041f691 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -256,12 +256,14 @@ SECTIONS
 
 	/* code sections that are never executed via the kernel mapping */
 	.rodata.text : {
+		_srodatatext = .;
 		TRAMP_TEXT
 		HIBERNATE_TEXT
 		KEXEC_TEXT
 		IDMAP_TEXT
 		. = ALIGN(PAGE_SIZE);
 	}
+	_erodatatext = .;
 
 	idmap_pg_dir = .;
 	. += PAGE_SIZE;
-- 
2.56.0.679.gc527c65306-goog


  parent reply	other threads:[~2026-11-17  4:35 UTC|newest]

Thread overview: 13+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-17  4:34 [RFC PATCH 00/12] arm64, unwind: add .lr_unwind unwinder for reliable stacktrace Dylan Hatch
2026-11-17  4:34 ` [RFC PATCH 01/12] arm64, unwind: build with .eh_frame and add gen_lr_unwind skeleton Dylan Hatch
2026-11-17  4:34 ` [RFC PATCH 02/12] arm64: factor .eh_frame parser out of patch-scs.c into <asm/eh_frame.h> Dylan Hatch
2026-11-17  4:34 ` [RFC PATCH 03/12] arm64: eh_frame: track CFA and x29/x30 register state in CFI parser Dylan Hatch
2026-11-17  4:34 ` [RFC PATCH 04/12] arm64, unwind: parse .eh_frame into .lr_unwind entries in gen_lr_unwind Dylan Hatch
2026-11-17  4:34 ` [RFC PATCH 05/12] scripts/gen_lr_unwind: emit .lr_unwind_fde and .lr_unwind_ranges Dylan Hatch
2026-11-17  4:34 ` [RFC PATCH 06/12] scripts/gen_lr_unwind: add --dump option to inspect .lr_unwind tables Dylan Hatch
2026-11-17  4:34 ` [RFC PATCH 07/12] scripts/sorttable: sort .lr_unwind_fde by start_offset Dylan Hatch
2026-11-17  4:35 ` Dylan Hatch [this message]
2026-11-17  4:35 ` [RFC PATCH 09/12] arm64/module, unwind: add .lr_unwind support for modules Dylan Hatch
2026-11-17  4:35 ` [RFC PATCH 10/12] arm64: entry: add unwind info for call_on_irq_stack() Dylan Hatch
2026-11-17  4:35 ` [RFC PATCH 11/12] arm64, crypto/lib: annotate leaf functions with CFI info Dylan Hatch
2026-11-17  4:35 ` [RFC PATCH 12/12] unwind: arm64: use .lr_unwind to unwind interrupt frames Dylan Hatch

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261117043504.1021063-9-dylanbhatch@google.com \
    --to=dylanbhatch@google.com \
    --cc=ardb@kernel.org \
    --cc=catalin.marinas@arm.com \
    --cc=davem@davemloft.net \
    --cc=herbert@gondor.apana.org.au \
    --cc=ibhagatgnu@gmail.com \
    --cc=jpoimboe@kernel.org \
    --cc=jremus@linux.ibm.com \
    --cc=justinstitt@google.com \
    --cc=linux-arm-kernel@lists.infradead.org \
    --cc=linux-crypto@vger.kernel.org \
    --cc=linux-kbuild@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=live-patching@vger.kernel.org \
    --cc=llvm@lists.linux.dev \
    --cc=mark.rutland@arm.com \
    --cc=morbo@google.com \
    --cc=nathan@kernel.org \
    --cc=ndesaulniers@google.com \
    --cc=nsc@kernel.org \
    --cc=ptsm@linux.microsoft.com \
    --cc=rostedt@goodmis.org \
    --cc=will@kernel.org \
    --cc=wnliu@google.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox