[RFC PATCH v2 1/5] panic, sys_info: Fix duplicate and dropped all-CPU backtraces in panic

[Aaron Tomlin <atomlin@atomlin.com>]

When sys_info() was factored out of kernel/panic.c by
commit b76e89e50fc3 ("panic: generalize panic_print's function to show
sys info"), __sys_info() gained an unconditional call to
trigger_all_cpu_backtrace() when SYS_INFO_ALL_BT is set, while
panic_other_cpus_shutdown() retained its own call to
panic_trigger_all_cpu_backtrace().

This causes two problems during a kernel panic:

    1.  vpanic() invokes panic_trigger_all_cpu_backtrace() inside
        panic_other_cpus_shutdown() before stopping secondary CPUs, and
        then invokes trigger_all_cpu_backtrace() a second time inside
        sys_info(panic_print) after secondary CPUs have already been
        shut down, stalling for up to 10 seconds waiting for offline
        CPUs to respond to NMIs. Furthermore, if SYS_INFO_ALL_BT is
        configured via the kernel.kernel_sys_info sysctl rather than
        panic_print, panic_other_cpus_shutdown() misses it prior to CPU
        shutdown.

    2.  If sys_info(SYS_INFO_ALL_BT) is invoked on the panicking CPU
        prior to panic_other_cpus_shutdown(), calling
        trigger_all_cpu_backtrace() directly bypasses
        panic_trigger_all_cpu_backtrace() and leaves
        panic_triggering_all_cpu_backtrace unset. Because panic_cpu is
        already active, vprintk_emit() silently drops all NMI backtrace
        messages originating from non-panic CPUs.

Export panic_trigger_all_cpu_backtrace() with a panic_on_this_cpu() and
one-shot guard so all-CPU backtraces are only emitted once before
secondary CPUs are stopped, consult sys_info_get_mask(panic_print) in
panic_other_cpus_shutdown(), record prior Oops stack dumps in
panic_try_start(), and route SYS_INFO_ALL_BT in __sys_info()
through panic_trigger_all_cpu_backtrace() on the panicking CPU.

Fixes: b76e89e50fc3 ("panic: generalize panic_print's function to show sys info")
Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
---
 include/linux/panic.h    |  1 +
 include/linux/sys_info.h |  1 +
 kernel/panic.c           | 31 +++++++++++++++++++++++++++----
 lib/sys_info.c           | 17 ++++++++++++++---
 4 files changed, 43 insertions(+), 7 deletions(-)

diff --git a/include/linux/panic.h b/include/linux/panic.h
index 17e61b61c45f..3b56f873f1e2 100644
--- a/include/linux/panic.h
+++ b/include/linux/panic.h
@@ -57,6 +57,7 @@ void panic_reset(void);
 bool panic_in_progress(void);
 bool panic_on_this_cpu(void);
 bool panic_on_other_cpu(void);
+void panic_trigger_all_cpu_backtrace(void);
 
 /*
  * Only to be used by arch init code. If the user over-wrote the default
diff --git a/include/linux/sys_info.h b/include/linux/sys_info.h
index a5bc3ea3d44b..e7699b82aa49 100644
--- a/include/linux/sys_info.h
+++ b/include/linux/sys_info.h
@@ -18,6 +18,7 @@
 #define SYS_INFO_BLOCKED_TASKS		0x00000080
 
 void sys_info(unsigned long si_mask);
+unsigned long sys_info_get_mask(unsigned long si_mask);
 unsigned long sys_info_parse_param(char *str);
 
 #ifdef CONFIG_SYSCTL
diff --git a/kernel/panic.c b/kernel/panic.c
index 574ee5a12f92..4cb26914b2af 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -70,6 +70,7 @@ static bool panic_console_replay;
 
 bool panic_triggering_all_cpu_backtrace;
 static bool panic_this_cpu_backtrace_printed;
+static bool panic_all_cpu_backtrace_printed;
 
 int panic_timeout = CONFIG_PANIC_TIMEOUT;
 EXPORT_SYMBOL_GPL(panic_timeout);
@@ -451,13 +452,21 @@ bool panic_try_start(void)
 	old_cpu = PANIC_CPU_INVALID;
 	this_cpu = raw_smp_processor_id();
 
-	return atomic_try_cmpxchg(&panic_cpu, &old_cpu, this_cpu);
+	if (!atomic_try_cmpxchg(&panic_cpu, &old_cpu, this_cpu))
+		return false;
+
+	if (atomic_read(&panic_redirect_cpu) == PANIC_CPU_INVALID &&
+	    (test_taint(TAINT_DIE) || oops_in_progress))
+		panic_this_cpu_backtrace_printed = true;
+
+	return true;
 }
 EXPORT_SYMBOL(panic_try_start);
 
 void panic_reset(void)
 {
-	atomic_set(&panic_cpu, PANIC_CPU_INVALID);
+	panic_this_cpu_backtrace_printed = false;
+	atomic_set_release(&panic_cpu, PANIC_CPU_INVALID);
 }
 EXPORT_SYMBOL(panic_reset);
 
@@ -538,8 +547,21 @@ void check_panic_on_warn(const char *origin)
 		      origin, limit);
 }
 
-static void panic_trigger_all_cpu_backtrace(void)
+/**
+ * panic_trigger_all_cpu_backtrace - trigger NMI backtraces on all CPUs once
+ *
+ * Temporarily permit non-panic CPUs to emit messages into the printk ring
+ * buffer while capturing all-CPU backtraces during a panic. Subsequent calls
+ * are ignored so backtraces are not re-triggered after secondary CPUs have
+ * been shut down.
+ */
+void panic_trigger_all_cpu_backtrace(void)
 {
+	if (!panic_on_this_cpu() || panic_all_cpu_backtrace_printed)
+		return;
+
+	panic_all_cpu_backtrace_printed = true;
+
 	/* Temporary allow non-panic CPUs to write their backtraces. */
 	panic_triggering_all_cpu_backtrace = true;
 
@@ -558,8 +580,9 @@ static void panic_trigger_all_cpu_backtrace(void)
  */
 static void panic_other_cpus_shutdown(bool crash_kexec)
 {
-	if (panic_print & SYS_INFO_ALL_BT)
+	if (sys_info_get_mask(panic_print) & SYS_INFO_ALL_BT)
 		panic_trigger_all_cpu_backtrace();
+	panic_all_cpu_backtrace_printed = true;
 
 	/*
 	 * Note that smp_send_stop() is the usual SMP shutdown function,
diff --git a/lib/sys_info.c b/lib/sys_info.c
index f32a06ec9ed4..496893eb776b 100644
--- a/lib/sys_info.c
+++ b/lib/sys_info.c
@@ -7,6 +7,7 @@
 #include <linux/kernel.h>
 #include <linux/ftrace.h>
 #include <linux/nmi.h>
+#include <linux/panic.h>
 #include <linux/sched/debug.h>
 #include <linux/string.h>
 #include <linux/sysctl.h>
@@ -153,14 +154,24 @@ static void __sys_info(unsigned long si_mask)
 	if (si_mask & SYS_INFO_FTRACE)
 		ftrace_dump(DUMP_ALL);
 
-	if (si_mask & SYS_INFO_ALL_BT)
-		trigger_all_cpu_backtrace();
+	if (si_mask & SYS_INFO_ALL_BT) {
+		if (panic_on_this_cpu())
+			panic_trigger_all_cpu_backtrace();
+		else if (!panic_in_progress())
+			trigger_all_cpu_backtrace();
+	}
 
 	if (si_mask & SYS_INFO_BLOCKED_TASKS)
 		show_state_filter(TASK_UNINTERRUPTIBLE);
 }
 
+unsigned long sys_info_get_mask(unsigned long si_mask)
+{
+	return (si_mask & ~SYS_INFO_PANIC_CONSOLE_REPLAY) ? :
+		READ_ONCE(kernel_si_mask);
+}
+
 void sys_info(unsigned long si_mask)
 {
-	__sys_info(si_mask ? : kernel_si_mask);
+	__sys_info(sys_info_get_mask(si_mask));
 }
-- 
2.55.0