The Linux Kernel Mailing List
 help / color / mirror / Atom feed
* [PATCH RFC] RAS: hwerr_tracking: move recoverable hardware error tracking out of vmcoreinfo
@ 2026-07-07 14:02 Breno Leitao
  2026-07-07 16:02 ` Borislav Petkov
  2026-07-10 20:35 ` Bjorn Helgaas
  0 siblings, 2 replies; 6+ messages in thread
From: Breno Leitao @ 2026-07-07 14:02 UTC (permalink / raw)
  To: Tony Luck, Borislav Petkov, Thomas Gleixner, Ingo Molnar,
	Dave Hansen, x86, H. Peter Anvin, Rafael J. Wysocki, Hanjun Guo,
	Mauro Carvalho Chehab, Shuai Xue, Len Brown, Mahesh J Salgaonkar,
	Oliver O'Halloran, Bjorn Helgaas, Breno Leitao
  Cc: linux-kernel, linux-edac, linux-acpi, linuxppc-dev, linux-pci,
	kernel-team, Breno Leitao

The recoverable hardware error tracking (hwerr_log_error_type() and the
hwerr_data[] counters) was added under vmcoreinfo, but it uses none of the
vmcoreinfo note machinery: hwerr_data[] is a plain global array that crash
tools read from the vmcore by symbol, like any other global.  Functionally
it is RAS code, fed only by the hardware error paths (x86 MCE, APEI GHES
and PCIe AER).

I wanted to expand it, and Baoquan suggested moving it away from vmcore
info, which makes sense. [1]

Move the implementation to drivers/ras/hwerr_tracking.c and the
declaration (with its no-op stub) to <linux/ras.h>.  Give it a dedicated
CONFIG_RAS_HWERR (bool, under RAS, default y) rather than riding
CONFIG_VMCORE_INFO, so it is a first-class RAS feature that can be turned
off on its own.  The producers now reach hwerr_log_error_type() through
<linux/ras.h>: x86 MCE and APEI GHES already include it, so drop their
<linux/vmcore_info.h> include; PCIe AER switches its include from
<linux/vmcore_info.h> to <linux/ras.h>.

enum hwerr_error_type stays in <uapi/linux/vmcore.h> as it has been part
of the UAPI since the feature shipped; <linux/ras.h> includes it from
there.

hwerr_data[] keeps its name and layout, so existing crash/drgn recipes
keep working.  The config gate moves from CONFIG_VMCORE_INFO to
CONFIG_RAS_HWERR (default y).

Link: https://lore.kernel.org/all/aYvi4Y_HNqk_u1-v@fedora/ [1]
Signed-off-by: Breno Leitao <leitao@debian.org>
---
Once we move it outside of vmcore info, I am planning to add new
features that are in the limbo now, given they don't belong to vmcore
info, such as:

Track fatal hardware errors
	https://lore.kernel.org/all/20260617-hwerr-v1-0-ff131cd6203c@debian.org/

Expose hardware error recovery statistics via sysfs
	https://lore.kernel.org/all/20260202-vmcoreinfo_sysfs-v2-0-8f3b5308b894@debian.org/
---
 MAINTAINERS                    |  7 +++++++
 arch/x86/kernel/cpu/mce/core.c |  1 -
 drivers/acpi/apei/ghes.c       |  1 -
 drivers/pci/pcie/aer.c         |  2 +-
 drivers/ras/Kconfig            | 12 ++++++++++++
 drivers/ras/Makefile           |  1 +
 drivers/ras/hwerr_tracking.c   | 35 +++++++++++++++++++++++++++++++++++
 include/linux/ras.h            |  7 +++++++
 include/linux/vmcore_info.h    |  7 -------
 kernel/vmcore_info.c           | 21 ---------------------
 10 files changed, 63 insertions(+), 31 deletions(-)

diff --git a/MAINTAINERS b/MAINTAINERS
index 1705eb823dd00..356a51032e4b0 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -22539,6 +22539,13 @@ L:	linux-edac@vger.kernel.org
 S:	Maintained
 F:	drivers/ras/amd/fmpm.c
 
+RAS RECOVERABLE HARDWARE ERROR TRACKING
+M:	Breno Leitao <leitao@kernel.org>
+L:	linux-edac@vger.kernel.org
+S:	Maintained
+F:	Documentation/driver-api/hw-recoverable-errors.rst
+F:	drivers/ras/hwerr_tracking.c
+
 RASPBERRY PI PISP BACK END
 M:	Jacopo Mondi <jacopo.mondi@ideasonboard.com>
 R:	Raspberry Pi Kernel Maintenance <kernel-list@raspberrypi.com>
diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c
index 9bba1e2f03af7..58f1d7a601883 100644
--- a/arch/x86/kernel/cpu/mce/core.c
+++ b/arch/x86/kernel/cpu/mce/core.c
@@ -45,7 +45,6 @@
 #include <linux/task_work.h>
 #include <linux/hardirq.h>
 #include <linux/kexec.h>
-#include <linux/vmcore_info.h>
 
 #include <asm/fred.h>
 #include <asm/cpu_device_id.h>
diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c
index 3236a3ce79d6b..4b6666bc19c77 100644
--- a/drivers/acpi/apei/ghes.c
+++ b/drivers/acpi/apei/ghes.c
@@ -45,7 +45,6 @@
 #include <linux/uuid.h>
 #include <linux/ras.h>
 #include <linux/task_work.h>
-#include <linux/vmcore_info.h>
 
 #include <acpi/actbl1.h>
 #include <acpi/ghes.h>
diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c
index c4fd9c0b2a548..00cdca26a5114 100644
--- a/drivers/pci/pcie/aer.c
+++ b/drivers/pci/pcie/aer.c
@@ -30,7 +30,7 @@
 #include <linux/kfifo.h>
 #include <linux/ratelimit.h>
 #include <linux/slab.h>
-#include <linux/vmcore_info.h>
+#include <linux/ras.h>
 #include <acpi/apei.h>
 #include <acpi/ghes.h>
 #include <ras/ras_event.h>
diff --git a/drivers/ras/Kconfig b/drivers/ras/Kconfig
index fc4f4bb94a4c6..241642679c1f1 100644
--- a/drivers/ras/Kconfig
+++ b/drivers/ras/Kconfig
@@ -34,6 +34,18 @@ if RAS
 source "arch/x86/ras/Kconfig"
 source "drivers/ras/amd/atl/Kconfig"
 
+config RAS_HWERR
+	bool "Track hardware errors for crash analysis"
+	default y
+	help
+	  Record the count and timestamp of the most recent recoverable
+	  hardware error for each source (CPU, memory, PCI, CXL, ...).  The
+	  data is written at runtime and read post-mortem from a vmcore by
+	  tools such as crash or drgn, to correlate recoverable errors with a
+	  later panic.
+
+	  If unsure, say Y.
+
 config RAS_FMPM
 	tristate "FRU Memory Poison Manager"
 	default m
diff --git a/drivers/ras/Makefile b/drivers/ras/Makefile
index 11f95d59d3972..4217bee75d910 100644
--- a/drivers/ras/Makefile
+++ b/drivers/ras/Makefile
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0-only
 obj-$(CONFIG_RAS)	+= ras.o
+obj-$(CONFIG_RAS_HWERR)	+= hwerr_tracking.o
 obj-$(CONFIG_DEBUG_FS)	+= debugfs.o
 obj-$(CONFIG_RAS_CEC)	+= cec.o
 
diff --git a/drivers/ras/hwerr_tracking.c b/drivers/ras/hwerr_tracking.c
new file mode 100644
index 0000000000000..847c01fb24d55
--- /dev/null
+++ b/drivers/ras/hwerr_tracking.c
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Track recoverable hardware errors (visible to the OS but not fatal) so that
+ * crash tools like crash/drgn can read the count and timestamp of the last
+ * occurrence from a vmcore and correlate them with a subsequent panic.
+ *
+ * Copyright (c) 2026 Meta Platforms, Inc. and affiliates
+ * Copyright (c) 2026 Breno Leitao <leitao@kernel.org>
+ */
+
+#include <linux/atomic.h>
+#include <linux/export.h>
+#include <linux/ras.h>
+#include <linux/timekeeping.h>
+
+struct hwerr_info {
+	atomic_t count;
+	time64_t timestamp;
+};
+
+/*
+ * Keep hwerr_data[] at global scope so it stays accessible from the vmcore
+ * (via crash/drgn) even when Link Time Optimization (LTO) is enabled.
+ */
+struct hwerr_info hwerr_data[HWERR_RECOV_MAX];
+
+void hwerr_log_error_type(enum hwerr_error_type src)
+{
+	if (src < 0 || src >= HWERR_RECOV_MAX)
+		return;
+
+	atomic_inc(&hwerr_data[src].count);
+	WRITE_ONCE(hwerr_data[src].timestamp, ktime_get_real_seconds());
+}
+EXPORT_SYMBOL_GPL(hwerr_log_error_type);
diff --git a/include/linux/ras.h b/include/linux/ras.h
index 468941bfe855f..1019183c00342 100644
--- a/include/linux/ras.h
+++ b/include/linux/ras.h
@@ -5,6 +5,7 @@
 #include <asm/errno.h>
 #include <linux/uuid.h>
 #include <linux/cper.h>
+#include <uapi/linux/vmcore.h>
 
 #ifdef CONFIG_DEBUG_FS
 int ras_userspace_consumers(void);
@@ -35,6 +36,12 @@ static inline void
 log_arm_hw_error(struct cper_sec_proc_arm *err, const u8 sev) { return; }
 #endif
 
+#ifdef CONFIG_RAS_HWERR
+void hwerr_log_error_type(enum hwerr_error_type src);
+#else
+static inline void hwerr_log_error_type(enum hwerr_error_type src) { }
+#endif
+
 struct atl_err {
 	u64 addr;
 	u64 ipid;
diff --git a/include/linux/vmcore_info.h b/include/linux/vmcore_info.h
index e71518caacdfc..fb6f29b7202e3 100644
--- a/include/linux/vmcore_info.h
+++ b/include/linux/vmcore_info.h
@@ -5,7 +5,6 @@
 #include <linux/linkage.h>
 #include <linux/elfcore.h>
 #include <linux/elf.h>
-#include <uapi/linux/vmcore.h>
 
 #define CRASH_CORE_NOTE_HEAD_BYTES ALIGN(sizeof(struct elf_note), 4)
 #define CRASH_CORE_NOTE_NAME_BYTES ALIGN(sizeof(NN_PRSTATUS), 4)
@@ -79,10 +78,4 @@ Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type,
 			  void *data, size_t data_len);
 void final_note(Elf_Word *buf);
 
-#ifdef CONFIG_VMCORE_INFO
-void hwerr_log_error_type(enum hwerr_error_type src);
-#else
-static inline void hwerr_log_error_type(enum hwerr_error_type src) {};
-#endif
-
 #endif /* LINUX_VMCORE_INFO_H */
diff --git a/kernel/vmcore_info.c b/kernel/vmcore_info.c
index 8614430ca212a..5c288796bdbf5 100644
--- a/kernel/vmcore_info.c
+++ b/kernel/vmcore_info.c
@@ -29,17 +29,6 @@ u32 *vmcoreinfo_note;
 /* trusted vmcoreinfo, e.g. we can make a copy in the crash memory */
 static unsigned char *vmcoreinfo_data_safecopy;
 
-struct hwerr_info {
-	atomic_t count;
-	time64_t timestamp;
-};
-
-/*
- * The hwerr_data[] array is declared with global scope so that it remains
- * accessible to vmcoreinfo even when Link Time Optimization (LTO) is enabled.
- */
-struct hwerr_info hwerr_data[HWERR_RECOV_MAX];
-
 Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type,
 			  void *data, size_t data_len)
 {
@@ -127,16 +116,6 @@ phys_addr_t __weak paddr_vmcoreinfo_note(void)
 }
 EXPORT_SYMBOL(paddr_vmcoreinfo_note);
 
-void hwerr_log_error_type(enum hwerr_error_type src)
-{
-	if (src < 0 || src >= HWERR_RECOV_MAX)
-		return;
-
-	atomic_inc(&hwerr_data[src].count);
-	WRITE_ONCE(hwerr_data[src].timestamp, ktime_get_real_seconds());
-}
-EXPORT_SYMBOL_GPL(hwerr_log_error_type);
-
 static int __init crash_save_vmcoreinfo_init(void)
 {
 	int order;

---
base-commit: 3d5670d672ae08b8c534b7beed6f57c8b44e7b43
change-id: 20260629-hwerr-ras-e26664926c58

Best regards,
--  
Breno Leitao <leitao@debian.org>


^ permalink raw reply related	[flat|nested] 6+ messages in thread

* Re: [PATCH RFC] RAS: hwerr_tracking: move recoverable hardware error tracking out of vmcoreinfo
  2026-07-07 14:02 [PATCH RFC] RAS: hwerr_tracking: move recoverable hardware error tracking out of vmcoreinfo Breno Leitao
@ 2026-07-07 16:02 ` Borislav Petkov
  2026-07-07 16:53   ` Breno Leitao
  2026-07-10 20:35 ` Bjorn Helgaas
  1 sibling, 1 reply; 6+ messages in thread
From: Borislav Petkov @ 2026-07-07 16:02 UTC (permalink / raw)
  To: Breno Leitao
  Cc: Tony Luck, Thomas Gleixner, Ingo Molnar, Dave Hansen, x86,
	H. Peter Anvin, Rafael J. Wysocki, Hanjun Guo,
	Mauro Carvalho Chehab, Shuai Xue, Len Brown, Mahesh J Salgaonkar,
	Oliver O'Halloran, Bjorn Helgaas, Breno Leitao, linux-kernel,
	linux-edac, linux-acpi, linuxppc-dev, linux-pci, kernel-team

On Tue, Jul 07, 2026 at 07:02:34AM -0700, Breno Leitao wrote:
> The recoverable hardware error tracking (hwerr_log_error_type() and the
> hwerr_data[] counters) was added under vmcoreinfo, but it uses none of the
> vmcoreinfo note machinery: hwerr_data[] is a plain global array that crash
> tools read from the vmcore by symbol, like any other global.  Functionally
> it is RAS code, fed only by the hardware error paths (x86 MCE, APEI GHES
> and PCIe AER).
> 
> I wanted to expand it, and Baoquan suggested moving it away from vmcore
> info, which makes sense. [1]
> 
> Move the implementation to drivers/ras/hwerr_tracking.c and the
> declaration (with its no-op stub) to <linux/ras.h>.  Give it a dedicated
> CONFIG_RAS_HWERR (bool, under RAS, default y) rather than riding

Definitely not default y.

> CONFIG_VMCORE_INFO, so it is a first-class RAS feature that can be turned
> off on its own.  The producers now reach hwerr_log_error_type() through
> <linux/ras.h>: x86 MCE and APEI GHES already include it, so drop their
> <linux/vmcore_info.h> include; PCIe AER switches its include from
> <linux/vmcore_info.h> to <linux/ras.h>.
> 
> enum hwerr_error_type stays in <uapi/linux/vmcore.h> as it has been part
> of the UAPI since the feature shipped; <linux/ras.h> includes it from
> there.
> 
> hwerr_data[] keeps its name and layout, so existing crash/drgn recipes
> keep working.  The config gate moves from CONFIG_VMCORE_INFO to
> CONFIG_RAS_HWERR (default y).

Do not explain the WHAT - that's visible from the diff below; explain the WHY.

> Link: https://lore.kernel.org/all/aYvi4Y_HNqk_u1-v@fedora/ [1]
> Signed-off-by: Breno Leitao <leitao@debian.org>
> ---
> Once we move it outside of vmcore info, I am planning to add new
> features that are in the limbo now, given they don't belong to vmcore
> info, such as:
> 
> Track fatal hardware errors
> 	https://lore.kernel.org/all/20260617-hwerr-v1-0-ff131cd6203c@debian.org/
> 
> Expose hardware error recovery statistics via sysfs
> 	https://lore.kernel.org/all/20260202-vmcoreinfo_sysfs-v2-0-8f3b5308b894@debian.org/

We already have rasdaemon and a whole pile of infrastructure around reporting
errors. Why isn't what we have, enough?

The stick-the-error-into-vmcore makes sense as a use case, sure. But this
other information we already have plenty. I think you should use/extend that
instead of adding more.

Thx.

-- 
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette

^ permalink raw reply	[flat|nested] 6+ messages in thread

* Re: [PATCH RFC] RAS: hwerr_tracking: move recoverable hardware error tracking out of vmcoreinfo
  2026-07-07 16:02 ` Borislav Petkov
@ 2026-07-07 16:53   ` Breno Leitao
  2026-07-07 18:35     ` Borislav Petkov
  0 siblings, 1 reply; 6+ messages in thread
From: Breno Leitao @ 2026-07-07 16:53 UTC (permalink / raw)
  To: Borislav Petkov
  Cc: Tony Luck, Thomas Gleixner, Ingo Molnar, Dave Hansen, x86,
	H. Peter Anvin, Rafael J. Wysocki, Hanjun Guo,
	Mauro Carvalho Chehab, Shuai Xue, Len Brown, Mahesh J Salgaonkar,
	Oliver O'Halloran, Bjorn Helgaas, Breno Leitao, linux-kernel,
	linux-edac, linux-acpi, linuxppc-dev, linux-pci, kernel-team

On Tue, Jul 07, 2026 at 09:02:47AM -0700, Borislav Petkov wrote:
> On Tue, Jul 07, 2026 at 07:02:34AM -0700, Breno Leitao wrote:
> > Move the implementation to drivers/ras/hwerr_tracking.c and the
> > declaration (with its no-op stub) to <linux/ras.h>.  Give it a dedicated
> > CONFIG_RAS_HWERR (bool, under RAS, default y) rather than riding
> 
> Definitely not default y.

The current vmcoreinfo implementation defaults to enabled, so I wanted to
preserve that behavior to avoid silently removing symbols that existing
tools may depend on. Would you prefer a different default?

> Do not explain the WHAT - that's visible from the diff below; explain the WHY.

Will do.

> > Link: https://lore.kernel.org/all/aYvi4Y_HNqk_u1-v@fedora/ [1]
> > Signed-off-by: Breno Leitao <leitao@debian.org>
> > ---
> > Once we move it outside of vmcore info, I am planning to add new
> > features that are in the limbo now, given they don't belong to vmcore
> > info, such as:
> > 
> > Track fatal hardware errors
> > 	https://lore.kernel.org/all/20260617-hwerr-v1-0-ff131cd6203c@debian.org/
> > 
> > Expose hardware error recovery statistics via sysfs
> > 	https://lore.kernel.org/all/20260202-vmcoreinfo_sysfs-v2-0-8f3b5308b894@debian.org/
> 
> We already have rasdaemon and a whole pile of infrastructure around reporting
> errors. Why isn't what we have, enough?
> 
> The stick-the-error-into-vmcore makes sense as a use case, sure. But this
> other information we already have plenty. I think you should use/extend that
> instead of adding more.

I agree the sysfs exposure duplicates existing infrastructure, points
taken. However, tracking fatal hardware errors provides value at crash
analysis time—it lets us quickly determine whether a fatal hardware
error occurred during the kernel's lifetime, which is useful for
root-cause attribution.

> The stick-the-error-into-vmcore makes sense as a use case

For this, would you like to keep it in vmcore info (as of today), or
move to RAS subsystem?

Thanks for the review and direction,
--breno

^ permalink raw reply	[flat|nested] 6+ messages in thread

* Re: [PATCH RFC] RAS: hwerr_tracking: move recoverable hardware error tracking out of vmcoreinfo
  2026-07-07 16:53   ` Breno Leitao
@ 2026-07-07 18:35     ` Borislav Petkov
  2026-07-08 10:13       ` Breno Leitao
  0 siblings, 1 reply; 6+ messages in thread
From: Borislav Petkov @ 2026-07-07 18:35 UTC (permalink / raw)
  To: Breno Leitao
  Cc: Tony Luck, Thomas Gleixner, Ingo Molnar, Dave Hansen, x86,
	H. Peter Anvin, Rafael J. Wysocki, Hanjun Guo,
	Mauro Carvalho Chehab, Shuai Xue, Len Brown, Mahesh J Salgaonkar,
	Oliver O'Halloran, Bjorn Helgaas, Breno Leitao, linux-kernel,
	linux-edac, linux-acpi, linuxppc-dev, linux-pci, kernel-team

On Tue, Jul 07, 2026 at 09:53:34AM -0700, Breno Leitao wrote:
> The current vmcoreinfo implementation defaults to enabled, so I wanted to
> preserve that behavior to avoid silently removing symbols that existing
> tools may depend on. Would you prefer a different default?

You do know Linus' stance on enabling things by default which are not
ubiquitous, right?

Logging hw errors in vmcore is really necessary to be enabled everywhere?

> I agree the sysfs exposure duplicates existing infrastructure, points
> taken. However, tracking fatal hardware errors provides value at crash
> analysis time—it lets us quickly determine whether a fatal hardware
> error occurred during the kernel's lifetime, which is useful for
> root-cause attribution.

Yes, that's why you put it in vmcore. You can't read sysfs if you encounter
a fatal hw error.

> For this, would you like to keep it in vmcore info (as of today), or
> move to RAS subsystem?

You mean, would I like to pay attention to more patches than now?

Not really - I can barely manage as it is.

:-)

-- 
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette

^ permalink raw reply	[flat|nested] 6+ messages in thread

* Re: [PATCH RFC] RAS: hwerr_tracking: move recoverable hardware error tracking out of vmcoreinfo
  2026-07-07 18:35     ` Borislav Petkov
@ 2026-07-08 10:13       ` Breno Leitao
  0 siblings, 0 replies; 6+ messages in thread
From: Breno Leitao @ 2026-07-08 10:13 UTC (permalink / raw)
  To: Borislav Petkov
  Cc: Tony Luck, Thomas Gleixner, Ingo Molnar, Dave Hansen, x86,
	H. Peter Anvin, Rafael J. Wysocki, Hanjun Guo,
	Mauro Carvalho Chehab, Shuai Xue, Len Brown, Mahesh J Salgaonkar,
	Oliver O'Halloran, Bjorn Helgaas, Breno Leitao, linux-kernel,
	linux-edac, linux-acpi, linuxppc-dev, linux-pci, kernel-team

On Tue, Jul 07, 2026 at 11:35:17AM -0700, Borislav Petkov wrote:
> On Tue, Jul 07, 2026 at 09:53:34AM -0700, Breno Leitao wrote:
> > The current vmcoreinfo implementation defaults to enabled, so I wanted to
> > preserve that behavior to avoid silently removing symbols that existing
> > tools may depend on. Would you prefer a different default?
> 
> You do know Linus' stance on enabling things by default which are not
> ubiquitous, right?

Oh yea, I am familiar with it. :-)

> Logging hw errors in vmcore is really necessary to be enabled everywhere?

I am happy to make it no, if we transform it to a KCONFIG. Right now it
is not a Kconfig, so, it comes with VMCORE set of exported fields.

> > I agree the sysfs exposure duplicates existing infrastructure, points
> > taken. However, tracking fatal hardware errors provides value at crash
> > analysis time—it lets us quickly determine whether a fatal hardware
> > error occurred during the kernel's lifetime, which is useful for
> > root-cause attribution.
> 
> Yes, that's why you put it in vmcore. You can't read sysfs if you encounter
> a fatal hw error.

Ack. Right now it is not tracking fatal error (just recoverable error),
although I think it is a good idea to also track Fatal error (of course
that not on sysfs).

> > For this, would you like to keep it in vmcore info (as of today), or
> > move to RAS subsystem?
> 
> You mean, would I like to pay attention to more patches than now?
>
> Not really - I can barely manage as it is.

Come one Borislav, we need your insights/review/opinions here as well.
:-)

^ permalink raw reply	[flat|nested] 6+ messages in thread

* Re: [PATCH RFC] RAS: hwerr_tracking: move recoverable hardware error tracking out of vmcoreinfo
  2026-07-07 14:02 [PATCH RFC] RAS: hwerr_tracking: move recoverable hardware error tracking out of vmcoreinfo Breno Leitao
  2026-07-07 16:02 ` Borislav Petkov
@ 2026-07-10 20:35 ` Bjorn Helgaas
  1 sibling, 0 replies; 6+ messages in thread
From: Bjorn Helgaas @ 2026-07-10 20:35 UTC (permalink / raw)
  To: Breno Leitao
  Cc: Tony Luck, Borislav Petkov, Thomas Gleixner, Ingo Molnar,
	Dave Hansen, x86, H. Peter Anvin, Rafael J. Wysocki, Hanjun Guo,
	Mauro Carvalho Chehab, Shuai Xue, Len Brown, Mahesh J Salgaonkar,
	Oliver O'Halloran, Bjorn Helgaas, Breno Leitao, linux-kernel,
	linux-edac, linux-acpi, linuxppc-dev, linux-pci, kernel-team

On Tue, Jul 07, 2026 at 07:02:34AM -0700, Breno Leitao wrote:
> The recoverable hardware error tracking (hwerr_log_error_type() and the
> hwerr_data[] counters) was added under vmcoreinfo, but it uses none of the
> vmcoreinfo note machinery: hwerr_data[] is a plain global array that crash
> tools read from the vmcore by symbol, like any other global.  Functionally
> it is RAS code, fed only by the hardware error paths (x86 MCE, APEI GHES
> and PCIe AER).
> 
> I wanted to expand it, and Baoquan suggested moving it away from vmcore
> info, which makes sense. [1]
> 
> Move the implementation to drivers/ras/hwerr_tracking.c and the
> declaration (with its no-op stub) to <linux/ras.h>.  Give it a dedicated
> CONFIG_RAS_HWERR (bool, under RAS, default y) rather than riding
> CONFIG_VMCORE_INFO, so it is a first-class RAS feature that can be turned
> off on its own.  The producers now reach hwerr_log_error_type() through
> <linux/ras.h>: x86 MCE and APEI GHES already include it, so drop their
> <linux/vmcore_info.h> include; PCIe AER switches its include from
> <linux/vmcore_info.h> to <linux/ras.h>.
> 
> enum hwerr_error_type stays in <uapi/linux/vmcore.h> as it has been part
> of the UAPI since the feature shipped; <linux/ras.h> includes it from
> there.
> 
> hwerr_data[] keeps its name and layout, so existing crash/drgn recipes
> keep working.  The config gate moves from CONFIG_VMCORE_INFO to
> CONFIG_RAS_HWERR (default y).
> 
> Link: https://lore.kernel.org/all/aYvi4Y_HNqk_u1-v@fedora/ [1]
> Signed-off-by: Breno Leitao <leitao@debian.org>
> ---
> Once we move it outside of vmcore info, I am planning to add new
> features that are in the limbo now, given they don't belong to vmcore
> info, such as:
> 
> Track fatal hardware errors
> 	https://lore.kernel.org/all/20260617-hwerr-v1-0-ff131cd6203c@debian.org/
> 
> Expose hardware error recovery statistics via sysfs
> 	https://lore.kernel.org/all/20260202-vmcoreinfo_sysfs-v2-0-8f3b5308b894@debian.org/
> ---
>  MAINTAINERS                    |  7 +++++++
>  arch/x86/kernel/cpu/mce/core.c |  1 -
>  drivers/acpi/apei/ghes.c       |  1 -
>  drivers/pci/pcie/aer.c         |  2 +-

Acked-by: Bjorn Helgaas <bhelgaas@google.com> # drivers/pci

>  drivers/ras/Kconfig            | 12 ++++++++++++
>  drivers/ras/Makefile           |  1 +
>  drivers/ras/hwerr_tracking.c   | 35 +++++++++++++++++++++++++++++++++++
>  include/linux/ras.h            |  7 +++++++
>  include/linux/vmcore_info.h    |  7 -------
>  kernel/vmcore_info.c           | 21 ---------------------
>  10 files changed, 63 insertions(+), 31 deletions(-)
> 
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 1705eb823dd00..356a51032e4b0 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -22539,6 +22539,13 @@ L:	linux-edac@vger.kernel.org
>  S:	Maintained
>  F:	drivers/ras/amd/fmpm.c
>  
> +RAS RECOVERABLE HARDWARE ERROR TRACKING
> +M:	Breno Leitao <leitao@kernel.org>
> +L:	linux-edac@vger.kernel.org
> +S:	Maintained
> +F:	Documentation/driver-api/hw-recoverable-errors.rst
> +F:	drivers/ras/hwerr_tracking.c
> +
>  RASPBERRY PI PISP BACK END
>  M:	Jacopo Mondi <jacopo.mondi@ideasonboard.com>
>  R:	Raspberry Pi Kernel Maintenance <kernel-list@raspberrypi.com>
> diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c
> index 9bba1e2f03af7..58f1d7a601883 100644
> --- a/arch/x86/kernel/cpu/mce/core.c
> +++ b/arch/x86/kernel/cpu/mce/core.c
> @@ -45,7 +45,6 @@
>  #include <linux/task_work.h>
>  #include <linux/hardirq.h>
>  #include <linux/kexec.h>
> -#include <linux/vmcore_info.h>
>  
>  #include <asm/fred.h>
>  #include <asm/cpu_device_id.h>
> diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c
> index 3236a3ce79d6b..4b6666bc19c77 100644
> --- a/drivers/acpi/apei/ghes.c
> +++ b/drivers/acpi/apei/ghes.c
> @@ -45,7 +45,6 @@
>  #include <linux/uuid.h>
>  #include <linux/ras.h>
>  #include <linux/task_work.h>
> -#include <linux/vmcore_info.h>
>  
>  #include <acpi/actbl1.h>
>  #include <acpi/ghes.h>
> diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c
> index c4fd9c0b2a548..00cdca26a5114 100644
> --- a/drivers/pci/pcie/aer.c
> +++ b/drivers/pci/pcie/aer.c
> @@ -30,7 +30,7 @@
>  #include <linux/kfifo.h>
>  #include <linux/ratelimit.h>
>  #include <linux/slab.h>
> -#include <linux/vmcore_info.h>
> +#include <linux/ras.h>
>  #include <acpi/apei.h>
>  #include <acpi/ghes.h>
>  #include <ras/ras_event.h>
> diff --git a/drivers/ras/Kconfig b/drivers/ras/Kconfig
> index fc4f4bb94a4c6..241642679c1f1 100644
> --- a/drivers/ras/Kconfig
> +++ b/drivers/ras/Kconfig
> @@ -34,6 +34,18 @@ if RAS
>  source "arch/x86/ras/Kconfig"
>  source "drivers/ras/amd/atl/Kconfig"
>  
> +config RAS_HWERR
> +	bool "Track hardware errors for crash analysis"
> +	default y
> +	help
> +	  Record the count and timestamp of the most recent recoverable
> +	  hardware error for each source (CPU, memory, PCI, CXL, ...).  The
> +	  data is written at runtime and read post-mortem from a vmcore by
> +	  tools such as crash or drgn, to correlate recoverable errors with a
> +	  later panic.
> +
> +	  If unsure, say Y.
> +
>  config RAS_FMPM
>  	tristate "FRU Memory Poison Manager"
>  	default m
> diff --git a/drivers/ras/Makefile b/drivers/ras/Makefile
> index 11f95d59d3972..4217bee75d910 100644
> --- a/drivers/ras/Makefile
> +++ b/drivers/ras/Makefile
> @@ -1,5 +1,6 @@
>  # SPDX-License-Identifier: GPL-2.0-only
>  obj-$(CONFIG_RAS)	+= ras.o
> +obj-$(CONFIG_RAS_HWERR)	+= hwerr_tracking.o
>  obj-$(CONFIG_DEBUG_FS)	+= debugfs.o
>  obj-$(CONFIG_RAS_CEC)	+= cec.o
>  
> diff --git a/drivers/ras/hwerr_tracking.c b/drivers/ras/hwerr_tracking.c
> new file mode 100644
> index 0000000000000..847c01fb24d55
> --- /dev/null
> +++ b/drivers/ras/hwerr_tracking.c
> @@ -0,0 +1,35 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/*
> + * Track recoverable hardware errors (visible to the OS but not fatal) so that
> + * crash tools like crash/drgn can read the count and timestamp of the last
> + * occurrence from a vmcore and correlate them with a subsequent panic.
> + *
> + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates
> + * Copyright (c) 2026 Breno Leitao <leitao@kernel.org>
> + */
> +
> +#include <linux/atomic.h>
> +#include <linux/export.h>
> +#include <linux/ras.h>
> +#include <linux/timekeeping.h>
> +
> +struct hwerr_info {
> +	atomic_t count;
> +	time64_t timestamp;
> +};
> +
> +/*
> + * Keep hwerr_data[] at global scope so it stays accessible from the vmcore
> + * (via crash/drgn) even when Link Time Optimization (LTO) is enabled.
> + */
> +struct hwerr_info hwerr_data[HWERR_RECOV_MAX];
> +
> +void hwerr_log_error_type(enum hwerr_error_type src)
> +{
> +	if (src < 0 || src >= HWERR_RECOV_MAX)
> +		return;
> +
> +	atomic_inc(&hwerr_data[src].count);
> +	WRITE_ONCE(hwerr_data[src].timestamp, ktime_get_real_seconds());
> +}
> +EXPORT_SYMBOL_GPL(hwerr_log_error_type);
> diff --git a/include/linux/ras.h b/include/linux/ras.h
> index 468941bfe855f..1019183c00342 100644
> --- a/include/linux/ras.h
> +++ b/include/linux/ras.h
> @@ -5,6 +5,7 @@
>  #include <asm/errno.h>
>  #include <linux/uuid.h>
>  #include <linux/cper.h>
> +#include <uapi/linux/vmcore.h>
>  
>  #ifdef CONFIG_DEBUG_FS
>  int ras_userspace_consumers(void);
> @@ -35,6 +36,12 @@ static inline void
>  log_arm_hw_error(struct cper_sec_proc_arm *err, const u8 sev) { return; }
>  #endif
>  
> +#ifdef CONFIG_RAS_HWERR
> +void hwerr_log_error_type(enum hwerr_error_type src);
> +#else
> +static inline void hwerr_log_error_type(enum hwerr_error_type src) { }
> +#endif
> +
>  struct atl_err {
>  	u64 addr;
>  	u64 ipid;
> diff --git a/include/linux/vmcore_info.h b/include/linux/vmcore_info.h
> index e71518caacdfc..fb6f29b7202e3 100644
> --- a/include/linux/vmcore_info.h
> +++ b/include/linux/vmcore_info.h
> @@ -5,7 +5,6 @@
>  #include <linux/linkage.h>
>  #include <linux/elfcore.h>
>  #include <linux/elf.h>
> -#include <uapi/linux/vmcore.h>
>  
>  #define CRASH_CORE_NOTE_HEAD_BYTES ALIGN(sizeof(struct elf_note), 4)
>  #define CRASH_CORE_NOTE_NAME_BYTES ALIGN(sizeof(NN_PRSTATUS), 4)
> @@ -79,10 +78,4 @@ Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type,
>  			  void *data, size_t data_len);
>  void final_note(Elf_Word *buf);
>  
> -#ifdef CONFIG_VMCORE_INFO
> -void hwerr_log_error_type(enum hwerr_error_type src);
> -#else
> -static inline void hwerr_log_error_type(enum hwerr_error_type src) {};
> -#endif
> -
>  #endif /* LINUX_VMCORE_INFO_H */
> diff --git a/kernel/vmcore_info.c b/kernel/vmcore_info.c
> index 8614430ca212a..5c288796bdbf5 100644
> --- a/kernel/vmcore_info.c
> +++ b/kernel/vmcore_info.c
> @@ -29,17 +29,6 @@ u32 *vmcoreinfo_note;
>  /* trusted vmcoreinfo, e.g. we can make a copy in the crash memory */
>  static unsigned char *vmcoreinfo_data_safecopy;
>  
> -struct hwerr_info {
> -	atomic_t count;
> -	time64_t timestamp;
> -};
> -
> -/*
> - * The hwerr_data[] array is declared with global scope so that it remains
> - * accessible to vmcoreinfo even when Link Time Optimization (LTO) is enabled.
> - */
> -struct hwerr_info hwerr_data[HWERR_RECOV_MAX];
> -
>  Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type,
>  			  void *data, size_t data_len)
>  {
> @@ -127,16 +116,6 @@ phys_addr_t __weak paddr_vmcoreinfo_note(void)
>  }
>  EXPORT_SYMBOL(paddr_vmcoreinfo_note);
>  
> -void hwerr_log_error_type(enum hwerr_error_type src)
> -{
> -	if (src < 0 || src >= HWERR_RECOV_MAX)
> -		return;
> -
> -	atomic_inc(&hwerr_data[src].count);
> -	WRITE_ONCE(hwerr_data[src].timestamp, ktime_get_real_seconds());
> -}
> -EXPORT_SYMBOL_GPL(hwerr_log_error_type);
> -
>  static int __init crash_save_vmcoreinfo_init(void)
>  {
>  	int order;
> 
> ---
> base-commit: 3d5670d672ae08b8c534b7beed6f57c8b44e7b43
> change-id: 20260629-hwerr-ras-e26664926c58
> 
> Best regards,
> --  
> Breno Leitao <leitao@debian.org>
> 

^ permalink raw reply	[flat|nested] 6+ messages in thread

end of thread, other threads:[~2026-07-10 20:35 UTC | newest]

Thread overview: 6+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-07-07 14:02 [PATCH RFC] RAS: hwerr_tracking: move recoverable hardware error tracking out of vmcoreinfo Breno Leitao
2026-07-07 16:02 ` Borislav Petkov
2026-07-07 16:53   ` Breno Leitao
2026-07-07 18:35     ` Borislav Petkov
2026-07-08 10:13       ` Breno Leitao
2026-07-10 20:35 ` Bjorn Helgaas

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox