[PATCH 2/2] x86/mce: Add mce=panic_on_ce_count to panic on a corrected error flood

[Breno Leitao <leitao@debian.org>]

A machine check bank producing corrected errors faster than the kernel can
drain them is not a machine anyone wants to keep in service, but nothing
takes it out. mce_track_storm() throttles CMCI for the bank and the
machine stays up.

Count corrected errors per bank and add mce=panic_on_ce_count=<count>
and panic the host if we have more events than set.

Off by default, for obvious reasons.

Signed-off-by: Breno Leitao <leitao@debian.org>
---
 Documentation/ABI/testing/sysfs-mce             | 12 ++++++++++++
 Documentation/admin-guide/kernel-parameters.txt | 10 ++++++++++
 arch/x86/kernel/cpu/mce/core.c                  | 14 ++++++++++++--
 arch/x86/kernel/cpu/mce/internal.h              |  6 ++++++
 arch/x86/kernel/cpu/mce/threshold.c             | 23 +++++++++++++++++++++++
 5 files changed, 63 insertions(+), 2 deletions(-)

diff --git a/Documentation/ABI/testing/sysfs-mce b/Documentation/ABI/testing/sysfs-mce
index 83172f50e27c6..02009b5d581a4 100644
--- a/Documentation/ABI/testing/sysfs-mce
+++ b/Documentation/ABI/testing/sysfs-mce
@@ -95,3 +95,15 @@ Contact:	Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
 Date:		Jun 2009
 Description:
 		Disables the CMCI feature.
+
+What:		/sys/devices/system/machinecheck/machinecheckX/panic_on_ce_count
+Contact:	Breno Leitao <leitao@debian.org>
+Date:		Aug 2026
+Description:
+		Panic once a machine check bank has logged this many corrected
+		errors. 0, the default, disables it.
+
+		The setting is global rather than per-CPU, and it is compared
+		against a running total kept per CPU and per bank. Lowering it
+		below a total a bank has already reached takes the machine
+		down on that bank's next corrected error.
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index d55524e3b7246..d5005bfefb7be 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -3924,6 +3924,16 @@ Kernel parameters
 		print_all
 			print all machine check logs to the console.
 
+		panic_on_ce_count=<count>
+			panic once a machine check bank has logged this many
+			corrected errors. 0, the default, disables it.
+
+			This is a running total per CPU and per bank for the
+			life of the boot, not a rate, so a machine that logs
+			a slow trickle for long enough will reach any value
+			eventually. Pick one a failing part reaches in
+			minutes and a healthy one does not reach at all.
+
 		monarchtimeout (number)
 			sets the time in us to wait for other CPUs on machine
 			checks. 0 to disable.
diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c
index ab469605fc893..43dcdef1a7bbe 100644
--- a/arch/x86/kernel/cpu/mce/core.c
+++ b/arch/x86/kernel/cpu/mce/core.c
@@ -261,7 +261,7 @@ static const char *mce_dump_aux_info(struct mce *m)
 	return NULL;
 }
 
-static noinstr void mce_panic(const char *msg, struct mce_hw_err *final, char *exp)
+noinstr void mce_panic(const char *msg, struct mce_hw_err *final, char *exp)
 {
 	struct llist_node *pending;
 	struct mce_evt_llist *l;
@@ -813,6 +813,10 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
 		barrier();
 		m->status = mce_rdmsrq(mca_msr_reg(i, MCA_STATUS));
 
+		/* The boot time poll replays errors from before this boot. */
+		if (!(flags & MCP_QUEUE_LOG))
+			mce_track_ce_count(m);
+
 		/*
 		 * Update storm tracking here, before checking for the
 		 * MCI_STATUS_VAL bit. Valid corrected errors count
@@ -2326,6 +2330,7 @@ void mce_disable_bank(int bank)
  * mce=nobootlog Don't log MCEs from before booting.
  * mce=bios_cmci_threshold Don't program the CMCI threshold
  * mce=recovery force enable copy_mc_fragile()
+ * mce=panic_on_ce_count=<n> Panic after n corrected errors on one bank
  */
 static int __init mcheck_enable(char *str)
 {
@@ -2349,7 +2354,10 @@ static int __init mcheck_enable(char *str)
 		cfg->print_all = true;
 	else if (!strcmp(str, "ignore_ce"))
 		cfg->ignore_ce = true;
-	else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
+	else if (str_has_prefix(str, "panic_on_ce_count=")) {
+		str += strlen("panic_on_ce_count=");
+		get_option(&str, &cfg->panic_on_ce_count);
+	} else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
 		cfg->bootlog = (str[0] == 'b');
 	else if (!strcmp(str, "bios_cmci_threshold"))
 		cfg->bios_cmci_threshold = 1;
@@ -2617,6 +2625,7 @@ static ssize_t store_int_with_restart(struct device *s,
 }
 
 static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout);
+static DEVICE_INT_ATTR(panic_on_ce_count, 0644, mca_cfg.panic_on_ce_count);
 static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce);
 static DEVICE_BOOL_ATTR(print_all, 0644, mca_cfg.print_all);
 
@@ -2641,6 +2650,7 @@ static struct device_attribute *mce_device_attrs[] = {
 	&dev_attr_trigger,
 #endif
 	&dev_attr_monarch_timeout.attr,
+	&dev_attr_panic_on_ce_count.attr,
 	&dev_attr_dont_log_ce.attr,
 	&dev_attr_print_all.attr,
 	&dev_attr_ignore_ce.attr,
diff --git a/arch/x86/kernel/cpu/mce/internal.h b/arch/x86/kernel/cpu/mce/internal.h
index a31cf984619ca..7ebf87f6ca346 100644
--- a/arch/x86/kernel/cpu/mce/internal.h
+++ b/arch/x86/kernel/cpu/mce/internal.h
@@ -37,6 +37,7 @@ struct llist_node *mce_gen_pool_prepare_records(void);
 
 int mce_severity(struct mce *a, struct pt_regs *regs, char **msg, bool is_excp);
 struct dentry *mce_get_debugfs_dir(void);
+noinstr void mce_panic(const char *msg, struct mce_hw_err *final, char *exp);
 
 extern mce_banks_t mce_banks_ce_disabled;
 
@@ -64,6 +65,7 @@ void mce_timer_kick(bool storm);
 void cmci_storm_begin(unsigned int bank);
 void cmci_storm_end(unsigned int bank);
 void mce_track_storm(struct mce *mce);
+void mce_track_ce_count(struct mce *mce);
 void mce_inherit_storm(unsigned int bank);
 bool mce_get_storm_mode(void);
 void mce_set_storm_mode(bool storm);
@@ -72,6 +74,7 @@ u32  mce_get_apei_thr_limit(void);
 static inline void cmci_storm_begin(unsigned int bank) {}
 static inline void cmci_storm_end(unsigned int bank) {}
 static inline void mce_track_storm(struct mce *mce) {}
+static inline void mce_track_ce_count(struct mce *mce) {}
 static inline void mce_inherit_storm(unsigned int bank) {}
 static inline bool mce_get_storm_mode(void) { return false; }
 static inline void mce_set_storm_mode(bool storm) {}
@@ -83,12 +86,14 @@ static inline u32  mce_get_apei_thr_limit(void) { return 0; }
  *			represents an error seen.
  *
  * timestamp:		Last time (in jiffies) that the bank was polled.
+ * ce_count:		Corrected errors logged since boot.
  * in_storm_mode:	Is this bank in storm mode?
  * poll_only:		Bank does not support CMCI, skip storm tracking.
  */
 struct storm_bank {
 	u64 history;
 	u64 timestamp;
+	u64 ce_count;
 	bool in_storm_mode;
 	bool poll_only;
 };
@@ -183,6 +188,7 @@ struct mca_config {
 	bool ignore_ce;
 	bool print_all;
 
+	int panic_on_ce_count;
 	int monarch_timeout;
 	int panic_timeout;
 	u32 rip_msr;
diff --git a/arch/x86/kernel/cpu/mce/threshold.c b/arch/x86/kernel/cpu/mce/threshold.c
index 8591715ce2430..208dd6c8a5b18 100644
--- a/arch/x86/kernel/cpu/mce/threshold.c
+++ b/arch/x86/kernel/cpu/mce/threshold.c
@@ -119,6 +119,29 @@ void cmci_storm_end(unsigned int bank)
 static DEFINE_RATELIMIT_STATE(storm_rs, DEFAULT_RATELIMIT_INTERVAL,
 			      DEFAULT_RATELIMIT_BURST);
 
+/*
+ * A bank that keeps reporting corrected errors is repairing them faster than
+ * anything acts on it. Count them and let the admin put a ceiling on it.
+ */
+void mce_track_ce_count(struct mce *mce)
+{
+	struct storm_bank *bank = &this_cpu_ptr(&storm_desc)->banks[mce->bank];
+	int limit = READ_ONCE(mca_cfg.panic_on_ce_count);
+
+	if (limit <= 0)
+		return;
+
+	if (!(mce->status & MCI_STATUS_VAL) || !mce_is_correctable(mce))
+		return;
+
+	if (++bank->ce_count < (u64)limit)
+		return;
+
+	printk_deferred(KERN_EMERG "CPU%d BANK%d logged %llu corrected errors\n",
+			smp_processor_id(), mce->bank, bank->ce_count);
+	mce_panic("Too many corrected errors", NULL, NULL);
+}
+
 void mce_track_storm(struct mce *mce)
 {
 	struct mca_storm_desc *storm = this_cpu_ptr(&storm_desc);

-- 
2.53.0-Meta