[PATCH v4 05/11] x86/irq: Process nmi sources in NMI handler

[Jacob Pan <jacob.jun.pan@linux.intel.com>]

When NMI-source reporting is enabled, the vector 2 NMI handler can
prioritize the handling of explicitly reported sources. If the source
is unknown, it will continue with the existing processing flow, meaning
all NMI handlers will be invoked.

Signed-off-by: Jacob Pan <jacob.jun.pan@linux.intel.com>

---
v4:
   - Coding style changes (Li Xin)
   - Renamed handled_mask to partial_bitmap (Nikolay)
v3:
   - Use a static flag to disable NMIs in case of HW failure
   - Optimize the case when unknown NMIs are mixed with known NMIs(HPA)
v2:
   - Disable NMI source reporting once garbage data is given in FRED
return stack. (HPA)

process nmi

Signed-off-by: Jacob Pan <jacob.jun.pan@linux.intel.com>
---
 arch/x86/kernel/nmi.c | 83 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 82 insertions(+), 1 deletion(-)

diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index b96667eed106..edb169289a1d 100644
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -149,12 +149,89 @@ static inline int do_handle_nmi(struct nmiaction *a, struct pt_regs *regs, unsig
 	return thishandled;
 }
 
+static int nmi_handle_src(unsigned int type, struct pt_regs *regs, unsigned long *partial_bitmap)
+{
+	static bool nmi_source_disabled;
+	bool has_unknown_src = false;
+	unsigned long source_bitmap;
+	struct nmiaction *a;
+	int handled = 0;
+	int vec;
+
+	if (!cpu_feature_enabled(X86_FEATURE_NMI_SOURCE) || type != NMI_LOCAL || nmi_source_disabled)
+		return 0;
+
+	source_bitmap = fred_event_data(regs);
+	if (unlikely(!source_bitmap)) {
+		pr_warn("Buggy hardware! Disable NMI-source handling.\n");
+		nmi_source_disabled = true;
+		return 0;
+	}
+
+	/*
+	 * There is no guarantee that a valid NMI-source vector is always
+	 * delivered, even when the originator specified one. It is software's
+	 * responsibility to check all available NMI sources when bit 0 is set
+	 * in the NMI-source reporting bitmap. I.e. we have to call every
+	 * handler as if there is no NMI-source reporting feature enabled.
+	 *
+	 * In this case, handlers for the known NMI sources will be called
+	 * first, followed by the remaining handlers, which are called
+	 * during the subsequent polling code.
+	 *
+	 * Conversely, if non-zero vectors appear in the source bitmap, we
+	 * can precisely identify the sources. Therefore, we only invoke the
+	 * handlers for which the corresponding bits are set.
+	 */
+	if (unlikely(source_bitmap & BIT(NMI_SOURCE_VEC_UNKNOWN))) {
+		pr_warn_ratelimited("NMI received with unknown sources\n");
+		has_unknown_src = true;
+	}
+
+	rcu_read_lock();
+
+	/* Bit 0 is for unknown NMI sources, skip it. */
+	vec = 1;
+	for_each_set_bit_from(vec, &source_bitmap, NR_NMI_SOURCE_VECTORS) {
+		a = rcu_dereference(nmiaction_src_table[vec]);
+		if (!a) {
+			pr_warn_ratelimited("NMI-source vector %d has no handler!", vec);
+			continue;
+		}
+
+		handled += do_handle_nmi(a, regs, type);
+
+		/*
+		 * Needs polling if the unknown source bit is set.
+		 * partial_bitmap is used to tell the polling code which
+		 * NMIs have already been handled based on explicit source
+		 * thus can be skipped.
+		 */
+		if (has_unknown_src)
+			*partial_bitmap |= BIT(vec);
+	}
+
+	rcu_read_unlock();
+
+	return handled;
+}
+
 static int nmi_handle(unsigned int type, struct pt_regs *regs)
 {
 	struct nmi_desc *desc = nmi_to_desc(type);
+	unsigned long partial_bitmap = 0;
 	struct nmiaction *a;
 	int handled=0;
 
+	/*
+	 * Check if the NMI source handling is complete, otherwise polling is
+	 * still required. partial_bitmap is non-zero if NMI source handling is
+	 * partial due to unknown NMI sources.
+	 */
+	handled = nmi_handle_src(type, regs, &partial_bitmap);
+	if (handled && !partial_bitmap)
+		return handled;
+
 	rcu_read_lock();
 
 	/*
@@ -163,8 +240,12 @@ static int nmi_handle(unsigned int type, struct pt_regs *regs)
 	 * can be latched at any given time.  Walk the whole list
 	 * to handle those situations.
 	 */
-	list_for_each_entry_rcu(a, &desc->head, list)
+	list_for_each_entry_rcu(a, &desc->head, list) {
+		/* Skip NMIs handled earlier with source info */
+		if (BIT(a->source_vec) & partial_bitmap)
+			continue;
 		handled += do_handle_nmi(a, regs, type);
+	}
 
 	rcu_read_unlock();
 
-- 
2.25.1