[PATCH -v3] x86: Disable x2apic if nox2apic is specified

[Yinghai Lu <yinghai@kernel.org>]

For
1. x2apic preenabled system
2. first kernel have x2apic enabled, and try to kexec second kernel with "nox2apic"

Will put back cpu with apic id < 255 into xapic mode, instead of panic.

-v2: use x2apic_disabled instead of nox2apic, Suggested by Thomas
     update x2apic_supported with x2apic_disabled, Suggested by Thomas

-v3: add checking for boot cpu apic id > 255. in that case will just panic
     --- pointed out by Suresh.

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 arch/x86/include/asm/apic.h      |    6 +++-
 arch/x86/include/asm/apicdef.h   |    1 
 arch/x86/include/asm/processor.h |    1 
 arch/x86/kernel/acpi/boot.c      |   10 ++++++-
 arch/x86/kernel/apic/apic.c      |   51 +++++++++++++++++++++++++++++++++------
 arch/x86/kernel/cpu/topology.c   |   21 ++++++++++++++++
 arch/x86/mm/srat_64.c            |    7 ++++-
 7 files changed, 85 insertions(+), 12 deletions(-)

Index: linux-2.6/arch/x86/include/asm/apic.h
===================================================================
--- linux-2.6.orig/arch/x86/include/asm/apic.h
+++ linux-2.6/arch/x86/include/asm/apic.h
@@ -176,6 +176,7 @@ static inline u64 native_x2apic_icr_read
 }
 
 extern int x2apic_phys;
+extern int x2apic_disabled;
 extern void check_x2apic(void);
 extern void enable_x2apic(void);
 extern void x2apic_icr_write(u32 low, u32 id);
@@ -183,7 +184,7 @@ static inline int x2apic_enabled(void)
 {
 	int msr, msr2;
 
-	if (!cpu_has_x2apic)
+	if (!cpu_has_x2apic || x2apic_disabled)
 		return 0;
 
 	rdmsr(MSR_IA32_APICBASE, msr, msr2);
@@ -192,7 +193,7 @@ static inline int x2apic_enabled(void)
 	return 0;
 }
 
-#define x2apic_supported()	(cpu_has_x2apic)
+#define x2apic_supported()	(cpu_has_x2apic && !x2apic_disabled)
 static inline void x2apic_force_phys(void)
 {
 	x2apic_phys = 1;
@@ -214,6 +215,7 @@ static inline void x2apic_force_phys(voi
 
 #define	x2apic_preenabled 0
 #define	x2apic_supported()	0
+#define	x2apic_disabled	1
 #endif
 
 extern void enable_IR_x2apic(void);
Index: linux-2.6/arch/x86/kernel/acpi/boot.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/acpi/boot.c
+++ linux-2.6/arch/x86/kernel/acpi/boot.c
@@ -213,6 +213,8 @@ static int __init
 acpi_parse_x2apic(struct acpi_subtable_header *header, const unsigned long end)
 {
 	struct acpi_madt_local_x2apic *processor = NULL;
+	int apic_id;
+	u8 enabled;
 
 	processor = (struct acpi_madt_local_x2apic *)header;
 
@@ -221,6 +223,8 @@ acpi_parse_x2apic(struct acpi_subtable_h
 
 	acpi_table_print_madt_entry(header);
 
+	apic_id = processor->local_apic_id;
+	enabled = processor->lapic_flags & ACPI_MADT_ENABLED;
 #ifdef CONFIG_X86_X2APIC
 	/*
 	 * We need to register disabled CPU as well to permit
@@ -229,8 +233,10 @@ acpi_parse_x2apic(struct acpi_subtable_h
 	 * to not preallocating memory for all NR_CPUS
 	 * when we use CPU hotplug.
 	 */
-	acpi_register_lapic(processor->local_apic_id,	/* APIC ID */
-			    processor->lapic_flags & ACPI_MADT_ENABLED);
+	if (x2apic_disabled && (apic_id >= 0xff) && enabled)
+		printk(KERN_WARNING PREFIX "x2apic entry ignored\n");
+	else
+		acpi_register_lapic(apic_id, enabled);
 #else
 	printk(KERN_WARNING PREFIX "x2apic entry ignored\n");
 #endif
Index: linux-2.6/arch/x86/kernel/apic/apic.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/apic/apic.c
+++ linux-2.6/arch/x86/kernel/apic/apic.c
@@ -138,15 +138,14 @@ int x2apic_mode;
 #ifdef CONFIG_X86_X2APIC
 /* x2apic enabled before OS handover */
 static int x2apic_preenabled;
+int x2apic_disabled;
 static __init int setup_nox2apic(char *str)
 {
-	if (x2apic_enabled()) {
-		pr_warning("Bios already enabled x2apic, "
-			   "can't enforce nox2apic");
-		return 0;
-	}
+	if (x2apic_enabled())
+		pr_warning("Bios already enabled x2apic, will disable it");
+
+	x2apic_disabled = 1;
 
-	setup_clear_cpu_cap(X86_FEATURE_X2APIC);
 	return 0;
 }
 early_param("nox2apic", setup_nox2apic);
@@ -1394,8 +1393,38 @@ void __cpuinit end_local_APIC_setup(void
 }
 
 #ifdef CONFIG_X86_X2APIC
+
+static void disable_x2apic(void)
+{
+	int msr, msr2;
+
+	if (!cpu_has_x2apic)
+		return;
+
+	rdmsr(MSR_IA32_APICBASE, msr, msr2);
+	if (msr & X2APIC_ENABLE) {
+		u32 x2apic_id = x2apic_cpuid_initial_apicid();
+
+		if (x2apic_id > 255)
+			panic("Can not disable x2apic, id: %08x\n", x2apic_id);
+
+		pr_info("Disabling x2apic\n");
+		/*
+		 * Need to disable xapic and x2apic at the same time at first
+		 *  then enable xapic
+		 */
+		wrmsr(MSR_IA32_APICBASE, msr & ~(X2APIC_ENABLE | XAPIC_ENABLE),
+			 0);
+		wrmsr(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE, 0);
+	}
+}
 void check_x2apic(void)
 {
+	if (x2apic_disabled) {
+		disable_x2apic();
+		return;
+	}
+
 	if (x2apic_enabled()) {
 		pr_info("x2apic enabled by BIOS, switching to x2apic ops\n");
 		x2apic_preenabled = x2apic_mode = 1;
@@ -1406,6 +1435,11 @@ void enable_x2apic(void)
 {
 	int msr, msr2;
 
+	if (x2apic_disabled) {
+		disable_x2apic();
+		return;
+	}
+
 	if (!x2apic_mode)
 		return;
 
@@ -1474,6 +1508,9 @@ void __init enable_IR_x2apic(void)
 	else
 		ret = enable_IR();
 
+	if (x2apic_disabled)
+		goto nox2apic;
+
 	if (!ret) {
 		/* IR is required if there is APIC ID > 255 even when running
 		 * under KVM
@@ -1510,7 +1547,7 @@ out:
 
 	if (x2apic_preenabled)
 		panic("x2apic: enabled by BIOS but kernel init failed.");
-	else if (cpu_has_x2apic)
+	else if (cpu_has_x2apic && !x2apic_disabled)
 		pr_info("Not enabling x2apic, Intr-remapping init failed.\n");
 }
 
Index: linux-2.6/arch/x86/mm/srat_64.c
===================================================================
--- linux-2.6.orig/arch/x86/mm/srat_64.c
+++ linux-2.6/arch/x86/mm/srat_64.c
@@ -126,6 +126,12 @@ acpi_numa_x2apic_affinity_init(struct ac
 	if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
 		return;
 	pxm = pa->proximity_domain;
+	apic_id = pa->apic_id;
+	if (x2apic_disabled && (apic_id >= 0xff)) {
+		printk(KERN_INFO "SRAT: PXM %u -> X2APIC 0x%04x ignored\n",
+			 pxm, apic_id);
+		return;
+	}
 	node = setup_node(pxm);
 	if (node < 0) {
 		printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
@@ -133,7 +139,6 @@ acpi_numa_x2apic_affinity_init(struct ac
 		return;
 	}
 
-	apic_id = pa->apic_id;
 	apicid_to_node[apic_id] = node;
 	node_set(node, cpu_nodes_parsed);
 	acpi_numa = 1;
Index: linux-2.6/arch/x86/include/asm/apicdef.h
===================================================================
--- linux-2.6.orig/arch/x86/include/asm/apicdef.h
+++ linux-2.6/arch/x86/include/asm/apicdef.h
@@ -141,6 +141,7 @@
 
 #define APIC_BASE (fix_to_virt(FIX_APIC_BASE))
 #define APIC_BASE_MSR	0x800
+#define XAPIC_ENABLE	(1UL << 11)
 #define X2APIC_ENABLE	(1UL << 10)
 
 #ifdef CONFIG_X86_32
Index: linux-2.6/arch/x86/include/asm/processor.h
===================================================================
--- linux-2.6.orig/arch/x86/include/asm/processor.h
+++ linux-2.6/arch/x86/include/asm/processor.h
@@ -172,6 +172,7 @@ extern void init_scattered_cpuid_feature
 extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c);
 extern unsigned short num_cache_leaves;
 
+u32 x2apic_cpuid_initial_apicid(void);
 extern void detect_extended_topology(struct cpuinfo_x86 *c);
 extern void detect_ht(struct cpuinfo_x86 *c);
 
Index: linux-2.6/arch/x86/kernel/cpu/topology.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/cpu/topology.c
+++ linux-2.6/arch/x86/kernel/cpu/topology.c
@@ -21,6 +21,27 @@
 #define BITS_SHIFT_NEXT_LEVEL(eax)	((eax) & 0x1f)
 #define LEVEL_MAX_SIBLINGS(ebx)		((ebx) & 0xffff)
 
+u32 x2apic_cpuid_initial_apicid(void)
+{
+	unsigned int eax, ebx, ecx, edx;
+
+	if (boot_cpu_data.cpuid_level < 0xb)
+		return 0;
+
+	cpuid_count(0xb, SMT_LEVEL, &eax, &ebx, &ecx, &edx);
+
+	/*
+	 * check if the cpuid leaf 0xb is actually implemented.
+	 */
+	if (ebx == 0 || (LEAFB_SUBTYPE(ecx) != SMT_TYPE))
+		return 0;
+
+	/*
+	 * initial apic id, which also represents 32-bit extended x2apic id.
+	 */
+	return edx;
+}
+
 /*
  * Check for extended topology enumeration cpuid leaf 0xb and if it
  * exists, use it for populating initial_apicid and cpu topology