[PATCH] MSI interrupts: disallow when no LAPIC/IOAPIC support

[PATCH] MSI interrupts: disallow when no LAPIC/IOAPIC support

[Randy.Dunlap]

From: Randy Dunlap <rdunlap@xenotime.net>

MSI requires local APIC + IO APIC support (according to
drivers/pci/Kconfig), but if a kernel is built with all of
those (APICs + CONFIG_PCI_MSI) and then booted with "nosmp" or
"max_cpus=0|1" or "noapic" or "nolapic", MSI also should be
disabled, otherwise the interrupt routing is bad (so that only
using "irqpoll" helps).

Signed-off-by: Randy Dunlap <rdunlap@xenotime.net>
---

 arch/i386/kernel/io_apic.c   |    2 ++
 arch/i386/kernel/smpboot.c   |    6 +++++-
 arch/x86_64/kernel/apic.c    |    6 +++++-
 arch/x86_64/kernel/io_apic.c |    2 ++
 arch/x86_64/kernel/setup.c   |    4 +++-
 arch/x86_64/kernel/smpboot.c |   11 +++++++++--
 drivers/pci/msi.c            |    5 +++++
 include/linux/pci.h          |    2 ++
 8 files changed, 33 insertions(+), 5 deletions(-)

diff -Naurp -X linux-2614-rc2/Documentation/dontdiff linux-2614-rc2-git4-pv/arch/i386/kernel/io_apic.c linux-2614-rc2-git4/arch/i386/kernel/io_apic.c
--- linux-2614-rc2-git4-pv/arch/i386/kernel/io_apic.c	2005-09-25 13:18:08.000000000 -0700
+++ linux-2614-rc2-git4/arch/i386/kernel/io_apic.c	2005-09-25 14:54:45.000000000 -0700
@@ -31,6 +31,7 @@
 #include <linux/mc146818rtc.h>
 #include <linux/compiler.h>
 #include <linux/acpi.h>
+#include <linux/pci.h>
 #include <linux/module.h>
 #include <linux/sysdev.h>
 
@@ -684,6 +685,7 @@ int skip_ioapic_setup;
 static int __init ioapic_setup(char *str)
 {
 	skip_ioapic_setup = 1;
+	msi_off();
 	return 1;
 }
 
diff -Naurp -X linux-2614-rc2/Documentation/dontdiff linux-2614-rc2-git4-pv/arch/i386/kernel/smpboot.c linux-2614-rc2-git4/arch/i386/kernel/smpboot.c
--- linux-2614-rc2-git4-pv/arch/i386/kernel/smpboot.c	2005-09-25 13:18:08.000000000 -0700
+++ linux-2614-rc2-git4/arch/i386/kernel/smpboot.c	2005-09-25 14:54:51.000000000 -0700
@@ -46,6 +46,7 @@
 #include <linux/bootmem.h>
 #include <linux/notifier.h>
 #include <linux/cpu.h>
+#include <linux/pci.h>
 #include <linux/percpu.h>
 
 #include <linux/delay.h>
@@ -1107,9 +1108,11 @@ static void __init smp_boot_cpus(unsigne
 		printk(KERN_NOTICE "SMP motherboard not detected.\n");
 		smpboot_clear_io_apic_irqs();
 		phys_cpu_present_map = physid_mask_of_physid(0);
-		if (APIC_init_uniprocessor())
+		if (APIC_init_uniprocessor()) {
 			printk(KERN_NOTICE "Local APIC not detected."
 					   " Using dummy APIC emulation.\n");
+			msi_off();
+		}
 		map_cpu_to_logical_apicid();
 		cpu_set(0, cpu_sibling_map[0]);
 		cpu_set(0, cpu_core_map[0]);
@@ -1150,6 +1153,7 @@ static void __init smp_boot_cpus(unsigne
 		smp_found_config = 0;
 		printk(KERN_INFO "SMP mode deactivated, forcing use of dummy APIC emulation.\n");
 		smpboot_clear_io_apic_irqs();
+		msi_off();
 		phys_cpu_present_map = physid_mask_of_physid(0);
 		cpu_set(0, cpu_sibling_map[0]);
 		cpu_set(0, cpu_core_map[0]);
diff -Naurp -X linux-2614-rc2/Documentation/dontdiff linux-2614-rc2-git4-pv/arch/x86_64/kernel/apic.c linux-2614-rc2-git4/arch/x86_64/kernel/apic.c
--- linux-2614-rc2-git4-pv/arch/x86_64/kernel/apic.c	2005-09-25 13:18:11.000000000 -0700
+++ linux-2614-rc2-git4/arch/x86_64/kernel/apic.c	2005-09-25 14:55:20.000000000 -0700
@@ -24,6 +24,7 @@
 #include <linux/interrupt.h>
 #include <linux/mc146818rtc.h>
 #include <linux/kernel_stat.h>
+#include <linux/pci.h>
 #include <linux/sysdev.h>
 
 #include <asm/atomic.h>
@@ -1057,8 +1058,10 @@ int __init APIC_init_uniprocessor (void)
 #ifdef CONFIG_X86_IO_APIC
 	if (smp_found_config && !skip_ioapic_setup && nr_ioapics)
 			setup_IO_APIC();
-	else
+	else {
 		nr_ioapics = 0;
+		msi_off();
+	}
 #endif
 	setup_boot_APIC_clock();
 	check_nmi_watchdog();
@@ -1074,6 +1077,7 @@ static __init int setup_disableapic(char
 static __init int setup_nolapic(char *str) 
 { 
 	disable_apic = 1;
+	msi_off();
 	return 0;
 } 
 
diff -Naurp -X linux-2614-rc2/Documentation/dontdiff linux-2614-rc2-git4-pv/arch/x86_64/kernel/io_apic.c linux-2614-rc2-git4/arch/x86_64/kernel/io_apic.c
--- linux-2614-rc2-git4-pv/arch/x86_64/kernel/io_apic.c	2005-09-25 13:18:11.000000000 -0700
+++ linux-2614-rc2-git4/arch/x86_64/kernel/io_apic.c	2005-09-25 14:55:22.000000000 -0700
@@ -29,6 +29,7 @@
 #include <linux/smp_lock.h>
 #include <linux/mc146818rtc.h>
 #include <linux/acpi.h>
+#include <linux/pci.h>
 #include <linux/sysdev.h>
 
 #include <asm/io.h>
@@ -227,6 +228,7 @@ int ioapic_force;
 static int __init disable_ioapic_setup(char *str)
 {
 	skip_ioapic_setup = 1;
+	msi_off();
 	return 1;
 }
 
diff -Naurp -X linux-2614-rc2/Documentation/dontdiff linux-2614-rc2-git4-pv/arch/x86_64/kernel/setup.c linux-2614-rc2-git4/arch/x86_64/kernel/setup.c
--- linux-2614-rc2-git4-pv/arch/x86_64/kernel/setup.c	2005-09-25 13:18:11.000000000 -0700
+++ linux-2614-rc2-git4/arch/x86_64/kernel/setup.c	2005-09-25 14:55:27.000000000 -0700
@@ -345,8 +345,10 @@ static __init void parse_cmdline_early (
 		    !memcmp(from, "disableapic", 11))
 			disable_apic = 1;
 
-		if (!memcmp(from, "noapic", 6)) 
+		if (!memcmp(from, "noapic", 6)) {
 			skip_ioapic_setup = 1;
+			msi_off();
+		}
 
 		if (!memcmp(from, "apic", 4)) { 
 			skip_ioapic_setup = 0;
diff -Naurp -X linux-2614-rc2/Documentation/dontdiff linux-2614-rc2-git4-pv/arch/x86_64/kernel/smpboot.c linux-2614-rc2-git4/arch/x86_64/kernel/smpboot.c
--- linux-2614-rc2-git4-pv/arch/x86_64/kernel/smpboot.c	2005-09-25 13:18:11.000000000 -0700
+++ linux-2614-rc2-git4/arch/x86_64/kernel/smpboot.c	2005-09-25 14:55:33.000000000 -0700
@@ -47,6 +47,7 @@
 #include <linux/bootmem.h>
 #include <linux/thread_info.h>
 #include <linux/module.h>
+#include <linux/pci.h>
 
 #include <linux/delay.h>
 #include <linux/mc146818rtc.h>
@@ -918,9 +919,11 @@ static int __init smp_sanity_check(unsig
 	if (!smp_found_config) {
 		printk(KERN_NOTICE "SMP motherboard not detected.\n");
 		disable_smp();
-		if (APIC_init_uniprocessor())
+		if (APIC_init_uniprocessor()) {
 			printk(KERN_NOTICE "Local APIC not detected."
 					   " Using dummy APIC emulation.\n");
+			msi_off();
+		}
 		return -1;
 	}
 
@@ -942,6 +945,7 @@ static int __init smp_sanity_check(unsig
 			boot_cpu_id);
 		printk(KERN_ERR "... forcing use of dummy APIC emulation. (tell your hw vendor)\n");
 		nr_ioapics = 0;
+		msi_off();
 		return -1;
 	}
 
@@ -951,6 +955,7 @@ static int __init smp_sanity_check(unsig
 	if (!max_cpus) {
 		printk(KERN_INFO "SMP mode deactivated, forcing use of dummy APIC emulation.\n");
 		nr_ioapics = 0;
+		msi_off();
 		return -1;
 	}
 
@@ -995,8 +1000,10 @@ void __init smp_prepare_cpus(unsigned in
 	 */
 	if (!skip_ioapic_setup && nr_ioapics)
 		setup_IO_APIC();
-	else
+	else {
 		nr_ioapics = 0;
+		msi_off();
+	}
 
 	/*
 	 * Set up local APIC timer on boot CPU.
diff -Naurp -X linux-2614-rc2/Documentation/dontdiff linux-2614-rc2-git4-pv/drivers/pci/msi.c linux-2614-rc2-git4/drivers/pci/msi.c
--- linux-2614-rc2-git4-pv/drivers/pci/msi.c	2005-09-25 13:18:14.000000000 -0700
+++ linux-2614-rc2-git4/drivers/pci/msi.c	2005-09-25 14:53:53.000000000 -0700
@@ -1113,6 +1113,11 @@ void msi_remove_pci_irq_vectors(struct p
 	}
 }
 
+void msi_off(void)
+{
+	pci_msi_enable = 0;
+}
+
 EXPORT_SYMBOL(pci_enable_msi);
 EXPORT_SYMBOL(pci_disable_msi);
 EXPORT_SYMBOL(pci_enable_msix);
diff -Naurp -X linux-2614-rc2/Documentation/dontdiff linux-2614-rc2-git4-pv/include/linux/pci.h linux-2614-rc2-git4/include/linux/pci.h
--- linux-2614-rc2-git4-pv/include/linux/pci.h	2005-09-25 13:18:20.000000000 -0700
+++ linux-2614-rc2-git4/include/linux/pci.h	2005-09-25 14:53:27.000000000 -0700
@@ -480,6 +480,7 @@ static inline int pci_enable_msix(struct
 	struct msix_entry *entries, int nvec) {return -1;}
 static inline void pci_disable_msix(struct pci_dev *dev) {}
 static inline void msi_remove_pci_irq_vectors(struct pci_dev *dev) {}
+static inline void msi_off(void) {}
 #else
 extern void pci_scan_msi_device(struct pci_dev *dev);
 extern int pci_enable_msi(struct pci_dev *dev);
@@ -488,6 +489,7 @@ extern int pci_enable_msix(struct pci_de
 	struct msix_entry *entries, int nvec);
 extern void pci_disable_msix(struct pci_dev *dev);
 extern void msi_remove_pci_irq_vectors(struct pci_dev *dev);
+extern void msi_off(void);
 #endif
 
 /*


---

Re: [PATCH] MSI interrupts: disallow when no LAPIC/IOAPIC support

[Grant Grundler]

On Mon, Sep 26, 2005 at 08:11:56PM -0700, Randy.Dunlap wrote:
> From: Randy Dunlap <rdunlap@xenotime.net>
> 
> MSI requires local APIC + IO APIC support (according to
> drivers/pci/Kconfig),

The purpose of MSI is to bypass the IO APIC.
I'm not aware of any dependency on IO APIC in HW.
I suspect the dependency in the Kconfig is historical
because of how the x86 source code is structured:
config X86_LOCAL_APIC
        bool
        depends on X86_UP_APIC || ((X86_VISWS || SMP) && !X86_VOYAGER)
        default y

essentially, LOCAL_APIC gets enabled if "UP_APIC or SMP".
I've no clue why folks thought it was better to ignore
the IO APIC on UP kernels.

I'm pretty sure one only needs a local APIC support.
Local APIC responds to 0xfee00000 range of addresses and
directs the interrupt to a CPU.


> but if a kernel is built with all of
> those (APICs + CONFIG_PCI_MSI) and then booted with "nosmp" or
> "max_cpus=0|1" or "noapic" or "nolapic", MSI also should be
> disabled, otherwise the interrupt routing is bad (so that only
> using "irqpoll" helps).

UP vs SMP is orthogonal to MSI support wrt HW.
I suppose this is another historical artifact of
APICs being used with SMP x86 boxes first.

thanks,
grant

Re: [PATCH] MSI interrupts: disallow when no LAPIC/IOAPIC support

[Randy.Dunlap]

On Mon, 26 Sep 2005 22:48:40 -0600 Grant Grundler wrote:

> On Mon, Sep 26, 2005 at 08:11:56PM -0700, Randy.Dunlap wrote:
> > From: Randy Dunlap <rdunlap@xenotime.net>
> > 
> > MSI requires local APIC + IO APIC support (according to
> > drivers/pci/Kconfig),
> 
> The purpose of MSI is to bypass the IO APIC.
> I'm not aware of any dependency on IO APIC in HW.
> I suspect the dependency in the Kconfig is historical
> because of how the x86 source code is structured:
> config X86_LOCAL_APIC
>         bool
>         depends on X86_UP_APIC || ((X86_VISWS || SMP) && !X86_VOYAGER)
>         default y
> 
> essentially, LOCAL_APIC gets enabled if "UP_APIC or SMP".
> I've no clue why folks thought it was better to ignore
> the IO APIC on UP kernels.
> 
> I'm pretty sure one only needs a local APIC support.
> Local APIC responds to 0xfee00000 range of addresses and
> directs the interrupt to a CPU.

Thanks.  Can't say I'm surprised at that info.

> > but if a kernel is built with all of
> > those (APICs + CONFIG_PCI_MSI) and then booted with "nosmp" or
> > "max_cpus=0|1" or "noapic" or "nolapic", MSI also should be
> > disabled, otherwise the interrupt routing is bad (so that only
> > using "irqpoll" helps).
> 
> UP vs SMP is orthogonal to MSI support wrt HW.
> I suppose this is another historical artifact of
> APICs being used with SMP x86 boxes first.

"nosmp" (currently) means 1 CPU and no LAPICs/no IOAPICs.
It doesn't have to be that way, but yes, I suspect that it's
mostly historical, plus a method of getting to a lowest common
hardware boot sequence, which is sometimes nice for debugging
or installation.

Nevertheless, there is a problem here.  What do you suggest
to solve it?  Just making PCI_MSI depend on Local APIC support,
or something else?

[Kernel is assigning MSI interrupts, but then they are "lost."
Using "irqpoll" will find them, but that's a performance penalty.]

---
~Randy
You can't do anything without having to do something else first.
-- Belefant's Law

Re: [PATCH] MSI interrupts: disallow when no LAPIC/IOAPIC support

[Grant Grundler]

On Mon, Sep 26, 2005 at 09:52:45PM -0700, Randy.Dunlap wrote:
> "nosmp" (currently) means 1 CPU and no LAPICs/no IOAPICs.

ok - ie we ignore HW that may be present.

In a way, this makes sense since LAPIC was used for
IRQ "load balancing" - ie IRQs could get redirected
to another "less busy" CPU.  IIRC, discussions about
"XPR" should explain how that works.

> It doesn't have to be that way, but yes, I suspect that it's
> mostly historical, plus a method of getting to a lowest common
> hardware boot sequence, which is sometimes nice for debugging
> or installation.
> 
> Nevertheless, there is a problem here.  What do you suggest
> to solve it?  Just making PCI_MSI depend on Local APIC support,
> or something else?

Yeah, my preference would be PCI_MSI only depend on Local APIC.
(Note that having a Local APIC implies having an IO APIC as well
- but MSI should completely ignore it.)

But I don't know enough x86 history to know if thats feasible
or not. And I'm not blessed with the time to unravel the x86
APIC support - if the dependencies are necessary, so be it.

> [Kernel is assigning MSI interrupts, but then they are "lost."
> Using "irqpoll" will find them, but that's a performance penalty.]

Interesting. I'm suprised an MSI can get "lost".
It implies MSI code is allocating a CPU vector but the vector is
not getting enabled/unmasked or the Local APIC is ignoring it.

Yeah, we don't want to be using irqpoll for this - entirely
defeats the purpose of MSI.

thanks,
grant

Re: [PATCH] MSI interrupts: disallow when no LAPIC/IOAPIC support

[Mikael Pettersson]

Randy.Dunlap writes:
 > "nosmp" (currently) means 1 CPU and no LAPICs/no IOAPICs.

nosmp shouldn't disable the local APIC or the I/O APIC(s).
If it really does, then that's a bug.  We have noapic/nolapic/lapic
to control enablement of the APICs.

Re: [PATCH] MSI interrupts: disallow when no LAPIC/IOAPIC support

[Mikael Pettersson]

Grant Grundler writes:
 > Yeah, my preference would be PCI_MSI only depend on Local APIC.
 > (Note that having a Local APIC implies having an IO APIC as well

Not true. Lots of systems have a local APIC but no I/O APICs.
However, having an I/O APIC pretty much implies having a local APIC.

Re: [PATCH] MSI interrupts: disallow when no LAPIC/IOAPIC support

[Grant Grundler]

On Tue, Sep 27, 2005 at 11:14:11AM +0200, Mikael Pettersson wrote:
> Grant Grundler writes:
>  > Yeah, my preference would be PCI_MSI only depend on Local APIC.
>  > (Note that having a Local APIC implies having an IO APIC as well
> 
> Not true. Lots of systems have a local APIC but no I/O APICs.

Eh?!!!
Do systems with Local APIC still have IRQ lines going to the CPU?
How does PCI IRQ line otherwise generate an interrupt?

> However, having an I/O APIC pretty much implies having a local APIC.

Ok - That sounds right for x86 and is true for ia64.
parisc has IO SAPICs but no local APIC.

parisc and (IIRC) alpha has the functional equivalent of Local APIC
but the implementation details are different. ie they can't share
code with x86/ia64.

thanks,
grant

Re: [PATCH] MSI interrupts: disallow when no LAPIC/IOAPIC support

[Jeff Garzik]

Grant Grundler wrote:
> I've no clue why folks thought it was better to ignore
> the IO APIC on UP kernels.

Hysterical raisins:  the -majority- of the early uniprocessor systems 
that claimed IOAPIC support were broken.

	Jeff

Re: [PATCH] MSI interrupts: disallow when no LAPIC/IOAPIC support

[Alan Cox]

On Maw, 2005-09-27 at 17:26 -0400, Jeff Garzik wrote:
> Grant Grundler wrote:
> > I've no clue why folks thought it was better to ignore
> > the IO APIC on UP kernels.
> 
> Hysterical raisins:  the -majority- of the early uniprocessor systems 
> that claimed IOAPIC support were broken.

Not really broken in most cases, but since nobody was using the APIC
board makers didn't bother wiring for it.

Re: [PATCH] MSI interrupts: disallow when no LAPIC/IOAPIC support

[Grant Grundler]

On Tue, Sep 27, 2005 at 11:37:50PM +0100, Alan Cox wrote:
> On Maw, 2005-09-27 at 17:26 -0400, Jeff Garzik wrote:
> > Grant Grundler wrote:
> > > I've no clue why folks thought it was better to ignore
> > > the IO APIC on UP kernels.
> > 
> > Hysterical raisins:  the -majority- of the early uniprocessor systems 
> > that claimed IOAPIC support were broken.
> 
> Not really broken in most cases, but since nobody was using the APIC
> board makers didn't bother wiring for it.

ok. Any clue how PCI IRQs got routed/handled on those boxes?
Did UP boards have an 8259 PIC and an IRQ line to the CPU?
Could an 8529 PIC even co-exist with an IO APIC?

Or was it something more silly like BIOS mfgs had no version
of Windows that could grok a IRQ routing table and thus no
incentive to enable that feature?

thanks,
grant

Re: [PATCH] MSI interrupts: disallow when no LAPIC/IOAPIC support

[Eric W. Biederman]

Grant Grundler <grundler@parisc-linux.org> writes:

> On Tue, Sep 27, 2005 at 11:37:50PM +0100, Alan Cox wrote:
>> On Maw, 2005-09-27 at 17:26 -0400, Jeff Garzik wrote:
>> > Grant Grundler wrote:
>> > > I've no clue why folks thought it was better to ignore
>> > > the IO APIC on UP kernels.
>> > 
>> > Hysterical raisins:  the -majority- of the early uniprocessor systems 
>> > that claimed IOAPIC support were broken.
>> 
>> Not really broken in most cases, but since nobody was using the APIC
>> board makers didn't bother wiring for it.
>
> ok. Any clue how PCI IRQs got routed/handled on those boxes?
> Did UP boards have an 8259 PIC and an IRQ line to the CPU?
> Could an 8529 PIC even co-exist with an IO APIC?

x86 linux can't even boot without using an 8259 PIC.
The 8259 PIC if you don't have an IOAPIC is wired to the 
local apic and the appropriate pin is placed in ExtINT mode.

Most current systems have an 8259 PIC, their primary PIC, and a
PIRQ router all in the same chip.

Eric