[PATCH 0/3] x86 PCI domain support

[PATCH 0/3] x86 PCI domain support

[Jeff Garzik]

ACPI PCI support stopped short of supporting multiple PCI domains,
which is something I need in order to support a current machine
configuration, and something many will soon need, to support upcoming
systems.

This is a minimal, untested implementation.  But it should work,
provided your PCI op hooks (direct, BIOS, mmconfig) support PCI domains
(mmconfig).

Patches:

1) Fix: don't hardcode PCI domain to zero
2) Introduce struct pci_sysdata
3) Add PCI domain to struct pci_sysdata

[PATCH 1/3] x86 PCI domain support: a humble fix

[Jeff Garzik]

commit 6bc310571a421a755822f1e65815399caddcb400
Author: Jeff Garzik <jgarzik@pobox.com>
Date:   Fri Dec 2 19:20:52 2005 -0500

    [x86, PCI] pass PCI domain number to PCI config read/write hooks
    
    Don't hardcode zero, since modern x86 (with special ACPI sauce)
    can support multiple "PCI segments", aka PCI domains.

 arch/i386/pci/common.c |    6 ++++--
 1 files changed, 4 insertions(+), 2 deletions(-)


6bc310571a421a755822f1e65815399caddcb400
diff --git a/arch/i386/pci/common.c b/arch/i386/pci/common.c
index f6bc48d..6a18a8a 100644
--- a/arch/i386/pci/common.c
+++ b/arch/i386/pci/common.c
@@ -31,12 +31,14 @@ struct pci_raw_ops *raw_pci_ops;
 
 static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *value)
 {
-	return raw_pci_ops->read(0, bus->number, devfn, where, size, value);
+	return raw_pci_ops->read(pci_domain_nr(bus), bus->number,
+				 devfn, where, size, value);
 }
 
 static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 value)
 {
-	return raw_pci_ops->write(0, bus->number, devfn, where, size, value);
+	return raw_pci_ops->write(pci_domain_nr(bus), bus->number,
+				  devfn, where, size, value);
 }
 
 struct pci_ops pci_root_ops = {

[PATCH 2/3] x86 PCI domain support: struct pci_sysdata

[Jeff Garzik]

commit a5e17f3017ae2a4bd10454057bf63bd56bdd506c
Author: Jeff Garzik <jgarzik@pobox.com>
Date:   Fri Dec 2 20:12:52 2005 -0500

    [x86, PCI] Switch pci_bus::sysdata from NUMA node integer to a pointer
    
    On x86[-64], struct pci_bus::sysdata is only used on NUMA platforms,
    to store the associated NUMA node.
    
    Preparing for the future when we'll want to do other things with
    sysdata, struct pci_sysdata was created.  An allocated structure
    replaces the cast-pointer-to-int NUMA usage.  Updated all NUMA users.

 arch/i386/pci/acpi.c          |    7 ++++---
 arch/i386/pci/common.c        |   13 ++++++++++++-
 arch/x86_64/pci/k8-bus.c      |    6 +++++-
 include/asm-i386/pci.h        |    5 +++++
 include/asm-i386/topology.h   |    2 +-
 include/asm-x86_64/pci.h      |    4 ++++
 include/asm-x86_64/topology.h |    2 +-
 7 files changed, 32 insertions(+), 7 deletions(-)


a5e17f3017ae2a4bd10454057bf63bd56bdd506c
diff --git a/arch/i386/pci/acpi.c b/arch/i386/pci/acpi.c
index 4c4522b..107b18d 100644
--- a/arch/i386/pci/acpi.c
+++ b/arch/i386/pci/acpi.c
@@ -19,9 +19,10 @@ struct pci_bus * __devinit pci_acpi_scan
 	if (bus != NULL) {
 		int pxm = acpi_get_pxm(device->handle);
 		if (pxm >= 0) {
-			bus->sysdata = (void *)(unsigned long)pxm_to_node(pxm);
-			printk("bus %d -> pxm %d -> node %ld\n",
-				busnum, pxm, (long)(bus->sysdata));
+			struct pci_sysdata *sd = bus->sysdata;
+			sd->node = pxm_to_node(pxm);
+			printk("bus %d -> pxm %d -> node %d\n",
+				busnum, pxm, sd->node);
 		}
 	}
 #endif
diff --git a/arch/i386/pci/common.c b/arch/i386/pci/common.c
index 6a18a8a..ab48fc7 100644
--- a/arch/i386/pci/common.c
+++ b/arch/i386/pci/common.c
@@ -126,6 +126,7 @@ void __devinit  pcibios_fixup_bus(struct
 struct pci_bus * __devinit pcibios_scan_root(int busnum)
 {
 	struct pci_bus *bus = NULL;
+	struct pci_sysdata *sd;
 
 	while ((bus = pci_find_next_bus(bus)) != NULL) {
 		if (bus->number == busnum) {
@@ -134,9 +135,19 @@ struct pci_bus * __devinit pcibios_scan_
 		}
 	}
 
+	/* Allocate per-root-bus (not per bus) arch-specific data.
+	 * TODO: leak; this memory is never freed.
+	 * It's arguable whether it's worth the trouble to care.
+	 */
+	sd = kzalloc(sizeof(*sd), GFP_KERNEL);
+	if (!sd) {
+		printk(KERN_ERR "PCI: OOM, not probing PCI bus %02x\n", busnum);
+		return NULL;
+	}
+
 	printk(KERN_DEBUG "PCI: Probing PCI hardware (bus %02x)\n", busnum);
 
-	return pci_scan_bus_parented(NULL, busnum, &pci_root_ops, NULL);
+	return pci_scan_bus_parented(NULL, busnum, &pci_root_ops, sd);
 }
 
 extern u8 pci_cache_line_size;
diff --git a/arch/x86_64/pci/k8-bus.c b/arch/x86_64/pci/k8-bus.c
index 3acf60d..9cc813e 100644
--- a/arch/x86_64/pci/k8-bus.c
+++ b/arch/x86_64/pci/k8-bus.c
@@ -59,6 +59,8 @@ fill_mp_bus_to_cpumask(void)
 				     j <= SUBORDINATE_LDT_BUS_NUMBER(ldtbus);
 				     j++) { 
 					struct pci_bus *bus;
+					struct pci_sysdata *sd;
+
 					long node = NODE_ID(nid);
 					/* Algorithm a bit dumb, but
  					   it shouldn't matter here */
@@ -67,7 +69,9 @@ fill_mp_bus_to_cpumask(void)
 						continue;
 					if (!node_online(node))
 						node = 0;
-					bus->sysdata = (void *)node;
+
+					sd = bus->sysdata;
+					sd->node = node;
 				}		
 			}
 		}
diff --git a/include/asm-i386/pci.h b/include/asm-i386/pci.h
index 78c8598..f56a158 100644
--- a/include/asm-i386/pci.h
+++ b/include/asm-i386/pci.h
@@ -4,6 +4,11 @@
 #include <linux/config.h>
 
 #ifdef __KERNEL__
+
+struct pci_sysdata {
+	int		node;		/* NUMA node */
+};
+
 #include <linux/mm.h>		/* for struct page */
 
 /* Can be used to override the logic in pci_scan_bus for skipping
diff --git a/include/asm-i386/topology.h b/include/asm-i386/topology.h
index 0ec27c9..4dc8580 100644
--- a/include/asm-i386/topology.h
+++ b/include/asm-i386/topology.h
@@ -60,7 +60,7 @@ static inline int node_to_first_cpu(int 
 	return first_cpu(mask);
 }
 
-#define pcibus_to_node(bus) ((long) (bus)->sysdata)
+#define pcibus_to_node(bus) ((struct pci_sysdata *)((bus)->sysdata))->node
 #define pcibus_to_cpumask(bus) node_to_cpumask(pcibus_to_node(bus))
 
 /* sched_domains SD_NODE_INIT for NUMAQ machines */
diff --git a/include/asm-x86_64/pci.h b/include/asm-x86_64/pci.h
index eeb3088..424c1ee 100644
--- a/include/asm-x86_64/pci.h
+++ b/include/asm-x86_64/pci.h
@@ -6,6 +6,10 @@
 
 #ifdef __KERNEL__
 
+struct pci_sysdata {
+	int		node;		/* NUMA node */
+};
+
 #include <linux/mm.h> /* for struct page */
 
 /* Can be used to override the logic in pci_scan_bus for skipping
diff --git a/include/asm-x86_64/topology.h b/include/asm-x86_64/topology.h
index d39ebd5..0d1d61c 100644
--- a/include/asm-x86_64/topology.h
+++ b/include/asm-x86_64/topology.h
@@ -25,7 +25,7 @@ extern int __node_distance(int, int);
 #define parent_node(node)		(node)
 #define node_to_first_cpu(node) 	(__ffs(node_to_cpumask[node]))
 #define node_to_cpumask(node)		(node_to_cpumask[node])
-#define pcibus_to_node(bus)		((long)(bus->sysdata))	
+#define pcibus_to_node(bus)	((struct pci_sysdata *)((bus)->sysdata))->node
 #define pcibus_to_cpumask(bus)		node_to_cpumask(pcibus_to_node(bus));
 
 #define numa_node_id()			read_pda(nodenumber)

[PATCH 3/3] x86 PCI domain support: the meat

[Jeff Garzik]

commit 31c93c154ca9a86a865b4ef28b7ba5d8e48f7b13
Author: Jeff Garzik <jgarzik@pobox.com>
Date:   Fri Dec 2 20:30:48 2005 -0500

    [x86, PCI] add PCI domain support
    
    * Store PCI domain in struct pci_sysdata
    * Add magic inlines to pass PCI domain to read/write config hooks
    * Support ACPI's notion of PCI domains (PCI segments)

 arch/i386/pci/acpi.c     |   20 ++++++++++++++++++--
 include/asm-i386/pci.h   |   14 ++++++++++++++
 include/asm-x86_64/pci.h |   14 ++++++++++++++
 3 files changed, 46 insertions(+), 2 deletions(-)


31c93c154ca9a86a865b4ef28b7ba5d8e48f7b13
diff --git a/arch/i386/pci/acpi.c b/arch/i386/pci/acpi.c
index 107b18d..38a752d 100644
--- a/arch/i386/pci/acpi.c
+++ b/arch/i386/pci/acpi.c
@@ -8,18 +8,34 @@
 struct pci_bus * __devinit pci_acpi_scan_root(struct acpi_device *device, int domain, int busnum)
 {
 	struct pci_bus *bus;
+	struct pci_sysdata *sd;
 
+	/* Allocate per-root-bus (not per bus) arch-specific data.
+	 * TODO: leak; this memory is never freed.
+	 * It's arguable whether it's worth the trouble to care.
+	 */
+	sd = kzalloc(sizeof(*sd), GFP_KERNEL);
+	if (!sd) {
+		printk(KERN_ERR "PCI: OOM, not probing PCI bus %02x\n", busnum);
+		return NULL;
+	}
+
+#ifdef CONFIG_PCI_DOMAINS
+	sd->domain = domain;
+#else
 	if (domain != 0) {
 		printk(KERN_WARNING "PCI: Multiple domains not supported\n");
 		return NULL;
 	}
+#endif /* CONFIG_PCI_DOMAINS */
 
-	bus = pcibios_scan_root(busnum);
+	bus = pci_scan_bus_parented(NULL, busnum, &pci_root_ops, sd);
+	if (!bus)
+		kfree(sd);
 #ifdef CONFIG_ACPI_NUMA
 	if (bus != NULL) {
 		int pxm = acpi_get_pxm(device->handle);
 		if (pxm >= 0) {
-			struct pci_sysdata *sd = bus->sysdata;
 			sd->node = pxm_to_node(pxm);
 			printk("bus %d -> pxm %d -> node %d\n",
 				busnum, pxm, sd->node);
diff --git a/include/asm-i386/pci.h b/include/asm-i386/pci.h
index f56a158..4068d1c 100644
--- a/include/asm-i386/pci.h
+++ b/include/asm-i386/pci.h
@@ -6,9 +6,23 @@
 #ifdef __KERNEL__
 
 struct pci_sysdata {
+	int		domain;		/* PCI domain */
 	int		node;		/* NUMA node */
 };
 
+#ifdef CONFIG_PCI_DOMAINS
+static inline int pci_domain_nr(struct pci_bus *bus)
+{
+	struct pci_sysdata *sd = bus->sysdata;
+	return sd->domain;
+}
+
+static inline int pci_proc_domain(struct pci_bus *bus)
+{
+	return pci_domain_nr(bus);
+}
+#endif /* CONFIG_PCI_DOMAINS */
+
 #include <linux/mm.h>		/* for struct page */
 
 /* Can be used to override the logic in pci_scan_bus for skipping
diff --git a/include/asm-x86_64/pci.h b/include/asm-x86_64/pci.h
index 424c1ee..bcbf9aa 100644
--- a/include/asm-x86_64/pci.h
+++ b/include/asm-x86_64/pci.h
@@ -7,9 +7,23 @@
 #ifdef __KERNEL__
 
 struct pci_sysdata {
+	int		domain;		/* PCI domain */
 	int		node;		/* NUMA node */
 };
 
+#ifdef CONFIG_PCI_DOMAINS
+static inline int pci_domain_nr(struct pci_bus *bus)
+{
+	struct pci_sysdata *sd = bus->sysdata;
+	return sd->domain;
+}
+
+static inline int pci_proc_domain(struct pci_bus *bus)
+{
+	return pci_domain_nr(bus);
+}
+#endif /* CONFIG_PCI_DOMAINS */
+
 #include <linux/mm.h> /* for struct page */
 
 /* Can be used to override the logic in pci_scan_bus for skipping

Re: [PATCH 0/3] x86 PCI domain support

[Jeff Garzik]

On Fri, Dec 02, 2005 at 08:39:04PM -0500, Jeff Garzik wrote:
> ACPI PCI support stopped short of supporting multiple PCI domains,
> which is something I need in order to support a current machine
> configuration, and something many will soon need, to support upcoming
> systems.

> Patches:
> 
> 1) Fix: don't hardcode PCI domain to zero
> 2) Introduce struct pci_sysdata
> 3) Add PCI domain to struct pci_sysdata


NOTE:  This should have the DO NOT APPLY label.  This is just for
comment, and testing if there are brave souls.

	Jeff

Re: [PATCH 0/3] x86 PCI domain support

[Andi Kleen]

On Fri, Dec 02, 2005 at 08:39:04PM -0500, Jeff Garzik wrote:
> 
> ACPI PCI support stopped short of supporting multiple PCI domains,
> which is something I need in order to support a current machine
> configuration, and something many will soon need, to support upcoming
> systems.
> 
> This is a minimal, untested implementation.  But it should work,
> provided your PCI op hooks (direct, BIOS, mmconfig) support PCI domains
> (mmconfig).

It looks like a good start.  Thanks for doing this.

It actually needs some more fixes - e.g. falling back to 
type1 if the bus is not covered in MCFG (needed for the 
K8 internal busses) and a workaround for buggy Asus BIOS with wrong MCFG.
I have that in the works.

But your changes are needed too - or at least they are correct
according to the spec. I don't know of a system that actually
has different mmconfig apertures for different busses yet.
The only case that's interesting right now is that some busses
don't support it at all, but these don't need a seg number,
just a non listing in MCFG.

Greg are you queueing this up? 

-Andi

Re: [PATCH 0/3] x86 PCI domain support

[Jeff Garzik]

Andi Kleen wrote:
> On Fri, Dec 02, 2005 at 08:39:04PM -0500, Jeff Garzik wrote:
> 
>>ACPI PCI support stopped short of supporting multiple PCI domains,
>>which is something I need in order to support a current machine
>>configuration, and something many will soon need, to support upcoming
>>systems.
>>
>>This is a minimal, untested implementation.  But it should work,
>>provided your PCI op hooks (direct, BIOS, mmconfig) support PCI domains
>>(mmconfig).
> 
> 
> It looks like a good start.  Thanks for doing this.
> 
> It actually needs some more fixes - e.g. falling back to 
> type1 if the bus is not covered in MCFG (needed for the 
> K8 internal busses) and a workaround for buggy Asus BIOS with wrong MCFG.
> I have that in the works.
> 
> But your changes are needed too - or at least they are correct
> according to the spec. I don't know of a system that actually
> has different mmconfig apertures for different busses yet.
> The only case that's interesting right now is that some busses
> don't support it at all, but these don't need a seg number,
> just a non listing in MCFG.
> 
> Greg are you queueing this up? 

The first two patches could go in immediately, the last should probably 
wait a bit...

	Jeff

Re: [PATCH 0/3] x86 PCI domain support

[Greg KH]

On Sat, Dec 03, 2005 at 03:11:54PM -0500, Jeff Garzik wrote:
> Andi Kleen wrote:
> >On Fri, Dec 02, 2005 at 08:39:04PM -0500, Jeff Garzik wrote:
> >
> >>ACPI PCI support stopped short of supporting multiple PCI domains,
> >>which is something I need in order to support a current machine
> >>configuration, and something many will soon need, to support upcoming
> >>systems.
> >>
> >>This is a minimal, untested implementation.  But it should work,
> >>provided your PCI op hooks (direct, BIOS, mmconfig) support PCI domains
> >>(mmconfig).
> >
> >
> >It looks like a good start.  Thanks for doing this.
> >
> >It actually needs some more fixes - e.g. falling back to 
> >type1 if the bus is not covered in MCFG (needed for the 
> >K8 internal busses) and a workaround for buggy Asus BIOS with wrong MCFG.
> >I have that in the works.
> >
> >But your changes are needed too - or at least they are correct
> >according to the spec. I don't know of a system that actually
> >has different mmconfig apertures for different busses yet.
> >The only case that's interesting right now is that some busses
> >don't support it at all, but these don't need a seg number,
> >just a non listing in MCFG.
> >
> >Greg are you queueing this up? 
> 
> The first two patches could go in immediately, the last should probably 
> wait a bit...

Ok, I'll queue up all of them for testing in the next few -mm releases
before sending them on.

Care to add some "Signed-off-by:" lines to the patches?

thanks,

greg k-h

Re: [PATCH 0/3] x86 PCI domain support

[Greg KH]

On Sat, Dec 03, 2005 at 03:11:54PM -0500, Jeff Garzik wrote:
> The first two patches could go in immediately, the last should probably 
> wait a bit...

What is the rush?  These seem pretty late for the -rc series :)

I'll send them in after 2.6.15 is out, is that ok?

thanks,

greg k-h

Re: [PATCH 0/3] x86 PCI domain support

[Andi Kleen]

On Tue, Dec 06, 2005 at 04:39:22PM -0800, Greg KH wrote:
> On Sat, Dec 03, 2005 at 03:11:54PM -0500, Jeff Garzik wrote:
> > The first two patches could go in immediately, the last should probably 
> > wait a bit...
> 
> What is the rush?  These seem pretty late for the -rc series :)
> 
> I'll send them in after 2.6.15 is out, is that ok?

It's ok, but the two related workaround patches I posted earlier should
go into .15 (because they fix broken boot on some machines) 

-Andi

Re: [PATCH 0/3] x86 PCI domain support

[Greg KH]

On Wed, Dec 07, 2005 at 02:32:02AM +0100, Andi Kleen wrote:
> On Tue, Dec 06, 2005 at 04:39:22PM -0800, Greg KH wrote:
> > On Sat, Dec 03, 2005 at 03:11:54PM -0500, Jeff Garzik wrote:
> > > The first two patches could go in immediately, the last should probably 
> > > wait a bit...
> > 
> > What is the rush?  These seem pretty late for the -rc series :)
> > 
> > I'll send them in after 2.6.15 is out, is that ok?
> 
> It's ok, but the two related workaround patches I posted earlier should
> go into .15 (because they fix broken boot on some machines) 

Ok, that's fine.  Thanks for fixing those issues, I appreciate it.

greg k-h

Re: [PATCH 0/3] x86 PCI domain support

[Jeff Garzik]

Greg KH wrote:
> On Sat, Dec 03, 2005 at 03:11:54PM -0500, Jeff Garzik wrote:
> 
>>The first two patches could go in immediately, the last should probably 
>>wait a bit...
> 
> 
> What is the rush?  These seem pretty late for the -rc series :)
> 
> I'll send them in after 2.6.15 is out, is that ok?

You were supposed to read my mind :)  "immediately" meant "ok for 
upstream when -rc cycle closes" :)  The third patch I don't consider 
ready for upstream, -rc or no.

So post-2.6.15 is quite fine, that's what I expected.

	Jeff

Re: [PATCH 0/3] x86 PCI domain support

[Greg KH]

On Tue, Dec 06, 2005 at 09:14:30PM -0500, Jeff Garzik wrote:
> Greg KH wrote:
> >On Sat, Dec 03, 2005 at 03:11:54PM -0500, Jeff Garzik wrote:
> >
> >>The first two patches could go in immediately, the last should probably 
> >>wait a bit...
> >
> >
> >What is the rush?  These seem pretty late for the -rc series :)
> >
> >I'll send them in after 2.6.15 is out, is that ok?
> 
> You were supposed to read my mind :)  "immediately" meant "ok for 
> upstream when -rc cycle closes" :)  The third patch I don't consider 
> ready for upstream, -rc or no.

Ok, thanks.  But I did just include the third patch in my tree, so it
will get tested in -mm.  If you don't want this to happen, just let me
know and I'll drop it.

thanks,

greg k-h

Re: [PATCH 0/3] x86 PCI domain support

[Jeff Garzik]

Greg KH wrote:
> On Tue, Dec 06, 2005 at 09:14:30PM -0500, Jeff Garzik wrote:
> 
>>Greg KH wrote:
>>
>>>On Sat, Dec 03, 2005 at 03:11:54PM -0500, Jeff Garzik wrote:
>>>
>>>
>>>>The first two patches could go in immediately, the last should probably 
>>>>wait a bit...
>>>
>>>
>>>What is the rush?  These seem pretty late for the -rc series :)
>>>
>>>I'll send them in after 2.6.15 is out, is that ok?
>>
>>You were supposed to read my mind :)  "immediately" meant "ok for 
>>upstream when -rc cycle closes" :)  The third patch I don't consider 
>>ready for upstream, -rc or no.
> 
> 
> Ok, thanks.  But I did just include the third patch in my tree, so it
> will get tested in -mm.  If you don't want this to happen, just let me
> know and I'll drop it.

There's no ultimate harm in it, because nothing turns on 
CONFIG_PCI_DOMAINS in x86[-64] yet...

	Jeff

Re: [PATCH 0/3] x86 PCI domain support

[Greg KH]

On Tue, Dec 06, 2005 at 11:12:07PM -0500, Jeff Garzik wrote:
> Greg KH wrote:
> >On Tue, Dec 06, 2005 at 09:14:30PM -0500, Jeff Garzik wrote:
> >
> >>Greg KH wrote:
> >>
> >>>On Sat, Dec 03, 2005 at 03:11:54PM -0500, Jeff Garzik wrote:
> >>>
> >>>
> >>>>The first two patches could go in immediately, the last should probably 
> >>>>wait a bit...
> >>>
> >>>
> >>>What is the rush?  These seem pretty late for the -rc series :)
> >>>
> >>>I'll send them in after 2.6.15 is out, is that ok?
> >>
> >>You were supposed to read my mind :)  "immediately" meant "ok for 
> >>upstream when -rc cycle closes" :)  The third patch I don't consider 
> >>ready for upstream, -rc or no.
> >
> >
> >Ok, thanks.  But I did just include the third patch in my tree, so it
> >will get tested in -mm.  If you don't want this to happen, just let me
> >know and I'll drop it.
> 
> There's no ultimate harm in it, because nothing turns on 
> CONFIG_PCI_DOMAINS in x86[-64] yet...

I'm guessing that will be a follow-on patch?  :)

greg k-h