[PATCH] pcibus_to_node implementation for IA64

[PATCH] pcibus_to_node implementation for IA64

[Christoph Lameter]

pcibus_to_node provides a way for the Linux kernel to identify to which node
a certain pcibus connect to. Allocations of control structures for devices can then be
made on the node where the pci bus is located to allow local access during interrupt and
other device manipulation.

The numa aware block layer patch now in Linus tree relies on 
pcibus_to_node to allocate block device control structures on the correct 
node. There is a pending patch for an ethernet driver to allocate the 
descriptors on the node of each ethernet device.

pcibus_to_node was implemented for IA64 by adding an additional node field 
to the pci_controller structure and then have each subarchitecture set 
this field.

I hope I got the implementation for HP right. Please check.

The implementation for Altix is currently only returning the correct node 
id for devices that are located on nodes with memory. For devices from 
nodes without memory pcibus_to_node will return -1 (undetermined) which 
will then result in the fallback to old any node (or cpu local node) 
allocation strategy. We will fix that as soon as we can figure out how to
get to valid Linux node numbers from the io node numbers.

Signed-off-by: Christoph Lameter <clameter@sgi.com>

Index: linux-2.6.12/include/asm-ia64/pci.h
=================================--- linux-2.6.12.orig/include/asm-ia64/pci.h	2005-06-17 12:48:29.000000000 -0700
+++ linux-2.6.12/include/asm-ia64/pci.h	2005-06-23 13:55:01.000000000 -0700
@@ -109,6 +109,7 @@ struct pci_controller {
 	void *acpi_handle;
 	void *iommu;
 	int segment;
+	int node;		/* nearest node with memory or -1 for global allocation */
 
 	unsigned int windows;
 	struct pci_window *window;
Index: linux-2.6.12/include/asm-ia64/topology.h
=================================--- linux-2.6.12.orig/include/asm-ia64/topology.h	2005-06-17 12:48:29.000000000 -0700
+++ linux-2.6.12/include/asm-ia64/topology.h	2005-06-23 13:55:01.000000000 -0700
@@ -40,6 +40,11 @@
  */
 #define node_to_first_cpu(node) (__ffs(node_to_cpumask(node)))
 
+/*
+ * Determines the node for a given pci bus
+ */
+#define pcibus_to_node(bus) PCI_CONTROLLER(bus)->node
+
 void build_cpu_to_node_map(void);
 
 /* sched_domains SD_NODE_INIT for IA64 NUMA machines */
Index: linux-2.6.12/arch/ia64/pci/pci.c
=================================--- linux-2.6.12.orig/arch/ia64/pci/pci.c	2005-06-17 12:48:29.000000000 -0700
+++ linux-2.6.12/arch/ia64/pci/pci.c	2005-06-23 13:55:01.000000000 -0700
@@ -175,6 +175,7 @@ alloc_pci_controller (int seg)
 
 	memset(controller, 0, sizeof(*controller));
 	controller->segment = seg;
+	controller->node = -1;
 	return controller;
 }
 
Index: linux-2.6.12/arch/ia64/sn/kernel/io_init.c
=================================--- linux-2.6.12.orig/arch/ia64/sn/kernel/io_init.c	2005-06-17 12:48:29.000000000 -0700
+++ linux-2.6.12/arch/ia64/sn/kernel/io_init.c	2005-06-23 13:55:02.000000000 -0700
@@ -364,6 +364,8 @@ static void sn_pci_controller_fixup(int 
 
 	nasid = NASID_GET(SN_PCIBUS_BUSSOFT(bus)->bs_base);
 	cnode = nasid_to_cnodeid(nasid);
+	if (cnode < num_online_nodes() && cnode < numionodes)
+		controller->node = cnode;
 	hubdev_info = (struct hubdev_info *)(NODEPDA(cnode)->pdinfo);
 	SN_PCIBUS_BUSSOFT(bus)->bs_xwidget_info  	    &(hubdev_info->hdi_xwidget_info[SN_PCIBUS_BUSSOFT(bus)->bs_xid]);
Index: linux-2.6.12/arch/ia64/sn/pci/pci_dma.c
=================================--- linux-2.6.12.orig/arch/ia64/sn/pci/pci_dma.c	2005-06-17 12:48:29.000000000 -0700
+++ linux-2.6.12/arch/ia64/sn/pci/pci_dma.c	2005-06-23 13:55:02.000000000 -0700
@@ -78,6 +78,7 @@ void *sn_dma_alloc_coherent(struct devic
 {
 	void *cpuaddr;
 	unsigned long phys_addr;
+	int node;
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct sn_pcibus_provider *provider = SN_PCIDEV_BUSPROVIDER(pdev);
 
@@ -85,10 +86,19 @@ void *sn_dma_alloc_coherent(struct devic
 
 	/*
 	 * Allocate the memory.
-	 * FIXME: We should be doing alloc_pages_node for the node closest
-	 *        to the PCI device.
 	 */
-	if (!(cpuaddr = (void *)__get_free_pages(GFP_ATOMIC, get_order(size))))
+	node = pcibus_to_node(pdev->bus);
+	if (likely(node >=0)) {
+		struct page *p = alloc_pages_node(node, GFP_ATOMIC, get_order(size));
+
+		if (likely(p))
+			cpuaddr = page_address(p);
+		else
+			return NULL;
+	} else
+		cpuaddr = (void *)__get_free_pages(GFP_ATOMIC, get_order(size));
+
+	if (unlikely(!cpuaddr))
 		return NULL;
 
 	memset(cpuaddr, 0x0, size);
Index: linux-2.6.12/arch/ia64/hp/common/sba_iommu.c
=================================--- linux-2.6.12.orig/arch/ia64/hp/common/sba_iommu.c	2005-06-17 12:48:29.000000000 -0700
+++ linux-2.6.12/arch/ia64/hp/common/sba_iommu.c	2005-06-23 13:55:02.000000000 -0700
@@ -1930,6 +1930,9 @@ sba_connect_bus(struct pci_bus *bus)
 		for (ioc = ioc_list; ioc; ioc = ioc->next)
 			if (ioc->handle = handle) {
 				PCI_CONTROLLER(bus)->iommu = ioc;
+#ifdef CONFIG_NUMA
+				PCI_CONTROLLER(bus)->node = ioc->node;
+#endif
 				return;
 			}
 

Re: [PATCH] pcibus_to_node implementation for IA64

[Jesse Barnes]

On Thursday, June 23, 2005 2:02 pm, Christoph Lameter wrote:
> -0700 @@ -109,6 +109,7 @@ struct pci_controller {
>  	void *acpi_handle;
>  	void *iommu;
>  	int segment;
> +	int node;		/* nearest node with memory or -1 for global allocation
> */
...
> +#ifdef CONFIG_NUMA
> +				PCI_CONTROLLER(bus)->node = ioc->node;
> +#endif
>  				return;
>  			}

It looks like you're adding the new 'node' field unconditionally, but 
protecting the sba assignment by CONFIG_NUMA?  Which one is wrong? :)

Jesse

Re: [PATCH] pcibus_to_node implementation for IA64

[Alex Williamson]

On Thu, 2005-06-23 at 14:02 -0700, Christoph Lameter wrote:
> --- linux-2.6.12.orig/arch/ia64/hp/common/sba_iommu.c	2005-06-17 12:48:29.000000000 -0700
> +++ linux-2.6.12/arch/ia64/hp/common/sba_iommu.c	2005-06-23 13:55:02.000000000 -0700
> @@ -1930,6 +1930,9 @@ sba_connect_bus(struct pci_bus *bus)
>  		for (ioc = ioc_list; ioc; ioc = ioc->next)
>  			if (ioc->handle = handle) {
>  				PCI_CONTROLLER(bus)->iommu = ioc;
> +#ifdef CONFIG_NUMA
> +				PCI_CONTROLLER(bus)->node = ioc->node;
> +#endif
>  				return;
>  			}

   This will work, but there's no reason to hide this in the platform
specific code.  The PCI root bridges are in ACPI namespace, a
pxm_to_node() lookup should be done there when the buses are discovered.
Then it would work on HP and DIG boxes.  Thanks,

	Alex

-- 
Alex Williamson                             HP Linux & Open Source Lab

Re: [PATCH] pcibus_to_node implementation for IA64

[Christoph Lameter]

On Thu, 23 Jun 2005, Alex Williamson wrote:

>    This will work, but there's no reason to hide this in the platform
> specific code.  The PCI root bridges are in ACPI namespace, a
> pxm_to_node() lookup should be done there when the buses are discovered.
> Then it would work on HP and DIG boxes.  Thanks,

Could you do that? We only provide minimal ACPI support which does not 
include pci. For our needs this will not work.

Re: [PATCH] pcibus_to_node implementation for IA64

[Christoph Lameter]

On Thu, 23 Jun 2005, Jesse Barnes wrote:

> It looks like you're adding the new 'node' field unconditionally, but 
> protecting the sba assignment by CONFIG_NUMA?  Which one is wrong? :)

if !CONFIG_NUMA then the node field exists but is always -1, which will 
result in node unspecific allocations.

Re: [PATCH] pcibus_to_node implementation for IA64

[Alex Williamson]

On Thu, 2005-06-23 at 14:19 -0700, Christoph Lameter wrote:
> On Thu, 23 Jun 2005, Alex Williamson wrote:
> 
> >    This will work, but there's no reason to hide this in the platform
> > specific code.  The PCI root bridges are in ACPI namespace, a
> > pxm_to_node() lookup should be done there when the buses are discovered.
> > Then it would work on HP and DIG boxes.  Thanks,
> 
> Could you do that? We only provide minimal ACPI support which does not 
> include pci. For our needs this will not work.

   I believe the below (untested) will work.  I'll try to test it on a
box.  Thanks,

	Alex

-- 
Alex Williamson                             HP Linux & Open Source Lab


diff --git a/arch/ia64/pci/pci.c b/arch/ia64/pci/pci.c
--- a/arch/ia64/pci/pci.c
+++ b/arch/ia64/pci/pci.c
@@ -306,6 +306,7 @@ pci_acpi_scan_root(struct acpi_device *d
 	unsigned int windows = 0;
 	struct pci_bus *pbus;
 	char *name;
+	int pxm;
 
 	controller = alloc_pci_controller(domain);
 	if (!controller)
@@ -313,6 +314,10 @@ pci_acpi_scan_root(struct acpi_device *d
 
 	controller->acpi_handle = device->handle;
 
+	pxm = acpi_get_pxm(controller->acpi_handle);
+	if (pxm >= 0)
+		controller->node = pxm_to_nid_map[pxm];
+
 	acpi_walk_resources(device->handle, METHOD_NAME__CRS, count_window,
 			&windows);
 	controller->window = kmalloc(sizeof(*controller->window) * windows,

Re: [PATCH] pcibus_to_node implementation for IA64

[Christoph Lameter]

On Thu, 23 Jun 2005, Alex Williamson wrote:

>    I believe the below (untested) will work.  I'll try to test it on a
> box.  Thanks,

Yes that is how other arches do it.

Re: [PATCH] pcibus_to_node implementation for IA64

[Alex Williamson]

On Thu, 2005-06-23 at 14:47 -0700, Christoph Lameter wrote:
> On Thu, 23 Jun 2005, Alex Williamson wrote:
> 
> >    I believe the below (untested) will work.  I'll try to test it on a
> > box.  Thanks,
> 
> Yes that is how other arches do it.

  Cool.  It works on an rx8620 here.  I forgot to wrap the generic setup
in a NUMA config.  The below chunks should replace the sba_iommu chunk
in the original patch.  Thanks,

	Alex

Signed-off-by: Alex Williamson <alex.williamson@hp.com>

diff --git a/arch/ia64/pci/pci.c b/arch/ia64/pci/pci.c
--- a/arch/ia64/pci/pci.c
+++ b/arch/ia64/pci/pci.c
@@ -288,6 +289,7 @@ pci_acpi_scan_root(struct acpi_device *d
 	unsigned int windows = 0;
 	struct pci_bus *pbus;
 	char *name;
+	int pxm;
 
 	controller = alloc_pci_controller(domain);
 	if (!controller)
@@ -295,6 +297,12 @@ pci_acpi_scan_root(struct acpi_device *d
 
 	controller->acpi_handle = device->handle;
 
+	pxm = acpi_get_pxm(controller->acpi_handle);
+#ifdef CONFIG_NUMA
+	if (pxm >= 0)
+		controller->node = pxm_to_nid_map[pxm];
+#endif
+
 	acpi_walk_resources(device->handle, METHOD_NAME__CRS, count_window,
 			&windows);
 	controller->window = kmalloc(sizeof(*controller->window) * windows,

Re: [PATCH] pcibus_to_node implementation for IA64

[Christoph Lameter]

On Thu, 23 Jun 2005, Alex Williamson wrote:

>   Cool.  It works on an rx8620 here.  I forgot to wrap the generic setup
> in a NUMA config.  The below chunks should replace the sba_iommu chunk
> in the original patch.  Thanks,

Great. Tony I am going to split off my patch into two pieces, one generic 
and one Altix specific. Alex's patch implements the HP subarch support 
and the patch that I am going to send separately will implement Altix 
subarch support.

Here is the generic stuff for ia64 pcibus to node support:

----

[PATCH] pcibus_to_node implementation for IA64

pcibus_to_node provides a way for the Linux kernel to identify to which node
a certain pcibus connect to. Allocations of control structures for devices can then be
made on the node where the pci bus is located to allow local access during interrupt and
other device manipulation.

The numa aware block layer patch now in Linus tree relies on pcibus_to_node
to allocate block device control structures on the correct node. There is a pending
patch for an ethernet driver to allocate the descriptors on the node of each ethernet device.

This patch only provides an extension of the the pci_controller structure and will make
pcibus_to_node return -1 (undetermined) by default. Separate patches for HP and SGI will
be needed to make subarches return useful node information.

Signed-off-by: Christoph Lameter <clameter@sgi.com>

Index: linux-2.6.12/include/asm-ia64/pci.h
=================================--- linux-2.6.12.orig/include/asm-ia64/pci.h	2005-06-17 12:48:29.000000000 -0700
+++ linux-2.6.12/include/asm-ia64/pci.h	2005-06-23 15:57:26.000000000 -0700
@@ -109,6 +109,7 @@ struct pci_controller {
 	void *acpi_handle;
 	void *iommu;
 	int segment;
+	int node;		/* nearest node with memory or -1 for global allocation */
 
 	unsigned int windows;
 	struct pci_window *window;
Index: linux-2.6.12/include/asm-ia64/topology.h
=================================--- linux-2.6.12.orig/include/asm-ia64/topology.h	2005-06-17 12:48:29.000000000 -0700
+++ linux-2.6.12/include/asm-ia64/topology.h	2005-06-23 15:57:26.000000000 -0700
@@ -40,6 +40,11 @@
  */
 #define node_to_first_cpu(node) (__ffs(node_to_cpumask(node)))
 
+/*
+ * Determines the node for a given pci bus
+ */
+#define pcibus_to_node(bus) PCI_CONTROLLER(bus)->node
+
 void build_cpu_to_node_map(void);
 
 /* sched_domains SD_NODE_INIT for IA64 NUMA machines */
Index: linux-2.6.12/arch/ia64/pci/pci.c
=================================--- linux-2.6.12.orig/arch/ia64/pci/pci.c	2005-06-23 15:53:53.000000000 -0700
+++ linux-2.6.12/arch/ia64/pci/pci.c	2005-06-23 15:57:26.000000000 -0700
@@ -157,6 +157,7 @@ alloc_pci_controller (int seg)
 
 	memset(controller, 0, sizeof(*controller));
 	controller->segment = seg;
+	controller->node = -1;
 	return controller;
 }
 

Re: [PATCH] pcibus_to_node implementation for IA64

[Jack Steiner]

> Index: linux-2.6.12/arch/ia64/sn/kernel/io_init.c
> =================================> --- linux-2.6.12.orig/arch/ia64/sn/kernel/io_init.c	2005-06-17 12:48:29.000000000 -0700
> +++ linux-2.6.12/arch/ia64/sn/kernel/io_init.c	2005-06-23 13:55:02.000000000 -0700
> @@ -364,6 +364,8 @@ static void sn_pci_controller_fixup(int 
>  
>  	nasid = NASID_GET(SN_PCIBUS_BUSSOFT(bus)->bs_base);
>  	cnode = nasid_to_cnodeid(nasid);
> +	if (cnode < num_online_nodes() && cnode < numionodes)
> +		controller->node = cnode;

This doesn't look right. 

The code works for normal nodes (not TIO). For TIO nodes, the node number that
is returned by physical_node_map() is NOT a node number that is known to 
VM. 


>  	hubdev_info = (struct hubdev_info *)(NODEPDA(cnode)->pdinfo);
>  	SN_PCIBUS_BUSSOFT(bus)->bs_xwidget_info >  	    &(hubdev_info->hdi_xwidget_info[SN_PCIBUS_BUSSOFT(bus)->bs_xid]);

-- 
Thanks

Jack Steiner (steiner@sgi.com)          651-683-5302
Principal Engineer                      SGI - Silicon Graphics, Inc.

Re: [PATCH] pcibus_to_node implementation for IA64

[Christoph Lameter]

On Thu, 23 Jun 2005, Jack Steiner wrote:

> This doesn't look right. 

Yup, I figured that out too ..

Could you have a look at the latest generic patch posted to 
linux-ia64 and at the last patch posted to the PV?

[PATCH] pcibus_to_node implementation for IA64

[Christoph Lameter]

pcibus_to_node provides a way for the Linux kernel to identify to which node
a certain pcibus connects to. Allocations of control structures for devices can then be
made on the node where the pci bus is located to allow local access during interrupt and
other device manipulation.

This patch provides a new "node" field in the the pci_controller 
structure. The node field will be set based on ACPI information (thanks to 
Alex Williamson  <alex.williamson@hp.com for that piece).

Signed-off-by: Christoph Lameter <clameter@sgi.com>

Index: linux-2.6.13-rc2/include/asm-ia64/pci.h
=================================--- linux-2.6.13-rc2.orig/include/asm-ia64/pci.h	2005-07-05 20:46:33.000000000 -0700
+++ linux-2.6.13-rc2/include/asm-ia64/pci.h	2005-07-06 12:20:02.000000000 -0700
@@ -128,6 +128,7 @@ struct pci_controller {
 	void *acpi_handle;
 	void *iommu;
 	int segment;
+	int node;		/* nearest node with memory or -1 for global allocation */
 
 	unsigned int windows;
 	struct pci_window *window;
Index: linux-2.6.13-rc2/include/asm-ia64/topology.h
=================================--- linux-2.6.13-rc2.orig/include/asm-ia64/topology.h	2005-07-05 20:46:33.000000000 -0700
+++ linux-2.6.13-rc2/include/asm-ia64/topology.h	2005-07-06 12:20:02.000000000 -0700
@@ -40,6 +40,11 @@
  */
 #define node_to_first_cpu(node) (__ffs(node_to_cpumask(node)))
 
+/*
+ * Determines the node for a given pci bus
+ */
+#define pcibus_to_node(bus) PCI_CONTROLLER(bus)->node
+
 void build_cpu_to_node_map(void);
 
 #define SD_CPU_INIT (struct sched_domain) {		\
Index: linux-2.6.13-rc2/arch/ia64/pci/pci.c
=================================--- linux-2.6.13-rc2.orig/arch/ia64/pci/pci.c	2005-07-05 20:46:33.000000000 -0700
+++ linux-2.6.13-rc2/arch/ia64/pci/pci.c	2005-07-07 16:35:27.000000000 -0700
@@ -157,6 +157,7 @@ alloc_pci_controller (int seg)
 
 	memset(controller, 0, sizeof(*controller));
 	controller->segment = seg;
+	controller->node = -1;
 	return controller;
 }
 
@@ -288,6 +289,7 @@ pci_acpi_scan_root(struct acpi_device *d
 	unsigned int windows = 0;
 	struct pci_bus *pbus;
 	char *name;
+	int pxm;
 
 	controller = alloc_pci_controller(domain);
 	if (!controller)
@@ -295,10 +297,16 @@ pci_acpi_scan_root(struct acpi_device *d
 
 	controller->acpi_handle = device->handle;
 
+	pxm = acpi_get_pxm(controller->acpi_handle);
+#ifdef CONFIG_NUMA
+	if (pxm >= 0)
+		controller->node = pxm_to_nid_map[pxm];
+#endif
+
 	acpi_walk_resources(device->handle, METHOD_NAME__CRS, count_window,
 			&windows);
-	controller->window = kmalloc(sizeof(*controller->window) * windows,
-			GFP_KERNEL);
+	controller->window = kmalloc_node(sizeof(*controller->window) * windows,
+			GFP_KERNEL, controller->node);
 	if (!controller->window)
 		goto out2;