[PATCH 1/2] PCI: Split pci_assign_unassigned_resources to per root bus

[PATCH 1/2] PCI: Split pci_assign_unassigned_resources to per root bus

[Yinghai Lu]

BenH reported that there is some assign unassigned resource problem
in powerpc.

It turns out after
| commit 0c5be0cb0edfe3b5c4b62eac68aa2aa15ec681af
| Date:   Thu Feb 23 19:23:29 2012 -0800
|
|    PCI: Retry on IORESOURCE_IO type allocations

even the root bus does not have io port range, it will keep retrying
to realloc with mmio.

Current retry logic is : try with must+optional at first, and if
it fails will try must then try to extend must with optional.
That will fail as mmio-non-pref and mmio-pref for bridge will
be next to each other. So we have no chance to extend mmio-non-pref.

We should not fall into retry in this case, as root bus does
not io port range.

Before we do that we need to split pci_assign_unassiged_resource
to every root bus, so we can stop early for root bus without ioport
range, and still continue to retry on buses that do have ioport range.

This will be become more often when we have x86 8 sockets or 32 sockets
system, and those system will have one root bus per socket.
They will have some root buses do not have ioport range.

For the retry failing, we could allocate mmio-non-pref bottom-up
and mmio-pref will be top-down, but that could not be material for v3.10.

Reported-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |  101 +++++++++++++++++++++++-------------------------
 1 file changed, 49 insertions(+), 52 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1315,21 +1315,6 @@ static int __init pci_bus_get_depth(stru
 
 	return depth;
 }
-static int __init pci_get_max_depth(void)
-{
-	int depth = 0;
-	struct pci_bus *bus;
-
-	list_for_each_entry(bus, &pci_root_buses, node) {
-		int ret;
-
-		ret = pci_bus_get_depth(bus);
-		if (ret > depth)
-			depth = ret;
-	}
-
-	return depth;
-}
 
 /*
  * -1: undefined, will auto detect later
@@ -1354,34 +1339,41 @@ void __init pci_realloc_get_opt(char *st
 	else if (!strncmp(str, "on", 2))
 		pci_realloc_enable = user_enabled;
 }
-static bool __init pci_realloc_enabled(void)
+static bool __init pci_realloc_enabled(enum enable_type enable)
 {
-	return pci_realloc_enable >= user_enabled;
+	return enable >= user_enabled;
 }
 
-static void __init pci_realloc_detect(void)
+static enum enable_type __init pci_realloc_detect(struct pci_bus *bus,
+			 enum enable_type enable_local)
 {
 #if defined(CONFIG_PCI_IOV) && defined(CONFIG_PCI_REALLOC_ENABLE_AUTO)
-	struct pci_dev *dev = NULL;
+	struct pci_dev *dev;
 
-	if (pci_realloc_enable != undefined)
-		return;
+	if (enable_local != undefined)
+		return enable_local;
 
-	for_each_pci_dev(dev) {
+	list_for_each_entry(dev, &bus->devices, bus_list) {
 		int i;
 
 		for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++) {
 			struct resource *r = &dev->resource[i];
 
 			/* Not assigned, or rejected by kernel ? */
-			if (r->flags && !r->start) {
-				pci_realloc_enable = auto_enabled;
-
-				return;
-			}
+			if (r->flags && !r->start)
+				return auto_enabled;
 		}
 	}
+
+	list_for_each_entry(dev, &bus->devices, bus_list) {
+		struct pci_bus *child = dev->subordinate;
+
+		if (child &&
+		    pci_realloc_detect(child, enable_local) == auto_enabled)
+			return auto_enabled;
+	}
 #endif
+	return enable_local;
 }
 
 /*
@@ -1389,10 +1381,9 @@ static void __init pci_realloc_detect(vo
  * second  and later try will clear small leaf bridge res
  * will stop till to the max  deepth if can not find good one
  */
-void __init
-pci_assign_unassigned_resources(void)
+static void __init
+pci_assign_unassigned_root_bus_resources(struct pci_bus *bus)
 {
-	struct pci_bus *bus;
 	LIST_HEAD(realloc_head); /* list of resources that
 					want additional resources */
 	struct list_head *add_list = NULL;
@@ -1403,15 +1394,17 @@ pci_assign_unassigned_resources(void)
 	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
 				  IORESOURCE_PREFETCH;
 	int pci_try_num = 1;
+	enum enable_type enable_local;
 
 	/* don't realloc if asked to do so */
-	pci_realloc_detect();
-	if (pci_realloc_enabled()) {
-		int max_depth = pci_get_max_depth();
+	enable_local = pci_realloc_detect(bus, pci_realloc_enable);
+	if (pci_realloc_enabled(enable_local)) {
+		int max_depth = pci_bus_get_depth(bus);
 
 		pci_try_num = max_depth + 1;
-		printk(KERN_DEBUG "PCI: max bus depth: %d pci_try_num: %d\n",
-			 max_depth, pci_try_num);
+		dev_printk(KERN_DEBUG, &bus->dev,
+			   "max bus depth: %d pci_try_num: %d\n",
+			   max_depth, pci_try_num);
 	}
 
 again:
@@ -1423,12 +1416,10 @@ again:
 		add_list = &realloc_head;
 	/* Depth first, calculate sizes and alignments of all
 	   subordinate buses. */
-	list_for_each_entry(bus, &pci_root_buses, node)
-		__pci_bus_size_bridges(bus, add_list);
+	__pci_bus_size_bridges(bus, add_list);
 
 	/* Depth last, allocate resources and update the hardware. */
-	list_for_each_entry(bus, &pci_root_buses, node)
-		__pci_bus_assign_resources(bus, add_list, &fail_head);
+	__pci_bus_assign_resources(bus, add_list, &fail_head);
 	if (add_list)
 		BUG_ON(!list_empty(add_list));
 	tried_times++;
@@ -1438,17 +1429,17 @@ again:
 		goto enable_and_dump;
 
 	if (tried_times >= pci_try_num) {
-		if (pci_realloc_enable == undefined)
-			printk(KERN_INFO "Some PCI device resources are unassigned, try booting with pci=realloc\n");
-		else if (pci_realloc_enable == auto_enabled)
-			printk(KERN_INFO "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n");
+		if (enable_local == undefined)
+			dev_info(&bus->dev, "Some PCI device resources are unassigned, try booting with pci=realloc\n");
+		else if (enable_local == auto_enabled)
+			dev_info(&bus->dev, "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n");
 
 		free_list(&fail_head);
 		goto enable_and_dump;
 	}
 
-	printk(KERN_DEBUG "PCI: No. %d try to assign unassigned res\n",
-			 tried_times + 1);
+	dev_printk(KERN_DEBUG, &bus->dev,
+		   "No. %d try to assign unassigned res\n", tried_times + 1);
 
 	/* third times and later will not check if it is leaf */
 	if ((tried_times + 1) > 2)
@@ -1458,12 +1449,11 @@ again:
 	 * Try to release leaf bridge's resources that doesn't fit resource of
 	 * child device under that bridge
 	 */
-	list_for_each_entry(fail_res, &fail_head, list) {
-		bus = fail_res->dev->bus;
-		pci_bus_release_bridge_resources(bus,
+	list_for_each_entry(fail_res, &fail_head, list)
+		pci_bus_release_bridge_resources(fail_res->dev->bus,
 						 fail_res->flags & type_mask,
 						 rel_type);
-	}
+
 	/* restore size and flags */
 	list_for_each_entry(fail_res, &fail_head, list) {
 		struct resource *res = fail_res->res;
@@ -1480,12 +1470,19 @@ again:
 
 enable_and_dump:
 	/* Depth last, update the hardware. */
-	list_for_each_entry(bus, &pci_root_buses, node)
-		pci_enable_bridges(bus);
+	pci_enable_bridges(bus);
 
 	/* dump the resource on buses */
+	pci_bus_dump_resources(bus);
+}
+
+void __init
+pci_assign_unassigned_resources(void)
+{
+	struct pci_bus *bus;
+
 	list_for_each_entry(bus, &pci_root_buses, node)
-		pci_bus_dump_resources(bus);
+		pci_assign_unassigned_root_bus_resources(bus);
 }
 
 void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge)

[PATCH 2/2] PCI: Skip IORESOURCE_IO size and allocation for root bus without ioport range

[Yinghai Lu]

BenH reported that there is some assign unassigned resource problem
in powerpc.

It turns out after
| commit 0c5be0cb0edfe3b5c4b62eac68aa2aa15ec681af
| Date:   Thu Feb 23 19:23:29 2012 -0800
|
|    PCI: Retry on IORESOURCE_IO type allocations

even the root bus does not have io port range, it will keep retrying
to realloc with mmio.

Current retry logic is : try with must+optional at first, and if
it fails will try must then try to extend must with optional.
That will fail as mmio-non-pref and mmio-pref for bridge will
be next to each other. So we have no chance to extend mmio-non-pref.

We should not fall into retry in this case, as root bus does
not io port range.

We check if the root bus has ioport range, and set bus_res_type_mask,
and pass it to __bus_size_bridges and skip io port resources.

For the retry failing, we could allocate mmio-non-pref bottom-up
and mmio-pref will be top-down, but that could not be material for v3.10.

Reported-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |   39 +++++++++++++++++++++++++++++++--------
 1 file changed, 31 insertions(+), 8 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1045,7 +1045,8 @@ handle_done:
 }
 
 static void __ref __pci_bus_size_bridges(struct pci_bus *bus,
-			struct list_head *realloc_head)
+			struct list_head *realloc_head,
+			unsigned long res_type_mask)
 {
 	struct pci_dev *dev;
 	unsigned long mask, prefmask;
@@ -1063,7 +1064,7 @@ static void __ref __pci_bus_size_bridges
 
 		case PCI_CLASS_BRIDGE_PCI:
 		default:
-			__pci_bus_size_bridges(b, realloc_head);
+			__pci_bus_size_bridges(b, realloc_head, res_type_mask);
 			break;
 		}
 	}
@@ -1087,8 +1088,9 @@ static void __ref __pci_bus_size_bridges
 		 * Follow thru
 		 */
 	default:
-		pbus_size_io(bus, realloc_head ? 0 : additional_io_size,
-			     additional_io_size, realloc_head);
+		if (res_type_mask & IORESOURCE_IO)
+			pbus_size_io(bus, realloc_head ? 0 : additional_io_size,
+				     additional_io_size, realloc_head);
 		/* If the bridge supports prefetchable range, size it
 		   separately. If it doesn't, or its prefetchable window
 		   has already been allocated by arch code, try
@@ -1111,7 +1113,10 @@ static void __ref __pci_bus_size_bridges
 
 void __ref pci_bus_size_bridges(struct pci_bus *bus)
 {
-	__pci_bus_size_bridges(bus, NULL);
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	__pci_bus_size_bridges(bus, NULL, type_mask);
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
@@ -1376,6 +1381,21 @@ static enum enable_type __init pci_reall
 	return enable_local;
 }
 
+static unsigned int __init pci_bus_res_type_mask(struct pci_bus *bus)
+{
+	int i;
+	struct resource *r;
+	unsigned long mask = 0;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	pci_bus_for_each_resource(bus, r, i)
+		if (r)
+			mask |= r->flags & type_mask;
+
+	return mask;
+}
+
 /*
  * first try will not touch pci bridge res
  * second  and later try will clear small leaf bridge res
@@ -1395,6 +1415,7 @@ pci_assign_unassigned_root_bus_resources
 				  IORESOURCE_PREFETCH;
 	int pci_try_num = 1;
 	enum enable_type enable_local;
+	unsigned long bus_res_type_mask = pci_bus_res_type_mask(bus);
 
 	/* don't realloc if asked to do so */
 	enable_local = pci_realloc_detect(bus, pci_realloc_enable);
@@ -1416,7 +1437,7 @@ again:
 		add_list = &realloc_head;
 	/* Depth first, calculate sizes and alignments of all
 	   subordinate buses. */
-	__pci_bus_size_bridges(bus, add_list);
+	__pci_bus_size_bridges(bus, add_list, bus_res_type_mask);
 
 	/* Depth last, allocate resources and update the hardware. */
 	__pci_bus_assign_resources(bus, add_list, &fail_head);
@@ -1496,9 +1517,10 @@ void pci_assign_unassigned_bridge_resour
 	int retval;
 	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
 				  IORESOURCE_PREFETCH;
+	unsigned long bus_res_type_mask = pci_bus_res_type_mask(bridge->bus);
 
 again:
-	__pci_bus_size_bridges(parent, &add_list);
+	__pci_bus_size_bridges(parent, &add_list, bus_res_type_mask);
 	__pci_bridge_assign_resources(bridge, &add_list, &fail_head);
 	BUG_ON(!list_empty(&add_list));
 	tried_times++;
@@ -1554,6 +1576,7 @@ void pci_assign_unassigned_bus_resources
 	struct pci_dev *dev;
 	LIST_HEAD(add_list); /* list of resources that
 					want additional resources */
+	unsigned long bus_res_type_mask = pci_bus_res_type_mask(bus);
 
 	down_read(&pci_bus_sem);
 	list_for_each_entry(dev, &bus->devices, bus_list)
@@ -1561,7 +1584,7 @@ void pci_assign_unassigned_bus_resources
 		    dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)
 			if (dev->subordinate)
 				__pci_bus_size_bridges(dev->subordinate,
-							 &add_list);
+						 &add_list, bus_res_type_mask);
 	up_read(&pci_bus_sem);
 	__pci_bus_assign_resources(bus, &add_list, NULL);
 	BUG_ON(!list_empty(&add_list));

Re: [PATCH 1/2] PCI: Split pci_assign_unassigned_resources to per root bus

[Benjamin Herrenschmidt]

On Mon, 2013-05-06 at 16:15 -0700, Yinghai Lu wrote:
> BenH reported that there is some assign unassigned resource problem
> in powerpc.
> 
> It turns out after
> | commit 0c5be0cb0edfe3b5c4b62eac68aa2aa15ec681af
> | Date:   Thu Feb 23 19:23:29 2012 -0800
> |
> |    PCI: Retry on IORESOURCE_IO type allocations
> 
> even the root bus does not have io port range, it will keep retrying
> to realloc with mmio.
> 
> Current retry logic is : try with must+optional at first, and if
> it fails will try must then try to extend must with optional.
> That will fail as mmio-non-pref and mmio-pref for bridge will
> be next to each other. So we have no chance to extend mmio-non-pref.

Shouldn't you do completely separate passes for IO and MMIO through the
whole lot instead since they are completely separate address spaces ?

IE. Even a legit setup with IOs, in case of failures on the IO side
(resource exhaustion, FW bugs, ...) we shouldn't retry the memory side
and vice-versa.

Note that I'm a bit worried by how invasive your proposed patch is. I
will let Gavin test it and if it works I'm happy but keep in mind that I
really need that fixed in 3.10 ... (especially since there are noises
around about a certain enterprise distro using that version...)

Cheers,
Ben.

Re: [PATCH 1/2] PCI: Split pci_assign_unassigned_resources to per root bus

[Yinghai Lu]

[-- Attachment #1: Type: text/plain, Size: 299 bytes --]

On Mon, May 6, 2013 at 6:33 PM, Gavin Shan <shangw@linux.vnet.ibm.com> wrote:
>
> Run the kernel with Yinghai's code on the simulator and it's failed to
> enable PCI devices (including p2p bridges) as the attached kernel log
> indicates.

Please check attached v2 for second patch.

Thanks

Yinghai

[-- Attachment #2: root_bus_ioport_skip_2.patch --]
[-- Type: application/octet-stream, Size: 7451 bytes --]

Subject: [PATCH 2/2] PCI: Skip IORESOURCE_IO size and allocation for root bus without ioport range

BenH reported that there is some assign unassigned resource problem
in powerpc.

It turns out after
| commit 0c5be0cb0edfe3b5c4b62eac68aa2aa15ec681af
| Date:   Thu Feb 23 19:23:29 2012 -0800
|
|    PCI: Retry on IORESOURCE_IO type allocations

even the root bus does not have io port range, it will keep retrying
to realloc with mmio.

Current retry logic is : try with must+optional at first, and if
it fails will try must then try to extend must with optional.
That will fail as mmio-non-pref and mmio-pref for bridge will
be next to each other. So we have no chance to extend mmio-non-pref.

We should not fall into retry in this case, as root bus does
not io port range.

We check if the root bus has ioport range, and set bus_res_type_mask,
and pass it to __bus_size_bridges and skip io port resources.

For the retry failing, we could allocate mmio-non-pref bottom-up
and mmio-pref will be top-down, but that could not be material for v3.10.

-v2: remove wrong __init with pci_bus_res_type_mask()
     don't check bridge size/flags, and clear bus io resources.

Reported-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |   81 +++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 64 insertions(+), 17 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -586,7 +586,8 @@ void pci_setup_bridge(struct pci_bus *bu
 /* Check whether the bridge supports optional I/O and
    prefetchable memory ranges. If not, the respective
    base/limit registers must be read-only and read as 0. */
-static void pci_bridge_check_ranges(struct pci_bus *bus)
+static void pci_bridge_check_ranges(struct pci_bus *bus,
+				    unsigned long res_type_mask)
 {
 	u16 io;
 	u32 pmem;
@@ -596,14 +597,17 @@ static void pci_bridge_check_ranges(stru
 	b_res = &bridge->resource[PCI_BRIDGE_RESOURCES];
 	b_res[1].flags |= IORESOURCE_MEM;
 
-	pci_read_config_word(bridge, PCI_IO_BASE, &io);
-	if (!io) {
-		pci_write_config_word(bridge, PCI_IO_BASE, 0xf0f0);
+	if (res_type_mask & IORESOURCE_IO) {
 		pci_read_config_word(bridge, PCI_IO_BASE, &io);
- 		pci_write_config_word(bridge, PCI_IO_BASE, 0x0);
- 	}
- 	if (io)
-		b_res[0].flags |= IORESOURCE_IO;
+		if (!io) {
+			pci_write_config_word(bridge, PCI_IO_BASE, 0xf0f0);
+			pci_read_config_word(bridge, PCI_IO_BASE, &io);
+			pci_write_config_word(bridge, PCI_IO_BASE, 0x0);
+		}
+		if (io)
+			b_res[0].flags |= IORESOURCE_IO;
+	}
+
 	/*  DECchip 21050 pass 2 errata: the bridge may miss an address
 	    disconnect boundary by one PCI data phase.
 	    Workaround: do not use prefetching on this device. */
@@ -812,6 +816,24 @@ static void pbus_size_io(struct pci_bus
 	}
 }
 
+static void pbus_clear_io(struct pci_bus *bus)
+{
+	struct pci_dev *dev;
+
+	list_for_each_entry(dev, &bus->devices, bus_list) {
+		int i;
+
+		for (i = 0; i < PCI_NUM_RESOURCES; i++) {
+			struct resource *r = &dev->resource[i];
+
+			if (r->parent || !(r->flags & IORESOURCE_IO))
+				continue;
+
+			reset_resource(r);
+		}
+	}
+}
+
 static inline resource_size_t calculate_mem_align(resource_size_t *aligns,
 						  int max_order)
 {
@@ -1045,7 +1067,8 @@ handle_done:
 }
 
 static void __ref __pci_bus_size_bridges(struct pci_bus *bus,
-			struct list_head *realloc_head)
+			struct list_head *realloc_head,
+			unsigned long res_type_mask)
 {
 	struct pci_dev *dev;
 	unsigned long mask, prefmask;
@@ -1063,7 +1086,7 @@ static void __ref __pci_bus_size_bridges
 
 		case PCI_CLASS_BRIDGE_PCI:
 		default:
-			__pci_bus_size_bridges(b, realloc_head);
+			__pci_bus_size_bridges(b, realloc_head, res_type_mask);
 			break;
 		}
 	}
@@ -1078,7 +1101,7 @@ static void __ref __pci_bus_size_bridges
 		break;
 
 	case PCI_CLASS_BRIDGE_PCI:
-		pci_bridge_check_ranges(bus);
+		pci_bridge_check_ranges(bus, res_type_mask);
 		if (bus->self->is_hotplug_bridge) {
 			additional_io_size  = pci_hotplug_io_size;
 			additional_mem_size = pci_hotplug_mem_size;
@@ -1087,8 +1110,11 @@ static void __ref __pci_bus_size_bridges
 		 * Follow thru
 		 */
 	default:
-		pbus_size_io(bus, realloc_head ? 0 : additional_io_size,
-			     additional_io_size, realloc_head);
+		if (res_type_mask & IORESOURCE_IO)
+			pbus_size_io(bus, realloc_head ? 0 : additional_io_size,
+				     additional_io_size, realloc_head);
+		else
+			pbus_clear_io(bus);
 		/* If the bridge supports prefetchable range, size it
 		   separately. If it doesn't, or its prefetchable window
 		   has already been allocated by arch code, try
@@ -1111,7 +1137,10 @@ static void __ref __pci_bus_size_bridges
 
 void __ref pci_bus_size_bridges(struct pci_bus *bus)
 {
-	__pci_bus_size_bridges(bus, NULL);
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	__pci_bus_size_bridges(bus, NULL, type_mask);
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
@@ -1376,6 +1405,21 @@ static enum enable_type __init pci_reall
 	return enable_local;
 }
 
+static unsigned long pci_bus_res_type_mask(struct pci_bus *bus)
+{
+	int i;
+	struct resource *r;
+	unsigned long mask = 0;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	pci_bus_for_each_resource(bus, r, i)
+		if (r)
+			mask |= r->flags & type_mask;
+
+	return mask;
+}
+
 /*
  * first try will not touch pci bridge res
  * second  and later try will clear small leaf bridge res
@@ -1395,6 +1439,7 @@ pci_assign_unassigned_root_bus_resources
 				  IORESOURCE_PREFETCH;
 	int pci_try_num = 1;
 	enum enable_type enable_local;
+	unsigned long bus_res_type_mask = pci_bus_res_type_mask(bus);
 
 	/* don't realloc if asked to do so */
 	enable_local = pci_realloc_detect(bus, pci_realloc_enable);
@@ -1416,7 +1461,7 @@ again:
 		add_list = &realloc_head;
 	/* Depth first, calculate sizes and alignments of all
 	   subordinate buses. */
-	__pci_bus_size_bridges(bus, add_list);
+	__pci_bus_size_bridges(bus, add_list, bus_res_type_mask);
 
 	/* Depth last, allocate resources and update the hardware. */
 	__pci_bus_assign_resources(bus, add_list, &fail_head);
@@ -1496,9 +1541,10 @@ void pci_assign_unassigned_bridge_resour
 	int retval;
 	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
 				  IORESOURCE_PREFETCH;
+	unsigned long bus_res_type_mask = pci_bus_res_type_mask(bridge->bus);
 
 again:
-	__pci_bus_size_bridges(parent, &add_list);
+	__pci_bus_size_bridges(parent, &add_list, bus_res_type_mask);
 	__pci_bridge_assign_resources(bridge, &add_list, &fail_head);
 	BUG_ON(!list_empty(&add_list));
 	tried_times++;
@@ -1554,6 +1600,7 @@ void pci_assign_unassigned_bus_resources
 	struct pci_dev *dev;
 	LIST_HEAD(add_list); /* list of resources that
 					want additional resources */
+	unsigned long bus_res_type_mask = pci_bus_res_type_mask(bus);
 
 	down_read(&pci_bus_sem);
 	list_for_each_entry(dev, &bus->devices, bus_list)
@@ -1561,7 +1608,7 @@ void pci_assign_unassigned_bus_resources
 		    dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)
 			if (dev->subordinate)
 				__pci_bus_size_bridges(dev->subordinate,
-							 &add_list);
+						 &add_list, bus_res_type_mask);
 	up_read(&pci_bus_sem);
 	__pci_bus_assign_resources(bus, &add_list, NULL);
 	BUG_ON(!list_empty(&add_list));

Re: [PATCH 1/2] PCI: Split pci_assign_unassigned_resources to per root bus

[Bjorn Helgaas]

On Mon, May 06, 2013 at 04:15:29PM -0700, Yinghai Lu wrote:
> BenH reported that there is some assign unassigned resource problem
> in powerpc.
> 
> It turns out after
> | commit 0c5be0cb0edfe3b5c4b62eac68aa2aa15ec681af
> | Date:   Thu Feb 23 19:23:29 2012 -0800
> |
> |    PCI: Retry on IORESOURCE_IO type allocations
> 
> even the root bus does not have io port range, it will keep retrying
> to realloc with mmio.
> 
> Current retry logic is : try with must+optional at first, and if
> it fails will try must then try to extend must with optional.
> That will fail as mmio-non-pref and mmio-pref for bridge will
> be next to each other. So we have no chance to extend mmio-non-pref.
> 
> We should not fall into retry in this case, as root bus does
> not io port range.
> 
> Before we do that we need to split pci_assign_unassiged_resource
> to every root bus, so we can stop early for root bus without ioport
> range, and still continue to retry on buses that do have ioport range.
> 
> This will be become more often when we have x86 8 sockets or 32 sockets
> system, and those system will have one root bus per socket.
> They will have some root buses do not have ioport range.
> 
> For the retry failing, we could allocate mmio-non-pref bottom-up
> and mmio-pref will be top-down, but that could not be material for v3.10.

If I understand correctly, this particular patch makes no functional
changes, so the changelog above should be saved for the patches that *do*
actually fix problems.

> 
> Reported-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
> Signed-off-by: Yinghai Lu <yinghai@kernel.org>
> 
> ---
>  drivers/pci/setup-bus.c |  101 +++++++++++++++++++++++-------------------------
>  1 file changed, 49 insertions(+), 52 deletions(-)
> 
> Index: linux-2.6/drivers/pci/setup-bus.c
> ===================================================================
> --- linux-2.6.orig/drivers/pci/setup-bus.c
> +++ linux-2.6/drivers/pci/setup-bus.c
> @@ -1315,21 +1315,6 @@ static int __init pci_bus_get_depth(stru
>  
>  	return depth;
>  }
> -static int __init pci_get_max_depth(void)
> -{
> -	int depth = 0;
> -	struct pci_bus *bus;
> -
> -	list_for_each_entry(bus, &pci_root_buses, node) {
> -		int ret;
> -
> -		ret = pci_bus_get_depth(bus);
> -		if (ret > depth)
> -			depth = ret;
> -	}
> -
> -	return depth;
> -}
>  
>  /*
>   * -1: undefined, will auto detect later
> @@ -1354,34 +1339,41 @@ void __init pci_realloc_get_opt(char *st
>  	else if (!strncmp(str, "on", 2))
>  		pci_realloc_enable = user_enabled;
>  }
> -static bool __init pci_realloc_enabled(void)
> +static bool __init pci_realloc_enabled(enum enable_type enable)
>  {
> -	return pci_realloc_enable >= user_enabled;
> +	return enable >= user_enabled;
>  }
>  
> -static void __init pci_realloc_detect(void)
> +static enum enable_type __init pci_realloc_detect(struct pci_bus *bus,
> +			 enum enable_type enable_local)
>  {
>  #if defined(CONFIG_PCI_IOV) && defined(CONFIG_PCI_REALLOC_ENABLE_AUTO)
> -	struct pci_dev *dev = NULL;
> +	struct pci_dev *dev;
>  
> -	if (pci_realloc_enable != undefined)
> -		return;
> +	if (enable_local != undefined)
> +		return enable_local;
>  
> -	for_each_pci_dev(dev) {
> +	list_for_each_entry(dev, &bus->devices, bus_list) {
>  		int i;
>  
>  		for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++) {
>  			struct resource *r = &dev->resource[i];
>  
>  			/* Not assigned, or rejected by kernel ? */
> -			if (r->flags && !r->start) {
> -				pci_realloc_enable = auto_enabled;
> -
> -				return;
> -			}
> +			if (r->flags && !r->start)
> +				return auto_enabled;
>  		}
>  	}
> +
> +	list_for_each_entry(dev, &bus->devices, bus_list) {
> +		struct pci_bus *child = dev->subordinate;
> +
> +		if (child &&
> +		    pci_realloc_detect(child, enable_local) == auto_enabled)
> +			return auto_enabled;
> +	}

This uses recursion and basically does the same thing as pci_walk_bus().
I think it will be clearer if you make it look something like this:

        static int count_unassigned_resources(struct pci_dev *dev, void *data)
        {
          int *count = data;

          for (i = PCI_IOV_RESOURCES; ...)
            if (r->flags && !r->start)
              *count++;

          return 0;
        }

        static pci_realloc_detect(struct pci_bus *bus, ...  enable_local)
        {
	  int unassigned;

          if (enable_local != undefined)
            return enable_local;

	  unassigned = 0;
	  pci_walk_bus(bus, count_unassigned_resources, &unassigned);
	  if (unassigned)
	    return auto_enabled;

          return enable_local;
        }


>  #endif
> +	return enable_local;
>  }
>  
>  /*
> @@ -1389,10 +1381,9 @@ static void __init pci_realloc_detect(vo
>   * second  and later try will clear small leaf bridge res
>   * will stop till to the max  deepth if can not find good one
>   */
> -void __init
> -pci_assign_unassigned_resources(void)
> +static void __init
> +pci_assign_unassigned_root_bus_resources(struct pci_bus *bus)
>  {
> -	struct pci_bus *bus;
>  	LIST_HEAD(realloc_head); /* list of resources that
>  					want additional resources */
>  	struct list_head *add_list = NULL;
> @@ -1403,15 +1394,17 @@ pci_assign_unassigned_resources(void)
>  	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
>  				  IORESOURCE_PREFETCH;
>  	int pci_try_num = 1;
> +	enum enable_type enable_local;
>  
>  	/* don't realloc if asked to do so */
> -	pci_realloc_detect();
> -	if (pci_realloc_enabled()) {
> -		int max_depth = pci_get_max_depth();
> +	enable_local = pci_realloc_detect(bus, pci_realloc_enable);
> +	if (pci_realloc_enabled(enable_local)) {
> +		int max_depth = pci_bus_get_depth(bus);
>  
>  		pci_try_num = max_depth + 1;
> -		printk(KERN_DEBUG "PCI: max bus depth: %d pci_try_num: %d\n",
> -			 max_depth, pci_try_num);
> +		dev_printk(KERN_DEBUG, &bus->dev,
> +			   "max bus depth: %d pci_try_num: %d\n",
> +			   max_depth, pci_try_num);
>  	}
>  
>  again:
> @@ -1423,12 +1416,10 @@ again:
>  		add_list = &realloc_head;
>  	/* Depth first, calculate sizes and alignments of all
>  	   subordinate buses. */
> -	list_for_each_entry(bus, &pci_root_buses, node)
> -		__pci_bus_size_bridges(bus, add_list);
> +	__pci_bus_size_bridges(bus, add_list);
>  
>  	/* Depth last, allocate resources and update the hardware. */
> -	list_for_each_entry(bus, &pci_root_buses, node)
> -		__pci_bus_assign_resources(bus, add_list, &fail_head);
> +	__pci_bus_assign_resources(bus, add_list, &fail_head);
>  	if (add_list)
>  		BUG_ON(!list_empty(add_list));
>  	tried_times++;
> @@ -1438,17 +1429,17 @@ again:
>  		goto enable_and_dump;
>  
>  	if (tried_times >= pci_try_num) {
> -		if (pci_realloc_enable == undefined)
> -			printk(KERN_INFO "Some PCI device resources are unassigned, try booting with pci=realloc\n");
> -		else if (pci_realloc_enable == auto_enabled)
> -			printk(KERN_INFO "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n");
> +		if (enable_local == undefined)
> +			dev_info(&bus->dev, "Some PCI device resources are unassigned, try booting with pci=realloc\n");
> +		else if (enable_local == auto_enabled)
> +			dev_info(&bus->dev, "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n");

I think you can add enable_local and the pci_realloc_enabled() parameter
in a separate patch.  That will remove distractions from the main patch.

>  
>  		free_list(&fail_head);
>  		goto enable_and_dump;
>  	}
>  
> -	printk(KERN_DEBUG "PCI: No. %d try to assign unassigned res\n",
> -			 tried_times + 1);
> +	dev_printk(KERN_DEBUG, &bus->dev,
> +		   "No. %d try to assign unassigned res\n", tried_times + 1);
>  
>  	/* third times and later will not check if it is leaf */
>  	if ((tried_times + 1) > 2)
> @@ -1458,12 +1449,11 @@ again:
>  	 * Try to release leaf bridge's resources that doesn't fit resource of
>  	 * child device under that bridge
>  	 */
> -	list_for_each_entry(fail_res, &fail_head, list) {
> -		bus = fail_res->dev->bus;
> -		pci_bus_release_bridge_resources(bus,
> +	list_for_each_entry(fail_res, &fail_head, list)
> +		pci_bus_release_bridge_resources(fail_res->dev->bus,

This change is gratuitous and distracting.  Please move it to a
separate patch.

>  						 fail_res->flags & type_mask,
>  						 rel_type);
> -	}
> +
>  	/* restore size and flags */
>  	list_for_each_entry(fail_res, &fail_head, list) {
>  		struct resource *res = fail_res->res;
> @@ -1480,12 +1470,19 @@ again:
>  
>  enable_and_dump:
>  	/* Depth last, update the hardware. */
> -	list_for_each_entry(bus, &pci_root_buses, node)
> -		pci_enable_bridges(bus);
> +	pci_enable_bridges(bus);
>  
>  	/* dump the resource on buses */
> +	pci_bus_dump_resources(bus);
> +}
> +
> +void __init
> +pci_assign_unassigned_resources(void)
> +{
> +	struct pci_bus *bus;
> +
>  	list_for_each_entry(bus, &pci_root_buses, node)
> -		pci_bus_dump_resources(bus);
> +		pci_assign_unassigned_root_bus_resources(bus);
>  }
>  
>  void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge)

I think this should be split up into something like the following patches
so we can see what's going on here:

    - Remove "bus" temporary when calling pci_bus_release_bridge_resources()

    - Add pci_realloc_enabled() parameter and enable_local

    - Add pci_realloc_detect() parameters.  The "bus" parameter is 
      ignored for now.

    - Change pci_realloc_detect() to iterate over pci_root_buses and
      call pci_walk_bus() to find any unassigned resources instead of
      using for_each_pci_dev().

    - Split pci_assign_unassigned_resources() into iterating over
      pci_root_buses and calling pci_assign_unassigned_root_bus_resources(bus).
      Change pci_realloc_detect() to only walk the supplied bus instead
      of everything in pci_root_buses.  This will be basically just removing
      list_for_each_entry(bus, &pci_root_buses, node) loops.

Bjorn

[PATCH v3 0/5] PCI: Skip resource allocation for root bus without conresponding type resource

[Yinghai Lu]

BenH reported that there is some assign unassigned resource problem
in powerpc.

It turns out after
| commit 0c5be0cb0edfe3b5c4b62eac68aa2aa15ec681af
| Date:   Thu Feb 23 19:23:29 2012 -0800
|
|    PCI: Retry on IORESOURCE_IO type allocations

even the root bus does not have io port range, it will keep retrying
to realloc with mmio.

Current retry logic is : try with must+optional at first, and if
it fails will try must then try to extend must with optional.
That will fail as mmio-non-pref and mmio-pref for bridge will
be next to each other. So we have no chance to extend mmio-non-pref.

This will be become more often when we have x86 8 sockets or 32 sockets
system, and those system will have one root bus per socket.
They will have some root buses do not have ioport range.

We should not fall into retry in this case, as root bus does
not io port range.

We check if the root bus has ioport range, and set bus_res_type_mask,
and pass it to assign_resources and don't add ioport res to failed list
for root bus that does not have ioport range.
So even BIOS set wrong value to pci devices and bridges will still
get cleared.

Then also check mmio-nonpref resource for root buses.

First two are for 3.10, and others are targeted to 3.11

 PCI: Split pci_assign_unassigned_resources to per root bus
 PCI: Skip IORESOURCE_IO allocation for root bus without ioport range
 PCI: Skip IORESOURCE_MMIO allocation for root bus without MMIO range
 PCI: Enable pci bridge when it is needed
 PCI: Retry assign unassigned resources for hotadd root bus

Thanks

Yinghai

[PATCH v3 1/5] PCI: Split pci_assign_unassigned_resources to per root bus

[Yinghai Lu]

BenH reported that there is some assign unassigned resource problem
in powerpc.

It turns out after
| commit 0c5be0cb0edfe3b5c4b62eac68aa2aa15ec681af
| Date:   Thu Feb 23 19:23:29 2012 -0800
|
|    PCI: Retry on IORESOURCE_IO type allocations

even the root bus does not have io port range, it will keep retrying
to realloc with mmio.

Current retry logic is : try with must+optional at first, and if
it fails will try must then try to extend must with optional.
That will fail as mmio-non-pref and mmio-pref for bridge will
be next to each other. So we have no chance to extend mmio-non-pref.

This will be become more often when we have x86 8 sockets or 32 sockets
system, and those system will have one root bus per socket.
They will have some root buses do not have ioport range.

We should not fall into retry in this case, as root bus does
not io port range.

Before that we need to split pci_assign_unassiged_resource
to every root bus, so we can stop early for root bus without ioport
range, and still continue to retry on buses that do have ioport range.

Also later we could let root bus hot add and booting path use same code.

Reported-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |  101 +++++++++++++++++++++++-------------------------
 1 file changed, 49 insertions(+), 52 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1315,21 +1315,6 @@ static int __init pci_bus_get_depth(stru
 
 	return depth;
 }
-static int __init pci_get_max_depth(void)
-{
-	int depth = 0;
-	struct pci_bus *bus;
-
-	list_for_each_entry(bus, &pci_root_buses, node) {
-		int ret;
-
-		ret = pci_bus_get_depth(bus);
-		if (ret > depth)
-			depth = ret;
-	}
-
-	return depth;
-}
 
 /*
  * -1: undefined, will auto detect later
@@ -1354,34 +1339,41 @@ void __init pci_realloc_get_opt(char *st
 	else if (!strncmp(str, "on", 2))
 		pci_realloc_enable = user_enabled;
 }
-static bool __init pci_realloc_enabled(void)
+static bool __init pci_realloc_enabled(enum enable_type enable)
 {
-	return pci_realloc_enable >= user_enabled;
+	return enable >= user_enabled;
 }
 
-static void __init pci_realloc_detect(void)
+static enum enable_type __init pci_realloc_detect(struct pci_bus *bus,
+			 enum enable_type enable_local)
 {
 #if defined(CONFIG_PCI_IOV) && defined(CONFIG_PCI_REALLOC_ENABLE_AUTO)
-	struct pci_dev *dev = NULL;
+	struct pci_dev *dev;
 
-	if (pci_realloc_enable != undefined)
-		return;
+	if (enable_local != undefined)
+		return enable_local;
 
-	for_each_pci_dev(dev) {
+	list_for_each_entry(dev, &bus->devices, bus_list) {
 		int i;
 
 		for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++) {
 			struct resource *r = &dev->resource[i];
 
 			/* Not assigned, or rejected by kernel ? */
-			if (r->flags && !r->start) {
-				pci_realloc_enable = auto_enabled;
-
-				return;
-			}
+			if (r->flags && !r->start)
+				return auto_enabled;
 		}
 	}
+
+	list_for_each_entry(dev, &bus->devices, bus_list) {
+		struct pci_bus *child = dev->subordinate;
+
+		if (child &&
+		    pci_realloc_detect(child, enable_local) == auto_enabled)
+			return auto_enabled;
+	}
 #endif
+	return enable_local;
 }
 
 /*
@@ -1389,10 +1381,9 @@ static void __init pci_realloc_detect(vo
  * second  and later try will clear small leaf bridge res
  * will stop till to the max  deepth if can not find good one
  */
-void __init
-pci_assign_unassigned_resources(void)
+static void __init
+pci_assign_unassigned_root_bus_resources(struct pci_bus *bus)
 {
-	struct pci_bus *bus;
 	LIST_HEAD(realloc_head); /* list of resources that
 					want additional resources */
 	struct list_head *add_list = NULL;
@@ -1403,15 +1394,17 @@ pci_assign_unassigned_resources(void)
 	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
 				  IORESOURCE_PREFETCH;
 	int pci_try_num = 1;
+	enum enable_type enable_local;
 
 	/* don't realloc if asked to do so */
-	pci_realloc_detect();
-	if (pci_realloc_enabled()) {
-		int max_depth = pci_get_max_depth();
+	enable_local = pci_realloc_detect(bus, pci_realloc_enable);
+	if (pci_realloc_enabled(enable_local)) {
+		int max_depth = pci_bus_get_depth(bus);
 
 		pci_try_num = max_depth + 1;
-		printk(KERN_DEBUG "PCI: max bus depth: %d pci_try_num: %d\n",
-			 max_depth, pci_try_num);
+		dev_printk(KERN_DEBUG, &bus->dev,
+			   "max bus depth: %d pci_try_num: %d\n",
+			   max_depth, pci_try_num);
 	}
 
 again:
@@ -1423,12 +1416,10 @@ again:
 		add_list = &realloc_head;
 	/* Depth first, calculate sizes and alignments of all
 	   subordinate buses. */
-	list_for_each_entry(bus, &pci_root_buses, node)
-		__pci_bus_size_bridges(bus, add_list);
+	__pci_bus_size_bridges(bus, add_list);
 
 	/* Depth last, allocate resources and update the hardware. */
-	list_for_each_entry(bus, &pci_root_buses, node)
-		__pci_bus_assign_resources(bus, add_list, &fail_head);
+	__pci_bus_assign_resources(bus, add_list, &fail_head);
 	if (add_list)
 		BUG_ON(!list_empty(add_list));
 	tried_times++;
@@ -1438,17 +1429,17 @@ again:
 		goto enable_and_dump;
 
 	if (tried_times >= pci_try_num) {
-		if (pci_realloc_enable == undefined)
-			printk(KERN_INFO "Some PCI device resources are unassigned, try booting with pci=realloc\n");
-		else if (pci_realloc_enable == auto_enabled)
-			printk(KERN_INFO "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n");
+		if (enable_local == undefined)
+			dev_info(&bus->dev, "Some PCI device resources are unassigned, try booting with pci=realloc\n");
+		else if (enable_local == auto_enabled)
+			dev_info(&bus->dev, "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n");
 
 		free_list(&fail_head);
 		goto enable_and_dump;
 	}
 
-	printk(KERN_DEBUG "PCI: No. %d try to assign unassigned res\n",
-			 tried_times + 1);
+	dev_printk(KERN_DEBUG, &bus->dev,
+		   "No. %d try to assign unassigned res\n", tried_times + 1);
 
 	/* third times and later will not check if it is leaf */
 	if ((tried_times + 1) > 2)
@@ -1458,12 +1449,11 @@ again:
 	 * Try to release leaf bridge's resources that doesn't fit resource of
 	 * child device under that bridge
 	 */
-	list_for_each_entry(fail_res, &fail_head, list) {
-		bus = fail_res->dev->bus;
-		pci_bus_release_bridge_resources(bus,
+	list_for_each_entry(fail_res, &fail_head, list)
+		pci_bus_release_bridge_resources(fail_res->dev->bus,
 						 fail_res->flags & type_mask,
 						 rel_type);
-	}
+
 	/* restore size and flags */
 	list_for_each_entry(fail_res, &fail_head, list) {
 		struct resource *res = fail_res->res;
@@ -1480,12 +1470,19 @@ again:
 
 enable_and_dump:
 	/* Depth last, update the hardware. */
-	list_for_each_entry(bus, &pci_root_buses, node)
-		pci_enable_bridges(bus);
+	pci_enable_bridges(bus);
 
 	/* dump the resource on buses */
+	pci_bus_dump_resources(bus);
+}
+
+void __init
+pci_assign_unassigned_resources(void)
+{
+	struct pci_bus *bus;
+
 	list_for_each_entry(bus, &pci_root_buses, node)
-		pci_bus_dump_resources(bus);
+		pci_assign_unassigned_root_bus_resources(bus);
 }
 
 void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge)

[PATCH v3 2/5] PCI: Skip IORESOURCE_IO allocation for root bus without ioport range

[Yinghai Lu]

BenH reported that there is some assign unassigned resource problem
in powerpc.

It turns out after
| commit 0c5be0cb0edfe3b5c4b62eac68aa2aa15ec681af
| Date:   Thu Feb 23 19:23:29 2012 -0800
|
|    PCI: Retry on IORESOURCE_IO type allocations

even the root bus does not have io port range, it will keep retrying
to realloc with mmio.

Current retry logic is : try with must+optional at first, and if
it fails will try must then try to extend must with optional.
That will fail as mmio-non-pref and mmio-pref for bridge will
be next to each other. So we have no chance to extend mmio-non-pref.

This will be become more often when we have x86 8 sockets or 32 sockets
system, and those system will have one root bus per socket.
They will have some root buses do not have ioport range.

We should not fall into retry in this case, as root bus does
not io port range.

We check if the root bus has ioport range, and set bus_res_type_mask,
and pass it to assign_resources and don't add ioport res to failed list
for root bus that does not have ioport range.
So even BIOS set wrong value to pci devices and bridges will still
get cleared.

For the retry failing, we could allocate mmio-non-pref bottom-up
and mmio-pref will be top-down, but that is not easy and could not be
material for v3.10.

-v2: remove wrong __init with pci_bus_res_type_mask()
     don't check bridge size/flags, and clear bus io resources.
-v3: change to skip adding to failed_list instead.

Reported-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |   86 ++++++++++++++++++++++++++++++++++++------------
 1 file changed, 66 insertions(+), 20 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -272,7 +272,8 @@ out:
  * requests that could not satisfied to the failed_list.
  */
 static void assign_requested_resources_sorted(struct list_head *head,
-				 struct list_head *fail_head)
+				 struct list_head *fail_head,
+				 unsigned long bus_res_type_mask)
 {
 	struct resource *res;
 	struct pci_dev_resource *dev_res;
@@ -288,8 +289,19 @@ static void assign_requested_resources_s
 				 * if the failed res is for ROM BAR, and it will
 				 * be enabled later, don't add it to the list
 				 */
-				if (!((idx == PCI_ROM_RESOURCE) &&
-				      (!(res->flags & IORESOURCE_ROM_ENABLE))))
+				bool is_rom_res_not_enabled =
+					 (idx == PCI_ROM_RESOURCE) &&
+					 (!(res->flags & IORESOURCE_ROM_ENABLE));
+				/*
+				 * if the failed res is io port, but bus does
+				 * not have io port support, don't add it
+				 */
+				bool is_ioport_res_without_bus_support =
+					 (!(bus_res_type_mask & IORESOURCE_IO)) &&
+					 (res->flags & IORESOURCE_IO);
+
+				if (!is_rom_res_not_enabled &&
+				    !is_ioport_res_without_bus_support)
 					add_to_list(fail_head,
 						    dev_res->dev, res,
 						    0 /* dont care */,
@@ -302,7 +314,8 @@ static void assign_requested_resources_s
 
 static void __assign_resources_sorted(struct list_head *head,
 				 struct list_head *realloc_head,
-				 struct list_head *fail_head)
+				 struct list_head *fail_head,
+				 unsigned long bus_res_type_mask)
 {
 	/*
 	 * Should not assign requested resources at first.
@@ -336,7 +349,8 @@ static void __assign_resources_sorted(st
 							dev_res->res);
 
 	/* Try updated head list with add_size added */
-	assign_requested_resources_sorted(head, &local_fail_head);
+	assign_requested_resources_sorted(head, &local_fail_head,
+					  bus_res_type_mask);
 
 	/* all assigned with add_size ? */
 	if (list_empty(&local_fail_head)) {
@@ -365,7 +379,8 @@ static void __assign_resources_sorted(st
 
 requested_and_reassign:
 	/* Satisfy the must-have resource requests */
-	assign_requested_resources_sorted(head, fail_head);
+	assign_requested_resources_sorted(head, fail_head,
+					  bus_res_type_mask);
 
 	/* Try to satisfy any additional optional resource
 		requests */
@@ -376,18 +391,21 @@ requested_and_reassign:
 
 static void pdev_assign_resources_sorted(struct pci_dev *dev,
 				 struct list_head *add_head,
-				 struct list_head *fail_head)
+				 struct list_head *fail_head,
+				 unsigned long bus_res_type_mask)
 {
 	LIST_HEAD(head);
 
 	__dev_sort_resources(dev, &head);
-	__assign_resources_sorted(&head, add_head, fail_head);
+	__assign_resources_sorted(&head, add_head, fail_head,
+					bus_res_type_mask);
 
 }
 
 static void pbus_assign_resources_sorted(const struct pci_bus *bus,
 					 struct list_head *realloc_head,
-					 struct list_head *fail_head)
+					 struct list_head *fail_head,
+					 unsigned long bus_res_type_mask)
 {
 	struct pci_dev *dev;
 	LIST_HEAD(head);
@@ -395,7 +413,8 @@ static void pbus_assign_resources_sorted
 	list_for_each_entry(dev, &bus->devices, bus_list)
 		__dev_sort_resources(dev, &head);
 
-	__assign_resources_sorted(&head, realloc_head, fail_head);
+	__assign_resources_sorted(&head, realloc_head, fail_head,
+					bus_res_type_mask);
 }
 
 void pci_setup_cardbus(struct pci_bus *bus)
@@ -1117,19 +1136,22 @@ EXPORT_SYMBOL(pci_bus_size_bridges);
 
 static void __ref __pci_bus_assign_resources(const struct pci_bus *bus,
 					 struct list_head *realloc_head,
-					 struct list_head *fail_head)
+					 struct list_head *fail_head,
+					 unsigned long bus_res_type_mask)
 {
 	struct pci_bus *b;
 	struct pci_dev *dev;
 
-	pbus_assign_resources_sorted(bus, realloc_head, fail_head);
+	pbus_assign_resources_sorted(bus, realloc_head, fail_head,
+					bus_res_type_mask);
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
 		b = dev->subordinate;
 		if (!b)
 			continue;
 
-		__pci_bus_assign_resources(b, realloc_head, fail_head);
+		__pci_bus_assign_resources(b, realloc_head, fail_head,
+						 bus_res_type_mask);
 
 		switch (dev->class >> 8) {
 		case PCI_CLASS_BRIDGE_PCI:
@@ -1151,24 +1173,28 @@ static void __ref __pci_bus_assign_resou
 
 void __ref pci_bus_assign_resources(const struct pci_bus *bus)
 {
-	__pci_bus_assign_resources(bus, NULL, NULL);
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	__pci_bus_assign_resources(bus, NULL, NULL, type_mask);
 }
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
 static void __ref __pci_bridge_assign_resources(const struct pci_dev *bridge,
 					 struct list_head *add_head,
-					 struct list_head *fail_head)
+					 struct list_head *fail_head,
+					 unsigned long bus_res_type_mask)
 {
 	struct pci_bus *b;
 
 	pdev_assign_resources_sorted((struct pci_dev *)bridge,
-					 add_head, fail_head);
+				     add_head, fail_head, bus_res_type_mask);
 
 	b = bridge->subordinate;
 	if (!b)
 		return;
 
-	__pci_bus_assign_resources(b, add_head, fail_head);
+	__pci_bus_assign_resources(b, add_head, fail_head, bus_res_type_mask);
 
 	switch (bridge->class >> 8) {
 	case PCI_CLASS_BRIDGE_PCI:
@@ -1376,6 +1402,21 @@ static enum enable_type __init pci_reall
 	return enable_local;
 }
 
+static unsigned long pci_bus_res_type_mask(struct pci_bus *bus)
+{
+	int i;
+	struct resource *r;
+	unsigned long mask = 0;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	pci_bus_for_each_resource(bus, r, i)
+		if (r)
+			mask |= r->flags & type_mask;
+
+	return mask;
+}
+
 /*
  * first try will not touch pci bridge res
  * second  and later try will clear small leaf bridge res
@@ -1395,6 +1436,7 @@ pci_assign_unassigned_root_bus_resources
 				  IORESOURCE_PREFETCH;
 	int pci_try_num = 1;
 	enum enable_type enable_local;
+	unsigned long bus_res_type_mask = pci_bus_res_type_mask(bus);
 
 	/* don't realloc if asked to do so */
 	enable_local = pci_realloc_detect(bus, pci_realloc_enable);
@@ -1419,7 +1461,8 @@ again:
 	__pci_bus_size_bridges(bus, add_list);
 
 	/* Depth last, allocate resources and update the hardware. */
-	__pci_bus_assign_resources(bus, add_list, &fail_head);
+	__pci_bus_assign_resources(bus, add_list, &fail_head,
+				   bus_res_type_mask);
 	if (add_list)
 		BUG_ON(!list_empty(add_list));
 	tried_times++;
@@ -1496,10 +1539,12 @@ void pci_assign_unassigned_bridge_resour
 	int retval;
 	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
 				  IORESOURCE_PREFETCH;
+	unsigned long bus_res_type_mask = pci_bus_res_type_mask(bridge->bus);
 
 again:
 	__pci_bus_size_bridges(parent, &add_list);
-	__pci_bridge_assign_resources(bridge, &add_list, &fail_head);
+	__pci_bridge_assign_resources(bridge, &add_list, &fail_head,
+					bus_res_type_mask);
 	BUG_ON(!list_empty(&add_list));
 	tried_times++;
 
@@ -1554,6 +1599,7 @@ void pci_assign_unassigned_bus_resources
 	struct pci_dev *dev;
 	LIST_HEAD(add_list); /* list of resources that
 					want additional resources */
+	unsigned long bus_res_type_mask = pci_bus_res_type_mask(bus);
 
 	down_read(&pci_bus_sem);
 	list_for_each_entry(dev, &bus->devices, bus_list)
@@ -1563,6 +1609,6 @@ void pci_assign_unassigned_bus_resources
 				__pci_bus_size_bridges(dev->subordinate,
 							 &add_list);
 	up_read(&pci_bus_sem);
-	__pci_bus_assign_resources(bus, &add_list, NULL);
+	__pci_bus_assign_resources(bus, &add_list, NULL, bus_res_type_mask);
 	BUG_ON(!list_empty(&add_list));
 }

[PATCH v3 3/5] PCI: Skip IORESOURCE_MMIO allocation for root bus without MMIO range

[Yinghai Lu]

For x86 8 sockets or 32 sockets system that will have one root bus per socket,
They may have some root buses do not have mmio non-pref range.

We should not fall into retry in this case, as root bus does
not mmio non-pref range.

We check if the root bus has mmio-nonpref range, and set bus_res_type_mask,
and pass it to assign_resources and don't add mmio-nonpref res to failed list
for root bus that does not have mmio-nonpref range.
So even BIOS set wrong value to pci devices and bridges will still
get cleared.

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |   32 ++++++++++++++++++++++++++------
 1 file changed, 26 insertions(+), 6 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -299,9 +299,17 @@ static void assign_requested_resources_s
 				bool is_ioport_res_without_bus_support =
 					 (!(bus_res_type_mask & IORESOURCE_IO)) &&
 					 (res->flags & IORESOURCE_IO);
+				/*
+				 * if the failed res is mmio, but bus does
+				 * not have io port support, don't add it
+				 */
+				bool is_mmio_nonpref_res_without_bus_support =
+					 (!(bus_res_type_mask & IORESOURCE_MEM)) &&
+					 ((res->flags & (IORESOURCE_MEM | IORESOURCE_PREFETCH)) == IORESOURCE_MEM);
 
 				if (!is_rom_res_not_enabled &&
-				    !is_ioport_res_without_bus_support)
+				    !is_ioport_res_without_bus_support &&
+				    !is_mmio_nonpref_res_without_bus_support)
 					add_to_list(fail_head,
 						    dev_res->dev, res,
 						    0 /* dont care */,
@@ -1407,12 +1415,24 @@ static unsigned long pci_bus_res_type_ma
 	int i;
 	struct resource *r;
 	unsigned long mask = 0;
-	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
-				  IORESOURCE_PREFETCH;
 
-	pci_bus_for_each_resource(bus, r, i)
-		if (r)
-			mask |= r->flags & type_mask;
+	pci_bus_for_each_resource(bus, r, i) {
+		if (!r)
+			continue;
+
+		if (r->flags & IORESOURCE_IO) {
+			mask |= IORESOURCE_IO;
+			continue;
+		}
+		if (r->flags & IORESOURCE_PREFETCH) {
+			mask |= IORESOURCE_PREFETCH;
+			continue;
+		}
+		if ((r->flags & (IORESOURCE_MEM | IORESOURCE_PREFETCH)) == IORESOURCE_MEM) {
+			mask |= IORESOURCE_MEM; /* nonpref only */
+			continue;
+		}
+	}
 
 	return mask;
 }

Re: [PATCH v3 3/5] PCI: Skip IORESOURCE_MMIO allocation for root bus without MMIO range

[Benjamin Herrenschmidt]

On Tue, 2013-05-07 at 15:17 -0700, Yinghai Lu wrote:
> For x86 8 sockets or 32 sockets system that will have one root bus per socket,
> They may have some root buses do not have mmio non-pref range.

That seems very odd. Most device registers are non-prefetchable. I know
of no adapter today that would work in a prefetchable-only environment.

Are you sure that isn't the other way around ?

Regarding your 3.10 patches, me and Gavin will test your v3 later today
(ASAP) and will give you an Ack if they work, in which case they should
hit Linus as soon as Bjorn is comfortable with :-)

Cheers,
Ben.

> We should not fall into retry in this case, as root bus does
> not mmio non-pref range.
> 
> We check if the root bus has mmio-nonpref range, and set bus_res_type_mask,
> and pass it to assign_resources and don't add mmio-nonpref res to failed list
> for root bus that does not have mmio-nonpref range.
> So even BIOS set wrong value to pci devices and bridges will still
> get cleared.
> 
> Signed-off-by: Yinghai Lu <yinghai@kernel.org>
> 
> ---
>  drivers/pci/setup-bus.c |   32 ++++++++++++++++++++++++++------
>  1 file changed, 26 insertions(+), 6 deletions(-)
> 
> Index: linux-2.6/drivers/pci/setup-bus.c
> ===================================================================
> --- linux-2.6.orig/drivers/pci/setup-bus.c
> +++ linux-2.6/drivers/pci/setup-bus.c
> @@ -299,9 +299,17 @@ static void assign_requested_resources_s
>  				bool is_ioport_res_without_bus_support =
>  					 (!(bus_res_type_mask & IORESOURCE_IO)) &&
>  					 (res->flags & IORESOURCE_IO);
> +				/*
> +				 * if the failed res is mmio, but bus does
> +				 * not have io port support, don't add it
> +				 */
> +				bool is_mmio_nonpref_res_without_bus_support =
> +					 (!(bus_res_type_mask & IORESOURCE_MEM)) &&
> +					 ((res->flags & (IORESOURCE_MEM | IORESOURCE_PREFETCH)) == IORESOURCE_MEM);
>  
>  				if (!is_rom_res_not_enabled &&
> -				    !is_ioport_res_without_bus_support)
> +				    !is_ioport_res_without_bus_support &&
> +				    !is_mmio_nonpref_res_without_bus_support)
>  					add_to_list(fail_head,
>  						    dev_res->dev, res,
>  						    0 /* dont care */,
> @@ -1407,12 +1415,24 @@ static unsigned long pci_bus_res_type_ma
>  	int i;
>  	struct resource *r;
>  	unsigned long mask = 0;
> -	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
> -				  IORESOURCE_PREFETCH;
>  
> -	pci_bus_for_each_resource(bus, r, i)
> -		if (r)
> -			mask |= r->flags & type_mask;
> +	pci_bus_for_each_resource(bus, r, i) {
> +		if (!r)
> +			continue;
> +
> +		if (r->flags & IORESOURCE_IO) {
> +			mask |= IORESOURCE_IO;
> +			continue;
> +		}
> +		if (r->flags & IORESOURCE_PREFETCH) {
> +			mask |= IORESOURCE_PREFETCH;
> +			continue;
> +		}
> +		if ((r->flags & (IORESOURCE_MEM | IORESOURCE_PREFETCH)) == IORESOURCE_MEM) {
> +			mask |= IORESOURCE_MEM; /* nonpref only */
> +			continue;
> +		}
> +	}
>  
>  	return mask;
>  }

Re: [PATCH v3 3/5] PCI: Skip IORESOURCE_MMIO allocation for root bus without MMIO range

[Yinghai Lu]

On Tue, May 7, 2013 at 3:28 PM, Benjamin Herrenschmidt
<benh@kernel.crashing.org> wrote:
> On Tue, 2013-05-07 at 15:17 -0700, Yinghai Lu wrote:
>> For x86 8 sockets or 32 sockets system that will have one root bus per socket,
>> They may have some root buses do not have mmio non-pref range.
>
> That seems very odd. Most device registers are non-prefetchable. I know
> of no adapter today that would work in a prefetchable-only environment.

FC/10G/FCoE from Qlogic and Emulex can be used with 64bit mmio-pref only.

pci bridge will only support 32bit non-pref mmio, so we have them under 4G,
and it could be only 2G at most.

x86 32 socket system, we may need to leave more mmiol for only several
sockets to make them work with cards that does not support mmio 64 bit pref.

we can always have enough 64bit mmio pref above 4G....

>
> Are you sure that isn't the other way around ?
>
> Regarding your 3.10 patches, me and Gavin will test your v3 later today
> (ASAP) and will give you an Ack if they work, in which case they should
> hit Linus as soon as Bjorn is comfortable with :-)

Thanks

Yinghai

Re: [PATCH v3 3/5] PCI: Skip IORESOURCE_MMIO allocation for root bus without MMIO range

[Benjamin Herrenschmidt]

On Tue, 2013-05-07 at 15:44 -0700, Yinghai Lu wrote:
> x86 32 socket system, we may need to leave more mmiol for only several
> sockets to make them work with cards that does not support mmio 64 bit
> pref.

Ok, while on POWER each root bridge has its own distinct 32-bit space
(mapped elsewhere in CPU space). So at least we don't have *that*
specific problem :-)

Cheers,
Ben.

Re: [PATCH v3 3/5] PCI: Skip IORESOURCE_MMIO allocation for root bus without MMIO range

[Yinghai Lu]

On Tue, May 7, 2013 at 6:16 PM, Benjamin Herrenschmidt
<benh@kernel.crashing.org> wrote:
> On Tue, 2013-05-07 at 15:44 -0700, Yinghai Lu wrote:
>> x86 32 socket system, we may need to leave more mmiol for only several
>> sockets to make them work with cards that does not support mmio 64 bit
>> pref.
>
> Ok, while on POWER each root bridge has its own distinct 32-bit space
> (mapped elsewhere in CPU space). So at least we don't have *that*
> specific problem :-)

Hope x86 could support that.

Yinghai

[PATCH v3 4/5] PCI: Enable pci bridge when it is needed

[Yinghai Lu]

Current we enable bridges after bus scan and assign resources.
and it is spreaded a lot of places.

We can move it to where pci device is enabled, and need
to go up to root bus and enable bridge one by one down to pci
dev.

So that will delay enable bridge late as needed bassis,
also kill one inconsistent between boot path and hotplug
path in acpi_pci_root_add().

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 arch/arm/kernel/bios32.c           |    5 -----
 arch/m68k/platform/coldfire/pci.c  |    1 -
 arch/mips/pci/pci.c                |    1 -
 arch/sh/drivers/pci/pci.c          |    1 -
 drivers/acpi/pci_root.c            |    4 ----
 drivers/parisc/lba_pci.c           |    1 -
 drivers/pci/bus.c                  |   19 -------------------
 drivers/pci/hotplug/acpiphp_glue.c |    1 -
 drivers/pci/pci.c                  |   20 ++++++++++++++++++++
 drivers/pci/probe.c                |    1 -
 drivers/pci/setup-bus.c            |   10 +++-------
 drivers/pcmcia/cardbus.c           |    1 -
 include/linux/pci.h                |    1 -
 13 files changed, 23 insertions(+), 43 deletions(-)

Index: linux-2.6/arch/arm/kernel/bios32.c
===================================================================
--- linux-2.6.orig/arch/arm/kernel/bios32.c
+++ linux-2.6/arch/arm/kernel/bios32.c
@@ -524,11 +524,6 @@ void pci_common_init(struct hw_pci *hw)
 			 * Assign resources.
 			 */
 			pci_bus_assign_resources(bus);
-
-			/*
-			 * Enable bridges
-			 */
-			pci_enable_bridges(bus);
 		}
 
 		/*
Index: linux-2.6/arch/m68k/platform/coldfire/pci.c
===================================================================
--- linux-2.6.orig/arch/m68k/platform/coldfire/pci.c
+++ linux-2.6/arch/m68k/platform/coldfire/pci.c
@@ -319,7 +319,6 @@ static int __init mcf_pci_init(void)
 	pci_fixup_irqs(pci_common_swizzle, mcf_pci_map_irq);
 	pci_bus_size_bridges(rootbus);
 	pci_bus_assign_resources(rootbus);
-	pci_enable_bridges(rootbus);
 	pci_bus_add_devices(rootbus);
 	return 0;
 }
Index: linux-2.6/arch/mips/pci/pci.c
===================================================================
--- linux-2.6.orig/arch/mips/pci/pci.c
+++ linux-2.6/arch/mips/pci/pci.c
@@ -113,7 +113,6 @@ static void pcibios_scanbus(struct pci_c
 		if (!pci_has_flag(PCI_PROBE_ONLY)) {
 			pci_bus_size_bridges(bus);
 			pci_bus_assign_resources(bus);
-			pci_enable_bridges(bus);
 		}
 	}
 }
Index: linux-2.6/arch/sh/drivers/pci/pci.c
===================================================================
--- linux-2.6.orig/arch/sh/drivers/pci/pci.c
+++ linux-2.6/arch/sh/drivers/pci/pci.c
@@ -69,7 +69,6 @@ static void pcibios_scanbus(struct pci_c
 
 		pci_bus_size_bridges(bus);
 		pci_bus_assign_resources(bus);
-		pci_enable_bridges(bus);
 	} else {
 		pci_free_resource_list(&resources);
 	}
Index: linux-2.6/drivers/acpi/pci_root.c
===================================================================
--- linux-2.6.orig/drivers/acpi/pci_root.c
+++ linux-2.6/drivers/acpi/pci_root.c
@@ -538,10 +538,6 @@ static int acpi_pci_root_add(struct acpi
 		pci_assign_unassigned_bus_resources(root->bus);
 	}
 
-	/* need to after hot-added ioapic is registered */
-	if (system_state != SYSTEM_BOOTING)
-		pci_enable_bridges(root->bus);
-
 	pci_bus_add_devices(root->bus);
 	return 1;
 
Index: linux-2.6/drivers/parisc/lba_pci.c
===================================================================
--- linux-2.6.orig/drivers/parisc/lba_pci.c
+++ linux-2.6/drivers/parisc/lba_pci.c
@@ -1533,7 +1533,6 @@ lba_driver_probe(struct parisc_device *d
 		lba_dump_res(&lba_dev->hba.lmmio_space, 2);
 #endif
 	}
-	pci_enable_bridges(lba_bus);
 
 	/*
 	** Once PCI register ops has walked the bus, access to config
Index: linux-2.6/drivers/pci/bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/bus.c
+++ linux-2.6/drivers/pci/bus.c
@@ -215,24 +215,6 @@ void pci_bus_add_devices(const struct pc
 	}
 }
 
-void pci_enable_bridges(struct pci_bus *bus)
-{
-	struct pci_dev *dev;
-	int retval;
-
-	list_for_each_entry(dev, &bus->devices, bus_list) {
-		if (dev->subordinate) {
-			if (!pci_is_enabled(dev)) {
-				retval = pci_enable_device(dev);
-				if (retval)
-					dev_err(&dev->dev, "Error enabling bridge (%d), continuing\n", retval);
-				pci_set_master(dev);
-			}
-			pci_enable_bridges(dev->subordinate);
-		}
-	}
-}
-
 /** pci_walk_bus - walk devices on/under bus, calling callback.
  *  @top      bus whose devices should be walked
  *  @cb       callback to be called for each device found
@@ -285,4 +267,3 @@ EXPORT_SYMBOL_GPL(pci_walk_bus);
 EXPORT_SYMBOL(pci_bus_alloc_resource);
 EXPORT_SYMBOL_GPL(pci_bus_add_device);
 EXPORT_SYMBOL(pci_bus_add_devices);
-EXPORT_SYMBOL(pci_enable_bridges);
Index: linux-2.6/drivers/pci/hotplug/acpiphp_glue.c
===================================================================
--- linux-2.6.orig/drivers/pci/hotplug/acpiphp_glue.c
+++ linux-2.6/drivers/pci/hotplug/acpiphp_glue.c
@@ -704,7 +704,6 @@ static int __ref enable_device(struct ac
 	acpiphp_sanitize_bus(bus);
 	acpiphp_set_hpp_values(bus);
 	acpiphp_set_acpi_region(slot);
-	pci_enable_bridges(bus);
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
 		/* Assume that newly added devices are powered on already. */
Index: linux-2.6/drivers/pci/pci.c
===================================================================
--- linux-2.6.orig/drivers/pci/pci.c
+++ linux-2.6/drivers/pci/pci.c
@@ -1145,6 +1145,24 @@ int pci_reenable_device(struct pci_dev *
 	return 0;
 }
 
+static void pci_enable_bridge(struct pci_dev *dev)
+{
+	int retval;
+
+	if (!dev)
+		return;
+
+	pci_enable_bridge(dev->bus->self);
+
+	if (pci_is_enabled(dev))
+		return;
+	retval = pci_enable_device(dev);
+	if (retval)
+		dev_err(&dev->dev, "Error enabling bridge (%d), continuing\n",
+			retval);
+	pci_set_master(dev);
+}
+
 static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags)
 {
 	int err;
@@ -1165,6 +1183,8 @@ static int pci_enable_device_flags(struc
 	if (atomic_inc_return(&dev->enable_cnt) > 1)
 		return 0;		/* already enabled */
 
+	pci_enable_bridge(dev->bus->self);
+
 	/* only skip sriov related */
 	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
 		if (dev->resource[i].flags & flags)
Index: linux-2.6/drivers/pci/probe.c
===================================================================
--- linux-2.6.orig/drivers/pci/probe.c
+++ linux-2.6/drivers/pci/probe.c
@@ -1949,7 +1949,6 @@ unsigned int __ref pci_rescan_bus(struct
 
 	max = pci_scan_child_bus(bus);
 	pci_assign_unassigned_bus_resources(bus);
-	pci_enable_bridges(bus);
 	pci_bus_add_devices(bus);
 
 	return max;
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1489,7 +1489,7 @@ again:
 
 	/* any device complain? */
 	if (list_empty(&fail_head))
-		goto enable_and_dump;
+		goto dump;
 
 	if (tried_times >= pci_try_num) {
 		if (enable_local == undefined)
@@ -1498,7 +1498,7 @@ again:
 			dev_info(&bus->dev, "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n");
 
 		free_list(&fail_head);
-		goto enable_and_dump;
+		goto dump;
 	}
 
 	dev_printk(KERN_DEBUG, &bus->dev,
@@ -1531,10 +1531,7 @@ again:
 
 	goto again;
 
-enable_and_dump:
-	/* Depth last, update the hardware. */
-	pci_enable_bridges(bus);
-
+dump:
 	/* dump the resource on buses */
 	pci_bus_dump_resources(bus);
 }
@@ -1610,7 +1607,6 @@ enable_all:
 	if (retval)
 		dev_err(&bridge->dev, "Error reenabling bridge (%d)\n", retval);
 	pci_set_master(bridge);
-	pci_enable_bridges(parent);
 }
 EXPORT_SYMBOL_GPL(pci_assign_unassigned_bridge_resources);
 
Index: linux-2.6/drivers/pcmcia/cardbus.c
===================================================================
--- linux-2.6.orig/drivers/pcmcia/cardbus.c
+++ linux-2.6/drivers/pcmcia/cardbus.c
@@ -91,7 +91,6 @@ int __ref cb_alloc(struct pcmcia_socket
 	if (s->tune_bridge)
 		s->tune_bridge(s, bus);
 
-	pci_enable_bridges(bus);
 	pci_bus_add_devices(bus);
 
 	return 0;
Index: linux-2.6/include/linux/pci.h
===================================================================
--- linux-2.6.orig/include/linux/pci.h
+++ linux-2.6/include/linux/pci.h
@@ -1040,7 +1040,6 @@ int __must_check pci_bus_alloc_resource(
 						  resource_size_t,
 						  resource_size_t),
 			void *alignf_data);
-void pci_enable_bridges(struct pci_bus *bus);
 
 /* Proper probing supporting hot-pluggable devices */
 int __must_check __pci_register_driver(struct pci_driver *, struct module *,

[PATCH v3 5/5] PCI: Retry assign unassigned resources for hotadd root bus

[Yinghai Lu]

Let root bus hotadd path use same code for booting path.
As driver is not loaded yet, we could retry to make sure
all pci devices get resources allocated.

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/acpi/pci_root.c |    2 +-
 drivers/pci/setup-bus.c |   11 +++++------
 include/linux/pci.h     |    1 +
 3 files changed, 7 insertions(+), 7 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1331,7 +1331,7 @@ static void pci_bus_dump_resources(struc
 	}
 }
 
-static int __init pci_bus_get_depth(struct pci_bus *bus)
+static int pci_bus_get_depth(struct pci_bus *bus)
 {
 	int depth = 0;
 	struct pci_dev *dev;
@@ -1365,7 +1365,7 @@ enum enable_type {
 	auto_enabled,
 };
 
-static enum enable_type pci_realloc_enable __initdata = undefined;
+static enum enable_type pci_realloc_enable = undefined;
 void __init pci_realloc_get_opt(char *str)
 {
 	if (!strncmp(str, "off", 3))
@@ -1373,12 +1373,12 @@ void __init pci_realloc_get_opt(char *st
 	else if (!strncmp(str, "on", 2))
 		pci_realloc_enable = user_enabled;
 }
-static bool __init pci_realloc_enabled(enum enable_type enable)
+static bool pci_realloc_enabled(enum enable_type enable)
 {
 	return enable >= user_enabled;
 }
 
-static enum enable_type __init pci_realloc_detect(struct pci_bus *bus,
+static enum enable_type pci_realloc_detect(struct pci_bus *bus,
 			 enum enable_type enable_local)
 {
 #if defined(CONFIG_PCI_IOV) && defined(CONFIG_PCI_REALLOC_ENABLE_AUTO)
@@ -1442,8 +1442,7 @@ static unsigned long pci_bus_res_type_ma
  * second  and later try will clear small leaf bridge res
  * will stop till to the max  deepth if can not find good one
  */
-static void __init
-pci_assign_unassigned_root_bus_resources(struct pci_bus *bus)
+void pci_assign_unassigned_root_bus_resources(struct pci_bus *bus)
 {
 	LIST_HEAD(realloc_head); /* list of resources that
 					want additional resources */
Index: linux-2.6/drivers/acpi/pci_root.c
===================================================================
--- linux-2.6.orig/drivers/acpi/pci_root.c
+++ linux-2.6/drivers/acpi/pci_root.c
@@ -535,7 +535,7 @@ static int acpi_pci_root_add(struct acpi
 
 	if (system_state != SYSTEM_BOOTING) {
 		pcibios_resource_survey_bus(root->bus);
-		pci_assign_unassigned_bus_resources(root->bus);
+		pci_assign_unassigned_root_bus_resources(root->bus);
 	}
 
 	pci_bus_add_devices(root->bus);
Index: linux-2.6/include/linux/pci.h
===================================================================
--- linux-2.6.orig/include/linux/pci.h
+++ linux-2.6/include/linux/pci.h
@@ -1002,6 +1002,7 @@ int pci_claim_resource(struct pci_dev *,
 void pci_assign_unassigned_resources(void);
 void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge);
 void pci_assign_unassigned_bus_resources(struct pci_bus *bus);
+void pci_assign_unassigned_root_bus_resources(struct pci_bus *bus);
 void pdev_enable_device(struct pci_dev *);
 int pci_enable_resources(struct pci_dev *, int mask);
 void pci_fixup_irqs(u8 (*)(struct pci_dev *, u8 *),

[PATCH v4 0/8] PCI: Skip resource allocation for root bus without conresponding type resource

[Yinghai Lu]

BenH reported that there is some assign unassigned resource problem
in powerpc.

It turns out after
| commit 0c5be0cb0edfe3b5c4b62eac68aa2aa15ec681af
| Date:   Thu Feb 23 19:23:29 2012 -0800
|
|    PCI: Retry on IORESOURCE_IO type allocations

even the root bus does not have io port range, it will keep retrying
to realloc with mmio.

Current retry logic is : try with must+optional at first, and if
it fails will try must then try to extend must with optional.
That will fail as mmio-non-pref and mmio-pref for bridge will
be next to each other. So we have no chance to extend mmio-non-pref.

This will be become more often when we have x86 8 sockets or 32 sockets
system, and those system will have one root bus per socket.
They will have some root buses do not have ioport range.

We should not fall into retry in this case, as root bus does
not io port range.

We check if the root bus has ioport range, and set bus_res_type_mask,
and pass it to assign_resources and don't add ioport res to failed list
for root bus that does not have ioport range.
So even BIOS set wrong value to pci devices and bridges will still
get cleared.

Then also check mmio-nonpref resource for root buses.

First five are for 3.10, and others are targeted to 3.11

-v4: split first patch into 4 patches per Bjorn.

 PCI: Don't use temp bus for pci_bus_release_bridge_resources
 PCI: Use pci_walk_bus to detect unassigned resources
 PCI: Introduce enable_local to prepare per root bus handling
 PCI: Split pci_assign_unassigned_resources to per root bus
 PCI: Skip IORESOURCE_IO allocation for root bus without ioport range
 PCI: Skip IORESOURCE_MMIO allocation for root bus without MMIO range
 PCI: Enable pci bridge when it is needed
 PCI: Retry assign unassigned resources for hotadd root bus

Thanks

Yinghai

[PATCH v4 1/8] PCI: Don't use temp bus for pci_bus_release_bridge_resources

[Yinghai Lu]

as later bus can not be used as temp variable after we change to
per root bus handling with assign unassigned resources.

Per Bjorn, separated out different patch.

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |    7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1458,12 +1458,11 @@ again:
 	 * Try to release leaf bridge's resources that doesn't fit resource of
 	 * child device under that bridge
 	 */
-	list_for_each_entry(fail_res, &fail_head, list) {
-		bus = fail_res->dev->bus;
-		pci_bus_release_bridge_resources(bus,
+	list_for_each_entry(fail_res, &fail_head, list)
+		pci_bus_release_bridge_resources(fail_res->dev->bus,
 						 fail_res->flags & type_mask,
 						 rel_type);
-	}
+
 	/* restore size and flags */
 	list_for_each_entry(fail_res, &fail_head, list) {
 		struct resource *res = fail_res->res;

[PATCH v4 2/8] PCI: Use pci_walk_bus to detect unassigned resources

[Yinghai Lu]

Per Bjorn, use pci_walk_bus instead of for_each_pci_dev or
calling pci_realloc_detect() recursively.

And separate it to different patch.

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |   46 +++++++++++++++++++++++++++++++---------------
 1 file changed, 31 insertions(+), 15 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1359,30 +1359,46 @@ static bool __init pci_realloc_enabled(v
 	return pci_realloc_enable >= user_enabled;
 }
 
-static void __init pci_realloc_detect(void)
-{
 #if defined(CONFIG_PCI_IOV) && defined(CONFIG_PCI_REALLOC_ENABLE_AUTO)
-	struct pci_dev *dev = NULL;
+static int __init check_unassigned_resources(struct pci_dev *dev, void *data)
+{
+	int i;
+	int *unassigned = data;
 
-	if (pci_realloc_enable != undefined)
-		return;
+	for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++) {
+		struct resource *r = &dev->resource[i];
 
-	for_each_pci_dev(dev) {
-		int i;
+		/* Not assigned, or rejected by kernel ? */
+		if (r->flags && !r->start) {
+			(*unassigned)++;
+			return 1; /* return early from pci_walk_bus */
+		}
+	}
 
-		for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++) {
-			struct resource *r = &dev->resource[i];
+	return 0;
+}
 
-			/* Not assigned, or rejected by kernel ? */
-			if (r->flags && !r->start) {
-				pci_realloc_enable = auto_enabled;
+static void  __init pci_realloc_detect(void)
+{
+	int unassigned = 0;
+	struct pci_bus *bus;
 
-				return;
-			}
+	if (pci_realloc_enable != undefined)
+		return;
+
+	list_for_each_entry(bus, &pci_root_buses, node) {
+		pci_walk_bus(bus, check_unassigned_resources, &unassigned);
+		if (unassigned) {
+			pci_realloc_enable = auto_enabled;
+			return;
 		}
 	}
-#endif
 }
+#else
+static void __init pci_realloc_detect(void)
+{
+}
+#endif
 
 /*
  * first try will not touch pci bridge res

[PATCH v4 3/8] PCI: Introduce enable_local to prepare per root bus handling

[Yinghai Lu]

Add enable_local to prepare assign unassigned resource
for per root bus.

Per Bjorn, separate it to different patch.

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |   40 +++++++++++++++++++++-------------------
 1 file changed, 21 insertions(+), 19 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1354,9 +1354,9 @@ void __init pci_realloc_get_opt(char *st
 	else if (!strncmp(str, "on", 2))
 		pci_realloc_enable = user_enabled;
 }
-static bool __init pci_realloc_enabled(void)
+static bool __init pci_realloc_enabled(enum enable_type enable)
 {
-	return pci_realloc_enable >= user_enabled;
+	return enable >= user_enabled;
 }
 
 #if defined(CONFIG_PCI_IOV) && defined(CONFIG_PCI_REALLOC_ENABLE_AUTO)
@@ -1378,25 +1378,25 @@ static int __init check_unassigned_resou
 	return 0;
 }
 
-static void  __init pci_realloc_detect(void)
+static enum enable_type __init pci_realloc_detect(struct pci_bus *bus,
+			 enum enable_type enable_local)
 {
 	int unassigned = 0;
-	struct pci_bus *bus;
 
-	if (pci_realloc_enable != undefined)
-		return;
+	if (enable_local != undefined)
+		return enable_local;
 
-	list_for_each_entry(bus, &pci_root_buses, node) {
-		pci_walk_bus(bus, check_unassigned_resources, &unassigned);
-		if (unassigned) {
-			pci_realloc_enable = auto_enabled;
-			return;
-		}
-	}
+	pci_walk_bus(bus, check_unassigned_resources, &unassigned);
+	if (unassigned)
+		return auto_enabled;
+
+	return enable_local;
 }
 #else
-static void __init pci_realloc_detect(void)
+static enum enable_type __init pci_realloc_detect(struct pci_bus *bus,
+			 enum enable_type enable_local)
 {
+	return enable_local;
 }
 #endif
 
@@ -1419,10 +1419,12 @@ pci_assign_unassigned_resources(void)
 	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
 				  IORESOURCE_PREFETCH;
 	int pci_try_num = 1;
+	enum enable_type enable_local = pci_realloc_enable;
+
+	list_for_each_entry(bus, &pci_root_buses, node)
+		enable_local = pci_realloc_detect(bus, enable_local);
 
-	/* don't realloc if asked to do so */
-	pci_realloc_detect();
-	if (pci_realloc_enabled()) {
+	if (pci_realloc_enabled(enable_local)) {
 		int max_depth = pci_get_max_depth();
 
 		pci_try_num = max_depth + 1;
@@ -1454,9 +1456,9 @@ again:
 		goto enable_and_dump;
 
 	if (tried_times >= pci_try_num) {
-		if (pci_realloc_enable == undefined)
+		if (enable_local == undefined)
 			printk(KERN_INFO "Some PCI device resources are unassigned, try booting with pci=realloc\n");
-		else if (pci_realloc_enable == auto_enabled)
+		else if (enable_local == auto_enabled)
 			printk(KERN_INFO "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n");
 
 		free_list(&fail_head);

[PATCH v4 4/8] PCI: Split pci_assign_unassigned_resources to per root bus

[Yinghai Lu]

We need to split pci_assign_unassiged_resource to every root bus, so in
next patch we can stop early for root bus without ioport range, and still
continue to retry on buses that do have ioport range.

Also later we could let root bus hot add and booting path use same code.

-v2: separate enable_local and pci_release_bridge_resources to
     other patches requested by Bjorn.

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |   62 +++++++++++++++++++-----------------------------
 1 file changed, 25 insertions(+), 37 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1315,21 +1315,6 @@ static int __init pci_bus_get_depth(stru
 
 	return depth;
 }
-static int __init pci_get_max_depth(void)
-{
-	int depth = 0;
-	struct pci_bus *bus;
-
-	list_for_each_entry(bus, &pci_root_buses, node) {
-		int ret;
-
-		ret = pci_bus_get_depth(bus);
-		if (ret > depth)
-			depth = ret;
-	}
-
-	return depth;
-}
 
 /*
  * -1: undefined, will auto detect later
@@ -1405,10 +1390,9 @@ static enum enable_type __init pci_reall
  * second  and later try will clear small leaf bridge res
  * will stop till to the max  deepth if can not find good one
  */
-void __init
-pci_assign_unassigned_resources(void)
+static void __init
+pci_assign_unassigned_root_bus_resources(struct pci_bus *bus)
 {
-	struct pci_bus *bus;
 	LIST_HEAD(realloc_head); /* list of resources that
 					want additional resources */
 	struct list_head *add_list = NULL;
@@ -1419,17 +1403,17 @@ pci_assign_unassigned_resources(void)
 	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
 				  IORESOURCE_PREFETCH;
 	int pci_try_num = 1;
-	enum enable_type enable_local = pci_realloc_enable;
-
-	list_for_each_entry(bus, &pci_root_buses, node)
-		enable_local = pci_realloc_detect(bus, enable_local);
+	enum enable_type enable_local;
 
+	/* don't realloc if asked to do so */
+	enable_local = pci_realloc_detect(bus, pci_realloc_enable);
 	if (pci_realloc_enabled(enable_local)) {
-		int max_depth = pci_get_max_depth();
+		int max_depth = pci_bus_get_depth(bus);
 
 		pci_try_num = max_depth + 1;
-		printk(KERN_DEBUG "PCI: max bus depth: %d pci_try_num: %d\n",
-			 max_depth, pci_try_num);
+		dev_printk(KERN_DEBUG, &bus->dev,
+			   "max bus depth: %d pci_try_num: %d\n",
+			   max_depth, pci_try_num);
 	}
 
 again:
@@ -1441,12 +1425,10 @@ again:
 		add_list = &realloc_head;
 	/* Depth first, calculate sizes and alignments of all
 	   subordinate buses. */
-	list_for_each_entry(bus, &pci_root_buses, node)
-		__pci_bus_size_bridges(bus, add_list);
+	__pci_bus_size_bridges(bus, add_list);
 
 	/* Depth last, allocate resources and update the hardware. */
-	list_for_each_entry(bus, &pci_root_buses, node)
-		__pci_bus_assign_resources(bus, add_list, &fail_head);
+	__pci_bus_assign_resources(bus, add_list, &fail_head);
 	if (add_list)
 		BUG_ON(!list_empty(add_list));
 	tried_times++;
@@ -1457,16 +1439,16 @@ again:
 
 	if (tried_times >= pci_try_num) {
 		if (enable_local == undefined)
-			printk(KERN_INFO "Some PCI device resources are unassigned, try booting with pci=realloc\n");
+			dev_info(&bus->dev, "Some PCI device resources are unassigned, try booting with pci=realloc\n");
 		else if (enable_local == auto_enabled)
-			printk(KERN_INFO "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n");
+			dev_info(&bus->dev, "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n");
 
 		free_list(&fail_head);
 		goto enable_and_dump;
 	}
 
-	printk(KERN_DEBUG "PCI: No. %d try to assign unassigned res\n",
-			 tried_times + 1);
+	dev_printk(KERN_DEBUG, &bus->dev,
+		   "No. %d try to assign unassigned res\n", tried_times + 1);
 
 	/* third times and later will not check if it is leaf */
 	if ((tried_times + 1) > 2)
@@ -1497,12 +1479,18 @@ again:
 
 enable_and_dump:
 	/* Depth last, update the hardware. */
-	list_for_each_entry(bus, &pci_root_buses, node)
-		pci_enable_bridges(bus);
+	pci_enable_bridges(bus);
 
 	/* dump the resource on buses */
-	list_for_each_entry(bus, &pci_root_buses, node)
-		pci_bus_dump_resources(bus);
+	pci_bus_dump_resources(bus);
+}
+
+void __init pci_assign_unassigned_resources(void)
+{
+	struct pci_bus *root_bus;
+
+	list_for_each_entry(root_bus, &pci_root_buses, node)
+		pci_assign_unassigned_root_bus_resources(root_bus);
 }
 
 void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge)

[PATCH v4 5/8] PCI: Skip IORESOURCE_IO allocation for root bus without ioport range

[Yinghai Lu]

BenH reported that there is some assign unassigned resource problem
in powerpc.

It turns out after
| commit 0c5be0cb0edfe3b5c4b62eac68aa2aa15ec681af
| Date:   Thu Feb 23 19:23:29 2012 -0800
|
|    PCI: Retry on IORESOURCE_IO type allocations

even the root bus does not have io port range, it will keep retrying
to realloc with mmio.

Current retry logic is : try with must+optional at first, and if
it fails will try must then try to extend must with optional.
That will fail as mmio-non-pref and mmio-pref for bridge will
be next to each other. So we have no chance to extend mmio-non-pref.

This will be become more often when we have x86 8 sockets or 32 sockets
system, and those system will have one root bus per socket.
They will have some root buses do not have ioport range.

We should not fall into retry in this case, as root bus does
not io port range.

We check if the root bus has ioport range, and set bus_res_type_mask,
and pass it to assign_resources and don't add ioport res to failed list
for root bus that does not have ioport range.
So even BIOS set wrong value to pci devices and bridges will still
get cleared.

For the retry failing, we could allocate mmio-non-pref bottom-up
and mmio-pref will be top-down, but that is not easy and could not be
material for v3.10.

-v2: remove wrong __init with pci_bus_res_type_mask()
     don't check bridge size/flags, and clear bus io resources.
-v3: change to skip adding to failed_list instead.

Reported-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |   86 ++++++++++++++++++++++++++++++++++++------------
 1 file changed, 66 insertions(+), 20 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -272,7 +272,8 @@ out:
  * requests that could not satisfied to the failed_list.
  */
 static void assign_requested_resources_sorted(struct list_head *head,
-				 struct list_head *fail_head)
+				 struct list_head *fail_head,
+				 unsigned long bus_res_type_mask)
 {
 	struct resource *res;
 	struct pci_dev_resource *dev_res;
@@ -288,8 +289,19 @@ static void assign_requested_resources_s
 				 * if the failed res is for ROM BAR, and it will
 				 * be enabled later, don't add it to the list
 				 */
-				if (!((idx == PCI_ROM_RESOURCE) &&
-				      (!(res->flags & IORESOURCE_ROM_ENABLE))))
+				bool is_rom_res_not_enabled =
+					 (idx == PCI_ROM_RESOURCE) &&
+					 (!(res->flags & IORESOURCE_ROM_ENABLE));
+				/*
+				 * if the failed res is io port, but bus does
+				 * not have io port support, don't add it
+				 */
+				bool is_ioport_res_without_bus_support =
+					 (!(bus_res_type_mask & IORESOURCE_IO)) &&
+					 (res->flags & IORESOURCE_IO);
+
+				if (!is_rom_res_not_enabled &&
+				    !is_ioport_res_without_bus_support)
 					add_to_list(fail_head,
 						    dev_res->dev, res,
 						    0 /* dont care */,
@@ -302,7 +314,8 @@ static void assign_requested_resources_s
 
 static void __assign_resources_sorted(struct list_head *head,
 				 struct list_head *realloc_head,
-				 struct list_head *fail_head)
+				 struct list_head *fail_head,
+				 unsigned long bus_res_type_mask)
 {
 	/*
 	 * Should not assign requested resources at first.
@@ -336,7 +349,8 @@ static void __assign_resources_sorted(st
 							dev_res->res);
 
 	/* Try updated head list with add_size added */
-	assign_requested_resources_sorted(head, &local_fail_head);
+	assign_requested_resources_sorted(head, &local_fail_head,
+					  bus_res_type_mask);
 
 	/* all assigned with add_size ? */
 	if (list_empty(&local_fail_head)) {
@@ -365,7 +379,8 @@ static void __assign_resources_sorted(st
 
 requested_and_reassign:
 	/* Satisfy the must-have resource requests */
-	assign_requested_resources_sorted(head, fail_head);
+	assign_requested_resources_sorted(head, fail_head,
+					  bus_res_type_mask);
 
 	/* Try to satisfy any additional optional resource
 		requests */
@@ -376,18 +391,21 @@ requested_and_reassign:
 
 static void pdev_assign_resources_sorted(struct pci_dev *dev,
 				 struct list_head *add_head,
-				 struct list_head *fail_head)
+				 struct list_head *fail_head,
+				 unsigned long bus_res_type_mask)
 {
 	LIST_HEAD(head);
 
 	__dev_sort_resources(dev, &head);
-	__assign_resources_sorted(&head, add_head, fail_head);
+	__assign_resources_sorted(&head, add_head, fail_head,
+					bus_res_type_mask);
 
 }
 
 static void pbus_assign_resources_sorted(const struct pci_bus *bus,
 					 struct list_head *realloc_head,
-					 struct list_head *fail_head)
+					 struct list_head *fail_head,
+					 unsigned long bus_res_type_mask)
 {
 	struct pci_dev *dev;
 	LIST_HEAD(head);
@@ -395,7 +413,8 @@ static void pbus_assign_resources_sorted
 	list_for_each_entry(dev, &bus->devices, bus_list)
 		__dev_sort_resources(dev, &head);
 
-	__assign_resources_sorted(&head, realloc_head, fail_head);
+	__assign_resources_sorted(&head, realloc_head, fail_head,
+					bus_res_type_mask);
 }
 
 void pci_setup_cardbus(struct pci_bus *bus)
@@ -1117,19 +1136,22 @@ EXPORT_SYMBOL(pci_bus_size_bridges);
 
 static void __ref __pci_bus_assign_resources(const struct pci_bus *bus,
 					 struct list_head *realloc_head,
-					 struct list_head *fail_head)
+					 struct list_head *fail_head,
+					 unsigned long bus_res_type_mask)
 {
 	struct pci_bus *b;
 	struct pci_dev *dev;
 
-	pbus_assign_resources_sorted(bus, realloc_head, fail_head);
+	pbus_assign_resources_sorted(bus, realloc_head, fail_head,
+					bus_res_type_mask);
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
 		b = dev->subordinate;
 		if (!b)
 			continue;
 
-		__pci_bus_assign_resources(b, realloc_head, fail_head);
+		__pci_bus_assign_resources(b, realloc_head, fail_head,
+						 bus_res_type_mask);
 
 		switch (dev->class >> 8) {
 		case PCI_CLASS_BRIDGE_PCI:
@@ -1151,24 +1173,28 @@ static void __ref __pci_bus_assign_resou
 
 void __ref pci_bus_assign_resources(const struct pci_bus *bus)
 {
-	__pci_bus_assign_resources(bus, NULL, NULL);
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	__pci_bus_assign_resources(bus, NULL, NULL, type_mask);
 }
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
 static void __ref __pci_bridge_assign_resources(const struct pci_dev *bridge,
 					 struct list_head *add_head,
-					 struct list_head *fail_head)
+					 struct list_head *fail_head,
+					 unsigned long bus_res_type_mask)
 {
 	struct pci_bus *b;
 
 	pdev_assign_resources_sorted((struct pci_dev *)bridge,
-					 add_head, fail_head);
+				     add_head, fail_head, bus_res_type_mask);
 
 	b = bridge->subordinate;
 	if (!b)
 		return;
 
-	__pci_bus_assign_resources(b, add_head, fail_head);
+	__pci_bus_assign_resources(b, add_head, fail_head, bus_res_type_mask);
 
 	switch (bridge->class >> 8) {
 	case PCI_CLASS_BRIDGE_PCI:
@@ -1385,6 +1411,21 @@ static enum enable_type __init pci_reall
 }
 #endif
 
+static unsigned long pci_bus_res_type_mask(struct pci_bus *bus)
+{
+	int i;
+	struct resource *r;
+	unsigned long mask = 0;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	pci_bus_for_each_resource(bus, r, i)
+		if (r)
+			mask |= r->flags & type_mask;
+
+	return mask;
+}
+
 /*
  * first try will not touch pci bridge res
  * second  and later try will clear small leaf bridge res
@@ -1404,6 +1445,7 @@ pci_assign_unassigned_root_bus_resources
 				  IORESOURCE_PREFETCH;
 	int pci_try_num = 1;
 	enum enable_type enable_local;
+	unsigned long bus_res_type_mask = pci_bus_res_type_mask(bus);
 
 	/* don't realloc if asked to do so */
 	enable_local = pci_realloc_detect(bus, pci_realloc_enable);
@@ -1428,7 +1470,8 @@ again:
 	__pci_bus_size_bridges(bus, add_list);
 
 	/* Depth last, allocate resources and update the hardware. */
-	__pci_bus_assign_resources(bus, add_list, &fail_head);
+	__pci_bus_assign_resources(bus, add_list, &fail_head,
+				   bus_res_type_mask);
 	if (add_list)
 		BUG_ON(!list_empty(add_list));
 	tried_times++;
@@ -1504,10 +1547,12 @@ void pci_assign_unassigned_bridge_resour
 	int retval;
 	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
 				  IORESOURCE_PREFETCH;
+	unsigned long bus_res_type_mask = pci_bus_res_type_mask(bridge->bus);
 
 again:
 	__pci_bus_size_bridges(parent, &add_list);
-	__pci_bridge_assign_resources(bridge, &add_list, &fail_head);
+	__pci_bridge_assign_resources(bridge, &add_list, &fail_head,
+					bus_res_type_mask);
 	BUG_ON(!list_empty(&add_list));
 	tried_times++;
 
@@ -1562,6 +1607,7 @@ void pci_assign_unassigned_bus_resources
 	struct pci_dev *dev;
 	LIST_HEAD(add_list); /* list of resources that
 					want additional resources */
+	unsigned long bus_res_type_mask = pci_bus_res_type_mask(bus);
 
 	down_read(&pci_bus_sem);
 	list_for_each_entry(dev, &bus->devices, bus_list)
@@ -1571,6 +1617,6 @@ void pci_assign_unassigned_bus_resources
 				__pci_bus_size_bridges(dev->subordinate,
 							 &add_list);
 	up_read(&pci_bus_sem);
-	__pci_bus_assign_resources(bus, &add_list, NULL);
+	__pci_bus_assign_resources(bus, &add_list, NULL, bus_res_type_mask);
 	BUG_ON(!list_empty(&add_list));
 }

[PATCH v4 6/8] PCI: Skip IORESOURCE_MMIO allocation for root bus without MMIO range

[Yinghai Lu]

For x86 8 sockets or 32 sockets system that will have one root bus per socket,
They may have some root buses do not have mmio non-pref range.

We should not fall into retry in this case, as root bus does
not mmio non-pref range.

We check if the root bus has mmio-nonpref range, and set bus_res_type_mask,
and pass it to assign_resources and don't add mmio-nonpref res to failed list
for root bus that does not have mmio-nonpref range.
So even BIOS set wrong value to pci devices and bridges will still
get cleared.

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |   32 ++++++++++++++++++++++++++------
 1 file changed, 26 insertions(+), 6 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -299,9 +299,17 @@ static void assign_requested_resources_s
 				bool is_ioport_res_without_bus_support =
 					 (!(bus_res_type_mask & IORESOURCE_IO)) &&
 					 (res->flags & IORESOURCE_IO);
+				/*
+				 * if the failed res is mmio, but bus does
+				 * not have io port support, don't add it
+				 */
+				bool is_mmio_nonpref_res_without_bus_support =
+					 (!(bus_res_type_mask & IORESOURCE_MEM)) &&
+					 ((res->flags & (IORESOURCE_MEM | IORESOURCE_PREFETCH)) == IORESOURCE_MEM);
 
 				if (!is_rom_res_not_enabled &&
-				    !is_ioport_res_without_bus_support)
+				    !is_ioport_res_without_bus_support &&
+				    !is_mmio_nonpref_res_without_bus_support)
 					add_to_list(fail_head,
 						    dev_res->dev, res,
 						    0 /* dont care */,
@@ -1416,12 +1424,24 @@ static unsigned long pci_bus_res_type_ma
 	int i;
 	struct resource *r;
 	unsigned long mask = 0;
-	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
-				  IORESOURCE_PREFETCH;
 
-	pci_bus_for_each_resource(bus, r, i)
-		if (r)
-			mask |= r->flags & type_mask;
+	pci_bus_for_each_resource(bus, r, i) {
+		if (!r)
+			continue;
+
+		if (r->flags & IORESOURCE_IO) {
+			mask |= IORESOURCE_IO;
+			continue;
+		}
+		if (r->flags & IORESOURCE_PREFETCH) {
+			mask |= IORESOURCE_PREFETCH;
+			continue;
+		}
+		if ((r->flags & (IORESOURCE_MEM | IORESOURCE_PREFETCH)) == IORESOURCE_MEM) {
+			mask |= IORESOURCE_MEM; /* nonpref only */
+			continue;
+		}
+	}
 
 	return mask;
 }

[PATCH v4 7/8] PCI: Enable pci bridge when it is needed

[Yinghai Lu]

Current we enable bridges after bus scan and assign resources.
and it is spreaded a lot of places.

We can move it to where pci device is enabled, and need
to go up to root bus and enable bridge one by one down to pci
dev.

So that will delay enable bridge late as needed bassis,
also kill one inconsistent between boot path and hotplug
path in acpi_pci_root_add().

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 arch/arm/kernel/bios32.c           |    5 -----
 arch/m68k/platform/coldfire/pci.c  |    1 -
 arch/mips/pci/pci.c                |    1 -
 arch/sh/drivers/pci/pci.c          |    1 -
 drivers/acpi/pci_root.c            |    4 ----
 drivers/parisc/lba_pci.c           |    1 -
 drivers/pci/bus.c                  |   19 -------------------
 drivers/pci/hotplug/acpiphp_glue.c |    1 -
 drivers/pci/pci.c                  |   20 ++++++++++++++++++++
 drivers/pci/probe.c                |    1 -
 drivers/pci/setup-bus.c            |   10 +++-------
 drivers/pcmcia/cardbus.c           |    1 -
 include/linux/pci.h                |    1 -
 13 files changed, 23 insertions(+), 43 deletions(-)

Index: linux-2.6/arch/arm/kernel/bios32.c
===================================================================
--- linux-2.6.orig/arch/arm/kernel/bios32.c
+++ linux-2.6/arch/arm/kernel/bios32.c
@@ -524,11 +524,6 @@ void pci_common_init(struct hw_pci *hw)
 			 * Assign resources.
 			 */
 			pci_bus_assign_resources(bus);
-
-			/*
-			 * Enable bridges
-			 */
-			pci_enable_bridges(bus);
 		}
 
 		/*
Index: linux-2.6/arch/m68k/platform/coldfire/pci.c
===================================================================
--- linux-2.6.orig/arch/m68k/platform/coldfire/pci.c
+++ linux-2.6/arch/m68k/platform/coldfire/pci.c
@@ -319,7 +319,6 @@ static int __init mcf_pci_init(void)
 	pci_fixup_irqs(pci_common_swizzle, mcf_pci_map_irq);
 	pci_bus_size_bridges(rootbus);
 	pci_bus_assign_resources(rootbus);
-	pci_enable_bridges(rootbus);
 	pci_bus_add_devices(rootbus);
 	return 0;
 }
Index: linux-2.6/arch/mips/pci/pci.c
===================================================================
--- linux-2.6.orig/arch/mips/pci/pci.c
+++ linux-2.6/arch/mips/pci/pci.c
@@ -113,7 +113,6 @@ static void pcibios_scanbus(struct pci_c
 		if (!pci_has_flag(PCI_PROBE_ONLY)) {
 			pci_bus_size_bridges(bus);
 			pci_bus_assign_resources(bus);
-			pci_enable_bridges(bus);
 		}
 	}
 }
Index: linux-2.6/arch/sh/drivers/pci/pci.c
===================================================================
--- linux-2.6.orig/arch/sh/drivers/pci/pci.c
+++ linux-2.6/arch/sh/drivers/pci/pci.c
@@ -69,7 +69,6 @@ static void pcibios_scanbus(struct pci_c
 
 		pci_bus_size_bridges(bus);
 		pci_bus_assign_resources(bus);
-		pci_enable_bridges(bus);
 	} else {
 		pci_free_resource_list(&resources);
 	}
Index: linux-2.6/drivers/acpi/pci_root.c
===================================================================
--- linux-2.6.orig/drivers/acpi/pci_root.c
+++ linux-2.6/drivers/acpi/pci_root.c
@@ -537,10 +537,6 @@ static int acpi_pci_root_add(struct acpi
 		pci_assign_unassigned_bus_resources(root->bus);
 	}
 
-	/* need to after hot-added ioapic is registered */
-	if (system_state != SYSTEM_BOOTING)
-		pci_enable_bridges(root->bus);
-
 	pci_bus_add_devices(root->bus);
 	return 1;
 
Index: linux-2.6/drivers/parisc/lba_pci.c
===================================================================
--- linux-2.6.orig/drivers/parisc/lba_pci.c
+++ linux-2.6/drivers/parisc/lba_pci.c
@@ -1533,7 +1533,6 @@ lba_driver_probe(struct parisc_device *d
 		lba_dump_res(&lba_dev->hba.lmmio_space, 2);
 #endif
 	}
-	pci_enable_bridges(lba_bus);
 
 	/*
 	** Once PCI register ops has walked the bus, access to config
Index: linux-2.6/drivers/pci/bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/bus.c
+++ linux-2.6/drivers/pci/bus.c
@@ -216,24 +216,6 @@ void pci_bus_add_devices(const struct pc
 	}
 }
 
-void pci_enable_bridges(struct pci_bus *bus)
-{
-	struct pci_dev *dev;
-	int retval;
-
-	list_for_each_entry(dev, &bus->devices, bus_list) {
-		if (dev->subordinate) {
-			if (!pci_is_enabled(dev)) {
-				retval = pci_enable_device(dev);
-				if (retval)
-					dev_err(&dev->dev, "Error enabling bridge (%d), continuing\n", retval);
-				pci_set_master(dev);
-			}
-			pci_enable_bridges(dev->subordinate);
-		}
-	}
-}
-
 /** pci_walk_bus - walk devices on/under bus, calling callback.
  *  @top      bus whose devices should be walked
  *  @cb       callback to be called for each device found
@@ -286,4 +268,3 @@ EXPORT_SYMBOL_GPL(pci_walk_bus);
 EXPORT_SYMBOL(pci_bus_alloc_resource);
 EXPORT_SYMBOL_GPL(pci_bus_add_device);
 EXPORT_SYMBOL(pci_bus_add_devices);
-EXPORT_SYMBOL(pci_enable_bridges);
Index: linux-2.6/drivers/pci/hotplug/acpiphp_glue.c
===================================================================
--- linux-2.6.orig/drivers/pci/hotplug/acpiphp_glue.c
+++ linux-2.6/drivers/pci/hotplug/acpiphp_glue.c
@@ -704,7 +704,6 @@ static int __ref enable_device(struct ac
 	acpiphp_sanitize_bus(bus);
 	acpiphp_set_hpp_values(bus);
 	acpiphp_set_acpi_region(slot);
-	pci_enable_bridges(bus);
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
 		/* Assume that newly added devices are powered on already. */
Index: linux-2.6/drivers/pci/pci.c
===================================================================
--- linux-2.6.orig/drivers/pci/pci.c
+++ linux-2.6/drivers/pci/pci.c
@@ -1145,6 +1145,24 @@ int pci_reenable_device(struct pci_dev *
 	return 0;
 }
 
+static void pci_enable_bridge(struct pci_dev *dev)
+{
+	int retval;
+
+	if (!dev)
+		return;
+
+	pci_enable_bridge(dev->bus->self);
+
+	if (pci_is_enabled(dev))
+		return;
+	retval = pci_enable_device(dev);
+	if (retval)
+		dev_err(&dev->dev, "Error enabling bridge (%d), continuing\n",
+			retval);
+	pci_set_master(dev);
+}
+
 static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags)
 {
 	int err;
@@ -1165,6 +1183,8 @@ static int pci_enable_device_flags(struc
 	if (atomic_inc_return(&dev->enable_cnt) > 1)
 		return 0;		/* already enabled */
 
+	pci_enable_bridge(dev->bus->self);
+
 	/* only skip sriov related */
 	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
 		if (dev->resource[i].flags & flags)
Index: linux-2.6/drivers/pci/probe.c
===================================================================
--- linux-2.6.orig/drivers/pci/probe.c
+++ linux-2.6/drivers/pci/probe.c
@@ -1948,7 +1948,6 @@ unsigned int __ref pci_rescan_bus(struct
 
 	max = pci_scan_child_bus(bus);
 	pci_assign_unassigned_bus_resources(bus);
-	pci_enable_bridges(bus);
 	pci_bus_add_devices(bus);
 
 	return max;
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1498,7 +1498,7 @@ again:
 
 	/* any device complain? */
 	if (list_empty(&fail_head))
-		goto enable_and_dump;
+		goto dump;
 
 	if (tried_times >= pci_try_num) {
 		if (enable_local == undefined)
@@ -1507,7 +1507,7 @@ again:
 			dev_info(&bus->dev, "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n");
 
 		free_list(&fail_head);
-		goto enable_and_dump;
+		goto dump;
 	}
 
 	dev_printk(KERN_DEBUG, &bus->dev,
@@ -1540,10 +1540,7 @@ again:
 
 	goto again;
 
-enable_and_dump:
-	/* Depth last, update the hardware. */
-	pci_enable_bridges(bus);
-
+dump:
 	/* dump the resource on buses */
 	pci_bus_dump_resources(bus);
 }
@@ -1618,7 +1615,6 @@ enable_all:
 	if (retval)
 		dev_err(&bridge->dev, "Error reenabling bridge (%d)\n", retval);
 	pci_set_master(bridge);
-	pci_enable_bridges(parent);
 }
 EXPORT_SYMBOL_GPL(pci_assign_unassigned_bridge_resources);
 
Index: linux-2.6/drivers/pcmcia/cardbus.c
===================================================================
--- linux-2.6.orig/drivers/pcmcia/cardbus.c
+++ linux-2.6/drivers/pcmcia/cardbus.c
@@ -91,7 +91,6 @@ int __ref cb_alloc(struct pcmcia_socket
 	if (s->tune_bridge)
 		s->tune_bridge(s, bus);
 
-	pci_enable_bridges(bus);
 	pci_bus_add_devices(bus);
 
 	return 0;
Index: linux-2.6/include/linux/pci.h
===================================================================
--- linux-2.6.orig/include/linux/pci.h
+++ linux-2.6/include/linux/pci.h
@@ -1040,7 +1040,6 @@ int __must_check pci_bus_alloc_resource(
 						  resource_size_t,
 						  resource_size_t),
 			void *alignf_data);
-void pci_enable_bridges(struct pci_bus *bus);
 
 /* Proper probing supporting hot-pluggable devices */
 int __must_check __pci_register_driver(struct pci_driver *, struct module *,

[PATCH v4 8/8] PCI: Retry assign unassigned resources for hotadd root bus

[Yinghai Lu]

Let root bus hotadd path use same code for booting path.
As driver is not loaded yet, we could retry to make sure
all pci devices get resources allocated.

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/acpi/pci_root.c |    2 +-
 drivers/pci/setup-bus.c |   15 +++++++--------
 include/linux/pci.h     |    1 +
 3 files changed, 9 insertions(+), 9 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1331,7 +1331,7 @@ static void pci_bus_dump_resources(struc
 	}
 }
 
-static int __init pci_bus_get_depth(struct pci_bus *bus)
+static int pci_bus_get_depth(struct pci_bus *bus)
 {
 	int depth = 0;
 	struct pci_dev *dev;
@@ -1365,7 +1365,7 @@ enum enable_type {
 	auto_enabled,
 };
 
-static enum enable_type pci_realloc_enable __initdata = undefined;
+static enum enable_type pci_realloc_enable = undefined;
 void __init pci_realloc_get_opt(char *str)
 {
 	if (!strncmp(str, "off", 3))
@@ -1373,13 +1373,13 @@ void __init pci_realloc_get_opt(char *st
 	else if (!strncmp(str, "on", 2))
 		pci_realloc_enable = user_enabled;
 }
-static bool __init pci_realloc_enabled(enum enable_type enable)
+static bool pci_realloc_enabled(enum enable_type enable)
 {
 	return enable >= user_enabled;
 }
 
 #if defined(CONFIG_PCI_IOV) && defined(CONFIG_PCI_REALLOC_ENABLE_AUTO)
-static int __init check_unassigned_resources(struct pci_dev *dev, void *data)
+static int check_unassigned_resources(struct pci_dev *dev, void *data)
 {
 	int i;
 	int *unassigned = data;
@@ -1397,7 +1397,7 @@ static int __init check_unassigned_resou
 	return 0;
 }
 
-static enum enable_type __init pci_realloc_detect(struct pci_bus *bus,
+static enum enable_type pci_realloc_detect(struct pci_bus *bus,
 			 enum enable_type enable_local)
 {
 	int unassigned = 0;
@@ -1412,7 +1412,7 @@ static enum enable_type __init pci_reall
 	return enable_local;
 }
 #else
-static enum enable_type __init pci_realloc_detect(struct pci_bus *bus,
+static enum enable_type pci_realloc_detect(struct pci_bus *bus,
 			 enum enable_type enable_local)
 {
 	return enable_local;
@@ -1451,8 +1451,7 @@ static unsigned long pci_bus_res_type_ma
  * second  and later try will clear small leaf bridge res
  * will stop till to the max  deepth if can not find good one
  */
-static void __init
-pci_assign_unassigned_root_bus_resources(struct pci_bus *bus)
+void pci_assign_unassigned_root_bus_resources(struct pci_bus *bus)
 {
 	LIST_HEAD(realloc_head); /* list of resources that
 					want additional resources */
Index: linux-2.6/drivers/acpi/pci_root.c
===================================================================
--- linux-2.6.orig/drivers/acpi/pci_root.c
+++ linux-2.6/drivers/acpi/pci_root.c
@@ -534,7 +534,7 @@ static int acpi_pci_root_add(struct acpi
 
 	if (system_state != SYSTEM_BOOTING) {
 		pcibios_resource_survey_bus(root->bus);
-		pci_assign_unassigned_bus_resources(root->bus);
+		pci_assign_unassigned_root_bus_resources(root->bus);
 	}
 
 	pci_bus_add_devices(root->bus);
Index: linux-2.6/include/linux/pci.h
===================================================================
--- linux-2.6.orig/include/linux/pci.h
+++ linux-2.6/include/linux/pci.h
@@ -1002,6 +1002,7 @@ int pci_claim_resource(struct pci_dev *,
 void pci_assign_unassigned_resources(void);
 void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge);
 void pci_assign_unassigned_bus_resources(struct pci_bus *bus);
+void pci_assign_unassigned_root_bus_resources(struct pci_bus *bus);
 void pdev_enable_device(struct pci_dev *);
 int pci_enable_resources(struct pci_dev *, int mask);
 void pci_fixup_irqs(u8 (*)(struct pci_dev *, u8 *),

[PATCH v5 0/7] PCI: Change assign unassigned resources per root bus bassis

[Yinghai Lu]

BenH reported that there is some assign unassigned resource problem
in powerpc.

It turns out after
| commit 0c5be0cb0edfe3b5c4b62eac68aa2aa15ec681af
| Date:   Thu Feb 23 19:23:29 2012 -0800
|
|    PCI: Retry on IORESOURCE_IO type allocations

even the root bus does not have io port range, it will keep retrying
to realloc with mmio.

After checking the code, found that we bound io port and mmio fail
path together.
First patch fix the problem, that will not make mmio fall back to must-only
when only have io port fail with must+optional.

During we found the fix for that problem, found that we can separate assign
unassigned resources to per root bus.
that will make the code simple, also could reuse it for hotadd path.

These patches are targeted to 3.11

-v4: split first patch into 4 patches per Bjorn.
-v5: drop two patches that will pass root bus resource mask after we found
     simple and less intrusive way to fix the problem.

 PCI: Don't let mmio fallback to must-only, if ioport fails with must+optional
 PCI: Don't use temp bus for pci_bus_release_bridge_resources
 PCI: Use pci_walk_bus to detect unassigned resources
 PCI: Introduce enable_local to prepare per root bus handling
 PCI: Split pci_assign_unassigned_resources to per root bus
 PCI: Enable pci bridge when it is needed
 PCI: Retry assign unassigned resources for hotadd root bus

Thanks

Yinghai

[PATCH v5 2/7] PCI: Don't use temp bus for pci_bus_release_bridge_resources

[Yinghai Lu]

as later bus can not be used as temp variable after we change to
per root bus handling with assign unassigned resources.

Per Bjorn, separate it from big patch that handing assign_unssigned per root bus.

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |    7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1526,12 +1526,11 @@ again:
 	 * Try to release leaf bridge's resources that doesn't fit resource of
 	 * child device under that bridge
 	 */
-	list_for_each_entry(fail_res, &fail_head, list) {
-		bus = fail_res->dev->bus;
-		pci_bus_release_bridge_resources(bus,
+	list_for_each_entry(fail_res, &fail_head, list)
+		pci_bus_release_bridge_resources(fail_res->dev->bus,
 						 fail_res->flags & type_mask,
 						 rel_type);
-	}
+
 	/* restore size and flags */
 	list_for_each_entry(fail_res, &fail_head, list) {
 		struct resource *res = fail_res->res;

[PATCH v5 3/7] PCI: Use pci_walk_bus to detect unassigned resources

[Yinghai Lu]

Per Bjorn, use pci_walk_bus instead of for_each_pci_dev or
calling pci_realloc_detect() recursively, that will make code more readable.

Per Bjorn, separate it from big patch that handing assign_unssigned per root bus.

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |   46 +++++++++++++++++++++++++++++++---------------
 1 file changed, 31 insertions(+), 15 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1427,30 +1427,46 @@ static bool __init pci_realloc_enabled(v
 	return pci_realloc_enable >= user_enabled;
 }
 
-static void __init pci_realloc_detect(void)
-{
 #if defined(CONFIG_PCI_IOV) && defined(CONFIG_PCI_REALLOC_ENABLE_AUTO)
-	struct pci_dev *dev = NULL;
+static int __init check_unassigned_resources(struct pci_dev *dev, void *data)
+{
+	int i;
+	int *unassigned = data;
 
-	if (pci_realloc_enable != undefined)
-		return;
+	for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++) {
+		struct resource *r = &dev->resource[i];
 
-	for_each_pci_dev(dev) {
-		int i;
+		/* Not assigned, or rejected by kernel ? */
+		if (r->flags && !r->start) {
+			(*unassigned)++;
+			return 1; /* return early from pci_walk_bus */
+		}
+	}
 
-		for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++) {
-			struct resource *r = &dev->resource[i];
+	return 0;
+}
 
-			/* Not assigned, or rejected by kernel ? */
-			if (r->flags && !r->start) {
-				pci_realloc_enable = auto_enabled;
+static void  __init pci_realloc_detect(void)
+{
+	int unassigned = 0;
+	struct pci_bus *bus;
 
-				return;
-			}
+	if (pci_realloc_enable != undefined)
+		return;
+
+	list_for_each_entry(bus, &pci_root_buses, node) {
+		pci_walk_bus(bus, check_unassigned_resources, &unassigned);
+		if (unassigned) {
+			pci_realloc_enable = auto_enabled;
+			return;
 		}
 	}
-#endif
 }
+#else
+static void __init pci_realloc_detect(void)
+{
+}
+#endif
 
 /*
  * first try will not touch pci bridge res

Re: [PATCH v5 3/7] PCI: Use pci_walk_bus to detect unassigned resources

[Bjorn Helgaas]

On Fri, May 31, 2013 at 11:03:08PM -0700, Yinghai Lu wrote:
> Per Bjorn, use pci_walk_bus instead of for_each_pci_dev or
> calling pci_realloc_detect() recursively, that will make code more readable.
> 
> Per Bjorn, separate it from big patch that handing assign_unssigned per root bus.
> 
> Signed-off-by: Yinghai Lu <yinghai@kernel.org>
> 
> ---
>  drivers/pci/setup-bus.c |   46 +++++++++++++++++++++++++++++++---------------
>  1 file changed, 31 insertions(+), 15 deletions(-)
> 
> Index: linux-2.6/drivers/pci/setup-bus.c
> ===================================================================
> --- linux-2.6.orig/drivers/pci/setup-bus.c
> +++ linux-2.6/drivers/pci/setup-bus.c
> @@ -1427,30 +1427,46 @@ static bool __init pci_realloc_enabled(v
>  	return pci_realloc_enable >= user_enabled;
>  }
>  
> -static void __init pci_realloc_detect(void)
> -{
>  #if defined(CONFIG_PCI_IOV) && defined(CONFIG_PCI_REALLOC_ENABLE_AUTO)
> -	struct pci_dev *dev = NULL;
> +static int __init check_unassigned_resources(struct pci_dev *dev, void *data)

I'm not going to add a function named "check_*()" because the name gives no
clue about what the return value means.  If it's a boolean function, the
name should be something like a question that has a yes/no answer.

> +{
> +	int i;
> +	int *unassigned = data;
>  
> -	if (pci_realloc_enable != undefined)
> -		return;
> +	for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++) {
> +		struct resource *r = &dev->resource[i];
>  
> -	for_each_pci_dev(dev) {
> -		int i;
> +		/* Not assigned, or rejected by kernel ? */
> +		if (r->flags && !r->start) {
> +			(*unassigned)++;
> +			return 1; /* return early from pci_walk_bus */
> +		}
> +	}
>  
> -		for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++) {
> -			struct resource *r = &dev->resource[i];
> +	return 0;
> +}
>  
> -			/* Not assigned, or rejected by kernel ? */
> -			if (r->flags && !r->start) {
> -				pci_realloc_enable = auto_enabled;
> +static void  __init pci_realloc_detect(void)
> +{
> +	int unassigned = 0;
> +	struct pci_bus *bus;
>  
> -				return;
> -			}
> +	if (pci_realloc_enable != undefined)
> +		return;
> +
> +	list_for_each_entry(bus, &pci_root_buses, node) {
> +		pci_walk_bus(bus, check_unassigned_resources, &unassigned);
> +		if (unassigned) {
> +			pci_realloc_enable = auto_enabled;
> +			return;
>  		}
>  	}
> -#endif
>  }
> +#else
> +static void __init pci_realloc_detect(void)
> +{
> +}
> +#endif
>  
>  /*
>   * first try will not touch pci bridge res

Re: [PATCH v5 3/7] PCI: Use pci_walk_bus to detect unassigned resources

[Benjamin Herrenschmidt]

On Tue, 2013-06-25 at 15:15 -0600, Bjorn Helgaas wrote:
> -     for_each_pci_dev(dev) {
> > -             int i;
> > +             /* Not assigned, or rejected by kernel ? */
> > +             if (r->flags && !r->start) {
> > +                     (*unassigned)++;
> > +                     return 1; /* return early from pci_walk_bus */
> > +             }
> > +     }

BTW. I'm aware you didn't change that logic but ... it's somewhat broken
in the case where the aperture has an offset. You should compare
r->start with the offset, not with 0.

Cheers,
Ben.

Re: [PATCH v5 3/7] PCI: Use pci_walk_bus to detect unassigned resources

[Bjorn Helgaas]

On Tue, Jun 25, 2013 at 3:38 PM, Benjamin Herrenschmidt
<benh@kernel.crashing.org> wrote:
> On Tue, 2013-06-25 at 15:15 -0600, Bjorn Helgaas wrote:
>> -     for_each_pci_dev(dev) {
>> > -             int i;
>> > +             /* Not assigned, or rejected by kernel ? */
>> > +             if (r->flags && !r->start) {
>> > +                     (*unassigned)++;
>> > +                     return 1; /* return early from pci_walk_bus */
>> > +             }
>> > +     }
>
> BTW. I'm aware you didn't change that logic but ... it's somewhat broken
> in the case where the aperture has an offset. You should compare
> r->start with the offset, not with 0.

Yes, please fix that in a separate patch that contains only the bugfix.

Re: [PATCH v5 3/7] PCI: Use pci_walk_bus to detect unassigned resources

[Yinghai Lu]

[-- Attachment #1: Type: text/plain, Size: 2240 bytes --]

On Tue, Jun 25, 2013 at 2:46 PM, Bjorn Helgaas <bhelgaas@google.com> wrote:
> On Tue, Jun 25, 2013 at 3:38 PM, Benjamin Herrenschmidt
> <benh@kernel.crashing.org> wrote:
>> On Tue, 2013-06-25 at 15:15 -0600, Bjorn Helgaas wrote:
>>> -     for_each_pci_dev(dev) {
>>> > -             int i;
>>> > +             /* Not assigned, or rejected by kernel ? */
>>> > +             if (r->flags && !r->start) {
>>> > +                     (*unassigned)++;
>>> > +                     return 1; /* return early from pci_walk_bus */
>>> > +             }
>>> > +     }
>>
>> BTW. I'm aware you didn't change that logic but ... it's somewhat broken
>> in the case where the aperture has an offset. You should compare
>> r->start with the offset, not with 0.
>
> Yes, please fix that in a separate patch that contains only the bugfix.

Please check inline word warped patch.

Subject: [PATCH] PCI: check pci bus address for unassigned res

We should compare res->start with root bus window offset.
Otherwise will have problem with arch that support hostbridge
resource offset.

BenH pointed out that during reviewing patchset that separate
assign unassigned to per root buses.

According to Bjorn, have it in separated patch.

Use pcibios_resource_to_bus to get region at first, and check
region.start instead.

Suggested-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |    7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1420,9 +1420,14 @@ static int check_unassigned_resources(st

     for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++) {
         struct resource *r = &dev->resource[i];
+        struct pci_bus_region region;

         /* Not assigned, or rejected by kernel ? */
-        if (r->flags && !r->start) {
+        if (!r->flags)
+            continue;
+
+        pcibios_resource_to_bus(dev, &region, res);
+        if (!region.start) {
             (*unassigned)++;
             return 1; /* return early from pci_walk_bus */
         }

[-- Attachment #2: root_bus_ioport_skip_2_a.patch --]
[-- Type: application/octet-stream, Size: 1300 bytes --]

Subject: [PATCH] PCI: check pci bus address for unassigned res

We should compare res->start with root bus window offset.
Otherwise will have problem with arch that support hostbridge
resource offset.

BenH pointed out that during reviewing patchset that separate
assign unassigned to per root buses.

According to Bjorn, have it in separated patch.

Use pcibios_resource_to_bus to get region at first, and check
region.start instead.

Suggested-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |    7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1420,9 +1420,14 @@ static int check_unassigned_resources(st
 
 	for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++) {
 		struct resource *r = &dev->resource[i];
+		struct pci_bus_region region;
 
 		/* Not assigned, or rejected by kernel ? */
-		if (r->flags && !r->start) {
+		if (!r->flags)
+			continue;
+
+		pcibios_resource_to_bus(dev, &region, res);
+		if (!region.start) {
 			(*unassigned)++;
 			return 1; /* return early from pci_walk_bus */
 		}

Re: [PATCH v5 3/7] PCI: Use pci_walk_bus to detect unassigned resources

[Yinghai Lu]

On Tue, Jun 25, 2013 at 2:15 PM, Bjorn Helgaas <bhelgaas@google.com> wrote:
>> +static int __init check_unassigned_resources(struct pci_dev *dev, void *data)
>
> I'm not going to add a function named "check_*()" because the name gives no
> clue about what the return value means.  If it's a boolean function, the
> name should be something like a question that has a yes/no answer.

that prototype return int is required by pci_walk_bus().

drivers/pci/bus.c:void pci_walk_bus(struct pci_bus *top, int
(*cb)(struct pci_dev *, void *)

return 1, will return early from pci_walk_bus().

count_unassigned_resources() is not good name too, as we bail out early.
find_unassigned_resources() is more weird, looks like it want to return resource

Yinghai

[PATCH v5 4/7] PCI: Introduce enable_local to prepare per root bus handling

[Yinghai Lu]

Add enable_local to prepare assign unassigned resource
for per root bus.

Per Bjorn, separate it from big patch that handing assign_unssigned per root bus.

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |   40 +++++++++++++++++++++-------------------
 1 file changed, 21 insertions(+), 19 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1422,9 +1422,9 @@ void __init pci_realloc_get_opt(char *st
 	else if (!strncmp(str, "on", 2))
 		pci_realloc_enable = user_enabled;
 }
-static bool __init pci_realloc_enabled(void)
+static bool __init pci_realloc_enabled(enum enable_type enable)
 {
-	return pci_realloc_enable >= user_enabled;
+	return enable >= user_enabled;
 }
 
 #if defined(CONFIG_PCI_IOV) && defined(CONFIG_PCI_REALLOC_ENABLE_AUTO)
@@ -1446,25 +1446,25 @@ static int __init check_unassigned_resou
 	return 0;
 }
 
-static void  __init pci_realloc_detect(void)
+static enum enable_type __init pci_realloc_detect(struct pci_bus *bus,
+			 enum enable_type enable_local)
 {
 	int unassigned = 0;
-	struct pci_bus *bus;
 
-	if (pci_realloc_enable != undefined)
-		return;
+	if (enable_local != undefined)
+		return enable_local;
 
-	list_for_each_entry(bus, &pci_root_buses, node) {
-		pci_walk_bus(bus, check_unassigned_resources, &unassigned);
-		if (unassigned) {
-			pci_realloc_enable = auto_enabled;
-			return;
-		}
-	}
+	pci_walk_bus(bus, check_unassigned_resources, &unassigned);
+	if (unassigned)
+		return auto_enabled;
+
+	return enable_local;
 }
 #else
-static void __init pci_realloc_detect(void)
+static enum enable_type __init pci_realloc_detect(struct pci_bus *bus,
+			 enum enable_type enable_local)
 {
+	return enable_local;
 }
 #endif
 
@@ -1487,10 +1487,12 @@ pci_assign_unassigned_resources(void)
 	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
 				  IORESOURCE_PREFETCH;
 	int pci_try_num = 1;
+	enum enable_type enable_local = pci_realloc_enable;
+
+	list_for_each_entry(bus, &pci_root_buses, node)
+		enable_local = pci_realloc_detect(bus, enable_local);
 
-	/* don't realloc if asked to do so */
-	pci_realloc_detect();
-	if (pci_realloc_enabled()) {
+	if (pci_realloc_enabled(enable_local)) {
 		int max_depth = pci_get_max_depth();
 
 		pci_try_num = max_depth + 1;
@@ -1522,9 +1524,9 @@ again:
 		goto enable_and_dump;
 
 	if (tried_times >= pci_try_num) {
-		if (pci_realloc_enable == undefined)
+		if (enable_local == undefined)
 			printk(KERN_INFO "Some PCI device resources are unassigned, try booting with pci=realloc\n");
-		else if (pci_realloc_enable == auto_enabled)
+		else if (enable_local == auto_enabled)
 			printk(KERN_INFO "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n");
 
 		free_list(&fail_head);

[PATCH v5 5/7] PCI: Split pci_assign_unassigned_resources to per root bus

[Yinghai Lu]

We need to split pci_assign_unassiged_resource to every root bus, so can
have different retry for assign_unassigned per root bus

Also we need root bus hot add and booting path use same code.

-v2: separate enable_local and pci_release_bridge_resources to
     other patches requested by Bjorn.

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |   62 +++++++++++++++++++-----------------------------
 1 file changed, 25 insertions(+), 37 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1383,21 +1383,6 @@ static int __init pci_bus_get_depth(stru
 
 	return depth;
 }
-static int __init pci_get_max_depth(void)
-{
-	int depth = 0;
-	struct pci_bus *bus;
-
-	list_for_each_entry(bus, &pci_root_buses, node) {
-		int ret;
-
-		ret = pci_bus_get_depth(bus);
-		if (ret > depth)
-			depth = ret;
-	}
-
-	return depth;
-}
 
 /*
  * -1: undefined, will auto detect later
@@ -1473,10 +1458,9 @@ static enum enable_type __init pci_reall
  * second  and later try will clear small leaf bridge res
  * will stop till to the max  deepth if can not find good one
  */
-void __init
-pci_assign_unassigned_resources(void)
+static void __init
+pci_assign_unassigned_root_bus_resources(struct pci_bus *bus)
 {
-	struct pci_bus *bus;
 	LIST_HEAD(realloc_head); /* list of resources that
 					want additional resources */
 	struct list_head *add_list = NULL;
@@ -1487,17 +1471,17 @@ pci_assign_unassigned_resources(void)
 	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
 				  IORESOURCE_PREFETCH;
 	int pci_try_num = 1;
-	enum enable_type enable_local = pci_realloc_enable;
-
-	list_for_each_entry(bus, &pci_root_buses, node)
-		enable_local = pci_realloc_detect(bus, enable_local);
+	enum enable_type enable_local;
 
+	/* don't realloc if asked to do so */
+	enable_local = pci_realloc_detect(bus, pci_realloc_enable);
 	if (pci_realloc_enabled(enable_local)) {
-		int max_depth = pci_get_max_depth();
+		int max_depth = pci_bus_get_depth(bus);
 
 		pci_try_num = max_depth + 1;
-		printk(KERN_DEBUG "PCI: max bus depth: %d pci_try_num: %d\n",
-			 max_depth, pci_try_num);
+		dev_printk(KERN_DEBUG, &bus->dev,
+			   "max bus depth: %d pci_try_num: %d\n",
+			   max_depth, pci_try_num);
 	}
 
 again:
@@ -1509,12 +1493,10 @@ again:
 		add_list = &realloc_head;
 	/* Depth first, calculate sizes and alignments of all
 	   subordinate buses. */
-	list_for_each_entry(bus, &pci_root_buses, node)
-		__pci_bus_size_bridges(bus, add_list);
+	__pci_bus_size_bridges(bus, add_list);
 
 	/* Depth last, allocate resources and update the hardware. */
-	list_for_each_entry(bus, &pci_root_buses, node)
-		__pci_bus_assign_resources(bus, add_list, &fail_head);
+	__pci_bus_assign_resources(bus, add_list, &fail_head);
 	if (add_list)
 		BUG_ON(!list_empty(add_list));
 	tried_times++;
@@ -1525,16 +1507,16 @@ again:
 
 	if (tried_times >= pci_try_num) {
 		if (enable_local == undefined)
-			printk(KERN_INFO "Some PCI device resources are unassigned, try booting with pci=realloc\n");
+			dev_info(&bus->dev, "Some PCI device resources are unassigned, try booting with pci=realloc\n");
 		else if (enable_local == auto_enabled)
-			printk(KERN_INFO "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n");
+			dev_info(&bus->dev, "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n");
 
 		free_list(&fail_head);
 		goto enable_and_dump;
 	}
 
-	printk(KERN_DEBUG "PCI: No. %d try to assign unassigned res\n",
-			 tried_times + 1);
+	dev_printk(KERN_DEBUG, &bus->dev,
+		   "No. %d try to assign unassigned res\n", tried_times + 1);
 
 	/* third times and later will not check if it is leaf */
 	if ((tried_times + 1) > 2)
@@ -1565,12 +1547,18 @@ again:
 
 enable_and_dump:
 	/* Depth last, update the hardware. */
-	list_for_each_entry(bus, &pci_root_buses, node)
-		pci_enable_bridges(bus);
+	pci_enable_bridges(bus);
 
 	/* dump the resource on buses */
-	list_for_each_entry(bus, &pci_root_buses, node)
-		pci_bus_dump_resources(bus);
+	pci_bus_dump_resources(bus);
+}
+
+void __init pci_assign_unassigned_resources(void)
+{
+	struct pci_bus *root_bus;
+
+	list_for_each_entry(root_bus, &pci_root_buses, node)
+		pci_assign_unassigned_root_bus_resources(root_bus);
 }
 
 void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge)

[PATCH v5 6/7] PCI: Enable pci bridge when it is needed

[Yinghai Lu]

Current we enable bridges after bus scan and assign resources.
and it is spreaded a lot of places.

We can enable them later when their children pci device is enabled.
Need to go up to root bus and enable bridge one by one down to pci
device.

So that will delay enable bridge as needed bassis,
also kill one inconsistent between boot path and hot-add
path in acpi_pci_root_add().

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 arch/arm/kernel/bios32.c           |    5 -----
 arch/m68k/platform/coldfire/pci.c  |    1 -
 arch/mips/pci/pci.c                |    1 -
 arch/sh/drivers/pci/pci.c          |    1 -
 drivers/acpi/pci_root.c            |    4 ----
 drivers/parisc/lba_pci.c           |    1 -
 drivers/pci/bus.c                  |   19 -------------------
 drivers/pci/hotplug/acpiphp_glue.c |    1 -
 drivers/pci/pci.c                  |   20 ++++++++++++++++++++
 drivers/pci/probe.c                |    1 -
 drivers/pci/setup-bus.c            |   10 +++-------
 drivers/pcmcia/cardbus.c           |    1 -
 include/linux/pci.h                |    1 -
 13 files changed, 23 insertions(+), 43 deletions(-)

Index: linux-2.6/arch/arm/kernel/bios32.c
===================================================================
--- linux-2.6.orig/arch/arm/kernel/bios32.c
+++ linux-2.6/arch/arm/kernel/bios32.c
@@ -524,11 +524,6 @@ void pci_common_init(struct hw_pci *hw)
 			 * Assign resources.
 			 */
 			pci_bus_assign_resources(bus);
-
-			/*
-			 * Enable bridges
-			 */
-			pci_enable_bridges(bus);
 		}
 
 		/*
Index: linux-2.6/arch/m68k/platform/coldfire/pci.c
===================================================================
--- linux-2.6.orig/arch/m68k/platform/coldfire/pci.c
+++ linux-2.6/arch/m68k/platform/coldfire/pci.c
@@ -319,7 +319,6 @@ static int __init mcf_pci_init(void)
 	pci_fixup_irqs(pci_common_swizzle, mcf_pci_map_irq);
 	pci_bus_size_bridges(rootbus);
 	pci_bus_assign_resources(rootbus);
-	pci_enable_bridges(rootbus);
 
 	return 0;
 }
Index: linux-2.6/arch/mips/pci/pci.c
===================================================================
--- linux-2.6.orig/arch/mips/pci/pci.c
+++ linux-2.6/arch/mips/pci/pci.c
@@ -113,7 +113,6 @@ static void pcibios_scanbus(struct pci_c
 		if (!pci_has_flag(PCI_PROBE_ONLY)) {
 			pci_bus_size_bridges(bus);
 			pci_bus_assign_resources(bus);
-			pci_enable_bridges(bus);
 		}
 	}
 }
Index: linux-2.6/arch/sh/drivers/pci/pci.c
===================================================================
--- linux-2.6.orig/arch/sh/drivers/pci/pci.c
+++ linux-2.6/arch/sh/drivers/pci/pci.c
@@ -69,7 +69,6 @@ static void pcibios_scanbus(struct pci_c
 
 		pci_bus_size_bridges(bus);
 		pci_bus_assign_resources(bus);
-		pci_enable_bridges(bus);
 	} else {
 		pci_free_resource_list(&resources);
 	}
Index: linux-2.6/drivers/parisc/lba_pci.c
===================================================================
--- linux-2.6.orig/drivers/parisc/lba_pci.c
+++ linux-2.6/drivers/parisc/lba_pci.c
@@ -1533,7 +1533,6 @@ lba_driver_probe(struct parisc_device *d
 		lba_dump_res(&lba_dev->hba.lmmio_space, 2);
 #endif
 	}
-	pci_enable_bridges(lba_bus);
 
 	/*
 	** Once PCI register ops has walked the bus, access to config
Index: linux-2.6/drivers/pci/bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/bus.c
+++ linux-2.6/drivers/pci/bus.c
@@ -216,24 +216,6 @@ void pci_bus_add_devices(const struct pc
 	}
 }
 
-void pci_enable_bridges(struct pci_bus *bus)
-{
-	struct pci_dev *dev;
-	int retval;
-
-	list_for_each_entry(dev, &bus->devices, bus_list) {
-		if (dev->subordinate) {
-			if (!pci_is_enabled(dev)) {
-				retval = pci_enable_device(dev);
-				if (retval)
-					dev_err(&dev->dev, "Error enabling bridge (%d), continuing\n", retval);
-				pci_set_master(dev);
-			}
-			pci_enable_bridges(dev->subordinate);
-		}
-	}
-}
-
 /** pci_walk_bus - walk devices on/under bus, calling callback.
  *  @top      bus whose devices should be walked
  *  @cb       callback to be called for each device found
@@ -301,4 +283,3 @@ EXPORT_SYMBOL(pci_bus_put);
 EXPORT_SYMBOL(pci_bus_alloc_resource);
 EXPORT_SYMBOL_GPL(pci_bus_add_device);
 EXPORT_SYMBOL(pci_bus_add_devices);
-EXPORT_SYMBOL(pci_enable_bridges);
Index: linux-2.6/drivers/pci/hotplug/acpiphp_glue.c
===================================================================
--- linux-2.6.orig/drivers/pci/hotplug/acpiphp_glue.c
+++ linux-2.6/drivers/pci/hotplug/acpiphp_glue.c
@@ -704,7 +704,6 @@ static int __ref enable_device(struct ac
 	acpiphp_sanitize_bus(bus);
 	acpiphp_set_hpp_values(bus);
 	acpiphp_set_acpi_region(slot);
-	pci_enable_bridges(bus);
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
 		/* Assume that newly added devices are powered on already. */
Index: linux-2.6/drivers/pci/pci.c
===================================================================
--- linux-2.6.orig/drivers/pci/pci.c
+++ linux-2.6/drivers/pci/pci.c
@@ -1145,6 +1145,24 @@ int pci_reenable_device(struct pci_dev *
 	return 0;
 }
 
+static void pci_enable_bridge(struct pci_dev *dev)
+{
+	int retval;
+
+	if (!dev)
+		return;
+
+	pci_enable_bridge(dev->bus->self);
+
+	if (pci_is_enabled(dev))
+		return;
+	retval = pci_enable_device(dev);
+	if (retval)
+		dev_err(&dev->dev, "Error enabling bridge (%d), continuing\n",
+			retval);
+	pci_set_master(dev);
+}
+
 static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags)
 {
 	int err;
@@ -1165,6 +1183,8 @@ static int pci_enable_device_flags(struc
 	if (atomic_inc_return(&dev->enable_cnt) > 1)
 		return 0;		/* already enabled */
 
+	pci_enable_bridge(dev->bus->self);
+
 	/* only skip sriov related */
 	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
 		if (dev->resource[i].flags & flags)
Index: linux-2.6/drivers/pci/probe.c
===================================================================
--- linux-2.6.orig/drivers/pci/probe.c
+++ linux-2.6/drivers/pci/probe.c
@@ -1964,7 +1964,6 @@ unsigned int __ref pci_rescan_bus(struct
 
 	max = pci_scan_child_bus(bus);
 	pci_assign_unassigned_bus_resources(bus);
-	pci_enable_bridges(bus);
 	pci_bus_add_devices(bus);
 
 	return max;
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1503,7 +1503,7 @@ again:
 
 	/* any device complain? */
 	if (list_empty(&fail_head))
-		goto enable_and_dump;
+		goto dump;
 
 	if (tried_times >= pci_try_num) {
 		if (enable_local == undefined)
@@ -1512,7 +1512,7 @@ again:
 			dev_info(&bus->dev, "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n");
 
 		free_list(&fail_head);
-		goto enable_and_dump;
+		goto dump;
 	}
 
 	dev_printk(KERN_DEBUG, &bus->dev,
@@ -1545,10 +1545,7 @@ again:
 
 	goto again;
 
-enable_and_dump:
-	/* Depth last, update the hardware. */
-	pci_enable_bridges(bus);
-
+dump:
 	/* dump the resource on buses */
 	pci_bus_dump_resources(bus);
 }
@@ -1621,7 +1618,6 @@ enable_all:
 	if (retval)
 		dev_err(&bridge->dev, "Error reenabling bridge (%d)\n", retval);
 	pci_set_master(bridge);
-	pci_enable_bridges(parent);
 }
 EXPORT_SYMBOL_GPL(pci_assign_unassigned_bridge_resources);
 
Index: linux-2.6/drivers/pcmcia/cardbus.c
===================================================================
--- linux-2.6.orig/drivers/pcmcia/cardbus.c
+++ linux-2.6/drivers/pcmcia/cardbus.c
@@ -91,7 +91,6 @@ int __ref cb_alloc(struct pcmcia_socket
 	if (s->tune_bridge)
 		s->tune_bridge(s, bus);
 
-	pci_enable_bridges(bus);
 	pci_bus_add_devices(bus);
 
 	return 0;
Index: linux-2.6/include/linux/pci.h
===================================================================
--- linux-2.6.orig/include/linux/pci.h
+++ linux-2.6/include/linux/pci.h
@@ -1043,7 +1043,6 @@ int __must_check pci_bus_alloc_resource(
 						  resource_size_t,
 						  resource_size_t),
 			void *alignf_data);
-void pci_enable_bridges(struct pci_bus *bus);
 
 /* Proper probing supporting hot-pluggable devices */
 int __must_check __pci_register_driver(struct pci_driver *, struct module *,
Index: linux-2.6/drivers/acpi/pci_root.c
===================================================================
--- linux-2.6.orig/drivers/acpi/pci_root.c
+++ linux-2.6/drivers/acpi/pci_root.c
@@ -537,10 +537,6 @@ static int acpi_pci_root_add(struct acpi
 		pci_assign_unassigned_bus_resources(root->bus);
 	}
 
-	/* need to after hot-added ioapic is registered */
-	if (system_state != SYSTEM_BOOTING)
-		pci_enable_bridges(root->bus);
-
 	pci_bus_add_devices(root->bus);
 	return 1;
 

[PATCH v5 7/7] PCI: Retry assign unassigned resources for hotadd root bus

[Yinghai Lu]

Let root bus hotadd path use same code for booting path.
As driver is not loaded yet, we could retry to make sure
all pci devices get resources allocated.
We need this as during hotadd, firmware could assign some bars before
handle over.

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/acpi/pci_root.c |    2 +-
 drivers/pci/setup-bus.c |   15 +++++++--------
 include/linux/pci.h     |    1 +
 3 files changed, 9 insertions(+), 9 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -1365,7 +1365,7 @@ static void pci_bus_dump_resources(struc
 	}
 }
 
-static int __init pci_bus_get_depth(struct pci_bus *bus)
+static int pci_bus_get_depth(struct pci_bus *bus)
 {
 	int depth = 0;
 	struct pci_dev *dev;
@@ -1399,7 +1399,7 @@ enum enable_type {
 	auto_enabled,
 };
 
-static enum enable_type pci_realloc_enable __initdata = undefined;
+static enum enable_type pci_realloc_enable = undefined;
 void __init pci_realloc_get_opt(char *str)
 {
 	if (!strncmp(str, "off", 3))
@@ -1407,13 +1407,13 @@ void __init pci_realloc_get_opt(char *st
 	else if (!strncmp(str, "on", 2))
 		pci_realloc_enable = user_enabled;
 }
-static bool __init pci_realloc_enabled(enum enable_type enable)
+static bool pci_realloc_enabled(enum enable_type enable)
 {
 	return enable >= user_enabled;
 }
 
 #if defined(CONFIG_PCI_IOV) && defined(CONFIG_PCI_REALLOC_ENABLE_AUTO)
-static int __init check_unassigned_resources(struct pci_dev *dev, void *data)
+static int check_unassigned_resources(struct pci_dev *dev, void *data)
 {
 	int i;
 	int *unassigned = data;
@@ -1431,7 +1431,7 @@ static int __init check_unassigned_resou
 	return 0;
 }
 
-static enum enable_type __init pci_realloc_detect(struct pci_bus *bus,
+static enum enable_type pci_realloc_detect(struct pci_bus *bus,
 			 enum enable_type enable_local)
 {
 	int unassigned = 0;
@@ -1446,7 +1446,7 @@ static enum enable_type __init pci_reall
 	return enable_local;
 }
 #else
-static enum enable_type __init pci_realloc_detect(struct pci_bus *bus,
+static enum enable_type pci_realloc_detect(struct pci_bus *bus,
 			 enum enable_type enable_local)
 {
 	return enable_local;
@@ -1458,8 +1458,7 @@ static enum enable_type __init pci_reall
  * second  and later try will clear small leaf bridge res
  * will stop till to the max  deepth if can not find good one
  */
-static void __init
-pci_assign_unassigned_root_bus_resources(struct pci_bus *bus)
+void pci_assign_unassigned_root_bus_resources(struct pci_bus *bus)
 {
 	LIST_HEAD(realloc_head); /* list of resources that
 					want additional resources */
Index: linux-2.6/drivers/acpi/pci_root.c
===================================================================
--- linux-2.6.orig/drivers/acpi/pci_root.c
+++ linux-2.6/drivers/acpi/pci_root.c
@@ -534,7 +534,7 @@ static int acpi_pci_root_add(struct acpi
 
 	if (system_state != SYSTEM_BOOTING) {
 		pcibios_resource_survey_bus(root->bus);
-		pci_assign_unassigned_bus_resources(root->bus);
+		pci_assign_unassigned_root_bus_resources(root->bus);
 	}
 
 	pci_bus_add_devices(root->bus);
Index: linux-2.6/include/linux/pci.h
===================================================================
--- linux-2.6.orig/include/linux/pci.h
+++ linux-2.6/include/linux/pci.h
@@ -1003,6 +1003,7 @@ int pci_claim_resource(struct pci_dev *,
 void pci_assign_unassigned_resources(void);
 void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge);
 void pci_assign_unassigned_bus_resources(struct pci_bus *bus);
+void pci_assign_unassigned_root_bus_resources(struct pci_bus *bus);
 void pdev_enable_device(struct pci_dev *);
 int pci_enable_resources(struct pci_dev *, int mask);
 void pci_fixup_irqs(u8 (*)(struct pci_dev *, u8 *),

[PATCH v5 1/7] PCI: Don't let mmio fallback to must-only, if ioport fails with must+optional

[Yinghai Lu]

BenH reported that there is some assign unassigned resource problem
in powerpc.

It turns out after
| commit 0c5be0cb0edfe3b5c4b62eac68aa2aa15ec681af
| Date:   Thu Feb 23 19:23:29 2012 -0800
|
|    PCI: Retry on IORESOURCE_IO type allocations

even the root bus does not have io port range, it will keep retrying
to realloc with mmio.

Current retry logic is : try with must+optional at first, and if
it fails with any ioport or mmio, it will try must then try to extend
must with optional. The reassign will put extended mmio or pref mmio
in the middle of parent resource range.
That will prevent other sibling resources to get must-have resources
or get extended properly.

We can check fail type to see if can we need fall back to must-have
only, that will keep not needed release resource to be must+optional.

Separate three resource type checking if we need to release
assigned resource after requested + add_size try.
1. if there is io port assign fail, will release assigned io port.
2. if there is pref mmio assign fail, release assigned pref mmio.
   if assigned pref mmio's parent is non-pref mmio and there
   is non-pref mmio assign fail, will release that assigned pref mmio.
3. if there is non-pref mmio assign fail or pref mmio assigned fail,
   will release assigned non-pref mmio.

This will be become more often when we have x86 8 sockets or 32 sockets
system, and those system will have one root bus per socket.
They will have some root buses do not have ioport range or 32bit mmio.

-v2: need to remove assigned entries from optional list too.
-v3: not just checking ioport related, requested by Bjorn.

Reported-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Tested-by: Gavin Shan <shangw@linux.vnet.ibm.com>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 drivers/pci/setup-bus.c |   70 +++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 69 insertions(+), 1 deletion(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
--- linux-2.6.orig/drivers/pci/setup-bus.c
+++ linux-2.6/drivers/pci/setup-bus.c
@@ -300,6 +300,48 @@ static void assign_requested_resources_s
 	}
 }
 
+static unsigned long pci_fail_res_type_mask(struct list_head *fail_head)
+{
+	struct pci_dev_resource *fail_res;
+	unsigned long mask = 0;
+
+	/* check failed type */
+	list_for_each_entry(fail_res, fail_head, list)
+		mask |= fail_res->flags;
+
+	/*
+	 * one pref failed resource will set IORESOURCE_MEM,
+	 * as we can allocate pref in non-pref range.
+	 * Will release all asssigned non-pref sibling resources
+	 * according to that bit.
+	 */
+	return mask & (IORESOURCE_IO | IORESOURCE_MEM | IORESOURCE_PREFETCH);
+}
+
+static bool pci_need_to_release(unsigned long mask, struct resource *res)
+{
+	if (res->flags & IORESOURCE_IO)
+		return !!(mask & IORESOURCE_IO);
+
+	/* check pref at first */
+	if (res->flags & IORESOURCE_PREFETCH) {
+		if (mask & IORESOURCE_PREFETCH)
+			return true;
+		/* count pref if its parent is non-pref */
+		else if ((mask & IORESOURCE_MEM) &&
+			 !(res->parent->flags & IORESOURCE_PREFETCH))
+			return true;
+		else
+			return false;
+	}
+
+	if (res->flags & IORESOURCE_MEM)
+		return !!(mask & IORESOURCE_MEM);
+
+	/* should not get here */
+	return false;
+}
+
 static void __assign_resources_sorted(struct list_head *head,
 				 struct list_head *realloc_head,
 				 struct list_head *fail_head)
@@ -312,11 +354,24 @@ static void __assign_resources_sorted(st
 	 *  if could do that, could get out early.
 	 *  if could not do that, we still try to assign requested at first,
 	 *    then try to reassign add_size for some resources.
+	 *
+	 * Separate three resource type checking if we need to release
+	 *  assigned resource after requested + add_size try.
+	 *	1. if there is io port assign fail, will release assigned
+	 *	   io port.
+	 *	2. if there is pref mmio assign fail, release assigned
+	 *	   pref mmio.
+	 *	   if assigned pref mmio's parent is non-pref mmio and there
+	 *	   is non-pref mmio assign fail, will release that assigned
+	 *	   pref mmio.
+	 *	3. if there is non-pref mmio assign fail or pref mmio
+	 *	   assigned fail, will release assigned non-pref mmio.
 	 */
 	LIST_HEAD(save_head);
 	LIST_HEAD(local_fail_head);
 	struct pci_dev_resource *save_res;
-	struct pci_dev_resource *dev_res;
+	struct pci_dev_resource *dev_res, *tmp_res;
+	unsigned long fail_type;
 
 	/* Check if optional add_size is there */
 	if (!realloc_head || list_empty(realloc_head))
@@ -348,6 +403,19 @@ static void __assign_resources_sorted(st
 		return;
 	}
 
+	/* check failed type */
+	fail_type = pci_fail_res_type_mask(&local_fail_head);
+	/* remove not need to be released assigned res from head list etc */
+	list_for_each_entry_safe(dev_res, tmp_res, head, list)
+		if (dev_res->res->parent &&
+		    !pci_need_to_release(fail_type, dev_res->res)) {
+			/* remove it from realloc_head list */
+			remove_from_list(realloc_head, dev_res->res);
+			remove_from_list(&save_head, dev_res->res);
+			list_del(&dev_res->list);
+			kfree(dev_res);
+		}
+
 	free_list(&local_fail_head);
 	/* Release assigned resource */
 	list_for_each_entry(dev_res, head, list)

Re: [PATCH v5 0/7] PCI: Change assign unassigned resources per root bus bassis

[Yinghai Lu]

On Fri, May 31, 2013 at 11:03 PM, Yinghai Lu <yinghai@kernel.org> wrote:
> BenH reported that there is some assign unassigned resource problem
> in powerpc.
>
> It turns out after
> | commit 0c5be0cb0edfe3b5c4b62eac68aa2aa15ec681af
> | Date:   Thu Feb 23 19:23:29 2012 -0800
> |
> |    PCI: Retry on IORESOURCE_IO type allocations
>
> even the root bus does not have io port range, it will keep retrying
> to realloc with mmio.
>
> After checking the code, found that we bound io port and mmio fail
> path together.
> First patch fix the problem, that will not make mmio fall back to must-only
> when only have io port fail with must+optional.
>
> During we found the fix for that problem, found that we can separate assign
> unassigned resources to per root bus.
> that will make the code simple, also could reuse it for hotadd path.
>
> These patches are targeted to 3.11
>
> -v4: split first patch into 4 patches per Bjorn.
> -v5: drop two patches that will pass root bus resource mask after we found
>      simple and less intrusive way to fix the problem.
>
>  PCI: Don't let mmio fallback to must-only, if ioport fails with must+optional
>  PCI: Don't use temp bus for pci_bus_release_bridge_resources
>  PCI: Use pci_walk_bus to detect unassigned resources
>  PCI: Introduce enable_local to prepare per root bus handling
>  PCI: Split pci_assign_unassigned_resources to per root bus
>  PCI: Enable pci bridge when it is needed
>  PCI: Retry assign unassigned resources for hotadd root bus

Hi, Bjorn,

Can you put this patchset in pci/next for 3.11?

Found another pciehp will need this one two. the pcie bridge does not
have io port range and it cause mmio get clear and retry.

Thanks

Yinghai