[PATCH V3 0/8] ppc/pnv: Rework on pnv P7IOC initialization

[PATCH V3 0/8] ppc/pnv: Rework on pnv P7IOC initialization

[Gavin Shan]

The rework is done based on Ben's initial ideas on how PE and resource
assignment is done on top of PCI core instead of doing resource assignment
by powernv platform. With the series of patches, the following aspects will
be covered:

	- Only create PE based on PCI bus. Basically, there will be 2 types
	  of PEs built on PCI bus: (A) PE only includes single PCI bus. (B)
	  PE includes the PCI bus as well as its subordinate PCI buses.
	- Make the I/O and MMIO resources of P2P bridge aligned according to
	  the PHB's resource layout. For example, MMIO resource of P2P bridges
	  will be aligned with 16MB if PHB has 16MB M32 segment.
	- The resource allocation will be done on basis of PE.
	- Trivial fixup on the hook of enabling PCI device, PE check etc.

The series of patches has been verified on Firebird-L and Juno-IOC-L based on
kernel 3.4.RC6. During the kernel boots, USB HUBs can be detected and initialized
correctly. Also, the disk drive as well as network interfaces can work successfully
after booting into shell environment.

V1 -> V2
	- Add separate field into "struct pci_host_bridge" to trace the minimal
	  alignment of prefetchable MMIO bars for P2P bridges (Benjamin Herrenschmidt)
	- Reduced variable names while measuring the MMIO bars (Benjamin Herrenschmidt)
	- Change the commit log for more clear description on [PATCH 2/8] (Benjamin Herrenschmidt)
	- Change the commit log for more clear description on the types of PCI bus
	  sensitive PEs on [PATCH 5/8] (Richard Yang)

V2 -> V3
	- Rework the sequence of the patches so that it won't break "git bitsect" (Benjamin Herrenschmidt)
	- Cleanup on variables name so that they have shorter length (Benjamin Herrenschmidt)
	- Rework on PE's resource calculation based on the PCI bus (Benjamin Herrenschmidt)
	- Cleanup on changelog (Benjamin Herrenschmidt)
	

arch/powerpc/platforms/powernv/pci-ioda.c |  708 ++++++++---------------------
arch/powerpc/platforms/powernv/pci.h      |   21 +-
2 files changed, 213 insertions(+), 516 deletions(-)

Thanks,
Gavin

[PATCH 1/8] ppc/pnv: create bus sensitive PEs

[Gavin Shan]

Basically, there're 2 types of PCI bus sensitive PEs: (A) The PE
includes single PCI bus. (B) The PE includes the PCI bus and all
the subordinate PCI buses. At present, we'd like to put PCI bus
originated by PCI-e link to form PE that contains single PCI bus,
and the PCIe-to-PCI bridge will form the 2nd type of PE. We don't
figure out to detect PLX bridge yet. Once we can detect PLX bridge
some day, we have to put PCI buses originated from the downstream
port of PLX bridge to the 2nd type of PE.

The patch changes the original implementation for a little bit
to support 2 types of PCI bus sensitive PEs described as above.
Also, the function used to retrieve the corresponding PE according
to the given PCI device has been changed based on that because each
PCI device should trace the directly associated PE.

Signed-off-by: Gavin Shan <shangw@linux.vnet.ibm.com>
Reviewed-by: Ram Pai <linuxram@us.ibm.com>
Reviewed-by: Richard Yang <weiyang@linux.vnet.ibm.com>
---
 arch/powerpc/platforms/powernv/pci-ioda.c |   97 +++++++++++++++++------------
 arch/powerpc/platforms/powernv/pci.h      |   10 +--
 2 files changed, 63 insertions(+), 44 deletions(-)

diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index fbdd74d..1504795 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -548,7 +548,7 @@ static void __devinit pnv_ioda_free_pe(struct pnv_phb *phb, int pe)
  * but in the meantime, we need to protect them to avoid warnings
  */
 #ifdef CONFIG_PCI_MSI
-static struct pnv_ioda_pe * __devinit __pnv_ioda_get_one_pe(struct pci_dev *dev)
+static struct pnv_ioda_pe * __devinit pnv_ioda_get_pe(struct pci_dev *dev)
 {
 	struct pci_controller *hose = pci_bus_to_host(dev->bus);
 	struct pnv_phb *phb = hose->private_data;
@@ -560,19 +560,6 @@ static struct pnv_ioda_pe * __devinit __pnv_ioda_get_one_pe(struct pci_dev *dev)
 		return NULL;
 	return &phb->ioda.pe_array[pdn->pe_number];
 }
-
-static struct pnv_ioda_pe * __devinit pnv_ioda_get_pe(struct pci_dev *dev)
-{
-	struct pnv_ioda_pe *pe = __pnv_ioda_get_one_pe(dev);
-
-	while (!pe && dev->bus->self) {
-		dev = dev->bus->self;
-		pe = __pnv_ioda_get_one_pe(dev);
-		if (pe)
-			pe = pe->bus_pe;
-	}
-	return pe;
-}
 #endif /* CONFIG_PCI_MSI */
 
 static int __devinit pnv_ioda_configure_pe(struct pnv_phb *phb,
@@ -589,7 +576,11 @@ static int __devinit pnv_ioda_configure_pe(struct pnv_phb *phb,
 		dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
 		fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
 		parent = pe->pbus->self;
-		count = pe->pbus->subordinate - pe->pbus->secondary + 1;
+		if (pe->flags & PNV_IODA_PE_BUS_ALL)
+			count = pe->pbus->subordinate - pe->pbus->secondary + 1;
+		else
+			count = 1;
+
 		switch(count) {
 		case  1: bcomp = OpalPciBusAll;		break;
 		case  2: bcomp = OpalPciBus7Bits;	break;
@@ -699,6 +690,7 @@ static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev)
 	return 10;
 }
 
+#if 0
 static struct pnv_ioda_pe * __devinit pnv_ioda_setup_dev_PE(struct pci_dev *dev)
 {
 	struct pci_controller *hose = pci_bus_to_host(dev->bus);
@@ -767,6 +759,7 @@ static struct pnv_ioda_pe * __devinit pnv_ioda_setup_dev_PE(struct pci_dev *dev)
 
 	return pe;
 }
+#endif /* Useful for SRIOV case */
 
 static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
 {
@@ -784,43 +777,47 @@ static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
 		pdn->pcidev = dev;
 		pdn->pe_number = pe->pe_number;
 		pe->dma_weight += pnv_ioda_dma_weight(dev);
-		if (dev->subordinate)
+		if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
 			pnv_ioda_setup_same_PE(dev->subordinate, pe);
 	}
 }
 
-static void __devinit pnv_ioda_setup_bus_PE(struct pci_dev *dev,
-					    struct pnv_ioda_pe *ppe)
+/*
+ * There're 2 types of PCI bus sensitive PEs: One that is compromised of
+ * single PCI bus. Another one that contains the primary PCI bus and its
+ * subordinate PCI devices and buses. The second type of PE is normally
+ * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
+ */
+static void __devinit pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all)
 {
-	struct pci_controller *hose = pci_bus_to_host(dev->bus);
+	struct pci_controller *hose = pci_bus_to_host(bus);
 	struct pnv_phb *phb = hose->private_data;
-	struct pci_bus *bus = dev->subordinate;
 	struct pnv_ioda_pe *pe;
 	int pe_num;
 
-	if (!bus) {
-		pr_warning("%s: Bridge without a subordinate bus !\n",
-			   pci_name(dev));
-		return;
-	}
 	pe_num = pnv_ioda_alloc_pe(phb);
 	if (pe_num == IODA_INVALID_PE) {
-		pr_warning("%s: Not enough PE# available, disabling bus\n",
-			   pci_name(dev));
+		pr_warning("%s: Not enough PE# available for PCI bus %04x:%02x\n",
+			__func__, pci_domain_nr(bus), bus->number);
 		return;
 	}
 
 	pe = &phb->ioda.pe_array[pe_num];
-	ppe->bus_pe = pe;
+	pe->flags = (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
 	pe->pbus = bus;
+	pe->pe_number = pe_num;
 	pe->pdev = NULL;
 	pe->tce32_seg = -1;
 	pe->mve_number = -1;
 	pe->rid = bus->secondary << 8;
 	pe->dma_weight = 0;
 
-	pe_info(pe, "Secondary busses %d..%d associated with PE\n",
-		bus->secondary, bus->subordinate);
+	if (all)
+		pe_info(pe, "Secondary busses %d..%d associated with PE#%d\n",
+			bus->secondary, bus->subordinate, pe_num);
+	else
+		pe_info(pe, "Secondary busses %d associated with PE#%d\n",
+			bus->secondary, pe_num);
 
 	if (pnv_ioda_configure_pe(phb, pe)) {
 		/* XXX What do we do here ? */
@@ -848,17 +845,33 @@ static void __devinit pnv_ioda_setup_bus_PE(struct pci_dev *dev,
 static void __devinit pnv_ioda_setup_PEs(struct pci_bus *bus)
 {
 	struct pci_dev *dev;
-	struct pnv_ioda_pe *pe;
+
+	pnv_ioda_setup_bus_PE(bus, 0);
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
-		pe = pnv_ioda_setup_dev_PE(dev);
-		if (pe == NULL)
-			continue;
-		/* Leaving the PCIe domain ... single PE# */
-		if (dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
-			pnv_ioda_setup_bus_PE(dev, pe);
-		else if (dev->subordinate)
-			pnv_ioda_setup_PEs(dev->subordinate);
+		if (dev->subordinate) {
+			if (dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
+				pnv_ioda_setup_bus_PE(dev->subordinate, 1);
+			else
+				pnv_ioda_setup_PEs(dev->subordinate);
+		}
+	}
+}
+
+/*
+ * Configure PEs so that the downstream PCI buses and devices
+ * could have their associated PE#. Unfortunately, we didn't
+ * figure out the way to identify the PLX bridge yet. So we
+ * simply put the PCI bus and the subordinate behind the root
+ * port to PE# here. The game rule here is expected to be changed
+ * as soon as we can detected PLX bridge correctly.
+ */
+static void __devinit pnv_pci_ioda_setup_PEs(void)
+{
+	struct pci_controller *hose, *tmp;
+
+	list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
+		pnv_ioda_setup_PEs(hose->bus);
 	}
 }
 
@@ -1139,6 +1152,11 @@ static void __devinit pnv_pci_ioda_fixup_phb(struct pci_controller *hose)
 	}
 }
 
+static void __devinit pnv_pci_ioda_fixup(void)
+{
+	pnv_pci_ioda_setup_PEs();
+}
+
 /* Prevent enabling devices for which we couldn't properly
  * assign a PE
  */
@@ -1305,6 +1323,7 @@ void __init pnv_pci_init_ioda1_phb(struct device_node *np)
 	 * ourselves here
 	 */
 	ppc_md.pcibios_fixup_phb = pnv_pci_ioda_fixup_phb;
+	ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
 	ppc_md.pcibios_enable_device_hook = pnv_pci_enable_device_hook;
 	pci_add_flags(PCI_PROBE_ONLY | PCI_REASSIGN_ALL_RSRC);
 
diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
index 8bc4796..0cb760c 100644
--- a/arch/powerpc/platforms/powernv/pci.h
+++ b/arch/powerpc/platforms/powernv/pci.h
@@ -17,9 +17,14 @@ enum pnv_phb_model {
 };
 
 #define PNV_PCI_DIAG_BUF_SIZE	4096
+#define PNV_IODA_PE_DEV		(1 << 0)	/* PE has single PCI device	*/
+#define PNV_IODA_PE_BUS		(1 << 1)	/* PE has primary PCI bus	*/
+#define PNV_IODA_PE_BUS_ALL	(1 << 2)	/* PE has subordinate buses	*/
 
 /* Data associated with a PE, including IOMMU tracking etc.. */
 struct pnv_ioda_pe {
+	unsigned long		flags;
+
 	/* A PE can be associated with a single device or an
 	 * entire bus (& children). In the former case, pdev
 	 * is populated, in the later case, pbus is.
@@ -40,11 +45,6 @@ struct pnv_ioda_pe {
 	 */
 	unsigned int		dma_weight;
 
-	/* This is a PCI-E -> PCI-X bridge, this points to the
-	 * corresponding bus PE
-	 */
-	struct pnv_ioda_pe	*bus_pe;
-
 	/* "Base" iommu table, ie, 4K TCEs, 32-bit DMA */
 	int			tce32_seg;
 	int			tce32_segcount;
-- 
1.7.9.5

Re: [PATCH 1/8] ppc/pnv: create bus sensitive PEs

[Richard Yang]

On Mon, Jun 25, 2012 at 11:43:14PM +0800, Gavin Shan wrote:
>Basically, there're 2 types of PCI bus sensitive PEs: (A) The PE
>includes single PCI bus. (B) The PE includes the PCI bus and all
>the subordinate PCI buses. At present, we'd like to put PCI bus
>originated by PCI-e link to form PE that contains single PCI bus,
>and the PCIe-to-PCI bridge will form the 2nd type of PE. We don't
>figure out to detect PLX bridge yet. Once we can detect PLX bridge
>some day, we have to put PCI buses originated from the downstream
>port of PLX bridge to the 2nd type of PE.
>
>The patch changes the original implementation for a little bit
>to support 2 types of PCI bus sensitive PEs described as above.
>Also, the function used to retrieve the corresponding PE according
>to the given PCI device has been changed based on that because each
>PCI device should trace the directly associated PE.
>
>Signed-off-by: Gavin Shan <shangw@linux.vnet.ibm.com>
>Reviewed-by: Ram Pai <linuxram@us.ibm.com>
>Reviewed-by: Richard Yang <weiyang@linux.vnet.ibm.com>
>---
> arch/powerpc/platforms/powernv/pci-ioda.c |   97 +++++++++++++++++------------
> arch/powerpc/platforms/powernv/pci.h      |   10 +--
> 2 files changed, 63 insertions(+), 44 deletions(-)
>
>diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
>index fbdd74d..1504795 100644
>--- a/arch/powerpc/platforms/powernv/pci-ioda.c
>+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
>@@ -548,7 +548,7 @@ static void __devinit pnv_ioda_free_pe(struct pnv_phb *phb, int pe)
>  * but in the meantime, we need to protect them to avoid warnings
>  */
> #ifdef CONFIG_PCI_MSI
>-static struct pnv_ioda_pe * __devinit __pnv_ioda_get_one_pe(struct pci_dev *dev)
>+static struct pnv_ioda_pe * __devinit pnv_ioda_get_pe(struct pci_dev *dev)
> {
> 	struct pci_controller *hose = pci_bus_to_host(dev->bus);
> 	struct pnv_phb *phb = hose->private_data;
>@@ -560,19 +560,6 @@ static struct pnv_ioda_pe * __devinit __pnv_ioda_get_one_pe(struct pci_dev *dev)
> 		return NULL;
> 	return &phb->ioda.pe_array[pdn->pe_number];
> }
>-
>-static struct pnv_ioda_pe * __devinit pnv_ioda_get_pe(struct pci_dev *dev)
>-{
>-	struct pnv_ioda_pe *pe = __pnv_ioda_get_one_pe(dev);
>-
>-	while (!pe && dev->bus->self) {
>-		dev = dev->bus->self;
>-		pe = __pnv_ioda_get_one_pe(dev);
>-		if (pe)
>-			pe = pe->bus_pe;
>-	}
>-	return pe;
>-}
> #endif /* CONFIG_PCI_MSI */
>
> static int __devinit pnv_ioda_configure_pe(struct pnv_phb *phb,
>@@ -589,7 +576,11 @@ static int __devinit pnv_ioda_configure_pe(struct pnv_phb *phb,
> 		dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
> 		fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
> 		parent = pe->pbus->self;
>-		count = pe->pbus->subordinate - pe->pbus->secondary + 1;
>+		if (pe->flags & PNV_IODA_PE_BUS_ALL)
>+			count = pe->pbus->subordinate - pe->pbus->secondary + 1;
>+		else
>+			count = 1;
>+
> 		switch(count) {
> 		case  1: bcomp = OpalPciBusAll;		break;
> 		case  2: bcomp = OpalPciBus7Bits;	break;
>@@ -699,6 +690,7 @@ static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev)
> 	return 10;
> }
>
>+#if 0
> static struct pnv_ioda_pe * __devinit pnv_ioda_setup_dev_PE(struct pci_dev *dev)
> {
> 	struct pci_controller *hose = pci_bus_to_host(dev->bus);
>@@ -767,6 +759,7 @@ static struct pnv_ioda_pe * __devinit pnv_ioda_setup_dev_PE(struct pci_dev *dev)
>
> 	return pe;
> }
>+#endif /* Useful for SRIOV case */
>
> static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
> {
>@@ -784,43 +777,47 @@ static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
> 		pdn->pcidev = dev;
> 		pdn->pe_number = pe->pe_number;
> 		pe->dma_weight += pnv_ioda_dma_weight(dev);
>-		if (dev->subordinate)
>+		if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
> 			pnv_ioda_setup_same_PE(dev->subordinate, pe);
> 	}
> }
>
>-static void __devinit pnv_ioda_setup_bus_PE(struct pci_dev *dev,
>-					    struct pnv_ioda_pe *ppe)
>+/*
>+ * There're 2 types of PCI bus sensitive PEs: One that is compromised of
>+ * single PCI bus. Another one that contains the primary PCI bus and its
>+ * subordinate PCI devices and buses. The second type of PE is normally
>+ * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
>+ */
>+static void __devinit pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all)
> {
>-	struct pci_controller *hose = pci_bus_to_host(dev->bus);
>+	struct pci_controller *hose = pci_bus_to_host(bus);
> 	struct pnv_phb *phb = hose->private_data;
>-	struct pci_bus *bus = dev->subordinate;
> 	struct pnv_ioda_pe *pe;
> 	int pe_num;
>
>-	if (!bus) {
>-		pr_warning("%s: Bridge without a subordinate bus !\n",
>-			   pci_name(dev));
>-		return;
>-	}
> 	pe_num = pnv_ioda_alloc_pe(phb);
> 	if (pe_num == IODA_INVALID_PE) {
>-		pr_warning("%s: Not enough PE# available, disabling bus\n",
>-			   pci_name(dev));
>+		pr_warning("%s: Not enough PE# available for PCI bus %04x:%02x\n",
>+			__func__, pci_domain_nr(bus), bus->number);
> 		return;
> 	}
>
> 	pe = &phb->ioda.pe_array[pe_num];
>-	ppe->bus_pe = pe;
>+	pe->flags = (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
> 	pe->pbus = bus;
>+	pe->pe_number = pe_num;

Gavin, 

Sorry for the late reply. I am not sure I a replying on the latest code. If
not, please point me out. 

I think we don't need to add this line. the pe->pe_number is already set in
pnv_ioda_alloc_pe().

> 	pe->pdev = NULL;
> 	pe->tce32_seg = -1;
> 	pe->mve_number = -1;
> 	pe->rid = bus->secondary << 8;
> 	pe->dma_weight = 0;
>
>-	pe_info(pe, "Secondary busses %d..%d associated with PE\n",
>-		bus->secondary, bus->subordinate);
>+	if (all)
>+		pe_info(pe, "Secondary busses %d..%d associated with PE#%d\n",
>+			bus->secondary, bus->subordinate, pe_num);
>+	else
>+		pe_info(pe, "Secondary busses %d associated with PE#%d\n",
>+			bus->secondary, pe_num);
>
> 	if (pnv_ioda_configure_pe(phb, pe)) {
> 		/* XXX What do we do here ? */
>@@ -848,17 +845,33 @@ static void __devinit pnv_ioda_setup_bus_PE(struct pci_dev *dev,
> static void __devinit pnv_ioda_setup_PEs(struct pci_bus *bus)
> {
> 	struct pci_dev *dev;
>-	struct pnv_ioda_pe *pe;
>+
>+	pnv_ioda_setup_bus_PE(bus, 0);
>
> 	list_for_each_entry(dev, &bus->devices, bus_list) {
>-		pe = pnv_ioda_setup_dev_PE(dev);
>-		if (pe == NULL)
>-			continue;
>-		/* Leaving the PCIe domain ... single PE# */
>-		if (dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
>-			pnv_ioda_setup_bus_PE(dev, pe);
>-		else if (dev->subordinate)
>-			pnv_ioda_setup_PEs(dev->subordinate);
>+		if (dev->subordinate) {
>+			if (dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
>+				pnv_ioda_setup_bus_PE(dev->subordinate, 1);
>+			else
>+				pnv_ioda_setup_PEs(dev->subordinate);
>+		}
>+	}
>+}
>+
>+/*
>+ * Configure PEs so that the downstream PCI buses and devices
>+ * could have their associated PE#. Unfortunately, we didn't
>+ * figure out the way to identify the PLX bridge yet. So we
>+ * simply put the PCI bus and the subordinate behind the root
>+ * port to PE# here. The game rule here is expected to be changed
>+ * as soon as we can detected PLX bridge correctly.
>+ */
>+static void __devinit pnv_pci_ioda_setup_PEs(void)
>+{
>+	struct pci_controller *hose, *tmp;
>+
>+	list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
>+		pnv_ioda_setup_PEs(hose->bus);
> 	}
> }
>
>@@ -1139,6 +1152,11 @@ static void __devinit pnv_pci_ioda_fixup_phb(struct pci_controller *hose)
> 	}
> }
>
>+static void __devinit pnv_pci_ioda_fixup(void)
>+{
>+	pnv_pci_ioda_setup_PEs();
>+}
>+
> /* Prevent enabling devices for which we couldn't properly
>  * assign a PE
>  */
>@@ -1305,6 +1323,7 @@ void __init pnv_pci_init_ioda1_phb(struct device_node *np)
> 	 * ourselves here
> 	 */
> 	ppc_md.pcibios_fixup_phb = pnv_pci_ioda_fixup_phb;
>+	ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
> 	ppc_md.pcibios_enable_device_hook = pnv_pci_enable_device_hook;
> 	pci_add_flags(PCI_PROBE_ONLY | PCI_REASSIGN_ALL_RSRC);
>
>diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
>index 8bc4796..0cb760c 100644
>--- a/arch/powerpc/platforms/powernv/pci.h
>+++ b/arch/powerpc/platforms/powernv/pci.h
>@@ -17,9 +17,14 @@ enum pnv_phb_model {
> };
>
> #define PNV_PCI_DIAG_BUF_SIZE	4096
>+#define PNV_IODA_PE_DEV		(1 << 0)	/* PE has single PCI device	*/
>+#define PNV_IODA_PE_BUS		(1 << 1)	/* PE has primary PCI bus	*/
>+#define PNV_IODA_PE_BUS_ALL	(1 << 2)	/* PE has subordinate buses	*/
>
> /* Data associated with a PE, including IOMMU tracking etc.. */
> struct pnv_ioda_pe {
>+	unsigned long		flags;
>+
> 	/* A PE can be associated with a single device or an
> 	 * entire bus (& children). In the former case, pdev
> 	 * is populated, in the later case, pbus is.
>@@ -40,11 +45,6 @@ struct pnv_ioda_pe {
> 	 */
> 	unsigned int		dma_weight;
>
>-	/* This is a PCI-E -> PCI-X bridge, this points to the
>-	 * corresponding bus PE
>-	 */
>-	struct pnv_ioda_pe	*bus_pe;
>-
> 	/* "Base" iommu table, ie, 4K TCEs, 32-bit DMA */
> 	int			tce32_seg;
> 	int			tce32_segcount;
>-- 
>1.7.9.5
>
>_______________________________________________
>Linuxppc-dev mailing list
>Linuxppc-dev@lists.ozlabs.org
>https://lists.ozlabs.org/listinfo/linuxppc-dev

-- 
Richard Yang
Help you, Help me

Re: [PATCH 1/8] ppc/pnv: create bus sensitive PEs

[Gavin Shan]

On Wed, Aug 01, 2012 at 03:49:41AM -0400, Richard Yang wrote:
>On Mon, Jun 25, 2012 at 11:43:14PM +0800, Gavin Shan wrote:
>>Basically, there're 2 types of PCI bus sensitive PEs: (A) The PE
>>includes single PCI bus. (B) The PE includes the PCI bus and all
>>the subordinate PCI buses. At present, we'd like to put PCI bus
>>originated by PCI-e link to form PE that contains single PCI bus,
>>and the PCIe-to-PCI bridge will form the 2nd type of PE. We don't
>>figure out to detect PLX bridge yet. Once we can detect PLX bridge
>>some day, we have to put PCI buses originated from the downstream
>>port of PLX bridge to the 2nd type of PE.
>>
>>The patch changes the original implementation for a little bit
>>to support 2 types of PCI bus sensitive PEs described as above.
>>Also, the function used to retrieve the corresponding PE according
>>to the given PCI device has been changed based on that because each
>>PCI device should trace the directly associated PE.
>>
>>Signed-off-by: Gavin Shan <shangw@linux.vnet.ibm.com>
>>Reviewed-by: Ram Pai <linuxram@us.ibm.com>
>>Reviewed-by: Richard Yang <weiyang@linux.vnet.ibm.com>
>>---
>> arch/powerpc/platforms/powernv/pci-ioda.c |   97 +++++++++++++++++------------
>> arch/powerpc/platforms/powernv/pci.h      |   10 +--
>> 2 files changed, 63 insertions(+), 44 deletions(-)
>>
>>diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
>>index fbdd74d..1504795 100644
>>--- a/arch/powerpc/platforms/powernv/pci-ioda.c
>>+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
>>@@ -548,7 +548,7 @@ static void __devinit pnv_ioda_free_pe(struct pnv_phb *phb, int pe)
>>  * but in the meantime, we need to protect them to avoid warnings
>>  */
>> #ifdef CONFIG_PCI_MSI
>>-static struct pnv_ioda_pe * __devinit __pnv_ioda_get_one_pe(struct pci_dev *dev)
>>+static struct pnv_ioda_pe * __devinit pnv_ioda_get_pe(struct pci_dev *dev)
>> {
>> 	struct pci_controller *hose = pci_bus_to_host(dev->bus);
>> 	struct pnv_phb *phb = hose->private_data;
>>@@ -560,19 +560,6 @@ static struct pnv_ioda_pe * __devinit __pnv_ioda_get_one_pe(struct pci_dev *dev)
>> 		return NULL;
>> 	return &phb->ioda.pe_array[pdn->pe_number];
>> }
>>-
>>-static struct pnv_ioda_pe * __devinit pnv_ioda_get_pe(struct pci_dev *dev)
>>-{
>>-	struct pnv_ioda_pe *pe = __pnv_ioda_get_one_pe(dev);
>>-
>>-	while (!pe && dev->bus->self) {
>>-		dev = dev->bus->self;
>>-		pe = __pnv_ioda_get_one_pe(dev);
>>-		if (pe)
>>-			pe = pe->bus_pe;
>>-	}
>>-	return pe;
>>-}
>> #endif /* CONFIG_PCI_MSI */
>>
>> static int __devinit pnv_ioda_configure_pe(struct pnv_phb *phb,
>>@@ -589,7 +576,11 @@ static int __devinit pnv_ioda_configure_pe(struct pnv_phb *phb,
>> 		dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
>> 		fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
>> 		parent = pe->pbus->self;
>>-		count = pe->pbus->subordinate - pe->pbus->secondary + 1;
>>+		if (pe->flags & PNV_IODA_PE_BUS_ALL)
>>+			count = pe->pbus->subordinate - pe->pbus->secondary + 1;
>>+		else
>>+			count = 1;
>>+
>> 		switch(count) {
>> 		case  1: bcomp = OpalPciBusAll;		break;
>> 		case  2: bcomp = OpalPciBus7Bits;	break;
>>@@ -699,6 +690,7 @@ static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev)
>> 	return 10;
>> }
>>
>>+#if 0
>> static struct pnv_ioda_pe * __devinit pnv_ioda_setup_dev_PE(struct pci_dev *dev)
>> {
>> 	struct pci_controller *hose = pci_bus_to_host(dev->bus);
>>@@ -767,6 +759,7 @@ static struct pnv_ioda_pe * __devinit pnv_ioda_setup_dev_PE(struct pci_dev *dev)
>>
>> 	return pe;
>> }
>>+#endif /* Useful for SRIOV case */
>>
>> static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
>> {
>>@@ -784,43 +777,47 @@ static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
>> 		pdn->pcidev = dev;
>> 		pdn->pe_number = pe->pe_number;
>> 		pe->dma_weight += pnv_ioda_dma_weight(dev);
>>-		if (dev->subordinate)
>>+		if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
>> 			pnv_ioda_setup_same_PE(dev->subordinate, pe);
>> 	}
>> }
>>
>>-static void __devinit pnv_ioda_setup_bus_PE(struct pci_dev *dev,
>>-					    struct pnv_ioda_pe *ppe)
>>+/*
>>+ * There're 2 types of PCI bus sensitive PEs: One that is compromised of
>>+ * single PCI bus. Another one that contains the primary PCI bus and its
>>+ * subordinate PCI devices and buses. The second type of PE is normally
>>+ * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
>>+ */
>>+static void __devinit pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all)
>> {
>>-	struct pci_controller *hose = pci_bus_to_host(dev->bus);
>>+	struct pci_controller *hose = pci_bus_to_host(bus);
>> 	struct pnv_phb *phb = hose->private_data;
>>-	struct pci_bus *bus = dev->subordinate;
>> 	struct pnv_ioda_pe *pe;
>> 	int pe_num;
>>
>>-	if (!bus) {
>>-		pr_warning("%s: Bridge without a subordinate bus !\n",
>>-			   pci_name(dev));
>>-		return;
>>-	}
>> 	pe_num = pnv_ioda_alloc_pe(phb);
>> 	if (pe_num == IODA_INVALID_PE) {
>>-		pr_warning("%s: Not enough PE# available, disabling bus\n",
>>-			   pci_name(dev));
>>+		pr_warning("%s: Not enough PE# available for PCI bus %04x:%02x\n",
>>+			__func__, pci_domain_nr(bus), bus->number);
>> 		return;
>> 	}
>>
>> 	pe = &phb->ioda.pe_array[pe_num];
>>-	ppe->bus_pe = pe;
>>+	pe->flags = (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
>> 	pe->pbus = bus;
>>+	pe->pe_number = pe_num;
>
>Gavin, 
>
>Sorry for the late reply. I am not sure I a replying on the latest code. If
>not, please point me out. 
>
>I think we don't need to add this line. the pe->pe_number is already set in
>pnv_ioda_alloc_pe().
>

Thanks, Richard. I think we probablly need remove the following line in pnv_ioda_alloc_pe()
instead of the line you pointed because pnv_ioda_alloc_pe() might return invalid
PE number (-1). That will eventually cause data corruption while using "-1" to
referring phb->ioda.pe_array[], even the situation shouldn't happen for now :-)

	phb->ioda.pe_array[pe].pe_number = pe;

Let me change it accordingly in next version. The series of patches is pending
for the patches against PCI core change. The later one is waiting for Bjorn's
confirm.

Thanks,
Gavin

>> 	pe->pdev = NULL;
>> 	pe->tce32_seg = -1;
>> 	pe->mve_number = -1;
>> 	pe->rid = bus->secondary << 8;
>> 	pe->dma_weight = 0;
>>
>>-	pe_info(pe, "Secondary busses %d..%d associated with PE\n",
>>-		bus->secondary, bus->subordinate);
>>+	if (all)
>>+		pe_info(pe, "Secondary busses %d..%d associated with PE#%d\n",
>>+			bus->secondary, bus->subordinate, pe_num);
>>+	else
>>+		pe_info(pe, "Secondary busses %d associated with PE#%d\n",
>>+			bus->secondary, pe_num);
>>
>> 	if (pnv_ioda_configure_pe(phb, pe)) {
>> 		/* XXX What do we do here ? */
>>@@ -848,17 +845,33 @@ static void __devinit pnv_ioda_setup_bus_PE(struct pci_dev *dev,
>> static void __devinit pnv_ioda_setup_PEs(struct pci_bus *bus)
>> {
>> 	struct pci_dev *dev;
>>-	struct pnv_ioda_pe *pe;
>>+
>>+	pnv_ioda_setup_bus_PE(bus, 0);
>>
>> 	list_for_each_entry(dev, &bus->devices, bus_list) {
>>-		pe = pnv_ioda_setup_dev_PE(dev);
>>-		if (pe == NULL)
>>-			continue;
>>-		/* Leaving the PCIe domain ... single PE# */
>>-		if (dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
>>-			pnv_ioda_setup_bus_PE(dev, pe);
>>-		else if (dev->subordinate)
>>-			pnv_ioda_setup_PEs(dev->subordinate);
>>+		if (dev->subordinate) {
>>+			if (dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
>>+				pnv_ioda_setup_bus_PE(dev->subordinate, 1);
>>+			else
>>+				pnv_ioda_setup_PEs(dev->subordinate);
>>+		}
>>+	}
>>+}
>>+
>>+/*
>>+ * Configure PEs so that the downstream PCI buses and devices
>>+ * could have their associated PE#. Unfortunately, we didn't
>>+ * figure out the way to identify the PLX bridge yet. So we
>>+ * simply put the PCI bus and the subordinate behind the root
>>+ * port to PE# here. The game rule here is expected to be changed
>>+ * as soon as we can detected PLX bridge correctly.
>>+ */
>>+static void __devinit pnv_pci_ioda_setup_PEs(void)
>>+{
>>+	struct pci_controller *hose, *tmp;
>>+
>>+	list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
>>+		pnv_ioda_setup_PEs(hose->bus);
>> 	}
>> }
>>
>>@@ -1139,6 +1152,11 @@ static void __devinit pnv_pci_ioda_fixup_phb(struct pci_controller *hose)
>> 	}
>> }
>>
>>+static void __devinit pnv_pci_ioda_fixup(void)
>>+{
>>+	pnv_pci_ioda_setup_PEs();
>>+}
>>+
>> /* Prevent enabling devices for which we couldn't properly
>>  * assign a PE
>>  */
>>@@ -1305,6 +1323,7 @@ void __init pnv_pci_init_ioda1_phb(struct device_node *np)
>> 	 * ourselves here
>> 	 */
>> 	ppc_md.pcibios_fixup_phb = pnv_pci_ioda_fixup_phb;
>>+	ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
>> 	ppc_md.pcibios_enable_device_hook = pnv_pci_enable_device_hook;
>> 	pci_add_flags(PCI_PROBE_ONLY | PCI_REASSIGN_ALL_RSRC);
>>
>>diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
>>index 8bc4796..0cb760c 100644
>>--- a/arch/powerpc/platforms/powernv/pci.h
>>+++ b/arch/powerpc/platforms/powernv/pci.h
>>@@ -17,9 +17,14 @@ enum pnv_phb_model {
>> };
>>
>> #define PNV_PCI_DIAG_BUF_SIZE	4096
>>+#define PNV_IODA_PE_DEV		(1 << 0)	/* PE has single PCI device	*/
>>+#define PNV_IODA_PE_BUS		(1 << 1)	/* PE has primary PCI bus	*/
>>+#define PNV_IODA_PE_BUS_ALL	(1 << 2)	/* PE has subordinate buses	*/
>>
>> /* Data associated with a PE, including IOMMU tracking etc.. */
>> struct pnv_ioda_pe {
>>+	unsigned long		flags;
>>+
>> 	/* A PE can be associated with a single device or an
>> 	 * entire bus (& children). In the former case, pdev
>> 	 * is populated, in the later case, pbus is.
>>@@ -40,11 +45,6 @@ struct pnv_ioda_pe {
>> 	 */
>> 	unsigned int		dma_weight;
>>
>>-	/* This is a PCI-E -> PCI-X bridge, this points to the
>>-	 * corresponding bus PE
>>-	 */
>>-	struct pnv_ioda_pe	*bus_pe;
>>-
>> 	/* "Base" iommu table, ie, 4K TCEs, 32-bit DMA */
>> 	int			tce32_seg;
>> 	int			tce32_segcount;
>>-- 
>>1.7.9.5
>>
>>_______________________________________________
>>Linuxppc-dev mailing list
>>Linuxppc-dev@lists.ozlabs.org
>>https://lists.ozlabs.org/listinfo/linuxppc-dev
>
>-- 
>Richard Yang
>Help you, Help me

Re: [PATCH 1/8] ppc/pnv: create bus sensitive PEs

[Richard Yang]

On Wed, Aug 01, 2012 at 04:26:54PM +0800, Gavin Shan wrote:
>On Wed, Aug 01, 2012 at 03:49:41AM -0400, Richard Yang wrote:
>>On Mon, Jun 25, 2012 at 11:43:14PM +0800, Gavin Shan wrote:
>>>Basically, there're 2 types of PCI bus sensitive PEs: (A) The PE
>>>includes single PCI bus. (B) The PE includes the PCI bus and all
>>>the subordinate PCI buses. At present, we'd like to put PCI bus
>>>originated by PCI-e link to form PE that contains single PCI bus,
>>>and the PCIe-to-PCI bridge will form the 2nd type of PE. We don't
>>>figure out to detect PLX bridge yet. Once we can detect PLX bridge
>>>some day, we have to put PCI buses originated from the downstream
>>>port of PLX bridge to the 2nd type of PE.
>>>
>>>The patch changes the original implementation for a little bit
>>>to support 2 types of PCI bus sensitive PEs described as above.
>>>Also, the function used to retrieve the corresponding PE according
>>>to the given PCI device has been changed based on that because each
>>>PCI device should trace the directly associated PE.
>>>
>>>Signed-off-by: Gavin Shan <shangw@linux.vnet.ibm.com>
>>>Reviewed-by: Ram Pai <linuxram@us.ibm.com>
>>>Reviewed-by: Richard Yang <weiyang@linux.vnet.ibm.com>
>>>---
>>> arch/powerpc/platforms/powernv/pci-ioda.c |   97 +++++++++++++++++------------
>>> arch/powerpc/platforms/powernv/pci.h      |   10 +--
>>> 2 files changed, 63 insertions(+), 44 deletions(-)
>>>
>>>diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
>>>index fbdd74d..1504795 100644
>>>--- a/arch/powerpc/platforms/powernv/pci-ioda.c
>>>+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
>>>@@ -548,7 +548,7 @@ static void __devinit pnv_ioda_free_pe(struct pnv_phb *phb, int pe)
>>>  * but in the meantime, we need to protect them to avoid warnings
>>>  */
>>> #ifdef CONFIG_PCI_MSI
>>>-static struct pnv_ioda_pe * __devinit __pnv_ioda_get_one_pe(struct pci_dev *dev)
>>>+static struct pnv_ioda_pe * __devinit pnv_ioda_get_pe(struct pci_dev *dev)
>>> {
>>> 	struct pci_controller *hose = pci_bus_to_host(dev->bus);
>>> 	struct pnv_phb *phb = hose->private_data;
>>>@@ -560,19 +560,6 @@ static struct pnv_ioda_pe * __devinit __pnv_ioda_get_one_pe(struct pci_dev *dev)
>>> 		return NULL;
>>> 	return &phb->ioda.pe_array[pdn->pe_number];
>>> }
>>>-
>>>-static struct pnv_ioda_pe * __devinit pnv_ioda_get_pe(struct pci_dev *dev)
>>>-{
>>>-	struct pnv_ioda_pe *pe = __pnv_ioda_get_one_pe(dev);
>>>-
>>>-	while (!pe && dev->bus->self) {
>>>-		dev = dev->bus->self;
>>>-		pe = __pnv_ioda_get_one_pe(dev);
>>>-		if (pe)
>>>-			pe = pe->bus_pe;
>>>-	}
>>>-	return pe;
>>>-}
>>> #endif /* CONFIG_PCI_MSI */
>>>
>>> static int __devinit pnv_ioda_configure_pe(struct pnv_phb *phb,
>>>@@ -589,7 +576,11 @@ static int __devinit pnv_ioda_configure_pe(struct pnv_phb *phb,
>>> 		dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
>>> 		fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
>>> 		parent = pe->pbus->self;
>>>-		count = pe->pbus->subordinate - pe->pbus->secondary + 1;
>>>+		if (pe->flags & PNV_IODA_PE_BUS_ALL)
>>>+			count = pe->pbus->subordinate - pe->pbus->secondary + 1;
>>>+		else
>>>+			count = 1;
>>>+
>>> 		switch(count) {
>>> 		case  1: bcomp = OpalPciBusAll;		break;
>>> 		case  2: bcomp = OpalPciBus7Bits;	break;
>>>@@ -699,6 +690,7 @@ static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev)
>>> 	return 10;
>>> }
>>>
>>>+#if 0
>>> static struct pnv_ioda_pe * __devinit pnv_ioda_setup_dev_PE(struct pci_dev *dev)
>>> {
>>> 	struct pci_controller *hose = pci_bus_to_host(dev->bus);
>>>@@ -767,6 +759,7 @@ static struct pnv_ioda_pe * __devinit pnv_ioda_setup_dev_PE(struct pci_dev *dev)
>>>
>>> 	return pe;
>>> }
>>>+#endif /* Useful for SRIOV case */
>>>
>>> static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
>>> {
>>>@@ -784,43 +777,47 @@ static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
>>> 		pdn->pcidev = dev;
>>> 		pdn->pe_number = pe->pe_number;
>>> 		pe->dma_weight += pnv_ioda_dma_weight(dev);
>>>-		if (dev->subordinate)
>>>+		if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
>>> 			pnv_ioda_setup_same_PE(dev->subordinate, pe);
>>> 	}
>>> }
>>>
>>>-static void __devinit pnv_ioda_setup_bus_PE(struct pci_dev *dev,
>>>-					    struct pnv_ioda_pe *ppe)
>>>+/*
>>>+ * There're 2 types of PCI bus sensitive PEs: One that is compromised of
>>>+ * single PCI bus. Another one that contains the primary PCI bus and its
>>>+ * subordinate PCI devices and buses. The second type of PE is normally
>>>+ * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
>>>+ */
>>>+static void __devinit pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all)
>>> {
>>>-	struct pci_controller *hose = pci_bus_to_host(dev->bus);
>>>+	struct pci_controller *hose = pci_bus_to_host(bus);
>>> 	struct pnv_phb *phb = hose->private_data;
>>>-	struct pci_bus *bus = dev->subordinate;
>>> 	struct pnv_ioda_pe *pe;
>>> 	int pe_num;
>>>
>>>-	if (!bus) {
>>>-		pr_warning("%s: Bridge without a subordinate bus !\n",
>>>-			   pci_name(dev));
>>>-		return;
>>>-	}
>>> 	pe_num = pnv_ioda_alloc_pe(phb);
>>> 	if (pe_num == IODA_INVALID_PE) {
>>>-		pr_warning("%s: Not enough PE# available, disabling bus\n",
>>>-			   pci_name(dev));
>>>+		pr_warning("%s: Not enough PE# available for PCI bus %04x:%02x\n",
>>>+			__func__, pci_domain_nr(bus), bus->number);
>>> 		return;
>>> 	}
>>>
>>> 	pe = &phb->ioda.pe_array[pe_num];
>>>-	ppe->bus_pe = pe;
>>>+	pe->flags = (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
>>> 	pe->pbus = bus;
>>>+	pe->pe_number = pe_num;
>>
>>Gavin, 
>>
>>Sorry for the late reply. I am not sure I a replying on the latest code. If
>>not, please point me out. 
>>
>>I think we don't need to add this line. the pe->pe_number is already set in
>>pnv_ioda_alloc_pe().
>>
>
>Thanks, Richard. I think we probablly need remove the following line in pnv_ioda_alloc_pe()
>instead of the line you pointed because pnv_ioda_alloc_pe() might return invalid
>PE number (-1). That will eventually cause data corruption while using "-1" to
>referring phb->ioda.pe_array[], even the situation shouldn't happen for now :-)
>
>	phb->ioda.pe_array[pe].pe_number = pe;

oh, so it is not proper to set pe_number = -1 in the pe_array, right?

>
>Let me change it accordingly in next version. The series of patches is pending
>for the patches against PCI core change. The later one is waiting for Bjorn's
>confirm.
>
>Thanks,
>Gavin
>
>>> 	pe->pdev = NULL;
>>> 	pe->tce32_seg = -1;
>>> 	pe->mve_number = -1;
>>> 	pe->rid = bus->secondary << 8;
>>> 	pe->dma_weight = 0;
>>>
>>>-	pe_info(pe, "Secondary busses %d..%d associated with PE\n",
>>>-		bus->secondary, bus->subordinate);
>>>+	if (all)
>>>+		pe_info(pe, "Secondary busses %d..%d associated with PE#%d\n",
>>>+			bus->secondary, bus->subordinate, pe_num);
>>>+	else
>>>+		pe_info(pe, "Secondary busses %d associated with PE#%d\n",
>>>+			bus->secondary, pe_num);
>>>
>>> 	if (pnv_ioda_configure_pe(phb, pe)) {
>>> 		/* XXX What do we do here ? */
>>>@@ -848,17 +845,33 @@ static void __devinit pnv_ioda_setup_bus_PE(struct pci_dev *dev,
>>> static void __devinit pnv_ioda_setup_PEs(struct pci_bus *bus)
>>> {
>>> 	struct pci_dev *dev;
>>>-	struct pnv_ioda_pe *pe;
>>>+
>>>+	pnv_ioda_setup_bus_PE(bus, 0);
>>>
>>> 	list_for_each_entry(dev, &bus->devices, bus_list) {
>>>-		pe = pnv_ioda_setup_dev_PE(dev);
>>>-		if (pe == NULL)
>>>-			continue;
>>>-		/* Leaving the PCIe domain ... single PE# */
>>>-		if (dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
>>>-			pnv_ioda_setup_bus_PE(dev, pe);
>>>-		else if (dev->subordinate)
>>>-			pnv_ioda_setup_PEs(dev->subordinate);
>>>+		if (dev->subordinate) {
>>>+			if (dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
>>>+				pnv_ioda_setup_bus_PE(dev->subordinate, 1);
>>>+			else
>>>+				pnv_ioda_setup_PEs(dev->subordinate);
>>>+		}
>>>+	}
>>>+}
>>>+
>>>+/*
>>>+ * Configure PEs so that the downstream PCI buses and devices
>>>+ * could have their associated PE#. Unfortunately, we didn't
>>>+ * figure out the way to identify the PLX bridge yet. So we
>>>+ * simply put the PCI bus and the subordinate behind the root
>>>+ * port to PE# here. The game rule here is expected to be changed
>>>+ * as soon as we can detected PLX bridge correctly.
>>>+ */
>>>+static void __devinit pnv_pci_ioda_setup_PEs(void)
>>>+{
>>>+	struct pci_controller *hose, *tmp;
>>>+
>>>+	list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
>>>+		pnv_ioda_setup_PEs(hose->bus);
>>> 	}
>>> }
>>>
>>>@@ -1139,6 +1152,11 @@ static void __devinit pnv_pci_ioda_fixup_phb(struct pci_controller *hose)
>>> 	}
>>> }
>>>
>>>+static void __devinit pnv_pci_ioda_fixup(void)
>>>+{
>>>+	pnv_pci_ioda_setup_PEs();
>>>+}
>>>+
>>> /* Prevent enabling devices for which we couldn't properly
>>>  * assign a PE
>>>  */
>>>@@ -1305,6 +1323,7 @@ void __init pnv_pci_init_ioda1_phb(struct device_node *np)
>>> 	 * ourselves here
>>> 	 */
>>> 	ppc_md.pcibios_fixup_phb = pnv_pci_ioda_fixup_phb;
>>>+	ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
>>> 	ppc_md.pcibios_enable_device_hook = pnv_pci_enable_device_hook;
>>> 	pci_add_flags(PCI_PROBE_ONLY | PCI_REASSIGN_ALL_RSRC);
>>>
>>>diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
>>>index 8bc4796..0cb760c 100644
>>>--- a/arch/powerpc/platforms/powernv/pci.h
>>>+++ b/arch/powerpc/platforms/powernv/pci.h
>>>@@ -17,9 +17,14 @@ enum pnv_phb_model {
>>> };
>>>
>>> #define PNV_PCI_DIAG_BUF_SIZE	4096
>>>+#define PNV_IODA_PE_DEV		(1 << 0)	/* PE has single PCI device	*/
>>>+#define PNV_IODA_PE_BUS		(1 << 1)	/* PE has primary PCI bus	*/
>>>+#define PNV_IODA_PE_BUS_ALL	(1 << 2)	/* PE has subordinate buses	*/
>>>
>>> /* Data associated with a PE, including IOMMU tracking etc.. */
>>> struct pnv_ioda_pe {
>>>+	unsigned long		flags;
>>>+
>>> 	/* A PE can be associated with a single device or an
>>> 	 * entire bus (& children). In the former case, pdev
>>> 	 * is populated, in the later case, pbus is.
>>>@@ -40,11 +45,6 @@ struct pnv_ioda_pe {
>>> 	 */
>>> 	unsigned int		dma_weight;
>>>
>>>-	/* This is a PCI-E -> PCI-X bridge, this points to the
>>>-	 * corresponding bus PE
>>>-	 */
>>>-	struct pnv_ioda_pe	*bus_pe;
>>>-
>>> 	/* "Base" iommu table, ie, 4K TCEs, 32-bit DMA */
>>> 	int			tce32_seg;
>>> 	int			tce32_segcount;
>>>-- 
>>>1.7.9.5
>>>
>>>_______________________________________________
>>>Linuxppc-dev mailing list
>>>Linuxppc-dev@lists.ozlabs.org
>>>https://lists.ozlabs.org/listinfo/linuxppc-dev
>>
>>-- 
>>Richard Yang
>>Help you, Help me
>
>_______________________________________________
>Linuxppc-dev mailing list
>Linuxppc-dev@lists.ozlabs.org
>https://lists.ozlabs.org/listinfo/linuxppc-dev

-- 
Richard Yang
Help you, Help me

Re: [PATCH 1/8] ppc/pnv: create bus sensitive PEs

[Gavin Shan]

On Wed, Aug 01, 2012 at 05:04:46AM -0400, Richard Yang wrote:
>On Wed, Aug 01, 2012 at 04:26:54PM +0800, Gavin Shan wrote:
>>On Wed, Aug 01, 2012 at 03:49:41AM -0400, Richard Yang wrote:
>>>On Mon, Jun 25, 2012 at 11:43:14PM +0800, Gavin Shan wrote:
>>>>Basically, there're 2 types of PCI bus sensitive PEs: (A) The PE
>>>>includes single PCI bus. (B) The PE includes the PCI bus and all
>>>>the subordinate PCI buses. At present, we'd like to put PCI bus
>>>>originated by PCI-e link to form PE that contains single PCI bus,
>>>>and the PCIe-to-PCI bridge will form the 2nd type of PE. We don't
>>>>figure out to detect PLX bridge yet. Once we can detect PLX bridge
>>>>some day, we have to put PCI buses originated from the downstream
>>>>port of PLX bridge to the 2nd type of PE.
>>>>
>>>>The patch changes the original implementation for a little bit
>>>>to support 2 types of PCI bus sensitive PEs described as above.
>>>>Also, the function used to retrieve the corresponding PE according
>>>>to the given PCI device has been changed based on that because each
>>>>PCI device should trace the directly associated PE.
>>>>
>>>>Signed-off-by: Gavin Shan <shangw@linux.vnet.ibm.com>
>>>>Reviewed-by: Ram Pai <linuxram@us.ibm.com>
>>>>Reviewed-by: Richard Yang <weiyang@linux.vnet.ibm.com>
>>>>---
>>>> arch/powerpc/platforms/powernv/pci-ioda.c |   97 +++++++++++++++++------------
>>>> arch/powerpc/platforms/powernv/pci.h      |   10 +--
>>>> 2 files changed, 63 insertions(+), 44 deletions(-)
>>>>
>>>>diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
>>>>index fbdd74d..1504795 100644
>>>>--- a/arch/powerpc/platforms/powernv/pci-ioda.c
>>>>+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
>>>>@@ -548,7 +548,7 @@ static void __devinit pnv_ioda_free_pe(struct pnv_phb *phb, int pe)
>>>>  * but in the meantime, we need to protect them to avoid warnings
>>>>  */
>>>> #ifdef CONFIG_PCI_MSI
>>>>-static struct pnv_ioda_pe * __devinit __pnv_ioda_get_one_pe(struct pci_dev *dev)
>>>>+static struct pnv_ioda_pe * __devinit pnv_ioda_get_pe(struct pci_dev *dev)
>>>> {
>>>> 	struct pci_controller *hose = pci_bus_to_host(dev->bus);
>>>> 	struct pnv_phb *phb = hose->private_data;
>>>>@@ -560,19 +560,6 @@ static struct pnv_ioda_pe * __devinit __pnv_ioda_get_one_pe(struct pci_dev *dev)
>>>> 		return NULL;
>>>> 	return &phb->ioda.pe_array[pdn->pe_number];
>>>> }
>>>>-
>>>>-static struct pnv_ioda_pe * __devinit pnv_ioda_get_pe(struct pci_dev *dev)
>>>>-{
>>>>-	struct pnv_ioda_pe *pe = __pnv_ioda_get_one_pe(dev);
>>>>-
>>>>-	while (!pe && dev->bus->self) {
>>>>-		dev = dev->bus->self;
>>>>-		pe = __pnv_ioda_get_one_pe(dev);
>>>>-		if (pe)
>>>>-			pe = pe->bus_pe;
>>>>-	}
>>>>-	return pe;
>>>>-}
>>>> #endif /* CONFIG_PCI_MSI */
>>>>
>>>> static int __devinit pnv_ioda_configure_pe(struct pnv_phb *phb,
>>>>@@ -589,7 +576,11 @@ static int __devinit pnv_ioda_configure_pe(struct pnv_phb *phb,
>>>> 		dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
>>>> 		fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
>>>> 		parent = pe->pbus->self;
>>>>-		count = pe->pbus->subordinate - pe->pbus->secondary + 1;
>>>>+		if (pe->flags & PNV_IODA_PE_BUS_ALL)
>>>>+			count = pe->pbus->subordinate - pe->pbus->secondary + 1;
>>>>+		else
>>>>+			count = 1;
>>>>+
>>>> 		switch(count) {
>>>> 		case  1: bcomp = OpalPciBusAll;		break;
>>>> 		case  2: bcomp = OpalPciBus7Bits;	break;
>>>>@@ -699,6 +690,7 @@ static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev)
>>>> 	return 10;
>>>> }
>>>>
>>>>+#if 0
>>>> static struct pnv_ioda_pe * __devinit pnv_ioda_setup_dev_PE(struct pci_dev *dev)
>>>> {
>>>> 	struct pci_controller *hose = pci_bus_to_host(dev->bus);
>>>>@@ -767,6 +759,7 @@ static struct pnv_ioda_pe * __devinit pnv_ioda_setup_dev_PE(struct pci_dev *dev)
>>>>
>>>> 	return pe;
>>>> }
>>>>+#endif /* Useful for SRIOV case */
>>>>
>>>> static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
>>>> {
>>>>@@ -784,43 +777,47 @@ static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
>>>> 		pdn->pcidev = dev;
>>>> 		pdn->pe_number = pe->pe_number;
>>>> 		pe->dma_weight += pnv_ioda_dma_weight(dev);
>>>>-		if (dev->subordinate)
>>>>+		if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
>>>> 			pnv_ioda_setup_same_PE(dev->subordinate, pe);
>>>> 	}
>>>> }
>>>>
>>>>-static void __devinit pnv_ioda_setup_bus_PE(struct pci_dev *dev,
>>>>-					    struct pnv_ioda_pe *ppe)
>>>>+/*
>>>>+ * There're 2 types of PCI bus sensitive PEs: One that is compromised of
>>>>+ * single PCI bus. Another one that contains the primary PCI bus and its
>>>>+ * subordinate PCI devices and buses. The second type of PE is normally
>>>>+ * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
>>>>+ */
>>>>+static void __devinit pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all)
>>>> {
>>>>-	struct pci_controller *hose = pci_bus_to_host(dev->bus);
>>>>+	struct pci_controller *hose = pci_bus_to_host(bus);
>>>> 	struct pnv_phb *phb = hose->private_data;
>>>>-	struct pci_bus *bus = dev->subordinate;
>>>> 	struct pnv_ioda_pe *pe;
>>>> 	int pe_num;
>>>>
>>>>-	if (!bus) {
>>>>-		pr_warning("%s: Bridge without a subordinate bus !\n",
>>>>-			   pci_name(dev));
>>>>-		return;
>>>>-	}
>>>> 	pe_num = pnv_ioda_alloc_pe(phb);
>>>> 	if (pe_num == IODA_INVALID_PE) {
>>>>-		pr_warning("%s: Not enough PE# available, disabling bus\n",
>>>>-			   pci_name(dev));
>>>>+		pr_warning("%s: Not enough PE# available for PCI bus %04x:%02x\n",
>>>>+			__func__, pci_domain_nr(bus), bus->number);
>>>> 		return;
>>>> 	}
>>>>
>>>> 	pe = &phb->ioda.pe_array[pe_num];
>>>>-	ppe->bus_pe = pe;
>>>>+	pe->flags = (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
>>>> 	pe->pbus = bus;
>>>>+	pe->pe_number = pe_num;
>>>
>>>Gavin, 
>>>
>>>Sorry for the late reply. I am not sure I a replying on the latest code. If
>>>not, please point me out. 
>>>
>>>I think we don't need to add this line. the pe->pe_number is already set in
>>>pnv_ioda_alloc_pe().
>>>
>>
>>Thanks, Richard. I think we probablly need remove the following line in pnv_ioda_alloc_pe()
>>instead of the line you pointed because pnv_ioda_alloc_pe() might return invalid
>>PE number (-1). That will eventually cause data corruption while using "-1" to
>>referring phb->ioda.pe_array[], even the situation shouldn't happen for now :-)
>>
>>	phb->ioda.pe_array[pe].pe_number = pe;
>
>oh, so it is not proper to set pe_number = -1 in the pe_array, right?
>

It seems that I missed something. Anyway, moving the line from pnv_ioda_alloc_pe
or that one you pointed is ok. I will remove the line you pointed in next version.
Thanks a lot, Richard.

"-1" means invalid PE number. In previous reply, I tried to say that following code
will cause data corruption, which will never happen after looking into the code
again :-)

	phb->ioda.pe_array[-1].pe_number = -1;

Thanks,
Gavin

>>
>>Let me change it accordingly in next version. The series of patches is pending
>>for the patches against PCI core change. The later one is waiting for Bjorn's
>>confirm.
>>
>>Thanks,
>>Gavin
>>
>>>> 	pe->pdev = NULL;
>>>> 	pe->tce32_seg = -1;
>>>> 	pe->mve_number = -1;
>>>> 	pe->rid = bus->secondary << 8;
>>>> 	pe->dma_weight = 0;
>>>>
>>>>-	pe_info(pe, "Secondary busses %d..%d associated with PE\n",
>>>>-		bus->secondary, bus->subordinate);
>>>>+	if (all)
>>>>+		pe_info(pe, "Secondary busses %d..%d associated with PE#%d\n",
>>>>+			bus->secondary, bus->subordinate, pe_num);
>>>>+	else
>>>>+		pe_info(pe, "Secondary busses %d associated with PE#%d\n",
>>>>+			bus->secondary, pe_num);
>>>>
>>>> 	if (pnv_ioda_configure_pe(phb, pe)) {
>>>> 		/* XXX What do we do here ? */
>>>>@@ -848,17 +845,33 @@ static void __devinit pnv_ioda_setup_bus_PE(struct pci_dev *dev,
>>>> static void __devinit pnv_ioda_setup_PEs(struct pci_bus *bus)
>>>> {
>>>> 	struct pci_dev *dev;
>>>>-	struct pnv_ioda_pe *pe;
>>>>+
>>>>+	pnv_ioda_setup_bus_PE(bus, 0);
>>>>
>>>> 	list_for_each_entry(dev, &bus->devices, bus_list) {
>>>>-		pe = pnv_ioda_setup_dev_PE(dev);
>>>>-		if (pe == NULL)
>>>>-			continue;
>>>>-		/* Leaving the PCIe domain ... single PE# */
>>>>-		if (dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
>>>>-			pnv_ioda_setup_bus_PE(dev, pe);
>>>>-		else if (dev->subordinate)
>>>>-			pnv_ioda_setup_PEs(dev->subordinate);
>>>>+		if (dev->subordinate) {
>>>>+			if (dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
>>>>+				pnv_ioda_setup_bus_PE(dev->subordinate, 1);
>>>>+			else
>>>>+				pnv_ioda_setup_PEs(dev->subordinate);
>>>>+		}
>>>>+	}
>>>>+}
>>>>+
>>>>+/*
>>>>+ * Configure PEs so that the downstream PCI buses and devices
>>>>+ * could have their associated PE#. Unfortunately, we didn't
>>>>+ * figure out the way to identify the PLX bridge yet. So we
>>>>+ * simply put the PCI bus and the subordinate behind the root
>>>>+ * port to PE# here. The game rule here is expected to be changed
>>>>+ * as soon as we can detected PLX bridge correctly.
>>>>+ */
>>>>+static void __devinit pnv_pci_ioda_setup_PEs(void)
>>>>+{
>>>>+	struct pci_controller *hose, *tmp;
>>>>+
>>>>+	list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
>>>>+		pnv_ioda_setup_PEs(hose->bus);
>>>> 	}
>>>> }
>>>>
>>>>@@ -1139,6 +1152,11 @@ static void __devinit pnv_pci_ioda_fixup_phb(struct pci_controller *hose)
>>>> 	}
>>>> }
>>>>
>>>>+static void __devinit pnv_pci_ioda_fixup(void)
>>>>+{
>>>>+	pnv_pci_ioda_setup_PEs();
>>>>+}
>>>>+
>>>> /* Prevent enabling devices for which we couldn't properly
>>>>  * assign a PE
>>>>  */
>>>>@@ -1305,6 +1323,7 @@ void __init pnv_pci_init_ioda1_phb(struct device_node *np)
>>>> 	 * ourselves here
>>>> 	 */
>>>> 	ppc_md.pcibios_fixup_phb = pnv_pci_ioda_fixup_phb;
>>>>+	ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
>>>> 	ppc_md.pcibios_enable_device_hook = pnv_pci_enable_device_hook;
>>>> 	pci_add_flags(PCI_PROBE_ONLY | PCI_REASSIGN_ALL_RSRC);
>>>>
>>>>diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
>>>>index 8bc4796..0cb760c 100644
>>>>--- a/arch/powerpc/platforms/powernv/pci.h
>>>>+++ b/arch/powerpc/platforms/powernv/pci.h
>>>>@@ -17,9 +17,14 @@ enum pnv_phb_model {
>>>> };
>>>>
>>>> #define PNV_PCI_DIAG_BUF_SIZE	4096
>>>>+#define PNV_IODA_PE_DEV		(1 << 0)	/* PE has single PCI device	*/
>>>>+#define PNV_IODA_PE_BUS		(1 << 1)	/* PE has primary PCI bus	*/
>>>>+#define PNV_IODA_PE_BUS_ALL	(1 << 2)	/* PE has subordinate buses	*/
>>>>
>>>> /* Data associated with a PE, including IOMMU tracking etc.. */
>>>> struct pnv_ioda_pe {
>>>>+	unsigned long		flags;
>>>>+
>>>> 	/* A PE can be associated with a single device or an
>>>> 	 * entire bus (& children). In the former case, pdev
>>>> 	 * is populated, in the later case, pbus is.
>>>>@@ -40,11 +45,6 @@ struct pnv_ioda_pe {
>>>> 	 */
>>>> 	unsigned int		dma_weight;
>>>>
>>>>-	/* This is a PCI-E -> PCI-X bridge, this points to the
>>>>-	 * corresponding bus PE
>>>>-	 */
>>>>-	struct pnv_ioda_pe	*bus_pe;
>>>>-
>>>> 	/* "Base" iommu table, ie, 4K TCEs, 32-bit DMA */
>>>> 	int			tce32_seg;
>>>> 	int			tce32_segcount;
>>>>-- 
>>>>1.7.9.5
>>>>
>>>>_______________________________________________
>>>>Linuxppc-dev mailing list
>>>>Linuxppc-dev@lists.ozlabs.org
>>>>https://lists.ozlabs.org/listinfo/linuxppc-dev
>>>
>>>-- 
>>>Richard Yang
>>>Help you, Help me
>>
>>_______________________________________________
>>Linuxppc-dev mailing list
>>Linuxppc-dev@lists.ozlabs.org
>>https://lists.ozlabs.org/listinfo/linuxppc-dev
>
>-- 
>Richard Yang
>Help you, Help me

[PATCH 2/8] ppc/pnv: PE list based on creation order

[Gavin Shan]

The resource (I/O and MMIO) will be assigned on basis of PE from
top to bottom so that we can implement the trick here: the resource
that has been assigned to parent PE could be taken by child PE if
necessary.

The current implementation already has PE list per PHB basis, but
the list doesn't meet our requirment: tracing PE based on their
cration time from top to bottom. So the patch does rename for the
DMA based PE list and introduces the list to trace the PEs sequentially
based on their creation time.

Signed-off-by: Gavin Shan <shangw@linux.vnet.ibm.com>
---
 arch/powerpc/platforms/powernv/pci-ioda.c |   12 ++++++++----
 arch/powerpc/platforms/powernv/pci.h      |   10 ++++++++--
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index 1504795..6a7672e 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -657,13 +657,13 @@ static void __devinit pnv_ioda_link_pe_by_weight(struct pnv_phb *phb,
 {
 	struct pnv_ioda_pe *lpe;
 
-	list_for_each_entry(lpe, &phb->ioda.pe_list, link) {
+	list_for_each_entry(lpe, &phb->ioda.pe_dma_list, dma_link) {
 		if (lpe->dma_weight < pe->dma_weight) {
-			list_add_tail(&pe->link, &lpe->link);
+			list_add_tail(&pe->dma_link, &lpe->dma_link);
 			return;
 		}
 	}
-	list_add_tail(&pe->link, &phb->ioda.pe_list);
+	list_add_tail(&pe->dma_link, &phb->ioda.pe_dma_list);
 }
 
 static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev)
@@ -830,6 +830,9 @@ static void __devinit pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all)
 	/* Associate it with all child devices */
 	pnv_ioda_setup_same_PE(bus, pe);
 
+	/* Put PE to the list */
+	list_add_tail(&pe->list, &phb->ioda.pe_list);
+
 	/* Account for one DMA PE if at least one DMA capable device exist
 	 * below the bridge
 	 */
@@ -1013,7 +1016,7 @@ static void __devinit pnv_ioda_setup_dma(struct pnv_phb *phb)
 	remaining = phb->ioda.tce32_count;
 	tw = phb->ioda.dma_weight;
 	base = 0;
-	list_for_each_entry(pe, &phb->ioda.pe_list, link) {
+	list_for_each_entry(pe, &phb->ioda.pe_dma_list, dma_link) {
 		if (!pe->dma_weight)
 			continue;
 		if (!remaining) {
@@ -1269,6 +1272,7 @@ void __init pnv_pci_init_ioda1_phb(struct device_node *np)
 	phb->ioda.pe_array = aux + pemap_off;
 	set_bit(0, phb->ioda.pe_alloc);
 
+	INIT_LIST_HEAD(&phb->ioda.pe_dma_list);
 	INIT_LIST_HEAD(&phb->ioda.pe_list);
 
 	/* Calculate how many 32-bit TCE segments we have */
diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
index 0cb760c..b70720b 100644
--- a/arch/powerpc/platforms/powernv/pci.h
+++ b/arch/powerpc/platforms/powernv/pci.h
@@ -59,7 +59,8 @@ struct pnv_ioda_pe {
 	int			mve_number;
 
 	/* Link in list of PE#s */
-	struct list_head	link;
+	struct list_head	dma_link;
+	struct list_head	list;
 };
 
 struct pnv_phb {
@@ -107,6 +108,11 @@ struct pnv_phb {
 			unsigned int		*io_segmap;
 			struct pnv_ioda_pe	*pe_array;
 
+			/* Sorted list of used PE's based
+			 * on the sequence of creation
+			 */
+			struct list_head	pe_list;
+
 			/* Reverse map of PEs, will have to extend if
 			 * we are to support more than 256 PEs, indexed
 			 * bus { bus, devfn }
@@ -125,7 +131,7 @@ struct pnv_phb {
 			/* Sorted list of used PE's, sorted at
 			 * boot for resource allocation purposes
 			 */
-			struct list_head	pe_list;
+			struct list_head	pe_dma_list;
 		} ioda;
 	};
 
-- 
1.7.9.5

[PATCH 3/8] ppc/pnv: I/O and MMIO resource assignment for PEs

[Gavin Shan]

There're 2 types of PCI bus sensitive PEs: (A) The PE includes
single PCI bus. (B) The PE includes the PCI bus and all the subordinate
PCI buses, and the patch tries to assign I/O and MMIO resources
based on created PEs. Fortunately, we figured out unified scheme
to do resource assignment for all types of PCI bus based PEs according
to Ben's idea:

	- Resource assignment based on PE from top to bottom.
	- The soureces, either I/O or MMIO, of the PE are figured out
	  from the assigned PCI bus.
	- The occupied resource by parent PE could possibilly be overrided
	  by children PEs.

Signed-off-by: Gavin Shan <shangw@linux.vnet.ibm.com>
Reviewed-by: Ram Pai <linuxram@us.ibm.com>
Reviewed-by: Richard Yang <weiyang@linux.vnet.ibm.com>
---
 arch/powerpc/platforms/powernv/pci-ioda.c |   89 +++++++++++++++++++++++++++++
 1 file changed, 89 insertions(+)

diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index 6a7672e..1071820 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -1155,9 +1155,98 @@ static void __devinit pnv_pci_ioda_fixup_phb(struct pci_controller *hose)
 	}
 }
 
+/*
+ * This function is supposed to be called on basis of PE from top
+ * to bottom style. So the the I/O or MMIO segment assigned to
+ * parent PE could be overrided by its child PEs if necessary.
+ */
+static void __devinit pnv_ioda_setup_pe_seg(struct pci_controller *hose,
+                                struct pnv_ioda_pe *pe)
+{
+	struct pnv_phb *phb = hose->private_data;
+	struct pci_bus_region region;
+	struct resource *res;
+	int i, index;
+	int rc;
+
+	/*
+	 * NOTE: We only care PCI bus based PE for now. For PCI
+	 * device based PE, for example SRIOV sensitive VF should
+	 * be figured out later.
+	 */
+	BUG_ON(!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)));
+
+	pci_bus_for_each_resource(pe->pbus, res, i) {
+		if (!res || !res->flags ||
+		    res->start > res->end)
+			continue;
+
+		if (res->flags & IORESOURCE_IO) {
+			region.start = res->start - phb->ioda.io_pci_base;
+			region.end   = res->end - phb->ioda.io_pci_base;
+			index = region.start / phb->ioda.io_segsize;
+
+			while (index < phb->ioda.total_pe &&
+			       region.start <= region.end) {
+				phb->ioda.io_segmap[index] = pe->pe_number;
+				rc = opal_pci_map_pe_mmio_window(phb->opal_id,
+					pe->pe_number, OPAL_IO_WINDOW_TYPE, 0, index);
+				if (rc != OPAL_SUCCESS) {
+					pr_err("%s: OPAL error %d when mapping IO "
+					       "segment #%d to PE#%d\n",
+						__func__, rc, index, pe->pe_number);
+					break;
+				}
+
+				region.start += phb->ioda.io_segsize;
+				index++;
+			}
+		} else if (res->flags & IORESOURCE_MEM) {
+			region.start = res->start -
+				       hose->pci_mem_offset -
+				       phb->ioda.m32_pci_base;
+			region.end   = res->end -
+				       hose->pci_mem_offset -
+				       phb->ioda.m32_pci_base;
+			index = region.start / phb->ioda.m32_segsize;
+
+			while (index < phb->ioda.total_pe &&
+			       region.start <= region.end) {
+				phb->ioda.m32_segmap[index] = pe->pe_number;
+				rc = opal_pci_map_pe_mmio_window(phb->opal_id,
+					pe->pe_number, OPAL_M32_WINDOW_TYPE, 0, index);
+				if (rc != OPAL_SUCCESS) {
+					pr_err("%s: OPAL error %d when mapping M32 "
+					       "segment#%d to PE#%d",
+					       __func__, rc, index, pe->pe_number);
+					break;
+				}
+
+				region.start += phb->ioda.m32_segsize;
+				index++;
+			}
+		}
+	}
+}
+
+static void __devinit pnv_pci_ioda_setup_seg(void)
+{
+	struct pci_controller *tmp, *hose;
+	struct pnv_phb *phb;
+	struct pnv_ioda_pe *pe;
+
+	list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
+		phb = hose->private_data;
+		list_for_each_entry(pe, &phb->ioda.pe_list, list) {
+			pnv_ioda_setup_pe_seg(hose, pe);
+		}
+	}
+}
+
 static void __devinit pnv_pci_ioda_fixup(void)
 {
 	pnv_pci_ioda_setup_PEs();
+	pnv_pci_ioda_setup_seg();
 }
 
 /* Prevent enabling devices for which we couldn't properly
-- 
1.7.9.5

[PATCH 4/8] ppc/pnv: initialize DMA for PEs

[Gavin Shan]

The patch introduces additional wrapper function to call the original
implementation so that the DMA can be configured for all existing PEs.

Signed-off-by: Gavin Shan <shangw@linux.vnet.ibm.com>
Reviewed-by: Ram Pai <linuxram@us.ibm.com>
Reviewed-by: Richard Yang <weiyang@linux.vnet.ibm.com>
---
 arch/powerpc/platforms/powernv/pci-ioda.c |   10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index 1071820..f0ae647 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -1243,10 +1243,20 @@ static void __devinit pnv_pci_ioda_setup_seg(void)
 	}
 }
 
+static void __devinit pnv_pci_ioda_setup_DMA(void)
+{
+	struct pci_controller *hose, *tmp;
+
+	list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
+		pnv_ioda_setup_dma(hose->private_data);
+	}
+}
+
 static void __devinit pnv_pci_ioda_fixup(void)
 {
 	pnv_pci_ioda_setup_PEs();
 	pnv_pci_ioda_setup_seg();
+	pnv_pci_ioda_setup_DMA();
 }
 
 /* Prevent enabling devices for which we couldn't properly
-- 
1.7.9.5

[PATCH 5/8] ppc/pnv: skip check on PE if necessary

[Gavin Shan]

While the device driver or PCI core tries to enable PCI device, the
platform dependent callback "ppc_md.pcibios_enable_device_hook" will
be called to check if there has one associated PE for the PCI device.
If we don't have the associated PE for the PCI device, it's not allowed
to enable the PCI device. Unfortunately, there might have some cases
we have to enable the PCI device (e.g. P2P bridge), but the PEs have
not been created yet.

The patch handles the unfortunate cases. Each PHB (struct pnv_phb)
has one field "initialized" to trace if the PEs have been created
and configured or not. When the PEs are not available, we won't check
the associated PE for the PCI device to be enabled.

Signed-off-by: Gavin Shan <shangw@linux.vnet.ibm.com>
Reviewed-by: Ram Pai <linuxram@us.ibm.com>
Reviewed-by: Richard Yang <weiyang@linux.vnet.ibm.com>
---
 arch/powerpc/platforms/powernv/pci-ioda.c |   19 ++++++++++++++++++-
 arch/powerpc/platforms/powernv/pci.h      |    1 +
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index f0ae647..052a50f 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -1246,9 +1246,14 @@ static void __devinit pnv_pci_ioda_setup_seg(void)
 static void __devinit pnv_pci_ioda_setup_DMA(void)
 {
 	struct pci_controller *hose, *tmp;
+	struct pnv_phb *phb;
 
 	list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
 		pnv_ioda_setup_dma(hose->private_data);
+
+		/* Mark the PHB initialization done */
+		phb = hose->private_data;
+		phb->initialized = 1;
 	}
 }
 
@@ -1264,10 +1269,22 @@ static void __devinit pnv_pci_ioda_fixup(void)
  */
 static int __devinit pnv_pci_enable_device_hook(struct pci_dev *dev)
 {
-	struct pci_dn *pdn = pnv_ioda_get_pdn(dev);
+	struct pci_controller *hose = pci_bus_to_host(dev->bus);
+	struct pnv_phb *phb = hose->private_data;
+	struct pci_dn *pdn;
 
+	/* The function is probably called while the PEs have
+	 * not be created yet. For example, resource reassignment
+	 * during PCI probe period. We just skip the check if
+	 * PEs isn't ready.
+	 */
+	if (!phb->initialized)
+		return 0;
+
+	pdn = pnv_ioda_get_pdn(dev);
 	if (!pdn || pdn->pe_number == IODA_INVALID_PE)
 		return -EINVAL;
+
 	return 0;
 }
 
diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
index b70720b..7cfb7c8 100644
--- a/arch/powerpc/platforms/powernv/pci.h
+++ b/arch/powerpc/platforms/powernv/pci.h
@@ -69,6 +69,7 @@ struct pnv_phb {
 	enum pnv_phb_model	model;
 	u64			opal_id;
 	void __iomem		*regs;
+	int			initialized;
 	spinlock_t		lock;
 
 #ifdef CONFIG_PCI_MSI
-- 
1.7.9.5

[PATCH 6/8] ppc/pnv: fix overrunning segment tracing array

[Gavin Shan]

There're 2 arrays introduced to trace which PE has occupied the
corresponding resource (I/O or MMIO) segment. However, we didn't
allocate enough memory for them and that possiblly leads to PE
descriptor corruption.

The patch fixes that by allocating enough memory for those 2 arrays.

Signed-off-by: Gavin Shan <shangw@linux.vnet.ibm.com>
---
 arch/powerpc/platforms/powernv/pci-ioda.c |    4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index 052a50f..ed8cf09 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -1375,9 +1375,9 @@ void __init pnv_pci_init_ioda1_phb(struct device_node *np)
 	/* Allocate aux data & arrays */
 	size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
 	m32map_off = size;
-	size += phb->ioda.total_pe;
+	size += phb->ioda.total_pe * sizeof(phb->ioda.m32_segmap[0]);
 	iomap_off = size;
-	size += phb->ioda.total_pe;
+	size += phb->ioda.total_pe * sizeof(phb->ioda.io_segmap[0]);
 	pemap_off = size;
 	size += phb->ioda.total_pe * sizeof(struct pnv_ioda_pe);
 	aux = alloc_bootmem(size);
-- 
1.7.9.5

[PATCH 7/8] ppc/pnv: using PCI core to do resource assignment

[Gavin Shan]

Currently, the PCI probe flags "PCI_PROBE_ONLY | PCI_REASSIGN_ALL_RSRC"
used on powernv platform. That means the platform has to do the PCI
resource assignment by itself.

The patch changes the PCI probe flag to "PCI_REASSIGN_ALL_RSRC" so
that the PCI core will do the resource assignment. Also, the I/O
and MMIO minimal alignment for P2P bridges have been configured
while doing fixup for the PHBs.

Signed-off-by: Gavin Shan <shangw@linux.vnet.ibm.com>
Reviewed-by: Ram Pai <linuxram@us.ibm.com>
Reviewed-by: Richard Yang <weiyang@linux.vnet.ibm.com>
---
 arch/powerpc/platforms/powernv/pci-ioda.c |   46 ++++++++++++-----------------
 1 file changed, 19 insertions(+), 27 deletions(-)

diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index ed8cf09..1368804 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -1130,29 +1130,19 @@ static void pnv_pci_init_ioda_msis(struct pnv_phb *phb) { }
  */
 static void __devinit pnv_pci_ioda_fixup_phb(struct pci_controller *hose)
 {
-	resource_size_t size, align;
-	struct pci_bus *child;
-
-	/* Associate PEs per functions */
-	pnv_ioda_setup_PEs(hose->bus);
-
-	/* Calculate all resources */
-	pnv_ioda_calc_bus(hose->bus, IORESOURCE_IO, &size, &align);
-	pnv_ioda_calc_bus(hose->bus, IORESOURCE_MEM, &size, &align);
-
-	/* Apply then to HW */
-	pnv_ioda_update_resources(hose->bus);
-
-	/* Setup DMA */
-	pnv_ioda_setup_dma(hose->private_data);
+	struct pnv_phb *phb = hose->private_data;
+	struct pci_host_bridge *host_bridge;
 
-	/* Configure PCI Express settings */
-	list_for_each_entry(child, &hose->bus->children, node) {
-		struct pci_dev *self = child->self;
-		if (!self)
-			continue;
-		pcie_bus_configure_settings(child, self->pcie_mpss);
-	}
+	host_bridge = pci_bus_host_bridge(hose->bus);
+	host_bridge->io_align_shift = __ffs(phb->ioda.io_segsize);
+	host_bridge->mem_align_shift = __ffs(phb->ioda.m32_segsize);
+	/*
+	 * Note: We do not use the M64 window yet (64-bit MMIO window),
+	 * so we leave the alignment setting alone. In the long run,
+	 * we might use it for prefetchable memory regions in which
+	 * case pmem_align_shift will have to be adjusted as well.
+	 */
+	host_bridge->pmem_align_shift = host_bridge->mem_align_shift;
 }
 
 /*
@@ -1437,15 +1427,17 @@ void __init pnv_pci_init_ioda1_phb(struct device_node *np)
 	/* Setup MSI support */
 	pnv_pci_init_ioda_msis(phb);
 
-	/* We set both PCI_PROBE_ONLY and PCI_REASSIGN_ALL_RSRC. This is an
-	 * odd combination which essentially means that we skip all resource
-	 * fixups and assignments in the generic code, and do it all
-	 * ourselves here
+	/*
+	 * We pass the PCI probe flag PCI_REASSIGN_ALL_RSRC here
+	 * to let the PCI core do resource assignment. It's supposed
+	 * that the PCI core will do correct I/O and MMIO alignment
+	 * for the P2P bridge bars so that each PCI bus (excluding
+	 * the child P2P bridges) can form individual PE.
 	 */
 	ppc_md.pcibios_fixup_phb = pnv_pci_ioda_fixup_phb;
 	ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
 	ppc_md.pcibios_enable_device_hook = pnv_pci_enable_device_hook;
-	pci_add_flags(PCI_PROBE_ONLY | PCI_REASSIGN_ALL_RSRC);
+	pci_add_flags(PCI_REASSIGN_ALL_RSRC);
 
 	/* Reset IODA tables to a clean state */
 	rc = opal_pci_reset(phb_id, OPAL_PCI_IODA_TABLE_RESET, OPAL_ASSERT_RESET);
-- 
1.7.9.5

[PATCH 8/8] ppc/pnv: remove unused functions

[Gavin Shan]

We don't need them anymore. The patch removes those functions.

Signed-off-by: Gavin Shan <shangw@linux.vnet.ibm.com>
Reviewed-by: Ram Pai <linuxram@us.ibm.com>
Reviewed-by: Richard Yang <weiyang@linux.vnet.ibm.com>
---
 arch/powerpc/platforms/powernv/pci-ioda.c |  441 -----------------------------
 1 file changed, 441 deletions(-)

diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index 1368804..a40071f 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -35,14 +35,6 @@
 #include "powernv.h"
 #include "pci.h"
 
-struct resource_wrap {
-	struct list_head	link;
-	resource_size_t		size;
-	resource_size_t		align;
-	struct pci_dev		*dev;	/* Set if it's a device */
-	struct pci_bus		*bus;	/* Set if it's a bridge */
-};
-
 static int __pe_printk(const char *level, const struct pnv_ioda_pe *pe,
 		       struct va_format *vaf)
 {
@@ -78,273 +70,6 @@ define_pe_printk_level(pe_err, KERN_ERR);
 define_pe_printk_level(pe_warn, KERN_WARNING);
 define_pe_printk_level(pe_info, KERN_INFO);
 
-
-/* Calculate resource usage & alignment requirement of a single
- * device. This will also assign all resources within the device
- * for a given type starting at 0 for the biggest one and then
- * assigning in decreasing order of size.
- */
-static void __devinit pnv_ioda_calc_dev(struct pci_dev *dev, unsigned int flags,
-					resource_size_t *size,
-					resource_size_t *align)
-{
-	resource_size_t start;
-	struct resource *r;
-	int i;
-
-	pr_devel("  -> CDR %s\n", pci_name(dev));
-
-	*size = *align = 0;
-
-	/* Clear the resources out and mark them all unset */
-	for (i = 0; i <= PCI_ROM_RESOURCE; i++) {
-		r = &dev->resource[i];
-		if (!(r->flags & flags))
-		    continue;
-		if (r->start) {
-			r->end -= r->start;
-			r->start = 0;
-		}
-		r->flags |= IORESOURCE_UNSET;
-	}
-
-	/* We currently keep all memory resources together, we
-	 * will handle prefetch & 64-bit separately in the future
-	 * but for now we stick everybody in M32
-	 */
-	start = 0;
-	for (;;) {
-		resource_size_t max_size = 0;
-		int max_no = -1;
-
-		/* Find next biggest resource */
-		for (i = 0; i <= PCI_ROM_RESOURCE; i++) {
-			r = &dev->resource[i];
-			if (!(r->flags & IORESOURCE_UNSET) ||
-			    !(r->flags & flags))
-				continue;
-			if (resource_size(r) > max_size) {
-				max_size = resource_size(r);
-				max_no = i;
-			}
-		}
-		if (max_no < 0)
-			break;
-		r = &dev->resource[max_no];
-		if (max_size > *align)
-			*align = max_size;
-		*size += max_size;
-		r->start = start;
-		start += max_size;
-		r->end = r->start + max_size - 1;
-		r->flags &= ~IORESOURCE_UNSET;
-		pr_devel("  ->     R%d %016llx..%016llx\n",
-			 max_no, r->start, r->end);
-	}
-	pr_devel("  <- CDR %s size=%llx align=%llx\n",
-		 pci_name(dev), *size, *align);
-}
-
-/* Allocate a resource "wrap" for a given device or bridge and
- * insert it at the right position in the sorted list
- */
-static void __devinit pnv_ioda_add_wrap(struct list_head *list,
-					struct pci_bus *bus,
-					struct pci_dev *dev,
-					resource_size_t size,
-					resource_size_t align)
-{
-	struct resource_wrap *w1, *w = kzalloc(sizeof(*w), GFP_KERNEL);
-
-	w->size = size;
-	w->align = align;
-	w->dev = dev;
-	w->bus = bus;
-
-	list_for_each_entry(w1, list, link) {
-		if (w1->align < align) {
-			list_add_tail(&w->link, &w1->link);
-			return;
-		}
-	}
-	list_add_tail(&w->link, list);
-}
-
-/* Offset device resources of a given type */
-static void __devinit pnv_ioda_offset_dev(struct pci_dev *dev,
-					  unsigned int flags,
-					  resource_size_t offset)
-{
-	struct resource *r;
-	int i;
-
-	pr_devel("  -> ODR %s [%x] +%016llx\n", pci_name(dev), flags, offset);
-
-	for (i = 0; i <= PCI_ROM_RESOURCE; i++) {
-		r = &dev->resource[i];
-		if (r->flags & flags) {
-			dev->resource[i].start += offset;
-			dev->resource[i].end += offset;
-		}
-	}
-
-	pr_devel("  <- ODR %s [%x] +%016llx\n", pci_name(dev), flags, offset);
-}
-
-/* Offset bus resources (& all children) of a given type */
-static void __devinit pnv_ioda_offset_bus(struct pci_bus *bus,
-					  unsigned int flags,
-					  resource_size_t offset)
-{
-	struct resource *r;
-	struct pci_dev *dev;
-	struct pci_bus *cbus;
-	int i;
-
-	pr_devel("  -> OBR %s [%x] +%016llx\n",
-		 bus->self ? pci_name(bus->self) : "root", flags, offset);
-
-	pci_bus_for_each_resource(bus, r, i) {
-		if (r && (r->flags & flags)) {
-			r->start += offset;
-			r->end += offset;
-		}
-	}
-	list_for_each_entry(dev, &bus->devices, bus_list)
-		pnv_ioda_offset_dev(dev, flags, offset);
-	list_for_each_entry(cbus, &bus->children, node)
-		pnv_ioda_offset_bus(cbus, flags, offset);
-
-	pr_devel("  <- OBR %s [%x]\n",
-		 bus->self ? pci_name(bus->self) : "root", flags);
-}
-
-/* This is the guts of our IODA resource allocation. This is called
- * recursively for each bus in the system. It calculates all the
- * necessary size and requirements for children and assign them
- * resources such that:
- *
- *   - Each function fits in it's own contiguous set of IO/M32
- *     segment
- *
- *   - All segments behind a P2P bridge are contiguous and obey
- *     alignment constraints of those bridges
- */
-static void __devinit pnv_ioda_calc_bus(struct pci_bus *bus, unsigned int flags,
-					resource_size_t *size,
-					resource_size_t *align)
-{
-	struct pci_controller *hose = pci_bus_to_host(bus);
-	struct pnv_phb *phb = hose->private_data;
-	resource_size_t dev_size, dev_align, start;
-	resource_size_t min_align, min_balign;
-	struct pci_dev *cdev;
-	struct pci_bus *cbus;
-	struct list_head head;
-	struct resource_wrap *w;
-	unsigned int bres;
-
-	*size = *align = 0;
-
-	pr_devel("-> CBR %s [%x]\n",
-		 bus->self ? pci_name(bus->self) : "root", flags);
-
-	/* Calculate alignment requirements based on the type
-	 * of resource we are working on
-	 */
-	if (flags & IORESOURCE_IO) {
-		bres = 0;
-		min_align = phb->ioda.io_segsize;
-		min_balign = 0x1000;
-	} else {
-		bres = 1;
-		min_align = phb->ioda.m32_segsize;
-		min_balign = 0x100000;
-	}
-
-	/* Gather all our children resources ordered by alignment */
-	INIT_LIST_HEAD(&head);
-
-	/*   - Busses */
-	list_for_each_entry(cbus, &bus->children, node) {
-		pnv_ioda_calc_bus(cbus, flags, &dev_size, &dev_align);
-		pnv_ioda_add_wrap(&head, cbus, NULL, dev_size, dev_align);
-	}
-
-	/*   - Devices */
-	list_for_each_entry(cdev, &bus->devices, bus_list) {
-		pnv_ioda_calc_dev(cdev, flags, &dev_size, &dev_align);
-		/* Align them to segment size */
-		if (dev_align < min_align)
-			dev_align = min_align;
-		pnv_ioda_add_wrap(&head, NULL, cdev, dev_size, dev_align);
-	}
-	if (list_empty(&head))
-		goto empty;
-
-	/* Now we can do two things: assign offsets to them within that
-	 * level and get our total alignment & size requirements. The
-	 * assignment algorithm is going to be uber-trivial for now, we
-	 * can try to be smarter later at filling out holes.
-	 */
-	if (bus->self) {
-		/* No offset for downstream bridges */
-		start = 0;
-	} else {
-		/* Offset from the root */
-		if (flags & IORESOURCE_IO)
-			/* Don't hand out IO 0 */
-			start = hose->io_resource.start + 0x1000;
-		else
-			start = hose->mem_resources[0].start;
-	}
-	while(!list_empty(&head)) {
-		w = list_first_entry(&head, struct resource_wrap, link);
-		list_del(&w->link);
-		if (w->size) {
-			if (start) {
-				start = ALIGN(start, w->align);
-				if (w->dev)
-					pnv_ioda_offset_dev(w->dev,flags,start);
-				else if (w->bus)
-					pnv_ioda_offset_bus(w->bus,flags,start);
-			}
-			if (w->align > *align)
-				*align = w->align;
-		}
-		start += w->size;
-		kfree(w);
-	}
-	*size = start;
-
-	/* Align and setup bridge resources */
-	*align = max_t(resource_size_t, *align,
-		       max_t(resource_size_t, min_align, min_balign));
-	*size = ALIGN(*size,
-		      max_t(resource_size_t, min_align, min_balign));
- empty:
-	/* Only setup P2P's, not the PHB itself */
-	if (bus->self) {
-		struct resource *res = bus->resource[bres];
-
-		if (WARN_ON(res == NULL))
-			return;
-
-		/*
-		 * FIXME: We should probably export and call
-		 * pci_bridge_check_ranges() to properly re-initialize
-		 * the PCI portion of the flags here, and to detect
-		 * what the bridge actually supports.
-		 */
-		res->start = 0;
-		res->flags = (*size) ? flags : 0;
-		res->end = (*size) ? (*size - 1) : 0;
-	}
-
-	pr_devel("<- CBR %s [%x] *size=%016llx *align=%016llx\n",
-		 bus->self ? pci_name(bus->self) : "root", flags,*size,*align);
-}
-
 static struct pci_dn *pnv_ioda_get_pdn(struct pci_dev *dev)
 {
 	struct device_node *np;
@@ -355,172 +80,6 @@ static struct pci_dn *pnv_ioda_get_pdn(struct pci_dev *dev)
 	return PCI_DN(np);
 }
 
-static void __devinit pnv_ioda_setup_pe_segments(struct pci_dev *dev)
-{
-	struct pci_controller *hose = pci_bus_to_host(dev->bus);
-	struct pnv_phb *phb = hose->private_data;
-	struct pci_dn *pdn = pnv_ioda_get_pdn(dev);
-	unsigned int pe, i;
-	resource_size_t pos;
-	struct resource io_res;
-	struct resource m32_res;
-	struct pci_bus_region region;
-	int rc;
-
-	/* Anything not referenced in the device-tree gets PE#0 */
-	pe = pdn ? pdn->pe_number : 0;
-
-	/* Calculate the device min/max */
-	io_res.start = m32_res.start = (resource_size_t)-1;
-	io_res.end = m32_res.end = 0;
-	io_res.flags = IORESOURCE_IO;
-	m32_res.flags = IORESOURCE_MEM;
-
-	for (i = 0; i <= PCI_ROM_RESOURCE; i++) {
-		struct resource *r = NULL;
-		if (dev->resource[i].flags & IORESOURCE_IO)
-			r = &io_res;
-		if (dev->resource[i].flags & IORESOURCE_MEM)
-			r = &m32_res;
-		if (!r)
-			continue;
-		if (dev->resource[i].start < r->start)
-			r->start = dev->resource[i].start;
-		if (dev->resource[i].end > r->end)
-			r->end = dev->resource[i].end;
-	}
-
-	/* Setup IO segments */
-	if (io_res.start < io_res.end) {
-		pcibios_resource_to_bus(dev, &region, &io_res);
-		pos = region.start;
-		i = pos / phb->ioda.io_segsize;
-		while(i < phb->ioda.total_pe && pos <= region.end) {
-			if (phb->ioda.io_segmap[i]) {
-				pr_err("%s: Trying to use IO seg #%d which is"
-				       " already used by PE# %d\n",
-				       pci_name(dev), i,
-				       phb->ioda.io_segmap[i]);
-				/* XXX DO SOMETHING TO DISABLE DEVICE ? */
-				break;
-			}
-			phb->ioda.io_segmap[i] = pe;
-			rc = opal_pci_map_pe_mmio_window(phb->opal_id, pe,
-							 OPAL_IO_WINDOW_TYPE,
-							 0, i);
-			if (rc != OPAL_SUCCESS) {
-				pr_err("%s: OPAL error %d setting up mapping"
-				       " for IO seg# %d\n",
-				       pci_name(dev), rc, i);
-				/* XXX DO SOMETHING TO DISABLE DEVICE ? */
-				break;
-			}
-			pos += phb->ioda.io_segsize;
-			i++;
-		};
-	}
-
-	/* Setup M32 segments */
-	if (m32_res.start < m32_res.end) {
-		pcibios_resource_to_bus(dev, &region, &m32_res);
-		pos = region.start;
-		i = pos / phb->ioda.m32_segsize;
-		while(i < phb->ioda.total_pe && pos <= region.end) {
-			if (phb->ioda.m32_segmap[i]) {
-				pr_err("%s: Trying to use M32 seg #%d which is"
-				       " already used by PE# %d\n",
-				       pci_name(dev), i,
-				       phb->ioda.m32_segmap[i]);
-				/* XXX DO SOMETHING TO DISABLE DEVICE ? */
-				break;
-			}
-			phb->ioda.m32_segmap[i] = pe;
-			rc = opal_pci_map_pe_mmio_window(phb->opal_id, pe,
-							 OPAL_M32_WINDOW_TYPE,
-							 0, i);
-			if (rc != OPAL_SUCCESS) {
-				pr_err("%s: OPAL error %d setting up mapping"
-				       " for M32 seg# %d\n",
-				       pci_name(dev), rc, i);
-				/* XXX DO SOMETHING TO DISABLE DEVICE ? */
-				break;
-			}
-			pos += phb->ioda.m32_segsize;
-			i++;
-		}
-	}
-}
-
-/* Check if a resource still fits in the total IO or M32 range
- * for a given PHB
- */
-static int __devinit pnv_ioda_resource_fit(struct pci_controller *hose,
-					   struct resource *r)
-{
-	struct resource *bounds;
-
-	if (r->flags & IORESOURCE_IO)
-		bounds = &hose->io_resource;
-	else if (r->flags & IORESOURCE_MEM)
-		bounds = &hose->mem_resources[0];
-	else
-		return 1;
-
-	if (r->start >= bounds->start && r->end <= bounds->end)
-		return 1;
-	r->flags = 0;
-	return 0;
-}
-
-static void __devinit pnv_ioda_update_resources(struct pci_bus *bus)
-{
-	struct pci_controller *hose = pci_bus_to_host(bus);
-	struct pci_bus *cbus;
-	struct pci_dev *cdev;
-	unsigned int i;
-
-	/* We used to clear all device enables here. However it looks like
-	 * clearing MEM enable causes Obsidian (IPR SCS) to go bonkers,
-	 * and shoot fatal errors to the PHB which in turns fences itself
-	 * and we can't recover from that ... yet. So for now, let's leave
-	 * the enables as-is and hope for the best.
-	 */
-
-	/* Check if bus resources fit in our IO or M32 range */
-	for (i = 0; bus->self && (i < 2); i++) {
-		struct resource *r = bus->resource[i];
-		if (r && !pnv_ioda_resource_fit(hose, r))
-			pr_err("%s: Bus %d resource %d disabled, no room\n",
-			       pci_name(bus->self), bus->number, i);
-	}
-
-	/* Update self if it's not a PHB */
-	if (bus->self)
-		pci_setup_bridge(bus);
-
-	/* Update child devices */
-	list_for_each_entry(cdev, &bus->devices, bus_list) {
-		/* Check if resource fits, if not, disabled it */
-		for (i = 0; i <= PCI_ROM_RESOURCE; i++) {
-			struct resource *r = &cdev->resource[i];
-			if (!pnv_ioda_resource_fit(hose, r))
-				pr_err("%s: Resource %d disabled, no room\n",
-				       pci_name(cdev), i);
-		}
-
-		/* Assign segments */
-		pnv_ioda_setup_pe_segments(cdev);
-
-		/* Update HW BARs */
-		for (i = 0; i <= PCI_ROM_RESOURCE; i++)
-			pci_update_resource(cdev, i);
-	}
-
-	/* Update child busses */
-	list_for_each_entry(cbus, &bus->children, node)
-		pnv_ioda_update_resources(cbus);
-}
-
 static int __devinit pnv_ioda_alloc_pe(struct pnv_phb *phb)
 {
 	unsigned long pe;
-- 
1.7.9.5

[PATCH V4 0/8] Rework on PowerNV P7IOC initialization

[Gavin Shan]

The rework is done based on Ben's initial ideas on how PE and resource
assignment is done on top of PCI core instead of doing resource assignment
by powernv platform. With the series of patches, the following aspects will
be covered:

	- Only create PE based on PCI bus. Basically, there will be 2 types
	  of PEs built on PCI bus: (A) PE only includes single PCI bus. (B)
	  PE includes the PCI bus as well as its subordinate PCI buses.
	- Make the I/O and MMIO resources of P2P bridge aligned according to
	  the PHB's resource layout. For example, MMIO resource of P2P bridges
	  will be aligned with 16MB if PHB has 16MB M32 segment.
	- The resource allocation will be done on basis of PE.
	- Trivial fixup on the hook of enabling PCI device, PE check etc.

The series of patches has been verified on Firebird-L based on kernel 3.6.RC2. During
the kernel boots, USB HUBs can be detected and initialized correctly. Also, the disk
drive as well as network interfaces can work successfully after booting into shell
environment.

V1 -> V2
	- Add separate field into "struct pci_host_bridge" to trace the minimal
	  alignment of prefetchable MMIO bars for P2P bridges (Benjamin Herrenschmidt)
	- Reduced variable names while measuring the MMIO bars (Benjamin Herrenschmidt)
	- Change the commit log for more clear description on [PATCH 2/8] (Benjamin Herrenschmidt)
	- Change the commit log for more clear description on the types of PCI bus
	  sensitive PEs on [PATCH 5/8] (Richard Yang)

V2 -> V3
	- Rework the sequence of the patches so that it won't break "git bitsect" (Benjamin Herrenschmidt)
	- Cleanup on variables name so that they have shorter length (Benjamin Herrenschmidt)
	- Rework on PE's resource calculation based on the PCI bus (Benjamin Herrenschmidt)
	- Cleanup on changelog (Benjamin Herrenschmidt)

V3 -> V4
	- Rebase to 3.6.RC2
	- The series of patches has been reworked to comply with the generic part
	  "[PATCH v8 0/8] Minimal alignment for p2p bars"	
	- Remove the duplicate assignment for PE number (Richard Yang)

-----

arch/powerpc/platforms/powernv/pci-ioda.c |  694 ++++++++---------------------
arch/powerpc/platforms/powernv/pci.h      |   21 +-
2 files changed, 195 insertions(+), 520 deletions(-)

Thanks,
Gavin

[PATCH 7/8] ppc/pnv: using PCI core to do resource assignment

[Gavin Shan]

Currently, the PCI probe flags "PCI_PROBE_ONLY | PCI_REASSIGN_ALL_RSRC"
used on powernv platform. That means the platform has to do the PCI
resource assignment by itself.

The patch changes the PCI probe flag to "PCI_REASSIGN_ALL_RSRC" so
that the PCI core will do the resource assignment. Also, the I/O
and MMIO minimal alignment for P2P bridges have been configured
while doing fixup for the PHBs.

Signed-off-by: Gavin Shan <shangw@linux.vnet.ibm.com>
Reviewed-by: Ram Pai <linuxram@us.ibm.com>
Reviewed-by: Richard Yang <weiyang@linux.vnet.ibm.com>
---
 arch/powerpc/platforms/powernv/pci-ioda.c |   43 +++++------------------------
 1 files changed, 7 insertions(+), 36 deletions(-)

diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index c856f90..ead4eff 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -1124,36 +1124,6 @@ static void pnv_pci_init_ioda_msis(struct pnv_phb *phb)
 static void pnv_pci_init_ioda_msis(struct pnv_phb *phb) { }
 #endif /* CONFIG_PCI_MSI */
 
-/* This is the starting point of our IODA specific resource
- * allocation process
- */
-static void __devinit pnv_pci_ioda_fixup_phb(struct pci_controller *hose)
-{
-	resource_size_t size, align;
-	struct pci_bus *child;
-
-	/* Associate PEs per functions */
-	pnv_ioda_setup_PEs(hose->bus);
-
-	/* Calculate all resources */
-	pnv_ioda_calc_bus(hose->bus, IORESOURCE_IO, &size, &align);
-	pnv_ioda_calc_bus(hose->bus, IORESOURCE_MEM, &size, &align);
-
-	/* Apply then to HW */
-	pnv_ioda_update_resources(hose->bus);
-
-	/* Setup DMA */
-	pnv_ioda_setup_dma(hose->private_data);
-
-	/* Configure PCI Express settings */
-	list_for_each_entry(child, &hose->bus->children, node) {
-		struct pci_dev *self = child->self;
-		if (!self)
-			continue;
-		pcie_bus_configure_settings(child, self->pcie_mpss);
-	}
-}
-
 /*
  * This function is supposed to be called on basis of PE from top
  * to bottom style. So the the I/O or MMIO segment assigned to
@@ -1473,16 +1443,17 @@ void __init pnv_pci_init_ioda1_phb(struct device_node *np)
 	/* Setup MSI support */
 	pnv_pci_init_ioda_msis(phb);
 
-	/* We set both PCI_PROBE_ONLY and PCI_REASSIGN_ALL_RSRC. This is an
-	 * odd combination which essentially means that we skip all resource
-	 * fixups and assignments in the generic code, and do it all
-	 * ourselves here
+	/*
+	 * We pass the PCI probe flag PCI_REASSIGN_ALL_RSRC here
+	 * to let the PCI core do resource assignment. It's supposed
+	 * that the PCI core will do correct I/O and MMIO alignment
+	 * for the P2P bridge bars so that each PCI bus (excluding
+	 * the child P2P bridges) can form individual PE.
 	 */
-	ppc_md.pcibios_fixup_phb = pnv_pci_ioda_fixup_phb;
 	ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
 	ppc_md.pcibios_enable_device_hook = pnv_pci_enable_device_hook;
 	ppc_md.pcibios_window_alignment = pnv_pci_window_alignment;
-	pci_add_flags(PCI_PROBE_ONLY | PCI_REASSIGN_ALL_RSRC);
+	pci_add_flags(PCI_REASSIGN_ALL_RSRC);
 
 	/* Reset IODA tables to a clean state */
 	rc = opal_pci_reset(phb_id, OPAL_PCI_IODA_TABLE_RESET, OPAL_ASSERT_RESET);
-- 
1.7.5.4