[PATCH 0/6] Tegra PCIe end point config space map code refactoring

[PATCH 0/6] Tegra PCIe end point config space map code refactoring

[Vidya Sagar]

PCIe host controller in Tegra SoCs has 1GB of aperture available
for mapping end points config space, IO and BARs. In that, currently
256MB is being reserved for mapping end points configuration space
which leaves less memory space available for mapping end points BARs
on some of the platforms.
This patch series attempts to use only 4K space from 1GB aperture to
access end points configuration space.

Currently, this change benefits T20 and future chips in saving (i.e. repurposed
to use for BAR mapping) physical space as well as kernel virtual mapping space,
it saves only kernel virtual address space in T30, T124, T132 and T210.

Testing Done on T210:
 Enumeration is and basic functionality of immediate devices
 Enumeration of devices behind a PCIe switch
 Complete 4K configuration space access

Vidya Sagar (6):
  PCI: tegra: refactor config space mapping code
  ARM: tegra: limit PCIe config space mapping to 4K for T20
  ARM: tegra: limit PCIe config space mapping to 4K for T30
  ARM: tegra: limit PCIe config space mapping to 4K for T124
  ARM64: tegra: limit PCIe config space mapping to 4K for T132
  ARM64: tegra: limit PCIe config space mapping to 4K for T210

 arch/arm/boot/dts/tegra124.dtsi          |  2 +-
 arch/arm/boot/dts/tegra20.dtsi           |  8 +--
 arch/arm/boot/dts/tegra30.dtsi           |  2 +-
 arch/arm64/boot/dts/nvidia/tegra132.dtsi |  2 +-
 arch/arm64/boot/dts/nvidia/tegra210.dtsi |  2 +-
 drivers/pci/host/pci-tegra.c             | 85 +++++++++-----------------------
 6 files changed, 31 insertions(+), 70 deletions(-)

-- 
2.7.4

[PATCH 1/6] PCI: tegra: refactor config space mapping code

[Vidya Sagar]

use only 4K space from available 1GB PCIe aperture to access
end points configuration space by dynamically moving AFI_AXI_BAR
base address and always making sure that the desired location
to be accessed for generating required config space access falls
in the 4K space reserved for this purpose. This would give more
space for mapping end point device's BARs on some of Tegra platforms

Signed-off-by: Vidya Sagar <vidyas@nvidia.com>
---
 drivers/pci/host/pci-tegra.c | 85 ++++++++++++--------------------------------
 1 file changed, 23 insertions(+), 62 deletions(-)

diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
index 9c40da54f88a..ebdcfca10e17 100644
--- a/drivers/pci/host/pci-tegra.c
+++ b/drivers/pci/host/pci-tegra.c
@@ -269,6 +269,8 @@ struct tegra_pcie {
 	struct list_head buses;
 	struct resource *cs;
 
+	void __iomem *cfg_va_base;
+
 	struct resource io;
 	struct resource pio;
 	struct resource mem;
@@ -317,7 +319,6 @@ struct tegra_pcie_port {
 };
 
 struct tegra_pcie_bus {
-	struct vm_struct *area;
 	struct list_head list;
 	unsigned int nr;
 };
@@ -357,34 +358,16 @@ static inline u32 pads_readl(struct tegra_pcie *pcie, unsigned long offset)
  *
  * Mapping the whole extended configuration space would require 256 MiB of
  * virtual address space, only a small part of which will actually be used.
- * To work around this, a 1 MiB of virtual addresses are allocated per bus
- * when the bus is first accessed. When the physical range is mapped, the
- * the bus number bits are hidden so that the extended register number bits
- * appear as bits [19:16]. Therefore the virtual mapping looks like this:
- *
- *    [19:16] extended register number
- *    [15:11] device number
- *    [10: 8] function number
- *    [ 7: 0] register number
- *
- * This is achieved by stitching together 16 chunks of 64 KiB of physical
- * address space via the MMU.
+ * To work around this, a 4K of region is used to generate required
+ * configuration transaction with relevant B:D:F values. This is achieved by
+ * dynamically programming base address and size of AFI_AXI_BAR used for
+ * end point config space mapping to make sure that the address (access to
+ * which generates correct config transaction) falls in this 4K region
  */
-static unsigned long tegra_pcie_conf_offset(unsigned int devfn, int where)
-{
-	return ((where & 0xf00) << 8) | (PCI_SLOT(devfn) << 11) |
-	       (PCI_FUNC(devfn) << 8) | (where & 0xfc);
-}
-
 static struct tegra_pcie_bus *tegra_pcie_bus_alloc(struct tegra_pcie *pcie,
 						   unsigned int busnr)
 {
-	struct device *dev = pcie->dev;
-	pgprot_t prot = pgprot_noncached(PAGE_KERNEL);
-	phys_addr_t cs = pcie->cs->start;
 	struct tegra_pcie_bus *bus;
-	unsigned int i;
-	int err;
 
 	bus = kzalloc(sizeof(*bus), GFP_KERNEL);
 	if (!bus)
@@ -393,33 +376,16 @@ static struct tegra_pcie_bus *tegra_pcie_bus_alloc(struct tegra_pcie *pcie,
 	INIT_LIST_HEAD(&bus->list);
 	bus->nr = busnr;
 
-	/* allocate 1 MiB of virtual addresses */
-	bus->area = get_vm_area(SZ_1M, VM_IOREMAP);
-	if (!bus->area) {
-		err = -ENOMEM;
-		goto free;
-	}
-
-	/* map each of the 16 chunks of 64 KiB each */
-	for (i = 0; i < 16; i++) {
-		unsigned long virt = (unsigned long)bus->area->addr +
-				     i * SZ_64K;
-		phys_addr_t phys = cs + i * SZ_16M + busnr * SZ_64K;
-
-		err = ioremap_page_range(virt, virt + SZ_64K, phys, prot);
-		if (err < 0) {
-			dev_err(dev, "ioremap_page_range() failed: %d\n", err);
-			goto unmap;
+	if (!pcie->cfg_va_base) {
+		pcie->cfg_va_base = ioremap(pcie->cs->start, SZ_4K);
+		if (!pcie->cfg_va_base) {
+			dev_err(pcie->dev, "failed to ioremap config space\n");
+			kfree(bus);
+			bus = (struct tegra_pcie_bus *)-ENOMEM;
 		}
 	}
 
 	return bus;
-
-unmap:
-	vunmap(bus->area->addr);
-free:
-	kfree(bus);
-	return ERR_PTR(err);
 }
 
 static int tegra_pcie_add_bus(struct pci_bus *bus)
@@ -445,12 +411,12 @@ static void tegra_pcie_remove_bus(struct pci_bus *child)
 
 	list_for_each_entry_safe(bus, tmp, &pcie->buses, list) {
 		if (bus->nr == child->number) {
-			vunmap(bus->area->addr);
 			list_del(&bus->list);
 			kfree(bus);
 			break;
 		}
 	}
+	iounmap(pcie->cfg_va_base);
 }
 
 static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
@@ -459,8 +425,8 @@ static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
 {
 	struct pci_host_bridge *host = pci_find_host_bridge(bus);
 	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
-	struct device *dev = pcie->dev;
 	void __iomem *addr = NULL;
+	u32 val = 0;
 
 	if (bus->number == 0) {
 		unsigned int slot = PCI_SLOT(devfn);
@@ -473,19 +439,14 @@ static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
 			}
 		}
 	} else {
-		struct tegra_pcie_bus *b;
-
-		list_for_each_entry(b, &pcie->buses, list)
-			if (b->nr == bus->number)
-				addr = (void __iomem *)b->area->addr;
-
-		if (!addr) {
-			dev_err(dev, "failed to map cfg. space for bus %u\n",
-				bus->number);
-			return NULL;
-		}
-
-		addr += tegra_pcie_conf_offset(devfn, where);
+		addr = pcie->cfg_va_base;
+		val = ((((u32)where & 0xf00) >> 8) << 24) |
+		      (bus->number << 16) | (PCI_SLOT(devfn) << 11) |
+		      (PCI_FUNC(devfn) << 8) | (where & 0xff);
+		addr = (val & (SZ_4K - 1)) + addr;
+		val = val & ~(SZ_4K - 1);
+		afi_writel(pcie, pcie->cs->start - val, AFI_AXI_BAR0_START);
+		afi_writel(pcie, (val + SZ_4K) >> 12, AFI_AXI_BAR0_SZ);
 	}
 
 	return addr;
-- 
2.7.4

Re: [PATCH 1/6] PCI: tegra: refactor config space mapping code

[Bjorn Helgaas]

On Fri, Oct 13, 2017 at 12:20:06AM +0530, Vidya Sagar wrote:
> use only 4K space from available 1GB PCIe aperture to access
> end points configuration space by dynamically moving AFI_AXI_BAR
> base address and always making sure that the desired location
> to be accessed for generating required config space access falls
> in the 4K space reserved for this purpose. This would give more
> space for mapping end point device's BARs on some of Tegra platforms

Do you now have a revlock issue between this driver change and the
corresponding DT changes?  I.e., what happens when you run this driver
change with an older DT that specifies a 256MB config aperture?  What
happens if you run the older driver with a new DT that only specifies
a 4KB config aperture?

> Signed-off-by: Vidya Sagar <vidyas@nvidia.com>
> ---
>  drivers/pci/host/pci-tegra.c | 85 ++++++++++++--------------------------------
>  1 file changed, 23 insertions(+), 62 deletions(-)
> 
> diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
> index 9c40da54f88a..ebdcfca10e17 100644
> --- a/drivers/pci/host/pci-tegra.c
> +++ b/drivers/pci/host/pci-tegra.c
> @@ -269,6 +269,8 @@ struct tegra_pcie {
>  	struct list_head buses;
>  	struct resource *cs;
>  
> +	void __iomem *cfg_va_base;
> +
>  	struct resource io;
>  	struct resource pio;
>  	struct resource mem;
> @@ -317,7 +319,6 @@ struct tegra_pcie_port {
>  };
>  
>  struct tegra_pcie_bus {
> -	struct vm_struct *area;
>  	struct list_head list;
>  	unsigned int nr;
>  };
> @@ -357,34 +358,16 @@ static inline u32 pads_readl(struct tegra_pcie *pcie, unsigned long offset)
>   *
>   * Mapping the whole extended configuration space would require 256 MiB of
>   * virtual address space, only a small part of which will actually be used.
> - * To work around this, a 1 MiB of virtual addresses are allocated per bus
> - * when the bus is first accessed. When the physical range is mapped, the
> - * the bus number bits are hidden so that the extended register number bits
> - * appear as bits [19:16]. Therefore the virtual mapping looks like this:
> - *
> - *    [19:16] extended register number
> - *    [15:11] device number
> - *    [10: 8] function number
> - *    [ 7: 0] register number
> - *
> - * This is achieved by stitching together 16 chunks of 64 KiB of physical
> - * address space via the MMU.
> + * To work around this, a 4K of region is used to generate required
> + * configuration transaction with relevant B:D:F values. This is achieved by
> + * dynamically programming base address and size of AFI_AXI_BAR used for
> + * end point config space mapping to make sure that the address (access to
> + * which generates correct config transaction) falls in this 4K region
>   */
> -static unsigned long tegra_pcie_conf_offset(unsigned int devfn, int where)
> -{
> -	return ((where & 0xf00) << 8) | (PCI_SLOT(devfn) << 11) |
> -	       (PCI_FUNC(devfn) << 8) | (where & 0xfc);
> -}
> -
>  static struct tegra_pcie_bus *tegra_pcie_bus_alloc(struct tegra_pcie *pcie,
>  						   unsigned int busnr)
>  {
> -	struct device *dev = pcie->dev;
> -	pgprot_t prot = pgprot_noncached(PAGE_KERNEL);
> -	phys_addr_t cs = pcie->cs->start;
>  	struct tegra_pcie_bus *bus;
> -	unsigned int i;
> -	int err;
>  
>  	bus = kzalloc(sizeof(*bus), GFP_KERNEL);
>  	if (!bus)
> @@ -393,33 +376,16 @@ static struct tegra_pcie_bus *tegra_pcie_bus_alloc(struct tegra_pcie *pcie,
>  	INIT_LIST_HEAD(&bus->list);
>  	bus->nr = busnr;
>  
> -	/* allocate 1 MiB of virtual addresses */
> -	bus->area = get_vm_area(SZ_1M, VM_IOREMAP);
> -	if (!bus->area) {
> -		err = -ENOMEM;
> -		goto free;
> -	}
> -
> -	/* map each of the 16 chunks of 64 KiB each */
> -	for (i = 0; i < 16; i++) {
> -		unsigned long virt = (unsigned long)bus->area->addr +
> -				     i * SZ_64K;
> -		phys_addr_t phys = cs + i * SZ_16M + busnr * SZ_64K;
> -
> -		err = ioremap_page_range(virt, virt + SZ_64K, phys, prot);
> -		if (err < 0) {
> -			dev_err(dev, "ioremap_page_range() failed: %d\n", err);
> -			goto unmap;
> +	if (!pcie->cfg_va_base) {
> +		pcie->cfg_va_base = ioremap(pcie->cs->start, SZ_4K);
> +		if (!pcie->cfg_va_base) {
> +			dev_err(pcie->dev, "failed to ioremap config space\n");
> +			kfree(bus);
> +			bus = (struct tegra_pcie_bus *)-ENOMEM;

tegra_pcie_bus_alloc() used to do some bus-specific work -- it allocated VM
space and mapped it.  This pcie->cfg_va_base ioremap() is not bus-specific,
so I don't think it fits here.  It looks like something that should be done
in the tegra_pcie_probe() path instead.

If you move that, then I think you might as well inline the kzalloc() stuff
into tegra_pcie_add_bus().

>  		}
>  	}
>  
>  	return bus;
> -
> -unmap:
> -	vunmap(bus->area->addr);
> -free:
> -	kfree(bus);
> -	return ERR_PTR(err);
>  }
>  
>  static int tegra_pcie_add_bus(struct pci_bus *bus)
> @@ -445,12 +411,12 @@ static void tegra_pcie_remove_bus(struct pci_bus *child)
>  
>  	list_for_each_entry_safe(bus, tmp, &pcie->buses, list) {
>  		if (bus->nr == child->number) {
> -			vunmap(bus->area->addr);
>  			list_del(&bus->list);
>  			kfree(bus);
>  			break;
>  		}
>  	}
> +	iounmap(pcie->cfg_va_base);
>  }
>  
>  static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
> @@ -459,8 +425,8 @@ static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
>  {
>  	struct pci_host_bridge *host = pci_find_host_bridge(bus);
>  	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
> -	struct device *dev = pcie->dev;
>  	void __iomem *addr = NULL;
> +	u32 val = 0;
>  
>  	if (bus->number == 0) {
>  		unsigned int slot = PCI_SLOT(devfn);
> @@ -473,19 +439,14 @@ static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
>  			}
>  		}
>  	} else {
> -		struct tegra_pcie_bus *b;
> -
> -		list_for_each_entry(b, &pcie->buses, list)
> -			if (b->nr == bus->number)
> -				addr = (void __iomem *)b->area->addr;
> -
> -		if (!addr) {
> -			dev_err(dev, "failed to map cfg. space for bus %u\n",
> -				bus->number);
> -			return NULL;
> -		}
> -
> -		addr += tegra_pcie_conf_offset(devfn, where);
> +		addr = pcie->cfg_va_base;
> +		val = ((((u32)where & 0xf00) >> 8) << 24) |
> +		      (bus->number << 16) | (PCI_SLOT(devfn) << 11) |
> +		      (PCI_FUNC(devfn) << 8) | (where & 0xff);

I think it would be useful to keep tegra_pcie_conf_offset() (extending
it to take the bus number) near the comment describing the mapping.

> +		addr = (val & (SZ_4K - 1)) + addr;
> +		val = val & ~(SZ_4K - 1);

  val &= ~(SZ_4K - 1);

I *think* what you're doing is computing a config space offset and
mapping the 4KB window that contains it.

"addr" is not used at all in the AFI_AXI_BAR0 programming, so this
would be clearer as:

    val = ...
    afi_writel(...)
    afi_writel(...)

    addr = pcie->cfg_va_base + (val & (SZ_4K - 1));
  }

  return addr;

> +		afi_writel(pcie, pcie->cs->start - val, AFI_AXI_BAR0_START);
> +		afi_writel(pcie, (val + SZ_4K) >> 12, AFI_AXI_BAR0_SZ);
>  	}
>  
>  	return addr;
> -- 
> 2.7.4
> 

Re: [PATCH 1/6] PCI: tegra: refactor config space mapping code

[Vidya Sagar]

On Saturday 21 October 2017 12:38 AM, Bjorn Helgaas wrote:
> On Fri, Oct 13, 2017 at 12:20:06AM +0530, Vidya Sagar wrote:
>> use only 4K space from available 1GB PCIe aperture to access
>> end points configuration space by dynamically moving AFI_AXI_BAR
>> base address and always making sure that the desired location
>> to be accessed for generating required config space access falls
>> in the 4K space reserved for this purpose. This would give more
>> space for mapping end point device's BARs on some of Tegra platforms
> Do you now have a revlock issue between this driver change and the
> corresponding DT changes?  I.e., what happens when you run this driver
> change with an older DT that specifies a 256MB config aperture?  What
> happens if you run the older driver with a new DT that only specifies
> a 4KB config aperture?
Yes. Both driver and DT changes must go together. New driver and old DT (or)
Old driver and new DT would break config space access.
>
>> Signed-off-by: Vidya Sagar <vidyas@nvidia.com>
>> ---
>>   drivers/pci/host/pci-tegra.c | 85 ++++++++++++--------------------------------
>>   1 file changed, 23 insertions(+), 62 deletions(-)
>>
>> diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
>> index 9c40da54f88a..ebdcfca10e17 100644
>> --- a/drivers/pci/host/pci-tegra.c
>> +++ b/drivers/pci/host/pci-tegra.c
>> @@ -269,6 +269,8 @@ struct tegra_pcie {
>>   	struct list_head buses;
>>   	struct resource *cs;
>>   
>> +	void __iomem *cfg_va_base;
>> +
>>   	struct resource io;
>>   	struct resource pio;
>>   	struct resource mem;
>> @@ -317,7 +319,6 @@ struct tegra_pcie_port {
>>   };
>>   
>>   struct tegra_pcie_bus {
>> -	struct vm_struct *area;
>>   	struct list_head list;
>>   	unsigned int nr;
>>   };
>> @@ -357,34 +358,16 @@ static inline u32 pads_readl(struct tegra_pcie *pcie, unsigned long offset)
>>    *
>>    * Mapping the whole extended configuration space would require 256 MiB of
>>    * virtual address space, only a small part of which will actually be used.
>> - * To work around this, a 1 MiB of virtual addresses are allocated per bus
>> - * when the bus is first accessed. When the physical range is mapped, the
>> - * the bus number bits are hidden so that the extended register number bits
>> - * appear as bits [19:16]. Therefore the virtual mapping looks like this:
>> - *
>> - *    [19:16] extended register number
>> - *    [15:11] device number
>> - *    [10: 8] function number
>> - *    [ 7: 0] register number
>> - *
>> - * This is achieved by stitching together 16 chunks of 64 KiB of physical
>> - * address space via the MMU.
>> + * To work around this, a 4K of region is used to generate required
>> + * configuration transaction with relevant B:D:F values. This is achieved by
>> + * dynamically programming base address and size of AFI_AXI_BAR used for
>> + * end point config space mapping to make sure that the address (access to
>> + * which generates correct config transaction) falls in this 4K region
>>    */
>> -static unsigned long tegra_pcie_conf_offset(unsigned int devfn, int where)
>> -{
>> -	return ((where & 0xf00) << 8) | (PCI_SLOT(devfn) << 11) |
>> -	       (PCI_FUNC(devfn) << 8) | (where & 0xfc);
>> -}
>> -
>>   static struct tegra_pcie_bus *tegra_pcie_bus_alloc(struct tegra_pcie *pcie,
>>   						   unsigned int busnr)
>>   {
>> -	struct device *dev = pcie->dev;
>> -	pgprot_t prot = pgprot_noncached(PAGE_KERNEL);
>> -	phys_addr_t cs = pcie->cs->start;
>>   	struct tegra_pcie_bus *bus;
>> -	unsigned int i;
>> -	int err;
>>   
>>   	bus = kzalloc(sizeof(*bus), GFP_KERNEL);
>>   	if (!bus)
>> @@ -393,33 +376,16 @@ static struct tegra_pcie_bus *tegra_pcie_bus_alloc(struct tegra_pcie *pcie,
>>   	INIT_LIST_HEAD(&bus->list);
>>   	bus->nr = busnr;
>>   
>> -	/* allocate 1 MiB of virtual addresses */
>> -	bus->area = get_vm_area(SZ_1M, VM_IOREMAP);
>> -	if (!bus->area) {
>> -		err = -ENOMEM;
>> -		goto free;
>> -	}
>> -
>> -	/* map each of the 16 chunks of 64 KiB each */
>> -	for (i = 0; i < 16; i++) {
>> -		unsigned long virt = (unsigned long)bus->area->addr +
>> -				     i * SZ_64K;
>> -		phys_addr_t phys = cs + i * SZ_16M + busnr * SZ_64K;
>> -
>> -		err = ioremap_page_range(virt, virt + SZ_64K, phys, prot);
>> -		if (err < 0) {
>> -			dev_err(dev, "ioremap_page_range() failed: %d\n", err);
>> -			goto unmap;
>> +	if (!pcie->cfg_va_base) {
>> +		pcie->cfg_va_base = ioremap(pcie->cs->start, SZ_4K);
>> +		if (!pcie->cfg_va_base) {
>> +			dev_err(pcie->dev, "failed to ioremap config space\n");
>> +			kfree(bus);
>> +			bus = (struct tegra_pcie_bus *)-ENOMEM;
> tegra_pcie_bus_alloc() used to do some bus-specific work -- it allocated VM
> space and mapped it.  This pcie->cfg_va_base ioremap() is not bus-specific,
> so I don't think it fits here.  It looks like something that should be done
> in the tegra_pcie_probe() path instead.
>
> If you move that, then I think you might as well inline the kzalloc() stuff
> into tegra_pcie_add_bus().
I'll take care of it in next patch.
>>   		}
>>   	}
>>   
>>   	return bus;
>> -
>> -unmap:
>> -	vunmap(bus->area->addr);
>> -free:
>> -	kfree(bus);
>> -	return ERR_PTR(err);
>>   }
>>   
>>   static int tegra_pcie_add_bus(struct pci_bus *bus)
>> @@ -445,12 +411,12 @@ static void tegra_pcie_remove_bus(struct pci_bus *child)
>>   
>>   	list_for_each_entry_safe(bus, tmp, &pcie->buses, list) {
>>   		if (bus->nr == child->number) {
>> -			vunmap(bus->area->addr);
>>   			list_del(&bus->list);
>>   			kfree(bus);
>>   			break;
>>   		}
>>   	}
>> +	iounmap(pcie->cfg_va_base);
>>   }
>>   
>>   static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
>> @@ -459,8 +425,8 @@ static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
>>   {
>>   	struct pci_host_bridge *host = pci_find_host_bridge(bus);
>>   	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
>> -	struct device *dev = pcie->dev;
>>   	void __iomem *addr = NULL;
>> +	u32 val = 0;
>>   
>>   	if (bus->number == 0) {
>>   		unsigned int slot = PCI_SLOT(devfn);
>> @@ -473,19 +439,14 @@ static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
>>   			}
>>   		}
>>   	} else {
>> -		struct tegra_pcie_bus *b;
>> -
>> -		list_for_each_entry(b, &pcie->buses, list)
>> -			if (b->nr == bus->number)
>> -				addr = (void __iomem *)b->area->addr;
>> -
>> -		if (!addr) {
>> -			dev_err(dev, "failed to map cfg. space for bus %u\n",
>> -				bus->number);
>> -			return NULL;
>> -		}
>> -
>> -		addr += tegra_pcie_conf_offset(devfn, where);
>> +		addr = pcie->cfg_va_base;
>> +		val = ((((u32)where & 0xf00) >> 8) << 24) |
>> +		      (bus->number << 16) | (PCI_SLOT(devfn) << 11) |
>> +		      (PCI_FUNC(devfn) << 8) | (where & 0xff);
> I think it would be useful to keep tegra_pcie_conf_offset() (extending
> it to take the bus number) near the comment describing the mapping.
I'll take care of it in next patch.
>
>> +		addr = (val & (SZ_4K - 1)) + addr;
>> +		val = val & ~(SZ_4K - 1);
>    val &= ~(SZ_4K - 1);
>
> I *think* what you're doing is computing a config space offset and
> mapping the 4KB window that contains it.
>
> "addr" is not used at all in the AFI_AXI_BAR0 programming, so this
> would be clearer as:
>
>      val = ...
>      afi_writel(...)
>      afi_writel(...)
>
>      addr = pcie->cfg_va_base + (val & (SZ_4K - 1));
but, addr is using the intermediate 'val' i.e. before 'val &= ~(SZ_4K - 
1)' operation is performed on it.
>    }
>
>    return addr;
>
>> +		afi_writel(pcie, pcie->cs->start - val, AFI_AXI_BAR0_START);
>> +		afi_writel(pcie, (val + SZ_4K) >> 12, AFI_AXI_BAR0_SZ);
>>   	}
>>   
>>   	return addr;
>> -- 
>> 2.7.4
>>

Re: [PATCH 1/6] PCI: tegra: refactor config space mapping code

[Bjorn Helgaas]

[+cc Rob, devicetree list]

On Mon, Oct 23, 2017 at 11:09:50AM +0530, Vidya Sagar wrote:
> On Saturday 21 October 2017 12:38 AM, Bjorn Helgaas wrote:
> >On Fri, Oct 13, 2017 at 12:20:06AM +0530, Vidya Sagar wrote:
> >>use only 4K space from available 1GB PCIe aperture to access
> >>end points configuration space by dynamically moving AFI_AXI_BAR
> >>base address and always making sure that the desired location
> >>to be accessed for generating required config space access falls
> >>in the 4K space reserved for this purpose. This would give more
> >>space for mapping end point device's BARs on some of Tegra platforms
> >
> >Do you now have a revlock issue between this driver change and the
> >corresponding DT changes?  I.e., what happens when you run this driver
> >change with an older DT that specifies a 256MB config aperture?  What
> >happens if you run the older driver with a new DT that only specifies
> >a 4KB config aperture?
>
> Yes. Both driver and DT changes must go together. New driver and old DT (or)
> Old driver and new DT would break config space access.

This revlock sounds like a pretty major issue.  I don't know if the DT
conventions even allow for this.  If they do, it definitely needs to
be mentioned in the changelog.

Looking for acks from Thierry and Rob.

> >>-		addr += tegra_pcie_conf_offset(devfn, where);
> >>+		addr = pcie->cfg_va_base;
> >>+		val = ((((u32)where & 0xf00) >> 8) << 24) |
> >>+		      (bus->number << 16) | (PCI_SLOT(devfn) << 11) |
> >>+		      (PCI_FUNC(devfn) << 8) | (where & 0xff);
> >I think it would be useful to keep tegra_pcie_conf_offset() (extending
> >it to take the bus number) near the comment describing the mapping.
> I'll take care of it in next patch.
> >
> >>+		addr = (val & (SZ_4K - 1)) + addr;
> >>+		val = val & ~(SZ_4K - 1);
> >   val &= ~(SZ_4K - 1);
> >
> >I *think* what you're doing is computing a config space offset and
> >mapping the 4KB window that contains it.
> >
> >"addr" is not used at all in the AFI_AXI_BAR0 programming, so this
> >would be clearer as:
> >
> >     val = ...
> >     afi_writel(...)
> >     afi_writel(...)
> >
> >     addr = pcie->cfg_va_base + (val & (SZ_4K - 1));
> but, addr is using the intermediate 'val' i.e. before 'val &=
> ~(SZ_4K - 1)' operation is performed on it.

Ah, right.  I think adding another temporary variable here would make
it clearer.

> >   }
> >
> >   return addr;
> >
> >>+		afi_writel(pcie, pcie->cs->start - val, AFI_AXI_BAR0_START);
> >>+		afi_writel(pcie, (val + SZ_4K) >> 12, AFI_AXI_BAR0_SZ);
> >>  	}
> >>  	return addr;
> >>-- 
> >>2.7.4
> >>
> 

[PATCH 2/6] ARM: tegra: limit PCIe config space mapping to 4K for T20

[Vidya Sagar]

reduces PCIe end point config space mapping size from its
current 256MB to 4K to give more space for BAR mapping

Signed-off-by: Vidya Sagar <vidyas@nvidia.com>
---
 arch/arm/boot/dts/tegra20.dtsi | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/arch/arm/boot/dts/tegra20.dtsi b/arch/arm/boot/dts/tegra20.dtsi
index 7c85f97f72ea..4c761a60cdf7 100644
--- a/arch/arm/boot/dts/tegra20.dtsi
+++ b/arch/arm/boot/dts/tegra20.dtsi
@@ -591,7 +591,7 @@
 		device_type = "pci";
 		reg = <0x80003000 0x00000800   /* PADS registers */
 		       0x80003800 0x00000200   /* AFI registers */
-		       0x90000000 0x10000000>; /* configuration space */
+		       0x82000000 0x00001000>; /* configuration space */
 		reg-names = "pads", "afi", "cs";
 		interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH   /* controller interrupt */
 			      GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
@@ -607,9 +607,9 @@
 
 		ranges = <0x82000000 0 0x80000000 0x80000000 0 0x00001000   /* port 0 registers */
 			  0x82000000 0 0x80001000 0x80001000 0 0x00001000   /* port 1 registers */
-			  0x81000000 0 0          0x82000000 0 0x00010000   /* downstream I/O */
-			  0x82000000 0 0xa0000000 0xa0000000 0 0x08000000   /* non-prefetchable memory */
-			  0xc2000000 0 0xa8000000 0xa8000000 0 0x18000000>; /* prefetchable memory */
+			  0x81000000 0 0          0x82001000 0 0x00010000   /* downstream I/O */
+			  0x82000000 0 0x82100000 0x82100000 0 0x05F00000   /* non-prefetchable memory */
+			  0xc2000000 0 0x88000000 0x88000000 0 0x38000000>; /* prefetchable memory */
 
 		clocks = <&tegra_car TEGRA20_CLK_PEX>,
 			 <&tegra_car TEGRA20_CLK_AFI>,
-- 
2.7.4

[PATCH 3/6] ARM: tegra: limit PCIe config space mapping to 4K for T30

[Vidya Sagar]

reduces PCIe config space mapping size from its current
256MB to 4K to have only 4K of virtual memory mapping and to be
in line with driver implementation

Signed-off-by: Vidya Sagar <vidyas@nvidia.com>
---
 arch/arm/boot/dts/tegra30.dtsi | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/arm/boot/dts/tegra30.dtsi b/arch/arm/boot/dts/tegra30.dtsi
index 13960fda7471..b23171f2b86f 100644
--- a/arch/arm/boot/dts/tegra30.dtsi
+++ b/arch/arm/boot/dts/tegra30.dtsi
@@ -15,7 +15,7 @@
 		device_type = "pci";
 		reg = <0x00003000 0x00000800   /* PADS registers */
 		       0x00003800 0x00000200   /* AFI registers */
-		       0x10000000 0x10000000>; /* configuration space */
+		       0x1FFFF000 0x00001000>; /* configuration space */
 		reg-names = "pads", "afi", "cs";
 		interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH   /* controller interrupt */
 			      GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
-- 
2.7.4

[PATCH 4/6] ARM: tegra: limit PCIe config space mapping to 4K for T124

[Vidya Sagar]

reduces PCIe config space mapping size from its current 256MB
to 4K to have only 4K of virtual memory mapping and to be
in line with driver implementation

Signed-off-by: Vidya Sagar <vidyas@nvidia.com>
---
 arch/arm/boot/dts/tegra124.dtsi | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/arm/boot/dts/tegra124.dtsi b/arch/arm/boot/dts/tegra124.dtsi
index 8baf00b89efb..e3cd9dca57cb 100644
--- a/arch/arm/boot/dts/tegra124.dtsi
+++ b/arch/arm/boot/dts/tegra124.dtsi
@@ -19,7 +19,7 @@
 		device_type = "pci";
 		reg = <0x0 0x01003000 0x0 0x00000800   /* PADS registers */
 		       0x0 0x01003800 0x0 0x00000800   /* AFI registers */
-		       0x0 0x02000000 0x0 0x10000000>; /* configuration space */
+		       0x0 0x11FFF000 0x0 0x00001000>; /* configuration space */
 		reg-names = "pads", "afi", "cs";
 		interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
 			     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
-- 
2.7.4

[PATCH 5/6] ARM64: tegra: limit PCIe config space mapping to 4K for T132

[Vidya Sagar]

reduces PCIe config space mapping size from its current 256MB
to 4K to have only 4K of virtual memory mapping and to be
in line with driver implementation

Signed-off-by: Vidya Sagar <vidyas@nvidia.com>
---
 arch/arm64/boot/dts/nvidia/tegra132.dtsi | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/arm64/boot/dts/nvidia/tegra132.dtsi b/arch/arm64/boot/dts/nvidia/tegra132.dtsi
index c2f0f2743578..2f0ff087112e 100644
--- a/arch/arm64/boot/dts/nvidia/tegra132.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra132.dtsi
@@ -17,7 +17,7 @@
 		device_type = "pci";
 		reg = <0x0 0x01003000 0x0 0x00000800   /* PADS registers */
 		       0x0 0x01003800 0x0 0x00000800   /* AFI registers */
-		       0x0 0x02000000 0x0 0x10000000>; /* configuration space */
+		       0x0 0x11FFF000 0x0 0x00001000>; /* configuration space */
 		reg-names = "pads", "afi", "cs";
 		interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
 			     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
-- 
2.7.4

[PATCH 6/6] ARM64: tegra: limit PCIe config space mapping to 4K for T210

[Vidya Sagar]

reduces PCIe config space mapping size from its current 256MB
to 4K to have only 4K of virtual memory mapping and to be
in line with driver implementation

Signed-off-by: Vidya Sagar <vidyas@nvidia.com>
---
 arch/arm64/boot/dts/nvidia/tegra210.dtsi | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/arm64/boot/dts/nvidia/tegra210.dtsi b/arch/arm64/boot/dts/nvidia/tegra210.dtsi
index 29f471e0f22a..8cad2516597b 100644
--- a/arch/arm64/boot/dts/nvidia/tegra210.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra210.dtsi
@@ -16,7 +16,7 @@
 		device_type = "pci";
 		reg = <0x0 0x01003000 0x0 0x00000800   /* PADS registers */
 		       0x0 0x01003800 0x0 0x00000800   /* AFI registers */
-		       0x0 0x02000000 0x0 0x10000000>; /* configuration space */
+		       0x0 0x11FFF000 0x0 0x00001000>; /* configuration space */
 		reg-names = "pads", "afi", "cs";
 		interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
 			     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
-- 
2.7.4

Re: [PATCH 0/6] Tegra PCIe end point config space map code refactoring

[Manikanta Maddireddy]

These series of patches are tested on Tegra210

Reviewed-by: Manikanta Maddireddy<mmaddireddy@nvidia.com>
Tested-by: Manikanta Maddireddy<mmaddireddy@nvidia.com>


On 13-Oct-17 12:20 AM, Vidya Sagar wrote:
> PCIe host controller in Tegra SoCs has 1GB of aperture available
> for mapping end points config space, IO and BARs. In that, currently
> 256MB is being reserved for mapping end points configuration space
> which leaves less memory space available for mapping end points BARs
> on some of the platforms.
> This patch series attempts to use only 4K space from 1GB aperture to
> access end points configuration space.
> 
> Currently, this change benefits T20 and future chips in saving (i.e. repurposed
> to use for BAR mapping) physical space as well as kernel virtual mapping space,
> it saves only kernel virtual address space in T30, T124, T132 and T210.
> 
> Testing Done on T210:
>  Enumeration is and basic functionality of immediate devices
>  Enumeration of devices behind a PCIe switch
>  Complete 4K configuration space access
> 
> Vidya Sagar (6):
>   PCI: tegra: refactor config space mapping code
>   ARM: tegra: limit PCIe config space mapping to 4K for T20
>   ARM: tegra: limit PCIe config space mapping to 4K for T30
>   ARM: tegra: limit PCIe config space mapping to 4K for T124
>   ARM64: tegra: limit PCIe config space mapping to 4K for T132
>   ARM64: tegra: limit PCIe config space mapping to 4K for T210
> 
>  arch/arm/boot/dts/tegra124.dtsi          |  2 +-
>  arch/arm/boot/dts/tegra20.dtsi           |  8 +--
>  arch/arm/boot/dts/tegra30.dtsi           |  2 +-
>  arch/arm64/boot/dts/nvidia/tegra132.dtsi |  2 +-
>  arch/arm64/boot/dts/nvidia/tegra210.dtsi |  2 +-
>  drivers/pci/host/pci-tegra.c             | 85 +++++++++-----------------------
>  6 files changed, 31 insertions(+), 70 deletions(-)
>