[PATCH 1/2] PCI: tegra: Support MSI 64-bit addressing

[PATCH 1/2] PCI: tegra: Support MSI 64-bit addressing

[Thierry Reding]

From: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>

The MSI target address can reside beyond the 32-bit boundary on devices
with more than 2 GiB of system memory. The PCI host bridge on Tegra can
easily support 64-bit addresses, so make sure to pass the upper 32 bits
of the target address to endpoints when allocating MSI entries.

Signed-off-by: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
---
 drivers/pci/host/pci-tegra.c | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
index ecf8b4f1bf05..e69f828a9ab2 100644
--- a/drivers/pci/host/pci-tegra.c
+++ b/drivers/pci/host/pci-tegra.c
@@ -235,6 +235,7 @@ struct tegra_msi {
 	struct irq_domain *domain;
 	unsigned long pages;
 	struct mutex lock;
+	u64 phys;
 	int irq;
 };
 
@@ -1468,9 +1469,8 @@ static int tegra_msi_setup_irq(struct msi_controller *chip,
 
 	irq_set_msi_desc(irq, desc);
 
-	msg.address_lo = virt_to_phys((void *)msi->pages);
-	/* 32 bit address only */
-	msg.address_hi = 0;
+	msg.address_lo = lower_32_bits(msi->phys);
+	msg.address_hi = upper_32_bits(msi->phys);
 	msg.data = hwirq;
 
 	pci_write_msi_msg(irq, &msg);
@@ -1519,7 +1519,6 @@ static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
 	const struct tegra_pcie_soc *soc = pcie->soc;
 	struct tegra_msi *msi = &pcie->msi;
 	struct device *dev = pcie->dev;
-	unsigned long base;
 	int err;
 	u32 reg;
 
@@ -1553,10 +1552,10 @@ static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
 
 	/* setup AFI/FPCI range */
 	msi->pages = __get_free_pages(GFP_KERNEL, 0);
-	base = virt_to_phys((void *)msi->pages);
+	msi->phys = virt_to_phys((void *)msi->pages);
 
-	afi_writel(pcie, base >> soc->msi_base_shift, AFI_MSI_FPCI_BAR_ST);
-	afi_writel(pcie, base, AFI_MSI_AXI_BAR_ST);
+	afi_writel(pcie, msi->phys >> soc->msi_base_shift, AFI_MSI_FPCI_BAR_ST);
+	afi_writel(pcie, msi->phys, AFI_MSI_AXI_BAR_ST);
 	/* this register is in 4K increments */
 	afi_writel(pcie, 1, AFI_MSI_BAR_SZ);
 
-- 
2.12.0

[PATCH 2/2] PCI: tegra: Do not allocate MSI target memory

[Thierry Reding]

From: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>

The PCI host bridge found on Tegra SoCs doesn't require the MSI target
address to be backed by physical system memory. Writes are intercepted
within the controller and never make it to the memory pointed to.

Since no actual system memory is required, remove the allocation of a
single page and hardcode the MSI target address with a special address
that maps to the last 4 KiB page within the range that is reserved for
system memory and memory-mapped I/O in the FPCI address map.

Signed-off-by: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
---
 drivers/pci/host/pci-tegra.c | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
index e69f828a9ab2..4f795a5dcbed 100644
--- a/drivers/pci/host/pci-tegra.c
+++ b/drivers/pci/host/pci-tegra.c
@@ -233,7 +233,6 @@ struct tegra_msi {
 	struct msi_controller chip;
 	DECLARE_BITMAP(used, INT_PCI_MSI_NR);
 	struct irq_domain *domain;
-	unsigned long pages;
 	struct mutex lock;
 	u64 phys;
 	int irq;
@@ -1550,9 +1549,22 @@ static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
 		goto err;
 	}
 
-	/* setup AFI/FPCI range */
-	msi->pages = __get_free_pages(GFP_KERNEL, 0);
-	msi->phys = virt_to_phys((void *)msi->pages);
+	/*
+	 * The PCI host bridge on Tegra contains some logic that intercepts
+	 * MSI writes, which means that the MSI target address doesn't have
+	 * to point to actual physical memory. Rather than allocating one 4
+	 * KiB page of system memory that's never used, we can simply pick
+	 * an arbitrary address within an area reserved for system memory
+	 * in the FPCI address map.
+	 *
+	 * However, in order to avoid confusion, we pick an address that
+	 * doesn't map to physical memory. The FPCI address map reserves a
+	 * 1012 GiB region for system memory and memory-mapped I/O. Since
+	 * none of the Tegra SoCs that contain this PCI host bridge can
+	 * address more than 16 GiB of system memory, the last 4 KiB of
+	 * these 1012 GiB is a good candidate.
+	 */
+	msi->phys = 0xfcfffff000;
 
 	afi_writel(pcie, msi->phys >> soc->msi_base_shift, AFI_MSI_FPCI_BAR_ST);
 	afi_writel(pcie, msi->phys, AFI_MSI_AXI_BAR_ST);
@@ -1604,8 +1616,6 @@ static int tegra_pcie_disable_msi(struct tegra_pcie *pcie)
 	afi_writel(pcie, 0, AFI_MSI_EN_VEC6);
 	afi_writel(pcie, 0, AFI_MSI_EN_VEC7);
 
-	free_pages(msi->pages, 0);
-
 	if (msi->irq > 0)
 		free_irq(msi->irq, pcie);
 
-- 
2.12.0

Re: [PATCH 1/2] PCI: tegra: Support MSI 64-bit addressing

[Stephen Warren]

On 05/04/2017 02:10 PM, Thierry Reding wrote:
> From: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
>
> The MSI target address can reside beyond the 32-bit boundary on devices
> with more than 2 GiB of system memory. The PCI host bridge on Tegra can
> easily support 64-bit addresses, so make sure to pass the upper 32 bits
> of the target address to endpoints when allocating MSI entries.

The series,
Acked-by: Stephen Warren <swarren-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>

Should the first patch in the series be CC: stable since it fixes a bug?

Re: [PATCH 1/2] PCI: tegra: Support MSI 64-bit addressing

[Bjorn Helgaas]

On Thu, May 04, 2017 at 10:10:31PM +0200, Thierry Reding wrote:
> From: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> 
> The MSI target address can reside beyond the 32-bit boundary on devices
> with more than 2 GiB of system memory. The PCI host bridge on Tegra can
> easily support 64-bit addresses, so make sure to pass the upper 32 bits
> of the target address to endpoints when allocating MSI entries.
> 
> Signed-off-by: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>

Applied both with Stephen's ack to pci/host-tegra for v4.13, thanks!

> ---
>  drivers/pci/host/pci-tegra.c | 13 ++++++-------
>  1 file changed, 6 insertions(+), 7 deletions(-)
> 
> diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
> index ecf8b4f1bf05..e69f828a9ab2 100644
> --- a/drivers/pci/host/pci-tegra.c
> +++ b/drivers/pci/host/pci-tegra.c
> @@ -235,6 +235,7 @@ struct tegra_msi {
>  	struct irq_domain *domain;
>  	unsigned long pages;
>  	struct mutex lock;
> +	u64 phys;
>  	int irq;
>  };
>  
> @@ -1468,9 +1469,8 @@ static int tegra_msi_setup_irq(struct msi_controller *chip,
>  
>  	irq_set_msi_desc(irq, desc);
>  
> -	msg.address_lo = virt_to_phys((void *)msi->pages);
> -	/* 32 bit address only */
> -	msg.address_hi = 0;
> +	msg.address_lo = lower_32_bits(msi->phys);
> +	msg.address_hi = upper_32_bits(msi->phys);
>  	msg.data = hwirq;
>  
>  	pci_write_msi_msg(irq, &msg);
> @@ -1519,7 +1519,6 @@ static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
>  	const struct tegra_pcie_soc *soc = pcie->soc;
>  	struct tegra_msi *msi = &pcie->msi;
>  	struct device *dev = pcie->dev;
> -	unsigned long base;
>  	int err;
>  	u32 reg;
>  
> @@ -1553,10 +1552,10 @@ static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
>  
>  	/* setup AFI/FPCI range */
>  	msi->pages = __get_free_pages(GFP_KERNEL, 0);
> -	base = virt_to_phys((void *)msi->pages);
> +	msi->phys = virt_to_phys((void *)msi->pages);
>  
> -	afi_writel(pcie, base >> soc->msi_base_shift, AFI_MSI_FPCI_BAR_ST);
> -	afi_writel(pcie, base, AFI_MSI_AXI_BAR_ST);
> +	afi_writel(pcie, msi->phys >> soc->msi_base_shift, AFI_MSI_FPCI_BAR_ST);
> +	afi_writel(pcie, msi->phys, AFI_MSI_AXI_BAR_ST);
>  	/* this register is in 4K increments */
>  	afi_writel(pcie, 1, AFI_MSI_BAR_SZ);
>  
> -- 
> 2.12.0
>