[PATCH 0/4] PCI: dwc: .cpu_addr_fixup() sketch

[PATCH 0/4] PCI: dwc: .cpu_addr_fixup() sketch

[Bjorn Helgaas]

From: Bjorn Helgaas <bhelgaas@google.com>

This is not complete or ready to merge at all.  It's just a sketch to see
if this approach to moving away from the .cpu_addr_fixup() approach is
feasible.

This is based on Frank's series at
https://lore.kernel.org/r/20250128-pci_fixup_addr-v9-0-3c4bb506f665@nxp.com,
and these apply on top of these patches from that series, which applies on
v6.15-rc1:

      PCI: dwc: Use resource start as iomap() input in dw_pcie_pme_turn_off()
      PCI: dwc: Rename cpu_addr to parent_bus_addr for ATU configuration
      PCI: Add parent_bus_offset to resource_entry

The idea is that on some systems the PCI controller lives in a different
address space (an "intermediate" address space) than the CPU physical
address space.  Frank has a beautiful picture of this at
https://lore.kernel.org/r/20250128-pci_fixup_addr-v9-4-3c4bb506f665@nxp.com

The devicetree *should* describe the fabric address translation between the
CPU and intermediate address spaces, but historically we haven't taken
advantage of that and have used .cpu_addr_fixup() functions to convert CPU
to intermediate addresses each time we program the ATU.

What I tried to do here was to:

  - Try to extract the offset from devicetree using the 'reg' property
    (this is from Frank's patch at the link above)

  - If a .cpu_addr_fixup() function exists, run that as well and compare
    with the offset from devicetree

  - Save the offset in struct dw_pcie_rp and apply that instead of calling
    .cpu_addr_fixup() in the future

  - Emit messages about devicetree offsets that don't match what
    .cpu_addr_fixup() did, or about .cpu_addr_fixup() methods that are
    superfluous because they *do* match the devicetree offset

I only looked at the RC side and didn't do anything at all with the EP
side.

Bjorn Helgaas (4):
  PCI: dwc: Move cfg0 setup to dw_pcie_cfg0_setup()
  PCI: dwc: Delay cfg0 setup until after discovering bridge windows
  PCI: dwc: Look up 'config' address
  PCI: dwc: Use parent_bus_offset

 .../pci/controller/dwc/pcie-designware-host.c | 88 +++++++++++++++----
 drivers/pci/controller/dwc/pcie-designware.c  |  3 -
 drivers/pci/controller/dwc/pcie-designware.h  |  1 +
 3 files changed, 72 insertions(+), 20 deletions(-)

-- 
2.34.1

[PATCH 1/4] PCI: dwc: Move cfg0 setup to dw_pcie_cfg0_setup()

[Bjorn Helgaas]

From: Bjorn Helgaas <bhelgaas@google.com>

Move pp->cfg0 setup to dw_pcie_cfg0_setup().  No functional change
intended.
---
 .../pci/controller/dwc/pcie-designware-host.c | 34 +++++++++++++------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
index 1206b26bff3f..de2f2dcf5c40 100644
--- a/drivers/pci/controller/dwc/pcie-designware-host.c
+++ b/drivers/pci/controller/dwc/pcie-designware-host.c
@@ -418,22 +418,12 @@ static void dw_pcie_host_request_msg_tlp_res(struct dw_pcie_rp *pp)
 	}
 }
 
-int dw_pcie_host_init(struct dw_pcie_rp *pp)
+static int dw_pcie_cfg0_setup(struct dw_pcie_rp *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	struct device *dev = pci->dev;
-	struct device_node *np = dev->of_node;
 	struct platform_device *pdev = to_platform_device(dev);
-	struct resource_entry *win;
-	struct pci_host_bridge *bridge;
 	struct resource *res;
-	int ret;
-
-	raw_spin_lock_init(&pp->lock);
-
-	ret = dw_pcie_get_resources(pci);
-	if (ret)
-		return ret;
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config");
 	if (!res) {
@@ -448,6 +438,28 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
 	if (IS_ERR(pp->va_cfg0_base))
 		return PTR_ERR(pp->va_cfg0_base);
 
+	return 0;
+}
+
+int dw_pcie_host_init(struct dw_pcie_rp *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct device *dev = pci->dev;
+	struct device_node *np = dev->of_node;
+	struct resource_entry *win;
+	struct pci_host_bridge *bridge;
+	int ret;
+
+	raw_spin_lock_init(&pp->lock);
+
+	ret = dw_pcie_get_resources(pci);
+	if (ret)
+		return ret;
+
+	ret = dw_pcie_cfg0_setup(pp);
+	if (ret)
+		return ret;
+
 	bridge = devm_pci_alloc_host_bridge(dev, 0);
 	if (!bridge)
 		return -ENOMEM;
-- 
2.34.1

[PATCH 2/4] PCI: dwc: Delay cfg0 setup until after discovering bridge windows

[Bjorn Helgaas]

From: Bjorn Helgaas <bhelgaas@google.com>

devm_pci_alloc_host_bridge() reads host bridge windows and any translation
offsets.  Some .cpu_addr_fixup() implementations depend on the window
offset, e.g., imx_pcie_cpu_addr_fixup() uses the offset of the first bridge
window.
---
 drivers/pci/controller/dwc/pcie-designware-host.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
index de2f2dcf5c40..b9eaba157dae 100644
--- a/drivers/pci/controller/dwc/pcie-designware-host.c
+++ b/drivers/pci/controller/dwc/pcie-designware-host.c
@@ -456,14 +456,14 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
 	if (ret)
 		return ret;
 
-	ret = dw_pcie_cfg0_setup(pp);
-	if (ret)
-		return ret;
-
 	bridge = devm_pci_alloc_host_bridge(dev, 0);
 	if (!bridge)
 		return -ENOMEM;
 
+	ret = dw_pcie_cfg0_setup(pp);
+	if (ret)
+		return ret;
+
 	pp->bridge = bridge;
 
 	/* Get the I/O range from DT */
-- 
2.34.1

[PATCH 3/4] PCI: dwc: Look up 'config' address

[Bjorn Helgaas]

From: Bjorn Helgaas <bhelgaas@google.com>

Set pp->parent_bus_offset based on the parent bus address from the "config"
reg entry if it exists.

.cpu_addr_fixup(res->start) (if implemented) should return the parent bus
address corresponding to res->start.

Sets pp->parent_bus_offset, but doesn't use it, so no functional change
intended yet.
---
 .../pci/controller/dwc/pcie-designware-host.c | 42 +++++++++++++++++++
 drivers/pci/controller/dwc/pcie-designware.h  |  1 +
 2 files changed, 43 insertions(+)

diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
index b9eaba157dae..e22f650ada5a 100644
--- a/drivers/pci/controller/dwc/pcie-designware-host.c
+++ b/drivers/pci/controller/dwc/pcie-designware-host.c
@@ -423,7 +423,11 @@ static int dw_pcie_cfg0_setup(struct dw_pcie_rp *pp)
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	struct device *dev = pci->dev;
 	struct platform_device *pdev = to_platform_device(dev);
+	struct device_node *np = dev->of_node;
 	struct resource *res;
+	int index;
+	u64 reg_addr, fixup_addr;
+	u64 (*fixup)(struct dw_pcie *pcie, u64 cpu_addr);
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config");
 	if (!res) {
@@ -434,6 +438,44 @@ static int dw_pcie_cfg0_setup(struct dw_pcie_rp *pp)
 	pp->cfg0_size = resource_size(res);
 	pp->cfg0_base = res->start;
 
+	/* [mem 0x7ff00000-] in example */
+	dev_info(dev, "%pR config CPU physical\n", res);
+
+	/* Look up "config" address on parent bus */
+	reg_addr = 0;
+	index = of_property_match_string(np, "reg-names", "config");
+	if (index >= 0) {
+		of_property_read_reg(np, index, &reg_addr, NULL);
+		/* [ia  0x8ff00000-] in example */
+		dev_info(dev, "%#010llx config reg[%d] parent bus addr\n",
+			 reg_addr, index);
+	} else {
+		reg_addr = res->start;
+		dev_warn(dev, "%#010llx assumed parent bus addr (no config reg-names entry)\n",
+			 reg_addr);
+	}
+
+	fixup = pci->ops->cpu_addr_fixup;
+	if (fixup) {
+		fixup_addr = fixup(pci, res->start);
+		dev_info(dev, "%#010llx result of %ps(%#010llx)\n",
+			 fixup_addr, fixup, res->start);
+		if (reg_addr == fixup_addr) {
+			dev_info(dev, "%#010llx config reg[%d] == %#010llx; %ps is redundant\n",
+				 reg_addr, index, fixup_addr, fixup);
+		} else {
+			dev_warn(dev, "%#010llx config reg[%d] != %#010llx fixed up addr; DT is broken\n",
+				 reg_addr, index, fixup_addr);
+			reg_addr = fixup_addr;
+		}
+	}
+
+	/* 0x7ff00000 - 0x8ff00000 == 0xf0000000 */
+	pp->parent_bus_offset = res->start - reg_addr;
+	dev_info(dev, "%#010llx config parent bus offset\n",
+		 pp->parent_bus_offset);
+
+
 	pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res);
 	if (IS_ERR(pp->va_cfg0_base))
 		return PTR_ERR(pp->va_cfg0_base);
diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
index ac23604c829f..eeca38ec3a2b 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -362,6 +362,7 @@ struct dw_pcie_rp {
 	u64			cfg0_base;
 	void __iomem		*va_cfg0_base;
 	u32			cfg0_size;
+	u64			parent_bus_offset;
 	resource_size_t		io_base;
 	phys_addr_t		io_bus_addr;
 	u32			io_size;
-- 
2.34.1

[PATCH 4/4] PCI: dwc: Use parent_bus_offset

[Bjorn Helgaas]

From: Bjorn Helgaas <bhelgaas@google.com>

We know the parent_bus_offset, either computed from a DT reg property (the
offset is the CPU physical addr - the 'config' address on the parent bus)
or from a .cpu_addr_fixup() (which may have used a host bridge window
offset).

Apply that parent_bus_offset instead of calling .cpu_addr_fixup() again.

This assumes that all intermediate addresses are at the same offset from
the CPU physical addresses.
---
 drivers/pci/controller/dwc/pcie-designware-host.c | 12 ++++++------
 drivers/pci/controller/dwc/pcie-designware.c      |  3 ---
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
index e22f650ada5a..3378f905b3bd 100644
--- a/drivers/pci/controller/dwc/pcie-designware-host.c
+++ b/drivers/pci/controller/dwc/pcie-designware-host.c
@@ -670,7 +670,7 @@ static void __iomem *dw_pcie_other_conf_map_bus(struct pci_bus *bus,
 		type = PCIE_ATU_TYPE_CFG1;
 
 	atu.type = type;
-	atu.parent_bus_addr = pp->cfg0_base;
+	atu.parent_bus_addr = pp->cfg0_base - pp->parent_bus_offset;
 	atu.pci_addr = busdev;
 	atu.size = pp->cfg0_size;
 
@@ -695,7 +695,7 @@ static int dw_pcie_rd_other_conf(struct pci_bus *bus, unsigned int devfn,
 
 	if (pp->cfg0_io_shared) {
 		atu.type = PCIE_ATU_TYPE_IO;
-		atu.parent_bus_addr = pp->io_base;
+		atu.parent_bus_addr = pp->io_base - pp->parent_bus_offset;
 		atu.pci_addr = pp->io_bus_addr;
 		atu.size = pp->io_size;
 
@@ -721,7 +721,7 @@ static int dw_pcie_wr_other_conf(struct pci_bus *bus, unsigned int devfn,
 
 	if (pp->cfg0_io_shared) {
 		atu.type = PCIE_ATU_TYPE_IO;
-		atu.parent_bus_addr = pp->io_base;
+		atu.parent_bus_addr = pp->io_base - pp->parent_bus_offset;
 		atu.pci_addr = pp->io_bus_addr;
 		atu.size = pp->io_size;
 
@@ -790,7 +790,7 @@ static int dw_pcie_iatu_setup(struct dw_pcie_rp *pp)
 
 		atu.index = i;
 		atu.type = PCIE_ATU_TYPE_MEM;
-		atu.parent_bus_addr = entry->res->start;
+		atu.parent_bus_addr = entry->res->start - pp->parent_bus_offset;
 		atu.pci_addr = entry->res->start - entry->offset;
 
 		/* Adjust iATU size if MSG TLP region was allocated before */
@@ -812,7 +812,7 @@ static int dw_pcie_iatu_setup(struct dw_pcie_rp *pp)
 		if (pci->num_ob_windows > ++i) {
 			atu.index = i;
 			atu.type = PCIE_ATU_TYPE_IO;
-			atu.parent_bus_addr = pp->io_base;
+			atu.parent_bus_addr = pp->io_base - pp->parent_bus_offset;
 			atu.pci_addr = pp->io_bus_addr;
 			atu.size = pp->io_size;
 
@@ -956,7 +956,7 @@ static int dw_pcie_pme_turn_off(struct dw_pcie *pci)
 	atu.size = resource_size(pci->pp.msg_res);
 	atu.index = pci->pp.msg_atu_index;
 
-	atu.parent_bus_addr = pci->pp.msg_res->start;
+	atu.parent_bus_addr = pci->pp.msg_res->start - pci->pp.parent_bus_offset;
 
 	ret = dw_pcie_prog_outbound_atu(pci, &atu);
 	if (ret)
diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c
index 9d0a5f75effc..640caf4a084f 100644
--- a/drivers/pci/controller/dwc/pcie-designware.c
+++ b/drivers/pci/controller/dwc/pcie-designware.c
@@ -474,9 +474,6 @@ int dw_pcie_prog_outbound_atu(struct dw_pcie *pci,
 	u32 retries, val;
 	u64 limit_addr;
 
-	if (pci->ops && pci->ops->cpu_addr_fixup)
-		parent_bus_addr = pci->ops->cpu_addr_fixup(pci, parent_bus_addr);
-
 	limit_addr = parent_bus_addr + atu->size - 1;
 
 	if ((limit_addr & ~pci->region_limit) != (parent_bus_addr & ~pci->region_limit) ||
-- 
2.34.1