Re: [PATCH V6 09/13] pci, acpi: Support for ACPI based generic PCI host controller

[Bjorn Helgaas <helgaas@kernel.org>]

Hey, I really kind of like this.  I think this might work out well.

On Fri, Apr 15, 2016 at 07:06:44PM +0200, Tomasz Nowicki wrote:
> This patch is going to implement generic PCI host controller for
> ACPI world, similar to what pci-host-generic.c driver does for DT world.
> 
> All such drivers, which we have seen so far, were implemented within
> arch/ directory since they had some arch assumptions (x86 and ia64).
> However, they all are doing similar thing, so it makes sense to find
> some common code and abstract it into the generic driver.
> 
> In order to handle PCI config space regions properly, we define new
> MCFG interface which parses MCFG table and keep its entries
> in a list. New pci_mcfg_init call is defined so that we do not depend
> on PCI_MMCONFIG. Regions are not mapped until host bridge ask for it.
> 
> The implementation of pci_acpi_scan_root() looks up the saved MCFG entries
> and sets up a new mapping. Generic PCI functions are used for
> accessing config space. Driver selects PCI_GENERIC_ECAM and uses functions
> from drivers/pci/ecam.h to create and access ECAM mappings.
> 
> As mentioned in Kconfig help section, ACPI_PCI_HOST_GENERIC choice
> should be made on a per-architecture basis.
> 
> This patch is heavily based on the updated version from Jayachandran C:
> https://lkml.org/lkml/2016/4/11/908
> git: https://github.com/jchandra-brcm/linux/ (arm64-acpi-pci-v3)
> 
> Signed-off-by: Tomasz Nowicki <tn@semihalf.com>
> Signed-off-by: Jayachandran C <jchandra@broadcom.com>
> ---
>  drivers/acpi/Kconfig        |   8 ++
>  drivers/acpi/Makefile       |   1 +
>  drivers/acpi/bus.c          |   1 +
>  drivers/acpi/pci_gen_host.c | 231 ++++++++++++++++++++++++++++++++++++++++++++
>  include/linux/pci.h         |   6 ++
>  5 files changed, 247 insertions(+)
>  create mode 100644 drivers/acpi/pci_gen_host.c
> 
> diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig
> index 183ffa3..70272c5 100644
> --- a/drivers/acpi/Kconfig
> +++ b/drivers/acpi/Kconfig
> @@ -346,6 +346,14 @@ config ACPI_PCI_SLOT
>  	  i.e., segment/bus/device/function tuples, with physical slots in
>  	  the system.  If you are unsure, say N.
>  
> +config ACPI_PCI_HOST_GENERIC
> +	bool
> +	select PCI_GENERIC_ECAM
> +	help
> +	  Select this config option from the architecture Kconfig,
> +	  if it is preferred to enable ACPI PCI host controller driver which
> +	  has no arch-specific assumptions.
> +
>  config X86_PM_TIMER
>  	bool "Power Management Timer Support" if EXPERT
>  	depends on X86
> diff --git a/drivers/acpi/Makefile b/drivers/acpi/Makefile
> index 81e5cbc..b12fa64 100644
> --- a/drivers/acpi/Makefile
> +++ b/drivers/acpi/Makefile
> @@ -40,6 +40,7 @@ acpi-$(CONFIG_ARCH_MIGHT_HAVE_ACPI_PDC) += processor_pdc.o
>  acpi-y				+= ec.o
>  acpi-$(CONFIG_ACPI_DOCK)	+= dock.o
>  acpi-y				+= pci_root.o pci_link.o pci_irq.o
> +obj-$(CONFIG_ACPI_PCI_HOST_GENERIC)	+= pci_gen_host.o

Maybe "pci_root_generic.c" so it shows up next to and is obviously
related to pci_root.c?

>  acpi-y				+= acpi_lpss.o acpi_apd.o
>  acpi-y				+= acpi_platform.o
>  acpi-y				+= acpi_pnp.o
> diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c
> index c068c82..803a1d7 100644
> --- a/drivers/acpi/bus.c
> +++ b/drivers/acpi/bus.c
> @@ -1107,6 +1107,7 @@ static int __init acpi_init(void)
>  	}
>  
>  	pci_mmcfg_late_init();
> +	pci_mcfg_init();
>  	acpi_scan_init();
>  	acpi_ec_init();
>  	acpi_debugfs_init();
> diff --git a/drivers/acpi/pci_gen_host.c b/drivers/acpi/pci_gen_host.c
> new file mode 100644
> index 0000000..fd360b5
> --- /dev/null
> +++ b/drivers/acpi/pci_gen_host.c
> @@ -0,0 +1,231 @@
> +/*
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License, version 2, as
> + * published by the Free Software Foundation (the "GPL").
> + *
> + * This program is distributed in the hope that it will be useful, but
> + * WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * General Public License version 2 (GPLv2) for more details.
> + *
> + * You should have received a copy of the GNU General Public License
> + * version 2 (GPLv2) along with this source code.
> + */
> +#include <linux/kernel.h>
> +#include <linux/pci.h>
> +#include <linux/pci-acpi.h>
> +#include <linux/sfi_acpi.h>
> +#include <linux/slab.h>
> +
> +#include "../pci/ecam.h"
> +
> +#define PREFIX	"ACPI: "
> +
> +/* Structure to hold entries from the MCFG table */
> +struct mcfg_entry {
> +	struct list_head	list;
> +	phys_addr_t		addr;
> +	u16			segment;
> +	u8			bus_start;
> +	u8			bus_end;
> +};
> +
> +/* List to save mcfg entries */
> +static LIST_HEAD(pci_mcfg_list);
> +static DEFINE_MUTEX(pci_mcfg_lock);
> +
> +/* ACPI info for generic ACPI PCI controller */
> +struct acpi_pci_generic_root_info {
> +	struct acpi_pci_root_info	common;
> +	struct pci_config_window	*cfg;	/* config space mapping */
> +};
> +
> +/* Find the entry in mcfg list which contains range bus_start */
> +static struct mcfg_entry *pci_mcfg_lookup(u16 seg, u8 bus_start)
> +{
> +	struct mcfg_entry *e;
> +
> +	list_for_each_entry(e, &pci_mcfg_list, list) {
> +		if (e->segment == seg &&
> +		    e->bus_start <= bus_start && bus_start <= e->bus_end)
> +			return e;
> +	}
> +
> +	return NULL;
> +}

Can you put the MCFG parsing, caching, and searching in a different
file, e.g., drivers/acpi/pci_mcfg.c?

> +/*
> + * Lookup the bus range for the domain in MCFG, and set up config space
> + * mapping.
> + */
> +static int pci_acpi_setup_ecam_mapping(struct acpi_pci_root *root,
> +				       struct acpi_pci_generic_root_info *ri)
> +{
> +	u16 seg = root->segment;
> +	u8 bus_start = root->secondary.start;
> +	u8 bus_end = root->secondary.end;
> +	struct pci_config_window *cfg;
> +	struct mcfg_entry *e;
> +	phys_addr_t addr;
> +	int err = 0;
> +
> +	mutex_lock(&pci_mcfg_lock);

What does this lock protect?  The pci_mcfg_list should already be
initialized by the time we get there, and it should be immutable for
the life of the system.  In fact, I would prefer if we could just
search the static table itself whenever we need it rather than caching
it in our own list.  But I don't think we can easily do that because
acpi_table_parse() is __init.

> +	e = pci_mcfg_lookup(seg, bus_start);

I would argue that we should check for _CBA first, and fall back to
MCFG if _CBA doesn't exist.

> +	if (!e) {
> +		addr = acpi_pci_root_get_mcfg_addr(root->device->handle);

IMO, acpi_pci_root_get_mcfg_addr() is misnamed.  It should be
acpi_pci_config_base_addr() or similar.  It definitely is not related
to MCFG.  Not your fault, obviously.

> +		if (addr == 0) {
> +			pr_err(PREFIX"%04x:%02x-%02x bus range error\n",
> +			       seg, bus_start, bus_end);
> +			err = -ENOENT;
> +			goto err_out;
> +		}
> +	} else {
> +		if (bus_start != e->bus_start) {
> +			pr_err("%04x:%02x-%02x bus range mismatch %02x\n",
> +			       seg, bus_start, bus_end, e->bus_start);
> +			err = -EINVAL;
> +			goto err_out;
> +		} else if (bus_end != e->bus_end) {
> +			pr_warn("%04x:%02x-%02x bus end mismatch %02x\n",
> +				seg, bus_start, bus_end, e->bus_end);
> +			bus_end = min(bus_end, e->bus_end);
> +		}
> +		addr = e->addr;
> +	}

I really don't think you need a lock around this, so you can factor
out the address lookup into something like:

  addr = acpi_pci_config_base_addr(...);
  if (addr)
    return addr;

  return acpi_pci_mcfg_lookup(seg, busn_res);

You can check inside acpi_pci_mcfg_lookup() to make sure the entry you
find covers the entire [busn_res.start-busn_res.end] range and return
failure if it doesn't.  At this point, I'm not sure it's worth it to
truncate the host bridge bus range to match something we find in MCFG.

If the MCFG entry covers *more* than the host bridge range from _CRS,
that's fine.  In any case, we have to be careful with the start address,
because the MCFG start address is always based on bus 0, but I think
pci_generic_ecam_create() expects the start address based on the
bus_start you pass to it.

> +
> +	cfg = pci_generic_ecam_create(&root->device->dev, addr, bus_start,
> +				      bus_end, &pci_generic_ecam_default_ops);
> +	if (IS_ERR(cfg)) {
> +		err = PTR_ERR(cfg);
> +		pr_err("%04x:%02x-%02x error %d mapping CAM\n", seg,
> +			bus_start, bus_end, err);
> +		goto err_out;
> +	}
> +
> +	cfg->domain = seg;
> +	ri->cfg = cfg;
> +err_out:
> +	mutex_unlock(&pci_mcfg_lock);
> +	return err;
> +}
> +
> +/* release_info: free resrouces allocated by init_info */
> +static void pci_acpi_generic_release_info(struct acpi_pci_root_info *ci)
> +{
> +	struct acpi_pci_generic_root_info *ri;
> +
> +	ri = container_of(ci, struct acpi_pci_generic_root_info, common);
> +	pci_generic_ecam_free(ri->cfg);
> +	kfree(ri);
> +}
> +
> +static struct acpi_pci_root_ops acpi_pci_root_ops = {
> +	.release_info = pci_acpi_generic_release_info,
> +};
> +
> +/* Interface called from ACPI code to setup PCI host controller */
> +struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root)
> +{
> +	int node = acpi_get_node(root->device->handle);
> +	struct acpi_pci_generic_root_info *ri;
> +	struct pci_bus *bus, *child;
> +	int err;
> +
> +	ri = kzalloc_node(sizeof(*ri), GFP_KERNEL, node);
> +	if (!ri)
> +		return NULL;
> +
> +	err = pci_acpi_setup_ecam_mapping(root, ri);
> +	if (err)
> +		return NULL;
> +
> +	acpi_pci_root_ops.pci_ops = &ri->cfg->ops->pci_ops;
> +	bus = acpi_pci_root_create(root, &acpi_pci_root_ops, &ri->common,
> +				   ri->cfg);
> +	if (!bus)
> +		return NULL;
> +
> +	pci_bus_size_bridges(bus);
> +	pci_bus_assign_resources(bus);
> +
> +	list_for_each_entry(child, &bus->children, node)
> +		pcie_bus_configure_settings(child);
> +
> +	return bus;
> +}
> +
> +/* handle MCFG table entries */
> +static __init int pci_mcfg_parse(struct acpi_table_header *header)
> +{
> +	struct acpi_table_mcfg *mcfg;
> +	struct acpi_mcfg_allocation *mptr;
> +	struct mcfg_entry *e, *arr;
> +	int i, n;
> +
> +	if (!header)
> +		return -EINVAL;
> +
> +	mcfg = (struct acpi_table_mcfg *)header;
> +	mptr = (struct acpi_mcfg_allocation *) &mcfg[1];
> +	n = (header->length - sizeof(*mcfg)) / sizeof(*mptr);
> +	if (n <= 0 || n > 255) {
> +		pr_err(PREFIX " MCFG has incorrect entries (%d).\n", n);
> +		return -EINVAL;
> +	}
> +
> +	arr = kcalloc(n, sizeof(*arr), GFP_KERNEL);
> +	if (!arr)
> +		return -ENOMEM;
> +
> +	for (i = 0, e = arr; i < n; i++, mptr++, e++) {
> +		e->segment = mptr->pci_segment;
> +		e->addr =  mptr->address;
> +		e->bus_start = mptr->start_bus_number;
> +		e->bus_end = mptr->end_bus_number;
> +		list_add(&e->list, &pci_mcfg_list);
> +		pr_info(PREFIX
> +			"MCFG entry for domain %04x [bus %02x-%02x] (base %pa)\n",
> +			e->segment, e->bus_start, e->bus_end, &e->addr);

Ah, this is information similar to what I suggested we might print in
pci_generic_ecam_create().  Could go either way, but personally I
think I'd put it in pci_generic_ecam_create() instead, because then we
only print the info we're actually using.  And I think it'd be nice to
have the actual MMIO resource ("[mem 0x...-0x...]") instead of just
the base.

> +	}
> +
> +	return 0;
> +}
> +
> +/* Interface called by ACPI - parse and save MCFG table */
> +void __init pci_mcfg_init(void)
> +{
> +	int err = acpi_table_parse(ACPI_SIG_MCFG, pci_mcfg_parse);
> +	if (err)
> +		pr_err(PREFIX "Failed to parse MCFG (%d)\n", err);
> +	else if (list_empty(&pci_mcfg_list))
> +		pr_info(PREFIX "No valid entries in MCFG table.\n");
> +	else {
> +		struct mcfg_entry *e;
> +		int i = 0;
> +		list_for_each_entry(e, &pci_mcfg_list, list)
> +			i++;
> +		pr_info(PREFIX "MCFG table loaded, %d entries\n", i);
> +	}
> +}
> +
> +/* Raw operations, works only for MCFG entries with an associated bus */

Can you elaborate a little on the connection with MCFG?  I don't quite
see how they're related.  Obviously the device has to be on a bus
we've already enumerated and assigned bus->ops for, but it seems like
we don't really know or care what bus->ops actually is.  Given that
these are in this file, acpi_pci_root_ops is the only possibility,
since that's what we pass to acpi_pci_root_create(), but it doesn't
seem worth mentioning specifically in a comment.

> +int raw_pci_read(unsigned int domain, unsigned int busn, unsigned int devfn,
> +		 int reg, int len, u32 *val)
> +{
> +	struct pci_bus *bus = pci_find_bus(domain, busn);
> +
> +	if (!bus)
> +		return PCIBIOS_DEVICE_NOT_FOUND;
> +	return bus->ops->read(bus, devfn, reg, len, val);
> +}
> +
> +int raw_pci_write(unsigned int domain, unsigned int busn, unsigned int devfn,
> +		  int reg, int len, u32 val)
> +{
> +	struct pci_bus *bus = pci_find_bus(domain, busn);
> +
> +	if (!bus)
> +		return PCIBIOS_DEVICE_NOT_FOUND;
> +	return bus->ops->write(bus, devfn, reg, len, val);
> +}
> diff --git a/include/linux/pci.h b/include/linux/pci.h
> index df1f33d..c0422ea 100644
> --- a/include/linux/pci.h
> +++ b/include/linux/pci.h
> @@ -1729,6 +1729,12 @@ static inline void pci_mmcfg_early_init(void) { }
>  static inline void pci_mmcfg_late_init(void) { }
>  #endif
>  
> +#ifdef CONFIG_ACPI_PCI_HOST_GENERIC
> +void __init pci_mcfg_init(void);
> +#else
> +static inline void pci_mcfg_init(void) { return; }
> +#endif
> +
>  int pci_ext_cfg_avail(void);
>  
>  void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar);
> -- 
> 1.9.1
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-pci" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html