Re: [PATCH v5 2/4] PCI: Add support for Enhanced Allocation devices

[Bjorn Helgaas <helgaas@kernel.org>]

On Tue, Oct 06, 2015 at 04:50:36PM -0700, David Daney wrote:
> From: "Sean O. Stalley" <sean.stalley@intel.com>
> 
> Add support for devices using Enhanced Allocation entries instead of BARs.
> This patch allows the kernel to parse the EA Extended Capability structure
> in PCI configspace and claim the BAR-equivalent resources.
> 
> Signed-off-by: Sean O. Stalley <sean.stalley@intel.com>
> [david.daney@cavium.com: Add more support/checking for Entry Properties,
> allow EA behind bridges, rewrite some error messages.]
> Signed-off-by: David Daney <david.daney@cavium.com>
> ---
>  drivers/pci/pci.c   | 182 ++++++++++++++++++++++++++++++++++++++++++++++++++++
>  drivers/pci/pci.h   |   1 +
>  drivers/pci/probe.c |   3 +
>  3 files changed, 186 insertions(+)
> 
> diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
> index 6a9a111..30a90d1 100644
> --- a/drivers/pci/pci.c
> +++ b/drivers/pci/pci.c
> @@ -2148,6 +2148,188 @@ void pci_pm_init(struct pci_dev *dev)
>  	}
>  }
>  
> +static unsigned long pci_ea_set_flags(struct pci_dev *dev, u8 prop)
> +{
> +	unsigned long flags = IORESOURCE_PCI_FIXED;
> +
> +	switch (prop) {
> +	case PCI_EA_P_MEM:
> +	case PCI_EA_P_VIRT_MEM:
> +		flags |= IORESOURCE_MEM;
> +		break;
> +	case PCI_EA_P_MEM_PREFETCH:
> +	case PCI_EA_P_VIRT_MEM_PREFETCH:
> +		flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH;
> +		break;
> +	case PCI_EA_P_IO:
> +		flags |= IORESOURCE_IO;
> +		break;
> +	default:
> +		return 0;
> +	}
> +
> +	return flags;
> +}
> +
> +static struct resource *pci_ea_get_resource(struct pci_dev *dev, u8 bei,
> +					    u8 prop)
> +{
> +	if (bei <= PCI_EA_BEI_BAR5 && prop <= PCI_EA_P_IO)
> +		return &dev->resource[bei];
> +	else if (bei == PCI_EA_BEI_ROM)
> +		return &dev->resource[PCI_ROM_RESOURCE];
> +	else
> +		return NULL;
> +}
> +
> +/* Read an Enhanced Allocation (EA) entry */
> +static int pci_ea_read(struct pci_dev *dev, int offset)
> +{
> +	struct resource *res;
> +	int ent_offset = offset;
> +	int ent_size;
> +	resource_size_t start;
> +	resource_size_t end;
> +	unsigned long flags;
> +	u32 dw0;
> +	u32 base;
> +	u32 max_offset;
> +	u8 prop;
> +	bool support_64 = (sizeof(resource_size_t) >= 8);
> +
> +	pci_read_config_dword(dev, ent_offset, &dw0);
> +	ent_offset += 4;
> +
> +	/* Entry size field indicates DWORDs after 1st */
> +	ent_size = ((dw0 & PCI_EA_ES) + 1) << 2;
> +
> +	if (!(dw0 & PCI_EA_ENABLE)) /* Entry not enabled */
> +		goto out;
> +
> +	prop = PCI_EA_PP(dw0);
> +	/*
> +	 * If the Property is in the reserved range, try the Secondary
> +	 * Property instead.
> +	 */
> +	if (prop > PCI_EA_P_BRIDGE_IO && prop < PCI_EA_P_MEM_RESERVED)
> +		prop = PCI_EA_SP(dw0);
> +	if (prop > PCI_EA_P_BRIDGE_IO)
> +		goto out;
> +
> +	res = pci_ea_get_resource(dev, PCI_EA_BEI(dw0), prop);
> +	if (!res) {
> +		dev_err(&dev->dev, "Unsupported EA entry BEI: %u\n",
> +			PCI_EA_BEI(dw0));
> +		goto out;
> +	}
> +
> +	flags = pci_ea_set_flags(dev, prop);
> +	if (!flags) {
> +		dev_err(&dev->dev, "Unsupported EA properties: %u\n", prop);
> +		goto out;
> +	}
> +
> +	/* Read Base */
> +	pci_read_config_dword(dev, ent_offset, &base);
> +	start = (base & PCI_EA_FIELD_MASK);
> +	ent_offset += 4;
> +
> +	/* Read MaxOffset */
> +	pci_read_config_dword(dev, ent_offset, &max_offset);
> +	ent_offset += 4;
> +
> +	/* Read Base MSBs (if 64-bit entry) */
> +	if (base & PCI_EA_IS_64) {
> +		u32 base_upper;
> +
> +		pci_read_config_dword(dev, ent_offset, &base_upper);
> +		ent_offset += 4;
> +
> +		flags |= IORESOURCE_MEM_64;
> +
> +		/* entry starts above 32-bit boundary, can't use */
> +		if (!support_64 && base_upper)
> +			goto out;
> +
> +		if (support_64)
> +			start |= ((u64)base_upper << 32);
> +	}
> +
> +	dev_dbg(&dev->dev,
> +		"EA (%u,%u) start = %pa\n", PCI_EA_BEI(dw0), prop, &start);
> +
> +	end = start + (max_offset | 0x03);
> +
> +	/* Read MaxOffset MSBs (if 64-bit entry) */
> +	if (max_offset & PCI_EA_IS_64) {
> +		u32 max_offset_upper;
> +
> +		pci_read_config_dword(dev, ent_offset, &max_offset_upper);
> +		ent_offset += 4;
> +
> +		flags |= IORESOURCE_MEM_64;
> +
> +		/* entry too big, can't use */
> +		if (!support_64 && max_offset_upper)
> +			goto out;
> +
> +		if (support_64)
> +			end += ((u64)max_offset_upper << 32);
> +	}
> +
> +	dev_dbg(&dev->dev,
> +		"EA (%u,%u) end = %pa\n", PCI_EA_BEI(dw0), prop, &end);
> +
> +	if (end < start) {
> +		dev_err(&dev->dev, "EA Entry crosses address boundary\n");
> +		goto out;
> +	}
> +
> +	if (ent_size != ent_offset - offset) {
> +		dev_err(&dev->dev,
> +			"EA Entry Size (%d) does not match length read (%d)\n",
> +			ent_size, ent_offset - offset);
> +		goto out;
> +	}
> +
> +	res->name = pci_name(dev);
> +	res->start = start;
> +	res->end = end;
> +	res->flags = flags;

This is similar to reading a BAR from a normal device; can you print what
we found so it looks similar in dmesg, e.g., similar to what
__pci_read_base() does?  Note that "dev_dbg" is not equivalent to
dev_printk(KERN_DEBUG).  I want the output in dmesg all the time, without
having to enable something via dyndbg.

> +
> +out:
> +	return offset + ent_size;
> +}
> +
> +/* Enhanced Allocation Initalization */
> +void pci_ea_init(struct pci_dev *dev)
> +{
> +	int ea;
> +	u8 num_ent;
> +	int offset;
> +	int i;
> +
> +	/* find PCI EA capability in list */
> +	ea = pci_find_capability(dev, PCI_CAP_ID_EA);
> +	if (!ea)
> +		return;
> +
> +	/* determine the number of entries */
> +	pci_bus_read_config_byte(dev->bus, dev->devfn, ea + PCI_EA_NUM_ENT,
> +					&num_ent);
> +	num_ent &= PCI_EA_NUM_ENT_MASK;
> +
> +	offset = ea + PCI_EA_FIRST_ENT;
> +
> +	/* Skip DWORD 2 for type 1 functions */
> +	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE)
> +		offset += 4;
> +
> +	/* parse each EA entry */
> +	for (i = 0; i < num_ent; ++i)
> +		offset = pci_ea_read(dev, offset);
> +}
> +
>  static void pci_add_saved_cap(struct pci_dev *pci_dev,
>  	struct pci_cap_saved_state *new_cap)
>  {
> diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
> index 24ba9dc..a160733 100644
> --- a/drivers/pci/pci.h
> +++ b/drivers/pci/pci.h
> @@ -78,6 +78,7 @@ bool pci_dev_keep_suspended(struct pci_dev *dev);
>  void pci_config_pm_runtime_get(struct pci_dev *dev);
>  void pci_config_pm_runtime_put(struct pci_dev *dev);
>  void pci_pm_init(struct pci_dev *dev);
> +void pci_ea_init(struct pci_dev *dev);
>  void pci_allocate_cap_save_buffers(struct pci_dev *dev);
>  void pci_free_cap_save_buffers(struct pci_dev *dev);
>  
> diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
> index 8361d27..4c4af78 100644
> --- a/drivers/pci/probe.c
> +++ b/drivers/pci/probe.c
> @@ -1597,6 +1597,9 @@ static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn)
>  
>  static void pci_init_capabilities(struct pci_dev *dev)
>  {
> +	/* Enhanced Allocation */
> +	pci_ea_init(dev);
> +
>  	/* MSI/MSI-X list */
>  	pci_msi_init_pci_dev(dev);
>  
> -- 
> 1.9.1
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-pci" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html