[RFC PATCH 0/4] Enable new VMD device

[RFC PATCH 0/4] Enable new VMD device

[Jon Derrick]

This patchset enables a new VMD device with the existing VMD driver. The
new device offers a few additional features such as direct-assign
capabilities.

Jon Derrick (4):
  PCI/VMD: Assign membar addresses from shadow registers
  PCI/VMD: Add offset to bus numbers if necessary
  x86/PCI: Add new VMD device root ports to VMD AER quirk
  PCI/VMD: Add new VMD device id to driver device id table

 arch/x86/pci/fixup.c   |  4 +++
 drivers/pci/host/vmd.c | 79 +++++++++++++++++++++++++++++++++++++++++---------
 2 files changed, 70 insertions(+), 13 deletions(-)

-- 
2.14.3

[RFC PATCH 1/4] PCI/VMD: Assign membar addresses from shadow registers

[Jon Derrick]

The new VMD device has registers within membar 2 which may shadow the
membar 1 and membar 2 addresses. These are intended to be used in
virtualization, where assigning a guest address wouldn't be translated
in the assignment to root port and child devices because the addresses
exist within the assignment message.

These values will only reflect the membars when enabled in the BIOS, as
determined by a register in the VMD device.

Signed-off-by: Jon Derrick <jonathan.derrick@intel.com>
---
 drivers/pci/host/vmd.c | 52 +++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 43 insertions(+), 9 deletions(-)

diff --git a/drivers/pci/host/vmd.c b/drivers/pci/host/vmd.c
index 930a8fa08bd6..62270aeb7a2e 100644
--- a/drivers/pci/host/vmd.c
+++ b/drivers/pci/host/vmd.c
@@ -24,6 +24,9 @@
 #define VMD_MEMBAR1	2
 #define VMD_MEMBAR2	4
 
+#define PCI_REG_VMLOCK		0x70
+#define MB2_SHADOW_EN(vmlock)	(vmlock & 0x2)
+
 /*
  * Lock for manipulating VMD IRQ lists.
  */
@@ -551,9 +554,9 @@ static int vmd_enable_domain(struct vmd_dev *vmd)
 	struct pci_sysdata *sd = &vmd->sysdata;
 	struct fwnode_handle *fn;
 	struct resource *res;
-	u32 upper_bits;
 	unsigned long flags;
 	LIST_HEAD(resources);
+	resource_size_t start, end, membar2_offset;
 
 	res = &vmd->dev->resource[VMD_CFGBAR];
 	vmd->resources[0] = (struct resource) {
@@ -581,27 +584,34 @@ static int vmd_enable_domain(struct vmd_dev *vmd)
 	 * according to the platform needs.
 	 */
 	res = &vmd->dev->resource[VMD_MEMBAR1];
-	upper_bits = upper_32_bits(res->end);
+	start = vmd->resources[1].start;
+	end = start + resource_size(res) - 1;
 	flags = res->flags & ~IORESOURCE_SIZEALIGN;
-	if (!upper_bits)
+	if (upper_32_bits(end))
+		flags |= IORESOURCE_MEM_64;
+	else
 		flags &= ~IORESOURCE_MEM_64;
 	vmd->resources[1] = (struct resource) {
 		.name  = "VMD MEMBAR1",
-		.start = res->start,
-		.end   = res->end,
+		.start = start,
+		.end   = end,
 		.flags = flags,
 		.parent = res,
 	};
 
 	res = &vmd->dev->resource[VMD_MEMBAR2];
-	upper_bits = upper_32_bits(res->end);
+	start = vmd->resources[2].start;
+	end = start + resource_size(res) - 1;
 	flags = res->flags & ~IORESOURCE_SIZEALIGN;
-	if (!upper_bits)
+	if (upper_32_bits(end))
+		flags |= IORESOURCE_MEM_64;
+	else
 		flags &= ~IORESOURCE_MEM_64;
+	membar2_offset = (vmd->dev->device == 0x28c0) ? 0x2018 : 0x2000;
 	vmd->resources[2] = (struct resource) {
 		.name  = "VMD MEMBAR2",
-		.start = res->start + 0x2000,
-		.end   = res->end,
+		.start = start + membar2_offset,
+		.end   = end,
 		.flags = flags,
 		.parent = res,
 	};
@@ -662,6 +672,7 @@ static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id)
 {
 	struct vmd_dev *vmd;
 	int i, err;
+	u32 vmlock = 0;
 
 	if (resource_size(&dev->resource[VMD_CFGBAR]) < (1 << 20))
 		return -ENOMEM;
@@ -711,6 +722,29 @@ static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id)
 			return err;
 	}
 
+	/*
+	 * For the new VMD device id, shadow registers exist within the MEMBAR2
+	 * which allow guests to correctly assign host physical addresses to
+	 * the root ports and child devices. These registers will either return
+	 * the host value or 0, depending on an enable bit in the VMD device.
+	 */
+	if (id->device == 0x28c0)
+		pci_read_config_dword(dev, PCI_REG_VMLOCK, &vmlock);
+
+	if (MB2_SHADOW_EN(vmlock)) {
+		void __iomem *membar2;
+
+		membar2 = pci_iomap(dev, VMD_MEMBAR2, 0);
+		if (!membar2)
+			return -ENOMEM;
+		vmd->resources[1].start = readq(membar2 + 0x2008);
+		vmd->resources[2].start = readq(membar2 + 0x2010);
+		pci_iounmap(dev, membar2);
+	} else {
+		vmd->resources[1].start = vmd->dev->resource[VMD_MEMBAR1].start;
+		vmd->resources[2].start = vmd->dev->resource[VMD_MEMBAR2].start;
+	}
+
 	spin_lock_init(&vmd->cfg_lock);
 	pci_set_drvdata(dev, vmd);
 	err = vmd_enable_domain(vmd);
-- 
2.14.3

Re: [RFC PATCH 1/4] PCI/VMD: Assign membar addresses from shadow registers

[Christoph Hellwig]

> +	 * For the new VMD device id, shadow registers exist within the MEMBAR2

It won't be new rather soon.  Better us a more descriptive name.

[RFC PATCH 2/4] PCI/VMD: Add offset to bus numbers if necessary

[Jon Derrick]

Depending on platform configuration, the new VMD device's bus numbers
may require being offset by 128. We determine this requirement by
checking the value of two vendor specific capability registers in the
VMD endpoint:

 VMCAP[0] | VMCONFIG[9:8] | Bus Numbers
----------------------------------------
    0     |       *       |     0-255
    1     |      00       |     0-127
    1     |      01       |   128-255
    1     |      10       |     0-255

Signed-off-by: Jon Derrick <jonathan.derrick@intel.com>
---
 drivers/pci/host/vmd.c | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/drivers/pci/host/vmd.c b/drivers/pci/host/vmd.c
index 62270aeb7a2e..fdfc286a1c9e 100644
--- a/drivers/pci/host/vmd.c
+++ b/drivers/pci/host/vmd.c
@@ -24,6 +24,10 @@
 #define VMD_MEMBAR1	2
 #define VMD_MEMBAR2	4
 
+#define PCI_REG_VMCAP		0x40
+#define BUS_RESTRICT_CAP(vmcap)	(vmcap & 0x1)
+#define PCI_REG_VMCONFIG	0x44
+#define BUS_RESTRICT_CFG(vmcfg)	((vmcfg >> 8) & 0x3)
 #define PCI_REG_VMLOCK		0x70
 #define MB2_SHADOW_EN(vmlock)	(vmlock & 0x2)
 
@@ -556,13 +560,27 @@ static int vmd_enable_domain(struct vmd_dev *vmd)
 	struct resource *res;
 	unsigned long flags;
 	LIST_HEAD(resources);
-	resource_size_t start, end, membar2_offset;
+	resource_size_t start, end, membar2_offset, busn_start = 0;
+
+	/*
+	 * Depending on the root port configuration, VMD may assign bus numbers
+	 * between 0-127 or 128-255
+	 */
+	if (vmd->dev->device == 0x28c0) {
+		u32 vmcap, vmconfig;
+
+		pci_read_config_dword(vmd->dev, PCI_REG_VMCAP, &vmcap);
+		pci_read_config_dword(vmd->dev, PCI_REG_VMCONFIG, &vmconfig);
+		if (BUS_RESTRICT_CAP(vmcap) &&
+		    (BUS_RESTRICT_CFG(vmconfig) == 0x1))
+			busn_start = 128;
+	}
 
 	res = &vmd->dev->resource[VMD_CFGBAR];
 	vmd->resources[0] = (struct resource) {
 		.name  = "VMD CFGBAR",
-		.start = 0,
-		.end   = (resource_size(res) >> 20) - 1,
+		.start = busn_start,
+		.end   = busn_start + (resource_size(res) >> 20) - 1,
 		.flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED,
 	};
 
@@ -636,8 +654,8 @@ static int vmd_enable_domain(struct vmd_dev *vmd)
 	pci_add_resource(&resources, &vmd->resources[0]);
 	pci_add_resource(&resources, &vmd->resources[1]);
 	pci_add_resource(&resources, &vmd->resources[2]);
-	vmd->bus = pci_create_root_bus(&vmd->dev->dev, 0, &vmd_ops, sd,
-				       &resources);
+	vmd->bus = pci_create_root_bus(&vmd->dev->dev, busn_start, &vmd_ops,
+				       sd, &resources);
 	if (!vmd->bus) {
 		pci_free_resource_list(&resources);
 		irq_domain_remove(vmd->irq_domain);
-- 
2.14.3

[RFC PATCH 3/4] x86/PCI: Add new VMD device root ports to VMD AER quirk

[Jon Derrick]

VMD devices change the source id of messages from child devices to the
VMD endpoint. This patch adds the new VMD device id to the AER quirk
which requires walking the bus to determine which devices were throwing
the error.

Signed-off-by: Jon Derrick <jonathan.derrick@intel.com>
---
 arch/x86/pci/fixup.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/arch/x86/pci/fixup.c b/arch/x86/pci/fixup.c
index 54ef19e90705..13f4485ca388 100644
--- a/arch/x86/pci/fixup.c
+++ b/arch/x86/pci/fixup.c
@@ -636,6 +636,10 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2030, quirk_no_aersid);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2031, quirk_no_aersid);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2032, quirk_no_aersid);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2033, quirk_no_aersid);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x334a, quirk_no_aersid);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x334b, quirk_no_aersid);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x334c, quirk_no_aersid);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x334d, quirk_no_aersid);
 
 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 
-- 
2.14.3

[RFC PATCH 4/4] PCI/VMD: Add new VMD device id to driver device id table

[Jon Derrick]

Allow new VMD hardware to bind to VMD driver.

Signed-off-by: Jon Derrick <jonathan.derrick@intel.com>
---
 drivers/pci/host/vmd.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/drivers/pci/host/vmd.c b/drivers/pci/host/vmd.c
index fdfc286a1c9e..add940eedc56 100644
--- a/drivers/pci/host/vmd.c
+++ b/drivers/pci/host/vmd.c
@@ -831,6 +831,7 @@ static SIMPLE_DEV_PM_OPS(vmd_dev_pm_ops, vmd_suspend, vmd_resume);
 
 static const struct pci_device_id vmd_ids[] = {
 	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x201d),},
+	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x28c0),},
 	{0,}
 };
 MODULE_DEVICE_TABLE(pci, vmd_ids);
-- 
2.14.3