[PATCH 1/2][VTD] pci mmconfig support to be used for ATS

[PATCH 1/2][VTD] pci mmconfig support to be used for ATS

[Kay, Allen M]

[-- Attachment #1: Type: text/plain, Size: 410 bytes --]

ATS (Address Translation Service) is a PCI specification used for synchronizing iotlb translations between chipset iommu such as vt-d with iotlb on board a PCI device.

This patch enables PCI mmconfig for Intel64 systems.  Most of the code were copied from Linux.  This functionality is need for parsing ATS capability in PCIe extended configuration space.

Signed-off-by: Allen Kay <allen.m.kay@intel.com>

[-- Attachment #2: mmconfig_1119.patch --]
[-- Type: application/octet-stream, Size: 22166 bytes --]

diff -r 98d48f7680db xen/arch/x86/acpi/boot.c
--- a/xen/arch/x86/acpi/boot.c	Wed Nov 19 19:13:22 2008 +0000
+++ b/xen/arch/x86/acpi/boot.c	Wed Nov 19 04:04:50 2008 -0800
@@ -986,6 +986,8 @@ int __init acpi_boot_init(void)
 
 	acpi_dmar_init();
 
+	acpi_mmcfg_init();
+
 	return 0;
 }
 
diff -r 98d48f7680db xen/drivers/passthrough/vtd/x86/Makefile
--- a/xen/drivers/passthrough/vtd/x86/Makefile	Wed Nov 19 19:13:22 2008 +0000
+++ b/xen/drivers/passthrough/vtd/x86/Makefile	Wed Nov 19 04:24:55 2008 -0800
@@ -1,1 +1,4 @@ obj-y += vtd.o
 obj-y += vtd.o
+obj-y += acpi_mmcfg.o
+obj-y += mmconfig-shared.o
+obj-y += mmconfig_64.o
diff -r 98d48f7680db xen/include/asm-x86/acpi.h
--- a/xen/include/asm-x86/acpi.h	Wed Nov 19 19:13:22 2008 +0000
+++ b/xen/include/asm-x86/acpi.h	Wed Nov 19 04:04:50 2008 -0800
@@ -164,7 +164,8 @@ extern u8 x86_acpiid_to_apicid[];
 extern u8 x86_acpiid_to_apicid[];
 #define MAX_LOCAL_APIC 256
 
-extern int acpi_dmar_init(void);
+int acpi_dmar_init(void);
+void acpi_mmcfg_init(void);
 
 /* Incremented whenever we transition through S3. Value is 1 during boot. */
 extern uint32_t system_reset_counter;
diff -r 98d48f7680db xen/include/asm-x86/fixmap.h
--- a/xen/include/asm-x86/fixmap.h	Wed Nov 19 19:13:22 2008 +0000
+++ b/xen/include/asm-x86/fixmap.h	Wed Nov 19 04:04:50 2008 -0800
@@ -20,6 +20,7 @@
 #include <xen/iommu.h>
 #include <asm/amd-iommu.h>
 #include <asm/msi.h>
+#include <xen/pci.h>
 
 /*
  * Here we define all the compile-time 'special' virtual
@@ -51,6 +52,8 @@ enum fixed_addresses {
     FIX_TBOOT_SHARED_BASE,
     FIX_MSIX_IO_RESERV_BASE,
     FIX_MSIX_IO_RESERV_END = FIX_MSIX_IO_RESERV_BASE + MAX_MSIX_PAGES -1,
+    FIX_PCI_MMCFG_BASE,
+    FIX_PCI_MMCFG_END = FIX_PCI_MMCFG_BASE + MAX_SEGMENTS * PER_SEGMENT_ADDR_SPACE,
     __end_of_fixed_addresses
 };
 
diff -r 98d48f7680db xen/include/xen/pci.h
--- a/xen/include/xen/pci.h	Wed Nov 19 19:13:22 2008 +0000
+++ b/xen/include/xen/pci.h	Wed Nov 19 04:04:50 2008 -0800
@@ -12,6 +12,14 @@
 #include <xen/list.h>
 #include <xen/spinlock.h>
 
+#if defined(__x86_64__)
+#define MAX_SEGMENTS    4    /* support 4 pci segments for now */
+#else
+#define MAX_SEGMENTS    0
+#endif
+
+#define PER_SEGMENT_ADDR_SPACE     ((256 << 20) >> PAGE_SHIFT)
+ 
 /*
  * The PCI interface treats multi-function devices as independent
  * devices.  The slot/function address of each device is encoded
@@ -30,12 +38,14 @@
 #define PCI_BDF2(b,df)  ((((b) & 0xff) << 8) | ((df) & 0xff))
 
 struct pci_dev {
+    struct list_head list;          /* non-shared generic list head */
     struct list_head alldevs_list;
     struct list_head domain_list;
     struct list_head msi_list;
     struct domain *domain;
-    const u8 bus;
-    const u8 devfn;
+    u8 bus;
+    u8 devfn;
+    u16 ats_queue_depth;    /* ATS device invalidation queue depth */
     spinlock_t lock;
 };
 
diff -r 98d48f7680db xen/drivers/passthrough/vtd/x86/acpi_mmcfg.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/xen/drivers/passthrough/vtd/x86/acpi_mmcfg.c	Wed Nov 19 04:04:50 2008 -0800
@@ -0,0 +1,103 @@
+/*
+ *  acpi_mmconfig.c - Architecture-Specific Low-Level ACPI Boot Support
+ *
+ *  Copyright (C) 2001, 2002 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com>
+ *  Copyright (C) 2001 Jun Nakajima <jun.nakajima@intel.com>
+ *
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ *
+ * copied from Linux
+ */
+
+#include <xen/config.h>
+#include <xen/errno.h>
+#include <xen/init.h>
+#include <xen/acpi.h>
+#include <xen/irq.h>
+#include <xen/dmi.h>
+#include <asm/fixmap.h>
+#include <asm/page.h>
+#include <asm/apic.h>
+#include <asm/io_apic.h>
+#include <asm/apic.h>
+#include <asm/io.h>
+#include <asm/mpspec.h>
+#include <asm/processor.h>
+#include <mach_apic.h>
+#include <mach_mpparse.h>
+
+#include "pci.h"
+
+/* The physical address of the MMCONFIG aperture.  Set from ACPI tables. */
+struct acpi_mcfg_allocation *pci_mmcfg_config;
+int pci_mmcfg_config_num;
+
+static int acpi_mcfg_64bit_base_addr __initdata = FALSE;
+
+int acpi_parse_mcfg(struct acpi_table_header *header)
+{
+	struct acpi_table_mcfg *mcfg;
+	unsigned long i;
+	int config_size;
+
+#if !defined(__x86_64__)
+        /* only supporting x86_64 for now */ 
+	return -EINVAL;
+#endif
+
+	if (!header)
+		return -EINVAL;
+
+	mcfg = (struct acpi_table_mcfg *)header;
+
+	/* how many config structures do we have */
+	pci_mmcfg_config_num = 0;
+	i = header->length - sizeof(struct acpi_table_mcfg);
+	while (i >= sizeof(struct acpi_mcfg_allocation)) {
+		++pci_mmcfg_config_num;
+		i -= sizeof(struct acpi_mcfg_allocation);
+	};
+	if (pci_mmcfg_config_num == 0) {
+		printk(KERN_ERR PREFIX "MMCONFIG has no entries\n");
+		return -ENODEV;
+	}
+
+	config_size = pci_mmcfg_config_num * sizeof(*pci_mmcfg_config);
+	pci_mmcfg_config = xmalloc_bytes(config_size);
+	if (!pci_mmcfg_config) {
+		printk(KERN_WARNING PREFIX
+		       "No memory for MCFG config tables\n");
+		return -ENOMEM;
+	}
+
+	memcpy(pci_mmcfg_config, &mcfg[1], config_size);
+
+	for (i = 0; i < pci_mmcfg_config_num; ++i) {
+		if ((pci_mmcfg_config[i].address > 0xFFFFFFFF) &&
+		    !acpi_mcfg_64bit_base_addr) {
+			printk(KERN_ERR PREFIX
+			       "MMCONFIG not in low 4GB of memory\n");
+			xfree(pci_mmcfg_config);
+			pci_mmcfg_config_num = 0;
+			return -ENODEV;
+		}
+	}
+
+	return 0;
+}
diff -r 98d48f7680db xen/drivers/passthrough/vtd/x86/mmconfig-shared.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/xen/drivers/passthrough/vtd/x86/mmconfig-shared.c	Wed Nov 19 04:04:50 2008 -0800
@@ -0,0 +1,327 @@
+/*
+ * mmconfig-shared.c - Low-level direct PCI config space access via
+ *                     MMCONFIG - common code between i386 and x86-64.
+ *
+ * This code does:
+ * - known chipset handling
+ * - ACPI decoding and validation
+ *
+ * Per-architecture code takes care of the mappings and accesses
+ * themselves.
+ *
+ * Author: Allen Kay <allen.m.kay@intel.com> -  adapted to xen from Linux
+ */
+
+#include <xen/mm.h>
+#include <xen/acpi.h>
+#include <xen/xmalloc.h>
+#include <xen/pci.h>
+#include <xen/pci_regs.h>
+
+#include "pci.h"
+
+struct pci_raw_ops *raw_pci_ext_ops;
+struct pci_raw_ops *raw_pci_ops;
+
+/* aperture is up to 256MB but BIOS may reserve less */
+#define MMCONFIG_APER_MIN	(2 * 1024*1024)
+#define MMCONFIG_APER_MAX	(256 * 1024*1024)
+
+/* Indicate if the mmcfg resources have been placed into the resource table. */
+static int __initdata pci_mmcfg_resources_inserted;
+
+static const char __init *pci_mmcfg_e7520(void)
+{
+	u32 win;
+
+	raw_pci_ops->read(0, 0, PCI_DEVFN(0, 0), 0xce, 2, &win);
+
+	win = win & 0xf000;
+	if(win == 0x0000 || win == 0xf000)
+		pci_mmcfg_config_num = 0;
+	else {
+		pci_mmcfg_config_num = 1;
+		pci_mmcfg_config = xmalloc_bytes(sizeof(pci_mmcfg_config[0]));
+		if (!pci_mmcfg_config)
+			return NULL;
+		memset(pci_mmcfg_config, 0, sizeof(pci_mmcfg_config[0]));
+		pci_mmcfg_config[0].address = win << 16;
+		pci_mmcfg_config[0].pci_segment = 0;
+		pci_mmcfg_config[0].start_bus_number = 0;
+		pci_mmcfg_config[0].end_bus_number = 255;
+	}
+
+	return "Intel Corporation E7520 Memory Controller Hub";
+}
+
+static const char __init *pci_mmcfg_intel_945(void)
+{
+	u32 pciexbar, mask = 0, len = 0;
+
+	pci_mmcfg_config_num = 1;
+
+	raw_pci_ops->read(0, 0, PCI_DEVFN(0, 0), 0x48, 4, &pciexbar);
+
+	/* Enable bit */
+	if (!(pciexbar & 1))
+		pci_mmcfg_config_num = 0;
+
+	/* Size bits */
+	switch ((pciexbar >> 1) & 3) {
+	case 0:
+		mask = 0xf0000000U;
+		len  = 0x10000000U;
+		break;
+	case 1:
+		mask = 0xf8000000U;
+		len  = 0x08000000U;
+		break;
+	case 2:
+		mask = 0xfc000000U;
+		len  = 0x04000000U;
+		break;
+	default:
+		pci_mmcfg_config_num = 0;
+	}
+
+	/* Errata #2, things break when not aligned on a 256Mb boundary */
+	/* Can only happen in 64M/128M mode */
+
+	if ((pciexbar & mask) & 0x0fffffffU)
+		pci_mmcfg_config_num = 0;
+
+	/* Don't hit the APIC registers and their friends */
+	if ((pciexbar & mask) >= 0xf0000000U)
+		pci_mmcfg_config_num = 0;
+
+	if (pci_mmcfg_config_num) {
+		pci_mmcfg_config = xmalloc_bytes(sizeof(pci_mmcfg_config[0]));
+		if (!pci_mmcfg_config)
+			return NULL;
+		memset(pci_mmcfg_config, 0, sizeof(pci_mmcfg_config[0]));
+		pci_mmcfg_config[0].address = pciexbar & mask;
+		pci_mmcfg_config[0].pci_segment = 0;
+		pci_mmcfg_config[0].start_bus_number = 0;
+		pci_mmcfg_config[0].end_bus_number = (len >> 20) - 1;
+	}
+
+	return "Intel Corporation 945G/GZ/P/PL Express Memory Controller Hub";
+}
+
+struct pci_mmcfg_hostbridge_probe {
+	u32 bus;
+	u32 devfn;
+	u32 vendor;
+	u32 device;
+	const char *(*probe)(void);
+};
+
+static struct pci_mmcfg_hostbridge_probe pci_mmcfg_probes[] __initdata = {
+	{ 0, PCI_DEVFN(0, 0), PCI_VENDOR_ID_INTEL,
+	  PCI_DEVICE_ID_INTEL_E7520_MCH, pci_mmcfg_e7520 },
+	{ 0, PCI_DEVFN(0, 0), PCI_VENDOR_ID_INTEL,
+	  PCI_DEVICE_ID_INTEL_82945G_HB, pci_mmcfg_intel_945 },
+};
+
+static int __init pci_mmcfg_check_hostbridge(void)
+{
+	u32 l;
+	u32 bus, devfn;
+	u16 vendor, device;
+	int i;
+	const char *name;
+
+	if (!raw_pci_ops)
+		return 0;
+
+	pci_mmcfg_config_num = 0;
+	pci_mmcfg_config = NULL;
+	name = NULL;
+
+	for (i = 0; !name && i < ARRAY_SIZE(pci_mmcfg_probes); i++) {
+		bus =  pci_mmcfg_probes[i].bus;
+		devfn = pci_mmcfg_probes[i].devfn;
+		raw_pci_ops->read(0, bus, devfn, 0, 4, &l);
+		vendor = l & 0xffff;
+		device = (l >> 16) & 0xffff;
+
+		if (pci_mmcfg_probes[i].vendor == vendor &&
+		    pci_mmcfg_probes[i].device == device)
+			name = pci_mmcfg_probes[i].probe();
+	}
+
+	if (name) {
+		printk(KERN_INFO "PCI: Found %s %s MMCONFIG support.\n",
+		       name, pci_mmcfg_config_num ? "with" : "without");
+	}
+
+	return name != NULL;
+}
+
+static int __initdata known_bridge;
+
+static struct pci_raw_ops pci_iocfg = {
+	.read =		pci_conf1_read,
+	.write =	pci_conf1_write,
+};
+
+unsigned int pci_probe = PCI_PROBE_CONF1 | PCI_PROBE_MMCONF;
+
+static void __init __pci_mmcfg_init(int early)
+{
+	raw_pci_ops = &pci_iocfg;
+	/* MMCONFIG disabled */
+	if ((pci_probe & PCI_PROBE_MMCONF) == 0)
+		return;
+
+	/* MMCONFIG already enabled */
+	if (!early && !(pci_probe & PCI_PROBE_MASK & ~PCI_PROBE_MMCONF))
+		return;
+
+	/* for late to exit */
+	if (known_bridge)
+		return;
+
+	if (early) {
+		if (pci_mmcfg_check_hostbridge())
+			known_bridge = 1;
+	}
+
+	if (!known_bridge) {
+		acpi_table_parse(ACPI_SIG_MCFG, acpi_parse_mcfg);
+	}
+
+	if ((pci_mmcfg_config_num == 0) ||
+	    (pci_mmcfg_config == NULL) ||
+	    (pci_mmcfg_config[0].address == 0))
+		return;
+
+	if (pci_mmcfg_arch_init()) {
+		pci_probe = (pci_probe & ~PCI_PROBE_MASK) | PCI_PROBE_MMCONF;
+	} else {
+		/*
+		 * Signal not to attempt to insert mmcfg resources because
+		 * the architecture mmcfg setup could not initialize.
+		 */
+		pci_mmcfg_resources_inserted = 1;
+	}
+}
+
+void __init pci_mmcfg_early_init(void)
+{
+	__pci_mmcfg_init(1);
+}
+
+void __init pci_mmcfg_late_init(void)
+{
+	__pci_mmcfg_init(0);
+}
+
+void acpi_mmcfg_init(void)
+{
+	__pci_mmcfg_init(1);
+}
+
+void __iomem * ioremap_nocache(unsigned long offset, unsigned long size)
+{
+	void __iomem *addr = NULL;
+
+	BUG_ON((FIX_PCI_MMCFG_BASE + (size >> PAGE_SHIFT)) >= 
+		__end_of_fixed_addresses);
+
+	map_pages_to_xen(fix_to_virt(FIX_PCI_MMCFG_END),
+			(offset >> PAGE_SHIFT),
+			(size >> PAGE_SHIFT),
+			PAGE_HYPERVISOR_NOCACHE);
+	addr = (void __iomem *) fix_to_virt(FIX_PCI_MMCFG_END);
+
+	return(addr);
+}
+
+int pci_conf1_read(unsigned int seg, unsigned int bus,
+		unsigned int devfn, int reg, int len, u32 *value)
+{
+	switch (len) {
+	case 1:
+		*value = pci_conf_read8(bus, PCI_SLOT(devfn),
+					PCI_FUNC(devfn), reg);
+		break;
+	case 2:
+		*value = pci_conf_read16(bus, PCI_SLOT(devfn),
+					PCI_FUNC(devfn), reg);
+		break;
+	case 4:
+		*value = pci_conf_read32(bus, PCI_SLOT(devfn),
+					PCI_FUNC(devfn), reg);
+		break;
+	}
+	return 0;
+}
+
+int pci_conf1_write(unsigned int seg, unsigned int bus,
+		unsigned int devfn, int reg, int len, u32 value)
+{
+	switch (len) {
+	case 1:
+		pci_conf_write8(bus, PCI_SLOT(devfn),
+				PCI_FUNC(devfn), reg, value);
+		break;
+	case 2:
+		pci_conf_write16(bus, PCI_SLOT(devfn),
+				PCI_FUNC(devfn), reg, value);
+		break;
+	case 4:
+		pci_conf_write32(bus, PCI_SLOT(devfn),
+				PCI_FUNC(devfn), reg, value);
+		break;
+	}
+	return 0;
+}
+
+/**
+ * pci_find_ext_capability - Find an extended capability
+ * @dev: PCI device to query
+ * @cap: capability code
+ *
+ * Returns the address of the requested extended capability structure
+ * within the device's PCI configuration space or 0 if the device does
+ * not support it.  Possible values for @cap:
+ *
+ *  %PCI_EXT_CAP_ID_ERR         Advanced Error Reporting
+ *  %PCI_EXT_CAP_ID_VC          Virtual Channel
+ *  %PCI_EXT_CAP_ID_DSN         Device Serial Number
+ *  %PCI_EXT_CAP_ID_PWR         Power Budgeting
+ */
+int pci_find_ext_capability(int seg, int bus, int devfn, int cap)
+{
+    u32 header;
+    int ttl = 480; /* 3840 bytes, minimum 8 bytes per capability */
+    int pos = 0x100;
+
+    if ( !raw_pci_ext_ops)
+        return 0; 
+
+    raw_pci_ext_ops->read(0, bus, devfn, pos, 4, &header);
+
+    /*
+     * If we have no capabilities, this is indicated by cap ID,
+     * cap version and next pointer all being 0.
+     */
+    if ( (header == 0) || (header == -1) )
+    {
+        dprintk(XENLOG_INFO VTDPREFIX,
+                "next cap:%x:%x.%x:  no extended config\n",
+                bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
+        return 0;
+    }
+
+    while ( ttl-- > 0 ) {
+        if ( PCI_EXT_CAP_ID(header) == cap )
+            return pos;
+        pos = PCI_EXT_CAP_NEXT(header);
+        if ( pos < 0x100 )
+            break;
+        raw_pci_ext_ops->read(0, bus, devfn, pos, 4, &header);
+    }
+    return 0;
+}
diff -r 98d48f7680db xen/drivers/passthrough/vtd/x86/mmconfig_64.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/xen/drivers/passthrough/vtd/x86/mmconfig_64.c	Wed Nov 19 04:04:50 2008 -0800
@@ -0,0 +1,171 @@
+/*
+ * mmconfig.c - Low-level direct PCI config space access via MMCONFIG
+ *
+ * This is an 64bit optimized version that always keeps the full mmconfig
+ * space mapped. This allows lockless config space operation.
+ *
+ * copied from Linux
+ */
+
+#include <xen/mm.h>
+#include <xen/acpi.h>
+#include <xen/xmalloc.h>
+#include <xen/pci.h>
+#include <xen/pci_regs.h>
+
+#include "pci.h"
+
+/* Static virtual mapping of the MMCONFIG aperture */
+struct mmcfg_virt {
+	struct acpi_mcfg_allocation *cfg;
+	char __iomem *virt;
+};
+static struct mmcfg_virt *pci_mmcfg_virt;
+
+static char __iomem *get_virt(unsigned int seg, unsigned bus)
+{
+	struct acpi_mcfg_allocation *cfg;
+	int cfg_num;
+
+	for (cfg_num = 0; cfg_num < pci_mmcfg_config_num; cfg_num++) {
+		cfg = pci_mmcfg_virt[cfg_num].cfg;
+		if (cfg->pci_segment == seg &&
+		    (cfg->start_bus_number <= bus) &&
+		    (cfg->end_bus_number >= bus))
+			return pci_mmcfg_virt[cfg_num].virt;
+	}
+
+	/* Fall back to type 0 */
+	return NULL;
+}
+
+static char __iomem *pci_dev_base(unsigned int seg, unsigned int bus, unsigned int devfn)
+{
+	char __iomem *addr;
+
+	addr = get_virt(seg, bus);
+	if (!addr)
+		return NULL;
+ 	return addr + ((bus << 20) | (devfn << 12));
+}
+
+static int pci_mmcfg_read(unsigned int seg, unsigned int bus,
+			  unsigned int devfn, int reg, int len, u32 *value)
+{
+	char __iomem *addr;
+
+	/* Why do we have this when nobody checks it. How about a BUG()!? -AK */
+	if (unlikely((bus > 255) || (devfn > 255) || (reg > 4095))) {
+err:		*value = -1;
+		return -EINVAL;
+	}
+
+	addr = pci_dev_base(seg, bus, devfn);
+	if (!addr)
+		goto err;
+
+	switch (len) {
+	case 1:
+		*value = mmio_config_readb(addr + reg);
+		break;
+	case 2:
+		*value = mmio_config_readw(addr + reg);
+		break;
+	case 4:
+		*value = mmio_config_readl(addr + reg);
+		break;
+	}
+
+	return 0;
+}
+
+static int pci_mmcfg_write(unsigned int seg, unsigned int bus,
+			   unsigned int devfn, int reg, int len, u32 value)
+{
+	char __iomem *addr;
+
+	/* Why do we have this when nobody checks it. How about a BUG()!? -AK */
+	if (unlikely((bus > 255) || (devfn > 255) || (reg > 4095)))
+		return -EINVAL;
+
+	addr = pci_dev_base(seg, bus, devfn);
+	if (!addr)
+		return -EINVAL;
+
+	switch (len) {
+	case 1:
+		mmio_config_writeb(addr + reg, value);
+		break;
+	case 2:
+		mmio_config_writew(addr + reg, value);
+		break;
+	case 4:
+		mmio_config_writel(addr + reg, value);
+		break;
+	}
+
+	return 0;
+}
+
+static struct pci_raw_ops pci_mmcfg = {
+	.read =		pci_mmcfg_read,
+	.write =	pci_mmcfg_write,
+};
+
+static void __iomem * __init mcfg_ioremap(struct acpi_mcfg_allocation *cfg)
+{
+	void __iomem *addr;
+	u32 size;
+
+	size = (cfg->end_bus_number + 1) << 20;
+	addr = ioremap_nocache(cfg->address, size);
+	if (addr) {
+		printk(KERN_INFO "PCI: Using MMCONFIG at %"PRIx64" - %"PRIx64"\n",
+		       cfg->address, cfg->address + size - 1);
+	}
+	return addr;
+}
+
+int __init pci_mmcfg_arch_init(void)
+{
+	int i;
+	pci_mmcfg_virt = xmalloc_bytes(sizeof(*pci_mmcfg_virt) * pci_mmcfg_config_num);
+	if (pci_mmcfg_virt == NULL) {
+		printk(KERN_ERR "PCI: Can not allocate memory for mmconfig structures\n");
+		return 0;
+	}
+	memset(pci_mmcfg_virt, 0, sizeof(*pci_mmcfg_virt) * pci_mmcfg_config_num);
+
+	for (i = 0; i < pci_mmcfg_config_num; ++i) {
+		pci_mmcfg_virt[i].cfg = &pci_mmcfg_config[i];
+		pci_mmcfg_virt[i].virt = mcfg_ioremap(&pci_mmcfg_config[i]);
+		if (!pci_mmcfg_virt[i].virt) {
+			printk(KERN_ERR "PCI: Cannot map mmconfig aperture for "
+					"segment %d\n",
+				pci_mmcfg_config[i].pci_segment);
+			pci_mmcfg_arch_free();
+			return 0;
+		}
+	}
+	raw_pci_ext_ops = &pci_mmcfg;
+	return 1;
+}
+
+void __init pci_mmcfg_arch_free(void)
+{
+	int i;
+
+	if (pci_mmcfg_virt == NULL)
+		return;
+
+	for (i = 0; i < pci_mmcfg_config_num; ++i) {
+		if (pci_mmcfg_virt[i].virt) {
+			iounmap(pci_mmcfg_virt[i].virt);
+			pci_mmcfg_virt[i].virt = NULL;
+			pci_mmcfg_virt[i].cfg = NULL;
+		}
+	}
+
+	xfree(pci_mmcfg_virt);
+	pci_mmcfg_virt = NULL;
+}
diff -r 98d48f7680db xen/drivers/passthrough/vtd/x86/pci.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/xen/drivers/passthrough/vtd/x86/pci.h	Wed Nov 19 04:04:50 2008 -0800
@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2006, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ *
+ * Author: Allen Kay <allen.m.kay@intel.com> - adapted from linux
+ */
+
+#define PCI_VENDOR_ID_INTEL		0x8086
+#define PCI_DEVICE_ID_INTEL_E7520_MCH	0x3590
+#define PCI_DEVICE_ID_INTEL_82945G_HB	0x2770
+
+/* ioport ends */
+#define PCI_PROBE_BIOS		0x0001
+#define PCI_PROBE_CONF1		0x0002
+#define PCI_PROBE_CONF2		0x0004
+#define PCI_PROBE_MMCONF	0x0008
+#define PCI_PROBE_MASK		0x000f
+#define PCI_PROBE_NOEARLY	0x0010
+
+struct pci_raw_ops {
+        int (*read)(unsigned int domain, unsigned int bus, unsigned int devfn,
+                                                int reg, int len, u32 *val);
+        int (*write)(unsigned int domain, unsigned int bus, unsigned int devfn,
+                                                int reg, int len, u32 val);
+};
+
+static inline unsigned char mmio_config_readb(void __iomem *pos)
+{
+	u8 val;
+	asm volatile("movb (%1),%%al" : "=a" (val) : "r" (pos));
+	return val;
+}
+
+static inline unsigned short mmio_config_readw(void __iomem *pos)
+{
+	u16 val;
+	asm volatile("movw (%1),%%ax" : "=a" (val) : "r" (pos));
+	return val;
+}
+
+static inline unsigned int mmio_config_readl(void __iomem *pos)
+{
+	u32 val;
+	asm volatile("movl (%1),%%eax" : "=a" (val) : "r" (pos));
+	return val;
+}
+
+static inline void mmio_config_writeb(void __iomem *pos, u8 val)
+{
+	asm volatile("movb %%al,(%1)" :: "a" (val), "r" (pos) : "memory");
+}
+
+static inline void mmio_config_writew(void __iomem *pos, u16 val)
+{
+	asm volatile("movw %%ax,(%1)" :: "a" (val), "r" (pos) : "memory");
+}
+
+static inline void mmio_config_writel(void __iomem *pos, u32 val)
+{
+	asm volatile("movl %%eax,(%1)" :: "a" (val), "r" (pos) : "memory");
+}
+
+/* external variable defines */
+extern unsigned int pci_probe;
+extern int pci_mmcfg_config_num;
+extern struct acpi_mcfg_allocation *pci_mmcfg_config;
+extern struct pci_raw_ops *raw_pci_ext_ops;
+
+/* fucntion prototypes */
+int __init acpi_parse_mcfg(struct acpi_table_header *header);
+int pci_conf1_read(unsigned int seg, unsigned int bus,
+                   unsigned int devfn, int reg, int len, u32 *value);
+int pci_conf1_write(unsigned int seg, unsigned int bus,
+                    unsigned int devfn, int reg, int len, u32 value);
+int __init pci_mmcfg_arch_init(void);
+void __init pci_mmcfg_arch_free(void);
+void __iomem * ioremap_nocache(unsigned long offset, unsigned long size);
+int pci_find_ext_capability(int seg, int bus, int devfn, int cap);

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [PATCH 1/2][VTD] pci mmconfig support to be used for ATS

[Jan Beulich]

>>> "Kay, Allen M" <allen.m.kay@intel.com> 20.11.08 02:42 >>>
>ATS (Address Translation Service) is a PCI specification used for synchronizing iotlb translations between chipset iommu such as vt-d with iotlb on board a PCI device.
>
>This patch enables PCI mmconfig for Intel64 systems.  Most of the code were copied from Linux.  This functionality is need for parsing ATS capability in PCIe extended configuration space.

If this is x86-64 only anyway, why do you need to use fixmap space for this (and limit things arbitrarily to 4 segments) rather than using a 256Mb block from one of the unused virtual address regions in hypervisor space?

Also, using obj-$(CONFIG_X86_64) in the Makefile changes would avoid building dead code for x86-32...

Jan

Re: [PATCH 1/2][VTD] pci mmconfig support to be used for ATS

[Espen Skoglund]

I see that you have sort of started to introduce pci_ops functionality
into Xen.  However, the current patch looks very hackish and ugly in
my eyes.  I do undestand that you somehow need to get access to the
PCIe extended configuration space, but I would rather see this happen
in one of the following ways:

  - Use the regular PCI config access functions, and if accesses above
    the regular 255 registers are detected then use mmio (if
    available) for accesses.  The regular PCI configuration space is
    still always accessible using standard mechanisms.

  - Introduce pci_ops integrated properly into Xen internals.  You may
    even want to pci_ops on a per device or per bus basis.

The current proposal very much looks like a quick cut-n-paste hack.

I also see that you've modified struct pci_dev to accomomodate for
specifc ATS needs.  The 'bus' and 'devfn' fields have been made non
constant.  They were marked as 'const' for a reason --- modifying them
leads to race conditions.  Further, you don't actually allocate the
pci_dev using the proper allocate function.  If you don't want the ATS
unit to be treated as a PCI device in Xen then don't use struct
pci_dev.  Create a separate struct ats_unit containing the fields you
need instead.

	eSk



[Allen M Kay]
> ATS (Address Translation Service) is a PCI specification used for
> synchronizing iotlb translations between chipset iommu such as vt-d
> with iotlb on board a PCI device.

> This patch enables PCI mmconfig for Intel64 systems.  Most of the
> code were copied from Linux.  This functionality is need for parsing
> ATS capability in PCIe extended configuration space.

> Signed-off-by: Allen Kay <allen.m.kay@intel.com>

RE: [PATCH 1/2][VTD] pci mmconfig support to be used for ATS

[Kay, Allen M]

>
>  - Use the regular PCI config access functions, and if accesses above
>    the regular 255 registers are detected then use mmio (if
>    available) for accesses.  The regular PCI configuration space is
>    still always accessible using standard mechanisms.
>
>  - Introduce pci_ops integrated properly into Xen internals.  You may
>    even want to pci_ops on a per device or per bus basis.
>

I will look into to see which of the above method is cleaner to implement.

>I also see that you've modified struct pci_dev to accomomodate for
>specifc ATS needs.  The 'bus' and 'devfn' fields have been made non
>constant.  They were marked as 'const' for a reason --- modifying them
>leads to race conditions.  Further, you don't actually allocate the
>pci_dev using the proper allocate function.  If you don't want the ATS
>unit to be treated as a PCI device in Xen then don't use struct
>pci_dev.  Create a separate struct ats_unit containing the fields you
>need instead.
>

I will define a new ATS structure instead of using the existing pci_dev structure so that it will not disturb existing use model for pci_dev structure.

Allen

RE: [PATCH 1/2][VTD] pci mmconfig support to be used for ATS

[Kay, Allen M]

>
>If this is x86-64 only anyway, why do you need to use fixmap 
>space for this (and limit things arbitrarily to 4 segments) 
>rather than using a 256Mb block from one of the unused virtual 
>address regions in hypervisor space?

Can you point me how to use unused virtual address region to do ioremap without using fixmap?  Is there a existing code that is already doing this? 

>
>Also, using obj-$(CONFIG_X86_64) in the Makefile changes would 
>avoid building dead code for x86-32...
>

Will do.  Thanks!

Allen

Re: [PATCH 1/2][VTD] pci mmconfig support to be used for ATS

[Keir Fraser]

On 20/11/08 20:10, "Kay, Allen M" <allen.m.kay@intel.com> wrote:

>> If this is x86-64 only anyway, why do you need to use fixmap
>> space for this (and limit things arbitrarily to 4 segments)
>> rather than using a 256Mb block from one of the unused virtual
>> address regions in hypervisor space?
> 
> Can you point me how to use unused virtual address region to do ioremap
> without using fixmap?  Is there a existing code that is already doing this?

Pick yourself a nice piece of virtual address space (see the layout in
include/asm-x86/config.h -- 0xffff840000000000 would be fine) and then use
map_pages_to_xen().

 -- Keir

Re: [PATCH 1/2][VTD] pci mmconfig support to be usedfor ATS

[Jan Beulich]

>>> Keir Fraser <keir.fraser@eu.citrix.com> 20.11.08 21:20 >>>
>On 20/11/08 20:10, "Kay, Allen M" <allen.m.kay@intel.com> wrote:
>
>>> If this is x86-64 only anyway, why do you need to use fixmap
>>> space for this (and limit things arbitrarily to 4 segments)
>>> rather than using a 256Mb block from one of the unused virtual
>>> address regions in hypervisor space?
>> 
>> Can you point me how to use unused virtual address region to do ioremap
>> without using fixmap?  Is there a existing code that is already doing this?
>
>Pick yourself a nice piece of virtual address space (see the layout in
>include/asm-x86/config.h -- 0xffff840000000000 would be fine) and then use
>map_pages_to_xen().

While it can be adjusted later anyway, I'd suggest not using a part of this
largest available range - sooner or later we'll want to use this to extend
the 1:1 mapping to 4Tb. Instead, I'd pick another 1Gb range out of the
461Gb reserved region.

Jan