VT-d pci passthrough patches

VT-d pci passthrough patches

[Ben-Ami Yassour]

The following two patches contains the VT-d support for pci passthrough
devices. 
The first patch contains the changes that are required to the generic
VT-d code.
The second patch contains the changes to KVM.

These patches contains fixes based on comments by Anthony,
on the pervious version.

The VT-d code is used by the pci-passthrough code to map the entire
guest memory in the IOMMU.
Once mmu-notifiers go in, we will be able to support finer grained forms
of protection.

The main differences from the previous version, sent by Weidong, are
fixes for the page pinning.
In the destruction path we need to call put_page for each page that was
mapped in the iommu and pinned during initialization..
The KVM process page tables are already released by the time the KVM vm
destroy path is executed.
Therefore we can not use gfn_to_pfn (which uses get_user_pages) to get
the pages.
The approach in the current patch is to find the pages to unpin by
looking at the IOMMU page table.

This patch applies to the Amit's PCI passthrough tree.

Any comments are welcome.

Regards,
Ben

[PATCH] VT-d : changes to support KVM

[Ben-Ami Yassour]

From: Kay, Allen M <allen.m.kay@intel.com>

This patch extends the VT-d driver to support KVM

[Ben: fixed memory pinning]

Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
Signed-off-by: Weidong Han <weidong.han@intel.com>
Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
---
 drivers/pci/dmar.c                           |    4 +-
 drivers/pci/intel-iommu.c                    |  117 +++++++++++++++++++++++++-
 drivers/pci/iova.c                           |    2 +-
 {drivers/pci => include/linux}/intel-iommu.h |   11 +++
 {drivers/pci => include/linux}/iova.h        |    0 
 5 files changed, 127 insertions(+), 7 deletions(-)
 rename {drivers/pci => include/linux}/intel-iommu.h (94%)
 rename {drivers/pci => include/linux}/iova.h (100%)

diff --git a/drivers/pci/dmar.c b/drivers/pci/dmar.c
index f941f60..a58a5b0 100644
--- a/drivers/pci/dmar.c
+++ b/drivers/pci/dmar.c
@@ -26,8 +26,8 @@
 
 #include <linux/pci.h>
 #include <linux/dmar.h>
-#include "iova.h"
-#include "intel-iommu.h"
+#include <linux/iova.h>
+#include <linux/intel-iommu.h>
 
 #undef PREFIX
 #define PREFIX "DMAR:"
diff --git a/drivers/pci/intel-iommu.c b/drivers/pci/intel-iommu.c
index 66c0fd2..6ad2c75 100644
--- a/drivers/pci/intel-iommu.c
+++ b/drivers/pci/intel-iommu.c
@@ -20,6 +20,7 @@
  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  */
 
+#undef DEBUG
 #include <linux/init.h>
 #include <linux/bitmap.h>
 #include <linux/debugfs.h>
@@ -33,8 +34,8 @@
 #include <linux/dma-mapping.h>
 #include <linux/mempool.h>
 #include <linux/timer.h>
-#include "iova.h"
-#include "intel-iommu.h"
+#include <linux/iova.h>
+#include <linux/intel-iommu.h>
 #include <asm/proto.h> /* force_iommu in this header in x86-64*/
 #include <asm/cacheflush.h>
 #include <asm/gart.h>
@@ -160,7 +161,7 @@ static inline void *alloc_domain_mem(void)
 	return iommu_kmem_cache_alloc(iommu_domain_cache);
 }
 
-static inline void free_domain_mem(void *vaddr)
+static void free_domain_mem(void *vaddr)
 {
 	kmem_cache_free(iommu_domain_cache, vaddr);
 }
@@ -1414,7 +1415,7 @@ static void domain_remove_dev_info(struct dmar_domain *domain)
  * find_domain
  * Note: we use struct pci_dev->dev.archdata.iommu stores the info
  */
-struct dmar_domain *
+static struct dmar_domain *
 find_domain(struct pci_dev *pdev)
 {
 	struct device_domain_info *info;
@@ -2408,3 +2409,111 @@ int __init intel_iommu_init(void)
 	return 0;
 }
 
+void intel_iommu_domain_exit(struct dmar_domain *domain)
+{
+	u64 end;
+
+	/* Domain 0 is reserved, so dont process it */
+	if (!domain)
+		return;
+
+	end = DOMAIN_MAX_ADDR(domain->gaw);
+	end = end & (~PAGE_MASK_4K);
+
+	/* clear ptes */
+	dma_pte_clear_range(domain, 0, end);
+
+	/* free page tables */
+	dma_pte_free_pagetable(domain, 0, end);
+
+	iommu_free_domain(domain);
+	free_domain_mem(domain);
+}
+EXPORT_SYMBOL_GPL(intel_iommu_domain_exit);
+
+struct dmar_domain *intel_iommu_domain_alloc(struct pci_dev *pdev)
+{
+	struct dmar_drhd_unit *drhd;
+	struct dmar_domain *domain;
+	struct intel_iommu *iommu;
+
+	drhd = dmar_find_matched_drhd_unit(pdev);
+	if (!drhd) {
+		printk(KERN_ERR "intel_iommu_domain_alloc: drhd == NULL\n");
+		return NULL;
+	}
+
+	iommu = drhd->iommu;
+	if (!iommu) {
+		printk(KERN_ERR
+			"intel_iommu_domain_alloc: iommu == NULL\n");
+		return NULL;
+	}
+	domain = iommu_alloc_domain(iommu);
+	if (!domain) {
+		printk(KERN_ERR
+			"intel_iommu_domain_alloc: domain == NULL\n");
+		return NULL;
+	}
+	if (domain_init(domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
+		printk(KERN_ERR
+			"intel_iommu_domain_alloc: domain_init() failed\n");
+		intel_iommu_domain_exit(domain);
+		return NULL;
+	}
+	return domain;
+}
+EXPORT_SYMBOL_GPL(intel_iommu_domain_alloc);
+
+int intel_iommu_context_mapping(
+	struct dmar_domain *domain, struct pci_dev *pdev)
+{
+	int rc;
+	rc = domain_context_mapping(domain, pdev);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(intel_iommu_context_mapping);
+
+int intel_iommu_page_mapping(
+	struct dmar_domain *domain, dma_addr_t iova,
+	u64 hpa, size_t size, int prot)
+{
+	int rc;
+	rc = domain_page_mapping(domain, iova, hpa, size, prot);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(intel_iommu_page_mapping);
+
+void intel_iommu_detach_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
+{
+	detach_domain_for_dev(domain, bus, devfn);
+}
+EXPORT_SYMBOL_GPL(intel_iommu_detach_dev);
+
+struct dmar_domain *
+intel_iommu_find_domain(struct pci_dev *pdev)
+{
+	return find_domain(pdev);
+}
+EXPORT_SYMBOL_GPL(intel_iommu_find_domain);
+
+int intel_iommu_found(void)
+{
+	return g_num_of_iommus;
+}
+EXPORT_SYMBOL_GPL(intel_iommu_found);
+
+u64 intel_iommu_iova_to_pfn(struct dmar_domain *domain, u64 iova)
+{
+	struct dma_pte *pte;
+	u64 pfn;
+
+	pfn = 0;
+	pte = addr_to_dma_pte(domain, iova);
+
+	if (pte)
+		pfn = dma_pte_addr(*pte);
+
+	return pfn >> PAGE_SHIFT_4K;
+}
+EXPORT_SYMBOL_GPL(intel_iommu_iova_to_pfn);
diff --git a/drivers/pci/iova.c b/drivers/pci/iova.c
index 3ef4ac0..2287116 100644
--- a/drivers/pci/iova.c
+++ b/drivers/pci/iova.c
@@ -7,7 +7,7 @@
  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  */
 
-#include "iova.h"
+#include <linux/iova.h>
 
 void
 init_iova_domain(struct iova_domain *iovad, unsigned long pfn_32bit)
diff --git a/drivers/pci/intel-iommu.h b/include/linux/intel-iommu.h
similarity index 94%
rename from drivers/pci/intel-iommu.h
rename to include/linux/intel-iommu.h
index afc0ad9..1490fc0 100644
--- a/drivers/pci/intel-iommu.h
+++ b/include/linux/intel-iommu.h
@@ -341,4 +341,15 @@ static inline void iommu_prepare_gfx_mapping(void)
 }
 #endif /* !CONFIG_DMAR_GFX_WA */
 
+void intel_iommu_domain_exit(struct dmar_domain *domain);
+struct dmar_domain *intel_iommu_domain_alloc(struct pci_dev *pdev);
+int intel_iommu_context_mapping(struct dmar_domain *domain,
+				struct pci_dev *pdev);
+int intel_iommu_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
+			     u64 hpa, size_t size, int prot);
+void intel_iommu_detach_dev(struct dmar_domain *domain, u8 bus, u8 devfn);
+struct dmar_domain *intel_iommu_find_domain(struct pci_dev *pdev);
+int intel_iommu_found(void);
+u64 intel_iommu_iova_to_pfn(struct dmar_domain *domain, u64 iova);
+
 #endif
diff --git a/drivers/pci/iova.h b/include/linux/iova.h
similarity index 100%
rename from drivers/pci/iova.h
rename to include/linux/iova.h
-- 
1.5.6

[PATCH] KVM: PCIPT: VT-d support

[Ben-Ami Yassour]

From: Kay, Allen M <allen.m.kay@intel.com>

This patch includes the functions to support VT-d for passthrough
devices.

[Ben: fixed memory pinning]

Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
Signed-off-by: Weidong Han <weidong.han@intel.com>
Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
---
 arch/x86/kvm/Makefile      |    2 +-
 arch/x86/kvm/vtd.c         |  176 ++++++++++++++++++++++++++++++++++++++++++++
 arch/x86/kvm/x86.c         |   10 +++
 include/asm-x86/kvm_host.h |    1 +
 include/linux/kvm_host.h   |    6 ++
 virt/kvm/kvm_main.c        |    6 ++
 6 files changed, 200 insertions(+), 1 deletions(-)
 create mode 100644 arch/x86/kvm/vtd.c

diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index d0e940b..5d9d079 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -11,7 +11,7 @@ endif
 EXTRA_CFLAGS += -Ivirt/kvm -Iarch/x86/kvm
 
 kvm-objs := $(common-objs) x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o \
-	i8254.o
+	i8254.o vtd.o
 obj-$(CONFIG_KVM) += kvm.o
 kvm-intel-objs = vmx.o
 obj-$(CONFIG_KVM_INTEL) += kvm-intel.o
diff --git a/arch/x86/kvm/vtd.c b/arch/x86/kvm/vtd.c
new file mode 100644
index 0000000..83efb8a
--- /dev/null
+++ b/arch/x86/kvm/vtd.c
@@ -0,0 +1,176 @@
+/*
+ * Copyright (c) 2006, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ *
+ * Copyright (C) 2006-2008 Intel Corporation
+ * Author: Allen M. Kay <allen.m.kay@intel.com>
+ * Author: Weidong Han <weidong.han@intel.com>
+ */
+
+#include <linux/list.h>
+#include <linux/kvm_host.h>
+#include <linux/pci.h>
+#include <linux/dmar.h>
+#include <linux/intel-iommu.h>
+
+static int kvm_iommu_unmap_memslots(struct kvm *kvm);
+
+int kvm_iommu_map_pages(struct kvm *kvm,
+			gfn_t base_gfn, unsigned long npages)
+{
+	gfn_t gfn = base_gfn;
+	pfn_t pfn;
+	int i, rc;
+	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
+
+	if (!domain)
+		return -EFAULT;
+
+	for (i = 0; i < npages; i++) {
+		pfn = gfn_to_pfn(kvm, gfn);
+		rc = intel_iommu_page_mapping(domain,
+					      gfn << PAGE_SHIFT,
+						      pfn << PAGE_SHIFT,
+					      PAGE_SIZE,
+					      DMA_PTE_READ |
+					      DMA_PTE_WRITE);
+		if (rc)
+			kvm_release_pfn_clean(pfn);
+
+		gfn++;
+	}
+	return 0;
+}
+
+static int kvm_iommu_map_memslots(struct kvm *kvm)
+{
+	int i, rc;
+	for (i = 0; i < kvm->nmemslots; i++) {
+		rc = kvm_iommu_map_pages(kvm, kvm->memslots[i].base_gfn,
+					 kvm->memslots[i].npages);
+		if (rc)
+			return rc;
+	}
+	return 0;
+}
+
+int kvm_iommu_map_guest(struct kvm *kvm,
+			struct kvm_pci_passthrough_dev *pci_pt_dev)
+{
+	struct pci_dev *pdev = NULL;
+
+	printk(KERN_DEBUG "VT-d direct map: host bdf = %x:%x:%x\n",
+	       pci_pt_dev->host.busnr,
+	       PCI_SLOT(pci_pt_dev->host.devfn),
+	       PCI_FUNC(pci_pt_dev->host.devfn));
+
+	for_each_pci_dev(pdev) {
+		if ((pdev->bus->number == pci_pt_dev->host.busnr) &&
+		    (pdev->devfn == pci_pt_dev->host.devfn)) {
+			break;
+		}
+	}
+
+	if (pdev == NULL) {
+		if (kvm->arch.intel_iommu_domain) {
+			intel_iommu_domain_exit(kvm->arch.intel_iommu_domain);
+			kvm->arch.intel_iommu_domain = NULL;
+		}
+		return -ENODEV;
+	}
+
+	kvm->arch.intel_iommu_domain = intel_iommu_domain_alloc(pdev);
+
+	if (kvm_iommu_map_memslots(kvm)) {
+		kvm_iommu_unmap_memslots(kvm);
+		return -EFAULT;
+	}
+
+	intel_iommu_detach_dev(kvm->arch.intel_iommu_domain,
+			       pdev->bus->number, pdev->devfn);
+
+	if (intel_iommu_context_mapping(kvm->arch.intel_iommu_domain,
+					pdev)) {
+		printk(KERN_ERR "Domain context map for %s failed",
+		       pci_name(pdev));
+		return -EFAULT;
+	}
+	return 0;
+}
+
+static int kvm_iommu_put_pages(struct kvm *kvm,
+			       gfn_t base_gfn, unsigned long npages)
+{
+	gfn_t gfn = base_gfn;
+	pfn_t pfn;
+	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
+	int i;
+
+	if (!domain)
+		return -EFAULT;
+
+	for (i = 0; i < npages; i++) {
+		pfn = (pfn_t)intel_iommu_iova_to_pfn(domain,
+						     gfn << PAGE_SHIFT);
+		kvm_release_pfn_clean(pfn);
+		gfn++;
+	}
+	return 0;
+}
+
+static int kvm_iommu_unmap_memslots(struct kvm *kvm)
+{
+	int i, rc;
+	for (i = 0; i < kvm->nmemslots; i++) {
+		rc = kvm_iommu_put_pages(kvm, kvm->memslots[i].base_gfn,
+					 kvm->memslots[i].npages);
+		if (rc)
+			return rc;
+	}
+	return 0;
+}
+
+int kvm_iommu_unmap_guest(struct kvm *kvm)
+{
+	struct kvm_pci_pt_dev_list *entry;
+	struct pci_dev *pdev = NULL;
+	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
+
+	if (!domain)
+		return 0;
+
+	list_for_each_entry(entry, &kvm->arch.pci_pt_dev_head, list) {
+		printk(KERN_DEBUG "VT-d unmap: host bdf = %x:%x:%x\n",
+		       entry->pt_dev.host.busnr,
+		       PCI_SLOT(entry->pt_dev.host.devfn),
+		       PCI_FUNC(entry->pt_dev.host.devfn));
+
+		for_each_pci_dev(pdev) {
+			if ((pdev->bus->number == entry->pt_dev.host.busnr) &&
+			    (pdev->devfn == entry->pt_dev.host.devfn))
+				break;
+		}
+
+		if (pdev == NULL)
+			return -ENODEV;
+
+		/* detach kvm dmar domain */
+		intel_iommu_detach_dev(domain,
+				       pdev->bus->number, pdev->devfn);
+	}
+	kvm_iommu_unmap_memslots(kvm);
+	intel_iommu_domain_exit(domain);
+	return 0;
+}
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index d4d4e0c..19d2ae3 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -32,6 +32,7 @@
 #include <linux/module.h>
 #include <linux/mman.h>
 #include <linux/highmem.h>
+#include <linux/intel-iommu.h>
 
 #include <asm/uaccess.h>
 #include <asm/msr.h>
@@ -333,6 +334,13 @@ static int kvm_vm_ioctl_pci_pt_dev(struct kvm *kvm,
 	if (irqchip_in_kernel(kvm))
 		set_bit(dev->irq, pt_irq_handled);
 	write_unlock_irqrestore(&kvm_pci_pt_lock, flags);
+
+	if (intel_iommu_found()) {
+		r = kvm_iommu_map_guest(kvm, pci_pt_dev);
+		if (r)
+			goto out_put;
+	}
+
 out:
 	return r;
 out_put:
@@ -4272,6 +4280,8 @@ static void kvm_free_vcpus(struct kvm *kvm)
 
 void kvm_arch_destroy_vm(struct kvm *kvm)
 {
+	if (intel_iommu_found())
+		kvm_iommu_unmap_guest(kvm);
 	kvm_free_pci_passthrough(kvm);
 	kvm_free_pit(kvm);
 	kfree(kvm->arch.vpic);
diff --git a/include/asm-x86/kvm_host.h b/include/asm-x86/kvm_host.h
index 0c6699f..953080e 100644
--- a/include/asm-x86/kvm_host.h
+++ b/include/asm-x86/kvm_host.h
@@ -354,6 +354,7 @@ struct kvm_arch{
 	 */
 	struct list_head active_mmu_pages;
 	struct list_head pci_pt_dev_head;
+	struct dmar_domain *intel_iommu_domain;
 	struct kvm_pic *vpic;
 	struct kvm_ioapic *vioapic;
 	struct kvm_pit *vpit;
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index d220b49..a16e2e3 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -278,6 +278,12 @@ int kvm_cpu_has_interrupt(struct kvm_vcpu *v);
 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
 void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
 
+int kvm_iommu_map_pages(struct kvm *kvm, gfn_t base_gfn,
+			unsigned long npages);
+int kvm_iommu_map_guest(struct kvm *kvm,
+			struct kvm_pci_passthrough_dev *pci_pt_dev);
+int kvm_iommu_unmap_guest(struct kvm *kvm);
+
 static inline void kvm_guest_enter(void)
 {
 	account_system_vtime(current);
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 531d635..dc67d90 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -41,6 +41,7 @@
 #include <linux/pagemap.h>
 #include <linux/mman.h>
 #include <linux/swap.h>
+#include <linux/intel-iommu.h>
 
 #include <asm/processor.h>
 #include <asm/io.h>
@@ -422,6 +423,11 @@ int __kvm_set_memory_region(struct kvm *kvm,
 	}
 
 	kvm_free_physmem_slot(&old, &new);
+
+	/* map the pages in iommu page table */
+	if (intel_iommu_found())
+		kvm_iommu_map_pages(kvm, base_gfn, npages);
+
 	return 0;
 
 out_free:
-- 
1.5.6

Re: [PATCH] KVM: PCIPT: VT-d support

[Yang, Sheng]

On Thursday 10 July 2008 17:14:42 Ben-Ami Yassour wrote:
> From: Kay, Allen M <allen.m.kay@intel.com>
>
> This patch includes the functions to support VT-d for passthrough
> devices.
>
> [Ben: fixed memory pinning]
>
> Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
> Signed-off-by: Weidong Han <weidong.han@intel.com>
> Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
> ---
>  arch/x86/kvm/Makefile      |    2 +-
>  arch/x86/kvm/vtd.c         |  176
> ++++++++++++++++++++++++++++++++++++++++++++ arch/x86/kvm/x86.c    
>     |   10 +++
>  include/asm-x86/kvm_host.h |    1 +
>  include/linux/kvm_host.h   |    6 ++
>  virt/kvm/kvm_main.c        |    6 ++
>  6 files changed, 200 insertions(+), 1 deletions(-)
>  create mode 100644 arch/x86/kvm/vtd.c
>
> diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
> index 531d635..dc67d90 100644
> --- a/virt/kvm/kvm_main.c
> +++ b/virt/kvm/kvm_main.c
> @@ -41,6 +41,7 @@
>  #include <linux/pagemap.h>
>  #include <linux/mman.h>
>  #include <linux/swap.h>
> +#include <linux/intel-iommu.h>
>
>  #include <asm/processor.h>
>  #include <asm/io.h>
> @@ -422,6 +423,11 @@ int __kvm_set_memory_region(struct kvm *kvm,
>  	}
>
>  	kvm_free_physmem_slot(&old, &new);
> +
> +	/* map the pages in iommu page table */
> +	if (intel_iommu_found())
> +		kvm_iommu_map_pages(kvm, base_gfn, npages);
> +

I don't understand why we need this along with  
kvm_iommu_map_memslots(). This works during the memory setup, and in 
kvm_iommu_map_guest() we do it again with the overlapped memory 
region?

I think even if we need pin all pages, we still just need do it 
once...

--
Thanks
Yang, Sheng

Re: [PATCH] KVM: PCIPT: VT-d support

[Ben-Ami Yassour]

On Thu, 2008-07-10 at 17:30 +0800, Yang, Sheng wrote:
> On Thursday 10 July 2008 17:14:42 Ben-Ami Yassour wrote:
> > From: Kay, Allen M <allen.m.kay@intel.com>
> >
> > This patch includes the functions to support VT-d for passthrough
> > devices.
> >
> > [Ben: fixed memory pinning]
> >
> > Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
> > Signed-off-by: Weidong Han <weidong.han@intel.com>
> > Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
> > ---
> >  arch/x86/kvm/Makefile      |    2 +-
> >  arch/x86/kvm/vtd.c         |  176
> > ++++++++++++++++++++++++++++++++++++++++++++ arch/x86/kvm/x86.c    
> >     |   10 +++
> >  include/asm-x86/kvm_host.h |    1 +
> >  include/linux/kvm_host.h   |    6 ++
> >  virt/kvm/kvm_main.c        |    6 ++
> >  6 files changed, 200 insertions(+), 1 deletions(-)
> >  create mode 100644 arch/x86/kvm/vtd.c
> >
> > diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
> > index 531d635..dc67d90 100644
> > --- a/virt/kvm/kvm_main.c
> > +++ b/virt/kvm/kvm_main.c
> > @@ -41,6 +41,7 @@
> >  #include <linux/pagemap.h>
> >  #include <linux/mman.h>
> >  #include <linux/swap.h>
> > +#include <linux/intel-iommu.h>
> >
> >  #include <asm/processor.h>
> >  #include <asm/io.h>
> > @@ -422,6 +423,11 @@ int __kvm_set_memory_region(struct kvm *kvm,
> >  	}
> >
> >  	kvm_free_physmem_slot(&old, &new);
> > +
> > +	/* map the pages in iommu page table */
> > +	if (intel_iommu_found())
> > +		kvm_iommu_map_pages(kvm, base_gfn, npages);
> > +
> 
> I don't understand why we need this along with  
> kvm_iommu_map_memslots(). This works during the memory setup, and in 
> kvm_iommu_map_guest() we do it again with the overlapped memory 
> region?
> 
> I think even if we need pin all pages, we still just need do it 
> once...

We map the entire guest memory on initialization by going over all the
existing memory slots.
If a new slot is created later then we need to map it as well, this is
the call that you see here.

Regards,
Ben

Re: [PATCH] KVM: PCIPT: VT-d support

[Yang, Sheng]

On Thursday 10 July 2008 17:51:53 Ben-Ami Yassour wrote:
> On Thu, 2008-07-10 at 17:30 +0800, Yang, Sheng wrote:
> > On Thursday 10 July 2008 17:14:42 Ben-Ami Yassour wrote:
> > > From: Kay, Allen M <allen.m.kay@intel.com>
> > >
> > > This patch includes the functions to support VT-d for
> > > passthrough devices.
> > >
> > > [Ben: fixed memory pinning]
> > >
> > > Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
> > > Signed-off-by: Weidong Han <weidong.han@intel.com>
> > > Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
> > > ---
> > >  arch/x86/kvm/Makefile      |    2 +-
> > >  arch/x86/kvm/vtd.c         |  176
> > > ++++++++++++++++++++++++++++++++++++++++++++ arch/x86/kvm/x86.c
> > >
> > >     |   10 +++
> > >
> > >  include/asm-x86/kvm_host.h |    1 +
> > >  include/linux/kvm_host.h   |    6 ++
> > >  virt/kvm/kvm_main.c        |    6 ++
> > >  6 files changed, 200 insertions(+), 1 deletions(-)
> > >  create mode 100644 arch/x86/kvm/vtd.c
> > >
> > > diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
> > > index 531d635..dc67d90 100644
> > > --- a/virt/kvm/kvm_main.c
> > > +++ b/virt/kvm/kvm_main.c
> > > @@ -41,6 +41,7 @@
> > >  #include <linux/pagemap.h>
> > >  #include <linux/mman.h>
> > >  #include <linux/swap.h>
> > > +#include <linux/intel-iommu.h>
> > >
> > >  #include <asm/processor.h>
> > >  #include <asm/io.h>
> > > @@ -422,6 +423,11 @@ int __kvm_set_memory_region(struct kvm
> > > *kvm, }
> > >
> > >  	kvm_free_physmem_slot(&old, &new);
> > > +
> > > +	/* map the pages in iommu page table */
> > > +	if (intel_iommu_found())
> > > +		kvm_iommu_map_pages(kvm, base_gfn, npages);
> > > +
> >
> > I don't understand why we need this along with
> > kvm_iommu_map_memslots(). This works during the memory setup, and
> > in kvm_iommu_map_guest() we do it again with the overlapped
> > memory region?
> >
> > I think even if we need pin all pages, we still just need do it
> > once...
>
> We map the entire guest memory on initialization by going over all
> the existing memory slots.
> If a new slot is created later then we need to map it as well, this
> is the call that you see here.

I think it's may be unnecessary to map pages when device assigned. The 
table can be set up along with set_memory_region(), it covered all 
memory slots already, or I miss something here?

-- 
Thanks
Yang, Sheng

Re: [PATCH] KVM: PCIPT: VT-d support

[Muli Ben-Yehuda]

On Thu, Jul 10, 2008 at 06:07:27PM +0800, Yang, Sheng wrote:

> I think it's may be unnecessary to map pages when device
> assigned. The table can be set up along with set_memory_region(), it
> covered all memory slots already, or I miss something here?

VT-d is only initialized after the slots are originally created, so
when VT-d is initialized is map all of the existing slots separately,
and then for each new slot that may be added we'll catch it via
set_memory_region().

Cheers,
Muli

Re: [PATCH] KVM: PCIPT: VT-d support

[Avi Kivity]

Muli Ben-Yehuda wrote:
> On Thu, Jul 10, 2008 at 06:07:27PM +0800, Yang, Sheng wrote:
>
>   
>> I think it's may be unnecessary to map pages when device
>> assigned. The table can be set up along with set_memory_region(), it
>> covered all memory slots already, or I miss something here?
>>     
>
> VT-d is only initialized after the slots are originally created, so
> when VT-d is initialized is map all of the existing slots separately,
> and then for each new slot that may be added we'll catch it via
> set_memory_region().
>   

It makes sense to initialize VT-d before.  Since memslots can be created 
and destroyed dynamically, with the current implementation we can see

  create slot
  create slot
  init VT-d
  create slot

which means we need to support both slot-creation-after-VT-d and 
init-VT-d-after-slot-creation.  If we initialize VT-d up front, we only 
need to support (and test) one scenario.

On the other hand, this means that you will not be able to assign 
devices unless you specified this when creating the VM; but I think this 
is fair.

-- 
error compiling committee.c: too many arguments to function

Re: [PATCH] KVM: PCIPT: VT-d support

[Muli Ben-Yehuda]

On Thu, Jul 10, 2008 at 05:57:56PM +0300, Avi Kivity wrote:

> It makes sense to initialize VT-d before.  Since memslots can be
> created and destroyed dynamically, with the current implementation
> we can see
>
>  create slot
>  create slot
>  init VT-d
>  create slot
>
> which means we need to support both slot-creation-after-VT-d and
> init-VT-d-after-slot-creation.  If we initialize VT-d up front, we
> only need to support (and test) one scenario.

Fair enough---we'll take a look at whether initializing VT-d first
introduces any complications other than hot-plug (see below).

> On the other hand, this means that you will not be able to assign
> devices unless you specified this when creating the VM; but I think
> this is fair.

It will be nice to support hot-plugged pass-through devices some time
in the future.

Cheers,
Muli

Re: [PATCH] KVM: PCIPT: VT-d support

[Avi Kivity]

Muli Ben-Yehuda wrote:
>
>> On the other hand, this means that you will not be able to assign
>> devices unless you specified this when creating the VM; but I think
>> this is fair.
>>     
>
> It will be nice to support hot-plugged pass-through devices some time
> in the future.
>
>   

And it should be easy to do too.  The question is whether we need to 
allocate the iommu domain up front (which would mean that a command line 
parameter would be needed, meaning roughly 'support pci device 
assignment on this guest'.  Kind of like -usb, which means 'support usb 
device assignment on this guest'.

-- 
error compiling committee.c: too many arguments to function

Re: [PATCH] KVM: PCIPT: VT-d support

[Ben-Ami Yassour]

On Sun, 2008-07-13 at 10:34 +0300, Avi Kivity wrote:
> Muli Ben-Yehuda wrote:
> >
> >> On the other hand, this means that you will not be able to assign
> >> devices unless you specified this when creating the VM; but I think
> >> this is fair.
> >>     
> >
> > It will be nice to support hot-plugged pass-through devices some time
> > in the future.
> >
> >   
> 
> And it should be easy to do too.  The question is whether we need to 
> allocate the iommu domain up front (which would mean that a command line 
> parameter would be needed, meaning roughly 'support pci device 
> assignment on this guest'.  Kind of like -usb, which means 'support usb 
> device assignment on this guest'.
> 

I don't think that its that simple...
On a single machine you can have multiple iommu units, each of which controls certain PCI slots.
To configure the iommu for a specific device you need to configure the iommu unit that controls that device.
This means that we need to know the bus/dev/func of the device before we create the VT-d domain.
For hot-plug devices it means that by the time we can create the domain we already have memory slots that we need to add to the VT-d mapping.

In addition, some times it is desirable to be able to map the memory as late as possible.
For example if the guest supports PVDMA then there is no need to map the entire guest memory, 
however the host can not tell if the guest supports PVDMA before the memory slots are created.
In this case we would like to postpone the mapping of the entire guest memory until we know we have to.

Regards,
Ben

Re: [PATCH] KVM: PCIPT: VT-d support

[Avi Kivity]

Ben-Ami Yassour wrote:
> On Sun, 2008-07-13 at 10:34 +0300, Avi Kivity wrote:
>   
>> Muli Ben-Yehuda wrote:
>>     
>>>> On the other hand, this means that you will not be able to assign
>>>> devices unless you specified this when creating the VM; but I think
>>>> this is fair.
>>>>     
>>>>         
>>> It will be nice to support hot-plugged pass-through devices some time
>>> in the future.
>>>
>>>   
>>>       
>> And it should be easy to do too.  The question is whether we need to 
>> allocate the iommu domain up front (which would mean that a command line 
>> parameter would be needed, meaning roughly 'support pci device 
>> assignment on this guest'.  Kind of like -usb, which means 'support usb 
>> device assignment on this guest'.
>>
>>     
>
> I don't think that its that simple...
> On a single machine you can have multiple iommu units, each of which controls certain PCI slots.
> To configure the iommu for a specific device you need to configure the iommu unit that controls that device.
> This means that we need to know the bus/dev/func of the device before we create the VT-d domain.
> For hot-plug devices it means that by the time we can create the domain we already have memory slots that we need to add to the VT-d mapping.
>
>   

Thanks for the info.  Looks like we have to support both 
memslot-before-iommu and memslot-after-iommu then.

-- 
error compiling committee.c: too many arguments to function

VT-d pci passthrough patches

[Ben-Ami Yassour]

The following two patches contains the VT-d support for pci passthrough
devices. 
The first patch contains the changes that are required to the generic
VT-d code.
The second patch contains the changes to KVM.

These patches contains fixes of the previous version, based on comments 
from Avi and Weidong.

The VT-d code is used by the pci-passthrough code to map the entire
guest memory in the IOMMU.
Once mmu-notifiers go in, we will be able to support finer grained forms
of protection.

The main differences from the previous version, sent by Weidong, are
fixes for the page pinning.
In the destruction path we need to call put_page for each page that was
mapped in the iommu and pinned during initialization..
The KVM process page tables are already released by the time the KVM vm
destroy path is executed.
Therefore we can not use gfn_to_pfn (which uses get_user_pages) to get
the pages.
The approach in the current patch is to find the pages to unpin by
looking at the IOMMU page table.

This patch applies to the main KVM tree.
The code is not useable without the pci-passthrough code, however we
separate it from the rest of the pci-passthrough code, so it can be
reviewed and merged independently.

Any comments are welcome.

Regards,
Ben

[PATCH] VT-d : changes to support KVM

[Ben-Ami Yassour]

From: Kay, Allen M <allen.m.kay@intel.com>

This patch extends the VT-d driver to support KVM

[Ben: fixed memory pinning]

Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
Signed-off-by: Weidong Han <weidong.han@intel.com>
Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
---
 drivers/pci/dmar.c                           |    4 +-
 drivers/pci/intel-iommu.c                    |  117 +++++++++++++++++++++++++-
 drivers/pci/iova.c                           |    2 +-
 {drivers/pci => include/linux}/intel-iommu.h |   11 +++
 {drivers/pci => include/linux}/iova.h        |    0 
 5 files changed, 127 insertions(+), 7 deletions(-)
 rename {drivers/pci => include/linux}/intel-iommu.h (94%)
 rename {drivers/pci => include/linux}/iova.h (100%)

diff --git a/drivers/pci/dmar.c b/drivers/pci/dmar.c
index f941f60..a58a5b0 100644
--- a/drivers/pci/dmar.c
+++ b/drivers/pci/dmar.c
@@ -26,8 +26,8 @@
 
 #include <linux/pci.h>
 #include <linux/dmar.h>
-#include "iova.h"
-#include "intel-iommu.h"
+#include <linux/iova.h>
+#include <linux/intel-iommu.h>
 
 #undef PREFIX
 #define PREFIX "DMAR:"
diff --git a/drivers/pci/intel-iommu.c b/drivers/pci/intel-iommu.c
index 66c0fd2..6ad2c75 100644
--- a/drivers/pci/intel-iommu.c
+++ b/drivers/pci/intel-iommu.c
@@ -20,6 +20,7 @@
  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  */
 
+#undef DEBUG
 #include <linux/init.h>
 #include <linux/bitmap.h>
 #include <linux/debugfs.h>
@@ -33,8 +34,8 @@
 #include <linux/dma-mapping.h>
 #include <linux/mempool.h>
 #include <linux/timer.h>
-#include "iova.h"
-#include "intel-iommu.h"
+#include <linux/iova.h>
+#include <linux/intel-iommu.h>
 #include <asm/proto.h> /* force_iommu in this header in x86-64*/
 #include <asm/cacheflush.h>
 #include <asm/gart.h>
@@ -160,7 +161,7 @@ static inline void *alloc_domain_mem(void)
 	return iommu_kmem_cache_alloc(iommu_domain_cache);
 }
 
-static inline void free_domain_mem(void *vaddr)
+static void free_domain_mem(void *vaddr)
 {
 	kmem_cache_free(iommu_domain_cache, vaddr);
 }
@@ -1414,7 +1415,7 @@ static void domain_remove_dev_info(struct dmar_domain *domain)
  * find_domain
  * Note: we use struct pci_dev->dev.archdata.iommu stores the info
  */
-struct dmar_domain *
+static struct dmar_domain *
 find_domain(struct pci_dev *pdev)
 {
 	struct device_domain_info *info;
@@ -2408,3 +2409,111 @@ int __init intel_iommu_init(void)
 	return 0;
 }
 
+void intel_iommu_domain_exit(struct dmar_domain *domain)
+{
+	u64 end;
+
+	/* Domain 0 is reserved, so dont process it */
+	if (!domain)
+		return;
+
+	end = DOMAIN_MAX_ADDR(domain->gaw);
+	end = end & (~PAGE_MASK_4K);
+
+	/* clear ptes */
+	dma_pte_clear_range(domain, 0, end);
+
+	/* free page tables */
+	dma_pte_free_pagetable(domain, 0, end);
+
+	iommu_free_domain(domain);
+	free_domain_mem(domain);
+}
+EXPORT_SYMBOL_GPL(intel_iommu_domain_exit);
+
+struct dmar_domain *intel_iommu_domain_alloc(struct pci_dev *pdev)
+{
+	struct dmar_drhd_unit *drhd;
+	struct dmar_domain *domain;
+	struct intel_iommu *iommu;
+
+	drhd = dmar_find_matched_drhd_unit(pdev);
+	if (!drhd) {
+		printk(KERN_ERR "intel_iommu_domain_alloc: drhd == NULL\n");
+		return NULL;
+	}
+
+	iommu = drhd->iommu;
+	if (!iommu) {
+		printk(KERN_ERR
+			"intel_iommu_domain_alloc: iommu == NULL\n");
+		return NULL;
+	}
+	domain = iommu_alloc_domain(iommu);
+	if (!domain) {
+		printk(KERN_ERR
+			"intel_iommu_domain_alloc: domain == NULL\n");
+		return NULL;
+	}
+	if (domain_init(domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
+		printk(KERN_ERR
+			"intel_iommu_domain_alloc: domain_init() failed\n");
+		intel_iommu_domain_exit(domain);
+		return NULL;
+	}
+	return domain;
+}
+EXPORT_SYMBOL_GPL(intel_iommu_domain_alloc);
+
+int intel_iommu_context_mapping(
+	struct dmar_domain *domain, struct pci_dev *pdev)
+{
+	int rc;
+	rc = domain_context_mapping(domain, pdev);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(intel_iommu_context_mapping);
+
+int intel_iommu_page_mapping(
+	struct dmar_domain *domain, dma_addr_t iova,
+	u64 hpa, size_t size, int prot)
+{
+	int rc;
+	rc = domain_page_mapping(domain, iova, hpa, size, prot);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(intel_iommu_page_mapping);
+
+void intel_iommu_detach_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
+{
+	detach_domain_for_dev(domain, bus, devfn);
+}
+EXPORT_SYMBOL_GPL(intel_iommu_detach_dev);
+
+struct dmar_domain *
+intel_iommu_find_domain(struct pci_dev *pdev)
+{
+	return find_domain(pdev);
+}
+EXPORT_SYMBOL_GPL(intel_iommu_find_domain);
+
+int intel_iommu_found(void)
+{
+	return g_num_of_iommus;
+}
+EXPORT_SYMBOL_GPL(intel_iommu_found);
+
+u64 intel_iommu_iova_to_pfn(struct dmar_domain *domain, u64 iova)
+{
+	struct dma_pte *pte;
+	u64 pfn;
+
+	pfn = 0;
+	pte = addr_to_dma_pte(domain, iova);
+
+	if (pte)
+		pfn = dma_pte_addr(*pte);
+
+	return pfn >> PAGE_SHIFT_4K;
+}
+EXPORT_SYMBOL_GPL(intel_iommu_iova_to_pfn);
diff --git a/drivers/pci/iova.c b/drivers/pci/iova.c
index 3ef4ac0..2287116 100644
--- a/drivers/pci/iova.c
+++ b/drivers/pci/iova.c
@@ -7,7 +7,7 @@
  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  */
 
-#include "iova.h"
+#include <linux/iova.h>
 
 void
 init_iova_domain(struct iova_domain *iovad, unsigned long pfn_32bit)
diff --git a/drivers/pci/intel-iommu.h b/include/linux/intel-iommu.h
similarity index 94%
rename from drivers/pci/intel-iommu.h
rename to include/linux/intel-iommu.h
index afc0ad9..1490fc0 100644
--- a/drivers/pci/intel-iommu.h
+++ b/include/linux/intel-iommu.h
@@ -341,4 +341,15 @@ static inline void iommu_prepare_gfx_mapping(void)
 }
 #endif /* !CONFIG_DMAR_GFX_WA */
 
+void intel_iommu_domain_exit(struct dmar_domain *domain);
+struct dmar_domain *intel_iommu_domain_alloc(struct pci_dev *pdev);
+int intel_iommu_context_mapping(struct dmar_domain *domain,
+				struct pci_dev *pdev);
+int intel_iommu_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
+			     u64 hpa, size_t size, int prot);
+void intel_iommu_detach_dev(struct dmar_domain *domain, u8 bus, u8 devfn);
+struct dmar_domain *intel_iommu_find_domain(struct pci_dev *pdev);
+int intel_iommu_found(void);
+u64 intel_iommu_iova_to_pfn(struct dmar_domain *domain, u64 iova);
+
 #endif
diff --git a/drivers/pci/iova.h b/include/linux/iova.h
similarity index 100%
rename from drivers/pci/iova.h
rename to include/linux/iova.h
-- 
1.5.5.1

[PATCH] KVM: PCIPT: VT-d support

[Ben-Ami Yassour]

From: Kay, Allen M <allen.m.kay@intel.com>

This patch includes the functions to support VT-d for passthrough
devices.

[Ben: fixed memory pinning]

Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
Signed-off-by: Weidong Han <weidong.han@intel.com>
Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
---
 arch/x86/kvm/Makefile      |    2 +-
 arch/x86/kvm/vtd.c         |  189 ++++++++++++++++++++++++++++++++++++++++++++
 include/asm-x86/kvm_host.h |   17 ++++
 include/asm-x86/kvm_para.h |   14 +++
 include/linux/kvm_host.h   |    6 ++
 5 files changed, 227 insertions(+), 1 deletions(-)
 create mode 100644 arch/x86/kvm/vtd.c

diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index d0e940b..5d9d079 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -11,7 +11,7 @@ endif
 EXTRA_CFLAGS += -Ivirt/kvm -Iarch/x86/kvm
 
 kvm-objs := $(common-objs) x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o \
-	i8254.o
+	i8254.o vtd.o
 obj-$(CONFIG_KVM) += kvm.o
 kvm-intel-objs = vmx.o
 obj-$(CONFIG_KVM_INTEL) += kvm-intel.o
diff --git a/arch/x86/kvm/vtd.c b/arch/x86/kvm/vtd.c
new file mode 100644
index 0000000..5abeef1
--- /dev/null
+++ b/arch/x86/kvm/vtd.c
@@ -0,0 +1,189 @@
+/*
+ * Copyright (c) 2006, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ *
+ * Copyright (C) 2006-2008 Intel Corporation
+ * Author: Allen M. Kay <allen.m.kay@intel.com>
+ * Author: Weidong Han <weidong.han@intel.com>
+ */
+
+#include <linux/list.h>
+#include <linux/kvm_host.h>
+#include <linux/pci.h>
+#include <linux/dmar.h>
+#include <linux/intel-iommu.h>
+
+static int kvm_iommu_unmap_memslots(struct kvm *kvm);
+
+int kvm_iommu_map_pages(struct kvm *kvm,
+			gfn_t base_gfn, unsigned long npages)
+{
+	gfn_t gfn = base_gfn;
+	pfn_t pfn;
+	struct page *page;
+	int i, rc;
+	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
+
+	if (!domain)
+		return -EFAULT;
+
+	for (i = 0; i < npages; i++) {
+		pfn = gfn_to_pfn(kvm, gfn);
+		if (pfn_valid(pfn)) {
+			rc = intel_iommu_page_mapping(domain,
+						      gfn << PAGE_SHIFT,
+						      pfn << PAGE_SHIFT,
+						      PAGE_SIZE,
+						      DMA_PTE_READ |
+						      DMA_PTE_WRITE);
+			if (rc) {
+				page = pfn_to_page(pfn);
+				put_page(page);
+			}
+		} else {
+			printk(KERN_DEBUG "kvm_iommu_map_page:"
+			       "invalid pfn=%lx\n", pfn);
+			return 0;
+		}
+		gfn++;
+	}
+	return 0;
+}
+
+static int kvm_iommu_map_memslots(struct kvm *kvm)
+{
+	int i, rc;
+	for (i = 0; i < kvm->nmemslots; i++) {
+		rc = kvm_iommu_map_pages(kvm, kvm->memslots[i].base_gfn,
+					 kvm->memslots[i].npages);
+		if (rc)
+			return rc;
+	}
+	return 0;
+}
+
+int kvm_iommu_map_guest(struct kvm *kvm,
+			struct kvm_pci_passthrough_dev *pci_pt_dev)
+{
+	struct pci_dev *pdev = NULL;
+
+	printk(KERN_DEBUG "VT-d direct map: host bdf = %x:%x:%x\n",
+	       pci_pt_dev->host.busnr,
+	       PCI_SLOT(pci_pt_dev->host.devfn),
+	       PCI_FUNC(pci_pt_dev->host.devfn));
+
+	for_each_pci_dev(pdev) {
+		if ((pdev->bus->number == pci_pt_dev->host.busnr) &&
+		    (pdev->devfn == pci_pt_dev->host.devfn)) {
+			break;
+		}
+	}
+
+	if (pdev == NULL) {
+		if (kvm->arch.intel_iommu_domain) {
+			intel_iommu_domain_exit(kvm->arch.intel_iommu_domain);
+			kvm->arch.intel_iommu_domain = NULL;
+		}
+		return -ENODEV;
+	}
+
+	kvm->arch.intel_iommu_domain = intel_iommu_domain_alloc(pdev);
+
+	if (kvm_iommu_map_memslots(kvm)) {
+		kvm_iommu_unmap_memslots(kvm);
+		return -EFAULT;
+	}
+
+	intel_iommu_detach_dev(kvm->arch.intel_iommu_domain,
+			       pdev->bus->number, pdev->devfn);
+
+	if (intel_iommu_context_mapping(kvm->arch.intel_iommu_domain,
+					pdev)) {
+		printk(KERN_ERR "Domain context map for %s failed",
+		       pci_name(pdev));
+		return -EFAULT;
+	}
+	return 0;
+}
+
+static int kvm_iommu_put_pages(struct kvm *kvm,
+			       gfn_t base_gfn, unsigned long npages)
+{
+	gfn_t gfn = base_gfn;
+	pfn_t pfn;
+	struct page *page;
+	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
+	int i;
+
+	if (!domain)
+		return -EFAULT;
+
+	for (i = 0; i < npages; i++) {
+		pfn = (pfn_t)intel_iommu_iova_to_pfn(domain,
+						     gfn << PAGE_SHIFT);
+
+		if (pfn && pfn_valid(pfn)) {
+			page = pfn_to_page(pfn);
+			put_page(page);
+		}
+		gfn++;
+	}
+	return 0;
+}
+
+static int kvm_iommu_unmap_memslots(struct kvm *kvm)
+{
+	int i, rc;
+	for (i = 0; i < kvm->nmemslots; i++) {
+		rc = kvm_iommu_put_pages(kvm, kvm->memslots[i].base_gfn,
+					 kvm->memslots[i].npages);
+		if (rc)
+			return rc;
+	}
+	return 0;
+}
+
+int kvm_iommu_unmap_guest(struct kvm *kvm)
+{
+	struct kvm_pci_pt_dev_list *entry;
+	struct pci_dev *pdev = NULL;
+	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
+
+	if (!domain)
+		return 0;
+
+	list_for_each_entry(entry, &kvm->arch.pci_pt_dev_head, list) {
+		printk(KERN_DEBUG "VT-d unmap: host bdf = %x:%x:%x\n",
+		       entry->pt_dev.host.busnr,
+		       PCI_SLOT(entry->pt_dev.host.devfn),
+		       PCI_FUNC(entry->pt_dev.host.devfn));
+
+		for_each_pci_dev(pdev) {
+			if ((pdev->bus->number == entry->pt_dev.host.busnr) &&
+			    (pdev->devfn == entry->pt_dev.host.devfn))
+				break;
+		}
+
+		if (pdev == NULL)
+			return -ENODEV;
+
+		/* detach kvm dmar domain */
+		intel_iommu_detach_dev(domain,
+				       pdev->bus->number, pdev->devfn);
+	}
+	kvm_iommu_unmap_memslots(kvm);
+	intel_iommu_domain_exit(domain);
+	return 0;
+}
diff --git a/include/asm-x86/kvm_host.h b/include/asm-x86/kvm_host.h
index 41332f6..9391e57 100644
--- a/include/asm-x86/kvm_host.h
+++ b/include/asm-x86/kvm_host.h
@@ -308,6 +308,21 @@ struct kvm_mem_alias {
 	gfn_t target_gfn;
 };
 
+struct kvm_pci_passthrough_dev_kernel {
+	struct kvm_pci_pt_info guest;
+	struct kvm_pci_pt_info host;
+	struct pci_dev *dev;
+};
+
+/* This list is to store the guest bus:device:function-irq and host
+ * bus:device:function-irq mapping for assigned devices.
+ */
+struct kvm_pci_pt_dev_list {
+	struct list_head list;
+	struct kvm_pci_passthrough_dev_kernel pt_dev;
+};
+
+
 struct kvm_arch{
 	int naliases;
 	struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS];
@@ -320,6 +335,8 @@ struct kvm_arch{
 	 * Hash table of struct kvm_mmu_page.
 	 */
 	struct list_head active_mmu_pages;
+	struct list_head pci_pt_dev_head;
+	struct dmar_domain *intel_iommu_domain;
 	struct kvm_pic *vpic;
 	struct kvm_ioapic *vioapic;
 	struct kvm_pit *vpit;
diff --git a/include/asm-x86/kvm_para.h b/include/asm-x86/kvm_para.h
index 76f3921..88153f4 100644
--- a/include/asm-x86/kvm_para.h
+++ b/include/asm-x86/kvm_para.h
@@ -144,4 +144,18 @@ static inline unsigned int kvm_arch_para_features(void)
 
 #endif
 
+/* Stores information for identifying host PCI devices assigned to the
+ * guest: this is used in the host kernel and in the userspace.
+ */
+struct kvm_pci_pt_info {
+	unsigned char busnr;
+	unsigned int devfn;
+	__u32 irq;
+};
+
+/* Mapping between host and guest PCI device */
+struct kvm_pci_passthrough_dev {
+	struct kvm_pci_pt_info guest;
+	struct kvm_pci_pt_info host;
+};
 #endif
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index fc685c5..424534b 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -278,6 +278,12 @@ int kvm_cpu_has_interrupt(struct kvm_vcpu *v);
 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
 void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
 
+int kvm_iommu_map_pages(struct kvm *kvm, gfn_t base_gfn,
+			unsigned long npages);
+int kvm_iommu_map_guest(struct kvm *kvm,
+			struct kvm_pci_passthrough_dev *pci_pt_dev);
+int kvm_iommu_unmap_guest(struct kvm *kvm);
+
 static inline void kvm_guest_enter(void)
 {
 	account_system_vtime(current);
-- 
1.5.5.1

Re: [PATCH] KVM: PCIPT: VT-d support

[Anthony Liguori]

Ben-Ami Yassour wrote:
> From: Kay, Allen M <allen.m.kay@intel.com>
>
> This patch includes the functions to support VT-d for passthrough
> devices.
>
> [Ben: fixed memory pinning]
>
> Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
> Signed-off-by: Weidong Han <weidong.han@intel.com>
> Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
> ---
>  arch/x86/kvm/Makefile      |    2 +-
>  arch/x86/kvm/vtd.c         |  189 ++++++++++++++++++++++++++++++++++++++++++++
>  include/asm-x86/kvm_host.h |   17 ++++
>  include/asm-x86/kvm_para.h |   14 +++
>  include/linux/kvm_host.h   |    6 ++
>  5 files changed, 227 insertions(+), 1 deletions(-)
>  create mode 100644 arch/x86/kvm/vtd.c
>
> diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
> index d0e940b..5d9d079 100644
> --- a/arch/x86/kvm/Makefile
> +++ b/arch/x86/kvm/Makefile
> @@ -11,7 +11,7 @@ endif
>  EXTRA_CFLAGS += -Ivirt/kvm -Iarch/x86/kvm
>  
>  kvm-objs := $(common-objs) x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o \
> -	i8254.o
> +	i8254.o vtd.o
>  obj-$(CONFIG_KVM) += kvm.o
>  kvm-intel-objs = vmx.o
>  obj-$(CONFIG_KVM_INTEL) += kvm-intel.o
> diff --git a/arch/x86/kvm/vtd.c b/arch/x86/kvm/vtd.c
> new file mode 100644
> index 0000000..5abeef1
> --- /dev/null
> +++ b/arch/x86/kvm/vtd.c
> @@ -0,0 +1,189 @@
> +/*
> + * Copyright (c) 2006, Intel Corporation.
> + *
> + * This program is free software; you can redistribute it and/or modify it
> + * under the terms and conditions of the GNU General Public License,
> + * version 2, as published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope it will be useful, but WITHOUT
> + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
> + * more details.
> + *
> + * You should have received a copy of the GNU General Public License along with
> + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
> + * Place - Suite 330, Boston, MA 02111-1307 USA.
> + *
> + * Copyright (C) 2006-2008 Intel Corporation
> + * Author: Allen M. Kay <allen.m.kay@intel.com>
> + * Author: Weidong Han <weidong.han@intel.com>
> + */
> +
> +#include <linux/list.h>
> +#include <linux/kvm_host.h>
> +#include <linux/pci.h>
> +#include <linux/dmar.h>
> +#include <linux/intel-iommu.h>
> +
> +static int kvm_iommu_unmap_memslots(struct kvm *kvm);
> +
> +int kvm_iommu_map_pages(struct kvm *kvm,
> +			gfn_t base_gfn, unsigned long npages)
> +{
> +	gfn_t gfn = base_gfn;
> +	pfn_t pfn;
> +	struct page *page;
> +	int i, rc;
> +	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
> +
> +	if (!domain)
> +		return -EFAULT;
> +
> +	for (i = 0; i < npages; i++) {
> +		pfn = gfn_to_pfn(kvm, gfn);
> +		if (pfn_valid(pfn)) {
>   

As I've mentioned before, this is wrong.  We should add MMIO pages to 
the VT-d tables but at any rate, pfn_valid() doesn't work for checking 
if something is MMIO.

> +			rc = intel_iommu_page_mapping(domain,
> +						      gfn << PAGE_SHIFT,
> +						      pfn << PAGE_SHIFT,
> +						      PAGE_SIZE,
> +						      DMA_PTE_READ |
> +						      DMA_PTE_WRITE);
> +			if (rc) {
> +				page = pfn_to_page(pfn);
> +				put_page(page);
>   

This should be kvm_release_pfn_clean().

> +			}
> +		} else {
> +			printk(KERN_DEBUG "kvm_iommu_map_page:"
> +			       "invalid pfn=%lx\n", pfn);
> +			return 0;
> +		}
> +		gfn++;
> +	}
> +	return 0;
> +}
> +
> +static int kvm_iommu_map_memslots(struct kvm *kvm)
> +{
> +	int i, rc;
> +	for (i = 0; i < kvm->nmemslots; i++) {
> +		rc = kvm_iommu_map_pages(kvm, kvm->memslots[i].base_gfn,
> +					 kvm->memslots[i].npages);
> +		if (rc)
> +			return rc;
> +	}
> +	return 0;
> +}
> +
> +int kvm_iommu_map_guest(struct kvm *kvm,
> +			struct kvm_pci_passthrough_dev *pci_pt_dev)
> +{
> +	struct pci_dev *pdev = NULL;
> +
> +	printk(KERN_DEBUG "VT-d direct map: host bdf = %x:%x:%x\n",
> +	       pci_pt_dev->host.busnr,
> +	       PCI_SLOT(pci_pt_dev->host.devfn),
> +	       PCI_FUNC(pci_pt_dev->host.devfn));
> +
> +	for_each_pci_dev(pdev) {
> +		if ((pdev->bus->number == pci_pt_dev->host.busnr) &&
> +		    (pdev->devfn == pci_pt_dev->host.devfn)) {
> +			break;
> +		}
> +	}
> +
> +	if (pdev == NULL) {
> +		if (kvm->arch.intel_iommu_domain) {
> +			intel_iommu_domain_exit(kvm->arch.intel_iommu_domain);
> +			kvm->arch.intel_iommu_domain = NULL;
> +		}
> +		return -ENODEV;
> +	}
> +
> +	kvm->arch.intel_iommu_domain = intel_iommu_domain_alloc(pdev);
> +
> +	if (kvm_iommu_map_memslots(kvm)) {
> +		kvm_iommu_unmap_memslots(kvm);
> +		return -EFAULT;
> +	}
> +
> +	intel_iommu_detach_dev(kvm->arch.intel_iommu_domain,
> +			       pdev->bus->number, pdev->devfn);
> +
> +	if (intel_iommu_context_mapping(kvm->arch.intel_iommu_domain,
> +					pdev)) {
> +		printk(KERN_ERR "Domain context map for %s failed",
> +		       pci_name(pdev));
> +		return -EFAULT;
> +	}
> +	return 0;
> +}
> +
> +static int kvm_iommu_put_pages(struct kvm *kvm,
> +			       gfn_t base_gfn, unsigned long npages)
> +{
> +	gfn_t gfn = base_gfn;
> +	pfn_t pfn;
> +	struct page *page;
> +	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
> +	int i;
> +
> +	if (!domain)
> +		return -EFAULT;
> +
> +	for (i = 0; i < npages; i++) {
> +		pfn = (pfn_t)intel_iommu_iova_to_pfn(domain,
> +						     gfn << PAGE_SHIFT);
> +
> +		if (pfn && pfn_valid(pfn)) {
> +			page = pfn_to_page(pfn);
> +			put_page(page);
> +		}
> +		gfn++;
> +	}
> +	return 0;
> +}
> +
> +static int kvm_iommu_unmap_memslots(struct kvm *kvm)
> +{
> +	int i, rc;
> +	for (i = 0; i < kvm->nmemslots; i++) {
> +		rc = kvm_iommu_put_pages(kvm, kvm->memslots[i].base_gfn,
> +					 kvm->memslots[i].npages);
> +		if (rc)
> +			return rc;
> +	}
> +	return 0;
> +}
> +
> +int kvm_iommu_unmap_guest(struct kvm *kvm)
> +{
> +	struct kvm_pci_pt_dev_list *entry;
> +	struct pci_dev *pdev = NULL;
> +	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
> +
> +	if (!domain)
> +		return 0;
> +
> +	list_for_each_entry(entry, &kvm->arch.pci_pt_dev_head, list) {
> +		printk(KERN_DEBUG "VT-d unmap: host bdf = %x:%x:%x\n",
> +		       entry->pt_dev.host.busnr,
> +		       PCI_SLOT(entry->pt_dev.host.devfn),
> +		       PCI_FUNC(entry->pt_dev.host.devfn));
> +
> +		for_each_pci_dev(pdev) {
> +			if ((pdev->bus->number == entry->pt_dev.host.busnr) &&
> +			    (pdev->devfn == entry->pt_dev.host.devfn))
> +				break;
> +		}
> +
> +		if (pdev == NULL)
> +			return -ENODEV;
> +
> +		/* detach kvm dmar domain */
> +		intel_iommu_detach_dev(domain,
> +				       pdev->bus->number, pdev->devfn);
> +	}
> +	kvm_iommu_unmap_memslots(kvm);
> +	intel_iommu_domain_exit(domain);
> +	return 0;
> +}
> diff --git a/include/asm-x86/kvm_host.h b/include/asm-x86/kvm_host.h
> index 41332f6..9391e57 100644
> --- a/include/asm-x86/kvm_host.h
> +++ b/include/asm-x86/kvm_host.h
> @@ -308,6 +308,21 @@ struct kvm_mem_alias {
>  	gfn_t target_gfn;
>  };
>  
> +struct kvm_pci_passthrough_dev_kernel {
> +	struct kvm_pci_pt_info guest;
> +	struct kvm_pci_pt_info host;
> +	struct pci_dev *dev;
> +};
> +
> +/* This list is to store the guest bus:device:function-irq and host
> + * bus:device:function-irq mapping for assigned devices.
> + */
> +struct kvm_pci_pt_dev_list {
> +	struct list_head list;
> +	struct kvm_pci_passthrough_dev_kernel pt_dev;
> +};
> +
> +
>  struct kvm_arch{
>  	int naliases;
>  	struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS];
> @@ -320,6 +335,8 @@ struct kvm_arch{
>  	 * Hash table of struct kvm_mmu_page.
>  	 */
>  	struct list_head active_mmu_pages;
> +	struct list_head pci_pt_dev_head;
> +	struct dmar_domain *intel_iommu_domain;
>  	struct kvm_pic *vpic;
>  	struct kvm_ioapic *vioapic;
>  	struct kvm_pit *vpit;
> diff --git a/include/asm-x86/kvm_para.h b/include/asm-x86/kvm_para.h
> index 76f3921..88153f4 100644
> --- a/include/asm-x86/kvm_para.h
> +++ b/include/asm-x86/kvm_para.h
> @@ -144,4 +144,18 @@ static inline unsigned int kvm_arch_para_features(void)
>  
>  #endif
>  
> +/* Stores information for identifying host PCI devices assigned to the
> + * guest: this is used in the host kernel and in the userspace.
> + */
> +struct kvm_pci_pt_info {
> +	unsigned char busnr;
> +	unsigned int devfn;
> +	__u32 irq;
>   

Userspace should not have to provide the IRQ.  Just use the PCI device 
structure to find it.  We should use different structures for host/guest 
devices.

> +};
> +
> +/* Mapping between host and guest PCI device */
> +struct kvm_pci_passthrough_dev {
> +	struct kvm_pci_pt_info guest;
> +	struct kvm_pci_pt_info host;
> +};
>  #endif
> diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
> index fc685c5..424534b 100644
> --- a/include/linux/kvm_host.h
> +++ b/include/linux/kvm_host.h
> @@ -278,6 +278,12 @@ int kvm_cpu_has_interrupt(struct kvm_vcpu *v);
>  int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
>  void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
>  
> +int kvm_iommu_map_pages(struct kvm *kvm, gfn_t base_gfn,
> +			unsigned long npages);
> +int kvm_iommu_map_guest(struct kvm *kvm,
> +			struct kvm_pci_passthrough_dev *pci_pt_dev);
> +int kvm_iommu_unmap_guest(struct kvm *kvm);
> +
>  static inline void kvm_guest_enter(void)
>  {
>  	account_system_vtime(current);
>   

I'm not seeing these functions called anywhere?  Am I missing something 
obvious or is part of your patch missing?

Regards,

Anthony Liguori

Re: [PATCH] KVM: PCIPT: VT-d support

[Ben-Ami Yassour]

On Wed, 2008-07-09 at 10:49 -0500, Anthony Liguori wrote:
> Ben-Ami Yassour wrote:
> > From: Kay, Allen M <allen.m.kay@intel.com>
> >
> > This patch includes the functions to support VT-d for passthrough
> > devices.
> >
> > [Ben: fixed memory pinning]
> >
> > Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
> > Signed-off-by: Weidong Han <weidong.han@intel.com>
> > Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
> > ---
> >  arch/x86/kvm/Makefile      |    2 +-
> >  arch/x86/kvm/vtd.c         |  189 ++++++++++++++++++++++++++++++++++++++++++++
> >  include/asm-x86/kvm_host.h |   17 ++++
> >  include/asm-x86/kvm_para.h |   14 +++
> >  include/linux/kvm_host.h   |    6 ++
> >  5 files changed, 227 insertions(+), 1 deletions(-)
> >  create mode 100644 arch/x86/kvm/vtd.c
> >
> > diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
> > index d0e940b..5d9d079 100644
> > --- a/arch/x86/kvm/Makefile
> > +++ b/arch/x86/kvm/Makefile
> > @@ -11,7 +11,7 @@ endif
> >  EXTRA_CFLAGS += -Ivirt/kvm -Iarch/x86/kvm
> >  
> >  kvm-objs := $(common-objs) x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o \
> > -	i8254.o
> > +	i8254.o vtd.o
> >  obj-$(CONFIG_KVM) += kvm.o
> >  kvm-intel-objs = vmx.o
> >  obj-$(CONFIG_KVM_INTEL) += kvm-intel.o
> > diff --git a/arch/x86/kvm/vtd.c b/arch/x86/kvm/vtd.c
> > new file mode 100644
> > index 0000000..5abeef1
> > --- /dev/null
> > +++ b/arch/x86/kvm/vtd.c
> > @@ -0,0 +1,189 @@
> > +/*
> > + * Copyright (c) 2006, Intel Corporation.
> > + *
> > + * This program is free software; you can redistribute it and/or modify it
> > + * under the terms and conditions of the GNU General Public License,
> > + * version 2, as published by the Free Software Foundation.
> > + *
> > + * This program is distributed in the hope it will be useful, but WITHOUT
> > + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> > + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
> > + * more details.
> > + *
> > + * You should have received a copy of the GNU General Public License along with
> > + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
> > + * Place - Suite 330, Boston, MA 02111-1307 USA.
> > + *
> > + * Copyright (C) 2006-2008 Intel Corporation
> > + * Author: Allen M. Kay <allen.m.kay@intel.com>
> > + * Author: Weidong Han <weidong.han@intel.com>
> > + */
> > +
> > +#include <linux/list.h>
> > +#include <linux/kvm_host.h>
> > +#include <linux/pci.h>
> > +#include <linux/dmar.h>
> > +#include <linux/intel-iommu.h>
> > +
> > +static int kvm_iommu_unmap_memslots(struct kvm *kvm);
> > +
> > +int kvm_iommu_map_pages(struct kvm *kvm,
> > +			gfn_t base_gfn, unsigned long npages)
> > +{
> > +	gfn_t gfn = base_gfn;
> > +	pfn_t pfn;
> > +	struct page *page;
> > +	int i, rc;
> > +	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
> > +
> > +	if (!domain)
> > +		return -EFAULT;
> > +
> > +	for (i = 0; i < npages; i++) {
> > +		pfn = gfn_to_pfn(kvm, gfn);
> > +		if (pfn_valid(pfn)) {
> >   
> 
> As I've mentioned before, this is wrong.  We should add MMIO pages to 
> the VT-d tables but at any rate, pfn_valid() doesn't work for checking 
> if something is MMIO.

removing the check.

> 
> > +			rc = intel_iommu_page_mapping(domain,
> > +						      gfn << PAGE_SHIFT,
> > +						      pfn << PAGE_SHIFT,
> > +						      PAGE_SIZE,
> > +						      DMA_PTE_READ |
> > +						      DMA_PTE_WRITE);
> > +			if (rc) {
> > +				page = pfn_to_page(pfn);
> > +				put_page(page);
> >   
> 
> This should be kvm_release_pfn_clean().

fixed in the new version.

> 
> > +			}
> > +		} else {
> > +			printk(KERN_DEBUG "kvm_iommu_map_page:"
> > +			       "invalid pfn=%lx\n", pfn);
> > +			return 0;
> > +		}
> > +		gfn++;
> > +	}
> > +	return 0;
> > +}
> > +
> > +static int kvm_iommu_map_memslots(struct kvm *kvm)
> > +{
> > +	int i, rc;
> > +	for (i = 0; i < kvm->nmemslots; i++) {
> > +		rc = kvm_iommu_map_pages(kvm, kvm->memslots[i].base_gfn,
> > +					 kvm->memslots[i].npages);
> > +		if (rc)
> > +			return rc;
> > +	}
> > +	return 0;
> > +}
> > +
> > +int kvm_iommu_map_guest(struct kvm *kvm,
> > +			struct kvm_pci_passthrough_dev *pci_pt_dev)
> > +{
> > +	struct pci_dev *pdev = NULL;
> > +
> > +	printk(KERN_DEBUG "VT-d direct map: host bdf = %x:%x:%x\n",
> > +	       pci_pt_dev->host.busnr,
> > +	       PCI_SLOT(pci_pt_dev->host.devfn),
> > +	       PCI_FUNC(pci_pt_dev->host.devfn));
> > +
> > +	for_each_pci_dev(pdev) {
> > +		if ((pdev->bus->number == pci_pt_dev->host.busnr) &&
> > +		    (pdev->devfn == pci_pt_dev->host.devfn)) {
> > +			break;
> > +		}
> > +	}
> > +
> > +	if (pdev == NULL) {
> > +		if (kvm->arch.intel_iommu_domain) {
> > +			intel_iommu_domain_exit(kvm->arch.intel_iommu_domain);
> > +			kvm->arch.intel_iommu_domain = NULL;
> > +		}
> > +		return -ENODEV;
> > +	}
> > +
> > +	kvm->arch.intel_iommu_domain = intel_iommu_domain_alloc(pdev);
> > +
> > +	if (kvm_iommu_map_memslots(kvm)) {
> > +		kvm_iommu_unmap_memslots(kvm);
> > +		return -EFAULT;
> > +	}
> > +
> > +	intel_iommu_detach_dev(kvm->arch.intel_iommu_domain,
> > +			       pdev->bus->number, pdev->devfn);
> > +
> > +	if (intel_iommu_context_mapping(kvm->arch.intel_iommu_domain,
> > +					pdev)) {
> > +		printk(KERN_ERR "Domain context map for %s failed",
> > +		       pci_name(pdev));
> > +		return -EFAULT;
> > +	}
> > +	return 0;
> > +}
> > +
> > +static int kvm_iommu_put_pages(struct kvm *kvm,
> > +			       gfn_t base_gfn, unsigned long npages)
> > +{
> > +	gfn_t gfn = base_gfn;
> > +	pfn_t pfn;
> > +	struct page *page;
> > +	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
> > +	int i;
> > +
> > +	if (!domain)
> > +		return -EFAULT;
> > +
> > +	for (i = 0; i < npages; i++) {
> > +		pfn = (pfn_t)intel_iommu_iova_to_pfn(domain,
> > +						     gfn << PAGE_SHIFT);
> > +
> > +		if (pfn && pfn_valid(pfn)) {
> > +			page = pfn_to_page(pfn);
> > +			put_page(page);

based on the comment above, changing this to: kvm_release_pfn_clean

> > +		}
> > +		gfn++;
> > +	}
> > +	return 0;
> > +}
> > +
> > +static int kvm_iommu_unmap_memslots(struct kvm *kvm)
> > +{
> > +	int i, rc;
> > +	for (i = 0; i < kvm->nmemslots; i++) {
> > +		rc = kvm_iommu_put_pages(kvm, kvm->memslots[i].base_gfn,
> > +					 kvm->memslots[i].npages);
> > +		if (rc)
> > +			return rc;
> > +	}
> > +	return 0;
> > +}
> > +
> > +int kvm_iommu_unmap_guest(struct kvm *kvm)
> > +{
> > +	struct kvm_pci_pt_dev_list *entry;
> > +	struct pci_dev *pdev = NULL;
> > +	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
> > +
> > +	if (!domain)
> > +		return 0;
> > +
> > +	list_for_each_entry(entry, &kvm->arch.pci_pt_dev_head, list) {
> > +		printk(KERN_DEBUG "VT-d unmap: host bdf = %x:%x:%x\n",
> > +		       entry->pt_dev.host.busnr,
> > +		       PCI_SLOT(entry->pt_dev.host.devfn),
> > +		       PCI_FUNC(entry->pt_dev.host.devfn));
> > +
> > +		for_each_pci_dev(pdev) {
> > +			if ((pdev->bus->number == entry->pt_dev.host.busnr) &&
> > +			    (pdev->devfn == entry->pt_dev.host.devfn))
> > +				break;
> > +		}
> > +
> > +		if (pdev == NULL)
> > +			return -ENODEV;
> > +
> > +		/* detach kvm dmar domain */
> > +		intel_iommu_detach_dev(domain,
> > +				       pdev->bus->number, pdev->devfn);
> > +	}
> > +	kvm_iommu_unmap_memslots(kvm);
> > +	intel_iommu_domain_exit(domain);
> > +	return 0;
> > +}
> > diff --git a/include/asm-x86/kvm_host.h b/include/asm-x86/kvm_host.h
> > index 41332f6..9391e57 100644
> > --- a/include/asm-x86/kvm_host.h
> > +++ b/include/asm-x86/kvm_host.h
> > @@ -308,6 +308,21 @@ struct kvm_mem_alias {
> >  	gfn_t target_gfn;
> >  };
> >  
> > +struct kvm_pci_passthrough_dev_kernel {
> > +	struct kvm_pci_pt_info guest;
> > +	struct kvm_pci_pt_info host;
> > +	struct pci_dev *dev;
> > +};
> > +
> > +/* This list is to store the guest bus:device:function-irq and host
> > + * bus:device:function-irq mapping for assigned devices.
> > + */
> > +struct kvm_pci_pt_dev_list {
> > +	struct list_head list;
> > +	struct kvm_pci_passthrough_dev_kernel pt_dev;
> > +};
> > +
> > +
> >  struct kvm_arch{
> >  	int naliases;
> >  	struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS];
> > @@ -320,6 +335,8 @@ struct kvm_arch{
> >  	 * Hash table of struct kvm_mmu_page.
> >  	 */
> >  	struct list_head active_mmu_pages;
> > +	struct list_head pci_pt_dev_head;
> > +	struct dmar_domain *intel_iommu_domain;
> >  	struct kvm_pic *vpic;
> >  	struct kvm_ioapic *vioapic;
> >  	struct kvm_pit *vpit;
> > diff --git a/include/asm-x86/kvm_para.h b/include/asm-x86/kvm_para.h
> > index 76f3921..88153f4 100644
> > --- a/include/asm-x86/kvm_para.h
> > +++ b/include/asm-x86/kvm_para.h
> > @@ -144,4 +144,18 @@ static inline unsigned int kvm_arch_para_features(void)
> >  
> >  #endif
> >  
> > +/* Stores information for identifying host PCI devices assigned to the
> > + * guest: this is used in the host kernel and in the userspace.
> > + */
> > +struct kvm_pci_pt_info {
> > +	unsigned char busnr;
> > +	unsigned int devfn;
> > +	__u32 irq;
> >   
> 
> Userspace should not have to provide the IRQ.  Just use the PCI device 
> structure to find it.  We should use different structures for host/guest 
> devices.

This is part of the PCIPT code (not the VT-d code).
I agree that it needs to change, but its not related to the VT-d code.
This struct was included in the patch just so the VT-d code can be
compiled we on the main KVM tree without the PCI passthrough code.
The new patch that I sent is rebased on Amit's tree.

> 
> > +};
> > +
> > +/* Mapping between host and guest PCI device */
> > +struct kvm_pci_passthrough_dev {
> > +	struct kvm_pci_pt_info guest;
> > +	struct kvm_pci_pt_info host;
> > +};
> >  #endif
> > diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
> > index fc685c5..424534b 100644
> > --- a/include/linux/kvm_host.h
> > +++ b/include/linux/kvm_host.h
> > @@ -278,6 +278,12 @@ int kvm_cpu_has_interrupt(struct kvm_vcpu *v);
> >  int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
> >  void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
> >  
> > +int kvm_iommu_map_pages(struct kvm *kvm, gfn_t base_gfn,
> > +			unsigned long npages);
> > +int kvm_iommu_map_guest(struct kvm *kvm,
> > +			struct kvm_pci_passthrough_dev *pci_pt_dev);
> > +int kvm_iommu_unmap_guest(struct kvm *kvm);
> > +
> >  static inline void kvm_guest_enter(void)
> >  {
> >  	account_system_vtime(current);
> >   
> 
> I'm not seeing these functions called anywhere?  Am I missing something 
> obvious or is part of your patch missing?

The 3 APIs above are the interface between the PCI passthrough code and
the VT-d code. I am resending the patch rebased on Amit's tree so you
can see the way that the PCI passthrough code is calling this interface.

Thanks,
Ben

> 
> Regards,
> 
> Anthony Liguori

VT-d pci passthrough patches

[Ben-Ami Yassour]

The following two patches contains the VT-d support for pci passthrough
devices.
The first patch contains the changes that are required to the generic
VT-d code.
The second patch contains the changes to KVM.

The VT-d code is used by the pci-passthrough code to map the entire
guest memory in the IOMMU.
Once mmu-notifiers go in, we will be able to support finer grained forms
of protection.

The main differences from the previous version, sent by Weidong, are
fixes for the page pinning.
In the destruction path we need to call put_page for each page that was
mapped in the iommu and pinned during initialization..
The KVM process page tables are already released by the time the KVM vm
destroy path is executed.
Therefore we can not use gfn_to_pfn (which uses get_user_pages) to get
the pages.
The approach in the current patch is to find the pages to unpin by
looking at the IOMMU page table.

This patch applies to the main KVM tree.
The code is not useable without the pci-passthrough code, however we
separate it from the rest of the pci-passthrough code, so it can be
reviewed and merged independently.

Any comments are welcome.

Regards,
Ben

[PATCH] VT-d : changes to support KVM

[Ben-Ami Yassour]

From: Kay, Allen M <allen.m.kay@intel.com>

This patch extends the VT-d driver to support KVM

[Ben: fixed memory pinning]

Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
Signed-off-by: Weidong Han <weidong.han@intel.com>
Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
---
 drivers/pci/dmar.c          |    4 +-
 drivers/pci/intel-iommu.c   |  117 ++++++++++++++-
 drivers/pci/intel-iommu.h   |  344 -------------------------------------------
 drivers/pci/iova.c          |    2 +-
 drivers/pci/iova.h          |   52 -------
 include/linux/intel-iommu.h |  344 +++++++++++++++++++++++++++++++++++++++++++
 include/linux/iova.h        |   52 +++++++
 7 files changed, 512 insertions(+), 403 deletions(-)
 delete mode 100644 drivers/pci/intel-iommu.h
 delete mode 100644 drivers/pci/iova.h
 create mode 100644 include/linux/intel-iommu.h
 create mode 100644 include/linux/iova.h

diff --git a/drivers/pci/dmar.c b/drivers/pci/dmar.c
index f941f60..a58a5b0 100644
--- a/drivers/pci/dmar.c
+++ b/drivers/pci/dmar.c
@@ -26,8 +26,8 @@
 
 #include <linux/pci.h>
 #include <linux/dmar.h>
-#include "iova.h"
-#include "intel-iommu.h"
+#include <linux/iova.h>
+#include <linux/intel-iommu.h>
 
 #undef PREFIX
 #define PREFIX "DMAR:"
diff --git a/drivers/pci/intel-iommu.c b/drivers/pci/intel-iommu.c
index 66c0fd2..b53af37 100644
--- a/drivers/pci/intel-iommu.c
+++ b/drivers/pci/intel-iommu.c
@@ -20,6 +20,7 @@
  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  */
 
+#undef DEBUG
 #include <linux/init.h>
 #include <linux/bitmap.h>
 #include <linux/debugfs.h>
@@ -33,8 +34,8 @@
 #include <linux/dma-mapping.h>
 #include <linux/mempool.h>
 #include <linux/timer.h>
-#include "iova.h"
-#include "intel-iommu.h"
+#include <linux/iova.h>
+#include <linux/intel-iommu.h>
 #include <asm/proto.h> /* force_iommu in this header in x86-64*/
 #include <asm/cacheflush.h>
 #include <asm/gart.h>
@@ -160,7 +161,7 @@ static inline void *alloc_domain_mem(void)
 	return iommu_kmem_cache_alloc(iommu_domain_cache);
 }
 
-static inline void free_domain_mem(void *vaddr)
+static void free_domain_mem(void *vaddr)
 {
 	kmem_cache_free(iommu_domain_cache, vaddr);
 }
@@ -1414,7 +1415,7 @@ static void domain_remove_dev_info(struct dmar_domain *domain)
  * find_domain
  * Note: we use struct pci_dev->dev.archdata.iommu stores the info
  */
-struct dmar_domain *
+static struct dmar_domain *
 find_domain(struct pci_dev *pdev)
 {
 	struct device_domain_info *info;
@@ -2408,3 +2409,111 @@ int __init intel_iommu_init(void)
 	return 0;
 }
 
+void kvm_intel_iommu_domain_exit(struct dmar_domain *domain)
+{
+	u64 end;
+
+	/* Domain 0 is reserved, so dont process it */
+	if (!domain)
+		return;
+
+	end = DOMAIN_MAX_ADDR(domain->gaw);
+	end = end & (~PAGE_MASK_4K);
+
+	/* clear ptes */
+	dma_pte_clear_range(domain, 0, end);
+
+	/* free page tables */
+	dma_pte_free_pagetable(domain, 0, end);
+
+	iommu_free_domain(domain);
+	free_domain_mem(domain);
+}
+EXPORT_SYMBOL_GPL(kvm_intel_iommu_domain_exit);
+
+struct dmar_domain *kvm_intel_iommu_domain_alloc(struct pci_dev *pdev)
+{
+	struct dmar_drhd_unit *drhd;
+	struct dmar_domain *domain;
+	struct intel_iommu *iommu;
+
+	drhd = dmar_find_matched_drhd_unit(pdev);
+	if (!drhd) {
+		printk(KERN_ERR "kvm_intel_iommu_domain_alloc: drhd == NULL\n");
+		return NULL;
+	}
+
+	iommu = drhd->iommu;
+	if (!iommu) {
+		printk(KERN_ERR
+			"kvm_intel_iommu_domain_alloc: iommu == NULL\n");
+		return NULL;
+	}
+	domain = iommu_alloc_domain(iommu);
+	if (!domain) {
+		printk(KERN_ERR
+			"kvm_intel_iommu_domain_alloc: domain == NULL\n");
+		return NULL;
+	}
+	if (domain_init(domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
+		printk(KERN_ERR
+			"kvm_intel_iommu_domain_alloc: domain_init() failed\n");
+		kvm_intel_iommu_domain_exit(domain);
+		return NULL;
+	}
+	return domain;
+}
+EXPORT_SYMBOL_GPL(kvm_intel_iommu_domain_alloc);
+
+int kvm_intel_iommu_context_mapping(
+	struct dmar_domain *domain, struct pci_dev *pdev)
+{
+	int rc;
+	rc = domain_context_mapping(domain, pdev);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(kvm_intel_iommu_context_mapping);
+
+int kvm_intel_iommu_page_mapping(
+	struct dmar_domain *domain, dma_addr_t iova,
+	u64 hpa, size_t size, int prot)
+{
+	int rc;
+	rc = domain_page_mapping(domain, iova, hpa, size, prot);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(kvm_intel_iommu_page_mapping);
+
+void kvm_intel_iommu_detach_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
+{
+	detach_domain_for_dev(domain, bus, devfn);
+}
+EXPORT_SYMBOL_GPL(kvm_intel_iommu_detach_dev);
+
+struct dmar_domain *
+kvm_intel_iommu_find_domain(struct pci_dev *pdev)
+{
+	return find_domain(pdev);
+}
+EXPORT_SYMBOL_GPL(kvm_intel_iommu_find_domain);
+
+int kvm_intel_iommu_found(void)
+{
+	return g_num_of_iommus;
+}
+EXPORT_SYMBOL_GPL(kvm_intel_iommu_found);
+
+u64 kvm_intel_iommu_gfn_to_pfn(struct dmar_domain *domain, u64 gfn)
+{
+	struct dma_pte *pte;
+	u64 pfn;
+
+	pfn = 0;
+	pte = addr_to_dma_pte(domain, gfn << PAGE_SHIFT_4K);
+
+	if (pte)
+		pfn = dma_pte_addr(*pte);
+
+	return pfn >> PAGE_SHIFT_4K;
+}
+EXPORT_SYMBOL_GPL(kvm_intel_iommu_gfn_to_pfn);
diff --git a/drivers/pci/intel-iommu.h b/drivers/pci/intel-iommu.h
deleted file mode 100644
index afc0ad9..0000000
--- a/drivers/pci/intel-iommu.h
+++ /dev/null
@@ -1,344 +0,0 @@
-/*
- * Copyright (c) 2006, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
- * Place - Suite 330, Boston, MA 02111-1307 USA.
- *
- * Copyright (C) 2006-2008 Intel Corporation
- * Author: Ashok Raj <ashok.raj@intel.com>
- * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
- */
-
-#ifndef _INTEL_IOMMU_H_
-#define _INTEL_IOMMU_H_
-
-#include <linux/types.h>
-#include <linux/msi.h>
-#include <linux/sysdev.h>
-#include "iova.h"
-#include <linux/io.h>
-
-/*
- * We need a fixed PAGE_SIZE of 4K irrespective of
- * arch PAGE_SIZE for IOMMU page tables.
- */
-#define PAGE_SHIFT_4K		(12)
-#define PAGE_SIZE_4K		(1UL << PAGE_SHIFT_4K)
-#define PAGE_MASK_4K		(((u64)-1) << PAGE_SHIFT_4K)
-#define PAGE_ALIGN_4K(addr)	(((addr) + PAGE_SIZE_4K - 1) & PAGE_MASK_4K)
-
-#define IOVA_PFN(addr)		((addr) >> PAGE_SHIFT_4K)
-#define DMA_32BIT_PFN		IOVA_PFN(DMA_32BIT_MASK)
-#define DMA_64BIT_PFN		IOVA_PFN(DMA_64BIT_MASK)
-
-/*
- * Intel IOMMU register specification per version 1.0 public spec.
- */
-
-#define	DMAR_VER_REG	0x0	/* Arch version supported by this IOMMU */
-#define	DMAR_CAP_REG	0x8	/* Hardware supported capabilities */
-#define	DMAR_ECAP_REG	0x10	/* Extended capabilities supported */
-#define	DMAR_GCMD_REG	0x18	/* Global command register */
-#define	DMAR_GSTS_REG	0x1c	/* Global status register */
-#define	DMAR_RTADDR_REG	0x20	/* Root entry table */
-#define	DMAR_CCMD_REG	0x28	/* Context command reg */
-#define	DMAR_FSTS_REG	0x34	/* Fault Status register */
-#define	DMAR_FECTL_REG	0x38	/* Fault control register */
-#define	DMAR_FEDATA_REG	0x3c	/* Fault event interrupt data register */
-#define	DMAR_FEADDR_REG	0x40	/* Fault event interrupt addr register */
-#define	DMAR_FEUADDR_REG 0x44	/* Upper address register */
-#define	DMAR_AFLOG_REG	0x58	/* Advanced Fault control */
-#define	DMAR_PMEN_REG	0x64	/* Enable Protected Memory Region */
-#define	DMAR_PLMBASE_REG 0x68	/* PMRR Low addr */
-#define	DMAR_PLMLIMIT_REG 0x6c	/* PMRR low limit */
-#define	DMAR_PHMBASE_REG 0x70	/* pmrr high base addr */
-#define	DMAR_PHMLIMIT_REG 0x78	/* pmrr high limit */
-
-#define OFFSET_STRIDE		(9)
-/*
-#define dmar_readl(dmar, reg) readl(dmar + reg)
-#define dmar_readq(dmar, reg) ({ \
-		u32 lo, hi; \
-		lo = readl(dmar + reg); \
-		hi = readl(dmar + reg + 4); \
-		(((u64) hi) << 32) + lo; })
-*/
-static inline u64 dmar_readq(void __iomem *addr)
-{
-	u32 lo, hi;
-	lo = readl(addr);
-	hi = readl(addr + 4);
-	return (((u64) hi) << 32) + lo;
-}
-
-static inline void dmar_writeq(void __iomem *addr, u64 val)
-{
-	writel((u32)val, addr);
-	writel((u32)(val >> 32), addr + 4);
-}
-
-#define DMAR_VER_MAJOR(v)		(((v) & 0xf0) >> 4)
-#define DMAR_VER_MINOR(v)		((v) & 0x0f)
-
-/*
- * Decoding Capability Register
- */
-#define cap_read_drain(c)	(((c) >> 55) & 1)
-#define cap_write_drain(c)	(((c) >> 54) & 1)
-#define cap_max_amask_val(c)	(((c) >> 48) & 0x3f)
-#define cap_num_fault_regs(c)	((((c) >> 40) & 0xff) + 1)
-#define cap_pgsel_inv(c)	(((c) >> 39) & 1)
-
-#define cap_super_page_val(c)	(((c) >> 34) & 0xf)
-#define cap_super_offset(c)	(((find_first_bit(&cap_super_page_val(c), 4)) \
-					* OFFSET_STRIDE) + 21)
-
-#define cap_fault_reg_offset(c)	((((c) >> 24) & 0x3ff) * 16)
-#define cap_max_fault_reg_offset(c) \
-	(cap_fault_reg_offset(c) + cap_num_fault_regs(c) * 16)
-
-#define cap_zlr(c)		(((c) >> 22) & 1)
-#define cap_isoch(c)		(((c) >> 23) & 1)
-#define cap_mgaw(c)		((((c) >> 16) & 0x3f) + 1)
-#define cap_sagaw(c)		(((c) >> 8) & 0x1f)
-#define cap_caching_mode(c)	(((c) >> 7) & 1)
-#define cap_phmr(c)		(((c) >> 6) & 1)
-#define cap_plmr(c)		(((c) >> 5) & 1)
-#define cap_rwbf(c)		(((c) >> 4) & 1)
-#define cap_afl(c)		(((c) >> 3) & 1)
-#define cap_ndoms(c)		(((unsigned long)1) << (4 + 2 * ((c) & 0x7)))
-/*
- * Extended Capability Register
- */
-
-#define ecap_niotlb_iunits(e)	((((e) >> 24) & 0xff) + 1)
-#define ecap_iotlb_offset(e) 	((((e) >> 8) & 0x3ff) * 16)
-#define ecap_max_iotlb_offset(e) \
-	(ecap_iotlb_offset(e) + ecap_niotlb_iunits(e) * 16)
-#define ecap_coherent(e)	((e) & 0x1)
-
-
-/* IOTLB_REG */
-#define DMA_TLB_GLOBAL_FLUSH (((u64)1) << 60)
-#define DMA_TLB_DSI_FLUSH (((u64)2) << 60)
-#define DMA_TLB_PSI_FLUSH (((u64)3) << 60)
-#define DMA_TLB_IIRG(type) ((type >> 60) & 7)
-#define DMA_TLB_IAIG(val) (((val) >> 57) & 7)
-#define DMA_TLB_READ_DRAIN (((u64)1) << 49)
-#define DMA_TLB_WRITE_DRAIN (((u64)1) << 48)
-#define DMA_TLB_DID(id)	(((u64)((id) & 0xffff)) << 32)
-#define DMA_TLB_IVT (((u64)1) << 63)
-#define DMA_TLB_IH_NONLEAF (((u64)1) << 6)
-#define DMA_TLB_MAX_SIZE (0x3f)
-
-/* PMEN_REG */
-#define DMA_PMEN_EPM (((u32)1)<<31)
-#define DMA_PMEN_PRS (((u32)1)<<0)
-
-/* GCMD_REG */
-#define DMA_GCMD_TE (((u32)1) << 31)
-#define DMA_GCMD_SRTP (((u32)1) << 30)
-#define DMA_GCMD_SFL (((u32)1) << 29)
-#define DMA_GCMD_EAFL (((u32)1) << 28)
-#define DMA_GCMD_WBF (((u32)1) << 27)
-
-/* GSTS_REG */
-#define DMA_GSTS_TES (((u32)1) << 31)
-#define DMA_GSTS_RTPS (((u32)1) << 30)
-#define DMA_GSTS_FLS (((u32)1) << 29)
-#define DMA_GSTS_AFLS (((u32)1) << 28)
-#define DMA_GSTS_WBFS (((u32)1) << 27)
-
-/* CCMD_REG */
-#define DMA_CCMD_ICC (((u64)1) << 63)
-#define DMA_CCMD_GLOBAL_INVL (((u64)1) << 61)
-#define DMA_CCMD_DOMAIN_INVL (((u64)2) << 61)
-#define DMA_CCMD_DEVICE_INVL (((u64)3) << 61)
-#define DMA_CCMD_FM(m) (((u64)((m) & 0x3)) << 32)
-#define DMA_CCMD_MASK_NOBIT 0
-#define DMA_CCMD_MASK_1BIT 1
-#define DMA_CCMD_MASK_2BIT 2
-#define DMA_CCMD_MASK_3BIT 3
-#define DMA_CCMD_SID(s) (((u64)((s) & 0xffff)) << 16)
-#define DMA_CCMD_DID(d) ((u64)((d) & 0xffff))
-
-/* FECTL_REG */
-#define DMA_FECTL_IM (((u32)1) << 31)
-
-/* FSTS_REG */
-#define DMA_FSTS_PPF ((u32)2)
-#define DMA_FSTS_PFO ((u32)1)
-#define dma_fsts_fault_record_index(s) (((s) >> 8) & 0xff)
-
-/* FRCD_REG, 32 bits access */
-#define DMA_FRCD_F (((u32)1) << 31)
-#define dma_frcd_type(d) ((d >> 30) & 1)
-#define dma_frcd_fault_reason(c) (c & 0xff)
-#define dma_frcd_source_id(c) (c & 0xffff)
-#define dma_frcd_page_addr(d) (d & (((u64)-1) << 12)) /* low 64 bit */
-
-/*
- * 0: Present
- * 1-11: Reserved
- * 12-63: Context Ptr (12 - (haw-1))
- * 64-127: Reserved
- */
-struct root_entry {
-	u64	val;
-	u64	rsvd1;
-};
-#define ROOT_ENTRY_NR (PAGE_SIZE_4K/sizeof(struct root_entry))
-static inline bool root_present(struct root_entry *root)
-{
-	return (root->val & 1);
-}
-static inline void set_root_present(struct root_entry *root)
-{
-	root->val |= 1;
-}
-static inline void set_root_value(struct root_entry *root, unsigned long value)
-{
-	root->val |= value & PAGE_MASK_4K;
-}
-
-struct context_entry;
-static inline struct context_entry *
-get_context_addr_from_root(struct root_entry *root)
-{
-	return (struct context_entry *)
-		(root_present(root)?phys_to_virt(
-		root->val & PAGE_MASK_4K):
-		NULL);
-}
-
-/*
- * low 64 bits:
- * 0: present
- * 1: fault processing disable
- * 2-3: translation type
- * 12-63: address space root
- * high 64 bits:
- * 0-2: address width
- * 3-6: aval
- * 8-23: domain id
- */
-struct context_entry {
-	u64 lo;
-	u64 hi;
-};
-#define context_present(c) ((c).lo & 1)
-#define context_fault_disable(c) (((c).lo >> 1) & 1)
-#define context_translation_type(c) (((c).lo >> 2) & 3)
-#define context_address_root(c) ((c).lo & PAGE_MASK_4K)
-#define context_address_width(c) ((c).hi &  7)
-#define context_domain_id(c) (((c).hi >> 8) & ((1 << 16) - 1))
-
-#define context_set_present(c) do {(c).lo |= 1;} while (0)
-#define context_set_fault_enable(c) \
-	do {(c).lo &= (((u64)-1) << 2) | 1;} while (0)
-#define context_set_translation_type(c, val) \
-	do { \
-		(c).lo &= (((u64)-1) << 4) | 3; \
-		(c).lo |= ((val) & 3) << 2; \
-	} while (0)
-#define CONTEXT_TT_MULTI_LEVEL 0
-#define context_set_address_root(c, val) \
-	do {(c).lo |= (val) & PAGE_MASK_4K;} while (0)
-#define context_set_address_width(c, val) do {(c).hi |= (val) & 7;} while (0)
-#define context_set_domain_id(c, val) \
-	do {(c).hi |= ((val) & ((1 << 16) - 1)) << 8;} while (0)
-#define context_clear_entry(c) do {(c).lo = 0; (c).hi = 0;} while (0)
-
-/*
- * 0: readable
- * 1: writable
- * 2-6: reserved
- * 7: super page
- * 8-11: available
- * 12-63: Host physcial address
- */
-struct dma_pte {
-	u64 val;
-};
-#define dma_clear_pte(p)	do {(p).val = 0;} while (0)
-
-#define DMA_PTE_READ (1)
-#define DMA_PTE_WRITE (2)
-
-#define dma_set_pte_readable(p) do {(p).val |= DMA_PTE_READ;} while (0)
-#define dma_set_pte_writable(p) do {(p).val |= DMA_PTE_WRITE;} while (0)
-#define dma_set_pte_prot(p, prot) \
-		do {(p).val = ((p).val & ~3) | ((prot) & 3); } while (0)
-#define dma_pte_addr(p) ((p).val & PAGE_MASK_4K)
-#define dma_set_pte_addr(p, addr) do {\
-		(p).val |= ((addr) & PAGE_MASK_4K); } while (0)
-#define dma_pte_present(p) (((p).val & 3) != 0)
-
-struct intel_iommu;
-
-struct dmar_domain {
-	int	id;			/* domain id */
-	struct intel_iommu *iommu;	/* back pointer to owning iommu */
-
-	struct list_head devices; 	/* all devices' list */
-	struct iova_domain iovad;	/* iova's that belong to this domain */
-
-	struct dma_pte	*pgd;		/* virtual address */
-	spinlock_t	mapping_lock;	/* page table lock */
-	int		gaw;		/* max guest address width */
-
-	/* adjusted guest address width, 0 is level 2 30-bit */
-	int		agaw;
-
-#define DOMAIN_FLAG_MULTIPLE_DEVICES 1
-	int		flags;
-};
-
-/* PCI domain-device relationship */
-struct device_domain_info {
-	struct list_head link;	/* link to domain siblings */
-	struct list_head global; /* link to global list */
-	u8 bus;			/* PCI bus numer */
-	u8 devfn;		/* PCI devfn number */
-	struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
-	struct dmar_domain *domain; /* pointer to domain */
-};
-
-extern int init_dmars(void);
-
-struct intel_iommu {
-	void __iomem	*reg; /* Pointer to hardware regs, virtual addr */
-	u64		cap;
-	u64		ecap;
-	unsigned long 	*domain_ids; /* bitmap of domains */
-	struct dmar_domain **domains; /* ptr to domains */
-	int		seg;
-	u32		gcmd; /* Holds TE, EAFL. Don't need SRTP, SFL, WBF */
-	spinlock_t	lock; /* protect context, domain ids */
-	spinlock_t	register_lock; /* protect register handling */
-	struct root_entry *root_entry; /* virtual address */
-
-	unsigned int irq;
-	unsigned char name[7];    /* Device Name */
-	struct msi_msg saved_msg;
-	struct sys_device sysdev;
-};
-
-#ifndef CONFIG_DMAR_GFX_WA
-static inline void iommu_prepare_gfx_mapping(void)
-{
-	return;
-}
-#endif /* !CONFIG_DMAR_GFX_WA */
-
-#endif
diff --git a/drivers/pci/iova.c b/drivers/pci/iova.c
index 3ef4ac0..2287116 100644
--- a/drivers/pci/iova.c
+++ b/drivers/pci/iova.c
@@ -7,7 +7,7 @@
  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  */
 
-#include "iova.h"
+#include <linux/iova.h>
 
 void
 init_iova_domain(struct iova_domain *iovad, unsigned long pfn_32bit)
diff --git a/drivers/pci/iova.h b/drivers/pci/iova.h
deleted file mode 100644
index 228f6c9..0000000
--- a/drivers/pci/iova.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright (c) 2006, Intel Corporation.
- *
- * This file is released under the GPLv2.
- *
- * Copyright (C) 2006-2008 Intel Corporation
- * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
- *
- */
-
-#ifndef _IOVA_H_
-#define _IOVA_H_
-
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/rbtree.h>
-#include <linux/dma-mapping.h>
-
-/* IO virtual address start page frame number */
-#define IOVA_START_PFN		(1)
-
-/* iova structure */
-struct iova {
-	struct rb_node	node;
-	unsigned long	pfn_hi; /* IOMMU dish out addr hi */
-	unsigned long	pfn_lo; /* IOMMU dish out addr lo */
-};
-
-/* holds all the iova translations for a domain */
-struct iova_domain {
-	spinlock_t	iova_alloc_lock;/* Lock to protect iova  allocation */
-	spinlock_t	iova_rbtree_lock; /* Lock to protect update of rbtree */
-	struct rb_root	rbroot;		/* iova domain rbtree root */
-	struct rb_node	*cached32_node; /* Save last alloced node */
-	unsigned long	dma_32bit_pfn;
-};
-
-struct iova *alloc_iova_mem(void);
-void free_iova_mem(struct iova *iova);
-void free_iova(struct iova_domain *iovad, unsigned long pfn);
-void __free_iova(struct iova_domain *iovad, struct iova *iova);
-struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size,
-	unsigned long limit_pfn,
-	bool size_aligned);
-struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,
-	unsigned long pfn_hi);
-void copy_reserved_iova(struct iova_domain *from, struct iova_domain *to);
-void init_iova_domain(struct iova_domain *iovad, unsigned long pfn_32bit);
-struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn);
-void put_iova_domain(struct iova_domain *iovad);
-
-#endif
diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h
new file mode 100644
index 0000000..afc0ad9
--- /dev/null
+++ b/include/linux/intel-iommu.h
@@ -0,0 +1,344 @@
+/*
+ * Copyright (c) 2006, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ *
+ * Copyright (C) 2006-2008 Intel Corporation
+ * Author: Ashok Raj <ashok.raj@intel.com>
+ * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
+ */
+
+#ifndef _INTEL_IOMMU_H_
+#define _INTEL_IOMMU_H_
+
+#include <linux/types.h>
+#include <linux/msi.h>
+#include <linux/sysdev.h>
+#include "iova.h"
+#include <linux/io.h>
+
+/*
+ * We need a fixed PAGE_SIZE of 4K irrespective of
+ * arch PAGE_SIZE for IOMMU page tables.
+ */
+#define PAGE_SHIFT_4K		(12)
+#define PAGE_SIZE_4K		(1UL << PAGE_SHIFT_4K)
+#define PAGE_MASK_4K		(((u64)-1) << PAGE_SHIFT_4K)
+#define PAGE_ALIGN_4K(addr)	(((addr) + PAGE_SIZE_4K - 1) & PAGE_MASK_4K)
+
+#define IOVA_PFN(addr)		((addr) >> PAGE_SHIFT_4K)
+#define DMA_32BIT_PFN		IOVA_PFN(DMA_32BIT_MASK)
+#define DMA_64BIT_PFN		IOVA_PFN(DMA_64BIT_MASK)
+
+/*
+ * Intel IOMMU register specification per version 1.0 public spec.
+ */
+
+#define	DMAR_VER_REG	0x0	/* Arch version supported by this IOMMU */
+#define	DMAR_CAP_REG	0x8	/* Hardware supported capabilities */
+#define	DMAR_ECAP_REG	0x10	/* Extended capabilities supported */
+#define	DMAR_GCMD_REG	0x18	/* Global command register */
+#define	DMAR_GSTS_REG	0x1c	/* Global status register */
+#define	DMAR_RTADDR_REG	0x20	/* Root entry table */
+#define	DMAR_CCMD_REG	0x28	/* Context command reg */
+#define	DMAR_FSTS_REG	0x34	/* Fault Status register */
+#define	DMAR_FECTL_REG	0x38	/* Fault control register */
+#define	DMAR_FEDATA_REG	0x3c	/* Fault event interrupt data register */
+#define	DMAR_FEADDR_REG	0x40	/* Fault event interrupt addr register */
+#define	DMAR_FEUADDR_REG 0x44	/* Upper address register */
+#define	DMAR_AFLOG_REG	0x58	/* Advanced Fault control */
+#define	DMAR_PMEN_REG	0x64	/* Enable Protected Memory Region */
+#define	DMAR_PLMBASE_REG 0x68	/* PMRR Low addr */
+#define	DMAR_PLMLIMIT_REG 0x6c	/* PMRR low limit */
+#define	DMAR_PHMBASE_REG 0x70	/* pmrr high base addr */
+#define	DMAR_PHMLIMIT_REG 0x78	/* pmrr high limit */
+
+#define OFFSET_STRIDE		(9)
+/*
+#define dmar_readl(dmar, reg) readl(dmar + reg)
+#define dmar_readq(dmar, reg) ({ \
+		u32 lo, hi; \
+		lo = readl(dmar + reg); \
+		hi = readl(dmar + reg + 4); \
+		(((u64) hi) << 32) + lo; })
+*/
+static inline u64 dmar_readq(void __iomem *addr)
+{
+	u32 lo, hi;
+	lo = readl(addr);
+	hi = readl(addr + 4);
+	return (((u64) hi) << 32) + lo;
+}
+
+static inline void dmar_writeq(void __iomem *addr, u64 val)
+{
+	writel((u32)val, addr);
+	writel((u32)(val >> 32), addr + 4);
+}
+
+#define DMAR_VER_MAJOR(v)		(((v) & 0xf0) >> 4)
+#define DMAR_VER_MINOR(v)		((v) & 0x0f)
+
+/*
+ * Decoding Capability Register
+ */
+#define cap_read_drain(c)	(((c) >> 55) & 1)
+#define cap_write_drain(c)	(((c) >> 54) & 1)
+#define cap_max_amask_val(c)	(((c) >> 48) & 0x3f)
+#define cap_num_fault_regs(c)	((((c) >> 40) & 0xff) + 1)
+#define cap_pgsel_inv(c)	(((c) >> 39) & 1)
+
+#define cap_super_page_val(c)	(((c) >> 34) & 0xf)
+#define cap_super_offset(c)	(((find_first_bit(&cap_super_page_val(c), 4)) \
+					* OFFSET_STRIDE) + 21)
+
+#define cap_fault_reg_offset(c)	((((c) >> 24) & 0x3ff) * 16)
+#define cap_max_fault_reg_offset(c) \
+	(cap_fault_reg_offset(c) + cap_num_fault_regs(c) * 16)
+
+#define cap_zlr(c)		(((c) >> 22) & 1)
+#define cap_isoch(c)		(((c) >> 23) & 1)
+#define cap_mgaw(c)		((((c) >> 16) & 0x3f) + 1)
+#define cap_sagaw(c)		(((c) >> 8) & 0x1f)
+#define cap_caching_mode(c)	(((c) >> 7) & 1)
+#define cap_phmr(c)		(((c) >> 6) & 1)
+#define cap_plmr(c)		(((c) >> 5) & 1)
+#define cap_rwbf(c)		(((c) >> 4) & 1)
+#define cap_afl(c)		(((c) >> 3) & 1)
+#define cap_ndoms(c)		(((unsigned long)1) << (4 + 2 * ((c) & 0x7)))
+/*
+ * Extended Capability Register
+ */
+
+#define ecap_niotlb_iunits(e)	((((e) >> 24) & 0xff) + 1)
+#define ecap_iotlb_offset(e) 	((((e) >> 8) & 0x3ff) * 16)
+#define ecap_max_iotlb_offset(e) \
+	(ecap_iotlb_offset(e) + ecap_niotlb_iunits(e) * 16)
+#define ecap_coherent(e)	((e) & 0x1)
+
+
+/* IOTLB_REG */
+#define DMA_TLB_GLOBAL_FLUSH (((u64)1) << 60)
+#define DMA_TLB_DSI_FLUSH (((u64)2) << 60)
+#define DMA_TLB_PSI_FLUSH (((u64)3) << 60)
+#define DMA_TLB_IIRG(type) ((type >> 60) & 7)
+#define DMA_TLB_IAIG(val) (((val) >> 57) & 7)
+#define DMA_TLB_READ_DRAIN (((u64)1) << 49)
+#define DMA_TLB_WRITE_DRAIN (((u64)1) << 48)
+#define DMA_TLB_DID(id)	(((u64)((id) & 0xffff)) << 32)
+#define DMA_TLB_IVT (((u64)1) << 63)
+#define DMA_TLB_IH_NONLEAF (((u64)1) << 6)
+#define DMA_TLB_MAX_SIZE (0x3f)
+
+/* PMEN_REG */
+#define DMA_PMEN_EPM (((u32)1)<<31)
+#define DMA_PMEN_PRS (((u32)1)<<0)
+
+/* GCMD_REG */
+#define DMA_GCMD_TE (((u32)1) << 31)
+#define DMA_GCMD_SRTP (((u32)1) << 30)
+#define DMA_GCMD_SFL (((u32)1) << 29)
+#define DMA_GCMD_EAFL (((u32)1) << 28)
+#define DMA_GCMD_WBF (((u32)1) << 27)
+
+/* GSTS_REG */
+#define DMA_GSTS_TES (((u32)1) << 31)
+#define DMA_GSTS_RTPS (((u32)1) << 30)
+#define DMA_GSTS_FLS (((u32)1) << 29)
+#define DMA_GSTS_AFLS (((u32)1) << 28)
+#define DMA_GSTS_WBFS (((u32)1) << 27)
+
+/* CCMD_REG */
+#define DMA_CCMD_ICC (((u64)1) << 63)
+#define DMA_CCMD_GLOBAL_INVL (((u64)1) << 61)
+#define DMA_CCMD_DOMAIN_INVL (((u64)2) << 61)
+#define DMA_CCMD_DEVICE_INVL (((u64)3) << 61)
+#define DMA_CCMD_FM(m) (((u64)((m) & 0x3)) << 32)
+#define DMA_CCMD_MASK_NOBIT 0
+#define DMA_CCMD_MASK_1BIT 1
+#define DMA_CCMD_MASK_2BIT 2
+#define DMA_CCMD_MASK_3BIT 3
+#define DMA_CCMD_SID(s) (((u64)((s) & 0xffff)) << 16)
+#define DMA_CCMD_DID(d) ((u64)((d) & 0xffff))
+
+/* FECTL_REG */
+#define DMA_FECTL_IM (((u32)1) << 31)
+
+/* FSTS_REG */
+#define DMA_FSTS_PPF ((u32)2)
+#define DMA_FSTS_PFO ((u32)1)
+#define dma_fsts_fault_record_index(s) (((s) >> 8) & 0xff)
+
+/* FRCD_REG, 32 bits access */
+#define DMA_FRCD_F (((u32)1) << 31)
+#define dma_frcd_type(d) ((d >> 30) & 1)
+#define dma_frcd_fault_reason(c) (c & 0xff)
+#define dma_frcd_source_id(c) (c & 0xffff)
+#define dma_frcd_page_addr(d) (d & (((u64)-1) << 12)) /* low 64 bit */
+
+/*
+ * 0: Present
+ * 1-11: Reserved
+ * 12-63: Context Ptr (12 - (haw-1))
+ * 64-127: Reserved
+ */
+struct root_entry {
+	u64	val;
+	u64	rsvd1;
+};
+#define ROOT_ENTRY_NR (PAGE_SIZE_4K/sizeof(struct root_entry))
+static inline bool root_present(struct root_entry *root)
+{
+	return (root->val & 1);
+}
+static inline void set_root_present(struct root_entry *root)
+{
+	root->val |= 1;
+}
+static inline void set_root_value(struct root_entry *root, unsigned long value)
+{
+	root->val |= value & PAGE_MASK_4K;
+}
+
+struct context_entry;
+static inline struct context_entry *
+get_context_addr_from_root(struct root_entry *root)
+{
+	return (struct context_entry *)
+		(root_present(root)?phys_to_virt(
+		root->val & PAGE_MASK_4K):
+		NULL);
+}
+
+/*
+ * low 64 bits:
+ * 0: present
+ * 1: fault processing disable
+ * 2-3: translation type
+ * 12-63: address space root
+ * high 64 bits:
+ * 0-2: address width
+ * 3-6: aval
+ * 8-23: domain id
+ */
+struct context_entry {
+	u64 lo;
+	u64 hi;
+};
+#define context_present(c) ((c).lo & 1)
+#define context_fault_disable(c) (((c).lo >> 1) & 1)
+#define context_translation_type(c) (((c).lo >> 2) & 3)
+#define context_address_root(c) ((c).lo & PAGE_MASK_4K)
+#define context_address_width(c) ((c).hi &  7)
+#define context_domain_id(c) (((c).hi >> 8) & ((1 << 16) - 1))
+
+#define context_set_present(c) do {(c).lo |= 1;} while (0)
+#define context_set_fault_enable(c) \
+	do {(c).lo &= (((u64)-1) << 2) | 1;} while (0)
+#define context_set_translation_type(c, val) \
+	do { \
+		(c).lo &= (((u64)-1) << 4) | 3; \
+		(c).lo |= ((val) & 3) << 2; \
+	} while (0)
+#define CONTEXT_TT_MULTI_LEVEL 0
+#define context_set_address_root(c, val) \
+	do {(c).lo |= (val) & PAGE_MASK_4K;} while (0)
+#define context_set_address_width(c, val) do {(c).hi |= (val) & 7;} while (0)
+#define context_set_domain_id(c, val) \
+	do {(c).hi |= ((val) & ((1 << 16) - 1)) << 8;} while (0)
+#define context_clear_entry(c) do {(c).lo = 0; (c).hi = 0;} while (0)
+
+/*
+ * 0: readable
+ * 1: writable
+ * 2-6: reserved
+ * 7: super page
+ * 8-11: available
+ * 12-63: Host physcial address
+ */
+struct dma_pte {
+	u64 val;
+};
+#define dma_clear_pte(p)	do {(p).val = 0;} while (0)
+
+#define DMA_PTE_READ (1)
+#define DMA_PTE_WRITE (2)
+
+#define dma_set_pte_readable(p) do {(p).val |= DMA_PTE_READ;} while (0)
+#define dma_set_pte_writable(p) do {(p).val |= DMA_PTE_WRITE;} while (0)
+#define dma_set_pte_prot(p, prot) \
+		do {(p).val = ((p).val & ~3) | ((prot) & 3); } while (0)
+#define dma_pte_addr(p) ((p).val & PAGE_MASK_4K)
+#define dma_set_pte_addr(p, addr) do {\
+		(p).val |= ((addr) & PAGE_MASK_4K); } while (0)
+#define dma_pte_present(p) (((p).val & 3) != 0)
+
+struct intel_iommu;
+
+struct dmar_domain {
+	int	id;			/* domain id */
+	struct intel_iommu *iommu;	/* back pointer to owning iommu */
+
+	struct list_head devices; 	/* all devices' list */
+	struct iova_domain iovad;	/* iova's that belong to this domain */
+
+	struct dma_pte	*pgd;		/* virtual address */
+	spinlock_t	mapping_lock;	/* page table lock */
+	int		gaw;		/* max guest address width */
+
+	/* adjusted guest address width, 0 is level 2 30-bit */
+	int		agaw;
+
+#define DOMAIN_FLAG_MULTIPLE_DEVICES 1
+	int		flags;
+};
+
+/* PCI domain-device relationship */
+struct device_domain_info {
+	struct list_head link;	/* link to domain siblings */
+	struct list_head global; /* link to global list */
+	u8 bus;			/* PCI bus numer */
+	u8 devfn;		/* PCI devfn number */
+	struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
+	struct dmar_domain *domain; /* pointer to domain */
+};
+
+extern int init_dmars(void);
+
+struct intel_iommu {
+	void __iomem	*reg; /* Pointer to hardware regs, virtual addr */
+	u64		cap;
+	u64		ecap;
+	unsigned long 	*domain_ids; /* bitmap of domains */
+	struct dmar_domain **domains; /* ptr to domains */
+	int		seg;
+	u32		gcmd; /* Holds TE, EAFL. Don't need SRTP, SFL, WBF */
+	spinlock_t	lock; /* protect context, domain ids */
+	spinlock_t	register_lock; /* protect register handling */
+	struct root_entry *root_entry; /* virtual address */
+
+	unsigned int irq;
+	unsigned char name[7];    /* Device Name */
+	struct msi_msg saved_msg;
+	struct sys_device sysdev;
+};
+
+#ifndef CONFIG_DMAR_GFX_WA
+static inline void iommu_prepare_gfx_mapping(void)
+{
+	return;
+}
+#endif /* !CONFIG_DMAR_GFX_WA */
+
+#endif
diff --git a/include/linux/iova.h b/include/linux/iova.h
new file mode 100644
index 0000000..228f6c9
--- /dev/null
+++ b/include/linux/iova.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2006, Intel Corporation.
+ *
+ * This file is released under the GPLv2.
+ *
+ * Copyright (C) 2006-2008 Intel Corporation
+ * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
+ *
+ */
+
+#ifndef _IOVA_H_
+#define _IOVA_H_
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/rbtree.h>
+#include <linux/dma-mapping.h>
+
+/* IO virtual address start page frame number */
+#define IOVA_START_PFN		(1)
+
+/* iova structure */
+struct iova {
+	struct rb_node	node;
+	unsigned long	pfn_hi; /* IOMMU dish out addr hi */
+	unsigned long	pfn_lo; /* IOMMU dish out addr lo */
+};
+
+/* holds all the iova translations for a domain */
+struct iova_domain {
+	spinlock_t	iova_alloc_lock;/* Lock to protect iova  allocation */
+	spinlock_t	iova_rbtree_lock; /* Lock to protect update of rbtree */
+	struct rb_root	rbroot;		/* iova domain rbtree root */
+	struct rb_node	*cached32_node; /* Save last alloced node */
+	unsigned long	dma_32bit_pfn;
+};
+
+struct iova *alloc_iova_mem(void);
+void free_iova_mem(struct iova *iova);
+void free_iova(struct iova_domain *iovad, unsigned long pfn);
+void __free_iova(struct iova_domain *iovad, struct iova *iova);
+struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size,
+	unsigned long limit_pfn,
+	bool size_aligned);
+struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,
+	unsigned long pfn_hi);
+void copy_reserved_iova(struct iova_domain *from, struct iova_domain *to);
+void init_iova_domain(struct iova_domain *iovad, unsigned long pfn_32bit);
+struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn);
+void put_iova_domain(struct iova_domain *iovad);
+
+#endif
-- 
1.5.5.1

[PATCH] KVM: PCIPT: VT-d support

[Ben-Ami Yassour]

From: Kay, Allen M <allen.m.kay@intel.com>

This patch includes the functions to support VT-d for passthrough
devices.

[Ben: fixed memory pinning]

Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
Signed-off-by: Weidong Han <weidong.han@intel.com>
Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
---
 arch/x86/kvm/Makefile      |    2 +-
 arch/x86/kvm/vtd.c         |  191 ++++++++++++++++++++++++++++++++++++++++++++
 arch/x86/kvm/vtd.h         |   36 ++++++++
 include/asm-x86/kvm_host.h |   17 ++++
 include/asm-x86/kvm_para.h |   14 +++
 include/linux/kvm_host.h   |    8 ++
 6 files changed, 267 insertions(+), 1 deletions(-)
 create mode 100644 arch/x86/kvm/vtd.c
 create mode 100644 arch/x86/kvm/vtd.h

diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index d0e940b..5d9d079 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -11,7 +11,7 @@ endif
 EXTRA_CFLAGS += -Ivirt/kvm -Iarch/x86/kvm
 
 kvm-objs := $(common-objs) x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o \
-	i8254.o
+	i8254.o vtd.o
 obj-$(CONFIG_KVM) += kvm.o
 kvm-intel-objs = vmx.o
 obj-$(CONFIG_KVM_INTEL) += kvm-intel.o
diff --git a/arch/x86/kvm/vtd.c b/arch/x86/kvm/vtd.c
new file mode 100644
index 0000000..f665357
--- /dev/null
+++ b/arch/x86/kvm/vtd.c
@@ -0,0 +1,191 @@
+/*
+ * Copyright (c) 2006, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ *
+ * Copyright (C) 2006-2008 Intel Corporation
+ * Author: Allen M. Kay <allen.m.kay@intel.com>
+ * Author: Weidong Han <weidong.han@intel.com>
+ */
+
+#include <linux/list.h>
+#include <linux/kvm_host.h>
+#include <linux/pci.h>
+#include <linux/dmar.h>
+#include <linux/intel-iommu.h>
+#include "vtd.h"
+
+static int kvm_iommu_unmap_memslots(struct kvm *kvm);
+
+int kvm_iommu_map_pages(struct kvm *kvm,
+			gfn_t base_gfn, unsigned long npages)
+{
+	gfn_t gfn = base_gfn;
+	pfn_t pfn;
+	struct page *page;
+	int i, rc;
+	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
+
+	if (!domain)
+		return -EFAULT;
+
+	for (i = 0; i < npages; i++) {
+		pfn = gfn_to_pfn(kvm, gfn);
+		if (pfn_valid(pfn)) {
+			rc = kvm_intel_iommu_page_mapping(domain,
+							  gfn << PAGE_SHIFT,
+							  pfn << PAGE_SHIFT,
+							  PAGE_SIZE,
+							  DMA_PTE_READ |
+							  DMA_PTE_WRITE);
+			if (rc) {
+				page = pfn_to_page(pfn);
+				put_page(page);
+			}
+		} else {
+			printk(KERN_DEBUG "kvm_iommu_map_page:"
+			       "invalid pfn=%lx\n", pfn);
+			return 0;
+		}
+		gfn++;
+	}
+	return 0;
+}
+
+static int kvm_iommu_map_memslots(struct kvm *kvm)
+{
+	int i, rc;
+	for (i = 0; i < kvm->nmemslots; i++) {
+		rc = kvm_iommu_map_pages(kvm, kvm->memslots[i].base_gfn,
+					 kvm->memslots[i].npages);
+		if (rc)
+			return rc;
+	}
+	return 0;
+}
+
+int kvm_iommu_map_guest(struct kvm *kvm,
+			struct kvm_pci_passthrough_dev *pci_pt_dev)
+{
+	struct pci_dev *pdev = NULL;
+
+	printk(KERN_DEBUG "VT-d direct map: host bdf = %x:%x:%x\n",
+	       pci_pt_dev->host.busnr,
+	       PCI_SLOT(pci_pt_dev->host.devfn),
+	       PCI_FUNC(pci_pt_dev->host.devfn));
+
+	for_each_pci_dev(pdev) {
+		if ((pdev->bus->number == pci_pt_dev->host.busnr) &&
+		    (pdev->devfn == pci_pt_dev->host.devfn)) {
+			break;
+		}
+	}
+
+	if (pdev == NULL) {
+		if (kvm->arch.intel_iommu_domain) {
+			kvm_intel_iommu_domain_exit(kvm->arch.
+						    intel_iommu_domain);
+			kvm->arch.intel_iommu_domain = NULL;
+		}
+		return -ENODEV;
+	}
+
+	kvm->arch.intel_iommu_domain = kvm_intel_iommu_domain_alloc(pdev);
+
+	if (kvm_iommu_map_memslots(kvm)) {
+		kvm_iommu_unmap_memslots(kvm);
+		return -EFAULT;
+	}
+
+	kvm_intel_iommu_detach_dev(kvm->arch.intel_iommu_domain,
+				   pdev->bus->number, pdev->devfn);
+
+	if (kvm_intel_iommu_context_mapping(kvm->arch.intel_iommu_domain,
+					    pdev)) {
+		printk(KERN_ERR "Domain context map for %s failed",
+		       pci_name(pdev));
+		return -EFAULT;
+	}
+	return 0;
+}
+
+static int kvm_iommu_put_pages(struct kvm *kvm,
+			       gfn_t base_gfn, unsigned long npages)
+{
+	gfn_t gfn = base_gfn;
+	pfn_t pfn;
+	struct page *page;
+	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
+	int i;
+
+	if (!domain)
+		return -EFAULT;
+
+	for (i = 0; i < npages; i++) {
+		pfn = (pfn_t)kvm_intel_iommu_gfn_to_pfn(domain,
+							gfn);
+
+		if (pfn && pfn_valid(pfn)) {
+			page = pfn_to_page(pfn);
+			put_page(page);
+		}
+		gfn++;
+	}
+	return 0;
+}
+
+static int kvm_iommu_unmap_memslots(struct kvm *kvm)
+{
+	int i, rc;
+	for (i = 0; i < kvm->nmemslots; i++) {
+		rc = kvm_iommu_put_pages(kvm, kvm->memslots[i].base_gfn,
+					 kvm->memslots[i].npages);
+		if (rc)
+			return rc;
+	}
+	return 0;
+}
+
+int kvm_iommu_unmap_guest(struct kvm *kvm)
+{
+	struct kvm_pci_pt_dev_list *entry;
+	struct pci_dev *pdev = NULL;
+	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
+
+	if (!domain)
+		return 0;
+
+	list_for_each_entry(entry, &kvm->arch.pci_pt_dev_head, list) {
+		printk(KERN_DEBUG "VT-d unmap: host bdf = %x:%x:%x\n",
+		       entry->pt_dev.host.busnr,
+		       PCI_SLOT(entry->pt_dev.host.devfn),
+		       PCI_FUNC(entry->pt_dev.host.devfn));
+
+		for_each_pci_dev(pdev) {
+			if ((pdev->bus->number == entry->pt_dev.host.busnr) &&
+			    (pdev->devfn == entry->pt_dev.host.devfn))
+				break;
+		}
+
+		if (pdev == NULL)
+			return -ENODEV;
+
+		/* detach kvm dmar domain */
+		kvm_intel_iommu_detach_dev(domain,
+					   pdev->bus->number, pdev->devfn);
+	}
+	kvm_iommu_unmap_memslots(kvm);
+	kvm_intel_iommu_domain_exit(domain);
+	return 0;
+}
diff --git a/arch/x86/kvm/vtd.h b/arch/x86/kvm/vtd.h
new file mode 100644
index 0000000..9823cfd
--- /dev/null
+++ b/arch/x86/kvm/vtd.h
@@ -0,0 +1,36 @@
+#ifndef __KVM_INTEL_IOMMU_H
+#define __KVM_INTEL_IOMMU_H
+
+/*
+ * Copyright (c) 2006, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ *
+ * Copyright (C) 2006-2008 Intel Corporation
+ * Author: Allen M. Kay <allen.m.kay@intel.com>
+ * Author: Weidong Han <weidong.han@intel.com>
+ */
+
+/* #define DEBUG */
+
+int kvm_intel_iommu_context_mapping(struct dmar_domain *d,
+			struct pci_dev *pdev);
+int kvm_intel_iommu_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
+			u64 hpa, size_t size, int prot);
+void kvm_intel_iommu_detach_dev(struct dmar_domain *domain, u8 bus, u8 devfn);
+struct dmar_domain *kvm_intel_iommu_domain_alloc(struct pci_dev *pdev);
+void kvm_intel_iommu_domain_exit(struct dmar_domain *domain);
+struct dmar_domain *kvm_intel_iommu_find_domain(struct pci_dev *pdev);
+
+#endif
diff --git a/include/asm-x86/kvm_host.h b/include/asm-x86/kvm_host.h
index bae1b76..19194a2 100644
--- a/include/asm-x86/kvm_host.h
+++ b/include/asm-x86/kvm_host.h
@@ -308,6 +308,21 @@ struct kvm_mem_alias {
 	gfn_t target_gfn;
 };
 
+struct kvm_pci_passthrough_dev_kernel {
+	struct kvm_pci_pt_info guest;
+	struct kvm_pci_pt_info host;
+	struct pci_dev *dev;
+};
+
+/* This list is to store the guest bus:device:function-irq and host
+ * bus:device:function-irq mapping for assigned devices.
+ */
+struct kvm_pci_pt_dev_list {
+	struct list_head list;
+	struct kvm_pci_passthrough_dev_kernel pt_dev;
+};
+
+
 struct kvm_arch{
 	int naliases;
 	struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS];
@@ -320,6 +335,8 @@ struct kvm_arch{
 	 * Hash table of struct kvm_mmu_page.
 	 */
 	struct list_head active_mmu_pages;
+	struct list_head pci_pt_dev_head;
+	struct dmar_domain *intel_iommu_domain;
 	struct kvm_pic *vpic;
 	struct kvm_ioapic *vioapic;
 	struct kvm_pit *vpit;
diff --git a/include/asm-x86/kvm_para.h b/include/asm-x86/kvm_para.h
index bfd9900..a833f6b 100644
--- a/include/asm-x86/kvm_para.h
+++ b/include/asm-x86/kvm_para.h
@@ -139,4 +139,18 @@ static inline unsigned int kvm_arch_para_features(void)
 
 #endif
 
+/* Stores information for identifying host PCI devices assigned to the
+ * guest: this is used in the host kernel and in the userspace.
+ */
+struct kvm_pci_pt_info {
+	unsigned char busnr;
+	unsigned int devfn;
+	__u32 irq;
+};
+
+/* Mapping between host and guest PCI device */
+struct kvm_pci_passthrough_dev {
+	struct kvm_pci_pt_info guest;
+	struct kvm_pci_pt_info host;
+};
 #endif
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index d220b49..39b7c32 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -278,6 +278,14 @@ int kvm_cpu_has_interrupt(struct kvm_vcpu *v);
 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
 void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
 
+int kvm_iommu_map_pages(struct kvm *kvm, gfn_t base_gfn,
+			unsigned long npages);
+int kvm_iommu_map_guest(struct kvm *kvm,
+			struct kvm_pci_passthrough_dev *pci_pt_dev);
+int kvm_iommu_unmap_guest(struct kvm *kvm);
+int kvm_intel_iommu_found(void);
+u64 kvm_intel_iommu_gfn_to_pfn(struct dmar_domain *domain, u64 gfn);
+
 static inline void kvm_guest_enter(void)
 {
 	account_system_vtime(current);
-- 
1.5.5.1

RE: [PATCH] KVM: PCIPT: VT-d support

[Han, Weidong]

Ben-Ami Yassour wrote:
> From: Kay, Allen M <allen.m.kay@intel.com>
> 
> This patch includes the functions to support VT-d for passthrough
> devices.
> 
> [Ben: fixed memory pinning]
> 
> Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
> Signed-off-by: Weidong Han <weidong.han@intel.com>
> Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
> ---
>  arch/x86/kvm/Makefile      |    2 +-
>  arch/x86/kvm/vtd.c         |  191
>  ++++++++++++++++++++++++++++++++++++++++++++ arch/x86/kvm/vtd.h     
>  |   36 ++++++++ include/asm-x86/kvm_host.h |   17 ++++
>  include/asm-x86/kvm_para.h |   14 +++
>  include/linux/kvm_host.h   |    8 ++
>  6 files changed, 267 insertions(+), 1 deletions(-)
>  create mode 100644 arch/x86/kvm/vtd.c
>  create mode 100644 arch/x86/kvm/vtd.h
> 
> diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
> index d220b49..39b7c32 100644
> --- a/include/linux/kvm_host.h
> +++ b/include/linux/kvm_host.h
> @@ -278,6 +278,14 @@ int kvm_cpu_has_interrupt(struct kvm_vcpu *v);
>  int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
>  void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
> 
> +int kvm_iommu_map_pages(struct kvm *kvm, gfn_t base_gfn,
> +			unsigned long npages);
> +int kvm_iommu_map_guest(struct kvm *kvm,
> +			struct kvm_pci_passthrough_dev *pci_pt_dev);
> +int kvm_iommu_unmap_guest(struct kvm *kvm);
> +int kvm_intel_iommu_found(void);
> +u64 kvm_intel_iommu_gfn_to_pfn(struct dmar_domain *domain, u64 gfn);
> +

Needn't to add kvm_intel_iommu_gfn_to_pfn() declaration here. It's
exposed by VT-d driver.

RE: [PATCH] KVM: PCIPT: VT-d support

[Ben-Ami Yassour]

On Thu, 2008-07-03 at 12:55 +0800, Han, Weidong wrote:
> Ben-Ami Yassour wrote:
> > From: Kay, Allen M <allen.m.kay@intel.com>
> > 
> > This patch includes the functions to support VT-d for passthrough
> > devices.
> > 
> > [Ben: fixed memory pinning]
> > 
> > Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
> > Signed-off-by: Weidong Han <weidong.han@intel.com>
> > Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
> > ---
> >  arch/x86/kvm/Makefile      |    2 +-
> >  arch/x86/kvm/vtd.c         |  191
> >  ++++++++++++++++++++++++++++++++++++++++++++ arch/x86/kvm/vtd.h     
> >  |   36 ++++++++ include/asm-x86/kvm_host.h |   17 ++++
> >  include/asm-x86/kvm_para.h |   14 +++
> >  include/linux/kvm_host.h   |    8 ++
> >  6 files changed, 267 insertions(+), 1 deletions(-)
> >  create mode 100644 arch/x86/kvm/vtd.c
> >  create mode 100644 arch/x86/kvm/vtd.h
> > 
> > diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
> > index d220b49..39b7c32 100644
> > --- a/include/linux/kvm_host.h
> > +++ b/include/linux/kvm_host.h
> > @@ -278,6 +278,14 @@ int kvm_cpu_has_interrupt(struct kvm_vcpu *v);
> >  int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
> >  void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
> > 
> > +int kvm_iommu_map_pages(struct kvm *kvm, gfn_t base_gfn,
> > +			unsigned long npages);
> > +int kvm_iommu_map_guest(struct kvm *kvm,
> > +			struct kvm_pci_passthrough_dev *pci_pt_dev);
> > +int kvm_iommu_unmap_guest(struct kvm *kvm);
> > +int kvm_intel_iommu_found(void);
> > +u64 kvm_intel_iommu_gfn_to_pfn(struct dmar_domain *domain, u64 gfn);
> > +
> 
> Needn't to add kvm_intel_iommu_gfn_to_pfn() declaration here. It's
> exposed by VT-d driver.

I sent a new version of the patches in which all the prototypes of the functions that are implemented in intel-iommu.c are in intel-iommu.h

> 
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [PATCH] KVM: PCIPT: VT-d support

[Han, Weidong]

Han, Weidong wrote:
> Ben-Ami Yassour wrote:
>> From: Kay, Allen M <allen.m.kay@intel.com>
>> 
>> This patch includes the functions to support VT-d for passthrough
>> devices. 
>> 
>> [Ben: fixed memory pinning]
>> 
>> Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
>> Signed-off-by: Weidong Han <weidong.han@intel.com>
>> Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
>> ---
>>  arch/x86/kvm/Makefile      |    2 +-
>>  arch/x86/kvm/vtd.c         |  191
>>  ++++++++++++++++++++++++++++++++++++++++++++ arch/x86/kvm/vtd.h
>>  |   36 ++++++++ include/asm-x86/kvm_host.h |   17 ++++
>>  include/asm-x86/kvm_para.h |   14 +++
>>  include/linux/kvm_host.h   |    8 ++
>>  6 files changed, 267 insertions(+), 1 deletions(-)
>>  create mode 100644 arch/x86/kvm/vtd.c
>>  create mode 100644 arch/x86/kvm/vtd.h
>> 
>> diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
>> index d220b49..39b7c32 100644 --- a/include/linux/kvm_host.h
>> +++ b/include/linux/kvm_host.h
>> @@ -278,6 +278,14 @@ int kvm_cpu_has_interrupt(struct kvm_vcpu *v);
>>  int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
>>  void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
>> 
>> +int kvm_iommu_map_pages(struct kvm *kvm, gfn_t base_gfn,
>> +			unsigned long npages); +int
kvm_iommu_map_guest(struct kvm *kvm,
>> +			struct kvm_pci_passthrough_dev *pci_pt_dev);
>> +int kvm_iommu_unmap_guest(struct kvm *kvm);
>> +int kvm_intel_iommu_found(void);
>> +u64 kvm_intel_iommu_gfn_to_pfn(struct dmar_domain *domain, u64 gfn);
>> +
> 
> Needn't to add kvm_intel_iommu_gfn_to_pfn() declaration here. It's
> exposed by VT-d driver. 

I think it's better to declare kvm_intel_iommu_gfn_to_pfn() in vtd.h,
because it's only used in vtd.c.

Randy (Weidong)

RE: [PATCH] KVM: PCIPT: VT-d support

[Ben-Ami Yassour]

On Thu, 2008-07-03 at 14:16 +0800, Han, Weidong wrote:
> Han, Weidong wrote:
> > Ben-Ami Yassour wrote:
> >> From: Kay, Allen M <allen.m.kay@intel.com>
> >> 
> >> This patch includes the functions to support VT-d for passthrough
> >> devices. 
> >> 
> >> [Ben: fixed memory pinning]
> >> 
> >> Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
> >> Signed-off-by: Weidong Han <weidong.han@intel.com>
> >> Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
> >> ---
> >>  arch/x86/kvm/Makefile      |    2 +-
> >>  arch/x86/kvm/vtd.c         |  191
> >>  ++++++++++++++++++++++++++++++++++++++++++++ arch/x86/kvm/vtd.h
> >>  |   36 ++++++++ include/asm-x86/kvm_host.h |   17 ++++
> >>  include/asm-x86/kvm_para.h |   14 +++
> >>  include/linux/kvm_host.h   |    8 ++
> >>  6 files changed, 267 insertions(+), 1 deletions(-)
> >>  create mode 100644 arch/x86/kvm/vtd.c
> >>  create mode 100644 arch/x86/kvm/vtd.h
> >> 
> >> diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
> >> index d220b49..39b7c32 100644 --- a/include/linux/kvm_host.h
> >> +++ b/include/linux/kvm_host.h
> >> @@ -278,6 +278,14 @@ int kvm_cpu_has_interrupt(struct kvm_vcpu *v);
> >>  int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
> >>  void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
> >> 
> >> +int kvm_iommu_map_pages(struct kvm *kvm, gfn_t base_gfn,
> >> +			unsigned long npages); +int
> kvm_iommu_map_guest(struct kvm *kvm,
> >> +			struct kvm_pci_passthrough_dev *pci_pt_dev);
> >> +int kvm_iommu_unmap_guest(struct kvm *kvm);
> >> +int kvm_intel_iommu_found(void);
> >> +u64 kvm_intel_iommu_gfn_to_pfn(struct dmar_domain *domain, u64 gfn);
> >> +
> > 
> > Needn't to add kvm_intel_iommu_gfn_to_pfn() declaration here. It's
> > exposed by VT-d driver. 
> 
> I think it's better to declare kvm_intel_iommu_gfn_to_pfn() in vtd.h,
> because it's only used in vtd.c.
Agree.
Another option is to declare these functions in intel-iommu.h, since it
is implemented in intel-iommu.c.

Regards,
Ben

> 
> Randy (Weidong)