VT-d support for device assignment

VT-d support for device assignment

[Ben-Ami Yassour]

Avi,

I am re-sending the VT-d patches, please apply.

The following two patches contains the VT-d support for device
assignment.
The first patch contains the changes that are required to the generic
VT-d code.
The second patch contains the changes to KVM.

Regards,
Ben

VT-d support for device assignment

[Amit Shah]

The following two patches contain VT-d support for device assignment
for KVM guests.

The first patch contains the changes that are required to the generic
VT-d code.

The second patch contains the changes to KVM.

I've updated the 2nd patch to use VT-d only when requested by a parameter
on the command line, making it easier to support iommu with pvdma and
multiple iommu types.

The command line currently should be invoked as:

-pcidevice dev=00:13.0,vtd=on

[PATCH] VT-d: changes to support KVM

[Amit Shah]

From: Kay, Allen M <allen.m.kay@intel.com>

This patch extends the VT-d driver to support KVM

[Ben: fixed memory pinning]

Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
Signed-off-by: Weidong Han <weidong.han@intel.com>
Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
Signed-off-by: Amit Shah <amit.shah@qumranet.com>
---
 drivers/pci/dmar.c          |    4 +-
 drivers/pci/intel-iommu.c   |  117 ++++++++++++++-
 drivers/pci/intel-iommu.h   |  344 -----------------------------------------
 drivers/pci/iova.c          |    2 +-
 drivers/pci/iova.h          |   52 -------
 include/linux/intel-iommu.h |  355 +++++++++++++++++++++++++++++++++++++++++++
 include/linux/iova.h        |   52 +++++++
 7 files changed, 523 insertions(+), 403 deletions(-)
 delete mode 100644 drivers/pci/intel-iommu.h
 delete mode 100644 drivers/pci/iova.h
 create mode 100644 include/linux/intel-iommu.h
 create mode 100644 include/linux/iova.h

diff --git a/drivers/pci/dmar.c b/drivers/pci/dmar.c
index 8bf86ae..1df28ea 100644
--- a/drivers/pci/dmar.c
+++ b/drivers/pci/dmar.c
@@ -26,8 +26,8 @@
 
 #include <linux/pci.h>
 #include <linux/dmar.h>
-#include "iova.h"
-#include "intel-iommu.h"
+#include <linux/iova.h>
+#include <linux/intel-iommu.h>
 
 #undef PREFIX
 #define PREFIX "DMAR:"
diff --git a/drivers/pci/intel-iommu.c b/drivers/pci/intel-iommu.c
index 8d0e60a..1eefc60 100644
--- a/drivers/pci/intel-iommu.c
+++ b/drivers/pci/intel-iommu.c
@@ -20,6 +20,7 @@
  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  */
 
+#undef DEBUG
 #include <linux/init.h>
 #include <linux/bitmap.h>
 #include <linux/debugfs.h>
@@ -33,8 +34,8 @@
 #include <linux/dma-mapping.h>
 #include <linux/mempool.h>
 #include <linux/timer.h>
-#include "iova.h"
-#include "intel-iommu.h"
+#include <linux/iova.h>
+#include <linux/intel-iommu.h>
 #include <asm/proto.h> /* force_iommu in this header in x86-64*/
 #include <asm/cacheflush.h>
 #include <asm/iommu.h>
@@ -160,7 +161,7 @@ static inline void *alloc_domain_mem(void)
 	return iommu_kmem_cache_alloc(iommu_domain_cache);
 }
 
-static inline void free_domain_mem(void *vaddr)
+static void free_domain_mem(void *vaddr)
 {
 	kmem_cache_free(iommu_domain_cache, vaddr);
 }
@@ -1414,7 +1415,7 @@ static void domain_remove_dev_info(struct dmar_domain *domain)
  * find_domain
  * Note: we use struct pci_dev->dev.archdata.iommu stores the info
  */
-struct dmar_domain *
+static struct dmar_domain *
 find_domain(struct pci_dev *pdev)
 {
 	struct device_domain_info *info;
@@ -2430,3 +2431,111 @@ int __init intel_iommu_init(void)
 	return 0;
 }
 
+void intel_iommu_domain_exit(struct dmar_domain *domain)
+{
+	u64 end;
+
+	/* Domain 0 is reserved, so dont process it */
+	if (!domain)
+		return;
+
+	end = DOMAIN_MAX_ADDR(domain->gaw);
+	end = end & (~PAGE_MASK_4K);
+
+	/* clear ptes */
+	dma_pte_clear_range(domain, 0, end);
+
+	/* free page tables */
+	dma_pte_free_pagetable(domain, 0, end);
+
+	iommu_free_domain(domain);
+	free_domain_mem(domain);
+}
+EXPORT_SYMBOL_GPL(intel_iommu_domain_exit);
+
+struct dmar_domain *intel_iommu_domain_alloc(struct pci_dev *pdev)
+{
+	struct dmar_drhd_unit *drhd;
+	struct dmar_domain *domain;
+	struct intel_iommu *iommu;
+
+	drhd = dmar_find_matched_drhd_unit(pdev);
+	if (!drhd) {
+		printk(KERN_ERR "intel_iommu_domain_alloc: drhd == NULL\n");
+		return NULL;
+	}
+
+	iommu = drhd->iommu;
+	if (!iommu) {
+		printk(KERN_ERR
+			"intel_iommu_domain_alloc: iommu == NULL\n");
+		return NULL;
+	}
+	domain = iommu_alloc_domain(iommu);
+	if (!domain) {
+		printk(KERN_ERR
+			"intel_iommu_domain_alloc: domain == NULL\n");
+		return NULL;
+	}
+	if (domain_init(domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
+		printk(KERN_ERR
+			"intel_iommu_domain_alloc: domain_init() failed\n");
+		intel_iommu_domain_exit(domain);
+		return NULL;
+	}
+	return domain;
+}
+EXPORT_SYMBOL_GPL(intel_iommu_domain_alloc);
+
+int intel_iommu_context_mapping(
+	struct dmar_domain *domain, struct pci_dev *pdev)
+{
+	int rc;
+	rc = domain_context_mapping(domain, pdev);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(intel_iommu_context_mapping);
+
+int intel_iommu_page_mapping(
+	struct dmar_domain *domain, dma_addr_t iova,
+	u64 hpa, size_t size, int prot)
+{
+	int rc;
+	rc = domain_page_mapping(domain, iova, hpa, size, prot);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(intel_iommu_page_mapping);
+
+void intel_iommu_detach_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
+{
+	detach_domain_for_dev(domain, bus, devfn);
+}
+EXPORT_SYMBOL_GPL(intel_iommu_detach_dev);
+
+struct dmar_domain *
+intel_iommu_find_domain(struct pci_dev *pdev)
+{
+	return find_domain(pdev);
+}
+EXPORT_SYMBOL_GPL(intel_iommu_find_domain);
+
+int intel_iommu_found(void)
+{
+	return g_num_of_iommus;
+}
+EXPORT_SYMBOL_GPL(intel_iommu_found);
+
+u64 intel_iommu_iova_to_pfn(struct dmar_domain *domain, u64 iova)
+{
+	struct dma_pte *pte;
+	u64 pfn;
+
+	pfn = 0;
+	pte = addr_to_dma_pte(domain, iova);
+
+	if (pte)
+		pfn = dma_pte_addr(*pte);
+
+	return pfn >> PAGE_SHIFT_4K;
+}
+EXPORT_SYMBOL_GPL(intel_iommu_iova_to_pfn);
diff --git a/drivers/pci/intel-iommu.h b/drivers/pci/intel-iommu.h
deleted file mode 100644
index afc0ad9..0000000
--- a/drivers/pci/intel-iommu.h
+++ /dev/null
@@ -1,344 +0,0 @@
-/*
- * Copyright (c) 2006, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
- * Place - Suite 330, Boston, MA 02111-1307 USA.
- *
- * Copyright (C) 2006-2008 Intel Corporation
- * Author: Ashok Raj <ashok.raj@intel.com>
- * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
- */
-
-#ifndef _INTEL_IOMMU_H_
-#define _INTEL_IOMMU_H_
-
-#include <linux/types.h>
-#include <linux/msi.h>
-#include <linux/sysdev.h>
-#include "iova.h"
-#include <linux/io.h>
-
-/*
- * We need a fixed PAGE_SIZE of 4K irrespective of
- * arch PAGE_SIZE for IOMMU page tables.
- */
-#define PAGE_SHIFT_4K		(12)
-#define PAGE_SIZE_4K		(1UL << PAGE_SHIFT_4K)
-#define PAGE_MASK_4K		(((u64)-1) << PAGE_SHIFT_4K)
-#define PAGE_ALIGN_4K(addr)	(((addr) + PAGE_SIZE_4K - 1) & PAGE_MASK_4K)
-
-#define IOVA_PFN(addr)		((addr) >> PAGE_SHIFT_4K)
-#define DMA_32BIT_PFN		IOVA_PFN(DMA_32BIT_MASK)
-#define DMA_64BIT_PFN		IOVA_PFN(DMA_64BIT_MASK)
-
-/*
- * Intel IOMMU register specification per version 1.0 public spec.
- */
-
-#define	DMAR_VER_REG	0x0	/* Arch version supported by this IOMMU */
-#define	DMAR_CAP_REG	0x8	/* Hardware supported capabilities */
-#define	DMAR_ECAP_REG	0x10	/* Extended capabilities supported */
-#define	DMAR_GCMD_REG	0x18	/* Global command register */
-#define	DMAR_GSTS_REG	0x1c	/* Global status register */
-#define	DMAR_RTADDR_REG	0x20	/* Root entry table */
-#define	DMAR_CCMD_REG	0x28	/* Context command reg */
-#define	DMAR_FSTS_REG	0x34	/* Fault Status register */
-#define	DMAR_FECTL_REG	0x38	/* Fault control register */
-#define	DMAR_FEDATA_REG	0x3c	/* Fault event interrupt data register */
-#define	DMAR_FEADDR_REG	0x40	/* Fault event interrupt addr register */
-#define	DMAR_FEUADDR_REG 0x44	/* Upper address register */
-#define	DMAR_AFLOG_REG	0x58	/* Advanced Fault control */
-#define	DMAR_PMEN_REG	0x64	/* Enable Protected Memory Region */
-#define	DMAR_PLMBASE_REG 0x68	/* PMRR Low addr */
-#define	DMAR_PLMLIMIT_REG 0x6c	/* PMRR low limit */
-#define	DMAR_PHMBASE_REG 0x70	/* pmrr high base addr */
-#define	DMAR_PHMLIMIT_REG 0x78	/* pmrr high limit */
-
-#define OFFSET_STRIDE		(9)
-/*
-#define dmar_readl(dmar, reg) readl(dmar + reg)
-#define dmar_readq(dmar, reg) ({ \
-		u32 lo, hi; \
-		lo = readl(dmar + reg); \
-		hi = readl(dmar + reg + 4); \
-		(((u64) hi) << 32) + lo; })
-*/
-static inline u64 dmar_readq(void __iomem *addr)
-{
-	u32 lo, hi;
-	lo = readl(addr);
-	hi = readl(addr + 4);
-	return (((u64) hi) << 32) + lo;
-}
-
-static inline void dmar_writeq(void __iomem *addr, u64 val)
-{
-	writel((u32)val, addr);
-	writel((u32)(val >> 32), addr + 4);
-}
-
-#define DMAR_VER_MAJOR(v)		(((v) & 0xf0) >> 4)
-#define DMAR_VER_MINOR(v)		((v) & 0x0f)
-
-/*
- * Decoding Capability Register
- */
-#define cap_read_drain(c)	(((c) >> 55) & 1)
-#define cap_write_drain(c)	(((c) >> 54) & 1)
-#define cap_max_amask_val(c)	(((c) >> 48) & 0x3f)
-#define cap_num_fault_regs(c)	((((c) >> 40) & 0xff) + 1)
-#define cap_pgsel_inv(c)	(((c) >> 39) & 1)
-
-#define cap_super_page_val(c)	(((c) >> 34) & 0xf)
-#define cap_super_offset(c)	(((find_first_bit(&cap_super_page_val(c), 4)) \
-					* OFFSET_STRIDE) + 21)
-
-#define cap_fault_reg_offset(c)	((((c) >> 24) & 0x3ff) * 16)
-#define cap_max_fault_reg_offset(c) \
-	(cap_fault_reg_offset(c) + cap_num_fault_regs(c) * 16)
-
-#define cap_zlr(c)		(((c) >> 22) & 1)
-#define cap_isoch(c)		(((c) >> 23) & 1)
-#define cap_mgaw(c)		((((c) >> 16) & 0x3f) + 1)
-#define cap_sagaw(c)		(((c) >> 8) & 0x1f)
-#define cap_caching_mode(c)	(((c) >> 7) & 1)
-#define cap_phmr(c)		(((c) >> 6) & 1)
-#define cap_plmr(c)		(((c) >> 5) & 1)
-#define cap_rwbf(c)		(((c) >> 4) & 1)
-#define cap_afl(c)		(((c) >> 3) & 1)
-#define cap_ndoms(c)		(((unsigned long)1) << (4 + 2 * ((c) & 0x7)))
-/*
- * Extended Capability Register
- */
-
-#define ecap_niotlb_iunits(e)	((((e) >> 24) & 0xff) + 1)
-#define ecap_iotlb_offset(e) 	((((e) >> 8) & 0x3ff) * 16)
-#define ecap_max_iotlb_offset(e) \
-	(ecap_iotlb_offset(e) + ecap_niotlb_iunits(e) * 16)
-#define ecap_coherent(e)	((e) & 0x1)
-
-
-/* IOTLB_REG */
-#define DMA_TLB_GLOBAL_FLUSH (((u64)1) << 60)
-#define DMA_TLB_DSI_FLUSH (((u64)2) << 60)
-#define DMA_TLB_PSI_FLUSH (((u64)3) << 60)
-#define DMA_TLB_IIRG(type) ((type >> 60) & 7)
-#define DMA_TLB_IAIG(val) (((val) >> 57) & 7)
-#define DMA_TLB_READ_DRAIN (((u64)1) << 49)
-#define DMA_TLB_WRITE_DRAIN (((u64)1) << 48)
-#define DMA_TLB_DID(id)	(((u64)((id) & 0xffff)) << 32)
-#define DMA_TLB_IVT (((u64)1) << 63)
-#define DMA_TLB_IH_NONLEAF (((u64)1) << 6)
-#define DMA_TLB_MAX_SIZE (0x3f)
-
-/* PMEN_REG */
-#define DMA_PMEN_EPM (((u32)1)<<31)
-#define DMA_PMEN_PRS (((u32)1)<<0)
-
-/* GCMD_REG */
-#define DMA_GCMD_TE (((u32)1) << 31)
-#define DMA_GCMD_SRTP (((u32)1) << 30)
-#define DMA_GCMD_SFL (((u32)1) << 29)
-#define DMA_GCMD_EAFL (((u32)1) << 28)
-#define DMA_GCMD_WBF (((u32)1) << 27)
-
-/* GSTS_REG */
-#define DMA_GSTS_TES (((u32)1) << 31)
-#define DMA_GSTS_RTPS (((u32)1) << 30)
-#define DMA_GSTS_FLS (((u32)1) << 29)
-#define DMA_GSTS_AFLS (((u32)1) << 28)
-#define DMA_GSTS_WBFS (((u32)1) << 27)
-
-/* CCMD_REG */
-#define DMA_CCMD_ICC (((u64)1) << 63)
-#define DMA_CCMD_GLOBAL_INVL (((u64)1) << 61)
-#define DMA_CCMD_DOMAIN_INVL (((u64)2) << 61)
-#define DMA_CCMD_DEVICE_INVL (((u64)3) << 61)
-#define DMA_CCMD_FM(m) (((u64)((m) & 0x3)) << 32)
-#define DMA_CCMD_MASK_NOBIT 0
-#define DMA_CCMD_MASK_1BIT 1
-#define DMA_CCMD_MASK_2BIT 2
-#define DMA_CCMD_MASK_3BIT 3
-#define DMA_CCMD_SID(s) (((u64)((s) & 0xffff)) << 16)
-#define DMA_CCMD_DID(d) ((u64)((d) & 0xffff))
-
-/* FECTL_REG */
-#define DMA_FECTL_IM (((u32)1) << 31)
-
-/* FSTS_REG */
-#define DMA_FSTS_PPF ((u32)2)
-#define DMA_FSTS_PFO ((u32)1)
-#define dma_fsts_fault_record_index(s) (((s) >> 8) & 0xff)
-
-/* FRCD_REG, 32 bits access */
-#define DMA_FRCD_F (((u32)1) << 31)
-#define dma_frcd_type(d) ((d >> 30) & 1)
-#define dma_frcd_fault_reason(c) (c & 0xff)
-#define dma_frcd_source_id(c) (c & 0xffff)
-#define dma_frcd_page_addr(d) (d & (((u64)-1) << 12)) /* low 64 bit */
-
-/*
- * 0: Present
- * 1-11: Reserved
- * 12-63: Context Ptr (12 - (haw-1))
- * 64-127: Reserved
- */
-struct root_entry {
-	u64	val;
-	u64	rsvd1;
-};
-#define ROOT_ENTRY_NR (PAGE_SIZE_4K/sizeof(struct root_entry))
-static inline bool root_present(struct root_entry *root)
-{
-	return (root->val & 1);
-}
-static inline void set_root_present(struct root_entry *root)
-{
-	root->val |= 1;
-}
-static inline void set_root_value(struct root_entry *root, unsigned long value)
-{
-	root->val |= value & PAGE_MASK_4K;
-}
-
-struct context_entry;
-static inline struct context_entry *
-get_context_addr_from_root(struct root_entry *root)
-{
-	return (struct context_entry *)
-		(root_present(root)?phys_to_virt(
-		root->val & PAGE_MASK_4K):
-		NULL);
-}
-
-/*
- * low 64 bits:
- * 0: present
- * 1: fault processing disable
- * 2-3: translation type
- * 12-63: address space root
- * high 64 bits:
- * 0-2: address width
- * 3-6: aval
- * 8-23: domain id
- */
-struct context_entry {
-	u64 lo;
-	u64 hi;
-};
-#define context_present(c) ((c).lo & 1)
-#define context_fault_disable(c) (((c).lo >> 1) & 1)
-#define context_translation_type(c) (((c).lo >> 2) & 3)
-#define context_address_root(c) ((c).lo & PAGE_MASK_4K)
-#define context_address_width(c) ((c).hi &  7)
-#define context_domain_id(c) (((c).hi >> 8) & ((1 << 16) - 1))
-
-#define context_set_present(c) do {(c).lo |= 1;} while (0)
-#define context_set_fault_enable(c) \
-	do {(c).lo &= (((u64)-1) << 2) | 1;} while (0)
-#define context_set_translation_type(c, val) \
-	do { \
-		(c).lo &= (((u64)-1) << 4) | 3; \
-		(c).lo |= ((val) & 3) << 2; \
-	} while (0)
-#define CONTEXT_TT_MULTI_LEVEL 0
-#define context_set_address_root(c, val) \
-	do {(c).lo |= (val) & PAGE_MASK_4K;} while (0)
-#define context_set_address_width(c, val) do {(c).hi |= (val) & 7;} while (0)
-#define context_set_domain_id(c, val) \
-	do {(c).hi |= ((val) & ((1 << 16) - 1)) << 8;} while (0)
-#define context_clear_entry(c) do {(c).lo = 0; (c).hi = 0;} while (0)
-
-/*
- * 0: readable
- * 1: writable
- * 2-6: reserved
- * 7: super page
- * 8-11: available
- * 12-63: Host physcial address
- */
-struct dma_pte {
-	u64 val;
-};
-#define dma_clear_pte(p)	do {(p).val = 0;} while (0)
-
-#define DMA_PTE_READ (1)
-#define DMA_PTE_WRITE (2)
-
-#define dma_set_pte_readable(p) do {(p).val |= DMA_PTE_READ;} while (0)
-#define dma_set_pte_writable(p) do {(p).val |= DMA_PTE_WRITE;} while (0)
-#define dma_set_pte_prot(p, prot) \
-		do {(p).val = ((p).val & ~3) | ((prot) & 3); } while (0)
-#define dma_pte_addr(p) ((p).val & PAGE_MASK_4K)
-#define dma_set_pte_addr(p, addr) do {\
-		(p).val |= ((addr) & PAGE_MASK_4K); } while (0)
-#define dma_pte_present(p) (((p).val & 3) != 0)
-
-struct intel_iommu;
-
-struct dmar_domain {
-	int	id;			/* domain id */
-	struct intel_iommu *iommu;	/* back pointer to owning iommu */
-
-	struct list_head devices; 	/* all devices' list */
-	struct iova_domain iovad;	/* iova's that belong to this domain */
-
-	struct dma_pte	*pgd;		/* virtual address */
-	spinlock_t	mapping_lock;	/* page table lock */
-	int		gaw;		/* max guest address width */
-
-	/* adjusted guest address width, 0 is level 2 30-bit */
-	int		agaw;
-
-#define DOMAIN_FLAG_MULTIPLE_DEVICES 1
-	int		flags;
-};
-
-/* PCI domain-device relationship */
-struct device_domain_info {
-	struct list_head link;	/* link to domain siblings */
-	struct list_head global; /* link to global list */
-	u8 bus;			/* PCI bus numer */
-	u8 devfn;		/* PCI devfn number */
-	struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
-	struct dmar_domain *domain; /* pointer to domain */
-};
-
-extern int init_dmars(void);
-
-struct intel_iommu {
-	void __iomem	*reg; /* Pointer to hardware regs, virtual addr */
-	u64		cap;
-	u64		ecap;
-	unsigned long 	*domain_ids; /* bitmap of domains */
-	struct dmar_domain **domains; /* ptr to domains */
-	int		seg;
-	u32		gcmd; /* Holds TE, EAFL. Don't need SRTP, SFL, WBF */
-	spinlock_t	lock; /* protect context, domain ids */
-	spinlock_t	register_lock; /* protect register handling */
-	struct root_entry *root_entry; /* virtual address */
-
-	unsigned int irq;
-	unsigned char name[7];    /* Device Name */
-	struct msi_msg saved_msg;
-	struct sys_device sysdev;
-};
-
-#ifndef CONFIG_DMAR_GFX_WA
-static inline void iommu_prepare_gfx_mapping(void)
-{
-	return;
-}
-#endif /* !CONFIG_DMAR_GFX_WA */
-
-#endif
diff --git a/drivers/pci/iova.c b/drivers/pci/iova.c
index 3ef4ac0..2287116 100644
--- a/drivers/pci/iova.c
+++ b/drivers/pci/iova.c
@@ -7,7 +7,7 @@
  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  */
 
-#include "iova.h"
+#include <linux/iova.h>
 
 void
 init_iova_domain(struct iova_domain *iovad, unsigned long pfn_32bit)
diff --git a/drivers/pci/iova.h b/drivers/pci/iova.h
deleted file mode 100644
index 228f6c9..0000000
--- a/drivers/pci/iova.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright (c) 2006, Intel Corporation.
- *
- * This file is released under the GPLv2.
- *
- * Copyright (C) 2006-2008 Intel Corporation
- * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
- *
- */
-
-#ifndef _IOVA_H_
-#define _IOVA_H_
-
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/rbtree.h>
-#include <linux/dma-mapping.h>
-
-/* IO virtual address start page frame number */
-#define IOVA_START_PFN		(1)
-
-/* iova structure */
-struct iova {
-	struct rb_node	node;
-	unsigned long	pfn_hi; /* IOMMU dish out addr hi */
-	unsigned long	pfn_lo; /* IOMMU dish out addr lo */
-};
-
-/* holds all the iova translations for a domain */
-struct iova_domain {
-	spinlock_t	iova_alloc_lock;/* Lock to protect iova  allocation */
-	spinlock_t	iova_rbtree_lock; /* Lock to protect update of rbtree */
-	struct rb_root	rbroot;		/* iova domain rbtree root */
-	struct rb_node	*cached32_node; /* Save last alloced node */
-	unsigned long	dma_32bit_pfn;
-};
-
-struct iova *alloc_iova_mem(void);
-void free_iova_mem(struct iova *iova);
-void free_iova(struct iova_domain *iovad, unsigned long pfn);
-void __free_iova(struct iova_domain *iovad, struct iova *iova);
-struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size,
-	unsigned long limit_pfn,
-	bool size_aligned);
-struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,
-	unsigned long pfn_hi);
-void copy_reserved_iova(struct iova_domain *from, struct iova_domain *to);
-void init_iova_domain(struct iova_domain *iovad, unsigned long pfn_32bit);
-struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn);
-void put_iova_domain(struct iova_domain *iovad);
-
-#endif
diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h
new file mode 100644
index 0000000..1490fc0
--- /dev/null
+++ b/include/linux/intel-iommu.h
@@ -0,0 +1,355 @@
+/*
+ * Copyright (c) 2006, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ *
+ * Copyright (C) 2006-2008 Intel Corporation
+ * Author: Ashok Raj <ashok.raj@intel.com>
+ * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
+ */
+
+#ifndef _INTEL_IOMMU_H_
+#define _INTEL_IOMMU_H_
+
+#include <linux/types.h>
+#include <linux/msi.h>
+#include <linux/sysdev.h>
+#include "iova.h"
+#include <linux/io.h>
+
+/*
+ * We need a fixed PAGE_SIZE of 4K irrespective of
+ * arch PAGE_SIZE for IOMMU page tables.
+ */
+#define PAGE_SHIFT_4K		(12)
+#define PAGE_SIZE_4K		(1UL << PAGE_SHIFT_4K)
+#define PAGE_MASK_4K		(((u64)-1) << PAGE_SHIFT_4K)
+#define PAGE_ALIGN_4K(addr)	(((addr) + PAGE_SIZE_4K - 1) & PAGE_MASK_4K)
+
+#define IOVA_PFN(addr)		((addr) >> PAGE_SHIFT_4K)
+#define DMA_32BIT_PFN		IOVA_PFN(DMA_32BIT_MASK)
+#define DMA_64BIT_PFN		IOVA_PFN(DMA_64BIT_MASK)
+
+/*
+ * Intel IOMMU register specification per version 1.0 public spec.
+ */
+
+#define	DMAR_VER_REG	0x0	/* Arch version supported by this IOMMU */
+#define	DMAR_CAP_REG	0x8	/* Hardware supported capabilities */
+#define	DMAR_ECAP_REG	0x10	/* Extended capabilities supported */
+#define	DMAR_GCMD_REG	0x18	/* Global command register */
+#define	DMAR_GSTS_REG	0x1c	/* Global status register */
+#define	DMAR_RTADDR_REG	0x20	/* Root entry table */
+#define	DMAR_CCMD_REG	0x28	/* Context command reg */
+#define	DMAR_FSTS_REG	0x34	/* Fault Status register */
+#define	DMAR_FECTL_REG	0x38	/* Fault control register */
+#define	DMAR_FEDATA_REG	0x3c	/* Fault event interrupt data register */
+#define	DMAR_FEADDR_REG	0x40	/* Fault event interrupt addr register */
+#define	DMAR_FEUADDR_REG 0x44	/* Upper address register */
+#define	DMAR_AFLOG_REG	0x58	/* Advanced Fault control */
+#define	DMAR_PMEN_REG	0x64	/* Enable Protected Memory Region */
+#define	DMAR_PLMBASE_REG 0x68	/* PMRR Low addr */
+#define	DMAR_PLMLIMIT_REG 0x6c	/* PMRR low limit */
+#define	DMAR_PHMBASE_REG 0x70	/* pmrr high base addr */
+#define	DMAR_PHMLIMIT_REG 0x78	/* pmrr high limit */
+
+#define OFFSET_STRIDE		(9)
+/*
+#define dmar_readl(dmar, reg) readl(dmar + reg)
+#define dmar_readq(dmar, reg) ({ \
+		u32 lo, hi; \
+		lo = readl(dmar + reg); \
+		hi = readl(dmar + reg + 4); \
+		(((u64) hi) << 32) + lo; })
+*/
+static inline u64 dmar_readq(void __iomem *addr)
+{
+	u32 lo, hi;
+	lo = readl(addr);
+	hi = readl(addr + 4);
+	return (((u64) hi) << 32) + lo;
+}
+
+static inline void dmar_writeq(void __iomem *addr, u64 val)
+{
+	writel((u32)val, addr);
+	writel((u32)(val >> 32), addr + 4);
+}
+
+#define DMAR_VER_MAJOR(v)		(((v) & 0xf0) >> 4)
+#define DMAR_VER_MINOR(v)		((v) & 0x0f)
+
+/*
+ * Decoding Capability Register
+ */
+#define cap_read_drain(c)	(((c) >> 55) & 1)
+#define cap_write_drain(c)	(((c) >> 54) & 1)
+#define cap_max_amask_val(c)	(((c) >> 48) & 0x3f)
+#define cap_num_fault_regs(c)	((((c) >> 40) & 0xff) + 1)
+#define cap_pgsel_inv(c)	(((c) >> 39) & 1)
+
+#define cap_super_page_val(c)	(((c) >> 34) & 0xf)
+#define cap_super_offset(c)	(((find_first_bit(&cap_super_page_val(c), 4)) \
+					* OFFSET_STRIDE) + 21)
+
+#define cap_fault_reg_offset(c)	((((c) >> 24) & 0x3ff) * 16)
+#define cap_max_fault_reg_offset(c) \
+	(cap_fault_reg_offset(c) + cap_num_fault_regs(c) * 16)
+
+#define cap_zlr(c)		(((c) >> 22) & 1)
+#define cap_isoch(c)		(((c) >> 23) & 1)
+#define cap_mgaw(c)		((((c) >> 16) & 0x3f) + 1)
+#define cap_sagaw(c)		(((c) >> 8) & 0x1f)
+#define cap_caching_mode(c)	(((c) >> 7) & 1)
+#define cap_phmr(c)		(((c) >> 6) & 1)
+#define cap_plmr(c)		(((c) >> 5) & 1)
+#define cap_rwbf(c)		(((c) >> 4) & 1)
+#define cap_afl(c)		(((c) >> 3) & 1)
+#define cap_ndoms(c)		(((unsigned long)1) << (4 + 2 * ((c) & 0x7)))
+/*
+ * Extended Capability Register
+ */
+
+#define ecap_niotlb_iunits(e)	((((e) >> 24) & 0xff) + 1)
+#define ecap_iotlb_offset(e) 	((((e) >> 8) & 0x3ff) * 16)
+#define ecap_max_iotlb_offset(e) \
+	(ecap_iotlb_offset(e) + ecap_niotlb_iunits(e) * 16)
+#define ecap_coherent(e)	((e) & 0x1)
+
+
+/* IOTLB_REG */
+#define DMA_TLB_GLOBAL_FLUSH (((u64)1) << 60)
+#define DMA_TLB_DSI_FLUSH (((u64)2) << 60)
+#define DMA_TLB_PSI_FLUSH (((u64)3) << 60)
+#define DMA_TLB_IIRG(type) ((type >> 60) & 7)
+#define DMA_TLB_IAIG(val) (((val) >> 57) & 7)
+#define DMA_TLB_READ_DRAIN (((u64)1) << 49)
+#define DMA_TLB_WRITE_DRAIN (((u64)1) << 48)
+#define DMA_TLB_DID(id)	(((u64)((id) & 0xffff)) << 32)
+#define DMA_TLB_IVT (((u64)1) << 63)
+#define DMA_TLB_IH_NONLEAF (((u64)1) << 6)
+#define DMA_TLB_MAX_SIZE (0x3f)
+
+/* PMEN_REG */
+#define DMA_PMEN_EPM (((u32)1)<<31)
+#define DMA_PMEN_PRS (((u32)1)<<0)
+
+/* GCMD_REG */
+#define DMA_GCMD_TE (((u32)1) << 31)
+#define DMA_GCMD_SRTP (((u32)1) << 30)
+#define DMA_GCMD_SFL (((u32)1) << 29)
+#define DMA_GCMD_EAFL (((u32)1) << 28)
+#define DMA_GCMD_WBF (((u32)1) << 27)
+
+/* GSTS_REG */
+#define DMA_GSTS_TES (((u32)1) << 31)
+#define DMA_GSTS_RTPS (((u32)1) << 30)
+#define DMA_GSTS_FLS (((u32)1) << 29)
+#define DMA_GSTS_AFLS (((u32)1) << 28)
+#define DMA_GSTS_WBFS (((u32)1) << 27)
+
+/* CCMD_REG */
+#define DMA_CCMD_ICC (((u64)1) << 63)
+#define DMA_CCMD_GLOBAL_INVL (((u64)1) << 61)
+#define DMA_CCMD_DOMAIN_INVL (((u64)2) << 61)
+#define DMA_CCMD_DEVICE_INVL (((u64)3) << 61)
+#define DMA_CCMD_FM(m) (((u64)((m) & 0x3)) << 32)
+#define DMA_CCMD_MASK_NOBIT 0
+#define DMA_CCMD_MASK_1BIT 1
+#define DMA_CCMD_MASK_2BIT 2
+#define DMA_CCMD_MASK_3BIT 3
+#define DMA_CCMD_SID(s) (((u64)((s) & 0xffff)) << 16)
+#define DMA_CCMD_DID(d) ((u64)((d) & 0xffff))
+
+/* FECTL_REG */
+#define DMA_FECTL_IM (((u32)1) << 31)
+
+/* FSTS_REG */
+#define DMA_FSTS_PPF ((u32)2)
+#define DMA_FSTS_PFO ((u32)1)
+#define dma_fsts_fault_record_index(s) (((s) >> 8) & 0xff)
+
+/* FRCD_REG, 32 bits access */
+#define DMA_FRCD_F (((u32)1) << 31)
+#define dma_frcd_type(d) ((d >> 30) & 1)
+#define dma_frcd_fault_reason(c) (c & 0xff)
+#define dma_frcd_source_id(c) (c & 0xffff)
+#define dma_frcd_page_addr(d) (d & (((u64)-1) << 12)) /* low 64 bit */
+
+/*
+ * 0: Present
+ * 1-11: Reserved
+ * 12-63: Context Ptr (12 - (haw-1))
+ * 64-127: Reserved
+ */
+struct root_entry {
+	u64	val;
+	u64	rsvd1;
+};
+#define ROOT_ENTRY_NR (PAGE_SIZE_4K/sizeof(struct root_entry))
+static inline bool root_present(struct root_entry *root)
+{
+	return (root->val & 1);
+}
+static inline void set_root_present(struct root_entry *root)
+{
+	root->val |= 1;
+}
+static inline void set_root_value(struct root_entry *root, unsigned long value)
+{
+	root->val |= value & PAGE_MASK_4K;
+}
+
+struct context_entry;
+static inline struct context_entry *
+get_context_addr_from_root(struct root_entry *root)
+{
+	return (struct context_entry *)
+		(root_present(root)?phys_to_virt(
+		root->val & PAGE_MASK_4K):
+		NULL);
+}
+
+/*
+ * low 64 bits:
+ * 0: present
+ * 1: fault processing disable
+ * 2-3: translation type
+ * 12-63: address space root
+ * high 64 bits:
+ * 0-2: address width
+ * 3-6: aval
+ * 8-23: domain id
+ */
+struct context_entry {
+	u64 lo;
+	u64 hi;
+};
+#define context_present(c) ((c).lo & 1)
+#define context_fault_disable(c) (((c).lo >> 1) & 1)
+#define context_translation_type(c) (((c).lo >> 2) & 3)
+#define context_address_root(c) ((c).lo & PAGE_MASK_4K)
+#define context_address_width(c) ((c).hi &  7)
+#define context_domain_id(c) (((c).hi >> 8) & ((1 << 16) - 1))
+
+#define context_set_present(c) do {(c).lo |= 1;} while (0)
+#define context_set_fault_enable(c) \
+	do {(c).lo &= (((u64)-1) << 2) | 1;} while (0)
+#define context_set_translation_type(c, val) \
+	do { \
+		(c).lo &= (((u64)-1) << 4) | 3; \
+		(c).lo |= ((val) & 3) << 2; \
+	} while (0)
+#define CONTEXT_TT_MULTI_LEVEL 0
+#define context_set_address_root(c, val) \
+	do {(c).lo |= (val) & PAGE_MASK_4K;} while (0)
+#define context_set_address_width(c, val) do {(c).hi |= (val) & 7;} while (0)
+#define context_set_domain_id(c, val) \
+	do {(c).hi |= ((val) & ((1 << 16) - 1)) << 8;} while (0)
+#define context_clear_entry(c) do {(c).lo = 0; (c).hi = 0;} while (0)
+
+/*
+ * 0: readable
+ * 1: writable
+ * 2-6: reserved
+ * 7: super page
+ * 8-11: available
+ * 12-63: Host physcial address
+ */
+struct dma_pte {
+	u64 val;
+};
+#define dma_clear_pte(p)	do {(p).val = 0;} while (0)
+
+#define DMA_PTE_READ (1)
+#define DMA_PTE_WRITE (2)
+
+#define dma_set_pte_readable(p) do {(p).val |= DMA_PTE_READ;} while (0)
+#define dma_set_pte_writable(p) do {(p).val |= DMA_PTE_WRITE;} while (0)
+#define dma_set_pte_prot(p, prot) \
+		do {(p).val = ((p).val & ~3) | ((prot) & 3); } while (0)
+#define dma_pte_addr(p) ((p).val & PAGE_MASK_4K)
+#define dma_set_pte_addr(p, addr) do {\
+		(p).val |= ((addr) & PAGE_MASK_4K); } while (0)
+#define dma_pte_present(p) (((p).val & 3) != 0)
+
+struct intel_iommu;
+
+struct dmar_domain {
+	int	id;			/* domain id */
+	struct intel_iommu *iommu;	/* back pointer to owning iommu */
+
+	struct list_head devices; 	/* all devices' list */
+	struct iova_domain iovad;	/* iova's that belong to this domain */
+
+	struct dma_pte	*pgd;		/* virtual address */
+	spinlock_t	mapping_lock;	/* page table lock */
+	int		gaw;		/* max guest address width */
+
+	/* adjusted guest address width, 0 is level 2 30-bit */
+	int		agaw;
+
+#define DOMAIN_FLAG_MULTIPLE_DEVICES 1
+	int		flags;
+};
+
+/* PCI domain-device relationship */
+struct device_domain_info {
+	struct list_head link;	/* link to domain siblings */
+	struct list_head global; /* link to global list */
+	u8 bus;			/* PCI bus numer */
+	u8 devfn;		/* PCI devfn number */
+	struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
+	struct dmar_domain *domain; /* pointer to domain */
+};
+
+extern int init_dmars(void);
+
+struct intel_iommu {
+	void __iomem	*reg; /* Pointer to hardware regs, virtual addr */
+	u64		cap;
+	u64		ecap;
+	unsigned long 	*domain_ids; /* bitmap of domains */
+	struct dmar_domain **domains; /* ptr to domains */
+	int		seg;
+	u32		gcmd; /* Holds TE, EAFL. Don't need SRTP, SFL, WBF */
+	spinlock_t	lock; /* protect context, domain ids */
+	spinlock_t	register_lock; /* protect register handling */
+	struct root_entry *root_entry; /* virtual address */
+
+	unsigned int irq;
+	unsigned char name[7];    /* Device Name */
+	struct msi_msg saved_msg;
+	struct sys_device sysdev;
+};
+
+#ifndef CONFIG_DMAR_GFX_WA
+static inline void iommu_prepare_gfx_mapping(void)
+{
+	return;
+}
+#endif /* !CONFIG_DMAR_GFX_WA */
+
+void intel_iommu_domain_exit(struct dmar_domain *domain);
+struct dmar_domain *intel_iommu_domain_alloc(struct pci_dev *pdev);
+int intel_iommu_context_mapping(struct dmar_domain *domain,
+				struct pci_dev *pdev);
+int intel_iommu_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
+			     u64 hpa, size_t size, int prot);
+void intel_iommu_detach_dev(struct dmar_domain *domain, u8 bus, u8 devfn);
+struct dmar_domain *intel_iommu_find_domain(struct pci_dev *pdev);
+int intel_iommu_found(void);
+u64 intel_iommu_iova_to_pfn(struct dmar_domain *domain, u64 iova);
+
+#endif
diff --git a/include/linux/iova.h b/include/linux/iova.h
new file mode 100644
index 0000000..228f6c9
--- /dev/null
+++ b/include/linux/iova.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2006, Intel Corporation.
+ *
+ * This file is released under the GPLv2.
+ *
+ * Copyright (C) 2006-2008 Intel Corporation
+ * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
+ *
+ */
+
+#ifndef _IOVA_H_
+#define _IOVA_H_
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/rbtree.h>
+#include <linux/dma-mapping.h>
+
+/* IO virtual address start page frame number */
+#define IOVA_START_PFN		(1)
+
+/* iova structure */
+struct iova {
+	struct rb_node	node;
+	unsigned long	pfn_hi; /* IOMMU dish out addr hi */
+	unsigned long	pfn_lo; /* IOMMU dish out addr lo */
+};
+
+/* holds all the iova translations for a domain */
+struct iova_domain {
+	spinlock_t	iova_alloc_lock;/* Lock to protect iova  allocation */
+	spinlock_t	iova_rbtree_lock; /* Lock to protect update of rbtree */
+	struct rb_root	rbroot;		/* iova domain rbtree root */
+	struct rb_node	*cached32_node; /* Save last alloced node */
+	unsigned long	dma_32bit_pfn;
+};
+
+struct iova *alloc_iova_mem(void);
+void free_iova_mem(struct iova *iova);
+void free_iova(struct iova_domain *iovad, unsigned long pfn);
+void __free_iova(struct iova_domain *iovad, struct iova *iova);
+struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size,
+	unsigned long limit_pfn,
+	bool size_aligned);
+struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,
+	unsigned long pfn_hi);
+void copy_reserved_iova(struct iova_domain *from, struct iova_domain *to);
+void init_iova_domain(struct iova_domain *iovad, unsigned long pfn_32bit);
+struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn);
+void put_iova_domain(struct iova_domain *iovad);
+
+#endif
-- 
1.5.4.3

[PATCH] KVM: Device Assignment with VT-d

[Amit Shah]

Based on a patch by: Kay, Allen M <allen.m.kay@intel.com>

This patch enables PCI device assignment based on VT-d support.
When a device is assigned to the guest, the guest memory is pinned and
the mapping is updated in the VT-d IOMMU.

Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
Signed-off-by: Weidong Han <weidong.han@intel.com>
Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
Signed-off-by: Amit Shah <amit.shah@qumranet.com>
---
 arch/x86/kvm/Makefile      |    3 +
 arch/x86/kvm/vtd.c         |  203 ++++++++++++++++++++++++++++++++++++++++++++
 arch/x86/kvm/x86.c         |   11 +++
 include/asm-x86/kvm_host.h |    3 +
 include/linux/kvm.h        |    2 +
 include/linux/kvm_host.h   |   32 +++++++
 virt/kvm/kvm_main.c        |    9 ++-
 7 files changed, 262 insertions(+), 1 deletions(-)
 create mode 100644 arch/x86/kvm/vtd.c

diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index d0e940b..3072b17 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -12,6 +12,9 @@ EXTRA_CFLAGS += -Ivirt/kvm -Iarch/x86/kvm
 
 kvm-objs := $(common-objs) x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o \
 	i8254.o
+ifeq ($(CONFIG_DMAR),y)
+kvm-objs += vtd.o
+endif
 obj-$(CONFIG_KVM) += kvm.o
 kvm-intel-objs = vmx.o
 obj-$(CONFIG_KVM_INTEL) += kvm-intel.o
diff --git a/arch/x86/kvm/vtd.c b/arch/x86/kvm/vtd.c
new file mode 100644
index 0000000..4336769
--- /dev/null
+++ b/arch/x86/kvm/vtd.c
@@ -0,0 +1,203 @@
+/*
+ * Copyright (c) 2006, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ *
+ * Copyright (C) 2006-2008 Intel Corporation
+ * Copyright IBM Corporation, 2008
+ * Author: Allen M. Kay <allen.m.kay@intel.com>
+ * Author: Weidong Han <weidong.han@intel.com>
+ * Author: Ben-Ami Yassour <benami@il.ibm.com>
+ */
+
+#include <linux/list.h>
+#include <linux/kvm_host.h>
+#include <linux/pci.h>
+#include <linux/dmar.h>
+#include <linux/intel-iommu.h>
+
+static int kvm_iommu_unmap_memslots(struct kvm *kvm);
+
+int kvm_iommu_map_pages(struct kvm *kvm,
+			  gfn_t base_gfn, unsigned long npages)
+{
+	gfn_t gfn = base_gfn;
+	pfn_t pfn;
+	int i, rc;
+	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
+
+	/* check if iommu exists and in use */
+	if (!domain)
+		return 0;
+
+	for (i = 0; i < npages; i++) {
+		/* check if already mapped */
+		pfn = (pfn_t)intel_iommu_iova_to_pfn(domain,
+						     gfn_to_gpa(gfn));
+		if (pfn && !is_mmio_pfn(pfn))
+			continue;
+
+		pfn = gfn_to_pfn(kvm, gfn);
+		if (!is_mmio_pfn(pfn)) {
+			rc = intel_iommu_page_mapping(domain,
+						      gfn_to_gpa(gfn),
+						      pfn_to_hpa(pfn),
+						      PAGE_SIZE,
+						      DMA_PTE_READ |
+						      DMA_PTE_WRITE);
+			if (rc) {
+				kvm_release_pfn_clean(pfn);
+				printk(KERN_DEBUG "kvm_iommu_map_pages:"
+				       "iommu failed to map pfn=%lx\n", pfn);
+				return rc;
+			}
+		} else {
+			printk(KERN_DEBUG "kvm_iommu_map_page:"
+			       "invalid pfn=%lx\n", pfn);
+			return 0;
+		}
+
+		gfn++;
+	}
+	return 0;
+}
+
+static int kvm_iommu_map_memslots(struct kvm *kvm)
+{
+	int i, rc;
+
+	down_read(&kvm->slots_lock);
+	for (i = 0; i < kvm->nmemslots; i++) {
+		rc = kvm_iommu_map_pages(kvm, kvm->memslots[i].base_gfn,
+					 kvm->memslots[i].npages);
+		if (rc) {
+			up_read(&kvm->slots_lock);
+			return rc;
+		}
+	}
+	up_read(&kvm->slots_lock);
+	return 0;
+}
+
+int kvm_iommu_map_guest(struct kvm *kvm,
+			struct kvm_assigned_dev_kernel *assigned_dev)
+{
+	struct pci_dev *pdev = NULL;
+	int rc;
+
+	if (!intel_iommu_found()) {
+		printk(KERN_ERR "intel iommu not found\n");
+		return -ENODEV;
+	}
+
+	printk(KERN_DEBUG "VT-d direct map: host bdf = %x:%x:%x\n",
+	       assigned_dev->host_busnr,
+	       PCI_SLOT(assigned_dev->host_devfn),
+	       PCI_FUNC(assigned_dev->host_devfn));
+
+	pdev = assigned_dev->dev;
+
+	if (pdev == NULL) {
+		if (kvm->arch.intel_iommu_domain) {
+			intel_iommu_domain_exit(kvm->arch.intel_iommu_domain);
+			kvm->arch.intel_iommu_domain = NULL;
+		}
+		return -ENODEV;
+	}
+
+	kvm->arch.intel_iommu_domain = intel_iommu_domain_alloc(pdev);
+
+	rc = kvm_iommu_map_memslots(kvm);
+	if (rc)
+		goto out_unmap;
+
+	intel_iommu_detach_dev(kvm->arch.intel_iommu_domain,
+			       pdev->bus->number, pdev->devfn);
+
+	rc = intel_iommu_context_mapping(kvm->arch.intel_iommu_domain,
+					 pdev);
+	if (rc) {
+		printk(KERN_ERR "Domain context map for %s failed",
+		       pci_name(pdev));
+		goto out_unmap;
+	}
+	return 0;
+
+out_unmap:
+	kvm_iommu_unmap_memslots(kvm);
+	return rc;
+}
+
+static void kvm_iommu_put_pages(struct kvm *kvm,
+			       gfn_t base_gfn, unsigned long npages)
+{
+	gfn_t gfn = base_gfn;
+	pfn_t pfn;
+	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
+	int i;
+
+	for (i = 0; i < npages; i++) {
+		pfn = (pfn_t)intel_iommu_iova_to_pfn(domain,
+						     gfn_to_gpa(gfn));
+		kvm_release_pfn_clean(pfn);
+		gfn++;
+	}
+}
+
+static int kvm_iommu_unmap_memslots(struct kvm *kvm)
+{
+	int i;
+	down_read(&kvm->slots_lock);
+	for (i = 0; i < kvm->nmemslots; i++) {
+		kvm_iommu_put_pages(kvm, kvm->memslots[i].base_gfn,
+				    kvm->memslots[i].npages);
+	}
+	up_read(&kvm->slots_lock);
+
+	return 0;
+}
+
+int kvm_iommu_unmap_guest(struct kvm *kvm)
+{
+	struct kvm_assigned_dev_kernel *entry;
+	struct pci_dev *pdev = NULL;
+	struct dmar_domain *domain = kvm->arch.intel_iommu_domain;
+
+	/* check if iommu exists and in use */
+	if (!domain)
+		return 0;
+
+	list_for_each_entry(entry, &kvm->arch.assigned_dev_head, list) {
+		printk(KERN_DEBUG "VT-d unmap: host bdf = %x:%x:%x\n",
+		       entry->host_busnr,
+		       PCI_SLOT(entry->host_devfn),
+		       PCI_FUNC(entry->host_devfn));
+
+		for_each_pci_dev(pdev) {
+			if ((pdev->bus->number == entry->host_busnr) &&
+			    (pdev->devfn == entry->host_devfn))
+				break;
+		}
+
+		if (pdev == NULL)
+			return -ENODEV;
+
+		/* detach kvm dmar domain */
+		intel_iommu_detach_dev(domain,
+				       pdev->bus->number, pdev->devfn);
+	}
+	kvm_iommu_unmap_memslots(kvm);
+	intel_iommu_domain_exit(domain);
+	return 0;
+}
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index bfc7c33..40782df 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -35,6 +35,7 @@
 #include <linux/module.h>
 #include <linux/mman.h>
 #include <linux/highmem.h>
+#include <linux/intel-iommu.h>
 
 #include <asm/uaccess.h>
 #include <asm/msr.h>
@@ -276,9 +277,18 @@ static int kvm_vm_ioctl_assign_device(struct kvm *kvm,
 
 	list_add(&match->list, &kvm->arch.assigned_dev_head);
 
+	if (assigned_dev->flags & KVM_DEV_ASSIGN_USE_VTD) {
+		r = kvm_iommu_map_guest(kvm, match);
+		if (r)
+			goto out_list_del;
+	}
+
 out:
 	mutex_unlock(&kvm->lock);
 	return r;
+out_list_del:
+	list_del(&match->list);
+	pci_release_regions(dev);
 out_disable:
 	pci_disable_device(dev);
 out_put:
@@ -4264,6 +4274,7 @@ static void kvm_free_vcpus(struct kvm *kvm)
 
 void kvm_arch_destroy_vm(struct kvm *kvm)
 {
+	kvm_iommu_unmap_guest(kvm);
 	kvm_free_assigned_devices(kvm);
 	kvm_free_pit(kvm);
 	kfree(kvm->arch.vpic);
diff --git a/include/asm-x86/kvm_host.h b/include/asm-x86/kvm_host.h
index 1e28880..a48cd0a 100644
--- a/include/asm-x86/kvm_host.h
+++ b/include/asm-x86/kvm_host.h
@@ -364,6 +364,7 @@ struct kvm_arch{
 	 */
 	struct list_head active_mmu_pages;
 	struct list_head assigned_dev_head;
+	struct dmar_domain *intel_iommu_domain;
 	struct kvm_pic *vpic;
 	struct kvm_ioapic *vioapic;
 	struct kvm_pit *vpit;
@@ -513,6 +514,8 @@ int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
 int kvm_pv_mmu_op(struct kvm_vcpu *vcpu, unsigned long bytes,
 		  gpa_t addr, unsigned long *ret);
 
+int is_mmio_pfn(pfn_t pfn);
+
 extern bool tdp_enabled;
 
 enum emulation_result {
diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index d9ef7d3..2956e35 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -495,4 +495,6 @@ struct kvm_assigned_irq {
 	__u32 flags;
 };
 
+#define KVM_DEV_ASSIGN_USE_VTD	(1 << 1)
+
 #endif
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index a18aaad..b703890 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -285,6 +285,33 @@ int kvm_cpu_has_interrupt(struct kvm_vcpu *v);
 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
 void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
 
+#ifdef CONFIG_DMAR
+int kvm_iommu_map_pages(struct kvm *kvm, gfn_t base_gfn,
+			unsigned long npages);
+int kvm_iommu_map_guest(struct kvm *kvm,
+			struct kvm_assigned_dev_kernel *assigned_dev);
+int kvm_iommu_unmap_guest(struct kvm *kvm);
+#else /* CONFIG_DMAR */
+static inline int kvm_iommu_map_pages(struct kvm *kvm,
+				      gfn_t base_gfn,
+				      unsigned long npages)
+{
+	return 0;
+}
+
+static inline int kvm_iommu_map_guest(struct kvm *kvm,
+				      struct kvm_assigned_dev_kernel
+				      *assigned_dev)
+{
+	return -ENODEV;
+}
+
+static inline int kvm_iommu_unmap_guest(struct kvm *kvm)
+{
+	return 0;
+}
+#endif /* CONFIG_DMAR */
+
 static inline void kvm_guest_enter(void)
 {
 	account_system_vtime(current);
@@ -307,6 +334,11 @@ static inline gpa_t gfn_to_gpa(gfn_t gfn)
 	return (gpa_t)gfn << PAGE_SHIFT;
 }
 
+static inline hpa_t pfn_to_hpa(pfn_t pfn)
+{
+	return (hpa_t)pfn << PAGE_SHIFT;
+}
+
 static inline void kvm_migrate_timers(struct kvm_vcpu *vcpu)
 {
 	set_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests);
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 0309571..191bfe1 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -41,6 +41,7 @@
 #include <linux/pagemap.h>
 #include <linux/mman.h>
 #include <linux/swap.h>
+#include <linux/intel-iommu.h>
 
 #include <asm/processor.h>
 #include <asm/io.h>
@@ -76,7 +77,7 @@ static inline int valid_vcpu(int n)
 	return likely(n >= 0 && n < KVM_MAX_VCPUS);
 }
 
-static inline int is_mmio_pfn(pfn_t pfn)
+inline int is_mmio_pfn(pfn_t pfn)
 {
 	if (pfn_valid(pfn))
 		return PageReserved(pfn_to_page(pfn));
@@ -578,6 +579,12 @@ int __kvm_set_memory_region(struct kvm *kvm,
 	}
 
 	kvm_free_physmem_slot(&old, &new);
+
+	/* map the pages in iommu page table */
+	r = kvm_iommu_map_pages(kvm, base_gfn, npages);
+	if (r)
+		goto out_free;
+
 	return 0;
 
 out_free:
-- 
1.5.4.3

Re: VT-d support for device assignment

[Byron Stanoszek]

On Fri, 22 Aug 2008, Amit Shah wrote:

> The following two patches contain VT-d support for device assignment
> for KVM guests.
>
> The first patch contains the changes that are required to the generic
> VT-d code.
>
> The second patch contains the changes to KVM.
>
> I've updated the 2nd patch to use VT-d only when requested by a parameter
> on the command line, making it easier to support iommu with pvdma and
> multiple iommu types.
>
> The command line currently should be invoked as:
>
> -pcidevice dev=00:13.0,vtd=on

Amit,

Is it possible to log any physical PCI accesses from the guest?
This could be handy for reverse-engineering device drivers, among other things.

Thanks,
  -Byron

Re: VT-d support for device assignment

[Amit Shah]

* On Friday 22 Aug 2008 19:24:13 Byron Stanoszek wrote:
> On Fri, 22 Aug 2008, Amit Shah wrote:
> > The following two patches contain VT-d support for device assignment
> > for KVM guests.
> >
> > The first patch contains the changes that are required to the generic
> > VT-d code.
> >
> > The second patch contains the changes to KVM.
> >
> > I've updated the 2nd patch to use VT-d only when requested by a parameter
> > on the command line, making it easier to support iommu with pvdma and
> > multiple iommu types.
> >
> > The command line currently should be invoked as:
> >
> > -pcidevice dev=00:13.0,vtd=on
>
> Amit,
>
> Is it possible to log any physical PCI accesses from the guest?
> This could be handy for reverse-engineering device drivers, among other
> things.

Hello Byron,

All the PCI accesses happen from the userspace components in qemu; it's 
certainly possible to log activity.

Amit

Re: VT-d support for device assignment

[Muli Ben-Yehuda]

On Fri, Aug 22, 2008 at 09:54:13AM -0400, Byron Stanoszek wrote:

> Is it possible to log any physical PCI accesses from the guest?
> This could be handy for reverse-engineering device drivers, among
> other things.

With PCI passthrough It is possible in theory to log mmio's and pio's
executed by the guest, as well as interrupts delivered to the guest,
although it will have a performance impact. It is not possible to log
DMAs in a simple manner, but you could hook up a PCI analyzer to get
the missing bits.

Cheers,
Muli
-- 
Workshop on I/O Virtualization (WIOV '08)
Co-located with OSDI '08, Dec 2008, San Diego, CA
http://www.usenix.org/wiov08

Re: VT-d support for device assignment

[Avi Kivity]

Muli Ben-Yehuda wrote:
> With PCI passthrough It is possible in theory to log mmio's and pio's
> executed by the guest, as well as interrupts delivered to the guest,
> although it will have a performance impact. It is not possible to log
> DMAs in a simple manner, but you could hook up a PCI analyzer to get
> the missing bits.
>   

Well, with a pci analyzer you can log anything going in and out of the
device; you don't need a virtual machine monitor.

-- 
Do not meddle in the internals of kernels, for they are subtle and quick to panic.

Re: VT-d support for device assignment

[Avi Kivity]

Amit Shah wrote:
> The following two patches contain VT-d support for device assignment
> for KVM guests.
>
> The first patch contains the changes that are required to the generic
> VT-d code.
>
> The second patch contains the changes to KVM.
>
> I've updated the 2nd patch to use VT-d only when requested by a parameter
> on the command line, making it easier to support iommu with pvdma and
> multiple iommu types.
>
> The command line currently should be invoked as:
>
> -pcidevice dev=00:13.0,vtd=on
>   

You mean, iommu=on.

Or rather

  dma=iommu
  dma=none (1:1 mapping, or dma-less devices, or I'm Feeling Lucky)
  dma=cooperative (paravirt)

-- 
Do not meddle in the internals of kernels, for they are subtle and quick to panic.

Re: VT-d support for device assignment

[Muli Ben-Yehuda]

On Fri, Aug 22, 2008 at 09:12:32PM +0300, Avi Kivity wrote:
> Muli Ben-Yehuda wrote:
> > With PCI passthrough It is possible in theory to log mmio's and pio's
> > executed by the guest, as well as interrupts delivered to the guest,
> > although it will have a performance impact. It is not possible to log
> > DMAs in a simple manner, but you could hook up a PCI analyzer to get
> > the missing bits.
> >   
> 
> Well, with a pci analyzer you can log anything going in and out of
> the device; you don't need a virtual machine monitor.

Yep, that's true. I was thinking along the lines of doing the bulk of
reverse engineering without a PCI analyzer (if you understand the
device well enough, looking at the DMA data may not be all that
important) and then using a PCI analyzer just for any remaining bits.

By the way, with effort we should be able to capture some DMAs too,
either by taking "before" and "after" snapshots of VM memory (for DMA
write to memory) and/or by triggering DMA faults (by not mapping the
VM's memory in an isolation-capable IOMMU and enabling the IOMMU).

Cheers,
Muli
-- 
Workshop on I/O Virtualization (WIOV '08)
Co-located with OSDI '08, Dec 2008, San Diego, CA
http://www.usenix.org/wiov08

Re: VT-d support for device assignment

[Muli Ben-Yehuda]

On Fri, Aug 22, 2008 at 09:21:15PM +0300, Avi Kivity wrote:

> You mean, iommu=on.

Agreed. This should also be the default if not dma=xxx is specfied and
a capable IOMMU is found.

> Or rather
> 
>   dma=iommu
>   dma=none (1:1 mapping, or dma-less devices, or I'm Feeling Lucky)
>   dma=cooperative (paravirt)

What does the 'cooperative' mean? I assume you are thinking of
PVDMA-without-IOMMU, but in that case the name is misleading, since
you can do cooperative (PVDMA) just fine with an IOMMU. Perhaps
"dma=itrusttheguest", since with PVDMA0-without-IOMMU the guest is
trusted to not screw up the system?

Cheers,
Muli
-- 
Workshop on I/O Virtualization (WIOV '08)
Co-located with OSDI '08, Dec 2008, San Diego, CA
http://www.usenix.org/wiov08

Re: VT-d support for device assignment

[Muli Ben-Yehuda]

On Fri, Aug 22, 2008 at 09:02:33PM +0530, Amit Shah wrote:

> All the PCI accesses happen from the userspace components in qemu;
> it's certainly possible to log activity.

Actually, with direct-mmio we bypass QEMU and get a nice boost in
performance, and it should be possible to bypass QEMU for (most) PIO's
too.

Cheers,
Muli
-- 
Workshop on I/O Virtualization (WIOV '08)
Co-located with OSDI '08, Dec 2008, San Diego, CA
http://www.usenix.org/wiov08

Re: VT-d support for device assignment

[Avi Kivity]

Muli Ben-Yehuda wrote:
> On Fri, Aug 22, 2008 at 09:02:33PM +0530, Amit Shah wrote:
>
>   
>> All the PCI accesses happen from the userspace components in qemu;
>> it's certainly possible to log activity.
>>     
>
> Actually, with direct-mmio we bypass QEMU and get a nice boost in
> performance, and it should be possible to bypass QEMU for (most) PIO's
> too.
>   

Since there is no pio remapping, there's no way to ensure there's a free
pio range where the guest maps its ports.

Luckily pio is out of fashion these days.

-- 
Do not meddle in the internals of kernels, for they are subtle and quick to panic.

Re: VT-d support for device assignment

[Muli Ben-Yehuda]

On Fri, Aug 22, 2008 at 09:37:16PM +0300, Avi Kivity wrote:

> > Actually, with direct-mmio we bypass QEMU and get a nice boost in
> > performance, and it should be possible to bypass QEMU for (most)
> > PIO's too.
> 
> Since there is no pio remapping, there's no way to ensure there's a
> free pio range where the guest maps its ports.

True, and we also need to trap pio's for things like configuration
space accesses that the host needs to know about, such as BAR
re-mappings. Hence my (parenthetical) comment with regards to
"most". It's not at all clear that the gain will be worth the
complexity to decide which pio's to trap and which to let through
directly.

Cheers,
Muli
-- 
Workshop on I/O Virtualization (WIOV '08)
Co-located with OSDI '08, Dec 2008, San Diego, CA
http://www.usenix.org/wiov08

Re: VT-d support for device assignment

[Amit Shah]

* On Friday 22 Aug 2008 23:51:15 Avi Kivity wrote:
> Amit Shah wrote:
> > The following two patches contain VT-d support for device assignment
> > for KVM guests.
> >
> > The first patch contains the changes that are required to the generic
> > VT-d code.
> >
> > The second patch contains the changes to KVM.
> >
> > I've updated the 2nd patch to use VT-d only when requested by a parameter
> > on the command line, making it easier to support iommu with pvdma and
> > multiple iommu types.
> >
> > The command line currently should be invoked as:
> >
> > -pcidevice dev=00:13.0,vtd=on
>
> You mean, iommu=on.

I did mean vtd=on, since we'll also have AMD's iommu implementation here.

So something like:

-pcidevice dev=00:13.0,vtd=on,pvdma=on

or

-pcidevice dev=00:13.0,amd-iommu=on,pvdma=on

or do you mean we should autodetect which IOMMU we have and then select the 
appropriate one instead of bothering the user with it? Hmm, that seems a 
better UI and also such startup scripts can be ported across architectures.

> Or rather
>
>   dma=iommu
>   dma=none (1:1 mapping, or dma-less devices, or I'm Feeling Lucky)
>   dma=cooperative (paravirt)

This looks much better!

Once we have KVM_CAP_VTD,  KVM_CAP_AMD_IOMMU and KVM_CAP_PVDMA,

dma=iommu means use either of VTD or AMD, whichever one is available
dma=none means I'm feeling lucky

PVDMA will automatically get used if the guest has PVDMA support compiled in. 
Enabling/disabling pvdma would be a guest option rather than a host option, I 
think (host only exposes CAP_PVDMA).

Is this ok?

Amit

Re: VT-d support for device assignment

[Avi Kivity]

Amit Shah wrote:
> * On Friday 22 Aug 2008 23:51:15 Avi Kivity wrote:
>   
>> Amit Shah wrote:
>>     
>>> The following two patches contain VT-d support for device assignment
>>> for KVM guests.
>>>
>>> The first patch contains the changes that are required to the generic
>>> VT-d code.
>>>
>>> The second patch contains the changes to KVM.
>>>
>>> I've updated the 2nd patch to use VT-d only when requested by a parameter
>>> on the command line, making it easier to support iommu with pvdma and
>>> multiple iommu types.
>>>
>>> The command line currently should be invoked as:
>>>
>>> -pcidevice dev=00:13.0,vtd=on
>>>       
>> You mean, iommu=on.
>>     
>
> I did mean vtd=on, since we'll also have AMD's iommu implementation here.
>
> So something like:
>
> -pcidevice dev=00:13.0,vtd=on,pvdma=on
>
> or
>
> -pcidevice dev=00:13.0,amd-iommu=on,pvdma=on
>
> or do you mean we should autodetect which IOMMU we have and then select the 
> appropriate one instead of bothering the user with it? Hmm, that seems a 
> better UI and also such startup scripts can be ported across architectures.
>
>   

Yes of course.  Note there's no need for kvm to autodetect the iommu 
either; I won't let the amd iommu in without a proper abstraction via an 
iommu api.

>> Or rather
>>
>>   dma=iommu
>>   dma=none (1:1 mapping, or dma-less devices, or I'm Feeling Lucky)
>>   dma=cooperative (paravirt)
>>     
>
> This looks much better!
>
> Once we have KVM_CAP_VTD,  KVM_CAP_AMD_IOMMU and KVM_CAP_PVDMA,
>   

Why KVM_CAP_VTD and KVM_CAP_AMD_IOMMU?  Do they actually have 
differences?  if so, the capabilities should report the differences as 
features, not as vendor identifiers.

> dma=iommu means use either of VTD or AMD, whichever one is available
> dma=none means I'm feeling lucky
>
> PVDMA will automatically get used if the guest has PVDMA support compiled in. 
> Enabling/disabling pvdma would be a guest option rather than a host option, I 
> think (host only exposes CAP_PVDMA).
>
> Is this ok?
>   

So long as there is no potential for performance or security impact, 
having pvdma turned on automatically is better.  We could still have 
dma=noparavirt to disable it.

-- 
I have a truly marvellous patch that fixes the bug which this
signature is too narrow to contain.

Re: VT-d support for device assignment

[Amit Shah]

* On Saturday 23 Aug 2008 15:03:46 Avi Kivity wrote:
> Amit Shah wrote:
> > * On Friday 22 Aug 2008 23:51:15 Avi Kivity wrote:
> >> Amit Shah wrote:
> >>> The following two patches contain VT-d support for device assignment
> >>> for KVM guests.
> >>>
> >>> The first patch contains the changes that are required to the generic
> >>> VT-d code.
> >>>
> >>> The second patch contains the changes to KVM.
> >>>
> >>> I've updated the 2nd patch to use VT-d only when requested by a
> >>> parameter on the command line, making it easier to support iommu with
> >>> pvdma and multiple iommu types.
> >>>
> >>> The command line currently should be invoked as:
> >>>
> >>> -pcidevice dev=00:13.0,vtd=on
> >>
> >> You mean, iommu=on.
> >
> > I did mean vtd=on, since we'll also have AMD's iommu implementation here.
> >
> > So something like:
> >
> > -pcidevice dev=00:13.0,vtd=on,pvdma=on
> >
> > or
> >
> > -pcidevice dev=00:13.0,amd-iommu=on,pvdma=on
> >
> > or do you mean we should autodetect which IOMMU we have and then select
> > the appropriate one instead of bothering the user with it? Hmm, that
> > seems a better UI and also such startup scripts can be ported across
> > architectures.
>
> Yes of course.  Note there's no need for kvm to autodetect the iommu
> either; I won't let the amd iommu in without a proper abstraction via an
> iommu api.

Yes; we're going to have an API once the Intel IOMMU support goes in.

> >> Or rather
> >>
> >>   dma=iommu
> >>   dma=none (1:1 mapping, or dma-less devices, or I'm Feeling Lucky)
> >>   dma=cooperative (paravirt)
> >
> > This looks much better!
> >
> > Once we have KVM_CAP_VTD,  KVM_CAP_AMD_IOMMU and KVM_CAP_PVDMA,
>
> Why KVM_CAP_VTD and KVM_CAP_AMD_IOMMU?  Do they actually have
> differences?  if so, the capabilities should report the differences as
> features, not as vendor identifiers.

Agreed.

> > dma=iommu means use either of VTD or AMD, whichever one is available
> > dma=none means I'm feeling lucky
> >
> > PVDMA will automatically get used if the guest has PVDMA support compiled
> > in. Enabling/disabling pvdma would be a guest option rather than a host
> > option, I think (host only exposes CAP_PVDMA).
> >
> > Is this ok?
>
> So long as there is no potential for performance or security impact,
> having pvdma turned on automatically is better.  We could still have
> dma=noparavirt to disable it.

So we fail the hypercalls once the guest tries to contact us?

Amit

Re: VT-d support for device assignment

[Muli Ben-Yehuda]

On Sat, Aug 23, 2008 at 12:33:46PM +0300, Avi Kivity wrote:

> So long as there is no potential for performance or security impact,
> having pvdma turned on automatically is better.  We could still have
> dma=noparavirt to disable it.

pvdma has advantages and disadvantages. It has a significant
performance impact (compared to direct map) but it also let us avoid
pinning all of the guest's memory. I this this should be a guest
decision whether to use it or not, so 'dma=noparavirt' to disable it
seems reasonable.

Cheers,
Muli
-- 
Workshop on I/O Virtualization (WIOV '08)
Co-located with OSDI '08, Dec 2008, San Diego, CA
http://www.usenix.org/wiov08

Re: VT-d support for device assignment

[Muli Ben-Yehuda]

On Sat, Aug 23, 2008 at 12:50:25PM +0300, Muli Ben-Yehuda wrote:
> On Sat, Aug 23, 2008 at 12:33:46PM +0300, Avi Kivity wrote:
> 
> > So long as there is no potential for performance or security impact,
> > having pvdma turned on automatically is better.  We could still have
> > dma=noparavirt to disable it.
> 
> pvdma has advantages and disadvantages. It has a significant
> performance impact (compared to direct map) but it also let us avoid
> pinning all of the guest's memory. I this this should be a guest
> decision whether to use it or not, so 'dma=noparavirt' to disable it
> seems reasonable.

Just to clarify---I am talking about pvdma WITH an iommu, where there
are no protection implications to having pvdma. pvdma without an IOMMU
is a whole different ballgame and means we trust the guest---that
should definitely be a host option, not a default.

Cheers,
Muli
-- 
Workshop on I/O Virtualization (WIOV '08)
Co-located with OSDI '08, Dec 2008, San Diego, CA
http://www.usenix.org/wiov08

Re: VT-d support for device assignment

[Muli Ben-Yehuda]

On Fri, Aug 22, 2008 at 10:10:52AM +0300, Amit Shah wrote:

> The second patch contains the changes to KVM.
> 
> I've updated the 2nd patch to use VT-d only when requested by a
> parameter on the command line, making it easier to support iommu
> with pvdma and multiple iommu types.
> 
> The command line currently should be invoked as:
> 
> -pcidevice dev=00:13.0,vtd=on

Could you please send your changes as a seperate, follow-on patch?
The original patch is complicated enough, and that way the authorship
information does not get muddled further, and it's much easier to see
what you changed.

Cheers,
Muli
-- 
Workshop on I/O Virtualization (WIOV '08)
Co-located with OSDI '08, Dec 2008, San Diego, CA
http://www.usenix.org/wiov08

Re: VT-d support for device assignment

[Amit Shah]

* On Saturday 23 Aug 2008 15:27:47 Muli Ben-Yehuda wrote:
> On Fri, Aug 22, 2008 at 10:10:52AM +0300, Amit Shah wrote:
> > The second patch contains the changes to KVM.
> >
> > I've updated the 2nd patch to use VT-d only when requested by a
> > parameter on the command line, making it easier to support iommu
> > with pvdma and multiple iommu types.
> >
> > The command line currently should be invoked as:
> >
> > -pcidevice dev=00:13.0,vtd=on
>
> Could you please send your changes as a seperate, follow-on patch?
> The original patch is complicated enough, and that way the authorship
> information does not get muddled further, and it's much easier to see
> what you changed.

I already sent my patch in a separate mail.

The authorship info and the commit log stays the same; just contains my 
signoff.

A new patch based on our current discussion will have to be spun anyway.

Amit

Re: VT-d support for device assignment

[Muli Ben-Yehuda]

On Sat, Aug 23, 2008 at 03:55:25PM +0530, Amit Shah wrote:

> The authorship info and the commit log stays the same; just contains
> my signoff.

Actually, unless you add an explicit 'From:' header, the email From
header is used by git as the author of the patch.

Cheers,
Muli
-- 
Workshop on I/O Virtualization (WIOV '08)
Co-located with OSDI '08, Dec 2008, San Diego, CA
http://www.usenix.org/wiov08

Re: VT-d support for device assignment

[Amit Shah]

* On Saturday 23 Aug 2008 16:10:54 Muli Ben-Yehuda wrote:
> On Sat, Aug 23, 2008 at 03:55:25PM +0530, Amit Shah wrote:
> > The authorship info and the commit log stays the same; just contains
> > my signoff.
>
> Actually, unless you add an explicit 'From:' header, the email From
> header is used by git as the author of the patch.

If you notice, Patch 1/2, VT-d one has the From: line set to Allen's name (as 
Ben had sent). The patch 2/2 was modified by me, so it gets my 'From'.

Amit

Re: VT-d support for device assignment

[Muli Ben-Yehuda]

On Sat, Aug 23, 2008 at 04:41:02PM +0530, Amit Shah wrote:
> * On Saturday 23 Aug 2008 16:10:54 Muli Ben-Yehuda wrote:
> > On Sat, Aug 23, 2008 at 03:55:25PM +0530, Amit Shah wrote:
> > > The authorship info and the commit log stays the same; just contains
> > > my signoff.
> >
> > Actually, unless you add an explicit 'From:' header, the email From
> > header is used by git as the author of the patch.
> 
> If you notice, Patch 1/2, VT-d one has the From: line set to Allen's
> name (as Ben had sent). The patch 2/2 was modified by me, so it gets
> my 'From'.

Hence my comment about muddled authorship information and why it was
better to keep your changes as a separate patch.

Anyway, moving this thread to more constructive avenues: I think you
said you are planning to implement and submit Avi's dma=xxx
suggestion, and also had an outstanding userspace patch?

Cheers,
Muli
-- 
Workshop on I/O Virtualization (WIOV '08)
Co-located with OSDI '08, Dec 2008, San Diego, CA
http://www.usenix.org/wiov08

Re: VT-d support for device assignment

[Amit Shah]

* On Saturday 23 Aug 2008 17:41:32 Muli Ben-Yehuda wrote:
> On Sat, Aug 23, 2008 at 04:41:02PM +0530, Amit Shah wrote:
> > * On Saturday 23 Aug 2008 16:10:54 Muli Ben-Yehuda wrote:
> > > On Sat, Aug 23, 2008 at 03:55:25PM +0530, Amit Shah wrote:
> > > > The authorship info and the commit log stays the same; just contains
> > > > my signoff.
> > >
> > > Actually, unless you add an explicit 'From:' header, the email From
> > > header is used by git as the author of the patch.
> >
> > If you notice, Patch 1/2, VT-d one has the From: line set to Allen's
> > name (as Ben had sent). The patch 2/2 was modified by me, so it gets
> > my 'From'.
>
> Hence my comment about muddled authorship information and why it was
> better to keep your changes as a separate patch.

I don't know what you mean; isn't that the way patches flow? This is how 
collaboration is shown.

> Anyway, moving this thread to more constructive avenues: I think you
> said you are planning to implement and submit Avi's dma=xxx
> suggestion, and also had an outstanding userspace patch?

Yes; both are in the baking oven.

Amit.

Re: [PATCH] VT-d: changes to support KVM

[Jesse Barnes]

On Friday, August 22, 2008 12:10 am Amit Shah wrote:
> From: Kay, Allen M <allen.m.kay@intel.com>
>
> This patch extends the VT-d driver to support KVM
>
> [Ben: fixed memory pinning]
>
> Signed-off-by: Kay, Allen M <allen.m.kay@intel.com>
> Signed-off-by: Weidong Han <weidong.han@intel.com>
> Signed-off-by: Ben-Ami Yassour <benami@il.ibm.com>
> Signed-off-by: Amit Shah <amit.shah@qumranet.com>
> ---
>  drivers/pci/dmar.c          |    4 +-
>  drivers/pci/intel-iommu.c   |  117 ++++++++++++++-
>  drivers/pci/intel-iommu.h   |  344
> ----------------------------------------- drivers/pci/iova.c          |   
> 2 +-
>  drivers/pci/iova.h          |   52 -------
>  include/linux/intel-iommu.h |  355
> +++++++++++++++++++++++++++++++++++++++++++ include/linux/iova.h        |  
> 52 +++++++
>  7 files changed, 523 insertions(+), 403 deletions(-)
>  delete mode 100644 drivers/pci/intel-iommu.h
>  delete mode 100644 drivers/pci/iova.h
>  create mode 100644 include/linux/intel-iommu.h
>  create mode 100644 include/linux/iova.h

Assuming Mark is ok with this, I can put this part into linux-next...  But 
you'll have to re-send to my personal account (jbarnes@virtuousgeek.org) 
since this one got whitespace mangled by Outlook/Exchange.

Thanks,
-- 
Jesse Barnes, Intel Open Source Technology Center

Re: VT-d support for device assignment

[Avi Kivity]

Amit Shah wrote:
>> So long as there is no potential for performance or security impact,
>> having pvdma turned on automatically is better.  We could still have
>> dma=noparavirt to disable it.
>>     
>
> So we fail the hypercalls once the guest tries to contact us?
>   

Yes.  Also don't expose the feature bit to the guest.

-- 
error compiling committee.c: too many arguments to function

Re: VT-d support for device assignment

[Avi Kivity]

Muli Ben-Yehuda wrote:
> On Sat, Aug 23, 2008 at 12:33:46PM +0300, Avi Kivity wrote:
>
>   
>> So long as there is no potential for performance or security impact,
>> having pvdma turned on automatically is better.  We could still have
>> dma=noparavirt to disable it.
>>     
>
> pvdma has advantages and disadvantages. It has a significant
> performance impact (compared to direct map) 

That needs to be fixed.


-- 
error compiling committee.c: too many arguments to function

VT-d support for device assignment

[Amit Shah]

The following two patches contain VT-d support for device assignment for KVM guests.

The first patch contains the changes that are required to the generic VT-d code.

The second patch contains the changes to KVM.

Since the last send, I've updated the 2nd patch to expose KVM_CAP_IOMMU and check for the existence of an IOMMU before using one. I've also preserved the 'From' field for both the patches so that the original authors are credited for the patches.

The command line currently should be invoked with the parameter:

-pcidevice dev=00:13.0,dma=iommu

Re: VT-d support for device assignment

[Avi Kivity]

Amit Shah wrote:
> The following two patches contain VT-d support for device assignment for KVM guests.
>
> The first patch contains the changes that are required to the generic VT-d code.
>
> The second patch contains the changes to KVM.
>
> Since the last send, I've updated the 2nd patch to expose KVM_CAP_IOMMU and check for the existence of an IOMMU before using one. I've also preserved the 'From' field for both the patches so that the original authors are credited for the patches.
>
> The command line currently should be invoked with the parameter:
>
> -pcidevice dev=00:13.0,dma=iommu
>   

Why not default to iommu if one is available?

btw, -usbdevice prefixes host devices with host:, no?  maybe adopt the 
convention for -pcidevice?

-- 
error compiling committee.c: too many arguments to function

Re: VT-d support for device assignment

[Amit Shah]

* On Tuesday 26 Aug 2008 14:40:57 Avi Kivity wrote:
> Amit Shah wrote:
> > The following two patches contain VT-d support for device assignment for
> > KVM guests.
> >
> > The first patch contains the changes that are required to the generic
> > VT-d code.
> >
> > The second patch contains the changes to KVM.
> >
> > Since the last send, I've updated the 2nd patch to expose KVM_CAP_IOMMU
> > and check for the existence of an IOMMU before using one. I've also
> > preserved the 'From' field for both the patches so that the original
> > authors are credited for the patches.
> >
> > The command line currently should be invoked with the parameter:
> >
> > -pcidevice dev=00:13.0,dma=iommu
>
> Why not default to iommu if one is available?

I've set that as default now.

What should the parameter be to disable just the iommu? dma=noiommu looks 
ugly.

Or do we not want to disable an iommu at all (even for debugging or testing 
something else, like pvdma)?

> btw, -usbdevice prefixes host devices with host:, no?  maybe adopt the
> convention for -pcidevice?

Sounds ok.

Re: VT-d support for device assignment

[Avi Kivity]

Amit Shah wrote:
>>> -pcidevice dev=00:13.0,dma=iommu
>>>       
>> Why not default to iommu if one is available?
>>     
>
> I've set that as default now.
>
> What should the parameter be to disable just the iommu? dma=noiommu looks 
> ugly.
>   

dma=none ?

> Or do we not want to disable an iommu at all (even for debugging or testing 
> something else, like pvdma)?
>
>   

Good question.  Maybe it's safer to always enable it, and to let 
developers hack it out.

We must be careful of people's feet.

-- 
error compiling committee.c: too many arguments to function

Re: VT-d support for device assignment

[Joerg Roedel]

On Fri, Aug 22, 2008 at 10:10:52AM +0300, Amit Shah wrote:
> 
> The following two patches contain VT-d support for device assignment
> for KVM guests.
> 
> The first patch contains the changes that are required to the generic
> VT-d code.
> 
> The second patch contains the changes to KVM.
> 
> I've updated the 2nd patch to use VT-d only when requested by a parameter
> on the command line, making it easier to support iommu with pvdma and
> multiple iommu types.
> 
> The command line currently should be invoked as:
> 
> -pcidevice dev=00:13.0,vtd=on


I don't like the vtd=on flag. I think its not good to have seperate
parameters for Intel and AMD IOMMUs. So I think a parameter called
iommu=on is the better choice.

Joerg

VT-d support for device assignment

[Han, Weidong]

The following two patches contain VT-d support for device assignment for
KVM guests.

The first patch contains the changes that are required to the generic
VT-d code.

The second patch contains the changes to KVM.

Mark has acked on these two patches. Since the last send, I've updated
the first patch to remove "+#undef DEBUG", and added some error handling
on the second patch according to Amit's comments.

Becuase Amit is busy on other things this week, instead I send out these
patches.

VT-d support for device assignment

[Amit Shah]

These are based on the patches Weidong just sent, with some whitespace cleanups and one checkpatch fix for the 2/2 patch.

Please apply.

Amit.

VT-d support for device assignment

[Amit Shah]

Sorry for the resends; this one fixes two compile errors introduced by me and a warning.

Re: VT-d support for device assignment

[Avi Kivity]

Amit Shah wrote:
> Sorry for the resends; this one fixes two compile errors introduced by me and a warning.
>   


Applied both, thanks.

-- 
I have a truly marvellous patch that fixes the bug which this
signature is too narrow to contain.

Re: VT-d support for device assignment

[Jesse Barnes]

On Saturday, September 13, 2008 5:46 pm Avi Kivity wrote:
> Amit Shah wrote:
> > Sorry for the resends; this one fixes two compile errors introduced by me
> > and a warning.
>
> Applied both, thanks.

You can add my s-o-b to the IOMMU patch if you want Avi, it's fine with me if 
you push those particular PCI bits (along with the recent build fix).

Thanks,
Jesse