[PATCH 08/13] allocation and free functions of virtual machine domain

[PATCH 08/13] allocation and free functions of virtual machine domain

[Han, Weidong]

[-- Attachment #1: Type: text/plain, Size: 3719 bytes --]


Signed-off-by: Weidong Han <weidong.han@intel.com>
---
 drivers/pci/intel-iommu.c |  104 ++++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 103 insertions(+), 1 deletions(-)

diff --git a/drivers/pci/intel-iommu.c b/drivers/pci/intel-iommu.c
index b00a8f2..e96b3bc 100644
--- a/drivers/pci/intel-iommu.c
+++ b/drivers/pci/intel-iommu.c
@@ -947,6 +947,7 @@ static int iommu_init_domains(struct intel_iommu *iommu)
 
 
 static void domain_exit(struct dmar_domain *domain);
+static void vm_domain_exit(struct dmar_domain *domain);
 
 void free_dmar_iommu(struct intel_iommu *iommu)
 {
@@ -957,8 +958,13 @@ void free_dmar_iommu(struct intel_iommu *iommu)
 	for (; i < cap_ndoms(iommu->cap); ) {
 		domain = iommu->domains[i];
 		clear_bit(i, iommu->domain_ids);
-		if (--domain->iommu_count == 0)
+
+		if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) {
+			if (--domain->iommu_count == 0)
+				vm_domain_exit(domain);
+		} else
 			domain_exit(domain);
+
 		i = find_next_bit(iommu->domain_ids,
 			cap_ndoms(iommu->cap), i+1);
 	}
@@ -2492,6 +2498,102 @@ static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
 	spin_unlock_irqrestore(&device_domain_lock, flags);
 }
 
+/* domain id for virtual machine, it won't be set in context */
+static unsigned long vm_domid;
+
+static struct dmar_domain *iommu_alloc_vm_domain(void)
+{
+	struct dmar_domain *domain;
+
+	domain = alloc_domain_mem();
+	if (!domain)
+		return NULL;
+
+	domain->id = vm_domid++;
+	domain->iommu_count = 0;
+	domain->iommu_coherency = 0;
+	memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
+	domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
+
+	return domain;
+}
+
+static int vm_domain_init(struct dmar_domain *domain, int guest_width)
+{
+	int adjust_width;
+
+	init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
+	spin_lock_init(&domain->mapping_lock);
+
+	domain_reserve_special_ranges(domain);
+
+	/* calculate AGAW */
+	domain->gaw = guest_width;
+	adjust_width = guestwidth_to_adjustwidth(guest_width);
+	domain->agaw = width_to_agaw(adjust_width);
+
+	INIT_LIST_HEAD(&domain->devices);
+
+	/* always allocate the top pgd */
+	domain->pgd = (struct dma_pte *)alloc_pgtable_page();
+	if (!domain->pgd)
+		return -ENOMEM;
+	domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
+	return 0;
+}
+
+static void iommu_free_vm_domain(struct dmar_domain *domain)
+{
+	unsigned long flags;
+	struct dmar_drhd_unit *drhd;
+	struct intel_iommu *iommu;
+	unsigned long i;
+	unsigned long ndomains;
+
+	for_each_drhd_unit(drhd) {
+		if (drhd->ignored)
+			continue;
+		iommu = drhd->iommu;
+
+		ndomains = cap_ndoms(iommu->cap);
+		i = find_first_bit(iommu->domain_ids, ndomains);
+		for (; i < ndomains; ) {
+			if (iommu->domains[i] == domain) {
+				spin_lock_irqsave(&iommu->lock, flags);
+				clear_bit(i, iommu->domain_ids);
+				iommu->domains[i] = NULL;
+				spin_unlock_irqrestore(&iommu->lock, flags);
+				break;
+			}
+			i = find_next_bit(iommu->domain_ids, ndomains, i+1);
+		}
+	}
+}
+
+static void vm_domain_exit(struct dmar_domain *domain)
+{
+	u64 end;
+
+	/* Domain 0 is reserved, so dont process it */
+	if (!domain)
+		return;
+
+	vm_domain_remove_all_dev_info(domain);
+	/* destroy iovas */
+	put_iova_domain(&domain->iovad);
+	end = DOMAIN_MAX_ADDR(domain->gaw);
+	end = end & (~VTD_PAGE_MASK);
+
+	/* clear ptes */
+	dma_pte_clear_range(domain, 0, end);
+
+	/* free page tables */
+	dma_pte_free_pagetable(domain, 0, end);
+
+	iommu_free_vm_domain(domain);
+	free_domain_mem(domain);
+}
+
 void intel_iommu_domain_exit(struct dmar_domain *domain)
 {
 	u64 end;
-- 
1.5.1

[-- Attachment #2: 0008-allocation-and-free-functions-of-virtual-machine-do.patch --]
[-- Type: application/octet-stream, Size: 3806 bytes --]

From fbd3b41211945db892f966afebd76534bcf8fc7b Mon Sep 17 00:00:00 2001
From: Weidong Han <weidong.han@intel.com>
Date: Tue, 2 Dec 2008 16:43:48 +0800
Subject: [PATCH] allocation and free functions of virtual machine domain


Signed-off-by: Weidong Han <weidong.han@intel.com>
---
 drivers/pci/intel-iommu.c |  104 ++++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 103 insertions(+), 1 deletions(-)

diff --git a/drivers/pci/intel-iommu.c b/drivers/pci/intel-iommu.c
index b00a8f2..e96b3bc 100644
--- a/drivers/pci/intel-iommu.c
+++ b/drivers/pci/intel-iommu.c
@@ -947,6 +947,7 @@ static int iommu_init_domains(struct intel_iommu *iommu)
 
 
 static void domain_exit(struct dmar_domain *domain);
+static void vm_domain_exit(struct dmar_domain *domain);
 
 void free_dmar_iommu(struct intel_iommu *iommu)
 {
@@ -957,8 +958,13 @@ void free_dmar_iommu(struct intel_iommu *iommu)
 	for (; i < cap_ndoms(iommu->cap); ) {
 		domain = iommu->domains[i];
 		clear_bit(i, iommu->domain_ids);
-		if (--domain->iommu_count == 0)
+
+		if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) {
+			if (--domain->iommu_count == 0)
+				vm_domain_exit(domain);
+		} else
 			domain_exit(domain);
+
 		i = find_next_bit(iommu->domain_ids,
 			cap_ndoms(iommu->cap), i+1);
 	}
@@ -2492,6 +2498,102 @@ static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
 	spin_unlock_irqrestore(&device_domain_lock, flags);
 }
 
+/* domain id for virtual machine, it won't be set in context */
+static unsigned long vm_domid;
+
+static struct dmar_domain *iommu_alloc_vm_domain(void)
+{
+	struct dmar_domain *domain;
+
+	domain = alloc_domain_mem();
+	if (!domain)
+		return NULL;
+
+	domain->id = vm_domid++;
+	domain->iommu_count = 0;
+	domain->iommu_coherency = 0;
+	memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
+	domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
+
+	return domain;
+}
+
+static int vm_domain_init(struct dmar_domain *domain, int guest_width)
+{
+	int adjust_width;
+
+	init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
+	spin_lock_init(&domain->mapping_lock);
+
+	domain_reserve_special_ranges(domain);
+
+	/* calculate AGAW */
+	domain->gaw = guest_width;
+	adjust_width = guestwidth_to_adjustwidth(guest_width);
+	domain->agaw = width_to_agaw(adjust_width);
+
+	INIT_LIST_HEAD(&domain->devices);
+
+	/* always allocate the top pgd */
+	domain->pgd = (struct dma_pte *)alloc_pgtable_page();
+	if (!domain->pgd)
+		return -ENOMEM;
+	domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
+	return 0;
+}
+
+static void iommu_free_vm_domain(struct dmar_domain *domain)
+{
+	unsigned long flags;
+	struct dmar_drhd_unit *drhd;
+	struct intel_iommu *iommu;
+	unsigned long i;
+	unsigned long ndomains;
+
+	for_each_drhd_unit(drhd) {
+		if (drhd->ignored)
+			continue;
+		iommu = drhd->iommu;
+
+		ndomains = cap_ndoms(iommu->cap);
+		i = find_first_bit(iommu->domain_ids, ndomains);
+		for (; i < ndomains; ) {
+			if (iommu->domains[i] == domain) {
+				spin_lock_irqsave(&iommu->lock, flags);
+				clear_bit(i, iommu->domain_ids);
+				iommu->domains[i] = NULL;
+				spin_unlock_irqrestore(&iommu->lock, flags);
+				break;
+			}
+			i = find_next_bit(iommu->domain_ids, ndomains, i+1);
+		}
+	}
+}
+
+static void vm_domain_exit(struct dmar_domain *domain)
+{
+	u64 end;
+
+	/* Domain 0 is reserved, so dont process it */
+	if (!domain)
+		return;
+
+	vm_domain_remove_all_dev_info(domain);
+	/* destroy iovas */
+	put_iova_domain(&domain->iovad);
+	end = DOMAIN_MAX_ADDR(domain->gaw);
+	end = end & (~VTD_PAGE_MASK);
+
+	/* clear ptes */
+	dma_pte_clear_range(domain, 0, end);
+
+	/* free page tables */
+	dma_pte_free_pagetable(domain, 0, end);
+
+	iommu_free_vm_domain(domain);
+	free_domain_mem(domain);
+}
+
 void intel_iommu_domain_exit(struct dmar_domain *domain)
 {
 	u64 end;
-- 
1.5.1

Re: [PATCH 08/13] allocation and free functions of virtual machine domain

[Mark McLoughlin]

On Tue, 2008-12-02 at 22:22 +0800, Han, Weidong wrote:
> Signed-off-by: Weidong Han <weidong.han@intel.com>
> ---
>  drivers/pci/intel-iommu.c |  104 ++++++++++++++++++++++++++++++++++++++++++++-
>  1 files changed, 103 insertions(+), 1 deletions(-)
> 
> diff --git a/drivers/pci/intel-iommu.c b/drivers/pci/intel-iommu.c
> index b00a8f2..e96b3bc 100644
> --- a/drivers/pci/intel-iommu.c
> +++ b/drivers/pci/intel-iommu.c
> @@ -947,6 +947,7 @@ static int iommu_init_domains(struct intel_iommu *iommu)
>  
> 
>  static void domain_exit(struct dmar_domain *domain);
> +static void vm_domain_exit(struct dmar_domain *domain);
>  
>  void free_dmar_iommu(struct intel_iommu *iommu)
>  {
> @@ -957,8 +958,13 @@ void free_dmar_iommu(struct intel_iommu *iommu)
>  	for (; i < cap_ndoms(iommu->cap); ) {
>  		domain = iommu->domains[i];
>  		clear_bit(i, iommu->domain_ids);
> -		if (--domain->iommu_count == 0)
> +
> +		if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) {
> +			if (--domain->iommu_count == 0)
> +				vm_domain_exit(domain);
> +		} else
>  			domain_exit(domain);
> +

Again, these new functions are copies of existing code with minor
modifications. I'd much rather see the existing code refactored and then
modified to handle the DOMAIN_FLAG_VIRTUAL_MACHINE case.

Cheers,
Mark.

Re: [PATCH 08/13] allocation and free functions of virtual machine domain

[Joerg Roedel]

On Thu, Dec 04, 2008 at 05:13:00PM +0000, Mark McLoughlin wrote:
> On Tue, 2008-12-02 at 22:22 +0800, Han, Weidong wrote:
> > Signed-off-by: Weidong Han <weidong.han@intel.com>
> > ---
> >  drivers/pci/intel-iommu.c |  104 ++++++++++++++++++++++++++++++++++++++++++++-
> >  1 files changed, 103 insertions(+), 1 deletions(-)
> > 
> > diff --git a/drivers/pci/intel-iommu.c b/drivers/pci/intel-iommu.c
> > index b00a8f2..e96b3bc 100644
> > --- a/drivers/pci/intel-iommu.c
> > +++ b/drivers/pci/intel-iommu.c
> > @@ -947,6 +947,7 @@ static int iommu_init_domains(struct intel_iommu *iommu)
> >  
> > 
> >  static void domain_exit(struct dmar_domain *domain);
> > +static void vm_domain_exit(struct dmar_domain *domain);
> >  
> >  void free_dmar_iommu(struct intel_iommu *iommu)
> >  {
> > @@ -957,8 +958,13 @@ void free_dmar_iommu(struct intel_iommu *iommu)
> >  	for (; i < cap_ndoms(iommu->cap); ) {
> >  		domain = iommu->domains[i];
> >  		clear_bit(i, iommu->domain_ids);
> > -		if (--domain->iommu_count == 0)
> > +
> > +		if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) {
> > +			if (--domain->iommu_count == 0)
> > +				vm_domain_exit(domain);
> > +		} else
> >  			domain_exit(domain);
> > +
> 
> Again, these new functions are copies of existing code with minor
> modifications. I'd much rather see the existing code refactored and then
> modified to handle the DOMAIN_FLAG_VIRTUAL_MACHINE case.

Hey Mark,

can your objections be fixed by follow-up patches bei Han or is anything
critical in it? My AMD IOMMU patches for KVM support depend on these
patches and everything they are changed I have to rebase by work. So I
would prefer if Han can fix the issues found by follow-up patches :)

Joerg

Re: [PATCH 08/13] allocation and free functions of virtual machine domain

[Mark McLoughlin]

Hi Joerg,

On Fri, 2008-12-05 at 00:17 +0100, Joerg Roedel wrote:

> > Again, these new functions are copies of existing code with minor
> > modifications. I'd much rather see the existing code refactored and then
> > modified to handle the DOMAIN_FLAG_VIRTUAL_MACHINE case.
> 
> Hey Mark,
> 
> can your objections be fixed by follow-up patches bei Han or is anything
> critical in it? My AMD IOMMU patches for KVM support depend on these
> patches and everything they are changed I have to rebase by work. So I
> would prefer if Han can fix the issues found by follow-up patches :)

Well, Weidong is going to have to rebase to dwmw2's tree anyway. I
assume he'll fix at least some of the issues in the process of doing
that.

But you're right in that nothing I pointed out was a complete
showstopper, just ways in which things could be done more cleanly.

Cheers,
Mark.

RE: [PATCH 08/13] allocation and free functions of virtual machine domain

[Han, Weidong]

Mark McLoughlin wrote:
> Hi Joerg,
> 
> On Fri, 2008-12-05 at 00:17 +0100, Joerg Roedel wrote:
> 
>>> Again, these new functions are copies of existing code with minor
>>> modifications. I'd much rather see the existing code refactored and
>>> then modified to handle the DOMAIN_FLAG_VIRTUAL_MACHINE case.
>> 
>> Hey Mark,
>> 
>> can your objections be fixed by follow-up patches bei Han or is
>> anything critical in it? My AMD IOMMU patches for KVM support depend
>> on these patches and everything they are changed I have to rebase by
>> work. So I would prefer if Han can fix the issues found by follow-up
>> patches :) 
> 
> Well, Weidong is going to have to rebase to dwmw2's tree anyway. I
> assume he'll fix at least some of the issues in the process of doing
> that.
> 
> But you're right in that nothing I pointed out was a complete
> showstopper, just ways in which things could be done more cleanly.
> 

Will Avi merge intel-iommu changes into his master tree? If yes, we can push my patches and Joerg's patches together. Otherwise, I need rebase my patches to dwmw2's tree and push it into there first.

Regards,
Weidong

> Cheers,
> Mark.

Re: [PATCH 08/13] allocation and free functions of virtual machine domain

[Avi Kivity]

Han, Weidong wrote:
>
> Will Avi merge intel-iommu changes into his master tree? If yes, we can push my patches and Joerg's patches together. Otherwise, I need rebase my patches to dwmw2's tree and push it into there first.
>   

I think the iommu patches should be merged into Dave's tree first; I'll 
pull from that and apply the kvm patches.

It all needs to be done soonish as 2.6.28 is imminent.

-- 
I have a truly marvellous patch that fixes the bug which this
signature is too narrow to contain.

Re: [PATCH 08/13] allocation and free functions of virtual machine domain

['Joerg Roedel']

On Fri, Dec 05, 2008 at 06:48:09PM +0200, Avi Kivity wrote:
> Han, Weidong wrote:
> >
> >Will Avi merge intel-iommu changes into his master tree? If yes, we can 
> >push my patches and Joerg's patches together. Otherwise, I need rebase my 
> >patches to dwmw2's tree and push it into there first.
> >  
> 
> I think the iommu patches should be merged into Dave's tree first; I'll 
> pull from that and apply the kvm patches.
> 
> It all needs to be done soonish as 2.6.28 is imminent.

Ok, I will send the AMD IOMMU specific parts upstream via Ingo's tree in
a way that does not conflict with Han's patches and the IOMMU-API code
when Linus pulls it.
And as soon as Han sends out his rebased patchset I will rebase the
IOMMU-API too and then everything should be fine.

Joerg

RE: [PATCH 08/13] allocation and free functions of virtual machine domain

[Han, Weidong]

Avi Kivity wrote:
> Han, Weidong wrote:
>> 
>> Will Avi merge intel-iommu changes into his master tree? If yes, we
>> can push my patches and Joerg's patches together. Otherwise, I need
>> rebase my patches to dwmw2's tree and push it into there first.  
>> 
> 
> I think the iommu patches should be merged into Dave's tree first;
> I'll pull from that and apply the kvm patches.
> 

> It all needs to be done soonish as 2.6.28 is imminent.

Okay, I will rebase patchset to Dave's tree. Thanks.

Regards,
Weidong