[PATCH 0/5] Optimize page table walk

[PATCH 0/5] Optimize page table walk

[Avi Kivity]

(resend due to mail server malfunction)

The page table walk has gotten crufty over the years and is threatening to become
even more crufty when SMAP is introduced.  Clean it up (and optimize it) somewhat.

Avi Kivity (5):
  KVM: MMU: Push clean gpte write protection out of gpte_access()
  KVM: MMU: Optimize gpte_access() slightly
  KVM: MMU: Move gpte_access() out of paging_tmpl.h
  KVM: MMU: Optimize pte permission checks
  KVM: MMU: Simplify walk_addr_generic() loop

 arch/x86/include/asm/kvm_host.h |   7 +++
 arch/x86/kvm/mmu.c              |  60 ++++++++++++++++++++++
 arch/x86/kvm/mmu.h              |  16 +-----
 arch/x86/kvm/paging_tmpl.h      | 109 +++++++++++++---------------------------
 arch/x86/kvm/x86.c              |  12 ++---
 5 files changed, 110 insertions(+), 94 deletions(-)

-- 
1.7.11.3

[PATCH 1/5] KVM: MMU: Push clean gpte write protection out of gpte_access()

[Avi Kivity]

gpte_access() computes the access permissions of a guest pte and also
write-protects clean gptes.  This is wrong when we are servicing a
write fault (since we'll be setting the dirty bit momentarily) but
correct when instantiating a speculative spte, or when servicing a
read fault (since we'll want to trap a following write in order to
set the dirty bit).

It doesn't seem to hurt in practice, but in order to make the code
readable, push the write protection out of gpte_access() and into
a new protect_clean_gpte() which is called explicitly when needed.

Signed-off-by: Avi Kivity <avi@redhat.com>
---
 arch/x86/kvm/mmu.c         | 12 ++++++++++++
 arch/x86/kvm/mmu.h         |  3 ++-
 arch/x86/kvm/paging_tmpl.h | 24 ++++++++++++------------
 3 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index aa0b469..54c9cb4 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -3408,6 +3408,18 @@ static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level)
 	return (gpte & mmu->rsvd_bits_mask[bit7][level-1]) != 0;
 }
 
+static inline void protect_clean_gpte(unsigned *access, unsigned gpte)
+{
+	unsigned mask;
+
+	BUILD_BUG_ON(PT_WRITABLE_MASK != ACC_WRITE_MASK);
+
+	mask = (unsigned)~ACC_WRITE_MASK;
+	/* Allow write access to dirty gptes */
+	mask |= (gpte >> (PT_DIRTY_SHIFT - PT_WRITABLE_SHIFT)) & PT_WRITABLE_MASK;
+	*access &= mask;
+}
+
 static bool sync_mmio_spte(u64 *sptep, gfn_t gfn, unsigned access,
 			   int *nr_present)
 {
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index e374db9..2832081 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -18,7 +18,8 @@
 #define PT_PCD_MASK (1ULL << 4)
 #define PT_ACCESSED_SHIFT 5
 #define PT_ACCESSED_MASK (1ULL << PT_ACCESSED_SHIFT)
-#define PT_DIRTY_MASK (1ULL << 6)
+#define PT_DIRTY_SHIFT 6
+#define PT_DIRTY_MASK (1ULL << PT_DIRTY_SHIFT)
 #define PT_PAGE_SIZE_MASK (1ULL << 7)
 #define PT_PAT_MASK (1ULL << 7)
 #define PT_GLOBAL_MASK (1ULL << 8)
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index bf8c42b..bf7b4ff 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -101,14 +101,11 @@ static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 	return (ret != orig_pte);
 }
 
-static unsigned FNAME(gpte_access)(struct kvm_vcpu *vcpu, pt_element_t gpte,
-				   bool last)
+static unsigned FNAME(gpte_access)(struct kvm_vcpu *vcpu, pt_element_t gpte)
 {
 	unsigned access;
 
 	access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK;
-	if (last && !is_dirty_gpte(gpte))
-		access &= ~ACC_WRITE_MASK;
 
 #if PTTYPE == 64
 	if (vcpu->arch.mmu.nx)
@@ -222,8 +219,7 @@ retry_walk:
 
 		last_gpte = FNAME(is_last_gpte)(walker, vcpu, mmu, pte);
 		if (last_gpte) {
-			pte_access = pt_access &
-				     FNAME(gpte_access)(vcpu, pte, true);
+			pte_access = pt_access & FNAME(gpte_access)(vcpu, pte);
 			/* check if the kernel is fetching from user page */
 			if (unlikely(pte_access & PT_USER_MASK) &&
 			    kvm_read_cr4_bits(vcpu, X86_CR4_SMEP))
@@ -274,7 +270,7 @@ retry_walk:
 			break;
 		}
 
-		pt_access &= FNAME(gpte_access)(vcpu, pte, false);
+		pt_access &= FNAME(gpte_access)(vcpu, pte);
 		--walker->level;
 	}
 
@@ -283,7 +279,9 @@ retry_walk:
 		goto error;
 	}
 
-	if (write_fault && unlikely(!is_dirty_gpte(pte))) {
+	if (!write_fault)
+		protect_clean_gpte(&pte_access, pte);
+	else if (unlikely(!is_dirty_gpte(pte))) {
 		int ret;
 
 		trace_kvm_mmu_set_dirty_bit(table_gfn, index, sizeof(pte));
@@ -368,7 +366,8 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
 		return;
 
 	pgprintk("%s: gpte %llx spte %p\n", __func__, (u64)gpte, spte);
-	pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte, true);
+	pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte);
+	protect_clean_gpte(&pte_access, gpte);
 	pfn = gfn_to_pfn_atomic(vcpu->kvm, gpte_to_gfn(gpte));
 	if (mmu_invalid_pfn(pfn))
 		return;
@@ -441,8 +440,8 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw,
 		if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte))
 			continue;
 
-		pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte,
-								  true);
+		pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte);
+		protect_clean_gpte(&pte_access, gpte);
 		gfn = gpte_to_gfn(gpte);
 		pfn = pte_prefetch_gfn_to_pfn(vcpu, gfn,
 				      pte_access & ACC_WRITE_MASK);
@@ -794,7 +793,8 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
 
 		gfn = gpte_to_gfn(gpte);
 		pte_access = sp->role.access;
-		pte_access &= FNAME(gpte_access)(vcpu, gpte, true);
+		pte_access &= FNAME(gpte_access)(vcpu, gpte);
+		protect_clean_gpte(&pte_access, gpte);
 
 		if (sync_mmio_spte(&sp->spt[i], gfn, pte_access, &nr_present))
 			continue;
-- 
1.7.11.3

Re: [PATCH 1/5] KVM: MMU: Push clean gpte write protection out of gpte_access()

[Xiao Guangrong]

On 09/12/2012 10:29 PM, Avi Kivity wrote:
> gpte_access() computes the access permissions of a guest pte and also
> write-protects clean gptes.  This is wrong when we are servicing a
> write fault (since we'll be setting the dirty bit momentarily) but
> correct when instantiating a speculative spte, or when servicing a
> read fault (since we'll want to trap a following write in order to
> set the dirty bit).
> 
> It doesn't seem to hurt in practice, but in order to make the code

In current code, it seems that we will get two #PF if guest write memory
through clean pte: one mark the dirty bit, then fault again, set W bit.

> readable, push the write protection out of gpte_access() and into
> a new protect_clean_gpte() which is called explicitly when needed.

Reviewed-by: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>

[PATCH 2/5] KVM: MMU: Optimize gpte_access() slightly

[Avi Kivity]

If nx is disabled, then is gpte[63] is set we will hit a reserved
bit set fault before checking permissions; so we can ignore the
setting of efer.nxe.

Signed-off-by: Avi Kivity <avi@redhat.com>
---
 arch/x86/kvm/paging_tmpl.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index bf7b4ff..064bcb3 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -106,10 +106,8 @@ static unsigned FNAME(gpte_access)(struct kvm_vcpu *vcpu, pt_element_t gpte)
 	unsigned access;
 
 	access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK;
-
 #if PTTYPE == 64
-	if (vcpu->arch.mmu.nx)
-		access &= ~(gpte >> PT64_NX_SHIFT);
+	access &= ~(gpte >> PT64_NX_SHIFT);
 #endif
 	return access;
 }
-- 
1.7.11.3

Re: [PATCH 2/5] KVM: MMU: Optimize gpte_access() slightly

[Xiao Guangrong]

On 09/12/2012 10:29 PM, Avi Kivity wrote:
> If nx is disabled, then is gpte[63] is set we will hit a reserved
> bit set fault before checking permissions; so we can ignore the
> setting of efer.nxe.

Reviewed-by: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>

[PATCH 3/5] KVM: MMU: Move gpte_access() out of paging_tmpl.h

[Avi Kivity]

We no longer rely on paging_tmpl.h defines; so we can move the function
to mmu.c.

Rely on zero extension to 64 bits to get the correct nx behaviour.

Signed-off-by: Avi Kivity <avi@redhat.com>
---
 arch/x86/kvm/mmu.c         | 10 ++++++++++
 arch/x86/kvm/paging_tmpl.h | 21 +++++----------------
 2 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 54c9cb4..f297a2c 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -3437,6 +3437,16 @@ static bool sync_mmio_spte(u64 *sptep, gfn_t gfn, unsigned access,
 	return false;
 }
 
+static inline unsigned gpte_access(struct kvm_vcpu *vcpu, u64 gpte)
+{
+	unsigned access;
+
+	access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK;
+	access &= ~(gpte >> PT64_NX_SHIFT);
+
+	return access;
+}
+
 #define PTTYPE 64
 #include "paging_tmpl.h"
 #undef PTTYPE
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index 064bcb3..1cbf576 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -101,17 +101,6 @@ static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 	return (ret != orig_pte);
 }
 
-static unsigned FNAME(gpte_access)(struct kvm_vcpu *vcpu, pt_element_t gpte)
-{
-	unsigned access;
-
-	access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK;
-#if PTTYPE == 64
-	access &= ~(gpte >> PT64_NX_SHIFT);
-#endif
-	return access;
-}
-
 static bool FNAME(is_last_gpte)(struct guest_walker *walker,
 				struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 				pt_element_t gpte)
@@ -217,7 +206,7 @@ retry_walk:
 
 		last_gpte = FNAME(is_last_gpte)(walker, vcpu, mmu, pte);
 		if (last_gpte) {
-			pte_access = pt_access & FNAME(gpte_access)(vcpu, pte);
+			pte_access = pt_access & gpte_access(vcpu, pte);
 			/* check if the kernel is fetching from user page */
 			if (unlikely(pte_access & PT_USER_MASK) &&
 			    kvm_read_cr4_bits(vcpu, X86_CR4_SMEP))
@@ -268,7 +257,7 @@ retry_walk:
 			break;
 		}
 
-		pt_access &= FNAME(gpte_access)(vcpu, pte);
+		pt_access &= gpte_access(vcpu, pte);
 		--walker->level;
 	}
 
@@ -364,7 +353,7 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
 		return;
 
 	pgprintk("%s: gpte %llx spte %p\n", __func__, (u64)gpte, spte);
-	pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte);
+	pte_access = sp->role.access & gpte_access(vcpu, gpte);
 	protect_clean_gpte(&pte_access, gpte);
 	pfn = gfn_to_pfn_atomic(vcpu->kvm, gpte_to_gfn(gpte));
 	if (mmu_invalid_pfn(pfn))
@@ -438,7 +427,7 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw,
 		if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte))
 			continue;
 
-		pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte);
+		pte_access = sp->role.access & gpte_access(vcpu, gpte);
 		protect_clean_gpte(&pte_access, gpte);
 		gfn = gpte_to_gfn(gpte);
 		pfn = pte_prefetch_gfn_to_pfn(vcpu, gfn,
@@ -791,7 +780,7 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
 
 		gfn = gpte_to_gfn(gpte);
 		pte_access = sp->role.access;
-		pte_access &= FNAME(gpte_access)(vcpu, gpte);
+		pte_access &= gpte_access(vcpu, gpte);
 		protect_clean_gpte(&pte_access, gpte);
 
 		if (sync_mmio_spte(&sp->spt[i], gfn, pte_access, &nr_present))
-- 
1.7.11.3

Re: [PATCH 3/5] KVM: MMU: Move gpte_access() out of paging_tmpl.h

[Xiao Guangrong]

On 09/12/2012 10:29 PM, Avi Kivity wrote:

>  static bool FNAME(is_last_gpte)(struct guest_walker *walker,
>  				struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
>  				pt_element_t gpte)
> @@ -217,7 +206,7 @@ retry_walk:
> 
>  		last_gpte = FNAME(is_last_gpte)(walker, vcpu, mmu, pte);
>  		if (last_gpte) {
> -			pte_access = pt_access & FNAME(gpte_access)(vcpu, pte);
> +			pte_access = pt_access & gpte_access(vcpu, pte);

It can pass 32bit variable to gpte_access without cast, no warning?

Re: [PATCH 3/5] KVM: MMU: Move gpte_access() out of paging_tmpl.h

[Avi Kivity]

On 09/13/2012 02:48 PM, Xiao Guangrong wrote:
> On 09/12/2012 10:29 PM, Avi Kivity wrote:
> 
>>  static bool FNAME(is_last_gpte)(struct guest_walker *walker,
>>  				struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
>>  				pt_element_t gpte)
>> @@ -217,7 +206,7 @@ retry_walk:
>> 
>>  		last_gpte = FNAME(is_last_gpte)(walker, vcpu, mmu, pte);
>>  		if (last_gpte) {
>> -			pte_access = pt_access & FNAME(gpte_access)(vcpu, pte);
>> +			pte_access = pt_access & gpte_access(vcpu, pte);
> 
> It can pass 32bit variable to gpte_access without cast, no warning?

No warning.


-- 
error compiling committee.c: too many arguments to function

[PATCH 4/5] KVM: MMU: Optimize pte permission checks

[Avi Kivity]

walk_addr_generic() permission checks are a maze of branchy code, which is
performed four times per lookup.  It depends on the type of access, efer.nxe,
cr0.wp, cr4.smep, and in the near future, cr4.smap.

Optimize this away by precalculating all variants and storing them in a
bitmap.  The bitmap is recalculated when rarely-changing variables change
(cr0, cr4) and is indexed by the often-changing variables (page fault error
code, pte access permissions).

The result is short, branch-free code.

Signed-off-by: Avi Kivity <avi@redhat.com>
---
 arch/x86/include/asm/kvm_host.h |  7 +++++++
 arch/x86/kvm/mmu.c              | 38 ++++++++++++++++++++++++++++++++++++++
 arch/x86/kvm/mmu.h              | 13 -------------
 arch/x86/kvm/paging_tmpl.h      | 22 ++++------------------
 arch/x86/kvm/x86.c              | 12 +++++-------
 5 files changed, 54 insertions(+), 38 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 64adb61..3318bde 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -287,6 +287,13 @@ struct kvm_mmu {
 	union kvm_mmu_page_role base_role;
 	bool direct_map;
 
+	/*
+	 * Bitmap; bit set = permission fault
+	 * Byte index: page fault error code [4:1]
+	 * Bit index: pte permissions in ACC_* format
+	 */
+	u8 permissions[16];
+
 	u64 *pae_root;
 	u64 *lm_root;
 	u64 rsvd_bits_mask[2][4];
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index f297a2c..ce78408 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -3516,6 +3516,38 @@ static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
 	}
 }
 
+static void update_permission_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
+{
+	unsigned bit, byte, pfec;
+	u8 map;
+	bool fault, x, w, u, wf, uf, ff, smep;
+
+	smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
+	for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
+		pfec = byte << 1;
+		map = 0;
+		wf = pfec & PFERR_WRITE_MASK;
+		uf = pfec & PFERR_USER_MASK;
+		ff = pfec & PFERR_FETCH_MASK;
+		for (bit = 0; bit < 8; ++bit) {
+			x = bit & ACC_EXEC_MASK;
+			w = bit & ACC_WRITE_MASK;
+			u = bit & ACC_USER_MASK;
+
+			/* Not really needed: !nx will cause pte.nx to fault */
+			x |= !mmu->nx;
+			/* Allow supervisor writes if !cr0.wp */
+			w |= !is_write_protection(vcpu) && !uf;
+			/* Disallow supervisor fetches if cr4.smep */
+			x &= !(smep && !uf);
+
+			fault = (ff && !x) || (uf && !u) || (wf && !w);
+			map |= fault << bit;
+		}
+		mmu->permissions[byte] = map;
+	}
+}
+
 static int paging64_init_context_common(struct kvm_vcpu *vcpu,
 					struct kvm_mmu *context,
 					int level)
@@ -3524,6 +3556,7 @@ static int paging64_init_context_common(struct kvm_vcpu *vcpu,
 	context->root_level = level;
 
 	reset_rsvds_bits_mask(vcpu, context);
+	update_permission_bitmask(vcpu, context);
 
 	ASSERT(is_pae(vcpu));
 	context->new_cr3 = paging_new_cr3;
@@ -3552,6 +3585,7 @@ static int paging32_init_context(struct kvm_vcpu *vcpu,
 	context->root_level = PT32_ROOT_LEVEL;
 
 	reset_rsvds_bits_mask(vcpu, context);
+	update_permission_bitmask(vcpu, context);
 
 	context->new_cr3 = paging_new_cr3;
 	context->page_fault = paging32_page_fault;
@@ -3612,6 +3646,8 @@ static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
 		context->gva_to_gpa = paging32_gva_to_gpa;
 	}
 
+	update_permission_bitmask(vcpu, context);
+
 	return 0;
 }
 
@@ -3687,6 +3723,8 @@ static int init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
 		g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
 	}
 
+	update_permission_bitmask(vcpu, g_context);
+
 	return 0;
 }
 
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index 2832081..143ee70 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -89,17 +89,4 @@ static inline bool is_write_protection(struct kvm_vcpu *vcpu)
 	return kvm_read_cr0_bits(vcpu, X86_CR0_WP);
 }
 
-static inline bool check_write_user_access(struct kvm_vcpu *vcpu,
-					   bool write_fault, bool user_fault,
-					   unsigned long pte)
-{
-	if (unlikely(write_fault && !is_writable_pte(pte)
-	      && (user_fault || is_write_protection(vcpu))))
-		return false;
-
-	if (unlikely(user_fault && !(pte & PT_USER_MASK)))
-		return false;
-
-	return true;
-}
 #endif
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index 1cbf576..59c4319 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -129,7 +129,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 	pt_element_t pte;
 	pt_element_t __user *uninitialized_var(ptep_user);
 	gfn_t table_gfn;
-	unsigned index, pt_access, uninitialized_var(pte_access);
+	unsigned index, pt_access, pte_access;
 	gpa_t pte_gpa;
 	bool eperm, last_gpte;
 	int offset;
@@ -195,24 +195,10 @@ retry_walk:
 			goto error;
 		}
 
-		if (!check_write_user_access(vcpu, write_fault, user_fault,
-					  pte))
-			eperm = true;
-
-#if PTTYPE == 64
-		if (unlikely(fetch_fault && (pte & PT64_NX_MASK)))
-			eperm = true;
-#endif
+		pte_access = pt_access & gpte_access(vcpu, pte);
+		eperm |= (mmu->permissions[access >> 1] >> pte_access) & 1;
 
 		last_gpte = FNAME(is_last_gpte)(walker, vcpu, mmu, pte);
-		if (last_gpte) {
-			pte_access = pt_access & gpte_access(vcpu, pte);
-			/* check if the kernel is fetching from user page */
-			if (unlikely(pte_access & PT_USER_MASK) &&
-			    kvm_read_cr4_bits(vcpu, X86_CR4_SMEP))
-				if (fetch_fault && !user_fault)
-					eperm = true;
-		}
 
 		if (!eperm && unlikely(!(pte & PT_ACCESSED_MASK))) {
 			int ret;
@@ -257,7 +243,7 @@ retry_walk:
 			break;
 		}
 
-		pt_access &= gpte_access(vcpu, pte);
+		pt_access &= pte_access;
 		--walker->level;
 	}
 
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index c4d451e..e59b9fe 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -3672,20 +3672,18 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
 				gpa_t *gpa, struct x86_exception *exception,
 				bool write)
 {
-	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
+	u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
+		| (write ? PFERR_WRITE_MASK : 0);
+	u8 bit = vcpu->arch.access;
 
-	if (vcpu_match_mmio_gva(vcpu, gva) &&
-		  check_write_user_access(vcpu, write, access,
-		  vcpu->arch.access)) {
+	if (vcpu_match_mmio_gva(vcpu, gva)
+	    && ((vcpu->arch.walk_mmu->permissions[access >> 1] >> bit) & 1)) {
 		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
 					(gva & (PAGE_SIZE - 1));
 		trace_vcpu_match_mmio(gva, *gpa, write, false);
 		return 1;
 	}
 
-	if (write)
-		access |= PFERR_WRITE_MASK;
-
 	*gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
 
 	if (*gpa == UNMAPPED_GVA)
-- 
1.7.11.3

Re: [PATCH 4/5] KVM: MMU: Optimize pte permission checks

[Xiao Guangrong]

On 09/12/2012 10:29 PM, Avi Kivity wrote:
> walk_addr_generic() permission checks are a maze of branchy code, which is
> performed four times per lookup.  It depends on the type of access, efer.nxe,
> cr0.wp, cr4.smep, and in the near future, cr4.smap.
> 
> Optimize this away by precalculating all variants and storing them in a
> bitmap.  The bitmap is recalculated when rarely-changing variables change
> (cr0, cr4) and is indexed by the often-changing variables (page fault error
> code, pte access permissions).

Really graceful!

> 
> The result is short, branch-free code.
> 
> Signed-off-by: Avi Kivity <avi@redhat.com>

> +static void update_permission_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
> +{
> +	unsigned bit, byte, pfec;
> +	u8 map;
> +	bool fault, x, w, u, wf, uf, ff, smep;
> +
> +	smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
> +	for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
> +		pfec = byte << 1;
> +		map = 0;
> +		wf = pfec & PFERR_WRITE_MASK;
> +		uf = pfec & PFERR_USER_MASK;
> +		ff = pfec & PFERR_FETCH_MASK;
> +		for (bit = 0; bit < 8; ++bit) {
> +			x = bit & ACC_EXEC_MASK;
> +			w = bit & ACC_WRITE_MASK;
> +			u = bit & ACC_USER_MASK;
> +
> +			/* Not really needed: !nx will cause pte.nx to fault */
> +			x |= !mmu->nx;
> +			/* Allow supervisor writes if !cr0.wp */
> +			w |= !is_write_protection(vcpu) && !uf;
> +			/* Disallow supervisor fetches if cr4.smep */
> +			x &= !(smep && !uf);

In the case of smep, supervisor mode can fetch the memory if pte.u == 0,
so, it should be x &= !(smep && !uf && u)?

> @@ -3672,20 +3672,18 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
>  				gpa_t *gpa, struct x86_exception *exception,
>  				bool write)
>  {
> -	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
> +	u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
> +		| (write ? PFERR_WRITE_MASK : 0);
> +	u8 bit = vcpu->arch.access;
> 
> -	if (vcpu_match_mmio_gva(vcpu, gva) &&
> -		  check_write_user_access(vcpu, write, access,
> -		  vcpu->arch.access)) {
> +	if (vcpu_match_mmio_gva(vcpu, gva)
> +	    && ((vcpu->arch.walk_mmu->permissions[access >> 1] >> bit) & 1)) {

!((vcpu->arch.walk_mmu->permissions[access >> 1] >> bit) & 1) ?

It is better introducing a function to do the permission check?

Re: [PATCH 4/5] KVM: MMU: Optimize pte permission checks

[Avi Kivity]

On 09/13/2012 03:09 PM, Xiao Guangrong wrote:
>> 
>> The result is short, branch-free code.
>> 
>> Signed-off-by: Avi Kivity <avi@redhat.com>
> 
>> +static void update_permission_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
>> +{
>> +	unsigned bit, byte, pfec;
>> +	u8 map;
>> +	bool fault, x, w, u, wf, uf, ff, smep;
>> +
>> +	smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
>> +	for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
>> +		pfec = byte << 1;
>> +		map = 0;
>> +		wf = pfec & PFERR_WRITE_MASK;
>> +		uf = pfec & PFERR_USER_MASK;
>> +		ff = pfec & PFERR_FETCH_MASK;
>> +		for (bit = 0; bit < 8; ++bit) {
>> +			x = bit & ACC_EXEC_MASK;
>> +			w = bit & ACC_WRITE_MASK;
>> +			u = bit & ACC_USER_MASK;
>> +
>> +			/* Not really needed: !nx will cause pte.nx to fault */
>> +			x |= !mmu->nx;
>> +			/* Allow supervisor writes if !cr0.wp */
>> +			w |= !is_write_protection(vcpu) && !uf;
>> +			/* Disallow supervisor fetches if cr4.smep */
>> +			x &= !(smep && !uf);
> 
> In the case of smep, supervisor mode can fetch the memory if pte.u == 0,
> so, it should be x &= !(smep && !uf && u)?

Good catch, will fix.

> 
>> @@ -3672,20 +3672,18 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
>>  				gpa_t *gpa, struct x86_exception *exception,
>>  				bool write)
>>  {
>> -	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
>> +	u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
>> +		| (write ? PFERR_WRITE_MASK : 0);
>> +	u8 bit = vcpu->arch.access;
>> 
>> -	if (vcpu_match_mmio_gva(vcpu, gva) &&
>> -		  check_write_user_access(vcpu, write, access,
>> -		  vcpu->arch.access)) {
>> +	if (vcpu_match_mmio_gva(vcpu, gva)
>> +	    && ((vcpu->arch.walk_mmu->permissions[access >> 1] >> bit) & 1)) {
> 
> !((vcpu->arch.walk_mmu->permissions[access >> 1] >> bit) & 1) ?
> 
> It is better introducing a function to do the permission check?
> 

Probably, I'll rethink it.

-- 
error compiling committee.c: too many arguments to function

Re: [PATCH 4/5] KVM: MMU: Optimize pte permission checks

[Xiao Guangrong]

On 09/12/2012 10:29 PM, Avi Kivity wrote:

> +		pte_access = pt_access & gpte_access(vcpu, pte);
> +		eperm |= (mmu->permissions[access >> 1] >> pte_access) & 1;
> 
>  		last_gpte = FNAME(is_last_gpte)(walker, vcpu, mmu, pte);
> -		if (last_gpte) {
> -			pte_access = pt_access & gpte_access(vcpu, pte);
> -			/* check if the kernel is fetching from user page */
> -			if (unlikely(pte_access & PT_USER_MASK) &&
> -			    kvm_read_cr4_bits(vcpu, X86_CR4_SMEP))
> -				if (fetch_fault && !user_fault)
> -					eperm = true;
> -		}

I see this in the SDM:

If CR4.SMEP = 1, instructions may be fetched from any linear
address with a valid translation for which the U/S flag (bit 2) is 0 in at
least one of the paging-structure entries controlling the translation.

This patch checks smep on every levels, breaks this rule.
(current code checks smep on the last level).

Re: [PATCH 4/5] KVM: MMU: Optimize pte permission checks

[Avi Kivity]

On 09/13/2012 03:41 PM, Xiao Guangrong wrote:
> On 09/12/2012 10:29 PM, Avi Kivity wrote:
> 
>> +		pte_access = pt_access & gpte_access(vcpu, pte);
>> +		eperm |= (mmu->permissions[access >> 1] >> pte_access) & 1;
>> 
>>  		last_gpte = FNAME(is_last_gpte)(walker, vcpu, mmu, pte);
>> -		if (last_gpte) {
>> -			pte_access = pt_access & gpte_access(vcpu, pte);
>> -			/* check if the kernel is fetching from user page */
>> -			if (unlikely(pte_access & PT_USER_MASK) &&
>> -			    kvm_read_cr4_bits(vcpu, X86_CR4_SMEP))
>> -				if (fetch_fault && !user_fault)
>> -					eperm = true;
>> -		}
> 
> I see this in the SDM:
> 
> If CR4.SMEP = 1, instructions may be fetched from any linear
> address with a valid translation for which the U/S flag (bit 2) is 0 in at
> least one of the paging-structure entries controlling the translation.

Another good catch.

> 
> This patch checks smep on every levels, breaks this rule.
> (current code checks smep on the last level).
> 

We can just move the permission check to the end of the loop.  We used
to terminate the loop on a permission error, but now we do the whole
thing anyway.

It does mean that we'll need to set accessed bits after the loop is
complete.

-- 
error compiling committee.c: too many arguments to function

[PATCH 5/5] KVM: MMU: Simplify walk_addr_generic() loop

[Avi Kivity]

The page table walk is coded as an infinite loop, with a special
case on the last pte.

Code it as an ordinary loop with a termination condition on the last
pte (large page or walk length exhausted), and put the last pte handling
code after the loop where it belongs.

Signed-off-by: Avi Kivity <avi@redhat.com>
---
 arch/x86/kvm/paging_tmpl.h | 58 +++++++++++++++++++---------------------------
 1 file changed, 24 insertions(+), 34 deletions(-)

diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index 59c4319..eb4a668 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -131,12 +131,15 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 	gfn_t table_gfn;
 	unsigned index, pt_access, pte_access;
 	gpa_t pte_gpa;
-	bool eperm, last_gpte;
+	bool eperm;
 	int offset;
 	const int write_fault = access & PFERR_WRITE_MASK;
 	const int user_fault  = access & PFERR_USER_MASK;
 	const int fetch_fault = access & PFERR_FETCH_MASK;
 	u16 errcode = 0;
+	gpa_t real_gpa;
+	gfn_t gfn;
+	u32 ac;
 
 	trace_kvm_mmu_pagetable_walk(addr, access);
 retry_walk:
@@ -156,12 +159,16 @@ retry_walk:
 	ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) ||
 	       (mmu->get_cr3(vcpu) & CR3_NONPAE_RESERVED_BITS) == 0);
 
-	pt_access = ACC_ALL;
+	pt_access = pte_access = ACC_ALL;
+	++walker->level;
 
-	for (;;) {
+	do {
 		gfn_t real_gfn;
 		unsigned long host_addr;
 
+		pt_access &= pte_access;
+		--walker->level;
+
 		index = PT_INDEX(addr, walker->level);
 
 		table_gfn = gpte_to_gfn(pte);
@@ -198,8 +205,6 @@ retry_walk:
 		pte_access = pt_access & gpte_access(vcpu, pte);
 		eperm |= (mmu->permissions[access >> 1] >> pte_access) & 1;
 
-		last_gpte = FNAME(is_last_gpte)(walker, vcpu, mmu, pte);
-
 		if (!eperm && unlikely(!(pte & PT_ACCESSED_MASK))) {
 			int ret;
 			trace_kvm_mmu_set_accessed_bit(table_gfn, index,
@@ -216,41 +221,26 @@ retry_walk:
 		}
 
 		walker->ptes[walker->level - 1] = pte;
+	} while (!FNAME(is_last_gpte)(walker, vcpu, mmu, pte));
 
-		if (last_gpte) {
-			int lvl = walker->level;
-			gpa_t real_gpa;
-			gfn_t gfn;
-			u32 ac;
-
-			gfn = gpte_to_gfn_lvl(pte, lvl);
-			gfn += (addr & PT_LVL_OFFSET_MASK(lvl)) >> PAGE_SHIFT;
-
-			if (PTTYPE == 32 &&
-			    walker->level == PT_DIRECTORY_LEVEL &&
-			    is_cpuid_PSE36())
-				gfn += pse36_gfn_delta(pte);
-
-			ac = write_fault | fetch_fault | user_fault;
+	if (unlikely(eperm)) {
+		errcode |= PFERR_PRESENT_MASK;
+		goto error;
+	}
 
-			real_gpa = mmu->translate_gpa(vcpu, gfn_to_gpa(gfn),
-						      ac);
-			if (real_gpa == UNMAPPED_GVA)
-				return 0;
+	gfn = gpte_to_gfn_lvl(pte, walker->level);
+	gfn += (addr & PT_LVL_OFFSET_MASK(walker->level)) >> PAGE_SHIFT;
 
-			walker->gfn = real_gpa >> PAGE_SHIFT;
+	if (PTTYPE == 32 && walker->level == PT_DIRECTORY_LEVEL && is_cpuid_PSE36())
+		gfn += pse36_gfn_delta(pte);
 
-			break;
-		}
+	ac = write_fault | fetch_fault | user_fault;
 
-		pt_access &= pte_access;
-		--walker->level;
-	}
+	real_gpa = mmu->translate_gpa(vcpu, gfn_to_gpa(gfn), ac);
+	if (real_gpa == UNMAPPED_GVA)
+		return 0;
 
-	if (unlikely(eperm)) {
-		errcode |= PFERR_PRESENT_MASK;
-		goto error;
-	}
+	walker->gfn = real_gpa >> PAGE_SHIFT;
 
 	if (!write_fault)
 		protect_clean_gpte(&pte_access, pte);
-- 
1.7.11.3

[PATCH 6/5] KVM: MMU: Optimize is_last_gpte()

[Avi Kivity]

Instead of branchy code depending on level, gpte.ps, and mmu configuration,
prepare everything in a bitmap during mode changes and look it up during
runtime.

Signed-off-by: Avi Kivity <avi@redhat.com>
---
 arch/x86/include/asm/kvm_host.h |  7 +++++++
 arch/x86/kvm/mmu.c              | 20 ++++++++++++++++++++
 arch/x86/kvm/mmu.h              |  3 ++-
 arch/x86/kvm/paging_tmpl.h      | 25 +++++--------------------
 4 files changed, 34 insertions(+), 21 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 3318bde..78525f6 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -298,6 +298,13 @@ struct kvm_mmu {
 	u64 *lm_root;
 	u64 rsvd_bits_mask[2][4];
 
+	/*
+	 * Bitmap: bit set = last pte in walk
+	 * index[0]: pte.ps
+	 * index[1:2]: level
+	 */
+	u8 last_pte;
+
 	bool nx;
 
 	u64 pdptrs[4]; /* pae */
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index ce78408..30a574f 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -3548,6 +3548,22 @@ static void update_permission_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu
 	}
 }
 
+static void update_last_pte_bitmap(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
+{
+	u8 map = 0;
+	unsigned level, root_level = mmu->root_level;
+
+	if (root_level == PT32E_ROOT_LEVEL)
+		--root_level;
+	map |= 3;  /* PT_PAGE_TABLE_LEVEL always terminates */
+	for (level = PT_DIRECTORY_LEVEL; level <= root_level; ++level) {
+		if (level <= PT_PDPE_LEVEL
+		    && (mmu->root_level >= PT32E_ROOT_LEVEL || is_pse(vcpu)))
+			map |= 2 << (2 * level - 1);
+	}
+	mmu->last_pte = map;
+}
+
 static int paging64_init_context_common(struct kvm_vcpu *vcpu,
 					struct kvm_mmu *context,
 					int level)
@@ -3557,6 +3573,7 @@ static int paging64_init_context_common(struct kvm_vcpu *vcpu,
 
 	reset_rsvds_bits_mask(vcpu, context);
 	update_permission_bitmask(vcpu, context);
+	update_last_pte_bitmap(vcpu, context);
 
 	ASSERT(is_pae(vcpu));
 	context->new_cr3 = paging_new_cr3;
@@ -3586,6 +3603,7 @@ static int paging32_init_context(struct kvm_vcpu *vcpu,
 
 	reset_rsvds_bits_mask(vcpu, context);
 	update_permission_bitmask(vcpu, context);
+	update_last_pte_bitmap(vcpu, context);
 
 	context->new_cr3 = paging_new_cr3;
 	context->page_fault = paging32_page_fault;
@@ -3647,6 +3665,7 @@ static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
 	}
 
 	update_permission_bitmask(vcpu, context);
+	update_last_pte_bitmap(vcpu, context);
 
 	return 0;
 }
@@ -3724,6 +3743,7 @@ static int init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
 	}
 
 	update_permission_bitmask(vcpu, g_context);
+	update_last_pte_bitmap(vcpu, g_context);
 
 	return 0;
 }
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index 143ee70..b08dd34 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -20,7 +20,8 @@
 #define PT_ACCESSED_MASK (1ULL << PT_ACCESSED_SHIFT)
 #define PT_DIRTY_SHIFT 6
 #define PT_DIRTY_MASK (1ULL << PT_DIRTY_SHIFT)
-#define PT_PAGE_SIZE_MASK (1ULL << 7)
+#define PT_PAGE_SIZE_SHIFT 7
+#define PT_PAGE_SIZE_MASK (1ULL << PT_PAGE_SIZE_SHIFT)
 #define PT_PAT_MASK (1ULL << 7)
 #define PT_GLOBAL_MASK (1ULL << 8)
 #define PT64_NX_SHIFT 63
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index eb4a668..dd89404 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -101,24 +101,6 @@ static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 	return (ret != orig_pte);
 }
 
-static bool FNAME(is_last_gpte)(struct guest_walker *walker,
-				struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
-				pt_element_t gpte)
-{
-	if (walker->level == PT_PAGE_TABLE_LEVEL)
-		return true;
-
-	if ((walker->level == PT_DIRECTORY_LEVEL) && is_large_pte(gpte) &&
-	    (PTTYPE == 64 || is_pse(vcpu)))
-		return true;
-
-	if ((walker->level == PT_PDPE_LEVEL) && is_large_pte(gpte) &&
-	    (mmu->root_level == PT64_ROOT_LEVEL))
-		return true;
-
-	return false;
-}
-
 /*
  * Fetch a guest pte for a guest virtual address
  */
@@ -140,6 +122,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 	gpa_t real_gpa;
 	gfn_t gfn;
 	u32 ac;
+	bool last;
 
 	trace_kvm_mmu_pagetable_walk(addr, access);
 retry_walk:
@@ -220,8 +203,10 @@ retry_walk:
 			pte |= PT_ACCESSED_MASK;
 		}
 
-		walker->ptes[walker->level - 1] = pte;
-	} while (!FNAME(is_last_gpte)(walker, vcpu, mmu, pte));
+		index = (walker->level - 1) << 1;
+		index |= (pte >> PT_PAGE_SIZE_SHIFT) & 1;
+		last = mmu->last_pte & (1 << index);
+	} while (!last);
 
 	if (unlikely(eperm)) {
 		errcode |= PFERR_PRESENT_MASK;
-- 
1.7.11.3

Re: [PATCH 6/5] KVM: MMU: Optimize is_last_gpte()

[Avi Kivity]

On 09/12/2012 08:49 PM, Avi Kivity wrote:
> Instead of branchy code depending on level, gpte.ps, and mmu configuration,
> prepare everything in a bitmap during mode changes and look it up during
> runtime.

6/5 is buggy, sorry, will update it tomorrow.


-- 
error compiling committee.c: too many arguments to function

Re: [PATCH 6/5] KVM: MMU: Optimize is_last_gpte()

[Avi Kivity]

On 09/12/2012 09:03 PM, Avi Kivity wrote:
> On 09/12/2012 08:49 PM, Avi Kivity wrote:
>> Instead of branchy code depending on level, gpte.ps, and mmu configuration,
>> prepare everything in a bitmap during mode changes and look it up during
>> runtime.
> 
> 6/5 is buggy, sorry, will update it tomorrow.
> 
> 

--------8<----------8<--------------

From: Avi Kivity <avi@redhat.com>
Date: Wed, 12 Sep 2012 20:46:56 +0300
Subject: [PATCH v2 6/5] KVM: MMU: Optimize is_last_gpte()

Instead of branchy code depending on level, gpte.ps, and mmu configuration,
prepare everything in a bitmap during mode changes and look it up during
runtime.

Signed-off-by: Avi Kivity <avi@redhat.com>
---

v2: rearrange bitmap (one less shift)
    avoid stomping on local variable
    fix index calculation
    move check back to a function

 arch/x86/include/asm/kvm_host.h |  7 +++++++
 arch/x86/kvm/mmu.c              | 31 +++++++++++++++++++++++++++++++
 arch/x86/kvm/mmu.h              |  3 ++-
 arch/x86/kvm/paging_tmpl.h      | 22 +---------------------
 4 files changed, 41 insertions(+), 22 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 3318bde..f9a48cf 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -298,6 +298,13 @@ struct kvm_mmu {
 	u64 *lm_root;
 	u64 rsvd_bits_mask[2][4];
 
+	/*
+	 * Bitmap: bit set = last pte in walk
+	 * index[0]: pte.ps
+	 * index[1:2]: level
+	 */
+	u8 last_pte_bitmap;
+
 	bool nx;
 
 	u64 pdptrs[4]; /* pae */
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index ce78408..32fe597 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -3447,6 +3447,15 @@ static inline unsigned gpte_access(struct kvm_vcpu *vcpu, u64 gpte)
 	return access;
 }
 
+static inline bool is_last_gpte(struct kvm_mmu *mmu, unsigned level, unsigned gpte)
+{
+	unsigned index;
+
+	index = level - 1;
+	index |= (gpte & PT_PAGE_SIZE_MASK) >> (PT_PAGE_SIZE_SHIFT - 2);
+	return mmu->last_pte_bitmap & (1 << index);
+}
+
 #define PTTYPE 64
 #include "paging_tmpl.h"
 #undef PTTYPE
@@ -3548,6 +3557,24 @@ static void update_permission_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu
 	}
 }
 
+static void update_last_pte_bitmap(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
+{
+	u8 map;
+	unsigned level, root_level = mmu->root_level;
+	const unsigned ps_set_index = 1 << 2;  /* bit 2 of index: ps */
+
+	if (root_level == PT32E_ROOT_LEVEL)
+		--root_level;
+	/* PT_PAGE_TABLE_LEVEL always terminates */
+	map = 1 | (1 << ps_set_index);
+	for (level = PT_DIRECTORY_LEVEL; level <= root_level; ++level) {
+		if (level <= PT_PDPE_LEVEL
+		    && (mmu->root_level >= PT32E_ROOT_LEVEL || is_pse(vcpu)))
+			map |= 1 << (ps_set_index | (level - 1));
+	}
+	mmu->last_pte_bitmap = map;
+}
+
 static int paging64_init_context_common(struct kvm_vcpu *vcpu,
 					struct kvm_mmu *context,
 					int level)
@@ -3557,6 +3584,7 @@ static int paging64_init_context_common(struct kvm_vcpu *vcpu,
 
 	reset_rsvds_bits_mask(vcpu, context);
 	update_permission_bitmask(vcpu, context);
+	update_last_pte_bitmap(vcpu, context);
 
 	ASSERT(is_pae(vcpu));
 	context->new_cr3 = paging_new_cr3;
@@ -3586,6 +3614,7 @@ static int paging32_init_context(struct kvm_vcpu *vcpu,
 
 	reset_rsvds_bits_mask(vcpu, context);
 	update_permission_bitmask(vcpu, context);
+	update_last_pte_bitmap(vcpu, context);
 
 	context->new_cr3 = paging_new_cr3;
 	context->page_fault = paging32_page_fault;
@@ -3647,6 +3676,7 @@ static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
 	}
 
 	update_permission_bitmask(vcpu, context);
+	update_last_pte_bitmap(vcpu, context);
 
 	return 0;
 }
@@ -3724,6 +3754,7 @@ static int init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
 	}
 
 	update_permission_bitmask(vcpu, g_context);
+	update_last_pte_bitmap(vcpu, g_context);
 
 	return 0;
 }
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index 143ee70..b08dd34 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -20,7 +20,8 @@
 #define PT_ACCESSED_MASK (1ULL << PT_ACCESSED_SHIFT)
 #define PT_DIRTY_SHIFT 6
 #define PT_DIRTY_MASK (1ULL << PT_DIRTY_SHIFT)
-#define PT_PAGE_SIZE_MASK (1ULL << 7)
+#define PT_PAGE_SIZE_SHIFT 7
+#define PT_PAGE_SIZE_MASK (1ULL << PT_PAGE_SIZE_SHIFT)
 #define PT_PAT_MASK (1ULL << 7)
 #define PT_GLOBAL_MASK (1ULL << 8)
 #define PT64_NX_SHIFT 63
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index eb4a668..ec1e101 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -101,24 +101,6 @@ static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 	return (ret != orig_pte);
 }
 
-static bool FNAME(is_last_gpte)(struct guest_walker *walker,
-				struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
-				pt_element_t gpte)
-{
-	if (walker->level == PT_PAGE_TABLE_LEVEL)
-		return true;
-
-	if ((walker->level == PT_DIRECTORY_LEVEL) && is_large_pte(gpte) &&
-	    (PTTYPE == 64 || is_pse(vcpu)))
-		return true;
-
-	if ((walker->level == PT_PDPE_LEVEL) && is_large_pte(gpte) &&
-	    (mmu->root_level == PT64_ROOT_LEVEL))
-		return true;
-
-	return false;
-}
-
 /*
  * Fetch a guest pte for a guest virtual address
  */
@@ -219,9 +201,7 @@ retry_walk:
 			mark_page_dirty(vcpu->kvm, table_gfn);
 			pte |= PT_ACCESSED_MASK;
 		}
 
 		walker->ptes[walker->level - 1] = pte;
-	} while (!FNAME(is_last_gpte)(walker, vcpu, mmu, pte));
+	} while (is_last_gpte(mmu, walker->level, pte));
 
 	if (unlikely(eperm)) {
 		errcode |= PFERR_PRESENT_MASK;
-- 


-- 
error compiling committee.c: too many arguments to function

Re: [PATCH 6/5] KVM: MMU: Optimize is_last_gpte()

[Xiao Guangrong]

On 09/13/2012 05:47 PM, Avi Kivity wrote:
> On 09/12/2012 09:03 PM, Avi Kivity wrote:
>> On 09/12/2012 08:49 PM, Avi Kivity wrote:
>>> Instead of branchy code depending on level, gpte.ps, and mmu configuration,
>>> prepare everything in a bitmap during mode changes and look it up during
>>> runtime.
>>
>> 6/5 is buggy, sorry, will update it tomorrow.
>>
>>
> 
> --------8<----------8<--------------
> 
> From: Avi Kivity <avi@redhat.com>
> Date: Wed, 12 Sep 2012 20:46:56 +0300
> Subject: [PATCH v2 6/5] KVM: MMU: Optimize is_last_gpte()
> 
> Instead of branchy code depending on level, gpte.ps, and mmu configuration,
> prepare everything in a bitmap during mode changes and look it up during
> runtime.
> 
> Signed-off-by: Avi Kivity <avi@redhat.com>
> ---
> 
> v2: rearrange bitmap (one less shift)
>     avoid stomping on local variable
>     fix index calculation
>     move check back to a function
> 
>  arch/x86/include/asm/kvm_host.h |  7 +++++++
>  arch/x86/kvm/mmu.c              | 31 +++++++++++++++++++++++++++++++
>  arch/x86/kvm/mmu.h              |  3 ++-
>  arch/x86/kvm/paging_tmpl.h      | 22 +---------------------
>  4 files changed, 41 insertions(+), 22 deletions(-)
> 
> diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
> index 3318bde..f9a48cf 100644
> --- a/arch/x86/include/asm/kvm_host.h
> +++ b/arch/x86/include/asm/kvm_host.h
> @@ -298,6 +298,13 @@ struct kvm_mmu {
>  	u64 *lm_root;
>  	u64 rsvd_bits_mask[2][4];
> 
> +	/*
> +	 * Bitmap: bit set = last pte in walk
> +	 * index[0]: pte.ps
> +	 * index[1:2]: level
> +	 */

Opposite? index[2]: pte.pse?

Reviewed-by: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>

Re: [PATCH 6/5] KVM: MMU: Optimize is_last_gpte()

[Avi Kivity]

On 09/13/2012 04:39 PM, Xiao Guangrong wrote:

>> diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
>> index 3318bde..f9a48cf 100644
>> --- a/arch/x86/include/asm/kvm_host.h
>> +++ b/arch/x86/include/asm/kvm_host.h
>> @@ -298,6 +298,13 @@ struct kvm_mmu {
>>  	u64 *lm_root;
>>  	u64 rsvd_bits_mask[2][4];
>> 
>> +	/*
>> +	 * Bitmap: bit set = last pte in walk
>> +	 * index[0]: pte.ps
>> +	 * index[1:2]: level
>> +	 */
> 
> Opposite? index[2]: pte.pse?

Yes.  Fixed.


-- 
error compiling committee.c: too many arguments to function

Re: [PATCH 0/5] Optimize page table walk

[Marcelo Tosatti]

On Wed, Sep 12, 2012 at 05:29:49PM +0300, Avi Kivity wrote:
> (resend due to mail server malfunction)
> 
> The page table walk has gotten crufty over the years and is threatening to become
> even more crufty when SMAP is introduced.  Clean it up (and optimize it) somewhat.

What is SMAP?

Re: [PATCH 0/5] Optimize page table walk

[Avi Kivity]

On 09/13/2012 01:20 AM, Marcelo Tosatti wrote:
> On Wed, Sep 12, 2012 at 05:29:49PM +0300, Avi Kivity wrote:
>> (resend due to mail server malfunction)
>> 
>> The page table walk has gotten crufty over the years and is threatening to become
>> even more crufty when SMAP is introduced.  Clean it up (and optimize it) somewhat.
> 
> What is SMAP?
> 

Supervisor Mode Access Prevention, see
http://software.intel.com/sites/default/files/319433-014.pdf.

-- 
error compiling committee.c: too many arguments to function