Re: [PATCH v6 3/3] KVM: MMU: prefetch ptes when intercepted guest #PF

[Marcelo Tosatti <mtosatti@redhat.com>]

On Wed, Aug 04, 2010 at 11:33:29AM +0800, Xiao Guangrong wrote:
> Support prefetch ptes when intercept guest #PF, avoid to #PF by later
> access
> 
> If we meet any failure in the prefetch path, we will exit it and
> not try other ptes to avoid become heavy path
> 
> Signed-off-by: Xiao Guangrong <xiaoguangrong@cn.fujitsu.com>
> ---
>  arch/x86/kvm/mmu.c         |  107 +++++++++++++++++++++++++++++++++++++++++++-
>  arch/x86/kvm/paging_tmpl.h |   81 ++++++++++++++++++++++++++++++++-
>  2 files changed, 184 insertions(+), 4 deletions(-)
> 
> diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
> index 9c69725..4501734 100644
> --- a/arch/x86/kvm/mmu.c
> +++ b/arch/x86/kvm/mmu.c
> @@ -89,6 +89,8 @@ module_param(oos_shadow, bool, 0644);
>  	}
>  #endif
>  
> +#define PTE_PREFETCH_NUM		8
> +
>  #define PT_FIRST_AVAIL_BITS_SHIFT 9
>  #define PT64_SECOND_AVAIL_BITS_SHIFT 52
>  
> @@ -403,7 +405,7 @@ static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
>  	if (r)
>  		goto out;
>  	r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache,
> -				   rmap_desc_cache, 4);
> +				   rmap_desc_cache, 4 + PTE_PREFETCH_NUM);
>  	if (r)
>  		goto out;
>  	r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
> @@ -2090,6 +2092,108 @@ static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
>  {
>  }
>  
> +static struct kvm_memory_slot *
> +pte_prefetch_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn, bool no_dirty_log)
> +{
> +	struct kvm_memory_slot *slot;
> +
> +	slot = gfn_to_memslot(vcpu->kvm, gfn);
> +	if (!slot || slot->flags & KVM_MEMSLOT_INVALID ||
> +	      (no_dirty_log && slot->dirty_bitmap))
> +		slot = NULL;

Why is this no_dirty_log optimization worthwhile?

> +
> +	return slot;
> +}
> +
> +static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
> +				     bool no_dirty_log)
> +{
> +	struct kvm_memory_slot *slot;
> +	unsigned long hva;
> +
> +	slot = pte_prefetch_gfn_to_memslot(vcpu, gfn, no_dirty_log);
> +	if (!slot) {
> +		get_page(bad_page);
> +		return page_to_pfn(bad_page);
> +	}
> +
> +	hva = gfn_to_hva_memslot(slot, gfn);
> +
> +	return hva_to_pfn_atomic(vcpu->kvm, hva);
> +}
> +
> +static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
> +				    struct kvm_mmu_page *sp,
> +				    u64 *start, u64 *end)
> +{
> +	struct page *pages[PTE_PREFETCH_NUM];
> +	struct kvm_memory_slot *slot;
> +	unsigned hva, access = sp->role.access;
> +	int i, ret, npages = end - start;
> +	gfn_t gfn;
> +
> +	gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
> +	slot = pte_prefetch_gfn_to_memslot(vcpu, gfn, access & ACC_WRITE_MASK);
> +	if (!slot || slot->npages - (gfn - slot->base_gfn) != npages)
> +		return -1;
> +
> +	hva = gfn_to_hva_memslot(slot, gfn);
> +	ret = __get_user_pages_fast(hva, npages, 1, pages);
> +	if (ret <= 0)
> +		return -1;

Better do one at a time with hva_to_pfn_atomic. Or, if you measure that
its worthwhile, do on a separate patch (using a helper as discussed
previously).

> +
> +	for (i = 0; i < ret; i++, gfn++, start++)
> +		mmu_set_spte(vcpu, start, ACC_ALL,
> +			     access, 0, 0, 1, NULL,
> +			     sp->role.level, gfn,
> +			     page_to_pfn(pages[i]), true, true);
> +
> +	return ret == npages ? 0 : -1;
> +}
> +
> +static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
> +				  struct kvm_mmu_page *sp, u64 *sptep)
> +{
> +	u64 *spte, *start = NULL;
> +	int i;
> +
> +	WARN_ON(!sp->role.direct);
> +
> +	i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
> +	spte = sp->spt + i;
> +
> +	for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
> +		if (*spte != shadow_trap_nonpresent_pte || spte == sptep) {
> +			if (!start)
> +				continue;
> +			if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
> +				break;
> +			start = NULL;
> +		} else if (!start)
> +			start = spte;
> +	}
> +}
> +
> +static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
> +{
> +	struct kvm_mmu_page *sp;
> +
> +	/*
> +	 * Since it's no accessed bit on EPT, it's no way to
> +	 * distinguish between actually accessed translations
> +	 * and prefetched, so disable pte prefetch if EPT is
> +	 * enabled.
> +	 */
> +	if (!shadow_accessed_mask)
> +		return;
> +
> +	sp = page_header(__pa(sptep));
> +	if (sp->role.level > PT_PAGE_TABLE_LEVEL)
> +		return;
> +
> +	__direct_pte_prefetch(vcpu, sp, sptep);
> +}
> +
>  static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
>  			int level, gfn_t gfn, pfn_t pfn)
>  {
> @@ -2103,6 +2207,7 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
>  			mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, ACC_ALL,
>  				     0, write, 1, &pt_write,
>  				     level, gfn, pfn, false, true);
> +			direct_pte_prefetch(vcpu, iterator.sptep);
>  			++vcpu->stat.pf_fixed;
>  			break;
>  		}
> diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
> index 51ef909..114c17d 100644
> --- a/arch/x86/kvm/paging_tmpl.h
> +++ b/arch/x86/kvm/paging_tmpl.h
> @@ -67,6 +67,7 @@ struct guest_walker {
>  	int level;
>  	gfn_t table_gfn[PT_MAX_FULL_LEVELS];
>  	pt_element_t ptes[PT_MAX_FULL_LEVELS];
> +	pt_element_t prefetch_ptes[PTE_PREFETCH_NUM];
>  	gpa_t pte_gpa[PT_MAX_FULL_LEVELS];
>  	unsigned pt_access;
>  	unsigned pte_access;
> @@ -302,14 +303,87 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
>  static bool FNAME(gpte_changed)(struct kvm_vcpu *vcpu,
>  				struct guest_walker *gw, int level)
>  {
> -	int r;
>  	pt_element_t curr_pte;
> -
> -	r = kvm_read_guest_atomic(vcpu->kvm, gw->pte_gpa[level - 1],
> +	gpa_t base_gpa, pte_gpa = gw->pte_gpa[level - 1];
> +	u64 mask;
> +	int r, index;
> +
> +	if (level == PT_PAGE_TABLE_LEVEL) {
> +		mask = PTE_PREFETCH_NUM * sizeof(pt_element_t) - 1;
> +		base_gpa = pte_gpa & ~mask;
> +		index = (pte_gpa - base_gpa) / sizeof(pt_element_t);
> +
> +		r = kvm_read_guest_atomic(vcpu->kvm, base_gpa,
> +				gw->prefetch_ptes, sizeof(gw->prefetch_ptes));
> +		curr_pte = gw->prefetch_ptes[index];

This can slowdown a single non-prefetchable pte fault. Maybe its
irrelevant, but please have kvm_read_guest_atomic in the first patch and
then later optimize, its easier to review and bisectable.