From: christoffer.dall@linaro.org (Christoffer Dall)
To: linux-arm-kernel@lists.infradead.org
Subject: [PATCH v3 04/11] arm64: KVM: flush VM pages before letting the guest enable caches
Date: Thu, 6 Feb 2014 20:08:28 -0800 [thread overview]
Message-ID: <20140207040828.GO9157@cbox> (raw)
In-Reply-To: <1391630151-7875-5-git-send-email-marc.zyngier@arm.com>
On Wed, Feb 05, 2014 at 07:55:44PM +0000, Marc Zyngier wrote:
> When the guest runs with caches disabled (like in an early boot
> sequence, for example), all the writes are diectly going to RAM,
> bypassing the caches altogether.
>
> Once the MMU and caches are enabled, whatever sits in the cache
> becomes suddenly visible, which isn't what the guest expects.
>
> A way to avoid this potential disaster is to invalidate the cache
> when the MMU is being turned on. For this, we hook into the SCTLR_EL1
> trapping code, and scan the stage-2 page tables, invalidating the
> pages/sections that have already been mapped in.
>
> Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
> Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
> ---
> arch/arm/include/asm/kvm_mmu.h | 8 ++++
> arch/arm/kvm/mmu.c | 93 ++++++++++++++++++++++++++++++++++++++++
> arch/arm64/include/asm/kvm_mmu.h | 4 ++
> arch/arm64/kvm/sys_regs.c | 5 ++-
> 4 files changed, 109 insertions(+), 1 deletion(-)
>
> diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
> index 6d0f3d3..0931cda 100644
> --- a/arch/arm/include/asm/kvm_mmu.h
> +++ b/arch/arm/include/asm/kvm_mmu.h
> @@ -114,6 +114,12 @@ static inline void kvm_set_s2pmd_writable(pmd_t *pmd)
> pmd_val(*pmd) |= L_PMD_S2_RDWR;
> }
>
> +/* Open coded pgd_addr_end that can deal with 64bit addresses */
> +#define kvm_pgd_addr_end(addr, end) \
> +({ u64 __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \
> + (__boundary - 1 < (end) - 1)? __boundary: (end); \
> +})
> +
> struct kvm;
>
> static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,
> @@ -142,6 +148,8 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,
> #define kvm_flush_dcache_to_poc(a,l) __cpuc_flush_dcache_area((a), (l))
> #define kvm_virt_to_phys(x) virt_to_idmap((unsigned long)(x))
>
> +void stage2_flush_vm(struct kvm *kvm);
> +
> #endif /* !__ASSEMBLY__ */
>
> #endif /* __ARM_KVM_MMU_H__ */
> diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
> index fc71a8d..ea21b6a 100644
> --- a/arch/arm/kvm/mmu.c
> +++ b/arch/arm/kvm/mmu.c
> @@ -187,6 +187,99 @@ static void unmap_range(struct kvm *kvm, pgd_t *pgdp,
> }
> }
>
> +static void stage2_flush_ptes(struct kvm *kvm, pmd_t *pmd,
> + phys_addr_t addr, phys_addr_t end)
> +{
> + pte_t *pte;
> +
> + pte = pte_offset_kernel(pmd, addr);
> + do {
> + if (!pte_none(*pte)) {
> + hva_t hva = gfn_to_hva(kvm, addr >> PAGE_SHIFT);
> + kvm_flush_dcache_to_poc((void*)hva, PAGE_SIZE);
> + }
> + } while (pte++, addr += PAGE_SIZE, addr != end);
> +}
> +
> +static void stage2_flush_pmds(struct kvm *kvm, pud_t *pud,
> + phys_addr_t addr, phys_addr_t end)
> +{
> + pmd_t *pmd;
> + phys_addr_t next;
> +
> + pmd = pmd_offset(pud, addr);
> + do {
> + next = pmd_addr_end(addr, end);
> + if (!pmd_none(*pmd)) {
> + if (kvm_pmd_huge(*pmd)) {
> + hva_t hva = gfn_to_hva(kvm, addr >> PAGE_SHIFT);
> + kvm_flush_dcache_to_poc((void*)hva, PMD_SIZE);
> + } else {
> + stage2_flush_ptes(kvm, pmd, addr, next);
> + }
> + }
> + } while (pmd++, addr = next, addr != end);
> +}
> +
> +static void stage2_flush_puds(struct kvm *kvm, pgd_t *pgd,
> + phys_addr_t addr, phys_addr_t end)
> +{
> + pud_t *pud;
> + phys_addr_t next;
> +
> + pud = pud_offset(pgd, addr);
> + do {
> + next = pud_addr_end(addr, end);
> + if (!pud_none(*pud)) {
> + if (pud_huge(*pud)) {
> + hva_t hva = gfn_to_hva(kvm, addr >> PAGE_SHIFT);
> + kvm_flush_dcache_to_poc((void*)hva, PUD_SIZE);
> + } else {
> + stage2_flush_pmds(kvm, pud, addr, next);
> + }
> + }
> + } while(pud++, addr = next, addr != end);
you missed one space after this while, but no need to respin just
because of that.
> +}
> +
> +static void stage2_flush_memslot(struct kvm *kvm,
> + struct kvm_memory_slot *memslot)
> +{
> + phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
> + phys_addr_t end = addr + PAGE_SIZE * memslot->npages;
> + phys_addr_t next;
> + pgd_t *pgd;
> +
> + pgd = kvm->arch.pgd + pgd_index(addr);
> + do {
> + next = kvm_pgd_addr_end(addr, end);
> + stage2_flush_puds(kvm, pgd, addr, next);
> + } while (pgd++, addr = next, addr != end);
> +}
> +
> +/**
> + * stage2_flush_vm - Invalidate cache for pages mapped in stage 2
> + * @kvm: The struct kvm pointer
> + *
> + * Go through the stage 2 page tables and invalidate any cache lines
> + * backing memory already mapped to the VM.
> + */
> +void stage2_flush_vm(struct kvm *kvm)
> +{
> + struct kvm_memslots *slots;
> + struct kvm_memory_slot *memslot;
> + int idx;
> +
> + idx = srcu_read_lock(&kvm->srcu);
> + spin_lock(&kvm->mmu_lock);
> +
> + slots = kvm_memslots(kvm);
> + kvm_for_each_memslot(memslot, slots)
> + stage2_flush_memslot(kvm, memslot);
> +
> + spin_unlock(&kvm->mmu_lock);
> + srcu_read_unlock(&kvm->srcu, idx);
> +}
> +
> /**
> * free_boot_hyp_pgd - free HYP boot page tables
> *
> diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
> index 6eaf69b..e78d050 100644
> --- a/arch/arm64/include/asm/kvm_mmu.h
> +++ b/arch/arm64/include/asm/kvm_mmu.h
> @@ -121,6 +121,8 @@ static inline void kvm_set_s2pmd_writable(pmd_t *pmd)
> pmd_val(*pmd) |= PMD_S2_RDWR;
> }
>
> +#define kvm_pgd_addr_end(addr, end) pgd_addr_end(addr, end)
> +
> struct kvm;
>
> #define kvm_flush_dcache_to_poc(a,l) __flush_dcache_area((a), (l))
> @@ -146,5 +148,7 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,
>
> #define kvm_virt_to_phys(x) __virt_to_phys((unsigned long)(x))
>
> +void stage2_flush_vm(struct kvm *kvm);
> +
> #endif /* __ASSEMBLY__ */
> #endif /* __ARM64_KVM_MMU_H__ */
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index 2097e5e..0324458 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -27,6 +27,7 @@
> #include <asm/kvm_host.h>
> #include <asm/kvm_emulate.h>
> #include <asm/kvm_coproc.h>
> +#include <asm/kvm_mmu.h>
> #include <asm/cacheflush.h>
> #include <asm/cputype.h>
> #include <trace/events/kvm.h>
> @@ -154,8 +155,10 @@ static bool access_sctlr(struct kvm_vcpu *vcpu,
> {
> access_vm_reg(vcpu, p, r);
>
> - if (vcpu_has_cache_enabled(vcpu)) /* MMU+Caches enabled? */
> + if (vcpu_has_cache_enabled(vcpu)) { /* MMU+Caches enabled? */
> vcpu->arch.hcr_el2 &= ~HCR_TVM;
> + stage2_flush_vm(vcpu->kvm);
> + }
>
> return true;
> }
> --
> 1.8.3.4
>
Reviewed-by: Christoffer Dall <christoffer.dall@linaro.org>
WARNING: multiple messages have this Message-ID (diff)
From: Christoffer Dall <christoffer.dall@linaro.org>
To: Marc Zyngier <marc.zyngier@arm.com>
Cc: linux-arm-kernel@lists.infradead.org,
kvmarm@lists.cs.columbia.edu, kvm@vger.kernel.org,
catalin.marinas@arm.com
Subject: Re: [PATCH v3 04/11] arm64: KVM: flush VM pages before letting the guest enable caches
Date: Thu, 6 Feb 2014 20:08:28 -0800 [thread overview]
Message-ID: <20140207040828.GO9157@cbox> (raw)
In-Reply-To: <1391630151-7875-5-git-send-email-marc.zyngier@arm.com>
On Wed, Feb 05, 2014 at 07:55:44PM +0000, Marc Zyngier wrote:
> When the guest runs with caches disabled (like in an early boot
> sequence, for example), all the writes are diectly going to RAM,
> bypassing the caches altogether.
>
> Once the MMU and caches are enabled, whatever sits in the cache
> becomes suddenly visible, which isn't what the guest expects.
>
> A way to avoid this potential disaster is to invalidate the cache
> when the MMU is being turned on. For this, we hook into the SCTLR_EL1
> trapping code, and scan the stage-2 page tables, invalidating the
> pages/sections that have already been mapped in.
>
> Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
> Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
> ---
> arch/arm/include/asm/kvm_mmu.h | 8 ++++
> arch/arm/kvm/mmu.c | 93 ++++++++++++++++++++++++++++++++++++++++
> arch/arm64/include/asm/kvm_mmu.h | 4 ++
> arch/arm64/kvm/sys_regs.c | 5 ++-
> 4 files changed, 109 insertions(+), 1 deletion(-)
>
> diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
> index 6d0f3d3..0931cda 100644
> --- a/arch/arm/include/asm/kvm_mmu.h
> +++ b/arch/arm/include/asm/kvm_mmu.h
> @@ -114,6 +114,12 @@ static inline void kvm_set_s2pmd_writable(pmd_t *pmd)
> pmd_val(*pmd) |= L_PMD_S2_RDWR;
> }
>
> +/* Open coded pgd_addr_end that can deal with 64bit addresses */
> +#define kvm_pgd_addr_end(addr, end) \
> +({ u64 __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \
> + (__boundary - 1 < (end) - 1)? __boundary: (end); \
> +})
> +
> struct kvm;
>
> static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,
> @@ -142,6 +148,8 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,
> #define kvm_flush_dcache_to_poc(a,l) __cpuc_flush_dcache_area((a), (l))
> #define kvm_virt_to_phys(x) virt_to_idmap((unsigned long)(x))
>
> +void stage2_flush_vm(struct kvm *kvm);
> +
> #endif /* !__ASSEMBLY__ */
>
> #endif /* __ARM_KVM_MMU_H__ */
> diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
> index fc71a8d..ea21b6a 100644
> --- a/arch/arm/kvm/mmu.c
> +++ b/arch/arm/kvm/mmu.c
> @@ -187,6 +187,99 @@ static void unmap_range(struct kvm *kvm, pgd_t *pgdp,
> }
> }
>
> +static void stage2_flush_ptes(struct kvm *kvm, pmd_t *pmd,
> + phys_addr_t addr, phys_addr_t end)
> +{
> + pte_t *pte;
> +
> + pte = pte_offset_kernel(pmd, addr);
> + do {
> + if (!pte_none(*pte)) {
> + hva_t hva = gfn_to_hva(kvm, addr >> PAGE_SHIFT);
> + kvm_flush_dcache_to_poc((void*)hva, PAGE_SIZE);
> + }
> + } while (pte++, addr += PAGE_SIZE, addr != end);
> +}
> +
> +static void stage2_flush_pmds(struct kvm *kvm, pud_t *pud,
> + phys_addr_t addr, phys_addr_t end)
> +{
> + pmd_t *pmd;
> + phys_addr_t next;
> +
> + pmd = pmd_offset(pud, addr);
> + do {
> + next = pmd_addr_end(addr, end);
> + if (!pmd_none(*pmd)) {
> + if (kvm_pmd_huge(*pmd)) {
> + hva_t hva = gfn_to_hva(kvm, addr >> PAGE_SHIFT);
> + kvm_flush_dcache_to_poc((void*)hva, PMD_SIZE);
> + } else {
> + stage2_flush_ptes(kvm, pmd, addr, next);
> + }
> + }
> + } while (pmd++, addr = next, addr != end);
> +}
> +
> +static void stage2_flush_puds(struct kvm *kvm, pgd_t *pgd,
> + phys_addr_t addr, phys_addr_t end)
> +{
> + pud_t *pud;
> + phys_addr_t next;
> +
> + pud = pud_offset(pgd, addr);
> + do {
> + next = pud_addr_end(addr, end);
> + if (!pud_none(*pud)) {
> + if (pud_huge(*pud)) {
> + hva_t hva = gfn_to_hva(kvm, addr >> PAGE_SHIFT);
> + kvm_flush_dcache_to_poc((void*)hva, PUD_SIZE);
> + } else {
> + stage2_flush_pmds(kvm, pud, addr, next);
> + }
> + }
> + } while(pud++, addr = next, addr != end);
you missed one space after this while, but no need to respin just
because of that.
> +}
> +
> +static void stage2_flush_memslot(struct kvm *kvm,
> + struct kvm_memory_slot *memslot)
> +{
> + phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
> + phys_addr_t end = addr + PAGE_SIZE * memslot->npages;
> + phys_addr_t next;
> + pgd_t *pgd;
> +
> + pgd = kvm->arch.pgd + pgd_index(addr);
> + do {
> + next = kvm_pgd_addr_end(addr, end);
> + stage2_flush_puds(kvm, pgd, addr, next);
> + } while (pgd++, addr = next, addr != end);
> +}
> +
> +/**
> + * stage2_flush_vm - Invalidate cache for pages mapped in stage 2
> + * @kvm: The struct kvm pointer
> + *
> + * Go through the stage 2 page tables and invalidate any cache lines
> + * backing memory already mapped to the VM.
> + */
> +void stage2_flush_vm(struct kvm *kvm)
> +{
> + struct kvm_memslots *slots;
> + struct kvm_memory_slot *memslot;
> + int idx;
> +
> + idx = srcu_read_lock(&kvm->srcu);
> + spin_lock(&kvm->mmu_lock);
> +
> + slots = kvm_memslots(kvm);
> + kvm_for_each_memslot(memslot, slots)
> + stage2_flush_memslot(kvm, memslot);
> +
> + spin_unlock(&kvm->mmu_lock);
> + srcu_read_unlock(&kvm->srcu, idx);
> +}
> +
> /**
> * free_boot_hyp_pgd - free HYP boot page tables
> *
> diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
> index 6eaf69b..e78d050 100644
> --- a/arch/arm64/include/asm/kvm_mmu.h
> +++ b/arch/arm64/include/asm/kvm_mmu.h
> @@ -121,6 +121,8 @@ static inline void kvm_set_s2pmd_writable(pmd_t *pmd)
> pmd_val(*pmd) |= PMD_S2_RDWR;
> }
>
> +#define kvm_pgd_addr_end(addr, end) pgd_addr_end(addr, end)
> +
> struct kvm;
>
> #define kvm_flush_dcache_to_poc(a,l) __flush_dcache_area((a), (l))
> @@ -146,5 +148,7 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,
>
> #define kvm_virt_to_phys(x) __virt_to_phys((unsigned long)(x))
>
> +void stage2_flush_vm(struct kvm *kvm);
> +
> #endif /* __ASSEMBLY__ */
> #endif /* __ARM64_KVM_MMU_H__ */
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index 2097e5e..0324458 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -27,6 +27,7 @@
> #include <asm/kvm_host.h>
> #include <asm/kvm_emulate.h>
> #include <asm/kvm_coproc.h>
> +#include <asm/kvm_mmu.h>
> #include <asm/cacheflush.h>
> #include <asm/cputype.h>
> #include <trace/events/kvm.h>
> @@ -154,8 +155,10 @@ static bool access_sctlr(struct kvm_vcpu *vcpu,
> {
> access_vm_reg(vcpu, p, r);
>
> - if (vcpu_has_cache_enabled(vcpu)) /* MMU+Caches enabled? */
> + if (vcpu_has_cache_enabled(vcpu)) { /* MMU+Caches enabled? */
> vcpu->arch.hcr_el2 &= ~HCR_TVM;
> + stage2_flush_vm(vcpu->kvm);
> + }
>
> return true;
> }
> --
> 1.8.3.4
>
Reviewed-by: Christoffer Dall <christoffer.dall@linaro.org>
next prev parent reply other threads:[~2014-02-07 4:08 UTC|newest]
Thread overview: 50+ messages / expand[flat|nested] mbox.gz Atom feed top
2014-02-05 19:55 [PATCH v3 00/11] arm/arm64: KVM: host cache maintenance when guest caches are off Marc Zyngier
2014-02-05 19:55 ` Marc Zyngier
2014-02-05 19:55 ` [PATCH v3 01/11] arm64: KVM: force cache clean on page fault when " Marc Zyngier
2014-02-05 19:55 ` Marc Zyngier
2014-02-05 19:55 ` [PATCH v3 02/11] arm64: KVM: allows discrimination of AArch32 sysreg access Marc Zyngier
2014-02-05 19:55 ` Marc Zyngier
2014-02-05 19:55 ` [PATCH v3 03/11] arm64: KVM: trap VM system registers until MMU and caches are ON Marc Zyngier
2014-02-05 19:55 ` Marc Zyngier
2014-02-05 19:55 ` [PATCH v3 04/11] arm64: KVM: flush VM pages before letting the guest enable caches Marc Zyngier
2014-02-05 19:55 ` Marc Zyngier
2014-02-07 4:08 ` Christoffer Dall [this message]
2014-02-07 4:08 ` Christoffer Dall
2014-02-05 19:55 ` [PATCH v3 05/11] ARM: LPAE: provide an IPA capable pmd_addr_end Marc Zyngier
2014-02-05 19:55 ` Marc Zyngier
2014-02-06 10:43 ` Catalin Marinas
2014-02-06 10:43 ` Catalin Marinas
2014-02-07 4:04 ` Christoffer Dall
2014-02-07 4:04 ` Christoffer Dall
2014-02-07 15:44 ` Catalin Marinas
2014-02-07 15:44 ` Catalin Marinas
2014-02-07 17:10 ` Christoffer Dall
2014-02-07 17:10 ` Christoffer Dall
2014-02-11 9:07 ` Marc Zyngier
2014-02-11 9:07 ` Marc Zyngier
2014-02-05 19:55 ` [PATCH v3 06/11] ARM: KVM: force cache clean on page fault when caches are off Marc Zyngier
2014-02-05 19:55 ` Marc Zyngier
2014-02-06 10:49 ` Catalin Marinas
2014-02-06 10:49 ` Catalin Marinas
2014-02-05 19:55 ` [PATCH v3 07/11] ARM: KVM: fix handling of trapped 64bit coprocessor accesses Marc Zyngier
2014-02-05 19:55 ` Marc Zyngier
2014-02-06 10:49 ` Catalin Marinas
2014-02-06 10:49 ` Catalin Marinas
2014-02-05 19:55 ` [PATCH v3 08/11] ARM: KVM: fix ordering of " Marc Zyngier
2014-02-05 19:55 ` Marc Zyngier
2014-02-06 10:50 ` Catalin Marinas
2014-02-06 10:50 ` Catalin Marinas
2014-02-05 19:55 ` [PATCH v3 09/11] ARM: KVM: introduce per-vcpu HYP Configuration Register Marc Zyngier
2014-02-05 19:55 ` Marc Zyngier
2014-02-06 11:01 ` Catalin Marinas
2014-02-06 11:01 ` Catalin Marinas
2014-02-05 19:55 ` [PATCH v3 10/11] ARM: KVM: add world-switch for AMAIR{0,1} Marc Zyngier
2014-02-05 19:55 ` Marc Zyngier
2014-02-06 11:02 ` Catalin Marinas
2014-02-06 11:02 ` Catalin Marinas
2014-02-05 19:55 ` [PATCH v3 11/11] ARM: KVM: trap VM system registers until MMU and caches are ON Marc Zyngier
2014-02-05 19:55 ` Marc Zyngier
2014-02-06 11:03 ` Catalin Marinas
2014-02-06 11:03 ` Catalin Marinas
2014-02-07 4:08 ` Christoffer Dall
2014-02-07 4:08 ` Christoffer Dall
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20140207040828.GO9157@cbox \
--to=christoffer.dall@linaro.org \
--cc=linux-arm-kernel@lists.infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.