* [RFC 29/55] KVM: arm/arm64: Set up the prepared vgic state
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
Since vgic state is properly prepared and is pointed by hw_v2_cpu_if,
let's use it to manipulate vgic.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
virt/kvm/arm/hyp/vgic-v2-sr.c | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/virt/kvm/arm/hyp/vgic-v2-sr.c b/virt/kvm/arm/hyp/vgic-v2-sr.c
index c8aeb7b..5d4898f 100644
--- a/virt/kvm/arm/hyp/vgic-v2-sr.c
+++ b/virt/kvm/arm/hyp/vgic-v2-sr.c
@@ -22,10 +22,15 @@
#include <asm/kvm_emulate.h>
#include <asm/kvm_hyp.h>
+static __hyp_text struct vgic_v2_cpu_if *__hyp_get_cpu_if(struct kvm_vcpu *vcpu)
+{
+ return kern_hyp_va(vcpu->arch.vgic_cpu.hw_v2_cpu_if);
+}
+
static void __hyp_text save_maint_int_state(struct kvm_vcpu *vcpu,
void __iomem *base)
{
- struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
+ struct vgic_v2_cpu_if *cpu_if = __hyp_get_cpu_if(vcpu);
int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
u32 eisr0, eisr1;
int i;
@@ -67,7 +72,7 @@ static void __hyp_text save_maint_int_state(struct kvm_vcpu *vcpu,
static void __hyp_text save_elrsr(struct kvm_vcpu *vcpu, void __iomem *base)
{
- struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
+ struct vgic_v2_cpu_if *cpu_if = __hyp_get_cpu_if(vcpu);
int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
u32 elrsr0, elrsr1;
@@ -86,7 +91,7 @@ static void __hyp_text save_elrsr(struct kvm_vcpu *vcpu, void __iomem *base)
static void __hyp_text save_lrs(struct kvm_vcpu *vcpu, void __iomem *base)
{
- struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
+ struct vgic_v2_cpu_if *cpu_if = __hyp_get_cpu_if(vcpu);
int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
int i;
@@ -107,7 +112,7 @@ static void __hyp_text save_lrs(struct kvm_vcpu *vcpu, void __iomem *base)
void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu)
{
struct kvm *kvm = kern_hyp_va(vcpu->kvm);
- struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
+ struct vgic_v2_cpu_if *cpu_if = __hyp_get_cpu_if(vcpu);
struct vgic_dist *vgic = &kvm->arch.vgic;
void __iomem *base = kern_hyp_va(vgic->vctrl_base);
@@ -138,7 +143,7 @@ void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu)
void __hyp_text __vgic_v2_restore_state(struct kvm_vcpu *vcpu)
{
struct kvm *kvm = kern_hyp_va(vcpu->kvm);
- struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
+ struct vgic_v2_cpu_if *cpu_if = __hyp_get_cpu_if(vcpu);
struct vgic_dist *vgic = &kvm->arch.vgic;
void __iomem *base = kern_hyp_va(vgic->vctrl_base);
int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
--
1.9.1
^ permalink raw reply related
* [RFC 30/55] KVM: arm/arm64: Inject irqs to the guest hypervisor
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
If we have a pending IRQ for the guest and the guest expects IRQs
to be handled in its virtual EL2 mode (the virtual IMO bit is set)
and it is not already running in virtual EL2 mode, then we have to
emulate an IRQ exception.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
virt/kvm/arm/vgic/vgic.c | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/virt/kvm/arm/vgic/vgic.c b/virt/kvm/arm/vgic/vgic.c
index 6440b56..4a98654 100644
--- a/virt/kvm/arm/vgic/vgic.c
+++ b/virt/kvm/arm/vgic/vgic.c
@@ -17,6 +17,7 @@
#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <linux/list_sort.h>
+#include <asm/kvm_emulate.h>
#include "vgic.h"
@@ -652,6 +653,28 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu)
/* Nuke remaining LRs */
for ( ; count < kvm_vgic_global_state.nr_lr; count++)
vgic_clear_lr(vcpu, count);
+
+ /*
+ * If we have any pending IRQ for the guest and the guest expects IRQs
+ * to be handled in its virtual EL2 mode (the virtual IMO bit is set)
+ * and it is not already running in virtual EL2 mode, then we have to
+ * emulate an IRQ exception to virtual IRQ. Note that a pending IRQ
+ * means an irq of which state is pending but not active.
+ */
+ if (vcpu_el2_imo_is_set(vcpu) && !vcpu_mode_el2(vcpu)) {
+ bool pending = false;
+
+ list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
+ spin_lock(&irq->irq_lock);
+ pending = irq->pending && irq->enabled && !irq->active;
+ spin_unlock(&irq->irq_lock);
+
+ if (pending) {
+ kvm_inject_nested_irq(vcpu);
+ break;
+ }
+ }
+ }
}
/* Sync back the hardware VGIC state into our emulation after a guest's run. */
--
1.9.1
^ permalink raw reply related
* [RFC 31/55] KVM: arm/arm64: Inject maintenance interrupts to the guest hypervisor
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
If we exit a nested VM with a pending maintenance interrupt from the
GIC, then we need to forward this to the guest hypervisor so that it can
re-sync the appropriate LRs and sample level triggered interrupts again.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/kvm/context.c | 3 +++
include/kvm/arm_vgic.h | 2 ++
virt/kvm/arm/vgic/vgic-v2-nested.c | 16 ++++++++++++++++
3 files changed, 21 insertions(+)
diff --git a/arch/arm64/kvm/context.c b/arch/arm64/kvm/context.c
index 7a94c9d..a93ffe4 100644
--- a/arch/arm64/kvm/context.c
+++ b/arch/arm64/kvm/context.c
@@ -140,6 +140,9 @@ static void sync_shadow_el1_state(struct kvm_vcpu *vcpu, bool setup)
void kvm_arm_setup_shadow_state(struct kvm_vcpu *vcpu)
{
struct kvm_cpu_context *ctxt = &vcpu->arch.ctxt;
+
+ vgic_handle_nested_maint_irq(vcpu);
+
if (unlikely(vcpu_mode_el2(vcpu))) {
ctxt->hw_pstate = *vcpu_cpsr(vcpu) & ~PSR_MODE_MASK;
diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
index 484f6b1..fc882d6 100644
--- a/include/kvm/arm_vgic.h
+++ b/include/kvm/arm_vgic.h
@@ -315,9 +315,11 @@ int kvm_vgic_inject_mapped_irq(struct kvm *kvm, int cpuid, unsigned int intid,
#ifdef CONFIG_KVM_ARM_NESTED_HYP
void vgic_v2_setup_shadow_state(struct kvm_vcpu *vcpu);
void vgic_v2_restore_shadow_state(struct kvm_vcpu *vcpu);
+void vgic_handle_nested_maint_irq(struct kvm_vcpu *vcpu);
#else
static inline void vgic_v2_setup_shadow_state(struct kvm_vcpu *vcpu) { }
static inline void vgic_v2_restore_shadow_state(struct kvm_vcpu *vcpu) { }
+static inline void vgic_handle_nested_maint_irq(struct kvm_vcpu *vcpu) { }
#endif
#define irqchip_in_kernel(k) (!!((k)->arch.vgic.in_kernel))
diff --git a/virt/kvm/arm/vgic/vgic-v2-nested.c b/virt/kvm/arm/vgic/vgic-v2-nested.c
index a992da5..85f646b 100644
--- a/virt/kvm/arm/vgic/vgic-v2-nested.c
+++ b/virt/kvm/arm/vgic/vgic-v2-nested.c
@@ -300,6 +300,22 @@ void vgic_v2_restore_shadow_state(struct kvm_vcpu *vcpu)
vgic_cpu->nested_vgic_v2 = vgic_cpu->shadow_vgic_v2;
}
+void vgic_handle_nested_maint_irq(struct kvm_vcpu *vcpu)
+{
+ struct vgic_v2_cpu_if *cpu_if = vcpu_nested_if(vcpu);
+
+ /*
+ * If we exit a nested VM with a pending maintenance interrupt from the
+ * GIC, then we need to forward this to the guest hypervisor so that it
+ * can re-sync the appropriate LRs and sample level triggered interrupts
+ * again.
+ */
+ if (vcpu_el2_imo_is_set(vcpu) && !vcpu_mode_el2(vcpu) &&
+ (cpu_if->vgic_hcr & GICH_HCR_EN) &&
+ vgic_mmio_read_v2_misr(vcpu, 0, 0))
+ kvm_inject_nested_irq(vcpu);
+}
+
void vgic_init_nested(struct kvm_vcpu *vcpu)
{
vgic_v2_setup_shadow_state(vcpu);
--
1.9.1
^ permalink raw reply related
* [RFC 32/55] KVM: arm/arm64: register GICH iodev for the guest hypervisor
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
Register a device for the virtual interface control block(GICH) access
from the guest hypervisor.
TODO: Get GICH address from DT, which is hardcoded now.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/include/uapi/asm/kvm.h | 6 ++++++
include/kvm/arm_vgic.h | 5 ++++-
virt/kvm/arm/vgic/vgic-mmio.c | 6 ++++++
virt/kvm/arm/vgic/vgic-v2-nested.c | 24 ++++++++++++++++++++++++
virt/kvm/arm/vgic/vgic-v2.c | 7 +++++++
virt/kvm/arm/vgic/vgic.h | 6 ++++++
6 files changed, 53 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
index 78117bf..3995d3d 100644
--- a/arch/arm64/include/uapi/asm/kvm.h
+++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -99,6 +99,12 @@ struct kvm_regs {
#define KVM_ARM_VCPU_PMU_V3 3 /* Support guest PMUv3 */
#define KVM_ARM_VCPU_NESTED_VIRT 4 /* Support nested virtual EL2 */
+/* FIXME: This should come from DT */
+#ifdef CONFIG_KVM_ARM_NESTED_HYP
+#define KVM_VGIC_V2_GICH_BASE 0x08030000
+#define KVM_VGIC_V2_GICH_SIZE 0x2000
+#endif
+
struct kvm_vcpu_init {
__u32 target;
__u32 features[7];
diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
index fc882d6..5bda20c 100644
--- a/include/kvm/arm_vgic.h
+++ b/include/kvm/arm_vgic.h
@@ -125,7 +125,8 @@ enum iodev_type {
IODEV_CPUIF,
IODEV_DIST,
IODEV_REDIST,
- IODEV_ITS
+ IODEV_ITS,
+ IODEV_GICH,
};
struct vgic_io_device {
@@ -198,6 +199,8 @@ struct vgic_dist {
struct vgic_io_device dist_iodev;
+ struct vgic_io_device hyp_iodev;
+
bool has_its;
/*
diff --git a/virt/kvm/arm/vgic/vgic-mmio.c b/virt/kvm/arm/vgic/vgic-mmio.c
index 049c570..2e4097d 100644
--- a/virt/kvm/arm/vgic/vgic-mmio.c
+++ b/virt/kvm/arm/vgic/vgic-mmio.c
@@ -512,6 +512,9 @@ static int dispatch_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
case IODEV_ITS:
data = region->its_read(vcpu->kvm, iodev->its, addr, len);
break;
+ case IODEV_GICH:
+ data = region->read(vcpu, addr, len);
+ break;
}
vgic_data_host_to_mmio_bus(val, len, data);
@@ -543,6 +546,9 @@ static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
case IODEV_ITS:
region->its_write(vcpu->kvm, iodev->its, addr, len, data);
break;
+ case IODEV_GICH:
+ region->write(vcpu, addr, len, data);
+ break;
}
return 0;
diff --git a/virt/kvm/arm/vgic/vgic-v2-nested.c b/virt/kvm/arm/vgic/vgic-v2-nested.c
index 85f646b..cb55324 100644
--- a/virt/kvm/arm/vgic/vgic-v2-nested.c
+++ b/virt/kvm/arm/vgic/vgic-v2-nested.c
@@ -206,6 +206,30 @@ static void vgic_mmio_write_v2_gich(struct kvm_vcpu *vcpu,
4 * VGIC_V2_MAX_LRS, VGIC_ACCESS_32bit),
};
+int vgic_register_gich_iodev(struct kvm *kvm, struct vgic_dist *dist)
+{
+ struct vgic_io_device *io_device = &kvm->arch.vgic.hyp_iodev;
+ int ret = 0;
+ unsigned int len;
+
+ len = KVM_VGIC_V2_GICH_SIZE;
+
+ io_device->regions = vgic_v2_gich_registers;
+ io_device->nr_regions = ARRAY_SIZE(vgic_v2_gich_registers);
+ kvm_iodevice_init(&io_device->dev, &kvm_io_gic_ops);
+
+ io_device->base_addr = KVM_VGIC_V2_GICH_BASE;
+ io_device->iodev_type = IODEV_GICH;
+ io_device->redist_vcpu = NULL;
+
+ mutex_lock(&kvm->slots_lock);
+ ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, KVM_VGIC_V2_GICH_BASE,
+ len, &io_device->dev);
+ mutex_unlock(&kvm->slots_lock);
+
+ return ret;
+}
+
/*
* For LRs which have HW bit set such as timer interrupts, we modify them to
* have the host hardware interrupt number instead of the virtual one programmed
diff --git a/virt/kvm/arm/vgic/vgic-v2.c b/virt/kvm/arm/vgic/vgic-v2.c
index 9bab867..b8b73fd 100644
--- a/virt/kvm/arm/vgic/vgic-v2.c
+++ b/virt/kvm/arm/vgic/vgic-v2.c
@@ -280,6 +280,13 @@ int vgic_v2_map_resources(struct kvm *kvm)
goto out;
}
+ /* Register virtual GICH interface to kvm io bus */
+ ret = vgic_register_gich_iodev(kvm, dist);
+ if (ret) {
+ kvm_err("Unable to register VGIC GICH regions\n");
+ goto out;
+ }
+
if (!static_branch_unlikely(&vgic_v2_cpuif_trap)) {
ret = kvm_phys_addr_ioremap(kvm, dist->vgic_cpu_base,
kvm_vgic_global_state.vcpu_base,
diff --git a/virt/kvm/arm/vgic/vgic.h b/virt/kvm/arm/vgic/vgic.h
index 2aef680..11d61a7 100644
--- a/virt/kvm/arm/vgic/vgic.h
+++ b/virt/kvm/arm/vgic/vgic.h
@@ -121,8 +121,14 @@ static inline int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
int vgic_init(struct kvm *kvm);
#ifdef CONFIG_KVM_ARM_NESTED_HYP
+int vgic_register_gich_iodev(struct kvm *kvm, struct vgic_dist *dist);
void vgic_init_nested(struct kvm_vcpu *vcpu);
#else
+static inline int vgic_register_gich_iodev(struct kvm *kvm,
+ struct vgic_dist *dist)
+{
+ return 0;
+}
static inline void vgic_init_nested(struct kvm_vcpu *vcpu)
{
struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
--
1.9.1
^ permalink raw reply related
* [RFC 33/55] KVM: arm/arm64: Remove unused params in mmu functions
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
stage2_flush_xxx functions take a pointer to the kvm struct as the first
parameter but they are never used. Clean this up before modifying mmu
code for nested virtualization support.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/kvm/mmu.c | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index a5265ed..57cb671 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -300,7 +300,7 @@ static void unmap_stage2_range(struct kvm *kvm, phys_addr_t start, u64 size)
} while (pgd++, addr = next, addr != end);
}
-static void stage2_flush_ptes(struct kvm *kvm, pmd_t *pmd,
+static void stage2_flush_ptes(pmd_t *pmd,
phys_addr_t addr, phys_addr_t end)
{
pte_t *pte;
@@ -312,7 +312,7 @@ static void stage2_flush_ptes(struct kvm *kvm, pmd_t *pmd,
} while (pte++, addr += PAGE_SIZE, addr != end);
}
-static void stage2_flush_pmds(struct kvm *kvm, pud_t *pud,
+static void stage2_flush_pmds(pud_t *pud,
phys_addr_t addr, phys_addr_t end)
{
pmd_t *pmd;
@@ -325,12 +325,12 @@ static void stage2_flush_pmds(struct kvm *kvm, pud_t *pud,
if (pmd_thp_or_huge(*pmd))
kvm_flush_dcache_pmd(*pmd);
else
- stage2_flush_ptes(kvm, pmd, addr, next);
+ stage2_flush_ptes(pmd, addr, next);
}
} while (pmd++, addr = next, addr != end);
}
-static void stage2_flush_puds(struct kvm *kvm, pgd_t *pgd,
+static void stage2_flush_puds(pgd_t *pgd,
phys_addr_t addr, phys_addr_t end)
{
pud_t *pud;
@@ -343,7 +343,7 @@ static void stage2_flush_puds(struct kvm *kvm, pgd_t *pgd,
if (stage2_pud_huge(*pud))
kvm_flush_dcache_pud(*pud);
else
- stage2_flush_pmds(kvm, pud, addr, next);
+ stage2_flush_pmds(pud, addr, next);
}
} while (pud++, addr = next, addr != end);
}
@@ -359,7 +359,7 @@ static void stage2_flush_memslot(struct kvm *kvm,
pgd = kvm->arch.pgd + stage2_pgd_index(addr);
do {
next = stage2_pgd_addr_end(addr, end);
- stage2_flush_puds(kvm, pgd, addr, next);
+ stage2_flush_puds(pgd, addr, next);
} while (pgd++, addr = next, addr != end);
}
--
1.9.1
^ permalink raw reply related
* [RFC 34/55] KVM: arm/arm64: Abstract stage-2 MMU state into a separate structure
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
Abstract stage-2 MMU state into a separate structure and change all
callers referring to page tables, VMIDs, and the VTTBR to use this new
indirection.
This is about to become very handy when using shadow stage-2 page
tables.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/include/asm/kvm_asm.h | 7 +-
arch/arm/include/asm/kvm_host.h | 26 ++++---
arch/arm/kvm/arm.c | 34 +++++----
arch/arm/kvm/hyp/switch.c | 5 +-
arch/arm/kvm/hyp/tlb.c | 18 ++---
arch/arm/kvm/mmu.c | 146 +++++++++++++++++++++-----------------
arch/arm64/include/asm/kvm_asm.h | 7 +-
arch/arm64/include/asm/kvm_host.h | 10 ++-
arch/arm64/kvm/hyp/switch.c | 5 +-
arch/arm64/kvm/hyp/tlb.c | 20 +++---
10 files changed, 159 insertions(+), 119 deletions(-)
diff --git a/arch/arm/include/asm/kvm_asm.h b/arch/arm/include/asm/kvm_asm.h
index 8ef0538..36e3856 100644
--- a/arch/arm/include/asm/kvm_asm.h
+++ b/arch/arm/include/asm/kvm_asm.h
@@ -57,6 +57,7 @@
#ifndef __ASSEMBLY__
struct kvm;
struct kvm_vcpu;
+struct kvm_s2_mmu;
extern char __kvm_hyp_init[];
extern char __kvm_hyp_init_end[];
@@ -64,9 +65,9 @@
extern char __kvm_hyp_vector[];
extern void __kvm_flush_vm_context(void);
-extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
-extern void __kvm_tlb_flush_vmid(struct kvm *kvm);
-extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu);
+extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa);
+extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu);
+extern void __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu);
extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index d5423ab..f84a59c 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -53,9 +53,21 @@
int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
void kvm_reset_coprocs(struct kvm_vcpu *vcpu);
-struct kvm_arch {
- /* VTTBR value associated with below pgd and vmid */
+struct kvm_s2_mmu {
+ /* The VMID generation used for the virt. memory system */
+ u64 vmid_gen;
+ u32 vmid;
+
+ /* Stage-2 page table */
+ pgd_t *pgd;
+
+ /* VTTBR value associated with above pgd and vmid */
u64 vttbr;
+};
+
+struct kvm_arch {
+ /* Stage 2 paging state for the VM */
+ struct kvm_s2_mmu mmu;
/* The last vcpu id that ran on each physical CPU */
int __percpu *last_vcpu_ran;
@@ -68,13 +80,6 @@ struct kvm_arch {
* here.
*/
- /* The VMID generation used for the virt. memory system */
- u64 vmid_gen;
- u32 vmid;
-
- /* Stage-2 page table */
- pgd_t *pgd;
-
/* Interrupt controller */
struct vgic_dist vgic;
int max_vcpus;
@@ -188,6 +193,9 @@ struct kvm_vcpu_arch {
/* Detect first run of a vcpu */
bool has_run_once;
+
+ /* Stage 2 paging state used by the hardware on next switch */
+ struct kvm_s2_mmu *hw_mmu;
};
struct kvm_vm_stat {
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 436bf5a..eb3e709 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -139,7 +139,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
kvm_timer_init(kvm);
/* Mark the initial VMID generation invalid */
- kvm->arch.vmid_gen = 0;
+ kvm->arch.mmu.vmid_gen = 0;
/* The maximum number of VCPUs is limited by the host's GIC model */
kvm->arch.max_vcpus = vgic_present ?
@@ -321,6 +321,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
kvm_arm_reset_debug_ptr(vcpu);
+ vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
+
return 0;
}
@@ -335,7 +337,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
* over-invalidation doesn't affect correctness.
*/
if (*last_ran != vcpu->vcpu_id) {
- kvm_call_hyp(__kvm_tlb_flush_local_vmid, vcpu);
+ kvm_call_hyp(__kvm_tlb_flush_local_vmid, &vcpu->kvm->arch.mmu);
*last_ran = vcpu->vcpu_id;
}
@@ -423,25 +425,26 @@ void force_vm_exit(const cpumask_t *mask)
* VMID for the new generation, we must flush necessary caches and TLBs on all
* CPUs.
*/
-static bool need_new_vmid_gen(struct kvm *kvm)
+static bool need_new_vmid_gen(struct kvm_s2_mmu *mmu)
{
- return unlikely(kvm->arch.vmid_gen != atomic64_read(&kvm_vmid_gen));
+ return unlikely(mmu->vmid_gen != atomic64_read(&kvm_vmid_gen));
}
/**
* update_vttbr - Update the VTTBR with a valid VMID before the guest runs
- * @kvm The guest that we are about to run
+ * @kvm: The guest that we are about to run
+ * @mmu: The stage-2 translation context to update
*
* Called from kvm_arch_vcpu_ioctl_run before entering the guest to ensure the
* VM has a valid VMID, otherwise assigns a new one and flushes corresponding
* caches and TLBs.
*/
-static void update_vttbr(struct kvm *kvm)
+static void update_vttbr(struct kvm *kvm, struct kvm_s2_mmu *mmu)
{
phys_addr_t pgd_phys;
u64 vmid;
- if (!need_new_vmid_gen(kvm))
+ if (!need_new_vmid_gen(mmu))
return;
spin_lock(&kvm_vmid_lock);
@@ -451,7 +454,7 @@ static void update_vttbr(struct kvm *kvm)
* already allocated a valid vmid for this vm, then this vcpu should
* use the same vmid.
*/
- if (!need_new_vmid_gen(kvm)) {
+ if (!need_new_vmid_gen(mmu)) {
spin_unlock(&kvm_vmid_lock);
return;
}
@@ -475,16 +478,17 @@ static void update_vttbr(struct kvm *kvm)
kvm_call_hyp(__kvm_flush_vm_context);
}
- kvm->arch.vmid_gen = atomic64_read(&kvm_vmid_gen);
- kvm->arch.vmid = kvm_next_vmid;
+ mmu->vmid_gen = atomic64_read(&kvm_vmid_gen);
+ mmu->vmid = kvm_next_vmid;
kvm_next_vmid++;
kvm_next_vmid &= (1 << kvm_vmid_bits) - 1;
/* update vttbr to be used with the new vmid */
- pgd_phys = virt_to_phys(kvm->arch.pgd);
+ pgd_phys = virt_to_phys(mmu->pgd);
BUG_ON(pgd_phys & ~VTTBR_BADDR_MASK);
- vmid = ((u64)(kvm->arch.vmid) << VTTBR_VMID_SHIFT) & VTTBR_VMID_MASK(kvm_vmid_bits);
- kvm->arch.vttbr = pgd_phys | vmid;
+ vmid = ((u64)(mmu->vmid) << VTTBR_VMID_SHIFT) &
+ VTTBR_VMID_MASK(kvm_vmid_bits);
+ mmu->vttbr = pgd_phys | vmid;
spin_unlock(&kvm_vmid_lock);
}
@@ -611,7 +615,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
*/
cond_resched();
- update_vttbr(vcpu->kvm);
+ update_vttbr(vcpu->kvm, vcpu->arch.hw_mmu);
if (vcpu->arch.power_off || vcpu->arch.pause)
vcpu_sleep(vcpu);
@@ -636,7 +640,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
run->exit_reason = KVM_EXIT_INTR;
}
- if (ret <= 0 || need_new_vmid_gen(vcpu->kvm) ||
+ if (ret <= 0 || need_new_vmid_gen(vcpu->arch.hw_mmu) ||
vcpu->arch.power_off || vcpu->arch.pause) {
local_irq_enable();
kvm_pmu_sync_hwstate(vcpu);
diff --git a/arch/arm/kvm/hyp/switch.c b/arch/arm/kvm/hyp/switch.c
index 92678b7..6f99de1 100644
--- a/arch/arm/kvm/hyp/switch.c
+++ b/arch/arm/kvm/hyp/switch.c
@@ -73,8 +73,9 @@ static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
{
- struct kvm *kvm = kern_hyp_va(vcpu->kvm);
- write_sysreg(kvm->arch.vttbr, VTTBR);
+ struct kvm_s2_mmu *mmu = kern_hyp_va(vcpu->arch.hw_mmu);
+
+ write_sysreg(mmu->vttbr, VTTBR);
write_sysreg(vcpu->arch.midr, VPIDR);
}
diff --git a/arch/arm/kvm/hyp/tlb.c b/arch/arm/kvm/hyp/tlb.c
index 6d810af..56f0a49 100644
--- a/arch/arm/kvm/hyp/tlb.c
+++ b/arch/arm/kvm/hyp/tlb.c
@@ -34,13 +34,13 @@
* As v7 does not support flushing per IPA, just nuke the whole TLB
* instead, ignoring the ipa value.
*/
-void __hyp_text __kvm_tlb_flush_vmid(struct kvm *kvm)
+void __hyp_text __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu)
{
dsb(ishst);
/* Switch to requested VMID */
- kvm = kern_hyp_va(kvm);
- write_sysreg(kvm->arch.vttbr, VTTBR);
+ mmu = kern_hyp_va(mmu);
+ write_sysreg(mmu->vttbr, VTTBR);
isb();
write_sysreg(0, TLBIALLIS);
@@ -50,17 +50,17 @@ void __hyp_text __kvm_tlb_flush_vmid(struct kvm *kvm)
write_sysreg(0, VTTBR);
}
-void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
+void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
+ phys_addr_t ipa)
{
- __kvm_tlb_flush_vmid(kvm);
+ __kvm_tlb_flush_vmid(mmu);
}
-void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu)
+void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu)
{
- struct kvm *kvm = kern_hyp_va(kern_hyp_va(vcpu)->kvm);
-
/* Switch to requested VMID */
- write_sysreg(kvm->arch.vttbr, VTTBR);
+ mmu = kern_hyp_va(mmu);
+ write_sysreg(mmu->vttbr, VTTBR);
isb();
write_sysreg(0, TLBIALL);
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 57cb671..a27a204 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -63,9 +63,9 @@ void kvm_flush_remote_tlbs(struct kvm *kvm)
kvm_call_hyp(__kvm_tlb_flush_vmid, kvm);
}
-static void kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
+static void kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa)
{
- kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, kvm, ipa);
+ kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ipa);
}
/*
@@ -102,13 +102,14 @@ static bool kvm_is_device_pfn(unsigned long pfn)
* Function clears a PMD entry, flushes addr 1st and 2nd stage TLBs. Marks all
* pages in the range dirty.
*/
-static void stage2_dissolve_pmd(struct kvm *kvm, phys_addr_t addr, pmd_t *pmd)
+static void stage2_dissolve_pmd(struct kvm_s2_mmu *mmu, phys_addr_t addr,
+ pmd_t *pmd)
{
if (!pmd_thp_or_huge(*pmd))
return;
pmd_clear(pmd);
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
put_page(virt_to_page(pmd));
}
@@ -144,31 +145,34 @@ static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
return p;
}
-static void clear_stage2_pgd_entry(struct kvm *kvm, pgd_t *pgd, phys_addr_t addr)
+static void clear_stage2_pgd_entry(struct kvm_s2_mmu *mmu,
+ pgd_t *pgd, phys_addr_t addr)
{
pud_t *pud_table __maybe_unused = stage2_pud_offset(pgd, 0UL);
stage2_pgd_clear(pgd);
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
stage2_pud_free(pud_table);
put_page(virt_to_page(pgd));
}
-static void clear_stage2_pud_entry(struct kvm *kvm, pud_t *pud, phys_addr_t addr)
+static void clear_stage2_pud_entry(struct kvm_s2_mmu *mmu,
+ pud_t *pud, phys_addr_t addr)
{
pmd_t *pmd_table __maybe_unused = stage2_pmd_offset(pud, 0);
VM_BUG_ON(stage2_pud_huge(*pud));
stage2_pud_clear(pud);
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
stage2_pmd_free(pmd_table);
put_page(virt_to_page(pud));
}
-static void clear_stage2_pmd_entry(struct kvm *kvm, pmd_t *pmd, phys_addr_t addr)
+static void clear_stage2_pmd_entry(struct kvm_s2_mmu *mmu,
+ pmd_t *pmd, phys_addr_t addr)
{
pte_t *pte_table = pte_offset_kernel(pmd, 0);
VM_BUG_ON(pmd_thp_or_huge(*pmd));
pmd_clear(pmd);
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
pte_free_kernel(NULL, pte_table);
put_page(virt_to_page(pmd));
}
@@ -193,7 +197,7 @@ static void clear_stage2_pmd_entry(struct kvm *kvm, pmd_t *pmd, phys_addr_t addr
* the corresponding TLBs, we call kvm_flush_dcache_p*() to make sure
* the IO subsystem will never hit in the cache.
*/
-static void unmap_stage2_ptes(struct kvm *kvm, pmd_t *pmd,
+static void unmap_stage2_ptes(struct kvm_s2_mmu *mmu, pmd_t *pmd,
phys_addr_t addr, phys_addr_t end)
{
phys_addr_t start_addr = addr;
@@ -205,7 +209,7 @@ static void unmap_stage2_ptes(struct kvm *kvm, pmd_t *pmd,
pte_t old_pte = *pte;
kvm_set_pte(pte, __pte(0));
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
/* No need to invalidate the cache for device mappings */
if (!kvm_is_device_pfn(pte_pfn(old_pte)))
@@ -216,10 +220,10 @@ static void unmap_stage2_ptes(struct kvm *kvm, pmd_t *pmd,
} while (pte++, addr += PAGE_SIZE, addr != end);
if (stage2_pte_table_empty(start_pte))
- clear_stage2_pmd_entry(kvm, pmd, start_addr);
+ clear_stage2_pmd_entry(mmu, pmd, start_addr);
}
-static void unmap_stage2_pmds(struct kvm *kvm, pud_t *pud,
+static void unmap_stage2_pmds(struct kvm_s2_mmu *mmu, pud_t *pud,
phys_addr_t addr, phys_addr_t end)
{
phys_addr_t next, start_addr = addr;
@@ -233,22 +237,22 @@ static void unmap_stage2_pmds(struct kvm *kvm, pud_t *pud,
pmd_t old_pmd = *pmd;
pmd_clear(pmd);
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
kvm_flush_dcache_pmd(old_pmd);
put_page(virt_to_page(pmd));
} else {
- unmap_stage2_ptes(kvm, pmd, addr, next);
+ unmap_stage2_ptes(mmu, pmd, addr, next);
}
}
} while (pmd++, addr = next, addr != end);
if (stage2_pmd_table_empty(start_pmd))
- clear_stage2_pud_entry(kvm, pud, start_addr);
+ clear_stage2_pud_entry(mmu, pud, start_addr);
}
-static void unmap_stage2_puds(struct kvm *kvm, pgd_t *pgd,
+static void unmap_stage2_puds(struct kvm_s2_mmu *mmu, pgd_t *pgd,
phys_addr_t addr, phys_addr_t end)
{
phys_addr_t next, start_addr = addr;
@@ -262,17 +266,17 @@ static void unmap_stage2_puds(struct kvm *kvm, pgd_t *pgd,
pud_t old_pud = *pud;
stage2_pud_clear(pud);
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
kvm_flush_dcache_pud(old_pud);
put_page(virt_to_page(pud));
} else {
- unmap_stage2_pmds(kvm, pud, addr, next);
+ unmap_stage2_pmds(mmu, pud, addr, next);
}
}
} while (pud++, addr = next, addr != end);
if (stage2_pud_table_empty(start_pud))
- clear_stage2_pgd_entry(kvm, pgd, start_addr);
+ clear_stage2_pgd_entry(mmu, pgd, start_addr);
}
/**
@@ -286,17 +290,18 @@ static void unmap_stage2_puds(struct kvm *kvm, pgd_t *pgd,
* destroying the VM), otherwise another faulting VCPU may come in and mess
* with things behind our backs.
*/
-static void unmap_stage2_range(struct kvm *kvm, phys_addr_t start, u64 size)
+static void unmap_stage2_range(struct kvm_s2_mmu *mmu,
+ phys_addr_t start, u64 size)
{
pgd_t *pgd;
phys_addr_t addr = start, end = start + size;
phys_addr_t next;
- pgd = kvm->arch.pgd + stage2_pgd_index(addr);
+ pgd = mmu->pgd + stage2_pgd_index(addr);
do {
next = stage2_pgd_addr_end(addr, end);
if (!stage2_pgd_none(*pgd))
- unmap_stage2_puds(kvm, pgd, addr, next);
+ unmap_stage2_puds(mmu, pgd, addr, next);
} while (pgd++, addr = next, addr != end);
}
@@ -348,7 +353,7 @@ static void stage2_flush_puds(pgd_t *pgd,
} while (pud++, addr = next, addr != end);
}
-static void stage2_flush_memslot(struct kvm *kvm,
+static void stage2_flush_memslot(struct kvm_s2_mmu *mmu,
struct kvm_memory_slot *memslot)
{
phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
@@ -356,7 +361,7 @@ static void stage2_flush_memslot(struct kvm *kvm,
phys_addr_t next;
pgd_t *pgd;
- pgd = kvm->arch.pgd + stage2_pgd_index(addr);
+ pgd = mmu->pgd + stage2_pgd_index(addr);
do {
next = stage2_pgd_addr_end(addr, end);
stage2_flush_puds(pgd, addr, next);
@@ -381,7 +386,7 @@ static void stage2_flush_vm(struct kvm *kvm)
slots = kvm_memslots(kvm);
kvm_for_each_memslot(memslot, slots)
- stage2_flush_memslot(kvm, memslot);
+ stage2_flush_memslot(&kvm->arch.mmu, memslot);
spin_unlock(&kvm->mmu_lock);
srcu_read_unlock(&kvm->srcu, idx);
@@ -733,8 +738,9 @@ int create_hyp_io_mappings(void *from, void *to, phys_addr_t phys_addr)
int kvm_alloc_stage2_pgd(struct kvm *kvm)
{
pgd_t *pgd;
+ struct kvm_s2_mmu *mmu = &kvm->arch.mmu;
- if (kvm->arch.pgd != NULL) {
+ if (mmu->pgd != NULL) {
kvm_err("kvm_arch already initialized?\n");
return -EINVAL;
}
@@ -744,11 +750,12 @@ int kvm_alloc_stage2_pgd(struct kvm *kvm)
if (!pgd)
return -ENOMEM;
- kvm->arch.pgd = pgd;
+ mmu->pgd = pgd;
+
return 0;
}
-static void stage2_unmap_memslot(struct kvm *kvm,
+static void stage2_unmap_memslot(struct kvm_s2_mmu *mmu,
struct kvm_memory_slot *memslot)
{
hva_t hva = memslot->userspace_addr;
@@ -783,7 +790,7 @@ static void stage2_unmap_memslot(struct kvm *kvm,
if (!(vma->vm_flags & VM_PFNMAP)) {
gpa_t gpa = addr + (vm_start - memslot->userspace_addr);
- unmap_stage2_range(kvm, gpa, vm_end - vm_start);
+ unmap_stage2_range(mmu, gpa, vm_end - vm_start);
}
hva = vm_end;
} while (hva < reg_end);
@@ -807,7 +814,7 @@ void stage2_unmap_vm(struct kvm *kvm)
slots = kvm_memslots(kvm);
kvm_for_each_memslot(memslot, slots)
- stage2_unmap_memslot(kvm, memslot);
+ stage2_unmap_memslot(&kvm->arch.mmu, memslot);
spin_unlock(&kvm->mmu_lock);
srcu_read_unlock(&kvm->srcu, idx);
@@ -826,22 +833,25 @@ void stage2_unmap_vm(struct kvm *kvm)
*/
void kvm_free_stage2_pgd(struct kvm *kvm)
{
- if (kvm->arch.pgd == NULL)
+ struct kvm_s2_mmu *mmu = &kvm->arch.mmu;
+
+ if (mmu->pgd == NULL)
return;
- unmap_stage2_range(kvm, 0, KVM_PHYS_SIZE);
+ unmap_stage2_range(mmu, 0, KVM_PHYS_SIZE);
/* Free the HW pgd, one page at a time */
- free_pages_exact(kvm->arch.pgd, S2_PGD_SIZE);
- kvm->arch.pgd = NULL;
+ free_pages_exact(mmu->pgd, S2_PGD_SIZE);
+ mmu->pgd = NULL;
}
-static pud_t *stage2_get_pud(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
+static pud_t *stage2_get_pud(struct kvm_s2_mmu *mmu,
+ struct kvm_mmu_memory_cache *cache,
phys_addr_t addr)
{
pgd_t *pgd;
pud_t *pud;
- pgd = kvm->arch.pgd + stage2_pgd_index(addr);
+ pgd = mmu->pgd + stage2_pgd_index(addr);
if (WARN_ON(stage2_pgd_none(*pgd))) {
if (!cache)
return NULL;
@@ -853,13 +863,14 @@ static pud_t *stage2_get_pud(struct kvm *kvm, struct kvm_mmu_memory_cache *cache
return stage2_pud_offset(pgd, addr);
}
-static pmd_t *stage2_get_pmd(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
+static pmd_t *stage2_get_pmd(struct kvm_s2_mmu *mmu,
+ struct kvm_mmu_memory_cache *cache,
phys_addr_t addr)
{
pud_t *pud;
pmd_t *pmd;
- pud = stage2_get_pud(kvm, cache, addr);
+ pud = stage2_get_pud(mmu, cache, addr);
if (stage2_pud_none(*pud)) {
if (!cache)
return NULL;
@@ -871,12 +882,13 @@ static pmd_t *stage2_get_pmd(struct kvm *kvm, struct kvm_mmu_memory_cache *cache
return stage2_pmd_offset(pud, addr);
}
-static int stage2_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache
+static int stage2_set_pmd_huge(struct kvm_s2_mmu *mmu,
+ struct kvm_mmu_memory_cache
*cache, phys_addr_t addr, const pmd_t *new_pmd)
{
pmd_t *pmd, old_pmd;
- pmd = stage2_get_pmd(kvm, cache, addr);
+ pmd = stage2_get_pmd(mmu, cache, addr);
VM_BUG_ON(!pmd);
/*
@@ -893,7 +905,7 @@ static int stage2_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache
old_pmd = *pmd;
if (pmd_present(old_pmd)) {
pmd_clear(pmd);
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
} else {
get_page(virt_to_page(pmd));
}
@@ -902,7 +914,8 @@ static int stage2_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache
return 0;
}
-static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
+static int stage2_set_pte(struct kvm_s2_mmu *mmu,
+ struct kvm_mmu_memory_cache *cache,
phys_addr_t addr, const pte_t *new_pte,
unsigned long flags)
{
@@ -914,7 +927,7 @@ static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
VM_BUG_ON(logging_active && !cache);
/* Create stage-2 page table mapping - Levels 0 and 1 */
- pmd = stage2_get_pmd(kvm, cache, addr);
+ pmd = stage2_get_pmd(mmu, cache, addr);
if (!pmd) {
/*
* Ignore calls from kvm_set_spte_hva for unallocated
@@ -928,7 +941,7 @@ static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
* allocate page.
*/
if (logging_active)
- stage2_dissolve_pmd(kvm, addr, pmd);
+ stage2_dissolve_pmd(mmu, addr, pmd);
/* Create stage-2 page mappings - Level 2 */
if (pmd_none(*pmd)) {
@@ -948,7 +961,7 @@ static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
old_pte = *pte;
if (pte_present(old_pte)) {
kvm_set_pte(pte, __pte(0));
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
} else {
get_page(virt_to_page(pte));
}
@@ -1008,7 +1021,7 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
if (ret)
goto out;
spin_lock(&kvm->mmu_lock);
- ret = stage2_set_pte(kvm, &cache, addr, &pte,
+ ret = stage2_set_pte(&kvm->arch.mmu, &cache, addr, &pte,
KVM_S2PTE_FLAG_IS_IOMAP);
spin_unlock(&kvm->mmu_lock);
if (ret)
@@ -1146,12 +1159,13 @@ static void stage2_wp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end)
* @addr: Start address of range
* @end: End address of range
*/
-static void stage2_wp_range(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)
+static void stage2_wp_range(struct kvm *kvm, struct kvm_s2_mmu *mmu,
+ phys_addr_t addr, phys_addr_t end)
{
pgd_t *pgd;
phys_addr_t next;
- pgd = kvm->arch.pgd + stage2_pgd_index(addr);
+ pgd = mmu->pgd + stage2_pgd_index(addr);
do {
/*
* Release kvm_mmu_lock periodically if the memory region is
@@ -1190,7 +1204,7 @@ void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot)
phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;
spin_lock(&kvm->mmu_lock);
- stage2_wp_range(kvm, start, end);
+ stage2_wp_range(kvm, &kvm->arch.mmu, start, end);
spin_unlock(&kvm->mmu_lock);
kvm_flush_remote_tlbs(kvm);
}
@@ -1214,7 +1228,7 @@ static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT;
phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT;
- stage2_wp_range(kvm, start, end);
+ stage2_wp_range(kvm, &kvm->arch.mmu, start, end);
}
/*
@@ -1253,6 +1267,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
bool fault_ipa_uncached;
bool logging_active = memslot_is_logging(memslot);
unsigned long flags = 0;
+ struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu;
write_fault = kvm_is_write_fault(vcpu);
if (fault_status == FSC_PERM && !write_fault) {
@@ -1347,7 +1362,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
kvm_set_pfn_dirty(pfn);
}
coherent_cache_guest_page(vcpu, pfn, PMD_SIZE, fault_ipa_uncached);
- ret = stage2_set_pmd_huge(kvm, memcache, fault_ipa, &new_pmd);
+ ret = stage2_set_pmd_huge(mmu, memcache, fault_ipa, &new_pmd);
} else {
pte_t new_pte = pfn_pte(pfn, mem_type);
@@ -1357,7 +1372,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
mark_page_dirty(kvm, gfn);
}
coherent_cache_guest_page(vcpu, pfn, PAGE_SIZE, fault_ipa_uncached);
- ret = stage2_set_pte(kvm, memcache, fault_ipa, &new_pte, flags);
+ ret = stage2_set_pte(mmu, memcache, fault_ipa, &new_pte, flags);
}
out_unlock:
@@ -1385,7 +1400,7 @@ static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
spin_lock(&vcpu->kvm->mmu_lock);
- pmd = stage2_get_pmd(vcpu->kvm, NULL, fault_ipa);
+ pmd = stage2_get_pmd(vcpu->arch.hw_mmu, NULL, fault_ipa);
if (!pmd || pmd_none(*pmd)) /* Nothing there */
goto out;
@@ -1553,7 +1568,7 @@ static int handle_hva_to_gpa(struct kvm *kvm,
static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
{
- unmap_stage2_range(kvm, gpa, PAGE_SIZE);
+ unmap_stage2_range(&kvm->arch.mmu, gpa, PAGE_SIZE);
return 0;
}
@@ -1561,7 +1576,7 @@ int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
{
unsigned long end = hva + PAGE_SIZE;
- if (!kvm->arch.pgd)
+ if (!kvm->arch.mmu.pgd)
return 0;
trace_kvm_unmap_hva(hva);
@@ -1572,7 +1587,7 @@ int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
int kvm_unmap_hva_range(struct kvm *kvm,
unsigned long start, unsigned long end)
{
- if (!kvm->arch.pgd)
+ if (!kvm->arch.mmu.pgd)
return 0;
trace_kvm_unmap_hva_range(start, end);
@@ -1591,7 +1606,7 @@ static int kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
* therefore stage2_set_pte() never needs to clear out a huge PMD
* through this calling path.
*/
- stage2_set_pte(kvm, NULL, gpa, pte, 0);
+ stage2_set_pte(&kvm->arch.mmu, NULL, gpa, pte, 0);
return 0;
}
@@ -1601,7 +1616,7 @@ void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
unsigned long end = hva + PAGE_SIZE;
pte_t stage2_pte;
- if (!kvm->arch.pgd)
+ if (!kvm->arch.mmu.pgd)
return;
trace_kvm_set_spte_hva(hva);
@@ -1614,7 +1629,7 @@ static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
pmd_t *pmd;
pte_t *pte;
- pmd = stage2_get_pmd(kvm, NULL, gpa);
+ pmd = stage2_get_pmd(&kvm->arch.mmu, NULL, gpa);
if (!pmd || pmd_none(*pmd)) /* Nothing there */
return 0;
@@ -1633,7 +1648,7 @@ static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
pmd_t *pmd;
pte_t *pte;
- pmd = stage2_get_pmd(kvm, NULL, gpa);
+ pmd = stage2_get_pmd(&kvm->arch.mmu, NULL, gpa);
if (!pmd || pmd_none(*pmd)) /* Nothing there */
return 0;
@@ -1864,9 +1879,10 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
spin_lock(&kvm->mmu_lock);
if (ret)
- unmap_stage2_range(kvm, mem->guest_phys_addr, mem->memory_size);
+ unmap_stage2_range(&kvm->arch.mmu, mem->guest_phys_addr,
+ mem->memory_size);
else
- stage2_flush_memslot(kvm, memslot);
+ stage2_flush_memslot(&kvm->arch.mmu, memslot);
spin_unlock(&kvm->mmu_lock);
return ret;
}
@@ -1907,7 +1923,7 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
phys_addr_t size = slot->npages << PAGE_SHIFT;
spin_lock(&kvm->mmu_lock);
- unmap_stage2_range(kvm, gpa, size);
+ unmap_stage2_range(&kvm->arch.mmu, gpa, size);
spin_unlock(&kvm->mmu_lock);
}
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index ec3553eb..ed8139f 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -44,6 +44,7 @@
#ifndef __ASSEMBLY__
struct kvm;
struct kvm_vcpu;
+struct kvm_s2_mmu;
extern char __kvm_hyp_init[];
extern char __kvm_hyp_init_end[];
@@ -52,9 +53,9 @@
extern char __kvm_hyp_vector[];
extern void __kvm_flush_vm_context(void);
-extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
-extern void __kvm_tlb_flush_vmid(struct kvm *kvm);
-extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu);
+extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa);
+extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu);
+extern void __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu);
extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index ed78d73..954d6de 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -50,7 +50,7 @@
int kvm_arch_dev_ioctl_check_extension(struct kvm *kvm, long ext);
void __extended_idmap_trampoline(phys_addr_t boot_pgd, phys_addr_t idmap_start);
-struct kvm_arch {
+struct kvm_s2_mmu {
/* The VMID generation used for the virt. memory system */
u64 vmid_gen;
u32 vmid;
@@ -61,6 +61,11 @@ struct kvm_arch {
/* VTTBR value associated with above pgd and vmid */
u64 vttbr;
+};
+
+struct kvm_arch {
+ /* Stage 2 paging state for the VM */
+ struct kvm_s2_mmu mmu;
/* The last vcpu id that ran on each physical CPU */
int __percpu *last_vcpu_ran;
@@ -326,6 +331,9 @@ struct kvm_vcpu_arch {
/* Detect first run of a vcpu */
bool has_run_once;
+
+ /* Stage 2 paging state used by the hardware on next switch */
+ struct kvm_s2_mmu *hw_mmu;
};
#define vcpu_gp_regs(v) (&(v)->arch.ctxt.gp_regs)
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index b7c8c30..3207009a 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -135,8 +135,9 @@ static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
{
- struct kvm *kvm = kern_hyp_va(vcpu->kvm);
- write_sysreg(kvm->arch.vttbr, vttbr_el2);
+ struct kvm_s2_mmu *mmu = kern_hyp_va(vcpu->arch.hw_mmu);
+
+ write_sysreg(mmu->vttbr, vttbr_el2);
}
static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)
diff --git a/arch/arm64/kvm/hyp/tlb.c b/arch/arm64/kvm/hyp/tlb.c
index 88e2f2b..71a62ea 100644
--- a/arch/arm64/kvm/hyp/tlb.c
+++ b/arch/arm64/kvm/hyp/tlb.c
@@ -17,13 +17,14 @@
#include <asm/kvm_hyp.h>
-void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
+void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
+ phys_addr_t ipa)
{
dsb(ishst);
/* Switch to requested VMID */
- kvm = kern_hyp_va(kvm);
- write_sysreg(kvm->arch.vttbr, vttbr_el2);
+ mmu = kern_hyp_va(mmu);
+ write_sysreg(mmu->vttbr, vttbr_el2);
isb();
/*
@@ -48,13 +49,13 @@ void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
write_sysreg(0, vttbr_el2);
}
-void __hyp_text __kvm_tlb_flush_vmid(struct kvm *kvm)
+void __hyp_text __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu)
{
dsb(ishst);
/* Switch to requested VMID */
- kvm = kern_hyp_va(kvm);
- write_sysreg(kvm->arch.vttbr, vttbr_el2);
+ mmu = kern_hyp_va(mmu);
+ write_sysreg(mmu->vttbr, vttbr_el2);
isb();
asm volatile("tlbi vmalls12e1is" : : );
@@ -64,12 +65,11 @@ void __hyp_text __kvm_tlb_flush_vmid(struct kvm *kvm)
write_sysreg(0, vttbr_el2);
}
-void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu)
+void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu)
{
- struct kvm *kvm = kern_hyp_va(kern_hyp_va(vcpu)->kvm);
-
/* Switch to requested VMID */
- write_sysreg(kvm->arch.vttbr, vttbr_el2);
+ mmu = kern_hyp_va(mmu);
+ write_sysreg(mmu->vttbr, vttbr_el2);
isb();
asm volatile("tlbi vmalle1" : : );
--
1.9.1
^ permalink raw reply related
* [RFC 35/55] KVM: arm/arm64: Support mmu for the virtual EL2 execution
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
When running a guest hypervisor in virtual EL2, the translation context
has to be separate from the rest of the system, including the guest
EL1/0 translation regime, so we allocate a separate VMID for this mode.
Considering that we have two different vttbr values due to separate
VMIDs, it's racy to keep a vttbr value in a struct (kvm_s2_mmu) and
share it between multiple vcpus. So, keep the vttbr value per vcpu.
Hypercalls to flush tlb now have vttbr as a parameter instead of mmu,
since mmu structure does not have vttbr any more.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/include/asm/kvm_asm.h | 6 ++--
arch/arm/include/asm/kvm_emulate.h | 4 +++
arch/arm/include/asm/kvm_host.h | 14 ++++++---
arch/arm/include/asm/kvm_mmu.h | 11 +++++++
arch/arm/kvm/arm.c | 60 +++++++++++++++++++-----------------
arch/arm/kvm/hyp/switch.c | 4 +--
arch/arm/kvm/hyp/tlb.c | 15 ++++-----
arch/arm/kvm/mmu.c | 9 ++++--
arch/arm64/include/asm/kvm_asm.h | 6 ++--
arch/arm64/include/asm/kvm_emulate.h | 8 +++++
arch/arm64/include/asm/kvm_host.h | 14 ++++++---
arch/arm64/include/asm/kvm_mmu.h | 11 +++++++
arch/arm64/kvm/hyp/switch.c | 4 +--
arch/arm64/kvm/hyp/tlb.c | 16 ++++------
14 files changed, 112 insertions(+), 70 deletions(-)
diff --git a/arch/arm/include/asm/kvm_asm.h b/arch/arm/include/asm/kvm_asm.h
index 36e3856..aa214f7 100644
--- a/arch/arm/include/asm/kvm_asm.h
+++ b/arch/arm/include/asm/kvm_asm.h
@@ -65,9 +65,9 @@
extern char __kvm_hyp_vector[];
extern void __kvm_flush_vm_context(void);
-extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa);
-extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu);
-extern void __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu);
+extern void __kvm_tlb_flush_vmid_ipa(u64 vttbr, phys_addr_t ipa);
+extern void __kvm_tlb_flush_vmid(u64 vttbr);
+extern void __kvm_tlb_flush_local_vmid(u64 vttbr);
extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h
index 05d5906..6285f4f 100644
--- a/arch/arm/include/asm/kvm_emulate.h
+++ b/arch/arm/include/asm/kvm_emulate.h
@@ -305,4 +305,8 @@ static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu,
}
}
+static inline struct kvm_s2_vmid *vcpu_get_active_vmid(struct kvm_vcpu *vcpu)
+{
+ return &vcpu->kvm->arch.mmu.vmid;
+}
#endif /* __ARM_KVM_EMULATE_H__ */
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index f84a59c..da45394 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -53,16 +53,18 @@
int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
void kvm_reset_coprocs(struct kvm_vcpu *vcpu);
-struct kvm_s2_mmu {
+struct kvm_s2_vmid {
/* The VMID generation used for the virt. memory system */
u64 vmid_gen;
u32 vmid;
+};
+
+struct kvm_s2_mmu {
+ struct kvm_s2_vmid vmid;
+ struct kvm_s2_vmid el2_vmid;
/* Stage-2 page table */
pgd_t *pgd;
-
- /* VTTBR value associated with above pgd and vmid */
- u64 vttbr;
};
struct kvm_arch {
@@ -196,6 +198,9 @@ struct kvm_vcpu_arch {
/* Stage 2 paging state used by the hardware on next switch */
struct kvm_s2_mmu *hw_mmu;
+
+ /* VTTBR value used by the hardware on next switch */
+ u64 hw_vttbr;
};
struct kvm_vm_stat {
@@ -242,6 +247,7 @@ static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
{
}
+unsigned int get_kvm_vmid_bits(void);
struct kvm_vcpu *kvm_arm_get_running_vcpu(void);
struct kvm_vcpu __percpu **kvm_get_running_vcpus(void);
void kvm_arm_halt_guest(struct kvm *kvm);
diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
index 74a44727..1b3309c 100644
--- a/arch/arm/include/asm/kvm_mmu.h
+++ b/arch/arm/include/asm/kvm_mmu.h
@@ -230,6 +230,17 @@ static inline unsigned int kvm_get_vmid_bits(void)
return 8;
}
+static inline u64 kvm_get_vttbr(struct kvm_s2_vmid *vmid,
+ struct kvm_s2_mmu *mmu)
+{
+ u64 vmid_field, baddr;
+
+ baddr = virt_to_phys(mmu->pgd);
+ vmid_field = ((u64)vmid->vmid << VTTBR_VMID_SHIFT) &
+ VTTBR_VMID_MASK(get_kvm_vmid_bits());
+ return baddr | vmid_field;
+}
+
#endif /* !__ASSEMBLY__ */
#endif /* __ARM_KVM_MMU_H__ */
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index eb3e709..aa8771d 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -75,6 +75,11 @@ static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu)
__this_cpu_write(kvm_arm_running_vcpu, vcpu);
}
+unsigned int get_kvm_vmid_bits(void)
+{
+ return kvm_vmid_bits;
+}
+
/**
* kvm_arm_get_running_vcpu - get the vcpu running on the current CPU.
* Must be called from non-preemptible context
@@ -139,7 +144,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
kvm_timer_init(kvm);
/* Mark the initial VMID generation invalid */
- kvm->arch.mmu.vmid_gen = 0;
+ kvm->arch.mmu.vmid.vmid_gen = 0;
+ kvm->arch.mmu.el2_vmid.vmid_gen = 0;
/* The maximum number of VCPUs is limited by the host's GIC model */
kvm->arch.max_vcpus = vgic_present ?
@@ -312,6 +318,8 @@ void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
{
+ struct kvm_s2_mmu *mmu = &vcpu->kvm->arch.mmu;
+
/* Force users to call KVM_ARM_VCPU_INIT */
vcpu->arch.target = -1;
bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
@@ -321,7 +329,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
kvm_arm_reset_debug_ptr(vcpu);
- vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
+ vcpu->arch.hw_mmu = mmu;
+ vcpu->arch.hw_vttbr = kvm_get_vttbr(&mmu->vmid, mmu);
return 0;
}
@@ -337,7 +346,10 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
* over-invalidation doesn't affect correctness.
*/
if (*last_ran != vcpu->vcpu_id) {
- kvm_call_hyp(__kvm_tlb_flush_local_vmid, &vcpu->kvm->arch.mmu);
+ struct kvm_s2_mmu *mmu = &vcpu->kvm->arch.mmu;
+ u64 vttbr = kvm_get_vttbr(&mmu->vmid, mmu);
+
+ kvm_call_hyp(__kvm_tlb_flush_local_vmid, vttbr);
*last_ran = vcpu->vcpu_id;
}
@@ -415,36 +427,33 @@ void force_vm_exit(const cpumask_t *mask)
/**
* need_new_vmid_gen - check that the VMID is still valid
- * @kvm: The VM's VMID to check
+ * @vmid: The VMID to check
*
* return true if there is a new generation of VMIDs being used
*
- * The hardware supports only 256 values with the value zero reserved for the
- * host, so we check if an assigned value belongs to a previous generation,
- * which which requires us to assign a new value. If we're the first to use a
- * VMID for the new generation, we must flush necessary caches and TLBs on all
- * CPUs.
+ * The hardware supports a limited set of values with the value zero reserved
+ * for the host, so we check if an assigned value belongs to a previous
+ * generation, which which requires us to assign a new value. If we're the
+ * first to use a VMID for the new generation, we must flush necessary caches
+ * and TLBs on all CPUs.
*/
-static bool need_new_vmid_gen(struct kvm_s2_mmu *mmu)
+static bool need_new_vmid_gen(struct kvm_s2_vmid *vmid)
{
- return unlikely(mmu->vmid_gen != atomic64_read(&kvm_vmid_gen));
+ return unlikely(vmid->vmid_gen != atomic64_read(&kvm_vmid_gen));
}
/**
* update_vttbr - Update the VTTBR with a valid VMID before the guest runs
* @kvm: The guest that we are about to run
- * @mmu: The stage-2 translation context to update
+ * @vmid: The stage-2 VMID information struct
*
* Called from kvm_arch_vcpu_ioctl_run before entering the guest to ensure the
* VM has a valid VMID, otherwise assigns a new one and flushes corresponding
* caches and TLBs.
*/
-static void update_vttbr(struct kvm *kvm, struct kvm_s2_mmu *mmu)
+static void update_vttbr(struct kvm *kvm, struct kvm_s2_vmid *vmid)
{
- phys_addr_t pgd_phys;
- u64 vmid;
-
- if (!need_new_vmid_gen(mmu))
+ if (!need_new_vmid_gen(vmid))
return;
spin_lock(&kvm_vmid_lock);
@@ -454,7 +463,7 @@ static void update_vttbr(struct kvm *kvm, struct kvm_s2_mmu *mmu)
* already allocated a valid vmid for this vm, then this vcpu should
* use the same vmid.
*/
- if (!need_new_vmid_gen(mmu)) {
+ if (!need_new_vmid_gen(vmid)) {
spin_unlock(&kvm_vmid_lock);
return;
}
@@ -478,18 +487,11 @@ static void update_vttbr(struct kvm *kvm, struct kvm_s2_mmu *mmu)
kvm_call_hyp(__kvm_flush_vm_context);
}
- mmu->vmid_gen = atomic64_read(&kvm_vmid_gen);
- mmu->vmid = kvm_next_vmid;
+ vmid->vmid_gen = atomic64_read(&kvm_vmid_gen);
+ vmid->vmid = kvm_next_vmid;
kvm_next_vmid++;
kvm_next_vmid &= (1 << kvm_vmid_bits) - 1;
- /* update vttbr to be used with the new vmid */
- pgd_phys = virt_to_phys(mmu->pgd);
- BUG_ON(pgd_phys & ~VTTBR_BADDR_MASK);
- vmid = ((u64)(mmu->vmid) << VTTBR_VMID_SHIFT) &
- VTTBR_VMID_MASK(kvm_vmid_bits);
- mmu->vttbr = pgd_phys | vmid;
-
spin_unlock(&kvm_vmid_lock);
}
@@ -615,7 +617,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
*/
cond_resched();
- update_vttbr(vcpu->kvm, vcpu->arch.hw_mmu);
+ update_vttbr(vcpu->kvm, vcpu_get_active_vmid(vcpu));
if (vcpu->arch.power_off || vcpu->arch.pause)
vcpu_sleep(vcpu);
@@ -640,7 +642,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
run->exit_reason = KVM_EXIT_INTR;
}
- if (ret <= 0 || need_new_vmid_gen(vcpu->arch.hw_mmu) ||
+ if (ret <= 0 || need_new_vmid_gen(vcpu_get_active_vmid(vcpu)) ||
vcpu->arch.power_off || vcpu->arch.pause) {
local_irq_enable();
kvm_pmu_sync_hwstate(vcpu);
diff --git a/arch/arm/kvm/hyp/switch.c b/arch/arm/kvm/hyp/switch.c
index 6f99de1..65d0b5b 100644
--- a/arch/arm/kvm/hyp/switch.c
+++ b/arch/arm/kvm/hyp/switch.c
@@ -73,9 +73,7 @@ static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
{
- struct kvm_s2_mmu *mmu = kern_hyp_va(vcpu->arch.hw_mmu);
-
- write_sysreg(mmu->vttbr, VTTBR);
+ write_sysreg(vcpu->arch.hw_vttbr, VTTBR);
write_sysreg(vcpu->arch.midr, VPIDR);
}
diff --git a/arch/arm/kvm/hyp/tlb.c b/arch/arm/kvm/hyp/tlb.c
index 56f0a49..562ad0b 100644
--- a/arch/arm/kvm/hyp/tlb.c
+++ b/arch/arm/kvm/hyp/tlb.c
@@ -34,13 +34,12 @@
* As v7 does not support flushing per IPA, just nuke the whole TLB
* instead, ignoring the ipa value.
*/
-void __hyp_text __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu)
+void __hyp_text __kvm_tlb_flush_vmid(u64 vttbr)
{
dsb(ishst);
/* Switch to requested VMID */
- mmu = kern_hyp_va(mmu);
- write_sysreg(mmu->vttbr, VTTBR);
+ write_sysreg(vttbr, VTTBR);
isb();
write_sysreg(0, TLBIALLIS);
@@ -50,17 +49,15 @@ void __hyp_text __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu)
write_sysreg(0, VTTBR);
}
-void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
- phys_addr_t ipa)
+void __hyp_text __kvm_tlb_flush_vmid_ipa(u64 vttbr, phys_addr_t ipa)
{
- __kvm_tlb_flush_vmid(mmu);
+ __kvm_tlb_flush_vmid(vttbr);
}
-void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu)
+void __hyp_text __kvm_tlb_flush_local_vmid(u64 vttbr)
{
/* Switch to requested VMID */
- mmu = kern_hyp_va(mmu);
- write_sysreg(mmu->vttbr, VTTBR);
+ write_sysreg(vttbr, VTTBR);
isb();
write_sysreg(0, TLBIALL);
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index a27a204..5ca3a04 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -60,12 +60,17 @@ static bool memslot_is_logging(struct kvm_memory_slot *memslot)
*/
void kvm_flush_remote_tlbs(struct kvm *kvm)
{
- kvm_call_hyp(__kvm_tlb_flush_vmid, kvm);
+ struct kvm_s2_mmu *mmu = &kvm->arch.mmu;
+ u64 vttbr = kvm_get_vttbr(&mmu->vmid, mmu);
+
+ kvm_call_hyp(__kvm_tlb_flush_vmid, vttbr);
}
static void kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa)
{
- kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ipa);
+ u64 vttbr = kvm_get_vttbr(&mmu->vmid, mmu);
+
+ kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, vttbr, ipa);
}
/*
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index ed8139f..27dce47 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -53,9 +53,9 @@
extern char __kvm_hyp_vector[];
extern void __kvm_flush_vm_context(void);
-extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa);
-extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu);
-extern void __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu);
+extern void __kvm_tlb_flush_vmid_ipa(u64 vttbr, phys_addr_t ipa);
+extern void __kvm_tlb_flush_vmid(u64 vttbr);
+extern void __kvm_tlb_flush_local_vmid(u64 vttbr);
extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index a9c993f..94068e7 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -363,4 +363,12 @@ static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu,
return data; /* Leave LE untouched */
}
+static inline struct kvm_s2_vmid *vcpu_get_active_vmid(struct kvm_vcpu *vcpu)
+{
+ if (unlikely(vcpu_mode_el2(vcpu)))
+ return &vcpu->kvm->arch.mmu.el2_vmid;
+
+ return &vcpu->kvm->arch.mmu.vmid;
+}
+
#endif /* __ARM64_KVM_EMULATE_H__ */
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 954d6de..b33d35d 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -50,17 +50,19 @@
int kvm_arch_dev_ioctl_check_extension(struct kvm *kvm, long ext);
void __extended_idmap_trampoline(phys_addr_t boot_pgd, phys_addr_t idmap_start);
-struct kvm_s2_mmu {
+struct kvm_s2_vmid {
/* The VMID generation used for the virt. memory system */
u64 vmid_gen;
u32 vmid;
+};
+
+struct kvm_s2_mmu {
+ struct kvm_s2_vmid vmid;
+ struct kvm_s2_vmid el2_vmid;
/* 1-level 2nd stage table and lock */
spinlock_t pgd_lock;
pgd_t *pgd;
-
- /* VTTBR value associated with above pgd and vmid */
- u64 vttbr;
};
struct kvm_arch {
@@ -334,6 +336,9 @@ struct kvm_vcpu_arch {
/* Stage 2 paging state used by the hardware on next switch */
struct kvm_s2_mmu *hw_mmu;
+
+ /* VTTBR value used by the hardware on next switch */
+ u64 hw_vttbr;
};
#define vcpu_gp_regs(v) (&(v)->arch.ctxt.gp_regs)
@@ -391,6 +396,7 @@ static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
{
}
+unsigned int get_kvm_vmid_bits(void);
struct kvm_vcpu *kvm_arm_get_running_vcpu(void);
struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void);
void kvm_arm_halt_guest(struct kvm *kvm);
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 6f72fe8..e3455c4 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -314,5 +314,16 @@ static inline unsigned int kvm_get_vmid_bits(void)
return (cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR1_VMIDBITS_SHIFT) == 2) ? 16 : 8;
}
+static inline u64 kvm_get_vttbr(struct kvm_s2_vmid *vmid,
+ struct kvm_s2_mmu *mmu)
+{
+ u64 vmid_field, baddr;
+
+ baddr = virt_to_phys(mmu->pgd);
+ vmid_field = ((u64)vmid->vmid << VTTBR_VMID_SHIFT) &
+ VTTBR_VMID_MASK(get_kvm_vmid_bits());
+ return baddr | vmid_field;
+}
+
#endif /* __ASSEMBLY__ */
#endif /* __ARM64_KVM_MMU_H__ */
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index 3207009a..c80b2ae 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -135,9 +135,7 @@ static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
{
- struct kvm_s2_mmu *mmu = kern_hyp_va(vcpu->arch.hw_mmu);
-
- write_sysreg(mmu->vttbr, vttbr_el2);
+ write_sysreg(vcpu->arch.hw_vttbr, vttbr_el2);
}
static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)
diff --git a/arch/arm64/kvm/hyp/tlb.c b/arch/arm64/kvm/hyp/tlb.c
index 71a62ea..82350e7 100644
--- a/arch/arm64/kvm/hyp/tlb.c
+++ b/arch/arm64/kvm/hyp/tlb.c
@@ -17,14 +17,12 @@
#include <asm/kvm_hyp.h>
-void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
- phys_addr_t ipa)
+void __hyp_text __kvm_tlb_flush_vmid_ipa(u64 vttbr, phys_addr_t ipa)
{
dsb(ishst);
/* Switch to requested VMID */
- mmu = kern_hyp_va(mmu);
- write_sysreg(mmu->vttbr, vttbr_el2);
+ write_sysreg(vttbr, vttbr_el2);
isb();
/*
@@ -49,13 +47,12 @@ void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
write_sysreg(0, vttbr_el2);
}
-void __hyp_text __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu)
+void __hyp_text __kvm_tlb_flush_vmid(u64 vttbr)
{
dsb(ishst);
/* Switch to requested VMID */
- mmu = kern_hyp_va(mmu);
- write_sysreg(mmu->vttbr, vttbr_el2);
+ write_sysreg(vttbr, vttbr_el2);
isb();
asm volatile("tlbi vmalls12e1is" : : );
@@ -65,11 +62,10 @@ void __hyp_text __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu)
write_sysreg(0, vttbr_el2);
}
-void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu)
+void __hyp_text __kvm_tlb_flush_local_vmid(u64 vttbr)
{
/* Switch to requested VMID */
- mmu = kern_hyp_va(mmu);
- write_sysreg(mmu->vttbr, vttbr_el2);
+ write_sysreg(vttbr, vttbr_el2);
isb();
asm volatile("tlbi vmalle1" : : );
--
1.9.1
^ permalink raw reply related
* [RFC 36/55] KVM: arm64: Invalidate virtual EL2 TLB entries when needed
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
Sometimes when we are invalidating the TLB for a certain S2 MMU
context, this context can also have EL2 context associated with it and
we have to invalidate this too.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/kvm/arm.c | 6 ++++++
arch/arm/kvm/mmu.c | 16 ++++++++++++++++
2 files changed, 22 insertions(+)
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index aa8771d..371b38e7 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -350,6 +350,12 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
u64 vttbr = kvm_get_vttbr(&mmu->vmid, mmu);
kvm_call_hyp(__kvm_tlb_flush_local_vmid, vttbr);
+#ifndef CONFIG_KVM_ARM_NESTED_HYP
+ if (mmu->el2_vmid.vmid) {
+ vttbr = kvm_get_vttbr(&mmu->el2_vmid, mmu);
+ kvm_call_hyp(__kvm_tlb_flush_local_vmid, vttbr);
+ }
+#endif
*last_ran = vcpu->vcpu_id;
}
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 5ca3a04..56358fa 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -60,10 +60,20 @@ static bool memslot_is_logging(struct kvm_memory_slot *memslot)
*/
void kvm_flush_remote_tlbs(struct kvm *kvm)
{
+#ifndef CONFIG_KVM_ARM_NESTED_HYP
struct kvm_s2_mmu *mmu = &kvm->arch.mmu;
u64 vttbr = kvm_get_vttbr(&mmu->vmid, mmu);
kvm_call_hyp(__kvm_tlb_flush_vmid, vttbr);
+#else
+ /*
+ * When supporting nested virtualization, we can have multiple VMIDs
+ * in play for each VCPU in the VM, so it's really not worth it to try
+ * to quiesce the system and flush all the VMIDs that may be in use,
+ * instead just nuke the whole thing.
+ */
+ kvm_call_hyp(__kvm_flush_vm_context);
+#endif
}
static void kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa)
@@ -71,6 +81,12 @@ static void kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa)
u64 vttbr = kvm_get_vttbr(&mmu->vmid, mmu);
kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, vttbr, ipa);
+#ifdef CONFIG_KVM_ARM_NESTED_HYP
+ if (!mmu->el2_vmid.vmid)
+ return; /* only if this mmu has el2 context */
+ vttbr = kvm_get_vttbr(&mmu->el2_vmid, mmu);
+ kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, vttbr, ipa);
+#endif
}
/*
--
1.9.1
^ permalink raw reply related
* [RFC 37/55] KVM: arm64: Setup vttbr_el2 on each VM entry
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
Now that the vttbr value will be different depending on the VM's
exception level, we set it on each VM entry.
We only have one mmu instance at this point, but there will be
multiple of them when we run nested VMs.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/kvm/context.c | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/arch/arm64/kvm/context.c b/arch/arm64/kvm/context.c
index a93ffe4..b2c0220 100644
--- a/arch/arm64/kvm/context.c
+++ b/arch/arm64/kvm/context.c
@@ -18,6 +18,7 @@
#include <linux/kvm_host.h>
#include <asm/kvm_emulate.h>
#include <asm/esr.h>
+#include <asm/kvm_mmu.h>
struct el1_el2_map {
enum vcpu_sysreg el1;
@@ -88,6 +89,15 @@ static void create_shadow_el1_sysregs(struct kvm_vcpu *vcpu)
s_sys_regs[CPACR_EL1] = cptr_el2_to_cpacr_el1(el2_regs[CPTR_EL2]);
}
+static void setup_s2_mmu(struct kvm_vcpu *vcpu)
+{
+ struct kvm_s2_mmu *mmu = &vcpu->kvm->arch.mmu;
+ struct kvm_s2_vmid *vmid = vcpu_get_active_vmid(vcpu);
+
+ vcpu->arch.hw_vttbr = kvm_get_vttbr(vmid, mmu);
+ vcpu->arch.hw_mmu = mmu;
+}
+
/*
* List of EL1 registers which we allow the virtual EL2 mode to access
* directly without trapping and which haven't been paravirtualized.
@@ -166,6 +176,8 @@ void kvm_arm_setup_shadow_state(struct kvm_vcpu *vcpu)
}
vgic_v2_setup_shadow_state(vcpu);
+
+ setup_s2_mmu(vcpu);
}
/**
--
1.9.1
^ permalink raw reply related
* [RFC 38/55] KVM: arm/arm64: Make mmu functions non-static
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
Make mmu functions non-static so that we can reuse those functions
to support mmu for the nested VMs.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/kvm/mmu.c | 90 +++++++++++++++++++++++-----------------
arch/arm64/include/asm/kvm_mmu.h | 9 ++++
2 files changed, 61 insertions(+), 38 deletions(-)
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 56358fa..98b42e8 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -301,7 +301,7 @@ static void unmap_stage2_puds(struct kvm_s2_mmu *mmu, pgd_t *pgd,
}
/**
- * unmap_stage2_range -- Clear stage2 page table entries to unmap a range
+ * kvm_unmap_stage2_range -- Clear stage2 page table entries to unmap a range
* @kvm: The VM pointer
* @start: The intermediate physical base address of the range to unmap
* @size: The size of the area to unmap
@@ -311,8 +311,7 @@ static void unmap_stage2_puds(struct kvm_s2_mmu *mmu, pgd_t *pgd,
* destroying the VM), otherwise another faulting VCPU may come in and mess
* with things behind our backs.
*/
-static void unmap_stage2_range(struct kvm_s2_mmu *mmu,
- phys_addr_t start, u64 size)
+void kvm_unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size)
{
pgd_t *pgd;
phys_addr_t addr = start, end = start + size;
@@ -374,11 +373,10 @@ static void stage2_flush_puds(pgd_t *pgd,
} while (pud++, addr = next, addr != end);
}
-static void stage2_flush_memslot(struct kvm_s2_mmu *mmu,
- struct kvm_memory_slot *memslot)
+void kvm_stage2_flush_range(struct kvm_s2_mmu *mmu,
+ phys_addr_t start, phys_addr_t end)
{
- phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
- phys_addr_t end = addr + PAGE_SIZE * memslot->npages;
+ phys_addr_t addr = start;
phys_addr_t next;
pgd_t *pgd;
@@ -389,6 +387,15 @@ static void stage2_flush_memslot(struct kvm_s2_mmu *mmu,
} while (pgd++, addr = next, addr != end);
}
+static void stage2_flush_memslot(struct kvm_s2_mmu *mmu,
+ struct kvm_memory_slot *memslot)
+{
+ phys_addr_t start = memslot->base_gfn << PAGE_SHIFT;
+ phys_addr_t end = start + PAGE_SIZE * memslot->npages;
+
+ kvm_stage2_flush_range(mmu, start, end);
+}
+
/**
* stage2_flush_vm - Invalidate cache for pages mapped in stage 2
* @kvm: The struct kvm pointer
@@ -745,21 +752,9 @@ int create_hyp_io_mappings(void *from, void *to, phys_addr_t phys_addr)
__phys_to_pfn(phys_addr), PAGE_HYP_DEVICE);
}
-/**
- * kvm_alloc_stage2_pgd - allocate level-1 table for stage-2 translation.
- * @kvm: The KVM struct pointer for the VM.
- *
- * Allocates only the stage-2 HW PGD level table(s) (can support either full
- * 40-bit input addresses or limited to 32-bit input addresses). Clears the
- * allocated pages.
- *
- * Note we don't need locking here as this is only called when the VM is
- * created, which can only be done once.
- */
-int kvm_alloc_stage2_pgd(struct kvm *kvm)
+int __kvm_alloc_stage2_pgd(struct kvm_s2_mmu *mmu)
{
pgd_t *pgd;
- struct kvm_s2_mmu *mmu = &kvm->arch.mmu;
if (mmu->pgd != NULL) {
kvm_err("kvm_arch already initialized?\n");
@@ -776,6 +771,22 @@ int kvm_alloc_stage2_pgd(struct kvm *kvm)
return 0;
}
+/**
+ * kvm_alloc_stage2_pgd - allocate level-1 table for stage-2 translation.
+ * @kvm: The KVM struct pointer for the VM.
+ *
+ * Allocates only the stage-2 HW PGD level table(s) (can support either full
+ * 40-bit input addresses or limited to 32-bit input addresses). Clears the
+ * allocated pages.
+ *
+ * Note we don't need locking here as this is only called when the VM is
+ * created, which can only be done once.
+ */
+int kvm_alloc_stage2_pgd(struct kvm *kvm)
+{
+ return __kvm_alloc_stage2_pgd(&kvm->arch.mmu);
+}
+
static void stage2_unmap_memslot(struct kvm_s2_mmu *mmu,
struct kvm_memory_slot *memslot)
{
@@ -811,7 +822,7 @@ static void stage2_unmap_memslot(struct kvm_s2_mmu *mmu,
if (!(vma->vm_flags & VM_PFNMAP)) {
gpa_t gpa = addr + (vm_start - memslot->userspace_addr);
- unmap_stage2_range(mmu, gpa, vm_end - vm_start);
+ kvm_unmap_stage2_range(mmu, gpa, vm_end - vm_start);
}
hva = vm_end;
} while (hva < reg_end);
@@ -841,6 +852,17 @@ void stage2_unmap_vm(struct kvm *kvm)
srcu_read_unlock(&kvm->srcu, idx);
}
+void __kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu)
+{
+ if (mmu->pgd == NULL)
+ return;
+
+ kvm_unmap_stage2_range(mmu, 0, KVM_PHYS_SIZE);
+ /* Free the HW pgd, one page at a time */
+ free_pages_exact(mmu->pgd, S2_PGD_SIZE);
+ mmu->pgd = NULL;
+}
+
/**
* kvm_free_stage2_pgd - free all stage-2 tables
* @kvm: The KVM struct pointer for the VM.
@@ -854,15 +876,7 @@ void stage2_unmap_vm(struct kvm *kvm)
*/
void kvm_free_stage2_pgd(struct kvm *kvm)
{
- struct kvm_s2_mmu *mmu = &kvm->arch.mmu;
-
- if (mmu->pgd == NULL)
- return;
-
- unmap_stage2_range(mmu, 0, KVM_PHYS_SIZE);
- /* Free the HW pgd, one page at a time */
- free_pages_exact(mmu->pgd, S2_PGD_SIZE);
- mmu->pgd = NULL;
+ __kvm_free_stage2_pgd(&kvm->arch.mmu);
}
static pud_t *stage2_get_pud(struct kvm_s2_mmu *mmu,
@@ -1175,13 +1189,13 @@ static void stage2_wp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end)
}
/**
- * stage2_wp_range() - write protect stage2 memory region range
+ * kvm_stage2_wp_range() - write protect stage2 memory region range
* @kvm: The KVM pointer
* @addr: Start address of range
* @end: End address of range
*/
-static void stage2_wp_range(struct kvm *kvm, struct kvm_s2_mmu *mmu,
- phys_addr_t addr, phys_addr_t end)
+void kvm_stage2_wp_range(struct kvm *kvm, struct kvm_s2_mmu *mmu,
+ phys_addr_t addr, phys_addr_t end)
{
pgd_t *pgd;
phys_addr_t next;
@@ -1225,7 +1239,7 @@ void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot)
phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;
spin_lock(&kvm->mmu_lock);
- stage2_wp_range(kvm, &kvm->arch.mmu, start, end);
+ kvm_stage2_wp_range(kvm, &kvm->arch.mmu, start, end);
spin_unlock(&kvm->mmu_lock);
kvm_flush_remote_tlbs(kvm);
}
@@ -1249,7 +1263,7 @@ static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT;
phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT;
- stage2_wp_range(kvm, &kvm->arch.mmu, start, end);
+ kvm_stage2_wp_range(kvm, &kvm->arch.mmu, start, end);
}
/*
@@ -1589,7 +1603,7 @@ static int handle_hva_to_gpa(struct kvm *kvm,
static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
{
- unmap_stage2_range(&kvm->arch.mmu, gpa, PAGE_SIZE);
+ kvm_unmap_stage2_range(&kvm->arch.mmu, gpa, PAGE_SIZE);
return 0;
}
@@ -1900,7 +1914,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
spin_lock(&kvm->mmu_lock);
if (ret)
- unmap_stage2_range(&kvm->arch.mmu, mem->guest_phys_addr,
+ kvm_unmap_stage2_range(&kvm->arch.mmu, mem->guest_phys_addr,
mem->memory_size);
else
stage2_flush_memslot(&kvm->arch.mmu, memslot);
@@ -1944,7 +1958,7 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
phys_addr_t size = slot->npages << PAGE_SHIFT;
spin_lock(&kvm->mmu_lock);
- unmap_stage2_range(&kvm->arch.mmu, gpa, size);
+ kvm_unmap_stage2_range(&kvm->arch.mmu, gpa, size);
spin_unlock(&kvm->mmu_lock);
}
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index e3455c4..a504162 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -145,9 +145,18 @@ static inline unsigned long __kern_hyp_va(unsigned long v)
void stage2_unmap_vm(struct kvm *kvm);
int kvm_alloc_stage2_pgd(struct kvm *kvm);
+int __kvm_alloc_stage2_pgd(struct kvm_s2_mmu *mmu);
void kvm_free_stage2_pgd(struct kvm *kvm);
+void __kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu);
int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
phys_addr_t pa, unsigned long size, bool writable);
+void kvm_unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start,
+ u64 size);
+void kvm_stage2_wp_range(struct kvm *kvm, struct kvm_s2_mmu *mmu,
+ phys_addr_t addr, phys_addr_t end);
+void kvm_stage2_flush_range(struct kvm_s2_mmu *mmu,
+ phys_addr_t start, phys_addr_t end);
+
int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run);
--
1.9.1
^ permalink raw reply related
* [RFC 39/55] KVM: arm/arm64: Add mmu context for the nesting
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
Add the shadow stage-2 MMU context to be used for the nesting, but don't
do anything with it yet.
The host hypervisor maintains mmu structures for each nested VM. When
entering a nested VM, the host hypervisor searches for the nested VM's
mmu using vmid as a key. Note that this vmid is from the guest
hypervisor's point of view.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/include/asm/kvm_host.h | 3 ++
arch/arm/kvm/arm.c | 1 +
arch/arm64/include/asm/kvm_emulate.h | 13 ++++-----
arch/arm64/include/asm/kvm_host.h | 19 +++++++++++++
arch/arm64/include/asm/kvm_mmu.h | 31 ++++++++++++++++++++
arch/arm64/kvm/Makefile | 1 +
arch/arm64/kvm/context.c | 2 +-
arch/arm64/kvm/mmu-nested.c | 55 ++++++++++++++++++++++++++++++++++++
8 files changed, 116 insertions(+), 9 deletions(-)
create mode 100644 arch/arm64/kvm/mmu-nested.c
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index da45394..fbde48d 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -82,6 +82,9 @@ struct kvm_arch {
* here.
*/
+ /* Never used on arm but added to be compatible with arm64 */
+ struct list_head nested_mmu_list;
+
/* Interrupt controller */
struct vgic_dist vgic;
int max_vcpus;
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 371b38e7..147df97 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -146,6 +146,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
/* Mark the initial VMID generation invalid */
kvm->arch.mmu.vmid.vmid_gen = 0;
kvm->arch.mmu.el2_vmid.vmid_gen = 0;
+ INIT_LIST_HEAD(&kvm->arch.nested_mmu_list);
/* The maximum number of VCPUs is limited by the host's GIC model */
kvm->arch.max_vcpus = vgic_present ?
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 94068e7..abad676 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -183,6 +183,11 @@ static inline bool vcpu_el2_imo_is_set(const struct kvm_vcpu *vcpu)
return (vcpu_el2_reg(vcpu, HCR_EL2) & HCR_IMO);
}
+static inline bool vcpu_nested_stage2_enabled(const struct kvm_vcpu *vcpu)
+{
+ return (vcpu_el2_reg(vcpu, HCR_EL2) & HCR_VM);
+}
+
static inline u32 kvm_vcpu_get_hsr(const struct kvm_vcpu *vcpu)
{
return vcpu->arch.fault.esr_el2;
@@ -363,12 +368,4 @@ static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu,
return data; /* Leave LE untouched */
}
-static inline struct kvm_s2_vmid *vcpu_get_active_vmid(struct kvm_vcpu *vcpu)
-{
- if (unlikely(vcpu_mode_el2(vcpu)))
- return &vcpu->kvm->arch.mmu.el2_vmid;
-
- return &vcpu->kvm->arch.mmu.vmid;
-}
-
#endif /* __ARM64_KVM_EMULATE_H__ */
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index b33d35d..23e2267 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -65,6 +65,22 @@ struct kvm_s2_mmu {
pgd_t *pgd;
};
+/* Per nested VM mmu structure */
+struct kvm_nested_s2_mmu {
+ struct kvm_s2_mmu mmu;
+
+ /*
+ * The vttbr value set by the guest hypervisor for this nested VM.
+ * vmid field is used as a key to search for this mmu structure among
+ * all nested VM mmu structures by the host hypervisor.
+ * baddr field is used to determine if we need to unmap stage 2
+ * shadow page tables.
+ */
+ u64 virtual_vttbr;
+
+ struct list_head list;
+};
+
struct kvm_arch {
/* Stage 2 paging state for the VM */
struct kvm_s2_mmu mmu;
@@ -80,6 +96,9 @@ struct kvm_arch {
/* Timer */
struct arch_timer_kvm timer;
+
+ /* Stage 2 shadow paging contexts for nested L2 VM */
+ struct list_head nested_mmu_list;
};
#define KVM_NR_MEM_OBJS 40
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index a504162..d1ef650 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -112,6 +112,7 @@
#include <asm/cacheflush.h>
#include <asm/mmu_context.h>
#include <asm/pgtable.h>
+#include <asm/kvm_emulate.h>
static inline unsigned long __kern_hyp_va(unsigned long v)
{
@@ -323,6 +324,21 @@ static inline unsigned int kvm_get_vmid_bits(void)
return (cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR1_VMIDBITS_SHIFT) == 2) ? 16 : 8;
}
+#ifdef CONFIG_KVM_ARM_NESTED_HYP
+struct kvm_nested_s2_mmu *get_nested_mmu(struct kvm_vcpu *vcpu, u64 vttbr);
+struct kvm_s2_mmu *vcpu_get_active_s2_mmu(struct kvm_vcpu *vcpu);
+#else
+static inline struct kvm_nested_s2_mmu *get_nested_mmu(struct kvm_vcpu *vcpu,
+ u64 vttbr)
+{
+ return NULL;
+}
+static inline struct kvm_s2_mmu *vcpu_get_active_s2_mmu(struct kvm_vcpu *vcpu)
+{
+ return &vcpu->kvm->arch.mmu;
+}
+#endif
+
static inline u64 kvm_get_vttbr(struct kvm_s2_vmid *vmid,
struct kvm_s2_mmu *mmu)
{
@@ -334,5 +350,20 @@ static inline u64 kvm_get_vttbr(struct kvm_s2_vmid *vmid,
return baddr | vmid_field;
}
+static inline u64 get_vmid(u64 vttbr)
+{
+ return (vttbr & VTTBR_VMID_MASK(get_kvm_vmid_bits()))>>VTTBR_VMID_SHIFT;
+}
+
+static inline struct kvm_s2_vmid *vcpu_get_active_vmid(struct kvm_vcpu *vcpu)
+{
+ struct kvm_s2_mmu *mmu = vcpu_get_active_s2_mmu(vcpu);
+
+ if (unlikely(vcpu_mode_el2(vcpu)))
+ return &mmu->el2_vmid;
+ else
+ return &mmu->vmid;
+}
+
#endif /* __ASSEMBLY__ */
#endif /* __ARM64_KVM_MMU_H__ */
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index 8573faf..b0b1074 100644
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -36,5 +36,6 @@ kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o
kvm-$(CONFIG_KVM_ARM_PMU) += $(KVM)/arm/pmu.o
kvm-$(CONFIG_KVM_ARM_NESTED_HYP) += handle_exit_nested.o
+kvm-$(CONFIG_KVM_ARM_NESTED_HYP) += mmu-nested.o
kvm-$(CONFIG_KVM_ARM_NESTED_HYP) += emulate-nested.o
kvm-$(CONFIG_KVM_ARM_NESTED_HYP) += $(KVM)/arm/vgic/vgic-v2-nested.o
diff --git a/arch/arm64/kvm/context.c b/arch/arm64/kvm/context.c
index b2c0220..9ebc38f 100644
--- a/arch/arm64/kvm/context.c
+++ b/arch/arm64/kvm/context.c
@@ -91,7 +91,7 @@ static void create_shadow_el1_sysregs(struct kvm_vcpu *vcpu)
static void setup_s2_mmu(struct kvm_vcpu *vcpu)
{
- struct kvm_s2_mmu *mmu = &vcpu->kvm->arch.mmu;
+ struct kvm_s2_mmu *mmu = vcpu_get_active_s2_mmu(vcpu);
struct kvm_s2_vmid *vmid = vcpu_get_active_vmid(vcpu);
vcpu->arch.hw_vttbr = kvm_get_vttbr(vmid, mmu);
diff --git a/arch/arm64/kvm/mmu-nested.c b/arch/arm64/kvm/mmu-nested.c
new file mode 100644
index 0000000..d52078f
--- /dev/null
+++ b/arch/arm64/kvm/mmu-nested.c
@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2016 - Columbia University
+ * Author: Jintack Lim <jintack@cs.columbia.edu>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/kvm_host.h>
+
+#include <asm/kvm_arm.h>
+#include <asm/kvm_emulate.h>
+#include <asm/kvm_mmu.h>
+#include <asm/kvm_nested.h>
+
+struct kvm_nested_s2_mmu *get_nested_mmu(struct kvm_vcpu *vcpu, u64 vttbr)
+{
+ struct kvm_nested_s2_mmu *mmu;
+ u64 target_vmid = get_vmid(vttbr);
+ struct list_head *nested_mmu_list = &vcpu->kvm->arch.nested_mmu_list;
+
+ list_for_each_entry_rcu(mmu, nested_mmu_list, list) {
+ u64 vmid = get_vmid(mmu->virtual_vttbr);
+
+ if (target_vmid == vmid)
+ return mmu;
+ }
+ return NULL;
+}
+
+struct kvm_s2_mmu *vcpu_get_active_s2_mmu(struct kvm_vcpu *vcpu)
+{
+ struct kvm_nested_s2_mmu *nested_mmu;
+
+ /* If we are NOT entering the nested VM, return mmu in kvm_arch */
+ if (vcpu_mode_el2(vcpu) || !vcpu_nested_stage2_enabled(vcpu))
+ return &vcpu->kvm->arch.mmu;
+
+ /* Otherwise, search for nested_mmu in the list */
+ nested_mmu = get_nested_mmu(vcpu, vcpu_el2_reg(vcpu, VTTBR_EL2));
+
+ /* When this function is called, nested_mmu should be in the list */
+ BUG_ON(!nested_mmu);
+
+ return &nested_mmu->mmu;
+}
--
1.9.1
^ permalink raw reply related
* [RFC 40/55] KVM: arm/arm64: Handle vttbr_el2 write operation from the guest hypervisor
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
Each nested VM is supposed to have a mmu (i.e. shadow stage-2 page
table), and we create it when the guest hypervisor writes to vttbr_el2
with a new vmid.
In case the guest hypervisor writes to vttbr_el2 with existing vmid, we
check if the base address is changed. If so, then what we have in the
shadow page table is not valid any more. So ummap it.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/include/asm/kvm_host.h | 1 +
arch/arm/kvm/arm.c | 1 +
arch/arm64/include/asm/kvm_host.h | 1 +
arch/arm64/include/asm/kvm_mmu.h | 6 ++++
arch/arm64/kvm/mmu-nested.c | 71 +++++++++++++++++++++++++++++++++++++++
arch/arm64/kvm/sys_regs.c | 15 ++++++++-
6 files changed, 94 insertions(+), 1 deletion(-)
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index fbde48d..ebf2810 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -84,6 +84,7 @@ struct kvm_arch {
/* Never used on arm but added to be compatible with arm64 */
struct list_head nested_mmu_list;
+ spinlock_t mmu_list_lock;
/* Interrupt controller */
struct vgic_dist vgic;
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 147df97..6fa5754 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -147,6 +147,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
kvm->arch.mmu.vmid.vmid_gen = 0;
kvm->arch.mmu.el2_vmid.vmid_gen = 0;
INIT_LIST_HEAD(&kvm->arch.nested_mmu_list);
+ spin_lock_init(&kvm->arch.mmu_list_lock);
/* The maximum number of VCPUs is limited by the host's GIC model */
kvm->arch.max_vcpus = vgic_present ?
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 23e2267..52eea76 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -99,6 +99,7 @@ struct kvm_arch {
/* Stage 2 shadow paging contexts for nested L2 VM */
struct list_head nested_mmu_list;
+ spinlock_t mmu_list_lock;
};
#define KVM_NR_MEM_OBJS 40
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index d1ef650..fdc9327 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -327,6 +327,7 @@ static inline unsigned int kvm_get_vmid_bits(void)
#ifdef CONFIG_KVM_ARM_NESTED_HYP
struct kvm_nested_s2_mmu *get_nested_mmu(struct kvm_vcpu *vcpu, u64 vttbr);
struct kvm_s2_mmu *vcpu_get_active_s2_mmu(struct kvm_vcpu *vcpu);
+bool handle_vttbr_update(struct kvm_vcpu *vcpu, u64 vttbr);
#else
static inline struct kvm_nested_s2_mmu *get_nested_mmu(struct kvm_vcpu *vcpu,
u64 vttbr)
@@ -337,6 +338,11 @@ static inline struct kvm_s2_mmu *vcpu_get_active_s2_mmu(struct kvm_vcpu *vcpu)
{
return &vcpu->kvm->arch.mmu;
}
+
+static inline bool handle_vttbr_update(struct kvm_vcpu *vcpu, u64 vttbr)
+{
+ return false;
+}
#endif
static inline u64 kvm_get_vttbr(struct kvm_s2_vmid *vmid,
diff --git a/arch/arm64/kvm/mmu-nested.c b/arch/arm64/kvm/mmu-nested.c
index d52078f..0811d94 100644
--- a/arch/arm64/kvm/mmu-nested.c
+++ b/arch/arm64/kvm/mmu-nested.c
@@ -53,3 +53,74 @@ struct kvm_s2_mmu *vcpu_get_active_s2_mmu(struct kvm_vcpu *vcpu)
return &nested_mmu->mmu;
}
+
+static struct kvm_nested_s2_mmu *create_nested_mmu(struct kvm_vcpu *vcpu,
+ u64 vttbr)
+{
+ struct kvm_nested_s2_mmu *nested_mmu, *tmp_mmu;
+ struct list_head *nested_mmu_list = &vcpu->kvm->arch.nested_mmu_list;
+ bool need_free = false;
+ int ret;
+
+ nested_mmu = kzalloc(sizeof(struct kvm_nested_s2_mmu), GFP_KERNEL);
+ if (!nested_mmu)
+ return NULL;
+
+ ret = __kvm_alloc_stage2_pgd(&nested_mmu->mmu);
+ if (ret) {
+ kfree(nested_mmu);
+ return NULL;
+ }
+
+ spin_lock(&vcpu->kvm->arch.mmu_list_lock);
+ tmp_mmu = get_nested_mmu(vcpu, vttbr);
+ if (!tmp_mmu)
+ list_add_rcu(&nested_mmu->list, nested_mmu_list);
+ else /* Somebody already created and put a new nested_mmu to the list */
+ need_free = true;
+ spin_unlock(&vcpu->kvm->arch.mmu_list_lock);
+
+ if (need_free) {
+ __kvm_free_stage2_pgd(&nested_mmu->mmu);
+ kfree(nested_mmu);
+ nested_mmu = tmp_mmu;
+ }
+
+ return nested_mmu;
+}
+
+static void kvm_nested_s2_unmap(struct kvm_vcpu *vcpu)
+{
+ struct kvm_nested_s2_mmu *nested_mmu;
+ struct list_head *nested_mmu_list = &vcpu->kvm->arch.nested_mmu_list;
+
+ list_for_each_entry_rcu(nested_mmu, nested_mmu_list, list)
+ kvm_unmap_stage2_range(&nested_mmu->mmu, 0, KVM_PHYS_SIZE);
+}
+
+bool handle_vttbr_update(struct kvm_vcpu *vcpu, u64 vttbr)
+{
+ struct kvm_nested_s2_mmu *nested_mmu;
+
+ /* See if we can relax this */
+ if (!vttbr)
+ return true;
+
+ nested_mmu = (struct kvm_nested_s2_mmu *)get_nested_mmu(vcpu, vttbr);
+ if (!nested_mmu) {
+ nested_mmu = create_nested_mmu(vcpu, vttbr);
+ if (!nested_mmu)
+ return false;
+ } else {
+ /*
+ * unmap the shadow page table if vttbr_el2 is
+ * changed to different value
+ */
+ if (vttbr != nested_mmu->virtual_vttbr)
+ kvm_nested_s2_unmap(vcpu);
+ }
+
+ nested_mmu->virtual_vttbr = vttbr;
+
+ return true;
+}
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index e66f40d..ddb641c 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -960,6 +960,19 @@ static bool access_cpacr(struct kvm_vcpu *vcpu,
return true;
}
+static bool access_vttbr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
+ const struct sys_reg_desc *r)
+{
+ u64 vttbr = p->regval;
+
+ if (!p->is_write) {
+ p->regval = vcpu_el2_reg(vcpu, r->reg);
+ return true;
+ }
+
+ return handle_vttbr_update(vcpu, vttbr);
+}
+
static bool trap_el2_reg(struct kvm_vcpu *vcpu,
struct sys_reg_params *p,
const struct sys_reg_desc *r)
@@ -1306,7 +1319,7 @@ static bool trap_el2_reg(struct kvm_vcpu *vcpu,
trap_el2_reg, reset_el2_val, TCR_EL2, 0 },
/* VTTBR_EL2 */
{ Op0(0b11), Op1(0b100), CRn(0b0010), CRm(0b0001), Op2(0b000),
- trap_el2_reg, reset_el2_val, VTTBR_EL2, 0 },
+ access_vttbr, reset_el2_val, VTTBR_EL2, 0 },
/* VTCR_EL2 */
{ Op0(0b11), Op1(0b100), CRn(0b0010), CRm(0b0001), Op2(0b010),
trap_el2_reg, reset_el2_val, VTCR_EL2, 0 },
--
1.9.1
^ permalink raw reply related
* [RFC 41/55] KVM: arm/arm64: Unmap/flush shadow stage 2 page tables
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
Unmap/flush shadow stage 2 page tables for the nested VMs as well as the
stage 2 page table for the guest hypervisor.
Note: A bunch of the code in mmu.c relating to MMU notifiers is
currently dealt with in an extremely abrupt way, for example by clearing
out an entire shadow stage-2 table. Probably we can do smarter with
some sort of rmap structure.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/include/asm/kvm_mmu.h | 7 ++++
arch/arm/kvm/arm.c | 6 ++-
arch/arm/kvm/mmu.c | 11 +++++
arch/arm64/include/asm/kvm_mmu.h | 13 ++++++
arch/arm64/kvm/mmu-nested.c | 90 ++++++++++++++++++++++++++++++++++++----
5 files changed, 117 insertions(+), 10 deletions(-)
diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
index 1b3309c..ae3aa39 100644
--- a/arch/arm/include/asm/kvm_mmu.h
+++ b/arch/arm/include/asm/kvm_mmu.h
@@ -230,6 +230,13 @@ static inline unsigned int kvm_get_vmid_bits(void)
return 8;
}
+static inline void kvm_nested_s2_unmap(struct kvm_vcpu *vcpu) { }
+static inline int kvm_nested_s2_init(struct kvm_vcpu *vcpu) { return 0; }
+static inline void kvm_nested_s2_teardown(struct kvm_vcpu *vcpu) { }
+static inline void kvm_nested_s2_all_vcpus_wp(struct kvm *kvm) { }
+static inline void kvm_nested_s2_all_vcpus_unmap(struct kvm *kvm) { }
+static inline void kvm_nested_s2_all_vcpus_flush(struct kvm *kvm) { }
+
static inline u64 kvm_get_vttbr(struct kvm_s2_vmid *vmid,
struct kvm_s2_mmu *mmu)
{
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 6fa5754..dc2795f 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -191,6 +191,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
for (i = 0; i < KVM_MAX_VCPUS; ++i) {
if (kvm->vcpus[i]) {
+ kvm_nested_s2_teardown(kvm->vcpus[i]);
kvm_arch_vcpu_free(kvm->vcpus[i]);
kvm->vcpus[i] = NULL;
}
@@ -333,6 +334,7 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
vcpu->arch.hw_mmu = mmu;
vcpu->arch.hw_vttbr = kvm_get_vttbr(&mmu->vmid, mmu);
+ kvm_nested_s2_init(vcpu);
return 0;
}
@@ -871,8 +873,10 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
* Ensure a rebooted VM will fault in RAM pages and detect if the
* guest MMU is turned off and flush the caches as needed.
*/
- if (vcpu->arch.has_run_once)
+ if (vcpu->arch.has_run_once) {
stage2_unmap_vm(vcpu->kvm);
+ kvm_nested_s2_unmap(vcpu);
+ }
vcpu_reset_hcr(vcpu);
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 98b42e8..1677a87 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -416,6 +416,8 @@ static void stage2_flush_vm(struct kvm *kvm)
kvm_for_each_memslot(memslot, slots)
stage2_flush_memslot(&kvm->arch.mmu, memslot);
+ kvm_nested_s2_all_vcpus_flush(kvm);
+
spin_unlock(&kvm->mmu_lock);
srcu_read_unlock(&kvm->srcu, idx);
}
@@ -1240,6 +1242,7 @@ void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot)
spin_lock(&kvm->mmu_lock);
kvm_stage2_wp_range(kvm, &kvm->arch.mmu, start, end);
+ kvm_nested_s2_all_vcpus_wp(kvm);
spin_unlock(&kvm->mmu_lock);
kvm_flush_remote_tlbs(kvm);
}
@@ -1278,6 +1281,7 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
gfn_t gfn_offset, unsigned long mask)
{
kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
+ kvm_nested_s2_all_vcpus_wp(kvm);
}
static void coherent_cache_guest_page(struct kvm_vcpu *vcpu, kvm_pfn_t pfn,
@@ -1604,6 +1608,7 @@ static int handle_hva_to_gpa(struct kvm *kvm,
static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
{
kvm_unmap_stage2_range(&kvm->arch.mmu, gpa, PAGE_SIZE);
+ kvm_nested_s2_all_vcpus_unmap(kvm);
return 0;
}
@@ -1642,6 +1647,7 @@ static int kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
* through this calling path.
*/
stage2_set_pte(&kvm->arch.mmu, NULL, gpa, pte, 0);
+ kvm_nested_s2_all_vcpus_unmap(kvm);
return 0;
}
@@ -1675,6 +1681,8 @@ static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
if (pte_none(*pte))
return 0;
+ /* TODO: Handle nested_mmu structures here as well */
+
return stage2_ptep_test_and_clear_young(pte);
}
@@ -1694,6 +1702,8 @@ static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
if (!pte_none(*pte)) /* Just a page... */
return pte_young(*pte);
+ /* TODO: Handle nested_mmu structures here as well */
+
return 0;
}
@@ -1959,6 +1969,7 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
spin_lock(&kvm->mmu_lock);
kvm_unmap_stage2_range(&kvm->arch.mmu, gpa, size);
+ kvm_nested_s2_all_vcpus_unmap(kvm);
spin_unlock(&kvm->mmu_lock);
}
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index fdc9327..e4d5d54 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -328,6 +328,12 @@ static inline unsigned int kvm_get_vmid_bits(void)
struct kvm_nested_s2_mmu *get_nested_mmu(struct kvm_vcpu *vcpu, u64 vttbr);
struct kvm_s2_mmu *vcpu_get_active_s2_mmu(struct kvm_vcpu *vcpu);
bool handle_vttbr_update(struct kvm_vcpu *vcpu, u64 vttbr);
+void kvm_nested_s2_unmap(struct kvm_vcpu *vcpu);
+int kvm_nested_s2_init(struct kvm_vcpu *vcpu);
+void kvm_nested_s2_teardown(struct kvm_vcpu *vcpu);
+void kvm_nested_s2_all_vcpus_wp(struct kvm *kvm);
+void kvm_nested_s2_all_vcpus_unmap(struct kvm *kvm);
+void kvm_nested_s2_all_vcpus_flush(struct kvm *kvm);
#else
static inline struct kvm_nested_s2_mmu *get_nested_mmu(struct kvm_vcpu *vcpu,
u64 vttbr)
@@ -343,6 +349,13 @@ static inline bool handle_vttbr_update(struct kvm_vcpu *vcpu, u64 vttbr)
{
return false;
}
+
+static inline void kvm_nested_s2_unmap(struct kvm_vcpu *vcpu) { }
+static inline int kvm_nested_s2_init(struct kvm_vcpu *vcpu) { return 0; }
+static inline void kvm_nested_s2_teardown(struct kvm_vcpu *vcpu) { }
+static inline void kvm_nested_s2_all_vcpus_wp(struct kvm *kvm) { }
+static inline void kvm_nested_s2_all_vcpus_unmap(struct kvm *kvm) { }
+static inline void kvm_nested_s2_all_vcpus_flush(struct kvm *kvm) { }
#endif
static inline u64 kvm_get_vttbr(struct kvm_s2_vmid *vmid,
diff --git a/arch/arm64/kvm/mmu-nested.c b/arch/arm64/kvm/mmu-nested.c
index 0811d94..b22b78c 100644
--- a/arch/arm64/kvm/mmu-nested.c
+++ b/arch/arm64/kvm/mmu-nested.c
@@ -1,6 +1,7 @@
/*
* Copyright (C) 2016 - Columbia University
* Author: Jintack Lim <jintack@cs.columbia.edu>
+ * Author: Christoffer Dall <cdall@cs.columbia.edu>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
@@ -22,6 +23,86 @@
#include <asm/kvm_mmu.h>
#include <asm/kvm_nested.h>
+
+/* expects kvm->mmu_lock to be held */
+void kvm_nested_s2_all_vcpus_wp(struct kvm *kvm)
+{
+ int i;
+ struct kvm_vcpu *vcpu;
+ struct kvm_nested_s2_mmu *nested_mmu;
+ struct list_head *nested_mmu_list;
+
+ kvm_for_each_vcpu(i, vcpu, kvm) {
+ if (need_resched() || spin_needbreak(&kvm->mmu_lock))
+ cond_resched_lock(&kvm->mmu_lock);
+
+ nested_mmu_list = &vcpu->kvm->arch.nested_mmu_list;
+ list_for_each_entry_rcu(nested_mmu, nested_mmu_list, list)
+ kvm_stage2_wp_range(kvm, &nested_mmu->mmu,
+ 0, KVM_PHYS_SIZE);
+ }
+}
+
+/* expects kvm->mmu_lock to be held */
+void kvm_nested_s2_all_vcpus_unmap(struct kvm *kvm)
+{
+ int i;
+ struct kvm_vcpu *vcpu;
+ struct kvm_nested_s2_mmu *nested_mmu;
+ struct list_head *nested_mmu_list;
+
+ kvm_for_each_vcpu(i, vcpu, kvm) {
+ if (need_resched() || spin_needbreak(&kvm->mmu_lock))
+ cond_resched_lock(&kvm->mmu_lock);
+
+ nested_mmu_list = &vcpu->kvm->arch.nested_mmu_list;
+ list_for_each_entry_rcu(nested_mmu, nested_mmu_list, list)
+ kvm_unmap_stage2_range(&nested_mmu->mmu,
+ 0, KVM_PHYS_SIZE);
+ }
+}
+
+void kvm_nested_s2_all_vcpus_flush(struct kvm *kvm)
+{
+ int i;
+ struct kvm_vcpu *vcpu;
+ struct kvm_nested_s2_mmu *nested_mmu;
+ struct list_head *nested_mmu_list;
+
+ kvm_for_each_vcpu(i, vcpu, kvm) {
+ if (need_resched() || spin_needbreak(&kvm->mmu_lock))
+ cond_resched_lock(&kvm->mmu_lock);
+
+ nested_mmu_list = &vcpu->kvm->arch.nested_mmu_list;
+ list_for_each_entry_rcu(nested_mmu, nested_mmu_list, list)
+ kvm_stage2_flush_range(&nested_mmu->mmu,
+ 0, KVM_PHYS_SIZE);
+ }
+}
+
+void kvm_nested_s2_unmap(struct kvm_vcpu *vcpu)
+{
+ struct kvm_nested_s2_mmu *nested_mmu;
+ struct list_head *nested_mmu_list = &vcpu->kvm->arch.nested_mmu_list;
+
+ list_for_each_entry_rcu(nested_mmu, nested_mmu_list, list)
+ kvm_unmap_stage2_range(&nested_mmu->mmu, 0, KVM_PHYS_SIZE);
+}
+
+int kvm_nested_s2_init(struct kvm_vcpu *vcpu)
+{
+ return 0;
+}
+
+void kvm_nested_s2_teardown(struct kvm_vcpu *vcpu)
+{
+ struct kvm_nested_s2_mmu *nested_mmu;
+ struct list_head *nested_mmu_list = &vcpu->kvm->arch.nested_mmu_list;
+
+ list_for_each_entry_rcu(nested_mmu, nested_mmu_list, list)
+ __kvm_free_stage2_pgd(&nested_mmu->mmu);
+}
+
struct kvm_nested_s2_mmu *get_nested_mmu(struct kvm_vcpu *vcpu, u64 vttbr)
{
struct kvm_nested_s2_mmu *mmu;
@@ -89,15 +170,6 @@ static struct kvm_nested_s2_mmu *create_nested_mmu(struct kvm_vcpu *vcpu,
return nested_mmu;
}
-static void kvm_nested_s2_unmap(struct kvm_vcpu *vcpu)
-{
- struct kvm_nested_s2_mmu *nested_mmu;
- struct list_head *nested_mmu_list = &vcpu->kvm->arch.nested_mmu_list;
-
- list_for_each_entry_rcu(nested_mmu, nested_mmu_list, list)
- kvm_unmap_stage2_range(&nested_mmu->mmu, 0, KVM_PHYS_SIZE);
-}
-
bool handle_vttbr_update(struct kvm_vcpu *vcpu, u64 vttbr)
{
struct kvm_nested_s2_mmu *nested_mmu;
--
1.9.1
^ permalink raw reply related
* [RFC 42/55] KVM: arm64: Implement nested Stage-2 page table walk logic
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
Based on the pseudo-code in the ARM ARM, implement a stage 2 software
page table walker.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/include/asm/kvm_mmu.h | 11 ++
arch/arm64/include/asm/kvm_arm.h | 1 +
arch/arm64/include/asm/kvm_mmu.h | 13 +++
arch/arm64/kvm/mmu-nested.c | 223 +++++++++++++++++++++++++++++++++++++++
4 files changed, 248 insertions(+)
diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
index ae3aa39..ab41a10 100644
--- a/arch/arm/include/asm/kvm_mmu.h
+++ b/arch/arm/include/asm/kvm_mmu.h
@@ -230,6 +230,17 @@ static inline unsigned int kvm_get_vmid_bits(void)
return 8;
}
+struct kvm_s2_trans {
+ phys_addr_t output;
+ phys_addr_t block_size;
+};
+
+static inline int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
+ struct kvm_s2_trans *result)
+{
+ return 0;
+}
+
static inline void kvm_nested_s2_unmap(struct kvm_vcpu *vcpu) { }
static inline int kvm_nested_s2_init(struct kvm_vcpu *vcpu) { return 0; }
static inline void kvm_nested_s2_teardown(struct kvm_vcpu *vcpu) { }
diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h
index feded61..f9addf3 100644
--- a/arch/arm64/include/asm/kvm_arm.h
+++ b/arch/arm64/include/asm/kvm_arm.h
@@ -103,6 +103,7 @@
#define VTCR_EL2_RES1 (1 << 31)
#define VTCR_EL2_HD (1 << 22)
#define VTCR_EL2_HA (1 << 21)
+#define VTCR_EL2_PS_SHIFT TCR_EL2_PS_SHIFT
#define VTCR_EL2_PS_MASK TCR_EL2_PS_MASK
#define VTCR_EL2_TG0_MASK TCR_TG0_MASK
#define VTCR_EL2_TG0_4K TCR_TG0_4K
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index e4d5d54..bf94f0c 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -324,10 +324,17 @@ static inline unsigned int kvm_get_vmid_bits(void)
return (cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR1_VMIDBITS_SHIFT) == 2) ? 16 : 8;
}
+struct kvm_s2_trans {
+ phys_addr_t output;
+ phys_addr_t block_size;
+};
+
#ifdef CONFIG_KVM_ARM_NESTED_HYP
struct kvm_nested_s2_mmu *get_nested_mmu(struct kvm_vcpu *vcpu, u64 vttbr);
struct kvm_s2_mmu *vcpu_get_active_s2_mmu(struct kvm_vcpu *vcpu);
bool handle_vttbr_update(struct kvm_vcpu *vcpu, u64 vttbr);
+int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
+ struct kvm_s2_trans *result);
void kvm_nested_s2_unmap(struct kvm_vcpu *vcpu);
int kvm_nested_s2_init(struct kvm_vcpu *vcpu);
void kvm_nested_s2_teardown(struct kvm_vcpu *vcpu);
@@ -350,6 +357,12 @@ static inline bool handle_vttbr_update(struct kvm_vcpu *vcpu, u64 vttbr)
return false;
}
+static inline int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
+ struct kvm_s2_trans *result)
+{
+ return 0;
+}
+
static inline void kvm_nested_s2_unmap(struct kvm_vcpu *vcpu) { }
static inline int kvm_nested_s2_init(struct kvm_vcpu *vcpu) { return 0; }
static inline void kvm_nested_s2_teardown(struct kvm_vcpu *vcpu) { }
diff --git a/arch/arm64/kvm/mmu-nested.c b/arch/arm64/kvm/mmu-nested.c
index b22b78c..a2fab41 100644
--- a/arch/arm64/kvm/mmu-nested.c
+++ b/arch/arm64/kvm/mmu-nested.c
@@ -23,6 +23,229 @@
#include <asm/kvm_mmu.h>
#include <asm/kvm_nested.h>
+struct s2_walk_info {
+ unsigned int pgshift;
+ unsigned int pgsize;
+ unsigned int ps;
+ unsigned int sl;
+ unsigned int t0sz;
+};
+
+static unsigned int ps_to_output_size(unsigned int ps)
+{
+ switch (ps) {
+ case 0: return 32;
+ case 1: return 36;
+ case 2: return 40;
+ case 3: return 42;
+ case 4: return 44;
+ case 5:
+ default:
+ return 48;
+ }
+}
+
+static unsigned int pa_max(void)
+{
+ u64 parange = read_sysreg(id_aa64mmfr0_el1) & 7;
+
+ return ps_to_output_size(parange);
+}
+
+static int vcpu_inject_s2_trans_fault(struct kvm_vcpu *vcpu, gpa_t ipa,
+ int level)
+{
+ /* TODO: Implement */
+ return -EFAULT;
+}
+
+static int vcpu_inject_s2_addr_sz_fault(struct kvm_vcpu *vcpu, gpa_t ipa,
+ int level)
+{
+ /* TODO: Implement */
+ return -EFAULT;
+}
+
+static int vcpu_inject_s2_access_flag_fault(struct kvm_vcpu *vcpu, gpa_t ipa,
+ int level)
+{
+ /* TODO: Implement */
+ return -EFAULT;
+}
+
+static int check_base_s2_limits(struct kvm_vcpu *vcpu, struct s2_walk_info *wi,
+ int level, int input_size, int stride)
+{
+ int start_size;
+
+ /* Check translation limits */
+ switch (wi->pgsize) {
+ case SZ_64K:
+ if (level == 0 || (level == 1 && pa_max() <= 42))
+ return -EFAULT;
+ break;
+ case SZ_16K:
+ if (level == 0 || (level == 1 && pa_max() <= 40))
+ return -EFAULT;
+ break;
+ case SZ_4K:
+ if (level < 0 || (level == 0 && pa_max() <= 42))
+ return -EFAULT;
+ break;
+ }
+
+ /* Check input size limits */
+ if (input_size > pa_max() &&
+ (!vcpu_mode_is_32bit(vcpu) || input_size > 40))
+ return -EFAULT;
+
+ /* Check number of entries in starting level table */
+ start_size = input_size - ((3 - level) * stride + wi->pgshift);
+ if (start_size < 1 || start_size > stride + 4)
+ return -EFAULT;
+
+ return 0;
+}
+
+/* Check if output is within boundaries */
+static int check_output_size(struct kvm_vcpu *vcpu, struct s2_walk_info *wi,
+ phys_addr_t output)
+{
+ unsigned int output_size = ps_to_output_size(wi->ps);
+
+ if (output_size > pa_max())
+ output_size = pa_max();
+
+ if (output_size != 48 && (output & GENMASK_ULL(47, output_size)))
+ return -1;
+
+ return 0;
+}
+
+/*
+ * This is essentially a C-version of the pseudo code from the ARM ARM
+ * AArch64.TranslationTableWalk function. I strongly recommend looking at
+ * that pseudocode in trying to understand this.
+ *
+ * Must be called with the kvm->srcy read lock held
+ */
+static int walk_nested_s2_pgd(struct kvm_vcpu *vcpu, phys_addr_t ipa,
+ struct s2_walk_info *wi, struct kvm_s2_trans *out)
+{
+ u64 vttbr = vcpu->arch.ctxt.el2_regs[VTTBR_EL2];
+ int first_block_level, level, stride, input_size, base_lower_bound;
+ phys_addr_t base_addr;
+ unsigned int addr_top, addr_bottom;
+ u64 desc; /* page table entry */
+ int ret;
+ phys_addr_t paddr;
+
+ switch (wi->pgsize) {
+ case SZ_64K:
+ case SZ_16K:
+ level = 3 - wi->sl;
+ first_block_level = 2;
+ break;
+ case SZ_4K:
+ level = 2 - wi->sl;
+ first_block_level = 1;
+ break;
+ default:
+ /* GCC is braindead */
+ WARN(1, "Page size is none of 4K, 16K or 64K");
+ }
+
+ stride = wi->pgshift - 3;
+ input_size = 64 - wi->t0sz;
+ if (input_size > 48 || input_size < 25)
+ return -EFAULT;
+
+ ret = check_base_s2_limits(vcpu, wi, level, input_size, stride);
+ if (WARN_ON(ret))
+ return ret;
+
+ if (check_output_size(vcpu, wi, vttbr))
+ return vcpu_inject_s2_addr_sz_fault(vcpu, ipa, level);
+
+ base_lower_bound = 3 + input_size - ((3 - level) * stride +
+ wi->pgshift);
+ base_addr = vttbr & GENMASK_ULL(47, base_lower_bound);
+
+ addr_top = input_size - 1;
+
+ while (1) {
+ phys_addr_t index;
+
+ addr_bottom = (3 - level) * stride + wi->pgshift;
+ index = (ipa & GENMASK_ULL(addr_top, addr_bottom))
+ >> (addr_bottom - 3);
+
+ paddr = base_addr | index;
+ ret = kvm_read_guest(vcpu->kvm, paddr, &desc, sizeof(desc));
+ if (ret < 0)
+ return ret;
+
+ /* Check for valid descriptor at this point */
+ if (!(desc & 1) || ((desc & 3) == 1 && level == 3))
+ return vcpu_inject_s2_trans_fault(vcpu, ipa, level);
+
+ /* We're@the final level or block translation level */
+ if ((desc & 3) == 1 || level == 3)
+ break;
+
+ if (check_output_size(vcpu, wi, desc))
+ return vcpu_inject_s2_addr_sz_fault(vcpu, ipa, level);
+
+ base_addr = desc & GENMASK_ULL(47, wi->pgshift);
+
+ level += 1;
+ addr_top = addr_bottom - 1;
+ }
+
+ if (level < first_block_level)
+ return vcpu_inject_s2_trans_fault(vcpu, ipa, level);
+
+ /* TODO: Consider checking contiguous bit setting */
+
+ if (check_output_size(vcpu, wi, desc))
+ return vcpu_inject_s2_addr_sz_fault(vcpu, ipa, level);
+
+ if (!(desc & BIT(10)))
+ return vcpu_inject_s2_access_flag_fault(vcpu, ipa, level);
+
+ /* Calculate and return the result */
+ paddr = (desc & GENMASK_ULL(47, addr_bottom)) |
+ (ipa & GENMASK_ULL(addr_bottom - 1, 0));
+ out->output = paddr;
+ out->block_size = 1UL << ((3 - level) * stride + wi->pgshift);
+ return 0;
+}
+
+int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
+ struct kvm_s2_trans *result)
+{
+ u64 vtcr = vcpu->arch.ctxt.el2_regs[VTCR_EL2];
+ struct s2_walk_info wi;
+
+ wi.t0sz = vtcr & TCR_EL2_T0SZ_MASK;
+
+ switch (vtcr & VTCR_EL2_TG0_MASK) {
+ case VTCR_EL2_TG0_4K:
+ wi.pgshift = 12; break;
+ case VTCR_EL2_TG0_16K:
+ wi.pgshift = 14; break;
+ case VTCR_EL2_TG0_64K:
+ default:
+ wi.pgshift = 16; break;
+ }
+ wi.pgsize = 1UL << wi.pgshift;
+ wi.ps = (vtcr & VTCR_EL2_PS_MASK) >> VTCR_EL2_PS_SHIFT;
+ wi.sl = (vtcr & VTCR_EL2_SL0_MASK) >> VTCR_EL2_SL0_SHIFT;
+
+ /* TODO: Reversedescriptor if SCTLR_EL2.EE == 1 */
+
+ return walk_nested_s2_pgd(vcpu, gipa, &wi, result);
+}
/* expects kvm->mmu_lock to be held */
void kvm_nested_s2_all_vcpus_wp(struct kvm *kvm)
--
1.9.1
^ permalink raw reply related
* [RFC 43/55] KVM: arm/arm64: Handle shadow stage 2 page faults
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
If we are faulting on a shadow stage 2 translation, we have to take
extra care in faulting in a page, because we have to collapse the two
levels of stage 2 paging by walking the L2-to-L1 stage 2 page tables in
software.
This approach tries to integrate as much as possible with the existing
code.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/include/asm/kvm_emulate.h | 7 ++++
arch/arm/kvm/mmio.c | 12 +++---
arch/arm/kvm/mmu.c | 75 ++++++++++++++++++++++++++++--------
arch/arm64/include/asm/kvm_emulate.h | 9 +++++
4 files changed, 82 insertions(+), 21 deletions(-)
diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h
index 6285f4f..dfc53ce 100644
--- a/arch/arm/include/asm/kvm_emulate.h
+++ b/arch/arm/include/asm/kvm_emulate.h
@@ -309,4 +309,11 @@ static inline struct kvm_s2_vmid *vcpu_get_active_vmid(struct kvm_vcpu *vcpu)
{
return &vcpu->kvm->arch.mmu.vmid;
}
+
+/* arm architecture doesn't support the nesting */
+static inline bool kvm_is_shadow_s2_fault(struct kvm_vcpu *vcpu)
+{
+ return false;
+}
+
#endif /* __ARM_KVM_EMULATE_H__ */
diff --git a/arch/arm/kvm/mmio.c b/arch/arm/kvm/mmio.c
index b6e715f..a1009c2 100644
--- a/arch/arm/kvm/mmio.c
+++ b/arch/arm/kvm/mmio.c
@@ -153,7 +153,7 @@ static int decode_hsr(struct kvm_vcpu *vcpu, bool *is_write, int *len)
}
int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
- phys_addr_t fault_ipa)
+ phys_addr_t ipa)
{
unsigned long data;
unsigned long rt;
@@ -182,22 +182,22 @@ int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
data = vcpu_data_guest_to_host(vcpu, vcpu_get_reg(vcpu, rt),
len);
- trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, len, fault_ipa, data);
+ trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, len, ipa, data);
kvm_mmio_write_buf(data_buf, len, data);
- ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, fault_ipa, len,
+ ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, ipa, len,
data_buf);
} else {
trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, len,
- fault_ipa, 0);
+ ipa, 0);
- ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, fault_ipa, len,
+ ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, ipa, len,
data_buf);
}
/* Now prepare kvm_run for the potential return to userland. */
run->mmio.is_write = is_write;
- run->mmio.phys_addr = fault_ipa;
+ run->mmio.phys_addr = ipa;
run->mmio.len = len;
if (!ret) {
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 1677a87..710ae60 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -1072,10 +1072,10 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
return ret;
}
-static bool transparent_hugepage_adjust(kvm_pfn_t *pfnp, phys_addr_t *ipap)
+static bool transparent_hugepage_adjust(kvm_pfn_t *pfnp, gfn_t gfn,
+ phys_addr_t *ipap)
{
kvm_pfn_t pfn = *pfnp;
- gfn_t gfn = *ipap >> PAGE_SHIFT;
if (PageTransCompoundMap(pfn_to_page(pfn))) {
unsigned long mask;
@@ -1291,13 +1291,15 @@ static void coherent_cache_guest_page(struct kvm_vcpu *vcpu, kvm_pfn_t pfn,
}
static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
- struct kvm_memory_slot *memslot, unsigned long hva,
- unsigned long fault_status)
+ struct kvm_s2_trans *nested,
+ struct kvm_memory_slot *memslot,
+ unsigned long hva, unsigned long fault_status)
{
int ret;
bool write_fault, writable, hugetlb = false, force_pte = false;
unsigned long mmu_seq;
- gfn_t gfn = fault_ipa >> PAGE_SHIFT;
+ phys_addr_t ipa = fault_ipa;
+ gfn_t gfn;
struct kvm *kvm = vcpu->kvm;
struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
struct vm_area_struct *vma;
@@ -1323,9 +1325,23 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
return -EFAULT;
}
- if (is_vm_hugetlb_page(vma) && !logging_active) {
+ if (kvm_is_shadow_s2_fault(vcpu)) {
+ ipa = nested->output;
+
+ /*
+ * If we're about to create a shadow stage 2 entry, then we
+ * can only create huge mapings if the guest hypervior also
+ * uses a huge mapping.
+ */
+ if (nested->block_size != PMD_SIZE)
+ force_pte = true;
+ }
+ gfn = ipa >> PAGE_SHIFT;
+
+
+ if (!force_pte && is_vm_hugetlb_page(vma) && !logging_active) {
hugetlb = true;
- gfn = (fault_ipa & PMD_MASK) >> PAGE_SHIFT;
+ gfn = (ipa & PMD_MASK) >> PAGE_SHIFT;
} else {
/*
* Pages belonging to memslots that don't have the same
@@ -1389,7 +1405,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
goto out_unlock;
if (!hugetlb && !force_pte)
- hugetlb = transparent_hugepage_adjust(&pfn, &fault_ipa);
+ hugetlb = transparent_hugepage_adjust(&pfn, gfn, &fault_ipa);
fault_ipa_uncached = memslot->flags & KVM_MEMSLOT_INCOHERENT;
@@ -1435,6 +1451,12 @@ static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
kvm_pfn_t pfn;
bool pfn_valid = false;
+ /*
+ * TODO: Lookup nested S2 pgtable entry and if the access flag is set,
+ * then inject an access fault to the guest and invalidate the shadow
+ * entry.
+ */
+
trace_kvm_access_fault(fault_ipa);
spin_lock(&vcpu->kvm->mmu_lock);
@@ -1478,8 +1500,10 @@ static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
{
unsigned long fault_status;
- phys_addr_t fault_ipa;
+ phys_addr_t fault_ipa; /* The address we faulted on */
+ phys_addr_t ipa; /* Always the IPA in the L1 guest phys space */
struct kvm_memory_slot *memslot;
+ struct kvm_s2_trans nested_trans;
unsigned long hva;
bool is_iabt, write_fault, writable;
gfn_t gfn;
@@ -1491,7 +1515,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
return 1;
}
- fault_ipa = kvm_vcpu_get_fault_ipa(vcpu);
+ ipa = fault_ipa = kvm_vcpu_get_fault_ipa(vcpu);
trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_hsr(vcpu),
kvm_vcpu_get_hfar(vcpu), fault_ipa);
@@ -1500,6 +1524,10 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
fault_status = kvm_vcpu_trap_get_fault_type(vcpu);
if (fault_status != FSC_FAULT && fault_status != FSC_PERM &&
fault_status != FSC_ACCESS) {
+ /*
+ * TODO: Report address size faults from an L2 IPA which
+ * exceeds KVM_PHYS_SIZE to the L1 hypervisor.
+ */
kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n",
kvm_vcpu_trap_get_class(vcpu),
(unsigned long)kvm_vcpu_trap_get_fault(vcpu),
@@ -1509,7 +1537,23 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
idx = srcu_read_lock(&vcpu->kvm->srcu);
- gfn = fault_ipa >> PAGE_SHIFT;
+ /*
+ * We may have faulted on a shadow stage 2 page table if we are
+ * running a nested guest. In this case, we have to resovle the L2
+ * IPA to the L1 IPA first, before knowing what kind of memory should
+ * back the L1 IPA.
+ *
+ * If the shadow stage 2 page table walk faults, then we simply inject
+ * this to the guest and carry on.
+ */
+ if (kvm_is_shadow_s2_fault(vcpu)) {
+ ret = kvm_walk_nested_s2(vcpu, fault_ipa, &nested_trans);
+ if (ret)
+ goto out_unlock;
+ ipa = nested_trans.output;
+ }
+
+ gfn = ipa >> PAGE_SHIFT;
memslot = gfn_to_memslot(vcpu->kvm, gfn);
hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable);
write_fault = kvm_is_write_fault(vcpu);
@@ -1543,13 +1587,13 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
* faulting VA. This is always 12 bits, irrespective
* of the page size.
*/
- fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ((1 << 12) - 1);
- ret = io_mem_abort(vcpu, run, fault_ipa);
+ ipa |= kvm_vcpu_get_hfar(vcpu) & ((1 << 12) - 1);
+ ret = io_mem_abort(vcpu, run, ipa);
goto out_unlock;
}
/* Userspace should not be able to register out-of-bounds IPAs */
- VM_BUG_ON(fault_ipa >= KVM_PHYS_SIZE);
+ VM_BUG_ON(ipa >= KVM_PHYS_SIZE);
if (fault_status == FSC_ACCESS) {
handle_access_fault(vcpu, fault_ipa);
@@ -1557,7 +1601,8 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
goto out_unlock;
}
- ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status);
+ ret = user_mem_abort(vcpu, fault_ipa, &nested_trans,
+ memslot, hva, fault_status);
if (ret == 0)
ret = 1;
out_unlock:
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index abad676..2994410 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -368,4 +368,13 @@ static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu,
return data; /* Leave LE untouched */
}
+static inline bool kvm_is_shadow_s2_fault(struct kvm_vcpu *vcpu)
+{
+#ifdef CONFIG_KVM_ARM_NESTED_HYP
+ return (!vcpu_mode_el2(vcpu)) && vcpu_nested_stage2_enabled(vcpu);
+#else
+ return false;
+#endif
+}
+
#endif /* __ARM64_KVM_EMULATE_H__ */
--
1.9.1
^ permalink raw reply related
* [RFC 44/55] KVM: arm/arm64: Move kvm_is_write_fault to header file
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
Move this little function to the header files for arm/arm64 so other
code can make use of it directly.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/include/asm/kvm_emulate.h | 8 ++++++++
arch/arm/kvm/mmu.c | 8 --------
arch/arm64/include/asm/kvm_emulate.h | 8 ++++++++
3 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h
index dfc53ce..dde5335 100644
--- a/arch/arm/include/asm/kvm_emulate.h
+++ b/arch/arm/include/asm/kvm_emulate.h
@@ -235,6 +235,14 @@ static inline u8 kvm_vcpu_trap_get_fault_type(struct kvm_vcpu *vcpu)
return kvm_vcpu_get_hsr(vcpu) & HSR_FSC_TYPE;
}
+static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
+{
+ if (kvm_vcpu_trap_is_iabt(vcpu))
+ return false;
+
+ return kvm_vcpu_dabt_iswrite(vcpu);
+}
+
static inline u32 kvm_vcpu_hvc_get_imm(struct kvm_vcpu *vcpu)
{
return kvm_vcpu_get_hsr(vcpu) & HSR_HVC_IMM_MASK;
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 710ae60..abdf345 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -1113,14 +1113,6 @@ static bool transparent_hugepage_adjust(kvm_pfn_t *pfnp, gfn_t gfn,
return false;
}
-static bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
-{
- if (kvm_vcpu_trap_is_iabt(vcpu))
- return false;
-
- return kvm_vcpu_dabt_iswrite(vcpu);
-}
-
/**
* stage2_wp_ptes - write protect PMD range
* @pmd: pointer to pmd entry
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 2994410..17f4855 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -285,6 +285,14 @@ static inline u8 kvm_vcpu_trap_get_fault_type(const struct kvm_vcpu *vcpu)
return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_FSC_TYPE;
}
+static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
+{
+ if (kvm_vcpu_trap_is_iabt(vcpu))
+ return false;
+
+ return kvm_vcpu_dabt_iswrite(vcpu);
+}
+
static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu)
{
return vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK;
--
1.9.1
^ permalink raw reply related
* [RFC 45/55] KVM: arm64: KVM: Inject stage-2 page faults
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
Inject stage-2 page faults to the guest hypervisor.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/include/asm/esr.h | 1 +
arch/arm64/kvm/mmu-nested.c | 30 ++++++++++++++++++++++++------
2 files changed, 25 insertions(+), 6 deletions(-)
diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h
index f32e3a7..6104e31 100644
--- a/arch/arm64/include/asm/esr.h
+++ b/arch/arm64/include/asm/esr.h
@@ -107,6 +107,7 @@
#define ESR_ELx_CM (UL(1) << 8)
/* ISS field definitions for exceptions taken in to Hyp */
+#define ESR_ELx_FSC_ADDRSZ (0x00)
#define ESR_ELx_CV (UL(1) << 24)
#define ESR_ELx_COND_SHIFT (20)
#define ESR_ELx_COND_MASK (UL(0xF) << ESR_ELx_COND_SHIFT)
diff --git a/arch/arm64/kvm/mmu-nested.c b/arch/arm64/kvm/mmu-nested.c
index a2fab41..b161b55 100644
--- a/arch/arm64/kvm/mmu-nested.c
+++ b/arch/arm64/kvm/mmu-nested.c
@@ -55,22 +55,40 @@ static unsigned int pa_max(void)
static int vcpu_inject_s2_trans_fault(struct kvm_vcpu *vcpu, gpa_t ipa,
int level)
{
- /* TODO: Implement */
- return -EFAULT;
+ u32 esr;
+
+ vcpu->arch.ctxt.el2_regs[FAR_EL2] = vcpu->arch.fault.far_el2;
+ vcpu->arch.ctxt.el2_regs[HPFAR_EL2] = vcpu->arch.fault.hpfar_el2;
+ esr = kvm_vcpu_get_hsr(vcpu) & ~ESR_ELx_FSC;
+ esr |= ESR_ELx_FSC_FAULT;
+ esr |= level & 0x3;
+ return kvm_inject_nested_sync(vcpu, esr);
}
static int vcpu_inject_s2_addr_sz_fault(struct kvm_vcpu *vcpu, gpa_t ipa,
int level)
{
- /* TODO: Implement */
- return -EFAULT;
+ u32 esr;
+
+ vcpu->arch.ctxt.el2_regs[FAR_EL2] = vcpu->arch.fault.far_el2;
+ vcpu->arch.ctxt.el2_regs[HPFAR_EL2] = vcpu->arch.fault.hpfar_el2;
+ esr = kvm_vcpu_get_hsr(vcpu) & ~ESR_ELx_FSC;
+ esr |= ESR_ELx_FSC_ADDRSZ;
+ esr |= level & 0x3;
+ return kvm_inject_nested_sync(vcpu, esr);
}
static int vcpu_inject_s2_access_flag_fault(struct kvm_vcpu *vcpu, gpa_t ipa,
int level)
{
- /* TODO: Implement */
- return -EFAULT;
+ u32 esr;
+
+ vcpu->arch.ctxt.el2_regs[FAR_EL2] = vcpu->arch.fault.far_el2;
+ vcpu->arch.ctxt.el2_regs[HPFAR_EL2] = vcpu->arch.fault.hpfar_el2;
+ esr = kvm_vcpu_get_hsr(vcpu) & ~ESR_ELx_FSC;
+ esr |= ESR_ELx_FSC_ACCESS;
+ esr |= level & 0x3;
+ return kvm_inject_nested_sync(vcpu, esr);
}
static int check_base_s2_limits(struct kvm_vcpu *vcpu, struct s2_walk_info *wi,
--
1.9.1
^ permalink raw reply related
* [RFC 46/55] KVM: arm64: Add more info to the S2 translation result
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
When translating an L2 IPA to an L1 IPA, we some times need to know at
which level this translation occurred and what the resulting permissions
was, so populate the translation result structure with these additional
fields.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/include/asm/kvm_mmu.h | 3 +++
arch/arm64/kvm/mmu-nested.c | 3 +++
2 files changed, 6 insertions(+)
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index bf94f0c..2ac603d 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -327,6 +327,9 @@ static inline unsigned int kvm_get_vmid_bits(void)
struct kvm_s2_trans {
phys_addr_t output;
phys_addr_t block_size;
+ bool writable;
+ bool readable;
+ int level;
};
#ifdef CONFIG_KVM_ARM_NESTED_HYP
diff --git a/arch/arm64/kvm/mmu-nested.c b/arch/arm64/kvm/mmu-nested.c
index b161b55..b579d23 100644
--- a/arch/arm64/kvm/mmu-nested.c
+++ b/arch/arm64/kvm/mmu-nested.c
@@ -236,6 +236,9 @@ static int walk_nested_s2_pgd(struct kvm_vcpu *vcpu, phys_addr_t ipa,
(ipa & GENMASK_ULL(addr_bottom - 1, 0));
out->output = paddr;
out->block_size = 1UL << ((3 - level) * stride + wi->pgshift);
+ out->readable = desc & (0b01 << 6);
+ out->writable = desc & (0b10 << 6);
+ out->level = level;
return 0;
}
--
1.9.1
^ permalink raw reply related
* [RFC 47/55] KVM: arm/arm64: Forward the guest hypervisor's stage 2 permission faults
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
When faulting on a shadow stage 2 page table we have to check if the
fault was a permission fault and if so, if that fault needs to be
handled by the guest hypervisor before us, in case the guest hypervisor
has created a less permissive S2 entry than the operation required.
Check if this is the case, and inject a fault if it is.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/include/asm/kvm_mmu.h | 7 +++++++
arch/arm/kvm/mmu.c | 5 +++++
arch/arm64/include/asm/kvm_mmu.h | 9 +++++++++
arch/arm64/kvm/mmu-nested.c | 33 +++++++++++++++++++++++++++++++++
4 files changed, 54 insertions(+)
diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
index ab41a10..0d106ae 100644
--- a/arch/arm/include/asm/kvm_mmu.h
+++ b/arch/arm/include/asm/kvm_mmu.h
@@ -241,6 +241,13 @@ static inline int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
return 0;
}
+static inline int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu,
+ phys_addr_t fault_ipa,
+ struct kvm_s2_trans *trans)
+{
+ return 0;
+}
+
static inline void kvm_nested_s2_unmap(struct kvm_vcpu *vcpu) { }
static inline int kvm_nested_s2_init(struct kvm_vcpu *vcpu) { return 0; }
static inline void kvm_nested_s2_teardown(struct kvm_vcpu *vcpu) { }
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index abdf345..68fc8e8 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -1542,6 +1542,11 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
ret = kvm_walk_nested_s2(vcpu, fault_ipa, &nested_trans);
if (ret)
goto out_unlock;
+
+ ret = kvm_s2_handle_perm_fault(vcpu, fault_ipa, &nested_trans);
+ if (ret)
+ goto out_unlock;
+
ipa = nested_trans.output;
}
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 2ac603d..2086296 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -338,6 +338,8 @@ struct kvm_s2_trans {
bool handle_vttbr_update(struct kvm_vcpu *vcpu, u64 vttbr);
int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
struct kvm_s2_trans *result);
+int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
+ struct kvm_s2_trans *trans);
void kvm_nested_s2_unmap(struct kvm_vcpu *vcpu);
int kvm_nested_s2_init(struct kvm_vcpu *vcpu);
void kvm_nested_s2_teardown(struct kvm_vcpu *vcpu);
@@ -366,6 +368,13 @@ static inline int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
return 0;
}
+static inline int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu,
+ phys_addr_t fault_ipa,
+ struct kvm_s2_trans *trans)
+{
+ return 0;
+}
+
static inline void kvm_nested_s2_unmap(struct kvm_vcpu *vcpu) { }
static inline int kvm_nested_s2_init(struct kvm_vcpu *vcpu) { return 0; }
static inline void kvm_nested_s2_teardown(struct kvm_vcpu *vcpu) { }
diff --git a/arch/arm64/kvm/mmu-nested.c b/arch/arm64/kvm/mmu-nested.c
index b579d23..65ad0da 100644
--- a/arch/arm64/kvm/mmu-nested.c
+++ b/arch/arm64/kvm/mmu-nested.c
@@ -52,6 +52,19 @@ static unsigned int pa_max(void)
return ps_to_output_size(parange);
}
+static int vcpu_inject_s2_perm_fault(struct kvm_vcpu *vcpu, gpa_t ipa,
+ int level)
+{
+ u32 esr;
+
+ vcpu->arch.ctxt.el2_regs[FAR_EL2] = vcpu->arch.fault.far_el2;
+ vcpu->arch.ctxt.el2_regs[HPFAR_EL2] = vcpu->arch.fault.hpfar_el2;
+ esr = kvm_vcpu_get_hsr(vcpu) & ~ESR_ELx_FSC;
+ esr |= ESR_ELx_FSC_PERM;
+ esr |= level & 0x3;
+ return kvm_inject_nested_sync(vcpu, esr);
+}
+
static int vcpu_inject_s2_trans_fault(struct kvm_vcpu *vcpu, gpa_t ipa,
int level)
{
@@ -268,6 +281,26 @@ int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
return walk_nested_s2_pgd(vcpu, gipa, &wi, result);
}
+/*
+ * Returns non-zero if permission fault is handled by injecting it to the next
+ * level hypervisor.
+ */
+int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
+ struct kvm_s2_trans *trans)
+{
+ unsigned long fault_status = kvm_vcpu_trap_get_fault_type(vcpu);
+ bool write_fault = kvm_is_write_fault(vcpu);
+
+ if (fault_status != FSC_PERM)
+ return 0;
+
+ if ((write_fault && !trans->writable) ||
+ (!write_fault && !trans->readable))
+ return vcpu_inject_s2_perm_fault(vcpu, fault_ipa, trans->level);
+
+ return 0;
+}
+
/* expects kvm->mmu_lock to be held */
void kvm_nested_s2_all_vcpus_wp(struct kvm *kvm)
{
--
1.9.1
^ permalink raw reply related
* [RFC 48/55] KVM: arm64: Emulate TLBI instruction
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
Currently, we flush ALL shadow stage-2 page tables on the tlbi
instruction execution. We may be able to do this more efficiently by
considering the vttbr_el2 value of the guest hypervisor, but leave it
for now.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/kvm/sys_regs.c | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index ddb641c..b0a057d 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -2013,8 +2013,14 @@ static int emulate_sys_reg(struct kvm_vcpu *vcpu,
static int emulate_tlbi(struct kvm_vcpu *vcpu,
struct sys_reg_params *params)
{
- /* TODO: support tlbi instruction emulation*/
- kvm_inject_undefined(vcpu);
+ /*
+ * We unmap ALL stage-2 page tables on tlbi instruction.
+ * We may make it more efficient by looking at the exact tlbi
+ * instruction.
+ */
+ stage2_unmap_vm(vcpu->kvm);
+ kvm_nested_s2_unmap(vcpu);
+
return 1;
}
--
1.9.1
^ permalink raw reply related
* [RFC 49/55] KVM: arm64: Fixes to toggle_cache for nesting
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
So far we were flushing almost the entire universe whenever a VM would
load/unload the SCTLR_EL1 and the two versions of that register had
different MMU enabled settings. This turned out to be so slow that it
prevented forward progress for a nested VM, because a scheduler timer
tick interrupt would always be pending when we reached the nested VM.
To avoid this problem, we consider the SCTLR_EL2 when evaluating if
caches are on or off when entering virtual EL2 (because this is the
value that we end up shadowing onto the hardware EL1 register).
We also reduce the scope of the flush operation to only flush shadow
stage 2 page table state of the particular VCPU toggling the caches
instead of the shadow stage 2 state of all possible VCPUs.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/kvm/mmu.c | 31 ++++++++++++++++++++++++++++++-
arch/arm64/include/asm/kvm_mmu.h | 7 ++++++-
2 files changed, 36 insertions(+), 2 deletions(-)
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 68fc8e8..344bc01 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -422,6 +422,35 @@ static void stage2_flush_vm(struct kvm *kvm)
srcu_read_unlock(&kvm->srcu, idx);
}
+/**
+ * Same as above but only flushed shadow state for specific vcpu
+ */
+static void stage2_flush_vcpu(struct kvm_vcpu *vcpu)
+{
+ struct kvm *kvm = vcpu->kvm;
+ struct kvm_memslots *slots;
+ struct kvm_memory_slot *memslot;
+ int idx;
+ struct kvm_nested_s2_mmu __maybe_unused *nested_mmu;
+
+ idx = srcu_read_lock(&kvm->srcu);
+ spin_lock(&kvm->mmu_lock);
+
+ slots = kvm_memslots(kvm);
+ kvm_for_each_memslot(memslot, slots)
+ stage2_flush_memslot(&kvm->arch.mmu, memslot);
+
+#ifdef CONFIG_KVM_ARM_NESTED_HYP
+ list_for_each_entry_rcu(nested_mmu, &vcpu->kvm->arch.nested_mmu_list,
+ list) {
+ kvm_stage2_flush_range(&nested_mmu->mmu, 0, KVM_PHYS_SIZE);
+ }
+#endif
+
+ spin_unlock(&kvm->mmu_lock);
+ srcu_read_unlock(&kvm->srcu, idx);
+}
+
static void clear_hyp_pgd_entry(pgd_t *pgd)
{
pud_t *pud_table __maybe_unused = pud_offset(pgd, 0UL);
@@ -2074,7 +2103,7 @@ void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled)
* Clean + invalidate does the trick always.
*/
if (now_enabled != was_enabled)
- stage2_flush_vm(vcpu->kvm);
+ stage2_flush_vcpu(vcpu);
/* Caches are now on, stop trapping VM ops (until a S/W op) */
if (now_enabled)
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 2086296..7754f3e 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -241,7 +241,12 @@ static inline bool kvm_page_empty(void *ptr)
static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
{
- return (vcpu_sys_reg(vcpu, SCTLR_EL1) & 0b101) == 0b101;
+ u32 mode = vcpu->arch.ctxt.gp_regs.regs.pstate & PSR_MODE_MASK;
+
+ if (mode != PSR_MODE_EL2h && mode != PSR_MODE_EL2t)
+ return (vcpu_sys_reg(vcpu, SCTLR_EL1) & 0b101) == 0b101;
+ else
+ return (vcpu_el2_reg(vcpu, SCTLR_EL2) & 0b101) == 0b101;
}
static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu,
--
1.9.1
^ permalink raw reply related
* [RFC 50/55] KVM: arm/arm64: Abstract kvm_phys_addr_ioremap() function
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
The original kvm_phys_addr_ioremap function only uses mmu pointing to
the VM's mmu context. However, it would be very useful to reuse this
function for the nested mmu context. Therefore, create a function named
__kvm_phys_addr_ioremapp which takes mmu as an argument, and let
kvm_phys_addr_ioremap calls this function with the VM's mmu context.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/kvm/mmu.c | 18 +++++++++++++-----
arch/arm64/include/asm/kvm_mmu.h | 3 +++
2 files changed, 16 insertions(+), 5 deletions(-)
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 344bc01..2cd6a19 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -1058,15 +1058,16 @@ static int stage2_pmdp_test_and_clear_young(pmd_t *pmd)
}
/**
- * kvm_phys_addr_ioremap - map a device range to guest IPA
+ * __kvm_phys_addr_ioremap - map a device range to guest IPA
*
* @kvm: The KVM pointer
* @guest_ipa: The IPA at which to insert the mapping
* @pa: The physical address of the device
* @size: The size of the mapping
*/
-int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
- phys_addr_t pa, unsigned long size, bool writable)
+int __kvm_phys_addr_ioremap(struct kvm *kvm, struct kvm_s2_mmu *mmu,
+ phys_addr_t guest_ipa, phys_addr_t pa,
+ unsigned long size, bool writable)
{
phys_addr_t addr, end;
int ret = 0;
@@ -1087,8 +1088,8 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
if (ret)
goto out;
spin_lock(&kvm->mmu_lock);
- ret = stage2_set_pte(&kvm->arch.mmu, &cache, addr, &pte,
- KVM_S2PTE_FLAG_IS_IOMAP);
+ ret = stage2_set_pte(mmu, &cache, addr, &pte,
+ KVM_S2PTE_FLAG_IS_IOMAP);
spin_unlock(&kvm->mmu_lock);
if (ret)
goto out;
@@ -1101,6 +1102,13 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
return ret;
}
+int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
+ phys_addr_t pa, unsigned long size, bool writable)
+{
+ return __kvm_phys_addr_ioremap(kvm, &kvm->arch.mmu, guest_ipa, pa,
+ size, writable);
+}
+
static bool transparent_hugepage_adjust(kvm_pfn_t *pfnp, gfn_t gfn,
phys_addr_t *ipap)
{
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 7754f3e..ec9e5e9 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -149,6 +149,9 @@ static inline unsigned long __kern_hyp_va(unsigned long v)
int __kvm_alloc_stage2_pgd(struct kvm_s2_mmu *mmu);
void kvm_free_stage2_pgd(struct kvm *kvm);
void __kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu);
+int __kvm_phys_addr_ioremap(struct kvm *kvm, struct kvm_s2_mmu *mmu,
+ phys_addr_t guest_ipa, phys_addr_t pa,
+ unsigned long size, bool writable);
int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
phys_addr_t pa, unsigned long size, bool writable);
void kvm_unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start,
--
1.9.1
^ permalink raw reply related
* [RFC 51/55] KVM: arm64: Expose physical address of vcpu interface
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
Expose physical address of vgic virtual cpu interface.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
include/kvm/arm_vgic.h | 1 +
virt/kvm/arm/vgic/vgic-v2.c | 6 ++++++
2 files changed, 7 insertions(+)
diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
index 5bda20c..05c7811 100644
--- a/include/kvm/arm_vgic.h
+++ b/include/kvm/arm_vgic.h
@@ -331,6 +331,7 @@ static inline void vgic_handle_nested_maint_irq(struct kvm_vcpu *vcpu) { }
#define vgic_valid_spi(k, i) (((i) >= VGIC_NR_PRIVATE_IRQS) && \
((i) < (k)->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS))
+phys_addr_t vgic_vcpu_base(void);
bool kvm_vcpu_has_pending_irqs(struct kvm_vcpu *vcpu);
void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu);
void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu);
diff --git a/virt/kvm/arm/vgic/vgic-v2.c b/virt/kvm/arm/vgic/vgic-v2.c
index b8b73fd..5d85041 100644
--- a/virt/kvm/arm/vgic/vgic-v2.c
+++ b/virt/kvm/arm/vgic/vgic-v2.c
@@ -386,3 +386,9 @@ int vgic_v2_probe(const struct gic_kvm_info *info)
return ret;
}
+
+/* Return physical address of vgic virtual cpu interface */
+phys_addr_t vgic_vcpu_base(void)
+{
+ return kvm_vgic_global_state.vcpu_base;
+}
--
1.9.1
^ permalink raw reply related
* [RFC 52/55] KVM: arm/arm64: Create a vcpu mapping for the nested VM
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
Create a mapping from the nested VM's cpu interface to the hardware
virtual cpu interface. This is to allow the nested VM to access virtual
cpu interface directly.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/include/asm/kvm_mmu.h | 3 +++
arch/arm/kvm/mmu.c | 5 +++++
arch/arm64/include/asm/kvm_mmu.h | 5 +++++
arch/arm64/kvm/mmu-nested.c | 26 ++++++++++++++++++++++++++
4 files changed, 39 insertions(+)
diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
index 0d106ae..048a021 100644
--- a/arch/arm/include/asm/kvm_mmu.h
+++ b/arch/arm/include/asm/kvm_mmu.h
@@ -254,6 +254,9 @@ static inline void kvm_nested_s2_teardown(struct kvm_vcpu *vcpu) { }
static inline void kvm_nested_s2_all_vcpus_wp(struct kvm *kvm) { }
static inline void kvm_nested_s2_all_vcpus_unmap(struct kvm *kvm) { }
static inline void kvm_nested_s2_all_vcpus_flush(struct kvm *kvm) { }
+static inline int kvm_nested_mmio_ondemand(struct kvm_vcpu *vcpu,
+ phys_addr_t fault_ipa,
+ phys_addr_t ipa) { return 0; }
static inline u64 kvm_get_vttbr(struct kvm_s2_vmid *vmid,
struct kvm_s2_mmu *mmu)
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 2cd6a19..f7c2911 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -1615,6 +1615,11 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
goto out_unlock;
}
+ if (kvm_nested_mmio_ondemand(vcpu, fault_ipa, ipa)) {
+ ret = 1;
+ goto out_unlock;
+ }
+
/*
* The IPA is reported as [MAX:12], so we need to
* complement it with the bottom 12 bits from the
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index ec9e5e9..ee80a58 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -354,6 +354,8 @@ int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
void kvm_nested_s2_all_vcpus_wp(struct kvm *kvm);
void kvm_nested_s2_all_vcpus_unmap(struct kvm *kvm);
void kvm_nested_s2_all_vcpus_flush(struct kvm *kvm);
+int kvm_nested_mmio_ondemand(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
+ phys_addr_t ipa);
#else
static inline struct kvm_nested_s2_mmu *get_nested_mmu(struct kvm_vcpu *vcpu,
u64 vttbr)
@@ -389,6 +391,9 @@ static inline void kvm_nested_s2_teardown(struct kvm_vcpu *vcpu) { }
static inline void kvm_nested_s2_all_vcpus_wp(struct kvm *kvm) { }
static inline void kvm_nested_s2_all_vcpus_unmap(struct kvm *kvm) { }
static inline void kvm_nested_s2_all_vcpus_flush(struct kvm *kvm) { }
+static inline int kvm_nested_mmio_ondemand(struct kvm_vcpu *vcpu,
+ phys_addr_t fault_ipa,
+ phys_addr_t ipa) { return 0; }
#endif
static inline u64 kvm_get_vttbr(struct kvm_s2_vmid *vmid,
diff --git a/arch/arm64/kvm/mmu-nested.c b/arch/arm64/kvm/mmu-nested.c
index 65ad0da..bce0042 100644
--- a/arch/arm64/kvm/mmu-nested.c
+++ b/arch/arm64/kvm/mmu-nested.c
@@ -473,3 +473,29 @@ bool handle_vttbr_update(struct kvm_vcpu *vcpu, u64 vttbr)
return true;
}
+
+/*
+ * vcpu interface address. This address is supposed to come from the guest's
+ * device tree via QEMU. Here we just hardcoded it, but should be fixed.
+ */
+#define NESTED_VCPU_IF_ADDR 0x08010000
+int kvm_nested_mmio_ondemand(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
+ phys_addr_t ipa)
+{
+ int ret = 0;
+ phys_addr_t vcpu_base = vgic_vcpu_base();
+
+ /* Return if this fault is not from a nested VM */
+ if (vcpu->arch.hw_mmu == &vcpu->kvm->arch.mmu)
+ return ret;
+
+ if (ipa == NESTED_VCPU_IF_ADDR) {
+ ret = __kvm_phys_addr_ioremap(vcpu->kvm, vcpu->arch.hw_mmu,
+ fault_ipa, vcpu_base,
+ KVM_VGIC_V2_CPU_SIZE, true);
+ if (!ret)
+ ret = 1;
+ }
+
+ return ret;
+}
--
1.9.1
^ permalink raw reply related
* [RFC 53/55] KVM: arm64: Reflect shadow VMPIDR_EL2 value to MPIDR_EL1
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
A non-secure EL0 or EL1 read of MPIDR_EL1 should return the value of
VMPIDR_EL2. We emulate this by copying the virtual VMPIDR_EL2 value to
MPIDR_EL1 when entering VM's EL0 or EL1.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/kvm/context.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/arch/arm64/kvm/context.c b/arch/arm64/kvm/context.c
index 9ebc38f..dd79b0e 100644
--- a/arch/arm64/kvm/context.c
+++ b/arch/arm64/kvm/context.c
@@ -173,6 +173,12 @@ void kvm_arm_setup_shadow_state(struct kvm_vcpu *vcpu)
ctxt->hw_pstate = *vcpu_cpsr(vcpu);
ctxt->hw_sys_regs = ctxt->sys_regs;
ctxt->hw_sp_el1 = ctxt->gp_regs.sp_el1;
+
+ /*
+ * A non-secure EL0 or EL1 read of MPIDR_EL1 returns
+ * the value of VMPIDR_EL2.
+ */
+ ctxt->hw_sys_regs[MPIDR_EL1] = ctxt->el2_regs[VMPIDR_EL2];
}
vgic_v2_setup_shadow_state(vcpu);
--
1.9.1
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox