* [RFC 14/55] KVM: arm64: Take account of system instruction traps
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
When HCR.NV bit is set, execution of the EL2 translation regime Address
Translation instructions and TLB maintenance instructions are trapped to
EL2. In addition, execution of the EL1 translation regime Address
Translation instructions and TLB maintenance instructions that are only
accessible from EL2 and above are trapped to EL2. In these cases,
ESR_EL2.EC will be set to 0x18.
Take account of this and handle system instructions as well as MRS/MSR
instructions in the handler. Change the handler name to reflect this.
Emulation of those system instructions is to be done.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/include/asm/kvm_coproc.h | 2 +-
arch/arm64/kvm/handle_exit.c | 2 +-
arch/arm64/kvm/sys_regs.c | 49 ++++++++++++++++++++++++++++++++-----
arch/arm64/kvm/trace.h | 2 +-
4 files changed, 46 insertions(+), 9 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_coproc.h b/arch/arm64/include/asm/kvm_coproc.h
index 0b52377..1b3d21b 100644
--- a/arch/arm64/include/asm/kvm_coproc.h
+++ b/arch/arm64/include/asm/kvm_coproc.h
@@ -43,7 +43,7 @@ void kvm_register_target_sys_reg_table(unsigned int target,
int kvm_handle_cp14_64(struct kvm_vcpu *vcpu, struct kvm_run *run);
int kvm_handle_cp15_32(struct kvm_vcpu *vcpu, struct kvm_run *run);
int kvm_handle_cp15_64(struct kvm_vcpu *vcpu, struct kvm_run *run);
-int kvm_handle_sys_reg(struct kvm_vcpu *vcpu, struct kvm_run *run);
+int kvm_handle_sys(struct kvm_vcpu *vcpu, struct kvm_run *run);
#define kvm_coproc_table_init kvm_sys_reg_table_init
void kvm_sys_reg_table_init(void);
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index 4e4a915..a891684 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -147,7 +147,7 @@ static int kvm_handle_eret(struct kvm_vcpu *vcpu, struct kvm_run *run)
[ESR_ELx_EC_SMC32] = handle_smc,
[ESR_ELx_EC_HVC64] = handle_hvc,
[ESR_ELx_EC_SMC64] = handle_smc,
- [ESR_ELx_EC_SYS64] = kvm_handle_sys_reg,
+ [ESR_ELx_EC_SYS64] = kvm_handle_sys,
[ESR_ELx_EC_ERET] = kvm_handle_eret,
[ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort,
[ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort,
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 4158f2f..202f64d 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -1903,6 +1903,36 @@ static int emulate_sys_reg(struct kvm_vcpu *vcpu,
return 1;
}
+static int emulate_tlbi(struct kvm_vcpu *vcpu,
+ struct sys_reg_params *params)
+{
+ /* TODO: support tlbi instruction emulation*/
+ kvm_inject_undefined(vcpu);
+ return 1;
+}
+
+static int emulate_at(struct kvm_vcpu *vcpu,
+ struct sys_reg_params *params)
+{
+ /* TODO: support address translation instruction emulation */
+ kvm_inject_undefined(vcpu);
+ return 1;
+}
+
+static int emulate_sys_instr(struct kvm_vcpu *vcpu,
+ struct sys_reg_params *params)
+{
+ int ret;
+
+ /* TLB maintenance instructions*/
+ if (params->CRn == 0b1000)
+ ret = emulate_tlbi(vcpu, params);
+ /* Address Translation instructions */
+ else if (params->CRn == 0b0111 && params->CRm == 0b1000)
+ ret = emulate_at(vcpu, params);
+ return ret;
+}
+
static void reset_sys_reg_descs(struct kvm_vcpu *vcpu,
const struct sys_reg_desc *table, size_t num)
{
@@ -1914,18 +1944,19 @@ static void reset_sys_reg_descs(struct kvm_vcpu *vcpu,
}
/**
- * kvm_handle_sys_reg -- handles a mrs/msr trap on a guest sys_reg access
+ * kvm_handle_sys-- handles a system instruction or mrs/msr instruction trap
+ on a guest execution
* @vcpu: The VCPU pointer
* @run: The kvm_run struct
*/
-int kvm_handle_sys_reg(struct kvm_vcpu *vcpu, struct kvm_run *run)
+int kvm_handle_sys(struct kvm_vcpu *vcpu, struct kvm_run *run)
{
struct sys_reg_params params;
unsigned long esr = kvm_vcpu_get_hsr(vcpu);
int Rt = (esr >> 5) & 0x1f;
int ret;
- trace_kvm_handle_sys_reg(esr);
+ trace_kvm_handle_sys(esr);
params.is_aarch32 = false;
params.is_32bit = false;
@@ -1937,10 +1968,16 @@ int kvm_handle_sys_reg(struct kvm_vcpu *vcpu, struct kvm_run *run)
params.regval = vcpu_get_reg(vcpu, Rt);
params.is_write = !(esr & 1);
- ret = emulate_sys_reg(vcpu, ¶ms);
+ if (params.Op0 == 1) {
+ /* System instructions */
+ ret = emulate_sys_instr(vcpu, ¶ms);
+ } else {
+ /* MRS/MSR instructions */
+ ret = emulate_sys_reg(vcpu, ¶ms);
+ if (!params.is_write)
+ vcpu_set_reg(vcpu, Rt, params.regval);
+ }
- if (!params.is_write)
- vcpu_set_reg(vcpu, Rt, params.regval);
return ret;
}
diff --git a/arch/arm64/kvm/trace.h b/arch/arm64/kvm/trace.h
index 5f40987..192708e 100644
--- a/arch/arm64/kvm/trace.h
+++ b/arch/arm64/kvm/trace.h
@@ -134,7 +134,7 @@
TP_printk("%s %s reg %d (0x%08llx)", __entry->fn, __entry->is_write?"write to":"read from", __entry->reg, __entry->write_value)
);
-TRACE_EVENT(kvm_handle_sys_reg,
+TRACE_EVENT(kvm_handle_sys,
TP_PROTO(unsigned long hsr),
TP_ARGS(hsr),
--
1.9.1
^ permalink raw reply related
* [RFC 15/55] KVM: arm64: Trap EL1 VM register accesses in virtual EL2
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
When running in virtual EL2 mode, we actually run the hardware in EL1
and therefore have to use the EL1 registers to ensure correct operation.
By setting the HCR.TVM and HCR.TVRM we ensure that the virtual EL2 mode
doesn't shoot itself in the foot when setting up what it believes to be
a different mode's system register state (for example when preparing to
switch to a VM).
We can leverage the existing sysregs infrastructure to support trapped
accesses to these registers.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/kvm/hyp/switch.c | 2 ++
arch/arm64/kvm/sys_regs.c | 7 ++++++-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index 83037cd..c05c48f 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -82,6 +82,8 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
write_sysreg(1 << 30, fpexc32_el2);
isb();
}
+ if (vcpu_mode_el2(vcpu))
+ val |= HCR_TVM | HCR_TRVM;
write_sysreg(val, hcr_el2);
/* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
write_sysreg(1 << 15, hstr_el2);
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 202f64d..b8e993a 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -101,7 +101,12 @@ static bool access_vm_reg(struct kvm_vcpu *vcpu,
{
bool was_enabled = vcpu_has_cache_enabled(vcpu);
- BUG_ON(!p->is_write);
+ BUG_ON(!vcpu_mode_el2(vcpu) && !p->is_write);
+
+ if (!p->is_write) {
+ p->regval = vcpu_sys_reg(vcpu, r->reg);
+ return true;
+ }
if (!p->is_aarch32) {
vcpu_sys_reg(vcpu, r->reg) = p->regval;
--
1.9.1
^ permalink raw reply related
* [RFC 16/55] KVM: arm64: Forward VM reg traps to the guest hypervisor
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
Forward virtual memory register traps to the guest hypervisor
if it has set corresponding bits to the virtual HCR_EL2.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/kvm/sys_regs.c | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index b8e993a..0f5d21b 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -90,6 +90,23 @@ static bool access_dcsw(struct kvm_vcpu *vcpu,
return true;
}
+static bool forward_vm_traps(struct kvm_vcpu *vcpu, struct sys_reg_params *p)
+{
+ u64 hcr_el2 = vcpu_el2_reg(vcpu, HCR_EL2);
+
+ /* If this is a trap from the virtual EL2, the host handles */
+ if (vcpu_mode_el2(vcpu))
+ return false;
+
+ /* If the guest wants to trap on R/W operation, forward this trap */
+ if ((hcr_el2 & HCR_TVM) && p->is_write)
+ return true;
+ else if ((hcr_el2 & HCR_TRVM) && !p->is_write)
+ return true;
+
+ return false;
+}
+
/*
* Generic accessor for VM registers. Only called as long as HCR_TVM
* is set. If the guest enables the MMU, we stop trapping the VM
@@ -101,6 +118,9 @@ static bool access_vm_reg(struct kvm_vcpu *vcpu,
{
bool was_enabled = vcpu_has_cache_enabled(vcpu);
+ if (forward_vm_traps(vcpu, p))
+ return kvm_inject_nested_sync(vcpu, kvm_vcpu_get_hsr(vcpu));
+
BUG_ON(!vcpu_mode_el2(vcpu) && !p->is_write);
if (!p->is_write) {
--
1.9.1
^ permalink raw reply related
* [RFC 17/55] KVM: arm64: Trap SPSR_EL1, ELR_EL1 and VBAR_EL1 in virtual EL2
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
For the same reason we trap virtual memory register accesses in virtual
EL2, we need to trap SPSR_EL1, ELR_EL1 and VBAR_EL1 accesses. ARM v8.3
introduces the HCR_EL2.NV1 bit to be able to trap on those register
accesses in EL1. Do not set this bit until the whole nesting support is
complete.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/kvm/sys_regs.c | 41 ++++++++++++++++++++++++++++++++++++++++-
1 file changed, 40 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 0f5d21b..19d6a6e 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -898,6 +898,38 @@ static bool access_cntp_cval(struct kvm_vcpu *vcpu,
return true;
}
+static inline void access_rw(struct sys_reg_params *p, u64 *sysreg)
+{
+ if (!p->is_write)
+ p->regval = *sysreg;
+ else
+ *sysreg = p->regval;
+}
+
+static bool access_elr(struct kvm_vcpu *vcpu,
+ struct sys_reg_params *p,
+ const struct sys_reg_desc *r)
+{
+ access_rw(p, &vcpu->arch.ctxt.gp_regs.elr_el1);
+ return true;
+}
+
+static bool access_spsr(struct kvm_vcpu *vcpu,
+ struct sys_reg_params *p,
+ const struct sys_reg_desc *r)
+{
+ access_rw(p, &vcpu->arch.ctxt.gp_regs.spsr[KVM_SPSR_EL1]);
+ return true;
+}
+
+static bool access_vbar(struct kvm_vcpu *vcpu,
+ struct sys_reg_params *p,
+ const struct sys_reg_desc *r)
+{
+ access_rw(p, &vcpu_sys_reg(vcpu, r->reg));
+ return true;
+}
+
static bool trap_el2_reg(struct kvm_vcpu *vcpu,
struct sys_reg_params *p,
const struct sys_reg_desc *r)
@@ -1013,6 +1045,13 @@ static bool trap_el2_reg(struct kvm_vcpu *vcpu,
{ Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b010),
access_vm_reg, reset_val, TCR_EL1, 0 },
+ /* SPSR_EL1 */
+ { Op0(0b11), Op1(0b000), CRn(0b0100), CRm(0b0000), Op2(0b000),
+ access_spsr},
+ /* ELR_EL1 */
+ { Op0(0b11), Op1(0b000), CRn(0b0100), CRm(0b0000), Op2(0b001),
+ access_elr},
+
/* AFSR0_EL1 */
{ Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0001), Op2(0b000),
access_vm_reg, reset_unknown, AFSR0_EL1 },
@@ -1045,7 +1084,7 @@ static bool trap_el2_reg(struct kvm_vcpu *vcpu,
/* VBAR_EL1 */
{ Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b0000), Op2(0b000),
- NULL, reset_val, VBAR_EL1, 0 },
+ access_vbar, reset_val, VBAR_EL1, 0 },
/* ICC_SGI1R_EL1 */
{ Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b1011), Op2(0b101),
--
1.9.1
^ permalink raw reply related
* [RFC 18/55] KVM: arm64: Forward traps due to HCR_EL2.NV1 bit to the guest hypervisor
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
Forward ELR_EL1, SPSR_EL1 and VBAR_EL1 traps to the guest hypervisor if
it has set the NV1 bit to the virtual HCR_EL2. The guest hypervisor
would set this NV1 bit to run a hypervisor in its VM (i.e. another level
of nested hypervisor).
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/include/asm/kvm_arm.h | 1 +
arch/arm64/kvm/sys_regs.c | 17 +++++++++++++++++
2 files changed, 18 insertions(+)
diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h
index 2a2752b..feded61 100644
--- a/arch/arm64/include/asm/kvm_arm.h
+++ b/arch/arm64/include/asm/kvm_arm.h
@@ -23,6 +23,7 @@
#include <asm/types.h>
/* Hyp Configuration Register (HCR) bits */
+#define HCR_NV1 (UL(1) << 43)
#define HCR_E2H (UL(1) << 34)
#define HCR_ID (UL(1) << 33)
#define HCR_CD (UL(1) << 32)
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 19d6a6e..59f9cc6 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -906,10 +906,21 @@ static inline void access_rw(struct sys_reg_params *p, u64 *sysreg)
*sysreg = p->regval;
}
+static bool forward_nv1_traps(struct kvm_vcpu *vcpu, struct sys_reg_params *p)
+{
+ if (!vcpu_mode_el2(vcpu) && (vcpu_el2_reg(vcpu, HCR_EL2) & HCR_NV1))
+ return true;
+
+ return false;
+}
+
static bool access_elr(struct kvm_vcpu *vcpu,
struct sys_reg_params *p,
const struct sys_reg_desc *r)
{
+ if (forward_nv1_traps(vcpu, p))
+ return kvm_inject_nested_sync(vcpu, kvm_vcpu_get_hsr(vcpu));
+
access_rw(p, &vcpu->arch.ctxt.gp_regs.elr_el1);
return true;
}
@@ -918,6 +929,9 @@ static bool access_spsr(struct kvm_vcpu *vcpu,
struct sys_reg_params *p,
const struct sys_reg_desc *r)
{
+ if (forward_nv1_traps(vcpu, p))
+ return kvm_inject_nested_sync(vcpu, kvm_vcpu_get_hsr(vcpu));
+
access_rw(p, &vcpu->arch.ctxt.gp_regs.spsr[KVM_SPSR_EL1]);
return true;
}
@@ -926,6 +940,9 @@ static bool access_vbar(struct kvm_vcpu *vcpu,
struct sys_reg_params *p,
const struct sys_reg_desc *r)
{
+ if (forward_nv1_traps(vcpu, p))
+ return kvm_inject_nested_sync(vcpu, kvm_vcpu_get_hsr(vcpu));
+
access_rw(p, &vcpu_sys_reg(vcpu, r->reg));
return true;
}
--
1.9.1
^ permalink raw reply related
* [RFC 19/55] KVM: arm64: Trap CPACR_EL1 access in virtual EL2
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
For the same reason we trap virtual memory register accesses in virtual
EL2, we trap CPACR_EL1 access too. Basically, we don't want the guest
hypervisor to access the real CPACR_EL1, which is used to emulate
virtual EL2. Instead, we want it to access virtual CPACR_EL1 which is
used to run software in EL0/EL1 from the guest hypervisor's perspective.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/kvm/hyp/switch.c | 10 +++++++---
arch/arm64/kvm/sys_regs.c | 10 +++++++++-
2 files changed, 16 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index c05c48f..b7c8c30 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -41,7 +41,8 @@ bool __hyp_text __fpsimd_enabled(void)
return __fpsimd_is_enabled()();
}
-static void __hyp_text __activate_traps_vhe(void)
+static void __hyp_text __activate_traps_vhe(struct kvm_vcpu *vcpu)
+
{
u64 val;
@@ -53,12 +54,15 @@ static void __hyp_text __activate_traps_vhe(void)
write_sysreg(__kvm_hyp_vector, vbar_el1);
}
-static void __hyp_text __activate_traps_nvhe(void)
+static void __hyp_text __activate_traps_nvhe(struct kvm_vcpu *vcpu)
+
{
u64 val;
val = CPTR_EL2_DEFAULT;
val |= CPTR_EL2_TTA | CPTR_EL2_TFP;
+ if (vcpu_mode_el2(vcpu))
+ val |= CPTR_EL2_TCPAC;
write_sysreg(val, cptr_el2);
}
@@ -90,7 +94,7 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
/* Make sure we trap PMU access from EL0 to EL2 */
write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0);
write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2);
- __activate_traps_arch()();
+ __activate_traps_arch()(vcpu);
}
static void __hyp_text __deactivate_traps_vhe(void)
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 59f9cc6..321ecbc 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -947,6 +947,14 @@ static bool access_vbar(struct kvm_vcpu *vcpu,
return true;
}
+static bool access_cpacr(struct kvm_vcpu *vcpu,
+ struct sys_reg_params *p,
+ const struct sys_reg_desc *r)
+{
+ access_rw(p, &vcpu_sys_reg(vcpu, r->reg));
+ return true;
+}
+
static bool trap_el2_reg(struct kvm_vcpu *vcpu,
struct sys_reg_params *p,
const struct sys_reg_desc *r)
@@ -1051,7 +1059,7 @@ static bool trap_el2_reg(struct kvm_vcpu *vcpu,
access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
/* CPACR_EL1 */
{ Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b010),
- NULL, reset_val, CPACR_EL1, 0 },
+ access_cpacr, reset_val, CPACR_EL1, 0 },
/* TTBR0_EL1 */
{ Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b000),
access_vm_reg, reset_unknown, TTBR0_EL1 },
--
1.9.1
^ permalink raw reply related
* [RFC 20/55] KVM: arm64: Forward CPACR_EL1 traps to the guest hypervisor
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
Forward CPACR_EL1 traps to the guest hypervisor if it has configured the
virtual CPTR_EL2 to do so.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/kvm/sys_regs.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 321ecbc..e66f40d 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -951,6 +951,11 @@ static bool access_cpacr(struct kvm_vcpu *vcpu,
struct sys_reg_params *p,
const struct sys_reg_desc *r)
{
+ /* Forward this trap to the guest hypervisor if it expects */
+ if (!vcpu_mode_el2(vcpu) &&
+ (vcpu_el2_reg(vcpu, CPTR_EL2) & CPTR_EL2_TCPAC))
+ return kvm_inject_nested_sync(vcpu, kvm_vcpu_get_hsr(vcpu));
+
access_rw(p, &vcpu_sys_reg(vcpu, r->reg));
return true;
}
--
1.9.1
^ permalink raw reply related
* [RFC 21/55] KVM: arm64: Forward HVC instruction to the guest hypervisor
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
Forward exceptions due to hvc instruction to the guest hypervisor.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/include/asm/kvm_nested.h | 5 +++++
arch/arm64/kvm/Makefile | 1 +
arch/arm64/kvm/handle_exit.c | 11 +++++++++++
arch/arm64/kvm/handle_exit_nested.c | 27 +++++++++++++++++++++++++++
4 files changed, 44 insertions(+)
create mode 100644 arch/arm64/include/asm/kvm_nested.h
create mode 100644 arch/arm64/kvm/handle_exit_nested.c
diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h
new file mode 100644
index 0000000..620b4d3
--- /dev/null
+++ b/arch/arm64/include/asm/kvm_nested.h
@@ -0,0 +1,5 @@
+#ifndef __ARM64_KVM_NESTED_H__
+#define __ARM64_KVM_NESTED_H__
+
+int handle_hvc_nested(struct kvm_vcpu *vcpu);
+#endif
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index b342bdd..9c35e9a 100644
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -35,4 +35,5 @@ kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/irqchip.o
kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o
kvm-$(CONFIG_KVM_ARM_PMU) += $(KVM)/arm/pmu.o
+kvm-$(CONFIG_KVM_ARM_NESTED_HYP) += handle_exit_nested.o
kvm-$(CONFIG_KVM_ARM_NESTED_HYP) += emulate-nested.o
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index a891684..208be16 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -29,6 +29,10 @@
#include <asm/kvm_mmu.h>
#include <asm/kvm_psci.h>
+#ifdef CONFIG_KVM_ARM_NESTED_HYP
+#include <asm/kvm_nested.h>
+#endif
+
#define CREATE_TRACE_POINTS
#include "trace.h"
@@ -42,6 +46,13 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
kvm_vcpu_hvc_get_imm(vcpu));
vcpu->stat.hvc_exit_stat++;
+#ifdef CONFIG_KVM_ARM_NESTED_HYP
+ ret = handle_hvc_nested(vcpu);
+ if (ret < 0 && ret != -EINVAL)
+ return ret;
+ else if (ret >= 0)
+ return ret;
+#endif
ret = kvm_psci_call(vcpu);
if (ret < 0) {
kvm_inject_undefined(vcpu);
diff --git a/arch/arm64/kvm/handle_exit_nested.c b/arch/arm64/kvm/handle_exit_nested.c
new file mode 100644
index 0000000..a6ce23b
--- /dev/null
+++ b/arch/arm64/kvm/handle_exit_nested.c
@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2016 - Columbia University
+ * Author: Jintack Lim <jintack@cs.columbia.edu>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/kvm.h>
+#include <linux/kvm_host.h>
+
+#include <asm/kvm_emulate.h>
+
+/* We forward all hvc instruction to the guest hypervisor. */
+int handle_hvc_nested(struct kvm_vcpu *vcpu)
+{
+ return kvm_inject_nested_sync(vcpu, kvm_vcpu_get_hsr(vcpu));
+}
--
1.9.1
^ permalink raw reply related
* [RFC 22/55] KVM: arm64: Handle PSCI call from the guest
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
VMs used to execute hvc #0 for the psci call. However, when we come to
provide virtual EL2 to the VM, the host OS inside the VM also calls
kvm_call_hyp which is also hvc #0. So, it's hard to differentiate
between them from the host hypervisor's point of view.
So, let the VM execute smc for the psci call. On ARMv8.3, even if EL3 is
not implemented, a smc instruction executed at non-secure EL1 is trapped
to EL2 if HCR_EL2.TSC==1, rather than being treated as UNDEFINED. So,
the host hypervisor can handle this psci call without any confusion.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/kvm/handle_exit.c | 23 +++++++++++++++++++++--
1 file changed, 21 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index 208be16..ce6d2ef 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -64,8 +64,27 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run)
{
- kvm_inject_undefined(vcpu);
- return 1;
+ int ret;
+
+ /* If imm is non-zero, it's not defined */
+ if (kvm_vcpu_hvc_get_imm(vcpu)) {
+ kvm_inject_undefined(vcpu);
+ return 1;
+ }
+
+ /*
+ * If imm is zero, it's a psci call.
+ * Note that on ARMv8.3, even if EL3 is not implemented, SMC executed
+ * at Non-secure EL1 is trapped to EL2 if HCR_EL2.TSC==1, rather than
+ * being treated as UNDEFINED.
+ */
+ ret = kvm_psci_call(vcpu);
+ if (ret < 0) {
+ kvm_inject_undefined(vcpu);
+ return 1;
+ }
+
+ return ret;
}
/**
--
1.9.1
^ permalink raw reply related
* [RFC 23/55] KVM: arm64: Forward WFX to the guest hypervisor
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
Forward exceptions due to WFI or WFE to the guest hypervisor if the
guest hypervisor has set corresponding virtual HCR_EL2.TWX bits.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/include/asm/kvm_nested.h | 1 +
arch/arm64/kvm/handle_exit.c | 11 ++++++++++-
arch/arm64/kvm/handle_exit_nested.c | 18 ++++++++++++++++++
3 files changed, 29 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h
index 620b4d3..8d36935 100644
--- a/arch/arm64/include/asm/kvm_nested.h
+++ b/arch/arm64/include/asm/kvm_nested.h
@@ -2,4 +2,5 @@
#define __ARM64_KVM_NESTED_H__
int handle_hvc_nested(struct kvm_vcpu *vcpu);
+int handle_wfx_nested(struct kvm_vcpu *vcpu, bool is_wfe);
#endif
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index ce6d2ef..046fdf8 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -101,7 +101,16 @@ static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run)
*/
static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
{
- if (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WFx_ISS_WFE) {
+ bool is_wfe = !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WFx_ISS_WFE);
+#ifdef CONFIG_KVM_ARM_NESTED_HYP
+ int ret = handle_wfx_nested(vcpu, is_wfe);
+
+ if (ret < 0 && ret != -EINVAL)
+ return ret;
+ else if (ret >= 0)
+ return ret;
+#endif
+ if (is_wfe) {
trace_kvm_wfx_arm64(*vcpu_pc(vcpu), true);
vcpu->stat.wfe_exit_stat++;
kvm_vcpu_on_spin(vcpu);
diff --git a/arch/arm64/kvm/handle_exit_nested.c b/arch/arm64/kvm/handle_exit_nested.c
index a6ce23b..871ecfc 100644
--- a/arch/arm64/kvm/handle_exit_nested.c
+++ b/arch/arm64/kvm/handle_exit_nested.c
@@ -25,3 +25,21 @@ int handle_hvc_nested(struct kvm_vcpu *vcpu)
{
return kvm_inject_nested_sync(vcpu, kvm_vcpu_get_hsr(vcpu));
}
+
+/*
+ * Inject wfx to the nested hypervisor if this is from the nested VM and
+ * the virtual HCR_EL2.TWX is set. Otherwise, let the host hypervisor
+ * handle this.
+ */
+int handle_wfx_nested(struct kvm_vcpu *vcpu, bool is_wfe)
+{
+ u64 hcr_el2 = vcpu_el2_reg(vcpu, HCR_EL2);
+
+ if (vcpu_mode_el2(vcpu))
+ return -EINVAL;
+
+ if ((is_wfe && (hcr_el2 & HCR_TWE)) || (!is_wfe && (hcr_el2 & HCR_TWI)))
+ return kvm_inject_nested_sync(vcpu, kvm_vcpu_get_hsr(vcpu));
+
+ return -EINVAL;
+}
--
1.9.1
^ permalink raw reply related
* [RFC 24/55] KVM: arm64: Forward FP exceptions to the guest hypervisor
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
Forward exceptions due to floating-point register accesses to the guest
hypervisor if it has set CPTR_EL2.TFP bit.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/include/asm/kvm_nested.h | 1 +
arch/arm64/kernel/asm-offsets.c | 1 +
arch/arm64/kvm/handle_exit.c | 3 +++
arch/arm64/kvm/handle_exit_nested.c | 6 ++++++
arch/arm64/kvm/hyp/entry.S | 14 ++++++++++++++
arch/arm64/kvm/hyp/hyp-entry.S | 2 +-
6 files changed, 26 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h
index 8d36935..54c5ce5 100644
--- a/arch/arm64/include/asm/kvm_nested.h
+++ b/arch/arm64/include/asm/kvm_nested.h
@@ -3,4 +3,5 @@
int handle_hvc_nested(struct kvm_vcpu *vcpu);
int handle_wfx_nested(struct kvm_vcpu *vcpu, bool is_wfe);
+int kvm_handle_fp_asimd(struct kvm_vcpu *vcpu, struct kvm_run *run);
#endif
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 4a2f0f0..b635f1a 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -131,6 +131,7 @@ int main(void)
DEFINE(CPU_FP_REGS, offsetof(struct kvm_regs, fp_regs));
DEFINE(VCPU_FPEXC32_EL2, offsetof(struct kvm_vcpu, arch.ctxt.sys_regs[FPEXC32_EL2]));
DEFINE(VCPU_HOST_CONTEXT, offsetof(struct kvm_vcpu, arch.host_cpu_context));
+ DEFINE(VIRTUAL_CPTR_EL2, offsetof(struct kvm_vcpu, arch.ctxt.el2_regs[CPTR_EL2]));
#endif
#ifdef CONFIG_CPU_PM
DEFINE(CPU_SUSPEND_SZ, sizeof(struct cpu_suspend_ctx));
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index 046fdf8..308f5c5 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -195,6 +195,9 @@ static int kvm_handle_eret(struct kvm_vcpu *vcpu, struct kvm_run *run)
[ESR_ELx_EC_BREAKPT_LOW]= kvm_handle_guest_debug,
[ESR_ELx_EC_BKPT32] = kvm_handle_guest_debug,
[ESR_ELx_EC_BRK64] = kvm_handle_guest_debug,
+#ifdef CONFIG_KVM_ARM_NESTED_HYP
+ [ESR_ELx_EC_FP_ASIMD] = kvm_handle_fp_asimd,
+#endif
};
static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu)
diff --git a/arch/arm64/kvm/handle_exit_nested.c b/arch/arm64/kvm/handle_exit_nested.c
index 871ecfc..7544c6d 100644
--- a/arch/arm64/kvm/handle_exit_nested.c
+++ b/arch/arm64/kvm/handle_exit_nested.c
@@ -43,3 +43,9 @@ int handle_wfx_nested(struct kvm_vcpu *vcpu, bool is_wfe)
return -EINVAL;
}
+
+/* This is only called when virtual CPTR_EL2.TFP bit is set. */
+int kvm_handle_fp_asimd(struct kvm_vcpu *vcpu, struct kvm_run *run)
+{
+ return kvm_inject_nested_sync(vcpu, kvm_vcpu_get_hsr(vcpu));
+}
diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S
index 12ee62d..a76f102 100644
--- a/arch/arm64/kvm/hyp/entry.S
+++ b/arch/arm64/kvm/hyp/entry.S
@@ -158,6 +158,20 @@ abort_guest_exit_end:
1: ret
ENDPROC(__guest_exit)
+ENTRY(__fpsimd_guest_trap)
+#ifdef CONFIG_KVM_ARM_NESTED_HYP
+// If virtual CPTR_EL2.TFP is set, then foward it to the nested hyp.
+ mrs x1, tpidr_el2
+ ldr x0, [x1, #VIRTUAL_CPTR_EL2]
+ and x0, x0, #CPTR_EL2_TFP
+ cbnz x0, 1f
+#endif
+ b __fpsimd_guest_restore
+1:
+ mov x0, #ARM_EXCEPTION_TRAP
+ b __guest_exit
+ENDPROC(__fpsimd_guest_trap)
+
ENTRY(__fpsimd_guest_restore)
stp x2, x3, [sp, #-16]!
stp x4, lr, [sp, #-16]!
diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S
index 4e92399..d83494b 100644
--- a/arch/arm64/kvm/hyp/hyp-entry.S
+++ b/arch/arm64/kvm/hyp/hyp-entry.S
@@ -108,7 +108,7 @@ el1_trap:
/* Guest accessed VFP/SIMD registers, save host, restore Guest */
cmp x0, #ESR_ELx_EC_FP_ASIMD
- b.eq __fpsimd_guest_restore
+ b.eq __fpsimd_guest_trap
mrs x1, tpidr_el2
mov x0, #ARM_EXCEPTION_TRAP
--
1.9.1
^ permalink raw reply related
* [RFC 25/55] KVM: arm/arm64: Let vcpu thread modify its own active state
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
Currently, if a vcpu thread tries to change its own active state when
the irq is already in AP list, it'll loop forever. Since the VCPU thread
has already synced back LR state to the struct vgic_irq, let it modify
its own state safely.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
virt/kvm/arm/vgic/vgic-mmio.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/virt/kvm/arm/vgic/vgic-mmio.c b/virt/kvm/arm/vgic/vgic-mmio.c
index ebe1b9f..049c570 100644
--- a/virt/kvm/arm/vgic/vgic-mmio.c
+++ b/virt/kvm/arm/vgic/vgic-mmio.c
@@ -192,9 +192,9 @@ static void vgic_mmio_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq,
* If this virtual IRQ was written into a list register, we
* have to make sure the CPU that runs the VCPU thread has
* synced back LR state to the struct vgic_irq. We can only
- * know this for sure, when either this irq is not assigned to
+ * know this for sure, when this irq is not assigned to
* anyone's AP list anymore, or the VCPU thread is not
- * running on any CPUs.
+ * running on any CPUs, or current thread is the VCPU thread.
*
* In the opposite case, we know the VCPU thread may be on its
* way back from the guest and still has to sync back this
@@ -202,6 +202,7 @@ static void vgic_mmio_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq,
* other thread sync back the IRQ.
*/
while (irq->vcpu && /* IRQ may have state in an LR somewhere */
+ irq->vcpu != vcpu && /* Current thread is not the VCPU thread */
irq->vcpu->cpu != -1) /* VCPU thread is running */
cond_resched_lock(&irq->irq_lock);
--
1.9.1
^ permalink raw reply related
* [RFC 26/55] KVM: arm/arm64: Add VGIC data structures for the nesting
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
This adds a couple of extra data structures:
The nested_vgic_vX structures contain the data manipulated by the guest
hypervisor when it faults/traps on accesses to the GICH_ interface.
The shadow_vgic_vX arrays contain the shadow copies of the LRs. That
is, it is a modified version of the nested_vgic_vX->vgic_lr. The reason
why we need a modified version is that for interrupts with the HW bit
set (those for the timer) the interrupt number must be that of the host
hardware number, and not the virtual one programmed by the guest
hypervisor.
The hw_vX_cpu_if pointers point to the registers that the lowvisor (EL2)
code actually copied into hardware when switching to the guest, so at
init time we set:
vgic_cpu->hw_v2_cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
And we should change the vgic-sr function to read the LRs from the
hw_v2_lr pointer.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
include/kvm/arm_vgic.h | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
index 002f092..9a9cb27 100644
--- a/include/kvm/arm_vgic.h
+++ b/include/kvm/arm_vgic.h
@@ -246,6 +246,26 @@ struct vgic_cpu {
unsigned int used_lrs;
struct vgic_irq private_irqs[VGIC_NR_PRIVATE_IRQS];
+ /* CPU vif control registers for the virtual GICH interface */
+ union {
+ struct vgic_v2_cpu_if nested_vgic_v2;
+ struct vgic_v3_cpu_if nested_vgic_v3;
+ };
+
+ /*
+ * The shadow vif control register loaded to the hardware when
+ * running a sted L2 guest with the virtual IMO bit set.
+ */
+ union {
+ struct vgic_v2_cpu_if shadow_vgic_v2;
+ struct vgic_v3_cpu_if shadow_vgic_v3;
+ };
+
+ union {
+ struct vgic_v2_cpu_if *hw_v2_cpu_if;
+ struct vgic_v3_cpu_if *hw_v3_cpu_if;
+ };
+
spinlock_t ap_list_lock; /* Protects the ap_list */
/*
--
1.9.1
^ permalink raw reply related
* [RFC 27/55] KVM: arm/arm64: Emulate GICH interface on GICv2
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
Emulate GICH interface accesses from the guest hypervisor.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
Signed-off-by: Shih-Wei Li <shihwei@cs.columbia.edu>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm64/kvm/Makefile | 1 +
virt/kvm/arm/vgic/vgic-v2-nested.c | 207 +++++++++++++++++++++++++++++++++++++
2 files changed, 208 insertions(+)
create mode 100644 virt/kvm/arm/vgic/vgic-v2-nested.c
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index 9c35e9a..8573faf 100644
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -37,3 +37,4 @@ kvm-$(CONFIG_KVM_ARM_PMU) += $(KVM)/arm/pmu.o
kvm-$(CONFIG_KVM_ARM_NESTED_HYP) += handle_exit_nested.o
kvm-$(CONFIG_KVM_ARM_NESTED_HYP) += emulate-nested.o
+kvm-$(CONFIG_KVM_ARM_NESTED_HYP) += $(KVM)/arm/vgic/vgic-v2-nested.o
diff --git a/virt/kvm/arm/vgic/vgic-v2-nested.c b/virt/kvm/arm/vgic/vgic-v2-nested.c
new file mode 100644
index 0000000..b13128e
--- /dev/null
+++ b/virt/kvm/arm/vgic/vgic-v2-nested.c
@@ -0,0 +1,207 @@
+#include <linux/cpu.h>
+#include <linux/kvm.h>
+#include <linux/kvm_host.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/uaccess.h>
+
+#include <linux/irqchip/arm-gic.h>
+
+#include <asm/kvm_emulate.h>
+#include <asm/kvm_arm.h>
+#include <asm/kvm_mmu.h>
+#include <kvm/arm_vgic.h>
+
+#include "vgic.h"
+#include "vgic-mmio.h"
+
+static inline struct vgic_v2_cpu_if *vcpu_nested_if(struct kvm_vcpu *vcpu)
+{
+ return &vcpu->arch.vgic_cpu.nested_vgic_v2;
+}
+
+static inline struct vgic_v2_cpu_if *vcpu_shadow_if(struct kvm_vcpu *vcpu)
+{
+ return &vcpu->arch.vgic_cpu.shadow_vgic_v2;
+}
+
+static unsigned long vgic_mmio_read_v2_vtr(struct kvm_vcpu *vcpu,
+ gpa_t addr, unsigned int len)
+{
+ u32 reg;
+
+ reg = kvm_vgic_global_state.nr_lr - 1;
+ reg |= 0b100 << 26;
+ reg |= 0b100 << 29;
+
+ return reg;
+}
+
+static inline bool lr_triggers_eoi(u32 lr)
+{
+ return !(lr & (GICH_LR_STATE | GICH_LR_HW)) && (lr & GICH_LR_EOI);
+}
+
+static unsigned long get_eisr(struct kvm_vcpu *vcpu, bool upper_reg)
+{
+ struct vgic_v2_cpu_if *cpu_if = vcpu_nested_if(vcpu);
+ int max_lr = upper_reg ? 64 : 32;
+ int min_lr = upper_reg ? 32 : 0;
+ int nr_lr = min(kvm_vgic_global_state.nr_lr, max_lr);
+ int i;
+ u32 reg = 0;
+
+ for (i = min_lr; i < nr_lr; i++) {
+ if (lr_triggers_eoi(cpu_if->vgic_lr[i]))
+ reg |= BIT(i - min_lr);
+ }
+
+ return reg;
+}
+
+static unsigned long vgic_mmio_read_v2_eisr0(struct kvm_vcpu *vcpu,
+ gpa_t addr, unsigned int len)
+{
+ return get_eisr(vcpu, false);
+}
+
+static unsigned long vgic_mmio_read_v2_eisr1(struct kvm_vcpu *vcpu,
+ gpa_t addr, unsigned int len)
+{
+ return get_eisr(vcpu, true);
+}
+
+static u32 get_elrsr(struct kvm_vcpu *vcpu, bool upper_reg)
+{
+ struct vgic_v2_cpu_if *cpu_if = vcpu_nested_if(vcpu);
+ int max_lr = upper_reg ? 64 : 32;
+ int min_lr = upper_reg ? 32 : 0;
+ int nr_lr = min(kvm_vgic_global_state.nr_lr, max_lr);
+ u32 reg = 0;
+ int i;
+
+ for (i = min_lr; i < nr_lr; i++) {
+ if (!(cpu_if->vgic_lr[i] & GICH_LR_STATE))
+ reg |= BIT(i - min_lr);
+ }
+
+ return reg;
+}
+
+static unsigned long vgic_mmio_read_v2_elrsr0(struct kvm_vcpu *vcpu,
+ gpa_t addr, unsigned int len)
+{
+ return get_elrsr(vcpu, false);
+}
+
+static unsigned long vgic_mmio_read_v2_elrsr1(struct kvm_vcpu *vcpu,
+ gpa_t addr, unsigned int len)
+{
+ return get_elrsr(vcpu, true);
+}
+
+static unsigned long vgic_mmio_read_v2_misr(struct kvm_vcpu *vcpu,
+ gpa_t addr, unsigned int len)
+{
+ struct vgic_v2_cpu_if *cpu_if = vcpu_nested_if(vcpu);
+ int nr_lr = kvm_vgic_global_state.nr_lr;
+ u32 reg = 0;
+
+ if (vgic_mmio_read_v2_eisr0(vcpu, addr, len) ||
+ vgic_mmio_read_v2_eisr1(vcpu, addr, len))
+ reg |= GICH_MISR_EOI;
+
+ if (cpu_if->vgic_hcr & GICH_HCR_UIE) {
+ u32 elrsr0 = vgic_mmio_read_v2_elrsr0(vcpu, addr, len);
+ u32 elrsr1 = vgic_mmio_read_v2_elrsr1(vcpu, addr, len);
+ int used_lrs;
+
+ used_lrs = nr_lr - (hweight32(elrsr0) + hweight32(elrsr1));
+ if (used_lrs <= 1)
+ reg |= GICH_MISR_U;
+ }
+
+ /* TODO: Support remaining bits in this register */
+ return reg;
+}
+
+static unsigned long vgic_mmio_read_v2_gich(struct kvm_vcpu *vcpu,
+ gpa_t addr, unsigned int len)
+{
+ struct vgic_v2_cpu_if *cpu_if = vcpu_nested_if(vcpu);
+ u32 value;
+
+ switch (addr & 0xfff) {
+ case GICH_HCR:
+ value = cpu_if->vgic_hcr;
+ break;
+ case GICH_VMCR:
+ value = cpu_if->vgic_vmcr;
+ break;
+ case GICH_APR:
+ value = cpu_if->vgic_apr;
+ break;
+ case GICH_LR0 ... (GICH_LR0 + 4 * (VGIC_V2_MAX_LRS - 1)):
+ value = cpu_if->vgic_lr[(addr & 0xff) >> 2];
+ break;
+ default:
+ return 0;
+ }
+
+ return value;
+}
+
+static void vgic_mmio_write_v2_gich(struct kvm_vcpu *vcpu,
+ gpa_t addr, unsigned int len,
+ unsigned long val)
+{
+ struct vgic_v2_cpu_if *cpu_if = vcpu_nested_if(vcpu);
+
+ switch (addr & 0xfff) {
+ case GICH_HCR:
+ cpu_if->vgic_hcr = val;
+ break;
+ case GICH_VMCR:
+ cpu_if->vgic_vmcr = val;
+ break;
+ case GICH_APR:
+ cpu_if->vgic_apr = val;
+ break;
+ case GICH_LR0 ... (GICH_LR0 + 4 * (VGIC_V2_MAX_LRS - 1)):
+ cpu_if->vgic_lr[(addr & 0xff) >> 2] = val;
+ break;
+ }
+}
+
+static const struct vgic_register_region vgic_v2_gich_registers[] = {
+ REGISTER_DESC_WITH_LENGTH(GICH_HCR,
+ vgic_mmio_read_v2_gich, vgic_mmio_write_v2_gich, 4,
+ VGIC_ACCESS_32bit),
+ REGISTER_DESC_WITH_LENGTH(GICH_VTR,
+ vgic_mmio_read_v2_vtr, vgic_mmio_write_wi, 4,
+ VGIC_ACCESS_32bit),
+ REGISTER_DESC_WITH_LENGTH(GICH_VMCR,
+ vgic_mmio_read_v2_gich, vgic_mmio_write_v2_gich, 4,
+ VGIC_ACCESS_32bit),
+ REGISTER_DESC_WITH_LENGTH(GICH_MISR,
+ vgic_mmio_read_v2_misr, vgic_mmio_write_wi, 4,
+ VGIC_ACCESS_32bit),
+ REGISTER_DESC_WITH_LENGTH(GICH_EISR0,
+ vgic_mmio_read_v2_eisr0, vgic_mmio_write_wi, 4,
+ VGIC_ACCESS_32bit),
+ REGISTER_DESC_WITH_LENGTH(GICH_EISR1,
+ vgic_mmio_read_v2_eisr1, vgic_mmio_write_wi, 4,
+ VGIC_ACCESS_32bit),
+ REGISTER_DESC_WITH_LENGTH(GICH_ELRSR0,
+ vgic_mmio_read_v2_elrsr0, vgic_mmio_write_wi, 4,
+ VGIC_ACCESS_32bit),
+ REGISTER_DESC_WITH_LENGTH(GICH_ELRSR1,
+ vgic_mmio_read_v2_elrsr1, vgic_mmio_write_wi, 4,
+ VGIC_ACCESS_32bit),
+ REGISTER_DESC_WITH_LENGTH(GICH_APR,
+ vgic_mmio_read_v2_gich, vgic_mmio_write_v2_gich, 4,
+ VGIC_ACCESS_32bit),
+ REGISTER_DESC_WITH_LENGTH(GICH_LR0,
+ vgic_mmio_read_v2_gich, vgic_mmio_write_v2_gich,
+ 4 * VGIC_V2_MAX_LRS, VGIC_ACCESS_32bit),
+};
--
1.9.1
^ permalink raw reply related
* [RFC 28/55] KVM: arm/arm64: Prepare vgic state for the nested VM
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
When entering a nested VM, we set up the hypervisor control interface
based on what the guest hypervisor has set. Especially, we investigate
each list register written by the guest hypervisor whether HW bit is
set. If so, we translate hw irq number from the guest's point of view
to the real hardware irq number if there is a mapping.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/include/asm/kvm_emulate.h | 5 ++
arch/arm64/include/asm/kvm_emulate.h | 5 ++
arch/arm64/kvm/context.c | 4 ++
include/kvm/arm_vgic.h | 8 +++
virt/kvm/arm/vgic/vgic-init.c | 3 ++
virt/kvm/arm/vgic/vgic-v2-nested.c | 99 ++++++++++++++++++++++++++++++++++++
virt/kvm/arm/vgic/vgic.h | 11 ++++
7 files changed, 135 insertions(+)
diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h
index 0fa2f5a..05d5906 100644
--- a/arch/arm/include/asm/kvm_emulate.h
+++ b/arch/arm/include/asm/kvm_emulate.h
@@ -101,6 +101,11 @@ static inline bool vcpu_mode_el2(const struct kvm_vcpu *vcpu)
return false;
}
+static inline bool vcpu_el2_imo_is_set(const struct kvm_vcpu *vcpu)
+{
+ return false;
+}
+
static inline unsigned long *vcpu_pc(struct kvm_vcpu *vcpu)
{
return &vcpu->arch.ctxt.gp_regs.usr_regs.ARM_pc;
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 0987ee4..a9c993f 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -178,6 +178,11 @@ static inline bool vcpu_mode_el2(const struct kvm_vcpu *vcpu)
return mode == PSR_MODE_EL2h || mode == PSR_MODE_EL2t;
}
+static inline bool vcpu_el2_imo_is_set(const struct kvm_vcpu *vcpu)
+{
+ return (vcpu_el2_reg(vcpu, HCR_EL2) & HCR_IMO);
+}
+
static inline u32 kvm_vcpu_get_hsr(const struct kvm_vcpu *vcpu)
{
return vcpu->arch.fault.esr_el2;
diff --git a/arch/arm64/kvm/context.c b/arch/arm64/kvm/context.c
index 0025dd9..7a94c9d 100644
--- a/arch/arm64/kvm/context.c
+++ b/arch/arm64/kvm/context.c
@@ -161,6 +161,8 @@ void kvm_arm_setup_shadow_state(struct kvm_vcpu *vcpu)
ctxt->hw_sys_regs = ctxt->sys_regs;
ctxt->hw_sp_el1 = ctxt->gp_regs.sp_el1;
}
+
+ vgic_v2_setup_shadow_state(vcpu);
}
/**
@@ -179,6 +181,8 @@ void kvm_arm_restore_shadow_state(struct kvm_vcpu *vcpu)
*vcpu_cpsr(vcpu) = ctxt->hw_pstate;
ctxt->gp_regs.sp_el1 = ctxt->hw_sp_el1;
}
+
+ vgic_v2_restore_shadow_state(vcpu);
}
void kvm_arm_init_cpu_context(kvm_cpu_context_t *cpu_ctxt)
diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
index 9a9cb27..484f6b1 100644
--- a/include/kvm/arm_vgic.h
+++ b/include/kvm/arm_vgic.h
@@ -312,6 +312,14 @@ int kvm_vgic_inject_mapped_irq(struct kvm *kvm, int cpuid, unsigned int intid,
int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu);
+#ifdef CONFIG_KVM_ARM_NESTED_HYP
+void vgic_v2_setup_shadow_state(struct kvm_vcpu *vcpu);
+void vgic_v2_restore_shadow_state(struct kvm_vcpu *vcpu);
+#else
+static inline void vgic_v2_setup_shadow_state(struct kvm_vcpu *vcpu) { }
+static inline void vgic_v2_restore_shadow_state(struct kvm_vcpu *vcpu) { }
+#endif
+
#define irqchip_in_kernel(k) (!!((k)->arch.vgic.in_kernel))
#define vgic_initialized(k) ((k)->arch.vgic.initialized)
#define vgic_ready(k) ((k)->arch.vgic.ready)
diff --git a/virt/kvm/arm/vgic/vgic-init.c b/virt/kvm/arm/vgic/vgic-init.c
index 8cebfbc..06ab8a5 100644
--- a/virt/kvm/arm/vgic/vgic-init.c
+++ b/virt/kvm/arm/vgic/vgic-init.c
@@ -216,6 +216,9 @@ static void kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
irq->config = VGIC_CONFIG_LEVEL;
}
}
+
+ vgic_init_nested(vcpu);
+
if (kvm_vgic_global_state.type == VGIC_V2)
vgic_v2_enable(vcpu);
else
diff --git a/virt/kvm/arm/vgic/vgic-v2-nested.c b/virt/kvm/arm/vgic/vgic-v2-nested.c
index b13128e..a992da5 100644
--- a/virt/kvm/arm/vgic/vgic-v2-nested.c
+++ b/virt/kvm/arm/vgic/vgic-v2-nested.c
@@ -205,3 +205,102 @@ static void vgic_mmio_write_v2_gich(struct kvm_vcpu *vcpu,
vgic_mmio_read_v2_gich, vgic_mmio_write_v2_gich,
4 * VGIC_V2_MAX_LRS, VGIC_ACCESS_32bit),
};
+
+/*
+ * For LRs which have HW bit set such as timer interrupts, we modify them to
+ * have the host hardware interrupt number instead of the virtual one programmed
+ * by the guest hypervisor.
+ */
+static void vgic_v2_create_shadow_lr(struct kvm_vcpu *vcpu)
+{
+ int i;
+ struct vgic_v2_cpu_if *cpu_if = vcpu_nested_if(vcpu);
+ struct vgic_v2_cpu_if *s_cpu_if = vcpu_shadow_if(vcpu);
+ struct vgic_irq *irq;
+
+ int nr_lr = kvm_vgic_global_state.nr_lr;
+
+ for (i = 0; i < nr_lr; i++) {
+ u32 lr = cpu_if->vgic_lr[i];
+ int l1_irq;
+
+ if (!(lr & GICH_LR_HW))
+ goto next;
+
+ /* We have the HW bit set */
+ l1_irq = (lr & GICH_LR_PHYSID_CPUID) >>
+ GICH_LR_PHYSID_CPUID_SHIFT;
+ irq = vgic_get_irq(vcpu->kvm, vcpu, l1_irq);
+
+ if (!irq->hw) {
+ /* There was no real mapping, so nuke the HW bit */
+ lr &= ~GICH_LR_HW;
+ vgic_put_irq(vcpu->kvm, irq);
+ goto next;
+ }
+
+ /* Translate the virtual mapping to the real one */
+ lr &= ~GICH_LR_EOI;
+ lr &= ~GICH_LR_PHYSID_CPUID;
+ lr |= irq->hwintid << GICH_LR_PHYSID_CPUID_SHIFT;
+ vgic_put_irq(vcpu->kvm, irq);
+
+next:
+ s_cpu_if->vgic_lr[i] = lr;
+ }
+}
+
+/*
+ * Change the shadow HWIRQ field back to the virtual value before copying over
+ * the entire shadow struct to the nested state.
+ */
+static void vgic_v2_restore_shadow_lr(struct kvm_vcpu *vcpu)
+{
+ struct vgic_v2_cpu_if *cpu_if = vcpu_nested_if(vcpu);
+ struct vgic_v2_cpu_if *s_cpu_if = vcpu_shadow_if(vcpu);
+ int nr_lr = kvm_vgic_global_state.nr_lr;
+ int lr;
+
+ for (lr = 0; lr < nr_lr; lr++) {
+ s_cpu_if->vgic_lr[lr] &= ~GICH_LR_PHYSID_CPUID;
+ s_cpu_if->vgic_lr[lr] |= cpu_if->vgic_lr[lr] &
+ GICH_LR_PHYSID_CPUID;
+ }
+}
+
+void vgic_v2_setup_shadow_state(struct kvm_vcpu *vcpu)
+{
+ struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
+ struct vgic_v2_cpu_if *cpu_if;
+
+ if (vcpu_el2_imo_is_set(vcpu) && !vcpu_mode_el2(vcpu)) {
+ vgic_cpu->shadow_vgic_v2 = vgic_cpu->nested_vgic_v2;
+ vgic_v2_create_shadow_lr(vcpu);
+ cpu_if = vcpu_shadow_if(vcpu);
+ } else {
+ cpu_if = &vgic_cpu->vgic_v2;
+ }
+
+ vgic_cpu->hw_v2_cpu_if = cpu_if;
+}
+
+void vgic_v2_restore_shadow_state(struct kvm_vcpu *vcpu)
+{
+ struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
+
+ /* Not using shadow state: Nothing to do... */
+ if (vgic_cpu->hw_v2_cpu_if == &vgic_cpu->vgic_v2)
+ return;
+
+ /*
+ * Translate the shadow state HW fields back to the virtual ones
+ * before copying the shadow struct back to the nested one.
+ */
+ vgic_v2_restore_shadow_lr(vcpu);
+ vgic_cpu->nested_vgic_v2 = vgic_cpu->shadow_vgic_v2;
+}
+
+void vgic_init_nested(struct kvm_vcpu *vcpu)
+{
+ vgic_v2_setup_shadow_state(vcpu);
+}
diff --git a/virt/kvm/arm/vgic/vgic.h b/virt/kvm/arm/vgic/vgic.h
index 9d9e014..2aef680 100644
--- a/virt/kvm/arm/vgic/vgic.h
+++ b/virt/kvm/arm/vgic/vgic.h
@@ -120,4 +120,15 @@ static inline int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
int vgic_lazy_init(struct kvm *kvm);
int vgic_init(struct kvm *kvm);
+#ifdef CONFIG_KVM_ARM_NESTED_HYP
+void vgic_init_nested(struct kvm_vcpu *vcpu);
+#else
+static inline void vgic_init_nested(struct kvm_vcpu *vcpu)
+{
+ struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
+
+ vgic_cpu->hw_v2_cpu_if = &vgic_cpu->vgic_v2;
+}
+#endif
+
#endif
--
1.9.1
^ permalink raw reply related
* [RFC 29/55] KVM: arm/arm64: Set up the prepared vgic state
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
Since vgic state is properly prepared and is pointed by hw_v2_cpu_if,
let's use it to manipulate vgic.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
virt/kvm/arm/hyp/vgic-v2-sr.c | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/virt/kvm/arm/hyp/vgic-v2-sr.c b/virt/kvm/arm/hyp/vgic-v2-sr.c
index c8aeb7b..5d4898f 100644
--- a/virt/kvm/arm/hyp/vgic-v2-sr.c
+++ b/virt/kvm/arm/hyp/vgic-v2-sr.c
@@ -22,10 +22,15 @@
#include <asm/kvm_emulate.h>
#include <asm/kvm_hyp.h>
+static __hyp_text struct vgic_v2_cpu_if *__hyp_get_cpu_if(struct kvm_vcpu *vcpu)
+{
+ return kern_hyp_va(vcpu->arch.vgic_cpu.hw_v2_cpu_if);
+}
+
static void __hyp_text save_maint_int_state(struct kvm_vcpu *vcpu,
void __iomem *base)
{
- struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
+ struct vgic_v2_cpu_if *cpu_if = __hyp_get_cpu_if(vcpu);
int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
u32 eisr0, eisr1;
int i;
@@ -67,7 +72,7 @@ static void __hyp_text save_maint_int_state(struct kvm_vcpu *vcpu,
static void __hyp_text save_elrsr(struct kvm_vcpu *vcpu, void __iomem *base)
{
- struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
+ struct vgic_v2_cpu_if *cpu_if = __hyp_get_cpu_if(vcpu);
int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
u32 elrsr0, elrsr1;
@@ -86,7 +91,7 @@ static void __hyp_text save_elrsr(struct kvm_vcpu *vcpu, void __iomem *base)
static void __hyp_text save_lrs(struct kvm_vcpu *vcpu, void __iomem *base)
{
- struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
+ struct vgic_v2_cpu_if *cpu_if = __hyp_get_cpu_if(vcpu);
int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
int i;
@@ -107,7 +112,7 @@ static void __hyp_text save_lrs(struct kvm_vcpu *vcpu, void __iomem *base)
void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu)
{
struct kvm *kvm = kern_hyp_va(vcpu->kvm);
- struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
+ struct vgic_v2_cpu_if *cpu_if = __hyp_get_cpu_if(vcpu);
struct vgic_dist *vgic = &kvm->arch.vgic;
void __iomem *base = kern_hyp_va(vgic->vctrl_base);
@@ -138,7 +143,7 @@ void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu)
void __hyp_text __vgic_v2_restore_state(struct kvm_vcpu *vcpu)
{
struct kvm *kvm = kern_hyp_va(vcpu->kvm);
- struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
+ struct vgic_v2_cpu_if *cpu_if = __hyp_get_cpu_if(vcpu);
struct vgic_dist *vgic = &kvm->arch.vgic;
void __iomem *base = kern_hyp_va(vgic->vctrl_base);
int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
--
1.9.1
^ permalink raw reply related
* [RFC 30/55] KVM: arm/arm64: Inject irqs to the guest hypervisor
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
If we have a pending IRQ for the guest and the guest expects IRQs
to be handled in its virtual EL2 mode (the virtual IMO bit is set)
and it is not already running in virtual EL2 mode, then we have to
emulate an IRQ exception.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
virt/kvm/arm/vgic/vgic.c | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/virt/kvm/arm/vgic/vgic.c b/virt/kvm/arm/vgic/vgic.c
index 6440b56..4a98654 100644
--- a/virt/kvm/arm/vgic/vgic.c
+++ b/virt/kvm/arm/vgic/vgic.c
@@ -17,6 +17,7 @@
#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <linux/list_sort.h>
+#include <asm/kvm_emulate.h>
#include "vgic.h"
@@ -652,6 +653,28 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu)
/* Nuke remaining LRs */
for ( ; count < kvm_vgic_global_state.nr_lr; count++)
vgic_clear_lr(vcpu, count);
+
+ /*
+ * If we have any pending IRQ for the guest and the guest expects IRQs
+ * to be handled in its virtual EL2 mode (the virtual IMO bit is set)
+ * and it is not already running in virtual EL2 mode, then we have to
+ * emulate an IRQ exception to virtual IRQ. Note that a pending IRQ
+ * means an irq of which state is pending but not active.
+ */
+ if (vcpu_el2_imo_is_set(vcpu) && !vcpu_mode_el2(vcpu)) {
+ bool pending = false;
+
+ list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
+ spin_lock(&irq->irq_lock);
+ pending = irq->pending && irq->enabled && !irq->active;
+ spin_unlock(&irq->irq_lock);
+
+ if (pending) {
+ kvm_inject_nested_irq(vcpu);
+ break;
+ }
+ }
+ }
}
/* Sync back the hardware VGIC state into our emulation after a guest's run. */
--
1.9.1
^ permalink raw reply related
* [RFC 31/55] KVM: arm/arm64: Inject maintenance interrupts to the guest hypervisor
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
If we exit a nested VM with a pending maintenance interrupt from the
GIC, then we need to forward this to the guest hypervisor so that it can
re-sync the appropriate LRs and sample level triggered interrupts again.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/kvm/context.c | 3 +++
include/kvm/arm_vgic.h | 2 ++
virt/kvm/arm/vgic/vgic-v2-nested.c | 16 ++++++++++++++++
3 files changed, 21 insertions(+)
diff --git a/arch/arm64/kvm/context.c b/arch/arm64/kvm/context.c
index 7a94c9d..a93ffe4 100644
--- a/arch/arm64/kvm/context.c
+++ b/arch/arm64/kvm/context.c
@@ -140,6 +140,9 @@ static void sync_shadow_el1_state(struct kvm_vcpu *vcpu, bool setup)
void kvm_arm_setup_shadow_state(struct kvm_vcpu *vcpu)
{
struct kvm_cpu_context *ctxt = &vcpu->arch.ctxt;
+
+ vgic_handle_nested_maint_irq(vcpu);
+
if (unlikely(vcpu_mode_el2(vcpu))) {
ctxt->hw_pstate = *vcpu_cpsr(vcpu) & ~PSR_MODE_MASK;
diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
index 484f6b1..fc882d6 100644
--- a/include/kvm/arm_vgic.h
+++ b/include/kvm/arm_vgic.h
@@ -315,9 +315,11 @@ int kvm_vgic_inject_mapped_irq(struct kvm *kvm, int cpuid, unsigned int intid,
#ifdef CONFIG_KVM_ARM_NESTED_HYP
void vgic_v2_setup_shadow_state(struct kvm_vcpu *vcpu);
void vgic_v2_restore_shadow_state(struct kvm_vcpu *vcpu);
+void vgic_handle_nested_maint_irq(struct kvm_vcpu *vcpu);
#else
static inline void vgic_v2_setup_shadow_state(struct kvm_vcpu *vcpu) { }
static inline void vgic_v2_restore_shadow_state(struct kvm_vcpu *vcpu) { }
+static inline void vgic_handle_nested_maint_irq(struct kvm_vcpu *vcpu) { }
#endif
#define irqchip_in_kernel(k) (!!((k)->arch.vgic.in_kernel))
diff --git a/virt/kvm/arm/vgic/vgic-v2-nested.c b/virt/kvm/arm/vgic/vgic-v2-nested.c
index a992da5..85f646b 100644
--- a/virt/kvm/arm/vgic/vgic-v2-nested.c
+++ b/virt/kvm/arm/vgic/vgic-v2-nested.c
@@ -300,6 +300,22 @@ void vgic_v2_restore_shadow_state(struct kvm_vcpu *vcpu)
vgic_cpu->nested_vgic_v2 = vgic_cpu->shadow_vgic_v2;
}
+void vgic_handle_nested_maint_irq(struct kvm_vcpu *vcpu)
+{
+ struct vgic_v2_cpu_if *cpu_if = vcpu_nested_if(vcpu);
+
+ /*
+ * If we exit a nested VM with a pending maintenance interrupt from the
+ * GIC, then we need to forward this to the guest hypervisor so that it
+ * can re-sync the appropriate LRs and sample level triggered interrupts
+ * again.
+ */
+ if (vcpu_el2_imo_is_set(vcpu) && !vcpu_mode_el2(vcpu) &&
+ (cpu_if->vgic_hcr & GICH_HCR_EN) &&
+ vgic_mmio_read_v2_misr(vcpu, 0, 0))
+ kvm_inject_nested_irq(vcpu);
+}
+
void vgic_init_nested(struct kvm_vcpu *vcpu)
{
vgic_v2_setup_shadow_state(vcpu);
--
1.9.1
^ permalink raw reply related
* [RFC 32/55] KVM: arm/arm64: register GICH iodev for the guest hypervisor
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
Register a device for the virtual interface control block(GICH) access
from the guest hypervisor.
TODO: Get GICH address from DT, which is hardcoded now.
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/include/uapi/asm/kvm.h | 6 ++++++
include/kvm/arm_vgic.h | 5 ++++-
virt/kvm/arm/vgic/vgic-mmio.c | 6 ++++++
virt/kvm/arm/vgic/vgic-v2-nested.c | 24 ++++++++++++++++++++++++
virt/kvm/arm/vgic/vgic-v2.c | 7 +++++++
virt/kvm/arm/vgic/vgic.h | 6 ++++++
6 files changed, 53 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
index 78117bf..3995d3d 100644
--- a/arch/arm64/include/uapi/asm/kvm.h
+++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -99,6 +99,12 @@ struct kvm_regs {
#define KVM_ARM_VCPU_PMU_V3 3 /* Support guest PMUv3 */
#define KVM_ARM_VCPU_NESTED_VIRT 4 /* Support nested virtual EL2 */
+/* FIXME: This should come from DT */
+#ifdef CONFIG_KVM_ARM_NESTED_HYP
+#define KVM_VGIC_V2_GICH_BASE 0x08030000
+#define KVM_VGIC_V2_GICH_SIZE 0x2000
+#endif
+
struct kvm_vcpu_init {
__u32 target;
__u32 features[7];
diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
index fc882d6..5bda20c 100644
--- a/include/kvm/arm_vgic.h
+++ b/include/kvm/arm_vgic.h
@@ -125,7 +125,8 @@ enum iodev_type {
IODEV_CPUIF,
IODEV_DIST,
IODEV_REDIST,
- IODEV_ITS
+ IODEV_ITS,
+ IODEV_GICH,
};
struct vgic_io_device {
@@ -198,6 +199,8 @@ struct vgic_dist {
struct vgic_io_device dist_iodev;
+ struct vgic_io_device hyp_iodev;
+
bool has_its;
/*
diff --git a/virt/kvm/arm/vgic/vgic-mmio.c b/virt/kvm/arm/vgic/vgic-mmio.c
index 049c570..2e4097d 100644
--- a/virt/kvm/arm/vgic/vgic-mmio.c
+++ b/virt/kvm/arm/vgic/vgic-mmio.c
@@ -512,6 +512,9 @@ static int dispatch_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
case IODEV_ITS:
data = region->its_read(vcpu->kvm, iodev->its, addr, len);
break;
+ case IODEV_GICH:
+ data = region->read(vcpu, addr, len);
+ break;
}
vgic_data_host_to_mmio_bus(val, len, data);
@@ -543,6 +546,9 @@ static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
case IODEV_ITS:
region->its_write(vcpu->kvm, iodev->its, addr, len, data);
break;
+ case IODEV_GICH:
+ region->write(vcpu, addr, len, data);
+ break;
}
return 0;
diff --git a/virt/kvm/arm/vgic/vgic-v2-nested.c b/virt/kvm/arm/vgic/vgic-v2-nested.c
index 85f646b..cb55324 100644
--- a/virt/kvm/arm/vgic/vgic-v2-nested.c
+++ b/virt/kvm/arm/vgic/vgic-v2-nested.c
@@ -206,6 +206,30 @@ static void vgic_mmio_write_v2_gich(struct kvm_vcpu *vcpu,
4 * VGIC_V2_MAX_LRS, VGIC_ACCESS_32bit),
};
+int vgic_register_gich_iodev(struct kvm *kvm, struct vgic_dist *dist)
+{
+ struct vgic_io_device *io_device = &kvm->arch.vgic.hyp_iodev;
+ int ret = 0;
+ unsigned int len;
+
+ len = KVM_VGIC_V2_GICH_SIZE;
+
+ io_device->regions = vgic_v2_gich_registers;
+ io_device->nr_regions = ARRAY_SIZE(vgic_v2_gich_registers);
+ kvm_iodevice_init(&io_device->dev, &kvm_io_gic_ops);
+
+ io_device->base_addr = KVM_VGIC_V2_GICH_BASE;
+ io_device->iodev_type = IODEV_GICH;
+ io_device->redist_vcpu = NULL;
+
+ mutex_lock(&kvm->slots_lock);
+ ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, KVM_VGIC_V2_GICH_BASE,
+ len, &io_device->dev);
+ mutex_unlock(&kvm->slots_lock);
+
+ return ret;
+}
+
/*
* For LRs which have HW bit set such as timer interrupts, we modify them to
* have the host hardware interrupt number instead of the virtual one programmed
diff --git a/virt/kvm/arm/vgic/vgic-v2.c b/virt/kvm/arm/vgic/vgic-v2.c
index 9bab867..b8b73fd 100644
--- a/virt/kvm/arm/vgic/vgic-v2.c
+++ b/virt/kvm/arm/vgic/vgic-v2.c
@@ -280,6 +280,13 @@ int vgic_v2_map_resources(struct kvm *kvm)
goto out;
}
+ /* Register virtual GICH interface to kvm io bus */
+ ret = vgic_register_gich_iodev(kvm, dist);
+ if (ret) {
+ kvm_err("Unable to register VGIC GICH regions\n");
+ goto out;
+ }
+
if (!static_branch_unlikely(&vgic_v2_cpuif_trap)) {
ret = kvm_phys_addr_ioremap(kvm, dist->vgic_cpu_base,
kvm_vgic_global_state.vcpu_base,
diff --git a/virt/kvm/arm/vgic/vgic.h b/virt/kvm/arm/vgic/vgic.h
index 2aef680..11d61a7 100644
--- a/virt/kvm/arm/vgic/vgic.h
+++ b/virt/kvm/arm/vgic/vgic.h
@@ -121,8 +121,14 @@ static inline int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
int vgic_init(struct kvm *kvm);
#ifdef CONFIG_KVM_ARM_NESTED_HYP
+int vgic_register_gich_iodev(struct kvm *kvm, struct vgic_dist *dist);
void vgic_init_nested(struct kvm_vcpu *vcpu);
#else
+static inline int vgic_register_gich_iodev(struct kvm *kvm,
+ struct vgic_dist *dist)
+{
+ return 0;
+}
static inline void vgic_init_nested(struct kvm_vcpu *vcpu)
{
struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
--
1.9.1
^ permalink raw reply related
* [RFC 33/55] KVM: arm/arm64: Remove unused params in mmu functions
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
stage2_flush_xxx functions take a pointer to the kvm struct as the first
parameter but they are never used. Clean this up before modifying mmu
code for nested virtualization support.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/kvm/mmu.c | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index a5265ed..57cb671 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -300,7 +300,7 @@ static void unmap_stage2_range(struct kvm *kvm, phys_addr_t start, u64 size)
} while (pgd++, addr = next, addr != end);
}
-static void stage2_flush_ptes(struct kvm *kvm, pmd_t *pmd,
+static void stage2_flush_ptes(pmd_t *pmd,
phys_addr_t addr, phys_addr_t end)
{
pte_t *pte;
@@ -312,7 +312,7 @@ static void stage2_flush_ptes(struct kvm *kvm, pmd_t *pmd,
} while (pte++, addr += PAGE_SIZE, addr != end);
}
-static void stage2_flush_pmds(struct kvm *kvm, pud_t *pud,
+static void stage2_flush_pmds(pud_t *pud,
phys_addr_t addr, phys_addr_t end)
{
pmd_t *pmd;
@@ -325,12 +325,12 @@ static void stage2_flush_pmds(struct kvm *kvm, pud_t *pud,
if (pmd_thp_or_huge(*pmd))
kvm_flush_dcache_pmd(*pmd);
else
- stage2_flush_ptes(kvm, pmd, addr, next);
+ stage2_flush_ptes(pmd, addr, next);
}
} while (pmd++, addr = next, addr != end);
}
-static void stage2_flush_puds(struct kvm *kvm, pgd_t *pgd,
+static void stage2_flush_puds(pgd_t *pgd,
phys_addr_t addr, phys_addr_t end)
{
pud_t *pud;
@@ -343,7 +343,7 @@ static void stage2_flush_puds(struct kvm *kvm, pgd_t *pgd,
if (stage2_pud_huge(*pud))
kvm_flush_dcache_pud(*pud);
else
- stage2_flush_pmds(kvm, pud, addr, next);
+ stage2_flush_pmds(pud, addr, next);
}
} while (pud++, addr = next, addr != end);
}
@@ -359,7 +359,7 @@ static void stage2_flush_memslot(struct kvm *kvm,
pgd = kvm->arch.pgd + stage2_pgd_index(addr);
do {
next = stage2_pgd_addr_end(addr, end);
- stage2_flush_puds(kvm, pgd, addr, next);
+ stage2_flush_puds(pgd, addr, next);
} while (pgd++, addr = next, addr != end);
}
--
1.9.1
^ permalink raw reply related
* [RFC 34/55] KVM: arm/arm64: Abstract stage-2 MMU state into a separate structure
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
Abstract stage-2 MMU state into a separate structure and change all
callers referring to page tables, VMIDs, and the VTTBR to use this new
indirection.
This is about to become very handy when using shadow stage-2 page
tables.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/include/asm/kvm_asm.h | 7 +-
arch/arm/include/asm/kvm_host.h | 26 ++++---
arch/arm/kvm/arm.c | 34 +++++----
arch/arm/kvm/hyp/switch.c | 5 +-
arch/arm/kvm/hyp/tlb.c | 18 ++---
arch/arm/kvm/mmu.c | 146 +++++++++++++++++++++-----------------
arch/arm64/include/asm/kvm_asm.h | 7 +-
arch/arm64/include/asm/kvm_host.h | 10 ++-
arch/arm64/kvm/hyp/switch.c | 5 +-
arch/arm64/kvm/hyp/tlb.c | 20 +++---
10 files changed, 159 insertions(+), 119 deletions(-)
diff --git a/arch/arm/include/asm/kvm_asm.h b/arch/arm/include/asm/kvm_asm.h
index 8ef0538..36e3856 100644
--- a/arch/arm/include/asm/kvm_asm.h
+++ b/arch/arm/include/asm/kvm_asm.h
@@ -57,6 +57,7 @@
#ifndef __ASSEMBLY__
struct kvm;
struct kvm_vcpu;
+struct kvm_s2_mmu;
extern char __kvm_hyp_init[];
extern char __kvm_hyp_init_end[];
@@ -64,9 +65,9 @@
extern char __kvm_hyp_vector[];
extern void __kvm_flush_vm_context(void);
-extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
-extern void __kvm_tlb_flush_vmid(struct kvm *kvm);
-extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu);
+extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa);
+extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu);
+extern void __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu);
extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index d5423ab..f84a59c 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -53,9 +53,21 @@
int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
void kvm_reset_coprocs(struct kvm_vcpu *vcpu);
-struct kvm_arch {
- /* VTTBR value associated with below pgd and vmid */
+struct kvm_s2_mmu {
+ /* The VMID generation used for the virt. memory system */
+ u64 vmid_gen;
+ u32 vmid;
+
+ /* Stage-2 page table */
+ pgd_t *pgd;
+
+ /* VTTBR value associated with above pgd and vmid */
u64 vttbr;
+};
+
+struct kvm_arch {
+ /* Stage 2 paging state for the VM */
+ struct kvm_s2_mmu mmu;
/* The last vcpu id that ran on each physical CPU */
int __percpu *last_vcpu_ran;
@@ -68,13 +80,6 @@ struct kvm_arch {
* here.
*/
- /* The VMID generation used for the virt. memory system */
- u64 vmid_gen;
- u32 vmid;
-
- /* Stage-2 page table */
- pgd_t *pgd;
-
/* Interrupt controller */
struct vgic_dist vgic;
int max_vcpus;
@@ -188,6 +193,9 @@ struct kvm_vcpu_arch {
/* Detect first run of a vcpu */
bool has_run_once;
+
+ /* Stage 2 paging state used by the hardware on next switch */
+ struct kvm_s2_mmu *hw_mmu;
};
struct kvm_vm_stat {
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 436bf5a..eb3e709 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -139,7 +139,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
kvm_timer_init(kvm);
/* Mark the initial VMID generation invalid */
- kvm->arch.vmid_gen = 0;
+ kvm->arch.mmu.vmid_gen = 0;
/* The maximum number of VCPUs is limited by the host's GIC model */
kvm->arch.max_vcpus = vgic_present ?
@@ -321,6 +321,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
kvm_arm_reset_debug_ptr(vcpu);
+ vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
+
return 0;
}
@@ -335,7 +337,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
* over-invalidation doesn't affect correctness.
*/
if (*last_ran != vcpu->vcpu_id) {
- kvm_call_hyp(__kvm_tlb_flush_local_vmid, vcpu);
+ kvm_call_hyp(__kvm_tlb_flush_local_vmid, &vcpu->kvm->arch.mmu);
*last_ran = vcpu->vcpu_id;
}
@@ -423,25 +425,26 @@ void force_vm_exit(const cpumask_t *mask)
* VMID for the new generation, we must flush necessary caches and TLBs on all
* CPUs.
*/
-static bool need_new_vmid_gen(struct kvm *kvm)
+static bool need_new_vmid_gen(struct kvm_s2_mmu *mmu)
{
- return unlikely(kvm->arch.vmid_gen != atomic64_read(&kvm_vmid_gen));
+ return unlikely(mmu->vmid_gen != atomic64_read(&kvm_vmid_gen));
}
/**
* update_vttbr - Update the VTTBR with a valid VMID before the guest runs
- * @kvm The guest that we are about to run
+ * @kvm: The guest that we are about to run
+ * @mmu: The stage-2 translation context to update
*
* Called from kvm_arch_vcpu_ioctl_run before entering the guest to ensure the
* VM has a valid VMID, otherwise assigns a new one and flushes corresponding
* caches and TLBs.
*/
-static void update_vttbr(struct kvm *kvm)
+static void update_vttbr(struct kvm *kvm, struct kvm_s2_mmu *mmu)
{
phys_addr_t pgd_phys;
u64 vmid;
- if (!need_new_vmid_gen(kvm))
+ if (!need_new_vmid_gen(mmu))
return;
spin_lock(&kvm_vmid_lock);
@@ -451,7 +454,7 @@ static void update_vttbr(struct kvm *kvm)
* already allocated a valid vmid for this vm, then this vcpu should
* use the same vmid.
*/
- if (!need_new_vmid_gen(kvm)) {
+ if (!need_new_vmid_gen(mmu)) {
spin_unlock(&kvm_vmid_lock);
return;
}
@@ -475,16 +478,17 @@ static void update_vttbr(struct kvm *kvm)
kvm_call_hyp(__kvm_flush_vm_context);
}
- kvm->arch.vmid_gen = atomic64_read(&kvm_vmid_gen);
- kvm->arch.vmid = kvm_next_vmid;
+ mmu->vmid_gen = atomic64_read(&kvm_vmid_gen);
+ mmu->vmid = kvm_next_vmid;
kvm_next_vmid++;
kvm_next_vmid &= (1 << kvm_vmid_bits) - 1;
/* update vttbr to be used with the new vmid */
- pgd_phys = virt_to_phys(kvm->arch.pgd);
+ pgd_phys = virt_to_phys(mmu->pgd);
BUG_ON(pgd_phys & ~VTTBR_BADDR_MASK);
- vmid = ((u64)(kvm->arch.vmid) << VTTBR_VMID_SHIFT) & VTTBR_VMID_MASK(kvm_vmid_bits);
- kvm->arch.vttbr = pgd_phys | vmid;
+ vmid = ((u64)(mmu->vmid) << VTTBR_VMID_SHIFT) &
+ VTTBR_VMID_MASK(kvm_vmid_bits);
+ mmu->vttbr = pgd_phys | vmid;
spin_unlock(&kvm_vmid_lock);
}
@@ -611,7 +615,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
*/
cond_resched();
- update_vttbr(vcpu->kvm);
+ update_vttbr(vcpu->kvm, vcpu->arch.hw_mmu);
if (vcpu->arch.power_off || vcpu->arch.pause)
vcpu_sleep(vcpu);
@@ -636,7 +640,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
run->exit_reason = KVM_EXIT_INTR;
}
- if (ret <= 0 || need_new_vmid_gen(vcpu->kvm) ||
+ if (ret <= 0 || need_new_vmid_gen(vcpu->arch.hw_mmu) ||
vcpu->arch.power_off || vcpu->arch.pause) {
local_irq_enable();
kvm_pmu_sync_hwstate(vcpu);
diff --git a/arch/arm/kvm/hyp/switch.c b/arch/arm/kvm/hyp/switch.c
index 92678b7..6f99de1 100644
--- a/arch/arm/kvm/hyp/switch.c
+++ b/arch/arm/kvm/hyp/switch.c
@@ -73,8 +73,9 @@ static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
{
- struct kvm *kvm = kern_hyp_va(vcpu->kvm);
- write_sysreg(kvm->arch.vttbr, VTTBR);
+ struct kvm_s2_mmu *mmu = kern_hyp_va(vcpu->arch.hw_mmu);
+
+ write_sysreg(mmu->vttbr, VTTBR);
write_sysreg(vcpu->arch.midr, VPIDR);
}
diff --git a/arch/arm/kvm/hyp/tlb.c b/arch/arm/kvm/hyp/tlb.c
index 6d810af..56f0a49 100644
--- a/arch/arm/kvm/hyp/tlb.c
+++ b/arch/arm/kvm/hyp/tlb.c
@@ -34,13 +34,13 @@
* As v7 does not support flushing per IPA, just nuke the whole TLB
* instead, ignoring the ipa value.
*/
-void __hyp_text __kvm_tlb_flush_vmid(struct kvm *kvm)
+void __hyp_text __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu)
{
dsb(ishst);
/* Switch to requested VMID */
- kvm = kern_hyp_va(kvm);
- write_sysreg(kvm->arch.vttbr, VTTBR);
+ mmu = kern_hyp_va(mmu);
+ write_sysreg(mmu->vttbr, VTTBR);
isb();
write_sysreg(0, TLBIALLIS);
@@ -50,17 +50,17 @@ void __hyp_text __kvm_tlb_flush_vmid(struct kvm *kvm)
write_sysreg(0, VTTBR);
}
-void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
+void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
+ phys_addr_t ipa)
{
- __kvm_tlb_flush_vmid(kvm);
+ __kvm_tlb_flush_vmid(mmu);
}
-void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu)
+void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu)
{
- struct kvm *kvm = kern_hyp_va(kern_hyp_va(vcpu)->kvm);
-
/* Switch to requested VMID */
- write_sysreg(kvm->arch.vttbr, VTTBR);
+ mmu = kern_hyp_va(mmu);
+ write_sysreg(mmu->vttbr, VTTBR);
isb();
write_sysreg(0, TLBIALL);
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 57cb671..a27a204 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -63,9 +63,9 @@ void kvm_flush_remote_tlbs(struct kvm *kvm)
kvm_call_hyp(__kvm_tlb_flush_vmid, kvm);
}
-static void kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
+static void kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa)
{
- kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, kvm, ipa);
+ kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ipa);
}
/*
@@ -102,13 +102,14 @@ static bool kvm_is_device_pfn(unsigned long pfn)
* Function clears a PMD entry, flushes addr 1st and 2nd stage TLBs. Marks all
* pages in the range dirty.
*/
-static void stage2_dissolve_pmd(struct kvm *kvm, phys_addr_t addr, pmd_t *pmd)
+static void stage2_dissolve_pmd(struct kvm_s2_mmu *mmu, phys_addr_t addr,
+ pmd_t *pmd)
{
if (!pmd_thp_or_huge(*pmd))
return;
pmd_clear(pmd);
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
put_page(virt_to_page(pmd));
}
@@ -144,31 +145,34 @@ static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
return p;
}
-static void clear_stage2_pgd_entry(struct kvm *kvm, pgd_t *pgd, phys_addr_t addr)
+static void clear_stage2_pgd_entry(struct kvm_s2_mmu *mmu,
+ pgd_t *pgd, phys_addr_t addr)
{
pud_t *pud_table __maybe_unused = stage2_pud_offset(pgd, 0UL);
stage2_pgd_clear(pgd);
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
stage2_pud_free(pud_table);
put_page(virt_to_page(pgd));
}
-static void clear_stage2_pud_entry(struct kvm *kvm, pud_t *pud, phys_addr_t addr)
+static void clear_stage2_pud_entry(struct kvm_s2_mmu *mmu,
+ pud_t *pud, phys_addr_t addr)
{
pmd_t *pmd_table __maybe_unused = stage2_pmd_offset(pud, 0);
VM_BUG_ON(stage2_pud_huge(*pud));
stage2_pud_clear(pud);
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
stage2_pmd_free(pmd_table);
put_page(virt_to_page(pud));
}
-static void clear_stage2_pmd_entry(struct kvm *kvm, pmd_t *pmd, phys_addr_t addr)
+static void clear_stage2_pmd_entry(struct kvm_s2_mmu *mmu,
+ pmd_t *pmd, phys_addr_t addr)
{
pte_t *pte_table = pte_offset_kernel(pmd, 0);
VM_BUG_ON(pmd_thp_or_huge(*pmd));
pmd_clear(pmd);
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
pte_free_kernel(NULL, pte_table);
put_page(virt_to_page(pmd));
}
@@ -193,7 +197,7 @@ static void clear_stage2_pmd_entry(struct kvm *kvm, pmd_t *pmd, phys_addr_t addr
* the corresponding TLBs, we call kvm_flush_dcache_p*() to make sure
* the IO subsystem will never hit in the cache.
*/
-static void unmap_stage2_ptes(struct kvm *kvm, pmd_t *pmd,
+static void unmap_stage2_ptes(struct kvm_s2_mmu *mmu, pmd_t *pmd,
phys_addr_t addr, phys_addr_t end)
{
phys_addr_t start_addr = addr;
@@ -205,7 +209,7 @@ static void unmap_stage2_ptes(struct kvm *kvm, pmd_t *pmd,
pte_t old_pte = *pte;
kvm_set_pte(pte, __pte(0));
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
/* No need to invalidate the cache for device mappings */
if (!kvm_is_device_pfn(pte_pfn(old_pte)))
@@ -216,10 +220,10 @@ static void unmap_stage2_ptes(struct kvm *kvm, pmd_t *pmd,
} while (pte++, addr += PAGE_SIZE, addr != end);
if (stage2_pte_table_empty(start_pte))
- clear_stage2_pmd_entry(kvm, pmd, start_addr);
+ clear_stage2_pmd_entry(mmu, pmd, start_addr);
}
-static void unmap_stage2_pmds(struct kvm *kvm, pud_t *pud,
+static void unmap_stage2_pmds(struct kvm_s2_mmu *mmu, pud_t *pud,
phys_addr_t addr, phys_addr_t end)
{
phys_addr_t next, start_addr = addr;
@@ -233,22 +237,22 @@ static void unmap_stage2_pmds(struct kvm *kvm, pud_t *pud,
pmd_t old_pmd = *pmd;
pmd_clear(pmd);
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
kvm_flush_dcache_pmd(old_pmd);
put_page(virt_to_page(pmd));
} else {
- unmap_stage2_ptes(kvm, pmd, addr, next);
+ unmap_stage2_ptes(mmu, pmd, addr, next);
}
}
} while (pmd++, addr = next, addr != end);
if (stage2_pmd_table_empty(start_pmd))
- clear_stage2_pud_entry(kvm, pud, start_addr);
+ clear_stage2_pud_entry(mmu, pud, start_addr);
}
-static void unmap_stage2_puds(struct kvm *kvm, pgd_t *pgd,
+static void unmap_stage2_puds(struct kvm_s2_mmu *mmu, pgd_t *pgd,
phys_addr_t addr, phys_addr_t end)
{
phys_addr_t next, start_addr = addr;
@@ -262,17 +266,17 @@ static void unmap_stage2_puds(struct kvm *kvm, pgd_t *pgd,
pud_t old_pud = *pud;
stage2_pud_clear(pud);
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
kvm_flush_dcache_pud(old_pud);
put_page(virt_to_page(pud));
} else {
- unmap_stage2_pmds(kvm, pud, addr, next);
+ unmap_stage2_pmds(mmu, pud, addr, next);
}
}
} while (pud++, addr = next, addr != end);
if (stage2_pud_table_empty(start_pud))
- clear_stage2_pgd_entry(kvm, pgd, start_addr);
+ clear_stage2_pgd_entry(mmu, pgd, start_addr);
}
/**
@@ -286,17 +290,18 @@ static void unmap_stage2_puds(struct kvm *kvm, pgd_t *pgd,
* destroying the VM), otherwise another faulting VCPU may come in and mess
* with things behind our backs.
*/
-static void unmap_stage2_range(struct kvm *kvm, phys_addr_t start, u64 size)
+static void unmap_stage2_range(struct kvm_s2_mmu *mmu,
+ phys_addr_t start, u64 size)
{
pgd_t *pgd;
phys_addr_t addr = start, end = start + size;
phys_addr_t next;
- pgd = kvm->arch.pgd + stage2_pgd_index(addr);
+ pgd = mmu->pgd + stage2_pgd_index(addr);
do {
next = stage2_pgd_addr_end(addr, end);
if (!stage2_pgd_none(*pgd))
- unmap_stage2_puds(kvm, pgd, addr, next);
+ unmap_stage2_puds(mmu, pgd, addr, next);
} while (pgd++, addr = next, addr != end);
}
@@ -348,7 +353,7 @@ static void stage2_flush_puds(pgd_t *pgd,
} while (pud++, addr = next, addr != end);
}
-static void stage2_flush_memslot(struct kvm *kvm,
+static void stage2_flush_memslot(struct kvm_s2_mmu *mmu,
struct kvm_memory_slot *memslot)
{
phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
@@ -356,7 +361,7 @@ static void stage2_flush_memslot(struct kvm *kvm,
phys_addr_t next;
pgd_t *pgd;
- pgd = kvm->arch.pgd + stage2_pgd_index(addr);
+ pgd = mmu->pgd + stage2_pgd_index(addr);
do {
next = stage2_pgd_addr_end(addr, end);
stage2_flush_puds(pgd, addr, next);
@@ -381,7 +386,7 @@ static void stage2_flush_vm(struct kvm *kvm)
slots = kvm_memslots(kvm);
kvm_for_each_memslot(memslot, slots)
- stage2_flush_memslot(kvm, memslot);
+ stage2_flush_memslot(&kvm->arch.mmu, memslot);
spin_unlock(&kvm->mmu_lock);
srcu_read_unlock(&kvm->srcu, idx);
@@ -733,8 +738,9 @@ int create_hyp_io_mappings(void *from, void *to, phys_addr_t phys_addr)
int kvm_alloc_stage2_pgd(struct kvm *kvm)
{
pgd_t *pgd;
+ struct kvm_s2_mmu *mmu = &kvm->arch.mmu;
- if (kvm->arch.pgd != NULL) {
+ if (mmu->pgd != NULL) {
kvm_err("kvm_arch already initialized?\n");
return -EINVAL;
}
@@ -744,11 +750,12 @@ int kvm_alloc_stage2_pgd(struct kvm *kvm)
if (!pgd)
return -ENOMEM;
- kvm->arch.pgd = pgd;
+ mmu->pgd = pgd;
+
return 0;
}
-static void stage2_unmap_memslot(struct kvm *kvm,
+static void stage2_unmap_memslot(struct kvm_s2_mmu *mmu,
struct kvm_memory_slot *memslot)
{
hva_t hva = memslot->userspace_addr;
@@ -783,7 +790,7 @@ static void stage2_unmap_memslot(struct kvm *kvm,
if (!(vma->vm_flags & VM_PFNMAP)) {
gpa_t gpa = addr + (vm_start - memslot->userspace_addr);
- unmap_stage2_range(kvm, gpa, vm_end - vm_start);
+ unmap_stage2_range(mmu, gpa, vm_end - vm_start);
}
hva = vm_end;
} while (hva < reg_end);
@@ -807,7 +814,7 @@ void stage2_unmap_vm(struct kvm *kvm)
slots = kvm_memslots(kvm);
kvm_for_each_memslot(memslot, slots)
- stage2_unmap_memslot(kvm, memslot);
+ stage2_unmap_memslot(&kvm->arch.mmu, memslot);
spin_unlock(&kvm->mmu_lock);
srcu_read_unlock(&kvm->srcu, idx);
@@ -826,22 +833,25 @@ void stage2_unmap_vm(struct kvm *kvm)
*/
void kvm_free_stage2_pgd(struct kvm *kvm)
{
- if (kvm->arch.pgd == NULL)
+ struct kvm_s2_mmu *mmu = &kvm->arch.mmu;
+
+ if (mmu->pgd == NULL)
return;
- unmap_stage2_range(kvm, 0, KVM_PHYS_SIZE);
+ unmap_stage2_range(mmu, 0, KVM_PHYS_SIZE);
/* Free the HW pgd, one page at a time */
- free_pages_exact(kvm->arch.pgd, S2_PGD_SIZE);
- kvm->arch.pgd = NULL;
+ free_pages_exact(mmu->pgd, S2_PGD_SIZE);
+ mmu->pgd = NULL;
}
-static pud_t *stage2_get_pud(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
+static pud_t *stage2_get_pud(struct kvm_s2_mmu *mmu,
+ struct kvm_mmu_memory_cache *cache,
phys_addr_t addr)
{
pgd_t *pgd;
pud_t *pud;
- pgd = kvm->arch.pgd + stage2_pgd_index(addr);
+ pgd = mmu->pgd + stage2_pgd_index(addr);
if (WARN_ON(stage2_pgd_none(*pgd))) {
if (!cache)
return NULL;
@@ -853,13 +863,14 @@ static pud_t *stage2_get_pud(struct kvm *kvm, struct kvm_mmu_memory_cache *cache
return stage2_pud_offset(pgd, addr);
}
-static pmd_t *stage2_get_pmd(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
+static pmd_t *stage2_get_pmd(struct kvm_s2_mmu *mmu,
+ struct kvm_mmu_memory_cache *cache,
phys_addr_t addr)
{
pud_t *pud;
pmd_t *pmd;
- pud = stage2_get_pud(kvm, cache, addr);
+ pud = stage2_get_pud(mmu, cache, addr);
if (stage2_pud_none(*pud)) {
if (!cache)
return NULL;
@@ -871,12 +882,13 @@ static pmd_t *stage2_get_pmd(struct kvm *kvm, struct kvm_mmu_memory_cache *cache
return stage2_pmd_offset(pud, addr);
}
-static int stage2_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache
+static int stage2_set_pmd_huge(struct kvm_s2_mmu *mmu,
+ struct kvm_mmu_memory_cache
*cache, phys_addr_t addr, const pmd_t *new_pmd)
{
pmd_t *pmd, old_pmd;
- pmd = stage2_get_pmd(kvm, cache, addr);
+ pmd = stage2_get_pmd(mmu, cache, addr);
VM_BUG_ON(!pmd);
/*
@@ -893,7 +905,7 @@ static int stage2_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache
old_pmd = *pmd;
if (pmd_present(old_pmd)) {
pmd_clear(pmd);
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
} else {
get_page(virt_to_page(pmd));
}
@@ -902,7 +914,8 @@ static int stage2_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache
return 0;
}
-static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
+static int stage2_set_pte(struct kvm_s2_mmu *mmu,
+ struct kvm_mmu_memory_cache *cache,
phys_addr_t addr, const pte_t *new_pte,
unsigned long flags)
{
@@ -914,7 +927,7 @@ static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
VM_BUG_ON(logging_active && !cache);
/* Create stage-2 page table mapping - Levels 0 and 1 */
- pmd = stage2_get_pmd(kvm, cache, addr);
+ pmd = stage2_get_pmd(mmu, cache, addr);
if (!pmd) {
/*
* Ignore calls from kvm_set_spte_hva for unallocated
@@ -928,7 +941,7 @@ static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
* allocate page.
*/
if (logging_active)
- stage2_dissolve_pmd(kvm, addr, pmd);
+ stage2_dissolve_pmd(mmu, addr, pmd);
/* Create stage-2 page mappings - Level 2 */
if (pmd_none(*pmd)) {
@@ -948,7 +961,7 @@ static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
old_pte = *pte;
if (pte_present(old_pte)) {
kvm_set_pte(pte, __pte(0));
- kvm_tlb_flush_vmid_ipa(kvm, addr);
+ kvm_tlb_flush_vmid_ipa(mmu, addr);
} else {
get_page(virt_to_page(pte));
}
@@ -1008,7 +1021,7 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
if (ret)
goto out;
spin_lock(&kvm->mmu_lock);
- ret = stage2_set_pte(kvm, &cache, addr, &pte,
+ ret = stage2_set_pte(&kvm->arch.mmu, &cache, addr, &pte,
KVM_S2PTE_FLAG_IS_IOMAP);
spin_unlock(&kvm->mmu_lock);
if (ret)
@@ -1146,12 +1159,13 @@ static void stage2_wp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end)
* @addr: Start address of range
* @end: End address of range
*/
-static void stage2_wp_range(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)
+static void stage2_wp_range(struct kvm *kvm, struct kvm_s2_mmu *mmu,
+ phys_addr_t addr, phys_addr_t end)
{
pgd_t *pgd;
phys_addr_t next;
- pgd = kvm->arch.pgd + stage2_pgd_index(addr);
+ pgd = mmu->pgd + stage2_pgd_index(addr);
do {
/*
* Release kvm_mmu_lock periodically if the memory region is
@@ -1190,7 +1204,7 @@ void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot)
phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;
spin_lock(&kvm->mmu_lock);
- stage2_wp_range(kvm, start, end);
+ stage2_wp_range(kvm, &kvm->arch.mmu, start, end);
spin_unlock(&kvm->mmu_lock);
kvm_flush_remote_tlbs(kvm);
}
@@ -1214,7 +1228,7 @@ static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT;
phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT;
- stage2_wp_range(kvm, start, end);
+ stage2_wp_range(kvm, &kvm->arch.mmu, start, end);
}
/*
@@ -1253,6 +1267,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
bool fault_ipa_uncached;
bool logging_active = memslot_is_logging(memslot);
unsigned long flags = 0;
+ struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu;
write_fault = kvm_is_write_fault(vcpu);
if (fault_status == FSC_PERM && !write_fault) {
@@ -1347,7 +1362,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
kvm_set_pfn_dirty(pfn);
}
coherent_cache_guest_page(vcpu, pfn, PMD_SIZE, fault_ipa_uncached);
- ret = stage2_set_pmd_huge(kvm, memcache, fault_ipa, &new_pmd);
+ ret = stage2_set_pmd_huge(mmu, memcache, fault_ipa, &new_pmd);
} else {
pte_t new_pte = pfn_pte(pfn, mem_type);
@@ -1357,7 +1372,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
mark_page_dirty(kvm, gfn);
}
coherent_cache_guest_page(vcpu, pfn, PAGE_SIZE, fault_ipa_uncached);
- ret = stage2_set_pte(kvm, memcache, fault_ipa, &new_pte, flags);
+ ret = stage2_set_pte(mmu, memcache, fault_ipa, &new_pte, flags);
}
out_unlock:
@@ -1385,7 +1400,7 @@ static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
spin_lock(&vcpu->kvm->mmu_lock);
- pmd = stage2_get_pmd(vcpu->kvm, NULL, fault_ipa);
+ pmd = stage2_get_pmd(vcpu->arch.hw_mmu, NULL, fault_ipa);
if (!pmd || pmd_none(*pmd)) /* Nothing there */
goto out;
@@ -1553,7 +1568,7 @@ static int handle_hva_to_gpa(struct kvm *kvm,
static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
{
- unmap_stage2_range(kvm, gpa, PAGE_SIZE);
+ unmap_stage2_range(&kvm->arch.mmu, gpa, PAGE_SIZE);
return 0;
}
@@ -1561,7 +1576,7 @@ int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
{
unsigned long end = hva + PAGE_SIZE;
- if (!kvm->arch.pgd)
+ if (!kvm->arch.mmu.pgd)
return 0;
trace_kvm_unmap_hva(hva);
@@ -1572,7 +1587,7 @@ int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
int kvm_unmap_hva_range(struct kvm *kvm,
unsigned long start, unsigned long end)
{
- if (!kvm->arch.pgd)
+ if (!kvm->arch.mmu.pgd)
return 0;
trace_kvm_unmap_hva_range(start, end);
@@ -1591,7 +1606,7 @@ static int kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
* therefore stage2_set_pte() never needs to clear out a huge PMD
* through this calling path.
*/
- stage2_set_pte(kvm, NULL, gpa, pte, 0);
+ stage2_set_pte(&kvm->arch.mmu, NULL, gpa, pte, 0);
return 0;
}
@@ -1601,7 +1616,7 @@ void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
unsigned long end = hva + PAGE_SIZE;
pte_t stage2_pte;
- if (!kvm->arch.pgd)
+ if (!kvm->arch.mmu.pgd)
return;
trace_kvm_set_spte_hva(hva);
@@ -1614,7 +1629,7 @@ static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
pmd_t *pmd;
pte_t *pte;
- pmd = stage2_get_pmd(kvm, NULL, gpa);
+ pmd = stage2_get_pmd(&kvm->arch.mmu, NULL, gpa);
if (!pmd || pmd_none(*pmd)) /* Nothing there */
return 0;
@@ -1633,7 +1648,7 @@ static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
pmd_t *pmd;
pte_t *pte;
- pmd = stage2_get_pmd(kvm, NULL, gpa);
+ pmd = stage2_get_pmd(&kvm->arch.mmu, NULL, gpa);
if (!pmd || pmd_none(*pmd)) /* Nothing there */
return 0;
@@ -1864,9 +1879,10 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
spin_lock(&kvm->mmu_lock);
if (ret)
- unmap_stage2_range(kvm, mem->guest_phys_addr, mem->memory_size);
+ unmap_stage2_range(&kvm->arch.mmu, mem->guest_phys_addr,
+ mem->memory_size);
else
- stage2_flush_memslot(kvm, memslot);
+ stage2_flush_memslot(&kvm->arch.mmu, memslot);
spin_unlock(&kvm->mmu_lock);
return ret;
}
@@ -1907,7 +1923,7 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
phys_addr_t size = slot->npages << PAGE_SHIFT;
spin_lock(&kvm->mmu_lock);
- unmap_stage2_range(kvm, gpa, size);
+ unmap_stage2_range(&kvm->arch.mmu, gpa, size);
spin_unlock(&kvm->mmu_lock);
}
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index ec3553eb..ed8139f 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -44,6 +44,7 @@
#ifndef __ASSEMBLY__
struct kvm;
struct kvm_vcpu;
+struct kvm_s2_mmu;
extern char __kvm_hyp_init[];
extern char __kvm_hyp_init_end[];
@@ -52,9 +53,9 @@
extern char __kvm_hyp_vector[];
extern void __kvm_flush_vm_context(void);
-extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
-extern void __kvm_tlb_flush_vmid(struct kvm *kvm);
-extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu);
+extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa);
+extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu);
+extern void __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu);
extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index ed78d73..954d6de 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -50,7 +50,7 @@
int kvm_arch_dev_ioctl_check_extension(struct kvm *kvm, long ext);
void __extended_idmap_trampoline(phys_addr_t boot_pgd, phys_addr_t idmap_start);
-struct kvm_arch {
+struct kvm_s2_mmu {
/* The VMID generation used for the virt. memory system */
u64 vmid_gen;
u32 vmid;
@@ -61,6 +61,11 @@ struct kvm_arch {
/* VTTBR value associated with above pgd and vmid */
u64 vttbr;
+};
+
+struct kvm_arch {
+ /* Stage 2 paging state for the VM */
+ struct kvm_s2_mmu mmu;
/* The last vcpu id that ran on each physical CPU */
int __percpu *last_vcpu_ran;
@@ -326,6 +331,9 @@ struct kvm_vcpu_arch {
/* Detect first run of a vcpu */
bool has_run_once;
+
+ /* Stage 2 paging state used by the hardware on next switch */
+ struct kvm_s2_mmu *hw_mmu;
};
#define vcpu_gp_regs(v) (&(v)->arch.ctxt.gp_regs)
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index b7c8c30..3207009a 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -135,8 +135,9 @@ static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
{
- struct kvm *kvm = kern_hyp_va(vcpu->kvm);
- write_sysreg(kvm->arch.vttbr, vttbr_el2);
+ struct kvm_s2_mmu *mmu = kern_hyp_va(vcpu->arch.hw_mmu);
+
+ write_sysreg(mmu->vttbr, vttbr_el2);
}
static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)
diff --git a/arch/arm64/kvm/hyp/tlb.c b/arch/arm64/kvm/hyp/tlb.c
index 88e2f2b..71a62ea 100644
--- a/arch/arm64/kvm/hyp/tlb.c
+++ b/arch/arm64/kvm/hyp/tlb.c
@@ -17,13 +17,14 @@
#include <asm/kvm_hyp.h>
-void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
+void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
+ phys_addr_t ipa)
{
dsb(ishst);
/* Switch to requested VMID */
- kvm = kern_hyp_va(kvm);
- write_sysreg(kvm->arch.vttbr, vttbr_el2);
+ mmu = kern_hyp_va(mmu);
+ write_sysreg(mmu->vttbr, vttbr_el2);
isb();
/*
@@ -48,13 +49,13 @@ void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
write_sysreg(0, vttbr_el2);
}
-void __hyp_text __kvm_tlb_flush_vmid(struct kvm *kvm)
+void __hyp_text __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu)
{
dsb(ishst);
/* Switch to requested VMID */
- kvm = kern_hyp_va(kvm);
- write_sysreg(kvm->arch.vttbr, vttbr_el2);
+ mmu = kern_hyp_va(mmu);
+ write_sysreg(mmu->vttbr, vttbr_el2);
isb();
asm volatile("tlbi vmalls12e1is" : : );
@@ -64,12 +65,11 @@ void __hyp_text __kvm_tlb_flush_vmid(struct kvm *kvm)
write_sysreg(0, vttbr_el2);
}
-void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu)
+void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu)
{
- struct kvm *kvm = kern_hyp_va(kern_hyp_va(vcpu)->kvm);
-
/* Switch to requested VMID */
- write_sysreg(kvm->arch.vttbr, vttbr_el2);
+ mmu = kern_hyp_va(mmu);
+ write_sysreg(mmu->vttbr, vttbr_el2);
isb();
asm volatile("tlbi vmalle1" : : );
--
1.9.1
^ permalink raw reply related
* [RFC 35/55] KVM: arm/arm64: Support mmu for the virtual EL2 execution
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
When running a guest hypervisor in virtual EL2, the translation context
has to be separate from the rest of the system, including the guest
EL1/0 translation regime, so we allocate a separate VMID for this mode.
Considering that we have two different vttbr values due to separate
VMIDs, it's racy to keep a vttbr value in a struct (kvm_s2_mmu) and
share it between multiple vcpus. So, keep the vttbr value per vcpu.
Hypercalls to flush tlb now have vttbr as a parameter instead of mmu,
since mmu structure does not have vttbr any more.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/include/asm/kvm_asm.h | 6 ++--
arch/arm/include/asm/kvm_emulate.h | 4 +++
arch/arm/include/asm/kvm_host.h | 14 ++++++---
arch/arm/include/asm/kvm_mmu.h | 11 +++++++
arch/arm/kvm/arm.c | 60 +++++++++++++++++++-----------------
arch/arm/kvm/hyp/switch.c | 4 +--
arch/arm/kvm/hyp/tlb.c | 15 ++++-----
arch/arm/kvm/mmu.c | 9 ++++--
arch/arm64/include/asm/kvm_asm.h | 6 ++--
arch/arm64/include/asm/kvm_emulate.h | 8 +++++
arch/arm64/include/asm/kvm_host.h | 14 ++++++---
arch/arm64/include/asm/kvm_mmu.h | 11 +++++++
arch/arm64/kvm/hyp/switch.c | 4 +--
arch/arm64/kvm/hyp/tlb.c | 16 ++++------
14 files changed, 112 insertions(+), 70 deletions(-)
diff --git a/arch/arm/include/asm/kvm_asm.h b/arch/arm/include/asm/kvm_asm.h
index 36e3856..aa214f7 100644
--- a/arch/arm/include/asm/kvm_asm.h
+++ b/arch/arm/include/asm/kvm_asm.h
@@ -65,9 +65,9 @@
extern char __kvm_hyp_vector[];
extern void __kvm_flush_vm_context(void);
-extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa);
-extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu);
-extern void __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu);
+extern void __kvm_tlb_flush_vmid_ipa(u64 vttbr, phys_addr_t ipa);
+extern void __kvm_tlb_flush_vmid(u64 vttbr);
+extern void __kvm_tlb_flush_local_vmid(u64 vttbr);
extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h
index 05d5906..6285f4f 100644
--- a/arch/arm/include/asm/kvm_emulate.h
+++ b/arch/arm/include/asm/kvm_emulate.h
@@ -305,4 +305,8 @@ static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu,
}
}
+static inline struct kvm_s2_vmid *vcpu_get_active_vmid(struct kvm_vcpu *vcpu)
+{
+ return &vcpu->kvm->arch.mmu.vmid;
+}
#endif /* __ARM_KVM_EMULATE_H__ */
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index f84a59c..da45394 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -53,16 +53,18 @@
int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
void kvm_reset_coprocs(struct kvm_vcpu *vcpu);
-struct kvm_s2_mmu {
+struct kvm_s2_vmid {
/* The VMID generation used for the virt. memory system */
u64 vmid_gen;
u32 vmid;
+};
+
+struct kvm_s2_mmu {
+ struct kvm_s2_vmid vmid;
+ struct kvm_s2_vmid el2_vmid;
/* Stage-2 page table */
pgd_t *pgd;
-
- /* VTTBR value associated with above pgd and vmid */
- u64 vttbr;
};
struct kvm_arch {
@@ -196,6 +198,9 @@ struct kvm_vcpu_arch {
/* Stage 2 paging state used by the hardware on next switch */
struct kvm_s2_mmu *hw_mmu;
+
+ /* VTTBR value used by the hardware on next switch */
+ u64 hw_vttbr;
};
struct kvm_vm_stat {
@@ -242,6 +247,7 @@ static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
{
}
+unsigned int get_kvm_vmid_bits(void);
struct kvm_vcpu *kvm_arm_get_running_vcpu(void);
struct kvm_vcpu __percpu **kvm_get_running_vcpus(void);
void kvm_arm_halt_guest(struct kvm *kvm);
diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
index 74a44727..1b3309c 100644
--- a/arch/arm/include/asm/kvm_mmu.h
+++ b/arch/arm/include/asm/kvm_mmu.h
@@ -230,6 +230,17 @@ static inline unsigned int kvm_get_vmid_bits(void)
return 8;
}
+static inline u64 kvm_get_vttbr(struct kvm_s2_vmid *vmid,
+ struct kvm_s2_mmu *mmu)
+{
+ u64 vmid_field, baddr;
+
+ baddr = virt_to_phys(mmu->pgd);
+ vmid_field = ((u64)vmid->vmid << VTTBR_VMID_SHIFT) &
+ VTTBR_VMID_MASK(get_kvm_vmid_bits());
+ return baddr | vmid_field;
+}
+
#endif /* !__ASSEMBLY__ */
#endif /* __ARM_KVM_MMU_H__ */
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index eb3e709..aa8771d 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -75,6 +75,11 @@ static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu)
__this_cpu_write(kvm_arm_running_vcpu, vcpu);
}
+unsigned int get_kvm_vmid_bits(void)
+{
+ return kvm_vmid_bits;
+}
+
/**
* kvm_arm_get_running_vcpu - get the vcpu running on the current CPU.
* Must be called from non-preemptible context
@@ -139,7 +144,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
kvm_timer_init(kvm);
/* Mark the initial VMID generation invalid */
- kvm->arch.mmu.vmid_gen = 0;
+ kvm->arch.mmu.vmid.vmid_gen = 0;
+ kvm->arch.mmu.el2_vmid.vmid_gen = 0;
/* The maximum number of VCPUs is limited by the host's GIC model */
kvm->arch.max_vcpus = vgic_present ?
@@ -312,6 +318,8 @@ void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
{
+ struct kvm_s2_mmu *mmu = &vcpu->kvm->arch.mmu;
+
/* Force users to call KVM_ARM_VCPU_INIT */
vcpu->arch.target = -1;
bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
@@ -321,7 +329,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
kvm_arm_reset_debug_ptr(vcpu);
- vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
+ vcpu->arch.hw_mmu = mmu;
+ vcpu->arch.hw_vttbr = kvm_get_vttbr(&mmu->vmid, mmu);
return 0;
}
@@ -337,7 +346,10 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
* over-invalidation doesn't affect correctness.
*/
if (*last_ran != vcpu->vcpu_id) {
- kvm_call_hyp(__kvm_tlb_flush_local_vmid, &vcpu->kvm->arch.mmu);
+ struct kvm_s2_mmu *mmu = &vcpu->kvm->arch.mmu;
+ u64 vttbr = kvm_get_vttbr(&mmu->vmid, mmu);
+
+ kvm_call_hyp(__kvm_tlb_flush_local_vmid, vttbr);
*last_ran = vcpu->vcpu_id;
}
@@ -415,36 +427,33 @@ void force_vm_exit(const cpumask_t *mask)
/**
* need_new_vmid_gen - check that the VMID is still valid
- * @kvm: The VM's VMID to check
+ * @vmid: The VMID to check
*
* return true if there is a new generation of VMIDs being used
*
- * The hardware supports only 256 values with the value zero reserved for the
- * host, so we check if an assigned value belongs to a previous generation,
- * which which requires us to assign a new value. If we're the first to use a
- * VMID for the new generation, we must flush necessary caches and TLBs on all
- * CPUs.
+ * The hardware supports a limited set of values with the value zero reserved
+ * for the host, so we check if an assigned value belongs to a previous
+ * generation, which which requires us to assign a new value. If we're the
+ * first to use a VMID for the new generation, we must flush necessary caches
+ * and TLBs on all CPUs.
*/
-static bool need_new_vmid_gen(struct kvm_s2_mmu *mmu)
+static bool need_new_vmid_gen(struct kvm_s2_vmid *vmid)
{
- return unlikely(mmu->vmid_gen != atomic64_read(&kvm_vmid_gen));
+ return unlikely(vmid->vmid_gen != atomic64_read(&kvm_vmid_gen));
}
/**
* update_vttbr - Update the VTTBR with a valid VMID before the guest runs
* @kvm: The guest that we are about to run
- * @mmu: The stage-2 translation context to update
+ * @vmid: The stage-2 VMID information struct
*
* Called from kvm_arch_vcpu_ioctl_run before entering the guest to ensure the
* VM has a valid VMID, otherwise assigns a new one and flushes corresponding
* caches and TLBs.
*/
-static void update_vttbr(struct kvm *kvm, struct kvm_s2_mmu *mmu)
+static void update_vttbr(struct kvm *kvm, struct kvm_s2_vmid *vmid)
{
- phys_addr_t pgd_phys;
- u64 vmid;
-
- if (!need_new_vmid_gen(mmu))
+ if (!need_new_vmid_gen(vmid))
return;
spin_lock(&kvm_vmid_lock);
@@ -454,7 +463,7 @@ static void update_vttbr(struct kvm *kvm, struct kvm_s2_mmu *mmu)
* already allocated a valid vmid for this vm, then this vcpu should
* use the same vmid.
*/
- if (!need_new_vmid_gen(mmu)) {
+ if (!need_new_vmid_gen(vmid)) {
spin_unlock(&kvm_vmid_lock);
return;
}
@@ -478,18 +487,11 @@ static void update_vttbr(struct kvm *kvm, struct kvm_s2_mmu *mmu)
kvm_call_hyp(__kvm_flush_vm_context);
}
- mmu->vmid_gen = atomic64_read(&kvm_vmid_gen);
- mmu->vmid = kvm_next_vmid;
+ vmid->vmid_gen = atomic64_read(&kvm_vmid_gen);
+ vmid->vmid = kvm_next_vmid;
kvm_next_vmid++;
kvm_next_vmid &= (1 << kvm_vmid_bits) - 1;
- /* update vttbr to be used with the new vmid */
- pgd_phys = virt_to_phys(mmu->pgd);
- BUG_ON(pgd_phys & ~VTTBR_BADDR_MASK);
- vmid = ((u64)(mmu->vmid) << VTTBR_VMID_SHIFT) &
- VTTBR_VMID_MASK(kvm_vmid_bits);
- mmu->vttbr = pgd_phys | vmid;
-
spin_unlock(&kvm_vmid_lock);
}
@@ -615,7 +617,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
*/
cond_resched();
- update_vttbr(vcpu->kvm, vcpu->arch.hw_mmu);
+ update_vttbr(vcpu->kvm, vcpu_get_active_vmid(vcpu));
if (vcpu->arch.power_off || vcpu->arch.pause)
vcpu_sleep(vcpu);
@@ -640,7 +642,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
run->exit_reason = KVM_EXIT_INTR;
}
- if (ret <= 0 || need_new_vmid_gen(vcpu->arch.hw_mmu) ||
+ if (ret <= 0 || need_new_vmid_gen(vcpu_get_active_vmid(vcpu)) ||
vcpu->arch.power_off || vcpu->arch.pause) {
local_irq_enable();
kvm_pmu_sync_hwstate(vcpu);
diff --git a/arch/arm/kvm/hyp/switch.c b/arch/arm/kvm/hyp/switch.c
index 6f99de1..65d0b5b 100644
--- a/arch/arm/kvm/hyp/switch.c
+++ b/arch/arm/kvm/hyp/switch.c
@@ -73,9 +73,7 @@ static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
{
- struct kvm_s2_mmu *mmu = kern_hyp_va(vcpu->arch.hw_mmu);
-
- write_sysreg(mmu->vttbr, VTTBR);
+ write_sysreg(vcpu->arch.hw_vttbr, VTTBR);
write_sysreg(vcpu->arch.midr, VPIDR);
}
diff --git a/arch/arm/kvm/hyp/tlb.c b/arch/arm/kvm/hyp/tlb.c
index 56f0a49..562ad0b 100644
--- a/arch/arm/kvm/hyp/tlb.c
+++ b/arch/arm/kvm/hyp/tlb.c
@@ -34,13 +34,12 @@
* As v7 does not support flushing per IPA, just nuke the whole TLB
* instead, ignoring the ipa value.
*/
-void __hyp_text __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu)
+void __hyp_text __kvm_tlb_flush_vmid(u64 vttbr)
{
dsb(ishst);
/* Switch to requested VMID */
- mmu = kern_hyp_va(mmu);
- write_sysreg(mmu->vttbr, VTTBR);
+ write_sysreg(vttbr, VTTBR);
isb();
write_sysreg(0, TLBIALLIS);
@@ -50,17 +49,15 @@ void __hyp_text __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu)
write_sysreg(0, VTTBR);
}
-void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
- phys_addr_t ipa)
+void __hyp_text __kvm_tlb_flush_vmid_ipa(u64 vttbr, phys_addr_t ipa)
{
- __kvm_tlb_flush_vmid(mmu);
+ __kvm_tlb_flush_vmid(vttbr);
}
-void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu)
+void __hyp_text __kvm_tlb_flush_local_vmid(u64 vttbr)
{
/* Switch to requested VMID */
- mmu = kern_hyp_va(mmu);
- write_sysreg(mmu->vttbr, VTTBR);
+ write_sysreg(vttbr, VTTBR);
isb();
write_sysreg(0, TLBIALL);
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index a27a204..5ca3a04 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -60,12 +60,17 @@ static bool memslot_is_logging(struct kvm_memory_slot *memslot)
*/
void kvm_flush_remote_tlbs(struct kvm *kvm)
{
- kvm_call_hyp(__kvm_tlb_flush_vmid, kvm);
+ struct kvm_s2_mmu *mmu = &kvm->arch.mmu;
+ u64 vttbr = kvm_get_vttbr(&mmu->vmid, mmu);
+
+ kvm_call_hyp(__kvm_tlb_flush_vmid, vttbr);
}
static void kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa)
{
- kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ipa);
+ u64 vttbr = kvm_get_vttbr(&mmu->vmid, mmu);
+
+ kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, vttbr, ipa);
}
/*
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index ed8139f..27dce47 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -53,9 +53,9 @@
extern char __kvm_hyp_vector[];
extern void __kvm_flush_vm_context(void);
-extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa);
-extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu);
-extern void __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu);
+extern void __kvm_tlb_flush_vmid_ipa(u64 vttbr, phys_addr_t ipa);
+extern void __kvm_tlb_flush_vmid(u64 vttbr);
+extern void __kvm_tlb_flush_local_vmid(u64 vttbr);
extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index a9c993f..94068e7 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -363,4 +363,12 @@ static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu,
return data; /* Leave LE untouched */
}
+static inline struct kvm_s2_vmid *vcpu_get_active_vmid(struct kvm_vcpu *vcpu)
+{
+ if (unlikely(vcpu_mode_el2(vcpu)))
+ return &vcpu->kvm->arch.mmu.el2_vmid;
+
+ return &vcpu->kvm->arch.mmu.vmid;
+}
+
#endif /* __ARM64_KVM_EMULATE_H__ */
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 954d6de..b33d35d 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -50,17 +50,19 @@
int kvm_arch_dev_ioctl_check_extension(struct kvm *kvm, long ext);
void __extended_idmap_trampoline(phys_addr_t boot_pgd, phys_addr_t idmap_start);
-struct kvm_s2_mmu {
+struct kvm_s2_vmid {
/* The VMID generation used for the virt. memory system */
u64 vmid_gen;
u32 vmid;
+};
+
+struct kvm_s2_mmu {
+ struct kvm_s2_vmid vmid;
+ struct kvm_s2_vmid el2_vmid;
/* 1-level 2nd stage table and lock */
spinlock_t pgd_lock;
pgd_t *pgd;
-
- /* VTTBR value associated with above pgd and vmid */
- u64 vttbr;
};
struct kvm_arch {
@@ -334,6 +336,9 @@ struct kvm_vcpu_arch {
/* Stage 2 paging state used by the hardware on next switch */
struct kvm_s2_mmu *hw_mmu;
+
+ /* VTTBR value used by the hardware on next switch */
+ u64 hw_vttbr;
};
#define vcpu_gp_regs(v) (&(v)->arch.ctxt.gp_regs)
@@ -391,6 +396,7 @@ static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
{
}
+unsigned int get_kvm_vmid_bits(void);
struct kvm_vcpu *kvm_arm_get_running_vcpu(void);
struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void);
void kvm_arm_halt_guest(struct kvm *kvm);
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 6f72fe8..e3455c4 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -314,5 +314,16 @@ static inline unsigned int kvm_get_vmid_bits(void)
return (cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR1_VMIDBITS_SHIFT) == 2) ? 16 : 8;
}
+static inline u64 kvm_get_vttbr(struct kvm_s2_vmid *vmid,
+ struct kvm_s2_mmu *mmu)
+{
+ u64 vmid_field, baddr;
+
+ baddr = virt_to_phys(mmu->pgd);
+ vmid_field = ((u64)vmid->vmid << VTTBR_VMID_SHIFT) &
+ VTTBR_VMID_MASK(get_kvm_vmid_bits());
+ return baddr | vmid_field;
+}
+
#endif /* __ASSEMBLY__ */
#endif /* __ARM64_KVM_MMU_H__ */
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index 3207009a..c80b2ae 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -135,9 +135,7 @@ static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
{
- struct kvm_s2_mmu *mmu = kern_hyp_va(vcpu->arch.hw_mmu);
-
- write_sysreg(mmu->vttbr, vttbr_el2);
+ write_sysreg(vcpu->arch.hw_vttbr, vttbr_el2);
}
static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)
diff --git a/arch/arm64/kvm/hyp/tlb.c b/arch/arm64/kvm/hyp/tlb.c
index 71a62ea..82350e7 100644
--- a/arch/arm64/kvm/hyp/tlb.c
+++ b/arch/arm64/kvm/hyp/tlb.c
@@ -17,14 +17,12 @@
#include <asm/kvm_hyp.h>
-void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
- phys_addr_t ipa)
+void __hyp_text __kvm_tlb_flush_vmid_ipa(u64 vttbr, phys_addr_t ipa)
{
dsb(ishst);
/* Switch to requested VMID */
- mmu = kern_hyp_va(mmu);
- write_sysreg(mmu->vttbr, vttbr_el2);
+ write_sysreg(vttbr, vttbr_el2);
isb();
/*
@@ -49,13 +47,12 @@ void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
write_sysreg(0, vttbr_el2);
}
-void __hyp_text __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu)
+void __hyp_text __kvm_tlb_flush_vmid(u64 vttbr)
{
dsb(ishst);
/* Switch to requested VMID */
- mmu = kern_hyp_va(mmu);
- write_sysreg(mmu->vttbr, vttbr_el2);
+ write_sysreg(vttbr, vttbr_el2);
isb();
asm volatile("tlbi vmalls12e1is" : : );
@@ -65,11 +62,10 @@ void __hyp_text __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu)
write_sysreg(0, vttbr_el2);
}
-void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu)
+void __hyp_text __kvm_tlb_flush_local_vmid(u64 vttbr)
{
/* Switch to requested VMID */
- mmu = kern_hyp_va(mmu);
- write_sysreg(mmu->vttbr, vttbr_el2);
+ write_sysreg(vttbr, vttbr_el2);
isb();
asm volatile("tlbi vmalle1" : : );
--
1.9.1
^ permalink raw reply related
* [RFC 36/55] KVM: arm64: Invalidate virtual EL2 TLB entries when needed
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
Sometimes when we are invalidating the TLB for a certain S2 MMU
context, this context can also have EL2 context associated with it and
we have to invalidate this too.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/kvm/arm.c | 6 ++++++
arch/arm/kvm/mmu.c | 16 ++++++++++++++++
2 files changed, 22 insertions(+)
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index aa8771d..371b38e7 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -350,6 +350,12 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
u64 vttbr = kvm_get_vttbr(&mmu->vmid, mmu);
kvm_call_hyp(__kvm_tlb_flush_local_vmid, vttbr);
+#ifndef CONFIG_KVM_ARM_NESTED_HYP
+ if (mmu->el2_vmid.vmid) {
+ vttbr = kvm_get_vttbr(&mmu->el2_vmid, mmu);
+ kvm_call_hyp(__kvm_tlb_flush_local_vmid, vttbr);
+ }
+#endif
*last_ran = vcpu->vcpu_id;
}
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 5ca3a04..56358fa 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -60,10 +60,20 @@ static bool memslot_is_logging(struct kvm_memory_slot *memslot)
*/
void kvm_flush_remote_tlbs(struct kvm *kvm)
{
+#ifndef CONFIG_KVM_ARM_NESTED_HYP
struct kvm_s2_mmu *mmu = &kvm->arch.mmu;
u64 vttbr = kvm_get_vttbr(&mmu->vmid, mmu);
kvm_call_hyp(__kvm_tlb_flush_vmid, vttbr);
+#else
+ /*
+ * When supporting nested virtualization, we can have multiple VMIDs
+ * in play for each VCPU in the VM, so it's really not worth it to try
+ * to quiesce the system and flush all the VMIDs that may be in use,
+ * instead just nuke the whole thing.
+ */
+ kvm_call_hyp(__kvm_flush_vm_context);
+#endif
}
static void kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa)
@@ -71,6 +81,12 @@ static void kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa)
u64 vttbr = kvm_get_vttbr(&mmu->vmid, mmu);
kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, vttbr, ipa);
+#ifdef CONFIG_KVM_ARM_NESTED_HYP
+ if (!mmu->el2_vmid.vmid)
+ return; /* only if this mmu has el2 context */
+ vttbr = kvm_get_vttbr(&mmu->el2_vmid, mmu);
+ kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, vttbr, ipa);
+#endif
}
/*
--
1.9.1
^ permalink raw reply related
* [RFC 37/55] KVM: arm64: Setup vttbr_el2 on each VM entry
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
Now that the vttbr value will be different depending on the VM's
exception level, we set it on each VM entry.
We only have one mmu instance at this point, but there will be
multiple of them when we run nested VMs.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm64/kvm/context.c | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/arch/arm64/kvm/context.c b/arch/arm64/kvm/context.c
index a93ffe4..b2c0220 100644
--- a/arch/arm64/kvm/context.c
+++ b/arch/arm64/kvm/context.c
@@ -18,6 +18,7 @@
#include <linux/kvm_host.h>
#include <asm/kvm_emulate.h>
#include <asm/esr.h>
+#include <asm/kvm_mmu.h>
struct el1_el2_map {
enum vcpu_sysreg el1;
@@ -88,6 +89,15 @@ static void create_shadow_el1_sysregs(struct kvm_vcpu *vcpu)
s_sys_regs[CPACR_EL1] = cptr_el2_to_cpacr_el1(el2_regs[CPTR_EL2]);
}
+static void setup_s2_mmu(struct kvm_vcpu *vcpu)
+{
+ struct kvm_s2_mmu *mmu = &vcpu->kvm->arch.mmu;
+ struct kvm_s2_vmid *vmid = vcpu_get_active_vmid(vcpu);
+
+ vcpu->arch.hw_vttbr = kvm_get_vttbr(vmid, mmu);
+ vcpu->arch.hw_mmu = mmu;
+}
+
/*
* List of EL1 registers which we allow the virtual EL2 mode to access
* directly without trapping and which haven't been paravirtualized.
@@ -166,6 +176,8 @@ void kvm_arm_setup_shadow_state(struct kvm_vcpu *vcpu)
}
vgic_v2_setup_shadow_state(vcpu);
+
+ setup_s2_mmu(vcpu);
}
/**
--
1.9.1
^ permalink raw reply related
* [RFC 38/55] KVM: arm/arm64: Make mmu functions non-static
From: Jintack Lim @ 2017-01-09 6:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1483943091-1364-1-git-send-email-jintack@cs.columbia.edu>
From: Christoffer Dall <christoffer.dall@linaro.org>
Make mmu functions non-static so that we can reuse those functions
to support mmu for the nested VMs.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Jintack Lim <jintack@cs.columbia.edu>
---
arch/arm/kvm/mmu.c | 90 +++++++++++++++++++++++-----------------
arch/arm64/include/asm/kvm_mmu.h | 9 ++++
2 files changed, 61 insertions(+), 38 deletions(-)
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 56358fa..98b42e8 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -301,7 +301,7 @@ static void unmap_stage2_puds(struct kvm_s2_mmu *mmu, pgd_t *pgd,
}
/**
- * unmap_stage2_range -- Clear stage2 page table entries to unmap a range
+ * kvm_unmap_stage2_range -- Clear stage2 page table entries to unmap a range
* @kvm: The VM pointer
* @start: The intermediate physical base address of the range to unmap
* @size: The size of the area to unmap
@@ -311,8 +311,7 @@ static void unmap_stage2_puds(struct kvm_s2_mmu *mmu, pgd_t *pgd,
* destroying the VM), otherwise another faulting VCPU may come in and mess
* with things behind our backs.
*/
-static void unmap_stage2_range(struct kvm_s2_mmu *mmu,
- phys_addr_t start, u64 size)
+void kvm_unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size)
{
pgd_t *pgd;
phys_addr_t addr = start, end = start + size;
@@ -374,11 +373,10 @@ static void stage2_flush_puds(pgd_t *pgd,
} while (pud++, addr = next, addr != end);
}
-static void stage2_flush_memslot(struct kvm_s2_mmu *mmu,
- struct kvm_memory_slot *memslot)
+void kvm_stage2_flush_range(struct kvm_s2_mmu *mmu,
+ phys_addr_t start, phys_addr_t end)
{
- phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
- phys_addr_t end = addr + PAGE_SIZE * memslot->npages;
+ phys_addr_t addr = start;
phys_addr_t next;
pgd_t *pgd;
@@ -389,6 +387,15 @@ static void stage2_flush_memslot(struct kvm_s2_mmu *mmu,
} while (pgd++, addr = next, addr != end);
}
+static void stage2_flush_memslot(struct kvm_s2_mmu *mmu,
+ struct kvm_memory_slot *memslot)
+{
+ phys_addr_t start = memslot->base_gfn << PAGE_SHIFT;
+ phys_addr_t end = start + PAGE_SIZE * memslot->npages;
+
+ kvm_stage2_flush_range(mmu, start, end);
+}
+
/**
* stage2_flush_vm - Invalidate cache for pages mapped in stage 2
* @kvm: The struct kvm pointer
@@ -745,21 +752,9 @@ int create_hyp_io_mappings(void *from, void *to, phys_addr_t phys_addr)
__phys_to_pfn(phys_addr), PAGE_HYP_DEVICE);
}
-/**
- * kvm_alloc_stage2_pgd - allocate level-1 table for stage-2 translation.
- * @kvm: The KVM struct pointer for the VM.
- *
- * Allocates only the stage-2 HW PGD level table(s) (can support either full
- * 40-bit input addresses or limited to 32-bit input addresses). Clears the
- * allocated pages.
- *
- * Note we don't need locking here as this is only called when the VM is
- * created, which can only be done once.
- */
-int kvm_alloc_stage2_pgd(struct kvm *kvm)
+int __kvm_alloc_stage2_pgd(struct kvm_s2_mmu *mmu)
{
pgd_t *pgd;
- struct kvm_s2_mmu *mmu = &kvm->arch.mmu;
if (mmu->pgd != NULL) {
kvm_err("kvm_arch already initialized?\n");
@@ -776,6 +771,22 @@ int kvm_alloc_stage2_pgd(struct kvm *kvm)
return 0;
}
+/**
+ * kvm_alloc_stage2_pgd - allocate level-1 table for stage-2 translation.
+ * @kvm: The KVM struct pointer for the VM.
+ *
+ * Allocates only the stage-2 HW PGD level table(s) (can support either full
+ * 40-bit input addresses or limited to 32-bit input addresses). Clears the
+ * allocated pages.
+ *
+ * Note we don't need locking here as this is only called when the VM is
+ * created, which can only be done once.
+ */
+int kvm_alloc_stage2_pgd(struct kvm *kvm)
+{
+ return __kvm_alloc_stage2_pgd(&kvm->arch.mmu);
+}
+
static void stage2_unmap_memslot(struct kvm_s2_mmu *mmu,
struct kvm_memory_slot *memslot)
{
@@ -811,7 +822,7 @@ static void stage2_unmap_memslot(struct kvm_s2_mmu *mmu,
if (!(vma->vm_flags & VM_PFNMAP)) {
gpa_t gpa = addr + (vm_start - memslot->userspace_addr);
- unmap_stage2_range(mmu, gpa, vm_end - vm_start);
+ kvm_unmap_stage2_range(mmu, gpa, vm_end - vm_start);
}
hva = vm_end;
} while (hva < reg_end);
@@ -841,6 +852,17 @@ void stage2_unmap_vm(struct kvm *kvm)
srcu_read_unlock(&kvm->srcu, idx);
}
+void __kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu)
+{
+ if (mmu->pgd == NULL)
+ return;
+
+ kvm_unmap_stage2_range(mmu, 0, KVM_PHYS_SIZE);
+ /* Free the HW pgd, one page at a time */
+ free_pages_exact(mmu->pgd, S2_PGD_SIZE);
+ mmu->pgd = NULL;
+}
+
/**
* kvm_free_stage2_pgd - free all stage-2 tables
* @kvm: The KVM struct pointer for the VM.
@@ -854,15 +876,7 @@ void stage2_unmap_vm(struct kvm *kvm)
*/
void kvm_free_stage2_pgd(struct kvm *kvm)
{
- struct kvm_s2_mmu *mmu = &kvm->arch.mmu;
-
- if (mmu->pgd == NULL)
- return;
-
- unmap_stage2_range(mmu, 0, KVM_PHYS_SIZE);
- /* Free the HW pgd, one page at a time */
- free_pages_exact(mmu->pgd, S2_PGD_SIZE);
- mmu->pgd = NULL;
+ __kvm_free_stage2_pgd(&kvm->arch.mmu);
}
static pud_t *stage2_get_pud(struct kvm_s2_mmu *mmu,
@@ -1175,13 +1189,13 @@ static void stage2_wp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end)
}
/**
- * stage2_wp_range() - write protect stage2 memory region range
+ * kvm_stage2_wp_range() - write protect stage2 memory region range
* @kvm: The KVM pointer
* @addr: Start address of range
* @end: End address of range
*/
-static void stage2_wp_range(struct kvm *kvm, struct kvm_s2_mmu *mmu,
- phys_addr_t addr, phys_addr_t end)
+void kvm_stage2_wp_range(struct kvm *kvm, struct kvm_s2_mmu *mmu,
+ phys_addr_t addr, phys_addr_t end)
{
pgd_t *pgd;
phys_addr_t next;
@@ -1225,7 +1239,7 @@ void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot)
phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;
spin_lock(&kvm->mmu_lock);
- stage2_wp_range(kvm, &kvm->arch.mmu, start, end);
+ kvm_stage2_wp_range(kvm, &kvm->arch.mmu, start, end);
spin_unlock(&kvm->mmu_lock);
kvm_flush_remote_tlbs(kvm);
}
@@ -1249,7 +1263,7 @@ static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT;
phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT;
- stage2_wp_range(kvm, &kvm->arch.mmu, start, end);
+ kvm_stage2_wp_range(kvm, &kvm->arch.mmu, start, end);
}
/*
@@ -1589,7 +1603,7 @@ static int handle_hva_to_gpa(struct kvm *kvm,
static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
{
- unmap_stage2_range(&kvm->arch.mmu, gpa, PAGE_SIZE);
+ kvm_unmap_stage2_range(&kvm->arch.mmu, gpa, PAGE_SIZE);
return 0;
}
@@ -1900,7 +1914,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
spin_lock(&kvm->mmu_lock);
if (ret)
- unmap_stage2_range(&kvm->arch.mmu, mem->guest_phys_addr,
+ kvm_unmap_stage2_range(&kvm->arch.mmu, mem->guest_phys_addr,
mem->memory_size);
else
stage2_flush_memslot(&kvm->arch.mmu, memslot);
@@ -1944,7 +1958,7 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
phys_addr_t size = slot->npages << PAGE_SHIFT;
spin_lock(&kvm->mmu_lock);
- unmap_stage2_range(&kvm->arch.mmu, gpa, size);
+ kvm_unmap_stage2_range(&kvm->arch.mmu, gpa, size);
spin_unlock(&kvm->mmu_lock);
}
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index e3455c4..a504162 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -145,9 +145,18 @@ static inline unsigned long __kern_hyp_va(unsigned long v)
void stage2_unmap_vm(struct kvm *kvm);
int kvm_alloc_stage2_pgd(struct kvm *kvm);
+int __kvm_alloc_stage2_pgd(struct kvm_s2_mmu *mmu);
void kvm_free_stage2_pgd(struct kvm *kvm);
+void __kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu);
int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
phys_addr_t pa, unsigned long size, bool writable);
+void kvm_unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start,
+ u64 size);
+void kvm_stage2_wp_range(struct kvm *kvm, struct kvm_s2_mmu *mmu,
+ phys_addr_t addr, phys_addr_t end);
+void kvm_stage2_flush_range(struct kvm_s2_mmu *mmu,
+ phys_addr_t start, phys_addr_t end);
+
int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run);
--
1.9.1
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox