Re: [PATCH 00/59] KVM: arm64: ARMv8.3 Nested Virtualization support

Alexandru Elisei
2019-08-22T11:57:29Z

On 8/9/19 11:01 AM, Alexandru Elisei wrote:
> On 8/2/19 11:11 AM, Alexandru Elisei wrote:
>> Hi,
>>
>> On 6/21/19 10:37 AM, Marc Zyngier wrote:
>> When working on adding support for EL2 to kvm-unit-tests I was able to trigger
>> the following warning:
>>
>> # ./lkvm run -f psci.flat -m 128 -c 8 --console serial --irqchip gicv3 --nested
>> # lkvm run --firmware psci.flat -m 128 -c 8 --name guest-151
>> Info: Placing fdt at 0x80200000 - 0x80210000
>> # Warning: The maximum recommended amount of VCPUs is 4
>> chr_testdev_init: chr-testdev: can't find a virtio-console
>> INFO: PSCI version 1.0
>> PASS: invalid-function
>> PASS: affinity-info-on
>> PASS: affinity-info-off
>> [   24.381266] WARNING: CPU: 3 PID: 160 at
>> arch/arm64/kvm/../../../virt/kvm/arm/arch_timer.c:170
>> kvm_timer_irq_can_fire+0xc/0x30
>> [   24.381366] Modules linked in:
>> [   24.381466] CPU: 3 PID: 160 Comm: kvm-vcpu-1 Not tainted
>> 5.2.0-rc5-00060-g7dbce63bd1c7 #145
>> [   24.381566] Hardware name: Foundation-v8A (DT)
>> [   24.381566] pstate: 40400009 (nZcv daif +PAN -UAO)
>> [   24.381666] pc : kvm_timer_irq_can_fire+0xc/0x30
>> [   24.381766] lr : timer_emulate+0x24/0x98
>> [   24.381766] sp : ffff000013d8b780
>> [   24.381866] x29: ffff000013d8b780 x28: ffff80087a639b80
>> [   24.381966] x27: ffff000010ba8648 x26: ffff000010b71b40
>> [   24.382066] x25: ffff80087a63a100 x24: 0000000000000000
>> [   24.382111] x23: 000080086ca54000 x22: ffff0000100ce260
>> [   24.382166] x21: ffff800875e7c918 x20: ffff800875e7a800
>> [   24.382275] x19: ffff800875e7ca08 x18: 0000000000000000
>> [   24.382366] x17: 0000000000000000 x16: 0000000000000000
>> [   24.382466] x15: 0000000000000000 x14: 0000000000002118
>> [   24.382566] x13: 0000000000002190 x12: 0000000000002280
>> [   24.382566] x11: 0000000000002208 x10: 0000000000000040
>> [   24.382666] x9 : ffff000012dc3b38 x8 : 0000000000000000
>> [   24.382766] x7 : 0000000000000000 x6 : ffff80087ac00248
>> [   24.382866] x5 : 000080086ca54000 x4 : 0000000000002118
>> [   24.382966] x3 : eeeeeeeeeeeeeeef x2 : ffff800875e7c918
>> [   24.383066] x1 : 0000000000000001 x0 : ffff800875e7ca08
>> [   24.383066] Call trace:
>> [   24.383166]  kvm_timer_irq_can_fire+0xc/0x30
>> [   24.383266]  kvm_timer_vcpu_load+0x9c/0x1a0
>> [   24.383366]  kvm_arch_vcpu_load+0xb0/0x1f0
>> [   24.383366]  kvm_sched_in+0x1c/0x28
>> [   24.383466]  finish_task_switch+0xd8/0x1d8
>> [   24.383566]  __schedule+0x248/0x4a0
>> [   24.383666]  preempt_schedule_irq+0x60/0x90
>> [   24.383666]  el1_irq+0xd0/0x180
>> [   24.383766]  kvm_handle_guest_abort+0x0/0x3a0
>> [   24.383866]  kvm_arch_vcpu_ioctl_run+0x41c/0x688
>> [   24.383866]  kvm_vcpu_ioctl+0x4c0/0x838
>> [   24.383966]  do_vfs_ioctl+0xb8/0x878
>> [   24.384077]  ksys_ioctl+0x84/0x90
>> [   24.384166]  __arm64_sys_ioctl+0x18/0x28
>> [   24.384166]  el0_svc_common.constprop.0+0xb0/0x168
>> [   24.384266]  el0_svc_handler+0x28/0x78
>> [   24.384366]  el0_svc+0x8/0xc
>> [   24.384366] ---[ end trace 37a32293e43ac12c ]---
>> [   24.384666] WARNING: CPU: 3 PID: 160 at
>> arch/arm64/kvm/../../../virt/kvm/arm/arch_timer.c:170
>> kvm_timer_irq_can_fire+0xc/0x30
>> [   24.384766] Modules linked in:
>> [   24.384866] CPU: 3 PID: 160 Comm: kvm-vcpu-1 Tainted: G W
>> 5.2.0-rc5-00060-g7dbce63bd1c7 #145
>> [   24.384966] Hardware name: Foundation-v8A (DT)
>> [   24.384966] pstate: 40400009 (nZcv daif +PAN -UAO)
>> [   24.385066] pc : kvm_timer_irq_can_fire+0xc/0x30
>> [   24.385166] lr : timer_emulate+0x24/0x98
>> [   24.385166] sp : ffff000013d8b780
>> [   24.385266] x29: ffff000013d8b780 x28: ffff80087a639b80
>> [   24.385366] x27: ffff000010ba8648 x26: ffff000010b71b40
>> [   24.385466] x25: ffff80087a63a100 x24: 0000000000000000
>> [   24.385466] x23: 000080086ca54000 x22: ffff0000100ce260
>> [   24.385566] x21: ffff800875e7c918 x20: ffff800875e7a800
>> [   24.385666] x19: ffff800875e7ca80 x18: 0000000000000000
>> [   24.385766] x17: 0000000000000000 x16: 0000000000000000
>> [   24.385866] x15: 0000000000000000 x14: 0000000000002118
>> [   24.385966] x13: 0000000000002190 x12: 0000000000002280
>> [   24.385966] x11: 0000000000002208 x10: 0000000000000040
>> [   24.386066] x9 : ffff000012dc3b38 x8 : 0000000000000000
>> [   24.386166] x7 : 0000000000000000 x6 : ffff80087ac00248
>> [   24.386266] x5 : 000080086ca54000 x4 : 0000000000002118
>> [   24.386366] x3 : eeeeeeeeeeeeeeef x2 : ffff800875e7c918
>> [   24.386466] x1 : 0000000000000001 x0 : ffff800875e7ca80
>> [   24.386466] Call trace:
>> [   24.386566]  kvm_timer_irq_can_fire+0xc/0x30
>> [   24.386666]  kvm_timer_vcpu_load+0xa8/0x1a0
>> [   24.386666]  kvm_arch_vcpu_load+0xb0/0x1f0
>> [   24.386898]  kvm_sched_in+0x1c/0x28
>> [   24.386966]  finish_task_switch+0xd8/0x1d8
>> [   24.387166]  __schedule+0x248/0x4a0
>> [   24.387354]  preempt_schedule_irq+0x60/0x90
>> [   24.387366]  el1_irq+0xd0/0x180
>> [   24.387466]  kvm_handle_guest_abort+0x0/0x3a0
>> [   24.387566]  kvm_arch_vcpu_ioctl_run+0x41c/0x688
>> [   24.387566]  kvm_vcpu_ioctl+0x4c0/0x838
>> [   24.387666]  do_vfs_ioctl+0xb8/0x878
>> [   24.387766]  ksys_ioctl+0x84/0x90
>> [   24.387866]  __arm64_sys_ioctl+0x18/0x28
>> [   24.387866]  el0_svc_common.constprop.0+0xb0/0x168
>> [   24.387966]  el0_svc_handler+0x28/0x78
>> [   24.388066]  el0_svc+0x8/0xc
>> [   24.388066] ---[ end trace 37a32293e43ac12d ]---
>> PASS: cpu-on
>> SUMMARY: 4 te[   24.390266] WARNING: CPU: 3 PID: 160 at
>> arch/arm64/kvm/../../../virt/kvm/arm/arch_timer.c:170
>> kvm_timer_irq_can_fire+0xc/0x30
>> s[   24.390366] Modules linked in:
>> ts[   24.390366] CPU: 3 PID: 160 Comm: kvm-vcpu-1 Tainted: G W
>> 5.2.0-rc5-00060-g7dbce63bd1c7 #145
>> [   24.390566] Hardware name: Foundation-v8A (DT)
>>
>> [   24.390795] pstate: 40400009 (nZcv daif +PAN -UAO)
>> [   24.390866] pc : kvm_timer_irq_can_fire+0xc/0x30
>> [   24.390966] lr : timer_emulate+0x24/0x98
>> [   24.391066] sp : ffff000013d8b780
>> [   24.391066] x29: ffff000013d8b780 x28: ffff80087a639b80
>> [   24.391166] x27: ffff000010ba8648 x26: ffff000010b71b40
>> [   24.391266] x25: ffff80087a63a100 x24: 0000000000000000
>> [   24.391366] x23: 000080086ca54000 x22: 0000000000000003
>> [   24.391466] x21: ffff800875e7c918 x20: ffff800875e7a800
>> [   24.391466] x19: ffff800875e7ca08 x18: 0000000000000000
>> [   24.391566] x17: 0000000000000000 x16: 0000000000000000
>> [   24.391666] x15: 0000000000000000 x14: 0000000000002118
>> [   24.391766] x13: 0000000000002190 x12: 0000000000002280
>> [   24.391866] x11: 0000000000002208 x10: 0000000000000040
>> [   24.391942] x9 : ffff000012dc3b38 x8 : 0000000000000000
>> [   24.391966] x7 : 0000000000000000 x6 : ffff80087ac00248
>> [   24.392066] x5 : 000080086ca54000 x4 : 0000000000002118
>> [   24.392166] x3 : eeeeeeeeeeeeeeef x2 : ffff800875e7c918
>> [   24.392269] x1 : 0000000000000001 x0 : ffff800875e7ca08
>> [   24.392366] Call trace:
>> [   24.392433]  kvm_timer_irq_can_fire+0xc/0x30
>> [   24.392466]  kvm_timer_vcpu_load+0x9c/0x1a0
>> [   24.392597]  kvm_arch_vcpu_load+0xb0/0x1f0
>> [   24.392666]  kvm_sched_in+0x1c/0x28
>> [   24.392766]  finish_task_switch+0xd8/0x1d8
>> [   24.392766]  __schedule+0x248/0x4a0
>> [   24.392866]  preempt_schedule_irq+0x60/0x90
>> [   24.392966]  el1_irq+0xd0/0x180
>> [   24.392966]  kvm_handle_guest_abort+0x0/0x3a0
>> [   24.393066]  kvm_arch_vcpu_ioctl_run+0x41c/0x688
>> [   24.393166]  kvm_vcpu_ioctl+0x4c0/0x838
>> [   24.393266]  do_vfs_ioctl+0xb8/0x878
>> [   24.393266]  ksys_ioctl+0x84/0x90
>> [   24.393366]  __arm64_sys_ioctl+0x18/0x28
>> [   24.393466]  el0_svc_common.constprop.0+0xb0/0x168
>> [   24.393566]  el0_svc_handler+0x28/0x78
>> [   24.393566]  el0_svc+0x8/0xc
>> [   24.393666] ---[ end trace 37a32293e43ac12e ]---
>> [   24.393866] WARNING: CPU: 3 PID: 160 at
>> arch/arm64/kvm/../../../virt/kvm/arm/arch_timer.c:170
>> kvm_timer_irq_can_fire+0xc/0x30
>> [   24.394066] Modules linked in:
>> [   24.394266] CPU: 3 PID: 160 Comm: kvm-vcpu-1 Tainted: G W
>> 5.2.0-rc5-00060-g7dbce63bd1c7 #145
>> [   24.394366] Hardware name: Foundation-v8A (DT)
>> [   24.394466] pstate: 40400009 (nZcv daif +PAN -UAO)
>> [   24.394466] pc : kvm_timer_irq_can_fire+0xc/0x30
>> [   24.394566] lr : timer_emulate+0x24/0x98
>> [   24.394666] sp : ffff000013d8b780
>> [   24.394727] x29: ffff000013d8b780 x28: ffff80087a639b80
>> [   24.394766] x27: ffff000010ba8648 x26: ffff000010b71b40
>> [   24.394866] x25: ffff80087a63a100 x24: 0000000000000000
>> [   24.394966] x23: 000080086ca54000 x22: 0000000000000003
>> [   24.394966] x21: ffff800875e7c918 x20: ffff800875e7a800
>> [   24.395066] x19: ffff800875e7ca80 x18: 0000000000000000
>> [   24.395166] x17: 0000000000000000 x16: 0000000000000000
>> [   24.395266] x15: 0000000000000000 x14: 0000000000002118
>> [   24.395383] x13: 0000000000002190 x12: 0000000000002280
>> [   24.395466] x11: 0000000000002208 x10: 0000000000000040
>> [   24.395547] x9 : ffff000012dc3b38 x8 : 0000000000000000
>> [   24.395666] x7 : 0000000000000000 x6 : ffff80087ac00248
>> [   24.395866] x5 : 000080086ca54000 x4 : 0000000000002118
>> [   24.395966] x3 : eeeeeeeeeeeeeeef x2 : ffff800875e7c918
>> [   24.396066] x1 : 0000000000000001 x0 : ffff800875e7ca80
>> [   24.396066] Call trace:
>> [   24.396166]  kvm_timer_irq_can_fire+0xc/0x30
>> [   24.396266]  kvm_timer_vcpu_load+0xa8/0x1a0
>> [   24.396366]  kvm_arch_vcpu_load+0xb0/0x1f0
>> [   24.396366]  kvm_sched_in+0x1c/0x28
>> [   24.396466]  finish_task_switch+0xd8/0x1d8
>> [   24.396566]  __schedule+0x248/0x4a0
>> [   24.396666]  preempt_schedule_irq+0x60/0x90
>> [   24.396666]  el1_irq+0xd0/0x180
>> [   24.396766]  kvm_handle_guest_abort+0x0/0x3a0
>> [   24.396866]  kvm_arch_vcpu_ioctl_run+0x41c/0x688
>> [   24.396866]  kvm_vcpu_ioctl+0x4c0/0x838
>> [   24.397021]  do_vfs_ioctl+0xb8/0x878
>> [   24.397066]  ksys_ioctl+0x84/0x90
>> [   24.397166]  __arm64_sys_ioctl+0x18/0x28
>> [   24.397348]  el0_svc_common.constprop.0+0xb0/0x168
>> [   24.397366]  el0_svc_handler+0x28/0x78
>> [   24.397566]  el0_svc+0x8/0xc
>> [   24.397676] ---[ end trace 37a32293e43ac12f ]---
>>
>> # KVM compatibility warning.
>> virtio-9p device was not detected.
>> While you have requested a virtio-9p device, the guest kernel did not
>> initialize it.
>> Please make sure that the guest kernel was compiled with
>> CONFIG_NET_9P_VIRTIO=y enabled in .config.
>>
>> # KVM compatibility warning.
>> virtio-net device was not detected.
>> While you have requested a virtio-net device, the guest kernel did not
>> initialize it.
>> Please make sure that the guest kernel was compiled with CONFIG_VIRTIO_NET=y
>> enabled in .config.
>>
>> [..]
> Did some investigating and this was caused by a bug in kvm-unit-tests (the fix
> for it will be part of the EL2 patches for kvm-unit-tests). The guest was trying
> to fetch an instruction from address 0x200, which KVM interprets as a prefetch
> abort on an I/O address and ends up calling kvm_inject_pabt. The code from
> arch/arm64/kvm/inject_fault.c doesn't know anything about nested virtualization,
> and it sets the VCPU mode directly to PSR_MODE_EL1h. This makes_hyp_ctxt return
> false, and get_timer_map will return an incorrect mapping.
>
> On next kvm_timer_vcpu_put, the direct timers will be {p,v}timer, and
> h{p,v}timer->loaded will not be set to false. In the corresponding call to
> kvm_timer_vcpu_load, KVM will try to emulate the hptimer and hvtimer, which
> still have loaded = true. And this causes the warning I saw.

I tried to fix it with the following patch, inject_undef64 was similarly broken:

diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c
index fac962b467bd..aee8a9ef36d5 100644
--- a/arch/arm64/kvm/inject_fault.c
+++ b/arch/arm64/kvm/inject_fault.c
@@ -53,15 +53,7 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt,
unsigned long addr
{
unsigned long cpsr = *vcpu_cpsr(vcpu);
bool is_aarch32 = vcpu_mode_is_32bit(vcpu);
-    u32 esr = 0;
-
-    vcpu_write_elr_el1(vcpu, *vcpu_pc(vcpu));
-    *vcpu_pc(vcpu) = get_except_vector(vcpu, except_type_sync);
-
-    *vcpu_cpsr(vcpu) = PSTATE_FAULT_BITS_64;
-    vcpu_write_spsr(vcpu, cpsr);
-
-    vcpu_write_sys_reg(vcpu, addr, FAR_EL1);
+    u32 esr = ESR_ELx_FSC_EXTABT;

/*
* Build an {i,d}abort, depending on the level and the
@@ -82,13 +74,12 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool
is_iabt, unsigned long addr
if (!is_iabt)
esr |= ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT;

-    vcpu_write_sys_reg(vcpu, esr | ESR_ELx_FSC_EXTABT, ESR_EL1);
-}
+    if (nested_virt_in_use(vcpu)) {
+        kvm_inject_nested_sync(vcpu, esr);
+        return;
+    }

-static void inject_undef64(struct kvm_vcpu *vcpu)
-{
-    unsigned long cpsr = *vcpu_cpsr(vcpu);
-    u32 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT);
+    vcpu_write_sys_reg(vcpu, esr, ESR_EL1);

vcpu_write_elr_el1(vcpu, *vcpu_pc(vcpu));
*vcpu_pc(vcpu) = get_except_vector(vcpu, except_type_sync);
@@ -96,6 +87,14 @@ static void inject_undef64(struct kvm_vcpu *vcpu)
*vcpu_cpsr(vcpu) = PSTATE_FAULT_BITS_64;
vcpu_write_spsr(vcpu, cpsr);

+    vcpu_write_sys_reg(vcpu, addr, FAR_EL1);
+}
+
+static void inject_undef64(struct kvm_vcpu *vcpu)
+{
+    unsigned long cpsr = *vcpu_cpsr(vcpu);
+    u32 esr = ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT;
+
/*
* Build an unknown exception, depending on the instruction
* set.
@@ -103,7 +102,18 @@ static void inject_undef64(struct kvm_vcpu *vcpu)
if (kvm_vcpu_trap_il_is32bit(vcpu))
esr |= ESR_ELx_IL;

+    if (nested_virt_in_use(vcpu)) {
+        kvm_inject_nested_sync(vcpu, esr);
+        return;
+    }
+
vcpu_write_sys_reg(vcpu, esr, ESR_EL1);
+
+    vcpu_write_elr_el1(vcpu, *vcpu_pc(vcpu));
+    *vcpu_pc(vcpu) = get_except_vector(vcpu, except_type_sync);
+
+    *vcpu_cpsr(vcpu) = PSTATE_FAULT_BITS_64;
+    vcpu_write_spsr(vcpu, cpsr);
}

/**