[PATCH v2 2/2] KVM: x86: Wake blocked vCPUs before offlining a CPU

[Jinmeng Zhou <jinmengzhou22@gmail.com>]

From: Jinmeng Zhou <zhoujinmeng@bytedance.com>

Posted-interrupt wakeup state is tied to the physical CPU on which a vCPU
blocks.  When a CPU is offlined, sched_cpu_deactivate() removes it from the
active mask, but a blocked vCPU stays on that CPU's wakeup list and its
posted-interrupt descriptor still targets that CPU's APIC.

If a device posts an interrupt after the old CPU goes offline, the wakeup
notification cannot make the blocked vCPU runnable.  The vCPU cannot fix
the stale notification destination, because that happens only after the
vCPU is scheduled back in.

Add an optional x86 vendor cpu_offline() callback and call it from the
CPUHP_AP_X86_KVM_ONLINE teardown.  That teardown runs after CPUHP_AP_ACTIVE
has deactivated the CPU.  Have VMX wake every vCPU on the dying CPU's
posted-interrupt wakeup list, so that the scheduler places the vCPU on an
active CPU and vmx_vcpu_pi_load() rebuilds its wakeup list and
posted-interrupt destination state.

A single pass over the list is sufficient.  Once the CPU is inactive, the
scheduler won't select it for a wakeup, and balance_push() moves any task
that is switched in on it to an active CPU before the task can run.  Thus
no vCPU can block on the CPU and add itself to the list after the walk.

Use __kvm_vcpu_wake_up() rather than KVM_REQ_UNBLOCK.  A vCPU with no
pending event goes back to the block loop after its posted-interrupt state
has been migrated, and no spurious wakeup is reported to the guest.

Initialize the per-CPU posted-interrupt wakeup lists in vmx_init() before
calling kvm_x86_vendor_init().  Once the CPU hotplug state is registered,
a concurrent CPU offline can invoke the VMX callback, which must not walk
an uninitialized list.

Fixes: bf9f6ac8d749 ("KVM: Update Posted-Interrupts Descriptor when vCPU is blocked")
Cc: stable@vger.kernel.org
Co-developed-by: Guixiong Wei <weiguixiong@bytedance.com>
Signed-off-by: Guixiong Wei <weiguixiong@bytedance.com>
Signed-off-by: Jinmeng Zhou <zhoujinmeng@bytedance.com>
---
 arch/x86/include/asm/kvm-x86-ops.h |  1 +
 arch/x86/include/asm/kvm_host.h    |  1 +
 arch/x86/kvm/vmx/main.c            |  1 +
 arch/x86/kvm/vmx/posted_intr.c     | 18 ++++++++++++++++++
 arch/x86/kvm/vmx/posted_intr.h     |  1 +
 arch/x86/kvm/vmx/vmx.c             | 16 ++++++++++------
 arch/x86/kvm/x86.c                 |  3 ++-
 7 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h
index e213c9ae3e301..f2a805783f045 100644
--- a/arch/x86/include/asm/kvm-x86-ops.h
+++ b/arch/x86/include/asm/kvm-x86-ops.h
@@ -17,6 +17,7 @@
 KVM_X86_OP(check_processor_compatibility)
 KVM_X86_OP(enable_virtualization_cpu)
 KVM_X86_OP(disable_virtualization_cpu)
+KVM_X86_OP_OPTIONAL(cpu_offline)
 KVM_X86_OP(hardware_unsetup)
 KVM_X86_OP(has_emulated_msr)
 KVM_X86_OP(vcpu_after_set_cpuid)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 683bb8bf43a94..d47863ce2c5ae 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1509,6 +1509,7 @@ struct kvm_x86_ops {
 
 	int (*enable_virtualization_cpu)(void);
 	void (*disable_virtualization_cpu)(void);
+	void (*cpu_offline)(unsigned int cpu);
 	cpu_emergency_virt_cb *emergency_disable_virtualization_cpu;
 
 	void (*hardware_unsetup)(void);
diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c
index 4c52ab8d07869..5a3f77f42d20d 100644
--- a/arch/x86/kvm/vmx/main.c
+++ b/arch/x86/kvm/vmx/main.c
@@ -894,6 +894,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = {
 
 	.enable_virtualization_cpu = vmx_enable_virtualization_cpu,
 	.disable_virtualization_cpu = vt_op(disable_virtualization_cpu),
+	.cpu_offline = pi_cpu_offline,
 	.emergency_disable_virtualization_cpu = vmx_emergency_disable_virtualization_cpu,
 
 	.has_emulated_msr = vt_op(has_emulated_msr),
diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c
index 4a6d9a17da238..1905472b336d3 100644
--- a/arch/x86/kvm/vmx/posted_intr.c
+++ b/arch/x86/kvm/vmx/posted_intr.c
@@ -266,6 +266,24 @@ void pi_wakeup_handler(void)
 	raw_spin_unlock(spinlock);
 }
 
+void pi_cpu_offline(unsigned int cpu)
+{
+	struct list_head *wakeup_list = &per_cpu(wakeup_vcpus_on_cpu, cpu);
+	raw_spinlock_t *spinlock = &per_cpu(wakeup_vcpus_on_cpu_lock, cpu);
+	struct vcpu_vt *vt;
+	unsigned long flags;
+
+	/*
+	 * CPUHP_AP_ACTIVE has already deactivated the CPU.  Wake blocked vCPUs
+	 * so that they are scheduled on another CPU and rebuild their posted
+	 * interrupt wakeup state in vmx_vcpu_pi_load().
+	 */
+	raw_spin_lock_irqsave(spinlock, flags);
+	list_for_each_entry(vt, wakeup_list, pi_wakeup_list)
+		__kvm_vcpu_wake_up(vt_to_vcpu(vt));
+	raw_spin_unlock_irqrestore(spinlock, flags);
+}
+
 void __init pi_init_cpu(int cpu)
 {
 	INIT_LIST_HEAD(&per_cpu(wakeup_vcpus_on_cpu, cpu));
diff --git a/arch/x86/kvm/vmx/posted_intr.h b/arch/x86/kvm/vmx/posted_intr.h
index a4af39948cf04..530f1347f8b0d 100644
--- a/arch/x86/kvm/vmx/posted_intr.h
+++ b/arch/x86/kvm/vmx/posted_intr.h
@@ -11,6 +11,7 @@
 void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu);
 void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu);
 void pi_wakeup_handler(void);
+void pi_cpu_offline(unsigned int cpu);
 void __init pi_init_cpu(int cpu);
 void pi_apicv_pre_state_restore(struct kvm_vcpu *vcpu);
 bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu);
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 612ab07d4100f..ff1a6ccbd043b 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -8889,6 +8889,16 @@ int __init vmx_init(void)
 	if (setup_vmcs_config(&vmcs_config, &vmx_capability) < 0)
 		return -EIO;
 
+	/*
+	 * Initialize the per-CPU lists before common x86 init, which registers
+	 * the CPU hotplug callbacks that walk the posted-interrupt wakeup lists.
+	 */
+	for_each_possible_cpu(cpu) {
+		INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu));
+
+		pi_init_cpu(cpu);
+	}
+
 	r = kvm_x86_vendor_init(&vt_init_ops);
 	if (r)
 		return r;
@@ -8898,12 +8908,6 @@ int __init vmx_init(void)
 	if (r)
 		goto err_l1d_flush;
 
-	for_each_possible_cpu(cpu) {
-		INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu));
-
-		pi_init_cpu(cpu);
-	}
-
 	vmx_check_vmcs12_offsets();
 
 	return 0;
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index a08ae25f373fb..38bffe6ec9b4b 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -6823,8 +6823,9 @@ static int kvm_x86_cpu_online(unsigned int cpu)
 static int kvm_x86_cpu_offline(unsigned int cpu)
 {
 	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
-		return kvmclock_cpu_down_prep(cpu);
+		kvmclock_cpu_down_prep(cpu);
 
+	kvm_x86_call(cpu_offline)(cpu);
 	return 0;
 }
 
-- 
2.39.5