[PATCH] KVM: x86: Wake blocked vCPUs before offlining a CPU

[PATCH] KVM: x86: Wake blocked vCPUs before offlining a CPU

[Jinmeng Zhou]

Posted-interrupt wakeup state is tied to the physical CPU on which a vCPU
blocks. CPU hotplug migrates sleeping tasks before KVM's CPU offline
callback, but the vCPU remains on the old CPU's wakeup list and its posted
interrupt descriptor still targets the old physical APIC.

If a device posts an interrupt after the old CPU becomes unavailable, the
wakeup notification cannot make the blocked vCPU runnable. The vCPU cannot
repair the stale notification destination because that happens only after
the vCPU is scheduled back in.

Add an architecture hook to KVM's CPU offline path and a corresponding
optional x86 vendor callback. After CPUHP_AP_SCHED_WAIT_EMPTY has migrated
tasks away from the dying CPU, have VMX set KVM_REQ_UNBLOCK and wake every
vCPU on that CPU's posted-interrupt wakeup list. The vCPUs then unblock on
online CPUs and the existing load path rebuilds their wakeup-list and
posted-interrupt destination state.

Fixes: bf9f6ac8d749 ("KVM: Update Posted-Interrupts Descriptor when vCPU is blocked")
Cc: stable@vger.kernel.org
Signed-off-by: Jinmeng Zhou <zhoujinmeng@bytedance.com>
Signed-off-by: Guixiong Wei <weiguixiong@bytedance.com>
---
 arch/x86/include/asm/kvm-x86-ops.h |  1 +
 arch/x86/include/asm/kvm_host.h    |  1 +
 arch/x86/kvm/vmx/main.c            |  1 +
 arch/x86/kvm/vmx/posted_intr.c     | 23 +++++++++++++++++++++++
 arch/x86/kvm/vmx/posted_intr.h     |  1 +
 arch/x86/kvm/x86.c                 |  5 +++++
 include/linux/kvm_host.h           |  2 ++
 virt/kvm/kvm_main.c                |  5 +++++
 8 files changed, 39 insertions(+)

diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h
index e213c9ae3e301..27df8ebc97785 100644
--- a/arch/x86/include/asm/kvm-x86-ops.h
+++ b/arch/x86/include/asm/kvm-x86-ops.h
@@ -17,6 +17,7 @@
 KVM_X86_OP(check_processor_compatibility)
 KVM_X86_OP(enable_virtualization_cpu)
 KVM_X86_OP(disable_virtualization_cpu)
+KVM_X86_OP_OPTIONAL(prepare_cpu_offline)
 KVM_X86_OP(hardware_unsetup)
 KVM_X86_OP(has_emulated_msr)
 KVM_X86_OP(vcpu_after_set_cpuid)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 683bb8bf43a94..f230843e8566a 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1509,6 +1509,7 @@ struct kvm_x86_ops {
 
 	int (*enable_virtualization_cpu)(void);
 	void (*disable_virtualization_cpu)(void);
+	void (*prepare_cpu_offline)(unsigned int cpu);
 	cpu_emergency_virt_cb *emergency_disable_virtualization_cpu;
 
 	void (*hardware_unsetup)(void);
diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c
index 4c52ab8d07869..b3d61a85ba3bc 100644
--- a/arch/x86/kvm/vmx/main.c
+++ b/arch/x86/kvm/vmx/main.c
@@ -894,6 +894,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = {
 
 	.enable_virtualization_cpu = vmx_enable_virtualization_cpu,
 	.disable_virtualization_cpu = vt_op(disable_virtualization_cpu),
+	.prepare_cpu_offline = pi_wakeup_cpu_offline,
 	.emergency_disable_virtualization_cpu = vmx_emergency_disable_virtualization_cpu,
 
 	.has_emulated_msr = vt_op(has_emulated_msr),
diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c
index 4a6d9a17da238..2d6cca1606bd1 100644
--- a/arch/x86/kvm/vmx/posted_intr.c
+++ b/arch/x86/kvm/vmx/posted_intr.c
@@ -266,6 +266,29 @@ void pi_wakeup_handler(void)
 	raw_spin_unlock(spinlock);
 }
 
+void pi_wakeup_cpu_offline(unsigned int cpu)
+{
+	struct list_head *wakeup_list = &per_cpu(wakeup_vcpus_on_cpu, cpu);
+	raw_spinlock_t *spinlock = &per_cpu(wakeup_vcpus_on_cpu_lock, cpu);
+	struct vcpu_vt *vt;
+	unsigned long flags;
+
+	/*
+	 * CPUHP_AP_SCHED_WAIT_EMPTY has already migrated tasks away from the
+	 * dying CPU.  Force blocked vCPUs to leave the block loop so that their
+	 * PI wakeup state is rebuilt on an online CPU before the old notification
+	 * destination becomes unreachable.
+	 */
+	raw_spin_lock_irqsave(spinlock, flags);
+	list_for_each_entry(vt, wakeup_list, pi_wakeup_list) {
+		struct kvm_vcpu *vcpu = vt_to_vcpu(vt);
+
+		kvm_make_request(KVM_REQ_UNBLOCK, vcpu);
+		kvm_vcpu_wake_up(vcpu);
+	}
+	raw_spin_unlock_irqrestore(spinlock, flags);
+}
+
 void __init pi_init_cpu(int cpu)
 {
 	INIT_LIST_HEAD(&per_cpu(wakeup_vcpus_on_cpu, cpu));
diff --git a/arch/x86/kvm/vmx/posted_intr.h b/arch/x86/kvm/vmx/posted_intr.h
index a4af39948cf04..41414c583a063 100644
--- a/arch/x86/kvm/vmx/posted_intr.h
+++ b/arch/x86/kvm/vmx/posted_intr.h
@@ -11,6 +11,7 @@
 void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu);
 void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu);
 void pi_wakeup_handler(void);
+void pi_wakeup_cpu_offline(unsigned int cpu);
 void __init pi_init_cpu(int cpu);
 void pi_apicv_pre_state_restore(struct kvm_vcpu *vcpu);
 bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 79468ddfe4736..ea0ded5d64509 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -9820,6 +9820,11 @@ void kvm_arch_disable_virtualization_cpu(void)
 		__module_get(THIS_MODULE);
 }
 
+void kvm_arch_prepare_cpu_offline(unsigned int cpu)
+{
+	kvm_x86_call(prepare_cpu_offline)(cpu);
+}
+
 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)
 {
 	return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 03bfc92864b6e..d24fd09881dbf 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -1685,6 +1685,8 @@ void kvm_arch_disable_virtualization(void);
  */
 int kvm_arch_enable_virtualization_cpu(void);
 void kvm_arch_disable_virtualization_cpu(void);
+/* Called after tasks have migrated away from an offlining CPU. */
+void kvm_arch_prepare_cpu_offline(unsigned int cpu);
 #endif
 bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu);
 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu);
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 65eb26a0520d8..2169130302ca0 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -5610,6 +5610,10 @@ __weak void kvm_arch_disable_virtualization(void)
 
 }
 
+__weak void kvm_arch_prepare_cpu_offline(unsigned int cpu)
+{
+}
+
 static int kvm_enable_virtualization_cpu(void)
 {
 	if (__this_cpu_read(virtualization_enabled))
@@ -5647,6 +5651,7 @@ static void kvm_disable_virtualization_cpu(void *ign)
 
 static int kvm_offline_cpu(unsigned int cpu)
 {
+	kvm_arch_prepare_cpu_offline(cpu);
 	kvm_disable_virtualization_cpu(NULL);
 	return 0;
 }
-- 
2.39.5

Re: [PATCH] KVM: x86: Wake blocked vCPUs before offlining a CPU

[Sean Christopherson]

On Thu, Sep 24, 2026, Jinmeng Zhou wrote:
> Posted-interrupt wakeup state is tied to the physical CPU on which a vCPU
> blocks. CPU hotplug migrates sleeping tasks before KVM's CPU offline
> callback, but the vCPU remains on the old CPU's wakeup list and its posted
> interrupt descriptor still targets the old physical APIC.
> 
> If a device posts an interrupt after the old CPU becomes unavailable, the
> wakeup notification cannot make the blocked vCPU runnable. The vCPU cannot
> repair the stale notification destination because that happens only after
> the vCPU is scheduled back in.
> 
> Add an architecture hook to KVM's CPU offline path and a corresponding
> optional x86 vendor callback. After CPUHP_AP_SCHED_WAIT_EMPTY has migrated
> tasks away from the dying CPU, have VMX set KVM_REQ_UNBLOCK and wake every
> vCPU on that CPU's posted-interrupt wakeup list. The vCPUs then unblock on
> online CPUs and the existing load path rebuilds their wakeup-list and
> posted-interrupt destination state.
> 
> Fixes: bf9f6ac8d749 ("KVM: Update Posted-Interrupts Descriptor when vCPU is blocked")
> Cc: stable@vger.kernel.org
> Signed-off-by: Jinmeng Zhou <zhoujinmeng@bytedance.com>
> Signed-off-by: Guixiong Wei <weiguixiong@bytedance.com>

SoB chain is wrong.  Whoever sends the patch needs to come last.  And presumably
there's a missing "Co-developed-by: Guixiong Wei <weiguixiong@bytedance.com>".
The patch also needs a "From: Jinmeng Zhou <zhoujinmeng@bytedance.com>" since
you're using a different address to send the email.

> diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c
> index 4a6d9a17da238..2d6cca1606bd1 100644
> --- a/arch/x86/kvm/vmx/posted_intr.c
> +++ b/arch/x86/kvm/vmx/posted_intr.c
> @@ -266,6 +266,29 @@ void pi_wakeup_handler(void)
>  	raw_spin_unlock(spinlock);
>  }
>  
> +void pi_wakeup_cpu_offline(unsigned int cpu)
> +{
> +	struct list_head *wakeup_list = &per_cpu(wakeup_vcpus_on_cpu, cpu);
> +	raw_spinlock_t *spinlock = &per_cpu(wakeup_vcpus_on_cpu_lock, cpu);
> +	struct vcpu_vt *vt;
> +	unsigned long flags;
> +
> +	/*
> +	 * CPUHP_AP_SCHED_WAIT_EMPTY has already migrated tasks away from the

IIUC, CPUHP_AP_SCHED_WAIT_EMPTY just waits for task migrations to complete,
CPUHP_AP_ACTIVE => sched_cpu_deactivate() is what actually initiates the migration.
That matters because I think it means we can repurpose CPUHP_AP_X86_KVM_CLK_ONLINE.

> +	 * dying CPU.  Force blocked vCPUs to leave the block loop so that their
> +	 * PI wakeup state is rebuilt on an online CPU before the old notification
> +	 * destination becomes unreachable.
> +	 */
> +	raw_spin_lock_irqsave(spinlock, flags);
> +	list_for_each_entry(vt, wakeup_list, pi_wakeup_list) {
> +		struct kvm_vcpu *vcpu = vt_to_vcpu(vt);
> +
> +		kvm_make_request(KVM_REQ_UNBLOCK, vcpu);

Hrm, KVM_REQ_UNBLOCK is going to cause spurious wakeups for the vCPU.  That isn't
the end of the world, but it's definitely undesirable, especially since these
flows are shared with SUSPEND+RESUME.

If we attach to CPUHP_AP_X86_KVM_CLK_ONLINE, can we do a bare __kvm_vcpu_wake_up(),
so that the task (temporarily) wakes up and gets migrated to a new pCPU before this
pCPU goes down?

> +		kvm_vcpu_wake_up(vcpu);
> +	}
> +	raw_spin_unlock_irqrestore(spinlock, flags);
> +}
> +
>  void __init pi_init_cpu(int cpu)
>  {
>  	INIT_LIST_HEAD(&per_cpu(wakeup_vcpus_on_cpu, cpu));
> diff --git a/arch/x86/kvm/vmx/posted_intr.h b/arch/x86/kvm/vmx/posted_intr.h
> index a4af39948cf04..41414c583a063 100644
> --- a/arch/x86/kvm/vmx/posted_intr.h
> +++ b/arch/x86/kvm/vmx/posted_intr.h
> @@ -11,6 +11,7 @@
>  void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu);
>  void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu);
>  void pi_wakeup_handler(void);
> +void pi_wakeup_cpu_offline(unsigned int cpu);
>  void __init pi_init_cpu(int cpu);
>  void pi_apicv_pre_state_restore(struct kvm_vcpu *vcpu);
>  bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu);
> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index 79468ddfe4736..ea0ded5d64509 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -9820,6 +9820,11 @@ void kvm_arch_disable_virtualization_cpu(void)
>  		__module_get(THIS_MODULE);
>  }
>  
> +void kvm_arch_prepare_cpu_offline(unsigned int cpu)

I don't love adding an arch hook for this, because CPUHP_AP_KVM_ONLINE is tied to
KVM_GENERIC_HARDWARE_ENABLING=y.  I think I'd rather turn CPUHP_AP_X86_KVM_CLK_ONLINE
into a slightly more generic CPUHP_AP_X86_KVM_ONLINE

> +{
> +	kvm_x86_call(prepare_cpu_offline)(cpu);

My vote for the names would just be "cpu_offline", i.e. no "prepare".

[PATCH v2 0/2] KVM: x86: Wake blocked vCPUs before offlining a CPU

[Jinmeng Zhou]

From: Jinmeng Zhou <zhoujinmeng@bytedance.com>

A vCPU blocked with posted interrupts enabled stays on the wakeup list of
the physical CPU on which it blocked, and its posted-interrupt descriptor
keeps targeting that CPU's APIC.  If that CPU is offlined, a later device
interrupt sends the wakeup notification to a CPU that is gone, and the vCPU
is never woken.

Patch 1 turns CPUHP_AP_X86_KVM_CLK_ONLINE into a generic x86 KVM CPU
hotplug state that is always registered.  Today it is registered only on
hosts without a constant TSC.

Patch 2 adds an optional vendor cpu_offline() callback that runs from that
state's teardown, after the CPU has been deactivated.  VMX uses it to wake
the vCPUs on the dying CPU's wakeup list, so that they move to an active
CPU and rebuild their posted-interrupt state.

Patch 1 is a prerequisite for patch 2, so both are tagged for stable and
should be backported together.

Changes in v2:
- Reuse and generalize CPUHP_AP_X86_KVM_CLK_ONLINE instead of adding an
  arch hook to the generic CPUHP_AP_KVM_ONLINE path, which depends on
  KVM_GENERIC_HARDWARE_ENABLING=y.
- Register the x86 KVM CPU hotplug state unconditionally, so that the
  callback also runs on hosts with a constant TSC.
- Rename the vendor callback from prepare_cpu_offline() to cpu_offline().
- Use a bare __kvm_vcpu_wake_up() instead of KVM_REQ_UNBLOCK, so that the
  guest does not get spurious wakeups.
- Fix the changelog: the migration is started by CPUHP_AP_ACTIVE
  (sched_cpu_deactivate()), not CPUHP_AP_SCHED_WAIT_EMPTY.
- Initialize the per-CPU posted-interrupt wakeup lists before registering
  the CPU hotplug callbacks that walk them.
- Fix the trailers: add Co-developed-by and put the sender's SoB last, and
  add an in-body From: line for the author address.

v1: https://lore.kernel.org/kvm/20260924080648.51014-1-zhoujinmeng@bytedance.com/

Jinmeng Zhou (2):
  KVM: x86: Make the kvmclock CPU hotplug state generic
  KVM: x86: Wake blocked vCPUs before offlining a CPU

 arch/x86/include/asm/kvm-x86-ops.h |  1 +
 arch/x86/include/asm/kvm_host.h    |  1 +
 arch/x86/kvm/vmx/main.c            |  1 +
 arch/x86/kvm/vmx/posted_intr.c     | 18 ++++++++++++++++++
 arch/x86/kvm/vmx/posted_intr.h     |  1 +
 arch/x86/kvm/vmx/vmx.c             | 16 ++++++++++------
 arch/x86/kvm/x86.c                 | 30 ++++++++++++++++++++++++++----
 include/linux/cpuhotplug.h         |  2 +-
 8 files changed, 59 insertions(+), 11 deletions(-)


base-commit: a500db7819c50db59e55f1b4fa1c3baa5a2616f3
-- 
2.39.5

[PATCH v2 1/2] KVM: x86: Make the kvmclock CPU hotplug state generic

[Jinmeng Zhou]

From: Jinmeng Zhou <zhoujinmeng@bytedance.com>

Rename CPUHP_AP_X86_KVM_CLK_ONLINE to CPUHP_AP_X86_KVM_ONLINE so that the
state can be used for any x86 KVM work that needs to run when a CPU comes
online or goes offline, not just for kvmclock.

The state is currently registered by kvm_timer_init() only if the host
lacks X86_FEATURE_CONSTANT_TSC, so it is never installed on modern CPUs.
Register it unconditionally from kvm_x86_vendor_init(), and remove it from
kvm_x86_vendor_exit().  Add kvm_x86_cpu_online() and kvm_x86_cpu_offline()
wrappers that call the existing kvmclock helpers only on hosts without a
constant TSC.

Keep the registration after kvm_timer_init() so that the cpufreq notifier
is registered before kvmclock_cpu_online() samples the frequency of each
online CPU.

No functional change for kvmclock.  On hosts with a constant TSC, the
state is now registered but its callbacks do nothing.

Cc: stable@vger.kernel.org
Co-developed-by: Guixiong Wei <weiguixiong@bytedance.com>
Signed-off-by: Guixiong Wei <weiguixiong@bytedance.com>
Signed-off-by: Jinmeng Zhou <zhoujinmeng@bytedance.com>
---
 arch/x86/kvm/x86.c         | 29 +++++++++++++++++++++++++----
 include/linux/cpuhotplug.h |  2 +-
 2 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 79468ddfe4736..a08ae25f373fb 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -6812,6 +6812,22 @@ static int kvmclock_cpu_online(unsigned int cpu)
 	return 0;
 }
 
+static int kvm_x86_cpu_online(unsigned int cpu)
+{
+	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
+		return kvmclock_cpu_online(cpu);
+
+	return 0;
+}
+
+static int kvm_x86_cpu_offline(unsigned int cpu)
+{
+	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
+		return kvmclock_cpu_down_prep(cpu);
+
+	return 0;
+}
+
 static void kvm_timer_init(void)
 {
 	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
@@ -6832,9 +6848,6 @@ static void kvm_timer_init(void)
 		}
 		cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
 					  CPUFREQ_TRANSITION_NOTIFIER);
-
-		cpuhp_setup_state(CPUHP_AP_X86_KVM_CLK_ONLINE, "x86/kvm/clk:online",
-				  kvmclock_cpu_online, kvmclock_cpu_down_prep);
 	}
 }
 
@@ -7109,6 +7122,13 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops)
 	 */
 	kvm_timer_init();
 
+	/*
+	 * Register the CPU hotplug state after the cpufreq notifier so that
+	 * kvmclock_cpu_online() can't miss a frequency transition.
+	 */
+	cpuhp_setup_state(CPUHP_AP_X86_KVM_ONLINE, "x86/kvm:online",
+			  kvm_x86_cpu_online, kvm_x86_cpu_offline);
+
 	if (pi_inject_timer == -1)
 		pi_inject_timer = housekeeping_enabled(HK_TYPE_TIMER);
 #ifdef CONFIG_X86_64
@@ -7165,10 +7185,11 @@ void kvm_x86_vendor_exit(void)
 #endif
 	kvm_lapic_exit();
 
+	cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_ONLINE);
+
 	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
 		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
 					    CPUFREQ_TRANSITION_NOTIFIER);
-		cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE);
 	}
 #ifdef CONFIG_X86_64
 	pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index feb32949aeea6..91da42fd11161 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -241,7 +241,7 @@ enum cpuhp_state {
 	CPUHP_AP_ONLINE_DYN,
 	CPUHP_AP_ONLINE_DYN_END		= CPUHP_AP_ONLINE_DYN + 40,
 	CPUHP_AP_X86_HPET_ONLINE,
-	CPUHP_AP_X86_KVM_CLK_ONLINE,
+	CPUHP_AP_X86_KVM_ONLINE,
 	CPUHP_AP_ACTIVE,
 	CPUHP_ONLINE,
 };
-- 
2.39.5

[PATCH v2 2/2] KVM: x86: Wake blocked vCPUs before offlining a CPU

[Jinmeng Zhou]

From: Jinmeng Zhou <zhoujinmeng@bytedance.com>

Posted-interrupt wakeup state is tied to the physical CPU on which a vCPU
blocks.  When a CPU is offlined, sched_cpu_deactivate() removes it from the
active mask, but a blocked vCPU stays on that CPU's wakeup list and its
posted-interrupt descriptor still targets that CPU's APIC.

If a device posts an interrupt after the old CPU goes offline, the wakeup
notification cannot make the blocked vCPU runnable.  The vCPU cannot fix
the stale notification destination, because that happens only after the
vCPU is scheduled back in.

Add an optional x86 vendor cpu_offline() callback and call it from the
CPUHP_AP_X86_KVM_ONLINE teardown.  That teardown runs after CPUHP_AP_ACTIVE
has deactivated the CPU.  Have VMX wake every vCPU on the dying CPU's
posted-interrupt wakeup list, so that the scheduler places the vCPU on an
active CPU and vmx_vcpu_pi_load() rebuilds its wakeup list and
posted-interrupt destination state.

A single pass over the list is sufficient.  Once the CPU is inactive, the
scheduler won't select it for a wakeup, and balance_push() moves any task
that is switched in on it to an active CPU before the task can run.  Thus
no vCPU can block on the CPU and add itself to the list after the walk.

Use __kvm_vcpu_wake_up() rather than KVM_REQ_UNBLOCK.  A vCPU with no
pending event goes back to the block loop after its posted-interrupt state
has been migrated, and no spurious wakeup is reported to the guest.

Initialize the per-CPU posted-interrupt wakeup lists in vmx_init() before
calling kvm_x86_vendor_init().  Once the CPU hotplug state is registered,
a concurrent CPU offline can invoke the VMX callback, which must not walk
an uninitialized list.

Fixes: bf9f6ac8d749 ("KVM: Update Posted-Interrupts Descriptor when vCPU is blocked")
Cc: stable@vger.kernel.org
Co-developed-by: Guixiong Wei <weiguixiong@bytedance.com>
Signed-off-by: Guixiong Wei <weiguixiong@bytedance.com>
Signed-off-by: Jinmeng Zhou <zhoujinmeng@bytedance.com>
---
 arch/x86/include/asm/kvm-x86-ops.h |  1 +
 arch/x86/include/asm/kvm_host.h    |  1 +
 arch/x86/kvm/vmx/main.c            |  1 +
 arch/x86/kvm/vmx/posted_intr.c     | 18 ++++++++++++++++++
 arch/x86/kvm/vmx/posted_intr.h     |  1 +
 arch/x86/kvm/vmx/vmx.c             | 16 ++++++++++------
 arch/x86/kvm/x86.c                 |  3 ++-
 7 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h
index e213c9ae3e301..f2a805783f045 100644
--- a/arch/x86/include/asm/kvm-x86-ops.h
+++ b/arch/x86/include/asm/kvm-x86-ops.h
@@ -17,6 +17,7 @@
 KVM_X86_OP(check_processor_compatibility)
 KVM_X86_OP(enable_virtualization_cpu)
 KVM_X86_OP(disable_virtualization_cpu)
+KVM_X86_OP_OPTIONAL(cpu_offline)
 KVM_X86_OP(hardware_unsetup)
 KVM_X86_OP(has_emulated_msr)
 KVM_X86_OP(vcpu_after_set_cpuid)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 683bb8bf43a94..d47863ce2c5ae 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1509,6 +1509,7 @@ struct kvm_x86_ops {
 
 	int (*enable_virtualization_cpu)(void);
 	void (*disable_virtualization_cpu)(void);
+	void (*cpu_offline)(unsigned int cpu);
 	cpu_emergency_virt_cb *emergency_disable_virtualization_cpu;
 
 	void (*hardware_unsetup)(void);
diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c
index 4c52ab8d07869..5a3f77f42d20d 100644
--- a/arch/x86/kvm/vmx/main.c
+++ b/arch/x86/kvm/vmx/main.c
@@ -894,6 +894,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = {
 
 	.enable_virtualization_cpu = vmx_enable_virtualization_cpu,
 	.disable_virtualization_cpu = vt_op(disable_virtualization_cpu),
+	.cpu_offline = pi_cpu_offline,
 	.emergency_disable_virtualization_cpu = vmx_emergency_disable_virtualization_cpu,
 
 	.has_emulated_msr = vt_op(has_emulated_msr),
diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c
index 4a6d9a17da238..1905472b336d3 100644
--- a/arch/x86/kvm/vmx/posted_intr.c
+++ b/arch/x86/kvm/vmx/posted_intr.c
@@ -266,6 +266,24 @@ void pi_wakeup_handler(void)
 	raw_spin_unlock(spinlock);
 }
 
+void pi_cpu_offline(unsigned int cpu)
+{
+	struct list_head *wakeup_list = &per_cpu(wakeup_vcpus_on_cpu, cpu);
+	raw_spinlock_t *spinlock = &per_cpu(wakeup_vcpus_on_cpu_lock, cpu);
+	struct vcpu_vt *vt;
+	unsigned long flags;
+
+	/*
+	 * CPUHP_AP_ACTIVE has already deactivated the CPU.  Wake blocked vCPUs
+	 * so that they are scheduled on another CPU and rebuild their posted
+	 * interrupt wakeup state in vmx_vcpu_pi_load().
+	 */
+	raw_spin_lock_irqsave(spinlock, flags);
+	list_for_each_entry(vt, wakeup_list, pi_wakeup_list)
+		__kvm_vcpu_wake_up(vt_to_vcpu(vt));
+	raw_spin_unlock_irqrestore(spinlock, flags);
+}
+
 void __init pi_init_cpu(int cpu)
 {
 	INIT_LIST_HEAD(&per_cpu(wakeup_vcpus_on_cpu, cpu));
diff --git a/arch/x86/kvm/vmx/posted_intr.h b/arch/x86/kvm/vmx/posted_intr.h
index a4af39948cf04..530f1347f8b0d 100644
--- a/arch/x86/kvm/vmx/posted_intr.h
+++ b/arch/x86/kvm/vmx/posted_intr.h
@@ -11,6 +11,7 @@
 void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu);
 void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu);
 void pi_wakeup_handler(void);
+void pi_cpu_offline(unsigned int cpu);
 void __init pi_init_cpu(int cpu);
 void pi_apicv_pre_state_restore(struct kvm_vcpu *vcpu);
 bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu);
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 612ab07d4100f..ff1a6ccbd043b 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -8889,6 +8889,16 @@ int __init vmx_init(void)
 	if (setup_vmcs_config(&vmcs_config, &vmx_capability) < 0)
 		return -EIO;
 
+	/*
+	 * Initialize the per-CPU lists before common x86 init, which registers
+	 * the CPU hotplug callbacks that walk the posted-interrupt wakeup lists.
+	 */
+	for_each_possible_cpu(cpu) {
+		INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu));
+
+		pi_init_cpu(cpu);
+	}
+
 	r = kvm_x86_vendor_init(&vt_init_ops);
 	if (r)
 		return r;
@@ -8898,12 +8908,6 @@ int __init vmx_init(void)
 	if (r)
 		goto err_l1d_flush;
 
-	for_each_possible_cpu(cpu) {
-		INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu));
-
-		pi_init_cpu(cpu);
-	}
-
 	vmx_check_vmcs12_offsets();
 
 	return 0;
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index a08ae25f373fb..38bffe6ec9b4b 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -6823,8 +6823,9 @@ static int kvm_x86_cpu_online(unsigned int cpu)
 static int kvm_x86_cpu_offline(unsigned int cpu)
 {
 	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
-		return kvmclock_cpu_down_prep(cpu);
+		kvmclock_cpu_down_prep(cpu);
 
+	kvm_x86_call(cpu_offline)(cpu);
 	return 0;
 }
 
-- 
2.39.5

Re: [PATCH v2 2/2] KVM: x86: Wake blocked vCPUs before offlining a CPU

[sashiko-bot]

Thank you for your contribution! Sashiko AI review found 2 potential issue(s) to consider:
- [Critical] KVM: VMX: out-of-bounds per-CPU access in vmx_vcpu_pi_load() on TDX CPU offline
- [High] KVM: x86: pi_cpu_offline() runs before CPUHP_AP_SCHED_WAIT_EMPTY drains tasks
--

commit e59f5fb4482718a249da5dc7f86ef0922b551270
Author: Jinmeng Zhou <zhoujinmeng@bytedance.com>

KVM: x86: Wake blocked vCPUs before offlining a CPU

This commit adds an optional x86 vendor cpu_offline() callback invoked during
CPUHP_AP_X86_KVM_ONLINE teardown and implements it for VMX to wake blocked
vCPUs on the outgoing CPU's posted-interrupt wakeup list so their
posted-interrupt destination state is migrated to an active CPU.

> diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c
> index 4a6d9a17da238..1905472b336d3 100644
> --- a/arch/x86/kvm/vmx/posted_intr.c
> +++ b/arch/x86/kvm/vmx/posted_intr.c
> @@ -266,6 +266,24 @@ void pi_wakeup_handler(void)
>  	raw_spin_unlock(spinlock);
>  }
>  
> +void pi_cpu_offline(unsigned int cpu)
> +{
> +	struct list_head *wakeup_list = &per_cpu(wakeup_vcpus_on_cpu, cpu);
> +	raw_spinlock_t *spinlock = &per_cpu(wakeup_vcpus_on_cpu_lock, cpu);
> +	struct vcpu_vt *vt;
> +	unsigned long flags;
> +
> +	/*
> +	 * CPUHP_AP_ACTIVE has already deactivated the CPU.  Wake blocked vCPUs
> +	 * so that they are scheduled on another CPU and rebuild their posted
> +	 * interrupt wakeup state in vmx_vcpu_pi_load().
> +	 */
> +	raw_spin_lock_irqsave(spinlock, flags);
> +	list_for_each_entry(vt, wakeup_list, pi_wakeup_list)
> +		__kvm_vcpu_wake_up(vt_to_vcpu(vt));
> +	raw_spin_unlock_irqrestore(spinlock, flags);
> +}

[Severity: Critical]
Can this trigger an out-of-bounds per-CPU access in vmx_vcpu_pi_load() when
offlining a CPU with a blocked TDX vCPU that uses VT-d posted interrupts?

When a TDX vCPU blocks on CPU C with posted interrupts enabled,
tdx_vcpu_put() -> vmx_vcpu_pi_put() -> pi_enable_wakeup_handler() adds
vt->pi_wakeup_list to per_cpu(wakeup_vcpus_on_cpu, C) and sets pi_desc->nv to
POSTED_INTR_WAKEUP_VECTOR while the vCPU is on per_cpu(associated_tdvcpus, C)
with vcpu->cpu == C.

During CPU hot-unplug of CPU C, pi_cpu_offline(C) calls __kvm_vcpu_wake_up(),
which asynchronously wakes the vCPU task on another active CPU D without
removing vt->pi_wakeup_list or waiting for the vCPU task to schedule in on
CPU D.

As CPU C continues hotplug teardown to CPUHP_AP_KVM_ONLINE (since
CPUHP_AP_SCHED_WAIT_EMPTY only waits for CPU C's own runqueue to drain),
kvm_offline_cpu() -> kvm_disable_virtualization_cpu() ->
vt_disable_virtualization_cpu() -> tdx_disable_virtualization_cpu() ->
tdx_flush_vp() -> tdx_disassociate_vp() runs on CPU C and resets vcpu->cpu to
-1:

arch/x86/kvm/vmx/tdx.c:tdx_disassociate_vp() {
	lockdep_assert_irqs_disabled();

	list_del(&to_tdx(vcpu)->cpu_list);
	...
	smp_wmb();

	vcpu->cpu = -1;
}

When the woken TDX vCPU task subsequently schedules in on CPU D,
kvm_sched_in() -> kvm_arch_vcpu_load() -> vt_vcpu_load() -> tdx_vcpu_load()
calls vmx_vcpu_pi_load(vcpu, D) before vcpu->cpu is updated at the end of
kvm_arch_vcpu_load():

arch/x86/kvm/vmx/posted_intr.c:vmx_vcpu_pi_load() {
	...
	if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR) {
		raw_spinlock_t *spinlock = &per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu);

		raw_spin_lock(spinlock);
		spin_acquire(&spinlock->dep_map, PI_LOCK_SCHED_OUT, 0, _RET_IP_);
		list_del(&vt->pi_wakeup_list);
		spin_release(&spinlock->dep_map, _RET_IP_);
		raw_spin_unlock(spinlock);
	}
}

With vcpu->cpu == -1, doesn't &per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu)
index __per_cpu_offset[-1] out of bounds, pass an invalid spinlock pointer to
raw_spin_lock(), and delete vt->pi_wakeup_list without holding CPU C's
wakeup_vcpus_on_cpu_lock?

> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index a08ae25f373fb..38bffe6ec9b4b 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -6823,8 +6823,9 @@ static int kvm_x86_cpu_online(unsigned int cpu)
>  static int kvm_x86_cpu_offline(unsigned int cpu)
>  {
>  	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
> -		return kvmclock_cpu_down_prep(cpu);
> +		kvmclock_cpu_down_prep(cpu);
>  
> +	kvm_x86_call(cpu_offline)(cpu);
>  	return 0;
>  }

[Severity: High]
Could a vCPU task in a migrate_disable() region still block on the outgoing
CPU and add itself to wakeup_vcpus_on_cpu after kvm_x86_cpu_offline() has
already walked the list?

During CPU teardown, callbacks run in descending order of enum cpuhp_state:

include/linux/cpuhotplug.h:
	CPUHP_AP_KVM_ONLINE,
	CPUHP_AP_SCHED_WAIT_EMPTY,
	...
	CPUHP_AP_X86_KVM_ONLINE,
	CPUHP_AP_ACTIVE,

In CPUHP_AP_ACTIVE, sched_cpu_deactivate() clears cpu_active_mask and enables
balance_push_set(cpu, true), but draining tasks in migrate_disable() regions
on the outgoing CPU happens later at CPUHP_AP_SCHED_WAIT_EMPTY via
sched_cpu_wait_empty() -> balance_hotplug_wait().

If a vCPU task running vcpu_run() enters a migrate_disable() region (for
example on CONFIG_PREEMPT_RT where spinlock_t, rwlock_t such as kvm->mmu_lock,
and local_lock_t call migrate_disable(), or via bpf_prog_run_pin_on_cpu())
and gets preempted or blocks on an rtmutex, __schedule() calls
migrate_disable_switch(), setting p->cpus_ptr = cpumask_of(cpu) while
p->cpus_mask retains its normal affinity mask.

CPUHP_AP_X86_KVM_ONLINE then runs kvm_x86_cpu_offline() -> pi_cpu_offline(cpu)
while the vCPU is not in kvm_vcpu_block(), and the hotplug thread waits at
CPUHP_AP_SCHED_WAIT_EMPTY in balance_hotplug_wait() because
rq_has_pinned_tasks(rq) is true.

When the vCPU task switches back in on the dying CPU, finish_lock_switch() ->
__balance_callbacks() -> balance_push() returns early because migration is
disabled:

kernel/sched/core.c:balance_push() {
	...
	if (kthread_is_per_cpu(push_task) ||
	    is_migration_disabled(push_task)) {
		...
		return;
	}
}

When the vCPU task exits its migrate_disable() region, __migrate_enable() ->
___migrate_enable() -> __set_cpus_allowed_ptr() -> affine_move_task() checks
whether the current CPU is still in p->cpus_mask:

kernel/sched/core.c:affine_move_task() {
	...
	if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask) ||
	    (task_current_donor(rq, p) && !task_current(rq, p))) {
		struct task_struct *push_task = NULL;

		if ((flags & SCA_MIGRATE_ENABLE) &&
		    (p->migration_flags & MDF_PUSH) && !rq->push_busy) {
			rq->push_busy = true;
			push_task = get_task_struct(p);
		}
		...
		return 0;
	}
}

Because offline CPUs are not cleared from p->cpus_mask for tasks in
top_cpuset, and MDF_PUSH was not set (MDF_PUSH is only set by RT/DL
push_cpu_stop()), affine_move_task() returns 0 without migrating the task or
setting TIF_NEED_RESCHED.

If the vCPU task then reaches kvm_vcpu_halt() -> kvm_vcpu_block() ->
schedule() before another preemption occurs, kvm_sched_out() ->
vmx_vcpu_pi_put() -> pi_enable_wakeup_handler() will add vt->pi_wakeup_list to
per_cpu(wakeup_vcpus_on_cpu, cpu) with pi_desc.ndst targeting the offline CPU
after pi_cpu_offline(cpu) has already finished.

Should pi_cpu_offline() instead be invoked after CPUHP_AP_SCHED_WAIT_EMPTY
(for instance from the CPUHP_AP_KVM_ONLINE stage, which was placed below
CPUHP_AP_SCHED_WAIT_EMPTY so balance_hotplug_wait() has already drained all
vCPU tasks from the outgoing CPU)?

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20261020085401.80296-1-jinmengzhou22@gmail.com?part=2