[PATCH 1/2] KVM: register asynchronous page fault waits with interrupts disabled

[PATCH 1/2] KVM: register asynchronous page fault waits with interrupts disabled

[Paolo Bonzini]

Asynchronous page fault handling has some wonky code to detect page ready
notifications that are sent before the waiter is registered.  When page
ready notifications for asynchronous page faults were separated from
not-present notifications and turned into an interrupt, this code was
unwittingly made almost-redundant.  Page ready notification is now delayed
until after interrupts are enabled; and for an asynchronous page faults
that occur outside guest mode, Linux never enables interrupts between the
delivery of the #PF exception and the call to kvm_async_pf_queue_task().
Therefore, the dummy allocation path is fully dead on that common path.

For the nested virtualization case, however, vmx.c and svm.c do
record in the interrupt-disabled region whether a #PF vmexit is an
asynchronous page fault, but they defer registration of the waiter until
kvm_handle_page_fault().  Because of this, the vCPU thread can be preempted
before kvm_async_pf_task_wait_schedule() is called, and apf_task_wake_one()
may them not find the vCPU thread in the waiters list.  It then allocates
the struct kvm_task_sleep_node* on behalf of the task that received the
asynchronous page fault.  Besides keeping alive the wonky code, allocating
the node from an hardirq-disabled region is a problem for PREEMPT_RT kernels,
as duly reported by syzkaller.

Split registration from wait so that the caller decides where to
put the node and so that KVM can anticipate the registration before
interrupts are enabled.  KVM cannot use a node on the stack like
kvm_async_pf_task_wait_schedule() does, but it can place it in the vCPU
struct instead.

Reported-by: syzbot+f11512d1e5878ba5e60e@syzkaller.appspotmail.com
Link: https://lore.kernel.org/kvm/6adf2939.67317f00.38c4bc.00ae.GAE@google.com/
Cc: Sean Christopherson <seanjc@google.com>
Cc: stable@vger.kernel.org
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 arch/x86/include/asm/kvm_host.h |  1 +
 arch/x86/include/asm/kvm_para.h | 21 +++++++++++++++++++--
 arch/x86/kernel/kvm.c           | 31 ++++++++++++-------------------
 arch/x86/kvm/mmu.h              | 14 ++++++++++++++
 arch/x86/kvm/mmu/mmu.c          |  4 +++-
 arch/x86/kvm/svm/svm.c          |  3 +--
 arch/x86/kvm/vmx/main.c         |  2 +-
 7 files changed, 51 insertions(+), 25 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 1ee4b127004d..3f096a0f3387 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -951,6 +951,7 @@ struct kvm_vcpu_arch {
 		u16 vec;
 		u32 id;
 		u32 host_apf_flags;
+		struct kvm_task_sleep_node host_apf_node;
 		bool pageready_pending;
 	} apf;
 
diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h
index 4a47c16e2df8..1e28c89b1405 100644
--- a/arch/x86/include/asm/kvm_para.h
+++ b/arch/x86/include/asm/kvm_para.h
@@ -5,6 +5,7 @@
 #include <asm/processor.h>
 #include <asm/alternative.h>
 #include <linux/interrupt.h>
+#include <linux/swait.h>
 #include <uapi/asm/kvm_para.h>
 
 #include <asm/tdx.h>
@@ -117,13 +118,22 @@ static inline long kvm_sev_hypercall3(unsigned int nr, unsigned long p1,
 	return ret;
 }
 
+struct kvm_task_sleep_node {
+	struct hlist_node link;
+	struct swait_queue_head wq;
+	u32 token;
+	int cpu;
+	bool dummy;
+};
+
 #ifdef CONFIG_KVM_GUEST
 void kvmclock_init(void);
 void kvmclock_disable(void);
 bool kvm_para_available(void);
 unsigned int kvm_arch_para_features(void);
 unsigned int kvm_arch_para_hints(void);
-void kvm_async_pf_task_wait_schedule(u32 token);
+bool kvm_async_pf_queue_task(u32 token, struct kvm_task_sleep_node *n);
+void kvm_async_pf_task_wait_schedule(struct kvm_task_sleep_node *n);
 u32 kvm_read_and_reset_apf_flags(void);
 bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token);
 
@@ -146,7 +156,14 @@ static inline void kvm_spinlock_init(void)
 #endif /* CONFIG_PARAVIRT_SPINLOCKS */
 
 #else /* CONFIG_KVM_GUEST */
-#define kvm_async_pf_task_wait_schedule(T) do {} while(0)
+static inline bool kvm_async_pf_queue_task(u32 token, struct kvm_task_sleep_node *n)
+{
+	return false;
+}
+
+static inline void kvm_async_pf_task_wait_schedule(struct kvm_task_sleep_node *n)
+{
+}
 
 static inline bool kvm_para_available(void)
 {
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 9f0efbe94793..ca5dcdc7ffde 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -80,14 +80,6 @@ static int has_guest_poll = 0;
 #define KVM_TASK_SLEEP_HASHBITS 8
 #define KVM_TASK_SLEEP_HASHSIZE (1<<KVM_TASK_SLEEP_HASHBITS)
 
-struct kvm_task_sleep_node {
-	struct hlist_node link;
-	struct swait_queue_head wq;
-	u32 token;
-	int cpu;
-	bool dummy;
-};
-
 static struct kvm_task_sleep_head {
 	raw_spinlock_t lock;
 	struct hlist_head list;
@@ -108,12 +100,14 @@ static struct kvm_task_sleep_node *_find_apf_task(struct kvm_task_sleep_head *b,
 	return NULL;
 }
 
-static bool kvm_async_pf_queue_task(u32 token, struct kvm_task_sleep_node *n)
+bool kvm_async_pf_queue_task(u32 token, struct kvm_task_sleep_node *n)
 {
 	u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
 	struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
 	struct kvm_task_sleep_node *e;
 
+	lockdep_assert_irqs_disabled();
+
 	raw_spin_lock(&b->lock);
 	e = _find_apf_task(b, token);
 	if (e) {
@@ -142,34 +136,31 @@ static bool kvm_async_pf_queue_task(u32 token, struct kvm_task_sleep_node *n)
 	raw_spin_unlock(&b->lock);
 	return true;
 }
+EXPORT_SYMBOL_FOR_KVM(kvm_async_pf_queue_task);
 
 /*
  * kvm_async_pf_task_wait_schedule - Wait for pagefault to be handled
- * @token:	Token to identify the sleep node entry
+ * @n:	Previously queued sleep node.
  *
  * Invoked from the async pagefault handling code or from the VM exit page
  * fault handler. In both cases RCU is watching.
  */
-void kvm_async_pf_task_wait_schedule(u32 token)
+void kvm_async_pf_task_wait_schedule(struct kvm_task_sleep_node *n)
 {
-	struct kvm_task_sleep_node n;
 	DECLARE_SWAITQUEUE(wait);
 
 	lockdep_assert_irqs_disabled();
 
-	if (!kvm_async_pf_queue_task(token, &n))
-		return;
-
 	for (;;) {
-		prepare_to_swait_exclusive(&n.wq, &wait, TASK_UNINTERRUPTIBLE);
-		if (hlist_unhashed(&n.link))
+		prepare_to_swait_exclusive(&n->wq, &wait, TASK_UNINTERRUPTIBLE);
+		if (hlist_unhashed(&n->link))
 			break;
 
 		local_irq_enable();
 		schedule();
 		local_irq_disable();
 	}
-	finish_swait(&n.wq, &wait);
+	finish_swait(&n->wq, &wait);
 }
 EXPORT_SYMBOL_FOR_KVM(kvm_async_pf_task_wait_schedule);
 
@@ -273,6 +264,7 @@ static __always_inline bool kvm_apf_reason_pending(void)
 
 noinstr bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token)
 {
+	struct kvm_task_sleep_node n;
 	irqentry_state_t state;
 	u32 flags;
 
@@ -314,7 +306,8 @@ noinstr bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token)
 
 	if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) {
 		/* Page is swapped out by the host. */
-		kvm_async_pf_task_wait_schedule(token);
+		if (kvm_async_pf_queue_task(token, &n))
+			kvm_async_pf_task_wait_schedule(&n);
 	} else {
 		WARN_ONCE(1, "Unexpected async PF flags: %x\n", flags);
 	}
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index b2a295a9f084..3cb2616d88ac 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -162,6 +162,20 @@ void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level,
 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu);
 int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
 				u64 fault_address, char *insn, int insn_len);
+
+static inline void kvm_handle_host_apf_irqoff(struct kvm_vcpu *vcpu, u32 token)
+{
+	vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags();
+	if ((vcpu->arch.apf.host_apf_flags & KVM_PV_REASON_PAGE_NOT_PRESENT) &&
+	    !kvm_async_pf_queue_task(token, &vcpu->arch.apf.host_apf_node)) {
+		/*
+		 * Nothing to do but regular page fault handling must still
+		 * be suppressed, so leave host_apf_flags nonzero.
+		 */
+		vcpu->arch.apf.host_apf_flags = KVM_PV_REASON_PAGE_READY;
+	}
+}
+
 void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
 					struct kvm_pagewalk *pw);
 
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index dc2aef72ffdd..1f2ae73275aa 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -4979,8 +4979,10 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
 	} else if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) {
 		vcpu->arch.apf.host_apf_flags = 0;
 		local_irq_disable();
-		kvm_async_pf_task_wait_schedule(fault_address);
+		kvm_async_pf_task_wait_schedule(&vcpu->arch.apf.host_apf_node);
 		local_irq_enable();
+	} else if (flags & KVM_PV_REASON_PAGE_READY) {
+		vcpu->arch.apf.host_apf_flags = 0;
 	} else {
 		WARN_ONCE(1, "Unexpected host async PF flags: %x\n", flags);
 	}
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index 136069dfcd44..e0ff458761cf 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -4764,8 +4764,7 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 
 	/* if exit due to PF check for async PF */
 	if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
-		vcpu->arch.apf.host_apf_flags =
-			kvm_read_and_reset_apf_flags();
+		kvm_handle_host_apf_irqoff(vcpu, svm->vmcb->control.exit_info_2);
 
 	kvm_clear_available_registers(vcpu, SVM_REGS_LAZY_LOAD_SET);
 
diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c
index d75170a6199e..b46f8f60ce70 100644
--- a/arch/x86/kvm/vmx/main.c
+++ b/arch/x86/kvm/vmx/main.c
@@ -916,7 +916,7 @@ static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info)
 {
 	/* if exit due to PF check for async PF */
 	if (is_page_fault(intr_info))
-		vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags();
+		kvm_handle_host_apf_irqoff(vcpu, vt_get_exit_qual(vcpu));
 	/* if exit due to NM, handle before interrupts are enabled */
 	else if (is_nm_fault(intr_info))
 		handle_nm_fault_irqoff(vcpu);
-- 
2.56.0

Re: [PATCH 1/2] KVM: register asynchronous page fault waits with interrupts disabled

[Sean Christopherson]

On Tue, Nov 03, 2026, Paolo Bonzini wrote:
> KVM cannot use a node on the stack like kvm_async_pf_task_wait_schedule() does

Oh!  Yes, KVM can!  If we fully handle the PV async #PF in vcpu_enter_guest(),
then the node can be put on the stack, right?  Juggling preemption and guest
timing is heinous, but on the plus side it consolidates the PV async #PF code,
so I think it's a net positive?  Especially when factoring in the node stuff.

Not properly nested (we really need PV async #PF test coverage).

From: Sean Christopherson <seanjc@google.com>
Date: Tue, 3 Nov 2026 06:08:54 -0800
Subject: [PATCH] KVM: x86: Handle KVM-as-a-guest paravirt async #PFs before
 enabling IRQs

Fully handle PV async #PFs (when running as a KVM guest) before enabling
IRQs so that it's impossible for the current task, i.e. the vCPU task, to
be preempted before invoking kvm_async_pf_task_wait_schedule().  This will
allow the guest-side async #PF code to eliminate its dummy allocation,
which is needed only to handle the case where the page ready notification
is delivered before handling the not-present #PF.

Add dedicated handling for PV async #PFs instead of wrapping/piggybacking
kvm_x86_ops.handle_exit_irqoff(), as the act of handling an async #PF may
enable IRQs and schedule out the current task, i.e. needs to be done just
before KVM itself explicitly enables IRQs, and needs to enable preemption
before calling into the kernel's PV async #PF handler.

Opportunistically wrap the code with CONFIG_KVM_GUEST to help document the
context, and drop the CONFIG_KVM_GUEST=n stubs.

Suggested-by: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Sean Christopherson <seanjc@google.com>
---
 arch/x86/include/asm/kvm-x86-ops.h |  1 +
 arch/x86/include/asm/kvm_host.h    |  2 +-
 arch/x86/include/asm/kvm_para.h    |  7 ---
 arch/x86/kvm/mmu/mmu.c             | 19 +-------
 arch/x86/kvm/svm/nested.c          |  4 --
 arch/x86/kvm/svm/svm.c             | 16 +++++--
 arch/x86/kvm/vmx/main.c            | 14 ++++--
 arch/x86/kvm/vmx/nested.c          |  3 +-
 arch/x86/kvm/vmx/vmx.c             | 13 +++--
 arch/x86/kvm/vmx/x86_ops.h         |  1 +
 arch/x86/kvm/x86.c                 | 76 ++++++++++++++++++++++--------
 11 files changed, 92 insertions(+), 64 deletions(-)

diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h
index 48fa973b98d5..119fb588fc30 100644
--- a/arch/x86/include/asm/kvm-x86-ops.h
+++ b/arch/x86/include/asm/kvm-x86-ops.h
@@ -107,6 +107,7 @@ KVM_X86_OP(write_tsc_offset)
 KVM_X86_OP(write_tsc_multiplier)
 KVM_X86_OP(get_exit_info)
 KVM_X86_OP(get_entry_info)
+KVM_X86_OP(get_pf_exit_info)
 KVM_X86_OP(check_intercept)
 KVM_X86_OP(handle_exit_irqoff)
 KVM_X86_OP_OPTIONAL(update_cpu_dirty_logging)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 0378577d9d8b..63cc2026dfce 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -959,7 +959,6 @@ struct kvm_vcpu_arch {
 		u64 msr_int_val; /* MSR_KVM_ASYNC_PF_INT */
 		u16 vec;
 		u32 id;
-		u32 host_apf_flags;
 		bool pageready_pending;
 	} apf;
 
@@ -1683,6 +1682,7 @@ struct kvm_x86_ops {
 
 	void (*get_entry_info)(struct kvm_vcpu *vcpu,
 			       u32 *intr_info, u32 *error_code);
+	bool (*get_pf_exit_info)(struct kvm_vcpu *vcpu, u64 *fault_address);
 
 	int (*check_intercept)(struct kvm_vcpu *vcpu,
 			       struct x86_instruction_info *info,
diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h
index 4a47c16e2df8..857d4ac7dc42 100644
--- a/arch/x86/include/asm/kvm_para.h
+++ b/arch/x86/include/asm/kvm_para.h
@@ -146,8 +146,6 @@ static inline void kvm_spinlock_init(void)
 #endif /* CONFIG_PARAVIRT_SPINLOCKS */
 
 #else /* CONFIG_KVM_GUEST */
-#define kvm_async_pf_task_wait_schedule(T) do {} while(0)
-
 static inline bool kvm_para_available(void)
 {
 	return false;
@@ -163,11 +161,6 @@ static inline unsigned int kvm_arch_para_hints(void)
 	return 0;
 }
 
-static inline u32 kvm_read_and_reset_apf_flags(void)
-{
-	return 0;
-}
-
 static __always_inline bool kvm_handle_async_pf(struct pt_regs *regs, u32 token)
 {
 	return false;
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index f4b12d6aef55..3cc4bf506cb1 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -4954,9 +4954,6 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu,
 int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
 				u64 fault_address, char *insn, int insn_len)
 {
-	int r = 1;
-	u32 flags = vcpu->arch.apf.host_apf_flags;
-
 #ifndef CONFIG_X86_64
 	/* A 64-bit CR2 should be impossible on 32-bit KVM. */
 	if (KVM_BUG_ON(fault_address >> 32, vcpu->kvm))
@@ -4977,21 +4974,9 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
 	BUILD_BUG_ON(lower_32_bits(PFERR_SYNTHETIC_MASK));
 
 	kvm_request_l1tf_flush_l1d();
-	if (!flags) {
-		trace_kvm_page_fault(vcpu, fault_address, error_code);
+	trace_kvm_page_fault(vcpu, fault_address, error_code);
 
-		r = kvm_mmu_page_fault(vcpu, fault_address, error_code, insn,
-				insn_len);
-	} else if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) {
-		vcpu->arch.apf.host_apf_flags = 0;
-		local_irq_disable();
-		kvm_async_pf_task_wait_schedule(fault_address);
-		local_irq_enable();
-	} else {
-		WARN_ONCE(1, "Unexpected host async PF flags: %x\n", flags);
-	}
-
-	return r;
+	return kvm_mmu_page_fault(vcpu, fault_address, error_code, insn, insn_len);
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_handle_page_fault);
 
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index 07554089ec05..d4951dc8b1f7 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -1818,10 +1818,6 @@ int nested_svm_exit_special(struct vcpu_svm *svm)
 		if (svm->vmcb01.ptr->control.intercepts[INTERCEPT_EXCEPTION] &
 		    excp_bits)
 			return NESTED_EXIT_HOST;
-		else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR &&
-			 svm->vcpu.arch.apf.host_apf_flags)
-			/* Trap async PF even if not shadowing */
-			return NESTED_EXIT_HOST;
 		break;
 	}
 	case SVM_EXIT_VMMCALL:
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index 581bc3a23c95..32e1f6d0fc38 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -3811,7 +3811,17 @@ static void svm_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info,
 		*error_code = control->event_inj_err;
 	else
 		*error_code = 0;
+}
 
+static bool svm_get_pf_exit_info(struct kvm_vcpu *vcpu, u64 *fault_address)
+{
+	struct vcpu_svm *svm = to_svm(vcpu);
+
+	if (svm->vmcb->control.exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR)
+		return false;
+
+	*fault_address = svm->vmcb->control.exit_info_2;
+	return true;
 }
 
 static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
@@ -4758,11 +4768,6 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 			       vcpu_to_pmu(vcpu)->global_ctrl);
 	}
 
-	/* if exit due to PF check for async PF */
-	if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
-		vcpu->arch.apf.host_apf_flags =
-			kvm_read_and_reset_apf_flags();
-
 	kvm_clear_available_registers(vcpu, SVM_REGS_LAZY_LOAD_SET);
 
 	trace_kvm_exit(vcpu, KVM_ISA_SVM);
@@ -5544,6 +5549,7 @@ struct kvm_x86_ops svm_x86_ops __initdata = {
 
 	.get_exit_info = svm_get_exit_info,
 	.get_entry_info = svm_get_entry_info,
+	.get_pf_exit_info = svm_get_pf_exit_info,
 
 	.vcpu_after_set_cpuid = svm_vcpu_after_set_cpuid,
 
diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c
index d75170a6199e..370f649595f9 100644
--- a/arch/x86/kvm/vmx/main.c
+++ b/arch/x86/kvm/vmx/main.c
@@ -721,6 +721,14 @@ static void vt_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
 	vmx_get_exit_info(vcpu, reason, info1, info2, intr_info, error_code);
 }
 
+static bool vt_get_pf_exit_info(struct kvm_vcpu *vcpu, u64 *fault_address)
+{
+	if (is_td_vcpu(vcpu))
+		return false;
+
+	return vmx_get_pf_exit_info(vcpu, fault_address);
+}
+
 static void vt_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
 {
 	if (is_td_vcpu(vcpu))
@@ -914,11 +922,8 @@ static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu)
 
 static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info)
 {
-	/* if exit due to PF check for async PF */
-	if (is_page_fault(intr_info))
-		vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags();
 	/* if exit due to NM, handle before interrupts are enabled */
-	else if (is_nm_fault(intr_info))
+	if (is_nm_fault(intr_info))
 		handle_nm_fault_irqoff(vcpu);
 	/* Handle machine checks before interrupts are enabled */
 	else if (is_machine_check(intr_info))
@@ -1100,6 +1105,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = {
 
 	.get_exit_info = vt_op(get_exit_info),
 	.get_entry_info = vt_op(get_entry_info),
+	.get_pf_exit_info = vt_op(get_pf_exit_info),
 
 	.vcpu_after_set_cpuid = vt_op(vcpu_after_set_cpuid),
 
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index f0d75f5a968c..087c12e11d74 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -6500,8 +6500,7 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu,
 		if (is_nmi(intr_info))
 			return true;
 		else if (is_page_fault(intr_info))
-			return vcpu->arch.apf.host_apf_flags ||
-			       vmx_need_pf_intercept(vcpu);
+			return vmx_need_pf_intercept(vcpu);
 		else if (is_debug(intr_info) &&
 			 vcpu->guest_debug &
 			 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index b341af4c6057..464d6ed862d5 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -5396,9 +5396,6 @@ static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code)
 {
 	unsigned long cr2 = vt_get_exit_qual(vcpu);
 
-	if (vcpu->arch.apf.host_apf_flags)
-		goto handle_pf;
-
 	/* When using EPT, KVM intercepts #PF only to detect illegal GPAs. */
 	WARN_ON_ONCE(enable_ept && !allow_smaller_maxphyaddr);
 
@@ -5429,7 +5426,6 @@ static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code)
 		return 1;
 	}
 
-handle_pf:
 	return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0);
 }
 
@@ -6402,6 +6398,15 @@ void vmx_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *error_code)
 		*error_code = 0;
 }
 
+bool vmx_get_pf_exit_info(struct kvm_vcpu *vcpu, u64 *fault_address)
+{
+	if (!is_page_fault(vt_get_intr_info(vcpu)))
+		return false;
+
+	*fault_address = vt_get_exit_qual(vcpu);
+	return true;
+}
+
 static void vmx_destroy_pml_buffer(struct kvm_vcpu *vcpu)
 {
 	if (vcpu->arch.pml_page) {
diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h
index cd72684d075d..30a712266c4e 100644
--- a/arch/x86/kvm/vmx/x86_ops.h
+++ b/arch/x86/kvm/vmx/x86_ops.h
@@ -113,6 +113,7 @@ static inline bool vmx_tdp_has_smep(struct kvm *kvm)
 void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
 		       u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code);
 void vmx_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *error_code);
+bool vmx_get_pf_exit_info(struct kvm_vcpu *vcpu, u64 *fault_address);
 
 u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu);
 u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 26e7beaff8a0..0956bf8fbe65 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -8118,6 +8118,31 @@ static void kvm_update_cpu_dirty_logging(struct kvm_vcpu *vcpu)
 	kvm_x86_call(update_cpu_dirty_logging)(vcpu, enable);
 }
 
+static bool kvm_handle_pv_async_pf(struct kvm_vcpu *vcpu)
+{
+#ifdef CONFIG_KVM_GUEST
+	u32 host_apf_flags;
+	u64 fault_address;
+
+	if (!kvm_x86_call(get_pf_exit_info)(vcpu, &fault_address))
+		return false;
+
+	host_apf_flags = kvm_read_and_reset_apf_flags();
+	if (!host_apf_flags ||
+	    WARN_ON_ONCE(!(host_apf_flags & KVM_PV_REASON_PAGE_NOT_PRESENT)))
+		return false;
+
+	guest_timing_exit_irqoff();
+
+	preempt_enable();
+	kvm_async_pf_task_wait_schedule(fault_address);
+	local_irq_enable();
+	return true;
+#else
+	return false;
+#endif
+}
+
 /*
  * Called within kvm->srcu read side.
  * Returns 1 to let vcpu_run() continue the guest execution loop without
@@ -8521,29 +8546,40 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 		this_cpu_write(x86_ibpb_exit_to_user, true);
 
 	/*
-	 * Consume any pending interrupts, including the possible source of
-	 * VM-Exit on SVM and any ticks that occur between VM-Exit and now.
-	 * An instruction is required after local_irq_enable() to fully unblock
-	 * interrupts on processors that implement an interrupt shadow, the
-	 * stat.exits increment will do nicely.
+	 * Handle PV async #PFs, before enabling IRQs, to ensure the page ready
+	 * notification arrives after the async #PF is queued.  Note, the async
+	 * #PF handler is responsible for enabling preemption and IRQs, and for
+	 * exiting guest timing if the #PF is an async #PF (because handling PV
+	 * async #PFs enables IRQs and may schedule out this task).
 	 */
-	kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
-	local_irq_enable();
-	++vcpu->stat.exits;
-	local_irq_disable();
-	kvm_after_interrupt(vcpu);
+	if (exit_fastpath == EXIT_FASTPATH_NONE && kvm_handle_pv_async_pf(vcpu)) {
+		exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED;
+	} else {
+		/*
+		* Consume any pending interrupts, including the possible source
+		* of VM-Exit on SVM, and any ticks that occur between VM-Exit
+		* and now.  An instruction is required after local_irq_enable()
+		* to fully unblock interrupts on processors that implement an
+		* interrupt shadow, the stat.exits increment will do nicely.
+		*/
+		kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
+		local_irq_enable();
+		++vcpu->stat.exits;
+		local_irq_disable();
+		kvm_after_interrupt(vcpu);
 
-	/*
-	 * Wait until after servicing IRQs to account guest time so that any
-	 * ticks that occurred while running the guest are properly accounted
-	 * to the guest.  Waiting until IRQs are enabled degrades the accuracy
-	 * of accounting via context tracking, but the loss of accuracy is
-	 * acceptable for all known use cases.
-	 */
-	guest_timing_exit_irqoff();
+		/*
+		 * Wait until after servicing IRQs to account guest time so that
+		 * any ticks that occurred while running the guest are properly
+		 * accounted to the guest.  Waiting until IRQs are enabled
+		 * degrades the accuracy of accounting via context tracking, but
+		 * the loss of accuracy is acceptable for all known use cases.
+		 */
+		guest_timing_exit_irqoff();
 
-	local_irq_enable();
-	preempt_enable();
+		local_irq_enable();
+		preempt_enable();
+	}
 
 	kvm_vcpu_srcu_read_lock(vcpu);
 

base-commit: d21eef2084a9af87ac2a783bc7c2b175ba114e4e
--

Re: [PATCH 1/2] KVM: register asynchronous page fault waits with interrupts disabled

[Paolo Bonzini]

Il mar 3 nov 2026, 15:43 Sean Christopherson <seanjc@google.com> ha scritto:
>
> On Tue, Nov 03, 2026, Paolo Bonzini wrote:
> > KVM cannot use a node on the stack like kvm_async_pf_task_wait_schedule() does
>
> Oh!  Yes, KVM can!  If we fully handle the PV async #PF in vcpu_enter_guest(),
> then the node can be put on the stack, right?  Juggling preemption and guest
> timing is heinous, but on the plus side it consolidates the PV async #PF code,
> so I think it's a net positive?  Especially when factoring in the node stuff.
>
> Fully handle PV async #PFs (when running as a KVM guest) before enabling
> IRQs so that it's impossible for the current task, i.e. the vCPU task, to
> be preempted before invoking kvm_async_pf_task_wait_schedule().  This will
> allow the guest-side async #PF code to eliminate its dummy allocation,
> which is needed only to handle the case where the page ready notification
> is delivered before handling the not-present #PF.
>
> Add dedicated handling for PV async #PFs instead of wrapping/piggybacking
> kvm_x86_ops.handle_exit_irqoff(), as the act of handling an async #PF may
> enable IRQs and schedule out the current task, i.e. needs to be done just
> before KVM itself explicitly enables IRQs, and needs to enable preemption
> before calling into the kernel's PV async #PF handler.
>
> Opportunistically wrap the code with CONFIG_KVM_GUEST to help document the
> context, and drop the CONFIG_KVM_GUEST=n stubs.
>
> Suggested-by: Paolo Bonzini <pbonzini@redhat.com>

More like Instigated-by. :)

The main advantage would be that it bypasses the missing lock issue
that Sashiko reported, but I don't know... That;s surely nice, but at
the same time it is not great having to add a new callback, and to
duplicate the guest_time_exit/preempt_enable code into
kvm_handle_pv_async_pf().

Not that the extra field in vcpu->arch is beautiful, I admit that, but
it seems extra unwarranted complexity.

Unless you insist I will send a v2 that takes the lock properly.

Paolo

Re: [PATCH 1/2] KVM: register asynchronous page fault waits with interrupts disabled

[Sean Christopherson]

On Tue, Nov 03, 2026, Paolo Bonzini wrote:
> Il mar 3 nov 2026, 15:43 Sean Christopherson <seanjc@google.com> ha scritto:
> >
> > On Tue, Nov 03, 2026, Paolo Bonzini wrote:
> > > KVM cannot use a node on the stack like kvm_async_pf_task_wait_schedule() does
> >
> > Oh!  Yes, KVM can!  If we fully handle the PV async #PF in vcpu_enter_guest(),
> > then the node can be put on the stack, right?  Juggling preemption and guest
> > timing is heinous, but on the plus side it consolidates the PV async #PF code,
> > so I think it's a net positive?  Especially when factoring in the node stuff.
> >
> > Fully handle PV async #PFs (when running as a KVM guest) before enabling
> > IRQs so that it's impossible for the current task, i.e. the vCPU task, to
> > be preempted before invoking kvm_async_pf_task_wait_schedule().  This will
> > allow the guest-side async #PF code to eliminate its dummy allocation,
> > which is needed only to handle the case where the page ready notification
> > is delivered before handling the not-present #PF.
> >
> > Add dedicated handling for PV async #PFs instead of wrapping/piggybacking
> > kvm_x86_ops.handle_exit_irqoff(), as the act of handling an async #PF may
> > enable IRQs and schedule out the current task, i.e. needs to be done just
> > before KVM itself explicitly enables IRQs, and needs to enable preemption
> > before calling into the kernel's PV async #PF handler.
> >
> > Opportunistically wrap the code with CONFIG_KVM_GUEST to help document the
> > context, and drop the CONFIG_KVM_GUEST=n stubs.
> >
> > Suggested-by: Paolo Bonzini <pbonzini@redhat.com>
> 
> More like Instigated-by. :)
> 
> The main advantage would be that it bypasses the missing lock issue
> that Sashiko reported, but I don't know... That;s surely nice, but at
> the same time it is not great having to add a new callback, 

Agreed, though IMO it's better than adding another vCPU field.  E.g. we could
just as easily add apf.host_cr2 instead of host_apf_node.

> duplicate the guest_time_exit/preempt_enable code into kvm_handle_pv_async_pf().

Yeah, I was all excited about the idea, and that excitement definitely fizzled
when I realized preemption needed to be enabled.

> Not that the extra field in vcpu->arch is beautiful, I admit that, but
> it seems extra unwarranted complexity.
> 
> Unless you insist I will send a v2 that takes the lock properly.

I don't insist, I agree the guest_time_exit() and preempt_enable() code is too
ugly.  And the part that I actually care about cleaning up is not having a bunch
of apf.host_apf_flags checks sprinkled all over, especially in the nested code.
Which definitely doesn't require fully handling async #PF with IRQs disabled, we
just need to handle them before calling into vendor code (I think).

E.g. something like this?  Though to land it on top of your patch to add
host_apf_node, we'd want to define pass a struct, but that should only make the
code prettier.

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 0378577d9d8b..09416d3588d0 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -959,7 +959,6 @@ struct kvm_vcpu_arch {
 		u64 msr_int_val; /* MSR_KVM_ASYNC_PF_INT */
 		u16 vec;
 		u32 id;
-		u32 host_apf_flags;
 		bool pageready_pending;
 	} apf;
 
@@ -1688,7 +1687,8 @@ struct kvm_x86_ops {
 			       struct x86_instruction_info *info,
 			       enum x86_intercept_stage stage,
 			       struct x86_exception *exception);
-	void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu);
+	void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu, u32 *async_pf_flags,
+				   unsigned long *async_pf_cr2);
 
 	void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu, bool enable);
 
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index f4b12d6aef55..3cc4bf506cb1 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -4954,9 +4954,6 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu,
 int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
 				u64 fault_address, char *insn, int insn_len)
 {
-	int r = 1;
-	u32 flags = vcpu->arch.apf.host_apf_flags;
-
 #ifndef CONFIG_X86_64
 	/* A 64-bit CR2 should be impossible on 32-bit KVM. */
 	if (KVM_BUG_ON(fault_address >> 32, vcpu->kvm))
@@ -4977,21 +4974,9 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
 	BUILD_BUG_ON(lower_32_bits(PFERR_SYNTHETIC_MASK));
 
 	kvm_request_l1tf_flush_l1d();
-	if (!flags) {
-		trace_kvm_page_fault(vcpu, fault_address, error_code);
+	trace_kvm_page_fault(vcpu, fault_address, error_code);
 
-		r = kvm_mmu_page_fault(vcpu, fault_address, error_code, insn,
-				insn_len);
-	} else if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) {
-		vcpu->arch.apf.host_apf_flags = 0;
-		local_irq_disable();
-		kvm_async_pf_task_wait_schedule(fault_address);
-		local_irq_enable();
-	} else {
-		WARN_ONCE(1, "Unexpected host async PF flags: %x\n", flags);
-	}
-
-	return r;
+	return kvm_mmu_page_fault(vcpu, fault_address, error_code, insn, insn_len);
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_handle_page_fault);
 
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index 07554089ec05..d4951dc8b1f7 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -1818,10 +1818,6 @@ int nested_svm_exit_special(struct vcpu_svm *svm)
 		if (svm->vmcb01.ptr->control.intercepts[INTERCEPT_EXCEPTION] &
 		    excp_bits)
 			return NESTED_EXIT_HOST;
-		else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR &&
-			 svm->vcpu.arch.apf.host_apf_flags)
-			/* Trap async PF even if not shadowing */
-			return NESTED_EXIT_HOST;
 		break;
 	}
 	case SVM_EXIT_VMMCALL:
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index 581bc3a23c95..5a92a22e8e6a 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -3811,7 +3811,6 @@ static void svm_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info,
 		*error_code = control->event_inj_err;
 	else
 		*error_code = 0;
-
 }
 
 static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
@@ -4758,11 +4757,6 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 			       vcpu_to_pmu(vcpu)->global_ctrl);
 	}
 
-	/* if exit due to PF check for async PF */
-	if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
-		vcpu->arch.apf.host_apf_flags =
-			kvm_read_and_reset_apf_flags();
-
 	kvm_clear_available_registers(vcpu, SVM_REGS_LAZY_LOAD_SET);
 
 	trace_kvm_exit(vcpu, KVM_ISA_SVM);
@@ -5088,9 +5082,15 @@ static int svm_check_intercept(struct kvm_vcpu *vcpu,
 	return ret;
 }
 
-static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu)
+static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu, u32 *async_pf_flags,
+				   unsigned long *async_pf_cr2)
 {
 	switch (to_svm(vcpu)->vmcb->control.exit_code) {
+	case SVM_EXIT_EXCP_BASE + PF_VECTOR:
+		*async_pf_flags = kvm_read_and_reset_apf_flags();
+		if (*async_pf_flags)
+			*async_pf_cr2 = to_svm(vcpu)->vmcb->control.exit_info_2;
+		break;
 	case SVM_EXIT_EXCP_BASE + MC_VECTOR:
 		svm_handle_mce(vcpu);
 		break;
diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c
index d75170a6199e..05e1d600dbd8 100644
--- a/arch/x86/kvm/vmx/main.c
+++ b/arch/x86/kvm/vmx/main.c
@@ -912,17 +912,21 @@ static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu)
 		rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);
 }
 
-static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info)
+static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info,
+				    u32 *async_pf_flags,
+				    unsigned long *async_pf_cr2)
 {
-	/* if exit due to PF check for async PF */
-	if (is_page_fault(intr_info))
-		vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags();
 	/* if exit due to NM, handle before interrupts are enabled */
-	else if (is_nm_fault(intr_info))
+	if (is_page_fault(intr_info)) {
+		*async_pf_flags = kvm_read_and_reset_apf_flags();
+		if (*async_pf_flags)
+			*async_pf_cr2 = vt_get_exit_qual(vcpu);
+	} else if (is_nm_fault(intr_info)) {
 		handle_nm_fault_irqoff(vcpu);
 	/* Handle machine checks before interrupts are enabled */
-	else if (is_machine_check(intr_info))
+	} else if (is_machine_check(intr_info)) {
 		kvm_machine_check();
+	}
 }
 
 static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu,
@@ -941,7 +945,8 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu,
 	vcpu->arch.at_instruction_boundary = true;
 }
 
-static void vt_handle_exit_irqoff(struct kvm_vcpu *vcpu)
+static void vt_handle_exit_irqoff(struct kvm_vcpu *vcpu, u32 *async_pf_flags,
+				  unsigned long *async_pf_cr2)
 {
 	if (to_vt(vcpu)->emulation_required)
 		return;
@@ -951,7 +956,8 @@ static void vt_handle_exit_irqoff(struct kvm_vcpu *vcpu)
 		handle_external_interrupt_irqoff(vcpu, vt_get_intr_info(vcpu));
 		break;
 	case EXIT_REASON_EXCEPTION_NMI:
-		handle_exception_irqoff(vcpu, vt_get_intr_info(vcpu));
+		handle_exception_irqoff(vcpu, vt_get_intr_info(vcpu),
+					async_pf_flags, async_pf_cr2);
 		break;
 	case EXIT_REASON_MCE_DURING_VMENTRY:
 		kvm_machine_check();
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index f0d75f5a968c..087c12e11d74 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -6500,8 +6500,7 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu,
 		if (is_nmi(intr_info))
 			return true;
 		else if (is_page_fault(intr_info))
-			return vcpu->arch.apf.host_apf_flags ||
-			       vmx_need_pf_intercept(vcpu);
+			return vmx_need_pf_intercept(vcpu);
 		else if (is_debug(intr_info) &&
 			 vcpu->guest_debug &
 			 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index b341af4c6057..15f54474382b 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -5396,9 +5396,6 @@ static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code)
 {
 	unsigned long cr2 = vt_get_exit_qual(vcpu);
 
-	if (vcpu->arch.apf.host_apf_flags)
-		goto handle_pf;
-
 	/* When using EPT, KVM intercepts #PF only to detect illegal GPAs. */
 	WARN_ON_ONCE(enable_ept && !allow_smaller_maxphyaddr);
 
@@ -5429,7 +5426,6 @@ static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code)
 		return 1;
 	}
 
-handle_pf:
 	return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0);
 }
 
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 26e7beaff8a0..57aff240cdb6 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -8118,6 +8118,19 @@ static void kvm_update_cpu_dirty_logging(struct kvm_vcpu *vcpu)
 	kvm_x86_call(update_cpu_dirty_logging)(vcpu, enable);
 }
 
+static void kvm_handle_pv_async_pf(u32 flags, unsigned long cr2)
+{
+	if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_KVM_GUEST)))
+		return;
+
+	if (WARN_ON_ONCE(!(flags & KVM_PV_REASON_PAGE_NOT_PRESENT)))
+		return;
+
+	local_irq_disable();
+	kvm_async_pf_task_wait_schedule(cr2);
+	local_irq_enable();
+}
+
 /*
  * Called within kvm->srcu read side.
  * Returns 1 to let vcpu_run() continue the guest execution loop without
@@ -8130,9 +8143,10 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 	bool req_int_win =
 		dm_request_for_irq_injection(vcpu) &&
 		kvm_cpu_accept_dm_intr(vcpu);
+	unsigned long async_pf_cr2;
 	fastpath_t exit_fastpath;
 	u64 run_flags, debug_ctl;
-
+	u32 async_pf_flags = 0;
 	bool req_immediate_exit = false;
 
 	if (kvm_request_pending(vcpu)) {
@@ -8506,7 +8520,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 	if (vcpu->arch.xfd_no_write_intercept)
 		fpu_sync_guest_vmexit_xfd_state();
 
-	kvm_x86_call(handle_exit_irqoff)(vcpu);
+	kvm_x86_call(handle_exit_irqoff)(vcpu, &async_pf_flags, &async_pf_cr2);
 
 	if (vcpu->arch.guest_fpu.xfd_err)
 		wrmsrq(MSR_IA32_XFD_ERR, 0);
@@ -8568,6 +8582,11 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 	if (vcpu->arch.apic_attention)
 		kvm_lapic_sync_from_vapic(vcpu);
 
+	if (async_pf_flags && !WARN_ON_ONCE(exit_fastpath != EXIT_FASTPATH_NONE)) {
+		kvm_handle_pv_async_pf(async_pf_flags, async_pf_cr2);
+		exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED;
+	}
+
 	r = kvm_x86_call(handle_exit)(vcpu, exit_fastpath);
 	return r;