[RFC PATCH 2/5] KVM: nSVM: Add nested PML emulation

[Nikunj A Dadhania <nikunj@amd.com>]

Emulate Page Modification Logging (PML) for nested SVM guests, allowing
L1 hypervisors to use PML to track dirty pages in their L2 guests.
KVM emulates PML in software by write-protecting L2 SPTEs and logging
dirty GPAs to L1's PML buffer via the write_log_dirty hook.

PML_FULL exits are prioritized over NPF exits to ensure L1 processes
the PML buffer before continuing L2 execution.

Clear any pending PML Full on nested VMRUN.  The flag is derived from L1's
log index, and a fault that KVM discards instead of injecting can leave it
set after L1 has drained its buffer.  No event is lost: if the buffer is
genuinely full, the next dirty write raises PML Full again.

Assisted-by: LLM
Signed-off-by: Nikunj A Dadhania <nikunj@amd.com>
---
 arch/x86/kvm/mmu/paging_tmpl.h |  2 +-
 arch/x86/kvm/svm/nested.c      | 75 +++++++++++++++++++++++++++++++++-
 arch/x86/kvm/svm/svm.h         |  8 ++++
 3 files changed, 83 insertions(+), 2 deletions(-)

diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index 043d0ec02b27..c9ec05620e44 100644
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -234,7 +234,7 @@ static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu,
 		if (level == walker->level && write_fault &&
 				!(pte & PT_GUEST_DIRTY_MASK)) {
 			trace_kvm_mmu_set_dirty_bit(table_gfn, index, sizeof(pte));
-#if PTTYPE == PTTYPE_EPT
+#if (PTTYPE == PTTYPE_EPT) || (PTTYPE == 64)
 			/* Only the nested GPA walk yields an L2 GPA to log. */
 			if (w == &vcpu->arch.ngpa_walk &&
 			    kvm_nested_call(write_log_dirty)(vcpu, addr))
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index af0ef678433d..8251b38d1479 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -17,6 +17,7 @@
 #include <linux/kvm_types.h>
 #include <linux/kvm_host.h>
 #include <linux/kernel.h>
+#include <linux/mm.h>
 
 #include <asm/msr-index.h>
 #include <asm/debugreg.h>
@@ -63,7 +64,14 @@ static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
 	if (WARN_ON_ONCE(hweight64(fault_stage) != 1))
 		fault_stage = PFERR_GUEST_FINAL_MASK;
 
-	vmcb->control.exit_code = SVM_EXIT_NPF;
+	/*
+	 * Only the exit code differs for a PML Full; hardware leaves the
+	 * EXITINFO fields as the nested page fault that filled the buffer
+	 * would set them.
+	 */
+	vmcb->control.exit_code = svm->nested.pml_full ? SVM_EXIT_PML_FULL : SVM_EXIT_NPF;
+	svm->nested.pml_full = false;
+
 	vmcb->control.exit_info_1 = fault_stage |
 				    (fault->error_code & ~PFERR_GUEST_FAULT_STAGE_MASK);
 	vmcb->control.exit_info_2 = fault->address;
@@ -416,6 +424,16 @@ static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu,
 	       !kvm_vcpu_is_legal_gpa(vcpu, control->nested_cr3)))
 		return false;
 
+	if (control->misc_ctl & SVM_MISC_ENABLE_PML) {
+		if (CC(!(control->misc_ctl & SVM_MISC_ENABLE_NP)))
+			return false;
+		if (CC(!IS_ALIGNED(control->pml_addr, PAGE_SIZE)))
+			return false;
+		if (CC(!nested_svm_check_bitmap_pa(vcpu, control->pml_addr,
+						   PAGE_SIZE)))
+			return false;
+	}
+
 	if (CC(!nested_svm_check_bitmap_pa(vcpu, control->msrpm_base_pa,
 					   MSRPM_SIZE)))
 		return false;
@@ -556,6 +574,8 @@ void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu,
 	to->nested_cr3          = from->nested_cr3;
 	to->pause_filter_count  = from->pause_filter_count;
 	to->pause_filter_thresh = from->pause_filter_thresh;
+	to->pml_addr            = from->pml_addr;
+	to->pml_index           = from->pml_index;
 
 	/* Copy asid here because nested_vmcb_check_controls() will check it */
 	to->asid           = from->asid;
@@ -909,6 +929,13 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm)
 	if (pml)
 		vmcb02->control.pml_index = -1;
 
+	/*
+	 * Clear any pending PML Full, as L1's log index has just been
+	 * reloaded from vmcb12.  A fault that KVM discards instead of
+	 * injecting can leave the flag set after L1 drained its buffer.
+	 */
+	svm->nested.pml_full = false;
+
 	/*
 	 * Stash vmcb02's counter if the guest hasn't moved past the guilty
 	 * instruction; otherwise, reset the counter to '0'.
@@ -1311,6 +1338,8 @@ static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu)
 	vmcb12->control.event_inj	  = 0;
 	vmcb12->control.event_inj_err	  = 0;
 	vmcb12->control.int_ctl           = svm->nested.ctl.int_ctl;
+	vmcb12->control.pml_addr          = svm->nested.ctl.pml_addr;
+	vmcb12->control.pml_index         = svm->nested.ctl.pml_index;
 
 	trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code,
 				       vmcb12->control.exit_info_1,
@@ -1886,6 +1915,8 @@ static void nested_copy_vmcb_cache_to_control(struct vmcb_control_area *dst,
 	dst->misc_ctl2		  = from->misc_ctl2;
 	dst->pause_filter_count   = from->pause_filter_count;
 	dst->pause_filter_thresh  = from->pause_filter_thresh;
+	dst->pml_addr             = from->pml_addr;
+	dst->pml_index            = from->pml_index;
 	/* 'clean' and 'hv_enlightenments' are not changed by KVM */
 }
 
@@ -2169,6 +2200,47 @@ static gpa_t svm_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa,
 	return w->gva_to_gpa(vcpu, w, gpa, access, exception);
 }
 
+static int nested_svm_write_pml_buffer(struct kvm_vcpu *vcpu, gpa_t gpa)
+{
+	struct vcpu_svm *svm = to_svm(vcpu);
+	u16 index;
+	gpa_t dst;
+
+	if (WARN_ON_ONCE(!is_guest_mode(vcpu)) || !nested_pml_enabled(svm))
+		return 0;
+
+	/* The flag can be set here if the fault that raised it was discarded. */
+	if (svm->nested.pml_full)
+		return 1;
+
+	index = svm->nested.ctl.pml_index;
+	if (index >= PML_LOG_NR_ENTRIES) {
+		svm->nested.pml_full = true;
+		return 1;
+	}
+
+	gpa &= ~0xFFFULL;
+	dst = svm->nested.ctl.pml_addr + sizeof(u64) * index;
+
+	/*
+	 * Drop the entry if the write fails.  VMRUN only verifies that L1's
+	 * buffer address is a legal GPA, not that it is backed, so a failure
+	 * here means L1 has pointed PML at memory it does not have, and only
+	 * L1's own dirty tracking suffers.
+	 */
+	if (kvm_write_guest_page(vcpu->kvm, gpa_to_gfn(dst), &gpa,
+				 offset_in_page(dst), sizeof(gpa)))
+		return 0;
+
+	/*
+	 * Decrement PML_INDEX after logging the GPA (buffer fills backwards
+	 * from index 511 to 0)
+	 */
+	svm->nested.ctl.pml_index = index - 1;
+
+	return 0;
+}
+
 struct kvm_x86_nested_ops svm_nested_ops __initdata = {
 	.leave_nested = svm_leave_nested,
 	.translate_nested_gpa = svm_translate_nested_gpa,
@@ -2179,4 +2251,5 @@ struct kvm_x86_nested_ops svm_nested_ops __initdata = {
 	.get_state = svm_get_nested_state,
 	.set_state = svm_set_nested_state,
 	.hv_inject_synthetic_vmexit_post_tlb_flush = svm_hv_inject_synthetic_vmexit_post_tlb_flush,
+	.write_log_dirty = nested_svm_write_pml_buffer,
 };
diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h
index 6d0242966dc8..6a0359eb0c7c 100644
--- a/arch/x86/kvm/svm/svm.h
+++ b/arch/x86/kvm/svm/svm.h
@@ -196,6 +196,8 @@ struct vmcb_ctrl_area_cached {
 	u64 exit_info_2;
 	u32 exit_int_info;
 	u32 exit_int_info_err;
+	u64 pml_addr;
+	u16 pml_index;
 	u64 misc_ctl;
 	u32 event_inj;
 	u32 event_inj_err;
@@ -235,6 +237,7 @@ struct svm_nested_state {
 	struct vmcb_save_area_cached save;
 
 	bool initialized;
+	bool pml_full;
 
 	/*
 	 * Indicates whether MSR bitmap for L2 needs to be rebuilt due to
@@ -657,6 +660,11 @@ static inline bool nested_npt_enabled(struct vcpu_svm *svm)
 	return svm->nested.ctl.misc_ctl & SVM_MISC_ENABLE_NP;
 }
 
+static inline bool nested_pml_enabled(struct vcpu_svm *svm)
+{
+	return svm->nested.ctl.misc_ctl & SVM_MISC_ENABLE_PML;
+}
+
 static inline bool l2_has_separate_pat(struct kvm_vcpu *vcpu)
 {
 	/*
-- 
2.48.1