[PATCH v3 26/27] KVM: nSVM: Use different ASIDs for L1 and L2

[Yosry Ahmed <yosry@kernel.org>]

Now that TLB flushes are properly handled and tracked for L1 vs L2
ASIDs, allocate a separate new ASID for L2 for each vCPU, similar to how
VMX handles VPIDs. Drop the unconditional flushes and syncs on nested
transitions.

L1 and L2 can now only share an ASID in two cases:
- KVM runs out of ASIDs and uses the fallback ASID for both L1 and L2.
- SEV VMs always use the same ASID for SEV-specific requirements.

WARN if the ASID is shared otherwise.

Initialize last_asid to 0, such that the L2 ASID is always flushed on
first nested VM-Enter after it's allocated.

On SEV VM migration, free the L2 ASID on the destination vCPU (if nested
was already initialized) before overwriting it with the SEV ASID.

On a Turin CPU, this results in 8-15% performance boost in CPUID rate
microbenchmark [1] and netperf TCP_RR latency/throughput.

[1] https://lore.kernel.org/kvm/20231109180646.2963718-1-khorenko@virtuozzo.com/

Signed-off-by: Yosry Ahmed <yosry@kernel.org>
---
 arch/x86/kvm/svm/nested.c | 32 +++++++++++++++++++++++---------
 arch/x86/kvm/svm/sev.c    |  6 ++++--
 2 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index 4ffc10bf9232e..6fe68f2ae282d 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -711,6 +711,10 @@ static void nested_svm_entry_tlb_flush(struct kvm_vcpu *vcpu)
 	 * context is not tagged by the ASID, so the shadow NPTs cannot be
 	 * reused across different L2 ASIDs.
 	 *
+	 * Note, last_asid is initialized as 0, so the first nested VM-Enter
+	 * after setting EFER.SVME will always flush the TLB to avoid using
+	 * stale entries.
+	 *
 	 * If L1 requested a full TLB flush for all ASIDs (including its own),
 	 * L1's own ASID is also flushed on nested VM-Exit, before running L1.
 	 *
@@ -727,12 +731,10 @@ static void nested_svm_entry_tlb_flush(struct kvm_vcpu *vcpu)
 	 * If L1 and L2 share the same ASID in hardware (when using the fallback
 	 * ASID for both, or for SEV guests), flush it on nested transitions.
 	 */
-	if (svm->asid == svm->nested.asid02)
+	if (svm->asid == svm->nested.asid02) {
+		WARN_ON_ONCE(svm->asid != fallback_asid && !is_sev_guest(vcpu));
 		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
-
-	/* TODO: optimize unconditional TLB flush/MMU sync */
-	kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
-	kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
+	}
 }
 
 static void nested_svm_exit_tlb_flush(struct kvm_vcpu *vcpu)
@@ -747,9 +749,6 @@ static void nested_svm_exit_tlb_flush(struct kvm_vcpu *vcpu)
 
 	if (svm->asid == svm->nested.asid02)
 		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
-
-	kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
-	kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
 }
 
 /*
@@ -1566,7 +1565,20 @@ int svm_allocate_nested(struct vcpu_svm *svm)
 	svm->nested.vmcb02.ptr = page_address(vmcb02_page);
 	svm->nested.vmcb02.pa = __sme_set(page_to_pfn(vmcb02_page) << PAGE_SHIFT);
 
-	svm->nested.asid02 = svm->asid;
+	svm->nested.asid02 = allocate_asid(&svm->vcpu);
+
+	/*
+	 * KVM uses a fallback ASID when out of ASIDs, and fails vCPU creation
+	 * if there's no fallback ASID. ASID allocation must succeed here.
+	 */
+	KVM_BUG_ON(!svm->nested.asid02, svm->vcpu.kvm);
+
+	/*
+	 * Clear last_asid to ensure that the ASID is flushed on the first
+	 * nested VM-Enter. Otherwise, stale TLB entries from a previous life of
+	 * the ASID (e.g. different vCPU or even different VM) could be used.
+	 */
+	svm->nested.last_asid = 0;
 
 	return 0;
 
@@ -1583,6 +1595,8 @@ void svm_free_nested(struct vcpu_svm *svm)
 	if (WARN_ON_ONCE(svm->vmcb != svm->vmcb01.ptr))
 		svm_switch_vmcb(svm, &svm->vmcb01);
 
+	free_asid(svm->nested.asid02);
+
 	svm_vcpu_free_msrpm(svm->nested.msrpm);
 	svm->nested.msrpm = NULL;
 
diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c
index b2e6acfbb081f..646108e70e21a 100644
--- a/arch/x86/kvm/svm/sev.c
+++ b/arch/x86/kvm/svm/sev.c
@@ -2043,10 +2043,12 @@ static void sev_vcpu_migrate_asid(struct vcpu_svm *dst_svm,
 	dst_svm->asid = asid;
 
 	/*
-	 * If nested is already initialized on the destination vCPU, also update
-	 * the nested ASID to match the new SEV ASID.
+	 * If nested is already initialized on the destination vCPU, also free
+	 * the previous ASID (in case it was non-SEV) and update the nested ASID
+	 * to match the new SEV ASID.
 	 */
 	if (dst_svm->nested.vmcb02.ptr) {
+		free_asid(dst_svm->nested.asid02);
 		dst_svm->nested.asid02 = asid;
 		dst_svm->nested.vmcb02.ptr->control.asid = asid;
 	}
-- 
2.56.0.385.gd3acb90ef8-goog