[PATCH 4/5] KVM: Hold MMU lock exclusively when calling kvm_arch_flush_shadow_all()

James Houghton
2026-05-04T22:42:49Z

All architectures that non-trivially implement this function grab the
KVM MMU lock exclusively to prevent things like double-freeing of page
table entries and caches. Do so generically to somewhat simplify the
architecture-specific logic.

Without this change, it is possible for kvm_arch_flush_shadow_all() to
be called on the same `kvm` at the same time: if the `kvm`'s mm is
destroyed (exit_mm()) at the same time to the final reference to the
`kvm` is dropped.

Signed-off-by: James Houghton <jthoughton@google.com>
---
arch/arm64/kvm/nested.c | 2 +-
arch/loongarch/kvm/mmu.c | 4 +++-
arch/mips/kvm/mips.c | 2 +-
arch/riscv/kvm/mmu.c | 4 ++--
arch/riscv/kvm/vm.c | 2 ++
arch/x86/kvm/mmu/mmu.c | 4 +---
virt/kvm/kvm_main.c | 3 +++
7 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c
index 977598bff5e6..ba2e6c98bd45 100644
--- a/arch/arm64/kvm/nested.c
+++ b/arch/arm64/kvm/nested.c
@@ -1190,7 +1190,7 @@ void kvm_arch_flush_shadow_all(struct kvm *kvm)
{
int i;

- guard(write_lock)(&kvm->mmu_lock);
+ lockdep_assert_held_write(&kvm->mmu_lock);

for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
diff --git a/arch/loongarch/kvm/mmu.c b/arch/loongarch/kvm/mmu.c
index 5dbce9b18e1c..120001da26e4 100644
--- a/arch/loongarch/kvm/mmu.c
+++ b/arch/loongarch/kvm/mmu.c
@@ -486,7 +486,9 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,

void kvm_arch_flush_shadow_all(struct kvm *kvm)
{
- kvm_flush_range(kvm, 0, kvm->arch.gpa_size >> PAGE_SHIFT, 1);
+ lockdep_assert_held(&kvm->mmu_lock);
+
+ kvm_flush_range(kvm, 0, kvm->arch.gpa_size >> PAGE_SHIFT, 0);
}

void kvm_arch_flush_shadow_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c
index 463b6c4aa62c..4ad9e21a3321 100644
--- a/arch/mips/kvm/mips.c
+++ b/arch/mips/kvm/mips.c
@@ -180,7 +180,7 @@ long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl,

void kvm_arch_flush_shadow_all(struct kvm *kvm)
{
- guard(spinlock)(&kvm->mmu_lock);
+ lockdep_assert_held(&kvm->mmu_lock);

/* Flush whole GPA */
kvm_mips_flush_gpa_pt(kvm, 0, ~0);
diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c
index 2d3def024270..c1b9333f17eb 100644
--- a/arch/riscv/kvm/mmu.c
+++ b/arch/riscv/kvm/mmu.c
@@ -558,7 +558,8 @@ void kvm_riscv_mmu_free_pgd(struct kvm *kvm)
struct kvm_gstage gstage;
void *pgd = NULL;

- spin_lock(&kvm->mmu_lock);
+ lockdep_assert_held(&kvm->mmu_lock);
+
if (kvm->arch.pgd) {
kvm_riscv_gstage_init(&gstage, kvm);
kvm_riscv_gstage_unmap_range(&gstage, 0UL,
@@ -568,7 +569,6 @@ void kvm_riscv_mmu_free_pgd(struct kvm *kvm)
kvm->arch.pgd_phys = 0;
kvm->arch.pgd_levels = 0;
}
- spin_unlock(&kvm->mmu_lock);

if (pgd)
free_pages((unsigned long)pgd, get_order(kvm_riscv_gstage_pgd_size));
diff --git a/arch/riscv/kvm/vm.c b/arch/riscv/kvm/vm.c
index a9f083feeb76..f704a64bfc48 100644
--- a/arch/riscv/kvm/vm.c
+++ b/arch/riscv/kvm/vm.c
@@ -38,7 +38,9 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)

r = kvm_riscv_gstage_vmid_init(kvm);
if (r) {
+ spin_lock(&kvm->mmu_lock);
kvm_riscv_mmu_free_pgd(kvm);
+ spin_unlock(&kvm->mmu_lock);
return r;
}

diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 892246204435..6e6f94046b3f 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -7352,7 +7352,7 @@ static void kvm_mmu_zap_all(struct kvm *kvm)
LIST_HEAD(invalid_list);
int ign;

- write_lock(&kvm->mmu_lock);
+ lockdep_assert_held_write(&kvm->mmu_lock);
restart:
list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) {
if (WARN_ON_ONCE(sp->role.invalid))
@@ -7367,8 +7367,6 @@ static void kvm_mmu_zap_all(struct kvm *kvm)

if (tdp_mmu_enabled)
kvm_tdp_mmu_zap_all(kvm);
-
- write_unlock(&kvm->mmu_lock);
}

void kvm_arch_flush_shadow_all(struct kvm *kvm)
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 89489996fbc1..f5affd3bfda8 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -340,7 +340,10 @@ void kvm_flush_remote_tlbs_memslot(struct kvm *kvm,

static void kvm_flush_shadow_all(struct kvm *kvm)
{
+ KVM_MMU_LOCK(kvm);
kvm_arch_flush_shadow_all(kvm);
+ KVM_MMU_UNLOCK(kvm);
+
kvm_arch_guest_memory_reclaimed(kvm);
}

--
2.54.0.545.g6539524ca2-goog