* [PATCH 01/19] KVM: x86/hyperv: remove unnecessary mmu_is_nested() check
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
@ 2026-06-24 21:30 ` Paolo Bonzini
2026-06-24 21:30 ` [PATCH 02/19] KVM: x86/mmu: introduce struct kvm_pagewalk Paolo Bonzini
` (18 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:30 UTC (permalink / raw)
To: linux-kernel, kvm
Just always go through kvm_translate_gpa(), which will either invoke
the vendor check or just return hc->ingpa back.
This is a better way to fix the issue of commit 464af6fc2b1d ("KVM:
x86: check for nEPT/nNPT in slow flush hypercalls", 2026-05-03).
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/kvm/hyperv.c | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c
index fd4eb1e561f7..0b5050cccab4 100644
--- a/arch/x86/kvm/hyperv.c
+++ b/arch/x86/kvm/hyperv.c
@@ -2045,10 +2045,9 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
* flush). Translate the address here so the memory can be uniformly
* read with kvm_read_guest().
*/
- if (!hc->fast && mmu_is_nested(vcpu)) {
- hc->ingpa = kvm_x86_ops.nested_ops->translate_nested_gpa(
- vcpu, hc->ingpa,
- PFERR_GUEST_FINAL_MASK, NULL, 0);
+ if (!hc->fast) {
+ hc->ingpa = kvm_translate_gpa(vcpu, vcpu->arch.walk_mmu, hc->ingpa,
+ PFERR_GUEST_FINAL_MASK, NULL, 0);
if (unlikely(hc->ingpa == INVALID_GPA))
return HV_STATUS_INVALID_HYPERCALL_INPUT;
}
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 02/19] KVM: x86/mmu: introduce struct kvm_pagewalk
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
2026-06-24 21:30 ` [PATCH 01/19] KVM: x86/hyperv: remove unnecessary mmu_is_nested() check Paolo Bonzini
@ 2026-06-24 21:30 ` Paolo Bonzini
2026-06-24 21:30 ` [PATCH 03/19] KVM: x86/mmu: move get_guest_pgd to " Paolo Bonzini
` (17 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:30 UTC (permalink / raw)
To: linux-kernel, kvm
In preparation for separating walking and building of page tables,
introduce a dummy struct kvm_pagewalk and pass it around instead of
its containing kvm_mmu to functions that do not build the page tables.
Outermost functions retrieve the mmu via container_of, while internal
functions can pass around the struct kvm_pagewalk pointer. x86.c is
still (mostly) oblivious to the existence of struct kvm_pagewalk. There
are only a couple exceptions for now, which were done already here
for simplicity, but the plan is for the KVM code to use struct
kvm_pagewalk whenever dealing with guest page tables.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/include/asm/kvm_host.h | 7 +++++-
arch/x86/kvm/hyperv.c | 2 +-
arch/x86/kvm/mmu.h | 19 +++++++++------
arch/x86/kvm/mmu/mmu.c | 2 +-
arch/x86/kvm/mmu/paging_tmpl.h | 43 +++++++++++++++++++--------------
arch/x86/kvm/regs.c | 2 +-
arch/x86/kvm/x86.c | 2 +-
7 files changed, 46 insertions(+), 31 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index d8700eb848b4..57ea33f193d6 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -513,10 +513,15 @@ struct kvm_page_fault;
/*
* x86 supports 4 paging modes (5-level 64-bit, 4-level 64-bit, 3-level 32-bit,
- * and 2-level 32-bit). The kvm_mmu structure abstracts the details of the
+ * and 2-level 32-bit). The kvm_pagewalk structure abstracts the details of the
* current mmu mode.
*/
+struct kvm_pagewalk {
+};
+
struct kvm_mmu {
+ struct kvm_pagewalk w;
+
unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu);
u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
int (*page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c
index 0b5050cccab4..e4a0ca0f9fd4 100644
--- a/arch/x86/kvm/hyperv.c
+++ b/arch/x86/kvm/hyperv.c
@@ -2046,7 +2046,7 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
* read with kvm_read_guest().
*/
if (!hc->fast) {
- hc->ingpa = kvm_translate_gpa(vcpu, vcpu->arch.walk_mmu, hc->ingpa,
+ hc->ingpa = kvm_translate_gpa(vcpu, &vcpu->arch.walk_mmu->w, hc->ingpa,
PFERR_GUEST_FINAL_MASK, NULL, 0);
if (unlikely(hc->ingpa == INVALID_GPA))
return HV_STATUS_INVALID_HYPERCALL_INPUT;
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index 01846ef75d63..9384e626934b 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -246,21 +246,22 @@ static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu)
}
static inline void kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
- struct kvm_mmu *mmu)
+ struct kvm_pagewalk *w)
{
/*
* When EPT is enabled, KVM may passthrough CR0.WP to the guest, i.e.
- * @mmu's snapshot of CR0.WP and thus all related paging metadata may
+ * @w's snapshot of CR0.WP and thus all related paging metadata may
* be stale. Refresh CR0.WP and the metadata on-demand when checking
* for permission faults. Exempt nested MMUs, i.e. MMUs for shadowing
* nEPT and nNPT, as CR0.WP is ignored in both cases. Note, KVM does
* need to refresh nested_mmu, a.k.a. the walker used to translate L2
* GVAs to GPAs, as that "MMU" needs to honor L2's CR0.WP.
*/
- if (!tdp_enabled || mmu == &vcpu->arch.guest_mmu)
+ if (!tdp_enabled || w == &vcpu->arch.guest_mmu.w)
return;
- __kvm_mmu_refresh_passthrough_bits(vcpu, mmu);
+ __kvm_mmu_refresh_passthrough_bits(vcpu,
+ container_of(w, struct kvm_mmu, w));
}
/*
@@ -271,10 +272,12 @@ static inline void kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
* Return zero if the access does not fault; return the page fault error code
* if the access faults.
*/
-static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
+static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
unsigned pte_access, unsigned pte_pkey,
u64 access)
{
+ struct kvm_mmu *mmu = container_of(w, struct kvm_mmu, w);
+
/* strip nested paging fault error codes */
unsigned int pfec = access;
unsigned long rflags = kvm_x86_call(get_rflags)(vcpu);
@@ -297,7 +300,7 @@ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
u32 errcode = PFERR_PRESENT_MASK;
bool fault;
- kvm_mmu_refresh_passthrough_bits(vcpu, mmu);
+ kvm_mmu_refresh_passthrough_bits(vcpu, w);
fault = (mmu->permissions[index] >> pte_access) & 1;
@@ -377,12 +380,12 @@ static inline bool mmu_is_nested(struct kvm_vcpu *vcpu)
}
static inline gpa_t kvm_translate_gpa(struct kvm_vcpu *vcpu,
- struct kvm_mmu *mmu,
+ struct kvm_pagewalk *w,
gpa_t gpa, u64 access,
struct x86_exception *exception,
u64 pte_access)
{
- if (mmu != &vcpu->arch.nested_mmu)
+ if (w != &vcpu->arch.nested_mmu.w)
return gpa;
return kvm_x86_ops.nested_ops->translate_nested_gpa(vcpu, gpa, access,
exception,
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 0bf64e2722f0..3bc5195279fc 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -4378,7 +4378,7 @@ static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
* user-mode address if CR0.PG=0. Therefore *include* ACC_USER_MASK in
* the last argument to kvm_translate_gpa (which NPT does not use).
*/
- return kvm_translate_gpa(vcpu, mmu, vaddr, access | PFERR_GUEST_FINAL_MASK,
+ return kvm_translate_gpa(vcpu, &mmu->w, vaddr, access | PFERR_GUEST_FINAL_MASK,
exception, ACC_ALL);
}
diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index df3ae0c7ec2c..722567ae6b6a 100644
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -106,9 +106,10 @@ static gfn_t gpte_to_gfn_lvl(pt_element_t gpte, int lvl)
return (gpte & PT_LVL_ADDR_MASK(lvl)) >> PAGE_SHIFT;
}
-static inline void FNAME(protect_clean_gpte)(struct kvm_mmu *mmu, unsigned *access,
+static inline void FNAME(protect_clean_gpte)(struct kvm_pagewalk *w, unsigned *access,
unsigned gpte)
{
+ struct kvm_mmu __maybe_unused *mmu = container_of(w, struct kvm_mmu, w);
unsigned mask;
/* dirty bit is not supported, so no need to track it */
@@ -147,8 +148,10 @@ static bool FNAME(is_bad_mt_xwr)(struct rsvd_bits_validate *rsvd_check, u64 gpte
#endif
}
-static bool FNAME(is_rsvd_bits_set)(struct kvm_mmu *mmu, u64 gpte, int level)
+static bool FNAME(is_rsvd_bits_set)(struct kvm_pagewalk *w, u64 gpte, int level)
{
+ struct kvm_mmu *mmu = container_of(w, struct kvm_mmu, w);
+
return __is_rsvd_bits_set(&mmu->guest_rsvd_check, gpte, level) ||
FNAME(is_bad_mt_xwr)(&mmu->guest_rsvd_check, gpte);
}
@@ -165,7 +168,7 @@ static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu,
!(gpte & PT_GUEST_ACCESSED_MASK))
goto no_present;
- if (FNAME(is_rsvd_bits_set)(vcpu->arch.mmu, gpte, PG_LEVEL_4K))
+ if (FNAME(is_rsvd_bits_set)(&vcpu->arch.mmu->w, gpte, PG_LEVEL_4K))
goto no_present;
return false;
@@ -206,10 +209,11 @@ static inline unsigned FNAME(gpte_access)(u64 gpte)
}
static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu,
- struct kvm_mmu *mmu,
+ struct kvm_pagewalk *w,
struct guest_walker *walker,
gpa_t addr, int write_fault)
{
+ struct kvm_mmu __maybe_unused *mmu = container_of(w, struct kvm_mmu, w);
unsigned level, index;
pt_element_t pte, orig_pte;
pt_element_t __user *ptep_user;
@@ -278,9 +282,11 @@ static inline unsigned FNAME(gpte_pkeys)(struct kvm_vcpu *vcpu, u64 gpte)
return pkeys;
}
-static inline bool FNAME(is_last_gpte)(struct kvm_mmu *mmu,
+static inline bool FNAME(is_last_gpte)(struct kvm_pagewalk *w,
unsigned int level, unsigned int gpte)
{
+ struct kvm_mmu __maybe_unused *mmu = container_of(w, struct kvm_mmu, w);
+
/*
* For EPT and PAE paging (both variants), bit 7 is either reserved at
* all level or indicates a huge page (ignoring CR3/EPTP). In either
@@ -311,9 +317,10 @@ static inline bool FNAME(is_last_gpte)(struct kvm_mmu *mmu,
* Fetch a guest pte for a guest virtual address, or for an L2's GPA.
*/
static int FNAME(walk_addr_generic)(struct guest_walker *walker,
- struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
+ struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
gpa_t addr, u64 access)
{
+ struct kvm_mmu *mmu = container_of(w, struct kvm_mmu, w);
int ret;
pt_element_t pte;
pt_element_t __user *ptep_user;
@@ -393,7 +400,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
walker->table_gfn[walker->level - 1] = table_gfn;
walker->pte_gpa[walker->level - 1] = pte_gpa;
- real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(table_gfn),
+ real_gpa = kvm_translate_gpa(vcpu, w, gfn_to_gpa(table_gfn),
nested_access | PFERR_GUEST_PAGE_MASK,
&walker->fault, 0);
@@ -425,7 +432,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
if (unlikely(!FNAME(is_present_gpte)(mmu, pte)))
goto error;
- if (unlikely(FNAME(is_rsvd_bits_set)(mmu, pte, walker->level))) {
+ if (unlikely(FNAME(is_rsvd_bits_set)(w, pte, walker->level))) {
errcode = PFERR_RSVD_MASK | PFERR_PRESENT_MASK;
goto error;
}
@@ -434,14 +441,14 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
/* Convert to ACC_*_MASK flags for struct guest_walker. */
walker->pt_access[walker->level - 1] = FNAME(gpte_access)(pt_access ^ walk_nx_mask);
- } while (!FNAME(is_last_gpte)(mmu, walker->level, pte));
+ } while (!FNAME(is_last_gpte)(w, walker->level, pte));
pte_pkey = FNAME(gpte_pkeys)(vcpu, pte);
accessed_dirty = have_ad ? pte_access & PT_GUEST_ACCESSED_MASK : 0;
/* Convert to ACC_*_MASK flags for struct guest_walker. */
walker->pte_access = FNAME(gpte_access)(pte_access ^ walk_nx_mask);
- errcode = permission_fault(vcpu, mmu, walker->pte_access, pte_pkey, access);
+ errcode = permission_fault(vcpu, w, walker->pte_access, pte_pkey, access);
if (unlikely(errcode))
goto error;
@@ -453,7 +460,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
gfn += pse36_gfn_delta(pte);
#endif
- real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(gfn),
+ real_gpa = kvm_translate_gpa(vcpu, w, gfn_to_gpa(gfn),
access | PFERR_GUEST_FINAL_MASK,
&walker->fault, walker->pte_access);
if (real_gpa == INVALID_GPA)
@@ -462,7 +469,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
walker->gfn = real_gpa >> PAGE_SHIFT;
if (!write_fault)
- FNAME(protect_clean_gpte)(mmu, &walker->pte_access, pte);
+ FNAME(protect_clean_gpte)(w, &walker->pte_access, pte);
else
/*
* On a write fault, fold the dirty bit into accessed_dirty.
@@ -473,7 +480,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
(PT_GUEST_DIRTY_SHIFT - PT_GUEST_ACCESSED_SHIFT);
if (unlikely(!accessed_dirty)) {
- ret = FNAME(update_accessed_dirty_bits)(vcpu, mmu, walker,
+ ret = FNAME(update_accessed_dirty_bits)(vcpu, w, walker,
addr, write_fault);
if (unlikely(ret < 0))
goto error;
@@ -546,7 +553,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
}
#endif
walker->fault.address = addr;
- walker->fault.nested_page_fault = mmu != vcpu->arch.walk_mmu;
+ walker->fault.nested_page_fault = w != &vcpu->arch.walk_mmu->w;
walker->fault.async_page_fault = false;
#if PTTYPE != PTTYPE_EPT
@@ -561,7 +568,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
static int FNAME(walk_addr)(struct guest_walker *walker,
struct kvm_vcpu *vcpu, gpa_t addr, u64 access)
{
- return FNAME(walk_addr_generic)(walker, vcpu, vcpu->arch.mmu, addr,
+ return FNAME(walk_addr_generic)(walker, vcpu, &vcpu->arch.mmu->w, addr,
access);
}
@@ -577,7 +584,7 @@ FNAME(prefetch_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
gfn = gpte_to_gfn(gpte);
pte_access = sp->role.access & FNAME(gpte_access)(gpte);
- FNAME(protect_clean_gpte)(vcpu->arch.mmu, &pte_access, gpte);
+ FNAME(protect_clean_gpte)(&vcpu->arch.mmu->w, &pte_access, gpte);
return kvm_mmu_prefetch_sptes(vcpu, gfn, spte, 1, pte_access);
}
@@ -907,7 +914,7 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
WARN_ON_ONCE((addr >> 32) && mmu == vcpu->arch.walk_mmu);
#endif
- r = FNAME(walk_addr_generic)(&walker, vcpu, mmu, addr, access);
+ r = FNAME(walk_addr_generic)(&walker, vcpu, &mmu->w, addr, access);
if (r) {
gpa = gfn_to_gpa(walker.gfn);
@@ -957,7 +964,7 @@ static int FNAME(sync_spte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, int
gfn = gpte_to_gfn(gpte);
pte_access = sp->role.access;
pte_access &= FNAME(gpte_access)(gpte);
- FNAME(protect_clean_gpte)(vcpu->arch.mmu, &pte_access, gpte);
+ FNAME(protect_clean_gpte)(&vcpu->arch.mmu->w, &pte_access, gpte);
if (sync_mmio_spte(vcpu, &sp->spt[i], gfn, pte_access))
return 0;
diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c
index d2caf5a67dba..3b3515fd77e6 100644
--- a/arch/x86/kvm/regs.c
+++ b/arch/x86/kvm/regs.c
@@ -165,7 +165,7 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
* If the MMU is nested, CR3 holds an L2 GPA and needs to be translated
* to an L1 GPA.
*/
- real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn),
+ real_gpa = kvm_translate_gpa(vcpu, &mmu->w, gfn_to_gpa(pdpt_gfn),
PFERR_USER_MASK | PFERR_WRITE_MASK |
PFERR_GUEST_PAGE_MASK, NULL, 0);
if (real_gpa == INVALID_GPA)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 8dbc0fa302a8..e4d4d9ab3d50 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5007,7 +5007,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
* shadow page table for L2 guest.
*/
if (vcpu_match_mmio_gva(vcpu, gva) && (!is_paging(vcpu) ||
- !permission_fault(vcpu, vcpu->arch.walk_mmu,
+ !permission_fault(vcpu, &vcpu->arch.walk_mmu->w,
vcpu->arch.mmio_access, 0, access))) {
*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
(gva & (PAGE_SIZE - 1));
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 03/19] KVM: x86/mmu: move get_guest_pgd to struct kvm_pagewalk
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
2026-06-24 21:30 ` [PATCH 01/19] KVM: x86/hyperv: remove unnecessary mmu_is_nested() check Paolo Bonzini
2026-06-24 21:30 ` [PATCH 02/19] KVM: x86/mmu: introduce struct kvm_pagewalk Paolo Bonzini
@ 2026-06-24 21:30 ` Paolo Bonzini
2026-06-24 21:30 ` [PATCH 04/19] KVM: x86/mmu: move gva_to_gpa " Paolo Bonzini
` (16 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:30 UTC (permalink / raw)
To: linux-kernel, kvm
Start moving page walking functionality out of kvm_mmu. The easiest
target is the callbacks; change the kvm_mmu_get_guest_pgd() wrapper
to take a struct kvm_pagewalk too, and avoid the MMU indirection
whenever the caller has one.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/include/asm/kvm_host.h | 2 +-
arch/x86/kvm/mmu/mmu.c | 21 ++++++++++++---------
arch/x86/kvm/mmu/paging_tmpl.h | 2 +-
arch/x86/kvm/svm/nested.c | 4 +++-
arch/x86/kvm/vmx/nested.c | 3 ++-
5 files changed, 19 insertions(+), 13 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 57ea33f193d6..3a84ea81163f 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -517,12 +517,12 @@ struct kvm_page_fault;
* current mmu mode.
*/
struct kvm_pagewalk {
+ unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu);
};
struct kvm_mmu {
struct kvm_pagewalk w;
- unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu);
u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
int (*page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
void (*inject_page_fault)(struct kvm_vcpu *vcpu,
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 3bc5195279fc..adbd07b5af08 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -272,12 +272,12 @@ static unsigned long get_guest_cr3(struct kvm_vcpu *vcpu)
}
static inline unsigned long kvm_mmu_get_guest_pgd(struct kvm_vcpu *vcpu,
- struct kvm_mmu *mmu)
+ struct kvm_pagewalk *w)
{
- if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) && mmu->get_guest_pgd == get_guest_cr3)
+ if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) && w->get_guest_pgd == get_guest_cr3)
return kvm_read_cr3(vcpu);
- return mmu->get_guest_pgd(vcpu);
+ return w->get_guest_pgd(vcpu);
}
static inline bool kvm_available_flush_remote_tlbs_range(void)
@@ -4095,7 +4095,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
int quadrant, i, r;
hpa_t root;
- root_pgd = kvm_mmu_get_guest_pgd(vcpu, mmu);
+ root_pgd = kvm_mmu_get_guest_pgd(vcpu, &mmu->w);
root_gfn = (root_pgd & __PT_BASE_ADDR_MASK) >> PAGE_SHIFT;
if (!kvm_vcpu_is_visible_gfn(vcpu, root_gfn)) {
@@ -4567,7 +4567,7 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu,
if (arch.direct_map)
arch.cr3 = (unsigned long)INVALID_GPA;
else
- arch.cr3 = kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu);
+ arch.cr3 = kvm_mmu_get_guest_pgd(vcpu, &vcpu->arch.mmu->w);
return kvm_setup_async_pf(vcpu, fault->addr,
kvm_vcpu_gfn_to_hva(vcpu, fault->gfn), &arch);
@@ -5110,7 +5110,7 @@ void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
return;
if (!vcpu->arch.mmu->root_role.direct &&
- work->arch.cr3 != kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu))
+ work->arch.cr3 != kvm_mmu_get_guest_pgd(vcpu, &vcpu->arch.mmu->w))
return;
r = kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, work->arch.error_code,
@@ -5969,10 +5969,11 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
context->root_role.word = root_role.word;
context->page_fault = kvm_tdp_page_fault;
context->sync_spte = NULL;
- context->get_guest_pgd = get_guest_cr3;
context->get_pdptr = kvm_pdptr_read;
context->inject_page_fault = kvm_inject_page_fault;
+ context->w.get_guest_pgd = get_guest_cr3;
+
if (!is_cr0_pg(context))
context->gva_to_gpa = nonpaging_gva_to_gpa;
else if (is_cr4_pae(context))
@@ -6120,7 +6121,8 @@ static void init_kvm_softmmu(struct kvm_vcpu *vcpu,
kvm_init_shadow_mmu(vcpu, cpu_role);
- context->get_guest_pgd = get_guest_cr3;
+ context->w.get_guest_pgd = get_guest_cr3;
+
context->get_pdptr = kvm_pdptr_read;
context->inject_page_fault = kvm_inject_page_fault;
}
@@ -6134,10 +6136,11 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu,
return;
g_context->cpu_role.as_u64 = new_mode.as_u64;
- g_context->get_guest_pgd = get_guest_cr3;
g_context->get_pdptr = kvm_pdptr_read;
g_context->inject_page_fault = kvm_inject_page_fault;
+ g_context->w.get_guest_pgd = get_guest_cr3;
+
/*
* L2 page tables are never shadowed, so there is no need to sync
* SPTEs.
diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index 722567ae6b6a..14794026f9bf 100644
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -348,7 +348,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
trace_kvm_mmu_pagetable_walk(addr, access);
retry_walk:
walker->level = mmu->cpu_role.base.level;
- pte = kvm_mmu_get_guest_pgd(vcpu, mmu);
+ pte = kvm_mmu_get_guest_pgd(vcpu, w);
have_ad = PT_HAVE_ACCESSED_DIRTY(mmu);
#if PTTYPE == 64
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index c1485c3e691c..16849323d091 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -113,7 +113,9 @@ static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
svm->vmcb01.ptr->save.efer,
svm->nested.ctl.nested_cr3,
svm->nested.ctl.misc_ctl);
- vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3;
+
+ vcpu->arch.mmu->w.get_guest_pgd = nested_svm_get_tdp_cr3;
+
vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr;
vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit;
vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu;
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 23def2157bc5..b717201e9e58 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -512,7 +512,8 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
vcpu->arch.mmu = &vcpu->arch.guest_mmu;
nested_ept_new_eptp(vcpu);
- vcpu->arch.mmu->get_guest_pgd = nested_ept_get_eptp;
+ vcpu->arch.mmu->w.get_guest_pgd = nested_ept_get_eptp;
+
vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault;
vcpu->arch.mmu->get_pdptr = kvm_pdptr_read;
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 04/19] KVM: x86/mmu: move gva_to_gpa to struct kvm_pagewalk
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
` (2 preceding siblings ...)
2026-06-24 21:30 ` [PATCH 03/19] KVM: x86/mmu: move get_guest_pgd to " Paolo Bonzini
@ 2026-06-24 21:30 ` Paolo Bonzini
2026-06-24 21:30 ` [PATCH 05/19] KVM: x86/mmu: move get_pdptr " Paolo Bonzini
` (15 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:30 UTC (permalink / raw)
To: linux-kernel, kvm
gva_to_gpa is the main entry point into walk_mmu, which
is only used for guest page table walking (as opposed to building
the page tables). Moving gva_to_gpa to struct kvm_pagewalk
is a steps towards making walk_mmu a struct kvm_pagewalk.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/include/asm/kvm_host.h | 6 +++---
arch/x86/kvm/mmu/mmu.c | 26 +++++++++++++-------------
arch/x86/kvm/mmu/paging_tmpl.h | 6 +++---
arch/x86/kvm/svm/nested.c | 4 ++--
arch/x86/kvm/vmx/nested.c | 4 ++--
arch/x86/kvm/x86.c | 30 +++++++++++++++---------------
6 files changed, 38 insertions(+), 38 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 3a84ea81163f..95c2caf18bee 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -518,6 +518,9 @@ struct kvm_page_fault;
*/
struct kvm_pagewalk {
unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu);
+ gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
+ gpa_t gva_or_gpa, u64 access,
+ struct x86_exception *exception);
};
struct kvm_mmu {
@@ -528,9 +531,6 @@ struct kvm_mmu {
void (*inject_page_fault)(struct kvm_vcpu *vcpu,
struct x86_exception *fault,
bool from_hardware);
- gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
- gpa_t gva_or_gpa, u64 access,
- struct x86_exception *exception);
int (*sync_spte)(struct kvm_vcpu *vcpu,
struct kvm_mmu_page *sp, int i);
struct kvm_mmu_root_info root;
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index adbd07b5af08..11efca459509 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -4366,7 +4366,7 @@ void kvm_mmu_sync_prev_roots(struct kvm_vcpu *vcpu)
kvm_mmu_free_roots(vcpu->kvm, vcpu->arch.mmu, roots_to_free);
}
-static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
+static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
gpa_t vaddr, u64 access,
struct x86_exception *exception)
{
@@ -4378,7 +4378,7 @@ static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
* user-mode address if CR0.PG=0. Therefore *include* ACC_USER_MASK in
* the last argument to kvm_translate_gpa (which NPT does not use).
*/
- return kvm_translate_gpa(vcpu, &mmu->w, vaddr, access | PFERR_GUEST_FINAL_MASK,
+ return kvm_translate_gpa(vcpu, w, vaddr, access | PFERR_GUEST_FINAL_MASK,
exception, ACC_ALL);
}
@@ -5208,7 +5208,7 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_tdp_mmu_map_private_pfn);
static void nonpaging_init_context(struct kvm_mmu *context)
{
context->page_fault = nonpaging_page_fault;
- context->gva_to_gpa = nonpaging_gva_to_gpa;
+ context->w.gva_to_gpa = nonpaging_gva_to_gpa;
context->sync_spte = NULL;
}
@@ -5839,14 +5839,14 @@ static void reset_guest_paging_metadata(struct kvm_vcpu *vcpu,
static void paging64_init_context(struct kvm_mmu *context)
{
context->page_fault = paging64_page_fault;
- context->gva_to_gpa = paging64_gva_to_gpa;
+ context->w.gva_to_gpa = paging64_gva_to_gpa;
context->sync_spte = paging64_sync_spte;
}
static void paging32_init_context(struct kvm_mmu *context)
{
context->page_fault = paging32_page_fault;
- context->gva_to_gpa = paging32_gva_to_gpa;
+ context->w.gva_to_gpa = paging32_gva_to_gpa;
context->sync_spte = paging32_sync_spte;
}
@@ -5975,11 +5975,11 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
context->w.get_guest_pgd = get_guest_cr3;
if (!is_cr0_pg(context))
- context->gva_to_gpa = nonpaging_gva_to_gpa;
+ context->w.gva_to_gpa = nonpaging_gva_to_gpa;
else if (is_cr4_pae(context))
- context->gva_to_gpa = paging64_gva_to_gpa;
+ context->w.gva_to_gpa = paging64_gva_to_gpa;
else
- context->gva_to_gpa = paging32_gva_to_gpa;
+ context->w.gva_to_gpa = paging32_gva_to_gpa;
reset_guest_paging_metadata(vcpu, context);
reset_tdp_shadow_zero_bits_mask(context);
@@ -6101,7 +6101,7 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
context->root_role.word = new_mode.base.word;
context->page_fault = ept_page_fault;
- context->gva_to_gpa = ept_gva_to_gpa;
+ context->w.gva_to_gpa = ept_gva_to_gpa;
context->sync_spte = ept_sync_spte;
update_permission_bitmask(context, true, true);
@@ -6156,13 +6156,13 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu,
* the gva_to_gpa functions between mmu and nested_mmu are swapped.
*/
if (!is_paging(vcpu))
- g_context->gva_to_gpa = nonpaging_gva_to_gpa;
+ g_context->w.gva_to_gpa = nonpaging_gva_to_gpa;
else if (is_long_mode(vcpu))
- g_context->gva_to_gpa = paging64_gva_to_gpa;
+ g_context->w.gva_to_gpa = paging64_gva_to_gpa;
else if (is_pae(vcpu))
- g_context->gva_to_gpa = paging64_gva_to_gpa;
+ g_context->w.gva_to_gpa = paging64_gva_to_gpa;
else
- g_context->gva_to_gpa = paging32_gva_to_gpa;
+ g_context->w.gva_to_gpa = paging32_gva_to_gpa;
reset_guest_paging_metadata(vcpu, g_context);
}
diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index 14794026f9bf..49c4feed7cd2 100644
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -901,7 +901,7 @@ static gpa_t FNAME(get_level1_sp_gpa)(struct kvm_mmu_page *sp)
}
/* Note, @addr is a GPA when gva_to_gpa() translates an L2 GPA to an L1 GPA. */
-static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
+static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
gpa_t addr, u64 access,
struct x86_exception *exception)
{
@@ -911,10 +911,10 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
#ifndef CONFIG_X86_64
/* A 64-bit GVA should be impossible on 32-bit KVM. */
- WARN_ON_ONCE((addr >> 32) && mmu == vcpu->arch.walk_mmu);
+ WARN_ON_ONCE((addr >> 32) && w == &vcpu->arch.walk_mmu->w);
#endif
- r = FNAME(walk_addr_generic)(&walker, vcpu, &mmu->w, addr, access);
+ r = FNAME(walk_addr_generic)(&walker, vcpu, w, addr, access);
if (r) {
gpa = gfn_to_gpa(walker.gfn);
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index 16849323d091..be92c1a5144c 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -2153,7 +2153,7 @@ static gpa_t svm_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa,
u64 pte_access)
{
struct vcpu_svm *svm = to_svm(vcpu);
- struct kvm_mmu *mmu = vcpu->arch.mmu;
+ struct kvm_pagewalk *w = &vcpu->arch.mmu->w;
if (WARN_ON_ONCE(!mmu_is_nested(vcpu)))
return gpa;
@@ -2162,7 +2162,7 @@ static gpa_t svm_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa,
if (!(svm->nested.ctl.misc_ctl & SVM_MISC_ENABLE_GMET))
access |= PFERR_USER_MASK;
- return mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);
+ return w->gva_to_gpa(vcpu, w, gpa, access, exception);
}
struct kvm_x86_nested_ops svm_nested_ops = {
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index b717201e9e58..0e2d7714c6d2 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -7470,7 +7470,7 @@ static gpa_t vmx_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa,
struct x86_exception *exception,
u64 pte_access)
{
- struct kvm_mmu *mmu = vcpu->arch.mmu;
+ struct kvm_pagewalk *w = &vcpu->arch.mmu->w;
if (WARN_ON_ONCE(!mmu_is_nested(vcpu)))
return gpa;
@@ -7483,7 +7483,7 @@ static gpa_t vmx_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa,
if ((pte_access & ACC_USER_MASK) && (access & PFERR_GUEST_FINAL_MASK))
access |= PFERR_USER_MASK;
- return mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);
+ return w->gva_to_gpa(vcpu, w, gpa, access, exception);
}
struct kvm_x86_nested_ops vmx_nested_ops = {
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index e4d4d9ab3d50..bf0a65c76f71 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -4768,21 +4768,21 @@ static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
struct x86_exception *exception)
{
- struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
+ struct kvm_pagewalk *gva_walk = &vcpu->arch.walk_mmu->w;
u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
- return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
+ return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, exception);
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_gva_to_gpa_read);
gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
struct x86_exception *exception)
{
- struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
+ struct kvm_pagewalk *gva_walk = &vcpu->arch.walk_mmu->w;
u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
access |= PFERR_WRITE_MASK;
- return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
+ return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, exception);
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_gva_to_gpa_write);
@@ -4790,21 +4790,21 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_gva_to_gpa_write);
gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
struct x86_exception *exception)
{
- struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
+ struct kvm_pagewalk *gva_walk = &vcpu->arch.walk_mmu->w;
- return mmu->gva_to_gpa(vcpu, mmu, gva, 0, exception);
+ return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, 0, exception);
}
static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
struct kvm_vcpu *vcpu, u64 access,
struct x86_exception *exception)
{
- struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
+ struct kvm_pagewalk *gva_walk = &vcpu->arch.walk_mmu->w;
void *data = val;
int r = X86EMUL_CONTINUE;
while (bytes) {
- gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);
+ gpa_t gpa = gva_walk->gva_to_gpa(vcpu, gva_walk, addr, access, exception);
unsigned offset = addr & (PAGE_SIZE-1);
unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
int ret;
@@ -4832,14 +4832,14 @@ static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
struct x86_exception *exception)
{
struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
- struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
+ struct kvm_pagewalk *gva_walk = &vcpu->arch.walk_mmu->w;
u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
unsigned offset;
int ret;
/* Inline kvm_read_guest_virt_helper for speed. */
- gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access|PFERR_FETCH_MASK,
- exception);
+ gpa_t gpa = gva_walk->gva_to_gpa(vcpu, gva_walk, addr, access|PFERR_FETCH_MASK,
+ exception);
if (unlikely(gpa == INVALID_GPA))
return X86EMUL_PROPAGATE_FAULT;
@@ -4891,12 +4891,12 @@ static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes
struct kvm_vcpu *vcpu, u64 access,
struct x86_exception *exception)
{
- struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
+ struct kvm_pagewalk *gva_walk = &vcpu->arch.walk_mmu->w;
void *data = val;
int r = X86EMUL_CONTINUE;
while (bytes) {
- gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);
+ gpa_t gpa = gva_walk->gva_to_gpa(vcpu, gva_walk, addr, access, exception);
unsigned offset = addr & (PAGE_SIZE-1);
unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
int ret;
@@ -5015,7 +5015,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
return 1;
}
- *gpa = mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
+ *gpa = mmu->w.gva_to_gpa(vcpu, &mmu->w, gva, access, exception);
if (*gpa == INVALID_GPA)
return -1;
@@ -10606,7 +10606,7 @@ void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_c
(PFERR_WRITE_MASK | PFERR_FETCH_MASK | PFERR_USER_MASK);
if (!(error_code & PFERR_PRESENT_MASK) ||
- mmu->gva_to_gpa(vcpu, mmu, gva, access, &fault) != INVALID_GPA) {
+ mmu->w.gva_to_gpa(vcpu, &mmu->w, gva, access, &fault) != INVALID_GPA) {
/*
* If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page
* tables probably do not match the TLB. Just proceed
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 05/19] KVM: x86/mmu: move get_pdptr to struct kvm_pagewalk
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
` (3 preceding siblings ...)
2026-06-24 21:30 ` [PATCH 04/19] KVM: x86/mmu: move gva_to_gpa " Paolo Bonzini
@ 2026-06-24 21:30 ` Paolo Bonzini
2026-06-24 21:30 ` [PATCH 06/19] KVM: x86/mmu: move inject_page_fault " Paolo Bonzini
` (14 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:30 UTC (permalink / raw)
To: linux-kernel, kvm
Continue with yet another callback used in FNAME(walk_addr_generic),
as another step towards removing container_of() from there.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/include/asm/kvm_host.h | 2 +-
arch/x86/kvm/mmu/mmu.c | 8 ++++----
arch/x86/kvm/mmu/paging_tmpl.h | 2 +-
arch/x86/kvm/svm/nested.c | 2 +-
arch/x86/kvm/vmx/nested.c | 2 +-
5 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 95c2caf18bee..4ddfe9debf18 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -518,6 +518,7 @@ struct kvm_page_fault;
*/
struct kvm_pagewalk {
unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu);
+ u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
gpa_t gva_or_gpa, u64 access,
struct x86_exception *exception);
@@ -526,7 +527,6 @@ struct kvm_pagewalk {
struct kvm_mmu {
struct kvm_pagewalk w;
- u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
int (*page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
void (*inject_page_fault)(struct kvm_vcpu *vcpu,
struct x86_exception *fault,
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 11efca459509..ac9e1e95335a 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -4109,7 +4109,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
*/
if (mmu->cpu_role.base.level == PT32E_ROOT_LEVEL) {
for (i = 0; i < 4; ++i) {
- pdptrs[i] = mmu->get_pdptr(vcpu, i);
+ pdptrs[i] = mmu->w.get_pdptr(vcpu, i);
if (!(pdptrs[i] & PT_PRESENT_MASK))
continue;
@@ -5969,9 +5969,9 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
context->root_role.word = root_role.word;
context->page_fault = kvm_tdp_page_fault;
context->sync_spte = NULL;
- context->get_pdptr = kvm_pdptr_read;
context->inject_page_fault = kvm_inject_page_fault;
+ context->w.get_pdptr = kvm_pdptr_read;
context->w.get_guest_pgd = get_guest_cr3;
if (!is_cr0_pg(context))
@@ -6121,9 +6121,9 @@ static void init_kvm_softmmu(struct kvm_vcpu *vcpu,
kvm_init_shadow_mmu(vcpu, cpu_role);
+ context->w.get_pdptr = kvm_pdptr_read;
context->w.get_guest_pgd = get_guest_cr3;
- context->get_pdptr = kvm_pdptr_read;
context->inject_page_fault = kvm_inject_page_fault;
}
@@ -6136,9 +6136,9 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu,
return;
g_context->cpu_role.as_u64 = new_mode.as_u64;
- g_context->get_pdptr = kvm_pdptr_read;
g_context->inject_page_fault = kvm_inject_page_fault;
+ g_context->w.get_pdptr = kvm_pdptr_read;
g_context->w.get_guest_pgd = get_guest_cr3;
/*
diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index 49c4feed7cd2..cf13a8b608dc 100644
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -354,7 +354,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
#if PTTYPE == 64
walk_nx_mask = 1ULL << PT64_NX_SHIFT;
if (walker->level == PT32E_ROOT_LEVEL) {
- pte = mmu->get_pdptr(vcpu, (addr >> 30) & 3);
+ pte = w->get_pdptr(vcpu, (addr >> 30) & 3);
trace_kvm_mmu_paging_element(pte, walker->level);
if (!FNAME(is_present_gpte)(mmu, pte))
goto error;
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index be92c1a5144c..6634d6390f77 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -115,8 +115,8 @@ static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
svm->nested.ctl.misc_ctl);
vcpu->arch.mmu->w.get_guest_pgd = nested_svm_get_tdp_cr3;
+ vcpu->arch.mmu->w.get_pdptr = nested_svm_get_tdp_pdptr;
- vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr;
vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit;
vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu;
}
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 0e2d7714c6d2..ef54c444413e 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -513,9 +513,9 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
vcpu->arch.mmu = &vcpu->arch.guest_mmu;
nested_ept_new_eptp(vcpu);
vcpu->arch.mmu->w.get_guest_pgd = nested_ept_get_eptp;
+ vcpu->arch.mmu->w.get_pdptr = kvm_pdptr_read;
vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault;
- vcpu->arch.mmu->get_pdptr = kvm_pdptr_read;
vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu;
}
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 06/19] KVM: x86/mmu: move inject_page_fault to struct kvm_pagewalk
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
` (4 preceding siblings ...)
2026-06-24 21:30 ` [PATCH 05/19] KVM: x86/mmu: move get_pdptr " Paolo Bonzini
@ 2026-06-24 21:30 ` Paolo Bonzini
2026-06-24 21:30 ` [PATCH 07/19] KVM: x86/mmu: move CPU-related fields " Paolo Bonzini
` (13 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:30 UTC (permalink / raw)
To: linux-kernel, kvm
Injection of page faults is also part of accesses to guest
page tables. In particular, kvm_inject_emulated_page_fault
calls it on walk_mmu. Move it to struct kvm_pagewalk as
part of converting walk_mmu to a struct kvm_pagewalk.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/include/asm/kvm_host.h | 6 +++---
arch/x86/kvm/mmu/mmu.c | 8 +++-----
arch/x86/kvm/svm/nested.c | 2 +-
arch/x86/kvm/vmx/nested.c | 2 +-
arch/x86/kvm/x86.c | 4 ++--
5 files changed, 10 insertions(+), 12 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 4ddfe9debf18..7e397b5d93ce 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -519,6 +519,9 @@ struct kvm_page_fault;
struct kvm_pagewalk {
unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu);
u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
+ void (*inject_page_fault)(struct kvm_vcpu *vcpu,
+ struct x86_exception *fault,
+ bool from_hardware);
gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
gpa_t gva_or_gpa, u64 access,
struct x86_exception *exception);
@@ -528,9 +531,6 @@ struct kvm_mmu {
struct kvm_pagewalk w;
int (*page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
- void (*inject_page_fault)(struct kvm_vcpu *vcpu,
- struct x86_exception *fault,
- bool from_hardware);
int (*sync_spte)(struct kvm_vcpu *vcpu,
struct kvm_mmu_page *sp, int i);
struct kvm_mmu_root_info root;
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index ac9e1e95335a..ffc8acefd5c9 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -5969,8 +5969,8 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
context->root_role.word = root_role.word;
context->page_fault = kvm_tdp_page_fault;
context->sync_spte = NULL;
- context->inject_page_fault = kvm_inject_page_fault;
+ context->w.inject_page_fault = kvm_inject_page_fault;
context->w.get_pdptr = kvm_pdptr_read;
context->w.get_guest_pgd = get_guest_cr3;
@@ -6121,10 +6121,9 @@ static void init_kvm_softmmu(struct kvm_vcpu *vcpu,
kvm_init_shadow_mmu(vcpu, cpu_role);
+ context->w.inject_page_fault = kvm_inject_page_fault;
context->w.get_pdptr = kvm_pdptr_read;
context->w.get_guest_pgd = get_guest_cr3;
-
- context->inject_page_fault = kvm_inject_page_fault;
}
static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu,
@@ -6136,8 +6135,7 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu,
return;
g_context->cpu_role.as_u64 = new_mode.as_u64;
- g_context->inject_page_fault = kvm_inject_page_fault;
-
+ g_context->w.inject_page_fault = kvm_inject_page_fault;
g_context->w.get_pdptr = kvm_pdptr_read;
g_context->w.get_guest_pgd = get_guest_cr3;
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index 6634d6390f77..55eace8aa2c8 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -117,7 +117,7 @@ static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
vcpu->arch.mmu->w.get_guest_pgd = nested_svm_get_tdp_cr3;
vcpu->arch.mmu->w.get_pdptr = nested_svm_get_tdp_pdptr;
- vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit;
+ vcpu->arch.mmu->w.inject_page_fault = nested_svm_inject_npf_exit;
vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu;
}
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index ef54c444413e..466f838a05bc 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -515,7 +515,7 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
vcpu->arch.mmu->w.get_guest_pgd = nested_ept_get_eptp;
vcpu->arch.mmu->w.get_pdptr = kvm_pdptr_read;
- vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault;
+ vcpu->arch.mmu->w.inject_page_fault = nested_ept_inject_page_fault;
vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu;
}
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index bf0a65c76f71..484ed409e420 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -597,7 +597,7 @@ void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
kvm_mmu_invalidate_addr(vcpu, fault_mmu, fault->address,
KVM_MMU_ROOT_CURRENT);
- fault_mmu->inject_page_fault(vcpu, fault, from_hardware);
+ fault_mmu->w.inject_page_fault(vcpu, fault, from_hardware);
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_inject_emulated_page_fault);
@@ -10619,7 +10619,7 @@ void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_c
fault.address = gva;
fault.async_page_fault = false;
}
- vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault, true);
+ vcpu->arch.walk_mmu->w.inject_page_fault(vcpu, &fault, true);
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_fixup_and_inject_pf_error);
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 07/19] KVM: x86/mmu: move CPU-related fields to struct kvm_pagewalk
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
` (5 preceding siblings ...)
2026-06-24 21:30 ` [PATCH 06/19] KVM: x86/mmu: move inject_page_fault " Paolo Bonzini
@ 2026-06-24 21:30 ` Paolo Bonzini
2026-06-24 21:30 ` [PATCH 08/19] KVM: x86/mmu: change CPU-role accessor fields to take " Paolo Bonzini
` (12 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:30 UTC (permalink / raw)
To: linux-kernel, kvm
struct kvm_pagewalk's behavior depends on the CPU state and its
page format. Move related fields so that walk_mmu remains
self contained.
Note that for now, some of the accessors still use kvm_mmu
to split the churn.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/include/asm/kvm_host.h | 4 +--
arch/x86/kvm/mmu/mmu.c | 52 ++++++++++++++++-----------------
arch/x86/kvm/mmu/paging_tmpl.h | 40 ++++++++++++-------------
3 files changed, 46 insertions(+), 50 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 7e397b5d93ce..93b7e528d3c6 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -525,6 +525,8 @@ struct kvm_pagewalk {
gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
gpa_t gva_or_gpa, u64 access,
struct x86_exception *exception);
+ union kvm_cpu_role cpu_role;
+ struct rsvd_bits_validate guest_rsvd_check;
};
struct kvm_mmu {
@@ -535,7 +537,6 @@ struct kvm_mmu {
struct kvm_mmu_page *sp, int i);
struct kvm_mmu_root_info root;
hpa_t mirror_root_hpa;
- union kvm_cpu_role cpu_role;
union kvm_mmu_page_role root_role;
/*
@@ -565,7 +566,6 @@ struct kvm_mmu {
* the bits spte never used.
*/
struct rsvd_bits_validate shadow_zero_check;
- struct rsvd_bits_validate guest_rsvd_check;
};
enum pmc_type {
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index ffc8acefd5c9..92142b389b1a 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -229,7 +229,7 @@ BUILD_MMU_ROLE_REGS_ACCESSOR(efer, lma, EFER_LMA);
#define BUILD_MMU_ROLE_ACCESSOR(base_or_ext, reg, name) \
static inline bool __maybe_unused is_##reg##_##name(struct kvm_mmu *mmu) \
{ \
- return !!(mmu->cpu_role. base_or_ext . reg##_##name); \
+ return !!(mmu->w.cpu_role. base_or_ext . reg##_##name); \
}
BUILD_MMU_ROLE_ACCESSOR(base, cr0, wp);
BUILD_MMU_ROLE_ACCESSOR(ext, cr4, pse);
@@ -242,17 +242,17 @@ BUILD_MMU_ROLE_ACCESSOR(ext, efer, lma);
static inline bool has_pferr_fetch(struct kvm_mmu *mmu)
{
- return mmu->cpu_role.ext.has_pferr_fetch;
+ return mmu->w.cpu_role.ext.has_pferr_fetch;
}
static inline bool is_cr0_pg(struct kvm_mmu *mmu)
{
- return mmu->cpu_role.base.level > 0;
+ return mmu->w.cpu_role.base.level > 0;
}
static inline bool is_cr4_pae(struct kvm_mmu *mmu)
{
- return !mmu->cpu_role.base.has_4_byte_gpte;
+ return !mmu->w.cpu_role.base.has_4_byte_gpte;
}
static struct kvm_mmu_role_regs vcpu_to_role_regs(struct kvm_vcpu *vcpu)
@@ -2483,7 +2483,7 @@ static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterato
iterator->level = vcpu->arch.mmu->root_role.level;
if (iterator->level >= PT64_ROOT_4LEVEL &&
- vcpu->arch.mmu->cpu_role.base.level < PT64_ROOT_4LEVEL &&
+ vcpu->arch.mmu->w.cpu_role.base.level < PT64_ROOT_4LEVEL &&
!vcpu->arch.mmu->root_role.direct)
iterator->level = PT32E_ROOT_LEVEL;
@@ -4107,7 +4107,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
* On SVM, reading PDPTRs might access guest memory, which might fault
* and thus might sleep. Grab the PDPTRs before acquiring mmu_lock.
*/
- if (mmu->cpu_role.base.level == PT32E_ROOT_LEVEL) {
+ if (mmu->w.cpu_role.base.level == PT32E_ROOT_LEVEL) {
for (i = 0; i < 4; ++i) {
pdptrs[i] = mmu->w.get_pdptr(vcpu, i);
if (!(pdptrs[i] & PT_PRESENT_MASK))
@@ -4131,7 +4131,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
* Do we shadow a long mode page table? If so we need to
* write-protect the guests page table root.
*/
- if (mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL) {
+ if (mmu->w.cpu_role.base.level >= PT64_ROOT_4LEVEL) {
root = mmu_alloc_root(vcpu, root_gfn, 0,
mmu->root_role.level);
mmu->root.hpa = root;
@@ -4170,7 +4170,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
for (i = 0; i < 4; ++i) {
WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i]));
- if (mmu->cpu_role.base.level == PT32E_ROOT_LEVEL) {
+ if (mmu->w.cpu_role.base.level == PT32E_ROOT_LEVEL) {
if (!(pdptrs[i] & PT_PRESENT_MASK)) {
mmu->pae_root[i] = INVALID_PAE_ROOT;
continue;
@@ -4184,7 +4184,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
* directory. Othwerise each PAE page direct shadows one guest
* PAE page directory so that quadrant should be 0.
*/
- quadrant = (mmu->cpu_role.base.level == PT32_ROOT_LEVEL) ? i : 0;
+ quadrant = (mmu->w.cpu_role.base.level == PT32_ROOT_LEVEL) ? i : 0;
root = mmu_alloc_root(vcpu, root_gfn, quadrant, PT32_ROOT_LEVEL);
mmu->pae_root[i] = root | pm_mask;
@@ -4220,7 +4220,7 @@ static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu)
* on demand, as running a 32-bit L1 VMM on 64-bit KVM is very rare.
*/
if (mmu->root_role.direct ||
- mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL ||
+ mmu->w.cpu_role.base.level >= PT64_ROOT_4LEVEL ||
mmu->root_role.level < PT64_ROOT_4LEVEL)
return 0;
@@ -4325,7 +4325,7 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
- if (vcpu->arch.mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL) {
+ if (vcpu->arch.mmu->w.cpu_role.base.level >= PT64_ROOT_4LEVEL) {
hpa_t root = vcpu->arch.mmu->root.hpa;
if (!is_unsync_root(root))
@@ -5476,9 +5476,9 @@ static void __reset_rsvds_bits_mask(struct rsvd_bits_validate *rsvd_check,
static void reset_guest_rsvds_bits_mask(struct kvm_vcpu *vcpu,
struct kvm_mmu *context)
{
- __reset_rsvds_bits_mask(&context->guest_rsvd_check,
+ __reset_rsvds_bits_mask(&context->w.guest_rsvd_check,
vcpu->arch.reserved_gpa_bits,
- context->cpu_role.base.level, is_efer_nx(context),
+ context->w.cpu_role.base.level, is_efer_nx(context),
guest_cpu_cap_has(vcpu, X86_FEATURE_GBPAGES),
is_cr4_pse(context),
guest_cpuid_is_amd_compatible(vcpu));
@@ -5525,7 +5525,7 @@ static void __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
struct kvm_mmu *context, bool execonly, int huge_page_level)
{
- __reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,
+ __reset_rsvds_bits_mask_ept(&context->w.guest_rsvd_check,
vcpu->arch.reserved_gpa_bits, execonly,
huge_page_level);
}
@@ -5902,7 +5902,7 @@ void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
if (is_cr0_wp(mmu) == cr0_wp)
return;
- mmu->cpu_role.base.cr0_wp = cr0_wp;
+ mmu->w.cpu_role.base.cr0_wp = cr0_wp;
reset_guest_paging_metadata(vcpu, mmu);
}
@@ -5961,11 +5961,11 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
struct kvm_mmu *context = &vcpu->arch.root_mmu;
union kvm_mmu_page_role root_role = kvm_calc_tdp_mmu_root_page_role(vcpu, cpu_role);
- if (cpu_role.as_u64 == context->cpu_role.as_u64 &&
+ if (cpu_role.as_u64 == context->w.cpu_role.as_u64 &&
root_role.word == context->root_role.word)
return;
- context->cpu_role.as_u64 = cpu_role.as_u64;
+ context->w.cpu_role.as_u64 = cpu_role.as_u64;
context->root_role.word = root_role.word;
context->page_fault = kvm_tdp_page_fault;
context->sync_spte = NULL;
@@ -5989,11 +5989,11 @@ static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *conte
union kvm_cpu_role cpu_role,
union kvm_mmu_page_role root_role)
{
- if (cpu_role.as_u64 == context->cpu_role.as_u64 &&
+ if (cpu_role.as_u64 == context->w.cpu_role.as_u64 &&
root_role.word == context->root_role.word)
return;
- context->cpu_role.as_u64 = cpu_role.as_u64;
+ context->w.cpu_role.as_u64 = cpu_role.as_u64;
context->root_role.word = root_role.word;
if (!is_cr0_pg(context))
@@ -6095,9 +6095,9 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty,
execonly, level, mbec);
- if (new_mode.as_u64 != context->cpu_role.as_u64) {
+ if (new_mode.as_u64 != context->w.cpu_role.as_u64) {
/* EPT, and thus nested EPT, does not consume CR0, CR4, nor EFER. */
- context->cpu_role.as_u64 = new_mode.as_u64;
+ context->w.cpu_role.as_u64 = new_mode.as_u64;
context->root_role.word = new_mode.base.word;
context->page_fault = ept_page_fault;
@@ -6131,10 +6131,10 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu,
{
struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;
- if (new_mode.as_u64 == g_context->cpu_role.as_u64)
+ if (new_mode.as_u64 == g_context->w.cpu_role.as_u64)
return;
- g_context->cpu_role.as_u64 = new_mode.as_u64;
+ g_context->w.cpu_role.as_u64 = new_mode.as_u64;
g_context->w.inject_page_fault = kvm_inject_page_fault;
g_context->w.get_pdptr = kvm_pdptr_read;
g_context->w.get_guest_pgd = get_guest_cr3;
@@ -6196,9 +6196,9 @@ void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu)
vcpu->arch.root_mmu.root_role.invalid = 1;
vcpu->arch.guest_mmu.root_role.invalid = 1;
vcpu->arch.nested_mmu.root_role.invalid = 1;
- vcpu->arch.root_mmu.cpu_role.ext.valid = 0;
- vcpu->arch.guest_mmu.cpu_role.ext.valid = 0;
- vcpu->arch.nested_mmu.cpu_role.ext.valid = 0;
+ vcpu->arch.root_mmu.w.cpu_role.ext.valid = 0;
+ vcpu->arch.guest_mmu.w.cpu_role.ext.valid = 0;
+ vcpu->arch.nested_mmu.w.cpu_role.ext.valid = 0;
kvm_mmu_reset_context(vcpu);
KVM_BUG_ON(!kvm_can_set_cpuid_and_feature_msrs(vcpu), vcpu->kvm);
diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index cf13a8b608dc..3ea1ce11b025 100644
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -55,7 +55,7 @@
#define PT_LEVEL_BITS 9
#define PT_GUEST_DIRTY_SHIFT 9
#define PT_GUEST_ACCESSED_SHIFT 8
- #define PT_HAVE_ACCESSED_DIRTY(mmu) (!(mmu)->cpu_role.base.ad_disabled)
+ #define PT_HAVE_ACCESSED_DIRTY(w) (!(w)->cpu_role.base.ad_disabled)
#define PT_MAX_FULL_LEVELS PT64_ROOT_MAX_LEVEL
#else
#error Invalid PTTYPE value
@@ -109,11 +109,10 @@ static gfn_t gpte_to_gfn_lvl(pt_element_t gpte, int lvl)
static inline void FNAME(protect_clean_gpte)(struct kvm_pagewalk *w, unsigned *access,
unsigned gpte)
{
- struct kvm_mmu __maybe_unused *mmu = container_of(w, struct kvm_mmu, w);
unsigned mask;
/* dirty bit is not supported, so no need to track it */
- if (!PT_HAVE_ACCESSED_DIRTY(mmu))
+ if (!PT_HAVE_ACCESSED_DIRTY(w))
return;
BUILD_BUG_ON(PT_WRITABLE_MASK != ACC_WRITE_MASK);
@@ -125,7 +124,7 @@ static inline void FNAME(protect_clean_gpte)(struct kvm_pagewalk *w, unsigned *a
*access &= mask;
}
-static inline int FNAME(is_present_gpte)(struct kvm_mmu *mmu,
+static inline int FNAME(is_present_gpte)(struct kvm_pagewalk *w,
unsigned long pte)
{
#if PTTYPE != PTTYPE_EPT
@@ -135,7 +134,7 @@ static inline int FNAME(is_present_gpte)(struct kvm_mmu *mmu,
* For EPT, an entry is present if any of bits 2:0 are set.
* With mode-based execute control, bit 10 also indicates presence.
*/
- return pte & (7 | (mmu_has_mbec(mmu) ? VMX_EPT_USER_EXECUTABLE_MASK : 0));
+ return pte & (7 | (w->cpu_role.base.cr4_smep ? VMX_EPT_USER_EXECUTABLE_MASK : 0));
#endif
}
@@ -150,25 +149,25 @@ static bool FNAME(is_bad_mt_xwr)(struct rsvd_bits_validate *rsvd_check, u64 gpte
static bool FNAME(is_rsvd_bits_set)(struct kvm_pagewalk *w, u64 gpte, int level)
{
- struct kvm_mmu *mmu = container_of(w, struct kvm_mmu, w);
-
- return __is_rsvd_bits_set(&mmu->guest_rsvd_check, gpte, level) ||
- FNAME(is_bad_mt_xwr)(&mmu->guest_rsvd_check, gpte);
+ return __is_rsvd_bits_set(&w->guest_rsvd_check, gpte, level) ||
+ FNAME(is_bad_mt_xwr)(&w->guest_rsvd_check, gpte);
}
static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu,
struct kvm_mmu_page *sp, u64 *spte,
u64 gpte)
{
- if (!FNAME(is_present_gpte)(vcpu->arch.mmu, gpte))
+ struct kvm_pagewalk *w = &vcpu->arch.mmu->w;
+
+ if (!FNAME(is_present_gpte)(w, gpte))
goto no_present;
/* Prefetch only accessed entries (unless A/D bits are disabled). */
- if (PT_HAVE_ACCESSED_DIRTY(vcpu->arch.mmu) &&
+ if (PT_HAVE_ACCESSED_DIRTY(w) &&
!(gpte & PT_GUEST_ACCESSED_MASK))
goto no_present;
- if (FNAME(is_rsvd_bits_set)(&vcpu->arch.mmu->w, gpte, PG_LEVEL_4K))
+ if (FNAME(is_rsvd_bits_set)(w, gpte, PG_LEVEL_4K))
goto no_present;
return false;
@@ -213,7 +212,6 @@ static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu,
struct guest_walker *walker,
gpa_t addr, int write_fault)
{
- struct kvm_mmu __maybe_unused *mmu = container_of(w, struct kvm_mmu, w);
unsigned level, index;
pt_element_t pte, orig_pte;
pt_element_t __user *ptep_user;
@@ -221,7 +219,7 @@ static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu,
int ret;
/* dirty/accessed bits are not supported, so no need to update them */
- if (!PT_HAVE_ACCESSED_DIRTY(mmu))
+ if (!PT_HAVE_ACCESSED_DIRTY(w))
return 0;
for (level = walker->max_level; level >= walker->level; --level) {
@@ -285,8 +283,6 @@ static inline unsigned FNAME(gpte_pkeys)(struct kvm_vcpu *vcpu, u64 gpte)
static inline bool FNAME(is_last_gpte)(struct kvm_pagewalk *w,
unsigned int level, unsigned int gpte)
{
- struct kvm_mmu __maybe_unused *mmu = container_of(w, struct kvm_mmu, w);
-
/*
* For EPT and PAE paging (both variants), bit 7 is either reserved at
* all level or indicates a huge page (ignoring CR3/EPTP). In either
@@ -302,7 +298,7 @@ static inline bool FNAME(is_last_gpte)(struct kvm_pagewalk *w,
* is not reserved and does not indicate a large page at this level,
* so clear PT_PAGE_SIZE_MASK in gpte if that is the case.
*/
- gpte &= level - (PT32_ROOT_LEVEL + mmu->cpu_role.ext.cr4_pse);
+ gpte &= level - (PT32_ROOT_LEVEL + w->cpu_role.ext.cr4_pse);
#endif
/*
* PG_LEVEL_4K always terminates. The RHS has bit 7 set
@@ -347,16 +343,16 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
trace_kvm_mmu_pagetable_walk(addr, access);
retry_walk:
- walker->level = mmu->cpu_role.base.level;
+ walker->level = w->cpu_role.base.level;
pte = kvm_mmu_get_guest_pgd(vcpu, w);
- have_ad = PT_HAVE_ACCESSED_DIRTY(mmu);
+ have_ad = PT_HAVE_ACCESSED_DIRTY(w);
#if PTTYPE == 64
walk_nx_mask = 1ULL << PT64_NX_SHIFT;
if (walker->level == PT32E_ROOT_LEVEL) {
pte = w->get_pdptr(vcpu, (addr >> 30) & 3);
trace_kvm_mmu_paging_element(pte, walker->level);
- if (!FNAME(is_present_gpte)(mmu, pte))
+ if (!FNAME(is_present_gpte)(w, pte))
goto error;
--walker->level;
}
@@ -429,7 +425,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
*/
pte_access = pt_access & (pte ^ walk_nx_mask);
- if (unlikely(!FNAME(is_present_gpte)(mmu, pte)))
+ if (unlikely(!FNAME(is_present_gpte)(w, pte)))
goto error;
if (unlikely(FNAME(is_rsvd_bits_set)(w, pte, walker->level))) {
@@ -667,7 +663,7 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
WARN_ON_ONCE(gw->gfn != base_gfn);
direct_access = gw->pte_access;
- top_level = vcpu->arch.mmu->cpu_role.base.level;
+ top_level = vcpu->arch.mmu->w.cpu_role.base.level;
if (top_level == PT32E_ROOT_LEVEL)
top_level = PT32_ROOT_LEVEL;
/*
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 08/19] KVM: x86/mmu: change CPU-role accessor fields to take struct kvm_pagewalk
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
` (6 preceding siblings ...)
2026-06-24 21:30 ` [PATCH 07/19] KVM: x86/mmu: move CPU-related fields " Paolo Bonzini
@ 2026-06-24 21:30 ` Paolo Bonzini
2026-06-24 21:30 ` [PATCH 09/19] KVM: x86/mmu: move remaining permission fields to " Paolo Bonzini
` (11 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:30 UTC (permalink / raw)
To: linux-kernel, kvm
With this change, walk_addr_generic and its callees do not need to use
container_of() anymore.
The next step is removing it from permission_fault() and
kvm_mmu_refresh_passthrough_bits().
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/kvm/mmu/mmu.c | 44 +++++++++++++++++-----------------
arch/x86/kvm/mmu/paging_tmpl.h | 11 ++++-----
2 files changed, 27 insertions(+), 28 deletions(-)
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 92142b389b1a..1258f94d8d78 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -227,9 +227,9 @@ BUILD_MMU_ROLE_REGS_ACCESSOR(efer, lma, EFER_LMA);
* and the vCPU may be incorrect/irrelevant.
*/
#define BUILD_MMU_ROLE_ACCESSOR(base_or_ext, reg, name) \
-static inline bool __maybe_unused is_##reg##_##name(struct kvm_mmu *mmu) \
+static inline bool __maybe_unused is_##reg##_##name(struct kvm_pagewalk *w) \
{ \
- return !!(mmu->w.cpu_role. base_or_ext . reg##_##name); \
+ return !!(w->cpu_role. base_or_ext . reg##_##name); \
}
BUILD_MMU_ROLE_ACCESSOR(base, cr0, wp);
BUILD_MMU_ROLE_ACCESSOR(ext, cr4, pse);
@@ -240,19 +240,19 @@ BUILD_MMU_ROLE_ACCESSOR(ext, cr4, la57);
BUILD_MMU_ROLE_ACCESSOR(base, efer, nx);
BUILD_MMU_ROLE_ACCESSOR(ext, efer, lma);
-static inline bool has_pferr_fetch(struct kvm_mmu *mmu)
+static inline bool has_pferr_fetch(struct kvm_pagewalk *w)
{
- return mmu->w.cpu_role.ext.has_pferr_fetch;
+ return w->cpu_role.ext.has_pferr_fetch;
}
-static inline bool is_cr0_pg(struct kvm_mmu *mmu)
+static inline bool is_cr0_pg(struct kvm_pagewalk *w)
{
- return mmu->w.cpu_role.base.level > 0;
+ return w->cpu_role.base.level > 0;
}
-static inline bool is_cr4_pae(struct kvm_mmu *mmu)
+static inline bool is_cr4_pae(struct kvm_pagewalk *w)
{
- return !mmu->w.cpu_role.base.has_4_byte_gpte;
+ return !w->cpu_role.base.has_4_byte_gpte;
}
static struct kvm_mmu_role_regs vcpu_to_role_regs(struct kvm_vcpu *vcpu)
@@ -5478,9 +5478,9 @@ static void reset_guest_rsvds_bits_mask(struct kvm_vcpu *vcpu,
{
__reset_rsvds_bits_mask(&context->w.guest_rsvd_check,
vcpu->arch.reserved_gpa_bits,
- context->w.cpu_role.base.level, is_efer_nx(context),
+ context->w.cpu_role.base.level, is_efer_nx(&context->w),
guest_cpu_cap_has(vcpu, X86_FEATURE_GBPAGES),
- is_cr4_pse(context),
+ is_cr4_pse(&context->w),
guest_cpuid_is_amd_compatible(vcpu));
}
@@ -5662,10 +5662,10 @@ static void update_permission_bitmask(struct kvm_mmu *mmu, bool tdp, bool ept)
const u16 w = ACC_BITS_MASK(ACC_WRITE_MASK);
const u16 r = ACC_BITS_MASK(ACC_READ_MASK);
- bool cr4_smep = is_cr4_smep(mmu);
- bool cr4_smap = is_cr4_smap(mmu);
- bool cr0_wp = is_cr0_wp(mmu);
- bool efer_nx = is_efer_nx(mmu);
+ bool cr4_smep = is_cr4_smep(&mmu->w);
+ bool cr4_smap = is_cr4_smap(&mmu->w);
+ bool cr0_wp = is_cr0_wp(&mmu->w);
+ bool efer_nx = is_efer_nx(&mmu->w);
/*
* In hardware, page fault error codes are generated (as the name
@@ -5788,10 +5788,10 @@ static void update_pkru_bitmask(struct kvm_mmu *mmu)
mmu->pkru_mask = 0;
- if (!is_cr4_pke(mmu))
+ if (!is_cr4_pke(&mmu->w))
return;
- wp = is_cr0_wp(mmu);
+ wp = is_cr0_wp(&mmu->w);
for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) {
unsigned pfec, pkey_bits;
@@ -5828,7 +5828,7 @@ static void update_pkru_bitmask(struct kvm_mmu *mmu)
static void reset_guest_paging_metadata(struct kvm_vcpu *vcpu,
struct kvm_mmu *mmu)
{
- if (!is_cr0_pg(mmu))
+ if (!is_cr0_pg(&mmu->w))
return;
reset_guest_rsvds_bits_mask(vcpu, mmu);
@@ -5899,7 +5899,7 @@ void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
BUILD_BUG_ON((KVM_MMU_CR0_ROLE_BITS & KVM_POSSIBLE_CR0_GUEST_BITS) != X86_CR0_WP);
BUILD_BUG_ON((KVM_MMU_CR4_ROLE_BITS & KVM_POSSIBLE_CR4_GUEST_BITS));
- if (is_cr0_wp(mmu) == cr0_wp)
+ if (is_cr0_wp(&mmu->w) == cr0_wp)
return;
mmu->w.cpu_role.base.cr0_wp = cr0_wp;
@@ -5974,9 +5974,9 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
context->w.get_pdptr = kvm_pdptr_read;
context->w.get_guest_pgd = get_guest_cr3;
- if (!is_cr0_pg(context))
+ if (!is_cr0_pg(&context->w))
context->w.gva_to_gpa = nonpaging_gva_to_gpa;
- else if (is_cr4_pae(context))
+ else if (is_cr4_pae(&context->w))
context->w.gva_to_gpa = paging64_gva_to_gpa;
else
context->w.gva_to_gpa = paging32_gva_to_gpa;
@@ -5996,9 +5996,9 @@ static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *conte
context->w.cpu_role.as_u64 = cpu_role.as_u64;
context->root_role.word = root_role.word;
- if (!is_cr0_pg(context))
+ if (!is_cr0_pg(&context->w))
nonpaging_init_context(context);
- else if (is_cr4_pae(context))
+ else if (is_cr4_pae(&context->w))
paging64_init_context(context);
else
paging32_init_context(context);
diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index 3ea1ce11b025..e04b646f00d2 100644
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -134,7 +134,7 @@ static inline int FNAME(is_present_gpte)(struct kvm_pagewalk *w,
* For EPT, an entry is present if any of bits 2:0 are set.
* With mode-based execute control, bit 10 also indicates presence.
*/
- return pte & (7 | (w->cpu_role.base.cr4_smep ? VMX_EPT_USER_EXECUTABLE_MASK : 0));
+ return pte & (7 | (is_cr4_smep(w) ? VMX_EPT_USER_EXECUTABLE_MASK : 0));
#endif
}
@@ -316,7 +316,6 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
gpa_t addr, u64 access)
{
- struct kvm_mmu *mmu = container_of(w, struct kvm_mmu, w);
int ret;
pt_element_t pte;
pt_element_t __user *ptep_user;
@@ -488,7 +487,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
error:
errcode |= write_fault | user_fault;
- if (fetch_fault && has_pferr_fetch(mmu))
+ if (fetch_fault && has_pferr_fetch(w))
errcode |= PFERR_FETCH_MASK;
walker->fault.vector = PF_VECTOR;
@@ -543,7 +542,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
* ACC_*_MASK flags!
*/
walker->fault.exit_qualification |= EPT_VIOLATION_RWX_TO_PROT(pte_access);
- if (mmu_has_mbec(mmu))
+ if (is_cr4_smep(w))
walker->fault.exit_qualification |=
EPT_VIOLATION_USER_EXEC_TO_PROT(pte_access);
}
@@ -852,7 +851,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
* otherwise KVM will cache incorrect access information in the SPTE.
*/
if (fault->write && !(walker.pte_access & ACC_WRITE_MASK) &&
- !is_cr0_wp(vcpu->arch.mmu) && !fault->user && fault->slot) {
+ !is_cr0_wp(&vcpu->arch.mmu->w) && !fault->user && fault->slot) {
walker.pte_access |= ACC_WRITE_MASK;
walker.pte_access &= ~ACC_USER_MASK;
@@ -862,7 +861,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
* then we should prevent the kernel from executing it
* if SMEP is enabled.
*/
- if (is_cr4_smep(vcpu->arch.mmu))
+ if (is_cr4_smep(&vcpu->arch.mmu->w))
walker.pte_access &= ~ACC_EXEC_MASK;
}
#endif
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 09/19] KVM: x86/mmu: move remaining permission fields to struct kvm_pagewalk
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
` (7 preceding siblings ...)
2026-06-24 21:30 ` [PATCH 08/19] KVM: x86/mmu: change CPU-role accessor fields to take " Paolo Bonzini
@ 2026-06-24 21:30 ` Paolo Bonzini
2026-06-24 21:30 ` [PATCH 10/19] KVM: x86/mmu: pass struct kvm_pagewalk to kvm_mmu_invalidate_addr Paolo Bonzini
` (10 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:30 UTC (permalink / raw)
To: linux-kernel, kvm
As promised, this removes the remaining instances of
container_of(w, struct kvm_mmu, w).
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/include/asm/kvm_host.h | 30 ++++++++--------
arch/x86/kvm/mmu.h | 13 +++----
arch/x86/kvm/mmu/mmu.c | 62 ++++++++++++++++-----------------
3 files changed, 51 insertions(+), 54 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 93b7e528d3c6..bf1151c91372 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -527,6 +527,21 @@ struct kvm_pagewalk {
struct x86_exception *exception);
union kvm_cpu_role cpu_role;
struct rsvd_bits_validate guest_rsvd_check;
+
+ /*
+ * The pkru_mask indicates if protection key checks are needed. It
+ * consists of 16 domains indexed by page fault error code bits [4:1],
+ * with PFEC.RSVD replaced by ACC_USER_MASK from the page tables.
+ * Each domain has 2 bits which are ANDed with AD and WD from PKRU.
+ */
+ u32 pkru_mask;
+
+ /*
+ * Bitmap; bit set = permission fault
+ * Array index: page fault error code [4:1]
+ * Bit index: pte permissions in ACC_* format
+ */
+ u16 permissions[16];
};
struct kvm_mmu {
@@ -539,23 +554,8 @@ struct kvm_mmu {
hpa_t mirror_root_hpa;
union kvm_mmu_page_role root_role;
- /*
- * The pkru_mask indicates if protection key checks are needed. It
- * consists of 16 domains indexed by page fault error code bits [4:1],
- * with PFEC.RSVD replaced by ACC_USER_MASK from the page tables.
- * Each domain has 2 bits which are ANDed with AD and WD from PKRU.
- */
- u32 pkru_mask;
-
struct kvm_mmu_root_info prev_roots[KVM_MMU_NUM_PREV_ROOTS];
- /*
- * Bitmap; bit set = permission fault
- * Byte index: page fault error code [4:1]
- * Bit index: pte permissions in ACC_* format
- */
- u16 permissions[16];
-
u64 *pae_root;
u64 *pml4_root;
u64 *pml5_root;
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index 9384e626934b..c404ed9dcff2 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -163,7 +163,7 @@ bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu);
int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
u64 fault_address, char *insn, int insn_len);
void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
- struct kvm_mmu *mmu);
+ struct kvm_pagewalk *pw);
int kvm_mmu_load(struct kvm_vcpu *vcpu);
void kvm_mmu_unload(struct kvm_vcpu *vcpu);
@@ -260,8 +260,7 @@ static inline void kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
if (!tdp_enabled || w == &vcpu->arch.guest_mmu.w)
return;
- __kvm_mmu_refresh_passthrough_bits(vcpu,
- container_of(w, struct kvm_mmu, w));
+ __kvm_mmu_refresh_passthrough_bits(vcpu, w);
}
/*
@@ -276,8 +275,6 @@ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
unsigned pte_access, unsigned pte_pkey,
u64 access)
{
- struct kvm_mmu *mmu = container_of(w, struct kvm_mmu, w);
-
/* strip nested paging fault error codes */
unsigned int pfec = access;
unsigned long rflags = kvm_x86_call(get_rflags)(vcpu);
@@ -302,10 +299,10 @@ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
kvm_mmu_refresh_passthrough_bits(vcpu, w);
- fault = (mmu->permissions[index] >> pte_access) & 1;
+ fault = (w->permissions[index] >> pte_access) & 1;
WARN_ON_ONCE(pfec & (PFERR_PK_MASK | PFERR_SS_MASK | PFERR_RSVD_MASK));
- if (unlikely(mmu->pkru_mask)) {
+ if (unlikely(w->pkru_mask)) {
u32 pkru_bits, offset;
/*
@@ -319,7 +316,7 @@ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
/* clear present bit, replace PFEC.RSVD with ACC_USER_MASK. */
offset = (pfec & ~1) | ((pte_access & PT_USER_MASK) ? PFERR_RSVD_MASK : 0);
- pkru_bits &= mmu->pkru_mask >> offset;
+ pkru_bits &= w->pkru_mask >> offset;
errcode |= -pkru_bits & PFERR_PK_MASK;
fault |= (pkru_bits != 0);
}
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 1258f94d8d78..29f2948bd38d 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -5474,13 +5474,13 @@ static void __reset_rsvds_bits_mask(struct rsvd_bits_validate *rsvd_check,
}
static void reset_guest_rsvds_bits_mask(struct kvm_vcpu *vcpu,
- struct kvm_mmu *context)
+ struct kvm_pagewalk *w)
{
- __reset_rsvds_bits_mask(&context->w.guest_rsvd_check,
+ __reset_rsvds_bits_mask(&w->guest_rsvd_check,
vcpu->arch.reserved_gpa_bits,
- context->w.cpu_role.base.level, is_efer_nx(&context->w),
+ w->cpu_role.base.level, is_efer_nx(w),
guest_cpu_cap_has(vcpu, X86_FEATURE_GBPAGES),
- is_cr4_pse(&context->w),
+ is_cr4_pse(w),
guest_cpuid_is_amd_compatible(vcpu));
}
@@ -5655,17 +5655,17 @@ reset_ept_shadow_zero_bits_mask(struct kvm_mmu *context, bool execonly)
(14 & (access) ? 1 << 14 : 0) | \
(15 & (access) ? 1 << 15 : 0))
-static void update_permission_bitmask(struct kvm_mmu *mmu, bool tdp, bool ept)
+static void update_permission_bitmask(struct kvm_pagewalk *pw, bool tdp, bool ept)
{
unsigned index;
const u16 w = ACC_BITS_MASK(ACC_WRITE_MASK);
const u16 r = ACC_BITS_MASK(ACC_READ_MASK);
- bool cr4_smep = is_cr4_smep(&mmu->w);
- bool cr4_smap = is_cr4_smap(&mmu->w);
- bool cr0_wp = is_cr0_wp(&mmu->w);
- bool efer_nx = is_efer_nx(&mmu->w);
+ bool cr4_smep = is_cr4_smep(pw);
+ bool cr4_smap = is_cr4_smap(pw);
+ bool cr0_wp = is_cr0_wp(pw);
+ bool efer_nx = is_efer_nx(pw);
/*
* In hardware, page fault error codes are generated (as the name
@@ -5679,7 +5679,7 @@ static void update_permission_bitmask(struct kvm_mmu *mmu, bool tdp, bool ept)
* permission_fault() to indicate accesses that are *not* subject to
* SMAP restrictions.
*/
- for (index = 0; index < ARRAY_SIZE(mmu->permissions); ++index) {
+ for (index = 0; index < ARRAY_SIZE(pw->permissions); ++index) {
unsigned pfec = index << 1;
/*
@@ -5753,7 +5753,7 @@ static void update_permission_bitmask(struct kvm_mmu *mmu, bool tdp, bool ept)
smapf = (pfec & (PFERR_RSVD_MASK|PFERR_FETCH_MASK)) ? 0 : kf;
}
- mmu->permissions[index] = ff | uf | wf | rf | smapf;
+ pw->permissions[index] = ff | uf | wf | rf | smapf;
}
}
@@ -5781,19 +5781,19 @@ static void update_permission_bitmask(struct kvm_mmu *mmu, bool tdp, bool ept)
* away both AD and WD. For all reads or if the last condition holds, WD
* only will be masked away.
*/
-static void update_pkru_bitmask(struct kvm_mmu *mmu)
+static void update_pkru_bitmask(struct kvm_pagewalk *w)
{
unsigned bit;
bool wp;
- mmu->pkru_mask = 0;
+ w->pkru_mask = 0;
- if (!is_cr4_pke(&mmu->w))
+ if (!is_cr4_pke(w))
return;
- wp = is_cr0_wp(&mmu->w);
+ wp = is_cr0_wp(w);
- for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) {
+ for (bit = 0; bit < ARRAY_SIZE(w->permissions); ++bit) {
unsigned pfec, pkey_bits;
bool check_pkey, check_write, ff, uf, wf, pte_user;
@@ -5821,19 +5821,19 @@ static void update_pkru_bitmask(struct kvm_mmu *mmu)
/* PKRU.WD stops write access. */
pkey_bits |= (!!check_write) << 1;
- mmu->pkru_mask |= (pkey_bits & 3) << pfec;
+ w->pkru_mask |= (pkey_bits & 3) << pfec;
}
}
static void reset_guest_paging_metadata(struct kvm_vcpu *vcpu,
- struct kvm_mmu *mmu)
+ struct kvm_pagewalk *w)
{
- if (!is_cr0_pg(&mmu->w))
+ if (!is_cr0_pg(w))
return;
- reset_guest_rsvds_bits_mask(vcpu, mmu);
- update_permission_bitmask(mmu, mmu == &vcpu->arch.guest_mmu, false);
- update_pkru_bitmask(mmu);
+ reset_guest_rsvds_bits_mask(vcpu, w);
+ update_permission_bitmask(w, w == &vcpu->arch.guest_mmu.w, false);
+ update_pkru_bitmask(w);
}
static void paging64_init_context(struct kvm_mmu *context)
@@ -5892,18 +5892,18 @@ static union kvm_cpu_role kvm_calc_cpu_role(struct kvm_vcpu *vcpu,
}
void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
- struct kvm_mmu *mmu)
+ struct kvm_pagewalk *w)
{
const bool cr0_wp = kvm_is_cr0_bit_set(vcpu, X86_CR0_WP);
BUILD_BUG_ON((KVM_MMU_CR0_ROLE_BITS & KVM_POSSIBLE_CR0_GUEST_BITS) != X86_CR0_WP);
BUILD_BUG_ON((KVM_MMU_CR4_ROLE_BITS & KVM_POSSIBLE_CR4_GUEST_BITS));
- if (is_cr0_wp(&mmu->w) == cr0_wp)
+ if (is_cr0_wp(w) == cr0_wp)
return;
- mmu->w.cpu_role.base.cr0_wp = cr0_wp;
- reset_guest_paging_metadata(vcpu, mmu);
+ w->cpu_role.base.cr0_wp = cr0_wp;
+ reset_guest_paging_metadata(vcpu, w);
}
static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu)
@@ -5981,7 +5981,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
else
context->w.gva_to_gpa = paging32_gva_to_gpa;
- reset_guest_paging_metadata(vcpu, context);
+ reset_guest_paging_metadata(vcpu, &context->w);
reset_tdp_shadow_zero_bits_mask(context);
}
@@ -6003,7 +6003,7 @@ static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *conte
else
paging32_init_context(context);
- reset_guest_paging_metadata(vcpu, context);
+ reset_guest_paging_metadata(vcpu, &context->w);
reset_shadow_zero_bits_mask(vcpu, context);
}
@@ -6104,8 +6104,8 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
context->w.gva_to_gpa = ept_gva_to_gpa;
context->sync_spte = ept_sync_spte;
- update_permission_bitmask(context, true, true);
- context->pkru_mask = 0;
+ update_permission_bitmask(&context->w, true, true);
+ context->w.pkru_mask = 0;
reset_rsvds_bits_mask_ept(vcpu, context, execonly, huge_page_level);
reset_ept_shadow_zero_bits_mask(context, execonly);
}
@@ -6162,7 +6162,7 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu,
else
g_context->w.gva_to_gpa = paging32_gva_to_gpa;
- reset_guest_paging_metadata(vcpu, g_context);
+ reset_guest_paging_metadata(vcpu, &g_context->w);
}
void kvm_init_mmu(struct kvm_vcpu *vcpu)
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 10/19] KVM: x86/mmu: pass struct kvm_pagewalk to kvm_mmu_invalidate_addr
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
` (8 preceding siblings ...)
2026-06-24 21:30 ` [PATCH 09/19] KVM: x86/mmu: move remaining permission fields to " Paolo Bonzini
@ 2026-06-24 21:30 ` Paolo Bonzini
2026-06-24 21:30 ` [PATCH 11/19] KVM: x86/mmu: change walk_mmu to struct kvm_pagewalk Paolo Bonzini
` (9 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:30 UTC (permalink / raw)
To: linux-kernel, kvm
kvm_mmu_invalidate_addr only needs to know if what's being invalidated
is a GVA or GPA. This will ultimately be represented by two different
kvm_pagewalk structs, so adjust the type of the parameter.
For now the GVA case is represented by both root_mmu and nested_mmu.
Since nested_mmu never has a sync_spte callback, it would exit at its
check; but really nested_mmu should not be a kvm_mmu in the first place
and the container_of() would be bogus, so introduce a separate check
for whether the invalidation is happening for a nested GVA. In that
case there's nothing needed beyond kvm_x86_call(flush_tlb_gva).
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/kvm/mmu.h | 2 +-
arch/x86/kvm/mmu/mmu.c | 12 ++++++++----
arch/x86/kvm/vmx/nested.c | 2 +-
arch/x86/kvm/x86.c | 2 +-
4 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index c404ed9dcff2..4f48e2ca2dac 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -151,7 +151,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
void *insn, int insn_len);
void kvm_mmu_print_sptes(struct kvm_vcpu *vcpu, gpa_t gpa, const char *msg);
void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
-void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
+void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
u64 addr, unsigned long roots);
void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid);
void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd);
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 29f2948bd38d..30774b562fa1 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -6685,22 +6685,26 @@ static void __kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu
write_unlock(&vcpu->kvm->mmu_lock);
}
-void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
+void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
u64 addr, unsigned long roots)
{
+ struct kvm_mmu *mmu;
int i;
WARN_ON_ONCE(roots & ~KVM_MMU_ROOTS_ALL);
/* It's actually a GPA for vcpu->arch.guest_mmu. */
- if (mmu != &vcpu->arch.guest_mmu) {
+ if (w != &vcpu->arch.guest_mmu.w) {
/* INVLPG on a non-canonical address is a NOP according to the SDM. */
if (is_noncanonical_invlpg_address(addr, vcpu))
return;
kvm_x86_call(flush_tlb_gva)(vcpu, addr);
+ if (w == &vcpu->arch.nested_mmu.w)
+ return;
}
+ mmu = container_of(w, struct kvm_mmu, w);
if (!mmu->sync_spte)
return;
@@ -6726,7 +6730,7 @@ void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
* be synced when switching to that new cr3, so nothing needs to be
* done here for them.
*/
- kvm_mmu_invalidate_addr(vcpu, vcpu->arch.walk_mmu, gva, KVM_MMU_ROOTS_ALL);
+ kvm_mmu_invalidate_addr(vcpu, &vcpu->arch.walk_mmu->w, gva, KVM_MMU_ROOTS_ALL);
++vcpu->stat.invlpg;
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_invlpg);
@@ -6748,7 +6752,7 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
}
if (roots)
- kvm_mmu_invalidate_addr(vcpu, mmu, gva, roots);
+ kvm_mmu_invalidate_addr(vcpu, &mmu->w, gva, roots);
++vcpu->stat.invlpg;
/*
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 466f838a05bc..6c4f38cc9896 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -408,7 +408,7 @@ static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp,
roots |= KVM_MMU_ROOT_PREVIOUS(i);
}
if (roots)
- kvm_mmu_invalidate_addr(vcpu, vcpu->arch.mmu, addr, roots);
+ kvm_mmu_invalidate_addr(vcpu, &vcpu->arch.guest_mmu.w, addr, roots);
}
static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 484ed409e420..0f76e52e2695 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -594,7 +594,7 @@ void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
*/
if ((fault->error_code & PFERR_PRESENT_MASK) &&
!(fault->error_code & PFERR_RSVD_MASK))
- kvm_mmu_invalidate_addr(vcpu, fault_mmu, fault->address,
+ kvm_mmu_invalidate_addr(vcpu, &fault_mmu->w, fault->address,
KVM_MMU_ROOT_CURRENT);
fault_mmu->w.inject_page_fault(vcpu, fault, from_hardware);
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 11/19] KVM: x86/mmu: change walk_mmu to struct kvm_pagewalk
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
` (9 preceding siblings ...)
2026-06-24 21:30 ` [PATCH 10/19] KVM: x86/mmu: pass struct kvm_pagewalk to kvm_mmu_invalidate_addr Paolo Bonzini
@ 2026-06-24 21:30 ` Paolo Bonzini
2026-06-24 21:30 ` [PATCH 12/19] KVM: x86/mmu: change nested_mmu.w to ngva_walk Paolo Bonzini
` (8 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:30 UTC (permalink / raw)
To: linux-kernel, kvm
Now that walk_mmu is only accessed for its "w" member, store
directly the pointer to it. This also means that nested_mmu
is only accessed for its "w" member.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/include/asm/kvm_host.h | 2 +-
arch/x86/kvm/hyperv.c | 2 +-
arch/x86/kvm/mmu/mmu.c | 4 ++--
arch/x86/kvm/mmu/paging_tmpl.h | 4 ++--
arch/x86/kvm/regs.c | 7 ++++---
arch/x86/kvm/svm/nested.c | 4 ++--
arch/x86/kvm/vmx/nested.c | 4 ++--
arch/x86/kvm/x86.c | 37 +++++++++++++++++----------------
8 files changed, 33 insertions(+), 31 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index bf1151c91372..a64deb5c05eb 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -923,7 +923,7 @@ struct kvm_vcpu_arch {
* Pointer to the mmu context currently used for
* gva_to_gpa translations.
*/
- struct kvm_mmu *walk_mmu;
+ struct kvm_pagewalk *gva_walk;
u64 pdptrs[4]; /* pae */
diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c
index e4a0ca0f9fd4..51d812babe73 100644
--- a/arch/x86/kvm/hyperv.c
+++ b/arch/x86/kvm/hyperv.c
@@ -2046,7 +2046,7 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
* read with kvm_read_guest().
*/
if (!hc->fast) {
- hc->ingpa = kvm_translate_gpa(vcpu, &vcpu->arch.walk_mmu->w, hc->ingpa,
+ hc->ingpa = kvm_translate_gpa(vcpu, vcpu->arch.gva_walk, hc->ingpa,
PFERR_GUEST_FINAL_MASK, NULL, 0);
if (unlikely(hc->ingpa == INVALID_GPA))
return HV_STATUS_INVALID_HYPERCALL_INPUT;
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 30774b562fa1..8ed9876cf3b8 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -6730,7 +6730,7 @@ void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
* be synced when switching to that new cr3, so nothing needs to be
* done here for them.
*/
- kvm_mmu_invalidate_addr(vcpu, &vcpu->arch.walk_mmu->w, gva, KVM_MMU_ROOTS_ALL);
+ kvm_mmu_invalidate_addr(vcpu, vcpu->arch.gva_walk, gva, KVM_MMU_ROOTS_ALL);
++vcpu->stat.invlpg;
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_invlpg);
@@ -6867,7 +6867,7 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu)
vcpu->arch.mmu_shadow_page_cache.gfp_zero = __GFP_ZERO;
vcpu->arch.mmu = &vcpu->arch.root_mmu;
- vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
+ vcpu->arch.gva_walk = &vcpu->arch.root_mmu.w;
ret = __kvm_mmu_create(vcpu, &vcpu->arch.guest_mmu);
if (ret)
diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index e04b646f00d2..9cfae71cd3e6 100644
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -548,7 +548,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
}
#endif
walker->fault.address = addr;
- walker->fault.nested_page_fault = w != &vcpu->arch.walk_mmu->w;
+ walker->fault.nested_page_fault = w != vcpu->arch.gva_walk;
walker->fault.async_page_fault = false;
#if PTTYPE != PTTYPE_EPT
@@ -906,7 +906,7 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
#ifndef CONFIG_X86_64
/* A 64-bit GVA should be impossible on 32-bit KVM. */
- WARN_ON_ONCE((addr >> 32) && w == &vcpu->arch.walk_mmu->w);
+ WARN_ON_ONCE((addr >> 32) && w == vcpu->arch.gva_walk);
#endif
r = FNAME(walk_addr_generic)(&walker, vcpu, w, addr, access);
diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c
index 3b3515fd77e6..02adaa4ef64e 100644
--- a/arch/x86/kvm/regs.c
+++ b/arch/x86/kvm/regs.c
@@ -154,7 +154,7 @@ static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
*/
int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
{
- struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
+ struct kvm_pagewalk *w = vcpu->arch.gva_walk;
gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
gpa_t real_gpa;
int i;
@@ -165,7 +165,7 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
* If the MMU is nested, CR3 holds an L2 GPA and needs to be translated
* to an L1 GPA.
*/
- real_gpa = kvm_translate_gpa(vcpu, &mmu->w, gfn_to_gpa(pdpt_gfn),
+ real_gpa = kvm_translate_gpa(vcpu, w, gfn_to_gpa(pdpt_gfn),
PFERR_USER_MASK | PFERR_WRITE_MASK |
PFERR_GUEST_PAGE_MASK, NULL, 0);
if (real_gpa == INVALID_GPA)
@@ -189,7 +189,8 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
* Shadow page roots need to be reconstructed instead.
*/
if (!tdp_enabled && memcmp(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs)))
- kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT);
+ kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.root_mmu,
+ KVM_MMU_ROOT_CURRENT);
memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs));
kvm_register_mark_dirty(vcpu, VCPU_REG_PDPTR);
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index 55eace8aa2c8..188e4b06a279 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -118,13 +118,13 @@ static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
vcpu->arch.mmu->w.get_pdptr = nested_svm_get_tdp_pdptr;
vcpu->arch.mmu->w.inject_page_fault = nested_svm_inject_npf_exit;
- vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu;
+ vcpu->arch.gva_walk = &vcpu->arch.nested_mmu.w;
}
static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
{
vcpu->arch.mmu = &vcpu->arch.root_mmu;
- vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
+ vcpu->arch.gva_walk = &vcpu->arch.root_mmu.w;
}
static bool nested_vmcb_needs_vls_intercept(struct vcpu_svm *svm)
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 6c4f38cc9896..5adc6a7c6af4 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -517,13 +517,13 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
vcpu->arch.mmu->w.inject_page_fault = nested_ept_inject_page_fault;
- vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu;
+ vcpu->arch.gva_walk = &vcpu->arch.nested_mmu.w;
}
static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
{
vcpu->arch.mmu = &vcpu->arch.root_mmu;
- vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
+ vcpu->arch.gva_walk = &vcpu->arch.root_mmu.w;
}
static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 0f76e52e2695..43625cc9e934 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -582,11 +582,12 @@ void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
struct x86_exception *fault,
bool from_hardware)
{
- struct kvm_mmu *fault_mmu;
+ struct kvm_pagewalk *fault_walk;
+
WARN_ON_ONCE(fault->vector != PF_VECTOR);
- fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :
- vcpu->arch.walk_mmu;
+ fault_walk = fault->nested_page_fault ? &vcpu->arch.mmu->w :
+ vcpu->arch.gva_walk;
/*
* Invalidate the TLB entry for the faulting address, if it exists,
@@ -594,10 +595,10 @@ void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
*/
if ((fault->error_code & PFERR_PRESENT_MASK) &&
!(fault->error_code & PFERR_RSVD_MASK))
- kvm_mmu_invalidate_addr(vcpu, &fault_mmu->w, fault->address,
+ kvm_mmu_invalidate_addr(vcpu, fault_walk, fault->address,
KVM_MMU_ROOT_CURRENT);
- fault_mmu->w.inject_page_fault(vcpu, fault, from_hardware);
+ fault_walk->inject_page_fault(vcpu, fault, from_hardware);
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_inject_emulated_page_fault);
@@ -4768,7 +4769,7 @@ static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
struct x86_exception *exception)
{
- struct kvm_pagewalk *gva_walk = &vcpu->arch.walk_mmu->w;
+ struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, exception);
@@ -4778,7 +4779,7 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_gva_to_gpa_read);
gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
struct x86_exception *exception)
{
- struct kvm_pagewalk *gva_walk = &vcpu->arch.walk_mmu->w;
+ struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
access |= PFERR_WRITE_MASK;
@@ -4790,7 +4791,7 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_gva_to_gpa_write);
gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
struct x86_exception *exception)
{
- struct kvm_pagewalk *gva_walk = &vcpu->arch.walk_mmu->w;
+ struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, 0, exception);
}
@@ -4799,7 +4800,7 @@ static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
struct kvm_vcpu *vcpu, u64 access,
struct x86_exception *exception)
{
- struct kvm_pagewalk *gva_walk = &vcpu->arch.walk_mmu->w;
+ struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
void *data = val;
int r = X86EMUL_CONTINUE;
@@ -4832,7 +4833,7 @@ static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
struct x86_exception *exception)
{
struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
- struct kvm_pagewalk *gva_walk = &vcpu->arch.walk_mmu->w;
+ struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
unsigned offset;
int ret;
@@ -4891,7 +4892,7 @@ static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes
struct kvm_vcpu *vcpu, u64 access,
struct x86_exception *exception)
{
- struct kvm_pagewalk *gva_walk = &vcpu->arch.walk_mmu->w;
+ struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
void *data = val;
int r = X86EMUL_CONTINUE;
@@ -4997,7 +4998,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
gpa_t *gpa, struct x86_exception *exception,
bool write)
{
- struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
+ struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
u64 access = ((kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0)
| (write ? PFERR_WRITE_MASK : 0);
@@ -5007,7 +5008,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
* shadow page table for L2 guest.
*/
if (vcpu_match_mmio_gva(vcpu, gva) && (!is_paging(vcpu) ||
- !permission_fault(vcpu, &vcpu->arch.walk_mmu->w,
+ !permission_fault(vcpu, gva_walk,
vcpu->arch.mmio_access, 0, access))) {
*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
(gva & (PAGE_SIZE - 1));
@@ -5015,7 +5016,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
return 1;
}
- *gpa = mmu->w.gva_to_gpa(vcpu, &mmu->w, gva, access, exception);
+ *gpa = gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, exception);
if (*gpa == INVALID_GPA)
return -1;
@@ -10600,15 +10601,15 @@ void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end)
void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code)
{
- struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
+ struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
struct x86_exception fault;
u64 access = error_code &
(PFERR_WRITE_MASK | PFERR_FETCH_MASK | PFERR_USER_MASK);
if (!(error_code & PFERR_PRESENT_MASK) ||
- mmu->w.gva_to_gpa(vcpu, &mmu->w, gva, access, &fault) != INVALID_GPA) {
+ gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, &fault) != INVALID_GPA) {
/*
- * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page
+ * If gva_walk->gva_to_gpa succeeded, the page
* tables probably do not match the TLB. Just proceed
* with the error code that the processor gave.
*/
@@ -10619,7 +10620,7 @@ void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_c
fault.address = gva;
fault.async_page_fault = false;
}
- vcpu->arch.walk_mmu->w.inject_page_fault(vcpu, &fault, true);
+ gva_walk->inject_page_fault(vcpu, &fault, true);
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_fixup_and_inject_pf_error);
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 12/19] KVM: x86/mmu: change nested_mmu.w to ngva_walk
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
` (10 preceding siblings ...)
2026-06-24 21:30 ` [PATCH 11/19] KVM: x86/mmu: change walk_mmu to struct kvm_pagewalk Paolo Bonzini
@ 2026-06-24 21:30 ` Paolo Bonzini
2026-06-24 21:30 ` [PATCH 13/19] KVM: x86/mmu: pull struct kvm_pagewalk out of struct kvm_mmu Paolo Bonzini
` (7 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:30 UTC (permalink / raw)
To: linux-kernel, kvm
nested_mmu is now only used for its w member. Rename it,
and change its type.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/include/asm/kvm_host.h | 5 ++--
arch/x86/kvm/mmu.h | 6 ++---
arch/x86/kvm/mmu/mmu.c | 41 ++++++++++++++-------------------
arch/x86/kvm/svm/nested.c | 2 +-
arch/x86/kvm/vmx/nested.c | 2 +-
5 files changed, 24 insertions(+), 32 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index a64deb5c05eb..25288d73cce7 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -917,11 +917,10 @@ struct kvm_vcpu_arch {
* walking and not for faulting since we never handle l2 page faults on
* the host.
*/
- struct kvm_mmu nested_mmu;
+ struct kvm_pagewalk ngva_walk;
/*
- * Pointer to the mmu context currently used for
- * gva_to_gpa translations.
+ * Pagewalk context used for gva_to_gpa translations.
*/
struct kvm_pagewalk *gva_walk;
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index 4f48e2ca2dac..2610b996e144 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -254,8 +254,8 @@ static inline void kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
* be stale. Refresh CR0.WP and the metadata on-demand when checking
* for permission faults. Exempt nested MMUs, i.e. MMUs for shadowing
* nEPT and nNPT, as CR0.WP is ignored in both cases. Note, KVM does
- * need to refresh nested_mmu, a.k.a. the walker used to translate L2
- * GVAs to GPAs, as that "MMU" needs to honor L2's CR0.WP.
+ * need to refresh ngva_walk, a.k.a. the walker used to translate L2
+ * GVAs to GPAs, so as to honor L2's CR0.WP.
*/
if (!tdp_enabled || w == &vcpu->arch.guest_mmu.w)
return;
@@ -382,7 +382,7 @@ static inline gpa_t kvm_translate_gpa(struct kvm_vcpu *vcpu,
struct x86_exception *exception,
u64 pte_access)
{
- if (w != &vcpu->arch.nested_mmu.w)
+ if (w != &vcpu->arch.ngva_walk)
return gpa;
return kvm_x86_ops.nested_ops->translate_nested_gpa(vcpu, gpa, access,
exception,
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 8ed9876cf3b8..8b5ffd78565b 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -6126,43 +6126,37 @@ static void init_kvm_softmmu(struct kvm_vcpu *vcpu,
context->w.get_guest_pgd = get_guest_cr3;
}
-static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu,
+static void init_kvm_ngva_walk(struct kvm_vcpu *vcpu,
union kvm_cpu_role new_mode)
{
- struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;
+ struct kvm_pagewalk *g_context = &vcpu->arch.ngva_walk;
- if (new_mode.as_u64 == g_context->w.cpu_role.as_u64)
+ if (new_mode.as_u64 == g_context->cpu_role.as_u64)
return;
- g_context->w.cpu_role.as_u64 = new_mode.as_u64;
- g_context->w.inject_page_fault = kvm_inject_page_fault;
- g_context->w.get_pdptr = kvm_pdptr_read;
- g_context->w.get_guest_pgd = get_guest_cr3;
-
- /*
- * L2 page tables are never shadowed, so there is no need to sync
- * SPTEs.
- */
- g_context->sync_spte = NULL;
+ g_context->cpu_role.as_u64 = new_mode.as_u64;
+ g_context->inject_page_fault = kvm_inject_page_fault;
+ g_context->get_pdptr = kvm_pdptr_read;
+ g_context->get_guest_pgd = get_guest_cr3;
/*
* Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using
* L1's nested page tables (e.g. EPT12). The nested translation
- * of l2_gva to l1_gpa is done by arch.nested_mmu.gva_to_gpa using
+ * of l2_gva to l1_gpa is done by arch.ngva_walk.gva_to_gpa using
* L2's page tables as the first level of translation and L1's
* nested page tables as the second level of translation. Basically
- * the gva_to_gpa functions between mmu and nested_mmu are swapped.
+ * the gva_to_gpa functions between mmu and ngva_walk are swapped.
*/
if (!is_paging(vcpu))
- g_context->w.gva_to_gpa = nonpaging_gva_to_gpa;
+ g_context->gva_to_gpa = nonpaging_gva_to_gpa;
else if (is_long_mode(vcpu))
- g_context->w.gva_to_gpa = paging64_gva_to_gpa;
+ g_context->gva_to_gpa = paging64_gva_to_gpa;
else if (is_pae(vcpu))
- g_context->w.gva_to_gpa = paging64_gva_to_gpa;
+ g_context->gva_to_gpa = paging64_gva_to_gpa;
else
- g_context->w.gva_to_gpa = paging32_gva_to_gpa;
+ g_context->gva_to_gpa = paging32_gva_to_gpa;
- reset_guest_paging_metadata(vcpu, &g_context->w);
+ reset_guest_paging_metadata(vcpu, g_context);
}
void kvm_init_mmu(struct kvm_vcpu *vcpu)
@@ -6171,7 +6165,7 @@ void kvm_init_mmu(struct kvm_vcpu *vcpu)
union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, ®s);
if (mmu_is_nested(vcpu))
- init_kvm_nested_mmu(vcpu, cpu_role);
+ init_kvm_ngva_walk(vcpu, cpu_role);
else if (tdp_enabled)
init_kvm_tdp_mmu(vcpu, cpu_role);
else
@@ -6195,10 +6189,9 @@ void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu)
*/
vcpu->arch.root_mmu.root_role.invalid = 1;
vcpu->arch.guest_mmu.root_role.invalid = 1;
- vcpu->arch.nested_mmu.root_role.invalid = 1;
vcpu->arch.root_mmu.w.cpu_role.ext.valid = 0;
vcpu->arch.guest_mmu.w.cpu_role.ext.valid = 0;
- vcpu->arch.nested_mmu.w.cpu_role.ext.valid = 0;
+ vcpu->arch.ngva_walk.cpu_role.ext.valid = 0;
kvm_mmu_reset_context(vcpu);
KVM_BUG_ON(!kvm_can_set_cpuid_and_feature_msrs(vcpu), vcpu->kvm);
@@ -6700,7 +6693,7 @@ void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
return;
kvm_x86_call(flush_tlb_gva)(vcpu, addr);
- if (w == &vcpu->arch.nested_mmu.w)
+ if (w == &vcpu->arch.ngva_walk)
return;
}
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index 188e4b06a279..713b2508b8ca 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -118,7 +118,7 @@ static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
vcpu->arch.mmu->w.get_pdptr = nested_svm_get_tdp_pdptr;
vcpu->arch.mmu->w.inject_page_fault = nested_svm_inject_npf_exit;
- vcpu->arch.gva_walk = &vcpu->arch.nested_mmu.w;
+ vcpu->arch.gva_walk = &vcpu->arch.ngva_walk;
}
static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 5adc6a7c6af4..345ee3323a93 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -517,7 +517,7 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
vcpu->arch.mmu->w.inject_page_fault = nested_ept_inject_page_fault;
- vcpu->arch.gva_walk = &vcpu->arch.nested_mmu.w;
+ vcpu->arch.gva_walk = &vcpu->arch.ngva_walk;
}
static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 13/19] KVM: x86/mmu: pull struct kvm_pagewalk out of struct kvm_mmu
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
` (11 preceding siblings ...)
2026-06-24 21:30 ` [PATCH 12/19] KVM: x86/mmu: change nested_mmu.w to ngva_walk Paolo Bonzini
@ 2026-06-24 21:30 ` Paolo Bonzini
2026-06-24 21:30 ` [PATCH 14/19] KVM: x86/mmu: unify root_gva_walk and ngva_walk Paolo Bonzini
` (6 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:30 UTC (permalink / raw)
To: linux-kernel, kvm
Replace kvm_mmu's w field with a pointer to an external instance of
struct kvm_pagewalk. This is the first step towards using a single
kvm_pagewalk struct for all GVA walks, whether nested or not.
With this patch, non-MMU code basically does not use kvm_mmu anymore:
it does care about page walks, but it funnels (almost) all interactions
with the TLB to mmu.c.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/include/asm/kvm_host.h | 8 ++-
arch/x86/kvm/mmu.h | 2 +-
arch/x86/kvm/mmu/mmu.c | 113 ++++++++++++++++++--------------
arch/x86/kvm/mmu/paging_tmpl.h | 14 ++--
arch/x86/kvm/svm/nested.c | 11 ++--
arch/x86/kvm/vmx/nested.c | 13 ++--
arch/x86/kvm/x86.c | 2 +-
7 files changed, 88 insertions(+), 75 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 25288d73cce7..8b9cf364c9f6 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -545,11 +545,11 @@ struct kvm_pagewalk {
};
struct kvm_mmu {
- struct kvm_pagewalk w;
-
int (*page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
int (*sync_spte)(struct kvm_vcpu *vcpu,
struct kvm_mmu_page *sp, int i);
+ struct kvm_pagewalk *w;
+
struct kvm_mmu_root_info root;
hpa_t mirror_root_hpa;
union kvm_mmu_page_role root_role;
@@ -905,9 +905,11 @@ struct kvm_vcpu_arch {
/* Non-nested MMU for L1 */
struct kvm_mmu root_mmu;
+ struct kvm_pagewalk root_gva_walk;
- /* L1 MMU when running nested */
+ /* L1 TDP when running nested */
struct kvm_mmu guest_mmu;
+ struct kvm_pagewalk ngpa_walk;
/*
* Paging state of an L2 guest (used for nested npt)
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index 2610b996e144..1631fd43c9a1 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -257,7 +257,7 @@ static inline void kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
* need to refresh ngva_walk, a.k.a. the walker used to translate L2
* GVAs to GPAs, so as to honor L2's CR0.WP.
*/
- if (!tdp_enabled || w == &vcpu->arch.guest_mmu.w)
+ if (!tdp_enabled || w == &vcpu->arch.ngpa_walk)
return;
__kvm_mmu_refresh_passthrough_bits(vcpu, w);
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 8b5ffd78565b..3ffaa48b566e 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -2478,12 +2478,14 @@ static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterato
struct kvm_vcpu *vcpu, hpa_t root,
u64 addr)
{
+ struct kvm_pagewalk *w = vcpu->arch.mmu->w;
+
iterator->addr = addr;
iterator->shadow_addr = root;
iterator->level = vcpu->arch.mmu->root_role.level;
if (iterator->level >= PT64_ROOT_4LEVEL &&
- vcpu->arch.mmu->w.cpu_role.base.level < PT64_ROOT_4LEVEL &&
+ w->cpu_role.base.level < PT64_ROOT_4LEVEL &&
!vcpu->arch.mmu->root_role.direct)
iterator->level = PT32E_ROOT_LEVEL;
@@ -4090,12 +4092,13 @@ static int mmu_first_shadow_root_alloc(struct kvm *kvm)
static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
{
struct kvm_mmu *mmu = vcpu->arch.mmu;
+ struct kvm_pagewalk *w = mmu->w;
u64 pdptrs[4], pm_mask;
gfn_t root_gfn, root_pgd;
int quadrant, i, r;
hpa_t root;
- root_pgd = kvm_mmu_get_guest_pgd(vcpu, &mmu->w);
+ root_pgd = kvm_mmu_get_guest_pgd(vcpu, w);
root_gfn = (root_pgd & __PT_BASE_ADDR_MASK) >> PAGE_SHIFT;
if (!kvm_vcpu_is_visible_gfn(vcpu, root_gfn)) {
@@ -4107,9 +4110,9 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
* On SVM, reading PDPTRs might access guest memory, which might fault
* and thus might sleep. Grab the PDPTRs before acquiring mmu_lock.
*/
- if (mmu->w.cpu_role.base.level == PT32E_ROOT_LEVEL) {
+ if (w->cpu_role.base.level == PT32E_ROOT_LEVEL) {
for (i = 0; i < 4; ++i) {
- pdptrs[i] = mmu->w.get_pdptr(vcpu, i);
+ pdptrs[i] = w->get_pdptr(vcpu, i);
if (!(pdptrs[i] & PT_PRESENT_MASK))
continue;
@@ -4131,7 +4134,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
* Do we shadow a long mode page table? If so we need to
* write-protect the guests page table root.
*/
- if (mmu->w.cpu_role.base.level >= PT64_ROOT_4LEVEL) {
+ if (w->cpu_role.base.level >= PT64_ROOT_4LEVEL) {
root = mmu_alloc_root(vcpu, root_gfn, 0,
mmu->root_role.level);
mmu->root.hpa = root;
@@ -4170,7 +4173,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
for (i = 0; i < 4; ++i) {
WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i]));
- if (mmu->w.cpu_role.base.level == PT32E_ROOT_LEVEL) {
+ if (w->cpu_role.base.level == PT32E_ROOT_LEVEL) {
if (!(pdptrs[i] & PT_PRESENT_MASK)) {
mmu->pae_root[i] = INVALID_PAE_ROOT;
continue;
@@ -4184,7 +4187,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
* directory. Othwerise each PAE page direct shadows one guest
* PAE page directory so that quadrant should be 0.
*/
- quadrant = (mmu->w.cpu_role.base.level == PT32_ROOT_LEVEL) ? i : 0;
+ quadrant = (w->cpu_role.base.level == PT32_ROOT_LEVEL) ? i : 0;
root = mmu_alloc_root(vcpu, root_gfn, quadrant, PT32_ROOT_LEVEL);
mmu->pae_root[i] = root | pm_mask;
@@ -4208,6 +4211,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu)
{
struct kvm_mmu *mmu = vcpu->arch.mmu;
+ struct kvm_pagewalk *w = mmu->w;
bool need_pml5 = mmu->root_role.level > PT64_ROOT_4LEVEL;
u64 *pml5_root = NULL;
u64 *pml4_root = NULL;
@@ -4220,7 +4224,7 @@ static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu)
* on demand, as running a 32-bit L1 VMM on 64-bit KVM is very rare.
*/
if (mmu->root_role.direct ||
- mmu->w.cpu_role.base.level >= PT64_ROOT_4LEVEL ||
+ w->cpu_role.base.level >= PT64_ROOT_4LEVEL ||
mmu->root_role.level < PT64_ROOT_4LEVEL)
return 0;
@@ -4325,7 +4329,7 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
- if (vcpu->arch.mmu->w.cpu_role.base.level >= PT64_ROOT_4LEVEL) {
+ if (vcpu->arch.mmu->w->cpu_role.base.level >= PT64_ROOT_4LEVEL) {
hpa_t root = vcpu->arch.mmu->root.hpa;
if (!is_unsync_root(root))
@@ -4567,7 +4571,7 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu,
if (arch.direct_map)
arch.cr3 = (unsigned long)INVALID_GPA;
else
- arch.cr3 = kvm_mmu_get_guest_pgd(vcpu, &vcpu->arch.mmu->w);
+ arch.cr3 = kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu->w);
return kvm_setup_async_pf(vcpu, fault->addr,
kvm_vcpu_gfn_to_hva(vcpu, fault->gfn), &arch);
@@ -5110,7 +5114,7 @@ void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
return;
if (!vcpu->arch.mmu->root_role.direct &&
- work->arch.cr3 != kvm_mmu_get_guest_pgd(vcpu, &vcpu->arch.mmu->w))
+ work->arch.cr3 != kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu->w))
return;
r = kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, work->arch.error_code,
@@ -5208,7 +5212,7 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_tdp_mmu_map_private_pfn);
static void nonpaging_init_context(struct kvm_mmu *context)
{
context->page_fault = nonpaging_page_fault;
- context->w.gva_to_gpa = nonpaging_gva_to_gpa;
+ context->w->gva_to_gpa = nonpaging_gva_to_gpa;
context->sync_spte = NULL;
}
@@ -5523,9 +5527,9 @@ static void __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
}
static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
- struct kvm_mmu *context, bool execonly, int huge_page_level)
+ bool execonly, int huge_page_level)
{
- __reset_rsvds_bits_mask_ept(&context->w.guest_rsvd_check,
+ __reset_rsvds_bits_mask_ept(&vcpu->arch.ngpa_walk.guest_rsvd_check,
vcpu->arch.reserved_gpa_bits, execonly,
huge_page_level);
}
@@ -5832,21 +5836,21 @@ static void reset_guest_paging_metadata(struct kvm_vcpu *vcpu,
return;
reset_guest_rsvds_bits_mask(vcpu, w);
- update_permission_bitmask(w, w == &vcpu->arch.guest_mmu.w, false);
+ update_permission_bitmask(w, w == &vcpu->arch.ngpa_walk, false);
update_pkru_bitmask(w);
}
static void paging64_init_context(struct kvm_mmu *context)
{
context->page_fault = paging64_page_fault;
- context->w.gva_to_gpa = paging64_gva_to_gpa;
+ context->w->gva_to_gpa = paging64_gva_to_gpa;
context->sync_spte = paging64_sync_spte;
}
static void paging32_init_context(struct kvm_mmu *context)
{
context->page_fault = paging32_page_fault;
- context->w.gva_to_gpa = paging32_gva_to_gpa;
+ context->w->gva_to_gpa = paging32_gva_to_gpa;
context->sync_spte = paging32_sync_spte;
}
@@ -5961,27 +5965,27 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
struct kvm_mmu *context = &vcpu->arch.root_mmu;
union kvm_mmu_page_role root_role = kvm_calc_tdp_mmu_root_page_role(vcpu, cpu_role);
- if (cpu_role.as_u64 == context->w.cpu_role.as_u64 &&
+ if (cpu_role.as_u64 == context->w->cpu_role.as_u64 &&
root_role.word == context->root_role.word)
return;
- context->w.cpu_role.as_u64 = cpu_role.as_u64;
+ context->w->cpu_role.as_u64 = cpu_role.as_u64;
context->root_role.word = root_role.word;
context->page_fault = kvm_tdp_page_fault;
context->sync_spte = NULL;
- context->w.inject_page_fault = kvm_inject_page_fault;
- context->w.get_pdptr = kvm_pdptr_read;
- context->w.get_guest_pgd = get_guest_cr3;
+ context->w->inject_page_fault = kvm_inject_page_fault;
+ context->w->get_pdptr = kvm_pdptr_read;
+ context->w->get_guest_pgd = get_guest_cr3;
- if (!is_cr0_pg(&context->w))
- context->w.gva_to_gpa = nonpaging_gva_to_gpa;
- else if (is_cr4_pae(&context->w))
- context->w.gva_to_gpa = paging64_gva_to_gpa;
+ if (!is_cr0_pg(context->w))
+ context->w->gva_to_gpa = nonpaging_gva_to_gpa;
+ else if (is_cr4_pae(context->w))
+ context->w->gva_to_gpa = paging64_gva_to_gpa;
else
- context->w.gva_to_gpa = paging32_gva_to_gpa;
+ context->w->gva_to_gpa = paging32_gva_to_gpa;
- reset_guest_paging_metadata(vcpu, &context->w);
+ reset_guest_paging_metadata(vcpu, context->w);
reset_tdp_shadow_zero_bits_mask(context);
}
@@ -5989,21 +5993,21 @@ static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *conte
union kvm_cpu_role cpu_role,
union kvm_mmu_page_role root_role)
{
- if (cpu_role.as_u64 == context->w.cpu_role.as_u64 &&
+ if (cpu_role.as_u64 == context->w->cpu_role.as_u64 &&
root_role.word == context->root_role.word)
return;
- context->w.cpu_role.as_u64 = cpu_role.as_u64;
+ context->w->cpu_role.as_u64 = cpu_role.as_u64;
context->root_role.word = root_role.word;
- if (!is_cr0_pg(&context->w))
+ if (!is_cr0_pg(context->w))
nonpaging_init_context(context);
- else if (is_cr4_pae(&context->w))
+ else if (is_cr4_pae(context->w))
paging64_init_context(context);
else
paging32_init_context(context);
- reset_guest_paging_metadata(vcpu, &context->w);
+ reset_guest_paging_metadata(vcpu, context->w);
reset_shadow_zero_bits_mask(vcpu, context);
}
@@ -6095,18 +6099,20 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty,
execonly, level, mbec);
- if (new_mode.as_u64 != context->w.cpu_role.as_u64) {
+ struct kvm_pagewalk *ngpa_walk = &vcpu->arch.ngpa_walk;
+
+ if (new_mode.as_u64 != ngpa_walk->cpu_role.as_u64) {
/* EPT, and thus nested EPT, does not consume CR0, CR4, nor EFER. */
- context->w.cpu_role.as_u64 = new_mode.as_u64;
+ ngpa_walk->cpu_role.as_u64 = new_mode.as_u64;
context->root_role.word = new_mode.base.word;
context->page_fault = ept_page_fault;
- context->w.gva_to_gpa = ept_gva_to_gpa;
+ ngpa_walk->gva_to_gpa = ept_gva_to_gpa;
context->sync_spte = ept_sync_spte;
- update_permission_bitmask(&context->w, true, true);
- context->w.pkru_mask = 0;
- reset_rsvds_bits_mask_ept(vcpu, context, execonly, huge_page_level);
+ update_permission_bitmask(ngpa_walk, true, true);
+ ngpa_walk->pkru_mask = 0;
+ reset_rsvds_bits_mask_ept(vcpu, execonly, huge_page_level);
reset_ept_shadow_zero_bits_mask(context, execonly);
}
@@ -6121,9 +6127,9 @@ static void init_kvm_softmmu(struct kvm_vcpu *vcpu,
kvm_init_shadow_mmu(vcpu, cpu_role);
- context->w.inject_page_fault = kvm_inject_page_fault;
- context->w.get_pdptr = kvm_pdptr_read;
- context->w.get_guest_pgd = get_guest_cr3;
+ context->w->inject_page_fault = kvm_inject_page_fault;
+ context->w->get_pdptr = kvm_pdptr_read;
+ context->w->get_guest_pgd = get_guest_cr3;
}
static void init_kvm_ngva_walk(struct kvm_vcpu *vcpu,
@@ -6189,8 +6195,8 @@ void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu)
*/
vcpu->arch.root_mmu.root_role.invalid = 1;
vcpu->arch.guest_mmu.root_role.invalid = 1;
- vcpu->arch.root_mmu.w.cpu_role.ext.valid = 0;
- vcpu->arch.guest_mmu.w.cpu_role.ext.valid = 0;
+ vcpu->arch.root_gva_walk.cpu_role.ext.valid = 0;
+ vcpu->arch.ngpa_walk.cpu_role.ext.valid = 0;
vcpu->arch.ngva_walk.cpu_role.ext.valid = 0;
kvm_mmu_reset_context(vcpu);
@@ -6687,7 +6693,7 @@ void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
WARN_ON_ONCE(roots & ~KVM_MMU_ROOTS_ALL);
/* It's actually a GPA for vcpu->arch.guest_mmu. */
- if (w != &vcpu->arch.guest_mmu.w) {
+ if (w == vcpu->arch.gva_walk) {
/* INVLPG on a non-canonical address is a NOP according to the SDM. */
if (is_noncanonical_invlpg_address(addr, vcpu))
return;
@@ -6695,9 +6701,13 @@ void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
kvm_x86_call(flush_tlb_gva)(vcpu, addr);
if (w == &vcpu->arch.ngva_walk)
return;
+
+ mmu = &vcpu->arch.root_mmu;
+ } else {
+ mmu = &vcpu->arch.guest_mmu;
}
- mmu = container_of(w, struct kvm_mmu, w);
+ /* Invalidate shadow pages, whether GPA->GVA or nGPA->GPA. */
if (!mmu->sync_spte)
return;
@@ -6745,7 +6755,7 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
}
if (roots)
- kvm_mmu_invalidate_addr(vcpu, &mmu->w, gva, roots);
+ kvm_mmu_invalidate_addr(vcpu, mmu->w, gva, roots);
++vcpu->stat.invlpg;
/*
@@ -6790,11 +6800,12 @@ static void free_mmu_pages(struct kvm_mmu *mmu)
free_page((unsigned long)mmu->pml5_root);
}
-static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
+static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, struct kvm_pagewalk *w)
{
struct page *page;
int i;
+ mmu->w = w;
mmu->root.hpa = INVALID_PAGE;
mmu->root.pgd = 0;
mmu->mirror_root_hpa = INVALID_PAGE;
@@ -6860,13 +6871,13 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu)
vcpu->arch.mmu_shadow_page_cache.gfp_zero = __GFP_ZERO;
vcpu->arch.mmu = &vcpu->arch.root_mmu;
- vcpu->arch.gva_walk = &vcpu->arch.root_mmu.w;
+ vcpu->arch.gva_walk = &vcpu->arch.root_gva_walk;
- ret = __kvm_mmu_create(vcpu, &vcpu->arch.guest_mmu);
+ ret = __kvm_mmu_create(vcpu, &vcpu->arch.guest_mmu, &vcpu->arch.ngpa_walk);
if (ret)
return ret;
- ret = __kvm_mmu_create(vcpu, &vcpu->arch.root_mmu);
+ ret = __kvm_mmu_create(vcpu, &vcpu->arch.root_mmu, &vcpu->arch.root_gva_walk);
if (ret)
goto fail_allocate_root;
diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index 9cfae71cd3e6..115f0fd2d4ba 100644
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -157,7 +157,7 @@ static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu,
struct kvm_mmu_page *sp, u64 *spte,
u64 gpte)
{
- struct kvm_pagewalk *w = &vcpu->arch.mmu->w;
+ struct kvm_pagewalk *w = vcpu->arch.mmu->w;
if (!FNAME(is_present_gpte)(w, gpte))
goto no_present;
@@ -563,7 +563,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
static int FNAME(walk_addr)(struct guest_walker *walker,
struct kvm_vcpu *vcpu, gpa_t addr, u64 access)
{
- return FNAME(walk_addr_generic)(walker, vcpu, &vcpu->arch.mmu->w, addr,
+ return FNAME(walk_addr_generic)(walker, vcpu, vcpu->arch.mmu->w, addr,
access);
}
@@ -579,7 +579,7 @@ FNAME(prefetch_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
gfn = gpte_to_gfn(gpte);
pte_access = sp->role.access & FNAME(gpte_access)(gpte);
- FNAME(protect_clean_gpte)(&vcpu->arch.mmu->w, &pte_access, gpte);
+ FNAME(protect_clean_gpte)(vcpu->arch.mmu->w, &pte_access, gpte);
return kvm_mmu_prefetch_sptes(vcpu, gfn, spte, 1, pte_access);
}
@@ -662,7 +662,7 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
WARN_ON_ONCE(gw->gfn != base_gfn);
direct_access = gw->pte_access;
- top_level = vcpu->arch.mmu->w.cpu_role.base.level;
+ top_level = vcpu->arch.mmu->w->cpu_role.base.level;
if (top_level == PT32E_ROOT_LEVEL)
top_level = PT32_ROOT_LEVEL;
/*
@@ -851,7 +851,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
* otherwise KVM will cache incorrect access information in the SPTE.
*/
if (fault->write && !(walker.pte_access & ACC_WRITE_MASK) &&
- !is_cr0_wp(&vcpu->arch.mmu->w) && !fault->user && fault->slot) {
+ !is_cr0_wp(vcpu->arch.mmu->w) && !fault->user && fault->slot) {
walker.pte_access |= ACC_WRITE_MASK;
walker.pte_access &= ~ACC_USER_MASK;
@@ -861,7 +861,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
* then we should prevent the kernel from executing it
* if SMEP is enabled.
*/
- if (is_cr4_smep(&vcpu->arch.mmu->w))
+ if (is_cr4_smep(vcpu->arch.mmu->w))
walker.pte_access &= ~ACC_EXEC_MASK;
}
#endif
@@ -959,7 +959,7 @@ static int FNAME(sync_spte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, int
gfn = gpte_to_gfn(gpte);
pte_access = sp->role.access;
pte_access &= FNAME(gpte_access)(gpte);
- FNAME(protect_clean_gpte)(&vcpu->arch.mmu->w, &pte_access, gpte);
+ FNAME(protect_clean_gpte)(vcpu->arch.mmu->w, &pte_access, gpte);
if (sync_mmio_spte(vcpu, &sp->spt[i], gfn, pte_access))
return 0;
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index 713b2508b8ca..97d3fabb8c0d 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -114,17 +114,16 @@ static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
svm->nested.ctl.nested_cr3,
svm->nested.ctl.misc_ctl);
- vcpu->arch.mmu->w.get_guest_pgd = nested_svm_get_tdp_cr3;
- vcpu->arch.mmu->w.get_pdptr = nested_svm_get_tdp_pdptr;
-
- vcpu->arch.mmu->w.inject_page_fault = nested_svm_inject_npf_exit;
+ vcpu->arch.ngpa_walk.get_guest_pgd = nested_svm_get_tdp_cr3;
+ vcpu->arch.ngpa_walk.get_pdptr = nested_svm_get_tdp_pdptr;
+ vcpu->arch.ngpa_walk.inject_page_fault = nested_svm_inject_npf_exit;
vcpu->arch.gva_walk = &vcpu->arch.ngva_walk;
}
static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
{
vcpu->arch.mmu = &vcpu->arch.root_mmu;
- vcpu->arch.gva_walk = &vcpu->arch.root_mmu.w;
+ vcpu->arch.gva_walk = vcpu->arch.root_mmu.w;
}
static bool nested_vmcb_needs_vls_intercept(struct vcpu_svm *svm)
@@ -2153,7 +2152,7 @@ static gpa_t svm_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa,
u64 pte_access)
{
struct vcpu_svm *svm = to_svm(vcpu);
- struct kvm_pagewalk *w = &vcpu->arch.mmu->w;
+ struct kvm_pagewalk *w = &vcpu->arch.ngpa_walk;
if (WARN_ON_ONCE(!mmu_is_nested(vcpu)))
return gpa;
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 345ee3323a93..3596d15ae405 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -408,7 +408,7 @@ static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp,
roots |= KVM_MMU_ROOT_PREVIOUS(i);
}
if (roots)
- kvm_mmu_invalidate_addr(vcpu, &vcpu->arch.guest_mmu.w, addr, roots);
+ kvm_mmu_invalidate_addr(vcpu, &vcpu->arch.ngpa_walk, addr, roots);
}
static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
@@ -512,10 +512,10 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
vcpu->arch.mmu = &vcpu->arch.guest_mmu;
nested_ept_new_eptp(vcpu);
- vcpu->arch.mmu->w.get_guest_pgd = nested_ept_get_eptp;
- vcpu->arch.mmu->w.get_pdptr = kvm_pdptr_read;
+ vcpu->arch.ngpa_walk.get_guest_pgd = nested_ept_get_eptp;
+ vcpu->arch.ngpa_walk.get_pdptr = kvm_pdptr_read;
- vcpu->arch.mmu->w.inject_page_fault = nested_ept_inject_page_fault;
+ vcpu->arch.ngpa_walk.inject_page_fault = nested_ept_inject_page_fault;
vcpu->arch.gva_walk = &vcpu->arch.ngva_walk;
}
@@ -523,7 +523,7 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
{
vcpu->arch.mmu = &vcpu->arch.root_mmu;
- vcpu->arch.gva_walk = &vcpu->arch.root_mmu.w;
+ vcpu->arch.gva_walk = vcpu->arch.root_mmu.w;
}
static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
@@ -7465,12 +7465,13 @@ __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *))
return 0;
}
+
static gpa_t vmx_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa,
u64 access,
struct x86_exception *exception,
u64 pte_access)
{
- struct kvm_pagewalk *w = &vcpu->arch.mmu->w;
+ struct kvm_pagewalk *w = &vcpu->arch.ngpa_walk;
if (WARN_ON_ONCE(!mmu_is_nested(vcpu)))
return gpa;
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 43625cc9e934..d6ab17f17d69 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -586,7 +586,7 @@ void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
WARN_ON_ONCE(fault->vector != PF_VECTOR);
- fault_walk = fault->nested_page_fault ? &vcpu->arch.mmu->w :
+ fault_walk = fault->nested_page_fault ? &vcpu->arch.ngpa_walk :
vcpu->arch.gva_walk;
/*
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 14/19] KVM: x86/mmu: unify root_gva_walk and ngva_walk
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
` (12 preceding siblings ...)
2026-06-24 21:30 ` [PATCH 13/19] KVM: x86/mmu: pull struct kvm_pagewalk out of struct kvm_mmu Paolo Bonzini
@ 2026-06-24 21:30 ` Paolo Bonzini
2026-06-24 21:30 ` [PATCH 15/19] KVM: x86/mmu: cleanup functions that initialize shadow MMU Paolo Bonzini
` (5 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:30 UTC (permalink / raw)
To: linux-kernel, kvm
At this point, vcpu->arch.gva_walk and vcpu->arch.root_mmu.w contain
the same information (at least when KVM is not running a nested guest,
i.e. when root_mmu is actually in use); compare init_kvm_page_walk()
on one side with init_kvm_softmmu() + shadow_mmu_init_context() on
the other. root_mmu.w is still used by shadow paging, via
FNAME(walk_addr) and its callers.
Always use the same instance of kvm_pagewalk to do GVA->GPA translations,
instead of flipping the gva_walk pointer back and forth. After all the
page walking does behave the same no matter if you are in guest mode or
not; the difference lies in the behavior of kvm_translate_gpa and thus
in vcpu->arch.mmu, not in the page walker itself.
vcpu->arch.guest_mmu.w instead is used for both guest emulation
(kvm_translate_gpa) and shadow paging.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/include/asm/kvm_host.h | 13 +---
arch/x86/kvm/hyperv.c | 2 +-
arch/x86/kvm/mmu.h | 8 +--
arch/x86/kvm/mmu/mmu.c | 120 +++++++++++---------------------
arch/x86/kvm/mmu/paging_tmpl.h | 4 +-
arch/x86/kvm/regs.c | 2 +-
arch/x86/kvm/svm/nested.c | 2 -
arch/x86/kvm/vmx/nested.c | 3 -
arch/x86/kvm/x86.c | 18 ++---
9 files changed, 60 insertions(+), 112 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 8b9cf364c9f6..8a2126ca49c4 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -905,26 +905,15 @@ struct kvm_vcpu_arch {
/* Non-nested MMU for L1 */
struct kvm_mmu root_mmu;
- struct kvm_pagewalk root_gva_walk;
/* L1 TDP when running nested */
struct kvm_mmu guest_mmu;
struct kvm_pagewalk ngpa_walk;
- /*
- * Paging state of an L2 guest (used for nested npt)
- *
- * This context will save all necessary information to walk page tables
- * of an L2 guest. This context is only initialized for page table
- * walking and not for faulting since we never handle l2 page faults on
- * the host.
- */
- struct kvm_pagewalk ngva_walk;
-
/*
* Pagewalk context used for gva_to_gpa translations.
*/
- struct kvm_pagewalk *gva_walk;
+ struct kvm_pagewalk gva_walk;
u64 pdptrs[4]; /* pae */
diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c
index 51d812babe73..1ee0d23f8949 100644
--- a/arch/x86/kvm/hyperv.c
+++ b/arch/x86/kvm/hyperv.c
@@ -2046,7 +2046,7 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
* read with kvm_read_guest().
*/
if (!hc->fast) {
- hc->ingpa = kvm_translate_gpa(vcpu, vcpu->arch.gva_walk, hc->ingpa,
+ hc->ingpa = kvm_translate_gpa(vcpu, &vcpu->arch.gva_walk, hc->ingpa,
PFERR_GUEST_FINAL_MASK, NULL, 0);
if (unlikely(hc->ingpa == INVALID_GPA))
return HV_STATUS_INVALID_HYPERCALL_INPUT;
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index 1631fd43c9a1..9d00d0eb230b 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -253,9 +253,9 @@ static inline void kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
* @w's snapshot of CR0.WP and thus all related paging metadata may
* be stale. Refresh CR0.WP and the metadata on-demand when checking
* for permission faults. Exempt nested MMUs, i.e. MMUs for shadowing
- * nEPT and nNPT, as CR0.WP is ignored in both cases. Note, KVM does
- * need to refresh ngva_walk, a.k.a. the walker used to translate L2
- * GVAs to GPAs, so as to honor L2's CR0.WP.
+ * nEPT and nNPT, as CR0.WP is ignored in both cases. Note, KVM will
+ * still refresh gva_walk, so as to honor L2's CR0.WP when translating
+ * L2 GVAs to GPAs.
*/
if (!tdp_enabled || w == &vcpu->arch.ngpa_walk)
return;
@@ -382,7 +382,7 @@ static inline gpa_t kvm_translate_gpa(struct kvm_vcpu *vcpu,
struct x86_exception *exception,
u64 pte_access)
{
- if (w != &vcpu->arch.ngva_walk)
+ if (!mmu_is_nested(vcpu) || w == &vcpu->arch.ngpa_walk)
return gpa;
return kvm_x86_ops.nested_ops->translate_nested_gpa(vcpu, gpa, access,
exception,
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 3ffaa48b566e..a464e3ec26ee 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -5212,7 +5212,6 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_tdp_mmu_map_private_pfn);
static void nonpaging_init_context(struct kvm_mmu *context)
{
context->page_fault = nonpaging_page_fault;
- context->w->gva_to_gpa = nonpaging_gva_to_gpa;
context->sync_spte = NULL;
}
@@ -5843,14 +5842,12 @@ static void reset_guest_paging_metadata(struct kvm_vcpu *vcpu,
static void paging64_init_context(struct kvm_mmu *context)
{
context->page_fault = paging64_page_fault;
- context->w->gva_to_gpa = paging64_gva_to_gpa;
context->sync_spte = paging64_sync_spte;
}
static void paging32_init_context(struct kvm_mmu *context)
{
context->page_fault = paging32_page_fault;
- context->w->gva_to_gpa = paging32_gva_to_gpa;
context->sync_spte = paging32_sync_spte;
}
@@ -5965,39 +5962,22 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
struct kvm_mmu *context = &vcpu->arch.root_mmu;
union kvm_mmu_page_role root_role = kvm_calc_tdp_mmu_root_page_role(vcpu, cpu_role);
- if (cpu_role.as_u64 == context->w->cpu_role.as_u64 &&
- root_role.word == context->root_role.word)
+ if (root_role.word == context->root_role.word)
return;
- context->w->cpu_role.as_u64 = cpu_role.as_u64;
context->root_role.word = root_role.word;
context->page_fault = kvm_tdp_page_fault;
context->sync_spte = NULL;
- context->w->inject_page_fault = kvm_inject_page_fault;
- context->w->get_pdptr = kvm_pdptr_read;
- context->w->get_guest_pgd = get_guest_cr3;
-
- if (!is_cr0_pg(context->w))
- context->w->gva_to_gpa = nonpaging_gva_to_gpa;
- else if (is_cr4_pae(context->w))
- context->w->gva_to_gpa = paging64_gva_to_gpa;
- else
- context->w->gva_to_gpa = paging32_gva_to_gpa;
-
- reset_guest_paging_metadata(vcpu, context->w);
reset_tdp_shadow_zero_bits_mask(context);
}
static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *context,
- union kvm_cpu_role cpu_role,
union kvm_mmu_page_role root_role)
{
- if (cpu_role.as_u64 == context->w->cpu_role.as_u64 &&
- root_role.word == context->root_role.word)
+ if (root_role.word == context->root_role.word)
return;
- context->w->cpu_role.as_u64 = cpu_role.as_u64;
context->root_role.word = root_role.word;
if (!is_cr0_pg(context->w))
@@ -6007,7 +5987,6 @@ static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *conte
else
paging32_init_context(context);
- reset_guest_paging_metadata(vcpu, context->w);
reset_shadow_zero_bits_mask(vcpu, context);
}
@@ -6033,7 +6012,28 @@ static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu,
*/
root_role.efer_nx = true;
- shadow_mmu_init_context(vcpu, context, cpu_role, root_role);
+ shadow_mmu_init_context(vcpu, context, root_role);
+}
+
+static void init_kvm_page_walk(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
+ union kvm_cpu_role cpu_role)
+{
+ if (cpu_role.as_u64 == w->cpu_role.as_u64)
+ return;
+
+ w->cpu_role.as_u64 = cpu_role.as_u64;
+ w->inject_page_fault = kvm_inject_page_fault;
+ w->get_pdptr = kvm_pdptr_read;
+ w->get_guest_pgd = get_guest_cr3;
+
+ if (!is_cr0_pg(w))
+ w->gva_to_gpa = nonpaging_gva_to_gpa;
+ else if (is_cr4_pae(w))
+ w->gva_to_gpa = paging64_gva_to_gpa;
+ else
+ w->gva_to_gpa = paging32_gva_to_gpa;
+
+ reset_guest_paging_metadata(vcpu, w);
}
void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr4,
@@ -6052,13 +6052,15 @@ void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr4,
WARN_ON_ONCE(cpu_role.base.direct || !cpu_role.base.guest_mode);
cpu_role.base.cr4_smep = (misc_ctl & SVM_MISC_ENABLE_GMET) != 0;
+ init_kvm_page_walk(vcpu, &vcpu->arch.ngpa_walk, cpu_role);
+
root_role = cpu_role.base;
root_role.level = kvm_mmu_get_tdp_level(vcpu);
if (root_role.level == PT64_ROOT_5LEVEL &&
cpu_role.base.level == PT64_ROOT_4LEVEL)
root_role.passthrough = 1;
- shadow_mmu_init_context(vcpu, context, cpu_role, root_role);
+ shadow_mmu_init_context(vcpu, context, root_role);
kvm_mmu_new_pgd(vcpu, nested_cr3);
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_init_shadow_npt_mmu);
@@ -6123,46 +6125,7 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_init_shadow_ept_mmu);
static void init_kvm_softmmu(struct kvm_vcpu *vcpu,
union kvm_cpu_role cpu_role)
{
- struct kvm_mmu *context = &vcpu->arch.root_mmu;
-
kvm_init_shadow_mmu(vcpu, cpu_role);
-
- context->w->inject_page_fault = kvm_inject_page_fault;
- context->w->get_pdptr = kvm_pdptr_read;
- context->w->get_guest_pgd = get_guest_cr3;
-}
-
-static void init_kvm_ngva_walk(struct kvm_vcpu *vcpu,
- union kvm_cpu_role new_mode)
-{
- struct kvm_pagewalk *g_context = &vcpu->arch.ngva_walk;
-
- if (new_mode.as_u64 == g_context->cpu_role.as_u64)
- return;
-
- g_context->cpu_role.as_u64 = new_mode.as_u64;
- g_context->inject_page_fault = kvm_inject_page_fault;
- g_context->get_pdptr = kvm_pdptr_read;
- g_context->get_guest_pgd = get_guest_cr3;
-
- /*
- * Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using
- * L1's nested page tables (e.g. EPT12). The nested translation
- * of l2_gva to l1_gpa is done by arch.ngva_walk.gva_to_gpa using
- * L2's page tables as the first level of translation and L1's
- * nested page tables as the second level of translation. Basically
- * the gva_to_gpa functions between mmu and ngva_walk are swapped.
- */
- if (!is_paging(vcpu))
- g_context->gva_to_gpa = nonpaging_gva_to_gpa;
- else if (is_long_mode(vcpu))
- g_context->gva_to_gpa = paging64_gva_to_gpa;
- else if (is_pae(vcpu))
- g_context->gva_to_gpa = paging64_gva_to_gpa;
- else
- g_context->gva_to_gpa = paging32_gva_to_gpa;
-
- reset_guest_paging_metadata(vcpu, g_context);
}
void kvm_init_mmu(struct kvm_vcpu *vcpu)
@@ -6170,12 +6133,14 @@ void kvm_init_mmu(struct kvm_vcpu *vcpu)
struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu);
union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, ®s);
- if (mmu_is_nested(vcpu))
- init_kvm_ngva_walk(vcpu, cpu_role);
- else if (tdp_enabled)
- init_kvm_tdp_mmu(vcpu, cpu_role);
- else
- init_kvm_softmmu(vcpu, cpu_role);
+ init_kvm_page_walk(vcpu, &vcpu->arch.gva_walk, cpu_role);
+
+ if (!mmu_is_nested(vcpu)) {
+ if (tdp_enabled)
+ init_kvm_tdp_mmu(vcpu, cpu_role);
+ else
+ init_kvm_softmmu(vcpu, cpu_role);
+ }
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_init_mmu);
@@ -6195,9 +6160,8 @@ void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu)
*/
vcpu->arch.root_mmu.root_role.invalid = 1;
vcpu->arch.guest_mmu.root_role.invalid = 1;
- vcpu->arch.root_gva_walk.cpu_role.ext.valid = 0;
vcpu->arch.ngpa_walk.cpu_role.ext.valid = 0;
- vcpu->arch.ngva_walk.cpu_role.ext.valid = 0;
+ vcpu->arch.gva_walk.cpu_role.ext.valid = 0;
kvm_mmu_reset_context(vcpu);
KVM_BUG_ON(!kvm_can_set_cpuid_and_feature_msrs(vcpu), vcpu->kvm);
@@ -6693,13 +6657,14 @@ void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
WARN_ON_ONCE(roots & ~KVM_MMU_ROOTS_ALL);
/* It's actually a GPA for vcpu->arch.guest_mmu. */
- if (w == vcpu->arch.gva_walk) {
+ if (w == &vcpu->arch.gva_walk) {
/* INVLPG on a non-canonical address is a NOP according to the SDM. */
if (is_noncanonical_invlpg_address(addr, vcpu))
return;
kvm_x86_call(flush_tlb_gva)(vcpu, addr);
- if (w == &vcpu->arch.ngva_walk)
+
+ if (tdp_enabled)
return;
mmu = &vcpu->arch.root_mmu;
@@ -6733,7 +6698,7 @@ void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
* be synced when switching to that new cr3, so nothing needs to be
* done here for them.
*/
- kvm_mmu_invalidate_addr(vcpu, vcpu->arch.gva_walk, gva, KVM_MMU_ROOTS_ALL);
+ kvm_mmu_invalidate_addr(vcpu, &vcpu->arch.gva_walk, gva, KVM_MMU_ROOTS_ALL);
++vcpu->stat.invlpg;
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_invlpg);
@@ -6755,7 +6720,7 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
}
if (roots)
- kvm_mmu_invalidate_addr(vcpu, mmu->w, gva, roots);
+ kvm_mmu_invalidate_addr(vcpu, &vcpu->arch.gva_walk, gva, roots);
++vcpu->stat.invlpg;
/*
@@ -6871,13 +6836,12 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu)
vcpu->arch.mmu_shadow_page_cache.gfp_zero = __GFP_ZERO;
vcpu->arch.mmu = &vcpu->arch.root_mmu;
- vcpu->arch.gva_walk = &vcpu->arch.root_gva_walk;
ret = __kvm_mmu_create(vcpu, &vcpu->arch.guest_mmu, &vcpu->arch.ngpa_walk);
if (ret)
return ret;
- ret = __kvm_mmu_create(vcpu, &vcpu->arch.root_mmu, &vcpu->arch.root_gva_walk);
+ ret = __kvm_mmu_create(vcpu, &vcpu->arch.root_mmu, &vcpu->arch.gva_walk);
if (ret)
goto fail_allocate_root;
diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index 115f0fd2d4ba..a46384b7080f 100644
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -548,7 +548,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
}
#endif
walker->fault.address = addr;
- walker->fault.nested_page_fault = w != vcpu->arch.gva_walk;
+ walker->fault.nested_page_fault = w != &vcpu->arch.gva_walk;
walker->fault.async_page_fault = false;
#if PTTYPE != PTTYPE_EPT
@@ -906,7 +906,7 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
#ifndef CONFIG_X86_64
/* A 64-bit GVA should be impossible on 32-bit KVM. */
- WARN_ON_ONCE((addr >> 32) && w == vcpu->arch.gva_walk);
+ WARN_ON_ONCE((addr >> 32) && w == &vcpu->arch.gva_walk);
#endif
r = FNAME(walk_addr_generic)(&walker, vcpu, w, addr, access);
diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c
index 02adaa4ef64e..bd8147798cc3 100644
--- a/arch/x86/kvm/regs.c
+++ b/arch/x86/kvm/regs.c
@@ -154,7 +154,7 @@ static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
*/
int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
{
- struct kvm_pagewalk *w = vcpu->arch.gva_walk;
+ struct kvm_pagewalk *w = &vcpu->arch.gva_walk;
gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
gpa_t real_gpa;
int i;
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index 97d3fabb8c0d..ba985a02208a 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -117,13 +117,11 @@ static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
vcpu->arch.ngpa_walk.get_guest_pgd = nested_svm_get_tdp_cr3;
vcpu->arch.ngpa_walk.get_pdptr = nested_svm_get_tdp_pdptr;
vcpu->arch.ngpa_walk.inject_page_fault = nested_svm_inject_npf_exit;
- vcpu->arch.gva_walk = &vcpu->arch.ngva_walk;
}
static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
{
vcpu->arch.mmu = &vcpu->arch.root_mmu;
- vcpu->arch.gva_walk = vcpu->arch.root_mmu.w;
}
static bool nested_vmcb_needs_vls_intercept(struct vcpu_svm *svm)
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 3596d15ae405..0635e92471c8 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -516,14 +516,11 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
vcpu->arch.ngpa_walk.get_pdptr = kvm_pdptr_read;
vcpu->arch.ngpa_walk.inject_page_fault = nested_ept_inject_page_fault;
-
- vcpu->arch.gva_walk = &vcpu->arch.ngva_walk;
}
static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
{
vcpu->arch.mmu = &vcpu->arch.root_mmu;
- vcpu->arch.gva_walk = vcpu->arch.root_mmu.w;
}
static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index d6ab17f17d69..0626e835e9eb 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -587,7 +587,7 @@ void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
WARN_ON_ONCE(fault->vector != PF_VECTOR);
fault_walk = fault->nested_page_fault ? &vcpu->arch.ngpa_walk :
- vcpu->arch.gva_walk;
+ &vcpu->arch.gva_walk;
/*
* Invalidate the TLB entry for the faulting address, if it exists,
@@ -4769,7 +4769,7 @@ static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
struct x86_exception *exception)
{
- struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
+ struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, exception);
@@ -4779,7 +4779,7 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_gva_to_gpa_read);
gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
struct x86_exception *exception)
{
- struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
+ struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
access |= PFERR_WRITE_MASK;
@@ -4791,7 +4791,7 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_gva_to_gpa_write);
gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
struct x86_exception *exception)
{
- struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
+ struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, 0, exception);
}
@@ -4800,7 +4800,7 @@ static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
struct kvm_vcpu *vcpu, u64 access,
struct x86_exception *exception)
{
- struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
+ struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
void *data = val;
int r = X86EMUL_CONTINUE;
@@ -4833,7 +4833,7 @@ static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
struct x86_exception *exception)
{
struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
- struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
+ struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
unsigned offset;
int ret;
@@ -4892,7 +4892,7 @@ static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes
struct kvm_vcpu *vcpu, u64 access,
struct x86_exception *exception)
{
- struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
+ struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
void *data = val;
int r = X86EMUL_CONTINUE;
@@ -4998,7 +4998,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
gpa_t *gpa, struct x86_exception *exception,
bool write)
{
- struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
+ struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
u64 access = ((kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0)
| (write ? PFERR_WRITE_MASK : 0);
@@ -10601,7 +10601,7 @@ void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end)
void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code)
{
- struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
+ struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
struct x86_exception fault;
u64 access = error_code &
(PFERR_WRITE_MASK | PFERR_FETCH_MASK | PFERR_USER_MASK);
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 15/19] KVM: x86/mmu: cleanup functions that initialize shadow MMU
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
` (13 preceding siblings ...)
2026-06-24 21:30 ` [PATCH 14/19] KVM: x86/mmu: unify root_gva_walk and ngva_walk Paolo Bonzini
@ 2026-06-24 21:30 ` Paolo Bonzini
2026-06-24 21:30 ` [PATCH 16/19] KVM: x86/mmu: pull page format to a new struct Paolo Bonzini
` (4 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:30 UTC (permalink / raw)
To: linux-kernel, kvm
Now that the GVA->GPA page walker is initialized independently,
init_kvm_softmmu() does not do anything more than calling
kvm_init_shadow_mmu() so eliminate it from the call chain.
At the same time, rename kvm_init_shadow_mmu() to
init_kvm_shadow_mmu() for consistency with init_kvm_tdp_mmu().
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/kvm/mmu/mmu.c | 10 ++--------
1 file changed, 2 insertions(+), 8 deletions(-)
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index a464e3ec26ee..71ad933d34c9 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -5990,7 +5990,7 @@ static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *conte
reset_shadow_zero_bits_mask(vcpu, context);
}
-static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu,
+static void init_kvm_shadow_mmu(struct kvm_vcpu *vcpu,
union kvm_cpu_role cpu_role)
{
struct kvm_mmu *context = &vcpu->arch.root_mmu;
@@ -6122,12 +6122,6 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_init_shadow_ept_mmu);
-static void init_kvm_softmmu(struct kvm_vcpu *vcpu,
- union kvm_cpu_role cpu_role)
-{
- kvm_init_shadow_mmu(vcpu, cpu_role);
-}
-
void kvm_init_mmu(struct kvm_vcpu *vcpu)
{
struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu);
@@ -6139,7 +6133,7 @@ void kvm_init_mmu(struct kvm_vcpu *vcpu)
if (tdp_enabled)
init_kvm_tdp_mmu(vcpu, cpu_role);
else
- init_kvm_softmmu(vcpu, cpu_role);
+ init_kvm_shadow_mmu(vcpu, cpu_role);
}
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_init_mmu);
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 16/19] KVM: x86/mmu: pull page format to a new struct
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
` (14 preceding siblings ...)
2026-06-24 21:30 ` [PATCH 15/19] KVM: x86/mmu: cleanup functions that initialize shadow MMU Paolo Bonzini
@ 2026-06-24 21:30 ` Paolo Bonzini
2026-06-24 21:31 ` [PATCH 17/19] KVM: x86/mmu: merge struct rsvd_bits_validate into struct kvm_page_format Paolo Bonzini
` (3 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:30 UTC (permalink / raw)
To: linux-kernel, kvm
KVM is doing reserved bits checks on both guest and host page tables,
though the latter are only for consistency. Create a new struct
for this common code as well as for all data that is extracted from
the CPU role.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/include/asm/kvm_host.h | 25 +++++++++++++++----------
arch/x86/kvm/mmu.h | 7 ++++---
arch/x86/kvm/mmu/mmu.c | 16 ++++++++--------
arch/x86/kvm/mmu/paging_tmpl.h | 10 +++++-----
4 files changed, 32 insertions(+), 26 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 8a2126ca49c4..08aa1e2da582 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -516,16 +516,7 @@ struct kvm_page_fault;
* and 2-level 32-bit). The kvm_pagewalk structure abstracts the details of the
* current mmu mode.
*/
-struct kvm_pagewalk {
- unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu);
- u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
- void (*inject_page_fault)(struct kvm_vcpu *vcpu,
- struct x86_exception *fault,
- bool from_hardware);
- gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
- gpa_t gva_or_gpa, u64 access,
- struct x86_exception *exception);
- union kvm_cpu_role cpu_role;
+struct kvm_page_format {
struct rsvd_bits_validate guest_rsvd_check;
/*
@@ -544,6 +535,20 @@ struct kvm_pagewalk {
u16 permissions[16];
};
+struct kvm_pagewalk {
+ unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu);
+ u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
+ void (*inject_page_fault)(struct kvm_vcpu *vcpu,
+ struct x86_exception *fault,
+ bool from_hardware);
+ gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
+ gpa_t gva_or_gpa, u64 access,
+ struct x86_exception *exception);
+
+ union kvm_cpu_role cpu_role;
+ struct kvm_page_format fmt;
+};
+
struct kvm_mmu {
int (*page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
int (*sync_spte)(struct kvm_vcpu *vcpu,
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index 9d00d0eb230b..c9f628b97dae 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -294,15 +294,16 @@ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
u64 implicit_access = access & PFERR_IMPLICIT_ACCESS;
bool not_smap = ((rflags & X86_EFLAGS_AC) | implicit_access) == X86_EFLAGS_AC;
int index = (pfec | (not_smap ? PFERR_RSVD_MASK : 0)) >> 1;
+ struct kvm_page_format *fmt = &w->fmt;
u32 errcode = PFERR_PRESENT_MASK;
bool fault;
kvm_mmu_refresh_passthrough_bits(vcpu, w);
- fault = (w->permissions[index] >> pte_access) & 1;
+ fault = (fmt->permissions[index] >> pte_access) & 1;
WARN_ON_ONCE(pfec & (PFERR_PK_MASK | PFERR_SS_MASK | PFERR_RSVD_MASK));
- if (unlikely(w->pkru_mask)) {
+ if (unlikely(fmt->pkru_mask)) {
u32 pkru_bits, offset;
/*
@@ -316,7 +317,7 @@ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
/* clear present bit, replace PFEC.RSVD with ACC_USER_MASK. */
offset = (pfec & ~1) | ((pte_access & PT_USER_MASK) ? PFERR_RSVD_MASK : 0);
- pkru_bits &= w->pkru_mask >> offset;
+ pkru_bits &= fmt->pkru_mask >> offset;
errcode |= -pkru_bits & PFERR_PK_MASK;
fault |= (pkru_bits != 0);
}
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 71ad933d34c9..f70892b15680 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -5479,7 +5479,7 @@ static void __reset_rsvds_bits_mask(struct rsvd_bits_validate *rsvd_check,
static void reset_guest_rsvds_bits_mask(struct kvm_vcpu *vcpu,
struct kvm_pagewalk *w)
{
- __reset_rsvds_bits_mask(&w->guest_rsvd_check,
+ __reset_rsvds_bits_mask(&w->fmt.guest_rsvd_check,
vcpu->arch.reserved_gpa_bits,
w->cpu_role.base.level, is_efer_nx(w),
guest_cpu_cap_has(vcpu, X86_FEATURE_GBPAGES),
@@ -5528,7 +5528,7 @@ static void __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
bool execonly, int huge_page_level)
{
- __reset_rsvds_bits_mask_ept(&vcpu->arch.ngpa_walk.guest_rsvd_check,
+ __reset_rsvds_bits_mask_ept(&vcpu->arch.ngpa_walk.fmt.guest_rsvd_check,
vcpu->arch.reserved_gpa_bits, execonly,
huge_page_level);
}
@@ -5682,7 +5682,7 @@ static void update_permission_bitmask(struct kvm_pagewalk *pw, bool tdp, bool ep
* permission_fault() to indicate accesses that are *not* subject to
* SMAP restrictions.
*/
- for (index = 0; index < ARRAY_SIZE(pw->permissions); ++index) {
+ for (index = 0; index < ARRAY_SIZE(pw->fmt.permissions); ++index) {
unsigned pfec = index << 1;
/*
@@ -5756,7 +5756,7 @@ static void update_permission_bitmask(struct kvm_pagewalk *pw, bool tdp, bool ep
smapf = (pfec & (PFERR_RSVD_MASK|PFERR_FETCH_MASK)) ? 0 : kf;
}
- pw->permissions[index] = ff | uf | wf | rf | smapf;
+ pw->fmt.permissions[index] = ff | uf | wf | rf | smapf;
}
}
@@ -5789,14 +5789,14 @@ static void update_pkru_bitmask(struct kvm_pagewalk *w)
unsigned bit;
bool wp;
- w->pkru_mask = 0;
+ w->fmt.pkru_mask = 0;
if (!is_cr4_pke(w))
return;
wp = is_cr0_wp(w);
- for (bit = 0; bit < ARRAY_SIZE(w->permissions); ++bit) {
+ for (bit = 0; bit < ARRAY_SIZE(w->fmt.permissions); ++bit) {
unsigned pfec, pkey_bits;
bool check_pkey, check_write, ff, uf, wf, pte_user;
@@ -5824,7 +5824,7 @@ static void update_pkru_bitmask(struct kvm_pagewalk *w)
/* PKRU.WD stops write access. */
pkey_bits |= (!!check_write) << 1;
- w->pkru_mask |= (pkey_bits & 3) << pfec;
+ w->fmt.pkru_mask |= (pkey_bits & 3) << pfec;
}
}
@@ -6113,7 +6113,7 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
context->sync_spte = ept_sync_spte;
update_permission_bitmask(ngpa_walk, true, true);
- ngpa_walk->pkru_mask = 0;
+ ngpa_walk->fmt.pkru_mask = 0;
reset_rsvds_bits_mask_ept(vcpu, execonly, huge_page_level);
reset_ept_shadow_zero_bits_mask(context, execonly);
}
diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index a46384b7080f..58397f58320f 100644
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -147,10 +147,10 @@ static bool FNAME(is_bad_mt_xwr)(struct rsvd_bits_validate *rsvd_check, u64 gpte
#endif
}
-static bool FNAME(is_rsvd_bits_set)(struct kvm_pagewalk *w, u64 gpte, int level)
+static bool FNAME(is_rsvd_bits_set)(struct kvm_page_format *fmt, u64 gpte, int level)
{
- return __is_rsvd_bits_set(&w->guest_rsvd_check, gpte, level) ||
- FNAME(is_bad_mt_xwr)(&w->guest_rsvd_check, gpte);
+ return __is_rsvd_bits_set(&fmt->guest_rsvd_check, gpte, level) ||
+ FNAME(is_bad_mt_xwr)(&fmt->guest_rsvd_check, gpte);
}
static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu,
@@ -167,7 +167,7 @@ static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu,
!(gpte & PT_GUEST_ACCESSED_MASK))
goto no_present;
- if (FNAME(is_rsvd_bits_set)(w, gpte, PG_LEVEL_4K))
+ if (FNAME(is_rsvd_bits_set)(&w->fmt, gpte, PG_LEVEL_4K))
goto no_present;
return false;
@@ -427,7 +427,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
if (unlikely(!FNAME(is_present_gpte)(w, pte)))
goto error;
- if (unlikely(FNAME(is_rsvd_bits_set)(w, pte, walker->level))) {
+ if (unlikely(FNAME(is_rsvd_bits_set)(&w->fmt, pte, walker->level))) {
errcode = PFERR_RSVD_MASK | PFERR_PRESENT_MASK;
goto error;
}
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 17/19] KVM: x86/mmu: merge struct rsvd_bits_validate into struct kvm_page_format
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
` (15 preceding siblings ...)
2026-06-24 21:30 ` [PATCH 16/19] KVM: x86/mmu: pull page format to a new struct Paolo Bonzini
@ 2026-06-24 21:31 ` Paolo Bonzini
2026-06-24 21:31 ` [PATCH 18/19] KVM: x86/mmu: parameterize update_permission_bitmask() Paolo Bonzini
` (2 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:31 UTC (permalink / raw)
To: linux-kernel, kvm
Remove one level of indirection, and prepare for using the permission bitmask
machinery for shadow pages as well.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/include/asm/kvm_host.h | 38 +++++------
arch/x86/kvm/mmu/mmu.c | 116 ++++++++++++++++----------------
arch/x86/kvm/mmu/paging_tmpl.h | 8 +--
arch/x86/kvm/mmu/spte.c | 4 +-
arch/x86/kvm/mmu/spte.h | 18 ++---
arch/x86/kvm/vmx/vmx.c | 2 +-
6 files changed, 91 insertions(+), 95 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 08aa1e2da582..b517257a6315 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -487,9 +487,24 @@ struct kvm_pio_request {
#define PT64_ROOT_MAX_LEVEL 5
-struct rsvd_bits_validate {
+struct kvm_page_format {
u64 rsvd_bits_mask[2][PT64_ROOT_MAX_LEVEL];
u64 bad_mt_xwr;
+
+ /*
+ * The pkru_mask indicates if protection key checks are needed. It
+ * consists of 16 domains indexed by page fault error code bits [4:1],
+ * with PFEC.RSVD replaced by ACC_USER_MASK from the page tables.
+ * Each domain has 2 bits which are ANDed with AD and WD from PKRU.
+ */
+ u32 pkru_mask;
+
+ /*
+ * Bitmap; bit set = permission fault
+ * Array index: page fault error code [4:1]
+ * Bit index: pte permissions in ACC_* format
+ */
+ u16 permissions[16];
};
struct kvm_mmu_root_info {
@@ -516,25 +531,6 @@ struct kvm_page_fault;
* and 2-level 32-bit). The kvm_pagewalk structure abstracts the details of the
* current mmu mode.
*/
-struct kvm_page_format {
- struct rsvd_bits_validate guest_rsvd_check;
-
- /*
- * The pkru_mask indicates if protection key checks are needed. It
- * consists of 16 domains indexed by page fault error code bits [4:1],
- * with PFEC.RSVD replaced by ACC_USER_MASK from the page tables.
- * Each domain has 2 bits which are ANDed with AD and WD from PKRU.
- */
- u32 pkru_mask;
-
- /*
- * Bitmap; bit set = permission fault
- * Array index: page fault error code [4:1]
- * Bit index: pte permissions in ACC_* format
- */
- u16 permissions[16];
-};
-
struct kvm_pagewalk {
unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu);
u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
@@ -570,7 +566,7 @@ struct kvm_mmu {
* bits include not only hardware reserved bits but also
* the bits spte never used.
*/
- struct rsvd_bits_validate shadow_zero_check;
+ struct kvm_page_format fmt;
};
enum pmc_type {
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index f70892b15680..03e0bd3c8490 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -4446,7 +4446,7 @@ static int get_sptes_lockless(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes,
static bool get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)
{
u64 sptes[PT64_ROOT_MAX_LEVEL + 1];
- struct rsvd_bits_validate *rsvd_check;
+ struct kvm_page_format *rsvd_check;
int root, leaf, level;
bool reserved = false;
@@ -4467,7 +4467,7 @@ static bool get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)
if (!is_shadow_present_pte(sptes[leaf]))
leaf++;
- rsvd_check = &vcpu->arch.mmu->shadow_zero_check;
+ rsvd_check = &vcpu->arch.mmu->fmt;
for (level = root; level >= leaf; level--)
reserved |= is_rsvd_spte(rsvd_check, sptes[level], level);
@@ -5387,7 +5387,7 @@ static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
#include "paging_tmpl.h"
#undef PTTYPE
-static void __reset_rsvds_bits_mask(struct rsvd_bits_validate *rsvd_check,
+static void __reset_rsvds_bits_mask(struct kvm_page_format *fmt,
u64 pa_bits_rsvd, int level, bool nx,
bool gbpages, bool pse, bool amd)
{
@@ -5395,7 +5395,7 @@ static void __reset_rsvds_bits_mask(struct rsvd_bits_validate *rsvd_check,
u64 nonleaf_bit8_rsvd = 0;
u64 high_bits_rsvd;
- rsvd_check->bad_mt_xwr = 0;
+ fmt->bad_mt_xwr = 0;
if (!gbpages)
gbpages_bit_rsvd = rsvd_bits(7, 7);
@@ -5419,59 +5419,59 @@ static void __reset_rsvds_bits_mask(struct rsvd_bits_validate *rsvd_check,
switch (level) {
case PT32_ROOT_LEVEL:
/* no rsvd bits for 2 level 4K page table entries */
- rsvd_check->rsvd_bits_mask[0][1] = 0;
- rsvd_check->rsvd_bits_mask[0][0] = 0;
- rsvd_check->rsvd_bits_mask[1][0] =
- rsvd_check->rsvd_bits_mask[0][0];
+ fmt->rsvd_bits_mask[0][1] = 0;
+ fmt->rsvd_bits_mask[0][0] = 0;
+ fmt->rsvd_bits_mask[1][0] =
+ fmt->rsvd_bits_mask[0][0];
if (!pse) {
- rsvd_check->rsvd_bits_mask[1][1] = 0;
+ fmt->rsvd_bits_mask[1][1] = 0;
break;
}
if (is_cpuid_PSE36())
/* 36bits PSE 4MB page */
- rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
+ fmt->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
else
/* 32 bits PSE 4MB page */
- rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
+ fmt->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
break;
case PT32E_ROOT_LEVEL:
- rsvd_check->rsvd_bits_mask[0][2] = rsvd_bits(63, 63) |
+ fmt->rsvd_bits_mask[0][2] = rsvd_bits(63, 63) |
high_bits_rsvd |
rsvd_bits(5, 8) |
rsvd_bits(1, 2); /* PDPTE */
- rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd; /* PDE */
- rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd; /* PTE */
- rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |
+ fmt->rsvd_bits_mask[0][1] = high_bits_rsvd; /* PDE */
+ fmt->rsvd_bits_mask[0][0] = high_bits_rsvd; /* PTE */
+ fmt->rsvd_bits_mask[1][1] = high_bits_rsvd |
rsvd_bits(13, 20); /* large page */
- rsvd_check->rsvd_bits_mask[1][0] =
- rsvd_check->rsvd_bits_mask[0][0];
+ fmt->rsvd_bits_mask[1][0] =
+ fmt->rsvd_bits_mask[0][0];
break;
case PT64_ROOT_5LEVEL:
- rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd |
+ fmt->rsvd_bits_mask[0][4] = high_bits_rsvd |
nonleaf_bit8_rsvd |
rsvd_bits(7, 7);
- rsvd_check->rsvd_bits_mask[1][4] =
- rsvd_check->rsvd_bits_mask[0][4];
+ fmt->rsvd_bits_mask[1][4] =
+ fmt->rsvd_bits_mask[0][4];
fallthrough;
case PT64_ROOT_4LEVEL:
- rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd |
+ fmt->rsvd_bits_mask[0][3] = high_bits_rsvd |
nonleaf_bit8_rsvd |
rsvd_bits(7, 7);
- rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd |
+ fmt->rsvd_bits_mask[0][2] = high_bits_rsvd |
gbpages_bit_rsvd;
- rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;
- rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
- rsvd_check->rsvd_bits_mask[1][3] =
- rsvd_check->rsvd_bits_mask[0][3];
- rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd |
+ fmt->rsvd_bits_mask[0][1] = high_bits_rsvd;
+ fmt->rsvd_bits_mask[0][0] = high_bits_rsvd;
+ fmt->rsvd_bits_mask[1][3] =
+ fmt->rsvd_bits_mask[0][3];
+ fmt->rsvd_bits_mask[1][2] = high_bits_rsvd |
gbpages_bit_rsvd |
rsvd_bits(13, 29);
- rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |
+ fmt->rsvd_bits_mask[1][1] = high_bits_rsvd |
rsvd_bits(13, 20); /* large page */
- rsvd_check->rsvd_bits_mask[1][0] =
- rsvd_check->rsvd_bits_mask[0][0];
+ fmt->rsvd_bits_mask[1][0] =
+ fmt->rsvd_bits_mask[0][0];
break;
}
}
@@ -5479,7 +5479,7 @@ static void __reset_rsvds_bits_mask(struct rsvd_bits_validate *rsvd_check,
static void reset_guest_rsvds_bits_mask(struct kvm_vcpu *vcpu,
struct kvm_pagewalk *w)
{
- __reset_rsvds_bits_mask(&w->fmt.guest_rsvd_check,
+ __reset_rsvds_bits_mask(&w->fmt,
vcpu->arch.reserved_gpa_bits,
w->cpu_role.base.level, is_efer_nx(w),
guest_cpu_cap_has(vcpu, X86_FEATURE_GBPAGES),
@@ -5487,7 +5487,7 @@ static void reset_guest_rsvds_bits_mask(struct kvm_vcpu *vcpu,
guest_cpuid_is_amd_compatible(vcpu));
}
-static void __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
+static void __reset_rsvds_bits_mask_ept(struct kvm_page_format *fmt,
u64 pa_bits_rsvd, bool execonly,
int huge_page_level)
{
@@ -5500,18 +5500,18 @@ static void __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
if (huge_page_level < PG_LEVEL_2M)
large_2m_rsvd = rsvd_bits(7, 7);
- rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd | rsvd_bits(3, 7);
- rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd | rsvd_bits(3, 7);
- rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd | rsvd_bits(3, 6) | large_1g_rsvd;
- rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd | rsvd_bits(3, 6) | large_2m_rsvd;
- rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
+ fmt->rsvd_bits_mask[0][4] = high_bits_rsvd | rsvd_bits(3, 7);
+ fmt->rsvd_bits_mask[0][3] = high_bits_rsvd | rsvd_bits(3, 7);
+ fmt->rsvd_bits_mask[0][2] = high_bits_rsvd | rsvd_bits(3, 6) | large_1g_rsvd;
+ fmt->rsvd_bits_mask[0][1] = high_bits_rsvd | rsvd_bits(3, 6) | large_2m_rsvd;
+ fmt->rsvd_bits_mask[0][0] = high_bits_rsvd;
/* large page */
- rsvd_check->rsvd_bits_mask[1][4] = rsvd_check->rsvd_bits_mask[0][4];
- rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];
- rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd | rsvd_bits(12, 29) | large_1g_rsvd;
- rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd | rsvd_bits(12, 20) | large_2m_rsvd;
- rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];
+ fmt->rsvd_bits_mask[1][4] = fmt->rsvd_bits_mask[0][4];
+ fmt->rsvd_bits_mask[1][3] = fmt->rsvd_bits_mask[0][3];
+ fmt->rsvd_bits_mask[1][2] = high_bits_rsvd | rsvd_bits(12, 29) | large_1g_rsvd;
+ fmt->rsvd_bits_mask[1][1] = high_bits_rsvd | rsvd_bits(12, 20) | large_2m_rsvd;
+ fmt->rsvd_bits_mask[1][0] = fmt->rsvd_bits_mask[0][0];
bad_mt_xwr = 0xFFull << (2 * 8); /* bits 3..5 must not be 2 */
bad_mt_xwr |= 0xFFull << (3 * 8); /* bits 3..5 must not be 3 */
@@ -5522,13 +5522,13 @@ static void __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
/* bits 0..2 must not be 100 unless VMX capabilities allow it */
bad_mt_xwr |= REPEAT_BYTE(1ull << 4);
}
- rsvd_check->bad_mt_xwr = bad_mt_xwr;
+ fmt->bad_mt_xwr = bad_mt_xwr;
}
static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
bool execonly, int huge_page_level)
{
- __reset_rsvds_bits_mask_ept(&vcpu->arch.ngpa_walk.fmt.guest_rsvd_check,
+ __reset_rsvds_bits_mask_ept(&vcpu->arch.ngpa_walk.fmt,
vcpu->arch.reserved_gpa_bits, execonly,
huge_page_level);
}
@@ -5550,13 +5550,13 @@ static void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
bool is_amd = true;
/* KVM doesn't use 2-level page tables for the shadow MMU. */
bool is_pse = false;
- struct rsvd_bits_validate *shadow_zero_check;
+ struct kvm_page_format *fmt;
int i;
WARN_ON_ONCE(context->root_role.level < PT32E_ROOT_LEVEL);
- shadow_zero_check = &context->shadow_zero_check;
- __reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(),
+ fmt = &context->fmt;
+ __reset_rsvds_bits_mask(fmt, reserved_hpa_bits(),
context->root_role.level,
context->root_role.efer_nx,
guest_cpu_cap_has(vcpu, X86_FEATURE_GBPAGES),
@@ -5572,10 +5572,10 @@ static void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
* Bits in shadow_me_mask but not in shadow_me_value are
* not allowed to be set.
*/
- shadow_zero_check->rsvd_bits_mask[0][i] |= shadow_me_mask;
- shadow_zero_check->rsvd_bits_mask[1][i] |= shadow_me_mask;
- shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_value;
- shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_value;
+ fmt->rsvd_bits_mask[0][i] |= shadow_me_mask;
+ fmt->rsvd_bits_mask[1][i] |= shadow_me_mask;
+ fmt->rsvd_bits_mask[0][i] &= ~shadow_me_value;
+ fmt->rsvd_bits_mask[1][i] &= ~shadow_me_value;
}
}
@@ -5592,18 +5592,18 @@ static inline bool boot_cpu_is_amd(void)
*/
static void reset_tdp_shadow_zero_bits_mask(struct kvm_mmu *context)
{
- struct rsvd_bits_validate *shadow_zero_check;
+ struct kvm_page_format *fmt;
int i;
- shadow_zero_check = &context->shadow_zero_check;
+ fmt = &context->fmt;
if (boot_cpu_is_amd())
- __reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(),
+ __reset_rsvds_bits_mask(fmt, reserved_hpa_bits(),
context->root_role.level, true,
boot_cpu_has(X86_FEATURE_GBPAGES),
false, true);
else
- __reset_rsvds_bits_mask_ept(shadow_zero_check,
+ __reset_rsvds_bits_mask_ept(fmt,
reserved_hpa_bits(), false,
max_huge_page_level);
@@ -5611,8 +5611,8 @@ static void reset_tdp_shadow_zero_bits_mask(struct kvm_mmu *context)
return;
for (i = context->root_role.level; --i >= 0;) {
- shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask;
- shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask;
+ fmt->rsvd_bits_mask[0][i] &= ~shadow_me_mask;
+ fmt->rsvd_bits_mask[1][i] &= ~shadow_me_mask;
}
}
@@ -5623,7 +5623,7 @@ static void reset_tdp_shadow_zero_bits_mask(struct kvm_mmu *context)
static void
reset_ept_shadow_zero_bits_mask(struct kvm_mmu *context, bool execonly)
{
- __reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
+ __reset_rsvds_bits_mask_ept(&context->fmt,
reserved_hpa_bits(), execonly,
max_huge_page_level);
}
diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index 58397f58320f..e73fc09ec4db 100644
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -138,19 +138,19 @@ static inline int FNAME(is_present_gpte)(struct kvm_pagewalk *w,
#endif
}
-static bool FNAME(is_bad_mt_xwr)(struct rsvd_bits_validate *rsvd_check, u64 gpte)
+static bool FNAME(is_bad_mt_xwr)(struct kvm_page_format *fmt, u64 gpte)
{
#if PTTYPE != PTTYPE_EPT
return false;
#else
- return __is_bad_mt_xwr(rsvd_check, gpte);
+ return __is_bad_mt_xwr(fmt, gpte);
#endif
}
static bool FNAME(is_rsvd_bits_set)(struct kvm_page_format *fmt, u64 gpte, int level)
{
- return __is_rsvd_bits_set(&fmt->guest_rsvd_check, gpte, level) ||
- FNAME(is_bad_mt_xwr)(&fmt->guest_rsvd_check, gpte);
+ return __is_rsvd_bits_set(fmt, gpte, level) ||
+ FNAME(is_bad_mt_xwr)(fmt, gpte);
}
static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu,
diff --git a/arch/x86/kvm/mmu/spte.c b/arch/x86/kvm/mmu/spte.c
index d2f5f7dd8fe1..bdf72a98c19c 100644
--- a/arch/x86/kvm/mmu/spte.c
+++ b/arch/x86/kvm/mmu/spte.c
@@ -280,9 +280,9 @@ bool make_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
if (prefetch && !synchronizing)
spte = mark_spte_for_access_track(spte);
- WARN_ONCE(is_rsvd_spte(&vcpu->arch.mmu->shadow_zero_check, spte, level),
+ WARN_ONCE(is_rsvd_spte(&vcpu->arch.mmu->fmt, spte, level),
"spte = 0x%llx, level = %d, rsvd bits = 0x%llx", spte, level,
- get_rsvd_bits(&vcpu->arch.mmu->shadow_zero_check, spte, level));
+ get_rsvd_bits(&vcpu->arch.mmu->fmt, spte, level));
/*
* Mark the memslot dirty *after* modifying it for access tracking.
diff --git a/arch/x86/kvm/mmu/spte.h b/arch/x86/kvm/mmu/spte.h
index 13eea94dd212..918533e61b98 100644
--- a/arch/x86/kvm/mmu/spte.h
+++ b/arch/x86/kvm/mmu/spte.h
@@ -378,33 +378,33 @@ static inline bool is_accessed_spte(u64 spte)
return spte & shadow_accessed_mask;
}
-static inline u64 get_rsvd_bits(struct rsvd_bits_validate *rsvd_check, u64 pte,
+static inline u64 get_rsvd_bits(struct kvm_page_format *fmt, u64 pte,
int level)
{
int bit7 = (pte >> 7) & 1;
- return rsvd_check->rsvd_bits_mask[bit7][level-1];
+ return fmt->rsvd_bits_mask[bit7][level-1];
}
-static inline bool __is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check,
+static inline bool __is_rsvd_bits_set(struct kvm_page_format *fmt,
u64 pte, int level)
{
- return pte & get_rsvd_bits(rsvd_check, pte, level);
+ return pte & get_rsvd_bits(fmt, pte, level);
}
-static inline bool __is_bad_mt_xwr(struct rsvd_bits_validate *rsvd_check,
+static inline bool __is_bad_mt_xwr(struct kvm_page_format *fmt,
u64 pte)
{
if (pte & VMX_EPT_USER_EXECUTABLE_MASK)
pte |= VMX_EPT_EXECUTABLE_MASK;
- return rsvd_check->bad_mt_xwr & BIT_ULL(pte & 0x3f);
+ return fmt->bad_mt_xwr & BIT_ULL(pte & 0x3f);
}
-static __always_inline bool is_rsvd_spte(struct rsvd_bits_validate *rsvd_check,
+static __always_inline bool is_rsvd_spte(struct kvm_page_format *fmt,
u64 spte, int level)
{
- return __is_bad_mt_xwr(rsvd_check, spte) ||
- __is_rsvd_bits_set(rsvd_check, spte, level);
+ return __is_bad_mt_xwr(fmt, spte) ||
+ __is_rsvd_bits_set(fmt, spte, level);
}
/*
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index aded7039bd3e..3681d565f177 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -8719,7 +8719,7 @@ __init int vmx_hardware_setup(void)
/*
* Setup shadow_me_value/shadow_me_mask to include MKTME KeyID
- * bits to shadow_zero_check.
+ * bits into the MMU's struct kvm_page_format.
*/
vmx_setup_me_spte_mask();
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 18/19] KVM: x86/mmu: parameterize update_permission_bitmask()
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
` (16 preceding siblings ...)
2026-06-24 21:31 ` [PATCH 17/19] KVM: x86/mmu: merge struct rsvd_bits_validate into struct kvm_page_format Paolo Bonzini
@ 2026-06-24 21:31 ` Paolo Bonzini
2026-06-24 21:31 ` [PATCH 19/19] KVM: x86/mmu: use kvm_page_format to test SPTEs Paolo Bonzini
2026-07-15 14:43 ` [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:31 UTC (permalink / raw)
To: linux-kernel, kvm
Make it possible to apply the computation loop to both guest
and shadow PTEs formats; the latter do not have an extended role, so
pass the four parameters to the function one by one.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/kvm/mmu/mmu.c | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 03e0bd3c8490..11d86e34acf4 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -5658,18 +5658,15 @@ reset_ept_shadow_zero_bits_mask(struct kvm_mmu *context, bool execonly)
(14 & (access) ? 1 << 14 : 0) | \
(15 & (access) ? 1 << 15 : 0))
-static void update_permission_bitmask(struct kvm_pagewalk *pw, bool tdp, bool ept)
+static void __update_permission_bitmask(struct kvm_page_format *fmt, bool tdp,
+ bool ept, bool cr4_smep, bool cr4_smap,
+ bool cr0_wp, bool efer_nx)
{
unsigned index;
const u16 w = ACC_BITS_MASK(ACC_WRITE_MASK);
const u16 r = ACC_BITS_MASK(ACC_READ_MASK);
- bool cr4_smep = is_cr4_smep(pw);
- bool cr4_smap = is_cr4_smap(pw);
- bool cr0_wp = is_cr0_wp(pw);
- bool efer_nx = is_efer_nx(pw);
-
/*
* In hardware, page fault error codes are generated (as the name
* suggests) on any kind of page fault. permission_fault() and
@@ -5682,7 +5679,7 @@ static void update_permission_bitmask(struct kvm_pagewalk *pw, bool tdp, bool ep
* permission_fault() to indicate accesses that are *not* subject to
* SMAP restrictions.
*/
- for (index = 0; index < ARRAY_SIZE(pw->fmt.permissions); ++index) {
+ for (index = 0; index < ARRAY_SIZE(fmt->permissions); ++index) {
unsigned pfec = index << 1;
/*
@@ -5756,10 +5753,17 @@ static void update_permission_bitmask(struct kvm_pagewalk *pw, bool tdp, bool ep
smapf = (pfec & (PFERR_RSVD_MASK|PFERR_FETCH_MASK)) ? 0 : kf;
}
- pw->fmt.permissions[index] = ff | uf | wf | rf | smapf;
+ fmt->permissions[index] = ff | uf | wf | rf | smapf;
}
}
+static void update_permission_bitmask(struct kvm_pagewalk *w, bool tdp, bool ept)
+{
+ __update_permission_bitmask(&w->fmt, tdp, ept,
+ is_cr4_smep(w), is_cr4_smap(w),
+ is_cr0_wp(w), is_efer_nx(w));
+}
+
/*
* PKU is an additional mechanism by which the paging controls access to
* user-mode addresses based on the value in the PKRU register. Protection
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 19/19] KVM: x86/mmu: use kvm_page_format to test SPTEs
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
` (17 preceding siblings ...)
2026-06-24 21:31 ` [PATCH 18/19] KVM: x86/mmu: parameterize update_permission_bitmask() Paolo Bonzini
@ 2026-06-24 21:31 ` Paolo Bonzini
2026-07-15 14:43 ` [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-06-24 21:31 UTC (permalink / raw)
To: linux-kernel, kvm
is_access_allowed(), and is_executable_pte() within it, are effectively
a special version of permission_fault() that only supports a subset
of roles. In particular it does not allow SMEP, SMAP and PKE.
Replace its implementation with a modified version of permission_fault();
the new version will support SMEP (and hence AMD GMET) for free as soon
as update_spte_permission_bitmask() stops hardcoding cr4_smep == false.
This prepares for a possible future where TDP entries could have XS!=XU,
for example as part of implementing Hyper-V VSM natively inside KVM.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/kvm/mmu/mmu.c | 18 +++++++++++---
arch/x86/kvm/mmu/spte.h | 51 ++++++++++++++++++++++----------------
arch/x86/kvm/mmu/tdp_mmu.c | 3 ++-
3 files changed, 47 insertions(+), 25 deletions(-)
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 11d86e34acf4..6c13da942bfc 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -3694,6 +3694,7 @@ static u64 *fast_pf_get_last_sptep(struct kvm_vcpu *vcpu, gpa_t gpa, u64 *spte)
*/
static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
{
+ struct kvm_mmu *mmu;
struct kvm_mmu_page *sp;
int ret = RET_PF_INVALID;
u64 spte;
@@ -3703,6 +3704,7 @@ static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
if (!page_fault_can_be_fast(vcpu->kvm, fault))
return ret;
+ mmu = vcpu->arch.mmu;
walk_shadow_page_lockless_begin(vcpu);
do {
@@ -3738,7 +3740,7 @@ static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
* Need not check the access of upper level table entries since
* they are always ACC_ALL.
*/
- if (is_access_allowed(fault, spte)) {
+ if (!spte_permission_fault(mmu, spte, fault)) {
ret = RET_PF_SPURIOUS;
break;
}
@@ -3761,7 +3763,7 @@ static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
* that were write-protected for dirty-logging or access
* tracking are handled here. Don't bother checking if the
* SPTE is writable to prioritize running with A/D bits enabled.
- * The is_access_allowed() check above handles the common case
+ * The spte_permission_fault() check above handles the common case
* of the fault being spurious, and the SPTE is known to be
* shadow-present, i.e. except for access tracking restoration
* making the new SPTE writable, the check is wasteful.
@@ -3786,7 +3788,7 @@ static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
/* Verify that the fault can be handled in the fast path */
if (new_spte == spte ||
- !is_access_allowed(fault, new_spte))
+ spte_permission_fault(mmu, new_spte, fault))
break;
/*
@@ -5764,6 +5766,12 @@ static void update_permission_bitmask(struct kvm_pagewalk *w, bool tdp, bool ept
is_cr0_wp(w), is_efer_nx(w));
}
+static void update_spte_permission_bitmask(struct kvm_mmu *mmu, bool tdp, bool ept)
+{
+ __update_permission_bitmask(&mmu->fmt, tdp, ept,
+ mmu->root_role.cr4_smep, false, true, true);
+}
+
/*
* PKU is an additional mechanism by which the paging controls access to
* user-mode addresses based on the value in the PKRU register. Protection
@@ -5973,6 +5981,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
context->page_fault = kvm_tdp_page_fault;
context->sync_spte = NULL;
+ update_spte_permission_bitmask(context, true, shadow_xs_mask);
reset_tdp_shadow_zero_bits_mask(context);
}
@@ -5991,6 +6000,7 @@ static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *conte
else
paging32_init_context(context);
+ update_spte_permission_bitmask(context, context == &vcpu->arch.guest_mmu, false);
reset_shadow_zero_bits_mask(vcpu, context);
}
@@ -6119,6 +6129,8 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
update_permission_bitmask(ngpa_walk, true, true);
ngpa_walk->fmt.pkru_mask = 0;
reset_rsvds_bits_mask_ept(vcpu, execonly, huge_page_level);
+
+ update_spte_permission_bitmask(context, true, true);
reset_ept_shadow_zero_bits_mask(context, execonly);
}
diff --git a/arch/x86/kvm/mmu/spte.h b/arch/x86/kvm/mmu/spte.h
index 918533e61b98..e730717824b3 100644
--- a/arch/x86/kvm/mmu/spte.h
+++ b/arch/x86/kvm/mmu/spte.h
@@ -357,17 +357,6 @@ static inline bool is_last_spte(u64 pte, int level)
return (level == PG_LEVEL_4K) || is_large_pte(pte);
}
-static inline bool is_executable_pte(u64 spte)
-{
- /*
- * For now, return true if either the XS or XU bit is set
- * This function is only used for fast_page_fault,
- * which never processes shadow EPT, and regular page
- * tables always have XS==XU.
- */
- return (spte & (shadow_xs_mask | shadow_xu_mask | shadow_nx_mask)) != shadow_nx_mask;
-}
-
static inline kvm_pfn_t spte_to_pfn(u64 pte)
{
return (pte & SPTE_BASE_ADDR_MASK) >> PAGE_SHIFT;
@@ -496,20 +485,40 @@ static inline bool is_mmu_writable_spte(u64 spte)
}
/*
- * Returns true if the access indicated by @fault is allowed by the existing
- * SPTE protections. Note, the caller is responsible for checking that the
- * SPTE is a shadow-present, leaf SPTE (either before or after).
+ * Returns true if the access indicated by @fault is forbidden by the existing
+ * SPTE protections.
*/
-static inline bool is_access_allowed(struct kvm_page_fault *fault, u64 spte)
+static inline bool spte_permission_fault(struct kvm_mmu *mmu, u64 spte,
+ struct kvm_page_fault *fault)
{
- if (fault->exec)
- return is_executable_pte(spte);
+ unsigned pfec, pte_access;
- if (fault->write)
- return is_writable_pte(spte);
+ if (!is_shadow_present_pte(spte))
+ return true;
- /* Fault was on Read access */
- return spte & PT_PRESENT_MASK;
+ BUILD_BUG_ON(PT_PRESENT_MASK != ACC_READ_MASK);
+ BUILD_BUG_ON(PT_WRITABLE_MASK != ACC_WRITE_MASK);
+ BUILD_BUG_ON(VMX_EPT_READABLE_MASK != ACC_READ_MASK);
+ BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != ACC_WRITE_MASK);
+
+ /* strip nested paging fault error codes */
+ pte_access = spte & (PT_PRESENT_MASK | PT_WRITABLE_MASK);
+ if (shadow_nx_mask) {
+ pte_access |= spte & shadow_user_mask ? ACC_USER_MASK : 0;
+ pte_access |= spte & shadow_nx_mask ? 0 : ACC_EXEC_MASK;
+ } else {
+ pte_access |= spte & shadow_xs_mask ? ACC_EXEC_MASK : 0;
+ pte_access |= spte & shadow_xu_mask ? ACC_USER_EXEC_MASK : 0;
+ }
+
+ /*
+ * RSVD is handled elsewhere, and is used for SMAP in the context
+ * of accessing fmt.permissions[]. SPTEs never use PK or SS, as
+ * they are not supported for shadow paging and irrelevant for TDP.
+ */
+ pfec = fault->error_code & (
+ PFERR_WRITE_MASK | PFERR_USER_MASK | PFERR_FETCH_MASK);
+ return (mmu->fmt.permissions[pfec >> 1] >> pte_access) & 1;
}
/*
diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
index c1cbae65d239..ce3f2efadb05 100644
--- a/arch/x86/kvm/mmu/tdp_mmu.c
+++ b/arch/x86/kvm/mmu/tdp_mmu.c
@@ -1122,6 +1122,7 @@ static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu,
struct kvm_page_fault *fault,
struct tdp_iter *iter)
{
+ struct kvm_mmu *mmu = vcpu->arch.mmu;
struct kvm_mmu_page *sp = sptep_to_sp(rcu_dereference(iter->sptep));
u64 new_spte;
int ret = RET_PF_FIXED;
@@ -1131,7 +1132,7 @@ static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu,
return RET_PF_RETRY;
if (is_shadow_present_pte(iter->old_spte) &&
- (fault->prefetch || is_access_allowed(fault, iter->old_spte)) &&
+ (fault->prefetch || !spte_permission_fault(mmu, iter->old_spte, fault)) &&
is_last_spte(iter->old_spte, iter->level)) {
WARN_ON_ONCE(fault->pfn != spte_to_pfn(iter->old_spte));
return RET_PF_SPURIOUS;
--
2.52.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu
2026-06-24 21:30 [PATCH v2 00/19] KVM: apply chainsaw to struct kvm_mmu Paolo Bonzini
` (18 preceding siblings ...)
2026-06-24 21:31 ` [PATCH 19/19] KVM: x86/mmu: use kvm_page_format to test SPTEs Paolo Bonzini
@ 2026-07-15 14:43 ` Paolo Bonzini
19 siblings, 0 replies; 21+ messages in thread
From: Paolo Bonzini @ 2026-07-15 14:43 UTC (permalink / raw)
To: Paolo Bonzini; +Cc: linux-kernel, kvm
Queued, thanks.
Paolo
^ permalink raw reply [flat|nested] 21+ messages in thread