[PATCH 3/4] target/i386: emulate LAM in TCG

[Tim <mdspan.github@gmail.com>]

Linear Address Masking (LAM) lets software keep metadata in bits
62:57 or 62:48 of 64-bit data addresses.  CR3.LAM_U57 and CR3.LAM_U48
enable it for user addresses, and CR4.LAM_SUP for supervisor
addresses.  Implement it in TCG and add it to TCG_7_1_EAX_FEATURES.

get_physical_address() masks the address and runs the canonical check
and the page walk on the result.  Since masking keeps bit 63, this is
the canonical check that the SDM specifies for LAM.  CR2 receives the
masked address on #PF.  The TLB entry is installed under the tagged
address, which is the same way target/arm does it for top-byte-ignore.
Instruction fetches are not masked, so if masking changes the address,
an entry filled by a data access is not executable and one filled by
a fetch is not readable or writable.

Since each tag of a page has its own entry, INVLPG and INVLPGA flush
every tagged alias with tlb_flush_page_bits_by_mmuidx(), and a change
of CR4.LAM_SUP flushes the TLB.

MOV to CR3 and VMRUN accept the CR3 LAM bits if CPUID reports LAM.
Masking checks CPUID too, since RSM and gdbstub load CR3 without
checking reserved bits.

x86_cpu_translate_for_debug() masks addresses as for a data access,
for gdbstub, monitor, and qemu_plugin_translate_vaddr().  With LAM_U48
and 5-level paging, it therefore translates an instruction address at
or above 2^47 differently from a fetch.

Signed-off-by: Tim <mdspan.github@gmail.com>
---
 target/i386/cpu.c                    |  2 +-
 target/i386/cpu.h                    | 46 ++++++++++++++++++++++++++++
 target/i386/helper.c                 |  6 +++-
 target/i386/tcg/system/excp_helper.c | 30 ++++++++++++++++--
 target/i386/tcg/system/misc_helper.c | 24 +++++++++++++--
 target/i386/tcg/system/svm_helper.c  |  3 +-
 6 files changed, 102 insertions(+), 9 deletions(-)

diff --git a/target/i386/cpu.c b/target/i386/cpu.c
index eab0d5a..ad5d90d 100644
--- a/target/i386/cpu.c
+++ b/target/i386/cpu.c
@@ -1006,7 +1006,7 @@ void x86_cpu_vendor_words2str(char *dst, uint32_t vendor1,
 #define TCG_7_0_EDX_FEATURES (CPUID_7_0_EDX_FSRM | CPUID_7_0_EDX_KERNEL_FEATURES)
 
 #define TCG_7_1_EAX_FEATURES (CPUID_7_1_EAX_FZRM | CPUID_7_1_EAX_FSRS | \
-          CPUID_7_1_EAX_FSRC | CPUID_7_1_EAX_CMPCCXADD)
+          CPUID_7_1_EAX_FSRC | CPUID_7_1_EAX_CMPCCXADD | CPUID_7_1_EAX_LAM)
 #define TCG_7_1_ECX_FEATURES 0
 #define TCG_7_1_EDX_FEATURES CPUID_7_1_EDX_APXF
 #define TCG_7_2_EDX_FEATURES 0
diff --git a/target/i386/cpu.h b/target/i386/cpu.h
index 3945e90..f196d93 100644
--- a/target/i386/cpu.h
+++ b/target/i386/cpu.h
@@ -260,6 +260,9 @@ typedef enum X86Seg {
 #define CR0_CD_MASK  (1U << 30)
 #define CR0_PG_MASK  (1U << 31)
 
+#define CR3_LAM_U57_MASK (1ULL << 61)
+#define CR3_LAM_U48_MASK (1ULL << 62)
+
 #define CR4_VME_MASK  (1U << 0)
 #define CR4_PVI_MASK  (1U << 1)
 #define CR4_TSD_MASK  (1U << 2)
@@ -3111,6 +3114,49 @@ static inline uint64_t cr4_reserved_bits(CPUX86State *env)
     return reserved_bits;
 }
 
+/* Reserved CR3 bits in IA-32e mode */
+static inline uint64_t cr3_reserved_bits(CPUX86State *env)
+{
+    uint64_t reserved_bits = ~0ULL << env_archcpu(env)->phys_bits;
+
+    if (env->features[FEAT_7_1_EAX] & CPUID_7_1_EAX_LAM) {
+        reserved_bits &= ~(CR3_LAM_U57_MASK | CR3_LAM_U48_MASK);
+    }
+    return reserved_bits;
+}
+
+/* LAM width of @addr: 57 or 48, or 64 without LAM. IA-32e mode only. */
+static inline unsigned x86_lam_addr_bits(CPUX86State *env, uint64_t addr)
+{
+    if (!(env->features[FEAT_7_1_EAX] & CPUID_7_1_EAX_LAM)) {
+        return 64;
+    }
+    if (addr >> 63) {
+        if (!(env->cr[4] & CR4_LAM_SUP_MASK)) {
+            return 64;
+        }
+        return env->cr[4] & CR4_LA57_MASK ? 57 : 48;
+    }
+    if (env->cr[3] & CR3_LAM_U57_MASK) {
+        return 57;
+    }
+    if (env->cr[3] & CR3_LAM_U48_MASK) {
+        return 48;
+    }
+    return 64;
+}
+
+/* @addr with its LAM metadata masked, keeping bit 63. IA-32e mode only. */
+static inline uint64_t x86_lam_mask_addr(CPUX86State *env, uint64_t addr)
+{
+    unsigned lam_bits = x86_lam_addr_bits(env, addr);
+
+    if (lam_bits == 64) {
+        return addr;
+    }
+    return deposit64(sextract64(addr, 0, lam_bits), 63, 1, addr >> 63);
+}
+
 static inline bool ctl_has_irq(CPUX86State *env)
 {
     uint32_t int_prio;
diff --git a/target/i386/helper.c b/target/i386/helper.c
index 723e87f..c680e01 100644
--- a/target/i386/helper.c
+++ b/target/i386/helper.c
@@ -204,7 +204,8 @@ void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4)
 #endif
     if ((new_cr4 ^ env->cr[4]) &
         (CR4_PGE_MASK | CR4_PAE_MASK | CR4_PSE_MASK |
-         CR4_SMEP_MASK | CR4_SMAP_MASK | CR4_LA57_MASK)) {
+         CR4_SMEP_MASK | CR4_SMAP_MASK | CR4_LA57_MASK |
+         CR4_LAM_SUP_MASK)) {
         tlb_flush(env_cpu(env));
     }
 
@@ -290,6 +291,9 @@ bool x86_cpu_translate_for_debug(CPUState *cs, vaddr addr,
             uint64_t pml4e_addr, pml4e;
             int32_t sext;
 
+            /* LAM: translate as for a data access */
+            addr = x86_lam_mask_addr(env, addr);
+
             /* test virtual address sign extension */
             sext = la57 ? (int64_t)addr >> 56 : (int64_t)addr >> 47;
             if (sext != 0 && sext != -1) {
diff --git a/target/i386/tcg/system/excp_helper.c b/target/i386/tcg/system/excp_helper.c
index 6c48ccb..1c18eb7 100644
--- a/target/i386/tcg/system/excp_helper.c
+++ b/target/i386/tcg/system/excp_helper.c
@@ -581,15 +581,35 @@ static bool get_physical_address(CPUX86State *env, vaddr addr,
         }
 
         if (likely(env->cr[0] & CR0_PG_MASK || use_stage2)) {
+            int lam_prot_mask = 0;
+
             in.cr3 = env->cr[3];
             in.mmu_idx = mmu_idx;
             in.ptw_idx = use_stage2 ? MMU_NESTED_IDX : MMU_PHYS_IDX;
             in.pg_mode = get_pg_mode(env);
 
             if (in.pg_mode & PG_MODE_LMA) {
-                /* test virtual address sign extension */
+                vaddr masked = x86_lam_mask_addr(env, addr);
                 int shift = in.pg_mode & PG_MODE_LA57 ? 56 : 47;
-                int64_t sext = (int64_t)addr >> shift;
+                int64_t sext;
+
+                /*
+                 * LAM masks data addresses but not fetches.  Both install
+                 * the entry under the unmasked address, so if masking
+                 * changes it, keep only the permissions of this access.
+                 */
+                if (masked != addr) {
+                    if (access_type == MMU_INST_FETCH) {
+                        lam_prot_mask = PAGE_READ | PAGE_WRITE;
+                    } else {
+                        lam_prot_mask = PAGE_EXEC;
+                        addr = masked;
+                        in.addr = addr;
+                    }
+                }
+
+                /* test virtual address sign extension */
+                sext = (int64_t)addr >> shift;
                 if (sext != 0 && sext != -1) {
                     *err = (TranslateFault){
                         .exception_index = EXCP0D_GPF,
@@ -599,7 +619,11 @@ static bool get_physical_address(CPUX86State *env, vaddr addr,
                     return false;
                 }
             }
-            return mmu_translate(env, &in, out, err, ra);
+            if (!mmu_translate(env, &in, out, err, ra)) {
+                return false;
+            }
+            out->prot &= ~lam_prot_mask;
+            return true;
         }
         break;
     }
diff --git a/target/i386/tcg/system/misc_helper.c b/target/i386/tcg/system/misc_helper.c
index a77da59..be4f553 100644
--- a/target/i386/tcg/system/misc_helper.c
+++ b/target/i386/tcg/system/misc_helper.c
@@ -90,8 +90,7 @@ void HELPER(write_crN)(CPUX86State *env, int reg, target_ulong t0)
         cpu_x86_update_cr0(env, t0);
         break;
     case 3:
-        if ((env->efer & MSR_EFER_LMA) &&
-                (t0 & ((~0ULL) << env_archcpu(env)->phys_bits))) {
+        if ((env->efer & MSR_EFER_LMA) && (t0 & cr3_reserved_bits(env))) {
             cpu_vmexit(env, SVM_EXIT_ERR, 0, GETPC());
         }
         if (!(env->efer & MSR_EFER_LMA)) {
@@ -504,6 +503,27 @@ void HELPER(rdmsr)(CPUX86State *env)
 
 void HELPER(flush_page)(CPUX86State *env, target_ulong addr)
 {
+#ifdef TARGET_X86_64
+    if (env->hflags & HF_LMA_MASK) {
+        int shift = env->cr[4] & CR4_LA57_MASK ? 56 : 47;
+        int64_t sext = (int64_t)addr >> shift;
+        unsigned lam_bits;
+
+        /* no LAM for the operand: non-canonical is a no-op */
+        if (sext != 0 && sext != -1) {
+            return;
+        }
+
+        /* TLB entries are per tag: flush every tagged alias */
+        lam_bits = x86_lam_addr_bits(env, addr);
+        if (lam_bits < 64) {
+            tlb_flush_page_bits_by_mmuidx(env_cpu(env), addr,
+                                          MAKE_64BIT_MASK(0, NB_MMU_MODES),
+                                          lam_bits);
+            return;
+        }
+    }
+#endif
     tlb_flush_page(env_cpu(env), addr);
 }
 
diff --git a/target/i386/tcg/system/svm_helper.c b/target/i386/tcg/system/svm_helper.c
index 9c04042..0e3cf6f 100644
--- a/target/i386/tcg/system/svm_helper.c
+++ b/target/i386/tcg/system/svm_helper.c
@@ -315,8 +315,7 @@ void HELPER(vmrun)(CPUX86State *env, int aflag, int next_eip_addend)
         cpu_vmexit(env, SVM_EXIT_ERR, 0, GETPC());
     }
     new_cr3 = x86_ldq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.cr3));
-    if ((env->efer & MSR_EFER_LMA) &&
-            (new_cr3 & ((~0ULL) << cpu->phys_bits))) {
+    if ((env->efer & MSR_EFER_LMA) && (new_cr3 & cr3_reserved_bits(env))) {
         cpu_vmexit(env, SVM_EXIT_ERR, 0, GETPC());
     }
     new_cr4 = x86_ldq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.cr4));
-- 
2.47.3