[RFC PATCH 05/47] mm: asi: Make __get_current_cr3_fast() ASI-aware

Junaid Shahid
2022-02-23T05:24:16Z

When ASI is active, __get_current_cr3_fast() adjusts the returned CR3
value accordingly to reflect the actual ASI CR3.

Signed-off-by: Junaid Shahid <junaids@google.com>

---
arch/x86/include/asm/asi.h | 7 +++++++
arch/x86/mm/tlb.c | 20 ++++++++++++++++++--
2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/asi.h b/arch/x86/include/asm/asi.h
index 7702332c62e8..95557211dabd 100644
--- a/arch/x86/include/asm/asi.h
+++ b/arch/x86/include/asm/asi.h
@@ -112,6 +112,11 @@ static inline void asi_intr_exit(void)
}
}

+static inline pgd_t *asi_pgd(struct asi *asi)
+{
+ return asi->pgd;
+}
+
#else /* CONFIG_ADDRESS_SPACE_ISOLATION */

static inline void asi_intr_enter(void) { }
@@ -120,6 +125,8 @@ static inline void asi_intr_exit(void) { }

static inline void asi_init_thread_state(struct thread_struct *thread) { }

+static inline pgd_t *asi_pgd(struct asi *asi) { return NULL; }
+
#endif /* CONFIG_ADDRESS_SPACE_ISOLATION */

#endif
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 88d9298720dc..25bee959d1d3 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -17,6 +17,7 @@
#include <asm/cacheflush.h>
#include <asm/apic.h>
#include <asm/perf_event.h>
+#include <asm/asi.h>

#include "mm_internal.h"

@@ -1073,12 +1074,27 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
*/
unsigned long __get_current_cr3_fast(void)
{
- unsigned long cr3 = build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd,
- this_cpu_read(cpu_tlbstate.loaded_mm_asid));
+ unsigned long cr3;
+ pgd_t *pgd;
+ u16 asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
+ struct asi *asi = asi_get_current();
+
+ if (asi)
+ pgd = asi_pgd(asi);
+ else
+ pgd = this_cpu_read(cpu_tlbstate.loaded_mm)->pgd;
+
+ cr3 = build_cr3(pgd, asid);

/* For now, be very restrictive about when this can be called. */
VM_WARN_ON(in_nmi() || preemptible());

+ /*
+ * CR3 is unstable if the target ASI is unrestricted
+ * and a restricted ASI is currently loaded.
+ */
+ VM_WARN_ON_ONCE(asi && asi_is_target_unrestricted());
+
VM_BUG_ON(cr3 != __read_cr3());
return cr3;
}
--
2.35.1.473.g83b2b277ed-goog