[RFC PATCH 8/9] x86/mm: use RAR for kernel TLB flushes

Rik van Riel
2025-05-06T00:49:10Z

From: Rik van Riel <riel@fb.com>

Use Intel RAR for kernel TLB flushes, when enabled.

Pass in PCID 0 to smp_call_rar_many() to flush kernel memory.

Unfortunately RAR_INVPCID_ADDR excludes global PTE mappings, so only
full flushes with RAR_INVPCID_ALL will flush kernel mappings.

Signed-off-by: Rik van Riel <riel@surriel.com>
---
arch/x86/mm/tlb.c | 38 ++++++++++++++++++++++++++++++++++++++
1 file changed, 38 insertions(+)

diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 7c61bf11d472..a4f3941281b6 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -21,6 +21,7 @@
#include <asm/apic.h>
#include <asm/msr.h>
#include <asm/perf_event.h>
+#include <asm/rar.h>
#include <asm/tlb.h>

#include "mm_internal.h"
@@ -1451,6 +1452,18 @@ static void do_flush_tlb_all(void *info)
__flush_tlb_all();
}

+static void rar_full_flush(const cpumask_t *cpumask)
+{
+ guard(preempt)();
+ smp_call_rar_many(cpumask, 0, 0, TLB_FLUSH_ALL);
+ invpcid_flush_all();
+}
+
+static void rar_flush_all(void)
+{
+ rar_full_flush(cpu_online_mask);
+}
+
void flush_tlb_all(void)
{
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
@@ -1458,6 +1471,8 @@ void flush_tlb_all(void)
/* First try (faster) hardware-assisted TLB invalidation. */
if (cpu_feature_enabled(X86_FEATURE_INVLPGB))
invlpgb_flush_all();
+ else if (cpu_feature_enabled(X86_FEATURE_RAR))
+ rar_flush_all();
else
/* Fall back to the IPI-based invalidation. */
on_each_cpu(do_flush_tlb_all, NULL, 1);
@@ -1487,15 +1502,36 @@ static void do_kernel_range_flush(void *info)
struct flush_tlb_info *f = info;
unsigned long addr;

+ /*
+ * With PTI kernel TLB entries in all PCIDs need to be flushed.
+ * With RAR the PCID space becomes so large, we might as well flush it all.
+ *
+ * Either of the two by itself works with targeted flushes.
+ */
+ if (cpu_feature_enabled(X86_FEATURE_RAR) &&
+ cpu_feature_enabled(X86_FEATURE_PTI)) {
+ invpcid_flush_all();
+ return;
+ }
+
/* flush range by one by one 'invlpg' */
for (addr = f->start; addr < f->end; addr += PAGE_SIZE)
flush_tlb_one_kernel(addr);
}

+static void rar_kernel_range_flush(struct flush_tlb_info *info)
+{
+ guard(preempt)();
+ smp_call_rar_many(cpu_online_mask, 0, info->start, info->end);
+ do_kernel_range_flush(info);
+}
+
static void kernel_tlb_flush_all(struct flush_tlb_info *info)
{
if (cpu_feature_enabled(X86_FEATURE_INVLPGB))
invlpgb_flush_all();
+ else if (cpu_feature_enabled(X86_FEATURE_RAR))
+ rar_flush_all();
else
on_each_cpu(do_flush_tlb_all, NULL, 1);
}
@@ -1504,6 +1540,8 @@ static void kernel_tlb_flush_range(struct flush_tlb_info *info)
{
if (cpu_feature_enabled(X86_FEATURE_INVLPGB))
invlpgb_kernel_range_flush(info);
+ else if (cpu_feature_enabled(X86_FEATURE_RAR))
+ rar_kernel_range_flush(info);
else
on_each_cpu(do_kernel_range_flush, info, 1);
}
--
2.49.0