Re: [RFC PATCH v2 4/4] arch/x86: L1D flush, optimize the context switch

Singh, Balbir
2020-03-25T07:15:11Z

[-- Attachment #1: Type: text/plain, Size: 354 bytes --]

On Wed, 2020-03-25 at 18:11 +1100, Balbir Singh wrote:
> Use a static branch/jump label to optimize the code. Right now
> we don't ref count the users, but that could be done if needed
> in the future.
>
> Signed-off-by: Balbir Singh <sblbir@amazon.com>
>

I sent an older version of the patch, here is the updated version

Balbir Singh

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: v2-0004-arch-x86-L1D-flush-optimize-the-context-switch.patch --]
[-- Type: text/x-patch; name="v2-0004-arch-x86-L1D-flush-optimize-the-context-switch.patch", Size: 3493 bytes --]

From fa80e9691202183a2879a013e497f221b22305a6 Mon Sep 17 00:00:00 2001
From: Balbir Singh <sblbir@amazon.com>
Date: Wed, 25 Mar 2020 16:41:18 +1100
Subject: [RFC PATCH v2 4/4] arch/x86: L1D flush, optimize the context switch

Use a static branch/jump label to optimize the code. Right now
we don't ref count the users, but that could be done if needed
in the future.

Signed-off-by: Balbir Singh <sblbir@amazon.com>
---
arch/x86/include/asm/nospec-branch.h | 2 ++
arch/x86/mm/tlb.c | 52 +++++++++++++++++-----------
2 files changed, 34 insertions(+), 20 deletions(-)

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 07e95dcb40ad..371e28cea1f4 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -310,6 +310,8 @@ DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
DECLARE_STATIC_KEY_FALSE(mds_user_clear);
DECLARE_STATIC_KEY_FALSE(mds_idle_clear);

+DECLARE_STATIC_KEY_FALSE(switch_mm_l1d_flush);
+
#include <asm/segment.h>

/**
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 22f96c5f74f0..8f272e5921ce 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -155,6 +155,11 @@ EXPORT_SYMBOL_GPL(leave_mm);
static void *l1d_flush_pages;
static DEFINE_MUTEX(l1d_flush_mutex);

+/* Flush L1D on switch_mm() */
+DEFINE_STATIC_KEY_FALSE(switch_mm_l1d_flush);
+EXPORT_SYMBOL_GPL(switch_mm_l1d_flush);
+
+
int enable_l1d_flush_for_task(struct task_struct *tsk)
{
struct page *page;
@@ -170,6 +175,11 @@ int enable_l1d_flush_for_task(struct task_struct *tsk)
l1d_flush_pages = alloc_l1d_flush_pages();
if (!l1d_flush_pages)
return -ENOMEM;
+ /*
+ * We could do more accurate ref counting
+ * if needed
+ */
+ static_branch_enable(&switch_mm_l1d_flush);
}
mutex_unlock(&l1d_flush_mutex);
}
@@ -195,29 +205,31 @@ static void l1d_flush(struct mm_struct *next, struct task_struct *tsk)
{
struct mm_struct *real_prev = this_cpu_read(cpu_tlbstate.loaded_mm);

- /*
- * If we are not really switching mm's, we can just return
- */
- if (real_prev == next)
- return;
+ if (static_branch_unlikely(&switch_mm_l1d_flush)) {
+ /*
+ * If we are not really switching mm's, we can just return
+ */
+ if (real_prev == next)
+ return;

- /*
- * Do we need flushing for by the previous task
- */
- if (this_cpu_read(cpu_tlbstate.last_user_mm_l1d_flush) != 0) {
- if (!flush_l1d_cache_hw())
- flush_l1d_cache_sw(l1d_flush_pages);
- this_cpu_write(cpu_tlbstate.last_user_mm_l1d_flush, 0);
- /* Make sure we clear the values before we set it again */
- barrier();
- }
+ /*
+ * Do we need flushing for by the previous task
+ */
+ if (this_cpu_read(cpu_tlbstate.last_user_mm_l1d_flush) != 0) {
+ if (!flush_l1d_cache_hw())
+ flush_l1d_cache_sw(l1d_flush_pages);
+ this_cpu_write(cpu_tlbstate.last_user_mm_l1d_flush, 0);
+ /* Make sure we clear the values before we set it again */
+ barrier();
+ }

- if (tsk == NULL)
- return;
+ if (tsk == NULL)
+ return;

- /* We don't need stringent checks as we opt-in/opt-out */
- if (test_ti_thread_flag(&tsk->thread_info, TIF_L1D_FLUSH))
- this_cpu_write(cpu_tlbstate.last_user_mm_l1d_flush, 1);
+ /* We don't need stringent checks as we opt-in/opt-out */
+ if (test_ti_thread_flag(&tsk->thread_info, TIF_L1D_FLUSH))
+ this_cpu_write(cpu_tlbstate.last_user_mm_l1d_flush, 1);
+ }
}

void switch_mm(struct mm_struct *prev, struct mm_struct *next,
--
2.17.1