[RFC PATCH v12 based on mm-unstable as of Feb 21, 2025 09/25] mm: introduce API to perform tlb shootdown on exit from page allocator

Byungchul Park
2025-02-26T12:01:48Z

Functionally, no change. This is a preparation for luf mechanism that
performs tlb shootdown required on exit from page allocator.

This patch introduced a new API rather than making use of existing
try_to_unmap_flush() to avoid repeated and redundant tlb shootdown due
to frequent page allocations during a session of batched unmap flush.

Signed-off-by: Byungchul Park <byungchul@sk.com>
---
include/linux/sched.h | 1 +
mm/internal.h | 4 ++++
mm/rmap.c | 20 ++++++++++++++++++++
3 files changed, 25 insertions(+)

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 9632e3318e0d6..86ef426644639 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1401,6 +1401,7 @@ struct task_struct {
#endif

struct tlbflush_unmap_batch tlb_ubc;
+ struct tlbflush_unmap_batch tlb_ubc_takeoff;

/* Cache last used pipe for splice(): */
struct pipe_inode_info *splice_pipe;
diff --git a/mm/internal.h b/mm/internal.h
index 8ade04255dba3..8ad7e86c1c0e2 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1269,6 +1269,7 @@ extern struct workqueue_struct *mm_percpu_wq;
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
void try_to_unmap_flush(void);
void try_to_unmap_flush_dirty(void);
+void try_to_unmap_flush_takeoff(void);
void flush_tlb_batched_pending(struct mm_struct *mm);
void fold_batch(struct tlbflush_unmap_batch *dst, struct tlbflush_unmap_batch *src, bool reset);
void fold_luf_batch(struct luf_batch *dst, struct luf_batch *src);
@@ -1279,6 +1280,9 @@ static inline void try_to_unmap_flush(void)
static inline void try_to_unmap_flush_dirty(void)
{
}
+static inline void try_to_unmap_flush_takeoff(void)
+{
+}
static inline void flush_tlb_batched_pending(struct mm_struct *mm)
{
}
diff --git a/mm/rmap.c b/mm/rmap.c
index ac450a45257f6..61366b4570c9a 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -772,6 +772,26 @@ void fold_luf_batch(struct luf_batch *dst, struct luf_batch *src)
read_unlock_irqrestore(&src->lock, flags);
}

+void try_to_unmap_flush_takeoff(void)
+{
+ struct tlbflush_unmap_batch *tlb_ubc = &current->tlb_ubc;
+ struct tlbflush_unmap_batch *tlb_ubc_takeoff = &current->tlb_ubc_takeoff;
+
+ if (!tlb_ubc_takeoff->flush_required)
+ return;
+
+ arch_tlbbatch_flush(&tlb_ubc_takeoff->arch);
+
+ /*
+ * Now that tlb shootdown of tlb_ubc_takeoff has been performed,
+ * it's good chance to shrink tlb_ubc if possible.
+ */
+ if (arch_tlbbatch_done(&tlb_ubc->arch, &tlb_ubc_takeoff->arch))
+ reset_batch(tlb_ubc);
+
+ reset_batch(tlb_ubc_takeoff);
+}
+
/*
* Flush TLB entries for recently unmapped pages from remote CPUs. It is
* important if a PTE was dirty when it was unmapped that it's flushed
--
2.17.1