[RFC PATCH v12 based on v6.14-rc4 22/25] mm/page_alloc: not allow to tlb shootdown if !preemptable() && non_luf_pages_ok()

Byungchul Park
2025-02-26T12:03:57Z

Do not perform tlb shootdown if the context is in preempt disable and
there are already enough non luf pages, not to hurt preemptibility.

Signed-off-by: Byungchul Park <byungchul@sk.com>
---
mm/compaction.c | 6 +++---
mm/internal.h | 5 +++--
mm/page_alloc.c | 27 +++++++++++++++------------
mm/page_isolation.c | 2 +-
mm/page_reporting.c | 4 ++--
5 files changed, 24 insertions(+), 20 deletions(-)

diff --git a/mm/compaction.c b/mm/compaction.c
index b7a7a6feb9eac..aab400ec6a734 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -606,7 +606,7 @@ static unsigned long isolate_freepages_block(struct compact_control *cc,

page = pfn_to_page(blockpfn);

- luf_takeoff_start();
+ luf_takeoff_start(cc->zone);
/* Isolate free pages. */
for (; blockpfn < end_pfn; blockpfn += stride, page += stride) {
int isolated;
@@ -1603,7 +1603,7 @@ static void fast_isolate_freepages(struct compact_control *cc)
if (!area->nr_free)
continue;

- can_shootdown = luf_takeoff_start();
+ can_shootdown = luf_takeoff_start(cc->zone);
spin_lock_irqsave(&cc->zone->lock, flags);
freelist = &area->free_list[MIGRATE_MOVABLE];
retry:
@@ -2416,7 +2416,7 @@ static enum compact_result compact_finished(struct compact_control *cc)
* luf_takeoff_{start,end}() is required to identify whether
* this compaction context is tlb shootdownable for luf'd pages.
*/
- luf_takeoff_start();
+ luf_takeoff_start(cc->zone);
ret = __compact_finished(cc);
luf_takeoff_end(cc->zone);

diff --git a/mm/internal.h b/mm/internal.h
index 6d7b3b389810e..b5f1928732498 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1591,7 +1591,7 @@ static inline void accept_page(struct page *page)
#endif /* CONFIG_UNACCEPTED_MEMORY */
#if defined(CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH)
extern struct luf_batch luf_batch[];
-bool luf_takeoff_start(void);
+bool luf_takeoff_start(struct zone *zone);
void luf_takeoff_end(struct zone *zone);
bool luf_takeoff_no_shootdown(void);
bool luf_takeoff_check(struct zone *zone, struct page *page);
@@ -1605,6 +1605,7 @@ static inline bool non_luf_pages_ok(struct zone *zone)

return nr_free - nr_luf_pages > min_wm;
}
+
unsigned short fold_unmap_luf(void);

/*
@@ -1691,7 +1692,7 @@ static inline bool can_luf_vma(struct vm_area_struct *vma)
return true;
}
#else /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */
-static inline bool luf_takeoff_start(void) { return false; }
+static inline bool luf_takeoff_start(struct zone *zone) { return false; }
static inline void luf_takeoff_end(struct zone *zone) {}
static inline bool luf_takeoff_no_shootdown(void) { return true; }
static inline bool luf_takeoff_check(struct zone *zone, struct page *page) { return true; }
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 917a257ea5706..2a2103df2d88e 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -623,22 +623,25 @@ compaction_capture(struct capture_control *capc, struct page *page,
#endif /* CONFIG_COMPACTION */

#if defined(CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH)
-static bool no_shootdown_context(void)
+static bool no_shootdown_context(struct zone *zone)
{
/*
- * If it performs with irq disabled, that might cause a deadlock.
- * Avoid tlb shootdown in this case.
+ * Tries to avoid tlb shootdown if !preemptible(). However, it
+ * should be allowed under heavy memory pressure.
*/
+ if (zone && non_luf_pages_ok(zone))
+ return !(preemptible() && in_task());
+
return !(!irqs_disabled() && in_task());
}

/*
* Can be called with zone lock released and irq enabled.
*/
-bool luf_takeoff_start(void)
+bool luf_takeoff_start(struct zone *zone)
{
unsigned long flags;
- bool no_shootdown = no_shootdown_context();
+ bool no_shootdown = no_shootdown_context(zone);

local_irq_save(flags);

@@ -2591,7 +2594,7 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
* luf_takeoff_{start,end}() is required for
* get_page_from_free_area() to use luf_takeoff_check().
*/
- luf_takeoff_start();
+ luf_takeoff_start(zone);
spin_lock_irqsave(&zone->lock, flags);
for (order = 0; order < NR_PAGE_ORDERS; order++) {
struct free_area *area = &(zone->free_area[order]);
@@ -2796,7 +2799,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
unsigned long flags;
int i;

- luf_takeoff_start();
+ luf_takeoff_start(zone);
spin_lock_irqsave(&zone->lock, flags);
for (i = 0; i < count; ++i) {
struct page *page = __rmqueue(zone, order, migratetype,
@@ -3422,7 +3425,7 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone,

do {
page = NULL;
- luf_takeoff_start();
+ luf_takeoff_start(zone);
spin_lock_irqsave(&zone->lock, flags);
if (alloc_flags & ALLOC_HIGHATOMIC)
page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
@@ -3567,7 +3570,7 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone,
struct page *page;
unsigned long __maybe_unused UP_flags;

- luf_takeoff_start();
+ luf_takeoff_start(NULL);
/* spin_trylock may fail due to a parallel drain or IRQ reentrancy. */
pcp_trylock_prepare(UP_flags);
pcp = pcp_spin_trylock(zone->per_cpu_pageset);
@@ -5190,7 +5193,7 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,
if (unlikely(!zone))
goto failed;

- luf_takeoff_start();
+ luf_takeoff_start(NULL);
/* spin_trylock may fail due to a parallel drain or IRQ reentrancy. */
pcp_trylock_prepare(UP_flags);
pcp = pcp_spin_trylock(zone->per_cpu_pageset);
@@ -7294,7 +7297,7 @@ unsigned long __offline_isolated_pages(unsigned long start_pfn,

offline_mem_sections(pfn, end_pfn);
zone = page_zone(pfn_to_page(pfn));
- luf_takeoff_start();
+ luf_takeoff_start(zone);
spin_lock_irqsave(&zone->lock, flags);
while (pfn < end_pfn) {
page = pfn_to_page(pfn);
@@ -7412,7 +7415,7 @@ bool take_page_off_buddy(struct page *page)
unsigned int order;
bool ret = false;

- luf_takeoff_start();
+ luf_takeoff_start(zone);
spin_lock_irqsave(&zone->lock, flags);
for (order = 0; order < NR_PAGE_ORDERS; order++) {
struct page *page_head = page - (pfn & ((1 << order) - 1));
diff --git a/mm/page_isolation.c b/mm/page_isolation.c
index c34659b58ca6c..f4055c0a2ea89 100644
--- a/mm/page_isolation.c
+++ b/mm/page_isolation.c
@@ -211,7 +211,7 @@ static void unset_migratetype_isolate(struct page *page, int migratetype)
struct page *buddy;

zone = page_zone(page);
- luf_takeoff_start();
+ luf_takeoff_start(zone);
spin_lock_irqsave(&zone->lock, flags);
if (!is_migrate_isolate_page(page))
goto out;
diff --git a/mm/page_reporting.c b/mm/page_reporting.c
index b23d3ed34ec07..83b66e7f0d257 100644
--- a/mm/page_reporting.c
+++ b/mm/page_reporting.c
@@ -170,7 +170,7 @@ page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone,
if (free_area_empty(area, mt))
return err;

- can_shootdown = luf_takeoff_start();
+ can_shootdown = luf_takeoff_start(zone);
spin_lock_irq(&zone->lock);

/*
@@ -250,7 +250,7 @@ page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone,
/* update budget to reflect call to report function */
budget--;

- luf_takeoff_start();
+ luf_takeoff_start(zone);

/* reacquire zone lock and resume processing */
spin_lock_irq(&zone->lock);
--
2.17.1