[PATCH v2 1/4] mm/migrate: do not migrate folios mapped into VM_LOCKED VMAs under compaction

[Wandun Chen <chenwandun1@gmail.com>]

From: Wandun Chen <chenwandun@lixiang.com>

When compact_unevictable_allowed=0, unevictable pages should not be
migrated. However, mlock_folio_batch in the mlock[all] syscall introduces
a race, mlock_folio() sets PG_mlocked immediately but defers PG_unevictable
to mlock_folio_batch(), causing pages that are about to become unevictable
to be migrated, which violates the intent of compact_unevictable_allowed,
and causes spike latency in RT kernels [1].

In order to fix this, migration is forbidden for pages mapped into VMAs
marked with VM_LOCKED. In addition, two early-return paths are introduced,
filter out mlocked pages, return early to avoid unnecessary operations.

Fixes: 90d07210ab55 ("mm: mlock: use folios and a folio batch internally")
Reported-by: Alexander Krabler <Alexander.Krabler@kuka.com>
Closes: https://lore.kernel.org/all/DU0PR01MB10385345F7153F334100981888259A@DU0PR01MB10385.eurprd01.prod.exchangelabs.com/ [1]
Suggested-by: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Wandun Chen <chenwandun@lixiang.com>
Link: https://lore.kernel.org/linux-rt-users/33275585-f2db-4779-89f0-3ae24b455a67@suse.cz/#t
---
 include/linux/compaction.h |  6 ++++++
 include/linux/rmap.h       |  3 +++
 mm/compaction.c            |  8 +++++++-
 mm/migrate.c               | 23 +++++++++++++++++++----
 mm/rmap.c                  | 12 +++++++++---
 5 files changed, 44 insertions(+), 8 deletions(-)

diff --git a/include/linux/compaction.h b/include/linux/compaction.h
index f29ef0653546..04e60f65b976 100644
--- a/include/linux/compaction.h
+++ b/include/linux/compaction.h
@@ -106,6 +106,7 @@ bool compaction_zonelist_suitable(struct alloc_context *ac, int order,
 extern void __meminit kcompactd_run(int nid);
 extern void __meminit kcompactd_stop(int nid);
 extern void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx);
+extern bool compaction_allow_unevictable(void);
 
 #else
 static inline void reset_isolation_suitable(pg_data_t *pgdat)
@@ -131,6 +132,11 @@ static inline void wakeup_kcompactd(pg_data_t *pgdat,
 {
 }
 
+static inline bool compaction_allow_unevictable(void)
+{
+	return true;
+}
+
 #endif /* CONFIG_COMPACTION */
 
 struct node;
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 8dc0871e5f00..359c7426b6b9 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -102,6 +102,9 @@ enum ttu_flags {
 					 * do a final flush if necessary */
 	TTU_RMAP_LOCKED		= 0x80,	/* do not grab rmap lock:
 					 * caller holds it */
+	TTU_RESPECT_MLOCK	= 0x100,/* leave VM_LOCKED vmas mapped instead
+					 * of installing a migration entry
+					 */
 };
 
 #ifdef CONFIG_MMU
diff --git a/mm/compaction.c b/mm/compaction.c
index f08765ade014..5d256930e389 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -1116,7 +1116,8 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
 		is_unevictable = folio_test_unevictable(folio);
 
 		/* Compaction might skip unevictable pages but CMA takes them */
-		if (!(mode & ISOLATE_UNEVICTABLE) && is_unevictable)
+		if (!(mode & ISOLATE_UNEVICTABLE) &&
+		    (is_unevictable || folio_test_mlocked(folio)))
 			goto isolate_fail_put;
 
 		/*
@@ -1898,6 +1899,11 @@ typedef enum {
  * compactable pages.
  */
 static int sysctl_compact_unevictable_allowed __read_mostly = CONFIG_COMPACT_UNEVICTABLE_DEFAULT;
+
+bool compaction_allow_unevictable(void)
+{
+	return sysctl_compact_unevictable_allowed;
+}
 /*
  * Tunable for proactive compaction. It determines how
  * aggressively the kernel should compact memory in the
diff --git a/mm/migrate.c b/mm/migrate.c
index a786549551e3..3a15eb13e82b 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1202,7 +1202,7 @@ static void migrate_folio_done(struct folio *src,
 static int migrate_folio_unmap(new_folio_t get_new_folio,
 		free_folio_t put_new_folio, unsigned long private,
 		struct folio *src, struct folio **dstp, enum migrate_mode mode,
-		struct list_head *ret)
+		struct list_head *ret, enum migrate_reason reason)
 {
 	struct folio *dst;
 	int rc = -EAGAIN;
@@ -1210,6 +1210,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
 	struct anon_vma *anon_vma = NULL;
 	bool locked = false;
 	bool dst_locked = false;
+	enum ttu_flags ttu = 0;
 
 	dst = get_new_folio(src, private);
 	if (!dst)
@@ -1249,9 +1250,15 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
 		folio_lock(src);
 	}
 	locked = true;
-	if (folio_test_mlocked(src))
+	if (folio_test_mlocked(src)) {
 		old_folio_state |= FOLIO_WAS_MLOCKED;
 
+		if (reason == MR_COMPACTION && !compaction_allow_unevictable()) {
+			rc = -EBUSY;
+			goto out;
+		}
+	}
+
 	if (folio_test_writeback(src)) {
 		/*
 		 * Only in the case of a full synchronous migration is it
@@ -1324,7 +1331,14 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
 		/* Establish migration ptes */
 		VM_BUG_ON_FOLIO(folio_test_anon(src) &&
 			       !folio_test_ksm(src) && !anon_vma, src);
-		try_to_migrate(src, mode == MIGRATE_ASYNC ? TTU_BATCH_FLUSH : 0);
+
+		if (mode == MIGRATE_ASYNC)
+			ttu |= TTU_BATCH_FLUSH;
+
+		if (reason == MR_COMPACTION && !compaction_allow_unevictable())
+			ttu |= TTU_RESPECT_MLOCK;
+
+		try_to_migrate(src, ttu);
 		old_folio_state |= FOLIO_WAS_MAPPED;
 	}
 
@@ -1905,7 +1919,8 @@ static int migrate_pages_batch(struct list_head *from,
 			}
 
 			rc = migrate_folio_unmap(get_new_folio, put_new_folio,
-					private, folio, &dst, mode, ret_folios);
+					private, folio, &dst, mode, ret_folios,
+					reason);
 			/*
 			 * The rules are:
 			 *	0: folio will be put on unmap_folios list,
diff --git a/mm/rmap.c b/mm/rmap.c
index 0fb7a1b82cf3..3cb7f6337d38 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -2420,6 +2420,9 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
 	unsigned long pfn;
 	unsigned long hsz = 0;
 
+	if ((flags & TTU_RESPECT_MLOCK) && (vma->vm_flags & VM_LOCKED))
+		return false;
+
 	/*
 	 * When racing against e.g. zap_pte_range() on another cpu,
 	 * in between its ptep_get_and_clear_full() and folio_remove_rmap_*(),
@@ -2741,11 +2744,14 @@ void try_to_migrate(struct folio *folio, enum ttu_flags flags)
 	};
 
 	/*
-	 * Migration always ignores mlock and only supports TTU_RMAP_LOCKED and
-	 * TTU_SPLIT_HUGE_PMD, TTU_SYNC, and TTU_BATCH_FLUSH flags.
+	 * Migration normally ignores mlock, but TTU_RESPECT_MLOCK asks it to
+	 * leave folios mapped into VM_LOCKED vmas alone.  Only TTU_RMAP_LOCKED,
+	 * TTU_SPLIT_HUGE_PMD, TTU_SYNC, TTU_BATCH_FLUSH and TTU_RESPECT_MLOCK
+	 * are supported.
 	 */
 	if (WARN_ON_ONCE(flags & ~(TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD |
-					TTU_SYNC | TTU_BATCH_FLUSH)))
+					TTU_SYNC | TTU_BATCH_FLUSH |
+					TTU_RESPECT_MLOCK)))
 		return;
 
 	if (folio_is_zone_device(folio) &&
-- 
2.43.0