From: Shivank Garg <shivankg@amd.com>
To: Andrew Morton <akpm@linux-foundation.org>,
David Hildenbrand <david@kernel.org>,
Lorenzo Stoakes <ljs@kernel.org>, Rik van Riel <riel@surriel.com>,
"Liam R. Howlett" <liam@infradead.org>,
Vlastimil Babka <vbabka@kernel.org>, Harry Yoo <harry@kernel.org>,
Jann Horn <jannh@google.com>, Lance Yang <lance.yang@linux.dev>,
Mike Rapoport <rppt@kernel.org>,
Suren Baghdasaryan <surenb@google.com>,
Michal Hocko <mhocko@suse.com>, Zi Yan <ziy@nvidia.com>,
Matthew Brost <matthew.brost@intel.com>,
Joshua Hahn <joshua.hahnjy@gmail.com>,
Rakie Kim <rakie.kim@sk.com>, Byungchul Park <byungchul@sk.com>,
Gregory Price <gourry@gourry.net>,
Ying Huang <ying.huang@linux.alibaba.com>,
"Alistair Popple" <apopple@nvidia.com>
Cc: Karim Manaouil <kmanaouil.dev@gmail.com>,
Frank van der Linden <fvdl@google.com>,
Kinsey Ho <kinseyho@google.com>, Wei Xu <weixugc@google.com>,
Bharata B Rao <bharata@amd.com>,
David Rientjes <rientjes@google.com>, Dev Jain <dev.jain@arm.com>,
<linux-mm@kvack.org>, <linux-kernel@vger.kernel.org>,
Shivank Garg <shivankg@amd.com>
Subject: [PATCH 3/6] mm/migrate: split remove_migration_pte_hugetlb() out of remove_migration_pte()
Date: Sun, 12 Jul 2026 07:38:49 +0000 [thread overview]
Message-ID: <20260712-migrate-rmap-batch-v1-3-872a734431d1@amd.com> (raw)
In-Reply-To: <20260712-migrate-rmap-batch-v1-0-872a734431d1@amd.com>
remove_migration_pte() interleaves the regular folio path with the
hugetlb path via folio_test_hugetlb() and CONFIG_HUGETLB_PAGE
special cases. Simplify this by seprating the hugetlb handling into
remove_migration_pte_hugetlb().
Factor out working-PTE construction from migration entry into
migration_softleaf_entry_to_pte(), used by both paths.
While here, convert the VM_BUG_ON_FOLIO() to VM_WARN_ON_ONCE_FOLIO().
No functional change intended.
Signed-off-by: Shivank Garg <shivankg@amd.com>
---
mm/migrate.c | 153 ++++++++++++++++++++++++++++++++++++++---------------------
1 file changed, 98 insertions(+), 55 deletions(-)
diff --git a/mm/migrate.c b/mm/migrate.c
index 222c8c15f782..8a098402b8c9 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -345,6 +345,89 @@ struct rmap_walk_arg {
bool map_unused_to_zeropage;
};
+static pte_t migration_softleaf_entry_to_pte(struct folio *folio, struct page *new,
+ softleaf_t entry, pte_t old_pte, struct vm_area_struct *vma,
+ rmap_t *rmap_flags)
+{
+ pte_t pte = mk_pte(new, READ_ONCE(vma->vm_page_prot));
+
+ if (!softleaf_is_migration_young(entry))
+ pte = pte_mkold(pte);
+ if (folio_test_dirty(folio) && softleaf_is_migration_dirty(entry))
+ pte = pte_mkdirty(pte);
+ if (pte_swp_soft_dirty(old_pte))
+ pte = pte_mksoft_dirty(pte);
+ else
+ pte = pte_clear_soft_dirty(pte);
+
+ if (softleaf_is_migration_write(entry))
+ pte = pte_mkwrite(pte, vma);
+ else if (pte_swp_uffd(old_pte))
+ pte = pte_mkuffd(pte);
+
+ /* See do_swap_page(): restore PAGE_NONE for RWP */
+ if (pte_swp_uffd(old_pte) && userfaultfd_rwp(vma))
+ pte = pte_modify(pte, PAGE_NONE);
+
+ if (folio_test_anon(folio) && !softleaf_is_migration_read(entry))
+ *rmap_flags |= RMAP_EXCLUSIVE;
+
+ return pte;
+}
+
+/*
+ * Restore a potential migration pte to a working pte entry for hugetlb folios.
+ */
+#ifdef CONFIG_HUGETLB_PAGE
+static bool remove_migration_pte_hugetlb(struct folio *folio,
+ struct vm_area_struct *vma, unsigned long addr, void *arg)
+{
+ struct rmap_walk_arg *rmap_walk_arg = arg;
+ DEFINE_FOLIO_VMA_WALK(pvmw, rmap_walk_arg->folio, vma, addr, PVMW_SYNC | PVMW_MIGRATION);
+ struct hstate *h = hstate_vma(vma);
+ unsigned int shift = huge_page_shift(h);
+ unsigned long psize = huge_page_size(h);
+
+ if (page_vma_mapped_walk(&pvmw)) {
+ rmap_t rmap_flags = RMAP_NONE;
+ struct page *new = folio_page(folio, 0);
+ pte_t old_pte, pte;
+ softleaf_t entry;
+
+ old_pte = huge_ptep_get(vma->vm_mm, pvmw.address, pvmw.pte);
+ entry = softleaf_from_pte(old_pte);
+
+ folio_get(folio);
+ pte = migration_softleaf_entry_to_pte(folio, new, entry, old_pte,
+ vma, &rmap_flags);
+ pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
+ if (folio_test_anon(folio))
+ hugetlb_add_anon_rmap(folio, vma, pvmw.address,
+ rmap_flags);
+ else
+ hugetlb_add_file_rmap(folio);
+ set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte, psize);
+
+ if (READ_ONCE(vma->vm_flags) & VM_LOCKED)
+ mlock_drain_local();
+
+ trace_remove_migration_pte(pvmw.address, pte_val(pte),
+ compound_order(new));
+
+ /* No need to invalidate - it was non-present before */
+ update_mmu_cache(vma, pvmw.address, pvmw.pte);
+ }
+
+ return true;
+}
+#else
+static bool remove_migration_pte_hugetlb(struct folio *folio,
+ struct vm_area_struct *vma, unsigned long addr, void *arg)
+{
+ return false;
+}
+#endif /* CONFIG_HUGETLB_PAGE */
+
/*
* Restore a potential migration pte to a working pte entry
*/
@@ -363,52 +446,28 @@ static bool remove_migration_pte(struct folio *folio,
unsigned long idx = 0;
/* pgoff is invalid for ksm pages, but they are never large */
- if (folio_test_large(folio) && !folio_test_hugetlb(folio))
+ if (folio_test_large(folio))
idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff;
new = folio_page(folio, idx);
#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
/* PMD-mapped THP migration entry */
if (!pvmw.pte) {
- VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) ||
- !folio_test_pmd_mappable(folio), folio);
+ VM_WARN_ON_ONCE_FOLIO(!folio_test_pmd_mappable(folio), folio);
remove_migration_pmd(&pvmw, new);
continue;
}
#endif
- if (folio_test_hugetlb(folio))
- old_pte = huge_ptep_get(vma->vm_mm, pvmw.address,
- pvmw.pte);
- else
- old_pte = ptep_get(pvmw.pte);
+ old_pte = ptep_get(pvmw.pte);
if (rmap_walk_arg->map_unused_to_zeropage &&
try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx))
continue;
- folio_get(folio);
- pte = mk_pte(new, READ_ONCE(vma->vm_page_prot));
-
entry = softleaf_from_pte(old_pte);
- if (!softleaf_is_migration_young(entry))
- pte = pte_mkold(pte);
- if (folio_test_dirty(folio) && softleaf_is_migration_dirty(entry))
- pte = pte_mkdirty(pte);
- if (pte_swp_soft_dirty(old_pte))
- pte = pte_mksoft_dirty(pte);
- else
- pte = pte_clear_soft_dirty(pte);
- if (softleaf_is_migration_write(entry))
- pte = pte_mkwrite(pte, vma);
- else if (pte_swp_uffd(old_pte))
- pte = pte_mkuffd(pte);
-
- /* See do_swap_page(): restore PAGE_NONE for RWP */
- if (pte_swp_uffd(old_pte) && userfaultfd_rwp(vma))
- pte = pte_modify(pte, PAGE_NONE);
-
- if (folio_test_anon(folio) && !softleaf_is_migration_read(entry))
- rmap_flags |= RMAP_EXCLUSIVE;
+ folio_get(folio);
+ pte = migration_softleaf_entry_to_pte(folio, new, entry, old_pte,
+ vma, &rmap_flags);
if (unlikely(is_device_private_page(new))) {
if (pte_write(pte))
@@ -424,30 +483,12 @@ static bool remove_migration_pte(struct folio *folio,
pte = pte_swp_mkuffd(pte);
}
-#ifdef CONFIG_HUGETLB_PAGE
- if (folio_test_hugetlb(folio)) {
- struct hstate *h = hstate_vma(vma);
- unsigned int shift = huge_page_shift(h);
- unsigned long psize = huge_page_size(h);
-
- pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
- if (folio_test_anon(folio))
- hugetlb_add_anon_rmap(folio, vma, pvmw.address,
- rmap_flags);
- else
- hugetlb_add_file_rmap(folio);
- set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte,
- psize);
- } else
-#endif
- {
- if (folio_test_anon(folio))
- folio_add_anon_rmap_pte(folio, new, vma,
- pvmw.address, rmap_flags);
- else
- folio_add_file_rmap_pte(folio, new, vma);
- set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
- }
+ if (folio_test_anon(folio))
+ folio_add_anon_rmap_pte(folio, new, vma,
+ pvmw.address, rmap_flags);
+ else
+ folio_add_file_rmap_pte(folio, new, vma);
+ set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
if (READ_ONCE(vma->vm_flags) & VM_LOCKED)
mlock_drain_local();
@@ -474,7 +515,9 @@ void remove_migration_ptes(struct folio *src, struct folio *dst,
};
struct rmap_walk_control rwc = {
- .rmap_one = remove_migration_pte,
+ .rmap_one = folio_test_hugetlb(src) ?
+ remove_migration_pte_hugetlb :
+ remove_migration_pte,
.arg = &rmap_walk_arg,
};
--
2.43.0
next prev parent reply other threads:[~2026-07-12 7:41 UTC|newest]
Thread overview: 10+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-07-12 7:38 [PATCH 0/6] mm: batch rmap walks during large folio migration Shivank Garg
2026-07-12 7:38 ` [PATCH 1/6] mm/rmap: Add batched version of folio_try_share_anon_rmap_pte Shivank Garg
2026-07-12 7:38 ` [PATCH 2/6] mm: factor out generic PTE batch detection from swap_pte_batch() Shivank Garg
2026-07-12 7:38 ` Shivank Garg [this message]
2026-07-12 7:38 ` [PATCH 4/6] mm/migrate: batch the restore-side migration rmap walk Shivank Garg
2026-07-12 7:38 ` [PATCH 5/6] mm/rmap: split try_to_migrate_hugetlb_one() out of try_to_migrate_one() Shivank Garg
2026-07-12 7:38 ` [PATCH 6/6] mm/rmap: batch the unmap of large folios in try_to_migrate_one() Shivank Garg
2026-07-12 18:24 ` [syzbot ci] Re: mm: batch rmap walks during large folio migration syzbot ci
2026-07-14 2:05 ` [PATCH 0/6] " Andrew Morton
2026-07-14 5:34 ` Garg, Shivank
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260712-migrate-rmap-batch-v1-3-872a734431d1@amd.com \
--to=shivankg@amd.com \
--cc=akpm@linux-foundation.org \
--cc=apopple@nvidia.com \
--cc=bharata@amd.com \
--cc=byungchul@sk.com \
--cc=david@kernel.org \
--cc=dev.jain@arm.com \
--cc=fvdl@google.com \
--cc=gourry@gourry.net \
--cc=harry@kernel.org \
--cc=jannh@google.com \
--cc=joshua.hahnjy@gmail.com \
--cc=kinseyho@google.com \
--cc=kmanaouil.dev@gmail.com \
--cc=lance.yang@linux.dev \
--cc=liam@infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=ljs@kernel.org \
--cc=matthew.brost@intel.com \
--cc=mhocko@suse.com \
--cc=rakie.kim@sk.com \
--cc=riel@surriel.com \
--cc=rientjes@google.com \
--cc=rppt@kernel.org \
--cc=surenb@google.com \
--cc=vbabka@kernel.org \
--cc=weixugc@google.com \
--cc=ying.huang@linux.alibaba.com \
--cc=ziy@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox