[PATCH v11 2/7] mm/vmalloc: undo partial mappings inside the mapping functions

[Hao Ge <hao.ge@linux.dev>]

__vmap_pages_range_noflush() and friends can install some PTEs before
failing and leave them mapped, and the kernel callers do not agree on
who cleans them up. For example, pcpu_map_pages() and
kmsan_ioremap_page_range() unmap the leftovers themselves, while
vm_module_tags_populate() and the __GFP_NOFAIL retry loop in
__vmalloc_area_node() relied on the mapping functions cleaning up and
did not call anything like vunmap_range() themselves. When the same
range is mapped again, the attempt hits the leftovers and fails, with
BUG() in vmap_pte_range() for huge mappings.

After discussing with Suren and Ulad, we decided the cleanup belongs
to __vmap_pages_range_noflush() and friends, so the callers no longer
need to unmap the partial mappings themselves. Each function now
undoes the PTEs it installed itself.

The rollback calls the low-level __vunmap_range_noflush(), it just
clears the PTEs of the range it is given, which is all a rollback
needs. It cannot use vunmap_range_noflush() because these mapping
functions also map the KMSAN shadow and origin, and for a metadata
range its hook would look up the metadata of the metadata, get 0
and BUG() on addr >= end. The failed mappings were never accessed,
no TLB flush needed.

Fixes: 9376130c390a ("mm/vmalloc: add support for __GFP_NOFAIL")
Fixes: 0f9b685626da ("alloc_tag: populate memory for module tags as needed")
Reported-by: Sashiko <sashiko-bot@kernel.org>
Cc: stable@vger.kernel.org
Signed-off-by: Hao Ge <hao.ge@linux.dev>
---
 mm/kmsan/shadow.c |  4 ++++
 mm/vmalloc.c      | 31 +++++++++++++++++++++++++++++--
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/mm/kmsan/shadow.c b/mm/kmsan/shadow.c
index 0c88d89bf0d6..2166086d3dc3 100644
--- a/mm/kmsan/shadow.c
+++ b/mm/kmsan/shadow.c
@@ -258,6 +258,10 @@ int kmsan_vmap_pages_range_noflush(unsigned long start, unsigned long end,
 					    o_pages, page_shift);
 	kmsan_leave_runtime();
 	if (mapped) {
+		/* Undo the shadow mapping set up above. */
+		kmsan_enter_runtime();
+		__vunmap_range_noflush(shadow_start, shadow_end);
+		kmsan_leave_runtime();
 		err = mapped;
 		goto ret;
 	}
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 859e6d2d57a3..9bbf75706627 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -349,6 +349,10 @@ static int vmap_range_noflush(unsigned long addr, unsigned long end,
 	if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
 		arch_sync_kernel_mappings(start, end);
 
+	/* Undo the PTEs installed before the failure. */
+	if (err)
+		__vunmap_range_noflush(start, end);
+
 	return err;
 }
 
@@ -363,6 +367,9 @@ int vmap_page_range(unsigned long addr, unsigned long end,
 	if (!err)
 		err = kmsan_ioremap_page_range(addr, end, phys_addr, prot,
 					       ioremap_max_page_shift);
+	if (err)
+		__vunmap_range_noflush(addr, end);
+
 	return err;
 }
 
@@ -667,6 +674,10 @@ static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end,
 	if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
 		arch_sync_kernel_mappings(start, end);
 
+	/* Undo the PTEs installed before the failure. */
+	if (err)
+		__vunmap_range_noflush(start, end);
+
 	return err;
 }
 
@@ -683,6 +694,7 @@ int __vmap_pages_range_noflush(unsigned long addr, unsigned long end,
 		pgprot_t prot, struct page **pages, unsigned int page_shift)
 {
 	unsigned int i, nr = (end - addr) >> PAGE_SHIFT;
+	unsigned long start = addr;
 
 	WARN_ON(page_shift < PAGE_SHIFT);
 
@@ -696,8 +708,14 @@ int __vmap_pages_range_noflush(unsigned long addr, unsigned long end,
 		err = vmap_range_noflush(addr, addr + (1UL << page_shift),
 					page_to_phys(pages[i]), prot,
 					page_shift);
-		if (err)
+		if (err) {
+			/*
+			 * vmap_range_noflush() undoes its own partial work,
+			 * discard the chunks that already mapped.
+			 */
+			__vunmap_range_noflush(start, addr);
 			return err;
+		}
 
 		addr += 1UL << page_shift;
 	}
@@ -714,7 +732,16 @@ int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
 
 	if (ret)
 		return ret;
-	return __vmap_pages_range_noflush(addr, end, prot, pages, page_shift);
+
+	ret = __vmap_pages_range_noflush(addr, end, prot, pages, page_shift);
+	/*
+	 * The page tables undo themselves on failure. Tear down the
+	 * metadata that was fully set up before the mapping failed.
+	 */
+	if (ret)
+		kmsan_vunmap_range_noflush(addr, end);
+
+	return ret;
 }
 
 static int __vmap_pages_range(unsigned long addr, unsigned long end,
-- 
2.25.1