[PATCH 15/43] kmsan: mm: maintain KMSAN metadata for page operations

[PATCH 15/43] kmsan: mm: maintain KMSAN metadata for page operations

[Alexander Potapenko]

Insert KMSAN hooks that make the necessary bookkeeping changes:
 - poison page shadow and origins in alloc_pages()/free_page();
 - clear page shadow and origins in clear_page(), copy_user_highpage();
 - copy page metadata in copy_highpage(), wp_page_copy();
 - handle vmap()/vunmap()/iounmap();

Signed-off-by: Alexander Potapenko <glider@google.com>
---
Link: https://linux-review.googlesource.com/id/I6d4f53a0e7eab46fa29f0348f3095d9f2e326850
---
 arch/x86/include/asm/page_64.h | 13 +++++++++++++
 arch/x86/mm/ioremap.c          |  3 +++
 include/linux/highmem.h        |  3 +++
 mm/memory.c                    |  2 ++
 mm/page_alloc.c                | 14 ++++++++++++++
 mm/vmalloc.c                   | 20 ++++++++++++++++++--
 6 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h
index 4bde0dc66100c..c10547510f1f4 100644
--- a/arch/x86/include/asm/page_64.h
+++ b/arch/x86/include/asm/page_64.h
@@ -44,14 +44,27 @@ void clear_page_orig(void *page);
 void clear_page_rep(void *page);
 void clear_page_erms(void *page);
 
+/* This is an assembly header, avoid including too much of kmsan.h */
+#ifdef CONFIG_KMSAN
+void kmsan_unpoison_memory(const void *addr, size_t size);
+#endif
+__no_sanitize_memory
 static inline void clear_page(void *page)
 {
+#ifdef CONFIG_KMSAN
+	/* alternative_call_2() changes @page. */
+	void *page_copy = page;
+#endif
 	alternative_call_2(clear_page_orig,
 			   clear_page_rep, X86_FEATURE_REP_GOOD,
 			   clear_page_erms, X86_FEATURE_ERMS,
 			   "=D" (page),
 			   "0" (page)
 			   : "cc", "memory", "rax", "rcx");
+#ifdef CONFIG_KMSAN
+	/* Clear KMSAN shadow for the pages that have it. */
+	kmsan_unpoison_memory(page_copy, PAGE_SIZE);
+#endif
 }
 
 void copy_page(void *to, void *from);
diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c
index 026031b3b7829..4d0349ecc7cd7 100644
--- a/arch/x86/mm/ioremap.c
+++ b/arch/x86/mm/ioremap.c
@@ -17,6 +17,7 @@
 #include <linux/cc_platform.h>
 #include <linux/efi.h>
 #include <linux/pgtable.h>
+#include <linux/kmsan.h>
 
 #include <asm/set_memory.h>
 #include <asm/e820/api.h>
@@ -474,6 +475,8 @@ void iounmap(volatile void __iomem *addr)
 		return;
 	}
 
+	kmsan_iounmap_page_range((unsigned long)addr,
+		(unsigned long)addr + get_vm_area_size(p));
 	memtype_free(p->phys_addr, p->phys_addr + get_vm_area_size(p));
 
 	/* Finally remove it */
diff --git a/include/linux/highmem.h b/include/linux/highmem.h
index 39bb9b47fa9cd..3e1898a44d7e3 100644
--- a/include/linux/highmem.h
+++ b/include/linux/highmem.h
@@ -6,6 +6,7 @@
 #include <linux/kernel.h>
 #include <linux/bug.h>
 #include <linux/cacheflush.h>
+#include <linux/kmsan.h>
 #include <linux/mm.h>
 #include <linux/uaccess.h>
 #include <linux/hardirq.h>
@@ -277,6 +278,7 @@ static inline void copy_user_highpage(struct page *to, struct page *from,
 	vfrom = kmap_local_page(from);
 	vto = kmap_local_page(to);
 	copy_user_page(vto, vfrom, vaddr, to);
+	kmsan_unpoison_memory(page_address(to), PAGE_SIZE);
 	kunmap_local(vto);
 	kunmap_local(vfrom);
 }
@@ -292,6 +294,7 @@ static inline void copy_highpage(struct page *to, struct page *from)
 	vfrom = kmap_local_page(from);
 	vto = kmap_local_page(to);
 	copy_page(vto, vfrom);
+	kmsan_copy_page_meta(to, from);
 	kunmap_local(vto);
 	kunmap_local(vfrom);
 }
diff --git a/mm/memory.c b/mm/memory.c
index 8f1de811a1dcb..ea9e48daadb15 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -51,6 +51,7 @@
 #include <linux/highmem.h>
 #include <linux/pagemap.h>
 #include <linux/memremap.h>
+#include <linux/kmsan.h>
 #include <linux/ksm.h>
 #include <linux/rmap.h>
 #include <linux/export.h>
@@ -3003,6 +3004,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
 				put_page(old_page);
 			return 0;
 		}
+		kmsan_copy_page_meta(new_page, old_page);
 	}
 
 	if (mem_cgroup_charge(page_folio(new_page), mm, GFP_KERNEL))
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index c5952749ad40b..fa8029b714a81 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -26,6 +26,7 @@
 #include <linux/compiler.h>
 #include <linux/kernel.h>
 #include <linux/kasan.h>
+#include <linux/kmsan.h>
 #include <linux/module.h>
 #include <linux/suspend.h>
 #include <linux/pagevec.h>
@@ -1288,6 +1289,7 @@ static __always_inline bool free_pages_prepare(struct page *page,
 	VM_BUG_ON_PAGE(PageTail(page), page);
 
 	trace_mm_page_free(page, order);
+	kmsan_free_page(page, order);
 
 	if (unlikely(PageHWPoison(page)) && !order) {
 		/*
@@ -1734,6 +1736,9 @@ void __init memblock_free_pages(struct page *page, unsigned long pfn,
 {
 	if (early_page_uninitialised(pfn))
 		return;
+	if (!kmsan_memblock_free_pages(page, order))
+		/* KMSAN will take care of these pages. */
+		return;
 	__free_pages_core(page, order);
 }
 
@@ -3663,6 +3668,14 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone,
 /*
  * Allocate a page from the given zone. Use pcplists for order-0 allocations.
  */
+
+/*
+ * Do not instrument rmqueue() with KMSAN. This function may call
+ * __msan_poison_alloca() through a call to set_pfnblock_flags_mask().
+ * If __msan_poison_alloca() attempts to allocate pages for the stack depot, it
+ * may call rmqueue() again, which will result in a deadlock.
+ */
+__no_sanitize_memory
 static inline
 struct page *rmqueue(struct zone *preferred_zone,
 			struct zone *zone, unsigned int order,
@@ -5389,6 +5402,7 @@ struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid,
 	}
 
 	trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype);
+	kmsan_alloc_page(page, order, alloc_gfp);
 
 	return page;
 }
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index d2a00ad4e1dd1..333de26b3c56e 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -319,6 +319,9 @@ int ioremap_page_range(unsigned long addr, unsigned long end,
 	err = vmap_range_noflush(addr, end, phys_addr, pgprot_nx(prot),
 				 ioremap_max_page_shift);
 	flush_cache_vmap(addr, end);
+	if (!err)
+		kmsan_ioremap_page_range(addr, end, phys_addr, prot,
+					 ioremap_max_page_shift);
 	return err;
 }
 
@@ -418,7 +421,7 @@ static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
  *
  * This is an internal function only. Do not use outside mm/.
  */
-void vunmap_range_noflush(unsigned long start, unsigned long end)
+void __vunmap_range_noflush(unsigned long start, unsigned long end)
 {
 	unsigned long next;
 	pgd_t *pgd;
@@ -440,6 +443,12 @@ void vunmap_range_noflush(unsigned long start, unsigned long end)
 		arch_sync_kernel_mappings(start, end);
 }
 
+void vunmap_range_noflush(unsigned long start, unsigned long end)
+{
+	kmsan_vunmap_range_noflush(start, end);
+	__vunmap_range_noflush(start, end);
+}
+
 /**
  * vunmap_range - unmap kernel virtual addresses
  * @addr: start of the VM area to unmap
@@ -574,7 +583,7 @@ static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end,
  *
  * This is an internal function only. Do not use outside mm/.
  */
-int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
+int __vmap_pages_range_noflush(unsigned long addr, unsigned long end,
 		pgprot_t prot, struct page **pages, unsigned int page_shift)
 {
 	unsigned int i, nr = (end - addr) >> PAGE_SHIFT;
@@ -600,6 +609,13 @@ int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
 	return 0;
 }
 
+int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
+		pgprot_t prot, struct page **pages, unsigned int page_shift)
+{
+	kmsan_vmap_pages_range_noflush(addr, end, prot, pages, page_shift);
+	return __vmap_pages_range_noflush(addr, end, prot, pages, page_shift);
+}
+
 /**
  * vmap_pages_range - map pages to a kernel virtual address
  * @addr: start of the VM area to map
-- 
2.34.1.173.g76aa8bc2d0-goog

Re: [PATCH 15/43] kmsan: mm: maintain KMSAN metadata for page operations

[kernel test robot]

[-- Attachment #1: Type: text/plain, Size: 4361 bytes --]

CC: kbuild-all(a)lists.01.org
In-Reply-To: <20211214162050.660953-16-glider@google.com>
References: <20211214162050.660953-16-glider@google.com>
TO: Alexander Potapenko <glider@google.com>

Hi Alexander,

I love your patch! Perhaps something to improve:

[auto build test WARNING on tip/x86/mm]
[also build test WARNING on linus/master v5.16-rc5]
[cannot apply to tip/x86/core hnaz-mm/master next-20211214]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Alexander-Potapenko/Add-KernelMemorySanitizer-infrastructure/20211215-003033
base:   https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git 35fa745286ac44ee26ed100c2bd2553368ad193b
:::::: branch date: 21 hours ago
:::::: commit date: 21 hours ago
config: x86_64-randconfig-m001-20211214 (https://download.01.org/0day-ci/archive/20211215/202112152149.7ygega3j-lkp(a)intel.com/config)
compiler: gcc-9 (Debian 9.3.0-22) 9.3.0

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>
Reported-by: Dan Carpenter <dan.carpenter@oracle.com>

smatch warnings:
mm/vmalloc.c:442 __vunmap_range_noflush() warn: bitwise AND condition is false here

vim +442 mm/vmalloc.c

c2febafc67734a Kirill A. Shutemov  2017-03-09  411  
4ad0ae8c64ac8f Nicholas Piggin     2021-04-29  412  /*
4ad0ae8c64ac8f Nicholas Piggin     2021-04-29  413   * vunmap_range_noflush is similar to vunmap_range, but does not
4ad0ae8c64ac8f Nicholas Piggin     2021-04-29  414   * flush caches or TLBs.
b521c43f58e523 Christoph Hellwig   2020-06-01  415   *
4ad0ae8c64ac8f Nicholas Piggin     2021-04-29  416   * The caller is responsible for calling flush_cache_vmap() before calling
4ad0ae8c64ac8f Nicholas Piggin     2021-04-29  417   * this function, and flush_tlb_kernel_range after it has returned
4ad0ae8c64ac8f Nicholas Piggin     2021-04-29  418   * successfully (and before the addresses are expected to cause a page fault
4ad0ae8c64ac8f Nicholas Piggin     2021-04-29  419   * or be re-mapped for something else, if TLB flushes are being delayed or
4ad0ae8c64ac8f Nicholas Piggin     2021-04-29  420   * coalesced).
b521c43f58e523 Christoph Hellwig   2020-06-01  421   *
4ad0ae8c64ac8f Nicholas Piggin     2021-04-29  422   * This is an internal function only. Do not use outside mm/.
b521c43f58e523 Christoph Hellwig   2020-06-01  423   */
73c9fc6d9c6621 Alexander Potapenko 2021-12-14  424  void __vunmap_range_noflush(unsigned long start, unsigned long end)
^1da177e4c3f41 Linus Torvalds      2005-04-16  425  {
^1da177e4c3f41 Linus Torvalds      2005-04-16  426  	unsigned long next;
b521c43f58e523 Christoph Hellwig   2020-06-01  427  	pgd_t *pgd;
2ba3e6947aed9b Joerg Roedel        2020-06-01  428  	unsigned long addr = start;
2ba3e6947aed9b Joerg Roedel        2020-06-01  429  	pgtbl_mod_mask mask = 0;
^1da177e4c3f41 Linus Torvalds      2005-04-16  430  
^1da177e4c3f41 Linus Torvalds      2005-04-16  431  	BUG_ON(addr >= end);
^1da177e4c3f41 Linus Torvalds      2005-04-16  432  	pgd = pgd_offset_k(addr);
^1da177e4c3f41 Linus Torvalds      2005-04-16  433  	do {
^1da177e4c3f41 Linus Torvalds      2005-04-16  434  		next = pgd_addr_end(addr, end);
2ba3e6947aed9b Joerg Roedel        2020-06-01  435  		if (pgd_bad(*pgd))
2ba3e6947aed9b Joerg Roedel        2020-06-01  436  			mask |= PGTBL_PGD_MODIFIED;
^1da177e4c3f41 Linus Torvalds      2005-04-16  437  		if (pgd_none_or_clear_bad(pgd))
^1da177e4c3f41 Linus Torvalds      2005-04-16  438  			continue;
2ba3e6947aed9b Joerg Roedel        2020-06-01  439  		vunmap_p4d_range(pgd, addr, next, &mask);
^1da177e4c3f41 Linus Torvalds      2005-04-16  440  	} while (pgd++, addr = next, addr != end);
2ba3e6947aed9b Joerg Roedel        2020-06-01  441  
2ba3e6947aed9b Joerg Roedel        2020-06-01 @442  	if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
2ba3e6947aed9b Joerg Roedel        2020-06-01  443  		arch_sync_kernel_mappings(start, end);
^1da177e4c3f41 Linus Torvalds      2005-04-16  444  }
^1da177e4c3f41 Linus Torvalds      2005-04-16  445  

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org