[patch 18/38] mm/vmalloc: Provide huge page mappings

Thomas Gleixner
2022-07-16T23:18:43Z

From: Peter Zijlstra <peterz@infradead.org>

Provide VM_HUGE_VMAP, which unconditionally tries to use huge
mappings. Unlike VM_ALLOW_HUGE_VMAP it doesn't care about the number
of NUMA nodes or the size of the allocation.

If the page allocator fails to provide huge pages, it will silently
fall back.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
include/linux/vmalloc.h | 3 ++-
mm/vmalloc.c | 33 +++++++++++++++++++--------------
2 files changed, 21 insertions(+), 15 deletions(-)

--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -27,10 +27,11 @@ struct notifier_block; /* in notifier.h
#define VM_FLUSH_RESET_PERMS 0x00000100 /* reset direct map and flush TLB on unmap, can't be freed in atomic context */
#define VM_MAP_PUT_PAGES 0x00000200 /* put pages and free array in vfree */
#define VM_ALLOW_HUGE_VMAP 0x00000400 /* Allow for huge pages on archs with HAVE_ARCH_HUGE_VMALLOC */
+#define VM_HUGE_VMAP 0x00000800 /* Force for huge pages on archs with HAVE_ARCH_HUGE_VMALLOC */

#if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \
!defined(CONFIG_KASAN_VMALLOC)
-#define VM_DEFER_KMEMLEAK 0x00000800 /* defer kmemleak object creation */
+#define VM_DEFER_KMEMLEAK 0x00001000 /* defer kmemleak object creation */
#else
#define VM_DEFER_KMEMLEAK 0
#endif
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -3099,23 +3099,28 @@ void *__vmalloc_node_range(unsigned long
return NULL;
}

- if (vmap_allow_huge && (vm_flags & VM_ALLOW_HUGE_VMAP)) {
- unsigned long size_per_node;
+ if (vmap_allow_huge && (vm_flags & (VM_HUGE_VMAP|VM_ALLOW_HUGE_VMAP))) {

- /*
- * Try huge pages. Only try for PAGE_KERNEL allocations,
- * others like modules don't yet expect huge pages in
- * their allocations due to apply_to_page_range not
- * supporting them.
- */
+ if (vm_flags & VM_ALLOW_HUGE_VMAP) {
+ unsigned long size_per_node;

- size_per_node = size;
- if (node == NUMA_NO_NODE)
- size_per_node /= num_online_nodes();
- if (arch_vmap_pmd_supported(prot) && size_per_node >= PMD_SIZE)
+ /*
+ * Try huge pages. Only try for PAGE_KERNEL allocations,
+ * others like modules don't yet expect huge pages in
+ * their allocations due to apply_to_page_range not
+ * supporting them.
+ */
+
+ size_per_node = size;
+ if (node == NUMA_NO_NODE)
+ size_per_node /= num_online_nodes();
+ if (arch_vmap_pmd_supported(prot) && size_per_node >= PMD_SIZE)
+ shift = PMD_SHIFT;
+ else
+ shift = arch_vmap_pte_supported_shift(size_per_node);
+ } else {
shift = PMD_SHIFT;
- else
- shift = arch_vmap_pte_supported_shift(size_per_node);
+ }

align = max(real_align, 1UL << shift);
size = ALIGN(real_size, 1UL << shift);