[PATCH 1/6] mm/hugetlb: add Kconfig and basic cache infrastructure

[Sourav Panda <souravpanda@google.com>]

Introduce CONFIG_HUGETLB_CACHE Kconfig option and add the basic
infrastructure for the HugeTLB dynamic cache.

This includes:
- Adding new fields to struct hstate to track cached pages
  (nr_cached_hugepages, max_cached_huge_pages, and node-specific versions)
  and initializing them.
- Introducing a new hugetlb page flag HPG_cached to track cached folios.
- Updating remove_hugetlb_folio() to be cache-aware: if the folio has
  HPG_cached set, decrement cache counters instead of the free page
  counters. This prevents leaks during memory hotplug
  (dissolve_free_hugetlb_folios).
- Preventing inflation of free_huge_pages by cached pages to protect
  memory reservation guarantees.

Signed-off-by: Sourav Panda <souravpanda@google.com>
---
 fs/Kconfig              |  9 +++++
 include/linux/hugetlb.h | 14 +++++++
 mm/hugetlb.c            | 82 +++++++++++++++++++++++++++++++++++++----
 3 files changed, 98 insertions(+), 7 deletions(-)

diff --git a/fs/Kconfig b/fs/Kconfig
index 43cb06de297f..d33d1973ce8f 100644
--- a/fs/Kconfig
+++ b/fs/Kconfig
@@ -268,6 +268,15 @@ config HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON
 	  The HugeTLB Vmemmap Optimization (HVO) defaults to off. Say Y here to
 	  enable HVO by default. It can be disabled via hugetlb_free_vmemmap=off
 	  (boot command line) or hugetlb_optimize_vmemmap (sysctl).
+
+config HUGETLB_CACHE
+	bool "HugeTLB dynamic cache"
+	help
+	  Enables a dynamic, NUMA-aware hugepage cache for
+	  gigantic hugepages to allow faster allocation of surplus pages.
+	  Surplus pages are recycled into this cache upon release
+	  instead of being freed back to the buddy allocator immediately.
+	  Subsequent hugetlb allocations will prefer this cache.
 endif # HUGETLBFS
 
 config HUGETLB_PAGE
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 2abaf99321e9..4768f52ddd35 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -585,6 +585,9 @@ enum hugetlb_page_flags {
 	HPG_vmemmap_optimized,
 	HPG_raw_hwp_unreliable,
 	HPG_cma,
+#ifdef CONFIG_HUGETLB_CACHE
+	HPG_cached,
+#endif
 	__NR_HPAGEFLAGS,
 };
 
@@ -645,6 +648,9 @@ HPAGEFLAG(Freed, freed)
 HPAGEFLAG(VmemmapOptimized, vmemmap_optimized)
 HPAGEFLAG(RawHwpUnreliable, raw_hwp_unreliable)
 HPAGEFLAG(Cma, cma)
+#ifdef CONFIG_HUGETLB_CACHE
+HPAGEFLAG(Cached, cached)
+#endif
 
 #ifdef CONFIG_HUGETLB_PAGE
 
@@ -670,6 +676,14 @@ struct hstate {
 	unsigned int nr_huge_pages_node[MAX_NUMNODES];
 	unsigned int free_huge_pages_node[MAX_NUMNODES];
 	unsigned int surplus_huge_pages_node[MAX_NUMNODES];
+
+#ifdef CONFIG_HUGETLB_CACHE
+	unsigned long nr_cached_hugepages;
+	unsigned int nr_cached_hugepages_node[MAX_NUMNODES];
+	struct list_head hugepage_cache_lists[MAX_NUMNODES];
+	unsigned long max_cached_huge_pages;
+	unsigned int max_cached_huge_pages_node[MAX_NUMNODES];
+#endif
 	char name[HSTATE_NAME_LEN];
 };
 
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index c921287489de..15be3cf54606 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1381,6 +1381,60 @@ static struct folio *alloc_gigantic_frozen_folio(int order, gfp_t gfp_mask, int
 }
 #endif
 
+#ifdef CONFIG_HUGETLB_CACHE
+static inline bool hugetlb_folio_is_cached(struct folio *folio)
+{
+	return folio_test_hugetlb_cached(folio);
+}
+
+static void hugetlb_cache_add(struct hstate *h, struct folio *folio, bool from_surplus)
+{
+	int nid = folio_nid(folio);
+
+	list_move_tail(&folio->lru, &h->hugepage_cache_lists[nid]);
+	folio_set_hugetlb_freed(folio);
+	folio_set_hugetlb_cached(folio);
+	h->nr_cached_hugepages++;
+	h->nr_cached_hugepages_node[nid]++;
+	if (from_surplus) {
+		h->surplus_huge_pages--;
+		h->surplus_huge_pages_node[nid]--;
+	}
+}
+
+static void hugetlb_cache_remove(struct hstate *h, struct folio *folio, bool to_surplus)
+{
+	int nid = folio_nid(folio);
+
+	list_del_init(&folio->lru);
+	folio_clear_hugetlb_freed(folio);
+	folio_clear_hugetlb_cached(folio);
+	h->nr_cached_hugepages--;
+	h->nr_cached_hugepages_node[nid]--;
+	if (to_surplus) {
+		h->surplus_huge_pages++;
+		h->surplus_huge_pages_node[nid]++;
+	}
+}
+#else
+static inline bool hugetlb_folio_is_cached(struct folio *folio)
+{
+	return false;
+}
+
+static inline void hugetlb_cache_add(struct hstate *h,
+				     struct folio *folio,
+				     bool from_surplus)
+{
+}
+
+static inline void hugetlb_cache_remove(struct hstate *h,
+					struct folio *folio,
+					bool to_surplus)
+{
+}
+#endif
+
 /*
  * Remove hugetlb folio from lists.
  * If vmemmap exists for the folio, clear the hugetlb flag so that the
@@ -1401,12 +1455,16 @@ void remove_hugetlb_folio(struct hstate *h, struct folio *folio,
 	if (hstate_is_gigantic_no_runtime(h))
 		return;
 
-	list_del(&folio->lru);
-
-	if (folio_test_hugetlb_freed(folio)) {
-		folio_clear_hugetlb_freed(folio);
-		h->free_huge_pages--;
-		h->free_huge_pages_node[nid]--;
+	if (hugetlb_folio_is_cached(folio)) {
+		if (folio_test_hugetlb_freed(folio))
+			hugetlb_cache_remove(h, folio, false);
+	} else {
+		list_del(&folio->lru);
+		if (folio_test_hugetlb_freed(folio)) {
+			folio_clear_hugetlb_freed(folio);
+			h->free_huge_pages--;
+			h->free_huge_pages_node[nid]--;
+		}
 	}
 	if (adjust_surplus) {
 		h->surplus_huge_pages--;
@@ -4169,8 +4227,18 @@ void __init hugetlb_add_hstate(unsigned int order)
 	__mutex_init(&h->resize_lock, "resize mutex", &h->resize_key);
 	h->order = order;
 	h->mask = ~(huge_page_size(h) - 1);
-	for (i = 0; i < MAX_NUMNODES; ++i)
+	for (i = 0; i < MAX_NUMNODES; ++i) {
 		INIT_LIST_HEAD(&h->hugepage_freelists[i]);
+#ifdef CONFIG_HUGETLB_CACHE
+		INIT_LIST_HEAD(&h->hugepage_cache_lists[i]);
+		h->nr_cached_hugepages_node[i] = 0;
+		h->max_cached_huge_pages_node[i] = 0;
+#endif
+	}
+#ifdef CONFIG_HUGETLB_CACHE
+	h->nr_cached_hugepages = 0;
+	h->max_cached_huge_pages = 0;
+#endif
 	INIT_LIST_HEAD(&h->hugepage_activelist);
 	snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
 					huge_page_size(h)/SZ_1K);
-- 
2.55.0.rc0.799.gd6f94ed593-goog