[PATCH 2/6] mm/hugetlb: implement cache recycling and allocation

[Sourav Panda <souravpanda@google.com>]

Implement the core recycling and allocation logic for the HugeTLB dynamic
cache.

Recycle surplus hugepages in free_huge_folio() up to max_cached_huge_pages.
Allocate from the cache in alloc_surplus_hugetlb_folio() if pages are
available.

To ensure safety and architecture correctness:
- Skip recycling of hardware-poisoned pages in free_huge_folio() and
  skip poisoned pages in the cache during allocation.
- Call arch_clear_hugetlb_flags() before caching a page to clear stale
  architecture-specific metadata (e.g. ARM64 MTE tags, dcache dirty).
- Use MRU (Most Recently Used) policy for allocation (taking from the
  tail of the cache list) to prefer hot pages.

Signed-off-by: Sourav Panda <souravpanda@google.com>
---
 include/linux/hugetlb.h |  4 +++
 mm/hugetlb.c            | 76 +++++++++++++++++++++++++++++++++++++++--
 2 files changed, 77 insertions(+), 3 deletions(-)

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 4768f52ddd35..e882c99780b6 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -650,6 +650,10 @@ HPAGEFLAG(RawHwpUnreliable, raw_hwp_unreliable)
 HPAGEFLAG(Cma, cma)
 #ifdef CONFIG_HUGETLB_CACHE
 HPAGEFLAG(Cached, cached)
+#else
+static inline bool folio_test_hugetlb_cached(const struct folio *folio) { return false; }
+static inline void folio_clear_hugetlb_cached(struct folio *folio) { }
+static inline void folio_set_hugetlb_cached(struct folio *folio) { }
 #endif
 
 #ifdef CONFIG_HUGETLB_PAGE
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 15be3cf54606..d00aa67b8e13 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -125,6 +125,11 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
 		unsigned long start, unsigned long end, bool take_locks);
 static struct resv_map *vma_resv_map(struct vm_area_struct *vma);
 
+#ifdef CONFIG_HUGETLB_CACHE
+static void hugetlb_cache_add(struct hstate *h, struct folio *folio, bool from_surplus);
+static void hugetlb_cache_remove(struct hstate *h, struct folio *folio, bool to_surplus);
+#endif
+
 static inline bool subpool_is_free(struct hugepage_subpool *spool)
 {
 	if (spool->count)
@@ -1796,6 +1801,15 @@ void free_huge_folio(struct folio *folio)
 		spin_unlock_irqrestore(&hugetlb_lock, flags);
 		update_and_free_hugetlb_folio(h, folio, true);
 	} else if (h->surplus_huge_pages_node[nid]) {
+#ifdef CONFIG_HUGETLB_CACHE
+		if (h->nr_cached_hugepages < h->max_cached_huge_pages &&
+		    !folio_test_hwpoison(folio)) {
+			arch_clear_hugetlb_flags(folio);
+			hugetlb_cache_add(h, folio, true);
+			spin_unlock_irqrestore(&hugetlb_lock, flags);
+			return;
+		}
+#endif
 		/* remove the page from active list */
 		remove_hugetlb_folio(h, folio, true);
 		spin_unlock_irqrestore(&hugetlb_lock, flags);
@@ -2035,8 +2049,14 @@ int dissolve_free_hugetlb_folio(struct folio *folio)
 		struct hstate *h = folio_hstate(folio);
 		bool adjust_surplus = false;
 
-		if (!available_huge_pages(h))
+		if (!available_huge_pages(h) && !folio_test_hugetlb_cached(folio))
+			goto out;
+
+		/* If the folio is currently isolated for page reporting, skip it */
+		if (folio_test_hugetlb_cached(folio) && list_empty(&folio->lru)) {
+			rc = -EBUSY;
 			goto out;
+		}
 
 		/*
 		 * We should make sure that the page is already on the free list
@@ -2128,6 +2148,50 @@ int dissolve_free_hugetlb_folios(unsigned long start_pfn, unsigned long end_pfn)
 	return rc;
 }
 
+#ifdef CONFIG_HUGETLB_CACHE
+static struct folio *get_cached_folio(struct hstate *h, int nid, nodemask_t *nmask)
+{
+	int node;
+	struct folio *folio;
+
+	if (h->nr_cached_hugepages == 0)
+		return NULL;
+
+	if (nid != NUMA_NO_NODE && (!nmask || node_isset(nid, *nmask))) {
+		list_for_each_entry_reverse(folio, &h->hugepage_cache_lists[nid], lru) {
+			if (!folio_test_hwpoison(folio))
+				goto found;
+		}
+	}
+
+	if (nmask) {
+		for_each_node_mask(node, *nmask) {
+			list_for_each_entry_reverse(folio, &h->hugepage_cache_lists[node], lru) {
+				if (!folio_test_hwpoison(folio))
+					goto found;
+			}
+		}
+	} else {
+		for_each_node_state(node, N_MEMORY) {
+			list_for_each_entry_reverse(folio, &h->hugepage_cache_lists[node], lru) {
+				if (!folio_test_hwpoison(folio))
+					goto found;
+			}
+		}
+	}
+	return NULL;
+
+found:
+	hugetlb_cache_remove(h, folio, true);
+	return folio;
+}
+#else
+static inline struct folio *get_cached_folio(struct hstate *h, int nid, nodemask_t *nmask)
+{
+	return NULL;
+}
+#endif
+
 /*
  * Allocates a fresh surplus page from the page allocator.
  */
@@ -2136,10 +2200,16 @@ static struct folio *alloc_surplus_hugetlb_folio(struct hstate *h,
 {
 	struct folio *folio = NULL;
 
+	spin_lock_irq(&hugetlb_lock);
+	folio = get_cached_folio(h, nid, nmask);
+	if (folio) {
+		spin_unlock_irq(&hugetlb_lock);
+		return folio;
+	}
+
 	if (hstate_is_gigantic_no_runtime(h))
-		return NULL;
+		goto out_unlock;
 
-	spin_lock_irq(&hugetlb_lock);
 	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages)
 		goto out_unlock;
 	spin_unlock_irq(&hugetlb_lock);
-- 
2.55.0.rc0.799.gd6f94ed593-goog