[PATCH 3/6] mm/hugetlb: add sysfs interfaces for cache

[Sourav Panda <souravpanda@google.com>]

Introduce sysfs interfaces to control and monitor the HugeTLB dynamic
cache.

Expose the following attributes under:
/sys/kernel/mm/hugepages/hugepages-<size>/
- max_cached_huge_pages: Max limit of cached hugepages (global).
- nr_cached_hugepages: Current number of cached hugepages. Writing to
  this file allows manual expansion or contraction of the cache.
  These interfaces support writing delta values (e.g. "+1" or "-1")
  in addition to absolute values, allowing orchestrators (e.g. borglet)
  to dynamically scale the cache without racing against the kernel
  shrinker.

Also expose node-specific versions under node sysfs if CONFIG_NUMA.

Implement adjust_cached_huge_pages to handle the actual expansion
(allocating fresh hugepages to cache) and contraction
(freeing cached hugepages to buddy).

For expansion and contraction correctness:
- Global expansion interleaves allocations across allowed nodes.
- Node-specific expansion enforces the global cache limit ceiling and
  overcommit limits, uses __GFP_THISNODE to prevent fallback, and accounts
  to the actual allocated node using folio_nid().
- Contraction uses LRU (Least Recently Used) policy (taking from the
  head of the cache list) to discard cold pages first.

Signed-off-by: Sourav Panda <souravpanda@google.com>
---
 mm/hugetlb.c          | 230 ++++++++++++++++++++++++++++++++++++++++++
 mm/hugetlb_internal.h |   9 ++
 mm/hugetlb_sysfs.c    | 158 +++++++++++++++++++++++++++++
 3 files changed, 397 insertions(+)

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index d00aa67b8e13..f18a3123cbcb 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -4171,6 +4171,236 @@ long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed,
 	return -EBUSY;
 }
 
+#ifdef CONFIG_HUGETLB_CACHE
+static int adjust_cached_huge_pages(struct hstate *h, long count, bool is_delta, int nid,
+				    nodemask_t *nodes_allowed, bool update_limit)
+{
+	struct folio *folio;
+	int err = 0;
+
+	mutex_lock(&h->resize_lock);
+
+	if (is_delta) {
+		long current_val;
+
+		if (nid == NUMA_NO_NODE)
+			current_val = update_limit ? h->max_cached_huge_pages :
+						     h->nr_cached_hugepages;
+		else
+			current_val = update_limit ? h->max_cached_huge_pages_node[nid] :
+						     h->nr_cached_hugepages_node[nid];
+
+		current_val += count;
+		if (current_val < 0)
+			current_val = 0;
+		count = current_val;
+	}
+
+	if (nid == NUMA_NO_NODE) {
+		int temp_next_node = first_node(*nodes_allowed);
+
+		if (update_limit) {
+			h->max_cached_huge_pages = count;
+		} else if (count > h->max_cached_huge_pages) {
+			err = -EINVAL;
+			goto out;
+		}
+
+		/* Expansion */
+		while (!update_limit) {
+			unsigned long curr;
+
+			spin_lock_irq(&hugetlb_lock);
+			curr = h->nr_cached_hugepages;
+			spin_unlock_irq(&hugetlb_lock);
+
+			if (curr >= count)
+				break;
+
+			folio = alloc_pool_huge_folio(h, nodes_allowed,
+						      NULL, &temp_next_node);
+			if (!folio) {
+				err = -ENOMEM;
+				break;
+			}
+
+			spin_lock_irq(&hugetlb_lock);
+			if (h->nr_cached_hugepages >= count) {
+				spin_unlock_irq(&hugetlb_lock);
+				update_and_free_hugetlb_folio(h, folio, false);
+				break;
+			}
+
+			account_new_hugetlb_folio(h, folio);
+
+			hugetlb_cache_add(h, folio, false);
+			spin_unlock_irq(&hugetlb_lock);
+			cond_resched();
+		}
+
+		/* Contraction */
+		while (1) {
+			unsigned long curr;
+			struct folio *folio = NULL;
+
+			spin_lock_irq(&hugetlb_lock);
+			curr = h->nr_cached_hugepages;
+			if (curr <= count) {
+				spin_unlock_irq(&hugetlb_lock);
+				break;
+			}
+
+			int node;
+
+			for_each_node_mask(node, *nodes_allowed) {
+				if (!list_empty(&h->hugepage_cache_lists[node])) {
+					folio = list_first_entry(&h->hugepage_cache_lists[node],
+								 struct folio, lru);
+					break;
+				}
+			}
+
+			if (!folio) {
+				spin_unlock_irq(&hugetlb_lock);
+				break;
+			}
+
+			remove_hugetlb_folio(h, folio, false);
+			spin_unlock_irq(&hugetlb_lock);
+
+			update_and_free_hugetlb_folio(h, folio, false);
+			cond_resched();
+		}
+	} else {
+		if (update_limit) {
+			h->max_cached_huge_pages_node[nid] = count;
+		} else if (count > h->max_cached_huge_pages_node[nid]) {
+			err = -EINVAL;
+			goto out;
+		}
+
+		/* Node expansion */
+		while (!update_limit) {
+			unsigned long curr;
+			gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
+
+			spin_lock_irq(&hugetlb_lock);
+			curr = h->nr_cached_hugepages_node[nid];
+			if (h->nr_cached_hugepages >= h->max_cached_huge_pages) {
+				spin_unlock_irq(&hugetlb_lock);
+				err = -ENOSPC;
+				break;
+			}
+			spin_unlock_irq(&hugetlb_lock);
+
+			if (curr >= count)
+				break;
+
+			folio = alloc_fresh_hugetlb_folio(h, gfp_mask,
+							  nid, nodes_allowed);
+			if (!folio) {
+				err = -ENOMEM;
+				break;
+			}
+
+			spin_lock_irq(&hugetlb_lock);
+			if (h->nr_cached_hugepages_node[nid] >= count) {
+				spin_unlock_irq(&hugetlb_lock);
+				update_and_free_hugetlb_folio(h, folio, false);
+				break;
+			}
+
+			account_new_hugetlb_folio(h, folio);
+
+			hugetlb_cache_add(h, folio, false);
+			spin_unlock_irq(&hugetlb_lock);
+			cond_resched();
+		}
+
+		/* Node contraction */
+		while (1) {
+			unsigned long curr;
+			struct folio *folio = NULL;
+
+			spin_lock_irq(&hugetlb_lock);
+			curr = h->nr_cached_hugepages_node[nid];
+			if (curr <= count) {
+				spin_unlock_irq(&hugetlb_lock);
+				break;
+			}
+
+			if (!list_empty(&h->hugepage_cache_lists[nid]))
+				folio = list_first_entry(&h->hugepage_cache_lists[nid],
+							 struct folio, lru);
+
+			if (!folio) {
+				spin_unlock_irq(&hugetlb_lock);
+				break;
+			}
+
+			remove_hugetlb_folio(h, folio, false);
+			spin_unlock_irq(&hugetlb_lock);
+
+			update_and_free_hugetlb_folio(h, folio, false);
+			cond_resched();
+		}
+	}
+
+out:
+	mutex_unlock(&h->resize_lock);
+	return err;
+}
+
+ssize_t __nr_cached_hugepages_store_common(bool obey_mempolicy,
+					   struct hstate *h, int nid,
+					   long count, bool is_delta, size_t len)
+{
+	int err;
+	nodemask_t nodes_allowed, *n_mask;
+
+	if (hstate_is_gigantic_no_runtime(h))
+		return -EINVAL;
+
+	if (nid == NUMA_NO_NODE) {
+		if (!(obey_mempolicy &&
+		      init_nodemask_of_mempolicy(&nodes_allowed)))
+			n_mask = &node_states[N_MEMORY];
+		else
+			n_mask = &nodes_allowed;
+	} else {
+		init_nodemask_of_node(&nodes_allowed, nid);
+		n_mask = &nodes_allowed;
+	}
+
+	err = adjust_cached_huge_pages(h, count, is_delta, nid, n_mask, false);
+
+	return err ? err : len;
+}
+
+ssize_t __max_cached_huge_pages_store_common(bool obey_mempolicy,
+					     struct hstate *h, int nid,
+					     long count, bool is_delta, size_t len)
+{
+	int err;
+	nodemask_t nodes_allowed, *n_mask;
+
+	if (hstate_is_gigantic_no_runtime(h))
+		return -EINVAL;
+
+	if (nid == NUMA_NO_NODE) {
+		if (!(obey_mempolicy &&
+		      init_nodemask_of_mempolicy(&nodes_allowed)))
+			n_mask = &node_states[N_MEMORY];
+		else
+			n_mask = &nodes_allowed;
+	} else {
+		init_nodemask_of_node(&nodes_allowed, nid);
+		n_mask = &nodes_allowed;
+	}
+
+	err = adjust_cached_huge_pages(h, count, is_delta, nid, n_mask, true);
+
+	return err ? err : len;
+}
+#endif /* CONFIG_HUGETLB_CACHE */
+
 ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
 					   struct hstate *h, int nid,
 					   unsigned long count, size_t len)
diff --git a/mm/hugetlb_internal.h b/mm/hugetlb_internal.h
index 1d2f870deccf..16ed9ac9dc05 100644
--- a/mm/hugetlb_internal.h
+++ b/mm/hugetlb_internal.h
@@ -114,4 +114,13 @@ extern void hugetlb_sysctl_init(void);
 static inline void hugetlb_sysctl_init(void) { }
 #endif
 
+#ifdef CONFIG_HUGETLB_CACHE
+ssize_t __nr_cached_hugepages_store_common(bool obey_mempolicy,
+					   struct hstate *h, int nid,
+					   long count, bool is_delta, size_t len);
+ssize_t __max_cached_huge_pages_store_common(bool obey_mempolicy,
+					     struct hstate *h, int nid,
+					     long count, bool is_delta, size_t len);
+#endif
+
 #endif /* _LINUX_HUGETLB_INTERNAL_H */
diff --git a/mm/hugetlb_sysfs.c b/mm/hugetlb_sysfs.c
index 79ece91406bf..682a3805e6ff 100644
--- a/mm/hugetlb_sysfs.c
+++ b/mm/hugetlb_sysfs.c
@@ -277,6 +277,152 @@ static ssize_t demote_size_store(struct kobject *kobj,
 }
 HSTATE_ATTR(demote_size);
 
+#ifdef CONFIG_HUGETLB_CACHE
+static ssize_t nr_cached_hugepages_show_common(struct kobject *kobj,
+					       struct kobj_attribute *attr, char *buf)
+{
+	struct hstate *h;
+	unsigned long nr_cached;
+	int nid;
+
+	h = kobj_to_hstate(kobj, &nid);
+	if (nid == NUMA_NO_NODE)
+		nr_cached = h->nr_cached_hugepages;
+	else
+		nr_cached = h->nr_cached_hugepages_node[nid];
+
+	return sysfs_emit(buf, "%lu\n", nr_cached);
+}
+
+static ssize_t nr_cached_hugepages_store_common(bool obey_mempolicy,
+						struct kobject *kobj, const char *buf,
+						size_t len)
+{
+	struct hstate *h;
+	long count;
+	bool is_delta = false;
+	int nid;
+	int err;
+	const char *p = skip_spaces(buf);
+
+	if (*p == '+' || *p == '-') {
+		is_delta = true;
+		err = kstrtol(p, 10, &count);
+	} else {
+		unsigned long ucount;
+
+		err = kstrtoul(p, 10, &ucount);
+		count = (long)ucount;
+	}
+	if (err)
+		return err;
+
+	h = kobj_to_hstate(kobj, &nid);
+	return __nr_cached_hugepages_store_common(obey_mempolicy, h, nid, count, is_delta, len);
+}
+
+static ssize_t nr_cached_hugepages_show(struct kobject *kobj,
+					struct kobj_attribute *attr, char *buf)
+{
+	return nr_cached_hugepages_show_common(kobj, attr, buf);
+}
+
+static ssize_t nr_cached_hugepages_store(struct kobject *kobj,
+					 struct kobj_attribute *attr, const char *buf, size_t len)
+{
+	return nr_cached_hugepages_store_common(false, kobj, buf, len);
+}
+HSTATE_ATTR(nr_cached_hugepages);
+
+#ifdef CONFIG_NUMA
+static ssize_t nr_cached_hugepages_mempolicy_show(struct kobject *kobj,
+						  struct kobj_attribute *attr,
+						  char *buf)
+{
+	return nr_cached_hugepages_show_common(kobj, attr, buf);
+}
+
+static ssize_t nr_cached_hugepages_mempolicy_store(struct kobject *kobj,
+						   struct kobj_attribute *attr,
+						   const char *buf, size_t len)
+{
+	return nr_cached_hugepages_store_common(true, kobj, buf, len);
+}
+HSTATE_ATTR(nr_cached_hugepages_mempolicy);
+#endif
+
+static ssize_t max_cached_huge_pages_show_common(struct kobject *kobj,
+						 struct kobj_attribute *attr, char *buf)
+{
+	struct hstate *h;
+	unsigned long max_cached;
+	int nid;
+
+	h = kobj_to_hstate(kobj, &nid);
+	if (nid == NUMA_NO_NODE)
+		max_cached = h->max_cached_huge_pages;
+	else
+		max_cached = h->max_cached_huge_pages_node[nid];
+
+	return sysfs_emit(buf, "%lu\n", max_cached);
+}
+
+static ssize_t max_cached_huge_pages_store_common(bool obey_mempolicy,
+						  struct kobject *kobj, const char *buf,
+						  size_t len)
+{
+	struct hstate *h;
+	long count;
+	bool is_delta = false;
+	int nid;
+	int err;
+	const char *p = skip_spaces(buf);
+
+	if (*p == '+' || *p == '-') {
+		is_delta = true;
+		err = kstrtol(p, 10, &count);
+	} else {
+		unsigned long ucount;
+
+		err = kstrtoul(p, 10, &ucount);
+		count = (long)ucount;
+	}
+	if (err)
+		return err;
+
+	h = kobj_to_hstate(kobj, &nid);
+	return __max_cached_huge_pages_store_common(obey_mempolicy, h, nid, count, is_delta, len);
+}
+
+static ssize_t max_cached_huge_pages_show(struct kobject *kobj,
+					  struct kobj_attribute *attr, char *buf)
+{
+	return max_cached_huge_pages_show_common(kobj, attr, buf);
+}
+
+static ssize_t max_cached_huge_pages_store(struct kobject *kobj,
+					   struct kobj_attribute *attr, const char *buf, size_t len)
+{
+	return max_cached_huge_pages_store_common(false, kobj, buf, len);
+}
+HSTATE_ATTR(max_cached_huge_pages);
+
+#ifdef CONFIG_NUMA
+static ssize_t max_cached_huge_pages_mempolicy_show(struct kobject *kobj,
+						    struct kobj_attribute *attr,
+						    char *buf)
+{
+	return max_cached_huge_pages_show_common(kobj, attr, buf);
+}
+
+static ssize_t max_cached_huge_pages_mempolicy_store(struct kobject *kobj,
+						     struct kobj_attribute *attr,
+						     const char *buf, size_t len)
+{
+	return max_cached_huge_pages_store_common(true, kobj, buf, len);
+}
+HSTATE_ATTR(max_cached_huge_pages_mempolicy);
+#endif
+#endif /* CONFIG_HUGETLB_CACHE */
+
 static struct attribute *hstate_attrs[] = {
 	&nr_hugepages_attr.attr,
 	&nr_overcommit_hugepages_attr.attr,
@@ -285,6 +431,14 @@ static struct attribute *hstate_attrs[] = {
 	&surplus_hugepages_attr.attr,
 #ifdef CONFIG_NUMA
 	&nr_hugepages_mempolicy_attr.attr,
+#endif
+#ifdef CONFIG_HUGETLB_CACHE
+	&nr_cached_hugepages_attr.attr,
+	&max_cached_huge_pages_attr.attr,
+#ifdef CONFIG_NUMA
+	&nr_cached_hugepages_mempolicy_attr.attr,
+	&max_cached_huge_pages_mempolicy_attr.attr,
+#endif
 #endif
 	NULL,
 };
@@ -359,6 +513,10 @@ static struct attribute *per_node_hstate_attrs[] = {
 	&nr_hugepages_attr.attr,
 	&free_hugepages_attr.attr,
 	&surplus_hugepages_attr.attr,
+#ifdef CONFIG_HUGETLB_CACHE
+	&nr_cached_hugepages_attr.attr,
+	&max_cached_huge_pages_attr.attr,
+#endif
 	NULL,
 };
 
-- 
2.55.0.rc0.799.gd6f94ed593-goog