[PATCH v6 11/12] mm, swap: add vmstat and memcg counters for vswap

[Nhat Pham <nphamcs@gmail.com>]

Add nr_vswap to /proc/vmstat and vswap to cgroup's memory.stat to report
virtual swap usage.

These counters help diagnose swapping issues. Userspace can also use
vswap as a component of a workload's logical memory footprint when
making policy decisions.

Suggested-by: Johannes Weiner <hannes@cmpxchg.org>
Signed-off-by: Nhat Pham <nphamcs@gmail.com>
---
 .../admin-guide/cgroup-v1/memory.rst          |  1 +
 Documentation/admin-guide/cgroup-v2.rst       |  3 +++
 include/linux/memcontrol.h                    |  1 +
 include/linux/swap.h                          |  1 +
 include/linux/vmstat.h                        |  3 +++
 mm/memcontrol-v1.c                            |  6 +++++-
 mm/memcontrol.c                               | 20 +++++++++++++++----
 mm/swapfile.c                                 | 10 +++++++---
 mm/vmstat.c                                   |  6 ++++++
 9 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/Documentation/admin-guide/cgroup-v1/memory.rst b/Documentation/admin-guide/cgroup-v1/memory.rst
index 2a67356d25a0..8ca9ef2bca9f 100644
--- a/Documentation/admin-guide/cgroup-v1/memory.rst
+++ b/Documentation/admin-guide/cgroup-v1/memory.rst
@@ -555,6 +555,7 @@ memory.stat file includes following statistics:
                     cgroup.
     swap            # of bytes of physical swap usage
     swapcached      # of bytes of swap cached in memory
+    vswap           # of bytes of virtual swap usage
     dirty           # of bytes that are waiting to get written back to the disk.
     writeback       # of bytes of file/anon cache that are queued for syncing to
                     disk.
diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst
index 623d29ea8c41..b1f97a61a1dc 100644
--- a/Documentation/admin-guide/cgroup-v2.rst
+++ b/Documentation/admin-guide/cgroup-v2.rst
@@ -1633,6 +1633,9 @@ The following nested keys are defined.
 		against memory usage, and against swap usage when the swap
 		entry has physical backing.
 
+	  vswap
+		Amount of virtual swap usage.
+
 	  anon_thp
 		Amount of memory used in anonymous mappings backed by
 		transparent hugepages
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index b9072a3c9a2a..99beb676d8e7 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -41,6 +41,7 @@ enum memcg_stat_item {
 	MEMCG_ZSWAP_B,
 	MEMCG_ZSWAPPED,
 	MEMCG_ZSWAP_INCOMP,
+	MEMCG_VSWAP,
 	MEMCG_NR_STAT,
 };
 
diff --git a/include/linux/swap.h b/include/linux/swap.h
index a05189c92dae..891442264ca9 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -371,6 +371,7 @@ bool current_is_kswapd(void);
 
 #ifdef CONFIG_SWAP
 bool mem_cgroup_can_vswap(struct mem_cgroup *memcg);
+unsigned long vswap_usage_in_pages(void);
 
 int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
 		unsigned long nr_pages, sector_t start_block);
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index 5b31d8e7ae40..37aff1ffac08 100644
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -33,6 +33,9 @@ enum vm_stat_item {
 	NR_DIRTY_BG_THRESHOLD,
 	NR_MEMMAP_PAGES,	/* page metadata allocated through buddy allocator */
 	NR_MEMMAP_BOOT_PAGES,	/* page metadata allocated through boot allocator */
+#ifdef CONFIG_SWAP
+	NR_VSWAP,		/* pages occupying a virtual swap slot */
+#endif
 	NR_VM_STAT_ITEMS,
 };
 
diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c
index 4e4946728db5..8fa7b2fb109d 100644
--- a/mm/memcontrol-v1.c
+++ b/mm/memcontrol-v1.c
@@ -300,7 +300,9 @@ void __memcg1_swapout(struct folio *folio, struct swap_cluster_info *ci)
 	 */
 	nr_entries = folio_nr_pages(folio);
 	private_id = mem_cgroup_private_id_get(memcg, nr_entries);
-	if (!is_vswap_entry(folio->swap))
+	if (is_vswap_entry(folio->swap))
+		mod_memcg_state(memcg, MEMCG_VSWAP, nr_entries);
+	else
 		mod_memcg_state(memcg, MEMCG_SWAP, nr_entries);
 
 	__swap_cgroup_set(ci, swp_cluster_offset(folio->swap), nr_entries, private_id);
@@ -1842,6 +1844,7 @@ static const unsigned int memcg1_stats[] = {
 #ifdef CONFIG_SWAP
 	MEMCG_SWAP,
 	NR_SWAPCACHE,
+	MEMCG_VSWAP,
 #endif
 };
 
@@ -1860,6 +1863,7 @@ static const char *const memcg1_stat_names[] = {
 #ifdef CONFIG_SWAP
 	"swap",
 	"swapcached",
+	"vswap",
 #endif
 };
 
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index e2aa75a292ed..eedaa094a867 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -66,6 +66,7 @@
 #include "internal.h"
 #include "swap.h"
 #include "swap_table.h"
+#include "vswap.h"
 #include <net/sock.h>
 #include <net/ip.h>
 #include "slab.h"
@@ -459,6 +460,7 @@ static const unsigned int memcg_stat_items[] = {
 	MEMCG_ZSWAP_B,
 	MEMCG_ZSWAPPED,
 	MEMCG_ZSWAP_INCOMP,
+	MEMCG_VSWAP,
 };
 
 #define NR_MEMCG_NODE_STAT_ITEMS ARRAY_SIZE(memcg_node_stat_items)
@@ -1628,6 +1630,7 @@ static const struct memory_stat memory_stats[] = {
 	{ "file_writeback",		NR_WRITEBACK			},
 #ifdef CONFIG_SWAP
 	{ "swapcached",			NR_SWAPCACHE			},
+	{ "vswap",			MEMCG_VSWAP			},
 #endif
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 	{ "anon_thp",			NR_ANON_THPS			},
@@ -6004,6 +6007,7 @@ unsigned short __mem_cgroup_swap_get(struct folio *folio)
 	unsigned int nr_pages = folio_nr_pages(folio);
 	struct mem_cgroup *memcg;
 	struct obj_cgroup *objcg;
+	unsigned short id;
 
 	if (do_memsw_account())
 		return 0;
@@ -6020,7 +6024,12 @@ unsigned short __mem_cgroup_swap_get(struct folio *folio)
 		return 0;
 	}
 
-	return mem_cgroup_private_id_get(memcg, nr_pages);
+	id = mem_cgroup_private_id_get(memcg, nr_pages);
+	if (is_vswap_entry(folio->swap)) {
+		memcg = obj_cgroup_memcg(obj_cgroup_from_private_id(id));
+		mod_memcg_state(memcg, MEMCG_VSWAP, nr_pages);
+	}
+	return id;
 }
 
 /**
@@ -6109,7 +6118,7 @@ void __mem_cgroup_swap_uncharge(unsigned short id, unsigned int nr_pages)
  * __mem_cgroup_swap_put - drop the private ID refs taken for swap slots
  * @id: the pinned private ID
  * @nr_pages: number of refs to drop
- * @is_vswap: whether the released swap slots are virtual
+ * @is_vswap: release virtual-swap accounting along with the refs
  */
 void __mem_cgroup_swap_put(unsigned short id, unsigned int nr_pages,
 			   bool is_vswap)
@@ -6117,9 +6126,12 @@ void __mem_cgroup_swap_put(unsigned short id, unsigned int nr_pages,
 	struct mem_cgroup *memcg;
 
 	guard(rcu)();
-	if (do_memsw_account() && !is_vswap) {
+	if (is_vswap || do_memsw_account()) {
 		memcg = obj_cgroup_memcg(obj_cgroup_from_private_id(id));
-		mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
+		if (is_vswap)
+			mod_memcg_state(memcg, MEMCG_VSWAP, -nr_pages);
+		else
+			mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
 	}
 	mem_cgroup_private_id_put(id, nr_pages);
 }
diff --git a/mm/swapfile.c b/mm/swapfile.c
index e95f2441fdd7..fe5460d15f55 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -2612,9 +2612,8 @@ static void __swap_cluster_free_phys_backing(struct swap_info_struct *psi,
 }
 
 /*
- * Release the cgroup accounting of a batch of freed slots. For vswap the
- * physical swap was already uncharged by __vswap_release_backing(), so only
- * the ID ref is left to drop.
+ * Release cgroup accounting for freed swap slots. Physical backing of
+ * vswap entries was already uncharged by __vswap_release_backing().
  */
 static void memcg_swap_free(unsigned short id, unsigned int nr, bool is_vswap)
 {
@@ -4922,6 +4921,11 @@ early_param("vswap", early_vswap);
 /* vswap does no IO on its own. */
 static const struct swap_ops vswap_ops = { };
 
+unsigned long vswap_usage_in_pages(void)
+{
+	return vswap_is_enabled() ? swap_usage_in_pages(vswap_si) : 0;
+}
+
 static int vswap_used_get(void *data, u64 *val)
 {
 	*val = swap_usage_in_pages(vswap_si);
diff --git a/mm/vmstat.c b/mm/vmstat.c
index a3e809c57f29..b7077a6f244d 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1320,6 +1320,9 @@ const char * const vmstat_text[] = {
 	[I(NR_DIRTY_BG_THRESHOLD)]		= "nr_dirty_background_threshold",
 	[I(NR_MEMMAP_PAGES)]			= "nr_memmap_pages",
 	[I(NR_MEMMAP_BOOT_PAGES)]		= "nr_memmap_boot_pages",
+#ifdef CONFIG_SWAP
+	[I(NR_VSWAP)]				= "nr_vswap",
+#endif
 #undef I
 
 #if defined(CONFIG_VM_EVENT_COUNTERS)
@@ -1936,6 +1939,9 @@ static void *vmstat_start(struct seq_file *m, loff_t *pos)
 			    v + NR_DIRTY_THRESHOLD);
 	v[NR_MEMMAP_PAGES] = atomic_long_read(&nr_memmap_pages);
 	v[NR_MEMMAP_BOOT_PAGES] = atomic_long_read(&nr_memmap_boot_pages);
+#ifdef CONFIG_SWAP
+	v[NR_VSWAP] = vswap_usage_in_pages();
+#endif
 	v += NR_VM_STAT_ITEMS;
 
 #ifdef CONFIG_VM_EVENT_COUNTERS
-- 
2.53.0-Meta