[PATCH v2 0/2] mm/slub: batch partial slab list operations

[PATCH v2 0/2] mm/slub: batch partial slab list operations

[Hao Li]

The loops in get_partial_node_bulk() and __refill_objects_node() perform
individual list operations for each slab. This patchset optimize this.

Patch 1 reduces the number of list operations by processing slabs in
batches, thereby minimizing the time spent inside the critical section.

Patch 2 is a cleanup patch that wraps duplicated code into a helper
function.

Changes in v2:
- Patch 1 apply the same optimization to __refill_objects_node. (Thanks
  Harry and Vlastimil)
- Patch 2 introduce a helper to wraps duplicated code. (Thanks Harry)

Hao Li (2):
  mm/slub: detach and reattach partial slabs in batch
  mm/slub: introduce helper to decrement partial slab count and clear
    flag

 mm/slub.c | 43 +++++++++++++++++++++++++++++++------------
 1 file changed, 31 insertions(+), 12 deletions(-)

base-commit: 6b41dc6749232c14f3e069f8115084a7bbeee402
-- 
2.54.0

[PATCH v2 1/2] mm/slub: detach and reattach partial slabs in batch

[Hao Li]

get_partial_node_bulk() moves each selected slab from the node's
partial list to the local pc->slabs list using a remove_partial() and
list_add() pair. In practice, the loop often detaches several adjacent
slabs. Doing this individually repeatedly manipulates list pointers
while holding n->list_lock, which causes unnecessary churn.

To demonstrate this, the counts below show how often single vs. multiple
consecutive slabs are retrieved during a will-it-scale mmap stress test:

consecutive_slabs_count        frequency
= 1                            277345324
= 2                            335238023
= 3                            175717884
>= 4                           88862337

The data confirms that retrieving multiple contiguous slabs is highly
frequent.

To optimize this, track contiguous runs of matching slabs and move each
run in a single operation using list_bulk_move_tail(). This reduces list
pointer churn inside the lock critical section.

Apply the same optimization to __refill_objects_node() when reattaching
leftover partial slabs back to the node's partial list.

The will-it-scale mmap benchmark shows a 2% ~ 5% performance improvement
after applying this patch.

Signed-off-by: Hao Li <hao.li@linux.dev>
---
 mm/slub.c | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index b9f10b869914..eb60b3da23ff 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3739,6 +3739,7 @@ static bool get_partial_node_bulk(struct kmem_cache *s,
 				  bool allow_spin)
 {
 	struct slab *slab, *slab2;
+	struct slab *first = NULL, *last = NULL;
 	unsigned int total_free = 0;
 	unsigned long flags;
 
@@ -3757,8 +3758,15 @@ static bool get_partial_node_bulk(struct kmem_cache *s,
 		struct freelist_counters flc;
 		unsigned int slab_free;
 
-		if (!pfmemalloc_match(slab, pc->flags))
+		if (!pfmemalloc_match(slab, pc->flags)) {
+			if (first) {
+				list_bulk_move_tail(&pc->slabs,
+						    &first->slab_list,
+						    &last->slab_list);
+				first = NULL;
+			}
 			continue;
+		}
 
 		/*
 		 * determine the number of free objects in the slab racily
@@ -3775,15 +3783,21 @@ static bool get_partial_node_bulk(struct kmem_cache *s,
 		    && total_free + slab_free > pc->max_objects)
 			break;
 
-		remove_partial(n, slab);
-
-		list_add(&slab->slab_list, &pc->slabs);
+		if (!first)
+			first = slab;
+		last = slab;
+		slab_clear_node_partial(slab);
+		n->nr_partial--;
 
 		total_free += slab_free;
 		if (total_free >= pc->max_objects)
 			break;
 	}
 
+	if (first)
+		list_bulk_move_tail(&pc->slabs, &first->slab_list,
+				    &last->slab_list);
+
 	spin_unlock_irqrestore(&n->list_lock, flags);
 	return total_free > 0;
 }
@@ -7194,12 +7208,13 @@ __refill_objects_node(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int mi
 	if (!list_empty(&pc.slabs)) {
 		spin_lock_irqsave(&n->list_lock, flags);
 
-		list_for_each_entry_safe(slab, slab2, &pc.slabs, slab_list) {
-
-			list_del(&slab->slab_list);
-			add_partial(n, slab, ADD_TO_TAIL);
+		list_for_each_entry(slab, &pc.slabs, slab_list) {
+			slab_set_node_partial(slab);
+			n->nr_partial++;
 		}
 
+		list_splice_tail(&pc.slabs, &n->partial);
+
 		spin_unlock_irqrestore(&n->list_lock, flags);
 	}
 
-- 
2.54.0

[PATCH v2 2/2] mm/slub: introduce helper to decrement partial slab count and clear flag

[Hao Li]

Wrap the partial slab count decrement and slab_clear_node_partial() into
a helper function to reduce code duplication.

Suggested-by: Harry Yoo <harry@kernel.org>
Signed-off-by: Hao Li <hao.li@linux.dev>
---
 mm/slub.c | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index eb60b3da23ff..71305290c8ca 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3539,13 +3539,19 @@ static inline void add_partial(struct kmem_cache_node *n,
 	__add_partial(n, slab, mode);
 }
 
+static inline void clear_node_partial_state(struct kmem_cache_node *n,
+					struct slab *slab)
+{
+	slab_clear_node_partial(slab);
+	n->nr_partial--;
+}
+
 static inline void remove_partial(struct kmem_cache_node *n,
 					struct slab *slab)
 {
 	lockdep_assert_held(&n->list_lock);
 	list_del(&slab->slab_list);
-	slab_clear_node_partial(slab);
-	n->nr_partial--;
+	clear_node_partial_state(n, slab);
 }
 
 /*
@@ -3786,8 +3792,7 @@ static bool get_partial_node_bulk(struct kmem_cache *s,
 		if (!first)
 			first = slab;
 		last = slab;
-		slab_clear_node_partial(slab);
-		n->nr_partial--;
+		clear_node_partial_state(n, slab);
 
 		total_free += slab_free;
 		if (total_free >= pc->max_objects)
@@ -8286,8 +8291,7 @@ static int __kmem_cache_do_shrink(struct kmem_cache *s)
 
 			if (free == slab->objects) {
 				list_move(&slab->slab_list, &discard);
-				slab_clear_node_partial(slab);
-				n->nr_partial--;
+				clear_node_partial_state(n, slab);
 				dec_slabs_node(s, node, slab->objects);
 			} else if (free <= SHRINK_PROMOTE_MAX)
 				list_move(&slab->slab_list, promote + free - 1);
-- 
2.54.0