Re: [PATCH v3] mm: page_alloc: make defrag_mode retries follow the promoted order

[Kiryl Shutsemau <kas@kernel.org>]

On Fri, Oct 23, 2026 at 11:59:06AM +0100, Kiryl Shutsemau wrote:
> 64K allocation is still lower with defrag_mode. I will look into this
> further.

Direct reclaim has the same problem.

compaction_ready() and should_continue_reclaim() ask the same watermark
check whether the allocation can already succeed. The free order-4+
pages left in unmovable pageblocks make it say yes, and reclaim stops
early.

I tried to generalize the fix: __zone_watermark_ok() takes the request's
migratetype and, under ALLOC_NOFRAGMENT, does not count other
migratetypes' free pages below pageblock_order. Direct reclaim learns
ALLOC_NOFRAGMENT from the allocation.

Medians of 15 runs of "compact-bench 99 64 16 20 0":

                             defrag_mode=0  defrag_mode=1  defrag_mode=1
                                                           + diff
    compact_stall                    2135          32476          8022
    compact_fail                      668          32330          4880
    64K anon_fault_alloc            20689           6780         19649
    64K anon_fault_fallback           776          14502          1851

The diff below replaces the previous one. It needs cleanup and splitting
before it goes upstream.

Could you give it a try?

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 94f9c3ff5416..f924940b5475 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -1629,7 +1629,7 @@ static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat)
 void build_all_zonelists(pg_data_t *pgdat);
 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
 			 int highest_zoneidx, unsigned int alloc_flags,
-			 long free_pages);
+			 long free_pages, int migratetype);
 bool zone_watermark_ok(struct zone *z, unsigned int order,
 		unsigned long mark, int highest_zoneidx,
 		unsigned int alloc_flags);
diff --git a/mm/compaction.c b/mm/compaction.c
index a049415512c6..420ff6c735eb 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -2344,7 +2344,8 @@ static enum compact_result __compact_finished(struct compact_control *cc)
 					high_wmark_pages(cc->zone),
 					cc->highest_zoneidx, cc->alloc_flags,
 					zone_page_state(cc->zone,
-							NR_FREE_PAGES_BLOCKS)))
+							NR_FREE_PAGES_BLOCKS),
+					cc->migratetype))
 			return COMPACT_SUCCESS;
 
 		return COMPACT_CONTINUE;
@@ -2365,6 +2366,11 @@ static enum compact_result __compact_finished(struct compact_control *cc)
 			!free_area_empty(area, MIGRATE_CMA))
 			return COMPACT_SUCCESS;
 #endif
+		/* Under ALLOC_NOFRAGMENT it may only claim a whole block */
+		if ((cc->alloc_flags & ALLOC_NOFRAGMENT) &&
+		    order < pageblock_order)
+			continue;
+
 		/*
 		 * Job done if allocation would steal freepages from
 		 * other migratetype buddy lists.
@@ -2425,7 +2431,7 @@ static bool __compaction_suitable(struct zone *zone, int order,
 	if (order > PAGE_ALLOC_COSTLY_ORDER)
 		watermark += low_wmark_pages(zone) - min_wmark_pages(zone);
 	return __zone_watermark_ok(zone, 0, watermark, highest_zoneidx,
-				   ALLOC_CMA, free_pages);
+				   ALLOC_CMA, free_pages, MIGRATE_MOVABLE);
 }
 
 /*
@@ -2520,7 +2526,7 @@ bool compaction_zonelist_suitable(struct alloc_context *ac, int order,
 static enum compact_result
 compaction_suit_allocation_order(struct zone *zone, unsigned int order,
 				 int highest_zoneidx, unsigned int alloc_flags,
-				 bool async, bool kcompactd)
+				 int migratetype, bool async, bool kcompactd)
 {
 	unsigned long free_pages;
 	unsigned long watermark;
@@ -2532,7 +2538,7 @@ compaction_suit_allocation_order(struct zone *zone, unsigned int order,
 
 	watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
 	if (__zone_watermark_ok(zone, order, watermark, highest_zoneidx,
-				alloc_flags, free_pages))
+				alloc_flags, free_pages, migratetype))
 		return COMPACT_SUCCESS;
 
 	/*
@@ -2548,7 +2554,8 @@ compaction_suit_allocation_order(struct zone *zone, unsigned int order,
 	    !(alloc_flags & ALLOC_CMA)) {
 		if (!__zone_watermark_ok(zone, 0, watermark + compact_gap(order),
 					 highest_zoneidx, 0,
-					 zone_page_state(zone, NR_FREE_PAGES)))
+					 zone_page_state(zone, NR_FREE_PAGES),
+					 migratetype))
 			return COMPACT_SKIPPED;
 	}
 
@@ -2588,6 +2595,7 @@ compact_zone(struct compact_control *cc, struct capture_control *capc)
 		ret = compaction_suit_allocation_order(cc->zone, cc->order,
 						       cc->highest_zoneidx,
 						       cc->alloc_flags,
+						       cc->migratetype,
 						       cc->mode == MIGRATE_ASYNC,
 						       !cc->direct_compaction);
 		if (ret != COMPACT_CONTINUE)
@@ -3073,7 +3081,7 @@ static bool kcompactd_node_suitable(pg_data_t *pgdat)
 		ret = compaction_suit_allocation_order(zone,
 				pgdat->kcompactd_max_order,
 				highest_zoneidx, alloc_flags,
-				false, true);
+				MIGRATE_MOVABLE, false, true);
 		if (ret == COMPACT_CONTINUE)
 			return true;
 	}
@@ -3116,7 +3124,7 @@ static void kcompactd_do_work(pg_data_t *pgdat)
 
 		ret = compaction_suit_allocation_order(zone,
 				cc.order, zoneid, cc.alloc_flags,
-				false, true);
+				MIGRATE_MOVABLE, false, true);
 		if (ret != COMPACT_CONTINUE)
 			continue;
 
diff --git a/mm/internal.h b/mm/internal.h
index 38b1165212c9..5a7556999ad2 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -68,7 +68,8 @@ void folio_mark_lazyfree(struct folio *folio);
 /* mm/vmscan.c */
 unsigned long zone_reclaimable_pages(struct zone *zone);
 unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
-				gfp_t gfp_mask, const nodemask_t *mask);
+				gfp_t gfp_mask, const nodemask_t *mask,
+				bool nofragment);
 unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru,
 			      int zone_idx);
 
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 12fac9084c48..83a52acf9c9c 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -3597,7 +3597,7 @@ static inline long __zone_watermark_unusable_free(struct zone *z,
  */
 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
 			 int highest_zoneidx, unsigned int alloc_flags,
-			 long free_pages)
+			 long free_pages, int migratetype)
 {
 	long min = mark;
 	int o;
@@ -3655,6 +3655,14 @@ bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
 			continue;
 
 		for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
+			/*
+			 * Under ALLOC_NOFRAGMENT, a request below
+			 * pageblock_order can take another migratetype's page
+			 * only by claiming the whole block.
+			 */
+			if ((alloc_flags & ALLOC_NOFRAGMENT) &&
+			    mt != migratetype && o < pageblock_order)
+				continue;
 			if (!free_area_empty(area, mt))
 				return true;
 		}
@@ -3676,8 +3684,11 @@ bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
 		      int highest_zoneidx, unsigned int alloc_flags)
 {
-	return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
-					zone_page_state(z, NR_FREE_PAGES));
+	/* Without the request's migratetype, count every migratetype's pages */
+	return __zone_watermark_ok(z, order, mark, highest_zoneidx,
+				   alloc_flags & ~ALLOC_NOFRAGMENT,
+				   zone_page_state(z, NR_FREE_PAGES),
+				   MIGRATE_MOVABLE);
 }
 
 static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
@@ -3706,7 +3717,7 @@ static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
 	}
 
 	if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
-					free_pages))
+				free_pages, gfp_migratetype(gfp_mask)))
 		return true;
 
 	/*
@@ -3719,7 +3730,8 @@ static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
 		&& ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) {
 		mark = z->_watermark[WMARK_MIN];
 		return __zone_watermark_ok(z, order, mark, highest_zoneidx,
-					alloc_flags, free_pages);
+					alloc_flags, free_pages,
+					gfp_migratetype(gfp_mask));
 	}
 
 	return false;
@@ -4435,7 +4447,7 @@ static unsigned int check_retry_zonelist(unsigned int seq)
 
 /* Perform direct synchronous page reclaim */
 static unsigned long
-__perform_reclaim(gfp_t gfp_mask, unsigned int order,
+__perform_reclaim(gfp_t gfp_mask, unsigned int order, unsigned int alloc_flags,
 					const struct alloc_context *ac)
 {
 	unsigned int noreclaim_flag;
@@ -4449,7 +4461,8 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order,
 	noreclaim_flag = memalloc_noreclaim_save();
 
 	progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
-								ac->nodemask);
+				     ac->nodemask,
+				     alloc_flags & ALLOC_NOFRAGMENT);
 
 	memalloc_noreclaim_restore(noreclaim_flag);
 	fs_reclaim_release(gfp_mask);
@@ -4475,7 +4488,8 @@ __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
 		reclaim_order = max(order, pageblock_order);
 
 	psi_memstall_enter(&pflags);
-	*did_some_progress = __perform_reclaim(gfp_mask, reclaim_order, ac);
+	*did_some_progress = __perform_reclaim(gfp_mask, reclaim_order,
+					       alloc_flags, ac);
 	if (unlikely(!(*did_some_progress)))
 		goto out;
 
@@ -4682,10 +4696,12 @@ should_reclaim_retry(gfp_t gfp_mask, unsigned order,
 
 		/*
 		 * Would the allocation succeed if we reclaimed all
-		 * reclaimable pages?
+		 * reclaimable pages?  Count free pages of every migratetype.
 		 */
 		wmark = __zone_watermark_ok(zone, order, min_wmark,
-				ac->highest_zoneidx, alloc_flags, available);
+				ac->highest_zoneidx,
+				alloc_flags & ~ALLOC_NOFRAGMENT,
+				available, ac->migratetype);
 		trace_reclaim_retry_zone(z, order, reclaimable,
 				available, min_wmark, *no_progress_loops, wmark);
 		if (wmark) {
diff --git a/mm/vmscan.c b/mm/vmscan.c
index f11491ee9ed5..f7c561c40e55 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -142,6 +142,9 @@ struct scan_control {
 	/* One of the zones is ready for compaction */
 	unsigned int compaction_ready:1;
 
+	/* The allocation cannot fall back to other migratetypes' blocks */
+	unsigned int nofragment:1;
+
 	/* There is easily reclaimable cold cache in the current node */
 	unsigned int cache_trim_mode:1;
 
@@ -6098,6 +6101,17 @@ static bool in_reclaim_compaction(struct scan_control *sc)
 	return false;
 }
 
+static bool allocation_can_succeed(struct zone *zone, struct scan_control *sc,
+				   unsigned long mark)
+{
+	unsigned int alloc_flags = sc->nofragment ? ALLOC_NOFRAGMENT : 0;
+	long free_pages = zone_page_state(zone, NR_FREE_PAGES);
+	int migratetype = gfp_migratetype(sc->gfp_mask);
+
+	return __zone_watermark_ok(zone, sc->order, mark, sc->reclaim_idx,
+				   alloc_flags, free_pages, migratetype);
+}
+
 /*
  * Reclaim/compaction is used for high-order allocation requests. It reclaims
  * order-0 pages before compacting the zone. should_continue_reclaim() returns
@@ -6136,8 +6150,7 @@ static inline bool should_continue_reclaim(struct pglist_data *pgdat,
 		unsigned long watermark = min_wmark_pages(zone);
 
 		/* Allocation can already succeed, nothing to do */
-		if (zone_watermark_ok(zone, sc->order, watermark,
-				      sc->reclaim_idx, 0))
+		if (allocation_can_succeed(zone, sc, watermark))
 			return false;
 
 		if (compaction_suitable(zone, sc->order, watermark,
@@ -6362,8 +6375,7 @@ static inline bool compaction_ready(struct zone *zone, struct scan_control *sc)
 		return false;
 
 	/* Allocation can already succeed, nothing to do */
-	if (zone_watermark_ok(zone, sc->order, min_wmark_pages(zone),
-			      sc->reclaim_idx, 0))
+	if (allocation_can_succeed(zone, sc, min_wmark_pages(zone)))
 		return true;
 
 	/*
@@ -6767,7 +6779,8 @@ static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
 }
 
 unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
-				gfp_t gfp_mask, const nodemask_t *nodemask)
+				gfp_t gfp_mask, const nodemask_t *nodemask,
+				bool nofragment)
 {
 	unsigned long nr_reclaimed;
 	struct scan_control sc = {
@@ -6780,6 +6793,7 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
 		.may_writepage = 1,
 		.may_unmap = 1,
 		.may_swap = 1,
+		.nofragment = nofragment,
 	};
 
 	/*
@@ -7011,7 +7025,7 @@ static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx)
 			free_pages = zone_page_state_snapshot(zone, item);
 
 		if (__zone_watermark_ok(zone, order, mark, highest_zoneidx,
-					0, free_pages))
+					0, free_pages, MIGRATE_MOVABLE))
 			return true;
 	}
 
-- 
  Kiryl Shutsemau / Kirill A. Shutemov