Re: [PATCH] mm: Add Kcompressd for accelerated memory compression

[Harry Yoo <harry.yoo@oracle.com>]

On Wed, Apr 30, 2025 at 04:26:41PM +0800, Qun-Wei Lin wrote:
> This patch series introduces a new mechanism called kcompressd to
> improve the efficiency of memory reclaiming in the operating system.
> 
> Problem:
>   In the current system, the kswapd thread is responsible for both scanning
>   the LRU pages and handling memory compression tasks (such as those
>   involving ZSWAP/ZRAM, if enabled). This combined responsibility can lead
>   to significant performance bottlenecks, especially under high memory
>   pressure. The kswapd thread becomes a single point of contention, causing
>   delays in memory reclaiming and overall system performance degradation.
> 
> Solution:
>   Introduced kcompressd to handle asynchronous compression during memory
>   reclaim, improving efficiency by offloading compression tasks from
>   kswapd. This allows kswapd to focus on its primary task of page reclaim
>   without being burdened by the additional overhead of compression.
> 
> In our handheld devices, we found that applying this mechanism under high
> memory pressure scenarios can increase the rate of pgsteal_anon per second
> by over 260% compared to the situation with only kswapd. Additionally, we
> observed a reduction of over 50% in page allocation stall occurrences,
> further demonstrating the effectiveness of kcompressd in alleviating memory
> pressure and improving system responsiveness.
> 
> Co-developed-by: Barry Song <21cnbao@gmail.com>
> Signed-off-by: Barry Song <21cnbao@gmail.com>
> Signed-off-by: Qun-Wei Lin <qun-wei.lin@mediatek.com>
> Reference: Re: [PATCH 0/2] Improve Zram by separating compression context from kswapd - Barry Song
>            https://lore.kernel.org/lkml/20250313093005.13998-1-21cnbao@gmail.com/
> ---

+Cc Zi Yan, who might be interested in writing a framework (or improving
the existing one, padata) for parallelizing jobs (e.g. migration/compression)

>  include/linux/mmzone.h |  6 ++++
>  mm/mm_init.c           |  1 +
>  mm/page_io.c           | 71 ++++++++++++++++++++++++++++++++++++++++++
>  mm/swap.h              |  6 ++++
>  mm/vmscan.c            | 25 +++++++++++++++
>  5 files changed, 109 insertions(+)
> 
> diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
> index 6ccec1bf2896..93c9195a54ae 100644
> --- a/include/linux/mmzone.h
> +++ b/include/linux/mmzone.h
> @@ -23,6 +23,7 @@
>  #include <linux/page-flags.h>
>  #include <linux/local_lock.h>
>  #include <linux/zswap.h>
> +#include <linux/kfifo.h>
>  #include <asm/page.h>
>  
>  /* Free memory management - zoned buddy allocator.  */
> @@ -1398,6 +1399,11 @@ typedef struct pglist_data {
>  
>  	int kswapd_failures;		/* Number of 'reclaimed == 0' runs */
>  
> +#define KCOMPRESS_FIFO_SIZE 256
> +	wait_queue_head_t kcompressd_wait;
> +	struct task_struct *kcompressd;
> +	struct kfifo kcompress_fifo;
> +
>  #ifdef CONFIG_COMPACTION
>  	int kcompactd_max_order;
>  	enum zone_type kcompactd_highest_zoneidx;
> diff --git a/mm/mm_init.c b/mm/mm_init.c
> index 9659689b8ace..49bae1dd4584 100644
> --- a/mm/mm_init.c
> +++ b/mm/mm_init.c
> @@ -1410,6 +1410,7 @@ static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
>  	pgdat_init_kcompactd(pgdat);
>  
>  	init_waitqueue_head(&pgdat->kswapd_wait);
> +	init_waitqueue_head(&pgdat->kcompressd_wait);
>  	init_waitqueue_head(&pgdat->pfmemalloc_wait);
>  
>  	for (i = 0; i < NR_VMSCAN_THROTTLE; i++)
> diff --git a/mm/page_io.c b/mm/page_io.c
> index 4bce19df557b..d85deb494a6a 100644
> --- a/mm/page_io.c
> +++ b/mm/page_io.c
> @@ -233,6 +233,38 @@ static void swap_zeromap_folio_clear(struct folio *folio)
>  	}
>  }
>  
> +static bool swap_sched_async_compress(struct folio *folio)
> +{
> +	struct swap_info_struct *sis = swp_swap_info(folio->swap);
> +	int nid = numa_node_id();
> +	pg_data_t *pgdat = NODE_DATA(nid);
> +
> +	if (unlikely(!pgdat->kcompressd))
> +		return false;
> +
> +	if (!current_is_kswapd())
> +		return false;
> +
> +	if (!folio_test_anon(folio))
> +		return false;
> +	/*
> +	 * This case needs to synchronously return AOP_WRITEPAGE_ACTIVATE
> +	 */
> +	if (!mem_cgroup_zswap_writeback_enabled(folio_memcg(folio)))
> +		return false;
> +
> +	sis = swp_swap_info(folio->swap);
> +	if (zswap_is_enabled() || data_race(sis->flags & SWP_SYNCHRONOUS_IO)) {
> +		if (kfifo_avail(&pgdat->kcompress_fifo) >= sizeof(folio) &&
> +			kfifo_in(&pgdat->kcompress_fifo, &folio, sizeof(folio))) {
> +			wake_up_interruptible(&pgdat->kcompressd_wait);
> +			return true;
> +		}
> +	}
> +
> +	return false;
> +}
> +
>  /*
>   * We may have stale swap cache pages in memory: notice
>   * them here and get rid of the unnecessary final write.
> @@ -275,6 +307,15 @@ int swap_writepage(struct page *page, struct writeback_control *wbc)
>  		 */
>  		swap_zeromap_folio_clear(folio);
>  	}
> +
> +	/*
> +	 * Compression within zswap and zram might block rmap, unmap
> +	 * of both file and anon pages, try to do compression async
> +	 * if possible
> +	 */
> +	if (swap_sched_async_compress(folio))
> +		return 0;
> +
>  	if (zswap_store(folio)) {
>  		count_mthp_stat(folio_order(folio), MTHP_STAT_ZSWPOUT);
>  		folio_unlock(folio);
> @@ -289,6 +330,36 @@ int swap_writepage(struct page *page, struct writeback_control *wbc)
>  	return 0;
>  }
>  
> +int kcompressd(void *p)
> +{
> +	pg_data_t *pgdat = (pg_data_t *)p;
> +	struct folio *folio;
> +	struct writeback_control wbc = {
> +		.sync_mode = WB_SYNC_NONE,
> +		.nr_to_write = SWAP_CLUSTER_MAX,
> +		.range_start = 0,
> +		.range_end = LLONG_MAX,
> +		.for_reclaim = 1,
> +	};
> +
> +	while (!kthread_should_stop()) {
> +		wait_event_interruptible(pgdat->kcompressd_wait,
> +				!kfifo_is_empty(&pgdat->kcompress_fifo));
> +
> +		while (!kfifo_is_empty(&pgdat->kcompress_fifo)) {
> +			if (kfifo_out(&pgdat->kcompress_fifo, &folio, sizeof(folio))) {
> +				if (zswap_store(folio)) {
> +					count_mthp_stat(folio_order(folio), MTHP_STAT_ZSWPOUT);
> +					folio_unlock(folio);
> +					continue;
> +				}
> +				__swap_writepage(folio, &wbc);
> +			}
> +		}
> +	}
> +	return 0;
> +}
> +
>  static inline void count_swpout_vm_event(struct folio *folio)
>  {
>  #ifdef CONFIG_TRANSPARENT_HUGEPAGE
> diff --git a/mm/swap.h b/mm/swap.h
> index 6f4a3f927edb..3579da413dc2 100644
> --- a/mm/swap.h
> +++ b/mm/swap.h
> @@ -22,6 +22,7 @@ static inline void swap_read_unplug(struct swap_iocb *plug)
>  void swap_write_unplug(struct swap_iocb *sio);
>  int swap_writepage(struct page *page, struct writeback_control *wbc);
>  void __swap_writepage(struct folio *folio, struct writeback_control *wbc);
> +int kcompressd(void *p);
>  
>  /* linux/mm/swap_state.c */
>  /* One swap address space for each 64M swap space */
> @@ -199,6 +200,11 @@ static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr,
>  	return 0;
>  }
>  
> +static inline int kcompressd(void *p)
> +{
> +	return 0;
> +}
> +
>  #endif /* CONFIG_SWAP */
>  
>  #endif /* _MM_SWAP_H */
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> index 3783e45bfc92..2d7b9167bfd6 100644
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
> @@ -7420,6 +7420,7 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
>  void __meminit kswapd_run(int nid)
>  {
>  	pg_data_t *pgdat = NODE_DATA(nid);
> +	int ret;
>  
>  	pgdat_kswapd_lock(pgdat);
>  	if (!pgdat->kswapd) {
> @@ -7433,7 +7434,26 @@ void __meminit kswapd_run(int nid)
>  		} else {
>  			wake_up_process(pgdat->kswapd);
>  		}
> +		ret = kfifo_alloc(&pgdat->kcompress_fifo,
> +				KCOMPRESS_FIFO_SIZE * sizeof(struct folio *),
> +				GFP_KERNEL);
> +		if (ret) {
> +			pr_err("%s: fail to kfifo_alloc\n", __func__);
> +			goto out;
> +		}
> +
> +		pgdat->kcompressd = kthread_create_on_node(kcompressd, pgdat, nid,
> +				"kcompressd%d", nid);
> +		if (IS_ERR(pgdat->kcompressd)) {
> +			pr_err("Failed to start kcompressd on node %d，ret=%ld\n",
> +					nid, PTR_ERR(pgdat->kcompressd));
> +			pgdat->kcompressd = NULL;
> +			kfifo_free(&pgdat->kcompress_fifo);
> +		} else {
> +			wake_up_process(pgdat->kcompressd);
> +		}
>  	}
> +out:
>  	pgdat_kswapd_unlock(pgdat);
>  }
>  
> @@ -7452,6 +7472,11 @@ void __meminit kswapd_stop(int nid)
>  		kthread_stop(kswapd);
>  		pgdat->kswapd = NULL;
>  	}
> +	if (pgdat->kcompressd) {
> +		kthread_stop(pgdat->kcompressd);
> +		pgdat->kcompressd = NULL;
> +		kfifo_free(&pgdat->kcompress_fifo);
> +	}
>  	pgdat_kswapd_unlock(pgdat);
>  }
>  
> -- 
> 2.45.2
> 
> 

-- 
Cheers,
Harry / Hyeonggon