[RFC PATCH 1/1] mm: batch page copies in folio_copy() and folio_mc_copy()

[Shivank Garg <shivankg@amd.com>]

Rewrite folio_copy() and folio_mc_copy() as thin wrappers around new
batched helpers copy_highpages() and copy_mc_highpages().

The current implementations iterate copy_highpage() (or its #MC-aware
variant) per 4 KB page. For a single 2 MB folio that loop runs 512
times and pays, per page:

  - kmap_local_page() / kunmap_local()
  - cond_resched()
  - one invocation of the architecture copy_page()/memcpy() primitive

The new helpers issue a single copy_mc_to_kernel()/memcpy() over
the whole contiguous range when CONFIG_HIGHMEM is off and no
architecture overrides (__HAVE_ARCH_COPY_HIGHPAGE) copy_highpage().
HIGHMEM and arch overrides keep the existing per-page path.

Tested on dual-socket AMD EPYC 9655 (Zen 5) with a CXL.mem node.
In-kernel folio_mc_copy() microbenchmark on 2 MB folios, source
evicted from cache before each iteration and measured throughput:

  direction         baseline GB/s   optimized GB/s   speedup
  DRAM0 -> DRAM1     18.65 ± 1.37    38.03 ± 3.21     2.04x
  DRAM0 -> CXL       25.46 ± 2.89    39.29 ± 1.17     1.54x
  CXL   -> DRAM0     20.61 ± 3.95    35.07 ± 0.62     1.70x

End-to-end move_pages(2) throughput on anonymous 2 MB mTHP folios,
1 GB migrated per run:

  direction         baseline GB/s   optimized GB/s   speedup
  DRAM0 -> DRAM1      7.20 ± 0.03     8.01 ± 0.02     1.11x
  DRAM0 -> CXL       11.12 ± 0.15    13.07 ± 0.03     1.18x
  DRAM1 -> DRAM0      7.21 ± 0.02     7.95 ± 0.02     1.10x
  CXL   -> DRAM0      9.10 ± 0.05     9.49 ± 0.01     1.04x

On AMD EPYC 7713 (Zen 3 / Milan, REP_GOOD without FSRM/ERMS) the
folio_copy() bulk path regresses because memcpy() falls through to
memcpy_orig (an unrolled movq loop), which is slower than the
per-page copy_page() (microcoded rep movsq) it replaces. 
Same 2 MB folio_copy() microbench, source evicted before
each iteration:

  direction         baseline GB/s   optimized GB/s   speedup
  DRAM0 -> DRAM1     13.03 ± 0.76    11.59 ± 0.18     0.89x
  DRAM1 -> DRAM0     12.85 ± 0.25    11.02 ± 0.10     0.86x

Cover letter discusses introducing a copy_pages() helper to avoid
this regression. 

Signed-off-by: Shivank Garg <shivankg@amd.com>
---
 include/linux/highmem.h | 58 +++++++++++++++++++++++++++++++++++++++++
 mm/util.c               | 25 +++---------------
 2 files changed, 62 insertions(+), 21 deletions(-)

diff --git a/include/linux/highmem.h b/include/linux/highmem.h
index 871d817426bc..daee3f1863d1 100644
--- a/include/linux/highmem.h
+++ b/include/linux/highmem.h
@@ -439,6 +439,23 @@ static inline void copy_highpage(struct page *to, struct page *from)
 
 #endif
 
+static inline void copy_highpages(struct page *to, struct page *from,
+		unsigned long nr_pages)
+{
+	unsigned long i;
+
+#ifndef __HAVE_ARCH_COPY_HIGHPAGE
+	if (!IS_ENABLED(CONFIG_HIGHMEM)) {
+		memcpy(page_address(to), page_address(from), nr_pages << PAGE_SHIFT);
+		for (i = 0; i < nr_pages; i++)
+			kmsan_copy_page_meta(to + i, from + i);
+		return;
+	}
+#endif
+	for (i = 0; i < nr_pages; i++)
+		copy_highpage(to + i, from + i);
+}
+
 #ifdef copy_mc_to_kernel
 /*
  * If architecture supports machine check exception handling, define the
@@ -484,6 +501,40 @@ static inline int copy_mc_highpage(struct page *to, struct page *from)
 
 	return ret;
 }
+
+static inline int copy_mc_highpages(struct page *to, struct page *from,
+		unsigned long nr_pages)
+{
+	unsigned long i;
+
+#ifndef __HAVE_ARCH_COPY_HIGHPAGE
+	if (!IS_ENABLED(CONFIG_HIGHMEM)) {
+		unsigned long len = nr_pages << PAGE_SHIFT;
+		unsigned long ret;
+
+		ret = copy_mc_to_kernel(page_address(to),
+					page_address(from), len);
+		if (!ret) {
+			for (i = 0; i < nr_pages; i++)
+				kmsan_copy_page_meta(to + i, from + i);
+			return 0;
+		}
+		/*
+		 * copy_mc_to_kernel() returns the number bytes that were not copied,
+		 * counted from the end. The first failing page is therefore at
+		 * offset (len - ret) >> PAGE_SHIFT within the range.
+		 */
+		memory_failure_queue(page_to_pfn(from) +
+				     ((len - ret) >> PAGE_SHIFT), 0);
+		return -EHWPOISON;
+	}
+#endif
+
+	for (i = 0; i < nr_pages; i++)
+		if (copy_mc_highpage(to + i, from + i))
+			return -EHWPOISON;
+	return 0;
+}
 #else
 static inline int copy_mc_user_highpage(struct page *to, struct page *from,
 					unsigned long vaddr, struct vm_area_struct *vma)
@@ -497,6 +548,13 @@ static inline int copy_mc_highpage(struct page *to, struct page *from)
 	copy_highpage(to, from);
 	return 0;
 }
+
+static inline int copy_mc_highpages(struct page *to, struct page *from,
+		unsigned long nr_pages)
+{
+	copy_highpages(to, from, nr_pages);
+	return 0;
+}
 #endif
 
 static inline void memcpy_page(struct page *dst_page, size_t dst_off,
diff --git a/mm/util.c b/mm/util.c
index 3cc949a0b7ed..93f0d9daffce 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -749,32 +749,15 @@ EXPORT_SYMBOL(folio_mapping);
  */
 void folio_copy(struct folio *dst, struct folio *src)
 {
-	long i = 0;
-	long nr = folio_nr_pages(src);
-
-	for (;;) {
-		copy_highpage(folio_page(dst, i), folio_page(src, i));
-		if (++i == nr)
-			break;
-		cond_resched();
-	}
+	copy_highpages(folio_page(dst, 0), folio_page(src, 0),
+		       folio_nr_pages(src));
 }
 EXPORT_SYMBOL(folio_copy);
 
 int folio_mc_copy(struct folio *dst, struct folio *src)
 {
-	long nr = folio_nr_pages(src);
-	long i = 0;
-
-	for (;;) {
-		if (copy_mc_highpage(folio_page(dst, i), folio_page(src, i)))
-			return -EHWPOISON;
-		if (++i == nr)
-			break;
-		cond_resched();
-	}
-
-	return 0;
+	return copy_mc_highpages(folio_page(dst, 0), folio_page(src, 0),
+				 folio_nr_pages(src));
 }
 EXPORT_SYMBOL(folio_mc_copy);
 
-- 
2.43.0