[PATCH 1/9] huge_pages: get rid of process_huge_page()

Ankur Arora
2023-04-03T05:23:47Z

process_huge_pages() (and the subsequent call to process_subpage())
translate between base-addr[index], struct page *, first when calling
clear/copy_subpage() which then translates to the parameters
needed in clear/copy_user_highpage().

There's no runtime cost in doing this, but it's unnecessary complexity
for something that only has two users.

Accordingly, fold process_huge_page() back in its callers.

Link: https://lore.kernel.org/lkml/20220606202109.1306034-1-ankur.a.arora@oracle.com/
Suggested-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Ankur Arora <ankur.a.arora@oracle.com>
---
mm/memory.c | 126 ++++++++++++++++++++++++++++++----------------------
1 file changed, 73 insertions(+), 53 deletions(-)

diff --git a/mm/memory.c b/mm/memory.c
index f456f3b5049c..d54bc27a35ca 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -5629,63 +5629,13 @@ EXPORT_SYMBOL(__might_fault);
#endif

#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
-/*
- * Process all subpages of the specified huge page with the specified
- * operation. The target subpage will be processed last to keep its
- * cache lines hot.
- */
-static inline void process_huge_page(
- unsigned long addr_hint, unsigned int pages_per_huge_page,
- void (*process_subpage)(unsigned long addr, int idx, void *arg),
- void *arg)
-{
- int i, n, base, l;
- unsigned long addr = addr_hint &
- ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
-
- /* Process target subpage last to keep its cache lines hot */
- might_sleep();
- n = (addr_hint - addr) / PAGE_SIZE;
- if (2 * n <= pages_per_huge_page) {
- /* If target subpage in first half of huge page */
- base = 0;
- l = n;
- /* Process subpages at the end of huge page */
- for (i = pages_per_huge_page - 1; i >= 2 * n; i--) {
- cond_resched();
- process_subpage(addr + i * PAGE_SIZE, i, arg);
- }
- } else {
- /* If target subpage in second half of huge page */
- base = pages_per_huge_page - 2 * (pages_per_huge_page - n);
- l = pages_per_huge_page - n;
- /* Process subpages at the begin of huge page */
- for (i = 0; i < base; i++) {
- cond_resched();
- process_subpage(addr + i * PAGE_SIZE, i, arg);
- }
- }
- /*
- * Process remaining subpages in left-right-left-right pattern
- * towards the target subpage
- */
- for (i = 0; i < l; i++) {
- int left_idx = base + i;
- int right_idx = base + 2 * l - 1 - i;
-
- cond_resched();
- process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg);
- cond_resched();
- process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg);
- }
-}

static void clear_gigantic_page(struct page *page,
unsigned long addr,
unsigned int pages_per_huge_page)
{
int i;
- struct page *p;
+ struct page *p = page;

might_sleep();
for (i = 0; i < pages_per_huge_page; i++) {
@@ -5707,13 +5657,48 @@ void clear_huge_page(struct page *page,
{
unsigned long addr = addr_hint &
~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
+ int i, n, base, l;

if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
clear_gigantic_page(page, addr, pages_per_huge_page);
return;
}

- process_huge_page(addr_hint, pages_per_huge_page, clear_subpage, page);
+ /* Process target subpage last to keep its cache lines hot */
+ might_sleep();
+ n = (addr_hint - addr) / PAGE_SIZE;
+ if (2 * n <= pages_per_huge_page) {
+ /* If target subpage in first half of huge page */
+ base = 0;
+ l = n;
+ /* Process subpages at the end of huge page */
+ for (i = pages_per_huge_page - 1; i >= 2 * n; i--) {
+ cond_resched();
+ clear_subpage(addr + i * PAGE_SIZE, i, (void *)page);
+ }
+ } else {
+ /* If target subpage in second half of huge page */
+ base = pages_per_huge_page - 2 * (pages_per_huge_page - n);
+ l = pages_per_huge_page - n;
+ /* Process subpages at the begin of huge page */
+ for (i = 0; i < base; i++) {
+ cond_resched();
+ clear_subpage(addr + i * PAGE_SIZE, i, (void *)page);
+ }
+ }
+ /*
+ * Process remaining subpages in left-right-left-right pattern
+ * towards the target subpage
+ */
+ for (i = 0; i < l; i++) {
+ int left_idx = base + i;
+ int right_idx = base + 2 * l - 1 - i;
+
+ cond_resched();
+ clear_subpage(addr + left_idx * PAGE_SIZE, left_idx, (void *)page);
+ cond_resched();
+ clear_subpage(addr + right_idx * PAGE_SIZE, right_idx, (void *)page);
+ }
}

static void copy_user_gigantic_page(struct page *dst, struct page *src,
@@ -5759,6 +5744,7 @@ void copy_user_huge_page(struct page *dst, struct page *src,
.src = src,
.vma = vma,
};
+ int i, n, base, l;

if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
copy_user_gigantic_page(dst, src, addr, vma,
@@ -5766,7 +5752,41 @@ void copy_user_huge_page(struct page *dst, struct page *src,
return;
}

- process_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg);
+ /* Process target subpage last to keep its cache lines hot */
+ might_sleep();
+ n = (addr_hint - addr) / PAGE_SIZE;
+ if (2 * n <= pages_per_huge_page) {
+ /* If target subpage in first half of huge page */
+ base = 0;
+ l = n;
+ /* Process subpages at the end of huge page */
+ for (i = pages_per_huge_page - 1; i >= 2 * n; i--) {
+ cond_resched();
+ copy_subpage(addr + i * PAGE_SIZE, i, &arg);
+ }
+ } else {
+ /* If target subpage in second half of huge page */
+ base = pages_per_huge_page - 2 * (pages_per_huge_page - n);
+ l = pages_per_huge_page - n;
+ /* Process subpages at the begin of huge page */
+ for (i = 0; i < base; i++) {
+ cond_resched();
+ copy_subpage(addr + i * PAGE_SIZE, i, &arg);
+ }
+ }
+ /*
+ * Process remaining subpages in left-right-left-right pattern
+ * towards the target subpage
+ */
+ for (i = 0; i < l; i++) {
+ int left_idx = base + i;
+ int right_idx = base + 2 * l - 1 - i;
+
+ cond_resched();
+ copy_subpage(addr + left_idx * PAGE_SIZE, left_idx, &arg);
+ cond_resched();
+ copy_subpage(addr + right_idx * PAGE_SIZE, right_idx, &arg);
+ }
}

long copy_huge_page_from_user(struct page *dst_page,
--
2.31.1