[PATCH v3 00/17] Prepare slab for memdescs

[PATCH v3 00/17] Prepare slab for memdescs

[Matthew Wilcox (Oracle)]

When we separate struct folio, struct page and struct slab from each
other, converting to folios then to slabs will be nonsense.  It made
sense under the 'folio is just a head page' interpretation, but with
full separation, page_folio() will return NULL for a page which belongs
to a slab.

This patch series removes almost all mentions of folio from slab.
There are a few folio_test_slab() invocations left around the tree that
I haven't decided how to handle yet.  We're not yet quite at the point
of separately allocating struct slab, but that's what I'll be working
on next.

v3:
 - Rebased on next-20251022
 - Picked up review tags (thanks!)

Matthew Wilcox (Oracle) (17):
  slab: Reimplement page_slab()
  slab: Remove folio references from __ksize()
  slab: Remove folio references in memcg_slab_post_charge()
  slab: Remove folio references in slab alloc/free
  slab: Remove folio references from ___kmalloc_large_node()
  slab: Remove folio references from free_large_kmalloc()
  slab: Remove folio references from kvfree_rcu_cb()
  slab: Remove folio references from kvfree()
  slab: Remove folio references from __do_krealloc()
  slab: Remove folio references from build_detached_freelist()
  slab: Remove dead code from free_large_kmalloc()
  slab: Remove folio references from kfree_rcu_sheaf()
  slab: Remove folio references from kfree_nolock()
  usercopy: Remove folio references from check_heap_object()
  memcg: Convert mem_cgroup_from_obj_folio() to
    mem_cgroup_from_obj_slab()
  kasan: Remove references to folio in __kasan_mempool_poison_object()
  slab: Remove references to folios from virt_to_slab()

 include/linux/page-flags.h |  16 +----
 mm/kasan/common.c          |  12 ++--
 mm/kfence/core.c           |  12 ++--
 mm/memcontrol.c            |  36 ++++------
 mm/slab.h                  |  58 ++++++++--------
 mm/slab_common.c           |  29 ++++----
 mm/slub.c                  | 138 ++++++++++++++++++-------------------
 mm/usercopy.c              |  21 +++---
 8 files changed, 148 insertions(+), 174 deletions(-)

-- 
2.47.2

[PATCH v3 01/17] slab: Reimplement page_slab()

[Matthew Wilcox (Oracle)]

In order to separate slabs from folios, we need to convert from any page
in a slab to the slab directly without going through a page to folio
conversion first.  page_slab() is a little different from other memdesc
converters we have in that it will return NULL if the page is not part
of a slab.  This will be the normal style for memdesc converters in
the future.

kfence was the only user of page_slab(), so adjust it to the new way
of working.  It will need to be touched again when we separate slab
from page.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
 include/linux/page-flags.h | 14 +-------------
 mm/kfence/core.c           | 12 ++++++++----
 mm/slab.h                  | 28 ++++++++++++++++------------
 3 files changed, 25 insertions(+), 29 deletions(-)

diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 0091ad1986bf..6d5e44968eab 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -1048,19 +1048,7 @@ PAGE_TYPE_OPS(Table, table, pgtable)
  */
 PAGE_TYPE_OPS(Guard, guard, guard)
 
-FOLIO_TYPE_OPS(slab, slab)
-
-/**
- * PageSlab - Determine if the page belongs to the slab allocator
- * @page: The page to test.
- *
- * Context: Any context.
- * Return: True for slab pages, false for any other kind of page.
- */
-static inline bool PageSlab(const struct page *page)
-{
-	return folio_test_slab(page_folio(page));
-}
+PAGE_TYPE_OPS(Slab, slab, slab)
 
 #ifdef CONFIG_HUGETLB_PAGE
 FOLIO_TYPE_OPS(hugetlb, hugetlb)
diff --git a/mm/kfence/core.c b/mm/kfence/core.c
index 727c20c94ac5..b16e73fd5b68 100644
--- a/mm/kfence/core.c
+++ b/mm/kfence/core.c
@@ -612,13 +612,15 @@ static unsigned long kfence_init_pool(void)
 	 * enters __slab_free() slow-path.
 	 */
 	for (i = 0; i < KFENCE_POOL_SIZE / PAGE_SIZE; i++) {
+		struct page *page;
 		struct slab *slab;
 
 		if (!i || (i % 2))
 			continue;
 
-		slab = page_slab(pfn_to_page(start_pfn + i));
-		__folio_set_slab(slab_folio(slab));
+		page = pfn_to_page(start_pfn + i);
+		__SetPageSlab(page);
+		slab = page_slab(page);
 #ifdef CONFIG_MEMCG
 		slab->obj_exts = (unsigned long)&kfence_metadata_init[i / 2 - 1].obj_exts |
 				 MEMCG_DATA_OBJEXTS;
@@ -665,16 +667,18 @@ static unsigned long kfence_init_pool(void)
 
 reset_slab:
 	for (i = 0; i < KFENCE_POOL_SIZE / PAGE_SIZE; i++) {
+		struct page *page;
 		struct slab *slab;
 
 		if (!i || (i % 2))
 			continue;
 
-		slab = page_slab(pfn_to_page(start_pfn + i));
+		page = pfn_to_page(start_pfn + i);
+		slab = page_slab(page);
 #ifdef CONFIG_MEMCG
 		slab->obj_exts = 0;
 #endif
-		__folio_clear_slab(slab_folio(slab));
+		__ClearPageSlab(page);
 	}
 
 	return addr;
diff --git a/mm/slab.h b/mm/slab.h
index 078daecc7cf5..a64b9b2c8731 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -146,20 +146,24 @@ static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(freelist_aba_t)
 	struct slab *:		(struct folio *)s))
 
 /**
- * page_slab - Converts from first struct page to slab.
- * @p: The first (either head of compound or single) page of slab.
+ * page_slab - Converts from struct page to its slab.
+ * @page: A page which may or may not belong to a slab.
  *
- * A temporary wrapper to convert struct page to struct slab in situations where
- * we know the page is the compound head, or single order-0 page.
- *
- * Long-term ideally everything would work with struct slab directly or go
- * through folio to struct slab.
- *
- * Return: The slab which contains this page
+ * Return: The slab which contains this page or NULL if the page does
+ * not belong to a slab.  This includes pages returned from large kmalloc.
  */
-#define page_slab(p)		(_Generic((p),				\
-	const struct page *:	(const struct slab *)(p),		\
-	struct page *:		(struct slab *)(p)))
+static inline struct slab *page_slab(const struct page *page)
+{
+	unsigned long head;
+
+	head = READ_ONCE(page->compound_head);
+	if (head & 1)
+		page = (struct page *)(head - 1);
+	if (data_race(page->page_type >> 24) != PGTY_slab)
+		page = NULL;
+
+	return (struct slab *)page;
+}
 
 /**
  * slab_page - The first struct page allocated for a slab
-- 
2.47.2

Re: [PATCH v3 01/17] slab: Reimplement page_slab()

[Vlastimil Babka]

On 10/24/25 22:44, Matthew Wilcox (Oracle) wrote:
> In order to separate slabs from folios, we need to convert from any page
> in a slab to the slab directly without going through a page to folio
> conversion first.  page_slab() is a little different from other memdesc
> converters we have in that it will return NULL if the page is not part
> of a slab.  This will be the normal style for memdesc converters in
> the future.
> 
> kfence was the only user of page_slab(), so adjust it to the new way
> of working.  It will need to be touched again when we separate slab
> from page.

+Cc KFENCE folks.

> Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>

Otherwise LGTM.

> ---
>  include/linux/page-flags.h | 14 +-------------
>  mm/kfence/core.c           | 12 ++++++++----
>  mm/slab.h                  | 28 ++++++++++++++++------------
>  3 files changed, 25 insertions(+), 29 deletions(-)
> 
> diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
> index 0091ad1986bf..6d5e44968eab 100644
> --- a/include/linux/page-flags.h
> +++ b/include/linux/page-flags.h
> @@ -1048,19 +1048,7 @@ PAGE_TYPE_OPS(Table, table, pgtable)
>   */
>  PAGE_TYPE_OPS(Guard, guard, guard)
>  
> -FOLIO_TYPE_OPS(slab, slab)
> -
> -/**
> - * PageSlab - Determine if the page belongs to the slab allocator
> - * @page: The page to test.
> - *
> - * Context: Any context.
> - * Return: True for slab pages, false for any other kind of page.
> - */
> -static inline bool PageSlab(const struct page *page)
> -{
> -	return folio_test_slab(page_folio(page));
> -}
> +PAGE_TYPE_OPS(Slab, slab, slab)
>  
>  #ifdef CONFIG_HUGETLB_PAGE
>  FOLIO_TYPE_OPS(hugetlb, hugetlb)
> diff --git a/mm/kfence/core.c b/mm/kfence/core.c
> index 727c20c94ac5..b16e73fd5b68 100644
> --- a/mm/kfence/core.c
> +++ b/mm/kfence/core.c
> @@ -612,13 +612,15 @@ static unsigned long kfence_init_pool(void)
>  	 * enters __slab_free() slow-path.
>  	 */
>  	for (i = 0; i < KFENCE_POOL_SIZE / PAGE_SIZE; i++) {
> +		struct page *page;
>  		struct slab *slab;
>  
>  		if (!i || (i % 2))
>  			continue;
>  
> -		slab = page_slab(pfn_to_page(start_pfn + i));
> -		__folio_set_slab(slab_folio(slab));
> +		page = pfn_to_page(start_pfn + i);
> +		__SetPageSlab(page);
> +		slab = page_slab(page);
>  #ifdef CONFIG_MEMCG
>  		slab->obj_exts = (unsigned long)&kfence_metadata_init[i / 2 - 1].obj_exts |
>  				 MEMCG_DATA_OBJEXTS;
> @@ -665,16 +667,18 @@ static unsigned long kfence_init_pool(void)
>  
>  reset_slab:
>  	for (i = 0; i < KFENCE_POOL_SIZE / PAGE_SIZE; i++) {
> +		struct page *page;
>  		struct slab *slab;
>  
>  		if (!i || (i % 2))
>  			continue;
>  
> -		slab = page_slab(pfn_to_page(start_pfn + i));
> +		page = pfn_to_page(start_pfn + i);
> +		slab = page_slab(page);
>  #ifdef CONFIG_MEMCG
>  		slab->obj_exts = 0;
>  #endif
> -		__folio_clear_slab(slab_folio(slab));
> +		__ClearPageSlab(page);
>  	}
>  
>  	return addr;
> diff --git a/mm/slab.h b/mm/slab.h
> index 078daecc7cf5..a64b9b2c8731 100644
> --- a/mm/slab.h
> +++ b/mm/slab.h
> @@ -146,20 +146,24 @@ static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(freelist_aba_t)
>  	struct slab *:		(struct folio *)s))
>  
>  /**
> - * page_slab - Converts from first struct page to slab.
> - * @p: The first (either head of compound or single) page of slab.
> + * page_slab - Converts from struct page to its slab.
> + * @page: A page which may or may not belong to a slab.
>   *
> - * A temporary wrapper to convert struct page to struct slab in situations where
> - * we know the page is the compound head, or single order-0 page.
> - *
> - * Long-term ideally everything would work with struct slab directly or go
> - * through folio to struct slab.
> - *
> - * Return: The slab which contains this page
> + * Return: The slab which contains this page or NULL if the page does
> + * not belong to a slab.  This includes pages returned from large kmalloc.
>   */
> -#define page_slab(p)		(_Generic((p),				\
> -	const struct page *:	(const struct slab *)(p),		\
> -	struct page *:		(struct slab *)(p)))
> +static inline struct slab *page_slab(const struct page *page)
> +{
> +	unsigned long head;
> +
> +	head = READ_ONCE(page->compound_head);
> +	if (head & 1)
> +		page = (struct page *)(head - 1);
> +	if (data_race(page->page_type >> 24) != PGTY_slab)
> +		page = NULL;
> +
> +	return (struct slab *)page;
> +}
>  
>  /**
>   * slab_page - The first struct page allocated for a slab

[PATCH v3 02/17] slab: Remove folio references from __ksize()

[Matthew Wilcox (Oracle)]

In the future, we will separate slab, folio and page from each other
and calling virt_to_folio() on an address allocated from slab will
return NULL.  Delay the conversion from struct page to struct slab
until we know we're not dealing with a large kmalloc allocation.

This deprecates calling ksize() on memory allocated by alloc_pages().
Today it becomes a warning and support will be removed entirely in
the future.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
 include/linux/page-flags.h |  2 +-
 mm/slab.h                  | 10 ++++++++++
 mm/slab_common.c           | 23 ++++++++++++-----------
 3 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 6d5e44968eab..f7a0e4af0c73 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -1064,7 +1064,7 @@ PAGE_TYPE_OPS(Zsmalloc, zsmalloc, zsmalloc)
  * Serialized with zone lock.
  */
 PAGE_TYPE_OPS(Unaccepted, unaccepted, unaccepted)
-FOLIO_TYPE_OPS(large_kmalloc, large_kmalloc)
+PAGE_TYPE_OPS(LargeKmalloc, large_kmalloc, large_kmalloc)
 
 /**
  * PageHuge - Determine if the page belongs to hugetlbfs
diff --git a/mm/slab.h b/mm/slab.h
index a64b9b2c8731..31ccf0f6d3a1 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -605,6 +605,16 @@ static inline size_t slab_ksize(const struct kmem_cache *s)
 	return s->size;
 }
 
+static inline unsigned int large_kmalloc_order(const struct page *page)
+{
+	return page[1].flags.f & 0xff;
+}
+
+static inline size_t large_kmalloc_size(const struct page *page)
+{
+	return PAGE_SIZE << large_kmalloc_order(page);
+}
+
 #ifdef CONFIG_SLUB_DEBUG
 void dump_unreclaimable_slab(void);
 #else
diff --git a/mm/slab_common.c b/mm/slab_common.c
index d2824daa98cf..236b4e25fce0 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -997,26 +997,27 @@ void __init create_kmalloc_caches(void)
  */
 size_t __ksize(const void *object)
 {
-	struct folio *folio;
+	const struct page *page;
+	const struct slab *slab;
 
 	if (unlikely(object == ZERO_SIZE_PTR))
 		return 0;
 
-	folio = virt_to_folio(object);
+	page = virt_to_page(object);
 
-	if (unlikely(!folio_test_slab(folio))) {
-		if (WARN_ON(folio_size(folio) <= KMALLOC_MAX_CACHE_SIZE))
-			return 0;
-		if (WARN_ON(object != folio_address(folio)))
-			return 0;
-		return folio_size(folio);
-	}
+	if (unlikely(PageLargeKmalloc(page)))
+		return large_kmalloc_size(page);
+
+	slab = page_slab(page);
+	/* Delete this after we're sure there are no users */
+	if (WARN_ON(!slab))
+		return page_size(page);
 
 #ifdef CONFIG_SLUB_DEBUG
-	skip_orig_size_check(folio_slab(folio)->slab_cache, object);
+	skip_orig_size_check(slab->slab_cache, object);
 #endif
 
-	return slab_ksize(folio_slab(folio)->slab_cache);
+	return slab_ksize(slab->slab_cache);
 }
 
 gfp_t kmalloc_fix_flags(gfp_t flags)
-- 
2.47.2

[PATCH v3 03/17] slab: Remove folio references in memcg_slab_post_charge()

[Matthew Wilcox (Oracle)]

This allows us to skip the compound_head() call for large kmalloc
objects as the virt_to_page() call will always give us the head page
for the large kmalloc case.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
 mm/slub.c | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index 90b94b418010..c72795f4f433 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2363,33 +2363,34 @@ bool memcg_slab_post_charge(void *p, gfp_t flags)
 {
 	struct slabobj_ext *slab_exts;
 	struct kmem_cache *s;
-	struct folio *folio;
+	struct page *page;
 	struct slab *slab;
 	unsigned long off;
 
-	folio = virt_to_folio(p);
-	if (!folio_test_slab(folio)) {
+	page = virt_to_page(p);
+	if (PageLargeKmalloc(page)) {
+		unsigned int order;
 		int size;
 
-		if (folio_memcg_kmem(folio))
+		if (PageMemcgKmem(page))
 			return true;
 
-		if (__memcg_kmem_charge_page(folio_page(folio, 0), flags,
-					     folio_order(folio)))
+		order = large_kmalloc_order(page);
+		if (__memcg_kmem_charge_page(page, flags, order))
 			return false;
 
 		/*
-		 * This folio has already been accounted in the global stats but
+		 * This page has already been accounted in the global stats but
 		 * not in the memcg stats. So, subtract from the global and use
 		 * the interface which adds to both global and memcg stats.
 		 */
-		size = folio_size(folio);
-		node_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B, -size);
-		lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B, size);
+		size = PAGE_SIZE << order;
+		mod_node_page_state(page_pgdat(page), NR_SLAB_UNRECLAIMABLE_B, -size);
+		mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, size);
 		return true;
 	}
 
-	slab = folio_slab(folio);
+	slab = page_slab(page);
 	s = slab->slab_cache;
 
 	/*
-- 
2.47.2

Re: [PATCH v3 03/17] slab: Remove folio references in memcg_slab_post_charge()

[Harry Yoo]

On Fri, Oct 24, 2025 at 09:44:18PM +0100, Matthew Wilcox (Oracle) wrote:
> This allows us to skip the compound_head() call for large kmalloc
> objects as the virt_to_page() call will always give us the head page
> for the large kmalloc case.
>
> Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
> ---

Looks good to me,
Reviewed-by: Harry Yoo <harry.yoo@oracle.com>

-- 
Cheers,
Harry / Hyeonggon

[PATCH v3 04/17] slab: Remove folio references in slab alloc/free

[Matthew Wilcox (Oracle)]

Use pages directly to further the split between slab and folio.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
 mm/slub.c | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index c72795f4f433..fb4a43a16269 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3039,24 +3039,24 @@ static inline struct slab *alloc_slab_page(gfp_t flags, int node,
 					   struct kmem_cache_order_objects oo,
 					   bool allow_spin)
 {
-	struct folio *folio;
+	struct page *page;
 	struct slab *slab;
 	unsigned int order = oo_order(oo);
 
 	if (unlikely(!allow_spin))
-		folio = (struct folio *)alloc_frozen_pages_nolock(0/* __GFP_COMP is implied */,
+		page = alloc_frozen_pages_nolock(0/* __GFP_COMP is implied */,
 								  node, order);
 	else if (node == NUMA_NO_NODE)
-		folio = (struct folio *)alloc_frozen_pages(flags, order);
+		page = alloc_frozen_pages(flags, order);
 	else
-		folio = (struct folio *)__alloc_frozen_pages(flags, order, node, NULL);
+		page = __alloc_frozen_pages(flags, order, node, NULL);
 
-	if (!folio)
+	if (!page)
 		return NULL;
 
-	slab = folio_slab(folio);
-	__folio_set_slab(folio);
-	if (folio_is_pfmemalloc(folio))
+	__SetPageSlab(page);
+	slab = page_slab(page);
+	if (page_is_pfmemalloc(page))
 		slab_set_pfmemalloc(slab);
 
 	return slab;
@@ -3280,16 +3280,16 @@ static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node)
 
 static void __free_slab(struct kmem_cache *s, struct slab *slab)
 {
-	struct folio *folio = slab_folio(slab);
-	int order = folio_order(folio);
+	struct page *page = slab_page(slab);
+	int order = compound_order(page);
 	int pages = 1 << order;
 
 	__slab_clear_pfmemalloc(slab);
-	folio->mapping = NULL;
-	__folio_clear_slab(folio);
+	page->mapping = NULL;
+	__ClearPageSlab(page);
 	mm_account_reclaimed_pages(pages);
 	unaccount_slab(slab, order, s);
-	free_frozen_pages(&folio->page, order);
+	free_frozen_pages(page, order);
 }
 
 static void rcu_free_slab(struct rcu_head *h)
-- 
2.47.2

Re: [PATCH v3 04/17] slab: Remove folio references in slab alloc/free

[Harry Yoo]

On Fri, Oct 24, 2025 at 09:44:19PM +0100, Matthew Wilcox (Oracle) wrote:
> Use pages directly to further the split between slab and folio.
> 
> Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
> ---

Looks good to me,
Reviewed-by: Harry Yoo <harry.yoo@oracle.com>

-- 
Cheers,
Harry / Hyeonggon

[PATCH v3 05/17] slab: Remove folio references from ___kmalloc_large_node()

[Matthew Wilcox (Oracle)]

There's no need to use folio APIs here; just use a page directly.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
 mm/slub.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index fb4a43a16269..8bf7d818d13b 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -5566,7 +5566,7 @@ unsigned int kmem_cache_sheaf_size(struct slab_sheaf *sheaf)
  */
 static void *___kmalloc_large_node(size_t size, gfp_t flags, int node)
 {
-	struct folio *folio;
+	struct page *page;
 	void *ptr = NULL;
 	unsigned int order = get_order(size);
 
@@ -5576,15 +5576,15 @@ static void *___kmalloc_large_node(size_t size, gfp_t flags, int node)
 	flags |= __GFP_COMP;
 
 	if (node == NUMA_NO_NODE)
-		folio = (struct folio *)alloc_frozen_pages_noprof(flags, order);
+		page = alloc_frozen_pages_noprof(flags, order);
 	else
-		folio = (struct folio *)__alloc_frozen_pages_noprof(flags, order, node, NULL);
+		page = __alloc_frozen_pages_noprof(flags, order, node, NULL);
 
-	if (folio) {
-		ptr = folio_address(folio);
-		lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B,
+	if (page) {
+		ptr = page_address(page);
+		mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
 				      PAGE_SIZE << order);
-		__folio_set_large_kmalloc(folio);
+		__SetPageLargeKmalloc(page);
 	}
 
 	ptr = kasan_kmalloc_large(ptr, size, flags);
-- 
2.47.2

Re: [PATCH v3 05/17] slab: Remove folio references from ___kmalloc_large_node()

[Harry Yoo]

On Fri, Oct 24, 2025 at 09:44:20PM +0100, Matthew Wilcox (Oracle) wrote:
> There's no need to use folio APIs here; just use a page directly.
> 
> Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
> ---

Looks good to me,
Reviewed-by: Harry Yoo <harry.yoo@oracle.com>

-- 
Cheers,
Harry / Hyeonggon

[PATCH v3 06/17] slab: Remove folio references from free_large_kmalloc()

[Matthew Wilcox (Oracle)]

There's no need to use folio APIs here; just use a page directly.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
 mm/slub.c | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index 8bf7d818d13b..561944d39be9 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -6734,12 +6734,12 @@ void kmem_cache_free(struct kmem_cache *s, void *x)
 }
 EXPORT_SYMBOL(kmem_cache_free);
 
-static void free_large_kmalloc(struct folio *folio, void *object)
+static void free_large_kmalloc(struct page *page, void *object)
 {
-	unsigned int order = folio_order(folio);
+	unsigned int order = compound_order(page);
 
-	if (WARN_ON_ONCE(!folio_test_large_kmalloc(folio))) {
-		dump_page(&folio->page, "Not a kmalloc allocation");
+	if (WARN_ON_ONCE(!PageLargeKmalloc(page))) {
+		dump_page(page, "Not a kmalloc allocation");
 		return;
 	}
 
@@ -6750,10 +6750,10 @@ static void free_large_kmalloc(struct folio *folio, void *object)
 	kasan_kfree_large(object);
 	kmsan_kfree_large(object);
 
-	lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B,
+	mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
 			      -(PAGE_SIZE << order));
-	__folio_clear_large_kmalloc(folio);
-	free_frozen_pages(&folio->page, order);
+	__ClearPageLargeKmalloc(page);
+	free_frozen_pages(page, order);
 }
 
 /*
@@ -6781,7 +6781,7 @@ void kvfree_rcu_cb(struct rcu_head *head)
 		 * consider folio order
 		 */
 		obj = (void *) PAGE_ALIGN_DOWN((unsigned long)obj);
-		free_large_kmalloc(folio, obj);
+		free_large_kmalloc(&folio->page, obj);
 		return;
 	}
 
@@ -6821,7 +6821,7 @@ void kfree(const void *object)
 
 	folio = virt_to_folio(object);
 	if (unlikely(!folio_test_slab(folio))) {
-		free_large_kmalloc(folio, (void *)object);
+		free_large_kmalloc(&folio->page, (void *)object);
 		return;
 	}
 
@@ -7252,7 +7252,7 @@ int build_detached_freelist(struct kmem_cache *s, size_t size,
 	if (!s) {
 		/* Handle kalloc'ed objects */
 		if (unlikely(!folio_test_slab(folio))) {
-			free_large_kmalloc(folio, object);
+			free_large_kmalloc(&folio->page, object);
 			df->slab = NULL;
 			return size;
 		}
-- 
2.47.2

Re: [PATCH v3 06/17] slab: Remove folio references from free_large_kmalloc()

[Harry Yoo]

On Fri, Oct 24, 2025 at 09:44:21PM +0100, Matthew Wilcox (Oracle) wrote:
> There's no need to use folio APIs here; just use a page directly.
> 
> Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
> ---

Looks good to me,
Reviewed-by: Harry Yoo <harry.yoo@oracle.com>

-- 
Cheers,
Harry / Hyeonggon

Re: [PATCH v3 06/17] slab: Remove folio references from free_large_kmalloc()

[Harry Yoo]

On Fri, Oct 24, 2025 at 09:44:21PM +0100, Matthew Wilcox (Oracle) wrote:
> There's no need to use folio APIs here; just use a page directly.
> 
> Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
> ---

Looks good to me,
Reviewed-by: Harry Yoo <harry.yoo@oracle.com>

-- 
Cheers,
Harry / Hyeonggon

[PATCH v3 07/17] slab: Remove folio references from kvfree_rcu_cb()

[Matthew Wilcox (Oracle)]

As with memcg_slab_post_charge(), we save a call to compound_head()
for large kmallocs.  This has a slight change of behaviour in that
non-vmalloc, non-slab, non-kmalloc pointers will now cause a NULL
pointer dereference rather than a warning.  We could add that back if
really needed.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
 mm/slub.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index 561944d39be9..debe7132b343 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -6763,7 +6763,7 @@ static void free_large_kmalloc(struct page *page, void *object)
 void kvfree_rcu_cb(struct rcu_head *head)
 {
 	void *obj = head;
-	struct folio *folio;
+	struct page *page;
 	struct slab *slab;
 	struct kmem_cache *s;
 	void *slab_addr;
@@ -6774,20 +6774,20 @@ void kvfree_rcu_cb(struct rcu_head *head)
 		return;
 	}
 
-	folio = virt_to_folio(obj);
-	if (!folio_test_slab(folio)) {
+	page = virt_to_page(obj);
+	if (PageLargeKmalloc(page)) {
 		/*
 		 * rcu_head offset can be only less than page size so no need to
-		 * consider folio order
+		 * consider allocation order
 		 */
 		obj = (void *) PAGE_ALIGN_DOWN((unsigned long)obj);
-		free_large_kmalloc(&folio->page, obj);
+		free_large_kmalloc(page, obj);
 		return;
 	}
 
-	slab = folio_slab(folio);
+	slab = page_slab(page);
 	s = slab->slab_cache;
-	slab_addr = folio_address(folio);
+	slab_addr = slab_address(slab);
 
 	if (is_kfence_address(obj)) {
 		obj = kfence_object_start(obj);
-- 
2.47.2

[PATCH v3 08/17] slab: Remove folio references from kvfree()

[Matthew Wilcox (Oracle)]

As with memcg_slab_post_charge(), we save a call to compound_head()
for large kmallocs.  This has a slight change of behaviour in that
non-vmalloc, non-slab, non-kmalloc pointers will now cause a NULL
pointer dereference rather than a warning.  We could add that back if
really needed.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
 mm/slub.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index debe7132b343..100ed979312d 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -6809,7 +6809,7 @@ void kvfree_rcu_cb(struct rcu_head *head)
  */
 void kfree(const void *object)
 {
-	struct folio *folio;
+	struct page *page;
 	struct slab *slab;
 	struct kmem_cache *s;
 	void *x = (void *)object;
@@ -6819,13 +6819,13 @@ void kfree(const void *object)
 	if (unlikely(ZERO_OR_NULL_PTR(object)))
 		return;
 
-	folio = virt_to_folio(object);
-	if (unlikely(!folio_test_slab(folio))) {
-		free_large_kmalloc(&folio->page, (void *)object);
+	page = virt_to_page(object);
+	if (unlikely(PageLargeKmalloc(page))) {
+		free_large_kmalloc(page, (void *)object);
 		return;
 	}
 
-	slab = folio_slab(folio);
+	slab = page_slab(page);
 	s = slab->slab_cache;
 	slab_free(s, slab, x, _RET_IP_);
 }
-- 
2.47.2

Re: [PATCH v3 08/17] slab: Remove folio references from kvfree()

[Vlastimil Babka]

Subject says kvfree() but it's kfree()

On 10/24/25 22:44, Matthew Wilcox (Oracle) wrote:
> As with memcg_slab_post_charge(), we save a call to compound_head()
> for large kmallocs.

I'd consider small kmallocs the more common path and so...

> This has a slight change of behaviour in that
> non-vmalloc, non-slab, non-kmalloc pointers will now cause a NULL
> pointer dereference rather than a warning.  We could add that back if
> really needed.

... considering also this...
> Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>

I'd rather restructure it to try page_slab() first, and free_large_kmalloc()
if it's unlikely(NULL), without checking the PageLargeKmalloc flag (let
free_large_kmalloc() do it for the WARN).

kvfree_rcu_cb() in previous patch too then, for consistency

> ---
>  mm/slub.c | 10 +++++-----
>  1 file changed, 5 insertions(+), 5 deletions(-)
> 
> diff --git a/mm/slub.c b/mm/slub.c
> index debe7132b343..100ed979312d 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -6809,7 +6809,7 @@ void kvfree_rcu_cb(struct rcu_head *head)
>   */
>  void kfree(const void *object)
>  {
> -	struct folio *folio;
> +	struct page *page;
>  	struct slab *slab;
>  	struct kmem_cache *s;
>  	void *x = (void *)object;
> @@ -6819,13 +6819,13 @@ void kfree(const void *object)
>  	if (unlikely(ZERO_OR_NULL_PTR(object)))
>  		return;
>  
> -	folio = virt_to_folio(object);
> -	if (unlikely(!folio_test_slab(folio))) {
> -		free_large_kmalloc(&folio->page, (void *)object);
> +	page = virt_to_page(object);
> +	if (unlikely(PageLargeKmalloc(page))) {
> +		free_large_kmalloc(page, (void *)object);
>  		return;
>  	}
>  
> -	slab = folio_slab(folio);
> +	slab = page_slab(page);
>  	s = slab->slab_cache;
>  	slab_free(s, slab, x, _RET_IP_);
>  }

Re: [PATCH v3 08/17] slab: Remove folio references from kvfree()

[Matthew Wilcox]

On Mon, Oct 27, 2025 at 05:58:39PM +0100, Vlastimil Babka wrote:
> Subject says kvfree() but it's kfree()

Thanks, will fix

> I'd rather restructure it to try page_slab() first, and free_large_kmalloc()
> if it's unlikely(NULL), without checking the PageLargeKmalloc flag (let
> free_large_kmalloc() do it for the WARN).

        page = virt_to_page(object);
-       if (unlikely(PageLargeKmalloc(page))) {
+       slab = page_slab(page);
+       if (!slab) {
                free_large_kmalloc(page, (void *)object);
                return;
        }

-       slab = page_slab(page);
        s = slab->slab_cache;

I'm fine with that ... will be part of v4.

[PATCH v3 09/17] slab: Remove folio references from __do_krealloc()

[Matthew Wilcox (Oracle)]

As with memcg_slab_post_charge(), we save a call to compound_head()
for large kmallocs.  This has a slight change of behaviour in that
non-vmalloc, non-slab, non-kmalloc pointers will now cause a NULL
pointer dereference rather than a warning.  We could add that back if
really needed.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
 mm/slub.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index 100ed979312d..4e282e7782bb 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -6924,16 +6924,16 @@ __do_krealloc(const void *p, size_t new_size, unsigned long align, gfp_t flags,
 	if (is_kfence_address(p)) {
 		ks = orig_size = kfence_ksize(p);
 	} else {
-		struct folio *folio;
+		struct page *page;
 
-		folio = virt_to_folio(p);
-		if (unlikely(!folio_test_slab(folio))) {
+		page = virt_to_page(p);
+		if (unlikely(PageLargeKmalloc(page))) {
 			/* Big kmalloc object */
-			WARN_ON(folio_size(folio) <= KMALLOC_MAX_CACHE_SIZE);
-			WARN_ON(p != folio_address(folio));
-			ks = folio_size(folio);
+			ks = page_size(page);
+			WARN_ON(ks <= KMALLOC_MAX_CACHE_SIZE);
+			WARN_ON(p != page_address(page));
 		} else {
-			s = folio_slab(folio)->slab_cache;
+			s = page_slab(page)->slab_cache;
 			orig_size = get_orig_size(s, (void *)p);
 			ks = s->object_size;
 		}
-- 
2.47.2

Re: [PATCH v3 09/17] slab: Remove folio references from __do_krealloc()

[Vlastimil Babka]

On 10/24/25 22:44, Matthew Wilcox (Oracle) wrote:
> As with memcg_slab_post_charge(), we save a call to compound_head()
> for large kmallocs.  This has a slight change of behaviour in that
> non-vmalloc, non-slab, non-kmalloc pointers will now cause a NULL
> pointer dereference rather than a warning.  We could add that back if
> really needed.
> 
> Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>

This would be another one to change for consistency as proposed to 08/17

> ---
>  mm/slub.c | 14 +++++++-------
>  1 file changed, 7 insertions(+), 7 deletions(-)
> 
> diff --git a/mm/slub.c b/mm/slub.c
> index 100ed979312d..4e282e7782bb 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -6924,16 +6924,16 @@ __do_krealloc(const void *p, size_t new_size, unsigned long align, gfp_t flags,
>  	if (is_kfence_address(p)) {
>  		ks = orig_size = kfence_ksize(p);
>  	} else {
> -		struct folio *folio;
> +		struct page *page;
>  
> -		folio = virt_to_folio(p);
> -		if (unlikely(!folio_test_slab(folio))) {
> +		page = virt_to_page(p);
> +		if (unlikely(PageLargeKmalloc(page))) {
>  			/* Big kmalloc object */
> -			WARN_ON(folio_size(folio) <= KMALLOC_MAX_CACHE_SIZE);
> -			WARN_ON(p != folio_address(folio));
> -			ks = folio_size(folio);
> +			ks = page_size(page);
> +			WARN_ON(ks <= KMALLOC_MAX_CACHE_SIZE);
> +			WARN_ON(p != page_address(page));
>  		} else {
> -			s = folio_slab(folio)->slab_cache;
> +			s = page_slab(page)->slab_cache;
>  			orig_size = get_orig_size(s, (void *)p);
>  			ks = s->object_size;
>  		}

[PATCH v3 10/17] slab: Remove folio references from build_detached_freelist()

[Matthew Wilcox (Oracle)]

As with memcg_slab_post_charge(), we save a call to compound_head() for
large kmallocs.  This has a slight change of behaviour in that non-slab,
non-kmalloc pointers will now cause a NULL pointer dereference rather
than a warning.  We could add that back if really needed.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
 mm/slub.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index 4e282e7782bb..e72016ff0236 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -7244,23 +7244,23 @@ int build_detached_freelist(struct kmem_cache *s, size_t size,
 {
 	int lookahead = 3;
 	void *object;
-	struct folio *folio;
+	struct page *page;
 	size_t same;
 
 	object = p[--size];
-	folio = virt_to_folio(object);
+	page = virt_to_page(object);
 	if (!s) {
 		/* Handle kalloc'ed objects */
-		if (unlikely(!folio_test_slab(folio))) {
-			free_large_kmalloc(&folio->page, object);
+		if (unlikely(PageLargeKmalloc(page))) {
+			free_large_kmalloc(page, object);
 			df->slab = NULL;
 			return size;
 		}
 		/* Derive kmem_cache from object */
-		df->slab = folio_slab(folio);
+		df->slab = page_slab(page);
 		df->s = df->slab->slab_cache;
 	} else {
-		df->slab = folio_slab(folio);
+		df->slab = page_slab(page);
 		df->s = cache_from_obj(s, object); /* Support for memcg */
 	}
 
-- 
2.47.2

Re: [PATCH v3 10/17] slab: Remove folio references from build_detached_freelist()

[Vlastimil Babka]

On 10/24/25 22:44, Matthew Wilcox (Oracle) wrote:
> As with memcg_slab_post_charge(), we save a call to compound_head() for
> large kmallocs.  This has a slight change of behaviour in that non-slab,
> non-kmalloc pointers will now cause a NULL pointer dereference rather
> than a warning.  We could add that back if really needed.
> 
> Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>

Ditto

> ---
>  mm/slub.c | 12 ++++++------
>  1 file changed, 6 insertions(+), 6 deletions(-)
> 
> diff --git a/mm/slub.c b/mm/slub.c
> index 4e282e7782bb..e72016ff0236 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -7244,23 +7244,23 @@ int build_detached_freelist(struct kmem_cache *s, size_t size,
>  {
>  	int lookahead = 3;
>  	void *object;
> -	struct folio *folio;
> +	struct page *page;
>  	size_t same;
>  
>  	object = p[--size];
> -	folio = virt_to_folio(object);
> +	page = virt_to_page(object);
>  	if (!s) {
>  		/* Handle kalloc'ed objects */
> -		if (unlikely(!folio_test_slab(folio))) {
> -			free_large_kmalloc(&folio->page, object);
> +		if (unlikely(PageLargeKmalloc(page))) {
> +			free_large_kmalloc(page, object);
>  			df->slab = NULL;
>  			return size;
>  		}
>  		/* Derive kmem_cache from object */
> -		df->slab = folio_slab(folio);
> +		df->slab = page_slab(page);
>  		df->s = df->slab->slab_cache;
>  	} else {
> -		df->slab = folio_slab(folio);
> +		df->slab = page_slab(page);
>  		df->s = cache_from_obj(s, object); /* Support for memcg */
>  	}
>  

[PATCH v3 11/17] slab: Remove dead code from free_large_kmalloc()

[Matthew Wilcox (Oracle)]

All three callers of free_large_kmalloc() check PageLargeKmalloc
first so this warning is now unnecessary.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Reviewed-by: Christoph Lameter (Ampere) <cl@gentwo.org>
---
 mm/slub.c | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index e72016ff0236..e37f584352f2 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -6738,11 +6738,6 @@ static void free_large_kmalloc(struct page *page, void *object)
 {
 	unsigned int order = compound_order(page);
 
-	if (WARN_ON_ONCE(!PageLargeKmalloc(page))) {
-		dump_page(page, "Not a kmalloc allocation");
-		return;
-	}
-
 	if (WARN_ON_ONCE(order == 0))
 		pr_warn_once("object pointer: 0x%p\n", object);
 
-- 
2.47.2

Re: [PATCH v3 11/17] slab: Remove dead code from free_large_kmalloc()

[Vlastimil Babka]

On 10/24/25 22:44, Matthew Wilcox (Oracle) wrote:
> All three callers of free_large_kmalloc() check PageLargeKmalloc
> first so this warning is now unnecessary.
> 
> Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
> Reviewed-by: Christoph Lameter (Ampere) <cl@gentwo.org>

With testing page_slab() for NULL and fallback free_large_kmalloc() I'd then
not remove the warning.

> ---
>  mm/slub.c | 5 -----
>  1 file changed, 5 deletions(-)
> 
> diff --git a/mm/slub.c b/mm/slub.c
> index e72016ff0236..e37f584352f2 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -6738,11 +6738,6 @@ static void free_large_kmalloc(struct page *page, void *object)
>  {
>  	unsigned int order = compound_order(page);
>  
> -	if (WARN_ON_ONCE(!PageLargeKmalloc(page))) {
> -		dump_page(page, "Not a kmalloc allocation");
> -		return;
> -	}
> -
>  	if (WARN_ON_ONCE(order == 0))
>  		pr_warn_once("object pointer: 0x%p\n", object);
>  

[PATCH v3 12/17] slab: Remove folio references from kfree_rcu_sheaf()

[Matthew Wilcox (Oracle)]

In preparation for splitting struct slab from struct page and struct
folio, remove mentions of struct folio from this function.  Since
we don't need to handle large kmalloc objects specially here, we
can just use virt_to_slab().

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
 mm/slab_common.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/mm/slab_common.c b/mm/slab_common.c
index 236b4e25fce0..b613533b29e7 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -1615,17 +1615,15 @@ static void kfree_rcu_work(struct work_struct *work)
 static bool kfree_rcu_sheaf(void *obj)
 {
 	struct kmem_cache *s;
-	struct folio *folio;
 	struct slab *slab;
 
 	if (is_vmalloc_addr(obj))
 		return false;
 
-	folio = virt_to_folio(obj);
-	if (unlikely(!folio_test_slab(folio)))
+	slab = virt_to_slab(obj);
+	if (unlikely(!slab))
 		return false;
 
-	slab = folio_slab(folio);
 	s = slab->slab_cache;
 	if (s->cpu_sheaves) {
 		if (likely(!IS_ENABLED(CONFIG_NUMA) ||
-- 
2.47.2

[PATCH v3 13/17] slab: Remove folio references from kfree_nolock()

[Matthew Wilcox (Oracle)]

In preparation for splitting struct slab from struct page and struct
folio, remove mentions of struct folio from this function.  Since large
kmalloc objects are not supported here, we can just use virt_to_slab().

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
 mm/slub.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index e37f584352f2..39a26a970018 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -6837,7 +6837,6 @@ EXPORT_SYMBOL(kfree);
  */
 void kfree_nolock(const void *object)
 {
-	struct folio *folio;
 	struct slab *slab;
 	struct kmem_cache *s;
 	void *x = (void *)object;
@@ -6845,13 +6844,12 @@ void kfree_nolock(const void *object)
 	if (unlikely(ZERO_OR_NULL_PTR(object)))
 		return;
 
-	folio = virt_to_folio(object);
-	if (unlikely(!folio_test_slab(folio))) {
+	slab = virt_to_slab(object);
+	if (unlikely(!slab)) {
 		WARN_ONCE(1, "large_kmalloc is not supported by kfree_nolock()");
 		return;
 	}
 
-	slab = folio_slab(folio);
 	s = slab->slab_cache;
 
 	memcg_slab_free_hook(s, slab, &x, 1);
-- 
2.47.2

[PATCH v3 14/17] usercopy: Remove folio references from check_heap_object()

[Matthew Wilcox (Oracle)]

Because the pointer being checked may not lie within the first PAGE_SIZE
bytes of the object, we have to mark all pages as being LargeKmalloc.
We could use virt_to_head_page() instead, but that would pessimize
slab objects.  Once we move to memdescs properly, we'll tag each page
as being LargeKmalloc anyway, so this is more in keeping with how code
will be written in the future.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
 mm/slub.c     |  8 ++++++--
 mm/usercopy.c | 21 ++++++++++++---------
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index 39a26a970018..54e425efaa6f 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -5581,10 +5581,12 @@ static void *___kmalloc_large_node(size_t size, gfp_t flags, int node)
 		page = __alloc_frozen_pages_noprof(flags, order, node, NULL);
 
 	if (page) {
+		unsigned long i;
 		ptr = page_address(page);
 		mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
 				      PAGE_SIZE << order);
-		__SetPageLargeKmalloc(page);
+		for (i = 0; i < 1UL << order; i++)
+			__SetPageLargeKmalloc(page + i);
 	}
 
 	ptr = kasan_kmalloc_large(ptr, size, flags);
@@ -6737,6 +6739,7 @@ EXPORT_SYMBOL(kmem_cache_free);
 static void free_large_kmalloc(struct page *page, void *object)
 {
 	unsigned int order = compound_order(page);
+	unsigned long i;
 
 	if (WARN_ON_ONCE(order == 0))
 		pr_warn_once("object pointer: 0x%p\n", object);
@@ -6747,7 +6750,8 @@ static void free_large_kmalloc(struct page *page, void *object)
 
 	mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
 			      -(PAGE_SIZE << order));
-	__ClearPageLargeKmalloc(page);
+	for (i = 0; i < 1UL << order; i++)
+		__ClearPageLargeKmalloc(page + i);
 	free_frozen_pages(page, order);
 }
 
diff --git a/mm/usercopy.c b/mm/usercopy.c
index dbdcc43964fb..8d21635147a4 100644
--- a/mm/usercopy.c
+++ b/mm/usercopy.c
@@ -164,7 +164,7 @@ static inline void check_heap_object(const void *ptr, unsigned long n,
 {
 	unsigned long addr = (unsigned long)ptr;
 	unsigned long offset;
-	struct folio *folio;
+	struct page *page;
 
 	if (is_kmap_addr(ptr)) {
 		offset = offset_in_page(ptr);
@@ -189,15 +189,18 @@ static inline void check_heap_object(const void *ptr, unsigned long n,
 	if (!virt_addr_valid(ptr))
 		return;
 
-	folio = virt_to_folio(ptr);
-
-	if (folio_test_slab(folio)) {
+	page = virt_to_page(ptr);
+	if (PageLargeKmalloc(page)) {
+		page = compound_head(page);
+		offset = ptr - page_address(page);
+		if (n > page_size(page) - offset)
+			usercopy_abort("kmalloc", NULL, to_user, offset, n);
+		return;
+	} else {
+		struct slab *slab = page_slab(page);
 		/* Check slab allocator for flags and size. */
-		__check_heap_object(ptr, n, folio_slab(folio), to_user);
-	} else if (folio_test_large(folio)) {
-		offset = ptr - folio_address(folio);
-		if (n > folio_size(folio) - offset)
-			usercopy_abort("page alloc", NULL, to_user, offset, n);
+		if (slab)
+			__check_heap_object(ptr, n, slab, to_user);
 	}
 }
 
-- 
2.47.2

Re: [PATCH v3 14/17] usercopy: Remove folio references from check_heap_object()

[Vlastimil Babka]

On 10/24/25 22:44, Matthew Wilcox (Oracle) wrote:
> Because the pointer being checked may not lie within the first PAGE_SIZE
> bytes of the object, we have to mark all pages as being LargeKmalloc.
> We could use virt_to_head_page() instead, but that would pessimize
> slab objects.

But for slab objects we do page_slab() which does compound_head() anyway, so
that pessimization already happens? Or do you mean something else?

Again we could do page_slab() first and handle large kmalloc when the result
is NULL?

> Once we move to memdescs properly, we'll tag each page
> as being LargeKmalloc anyway, so this is more in keeping with how code
> will be written in the future.

Wouldn't this be the first case whre we would set page type on tail pages of
a compound page? If yes I'd rather postpone it until that's introduced
globally as part of the memdesc conversion, if the reason is mainly the one
disputed above.

> Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
> ---
>  mm/slub.c     |  8 ++++++--
>  mm/usercopy.c | 21 ++++++++++++---------
>  2 files changed, 18 insertions(+), 11 deletions(-)
> 
> diff --git a/mm/slub.c b/mm/slub.c
> index 39a26a970018..54e425efaa6f 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -5581,10 +5581,12 @@ static void *___kmalloc_large_node(size_t size, gfp_t flags, int node)
>  		page = __alloc_frozen_pages_noprof(flags, order, node, NULL);
>  
>  	if (page) {
> +		unsigned long i;
>  		ptr = page_address(page);
>  		mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
>  				      PAGE_SIZE << order);
> -		__SetPageLargeKmalloc(page);
> +		for (i = 0; i < 1UL << order; i++)
> +			__SetPageLargeKmalloc(page + i);
>  	}
>  
>  	ptr = kasan_kmalloc_large(ptr, size, flags);
> @@ -6737,6 +6739,7 @@ EXPORT_SYMBOL(kmem_cache_free);
>  static void free_large_kmalloc(struct page *page, void *object)
>  {
>  	unsigned int order = compound_order(page);
> +	unsigned long i;
>  
>  	if (WARN_ON_ONCE(order == 0))
>  		pr_warn_once("object pointer: 0x%p\n", object);
> @@ -6747,7 +6750,8 @@ static void free_large_kmalloc(struct page *page, void *object)
>  
>  	mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
>  			      -(PAGE_SIZE << order));
> -	__ClearPageLargeKmalloc(page);
> +	for (i = 0; i < 1UL << order; i++)
> +		__ClearPageLargeKmalloc(page + i);
>  	free_frozen_pages(page, order);
>  }
>  
> diff --git a/mm/usercopy.c b/mm/usercopy.c
> index dbdcc43964fb..8d21635147a4 100644
> --- a/mm/usercopy.c
> +++ b/mm/usercopy.c
> @@ -164,7 +164,7 @@ static inline void check_heap_object(const void *ptr, unsigned long n,
>  {
>  	unsigned long addr = (unsigned long)ptr;
>  	unsigned long offset;
> -	struct folio *folio;
> +	struct page *page;
>  
>  	if (is_kmap_addr(ptr)) {
>  		offset = offset_in_page(ptr);
> @@ -189,15 +189,18 @@ static inline void check_heap_object(const void *ptr, unsigned long n,
>  	if (!virt_addr_valid(ptr))
>  		return;
>  
> -	folio = virt_to_folio(ptr);
> -
> -	if (folio_test_slab(folio)) {
> +	page = virt_to_page(ptr);
> +	if (PageLargeKmalloc(page)) {
> +		page = compound_head(page);
> +		offset = ptr - page_address(page);
> +		if (n > page_size(page) - offset)
> +			usercopy_abort("kmalloc", NULL, to_user, offset, n);
> +		return;
> +	} else {
> +		struct slab *slab = page_slab(page);
>  		/* Check slab allocator for flags and size. */
> -		__check_heap_object(ptr, n, folio_slab(folio), to_user);
> -	} else if (folio_test_large(folio)) {
> -		offset = ptr - folio_address(folio);
> -		if (n > folio_size(folio) - offset)
> -			usercopy_abort("page alloc", NULL, to_user, offset, n);
> +		if (slab)
> +			__check_heap_object(ptr, n, slab, to_user);
>  	}
>  }
>  

[PATCH v3 15/17] memcg: Convert mem_cgroup_from_obj_folio() to mem_cgroup_from_obj_slab()

[Matthew Wilcox (Oracle)]

In preparation for splitting struct slab from struct page and struct
folio, convert the pointer to a slab rather than a folio.  This means
we can end up passing a NULL slab pointer to mem_cgroup_from_obj_slab()
if the pointer is not to a page allocated to slab, and we handle that
appropriately by returning NULL.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
 mm/memcontrol.c | 36 +++++++++++++-----------------------
 1 file changed, 13 insertions(+), 23 deletions(-)

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 1a95049d8b88..c3ba53b40375 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2558,38 +2558,28 @@ static inline void mod_objcg_mlstate(struct obj_cgroup *objcg,
 }
 
 static __always_inline
-struct mem_cgroup *mem_cgroup_from_obj_folio(struct folio *folio, void *p)
+struct mem_cgroup *mem_cgroup_from_obj_slab(struct slab *slab, void *p)
 {
 	/*
 	 * Slab objects are accounted individually, not per-page.
 	 * Memcg membership data for each individual object is saved in
 	 * slab->obj_exts.
 	 */
-	if (folio_test_slab(folio)) {
-		struct slabobj_ext *obj_exts;
-		struct slab *slab;
-		unsigned int off;
-
-		slab = folio_slab(folio);
-		obj_exts = slab_obj_exts(slab);
-		if (!obj_exts)
-			return NULL;
+	struct slabobj_ext *obj_exts;
+	unsigned int off;
 
-		off = obj_to_index(slab->slab_cache, slab, p);
-		if (obj_exts[off].objcg)
-			return obj_cgroup_memcg(obj_exts[off].objcg);
+	if (!slab)
+		return NULL;
 
+	obj_exts = slab_obj_exts(slab);
+	if (!obj_exts)
 		return NULL;
-	}
 
-	/*
-	 * folio_memcg_check() is used here, because in theory we can encounter
-	 * a folio where the slab flag has been cleared already, but
-	 * slab->obj_exts has not been freed yet
-	 * folio_memcg_check() will guarantee that a proper memory
-	 * cgroup pointer or NULL will be returned.
-	 */
-	return folio_memcg_check(folio);
+	off = obj_to_index(slab->slab_cache, slab, p);
+	if (obj_exts[off].objcg)
+		return obj_cgroup_memcg(obj_exts[off].objcg);
+
+	return NULL;
 }
 
 /*
@@ -2606,7 +2596,7 @@ struct mem_cgroup *mem_cgroup_from_slab_obj(void *p)
 	if (mem_cgroup_disabled())
 		return NULL;
 
-	return mem_cgroup_from_obj_folio(virt_to_folio(p), p);
+	return mem_cgroup_from_obj_slab(virt_to_slab(p), p);
 }
 
 static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)
-- 
2.47.2

Re: [PATCH v3 15/17] memcg: Convert mem_cgroup_from_obj_folio() to mem_cgroup_from_obj_slab()

[Vlastimil Babka]

On 10/24/25 22:44, Matthew Wilcox (Oracle) wrote:
> In preparation for splitting struct slab from struct page and struct
> folio, convert the pointer to a slab rather than a folio.  This means
> we can end up passing a NULL slab pointer to mem_cgroup_from_obj_slab()
> if the pointer is not to a page allocated to slab, and we handle that
> appropriately by returning NULL.
> 
> Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
> ---
>  mm/memcontrol.c | 36 +++++++++++++-----------------------
>  1 file changed, 13 insertions(+), 23 deletions(-)
> 
> diff --git a/mm/memcontrol.c b/mm/memcontrol.c
> index 1a95049d8b88..c3ba53b40375 100644
> --- a/mm/memcontrol.c
> +++ b/mm/memcontrol.c
> @@ -2558,38 +2558,28 @@ static inline void mod_objcg_mlstate(struct obj_cgroup *objcg,
>  }
>  
>  static __always_inline
> -struct mem_cgroup *mem_cgroup_from_obj_folio(struct folio *folio, void *p)
> +struct mem_cgroup *mem_cgroup_from_obj_slab(struct slab *slab, void *p)
>  {
>  	/*
>  	 * Slab objects are accounted individually, not per-page.
>  	 * Memcg membership data for each individual object is saved in
>  	 * slab->obj_exts.
>  	 */
> -	if (folio_test_slab(folio)) {
> -		struct slabobj_ext *obj_exts;
> -		struct slab *slab;
> -		unsigned int off;
> -
> -		slab = folio_slab(folio);
> -		obj_exts = slab_obj_exts(slab);
> -		if (!obj_exts)
> -			return NULL;
> +	struct slabobj_ext *obj_exts;
> +	unsigned int off;
>  
> -		off = obj_to_index(slab->slab_cache, slab, p);
> -		if (obj_exts[off].objcg)
> -			return obj_cgroup_memcg(obj_exts[off].objcg);
> +	if (!slab)
> +		return NULL;
>  
> +	obj_exts = slab_obj_exts(slab);
> +	if (!obj_exts)
>  		return NULL;
> -	}
>  
> -	/*
> -	 * folio_memcg_check() is used here, because in theory we can encounter
> -	 * a folio where the slab flag has been cleared already, but
> -	 * slab->obj_exts has not been freed yet
> -	 * folio_memcg_check() will guarantee that a proper memory
> -	 * cgroup pointer or NULL will be returned.
> -	 */
> -	return folio_memcg_check(folio);

The changelog should perhaps mention this removal so it doesn't give people
pause. I think the comment describes an impossible scenario, you probably
concluded the same. I'm not sure if it became impossible unnoticed at some
point, or was bogus since the beginning. I'll pretend I did not try git
blame that.

> +	off = obj_to_index(slab->slab_cache, slab, p);
> +	if (obj_exts[off].objcg)
> +		return obj_cgroup_memcg(obj_exts[off].objcg);
> +
> +	return NULL;
>  }
>  
>  /*
> @@ -2606,7 +2596,7 @@ struct mem_cgroup *mem_cgroup_from_slab_obj(void *p)
>  	if (mem_cgroup_disabled())
>  		return NULL;
>  
> -	return mem_cgroup_from_obj_folio(virt_to_folio(p), p);
> +	return mem_cgroup_from_obj_slab(virt_to_slab(p), p);
>  }
>  
>  static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)

[PATCH v3 16/17] kasan: Remove references to folio in __kasan_mempool_poison_object()

[Matthew Wilcox (Oracle)]

In preparation for splitting struct slab from struct page and struct
folio, remove mentions of struct folio from this function.  We can
discard the comment as using PageLargeKmalloc() rather than
!folio_test_slab() makes it obvious.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Acked-by: David Hildenbrand <david@redhat.com>
---
 mm/kasan/common.c | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/mm/kasan/common.c b/mm/kasan/common.c
index 22e5d67ff064..1d27f1bd260b 100644
--- a/mm/kasan/common.c
+++ b/mm/kasan/common.c
@@ -517,24 +517,20 @@ void __kasan_mempool_unpoison_pages(struct page *page, unsigned int order,
 
 bool __kasan_mempool_poison_object(void *ptr, unsigned long ip)
 {
-	struct folio *folio = virt_to_folio(ptr);
+	struct page *page = virt_to_page(ptr);
 	struct slab *slab;
 
-	/*
-	 * This function can be called for large kmalloc allocation that get
-	 * their memory from page_alloc. Thus, the folio might not be a slab.
-	 */
-	if (unlikely(!folio_test_slab(folio))) {
+	if (unlikely(PageLargeKmalloc(page))) {
 		if (check_page_allocation(ptr, ip))
 			return false;
-		kasan_poison(ptr, folio_size(folio), KASAN_PAGE_FREE, false);
+		kasan_poison(ptr, page_size(page), KASAN_PAGE_FREE, false);
 		return true;
 	}
 
 	if (is_kfence_address(ptr))
 		return true;
 
-	slab = folio_slab(folio);
+	slab = page_slab(page);
 
 	if (check_slab_allocation(slab->slab_cache, ptr, ip))
 		return false;
-- 
2.47.2

Re: [PATCH v3 16/17] kasan: Remove references to folio in __kasan_mempool_poison_object()

[Vlastimil Babka]

On 10/24/25 22:44, Matthew Wilcox (Oracle) wrote:
> In preparation for splitting struct slab from struct page and struct
> folio, remove mentions of struct folio from this function.  We can
> discard the comment as using PageLargeKmalloc() rather than
> !folio_test_slab() makes it obvious.
> 
> Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
> Acked-by: David Hildenbrand <david@redhat.com>

+Cc KASAN folks

This too could check page_slab() first, but it's not that important. Note
that it should be fine even with not marking all tail pages as
PageLargeKmalloc(), as ptr should be pointer to the head page here.

> ---
>  mm/kasan/common.c | 12 ++++--------
>  1 file changed, 4 insertions(+), 8 deletions(-)
> 
> diff --git a/mm/kasan/common.c b/mm/kasan/common.c
> index 22e5d67ff064..1d27f1bd260b 100644
> --- a/mm/kasan/common.c
> +++ b/mm/kasan/common.c
> @@ -517,24 +517,20 @@ void __kasan_mempool_unpoison_pages(struct page *page, unsigned int order,
>  
>  bool __kasan_mempool_poison_object(void *ptr, unsigned long ip)
>  {
> -	struct folio *folio = virt_to_folio(ptr);
> +	struct page *page = virt_to_page(ptr);
>  	struct slab *slab;
>  
> -	/*
> -	 * This function can be called for large kmalloc allocation that get
> -	 * their memory from page_alloc. Thus, the folio might not be a slab.
> -	 */
> -	if (unlikely(!folio_test_slab(folio))) {
> +	if (unlikely(PageLargeKmalloc(page))) {
>  		if (check_page_allocation(ptr, ip))
>  			return false;
> -		kasan_poison(ptr, folio_size(folio), KASAN_PAGE_FREE, false);
> +		kasan_poison(ptr, page_size(page), KASAN_PAGE_FREE, false);
>  		return true;
>  	}
>  
>  	if (is_kfence_address(ptr))
>  		return true;
>  
> -	slab = folio_slab(folio);
> +	slab = page_slab(page);
>  
>  	if (check_slab_allocation(slab->slab_cache, ptr, ip))
>  		return false;

[PATCH v3 17/17] slab: Remove references to folios from virt_to_slab()

[Matthew Wilcox (Oracle)]

Use page_slab() instead of virt_to_folio() which will work
perfectly when struct slab is separated from struct folio.

This was the last user of folio_slab(), so delete it.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
 mm/slab.h | 20 +-------------------
 1 file changed, 1 insertion(+), 19 deletions(-)

diff --git a/mm/slab.h b/mm/slab.h
index 31ccf0f6d3a1..6e3e80c90043 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -117,19 +117,6 @@ static_assert(sizeof(struct slab) <= sizeof(struct page));
 static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(freelist_aba_t)));
 #endif
 
-/**
- * folio_slab - Converts from folio to slab.
- * @folio: The folio.
- *
- * Currently struct slab is a different representation of a folio where
- * folio_test_slab() is true.
- *
- * Return: The slab which contains this folio.
- */
-#define folio_slab(folio)	(_Generic((folio),			\
-	const struct folio *:	(const struct slab *)(folio),		\
-	struct folio *:		(struct slab *)(folio)))
-
 /**
  * slab_folio - The folio allocated for a slab
  * @s: The slab.
@@ -192,12 +179,7 @@ static inline pg_data_t *slab_pgdat(const struct slab *slab)
 
 static inline struct slab *virt_to_slab(const void *addr)
 {
-	struct folio *folio = virt_to_folio(addr);
-
-	if (!folio_test_slab(folio))
-		return NULL;
-
-	return folio_slab(folio);
+	return page_slab(virt_to_page(addr));
 }
 
 static inline int slab_order(const struct slab *slab)
-- 
2.47.2