Netdev List
 help / color / mirror / Atom feed
From: Luigi Rizzo <lrizzo@google.com>
To: Andrew Morton <akpm@linux-foundation.org>,
	David Hildenbrand <david@kernel.org>,
	 Johannes Weiner <hannes@cmpxchg.org>,
	Vlastimil Babka <vbabka@kernel.org>,
	Joerg Roedel <joro@8bytes.org>,  Will Deacon <will@kernel.org>,
	Robin Murphy <robin.murphy@arm.com>,
	 Marek Szyprowski <m.szyprowski@samsung.com>,
	Jens Axboe <axboe@kernel.dk>,
	 "David S . Miller" <davem@davemloft.net>,
	Eric Dumazet <edumazet@kernel.org>,
	 Jakub Kicinski <kuba@kernel.org>,
	Paolo Abeni <pabeni@redhat.com>,
	Luigi Rizzo <rizzo.unipi@gmail.com>
Cc: Lorenzo Stoakes <ljs@kernel.org>,
	"Liam R . Howlett" <liam@infradead.org>,
	Mike Rapoport <rppt@kernel.org>,
	 Suren Baghdasaryan <surenb@google.com>,
	Michal Hocko <mhocko@suse.com>,
	 Brendan Jackman <brendan.jackman@linux.dev>,
	Zi Yan <ziy@nvidia.com>, Harry Yoo <harry@kernel.org>,
	 Christoph Lameter <cl@gentwo.org>,
	David Rientjes <rientjes@google.com>,
	 Roman Gushchin <roman.gushchin@linux.dev>,
	Hao Li <hao.li@linux.dev>,  Matthew Wilcox <willy@infradead.org>,
	Jan Kara <jack@suse.cz>, Jann Horn <jannh@google.com>,
	 Catalin Marinas <catalin.marinas@arm.com>,
	Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
	 "Rafael J . Wysocki" <rafael@kernel.org>,
	Danilo Krummrich <dakr@kernel.org>,
	Jonathan Corbet <corbet@lwn.net>,
	 Simon Horman <horms@kernel.org>,
	Jesper Dangaard Brouer <hawk@kernel.org>,
	 Ilias Apalodimas <ilias.apalodimas@linaro.org>,
	Willem de Bruijn <willemb@google.com>,
	 Kuniyuki Iwashima <kuniyu@google.com>,
	Joshua Washington <joshwash@google.com>,
	 Harshitha Ramamurthy <hramamurthy@google.com>,
	Alexander Lobakin <aleksander.lobakin@intel.com>,
	 Christoph Hellwig <hch@lst.de>, Jason Gunthorpe <jgg@ziepe.ca>,
	Dragos Tatulea <dtatulea@nvidia.com>,
	 Naman Gulati <namangulati@google.com>,
	Pranjal Shrivastava <praan@google.com>,
	 Mostafa Saleh <smostafa@google.com>,
	Randy Dunlap <rdunlap@infradead.org>,
	linux-mm@kvack.org,  iommu@lists.linux.dev,
	netdev@vger.kernel.org, linux-block@vger.kernel.org,
	 linux-fsdevel@vger.kernel.org, driver-core@lists.linux.dev,
	 linux-arm-kernel@lists.infradead.org, linux-api@vger.kernel.org,
	 linux-doc@vger.kernel.org, linux-kernel@vger.kernel.org,
	 Luigi Rizzo <lrizzo@google.com>
Subject: [RFC PATCH v3 09/25] mm: Add per-CPU block caches to PISA allocators
Date: Fri, 20 Nov 2026 01:06:39 +0000	[thread overview]
Message-ID: <20261120010655.467825-10-lrizzo@google.com> (raw)
In-Reply-To: <20261120010655.467825-1-lrizzo@google.com>

Every block allocation and free takes the allocator's lock. Highly
concurrent users e.g. socket tx fragments, can cause significant lock
contention.

Add pisa_enable_pcp(), an opt-in per-CPU front cache, similar
to the page allocator's per-CPU lists:

- Each per-CPU cache keeps up to 64 free blocks, and refills it from
  the main allocator in batches of 32.
- returned blocks belonging to the local NUMA node are kept in the local cache.
- returned blocks belonging to a remote NUMA node are returned to the
  proper per-node allocator.
The cache is fully integrated with the shrinker and CPU offlining to
drain the cache.

Enable per-CPU caches for the per-node order-0 RW and SKB_FRAG_PAGE_ORDER RO
allocators.

Signed-off-by: Luigi Rizzo <lrizzo@google.com>
---
 mm/pisa-alloc.c | 272 ++++++++++++++++++++++++++++++++++++++++++++++--
 mm/pisa-kunit.c |  79 ++++++++++++++
 mm/pisa-priv.h  |  11 +-
 3 files changed, 348 insertions(+), 14 deletions(-)

diff --git a/mm/pisa-alloc.c b/mm/pisa-alloc.c
index eda12d475c9a7..896709303f258 100644
--- a/mm/pisa-alloc.c
+++ b/mm/pisa-alloc.c
@@ -9,6 +9,7 @@
 #include <linux/bitmap.h>
 #include <linux/bitops.h>
 #include <linux/cache.h>
+#include <linux/cpuhotplug.h>
 #include <linux/device.h>
 #include <linux/dma-direct.h>
 #include <linux/dma-map-ops.h>
@@ -26,6 +27,7 @@
 #include <linux/moduleparam.h>
 #include <linux/mutex.h>
 #include <linux/page_owner.h>
+#include <linux/percpu.h>
 #include <linux/pgalloc_tag.h>
 #include <linux/page_table_check.h>
 #include <linux/pci.h>
@@ -324,13 +326,168 @@ static bool pisa_return_block_locked(struct pisa_alloc *pa, struct page *block)
 	return pisa_place_meta(meta);
 }
 
+/* Return @n freed blocks to @pa under a single acquisition of @pa->lock. */
+static void pisa_free_bulk(struct pisa_alloc *pa, unsigned int n, struct page **blocks)
+{
+	bool released = false;
+	unsigned long flags;
+
+	spin_lock_irqsave(&pa->lock, flags);
+	for (unsigned int i = 0; i < n; i++) {
+		if (likely(!is_pisa_block_poisoned(pa, blocks[i])))
+			released |= pisa_return_block_locked(pa, blocks[i]);
+	}
+	spin_unlock_irqrestore(&pa->lock, flags);
+
+	if (released)
+		pisa_schedule_reclaim();
+}
+
+/*
+ * Per-CPU block cache: each CPU caches up to PISA_PCP_HIGH free blocks
+ * (refcount 0, not in bitmaps), refilling and flushing in PISA_PCP_BATCH
+ * batches. Drained by the shrinker and CPU hotplug.
+ */
+#define PISA_PCP_BATCH	32U	/* blocks moved per refill or flush */
+
+struct pisa_pcp {
+	spinlock_t	lock;	/* by this CPU, or by a drain from any CPU */
+	unsigned int	count;
+	struct page	*blocks[PISA_PCP_HIGH];
+};
+
+/* Max blocks moved per per-CPU cache refill or flush (1..PISA_PCP_BATCH). */
+static unsigned int pisa_pcp_batch __read_mostly = PISA_PCP_BATCH;
+
+static int pisa_pcp_batch_set(const char *val, const struct kernel_param *kp)
+{
+	return param_set_uint_minmax(val, kp, 1, PISA_PCP_BATCH);
+}
+
+static const struct kernel_param_ops pisa_pcp_batch_ops = {
+	.set = pisa_pcp_batch_set,
+	.get = param_get_uint,
+};
+module_param_cb(pcp_batch, &pisa_pcp_batch_ops, &pisa_pcp_batch, 0644);
+
+static inline unsigned int pisa_pcp_batch_order(const struct pisa_alloc *pa)
+{
+	return max(1U, READ_ONCE(pisa_pcp_batch) >> pa->order);
+}
+
+/* Cache @n blocks at refcount 0 on this CPU, @n <= PISA_PCP_BATCH. */
+static void pisa_pcp_push(struct pisa_alloc *pa, unsigned int n, struct page **blocks)
+{
+	unsigned int batch = pisa_pcp_batch_order(pa);
+	unsigned int high = max(2U, PISA_PCP_HIGH >> pa->order);
+	struct pisa_pcp *pcp;
+	unsigned long flags;
+
+	/* @pcp->lock protects the cache: migrating to another CPU here is harmless. */
+	pcp = raw_cpu_ptr(pa->pcp);
+	spin_lock_irqsave(&pcp->lock, flags);
+	for (unsigned int i = 0; i < n; i++) {
+		if (pcp->count >= high) {
+			/* Keep the most recently freed, likely still in cache. */
+			pisa_free_bulk(pa, batch, pcp->blocks);
+			pcp->count -= batch;
+			memmove(pcp->blocks, pcp->blocks + batch,
+				pcp->count * sizeof(*pcp->blocks));
+		}
+		pcp->blocks[pcp->count++] = blocks[i];
+	}
+	spin_unlock_irqrestore(&pcp->lock, flags);
+}
+
+/* Take up to @nr blocks, at refcount 0, from this CPU's cache. */
+static unsigned int pisa_pcp_pop(struct pisa_alloc *pa, unsigned int nr, struct page **out)
+{
+	struct pisa_pcp *pcp;
+	unsigned int got = 0;
+	unsigned long flags;
+
+	/* @pcp->lock protects the cache: migrating to another CPU here is harmless. */
+	pcp = raw_cpu_ptr(pa->pcp);
+	spin_lock_irqsave(&pcp->lock, flags);
+	while (got < nr && pcp->count)
+		out[got++] = pcp->blocks[--pcp->count];
+	spin_unlock_irqrestore(&pcp->lock, flags);
+
+	return got;
+}
+
+/* Return the blocks @cpu caches for @pa. */
+static void pisa_pcp_drain_cpu(struct pisa_alloc *pa, int cpu)
+{
+	struct pisa_pcp *pcp = per_cpu_ptr(pa->pcp, cpu);
+
+	if (!READ_ONCE(pcp->count))
+		return;
+	guard(spinlock_irqsave)(&pcp->lock);
+	if (pcp->count) {
+		pisa_free_bulk(pa, pcp->count, pcp->blocks);
+		pcp->count = 0;
+	}
+}
+
+/* Drain every non-empty CPU cache of @pa. */
+void pisa_pcp_drain(struct pisa_alloc *pa)
+{
+	int cpu;
+
+	/* Pairs with smp_store_release() in pisa_enable_pcp(). */
+	if (!smp_load_acquire(&pa->pcp))
+		return;
+	for_each_possible_cpu(cpu)
+		pisa_pcp_drain_cpu(pa, cpu);
+}
+
+/* Blocks in @pa's caches, for shrinker and debugfs. */
+static unsigned long pisa_pcp_count(struct pisa_alloc *pa)
+{
+	unsigned long n = 0;
+	int cpu;
+
+	if (!pa->pcp)
+		return 0;
+	for_each_possible_cpu(cpu) {
+		/* A racy sum is good enough for reclaim and statistics. */
+		n += data_race(per_cpu_ptr(pa->pcp, cpu)->count);
+	}
+	return n;
+}
+
+/**
+ * pisa_enable_pcp - Put per-CPU block caches in front of an allocator
+ * @pa: Allocator to cache
+ *
+ * Return: 0, or -ENOMEM; @pa works without the cache.
+ */
+int pisa_enable_pcp(struct pisa_alloc *pa)
+{
+	struct pisa_pcp __percpu *pcp;
+	int cpu;
+
+	if (!pa || pa->pcp)
+		return 0;
+	pcp = alloc_percpu(struct pisa_pcp);
+	if (!pcp)
+		return -ENOMEM;
+	for_each_possible_cpu(cpu)
+		spin_lock_init(&per_cpu_ptr(pcp, cpu)->lock);
+	/* Publish initialized locks; readers depend on the pointer. */
+	if (cmpxchg_release(&pa->pcp, NULL, pcp))
+		free_percpu(pcp);
+	return 0;
+}
+
 static DEFINE_MUTEX(pisa_init_mutex);
 
 /* Return *@pap, creating and publishing it under @pisa_init_mutex if NULL. */
-__printf(5, 6)
+__printf(6, 7)
 struct pisa_alloc *pisa_alloc_ensure(struct pisa_alloc **pap, int nid,
 				     unsigned int order, enum dma_data_direction dir,
-				     const char *fmt, ...)
+				     bool pcp, const char *fmt, ...)
 {
 	/* Pairs with smp_store_release() below. */
 	struct pisa_alloc *pa = smp_load_acquire(pap);
@@ -349,6 +506,8 @@ struct pisa_alloc *pisa_alloc_ensure(struct pisa_alloc **pap, int nid,
 		va_start(args, fmt);
 		vsnprintf(pa->name, sizeof(pa->name), fmt, args);
 		va_end(args);
+		if (pcp)
+			pisa_enable_pcp(pa);
 		/* Pairs with smp_load_acquire() in lockless readers. */
 		smp_store_release(pap, pa);
 	}
@@ -392,7 +551,9 @@ bool __pisa_has_free_blocks_rx(unsigned int order, int nid)
 	struct pisa_alloc *pa = pisa_node_pa(nid, order, false);
 
 	return pa &&
-	       (!list_empty_careful(&pa->partial) ||
+	       /* Pairs with cmpxchg_release() in pisa_enable_pcp(). */
+	       ((smp_load_acquire(&pa->pcp) && raw_cpu_read(pa->pcp->count)) ||
+		!list_empty_careful(&pa->partial) ||
 		!list_empty_careful(&pa->partial_dirty) ||
 		!list_empty_careful(&pa->idle) ||
 		READ_ONCE(pisa_nodes[pa->nid].nr_pages));
@@ -636,6 +797,58 @@ static unsigned long __pisa_alloc_bulk(struct pisa_alloc *pa, gfp_t gfp, unsigne
 	return n;
 }
 
+/* Allocate @nr_pages blocks through this CPU's cache, refilling in batches on miss. */
+static unsigned long pisa_pcp_alloc(struct pisa_alloc *pa, gfp_t gfp, unsigned long nr_pages,
+				    struct page **page_array, bool frozen)
+{
+	unsigned int pcp_batch = pisa_pcp_batch_order(pa);
+	unsigned long got, j = 0;
+
+	got = pisa_pcp_pop(pa, nr_pages, page_array);
+	if (got < nr_pages) {
+		if (nr_pages - got >= pcp_batch) {
+			got += __pisa_alloc_bulk(pa, gfp, nr_pages - got,
+						 page_array + got, false, true);
+		} else {
+			struct page *batch[PISA_PCP_BATCH];
+			unsigned long n, i;
+
+			n = __pisa_alloc_bulk(pa, gfp, pcp_batch, batch, false, true);
+			for (i = 0; i < n && got < nr_pages; i++)
+				page_array[got++] = batch[i];
+			if (i < n) {
+				if (pa->nid == numa_mem_id())
+					pisa_pcp_push(pa, n - i, batch + i);
+				else
+					pisa_free_bulk(pa, n - i, batch + i);
+			}
+		}
+	}
+
+	for (unsigned long i = 0; i < got; i++) {
+		struct page *block = page_array[i];
+
+		page_array[i] = NULL;
+		/* Poisoned while cached: leak it, as pisa_scan_bitmap() does. */
+		if (is_pisa_block_poisoned(pa, block))
+			continue;
+		pisa_block_prep(block);
+		if (want_init_on_alloc(gfp)) {
+			memset(page_address(block), 0, PAGE_SIZE << pa->order);
+			if (frozen) /* Order zeroing before caller publishes the page. */
+				smp_wmb();
+		}
+		if (!frozen)
+			page_ref_unfreeze(block, 1);
+		page_array[j++] = block;
+	}
+	/* Replace blocks dropped as poisoned, so that callers do not see a short allocation. */
+	if (unlikely(j < got))
+		j += __pisa_alloc_bulk(pa, gfp, nr_pages - j, page_array + j,
+				       want_init_on_alloc(gfp), frozen);
+	return j;
+}
+
 /**
  * pisa_alloc_bulk - Allocate up to @nr_pages blocks from @pa
  * @pa: Allocator to allocate from
@@ -665,11 +878,16 @@ unsigned long pisa_alloc_bulk(struct pisa_alloc *pa, gfp_t gfp,
 			i++;
 			continue;
 		}
+
 		/* Fill the run of @n NULL entries at @i. */
 		for (n = 1; i + n < nr_pages && !page_array[i + n]; n++)
 			;
-		got = __pisa_alloc_bulk(pa, gfp, n, page_array + i,
-					want_init_on_alloc(gfp), frozen);
+		/* Pairs with smp_store_release() in pisa_enable_pcp(). */
+		if (smp_load_acquire(&pa->pcp) && pa->nid == numa_mem_id())
+			got = pisa_pcp_alloc(pa, gfp, n, page_array + i, frozen);
+		else
+			got = __pisa_alloc_bulk(pa, gfp, n, page_array + i,
+						want_init_on_alloc(gfp), frozen);
 		i += got;
 		if (got < n)
 			break;
@@ -812,6 +1030,7 @@ __pisa_alloc_map_coherent(struct device *dev, struct iommu_domain *domain,
 	pa = pisa_node_pa(dev_to_node(dev), 0, false);
 	if (!pa)
 		return NULL;
+	pisa_pcp_drain(pa);
 
 	nblocks = size >> PAGE_SHIFT;
 	align_mask = (1UL << get_order(size)) - 1;
@@ -911,6 +1130,16 @@ bool __pisa_free_page(struct page *page, unsigned int order)
 	if (want_init_on_free())
 		memset(page_address(page), 0, PAGE_SIZE << pa->order);
 
+	/* Pairs with cmpxchg_release() in pisa_enable_pcp(). */
+	if (smp_load_acquire(&pa->pcp) && pa->nid == numa_mem_id() &&
+	    page_to_nid(page) == numa_mem_id()) {
+		if (pisa_page_can_recycle(page)) {
+			pisa_pcp_push(pa, 1, &page);
+			return true;
+		}
+		pisa_pcp_drain_cpu(pa, raw_smp_processor_id());
+	}
+
 	spin_lock_irqsave(&pa->lock, flags);
 	should_release = pisa_return_block_locked(pa, page);
 	spin_unlock_irqrestore(&pa->lock, flags);
@@ -924,14 +1153,16 @@ bool __pisa_free_page(struct page *page, unsigned int order)
 /* Shrinker: releases reservoir pages first, then idle allocator pages. */
 static unsigned long pisa_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
 {
-	unsigned long nr = pisa_reservoir_count();
+	unsigned long nr = pisa_reservoir_count() << PISA_ORDER;
 	struct pisa_alloc *pa;
 
 	guard(read_lock)(&pisa_allocs_lock);
-	list_for_each_entry(pa, &pisa_allocs, node)
-		nr += READ_ONCE(pa->num_idle_pages);
+	list_for_each_entry(pa, &pisa_allocs, node) {
+		nr += (unsigned long)READ_ONCE(pa->num_idle_pages) << PISA_ORDER;
+		nr += pisa_pcp_count(pa) << pa->order;
+	}
 
-	return nr << PISA_ORDER;
+	return nr;
 }
 
 /* Detach PISA pages from @list onto @release_list up to @nr_to_scan base pages. */
@@ -967,6 +1198,8 @@ VISIBLE_IF_KUNIT unsigned long pisa_shrink_scan(struct shrinker *shrink, struct
 	if (freed < sc->nr_to_scan) {
 		read_lock(&pisa_allocs_lock);
 		list_for_each_entry(pa, &pisa_allocs, node) {
+			/* Cached blocks keep PISA pages from going idle. */
+			pisa_pcp_drain(pa);
 			freed += pisa_shrink_list(&pa->lock, &pa->idle,
 						  &pa->num_idle_pages, &release_list,
 						  sc->nr_to_scan - freed);
@@ -989,6 +1222,19 @@ VISIBLE_IF_KUNIT unsigned long pisa_shrink_scan(struct shrinker *shrink, struct
 	return freed ?: SHRINK_STOP;
 }
 
+/* Return the blocks a dead CPU cached, which nothing else would. */
+static int pisa_cpu_dead(unsigned int cpu)
+{
+	struct pisa_alloc *pa;
+
+	guard(read_lock)(&pisa_allocs_lock);
+	list_for_each_entry(pa, &pisa_allocs, node)
+		/* Pairs with smp_store_release() in pisa_enable_pcp(). */
+		if (smp_load_acquire(&pa->pcp))
+			pisa_pcp_drain_cpu(pa, cpu);
+	return 0;
+}
+
 static int __init pisa_init(void)
 {
 	struct shrinker *shrink;
@@ -1001,9 +1247,9 @@ static int __init pisa_init(void)
 	/* Preallocate the supported per-node allocators. */
 	for_each_node_state(nid, N_MEMORY) {
 		pisa_alloc_ensure(&pisa_nodes[nid].pa[0][0], nid, 0,
-				  DMA_BIDIRECTIONAL, "pisa_o0_n%d", nid);
+				  DMA_BIDIRECTIONAL, true, "pisa_o0_n%d", nid);
 		pisa_alloc_ensure(&pisa_nodes[nid].pa[1][SKB_FRAG_PAGE_ORDER],
-				  nid, SKB_FRAG_PAGE_ORDER, DMA_TO_DEVICE,
+				  nid, SKB_FRAG_PAGE_ORDER, DMA_TO_DEVICE, true,
 				  "pisa_ro_o%u_n%d", SKB_FRAG_PAGE_ORDER, nid);
 	}
 
@@ -1012,6 +1258,10 @@ static int __init pisa_init(void)
 	if (!pisa_wq)
 		pr_warn("pisa: no reclaim workqueue, falling back to system_wq\n");
 
+	if (cpuhp_setup_state_nocalls(CPUHP_BP_PREPARE_DYN, "iommu/pisa:dead", NULL,
+				      pisa_cpu_dead) < 0)
+		pr_warn("pisa: no CPU hotplug callback, offline CPUs keep cached blocks\n");
+
 	shrink = shrinker_alloc(0, "pisa");
 	if (!shrink) {
 		pr_warn("pisa: shrinker registration failed\n");
diff --git a/mm/pisa-kunit.c b/mm/pisa-kunit.c
index 1040786330cef..0f8cc9baf5d28 100644
--- a/mm/pisa-kunit.c
+++ b/mm/pisa-kunit.c
@@ -583,6 +583,83 @@ static void test_dev_pisa_attrs(struct kunit *test)
 	device_unregister(dev);
 }
 
+static void test_pcp_cache(struct kunit *test)
+{
+	struct page *blocks[PISA_PCP_HIGH + 1];
+	struct pisa_alloc *pa;
+	struct page *p1, *p2;
+	unsigned int i, n = 0;
+
+	KUNIT_EXPECT_EQ(test, pisa_enable_pcp(NULL), 0);
+	pa = pisa_node_pa(numa_mem_id(), 0, false);
+	KUNIT_ASSERT_NOT_NULL(test, pa);
+	KUNIT_ASSERT_EQ(test, pisa_enable_pcp(pa), 0);
+
+	p1 = pisa_alloc(pa, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, p1);
+	__free_page(p1);
+	p2 = pisa_alloc(pa, GFP_KERNEL | __GFP_ZERO);
+	KUNIT_EXPECT_PTR_EQ(test, p2, p1);
+	if (p2)
+		__free_page(p2);
+
+	/* Overflowing PISA_PCP_HIGH flushes a batch back to the allocator. */
+	for (i = 0; i < ARRAY_SIZE(blocks); i++) {
+		blocks[i] = pisa_alloc(pa, GFP_KERNEL);
+		if (!blocks[i])
+			break;
+		n++;
+	}
+	KUNIT_EXPECT_EQ(test, n, (unsigned int)ARRAY_SIZE(blocks));
+	for (i = 0; i < n; i++)
+		__free_page(blocks[i]);
+}
+
+static void test_gfp_pisa(struct kunit *test)
+{
+	struct shrink_control sc = { .nr_to_scan = ULONG_MAX };
+	struct device dev = { .numa_node = first_memory_node };
+	struct page *p, *p_ro, *bulk[4] = {};
+	unsigned long n;
+
+	KUNIT_ASSERT_EQ(test, pisa_meta_init(), 0);
+	pisa_shrink_scan(NULL, &sc);
+	KUNIT_EXPECT_EQ(test, pisa_rx_gfp(NULL), 0);
+	KUNIT_EXPECT_EQ(test, pisa_rx_gfp(&dev), 0);
+	KUNIT_EXPECT_FALSE(test, pisa_has_free_blocks_rx(&dev, 0, first_memory_node));
+
+	dev.pisa_rx_bufs = true;
+	KUNIT_EXPECT_EQ(test, pisa_rx_gfp(&dev), __GFP_PISA_RW);
+
+	p = alloc_pages_node(first_memory_node, GFP_KERNEL | pisa_rx_gfp(&dev), 0);
+	KUNIT_ASSERT_NOT_NULL(test, p);
+	if (p) {
+		KUNIT_EXPECT_TRUE(test, is_pisa_page(p));
+		KUNIT_EXPECT_EQ(test, pisa_meta_of_pfn(page_to_pfn(p))->prot,
+				IOMMU_READ | IOMMU_WRITE);
+		__free_page(p);
+		KUNIT_EXPECT_TRUE(test, pisa_has_free_blocks_rx(&dev, 0, first_memory_node));
+	}
+
+	p_ro = alloc_pages(GFP_KERNEL | __GFP_COMP | __GFP_PISA,
+			   SKB_FRAG_PAGE_ORDER);
+	KUNIT_ASSERT_NOT_NULL(test, p_ro);
+	if (p_ro) {
+		KUNIT_EXPECT_TRUE(test, is_pisa_page(p_ro));
+		KUNIT_EXPECT_EQ(test, pisa_meta_of_pfn(page_to_pfn(p_ro))->prot,
+				IOMMU_READ);
+		__free_pages(p_ro, SKB_FRAG_PAGE_ORDER);
+	}
+
+	n = alloc_pages_bulk_node(GFP_KERNEL | __GFP_PISA_RW, first_memory_node,
+				  ARRAY_SIZE(bulk), bulk);
+	KUNIT_EXPECT_EQ(test, n, ARRAY_SIZE(bulk));
+	for (int i = 0; i < n; i++) {
+		KUNIT_EXPECT_TRUE(test, is_pisa_page(bulk[i]));
+		__free_page(bulk[i]);
+	}
+}
+
 static void test_reservoir_and_scrubber(struct kunit *test)
 {
 	KUNIT_ASSERT_EQ(test, pisa_meta_init(), 0);
@@ -601,6 +678,8 @@ static struct kunit_case pisa_meta_test_cases[] = {
 #endif
 	KUNIT_CASE(test_coherent_alloc_and_free),
 	KUNIT_CASE(test_dev_pisa_attrs),
+	KUNIT_CASE(test_pcp_cache),
+	KUNIT_CASE(test_gfp_pisa),
 	KUNIT_CASE(test_reservoir_and_scrubber),
 	{}
 };
diff --git a/mm/pisa-priv.h b/mm/pisa-priv.h
index fb26eb3aa9b37..60120a05082a9 100644
--- a/mm/pisa-priv.h
+++ b/mm/pisa-priv.h
@@ -15,6 +15,7 @@
 #include <linux/topology.h>
 #include <linux/workqueue.h>
 
+struct pisa_pcp;
 
 #ifdef CONFIG_IOMMU_DMA
 int iommu_get_dma_cookie(struct iommu_domain *domain);
@@ -30,7 +31,7 @@ void iommu_put_dma_cookie(struct iommu_domain *domain);
  * Locking (disjoint groups; locks in different groups do not nest):
  *
  * 1. Allocators (mm/pisa-alloc.c, mm/pisa-node.c):
- *    pisa_allocs_lock (rwlock) -> pa->lock (spinlock)
+ *    pisa_allocs_lock (rwlock) -> pcp->lock (spinlock) -> pa->lock (spinlock)
  *    pisa_nodes[].lock (spinlock): leaf lock over a node's reservoir.
  *
  * 2. Metadata tables (mm/pisa-meta.c):
@@ -100,6 +101,7 @@ struct pisa_alloc {
 	spinlock_t		lock;
 	u8			order;		/* block order (<= PISA_ORDER) */
 	u8			prot;		/* IOMMU_READ and/or IOMMU_WRITE */
+	struct pisa_pcp __percpu *pcp;
 	struct list_head	partial;	/* partial pages with nr_clean > 0 */
 	struct list_head	partial_dirty;	/* partial pages with nr_clean == 0 */
 	struct list_head	idle;		/* completely free PISA pages */
@@ -200,10 +202,13 @@ static inline void pisa_alloc_backoff(struct pisa_node *node, bool can_block)
 struct pisa_alloc *pisa_alloc_create(int nid, unsigned int order, enum dma_data_direction dir);
 __printf(2, 3)
 void pisa_alloc_set_name(struct pisa_alloc *pa, const char *fmt, ...);
-__printf(5, 6)
+__printf(6, 7)
 struct pisa_alloc *pisa_alloc_ensure(struct pisa_alloc **pap, int nid,
 				     unsigned int order, enum dma_data_direction dir,
-				     const char *fmt, ...);
+				     bool pcp, const char *fmt, ...);
+#define PISA_PCP_HIGH		64U	/* blocks cached per CPU at most */
+int pisa_enable_pcp(struct pisa_alloc *pa);
+void pisa_pcp_drain(struct pisa_alloc *pa);
 bool pisa_place_meta(struct pisa_meta *meta);
 extern struct llist_head pisa_free_list;
 extern unsigned int pisa_idle_ms;
-- 
2.56.1.809.ga0b61035b5-goog


  parent reply	other threads:[~2026-11-20  1:07 UTC|newest]

Thread overview: 112+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-10-03 21:22 [RFC: DMA_PMD 00/22] DMA_PMD: PMD_SIZE-backed IO buffer pools Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 01/22] iommu/dma: introduce CONFIG_DMA_PMD and metadata table Luigi Rizzo
2026-10-03 21:44   ` Randy Dunlap
2026-10-04  9:14     ` Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 02/22] iommu/dma: add DMA_PMD pool lifecycle and page recycle hook Luigi Rizzo
2026-10-10  2:59   ` kernel test robot
2026-10-03 21:22 ` [RFC: DMA_PMD 03/22] mm: Add split_page_compound() Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 04/22] iommu/dma: add DMA_PMD pool block allocation Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 05/22] iommu/dma: Global cap and shrinker for DMA_PMD pool memory Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 06/22] iommu/dma: reserve a per-domain IOVA window for DMA_PMD pages Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 07/22] iommu/dma: release DMA_PMD domain mappings on domain teardown Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 08/22] iommu/dma: use per-domain IOVA window to map DMA_PMD memory Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 09/22] iommu/dma: Add DMA_PMD arena allocator Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 10/22] driver core: Add per-device dma_pmd_* sysfs attributes Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 11/22] dma-mapping: Use DMA_PMD arena for dma_alloc_attrs() Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 12/22] net/core: Use per-CPU DMA_PMD pools for skb_page_frag_refill() Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 13/22] net/core: Use DMA_PMD for page_pool memory Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 14/22] iommu/dma: Support decrypted and pinned DMA_PMD pages Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 15/22] iommu/dma: Add background page scrubber for DMA_PMD pools Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 16/22] iommu/dma: Add per-NUMA-node PMD page reservoir Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 17/22] net/gve: Use DMA_PMD memory for RX buffers Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 18/22] net/gve: Use DMA_PMD memory for tx header bounce buffers Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 19/22] net/mlx5e: " Luigi Rizzo
2026-10-14 17:22   ` Dragos Tatulea
2026-10-14 17:50     ` Dragos Tatulea
2026-10-14 18:28       ` Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 20/22] net/idpf: " Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 21/22] net/bnxt: " Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 22/22] iommu/dma: Add DMA_PMD statistics and debugfs Luigi Rizzo
2026-10-11  9:02 ` [RFC: DMA_PMD 00/22] DMA_PMD: PMD_SIZE-backed IO buffer pools Pranjal Shrivastava
2026-10-12 13:51 ` David Hildenbrand (Arm)
2026-10-12 14:45   ` Luigi Rizzo
2026-10-12 19:43     ` David Hildenbrand (Arm)
2026-10-13 10:42       ` Robin Murphy
2026-10-14 10:47         ` David Hildenbrand (Arm)
2026-10-14 11:05           ` Luigi Rizzo
2026-10-17 17:11       ` Luigi Rizzo
2026-10-22 20:12         ` Jason Gunthorpe
2026-10-22 21:51           ` Luigi Rizzo
2026-10-23 14:01             ` Jason Gunthorpe
2026-10-23 14:30               ` Luigi Rizzo
2026-10-23 14:50                 ` Jason Gunthorpe
2026-10-23 15:12                   ` Luigi Rizzo
2026-10-23 17:27                   ` Jakub Kicinski
2026-10-23 20:22                     ` Luigi Rizzo
2026-11-03  8:09                   ` Christoph Hellwig
2026-11-03  9:13                     ` Luigi Rizzo
2026-11-03 15:10                     ` Jason Gunthorpe
2026-11-06 17:21                       ` Naman Gulati
2026-11-06 17:30                         ` Jason Gunthorpe
2026-11-06 18:00                           ` Naman Gulati
2026-11-06 18:07                             ` Jason Gunthorpe
2026-11-06 18:17                               ` Naman Gulati
2026-11-06 18:43                                 ` Jason Gunthorpe
2026-11-06 19:01                                   ` Luigi Rizzo
2026-11-06 19:11                                   ` Naman Gulati
2026-10-12 14:49   ` Mostafa Saleh
2026-10-12 15:02     ` Luigi Rizzo
2026-10-14 17:28 ` Dragos Tatulea
2026-10-14 18:14   ` Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 00/25] PISA: Pre-mapped IOVA Superpage Allocator for DMA and CoCo Luigi Rizzo
2026-11-20  1:06   ` [RFC PATCH v3 " Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 01/25] mm: Introduce CONFIG_PISA and metadata table Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 02/25] mm: Add PISA allocator lifecycle and page recycle hook Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 03/25] mm: Add per-NUMA-node PISA page reservoir Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 04/25] mm/page_alloc: Extend alloc_pages() to __GFP_PISA and PISA block allocation Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 05/25] mm: Add global cap and shrinker for PISA pages Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 06/25] iommu/dma: Map PISA memory in a per-domain IOVA window Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 07/25] driver core: Add per-device pisa_* sysfs attributes for PISA optin Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 08/25] iommu/dma: Use PISA for coherent DMA allocations Luigi Rizzo
2026-11-20  1:06     ` Luigi Rizzo [this message]
2026-11-20  1:06     ` [RFC PATCH v3 10/25] mm: Add background page scrubber for PISA allocators Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 11/25] mm: Migrate user pages into PISA Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 12/25] block, mm: Support PISA for disk I/O and scrub migration Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 13/25] net: Cache leaf PISA TX flags on struct net_device Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 14/25] net: Use per-node PISA allocators for skb_page_frag_refill() Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 15/25] net: Use PISA for page_pool memory Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 16/25] mm/slub: Add SLAB_CACHE_PISA_RO for PISA-backed slab caches Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 17/25] net: Allocate small skb heads from PISA Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 18/25] net: Migrate user pages into PISA for MSG_ZEROCOPY Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 19/25] mm, iommu/dma: Support decrypted PISA pages Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 20/25] arm64/mm, mm: Support decrypted PISA pages in Realms Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 21/25] mm: Add PISA statistics and debugfs Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 22/25] net/libeth: Use GFP_KERNEL when pre-filling RX rings Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 23/25] net/gve: " Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 24/25] net/gve: Use PISA memory for RX buffers Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 25/25] Documentation: Add PISA architecture documentation Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 01/25] iommu/dma: Introduce CONFIG_PISA and metadata table Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 02/25] iommu/dma: Add PISA allocator lifecycle and page recycle hook Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 03/25] iommu/dma: Add per-NUMA-node PISA page reservoir Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 04/25] iommu/dma: Add PISA block allocation Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 05/25] iommu/dma: Add global cap and shrinker for PISA pages Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 06/25] iommu/dma: Reserve a per-domain IOVA window " Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 07/25] iommu/dma: Release PISA domain mappings on domain teardown Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 08/25] iommu/dma: Use per-domain IOVA window to map PISA memory Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 09/25] driver core: Add per-device pisa_* sysfs attributes for PISA optin Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 10/25] iommu/dma: Use PISA for coherent DMA allocations Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 11/25] iommu/dma: Add per-CPU block caches to PISA allocators Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 12/25] net/core: Use per-node PISA allocators for skb_page_frag_refill() Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 13/25] net/core: Use PISA for page_pool memory Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 14/25] iommu/dma: Support decrypted PISA pages Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 15/25] iommu/dma: Add background page scrubber for PISA allocators Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 16/25] mm/slub: Add SLAB_CACHE_PISA_RO for PISA-backed slab caches Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 17/25] net: Allocate small skb heads from PISA Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 18/25] iommu/dma: Migrate user pages into PISA for TX zerocopy Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 19/25] iommu/dma: Stop recycling old PISA pages when allocator waste is high Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 20/25] net/libeth: Use GFP_KERNEL when pre-filling RX rings Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 21/25] net/gve: " Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 22/25] net/gve: Use PISA memory for RX buffers Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 23/25] block, mm, iommu/dma: Support PISA for disk I/O and scrub migration Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 24/25] Documentation: Add PISA architecture documentation Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 25/25] iommu/dma: Add PISA statistics and debugfs Luigi Rizzo

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261120010655.467825-10-lrizzo@google.com \
    --to=lrizzo@google.com \
    --cc=akpm@linux-foundation.org \
    --cc=aleksander.lobakin@intel.com \
    --cc=axboe@kernel.dk \
    --cc=brendan.jackman@linux.dev \
    --cc=catalin.marinas@arm.com \
    --cc=cl@gentwo.org \
    --cc=corbet@lwn.net \
    --cc=dakr@kernel.org \
    --cc=davem@davemloft.net \
    --cc=david@kernel.org \
    --cc=driver-core@lists.linux.dev \
    --cc=dtatulea@nvidia.com \
    --cc=edumazet@kernel.org \
    --cc=gregkh@linuxfoundation.org \
    --cc=hannes@cmpxchg.org \
    --cc=hao.li@linux.dev \
    --cc=harry@kernel.org \
    --cc=hawk@kernel.org \
    --cc=hch@lst.de \
    --cc=horms@kernel.org \
    --cc=hramamurthy@google.com \
    --cc=ilias.apalodimas@linaro.org \
    --cc=iommu@lists.linux.dev \
    --cc=jack@suse.cz \
    --cc=jannh@google.com \
    --cc=jgg@ziepe.ca \
    --cc=joro@8bytes.org \
    --cc=joshwash@google.com \
    --cc=kuba@kernel.org \
    --cc=kuniyu@google.com \
    --cc=liam@infradead.org \
    --cc=linux-api@vger.kernel.org \
    --cc=linux-arm-kernel@lists.infradead.org \
    --cc=linux-block@vger.kernel.org \
    --cc=linux-doc@vger.kernel.org \
    --cc=linux-fsdevel@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-mm@kvack.org \
    --cc=ljs@kernel.org \
    --cc=m.szyprowski@samsung.com \
    --cc=mhocko@suse.com \
    --cc=namangulati@google.com \
    --cc=netdev@vger.kernel.org \
    --cc=pabeni@redhat.com \
    --cc=praan@google.com \
    --cc=rafael@kernel.org \
    --cc=rdunlap@infradead.org \
    --cc=rientjes@google.com \
    --cc=rizzo.unipi@gmail.com \
    --cc=robin.murphy@arm.com \
    --cc=roman.gushchin@linux.dev \
    --cc=rppt@kernel.org \
    --cc=smostafa@google.com \
    --cc=surenb@google.com \
    --cc=vbabka@kernel.org \
    --cc=will@kernel.org \
    --cc=willemb@google.com \
    --cc=willy@infradead.org \
    --cc=ziy@nvidia.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox