All of lore.kernel.org
 help / color / mirror / Atom feed
From: Luigi Rizzo <lrizzo@google.com>
To: Andrew Morton <akpm@linux-foundation.org>,
	David Hildenbrand <david@kernel.org>,
	 Johannes Weiner <hannes@cmpxchg.org>,
	Vlastimil Babka <vbabka@kernel.org>,
	Joerg Roedel <joro@8bytes.org>,  Will Deacon <will@kernel.org>,
	Robin Murphy <robin.murphy@arm.com>,
	 Marek Szyprowski <m.szyprowski@samsung.com>,
	Jens Axboe <axboe@kernel.dk>,
	 "David S . Miller" <davem@davemloft.net>,
	Eric Dumazet <edumazet@kernel.org>,
	 Jakub Kicinski <kuba@kernel.org>,
	Paolo Abeni <pabeni@redhat.com>,
	Luigi Rizzo <rizzo.unipi@gmail.com>
Cc: Lorenzo Stoakes <ljs@kernel.org>,
	"Liam R . Howlett" <liam@infradead.org>,
	Mike Rapoport <rppt@kernel.org>,
	 Suren Baghdasaryan <surenb@google.com>,
	Michal Hocko <mhocko@suse.com>,
	 Brendan Jackman <brendan.jackman@linux.dev>,
	Zi Yan <ziy@nvidia.com>, Harry Yoo <harry@kernel.org>,
	 Christoph Lameter <cl@gentwo.org>,
	David Rientjes <rientjes@google.com>,
	 Roman Gushchin <roman.gushchin@linux.dev>,
	Hao Li <hao.li@linux.dev>,  Matthew Wilcox <willy@infradead.org>,
	Jan Kara <jack@suse.cz>, Jann Horn <jannh@google.com>,
	 Catalin Marinas <catalin.marinas@arm.com>,
	Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
	 "Rafael J . Wysocki" <rafael@kernel.org>,
	Danilo Krummrich <dakr@kernel.org>,
	Jonathan Corbet <corbet@lwn.net>,
	 Simon Horman <horms@kernel.org>,
	Jesper Dangaard Brouer <hawk@kernel.org>,
	 Ilias Apalodimas <ilias.apalodimas@linaro.org>,
	Willem de Bruijn <willemb@google.com>,
	 Kuniyuki Iwashima <kuniyu@google.com>,
	Joshua Washington <joshwash@google.com>,
	 Harshitha Ramamurthy <hramamurthy@google.com>,
	Alexander Lobakin <aleksander.lobakin@intel.com>,
	 Christoph Hellwig <hch@lst.de>, Jason Gunthorpe <jgg@ziepe.ca>,
	Dragos Tatulea <dtatulea@nvidia.com>,
	 Naman Gulati <namangulati@google.com>,
	Pranjal Shrivastava <praan@google.com>,
	 Mostafa Saleh <smostafa@google.com>,
	Randy Dunlap <rdunlap@infradead.org>,
	linux-mm@kvack.org,  iommu@lists.linux.dev,
	netdev@vger.kernel.org, linux-block@vger.kernel.org,
	 linux-fsdevel@vger.kernel.org, driver-core@lists.linux.dev,
	 linux-arm-kernel@lists.infradead.org, linux-api@vger.kernel.org,
	 linux-doc@vger.kernel.org, linux-kernel@vger.kernel.org,
	 Luigi Rizzo <lrizzo@google.com>
Subject: [RFC PATCH v3 09/25] mm: Add per-CPU block caches to PISA allocators
Date: Fri, 20 Nov 2026 01:06:39 +0000	[thread overview]
Message-ID: <20261120010655.467825-10-lrizzo@google.com> (raw)
In-Reply-To: <20261120010655.467825-1-lrizzo@google.com>

Every block allocation and free takes the allocator's lock. Highly
concurrent users e.g. socket tx fragments, can cause significant lock
contention.

Add pisa_enable_pcp(), an opt-in per-CPU front cache, similar
to the page allocator's per-CPU lists:

- Each per-CPU cache keeps up to 64 free blocks, and refills it from
  the main allocator in batches of 32.
- returned blocks belonging to the local NUMA node are kept in the local cache.
- returned blocks belonging to a remote NUMA node are returned to the
  proper per-node allocator.
The cache is fully integrated with the shrinker and CPU offlining to
drain the cache.

Enable per-CPU caches for the per-node order-0 RW and SKB_FRAG_PAGE_ORDER RO
allocators.

Signed-off-by: Luigi Rizzo <lrizzo@google.com>
---
 mm/pisa-alloc.c | 272 ++++++++++++++++++++++++++++++++++++++++++++++--
 mm/pisa-kunit.c |  79 ++++++++++++++
 mm/pisa-priv.h  |  11 +-
 3 files changed, 348 insertions(+), 14 deletions(-)

diff --git a/mm/pisa-alloc.c b/mm/pisa-alloc.c
index eda12d475c9a7..896709303f258 100644
--- a/mm/pisa-alloc.c
+++ b/mm/pisa-alloc.c
@@ -9,6 +9,7 @@
 #include <linux/bitmap.h>
 #include <linux/bitops.h>
 #include <linux/cache.h>
+#include <linux/cpuhotplug.h>
 #include <linux/device.h>
 #include <linux/dma-direct.h>
 #include <linux/dma-map-ops.h>
@@ -26,6 +27,7 @@
 #include <linux/moduleparam.h>
 #include <linux/mutex.h>
 #include <linux/page_owner.h>
+#include <linux/percpu.h>
 #include <linux/pgalloc_tag.h>
 #include <linux/page_table_check.h>
 #include <linux/pci.h>
@@ -324,13 +326,168 @@ static bool pisa_return_block_locked(struct pisa_alloc *pa, struct page *block)
 	return pisa_place_meta(meta);
 }
 
+/* Return @n freed blocks to @pa under a single acquisition of @pa->lock. */
+static void pisa_free_bulk(struct pisa_alloc *pa, unsigned int n, struct page **blocks)
+{
+	bool released = false;
+	unsigned long flags;
+
+	spin_lock_irqsave(&pa->lock, flags);
+	for (unsigned int i = 0; i < n; i++) {
+		if (likely(!is_pisa_block_poisoned(pa, blocks[i])))
+			released |= pisa_return_block_locked(pa, blocks[i]);
+	}
+	spin_unlock_irqrestore(&pa->lock, flags);
+
+	if (released)
+		pisa_schedule_reclaim();
+}
+
+/*
+ * Per-CPU block cache: each CPU caches up to PISA_PCP_HIGH free blocks
+ * (refcount 0, not in bitmaps), refilling and flushing in PISA_PCP_BATCH
+ * batches. Drained by the shrinker and CPU hotplug.
+ */
+#define PISA_PCP_BATCH	32U	/* blocks moved per refill or flush */
+
+struct pisa_pcp {
+	spinlock_t	lock;	/* by this CPU, or by a drain from any CPU */
+	unsigned int	count;
+	struct page	*blocks[PISA_PCP_HIGH];
+};
+
+/* Max blocks moved per per-CPU cache refill or flush (1..PISA_PCP_BATCH). */
+static unsigned int pisa_pcp_batch __read_mostly = PISA_PCP_BATCH;
+
+static int pisa_pcp_batch_set(const char *val, const struct kernel_param *kp)
+{
+	return param_set_uint_minmax(val, kp, 1, PISA_PCP_BATCH);
+}
+
+static const struct kernel_param_ops pisa_pcp_batch_ops = {
+	.set = pisa_pcp_batch_set,
+	.get = param_get_uint,
+};
+module_param_cb(pcp_batch, &pisa_pcp_batch_ops, &pisa_pcp_batch, 0644);
+
+static inline unsigned int pisa_pcp_batch_order(const struct pisa_alloc *pa)
+{
+	return max(1U, READ_ONCE(pisa_pcp_batch) >> pa->order);
+}
+
+/* Cache @n blocks at refcount 0 on this CPU, @n <= PISA_PCP_BATCH. */
+static void pisa_pcp_push(struct pisa_alloc *pa, unsigned int n, struct page **blocks)
+{
+	unsigned int batch = pisa_pcp_batch_order(pa);
+	unsigned int high = max(2U, PISA_PCP_HIGH >> pa->order);
+	struct pisa_pcp *pcp;
+	unsigned long flags;
+
+	/* @pcp->lock protects the cache: migrating to another CPU here is harmless. */
+	pcp = raw_cpu_ptr(pa->pcp);
+	spin_lock_irqsave(&pcp->lock, flags);
+	for (unsigned int i = 0; i < n; i++) {
+		if (pcp->count >= high) {
+			/* Keep the most recently freed, likely still in cache. */
+			pisa_free_bulk(pa, batch, pcp->blocks);
+			pcp->count -= batch;
+			memmove(pcp->blocks, pcp->blocks + batch,
+				pcp->count * sizeof(*pcp->blocks));
+		}
+		pcp->blocks[pcp->count++] = blocks[i];
+	}
+	spin_unlock_irqrestore(&pcp->lock, flags);
+}
+
+/* Take up to @nr blocks, at refcount 0, from this CPU's cache. */
+static unsigned int pisa_pcp_pop(struct pisa_alloc *pa, unsigned int nr, struct page **out)
+{
+	struct pisa_pcp *pcp;
+	unsigned int got = 0;
+	unsigned long flags;
+
+	/* @pcp->lock protects the cache: migrating to another CPU here is harmless. */
+	pcp = raw_cpu_ptr(pa->pcp);
+	spin_lock_irqsave(&pcp->lock, flags);
+	while (got < nr && pcp->count)
+		out[got++] = pcp->blocks[--pcp->count];
+	spin_unlock_irqrestore(&pcp->lock, flags);
+
+	return got;
+}
+
+/* Return the blocks @cpu caches for @pa. */
+static void pisa_pcp_drain_cpu(struct pisa_alloc *pa, int cpu)
+{
+	struct pisa_pcp *pcp = per_cpu_ptr(pa->pcp, cpu);
+
+	if (!READ_ONCE(pcp->count))
+		return;
+	guard(spinlock_irqsave)(&pcp->lock);
+	if (pcp->count) {
+		pisa_free_bulk(pa, pcp->count, pcp->blocks);
+		pcp->count = 0;
+	}
+}
+
+/* Drain every non-empty CPU cache of @pa. */
+void pisa_pcp_drain(struct pisa_alloc *pa)
+{
+	int cpu;
+
+	/* Pairs with smp_store_release() in pisa_enable_pcp(). */
+	if (!smp_load_acquire(&pa->pcp))
+		return;
+	for_each_possible_cpu(cpu)
+		pisa_pcp_drain_cpu(pa, cpu);
+}
+
+/* Blocks in @pa's caches, for shrinker and debugfs. */
+static unsigned long pisa_pcp_count(struct pisa_alloc *pa)
+{
+	unsigned long n = 0;
+	int cpu;
+
+	if (!pa->pcp)
+		return 0;
+	for_each_possible_cpu(cpu) {
+		/* A racy sum is good enough for reclaim and statistics. */
+		n += data_race(per_cpu_ptr(pa->pcp, cpu)->count);
+	}
+	return n;
+}
+
+/**
+ * pisa_enable_pcp - Put per-CPU block caches in front of an allocator
+ * @pa: Allocator to cache
+ *
+ * Return: 0, or -ENOMEM; @pa works without the cache.
+ */
+int pisa_enable_pcp(struct pisa_alloc *pa)
+{
+	struct pisa_pcp __percpu *pcp;
+	int cpu;
+
+	if (!pa || pa->pcp)
+		return 0;
+	pcp = alloc_percpu(struct pisa_pcp);
+	if (!pcp)
+		return -ENOMEM;
+	for_each_possible_cpu(cpu)
+		spin_lock_init(&per_cpu_ptr(pcp, cpu)->lock);
+	/* Publish initialized locks; readers depend on the pointer. */
+	if (cmpxchg_release(&pa->pcp, NULL, pcp))
+		free_percpu(pcp);
+	return 0;
+}
+
 static DEFINE_MUTEX(pisa_init_mutex);
 
 /* Return *@pap, creating and publishing it under @pisa_init_mutex if NULL. */
-__printf(5, 6)
+__printf(6, 7)
 struct pisa_alloc *pisa_alloc_ensure(struct pisa_alloc **pap, int nid,
 				     unsigned int order, enum dma_data_direction dir,
-				     const char *fmt, ...)
+				     bool pcp, const char *fmt, ...)
 {
 	/* Pairs with smp_store_release() below. */
 	struct pisa_alloc *pa = smp_load_acquire(pap);
@@ -349,6 +506,8 @@ struct pisa_alloc *pisa_alloc_ensure(struct pisa_alloc **pap, int nid,
 		va_start(args, fmt);
 		vsnprintf(pa->name, sizeof(pa->name), fmt, args);
 		va_end(args);
+		if (pcp)
+			pisa_enable_pcp(pa);
 		/* Pairs with smp_load_acquire() in lockless readers. */
 		smp_store_release(pap, pa);
 	}
@@ -392,7 +551,9 @@ bool __pisa_has_free_blocks_rx(unsigned int order, int nid)
 	struct pisa_alloc *pa = pisa_node_pa(nid, order, false);
 
 	return pa &&
-	       (!list_empty_careful(&pa->partial) ||
+	       /* Pairs with cmpxchg_release() in pisa_enable_pcp(). */
+	       ((smp_load_acquire(&pa->pcp) && raw_cpu_read(pa->pcp->count)) ||
+		!list_empty_careful(&pa->partial) ||
 		!list_empty_careful(&pa->partial_dirty) ||
 		!list_empty_careful(&pa->idle) ||
 		READ_ONCE(pisa_nodes[pa->nid].nr_pages));
@@ -636,6 +797,58 @@ static unsigned long __pisa_alloc_bulk(struct pisa_alloc *pa, gfp_t gfp, unsigne
 	return n;
 }
 
+/* Allocate @nr_pages blocks through this CPU's cache, refilling in batches on miss. */
+static unsigned long pisa_pcp_alloc(struct pisa_alloc *pa, gfp_t gfp, unsigned long nr_pages,
+				    struct page **page_array, bool frozen)
+{
+	unsigned int pcp_batch = pisa_pcp_batch_order(pa);
+	unsigned long got, j = 0;
+
+	got = pisa_pcp_pop(pa, nr_pages, page_array);
+	if (got < nr_pages) {
+		if (nr_pages - got >= pcp_batch) {
+			got += __pisa_alloc_bulk(pa, gfp, nr_pages - got,
+						 page_array + got, false, true);
+		} else {
+			struct page *batch[PISA_PCP_BATCH];
+			unsigned long n, i;
+
+			n = __pisa_alloc_bulk(pa, gfp, pcp_batch, batch, false, true);
+			for (i = 0; i < n && got < nr_pages; i++)
+				page_array[got++] = batch[i];
+			if (i < n) {
+				if (pa->nid == numa_mem_id())
+					pisa_pcp_push(pa, n - i, batch + i);
+				else
+					pisa_free_bulk(pa, n - i, batch + i);
+			}
+		}
+	}
+
+	for (unsigned long i = 0; i < got; i++) {
+		struct page *block = page_array[i];
+
+		page_array[i] = NULL;
+		/* Poisoned while cached: leak it, as pisa_scan_bitmap() does. */
+		if (is_pisa_block_poisoned(pa, block))
+			continue;
+		pisa_block_prep(block);
+		if (want_init_on_alloc(gfp)) {
+			memset(page_address(block), 0, PAGE_SIZE << pa->order);
+			if (frozen) /* Order zeroing before caller publishes the page. */
+				smp_wmb();
+		}
+		if (!frozen)
+			page_ref_unfreeze(block, 1);
+		page_array[j++] = block;
+	}
+	/* Replace blocks dropped as poisoned, so that callers do not see a short allocation. */
+	if (unlikely(j < got))
+		j += __pisa_alloc_bulk(pa, gfp, nr_pages - j, page_array + j,
+				       want_init_on_alloc(gfp), frozen);
+	return j;
+}
+
 /**
  * pisa_alloc_bulk - Allocate up to @nr_pages blocks from @pa
  * @pa: Allocator to allocate from
@@ -665,11 +878,16 @@ unsigned long pisa_alloc_bulk(struct pisa_alloc *pa, gfp_t gfp,
 			i++;
 			continue;
 		}
+
 		/* Fill the run of @n NULL entries at @i. */
 		for (n = 1; i + n < nr_pages && !page_array[i + n]; n++)
 			;
-		got = __pisa_alloc_bulk(pa, gfp, n, page_array + i,
-					want_init_on_alloc(gfp), frozen);
+		/* Pairs with smp_store_release() in pisa_enable_pcp(). */
+		if (smp_load_acquire(&pa->pcp) && pa->nid == numa_mem_id())
+			got = pisa_pcp_alloc(pa, gfp, n, page_array + i, frozen);
+		else
+			got = __pisa_alloc_bulk(pa, gfp, n, page_array + i,
+						want_init_on_alloc(gfp), frozen);
 		i += got;
 		if (got < n)
 			break;
@@ -812,6 +1030,7 @@ __pisa_alloc_map_coherent(struct device *dev, struct iommu_domain *domain,
 	pa = pisa_node_pa(dev_to_node(dev), 0, false);
 	if (!pa)
 		return NULL;
+	pisa_pcp_drain(pa);
 
 	nblocks = size >> PAGE_SHIFT;
 	align_mask = (1UL << get_order(size)) - 1;
@@ -911,6 +1130,16 @@ bool __pisa_free_page(struct page *page, unsigned int order)
 	if (want_init_on_free())
 		memset(page_address(page), 0, PAGE_SIZE << pa->order);
 
+	/* Pairs with cmpxchg_release() in pisa_enable_pcp(). */
+	if (smp_load_acquire(&pa->pcp) && pa->nid == numa_mem_id() &&
+	    page_to_nid(page) == numa_mem_id()) {
+		if (pisa_page_can_recycle(page)) {
+			pisa_pcp_push(pa, 1, &page);
+			return true;
+		}
+		pisa_pcp_drain_cpu(pa, raw_smp_processor_id());
+	}
+
 	spin_lock_irqsave(&pa->lock, flags);
 	should_release = pisa_return_block_locked(pa, page);
 	spin_unlock_irqrestore(&pa->lock, flags);
@@ -924,14 +1153,16 @@ bool __pisa_free_page(struct page *page, unsigned int order)
 /* Shrinker: releases reservoir pages first, then idle allocator pages. */
 static unsigned long pisa_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
 {
-	unsigned long nr = pisa_reservoir_count();
+	unsigned long nr = pisa_reservoir_count() << PISA_ORDER;
 	struct pisa_alloc *pa;
 
 	guard(read_lock)(&pisa_allocs_lock);
-	list_for_each_entry(pa, &pisa_allocs, node)
-		nr += READ_ONCE(pa->num_idle_pages);
+	list_for_each_entry(pa, &pisa_allocs, node) {
+		nr += (unsigned long)READ_ONCE(pa->num_idle_pages) << PISA_ORDER;
+		nr += pisa_pcp_count(pa) << pa->order;
+	}
 
-	return nr << PISA_ORDER;
+	return nr;
 }
 
 /* Detach PISA pages from @list onto @release_list up to @nr_to_scan base pages. */
@@ -967,6 +1198,8 @@ VISIBLE_IF_KUNIT unsigned long pisa_shrink_scan(struct shrinker *shrink, struct
 	if (freed < sc->nr_to_scan) {
 		read_lock(&pisa_allocs_lock);
 		list_for_each_entry(pa, &pisa_allocs, node) {
+			/* Cached blocks keep PISA pages from going idle. */
+			pisa_pcp_drain(pa);
 			freed += pisa_shrink_list(&pa->lock, &pa->idle,
 						  &pa->num_idle_pages, &release_list,
 						  sc->nr_to_scan - freed);
@@ -989,6 +1222,19 @@ VISIBLE_IF_KUNIT unsigned long pisa_shrink_scan(struct shrinker *shrink, struct
 	return freed ?: SHRINK_STOP;
 }
 
+/* Return the blocks a dead CPU cached, which nothing else would. */
+static int pisa_cpu_dead(unsigned int cpu)
+{
+	struct pisa_alloc *pa;
+
+	guard(read_lock)(&pisa_allocs_lock);
+	list_for_each_entry(pa, &pisa_allocs, node)
+		/* Pairs with smp_store_release() in pisa_enable_pcp(). */
+		if (smp_load_acquire(&pa->pcp))
+			pisa_pcp_drain_cpu(pa, cpu);
+	return 0;
+}
+
 static int __init pisa_init(void)
 {
 	struct shrinker *shrink;
@@ -1001,9 +1247,9 @@ static int __init pisa_init(void)
 	/* Preallocate the supported per-node allocators. */
 	for_each_node_state(nid, N_MEMORY) {
 		pisa_alloc_ensure(&pisa_nodes[nid].pa[0][0], nid, 0,
-				  DMA_BIDIRECTIONAL, "pisa_o0_n%d", nid);
+				  DMA_BIDIRECTIONAL, true, "pisa_o0_n%d", nid);
 		pisa_alloc_ensure(&pisa_nodes[nid].pa[1][SKB_FRAG_PAGE_ORDER],
-				  nid, SKB_FRAG_PAGE_ORDER, DMA_TO_DEVICE,
+				  nid, SKB_FRAG_PAGE_ORDER, DMA_TO_DEVICE, true,
 				  "pisa_ro_o%u_n%d", SKB_FRAG_PAGE_ORDER, nid);
 	}
 
@@ -1012,6 +1258,10 @@ static int __init pisa_init(void)
 	if (!pisa_wq)
 		pr_warn("pisa: no reclaim workqueue, falling back to system_wq\n");
 
+	if (cpuhp_setup_state_nocalls(CPUHP_BP_PREPARE_DYN, "iommu/pisa:dead", NULL,
+				      pisa_cpu_dead) < 0)
+		pr_warn("pisa: no CPU hotplug callback, offline CPUs keep cached blocks\n");
+
 	shrink = shrinker_alloc(0, "pisa");
 	if (!shrink) {
 		pr_warn("pisa: shrinker registration failed\n");
diff --git a/mm/pisa-kunit.c b/mm/pisa-kunit.c
index 1040786330cef..0f8cc9baf5d28 100644
--- a/mm/pisa-kunit.c
+++ b/mm/pisa-kunit.c
@@ -583,6 +583,83 @@ static void test_dev_pisa_attrs(struct kunit *test)
 	device_unregister(dev);
 }
 
+static void test_pcp_cache(struct kunit *test)
+{
+	struct page *blocks[PISA_PCP_HIGH + 1];
+	struct pisa_alloc *pa;
+	struct page *p1, *p2;
+	unsigned int i, n = 0;
+
+	KUNIT_EXPECT_EQ(test, pisa_enable_pcp(NULL), 0);
+	pa = pisa_node_pa(numa_mem_id(), 0, false);
+	KUNIT_ASSERT_NOT_NULL(test, pa);
+	KUNIT_ASSERT_EQ(test, pisa_enable_pcp(pa), 0);
+
+	p1 = pisa_alloc(pa, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, p1);
+	__free_page(p1);
+	p2 = pisa_alloc(pa, GFP_KERNEL | __GFP_ZERO);
+	KUNIT_EXPECT_PTR_EQ(test, p2, p1);
+	if (p2)
+		__free_page(p2);
+
+	/* Overflowing PISA_PCP_HIGH flushes a batch back to the allocator. */
+	for (i = 0; i < ARRAY_SIZE(blocks); i++) {
+		blocks[i] = pisa_alloc(pa, GFP_KERNEL);
+		if (!blocks[i])
+			break;
+		n++;
+	}
+	KUNIT_EXPECT_EQ(test, n, (unsigned int)ARRAY_SIZE(blocks));
+	for (i = 0; i < n; i++)
+		__free_page(blocks[i]);
+}
+
+static void test_gfp_pisa(struct kunit *test)
+{
+	struct shrink_control sc = { .nr_to_scan = ULONG_MAX };
+	struct device dev = { .numa_node = first_memory_node };
+	struct page *p, *p_ro, *bulk[4] = {};
+	unsigned long n;
+
+	KUNIT_ASSERT_EQ(test, pisa_meta_init(), 0);
+	pisa_shrink_scan(NULL, &sc);
+	KUNIT_EXPECT_EQ(test, pisa_rx_gfp(NULL), 0);
+	KUNIT_EXPECT_EQ(test, pisa_rx_gfp(&dev), 0);
+	KUNIT_EXPECT_FALSE(test, pisa_has_free_blocks_rx(&dev, 0, first_memory_node));
+
+	dev.pisa_rx_bufs = true;
+	KUNIT_EXPECT_EQ(test, pisa_rx_gfp(&dev), __GFP_PISA_RW);
+
+	p = alloc_pages_node(first_memory_node, GFP_KERNEL | pisa_rx_gfp(&dev), 0);
+	KUNIT_ASSERT_NOT_NULL(test, p);
+	if (p) {
+		KUNIT_EXPECT_TRUE(test, is_pisa_page(p));
+		KUNIT_EXPECT_EQ(test, pisa_meta_of_pfn(page_to_pfn(p))->prot,
+				IOMMU_READ | IOMMU_WRITE);
+		__free_page(p);
+		KUNIT_EXPECT_TRUE(test, pisa_has_free_blocks_rx(&dev, 0, first_memory_node));
+	}
+
+	p_ro = alloc_pages(GFP_KERNEL | __GFP_COMP | __GFP_PISA,
+			   SKB_FRAG_PAGE_ORDER);
+	KUNIT_ASSERT_NOT_NULL(test, p_ro);
+	if (p_ro) {
+		KUNIT_EXPECT_TRUE(test, is_pisa_page(p_ro));
+		KUNIT_EXPECT_EQ(test, pisa_meta_of_pfn(page_to_pfn(p_ro))->prot,
+				IOMMU_READ);
+		__free_pages(p_ro, SKB_FRAG_PAGE_ORDER);
+	}
+
+	n = alloc_pages_bulk_node(GFP_KERNEL | __GFP_PISA_RW, first_memory_node,
+				  ARRAY_SIZE(bulk), bulk);
+	KUNIT_EXPECT_EQ(test, n, ARRAY_SIZE(bulk));
+	for (int i = 0; i < n; i++) {
+		KUNIT_EXPECT_TRUE(test, is_pisa_page(bulk[i]));
+		__free_page(bulk[i]);
+	}
+}
+
 static void test_reservoir_and_scrubber(struct kunit *test)
 {
 	KUNIT_ASSERT_EQ(test, pisa_meta_init(), 0);
@@ -601,6 +678,8 @@ static struct kunit_case pisa_meta_test_cases[] = {
 #endif
 	KUNIT_CASE(test_coherent_alloc_and_free),
 	KUNIT_CASE(test_dev_pisa_attrs),
+	KUNIT_CASE(test_pcp_cache),
+	KUNIT_CASE(test_gfp_pisa),
 	KUNIT_CASE(test_reservoir_and_scrubber),
 	{}
 };
diff --git a/mm/pisa-priv.h b/mm/pisa-priv.h
index fb26eb3aa9b37..60120a05082a9 100644
--- a/mm/pisa-priv.h
+++ b/mm/pisa-priv.h
@@ -15,6 +15,7 @@
 #include <linux/topology.h>
 #include <linux/workqueue.h>
 
+struct pisa_pcp;
 
 #ifdef CONFIG_IOMMU_DMA
 int iommu_get_dma_cookie(struct iommu_domain *domain);
@@ -30,7 +31,7 @@ void iommu_put_dma_cookie(struct iommu_domain *domain);
  * Locking (disjoint groups; locks in different groups do not nest):
  *
  * 1. Allocators (mm/pisa-alloc.c, mm/pisa-node.c):
- *    pisa_allocs_lock (rwlock) -> pa->lock (spinlock)
+ *    pisa_allocs_lock (rwlock) -> pcp->lock (spinlock) -> pa->lock (spinlock)
  *    pisa_nodes[].lock (spinlock): leaf lock over a node's reservoir.
  *
  * 2. Metadata tables (mm/pisa-meta.c):
@@ -100,6 +101,7 @@ struct pisa_alloc {
 	spinlock_t		lock;
 	u8			order;		/* block order (<= PISA_ORDER) */
 	u8			prot;		/* IOMMU_READ and/or IOMMU_WRITE */
+	struct pisa_pcp __percpu *pcp;
 	struct list_head	partial;	/* partial pages with nr_clean > 0 */
 	struct list_head	partial_dirty;	/* partial pages with nr_clean == 0 */
 	struct list_head	idle;		/* completely free PISA pages */
@@ -200,10 +202,13 @@ static inline void pisa_alloc_backoff(struct pisa_node *node, bool can_block)
 struct pisa_alloc *pisa_alloc_create(int nid, unsigned int order, enum dma_data_direction dir);
 __printf(2, 3)
 void pisa_alloc_set_name(struct pisa_alloc *pa, const char *fmt, ...);
-__printf(5, 6)
+__printf(6, 7)
 struct pisa_alloc *pisa_alloc_ensure(struct pisa_alloc **pap, int nid,
 				     unsigned int order, enum dma_data_direction dir,
-				     const char *fmt, ...);
+				     bool pcp, const char *fmt, ...);
+#define PISA_PCP_HIGH		64U	/* blocks cached per CPU at most */
+int pisa_enable_pcp(struct pisa_alloc *pa);
+void pisa_pcp_drain(struct pisa_alloc *pa);
 bool pisa_place_meta(struct pisa_meta *meta);
 extern struct llist_head pisa_free_list;
 extern unsigned int pisa_idle_ms;
-- 
2.56.1.809.ga0b61035b5-goog


  parent reply	other threads:[~2026-11-20  1:07 UTC|newest]

Thread overview: 144+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-10-03 21:22 [RFC: DMA_PMD 00/22] DMA_PMD: PMD_SIZE-backed IO buffer pools Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 01/22] iommu/dma: introduce CONFIG_DMA_PMD and metadata table Luigi Rizzo
2026-10-03 21:44   ` Randy Dunlap
2026-10-04  9:14     ` Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 02/22] iommu/dma: add DMA_PMD pool lifecycle and page recycle hook Luigi Rizzo
2026-10-10  2:59   ` kernel test robot
2026-10-03 21:22 ` [RFC: DMA_PMD 03/22] mm: Add split_page_compound() Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 04/22] iommu/dma: add DMA_PMD pool block allocation Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 05/22] iommu/dma: Global cap and shrinker for DMA_PMD pool memory Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 06/22] iommu/dma: reserve a per-domain IOVA window for DMA_PMD pages Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 07/22] iommu/dma: release DMA_PMD domain mappings on domain teardown Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 08/22] iommu/dma: use per-domain IOVA window to map DMA_PMD memory Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 09/22] iommu/dma: Add DMA_PMD arena allocator Luigi Rizzo
2026-10-08 11:25   ` kernel test robot
2026-10-03 21:22 ` [RFC: DMA_PMD 10/22] driver core: Add per-device dma_pmd_* sysfs attributes Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 11/22] dma-mapping: Use DMA_PMD arena for dma_alloc_attrs() Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 12/22] net/core: Use per-CPU DMA_PMD pools for skb_page_frag_refill() Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 13/22] net/core: Use DMA_PMD for page_pool memory Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 14/22] iommu/dma: Support decrypted and pinned DMA_PMD pages Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 15/22] iommu/dma: Add background page scrubber for DMA_PMD pools Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 16/22] iommu/dma: Add per-NUMA-node PMD page reservoir Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 17/22] net/gve: Use DMA_PMD memory for RX buffers Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 18/22] net/gve: Use DMA_PMD memory for tx header bounce buffers Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 19/22] net/mlx5e: " Luigi Rizzo
2026-10-14 17:22   ` Dragos Tatulea
2026-10-14 17:50     ` Dragos Tatulea
2026-10-14 18:28       ` Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 20/22] net/idpf: " Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 21/22] net/bnxt: " Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 22/22] iommu/dma: Add DMA_PMD statistics and debugfs Luigi Rizzo
2026-10-11  9:02 ` [RFC: DMA_PMD 00/22] DMA_PMD: PMD_SIZE-backed IO buffer pools Pranjal Shrivastava
2026-10-12 13:51 ` David Hildenbrand (Arm)
2026-10-12 14:45   ` Luigi Rizzo
2026-10-12 19:43     ` David Hildenbrand (Arm)
2026-10-13 10:42       ` Robin Murphy
2026-10-14 10:47         ` David Hildenbrand (Arm)
2026-10-14 11:05           ` Luigi Rizzo
2026-10-17 17:11       ` Luigi Rizzo
2026-10-22 20:12         ` Jason Gunthorpe
2026-10-22 21:51           ` Luigi Rizzo
2026-10-23 14:01             ` Jason Gunthorpe
2026-10-23 14:30               ` Luigi Rizzo
2026-10-23 14:50                 ` Jason Gunthorpe
2026-10-23 15:12                   ` Luigi Rizzo
2026-10-23 17:27                   ` Jakub Kicinski
2026-10-23 20:22                     ` Luigi Rizzo
2026-11-03  8:09                   ` Christoph Hellwig
2026-11-03  9:13                     ` Luigi Rizzo
2026-11-03 15:10                     ` Jason Gunthorpe
2026-11-06 17:21                       ` Naman Gulati
2026-11-06 17:30                         ` Jason Gunthorpe
2026-11-06 18:00                           ` Naman Gulati
2026-11-06 18:07                             ` Jason Gunthorpe
2026-11-06 18:17                               ` Naman Gulati
2026-11-06 18:43                                 ` Jason Gunthorpe
2026-11-06 19:01                                   ` Luigi Rizzo
2026-11-06 19:11                                   ` Naman Gulati
2026-10-12 14:49   ` Mostafa Saleh
2026-10-12 15:02     ` Luigi Rizzo
2026-10-14 17:28 ` Dragos Tatulea
2026-10-14 18:14   ` Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 00/25] PISA: Pre-mapped IOVA Superpage Allocator for DMA and CoCo Luigi Rizzo
2026-11-20  1:06   ` [RFC PATCH v3 " Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 01/25] mm: Introduce CONFIG_PISA and metadata table Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 02/25] mm: Add PISA allocator lifecycle and page recycle hook Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 03/25] mm: Add per-NUMA-node PISA page reservoir Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 04/25] mm/page_alloc: Extend alloc_pages() to __GFP_PISA and PISA block allocation Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 05/25] mm: Add global cap and shrinker for PISA pages Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 06/25] iommu/dma: Map PISA memory in a per-domain IOVA window Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 07/25] driver core: Add per-device pisa_* sysfs attributes for PISA optin Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 08/25] iommu/dma: Use PISA for coherent DMA allocations Luigi Rizzo
2026-11-20  1:06     ` Luigi Rizzo [this message]
2026-11-20  1:06     ` [RFC PATCH v3 10/25] mm: Add background page scrubber for PISA allocators Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 11/25] mm: Migrate user pages into PISA Luigi Rizzo
2026-11-20 14:10       ` Lorenzo Stoakes (ARM)
2026-11-20  1:06     ` [RFC PATCH v3 12/25] block, mm: Support PISA for disk I/O and scrub migration Luigi Rizzo
2026-11-20 16:17       ` Christoph Hellwig
2026-11-20  1:06     ` [RFC PATCH v3 13/25] net: Cache leaf PISA TX flags on struct net_device Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 14/25] net: Use per-node PISA allocators for skb_page_frag_refill() Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 15/25] net: Use PISA for page_pool memory Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 16/25] mm/slub: Add SLAB_CACHE_PISA_RO for PISA-backed slab caches Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 17/25] net: Allocate small skb heads from PISA Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 18/25] net: Migrate user pages into PISA for MSG_ZEROCOPY Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 19/25] mm, iommu/dma: Support decrypted PISA pages Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 20/25] arm64/mm, mm: Support decrypted PISA pages in Realms Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 21/25] mm: Add PISA statistics and debugfs Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 22/25] net/libeth: Use GFP_KERNEL when pre-filling RX rings Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 23/25] net/gve: " Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 24/25] net/gve: Use PISA memory for RX buffers Luigi Rizzo
2026-11-20  1:06     ` [RFC PATCH v3 25/25] Documentation: Add PISA architecture documentation Luigi Rizzo
2026-11-20 13:50     ` [RFC PATCH v3 00/25] PISA: Pre-mapped IOVA Superpage Allocator for DMA and CoCo Jason Gunthorpe
2026-11-20 14:18       ` Lorenzo Stoakes (ARM)
2026-11-20 14:19       ` Luigi Rizzo
2026-11-20 14:36         ` Lorenzo Stoakes (ARM)
2026-11-20 14:54     ` Lorenzo Stoakes (ARM)
2026-11-20 16:12       ` David Hildenbrand (Arm)
2026-11-20 18:16         ` Lorenzo Stoakes (ARM)
2026-11-20 15:00   ` [RFC PATCH v2 " Matthew Wilcox
2026-11-09 18:25 ` [RFC PATCH v2 01/25] iommu/dma: Introduce CONFIG_PISA and metadata table Luigi Rizzo
2026-11-10 23:12   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 02/25] iommu/dma: Add PISA allocator lifecycle and page recycle hook Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 03/25] iommu/dma: Add per-NUMA-node PISA page reservoir Luigi Rizzo
2026-11-10 23:12   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 04/25] iommu/dma: Add PISA block allocation Luigi Rizzo
2026-11-10 23:12   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 05/25] iommu/dma: Add global cap and shrinker for PISA pages Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 06/25] iommu/dma: Reserve a per-domain IOVA window " Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 07/25] iommu/dma: Release PISA domain mappings on domain teardown Luigi Rizzo
2026-11-10 23:12   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 08/25] iommu/dma: Use per-domain IOVA window to map PISA memory Luigi Rizzo
2026-11-10 23:12   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 09/25] driver core: Add per-device pisa_* sysfs attributes for PISA optin Luigi Rizzo
2026-11-10 23:12   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 10/25] iommu/dma: Use PISA for coherent DMA allocations Luigi Rizzo
2026-11-10 23:12   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 11/25] iommu/dma: Add per-CPU block caches to PISA allocators Luigi Rizzo
2026-11-10 23:12   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 12/25] net/core: Use per-node PISA allocators for skb_page_frag_refill() Luigi Rizzo
2026-11-10 23:50   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 13/25] net/core: Use PISA for page_pool memory Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 14/25] iommu/dma: Support decrypted PISA pages Luigi Rizzo
2026-11-11  0:19   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 15/25] iommu/dma: Add background page scrubber for PISA allocators Luigi Rizzo
2026-11-11  0:37   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 16/25] mm/slub: Add SLAB_CACHE_PISA_RO for PISA-backed slab caches Luigi Rizzo
2026-11-11  1:03   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 17/25] net: Allocate small skb heads from PISA Luigi Rizzo
2026-11-11  1:33   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 18/25] iommu/dma: Migrate user pages into PISA for TX zerocopy Luigi Rizzo
2026-11-11  2:41   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 19/25] iommu/dma: Stop recycling old PISA pages when allocator waste is high Luigi Rizzo
2026-11-11  2:57   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 20/25] net/libeth: Use GFP_KERNEL when pre-filling RX rings Luigi Rizzo
2026-11-11  3:13   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 21/25] net/gve: " Luigi Rizzo
2026-11-11  3:22   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 22/25] net/gve: Use PISA memory for RX buffers Luigi Rizzo
2026-11-11  3:43   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 23/25] block, mm, iommu/dma: Support PISA for disk I/O and scrub migration Luigi Rizzo
2026-11-11  4:16   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 24/25] Documentation: Add PISA architecture documentation Luigi Rizzo
2026-11-11  4:24   ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 25/25] iommu/dma: Add PISA statistics and debugfs Luigi Rizzo
2026-11-11  4:37   ` sashiko-bot

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261120010655.467825-10-lrizzo@google.com \
    --to=lrizzo@google.com \
    --cc=akpm@linux-foundation.org \
    --cc=aleksander.lobakin@intel.com \
    --cc=axboe@kernel.dk \
    --cc=brendan.jackman@linux.dev \
    --cc=catalin.marinas@arm.com \
    --cc=cl@gentwo.org \
    --cc=corbet@lwn.net \
    --cc=dakr@kernel.org \
    --cc=davem@davemloft.net \
    --cc=david@kernel.org \
    --cc=driver-core@lists.linux.dev \
    --cc=dtatulea@nvidia.com \
    --cc=edumazet@kernel.org \
    --cc=gregkh@linuxfoundation.org \
    --cc=hannes@cmpxchg.org \
    --cc=hao.li@linux.dev \
    --cc=harry@kernel.org \
    --cc=hawk@kernel.org \
    --cc=hch@lst.de \
    --cc=horms@kernel.org \
    --cc=hramamurthy@google.com \
    --cc=ilias.apalodimas@linaro.org \
    --cc=iommu@lists.linux.dev \
    --cc=jack@suse.cz \
    --cc=jannh@google.com \
    --cc=jgg@ziepe.ca \
    --cc=joro@8bytes.org \
    --cc=joshwash@google.com \
    --cc=kuba@kernel.org \
    --cc=kuniyu@google.com \
    --cc=liam@infradead.org \
    --cc=linux-api@vger.kernel.org \
    --cc=linux-arm-kernel@lists.infradead.org \
    --cc=linux-block@vger.kernel.org \
    --cc=linux-doc@vger.kernel.org \
    --cc=linux-fsdevel@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-mm@kvack.org \
    --cc=ljs@kernel.org \
    --cc=m.szyprowski@samsung.com \
    --cc=mhocko@suse.com \
    --cc=namangulati@google.com \
    --cc=netdev@vger.kernel.org \
    --cc=pabeni@redhat.com \
    --cc=praan@google.com \
    --cc=rafael@kernel.org \
    --cc=rdunlap@infradead.org \
    --cc=rientjes@google.com \
    --cc=rizzo.unipi@gmail.com \
    --cc=robin.murphy@arm.com \
    --cc=roman.gushchin@linux.dev \
    --cc=rppt@kernel.org \
    --cc=smostafa@google.com \
    --cc=surenb@google.com \
    --cc=vbabka@kernel.org \
    --cc=will@kernel.org \
    --cc=willemb@google.com \
    --cc=willy@infradead.org \
    --cc=ziy@nvidia.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.