From: Luigi Rizzo <lrizzo@google.com>
To: Andrew Morton <akpm@linux-foundation.org>,
David Hildenbrand <david@kernel.org>,
Johannes Weiner <hannes@cmpxchg.org>,
Vlastimil Babka <vbabka@kernel.org>,
Joerg Roedel <joro@8bytes.org>, Will Deacon <will@kernel.org>,
Robin Murphy <robin.murphy@arm.com>,
Marek Szyprowski <m.szyprowski@samsung.com>,
Jens Axboe <axboe@kernel.dk>,
"David S . Miller" <davem@davemloft.net>,
Eric Dumazet <edumazet@kernel.org>,
Jakub Kicinski <kuba@kernel.org>,
Paolo Abeni <pabeni@redhat.com>,
Luigi Rizzo <rizzo.unipi@gmail.com>
Cc: Lorenzo Stoakes <ljs@kernel.org>,
"Liam R . Howlett" <liam@infradead.org>,
Mike Rapoport <rppt@kernel.org>,
Suren Baghdasaryan <surenb@google.com>,
Michal Hocko <mhocko@suse.com>,
Brendan Jackman <brendan.jackman@linux.dev>,
Zi Yan <ziy@nvidia.com>, Harry Yoo <harry@kernel.org>,
Christoph Lameter <cl@gentwo.org>,
David Rientjes <rientjes@google.com>,
Roman Gushchin <roman.gushchin@linux.dev>,
Hao Li <hao.li@linux.dev>, Matthew Wilcox <willy@infradead.org>,
Jan Kara <jack@suse.cz>, Jann Horn <jannh@google.com>,
Catalin Marinas <catalin.marinas@arm.com>,
Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
"Rafael J . Wysocki" <rafael@kernel.org>,
Danilo Krummrich <dakr@kernel.org>,
Jonathan Corbet <corbet@lwn.net>,
Simon Horman <horms@kernel.org>,
Jesper Dangaard Brouer <hawk@kernel.org>,
Ilias Apalodimas <ilias.apalodimas@linaro.org>,
Willem de Bruijn <willemb@google.com>,
Kuniyuki Iwashima <kuniyu@google.com>,
Joshua Washington <joshwash@google.com>,
Harshitha Ramamurthy <hramamurthy@google.com>,
Alexander Lobakin <aleksander.lobakin@intel.com>,
Christoph Hellwig <hch@lst.de>, Jason Gunthorpe <jgg@ziepe.ca>,
Dragos Tatulea <dtatulea@nvidia.com>,
Naman Gulati <namangulati@google.com>,
Pranjal Shrivastava <praan@google.com>,
Mostafa Saleh <smostafa@google.com>,
Randy Dunlap <rdunlap@infradead.org>,
linux-mm@kvack.org, iommu@lists.linux.dev,
netdev@vger.kernel.org, linux-block@vger.kernel.org,
linux-fsdevel@vger.kernel.org, driver-core@lists.linux.dev,
linux-arm-kernel@lists.infradead.org, linux-api@vger.kernel.org,
linux-doc@vger.kernel.org, linux-kernel@vger.kernel.org,
Luigi Rizzo <lrizzo@google.com>
Subject: [RFC PATCH v3 08/25] iommu/dma: Use PISA for coherent DMA allocations
Date: Fri, 20 Nov 2026 01:06:38 +0000 [thread overview]
Message-ID: <20261120010655.467825-9-lrizzo@google.com> (raw)
In-Reply-To: <20261120010655.467825-1-lrizzo@google.com>
Coherent DMA allocations (e.g. TX/RX descriptor rings allocated via
dma_alloc_attrs() / dma_alloc_coherent()) are long-lived and accessed
on every packet or I/O operation, so they benefit from sharing 2MB IOMMU
leaf mappings.
When dev->pisa_coherent is enabled, serve coherent DMA allocations
(dma_alloc_attrs()) up to 2MB with contiguous memory from PISA pages.
Signed-off-by: Luigi Rizzo <lrizzo@google.com>
---
drivers/iommu/dma-iommu.c | 6 ++
drivers/iommu/pisa-map.c | 16 +++
include/linux/pisa-iommu.h | 7 ++
include/linux/pisa.h | 38 +++++++
kernel/dma/direct.c | 7 ++
mm/pisa-alloc.c | 205 +++++++++++++++++++++++++++++++++++++
mm/pisa-kunit.c | 71 +++++++++++++
mm/pisa-priv.h | 3 +
8 files changed, 353 insertions(+)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 9b26d0aee9a97..45e9432d2a020 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -1653,6 +1653,8 @@ static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr)
dma_common_free_remap(cpu_addr, alloc_size);
} else {
/* Lowmem means a coherent atomic or CMA allocation */
+ if (pisa_free_coherent(cpu_addr, alloc_size))
+ return;
page = virt_to_page(cpu_addr);
}
@@ -1723,6 +1725,10 @@ void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
gfp |= __GFP_ZERO;
+ cpu_addr = pisa_alloc_map_coherent(dev, size, handle, gfp, attrs);
+ if (cpu_addr)
+ return cpu_addr;
+
if (gfpflags_allow_blocking(gfp) &&
!(attrs & DMA_ATTR_FORCE_CONTIGUOUS)) {
return iommu_dma_alloc_remap(dev, size, handle, gfp, attrs);
diff --git a/drivers/iommu/pisa-map.c b/drivers/iommu/pisa-map.c
index 3ff9f1a6c5b85..bbf330877f824 100644
--- a/drivers/iommu/pisa-map.c
+++ b/drivers/iommu/pisa-map.c
@@ -10,6 +10,7 @@
#include <linux/dma-map-ops.h>
#include <linux/dma-mapping.h>
#include <linux/iommu.h>
+#include <linux/iommu-dma.h>
#include <linux/iova.h>
#include <linux/log2.h>
#include <linux/mm.h>
@@ -18,6 +19,8 @@
#include <linux/spinlock.h>
#include <linux/srcu.h>
+#include "iommu-priv.h"
+
/*
* Registry of PISA IOMMU domains. Each slot is FREE (!domain && !base),
@@ -191,6 +194,19 @@ void __pisa_domain_quiesce(struct iommu_domain *domain)
done, domain, idx);
}
+/*
+ * Set *@domain to the DMA domain of @dev, or to NULL if @dev does not use
+ * dma-iommu. Returns an error if a deferred attach of @dev fails.
+ */
+int pisa_dev_dma_domain(struct device *dev, struct iommu_domain **domain)
+{
+ *domain = NULL;
+ if (!use_dma_iommu(dev))
+ return 0;
+ *domain = iommu_get_dma_domain(dev);
+ return iommu_deferred_attach(dev, *domain);
+}
+
/* Return negative errno if @pi's domain was permanently/currently declined, or 0. */
int pisa_iova_declined(const struct pisa_iova *pi)
{
diff --git a/include/linux/pisa-iommu.h b/include/linux/pisa-iommu.h
index d4c01207dca72..568108747f575 100644
--- a/include/linux/pisa-iommu.h
+++ b/include/linux/pisa-iommu.h
@@ -96,6 +96,7 @@ static inline dma_addr_t pisa_phys_to_iova(const struct pisa_iova *pi,
extern unsigned int pisa_domains_generation;
void pisa_unmap_all_domains(struct pisa_meta *meta);
+int pisa_dev_dma_domain(struct device *dev, struct iommu_domain **domain);
int pisa_iova_declined(const struct pisa_iova *pi);
#else /* !CONFIG_PISA || !CONFIG_IOMMU_DMA */
@@ -103,5 +104,11 @@ int pisa_iova_declined(const struct pisa_iova *pi);
static inline void pisa_unmap_all_domains(struct pisa_meta *meta) {}
static inline int pisa_iova_declined(const struct pisa_iova *pi) { return -EOPNOTSUPP; }
+static inline int pisa_dev_dma_domain(struct device *dev, struct iommu_domain **domain)
+{
+ *domain = NULL;
+ return 0;
+}
+
#endif /* CONFIG_PISA && CONFIG_IOMMU_DMA */
#endif /* _LINUX_PISA_IOMMU_H */
diff --git a/include/linux/pisa.h b/include/linux/pisa.h
index 5222cb207257f..edd34771ad1c4 100644
--- a/include/linux/pisa.h
+++ b/include/linux/pisa.h
@@ -84,6 +84,9 @@ static inline u64 pisa_iova_size(void)
}
int pisa_meta_init(void);
+void *__pisa_alloc_map_coherent_dev(struct device *dev, size_t size,
+ dma_addr_t *dma_handle, gfp_t gfp,
+ unsigned long attrs);
bool __pisa_has_free_blocks_rx(unsigned int order, int nid);
bool __pisa_page_can_recycle(const struct page *page);
@@ -143,6 +146,29 @@ static inline struct page *pisa_virt_to_page(const void *addr)
return is_pisa_page(page) ? page : NULL;
}
+/* Hook for iommu_dma_alloc() and dma_direct_alloc() (guarded by @pisa_key). */
+static inline void *pisa_alloc_map_coherent(struct device *dev, size_t size,
+ dma_addr_t *dma_handle, gfp_t gfp,
+ unsigned long attrs)
+{
+ if (!pisa_enabled() || !READ_ONCE(dev->pisa_coherent))
+ return NULL;
+ return __pisa_alloc_map_coherent_dev(dev, size, dma_handle, gfp, attrs);
+}
+
+/* Hook for __iommu_dma_free() and dma_direct_free() (guarded by @pisa_key). */
+static inline bool pisa_free_coherent(const void *cpu_addr, size_t size)
+{
+ unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
+ struct page *page = pisa_virt_to_page(cpu_addr);
+
+ if (!page)
+ return false;
+ while (count--)
+ __free_page(page++);
+ return true;
+}
+
/* Hook for __page_pool_alloc_netmems_slow() (guarded by @pisa_key). */
static inline gfp_t pisa_rx_gfp(const struct device *dev)
{
@@ -174,6 +200,18 @@ static inline bool is_pisa_page(const struct page *page) { return false; }
static inline bool is_pisa_phys(phys_addr_t phys) { return false; }
static inline struct page *pisa_virt_to_page(const void *addr) { return NULL; }
+static inline void *pisa_alloc_map_coherent(struct device *dev, size_t size,
+ dma_addr_t *dma_handle, gfp_t gfp,
+ unsigned long attrs)
+{
+ return NULL;
+}
+
+static inline bool pisa_free_coherent(const void *cpu_addr, size_t size)
+{
+ return false;
+}
+
static inline gfp_t pisa_rx_gfp(const struct device *dev)
{
return 0;
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 25e9a86835fd3..9ac4912e4a0a1 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -15,6 +15,7 @@
#include <linux/slab.h>
#include <linux/pci-p2pdma.h>
#include <linux/cc_platform.h>
+#include <linux/pisa.h>
#include "direct.h"
@@ -244,6 +245,10 @@ void *dma_direct_alloc(struct device *dev, size_t size,
if (attrs & DMA_ATTR_NO_WARN)
gfp |= __GFP_NOWARN;
+ cpu_addr = pisa_alloc_map_coherent(dev, size, dma_handle, gfp, attrs);
+ if (cpu_addr)
+ return cpu_addr;
+
if (dma_direct_use_no_mapping(dev, attrs))
return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp);
@@ -424,6 +429,8 @@ void dma_direct_free(struct device *dev, size_t size,
if (is_vmalloc_addr(cpu_addr)) {
vunmap(cpu_addr);
+ } else if (pisa_free_coherent(cpu_addr, size)) {
+ return;
} else {
if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED))
arch_dma_clear_uncached(cpu_addr, size);
diff --git a/mm/pisa-alloc.c b/mm/pisa-alloc.c
index ff8565c3206ad..eda12d475c9a7 100644
--- a/mm/pisa-alloc.c
+++ b/mm/pisa-alloc.c
@@ -10,11 +10,14 @@
#include <linux/bitops.h>
#include <linux/cache.h>
#include <linux/device.h>
+#include <linux/dma-direct.h>
+#include <linux/dma-map-ops.h>
#include <linux/dma-mapping.h>
#include <linux/pisa.h>
#include <linux/export.h>
#include <linux/gfp.h>
#include <linux/iommu.h>
+#include <linux/iommu-dma.h>
#include <linux/kasan.h>
#include <linux/list.h>
#include <linux/llist.h>
@@ -25,6 +28,7 @@
#include <linux/page_owner.h>
#include <linux/pgalloc_tag.h>
#include <linux/page_table_check.h>
+#include <linux/pci.h>
#include <linux/rcupdate.h>
#include <linux/sched/mm.h>
#include <linux/shrinker.h>
@@ -673,6 +677,207 @@ unsigned long pisa_alloc_bulk(struct pisa_alloc *pa, gfp_t gfp,
return i;
}
+/* Find and reserve @nblocks aligned contiguous blocks in @meta under @pa->lock. */
+static struct page *pisa_take_contig(struct pisa_meta *meta, unsigned int nblocks,
+ unsigned long align_mask, bool *clean)
+{
+ DECLARE_BITMAP(busy, _PISA_MAX_BLOCKS);
+ struct pisa_alloc *pa = meta->pa;
+ unsigned int bpp = PISA_BLOCKS(0), dirty = 0;
+ bool poisoned = false;
+ unsigned long off;
+ struct page *page;
+
+ if (pisa_meta_avail(meta) < nblocks)
+ return NULL;
+
+ bitmap_or(busy, meta->clean_bitmap, meta->dirty_bitmap, bpp);
+ bitmap_complement(busy, busy, bpp);
+
+ off = 0;
+ for (;;) {
+ unsigned int i;
+
+ off = bitmap_find_next_zero_area(busy, bpp, off, nblocks, align_mask);
+ if (off >= bpp) {
+ if (poisoned) {
+ if (meta->on_list & PISA_LIST_IDLE) {
+ pa->num_idle_pages--;
+ WRITE_ONCE(meta->alloc_jiffies, jiffies);
+ }
+ pisa_place_meta(meta);
+ }
+ return NULL;
+ }
+
+ page = pisa_meta_to_page(meta) + off;
+ for (i = 0; i < nblocks; i++) {
+ if (!is_pisa_block_poisoned(pa, page + i))
+ continue;
+ if (__test_and_clear_bit(off + i, meta->clean_bitmap)) {
+ meta->nr_clean--;
+ pa->blocks_in_use++;
+ } else if (__test_and_clear_bit(off + i, meta->dirty_bitmap)) {
+ meta->nr_dirty--;
+ pa->blocks_in_use++;
+ }
+ __set_bit(off + i, busy);
+ poisoned = true;
+ off += i + 1;
+ break;
+ }
+ if (i == nblocks)
+ break;
+ }
+
+ if (meta->on_list & PISA_LIST_IDLE) {
+ pa->num_idle_pages--;
+ WRITE_ONCE(meta->alloc_jiffies, jiffies);
+ }
+
+ for (unsigned int i = 0; i < nblocks; i++) {
+ if (__test_and_clear_bit(off + i, meta->clean_bitmap)) {
+ meta->nr_clean--;
+ } else {
+ __clear_bit(off + i, meta->dirty_bitmap);
+ meta->nr_dirty--;
+ dirty++;
+ }
+ pisa_block_prep(page + i);
+ page_ref_unfreeze(page + i, 1);
+ }
+ pa->blocks_in_use += nblocks;
+ *clean = !dirty;
+ pisa_place_meta(meta);
+ return page;
+}
+
+/* The device checks of pisa_domain_init() and __pisa_map_phys(), before allocating. */
+static bool pisa_coherent_dev_fits(struct device *dev, const struct pisa_iova *pi)
+{
+ u64 limit = min_not_zero(dma_get_mask(dev), dev->bus_dma_limit);
+ /* Pairs with smp_store_release() in pisa_domain_init(). */
+ dma_addr_t base = smp_load_acquire(&pi->base);
+
+ limit = min_not_zero(limit, dev->coherent_dma_mask);
+ if (limit <= DMA_BIT_MASK(32))
+ return false;
+ return base ? base + pisa_iova_size() - 1 <= limit :
+ pisa_iova_size() <= limit - PISA_SIZE;
+}
+
+/*
+ * Specialized memory provider for iommu_dma_alloc() and dma_direct_alloc():
+ * allocate a zeroed, naturally aligned contiguous range of order-0 pages for
+ * @size bytes (<= PISA_SIZE) from @dev's per-node coherent allocator, map it
+ * into *@dma_handle, and return its kernel virtual address, or NULL.
+ */
+VISIBLE_IF_KUNIT void *
+__pisa_alloc_map_coherent(struct device *dev, struct iommu_domain *domain,
+ size_t size, dma_addr_t *dma_handle, gfp_t gfp,
+ unsigned long attrs)
+{
+ bool should_release = false, clean = false;
+ struct list_head *lists[3];
+ unsigned long align_mask, flags;
+ struct pisa_meta *meta, *tmp;
+ unsigned int nblocks;
+ struct pisa_alloc *pa;
+ struct page *page = NULL;
+
+ size = PAGE_ALIGN(size);
+ if (!dev_is_dma_coherent(dev) || !size || size > PISA_SIZE)
+ return NULL;
+ /* As in pisa_domain_init(): no shared PISA pages for untrusted devices. */
+ if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted)
+ return NULL;
+ if (min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit) <= DMA_BIT_MASK(32))
+ return NULL;
+ if ((current_gfp_context(gfp) & ~(__GFP_ZERO | __GFP_NOWARN)) != GFP_KERNEL ||
+ (attrs & ~(DMA_ATTR_NO_WARN | __DMA_ATTR_ALLOC_CC_SHARED)))
+ return NULL;
+
+ if (domain) {
+ struct pisa_iova *pi = pisa_dma_iova(domain);
+
+ if (!pi || pisa_iova_declined(pi) || !pisa_domain_pgsize_ok(domain) ||
+ !pisa_coherent_dev_fits(dev, pi))
+ return NULL;
+ } else if (is_swiotlb_for_alloc(dev) ||
+ force_dma_unencrypted(dev) ||
+ !dma_coherent_ok(dev, PFN_PHYS(max_pfn) - 1, 1)) {
+ return NULL;
+ }
+
+ pa = pisa_node_pa(dev_to_node(dev), 0, false);
+ if (!pa)
+ return NULL;
+
+ nblocks = size >> PAGE_SHIFT;
+ align_mask = (1UL << get_order(size)) - 1;
+ lists[0] = &pa->partial;
+ lists[1] = &pa->partial_dirty;
+ lists[2] = &pa->idle;
+
+ spin_lock_irqsave(&pa->lock, flags);
+ for (unsigned int i = 0; i < ARRAY_SIZE(lists) && !page; i++) {
+ list_for_each_entry_safe(meta, tmp, lists[i], list) {
+ page = pisa_take_contig(meta, nblocks, align_mask, &clean);
+ if (page)
+ break;
+ }
+ }
+ if (!page) {
+ spin_unlock_irqrestore(&pa->lock, flags);
+ meta = pisa_alloc_split_page(0, gfp, pa->nid);
+ if (IS_ERR(meta))
+ return NULL;
+ spin_lock_irqsave(&pa->lock, flags);
+ meta->pa = pa;
+ meta->prot = pa->prot;
+ WRITE_ONCE(meta->alloc_jiffies, jiffies);
+ pisa_meta_set_flags(meta, PISA_ACTIVE);
+ pa->stats.page_alloc++;
+ page = pisa_take_contig(meta, nblocks, align_mask, &clean);
+ if (unlikely(!page))
+ should_release = pisa_place_meta(meta);
+ }
+ spin_unlock_irqrestore(&pa->lock, flags);
+
+ if (should_release)
+ pisa_schedule_reclaim();
+ if (!page)
+ return NULL;
+ if (!domain && unlikely(!dma_coherent_ok(dev, page_to_phys(page), size))) {
+ /* E.g. a hole in dev->dma_range_map, or RAM hotplugged above max_pfn. */
+ pisa_free_coherent(page_address(page), size);
+ return NULL;
+ }
+ if (domain) {
+ if (!pisa_map_phys(dma_handle, dev, domain, page_to_phys(page),
+ size, IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE)) {
+ pisa_free_coherent(page_address(page), size);
+ return NULL;
+ }
+ } else {
+ *dma_handle = phys_to_dma(dev, page_to_phys(page));
+ }
+ if (!clean)
+ memset(page_address(page), 0, size);
+ return page_address(page);
+}
+
+void *__pisa_alloc_map_coherent_dev(struct device *dev, size_t size,
+ dma_addr_t *dma_handle, gfp_t gfp,
+ unsigned long attrs)
+{
+ struct iommu_domain *domain;
+
+ if (pisa_dev_dma_domain(dev, &domain))
+ return NULL;
+ return __pisa_alloc_map_coherent(dev, domain, size, dma_handle, gfp, attrs);
+}
+
/**
* __pisa_free_page - Recycle a freed block back into its PISA allocator
* @page: Block being freed (called from __free_pages_prepare() under RCU)
diff --git a/mm/pisa-kunit.c b/mm/pisa-kunit.c
index 2adaeb1ffe43e..1040786330cef 100644
--- a/mm/pisa-kunit.c
+++ b/mm/pisa-kunit.c
@@ -13,6 +13,7 @@
#include <linux/mm.h>
#include <linux/shrinker.h>
#include <linux/skbuff.h>
+#include <linux/swiotlb.h>
#include <net/sock.h>
#include "pisa-priv.h"
@@ -445,6 +446,75 @@ static void test_map_and_domain_lifecycle(struct kunit *test)
}
#endif /* CONFIG_IOMMU_DMA */
+static void test_coherent_alloc_and_free(struct kunit *test)
+{
+ struct device dev = { .init_name = "pisa_kunit", .numa_node = NUMA_NO_NODE,
+ .coherent_dma_mask = DMA_BIT_MASK(64),
+ .pisa_coherent = true };
+ dma_addr_t d1, d2, d3;
+ void *v1, *v2, *v3 = NULL;
+#ifdef CONFIG_IOMMU_DMA
+ struct pisa_iova *pi;
+#endif
+
+ dev.dma_mask = &dev.coherent_dma_mask;
+ /* NUMA_NO_NODE would follow the node of the CPU the test runs on. */
+ set_dev_node(&dev, first_memory_node);
+ /* is_swiotlb_for_alloc() dereferences dev->dma_io_tlb_mem. */
+ swiotlb_dev_init(&dev);
+ dev_set_dma_coherent(&dev);
+ KUNIT_ASSERT_EQ(test, pisa_meta_init(), 0);
+ KUNIT_EXPECT_NULL(test, pisa_alloc_map_coherent(&dev, SZ_4K, &d1,
+ GFP_KERNEL | __GFP_ACCOUNT, 0));
+ KUNIT_EXPECT_NULL(test, pisa_alloc_map_coherent(&dev, PISA_SIZE + SZ_4K, &d1,
+ GFP_KERNEL, 0));
+
+#ifdef CONFIG_IOMMU_DMA
+ /* Uninitialized domain has no DMA cookie/window yet. */
+ KUNIT_EXPECT_NULL(test, __pisa_alloc_map_coherent(&dev, &fake_domain, SZ_4K,
+ &d1, GFP_KERNEL | __GFP_NOWARN, 0));
+
+ KUNIT_ASSERT_EQ(test, fake_domain_setup(test), 0);
+ pi = pisa_dma_iova(&fake_domain);
+
+ v1 = __pisa_alloc_map_coherent(&dev, &fake_domain, SZ_64K, &d1, GFP_KERNEL, 0);
+ KUNIT_ASSERT_NOT_NULL(test, v1);
+ KUNIT_EXPECT_TRUE(test, is_pisa_iova(pi, d1));
+
+ v2 = __pisa_alloc_map_coherent(&dev, &fake_domain, SZ_64K, &d2, GFP_KERNEL, 0);
+ KUNIT_EXPECT_NOT_NULL(test, v2);
+ if (v2) {
+ KUNIT_EXPECT_PTR_EQ(test, v2, v1 + SZ_64K);
+ KUNIT_EXPECT_EQ(test, d2, d1 + SZ_64K);
+ }
+
+ KUNIT_EXPECT_TRUE(test, pisa_free_coherent(v1, SZ_64K));
+ v3 = __pisa_alloc_map_coherent(&dev, &fake_domain, SZ_64K, &d3, GFP_KERNEL, 0);
+ KUNIT_EXPECT_NOT_NULL(test, v3);
+ if (v3)
+ KUNIT_EXPECT_PTR_EQ(test, v3, v1);
+#else
+ v1 = pisa_alloc_map_coherent(&dev, SZ_64K, &d1, GFP_KERNEL, 0);
+ KUNIT_ASSERT_NOT_NULL(test, v1);
+ v2 = pisa_alloc_map_coherent(&dev, SZ_64K, &d2, GFP_KERNEL, 0);
+ KUNIT_EXPECT_NOT_NULL(test, v2);
+ if (v2)
+ KUNIT_EXPECT_PTR_EQ(test, v2, v1 + SZ_64K);
+
+ /* Freeing v1 returns its 16 order-0 pages; next 64K alloc reuses them. */
+ KUNIT_EXPECT_TRUE(test, pisa_free_coherent(v1, SZ_64K));
+ v3 = pisa_alloc_map_coherent(&dev, SZ_64K, &d3, GFP_KERNEL, 0);
+ KUNIT_EXPECT_NOT_NULL(test, v3);
+ if (v3)
+ KUNIT_EXPECT_PTR_EQ(test, v3, v1);
+#endif
+
+ if (v2)
+ KUNIT_EXPECT_TRUE(test, pisa_free_coherent(v2, SZ_64K));
+ if (v3)
+ KUNIT_EXPECT_TRUE(test, pisa_free_coherent(v3, SZ_64K));
+}
+
static void kunit_check_pisa_attr(struct kunit *test, struct device *dev,
const char *name, bool *field)
{
@@ -529,6 +599,7 @@ static struct kunit_case pisa_meta_test_cases[] = {
KUNIT_CASE(test_iova_helpers),
KUNIT_CASE(test_map_and_domain_lifecycle),
#endif
+ KUNIT_CASE(test_coherent_alloc_and_free),
KUNIT_CASE(test_dev_pisa_attrs),
KUNIT_CASE(test_reservoir_and_scrubber),
{}
diff --git a/mm/pisa-priv.h b/mm/pisa-priv.h
index 8736b8d168727..fb26eb3aa9b37 100644
--- a/mm/pisa-priv.h
+++ b/mm/pisa-priv.h
@@ -165,6 +165,9 @@ struct shrink_control;
struct shrinker;
#if IS_ENABLED(CONFIG_KUNIT)
unsigned long pisa_shrink_scan(struct shrinker *shrink, struct shrink_control *sc);
+void *__pisa_alloc_map_coherent(struct device *dev, struct iommu_domain *domain,
+ size_t size, dma_addr_t *dma_handle, gfp_t gfp,
+ unsigned long attrs);
#endif
/* Global PISA page budget across all allocators and per-node reservoirs. */
--
2.56.1.809.ga0b61035b5-goog
next prev parent reply other threads:[~2026-11-20 1:07 UTC|newest]
Thread overview: 144+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-10-03 21:22 [RFC: DMA_PMD 00/22] DMA_PMD: PMD_SIZE-backed IO buffer pools Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 01/22] iommu/dma: introduce CONFIG_DMA_PMD and metadata table Luigi Rizzo
2026-10-03 21:44 ` Randy Dunlap
2026-10-04 9:14 ` Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 02/22] iommu/dma: add DMA_PMD pool lifecycle and page recycle hook Luigi Rizzo
2026-10-10 2:59 ` kernel test robot
2026-10-03 21:22 ` [RFC: DMA_PMD 03/22] mm: Add split_page_compound() Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 04/22] iommu/dma: add DMA_PMD pool block allocation Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 05/22] iommu/dma: Global cap and shrinker for DMA_PMD pool memory Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 06/22] iommu/dma: reserve a per-domain IOVA window for DMA_PMD pages Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 07/22] iommu/dma: release DMA_PMD domain mappings on domain teardown Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 08/22] iommu/dma: use per-domain IOVA window to map DMA_PMD memory Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 09/22] iommu/dma: Add DMA_PMD arena allocator Luigi Rizzo
2026-10-08 11:25 ` kernel test robot
2026-10-03 21:22 ` [RFC: DMA_PMD 10/22] driver core: Add per-device dma_pmd_* sysfs attributes Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 11/22] dma-mapping: Use DMA_PMD arena for dma_alloc_attrs() Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 12/22] net/core: Use per-CPU DMA_PMD pools for skb_page_frag_refill() Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 13/22] net/core: Use DMA_PMD for page_pool memory Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 14/22] iommu/dma: Support decrypted and pinned DMA_PMD pages Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 15/22] iommu/dma: Add background page scrubber for DMA_PMD pools Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 16/22] iommu/dma: Add per-NUMA-node PMD page reservoir Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 17/22] net/gve: Use DMA_PMD memory for RX buffers Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 18/22] net/gve: Use DMA_PMD memory for tx header bounce buffers Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 19/22] net/mlx5e: " Luigi Rizzo
2026-10-14 17:22 ` Dragos Tatulea
2026-10-14 17:50 ` Dragos Tatulea
2026-10-14 18:28 ` Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 20/22] net/idpf: " Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 21/22] net/bnxt: " Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 22/22] iommu/dma: Add DMA_PMD statistics and debugfs Luigi Rizzo
2026-10-11 9:02 ` [RFC: DMA_PMD 00/22] DMA_PMD: PMD_SIZE-backed IO buffer pools Pranjal Shrivastava
2026-10-12 13:51 ` David Hildenbrand (Arm)
2026-10-12 14:45 ` Luigi Rizzo
2026-10-12 19:43 ` David Hildenbrand (Arm)
2026-10-13 10:42 ` Robin Murphy
2026-10-14 10:47 ` David Hildenbrand (Arm)
2026-10-14 11:05 ` Luigi Rizzo
2026-10-17 17:11 ` Luigi Rizzo
2026-10-22 20:12 ` Jason Gunthorpe
2026-10-22 21:51 ` Luigi Rizzo
2026-10-23 14:01 ` Jason Gunthorpe
2026-10-23 14:30 ` Luigi Rizzo
2026-10-23 14:50 ` Jason Gunthorpe
2026-10-23 15:12 ` Luigi Rizzo
2026-10-23 17:27 ` Jakub Kicinski
2026-10-23 20:22 ` Luigi Rizzo
2026-11-03 8:09 ` Christoph Hellwig
2026-11-03 9:13 ` Luigi Rizzo
2026-11-03 15:10 ` Jason Gunthorpe
2026-11-06 17:21 ` Naman Gulati
2026-11-06 17:30 ` Jason Gunthorpe
2026-11-06 18:00 ` Naman Gulati
2026-11-06 18:07 ` Jason Gunthorpe
2026-11-06 18:17 ` Naman Gulati
2026-11-06 18:43 ` Jason Gunthorpe
2026-11-06 19:01 ` Luigi Rizzo
2026-11-06 19:11 ` Naman Gulati
2026-10-12 14:49 ` Mostafa Saleh
2026-10-12 15:02 ` Luigi Rizzo
2026-10-14 17:28 ` Dragos Tatulea
2026-10-14 18:14 ` Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 00/25] PISA: Pre-mapped IOVA Superpage Allocator for DMA and CoCo Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 " Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 01/25] mm: Introduce CONFIG_PISA and metadata table Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 02/25] mm: Add PISA allocator lifecycle and page recycle hook Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 03/25] mm: Add per-NUMA-node PISA page reservoir Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 04/25] mm/page_alloc: Extend alloc_pages() to __GFP_PISA and PISA block allocation Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 05/25] mm: Add global cap and shrinker for PISA pages Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 06/25] iommu/dma: Map PISA memory in a per-domain IOVA window Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 07/25] driver core: Add per-device pisa_* sysfs attributes for PISA optin Luigi Rizzo
2026-11-20 1:06 ` Luigi Rizzo [this message]
2026-11-20 1:06 ` [RFC PATCH v3 09/25] mm: Add per-CPU block caches to PISA allocators Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 10/25] mm: Add background page scrubber for " Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 11/25] mm: Migrate user pages into PISA Luigi Rizzo
2026-11-20 14:10 ` Lorenzo Stoakes (ARM)
2026-11-20 1:06 ` [RFC PATCH v3 12/25] block, mm: Support PISA for disk I/O and scrub migration Luigi Rizzo
2026-11-20 16:17 ` Christoph Hellwig
2026-11-20 1:06 ` [RFC PATCH v3 13/25] net: Cache leaf PISA TX flags on struct net_device Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 14/25] net: Use per-node PISA allocators for skb_page_frag_refill() Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 15/25] net: Use PISA for page_pool memory Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 16/25] mm/slub: Add SLAB_CACHE_PISA_RO for PISA-backed slab caches Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 17/25] net: Allocate small skb heads from PISA Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 18/25] net: Migrate user pages into PISA for MSG_ZEROCOPY Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 19/25] mm, iommu/dma: Support decrypted PISA pages Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 20/25] arm64/mm, mm: Support decrypted PISA pages in Realms Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 21/25] mm: Add PISA statistics and debugfs Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 22/25] net/libeth: Use GFP_KERNEL when pre-filling RX rings Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 23/25] net/gve: " Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 24/25] net/gve: Use PISA memory for RX buffers Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 25/25] Documentation: Add PISA architecture documentation Luigi Rizzo
2026-11-20 13:50 ` [RFC PATCH v3 00/25] PISA: Pre-mapped IOVA Superpage Allocator for DMA and CoCo Jason Gunthorpe
2026-11-20 14:18 ` Lorenzo Stoakes (ARM)
2026-11-20 14:19 ` Luigi Rizzo
2026-11-20 14:36 ` Lorenzo Stoakes (ARM)
2026-11-20 14:54 ` Lorenzo Stoakes (ARM)
2026-11-20 16:12 ` David Hildenbrand (Arm)
2026-11-20 18:16 ` Lorenzo Stoakes (ARM)
2026-11-20 15:00 ` [RFC PATCH v2 " Matthew Wilcox
2026-11-09 18:25 ` [RFC PATCH v2 01/25] iommu/dma: Introduce CONFIG_PISA and metadata table Luigi Rizzo
2026-11-10 23:12 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 02/25] iommu/dma: Add PISA allocator lifecycle and page recycle hook Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 03/25] iommu/dma: Add per-NUMA-node PISA page reservoir Luigi Rizzo
2026-11-10 23:12 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 04/25] iommu/dma: Add PISA block allocation Luigi Rizzo
2026-11-10 23:12 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 05/25] iommu/dma: Add global cap and shrinker for PISA pages Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 06/25] iommu/dma: Reserve a per-domain IOVA window " Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 07/25] iommu/dma: Release PISA domain mappings on domain teardown Luigi Rizzo
2026-11-10 23:12 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 08/25] iommu/dma: Use per-domain IOVA window to map PISA memory Luigi Rizzo
2026-11-10 23:12 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 09/25] driver core: Add per-device pisa_* sysfs attributes for PISA optin Luigi Rizzo
2026-11-10 23:12 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 10/25] iommu/dma: Use PISA for coherent DMA allocations Luigi Rizzo
2026-11-10 23:12 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 11/25] iommu/dma: Add per-CPU block caches to PISA allocators Luigi Rizzo
2026-11-10 23:12 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 12/25] net/core: Use per-node PISA allocators for skb_page_frag_refill() Luigi Rizzo
2026-11-10 23:50 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 13/25] net/core: Use PISA for page_pool memory Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 14/25] iommu/dma: Support decrypted PISA pages Luigi Rizzo
2026-11-11 0:19 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 15/25] iommu/dma: Add background page scrubber for PISA allocators Luigi Rizzo
2026-11-11 0:37 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 16/25] mm/slub: Add SLAB_CACHE_PISA_RO for PISA-backed slab caches Luigi Rizzo
2026-11-11 1:03 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 17/25] net: Allocate small skb heads from PISA Luigi Rizzo
2026-11-11 1:33 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 18/25] iommu/dma: Migrate user pages into PISA for TX zerocopy Luigi Rizzo
2026-11-11 2:41 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 19/25] iommu/dma: Stop recycling old PISA pages when allocator waste is high Luigi Rizzo
2026-11-11 2:57 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 20/25] net/libeth: Use GFP_KERNEL when pre-filling RX rings Luigi Rizzo
2026-11-11 3:13 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 21/25] net/gve: " Luigi Rizzo
2026-11-11 3:22 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 22/25] net/gve: Use PISA memory for RX buffers Luigi Rizzo
2026-11-11 3:43 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 23/25] block, mm, iommu/dma: Support PISA for disk I/O and scrub migration Luigi Rizzo
2026-11-11 4:16 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 24/25] Documentation: Add PISA architecture documentation Luigi Rizzo
2026-11-11 4:24 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 25/25] iommu/dma: Add PISA statistics and debugfs Luigi Rizzo
2026-11-11 4:37 ` sashiko-bot
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261120010655.467825-9-lrizzo@google.com \
--to=lrizzo@google.com \
--cc=akpm@linux-foundation.org \
--cc=aleksander.lobakin@intel.com \
--cc=axboe@kernel.dk \
--cc=brendan.jackman@linux.dev \
--cc=catalin.marinas@arm.com \
--cc=cl@gentwo.org \
--cc=corbet@lwn.net \
--cc=dakr@kernel.org \
--cc=davem@davemloft.net \
--cc=david@kernel.org \
--cc=driver-core@lists.linux.dev \
--cc=dtatulea@nvidia.com \
--cc=edumazet@kernel.org \
--cc=gregkh@linuxfoundation.org \
--cc=hannes@cmpxchg.org \
--cc=hao.li@linux.dev \
--cc=harry@kernel.org \
--cc=hawk@kernel.org \
--cc=hch@lst.de \
--cc=horms@kernel.org \
--cc=hramamurthy@google.com \
--cc=ilias.apalodimas@linaro.org \
--cc=iommu@lists.linux.dev \
--cc=jack@suse.cz \
--cc=jannh@google.com \
--cc=jgg@ziepe.ca \
--cc=joro@8bytes.org \
--cc=joshwash@google.com \
--cc=kuba@kernel.org \
--cc=kuniyu@google.com \
--cc=liam@infradead.org \
--cc=linux-api@vger.kernel.org \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-block@vger.kernel.org \
--cc=linux-doc@vger.kernel.org \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=ljs@kernel.org \
--cc=m.szyprowski@samsung.com \
--cc=mhocko@suse.com \
--cc=namangulati@google.com \
--cc=netdev@vger.kernel.org \
--cc=pabeni@redhat.com \
--cc=praan@google.com \
--cc=rafael@kernel.org \
--cc=rdunlap@infradead.org \
--cc=rientjes@google.com \
--cc=rizzo.unipi@gmail.com \
--cc=robin.murphy@arm.com \
--cc=roman.gushchin@linux.dev \
--cc=rppt@kernel.org \
--cc=smostafa@google.com \
--cc=surenb@google.com \
--cc=vbabka@kernel.org \
--cc=will@kernel.org \
--cc=willemb@google.com \
--cc=willy@infradead.org \
--cc=ziy@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.