From: Luigi Rizzo <lrizzo@google.com>
To: Andrew Morton <akpm@linux-foundation.org>,
David Hildenbrand <david@kernel.org>,
Johannes Weiner <hannes@cmpxchg.org>,
Vlastimil Babka <vbabka@kernel.org>,
Joerg Roedel <joro@8bytes.org>, Will Deacon <will@kernel.org>,
Robin Murphy <robin.murphy@arm.com>,
Marek Szyprowski <m.szyprowski@samsung.com>,
Jens Axboe <axboe@kernel.dk>,
"David S . Miller" <davem@davemloft.net>,
Eric Dumazet <edumazet@kernel.org>,
Jakub Kicinski <kuba@kernel.org>,
Paolo Abeni <pabeni@redhat.com>,
Luigi Rizzo <rizzo.unipi@gmail.com>
Cc: Lorenzo Stoakes <ljs@kernel.org>,
"Liam R . Howlett" <liam@infradead.org>,
Mike Rapoport <rppt@kernel.org>,
Suren Baghdasaryan <surenb@google.com>,
Michal Hocko <mhocko@suse.com>,
Brendan Jackman <brendan.jackman@linux.dev>,
Zi Yan <ziy@nvidia.com>, Harry Yoo <harry@kernel.org>,
Christoph Lameter <cl@gentwo.org>,
David Rientjes <rientjes@google.com>,
Roman Gushchin <roman.gushchin@linux.dev>,
Hao Li <hao.li@linux.dev>, Matthew Wilcox <willy@infradead.org>,
Jan Kara <jack@suse.cz>, Jann Horn <jannh@google.com>,
Catalin Marinas <catalin.marinas@arm.com>,
Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
"Rafael J . Wysocki" <rafael@kernel.org>,
Danilo Krummrich <dakr@kernel.org>,
Jonathan Corbet <corbet@lwn.net>,
Simon Horman <horms@kernel.org>,
Jesper Dangaard Brouer <hawk@kernel.org>,
Ilias Apalodimas <ilias.apalodimas@linaro.org>,
Willem de Bruijn <willemb@google.com>,
Kuniyuki Iwashima <kuniyu@google.com>,
Joshua Washington <joshwash@google.com>,
Harshitha Ramamurthy <hramamurthy@google.com>,
Alexander Lobakin <aleksander.lobakin@intel.com>,
Christoph Hellwig <hch@lst.de>, Jason Gunthorpe <jgg@ziepe.ca>,
Dragos Tatulea <dtatulea@nvidia.com>,
Naman Gulati <namangulati@google.com>,
Pranjal Shrivastava <praan@google.com>,
Mostafa Saleh <smostafa@google.com>,
Randy Dunlap <rdunlap@infradead.org>,
linux-mm@kvack.org, iommu@lists.linux.dev,
netdev@vger.kernel.org, linux-block@vger.kernel.org,
linux-fsdevel@vger.kernel.org, driver-core@lists.linux.dev,
linux-arm-kernel@lists.infradead.org, linux-api@vger.kernel.org,
linux-doc@vger.kernel.org, linux-kernel@vger.kernel.org,
Luigi Rizzo <lrizzo@google.com>
Subject: [RFC PATCH v3 12/25] block, mm: Support PISA for disk I/O and scrub migration
Date: Fri, 20 Nov 2026 01:06:42 +0000 [thread overview]
Message-ID: <20261120010655.467825-13-lrizzo@google.com> (raw)
In-Reply-To: <20261120010655.467825-1-lrizzo@google.com>
Extend PISA user/page-cache allocators to support O_DIRECT and buffered
disk I/O, gated per block/controller device through pisa_rx_bufs (read)
and pisa_tx_bufs (write):
- Split per-node user_pa into DMA_BIDIRECTIONAL, DMA_TO_DEVICE, and
DMA_FROM_DEVICE allocators, promoting pages to DMA_BIDIRECTIONAL if
later accessed in the opposite direction.
- Upgrade pinned O_DIRECT user pages in bio_iov_iter_get_pages(), and
allocate/promote order-0 page-cache folios for buffered I/O when PISA
is enabled on the target block device.
- Add pisa.scrub_migrate to compact or demote lingering folios off old,
sparse user_pa 2MB pages.
Assisted-by: gemini:2.5-pro
Signed-off-by: Luigi Rizzo <lrizzo@google.com>
---
block/bio.c | 9 +
include/linux/pisa-mm.h | 76 ++++++
include/linux/pisa.h | 7 +-
mm/filemap.c | 12 +-
mm/page-writeback.c | 9 +
mm/pisa-alloc.c | 18 +-
mm/pisa-meta.c | 15 +-
mm/pisa-node.c | 10 +-
mm/pisa-priv.h | 9 +-
mm/pisa-user.c | 523 ++++++++++++++++++++++++++++++++++++----
mm/readahead.c | 3 +-
11 files changed, 635 insertions(+), 56 deletions(-)
diff --git a/block/bio.c b/block/bio.c
index b0fcc69de6535..ad8632661dddf 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -6,6 +6,7 @@
#include <linux/swap.h>
#include <linux/bio-integrity.h>
#include <linux/blkdev.h>
+#include <linux/pisa-mm.h>
#include <linux/uio.h>
#include <linux/iocontext.h>
#include <linux/slab.h>
@@ -1346,6 +1347,7 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
{
iov_iter_extraction_t flags = 0;
struct iov_iter saved_iter;
+ bool migrated = false;
int ret;
if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
@@ -1368,6 +1370,7 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
flags |= ITER_ALLOW_P2PDMA;
saved_iter = *iter;
+retry:
do {
ssize_t size;
@@ -1386,6 +1389,12 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
bio->bi_iter.bi_size += size;
} while (iov_iter_count(iter) && !bio_full(bio, 0));
+ /* Migrate suitable pages to PISA, re-extract if so. */
+ if (!migrated && pisa_bio_migrate_pages(bio, iter)) {
+ migrated = true;
+ goto retry;
+ }
+
if (is_pci_p2pdma_page(bio->bi_io_vec->bv_page))
bio->bi_opf |= REQ_NOMERGE;
ret = bio_iov_iter_align_down(bio, iter, len_align_mask);
diff --git a/include/linux/pisa-mm.h b/include/linux/pisa-mm.h
index 1e82e081b4004..8caa2afded6ff 100644
--- a/include/linux/pisa-mm.h
+++ b/include/linux/pisa-mm.h
@@ -3,14 +3,29 @@
#ifndef _LINUX_PISA_MM_H
#define _LINUX_PISA_MM_H
+#include <linux/bio.h>
#include <linux/pisa.h>
+struct address_space;
+struct folio_batch;
+
#ifdef CONFIG_PISA
unsigned long __pisa_alloc_pages_bulk(gfp_t gfp, unsigned int order,
int preferred_nid, int nr_pages,
struct page **page_array, bool frozen);
bool __pisa_free_page(struct page *page, unsigned int order);
+bool __pisa_bio_migrate_pages(struct bio *bio, struct iov_iter *iter,
+ enum dma_data_direction dir);
+bool pisa_bdev_enabled(const struct block_device *bdev,
+ enum dma_data_direction dir);
+struct folio *__pisa_filemap_alloc(const struct address_space *mapping,
+ gfp_t gfp);
+bool __pisa_filemap_upgrade(const struct address_space *mapping,
+ struct folio *folio);
+bool __pisa_writeback_upgrade(const struct address_space *mapping,
+ struct folio *folio, pgoff_t *index,
+ struct folio_batch *fbatch);
int pisa_madvise(struct vm_area_struct *vma, unsigned long start,
unsigned long end, bool to_pisa);
@@ -57,6 +72,46 @@ static inline bool pisa_free_page(struct page *page, unsigned int order)
return is_pisa_page(page) && __pisa_free_page(page, order);
}
+/*
+ * Hook for bio_iov_iter_get_pages() O_DIRECT migration (@pisa_user_key).
+ * Returns false if @bio and @iter were left untouched; returns true if the
+ * pages were unpinned, migrated to PISA, and @iter rewound for re-extraction.
+ */
+static inline bool pisa_bio_migrate_pages(struct bio *bio, struct iov_iter *iter)
+{
+ enum dma_data_direction dir;
+
+ if (!pisa_user_enabled() || !bio->bi_iter.bi_size ||
+ !bio_flagged(bio, BIO_PAGE_PINNED))
+ return false;
+ dir = op_is_write(bio_op(bio)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+ return pisa_bdev_enabled(bio->bi_bdev, dir) &&
+ __pisa_bio_migrate_pages(bio, iter, dir);
+}
+
+/* Hook for __filemap_get_folio_mpol() and ractl_alloc_folio() (@pisa_user_key). */
+#define pisa_filemap_alloc_folio(gfp, order, policy, mapping) \
+ ((pisa_user_enabled() && !(order) ? \
+ __pisa_filemap_alloc(mapping, gfp) : NULL) ?: \
+ filemap_alloc_folio(gfp, order, policy))
+
+/* Hook for __filemap_get_folio_mpol() existing folio upgrade (@pisa_user_key). */
+static inline bool pisa_filemap_upgrade(const struct address_space *mapping,
+ struct folio *folio, bool write_wait)
+{
+ return pisa_user_enabled() && write_wait &&
+ __pisa_filemap_upgrade(mapping, folio);
+}
+
+/* Hook for writeback_get_folio() upgrade (guarded by @pisa_user_key). */
+static inline bool pisa_writeback_upgrade(const struct address_space *mapping,
+ struct folio *folio, pgoff_t *index,
+ struct folio_batch *fbatch)
+{
+ return pisa_user_enabled() &&
+ __pisa_writeback_upgrade(mapping, folio, index, fbatch);
+}
+
#else /* !CONFIG_PISA */
static inline unsigned long
@@ -79,6 +134,27 @@ static inline bool pisa_free_page(struct page *page, unsigned int order)
return false;
}
+static inline bool pisa_bio_migrate_pages(struct bio *bio, struct iov_iter *iter)
+{
+ return false;
+}
+
+#define pisa_filemap_alloc_folio(gfp, order, policy, mapping) \
+ filemap_alloc_folio(gfp, order, policy)
+
+static inline bool pisa_filemap_upgrade(const struct address_space *mapping,
+ struct folio *folio, bool write_wait)
+{
+ return false;
+}
+
+static inline bool pisa_writeback_upgrade(const struct address_space *mapping,
+ struct folio *folio, pgoff_t *index,
+ struct folio_batch *fbatch)
+{
+ return false;
+}
+
static inline int pisa_madvise(struct vm_area_struct *vma, unsigned long start,
unsigned long end, bool to_pisa)
{
diff --git a/include/linux/pisa.h b/include/linux/pisa.h
index 9321e7ff00042..6d184e393e297 100644
--- a/include/linux/pisa.h
+++ b/include/linux/pisa.h
@@ -93,7 +93,8 @@ void *__pisa_alloc_map_coherent_dev(struct device *dev, size_t size,
bool __pisa_has_free_blocks_rx(unsigned int order, int nid);
bool __pisa_page_can_recycle(const struct page *page);
bool pisa_upgrade_user_pages(struct iov_iter *iter, struct page **pages,
- size_t size, size_t offset);
+ size_t size, size_t offset,
+ enum dma_data_direction dir, bool pinned);
static inline bool pisa_enabled(void)
{
@@ -241,7 +242,9 @@ static inline bool pisa_page_can_recycle(const struct page *page)
static inline bool pisa_upgrade_user_pages(struct iov_iter *iter,
struct page **pages, size_t size,
- size_t offset)
+ size_t offset,
+ enum dma_data_direction dir,
+ bool pinned)
{
return false;
}
diff --git a/mm/filemap.c b/mm/filemap.c
index f9e34144c53c7..7d870c22abe53 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -15,6 +15,7 @@
#include <linux/dax.h>
#include <linux/fs.h>
#include <linux/sched/signal.h>
+#include <linux/pisa-mm.h>
#include <linux/uaccess.h>
#include <linux/capability.h>
#include <linux/kernel_stat.h>
@@ -1976,6 +1977,7 @@ void *filemap_get_entry(struct address_space *mapping, pgoff_t index)
struct folio *__filemap_get_folio_mpol(struct address_space *mapping,
pgoff_t index, fgf_t fgp_flags, gfp_t gfp, struct mempolicy *policy)
{
+ bool tried_pisa = false;
struct folio *folio;
repeat:
@@ -1985,6 +1987,13 @@ struct folio *__filemap_get_folio_mpol(struct address_space *mapping,
if (!folio)
goto no_page;
+ if (!tried_pisa) {
+ tried_pisa = true;
+ if (pisa_filemap_upgrade(mapping, folio,
+ (fgp_flags & (FGP_WRITE | FGP_NOWAIT)) == FGP_WRITE))
+ goto repeat;
+ }
+
if (fgp_flags & FGP_LOCK) {
if (fgp_flags & FGP_NOWAIT) {
if (!folio_trylock(folio)) {
@@ -2044,7 +2053,8 @@ struct folio *__filemap_get_folio_mpol(struct address_space *mapping,
err = -ENOMEM;
if (order > min_order)
alloc_gfp |= __GFP_NORETRY | __GFP_NOWARN;
- folio = filemap_alloc_folio(alloc_gfp, order, policy);
+ folio = pisa_filemap_alloc_folio(alloc_gfp, order,
+ policy, mapping);
if (!folio)
continue;
diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index 0694ec4098f6d..5f1327c22c0e4 100644
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -37,6 +37,7 @@
#include <linux/sched/rt.h>
#include <linux/sched/signal.h>
#include <linux/mm_inline.h>
+#include <linux/pisa-mm.h>
#include <linux/shmem_fs.h>
#include <trace/events/writeback.h>
@@ -2409,6 +2410,7 @@ static pgoff_t wbc_end(struct writeback_control *wbc)
static struct folio *writeback_get_folio(struct address_space *mapping,
struct writeback_control *wbc)
{
+ bool tried_pisa = false;
struct folio *folio;
retry:
@@ -2423,6 +2425,13 @@ static struct folio *writeback_get_folio(struct address_space *mapping,
return NULL;
}
+ if (!tried_pisa &&
+ pisa_writeback_upgrade(mapping, folio, &wbc->index, &wbc->fbatch)) {
+ tried_pisa = true;
+ goto retry;
+ }
+ tried_pisa = false;
+
folio_lock(folio);
if (unlikely(!folio_prepare_writeback(mapping, wbc, folio))) {
folio_unlock(folio);
diff --git a/mm/pisa-alloc.c b/mm/pisa-alloc.c
index 2213a56de1242..8c4c0f1f1cf50 100644
--- a/mm/pisa-alloc.c
+++ b/mm/pisa-alloc.c
@@ -574,6 +574,20 @@ static unsigned int pisa_pcp_pop(struct pisa_alloc *pa, unsigned int nr, struct
return got;
}
+/* Free blocks in the per-CPU caches of @pa, which @pa->blocks_in_use counts as in use. */
+unsigned long pisa_pcp_cached(struct pisa_alloc *pa)
+{
+ unsigned long n = 0;
+ int cpu;
+
+ /* Pairs with smp_store_release() in pisa_enable_pcp(). */
+ if (!smp_load_acquire(&pa->pcp))
+ return 0;
+ for_each_possible_cpu(cpu)
+ n += READ_ONCE(per_cpu_ptr(pa->pcp, cpu)->count);
+ return n;
+}
+
/* Return the blocks @cpu caches for @pa. */
static void pisa_pcp_drain_cpu(struct pisa_alloc *pa, int cpu)
{
@@ -1316,7 +1330,9 @@ bool __pisa_free_page(struct page *page, unsigned int order)
pisa_pcp_push(pa, 1, &page);
return true;
}
- pisa_pcp_drain_cpu(pa, raw_smp_processor_id());
+ /* A compaction target drains the caches once, when it is done. */
+ if (!(READ_ONCE(meta->on_list) & PISA_LIST_HOLD))
+ pisa_pcp_drain_cpu(pa, raw_smp_processor_id());
}
spin_lock_irqsave(&pa->lock, flags);
diff --git a/mm/pisa-meta.c b/mm/pisa-meta.c
index 49fa5271a392f..25d3add7a4891 100644
--- a/mm/pisa-meta.c
+++ b/mm/pisa-meta.c
@@ -85,8 +85,8 @@ DEFINE_STATIC_KEY_FALSE(pisa_key);
EXPORT_SYMBOL(pisa_key);
static unsigned int pisa_compact __read_mostly;
-static unsigned int pisa_target_usage_percent __read_mostly;
-static unsigned int pisa_old_ms __read_mostly;
+unsigned int pisa_target_usage_percent __read_mostly;
+unsigned int pisa_old_ms __read_mostly;
/* Enabled when pisa.compact > 0 or pisa.target_usage_percent > 0. */
DEFINE_STATIC_KEY_FALSE(pisa_recycle_check_key);
@@ -189,6 +189,10 @@ bool pisa_place_meta(struct pisa_meta *meta)
lockdep_assert_held(&pa->lock);
+ /* A compaction target stays parked until pisa_user_scrub_alloc() is done. */
+ if (meta->on_list & PISA_LIST_HOLD)
+ return false;
+
if (avail == bpp) {
/* All blocks free: retire immediately if quarantined or over idle cap with idle_ms == 0. */
if (pa->num_idle_pages >= pa->max_idle_pages)
@@ -283,6 +287,10 @@ static bool pisa_meta_can_recycle(struct pisa_meta *meta)
if (unlikely(READ_ONCE(meta->flags) & PISA_QUARANTINED))
return false;
+ /* Blocks freed from a compaction target go back to it, not to a cache. */
+ if (READ_ONCE(meta->on_list) & PISA_LIST_HOLD)
+ return false;
+
if (!(READ_ONCE(meta->on_list) & PISA_LIST_SPARSE))
return true;
@@ -317,6 +325,9 @@ static bool pisa_meta_can_recycle(struct pisa_meta *meta)
}
WRITE_ONCE(meta->drain, true);
+ /* Let scrub migration move out what the callers still hold on @meta. */
+ if (READ_ONCE(pisa_scrub_migrate))
+ pisa_schedule_scrub(pa->nid);
return false;
}
diff --git a/mm/pisa-node.c b/mm/pisa-node.c
index c013e384b1c7e..8d6fa9f36bdcf 100644
--- a/mm/pisa-node.c
+++ b/mm/pisa-node.c
@@ -248,7 +248,7 @@ void pisa_schedule_scrub(int nid)
{
struct pisa_node *node;
- if (!READ_ONCE(pisa_scrub_max))
+ if (!READ_ONCE(pisa_scrub_max) && !READ_ONCE(pisa_scrub_migrate))
return;
node = &pisa_nodes[nid];
@@ -277,8 +277,12 @@ static void pisa_node_work_fn(struct work_struct *work)
if (pisa_enabled() && READ_ONCE(pisa_reservoir_max))
pisa_reservoir_refill_node(node, nid);
- if (test_and_clear_bit(PISA_NODE_SCRUB, &node->flags) && pisa_scrub_node(nid))
- pisa_schedule_scrub(nid);
+ if (test_and_clear_bit(PISA_NODE_SCRUB, &node->flags)) {
+ bool more = pisa_scrub_node(nid);
+
+ if (pisa_user_scrub_node(nid) || more)
+ pisa_schedule_scrub(nid);
+ }
}
/* Queue node @nid's work if its reservoir is below or above the target. */
diff --git a/mm/pisa-priv.h b/mm/pisa-priv.h
index b18d461198ac5..c7f4e30579070 100644
--- a/mm/pisa-priv.h
+++ b/mm/pisa-priv.h
@@ -55,6 +55,7 @@ enum {
PISA_LIST_DIRTY = BIT(3), /* on @pa->partial_dirty */
PISA_LIST_MASK = 0xf,
PISA_LIST_SPARSE = BIT(4), /* tail: list_move_tail() */
+ PISA_LIST_HOLD = BIT(5), /* parked on @pa->full by compaction */
};
/* Publish meta->flags into the pfn2slot array. */
@@ -133,10 +134,11 @@ static inline long pisa_alloc_held_pages(const struct pisa_alloc *pa)
struct pisa_node {
unsigned long next_alloc; /* atomic alloc backoff jiffies */
struct pisa_alloc *pa[2][PISA_MAX_ALLOC_ORDER + 1]; /* [0]=RW, [1]=RO */
- struct pisa_alloc *user_pa; /* user page migration pa */
+ struct pisa_alloc *user_pa[3]; /* user/page-cache allocators by dir */
spinlock_t lock; /* protects @pages and @nr_pages */
unsigned int nr_pages; /* entries in @pages */
+ unsigned int scrub_dir; /* round-robin user_pa scrub index */
struct list_head pages; /* reservoir */
unsigned long flags; /* PISA_NODE_* requests */
struct work_struct work; /* refill/trim/scrub, on this node */
@@ -212,6 +214,7 @@ struct pisa_alloc *pisa_alloc_ensure(struct pisa_alloc **pap, int nid,
#define PISA_PCP_HIGH 64U /* blocks cached per CPU at most */
int pisa_enable_pcp(struct pisa_alloc *pa);
void pisa_pcp_drain(struct pisa_alloc *pa);
+unsigned long pisa_pcp_cached(struct pisa_alloc *pa);
bool pisa_place_meta(struct pisa_meta *meta);
extern struct llist_head pisa_free_list;
extern unsigned int pisa_idle_ms;
@@ -229,6 +232,9 @@ static inline struct page *pisa_alloc(struct pisa_alloc *pa, gfp_t gfp)
}
#define PISA_OLD_MS_MAX 100000U
+extern unsigned int pisa_target_usage_percent;
+extern unsigned int pisa_old_ms;
+extern unsigned int pisa_scrub_migrate;
unsigned long pisa_shrink_list(spinlock_t *lock, struct list_head *list,
unsigned int *nr, struct list_head *release_list,
unsigned long nr_to_scan);
@@ -236,6 +242,7 @@ unsigned long pisa_shrink_list(spinlock_t *lock, struct list_head *list,
#define PISA_SCRUB_RUN_NS (50 * NSEC_PER_USEC)
extern unsigned int pisa_scrub_max;
bool pisa_scrub_node(int nid);
+bool pisa_user_scrub_node(int nid);
/* pisa-node.c: per-NUMA-node PISA page reservoir and work item */
extern unsigned int pisa_reservoir_max;
diff --git a/mm/pisa-user.c b/mm/pisa-user.c
index 6c3efaee4ecda..5fb1ec7c45204 100644
--- a/mm/pisa-user.c
+++ b/mm/pisa-user.c
@@ -3,14 +3,22 @@
* PISA user and page-cache folios: allocate or migrate order-0 folios into
* per-node PISA allocators so zero-copy TX and block I/O use the PISA IOMMU mappings.
*/
+#include <linux/blkdev.h>
+#include <linux/device.h>
#include <linux/pisa.h>
+#include <linux/fs.h>
#include <linux/gfp.h>
#include <linux/huge_mm.h>
+#include <linux/iommu.h>
+#include <linux/jiffies.h>
#include <linux/migrate.h>
#include <linux/mm.h>
#include <linux/moduleparam.h>
#include <linux/nodemask.h>
+#include <linux/pagemap.h>
#include <linux/pagewalk.h>
+#include <linux/folio_batch.h>
+#include <linux/sched/clock.h>
#include <linux/swap.h>
#include <linux/uio.h>
@@ -20,9 +28,31 @@
static unsigned long pisa_user_max_pages __read_mostly;
DEFINE_STATIC_KEY_FALSE(pisa_user_key);
+/*
+ * Scrub-time migration of old sparse user_pa PISA pages when usage is below
+ * pisa.target_usage_percent (0 = off, 1 = compact in allocator, 2 = demote to the buddy allocator).
+ */
+unsigned int pisa_scrub_migrate __read_mostly;
+
+static int pisa_scrub_migrate_set(const char *val, const struct kernel_param *kp)
+{
+ int ret = param_set_uint_minmax(val, kp, 0, 2);
+
+ if (pisa_enabled() && !ret && pisa_scrub_migrate)
+ pisa_schedule_scrub_all();
+ return ret;
+}
+
+static const struct kernel_param_ops pisa_scrub_migrate_ops = {
+ .set = pisa_scrub_migrate_set,
+ .get = param_get_uint,
+};
+module_param_cb(scrub_migrate, &pisa_scrub_migrate_ops, &pisa_scrub_migrate, 0644);
+
/* Create per-node user allocators and enable pisa_user_key (serialized by param_lock/init). */
static int pisa_user_init(void)
{
+ static const char * const dir_names[3] = { "bi", "tx", "rx" };
int nid;
if (pisa_user_enabled())
@@ -30,9 +60,11 @@ static int pisa_user_init(void)
if (pisa_meta_init())
return -ENOMEM;
for_each_node_state(nid, N_MEMORY) {
- if (!pisa_alloc_ensure(&pisa_nodes[nid].user_pa, nid, 0,
- DMA_TO_DEVICE, false, "user_tx_n%d", nid))
- return -ENOMEM;
+ for (unsigned int dir = 0; dir < ARRAY_SIZE(pisa_nodes[nid].user_pa); dir++) {
+ if (!pisa_alloc_ensure(&pisa_nodes[nid].user_pa[dir], nid, 0,
+ dir, true, "user_%s_n%d", dir_names[dir], nid))
+ return -ENOMEM;
+ }
}
static_branch_enable(&pisa_user_key);
return 0;
@@ -73,16 +105,25 @@ void __init pisa_user_boot_enable(void)
}
/* User pages currently held in PISA blocks. */
-static unsigned long pisa_user_in_use(void)
+/* Blocks in use; @exact leaves out the free ones held in per-CPU caches (slower). */
+static unsigned long pisa_user_in_use(bool exact)
{
- unsigned long used = 0;
+ unsigned long used = 0, n;
int nid;
for_each_node_state(nid, N_MEMORY) {
- /* Pairs with smp_store_release() in pisa_user_init(). */
- const struct pisa_alloc *pa = smp_load_acquire(&pisa_nodes[nid].user_pa);
-
- used += pa ? READ_ONCE(pa->blocks_in_use) : 0;
+ for (unsigned int dir = 0; dir < ARRAY_SIZE(pisa_nodes[nid].user_pa); dir++) {
+ struct pisa_alloc *pa;
+
+ /* Pairs with smp_store_release() in pisa_user_init(). */
+ pa = smp_load_acquire(&pisa_nodes[nid].user_pa[dir]);
+ if (!pa)
+ continue;
+ n = READ_ONCE(pa->blocks_in_use);
+ if (exact)
+ n -= min(n, pisa_pcp_cached(pa));
+ used += n;
+ }
}
return used;
}
@@ -90,18 +131,59 @@ static unsigned long pisa_user_in_use(void)
/* Pages that can still be moved in before reaching pisa.user_max_pages. */
static long pisa_user_budget(void)
{
- return (long)((READ_ONCE(pisa_user_max_pages) << PISA_ORDER) - pisa_user_in_use());
+ unsigned long max = READ_ONCE(pisa_user_max_pages) << PISA_ORDER;
+ long budget = (long)(max - pisa_user_in_use(false));
+
+ /* Free blocks in per-CPU caches count as in use: recount before refusing. */
+ if (budget <= 0)
+ budget = (long)(max - pisa_user_in_use(true));
+ return budget;
}
-/* Is @folio worth moving into (@to_pisa) or out of PISA memory? */
-static bool pisa_user_want(struct folio *folio, bool to_pisa)
+/*
+ * Return true if @folio should be migrated into (@to_pisa) or out of PISA
+ * memory, promoting existing PISA folios to DMA_BIDIRECTIONAL when needed.
+ */
+static bool pisa_user_want_dir(struct folio *folio, bool to_pisa,
+ enum dma_data_direction dir, bool allow_file)
{
- if (!folio_test_anon(folio) || folio_is_zone_device(folio) ||
- folio_test_ksm(folio) || is_huge_zero_folio(folio) ||
- (folio_test_large(folio) && !to_pisa) ||
- folio_maybe_mapped_shared(folio))
+ unsigned long pfn = folio_pfn(folio);
+
+ if (is_zero_folio(folio) || folio_is_zone_device(folio) ||
+ folio_test_ksm(folio) || is_huge_zero_folio(folio))
+ return false;
+ if (folio_test_large(folio) && (!to_pisa || !folio_test_anon(folio)))
return false;
- return is_pisa_pfn(folio_pfn(folio)) != to_pisa;
+ if (folio_test_anon(folio)) {
+ if (folio_maybe_mapped_shared(folio))
+ return false;
+ } else if (!allow_file || !folio_mapping(folio) ||
+ folio_test_locked(folio) || folio_test_writeback(folio)) {
+ return false;
+ }
+
+ if (!to_pisa)
+ return is_pisa_pfn(pfn);
+ if (!is_pisa_pfn(pfn)) {
+ struct address_space *mapping;
+ bool movable;
+
+ if (folio_test_anon(folio))
+ return true;
+ /* Only file folios whose mapping allows movable pages. */
+ rcu_read_lock();
+ mapping = folio_mapping(folio);
+ movable = mapping && (mapping_gfp_mask(mapping) & __GFP_MOVABLE);
+ rcu_read_unlock();
+ return movable;
+ }
+
+ if (dir == DMA_TO_DEVICE || dir == DMA_FROM_DEVICE) {
+ int need = (dir == DMA_FROM_DEVICE) ? IOMMU_WRITE : IOMMU_READ;
+
+ return (READ_ONCE(pisa_meta_of_pfn(pfn)->prot) & need) != need;
+ }
+ return false;
}
/* Return true if @folio has references beyond its mappings, swap cache, and @extra. */
@@ -110,24 +192,46 @@ static bool pisa_user_busy(struct folio *folio, int extra)
return folio_ref_count(folio) > folio_expected_ref_count(folio) + extra;
}
+/*
+ * Return true if @folio has references or pins beyond its mappings, swap
+ * cache, and the @refs entries in the caller's GUP batch.
+ */
+static bool pisa_user_batch_busy(struct folio *folio, int refs, bool pinned)
+{
+ int extra = (pinned && !folio_has_pincount(folio)) ?
+ refs * GUP_PIN_COUNTING_BIAS : refs;
+
+ if (folio_has_pincount(folio) &&
+ atomic_read(&folio->_pincount) > (pinned ? refs : 0))
+ return true;
+ return pisa_user_busy(folio, extra);
+}
+
struct pisa_user_mtc {
- gfp_t gfp;
- long budget;
+ gfp_t gfp;
+ long budget;
+ enum dma_data_direction dir;
+ bool same_alloc;
};
/* new_folio_t for migration into PISA memory. */
static struct folio *pisa_user_new_folio(struct folio *src, unsigned long private)
{
struct pisa_user_mtc *mtc = (void *)private;
+ bool already_pisa = is_pisa_pfn(folio_pfn(src));
+ enum dma_data_direction dir = mtc->dir;
int nid = folio_nid(src);
struct page *page;
/* Callers split large folios first (see pisa_user_split()); none is expected here. */
- if (mtc->budget <= 0 || folio_test_large(src))
+ if ((!already_pisa && mtc->budget <= 0) || folio_test_large(src))
return NULL;
- page = pisa_alloc(pisa_nodes[nid].user_pa, mtc->gfp);
+ if (already_pisa && !mtc->same_alloc)
+ dir = DMA_BIDIRECTIONAL;
+ page = pisa_alloc(pisa_nodes[nid].user_pa[dir], mtc->gfp);
if (!page)
return NULL;
+ /* Charged for every destination, so that pisa_user_put_folio() can undo it. */
mtc->budget--;
return page_folio(page);
}
@@ -144,17 +248,31 @@ static void pisa_user_put_folio(struct folio *dst, unsigned long private)
/* new_folio_t for migration out of PISA memory: allocators never serve these. */
static struct folio *pisa_user_old_folio(struct folio *src, unsigned long private)
{
- return __folio_alloc_node(GFP_HIGHUSER_MOVABLE | __GFP_NOWARN, 0, folio_nid(src));
+ struct address_space *mapping = folio_mapping(src);
+ gfp_t gfp = GFP_HIGHUSER_MOVABLE;
+
+ /* A file folio goes back to memory that its mapping allows. */
+ if (!folio_test_anon(src) && mapping)
+ gfp = mapping_gfp_constraint(mapping, gfp);
+ return __folio_alloc_node(gfp | __GFP_NOWARN, 0, folio_nid(src));
}
/* Migrate the isolated folios on @list; failures are put back. */
-static void pisa_user_migrate(struct list_head *list, bool to_pisa, gfp_t gfp)
+static unsigned int pisa_user_migrate_dir(struct list_head *list, bool to_pisa,
+ enum dma_data_direction dir,
+ bool same_alloc, gfp_t gfp)
{
- struct pisa_user_mtc mtc = { .gfp = gfp, .budget = LONG_MAX };
+ struct pisa_user_mtc mtc = {
+ .gfp = gfp,
+ .budget = LONG_MAX,
+ .dir = dir,
+ .same_alloc = same_alloc,
+ };
enum migrate_mode mode = MIGRATE_SYNC;
+ unsigned int done = 0;
if (list_empty(list))
- return;
+ return 0;
if (to_pisa) {
mtc.budget = pisa_user_budget();
if (!gfpflags_allow_blocking(gfp))
@@ -162,8 +280,9 @@ static void pisa_user_migrate(struct list_head *list, bool to_pisa, gfp_t gfp)
}
if (migrate_pages(list, to_pisa ? pisa_user_new_folio : pisa_user_old_folio,
to_pisa ? pisa_user_put_folio : NULL, (unsigned long)&mtc,
- mode, MR_SYSCALL, NULL))
+ mode, MR_SYSCALL, &done))
putback_movable_pages(list);
+ return done;
}
/*
@@ -221,22 +340,25 @@ static unsigned int pisa_user_split_isolated(struct folio *folio, struct list_he
}
/**
- * pisa_upgrade_user_pages - Migrate extracted user @pages into PISA
+ * pisa_upgrade_user_pages - Migrate extracted user @pages into PISA for @dir
* @iter: Iterator from which @pages were just extracted
- * @pages: Extracted page array, one reference each
+ * @pages: Extracted page array
* @size: Number of bytes extracted from @iter
* @offset: Byte offset within @pages[0]
+ * @dir: I/O direction (DMA_TO_DEVICE or DMA_FROM_DEVICE)
+ * @pinned: true if @pages are FOLL_PIN-pinned, false if refcounted (+1)
*
* Return: false if no pages were isolated (@pages and @iter are untouched);
* true otherwise, so the caller must re-extract @pages from the reverted @iter.
*/
bool pisa_upgrade_user_pages(struct iov_iter *iter, struct page **pages,
- size_t size, size_t offset)
+ size_t size, size_t offset,
+ enum dma_data_direction dir, bool pinned)
{
unsigned int nr = DIV_ROUND_UP(offset + size, PAGE_SIZE);
- unsigned int want = 0;
+ unsigned int want = 0, need_budget = 0;
struct folio *f, *next;
- bool drained = false;
+ bool drained = false, no_budget = false;
LIST_HEAD(small);
LIST_HEAD(list);
long budget;
@@ -247,15 +369,29 @@ bool pisa_upgrade_user_pages(struct iov_iter *iter, struct page **pages,
* First pass: check if any page needs migration and whether budget is
* available before draining per-CPU LRU caches or isolating folios.
*/
- for (unsigned int i = 0; i < nr; i++)
- want += pisa_user_want(page_folio(pages[i]), true);
- if (!want || pisa_user_budget() <= 0)
+ for (unsigned int i = 0; i < nr; i++) {
+ struct folio *folio = page_folio(pages[i]);
+
+ if (pisa_user_want_dir(folio, true, dir, false)) {
+ want++;
+ need_budget += !is_pisa_pfn(folio_pfn(folio));
+ }
+ }
+ if (!want)
return false;
+ if (need_budget && pisa_user_budget() <= 0) {
+ if (want == need_budget)
+ return false;
+ no_budget = true;
+ }
for (unsigned int i = 0; i < nr; i++) {
struct folio *folio = page_folio(pages[i]);
unsigned int refs = 0;
- if (!pisa_user_want(folio, true))
+ if (!pisa_user_want_dir(folio, true, dir, false))
+ continue;
+ /* Out of budget: only PISA folios that need another direction. */
+ if (no_budget && !is_pisa_pfn(folio_pfn(folio)))
continue;
/* Drain per-CPU LRU batches once so freshly faulted folios can be isolated. */
if (!drained && !folio_test_lru(folio)) {
@@ -266,8 +402,7 @@ bool pisa_upgrade_user_pages(struct iov_iter *iter, struct page **pages,
continue;
for (unsigned int j = 0; j < nr; j++)
refs += page_folio(pages[j]) == folio;
- /* The references from iov_iter_get_pages2() are expected. */
- if (folio_maybe_dma_pinned(folio) || pisa_user_busy(folio, refs))
+ if (pisa_user_batch_busy(folio, refs, pinned))
continue;
if (isolate_folio_to_list(folio, &list))
list_move_tail(&folio->lru, &list);
@@ -275,11 +410,15 @@ bool pisa_upgrade_user_pages(struct iov_iter *iter, struct page **pages,
if (list_empty(&list))
return false;
/*
- * Release the caller's references on @pages before migrating
+ * Release the caller's references/pins on @pages before migrating
* (migrate_pages() rejects folios with extra references).
*/
- for (unsigned int i = 0; i < nr; i++)
- put_page(pages[i]);
+ if (pinned) {
+ unpin_user_pages(pages, nr);
+ } else {
+ for (unsigned int i = 0; i < nr; i++)
+ put_page(pages[i]);
+ }
/*
* Split large folios while budget remains and keep only their subpages
* that are in @pages: migrate_pages() would split them regardless of
@@ -293,15 +432,307 @@ bool pisa_upgrade_user_pages(struct iov_iter *iter, struct page **pages,
budget -= !is_pisa_pfn(folio_pfn(f));
}
list_splice_tail(&small, &list);
- pisa_user_migrate(&list, true,
- GFP_NOWAIT | __GFP_HARDWALL | __GFP_NOWARN);
+ pisa_user_migrate_dir(&list, true, dir, false,
+ GFP_NOWAIT | __GFP_HARDWALL | __GFP_NOWARN);
/* Rewind @iter so the caller re-extracts the updated page pointers. */
iov_iter_revert(iter, size);
return true;
}
+/* Migrate the single-page segments already added to @bio; the caller then extracts them again. */
+bool __pisa_bio_migrate_pages(struct bio *bio, struct iov_iter *iter,
+ enum dma_data_direction dir)
+{
+ ssize_t size = bio->bi_iter.bi_size;
+ struct page *bv_pages[64];
+
+ if (!bio->bi_vcnt || bio->bi_vcnt > ARRAY_SIZE(bv_pages))
+ return false;
+ for (unsigned int i = 0; i < bio->bi_vcnt; i++) {
+ const struct bio_vec *bv = &bio->bi_io_vec[i];
+
+ /* Only segments that lie within one page. */
+ if ((bv->bv_offset + bv->bv_len - 1) / PAGE_SIZE != bv->bv_offset / PAGE_SIZE)
+ return false;
+ bv_pages[i] = bv->bv_page;
+ }
+ if (!pisa_upgrade_user_pages(iter, bv_pages, size,
+ (size_t)bio->bi_vcnt * PAGE_SIZE - size,
+ dir, true))
+ return false;
+ bio->bi_vcnt = 0;
+ bio->bi_iter.bi_size = 0;
+ return true;
+}
+
+/*
+ * Return true if @bdev (or one of its parent devices) has opted into PISA for
+ * @dir (pisa_rx_bufs for reads, pisa_tx_bufs for writes).
+ */
+bool pisa_bdev_enabled(const struct block_device *bdev,
+ enum dma_data_direction dir)
+{
+ if (!bdev || !bdev->bd_disk)
+ return false;
+ for (struct device *dev = disk_to_dev(bdev->bd_disk); dev; dev = dev->parent) {
+ if (dir == DMA_FROM_DEVICE ?
+ READ_ONCE(dev->pisa_rx_bufs) :
+ READ_ONCE(dev->pisa_tx_bufs))
+ return true;
+ }
+ return false;
+}
+
+static bool pisa_mapping_enabled(const struct address_space *mapping,
+ enum dma_data_direction dir)
+{
+ struct super_block *sb;
+
+ if (!mapping || !mapping->host || !mapping->host->i_sb)
+ return false;
+ sb = mapping->host->i_sb;
+ return pisa_bdev_enabled(sb_is_blkdev_sb(sb) ?
+ I_BDEV(mapping->host) : sb->s_bdev, dir);
+}
+
+/**
+ * __pisa_filemap_alloc - Allocate an order-0 page-cache folio from a PISA user allocator
+ * @mapping: Page-cache address_space
+ * @gfp: GFP flags (__GFP_WRITE indicates buffered write path)
+ *
+ * Return: Newly allocated order-0 folio, or NULL on failure.
+ */
+struct folio *__pisa_filemap_alloc(const struct address_space *mapping,
+ gfp_t gfp)
+{
+ bool is_write = gfp & __GFP_WRITE;
+ enum dma_data_direction dir = is_write ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+ struct page *page;
+ int nid;
+
+ /* User PISA pages are compacted by migration: no unmovable folios in them. */
+ if (!(gfp & __GFP_MOVABLE))
+ return NULL;
+ if (!pisa_mapping_enabled(mapping, dir))
+ return NULL;
+ if (pisa_mapping_enabled(mapping, is_write ? DMA_FROM_DEVICE : DMA_TO_DEVICE))
+ dir = DMA_BIDIRECTIONAL;
+ if (pisa_user_budget() <= 0)
+ return NULL;
+ nid = pisa_sanitize_nid(numa_node_id());
+ page = pisa_alloc(pisa_nodes[nid].user_pa[dir], gfp);
+ if (!page)
+ return NULL;
+ return page_folio(page);
+}
+
+static bool pisa_folio_upgrade(struct folio *folio, enum dma_data_direction dir,
+ struct folio_batch *fbatch)
+{
+ LIST_HEAD(list);
+
+ if (!pisa_user_want_dir(folio, true, dir, true) ||
+ (!is_pisa_pfn(folio_pfn(folio)) && pisa_user_budget() <= 0))
+ return false;
+ if (!folio_test_lru(folio))
+ lru_add_drain();
+ if (pisa_user_busy(folio, 1) || !isolate_folio_to_list(folio, &list))
+ return false;
+ /* Drop caller's reference so migrate_pages() sees only expected refs. */
+ if (fbatch)
+ folio_batch_release(fbatch);
+ else
+ folio_put(folio);
+ pisa_user_migrate_dir(&list, true, dir, false,
+ GFP_NOWAIT | __GFP_HARDWALL | __GFP_NOWARN);
+ return true;
+}
+
+bool __pisa_filemap_upgrade(const struct address_space *mapping,
+ struct folio *folio)
+{
+ return pisa_mapping_enabled(mapping, DMA_TO_DEVICE) &&
+ pisa_folio_upgrade(folio, DMA_TO_DEVICE, NULL);
+}
+
+bool __pisa_writeback_upgrade(const struct address_space *mapping,
+ struct folio *folio, pgoff_t *index,
+ struct folio_batch *fbatch)
+{
+ pgoff_t idx;
+
+ if (!pisa_mapping_enabled(mapping, DMA_TO_DEVICE))
+ return false;
+ idx = folio->index;
+ if (!pisa_folio_upgrade(folio, DMA_TO_DEVICE, fbatch))
+ return false;
+ *index = idx;
+ return true;
+}
+
#define PISA_USER_BATCH 512
+/* Under RCU: is @pfn still a block of @target, and @target still owned by @pa? */
+static bool pisa_user_block_of(unsigned long pfn, const struct pisa_meta *target,
+ const struct pisa_alloc *pa)
+{
+ return is_pisa_pfn(pfn) && pisa_meta_of_pfn(pfn) == target &&
+ READ_ONCE(target->pa) == pa;
+}
+
+static unsigned long pisa_user_scrub_alloc(struct pisa_alloc *pa,
+ enum dma_data_direction dir,
+ unsigned int mode,
+ unsigned long budget)
+{
+ unsigned int target_pct = READ_ONCE(pisa_target_usage_percent);
+ unsigned int old_ms = READ_ONCE(pisa_old_ms);
+ unsigned long bpp = PISA_BLOCKS(0), in_use;
+ unsigned long max_age, base_pfn = 0, flags;
+ DECLARE_BITMAP(inuse_map, _PISA_MAX_BLOCKS);
+ struct list_head *lists[] = { &pa->partial_dirty, &pa->partial };
+ struct pisa_meta *meta, *target = NULL;
+ unsigned int idx, count = 0;
+ unsigned long migrated = 0;
+ LIST_HEAD(list);
+ long held;
+
+ if (!target_pct || !old_ms)
+ return 0;
+
+ pisa_pcp_drain(pa);
+
+ held = pisa_alloc_held_pages(pa);
+ in_use = READ_ONCE(pa->blocks_in_use);
+ if (held <= 0 || in_use * 100 >= (unsigned long)held * bpp * target_pct)
+ return 0;
+
+ max_age = msecs_to_jiffies(min(old_ms, PISA_OLD_MS_MAX));
+
+ spin_lock_irqsave(&pa->lock, flags);
+ for (unsigned int i = 0; i < ARRAY_SIZE(lists) && !target; i++) {
+ list_for_each_entry_reverse(meta, lists[i], list) {
+ if (pisa_meta_avail(meta) < (bpp >> 1))
+ break;
+ /* Compaction needs a sparse page and another partial page to fill. */
+ if (mode == 1 && (!(meta->on_list & PISA_LIST_SPARSE) ||
+ (list_is_first(&meta->list, lists[i]) &&
+ list_empty(lists[!i]))))
+ break;
+ if (time_after(jiffies, meta->alloc_jiffies + max_age)) {
+ target = meta;
+ break;
+ }
+ }
+ }
+ if (target) {
+ /*
+ * Compaction: park @target on @pa->full, so that the blocks
+ * freed from it are neither cached nor handed out again while
+ * its folios move to other pages of @pa.
+ */
+ if (mode == 1) {
+ WRITE_ONCE(target->on_list, PISA_LIST_FULL | PISA_LIST_HOLD);
+ list_move(&target->list, &pa->full);
+ }
+ base_pfn = pisa_meta_to_pfn(target);
+ bitmap_or(inuse_map, target->clean_bitmap, target->dirty_bitmap, bpp);
+ bitmap_complement(inuse_map, inuse_map, bpp);
+ }
+ spin_unlock_irqrestore(&pa->lock, flags);
+
+ if (!target)
+ return 0;
+
+ for_each_set_bit(idx, inuse_map, bpp) {
+ unsigned long pfn = base_pfn + idx;
+ struct folio *folio;
+
+ rcu_read_lock();
+ if (!pisa_user_block_of(pfn, target, pa)) {
+ rcu_read_unlock();
+ break;
+ }
+ folio = page_folio(pfn_to_page(pfn));
+ if (!folio_try_get(folio)) {
+ rcu_read_unlock();
+ continue;
+ }
+ if (!pisa_user_block_of(pfn, target, pa) ||
+ folio != page_folio(pfn_to_page(pfn))) {
+ rcu_read_unlock();
+ folio_put(folio);
+ continue;
+ }
+ rcu_read_unlock();
+
+ if (!pisa_user_want_dir(folio, false, dir, true) ||
+ pisa_user_busy(folio, 1) ||
+ !isolate_folio_to_list(folio, &list)) {
+ folio_put(folio);
+ continue;
+ }
+ folio_put(folio);
+ if (++count >= min_t(unsigned long, budget, PISA_USER_BATCH))
+ break;
+ }
+
+ if (!list_empty(&list)) {
+ migrated = pisa_user_migrate_dir(&list, mode == 1, dir, true,
+ GFP_KERNEL | __GFP_NOWARN);
+ pisa_pcp_drain(pa);
+ }
+
+ if (mode == 1) {
+ bool should_release;
+
+ /* Put @target back on the list that its free blocks call for. */
+ spin_lock_irqsave(&pa->lock, flags);
+ WRITE_ONCE(target->on_list, PISA_LIST_FULL);
+ should_release = pisa_place_meta(target);
+ spin_unlock_irqrestore(&pa->lock, flags);
+ if (should_release)
+ pisa_schedule_reclaim();
+ }
+
+ /* Nothing moved: do not pick @target again before it ages. */
+ if (!migrated && READ_ONCE(target->pa) == pa)
+ WRITE_ONCE(target->alloc_jiffies, jiffies);
+
+ return migrated;
+}
+
+bool pisa_user_scrub_node(int nid)
+{
+ unsigned long budget = READ_ONCE(pisa_scrub_max) ?: 512;
+ unsigned int mode = READ_ONCE(pisa_scrub_migrate);
+ unsigned int n_dirs = ARRAY_SIZE(pisa_nodes[nid].user_pa);
+ u64 deadline = local_clock() + PISA_SCRUB_RUN_NS;
+ bool progress = false;
+
+ if (!mode || mode > 2)
+ return false;
+
+ for (unsigned int i = 0; i < n_dirs; i++) {
+ unsigned int dir = pisa_nodes[nid].scrub_dir++ % n_dirs;
+ /* Pairs with smp_store_release() in pisa_user_init(). */
+ struct pisa_alloc *pa = smp_load_acquire(&pisa_nodes[nid].user_pa[dir]);
+ unsigned long n;
+
+ if (!pa)
+ continue;
+ n = pisa_user_scrub_alloc(pa, dir, mode, budget);
+ if (n >= budget || local_clock() >= deadline)
+ return true;
+ if (n) {
+ budget -= n;
+ progress = true;
+ cond_resched();
+ }
+ }
+ return progress;
+}
+
struct pisa_madvise_walk {
struct list_head list; /* isolated folios waiting to be migrated */
unsigned int n; /* pages on @list */
@@ -329,7 +760,8 @@ static int pisa_madvise_pmd_entry(pmd_t *pmd, unsigned long addr, unsigned long
return 1;
folio = folio_walk_start(&fw, walk->vma, addr, 0);
if (folio) {
- bool want = pisa_user_want(folio, to_pisa) && folio_test_lru(folio);
+ bool want = pisa_user_want_dir(folio, to_pisa, DMA_TO_DEVICE, false) &&
+ folio_test_lru(folio);
bool busy = want && pisa_user_busy(folio, 0);
bool split = want && !busy && folio_test_large(folio);
@@ -352,7 +784,8 @@ static int pisa_madvise_pmd_entry(pmd_t *pmd, unsigned long addr, unsigned long
}
}
if (w->n >= PISA_USER_BATCH) {
- pisa_user_migrate(&w->list, to_pisa, PISA_MADVISE_GFP);
+ pisa_user_migrate_dir(&w->list, to_pisa, DMA_TO_DEVICE, false,
+ PISA_MADVISE_GFP);
w->n = 0;
}
addr += nr << PAGE_SHIFT;
@@ -389,6 +822,6 @@ int pisa_madvise(struct vm_area_struct *vma, unsigned long start, unsigned long
lru_add_drain();
/* The page walker skips the unpopulated page-table levels of sparse mappings. */
ret = walk_page_range_vma(vma, start, end, &pisa_madvise_walk_ops, &w);
- pisa_user_migrate(&w.list, to_pisa, PISA_MADVISE_GFP);
+ pisa_user_migrate_dir(&w.list, to_pisa, DMA_TO_DEVICE, false, PISA_MADVISE_GFP);
return ret < 0 ? ret : 0;
}
diff --git a/mm/readahead.c b/mm/readahead.c
index 6e5563290287e..82b03ac1161dd 100644
--- a/mm/readahead.c
+++ b/mm/readahead.c
@@ -114,6 +114,7 @@
*/
#include <linux/blk_plug.h>
+#include <linux/pisa-mm.h>
#include <linux/kernel.h>
#include <linux/dax.h>
#include <linux/gfp.h>
@@ -197,7 +198,7 @@ static struct folio *ractl_alloc_folio(struct readahead_control *ractl,
{
struct folio *folio;
- folio = filemap_alloc_folio(gfp_mask, order, NULL);
+ folio = pisa_filemap_alloc_folio(gfp_mask, order, NULL, ractl->mapping);
if (folio && ractl->dropbehind)
__folio_set_dropbehind(folio);
--
2.56.1.809.ga0b61035b5-goog
next prev parent reply other threads:[~2026-11-20 1:07 UTC|newest]
Thread overview: 144+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-10-03 21:22 [RFC: DMA_PMD 00/22] DMA_PMD: PMD_SIZE-backed IO buffer pools Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 01/22] iommu/dma: introduce CONFIG_DMA_PMD and metadata table Luigi Rizzo
2026-10-03 21:44 ` Randy Dunlap
2026-10-04 9:14 ` Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 02/22] iommu/dma: add DMA_PMD pool lifecycle and page recycle hook Luigi Rizzo
2026-10-10 2:59 ` kernel test robot
2026-10-03 21:22 ` [RFC: DMA_PMD 03/22] mm: Add split_page_compound() Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 04/22] iommu/dma: add DMA_PMD pool block allocation Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 05/22] iommu/dma: Global cap and shrinker for DMA_PMD pool memory Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 06/22] iommu/dma: reserve a per-domain IOVA window for DMA_PMD pages Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 07/22] iommu/dma: release DMA_PMD domain mappings on domain teardown Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 08/22] iommu/dma: use per-domain IOVA window to map DMA_PMD memory Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 09/22] iommu/dma: Add DMA_PMD arena allocator Luigi Rizzo
2026-10-08 11:25 ` kernel test robot
2026-10-03 21:22 ` [RFC: DMA_PMD 10/22] driver core: Add per-device dma_pmd_* sysfs attributes Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 11/22] dma-mapping: Use DMA_PMD arena for dma_alloc_attrs() Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 12/22] net/core: Use per-CPU DMA_PMD pools for skb_page_frag_refill() Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 13/22] net/core: Use DMA_PMD for page_pool memory Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 14/22] iommu/dma: Support decrypted and pinned DMA_PMD pages Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 15/22] iommu/dma: Add background page scrubber for DMA_PMD pools Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 16/22] iommu/dma: Add per-NUMA-node PMD page reservoir Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 17/22] net/gve: Use DMA_PMD memory for RX buffers Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 18/22] net/gve: Use DMA_PMD memory for tx header bounce buffers Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 19/22] net/mlx5e: " Luigi Rizzo
2026-10-14 17:22 ` Dragos Tatulea
2026-10-14 17:50 ` Dragos Tatulea
2026-10-14 18:28 ` Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 20/22] net/idpf: " Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 21/22] net/bnxt: " Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 22/22] iommu/dma: Add DMA_PMD statistics and debugfs Luigi Rizzo
2026-10-11 9:02 ` [RFC: DMA_PMD 00/22] DMA_PMD: PMD_SIZE-backed IO buffer pools Pranjal Shrivastava
2026-10-12 13:51 ` David Hildenbrand (Arm)
2026-10-12 14:45 ` Luigi Rizzo
2026-10-12 19:43 ` David Hildenbrand (Arm)
2026-10-13 10:42 ` Robin Murphy
2026-10-14 10:47 ` David Hildenbrand (Arm)
2026-10-14 11:05 ` Luigi Rizzo
2026-10-17 17:11 ` Luigi Rizzo
2026-10-22 20:12 ` Jason Gunthorpe
2026-10-22 21:51 ` Luigi Rizzo
2026-10-23 14:01 ` Jason Gunthorpe
2026-10-23 14:30 ` Luigi Rizzo
2026-10-23 14:50 ` Jason Gunthorpe
2026-10-23 15:12 ` Luigi Rizzo
2026-10-23 17:27 ` Jakub Kicinski
2026-10-23 20:22 ` Luigi Rizzo
2026-11-03 8:09 ` Christoph Hellwig
2026-11-03 9:13 ` Luigi Rizzo
2026-11-03 15:10 ` Jason Gunthorpe
2026-11-06 17:21 ` Naman Gulati
2026-11-06 17:30 ` Jason Gunthorpe
2026-11-06 18:00 ` Naman Gulati
2026-11-06 18:07 ` Jason Gunthorpe
2026-11-06 18:17 ` Naman Gulati
2026-11-06 18:43 ` Jason Gunthorpe
2026-11-06 19:01 ` Luigi Rizzo
2026-11-06 19:11 ` Naman Gulati
2026-10-12 14:49 ` Mostafa Saleh
2026-10-12 15:02 ` Luigi Rizzo
2026-10-14 17:28 ` Dragos Tatulea
2026-10-14 18:14 ` Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 00/25] PISA: Pre-mapped IOVA Superpage Allocator for DMA and CoCo Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 " Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 01/25] mm: Introduce CONFIG_PISA and metadata table Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 02/25] mm: Add PISA allocator lifecycle and page recycle hook Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 03/25] mm: Add per-NUMA-node PISA page reservoir Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 04/25] mm/page_alloc: Extend alloc_pages() to __GFP_PISA and PISA block allocation Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 05/25] mm: Add global cap and shrinker for PISA pages Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 06/25] iommu/dma: Map PISA memory in a per-domain IOVA window Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 07/25] driver core: Add per-device pisa_* sysfs attributes for PISA optin Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 08/25] iommu/dma: Use PISA for coherent DMA allocations Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 09/25] mm: Add per-CPU block caches to PISA allocators Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 10/25] mm: Add background page scrubber for " Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 11/25] mm: Migrate user pages into PISA Luigi Rizzo
2026-11-20 14:10 ` Lorenzo Stoakes (ARM)
2026-11-20 1:06 ` Luigi Rizzo [this message]
2026-11-20 16:17 ` [RFC PATCH v3 12/25] block, mm: Support PISA for disk I/O and scrub migration Christoph Hellwig
2026-11-20 1:06 ` [RFC PATCH v3 13/25] net: Cache leaf PISA TX flags on struct net_device Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 14/25] net: Use per-node PISA allocators for skb_page_frag_refill() Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 15/25] net: Use PISA for page_pool memory Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 16/25] mm/slub: Add SLAB_CACHE_PISA_RO for PISA-backed slab caches Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 17/25] net: Allocate small skb heads from PISA Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 18/25] net: Migrate user pages into PISA for MSG_ZEROCOPY Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 19/25] mm, iommu/dma: Support decrypted PISA pages Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 20/25] arm64/mm, mm: Support decrypted PISA pages in Realms Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 21/25] mm: Add PISA statistics and debugfs Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 22/25] net/libeth: Use GFP_KERNEL when pre-filling RX rings Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 23/25] net/gve: " Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 24/25] net/gve: Use PISA memory for RX buffers Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 25/25] Documentation: Add PISA architecture documentation Luigi Rizzo
2026-11-20 13:50 ` [RFC PATCH v3 00/25] PISA: Pre-mapped IOVA Superpage Allocator for DMA and CoCo Jason Gunthorpe
2026-11-20 14:18 ` Lorenzo Stoakes (ARM)
2026-11-20 14:19 ` Luigi Rizzo
2026-11-20 14:36 ` Lorenzo Stoakes (ARM)
2026-11-20 14:54 ` Lorenzo Stoakes (ARM)
2026-11-20 16:12 ` David Hildenbrand (Arm)
2026-11-20 18:16 ` Lorenzo Stoakes (ARM)
2026-11-20 15:00 ` [RFC PATCH v2 " Matthew Wilcox
2026-11-09 18:25 ` [RFC PATCH v2 01/25] iommu/dma: Introduce CONFIG_PISA and metadata table Luigi Rizzo
2026-11-10 23:12 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 02/25] iommu/dma: Add PISA allocator lifecycle and page recycle hook Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 03/25] iommu/dma: Add per-NUMA-node PISA page reservoir Luigi Rizzo
2026-11-10 23:12 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 04/25] iommu/dma: Add PISA block allocation Luigi Rizzo
2026-11-10 23:12 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 05/25] iommu/dma: Add global cap and shrinker for PISA pages Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 06/25] iommu/dma: Reserve a per-domain IOVA window " Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 07/25] iommu/dma: Release PISA domain mappings on domain teardown Luigi Rizzo
2026-11-10 23:12 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 08/25] iommu/dma: Use per-domain IOVA window to map PISA memory Luigi Rizzo
2026-11-10 23:12 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 09/25] driver core: Add per-device pisa_* sysfs attributes for PISA optin Luigi Rizzo
2026-11-10 23:12 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 10/25] iommu/dma: Use PISA for coherent DMA allocations Luigi Rizzo
2026-11-10 23:12 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 11/25] iommu/dma: Add per-CPU block caches to PISA allocators Luigi Rizzo
2026-11-10 23:12 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 12/25] net/core: Use per-node PISA allocators for skb_page_frag_refill() Luigi Rizzo
2026-11-10 23:50 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 13/25] net/core: Use PISA for page_pool memory Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 14/25] iommu/dma: Support decrypted PISA pages Luigi Rizzo
2026-11-11 0:19 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 15/25] iommu/dma: Add background page scrubber for PISA allocators Luigi Rizzo
2026-11-11 0:37 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 16/25] mm/slub: Add SLAB_CACHE_PISA_RO for PISA-backed slab caches Luigi Rizzo
2026-11-11 1:03 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 17/25] net: Allocate small skb heads from PISA Luigi Rizzo
2026-11-11 1:33 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 18/25] iommu/dma: Migrate user pages into PISA for TX zerocopy Luigi Rizzo
2026-11-11 2:41 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 19/25] iommu/dma: Stop recycling old PISA pages when allocator waste is high Luigi Rizzo
2026-11-11 2:57 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 20/25] net/libeth: Use GFP_KERNEL when pre-filling RX rings Luigi Rizzo
2026-11-11 3:13 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 21/25] net/gve: " Luigi Rizzo
2026-11-11 3:22 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 22/25] net/gve: Use PISA memory for RX buffers Luigi Rizzo
2026-11-11 3:43 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 23/25] block, mm, iommu/dma: Support PISA for disk I/O and scrub migration Luigi Rizzo
2026-11-11 4:16 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 24/25] Documentation: Add PISA architecture documentation Luigi Rizzo
2026-11-11 4:24 ` sashiko-bot
2026-11-09 18:25 ` [RFC PATCH v2 25/25] iommu/dma: Add PISA statistics and debugfs Luigi Rizzo
2026-11-11 4:37 ` sashiko-bot
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261120010655.467825-13-lrizzo@google.com \
--to=lrizzo@google.com \
--cc=akpm@linux-foundation.org \
--cc=aleksander.lobakin@intel.com \
--cc=axboe@kernel.dk \
--cc=brendan.jackman@linux.dev \
--cc=catalin.marinas@arm.com \
--cc=cl@gentwo.org \
--cc=corbet@lwn.net \
--cc=dakr@kernel.org \
--cc=davem@davemloft.net \
--cc=david@kernel.org \
--cc=driver-core@lists.linux.dev \
--cc=dtatulea@nvidia.com \
--cc=edumazet@kernel.org \
--cc=gregkh@linuxfoundation.org \
--cc=hannes@cmpxchg.org \
--cc=hao.li@linux.dev \
--cc=harry@kernel.org \
--cc=hawk@kernel.org \
--cc=hch@lst.de \
--cc=horms@kernel.org \
--cc=hramamurthy@google.com \
--cc=ilias.apalodimas@linaro.org \
--cc=iommu@lists.linux.dev \
--cc=jack@suse.cz \
--cc=jannh@google.com \
--cc=jgg@ziepe.ca \
--cc=joro@8bytes.org \
--cc=joshwash@google.com \
--cc=kuba@kernel.org \
--cc=kuniyu@google.com \
--cc=liam@infradead.org \
--cc=linux-api@vger.kernel.org \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-block@vger.kernel.org \
--cc=linux-doc@vger.kernel.org \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=ljs@kernel.org \
--cc=m.szyprowski@samsung.com \
--cc=mhocko@suse.com \
--cc=namangulati@google.com \
--cc=netdev@vger.kernel.org \
--cc=pabeni@redhat.com \
--cc=praan@google.com \
--cc=rafael@kernel.org \
--cc=rdunlap@infradead.org \
--cc=rientjes@google.com \
--cc=rizzo.unipi@gmail.com \
--cc=robin.murphy@arm.com \
--cc=roman.gushchin@linux.dev \
--cc=rppt@kernel.org \
--cc=smostafa@google.com \
--cc=surenb@google.com \
--cc=vbabka@kernel.org \
--cc=will@kernel.org \
--cc=willemb@google.com \
--cc=willy@infradead.org \
--cc=ziy@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.