From: Luigi Rizzo <lrizzo@google.com>
To: Andrew Morton <akpm@linux-foundation.org>,
David Hildenbrand <david@kernel.org>,
Johannes Weiner <hannes@cmpxchg.org>,
Vlastimil Babka <vbabka@kernel.org>,
Joerg Roedel <joro@8bytes.org>, Will Deacon <will@kernel.org>,
Robin Murphy <robin.murphy@arm.com>,
Marek Szyprowski <m.szyprowski@samsung.com>,
Jens Axboe <axboe@kernel.dk>,
"David S . Miller" <davem@davemloft.net>,
Eric Dumazet <edumazet@kernel.org>,
Jakub Kicinski <kuba@kernel.org>,
Paolo Abeni <pabeni@redhat.com>,
Luigi Rizzo <rizzo.unipi@gmail.com>
Cc: Lorenzo Stoakes <ljs@kernel.org>,
"Liam R . Howlett" <liam@infradead.org>,
Mike Rapoport <rppt@kernel.org>,
Suren Baghdasaryan <surenb@google.com>,
Michal Hocko <mhocko@suse.com>,
Brendan Jackman <brendan.jackman@linux.dev>,
Zi Yan <ziy@nvidia.com>, Harry Yoo <harry@kernel.org>,
Christoph Lameter <cl@gentwo.org>,
David Rientjes <rientjes@google.com>,
Roman Gushchin <roman.gushchin@linux.dev>,
Hao Li <hao.li@linux.dev>, Matthew Wilcox <willy@infradead.org>,
Jan Kara <jack@suse.cz>, Jann Horn <jannh@google.com>,
Catalin Marinas <catalin.marinas@arm.com>,
Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
"Rafael J . Wysocki" <rafael@kernel.org>,
Danilo Krummrich <dakr@kernel.org>,
Jonathan Corbet <corbet@lwn.net>,
Simon Horman <horms@kernel.org>,
Jesper Dangaard Brouer <hawk@kernel.org>,
Ilias Apalodimas <ilias.apalodimas@linaro.org>,
Willem de Bruijn <willemb@google.com>,
Kuniyuki Iwashima <kuniyu@google.com>,
Joshua Washington <joshwash@google.com>,
Harshitha Ramamurthy <hramamurthy@google.com>,
Alexander Lobakin <aleksander.lobakin@intel.com>,
Christoph Hellwig <hch@lst.de>, Jason Gunthorpe <jgg@ziepe.ca>,
Dragos Tatulea <dtatulea@nvidia.com>,
Naman Gulati <namangulati@google.com>,
Pranjal Shrivastava <praan@google.com>,
Mostafa Saleh <smostafa@google.com>,
Randy Dunlap <rdunlap@infradead.org>,
linux-mm@kvack.org, iommu@lists.linux.dev,
netdev@vger.kernel.org, linux-block@vger.kernel.org,
linux-fsdevel@vger.kernel.org, driver-core@lists.linux.dev,
linux-arm-kernel@lists.infradead.org, linux-api@vger.kernel.org,
linux-doc@vger.kernel.org, linux-kernel@vger.kernel.org,
Luigi Rizzo <lrizzo@google.com>
Subject: [RFC PATCH v3 10/25] mm: Add background page scrubber for PISA allocators
Date: Fri, 20 Nov 2026 01:06:40 +0000 [thread overview]
Message-ID: <20261120010655.467825-11-lrizzo@google.com> (raw)
In-Reply-To: <20261120010655.467825-1-lrizzo@google.com>
Blocks returned to a pisa_alloc are marked dirty and need to be zeroed
before they can serve __GFP_ZERO requests. Zeroing blocks inline at
allocation time adds CPU cost and latency.
Add an optional background scrubber (enabled via pisa.scrub_max) that
runs on the allocator's home NUMA node work item, zeroes dirty blocks
outside pa->lock in batches, and moves them to meta->clean_bitmap[].
Scrubbing is demand-driven by the number of dirty blocks that __GFP_ZERO
allocations had to zero inline.
Signed-off-by: Luigi Rizzo <lrizzo@google.com>
---
include/linux/pisa-iommu.h | 1 +
mm/pisa-alloc.c | 187 ++++++++++++++++++++++++++++++++++++-
mm/pisa-kunit.c | 51 +++++++++-
mm/pisa-meta.c | 7 +-
mm/pisa-node.c | 33 +++++++
mm/pisa-priv.h | 10 +-
6 files changed, 281 insertions(+), 8 deletions(-)
diff --git a/include/linux/pisa-iommu.h b/include/linux/pisa-iommu.h
index 568108747f575..7dd91ff47288a 100644
--- a/include/linux/pisa-iommu.h
+++ b/include/linux/pisa-iommu.h
@@ -55,6 +55,7 @@ struct pisa_meta {
u8 on_list; /* current placement in @pa (PISA_LIST_*) */
bool drain; /* sparse old page draining back to @idle */
u32 slot; /* dense index in @pisa_meta_array and IOVA windows */
+ u16 nr_scrub; /* blocks being zeroed by the scrubber, in neither bitmap */
DECLARE_BITMAP(domains_mapped, PISA_MAX_DOMAINS);
diff --git a/mm/pisa-alloc.c b/mm/pisa-alloc.c
index 896709303f258..ccba2bf01bd59 100644
--- a/mm/pisa-alloc.c
+++ b/mm/pisa-alloc.c
@@ -32,6 +32,7 @@
#include <linux/page_table_check.h>
#include <linux/pci.h>
#include <linux/rcupdate.h>
+#include <linux/sched/clock.h>
#include <linux/sched/mm.h>
#include <linux/shrinker.h>
#include <linux/skbuff.h>
@@ -80,6 +81,10 @@ static unsigned int pisa_max_idle __read_mostly = 1;
unsigned int pisa_idle_ms __read_mostly = 200;
module_param_named(idle_ms, pisa_idle_ms, uint, 0644);
+/* Max pages zeroed in each scrubber run (0 = disabled, capped at PISA_SCRUB_MAX_LIMIT). */
+unsigned int pisa_scrub_max __read_mostly;
+#define PISA_SCRUB_MAX_LIMIT 65536U
+
/* All live allocators, protected by pisa_allocs_lock. */
static LIST_HEAD(pisa_allocs);
static DEFINE_RWLOCK(pisa_allocs_lock);
@@ -113,6 +118,25 @@ static const struct kernel_param_ops pisa_max_idle_ops = {
};
module_param_cb(max_idle, &pisa_max_idle_ops, &pisa_max_idle, 0644);
+static int pisa_scrub_set(const char *val, const struct kernel_param *kp)
+{
+ int ret = param_set_uint(val, kp);
+
+ if (ret)
+ return ret;
+ WRITE_ONCE(pisa_scrub_max, min(pisa_scrub_max, PISA_SCRUB_MAX_LIMIT));
+ /* Serve demand accumulated while the scrubber was off (once initialized). */
+ if (pisa_enabled() && pisa_scrub_max)
+ pisa_schedule_scrub_all();
+ return 0;
+}
+
+static const struct kernel_param_ops pisa_scrub_ops = {
+ .set = pisa_scrub_set,
+ .get = param_get_uint,
+};
+module_param_cb(scrub_max, &pisa_scrub_ops, &pisa_scrub_max, 0644);
+
static void pisa_reclaim_work_fn(struct work_struct *work);
/* Asynchronous reclaim worker for pisa_free_list and expired idle pages. */
@@ -245,6 +269,140 @@ static bool is_pisa_block_poisoned(struct pisa_alloc *pa, struct page *block)
return false;
}
+/*
+ * Background page scrubber: pre-zeroes dirty blocks in @pa->nid's work item
+ * to match inline-zeroed demand (@pa->scrub_demand) when scrub_max > 0.
+ */
+static struct pisa_meta *pisa_pick_dirty_meta(struct pisa_alloc *pa)
+{
+ struct pisa_meta *meta;
+
+ meta = list_first_entry_or_null(&pa->partial_dirty, struct pisa_meta, list);
+ if (meta)
+ return meta;
+
+ list_for_each_entry(meta, &pa->partial, list) {
+ if (meta->nr_dirty)
+ return meta;
+ }
+
+ /* Idle pages are left alone: scrubbing one would restart its idle clock. */
+ return NULL;
+}
+
+/* Dirty blocks per allocator, inline-zeroed since the last run, that queue a run. */
+#define PISA_SCRUB_TRIGGER 32U
+/* Cap on @pa->scrub_demand, so a long-off scrubber does not owe too much. */
+#define PISA_SCRUB_DEMAND_MAX 4096U
+/* Bytes zeroed per batch: a few microseconds of memset(). */
+#define PISA_SCRUB_BYTES SZ_128K
+#define PISA_SCRUB_BATCH (PISA_SCRUB_BYTES >> PAGE_SHIFT)
+static_assert(PISA_SCRUB_BATCH * sizeof(u16) <= 128);
+
+/*
+ * Scrub one batch of dirty blocks in @pa outside @pa->lock.
+ * Returns the number of PAGE_SIZE pages taken off the dirty bitmap.
+ */
+static unsigned long pisa_alloc_scrub_blocks(struct pisa_alloc *pa)
+{
+ unsigned int bpp, order, idx, count = 0, want;
+ unsigned long flags, used = 0;
+ u16 idxs[PISA_SCRUB_BATCH];
+ struct page *page, *block;
+ struct pisa_meta *meta;
+ bool should_release;
+
+ spin_lock_irqsave(&pa->lock, flags);
+ if (!pa->scrub_demand) {
+ spin_unlock_irqrestore(&pa->lock, flags);
+ return 0;
+ }
+
+ meta = pisa_pick_dirty_meta(pa);
+ if (!meta) {
+ pa->scrub_demand = 0;
+ spin_unlock_irqrestore(&pa->lock, flags);
+ return 0;
+ }
+
+ order = pa->order;
+ bpp = PISA_BLOCKS(order);
+ page = pisa_meta_to_page(meta);
+ want = min(max(PISA_SCRUB_BATCH >> order, 1U), pa->scrub_demand);
+
+ for_each_set_bit(idx, meta->dirty_bitmap, bpp) {
+ block = page + (idx << order);
+ __clear_bit(idx, meta->dirty_bitmap);
+ used++;
+ if (!is_pisa_block_poisoned(pa, block))
+ idxs[count++] = idx;
+ if (used == meta->nr_dirty || count == want)
+ break;
+ }
+ meta->nr_dirty -= used;
+ pa->blocks_in_use += used - count;
+ meta->nr_scrub = count;
+ pisa_place_meta(meta);
+ spin_unlock_irqrestore(&pa->lock, flags);
+
+ if (unlikely(!count))
+ return used << order;
+
+ for (idx = 0; idx < count; idx++) {
+ block = page + (idxs[idx] << order);
+ memset(page_address(block), 0, PAGE_SIZE << order);
+ }
+
+ spin_lock_irqsave(&pa->lock, flags);
+ for (idx = 0; idx < count; idx++)
+ __set_bit(idxs[idx], meta->clean_bitmap);
+ meta->nr_clean += count;
+ meta->nr_scrub = 0;
+ pa->scrub_demand -= min(count, pa->scrub_demand);
+ should_release = pisa_place_meta(meta);
+ spin_unlock_irqrestore(&pa->lock, flags);
+
+ if (should_release)
+ pisa_schedule_reclaim();
+
+ return used << order;
+}
+
+/*
+ * Scrub dirty blocks for allocators homed on @nid up to @pisa_scrub_max pages or
+ * PISA_SCRUB_RUN_NS. Returns true if stopped early with demand remaining.
+ */
+bool pisa_scrub_node(int nid)
+{
+ unsigned long budget = READ_ONCE(pisa_scrub_max), n;
+ u64 deadline = local_clock() + PISA_SCRUB_RUN_NS;
+ struct pisa_alloc *pa;
+ bool progress;
+
+ if (!budget)
+ return false;
+
+ do {
+ progress = false;
+ read_lock(&pisa_allocs_lock);
+ list_for_each_entry(pa, &pisa_allocs, node) {
+ if (pa->nid != nid || !READ_ONCE(pa->scrub_demand))
+ continue;
+ read_unlock(&pisa_allocs_lock);
+ n = pisa_alloc_scrub_blocks(pa);
+ if (n >= budget || local_clock() >= deadline)
+ return true;
+ budget -= n;
+ progress |= n != 0;
+ cond_resched();
+ read_lock(&pisa_allocs_lock);
+ }
+ read_unlock(&pisa_allocs_lock);
+ } while (progress);
+
+ return false;
+}
+
/**
* pisa_alloc_create - Create a PISA page allocator
* @nid: Home NUMA node for this allocator
@@ -731,6 +889,7 @@ static unsigned int take_blocks(struct pisa_meta *meta, unsigned int want,
for (unsigned int i = got; i < got + dirty; i++)
set_page_private(out[i], 1);
got += dirty;
+ pa->scrub_demand = min(pa->scrub_demand + dirty, PISA_SCRUB_DEMAND_MAX);
} else {
/* First choice are dirty blocks, followed by clean */
got += pisa_scan_bitmap(meta, true, want, out, !frozen);
@@ -754,6 +913,7 @@ static unsigned int take_blocks(struct pisa_meta *meta, unsigned int want,
static unsigned long __pisa_alloc_bulk(struct pisa_alloc *pa, gfp_t gfp, unsigned long nr_pages,
struct page **pages, bool zero, bool frozen)
{
+ bool should_scrub = false;
unsigned long n, flags;
struct pisa_meta *meta;
unsigned int want;
@@ -767,6 +927,7 @@ static unsigned long __pisa_alloc_bulk(struct pisa_alloc *pa, gfp_t gfp, unsigne
continue;
/* Fallback to a new allocation */
+ should_scrub |= pa->scrub_demand >= PISA_SCRUB_TRIGGER;
spin_unlock_irqrestore(&pa->lock, flags);
/* With @zero, fresh dirty blocks are zeroed below. */
meta = pisa_alloc_split_page(pa->order, gfp, pa->nid);
@@ -780,6 +941,8 @@ static unsigned long __pisa_alloc_bulk(struct pisa_alloc *pa, gfp_t gfp, unsigne
pisa_meta_set_flags(meta, PISA_ACTIVE);
pa->stats.page_alloc++;
}
+ /* Queue the scrubber when the inline zeroing crosses the trigger. */
+ should_scrub |= pa->scrub_demand >= PISA_SCRUB_TRIGGER;
spin_unlock_irqrestore(&pa->lock, flags);
out:
@@ -794,6 +957,8 @@ static unsigned long __pisa_alloc_bulk(struct pisa_alloc *pa, gfp_t gfp, unsigne
else
page_ref_unfreeze(pages[i], 1);
}
+ if (should_scrub)
+ pisa_schedule_scrub(pa->nid);
return n;
}
@@ -882,8 +1047,14 @@ unsigned long pisa_alloc_bulk(struct pisa_alloc *pa, gfp_t gfp,
/* Fill the run of @n NULL entries at @i. */
for (n = 1; i + n < nr_pages && !page_array[i + n]; n++)
;
- /* Pairs with smp_store_release() in pisa_enable_pcp(). */
- if (smp_load_acquire(&pa->pcp) && pa->nid == numa_mem_id())
+ /*
+ * With the scrubber on, zeroed allocations skip the per-CPU
+ * cache: __pisa_alloc_bulk() prefers clean blocks and records
+ * the demand that drives the scrubber.
+ * smp_load_acquire() pairs with smp_store_release() in pisa_enable_pcp().
+ */
+ if (smp_load_acquire(&pa->pcp) && pa->nid == numa_mem_id() &&
+ !(want_init_on_alloc(gfp) && READ_ONCE(pisa_scrub_max)))
got = pisa_pcp_alloc(pa, gfp, n, page_array + i, frozen);
else
got = __pisa_alloc_bulk(pa, gfp, n, page_array + i,
@@ -965,6 +1136,7 @@ static struct page *pisa_take_contig(struct pisa_meta *meta, unsigned int nblock
page_ref_unfreeze(page + i, 1);
}
pa->blocks_in_use += nblocks;
+ pa->scrub_demand = min(pa->scrub_demand + dirty, PISA_SCRUB_DEMAND_MAX);
*clean = !dirty;
pisa_place_meta(meta);
return page;
@@ -995,7 +1167,7 @@ __pisa_alloc_map_coherent(struct device *dev, struct iommu_domain *domain,
size_t size, dma_addr_t *dma_handle, gfp_t gfp,
unsigned long attrs)
{
- bool should_release = false, clean = false;
+ bool should_scrub = false, should_release = false, clean = false;
struct list_head *lists[3];
unsigned long align_mask, flags;
struct pisa_meta *meta, *tmp;
@@ -1047,10 +1219,14 @@ __pisa_alloc_map_coherent(struct device *dev, struct iommu_domain *domain,
}
}
if (!page) {
+ should_scrub |= pa->scrub_demand >= PISA_SCRUB_TRIGGER;
spin_unlock_irqrestore(&pa->lock, flags);
meta = pisa_alloc_split_page(0, gfp, pa->nid);
- if (IS_ERR(meta))
+ if (IS_ERR(meta)) {
+ if (should_scrub)
+ pisa_schedule_scrub(pa->nid);
return NULL;
+ }
spin_lock_irqsave(&pa->lock, flags);
meta->pa = pa;
meta->prot = pa->prot;
@@ -1061,10 +1237,13 @@ __pisa_alloc_map_coherent(struct device *dev, struct iommu_domain *domain,
if (unlikely(!page))
should_release = pisa_place_meta(meta);
}
+ should_scrub |= pa->scrub_demand >= PISA_SCRUB_TRIGGER;
spin_unlock_irqrestore(&pa->lock, flags);
if (should_release)
pisa_schedule_reclaim();
+ if (should_scrub)
+ pisa_schedule_scrub(pa->nid);
if (!page)
return NULL;
if (!domain && unlikely(!dma_coherent_ok(dev, page_to_phys(page), size))) {
diff --git a/mm/pisa-kunit.c b/mm/pisa-kunit.c
index 0f8cc9baf5d28..4effbc8610079 100644
--- a/mm/pisa-kunit.c
+++ b/mm/pisa-kunit.c
@@ -662,8 +662,57 @@ static void test_gfp_pisa(struct kunit *test)
static void test_reservoir_and_scrubber(struct kunit *test)
{
+ unsigned int old_res = READ_ONCE(pisa_reservoir_max);
+ unsigned int old_scrub = READ_ONCE(pisa_scrub_max);
+ struct shrink_control sc = { .nr_to_scan = ULONG_MAX };
+ struct pisa_meta *meta;
+ struct pisa_alloc *pa;
+ struct page *p;
+ int nid;
+
KUNIT_ASSERT_EQ(test, pisa_meta_init(), 0);
- KUNIT_EXPECT_EQ(test, READ_ONCE(pisa_reservoir_max), 16U);
+ pisa_shrink_scan(NULL, &sc);
+
+ /* Refill reservoir to 1 PISA page per memory node, then consume one. */
+ WRITE_ONCE(pisa_reservoir_max, 1);
+ pisa_schedule_reservoir();
+ for_each_node_state(nid, N_MEMORY)
+ flush_work(&pisa_nodes[nid].work);
+ KUNIT_EXPECT_GE(test, pisa_reservoir_count(), 1UL);
+
+ pa = pisa_node_pa(first_memory_node, SKB_FRAG_PAGE_ORDER, true);
+ KUNIT_ASSERT_NOT_NULL(test, pa);
+ p = pisa_alloc(pa, GFP_KERNEL);
+ if (p) {
+ meta = pisa_meta_of_pfn(page_to_pfn(p));
+ /* Reservoir pages come pre-zeroed (all remaining blocks clean). */
+ KUNIT_EXPECT_EQ(test, meta->nr_dirty, 0U);
+ __free_pages(p, SKB_FRAG_PAGE_ORDER);
+ pisa_pcp_drain(pa);
+ if (!want_init_on_free()) {
+ KUNIT_EXPECT_GE(test, meta->nr_dirty, 1U);
+
+ /* Scrub the freed block and its PCP batch back into clean_bitmap. */
+ pa->scrub_demand = meta->nr_dirty;
+ WRITE_ONCE(pisa_scrub_max, 512);
+ pisa_scrub_node(pa->nid);
+ KUNIT_EXPECT_EQ(test, meta->nr_dirty, 0U);
+ }
+ }
+ pisa_shrink_scan(NULL, &sc);
+
+ /* Trim reservoirs back to 0. */
+ for_each_node_state(nid, N_MEMORY)
+ flush_work(&pisa_nodes[nid].work);
+ WRITE_ONCE(pisa_reservoir_max, 0);
+ pisa_schedule_reservoir();
+ for_each_node_state(nid, N_MEMORY)
+ flush_work(&pisa_nodes[nid].work);
+ KUNIT_EXPECT_EQ(test, pisa_reservoir_count(), 0UL);
+
+ WRITE_ONCE(pisa_reservoir_max, old_res);
+ pisa_schedule_reservoir();
+ WRITE_ONCE(pisa_scrub_max, old_scrub);
}
static struct kunit_case pisa_meta_test_cases[] = {
diff --git a/mm/pisa-meta.c b/mm/pisa-meta.c
index 30f0f2f6ef628..49fa5271a392f 100644
--- a/mm/pisa-meta.c
+++ b/mm/pisa-meta.c
@@ -213,10 +213,13 @@ bool pisa_place_meta(struct pisa_meta *meta)
* (head of partial list, preferred for allocation). In between,
* keep the current dense/sparse state.
*/
+ /* Blocks that the scrubber is zeroing are free: count them here. */
+ unsigned int nfree = avail + meta->nr_scrub;
+
to_list = meta->nr_clean ? PISA_LIST_CLEAN : PISA_LIST_DIRTY;
- if (avail >= bpp - (bpp >> 2))
+ if (nfree >= bpp - (bpp >> 2))
to_list |= PISA_LIST_SPARSE;
- else if (avail > (bpp >> 1))
+ else if (nfree > (bpp >> 1))
to_list |= meta->on_list & PISA_LIST_SPARSE;
}
diff --git a/mm/pisa-node.c b/mm/pisa-node.c
index a39251b952263..c013e384b1c7e 100644
--- a/mm/pisa-node.c
+++ b/mm/pisa-node.c
@@ -5,6 +5,9 @@
*/
#include <linux/atomic.h>
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/export.h>
#include <linux/pisa.h>
#include <linux/err.h>
#include <linux/gfp.h>
@@ -37,6 +40,9 @@ struct pisa_node pisa_nodes[MAX_NUMNODES] = {
},
};
+/* Bit number in pisa_node.flags: background scrub requested on this node. */
+#define PISA_NODE_SCRUB 0
+
static int pisa_reservoir_max_set(const char *val, const struct kernel_param *kp)
{
int ret = param_set_uint(val, kp);
@@ -237,6 +243,30 @@ static void pisa_reservoir_trim_node(struct pisa_node *node, unsigned int max)
}
}
+/* Schedule node @nid's work item to scrub allocators homed on @nid. */
+void pisa_schedule_scrub(int nid)
+{
+ struct pisa_node *node;
+
+ if (!READ_ONCE(pisa_scrub_max))
+ return;
+
+ node = &pisa_nodes[nid];
+ set_bit(PISA_NODE_SCRUB, &node->flags);
+ /* Pairs with the clearing of PENDING before the work runs. */
+ smp_mb__after_atomic();
+ if (!work_pending(&node->work))
+ queue_work_node(nid, system_dfl_wq, &node->work);
+}
+
+void pisa_schedule_scrub_all(void)
+{
+ int nid;
+
+ for_each_node_state(nid, N_MEMORY)
+ pisa_schedule_scrub(nid);
+}
+
/* Per-node worker: trim/refill the reservoir, then scrub allocators homed on @nid. */
static void pisa_node_work_fn(struct work_struct *work)
{
@@ -246,6 +276,9 @@ static void pisa_node_work_fn(struct work_struct *work)
pisa_reservoir_trim_node(node, READ_ONCE(pisa_reservoir_max));
if (pisa_enabled() && READ_ONCE(pisa_reservoir_max))
pisa_reservoir_refill_node(node, nid);
+
+ if (test_and_clear_bit(PISA_NODE_SCRUB, &node->flags) && pisa_scrub_node(nid))
+ pisa_schedule_scrub(nid);
}
/* Queue node @nid's work if its reservoir is below or above the target. */
diff --git a/mm/pisa-priv.h b/mm/pisa-priv.h
index 60120a05082a9..9f01173d30f5f 100644
--- a/mm/pisa-priv.h
+++ b/mm/pisa-priv.h
@@ -108,6 +108,7 @@ struct pisa_alloc {
struct list_head full; /* PISA pages with 0 free blocks */
unsigned int num_idle_pages;
unsigned int max_idle_pages;
+ unsigned int scrub_demand; /* dirty blocks zeroed inline */
unsigned long idle_jiffies; /* last change of @idle at/above @max_idle_pages */
unsigned long blocks_in_use; /* blocks handed out and not yet freed */
/* Cold. */
@@ -136,7 +137,8 @@ struct pisa_node {
spinlock_t lock; /* protects @pages and @nr_pages */
unsigned int nr_pages; /* entries in @pages */
struct list_head pages; /* reservoir */
- struct work_struct work; /* refill/trim, on this node */
+ unsigned long flags; /* PISA_NODE_* requests */
+ struct work_struct work; /* refill/trim/scrub, on this node */
} ____cacheline_aligned_in_smp;
/* Per-NUMA-node PISA state (ring/TX/user allocators, reservoir). */
@@ -229,11 +231,17 @@ static inline struct page *pisa_alloc(struct pisa_alloc *pa, gfp_t gfp)
unsigned long pisa_shrink_list(spinlock_t *lock, struct list_head *list,
unsigned int *nr, struct list_head *release_list,
unsigned long nr_to_scan);
+/* Time after which a scrubber run stops and requeues. */
+#define PISA_SCRUB_RUN_NS (50 * NSEC_PER_USEC)
+extern unsigned int pisa_scrub_max;
+bool pisa_scrub_node(int nid);
/* pisa-node.c: per-NUMA-node PISA page reservoir and work item */
extern unsigned int pisa_reservoir_max;
void pisa_node_init(void);
void pisa_schedule_reservoir(void);
+void pisa_schedule_scrub(int nid);
+void pisa_schedule_scrub_all(void);
struct pisa_meta *pisa_reservoir_pop(int nid, gfp_t gfp, bool *clean);
unsigned long pisa_reservoir_count(void);
unsigned long pisa_reservoir_shrink(struct list_head *release_list, unsigned long nr_to_scan);
--
2.56.1.809.ga0b61035b5-goog
next prev parent reply other threads:[~2026-11-20 1:07 UTC|newest]
Thread overview: 112+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-10-03 21:22 [RFC: DMA_PMD 00/22] DMA_PMD: PMD_SIZE-backed IO buffer pools Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 01/22] iommu/dma: introduce CONFIG_DMA_PMD and metadata table Luigi Rizzo
2026-10-03 21:44 ` Randy Dunlap
2026-10-04 9:14 ` Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 02/22] iommu/dma: add DMA_PMD pool lifecycle and page recycle hook Luigi Rizzo
2026-10-10 2:59 ` kernel test robot
2026-10-03 21:22 ` [RFC: DMA_PMD 03/22] mm: Add split_page_compound() Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 04/22] iommu/dma: add DMA_PMD pool block allocation Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 05/22] iommu/dma: Global cap and shrinker for DMA_PMD pool memory Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 06/22] iommu/dma: reserve a per-domain IOVA window for DMA_PMD pages Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 07/22] iommu/dma: release DMA_PMD domain mappings on domain teardown Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 08/22] iommu/dma: use per-domain IOVA window to map DMA_PMD memory Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 09/22] iommu/dma: Add DMA_PMD arena allocator Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 10/22] driver core: Add per-device dma_pmd_* sysfs attributes Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 11/22] dma-mapping: Use DMA_PMD arena for dma_alloc_attrs() Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 12/22] net/core: Use per-CPU DMA_PMD pools for skb_page_frag_refill() Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 13/22] net/core: Use DMA_PMD for page_pool memory Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 14/22] iommu/dma: Support decrypted and pinned DMA_PMD pages Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 15/22] iommu/dma: Add background page scrubber for DMA_PMD pools Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 16/22] iommu/dma: Add per-NUMA-node PMD page reservoir Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 17/22] net/gve: Use DMA_PMD memory for RX buffers Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 18/22] net/gve: Use DMA_PMD memory for tx header bounce buffers Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 19/22] net/mlx5e: " Luigi Rizzo
2026-10-14 17:22 ` Dragos Tatulea
2026-10-14 17:50 ` Dragos Tatulea
2026-10-14 18:28 ` Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 20/22] net/idpf: " Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 21/22] net/bnxt: " Luigi Rizzo
2026-10-03 21:22 ` [RFC: DMA_PMD 22/22] iommu/dma: Add DMA_PMD statistics and debugfs Luigi Rizzo
2026-10-11 9:02 ` [RFC: DMA_PMD 00/22] DMA_PMD: PMD_SIZE-backed IO buffer pools Pranjal Shrivastava
2026-10-12 13:51 ` David Hildenbrand (Arm)
2026-10-12 14:45 ` Luigi Rizzo
2026-10-12 19:43 ` David Hildenbrand (Arm)
2026-10-13 10:42 ` Robin Murphy
2026-10-14 10:47 ` David Hildenbrand (Arm)
2026-10-14 11:05 ` Luigi Rizzo
2026-10-17 17:11 ` Luigi Rizzo
2026-10-22 20:12 ` Jason Gunthorpe
2026-10-22 21:51 ` Luigi Rizzo
2026-10-23 14:01 ` Jason Gunthorpe
2026-10-23 14:30 ` Luigi Rizzo
2026-10-23 14:50 ` Jason Gunthorpe
2026-10-23 15:12 ` Luigi Rizzo
2026-10-23 17:27 ` Jakub Kicinski
2026-10-23 20:22 ` Luigi Rizzo
2026-11-03 8:09 ` Christoph Hellwig
2026-11-03 9:13 ` Luigi Rizzo
2026-11-03 15:10 ` Jason Gunthorpe
2026-11-06 17:21 ` Naman Gulati
2026-11-06 17:30 ` Jason Gunthorpe
2026-11-06 18:00 ` Naman Gulati
2026-11-06 18:07 ` Jason Gunthorpe
2026-11-06 18:17 ` Naman Gulati
2026-11-06 18:43 ` Jason Gunthorpe
2026-11-06 19:01 ` Luigi Rizzo
2026-11-06 19:11 ` Naman Gulati
2026-10-12 14:49 ` Mostafa Saleh
2026-10-12 15:02 ` Luigi Rizzo
2026-10-14 17:28 ` Dragos Tatulea
2026-10-14 18:14 ` Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 00/25] PISA: Pre-mapped IOVA Superpage Allocator for DMA and CoCo Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 " Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 01/25] mm: Introduce CONFIG_PISA and metadata table Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 02/25] mm: Add PISA allocator lifecycle and page recycle hook Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 03/25] mm: Add per-NUMA-node PISA page reservoir Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 04/25] mm/page_alloc: Extend alloc_pages() to __GFP_PISA and PISA block allocation Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 05/25] mm: Add global cap and shrinker for PISA pages Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 06/25] iommu/dma: Map PISA memory in a per-domain IOVA window Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 07/25] driver core: Add per-device pisa_* sysfs attributes for PISA optin Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 08/25] iommu/dma: Use PISA for coherent DMA allocations Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 09/25] mm: Add per-CPU block caches to PISA allocators Luigi Rizzo
2026-11-20 1:06 ` Luigi Rizzo [this message]
2026-11-20 1:06 ` [RFC PATCH v3 11/25] mm: Migrate user pages into PISA Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 12/25] block, mm: Support PISA for disk I/O and scrub migration Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 13/25] net: Cache leaf PISA TX flags on struct net_device Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 14/25] net: Use per-node PISA allocators for skb_page_frag_refill() Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 15/25] net: Use PISA for page_pool memory Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 16/25] mm/slub: Add SLAB_CACHE_PISA_RO for PISA-backed slab caches Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 17/25] net: Allocate small skb heads from PISA Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 18/25] net: Migrate user pages into PISA for MSG_ZEROCOPY Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 19/25] mm, iommu/dma: Support decrypted PISA pages Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 20/25] arm64/mm, mm: Support decrypted PISA pages in Realms Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 21/25] mm: Add PISA statistics and debugfs Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 22/25] net/libeth: Use GFP_KERNEL when pre-filling RX rings Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 23/25] net/gve: " Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 24/25] net/gve: Use PISA memory for RX buffers Luigi Rizzo
2026-11-20 1:06 ` [RFC PATCH v3 25/25] Documentation: Add PISA architecture documentation Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 01/25] iommu/dma: Introduce CONFIG_PISA and metadata table Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 02/25] iommu/dma: Add PISA allocator lifecycle and page recycle hook Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 03/25] iommu/dma: Add per-NUMA-node PISA page reservoir Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 04/25] iommu/dma: Add PISA block allocation Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 05/25] iommu/dma: Add global cap and shrinker for PISA pages Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 06/25] iommu/dma: Reserve a per-domain IOVA window " Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 07/25] iommu/dma: Release PISA domain mappings on domain teardown Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 08/25] iommu/dma: Use per-domain IOVA window to map PISA memory Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 09/25] driver core: Add per-device pisa_* sysfs attributes for PISA optin Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 10/25] iommu/dma: Use PISA for coherent DMA allocations Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 11/25] iommu/dma: Add per-CPU block caches to PISA allocators Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 12/25] net/core: Use per-node PISA allocators for skb_page_frag_refill() Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 13/25] net/core: Use PISA for page_pool memory Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 14/25] iommu/dma: Support decrypted PISA pages Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 15/25] iommu/dma: Add background page scrubber for PISA allocators Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 16/25] mm/slub: Add SLAB_CACHE_PISA_RO for PISA-backed slab caches Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 17/25] net: Allocate small skb heads from PISA Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 18/25] iommu/dma: Migrate user pages into PISA for TX zerocopy Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 19/25] iommu/dma: Stop recycling old PISA pages when allocator waste is high Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 20/25] net/libeth: Use GFP_KERNEL when pre-filling RX rings Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 21/25] net/gve: " Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 22/25] net/gve: Use PISA memory for RX buffers Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 23/25] block, mm, iommu/dma: Support PISA for disk I/O and scrub migration Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 24/25] Documentation: Add PISA architecture documentation Luigi Rizzo
2026-11-09 18:25 ` [RFC PATCH v2 25/25] iommu/dma: Add PISA statistics and debugfs Luigi Rizzo
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261120010655.467825-11-lrizzo@google.com \
--to=lrizzo@google.com \
--cc=akpm@linux-foundation.org \
--cc=aleksander.lobakin@intel.com \
--cc=axboe@kernel.dk \
--cc=brendan.jackman@linux.dev \
--cc=catalin.marinas@arm.com \
--cc=cl@gentwo.org \
--cc=corbet@lwn.net \
--cc=dakr@kernel.org \
--cc=davem@davemloft.net \
--cc=david@kernel.org \
--cc=driver-core@lists.linux.dev \
--cc=dtatulea@nvidia.com \
--cc=edumazet@kernel.org \
--cc=gregkh@linuxfoundation.org \
--cc=hannes@cmpxchg.org \
--cc=hao.li@linux.dev \
--cc=harry@kernel.org \
--cc=hawk@kernel.org \
--cc=hch@lst.de \
--cc=horms@kernel.org \
--cc=hramamurthy@google.com \
--cc=ilias.apalodimas@linaro.org \
--cc=iommu@lists.linux.dev \
--cc=jack@suse.cz \
--cc=jannh@google.com \
--cc=jgg@ziepe.ca \
--cc=joro@8bytes.org \
--cc=joshwash@google.com \
--cc=kuba@kernel.org \
--cc=kuniyu@google.com \
--cc=liam@infradead.org \
--cc=linux-api@vger.kernel.org \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-block@vger.kernel.org \
--cc=linux-doc@vger.kernel.org \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=ljs@kernel.org \
--cc=m.szyprowski@samsung.com \
--cc=mhocko@suse.com \
--cc=namangulati@google.com \
--cc=netdev@vger.kernel.org \
--cc=pabeni@redhat.com \
--cc=praan@google.com \
--cc=rafael@kernel.org \
--cc=rdunlap@infradead.org \
--cc=rientjes@google.com \
--cc=rizzo.unipi@gmail.com \
--cc=robin.murphy@arm.com \
--cc=roman.gushchin@linux.dev \
--cc=rppt@kernel.org \
--cc=smostafa@google.com \
--cc=surenb@google.com \
--cc=vbabka@kernel.org \
--cc=will@kernel.org \
--cc=willemb@google.com \
--cc=willy@infradead.org \
--cc=ziy@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox