* [RFC PATCH 13/16] mm: memcomp: add KUnit tests
2026-11-16 11:01 [RFC PATCH 00/16] mm: add in-memory compression of reclaimed pages nakixii
` (11 preceding siblings ...)
2026-11-16 11:01 ` [RFC PATCH 12/16] mm: memcomp: report precise token residency nakixii
@ 2026-11-16 11:01 ` nakixii
2026-11-16 11:01 ` [RFC PATCH 14/16] selftests/mm: add a memcomp pageout and refault test nakixii
` (4 subsequent siblings)
17 siblings, 0 replies; 19+ messages in thread
From: nakixii @ 2026-11-16 11:01 UTC (permalink / raw)
To: linux-mm
Cc: linux-kernel, cgroups, Andrew Morton, David Hildenbrand,
Johannes Weiner, Chris Li, Kairui Song, Nhat Pham, Yosry Ahmed,
Minchan Kim, Sergey Senozhatsky, Barry Song, Hugh Dickins,
Michal Hocko, Roman Gushchin, Shakeel Butt, Mike Rapoport,
Peter Xu, Suren Baghdasaryan
The tokens used by memcomp can outlive individual reclaim attempts and
name resident, compressed, same-filled or swap-backed contents. Exercise
their bounds, reference ownership and state transitions without relying
solely on a userspace workload observing a compressed PTE.
Add 425 named cases covering slot layouts, per-key storage, allocation
cancellation, lookup guards, resident records, rollback and fault
restoration. Check single-page and multi-page geometry, migration, page-
cache replacement, source-object-cgroup accounting, writeback and
swapoff lifetime rules.
Cover zero PTE and cache ownership separately from slot references,
including successful publication and losing transactions. Exercise
bounded metadata aging and writeback scans, asynchronous completion and
finish failures, same-filled values and incompressible content.
Retain assertions that only genuine -E2BIG results set the
incompressible marker, that rollback installs the folio hash and that
unchanged contents skip recompression. Reject KSM folios with a direct
eligibility check. Guard advice-dependent cases so configurations
without those syscalls still link and report their configuration-
specific skips.
Build fixtures and production-side probes only with MEMCOMP_KUNIT_TEST.
Include the split test headers from the production compilation unit and
provide a dedicated KUnit configuration. Controlled allocation, object-
cgroup and I/O models expose otherwise difficult lifetime paths; they do
not substitute for end-to-end runtime validation.
Signed-off-by: nakixii <nakixii@gmail.com>
---
mm/Kconfig | 13 +
mm/memcomp.c | 438 +++
mm/memcomp/anon_store.h | 16 +
mm/memcomp/cache_folio.h | 9 +
mm/memcomp/consumers.h | 4 +
mm/memcomp/defrag.h | 6 +
mm/memcomp/fault.h | 53 +
mm/memcomp/load.h | 4 +
mm/memcomp/migrate.h | 3 +
mm/memcomp/mincore.h | 6 +
mm/memcomp/resident.h | 3 +
mm/memcomp/storage.h | 35 +
mm/memcomp/tests/.kunitconfig | 16 +
mm/memcomp/tests/access_test.h | 197 +
mm/memcomp/tests/aging_test.h | 771 ++++
mm/memcomp/tests/anon_store_test.h | 704 ++++
mm/memcomp/tests/async_test.h | 703 ++++
mm/memcomp/tests/budget_packed_test.h | 435 +++
mm/memcomp/tests/budget_swapped_test.h | 292 ++
mm/memcomp/tests/budget_test.h | 542 +++
mm/memcomp/tests/cache_folio_test.h | 627 +++
mm/memcomp/tests/cache_reserve_test.h | 447 +++
mm/memcomp/tests/cache_store_test.h | 216 +
mm/memcomp/tests/cache_xarray_race_test.h | 193 +
mm/memcomp/tests/codec_scope_test.h | 94 +
mm/memcomp/tests/consumers_test.h | 478 +++
mm/memcomp/tests/core_test.h | 119 +
mm/memcomp/tests/defrag_test.h | 601 +++
mm/memcomp/tests/drain_test.h | 581 +++
mm/memcomp/tests/enospc_test.h | 236 ++
mm/memcomp/tests/fault_alloc_test.h | 119 +
mm/memcomp/tests/fault_test.h | 851 ++++
mm/memcomp/tests/folio_test.h | 769 ++++
mm/memcomp/tests/format_test.h | 520 +++
mm/memcomp/tests/late_async_test.h | 331 ++
mm/memcomp/tests/load_test.h | 133 +
mm/memcomp/tests/migrate_test.h | 795 ++++
mm/memcomp/tests/mincore_test.h | 462 +++
mm/memcomp/tests/owner_test.h | 400 ++
mm/memcomp/tests/payload_test.h | 220 ++
mm/memcomp/tests/public_api_test.h | 422 ++
mm/memcomp/tests/public_driver_test.h | 358 ++
mm/memcomp/tests/query_test.h | 480 +++
mm/memcomp/tests/refs_test.h | 224 ++
mm/memcomp/tests/replacement_test.h | 360 ++
mm/memcomp/tests/resident_test.h | 796 ++++
mm/memcomp/tests/rollback_test.h | 163 +
mm/memcomp/tests/scan_test.h | 422 ++
mm/memcomp/tests/shared_test.h | 3477 +++++++++++++++++
mm/memcomp/tests/shmem_retry_test.h | 340 ++
mm/memcomp/tests/shmem_zero_fault_test.h | 121 +
mm/memcomp/tests/span_test.h | 128 +
mm/memcomp/tests/storage_fixture.h | 84 +
mm/memcomp/tests/storage_test.h | 468 +++
mm/memcomp/tests/store_transaction_test.h | 587 +++
mm/memcomp/tests/suite.h | 693 ++++
mm/memcomp/tests/swapoff_test.h | 558 +++
mm/memcomp/tests/swapoff_walk_test.h | 562 +++
mm/memcomp/tests/swapoff_yield_test.h | 532 +++
mm/memcomp/tests/test_api.h | 41 +
mm/memcomp/tests/transport_reclaim_test.h | 298 ++
mm/memcomp/tests/transport_test.h | 114 +
mm/memcomp/tests/workingset_load_test.h | 1095 ++++++
mm/memcomp/tests/workingset_mthp_error_test.h | 164 +
mm/memcomp/tests/workingset_storage_test.h | 442 +++
mm/memcomp/tests/workingset_store_test.h | 572 +++
mm/memcomp/tests/writeback_batch_test.h | 688 ++++
mm/memcomp/tests/writeback_test.h | 540 +++
mm/memcomp/tests/zero_callers_test.h | 717 ++++
mm/memcomp/tests/zero_carrier_test.h | 405 ++
mm/memcomp/tests/zero_convert_test.h | 694 ++++
mm/memcomp/tests/zero_hint_test.h | 860 ++++
mm/memcomp/tests/zero_load_test.h | 645 +++
mm/memcomp/tests/zero_test.h | 313 ++
mm/memcomp/workingset_store.h | 10 +
mm/memcomp/writeback_batch.h | 12 +
mm/memcomp/zero_load.h | 20 +
mm/mincore.c | 21 +
78 files changed, 30868 insertions(+)
create mode 100644 mm/memcomp/tests/.kunitconfig
create mode 100644 mm/memcomp/tests/access_test.h
create mode 100644 mm/memcomp/tests/aging_test.h
create mode 100644 mm/memcomp/tests/anon_store_test.h
create mode 100644 mm/memcomp/tests/async_test.h
create mode 100644 mm/memcomp/tests/budget_packed_test.h
create mode 100644 mm/memcomp/tests/budget_swapped_test.h
create mode 100644 mm/memcomp/tests/budget_test.h
create mode 100644 mm/memcomp/tests/cache_folio_test.h
create mode 100644 mm/memcomp/tests/cache_reserve_test.h
create mode 100644 mm/memcomp/tests/cache_store_test.h
create mode 100644 mm/memcomp/tests/cache_xarray_race_test.h
create mode 100644 mm/memcomp/tests/codec_scope_test.h
create mode 100644 mm/memcomp/tests/consumers_test.h
create mode 100644 mm/memcomp/tests/core_test.h
create mode 100644 mm/memcomp/tests/defrag_test.h
create mode 100644 mm/memcomp/tests/drain_test.h
create mode 100644 mm/memcomp/tests/enospc_test.h
create mode 100644 mm/memcomp/tests/fault_alloc_test.h
create mode 100644 mm/memcomp/tests/fault_test.h
create mode 100644 mm/memcomp/tests/folio_test.h
create mode 100644 mm/memcomp/tests/format_test.h
create mode 100644 mm/memcomp/tests/late_async_test.h
create mode 100644 mm/memcomp/tests/load_test.h
create mode 100644 mm/memcomp/tests/migrate_test.h
create mode 100644 mm/memcomp/tests/mincore_test.h
create mode 100644 mm/memcomp/tests/owner_test.h
create mode 100644 mm/memcomp/tests/payload_test.h
create mode 100644 mm/memcomp/tests/public_api_test.h
create mode 100644 mm/memcomp/tests/public_driver_test.h
create mode 100644 mm/memcomp/tests/query_test.h
create mode 100644 mm/memcomp/tests/refs_test.h
create mode 100644 mm/memcomp/tests/replacement_test.h
create mode 100644 mm/memcomp/tests/resident_test.h
create mode 100644 mm/memcomp/tests/rollback_test.h
create mode 100644 mm/memcomp/tests/scan_test.h
create mode 100644 mm/memcomp/tests/shared_test.h
create mode 100644 mm/memcomp/tests/shmem_retry_test.h
create mode 100644 mm/memcomp/tests/shmem_zero_fault_test.h
create mode 100644 mm/memcomp/tests/span_test.h
create mode 100644 mm/memcomp/tests/storage_fixture.h
create mode 100644 mm/memcomp/tests/storage_test.h
create mode 100644 mm/memcomp/tests/store_transaction_test.h
create mode 100644 mm/memcomp/tests/suite.h
create mode 100644 mm/memcomp/tests/swapoff_test.h
create mode 100644 mm/memcomp/tests/swapoff_walk_test.h
create mode 100644 mm/memcomp/tests/swapoff_yield_test.h
create mode 100644 mm/memcomp/tests/test_api.h
create mode 100644 mm/memcomp/tests/transport_reclaim_test.h
create mode 100644 mm/memcomp/tests/transport_test.h
create mode 100644 mm/memcomp/tests/workingset_load_test.h
create mode 100644 mm/memcomp/tests/workingset_mthp_error_test.h
create mode 100644 mm/memcomp/tests/workingset_storage_test.h
create mode 100644 mm/memcomp/tests/workingset_store_test.h
create mode 100644 mm/memcomp/tests/writeback_batch_test.h
create mode 100644 mm/memcomp/tests/writeback_test.h
create mode 100644 mm/memcomp/tests/zero_callers_test.h
create mode 100644 mm/memcomp/tests/zero_carrier_test.h
create mode 100644 mm/memcomp/tests/zero_convert_test.h
create mode 100644 mm/memcomp/tests/zero_hint_test.h
create mode 100644 mm/memcomp/tests/zero_load_test.h
create mode 100644 mm/memcomp/tests/zero_test.h
diff --git a/mm/Kconfig b/mm/Kconfig
index 4bd19dc449490d226d1dd5ce94235c2426cbc7c1..06ecf5ec43c6aa2fe1a282b7fb3ec8d3e24cb459 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -232,6 +232,19 @@ config MEMCOMP_DEBUG_STATS
These counters add hot-path accounting and are intended for
diagnosis and tuning. If unsure, say N.
+config MEMCOMP_KUNIT_TEST
+ bool "KUnit tests for memcomp" if !KUNIT_ALL_TESTS
+ depends on MEMCOMP && KUNIT=y
+ select MEMCOMP_DEBUG_STATS
+ default KUNIT_ALL_TESTS
+ help
+ Build unit tests for memcomp token bounds, reference lifetime,
+ compression execution admission, payload states, allocation flags and
+ failure attribution, same-filled pages and swap writeback selection.
+ The tests run at boot and do not require a swap file.
+
+ If unsure, say N.
+
config ZBUD
tristate "2:1 compression allocator (zbud)"
depends on ZSWAP
diff --git a/mm/memcomp.c b/mm/memcomp.c
index a246c801910b72a5084d4db17d4183e0d69bb958..0fa44a3c05273876ee6abed7a641bc4f56b01b52 100644
--- a/mm/memcomp.c
+++ b/mm/memcomp.c
@@ -300,6 +300,11 @@ struct memcomp_ctx_timing_record {
struct memcomp_direct_staging {
unsigned char *dst;
struct memcomp_ctx_timing_record timing;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ int kunit_compress_preempt_count;
+ int kunit_detach_preempt_count;
+ void *kunit_compress_dst;
+#endif
};
/*
@@ -352,8 +357,33 @@ struct memcomp_store_ctx {
/* The span's anon_vma reference, for the caller to drop once stored. */
struct anon_vma *anon_vma;
struct mc16_zero_carrier zero;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ unsigned int kunit_zs_failures;
+ int kunit_refresh_commit_error;
+ struct memcomp_kunit_async_probe *kunit_async_probe;
+#endif
+};
+
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+/* One fixture's synchronous store call, not a production allocator ledger. */
+struct memcomp_kunit_async_probe {
+ struct task_struct *task;
+ struct folio *folio;
+ const struct memcomp_async_req *request;
+ struct zs_pool *pool;
+ unsigned long handle;
+ unsigned int allocs;
+ unsigned int frees;
+ unsigned int precomp_hits;
+ unsigned int commit_fail_hits;
+ int commit_error;
+ bool live;
+ bool bad_lifetime;
};
+static struct memcomp_kunit_async_probe __rcu *memcomp_kunit_async_probe;
+#endif
+
/*
* Hashed payload locks. One entry's state and payload are protected by
* the mutex its token hashes to. Never hold two token locks at once.
@@ -426,6 +456,13 @@ struct memcomp_reclaim_ctx {
bool borrowed;
bool background;
bool direct_staging;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ int kunit_compress_error;
+ int kunit_compress_preempt_count;
+ void *kunit_compress_dst;
+ int kunit_refresh_commit_error;
+ bool *kunit_detached;
+#endif
};
enum memcomp_ctx_timing_kind {
@@ -618,6 +655,9 @@ static atomic64_t memcomp_reclaim_ctx_early_reject_pages =
ATOMIC64_INIT(0);
static atomic64_t memcomp_direct_staging_miss = ATOMIC64_INIT(0);
static atomic64_t memcomp_direct_staging_detach = ATOMIC64_INIT(0);
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+static atomic_t memcomp_kunit_ctx_get_fail = ATOMIC_INIT(0);
+#endif
static atomic64_t memcomp_compress_fail = ATOMIC64_INIT(0);
static atomic64_t memcomp_zs_fast_fail = ATOMIC64_INIT(0);
static atomic64_t memcomp_zs_retries = ATOMIC64_INIT(0);
@@ -858,19 +898,132 @@ static void memcomp_arch_restore(unsigned long token, struct folio *folio)
/* "Payload can satisfy a fault without swap IO". */
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+static void memcomp_kunit_async_bind(struct memcomp_store_ctx *store,
+ struct folio *folio)
+{
+ struct memcomp_kunit_async_probe *probe;
+
+ rcu_read_lock();
+ probe = rcu_dereference(memcomp_kunit_async_probe);
+ if (probe && probe->task == current && probe->folio == folio &&
+ probe->request == store->precomp)
+ store->kunit_async_probe = probe;
+ rcu_read_unlock();
+}
+
+static void memcomp_kunit_async_alloc(struct memcomp_store_ctx *store,
+ struct zs_pool *pool, unsigned long handle)
+{
+ struct memcomp_kunit_async_probe *probe;
+
+ /* A bound store is synchronous in the fixture task, which owns probe. */
+ probe = store ? store->kunit_async_probe : NULL;
+ if (!probe || probe->task != current)
+ return;
+ probe->allocs++;
+ if (probe->allocs != 1) {
+ probe->bad_lifetime = true;
+ return;
+ }
+ probe->pool = pool;
+ probe->handle = handle;
+ probe->live = true;
+}
+
+static void memcomp_kunit_async_precomp(struct memcomp_store_ctx *store,
+ const struct memcomp_async_req *request)
+{
+ struct memcomp_kunit_async_probe *probe;
+
+ probe = store ? store->kunit_async_probe : NULL;
+ if (probe && probe->task == current && probe->request == request)
+ probe->precomp_hits++;
+}
+
+static void memcomp_kunit_async_free(struct zs_pool *pool, unsigned long handle)
+{
+ struct memcomp_kunit_async_probe *probe;
+
+ rcu_read_lock();
+ probe = rcu_dereference(memcomp_kunit_async_probe);
+ if (probe && probe->task == current &&
+ probe->pool == pool && probe->handle == handle) {
+ if (!probe->live)
+ probe->bad_lifetime = true;
+ probe->frees++;
+ probe->live = false;
+ }
+ rcu_read_unlock();
+}
+#endif
+
static void memcomp_zs_free(u8 pool_idx, unsigned long handle)
{
zs_free(memcomp_pools[pool_idx], handle);
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ memcomp_kunit_async_free(memcomp_pools[pool_idx], handle);
+#endif
}
+#if defined(CONFIG_MEMCOMP_KUNIT_TEST) && defined(CONFIG_MEMCG)
+/*
+ * An isolated owner models byte charges in KUnit. Only this identity is
+ * intercepted; real memcgs always use the production APIs. The test drains
+ * RCU callbacks before removing the owner so deferred frees see the model.
+ */
+struct memcomp_kunit_objcg {
+ struct obj_cgroup objcg;
+ atomic_long_t charged;
+ atomic_t charge_calls;
+ atomic_t uncharge_calls;
+ atomic_t get_calls;
+ atomic_t put_calls;
+ struct task_struct *task;
+ struct folio *folio;
+ unsigned int fail_at;
+ int error;
+};
+
+static struct memcomp_kunit_objcg __rcu *memcomp_kunit_objcg;
+#endif
+
static struct obj_cgroup *memcomp_objcg_get(struct folio *folio)
{
+#if defined(CONFIG_MEMCOMP_KUNIT_TEST) && defined(CONFIG_MEMCG)
+ struct memcomp_kunit_objcg *model;
+ struct obj_cgroup *objcg = NULL;
+
+ rcu_read_lock();
+ model = rcu_dereference(memcomp_kunit_objcg);
+ if (model && model->task == current && model->folio == folio) {
+ atomic_inc(&model->get_calls);
+ objcg = &model->objcg;
+ }
+ rcu_read_unlock();
+ if (objcg)
+ return objcg;
+#endif
return get_obj_cgroup_from_folio(folio);
}
/* Hold an additional metadata owner without bypassing the test model. */
static __maybe_unused void memcomp_objcg_get_ref(struct obj_cgroup *objcg)
{
+#if defined(CONFIG_MEMCOMP_KUNIT_TEST) && defined(CONFIG_MEMCG)
+ struct memcomp_kunit_objcg *model;
+ bool modeled = false;
+
+ rcu_read_lock();
+ model = rcu_dereference(memcomp_kunit_objcg);
+ if (model && objcg == &model->objcg) {
+ atomic_inc(&model->get_calls);
+ modeled = true;
+ }
+ rcu_read_unlock();
+ if (modeled)
+ return;
+#endif
#ifdef CONFIG_MEMCG
if (objcg)
obj_cgroup_get(objcg);
@@ -879,6 +1032,20 @@ static __maybe_unused void memcomp_objcg_get_ref(struct obj_cgroup *objcg)
static void memcomp_objcg_put(struct obj_cgroup *objcg)
{
+#if defined(CONFIG_MEMCOMP_KUNIT_TEST) && defined(CONFIG_MEMCG)
+ struct memcomp_kunit_objcg *model;
+ bool modeled = false;
+
+ rcu_read_lock();
+ model = rcu_dereference(memcomp_kunit_objcg);
+ if (model && objcg == &model->objcg) {
+ atomic_inc(&model->put_calls);
+ modeled = true;
+ }
+ rcu_read_unlock();
+ if (modeled)
+ return;
+#endif
obj_cgroup_put(objcg);
}
@@ -889,6 +1056,28 @@ static int memcomp_objcg_charge(struct obj_cgroup *objcg, gfp_t gfp,
if (!objcg || !size)
return 0;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ {
+ struct memcomp_kunit_objcg *model;
+ bool modeled = false;
+ int ret = 0;
+
+ rcu_read_lock();
+ model = rcu_dereference(memcomp_kunit_objcg);
+ if (model && objcg == &model->objcg) {
+ int call = atomic_inc_return(&model->charge_calls);
+
+ modeled = true;
+ if (call == model->fail_at)
+ ret = model->error;
+ else
+ atomic_long_add(size, &model->charged);
+ }
+ rcu_read_unlock();
+ if (modeled)
+ return ret;
+ }
+#endif
return obj_cgroup_charge(objcg, gfp, size);
#else
return 0;
@@ -900,6 +1089,23 @@ static void memcomp_objcg_uncharge(struct obj_cgroup *objcg, size_t size)
#if defined(CONFIG_MEMCG)
if (!objcg || !size)
return;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ {
+ struct memcomp_kunit_objcg *model;
+ bool modeled = false;
+
+ rcu_read_lock();
+ model = rcu_dereference(memcomp_kunit_objcg);
+ if (model && objcg == &model->objcg) {
+ atomic_inc(&model->uncharge_calls);
+ atomic_long_sub(size, &model->charged);
+ modeled = true;
+ }
+ rcu_read_unlock();
+ if (modeled)
+ return;
+ }
+#endif
obj_cgroup_uncharge(objcg, size);
#endif
}
@@ -907,7 +1113,15 @@ static void memcomp_objcg_uncharge(struct obj_cgroup *objcg, size_t size)
#include "memcomp/format.h"
#include "memcomp/zero.h"
#include "memcomp/lifetime.h"
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+static void *mc16_kunit_storage_alloc(size_t bytes, gfp_t gfp,
+ struct obj_cgroup *objcg);
+static void mc16_kunit_queue_rcu(struct rcu_head *head, rcu_callback_t func);
+#endif
#include "memcomp/storage.h"
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+#include "memcomp/tests/storage_fixture.h"
+#endif
#ifdef CONFIG_SWAP
/* An entry owns one reference on its raw or packed backing slot. */
@@ -922,6 +1136,52 @@ static void memcomp_objcg_uncharge(struct obj_cgroup *objcg, size_t size)
* build keeps the direct calls.
*/
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+/* Fixture mode; synthetic swap tests must not access a real device. */
+static bool mc16_kunit_real_swap;
+
+static atomic_t memcomp_kunit_slot_puts = ATOMIC_INIT(0);
+static atomic_t memcomp_kunit_slot_dups = ATOMIC_INIT(0);
+
+static int memcomp_slot_dup(swp_entry_t swap)
+{
+ if (READ_ONCE(mc16_kunit_real_swap))
+ return swap_duplicate(swap);
+ if (swap.val)
+ atomic_inc(&memcomp_kunit_slot_dups);
+ return 0;
+}
+
+static void memcomp_slot_put(swp_entry_t swap)
+{
+ if (READ_ONCE(mc16_kunit_real_swap)) {
+ swap_free(swap);
+ return;
+ }
+ if (swap.val)
+ atomic_inc(&memcomp_kunit_slot_puts);
+}
+
+static atomic_t memcomp_kunit_slot_reclaims = ATOMIC_INIT(0);
+
+static void memcomp_slot_put_packed(swp_entry_t swap)
+{
+ if (READ_ONCE(mc16_kunit_real_swap)) {
+ free_swap_and_cache(swap);
+ return;
+ }
+ if (swap.val)
+ atomic_inc(&memcomp_kunit_slot_reclaims);
+}
+
+/* No device, so no count: a fault never keeps a slot's page cached. */
+static int memcomp_slot_count(swp_entry_t swap)
+{
+ if (READ_ONCE(mc16_kunit_real_swap))
+ return __swap_count(swap);
+ return 0;
+}
+#else
static int memcomp_slot_dup(swp_entry_t swap)
{
return swap_duplicate(swap);
@@ -949,6 +1209,7 @@ static int memcomp_slot_count(swp_entry_t swap)
{
return __swap_count(swap);
}
+#endif
/*
* Detach a swap-backed entry from its slot. Caller holds the token lock
@@ -1770,6 +2031,11 @@ memcomp_direct_staging_try_get(void)
memcomp_direct_staging_active);
#endif
staging = &memcomp_direct_staging[idx];
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ staging->kunit_compress_preempt_count = -1;
+ staging->kunit_detach_preempt_count = -1;
+ staging->kunit_compress_dst = NULL;
+#endif
}
spin_unlock_irqrestore(&memcomp_direct_staging_lock, flags);
if (!staging)
@@ -2178,7 +2444,20 @@ memcomp_payload_zs_malloc(struct zs_pool *pool, unsigned int size,
unsigned long *handle, bool allow_slow,
struct memcomp_store_ctx *store)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ int ret;
+
+ if (store && store->kunit_zs_failures) {
+ store->kunit_zs_failures--;
+ return -ENOMEM;
+ }
+ ret = memcomp_zs_malloc(pool, size, handle, allow_slow);
+ if (!ret)
+ memcomp_kunit_async_alloc(store, pool, *handle);
+ return ret;
+#else
return memcomp_zs_malloc(pool, size, handle, allow_slow);
+#endif
}
static int memcomp_decompress_payload(const void *src,
@@ -2626,6 +2905,13 @@ memcomp_try_get_comp_ctx(bool background, bool allow_borrow,
ctx->borrowed = borrowed;
ctx->background = background;
ctx->direct_staging = false;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ ctx->kunit_compress_error = 0;
+ ctx->kunit_compress_preempt_count = -1;
+ ctx->kunit_compress_dst = NULL;
+ ctx->kunit_refresh_commit_error = 0;
+ ctx->kunit_detached = NULL;
+#endif
return ctx;
}
@@ -2658,6 +2944,15 @@ memcomp_get_comp_ctx(enum memcomp_payload_mode mode,
if (miss)
*miss = MEMCOMP_COMP_CTX_AVAILABLE;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ if (atomic_dec_if_positive(&memcomp_kunit_ctx_get_fail) >= 0) {
+ if (miss)
+ *miss = MEMCOMP_COMP_CTX_ALL_BUSY;
+ memcomp_debug_inc(&memcomp_ctx_busy);
+ return NULL;
+ }
+#endif
+
can_retry = memcomp_can_retry_alloc();
background = memcomp_comp_ctx_background(mode, source, can_retry);
if (mode != MEMCOMP_PAYLOAD_BEST_EFFORT &&
@@ -2721,6 +3016,13 @@ static void memcomp_put_comp_ctx(struct memcomp_reclaim_ctx *ctx)
ctx->borrowed = false;
ctx->background = false;
ctx->direct_staging = false;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ ctx->kunit_compress_error = 0;
+ ctx->kunit_compress_preempt_count = -1;
+ ctx->kunit_compress_dst = NULL;
+ ctx->kunit_refresh_commit_error = 0;
+ ctx->kunit_detached = NULL;
+#endif
/*
* Unlock before publishing the free bit: a claimer trylocks ctx->lock
* as soon as it sees the bit, and expects the previous owner to be
@@ -2806,6 +3108,9 @@ static int memcomp_alloc_raw_payload(u8 pool_idx, const void *src,
dst = zs_map_object(pool, *handle, ZS_MM_WO);
if (!dst) {
zs_free(pool, *handle);
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ memcomp_kunit_async_free(pool, *handle);
+#endif
return -EIO;
}
@@ -2895,12 +3200,22 @@ memcomp_detach_reclaim_ctx(struct memcomp_reclaim_ctx **ctxp,
lockdep_assert_preemption_disabled();
lockdep_assert_held(&ctx->lock);
staging->timing = ctx->timing;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ staging->kunit_compress_preempt_count =
+ ctx->kunit_compress_preempt_count;
+ staging->kunit_compress_dst = ctx->kunit_compress_dst;
+ if (ctx->kunit_detached)
+ *ctx->kunit_detached = true;
+#endif
ctx->timing.timing_active = false;
*ctxp = NULL;
memcomp_put_comp_ctx(ctx);
if (staging->timing.timing_active)
staging->timing.timing_lease_end_ns = ktime_get_ns();
memcomp_debug_inc(&memcomp_direct_staging_detach);
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ staging->kunit_detach_preempt_count = preempt_count();
+#endif
}
/*
@@ -2969,6 +3284,10 @@ memcomp_prepare_payload_locked(struct memcomp_reclaim_ctx **ctxp,
*/
if (precomp ? precomp->same_filled :
page_same_filled(src, &element)) {
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ if (precomp)
+ memcomp_kunit_async_precomp(store, precomp);
+#endif
*length = 0;
*handle = precomp ? precomp->element : element;
return 0;
@@ -3006,6 +3325,13 @@ memcomp_prepare_payload_locked(struct memcomp_reclaim_ctx **ctxp,
if (staging)
preempt_disable();
compress_timing = memcomp_compress_timing_start(timing);
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ ctx->kunit_compress_preempt_count = preempt_count();
+ ctx->kunit_compress_dst = compressed;
+ ret = ctx->kunit_compress_error;
+ ctx->kunit_compress_error = 0;
+ if (!ret)
+#endif
ret = crypto_comp_compress(ctx->tfm, src, src_len,
compressed, &dst_len);
ret = memcomp_compress_result(ret, &dst_len, src_len);
@@ -3088,12 +3414,20 @@ memcomp_prepare_payload_locked(struct memcomp_reclaim_ctx **ctxp,
dst = zs_map_object(pool, *handle, ZS_MM_WO);
if (!dst) {
zs_free(pool, *handle);
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ memcomp_kunit_async_free(pool, *handle);
+#endif
memcomp_ctx_timing_phase_end(timing, timing_start,
&timing->timing_zs_ns);
return -EIO;
}
memcpy(dst, compressed, dst_len);
zs_unmap_object(pool, *handle);
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ /* Observe the supplied bytes only after the real object copy. */
+ if (precomp && compressed == precomp->buf && dst_len == precomp->len)
+ memcomp_kunit_async_precomp(store, precomp);
+#endif
memcomp_ctx_timing_phase_end(timing, timing_start,
&timing->timing_zs_ns);
@@ -3132,9 +3466,87 @@ static int memcomp_prepare_payload(const void *src, unsigned int src_len,
return ret;
}
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+enum memcomp_kunit_fail_stage {
+ MEMCOMP_KUNIT_FAIL_NONE,
+ MEMCOMP_KUNIT_FAIL_ENTRY_ALLOC,
+ MEMCOMP_KUNIT_FAIL_TOKEN_FIRST,
+ MEMCOMP_KUNIT_FAIL_TOKEN_LATER,
+ MEMCOMP_KUNIT_FAIL_ARCH_PREPARE,
+ MEMCOMP_KUNIT_FAIL_REFRESH_COMMIT,
+};
+
+struct memcomp_kunit_fail {
+ struct task_struct *task;
+ enum memcomp_kunit_fail_stage stage;
+ unsigned int remaining;
+ int error;
+};
+
+static struct memcomp_kunit_fail memcomp_kunit_fail;
+
+static void
+memcomp_kunit_fail_next(enum memcomp_kunit_fail_stage stage,
+ unsigned int nth, int error)
+{
+ WARN_ON_ONCE(READ_ONCE(memcomp_kunit_fail.task));
+ if (WARN_ON_ONCE(!nth))
+ nth = 1;
+ if (WARN_ON_ONCE(stage == MEMCOMP_KUNIT_FAIL_NONE))
+ stage = MEMCOMP_KUNIT_FAIL_ENTRY_ALLOC;
+ if (stage == MEMCOMP_KUNIT_FAIL_ENTRY_ALLOC) {
+ error = -ENOMEM;
+ } else if (stage == MEMCOMP_KUNIT_FAIL_ARCH_PREPARE) {
+ if (WARN_ON_ONCE(error >= 0))
+ error = -ENOMEM;
+ } else if (stage == MEMCOMP_KUNIT_FAIL_REFRESH_COMMIT) {
+ if (WARN_ON_ONCE(error != -EAGAIN))
+ error = -EAGAIN;
+ } else if (WARN_ON_ONCE(error != -ENOMEM && error != -EBUSY)) {
+ error = -ENOMEM;
+ }
+
+ memcomp_kunit_fail.stage = stage;
+ memcomp_kunit_fail.remaining = nth;
+ memcomp_kunit_fail.error = error;
+ WRITE_ONCE(memcomp_kunit_fail.task, current);
+}
+
+static void memcomp_kunit_fail_clear(void)
+{
+ WRITE_ONCE(memcomp_kunit_fail.task, NULL);
+ memcomp_kunit_fail.stage = MEMCOMP_KUNIT_FAIL_NONE;
+ memcomp_kunit_fail.remaining = 0;
+ memcomp_kunit_fail.error = 0;
+}
+
+static int
+memcomp_kunit_should_fail(enum memcomp_kunit_fail_stage stage)
+{
+ int error;
+
+ if (READ_ONCE(memcomp_kunit_fail.task) != current ||
+ memcomp_kunit_fail.stage != stage)
+ return 0;
+ if (--memcomp_kunit_fail.remaining)
+ return 0;
+
+ error = memcomp_kunit_fail.error;
+ memcomp_kunit_fail_clear();
+ return error;
+}
+#endif
+
static __always_inline int
memcomp_arch_prepare_to_swap(struct folio *folio)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ int ret;
+
+ ret = memcomp_kunit_should_fail(MEMCOMP_KUNIT_FAIL_ARCH_PREPARE);
+ if (ret)
+ return ret;
+#endif
return arch_prepare_to_swap(folio);
}
@@ -3249,11 +3661,24 @@ static void memcomp_drop_swapcache_folio(struct folio *folio, bool dirty)
folio_put(folio);
}
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+/*
+ * The suite builds the pool with sentinel slots; the real free-list return
+ * would log an error per sentinel against UML's absent device.
+ */
+static void memcomp_writeback_slots_release(swp_entry_t *slots,
+ unsigned int n)
+{
+ if (READ_ONCE(mc16_kunit_real_swap))
+ swapcache_free_entries(slots, n);
+}
+#else
static void memcomp_writeback_slots_release(swp_entry_t *slots,
unsigned int n)
{
swapcache_free_entries(slots, n);
}
+#endif
/* Return every prefetched slot this pass did not spend. */
static void memcomp_writeback_slots_return(void)
@@ -3786,6 +4211,11 @@ int memcomp_swapoff(unsigned int type)
return mc16_swapoff_walk(type, &ops);
}
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+/* Isolate index and IO edges; the walk, refs and mutex operations stay real. */
+
+#endif
+
/*
* Conservative defragmentation of packed slots.
*
@@ -4379,6 +4809,11 @@ __memcomp_store_after_unmap(struct folio *folio,
return false;
ctx = *ctxp;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ store.kunit_refresh_commit_error = ctx->kunit_refresh_commit_error;
+ ctx->kunit_refresh_commit_error = 0;
+ memcomp_kunit_async_bind(&store, folio);
+#endif
memcomp_ctx_timing_store_start(&ctx->timing);
success = mc16_anon_store_internal(folio, ctxp, &store, &failure);
if (!success)
@@ -5741,3 +6176,6 @@ static int __init memcomp_sysfs_init(void)
}
subsys_initcall(memcomp_sysfs_init);
#endif
+
+#include "memcomp/tests/suite.h"
+#include "memcomp/tests/codec_scope_test.h"
diff --git a/mm/memcomp/anon_store.h b/mm/memcomp/anon_store.h
index a0c01f9c52b039b81f01f6823267a7bb217e71f7..e84057f0528e916bbf63fd8cb9872180b9befb67 100644
--- a/mm/memcomp/anon_store.h
+++ b/mm/memcomp/anon_store.h
@@ -118,7 +118,23 @@ mc16_refresh_validate(struct memcomp_ref *ref, struct folio **folio,
static inline int mc16_refresh_commit_error(struct memcomp_store_ctx *store)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ int ret;
+
+ if (store && store->kunit_refresh_commit_error) {
+ ret = store->kunit_refresh_commit_error;
+ store->kunit_refresh_commit_error = 0;
+ return ret;
+ }
+ ret = memcomp_kunit_should_fail(MEMCOMP_KUNIT_FAIL_REFRESH_COMMIT);
+ if (ret && store && store->kunit_async_probe) {
+ store->kunit_async_probe->commit_fail_hits++;
+ store->kunit_async_probe->commit_error = ret;
+ }
+ return ret;
+#else
return 0;
+#endif
}
/*
diff --git a/mm/memcomp/cache_folio.h b/mm/memcomp/cache_folio.h
index 2bdc37b682a7ea598ba9af7ad721e3d50ff6d6ec..ec1250f8c0e0e5cd5ee8d874fb55669ea262f9e9 100644
--- a/mm/memcomp/cache_folio.h
+++ b/mm/memcomp/cache_folio.h
@@ -5,7 +5,16 @@
#include <linux/highmem.h>
/* Include after cache_store.h and memcomp_arch_prepare_to_swap(). */
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+#include <kunit/static_stub.h>
+static int mc16_cache_folio_arch_prepare(struct folio *folio)
+{
+ KUNIT_STATIC_STUB_REDIRECT(mc16_cache_folio_arch_prepare, folio);
+ return memcomp_arch_prepare_to_swap(folio);
+}
+#else
#define mc16_cache_folio_arch_prepare memcomp_arch_prepare_to_swap
+#endif
/*
* Caller holds the order-0 source folio lock throughout preparation and XA
diff --git a/mm/memcomp/consumers.h b/mm/memcomp/consumers.h
index cdbd2262c632b089538c4abbbe5bcbb99bdc6c00..b57aee58c55d0325d4444035593e7bead4869288 100644
--- a/mm/memcomp/consumers.h
+++ b/mm/memcomp/consumers.h
@@ -74,6 +74,10 @@ static inline int mc16_cache_load(struct memcomp_ref *ref, struct folio *folio)
void *dst;
int ret;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_cache_load, ref, folio);
+#endif
+
if (!ref || !ref->slot)
return -ENOENT;
if (!folio || folio_test_large(folio) || !folio_test_locked(folio))
diff --git a/mm/memcomp/defrag.h b/mm/memcomp/defrag.h
index 39374f48c007f22ea728139027dd2992ff668689..c7e2ba516eb02597064492eee175b8622b64cffd 100644
--- a/mm/memcomp/defrag.h
+++ b/mm/memcomp/defrag.h
@@ -6,6 +6,9 @@
/* Include after memcomp_defrag_ops, its scan/move limits, swapoff.h and scan.h. */
static inline void *mc16_defrag_buffer_alloc(void)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_defrag_buffer_alloc);
+#endif
return kmalloc(PAGE_SIZE, GFP_NOWAIT | __GFP_NOWARN);
}
@@ -17,6 +20,9 @@ static inline int mc16_defrag_copy_payload(const void *src, unsigned int bytes,
unsigned int copied = 0;
int ret;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_defrag_copy_payload, src, bytes, payload);
+#endif
ret = memcomp_alloc_raw_payload(0, src, bytes, &copied, &handle,
false, NULL);
if (!ret)
diff --git a/mm/memcomp/fault.h b/mm/memcomp/fault.h
index e051b0ee2c350101bebe5bb75b8cb813a883752c..c6c6dc893073e2c2d499df60a8ea04006a414cdb 100644
--- a/mm/memcomp/fault.h
+++ b/mm/memcomp/fault.h
@@ -12,6 +12,24 @@
static inline vm_fault_t mc16_load_resident(struct vm_fault *vmf,
struct memcomp_ref *ref, bool *handled);
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+/* Failure/race injection only. Successful data, allocation and charge are real. */
+static struct {
+ struct task_struct *task;
+ bool fail_large_alloc;
+ bool fail_small_alloc;
+ bool fail_large_charge;
+ bool fail_small_charge;
+ unsigned int large_allocs;
+ unsigned int small_allocs;
+ unsigned int large_charges;
+ unsigned int small_charges;
+ void (*after_alloc)(struct vm_fault *vmf, void *data);
+ void (*before_install)(struct vm_fault *vmf, void *data);
+ void *data;
+} mc16_fault_probe;
+#endif
+
static inline vm_fault_t mc16_load(struct vm_fault *vmf, bool *locked)
{
const unsigned int max_pages = MEMCOMP_MAX_COMPRESS_PAGES;
@@ -105,14 +123,41 @@ static inline vm_fault_t mc16_load(struct vm_fault *vmf, bool *locked)
id_start = id;
fallback_alloc:
alloc_gfp = memcomp_fault_alloc_gfp(vma, nr_pages);
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ if (READ_ONCE(mc16_fault_probe.task) == current) {
+ if (nr_pages > 1)
+ mc16_fault_probe.large_allocs++;
+ else
+ mc16_fault_probe.small_allocs++;
+ if (nr_pages > 1 ? mc16_fault_probe.fail_large_alloc :
+ mc16_fault_probe.fail_small_alloc)
+ goto alloc_failed;
+ }
+#endif
folio = vma_alloc_folio(alloc_gfp, order_base_2(nr_pages),
vma, start_addr, false);
if (!folio) {
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+alloc_failed:
+#endif
if (nr_pages > 1)
goto fallback_single;
ret = VM_FAULT_OOM;
goto out_put;
}
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ if (READ_ONCE(mc16_fault_probe.task) == current) {
+ if (nr_pages > 1)
+ mc16_fault_probe.large_charges++;
+ else
+ mc16_fault_probe.small_charges++;
+ }
+ if (READ_ONCE(mc16_fault_probe.task) == current &&
+ (nr_pages > 1 ? mc16_fault_probe.fail_large_charge :
+ mc16_fault_probe.fail_small_charge))
+ err = -ENOMEM;
+ else
+#endif
err = mc16_user_charge(&refs[0], folio, vma->vm_mm, alloc_gfp);
if (err) {
if (nr_pages > 1) {
@@ -128,6 +173,10 @@ static inline vm_fault_t mc16_load(struct vm_fault *vmf, bool *locked)
*locked = true;
__folio_set_swapbacked(folio);
folio->swap.val = 0;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ if (READ_ONCE(mc16_fault_probe.task) == current && mc16_fault_probe.after_alloc)
+ mc16_fault_probe.after_alloc(vmf, mc16_fault_probe.data);
+#endif
if (nr_pages > 1) {
struct memcomp_ref fault_ref = refs[0];
bool batch_ok = true;
@@ -192,6 +241,10 @@ static inline vm_fault_t mc16_load(struct vm_fault *vmf, bool *locked)
ret = VM_FAULT_MAJOR;
memcomp_arch_restore(id_start, folio);
__folio_mark_uptodate(folio);
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ if (READ_ONCE(mc16_fault_probe.task) == current && mc16_fault_probe.before_install)
+ mc16_fault_probe.before_install(vmf, mc16_fault_probe.data);
+#endif
/* No T survives mc16_load_page; PTL is acquired only after it returns. */
if (!memcomp_pte_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
&vmf->pte, &vmf->ptl))
diff --git a/mm/memcomp/load.h b/mm/memcomp/load.h
index ea6d6efa448dbfac3c94621e8e68179cbc035046..518178b4150313ed00219ae7b380e1be1b55e0c0 100644
--- a/mm/memcomp/load.h
+++ b/mm/memcomp/load.h
@@ -12,6 +12,10 @@ static inline int mc16_load_page(struct memcomp_ref *ref, void *dst)
void *src;
int ret;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_load_page, ref, dst);
+#endif
+
if (!ref->slot || !dst)
return -EINVAL;
mutex_lock(memcomp_lock_of(ref->token));
diff --git a/mm/memcomp/migrate.h b/mm/memcomp/migrate.h
index 66c06ec4d34744869d1e310bad23b6e8fb6c3072..4fcfb6af0c0e40babb3ffcf724580df48bcfcdaf 100644
--- a/mm/memcomp/migrate.h
+++ b/mm/memcomp/migrate.h
@@ -25,6 +25,9 @@ static inline int
mc16_migrate_move_error(struct memcomp_ref *ref, struct folio *from,
struct folio *to, unsigned int index)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_migrate_move_error, ref, from, to, index);
+#endif
return 0;
}
diff --git a/mm/memcomp/mincore.h b/mm/memcomp/mincore.h
index ebba2424b65ef4f0ce5fba1892c6728f43a346b8..655afdd2140f01ef516e31a6c0320e38af7bd932 100644
--- a/mm/memcomp/mincore.h
+++ b/mm/memcomp/mincore.h
@@ -4,6 +4,9 @@
#include <linux/pagemap.h>
#include <linux/swap.h>
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+#include <kunit/static_stub.h>
+#endif
/* Include after storage/access/refs and the memcomp_slot_* helpers. */
#ifdef CONFIG_SWAP
@@ -12,6 +15,9 @@ static struct folio *mc16_mincore_swap_folio(swp_entry_t swap)
struct swap_info_struct *si;
struct folio *folio;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_mincore_swap_folio, swap);
+#endif
si = get_swap_device(swap);
if (!si)
return ERR_PTR(-ENOENT);
diff --git a/mm/memcomp/resident.h b/mm/memcomp/resident.h
index fad9ddb2cc8def141dd330578eae2c8abe6aff82..be1ad1629a518ba1d22eed8926fefc26d23fdb9c 100644
--- a/mm/memcomp/resident.h
+++ b/mm/memcomp/resident.h
@@ -6,6 +6,9 @@
static inline vm_fault_t
mc16_resident_lock_or_retry(struct folio *folio, struct vm_fault *vmf)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_resident_lock_or_retry, folio, vmf);
+#endif
return folio_lock_or_retry(folio, vmf);
}
diff --git a/mm/memcomp/storage.h b/mm/memcomp/storage.h
index 1379e0dd546bf95c65d78d6140b6561575ea7477..e1b012c2cc9ccd7c185486d75596b7b67fd3c296 100644
--- a/mm/memcomp/storage.h
+++ b/mm/memcomp/storage.h
@@ -35,6 +35,9 @@
#include <linux/xarray.h>
#include "format.h"
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+#include <kunit/static_stub.h>
+#endif
#define MC16_CLUSTER_SHIFT 9
#define MC16_CLUSTER_SLOTS BIT(MC16_CLUSTER_SHIFT)
@@ -176,11 +179,17 @@ static struct {
static unsigned long mc16_aging_now(void)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_aging_now);
+#endif
return jiffies;
}
static void mc16_queue_aging(unsigned long delay)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_queue_aging, delay);
+#endif
mod_delayed_work(system_unbound_wq, &mc16_retained_aging, delay);
}
@@ -188,6 +197,9 @@ static void mc16_schedule_aging(void)
{
unsigned long irqflags, now, delay;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_schedule_aging);
+#endif
xa_lock_irqsave(&mc16_clusters, irqflags);
if (mc16_aging.requested) {
xa_unlock_irqrestore(&mc16_clusters, irqflags);
@@ -257,12 +269,20 @@ static inline void *mc16_kmalloc_owner(size_t bytes, gfp_t gfp,
static inline void *mc16_owner_alloc(size_t bytes, gfp_t gfp,
struct obj_cgroup *objcg)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ return mc16_kunit_storage_alloc(bytes, gfp, objcg);
+#else
return mc16_kmalloc_owner(bytes, gfp, objcg);
+#endif
}
static inline void mc16_queue_rcu(struct rcu_head *head, rcu_callback_t func)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ mc16_kunit_queue_rcu(head, func);
+#else
call_rcu(head, func);
+#endif
}
static inline void *mc16_alloc_zero(enum mc16_storage_class type,
@@ -284,16 +304,25 @@ static inline void *mc16_alloc_zero(enum mc16_storage_class type,
*/
static inline int mc16_source_id_get(struct folio *folio, unsigned short *id)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_source_id_get, folio, id);
+#endif
return mem_cgroup_memcomp_id_get_checked(folio, id);
}
static inline bool mc16_owner_id_get_ref(unsigned short id)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_owner_id_get_ref, id);
+#endif
return mem_cgroup_memcomp_id_get_ref(id);
}
static inline void mc16_owner_id_put(unsigned short id)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_owner_id_put, id);
+#endif
mem_cgroup_memcomp_id_put(id);
}
@@ -331,6 +360,9 @@ mc16_key_find_id(struct obj_cgroup *objcg, unsigned int nid, u8 kind,
static inline int mc16_private_id_alloc(u32 *id)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_private_id_alloc, id);
+#endif
return __xa_alloc(&mc16_clusters, id, NULL,
XA_LIMIT(1, MC16_MAX_CLUSTER_ID),
GFP_NOWAIT | __GFP_NOWARN);
@@ -338,6 +370,9 @@ static inline int mc16_private_id_alloc(u32 *id)
static inline struct mc16_key *mc16_key_zalloc(void)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_key_zalloc);
+#endif
return kzalloc(sizeof(struct mc16_key), GFP_NOIO | __GFP_NOWARN);
}
diff --git a/mm/memcomp/tests/.kunitconfig b/mm/memcomp/tests/.kunitconfig
new file mode 100644
index 0000000000000000000000000000000000000000..b12961ed5179edac7536bd29acb37bcb36fbbad2
--- /dev/null
+++ b/mm/memcomp/tests/.kunitconfig
@@ -0,0 +1,16 @@
+CONFIG_KUNIT=y
+CONFIG_CRYPTO=y
+CONFIG_CRYPTO_ZSTD=y
+CONFIG_SWAP=y
+CONFIG_MMU=y
+CONFIG_64BIT=y
+CONFIG_ZSMALLOC=y
+CONFIG_ZRAM=n
+CONFIG_ZSWAP=n
+CONFIG_MEMCOMP=y
+CONFIG_MEMCOMP_KUNIT_TEST=y
+CONFIG_CRYPTO_LZ4=y
+# Compile the batched folio-release path with memory accounting enabled.
+CONFIG_CGROUPS=y
+CONFIG_MEMCG=y
+CONFIG_KSM=y
diff --git a/mm/memcomp/tests/access_test.h b/mm/memcomp/tests/access_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..8d9a19fcd7d39c22ab2cf5999d03c2063cd00415
--- /dev/null
+++ b/mm/memcomp/tests/access_test.h
@@ -0,0 +1,197 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_ACCESS_TEST_H
+#define _MM_MEMCOMP_TESTS_ACCESS_TEST_H
+#ifdef CONFIG_MEMCG
+static void memcomp_test_pending_final_guard(struct kunit *test)
+{
+ struct mc16_test_storage *f;
+ struct mc16_view view;
+ struct memcomp_ref guard;
+ spinlock_t ptl; /* Models the caller holding a PTE lock. */
+ unsigned long irqflags;
+ long record_charge, before, charged_after;
+ int puts_after, folio_refs_after;
+ bool last;
+
+ f = mc16_test_storage_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(f, 1), 0);
+ KUNIT_ASSERT_TRUE(test, mc16_get(f->refs[0].token, &guard));
+ refcount_inc(&guard.slot->refs); /* persistent PTE ownership */
+ mutex_lock(memcomp_lock_of(guard.token));
+ mc16_view_locked(&guard, &view);
+ record_charge = view.record->charge_bytes;
+ *view.flags &= ~MC16_F_OWNER;
+ mutex_unlock(memcomp_lock_of(guard.token));
+ /* Owner + PTE are gone; only the guard still owns this member. */
+ KUNIT_ASSERT_FALSE(test, refcount_sub_and_test(2, &guard.slot->refs));
+ f->refs[0] = (struct memcomp_ref) { };
+ before = atomic_long_read(&f->owner->model.charged);
+ spin_lock_init(&ptl);
+ rcu_read_lock();
+ spin_lock_irqsave(&ptl, irqflags);
+ last = refcount_dec_and_test(&guard.slot->refs);
+ if (last) {
+ mc16_release_resources(&guard);
+ mc16_retire_slot(&guard);
+ }
+ spin_unlock_irqrestore(&ptl, irqflags);
+ /* Capture before grace; report only after leaving atomic/RCU scope. */
+ charged_after = atomic_long_read(&f->owner->model.charged);
+ puts_after = atomic_read(&f->owner->model.put_calls);
+ folio_refs_after = folio_ref_count(f->owner->folio);
+ rcu_read_unlock();
+ KUNIT_EXPECT_EQ(test, charged_after, before - record_charge);
+ KUNIT_EXPECT_EQ(test, puts_after, 1);
+ KUNIT_EXPECT_EQ(test, folio_refs_after, 1);
+ KUNIT_EXPECT_TRUE(test, last);
+ rcu_barrier();
+ KUNIT_EXPECT_FALSE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_FALSE(test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_GT(test, atomic_long_read(&f->owner->model.charged), 0L);
+ /* Exact metadata balance is checked after explicit pressure reclaim. */
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->owner->model.charged), 0L);
+}
+
+static void memcomp_test_samefill_normalize(struct kunit *test)
+{
+ struct mc16_test_storage *f;
+ struct mc16_record *tail[2];
+ struct mc16_view view;
+ const u64 pattern = 0xfedcba9876543210ULL;
+ unsigned int i;
+ u16 flags;
+ bool matches;
+
+ f = mc16_test_storage_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(f, 2), 0);
+ for (i = 0; i < 2; i++) {
+ mutex_lock(memcomp_lock_of(f->refs[i].token));
+ mc16_view_locked(&f->refs[i], &view);
+ matches = mc16_owner_matches_locked(&f->refs[i], f->owner->folio, 2, i);
+ KUNIT_EXPECT_TRUE(test, matches);
+ matches = mc16_owner_matches_locked(&f->refs[i], NULL, 2, i);
+ KUNIT_EXPECT_FALSE(test, matches);
+ matches = mc16_owner_matches_locked(&f->refs[i], f->owner->folio, 1, i);
+ KUNIT_EXPECT_FALSE(test, matches);
+ folio_put(view.member->owner_folio);
+ *view.word = pattern;
+ *view.length = 0;
+ *view.flags = MC16_SAMEFILL | (1 << MC16_ORDER_SHIFT);
+ tail[i] = mc16_normalize_locked(&f->refs[i]);
+ mutex_unlock(memcomp_lock_of(f->refs[i].token));
+ KUNIT_ASSERT_NOT_NULL(test, tail[i]);
+ KUNIT_EXPECT_EQ(test, f->refs[i].slot->word, pattern);
+ /* Pair with normalize release to read the complete fill word. */
+ flags = smp_load_acquire(&f->refs[i].slot->flags);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(flags),
+ (unsigned int)MC16_SAMEFILL);
+ }
+ KUNIT_EXPECT_PTR_EQ(test, tail[0], tail[1]);
+ mc16_record_member_put(tail[1]);
+ KUNIT_EXPECT_EQ(test, refcount_read(&tail[0]->members), 1U);
+ mc16_record_member_put(tail[0]); /* producer/head detaches last */
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->owner->model.put_calls), 1);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->owner->folio), 1);
+ for (i = 0; i < 2; i++)
+ mc16_test_cancel_pending(&f->refs[i]);
+ /* Exact metadata balance is checked after explicit pressure reclaim. */
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->owner->model.charged), 0L);
+}
+
+static void memcomp_test_payload_final_release(struct kunit *test)
+{
+ struct mc16_test_storage *f;
+ struct mc16_view view;
+ struct mc16_record *tail;
+ unsigned long handle;
+ spinlock_t ptl; /* Models the caller holding a PTE lock. */
+ unsigned long irqflags;
+ int ret;
+
+ f = mc16_test_storage_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(f, 1), 0);
+ ret = memcomp_zs_malloc(memcomp_pools[0], PAGE_SIZE, &handle, false);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ ret = memcomp_objcg_charge(&f->owner->model.objcg, GFP_KERNEL, PAGE_SIZE);
+ if (ret) {
+ memcomp_zs_free(0, handle);
+ KUNIT_FAIL(test, "payload charge failed: %d", ret);
+ return;
+ }
+ mutex_lock(memcomp_lock_of(f->refs[0].token));
+ mc16_view_locked(&f->refs[0], &view);
+ folio_put(view.member->owner_folio);
+ *view.word = handle;
+ *view.length = PAGE_SIZE - 1;
+ *view.flags = MC16_STORED;
+ view.member->payload_charge = PAGE_SIZE;
+ tail = mc16_normalize_locked(&f->refs[0]);
+ mutex_unlock(memcomp_lock_of(f->refs[0].token));
+ mc16_record_member_put(tail);
+ f->payload.task = current;
+ f->payload.pool = memcomp_pools[0];
+ f->payload.handle = handle;
+ f->payload.live = true;
+ rcu_assign_pointer(memcomp_kunit_async_probe, &f->payload);
+ spin_lock_init(&ptl);
+ spin_lock_irqsave(&ptl, irqflags);
+ mc16_test_cancel_pending(&f->refs[0]);
+ spin_unlock_irqrestore(&ptl, irqflags);
+ KUNIT_EXPECT_EQ(test, f->payload.frees, 1U);
+ KUNIT_EXPECT_FALSE(test, f->payload.live);
+ KUNIT_EXPECT_FALSE(test, f->payload.bad_lifetime);
+ /* Exact metadata balance is checked after explicit pressure reclaim. */
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->owner->model.charged), 0L);
+}
+
+static void memcomp_test_record_outlives_key(struct kunit *test)
+{
+ struct mc16_test_storage *f;
+ struct memcomp_test_owner *tail;
+ struct mc16_view view;
+ u64 charge;
+ int ret;
+
+ f = mc16_test_storage_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ tail = kunit_kzalloc(test, sizeof(*tail), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, tail);
+ ret = kunit_add_action_or_reset(test, memcomp_test_cleanup, tail);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(f, 1), 0);
+ mutex_lock(memcomp_lock_of(f->refs[0].token));
+ mc16_view_locked(&f->refs[0], &view);
+ charge = view.record->charge_bytes;
+ folio_put(view.member->owner_folio);
+ *view.word = 0xfefefefefefefefeULL;
+ *view.length = 0;
+ *view.flags = MC16_SAMEFILL;
+ tail->record = mc16_normalize_locked(&f->refs[0]);
+ tail->remaining = 1;
+ mutex_unlock(memcomp_lock_of(f->refs[0].token));
+ mc16_test_cancel_pending(&f->refs[0]);
+ KUNIT_EXPECT_FALSE(test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_GT(test, atomic_long_read(&f->owner->model.charged),
+ (long)charge);
+ /* The independent record survives explicit header/key reclaim. */
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->owner->model.charged),
+ (long)charge);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->owner->model.put_calls), 1);
+ tail->remaining = 0;
+ memcomp_test_record_put(tail->record);
+ tail->record = NULL;
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->owner->model.charged), 0L);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->owner->model.put_calls), 2);
+}
+
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_ACCESS_TEST_H */
diff --git a/mm/memcomp/tests/aging_test.h b/mm/memcomp/tests/aging_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..5616b5a049cc4e0e2558cb461fceba180c96f6c7
--- /dev/null
+++ b/mm/memcomp/tests/aging_test.h
@@ -0,0 +1,771 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_AGING_TEST_H
+#define _MM_MEMCOMP_TESTS_AGING_TEST_H
+
+static atomic_t mc16_test_aging_schedules = ATOMIC_INIT(0);
+static unsigned long mc16_test_aging_clock;
+static unsigned long mc16_test_aging_delay;
+static void (*mc16_test_aging_queue_action)(void);
+static unsigned long mc16_test_aging_queue_next;
+static unsigned long mc16_test_aging_queue_deadline;
+static int mc16_test_aging_queue_duplicates;
+
+/* The case action runs after all descendant storage/objcg cleanups. */
+static void mc16_test_aging_reset(void)
+{
+ unsigned long flags;
+
+ cancel_delayed_work_sync(&mc16_retained_aging);
+ xa_lock_irqsave(&mc16_clusters, flags);
+ memset(&mc16_aging, 0, sizeof(mc16_aging));
+ mc16_aging.next_id = 1;
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ mc16_test_aging_clock = 0;
+ mc16_test_aging_delay = 0;
+ mc16_test_aging_queue_action = NULL;
+ mc16_test_aging_queue_next = 0;
+ mc16_test_aging_queue_deadline = 0;
+ mc16_test_aging_queue_duplicates = 0;
+}
+
+static void mc16_test_schedule_aging(void)
+{
+ atomic_inc(&mc16_test_aging_schedules);
+}
+
+#ifdef CONFIG_MEMCG
+static unsigned long mc16_test_aging_now(void)
+{
+ return mc16_test_aging_clock;
+}
+
+static void mc16_test_queue_aging(unsigned long delay)
+{
+ void (*action)(void) = mc16_test_aging_queue_action;
+
+ mc16_test_aging_delay = delay;
+ atomic_inc(&mc16_test_aging_schedules);
+ /* A nested producer can queue too; consume this boundary only once. */
+ mc16_test_aging_queue_action = NULL;
+ if (action)
+ action();
+}
+
+/* All traversal and transitions below execute the production step/helper. */
+#define MC16_TEST_AGING_IDS (4 * MC16_AGING_BATCH + 16)
+struct mc16_test_aging_batch {
+ struct kunit *test;
+ struct mc16_test_storage *storage;
+ unsigned long ids[MC16_TEST_AGING_IDS];
+ unsigned int nr;
+};
+
+struct mc16_test_aging_step {
+ struct mc16_cluster *detached[MC16_AGING_BATCH + 1];
+ unsigned int publications[MC16_AGING_BATCH + 1];
+ unsigned int publish_attempts;
+ unsigned int published;
+ unsigned int nr;
+ unsigned int checked;
+ unsigned long next_id;
+ bool active;
+ bool owned;
+ long delay;
+};
+
+/* Count every adapter publication attempt before forwarding to real RCU. */
+static void mc16_test_aging_publish(struct kunit *test,
+ struct mc16_test_aging_step *r,
+ unsigned int index)
+{
+ unsigned int first;
+
+ /* Compare addresses only: a previous callback may already have run. */
+ for (first = 0; first < index; first++)
+ if (r->detached[first] == r->detached[index])
+ break;
+ r->publish_attempts++;
+ r->publications[first]++;
+ KUNIT_EXPECT_EQ_MSG(test, r->publications[first], 1U,
+ "same header published more than once");
+ if (r->publications[first] != 1)
+ return;
+ r->published++;
+ mc16_queue_cluster(r->detached[index]);
+}
+
+static void mc16_test_aging_batch_cleanup(void *data)
+{
+ struct mc16_test_aging_batch *b = data;
+ struct mc16_cluster *cluster;
+ unsigned long flags, id, retained = 0;
+ unsigned int i;
+
+ cancel_delayed_work_sync(&mc16_retained_aging);
+ for (i = 0; i < b->nr; i++) {
+ xa_lock_irqsave(&mc16_clusters, flags);
+ cluster = __xa_erase(&mc16_clusters, b->ids[i]);
+ if (cluster && !xa_is_value(cluster) &&
+ !xa_is_internal(cluster)) {
+ if (cluster->key->current_cluster == cluster)
+ cluster->key->current_cluster = NULL;
+ INIT_LIST_HEAD(&cluster->fifo);
+ } else {
+ cluster = NULL;
+ }
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ if (cluster)
+ mc16_queue_cluster(cluster);
+ }
+ xa_lock_irqsave(&mc16_clusters, flags);
+ xa_for_each(&mc16_clusters, id, cluster) {
+ if (xa_is_value(cluster) || xa_is_internal(cluster))
+ continue;
+ if (cluster->key->current_cluster == cluster &&
+ !cluster->live_slots && cluster->next &&
+ cluster->next < MC16_CLUSTER_SLOTS)
+ retained++;
+ }
+ WRITE_ONCE(mc16_retained_empty_headers, retained);
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ rcu_barrier();
+}
+
+static struct mc16_test_aging_batch *
+mc16_test_aging_batch_begin(struct kunit *test)
+{
+ struct mc16_test_aging_batch *b;
+
+ b = kunit_kzalloc(test, sizeof(*b), GFP_KERNEL);
+ if (!b)
+ return NULL;
+ b->test = test;
+ b->storage = mc16_test_storage_begin(test);
+ if (!b->storage)
+ return NULL;
+ if (kunit_add_action_or_reset(test, mc16_test_aging_batch_cleanup, b))
+ return NULL;
+ kunit_activate_static_stub(test, mc16_aging_now, mc16_test_aging_now);
+ kunit_activate_static_stub(test, mc16_queue_aging, mc16_test_queue_aging);
+ return b;
+}
+
+/* Real header/key allocators and RCU destructor; no simulated aging engine. */
+static int mc16_test_aging_add(struct mc16_test_aging_batch *b,
+ unsigned long id, unsigned long parent)
+{
+ struct mc16_cluster *cluster, *previous;
+ struct mc16_key *key;
+ unsigned long flags;
+ int ret;
+
+ if (b->nr == ARRAY_SIZE(b->ids))
+ return -ENOSPC;
+ if (parent) {
+ previous = xa_load(&mc16_clusters, parent);
+ if (!previous)
+ return -ENOENT;
+ key = previous->key;
+ } else {
+ key = mc16_key_alloc(&b->storage->owner->model.objcg,
+ 0, MC16_KEY_ANON, false);
+ if (IS_ERR(key))
+ return PTR_ERR(key);
+ }
+ cluster = mc16_alloc_zero(MC16_META_CLUSTER, sizeof(*cluster),
+ &b->storage->owner->model.objcg);
+ if (!cluster) {
+ if (!parent)
+ mc16_key_free(key);
+ return -ENOMEM;
+ }
+ cluster->key = key;
+ cluster->id = id;
+ cluster->next = 1;
+ cluster->metadata_bytes = ksize(cluster);
+ INIT_LIST_HEAD(&cluster->fifo);
+ ret = xa_insert_irq(&mc16_clusters, id, cluster, GFP_KERNEL);
+ if (ret) {
+ mc16_meta_free(MC16_META_CLUSTER, cluster);
+ if (!parent)
+ mc16_key_free(key);
+ return ret;
+ }
+ xa_lock_irqsave(&mc16_clusters, flags);
+ if (!parent)
+ hash_add(mc16_keys, &key->node,
+ mc16_key_tag(key->objcg, 0, MC16_KEY_ANON, false));
+ previous = key->current_cluster;
+ if (previous) {
+ previous->next = MC16_CLUSTER_SLOTS;
+ mc16_retained_empty_dec_locked();
+ }
+ key->current_cluster = cluster;
+ key->clusters++;
+ mc16_retained_empty_inc_locked();
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ b->ids[b->nr++] = id;
+ return 0;
+}
+
+static int mc16_test_aging_marker(struct mc16_test_aging_batch *b,
+ unsigned long id, bool value)
+{
+ int ret;
+
+ if (b->nr == ARRAY_SIZE(b->ids))
+ return -ENOSPC;
+ ret = value ? xa_insert_irq(&mc16_clusters, id, xa_mk_value(7),
+ GFP_KERNEL) :
+ xa_reserve_irq(&mc16_clusters, id, GFP_KERNEL);
+ if (!ret)
+ b->ids[b->nr++] = id;
+ return ret;
+}
+
+static struct mc16_test_aging_step
+mc16_test_aging_step(struct kunit *test, bool publish)
+{
+ struct mc16_test_aging_step r = { .owned = true };
+ struct mc16_cluster *alias;
+ unsigned long flags, id;
+ unsigned int i, j;
+
+ xa_lock_irqsave(&mc16_clusters, flags);
+ r.delay = mc16_aging_step_locked(mc16_test_aging_clock, r.detached,
+ &r.nr, &r.checked);
+ r.next_id = mc16_aging.next_id;
+ r.active = mc16_aging.active;
+ for (i = 0; i < min_t(unsigned int, r.nr, MC16_AGING_BATCH); i++) {
+ struct mc16_cluster *c = r.detached[i];
+
+ r.owned &= c->key->current_cluster != c &&
+ !xa_load(&mc16_clusters, c->id) && c->key->clusters;
+ for (j = 0; j < i; j++)
+ r.owned &= c != r.detached[j];
+ /* Preserve failure evidence before repairing a variant for cleanup. */
+ if (c->key->current_cluster == c)
+ c->key->current_cluster = NULL;
+ /* An ID-guard mutation can erase the wrong slot, leaving an alias. */
+ xa_for_each(&mc16_clusters, id, alias) {
+ if (alias != c)
+ continue;
+ r.owned = false;
+ __xa_erase(&mc16_clusters, id);
+ /* Restore the fixture ID before its one permitted publication. */
+ c->id = id;
+ }
+ }
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ KUNIT_EXPECT_LE(test, r.checked, (unsigned int)MC16_AGING_BATCH);
+ KUNIT_EXPECT_LE(test, r.nr, (unsigned int)MC16_AGING_BATCH);
+ KUNIT_EXPECT_PTR_EQ(test, r.detached[MC16_AGING_BATCH], NULL);
+ KUNIT_EXPECT_TRUE(test, r.owned);
+ if (publish) {
+ for (i = 0; i < min_t(unsigned int, r.nr, MC16_AGING_BATCH); i++)
+ mc16_test_aging_publish(test, &r, i);
+ KUNIT_EXPECT_EQ(test, r.publish_attempts, r.nr);
+ KUNIT_EXPECT_EQ(test, r.published, r.publish_attempts);
+ }
+ return r;
+}
+
+static void mc16_test_aging_observe(struct kunit *test)
+{
+ struct mc16_test_aging_step r;
+ unsigned int i;
+
+ for (i = 0; i < MC16_TEST_AGING_IDS + 1; i++) {
+ r = mc16_test_aging_step(test, true);
+ if (r.delay)
+ return;
+ }
+ KUNIT_FAIL(test, "numeric cursor did not finish a bounded population");
+}
+
+/* Group 1: raw budget, private reservations, sparse/max IDs and key shapes. */
+static void memcomp_test_aging_batch_bounds(struct kunit *test)
+{
+ struct mc16_test_aging_batch *b = mc16_test_aging_batch_begin(test);
+ struct mc16_test_aging_step r;
+ unsigned int i, n;
+ unsigned long before;
+ struct mc16_cluster *c;
+
+ KUNIT_ASSERT_NOT_NULL(test, b);
+ for (i = 1; i <= MC16_AGING_BATCH + 1; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_marker(b, i, false), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_add(b, MC16_AGING_BATCH + 2, 0), 0);
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.checked, (unsigned int)MC16_AGING_BATCH);
+ KUNIT_EXPECT_EQ(test, r.nr, 0U);
+ KUNIT_EXPECT_EQ(test, r.next_id, MC16_AGING_BATCH + 1UL);
+ c = xa_load(&mc16_clusters, MC16_AGING_BATCH + 2);
+ KUNIT_ASSERT_NOT_NULL(test, c);
+ KUNIT_EXPECT_EQ(test, c->flags & MC16_CLUSTER_AGED, 0);
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.checked, 2U);
+ KUNIT_EXPECT_EQ(test, c->flags & MC16_CLUSTER_AGED, MC16_CLUSTER_AGED);
+ mc16_test_aging_batch_cleanup(b);
+ b->nr = 0;
+ mc16_test_aging_reset();
+ /* All-reserved arrays still consume B raw returns; holes are not units. */
+ for (i = 1; i <= MC16_AGING_BATCH + 1; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_marker(b, 2 * i, false), 0);
+ WRITE_ONCE(mc16_retained_empty_headers, 1);
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.checked, (unsigned int)MC16_AGING_BATCH);
+ KUNIT_EXPECT_EQ(test, r.next_id, 2UL * MC16_AGING_BATCH + 1);
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.checked, 1U);
+ mc16_test_aging_batch_cleanup(b);
+ b->nr = 0;
+ for (n = MC16_AGING_BATCH - 1; n <= MC16_AGING_BATCH + 1; n++) {
+ mc16_test_aging_reset();
+ for (i = 1; i <= n; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_add(b, i, 0), 0);
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.checked, min_t(unsigned int, n, MC16_AGING_BATCH));
+ KUNIT_EXPECT_EQ(test, r.active, n >= MC16_AGING_BATCH);
+ before = r.next_id;
+ if (r.active) {
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.checked, n - MC16_AGING_BATCH);
+ KUNIT_EXPECT_FALSE(test, r.active);
+ KUNIT_EXPECT_GT(test, before, 1UL);
+ }
+ mc16_test_aging_batch_cleanup(b);
+ b->nr = 0;
+ }
+ mc16_test_aging_reset();
+ for (i = 1; i <= 2 * MC16_AGING_BATCH + 1; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_add(b, i, i == 1 ? 0 : i - 1), 0);
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.next_id, MC16_AGING_BATCH + 1UL);
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.next_id, 2UL * MC16_AGING_BATCH + 1);
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.checked, 1U);
+ mc16_test_aging_batch_cleanup(b);
+ b->nr = 0;
+ mc16_test_aging_reset();
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_marker(b, 5, true), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_add(b, MC16_MAX_CLUSTER_ID, 0), 0);
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.checked, 2U);
+ KUNIT_EXPECT_FALSE(test, r.active);
+ KUNIT_EXPECT_EQ(test, r.next_id, 1UL);
+ mc16_test_aging_clock += HZ;
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.nr, 1U);
+}
+
+/* Group 2: real producer requests cannot shorten wrap-safe pass deadlines. */
+static void memcomp_test_aging_deadlines(struct kunit *test)
+{
+ struct mc16_test_aging_batch *b = mc16_test_aging_batch_begin(test);
+ struct mc16_test_aging_step r;
+ unsigned long deadline;
+ int queued;
+
+ KUNIT_ASSERT_NOT_NULL(test, b);
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_add(b, 1, 0), 0);
+ kunit_deactivate_static_stub(test, mc16_schedule_aging);
+ mc16_test_aging_clock = ULONG_MAX - HZ / 2;
+ mc16_schedule_aging();
+ deadline = mc16_aging.not_before;
+ queued = atomic_read(&mc16_test_aging_schedules);
+ mc16_schedule_aging();
+ KUNIT_EXPECT_EQ(test, mc16_aging.not_before, deadline);
+ KUNIT_EXPECT_EQ(test, atomic_read(&mc16_test_aging_schedules), queued);
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.checked, 0U);
+ KUNIT_EXPECT_EQ(test, r.delay, (long)HZ);
+ mc16_test_aging_clock = deadline;
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.nr, 0U);
+ KUNIT_EXPECT_EQ(test, r.checked, 1U);
+ deadline = mc16_aging.not_before;
+ mc16_schedule_aging();
+ mc16_test_aging_clock += HZ - 1;
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.checked, 0U);
+ KUNIT_EXPECT_EQ(test, r.delay, 1L);
+ KUNIT_EXPECT_EQ(test, mc16_aging.not_before, deadline);
+ mc16_test_aging_clock++;
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.nr, 1U);
+ KUNIT_EXPECT_EQ(test, r.delay, -1L);
+}
+
+/* Group 3: numeric continuation survives deletion, insertion and ID reuse. */
+static void memcomp_test_aging_mutation(struct kunit *test)
+{
+ struct mc16_test_aging_batch *b = mc16_test_aging_batch_begin(test);
+ struct mc16_test_aging_step r;
+ struct shrink_control sc = { .nr_to_scan = 1 };
+ struct mc16_cluster *c;
+ unsigned long flags;
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, b);
+ for (i = 1; i <= MC16_AGING_BATCH; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_marker(b, 2 * i, false), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_add(b, 200, 0), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_add(b, 201, 0), 0);
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.next_id, 2UL * MC16_AGING_BATCH + 1);
+ mc16_meta_shrinker_cursor = 200;
+ KUNIT_ASSERT_EQ(test, mc16_meta_shrinker_scan(NULL, &sc), 1UL);
+ rcu_barrier();
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_add(b, 200, 0), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_add(b, 1, 0), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_add(b, 300, 0), 0);
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.checked, 3U);
+ c = xa_load(&mc16_clusters, 1);
+ KUNIT_ASSERT_NOT_NULL(test, c);
+ KUNIT_EXPECT_EQ(test, c->flags & MC16_CLUSTER_AGED, 0);
+ c = xa_load(&mc16_clusters, 200);
+ KUNIT_ASSERT_NOT_NULL(test, c);
+ KUNIT_EXPECT_NE(test, c->flags & MC16_CLUSTER_AGED, 0);
+ /* A completed final-key callback did not leave a persistent pointer. */
+ mc16_test_aging_clock += HZ;
+ mc16_test_aging_observe(test);
+ KUNIT_EXPECT_NOT_NULL(test, xa_load(&mc16_clusters, 1));
+ KUNIT_EXPECT_PTR_EQ(test, xa_load(&mc16_clusters, 200), NULL);
+ mc16_meta_shrinker_cursor = 1;
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_scan(NULL, &sc), 1UL);
+ xa_lock_irqsave(&mc16_clusters, flags);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 0UL);
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ rcu_barrier();
+}
+
+/* Group 4: independently exercise every production eligibility predicate. */
+static void memcomp_test_aging_eligibility(struct kunit *test)
+{
+ struct mc16_test_aging_batch *b = mc16_test_aging_batch_begin(test);
+ struct mc16_test_aging_step r;
+ struct mc16_cluster *c;
+ struct mc16_key *key;
+ LIST_HEAD(fifo);
+ unsigned long flags;
+ unsigned int guard;
+
+ KUNIT_ASSERT_NOT_NULL(test, b);
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_add(b, 10, 0), 0);
+ for (guard = 0; guard < 7; guard++) {
+ xa_lock_irqsave(&mc16_clusters, flags);
+ c = xa_load(&mc16_clusters, 10);
+ if (!c) {
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ KUNIT_FAIL(test, "eligibility guard detached the header");
+ return;
+ }
+ key = c->key;
+ c->flags = MC16_CLUSTER_AGED;
+ if (guard == 0)
+ c->id = 11;
+ if (guard == 1)
+ key->current_cluster = NULL;
+ if (guard == 2)
+ c->live_slots = 1;
+ if (guard == 3)
+ c->next = 0;
+ if (guard == 4)
+ c->next = MC16_CLUSTER_SLOTS;
+ if (guard == 5)
+ c->eligible_slots = 1;
+ if (guard == 6)
+ list_add(&c->fifo, &fifo);
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.nr, 0U);
+ xa_lock_irqsave(&mc16_clusters, flags);
+ c = xa_load(&mc16_clusters, 10);
+ if (!c) {
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ KUNIT_FAIL(test, "eligibility mutation escaped its guard");
+ return;
+ }
+ c->id = 10;
+ key->current_cluster = c;
+ c->live_slots = 0;
+ c->next = 1;
+ c->eligible_slots = 0;
+ list_del_init(&c->fifo);
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ mc16_test_aging_clock += HZ;
+ }
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.nr, 1U);
+ /* Refill uses the real reserve/publish path, not a flags simulation. */
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(b->storage, 1), 0);
+ mc16_test_cancel_pending(&b->storage->refs[0]);
+ mc16_test_aging_clock += HZ;
+ mc16_test_aging_observe(test);
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(b->storage, 1), 0);
+ KUNIT_EXPECT_EQ(test, b->storage->refs[1].cluster->flags &
+ MC16_CLUSTER_AGED, 0);
+ mc16_test_cancel_pending(&b->storage->refs[1]);
+ mc16_test_aging_clock += HZ;
+ mc16_test_aging_observe(test);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 1UL);
+}
+
+/* Runs from the actual wrapper's queue call, after its locked step. */
+static void mc16_test_aging_producer_at_queue(void)
+{
+ int queued;
+
+ mc16_test_aging_queue_next = mc16_aging.next_id;
+ mc16_test_aging_queue_deadline = mc16_aging.not_before;
+ mc16_schedule_aging();
+ queued = atomic_read(&mc16_test_aging_schedules);
+ mc16_schedule_aging();
+ mc16_test_aging_queue_duplicates =
+ atomic_read(&mc16_test_aging_schedules) - queued;
+}
+
+/* Group 5: queue boundaries consume requests, duplicates preserve progress. */
+static void memcomp_test_aging_requests(struct kunit *test)
+{
+ struct mc16_test_aging_batch *b = mc16_test_aging_batch_begin(test);
+ struct mc16_test_aging_step r;
+ struct shrink_control sc = { .nr_to_scan = 1 };
+ unsigned long next, deadline;
+ unsigned int i;
+ int queued;
+
+ KUNIT_ASSERT_NOT_NULL(test, b);
+ kunit_deactivate_static_stub(test, mc16_schedule_aging);
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.delay, -1L);
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_add(b, 1000, 0), 0);
+ mc16_schedule_aging();
+ KUNIT_EXPECT_TRUE(test, mc16_aging.requested);
+ queued = atomic_read(&mc16_test_aging_schedules);
+ mc16_schedule_aging();
+ KUNIT_EXPECT_EQ(test, atomic_read(&mc16_test_aging_schedules), queued);
+ mc16_test_aging_clock += HZ;
+ for (i = 1; i <= MC16_AGING_BATCH + 1; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_marker(b, i, false), 0);
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.delay, 0L);
+ KUNIT_EXPECT_FALSE(test, mc16_aging.requested);
+ next = mc16_aging.next_id;
+ deadline = mc16_aging.not_before;
+ /* Producer obtains X after step unlock but before wrapper queueing. */
+ mc16_schedule_aging();
+ queued = atomic_read(&mc16_test_aging_schedules);
+ mc16_schedule_aging();
+ KUNIT_EXPECT_EQ(test, mc16_aging.next_id, next);
+ KUNIT_EXPECT_EQ(test, mc16_aging.not_before, deadline);
+ KUNIT_EXPECT_EQ(test, atomic_read(&mc16_test_aging_schedules), queued);
+ mc16_meta_shrinker_cursor = 1000;
+ KUNIT_ASSERT_EQ(test, mc16_meta_shrinker_scan(NULL, &sc), 1UL);
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.delay, -1L);
+ KUNIT_EXPECT_FALSE(test, r.active);
+ /* First retained header appears after the old callback chose idle. */
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_add(b, 2000, 0), 0);
+ mc16_schedule_aging();
+ KUNIT_EXPECT_TRUE(test, mc16_aging.requested);
+ KUNIT_EXPECT_GE(test, atomic_read(&mc16_test_aging_schedules), queued + 1);
+ mc16_test_aging_clock += HZ;
+ mc16_test_aging_observe(test);
+ mc16_test_aging_clock += HZ;
+ mc16_test_aging_observe(test);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 0UL);
+ queued = atomic_read(&mc16_test_aging_schedules);
+ mc16_retained_age_work(&mc16_retained_aging.work);
+ KUNIT_EXPECT_EQ(test, atomic_read(&mc16_test_aging_schedules), queued);
+ /* Exercise the real wrapper at a continuation queue boundary. */
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_add(b, 3000, 0), 0);
+ mc16_schedule_aging();
+ mc16_test_aging_clock += HZ;
+ mc16_test_aging_queue_duplicates = -1;
+ mc16_test_aging_queue_action = mc16_test_aging_producer_at_queue;
+ mc16_retained_age_work(&mc16_retained_aging.work);
+ KUNIT_EXPECT_TRUE(test, !mc16_test_aging_queue_action);
+ KUNIT_EXPECT_EQ(test, mc16_test_aging_queue_duplicates, 0);
+ KUNIT_EXPECT_EQ(test, mc16_aging.next_id, mc16_test_aging_queue_next);
+ KUNIT_EXPECT_EQ(test, mc16_aging.not_before,
+ mc16_test_aging_queue_deadline);
+ KUNIT_EXPECT_TRUE(test, mc16_aging.requested);
+ KUNIT_EXPECT_EQ(test, mc16_test_aging_delay, 0UL);
+ /* Finish this sweep; the queued request cannot bypass the next pass. */
+ mc16_retained_age_work(&mc16_retained_aging.work);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 1UL);
+ mc16_retained_age_work(&mc16_retained_aging.work);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 1UL);
+ KUNIT_EXPECT_EQ(test, mc16_test_aging_delay, (unsigned long)HZ);
+ mc16_test_aging_clock += HZ;
+ mc16_retained_age_work(&mc16_retained_aging.work);
+ mc16_retained_age_work(&mc16_retained_aging.work);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 0UL);
+}
+
+/* Group 6: detached headers retain keys/charges until the real RCU path. */
+static void memcomp_test_aging_rcu_ownership(struct kunit *test)
+{
+ struct mc16_test_aging_batch *b = mc16_test_aging_batch_begin(test);
+ struct mc16_test_aging_step r;
+ struct memcomp_kunit_objcg *model;
+ unsigned int i;
+ long charged, published_charge;
+ int published_puts;
+ unsigned long flags;
+ bool keys_owned = true;
+
+ KUNIT_ASSERT_NOT_NULL(test, b);
+ model = &b->storage->owner->model;
+ for (i = 1; i <= MC16_AGING_BATCH + 1; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_aging_add(b, i, 0), 0);
+ mc16_test_aging_observe(test);
+ charged = atomic_long_read(&model->charged);
+ mc16_test_aging_clock += HZ;
+ /* Deliberately defer publication, as permitted for detached headers. */
+ r = mc16_test_aging_step(test, false);
+ KUNIT_ASSERT_EQ(test, r.nr, (unsigned int)MC16_AGING_BATCH);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&model->charged), charged);
+ KUNIT_EXPECT_EQ(test, atomic_read(&model->put_calls), 0);
+ KUNIT_EXPECT_FALSE(test, hash_empty(mc16_keys));
+ /* Publish through call_rcu while a pre-existing real reader holds GP. */
+ rcu_read_lock();
+ for (i = 0; i < r.nr; i++)
+ mc16_test_aging_publish(test, &r, i);
+ published_charge = atomic_long_read(&model->charged);
+ published_puts = atomic_read(&model->put_calls);
+ xa_lock_irqsave(&mc16_clusters, flags);
+ for (i = 0; i < r.nr; i++)
+ keys_owned &= r.detached[i]->key->clusters == 1 &&
+ !hlist_unhashed(&r.detached[i]->key->node) &&
+ !r.detached[i]->key->current_cluster;
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ rcu_read_unlock();
+ KUNIT_EXPECT_EQ(test, r.publish_attempts, r.nr);
+ KUNIT_EXPECT_EQ(test, r.published, r.nr);
+ KUNIT_EXPECT_EQ(test, published_charge, charged);
+ KUNIT_EXPECT_EQ(test, published_puts, 0);
+ KUNIT_EXPECT_TRUE(test, keys_owned);
+ rcu_barrier();
+ KUNIT_EXPECT_EQ(test, atomic_read(&model->put_calls), MC16_AGING_BATCH);
+ r = mc16_test_aging_step(test, true);
+ KUNIT_EXPECT_EQ(test, r.nr, 1U);
+ rcu_barrier();
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&model->charged), 0L);
+ KUNIT_EXPECT_EQ(test, atomic_read(&model->get_calls),
+ atomic_read(&model->put_calls));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+}
+
+static struct mc16_test_storage *mc16_test_aging_begin(struct kunit *test)
+{
+ struct mc16_test_storage *s = mc16_test_storage_begin(test);
+
+ if (!s)
+ return NULL;
+ kunit_activate_static_stub(test, mc16_aging_now, mc16_test_aging_now);
+ kunit_activate_static_stub(test, mc16_queue_aging, mc16_test_queue_aging);
+ if (mc16_test_reserve(s, 1))
+ return NULL;
+ mc16_test_cancel_pending(&s->refs[0]);
+ return s;
+}
+
+static void memcomp_test_aging_teardown(struct kunit *test)
+{
+ struct mc16_test_storage *s = mc16_test_aging_begin(test);
+
+ KUNIT_ASSERT_NOT_NULL(test, s);
+ KUNIT_ASSERT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 1UL);
+ mc16_retained_age_work(&mc16_retained_aging.work);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 1UL);
+ mc16_test_aging_clock += HZ;
+ mc16_retained_age_work(&mc16_retained_aging.work);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 0UL);
+}
+
+static void memcomp_test_aging_reuse(struct kunit *test)
+{
+ struct mc16_test_storage *s = mc16_test_aging_begin(test);
+ u32 id;
+
+ KUNIT_ASSERT_NOT_NULL(test, s);
+ mc16_retained_age_work(&mc16_retained_aging.work);
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(s, 1), 0);
+ id = s->refs[1].cluster->id;
+ mc16_test_cancel_pending(&s->refs[1]);
+ mc16_test_aging_clock += HZ;
+ mc16_retained_age_work(&mc16_retained_aging.work);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 1UL);
+ KUNIT_EXPECT_NOT_NULL(test, xa_load(&mc16_clusters, id));
+}
+
+static void memcomp_test_aging_stops(struct kunit *test)
+{
+ struct mc16_test_storage *s = mc16_test_aging_begin(test);
+ int schedules;
+
+ KUNIT_ASSERT_NOT_NULL(test, s);
+ KUNIT_EXPECT_EQ(test, atomic_read(&mc16_test_aging_schedules), 1);
+ mc16_retained_age_work(&mc16_retained_aging.work);
+ KUNIT_EXPECT_EQ(test, atomic_read(&mc16_test_aging_schedules), 2);
+ mc16_test_aging_clock += HZ;
+ mc16_retained_age_work(&mc16_retained_aging.work);
+ KUNIT_ASSERT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 0UL);
+ schedules = atomic_read(&mc16_test_aging_schedules);
+ mc16_retained_age_work(&mc16_retained_aging.work);
+ KUNIT_EXPECT_EQ(test, atomic_read(&mc16_test_aging_schedules), schedules);
+ KUNIT_EXPECT_FALSE(test, delayed_work_pending(&mc16_retained_aging));
+}
+
+static void memcomp_test_aging_shrinker(struct kunit *test)
+{
+ struct mc16_test_storage *s = mc16_test_aging_begin(test);
+ struct shrink_control sc = { .nr_to_scan = 64 };
+
+ KUNIT_ASSERT_NOT_NULL(test, s);
+ mc16_retained_age_work(&mc16_retained_aging.work);
+ mc16_meta_shrinker_scan(NULL, &sc);
+ mc16_test_aging_clock += HZ;
+ mc16_retained_age_work(&mc16_retained_aging.work);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 0UL);
+ rcu_barrier();
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_EQ(test, atomic_read(&s->owner->model.get_calls),
+ atomic_read(&s->owner->model.put_calls));
+}
+
+static void memcomp_test_aging_real_work(struct kunit *test)
+{
+ struct mc16_test_storage *s = mc16_test_aging_begin(test);
+ unsigned long flags;
+
+ KUNIT_ASSERT_NOT_NULL(test, s);
+ kunit_deactivate_static_stub(test, mc16_aging_now);
+ kunit_deactivate_static_stub(test, mc16_schedule_aging);
+ kunit_deactivate_static_stub(test, mc16_queue_aging);
+ mc16_schedule_aging();
+ /* The deadline contract is checked separately with a controlled clock. */
+ xa_lock_irqsave(&mc16_clusters, flags);
+ mc16_aging.not_before = jiffies;
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ flush_delayed_work(&mc16_retained_aging);
+ xa_lock_irqsave(&mc16_clusters, flags);
+ mc16_aging.not_before = jiffies;
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ flush_delayed_work(&mc16_retained_aging);
+ cancel_delayed_work_sync(&mc16_retained_aging);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 0UL);
+ KUNIT_EXPECT_FALSE(test, delayed_work_pending(&mc16_retained_aging));
+ rcu_barrier();
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+}
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_AGING_TEST_H */
diff --git a/mm/memcomp/tests/anon_store_test.h b/mm/memcomp/tests/anon_store_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..00202259bcc52e57ff9475d2460e9cd6de540575
--- /dev/null
+++ b/mm/memcomp/tests/anon_store_test.h
@@ -0,0 +1,704 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_ANON_STORE_TEST_H
+#define _MM_MEMCOMP_TESTS_ANON_STORE_TEST_H
+
+/*
+ * Include in suite.h after refs_test.h. Uses its
+ * refs/storage/objcg fixtures and memcomp_test_reclaimer_enter()/exit().
+ * Register the cases in the enclosing suite.
+ */
+#ifdef CONFIG_MEMCG
+struct mc16_test_anon_store {
+ struct mc16_test_refs *refs;
+ struct memcomp_ref guards[2];
+ struct memcomp_reclaim_ctx *ctx;
+ struct memcomp_store_ctx store;
+ struct memcomp_async_req precomp[2];
+ struct folio *refresh_folio;
+ unsigned long token;
+ unsigned int reclaim_flags;
+ bool owner_moved[2];
+ bool owner_dropped[2];
+ bool reclaiming;
+ bool reserved;
+ bool folio_unlocked;
+ u8 *src;
+ u8 *dst;
+};
+
+static void mc16_test_anon_ctx_put(struct mc16_test_anon_store *f)
+{
+ if (f->ctx) {
+ f->ctx->kunit_compress_error = 0;
+ memcomp_reclaim_ctx_put(f->ctx);
+ f->ctx = NULL;
+ }
+ if (f->reclaiming) {
+ memcomp_test_reclaimer_exit(f->reclaim_flags);
+ f->reclaiming = false;
+ }
+}
+
+/* Runs before shared refs/storage/model teardown, including fatal asserts. */
+static void mc16_test_anon_store_cleanup(void *data)
+{
+ struct mc16_test_anon_store *f = data;
+ struct mc16_test_storage *s = f->refs->storage;
+ struct memcomp_ref owner;
+ unsigned int i;
+
+ memcomp_kunit_fail_clear();
+ s->owner->model.fail_at = 0;
+ s->probe.fail_at = 0;
+ /* Restore the parent fixture contract in the cleanup task. */
+ if (f->folio_unlocked) {
+ folio_lock(s->owner->folio);
+ f->folio_unlocked = false;
+ }
+ /* Actions may run after the test task exits; never unlock its ctx. */
+ KUNIT_EXPECT_PTR_EQ(s->test, f->ctx, NULL);
+ KUNIT_EXPECT_FALSE(s->test, f->reclaiming);
+ for (i = 0; i < ARRAY_SIZE(f->guards); i++) {
+ if (f->guards[i].slot)
+ mc16_ref_put(&f->guards[i]);
+ }
+ /* Real OWNER removal, siblings first and producer/head last. */
+ for (i = ARRAY_SIZE(f->guards); f->reserved && i; ) {
+ i--;
+ if (f->owner_dropped[i])
+ continue;
+ if (!mc16_get(f->token + i, &owner)) {
+ KUNIT_FAIL(s->test, "reserved member %u disappeared", i);
+ continue;
+ }
+ KUNIT_EXPECT_TRUE(s->test,
+ mc16_owner_drop_folio_held(&owner,
+ f->owner_moved[i] ?
+ f->refresh_folio :
+ s->owner->folio,
+ 2, i));
+ }
+ f->reserved = false;
+ if (f->refresh_folio) {
+ KUNIT_EXPECT_EQ(s->test, folio_ref_count(f->refresh_folio), 1);
+ folio_unlock(f->refresh_folio);
+ folio_put(f->refresh_folio);
+ f->refresh_folio = NULL;
+ }
+ KUNIT_EXPECT_EQ(s->test, folio_ref_count(s->owner->folio), 1);
+ /* Shared cleanup checks bytes, metadata, registry and objcg balance. */
+}
+
+static struct mc16_test_anon_store *
+mc16_test_anon_store_begin(struct kunit *test)
+{
+ struct mc16_test_anon_store *f;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->refs = mc16_test_refs_begin(test);
+ if (!f->refs || kunit_add_action_or_reset(test,
+ mc16_test_anon_store_cleanup, f))
+ return NULL;
+ f->src = kunit_kzalloc(test, 2 * PAGE_SIZE, GFP_KERNEL);
+ f->dst = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ if (!f->src || !f->dst)
+ return NULL;
+ /* Native source exit needs a real folio memcg charge. */
+ if (mem_cgroup_charge(f->refs->storage->owner->folio, NULL,
+ GFP_KERNEL)) {
+ KUNIT_FAIL(test, "anon source memcg charge failed");
+ return NULL;
+ }
+ /* Each reserve actor acquires/releases the real producer folio lock. */
+ folio_unlock(f->refs->storage->owner->folio);
+ f->folio_unlocked = true;
+ return f;
+}
+
+static int mc16_test_anon_reserve_call(struct mc16_test_anon_store *f,
+ unsigned long *token)
+{
+ struct folio *folio = f->refs->storage->owner->folio;
+ unsigned int i;
+ int ret;
+
+ folio_lock(folio);
+ /* The placement hint must inspect the same bytes supplied to refresh. */
+ for (i = 0; i < folio_nr_pages(folio); i++) {
+ void *dst = kmap_local_folio(folio, i * PAGE_SIZE);
+
+ memcpy(dst, f->src + i * PAGE_SIZE, PAGE_SIZE);
+ kunmap_local(dst);
+ }
+ ret = mc16_anon_reserve(folio, 2, NULL, token);
+ folio_unlock(folio);
+ return ret;
+}
+
+static int mc16_test_anon_reserve(struct mc16_test_anon_store *f)
+{
+ unsigned int i;
+ int ret;
+
+ /* No fabricated anon_vma: these transactions transfer no VMA hold. */
+ ret = mc16_test_anon_reserve_call(f, &f->token);
+ if (ret)
+ return ret;
+ /* Track all initial owners before any lookup or assertion can fail. */
+ f->reserved = true;
+ for (i = 0; i < ARRAY_SIZE(f->guards); i++) {
+ if (!mc16_get(f->token + i, &f->guards[i]))
+ return -ENOENT;
+ }
+ return 0;
+}
+
+static int mc16_test_anon_ctx_get(struct mc16_test_anon_store *f)
+{
+ f->reclaim_flags = memcomp_test_reclaimer_enter();
+ f->reclaiming = true;
+ f->ctx = memcomp_reclaim_ctx_get(2, MEMCOMP_RECLAIM_KSWAPD);
+ if (!f->ctx) {
+ mc16_test_anon_ctx_put(f);
+ return -EBUSY;
+ }
+ return 0;
+}
+
+static int mc16_test_anon_refresh(struct mc16_test_anon_store *f)
+{
+ struct folio *folio = f->refs->storage->owner->folio;
+ int ret;
+
+ folio_lock(folio);
+ ret = mc16_anon_refresh(&f->ctx, f->src, 2, &f->store, f->guards);
+ folio_unlock(folio);
+ return ret;
+}
+
+/* Keep every caller guard through real source exit and later payload loads. */
+static bool mc16_test_anon_finalize(struct mc16_test_anon_store *f)
+{
+ struct folio *folio = f->owner_moved[0] ? f->refresh_folio :
+ f->refs->storage->owner->folio;
+ bool locked = folio == f->refresh_folio;
+ bool swapbacked, released;
+ swp_entry_t saved_swap;
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(f->guards); i++) {
+ if (!f->guards[i].slot || f->owner_dropped[i] ||
+ f->owner_moved[i] != f->owner_moved[0])
+ return false;
+ }
+ if (!locked)
+ folio_lock(folio);
+ /* Low-level reserve returns the token without binding folio->swap. */
+ saved_swap = folio->swap;
+ swapbacked = folio_test_swapbacked(folio);
+ __folio_set_swapbacked(folio);
+ folio->swap = make_memcomp_entry(f->token);
+ released = mc16_release_folio(folio);
+ if (released) {
+ folio_ref_unfreeze(folio, 1);
+ for (i = 0; i < ARRAY_SIZE(f->guards); i++)
+ f->owner_dropped[i] = true;
+ }
+ /* Restore the allocator-only parent fixture, not a live source owner. */
+ folio->swap = saved_swap;
+ if (!swapbacked)
+ __folio_clear_swapbacked(folio);
+ if (!locked)
+ folio_unlock(folio);
+ return released;
+}
+
+/* Partial commit is rolled back, never recorded as a successful eviction. */
+static bool mc16_test_anon_rollback(struct mc16_test_anon_store *f)
+{
+ struct folio *folio = f->refs->storage->owner->folio;
+ unsigned int i;
+ bool dropped = true;
+
+ folio_lock(folio);
+ for (i = ARRAY_SIZE(f->guards); i; ) {
+ struct memcomp_ref owner = { };
+
+ i--;
+ if (f->owner_dropped[i])
+ continue;
+ if (!mc16_get(f->token + i, &owner)) {
+ dropped = false;
+ continue;
+ }
+ f->owner_dropped[i] =
+ mc16_owner_drop_folio_held(&owner, folio, 2, i);
+ dropped &= f->owner_dropped[i];
+ }
+ folio_unlock(folio);
+ return dropped;
+}
+
+static void mc16_test_anon_fill(struct mc16_test_anon_store *f, bool samefill)
+{
+ unsigned int i;
+
+ if (samefill) {
+ memset_l((unsigned long *)f->src, 0xfedcba9876543210UL,
+ PAGE_SIZE / sizeof(unsigned long));
+ memset_l((unsigned long *)(f->src + PAGE_SIZE), 0x13579bdf2468ace0UL,
+ PAGE_SIZE / sizeof(unsigned long));
+ } else {
+ for (i = 0; i < 2 * PAGE_SIZE; i++)
+ f->src[i] = (i % PAGE_SIZE) % 53;
+ }
+}
+
+/* Observe effective members under T, never the physical PENDING slot tag. */
+static void mc16_test_anon_expect_member(struct kunit *test,
+ struct memcomp_ref *ref,
+ unsigned int expected, u32 *bytes)
+{
+ struct mc16_view view;
+ unsigned int state = MC16_EMPTY;
+ bool valid, decoded = false, resident = false;
+
+ KUNIT_ASSERT_NOT_NULL(test, ref->slot);
+ *bytes = 0;
+ mutex_lock(memcomp_lock_of(ref->token));
+ valid = mc16_view_locked(ref, &view);
+ if (valid) {
+ state = mc16_flags_state(*view.flags);
+ resident = *view.flags & MC16_F_RESIDENT;
+ if (state == MC16_PENDING)
+ decoded = true;
+ else
+ decoded = mc16_payload_length_decode(state, *view.length,
+ bytes);
+ }
+ mutex_unlock(memcomp_lock_of(ref->token));
+ KUNIT_EXPECT_TRUE(test, valid);
+ KUNIT_EXPECT_TRUE(test, decoded);
+ KUNIT_EXPECT_EQ(test, state, expected);
+ KUNIT_EXPECT_EQ(test, resident, expected == MC16_PENDING);
+ /* Both the initial OWNER and the caller's lookup must survive. */
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), 2U);
+}
+
+static void mc16_test_anon_roundtrip(struct kunit *test, bool samefill)
+{
+ struct mc16_test_anon_store *f = mc16_test_anon_store_begin(test);
+ struct mc16_test_storage *s;
+ unsigned int i;
+ u32 bytes;
+ long charged;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ mc16_test_anon_fill(f, samefill);
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_reserve(f), 0);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(s->owner->folio), 3);
+ charged = atomic_long_read(&s->owner->model.charged);
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_ctx_get(f), 0);
+ /* Failed async requests must fall back to real synchronous compression. */
+ if (!samefill) {
+ f->precomp[0].ret = -EIO;
+ f->precomp[1].ret = -EIO;
+ f->store.precomp = f->precomp;
+ }
+ ret = mc16_test_anon_refresh(f);
+ mc16_test_anon_ctx_put(f);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(s->owner->folio), 1);
+ for (i = 0; i < ARRAY_SIZE(f->guards); i++) {
+ mc16_test_anon_expect_member(test, &f->guards[i],
+ samefill ? MC16_SAMEFILL : MC16_STORED, &bytes);
+ if (samefill) {
+ KUNIT_EXPECT_EQ(test, bytes, 0U);
+ } else {
+ KUNIT_EXPECT_GT(test, bytes, 0U);
+ KUNIT_EXPECT_LT(test, bytes, (u32)PAGE_SIZE);
+ }
+ charged += bytes;
+ }
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged),
+ charged);
+ /* All OWNER/refcount/charge observations precede source exit. */
+ KUNIT_ASSERT_TRUE(test, mc16_test_anon_finalize(f));
+ for (i = 0; i < ARRAY_SIZE(f->guards); i++) {
+ KUNIT_ASSERT_EQ(test, mc16_load_page(&f->guards[i], f->dst), 0);
+ KUNIT_EXPECT_EQ(test, memcmp(f->src + i * PAGE_SIZE,
+ f->dst, PAGE_SIZE), 0);
+ }
+}
+
+static void memcomp_test_anon_store_samefill(struct kunit *test)
+{
+ mc16_test_anon_roundtrip(test, true);
+}
+
+static void memcomp_test_anon_store_compressed(struct kunit *test)
+{
+ mc16_test_anon_roundtrip(test, false);
+}
+
+static void memcomp_test_anon_store_mixed_raw(struct kunit *test)
+{
+ struct mc16_test_anon_store *f = mc16_test_anon_store_begin(test);
+ unsigned int i;
+ u32 bytes;
+ int ret, compress_error;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_test_anon_fill(f, true);
+ for (i = 0; i < PAGE_SIZE; i++)
+ f->src[i] = i * 37 + (i >> 5);
+ /* Force a codec no-shrink verdict without depending on its ratio. */
+ f->precomp[0].ret = -ENOSPC;
+ f->precomp[1].same_filled = true;
+ f->precomp[1].element = 0x13579bdf2468ace0UL;
+ f->store.precomp = f->precomp;
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_reserve(f), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_ctx_get(f), 0);
+ f->ctx->kunit_compress_error = -EIO;
+ ret = mc16_test_anon_refresh(f);
+ compress_error = f->ctx->kunit_compress_error;
+ mc16_test_anon_ctx_put(f);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ /* Both precomputed decisions bypassed synchronous compression. */
+ KUNIT_EXPECT_EQ(test, compress_error, -EIO);
+ mc16_test_anon_expect_member(test, &f->guards[0], MC16_STORED, &bytes);
+ KUNIT_EXPECT_EQ(test, bytes, (u32)PAGE_SIZE);
+ mc16_test_anon_expect_member(test, &f->guards[1], MC16_SAMEFILL, &bytes);
+ KUNIT_EXPECT_EQ(test, bytes, 0U);
+ KUNIT_ASSERT_TRUE(test, mc16_test_anon_finalize(f));
+ for (i = 0; i < ARRAY_SIZE(f->guards); i++) {
+ KUNIT_ASSERT_EQ(test, mc16_load_page(&f->guards[i], f->dst), 0);
+ KUNIT_EXPECT_EQ(test, memcmp(f->src + i * PAGE_SIZE,
+ f->dst, PAGE_SIZE), 0);
+ }
+}
+
+static void memcomp_test_anon_store_reserve_failure(struct kunit *test)
+{
+ struct mc16_test_anon_store *f = mc16_test_anon_store_begin(test);
+ struct mc16_test_storage *s;
+ unsigned long token = ULONG_MAX;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ s->owner->model.fail_at = 1;
+ ret = mc16_test_anon_reserve_call(f, &token);
+ /* Track an unexpected success as well, before any fatal assertion. */
+ if (!ret) {
+ f->token = token;
+ f->reserved = true;
+ }
+ KUNIT_ASSERT_EQ(test, ret, -ENOMEM);
+ KUNIT_EXPECT_EQ(test, token, ULONG_MAX);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged), 0L);
+ s->owner->model.fail_at = 0;
+ s->probe.fail_at = s->probe.allocations + 1;
+ ret = mc16_test_anon_reserve_call(f, &token);
+ if (!ret) {
+ f->token = token;
+ f->reserved = true;
+ }
+ KUNIT_ASSERT_EQ(test, ret, -ENOMEM);
+ s->probe.fail_at = 0;
+ rcu_barrier();
+ KUNIT_EXPECT_EQ(test, token, ULONG_MAX);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(s->owner->folio), 1);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged), 0L);
+ KUNIT_EXPECT_EQ(test, atomic_read(&s->owner->model.get_calls),
+ atomic_read(&s->owner->model.put_calls));
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_reserve(f), 0);
+}
+
+static void memcomp_test_anon_store_charge_failure(struct kunit *test)
+{
+ struct mc16_test_anon_store *f = mc16_test_anon_store_begin(test);
+ struct mc16_test_storage *s;
+ unsigned int i;
+ u32 bytes;
+ long charged;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ mc16_test_anon_fill(f, false);
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_reserve(f), 0);
+ charged = atomic_long_read(&s->owner->model.charged);
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_ctx_get(f), 0);
+ /* The first payload charge must unwind when the second one fails. */
+ s->owner->model.fail_at =
+ atomic_read(&s->owner->model.charge_calls) + 2;
+ ret = mc16_test_anon_refresh(f);
+ s->owner->model.fail_at = 0;
+ mc16_test_anon_ctx_put(f);
+ KUNIT_ASSERT_EQ(test, ret, -ENOMEM);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged),
+ charged);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(s->owner->folio), 3);
+ for (i = 0; i < ARRAY_SIZE(f->guards); i++)
+ mc16_test_anon_expect_member(test, &f->guards[i], MC16_PENDING,
+ &bytes);
+ /* The same caller guards remain valid for a real retry. */
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_ctx_get(f), 0);
+ ret = mc16_test_anon_refresh(f);
+ mc16_test_anon_ctx_put(f);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ KUNIT_ASSERT_TRUE(test, mc16_test_anon_finalize(f));
+}
+
+static void memcomp_test_anon_store_partial_commit(struct kunit *test)
+{
+ struct mc16_test_anon_store *f = mc16_test_anon_store_begin(test);
+ struct mc16_test_storage *s;
+ u32 bytes, pending_bytes;
+ long charged;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ mc16_test_anon_fill(f, false);
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_reserve(f), 0);
+ charged = atomic_long_read(&s->owner->model.charged);
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_ctx_get(f), 0);
+ memcomp_kunit_fail_next(MEMCOMP_KUNIT_FAIL_REFRESH_COMMIT, 2, -EAGAIN);
+ ret = mc16_test_anon_refresh(f);
+ memcomp_kunit_fail_clear();
+ mc16_test_anon_ctx_put(f);
+ KUNIT_ASSERT_EQ(test, ret, -EAGAIN);
+ mc16_test_anon_expect_member(test, &f->guards[0], MC16_STORED, &bytes);
+ mc16_test_anon_expect_member(test, &f->guards[1], MC16_PENDING,
+ &pending_bytes);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged),
+ charged + bytes);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(s->owner->folio), 2);
+ /* Observe the resident member before rollback, with both guards live. */
+ memset(f->dst, 0x5a, PAGE_SIZE);
+ KUNIT_EXPECT_EQ(test, mc16_load_page(&f->guards[1], f->dst), -EAGAIN);
+ KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->dst, 0x5a, PAGE_SIZE), NULL);
+ KUNIT_ASSERT_TRUE(test, mc16_test_anon_rollback(f));
+ KUNIT_ASSERT_EQ(test, mc16_load_page(&f->guards[0], f->dst), 0);
+ KUNIT_EXPECT_EQ(test, memcmp(f->src, f->dst, PAGE_SIZE), 0);
+ /* Cleanup puts guards only; the real rollback already consumed OWNER. */
+}
+
+static void memcomp_test_anon_store_invalid_refs(struct kunit *test)
+{
+ struct mc16_test_anon_store *f = mc16_test_anon_store_begin(test);
+ struct memcomp_ref saved;
+ unsigned int i;
+ u32 bytes;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_test_anon_fill(f, true);
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_reserve(f), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_ctx_get(f), 0);
+ /* Restore the real guard before any assertion can run teardown. */
+ saved = f->guards[1];
+ f->guards[1] = (struct memcomp_ref) { };
+ ret = mc16_test_anon_refresh(f);
+ f->guards[1] = saved;
+ mc16_test_anon_ctx_put(f);
+ KUNIT_ASSERT_EQ(test, ret, -EINVAL);
+ for (i = 0; i < ARRAY_SIZE(f->guards); i++)
+ mc16_test_anon_expect_member(test, &f->guards[i], MC16_PENDING,
+ &bytes);
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_ctx_get(f), 0);
+ ret = mc16_test_anon_refresh(f);
+ mc16_test_anon_ctx_put(f);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ KUNIT_ASSERT_TRUE(test, mc16_test_anon_finalize(f));
+}
+
+static void memcomp_test_anon_store_precomp(struct kunit *test)
+{
+ struct mc16_test_anon_store *f = mc16_test_anon_store_begin(test);
+ unsigned int i, len;
+ int ret, compress_error;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_test_anon_fill(f, false);
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_reserve(f), 0);
+ /* Allocate before taking ctx: allocation assertions may abort. */
+ for (i = 0; i < ARRAY_SIZE(f->precomp); i++) {
+ f->precomp[i].buf = kunit_kmalloc(test,
+ MEMCOMP_MAX_COMPRESSED_SIZE,
+ GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, f->precomp[i].buf);
+ }
+ for (i = 0; i < ARRAY_SIZE(f->precomp); i++) {
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_ctx_get(f), 0);
+ len = MEMCOMP_MAX_COMPRESSED_SIZE;
+ ret = crypto_comp_compress(f->ctx->tfm, f->src + i * PAGE_SIZE,
+ PAGE_SIZE, f->precomp[i].buf, &len);
+ mc16_test_anon_ctx_put(f);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ KUNIT_ASSERT_LT(test, len, (unsigned int)PAGE_SIZE);
+ f->precomp[i].len = len;
+ }
+ f->store.precomp = f->precomp;
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_ctx_get(f), 0);
+ f->ctx->kunit_compress_error = -EIO;
+ ret = mc16_test_anon_refresh(f);
+ compress_error = f->ctx->kunit_compress_error;
+ mc16_test_anon_ctx_put(f);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ KUNIT_EXPECT_EQ(test, compress_error, -EIO);
+ KUNIT_ASSERT_TRUE(test, mc16_test_anon_finalize(f));
+ for (i = 0; i < ARRAY_SIZE(f->guards); i++) {
+ KUNIT_ASSERT_EQ(test, mc16_load_page(&f->guards[i], f->dst), 0);
+ KUNIT_EXPECT_EQ(test, memcmp(f->src + i * PAGE_SIZE,
+ f->dst, PAGE_SIZE), 0);
+ }
+}
+
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+static u64 mc16_test_refresh_cross_node_read(void)
+{
+ u64 pages = 0;
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ pages += READ_ONCE(per_cpu(mc16_refresh_cross_node_pages, cpu));
+ return pages;
+}
+
+static void memcomp_test_anon_store_cross_node_refresh(struct kunit *test)
+{
+ struct mc16_test_anon_store *f = mc16_test_anon_store_begin(test);
+ struct mc16_test_storage *s;
+ struct mc16_writeback snapshot = { };
+ struct mc16_writeback_tail tail = { };
+ struct mc16_swap_fields swap;
+ struct memcomp_lru *fifo_a, *fifo_b;
+ struct mc16_view view;
+ struct folio *source, *target;
+ u64 cross_before;
+ long fifo_a_before, fifo_b_before;
+ u16 eligible_before;
+ unsigned int bytes;
+ int nid, source_nid, target_nid = NUMA_NO_NODE;
+ bool valid;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ source = s->owner->folio;
+ source_nid = folio_nid(source);
+ for_each_node_state(nid, N_MEMORY) {
+ if (nid != source_nid) {
+ target_nid = nid;
+ break;
+ }
+ }
+ if (target_nid == NUMA_NO_NODE) {
+ kunit_skip(test, "requires two N_MEMORY nodes");
+ return;
+ }
+ target = __folio_alloc_node(GFP_KERNEL | __GFP_THISNODE, 1, target_nid);
+ if (!target) {
+ kunit_skip(test, "could not allocate target-node folio");
+ return;
+ }
+ folio_lock(target);
+ f->refresh_folio = target;
+ if (mem_cgroup_charge(target, NULL, GFP_KERNEL)) {
+ KUNIT_FAIL(test, "refresh source memcg charge failed");
+ return;
+ }
+ fifo_a = memcomp_lru_nid(source_nid);
+ fifo_b = memcomp_lru_nid(target_nid);
+ fifo_a_before = atomic_long_read(&fifo_a->nr_entries);
+ fifo_b_before = atomic_long_read(&fifo_b->nr_entries);
+ mc16_test_anon_fill(f, false);
+ for (nid = 0; nid < folio_nr_pages(target); nid++) {
+ void *dst = kmap_local_folio(target, nid * PAGE_SIZE);
+
+ memcpy(dst, f->src + nid * PAGE_SIZE, PAGE_SIZE);
+ kunmap_local(dst);
+ }
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_reserve(f), 0);
+ eligible_before = f->guards[0].cluster->eligible_slots;
+ KUNIT_ASSERT_PTR_EQ(test, f->guards[0].cluster,
+ f->guards[1].cluster);
+ for (nid = 0; nid < ARRAY_SIZE(f->guards); nid++) {
+ mutex_lock(memcomp_lock_of(f->guards[nid].token));
+ valid = mc16_view_locked(&f->guards[nid], &view) &&
+ view.member->owner_folio == source;
+ if (valid) {
+ folio_get(target);
+ view.member->owner_folio = target;
+ folio_put(source);
+ f->owner_moved[nid] = true;
+ }
+ mutex_unlock(memcomp_lock_of(f->guards[nid].token));
+ KUNIT_ASSERT_TRUE(test, valid);
+ }
+ KUNIT_EXPECT_EQ(test, folio_nid(target), target_nid);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(source), 1);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(target), 3);
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_ctx_get(f), 0);
+ cross_before = mc16_test_refresh_cross_node_read();
+ ret = mc16_test_anon_refresh(f);
+ mc16_test_anon_ctx_put(f);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ KUNIT_EXPECT_EQ(test, mc16_test_refresh_cross_node_read(), cross_before + 2);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(target), 1);
+ KUNIT_EXPECT_EQ(test, f->guards[0].cluster->key->nid, source_nid);
+ KUNIT_EXPECT_EQ(test, f->guards[1].cluster->key->nid, source_nid);
+ KUNIT_EXPECT_EQ(test, f->guards[0].cluster->eligible_slots,
+ (u16)(eligible_before + ARRAY_SIZE(f->guards)));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&fifo_a->nr_entries),
+ fifo_a_before + ARRAY_SIZE(f->guards));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&fifo_b->nr_entries), fifo_b_before);
+ /* Keep the NUMA/FIFO observations above in their prepared phase. */
+ KUNIT_ASSERT_TRUE(test, mc16_test_anon_finalize(f));
+
+ mutex_lock(memcomp_lock_of(f->guards[0].token));
+ valid = mc16_writeback_claim_locked(&f->guards[0], &snapshot);
+ mutex_unlock(memcomp_lock_of(f->guards[0].token));
+ KUNIT_ASSERT_TRUE(test, valid);
+ if (!mc16_length_decode(snapshot.length, &bytes)) {
+ mutex_lock(memcomp_lock_of(f->guards[0].token));
+ mc16_writeback_revert_locked(&f->guards[0]);
+ mutex_unlock(memcomp_lock_of(f->guards[0].token));
+ KUNIT_FAIL(test, "claimed writeback length did not decode");
+ return;
+ }
+ swap = (struct mc16_swap_fields) {
+ .type = MAX_SWAPFILES - 1,
+ .swap_offset = mc16_swap_offset_limit(),
+ .length = bytes,
+ };
+ memcomp_slot_dup(swp_entry(swap.type, swap.swap_offset));
+ mutex_lock(memcomp_lock_of(f->guards[0].token));
+ ret = mc16_writeback_settle_locked(&f->guards[0], &snapshot, &swap,
+ &tail);
+ if (ret)
+ mc16_writeback_revert_locked(&f->guards[0]);
+ mutex_unlock(memcomp_lock_of(f->guards[0].token));
+ if (ret) {
+ memcomp_slot_put(swp_entry(swap.type, swap.swap_offset));
+ KUNIT_FAIL(test, "writeback settle failed: %d", ret);
+ return;
+ }
+ mc16_writeback_finish(&f->guards[0], &tail);
+ mutex_lock(memcomp_lock_of(f->guards[0].token));
+ valid = mc16_view_locked(&f->guards[0], &view);
+ ret = valid ? mc16_flags_state(*view.flags) : MC16_EMPTY;
+ mutex_unlock(memcomp_lock_of(f->guards[0].token));
+ KUNIT_EXPECT_EQ(test, ret, (int)MC16_SWAPPED);
+}
+#endif
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_ANON_STORE_TEST_H */
diff --git a/mm/memcomp/tests/async_test.h b/mm/memcomp/tests/async_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..d02df557c24c511b60427bf67911f47a1ec76390
--- /dev/null
+++ b/mm/memcomp/tests/async_test.h
@@ -0,0 +1,703 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_ASYNC_TEST_H
+#define _MM_MEMCOMP_TESTS_ASYNC_TEST_H
+
+/*
+ * Include after shared_test.h. Definitions only.
+ * Run only with the public positive-token dispatch using clustered slots.
+ * The registered acomp frontend and codec work are real. Only objcg identity
+ * and charging use the shared explicit model; no swap device is involved.
+ */
+#ifdef CONFIG_MEMCG
+struct mc16_act_snapshot {
+ u64 word;
+ u32 refs;
+ u32 bytes;
+ u32 charge;
+ u16 flags;
+ bool attached;
+};
+
+struct mc16_act {
+ struct kunit *test;
+ struct memcomp_kunit_objcg model;
+ struct memcomp_kunit_async_probe probe;
+ struct crypto_acomp *tfm;
+ struct memcomp_async *async;
+ struct folio *folio[2];
+ struct memcomp_ref guards[2][MEMCOMP_MAX_COMPRESS_PAGES];
+ struct folio *dst;
+ unsigned long token[2];
+ unsigned int initialized;
+ unsigned int inflight[3];
+ bool ready_inflight[3];
+ bool stored[2];
+ bool finalized[2];
+ bool registered;
+ bool modeled;
+ bool observing;
+ bool ctx_cleared;
+ bool fail_consumed;
+ int acquisitions_left;
+ s64 stored_before;
+ s64 compressed_before;
+ u8 *packed;
+};
+
+/* Value/copy adapter only: a guard spans T, never a borrowed record pointer. */
+static bool mc16_act_snapshot(unsigned long token, struct mc16_act_snapshot *s,
+ void *packed)
+{
+ struct memcomp_ref ref = { };
+ struct mc16_view view;
+ void *src;
+ bool valid = false;
+
+ *s = (struct mc16_act_snapshot) { };
+ if (!memcomp_entry_get_ref(token, &ref))
+ return false;
+ mutex_lock(memcomp_lock_of(ref.token));
+ if (!mc16_view_locked(&ref, &view) || !mc16_flags_valid(*view.flags))
+ goto out;
+ s->word = *view.word;
+ s->flags = *view.flags;
+ s->refs = refcount_read(&ref.slot->refs);
+ s->attached = !!view.record;
+ s->charge = view.member ? view.member->payload_charge : 0;
+ if (mc16_flags_state(s->flags) == MC16_PENDING)
+ valid = !*view.length;
+ else
+ valid = mc16_payload_length_decode(mc16_flags_state(s->flags),
+ *view.length, &s->bytes);
+ if (!valid || !packed || mc16_flags_state(s->flags) != MC16_STORED)
+ goto out;
+ if (!s->word || !s->bytes || s->bytes > PAGE_SIZE) {
+ valid = false;
+ goto out;
+ }
+ src = zs_map_object(memcomp_pools[0], s->word, ZS_MM_RO);
+ if (!src) {
+ valid = false;
+ goto out;
+ }
+ memcpy(packed, src, s->bytes);
+ zs_unmap_object(memcomp_pools[0], s->word);
+out:
+ mutex_unlock(memcomp_lock_of(ref.token));
+ memcomp_entry_put_ref(&ref);
+ return valid;
+}
+
+static void mc16_act_cleanup(void *data)
+{
+ struct mc16_act *f = data;
+ unsigned int i, page;
+
+ /* Open every gate first: the shared frontend uses one serial workqueue. */
+ for (i = 0; i < f->initialized; i++) {
+ struct memcomp_test_acomp_req *r;
+
+ r = acomp_request_ctx(f->async->reqs[i].req);
+ complete_all(&r->release);
+ }
+ for (i = 0; i < f->initialized; i++) {
+ struct memcomp_test_acomp_req *r;
+
+ r = acomp_request_ctx(f->async->reqs[i].req);
+ flush_work(&r->work);
+ }
+ /* Fatal-assert actions can run in a different KUnit cleanup task. */
+ if (f->observing)
+ WRITE_ONCE(f->probe.task, current);
+ for (i = 0; i < ARRAY_SIZE(f->folio); i++) {
+ struct folio *folio = f->folio[i];
+ int refs;
+
+ if (!folio)
+ continue;
+ folio_lock(folio);
+ if (!f->finalized[i]) {
+ if (f->stored[i])
+ memcomp_invalidate_folio(folio);
+ else
+ memcomp_rollback_folio(folio);
+ }
+ /* OWNER is gone; guards are the final payload lifetime holders. */
+ for (page = 0; page < ARRAY_SIZE(f->guards[i]); page++) {
+ if (f->guards[i][page].slot)
+ memcomp_entry_put_ref(&f->guards[i][page]);
+ }
+ refs = folio_ref_count(folio);
+ folio_unlock(folio);
+ KUNIT_EXPECT_EQ(f->test, refs, 1);
+ folio_put(folio);
+ f->folio[i] = NULL;
+ }
+ if (f->observing) {
+ KUNIT_EXPECT_EQ(f->test, f->probe.frees, f->probe.allocs);
+ KUNIT_EXPECT_FALSE(f->test, f->probe.live);
+ KUNIT_EXPECT_FALSE(f->test, f->probe.bad_lifetime);
+ KUNIT_EXPECT_PTR_EQ(f->test,
+ rcu_access_pointer(memcomp_kunit_async_probe),
+ &f->probe);
+ rcu_assign_pointer(memcomp_kunit_async_probe, NULL);
+ synchronize_rcu();
+ }
+ if (f->async)
+ memcomp_async_free_batch(f->async);
+ if (f->tfm)
+ crypto_free_acomp(f->tfm);
+ if (f->registered)
+ crypto_unregister_acomp(&memcomp_test_acomp_alg);
+ if (f->dst)
+ folio_put(f->dst);
+ mc16_test_reclaim_retained();
+ if (f->modeled) {
+ KUNIT_EXPECT_TRUE(f->test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_EQ(f->test, atomic_long_read(&f->model.charged), 0L);
+ KUNIT_EXPECT_EQ(f->test, atomic_read(&f->model.get_calls),
+ atomic_read(&f->model.put_calls));
+ rcu_assign_pointer(memcomp_kunit_objcg, NULL);
+ synchronize_rcu();
+ }
+ KUNIT_EXPECT_EQ(f->test, atomic64_read(&memcomp_stored_bytes),
+ f->stored_before);
+ KUNIT_EXPECT_EQ(f->test, atomic64_read(&memcomp_compressed_bytes),
+ f->compressed_before);
+}
+
+/* Return unlocked, including every allocation/registration failure. */
+static struct mc16_act *mc16_act_begin(struct kunit *test, bool large,
+ bool prefix, bool samefill)
+{
+ struct mc16_act *f;
+ unsigned int i, page, nr = (large ? 2 : 1) + prefix;
+ int ret;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return NULL;
+ }
+ if (nr > MEMCOMP_ASYNC_PAGES || (large && MEMCOMP_MAX_COMPRESS_PAGES < 2)) {
+ kunit_skip(test, "two-page compression batch unavailable");
+ return NULL;
+ }
+ rcu_barrier();
+ if (rcu_access_pointer(memcomp_kunit_objcg) ||
+ rcu_access_pointer(memcomp_kunit_async_probe) || !xa_empty(&mc16_clusters)) {
+ KUNIT_FAIL(test, "async fixture requires idle storage/objcg/probe");
+ return NULL;
+ }
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ goto fail;
+ f->test = test;
+ f->stored_before = atomic64_read(&memcomp_stored_bytes);
+ f->compressed_before = atomic64_read(&memcomp_compressed_bytes);
+ if (kunit_add_action_or_reset(test, mc16_act_cleanup, f))
+ goto fail;
+ ret = crypto_register_acomp(&memcomp_test_acomp_alg);
+ if (ret) {
+ KUNIT_FAIL(test, "registered acomp frontend: %d", ret);
+ goto fail;
+ }
+ f->registered = true;
+ f->tfm = crypto_alloc_acomp("memcomp-kunit-acomp", CRYPTO_ALG_ASYNC,
+ CRYPTO_ALG_ASYNC | CRYPTO_NOLOAD);
+ if (IS_ERR(f->tfm)) {
+ ret = PTR_ERR(f->tfm);
+ f->tfm = NULL;
+ KUNIT_FAIL(test, "registered acomp tfm: %d", ret);
+ goto fail;
+ }
+ f->async = kzalloc(sizeof(*f->async), GFP_KERNEL);
+ if (!f->async)
+ goto fail;
+ init_completion(&f->async->done);
+ atomic_set(&f->async->pending, 1);
+ for (i = 0; i < nr; i++) {
+ struct memcomp_async_req *req = &f->async->reqs[i];
+ struct memcomp_test_acomp_req *r;
+
+ req->async = f->async;
+ req->req = acomp_request_alloc(f->tfm);
+ if (!req->req)
+ goto fail;
+ req->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
+ if (!req->buf)
+ goto fail;
+ r = acomp_request_ctx(req->req);
+ /* The crypto request's private bytes are not guaranteed zeroed. */
+ memset(r, 0, sizeof(*r));
+ INIT_WORK(&r->work, memcomp_test_acomp_work);
+ init_completion(&r->started);
+ init_completion(&r->release);
+ init_completion(&r->finished);
+ atomic_set(&r->submitted, 0);
+ atomic_set(&r->completed, 0);
+ f->initialized++;
+ }
+ f->dst = folio_alloc(GFP_KERNEL, 0);
+ f->packed = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ if (!f->dst || !f->packed)
+ goto fail;
+ f->folio[0] = folio_alloc(GFP_KERNEL | __GFP_ZERO, large ? 1 : 0);
+ if (prefix)
+ f->folio[1] = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0);
+ if (!f->folio[0] || (prefix && !f->folio[1]))
+ goto fail;
+ for (i = 0; i < ARRAY_SIZE(f->folio); i++) {
+ struct folio *folio = f->folio[i];
+
+ if (!folio)
+ continue;
+ folio->mapping = (struct address_space *)PAGE_MAPPING_ANON;
+ __folio_set_swapbacked(folio);
+ /* Finalize requires real source-folio memcg identity. */
+ if (mem_cgroup_charge(folio, NULL, GFP_KERNEL))
+ goto fail;
+ for (page = 0; page < folio_nr_pages(folio); page++) {
+ u8 *src = kmap_local_folio(folio, page * PAGE_SIZE);
+
+ memset(src, 0x31 + 0x17 * page + 0x23 * i, PAGE_SIZE);
+ if (!samefill)
+ src[17] ^= 0xff;
+ kunmap_local(src);
+ }
+ }
+ f->model.task = current;
+ f->model.error = -ENOMEM;
+ rcu_assign_pointer(memcomp_kunit_objcg, &f->model);
+ f->modeled = true;
+ return f;
+fail:
+ KUNIT_FAIL(test, "async fixture setup failed");
+ return NULL;
+}
+
+static bool mc16_act_reserve(struct mc16_act *f, unsigned int index)
+{
+ struct folio *folio = f->folio[index];
+ bool reserved;
+
+ f->model.folio = folio;
+ folio_lock(folio);
+ reserved = memcomp_reserve(folio);
+ if (reserved)
+ f->token[index] = swp_offset(folio->swap);
+ folio_unlock(folio);
+ return reserved;
+}
+
+/* Preserve the locked-source contract until each real callback has returned. */
+static int mc16_act_submit(struct mc16_act *f, unsigned int index)
+{
+ struct folio *folio = f->folio[index];
+ unsigned int first = f->async->nr, i;
+ unsigned long waited;
+ int ret = -EINVAL;
+
+ folio_lock(folio);
+ if (!memcomp_async_submit(f->async, folio))
+ goto out;
+ ret = 0;
+ for (i = first; i < f->async->nr; i++) {
+ struct memcomp_test_acomp_req *r;
+
+ r = acomp_request_ctx(f->async->reqs[i].req);
+ if (!atomic_read(&r->submitted))
+ continue;
+ waited = wait_for_completion_timeout(&r->started, HZ);
+ if (!waited) {
+ ret = -ETIMEDOUT;
+ goto out;
+ }
+ /* Progress (-EINPROGRESS) must not consume the terminal hold. */
+ f->inflight[i] = atomic_read(&f->async->pending);
+ f->ready_inflight[i] = memcomp_async_complete(f->async);
+ if (atomic_read(&r->completed)) {
+ ret = -EIO;
+ goto out;
+ }
+ complete_all(&r->release);
+ waited = wait_for_completion_timeout(&r->finished, HZ);
+ if (!waited) {
+ ret = -ETIMEDOUT;
+ goto out;
+ }
+ flush_work(&r->work);
+ }
+out:
+ /* Also open gates on failure before releasing the private source folio. */
+ for (i = first; i < f->async->nr; i++) {
+ struct memcomp_test_acomp_req *r;
+
+ r = acomp_request_ctx(f->async->reqs[i].req);
+ complete_all(&r->release);
+ }
+ for (i = first; i < f->async->nr; i++) {
+ struct memcomp_test_acomp_req *r;
+
+ r = acomp_request_ctx(f->async->reqs[i].req);
+ flush_work(&r->work);
+ }
+ folio_unlock(folio);
+ return ret;
+}
+
+static bool mc16_act_wait(struct mc16_act *f)
+{
+ /* Refuse an unbounded production wait if callback accounting is broken. */
+ if (!memcomp_async_complete(f->async) ||
+ atomic_read(&f->async->pending) != 1)
+ return false;
+ memcomp_async_wait(f->async);
+ return atomic_read(&f->async->pending) == 0;
+}
+
+/* No context/folio lock or modified task flags escape into an assertion. */
+static int mc16_act_store(struct mc16_act *f, unsigned int index,
+ bool finish_fail, bool lease_check, bool poison)
+{
+ struct folio *folio = f->folio[index];
+ struct memcomp_reclaim_ctx *ctx;
+ unsigned int nr = folio_nr_pages(folio), reclaim_flags = 0;
+ bool stored;
+
+ if (atomic_read(&memcomp_kunit_ctx_get_fail) ||
+ READ_ONCE(memcomp_kunit_fail.task))
+ return -EBUSY;
+ if (nr > 1)
+ reclaim_flags = memcomp_test_reclaimer_enter();
+ folio_lock(folio);
+ ctx = memcomp_reclaim_ctx_get(nr, nr > 1 ? MEMCOMP_RECLAIM_KSWAPD :
+ MEMCOMP_RECLAIM_DIRECT);
+ if (!ctx) {
+ folio_unlock(folio);
+ if (nr > 1)
+ memcomp_test_reclaimer_exit(reclaim_flags);
+ return -EBUSY;
+ }
+ /* An error injected in this precise leased object identifies its user. */
+ if (poison)
+ ctx->kunit_compress_error = -EBADMSG;
+ if (lease_check)
+ atomic_set(&memcomp_kunit_ctx_get_fail, 2);
+ if (finish_fail)
+ memcomp_kunit_fail_next(MEMCOMP_KUNIT_FAIL_REFRESH_COMMIT,
+ 1, -EAGAIN);
+ stored = memcomp_store_after_unmap_async(folio, &ctx, f->async);
+ f->stored[index] = stored;
+ f->ctx_cleared = !ctx;
+ f->fail_consumed = !READ_ONCE(memcomp_kunit_fail.task) &&
+ memcomp_kunit_fail.stage == MEMCOMP_KUNIT_FAIL_NONE &&
+ !memcomp_kunit_fail.remaining;
+ if (finish_fail)
+ memcomp_kunit_fail_clear();
+ if (lease_check) {
+ f->acquisitions_left = atomic_read(&memcomp_kunit_ctx_get_fail);
+ atomic_set(&memcomp_kunit_ctx_get_fail, 0);
+ }
+ if (ctx)
+ memcomp_reclaim_ctx_put(ctx);
+ folio_unlock(folio);
+ if (nr > 1)
+ memcomp_test_reclaimer_exit(reclaim_flags);
+ return stored;
+}
+
+/* Snapshot assertions run first: persistent guards would change refs. */
+static bool mc16_act_finalize(struct mc16_act *f, unsigned int index)
+{
+ struct folio *folio = f->folio[index];
+ unsigned int page, nr = folio_nr_pages(folio);
+ bool released;
+
+ if (!f->stored[index] || f->finalized[index])
+ return false;
+ for (page = 0; page < nr; page++) {
+ if (!f->guards[index][page].slot &&
+ !memcomp_entry_get_ref(f->token[index] + page,
+ &f->guards[index][page]))
+ return false;
+ }
+ /* These allocator sources were never on LRU; no isolation ref exists. */
+ folio_lock(folio);
+ released = memcomp_release_folio(folio);
+ if (released) {
+ folio_ref_unfreeze(folio, 1);
+ f->finalized[index] = true;
+ }
+ folio_unlock(folio);
+ return released;
+}
+
+static int mc16_act_load(struct mc16_act *f, unsigned int index,
+ unsigned int page, bool *same)
+{
+ struct memcomp_ref ref = { };
+ void *src, *dst;
+ int ret;
+
+ *same = false;
+ if (!memcomp_entry_get_ref(f->token[index] + page, &ref))
+ return -ENOENT;
+ folio_lock(f->dst);
+ ret = memcomp_load_folio(&ref, f->dst);
+ if (!ret) {
+ src = kmap_local_folio(f->folio[index], page * PAGE_SIZE);
+ dst = kmap_local_folio(f->dst, 0);
+ *same = !memcmp(src, dst, PAGE_SIZE);
+ kunmap_local(dst);
+ kunmap_local(src);
+ }
+ folio_unlock(f->dst);
+ memcomp_entry_put_ref(&ref);
+ return ret;
+}
+
+static void mc16_act_frontend(struct kunit *test, bool backlog, int error,
+ bool finish_fail, bool samefill)
+{
+ struct mc16_act *f = mc16_act_begin(test, false, false, samefill);
+ struct mc16_act_snapshot s;
+ struct memcomp_test_acomp_req *r;
+ struct memcomp_async_req *req;
+ unsigned long token;
+ long charged;
+ bool same;
+ int ret;
+
+ if (!f)
+ return;
+ req = &f->async->reqs[0];
+ r = acomp_request_ctx(req->req);
+ r->backlog = backlog;
+ r->error = error;
+ KUNIT_ASSERT_TRUE(test, mc16_act_reserve(f, 0));
+ token = f->token[0];
+ KUNIT_ASSERT_NE(test, token, 0UL);
+ charged = atomic_long_read(&f->model.charged);
+ KUNIT_ASSERT_EQ(test, mc16_act_submit(f, 0), 0);
+ KUNIT_EXPECT_EQ(test, f->async->nr, 1U);
+ KUNIT_EXPECT_EQ(test, atomic_read(&r->submitted), samefill ? 0 : 1);
+ KUNIT_EXPECT_EQ(test, atomic_read(&r->completed), samefill ? 0 : 1);
+ if (!samefill) {
+ KUNIT_EXPECT_EQ(test, f->inflight[0], 2U);
+ KUNIT_EXPECT_FALSE(test, f->ready_inflight[0]);
+ }
+ KUNIT_ASSERT_TRUE(test, mc16_act_wait(f));
+ KUNIT_EXPECT_EQ(test, req->ret, samefill ? 0 : error);
+ KUNIT_EXPECT_EQ(test, req->same_filled, samefill);
+ f->probe.task = current;
+ f->probe.folio = f->folio[0];
+ f->probe.request = req;
+ rcu_assign_pointer(memcomp_kunit_async_probe, &f->probe);
+ f->observing = true;
+ ret = mc16_act_store(f, 0, finish_fail, false, false);
+ KUNIT_EXPECT_EQ(test, ret, finish_fail ? 0 : 1);
+ KUNIT_EXPECT_TRUE(test, f->ctx_cleared);
+ KUNIT_EXPECT_TRUE(test, f->fail_consumed);
+ KUNIT_EXPECT_EQ(test, f->probe.commit_fail_hits, finish_fail ? 1U : 0U);
+ if (finish_fail)
+ KUNIT_EXPECT_EQ(test, f->probe.commit_error, -EAGAIN);
+ KUNIT_EXPECT_EQ(test, f->probe.precomp_hits, error ? 0U : 1U);
+ KUNIT_EXPECT_EQ(test, f->probe.allocs, samefill ? 0U : 1U);
+ KUNIT_EXPECT_EQ(test, f->probe.frees, finish_fail ? 1U : 0U);
+ KUNIT_EXPECT_EQ(test, f->probe.live, !samefill && !finish_fail);
+ KUNIT_EXPECT_FALSE(test, f->probe.bad_lifetime);
+ KUNIT_ASSERT_TRUE(test, mc16_act_snapshot(token, &s, f->packed));
+ KUNIT_EXPECT_TRUE(test, s.attached);
+ KUNIT_EXPECT_EQ(test, s.refs, 2U); /* owner + snapshot's lookup */
+ KUNIT_EXPECT_TRUE(test, s.flags & MC16_F_OWNER);
+ KUNIT_EXPECT_EQ(test, !!(s.flags & MC16_F_RESIDENT), finish_fail);
+ if (finish_fail) {
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(s.flags), MC16_PENDING);
+ KUNIT_EXPECT_EQ(test, s.charge, 0U);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->model.charged), charged);
+ } else {
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(s.flags),
+ samefill ? MC16_SAMEFILL : MC16_STORED);
+ KUNIT_EXPECT_EQ(test, s.charge, s.bytes);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->model.charged),
+ charged + s.bytes);
+ if (samefill)
+ KUNIT_EXPECT_EQ(test, s.word, (u64)req->element);
+ if (!samefill) {
+ KUNIT_EXPECT_PTR_EQ(test, f->probe.pool, memcomp_pools[0]);
+ KUNIT_EXPECT_EQ(test, s.word, (u64)f->probe.handle);
+ KUNIT_ASSERT_GT(test, s.bytes, 0U);
+ KUNIT_ASSERT_LT(test, s.bytes, (u32)PAGE_SIZE);
+ if (!error) {
+ KUNIT_ASSERT_EQ(test, s.bytes, req->len);
+ KUNIT_EXPECT_MEMEQ(test, f->packed, req->buf, s.bytes);
+ }
+ }
+ KUNIT_ASSERT_TRUE(test, mc16_act_finalize(f, 0));
+ ret = mc16_act_load(f, 0, 0, &same);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ KUNIT_EXPECT_TRUE(test, same);
+ }
+ kunit_release_action(test, mc16_act_cleanup, f);
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_present(token));
+}
+
+static void memcomp_test_async_registered_submit(struct kunit *test)
+{
+ mc16_act_frontend(test, false, 0, false, false);
+}
+
+static void memcomp_test_async_registered_backlog(struct kunit *test)
+{
+ mc16_act_frontend(test, true, 0, false, false);
+}
+
+static void memcomp_test_async_registered_error(struct kunit *test)
+{
+ mc16_act_frontend(test, false, -ENODATA, false, false);
+}
+
+static void memcomp_test_async_registered_finish_fail(struct kunit *test)
+{
+ mc16_act_frontend(test, false, 0, true, false);
+}
+
+static void memcomp_test_async_software_samefill(struct kunit *test)
+{
+ mc16_act_frontend(test, false, 0, false, true);
+}
+
+/* Both requests really fail in the registered codec; store must reuse ctx. */
+static void memcomp_test_async_lease_identity(struct kunit *test)
+{
+ struct mc16_act *f = mc16_act_begin(test, true, false, false);
+ struct mc16_act_snapshot s;
+ unsigned int i;
+ long charged;
+ bool same;
+ int ret;
+
+ if (!f)
+ return;
+ for (i = 0; i < 2; i++) {
+ struct memcomp_test_acomp_req *r;
+
+ r = acomp_request_ctx(f->async->reqs[i].req);
+ r->error = -ENODATA;
+ }
+ KUNIT_ASSERT_TRUE(test, mc16_act_reserve(f, 0));
+ charged = atomic_long_read(&f->model.charged);
+ KUNIT_ASSERT_EQ(test, mc16_act_submit(f, 0), 0);
+ KUNIT_ASSERT_TRUE(test, mc16_act_wait(f));
+ for (i = 0; i < 2; i++)
+ KUNIT_ASSERT_EQ(test, f->async->reqs[i].ret, -ENODATA);
+ /* Only the exact caller-held ctx contains this one-shot error. */
+ ret = mc16_act_store(f, 0, false, true, true);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ KUNIT_EXPECT_TRUE(test, f->ctx_cleared);
+ KUNIT_EXPECT_EQ(test, f->acquisitions_left, 2);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->model.charged), charged);
+ for (i = 0; i < 2; i++) {
+ KUNIT_ASSERT_TRUE(test, mc16_act_snapshot(f->token[0] + i, &s, NULL));
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(s.flags), MC16_PENDING);
+ KUNIT_EXPECT_EQ(test, s.charge, 0U);
+ }
+ /* Same live reservation and real outputs: unpoisoned lease must succeed. */
+ ret = mc16_act_store(f, 0, false, true, false);
+ KUNIT_ASSERT_EQ(test, ret, 1);
+ KUNIT_EXPECT_TRUE(test, f->ctx_cleared);
+ KUNIT_EXPECT_EQ(test, f->acquisitions_left, 2);
+ KUNIT_ASSERT_TRUE(test, mc16_act_finalize(f, 0));
+ for (i = 0; i < 2; i++) {
+ ret = mc16_act_load(f, 0, i, &same);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ KUNIT_EXPECT_TRUE(test, same);
+ }
+ kunit_release_action(test, mc16_act_cleanup, f);
+}
+
+/* A distinct folio precedes a two-page target; page 1 may require fallback. */
+static void mc16_act_precomp_order(struct kunit *test, bool mixed)
+{
+ struct mc16_act *f = mc16_act_begin(test, true, true, false);
+ struct mc16_act_snapshot s;
+ unsigned long token, prefix;
+ unsigned int i;
+ bool same;
+ int ret;
+
+ if (!f)
+ return;
+ if (mixed) {
+ struct memcomp_test_acomp_req *r;
+
+ r = acomp_request_ctx(f->async->reqs[2].req);
+ r->error = -EIO;
+ }
+ KUNIT_ASSERT_TRUE(test, mc16_act_reserve(f, 1));
+ KUNIT_ASSERT_TRUE(test, mc16_act_reserve(f, 0));
+ token = f->token[0];
+ prefix = f->token[1];
+ KUNIT_ASSERT_EQ(test, mc16_act_submit(f, 1), 0);
+ KUNIT_ASSERT_EQ(test, mc16_act_submit(f, 0), 0);
+ KUNIT_ASSERT_EQ(test, f->async->nr, 3U);
+ KUNIT_ASSERT_TRUE(test, mc16_act_wait(f));
+ KUNIT_EXPECT_PTR_EQ(test, f->async->reqs[0].folio, f->folio[1]);
+ KUNIT_EXPECT_PTR_EQ(test, f->async->reqs[1].folio, f->folio[0]);
+ KUNIT_EXPECT_PTR_EQ(test, f->async->reqs[2].folio, f->folio[0]);
+ /* Store out of submission order, exercising selection by folio identity. */
+ ret = mc16_act_store(f, 0, false, true, false);
+ KUNIT_ASSERT_EQ(test, ret, 1);
+ KUNIT_EXPECT_TRUE(test, f->ctx_cleared);
+ KUNIT_EXPECT_EQ(test, f->acquisitions_left, 2);
+ for (i = 0; i < 2; i++) {
+ struct memcomp_async_req *req = &f->async->reqs[i + 1];
+ struct memcomp_test_acomp_req *r = acomp_request_ctx(req->req);
+
+ KUNIT_EXPECT_EQ(test, atomic_read(&r->submitted), 1);
+ KUNIT_EXPECT_EQ(test, atomic_read(&r->completed), 1);
+ KUNIT_EXPECT_EQ(test, req->ret, mixed && i ? -EIO : 0);
+ KUNIT_ASSERT_TRUE(test, mc16_act_snapshot(token + i, &s, f->packed));
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(s.flags), MC16_STORED);
+ KUNIT_EXPECT_TRUE(test, s.flags & MC16_F_OWNER);
+ KUNIT_EXPECT_FALSE(test, s.flags & MC16_F_RESIDENT);
+ KUNIT_EXPECT_EQ(test, s.charge, s.bytes);
+ if (!mixed || !i) {
+ KUNIT_ASSERT_EQ(test, s.bytes, req->len);
+ KUNIT_EXPECT_MEMEQ(test, f->packed, req->buf, s.bytes);
+ }
+ }
+ /* Finish every original OWNER/payload snapshot before detaching the span. */
+ KUNIT_ASSERT_TRUE(test, mc16_act_finalize(f, 0));
+ for (i = 0; i < 2; i++) {
+ ret = mc16_act_load(f, 0, i, &same);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ KUNIT_EXPECT_TRUE(test, same);
+ }
+ /* Target publication must not consume the earlier folio's reservation. */
+ KUNIT_ASSERT_TRUE(test, mc16_act_snapshot(prefix, &s, NULL));
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(s.flags), MC16_PENDING);
+ KUNIT_EXPECT_TRUE(test, s.flags & MC16_F_RESIDENT);
+ ret = mc16_act_store(f, 1, false, false, false);
+ KUNIT_ASSERT_EQ(test, ret, 1);
+ KUNIT_ASSERT_TRUE(test, mc16_act_finalize(f, 1));
+ ret = mc16_act_load(f, 1, 0, &same);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ KUNIT_EXPECT_TRUE(test, same);
+ kunit_release_action(test, mc16_act_cleanup, f);
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_present(token));
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_present(token + 1));
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_present(prefix));
+}
+
+static void memcomp_test_async_precomp_offset(struct kunit *test)
+{
+ mc16_act_precomp_order(test, false);
+}
+
+static void memcomp_test_async_precomp_mixed(struct kunit *test)
+{
+ mc16_act_precomp_order(test, true);
+}
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_ASYNC_TEST_H */
diff --git a/mm/memcomp/tests/budget_packed_test.h b/mm/memcomp/tests/budget_packed_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..aa436a145deafcc24eb6b9fa1f6a4546d8982ecd
--- /dev/null
+++ b/mm/memcomp/tests/budget_packed_test.h
@@ -0,0 +1,435 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_BUDGET_PACKED_TEST_H
+#define _MM_MEMCOMP_TESTS_BUDGET_PACKED_TEST_H
+
+/* Include after the frozen swapped_test.h. Own mode: do not use raw cleanup. */
+#if defined(MC16_FOLIO_TEST_ENABLED) && defined(CONFIG_SWAP)
+struct mc16_bpp {
+ struct kunit *test;
+ struct mc16_bm *bm;
+ u32 tokens[MC16_TEST_WORKERS];
+ unsigned int nr;
+ struct swap_info_struct *device;
+ swp_entry_t swap;
+ u32 lengths[MC16_TEST_WORKERS];
+ bool active, ready;
+};
+
+static void mc16_bpp_cleanup(void *data)
+{
+ struct mc16_bpp *f = data;
+ unsigned int i;
+
+ kunit_release_action(f->test, mc16_bm_cleanup, f->bm);
+ kunit_release_action(f->test, mc16_flt_cleanup, f->bm->seed);
+ /* All fixture PTEs are removed before real swap mode ends. */
+ mc16_test_reclaim_retained();
+ if (f->device) {
+ KUNIT_EXPECT_EQ(f->test, __swap_count(f->swap), 0);
+ put_swap_device(f->device);
+ }
+ if (f->active) {
+ KUNIT_EXPECT_TRUE(f->test, xa_empty(&mc16_clusters));
+ if (f->ready) {
+ for (i = 0; i < f->nr; i++)
+ KUNIT_EXPECT_FALSE(f->test, memcomp_entry_present(f->tokens[i]));
+ }
+ WRITE_ONCE(mc16_kunit_real_swap, false);
+ synchronize_rcu();
+ f->active = false;
+ }
+}
+
+static struct mc16_bpp *mc16_bpp_begin(struct kunit *test, unsigned int nr)
+{
+ struct mc16_bpp *f;
+ bool idle;
+
+ if (get_nr_swap_pages() < 1) {
+ kunit_skip(test, "requires real guest swap and the raw bridge");
+ return NULL;
+ }
+ if (READ_ONCE(mc16_kunit_real_swap)) {
+ KUNIT_FAIL(test, "another real-swap fixture is active");
+ return NULL;
+ }
+ mutex_lock(&memcomp_writeback_lock);
+ idle = !mc16_writeback_batch_used && !mc16_writeback_members_used &&
+ !mc16_writeback_open_first && !mc16_writeback_open_bytes &&
+ !memcomp_writeback_slots_free && !memcomp_writeback_plugged;
+ mutex_unlock(&memcomp_writeback_lock);
+ if (!idle) {
+ KUNIT_FAIL(test, "writeback state is not idle");
+ return NULL;
+ }
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->test = test;
+ f->bm = mc16_bm_begin(test, nr, false);
+ if (!f->bm)
+ return NULL;
+ if (kunit_add_action_or_reset(test, mc16_bpp_cleanup, f))
+ return NULL;
+ f->nr = nr;
+ f->active = true;
+ WRITE_ONCE(mc16_kunit_real_swap, true);
+ return f;
+}
+
+static int mc16_bpp_fields(struct mc16_bpp *f, unsigned int index, bool swapped,
+ struct mc16_swap_fields *fields)
+{
+ struct memcomp_ref ref = { };
+ struct mc16_view view;
+ unsigned int state = swapped ? MC16_SWAPPED : MC16_STORED;
+ bool valid;
+
+ if (!mc16_get(f->tokens[index], &ref))
+ return -ENOENT;
+ mutex_lock(memcomp_lock_of(ref.token));
+ valid = mc16_view_locked(&ref, &view) && mc16_flags_valid(*view.flags) &&
+ mc16_flags_state(*view.flags) == state && !view.member &&
+ !(*view.flags & (MC16_F_OWNER | MC16_F_RESIDENT));
+ if (valid && swapped)
+ valid = mc16_swap_decode(*view.word, *view.length, fields) && fields->packed;
+ else if (valid)
+ valid = mc16_length_decode(*view.length, &f->lengths[index]);
+ mutex_unlock(memcomp_lock_of(ref.token));
+ mc16_ref_put(&ref);
+ return valid ? 0 : -EIO;
+}
+
+static void mc16_bpp_expect_distinct(struct kunit *test, struct mc16_bpp *f)
+{
+ u8 *first, *second;
+ int different;
+
+ first = kmap_local_folio(f->bm->page[0].folio, 0);
+ second = kmap_local_folio(f->bm->page[1].folio, 0);
+ different = memcmp(first, second, PAGE_SIZE);
+ kunmap_local(second);
+ kunmap_local(first);
+ KUNIT_EXPECT_NE(test, different, 0);
+}
+
+/* The first packed fault leaves the shared swap-cache folio available. */
+static void mc16_bpp_expect_tail(struct kunit *test, struct mc16_bpp *f)
+{
+ struct folio *folio;
+ u8 *data, *zero;
+ unsigned int i, off = 0;
+ bool allocated = false;
+
+ for (i = 0; i < f->nr; i++) {
+ KUNIT_ASSERT_GT(test, f->lengths[i], 0U);
+ off += f->lengths[i];
+ }
+ KUNIT_ASSERT_LT(test, off, PAGE_SIZE);
+ zero = kunit_kzalloc(test, PAGE_SIZE - off, GFP_KERNEL);
+ if (!zero)
+ return;
+ folio = __read_swap_cache_async(f->swap,
+ GFP_HIGHUSER_MOVABLE | __GFP_NOWARN,
+ get_task_policy(current), NO_INTERLEAVE_INDEX,
+ &allocated, false);
+ if (!folio) {
+ KUNIT_FAIL(test, "packed swap-cache lookup failed");
+ return;
+ }
+ if (allocated) {
+ folio_put(folio);
+ KUNIT_FAIL(test, "packed slot was not retained in swap cache");
+ return;
+ }
+ folio_lock(folio);
+ if (!folio_test_uptodate(folio)) {
+ KUNIT_FAIL(test, "packed swap-cache folio is not uptodate");
+ goto out_unlock;
+ }
+ data = kmap_local_folio(folio, 0);
+ KUNIT_EXPECT_EQ(test, memcmp(data + off, zero, PAGE_SIZE - off), 0);
+ kunmap_local(data);
+out_unlock:
+ folio_unlock(folio);
+ folio_put(folio);
+}
+
+static int mc16_bpp_drain(struct mc16_bpp *f)
+{
+ struct memcomp_lru *fifo = memcomp_lru_nid(folio_nid(f->bm->page[0].folio));
+ unsigned long scanned, freed = 0;
+ unsigned int noreclaim;
+ bool attempted = false;
+ int ret = 0;
+
+ if (!fifo)
+ return -EINVAL;
+ noreclaim = memalloc_noreclaim_save();
+ mutex_lock(&memcomp_writeback_lock);
+ scanned = mc16_writeback_drain_queue(folio_nid(f->bm->page[0].folio), fifo,
+ f->nr, &freed, &ret, &attempted);
+ mutex_unlock(&memcomp_writeback_lock);
+ memalloc_noreclaim_restore(noreclaim);
+ return ret ? ret : scanned == f->nr && freed == f->nr &&
+ attempted ? 0 : -EIO;
+}
+
+/* Page out each token, then validate the shared packed backing layout. */
+static int mc16_bpp_prepare(struct mc16_bpp *f)
+{
+ struct kunit *test = f->test;
+ unsigned int i, offset = 0;
+ int ret;
+
+ if (f->nr > 1)
+ mc16_bpp_expect_distinct(test, f);
+ for (i = 0; i < f->nr; i++) {
+ pte_t pte;
+
+ if (folio_nid(f->bm->page[i].folio) != folio_nid(f->bm->page[0].folio))
+ return -EXDEV;
+ ret = mc16_bm_pageout(&f->bm->page[i]);
+ if (ret)
+ return ret;
+ pte = mc16_flt_read(f->bm->seed, f->bm->page[i].addr);
+ if (!is_swap_pte(pte) || !is_memcomp_entry(pte_to_swp_entry(pte)))
+ return -EINVAL;
+ f->tokens[i] = swp_offset(pte_to_swp_entry(pte));
+ kunit_info(test, "packed source pageout index=%u token=%u\n", i, f->tokens[i]);
+ ret = mc16_bpp_fields(f, i, false, NULL);
+ if (ret)
+ return ret;
+ if (!f->lengths[i] || f->lengths[i] >= PAGE_SIZE ||
+ offset > PAGE_SIZE - f->lengths[i])
+ return -E2BIG;
+ offset += f->lengths[i];
+ }
+ ret = mc16_bpp_drain(f);
+ if (ret)
+ return ret;
+ offset = 0;
+ for (i = 0; i < f->nr; i++) {
+ struct mc16_swap_fields fields;
+ swp_entry_t swap;
+
+ ret = mc16_bpp_fields(f, i, true, &fields);
+ if (ret)
+ return ret;
+ swap = swp_entry(fields.type, fields.swap_offset);
+ if (fields.byte_offset != offset || fields.length != f->lengths[i])
+ return -EIO;
+ if (!i) {
+ f->swap = swap;
+ f->device = get_swap_device(swap);
+ if (!f->device)
+ return -ENODEV;
+ } else if (swap.val != f->swap.val) {
+ return -EIO;
+ }
+ offset += fields.length;
+ }
+ f->ready = true;
+ return __swap_count(f->swap) == f->nr ? 0 : -EIO;
+}
+
+static void mc16_bpp_expect_idle(struct kunit *test)
+{
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_slots_free, 0U);
+ KUNIT_EXPECT_EQ(test, mc16_writeback_members_used, 0U);
+ KUNIT_EXPECT_EQ(test, mc16_writeback_batch_used, 0U);
+}
+
+static void mc16_bpp_fault(struct kunit *test, struct mc16_bpp *f, unsigned int index)
+{
+ bool unlocked;
+ vm_fault_t ret;
+
+ ret = mc16_bm_fault(&f->bm->page[index], &unlocked);
+ KUNIT_ASSERT_EQ(test, (__force unsigned int)ret, (__force unsigned int)VM_FAULT_NOPAGE);
+ KUNIT_EXPECT_TRUE(test, unlocked);
+ mc16_bm_expect_bytes(test, f->bm, index, false);
+ if (__swap_count(f->swap) > 0)
+ mc16_bpp_expect_tail(test, f);
+}
+
+static void mc16_bpp_exit(struct kunit *test, struct mc16_bpp *f, unsigned int index)
+{
+ int ret;
+
+ ret = mc16_bm_pte(f->bm, f->bm->page[index].addr, false);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+}
+
+static void memcomp_test_budget_packed_writeback(struct kunit *test)
+{
+ struct mc16_bpp *f = mc16_bpp_begin(test, 2);
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_bpp_prepare(f), 0);
+ mc16_bpp_expect_idle(test);
+ for (i = 0; i < 2; i++) {
+ mc16_bpp_fault(test, f, i);
+ KUNIT_EXPECT_EQ(test, __swap_count(f->swap), (int)(1 - i));
+ }
+ mc16_bpp_expect_idle(test);
+ kunit_release_action(test, mc16_bpp_cleanup, f);
+}
+
+static void memcomp_test_budget_packed_refault1(struct kunit *test)
+{
+ struct mc16_bpp *f = mc16_bpp_begin(test, 2);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_bpp_prepare(f), 0);
+ mc16_bpp_fault(test, f, 0);
+ KUNIT_EXPECT_EQ(test, __swap_count(f->swap), 1);
+ mc16_bpp_expect_idle(test);
+ /* The last sibling exit exercises free_swap_and_cache on the retained page. */
+ mc16_bpp_exit(test, f, 1);
+ KUNIT_EXPECT_EQ(test, __swap_count(f->swap), 0);
+ mc16_bpp_expect_idle(test);
+ kunit_release_action(test, mc16_bpp_cleanup, f);
+}
+
+static void memcomp_test_budget_packed_exit(struct kunit *test)
+{
+ struct mc16_bpp *f = mc16_bpp_begin(test, 2);
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_bpp_prepare(f), 0);
+ for (i = 0; i < 2; i++) {
+ mc16_bpp_exit(test, f, i);
+ KUNIT_EXPECT_EQ(test, __swap_count(f->swap), (int)(1 - i));
+ }
+ mc16_bpp_expect_idle(test);
+ kunit_release_action(test, mc16_bpp_cleanup, f);
+}
+
+/* Parent and the unused +2 child PTE share the fixture's prevalidated PTL. */
+static int mc16_bpp_clone(struct mc16_bpp *f)
+{
+ struct mm_struct *mm = f->bm->seed->mm;
+ unsigned long address = f->bm->seed->addr;
+ pte_t *ptep, pte;
+ spinlock_t *ptl; /* Real parent ownership remains stable through publication. */
+ pmd_t *pmd;
+ int ret = -EINVAL;
+
+ mmap_read_lock(mm);
+ pmd = mm_find_pmd(mm, address);
+ if (!pmd)
+ goto out;
+ ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
+ if (!ptep)
+ goto out;
+ pte = ptep_get(ptep);
+ if ((address >> PMD_SHIFT) == ((address + 2 * PAGE_SIZE) >> PMD_SHIFT) &&
+ pte_none(ptep_get(ptep + 2)) && is_swap_pte(pte) &&
+ is_memcomp_entry(pte_to_swp_entry(pte)) &&
+ memcomp_entry_get(swp_offset(pte_to_swp_entry(pte)))) {
+ pte = pte_swp_clear_exclusive(pte);
+ set_pte_at(mm, address, ptep, pte);
+ set_pte_at(mm, address + 2 * PAGE_SIZE, ptep + 2, pte);
+ add_mm_counter(mm, MM_SWAPENTS, 1);
+ ret = 0;
+ }
+ pte_unmap_unlock(ptep, ptl);
+out:
+ mmap_read_unlock(mm);
+ return ret;
+}
+
+static void memcomp_test_budget_packed_fork_exit(struct kunit *test)
+{
+ struct mc16_bpp *f = mc16_bpp_begin(test, 2);
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_bpp_prepare(f), 0);
+ ret = mc16_bpp_clone(f);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ ret = mc16_bm_pte(f->bm, f->bm->seed->addr, false);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ KUNIT_EXPECT_EQ(test, __swap_count(f->swap), 2);
+ ret = mc16_bm_pte(f->bm, f->bm->seed->addr + 2 * PAGE_SIZE, false);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ KUNIT_EXPECT_EQ(test, __swap_count(f->swap), 1);
+ mc16_bpp_exit(test, f, 1);
+ KUNIT_EXPECT_EQ(test, __swap_count(f->swap), 0);
+ mc16_bpp_expect_idle(test);
+ kunit_release_action(test, mc16_bpp_cleanup, f);
+}
+
+static void memcomp_test_budget_packed_refault4(struct kunit *test)
+{
+ struct mc16_bpp *f = mc16_bpp_begin(test, 4);
+ unsigned int i;
+ bool done = true;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_bpp_prepare(f), 0);
+ for (i = 0; i < 4; i++) {
+ struct mc16_bm_worker *w = &f->bm->worker[i];
+
+ w->f = f->bm;
+ w->index = i;
+ init_completion(&w->ready);
+ init_completion(&w->done);
+ w->task = kthread_run(mc16_bm_worker, w, "mc16-packed-budget-%u", i);
+ if (IS_ERR(w->task)) {
+ w->task = NULL;
+ KUNIT_FAIL(test, "packed worker allocation failed");
+ return;
+ }
+ get_task_struct(w->task);
+ KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&w->ready, HZ), 0UL);
+ }
+ complete_all(&f->bm->go);
+ for (i = 0; i < 4; i++)
+ done &= !!wait_for_completion_timeout(&f->bm->worker[i].done, 10 * HZ);
+ if (!done) {
+ KUNIT_FAIL(test, "packed fault timed out");
+ return;
+ }
+ for (i = 0; i < 4; i++) {
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)f->bm->worker[i].ret,
+ (__force unsigned int)VM_FAULT_NOPAGE);
+ KUNIT_EXPECT_TRUE(test, f->bm->worker[i].unlocked);
+ mc16_bm_expect_bytes(test, f->bm, i, false);
+ }
+ KUNIT_EXPECT_EQ(test, __swap_count(f->swap), 0);
+ mc16_bpp_expect_idle(test);
+ kunit_release_action(test, mc16_bpp_cleanup, f);
+}
+
+/* A boot-time no-swap skip must not poison the later real-swap rerun. */
+static int mc16_bpp_suite_init(struct kunit_suite *suite)
+{
+ struct kunit_case *test_case;
+
+ for (test_case = suite->test_cases; test_case->run_case; test_case++)
+ test_case->status = KUNIT_SUCCESS;
+ return 0;
+}
+
+static struct kunit_case mc16_bpp_cases[] = {
+ KUNIT_CASE(memcomp_test_budget_packed_writeback),
+ KUNIT_CASE(memcomp_test_budget_packed_refault1),
+ KUNIT_CASE(memcomp_test_budget_packed_refault4),
+ KUNIT_CASE(memcomp_test_budget_packed_fork_exit),
+ KUNIT_CASE(memcomp_test_budget_packed_exit),
+ { }
+};
+
+static struct kunit_suite mc16_bpp_suite = {
+ .name = "swap-memcomp-packed-budget",
+ .suite_init = mc16_bpp_suite_init,
+ .test_cases = mc16_bpp_cases,
+};
+
+kunit_test_suite(mc16_bpp_suite);
+#endif /* MC16_FOLIO_TEST_ENABLED && CONFIG_SWAP */
+#endif /* _MM_MEMCOMP_TESTS_BUDGET_PACKED_TEST_H */
diff --git a/mm/memcomp/tests/budget_swapped_test.h b/mm/memcomp/tests/budget_swapped_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..1fb6bf8d47006a29dcf5c08ed19ed223ea441673
--- /dev/null
+++ b/mm/memcomp/tests/budget_swapped_test.h
@@ -0,0 +1,292 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_BUDGET_SWAPPED_TEST_H
+#define _MM_MEMCOMP_TESTS_BUDGET_SWAPPED_TEST_H
+
+/* Include after budget_test.h; only run serially in an isolated swap guest. */
+#if defined(MC16_FOLIO_TEST_ENABLED) && defined(CONFIG_SWAP)
+struct mc16_bw {
+ struct kunit *test;
+ struct mc16_bm *bm;
+ swp_entry_t swap[MC16_TEST_WORKERS];
+ struct swap_info_struct *device[MC16_TEST_WORKERS];
+ bool real;
+};
+
+static void mc16_bw_cleanup(void *data)
+{
+ struct mc16_bw *f = data;
+ unsigned int i;
+
+ /* Join workers, stop the budget, then remove ALL PTEs while puts are real. */
+ kunit_release_action(f->test, mc16_bm_cleanup, f->bm);
+ kunit_release_action(f->test, mc16_flt_cleanup, f->bm->seed);
+ /* All fixture PTEs are removed before real swap mode ends. */
+ mc16_test_reclaim_retained();
+ for (i = 0; i < MC16_TEST_WORKERS; i++) {
+ if (!f->device[i])
+ continue;
+ KUNIT_EXPECT_EQ(f->test, __swap_count(f->swap[i]), 0);
+ put_swap_device(f->device[i]);
+ f->device[i] = NULL;
+ }
+ if (f->real) {
+ KUNIT_EXPECT_TRUE(f->test, xa_empty(&mc16_clusters));
+ WRITE_ONCE(mc16_kunit_real_swap, false);
+ synchronize_rcu();
+ f->real = false;
+ }
+}
+
+static struct mc16_bw *mc16_bw_begin(struct kunit *test, unsigned int nr)
+{
+ struct mc16_bw *f;
+ bool idle;
+
+ if (get_nr_swap_pages() < nr) {
+ kunit_skip(test, "requires active real swap; rerun after guest swapon");
+ return NULL;
+ }
+ if (READ_ONCE(mc16_kunit_real_swap)) {
+ KUNIT_FAIL(test, "another real-swap fixture is active");
+ return NULL;
+ }
+ mutex_lock(&memcomp_writeback_lock);
+ idle = !mc16_writeback_batch_used && !mc16_writeback_members_used &&
+ !mc16_writeback_open_first && !mc16_writeback_open_bytes &&
+ !memcomp_writeback_slots_free && !memcomp_writeback_plugged;
+ mutex_unlock(&memcomp_writeback_lock);
+ if (!idle) {
+ KUNIT_FAIL(test, "writeback staging or reservation pool is not idle");
+ return NULL;
+ }
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->test = test;
+ f->bm = mc16_bm_begin(test, nr, false);
+ if (!f->bm)
+ return NULL;
+ if (kunit_add_action_or_reset(test, mc16_bw_cleanup, f))
+ return NULL;
+ f->real = true;
+ WRITE_ONCE(mc16_kunit_real_swap, true);
+ return f;
+}
+
+/* The real drain has one W scope and no empty final selector (limit is one). */
+static int mc16_bw_drain(struct mc16_flt *p)
+{
+ struct memcomp_lru *fifo = memcomp_lru_nid(folio_nid(p->folio));
+ unsigned long scanned, freed = 0;
+ unsigned int noreclaim;
+ bool attempted = false;
+ int ret = 0;
+
+ if (!fifo)
+ return -EINVAL;
+ noreclaim = memalloc_noreclaim_save();
+ mutex_lock(&memcomp_writeback_lock);
+ scanned = mc16_writeback_drain_queue(folio_nid(p->folio), fifo, 1,
+ &freed, &ret, &attempted);
+ mutex_unlock(&memcomp_writeback_lock);
+ memalloc_noreclaim_restore(noreclaim);
+ return ret ? ret : scanned == 1 && attempted && freed == 1 ? 0 : -EIO;
+}
+
+/* Capture effective fields under T, then retain only a real swap-device pin. */
+static int mc16_bw_capture(struct mc16_bw *f, unsigned int index)
+{
+ struct mc16_flt *p = &f->bm->page[index];
+ struct memcomp_ref ref = { };
+ struct mc16_swap_fields fields;
+ struct mc16_view view;
+ unsigned long token;
+ bool valid;
+
+ /* Successful source release clears folio->swap: read the actual token PTE. */
+ {
+ pte_t pte = mc16_flt_read(f->bm->seed, p->addr);
+
+ if (!is_swap_pte(pte) || !is_memcomp_entry(pte_to_swp_entry(pte)))
+ return -EINVAL;
+ token = swp_offset(pte_to_swp_entry(pte));
+ }
+ if (!mc16_get(token, &ref))
+ return -ENOENT;
+ mutex_lock(memcomp_lock_of(token));
+ valid = mc16_view_locked(&ref, &view) && mc16_flags_valid(*view.flags) &&
+ mc16_flags_state(*view.flags) == MC16_SWAPPED &&
+ !(*view.flags & (MC16_F_OWNER | MC16_F_RESIDENT)) && !view.member &&
+ mc16_swap_decode(*view.word, *view.length, &fields) &&
+ !fields.packed && !fields.byte_offset && fields.length;
+ mutex_unlock(memcomp_lock_of(token));
+ mc16_ref_put(&ref);
+ if (!valid)
+ return -EIO;
+ f->swap[index] = swp_entry(fields.type, fields.swap_offset);
+ f->device[index] = get_swap_device(f->swap[index]);
+ if (!f->device[index])
+ return -ENODEV;
+ return __swap_count(f->swap[index]) == 1 ? 0 : -EIO;
+}
+
+static void mc16_bw_expect_idle(struct kunit *test)
+{
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_slots_free, 0U);
+ KUNIT_EXPECT_EQ(test, mc16_writeback_batch_used, 0U);
+ KUNIT_EXPECT_EQ(test, mc16_writeback_members_used, 0U);
+}
+
+static int mc16_bw_prepare(struct mc16_bw *f, unsigned int index)
+{
+ int ret = mc16_bm_pageout(&f->bm->page[index]);
+
+ return ret ? ret : mc16_bw_drain(&f->bm->page[index]);
+}
+
+enum mc16_bw_op { MC16_BW_PAGEOUT, MC16_BW_REFAULT, MC16_BW_FORK, MC16_BW_EXIT };
+
+static void mc16_bw_one(struct kunit *test, enum mc16_bw_op op)
+{
+ struct mc16_bw *f = mc16_bw_begin(test, 1);
+ struct mc16_bm *bm;
+ bool unlocked = false;
+ vm_fault_t fault = 0;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ bm = f->bm;
+ if (op != MC16_BW_PAGEOUT) {
+ KUNIT_ASSERT_EQ(test, mc16_bw_prepare(f, 0), 0);
+ KUNIT_ASSERT_EQ(test, mc16_bw_capture(f, 0), 0);
+ }
+ switch (op) {
+ case MC16_BW_PAGEOUT:
+ ret = mc16_bw_prepare(f, 0);
+ break;
+ case MC16_BW_REFAULT:
+ fault = mc16_bm_fault(&bm->page[0], &unlocked);
+ ret = fault == VM_FAULT_NOPAGE ? 0 : -EIO;
+ break;
+ case MC16_BW_FORK:
+ ret = mc16_bm_pte(bm, bm->seed->addr + PAGE_SIZE, true);
+ break;
+ default:
+ ret = mc16_bm_pte(bm, bm->seed->addr, false);
+ break;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ switch (op) {
+ case MC16_BW_PAGEOUT:
+ KUNIT_ASSERT_EQ(test, mc16_bw_capture(f, 0), 0);
+ break;
+ case MC16_BW_REFAULT:
+ KUNIT_EXPECT_TRUE(test, unlocked);
+ mc16_bm_expect_bytes(test, bm, 0, false);
+ break;
+ case MC16_BW_FORK:
+ KUNIT_EXPECT_EQ(test, __swap_count(f->swap[0]), 1);
+ break;
+ default:
+ KUNIT_EXPECT_EQ(test, __swap_count(f->swap[0]), 0);
+ break;
+ }
+ mc16_bw_expect_idle(test);
+ kunit_release_action(test, mc16_bw_cleanup, f);
+}
+
+static void memcomp_test_budget_swapped_pageout(struct kunit *test)
+{
+ mc16_bw_one(test, MC16_BW_PAGEOUT);
+}
+
+static void memcomp_test_budget_swapped_refault1(struct kunit *test)
+{
+ mc16_bw_one(test, MC16_BW_REFAULT);
+}
+
+static void memcomp_test_budget_swapped_fork(struct kunit *test)
+{
+ mc16_bw_one(test, MC16_BW_FORK);
+}
+
+static void memcomp_test_budget_swapped_exit(struct kunit *test)
+{
+ mc16_bw_one(test, MC16_BW_EXIT);
+}
+
+static void memcomp_test_budget_swapped_refault4(struct kunit *test)
+{
+ struct mc16_bw *f = mc16_bw_begin(test, MC16_TEST_WORKERS);
+ struct mc16_bm *bm;
+ unsigned int i;
+ bool done = true;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ bm = f->bm;
+ for (i = 0; i < MC16_TEST_WORKERS; i++) {
+ struct mc16_bm_worker *w = &bm->worker[i];
+
+ /* Separate raw slots; setup is outside the refault measurement. */
+ KUNIT_ASSERT_EQ(test, mc16_bw_prepare(f, i), 0);
+ KUNIT_ASSERT_EQ(test, mc16_bw_capture(f, i), 0);
+ w->f = bm;
+ w->index = i;
+ init_completion(&w->ready);
+ init_completion(&w->done);
+ w->task = kthread_run(mc16_bm_worker, w, "mc16-swap-budget-%u", i);
+ if (IS_ERR(w->task)) {
+ w->task = NULL;
+ KUNIT_FAIL(test, "refault worker allocation failed");
+ return;
+ }
+ get_task_struct(w->task);
+ KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&w->ready, HZ), 0UL);
+ }
+ complete_all(&bm->go);
+ for (i = 0; i < MC16_TEST_WORKERS; i++)
+ done &= !!wait_for_completion_timeout(&bm->worker[i].done, 10 * HZ);
+ if (!done) {
+ KUNIT_FAIL(test, "real swap fault timed out");
+ return;
+ }
+ for (i = 0; i < MC16_TEST_WORKERS; i++) {
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)bm->worker[i].ret,
+ (__force unsigned int)VM_FAULT_NOPAGE);
+ KUNIT_EXPECT_TRUE(test, bm->worker[i].unlocked);
+ mc16_bm_expect_bytes(test, bm, i, false);
+ KUNIT_EXPECT_EQ(test, __swap_count(f->swap[i]), 0);
+ }
+ mc16_bw_expect_idle(test);
+ kunit_release_action(test, mc16_bw_cleanup, f);
+}
+
+/* A boot-time no-swap skip must not poison the later real-swap rerun. */
+static int mc16_bw_suite_init(struct kunit_suite *suite)
+{
+ struct kunit_case *test_case;
+
+ for (test_case = suite->test_cases; test_case->run_case; test_case++)
+ test_case->status = KUNIT_SUCCESS;
+ return 0;
+}
+
+/* This suite is deliberately excluded by the ordinary memcomp* filter. */
+static struct kunit_case mc16_bw_cases[] = {
+ KUNIT_CASE(memcomp_test_budget_swapped_pageout),
+ KUNIT_CASE(memcomp_test_budget_swapped_refault1),
+ KUNIT_CASE(memcomp_test_budget_swapped_refault4),
+ KUNIT_CASE(memcomp_test_budget_swapped_fork),
+ KUNIT_CASE(memcomp_test_budget_swapped_exit),
+ { }
+};
+
+static struct kunit_suite mc16_bw_suite = {
+ .name = "swap-memcomp-budget",
+ .suite_init = mc16_bw_suite_init,
+ .test_cases = mc16_bw_cases,
+};
+
+kunit_test_suite(mc16_bw_suite);
+#endif /* MC16_FOLIO_TEST_ENABLED && CONFIG_SWAP */
+#endif /* _MM_MEMCOMP_TESTS_BUDGET_SWAPPED_TEST_H */
diff --git a/mm/memcomp/tests/budget_test.h b/mm/memcomp/tests/budget_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..02c77091520be197578bea05b08abe7a0f88e68c
--- /dev/null
+++ b/mm/memcomp/tests/budget_test.h
@@ -0,0 +1,542 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_BUDGET_TEST_H
+#define _MM_MEMCOMP_TESTS_BUDGET_TEST_H
+
+/* Include after folio_test.h. Public positive-token dispatch uses clustered slots. */
+#ifdef MC16_FOLIO_TEST_ENABLED
+#define MC16_TEST_WORKERS 4
+struct mc16_bm;
+struct mc16_bm_worker {
+ struct mc16_bm *f;
+ unsigned int index;
+ struct task_struct *task;
+ struct completion ready;
+ struct completion done;
+ vm_fault_t ret;
+ bool unlocked;
+};
+
+struct mc16_bm {
+ struct mc16_flt *seed;
+ struct mc16_flt page[4];
+ unsigned int nr;
+ struct completion go;
+ struct mc16_bm_worker worker[4];
+};
+
+/* Actual present/token PTE transaction; no lookup retained for assertions. */
+static int mc16_bm_pte(struct mc16_bm *f, unsigned long address, bool clone)
+{
+ struct mm_struct *mm = f->seed->mm;
+ unsigned long parent = f->seed->addr;
+ pte_t orig, *ptep, *child;
+ spinlock_t *ptl; /* Serializes the real PTE ownership transaction. */
+ pmd_t *pmd;
+ int ret = -ENOENT;
+
+ mmap_read_lock(mm);
+ pmd = mm_find_pmd(mm, clone ? parent : address);
+ if (!pmd)
+ goto out;
+ ptep = pte_offset_map_lock(mm, pmd, clone ? parent : address, &ptl);
+ if (!ptep)
+ goto out;
+ orig = ptep_get(ptep);
+ if (clone) {
+ /* Fixture child is adjacent, in this same prevalidated PTE table. */
+ child = ptep + 1;
+ if (address != parent + PAGE_SIZE ||
+ (parent >> PMD_SHIFT) != (address >> PMD_SHIFT) ||
+ !pte_none(ptep_get(child)) || !is_swap_pte(orig) ||
+ !is_memcomp_entry(pte_to_swp_entry(orig))) {
+ ret = -EINVAL;
+ } else if (!memcomp_entry_get(swp_offset(pte_to_swp_entry(orig)))) {
+ ret = -ENOENT;
+ } else {
+ /* The parent PTL remains held across get and child publication. */
+ orig = pte_swp_clear_exclusive(orig);
+ set_pte_at(mm, parent, ptep, orig);
+ set_pte_at(mm, address, child, orig);
+ add_mm_counter(mm, MM_SWAPENTS, 1);
+ ret = 0;
+ }
+ } else {
+ ret = 0;
+ if (is_swap_pte(orig) && is_memcomp_entry(pte_to_swp_entry(orig))) {
+ ptep_get_and_clear(mm, address, ptep);
+ add_mm_counter(mm, MM_SWAPENTS, -1);
+ memcomp_invalidate(swp_offset(pte_to_swp_entry(orig)));
+ }
+ }
+ pte_unmap_unlock(ptep, ptl);
+out:
+ mmap_read_unlock(mm);
+ return ret;
+}
+
+static void mc16_bm_cleanup(void *data)
+{
+ struct mc16_bm *f = data;
+ unsigned int i;
+
+ complete_all(&f->go);
+ for (i = 0; i < ARRAY_SIZE(f->worker); i++) {
+ if (f->worker[i].task) {
+ kthread_stop(f->worker[i].task);
+ put_task_struct(f->worker[i].task);
+ }
+ }
+ /* The extra child branch lives inside the seed's existing mapping. */
+ mc16_bm_pte(f, f->seed->addr + PAGE_SIZE, false);
+ for (i = 1; i < f->nr; i++) {
+ struct mc16_flt *p = &f->page[i];
+
+ mc16_flt_zap(p, p->addr);
+ if (p->folio) {
+ folio_lock(p->folio);
+ memcomp_invalidate_folio(p->folio);
+ folio_unlock(p->folio);
+ mc16_test_dontneed(p->mm, p->addr, PAGE_SIZE);
+ lru_add_drain_all();
+ folio_put(p->folio);
+ }
+ if (p->anon_vma)
+ put_anon_vma(p->anon_vma);
+ }
+ f->seed->base->model.folio = f->seed->folio;
+}
+
+static struct mc16_bm *mc16_bm_begin(struct kunit *test, unsigned int nr,
+ bool samefill)
+{
+ struct mc16_bm *f;
+ unsigned int i;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->seed = mc16_flt_begin(test, 0);
+ if (!f->seed)
+ return NULL;
+ init_completion(&f->go);
+ if (kunit_add_action_or_reset(test, mc16_bm_cleanup, f))
+ return NULL;
+ f->nr = 1;
+ f->page[0] = *f->seed;
+ for (i = 1; i < nr; i++) {
+ struct mc16_flt *p = &f->page[i];
+
+ p->test = test;
+ p->base = f->seed->base;
+ p->mm = f->seed->mm;
+ p->addr = f->seed->addr + i * PAGE_SIZE;
+ f->nr++;
+ if (mc16_flt_map(p, 0))
+ return NULL;
+ }
+ for (i = 0; i < nr; i++) {
+ u8 *data = kmap_local_folio(f->page[i].folio, 0);
+
+ memset(data, 0x51 + i, PAGE_SIZE);
+ if (!samefill)
+ data[17] ^= 0xff;
+ kunmap_local(data);
+ }
+ return f;
+}
+
+/* Reserve -> actual rmap/PTE replacement -> public store -> source OWNER drop. */
+static int mc16_bm_pageout(struct mc16_flt *p)
+{
+ struct memcomp_reclaim_ctx *ctx;
+ unsigned int flags;
+ bool reserved, stored, released = false;
+
+ p->base->model.folio = p->folio;
+ folio_lock(p->folio);
+ reserved = memcomp_reserve(p->folio);
+ folio_unlock(p->folio);
+ if (!reserved)
+ return -ENOMEM;
+ if (mc16_flt_escape(p, 0))
+ return -EINVAL;
+ lru_add_drain_all();
+ if (folio_test_lru(p->folio)) {
+ if (!folio_isolate_lru(p->folio))
+ return -EBUSY;
+ /* Model reclaim isolation, not migration: the source is inactive. */
+ folio_clear_active(p->folio);
+ folio_put(p->folio); /* Keep only the original fixture hold. */
+ }
+ flags = memcomp_test_reclaimer_enter();
+ folio_lock(p->folio);
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_KSWAPD);
+ if (!ctx) {
+ folio_unlock(p->folio);
+ memcomp_test_reclaimer_exit(flags);
+ return -EBUSY;
+ }
+ stored = memcomp_store_after_unmap(p->folio, &ctx);
+ if (ctx)
+ memcomp_reclaim_ctx_put(ctx);
+ if (stored) {
+ released = memcomp_release_folio(p->folio);
+ /* Only keep the fixture's original physical folio alive. */
+ if (released)
+ folio_ref_unfreeze(p->folio, 1);
+ }
+ folio_unlock(p->folio);
+ memcomp_test_reclaimer_exit(flags);
+ return stored && released ? 0 : -EIO;
+}
+
+static vm_fault_t mc16_bm_fault(struct mc16_flt *p, bool *unlocked)
+{
+ struct vm_area_struct *vma;
+ vm_fault_t ret = VM_FAULT_SIGBUS;
+ bool locked = true;
+ pte_t orig;
+ pte_t *ptep;
+ spinlock_t *ptl; /* Serializes the real PTE ownership transaction. */
+ pmd_t *pmd;
+
+ *unlocked = false;
+ mmap_read_lock(p->mm);
+ vma = vma_lookup(p->mm, p->addr);
+ pmd = mm_find_pmd(p->mm, p->addr);
+ if (!vma || !pmd)
+ goto out;
+ ptep = pte_offset_map_lock(p->mm, pmd, p->addr, &ptl);
+ if (!ptep)
+ goto out;
+ orig = ptep_get(ptep);
+ pte_unmap_unlock(ptep, ptl);
+ {
+ struct vm_fault vmf = {
+ .vma = vma, .address = p->addr, .real_address = p->addr,
+ .flags = FAULT_FLAG_ORIG_PTE_VALID,
+ .pmd = pmd, .orig_pte = orig,
+ .gfp_mask = GFP_HIGHUSER_MOVABLE,
+ };
+
+ ret = memcomp_load(&vmf, &locked);
+ *unlocked = !locked && !vmf.pte;
+ }
+out:
+ mmap_read_unlock(p->mm);
+ return ret;
+}
+
+/* Pin the installed page under PTL; all oracle work follows measurement. */
+static void mc16_bm_expect_bytes(struct kunit *test, struct mc16_bm *f,
+ unsigned int index, bool samefill)
+{
+ struct mm_struct *mm = f->seed->mm;
+ unsigned long address = f->page[index].addr;
+ struct page *page = NULL;
+ spinlock_t *ptl; /* Protects the PTE until the page pin is acquired. */
+ u8 *expected, *actual;
+ pte_t *ptep, pte;
+ pmd_t *pmd;
+ int different;
+
+ expected = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, expected);
+ memset(expected, 0x51 + index, PAGE_SIZE);
+ if (!samefill)
+ expected[17] ^= 0xff;
+ mmap_read_lock(mm);
+ pmd = mm_find_pmd(mm, address);
+ if (!pmd)
+ goto out_unlock;
+ ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
+ if (!ptep)
+ goto out_unlock;
+ pte = ptep_get(ptep);
+ if (pte_present(pte) && !pte_special(pte)) {
+ page = pte_page(pte);
+ get_page(page);
+ }
+ pte_unmap_unlock(ptep, ptl);
+out_unlock:
+ mmap_read_unlock(mm);
+ KUNIT_ASSERT_NOT_NULL(test, page);
+ actual = kmap_local_page(page);
+ different = memcmp(actual, expected, PAGE_SIZE);
+ kunmap_local(actual);
+ put_page(page);
+ KUNIT_EXPECT_EQ(test, different, 0);
+}
+
+/* Real PTE operations remain covered without operation-budget probes. */
+enum mc16_bm_op { MC16_BM_PAGEOUT, MC16_BM_REFAULT, MC16_BM_FORK, MC16_BM_EXIT };
+
+static void mc16_bm_one(struct kunit *test, enum mc16_bm_op op, bool samefill)
+{
+ struct mc16_bm *f = mc16_bm_begin(test, 1, samefill);
+ bool unlocked = false;
+ vm_fault_t fault = 0;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ if (op != MC16_BM_PAGEOUT)
+ KUNIT_ASSERT_EQ(test, mc16_bm_pageout(&f->page[0]), 0);
+ switch (op) {
+ case MC16_BM_PAGEOUT:
+ ret = mc16_bm_pageout(&f->page[0]);
+ break;
+ case MC16_BM_REFAULT:
+ fault = mc16_bm_fault(&f->page[0], &unlocked);
+ ret = (fault & VM_FAULT_NOPAGE) && (fault & VM_FAULT_MAJOR) ?
+ 0 : -EIO;
+ break;
+ case MC16_BM_FORK:
+ ret = mc16_bm_pte(f, f->seed->addr + PAGE_SIZE, true);
+ break;
+ default:
+ ret = mc16_bm_pte(f, f->seed->addr, false);
+ break;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ switch (op) {
+ case MC16_BM_PAGEOUT:
+ break;
+ case MC16_BM_REFAULT:
+ KUNIT_EXPECT_TRUE(test, unlocked);
+ KUNIT_EXPECT_TRUE(test, pte_present(mc16_flt_read(f->seed, f->seed->addr)));
+ mc16_bm_expect_bytes(test, f, 0, samefill);
+ /* Stable non-PENDING tags skip the resident actor T. */
+ break;
+ case MC16_BM_FORK:
+ break;
+ default:
+ break;
+ }
+}
+
+static int mc16_bm_worker(void *data)
+{
+ struct mc16_bm_worker *w = data;
+ struct mc16_flt *p = &w->f->page[w->index];
+
+ /* Keep deterministic work scheduling attached to the live parent test. */
+ current->kunit_test = p->test;
+ complete(&w->ready);
+ wait_for_completion(&w->f->go);
+ kthread_use_mm(p->mm);
+ w->ret = mc16_bm_fault(p, &w->unlocked);
+ kthread_unuse_mm(p->mm);
+ KUNIT_EXPECT_FALSE(p->test, delayed_work_pending(&mc16_retained_aging));
+ current->kunit_test = NULL;
+ complete(&w->done);
+ return 0;
+}
+
+static void mc16_bm_four(struct kunit *test, bool samefill)
+{
+ struct mc16_bm *f = mc16_bm_begin(test, 4, samefill);
+ unsigned int i;
+ bool ready = true;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (i = 0; i < 4; i++) {
+ struct mc16_bm_worker *w = &f->worker[i];
+
+ KUNIT_ASSERT_EQ(test, mc16_bm_pageout(&f->page[i]), 0);
+ w->f = f;
+ w->index = i;
+ init_completion(&w->ready);
+ init_completion(&w->done);
+ w->task = kthread_run(mc16_bm_worker, w, "mc16-budget-%u", i);
+ if (IS_ERR(w->task)) {
+ w->task = NULL;
+ KUNIT_FAIL(test, "refault worker allocation failed");
+ return;
+ }
+ get_task_struct(w->task);
+ ready &= !!wait_for_completion_timeout(&w->ready, HZ);
+ }
+ KUNIT_ASSERT_TRUE(test, ready);
+ complete_all(&f->go);
+ for (i = 0; i < 4; i++)
+ ready &= !!wait_for_completion_timeout(&f->worker[i].done, 10 * HZ);
+ if (!ready) {
+ KUNIT_FAIL(test, "refault worker timed out");
+ return;
+ }
+ for (i = 0; i < 4; i++) {
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)f->worker[i].ret,
+ (__force unsigned int)(VM_FAULT_NOPAGE | VM_FAULT_MAJOR));
+ KUNIT_EXPECT_TRUE(test, f->worker[i].unlocked);
+ KUNIT_EXPECT_TRUE(test, pte_present(mc16_flt_read(f->seed,
+ f->page[i].addr)));
+ mc16_bm_expect_bytes(test, f, i, samefill);
+ }
+}
+
+static void memcomp_test_budget_pageout_stored(struct kunit *test)
+{
+ mc16_bm_one(test, MC16_BM_PAGEOUT, false);
+}
+
+static void memcomp_test_budget_pageout_samefill(struct kunit *test)
+{
+ mc16_bm_one(test, MC16_BM_PAGEOUT, true);
+}
+
+static void memcomp_test_budget_refault1_stored(struct kunit *test)
+{
+ mc16_bm_one(test, MC16_BM_REFAULT, false);
+}
+
+static void memcomp_test_budget_refault1_samefill(struct kunit *test)
+{
+ mc16_bm_one(test, MC16_BM_REFAULT, true);
+}
+
+static void memcomp_test_budget_fork_stored(struct kunit *test)
+{
+ mc16_bm_one(test, MC16_BM_FORK, false);
+}
+
+static void memcomp_test_budget_fork_samefill(struct kunit *test)
+{
+ mc16_bm_one(test, MC16_BM_FORK, true);
+}
+
+static void memcomp_test_budget_exit_stored(struct kunit *test)
+{
+ mc16_bm_one(test, MC16_BM_EXIT, false);
+}
+
+static void memcomp_test_budget_exit_samefill(struct kunit *test)
+{
+ mc16_bm_one(test, MC16_BM_EXIT, true);
+}
+
+static void memcomp_test_budget_refault4_stored(struct kunit *test)
+{
+ mc16_bm_four(test, false);
+}
+
+static void memcomp_test_budget_refault4_samefill(struct kunit *test)
+{
+ mc16_bm_four(test, true);
+}
+
+#ifdef CONFIG_MEMCG
+struct mc16_bs {
+ struct mc16_consumer_test *base;
+ unsigned long tokens[MC16_CLUSTER_SLOTS];
+ bool held[MC16_CLUSTER_SLOTS];
+};
+
+static void mc16_bs_cleanup(void *data)
+{
+ struct mc16_bs *f = data;
+ unsigned int i;
+
+ folio_lock(f->base->account);
+ memcomp_rollback_folio(f->base->account);
+ folio_unlock(f->base->account);
+ for (i = 0; i < MC16_CLUSTER_SLOTS; i++) {
+ if (f->held[i])
+ memcomp_invalidate(f->tokens[i]);
+ }
+ rcu_barrier();
+}
+
+/* A real reference holder, explicitly not a real PTE or the normal MM scene. */
+static int mc16_bs_publish(struct mc16_bs *f, unsigned int index)
+{
+ struct folio *folio = f->base->account;
+ struct memcomp_reclaim_ctx *ctx = NULL;
+ bool success = false;
+ int ret = -ENOMEM;
+
+ folio_lock(folio);
+ if (!memcomp_reserve(folio))
+ goto out;
+ f->tokens[index] = swp_offset(folio->swap);
+ if (!memcomp_entry_get(f->tokens[index]))
+ goto out;
+ f->held[index] = true;
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_DIRECT);
+ if (!ctx)
+ goto out;
+ if (!memcomp_store_after_unmap(folio, &ctx)) {
+ ret = -EIO;
+ goto out;
+ }
+ success = memcomp_release_folio(folio);
+ if (success)
+ folio_ref_unfreeze(folio, 1);
+ ret = success ? 0 : -EIO;
+out:
+ if (ctx)
+ memcomp_reclaim_ctx_put(ctx);
+ folio_unlock(folio);
+ return ret;
+}
+
+static void memcomp_test_budget_fixed512(struct kunit *test)
+{
+ struct mc16_bs *f;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ struct mc16_storage_debug before, after;
+ static const u64 expected[MC16_META_CLASSES] = {
+ [MC16_META_SEGMENT] = 8,
+ [MC16_META_CLUSTER] = 1,
+ [MC16_META_KEY] = 1,
+ [MC16_META_DESCRIPTOR] = 8,
+ [MC16_META_SHADOW] = 8,
+ };
+#endif
+ unsigned int i;
+ u8 *src;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->base = mc16_consumer_test_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f->base);
+ KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, mc16_bs_cleanup, f), 0);
+ f->base->account->mapping = (struct address_space *)PAGE_MAPPING_ANON;
+ __folio_set_swapbacked(f->base->account);
+ src = kmap_local_folio(f->base->account, 0);
+ memset(src, 0x67, PAGE_SIZE);
+ kunmap_local(src);
+ /* Start from an empty publication registry. */
+ KUNIT_ASSERT_TRUE(test, xa_empty(&mc16_clusters));
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ mc16_test_metadata_snapshot(&before);
+#endif
+ for (i = 0; i < MC16_CLUSTER_SLOTS; i++) {
+ int ret = mc16_bs_publish(f, i);
+
+ if (ret) {
+ KUNIT_FAIL(test, "structural page %u failed: %d", i, ret);
+ return;
+ }
+ }
+ for (i = 0; i < MC16_CLUSTER_SLOTS; i++) {
+ memcomp_invalidate(f->tokens[i]);
+ f->held[i] = false;
+ }
+ /* Wait for physical metadata frees before checking DEBUG_STATS. */
+ rcu_barrier();
+ /* Only memcomp-owned objects are counted; XA is not observed. */
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ mc16_test_metadata_snapshot(&after);
+ for (i = 0; i < MC16_META_CLASSES; i++) {
+ KUNIT_EXPECT_EQ(test, after.allocated[i] - before.allocated[i],
+ expected[i]);
+ KUNIT_EXPECT_EQ(test, after.freed[i] - before.freed[i],
+ expected[i]);
+ }
+#endif
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ kunit_release_action(test, mc16_bs_cleanup, f);
+}
+#endif /* CONFIG_MEMCG */
+#endif /* MC16_FOLIO_TEST_ENABLED */
+#endif /* _MM_MEMCOMP_TESTS_BUDGET_TEST_H */
diff --git a/mm/memcomp/tests/cache_folio_test.h b/mm/memcomp/tests/cache_folio_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..ab22ce6e5b220a5cae82ef81bde85bc212708d4e
--- /dev/null
+++ b/mm/memcomp/tests/cache_folio_test.h
@@ -0,0 +1,627 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_CACHE_FOLIO_TEST_H
+#define _MM_MEMCOMP_TESTS_CACHE_FOLIO_TEST_H
+
+#include <kunit/test.h>
+#include <kunit/static_stub.h>
+#if defined(CONFIG_ARM64) && defined(CONFIG_ARM64_MTE)
+#include <linux/cpufeature.h>
+#include <asm/mte.h>
+#endif
+#include "../cache_folio.h"
+
+/*
+ * Include in suite.h after storage_probe_impl and consumers.h/load.h
+ * are visible. CONFIG_MEMCOMP_KUNIT_TEST encloses this header; cases below run
+ * with MEMCG=n/y, except the two charge-model cases guarded by CONFIG_MEMCG.
+ * Real folios, codec, zs payloads, FIFO, refs and RCU retirement are exercised.
+ * Arch success is a synchronous model observing the temporary swap entry;
+ * failure can pass through the existing arch fail seam. MEMCG metadata and
+ * byte charges use the existing synthetic owner, not real memcg reclaim.
+ * All actors release folio locks and local mappings before any assertion.
+ */
+struct mc16_cf_test {
+ struct kunit *test;
+ struct mc16_storage_fixture probe;
+ struct memcomp_kunit_async_probe payload;
+#ifdef CONFIG_MEMCG
+ struct memcomp_kunit_objcg model;
+#endif
+ struct folio *src;
+ struct folio *large;
+ struct memcomp_ref guard;
+ unsigned long token;
+ bool cache_owner;
+ unsigned int zero_cache;
+ unsigned int zero_pte;
+ unsigned int arch_calls;
+ unsigned int initial_refs;
+ unsigned int state;
+ unsigned int eligible;
+ unsigned int length;
+ bool arch_contract;
+ bool arch_passthrough;
+ int arch_error;
+ s64 stored;
+ s64 compressed;
+ long fifo_entries;
+ unsigned long zs_pages;
+ struct mc16_zero_owners zero_before;
+};
+
+/* Only the test's synchronous static-stub invocation uses this pointer. */
+static struct mc16_cf_test *mc16_cf_actor;
+
+static int mc16_cf_arch(struct folio *folio)
+{
+ struct mc16_cf_test *f = mc16_cf_actor;
+ struct mc16_cluster *cluster;
+ struct mc16_slot *segment, *slot;
+ unsigned long token = swp_offset(folio->swap);
+ unsigned int index = token & (MC16_CLUSTER_SLOTS - 1);
+ u16 flags;
+
+ if (!f)
+ return -EINVAL;
+ f->arch_calls++;
+ f->arch_contract = folio == f->src && folio_test_locked(folio) &&
+ is_memcomp_entry(folio->swap) && token && token <= U32_MAX;
+ /* Observe under the continuous initial ref; do not get a second ref. */
+ rcu_read_lock();
+ cluster = xa_load(&mc16_clusters, token >> MC16_CLUSTER_SHIFT);
+ if (!cluster)
+ goto invalid;
+ /* Pair with reserve's segment installation while the initial ref lives. */
+ segment = smp_load_acquire(&cluster->segments[index >> MC16_SEGMENT_SHIFT]);
+ if (!segment)
+ goto invalid;
+ slot = &segment[index & (MC16_SEGMENT_SLOTS - 1)];
+ flags = READ_ONCE(slot->flags);
+ f->state = mc16_flags_state(flags);
+ f->initial_refs = refcount_read(&slot->refs);
+ f->eligible = READ_ONCE(cluster->eligible_slots);
+ f->length = READ_ONCE(slot->length);
+ f->arch_contract &= mc16_flags_valid(flags) &&
+ !(flags & (MC16_F_OWNER | MC16_F_RESIDENT)) &&
+ cluster->key->kind == MC16_KEY_SHMEM;
+ if (f->state == MC16_STORED) {
+ f->payload.handle = READ_ONCE(slot->word);
+ f->payload.pool = memcomp_pools[0];
+ f->payload.live = true;
+ }
+ rcu_read_unlock();
+ return f->arch_passthrough ? memcomp_arch_prepare_to_swap(folio) :
+ f->arch_error;
+invalid:
+ f->arch_contract = false;
+ rcu_read_unlock();
+ return -EINVAL;
+}
+
+static void mc16_cf_cleanup(void *data)
+{
+ struct mc16_cf_test *f = data;
+ struct mc16_zero_owners zero;
+ s64 delta;
+
+ /* Cleanup may run in a different KUnit task; it inherits no held lock. */
+ f->probe.task = current;
+ f->payload.task = current;
+ f->probe.fail_at = 0;
+#ifdef CONFIG_MEMCG
+ f->model.fail_at = 0;
+#endif
+ if (f->guard.slot)
+ mc16_ref_put(&f->guard);
+ if (f->cache_owner)
+ mc16_cache_drop(f->token);
+ mc16_zero_cache_put(f->zero_cache);
+ mc16_zero_pte_put(f->zero_pte);
+ mc16_test_drain_segments(&f->probe);
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_TRUE(f->test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(f->test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_EQ(f->test, atomic64_read(&memcomp_stored_bytes), f->stored);
+ KUNIT_EXPECT_EQ(f->test, atomic64_read(&memcomp_compressed_bytes),
+ f->compressed);
+ if (f->src)
+ KUNIT_EXPECT_EQ(f->test,
+ atomic_long_read(&memcomp_lrus[folio_nid(f->src)].nr_entries),
+ f->fifo_entries);
+ mc16_zero_owners_read(&zero);
+ KUNIT_EXPECT_EQ(f->test, zero.pte_pages, f->zero_before.pte_pages);
+ KUNIT_EXPECT_EQ(f->test, zero.cache_pages, f->zero_before.cache_pages);
+ /* Repair a failed ledger assertion too, so later cases stay isolated. */
+ delta = zero.pte_pages - f->zero_before.pte_pages;
+ if (delta > 0)
+ mc16_zero_pte_put(delta);
+ else if (delta < 0)
+ mc16_zero_pte_init(-delta);
+ delta = zero.cache_pages - f->zero_before.cache_pages;
+ if (delta > 0)
+ mc16_zero_cache_put(delta);
+ else if (delta < 0)
+ mc16_zero_cache_get(-delta);
+ /* Isolated pool: no mapped object or concurrent zs migration is retained. */
+ KUNIT_EXPECT_EQ(f->test, zs_get_total_pages(memcomp_pools[0]), f->zs_pages);
+ KUNIT_EXPECT_FALSE(f->test, f->payload.bad_lifetime);
+ KUNIT_EXPECT_FALSE(f->test, f->payload.live);
+#ifdef CONFIG_MEMCG
+ KUNIT_EXPECT_EQ(f->test, atomic_long_read(&f->model.charged), 0L);
+ KUNIT_EXPECT_EQ(f->test, atomic_read(&f->model.get_calls),
+ atomic_read(&f->model.put_calls));
+ if (rcu_access_pointer(memcomp_kunit_objcg) == &f->model)
+ rcu_assign_pointer(memcomp_kunit_objcg, NULL);
+#endif
+ if (rcu_access_pointer(mc16_storage_fixture) == &f->probe)
+ rcu_assign_pointer(mc16_storage_fixture, NULL);
+ if (rcu_access_pointer(memcomp_kunit_async_probe) == &f->payload)
+ rcu_assign_pointer(memcomp_kunit_async_probe, NULL);
+ synchronize_rcu();
+ if (f->large)
+ folio_put(f->large);
+ if (f->src) {
+ KUNIT_EXPECT_FALSE(f->test, folio_test_locked(f->src));
+ KUNIT_EXPECT_EQ(f->test, folio_ref_count(f->src), 1);
+ folio_put(f->src);
+ }
+}
+
+static struct mc16_cf_test *mc16_cf_begin(struct kunit *test)
+{
+ struct mc16_cf_test *f;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return NULL;
+ }
+ rcu_barrier();
+ if (rcu_access_pointer(mc16_storage_fixture) ||
+ rcu_access_pointer(memcomp_kunit_async_probe) ||
+ READ_ONCE(memcomp_kunit_fail.task) || !xa_empty(&mc16_clusters) ||
+ !hash_empty(mc16_keys)) {
+ KUNIT_FAIL(test, "cache folio fixture requires idle registry/probes");
+ return NULL;
+ }
+#ifdef CONFIG_MEMCG
+ if (rcu_access_pointer(memcomp_kunit_objcg)) {
+ KUNIT_FAIL(test, "cache folio fixture requires idle objcg model");
+ return NULL;
+ }
+#endif
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f) {
+ KUNIT_FAIL(test, "cache folio fixture allocation failed");
+ return NULL;
+ }
+ f->test = test;
+ f->stored = atomic64_read(&memcomp_stored_bytes);
+ f->compressed = atomic64_read(&memcomp_compressed_bytes);
+ f->zs_pages = zs_get_total_pages(memcomp_pools[0]);
+ mc16_zero_owners_read(&f->zero_before);
+ f->probe.task = current;
+ f->payload.task = current;
+ if (kunit_add_action_or_reset(test, mc16_cf_cleanup, f))
+ return NULL;
+ f->src = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0);
+ if (!f->src) {
+ KUNIT_FAIL(test, "source folio allocation failed");
+ return NULL;
+ }
+ f->fifo_entries = atomic_long_read(&memcomp_lrus[folio_nid(f->src)].nr_entries);
+ /* The source also needs real memcg identity for native eviction. */
+ if (mem_cgroup_charge(f->src, NULL, GFP_KERNEL)) {
+ KUNIT_FAIL(test, "cache source memcg charge failed");
+ return NULL;
+ }
+#ifdef CONFIG_MEMCG
+ f->model.task = current;
+ f->model.folio = f->src;
+ f->model.error = -ENOMEM;
+ rcu_assign_pointer(memcomp_kunit_objcg, &f->model);
+#endif
+ rcu_assign_pointer(mc16_storage_fixture, &f->probe);
+ rcu_assign_pointer(memcomp_kunit_async_probe, &f->payload);
+ kunit_activate_static_stub(test, mc16_cache_folio_arch_prepare, mc16_cf_arch);
+ return f;
+}
+
+enum mc16_cf_pattern {
+ MC16_CF_ZERO,
+ MC16_CF_BYTES,
+ MC16_CF_FILL,
+ MC16_CF_LAST_BYTE,
+};
+
+static void mc16_cf_fill(struct mc16_cf_test *f, enum mc16_cf_pattern pattern)
+{
+ u8 *src = kmap_local_folio(f->src, 0);
+ unsigned int i;
+
+ memset(src, 0, PAGE_SIZE);
+ if (pattern == MC16_CF_BYTES)
+ for (i = 0; i < PAGE_SIZE; i++)
+ src[i] = i % 53;
+ else if (pattern == MC16_CF_FILL)
+ memset(src, 0x5a, PAGE_SIZE);
+ else if (pattern == MC16_CF_LAST_BYTE)
+ src[PAGE_SIZE - 1] = 1;
+ kunmap_local(src);
+}
+
+static bool mc16_cf_store(struct mc16_cf_test *f)
+{
+ bool ok;
+
+ mc16_cf_actor = f;
+ folio_lock(f->src);
+ ok = mc16_store_cache_folio(f->src, &f->token);
+ folio_unlock(f->src);
+ mc16_cf_actor = NULL;
+ if (ok && f->token)
+ f->cache_owner = true;
+ return ok;
+}
+
+/* T, source mapping and destination allocation never escape this actor. */
+static int mc16_cf_roundtrip(struct mc16_cf_test *f)
+{
+ void *dst, *src;
+ int ret;
+
+ dst = kmalloc(PAGE_SIZE, GFP_KERNEL);
+ if (!dst)
+ return -ENOMEM;
+ if (!mc16_get(f->token, &f->guard)) {
+ kfree(dst);
+ return -ENOENT;
+ }
+ /* Byte roundtrip models a successful XA exit, not preparation alone. */
+ folio_lock(f->src);
+ folio_set_swapbacked(f->src);
+ mc16_publish_cache_eviction(f->src, &f->guard);
+ folio_unlock(f->src);
+ ret = mc16_load_page(&f->guard, dst);
+ if (!ret) {
+ src = kmap_local_folio(f->src, 0);
+ ret = memcmp(src, dst, PAGE_SIZE) ? -EILSEQ : 0;
+ kunmap_local(src);
+ }
+ mc16_ref_put(&f->guard);
+ kfree(dst);
+ return ret;
+}
+
+static void mc16_cf_expect_no_prepare(struct kunit *test, struct mc16_cf_test *f)
+{
+ KUNIT_EXPECT_EQ(test, f->arch_calls, 0U);
+ KUNIT_EXPECT_EQ(test, f->probe.allocations, 0U);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_stored_bytes), f->stored);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_compressed_bytes), f->compressed);
+#ifdef CONFIG_MEMCG
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->model.get_calls), 0);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->model.charge_calls), 0);
+#endif
+}
+
+static void mc16_cf_positive(struct kunit *test, enum mc16_cf_pattern pattern)
+{
+ struct mc16_cf_test *f = mc16_cf_begin(test);
+ u32 bytes;
+
+ if (!f)
+ return;
+ mc16_cf_fill(f, pattern);
+ KUNIT_ASSERT_TRUE(test, mc16_cf_store(f));
+ KUNIT_EXPECT_NE(test, f->token, 0UL);
+ KUNIT_EXPECT_TRUE(test, f->arch_contract);
+ KUNIT_EXPECT_EQ(test, f->arch_calls, 1U);
+ KUNIT_EXPECT_EQ(test, f->initial_refs, 1U);
+ KUNIT_EXPECT_EQ(test, f->state,
+ (unsigned int)(pattern == MC16_CF_FILL ? MC16_SAMEFILL : MC16_STORED));
+ KUNIT_EXPECT_EQ(test, f->eligible, pattern == MC16_CF_FILL ? 0U : 1U);
+ KUNIT_EXPECT_EQ(test, f->src->swap.val, 0UL);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->src), 1);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_stored_bytes),
+ f->stored + PAGE_SIZE);
+ KUNIT_ASSERT_TRUE(test, mc16_payload_length_decode(f->state, f->length, &bytes));
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_compressed_bytes),
+ f->compressed + bytes);
+ KUNIT_EXPECT_EQ(test, mc16_cf_roundtrip(f), 0);
+ KUNIT_ASSERT_TRUE(test, mc16_get(f->token, &f->guard));
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guard.slot->refs), 2U);
+ mc16_ref_put(&f->guard);
+ /* Model truncate after the successful exit used by the byte roundtrip. */
+ mc16_cache_drop(f->token);
+ f->cache_owner = false;
+ rcu_barrier();
+ KUNIT_EXPECT_EQ(test, f->payload.frees, pattern == MC16_CF_FILL ? 0U : 1U);
+ KUNIT_EXPECT_FALSE(test, mc16_entry_present(f->token));
+}
+
+static void memcomp_test_cache_folio_compressed(struct kunit *test)
+{
+ mc16_cf_positive(test, MC16_CF_BYTES);
+}
+
+static void memcomp_test_cache_folio_samefill(struct kunit *test)
+{
+ mc16_cf_positive(test, MC16_CF_FILL);
+}
+
+static void memcomp_test_cache_folio_last_byte(struct kunit *test)
+{
+ mc16_cf_positive(test, MC16_CF_LAST_BYTE);
+}
+
+static void mc16_cf_zero(struct kunit *test, bool winner)
+{
+ struct mc16_cf_test *f = mc16_cf_begin(test);
+ struct mc16_zero_owners zero;
+ bool pending;
+
+ if (!f)
+ return;
+ mc16_zero_pte_init(2);
+ f->zero_pte = 2;
+ f->token = ULONG_MAX;
+ memcomp_kunit_fail_next(MEMCOMP_KUNIT_FAIL_ARCH_PREPARE, 1, -EIO);
+ /* A zero preparation must not even reach the arch failure seam. */
+ if (!mc16_cf_store(f)) {
+ memcomp_kunit_fail_clear();
+ KUNIT_FAIL(test, "zero preparation failed");
+ return;
+ }
+ pending = memcomp_kunit_fail.stage == MEMCOMP_KUNIT_FAIL_ARCH_PREPARE &&
+ memcomp_kunit_fail.remaining == 1;
+ memcomp_kunit_fail_clear();
+ KUNIT_EXPECT_TRUE(test, pending);
+ KUNIT_EXPECT_EQ(test, f->token, 0UL);
+ mc16_cf_expect_no_prepare(test, f);
+ mc16_zero_owners_read(&zero);
+ KUNIT_EXPECT_EQ(test, zero.pte_pages, f->zero_before.pte_pages + 2);
+ KUNIT_EXPECT_EQ(test, zero.cache_pages, f->zero_before.cache_pages);
+ if (!winner)
+ return; /* A losing preparation performs no publish and owes no drop. */
+ /* Model only the successful XA winner's explicit publication. */
+ mc16_zero_cache_get(1);
+ f->zero_cache = 1;
+ mc16_zero_owners_read(&zero);
+ KUNIT_EXPECT_EQ(test, zero.cache_pages, f->zero_before.cache_pages + 1);
+ KUNIT_EXPECT_EQ(test, zero.pte_pages, f->zero_before.pte_pages + 2);
+ mc16_cache_drop(0);
+ f->zero_cache = 0;
+ mc16_zero_owners_read(&zero);
+ KUNIT_EXPECT_EQ(test, zero.cache_pages, f->zero_before.cache_pages);
+ KUNIT_EXPECT_EQ(test, zero.pte_pages, f->zero_before.pte_pages + 2);
+}
+
+static void memcomp_test_cache_folio_zero_abort(struct kunit *test)
+{
+ mc16_cf_zero(test, false);
+}
+
+static void memcomp_test_cache_folio_zero_publish(struct kunit *test)
+{
+ mc16_cf_zero(test, true);
+}
+
+static void mc16_cf_arch_fail(struct kunit *test, bool samefill, bool seam)
+{
+ struct mc16_cf_test *f = mc16_cf_begin(test);
+ bool ok;
+
+ if (!f)
+ return;
+ mc16_cf_fill(f, samefill ? MC16_CF_FILL : MC16_CF_BYTES);
+ f->token = ULONG_MAX;
+ f->arch_error = -EIO;
+ f->arch_passthrough = seam;
+ if (seam)
+ memcomp_kunit_fail_next(MEMCOMP_KUNIT_FAIL_ARCH_PREPARE, 1, -EIO);
+ ok = mc16_cf_store(f);
+ memcomp_kunit_fail_clear();
+ KUNIT_EXPECT_FALSE(test, ok);
+ KUNIT_EXPECT_EQ(test, f->token, ULONG_MAX);
+ KUNIT_EXPECT_EQ(test, f->src->swap.val, 0UL);
+ KUNIT_EXPECT_TRUE(test, f->arch_contract);
+ KUNIT_EXPECT_EQ(test, f->initial_refs, 1U);
+ KUNIT_EXPECT_EQ(test, f->arch_calls, 1U);
+ KUNIT_EXPECT_EQ(test, f->payload.frees, samefill ? 0U : 1U);
+ /* Rollback releases payloads while preserving the appendable header. */
+ KUNIT_EXPECT_FALSE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_FALSE(test, hash_empty(mc16_keys));
+ /* Cleanup explicitly reclaims metadata before checking charge refunds. */
+}
+
+static void memcomp_test_cache_folio_arch_failure(struct kunit *test)
+{
+ mc16_cf_arch_fail(test, false, false);
+}
+
+static void memcomp_test_cache_folio_arch_seam(struct kunit *test)
+{
+ mc16_cf_arch_fail(test, false, true);
+}
+
+static void memcomp_test_cache_folio_samefill_arch_failure(struct kunit *test)
+{
+ mc16_cf_arch_fail(test, true, false);
+}
+
+static void memcomp_test_cache_folio_disabled(struct kunit *test)
+{
+ struct mc16_cf_test *f = mc16_cf_begin(test);
+ bool saved, zero_ok, nonzero_ok;
+
+ if (!f)
+ return;
+ f->token = ULONG_MAX;
+ saved = READ_ONCE(memcomp_enabled);
+ /* Publish the temporary disabled state to the existing enabled reader. */
+ smp_store_release(&memcomp_enabled, false);
+ zero_ok = mc16_cf_store(f);
+ mc16_cf_fill(f, MC16_CF_BYTES);
+ nonzero_ok = mc16_cf_store(f);
+ /* Restore before assertions and before cleanup can call other helpers. */
+ smp_store_release(&memcomp_enabled, saved);
+ KUNIT_EXPECT_FALSE(test, zero_ok);
+ KUNIT_EXPECT_FALSE(test, nonzero_ok);
+ KUNIT_EXPECT_EQ(test, f->token, ULONG_MAX);
+ mc16_cf_expect_no_prepare(test, f);
+}
+
+/* Mapcount/writeback/mlock/ref-bias are eligibility models, not real GUP/IO. */
+static void memcomp_test_cache_folio_preflight(struct kunit *test)
+{
+ struct mc16_cf_test *f = mc16_cf_begin(test);
+ bool null_folio, null_token, unlocked, mapped, mlocked = false;
+ bool writeback, pinned;
+
+ if (!f)
+ return;
+ f->token = ULONG_MAX;
+ null_folio = mc16_store_cache_folio(NULL, &f->token);
+ unlocked = mc16_store_cache_folio(f->src, &f->token);
+ folio_lock(f->src);
+ null_token = mc16_store_cache_folio(f->src, NULL);
+ atomic_set(&f->src->_mapcount, 0);
+ mapped = mc16_store_cache_folio(f->src, &f->token);
+ atomic_set(&f->src->_mapcount, -1);
+#ifdef CONFIG_MMU
+ folio_set_mlocked(f->src);
+ mlocked = mc16_store_cache_folio(f->src, &f->token);
+ folio_clear_mlocked(f->src);
+#endif
+ /* Raw flag manipulation avoids pretending to start real writeback IO. */
+ __set_bit(PG_writeback, &f->src->flags);
+ writeback = mc16_store_cache_folio(f->src, &f->token);
+ __clear_bit(PG_writeback, &f->src->flags);
+ folio_ref_add(f->src, GUP_PIN_COUNTING_BIAS);
+ pinned = mc16_store_cache_folio(f->src, &f->token);
+ folio_ref_sub(f->src, GUP_PIN_COUNTING_BIAS);
+ folio_unlock(f->src);
+ KUNIT_EXPECT_FALSE(test, null_folio);
+ KUNIT_EXPECT_FALSE(test, null_token);
+ KUNIT_EXPECT_FALSE(test, unlocked);
+ KUNIT_EXPECT_FALSE(test, mapped);
+ KUNIT_EXPECT_FALSE(test, mlocked);
+ KUNIT_EXPECT_FALSE(test, writeback);
+ KUNIT_EXPECT_FALSE(test, pinned);
+ KUNIT_EXPECT_EQ(test, f->token, ULONG_MAX);
+ mc16_cf_expect_no_prepare(test, f);
+}
+
+static void memcomp_test_cache_folio_large(struct kunit *test)
+{
+ struct mc16_cf_test *f = mc16_cf_begin(test);
+ bool ok;
+
+ if (!f)
+ return;
+ f->large = folio_alloc(GFP_KERNEL | __GFP_ZERO, 1);
+ KUNIT_ASSERT_NOT_NULL(test, f->large);
+ f->token = ULONG_MAX;
+ folio_lock(f->large);
+ ok = mc16_store_cache_folio(f->large, &f->token);
+ folio_unlock(f->large);
+ KUNIT_EXPECT_FALSE(test, ok);
+ KUNIT_EXPECT_EQ(test, f->token, ULONG_MAX);
+ mc16_cf_expect_no_prepare(test, f);
+}
+
+static void mc16_cf_prepare_fail(struct kunit *test, unsigned int charge)
+{
+ struct mc16_cf_test *f = mc16_cf_begin(test);
+ bool ok;
+
+ if (!f)
+ return;
+ mc16_cf_fill(f, MC16_CF_BYTES);
+ f->token = ULONG_MAX;
+ if (!charge)
+ f->probe.fail_at = f->probe.allocations + 1;
+#ifdef CONFIG_MEMCG
+ else
+ f->model.fail_at = atomic_read(&f->model.charge_calls) + charge;
+#endif
+ ok = mc16_cf_store(f);
+ f->probe.fail_at = 0;
+#ifdef CONFIG_MEMCG
+ f->model.fail_at = 0;
+#endif
+ KUNIT_EXPECT_FALSE(test, ok);
+ KUNIT_EXPECT_EQ(test, f->token, ULONG_MAX);
+ KUNIT_EXPECT_EQ(test, f->arch_calls, 0U);
+ KUNIT_EXPECT_EQ(test, f->src->swap.val, 0UL);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ /* Payload preparation is real; deferred metadata refunds checked at exit. */
+}
+
+static void memcomp_test_cache_folio_metadata_failure(struct kunit *test)
+{
+ mc16_cf_prepare_fail(test, 0);
+}
+
+#ifdef CONFIG_MEMCG
+static void memcomp_test_cache_folio_charge_failure(struct kunit *test)
+{
+ mc16_cf_prepare_fail(test, 1);
+}
+
+static void memcomp_test_cache_folio_key_charge_failure(struct kunit *test)
+{
+ mc16_cf_prepare_fail(test, 2);
+}
+#endif
+
+#if defined(CONFIG_ARM64) && defined(CONFIG_ARM64_MTE)
+static void memcomp_test_cache_folio_tagged_zero(struct kunit *test)
+{
+ struct mc16_cf_test *f = mc16_cf_begin(test);
+ bool ok;
+
+ if (!f)
+ return;
+ if (!system_supports_mte()) {
+ kunit_skip(test, "requires the arm64 MTE pages_identical predicate");
+ return;
+ }
+ /* Predicate/flag model only: no real tag instructions or arch swap IO. */
+ set_page_mte_tagged(&f->src->page);
+ ok = mc16_cf_store(f);
+ clear_bit(PG_mte_tagged, &f->src->flags);
+ KUNIT_ASSERT_TRUE(test, ok);
+ KUNIT_EXPECT_NE(test, f->token, 0UL);
+ KUNIT_EXPECT_EQ(test, f->state, (unsigned int)MC16_SAMEFILL);
+ KUNIT_EXPECT_EQ(test, f->arch_calls, 1U);
+ KUNIT_EXPECT_EQ(test, mc16_cf_roundtrip(f), 0);
+}
+#endif
+
+/*
+ * Register in the enclosing memcomp_test_cases[] (no new suite):
+ *
+ * KUNIT_CASE(memcomp_test_cache_folio_compressed),
+ * KUNIT_CASE(memcomp_test_cache_folio_samefill),
+ * KUNIT_CASE(memcomp_test_cache_folio_last_byte),
+ * KUNIT_CASE(memcomp_test_cache_folio_zero_abort),
+ * KUNIT_CASE(memcomp_test_cache_folio_zero_publish),
+ * KUNIT_CASE(memcomp_test_cache_folio_arch_failure),
+ * KUNIT_CASE(memcomp_test_cache_folio_arch_seam),
+ * KUNIT_CASE(memcomp_test_cache_folio_samefill_arch_failure),
+ * KUNIT_CASE(memcomp_test_cache_folio_disabled),
+ * KUNIT_CASE(memcomp_test_cache_folio_preflight),
+ * KUNIT_CASE(memcomp_test_cache_folio_large),
+ * KUNIT_CASE(memcomp_test_cache_folio_metadata_failure),
+ *
+ * Additional CONFIG_MEMCG cases:
+ * KUNIT_CASE(memcomp_test_cache_folio_charge_failure),
+ * KUNIT_CASE(memcomp_test_cache_folio_key_charge_failure),
+ *
+ * Additional CONFIG_ARM64 && CONFIG_ARM64_MTE case (runtime MTE required):
+ * KUNIT_CASE(memcomp_test_cache_folio_tagged_zero),
+ */
+
+#endif /* _MM_MEMCOMP_TESTS_CACHE_FOLIO_TEST_H */
diff --git a/mm/memcomp/tests/cache_reserve_test.h b/mm/memcomp/tests/cache_reserve_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..b4e5acad96ba7f209497182a19c3fadfc876d5ad
--- /dev/null
+++ b/mm/memcomp/tests/cache_reserve_test.h
@@ -0,0 +1,447 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_CACHE_RESERVE_TEST_H
+#define _MM_MEMCOMP_TESTS_CACHE_RESERVE_TEST_H
+
+/* Include after storage_test.h; this header does not register any test cases. */
+#ifdef CONFIG_MEMCG
+struct mc16_test_cache_reserve {
+ struct mc16_test_storage *storage;
+ struct mc16_initial_payload initial[MC16_MAX_PAGES];
+ bool owned[MC16_MAX_PAGES];
+};
+
+static void mc16_test_cache_drop(struct mc16_test_storage *s);
+
+/* Prepared charges are private; stable slots use the storage cleanup action. */
+static void mc16_test_cache_cleanup(void *data)
+{
+ struct mc16_test_cache_reserve *f = data;
+ struct obj_cgroup *objcg = &f->storage->owner->model.objcg;
+ unsigned int i;
+
+ /* Teardown must not re-enter the fixture's allocation-window callback. */
+ f->storage->probe.fail_at = 0;
+ f->storage->probe.fail_descriptor = false;
+ f->storage->probe.allocation_window = NULL;
+
+ for (i = 0; i < MC16_MAX_PAGES; i++) {
+ if (!f->owned[i])
+ continue;
+ if (f->initial[i].bytes) {
+ memcomp_zs_free(0, f->initial[i].word);
+ memcomp_objcg_uncharge(objcg, f->initial[i].bytes);
+ }
+ }
+ mc16_test_cache_drop(f->storage);
+ /* Return the independent source hold after private storage cleanup. */
+ memcomp_objcg_put(objcg);
+}
+
+static struct mc16_test_cache_reserve *
+mc16_test_cache_begin(struct kunit *test)
+{
+ struct mc16_test_cache_reserve *f;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->storage = mc16_test_storage_begin(test);
+ if (!f->storage)
+ return NULL;
+ /* Explicitly model the source hold required across the allocation window. */
+ memcomp_objcg_get_ref(&f->storage->owner->model.objcg);
+ if (kunit_add_action_or_reset(test, mc16_test_cache_cleanup, f))
+ return NULL;
+ return f;
+}
+
+static int mc16_test_cache_prepare(struct mc16_test_cache_reserve *f,
+ unsigned int index, u32 bytes, u64 word)
+{
+ struct obj_cgroup *objcg = &f->storage->owner->model.objcg;
+ unsigned long handle;
+ void *mapped;
+ int ret;
+
+ if (index >= MC16_MAX_PAGES || f->owned[index] || bytes > PAGE_SIZE)
+ return -EINVAL;
+ if (bytes) {
+ ret = memcomp_zs_malloc(memcomp_pools[0], bytes, &handle, false);
+ if (ret)
+ return ret;
+ ret = memcomp_objcg_charge(objcg, GFP_NOWAIT | __GFP_NOWARN, bytes);
+ if (ret) {
+ memcomp_zs_free(0, handle);
+ return ret;
+ }
+ mapped = zs_map_object(memcomp_pools[0], handle, ZS_MM_WO);
+ memset(mapped, 0x5a, bytes);
+ zs_unmap_object(memcomp_pools[0], handle);
+ word = handle;
+ }
+ f->initial[index] = (struct mc16_initial_payload) {
+ .word = word, .bytes = bytes,
+ };
+ f->owned[index] = true;
+ return 0;
+}
+
+static int mc16_test_cache_reserve(struct mc16_test_cache_reserve *f,
+ unsigned int nr)
+{
+ struct mc16_test_storage *s = f->storage;
+ unsigned int i;
+ int ret;
+
+ if (nr > MC16_MAX_PAGES || s->nr + nr > MC16_TEST_REFS)
+ return -ENOSPC;
+ ret = mc16_reserve_cache(&s->owner->model.objcg,
+ folio_nid(s->owner->folio), f->initial, nr,
+ &s->refs[s->nr]);
+ if (!ret) {
+ for (i = 0; i < nr; i++)
+ f->owned[i] = false;
+ s->nr += nr;
+ }
+ return ret;
+}
+
+static void mc16_test_cache_drop(struct mc16_test_storage *s)
+{
+ unsigned int i;
+
+ /* No pending/stat/FIFO publication occurred in this storage-only fixture. */
+ for (i = 0; i < s->nr; i++) {
+ struct memcomp_ref *ref = &s->refs[i];
+
+ if (!ref->slot)
+ continue;
+ /* Private reservations never acquired an initial counted ref. */
+ mc16_release_resources(ref);
+ mc16_retire_slot(ref);
+ }
+ rcu_barrier();
+}
+
+static void memcomp_test_cache_reserve_payloads(struct kunit *test)
+{
+ const u64 fill = 0xfedcba9876543210ULL;
+ struct mc16_test_cache_reserve *f = mc16_test_cache_begin(test);
+ struct mc16_test_storage *s;
+ struct mc16_initial_payload saved[4];
+ struct mc16_view view;
+ unsigned int i, state;
+ u32 bytes;
+ long charge;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->storage;
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_prepare(f, 0, PAGE_SIZE, 0), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_prepare(f, 1, 0, fill), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_prepare(f, 2, 128, 0), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_prepare(f, 3, 0, 0), 0);
+ memcpy(saved, f->initial, sizeof(saved));
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_reserve(f, 4), 0);
+ for (i = 0; i < ARRAY_SIZE(saved); i++) {
+ struct memcomp_ref *ref = &s->refs[i];
+
+ state = saved[i].bytes ? MC16_STORED : MC16_SAMEFILL;
+ KUNIT_EXPECT_EQ(test, ref->slot->word, saved[i].word);
+ KUNIT_EXPECT_EQ(test, f->initial[i].word, saved[i].word);
+ KUNIT_EXPECT_EQ(test, f->initial[i].bytes, saved[i].bytes);
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), 0);
+ KUNIT_EXPECT_EQ(test, ref->slot->flags,
+ (u16)(state | (2 << MC16_ORDER_SHIFT)));
+ KUNIT_EXPECT_TRUE(test, mc16_payload_length_decode(state,
+ ref->slot->length, &bytes));
+ KUNIT_EXPECT_EQ(test, bytes, saved[i].bytes);
+ KUNIT_ASSERT_TRUE(test, mc16_view_owned(ref, &view));
+ KUNIT_EXPECT_PTR_EQ(test, view.record, NULL);
+ KUNIT_EXPECT_PTR_EQ(test, view.member, NULL);
+ KUNIT_EXPECT_PTR_EQ(test, ref->cluster, s->refs[0].cluster);
+ KUNIT_EXPECT_EQ(test, ref->token, s->refs[0].token + i);
+ }
+ KUNIT_EXPECT_EQ(test, s->refs[0].token & 3U, 0U);
+ KUNIT_EXPECT_EQ(test, s->refs[0].cluster->key->kind, (u8)MC16_KEY_SHMEM);
+ KUNIT_EXPECT_PTR_EQ(test, s->refs[0].cluster->key->objcg,
+ &s->owner->model.objcg);
+ KUNIT_EXPECT_EQ(test, s->refs[0].cluster->eligible_slots, (u16)0);
+ charge = PAGE_SIZE + 128 + s->refs[0].cluster->key->metadata_bytes;
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged), charge);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(s->owner->folio), 1);
+ KUNIT_EXPECT_EQ(test, atomic_read(&s->owner->model.get_calls), 2);
+ s->payload.task = current;
+ s->payload.pool = memcomp_pools[0];
+ s->payload.handle = saved[0].word;
+ s->payload.live = true;
+ rcu_assign_pointer(memcomp_kunit_async_probe, &s->payload);
+ mc16_test_cache_drop(s);
+ KUNIT_EXPECT_EQ(test, s->payload.frees, 1U);
+ KUNIT_EXPECT_FALSE(test, s->payload.live);
+ KUNIT_EXPECT_FALSE(test, s->payload.bad_lifetime);
+ KUNIT_EXPECT_FALSE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_FALSE(test, hash_empty(mc16_keys));
+ /* Source and key remain independent until explicit pressure reclaim. */
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged), 0L);
+ /* Only the caller's explicit source hold remains. */
+ KUNIT_EXPECT_EQ(test, atomic_read(&s->owner->model.get_calls) -
+ atomic_read(&s->owner->model.put_calls), 1);
+}
+
+static void memcomp_test_cache_reserve_boundaries(struct kunit *test)
+{
+ struct mc16_test_cache_reserve *f = mc16_test_cache_begin(test);
+ struct mc16_test_storage *s;
+ struct mc16_cluster *first;
+ struct memcomp_ref guard;
+ u32 base;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->storage;
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_reserve(f, 1), 0);
+ first = s->refs[0].cluster;
+ base = s->refs[0].token;
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_reserve(f, 2), 0);
+ KUNIT_EXPECT_EQ(test, s->refs[1].token, base + 2);
+ KUNIT_EXPECT_FALSE(test, mc16_get(base + 1, &guard));
+ /* Segment, shadow array and cluster; the second reserve reuses them. */
+ KUNIT_EXPECT_EQ(test, s->probe.allocations, 3U);
+ while (first->next < MC16_SEGMENT_SLOTS)
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_reserve(f, 1), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_reserve(f, MC16_MAX_PAGES), 0);
+ KUNIT_EXPECT_EQ(test, s->refs[s->nr - MC16_MAX_PAGES].token, base + 64);
+ KUNIT_EXPECT_EQ(test, s->refs[s->nr - 1].slot->flags,
+ (u16)(MC16_SAMEFILL |
+ (MC16_MAX_ORDER << MC16_ORDER_SHIFT)));
+ while (first->next < MC16_CLUSTER_SLOTS)
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_reserve(f, 1), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_reserve(f, 1), 0);
+ KUNIT_EXPECT_PTR_NE(test, s->refs[s->nr - 1].cluster, first);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(s->owner->folio), 1);
+}
+
+static void memcomp_test_cache_reserve_invalid(struct kunit *test)
+{
+ struct mc16_test_cache_reserve *f = mc16_test_cache_begin(test);
+ const unsigned int bad_nr[] = { 0, 3, MC16_MAX_PAGES + 1, 65537, UINT_MAX };
+ const unsigned int bad_nid[] = { MAX_NUMNODES, 65536, UINT_MAX };
+ struct memcomp_ref refs[MC16_MAX_PAGES];
+ struct mc16_test_storage *s;
+ struct obj_cgroup *objcg;
+ unsigned int i;
+ u64 handle;
+ long charge;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->storage;
+ objcg = &s->owner->model.objcg;
+ for (i = 0; i < ARRAY_SIZE(refs); i++)
+ refs[i] = (struct memcomp_ref) { .token = U32_MAX };
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_prepare(f, 0, 128, 0), 0);
+ handle = f->initial[0].word;
+ charge = atomic_long_read(&s->owner->model.charged);
+ for (i = 0; i < ARRAY_SIZE(bad_nr); i++)
+ KUNIT_EXPECT_EQ(test, mc16_reserve_cache(objcg, 0, f->initial,
+ bad_nr[i], refs), -EINVAL);
+ for (i = 0; i < ARRAY_SIZE(bad_nid); i++)
+ KUNIT_EXPECT_EQ(test, mc16_reserve_cache(objcg, bad_nid[i],
+ f->initial, 1, refs), -EINVAL);
+ KUNIT_EXPECT_EQ(test, mc16_reserve_cache(objcg, 0, NULL, 1, refs), -EINVAL);
+ KUNIT_EXPECT_EQ(test, mc16_reserve_cache(objcg, 0, f->initial, 1, NULL),
+ -EINVAL);
+ /* The last bad element must prevent publication of every earlier one. */
+ f->initial[MC16_MAX_PAGES - 1].bytes = PAGE_SIZE + 1;
+ f->initial[MC16_MAX_PAGES - 1].word = 1;
+ KUNIT_EXPECT_EQ(test, mc16_reserve_cache(objcg, 0, f->initial,
+ MC16_MAX_PAGES, refs), -EINVAL);
+ f->initial[MC16_MAX_PAGES - 1].bytes = U32_MAX;
+ KUNIT_EXPECT_EQ(test, mc16_reserve_cache(objcg, 0, f->initial,
+ MC16_MAX_PAGES, refs), -EINVAL);
+ f->initial[MC16_MAX_PAGES - 1].bytes = 1;
+ f->initial[MC16_MAX_PAGES - 1].word = 0;
+ KUNIT_EXPECT_EQ(test, mc16_reserve_cache(objcg, 0, f->initial,
+ MC16_MAX_PAGES, refs), -EINVAL);
+ for (i = 0; i < ARRAY_SIZE(refs); i++) {
+ KUNIT_EXPECT_EQ(test, refs[i].token, U32_MAX);
+ KUNIT_EXPECT_PTR_EQ(test, refs[i].slot, NULL);
+ }
+ KUNIT_EXPECT_EQ(test, f->initial[0].word, handle);
+ KUNIT_EXPECT_TRUE(test, f->owned[0]);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged), charge);
+ KUNIT_EXPECT_EQ(test, s->probe.allocations, 0U);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+}
+
+static int mc16_test_cache_id_fail(u32 *id)
+{
+ return -ENOSPC;
+}
+
+static struct mc16_key *mc16_test_cache_key_fail(void)
+{
+ return NULL;
+}
+
+static void memcomp_test_cache_reserve_nomem(struct kunit *test)
+{
+ struct mc16_test_cache_reserve *f = mc16_test_cache_begin(test);
+ struct mc16_test_storage *s;
+ unsigned int at;
+ u64 handle;
+ long charge;
+ u8 *mapped, first, last;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->storage;
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_prepare(f, 0, 128, 0), 0);
+ handle = f->initial[0].word;
+ charge = atomic_long_read(&s->owner->model.charged);
+ /* Keep header failure coverage after adding the shadow allocation. */
+ for (at = 1; at <= 3; at++) {
+ s->probe.fail_at = s->probe.allocations + at;
+ KUNIT_EXPECT_EQ(test, mc16_test_cache_reserve(f, 1), -ENOMEM);
+ s->probe.fail_at = 0;
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged),
+ charge);
+ }
+ kunit_activate_static_stub(test, mc16_key_zalloc, mc16_test_cache_key_fail);
+ KUNIT_EXPECT_EQ(test, mc16_test_cache_reserve(f, 1), -ENOMEM);
+ kunit_deactivate_static_stub(test, mc16_key_zalloc);
+ s->owner->model.fail_at = atomic_read(&s->owner->model.charge_calls) + 1;
+ KUNIT_EXPECT_EQ(test, mc16_test_cache_reserve(f, 1), -ENOMEM);
+ s->owner->model.fail_at = 0;
+ kunit_activate_static_stub(test, mc16_private_id_alloc, mc16_test_cache_id_fail);
+ KUNIT_EXPECT_EQ(test, mc16_test_cache_reserve(f, 1), -ENOSPC);
+ kunit_deactivate_static_stub(test, mc16_private_id_alloc);
+ KUNIT_EXPECT_EQ(test, f->initial[0].word, handle);
+ KUNIT_EXPECT_EQ(test, f->initial[0].bytes, 128U);
+ KUNIT_EXPECT_TRUE(test, f->owned[0]);
+ KUNIT_EXPECT_EQ(test, s->nr, 0U);
+ KUNIT_EXPECT_PTR_EQ(test, s->refs[0].slot, NULL);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged), charge);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(s->owner->folio), 1);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+ /* A rejected reserve must leave the real zs object mapped and intact. */
+ mapped = zs_map_object(memcomp_pools[0], handle, ZS_MM_RO);
+ first = mapped[0];
+ last = mapped[127];
+ zs_unmap_object(memcomp_pools[0], handle);
+ KUNIT_EXPECT_EQ(test, first, (u8)0x5a);
+ KUNIT_EXPECT_EQ(test, last, (u8)0x5a);
+}
+
+struct mc16_test_cache_window {
+ struct mc16_test_storage *storage;
+ int result;
+ int source_holds;
+};
+
+static void mc16_test_cache_compete(void *data)
+{
+ struct mc16_test_cache_window *window = data;
+ struct mc16_test_storage *s = window->storage;
+ struct mc16_initial_payload initial = { .word = U64_MAX };
+
+ window->source_holds = atomic_read(&s->owner->model.get_calls) -
+ atomic_read(&s->owner->model.put_calls);
+ window->result = mc16_reserve_cache(&s->owner->model.objcg,
+ folio_nid(s->owner->folio), &initial, 1,
+ &s->refs[s->nr]);
+ if (!window->result)
+ s->nr++;
+}
+
+static void memcomp_test_cache_reserve_race(struct kunit *test)
+{
+ struct mc16_test_cache_reserve *f = mc16_test_cache_begin(test);
+ struct mc16_test_cache_window window;
+ struct mc16_test_storage *s;
+ long charge;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->storage;
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_prepare(f, 0, 128, 0), 0);
+ window = (struct mc16_test_cache_window) { .storage = s };
+ s->probe.allocation_window = mc16_test_cache_compete;
+ s->probe.window_data = &window;
+ KUNIT_EXPECT_EQ(test, mc16_test_cache_reserve(f, 1), -EAGAIN);
+ KUNIT_EXPECT_EQ(test, window.result, 0);
+ KUNIT_EXPECT_EQ(test, window.source_holds, 1);
+ KUNIT_EXPECT_EQ(test, s->nr, 1U);
+ KUNIT_EXPECT_EQ(test, s->refs[0].slot->word, U64_MAX);
+ KUNIT_EXPECT_TRUE(test, f->owned[0]);
+ charge = 128 + s->refs[0].cluster->key->metadata_bytes;
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged), charge);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(s->owner->folio), 1);
+}
+
+static void mc16_test_cache_retire_window(void *data)
+{
+ struct mc16_test_cache_window *window = data;
+ struct mc16_test_storage *s = window->storage;
+
+ mc16_test_cache_drop(s);
+ window->source_holds = atomic_read(&s->owner->model.get_calls) -
+ atomic_read(&s->owner->model.put_calls);
+}
+
+static void memcomp_test_cache_reserve_retire(struct kunit *test)
+{
+ struct mc16_test_cache_reserve *f = mc16_test_cache_begin(test);
+ struct mc16_test_cache_window window;
+ struct mc16_test_storage *s;
+ struct mc16_cluster *cluster;
+ unsigned int i;
+ u32 base;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->storage;
+ for (i = 0; i < MC16_SEGMENT_SLOTS; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_reserve(f, 1), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_prepare(f, 0, 128, 0), 0);
+ cluster = s->refs[0].cluster;
+ base = s->refs[0].token;
+ window = (struct mc16_test_cache_window) { .storage = s };
+ s->probe.allocation_window = mc16_test_cache_retire_window;
+ s->probe.window_data = &window;
+ /* Last-live retirement preserves X1 geometry, so X2 can append. */
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_reserve(f, 1), 0);
+ KUNIT_EXPECT_EQ(test, window.source_holds, 2);
+ KUNIT_EXPECT_FALSE(test, f->owned[0]);
+ KUNIT_EXPECT_EQ(test, s->nr, MC16_SEGMENT_SLOTS + 1U);
+ KUNIT_EXPECT_PTR_EQ(test, s->refs[MC16_SEGMENT_SLOTS].cluster, cluster);
+ KUNIT_EXPECT_EQ(test, s->refs[MC16_SEGMENT_SLOTS].token,
+ base + MC16_SEGMENT_SLOTS);
+ KUNIT_EXPECT_EQ(test, cluster->live_slots, 1U);
+ KUNIT_EXPECT_EQ(test, cluster->next, MC16_SEGMENT_SLOTS + 1U);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged),
+ 128L + cluster->key->metadata_bytes);
+ KUNIT_EXPECT_FALSE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_FALSE(test, hash_empty(mc16_keys));
+}
+
+static void memcomp_test_cache_reserve_root(struct kunit *test)
+{
+ struct mc16_test_cache_reserve *f = mc16_test_cache_begin(test);
+ struct mc16_initial_payload initial = { .word = U64_MAX };
+ struct mc16_test_storage *s;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->storage;
+ KUNIT_ASSERT_EQ(test, mc16_reserve_cache(NULL, folio_nid(s->owner->folio),
+ &initial, 1, s->refs), 0);
+ s->nr = 1;
+ KUNIT_EXPECT_PTR_EQ(test, s->refs[0].cluster->key->objcg, NULL);
+ KUNIT_EXPECT_EQ(test, s->refs[0].slot->word, U64_MAX);
+ KUNIT_EXPECT_EQ(test, s->refs[0].slot->length, (u16)0);
+ KUNIT_EXPECT_EQ(test, s->refs[0].slot->flags, (u16)MC16_SAMEFILL);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged), 0L);
+ mc16_test_cache_drop(s);
+}
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_CACHE_RESERVE_TEST_H */
diff --git a/mm/memcomp/tests/cache_store_test.h b/mm/memcomp/tests/cache_store_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..1aefd8e8f0a575d1d59d7b5391ca011812b6559b
--- /dev/null
+++ b/mm/memcomp/tests/cache_store_test.h
@@ -0,0 +1,216 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_CACHE_STORE_TEST_H
+#define _MM_MEMCOMP_TESTS_CACHE_STORE_TEST_H
+#ifdef CONFIG_MEMCG
+struct mc16_test_cache_producer {
+ struct mc16_test_refs *refs;
+ struct memcomp_ref cache;
+ struct memcomp_ref private;
+};
+
+static void mc16_test_cache_producer_cleanup(void *data)
+{
+ struct mc16_test_cache_producer *f = data;
+
+ if (f->private.slot) {
+ mc16_release_resources(&f->private);
+ mc16_retire_slot(&f->private);
+ }
+ if (f->cache.slot)
+ mc16_ref_put(&f->cache);
+}
+
+static struct mc16_test_cache_producer *
+mc16_test_cache_producer_begin(struct kunit *test)
+{
+ struct mc16_test_cache_producer *f;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->refs = mc16_test_refs_begin(test);
+ if (!f->refs || kunit_add_action_or_reset(test,
+ mc16_test_cache_producer_cleanup, f))
+ return NULL;
+ return f;
+}
+
+static void mc16_test_cache_roundtrip(struct kunit *test, bool samefill)
+{
+ const unsigned long pattern = 0x9876543210fedcbaUL;
+ struct mc16_test_cache_producer *f = mc16_test_cache_producer_begin(test);
+ struct mc16_test_storage *storage;
+ void *src, *dst;
+ unsigned int i;
+ int folio_refs;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->refs->storage;
+ src = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ dst = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ KUNIT_ASSERT_NOT_NULL(test, dst);
+ if (samefill)
+ memset_l(src, pattern, PAGE_SIZE / sizeof(unsigned long));
+ else
+ for (i = 0; i < PAGE_SIZE; i++)
+ ((u8 *)src)[i] = i % 53;
+ folio_refs = folio_ref_count(storage->owner->folio);
+ KUNIT_ASSERT_EQ(test, mc16_cache_store(src, storage->owner->folio,
+ &f->cache), 0);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(f->cache.slot->flags),
+ (unsigned int)(samefill ? MC16_SAMEFILL : MC16_STORED));
+ KUNIT_EXPECT_FALSE(test, f->cache.slot->flags &
+ (MC16_F_OWNER | MC16_F_RESIDENT));
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->cache.slot->refs), 1U);
+ KUNIT_EXPECT_EQ(test, f->cache.cluster->key->kind, (u8)MC16_KEY_SHMEM);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(storage->owner->folio), folio_refs);
+ KUNIT_EXPECT_EQ(test, f->cache.cluster->eligible_slots,
+ (u16)(samefill ? 0 : 1));
+ KUNIT_ASSERT_EQ(test, mc16_load_page(&f->cache, dst), 0);
+ KUNIT_EXPECT_EQ(test, memcmp(src, dst, PAGE_SIZE), 0);
+ mc16_ref_put(&f->cache);
+ /* Payload release precedes explicit reclaim of the retained header. */
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&storage->owner->model.charged), 0L);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+}
+
+static void memcomp_test_cache_store_samefill(struct kunit *test)
+{
+ mc16_test_cache_roundtrip(test, true);
+}
+
+static void memcomp_test_cache_store_compressed(struct kunit *test)
+{
+ mc16_test_cache_roundtrip(test, false);
+}
+
+static void mc16_test_cache_prepare_failure(struct kunit *test, bool charge)
+{
+ struct mc16_test_cache_producer *f = mc16_test_cache_producer_begin(test);
+ struct mc16_test_storage *storage;
+ u8 *src;
+ unsigned int i;
+ int folio_refs;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->refs->storage;
+ src = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ for (i = 0; i < PAGE_SIZE; i++)
+ src[i] = i % 53;
+ folio_refs = folio_ref_count(storage->owner->folio);
+ if (charge)
+ storage->owner->model.fail_at =
+ atomic_read(&storage->owner->model.charge_calls) + 1;
+ else
+ storage->probe.fail_at = storage->probe.allocations + 1;
+ KUNIT_EXPECT_EQ(test, mc16_cache_store(src, storage->owner->folio,
+ &f->cache), -ENOMEM);
+ storage->owner->model.fail_at = 0;
+ storage->probe.fail_at = 0;
+ KUNIT_EXPECT_PTR_EQ(test, f->cache.slot, NULL);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(storage->owner->folio), folio_refs);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&storage->owner->model.charged), 0L);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_EQ(test, atomic_read(&storage->owner->model.get_calls),
+ atomic_read(&storage->owner->model.put_calls));
+}
+
+static void memcomp_test_cache_store_charge_failure(struct kunit *test)
+{
+ mc16_test_cache_prepare_failure(test, true);
+}
+
+static void memcomp_test_cache_store_reserve_failure(struct kunit *test)
+{
+ mc16_test_cache_prepare_failure(test, false);
+}
+
+/* Schedule the publication-window interleaving on an already linked cluster. */
+static void memcomp_test_cache_publish_window(struct kunit *test)
+{
+ struct mc16_test_cache_producer *f = mc16_test_cache_producer_begin(test);
+ struct mc16_initial_payload initial = { .bytes = 128 };
+ struct memcomp_ref guard, active;
+ struct mc16_test_storage *s;
+ struct memcomp_lru *fifo;
+ struct mc16_view view;
+ unsigned long handle;
+ unsigned int budget, i;
+ unsigned short owner_id;
+ u8 *src;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ src = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ for (i = 0; i < PAGE_SIZE; i++)
+ src[i] = i % 53;
+ KUNIT_ASSERT_EQ(test, mc16_cache_store(src, s->owner->folio, &f->cache), 0);
+ fifo = mc16_fifo_of(&f->cache);
+ KUNIT_ASSERT_EQ(test, memcomp_zs_malloc(memcomp_pools[0], initial.bytes,
+ &handle, false), 0);
+ initial.word = handle;
+ ret = memcomp_objcg_charge(&s->owner->model.objcg,
+ GFP_NOWAIT | __GFP_NOWARN, initial.bytes);
+ if (ret) {
+ memcomp_zs_free(0, handle);
+ KUNIT_FAIL(test, "payload charge failed: %d", ret);
+ return;
+ }
+ ret = mc16_source_id_get(s->owner->folio, &owner_id);
+ if (!ret) {
+ ret = mc16_reserve_cache_id(&s->owner->model.objcg, owner_id,
+ folio_nid(s->owner->folio), &initial, 1,
+ &f->private);
+ mem_cgroup_memcomp_id_put(owner_id);
+ }
+ if (ret) {
+ memcomp_objcg_uncharge(&s->owner->model.objcg, initial.bytes);
+ memcomp_zs_free(0, handle);
+ KUNIT_FAIL(test, "private reserve failed: %d", ret);
+ return;
+ }
+ KUNIT_EXPECT_PTR_EQ(test, f->private.cluster, f->cache.cluster);
+ KUNIT_EXPECT_EQ(test, f->private.cluster->live_slots, 2U);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->private.slot->refs), 0U);
+ KUNIT_EXPECT_FALSE(test, mc16_get(f->private.token, &guard));
+ f->private.cluster->wb_cursor = f->private.token & (MC16_CLUSTER_SLOTS - 1);
+ budget = 1;
+ KUNIT_EXPECT_FALSE(test, mc16_fifo_select(fifo, &budget, &guard));
+ KUNIT_EXPECT_EQ(test, budget, 0U);
+ KUNIT_EXPECT_EQ(test, f->private.cluster->eligible_slots, (u16)1);
+ /* The old last STORED may disappear before new eligibility is recorded. */
+ mc16_ref_put(&f->cache);
+ KUNIT_EXPECT_TRUE(test, list_empty(&fifo->entries));
+ KUNIT_EXPECT_EQ(test, f->private.cluster->live_slots, 1U);
+ mc16_cache_publish(&f->private, initial.bytes);
+ active = f->private;
+ f->private = (struct memcomp_ref) { };
+ f->cache = active;
+ KUNIT_EXPECT_EQ(test, active.cluster->eligible_slots, (u16)1);
+ active.cluster->wb_cursor = active.token & (MC16_CLUSTER_SLOTS - 1);
+ budget = 1;
+ KUNIT_ASSERT_TRUE(test, mc16_fifo_select(fifo, &budget, &guard));
+ KUNIT_EXPECT_PTR_EQ(test, guard.slot, active.slot);
+ mutex_lock(memcomp_lock_of(guard.token));
+ if (mc16_view_locked(&guard, &view)) {
+ mc16_fifo_transition(&guard, MC16_STORED, MC16_WRITEBACK);
+ *view.flags = MC16_WRITEBACK;
+ }
+ mutex_unlock(memcomp_lock_of(guard.token));
+ KUNIT_EXPECT_EQ(test, active.cluster->eligible_slots, (u16)0);
+ KUNIT_EXPECT_TRUE(test, list_empty(&fifo->entries));
+ mc16_ref_put(&guard);
+ mc16_ref_put(&f->cache);
+ /* Payload release precedes explicit reclaim of the retained header. */
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged), 0L);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+}
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_CACHE_STORE_TEST_H */
diff --git a/mm/memcomp/tests/cache_xarray_race_test.h b/mm/memcomp/tests/cache_xarray_race_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..4b6ff059db493592dd9fe886f2558729b14ff663
--- /dev/null
+++ b/mm/memcomp/tests/cache_xarray_race_test.h
@@ -0,0 +1,193 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_CACHE_XARRAY_RACE_TEST_H
+#define _MM_MEMCOMP_TESTS_CACHE_XARRAY_RACE_TEST_H
+
+/* Include after shmem_retry_test.h. No load/error return is synthesized. */
+#ifdef MC16_SHMEM_RETRY_TEST_ENABLED
+struct mc16_cxr {
+ struct mc16_sr_test *sr;
+ struct folio *lost;
+ unsigned int calls, refs_before, refs_truncated;
+ u64 before[2], after[2], truncated_word;
+ int errors[2];
+ bool locked, unmapped, uptodate, bytes, entry_matched, emptied;
+};
+
+static struct mc16_cxr *mc16_cxr_current;
+
+static bool mc16_cxr_bytes(struct folio *folio, struct folio *source)
+{
+ void *a = kmap_local_folio(source, 0);
+ void *b = kmap_local_folio(folio, 0);
+ bool same = !memcmp(a, b, PAGE_SIZE);
+
+ kunmap_local(b);
+ kunmap_local(a);
+ return same;
+}
+
+static void mc16_cxr_cleanup(void *data)
+{
+ struct mc16_cxr *p = data;
+
+ mc16_cxr_current = NULL;
+ if (p->lost) {
+ /* The original outer call owns unlock/put; retain only our pin. */
+ KUNIT_EXPECT_FALSE(p->sr->test, folio_test_locked(p->lost));
+ if (folio_test_locked(p->lost))
+ folio_unlock(p->lost);
+ folio_clear_active(p->lost);
+ folio_put(p->lost);
+ }
+ /* A failed native-state assertion must not leave active on final put. */
+ folio_clear_active(p->sr->probe);
+}
+
+static int mc16_cxr_load(struct memcomp_ref *ref, struct folio *folio)
+{
+ struct mc16_cxr *p = mc16_cxr_current;
+ struct mc16_sr_test *f = p->sr;
+ struct address_space *mapping = f->file->f_mapping;
+ unsigned int call = p->calls++;
+ void *entry;
+ int ret;
+
+ if (call < ARRAY_SIZE(p->before))
+ p->before[call] = mc16_sr_shadow(ref);
+ if (!call)
+ p->refs_before = refcount_read(&ref->slot->refs);
+ kunit_deactivate_static_stub(f->test, mc16_cache_load);
+ ret = mc16_cache_load(ref, folio);
+ if (call < ARRAY_SIZE(p->after)) {
+ p->after[call] = mc16_sr_shadow(ref);
+ p->errors[call] = ret;
+ }
+ if (!call && !ret) {
+ /* Native read/refault finished; this private folio is not in XA. */
+ folio_get(folio);
+ p->lost = folio;
+ p->locked = folio_test_locked(folio);
+ p->unmapped = !folio->mapping;
+ p->uptodate = folio_test_uptodate(folio);
+ p->bytes = mc16_cxr_bytes(folio, f->cache->src);
+ rcu_read_lock();
+ entry = xa_load(&mapping->i_pages, 0);
+ rcu_read_unlock();
+ p->entry_matched = entry == f->entry;
+ /* No T/XA/PTL/RCU section or local mapping survives this point. */
+ shmem_truncate_range(file_inode(f->file), 0, (loff_t)-1);
+ rcu_read_lock();
+ entry = xa_load(&mapping->i_pages, 0);
+ rcu_read_unlock();
+ p->emptied = !entry;
+ p->truncated_word = mc16_sr_shadow(ref);
+ p->refs_truncated = refcount_read(&ref->slot->refs);
+ }
+ kunit_activate_static_stub(f->test, mc16_cache_load, mc16_cxr_load);
+ return ret;
+}
+
+static void memcomp_test_cache_xarray_race_no_restore(struct kunit *test)
+{
+ struct mc16_sr_test *f = mc16_sr_begin(test);
+ struct mc16_cxr *p;
+ struct memcomp_ref stale = { };
+ struct address_space *mapping;
+ struct shmem_inode_info *info;
+ unsigned long token;
+ blkcnt_t blocks;
+ void *data, *entry;
+ bool zero, found;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ p = kunit_kzalloc(test, sizeof(*p), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, p);
+ p->sr = f;
+ KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, mc16_cxr_cleanup, p), 0);
+ mc16_cxr_current = p;
+ /* Use real eviction/publication; no PUBLISHING-delay or read-error stub. */
+ KUNIT_ASSERT_EQ(test, mc16_sr_pageout(f), 0);
+ mapping = f->file->f_mapping;
+ info = SHMEM_I(file_inode(f->file));
+ rcu_read_lock();
+ f->entry = xa_load(&mapping->i_pages, 0);
+ rcu_read_unlock();
+ KUNIT_ASSERT_TRUE(test, xa_is_value(f->entry));
+ KUNIT_ASSERT_TRUE(test, is_memcomp_entry(radix_to_swp_entry(f->entry)));
+ token = swp_offset(radix_to_swp_entry(f->entry));
+ f->cache->token = token;
+ KUNIT_ASSERT_NE(test, token, 0UL);
+ KUNIT_ASSERT_TRUE(test, mc16_get(token, &f->cache->guard));
+ KUNIT_ASSERT_TRUE(test, xa_is_value((void *)(unsigned long)
+ mc16_sr_shadow(&f->cache->guard)));
+ KUNIT_ASSERT_NE(test, mc16_sr_shadow(&f->cache->guard),
+ (u64)MC16_SHADOW_PUBLISHING);
+ KUNIT_ASSERT_EQ(test, refcount_read(&f->cache->guard.slot->refs), 2U);
+ blocks = file_inode(f->file)->i_blocks;
+ kunit_activate_static_stub(test, mc16_cache_load, mc16_cxr_load);
+ ret = mc16_sr_outer(f);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ KUNIT_ASSERT_EQ(test, p->calls, 1U);
+ KUNIT_ASSERT_NOT_NULL(test, p->lost);
+ KUNIT_ASSERT_NOT_NULL(test, f->result);
+ KUNIT_EXPECT_EQ(test, p->errors[0], 0);
+ KUNIT_EXPECT_TRUE(test, xa_is_value((void *)(unsigned long)p->before[0]));
+ KUNIT_EXPECT_EQ(test, p->after[0], 0ULL);
+ KUNIT_EXPECT_EQ(test, p->truncated_word, 0ULL);
+ KUNIT_EXPECT_TRUE(test, p->locked);
+ KUNIT_EXPECT_TRUE(test, p->unmapped);
+ KUNIT_EXPECT_TRUE(test, p->uptodate);
+ KUNIT_EXPECT_TRUE(test, p->bytes);
+ KUNIT_EXPECT_TRUE(test, p->entry_matched);
+ KUNIT_EXPECT_TRUE(test, p->emptied);
+ KUNIT_EXPECT_EQ(test, p->refs_before, 3U);
+ KUNIT_EXPECT_EQ(test, p->refs_truncated, 2U);
+ KUNIT_EXPECT_PTR_EQ(test, p->lost->mapping, NULL);
+ KUNIT_EXPECT_FALSE(test, folio_test_locked(p->lost));
+ KUNIT_EXPECT_EQ(test, folio_ref_count(p->lost), 1);
+ KUNIT_EXPECT_PTR_NE(test, p->lost, f->result);
+ KUNIT_EXPECT_TRUE(test, f->result_locked);
+ KUNIT_EXPECT_TRUE(test, f->result_uptodate);
+ KUNIT_EXPECT_TRUE(test, f->result_mapping);
+ KUNIT_EXPECT_EQ(test, folio_order(f->result), 0U);
+ data = kmap_local_folio(f->result, 0);
+ zero = !memchr_inv(data, 0, PAGE_SIZE);
+ kunmap_local(data);
+ KUNIT_EXPECT_TRUE(test, zero);
+ rcu_read_lock();
+ entry = xa_load(&mapping->i_pages, 0);
+ rcu_read_unlock();
+ KUNIT_EXPECT_PTR_EQ(test, entry, f->result);
+ KUNIT_EXPECT_EQ(test, info->alloced, 1UL);
+ KUNIT_EXPECT_EQ(test, info->swapped, 0UL);
+ KUNIT_EXPECT_EQ(test, mapping->nrpages, 1UL);
+ KUNIT_EXPECT_EQ(test, file_inode(f->file)->i_blocks, blocks);
+ KUNIT_EXPECT_EQ(test, mc16_sr_shadow(&f->cache->guard), 0ULL);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->cache->guard.slot->refs), 1U);
+ /* Old XA owner is gone; the retained real guard still pins its payload. */
+ ret = mc16_user_charge(&f->cache->guard, f->probe, NULL, GFP_KERNEL);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ folio_lock(f->probe);
+ folio_set_swapbacked(f->probe);
+ ret = mc16_cache_load(&f->cache->guard, f->probe);
+ folio_unlock(f->probe);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ KUNIT_EXPECT_EQ(test, p->calls, 2U);
+ KUNIT_EXPECT_EQ(test, p->errors[1], 0);
+ KUNIT_EXPECT_EQ(test, p->before[1], 0ULL);
+ KUNIT_EXPECT_EQ(test, p->after[1], 0ULL);
+ KUNIT_EXPECT_TRUE(test, folio_test_uptodate(f->probe));
+ KUNIT_EXPECT_TRUE(test, mc16_cxr_bytes(f->probe, f->cache->src));
+ KUNIT_EXPECT_FALSE(test, folio_test_workingset(f->probe));
+ KUNIT_EXPECT_FALSE(test, folio_test_active(f->probe));
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->cache->guard.slot->refs), 1U);
+ mc16_ref_put(&f->cache->guard);
+ KUNIT_EXPECT_PTR_EQ(test, f->cache->guard.slot, NULL);
+ found = mc16_get(token, &stale);
+ KUNIT_EXPECT_FALSE(test, found);
+ if (found)
+ mc16_ref_put(&stale);
+}
+#endif /* MC16_SHMEM_RETRY_TEST_ENABLED */
+#endif /* _MM_MEMCOMP_TESTS_CACHE_XARRAY_RACE_TEST_H */
diff --git a/mm/memcomp/tests/codec_scope_test.h b/mm/memcomp/tests/codec_scope_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..5f1732f20d50700cac08f1b64140b9e93e43e089
--- /dev/null
+++ b/mm/memcomp/tests/codec_scope_test.h
@@ -0,0 +1,94 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_CODEC_SCOPE_TEST_H
+#define _MM_MEMCOMP_TESTS_CODEC_SCOPE_TEST_H
+
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+#include <kunit/test.h>
+
+/* Include after the production async and store-precomp definitions. */
+static void memcomp_test_async_error_recycled(struct kunit *test)
+{
+ const int errors[] = { -EIO, -EHWPOISON };
+ struct memcomp_store_ctx store = {};
+ struct memcomp_async_pool *pool;
+ struct memcomp_async *batch, *got;
+ struct memcomp_async_req *r;
+ unsigned int i, wait_first;
+
+ pool = kunit_kzalloc(test, sizeof(*pool), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, pool);
+ batch = kunit_kzalloc(test, sizeof(*batch), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, batch);
+ r = &batch->reqs[0];
+ r->req = kunit_kzalloc(test, sizeof(*r->req), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, r->req);
+ r->async = batch;
+ /* Failed output must never become an accepted precomputed payload. */
+ r->buf = NULL;
+ store.precomp = r;
+ spin_lock_init(&pool->lock);
+ init_completion(&batch->done);
+ pool->nr = 1;
+ pool->free = BIT(0);
+ pool->batches[0] = batch;
+
+ for (i = 0; i < ARRAY_SIZE(errors); i++) {
+ for (wait_first = 0; wait_first < 2; wait_first++) {
+ got = memcomp_async_take(pool, 0);
+ KUNIT_ASSERT_PTR_EQ(test, got, batch);
+ KUNIT_EXPECT_EQ(test, got->nr, 0U);
+ KUNIT_EXPECT_EQ(test, pool->free, 0UL);
+ got->nr = 1;
+ r->ret = -EAGAIN;
+ r->len = 0;
+ r->req->dlen = PAGE_SIZE;
+ atomic_inc(&got->pending);
+ /* Model a waiter dropping its sentinel before completion. */
+ if (wait_first)
+ atomic_dec(&got->pending);
+
+ /* A backlog notification neither finishes nor frees a batch. */
+ memcomp_async_done(r, -EINPROGRESS);
+ KUNIT_ASSERT_EQ(test, atomic_read(&got->pending),
+ wait_first ? 1 : 2);
+ KUNIT_EXPECT_EQ(test, r->ret, -EAGAIN);
+ KUNIT_EXPECT_EQ(test, r->len, 0U);
+ KUNIT_EXPECT_FALSE(test, completion_done(&got->done));
+ KUNIT_EXPECT_EQ(test, pool->free, 0UL);
+
+ memcomp_async_done(r, errors[i]);
+ KUNIT_EXPECT_EQ(test, r->ret, errors[i]);
+ KUNIT_EXPECT_EQ(test, r->len, (unsigned int)PAGE_SIZE);
+ KUNIT_EXPECT_NULL(test, memcomp_store_precomp(&store, 0));
+ KUNIT_ASSERT_EQ(test, atomic_read(&got->pending),
+ wait_first ? 0 : 1);
+ KUNIT_EXPECT_EQ(test, completion_done(&got->done),
+ (bool)wait_first);
+ memcomp_async_give(pool, got);
+ KUNIT_EXPECT_EQ(test, atomic_read(&got->pending), 0);
+ KUNIT_ASSERT_EQ(test, pool->free, BIT(0));
+ }
+ }
+
+ /* The last error must leave the same batch available for another run. */
+ got = memcomp_async_take(pool, 0);
+ KUNIT_ASSERT_PTR_EQ(test, got, batch);
+ KUNIT_EXPECT_EQ(test, got->nr, 0U);
+ KUNIT_EXPECT_FALSE(test, completion_done(&got->done));
+ memcomp_async_give(pool, got);
+ KUNIT_EXPECT_EQ(test, pool->free, BIT(0));
+}
+
+static struct kunit_case memcomp_codec_scope_test_cases[] = {
+ KUNIT_CASE(memcomp_test_async_error_recycled),
+ {}
+};
+
+static struct kunit_suite memcomp_codec_scope_test_suite = {
+ .name = "memcomp-codec-scope",
+ .test_cases = memcomp_codec_scope_test_cases,
+};
+
+kunit_test_suite(memcomp_codec_scope_test_suite);
+#endif /* CONFIG_MEMCOMP_KUNIT_TEST */
+#endif /* _MM_MEMCOMP_TESTS_CODEC_SCOPE_TEST_H */
diff --git a/mm/memcomp/tests/consumers_test.h b/mm/memcomp/tests/consumers_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..e7dbf305f890352ada53cc01ea20aa6fa6b60c69
--- /dev/null
+++ b/mm/memcomp/tests/consumers_test.h
@@ -0,0 +1,478 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_CONSUMERS_TEST_H
+#define _MM_MEMCOMP_TESTS_CONSUMERS_TEST_H
+
+#include <kunit/test.h>
+#include "../consumers.h"
+
+/*
+ * Include in suite.h after the storage probe implementation. Register
+ * the cases in the enclosing suite. Load-touch assertions exercise the shared
+ * load helper used by the public clustered-slot dispatch.
+ */
+#ifdef CONFIG_MEMCG
+struct mc16_consumer_test {
+ struct kunit *test;
+ struct memcomp_kunit_objcg model;
+ struct mc16_storage_fixture probe;
+ struct folio *account;
+ struct folio *dst;
+ struct folio *large;
+ struct memcomp_ref owners[3];
+ struct memcomp_ref guard;
+ unsigned long tokens[3];
+ unsigned int pte_refs[3];
+ unsigned int zero_pte;
+ unsigned int zero_cache;
+ s64 stored_before;
+ s64 compressed_before;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ s64 pending_before;
+#endif
+};
+
+/* No test-task context, folio lock, T or local mapping survives an actor. */
+static void mc16_consumer_test_cleanup(void *data)
+{
+ struct mc16_consumer_test *f = data;
+ unsigned int i;
+
+ f->probe.fail_at = 0;
+ f->probe.fail_descriptor = false;
+ f->probe.allocation_window = NULL;
+ f->model.fail_at = 0;
+ mc16_cache_abort(&f->guard);
+ for (i = 0; i < ARRAY_SIZE(f->owners); i++) {
+ while (f->pte_refs[i]) {
+ mc16_pte_put_nr(f->tokens[i], 1);
+ f->pte_refs[i]--;
+ }
+ if (f->owners[i].slot)
+ mc16_ref_put(&f->owners[i]);
+ }
+ mc16_zero_pte_put(f->zero_pte);
+ mc16_zero_cache_put(f->zero_cache);
+ mc16_test_drain_segments(&f->probe);
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_TRUE(f->test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(f->test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_EQ(f->test, atomic_long_read(&f->model.charged), 0L);
+ KUNIT_EXPECT_EQ(f->test, atomic_read(&f->model.get_calls),
+ atomic_read(&f->model.put_calls));
+ KUNIT_EXPECT_EQ(f->test, atomic64_read(&memcomp_stored_bytes),
+ f->stored_before);
+ KUNIT_EXPECT_EQ(f->test, atomic64_read(&memcomp_compressed_bytes),
+ f->compressed_before);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ KUNIT_EXPECT_EQ(f->test, atomic64_read(&memcomp_pending_pages),
+ f->pending_before);
+#endif
+ if (rcu_access_pointer(mc16_storage_fixture) == &f->probe)
+ rcu_assign_pointer(mc16_storage_fixture, NULL);
+ if (rcu_access_pointer(memcomp_kunit_objcg) == &f->model)
+ rcu_assign_pointer(memcomp_kunit_objcg, NULL);
+ synchronize_rcu();
+ if (f->large) {
+ KUNIT_EXPECT_FALSE(f->test, folio_test_locked(f->large));
+ folio_put(f->large);
+ }
+ if (f->dst) {
+ KUNIT_EXPECT_FALSE(f->test, folio_test_locked(f->dst));
+ folio_put(f->dst);
+ }
+ if (f->account) {
+ KUNIT_EXPECT_FALSE(f->test, folio_test_locked(f->account));
+ KUNIT_EXPECT_EQ(f->test, folio_ref_count(f->account), 1);
+ folio_put(f->account);
+ }
+}
+
+/* Unlike the older objcg fixture, account/dst remain unlocked on return. */
+static struct mc16_consumer_test *mc16_consumer_test_begin(struct kunit *test)
+{
+ struct mc16_consumer_test *f;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return NULL;
+ }
+ rcu_barrier();
+ if (rcu_access_pointer(memcomp_kunit_objcg) ||
+ rcu_access_pointer(mc16_storage_fixture) || !xa_empty(&mc16_clusters)) {
+ KUNIT_FAIL(test, "consumer fixture requires an idle registry/model");
+ return NULL;
+ }
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->test = test;
+ f->stored_before = atomic64_read(&memcomp_stored_bytes);
+ f->compressed_before = atomic64_read(&memcomp_compressed_bytes);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ f->pending_before = atomic64_read(&memcomp_pending_pages);
+#endif
+ f->model.task = current;
+ f->model.error = -ENOMEM;
+ f->probe.task = current;
+ if (kunit_add_action_or_reset(test, mc16_consumer_test_cleanup, f))
+ return NULL;
+ f->account = folio_alloc(GFP_KERNEL, 0);
+ f->dst = folio_alloc(GFP_KERNEL, 0);
+ if (!f->account || !f->dst)
+ return NULL;
+ /* Native eviction uses source-folio charge, not the objcg model. */
+ if (mem_cgroup_charge(f->account, NULL, GFP_KERNEL)) {
+ KUNIT_FAIL(test, "consumer source memcg charge failed");
+ return NULL;
+ }
+ f->model.folio = f->account;
+ rcu_assign_pointer(memcomp_kunit_objcg, &f->model);
+ rcu_assign_pointer(mc16_storage_fixture, &f->probe);
+ return f;
+}
+
+static int mc16_consumer_test_reserve(struct mc16_consumer_test *f,
+ unsigned int index)
+{
+ struct mc16_record *record;
+ int ret;
+
+ record = mc16_record_alloc(f->account, 1, false);
+ if (IS_ERR(record))
+ return PTR_ERR(record);
+ ret = mc16_reserve(record, 1, MC16_KEY_ANON, &f->owners[index]);
+ if (ret) {
+ mc16_record_abort(record);
+ return ret;
+ }
+ mc16_account_reservation(&f->owners[index]);
+ f->tokens[index] = f->owners[index].token;
+ return 0;
+}
+
+static int mc16_consumer_test_store(struct mc16_consumer_test *f,
+ const void *src)
+{
+ int ret = mc16_cache_store(src, f->account, &f->owners[0]);
+
+ if (!ret)
+ f->tokens[0] = f->owners[0].token;
+ return ret;
+}
+
+/* Every lock/mapping is released before returning to KUnit assertions. */
+static int mc16_consumer_test_load(struct memcomp_ref *ref, struct folio *folio,
+ bool zero)
+{
+ int ret;
+
+ folio_lock(folio);
+ ret = zero ? mc16_cache_load_zero(folio) : mc16_cache_load(ref, folio);
+ folio_unlock(folio);
+ return ret;
+}
+
+static void mc16_consumer_test_fill(struct folio *folio, u8 byte)
+{
+ void *dst = kmap_local_folio(folio, 0);
+
+ memset(dst, byte, PAGE_SIZE);
+ kunmap_local(dst);
+ folio_clear_uptodate(folio);
+}
+
+static bool mc16_consumer_test_bytes(struct folio *folio, u8 byte)
+{
+ void *dst = kmap_local_folio(folio, 0);
+ bool same = !memchr_inv(dst, byte, PAGE_SIZE);
+
+ kunmap_local(dst);
+ return same;
+}
+
+static bool mc16_consumer_test_matches(struct folio *folio, const void *src)
+{
+ void *dst = kmap_local_folio(folio, 0);
+ bool same = !memcmp(dst, src, PAGE_SIZE);
+
+ kunmap_local(dst);
+ return same;
+}
+
+static bool mc16_consumer_test_touch(struct memcomp_ref *ref, bool set)
+{
+ struct mc16_view view;
+ bool touched = false;
+
+ mutex_lock(memcomp_lock_of(ref->token));
+ if (mc16_view_locked(ref, &view)) {
+ if (set)
+ *view.flags |= MC16_F_REFRESH_PENDING;
+ touched = !!(*view.flags & MC16_F_REFERENCED);
+ }
+ mutex_unlock(memcomp_lock_of(ref->token));
+ return touched;
+}
+
+static void memcomp_test_consumer_pte_shared(struct kunit *test)
+{
+ struct mc16_consumer_test *f = mc16_consumer_test_begin(test);
+ struct memcomp_ref *owner;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_consumer_test_reserve(f, 0), 0);
+ owner = &f->owners[0];
+ KUNIT_ASSERT_TRUE(test, mc16_pte_get(f->tokens[0]));
+ f->pte_refs[0]++;
+ KUNIT_ASSERT_TRUE(test, mc16_pte_get(f->tokens[0]));
+ f->pte_refs[0]++;
+ KUNIT_EXPECT_EQ(test, refcount_read(&owner->slot->refs), 3U);
+ mc16_pte_put_nr(f->tokens[0], 1);
+ f->pte_refs[0]--;
+ KUNIT_EXPECT_EQ(test, refcount_read(&owner->slot->refs), 2U);
+ mc16_pte_put_nr(f->tokens[0], 1);
+ f->pte_refs[0]--;
+ KUNIT_EXPECT_EQ(test, refcount_read(&owner->slot->refs), 1U);
+ KUNIT_EXPECT_TRUE(test, owner->slot->flags & MC16_F_OWNER);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->account), 2);
+}
+
+static void memcomp_test_consumer_pte_bounds(struct kunit *test)
+{
+ struct mc16_consumer_test *f = mc16_consumer_test_begin(test);
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (i = 0; i < ARRAY_SIZE(f->owners); i++) {
+ KUNIT_ASSERT_EQ(test, mc16_consumer_test_reserve(f, i), 0);
+ KUNIT_ASSERT_TRUE(test, mc16_pte_get(f->tokens[i]));
+ f->pte_refs[i]++;
+ KUNIT_ASSERT_EQ(test, f->tokens[i], f->tokens[0] + i);
+ }
+ mc16_pte_put_nr(f->tokens[0], 0);
+ mc16_pte_put_nr(U32_MAX, 2);
+ mc16_pte_put_nr(U32_MAX, UINT_MAX);
+ mc16_pte_put_nr(ULONG_MAX, 0);
+#if BITS_PER_LONG > 32
+ KUNIT_EXPECT_FALSE(test, mc16_pte_get(BIT_ULL(32) | f->tokens[0]));
+ mc16_pte_put_nr(BIT_ULL(32) | f->tokens[0], 3);
+ mc16_cache_drop(BIT_ULL(32) | f->tokens[0]);
+ KUNIT_EXPECT_FALSE(test, mc16_cache_pin(BIT_ULL(32) | f->tokens[0],
+ &f->guard));
+#endif
+ for (i = 0; i < ARRAY_SIZE(f->owners); i++)
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->owners[i].slot->refs), 2U);
+ mc16_pte_put_nr(f->tokens[0], ARRAY_SIZE(f->owners));
+ for (i = 0; i < ARRAY_SIZE(f->owners); i++) {
+ f->pte_refs[i]--;
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->owners[i].slot->refs), 1U);
+ }
+}
+
+static void mc16_consumer_test_roundtrip(struct kunit *test, bool samefill,
+ bool commit)
+{
+ const unsigned long pattern = 0x9876543210fedcbaUL;
+ struct mc16_consumer_test *f = mc16_consumer_test_begin(test);
+ struct mc16_slot *slot;
+ unsigned int i;
+ u8 *src;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ src = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ if (samefill)
+ memset_l((unsigned long *)src, pattern,
+ PAGE_SIZE / sizeof(unsigned long));
+ else
+ for (i = 0; i < PAGE_SIZE; i++)
+ src[i] = i % 53;
+ KUNIT_ASSERT_EQ(test, mc16_consumer_test_store(f, src), 0);
+ slot = f->owners[0].slot;
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(slot->flags),
+ (unsigned int)(samefill ? MC16_SAMEFILL : MC16_STORED));
+ if (commit) {
+ KUNIT_ASSERT_TRUE(test, mc16_pte_get(f->tokens[0]));
+ f->pte_refs[0]++;
+ }
+ KUNIT_ASSERT_TRUE(test, mc16_cache_pin(f->tokens[0], &f->guard));
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.slot, slot);
+ mc16_consumer_test_fill(f->dst, 0xa5);
+ KUNIT_ASSERT_EQ(test, mc16_consumer_test_load(&f->guard, f->dst, false), 0);
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.slot, slot);
+ KUNIT_EXPECT_TRUE(test, folio_test_uptodate(f->dst));
+ KUNIT_EXPECT_TRUE(test, mc16_consumer_test_matches(f->dst, src));
+ KUNIT_EXPECT_TRUE(test, mc16_consumer_test_touch(&f->guard, false));
+ if (!commit) {
+ mc16_cache_abort(&f->guard);
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.slot, NULL);
+ KUNIT_EXPECT_EQ(test, refcount_read(&slot->refs), 1U);
+ /* Abort retained the cache owner and its complete payload. */
+ KUNIT_ASSERT_TRUE(test, mc16_cache_pin(f->tokens[0], &f->guard));
+ mc16_consumer_test_fill(f->dst, 0x5a);
+ KUNIT_ASSERT_EQ(test,
+ mc16_consumer_test_load(&f->guard, f->dst, false), 0);
+ KUNIT_EXPECT_TRUE(test, mc16_consumer_test_matches(f->dst, src));
+ mc16_cache_abort(&f->guard);
+ mc16_cache_drop(f->tokens[0]);
+ f->owners[0] = (struct memcomp_ref) { };
+ } else {
+ KUNIT_EXPECT_EQ(test, refcount_read(&slot->refs), 3U);
+ /* Model the winner of the expected-shadow xarray replacement. */
+ mc16_cache_commit(&f->guard);
+ f->owners[0] = (struct memcomp_ref) { };
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.slot, NULL);
+ KUNIT_ASSERT_TRUE(test, mc16_get(f->tokens[0], &f->guard));
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guard.slot->refs), 2U);
+ mc16_cache_abort(&f->guard);
+ /* Only the shared PTE remains; its put releases the real zs payload. */
+ mc16_pte_put_nr(f->tokens[0], 1);
+ f->pte_refs[0]--;
+ }
+ KUNIT_EXPECT_FALSE(test, mc16_get(f->tokens[0], &f->guard));
+ /* Already-absent numeric tokens acquire no guard and consume nothing. */
+ mc16_cache_drop(f->tokens[0]);
+ mc16_pte_put_nr(f->tokens[0], 1);
+ /* Absent tokens do not imply the current header has been reclaimed. */
+ KUNIT_EXPECT_FALSE(test, xa_empty(&mc16_clusters));
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->model.charged), 0L);
+}
+
+static void memcomp_test_consumer_cache_abort(struct kunit *test)
+{
+ mc16_consumer_test_roundtrip(test, false, false);
+}
+
+static void memcomp_test_consumer_cache_samefill(struct kunit *test)
+{
+ mc16_consumer_test_roundtrip(test, true, false);
+}
+
+static void memcomp_test_consumer_cache_commit(struct kunit *test)
+{
+ mc16_consumer_test_roundtrip(test, false, true);
+}
+
+static void memcomp_test_consumer_cache_retry(struct kunit *test)
+{
+ const unsigned long pattern = 0x0123456789abcdefUL;
+ struct mc16_consumer_test *f = mc16_consumer_test_begin(test);
+ unsigned long *src;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ src = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ memset_l(src, pattern, PAGE_SIZE / sizeof(*src));
+ KUNIT_ASSERT_EQ(test, mc16_consumer_test_store(f, src), 0);
+ KUNIT_ASSERT_TRUE(test, mc16_cache_pin(f->tokens[0], &f->guard));
+ mc16_consumer_test_touch(&f->guard, true);
+ mc16_consumer_test_fill(f->dst, 0x5a);
+ KUNIT_EXPECT_EQ(test, mc16_consumer_test_load(&f->guard, f->dst, false),
+ -EAGAIN);
+ KUNIT_EXPECT_FALSE(test, folio_test_uptodate(f->dst));
+ KUNIT_EXPECT_TRUE(test, mc16_consumer_test_bytes(f->dst, 0x5a));
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guard.slot->refs), 2U);
+ mc16_cache_abort(&f->guard);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->owners[0].slot->refs), 1U);
+}
+
+static void memcomp_test_consumer_zero_domains(struct kunit *test)
+{
+ struct mc16_consumer_test *f = mc16_consumer_test_begin(test);
+ struct mc16_zero_owners before, after;
+ struct mc16_zero_diagnostics events, end;
+ unsigned int allocations;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_zero_owners_read(&before);
+ mc16_zero_diagnostics_read(&events);
+ allocations = f->probe.allocations;
+ KUNIT_ASSERT_TRUE(test, mc16_zero_pte_init(2));
+ f->zero_pte += 2;
+ KUNIT_ASSERT_TRUE(test, mc16_pte_get(0));
+ f->zero_pte++;
+ KUNIT_ASSERT_TRUE(test, mc16_zero_cache_get(2));
+ f->zero_cache += 2;
+ KUNIT_EXPECT_FALSE(test, mc16_cache_pin(0, &f->guard));
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.slot, NULL);
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.cluster, NULL);
+ mc16_cache_abort(&f->guard);
+ mc16_cache_commit(&f->guard);
+ mc16_pte_put_nr(0, 0);
+ mc16_consumer_test_fill(f->dst, 0x5a);
+ KUNIT_EXPECT_EQ(test, mc16_consumer_test_load(NULL, f->dst, true), 0);
+ KUNIT_EXPECT_TRUE(test, folio_test_uptodate(f->dst));
+ KUNIT_EXPECT_TRUE(test, mc16_consumer_test_bytes(f->dst, 0));
+ mc16_zero_owners_read(&after);
+ KUNIT_EXPECT_EQ(test, after.pte_pages, before.pte_pages + 3);
+ KUNIT_EXPECT_EQ(test, after.cache_pages, before.cache_pages + 2);
+ /* A failed xa replacement consumes neither zero owner. */
+ KUNIT_EXPECT_EQ(test, mc16_consumer_test_load(NULL, f->dst, true), 0);
+ mc16_pte_put_nr(0, 2);
+ f->zero_pte -= 2;
+ mc16_zero_owners_read(&after);
+ KUNIT_EXPECT_EQ(test, after.pte_pages, before.pte_pages + 1);
+ KUNIT_EXPECT_EQ(test, after.cache_pages, before.cache_pages + 2);
+ /* Replacement and truncate each explicitly drop one cache owner. */
+ mc16_cache_drop(0);
+ f->zero_cache--;
+ mc16_cache_drop(0);
+ f->zero_cache--;
+ mc16_zero_owners_read(&after);
+ KUNIT_EXPECT_EQ(test, after.pte_pages, before.pte_pages + 1);
+ KUNIT_EXPECT_EQ(test, after.cache_pages, before.cache_pages);
+ mc16_pte_put_nr(0, 1);
+ f->zero_pte--;
+ mc16_zero_owners_read(&after);
+ KUNIT_EXPECT_EQ(test, after.pte_pages, before.pte_pages);
+ KUNIT_EXPECT_EQ(test, after.cache_pages, before.cache_pages);
+ KUNIT_EXPECT_EQ(test, f->probe.allocations, allocations);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->model.charge_calls), 0);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ mc16_zero_diagnostics_read(&end);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ KUNIT_EXPECT_EQ(test, end.calls[MC16_ZERO_PTE_GET] -
+ events.calls[MC16_ZERO_PTE_GET], 1ULL);
+ KUNIT_EXPECT_EQ(test, end.pages[MC16_ZERO_PTE_PUT] -
+ events.pages[MC16_ZERO_PTE_PUT], 3ULL);
+ KUNIT_EXPECT_EQ(test, end.pages[MC16_ZERO_CACHE_PUT] -
+ events.pages[MC16_ZERO_CACHE_PUT], 2ULL);
+#endif
+}
+
+static void memcomp_test_consumer_load_bounds(struct kunit *test)
+{
+ struct mc16_consumer_test *f = mc16_consumer_test_begin(test);
+ const unsigned long pattern = 0x1234567812345678UL;
+ unsigned long *src;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_EXPECT_FALSE(test, mc16_cache_pin(0, NULL));
+ KUNIT_EXPECT_FALSE(test, mc16_cache_pin(ULONG_MAX, &f->guard));
+ KUNIT_EXPECT_EQ(test, mc16_cache_load(NULL, f->dst), -ENOENT);
+ KUNIT_EXPECT_EQ(test, mc16_cache_load(&f->guard, f->dst), -ENOENT);
+ KUNIT_EXPECT_EQ(test, mc16_cache_load_zero(NULL), -EINVAL);
+ mc16_consumer_test_fill(f->dst, 0xa5);
+ KUNIT_EXPECT_EQ(test, mc16_cache_load_zero(f->dst), -EINVAL);
+ KUNIT_EXPECT_TRUE(test, mc16_consumer_test_bytes(f->dst, 0xa5));
+ f->large = folio_alloc(GFP_KERNEL, 1);
+ KUNIT_ASSERT_NOT_NULL(test, f->large);
+ KUNIT_EXPECT_EQ(test, mc16_consumer_test_load(NULL, f->large, true), -EINVAL);
+ src = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ memset_l(src, pattern, PAGE_SIZE / sizeof(*src));
+ KUNIT_ASSERT_EQ(test, mc16_consumer_test_store(f, src), 0);
+ KUNIT_ASSERT_TRUE(test, mc16_cache_pin(f->tokens[0], &f->guard));
+ KUNIT_EXPECT_EQ(test, mc16_cache_load(&f->guard, NULL), -EINVAL);
+ KUNIT_EXPECT_EQ(test, mc16_cache_load(&f->guard, f->dst), -EINVAL);
+ KUNIT_EXPECT_EQ(test, mc16_consumer_test_load(&f->guard, f->large, false),
+ -EINVAL);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guard.slot->refs), 2U);
+ mc16_cache_abort(&f->guard);
+ mc16_cache_abort(&f->guard);
+ mc16_cache_commit(&f->guard);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->owners[0].slot->refs), 1U);
+}
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_CONSUMERS_TEST_H */
diff --git a/mm/memcomp/tests/core_test.h b/mm/memcomp/tests/core_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..86764e017bafdef358ed529736d0eb3b25ddab3c
--- /dev/null
+++ b/mm/memcomp/tests/core_test.h
@@ -0,0 +1,119 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_CORE_TEST_H
+#define _MM_MEMCOMP_TESTS_CORE_TEST_H
+
+#include "test_api.h"
+
+#ifdef CONFIG_MEMCG
+struct memcomp_test_owner {
+ void *record;
+ unsigned int remaining;
+};
+
+static void memcomp_test_cleanup(void *data)
+{
+ struct memcomp_test_owner *owner = data;
+
+ while (owner->remaining) {
+ owner->remaining--;
+ memcomp_test_record_put(owner->record);
+ }
+ owner->record = NULL;
+}
+
+static struct memcomp_test_owner *
+memcomp_test_new(struct kunit *test, struct folio *folio,
+ unsigned int nr_pages)
+{
+ struct memcomp_test_owner *owner;
+ int ret;
+
+ owner = kunit_kzalloc(test, sizeof(*owner), GFP_KERNEL);
+ if (!owner)
+ return NULL;
+ owner->record = memcomp_test_record_alloc(folio, nr_pages);
+ if (IS_ERR(owner->record)) {
+ KUNIT_FAIL(test, "record allocation failed: %ld",
+ PTR_ERR(owner->record));
+ return NULL;
+ }
+ owner->remaining = nr_pages;
+ ret = kunit_add_action_or_reset(test, memcomp_test_cleanup,
+ owner);
+ return ret ? NULL : owner;
+}
+
+static void memcomp_test_record_members(struct kunit *test)
+{
+ struct memcomp_test_objcg_fixture *f;
+ struct memcomp_test_owner *owner;
+ struct memcomp_record_info info;
+ spinlock_t lock; /* Models an atomic final-put caller. */
+ unsigned long flags;
+
+ f = memcomp_test_objcg_begin(test, 2);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ owner = memcomp_test_new(test, f->folio, 2);
+ KUNIT_ASSERT_NOT_NULL(test, owner);
+ info = memcomp_test_record_info(owner->record);
+ KUNIT_EXPECT_EQ(test, info.members, 2U);
+ KUNIT_EXPECT_EQ(test, info.charged_bytes, info.allocated_bytes);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->model.charged),
+ (long)info.charged_bytes);
+
+ owner->remaining--;
+ memcomp_test_record_put(owner->record);
+ info = memcomp_test_record_info(owner->record);
+ KUNIT_EXPECT_EQ(test, info.members, 1U);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->model.put_calls), 0);
+
+ /* Last member release must be valid in the atomic final-put context. */
+ spin_lock_init(&lock);
+ spin_lock_irqsave(&lock, flags);
+ owner->remaining--;
+ memcomp_test_record_put(owner->record);
+ owner->record = NULL;
+ spin_unlock_irqrestore(&lock, flags);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->model.charged), 0L);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->model.get_calls), 1);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->model.put_calls), 1);
+}
+
+static void memcomp_test_record_abort(struct kunit *test)
+{
+ struct memcomp_test_objcg_fixture *f;
+ struct memcomp_test_owner *owner;
+
+ f = memcomp_test_objcg_begin(test, 2);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ owner = memcomp_test_new(test, f->folio, 2);
+ KUNIT_ASSERT_NOT_NULL(test, owner);
+ owner->remaining = 0;
+ memcomp_test_record_handle_abort(owner->record);
+ owner->record = NULL;
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->model.charged), 0L);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->model.get_calls), 1);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->model.put_calls), 1);
+}
+
+static void memcomp_test_record_charge_failure(struct kunit *test)
+{
+ struct memcomp_test_objcg_fixture *f;
+ void *record;
+
+ f = memcomp_test_objcg_begin(test, 1);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->model.fail_at = 1;
+ record = memcomp_test_record_alloc(f->folio, 1);
+ if (!IS_ERR(record)) {
+ memcomp_test_record_handle_abort(record);
+ KUNIT_FAIL(test, "record metadata charge unexpectedly succeeded");
+ return;
+ }
+ KUNIT_EXPECT_EQ(test, PTR_ERR(record), (long)-ENOMEM);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->model.charged), 0L);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->model.get_calls), 1);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->model.put_calls), 1);
+}
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_CORE_TEST_H */
diff --git a/mm/memcomp/tests/defrag_test.h b/mm/memcomp/tests/defrag_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..4d693944f800f0f1d724e96e1c676e11eea03f16
--- /dev/null
+++ b/mm/memcomp/tests/defrag_test.h
@@ -0,0 +1,601 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_DEFRAG_TEST_H
+#define _MM_MEMCOMP_TESTS_DEFRAG_TEST_H
+
+#include <kunit/static_stub.h>
+#include "../defrag.h"
+
+/*
+ * Include after consumers_test.h for its unlocked accounting/probe fixture.
+ * Backing IO and slot counts are modeled. Encodings, zs objects, byte copies,
+ * decompression, guards, charging and the scanner are real. No device is used.
+ */
+#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP)
+#define MC16_DEFRAG_TEST_SLOTS (MEMCOMP_WRITEBACK_DEFRAG_SCAN + 4)
+
+struct mc16_defrag_test {
+ struct mc16_consumer_test *base;
+ struct memcomp_ref refs[MC16_DEFRAG_TEST_SLOTS];
+ struct memcomp_ref guard;
+ struct memcomp_kunit_async_probe payload;
+ unsigned int nr;
+ u32 markers[2];
+ u8 *image;
+ u8 *original;
+ u32 bytes;
+ int count;
+ int read_error;
+ bool fail_buffer;
+ bool fail_alloc;
+ bool fail_charge;
+ bool drop_owner;
+ unsigned int race;
+ unsigned int reads;
+ unsigned int copies;
+ bool callback_under_t;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ long swapped_before;
+#endif
+};
+
+static struct mc16_defrag_test *mc16_defrag_test_current;
+
+static void mc16_defrag_test_cleanup(void *data)
+{
+ struct mc16_defrag_test *f = data;
+ unsigned int i;
+
+ f->base->model.fail_at = 0;
+ if (mc16_defrag_test_current == f)
+ mc16_defrag_test_current = NULL;
+ if (f->guard.slot)
+ mc16_ref_put(&f->guard);
+ for (i = 0; i < f->nr; i++)
+ if (f->refs[i].slot)
+ mc16_ref_put(&f->refs[i]);
+ for (i = 0; i < ARRAY_SIZE(f->markers); i++)
+ if (f->markers[i])
+ xa_erase_irq(&mc16_clusters, f->markers[i]);
+ rcu_barrier();
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ KUNIT_EXPECT_EQ(f->base->test, atomic_long_read(&memcomp_swapped_entries),
+ f->swapped_before);
+#endif
+ if (rcu_access_pointer(memcomp_kunit_async_probe) == &f->payload) {
+ rcu_assign_pointer(memcomp_kunit_async_probe, NULL);
+ synchronize_rcu();
+ }
+}
+
+static struct mc16_defrag_test *mc16_defrag_test_begin(struct kunit *test)
+{
+ struct mc16_consumer_test *base = mc16_consumer_test_begin(test);
+ struct mc16_defrag_test *f;
+ unsigned long handle = 0;
+ unsigned int i;
+ void *mapped;
+ u8 pool = 0;
+ int ret;
+
+ if (!base)
+ return NULL;
+ if (rcu_access_pointer(memcomp_kunit_async_probe)) {
+ KUNIT_FAIL(test, "defrag fixture requires an idle payload probe");
+ return NULL;
+ }
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->base = base;
+ f->count = 1;
+ f->payload.task = current;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ f->swapped_before = atomic_long_read(&memcomp_swapped_entries);
+#endif
+ if (kunit_add_action_or_reset(test, mc16_defrag_test_cleanup, f))
+ return NULL;
+ f->image = kunit_kzalloc(test, PAGE_SIZE, GFP_KERNEL);
+ f->original = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ if (!f->image || !f->original)
+ return NULL;
+ for (i = 0; i < PAGE_SIZE; i++)
+ f->original[i] = i % 53;
+ ret = memcomp_prepare_payload(f->original, PAGE_SIZE, &f->bytes, &handle,
+ &pool, MEMCOMP_PAYLOAD_MUST_SUCCEED, false);
+ if (ret)
+ return NULL;
+ if (!f->bytes || f->bytes >= PAGE_SIZE - 38 || pool) {
+ if (f->bytes)
+ memcomp_zs_free(pool, handle);
+ return NULL;
+ }
+ mapped = zs_map_object(memcomp_pools[0], handle, ZS_MM_RO);
+ if (mapped) {
+ memcpy(f->image + 37, mapped, f->bytes);
+ zs_unmap_object(memcomp_pools[0], handle);
+ }
+ memcomp_zs_free(0, handle);
+ return mapped ? f : NULL;
+}
+
+/* 0 samefill/noncandidate, 1 stable packed, 2 ordinary, 3 attached packed. */
+static int mc16_defrag_test_insert(struct mc16_defrag_test *f, unsigned int mode)
+{
+ struct mc16_initial_payload initial = { .word = 0x12345678 };
+ struct mc16_consumer_test *b = f->base;
+ struct memcomp_ref *ref;
+ struct mc16_view view;
+ u64 word;
+ u16 length;
+ int ret;
+
+ if (f->nr == ARRAY_SIZE(f->refs))
+ return -ENOSPC;
+ ref = &f->refs[f->nr];
+ if (mode == 3) {
+ ret = mc16_consumer_test_reserve(b, 0);
+ if (ret)
+ return ret;
+ *ref = b->owners[0];
+ b->owners[0] = (struct memcomp_ref) { };
+ } else {
+ ret = mc16_reserve_cache(&b->model.objcg, folio_nid(b->account),
+ &initial, 1, ref);
+ if (ret)
+ return ret;
+ mc16_cache_publish(ref, 0);
+ }
+ f->nr++;
+ if (!mode)
+ return 0;
+ if (!mc16_swap_encode(0, 0x567 + f->nr, mode == 2 ? 0 : 37,
+ f->bytes, mode != 2, &word, &length))
+ return -EINVAL;
+ mutex_lock(memcomp_lock_of(ref->token));
+ if (!mc16_view_locked(ref, &view)) {
+ ret = -EIO;
+ goto unlock;
+ }
+ mc16_account_state(ref, false);
+ if (*view.flags & MC16_F_RESIDENT) {
+ folio_put(view.member->owner_folio);
+ memcomp_debug_dec(&memcomp_pending_pages);
+ }
+ *view.word = word;
+ *view.length = length;
+ *view.flags = MC16_SWAPPED | (*view.flags & MC16_F_OWNER) |
+ MC16_F_REFERENCED | MC16_F_INCOMPRESSIBLE;
+ memcomp_compressed_bytes_add(f->bytes);
+ memcomp_debug_long_inc(&memcomp_swapped_entries);
+ mc16_account_state(ref, true);
+ ret = 0;
+unlock:
+ mutex_unlock(memcomp_lock_of(ref->token));
+ return ret;
+}
+
+static int mc16_defrag_test_count(swp_entry_t swap)
+{
+ return swap.val ? mc16_defrag_test_current->count : 0;
+}
+
+static int mc16_defrag_test_read(swp_entry_t swap, void *dst)
+{
+ struct mc16_defrag_test *f = mc16_defrag_test_current;
+
+ f->reads++;
+ if (!swap.val)
+ return -EIO;
+ if (f->guard.slot && mutex_is_locked(memcomp_lock_of(f->guard.token)))
+ f->callback_under_t = true;
+ if (f->read_error)
+ return f->read_error;
+ memcpy(dst, f->image, PAGE_SIZE);
+ return 0;
+}
+
+static const struct memcomp_defrag_ops mc16_defrag_test_ops = {
+ .slot_count = mc16_defrag_test_count,
+ .read_page = mc16_defrag_test_read,
+};
+
+static void *mc16_defrag_test_buffer(void)
+{
+ if (mc16_defrag_test_current->fail_buffer)
+ return NULL;
+ return kmalloc(PAGE_SIZE, GFP_NOWAIT | __GFP_NOWARN);
+}
+
+static int mc16_defrag_test_copy(const void *src, unsigned int bytes,
+ struct mc16_payload *payload)
+{
+ struct mc16_defrag_test *f = mc16_defrag_test_current;
+ struct memcomp_store_ctx store = { .kunit_async_probe = &f->payload };
+ unsigned int copied = 0;
+ unsigned long handle;
+ int ret;
+
+ f->copies++;
+ if (f->fail_alloc)
+ return -ENOMEM;
+ ret = memcomp_alloc_raw_payload(0, src, bytes, &copied, &handle, false,
+ f->guard.slot ? &store : NULL);
+ if (ret)
+ return ret;
+ *payload = (struct mc16_payload) { .word = handle, .bytes = copied };
+ if (f->fail_charge)
+ f->base->model.fail_at = atomic_read(&f->base->model.charge_calls) + 1;
+ if (f->race) {
+ struct mc16_swap_fields fields;
+ struct mc16_view view;
+ u64 word;
+ u16 length;
+
+ mutex_lock(memcomp_lock_of(f->guard.token));
+ ret = mc16_view_locked(&f->guard, &view) &&
+ mc16_swap_decode(*view.word, *view.length, &fields) ? 0 : -EIO;
+ if (!ret) {
+ if (f->race == 1)
+ fields.byte_offset++;
+ else
+ fields.length++;
+ if (!mc16_swap_encode(fields.type, fields.swap_offset,
+ fields.byte_offset, fields.length,
+ fields.packed, &word, &length)) {
+ ret = -EINVAL;
+ } else {
+ *view.word = word;
+ *view.length = length;
+ if (f->race == 2)
+ memcomp_compressed_bytes_add(1);
+ }
+ }
+ mutex_unlock(memcomp_lock_of(f->guard.token));
+ if (ret)
+ return ret;
+ }
+ if (f->drop_owner) {
+ struct memcomp_ref owner;
+ bool dropped = false;
+
+ folio_lock(f->base->account);
+ if (mc16_get(f->guard.token, &owner))
+ dropped = mc16_owner_drop_folio_held(&owner,
+ f->base->account, 1, 0);
+ folio_unlock(f->base->account);
+ if (!dropped)
+ return -EIO;
+ f->refs[0] = (struct memcomp_ref) { };
+ }
+ return 0;
+}
+
+/* Stubs are disabled, and every T/folio lock is gone, before any assertion. */
+static int mc16_defrag_test_run(struct mc16_defrag_test *f, u64 *cursor)
+{
+ struct kunit *test = f->base->test;
+ int ret;
+
+ mc16_defrag_test_current = f;
+ if (f->guard.slot)
+ rcu_assign_pointer(memcomp_kunit_async_probe, &f->payload);
+ kunit_activate_static_stub(test, mc16_defrag_buffer_alloc,
+ mc16_defrag_test_buffer);
+ kunit_activate_static_stub(test, mc16_defrag_copy_payload,
+ mc16_defrag_test_copy);
+ ret = cursor ? mc16_defrag_scan(cursor, &mc16_defrag_test_ops) :
+ mc16_defrag_entry(&f->guard, &mc16_defrag_test_ops);
+ kunit_deactivate_static_stub(test, mc16_defrag_copy_payload);
+ kunit_deactivate_static_stub(test, mc16_defrag_buffer_alloc);
+ mc16_defrag_test_current = NULL;
+ return ret;
+}
+
+static unsigned int mc16_defrag_test_state(struct memcomp_ref *ref,
+ u64 *word, u16 *length, bool *attached)
+{
+ struct mc16_view view;
+ unsigned int state = MC16_EMPTY;
+
+ mutex_lock(memcomp_lock_of(ref->token));
+ if (mc16_view_locked(ref, &view)) {
+ state = mc16_flags_state(*view.flags);
+ if (word)
+ *word = *view.word;
+ if (length)
+ *length = *view.length;
+ if (attached)
+ *attached = !!view.record;
+ }
+ mutex_unlock(memcomp_lock_of(ref->token));
+ return state;
+}
+
+static void memcomp_test_defrag_roundtrip(struct kunit *test)
+{
+ struct mc16_defrag_test *f = mc16_defrag_test_begin(test);
+ u64 word;
+ u16 length;
+ void *mapped;
+ bool same;
+ s64 compressed;
+ int puts, packed, dups, ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_defrag_test_insert(f, 1), 0);
+ KUNIT_ASSERT_TRUE(test, mc16_get(f->refs[0].token, &f->guard));
+ puts = atomic_read(&memcomp_kunit_slot_puts);
+ packed = atomic_read(&memcomp_kunit_slot_reclaims);
+ dups = atomic_read(&memcomp_kunit_slot_dups);
+ compressed = atomic64_read(&memcomp_compressed_bytes);
+ KUNIT_ASSERT_EQ(test, mc16_defrag_test_run(f, NULL), 1);
+ KUNIT_EXPECT_FALSE(test, f->callback_under_t);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_dups) - dups, 1);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_puts) - puts, 1);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_reclaims) - packed, 1);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_compressed_bytes), compressed);
+ KUNIT_EXPECT_EQ(test, f->guard.cluster->eligible_slots, (u16)1);
+ KUNIT_ASSERT_EQ(test, mc16_defrag_test_state(&f->guard, &word, &length, NULL),
+ (unsigned int)MC16_STORED);
+ KUNIT_EXPECT_EQ(test, (u32)length + 1, f->bytes);
+ mapped = zs_map_object(memcomp_pools[0], word, ZS_MM_RO);
+ if (!mapped) {
+ KUNIT_FAIL(test, "copied payload did not map");
+ return;
+ }
+ same = !memcmp(mapped, f->image + 37, f->bytes);
+ zs_unmap_object(memcomp_pools[0], word);
+ KUNIT_EXPECT_TRUE(test, same);
+ ret = mc16_load_page(&f->guard, f->image);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ KUNIT_EXPECT_EQ(test, memcmp(f->image, f->original, PAGE_SIZE), 0);
+ mc16_ref_put(&f->guard);
+ mc16_ref_put(&f->refs[0]);
+ KUNIT_EXPECT_EQ(test, f->payload.allocs, 1U);
+ KUNIT_EXPECT_EQ(test, f->payload.frees, 1U);
+ KUNIT_EXPECT_FALSE(test, f->payload.live);
+ KUNIT_EXPECT_FALSE(test, f->payload.bad_lifetime);
+}
+
+static void memcomp_test_defrag_noncandidates(struct kunit *test)
+{
+ struct mc16_defrag_test *f = mc16_defrag_test_begin(test);
+ int dups;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_defrag_test_insert(f, 1), 0);
+ KUNIT_ASSERT_TRUE(test, mc16_get(f->refs[0].token, &f->guard));
+ dups = atomic_read(&memcomp_kunit_slot_dups);
+ f->count = 2;
+ KUNIT_EXPECT_EQ(test, mc16_defrag_test_run(f, NULL), 0);
+ KUNIT_EXPECT_EQ(test, f->reads, 0U);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_dups), dups);
+ mc16_ref_put(&f->guard);
+ KUNIT_ASSERT_EQ(test, mc16_defrag_test_insert(f, 2), 0);
+ f->count = 1;
+ KUNIT_ASSERT_TRUE(test, mc16_get(f->refs[1].token, &f->guard));
+ KUNIT_EXPECT_EQ(test, mc16_defrag_test_run(f, NULL), 0);
+ KUNIT_EXPECT_EQ(test, f->reads, 0U);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_dups), dups);
+}
+
+/* Buffer/read/copy/charge errors and independent full-word/length races. */
+static void mc16_defrag_test_failure(struct kunit *test, unsigned int failure)
+{
+ struct mc16_defrag_test *f = mc16_defrag_test_begin(test);
+ u64 before_word, word;
+ u16 before_length, length;
+ long charge;
+ int puts, packed, expected;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_defrag_test_insert(f, 1), 0);
+ KUNIT_ASSERT_TRUE(test, mc16_get(f->refs[0].token, &f->guard));
+ mc16_defrag_test_state(&f->guard, &before_word, &before_length, NULL);
+ puts = atomic_read(&memcomp_kunit_slot_puts);
+ packed = atomic_read(&memcomp_kunit_slot_reclaims);
+ charge = atomic_long_read(&f->base->model.charged);
+ f->fail_buffer = failure == 0;
+ f->read_error = failure == 1 ? -EIO : 0;
+ f->fail_alloc = failure == 2;
+ f->fail_charge = failure == 3;
+ f->race = failure >= 4 ? failure - 3 : 0;
+ expected = failure >= 4 ? 0 : failure == 1 ? -EIO : -ENOMEM;
+ KUNIT_EXPECT_EQ(test, mc16_defrag_test_run(f, NULL), expected);
+ f->base->model.fail_at = 0;
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->base->model.charged), charge);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_puts) - puts, 1);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_reclaims) - packed, 0);
+ KUNIT_EXPECT_EQ(test, mc16_defrag_test_state(&f->guard, &word, &length, NULL),
+ (unsigned int)MC16_SWAPPED);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guard.slot->refs), 2U);
+ if (failure < 4) {
+ KUNIT_EXPECT_EQ(test, word, before_word);
+ KUNIT_EXPECT_EQ(test, length, before_length);
+ } else if (failure == 4) {
+ KUNIT_EXPECT_NE(test, word, before_word);
+ KUNIT_EXPECT_EQ(test, length, before_length);
+ } else {
+ KUNIT_EXPECT_EQ(test, word, before_word);
+ KUNIT_EXPECT_NE(test, length, before_length);
+ }
+ KUNIT_EXPECT_EQ(test, f->payload.allocs, failure >= 3 ? 1U : 0U);
+ KUNIT_EXPECT_EQ(test, f->payload.frees, f->payload.allocs);
+ KUNIT_EXPECT_FALSE(test, f->payload.live);
+ KUNIT_EXPECT_FALSE(test, f->payload.bad_lifetime);
+}
+
+static void memcomp_test_defrag_buffer_failure(struct kunit *test)
+{
+ mc16_defrag_test_failure(test, 0);
+}
+
+static void memcomp_test_defrag_read_failure(struct kunit *test)
+{
+ mc16_defrag_test_failure(test, 1);
+}
+
+static void memcomp_test_defrag_alloc_failure(struct kunit *test)
+{
+ mc16_defrag_test_failure(test, 2);
+}
+
+static void memcomp_test_defrag_charge_failure(struct kunit *test)
+{
+ mc16_defrag_test_failure(test, 3);
+}
+
+static void memcomp_test_defrag_stale_word(struct kunit *test)
+{
+ mc16_defrag_test_failure(test, 4);
+}
+
+static void memcomp_test_defrag_stale_length(struct kunit *test)
+{
+ mc16_defrag_test_failure(test, 5);
+}
+
+static void memcomp_test_defrag_owner_normalize(struct kunit *test)
+{
+ struct mc16_defrag_test *f = mc16_defrag_test_begin(test);
+ bool attached = false;
+ int puts, packed;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_defrag_test_insert(f, 3), 0);
+ KUNIT_ASSERT_TRUE(test, mc16_get(f->refs[0].token, &f->guard));
+ mc16_defrag_test_state(&f->guard, NULL, NULL, &attached);
+ KUNIT_ASSERT_TRUE(test, attached);
+ puts = atomic_read(&memcomp_kunit_slot_puts);
+ packed = atomic_read(&memcomp_kunit_slot_reclaims);
+ f->drop_owner = true;
+ KUNIT_ASSERT_EQ(test, mc16_defrag_test_run(f, NULL), 1);
+ KUNIT_EXPECT_EQ(test, mc16_defrag_test_state(&f->guard, NULL, NULL, &attached),
+ (unsigned int)MC16_STORED);
+ KUNIT_EXPECT_FALSE(test, attached);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guard.slot->refs), 1U);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->base->account), 1);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_puts) - puts, 1);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_reclaims) - packed, 1);
+ KUNIT_ASSERT_EQ(test, mc16_load_page(&f->guard, f->image), 0);
+ KUNIT_EXPECT_EQ(test, memcmp(f->image, f->original, PAGE_SIZE), 0);
+}
+
+static int mc16_defrag_test_marker(struct mc16_defrag_test *f, unsigned int i,
+ u32 id)
+{
+ unsigned long flags;
+ u32 reserved;
+ int ret;
+
+ xa_lock_irqsave(&mc16_clusters, flags);
+ ret = __xa_alloc(&mc16_clusters, &reserved, NULL, XA_LIMIT(id, id),
+ GFP_NOWAIT | __GFP_NOWARN);
+ if (!ret)
+ f->markers[i] = reserved;
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ return ret;
+}
+
+static int mc16_defrag_test_pair(struct mc16_defrag_test *f)
+{
+ struct mc16_initial_payload initial[2] = { { .word = 7 }, { .word = 8 } };
+ unsigned int index = f->nr;
+ int ret;
+
+ ret = mc16_reserve_cache(&f->base->model.objcg,
+ folio_nid(f->base->account), initial, 2, &f->refs[index]);
+ if (ret)
+ return ret;
+ mc16_cache_publish(&f->refs[index], 0);
+ mc16_cache_publish(&f->refs[index + 1], 0);
+ f->nr += 2;
+ return 0;
+}
+
+static void memcomp_test_defrag_scan_budget(struct kunit *test)
+{
+ struct mc16_defrag_test *f = mc16_defrag_test_begin(test);
+ u64 cursor = 0, first;
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_defrag_test_marker(f, 0, 1), 0);
+ KUNIT_ASSERT_EQ(test, mc16_defrag_test_marker(f, 1, 2), 0);
+ KUNIT_ASSERT_EQ(test, mc16_defrag_test_insert(f, 0), 0);
+ first = f->refs[0].token;
+ KUNIT_ASSERT_EQ(test, mc16_defrag_test_pair(f), 0);
+ KUNIT_ASSERT_EQ(test, f->refs[1].token, f->refs[0].token + 2);
+ for (i = f->nr; i < MEMCOMP_WRITEBACK_DEFRAG_SCAN + 1; i++)
+ KUNIT_ASSERT_EQ(test, mc16_defrag_test_insert(f, 0), 0);
+ KUNIT_EXPECT_EQ(test, mc16_defrag_test_run(f, &cursor), 1);
+ /* Two XA markers and the real alignment hole all consume probes. */
+ KUNIT_EXPECT_EQ(test, cursor, first + MEMCOMP_WRITEBACK_DEFRAG_SCAN - 2);
+ KUNIT_EXPECT_EQ(test, f->reads, 0U);
+ KUNIT_EXPECT_EQ(test, mc16_defrag_test_run(f, &cursor), 0);
+ KUNIT_EXPECT_EQ(test, cursor, 0ULL);
+ for (i = 0; i < f->nr; i++)
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->refs[i].slot->refs), 1U);
+ cursor = U64_MAX;
+ KUNIT_EXPECT_EQ(test, mc16_defrag_test_run(f, &cursor), 0);
+ KUNIT_EXPECT_EQ(test, cursor, 0ULL);
+}
+
+static void memcomp_test_defrag_scan_moves(struct kunit *test)
+{
+ struct mc16_defrag_test *f = mc16_defrag_test_begin(test);
+ u64 cursor = 0;
+ unsigned int i;
+ int puts, packed;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (i = 0; i < MEMCOMP_WRITEBACK_DEFRAG_MOVE + 2; i++)
+ KUNIT_ASSERT_EQ(test, mc16_defrag_test_insert(f, 1), 0);
+ puts = atomic_read(&memcomp_kunit_slot_puts);
+ packed = atomic_read(&memcomp_kunit_slot_reclaims);
+ KUNIT_EXPECT_EQ(test, mc16_defrag_test_run(f, &cursor), 1);
+ KUNIT_EXPECT_EQ(test, f->reads, MEMCOMP_WRITEBACK_DEFRAG_MOVE);
+ KUNIT_EXPECT_EQ(test, cursor,
+ (u64)f->refs[MEMCOMP_WRITEBACK_DEFRAG_MOVE - 1].token + 1);
+ for (i = 0; i < f->nr; i++)
+ KUNIT_EXPECT_EQ(test, mc16_defrag_test_state(&f->refs[i], NULL, NULL, NULL),
+ (unsigned int)(i < MEMCOMP_WRITEBACK_DEFRAG_MOVE ?
+ MC16_STORED : MC16_SWAPPED));
+ KUNIT_EXPECT_EQ(test, mc16_defrag_test_run(f, &cursor), 0);
+ KUNIT_EXPECT_EQ(test, cursor, 0ULL);
+ KUNIT_EXPECT_EQ(test, f->reads, MEMCOMP_WRITEBACK_DEFRAG_MOVE + 2);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_puts) - puts,
+ (int)f->nr);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_reclaims) - packed,
+ (int)f->nr);
+}
+
+static void memcomp_test_defrag_scan_error(struct kunit *test)
+{
+ struct mc16_defrag_test *f = mc16_defrag_test_begin(test);
+ u64 cursor = 0;
+ int puts, packed;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_defrag_test_insert(f, 1), 0);
+ puts = atomic_read(&memcomp_kunit_slot_puts);
+ packed = atomic_read(&memcomp_kunit_slot_reclaims);
+ f->read_error = -EIO;
+ KUNIT_EXPECT_EQ(test, mc16_defrag_test_run(f, &cursor), 0);
+ KUNIT_EXPECT_EQ(test, cursor, (u64)f->refs[0].token + 1);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->refs[0].slot->refs), 1U);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_puts) - puts, 1);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_reclaims) - packed, 0);
+ KUNIT_EXPECT_EQ(test, mc16_defrag_test_state(&f->refs[0], NULL, NULL, NULL),
+ (unsigned int)MC16_SWAPPED);
+ /* A later outer-policy restart can move the unchanged survivor. */
+ f->read_error = 0;
+ cursor = 0;
+ KUNIT_EXPECT_EQ(test, mc16_defrag_test_run(f, &cursor), 0);
+ KUNIT_EXPECT_EQ(test, cursor, 0ULL);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->refs[0].slot->refs), 1U);
+ KUNIT_EXPECT_EQ(test, mc16_defrag_test_state(&f->refs[0], NULL, NULL, NULL),
+ (unsigned int)MC16_STORED);
+}
+#endif /* CONFIG_MEMCG && CONFIG_SWAP */
+#endif /* _MM_MEMCOMP_TESTS_DEFRAG_TEST_H */
diff --git a/mm/memcomp/tests/drain_test.h b/mm/memcomp/tests/drain_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..cd4a4b381ee995fc5f131abd1524357d646b53e1
--- /dev/null
+++ b/mm/memcomp/tests/drain_test.h
@@ -0,0 +1,581 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_DRAIN_TEST_H
+#define _MM_MEMCOMP_TESTS_DRAIN_TEST_H
+
+#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP)
+/*
+ * Include after writeback_batch_test.h, with drain.h already visible.
+ * FIFO, slot guards, zs payloads, packing and batch settlement are real.
+ * Backing references and synchronous IO use the WB fixture's device model.
+ * The pool-release seam is a no-op: these tests check pool exhaustion and
+ * restoration, not a real device's free-list accounting.
+ */
+struct mc16_test_drain {
+ struct mc16_test_wbb *wb;
+ unsigned long scanned;
+ unsigned int seed;
+ unsigned int fail_alloc_at;
+ unsigned int pool_before_alloc[2];
+ unsigned int pool_after_alloc[2];
+ unsigned int refs_before[2];
+ unsigned int refs_after[2];
+ unsigned int state[2];
+ unsigned int cursor_after;
+ unsigned int open_at_entry;
+ u16 eligible;
+ long fifo_count;
+ int initial_error;
+ int write_ret;
+ int stopped;
+ bool hot[2];
+ bool valid[2];
+ bool attempted;
+ bool prestage;
+ bool globals_restored;
+};
+
+static struct mc16_test_drain *mc16_test_drain_current;
+
+static int mc16_test_drain_alloc(struct folio **folio, swp_entry_t *swap)
+{
+ struct mc16_test_drain *d = mc16_test_drain_current;
+ unsigned int call;
+ int saved, ret;
+
+ if (!d || mc16_test_wbb_current != d->wb)
+ return -EINVAL;
+ call = d->wb->alloc_calls;
+ if (call < ARRAY_SIZE(d->pool_before_alloc))
+ d->pool_before_alloc[call] = memcomp_writeback_slots_free;
+ saved = d->wb->alloc_error;
+ if (d->fail_alloc_at == d->wb->alloc_calls + 1)
+ d->wb->alloc_error = -ENOMEM;
+ ret = mc16_test_wbb_alloc(folio, swap);
+ if (call < ARRAY_SIZE(d->pool_after_alloc))
+ d->pool_after_alloc[call] = memcomp_writeback_slots_free;
+ d->wb->alloc_error = saved;
+ return ret;
+}
+
+/* No assertions or borrowed view survives a T scope. */
+static bool mc16_test_drain_hot(struct mc16_test_drain *d, bool hot)
+{
+ struct mc16_view view;
+ unsigned int i;
+ bool valid = true;
+
+ for (i = 0; i < 2; i++) {
+ struct memcomp_ref *ref = &d->wb->survivor[i];
+
+ mutex_lock(memcomp_lock_of(ref->token));
+ if (!mc16_view_locked(ref, &view))
+ valid = false;
+ else if (hot)
+ *view.flags |= MC16_F_REFERENCED;
+ else
+ *view.flags &= ~MC16_F_REFERENCED;
+ mutex_unlock(memcomp_lock_of(ref->token));
+ }
+ return valid;
+}
+
+static struct mc16_test_drain *
+mc16_test_drain_begin(struct kunit *test, bool raw)
+{
+ struct mc16_test_drain *d;
+
+ d = kunit_kzalloc(test, sizeof(*d), GFP_KERNEL);
+ if (!d)
+ return NULL;
+ d->wb = mc16_test_wbb_begin(test, raw);
+ if (!d->wb || !mc16_test_drain_hot(d, false))
+ return NULL;
+ d->seed = 3;
+ kunit_activate_static_stub(test, mc16_writeback_folio_alloc,
+ mc16_test_drain_alloc);
+ return d;
+}
+
+/*
+ * Synchronous transaction: no assertion, test action or return inherits W,
+ * T, L, a folio lock, or the nofs/noreclaim scope. UINT_MAX keeps the cursor.
+ * Capture drain's postconditions before a defensive flush of leaked work.
+ */
+static int mc16_test_drain_run(struct mc16_test_drain *d, unsigned long limit,
+ unsigned int cursor)
+{
+ struct mc16_test_wbb *f = d->wb;
+ struct mc16_test_storage *s = f->refs->storage;
+ struct mc16_cluster *cluster = f->survivor[0].cluster;
+ struct memcomp_lru *fifo = mc16_fifo_of(&f->survivor[0]);
+ struct memcomp_kunit_async_probe *saved_async;
+ struct mc16_test_drain *saved_drain;
+ struct mc16_test_wbb *saved_current;
+ unsigned int nofs, noreclaim, saved_free, i, frees;
+ int nid = cluster->key->nid;
+ int saved_stopped, ret = 0, dups, puts, reclaims;
+ unsigned long irqflags;
+ struct mc16_view view;
+ bool own_fifo;
+
+ nofs = memalloc_nofs_save();
+ noreclaim = memalloc_noreclaim_save();
+ mutex_lock(&memcomp_writeback_lock);
+ if (mc16_writeback_batch_used || mc16_writeback_members_used ||
+ mc16_writeback_open_first || mc16_writeback_open_bytes ||
+ memcomp_writeback_plugged || mc16_test_wbb_current ||
+ mc16_test_drain_current || d->seed > 3 || nid >= MAX_NUMNODES) {
+ ret = -EBUSY;
+ goto unlock;
+ }
+ spin_lock_irqsave(&fifo->lock, irqflags);
+ own_fifo = cluster->eligible_slots ?
+ list_is_singular(&fifo->entries) &&
+ list_first_entry(&fifo->entries, struct mc16_cluster, fifo) == cluster :
+ list_empty(&fifo->entries);
+ if (own_fifo && cursor != UINT_MAX)
+ cluster->wb_cursor = cursor;
+ spin_unlock_irqrestore(&fifo->lock, irqflags);
+ if (!own_fifo) {
+ ret = -EBUSY;
+ goto unlock;
+ }
+
+ memcpy(f->saved_pool, memcomp_writeback_slot_pool, sizeof(f->saved_pool));
+ saved_free = memcomp_writeback_slots_free;
+ for (i = 0; i < d->seed; i++)
+ memcomp_writeback_slot_pool[i] = swp_entry(MAX_SWAPFILES - 1, 240 + i);
+ memcomp_writeback_slots_free = d->seed;
+ saved_stopped = atomic_read(&memcomp_writeback_stopped[nid]);
+ atomic_set(&memcomp_writeback_stopped[nid], 0);
+ saved_current = mc16_test_wbb_current;
+ saved_drain = mc16_test_drain_current;
+ mc16_test_wbb_current = f;
+ mc16_test_drain_current = d;
+ saved_async = rcu_access_pointer(memcomp_kunit_async_probe);
+ s->payload.task = current;
+ s->payload.pool = memcomp_pools[0];
+ mutex_lock(memcomp_lock_of(f->token[0]));
+ if (!mc16_view_locked(&f->survivor[0], &view)) {
+ ret = -EIO;
+ } else {
+ s->payload.handle = *view.word;
+ s->payload.live = mc16_flags_state(*view.flags) == MC16_STORED;
+ }
+ mutex_unlock(memcomp_lock_of(f->token[0]));
+ rcu_assign_pointer(memcomp_kunit_async_probe, &s->payload);
+ dups = atomic_read(&memcomp_kunit_slot_dups);
+ puts = atomic_read(&memcomp_kunit_slot_puts);
+ reclaims = atomic_read(&memcomp_kunit_slot_reclaims);
+ frees = s->payload.frees;
+ f->charged_before = atomic_long_read(&s->owner->model.charged);
+ f->freed = 0;
+ d->scanned = 0;
+ d->attempted = false;
+ d->write_ret = d->initial_error;
+ for (i = 0; i < 2; i++)
+ d->refs_before[i] = refcount_read(&f->survivor[i].slot->refs);
+ if (!ret && d->prestage) {
+ mutex_lock(memcomp_lock_of(f->token[0]));
+ ret = mc16_writeback_stage_locked(&f->survivor[0], &f->staged[0]);
+ mutex_unlock(memcomp_lock_of(f->token[0]));
+ if (!ret)
+ ret = mc16_writeback_stage(&f->staged[0]);
+ }
+ if (f->staged[0].ref.slot)
+ mc16_writeback_stage_abort(&f->staged[0]);
+ d->open_at_entry = mc16_writeback_members_used - mc16_writeback_open_first;
+ f->live_before_io = s->payload.live;
+ if (!ret)
+ d->scanned = mc16_writeback_drain_queue(nid, fifo, limit, &f->freed,
+ &d->write_ret, &d->attempted);
+ f->batch_empty = !mc16_writeback_batch_used &&
+ !mc16_writeback_members_used && !mc16_writeback_open_first &&
+ !mc16_writeback_open_bytes && !memcomp_writeback_plugged;
+ f->seed_after_flush = memcomp_writeback_slots_free;
+ /* A failed assertion later must not strand global work or guards. */
+ if (!f->batch_empty || memcomp_writeback_slots_free)
+ mc16_writeback_batch_flush(&f->freed);
+ f->dups = atomic_read(&memcomp_kunit_slot_dups) - dups;
+ f->puts = atomic_read(&memcomp_kunit_slot_puts) - puts;
+ f->reclaims = atomic_read(&memcomp_kunit_slot_reclaims) - reclaims;
+ f->probe_frees = s->payload.frees - frees;
+ f->live_after = s->payload.live;
+ f->bad_lifetime = s->payload.bad_lifetime;
+ for (i = 0; i < 2; i++) {
+ struct memcomp_ref *ref = &f->survivor[i];
+
+ d->refs_after[i] = refcount_read(&ref->slot->refs);
+ mutex_lock(memcomp_lock_of(ref->token));
+ d->valid[i] = mc16_view_locked(ref, &view);
+ if (d->valid[i]) {
+ d->valid[i] = mc16_flags_valid(*view.flags);
+ d->state[i] = mc16_flags_state(*view.flags);
+ d->hot[i] = !!(*view.flags & MC16_F_REFERENCED);
+ }
+ mutex_unlock(memcomp_lock_of(ref->token));
+ }
+ spin_lock_irqsave(&fifo->lock, irqflags);
+ d->eligible = cluster->eligible_slots;
+ d->fifo_count = atomic_long_read(&fifo->nr_entries);
+ d->cursor_after = cluster->wb_cursor;
+ spin_unlock_irqrestore(&fifo->lock, irqflags);
+ d->stopped = atomic_read(&memcomp_writeback_stopped[nid]);
+ atomic_set(&memcomp_writeback_stopped[nid], saved_stopped);
+ memcpy(memcomp_writeback_slot_pool, f->saved_pool, sizeof(f->saved_pool));
+ memcomp_writeback_slots_free = saved_free;
+ f->pool_restored = !memcmp(memcomp_writeback_slot_pool, f->saved_pool,
+ sizeof(f->saved_pool)) &&
+ memcomp_writeback_slots_free == saved_free;
+ rcu_assign_pointer(memcomp_kunit_async_probe, saved_async);
+ f->probe_restored = rcu_access_pointer(memcomp_kunit_async_probe) == saved_async;
+ mc16_test_wbb_current = saved_current;
+ mc16_test_drain_current = saved_drain;
+ d->globals_restored = mc16_test_wbb_current == saved_current &&
+ mc16_test_drain_current == saved_drain &&
+ atomic_read(&memcomp_writeback_stopped[nid]) == saved_stopped;
+unlock:
+ mutex_unlock(&memcomp_writeback_lock);
+ memalloc_noreclaim_restore(noreclaim);
+ memalloc_nofs_restore(nofs);
+ synchronize_rcu();
+ return ret;
+}
+
+static void mc16_test_drain_done(struct kunit *test, struct mc16_test_drain *d,
+ unsigned int eligible)
+{
+ struct mc16_test_wbb *f = d->wb;
+ unsigned int i;
+
+ KUNIT_EXPECT_TRUE(test, f->batch_empty);
+ KUNIT_EXPECT_EQ(test, f->seed_after_flush, 0U);
+ KUNIT_EXPECT_TRUE(test, f->pool_restored);
+ KUNIT_EXPECT_TRUE(test, f->probe_restored);
+ KUNIT_EXPECT_TRUE(test, d->globals_restored);
+ KUNIT_EXPECT_FALSE(test, f->bad_lifetime);
+ KUNIT_EXPECT_EQ(test, d->eligible, (u16)eligible);
+ KUNIT_EXPECT_EQ(test, d->fifo_count, (long)eligible);
+ KUNIT_EXPECT_EQ(test, f->submits, f->completed);
+ KUNIT_EXPECT_FALSE(test, folio_test_locked(f->refs->storage->owner->folio));
+ for (i = 0; i < 2; i++) {
+ KUNIT_EXPECT_TRUE(test, d->valid[i]);
+ KUNIT_EXPECT_EQ(test, d->refs_before[i], 2U);
+ KUNIT_EXPECT_EQ(test, d->refs_after[i], d->refs_before[i]);
+ KUNIT_EXPECT_PTR_EQ(test, f->staged[i].ref.slot, NULL);
+ KUNIT_EXPECT_PTR_EQ(test, f->caller[i].slot, NULL);
+ }
+ if (f->submits)
+ KUNIT_EXPECT_TRUE(test, f->submit_contract);
+}
+
+static void memcomp_test_drain_cold_packed(struct kunit *test)
+{
+ struct mc16_test_drain *d = mc16_test_drain_begin(test, false);
+ struct mc16_test_wbb *f;
+ unsigned int off = 0, i;
+
+ KUNIT_ASSERT_NOT_NULL(test, d);
+ f = d->wb;
+ KUNIT_ASSERT_LE(test, f->bytes[0] + f->bytes[1], (u32)PAGE_SIZE);
+ KUNIT_ASSERT_EQ(test, mc16_test_drain_run(d, 2, 0), 0);
+ mc16_test_drain_done(test, d, 0);
+ KUNIT_EXPECT_EQ(test, d->scanned, 2UL);
+ KUNIT_EXPECT_EQ(test, d->write_ret, 0);
+ KUNIT_EXPECT_TRUE(test, d->attempted);
+ KUNIT_EXPECT_EQ(test, d->stopped, 0);
+ KUNIT_EXPECT_EQ(test, f->alloc_calls, 1U);
+ KUNIT_EXPECT_EQ(test, d->pool_before_alloc[0], 3U);
+ KUNIT_EXPECT_EQ(test, d->pool_after_alloc[0], 2U);
+ KUNIT_EXPECT_EQ(test, f->submits, 1U);
+ KUNIT_EXPECT_EQ(test, f->freed, 2UL);
+ KUNIT_EXPECT_EQ(test, f->dups, 2);
+ KUNIT_EXPECT_EQ(test, f->puts, 0);
+ KUNIT_EXPECT_EQ(test, f->reclaims, 0);
+ KUNIT_EXPECT_EQ(test, f->probe_frees, 1U);
+ KUNIT_EXPECT_FALSE(test, f->live_after);
+ for (i = 0; i < 2; i++) {
+ KUNIT_EXPECT_EQ(test, memcmp(f->images + off, f->encoded[i], f->bytes[i]), 0);
+ KUNIT_ASSERT_EQ(test,
+ memcomp_decode_slot(f->images, true, off, f->bytes[i], f->dst), 0);
+ KUNIT_EXPECT_EQ(test, memcmp(f->dst, f->src + i * PAGE_SIZE, PAGE_SIZE), 0);
+ mc16_test_wbb_state(test, f, i, MC16_SWAPPED, true, off);
+ off += f->bytes[i];
+ }
+ KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->images + off, 0, PAGE_SIZE - off), NULL);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->refs->storage->owner->model.charged),
+ f->charged_before - f->bytes[0] - f->bytes[1]);
+}
+
+static void memcomp_test_drain_cold_raw(struct kunit *test)
+{
+ struct mc16_test_drain *d = mc16_test_drain_begin(test, true);
+ struct mc16_test_wbb *f;
+
+ KUNIT_ASSERT_NOT_NULL(test, d);
+ f = d->wb;
+ KUNIT_ASSERT_EQ(test, mc16_test_drain_run(d, 1, 0), 0);
+ mc16_test_drain_done(test, d, 1);
+ KUNIT_EXPECT_EQ(test, d->scanned, 1UL);
+ KUNIT_EXPECT_EQ(test, d->write_ret, 0);
+ KUNIT_EXPECT_TRUE(test, d->attempted);
+ KUNIT_EXPECT_EQ(test, f->submits, 1U);
+ KUNIT_EXPECT_EQ(test, f->freed, 1UL);
+ KUNIT_EXPECT_EQ(test, f->dups, 1);
+ KUNIT_EXPECT_EQ(test, f->puts, 0);
+ KUNIT_EXPECT_EQ(test, memcmp(f->images, f->src, PAGE_SIZE), 0);
+ mc16_test_wbb_state(test, f, 0, MC16_SWAPPED, false, 0);
+ mc16_test_wbb_state(test, f, 1, MC16_STORED, false, 0);
+}
+
+static void memcomp_test_drain_hot_second_chance(struct kunit *test)
+{
+ struct mc16_test_drain *d = mc16_test_drain_begin(test, false);
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, d);
+ KUNIT_ASSERT_TRUE(test, mc16_test_drain_hot(d, true));
+ KUNIT_ASSERT_EQ(test, mc16_test_drain_run(d, 2, 0), 0);
+ mc16_test_drain_done(test, d, 2);
+ KUNIT_EXPECT_EQ(test, d->scanned, 2UL);
+ KUNIT_EXPECT_EQ(test, d->write_ret, 0);
+ KUNIT_EXPECT_FALSE(test, d->attempted);
+ KUNIT_EXPECT_EQ(test, d->wb->alloc_calls, 0U);
+ KUNIT_EXPECT_EQ(test, d->wb->freed, 0UL);
+ for (i = 0; i < 2; i++) {
+ KUNIT_EXPECT_FALSE(test, d->hot[i]);
+ KUNIT_EXPECT_EQ(test, d->state[i], (unsigned int)MC16_STORED);
+ }
+ /* Preserve the real FIFO cursor: only a second pass may claim them. */
+ KUNIT_ASSERT_EQ(test, mc16_test_drain_run(d, 2, UINT_MAX), 0);
+ mc16_test_drain_done(test, d, 0);
+ KUNIT_EXPECT_EQ(test, d->scanned, 2UL);
+ KUNIT_EXPECT_TRUE(test, d->attempted);
+ KUNIT_EXPECT_EQ(test, d->write_ret, 0);
+ KUNIT_EXPECT_EQ(test, d->wb->freed, 2UL);
+ KUNIT_EXPECT_EQ(test, d->wb->submits, 1U);
+}
+
+static void memcomp_test_drain_zero_limit(struct kunit *test)
+{
+ struct mc16_test_drain *d = mc16_test_drain_begin(test, false);
+
+ KUNIT_ASSERT_NOT_NULL(test, d);
+ KUNIT_ASSERT_EQ(test, mc16_test_drain_run(d, 0, 0), 0);
+ mc16_test_drain_done(test, d, 2);
+ KUNIT_EXPECT_EQ(test, d->scanned, 0UL);
+ KUNIT_EXPECT_EQ(test, d->cursor_after, 0U);
+ KUNIT_EXPECT_FALSE(test, d->attempted);
+ KUNIT_EXPECT_EQ(test, d->write_ret, 0);
+ KUNIT_EXPECT_EQ(test, d->wb->alloc_calls, 0U);
+ KUNIT_EXPECT_EQ(test, d->wb->freed, 0UL);
+}
+
+static void memcomp_test_drain_empty_fifo(struct kunit *test)
+{
+ struct mc16_test_drain *d = mc16_test_drain_begin(test, false);
+ unsigned int allocs;
+
+ KUNIT_ASSERT_NOT_NULL(test, d);
+ KUNIT_ASSERT_EQ(test, mc16_test_drain_run(d, 2, 0), 0);
+ KUNIT_ASSERT_EQ(test, d->wb->freed, 2UL);
+ allocs = d->wb->alloc_calls;
+ /* This is the same actual FIFO after both entries became SWAPPED. */
+ KUNIT_ASSERT_EQ(test, mc16_test_drain_run(d, ULONG_MAX, UINT_MAX), 0);
+ mc16_test_drain_done(test, d, 0);
+ KUNIT_EXPECT_EQ(test, d->scanned, 0UL);
+ KUNIT_EXPECT_FALSE(test, d->attempted);
+ KUNIT_EXPECT_EQ(test, d->write_ret, 0);
+ KUNIT_EXPECT_EQ(test, d->wb->alloc_calls, allocs);
+ KUNIT_EXPECT_EQ(test, d->wb->freed, 0UL);
+ KUNIT_EXPECT_EQ(test, d->wb->dups, 0);
+ KUNIT_EXPECT_EQ(test, d->wb->puts, 0);
+}
+
+/* Add adjacent resident PENDING, SAMEFILL, and retired refs-zero positions. */
+static int mc16_test_drain_ineligible(struct mc16_test_drain *d)
+{
+ struct mc16_test_storage *s = d->wb->refs->storage;
+ unsigned int i;
+ int ret = 0;
+
+ folio_lock(s->owner->folio);
+ for (i = 0; !ret && i < 3; i++)
+ ret = mc16_test_refs_reserve(d->wb->refs, 1);
+ if (!ret)
+ ret = mc16_test_refs_payload(d->wb->refs, 3, MC16_SAMEFILL);
+ folio_unlock(s->owner->folio);
+ if (ret)
+ return ret;
+ mc16_ref_put(&s->refs[4]);
+ return 0;
+}
+
+static void memcomp_test_drain_ineligible_budget(struct kunit *test)
+{
+ struct mc16_test_drain *d = mc16_test_drain_begin(test, false);
+ struct mc16_test_storage *s;
+ struct memcomp_ref gone;
+
+ KUNIT_ASSERT_NOT_NULL(test, d);
+ KUNIT_ASSERT_EQ(test, mc16_test_drain_ineligible(d), 0);
+ s = d->wb->refs->storage;
+ KUNIT_ASSERT_PTR_EQ(test, s->refs[2].cluster, s->refs[0].cluster);
+ KUNIT_ASSERT_PTR_EQ(test, s->refs[3].cluster, s->refs[0].cluster);
+ KUNIT_ASSERT_EQ(test, s->refs[3].token, s->refs[0].token + 3);
+ KUNIT_ASSERT_EQ(test, mc16_test_drain_run(d, 3, 2), 0);
+ mc16_test_drain_done(test, d, 2);
+ KUNIT_EXPECT_EQ(test, d->scanned, 3UL);
+ KUNIT_EXPECT_EQ(test, d->cursor_after, 5U);
+ KUNIT_EXPECT_FALSE(test, d->attempted);
+ KUNIT_EXPECT_EQ(test, d->write_ret, 0);
+ KUNIT_EXPECT_EQ(test, d->wb->alloc_calls, 0U);
+ KUNIT_EXPECT_EQ(test, refcount_read(&s->refs[2].slot->refs), 1U);
+ KUNIT_EXPECT_EQ(test, refcount_read(&s->refs[3].slot->refs), 1U);
+ if (mc16_get(s->refs[0].token + 4, &gone)) {
+ mc16_ref_put(&gone);
+ KUNIT_FAIL(test, "retired position became visible again");
+ }
+ KUNIT_ASSERT_EQ(test, mc16_test_drain_run(d, 2, UINT_MAX), 0);
+ mc16_test_drain_done(test, d, 0);
+ KUNIT_EXPECT_EQ(test, d->scanned, 2UL);
+ KUNIT_EXPECT_EQ(test, d->wb->freed, 2UL);
+}
+
+static void mc16_test_drain_failed(struct kunit *test, bool raw, bool dirty,
+ bool enospc)
+{
+ struct mc16_test_drain *d = mc16_test_drain_begin(test, raw);
+ struct mc16_test_wbb *f;
+ unsigned int i;
+ int error = enospc ? -ENOSPC : dirty ? -EIO : -ENOMEM;
+
+ KUNIT_ASSERT_NOT_NULL(test, d);
+ f = d->wb;
+ f->dirty_io = dirty;
+ if (enospc)
+ d->seed = 0; /* The model allocator must not refill from a device. */
+ else if (!dirty)
+ f->alloc_error = -ENOMEM;
+ KUNIT_ASSERT_EQ(test, mc16_test_drain_run(d, 2, 0), 0);
+ mc16_test_drain_done(test, d, 2);
+ KUNIT_EXPECT_EQ(test, d->scanned, raw ? 1UL : 2UL);
+ KUNIT_EXPECT_EQ(test, d->write_ret, error);
+ KUNIT_EXPECT_TRUE(test, d->attempted);
+ KUNIT_EXPECT_EQ(test, d->stopped, enospc ? 1 : 0);
+ KUNIT_EXPECT_EQ(test, f->alloc_calls, 1U);
+ KUNIT_EXPECT_EQ(test, d->pool_before_alloc[0], d->seed);
+ KUNIT_EXPECT_EQ(test, d->pool_after_alloc[0], dirty ? d->seed - 1 : d->seed);
+ KUNIT_EXPECT_EQ(test, f->freed, 0UL);
+ KUNIT_EXPECT_EQ(test, f->submits, dirty ? 1U : 0U);
+ KUNIT_EXPECT_EQ(test, f->dups, dirty ? 1 : 0);
+ KUNIT_EXPECT_EQ(test, f->puts, dirty ? 1 : 0);
+ KUNIT_EXPECT_EQ(test, f->reclaims, 0);
+ KUNIT_EXPECT_EQ(test, f->probe_frees, 0U);
+ KUNIT_EXPECT_TRUE(test, f->live_after);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->refs->storage->owner->model.charged),
+ f->charged_before);
+ for (i = 0; i < 2; i++)
+ mc16_test_wbb_state(test, f, i, MC16_STORED, false, 0);
+}
+
+static void memcomp_test_drain_alloc_failure(struct kunit *test)
+{
+ mc16_test_drain_failed(test, false, false, false);
+}
+
+static void memcomp_test_drain_stage_alloc_failure(struct kunit *test)
+{
+ mc16_test_drain_failed(test, true, false, false);
+}
+
+static void memcomp_test_drain_dirty_io(struct kunit *test)
+{
+ mc16_test_drain_failed(test, false, true, false);
+}
+
+static void memcomp_test_drain_enospc(struct kunit *test)
+{
+ mc16_test_drain_failed(test, false, false, true);
+}
+
+static void memcomp_test_drain_stage_enospc(struct kunit *test)
+{
+ mc16_test_drain_failed(test, true, false, true);
+}
+
+static void memcomp_test_drain_zero_limit_flush(struct kunit *test)
+{
+ struct mc16_test_drain *d = mc16_test_drain_begin(test, false);
+ struct mc16_test_wbb *f;
+
+ KUNIT_ASSERT_NOT_NULL(test, d);
+ f = d->wb;
+ d->prestage = true;
+ KUNIT_ASSERT_EQ(test, mc16_test_drain_run(d, 0, 0), 0);
+ mc16_test_drain_done(test, d, 1);
+ KUNIT_EXPECT_EQ(test, d->open_at_entry, 1U);
+ KUNIT_EXPECT_EQ(test, d->scanned, 0UL);
+ KUNIT_EXPECT_FALSE(test, d->attempted);
+ KUNIT_EXPECT_EQ(test, d->write_ret, 0);
+ KUNIT_EXPECT_EQ(test, f->alloc_calls, 1U);
+ KUNIT_EXPECT_EQ(test, d->pool_before_alloc[0], 3U);
+ KUNIT_EXPECT_EQ(test, d->pool_after_alloc[0], 2U);
+ KUNIT_EXPECT_EQ(test, f->submits, 1U);
+ KUNIT_EXPECT_EQ(test, f->freed, 1UL);
+ KUNIT_EXPECT_EQ(test, f->dups, 1);
+ KUNIT_EXPECT_EQ(test, f->puts, 0);
+ KUNIT_EXPECT_EQ(test, memcmp(f->images, f->src, PAGE_SIZE), 0);
+ mc16_test_wbb_state(test, f, 0, MC16_SWAPPED, false, 0);
+ mc16_test_wbb_state(test, f, 1, MC16_STORED, false, 0);
+}
+
+static void memcomp_test_drain_partial_flush_failure(struct kunit *test)
+{
+ struct mc16_test_drain *d = mc16_test_drain_begin(test, true);
+ struct mc16_test_wbb *f;
+
+ KUNIT_ASSERT_NOT_NULL(test, d);
+ f = d->wb;
+ d->fail_alloc_at = 2;
+ KUNIT_ASSERT_EQ(test, mc16_test_drain_run(d, 2, 0), 0);
+ mc16_test_drain_done(test, d, 1);
+ KUNIT_EXPECT_EQ(test, d->scanned, 2UL);
+ KUNIT_EXPECT_EQ(test, d->write_ret, -ENOMEM);
+ KUNIT_EXPECT_TRUE(test, d->attempted);
+ KUNIT_EXPECT_EQ(test, d->stopped, 0);
+ KUNIT_EXPECT_EQ(test, f->alloc_calls, 2U);
+ KUNIT_EXPECT_EQ(test, d->pool_before_alloc[0], 3U);
+ KUNIT_EXPECT_EQ(test, d->pool_after_alloc[0], 2U);
+ KUNIT_EXPECT_EQ(test, d->pool_before_alloc[1], 2U);
+ KUNIT_EXPECT_EQ(test, d->pool_after_alloc[1], 2U);
+ KUNIT_EXPECT_EQ(test, f->submits, 1U);
+ KUNIT_EXPECT_EQ(test, f->freed, 1UL);
+ KUNIT_EXPECT_EQ(test, f->dups, 1);
+ KUNIT_EXPECT_EQ(test, f->puts, 0);
+ KUNIT_EXPECT_EQ(test, f->probe_frees, 1U);
+ KUNIT_EXPECT_EQ(test, memcmp(f->images, f->src, PAGE_SIZE), 0);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->refs->storage->owner->model.charged),
+ f->charged_before - f->bytes[0]);
+ mc16_test_wbb_state(test, f, 0, MC16_SWAPPED, false, 0);
+ mc16_test_wbb_state(test, f, 1, MC16_STORED, false, 0);
+}
+
+static void memcomp_test_drain_first_error(struct kunit *test)
+{
+ struct mc16_test_drain *d = mc16_test_drain_begin(test, false);
+
+ KUNIT_ASSERT_NOT_NULL(test, d);
+ d->initial_error = -EINTR;
+ d->wb->alloc_error = -ENOSPC;
+ KUNIT_ASSERT_EQ(test, mc16_test_drain_run(d, 2, 0), 0);
+ mc16_test_drain_done(test, d, 2);
+ KUNIT_EXPECT_EQ(test, d->scanned, 2UL);
+ KUNIT_EXPECT_TRUE(test, d->attempted);
+ KUNIT_EXPECT_EQ(test, d->write_ret, -EINTR);
+ KUNIT_EXPECT_EQ(test, d->stopped, 1);
+ KUNIT_EXPECT_EQ(test, d->wb->freed, 0UL);
+ KUNIT_EXPECT_EQ(test, d->wb->alloc_calls, 1U);
+}
+#endif /* CONFIG_MEMCG && CONFIG_SWAP */
+#endif /* _MM_MEMCOMP_TESTS_DRAIN_TEST_H */
diff --git a/mm/memcomp/tests/enospc_test.h b/mm/memcomp/tests/enospc_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..a973ebdd4ea9118e27391772c69f80c0f8d8985a
--- /dev/null
+++ b/mm/memcomp/tests/enospc_test.h
@@ -0,0 +1,236 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_ENOSPC_TEST_H
+#define _MM_MEMCOMP_TESTS_ENOSPC_TEST_H
+
+#ifdef CONFIG_MEMCG
+/* Include after the shared fixtures in suite.h; register there. */
+static void memcomp_test_enospc_result(struct kunit *test)
+{
+ static const struct {
+ int status;
+ int result;
+ unsigned int length;
+ bool reusable;
+ } cases[] = {
+ { 0, 0, 17, true },
+ { -E2BIG, 0, PAGE_SIZE, true },
+ { -ENOSPC, 0, PAGE_SIZE, true },
+ { -EIO, -EIO, 17, false },
+ { -ENODATA, -ENODATA, 17, false },
+ { -EHWPOISON, -EHWPOISON, 17, false },
+ };
+ struct memcomp_async_req request = {};
+ struct memcomp_store_ctx store = { .precomp = &request };
+ const struct memcomp_async_req *expected;
+ unsigned int i, length;
+ int ret;
+
+ for (i = 0; i < ARRAY_SIZE(cases); i++) {
+ length = 17;
+ ret = memcomp_compress_result(cases[i].status, &length,
+ PAGE_SIZE);
+ KUNIT_EXPECT_EQ(test, ret, cases[i].result);
+ KUNIT_EXPECT_EQ(test, length, cases[i].length);
+ request.ret = cases[i].status;
+ request.len = 17;
+ expected = NULL;
+ if (cases[i].reusable)
+ expected = &request;
+ KUNIT_EXPECT_PTR_EQ(test, memcomp_store_precomp(&store, 0), expected);
+ /* A codec error cannot manufacture a software scan result. */
+ KUNIT_EXPECT_FALSE(test, request.same_filled);
+ KUNIT_EXPECT_EQ(test, request.ret, cases[i].status);
+ KUNIT_EXPECT_EQ(test, request.len, 17U);
+ }
+ KUNIT_EXPECT_NULL(test, memcomp_store_precomp(NULL, 0));
+ store.precomp = NULL;
+ KUNIT_EXPECT_NULL(test, memcomp_store_precomp(&store, 0));
+}
+
+/* Leave the atomic zsmalloc mapping before KUnit can report a failure. */
+static bool memcomp_test_enospc_raw_matches(u8 pool_idx,
+ unsigned long handle,
+ const void *src)
+{
+ struct zs_pool *pool = memcomp_pools[pool_idx];
+ const void *payload;
+ bool matches;
+
+ payload = zs_map_object(pool, handle, ZS_MM_RO);
+ matches = payload && !memcmp(payload, src, PAGE_SIZE);
+ if (payload)
+ zs_unmap_object(pool, handle);
+ return matches;
+}
+
+/* Exercise raw policy through the real synchronous and precomputed paths. */
+static void memcomp_test_enospc_payload(struct kunit *test,
+ bool precomputed)
+{
+ static const int statuses[] = { -ENOSPC, -E2BIG, -EIO };
+ struct memcomp_async_req request = { .len = 17 };
+ struct memcomp_reclaim_ctx *ctx;
+ struct memcomp_store_ctx store = {};
+ const struct memcomp_async_req *selected;
+ unsigned int reclaim_flags, length, i, raw;
+ unsigned long handle;
+ s64 candidate, reject, fail;
+ u8 pool_idx;
+ u8 *src;
+ bool matches;
+ int ret;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+ src = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ memcomp_test_fill_zstd_ratio_page(src);
+ reclaim_flags = memcomp_test_reclaimer_enter();
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_KSWAPD);
+ if (!ctx) {
+ memcomp_test_reclaimer_exit(reclaim_flags);
+ KUNIT_FAIL(test, "failed to acquire a compressor context");
+ return;
+ }
+ for (i = 0; i < ARRAY_SIZE(statuses); i++) {
+ request.ret = statuses[i];
+ store.precomp = precomputed ? &request : NULL;
+ selected = memcomp_store_precomp(&store, 0);
+ for (raw = 0; raw <= 1; raw++) {
+ candidate =
+ atomic64_read(&memcomp_raw_candidate_ge_page);
+ reject = atomic64_read(&memcomp_raw_reject_ge_page);
+ fail = atomic64_read(&memcomp_compress_fail);
+ length = 0;
+ handle = 0;
+ pool_idx = 0;
+ /* Reused output must leave the sync sentinel intact. */
+ ctx->kunit_compress_error =
+ precomputed ? -EIO : statuses[i];
+ ret = memcomp_prepare_payload_locked(&ctx, src,
+ PAGE_SIZE, &length,
+ &handle, &pool_idx,
+ raw, NULL, selected);
+ if (statuses[i] == -EIO) {
+ KUNIT_EXPECT_EQ(test, ret, -EIO);
+ KUNIT_EXPECT_EQ(test, length, 0U);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_compress_fail),
+ fail + 1);
+ KUNIT_EXPECT_EQ(test,
+ ctx->kunit_compress_error, 0);
+ } else {
+ KUNIT_EXPECT_EQ(test, ret, raw ? 0 : -E2BIG);
+ KUNIT_EXPECT_EQ(test, length,
+ raw ? (unsigned int)PAGE_SIZE : 0U);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_compress_fail),
+ fail);
+ KUNIT_EXPECT_EQ(test, ctx->kunit_compress_error,
+ precomputed ? -EIO : 0);
+ candidate++;
+ reject += !raw;
+ }
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_raw_candidate_ge_page),
+ candidate);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_raw_reject_ge_page),
+ reject);
+ if (!ret && length) {
+ if (length == PAGE_SIZE) {
+ matches = memcomp_test_enospc_raw_matches(pool_idx,
+ handle, src);
+ KUNIT_EXPECT_TRUE(test, matches);
+ }
+ memcomp_zs_free(pool_idx, handle);
+ }
+ }
+ }
+ ctx->kunit_compress_error = 0;
+ memcomp_reclaim_ctx_put(ctx);
+ memcomp_test_reclaimer_exit(reclaim_flags);
+}
+
+static void memcomp_test_enospc_sync(struct kunit *test)
+{
+ memcomp_test_enospc_payload(test, false);
+}
+
+static void memcomp_test_enospc_precomp(struct kunit *test)
+{
+ memcomp_test_enospc_payload(test, true);
+}
+
+/* Result events precede admission: a later store refusal still counts. */
+static void memcomp_test_enospc_async_events(struct kunit *test)
+{
+ static const struct {
+ int status;
+ bool same_filled;
+ unsigned int payloads;
+ unsigned int failures;
+ } cases[] = {
+ { 0, false, 2, 0 },
+ { -ENOSPC, false, 2, 0 },
+ { -E2BIG, false, 2, 0 },
+ { -EIO, false, 0, 2 },
+ { -ENODATA, false, 0, 2 },
+ { -EHWPOISON, false, 0, 2 },
+ { 0, true, 0, 0 },
+ { -ENOSPC, true, 0, 0 },
+ };
+ struct memcomp_reclaim_ctx *ctx;
+ struct memcomp_async *async;
+ struct folio *folio;
+ unsigned int reclaim_flags, i, j;
+ s64 payloads, failures;
+ bool stored;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+ async = kunit_kzalloc(test, sizeof(*async), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, async);
+ folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 1);
+ KUNIT_ASSERT_NOT_NULL(test, folio);
+ folio_lock(folio);
+ /* A non-anon folio refuses preflight before reading payload buffers. */
+ async->nr = 3;
+ async->reqs[0].ret = -EIO;
+ for (j = 1; j < async->nr; j++)
+ async->reqs[j].folio = folio;
+ reclaim_flags = memcomp_test_reclaimer_enter();
+ for (i = 0; i < ARRAY_SIZE(cases); i++) {
+ for (j = 1; j < async->nr; j++) {
+ async->reqs[j].ret = cases[i].status;
+ async->reqs[j].same_filled = cases[i].same_filled;
+ }
+ ctx = memcomp_reclaim_ctx_get(2, MEMCOMP_RECLAIM_KSWAPD);
+ if (!ctx) {
+ KUNIT_FAIL(test, "no compressor context");
+ break;
+ }
+ payloads = atomic64_read(&memcomp_async_payloads);
+ failures = atomic64_read(&memcomp_async_fail);
+ stored = memcomp_store_after_unmap_async(folio, &ctx, async);
+ KUNIT_EXPECT_FALSE(test, stored);
+ KUNIT_EXPECT_PTR_EQ(test, ctx, NULL);
+ if (ctx)
+ memcomp_reclaim_ctx_put(ctx);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_async_payloads),
+ payloads + cases[i].payloads);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_async_fail),
+ failures + cases[i].failures);
+ }
+ memcomp_test_reclaimer_exit(reclaim_flags);
+ folio_unlock(folio);
+ folio_put(folio);
+}
+
+#endif /* CONFIG_MEMCG */
+
+#endif /* _MM_MEMCOMP_TESTS_ENOSPC_TEST_H */
diff --git a/mm/memcomp/tests/fault_alloc_test.h b/mm/memcomp/tests/fault_alloc_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..a8b91933402f33a0f501efbc75df829bebbb7fe9
--- /dev/null
+++ b/mm/memcomp/tests/fault_alloc_test.h
@@ -0,0 +1,119 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_FAULT_ALLOC_TEST_H
+#define _MM_MEMCOMP_TESTS_FAULT_ALLOC_TEST_H
+
+#include <kunit/test.h>
+#include <linux/mm.h>
+#include <linux/huge_mm.h>
+
+#include "../fault_alloc.h"
+
+static void memcomp_test_fault_alloc_single_page(struct kunit *test)
+{
+ struct vm_area_struct vma = { };
+
+ KUNIT_EXPECT_EQ(test, memcomp_fault_alloc_gfp(&vma, 1),
+ GFP_HIGHUSER_MOVABLE);
+ vm_flags_init(&vma, VM_HUGEPAGE);
+ KUNIT_EXPECT_EQ(test, memcomp_fault_alloc_gfp(&vma, 1),
+ GFP_HIGHUSER_MOVABLE);
+}
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+/* No assertions or allocations may interrupt the save/restore interval. */
+static gfp_t
+memcomp_test_fault_alloc_with_policy(struct vm_area_struct *vma,
+ unsigned int nr_pages, unsigned long policy)
+{
+ const unsigned long defrag_mask =
+ BIT(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG) |
+ BIT(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG) |
+ BIT(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG) |
+ BIT(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
+ unsigned long saved = READ_ONCE(transparent_hugepage_flags);
+ gfp_t gfp;
+
+ WRITE_ONCE(transparent_hugepage_flags, (saved & ~defrag_mask) | policy);
+ gfp = memcomp_fault_alloc_gfp(vma, nr_pages);
+ WRITE_ONCE(transparent_hugepage_flags, saved);
+
+ return gfp;
+}
+#endif
+
+static void memcomp_test_fault_alloc_defrag_policy(struct kunit *test)
+{
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ /* Expected reclaim/retry behavior, for ordinary and madvised VMAs. */
+ static const struct {
+ unsigned long policy;
+ gfp_t behavior[2];
+ } cases[] = {
+ { 0, { 0, 0 } }, /* never */
+ { BIT(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG),
+ { __GFP_DIRECT_RECLAIM | __GFP_NORETRY,
+ __GFP_DIRECT_RECLAIM } }, /* always */
+ { BIT(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG),
+ { __GFP_KSWAPD_RECLAIM, __GFP_KSWAPD_RECLAIM } }, /* defer */
+ /* defer+madvise */
+ { BIT(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG),
+ { __GFP_KSWAPD_RECLAIM, __GFP_DIRECT_RECLAIM } },
+ /* madvise */
+ { BIT(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG),
+ { 0, __GFP_DIRECT_RECLAIM } },
+ };
+ const gfp_t behavior_mask = __GFP_RECLAIM | __GFP_NORETRY |
+ __GFP_RETRY_MAYFAIL | __GFP_NOFAIL;
+ const gfp_t required = __GFP_COMP | __GFP_NOMEMALLOC | __GFP_NOWARN;
+ struct vm_area_struct vma = { };
+ unsigned int i, advised;
+ gfp_t gfp;
+
+ for (i = 0; i < ARRAY_SIZE(cases); i++) {
+ for (advised = 0; advised < 2; advised++) {
+ vm_flags_init(&vma, advised ? VM_HUGEPAGE : 0);
+ /* The historical regression used an order-four span. */
+ gfp = memcomp_test_fault_alloc_with_policy(&vma, 16,
+ cases[i].policy);
+ KUNIT_EXPECT_EQ_MSG(test, gfp & behavior_mask,
+ cases[i].behavior[advised],
+ "policy %u advised %u", i, advised);
+ KUNIT_EXPECT_EQ(test, gfp & required, required);
+ KUNIT_EXPECT_EQ(test, gfp & ~behavior_mask,
+ GFP_TRANSHUGE_LIGHT & ~behavior_mask);
+ gfp = memcomp_test_fault_alloc_with_policy(&vma, 1,
+ cases[i].policy);
+ KUNIT_EXPECT_EQ(test, gfp, GFP_HIGHUSER_MOVABLE);
+ }
+ }
+#else
+ kunit_skip(test, "THP defrag requires CONFIG_TRANSPARENT_HUGEPAGE");
+#endif
+}
+
+static void memcomp_test_fault_alloc_single_fallback(struct kunit *test)
+{
+ struct vm_area_struct vma = { };
+ gfp_t large, single;
+
+ vm_flags_init(&vma, VM_HUGEPAGE);
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ /* MADV_HUGEPAGE obeys never; order-0 fallback permits reclaim. */
+ large = memcomp_test_fault_alloc_with_policy(&vma, 16, 0);
+ single = memcomp_test_fault_alloc_with_policy(&vma, 1, 0);
+ KUNIT_EXPECT_EQ(test, large & __GFP_RECLAIM, (gfp_t)0);
+#else
+ large = memcomp_fault_alloc_gfp(&vma, 16);
+ single = memcomp_fault_alloc_gfp(&vma, 1);
+ KUNIT_EXPECT_EQ(test, large,
+ GFP_HIGHUSER_MOVABLE | __GFP_NORETRY | __GFP_NOWARN);
+#endif
+ KUNIT_EXPECT_EQ(test, single, GFP_HIGHUSER_MOVABLE);
+ KUNIT_EXPECT_NE(test, large, single);
+ KUNIT_EXPECT_EQ(test, single & __GFP_RECLAIM,
+ GFP_HIGHUSER_MOVABLE & __GFP_RECLAIM);
+ KUNIT_EXPECT_EQ(test, single & (__GFP_COMP | __GFP_NOMEMALLOC |
+ __GFP_NOWARN | __GFP_NORETRY), (gfp_t)0);
+}
+
+#endif /* _MM_MEMCOMP_TESTS_FAULT_ALLOC_TEST_H */
diff --git a/mm/memcomp/tests/fault_test.h b/mm/memcomp/tests/fault_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..789362b732660e8d9db0961c10af7510ff46b87c
--- /dev/null
+++ b/mm/memcomp/tests/fault_test.h
@@ -0,0 +1,851 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_FAULT_TEST_H
+#define _MM_MEMCOMP_TESTS_FAULT_TEST_H
+
+/*
+ * Include after zero_load_test.h and refs/payload/owner test fixtures.
+ * Real mmap, PTL, folio allocation/charge, codec/zs and anon rmap. Only the
+ * existing source objcg is modeled. No swap device or public dispatch is used.
+ */
+#if defined(MC16_ZERO_LOAD_MM_TEST_ENABLED) && defined(CONFIG_MEMCG)
+#define MC16_FAULT_MM_TEST_ENABLED
+#define MC16_FT_PAGES 4
+
+struct mc16_ft {
+ struct kunit *test;
+ struct mc16_test_refs *refs;
+ struct mm_struct *mm;
+ unsigned long mapping;
+ unsigned long addr;
+ vm_flags_t flags;
+ pgprot_t prot;
+ bool saved_vma;
+ struct mc16_zero_owners zero;
+ s64 swapents;
+ s64 anonpages;
+ u32 tokens[MC16_FT_PAGES];
+ unsigned int nr;
+ struct folio *pins[MC16_FT_PAGES];
+ unsigned int nr_pins;
+ struct mc16_payload prepared[MC16_FT_PAGES];
+ struct mc16_payload_tail tail[MC16_FT_PAGES];
+ u8 *src;
+ int actor_error;
+ bool actor_ran;
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ bool saved_orders;
+ unsigned long orders;
+#endif
+};
+
+/* The inherited source cleanup expects a locked folio; lock in this task. */
+static void mc16_ft_objcg_cleanup(void *data)
+{
+ struct memcomp_test_objcg_fixture *owner = data;
+
+ if (owner->folio)
+ folio_lock(owner->folio);
+ memcomp_test_objcg_cleanup(owner);
+}
+
+static int mc16_ft_storage(struct mc16_ft *f)
+{
+ struct kunit *test = f->test;
+ struct mc16_test_storage *s;
+ struct memcomp_test_objcg_fixture *owner;
+ struct folio *source;
+ int ret;
+
+ f->refs = kunit_kzalloc(test, sizeof(*f->refs), GFP_KERNEL);
+ s = kunit_kzalloc(test, sizeof(*s), GFP_KERNEL);
+ if (!f->refs || !s)
+ return -ENOMEM;
+ f->refs->storage = s;
+ owner = memcomp_test_objcg_begin(test, 1);
+ if (!owner)
+ return -ENOMEM;
+ folio_unlock(owner->folio);
+ kunit_remove_action(test, memcomp_test_objcg_cleanup, owner);
+ ret = kunit_add_action_or_reset(test, mc16_ft_objcg_cleanup, owner);
+ if (ret)
+ return ret;
+ /* A genuine four-page source for the immutable order-2 span fixture. */
+ source = folio_alloc(GFP_KERNEL, 2);
+ if (!source)
+ return -ENOMEM;
+ folio_put(owner->folio);
+ owner->folio = source;
+ owner->model.folio = source;
+ owner->nr_pages = MC16_FT_PAGES;
+ s->owner = owner;
+ s->test = test;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ mc16_test_metadata_snapshot(&s->baseline);
+#endif
+ s->probe.task = current;
+ ret = kunit_add_action_or_reset(test, mc16_test_storage_cleanup, s);
+ if (ret)
+ return ret;
+ rcu_assign_pointer(mc16_storage_fixture, &s->probe);
+ f->refs->stored_before = atomic64_read(&memcomp_stored_bytes);
+ f->refs->compressed_before = atomic64_read(&memcomp_compressed_bytes);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ f->refs->pending_before = atomic64_read(&memcomp_pending_pages);
+#endif
+ return kunit_add_action_or_reset(test, mc16_test_refs_cleanup, f->refs);
+}
+
+static void mc16_ft_zero_unchanged(struct mc16_ft *f)
+{
+ struct mc16_zero_owners after;
+
+ mc16_zero_owners_read(&after);
+ KUNIT_EXPECT_EQ(f->test, after.pte_pages, f->zero.pte_pages);
+ KUNIT_EXPECT_EQ(f->test, after.cache_pages, f->zero.cache_pages);
+}
+
+/* Caller holds mmap_read_lock; a removed positive PTE owns exactly one ref. */
+static int mc16_ft_zap(struct mc16_ft *f, unsigned int index)
+{
+ unsigned long addr = f->addr + index * PAGE_SIZE;
+ spinlock_t *ptl; /* Actual PTE removal and token ownership transaction. */
+ struct memcomp_ref ref;
+ pte_t *ptep, pte;
+ pmd_t *pmd;
+
+ pmd = mm_find_pmd(f->mm, addr);
+ if (!pmd)
+ return -ENOENT;
+ ptep = pte_offset_map_lock(f->mm, pmd, addr, &ptl);
+ if (!ptep)
+ return -ENOENT;
+ pte = ptep_get(ptep);
+ if (is_swap_pte(pte) && is_memcomp_entry(pte_to_swp_entry(pte))) {
+ unsigned long token = swp_offset(pte_to_swp_entry(pte));
+
+ ptep_get_and_clear(f->mm, addr, ptep);
+ add_mm_counter(f->mm, MM_SWAPENTS, -1);
+ if (!token)
+ mc16_zero_pte_put(1);
+ else if (mc16_get(token, &ref))
+ mc16_ref_put_count(&ref, 2);
+ }
+ pte_unmap_unlock(ptep, ptl);
+ return 0;
+}
+
+static void mc16_ft_cleanup(void *data)
+{
+ struct mc16_ft *f = data;
+ struct mc16_test_storage *s = f->refs->storage;
+ struct vm_area_struct *vma;
+ unsigned int i;
+ int ret;
+
+ mc16_fault_probe = (typeof(mc16_fault_probe)) { };
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ if (f->saved_orders)
+ WRITE_ONCE(huge_anon_orders_always, f->orders);
+#endif
+ mmap_read_lock(f->mm);
+ for (i = 0; i < MC16_FT_PAGES; i++)
+ mc16_ft_zap(f, i);
+ mmap_read_unlock(f->mm);
+ for (i = 0; i < s->nr; i++) {
+ if (!s->refs[i].slot)
+ continue;
+ mc16_payload_finish(&s->refs[i], &f->tail[i]);
+ mc16_payload_abort(&s->refs[i], &f->prepared[i]);
+ }
+ mmap_write_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (vma && f->saved_vma) {
+ vma_start_write(vma);
+ vm_flags_reset(vma, f->flags);
+ vma->vm_page_prot = f->prot;
+ }
+ mmap_write_unlock(f->mm);
+ /* Present pages use normal MM rmap/LRU cleanup, after token PTE removal. */
+ ret = mc16_test_dontneed(f->mm, f->mapping, 2 * MC16_FT_PAGES * PAGE_SIZE);
+ KUNIT_EXPECT_EQ(f->test, ret, 0);
+ for (i = 0; i < f->nr_pins; i++)
+ folio_put(f->pins[i]);
+ KUNIT_EXPECT_EQ(f->test, mc16_zl_rss(f->mm, MM_SWAPENTS), f->swapents);
+ KUNIT_EXPECT_EQ(f->test, mc16_zl_rss(f->mm, MM_ANONPAGES), f->anonpages);
+ mc16_ft_zero_unchanged(f);
+ mmput(f->mm);
+}
+
+static int mc16_ft_prepare(struct mc16_ft *f, unsigned int index, bool raw)
+{
+ struct mc16_test_storage *s = f->refs->storage;
+ struct memcomp_ref *ref = &s->refs[index];
+ struct mc16_payload *p = &f->prepared[index];
+ unsigned long handle;
+ unsigned int bytes;
+ u8 pool = 0;
+ int ret;
+
+ if (raw)
+ ret = memcomp_alloc_raw_payload(0, f->src + index * PAGE_SIZE,
+ PAGE_SIZE, &bytes, &handle, false, NULL);
+ else
+ ret = memcomp_prepare_payload(f->src + index * PAGE_SIZE, PAGE_SIZE,
+ &bytes, &handle, &pool,
+ MEMCOMP_PAYLOAD_MUST_SUCCEED, false);
+ if (ret)
+ return ret;
+ if (pool) {
+ memcomp_zs_free(pool, handle);
+ return -EINVAL;
+ }
+ *p = (struct mc16_payload) { .word = handle, .bytes = bytes };
+ if (!bytes || (!raw && bytes >= PAGE_SIZE))
+ return -E2BIG;
+ ret = mc16_payload_charge(ref, p);
+ if (ret)
+ return ret;
+ folio_lock(s->owner->folio);
+ mutex_lock(memcomp_lock_of(ref->token));
+ ret = mc16_payload_commit_locked(ref, s->owner->folio, f->nr, index,
+ p, &f->tail[index]);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ mc16_payload_finish(ref, &f->tail[index]);
+ folio_unlock(s->owner->folio);
+ return ret;
+}
+
+static pte_t mc16_ft_orig(struct mc16_ft *f, unsigned int index)
+{
+ return pte_swp_mksoft_dirty(swp_entry_to_pte(make_memcomp_entry(f->tokens[index])));
+}
+
+static int mc16_ft_install(struct mc16_ft *f, unsigned int index, pte_t pte)
+{
+ unsigned long addr = f->addr + index * PAGE_SIZE;
+ spinlock_t *ptl; /* Actual PTE owns a transferred mc16_get reference. */
+ struct memcomp_ref ref;
+ pte_t *ptep;
+ int ret = 0;
+
+ mmap_read_lock(f->mm);
+ ptep = get_locked_pte(f->mm, addr, &ptl);
+ if (!ptep) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ __acquire(RCU);
+ if (!pte_none(ptep_get(ptep))) {
+ ret = -EEXIST;
+ } else if (!mc16_get(swp_offset(pte_to_swp_entry(pte)), &ref)) {
+ ret = -ENOENT;
+ } else {
+ set_pte_at(f->mm, addr, ptep, pte);
+ add_mm_counter(f->mm, MM_SWAPENTS, 1);
+ /* ref ownership is now the installed PTE's. */
+ }
+ pte_unmap_unlock(ptep, ptl);
+out:
+ mmap_read_unlock(f->mm);
+ return ret;
+}
+
+static struct mc16_ft *mc16_ft_begin(struct kunit *test, unsigned int nr,
+ bool raw, bool normalize)
+{
+ struct mc16_ft *f;
+ struct mc16_test_storage *s;
+ struct vm_area_struct *vma;
+ unsigned int i;
+ int ret;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->test = test;
+ f->nr = nr;
+ if (mc16_ft_storage(f))
+ return NULL;
+ s = f->refs->storage;
+ f->mapping = kunit_vm_mmap(test, NULL, 0, 2 * MC16_FT_PAGES * PAGE_SIZE,
+ PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, 0);
+ if (!f->mapping || IS_ERR_VALUE(f->mapping))
+ return NULL;
+ f->addr = ALIGN(f->mapping, MC16_FT_PAGES * PAGE_SIZE);
+ f->mm = current->mm;
+ mmget(f->mm);
+ f->swapents = mc16_zl_rss(f->mm, MM_SWAPENTS);
+ f->anonpages = mc16_zl_rss(f->mm, MM_ANONPAGES);
+ mc16_zero_owners_read(&f->zero);
+ if (kunit_add_action_or_reset(test, mc16_ft_cleanup, f))
+ return NULL;
+ mmap_read_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (vma) {
+ f->flags = vma->vm_flags;
+ f->prot = vma->vm_page_prot;
+ f->saved_vma = true;
+ }
+ ret = vma ? anon_vma_prepare(vma) : -ENOENT;
+ mmap_read_unlock(f->mm);
+ if (ret)
+ return NULL;
+ f->src = kunit_kmalloc(test, nr * PAGE_SIZE, GFP_KERNEL);
+ if (!f->src)
+ return NULL;
+ for (i = 0; i < nr * PAGE_SIZE; i++)
+ f->src[i] = raw ? i * 37 + (i >> 5) : (i + i / PAGE_SIZE) % 53;
+ folio_lock(s->owner->folio);
+ ret = mc16_test_refs_reserve(f->refs, nr);
+ folio_unlock(s->owner->folio);
+ if (ret)
+ return NULL;
+ for (i = 0; i < nr; i++) {
+ f->tokens[i] = s->refs[i].token;
+ if (mc16_ft_prepare(f, i, raw) || mc16_ft_install(f, i, mc16_ft_orig(f, i)))
+ return NULL;
+ }
+ if (normalize) {
+ folio_lock(s->owner->folio);
+ for (i = nr; i; ) {
+ struct memcomp_ref ref;
+
+ i--;
+ if (!mc16_get(f->tokens[i], &ref)) {
+ ret = -ENOENT;
+ break;
+ }
+ if (!mc16_owner_drop_folio_held(&ref, s->owner->folio, nr, i)) {
+ ret = -ESTALE;
+ break;
+ }
+ s->refs[i] = (struct memcomp_ref) { };
+ }
+ folio_unlock(s->owner->folio);
+ if (ret)
+ return NULL;
+ }
+ return f;
+}
+
+/* All vm_fault const fields are initialized in the aggregate, as 6.12 needs. */
+static vm_fault_t mc16_ft_fault(struct kunit *test, struct mm_struct *mm,
+ unsigned long addr, pte_t orig, unsigned int flags)
+{
+ struct vm_area_struct *vma;
+ vm_fault_t ret = VM_FAULT_SIGBUS;
+ bool locked = true;
+
+ mmap_read_lock(mm);
+ vma = vma_lookup(mm, addr);
+ if (vma) {
+ struct vm_fault vmf = {
+ .vma = vma,
+ .address = addr,
+ .real_address = addr,
+ .flags = flags | FAULT_FLAG_ORIG_PTE_VALID,
+ .pmd = mm_find_pmd(mm, addr),
+ .orig_pte = orig,
+ .gfp_mask = GFP_HIGHUSER_MOVABLE,
+ };
+
+ ret = mc16_load(&vmf, &locked);
+ KUNIT_EXPECT_PTR_EQ(test, vmf.pte, NULL);
+ }
+ mmap_read_unlock(mm);
+ KUNIT_EXPECT_NOT_NULL(test, vma);
+ KUNIT_EXPECT_FALSE(test, locked);
+ return ret;
+}
+
+static pte_t mc16_ft_read(struct mc16_ft *f, unsigned int index)
+{
+ unsigned long addr = f->addr + index * PAGE_SIZE;
+ spinlock_t *ptl; /* Pin present folios under PTL, including ASSERT exits. */
+ pte_t *ptep, pte = __pte(0);
+ pmd_t *pmd;
+ unsigned int i;
+
+ mmap_read_lock(f->mm);
+ pmd = mm_find_pmd(f->mm, addr);
+ if (!pmd)
+ goto out;
+ ptep = pte_offset_map_lock(f->mm, pmd, addr, &ptl);
+ if (!ptep)
+ goto out;
+ pte = ptep_get(ptep);
+ if (pte_present(pte) && !pte_special(pte)) {
+ struct folio *folio = page_folio(pte_page(pte));
+
+ for (i = 0; i < f->nr_pins; i++) {
+ if (f->pins[i] == folio)
+ break;
+ }
+ if (i == f->nr_pins && i < ARRAY_SIZE(f->pins)) {
+ folio_get(folio);
+ f->pins[f->nr_pins++] = folio;
+ }
+ }
+ pte_unmap_unlock(ptep, ptl);
+out:
+ mmap_read_unlock(f->mm);
+ return pte;
+}
+
+static void mc16_ft_counts(struct mc16_ft *f, unsigned int swapped,
+ unsigned int present)
+{
+ KUNIT_EXPECT_EQ(f->test, mc16_zl_rss(f->mm, MM_SWAPENTS), f->swapents + swapped);
+ KUNIT_EXPECT_EQ(f->test, mc16_zl_rss(f->mm, MM_ANONPAGES), f->anonpages + present);
+ mc16_ft_zero_unchanged(f);
+}
+
+static void mc16_ft_present(struct mc16_ft *f, unsigned int index,
+ unsigned int order, unsigned int mapped)
+{
+ pte_t pte = mc16_ft_read(f, index);
+ struct page *page;
+ struct folio *folio;
+ unsigned int i;
+ void *data;
+ bool pinned = false;
+
+ KUNIT_ASSERT_TRUE(f->test, pte_present(pte));
+ KUNIT_ASSERT_FALSE(f->test, pte_special(pte));
+ page = pte_page(pte);
+ folio = page_folio(page);
+ for (i = 0; i < f->nr_pins; i++)
+ pinned |= f->pins[i] == folio;
+ KUNIT_ASSERT_TRUE(f->test, pinned);
+ KUNIT_EXPECT_EQ(f->test, folio_order(folio), order);
+ KUNIT_EXPECT_EQ(f->test, folio_mapcount(folio), mapped);
+ KUNIT_EXPECT_TRUE(f->test, folio_test_anon(folio));
+ KUNIT_EXPECT_TRUE(f->test, folio_test_swapbacked(folio));
+ KUNIT_EXPECT_TRUE(f->test, folio_test_uptodate(folio));
+ KUNIT_EXPECT_FALSE(f->test, folio_test_locked(folio));
+ KUNIT_EXPECT_TRUE(f->test, PageAnonExclusive(page));
+ data = kmap_local_page(page);
+ KUNIT_EXPECT_EQ(f->test, memcmp(data, f->src + index * PAGE_SIZE, PAGE_SIZE), 0);
+ kunmap_local(data);
+}
+
+static void mc16_ft_token_refs(struct mc16_ft *f, unsigned int index,
+ unsigned int owners)
+{
+ struct memcomp_ref ref;
+ bool found = mc16_get(f->tokens[index], &ref);
+
+ KUNIT_EXPECT_EQ(f->test, found, owners != 0);
+ if (found) {
+ KUNIT_EXPECT_EQ(f->test, refcount_read(&ref.slot->refs), owners + 1);
+ mc16_ref_put(&ref);
+ }
+}
+
+static void mc16_ft_probe_start(struct mc16_ft *f)
+{
+ mc16_fault_probe = (typeof(mc16_fault_probe)) { .task = current, .data = f };
+}
+
+static void mc16_ft_policy(struct mc16_ft *f, vm_flags_t set, vm_flags_t clear)
+{
+ struct vm_area_struct *vma;
+
+ mmap_write_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (vma) {
+ vma_start_write(vma);
+ vm_flags_reset(vma, (vma->vm_flags | set) & ~clear);
+ vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+ }
+ mmap_write_unlock(f->mm);
+ KUNIT_EXPECT_NOT_NULL(f->test, vma);
+}
+
+static void mc16_ft_single(struct kunit *test, bool raw, bool attached)
+{
+ struct mc16_ft *f = mc16_ft_begin(test, 1, raw, !attached);
+ pte_t orig, pte;
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ orig = mc16_ft_orig(f, 0);
+ mc16_ft_counts(f, 1, 0);
+ ret = mc16_ft_fault(test, f->mm, f->addr, orig, FAULT_FLAG_WRITE);
+ KUNIT_ASSERT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)(VM_FAULT_NOPAGE | VM_FAULT_MAJOR));
+ mc16_ft_present(f, 0, 0, 1);
+ pte = mc16_ft_read(f, 0);
+ KUNIT_EXPECT_TRUE(test, pte_write(pte));
+ KUNIT_EXPECT_TRUE(test, pte_dirty(pte));
+ if (pte_swp_soft_dirty(orig))
+ KUNIT_EXPECT_TRUE(test, pte_soft_dirty(pte));
+ mc16_ft_counts(f, 0, 1);
+ mc16_ft_token_refs(f, 0, attached ? 1 : 0);
+}
+
+static void memcomp_test_fault_single(struct kunit *test)
+{
+ mc16_ft_single(test, false, false);
+}
+
+static void memcomp_test_fault_raw(struct kunit *test)
+{
+ mc16_ft_single(test, true, false);
+}
+
+static void memcomp_test_fault_attached(struct kunit *test)
+{
+ mc16_ft_single(test, false, true);
+}
+
+static void memcomp_test_fault_shared(struct kunit *test)
+{
+ struct mc16_ft *f = mc16_ft_begin(test, 1, false, true);
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_ft_install(f, 1, mc16_ft_orig(f, 0)), 0);
+ ret = mc16_ft_fault(test, f->mm, f->addr, mc16_ft_orig(f, 0), FAULT_FLAG_WRITE);
+ KUNIT_ASSERT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)(VM_FAULT_NOPAGE | VM_FAULT_MAJOR));
+ mc16_ft_present(f, 0, 0, 1);
+ KUNIT_EXPECT_TRUE(test, is_swap_pte(mc16_ft_read(f, 1)));
+ mc16_ft_token_refs(f, 0, 1);
+ mc16_ft_counts(f, 1, 1);
+}
+
+/* Caller owns mmap_read_lock, and updates flags of the same nonpresent owner. */
+static int mc16_ft_change(struct mc16_ft *f, unsigned int index, pte_t pte)
+{
+ unsigned long addr = f->addr + index * PAGE_SIZE;
+ spinlock_t *ptl; /* Only flags change; ownership and swap RSS are neutral. */
+ pte_t *ptep, old;
+ pmd_t *pmd = mm_find_pmd(f->mm, addr);
+ int ret = -ENOENT;
+
+ if (!pmd)
+ return ret;
+ ptep = pte_offset_map_lock(f->mm, pmd, addr, &ptl);
+ if (!ptep)
+ return ret;
+ old = ptep_get(ptep);
+ if (is_swap_pte(old) && is_memcomp_entry(pte_to_swp_entry(old)) &&
+ swp_offset(pte_to_swp_entry(old)) == swp_offset(pte_to_swp_entry(pte))) {
+ set_pte_at(f->mm, addr, ptep, pte);
+ ret = 0;
+ }
+ pte_unmap_unlock(ptep, ptl);
+ return ret;
+}
+
+static void mc16_ft_protection(struct kunit *test, unsigned int policy)
+{
+ struct mc16_ft *f;
+ pte_t orig, pte;
+ vm_fault_t ret;
+ int err;
+
+ if (policy == 1 && !IS_ENABLED(CONFIG_MEM_SOFT_DIRTY)) {
+ kunit_skip(test, "soft-dirty tracking disabled");
+ return;
+ }
+ orig = pte_swp_mkuffd_wp(mc16_zl_zero_pte());
+ if (policy == 2 && (!IS_ENABLED(CONFIG_USERFAULTFD) || !pte_swp_uffd_wp(orig))) {
+ kunit_skip(test, "swap UFFD-WP unavailable");
+ return;
+ }
+ f = mc16_ft_begin(test, 1, false, true);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ orig = mc16_ft_orig(f, 0);
+ if (policy == 1) {
+ orig = pte_swp_clear_soft_dirty(orig);
+ mc16_ft_policy(f, 0, VM_SOFTDIRTY);
+ } else if (policy == 2) {
+ orig = pte_swp_mkuffd_wp(orig);
+ mc16_ft_policy(f, VM_UFFD_WP, 0);
+ } else if (policy == 3) {
+ mc16_ft_policy(f, 0, VM_WRITE);
+ }
+ mmap_read_lock(f->mm);
+ err = mc16_ft_change(f, 0, orig);
+ mmap_read_unlock(f->mm);
+ KUNIT_ASSERT_EQ(test, err, 0);
+ ret = mc16_ft_fault(test, f->mm, f->addr, orig, 0);
+ KUNIT_ASSERT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)(VM_FAULT_NOPAGE | VM_FAULT_MAJOR));
+ mc16_ft_present(f, 0, 0, 1);
+ pte = mc16_ft_read(f, 0);
+ if (policy) {
+ KUNIT_EXPECT_FALSE(test, pte_write(pte));
+ KUNIT_EXPECT_FALSE(test, pte_dirty(pte));
+ }
+ if (policy == 1)
+ KUNIT_EXPECT_FALSE(test, pte_soft_dirty(pte));
+ else if (pte_swp_soft_dirty(orig))
+ KUNIT_EXPECT_TRUE(test, pte_soft_dirty(pte));
+ if (policy == 2)
+ KUNIT_EXPECT_TRUE(test, pte_uffd_wp(pte));
+ mc16_ft_counts(f, 0, 1);
+ mc16_ft_token_refs(f, 0, 0);
+}
+
+static void memcomp_test_fault_softdirty(struct kunit *test)
+{
+ if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY)) {
+ kunit_skip(test, "soft-dirty tracking disabled");
+ return;
+ }
+ mc16_ft_protection(test, 0);
+}
+
+static void memcomp_test_fault_softdirty_wp(struct kunit *test)
+{
+ mc16_ft_protection(test, 1);
+}
+
+static void memcomp_test_fault_uffd_wp(struct kunit *test)
+{
+ mc16_ft_protection(test, 2);
+}
+
+static void memcomp_test_fault_readonly(struct kunit *test)
+{
+ mc16_ft_protection(test, 3);
+}
+
+static void mc16_ft_race(struct vm_fault *vmf, void *data)
+{
+ struct mc16_ft *f = data;
+
+ mc16_fault_probe.before_install = NULL;
+ f->actor_ran = true;
+ f->actor_error = mc16_ft_change(f, 0, pte_swp_mkexclusive(vmf->orig_pte));
+}
+
+static void memcomp_test_fault_changed(struct kunit *test)
+{
+ struct mc16_ft *f = mc16_ft_begin(test, 1, false, true);
+ pte_t orig;
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ orig = mc16_ft_orig(f, 0);
+ KUNIT_ASSERT_FALSE(test, pte_same(orig, pte_swp_mkexclusive(orig)));
+ mc16_ft_probe_start(f);
+ mc16_fault_probe.before_install = mc16_ft_race;
+ ret = mc16_ft_fault(test, f->mm, f->addr, orig, 0);
+ WRITE_ONCE(mc16_fault_probe.task, NULL);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_MAJOR);
+ KUNIT_EXPECT_TRUE(test, f->actor_ran);
+ KUNIT_EXPECT_EQ(test, f->actor_error, 0);
+ KUNIT_EXPECT_TRUE(test,
+ pte_same(mc16_ft_read(f, 0), pte_swp_mkexclusive(orig)));
+ mc16_ft_counts(f, 1, 0);
+ mc16_ft_token_refs(f, 0, 1);
+}
+
+static void mc16_ft_oom(struct kunit *test, bool charge)
+{
+ struct mc16_ft *f = mc16_ft_begin(test, 1, false, true);
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_ft_probe_start(f);
+ mc16_fault_probe.fail_small_alloc = !charge;
+ mc16_fault_probe.fail_small_charge = charge;
+ ret = mc16_ft_fault(test, f->mm, f->addr, mc16_ft_orig(f, 0), 0);
+ WRITE_ONCE(mc16_fault_probe.task, NULL);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_OOM);
+ KUNIT_EXPECT_EQ(test, mc16_fault_probe.small_allocs, 1U);
+ KUNIT_EXPECT_EQ(test, mc16_fault_probe.small_charges, charge ? 1U : 0U);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_ft_read(f, 0), mc16_ft_orig(f, 0)));
+ mc16_ft_token_refs(f, 0, 1);
+ mc16_ft_counts(f, 1, 0);
+}
+
+static void memcomp_test_fault_alloc_oom(struct kunit *test)
+{
+ mc16_ft_oom(test, false);
+}
+
+static void memcomp_test_fault_charge_oom(struct kunit *test)
+{
+ mc16_ft_oom(test, true);
+}
+
+/* Restore intentionally invalid metadata before any assertion or final put. */
+static void mc16_ft_load_error(struct kunit *test, bool retry)
+{
+ struct mc16_ft *f = mc16_ft_begin(test, 1, false, true);
+ struct memcomp_ref ref;
+ struct mc16_view view;
+ vm_fault_t ret;
+ u16 saved;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_get(f->tokens[0], &ref));
+ mutex_lock(memcomp_lock_of(ref.token));
+ mc16_view_locked(&ref, &view);
+ saved = *view.flags;
+ *view.flags |= retry ? MC16_F_REFRESH_PENDING : BIT(15);
+ mutex_unlock(memcomp_lock_of(ref.token));
+ ret = mc16_ft_fault(test, f->mm, f->addr, mc16_ft_orig(f, 0), 0);
+ mutex_lock(memcomp_lock_of(ref.token));
+ mc16_view_locked(&ref, &view);
+ *view.flags = saved;
+ mutex_unlock(memcomp_lock_of(ref.token));
+ mc16_ref_put(&ref);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ retry ? 0U : (__force unsigned int)VM_FAULT_SIGBUS);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_ft_read(f, 0), mc16_ft_orig(f, 0)));
+ mc16_ft_token_refs(f, 0, 1);
+ mc16_ft_counts(f, 1, 0);
+}
+
+static void memcomp_test_fault_payload_retry(struct kunit *test)
+{
+ mc16_ft_load_error(test, true);
+}
+
+static void memcomp_test_fault_payload_invalid(struct kunit *test)
+{
+ mc16_ft_load_error(test, false);
+}
+
+static bool mc16_ft_large_enable(struct mc16_ft *f)
+{
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ struct vm_area_struct *vma;
+ unsigned long orders = 0;
+
+ if (!(THP_ORDERS_ALL_ANON & BIT(2))) {
+ kunit_skip(f->test, "order-2 anonymous THP unsupported");
+ return false;
+ }
+ f->orders = READ_ONCE(huge_anon_orders_always);
+ f->saved_orders = true;
+ WRITE_ONCE(huge_anon_orders_always, f->orders | BIT(2));
+ mc16_ft_policy(f, VM_HUGEPAGE, VM_NOHUGEPAGE);
+ mmap_read_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (vma) {
+ orders = thp_vma_allowable_orders(vma, vma->vm_flags,
+ TVA_IN_PF | TVA_ENFORCE_SYSFS,
+ BIT(2));
+ orders = thp_vma_suitable_orders(vma, f->addr + 2 * PAGE_SIZE, orders);
+ }
+ mmap_read_unlock(f->mm);
+ if (!(orders & BIT(2))) {
+ kunit_skip(f->test, "real VMA policy disallows order-2 THP");
+ return false;
+ }
+ return true;
+#else
+ kunit_skip(f->test, "THP disabled");
+ return false;
+#endif
+}
+
+static void mc16_ft_sibling_gone(struct vm_fault *vmf, void *data)
+{
+ struct mc16_ft *f = data;
+
+ mc16_fault_probe.after_alloc = NULL;
+ f->actor_ran = true;
+ f->actor_error = mc16_ft_zap(f, 3);
+}
+
+static void mc16_ft_large(struct kunit *test, unsigned int fallback)
+{
+ struct mc16_ft *f = mc16_ft_begin(test, MC16_FT_PAGES, false, true);
+ unsigned int i, present = fallback ? 1 : MC16_FT_PAGES;
+ vm_fault_t ret;
+ int err = 0;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ if (!mc16_ft_large_enable(f))
+ return;
+ mc16_ft_probe_start(f);
+ mc16_fault_probe.fail_large_alloc = fallback == 1;
+ mc16_fault_probe.fail_large_charge = fallback == 2;
+ if (fallback == 3) {
+ mmap_read_lock(f->mm);
+ err = mc16_ft_change(f, 1, pte_swp_mkexclusive(mc16_ft_orig(f, 1)));
+ mmap_read_unlock(f->mm);
+ }
+ if (fallback == 4)
+ mc16_fault_probe.after_alloc = mc16_ft_sibling_gone;
+ KUNIT_ASSERT_EQ(test, err, 0);
+ /* Fault in the middle: wrong fault_index movement must fail this test. */
+ ret = mc16_ft_fault(test, f->mm, f->addr + 2 * PAGE_SIZE,
+ mc16_ft_orig(f, 2), FAULT_FLAG_WRITE);
+ WRITE_ONCE(mc16_fault_probe.task, NULL);
+ KUNIT_ASSERT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)(VM_FAULT_NOPAGE | VM_FAULT_MAJOR));
+ KUNIT_EXPECT_EQ(test, mc16_fault_probe.large_allocs, fallback == 3 ? 0U : 1U);
+ KUNIT_EXPECT_EQ(test, mc16_fault_probe.small_allocs, fallback ? 1U : 0U);
+ if (fallback == 2)
+ KUNIT_EXPECT_EQ(test, mc16_fault_probe.large_charges, 1U);
+ if (fallback == 4) {
+ KUNIT_EXPECT_TRUE(test, f->actor_ran);
+ KUNIT_EXPECT_EQ(test, f->actor_error, 0);
+ KUNIT_EXPECT_TRUE(test, pte_none(mc16_ft_read(f, 3)));
+ }
+ for (i = 0; i < MC16_FT_PAGES; i++) {
+ if (!fallback || i == 2) {
+ mc16_ft_present(f, i, fallback ? 0 : 2, present);
+ mc16_ft_token_refs(f, i, 0);
+ } else if (fallback == 4 && i == 3) {
+ mc16_ft_token_refs(f, i, 0);
+ } else {
+ KUNIT_EXPECT_TRUE(test, is_swap_pte(mc16_ft_read(f, i)));
+ mc16_ft_token_refs(f, i, 1);
+ }
+ }
+ mc16_ft_counts(f, MC16_FT_PAGES - present - (fallback == 4), present);
+}
+
+static void memcomp_test_fault_thp(struct kunit *test)
+{
+ mc16_ft_large(test, 0);
+}
+
+static void memcomp_test_fault_thp_alloc_fallback(struct kunit *test)
+{
+ mc16_ft_large(test, 1);
+}
+
+static void memcomp_test_fault_thp_charge_fallback(struct kunit *test)
+{
+ mc16_ft_large(test, 2);
+}
+
+static void memcomp_test_fault_thp_pte_fallback(struct kunit *test)
+{
+ mc16_ft_large(test, 3);
+}
+
+static void memcomp_test_fault_thp_sibling_gone(struct kunit *test)
+{
+ mc16_ft_large(test, 4);
+}
+
+static void memcomp_test_fault_zero(struct kunit *test)
+{
+ struct mc16_zl_mm *f = mc16_zl_begin(test);
+ pte_t orig = pte_swp_mksoft_dirty(mc16_zl_zero_pte());
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_zl_install(f, orig), 0);
+ ret = mc16_ft_fault(test, f->mm, f->addr, orig, FAULT_FLAG_WRITE);
+ KUNIT_ASSERT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_NOPAGE);
+ mc16_zl_present(f, mc16_zl_read(f));
+}
+#endif /* MC16_FAULT_MM_TEST_ENABLED */
+#endif /* _MM_MEMCOMP_TESTS_FAULT_TEST_H */
diff --git a/mm/memcomp/tests/folio_test.h b/mm/memcomp/tests/folio_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..111ffc3bcaef6160bf3288af39fcb08bca9ce116
--- /dev/null
+++ b/mm/memcomp/tests/folio_test.h
@@ -0,0 +1,769 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_FOLIO_TEST_H
+#define _MM_MEMCOMP_TESTS_FOLIO_TEST_H
+
+#include <kunit/resource.h>
+#include <linux/mman.h>
+#include <linux/rmap.h>
+#include <linux/sched/mm.h>
+
+/*
+ * Author: nakixii
+ *
+ * Include after folio.h, resident.h, load.h and consumers_test.h. Register the
+ * cases below under MC16_FOLIO_TEST_ENABLED. The objcg/slot-allocation probe
+ * is the existing consumer model.
+ * Anonymous VMAs, order-0/order-1 folios, PTEs, rmap, commits and loads are real.
+ * PTE installation, partial commit and MOVE deliberately model serial states;
+ * these tests do not invoke reclaim, fork or mremap, or claim concurrent races.
+ */
+#if defined(CONFIG_MEMCG) && defined(CONFIG_MMU) && \
+ defined(CONFIG_ADVISE_SYSCALLS)
+#define MC16_FOLIO_TEST_ENABLED
+
+struct mc16_flt {
+ struct kunit *test;
+ struct mc16_consumer_test *base;
+ struct mm_struct *mm;
+ struct folio *folio;
+ struct anon_vma *anon_vma;
+ struct memcomp_ref guards[2];
+ unsigned long mapping;
+ unsigned long addr;
+ unsigned int nr;
+ s64 swapents;
+ s64 anonpages;
+};
+
+static s64 mc16_flt_rss(struct mc16_flt *f, int member)
+{
+ return percpu_counter_sum(&f->mm->rss_stat[member]);
+}
+
+/* Remove token PTEs before releasing the fixture and its modeled objcg. */
+static int mc16_flt_zap(struct mc16_flt *f, unsigned long address)
+{
+ struct memcomp_ref removed = { };
+ spinlock_t *ptl; /* Actual token PTE ownership is protected by this PTL. */
+ pte_t *ptep, pte;
+ pmd_t *pmd;
+ int ret = 0;
+
+ mmap_read_lock(f->mm);
+ pmd = mm_find_pmd(f->mm, address);
+ if (!pmd)
+ goto out;
+ ptep = pte_offset_map_lock(f->mm, pmd, address, &ptl);
+ if (!ptep)
+ goto out;
+ pte = ptep_get(ptep);
+ if (is_swap_pte(pte) && is_memcomp_entry(pte_to_swp_entry(pte))) {
+ if (!mc16_get(swp_offset(pte_to_swp_entry(pte)), &removed))
+ ret = -ENOENT;
+ ptep_get_and_clear(f->mm, address, ptep);
+ add_mm_counter(f->mm, MM_SWAPENTS, -1);
+ }
+ pte_unmap_unlock(ptep, ptl);
+ if (removed.slot)
+ mc16_ref_put_count(&removed, 2); /* Lookup and removed PTE. */
+out:
+ mmap_read_unlock(f->mm);
+ return ret;
+}
+
+/* Each cleanup acquires its own locks; it never unlocks the expired test task. */
+static void mc16_flt_cleanup(void *data)
+{
+ struct mc16_flt *f = data;
+ unsigned int i;
+ int ret;
+
+ f->base->probe.fail_at = 0;
+ f->base->account->swap.val = 0;
+ if (f->base->large)
+ f->base->large->swap.val = 0;
+ for (i = 0; i < 4; i++) {
+ ret = mc16_flt_zap(f, f->addr + i * PAGE_SIZE);
+ KUNIT_EXPECT_EQ(f->test, ret, 0);
+ }
+ if (f->folio) {
+ folio_lock(f->folio);
+ mc16_invalidate_folio(f->folio);
+ f->folio->swap.val = 0;
+ folio_unlock(f->folio);
+ }
+ for (i = 0; i < ARRAY_SIZE(f->guards); i++) {
+ if (f->guards[i].slot)
+ mc16_ref_put(&f->guards[i]);
+ }
+ ret = mc16_test_dontneed(f->mm, f->mapping, 8 * PAGE_SIZE);
+ KUNIT_EXPECT_EQ(f->test, ret, 0);
+ lru_add_drain_all();
+ if (f->folio) {
+ KUNIT_EXPECT_FALSE(f->test, folio_test_locked(f->folio));
+ KUNIT_EXPECT_EQ(f->test, folio_ref_count(f->folio), 1);
+ folio_put(f->folio);
+ }
+ if (f->anon_vma)
+ put_anon_vma(f->anon_vma);
+ f->base->model.folio = f->base->account;
+ KUNIT_EXPECT_EQ(f->test, mc16_flt_rss(f, MM_SWAPENTS), f->swapents);
+ KUNIT_EXPECT_EQ(f->test, mc16_flt_rss(f, MM_ANONPAGES), f->anonpages);
+ mmput(f->mm);
+}
+
+/*
+ * Install a real, charged anonymous folio into fresh PTEs in a real VMA.
+ * Reserve one extra reference for the fixture, independently of the nr PTEs.
+ * Materialize/check the whole PTE range before publishing any rmap or PTE.
+ */
+static int mc16_flt_map(struct mc16_flt *f, unsigned int order)
+{
+ struct vm_area_struct *vma;
+ spinlock_t *ptl; /* All PTEs in this prevalidated range share one PTL. */
+ pte_t *ptep, entry;
+ unsigned int i;
+ int ret = -ENOENT;
+ void *bytes;
+
+ f->folio = folio_alloc(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, order);
+ if (!f->folio)
+ return -ENOMEM;
+ f->nr = folio_nr_pages(f->folio);
+ ret = mem_cgroup_charge(f->folio, f->mm, GFP_KERNEL);
+ if (ret)
+ return ret;
+ for (i = 0; i < f->nr; i++) {
+ bytes = kmap_local_folio(f->folio, i * PAGE_SIZE);
+ memset(bytes, 0x6b, PAGE_SIZE);
+ kunmap_local(bytes);
+ }
+ folio_mark_uptodate(f->folio);
+ flush_dcache_folio(f->folio);
+ mmap_write_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (!vma) {
+ ret = -ENOENT;
+ goto out;
+ }
+ vma_start_write(vma);
+ vm_flags_set(vma, VM_NOHUGEPAGE);
+#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
+ vm_flags_set(vma, VM_UFFD_WP);
+#endif
+ ret = anon_vma_prepare(vma);
+ if (ret)
+ goto out;
+ ptep = get_locked_pte(f->mm, f->addr, &ptl);
+ if (!ptep) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ __acquire(RCU);
+ for (i = 0; i < f->nr; i++) {
+ if (!pte_none(ptep_get(ptep + i))) {
+ ret = -EEXIST;
+ goto out_pte;
+ }
+ }
+ entry = pte_mkwrite(pte_mkdirty(mk_pte(&f->folio->page, vma->vm_page_prot)),
+ vma);
+ folio_ref_add(f->folio, f->nr);
+ add_mm_counter(f->mm, MM_ANONPAGES, f->nr);
+ folio_add_new_anon_rmap(f->folio, vma, f->addr, RMAP_EXCLUSIVE);
+ folio_add_lru_vma(f->folio, vma);
+ set_ptes(f->mm, f->addr, ptep, entry, f->nr);
+ update_mmu_cache_range(NULL, vma, f->addr, ptep, f->nr);
+ f->anon_vma = folio_get_anon_vma(f->folio);
+ ret = f->anon_vma ? 0 : -EINVAL;
+out_pte:
+ pte_unmap_unlock(ptep, ptl);
+out:
+ mmap_write_unlock(f->mm);
+ lru_add_drain_all();
+ f->base->model.folio = f->folio;
+ return ret;
+}
+
+static struct mc16_flt *mc16_flt_begin(struct kunit *test, unsigned int order)
+{
+ struct mc16_flt *f;
+
+ if (order && !IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
+ kunit_skip(test, "large anonymous folios need CONFIG_TRANSPARENT_HUGEPAGE");
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->test = test;
+ f->base = mc16_consumer_test_begin(test);
+ if (!f->base)
+ return NULL;
+ f->mapping = kunit_vm_mmap(test, NULL, 0, 8 * PAGE_SIZE,
+ PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, 0);
+ if (!f->mapping || IS_ERR_VALUE(f->mapping))
+ return NULL;
+ /* Four adjacent pages, aligned for order 1, within one PTE table. */
+ f->addr = ALIGN(f->mapping, 2 * PAGE_SIZE);
+ if ((f->addr >> PMD_SHIFT) != ((f->addr + 3 * PAGE_SIZE) >> PMD_SHIFT))
+ f->addr = ALIGN(f->addr, PMD_SIZE);
+ f->mm = current->mm;
+ mmget(f->mm);
+ f->swapents = mc16_flt_rss(f, MM_SWAPENTS);
+ f->anonpages = mc16_flt_rss(f, MM_ANONPAGES);
+ if (kunit_add_action_or_reset(test, mc16_flt_cleanup, f))
+ return NULL;
+ if (mc16_flt_map(f, order))
+ return NULL;
+ return f;
+}
+
+static bool mc16_flt_reserve(struct mc16_flt *f)
+{
+ bool ret;
+
+ folio_lock(f->folio);
+ ret = mc16_reserve_folio(f->folio);
+ folio_unlock(f->folio);
+ return ret;
+}
+
+static void mc16_flt_drop(struct mc16_flt *f, bool rollback)
+{
+ mmap_read_lock(f->mm);
+ folio_lock(f->folio);
+ if (rollback)
+ mc16_rollback_folio(f->folio);
+ else
+ mc16_invalidate_folio(f->folio);
+ folio_unlock(f->folio);
+ mmap_read_unlock(f->mm);
+}
+
+/* Every successful release is thawed before unlocking or any test assertion. */
+static bool mc16_flt_release(struct folio *folio)
+{
+ bool ret;
+
+ /* Reclaim releases an isolated source, with only its final hold left. */
+ lru_add_drain_all();
+ if (folio_test_lru(folio)) {
+ if (!folio_isolate_lru(folio))
+ return false;
+ /* Model reclaim isolation, not migration: the source is inactive. */
+ folio_clear_active(folio);
+ folio_put(folio); /* Drop isolation's additional hold, not the fixture. */
+ }
+ folio_lock(folio);
+ ret = mc16_release_folio(folio);
+ if (ret)
+ folio_ref_unfreeze(folio, 1);
+ folio_unlock(folio);
+ return ret;
+}
+
+static int mc16_flt_hold(struct mc16_flt *f)
+{
+ unsigned long token = swp_offset(f->folio->swap);
+ unsigned int i;
+
+ for (i = 0; i < f->nr; i++) {
+ if (!mc16_get(token + i, &f->guards[i]))
+ return -ENOENT;
+ }
+ return 0;
+}
+
+/* Actual pending -> SAMEFILL commit/finish; serial partial-store model. */
+static int mc16_flt_commit(struct mc16_flt *f, unsigned int i)
+{
+ struct mc16_payload payload = { .word = 0x5a5a5a5a5a5a5a5aULL };
+ struct mc16_payload_tail tail = { };
+ struct memcomp_ref *ref = &f->guards[i];
+ int ret;
+
+ folio_lock(f->folio);
+ mutex_lock(memcomp_lock_of(ref->token));
+ ret = mc16_payload_commit_locked(ref, f->folio, f->nr, i, &payload, &tail);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ if (!ret)
+ mc16_payload_finish(ref, &tail);
+ folio_unlock(f->folio);
+ return ret;
+}
+
+static pte_t mc16_flt_read(struct mc16_flt *f, unsigned long address)
+{
+ spinlock_t *ptl; /* Only the copied PTE leaves the sampling scope. */
+ pte_t *ptep, pte = __pte(0);
+ pmd_t *pmd;
+
+ mmap_read_lock(f->mm);
+ pmd = mm_find_pmd(f->mm, address);
+ if (pmd) {
+ ptep = pte_offset_map_lock(f->mm, pmd, address, &ptl);
+ if (ptep) {
+ pte = ptep_get(ptep);
+ pte_unmap_unlock(ptep, ptl);
+ }
+ }
+ mmap_read_unlock(f->mm);
+ return pte;
+}
+
+/* Transfer one lookup to a real token PTE, retire the old mapping's folio ref. */
+static int mc16_flt_escape(struct mc16_flt *f, unsigned int i)
+{
+ struct memcomp_ref pte_ref = { };
+ struct vm_area_struct *vma;
+ spinlock_t *ptl; /* Protects the present -> token PTE transaction. */
+ unsigned long address = f->addr + i * PAGE_SIZE;
+ pte_t *ptep, old, entry;
+ int ret = -EINVAL;
+
+ mmap_read_lock(f->mm);
+ vma = vma_lookup(f->mm, address);
+ if (!vma)
+ goto out;
+ folio_lock(f->folio);
+ ptep = get_locked_pte(f->mm, address, &ptl);
+ if (!ptep) {
+ ret = -ENOMEM;
+ goto out_folio;
+ }
+ __acquire(RCU);
+ old = ptep_get(ptep);
+ if (!pte_present(old) || pte_special(old) ||
+ pte_page(old) != folio_page(f->folio, i) ||
+ !mc16_get(swp_offset(f->folio->swap) + i, &pte_ref))
+ goto out_pte;
+ entry = swp_entry_to_pte(make_memcomp_entry(pte_ref.token));
+ entry = pte_swp_mkexclusive(pte_swp_mkuffd_wp(pte_swp_mksoft_dirty(entry)));
+ ptep_clear_flush(vma, address, ptep);
+ folio_remove_rmap_pte(f->folio, folio_page(f->folio, i), vma);
+ add_mm_counter(f->mm, MM_ANONPAGES, -1);
+ add_mm_counter(f->mm, MM_SWAPENTS, 1);
+ set_pte_at(f->mm, address, ptep, entry);
+ /* This get is now owned by the PTE, not by a C fixture reference. */
+ folio_put(f->folio);
+ ret = 0;
+out_pte:
+ pte_unmap_unlock(ptep, ptl);
+out_folio:
+ folio_unlock(f->folio);
+out:
+ mmap_read_unlock(f->mm);
+ return ret;
+}
+
+static unsigned int mc16_flt_wrong_token(struct mc16_flt *f)
+{
+ struct mc16_folio_restore walk = {
+ .token = swp_offset(f->folio->swap) + f->nr,
+ };
+ struct vm_area_struct *vma;
+
+ mmap_read_lock(f->mm);
+ folio_lock(f->folio);
+ vma = vma_lookup(f->mm, f->addr);
+ if (vma)
+ mc16_folio_restore_one(f->folio, vma, f->addr, &walk);
+ folio_unlock(f->folio);
+ mmap_read_unlock(f->mm);
+ return walk.found;
+}
+
+/* Snapshot only scalar state; no record/member pointer leaves T. */
+static unsigned int mc16_flt_flags(struct memcomp_ref *ref, bool mark)
+{
+ struct mc16_view view;
+ unsigned int flags = 0;
+
+ mutex_lock(memcomp_lock_of(ref->token));
+ if (mc16_view_locked(ref, &view)) {
+ if (mark)
+ *view.flags |= MC16_F_INCOMPRESSIBLE;
+ flags = *view.flags;
+ }
+ mutex_unlock(memcomp_lock_of(ref->token));
+ return flags;
+}
+
+static void mc16_flt_expect_load(struct mc16_flt *f, unsigned int i)
+{
+ void *dst = kunit_kmalloc(f->test, PAGE_SIZE, GFP_KERNEL);
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(f->test, dst);
+ memset(dst, 0xa7, PAGE_SIZE);
+ ret = mc16_load_page(&f->guards[i], dst);
+ KUNIT_ASSERT_EQ(f->test, ret, 0);
+ KUNIT_EXPECT_PTR_EQ(f->test, memchr_inv(dst, 0x5a, PAGE_SIZE), NULL);
+}
+
+static void mc16_flt_expect_present(struct mc16_flt *f, unsigned int i,
+ unsigned long address)
+{
+ pte_t pte = mc16_flt_read(f, address);
+
+ KUNIT_ASSERT_TRUE(f->test, pte_present(pte));
+ KUNIT_ASSERT_FALSE(f->test, pte_special(pte));
+ KUNIT_EXPECT_PTR_EQ(f->test, pte_page(pte), folio_page(f->folio, i));
+#ifdef CONFIG_MEM_SOFT_DIRTY
+ KUNIT_EXPECT_TRUE(f->test, pte_soft_dirty(pte));
+#endif
+#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
+ KUNIT_EXPECT_TRUE(f->test, pte_uffd_wp(pte));
+ KUNIT_EXPECT_FALSE(f->test, pte_write(pte));
+#endif
+ KUNIT_EXPECT_TRUE(f->test, PageAnonExclusive(folio_page(f->folio, i)));
+}
+
+static void memcomp_test_folio_reserve(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 0);
+ struct memcomp_ref probe = { };
+ unsigned long token;
+ unsigned int refs;
+ bool found;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_EXPECT_FALSE(test, mc16_reserve_folio(f->folio)); /* Unlocked. */
+ KUNIT_ASSERT_TRUE(test, mc16_flt_reserve(f));
+ token = swp_offset(f->folio->swap);
+ KUNIT_ASSERT_TRUE(test, mc16_get(token, &probe));
+ refs = refcount_read(&probe.slot->refs);
+ KUNIT_EXPECT_TRUE(test, mc16_flt_reserve(f));
+ KUNIT_EXPECT_EQ(test, swp_offset(f->folio->swap), token);
+ KUNIT_EXPECT_EQ(test, refcount_read(&probe.slot->refs), refs);
+ mc16_ref_put(&probe);
+ /* No PTE or test guard escaped: rollback must retire the whole span. */
+ mc16_flt_drop(f, true);
+ found = mc16_get(token, &probe);
+ if (probe.slot)
+ mc16_ref_put(&probe);
+ KUNIT_EXPECT_FALSE(test, found);
+ KUNIT_EXPECT_EQ(test, f->folio->swap.val, 0UL);
+ KUNIT_ASSERT_TRUE(test, mc16_flt_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ KUNIT_ASSERT_EQ(test, mc16_flt_commit(f, 0), 0);
+ token = f->guards[0].token;
+ /* A stored OWNER is rolled back before a fresh reservation is published. */
+ KUNIT_EXPECT_TRUE(test, mc16_flt_reserve(f));
+ KUNIT_EXPECT_NE(test, swp_offset(f->folio->swap), token);
+ KUNIT_EXPECT_FALSE(test, mc16_flt_flags(&f->guards[0], false) & MC16_F_OWNER);
+ mc16_flt_expect_load(f, 0);
+ mc16_flt_drop(f, true);
+}
+
+static void memcomp_test_folio_reserve_failure(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 0);
+ int pins, refs;
+ bool reserved;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ pins = atomic_read(&f->anon_vma->refcount);
+ refs = folio_ref_count(f->folio);
+ f->base->probe.fail_at = f->base->probe.allocations + 1;
+ reserved = mc16_flt_reserve(f);
+ f->base->probe.fail_at = 0;
+ KUNIT_EXPECT_FALSE(test, reserved);
+ KUNIT_EXPECT_EQ(test, f->folio->swap.val, 0UL);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->anon_vma->refcount), pins);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), refs);
+ KUNIT_ASSERT_TRUE(test, mc16_flt_reserve(f));
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->anon_vma->refcount), pins + 1);
+ mc16_flt_drop(f, false);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->anon_vma->refcount), pins);
+}
+
+static void memcomp_test_folio_rollback_rmap(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 0);
+ pte_t orig;
+ u32 hash;
+ int pins;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ pins = atomic_read(&f->anon_vma->refcount);
+ KUNIT_ASSERT_TRUE(test, mc16_flt_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, 0), 0);
+ KUNIT_EXPECT_FALSE(test, folio_mapped(f->folio));
+ orig = mc16_flt_read(f, f->addr);
+ KUNIT_EXPECT_EQ(test, mc16_flt_wrong_token(f), 0U);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_flt_read(f, f->addr), orig));
+ folio_lock(f->folio);
+ mc16_flt_flags(&f->guards[0], true);
+ hash = memcomp_folio_hash(f->folio);
+ folio_unlock(f->folio);
+ mc16_flt_drop(f, true); /* Runs the real rmap_walk(), even fully unmapped. */
+ mc16_flt_expect_present(f, 0, f->addr);
+ KUNIT_EXPECT_EQ(test, mc16_flt_rss(f, MM_SWAPENTS), f->swapents);
+ KUNIT_EXPECT_EQ(test, mc16_flt_rss(f, MM_ANONPAGES), f->anonpages + 1);
+ KUNIT_EXPECT_TRUE(test, folio_test_incompressible(f->folio));
+ KUNIT_EXPECT_EQ(test, folio_incompressible_hash(f->folio), hash);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guards[0].slot->refs), 1U);
+ KUNIT_EXPECT_FALSE(test, mc16_flt_flags(&f->guards[0], false) & MC16_F_OWNER);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->anon_vma->refcount), pins);
+}
+
+static void memcomp_test_folio_partial_rollback(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 1);
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_flt_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ for (i = 0; i < f->nr; i++)
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, i), 0);
+ /* Model a batch failure after the head has committed, before sibling 1. */
+ KUNIT_ASSERT_EQ(test, mc16_flt_commit(f, 0), 0);
+ mc16_flt_drop(f, true);
+ KUNIT_EXPECT_EQ(test, f->folio->swap.val, 0UL);
+ for (i = 0; i < f->nr; i++) {
+ mc16_flt_expect_present(f, i, f->addr + i * PAGE_SIZE);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guards[i].slot->refs), 1U);
+ KUNIT_EXPECT_FALSE(test, mc16_flt_flags(&f->guards[i], false) & MC16_F_OWNER);
+ }
+ KUNIT_EXPECT_EQ(test, mc16_flt_rss(f, MM_SWAPENTS), f->swapents);
+ KUNIT_EXPECT_EQ(test, mc16_flt_rss(f, MM_ANONPAGES), f->anonpages + 2);
+ mc16_flt_expect_load(f, 0);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(mc16_flt_flags(&f->guards[1], false)),
+ MC16_PENDING);
+}
+
+static void memcomp_test_folio_partial_invalidate(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 1);
+ unsigned int i;
+ int pins;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ pins = atomic_read(&f->anon_vma->refcount);
+ KUNIT_ASSERT_TRUE(test, mc16_flt_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ KUNIT_ASSERT_EQ(test, mc16_flt_commit(f, 0), 0);
+ KUNIT_EXPECT_FALSE(test, mc16_flt_release(f->folio)); /* Mixed span. */
+ mc16_flt_drop(f, false);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->anon_vma->refcount), pins);
+ KUNIT_EXPECT_EQ(test, f->folio->swap.val, 0UL);
+ for (i = 0; i < f->nr; i++) {
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guards[i].slot->refs), 1U);
+ KUNIT_EXPECT_FALSE(test, mc16_flt_flags(&f->guards[i], false) & MC16_F_OWNER);
+ }
+ mc16_flt_expect_load(f, 0);
+ KUNIT_EXPECT_TRUE(test, mc16_flt_flags(&f->guards[1], false) & MC16_F_RESIDENT);
+}
+
+static void memcomp_test_folio_release(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 0);
+ bool released;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_flt_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, 0), 0);
+ KUNIT_ASSERT_EQ(test, mc16_flt_zap(f, f->addr), 0);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), 2); /* Fixture + resident. */
+ KUNIT_EXPECT_FALSE(test, mc16_flt_release(f->folio));
+ KUNIT_ASSERT_EQ(test, mc16_flt_commit(f, 0), 0);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), 1);
+ folio_get(f->folio);
+ released = mc16_flt_release(f->folio);
+ folio_put(f->folio);
+ KUNIT_EXPECT_FALSE(test, released); /* Extra ref prevents freeze(1). */
+ KUNIT_EXPECT_FALSE(test, mc16_release_folio(f->folio)); /* Unlocked. */
+ KUNIT_EXPECT_TRUE(test, mc16_flt_release(f->folio));
+ KUNIT_EXPECT_EQ(test, f->folio->swap.val, 0UL);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), 1);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guards[0].slot->refs), 1U);
+ mc16_flt_expect_load(f, 0);
+}
+
+static void memcomp_test_folio_transport(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 0);
+ struct folio *transport;
+ bool released;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ transport = f->base->account;
+ transport->swap = make_memcomp_entry(0);
+ KUNIT_EXPECT_FALSE(test, mc16_release_folio(transport));
+ /* Mapcount-only rejection model; restore it before any assertion/put. */
+ atomic_inc(&transport->_mapcount);
+ released = mc16_flt_release(transport);
+ atomic_dec(&transport->_mapcount);
+ KUNIT_EXPECT_FALSE(test, released);
+ folio_get(transport);
+ released = mc16_flt_release(transport);
+ folio_put(transport);
+ KUNIT_EXPECT_FALSE(test, released);
+ f->base->large = folio_alloc(GFP_KERNEL, 1);
+ KUNIT_ASSERT_NOT_NULL(test, f->base->large);
+ f->base->large->swap = make_memcomp_entry(0);
+ KUNIT_EXPECT_FALSE(test, mc16_flt_release(f->base->large));
+ f->base->large->swap.val = 0;
+ f->folio->swap = make_memcomp_entry(0);
+ KUNIT_EXPECT_FALSE(test, mc16_flt_release(f->folio)); /* Real anon mapping. */
+ f->folio->swap.val = 0;
+ KUNIT_EXPECT_TRUE(test, mc16_flt_release(transport));
+ KUNIT_EXPECT_EQ(test, transport->swap.val, 0UL);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(transport), 1);
+}
+
+/* Move a nonpresent PTE outside the original folio's rmap address, same PTL. */
+static int mc16_flt_move(struct mc16_flt *f, bool back)
+{
+ spinlock_t *ptl; /* Source and destination share the validated PTE table. */
+ pte_t *ptep;
+ unsigned int src = back ? 2 : 0, dst = back ? 0 : 2;
+ int ret = -EINVAL;
+
+ mmap_write_lock(f->mm);
+ ptep = get_locked_pte(f->mm, f->addr, &ptl);
+ if (!ptep) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ __acquire(RCU);
+ if (is_swap_pte(ptep_get(ptep + src)) && pte_none(ptep_get(ptep + dst))) {
+ pte_t entry = ptep_get_and_clear(f->mm, f->addr + src * PAGE_SIZE,
+ ptep + src);
+
+ set_pte_at(f->mm, f->addr + dst * PAGE_SIZE, ptep + dst, entry);
+ ret = 0;
+ }
+ pte_unmap_unlock(ptep, ptl);
+out:
+ mmap_write_unlock(f->mm);
+ return ret;
+}
+
+static vm_fault_t mc16_flt_late_fault(struct mc16_flt *f, bool *handled)
+{
+ struct memcomp_ref lookup = { };
+ unsigned long address = f->addr;
+ struct vm_area_struct *vma;
+ vm_fault_t ret = VM_FAULT_SIGBUS;
+
+ *handled = false;
+ if (!mc16_get(f->guards[0].token, &lookup))
+ return ret;
+ mmap_read_lock(f->mm);
+ vma = vma_lookup(f->mm, address);
+ if (vma) {
+ struct vm_fault vmf = {
+ .vma = vma, .address = address, .real_address = address,
+ .flags = FAULT_FLAG_ORIG_PTE_VALID | FAULT_FLAG_RETRY_NOWAIT,
+ .pmd = mm_find_pmd(f->mm, address),
+ .gfp_mask = GFP_HIGHUSER_MOVABLE,
+ };
+ spinlock_t *ptl; /* Copy orig_pte before the fault takes its own PTL. */
+ pte_t *ptep;
+
+ ptep = pte_offset_map_lock(f->mm, vmf.pmd, address, &ptl);
+ if (ptep) {
+ vmf.orig_pte = ptep_get(ptep);
+ pte_unmap_unlock(ptep, ptl);
+ ret = mc16_load_resident(&vmf, &lookup, handled);
+ }
+ }
+ /* RETRY_NOWAIT retains mmap_read_lock even if folio_lock_or_retry fails. */
+ mmap_read_unlock(f->mm);
+ if (lookup.slot)
+ mc16_ref_put(&lookup);
+ return ret;
+}
+
+static void memcomp_test_folio_missed_rmap(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 0);
+ pte_t orig;
+ vm_fault_t fault;
+ bool handled;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_flt_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, 0), 0);
+ KUNIT_ASSERT_EQ(test, mc16_flt_move(f, false), 0);
+ orig = mc16_flt_read(f, f->addr + 2 * PAGE_SIZE);
+ mc16_flt_drop(f, true); /* Bounded walks miss the modeled MOVE destination. */
+ KUNIT_EXPECT_EQ(test, f->folio->swap.val, 0UL);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_flt_read(f, f->addr + 2 * PAGE_SIZE), orig));
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guards[0].slot->refs), 2U);
+ KUNIT_EXPECT_FALSE(test, mc16_flt_flags(&f->guards[0], false) & MC16_F_OWNER);
+ KUNIT_EXPECT_TRUE(test, mc16_flt_flags(&f->guards[0], false) & MC16_F_RESIDENT);
+ /*
+ * Move back before the real fault: this bare PTE-transfer model does not
+ * update VMA pgoff as mremap does. Resident rmap requires the original
+ * page index. The guard stays continuous across both moves and detach.
+ */
+ KUNIT_ASSERT_EQ(test, mc16_flt_move(f, true), 0);
+ fault = mc16_flt_late_fault(f, &handled);
+ KUNIT_EXPECT_TRUE(test, handled);
+ KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)VM_FAULT_NOPAGE);
+ mc16_flt_expect_present(f, 0, f->addr);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guards[0].slot->refs), 1U);
+ KUNIT_EXPECT_EQ(test, mc16_flt_rss(f, MM_SWAPENTS), f->swapents);
+}
+
+/* Invalid lengths model corrupt metadata; restore it before any cleanup. */
+static void mc16_flt_bad_release(struct kunit *test, unsigned int bad)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, bad ? 1 : 0);
+ struct mc16_view view;
+ unsigned long swap;
+ unsigned int i;
+ u16 length;
+ bool released;
+ int pins;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_flt_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ for (i = 0; i < f->nr; i++) {
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, i), 0);
+ KUNIT_ASSERT_EQ(test, mc16_flt_zap(f, f->addr + i * PAGE_SIZE), 0);
+ KUNIT_ASSERT_EQ(test, mc16_flt_commit(f, i), 0);
+ }
+ swap = f->folio->swap.val;
+ pins = atomic_read(&f->anon_vma->refcount);
+ mutex_lock(memcomp_lock_of(f->guards[bad].token));
+ mc16_view_locked(&f->guards[bad], &view);
+ length = *view.length;
+ *view.length = U16_MAX;
+ mutex_unlock(memcomp_lock_of(f->guards[bad].token));
+ released = mc16_flt_release(f->folio);
+ mutex_lock(memcomp_lock_of(f->guards[bad].token));
+ mc16_view_locked(&f->guards[bad], &view);
+ *view.length = length;
+ mutex_unlock(memcomp_lock_of(f->guards[bad].token));
+ KUNIT_EXPECT_FALSE(test, released);
+ KUNIT_EXPECT_EQ(test, f->folio->swap.val, swap);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), 1);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->anon_vma->refcount), pins);
+ KUNIT_EXPECT_TRUE(test, mc16_flt_flags(&f->guards[0], false) & MC16_F_OWNER);
+ KUNIT_EXPECT_TRUE(test, mc16_flt_flags(&f->guards[bad], false) & MC16_F_OWNER);
+ KUNIT_EXPECT_TRUE(test, mc16_flt_release(f->folio));
+ for (i = 0; i < f->nr; i++)
+ mc16_flt_expect_load(f, i);
+}
+
+static void memcomp_test_folio_release_bad_head(struct kunit *test)
+{
+ mc16_flt_bad_release(test, 0);
+}
+
+static void memcomp_test_folio_release_bad_sibling(struct kunit *test)
+{
+ mc16_flt_bad_release(test, 1);
+}
+#endif /* CONFIG_MEMCG && CONFIG_MMU && CONFIG_ADVISE_SYSCALLS */
+#endif /* _MM_MEMCOMP_TESTS_FOLIO_TEST_H */
diff --git a/mm/memcomp/tests/format_test.h b/mm/memcomp/tests/format_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..e9b4a05043f0ae76084eabceaa7baa47a56594bd
--- /dev/null
+++ b/mm/memcomp/tests/format_test.h
@@ -0,0 +1,520 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _MM_MEMCOMP_TESTS_FORMAT_TEST_H
+#define _MM_MEMCOMP_TESTS_FORMAT_TEST_H
+
+#include <kunit/test.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include "../format.h"
+
+/* Registration is supplied by the enclosing memcomp KUnit suite. */
+static void memcomp_test_format_layout(struct kunit *test)
+{
+ KUNIT_EXPECT_EQ(test, sizeof(struct mc16_slot), (size_t)16);
+ KUNIT_EXPECT_EQ(test, sizeof(struct mc16_member), (size_t)24);
+ KUNIT_EXPECT_EQ(test, sizeof(struct mc16_record), (size_t)64);
+ KUNIT_EXPECT_EQ(test, __alignof__(struct mc16_slot), (size_t)8);
+ KUNIT_EXPECT_EQ(test, __alignof__(struct mc16_member), (size_t)8);
+ KUNIT_EXPECT_EQ(test, __alignof__(struct mc16_record), (size_t)8);
+ KUNIT_EXPECT_EQ(test, offsetof(struct mc16_slot, refs), (size_t)8);
+ KUNIT_EXPECT_EQ(test, offsetof(struct mc16_slot, length), (size_t)12);
+ KUNIT_EXPECT_EQ(test, offsetof(struct mc16_slot, flags), (size_t)14);
+ KUNIT_EXPECT_EQ(test, offsetof(struct mc16_member, owner_folio), (size_t)8);
+ KUNIT_EXPECT_EQ(test, offsetof(struct mc16_member, length), (size_t)16);
+ KUNIT_EXPECT_EQ(test, offsetof(struct mc16_member, flags), (size_t)18);
+ KUNIT_EXPECT_EQ(test, offsetof(struct mc16_member, payload_charge), (size_t)20);
+ KUNIT_EXPECT_EQ(test, offsetof(struct mc16_record, member), (size_t)64);
+ KUNIT_EXPECT_EQ(test, (u64)MC16_STATE_MASK, 0x7ULL);
+ KUNIT_EXPECT_EQ(test, (u64)MC16_ORDER_MASK, 0x1c0ULL);
+ KUNIT_EXPECT_EQ(test, MC16_ORDER_SHIFT, 6);
+ KUNIT_EXPECT_EQ(test, (u64)MC16_RESERVED_MASK, 0xf800ULL);
+}
+
+static void memcomp_test_format_states_orders(struct kunit *test)
+{
+ static const u64 modifiers[] = {
+ 0,
+ MC16_F_OWNER,
+ MC16_F_RESIDENT,
+ MC16_F_REFRESH_PENDING,
+ MC16_F_REFERENCED,
+ MC16_F_INCOMPRESSIBLE,
+ MC16_MODIFIER_MASK,
+ };
+ static const enum mc16_state states[] = {
+ MC16_EMPTY, MC16_PENDING, MC16_STORED, MC16_WRITEBACK, MC16_SWAPPED, MC16_SAMEFILL,
+ };
+ unsigned int state, order, i;
+ u16 flags;
+
+ for (state = 0; state < ARRAY_SIZE(states); state++) {
+ KUNIT_EXPECT_EQ(test, (unsigned int)states[state], state);
+ for (order = 0; order <= MC16_MAX_ORDER; order++) {
+ KUNIT_ASSERT_TRUE(test, mc16_flags_encode(states[state], order, 0, &flags));
+ KUNIT_EXPECT_TRUE(test, mc16_flags_valid(flags));
+ KUNIT_EXPECT_EQ(test, (u64)flags, (u64)state | ((u64)order << 6));
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(flags), state);
+ KUNIT_EXPECT_EQ(test, mc16_flags_order(flags), order);
+ }
+ }
+ /* Attached effective payloads can retain OWNER in their member flags. */
+ for (i = 0; i < ARRAY_SIZE(modifiers); i++) {
+ KUNIT_ASSERT_TRUE(test, mc16_flags_encode(MC16_PENDING, 4, modifiers[i], &flags));
+ KUNIT_EXPECT_TRUE(test, mc16_flags_valid(flags));
+ KUNIT_EXPECT_EQ(test, (u64)(flags & MC16_MODIFIER_MASK), modifiers[i]);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(flags), (unsigned int)MC16_PENDING);
+ KUNIT_EXPECT_EQ(test, mc16_flags_order(flags), 4U);
+ }
+}
+
+static void memcomp_test_format_flags_reject(struct kunit *test)
+{
+ static const struct {
+ u64 state;
+ u64 order;
+ u64 modifiers;
+ } bad[] = {
+ { 6, 0, 0 },
+ { 7, 0, 0 },
+ { BIT_ULL(32), 0, 0 },
+ { U64_MAX, 0, 0 },
+ { MC16_PENDING, 5, 0 },
+ { MC16_PENDING, 7, 0 },
+ { MC16_PENDING, BIT_ULL(32), 0 },
+ { MC16_PENDING, U64_MAX, 0 },
+ { MC16_PENDING, 0, MC16_STATE_MASK },
+ { MC16_PENDING, 0, MC16_ORDER_MASK },
+ { MC16_PENDING, 0, BIT_ULL(11) },
+ { MC16_PENDING, 0, BIT_ULL(15) },
+ { MC16_PENDING, 0, BIT_ULL(16) },
+ { MC16_PENDING, 0, BIT_ULL(63) },
+ };
+ unsigned int i;
+ u16 flags;
+
+ for (i = 0; i < ARRAY_SIZE(bad); i++) {
+ flags = 0x5a5a;
+ KUNIT_EXPECT_FALSE(test, mc16_flags_encode(bad[i].state, bad[i].order,
+ bad[i].modifiers, &flags));
+ KUNIT_EXPECT_EQ(test, flags, (u16)0x5a5a);
+ }
+ for (i = 11; i < 64; i++)
+ KUNIT_EXPECT_FALSE(test, mc16_flags_valid(BIT_ULL(i)));
+ for (i = 5; i <= 7; i++)
+ KUNIT_EXPECT_FALSE(test, mc16_flags_valid((u64)i << MC16_ORDER_SHIFT));
+ KUNIT_EXPECT_FALSE(test, mc16_flags_valid(6));
+ KUNIT_EXPECT_FALSE(test, mc16_flags_valid(7));
+ KUNIT_EXPECT_FALSE(test, mc16_flags_valid(U64_MAX));
+}
+
+static void memcomp_test_format_length_bounds(struct kunit *test)
+{
+ static const u64 good[] = { 1, 2, PAGE_SIZE - 1, PAGE_SIZE };
+ static const u64 bad_bytes[] = {
+ 0, PAGE_SIZE + 1, BIT_ULL(16) + 1, BIT_ULL(32), U64_MAX,
+ };
+ static const u64 bad_encoded[] = {
+ PAGE_SIZE,
+ BIT_ULL(16),
+ BIT_ULL(32),
+ U64_MAX,
+ };
+ unsigned int i;
+ u16 encoded;
+ u32 bytes;
+
+ for (i = 0; i < ARRAY_SIZE(good); i++) {
+ KUNIT_ASSERT_TRUE(test, mc16_length_encode(good[i], &encoded));
+ KUNIT_EXPECT_EQ(test, (u64)encoded, good[i] - 1);
+ KUNIT_ASSERT_TRUE(test, mc16_length_decode(encoded, &bytes));
+ KUNIT_EXPECT_EQ(test, (u64)bytes, good[i]);
+ }
+ for (i = 0; i < ARRAY_SIZE(bad_bytes); i++) {
+ encoded = 0x5a5a;
+ KUNIT_EXPECT_FALSE(test, mc16_length_encode(bad_bytes[i], &encoded));
+ KUNIT_EXPECT_EQ(test, encoded, (u16)0x5a5a);
+ }
+ for (i = 0; i < ARRAY_SIZE(bad_encoded); i++) {
+ bytes = 0x12345678;
+ KUNIT_EXPECT_FALSE(test, mc16_length_decode(bad_encoded[i], &bytes));
+ KUNIT_EXPECT_EQ(test, bytes, 0x12345678U);
+ }
+ KUNIT_ASSERT_TRUE(test, mc16_length_decode(0, &bytes));
+ KUNIT_EXPECT_EQ(test, bytes, 1U);
+}
+
+static void memcomp_test_format_payload_states(struct kunit *test)
+{
+ static const u64 states[] = {
+ MC16_EMPTY, MC16_PENDING, MC16_STORED, MC16_WRITEBACK,
+ MC16_SWAPPED, MC16_SAMEFILL, 6, 7,
+ BIT_ULL(32), U64_MAX,
+ };
+ unsigned int i;
+ u16 encoded;
+ u32 bytes;
+ bool payload;
+
+ for (i = 0; i < ARRAY_SIZE(states); i++) {
+ payload = states[i] == MC16_STORED || states[i] == MC16_WRITEBACK ||
+ states[i] == MC16_SWAPPED;
+ encoded = 0x5a5a;
+ bytes = 0x12345678;
+ if (payload) {
+ KUNIT_ASSERT_TRUE(test, mc16_payload_length_encode(states[i], PAGE_SIZE,
+ &encoded));
+ KUNIT_EXPECT_EQ(test, (u64)encoded, (u64)PAGE_SIZE - 1);
+ KUNIT_ASSERT_TRUE(test,
+ mc16_payload_length_decode(states[i], encoded, &bytes));
+ KUNIT_EXPECT_EQ(test, (u64)bytes, (u64)PAGE_SIZE);
+ } else if (states[i] == MC16_SAMEFILL) {
+ KUNIT_ASSERT_TRUE(test, mc16_payload_length_encode(states[i], 0, &encoded));
+ KUNIT_EXPECT_EQ(test, encoded, (u16)0);
+ KUNIT_ASSERT_TRUE(test,
+ mc16_payload_length_decode(states[i], encoded, &bytes));
+ KUNIT_EXPECT_EQ(test, bytes, 0U);
+ } else {
+ KUNIT_EXPECT_FALSE(test,
+ mc16_payload_length_encode(states[i], 1, &encoded));
+ KUNIT_EXPECT_EQ(test, encoded, (u16)0x5a5a);
+ KUNIT_EXPECT_FALSE(test, mc16_payload_length_decode(states[i], 0, &bytes));
+ KUNIT_EXPECT_EQ(test, bytes, 0x12345678U);
+ }
+ }
+ encoded = 0x5a5a;
+ bytes = 0x12345678;
+ KUNIT_EXPECT_FALSE(test, mc16_payload_length_encode(MC16_SAMEFILL, 1, &encoded));
+ KUNIT_EXPECT_EQ(test, encoded, (u16)0x5a5a);
+ KUNIT_EXPECT_FALSE(test, mc16_payload_length_decode(MC16_SAMEFILL, 1, &bytes));
+ KUNIT_EXPECT_EQ(test, bytes, 0x12345678U);
+ KUNIT_EXPECT_FALSE(test, mc16_payload_length_encode(MC16_STORED, 0, &encoded));
+ KUNIT_EXPECT_EQ(test, encoded, (u16)0x5a5a);
+}
+
+static void memcomp_test_format_full_words(struct kunit *test)
+{
+ static const u64 words[] = {
+ U64_MAX,
+ BIT_ULL(63),
+ 0xfedcba9876543210ULL,
+ 0x0101010101010101ULL,
+ 0x8000000000000001ULL,
+ };
+ static const enum mc16_state states[] = {
+ MC16_SAMEFILL,
+ MC16_STORED,
+ MC16_WRITEBACK,
+ };
+ struct mc16_slot slot = {};
+ unsigned int i, j;
+ u32 bytes;
+
+ for (i = 0; i < ARRAY_SIZE(states); i++) {
+ for (j = 0; j < ARRAY_SIZE(words); j++) {
+ slot.word = words[j];
+ KUNIT_ASSERT_TRUE(test, mc16_flags_encode(states[i], 0, 0, &slot.flags));
+ bytes = states[i] == MC16_SAMEFILL ? 0 : PAGE_SIZE;
+ KUNIT_ASSERT_TRUE(test, mc16_payload_length_encode(states[i], bytes,
+ &slot.length));
+ KUNIT_ASSERT_TRUE(test, mc16_payload_length_decode(states[i], slot.length,
+ &bytes));
+ KUNIT_EXPECT_EQ(test, slot.word, words[j]);
+ KUNIT_EXPECT_EQ(test, bytes,
+ states[i] == MC16_SAMEFILL ? 0U : (u32)PAGE_SIZE);
+ if (states[i] == MC16_SAMEFILL)
+ KUNIT_EXPECT_EQ(test, slot.length, (u16)0);
+ }
+ }
+}
+
+static void memcomp_test_format_swap_raw(struct kunit *test)
+{
+ const u64 offsets[] = { 1, mc16_swap_offset_limit() };
+ static const u64 lengths[] = { 1, PAGE_SIZE - 1, PAGE_SIZE };
+ const u64 types[] = { 0, MAX_SWAPFILES - 1 };
+ struct mc16_swap_fields fields;
+ unsigned int i, j, k;
+ u16 length;
+ u64 word;
+
+ KUNIT_EXPECT_EQ(test, mc16_swap_offset_limit(),
+ min_t(u64, SWP_OFFSET_MASK, U64_MAX >> (PAGE_SHIFT + 6)));
+ for (i = 0; i < ARRAY_SIZE(types); i++) {
+ for (j = 0; j < ARRAY_SIZE(offsets); j++) {
+ for (k = 0; k < ARRAY_SIZE(lengths); k++) {
+ KUNIT_ASSERT_TRUE(test, mc16_swap_encode(types[i], offsets[j], 0,
+ lengths[k], false, &word,
+ &length));
+ KUNIT_EXPECT_EQ(test, word,
+ types[i] | (offsets[j] << (PAGE_SHIFT + 6)));
+ KUNIT_ASSERT_TRUE(test, mc16_swap_decode(word, length, &fields));
+ KUNIT_EXPECT_EQ(test, fields.swap_offset, offsets[j]);
+ KUNIT_EXPECT_EQ(test, (u64)fields.type, types[i]);
+ KUNIT_EXPECT_EQ(test, fields.byte_offset, 0U);
+ KUNIT_EXPECT_EQ(test, (u64)fields.length, lengths[k]);
+ KUNIT_EXPECT_FALSE(test, fields.packed);
+ }
+ }
+ }
+}
+
+static void memcomp_test_format_swap_packed(struct kunit *test)
+{
+ static const struct {
+ u64 byte_offset;
+ u64 bytes;
+ } edges[] = {
+ { 0, 1 },
+ { 0, PAGE_SIZE - 1 },
+ { 1, PAGE_SIZE - 1 },
+ { PAGE_SIZE - 1, 1 },
+ };
+ struct mc16_swap_fields fields;
+ u64 offset = mc16_swap_offset_limit();
+ u64 type = MAX_SWAPFILES - 1;
+ unsigned int i;
+ u16 length;
+ u64 word;
+
+ for (i = 0; i < ARRAY_SIZE(edges); i++) {
+ KUNIT_ASSERT_TRUE(test, mc16_swap_encode(type, offset, edges[i].byte_offset,
+ edges[i].bytes, true, &word, &length));
+ KUNIT_EXPECT_EQ(test, word,
+ type | (edges[i].byte_offset << 5) | BIT_ULL(PAGE_SHIFT + 5) |
+ (offset << (PAGE_SHIFT + 6)));
+ KUNIT_ASSERT_TRUE(test, mc16_swap_decode(word, length, &fields));
+ KUNIT_EXPECT_EQ(test, fields.swap_offset, offset);
+ KUNIT_EXPECT_EQ(test, (u64)fields.type, type);
+ KUNIT_EXPECT_EQ(test, (u64)fields.byte_offset, edges[i].byte_offset);
+ KUNIT_EXPECT_EQ(test, (u64)fields.length, edges[i].bytes);
+ KUNIT_EXPECT_TRUE(test, fields.packed);
+ }
+}
+
+static void mc16_format_expect_swap_reject(struct kunit *test, u64 type, u64 offset,
+ u64 byte_offset, u64 bytes, bool packed)
+{
+ u64 word = 0x123456789abcdef0ULL;
+ u16 length = 0x5a5a;
+
+ KUNIT_EXPECT_FALSE(test, mc16_swap_encode(type, offset, byte_offset, bytes, packed, &word,
+ &length));
+ KUNIT_EXPECT_EQ(test, word, 0x123456789abcdef0ULL);
+ KUNIT_EXPECT_EQ(test, length, (u16)0x5a5a);
+}
+
+static void memcomp_test_format_swap_encode_reject(struct kunit *test)
+{
+ static const u64 bad_types[] = {
+ MAX_SWAPFILES,
+ 32,
+ BIT_ULL(32),
+ U64_MAX,
+ };
+ static const u64 bad_bytes[] = {
+ 0, PAGE_SIZE + 1, BIT_ULL(16) + 1, BIT_ULL(32), U64_MAX,
+ };
+ static const u64 bad_byte_offsets[] = {
+ PAGE_SIZE,
+ BIT_ULL(16),
+ BIT_ULL(32),
+ U64_MAX,
+ };
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(bad_types); i++)
+ mc16_format_expect_swap_reject(test, bad_types[i], 1, 0, 1, false);
+ for (i = 0; i < ARRAY_SIZE(bad_bytes); i++)
+ mc16_format_expect_swap_reject(test, 0, 1, 0, bad_bytes[i], false);
+ for (i = 0; i < ARRAY_SIZE(bad_byte_offsets); i++)
+ mc16_format_expect_swap_reject(test, 0, 1, bad_byte_offsets[i], 1, true);
+ mc16_format_expect_swap_reject(test, 0, mc16_swap_offset_limit() + 1, 0, 1, false);
+ mc16_format_expect_swap_reject(test, 0, U64_MAX, 0, 1, false);
+ mc16_format_expect_swap_reject(test, 0, 1, 1, 1, false);
+ mc16_format_expect_swap_reject(test, 0, 1, 0, PAGE_SIZE, true);
+ mc16_format_expect_swap_reject(test, 0, 1, PAGE_SIZE - 1, 2, true);
+}
+
+static void mc16_format_expect_decode_reject(struct kunit *test, u64 word, u64 length)
+{
+ struct mc16_swap_fields fields;
+ u8 before[sizeof(fields)];
+
+ memset(&fields, 0xa5, sizeof(fields));
+ memcpy(before, &fields, sizeof(fields));
+ KUNIT_EXPECT_FALSE(test, mc16_swap_decode(word, length, &fields));
+ KUNIT_EXPECT_MEMEQ(test, &fields, before, sizeof(fields));
+}
+
+static void memcomp_test_format_swap_decode_reject(struct kunit *test)
+{
+ u16 length;
+ u64 word;
+
+ KUNIT_ASSERT_TRUE(test, mc16_swap_encode(0, 1, 0, 1, false, &word, &length));
+ mc16_format_expect_decode_reject(test, (word & ~MC16_SWAP_TYPE_MASK) | MAX_SWAPFILES,
+ length);
+ mc16_format_expect_decode_reject(test, word | BIT_ULL(MC16_SWAP_BYTE_SHIFT), length);
+ mc16_format_expect_decode_reject(test, word | MC16_SWAP_PACKED, PAGE_SIZE - 1);
+ word |= MC16_SWAP_PACKED;
+ mc16_format_expect_decode_reject(test,
+ word | ((u64)(PAGE_SIZE - 1) << MC16_SWAP_BYTE_SHIFT), 1);
+ mc16_format_expect_decode_reject(test, word, PAGE_SIZE);
+ mc16_format_expect_decode_reject(test, word, BIT_ULL(32));
+ mc16_format_expect_decode_reject(test, word, U64_MAX);
+}
+
+static void memcomp_test_format_pending_record(struct kunit *test)
+{
+ static const enum mc16_state effective[] = {
+ MC16_STORED,
+ MC16_WRITEBACK,
+ MC16_SWAPPED,
+ MC16_SAMEFILL,
+ };
+ struct mc16_slot slot = {};
+ struct mc16_record *record;
+ struct mc16_member *member;
+ struct folio *owner;
+ unsigned int i;
+ u32 index, bytes;
+ u16 flags;
+ u64 word;
+
+ record = kunit_kzalloc(test, sizeof(*record) + MC16_MAX_PAGES * sizeof(record->member[0]),
+ GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, record);
+ owner = kunit_kzalloc(test, sizeof(*owner), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, owner);
+ record->token_base = 32;
+ record->nr_pages = MC16_MAX_PAGES;
+ record->producer = owner;
+ record->record_flags = MC16_RECORD_ANON_VMA;
+ refcount_set(&record->members, MC16_MAX_PAGES);
+ refcount_set(&slot.refs, 1);
+ slot.word = (u64)(unsigned long)record;
+ KUNIT_ASSERT_TRUE(test,
+ mc16_flags_encode(MC16_PENDING, MC16_MAX_ORDER,
+ MC16_F_OWNER | MC16_F_RESIDENT | MC16_F_REFRESH_PENDING,
+ &slot.flags));
+ KUNIT_EXPECT_TRUE(test, mc16_flags_valid(slot.flags));
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(slot.flags), (unsigned int)MC16_PENDING);
+ KUNIT_EXPECT_EQ(test, mc16_flags_order(slot.flags), 4U);
+ KUNIT_EXPECT_PTR_EQ(test, (void *)(unsigned long)slot.word, record);
+ KUNIT_EXPECT_EQ(test, slot.length, (u16)0);
+
+ /* PENDING length is not a direct payload encoding. */
+ flags = 0x5a5a;
+ KUNIT_EXPECT_FALSE(test, mc16_payload_length_encode(MC16_PENDING, PAGE_SIZE, &flags));
+ KUNIT_EXPECT_EQ(test, flags, (u16)0x5a5a);
+ for (i = 0; i < record->nr_pages; i++) {
+ enum mc16_state state = effective[i % ARRAY_SIZE(effective)];
+
+ KUNIT_ASSERT_TRUE(test, mc16_member_index(record, record->token_base + i, &index));
+ KUNIT_ASSERT_EQ(test, index, i);
+ member = &record->member[index];
+ member->owner_folio = owner;
+ KUNIT_ASSERT_TRUE(test, mc16_flags_encode(state, MC16_MAX_ORDER, MC16_F_OWNER,
+ &member->flags));
+ word = 0x8000000000000001ULL + i;
+ if (state == MC16_SWAPPED) {
+ KUNIT_ASSERT_TRUE(test, mc16_swap_encode(0, 1, 0, 1, true, &word,
+ &member->length));
+ } else {
+ bytes = state == MC16_SAMEFILL ? 0 : 1;
+ KUNIT_ASSERT_TRUE(test, mc16_payload_length_encode(state, bytes,
+ &member->length));
+ }
+ member->word = word;
+ member->payload_charge = state == MC16_SAMEFILL ? 0 : 1;
+ KUNIT_EXPECT_TRUE(test, mc16_flags_valid(member->flags));
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(member->flags), (unsigned int)state);
+ KUNIT_EXPECT_TRUE(test, member->flags & MC16_F_OWNER);
+ KUNIT_EXPECT_PTR_EQ(test, member->owner_folio, owner);
+ KUNIT_EXPECT_EQ(test, member->word, word);
+ KUNIT_ASSERT_TRUE(test, mc16_payload_length_decode(state, member->length, &bytes));
+ KUNIT_EXPECT_EQ(test, bytes, state == MC16_SAMEFILL ? 0U : 1U);
+ }
+ KUNIT_EXPECT_EQ(test, refcount_read(&record->members), MC16_MAX_PAGES);
+ KUNIT_EXPECT_EQ(test, refcount_read(&slot.refs), 1U);
+ KUNIT_EXPECT_PTR_EQ(test, record->producer, owner);
+}
+
+static void memcomp_test_format_member_index_orders(struct kunit *test)
+{
+ struct mc16_record record = { .token_base = 32 };
+ unsigned int order, i;
+ u32 index;
+
+ for (order = 0; order <= MC16_MAX_ORDER; order++) {
+ record.nr_pages = BIT(order);
+ for (i = 0; i < record.nr_pages; i++) {
+ KUNIT_ASSERT_TRUE(test, mc16_member_index(&record, record.token_base + i,
+ &index));
+ KUNIT_EXPECT_EQ(test, index, i);
+ }
+ index = 0x12345678;
+ KUNIT_EXPECT_FALSE(test,
+ mc16_member_index(&record, record.token_base + record.nr_pages,
+ &index));
+ KUNIT_EXPECT_EQ(test, index, 0x12345678U);
+ }
+}
+
+static void memcomp_test_format_member_index_reject(struct kunit *test)
+{
+ static const struct {
+ u32 base;
+ u16 nr;
+ } bad[] = {
+ { 0, 1 }, { 32, 0 }, { 32, 3 }, { 32, 32 }, { 33, 16 }, { U32_MAX, 16 },
+ };
+ const u64 bad_tokens[] = { 0, 31, 48, BIT_ULL(32), U64_MAX };
+ struct mc16_record record = { .token_base = 32, .nr_pages = 16 };
+ unsigned int i;
+ u32 index;
+
+ for (i = 0; i < ARRAY_SIZE(bad_tokens); i++) {
+ index = 0x12345678;
+ KUNIT_EXPECT_FALSE(test, mc16_member_index(&record, bad_tokens[i], &index));
+ KUNIT_EXPECT_EQ(test, index, 0x12345678U);
+ }
+ for (i = 0; i < ARRAY_SIZE(bad); i++) {
+ record.token_base = bad[i].base;
+ record.nr_pages = bad[i].nr;
+ index = 0x12345678;
+ KUNIT_EXPECT_FALSE(test, mc16_member_index(&record, bad[i].base, &index));
+ KUNIT_EXPECT_EQ(test, index, 0x12345678U);
+ }
+ /* Highest aligned span ends exactly at the u32 token boundary. */
+ record.token_base = U32_MAX - 15;
+ record.nr_pages = 16;
+ KUNIT_ASSERT_TRUE(test, mc16_member_index(&record, U32_MAX, &index));
+ KUNIT_EXPECT_EQ(test, index, 15U);
+}
+
+static void memcomp_test_format_null_outputs(struct kunit *test)
+{
+ struct mc16_record record = { .token_base = 1, .nr_pages = 1 };
+ u64 word = 0x123456789abcdef0ULL;
+ u16 length = 0x5a5a;
+ u32 index = 0x12345678;
+
+ KUNIT_EXPECT_FALSE(test, mc16_flags_encode(MC16_PENDING, 0, 0, NULL));
+ KUNIT_EXPECT_FALSE(test, mc16_length_encode(1, NULL));
+ KUNIT_EXPECT_FALSE(test, mc16_length_decode(0, NULL));
+ KUNIT_EXPECT_FALSE(test, mc16_payload_length_encode(MC16_SAMEFILL, 0, NULL));
+ KUNIT_EXPECT_FALSE(test, mc16_payload_length_decode(MC16_SAMEFILL, 0, NULL));
+ KUNIT_EXPECT_FALSE(test, mc16_swap_encode(0, 1, 0, 1, false, NULL, &length));
+ KUNIT_EXPECT_EQ(test, length, (u16)0x5a5a);
+ KUNIT_EXPECT_FALSE(test, mc16_swap_encode(0, 1, 0, 1, false, &word, NULL));
+ KUNIT_EXPECT_EQ(test, word, 0x123456789abcdef0ULL);
+ KUNIT_EXPECT_FALSE(test, mc16_swap_decode(word, 0, NULL));
+ KUNIT_EXPECT_FALSE(test, mc16_member_index(&record, 1, NULL));
+ KUNIT_EXPECT_FALSE(test, mc16_member_index(NULL, 1, &index));
+ KUNIT_EXPECT_EQ(test, index, 0x12345678U);
+}
+
+#endif /* _MM_MEMCOMP_TESTS_FORMAT_TEST_H */
diff --git a/mm/memcomp/tests/late_async_test.h b/mm/memcomp/tests/late_async_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..b2c95d2ad251330a1fc4692d9c515343e1b1606e
--- /dev/null
+++ b/mm/memcomp/tests/late_async_test.h
@@ -0,0 +1,331 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _MM_MEMCOMP_TESTS_LATE_ASYNC_TEST_H
+#define _MM_MEMCOMP_TESTS_LATE_ASYNC_TEST_H
+
+/* Included after the shared KUnit async codec definitions. */
+/* A second real LOADED event must not replace an initialized pool. */
+static struct acomp_alg memcomp_test_acomp_again_alg = {
+ .compress = memcomp_test_acomp_compress,
+ .decompress = memcomp_test_acomp_decompress,
+ .init = memcomp_test_acomp_init,
+ .exit = memcomp_test_acomp_exit,
+ .reqsize = sizeof(struct memcomp_test_acomp_req),
+ .base = {
+ .cra_name = "memcomp-kunit-acomp",
+ .cra_driver_name = "memcomp-kunit-acomp-again",
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct memcomp_test_acomp_tfm),
+ .cra_module = THIS_MODULE,
+ },
+};
+
+/* An unrelated algorithm must never become this selector's transform. */
+static struct acomp_alg memcomp_test_acomp_other_alg = {
+ .compress = memcomp_test_acomp_compress,
+ .decompress = memcomp_test_acomp_decompress,
+ .init = memcomp_test_acomp_init,
+ .exit = memcomp_test_acomp_exit,
+ .reqsize = sizeof(struct memcomp_test_acomp_req),
+ .base = {
+ .cra_name = "memcomp-kunit-other",
+ .cra_driver_name = "memcomp-kunit-other-async",
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct memcomp_test_acomp_tfm),
+ .cra_module = THIS_MODULE,
+ },
+};
+
+/* The synchronous form uses the boot-selected codec without replacing it. */
+struct memcomp_test_comp_tfm {
+ struct crypto_comp *sync;
+};
+
+static int memcomp_test_comp_init(struct crypto_tfm *tfm)
+{
+ struct memcomp_test_comp_tfm *ctx = crypto_tfm_ctx(tfm);
+
+ ctx->sync = crypto_alloc_comp(memcomp_compressor, 0, 0);
+ return PTR_ERR_OR_ZERO(ctx->sync);
+}
+
+static void memcomp_test_comp_exit(struct crypto_tfm *tfm)
+{
+ struct memcomp_test_comp_tfm *ctx = crypto_tfm_ctx(tfm);
+
+ crypto_free_comp(ctx->sync);
+}
+
+static int memcomp_test_comp_compress(struct crypto_tfm *tfm, const u8 *src,
+ unsigned int slen, u8 *dst,
+ unsigned int *dlen)
+{
+ struct memcomp_test_comp_tfm *ctx = crypto_tfm_ctx(tfm);
+
+ return crypto_comp_compress(ctx->sync, src, slen, dst, dlen);
+}
+
+static int memcomp_test_comp_decompress(struct crypto_tfm *tfm, const u8 *src,
+ unsigned int slen, u8 *dst,
+ unsigned int *dlen)
+{
+ struct memcomp_test_comp_tfm *ctx = crypto_tfm_ctx(tfm);
+
+ return crypto_comp_decompress(ctx->sync, src, slen, dst, dlen);
+}
+
+static struct crypto_alg memcomp_test_comp_alg = {
+ .cra_name = "memcomp-kunit-acomp",
+ .cra_driver_name = "memcomp-kunit-acomp-sync",
+ .cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct memcomp_test_comp_tfm),
+ .cra_init = memcomp_test_comp_init,
+ .cra_exit = memcomp_test_comp_exit,
+ .cra_module = THIS_MODULE,
+ .cra_u = {
+ .compress = {
+ .coa_compress = memcomp_test_comp_compress,
+ .coa_decompress = memcomp_test_comp_decompress,
+ },
+ },
+};
+
+struct memcomp_test_late_async_fixture {
+ struct memcomp_compressor_select select;
+ struct memcomp_async_pool pool;
+ struct notifier_block loaded_nb;
+ struct completion loaded;
+ struct crypto_comp *sync;
+ struct memcomp_async *taken;
+ struct crypto_acomp *initial_tfm;
+ struct memcomp_async *initial_batches[BITS_PER_LONG];
+ unsigned int initial_nr;
+ struct kunit *test;
+ char compressor[MEMCOMP_COMPRESSOR_NAME_LEN];
+ int inits_before;
+ int exits_before;
+ bool sync_registered;
+ bool async_registered;
+ bool again_registered;
+ bool other_registered;
+ bool select_registered;
+ bool observer_registered;
+};
+
+static int memcomp_test_late_loaded(struct notifier_block *nb,
+ unsigned long msg, void *data)
+{
+ struct memcomp_test_late_async_fixture *f;
+ struct crypto_alg *alg = data;
+
+ if (msg != CRYPTO_MSG_ALG_LOADED ||
+ (alg != &memcomp_test_acomp_alg.base &&
+ alg != &memcomp_test_acomp_again_alg.base &&
+ alg != &memcomp_test_acomp_other_alg.base))
+ return NOTIFY_DONE;
+ f = container_of(nb, struct memcomp_test_late_async_fixture, loaded_nb);
+ /* Lower priority than the production callback: its queueing is done. */
+ complete(&f->loaded);
+ return NOTIFY_OK;
+}
+
+static void memcomp_test_late_async_cleanup(void *data)
+{
+ struct memcomp_test_late_async_fixture *f = data;
+ unsigned int i, j;
+
+ /* Never hold select_lock while waiting for either callback or work. */
+ if (f->select_registered)
+ crypto_unregister_notifier(&f->select.nb);
+ if (f->observer_registered)
+ crypto_unregister_notifier(&f->loaded_nb);
+ cancel_delayed_work_sync(&f->select.work);
+ if (f->taken)
+ memcomp_async_give(&f->pool, f->taken);
+ for (i = 0; i < f->pool.nr; i++)
+ memcomp_async_free_batch(f->pool.batches[i]);
+ /* Preserve ownership even when a broken retry replaced the first pool. */
+ for (i = 0; i < f->initial_nr; i++) {
+ for (j = 0; j < f->pool.nr; j++) {
+ if (f->initial_batches[i] == f->pool.batches[j])
+ break;
+ }
+ if (j == f->pool.nr)
+ memcomp_async_free_batch(f->initial_batches[i]);
+ }
+ if (f->pool.tfm)
+ crypto_free_acomp(f->pool.tfm);
+ if (f->initial_tfm && f->initial_tfm != f->pool.tfm)
+ crypto_free_acomp(f->initial_tfm);
+ if (f->other_registered)
+ crypto_unregister_acomp(&memcomp_test_acomp_other_alg);
+ if (f->again_registered)
+ crypto_unregister_acomp(&memcomp_test_acomp_again_alg);
+ if (f->async_registered)
+ crypto_unregister_acomp(&memcomp_test_acomp_alg);
+ if (f->sync)
+ crypto_free_comp(f->sync);
+ if (f->sync_registered)
+ crypto_unregister_alg(&memcomp_test_comp_alg);
+ KUNIT_EXPECT_EQ(f->test,
+ atomic_read(&memcomp_test_acomp_inits) - f->inits_before,
+ atomic_read(&memcomp_test_acomp_exits) - f->exits_before);
+}
+
+static void memcomp_test_async_late_registration(struct kunit *test)
+{
+ struct memcomp_test_late_async_fixture *f;
+ struct crypto_acomp *tfm, *boot_tfm;
+ struct memcomp_async *first;
+ const char *driver;
+ unsigned long waited;
+ unsigned int nr, i, j;
+ int ret;
+
+ KUNIT_ASSERT_TRUE(test, memcomp_is_enabled());
+ KUNIT_ASSERT_NE(test, strcmp(memcomp_compressor,
+ "memcomp-kunit-acomp"), 0);
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->test = test;
+ f->inits_before = atomic_read(&memcomp_test_acomp_inits);
+ f->exits_before = atomic_read(&memcomp_test_acomp_exits);
+ spin_lock_init(&f->pool.lock);
+ INIT_DELAYED_WORK(&f->select.work, memcomp_select_worker);
+ init_completion(&f->loaded);
+ strscpy(f->compressor, "memcomp-kunit-acomp", sizeof(f->compressor));
+ f->select.compressor = f->compressor;
+ f->select.pool = &f->pool;
+ f->select.nb.notifier_call = memcomp_crypto_notify;
+ f->loaded_nb.notifier_call = memcomp_test_late_loaded;
+ f->loaded_nb.priority = -1;
+ ret = kunit_add_action_or_reset(test, memcomp_test_late_async_cleanup, f);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ ret = crypto_register_alg(&memcomp_test_comp_alg);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ f->sync_registered = true;
+ f->sync = crypto_alloc_comp(f->compressor, 0, 0);
+ if (IS_ERR(f->sync)) {
+ ret = PTR_ERR(f->sync);
+ f->sync = NULL;
+ KUNIT_FAIL(test, "late async synchronous selection: %d", ret);
+ return;
+ }
+ KUNIT_EXPECT_STREQ(test, crypto_comp_name(f->sync), f->compressor);
+ ret = crypto_has_acomp(f->compressor, CRYPTO_ALG_ASYNC,
+ CRYPTO_ALG_ASYNC | CRYPTO_NOLOAD);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ /* Only boot does global startup; this context owns a selected sync form. */
+ mutex_lock(&memcomp_select_lock);
+ f->select.decided = true;
+ f->select.started = true;
+ mutex_unlock(&memcomp_select_lock);
+ /* Before publication there must not be a visible async transform. */
+ KUNIT_ASSERT_PTR_EQ(test, smp_load_acquire(&f->pool.tfm), NULL);
+ KUNIT_ASSERT_EQ(test, f->pool.nr, 0U);
+ /* Observe the production pool without replacing its release-published tfm. */
+ boot_tfm = smp_load_acquire(&memcomp_async_pool.tfm);
+
+ ret = crypto_register_notifier(&f->loaded_nb);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ f->observer_registered = true;
+ ret = crypto_register_notifier(&f->select.nb);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ f->select_registered = true;
+ /* Run a real event while only the selected synchronous form exists. */
+ ret = crypto_register_acomp(&memcomp_test_acomp_other_alg);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ f->other_registered = true;
+ waited = wait_for_completion_timeout(&f->loaded, 10 * HZ);
+ KUNIT_ASSERT_NE(test, waited, 0UL);
+ flush_delayed_work(&f->select.work);
+ /* No release publication is allowed for the unrelated async name. */
+ KUNIT_EXPECT_PTR_EQ(test, smp_load_acquire(&f->pool.tfm), NULL);
+ KUNIT_EXPECT_EQ(test, f->pool.nr, 0U);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_test_acomp_inits),
+ f->inits_before);
+ reinit_completion(&f->loaded);
+ /* No direct async_init/worker call, queueing or global pool replacement. */
+ ret = crypto_register_acomp(&memcomp_test_acomp_alg);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ f->async_registered = true;
+ waited = wait_for_completion_timeout(&f->loaded, 10 * HZ);
+ KUNIT_ASSERT_NE(test, waited, 0UL);
+ flush_delayed_work(&f->select.work);
+ /* Consume exactly the publication performed by the production worker. */
+ tfm = smp_load_acquire(&f->pool.tfm);
+ KUNIT_ASSERT_NOT_NULL(test, tfm);
+ driver = crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm));
+ KUNIT_EXPECT_STREQ(test, driver, "memcomp-kunit-acomp-async");
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_test_acomp_inits),
+ f->inits_before + 1);
+ nr = f->pool.nr;
+ KUNIT_ASSERT_GT(test, nr, 0U);
+ KUNIT_EXPECT_EQ(test, (unsigned int)hweight_long(f->pool.free), nr);
+ for (i = 0; i < nr; i++) {
+ struct memcomp_async *batch = f->pool.batches[i];
+
+ KUNIT_ASSERT_NOT_NULL(test, batch);
+ KUNIT_EXPECT_EQ(test, batch->id, i);
+ for (j = 0; j < MEMCOMP_ASYNC_PAGES; j++) {
+ KUNIT_EXPECT_NOT_NULL(test, batch->reqs[j].req);
+ KUNIT_EXPECT_NOT_NULL(test, batch->reqs[j].buf);
+ KUNIT_EXPECT_PTR_EQ(test, batch->reqs[j].async, batch);
+ }
+ }
+ f->initial_tfm = tfm;
+ f->initial_nr = nr;
+ memcpy(f->initial_batches, f->pool.batches,
+ nr * sizeof(f->initial_batches[0]));
+ first = f->pool.batches[0];
+ KUNIT_ASSERT_NOT_NULL(test, first);
+ f->taken = memcomp_async_take(&f->pool, 0);
+ KUNIT_ASSERT_NOT_NULL(test, f->taken);
+ memcomp_async_give(&f->pool, f->taken);
+ f->taken = NULL;
+
+ /* A second real registration must not allocate or replace anything. */
+ reinit_completion(&f->loaded);
+ ret = crypto_register_acomp(&memcomp_test_acomp_again_alg);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ f->again_registered = true;
+ waited = wait_for_completion_timeout(&f->loaded, 10 * HZ);
+ KUNIT_ASSERT_NE(test, waited, 0UL);
+ flush_delayed_work(&f->select.work);
+ /* Repeated events must leave the release-published transform unchanged. */
+ KUNIT_ASSERT_PTR_EQ(test, smp_load_acquire(&f->pool.tfm), tfm);
+ KUNIT_EXPECT_PTR_EQ(test, f->pool.batches[0], first);
+ KUNIT_ASSERT_EQ(test, f->pool.nr, nr);
+ for (i = 0; i < nr; i++)
+ KUNIT_EXPECT_PTR_EQ(test, f->pool.batches[i], f->initial_batches[i]);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_test_acomp_inits),
+ f->inits_before + 1);
+
+ /* Unregister first, drain second, then prove a new event cannot requeue. */
+ ret = crypto_unregister_notifier(&f->select.nb);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ f->select_registered = false;
+ cancel_delayed_work_sync(&f->select.work);
+ crypto_unregister_acomp(&memcomp_test_acomp_again_alg);
+ f->again_registered = false;
+ reinit_completion(&f->loaded);
+ ret = crypto_register_acomp(&memcomp_test_acomp_again_alg);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ f->again_registered = true;
+ waited = wait_for_completion_timeout(&f->loaded, 10 * HZ);
+ KUNIT_ASSERT_NE(test, waited, 0UL);
+ KUNIT_EXPECT_FALSE(test, flush_delayed_work(&f->select.work));
+ KUNIT_EXPECT_FALSE(test, delayed_work_pending(&f->select.work));
+ /* Repeated events must leave the release-published transform unchanged. */
+ KUNIT_EXPECT_PTR_EQ(test, smp_load_acquire(&f->pool.tfm), tfm);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_test_acomp_inits),
+ f->inits_before + 1);
+ /* The boot pool's independently published transform was never replaced. */
+ KUNIT_EXPECT_PTR_EQ(test, smp_load_acquire(&memcomp_async_pool.tfm),
+ boot_tfm);
+}
+
+#endif /* _MM_MEMCOMP_TESTS_LATE_ASYNC_TEST_H */
diff --git a/mm/memcomp/tests/load_test.h b/mm/memcomp/tests/load_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..2c7de62eb504774c5fa8b09455dbd75f5e5f081f
--- /dev/null
+++ b/mm/memcomp/tests/load_test.h
@@ -0,0 +1,133 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_LOAD_TEST_H
+#define _MM_MEMCOMP_TESTS_LOAD_TEST_H
+#ifdef CONFIG_MEMCG
+static void memcomp_test_load_pending(struct kunit *test)
+{
+ struct mc16_test_payload *f = mc16_test_payload_begin(test);
+ void *dst = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_NOT_NULL(test, dst);
+ memset(dst, 0x5a, PAGE_SIZE);
+ KUNIT_EXPECT_EQ(test, mc16_load_page(&f->refs->storage->refs[0], dst),
+ -EAGAIN);
+ KUNIT_EXPECT_PTR_EQ(test, memchr_inv(dst, 0x5a, PAGE_SIZE), NULL);
+}
+
+static void memcomp_test_load_samefill(struct kunit *test)
+{
+ const unsigned long pattern = 0xfedcba9876543210UL;
+ struct mc16_test_payload *f = mc16_test_payload_begin(test);
+ struct mc16_test_storage *storage;
+ struct memcomp_ref *ref;
+ struct mc16_record *detached;
+ struct mc16_view view;
+ unsigned long *dst;
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->refs->storage;
+ ref = &storage->refs[0];
+ dst = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, dst);
+ KUNIT_ASSERT_EQ(test, mc16_test_payload_prepare(f, 0, pattern), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_payload_commit(f, storage->owner->folio), 0);
+ mc16_payload_finish(ref, &f->tail);
+ KUNIT_ASSERT_EQ(test, mc16_load_page(ref, dst), 0);
+ for (i = 0; i < PAGE_SIZE / sizeof(*dst); i++)
+ KUNIT_EXPECT_EQ(test, dst[i], pattern);
+
+ /* Model the completed producer's detach while retaining slot ownership. */
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ mc16_account_state(ref, false);
+ *view.flags &= ~MC16_F_OWNER;
+ mc16_account_state(ref, true);
+ detached = mc16_normalize_locked(ref);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ KUNIT_ASSERT_NOT_NULL(test, detached);
+ mc16_record_member_put(detached);
+ memset(dst, 0, PAGE_SIZE);
+ KUNIT_ASSERT_EQ(test, mc16_load_page(ref, dst), 0);
+ for (i = 0; i < PAGE_SIZE / sizeof(*dst); i++)
+ KUNIT_EXPECT_EQ(test, dst[i], pattern);
+}
+
+static void mc16_test_load_payload(struct kunit *test, bool raw,
+ bool writeback)
+{
+ struct mc16_test_payload *f = mc16_test_payload_begin(test);
+ struct mc16_test_storage *storage;
+ struct memcomp_ref *ref;
+ struct mc16_view view;
+ unsigned int bytes = 0, i;
+ unsigned long handle = 0;
+ u8 pool = 0;
+ u8 *src, *dst;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->refs->storage;
+ ref = &storage->refs[0];
+ src = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ dst = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ KUNIT_ASSERT_NOT_NULL(test, dst);
+ for (i = 0; i < PAGE_SIZE; i++)
+ src[i] = raw ? (i * 37 + (i >> 5)) : (i % 53);
+ if (raw)
+ ret = memcomp_alloc_raw_payload(0, src, PAGE_SIZE, &bytes,
+ &handle, false, NULL);
+ else
+ ret = memcomp_prepare_payload(src, PAGE_SIZE, &bytes, &handle,
+ &pool, MEMCOMP_PAYLOAD_MUST_SUCCEED,
+ false);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ f->prepared = (struct mc16_payload) { .word = handle, .bytes = bytes };
+ KUNIT_ASSERT_EQ(test, pool, (u8)0);
+ if (raw) {
+ KUNIT_EXPECT_EQ(test, bytes, (unsigned int)PAGE_SIZE);
+ } else {
+ KUNIT_EXPECT_GT(test, bytes, 0U);
+ KUNIT_EXPECT_LT(test, bytes, (unsigned int)PAGE_SIZE);
+ }
+ KUNIT_ASSERT_EQ(test, mc16_payload_charge(ref, &f->prepared), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_payload_commit(f, storage->owner->folio), 0);
+ mc16_payload_finish(ref, &f->tail);
+ if (writeback) {
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ *view.flags = (*view.flags & ~MC16_STATE_MASK) | MC16_WRITEBACK;
+ mc16_fifo_transition(ref, MC16_STORED, MC16_WRITEBACK);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ }
+ memset(dst, 0, PAGE_SIZE);
+ KUNIT_ASSERT_EQ(test, mc16_load_page(ref, dst), 0);
+ KUNIT_EXPECT_EQ(test, memcmp(src, dst, PAGE_SIZE), 0);
+
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ *view.flags |= MC16_F_REFRESH_PENDING;
+ mutex_unlock(memcomp_lock_of(ref->token));
+ memset(dst, 0x5a, PAGE_SIZE);
+ KUNIT_EXPECT_EQ(test, mc16_load_page(ref, dst), -EAGAIN);
+ KUNIT_EXPECT_PTR_EQ(test, memchr_inv(dst, 0x5a, PAGE_SIZE), NULL);
+}
+
+static void memcomp_test_load_raw(struct kunit *test)
+{
+ mc16_test_load_payload(test, true, false);
+}
+
+static void memcomp_test_load_compressed(struct kunit *test)
+{
+ mc16_test_load_payload(test, false, false);
+}
+
+static void memcomp_test_load_writeback(struct kunit *test)
+{
+ mc16_test_load_payload(test, true, true);
+}
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_LOAD_TEST_H */
diff --git a/mm/memcomp/tests/migrate_test.h b/mm/memcomp/tests/migrate_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..dc961f9aecc11656583e523a34e18d7a4b436c44
--- /dev/null
+++ b/mm/memcomp/tests/migrate_test.h
@@ -0,0 +1,795 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_MIGRATE_TEST_H
+#define _MM_MEMCOMP_TESTS_MIGRATE_TEST_H
+
+/* Include after consumers_test.h; register these CONFIG_MEMCG cases in main. */
+#ifdef CONFIG_MEMCG
+#define MC16_MIGT_NR 4
+
+struct mc16_migt_saved {
+ u64 word;
+ u16 flags;
+ u16 length;
+};
+
+struct mc16_migt_snapshot {
+ struct mc16_migt_saved payload[MC16_MIGT_NR];
+ u32 charge[MC16_MIGT_NR];
+ int refs[MC16_MIGT_NR];
+ int members[MC16_MIGT_NR];
+ u16 nid[MC16_MIGT_NR];
+ long charged;
+ int charges;
+ int uncharges;
+ int gets;
+ int puts;
+ int src_refs;
+ int dst_refs;
+ s64 stored;
+ s64 compressed;
+ long eligible;
+};
+
+struct mc16_migt {
+ struct mc16_consumer_test *base;
+ struct memcomp_ref refs[MC16_MIGT_NR]; /* Initial OWNERs, not lookups. */
+ u32 tokens[MC16_MIGT_NR];
+ struct mc16_migt_saved saved[MC16_MIGT_NR];
+ bool swapped[MC16_MIGT_NR];
+ struct mc16_migt_snapshot before;
+ unsigned int nr;
+ u8 *data;
+ u8 *loaded;
+ unsigned int fail_index;
+ bool race;
+ bool changed;
+ u16 race_flags;
+ unsigned int trace[2 * MC16_MIGT_NR + 1];
+ unsigned int nr_trace;
+ bool all_pinned;
+ bool old_owner_seen;
+};
+
+static struct mc16_migt *mc16_migt_current;
+
+/* No swap IO is executed: temporarily install only its encoded representation.
+ * The real zs payload is retained privately and restored before any release.
+ * This checks that migration never touches backing ownership, not swap IO.
+ */
+static int mc16_migt_swapped(struct mc16_migt *f, unsigned int index, bool set)
+{
+ struct memcomp_ref *ref = &f->refs[index];
+ struct mc16_view view;
+ unsigned int old_state, new_state;
+ u64 word = 0;
+ u16 length = 0;
+ int ret = 0;
+
+ if (set && !mc16_swap_encode(0, 128 + index, 0,
+ f->before.payload[index].length + 1,
+ true, &word, &length))
+ return -EINVAL;
+ mutex_lock(memcomp_lock_of(ref->token));
+ if (!mc16_view_locked(ref, &view) || !view.member) {
+ ret = -EINVAL;
+ goto out;
+ }
+ old_state = mc16_flags_state(*view.flags);
+ mc16_account_state(ref, false);
+ if (set) {
+ f->saved[index] = (struct mc16_migt_saved) {
+ .word = *view.word, .length = *view.length, .flags = *view.flags,
+ };
+ *view.word = word;
+ *view.length = length;
+ *view.flags = (*view.flags & ~MC16_STATE_MASK) | MC16_SWAPPED;
+ memcomp_debug_long_inc(&memcomp_swapped_entries);
+ } else {
+ *view.word = f->saved[index].word;
+ *view.length = f->saved[index].length;
+ *view.flags = f->saved[index].flags;
+ memcomp_debug_long_dec(&memcomp_swapped_entries);
+ }
+ new_state = mc16_flags_state(*view.flags);
+ mc16_fifo_transition(ref, old_state, new_state);
+ mc16_account_state(ref, true);
+ f->swapped[index] = set;
+out:
+ mutex_unlock(memcomp_lock_of(ref->token));
+ return ret;
+}
+
+static void mc16_migt_restore_race(struct mc16_migt *f)
+{
+ struct memcomp_ref *ref;
+ struct mc16_view view;
+
+ if (!f->changed)
+ return;
+ ref = &f->refs[f->fail_index];
+ mutex_lock(memcomp_lock_of(ref->token));
+ if (mc16_view_locked(ref, &view))
+ *view.flags = f->race_flags;
+ mutex_unlock(memcomp_lock_of(ref->token));
+ f->changed = false;
+}
+
+static void mc16_migt_cleanup(void *data)
+{
+ struct mc16_migt *f = data;
+ struct folio *src = f->base->account, *dst = f->base->dst;
+ unsigned int i;
+
+ mc16_migt_current = NULL;
+ mc16_migt_restore_race(f);
+ for (i = 0; i < f->nr; i++) {
+ if (!f->refs[i].slot)
+ continue;
+ if (f->swapped[i])
+ mc16_migt_swapped(f, i, false);
+ mc16_ref_put(&f->refs[i]);
+ }
+ if (src) {
+ src->swap.val = 0;
+ src->mapping = NULL;
+ folio_clear_swapbacked(src);
+ }
+ if (dst) {
+ dst->swap.val = 0;
+ dst->mapping = NULL;
+ folio_clear_swapbacked(dst);
+ }
+}
+
+static int mc16_migt_payload(struct mc16_migt *f, unsigned int index)
+{
+ struct folio *src = f->base->account;
+ struct memcomp_ref *ref = &f->refs[index];
+ struct mc16_payload payload = { };
+ struct mc16_payload_tail tail = { };
+ unsigned long handle;
+ unsigned int bytes;
+ u8 pool = 0;
+ int ret;
+
+ ret = memcomp_prepare_payload(f->data + index * PAGE_SIZE, PAGE_SIZE,
+ &bytes, &handle, &pool,
+ MEMCOMP_PAYLOAD_MUST_SUCCEED, false);
+ if (ret)
+ return ret;
+ if (pool) {
+ memcomp_zs_free(pool, handle);
+ return -EINVAL;
+ }
+ payload = (struct mc16_payload) { .word = handle, .bytes = bytes };
+ ret = mc16_payload_charge(ref, &payload);
+ if (ret)
+ goto abort;
+ folio_lock(src);
+ mutex_lock(memcomp_lock_of(ref->token));
+ ret = mc16_payload_commit_locked(ref, src, f->nr, index, &payload, &tail);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ mc16_payload_finish(ref, &tail);
+ folio_unlock(src);
+abort:
+ mc16_payload_abort(ref, &payload);
+ return ret;
+}
+
+static struct mc16_migt *
+mc16_migt_begin(struct kunit *test, unsigned int nr, unsigned int state)
+{
+ struct mc16_migt *f;
+ struct mc16_record *record;
+ struct folio *src, *dst;
+ unsigned int i;
+ int ret;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->base = mc16_consumer_test_begin(test);
+ if (!f->base)
+ return NULL;
+ f->fail_index = UINT_MAX;
+ if (nr > 1) {
+ src = folio_alloc(GFP_KERNEL, ilog2(nr));
+ dst = folio_alloc(GFP_KERNEL, ilog2(nr));
+ if (!src || !dst) {
+ if (src)
+ folio_put(src);
+ if (dst)
+ folio_put(dst);
+ return NULL;
+ }
+ folio_put(f->base->account);
+ folio_put(f->base->dst);
+ f->base->account = src;
+ f->base->dst = dst;
+ f->base->model.folio = src;
+ }
+ if (kunit_add_action_or_reset(test, mc16_migt_cleanup, f))
+ return NULL;
+ src = f->base->account;
+ dst = f->base->dst;
+ src->mapping = (struct address_space *)PAGE_MAPPING_ANON;
+ dst->mapping = (struct address_space *)PAGE_MAPPING_ANON;
+ __folio_set_swapbacked(src);
+ __folio_set_swapbacked(dst);
+ f->data = kunit_kmalloc(test, nr * PAGE_SIZE, GFP_KERNEL);
+ f->loaded = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ if (!f->data || !f->loaded)
+ return NULL;
+ for (i = 0; i < nr * PAGE_SIZE; i++)
+ f->data[i] = state == MC16_SAMEFILL ? 0x6e : (i + i / PAGE_SIZE) % 47;
+ for (i = 0; i < nr; i++) {
+ void *mapped = kmap_local_folio(src, i * PAGE_SIZE);
+
+ memcpy(mapped, f->data + i * PAGE_SIZE, PAGE_SIZE);
+ kunmap_local(mapped);
+ }
+ record = mc16_record_alloc(src, nr, false);
+ if (IS_ERR(record))
+ return NULL;
+ ret = mc16_reserve(record, nr, MC16_KEY_ANON, f->refs);
+ if (ret) {
+ mc16_record_abort(record);
+ return NULL;
+ }
+ f->nr = nr;
+ for (i = 0; i < nr; i++) {
+ mc16_account_reservation(&f->refs[i]);
+ f->tokens[i] = f->refs[i].token;
+ }
+ src->swap = make_memcomp_entry(f->tokens[0]);
+ if (state == MC16_PENDING)
+ return f;
+ for (i = 0; i < nr; i++) {
+ if (mc16_migt_payload(f, i))
+ return NULL;
+ }
+ return f;
+}
+
+/* Snapshot scalars under T; no record/member pointer is kept across T. */
+static void mc16_migt_snapshot(struct mc16_migt *f,
+ struct mc16_migt_snapshot *s, struct folio *owner)
+{
+ struct memcomp_kunit_objcg *model = &f->base->model;
+ struct kunit *test = f->base->test;
+ struct mc16_view view;
+ unsigned int i;
+ bool valid;
+
+ for (i = 0; i < f->nr; i++) {
+ struct memcomp_ref *ref = &f->refs[i];
+
+ if (!ref->slot)
+ continue;
+ mutex_lock(memcomp_lock_of(ref->token));
+ valid = mc16_view_locked(ref, &view) && view.record;
+ if (valid) {
+ s->payload[i] = (struct mc16_migt_saved) {
+ .word = *view.word, .flags = *view.flags,
+ .length = *view.length,
+ };
+ s->charge[i] = view.member->payload_charge;
+ s->members[i] = refcount_read(&view.record->members);
+ valid = view.member->owner_folio == owner &&
+ view.record->producer == f->base->account &&
+ view.record->nid == ref->cluster->key->nid;
+ }
+ mutex_unlock(memcomp_lock_of(ref->token));
+ KUNIT_EXPECT_TRUE(test, valid);
+ s->refs[i] = refcount_read(&ref->slot->refs);
+ s->nid[i] = ref->cluster->key->nid;
+ }
+ s->charged = atomic_long_read(&model->charged);
+ s->charges = atomic_read(&model->charge_calls);
+ s->uncharges = atomic_read(&model->uncharge_calls);
+ s->gets = atomic_read(&model->get_calls);
+ s->puts = atomic_read(&model->put_calls);
+ s->src_refs = folio_ref_count(f->base->account);
+ s->dst_refs = folio_ref_count(f->base->dst);
+ s->stored = atomic64_read(&memcomp_stored_bytes);
+ s->compressed = atomic64_read(&memcomp_compressed_bytes);
+ s->eligible = atomic_long_read(&mc16_fifo_of(&f->refs[0])->nr_entries);
+}
+
+static void mc16_migt_unchanged(struct mc16_migt *f, struct folio *owner)
+{
+ struct mc16_migt_snapshot after = { };
+ unsigned int i;
+
+ mc16_migt_snapshot(f, &after, owner);
+ for (i = 0; i < f->nr; i++) {
+ KUNIT_EXPECT_EQ(f->base->test, after.payload[i].word,
+ f->before.payload[i].word);
+ KUNIT_EXPECT_EQ(f->base->test, after.payload[i].flags,
+ f->before.payload[i].flags);
+ KUNIT_EXPECT_EQ(f->base->test, after.payload[i].length,
+ f->before.payload[i].length);
+ KUNIT_EXPECT_EQ(f->base->test, after.charge[i],
+ f->before.charge[i]);
+ KUNIT_EXPECT_EQ(f->base->test, after.refs[i],
+ f->before.refs[i]);
+ KUNIT_EXPECT_EQ(f->base->test, after.members[i],
+ f->before.members[i]);
+ KUNIT_EXPECT_EQ(f->base->test, after.nid[i],
+ f->before.nid[i]);
+ }
+ KUNIT_EXPECT_EQ(f->base->test, after.charged, f->before.charged);
+ KUNIT_EXPECT_EQ(f->base->test, after.charges, f->before.charges);
+ KUNIT_EXPECT_EQ(f->base->test, after.uncharges, f->before.uncharges);
+ KUNIT_EXPECT_EQ(f->base->test, after.gets, f->before.gets);
+ KUNIT_EXPECT_EQ(f->base->test, after.puts, f->before.puts);
+ KUNIT_EXPECT_EQ(f->base->test, after.src_refs, f->before.src_refs);
+ KUNIT_EXPECT_EQ(f->base->test, after.dst_refs, f->before.dst_refs);
+ KUNIT_EXPECT_EQ(f->base->test, after.stored, f->before.stored);
+ KUNIT_EXPECT_EQ(f->base->test, after.compressed, f->before.compressed);
+ KUNIT_EXPECT_EQ(f->base->test, after.eligible, f->before.eligible);
+}
+
+static void mc16_migt_bytes(struct mc16_migt *f)
+{
+ unsigned int i;
+ int ret;
+
+ for (i = 0; i < f->nr; i++) {
+ if (!f->refs[i].slot)
+ continue;
+ if (f->swapped[i]) {
+ ret = mc16_migt_swapped(f, i, false);
+ KUNIT_ASSERT_EQ(f->base->test, ret, 0);
+ }
+ ret = mc16_load_page(&f->refs[i], f->loaded);
+ KUNIT_ASSERT_EQ(f->base->test, ret, 0);
+ KUNIT_EXPECT_EQ(f->base->test,
+ memcmp(f->loaded, f->data + i * PAGE_SIZE, PAGE_SIZE), 0);
+ }
+}
+
+static int mc16_migt_move_window(struct memcomp_ref *ref, struct folio *from,
+ struct folio *to, unsigned int index)
+{
+ struct mc16_migt *f = mc16_migt_current;
+ struct mc16_view view;
+ unsigned int i;
+ bool forward = from == f->base->account;
+
+ if (f->nr_trace < ARRAY_SIZE(f->trace))
+ f->trace[f->nr_trace++] = index | (forward ? 0 : 0x80);
+ if (!index && forward) {
+ f->all_pinned = true;
+ for (i = 0; i < f->nr; i++)
+ f->all_pinned &= refcount_read(&f->refs[i].slot->refs) == 2;
+ }
+ if (!forward || index != f->fail_index)
+ return 0;
+ mutex_lock(memcomp_lock_of(ref->token));
+ if (mc16_view_locked(ref, &view)) {
+ f->old_owner_seen = view.member->owner_folio == from;
+ if (f->race) {
+ f->race_flags = *view.flags;
+ *view.flags &= ~MC16_F_OWNER;
+ f->changed = true;
+ }
+ }
+ mutex_unlock(memcomp_lock_of(ref->token));
+ return f->race ? 0 : -EAGAIN;
+}
+
+/* Every acquired folio lock is released before any fatal test assertion. */
+static int mc16_migt_call(struct mc16_migt *f, bool rollback,
+ bool lock_src, bool lock_dst, bool inject)
+{
+ struct folio *src = f->base->account, *dst = f->base->dst;
+ int ret = 0;
+
+ if (inject) {
+ mc16_migt_current = f;
+ kunit_activate_static_stub(f->base->test, mc16_migrate_move_error,
+ mc16_migt_move_window);
+ }
+ if (lock_src)
+ folio_lock(src);
+ if (lock_dst)
+ folio_lock(dst);
+ if (rollback)
+ mc16_migrate_folio_rollback(dst, src);
+ else
+ ret = mc16_migrate_folio(dst, src);
+ if (lock_dst)
+ folio_unlock(dst);
+ if (lock_src)
+ folio_unlock(src);
+ if (inject) {
+ kunit_deactivate_static_stub(f->base->test, mc16_migrate_move_error);
+ mc16_migt_current = NULL;
+ }
+ mc16_migt_restore_race(f);
+ return ret;
+}
+
+struct mc16_migt_metadata {
+ unsigned long entry;
+ u32 token[MC16_MIGT_NR];
+ unsigned short source_id[MC16_MIGT_NR];
+ u64 shadow[MC16_MIGT_NR];
+};
+
+/* Existing OWNER refs pin slots/keys; only scalars leave the lock scope. */
+static bool
+mc16_migt_metadata_snapshot(struct mc16_migt *f, struct folio *owner,
+ struct mc16_migt_metadata *s, bool seed)
+{
+ unsigned long token;
+ unsigned int i;
+ bool valid;
+
+ folio_lock(owner);
+ s->entry = owner->swap.val;
+ token = swp_offset(owner->swap);
+ valid = folio_memcomp_reserved(owner);
+ for (i = 0; valid && i < f->nr; i++) {
+ struct memcomp_ref *ref = &f->refs[i];
+ u64 *shadow;
+
+ mutex_lock(memcomp_lock_of(ref->token));
+ shadow = mc16_shadow_ptr(ref);
+ valid = shadow && ref->token == token + i &&
+ mc16_owner_matches_locked(ref, owner, f->nr, i);
+ if (valid && seed) {
+ valid = !*shadow;
+ /* Opaque words; never passed to stock refault. */
+ if (valid)
+ *shadow = (unsigned long)xa_mk_value(0x100 + i);
+ }
+ if (valid) {
+ s->token[i] = ref->token;
+ s->source_id[i] = mc16_owner_id(ref);
+ s->shadow[i] = *shadow;
+ }
+ mutex_unlock(memcomp_lock_of(ref->token));
+ }
+ folio_unlock(owner);
+ return valid;
+}
+
+static void mc16_migt_success(struct kunit *test, unsigned int state)
+{
+ struct mc16_migt *f = mc16_migt_begin(test, MC16_MIGT_NR, state);
+ struct mc16_migt_metadata before = { }, after = { };
+ unsigned int i;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_migt_snapshot(f, &f->before, f->base->account);
+ if (state == MC16_SWAPPED) {
+ for (i = 0; i < f->nr; i++)
+ KUNIT_ASSERT_EQ(test, mc16_migt_swapped(f, i, true), 0);
+ mc16_migt_snapshot(f, &f->before, f->base->account);
+ }
+ KUNIT_ASSERT_TRUE(test,
+ mc16_migt_metadata_snapshot(f, f->base->account,
+ &before, true));
+ ret = mc16_migt_call(f, false, true, true, false);
+ KUNIT_ASSERT_EQ(test, ret, 1);
+ KUNIT_EXPECT_EQ(test, f->base->dst->swap.val, f->base->account->swap.val);
+ mc16_migt_unchanged(f, f->base->dst);
+ KUNIT_ASSERT_TRUE(test,
+ mc16_migt_metadata_snapshot(f, f->base->dst,
+ &after, false));
+ KUNIT_EXPECT_EQ(test, after.entry, before.entry);
+ for (i = 0; i < f->nr; i++) {
+ KUNIT_EXPECT_EQ(test, after.token[i], before.token[i]);
+ KUNIT_EXPECT_EQ(test, after.source_id[i],
+ before.source_id[i]);
+ KUNIT_EXPECT_EQ(test, after.shadow[i], before.shadow[i]);
+ }
+ mc16_migt_bytes(f);
+}
+
+static void memcomp_test_migrate_stored(struct kunit *test)
+{
+ mc16_migt_success(test, MC16_STORED);
+}
+
+static void memcomp_test_migrate_samefill(struct kunit *test)
+{
+ mc16_migt_success(test, MC16_SAMEFILL);
+}
+
+static void memcomp_test_migrate_swapped(struct kunit *test)
+{
+ mc16_migt_success(test, MC16_SWAPPED);
+}
+
+static void memcomp_test_migrate_rollback(struct kunit *test)
+{
+ struct mc16_migt *f = mc16_migt_begin(test, MC16_MIGT_NR, MC16_STORED);
+ unsigned int i;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_migt_snapshot(f, &f->before, f->base->account);
+ ret = mc16_migt_call(f, false, true, true, true);
+ KUNIT_ASSERT_EQ(test, ret, 1);
+ KUNIT_EXPECT_TRUE(test, f->all_pinned);
+ mc16_migt_unchanged(f, f->base->dst);
+ mc16_migt_call(f, true, true, true, true);
+ KUNIT_EXPECT_EQ(test, f->base->dst->swap.val, 0UL);
+ KUNIT_ASSERT_EQ(test, f->nr_trace, 2 * f->nr);
+ for (i = 0; i < f->nr; i++) {
+ KUNIT_EXPECT_EQ(test, f->trace[i], i);
+ KUNIT_EXPECT_EQ(test, f->trace[f->nr + i], 0x80 + f->nr - i - 1);
+ }
+ mc16_migt_unchanged(f, f->base->account);
+ mc16_migt_bytes(f);
+}
+
+static void mc16_migt_partial(struct kunit *test, bool race)
+{
+ struct mc16_migt *f = mc16_migt_begin(test, MC16_MIGT_NR, MC16_STORED);
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->fail_index = 2;
+ f->race = race;
+ f->base->dst->swap = swp_entry(0, 777);
+ mc16_migt_snapshot(f, &f->before, f->base->account);
+ ret = mc16_migt_call(f, false, true, true, true);
+ KUNIT_EXPECT_EQ(test, ret, -EAGAIN);
+ KUNIT_EXPECT_EQ(test, f->base->dst->swap.val, swp_entry(0, 777).val);
+ KUNIT_EXPECT_TRUE(test, f->all_pinned);
+ KUNIT_EXPECT_TRUE(test, f->old_owner_seen);
+ KUNIT_ASSERT_EQ(test, f->nr_trace, 5U);
+ KUNIT_EXPECT_EQ(test, f->trace[0], 0U);
+ KUNIT_EXPECT_EQ(test, f->trace[1], 1U);
+ KUNIT_EXPECT_EQ(test, f->trace[2], 2U);
+ KUNIT_EXPECT_EQ(test, f->trace[3], 0x81U);
+ KUNIT_EXPECT_EQ(test, f->trace[4], 0x80U);
+ mc16_migt_unchanged(f, f->base->account);
+ mc16_migt_bytes(f);
+}
+
+static void memcomp_test_migrate_partial(struct kunit *test)
+{
+ mc16_migt_partial(test, false);
+}
+
+static void memcomp_test_migrate_revalidate(struct kunit *test)
+{
+ mc16_migt_partial(test, true);
+}
+
+static void memcomp_test_migrate_pending(struct kunit *test)
+{
+ struct mc16_migt *f = mc16_migt_begin(test, MC16_MIGT_NR, MC16_PENDING);
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_migt_snapshot(f, &f->before, f->base->account);
+ ret = mc16_migt_call(f, false, true, true, true);
+ KUNIT_EXPECT_EQ(test, ret, -EAGAIN);
+ KUNIT_EXPECT_EQ(test, f->nr_trace, 0U);
+ KUNIT_EXPECT_EQ(test, f->base->dst->swap.val, 0UL);
+ mc16_migt_unchanged(f, f->base->account);
+}
+
+static void mc16_migt_reject(struct kunit *test, bool wrong_owner)
+{
+ struct mc16_migt *f = mc16_migt_begin(test, MC16_MIGT_NR, MC16_STORED);
+ struct memcomp_ref *ref;
+ struct mc16_view view;
+ u16 flags;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ ref = &f->refs[1];
+ mc16_migt_snapshot(f, &f->before, f->base->account);
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ flags = ref->slot->flags;
+ if (wrong_owner)
+ view.member->owner_folio = f->base->dst;
+ else
+ ref->slot->flags &= ~MC16_ORDER_MASK;
+ mutex_unlock(memcomp_lock_of(ref->token));
+ ret = mc16_migt_call(f, false, true, true, true);
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ if (wrong_owner)
+ view.member->owner_folio = f->base->account;
+ else
+ ref->slot->flags = flags;
+ mutex_unlock(memcomp_lock_of(ref->token));
+ KUNIT_EXPECT_EQ(test, ret, -EAGAIN);
+ KUNIT_EXPECT_EQ(test, f->nr_trace, 0U);
+ KUNIT_EXPECT_EQ(test, f->base->dst->swap.val, 0UL);
+ mc16_migt_unchanged(f, f->base->account);
+ mc16_migt_bytes(f);
+}
+
+static void memcomp_test_migrate_geometry(struct kunit *test)
+{
+ mc16_migt_reject(test, false);
+}
+
+static void memcomp_test_migrate_owner(struct kunit *test)
+{
+ mc16_migt_reject(test, true);
+}
+
+static void memcomp_test_migrate_missing(struct kunit *test)
+{
+ struct mc16_migt *f = mc16_migt_begin(test, MC16_MIGT_NR, MC16_STORED);
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_ref_put(&f->refs[3]);
+ mc16_migt_snapshot(f, &f->before, f->base->account);
+ ret = mc16_migt_call(f, false, true, true, true);
+ KUNIT_EXPECT_EQ(test, ret, -EAGAIN);
+ KUNIT_EXPECT_EQ(test, f->nr_trace, 0U);
+ KUNIT_EXPECT_EQ(test, f->base->dst->swap.val, 0UL);
+ mc16_migt_unchanged(f, f->base->account);
+ mc16_migt_bytes(f);
+}
+
+static void memcomp_test_migrate_locks(struct kunit *test)
+{
+ struct mc16_migt *f = mc16_migt_begin(test, 1, MC16_STORED);
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_migt_snapshot(f, &f->before, f->base->account);
+ ret = mc16_migt_call(f, false, false, true, false);
+ KUNIT_EXPECT_EQ(test, ret, -EAGAIN);
+ ret = mc16_migt_call(f, false, true, false, false);
+ KUNIT_EXPECT_EQ(test, ret, -EAGAIN);
+ KUNIT_EXPECT_EQ(test, f->base->dst->swap.val, 0UL);
+ mc16_migt_unchanged(f, f->base->account);
+ mc16_migt_bytes(f);
+}
+
+static void memcomp_test_migrate_rollback_missing(struct kunit *test)
+{
+ struct mc16_migt *f = mc16_migt_begin(test, MC16_MIGT_NR, MC16_STORED);
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ ret = mc16_migt_call(f, false, true, true, false);
+ KUNIT_ASSERT_EQ(test, ret, 1);
+ /* Actual final member retirement; rollback must restore every survivor. */
+ mc16_ref_put(&f->refs[3]);
+ mc16_migt_snapshot(f, &f->before, f->base->dst);
+ mc16_migt_call(f, true, true, true, false);
+ KUNIT_EXPECT_EQ(test, f->base->dst->swap.val, 0UL);
+ mc16_migt_unchanged(f, f->base->account);
+ mc16_migt_bytes(f);
+}
+
+static void memcomp_test_migrate_rollback_invalid(struct kunit *test)
+{
+ struct mc16_migt *f = mc16_migt_begin(test, MC16_MIGT_NR, MC16_STORED);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_migt_snapshot(f, &f->before, f->base->account);
+ f->base->dst->swap = make_memcomp_entry(U32_MAX);
+ mc16_migt_call(f, true, true, true, false);
+ KUNIT_EXPECT_EQ(test, f->base->dst->swap.val, 0UL);
+ mc16_migt_unchanged(f, f->base->account);
+}
+
+static void memcomp_test_migrate_incompressible(struct kunit *test)
+{
+ struct mc16_consumer_test *f = mc16_consumer_test_begin(test);
+ unsigned long marker, moved, rolled_back;
+ struct folio *src, *dst;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ src = f->account;
+ dst = f->dst;
+ src->mapping = (struct address_space *)PAGE_MAPPING_ANON;
+ __folio_set_swapbacked(src);
+ folio_lock(src);
+ folio_lock(dst);
+ folio_set_incompressible(src, memcomp_folio_hash(src));
+ marker = src->swap.val;
+ ret = mc16_migrate_folio(dst, src);
+ moved = dst->swap.val;
+ mc16_migrate_folio_rollback(dst, src);
+ rolled_back = dst->swap.val;
+ src->swap.val = 0;
+ dst->swap.val = 0;
+ src->mapping = NULL;
+ folio_clear_swapbacked(src);
+ folio_unlock(dst);
+ folio_unlock(src);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ KUNIT_EXPECT_EQ(test, moved, marker);
+ KUNIT_EXPECT_EQ(test, rolled_back, marker);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(src), 1);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(dst), 1);
+}
+
+static void memcomp_test_migrate_single(struct kunit *test)
+{
+ struct mc16_migt *f = mc16_migt_begin(test, 1, MC16_STORED);
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_migt_snapshot(f, &f->before, f->base->account);
+ ret = mc16_migt_call(f, false, true, true, false);
+ KUNIT_ASSERT_EQ(test, ret, 1);
+ mc16_migt_unchanged(f, f->base->dst);
+ mc16_migt_bytes(f);
+}
+
+static void memcomp_test_migrate_dst_geometry(struct kunit *test)
+{
+ struct mc16_migt *f = mc16_migt_begin(test, MC16_MIGT_NR, MC16_STORED);
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->base->large = folio_alloc(GFP_KERNEL, 0);
+ KUNIT_ASSERT_NOT_NULL(test, f->base->large);
+ mc16_migt_snapshot(f, &f->before, f->base->account);
+ folio_lock(f->base->account);
+ folio_lock(f->base->large);
+ ret = mc16_migrate_folio(f->base->large, f->base->account);
+ folio_unlock(f->base->large);
+ folio_unlock(f->base->account);
+ KUNIT_EXPECT_EQ(test, ret, -EAGAIN);
+ KUNIT_EXPECT_EQ(test, f->base->large->swap.val, 0UL);
+ mc16_migt_unchanged(f, f->base->account);
+ mc16_migt_bytes(f);
+}
+
+static void memcomp_test_migrate_range(struct kunit *test)
+{
+ struct mc16_migt *f = mc16_migt_begin(test, MC16_MIGT_NR, MC16_STORED);
+ swp_entry_t saved;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_migt_snapshot(f, &f->before, f->base->account);
+ saved = f->base->account->swap;
+ f->base->account->swap = make_memcomp_entry(U32_MAX);
+ ret = mc16_migt_call(f, false, true, true, false);
+ f->base->account->swap = saved;
+ KUNIT_EXPECT_EQ(test, ret, -EAGAIN);
+ KUNIT_EXPECT_EQ(test, f->base->dst->swap.val, 0UL);
+ mc16_migt_unchanged(f, f->base->account);
+}
+
+static void memcomp_test_migrate_record_geometry(struct kunit *test)
+{
+ struct mc16_migt *f = mc16_migt_begin(test, MC16_MIGT_NR, MC16_STORED);
+ struct memcomp_ref *ref;
+ struct mc16_view view;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ ref = &f->refs[0];
+ mc16_migt_snapshot(f, &f->before, f->base->account);
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ view.record->nr_pages = 2;
+ mutex_unlock(memcomp_lock_of(ref->token));
+ ret = mc16_migt_call(f, false, true, true, true);
+ /* Member zero remains in both geometries; restore before any assertion. */
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ view.record->nr_pages = f->nr;
+ mutex_unlock(memcomp_lock_of(ref->token));
+ KUNIT_EXPECT_EQ(test, ret, -EAGAIN);
+ KUNIT_EXPECT_EQ(test, f->nr_trace, 0U);
+ KUNIT_EXPECT_EQ(test, f->base->dst->swap.val, 0UL);
+ mc16_migt_unchanged(f, f->base->account);
+ mc16_migt_bytes(f);
+}
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_MIGRATE_TEST_H */
diff --git a/mm/memcomp/tests/mincore_test.h b/mm/memcomp/tests/mincore_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..9040c4f6ffab56f792c85ce7987f18cf6e799502
--- /dev/null
+++ b/mm/memcomp/tests/mincore_test.h
@@ -0,0 +1,462 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_MINCORE_TEST_H
+#define _MM_MEMCOMP_TESTS_MINCORE_TEST_H
+
+#include <linux/types.h>
+
+struct address_space;
+struct mm_struct;
+
+unsigned char memcomp_test_mincore_page(struct address_space *mapping,
+ pgoff_t index);
+int memcomp_test_mincore_walk(struct mm_struct *mm, unsigned long start,
+ unsigned long end, unsigned char *vec);
+
+#ifndef MC16_MINCORE_DECLARE_TEST_API
+#include <linux/file.h>
+#include <linux/shmem_fs.h>
+#include "../../swap.h"
+
+/* Include after query_test.h, writeback_test.h and fault_test.h in suite.h. */
+
+static void memcomp_test_mincore_zero(struct kunit *test)
+{
+ struct mc16_zero_owners before, after;
+
+ mc16_zero_owners_read(&before);
+ KUNIT_EXPECT_TRUE(test, memcomp_mincore(0));
+ KUNIT_EXPECT_FALSE(test, memcomp_mincore(ULONG_MAX));
+ mc16_zero_owners_read(&after);
+ KUNIT_EXPECT_EQ(test, after.pte_pages, before.pte_pages);
+ KUNIT_EXPECT_EQ(test, after.cache_pages, before.cache_pages);
+}
+
+#ifdef CONFIG_MEMCG
+static void memcomp_test_mincore_pending(struct kunit *test)
+{
+ struct mc16_test_refs *f = mc16_test_refs_begin(test);
+ struct mc16_test_query_observation before;
+ struct memcomp_ref *ref;
+ unsigned int refs;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f, 1), 0);
+ ref = &f->storage->refs[0];
+ refs = refcount_read(&ref->slot->refs);
+ mc16_test_query_observe(f->storage, &before);
+ KUNIT_EXPECT_TRUE(test, memcomp_mincore(ref->token));
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), refs);
+ mc16_test_query_unchanged(test, f->storage, &before);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_payload(f, 0, MC16_SAMEFILL), 0);
+ mc16_test_query_observe(f->storage, &before);
+ KUNIT_EXPECT_TRUE(test, memcomp_mincore(ref->token));
+ mc16_test_query_unchanged(test, f->storage, &before);
+}
+
+#define MC16_MINCORE_PENDING_TEST_ENABLED
+#endif
+
+#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP)
+/* Only device/cache availability is modeled; guards, T and folio flags are real. */
+struct mc16_mincore_test {
+ struct kunit *test;
+ struct folio *cache;
+ struct file *file;
+ unsigned long token;
+ int dups;
+ unsigned int lookups;
+ bool cached;
+ bool under_t;
+ bool atomic;
+ bool missing_pin;
+ bool change_xa;
+ bool changed;
+ int change_error;
+#ifdef MC16_FAULT_MM_TEST_ENABLED
+ struct mc16_ft *fault;
+#endif
+};
+
+static struct mc16_mincore_test *mc16_mincore_test_current;
+
+static void mc16_mincore_test_cleanup(void *data)
+{
+ struct mc16_mincore_test *f = data;
+ unsigned int i;
+
+ if (mc16_mincore_test_current == f)
+ mc16_mincore_test_current = NULL;
+ /* The synthetic XA values borrow live fixture owners; erase before fput. */
+ if (f->file) {
+ for (i = 0; i < MC16_MAX_PAGES; i++)
+ xa_erase(&f->file->f_mapping->i_pages, i);
+ fput(f->file);
+ }
+ folio_put(f->cache);
+}
+
+#ifdef MC16_FAULT_MM_TEST_ENABLED
+static int mc16_mincore_test_replace_pte(struct mc16_ft *f)
+{
+ struct memcomp_ref ref = { };
+ spinlock_t *ptl; /* A real PTE replacement while the query has dropped PTL. */
+ pte_t *ptep;
+ int ret = -ENOENT;
+
+ ptep = get_locked_pte(f->mm, f->addr, &ptl);
+ if (!ptep)
+ return -ENOMEM;
+ __acquire(RCU);
+ if (pte_same(ptep_get(ptep), mc16_ft_orig(f, 0)) &&
+ mc16_get(f->tokens[0], &ref)) {
+ mc16_zero_pte_init(1);
+ set_pte_at(f->mm, f->addr, ptep,
+ swp_entry_to_pte(make_memcomp_entry(0)));
+ ret = 0;
+ }
+ pte_unmap_unlock(ptep, ptl);
+ if (ref.slot)
+ mc16_ref_put_count(&ref, 2); /* PTE owner and this lookup. */
+ return ret;
+}
+#endif
+
+static struct folio *mc16_mincore_test_swap_folio(swp_entry_t swap)
+{
+ struct mc16_mincore_test *f = mc16_mincore_test_current;
+ struct mutex *lock = memcomp_lock_of(f->token);
+
+ f->lookups++;
+ f->atomic |= irqs_disabled();
+ if (mutex_trylock(lock))
+ mutex_unlock(lock);
+ else
+ f->under_t = true;
+ f->missing_pin |= atomic_read(&memcomp_kunit_slot_dups) <= f->dups;
+ if (f->change_xa && !f->changed) {
+ void *old = swp_to_radix_entry(make_memcomp_entry(f->token));
+ void *zero = swp_to_radix_entry(make_memcomp_entry(0));
+
+ f->changed = true;
+ old = xa_cmpxchg(&f->file->f_mapping->i_pages, 0, old, zero,
+ GFP_KERNEL);
+ if (xa_is_err(old))
+ f->change_error = xa_err(old);
+ else if (old != swp_to_radix_entry(make_memcomp_entry(f->token)))
+ f->change_error = -ESTALE;
+ }
+#ifdef MC16_FAULT_MM_TEST_ENABLED
+ if (f->fault && !f->changed) {
+ f->changed = true;
+ f->change_error = mc16_mincore_test_replace_pte(f->fault);
+ }
+#endif
+ if (!swap.val || !f->cached)
+ return ERR_PTR(-ENOENT);
+ folio_get(f->cache);
+ return f->cache;
+}
+
+static struct mc16_mincore_test *
+mc16_mincore_test_begin(struct kunit *test, unsigned long token)
+{
+ struct mc16_mincore_test *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+
+ if (!f)
+ return NULL;
+ f->cache = folio_alloc(GFP_KERNEL, 0);
+ if (!f->cache)
+ return NULL;
+ f->test = test;
+ f->token = token;
+ f->dups = atomic_read(&memcomp_kunit_slot_dups);
+ if (kunit_add_action_or_reset(test, mc16_mincore_test_cleanup, f))
+ return NULL;
+ mc16_mincore_test_current = f;
+ kunit_activate_static_stub(test, mc16_mincore_swap_folio,
+ mc16_mincore_test_swap_folio);
+ return f;
+}
+
+static void mc16_mincore_test_expect_unlocked(struct kunit *test,
+ struct mc16_mincore_test *f)
+{
+ KUNIT_EXPECT_FALSE(test, f->under_t);
+ KUNIT_EXPECT_FALSE(test, f->atomic);
+ KUNIT_EXPECT_FALSE(test, f->missing_pin);
+ KUNIT_EXPECT_EQ(test, f->change_error, 0);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->cache), 1);
+}
+
+static int mc16_mincore_test_writeback(struct mc16_test_writeback *w, bool packed)
+{
+ struct mc16_swap_fields swap;
+ int ret;
+
+ if (!mc16_test_writeback_claim(w))
+ return -EBUSY;
+ swap = mc16_test_writeback_backing(w, packed);
+ ret = mc16_test_writeback_settle(w, &swap);
+ if (ret)
+ mc16_test_writeback_revert(w);
+ else
+ mc16_writeback_finish(&w->writer, &w->tail);
+ return ret;
+}
+
+static void memcomp_test_mincore_pool(struct kunit *test)
+{
+ struct mc16_test_writeback *w = mc16_test_writeback_begin(test, false);
+ struct mc16_test_query_observation before;
+ struct mc16_mincore_test *f;
+ unsigned int refs;
+
+ KUNIT_ASSERT_NOT_NULL(test, w);
+ f = mc16_mincore_test_begin(test, w->writer.token);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ refs = refcount_read(&w->writer.slot->refs);
+ mc16_test_query_observe(w->refs->storage, &before);
+ KUNIT_EXPECT_TRUE(test, memcomp_mincore(w->writer.token));
+ KUNIT_ASSERT_TRUE(test, mc16_test_writeback_claim(w));
+ KUNIT_EXPECT_TRUE(test, memcomp_mincore(w->writer.token));
+ KUNIT_EXPECT_TRUE(test, mc16_test_writeback_revert(w));
+ KUNIT_EXPECT_EQ(test, f->lookups, 0U);
+ KUNIT_EXPECT_EQ(test, refcount_read(&w->writer.slot->refs), refs);
+ mc16_test_query_unchanged(test, w->refs->storage, &before);
+}
+
+static void mc16_mincore_test_swapped(struct kunit *test, bool cached, bool uptodate)
+{
+ struct mc16_test_writeback *w = mc16_test_writeback_begin(test, false);
+ struct mc16_test_query_observation before;
+ struct mc16_mincore_test *f;
+ unsigned int refs;
+ int dups, puts;
+
+ KUNIT_ASSERT_NOT_NULL(test, w);
+ KUNIT_ASSERT_EQ(test, mc16_mincore_test_writeback(w, true), 0);
+ f = mc16_mincore_test_begin(test, w->writer.token);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->cached = cached;
+ if (uptodate)
+ folio_mark_uptodate(f->cache);
+ else
+ folio_clear_uptodate(f->cache);
+ refs = refcount_read(&w->writer.slot->refs);
+ dups = atomic_read(&memcomp_kunit_slot_dups);
+ puts = atomic_read(&memcomp_kunit_slot_puts);
+ mc16_test_query_observe(w->refs->storage, &before);
+ KUNIT_EXPECT_EQ(test, memcomp_mincore(w->writer.token), cached && uptodate);
+ KUNIT_EXPECT_EQ(test, f->lookups, 1U);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_dups), dups + 1);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_puts), puts + 1);
+ KUNIT_EXPECT_EQ(test, refcount_read(&w->writer.slot->refs), refs);
+ mc16_test_query_unchanged(test, w->refs->storage, &before);
+ mc16_mincore_test_expect_unlocked(test, f);
+}
+
+static void memcomp_test_mincore_swapped_absent(struct kunit *test)
+{
+ mc16_mincore_test_swapped(test, false, false);
+}
+
+static void memcomp_test_mincore_swapped_cached(struct kunit *test)
+{
+ mc16_mincore_test_swapped(test, true, true);
+}
+
+static void memcomp_test_mincore_swapped_incomplete(struct kunit *test)
+{
+ mc16_mincore_test_swapped(test, true, false);
+}
+
+#define MC16_MINCORE_SWAP_TEST_ENABLED
+
+#ifdef CONFIG_SHMEM
+static void mc16_mincore_test_shmem(struct kunit *test, bool race)
+{
+ struct mc16_test_writeback *w = mc16_test_writeback_begin(test, false);
+ struct mc16_mincore_test *f;
+ struct address_space *mapping;
+ void *entry;
+ struct folio *folio;
+
+ KUNIT_ASSERT_NOT_NULL(test, w);
+ f = mc16_mincore_test_begin(test, w->writer.token);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->file = shmem_kernel_file_setup("memcomp-mincore", MC16_MAX_PAGES * PAGE_SIZE,
+ VM_NORESERVE);
+ if (IS_ERR(f->file)) {
+ int error = PTR_ERR(f->file);
+
+ f->file = NULL;
+ KUNIT_FAIL(test, "shmem fixture: %d", error);
+ return;
+ }
+ mapping = f->file->f_mapping;
+ entry = swp_to_radix_entry(make_memcomp_entry(w->writer.token));
+ KUNIT_ASSERT_FALSE(test, xa_is_err(xa_store(&mapping->i_pages, 0, entry,
+ GFP_KERNEL)));
+ folio = filemap_get_incore_folio(mapping, 0);
+ if (!IS_ERR(folio))
+ folio_put(folio);
+ KUNIT_EXPECT_PTR_EQ(test, folio, ERR_PTR(-EEXIST));
+ KUNIT_EXPECT_EQ(test, memcomp_test_mincore_page(mapping, 0), (unsigned char)1);
+ KUNIT_ASSERT_EQ(test, mc16_mincore_test_writeback(w, false), 0);
+ f->dups = atomic_read(&memcomp_kunit_slot_dups);
+ f->change_xa = race;
+ KUNIT_EXPECT_EQ(test, memcomp_test_mincore_page(mapping, 0),
+ (unsigned char)(race ? 1 : 0));
+ KUNIT_EXPECT_EQ(test, f->lookups, 1U);
+ if (race) {
+ KUNIT_EXPECT_TRUE(test, f->changed);
+ } else {
+ f->cached = true;
+ folio_mark_uptodate(f->cache);
+ KUNIT_EXPECT_EQ(test, memcomp_test_mincore_page(mapping, 0),
+ (unsigned char)1);
+ folio_clear_uptodate(f->cache);
+ KUNIT_EXPECT_EQ(test, memcomp_test_mincore_page(mapping, 0),
+ (unsigned char)0);
+ }
+ entry = swp_to_radix_entry(make_memcomp_entry(0));
+ KUNIT_ASSERT_FALSE(test, xa_is_err(xa_store(&mapping->i_pages, 1, entry,
+ GFP_KERNEL)));
+ KUNIT_EXPECT_EQ(test, memcomp_test_mincore_page(mapping, 1), (unsigned char)1);
+#ifdef MC16_FAULT_MM_TEST_ENABLED
+ {
+ unsigned char vec[2] = { };
+ unsigned long addr;
+ spinlock_t *ptl; /* Allocate empty PTEs, then drop PTL before walking. */
+ pte_t *ptep;
+ int ret;
+
+ /* Empty but allocated PTEs reach the PTL-dropping shmem branch. */
+ addr = kunit_vm_mmap(test, f->file, 0, 2 * PAGE_SIZE,
+ PROT_READ | PROT_WRITE, MAP_SHARED, 0);
+ KUNIT_ASSERT_FALSE(test, IS_ERR_VALUE(addr));
+ KUNIT_ASSERT_NE(test, addr, 0UL);
+ mmap_read_lock(current->mm);
+ ptep = get_locked_pte(current->mm, addr, &ptl);
+ if (ptep) {
+ __acquire(RCU);
+ pte_unmap_unlock(ptep, ptl);
+ }
+ f->cached = false;
+ ret = ptep ? memcomp_test_mincore_walk(current->mm, addr,
+ addr + 2 * PAGE_SIZE, vec) : -ENOMEM;
+ mmap_read_unlock(current->mm);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ KUNIT_EXPECT_EQ(test, vec[0], (unsigned char)(race ? 1 : 0));
+ KUNIT_EXPECT_EQ(test, vec[1], (unsigned char)1);
+ }
+#endif
+ mc16_mincore_test_expect_unlocked(test, f);
+}
+
+static void memcomp_test_mincore_shmem(struct kunit *test)
+{
+ mc16_mincore_test_shmem(test, false);
+}
+
+static void memcomp_test_mincore_shmem_replaced(struct kunit *test)
+{
+ mc16_mincore_test_shmem(test, true);
+}
+
+#define MC16_MINCORE_SHMEM_TEST_ENABLED
+#endif
+
+#ifdef MC16_FAULT_MM_TEST_ENABLED
+/* Change a real member to a synthetic device slot using the real WB transaction. */
+static int mc16_mincore_test_member_swap(struct mc16_ft *f, unsigned int index)
+{
+ struct memcomp_ref *ref = &f->refs->storage->refs[index];
+ struct mc16_writeback snapshot;
+ struct mc16_writeback_tail tail = { };
+ struct mc16_swap_fields fields = {
+ .type = MAX_SWAPFILES - 1, .swap_offset = 1024 + index,
+ .byte_offset = 0, .packed = false,
+ };
+ bool pinned = false;
+ int ret = -EBUSY;
+
+ mutex_lock(memcomp_lock_of(ref->token));
+ if (mc16_writeback_claim_locked(ref, &snapshot)) {
+ mc16_length_decode(snapshot.length, &fields.length);
+ ret = memcomp_slot_dup(swp_entry(fields.type, fields.swap_offset));
+ if (!ret) {
+ pinned = true;
+ ret = mc16_writeback_settle_locked(ref, &snapshot, &fields, &tail);
+ }
+ if (ret)
+ mc16_writeback_revert_locked(ref);
+ }
+ mutex_unlock(memcomp_lock_of(ref->token));
+ if (pinned && ret)
+ memcomp_slot_put(swp_entry(fields.type, fields.swap_offset));
+ if (!ret)
+ mc16_writeback_finish(ref, &tail);
+ return ret;
+}
+
+static void memcomp_test_mincore_mthp(struct kunit *test)
+{
+ struct mc16_ft *mm = mc16_ft_begin(test, MC16_FT_PAGES, false, false);
+ struct mc16_mincore_test *f;
+ unsigned char vec[MC16_FT_PAGES] = { };
+ unsigned int i;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, mm);
+ KUNIT_ASSERT_EQ(test, mc16_mincore_test_member_swap(mm, 1), 0);
+ KUNIT_ASSERT_EQ(test, mc16_mincore_test_member_swap(mm, 3), 0);
+ f = mc16_mincore_test_begin(test, mm->tokens[1]);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mmap_read_lock(mm->mm);
+ ret = memcomp_test_mincore_walk(mm->mm, mm->addr,
+ mm->addr + sizeof(vec) * PAGE_SIZE, vec);
+ mmap_read_unlock(mm->mm);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ for (i = 0; i < ARRAY_SIZE(vec); i++)
+ KUNIT_EXPECT_EQ(test, vec[i], (unsigned char)(!(i & 1)));
+ KUNIT_EXPECT_EQ(test, f->lookups, 2U);
+ f->cached = true;
+ folio_mark_uptodate(f->cache);
+ mmap_read_lock(mm->mm);
+ ret = memcomp_test_mincore_walk(mm->mm, mm->addr,
+ mm->addr + sizeof(vec) * PAGE_SIZE, vec);
+ mmap_read_unlock(mm->mm);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ for (i = 0; i < ARRAY_SIZE(vec); i++)
+ KUNIT_EXPECT_EQ(test, vec[i], (unsigned char)1);
+ mc16_mincore_test_expect_unlocked(test, f);
+}
+
+static void memcomp_test_mincore_pte_replaced(struct kunit *test)
+{
+ struct mc16_ft *mm = mc16_ft_begin(test, 1, false, false);
+ struct mc16_mincore_test *f;
+ unsigned char vec = 0;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, mm);
+ KUNIT_ASSERT_EQ(test, mc16_mincore_test_member_swap(mm, 0), 0);
+ f = mc16_mincore_test_begin(test, mm->tokens[0]);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->fault = mm;
+ mmap_read_lock(mm->mm);
+ ret = memcomp_test_mincore_walk(mm->mm, mm->addr, mm->addr + PAGE_SIZE, &vec);
+ mmap_read_unlock(mm->mm);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ KUNIT_EXPECT_TRUE(test, f->changed);
+ KUNIT_EXPECT_EQ(test, vec, (unsigned char)1);
+ KUNIT_EXPECT_EQ(test, f->lookups, 1U);
+ mc16_mincore_test_expect_unlocked(test, f);
+}
+
+#define MC16_MINCORE_MM_TEST_ENABLED
+#endif /* MC16_FAULT_MM_TEST_ENABLED */
+#endif /* CONFIG_MEMCG && CONFIG_SWAP */
+#endif /* !MC16_MINCORE_DECLARE_TEST_API */
+#endif /* _MM_MEMCOMP_TESTS_MINCORE_TEST_H */
diff --git a/mm/memcomp/tests/owner_test.h b/mm/memcomp/tests/owner_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..820bba73899809779a1c406391e6e05bd8935bec
--- /dev/null
+++ b/mm/memcomp/tests/owner_test.h
@@ -0,0 +1,400 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_OWNER_TEST_H
+#define _MM_MEMCOMP_TESTS_OWNER_TEST_H
+#ifdef CONFIG_MEMCG
+/* Actions run before refs/storage cleanup, including after a fatal assert. */
+struct mc16_test_owner {
+ struct mc16_test_refs *refs;
+ struct memcomp_ref lookup;
+ struct memcomp_ref survivor[2];
+ struct mc16_owner_tail tail[2];
+};
+
+static void mc16_test_owner_cleanup(void *data)
+{
+ struct mc16_test_owner *f = data;
+ unsigned int i;
+
+ if (f->lookup.slot)
+ mc16_ref_put(&f->lookup);
+ for (i = 0; i < ARRAY_SIZE(f->survivor); i++) {
+ if (f->survivor[i].slot)
+ mc16_ref_put(&f->survivor[i]);
+ }
+ /* Siblings first, producer/head last, also on assertion failure. */
+ for (i = ARRAY_SIZE(f->tail); i; i--)
+ mc16_owner_tail_put(&f->tail[i - 1]);
+}
+
+static struct mc16_test_owner *
+mc16_test_owner_begin(struct kunit *test, unsigned int nr)
+{
+ struct mc16_test_owner *f;
+ int ret;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->refs = mc16_test_refs_begin(test);
+ if (!f->refs)
+ return NULL;
+ ret = kunit_add_action_or_reset(test, mc16_test_owner_cleanup, f);
+ if (ret || mc16_test_refs_reserve(f->refs, nr))
+ return NULL;
+ return f;
+}
+
+static void mc16_test_owner_normalize(struct kunit *test, unsigned int state)
+{
+ struct mc16_test_owner *f = mc16_test_owner_begin(test, 2);
+ struct mc16_test_storage *storage;
+ struct mc16_record *record;
+ struct memcomp_ref *ref;
+ struct mc16_view view;
+ unsigned int i;
+ long record_charge;
+ u64 word;
+ u16 length, flags;
+ u32 base;
+ unsigned int index;
+ bool dropped;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ struct memcomp_state_stats before, after;
+#endif
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->refs->storage;
+ for (i = 0; i < 2; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_payload(f->refs, i, state), 0);
+ record = (void *)(unsigned long)storage->refs[0].slot->word;
+ record_charge = record->charge_bytes;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ memcomp_state_read(&before);
+#endif
+ /* Exercise producer/head-last, using real lookups for surviving holds. */
+ for (i = 2; i; ) {
+ i--;
+ ref = &storage->refs[i];
+ KUNIT_ASSERT_TRUE(test, mc16_get(ref->token, &f->survivor[i]));
+ KUNIT_ASSERT_TRUE(test, mc16_get(ref->token, &f->lookup));
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ word = *view.word;
+ length = *view.length;
+ flags = *view.flags & ~MC16_F_OWNER;
+ dropped = mc16_owner_drop_locked(&f->lookup, storage->owner->folio,
+ 2, i, &f->tail[i]);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ /* Complete consumption before any fatal assertion can unwind. */
+ mc16_ref_put_count(&f->lookup, dropped ? 2 : 1);
+ if (dropped)
+ *ref = (struct memcomp_ref) { };
+ KUNIT_ASSERT_TRUE(test, dropped);
+ ref = &f->survivor[i];
+ KUNIT_EXPECT_PTR_EQ(test, f->lookup.slot, NULL);
+ KUNIT_ASSERT_PTR_EQ(test, f->tail[i].record, record);
+ KUNIT_EXPECT_FALSE(test, f->tail[i].resident);
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), 1U);
+ KUNIT_EXPECT_EQ(test, ref->slot->word, word);
+ KUNIT_EXPECT_EQ(test, ref->slot->length, length);
+ /* Pair with normalize's release publication of word/length. */
+ KUNIT_EXPECT_EQ(test, smp_load_acquire(&ref->slot->flags), flags);
+ KUNIT_EXPECT_EQ(test, mc16_ref_geometry(ref, &base, &index), 2U);
+ KUNIT_EXPECT_EQ(test, base, record->token_base);
+ KUNIT_EXPECT_EQ(test, index, i);
+ KUNIT_EXPECT_EQ(test, record->member[i].payload_charge, 0U);
+ KUNIT_EXPECT_PTR_EQ(test, record->member[i].owner_folio, NULL);
+ KUNIT_EXPECT_EQ(test, refcount_read(&record->members), 2U);
+ }
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ memcomp_state_read(&after);
+ KUNIT_EXPECT_EQ(test, after.folio_owner_pages, before.folio_owner_pages - 2);
+#endif
+ /* Independent tails survive slot release and explicit key reclaim. */
+ mc16_ref_put(&f->survivor[1]);
+ mc16_ref_put(&f->survivor[0]);
+ /* Exact metadata balance is checked after explicit pressure reclaim. */
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&storage->owner->model.charged),
+ record_charge);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_compressed_bytes),
+ f->refs->compressed_before);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(storage->owner->folio), 1);
+ mc16_owner_tail_put(&f->tail[1]);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->tail[0].record->members), 1U);
+ mc16_owner_tail_put(&f->tail[0]);
+ /* No record access after the final put, including cleanup retries. */
+ mc16_owner_tail_put(&f->tail[0]);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&storage->owner->model.charged), 0L);
+}
+
+static void memcomp_test_owner_stored_normalize(struct kunit *test)
+{
+ mc16_test_owner_normalize(test, MC16_STORED);
+}
+
+static void memcomp_test_owner_samefill_normalize(struct kunit *test)
+{
+ mc16_test_owner_normalize(test, MC16_SAMEFILL);
+}
+
+/* A writer's batch guard survives OWNER and the detached record metadata. */
+static void memcomp_test_owner_writeback_guard(struct kunit *test)
+{
+ struct mc16_test_owner *f = mc16_test_owner_begin(test, 1);
+ struct mc16_test_storage *storage;
+ struct memcomp_ref *ref;
+ struct mc16_view view;
+ struct memcomp_lru *fifo;
+ long fifo_before, charged, record_charge;
+ unsigned long handle;
+ void *data;
+ u32 token;
+ u16 length;
+ u8 marker;
+ int puts_before;
+ bool dropped;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->refs->storage;
+ ref = &storage->refs[0];
+ token = ref->token;
+ fifo = mc16_fifo_of(ref);
+ fifo_before = atomic_long_read(&fifo->nr_entries);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_payload(f->refs, 0, MC16_STORED), 0);
+ KUNIT_EXPECT_EQ(test, ref->cluster->eligible_slots, (u16)1);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&fifo->nr_entries), fifo_before + 1);
+ /* survivor is the actual extra slot reference owned by the batch. */
+ KUNIT_ASSERT_TRUE(test, mc16_get(token, &f->survivor[0]));
+ KUNIT_ASSERT_TRUE(test, mc16_get(token, &f->lookup));
+ mutex_lock(memcomp_lock_of(token));
+ mc16_view_locked(ref, &view);
+ handle = *view.word;
+ length = *view.length;
+ record_charge = view.record->charge_bytes;
+ /* The claim performs its existing L removal before OWNER is dropped. */
+ mc16_fifo_transition(ref, MC16_STORED, MC16_WRITEBACK);
+ *view.flags = (*view.flags & ~MC16_STATE_MASK) | MC16_WRITEBACK;
+ mutex_unlock(memcomp_lock_of(token));
+ data = zs_map_object(memcomp_pools[0], handle, ZS_MM_WO);
+ memset(data, 0x5a, PAGE_SIZE);
+ zs_unmap_object(memcomp_pools[0], handle);
+ storage->payload.task = current;
+ storage->payload.pool = memcomp_pools[0];
+ storage->payload.handle = handle;
+ storage->payload.live = true;
+ rcu_assign_pointer(memcomp_kunit_async_probe, &storage->payload);
+ charged = atomic_long_read(&storage->owner->model.charged);
+ puts_before = atomic_read(&storage->owner->model.put_calls);
+
+ dropped = mc16_owner_drop_folio_held(&f->lookup, storage->owner->folio,
+ 1, 0);
+ if (dropped)
+ *ref = (struct memcomp_ref) { };
+ KUNIT_ASSERT_TRUE(test, dropped);
+ KUNIT_EXPECT_PTR_EQ(test, f->lookup.slot, NULL);
+ ref = &f->survivor[0];
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), 1U);
+ KUNIT_EXPECT_EQ(test, ref->slot->word, (u64)handle);
+ KUNIT_EXPECT_EQ(test, ref->slot->length, length);
+ KUNIT_EXPECT_EQ(test, ref->slot->flags, (u16)MC16_WRITEBACK);
+ /* Record's last member was put, without consuming the batch's handle. */
+ KUNIT_EXPECT_EQ(test, atomic_read(&storage->owner->model.put_calls),
+ puts_before + 1);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&storage->owner->model.charged),
+ charged - record_charge);
+ KUNIT_EXPECT_EQ(test, storage->payload.frees, 0U);
+ KUNIT_ASSERT_TRUE(test, storage->payload.live);
+ data = zs_map_object(memcomp_pools[0], handle, ZS_MM_RO);
+ marker = *(u8 *)data;
+ zs_unmap_object(memcomp_pools[0], handle);
+ KUNIT_EXPECT_EQ(test, marker, (u8)0x5a);
+ KUNIT_EXPECT_EQ(test, ref->cluster->eligible_slots, (u16)0);
+ KUNIT_EXPECT_TRUE(test, list_empty(&ref->cluster->fifo));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&fifo->nr_entries), fifo_before);
+ /* WRITEBACK -> EMPTY has no eligibility change and takes no extra L. */
+ mc16_ref_put(ref);
+ KUNIT_EXPECT_PTR_EQ(test, ref->slot, NULL);
+ KUNIT_EXPECT_EQ(test, storage->payload.frees, 1U);
+ KUNIT_EXPECT_FALSE(test, storage->payload.live);
+ KUNIT_EXPECT_FALSE(test, storage->payload.bad_lifetime);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&fifo->nr_entries), fifo_before);
+ KUNIT_EXPECT_FALSE(test, mc16_get(token, &f->lookup));
+ /* Exact metadata balance is checked after explicit pressure reclaim. */
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&storage->owner->model.charged), 0L);
+}
+
+static void memcomp_test_owner_resident_final_guard(struct kunit *test)
+{
+ struct mc16_test_owner *f = mc16_test_owner_begin(test, 1);
+ struct mc16_test_storage *storage;
+ struct mc16_view view;
+ struct memcomp_ref *ref;
+ /* Models the PTE lock around the final reference release. */
+ spinlock_t ptl;
+ unsigned long irqflags;
+ u32 token;
+ u16 flags;
+ bool dropped, attached;
+ int folio_refs;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->refs->storage;
+ ref = &storage->refs[0];
+ token = ref->token;
+ KUNIT_ASSERT_TRUE(test, mc16_get(token, &f->survivor[0]));
+ KUNIT_ASSERT_TRUE(test, mc16_get(token, &f->lookup));
+ dropped = mc16_owner_drop_folio_held(&f->lookup, storage->owner->folio,
+ 1, 0);
+ if (dropped)
+ *ref = (struct memcomp_ref) { };
+ KUNIT_ASSERT_TRUE(test, dropped);
+ KUNIT_EXPECT_PTR_EQ(test, f->lookup.slot, NULL);
+ ref = &f->survivor[0];
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), 1U);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(ref->slot->flags),
+ (unsigned int)MC16_PENDING);
+ mutex_lock(memcomp_lock_of(token));
+ mc16_view_locked(ref, &view);
+ attached = !!view.record;
+ flags = *view.flags;
+ mutex_unlock(memcomp_lock_of(token));
+ KUNIT_EXPECT_TRUE(test, attached);
+ KUNIT_EXPECT_FALSE(test, flags & MC16_F_OWNER);
+ KUNIT_EXPECT_TRUE(test, flags & MC16_F_RESIDENT);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(storage->owner->folio), 2);
+ /* Real refs-zero release while holding a modeled PTE spinlock. */
+ spin_lock_init(&ptl);
+ spin_lock_irqsave(&ptl, irqflags);
+ mc16_ref_put(ref);
+ folio_refs = folio_ref_count(storage->owner->folio);
+ spin_unlock_irqrestore(&ptl, irqflags);
+ KUNIT_EXPECT_EQ(test, folio_refs, 1);
+ KUNIT_EXPECT_PTR_EQ(test, ref->slot, NULL);
+ KUNIT_EXPECT_FALSE(test, mc16_get(token, &f->lookup));
+ /* Exact metadata balance is checked after explicit pressure reclaim. */
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&storage->owner->model.charged), 0L);
+}
+
+static void memcomp_test_owner_last_pair(struct kunit *test)
+{
+ struct mc16_test_owner *f = mc16_test_owner_begin(test, 1);
+ struct mc16_test_storage *storage;
+ struct memcomp_ref *ref;
+ bool dropped;
+ u32 token;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->refs->storage;
+ ref = &storage->refs[0];
+ token = ref->token;
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_payload(f->refs, 0, MC16_STORED), 0);
+ storage->payload.task = current;
+ storage->payload.pool = memcomp_pools[0];
+ storage->payload.handle = ((struct mc16_record *)
+ (unsigned long)ref->slot->word)->member[0].word;
+ storage->payload.live = true;
+ rcu_assign_pointer(memcomp_kunit_async_probe, &storage->payload);
+ KUNIT_ASSERT_TRUE(test, mc16_get(token, &f->lookup));
+ dropped = mc16_owner_drop_folio_held(&f->lookup, storage->owner->folio,
+ 1, 0);
+ if (dropped)
+ *ref = (struct memcomp_ref) { };
+ KUNIT_ASSERT_TRUE(test, dropped);
+ KUNIT_EXPECT_PTR_EQ(test, f->lookup.slot, NULL);
+ KUNIT_EXPECT_EQ(test, storage->payload.frees, 1U);
+ KUNIT_EXPECT_FALSE(test, storage->payload.live);
+ KUNIT_EXPECT_FALSE(test, storage->payload.bad_lifetime);
+ KUNIT_EXPECT_FALSE(test, mc16_get(token, &f->lookup));
+ /* Exact metadata balance is checked after explicit pressure reclaim. */
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&storage->owner->model.charged), 0L);
+}
+
+/* A stale attempt consumes exactly its lookup; the initial OWNER survives. */
+static void
+mc16_test_owner_stale(struct kunit *test, struct mc16_test_owner *f,
+ struct folio *folio, unsigned int nr, unsigned int index)
+{
+ struct mc16_test_storage *storage = f->refs->storage;
+ struct memcomp_ref *ref = &storage->refs[0];
+ struct mc16_member before;
+ struct mc16_record *record;
+ bool dropped;
+ u64 physical;
+ u16 flags;
+
+ KUNIT_ASSERT_TRUE(test, mc16_get(ref->token, &f->lookup));
+ mutex_lock(memcomp_lock_of(ref->token));
+ record = (void *)(unsigned long)ref->slot->word;
+ before = record->member[0];
+ physical = ref->slot->word;
+ flags = ref->slot->flags;
+ mutex_unlock(memcomp_lock_of(ref->token));
+ dropped = mc16_owner_drop_folio_held(&f->lookup, folio, nr, index);
+ /* Retain cleanup ownership even if an unexpected success consumed it. */
+ if (dropped)
+ *ref = (struct memcomp_ref) { };
+ KUNIT_ASSERT_FALSE(test, dropped);
+ KUNIT_EXPECT_PTR_EQ(test, f->lookup.slot, NULL);
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), 1U);
+ KUNIT_EXPECT_EQ(test, ref->slot->word, physical);
+ KUNIT_EXPECT_EQ(test, ref->slot->flags, flags);
+ KUNIT_EXPECT_EQ(test, record->member[0].word, before.word);
+ KUNIT_EXPECT_EQ(test, record->member[0].length, before.length);
+ KUNIT_EXPECT_EQ(test, record->member[0].flags, before.flags);
+ KUNIT_EXPECT_EQ(test, record->member[0].payload_charge, before.payload_charge);
+ KUNIT_EXPECT_PTR_EQ(test, record->member[0].owner_folio, before.owner_folio);
+ KUNIT_EXPECT_EQ(test, refcount_read(&record->members), 1U);
+}
+
+static void memcomp_test_owner_stale(struct kunit *test)
+{
+ struct mc16_test_owner *f = mc16_test_owner_begin(test, 1);
+ struct mc16_test_storage *storage;
+ struct mc16_record *record;
+ struct memcomp_ref *ref;
+ u16 flags;
+ bool dropped;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->refs->storage;
+ mc16_test_owner_stale(test, f, NULL, 1, 0);
+ KUNIT_ASSERT_NOT_NULL(test, storage->refs[0].slot);
+ mc16_test_owner_stale(test, f, storage->owner->folio, 2, 0);
+ KUNIT_ASSERT_NOT_NULL(test, storage->refs[0].slot);
+ mc16_test_owner_stale(test, f, storage->owner->folio, 1, 1);
+ KUNIT_ASSERT_NOT_NULL(test, storage->refs[0].slot);
+ ref = &storage->refs[0];
+ KUNIT_ASSERT_TRUE(test, mc16_get(ref->token, &f->lookup));
+ /* OWNER absent and nonresident PENDING are independently rejected. */
+ mutex_lock(memcomp_lock_of(ref->token));
+ record = (void *)(unsigned long)ref->slot->word;
+ flags = record->member[0].flags;
+ record->member[0].flags = flags & ~MC16_F_OWNER;
+ dropped = mc16_owner_drop_locked(&f->lookup, storage->owner->folio,
+ 1, 0, &f->tail[0]);
+ record->member[0].flags = flags;
+ mutex_unlock(memcomp_lock_of(ref->token));
+ KUNIT_EXPECT_FALSE(test, dropped);
+ KUNIT_EXPECT_PTR_EQ(test, f->tail[0].record, NULL);
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), 2U);
+ mutex_lock(memcomp_lock_of(ref->token));
+ record->member[0].flags = flags & ~MC16_F_RESIDENT;
+ dropped = mc16_owner_drop_locked(&f->lookup, storage->owner->folio,
+ 1, 0, &f->tail[0]);
+ record->member[0].flags = flags;
+ mutex_unlock(memcomp_lock_of(ref->token));
+ KUNIT_EXPECT_FALSE(test, dropped);
+ KUNIT_EXPECT_PTR_EQ(test, f->tail[0].record, NULL);
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), 2U);
+ mc16_ref_put(&f->lookup);
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), 1U);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(storage->owner->folio), 2);
+}
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_OWNER_TEST_H */
diff --git a/mm/memcomp/tests/payload_test.h b/mm/memcomp/tests/payload_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..46c1b81fadccb9582385c3ac738ea365adc07888
--- /dev/null
+++ b/mm/memcomp/tests/payload_test.h
@@ -0,0 +1,220 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_PAYLOAD_TEST_H
+#define _MM_MEMCOMP_TESTS_PAYLOAD_TEST_H
+#ifdef CONFIG_MEMCG
+struct mc16_test_payload {
+ struct mc16_test_refs *refs;
+ struct mc16_payload prepared;
+ struct mc16_payload_tail tail;
+};
+
+static void mc16_test_payload_cleanup(void *data)
+{
+ struct mc16_test_payload *f = data;
+ struct memcomp_ref *ref = &f->refs->storage->refs[0];
+
+ mc16_payload_finish(ref, &f->tail);
+ mc16_payload_abort(ref, &f->prepared);
+}
+
+static struct mc16_test_payload *mc16_test_payload_begin(struct kunit *test)
+{
+ struct mc16_test_payload *f;
+ int ret;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->refs = mc16_test_refs_begin(test);
+ if (!f->refs)
+ return NULL;
+ ret = mc16_test_refs_reserve(f->refs, 1);
+ if (ret)
+ return NULL;
+ ret = kunit_add_action_or_reset(test, mc16_test_payload_cleanup, f);
+ return ret ? NULL : f;
+}
+
+static int mc16_test_payload_prepare(struct mc16_test_payload *f, u32 bytes,
+ u64 pattern)
+{
+ struct memcomp_ref *ref = &f->refs->storage->refs[0];
+ unsigned long handle;
+ int ret;
+
+ if (bytes) {
+ ret = memcomp_zs_malloc(memcomp_pools[0], bytes, &handle, false);
+ if (ret)
+ return ret;
+ pattern = handle;
+ }
+ f->prepared = (struct mc16_payload) { .word = pattern, .bytes = bytes };
+ return mc16_payload_charge(ref, &f->prepared);
+}
+
+static int mc16_test_payload_commit(struct mc16_test_payload *f,
+ struct folio *folio)
+{
+ struct memcomp_ref *ref = &f->refs->storage->refs[0];
+ int ret;
+
+ mutex_lock(memcomp_lock_of(ref->token));
+ ret = mc16_payload_commit_locked(ref, folio, 1, 0,
+ &f->prepared, &f->tail);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ return ret;
+}
+
+static void memcomp_test_payload_samefill(struct kunit *test)
+{
+ const u64 pattern = 0xfedcba9876543210ULL;
+ struct mc16_test_payload *f = mc16_test_payload_begin(test);
+ struct mc16_test_storage *storage;
+ struct memcomp_ref *ref;
+ struct mc16_view view;
+ u64 word;
+ u16 flags;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->refs->storage;
+ ref = &storage->refs[0];
+ KUNIT_ASSERT_EQ(test, mc16_test_payload_prepare(f, 0, pattern), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_payload_commit(f, storage->owner->folio), 0);
+ KUNIT_EXPECT_EQ(test, f->prepared.word, 0ULL);
+ mc16_payload_finish(ref, &f->tail);
+ mc16_payload_finish(ref, &f->tail); /* Empty tail must be harmless. */
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ word = *view.word;
+ flags = *view.flags;
+ mutex_unlock(memcomp_lock_of(ref->token));
+ KUNIT_EXPECT_EQ(test, word, pattern);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(flags), (unsigned int)MC16_SAMEFILL);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(ref->slot->flags),
+ (unsigned int)MC16_PENDING);
+ KUNIT_EXPECT_FALSE(test, flags & MC16_F_RESIDENT);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(storage->owner->folio), 1);
+ KUNIT_EXPECT_EQ(test, ref->cluster->eligible_slots, (u16)0);
+}
+
+static void memcomp_test_payload_replace(struct kunit *test)
+{
+ struct mc16_test_payload *f = mc16_test_payload_begin(test);
+ struct mc16_test_storage *storage;
+ struct memcomp_ref *ref;
+ long charged;
+ u64 handle;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->refs->storage;
+ ref = &storage->refs[0];
+ charged = atomic_long_read(&storage->owner->model.charged);
+ KUNIT_ASSERT_EQ(test, mc16_test_payload_prepare(f, PAGE_SIZE, 0), 0);
+ handle = f->prepared.word;
+ KUNIT_ASSERT_EQ(test, mc16_test_payload_commit(f, storage->owner->folio), 0);
+ mc16_payload_finish(ref, &f->tail);
+ KUNIT_EXPECT_EQ(test, ref->cluster->eligible_slots, (u16)1);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&storage->owner->model.charged),
+ charged + PAGE_SIZE);
+
+ storage->payload.task = current;
+ storage->payload.pool = memcomp_pools[0];
+ storage->payload.handle = handle;
+ storage->payload.live = true;
+ rcu_assign_pointer(memcomp_kunit_async_probe, &storage->payload);
+ KUNIT_ASSERT_EQ(test, mc16_test_payload_prepare(f, 0, U64_MAX), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_payload_commit(f, storage->owner->folio), 0);
+ mc16_payload_finish(ref, &f->tail);
+ KUNIT_EXPECT_EQ(test, storage->payload.frees, 1U);
+ KUNIT_EXPECT_FALSE(test, storage->payload.live);
+ KUNIT_EXPECT_FALSE(test, storage->payload.bad_lifetime);
+ KUNIT_EXPECT_EQ(test, ref->cluster->eligible_slots, (u16)0);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&storage->owner->model.charged),
+ charged);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(storage->owner->folio), 1);
+}
+
+static void memcomp_test_payload_stale(struct kunit *test)
+{
+ struct mc16_test_payload *f = mc16_test_payload_begin(test);
+ struct mc16_test_storage *storage;
+ struct memcomp_ref *ref;
+ long charged;
+ u64 handle, record;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->refs->storage;
+ ref = &storage->refs[0];
+ charged = atomic_long_read(&storage->owner->model.charged);
+ record = ref->slot->word;
+ KUNIT_ASSERT_EQ(test, mc16_test_payload_prepare(f, 128, 0), 0);
+ handle = f->prepared.word;
+ KUNIT_EXPECT_EQ(test, mc16_test_payload_commit(f, NULL), -ESTALE);
+ KUNIT_EXPECT_EQ(test, ref->slot->word, record);
+ KUNIT_EXPECT_EQ(test, f->prepared.word, handle);
+ KUNIT_EXPECT_EQ(test, f->prepared.charge, 128U);
+ KUNIT_EXPECT_EQ(test, f->tail.flags, (u16)0);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(storage->owner->folio), 2);
+ mc16_payload_abort(ref, &f->prepared);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&storage->owner->model.charged),
+ charged);
+}
+
+static void memcomp_test_payload_charge_failure(struct kunit *test)
+{
+ struct mc16_test_payload *f = mc16_test_payload_begin(test);
+ struct mc16_test_storage *storage;
+ struct memcomp_ref *ref;
+ long charged;
+ u64 record;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->refs->storage;
+ ref = &storage->refs[0];
+ charged = atomic_long_read(&storage->owner->model.charged);
+ record = ref->slot->word;
+ storage->owner->model.fail_at =
+ atomic_read(&storage->owner->model.charge_calls) + 1;
+ KUNIT_EXPECT_EQ(test, mc16_test_payload_prepare(f, 128, 0), -ENOMEM);
+ KUNIT_EXPECT_EQ(test, f->prepared.charge, 0U);
+ KUNIT_EXPECT_EQ(test, ref->slot->word, record);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&storage->owner->model.charged),
+ charged);
+ mc16_payload_abort(ref, &f->prepared);
+ storage->owner->model.fail_at = 0;
+}
+
+static void memcomp_test_payload_writeback(struct kunit *test)
+{
+ struct mc16_test_payload *f = mc16_test_payload_begin(test);
+ struct mc16_test_storage *storage;
+ struct memcomp_ref *ref;
+ struct mc16_view view;
+ u64 word;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->refs->storage;
+ ref = &storage->refs[0];
+ KUNIT_ASSERT_EQ(test, mc16_test_payload_prepare(f, PAGE_SIZE, 0), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_payload_commit(f, storage->owner->folio), 0);
+ mc16_payload_finish(ref, &f->tail);
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ word = *view.word;
+ *view.flags = (*view.flags & ~MC16_STATE_MASK) | MC16_WRITEBACK;
+ mc16_fifo_transition(ref, MC16_STORED, MC16_WRITEBACK);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ KUNIT_ASSERT_EQ(test, mc16_test_payload_prepare(f, 0, U64_MAX), 0);
+ KUNIT_EXPECT_EQ(test, mc16_test_payload_commit(f, storage->owner->folio),
+ -EAGAIN);
+ KUNIT_EXPECT_EQ(test, f->prepared.word, U64_MAX);
+ KUNIT_EXPECT_EQ(test, f->tail.flags, (u16)0);
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ KUNIT_EXPECT_EQ(test, *view.word, word);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(*view.flags),
+ (unsigned int)MC16_WRITEBACK);
+ mutex_unlock(memcomp_lock_of(ref->token));
+}
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_PAYLOAD_TEST_H */
diff --git a/mm/memcomp/tests/public_api_test.h b/mm/memcomp/tests/public_api_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..062224d7e290b68495708d311eefcd795946e7c1
--- /dev/null
+++ b/mm/memcomp/tests/public_api_test.h
@@ -0,0 +1,422 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_PUBLIC_API_TEST_H
+#define _MM_MEMCOMP_TESTS_PUBLIC_API_TEST_H
+
+/* Include after consumers_test.h. Definitions only; register in the main suite. */
+#ifdef CONFIG_MEMCG
+struct mc16_ct_snapshot {
+ u64 word;
+ u32 refs;
+ u16 flags;
+ u16 length;
+ bool attached;
+};
+
+/* Copy values under T. Never export a record/member pointer to an assertion. */
+static bool
+mc16_ct_snapshot(struct memcomp_ref *ref, struct mc16_ct_snapshot *snapshot)
+{
+ struct mc16_view view;
+ bool valid;
+
+ *snapshot = (struct mc16_ct_snapshot) { };
+ if (!ref->slot)
+ return false;
+ mutex_lock(memcomp_lock_of(ref->token));
+ valid = mc16_view_locked(ref, &view);
+ if (valid) {
+ snapshot->word = *view.word;
+ snapshot->flags = *view.flags;
+ snapshot->length = *view.length;
+ snapshot->refs = refcount_read(&ref->slot->refs);
+ snapshot->attached = !!view.record;
+ }
+ mutex_unlock(memcomp_lock_of(ref->token));
+ return valid;
+}
+
+struct mc16_ct {
+ struct mc16_consumer_test *base;
+ struct memcomp_ref guard;
+ unsigned long token;
+ bool cache_owner;
+ bool pte_owner;
+};
+
+static void mc16_ct_cleanup(void *data)
+{
+ struct mc16_ct *f = data;
+
+ f->base->model.fail_at = 0;
+ if (f->guard.slot)
+ memcomp_entry_put_ref(&f->guard);
+ if (f->pte_owner)
+ memcomp_invalidate(f->token);
+ if (f->cache_owner && f->token)
+ memcomp_invalidate_cache(f->token);
+}
+
+static struct mc16_ct *mc16_ct_begin(struct kunit *test)
+{
+ struct mc16_ct *f;
+ void *data;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->base = mc16_consumer_test_begin(test);
+ if (!f->base)
+ return NULL;
+ if (kunit_add_action_or_reset(test, mc16_ct_cleanup, f))
+ return NULL;
+ data = kmap_local_folio(f->base->account, 0);
+ memcomp_test_fill_zstd_ratio_page(data);
+ kunmap_local(data);
+ return f;
+}
+
+/* All locks are released before returning to any KUnit assertion. */
+static bool mc16_ct_store(struct mc16_ct *f)
+{
+ bool stored;
+
+ folio_lock(f->base->account);
+ stored = memcomp_store_cache(f->base->account, &f->token);
+ folio_unlock(f->base->account);
+ f->cache_owner = stored && f->token;
+ return stored;
+}
+
+/* Model the successful XA replacer's unlocked publication tail. */
+static void mc16_ct_publish(struct mc16_ct *f)
+{
+ folio_lock(f->base->account);
+ folio_set_swapbacked(f->base->account);
+ memcomp_publish_cache_eviction(f->base->account, &f->guard);
+ folio_unlock(f->base->account);
+}
+
+static int mc16_ct_load(struct mc16_ct *f, bool *equal)
+{
+ void *src, *dst;
+ int ret;
+
+ *equal = false;
+ folio_lock(f->base->dst);
+ ret = memcomp_load_folio(&f->guard, f->base->dst);
+ if (!ret) {
+ src = kmap_local_folio(f->base->account, 0);
+ dst = kmap_local_folio(f->base->dst, 0);
+ *equal = !memcmp(src, dst, PAGE_SIZE);
+ kunmap_local(dst);
+ kunmap_local(src);
+ }
+ folio_unlock(f->base->dst);
+ return ret;
+}
+
+static void memcomp_test_public_cache_transaction(struct kunit *test)
+{
+ struct mc16_ct *f = mc16_ct_begin(test);
+ struct mc16_ct_snapshot snapshot;
+ bool equal;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_ct_store(f));
+ KUNIT_ASSERT_NE(test, f->token, 0UL);
+ KUNIT_EXPECT_TRUE(test, memcomp_entry_present(f->token));
+ KUNIT_ASSERT_TRUE(test, memcomp_load_folio_pin(f->token, &f->guard));
+ KUNIT_ASSERT_TRUE(test, mc16_ct_snapshot(&f->guard, &snapshot));
+ KUNIT_EXPECT_FALSE(test, snapshot.attached);
+ KUNIT_EXPECT_EQ(test, snapshot.refs, 2U);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(snapshot.flags), MC16_STORED);
+ mc16_ct_publish(f);
+ ret = mc16_ct_load(f, &equal);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ KUNIT_EXPECT_TRUE(test, equal);
+ KUNIT_EXPECT_NOT_NULL(test, f->guard.slot);
+ memcomp_load_folio_abort(&f->guard);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_TRUE(test, memcomp_entry_present(f->token));
+ KUNIT_ASSERT_TRUE(test, memcomp_load_folio_pin(f->token, &f->guard));
+ ret = mc16_ct_load(f, &equal);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ KUNIT_EXPECT_TRUE(test, equal);
+ if (ret)
+ return;
+ memcomp_load_folio_commit(&f->guard);
+ f->cache_owner = false;
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_present(f->token));
+}
+
+static void memcomp_test_public_ref_domains(struct kunit *test)
+{
+ struct mc16_ct *f = mc16_ct_begin(test);
+ struct mc16_ct_snapshot snapshot;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_ct_store(f));
+ KUNIT_ASSERT_NE(test, f->token, 0UL);
+ KUNIT_ASSERT_TRUE(test, memcomp_entry_get_ref(f->token, &f->guard));
+ KUNIT_ASSERT_TRUE(test, memcomp_entry_get(f->token));
+ f->pte_owner = true;
+ KUNIT_ASSERT_TRUE(test, mc16_ct_snapshot(&f->guard, &snapshot));
+ KUNIT_EXPECT_EQ(test, snapshot.refs, 3U);
+ memcomp_invalidate(f->token);
+ f->pte_owner = false;
+ KUNIT_ASSERT_TRUE(test, mc16_ct_snapshot(&f->guard, &snapshot));
+ KUNIT_EXPECT_EQ(test, snapshot.refs, 2U);
+ memcomp_entry_put_ref(&f->guard);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.cluster, NULL);
+ KUNIT_EXPECT_EQ(test, f->guard.token, 0U);
+ KUNIT_EXPECT_TRUE(test, memcomp_entry_present(f->token));
+ memcomp_invalidate_cache(f->token);
+ f->cache_owner = false;
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_present(f->token));
+}
+
+static void memcomp_test_public_query_busy(struct kunit *test)
+{
+ struct mc16_ct *f = mc16_ct_begin(test);
+ struct memcomp_ref resident = { };
+ unsigned long compressed = ULONG_MAX, base = 0;
+ int maps = -1, diagnostic_maps = -1, busy;
+ bool refs_ok, diagnostic_ok, present;
+ unsigned int nr;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_ct_store(f));
+ KUNIT_ASSERT_NE(test, f->token, 0UL);
+ KUNIT_ASSERT_TRUE(test, memcomp_entry_get_ref(f->token, &f->guard));
+ nr = memcomp_entry_nr_pages(f->token, &base);
+ KUNIT_EXPECT_EQ(test, nr, 1U);
+ KUNIT_EXPECT_EQ(test, base, f->token);
+ mutex_lock(memcomp_lock_of(f->token));
+ refs_ok = memcomp_entry_stat(f->token, NULL, &maps);
+ diagnostic_ok = memcomp_entry_stat(f->token, &compressed, &diagnostic_maps);
+ present = memcomp_entry_present(f->token);
+ busy = memcomp_entry_resident_get(f->token, &resident);
+ mutex_unlock(memcomp_lock_of(f->token));
+ /* Do not leave an unexpected result's guard outside cleanup ownership. */
+ if (resident.slot)
+ memcomp_entry_resident_put(&resident);
+ KUNIT_EXPECT_TRUE(test, refs_ok);
+ KUNIT_EXPECT_GE(test, maps, 1);
+ KUNIT_EXPECT_FALSE(test, diagnostic_ok);
+ KUNIT_EXPECT_EQ(test, compressed, 0UL);
+ KUNIT_EXPECT_EQ(test, diagnostic_maps, 0);
+ KUNIT_EXPECT_TRUE(test, present);
+ KUNIT_EXPECT_EQ(test, busy, -EAGAIN);
+ KUNIT_EXPECT_NULL(test, resident.slot);
+}
+
+static void memcomp_test_public_refresh_retry(struct kunit *test)
+{
+ struct mc16_ct *f = mc16_ct_begin(test);
+ struct mc16_ct_snapshot snapshot;
+ bool equal;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_ct_store(f));
+ KUNIT_ASSERT_NE(test, f->token, 0UL);
+ KUNIT_ASSERT_TRUE(test, memcomp_load_folio_pin(f->token, &f->guard));
+ mc16_ct_publish(f);
+ KUNIT_ASSERT_TRUE(test, memcomp_entry_set_refresh_pending(f->token));
+ ret = mc16_ct_load(f, &equal);
+ KUNIT_EXPECT_EQ(test, ret, -EAGAIN);
+ KUNIT_EXPECT_FALSE(test, equal);
+ KUNIT_EXPECT_NOT_NULL(test, f->guard.slot);
+ KUNIT_ASSERT_TRUE(test, mc16_ct_snapshot(&f->guard, &snapshot));
+ KUNIT_EXPECT_TRUE(test, snapshot.flags & MC16_F_REFRESH_PENDING);
+ KUNIT_EXPECT_EQ(test, snapshot.refs, 2U);
+ memcomp_entry_clear_refresh_pending(f->token);
+ ret = mc16_ct_load(f, &equal);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ KUNIT_EXPECT_TRUE(test, equal);
+}
+
+static void memcomp_test_public_cache_charge_failure(struct kunit *test)
+{
+ struct mc16_ct *f = mc16_ct_begin(test);
+ long charged;
+ bool stored;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ charged = atomic_long_read(&f->base->model.charged);
+ f->base->model.fail_at = atomic_read(&f->base->model.charge_calls) + 1;
+ stored = mc16_ct_store(f);
+ f->base->model.fail_at = 0;
+ KUNIT_EXPECT_FALSE(test, stored);
+ KUNIT_EXPECT_FALSE(test, f->cache_owner);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->base->model.charged), charged);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+}
+
+static void memcomp_test_public_invalid_refs(struct kunit *test)
+{
+ struct mc16_ct *f = mc16_ct_begin(test);
+ unsigned long compressed = ULONG_MAX;
+ int mapcount = -1;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_get_ref(0, &f->guard));
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_get_ref(U32_MAX, &f->guard));
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_present(U32_MAX));
+ memcomp_invalidate_nr(U32_MAX, 2);
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_stat(U32_MAX, &compressed, &mapcount));
+ KUNIT_EXPECT_EQ(test, compressed, 0UL);
+ KUNIT_EXPECT_EQ(test, mapcount, 0);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+}
+
+#ifdef MC16_FAULT_MM_TEST_ENABLED
+struct mc16_ct_extra {
+ u32 tokens[MC16_FT_PAGES];
+ unsigned int nr;
+};
+
+static void mc16_ct_extra_cleanup(void *data)
+{
+ struct mc16_ct_extra *extra = data;
+ unsigned int i;
+
+ for (i = 0; i < extra->nr; i++)
+ memcomp_invalidate(extra->tokens[i]);
+}
+
+static void mc16_ct_madv_free(struct kunit *test, bool partial, bool shared)
+{
+ struct mc16_ft *f = mc16_ft_begin(test, MC16_FT_PAGES, false, true);
+ struct mc16_ct_extra *extra;
+ unsigned short owner_id[MC16_FT_PAGES];
+ unsigned int i, first = partial ? 1 : 0;
+ unsigned int last = partial ? 3 : MC16_FT_PAGES;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ extra = kunit_kzalloc(test, sizeof(*extra), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, extra);
+ ret = kunit_add_action_or_reset(test, mc16_ct_extra_cleanup, extra);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ for (i = 0; i < f->nr; i++) {
+ pte_t pte = mc16_ft_orig(f, i);
+ struct memcomp_ref guard;
+
+ if (i == 2)
+ pte = pte_swp_mksoft_dirty(pte);
+ if (i == 3)
+ pte = pte_swp_mkuffd_wp(pte);
+ mmap_read_lock(f->mm);
+ ret = mc16_ft_change(f, i, pte);
+ mmap_read_unlock(f->mm);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ if (shared) {
+ /* Same extra-PTE-owner model as the old shared oracle. */
+ KUNIT_ASSERT_TRUE(test, memcomp_entry_get(f->tokens[i]));
+ extra->tokens[extra->nr++] = f->tokens[i];
+ }
+ /* Opaque lifetime data: this case never calls stock refault. */
+ KUNIT_ASSERT_TRUE(test, mc16_get(f->tokens[i], &guard));
+ mutex_lock(memcomp_lock_of(guard.token));
+ *mc16_shadow_ptr(&guard) = (unsigned long)xa_mk_value(i + 1);
+ owner_id[i] = mc16_owner_id(&guard);
+ mutex_unlock(memcomp_lock_of(guard.token));
+ mc16_ref_put(&guard);
+ }
+ ret = do_madvise(f->mm, f->addr + first * PAGE_SIZE,
+ (last - first) * PAGE_SIZE, MADV_FREE);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ mc16_ft_counts(f, partial ? 2 : 0, 0);
+ for (i = 0; i < f->nr; i++) {
+ bool discarded = i >= first && i < last;
+ struct memcomp_ref guard;
+ unsigned int refs = 0;
+ unsigned short id = 0;
+ u64 shadow = 0;
+ bool found;
+
+ KUNIT_EXPECT_EQ(test, pte_none(mc16_ft_read(f, i)), discarded);
+ KUNIT_EXPECT_EQ(test, memcomp_entry_present(f->tokens[i]),
+ shared || !discarded);
+ /* Short observations must not keep the final discarded owner alive. */
+ found = mc16_get(f->tokens[i], &guard);
+ if (found) {
+ mutex_lock(memcomp_lock_of(guard.token));
+ shadow = *mc16_shadow_ptr(&guard);
+ id = mc16_owner_id(&guard);
+ refs = refcount_read(&guard.slot->refs);
+ mutex_unlock(memcomp_lock_of(guard.token));
+ mc16_ref_put(&guard);
+ }
+ KUNIT_EXPECT_EQ(test, found, shared || !discarded);
+ if (found) {
+ KUNIT_EXPECT_EQ(test, shadow,
+ (u64)(unsigned long)xa_mk_value(i + 1));
+ KUNIT_EXPECT_EQ(test, id, owner_id[i]);
+ /* Include the just-released observer in the sampled count. */
+ KUNIT_EXPECT_EQ(test, refs, 1U + shared + !discarded);
+ }
+ }
+ /* Both extra owners and surviving PTEs are covered by fixture actions. */
+}
+
+static void memcomp_test_public_madv_free(struct kunit *test)
+{
+ mc16_ct_madv_free(test, false, false);
+}
+
+static void memcomp_test_public_madv_free_partial(struct kunit *test)
+{
+ mc16_ct_madv_free(test, true, false);
+}
+
+static void memcomp_test_public_madv_free_shared(struct kunit *test)
+{
+ mc16_ct_madv_free(test, false, true);
+}
+
+static void memcomp_test_public_fault(struct kunit *test)
+{
+ struct mc16_ft *f = mc16_ft_begin(test, 1, false, true);
+ struct vm_area_struct *vma;
+ pte_t *returned_pte = NULL;
+ vm_fault_t ret = VM_FAULT_SIGBUS;
+ bool locked = true;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mmap_read_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (vma) {
+ struct vm_fault vmf = {
+ .vma = vma,
+ .address = f->addr,
+ .real_address = f->addr,
+ .flags = FAULT_FLAG_WRITE | FAULT_FLAG_ORIG_PTE_VALID,
+ .pmd = mm_find_pmd(f->mm, f->addr),
+ .orig_pte = mc16_ft_orig(f, 0),
+ .gfp_mask = GFP_HIGHUSER_MOVABLE,
+ };
+
+ ret = memcomp_load(&vmf, &locked);
+ returned_pte = vmf.pte;
+ }
+ mmap_read_unlock(f->mm);
+ KUNIT_ASSERT_NOT_NULL(test, vma);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)(VM_FAULT_NOPAGE | VM_FAULT_MAJOR));
+ KUNIT_EXPECT_FALSE(test, locked);
+ KUNIT_EXPECT_PTR_EQ(test, returned_pte, NULL);
+ mc16_ft_counts(f, 0, 1);
+ mc16_ft_present(f, 0, 0, 1);
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_present(f->tokens[0]));
+}
+#endif /* MC16_FAULT_MM_TEST_ENABLED */
+
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_PUBLIC_API_TEST_H */
diff --git a/mm/memcomp/tests/public_driver_test.h b/mm/memcomp/tests/public_driver_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..136a1f00bb5c8e1450231f1eb541e65adf5443c7
--- /dev/null
+++ b/mm/memcomp/tests/public_driver_test.h
@@ -0,0 +1,358 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_PUBLIC_DRIVER_TEST_H
+#define _MM_MEMCOMP_TESTS_PUBLIC_DRIVER_TEST_H
+
+#include <kunit/test.h>
+#include <linux/highmem.h>
+
+/*
+ * Include after public_api_test.h and folio_test.h in suite.h, before the
+ * main case array. Requires the public slot-table dispatch in memcomp.c.
+ * No public wrapper, walker, restore actor or IO function is mocked here.
+ *
+ * Resident cases reuse the real MM/folio/rmap fixture. Swapoff cases reuse the
+ * unlocked consumer/public-API fixture and swapoff_test.h's encoded-backing setup.
+ * Only metadata objcg and backing reference tallies use existing models.
+ * The public driver has no device-read seam: type rejection and corrupt-state
+ * errors intentionally stop before IO. These are driver binding tests, not
+ * successful device restore tests or another copy of the swapoff model matrix.
+ * Every actor drops its MM/PTL/folio/T/W locks and mapping before assertions;
+ * fixture cleanup never inherits a lock from this header's test task.
+ */
+#ifdef MC16_FOLIO_TEST_ENABLED
+#define MC16_DRIVER_RESIDENT_TEST_ENABLED
+
+struct mc16_dct_resident {
+ struct mc16_ct_snapshot before;
+ struct mc16_ct_snapshot after;
+ pte_t orig;
+ pte_t result;
+ int query;
+ int lookup;
+ int restore;
+ bool consumed;
+ bool bytes_equal;
+ bool same_page;
+ bool metadata_restored;
+};
+
+static bool mc16_dct_public_reserve(struct mc16_flt *f)
+{
+ bool reserved;
+
+ folio_lock(f->folio);
+ reserved = memcomp_reserve(f->folio);
+ folio_unlock(f->folio);
+ return reserved;
+}
+
+/* The independent observer guard protects metadata while restore consumes its. */
+static bool mc16_dct_flags(struct memcomp_ref *observer, u16 *saved, bool restore)
+{
+ struct mc16_view view;
+ bool valid;
+
+ mutex_lock(memcomp_lock_of(observer->token));
+ valid = mc16_view_locked(observer, &view);
+ if (valid) {
+ if (restore) {
+ *view.flags = *saved;
+ } else {
+ *saved = *view.flags;
+ *view.flags |= BIT(11); /* Reserved bit: effective flags invalid. */
+ }
+ }
+ mutex_unlock(memcomp_lock_of(observer->token));
+ return valid;
+}
+
+static int mc16_dct_restore(struct mc16_flt *f, bool corrupt,
+ struct mc16_dct_resident *r)
+{
+ struct memcomp_ref *lookup = &f->base->guard;
+ struct vm_area_struct *vma;
+ spinlock_t *ptl; /* Copy orig/result PTEs only within their own PTL scope. */
+ unsigned long token = f->guards[0].token;
+ pte_t *ptep;
+ u16 saved = 0;
+ void *bytes;
+ bool injected = false;
+ int ret = -ENOENT;
+
+ memset(r, 0, sizeof(*r));
+ mmap_read_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (!vma)
+ goto out;
+ {
+ struct vm_fault vmf = {
+ .vma = vma, .address = f->addr, .real_address = f->addr,
+ .flags = FAULT_FLAG_ORIG_PTE_VALID | FAULT_FLAG_RETRY_NOWAIT,
+ .pmd = mm_find_pmd(f->mm, f->addr),
+ .gfp_mask = GFP_HIGHUSER_MOVABLE,
+ };
+ if (!vmf.pmd)
+ goto out;
+ ptep = pte_offset_map_lock(f->mm, vmf.pmd, f->addr, &ptl);
+ if (!ptep)
+ goto out;
+ r->orig = ptep_get(ptep);
+ vmf.orig_pte = r->orig;
+ pte_unmap_unlock(ptep, ptl);
+
+ r->query = memcomp_entry_resident(token);
+ r->lookup = memcomp_entry_resident_get(token, lookup);
+ if (r->lookup != 1)
+ goto out;
+ if (!mc16_ct_snapshot(&f->guards[0], &r->before))
+ goto out;
+ if (corrupt) {
+ injected = mc16_dct_flags(&f->guards[0], &saved, false);
+ if (!injected)
+ goto out;
+ }
+ /* Real wrapper consumes the public lookup, on success and SIGBUS alike. */
+ r->restore = memcomp_restore_resident(&vmf, lookup);
+ r->consumed = !lookup->slot && !lookup->cluster && !lookup->token;
+ if (injected) {
+ r->metadata_restored = mc16_dct_flags(&f->guards[0], &saved, true);
+ injected = false;
+ } else {
+ r->metadata_restored = true;
+ }
+ if (!mc16_ct_snapshot(&f->guards[0], &r->after))
+ goto out;
+ ptep = pte_offset_map_lock(f->mm, vmf.pmd, f->addr, &ptl);
+ if (!ptep)
+ goto out;
+ r->result = ptep_get(ptep);
+ if (pte_present(r->result) && !pte_special(r->result)) {
+ r->same_page = pte_page(r->result) == folio_page(f->folio, 0);
+ if (r->same_page) {
+ bytes = kmap_local_page(pte_page(r->result));
+ r->bytes_equal = !memchr_inv(bytes, 0x6b, PAGE_SIZE);
+ kunmap_local(bytes);
+ }
+ }
+ pte_unmap_unlock(ptep, ptl);
+ ret = 0;
+ }
+out:
+ if (injected)
+ mc16_dct_flags(&f->guards[0], &saved, true);
+ /* Also clean preparation failures before the folio fixture is torn down. */
+ if (lookup->slot)
+ memcomp_entry_put_ref(lookup);
+ /* RETRY_NOWAIT keeps mmap_read_lock on every folio-lock retry outcome. */
+ mmap_read_unlock(f->mm);
+ return ret;
+}
+
+static void memcomp_test_public_resident_restore(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 0);
+ struct mc16_dct_resident r;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_dct_public_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, 0), 0);
+ KUNIT_ASSERT_EQ(test, mc16_dct_restore(f, false, &r), 0);
+ KUNIT_EXPECT_EQ(test, r.query, 1);
+ KUNIT_EXPECT_EQ(test, r.lookup, 1);
+ KUNIT_EXPECT_EQ(test, r.restore, 0);
+ KUNIT_EXPECT_TRUE(test, r.consumed);
+ KUNIT_EXPECT_TRUE(test, r.metadata_restored);
+ KUNIT_EXPECT_TRUE(test, r.same_page);
+ KUNIT_EXPECT_TRUE(test, r.bytes_equal);
+ KUNIT_EXPECT_EQ(test, r.before.refs, 4U); /* OWNER, observer, PTE, lookup. */
+ KUNIT_EXPECT_EQ(test, r.after.refs, 2U); /* OWNER and observer survive. */
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(r.after.flags), MC16_PENDING);
+ KUNIT_EXPECT_TRUE(test, r.after.flags & MC16_F_RESIDENT);
+ KUNIT_EXPECT_EQ(test, mc16_flt_rss(f, MM_SWAPENTS), f->swapents);
+ KUNIT_EXPECT_EQ(test, mc16_flt_rss(f, MM_ANONPAGES), f->anonpages + 1);
+}
+
+static void memcomp_test_public_restore_error(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 0);
+ struct mc16_dct_resident r;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_dct_public_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, 0), 0);
+ KUNIT_ASSERT_EQ(test, mc16_dct_restore(f, true, &r), 0);
+ KUNIT_EXPECT_EQ(test, r.query, 1);
+ KUNIT_EXPECT_EQ(test, r.lookup, 1);
+ KUNIT_EXPECT_EQ(test, r.restore, -EFAULT);
+ KUNIT_EXPECT_TRUE(test, r.consumed);
+ KUNIT_EXPECT_TRUE(test, r.metadata_restored);
+ KUNIT_EXPECT_TRUE(test, pte_same(r.orig, r.result));
+ KUNIT_EXPECT_EQ(test, r.before.refs, 4U);
+ KUNIT_EXPECT_EQ(test, r.after.refs, 3U); /* Failed restore retains PTE. */
+ KUNIT_EXPECT_EQ(test, r.after.flags, r.before.flags);
+ KUNIT_EXPECT_EQ(test, mc16_flt_rss(f, MM_SWAPENTS), f->swapents + 1);
+ KUNIT_EXPECT_EQ(test, mc16_flt_rss(f, MM_ANONPAGES), f->anonpages);
+ KUNIT_EXPECT_EQ(test, memcomp_entry_resident(f->guards[0].token), 1);
+}
+#endif /* MC16_FOLIO_TEST_ENABLED */
+
+#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP)
+#define MC16_DRIVER_SWAPOFF_TEST_ENABLED
+
+/* Same encoded-backing setup as swapoff_test.h, on an unlocked base fixture. */
+static int mc16_dct_swapped(struct mc16_ct *f)
+{
+ struct memcomp_ref *owner = &f->base->owners[0];
+ struct mc16_view view;
+ u64 word;
+ u16 length;
+ int ret;
+
+ ret = mc16_consumer_test_reserve(f->base, 0);
+ if (ret)
+ return ret;
+ if (!mc16_swap_encode(7, 0x567, 0, PAGE_SIZE, false, &word, &length))
+ return -EINVAL;
+ mutex_lock(memcomp_lock_of(owner->token));
+ if (!mc16_view_locked(owner, &view) || !view.member) {
+ ret = -EINVAL;
+ goto out;
+ }
+ mc16_account_state(owner, false);
+ folio_put(view.member->owner_folio);
+ *view.word = word;
+ *view.length = length;
+ *view.flags = MC16_SWAPPED | MC16_F_OWNER | MC16_F_REFERENCED;
+ memcomp_debug_dec(&memcomp_pending_pages);
+ memcomp_debug_long_inc(&memcomp_swapped_entries);
+ memcomp_compressed_bytes_add(PAGE_SIZE);
+ mc16_account_state(owner, true);
+ ret = 0;
+out:
+ mutex_unlock(memcomp_lock_of(owner->token));
+ if (!ret) {
+ f->token = owner->token;
+ if (!memcomp_entry_get_ref(f->token, &f->guard))
+ ret = -ENOENT;
+ }
+ return ret;
+}
+
+struct mc16_dct_swapoff {
+ struct mc16_ct_snapshot before;
+ struct mc16_ct_snapshot after;
+ int result;
+ int dups;
+ int puts;
+ unsigned int flags_before;
+ unsigned int flags_after;
+ bool metadata_restored;
+ bool unlocked;
+};
+
+static int mc16_dct_swapoff(struct mc16_ct *f, bool corrupt,
+ struct mc16_dct_swapoff *r)
+{
+ struct mc16_view view;
+ u16 saved = 0;
+ bool injected = false;
+ int dups, puts;
+
+ memset(r, 0, sizeof(*r));
+ if (!mc16_ct_snapshot(&f->guard, &r->before))
+ return -EINVAL;
+ /* The public driver acquires W itself; never call it under an outer W. */
+ if (corrupt) {
+ mutex_lock(memcomp_lock_of(f->token));
+ if (mc16_view_locked(&f->guard, &view)) {
+ saved = *view.flags;
+ *view.flags |= BIT(11);
+ injected = true;
+ }
+ mutex_unlock(memcomp_lock_of(f->token));
+ if (!injected)
+ return -EINVAL;
+ }
+ dups = atomic_read(&memcomp_kunit_slot_dups);
+ puts = atomic_read(&memcomp_kunit_slot_puts);
+ r->flags_before = current->flags & (PF_MEMALLOC_NOFS | PF_MEMALLOC);
+ /* Valid backing type 7 must be rejected by the real restore binding. */
+ r->result = memcomp_swapoff(8);
+ r->flags_after = current->flags & (PF_MEMALLOC_NOFS | PF_MEMALLOC);
+ r->dups = atomic_read(&memcomp_kunit_slot_dups) - dups;
+ r->puts = atomic_read(&memcomp_kunit_slot_puts) - puts;
+ if (injected) {
+ mutex_lock(memcomp_lock_of(f->token));
+ if (mc16_view_locked(&f->guard, &view)) {
+ *view.flags = saved;
+ r->metadata_restored = true;
+ }
+ mutex_unlock(memcomp_lock_of(f->token));
+ } else {
+ r->metadata_restored = true;
+ }
+ r->unlocked = mutex_trylock(&memcomp_writeback_lock);
+ if (r->unlocked)
+ mutex_unlock(&memcomp_writeback_lock);
+ return mc16_ct_snapshot(&f->guard, &r->after) ? 0 : -EINVAL;
+}
+
+static void mc16_dct_driver(struct kunit *test, bool corrupt)
+{
+ struct mc16_ct *f = mc16_ct_begin(test);
+ struct mc16_dct_swapoff r;
+ s64 stored, compressed;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_dct_swapped(f), 0);
+ stored = atomic64_read(&memcomp_stored_bytes);
+ compressed = atomic64_read(&memcomp_compressed_bytes);
+ KUNIT_ASSERT_EQ(test, mc16_dct_swapoff(f, corrupt, &r), 0);
+ KUNIT_EXPECT_EQ(test, r.result, corrupt ? -EIO : 0);
+ KUNIT_EXPECT_TRUE(test, r.metadata_restored);
+ KUNIT_EXPECT_TRUE(test, r.unlocked);
+ KUNIT_EXPECT_EQ(test, r.flags_after, r.flags_before);
+ KUNIT_EXPECT_EQ(test, r.dups, corrupt ? 0 : 1);
+ KUNIT_EXPECT_EQ(test, r.puts, corrupt ? 0 : 1);
+ KUNIT_EXPECT_EQ(test, r.before.refs, 2U);
+ KUNIT_EXPECT_EQ(test, r.after.refs, r.before.refs);
+ KUNIT_EXPECT_EQ(test, r.after.word, r.before.word);
+ KUNIT_EXPECT_EQ(test, r.after.length, r.before.length);
+ KUNIT_EXPECT_EQ(test, r.after.flags, r.before.flags);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(r.after.flags), MC16_SWAPPED);
+ KUNIT_EXPECT_NOT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_stored_bytes), stored);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_compressed_bytes), compressed);
+ /* Explicitly prove public guard put leaves only the fixture's owner. */
+ memcomp_entry_put_ref(&f->guard);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->base->owners[0].slot->refs), 1U);
+}
+
+static void memcomp_test_public_swapoff_type_miss(struct kunit *test)
+{
+ mc16_dct_driver(test, false);
+}
+
+static void memcomp_test_public_swapoff_error(struct kunit *test)
+{
+ mc16_dct_driver(test, true);
+}
+#endif /* CONFIG_MEMCG && CONFIG_SWAP */
+
+/*
+ * Register these additions in the existing suite:
+ *
+ * #ifdef MC16_DRIVER_RESIDENT_TEST_ENABLED
+ * KUNIT_CASE(memcomp_test_public_resident_restore),
+ * KUNIT_CASE(memcomp_test_public_restore_error),
+ * #endif
+ * #ifdef MC16_DRIVER_SWAPOFF_TEST_ENABLED
+ * KUNIT_CASE(memcomp_test_public_swapoff_type_miss),
+ * KUNIT_CASE(memcomp_test_public_swapoff_error),
+ * #endif
+ */
+
+#endif /* _MM_MEMCOMP_TESTS_PUBLIC_DRIVER_TEST_H */
diff --git a/mm/memcomp/tests/query_test.h b/mm/memcomp/tests/query_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..bd5faa4f633a6ea4c5b8937ad7ad8c98ca124531
--- /dev/null
+++ b/mm/memcomp/tests/query_test.h
@@ -0,0 +1,480 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_QUERY_TEST_H
+#define _MM_MEMCOMP_TESTS_QUERY_TEST_H
+
+/*
+ * Include after cache_reserve/refs/payload/owner test fixtures and query.h.
+ * These call the slot-storage adapters directly; public dispatch stays separate.
+ * All cases use real slots/guards and the existing modeled objcg fixture.
+ */
+#ifdef CONFIG_MEMCG
+struct mc16_test_query_observation {
+ unsigned int allocations;
+ int gets;
+ int puts;
+ int charges;
+ int uncharges;
+ int folio_refs;
+ long charged;
+ s64 stored;
+ s64 compressed;
+};
+
+static void
+mc16_test_query_observe(struct mc16_test_storage *s,
+ struct mc16_test_query_observation *out)
+{
+ out->allocations = s->probe.allocations;
+ out->gets = atomic_read(&s->owner->model.get_calls);
+ out->puts = atomic_read(&s->owner->model.put_calls);
+ out->charges = atomic_read(&s->owner->model.charge_calls);
+ out->uncharges = atomic_read(&s->owner->model.uncharge_calls);
+ out->folio_refs = folio_ref_count(s->owner->folio);
+ out->charged = atomic_long_read(&s->owner->model.charged);
+ out->stored = atomic64_read(&memcomp_stored_bytes);
+ out->compressed = atomic64_read(&memcomp_compressed_bytes);
+}
+
+static void
+mc16_test_query_unchanged(struct kunit *test, struct mc16_test_storage *s,
+ const struct mc16_test_query_observation *before)
+{
+ struct mc16_test_query_observation after;
+
+ mc16_test_query_observe(s, &after);
+ KUNIT_EXPECT_EQ(test, after.allocations, before->allocations);
+ KUNIT_EXPECT_EQ(test, after.gets, before->gets);
+ KUNIT_EXPECT_EQ(test, after.puts, before->puts);
+ KUNIT_EXPECT_EQ(test, after.charges, before->charges);
+ KUNIT_EXPECT_EQ(test, after.uncharges, before->uncharges);
+ KUNIT_EXPECT_EQ(test, after.folio_refs, before->folio_refs);
+ KUNIT_EXPECT_EQ(test, after.charged, before->charged);
+ KUNIT_EXPECT_EQ(test, after.stored, before->stored);
+ KUNIT_EXPECT_EQ(test, after.compressed, before->compressed);
+}
+
+static void mc16_test_query_absent(struct kunit *test, unsigned long token)
+{
+ struct memcomp_ref ref = { };
+ unsigned long compressed = ULONG_MAX, base = ULONG_MAX, index = ULONG_MAX;
+ unsigned int nr;
+ int count = -1, resident;
+ bool stat;
+
+ KUNIT_EXPECT_FALSE(test, mc16_entry_present(token));
+ stat = mc16_entry_stat(token, &compressed, &count);
+ KUNIT_EXPECT_FALSE(test, stat);
+ KUNIT_EXPECT_EQ(test, compressed, 0UL);
+ KUNIT_EXPECT_EQ(test, count, 0);
+ nr = mc16_entry_geometry(token, &base, &index);
+ KUNIT_EXPECT_EQ(test, nr, 0U);
+ KUNIT_EXPECT_EQ(test, base, token);
+ KUNIT_EXPECT_EQ(test, index, 0UL);
+ resident = mc16_entry_resident_get(token, &ref);
+ KUNIT_EXPECT_EQ(test, resident, 0);
+ KUNIT_EXPECT_PTR_EQ(test, ref.slot, NULL);
+ KUNIT_EXPECT_PTR_EQ(test, ref.cluster, NULL);
+ KUNIT_EXPECT_EQ(test, ref.token, 0U);
+ /* Return any unexpected successful guard, even on a failing expectation. */
+ mc16_entry_resident_put(&ref);
+}
+
+/*
+ * Presence must work with T already held and under PTL/RCU. Check storage
+ * side effects and reference balance; the storage probe does not count slot
+ * ref RMW events, so unchanged refs alone do not prove absence of a get/put.
+ */
+static void memcomp_test_query_present(struct kunit *test)
+{
+ struct mc16_test_owner *f = mc16_test_owner_begin(test, 1);
+ struct mc16_test_query_observation before;
+ struct mc16_test_storage *s;
+ struct memcomp_ref *ref;
+ spinlock_t ptl; /* Models a PTL-only query caller. */
+ unsigned long irqflags;
+ unsigned int refs, i;
+ bool present = true;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ ref = &s->refs[0];
+ refs = refcount_read(&ref->slot->refs);
+ mc16_test_query_observe(s, &before);
+ spin_lock_init(&ptl);
+ mutex_lock(memcomp_lock_of(ref->token));
+ present = mc16_entry_present(ref->token);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ rcu_read_lock();
+ spin_lock_irqsave(&ptl, irqflags);
+ for (i = 0; i < 8; i++)
+ present &= mc16_entry_present(ref->token);
+ spin_unlock_irqrestore(&ptl, irqflags);
+ rcu_read_unlock();
+ KUNIT_EXPECT_TRUE(test, present);
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), refs);
+ mc16_test_query_unchanged(test, s, &before);
+}
+
+static void memcomp_test_query_absent(struct kunit *test)
+{
+ struct mc16_test_cache_reserve *f = mc16_test_cache_begin(test);
+ struct mc16_test_query_observation before;
+ struct mc16_test_storage *s;
+ struct memcomp_ref *ref;
+ unsigned long token;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_prepare(f, 0, 0, U64_MAX), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_cache_reserve(f, 1), 0);
+ s = f->storage;
+ ref = &s->refs[0];
+ token = ref->token;
+ KUNIT_ASSERT_EQ(test, refcount_read(&ref->slot->refs), 0U);
+ mc16_test_query_observe(s, &before);
+ /* Private reservation is published structurally but has no owner yet. */
+ mc16_test_query_absent(test, token);
+ mc16_test_query_absent(test, token + 1); /* beyond release-published next */
+ mc16_test_query_absent(test, token + MC16_SEGMENT_SLOTS);
+ mc16_test_query_absent(test, 1); /* cluster id zero */
+ mc16_test_query_absent(test, U32_MAX);
+#if BITS_PER_LONG > 32
+ mc16_test_query_absent(test, (1UL << 32) | token);
+#endif
+ mc16_test_query_absent(test, ULONG_MAX);
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), 0U);
+ mc16_test_query_unchanged(test, s, &before);
+ /* Same numeric token must remain absent before and after RCU retirement. */
+ mc16_release_resources(ref);
+ mc16_retire_slot(ref);
+ *ref = (struct memcomp_ref) { };
+ mc16_test_query_absent(test, token);
+ rcu_barrier();
+ mc16_test_query_absent(test, token);
+}
+
+static void memcomp_test_query_zero(struct kunit *test)
+{
+ struct mc16_test_refs *f = mc16_test_refs_begin(test);
+ struct mc16_test_query_observation before;
+ struct mc16_zero_owners owners, after;
+ struct memcomp_ref ref = { .token = U32_MAX };
+ unsigned long bytes = ULONG_MAX, base = ULONG_MAX, index = ULONG_MAX;
+ int count = -1, resident;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_test_query_observe(f->storage, &before);
+ mc16_zero_owners_read(&owners);
+ KUNIT_EXPECT_TRUE(test, mc16_entry_present(0));
+ KUNIT_EXPECT_TRUE(test, mc16_entry_stat(0, &bytes, &count));
+ KUNIT_EXPECT_EQ(test, bytes, 0UL);
+ KUNIT_EXPECT_EQ(test, count, 1);
+ KUNIT_EXPECT_TRUE(test, mc16_entry_stat(0, NULL, NULL));
+ KUNIT_EXPECT_EQ(test, mc16_entry_geometry(0, &base, &index), 1U);
+ KUNIT_EXPECT_EQ(test, base, 0UL);
+ KUNIT_EXPECT_EQ(test, index, 0UL);
+ KUNIT_EXPECT_EQ(test, mc16_entry_nr_pages(0, NULL), 1U);
+ resident = mc16_entry_resident_get(0, &ref);
+ KUNIT_EXPECT_EQ(test, resident, 0);
+ KUNIT_EXPECT_PTR_EQ(test, ref.slot, NULL);
+ KUNIT_EXPECT_PTR_EQ(test, ref.cluster, NULL);
+ KUNIT_EXPECT_EQ(test, ref.token, 0U);
+ mc16_entry_resident_put(&ref);
+ mc16_entry_resident_put(&ref);
+ mc16_zero_owners_read(&after);
+ KUNIT_EXPECT_EQ(test, after.pte_pages, owners.pte_pages);
+ KUNIT_EXPECT_EQ(test, after.cache_pages, owners.cache_pages);
+ mc16_test_query_unchanged(test, f->storage, &before);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+}
+
+static void memcomp_test_query_stat_ptl(struct kunit *test)
+{
+ struct mc16_test_owner *f = mc16_test_owner_begin(test, 1);
+ struct memcomp_ref *ref;
+ spinlock_t ptl; /* Models a PTL-only query caller. */
+ unsigned long irqflags;
+ unsigned int refs;
+ int count = -1, held_count = -1;
+ bool stat, optional, held_stat;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ ref = &f->refs->storage->refs[0];
+ /* OWNER plus two persistent PTEs; query's own lookup must disappear. */
+ KUNIT_ASSERT_TRUE(test, mc16_get(ref->token, &f->survivor[0]));
+ KUNIT_ASSERT_TRUE(test, mc16_get(ref->token, &f->survivor[1]));
+ refs = refcount_read(&ref->slot->refs);
+ spin_lock_init(&ptl);
+ /* Separate contexts: never manufacture a forbidden T -> PTL edge. */
+ mutex_lock(memcomp_lock_of(ref->token));
+ held_stat = mc16_entry_stat(ref->token, NULL, &held_count);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ rcu_read_lock();
+ spin_lock_irqsave(&ptl, irqflags);
+ stat = mc16_entry_stat(ref->token, NULL, &count);
+ optional = mc16_entry_stat(ref->token, NULL, NULL);
+ spin_unlock_irqrestore(&ptl, irqflags);
+ rcu_read_unlock();
+ KUNIT_EXPECT_TRUE(test, held_stat);
+ KUNIT_EXPECT_EQ(test, held_count, 2);
+ KUNIT_EXPECT_TRUE(test, stat);
+ KUNIT_EXPECT_TRUE(test, optional);
+ KUNIT_EXPECT_EQ(test, count, 2);
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), refs);
+}
+
+static void memcomp_test_query_stat_busy_invalid(struct kunit *test)
+{
+ struct mc16_test_owner *f = mc16_test_owner_begin(test, 1);
+ struct memcomp_ref *ref;
+ struct mc16_view view;
+ unsigned long bytes = ULONG_MAX;
+ unsigned int refs, i;
+ u16 flags, length;
+ int count = -1;
+ bool stat;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_payload(f->refs, 0, MC16_STORED), 0);
+ ref = &f->refs->storage->refs[0];
+ refs = refcount_read(&ref->slot->refs);
+ mutex_lock(memcomp_lock_of(ref->token));
+ stat = mc16_entry_stat(ref->token, &bytes, &count);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ KUNIT_EXPECT_FALSE(test, stat);
+ KUNIT_EXPECT_EQ(test, bytes, 0UL);
+ KUNIT_EXPECT_EQ(test, count, 0);
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), refs);
+ KUNIT_ASSERT_TRUE(test, mc16_entry_stat(ref->token, &bytes, &count));
+ KUNIT_EXPECT_EQ(test, bytes, (unsigned long)PAGE_SIZE);
+ KUNIT_EXPECT_EQ(test, count, 1);
+
+ /* Restore every corrupted field before any fatal assertion or teardown. */
+ for (i = 0; i < 3; i++) {
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ flags = *view.flags;
+ length = *view.length;
+ if (i == 0)
+ *view.flags |= BIT(15);
+ else if (i == 1)
+ *view.length = U16_MAX;
+ else
+ *view.flags |= MC16_F_RESIDENT;
+ mutex_unlock(memcomp_lock_of(ref->token));
+ bytes = ULONG_MAX;
+ count = -1;
+ stat = mc16_entry_stat(ref->token, &bytes, &count);
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ *view.flags = flags;
+ *view.length = length;
+ mutex_unlock(memcomp_lock_of(ref->token));
+ KUNIT_EXPECT_FALSE(test, stat);
+ KUNIT_EXPECT_EQ(test, bytes, 0UL);
+ KUNIT_EXPECT_EQ(test, count, 0);
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), refs);
+ }
+}
+
+static void memcomp_test_query_stat_compressed(struct kunit *test)
+{
+ struct mc16_test_payload *f = mc16_test_payload_begin(test);
+ struct memcomp_ref *ref;
+ unsigned long handle = 0, bytes;
+ unsigned int compressed = 0, i;
+ u8 pool = 0, *src;
+ int count, ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ ref = &f->refs->storage->refs[0];
+ src = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ for (i = 0; i < PAGE_SIZE; i++)
+ src[i] = i % 53;
+ ret = memcomp_prepare_payload(src, PAGE_SIZE, &compressed, &handle, &pool,
+ MEMCOMP_PAYLOAD_MUST_SUCCEED, false);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ f->prepared = (struct mc16_payload) { .word = handle, .bytes = compressed };
+ KUNIT_ASSERT_EQ(test, pool, (u8)0);
+ KUNIT_ASSERT_GT(test, compressed, 0U);
+ KUNIT_ASSERT_LT(test, compressed, (unsigned int)PAGE_SIZE);
+ KUNIT_ASSERT_EQ(test, mc16_payload_charge(ref, &f->prepared), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_payload_commit(f, f->refs->storage->owner->folio), 0);
+ mc16_payload_finish(ref, &f->tail);
+ /* Expected byte count comes from the real compressor's output. */
+ KUNIT_ASSERT_TRUE(test, mc16_entry_stat(ref->token, &bytes, &count));
+ KUNIT_EXPECT_EQ(test, bytes, (unsigned long)compressed);
+ KUNIT_EXPECT_EQ(test, count, 1);
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), 1U);
+ KUNIT_ASSERT_EQ(test, mc16_test_payload_prepare(f, 0, U64_MAX), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_payload_commit(f, f->refs->storage->owner->folio), 0);
+ mc16_payload_finish(ref, &f->tail);
+ bytes = ULONG_MAX;
+ KUNIT_ASSERT_TRUE(test, mc16_entry_stat(ref->token, &bytes, NULL));
+ KUNIT_EXPECT_EQ(test, bytes, 0UL);
+}
+
+static void memcomp_test_query_pending_owner_drop(struct kunit *test)
+{
+ struct mc16_test_owner *f = mc16_test_owner_begin(test, 1);
+ struct mc16_test_storage *s;
+ struct mc16_view view;
+ unsigned long bytes = ULONG_MAX;
+ u32 token;
+ u16 physical, effective;
+ int count = -1;
+ bool dropped;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ token = s->refs[0].token;
+ KUNIT_ASSERT_TRUE(test, mc16_get(token, &f->survivor[0]));
+ KUNIT_ASSERT_TRUE(test, mc16_get(token, &f->survivor[1]));
+ KUNIT_ASSERT_TRUE(test, mc16_get(token, &f->lookup));
+ dropped = mc16_owner_drop_folio_held(&f->lookup, s->owner->folio, 1, 0);
+ if (dropped)
+ s->refs[0] = (struct memcomp_ref) { };
+ KUNIT_ASSERT_TRUE(test, dropped);
+ mutex_lock(memcomp_lock_of(token));
+ physical = f->survivor[0].slot->flags;
+ mc16_view_locked(&f->survivor[0], &view);
+ effective = *view.flags;
+ mutex_unlock(memcomp_lock_of(token));
+ KUNIT_EXPECT_FALSE(test, physical & MC16_F_OWNER);
+ KUNIT_EXPECT_FALSE(test, effective & MC16_F_OWNER);
+ KUNIT_EXPECT_TRUE(test, effective & MC16_F_RESIDENT);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(physical), (unsigned int)MC16_PENDING);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->survivor[0].slot->refs), 2U);
+ KUNIT_ASSERT_TRUE(test, mc16_entry_stat(token, NULL, &count));
+ KUNIT_EXPECT_EQ(test, count, 2);
+ KUNIT_ASSERT_TRUE(test, mc16_entry_stat(token, &bytes, &count));
+ KUNIT_EXPECT_EQ(test, bytes, 0UL);
+ KUNIT_EXPECT_EQ(test, count, 2);
+}
+
+static void memcomp_test_query_geometry(struct kunit *test)
+{
+ struct mc16_test_owner *f = mc16_test_owner_begin(test, 2);
+ struct mc16_test_storage *s;
+ unsigned long base, index;
+ unsigned int i, nr, refs;
+ u32 first, token;
+ bool dropped;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ first = s->refs[0].token;
+ for (i = 0; i < 2; i++) {
+ token = s->refs[i].token;
+ refs = refcount_read(&s->refs[i].slot->refs);
+ mutex_lock(memcomp_lock_of(token));
+ nr = mc16_entry_geometry(token, &base, &index);
+ mutex_unlock(memcomp_lock_of(token));
+ KUNIT_EXPECT_EQ(test, nr, 2U);
+ KUNIT_EXPECT_EQ(test, base, (unsigned long)first);
+ KUNIT_EXPECT_EQ(test, index, (unsigned long)i);
+ KUNIT_EXPECT_EQ(test, mc16_entry_nr_pages(token, NULL), 2U);
+ KUNIT_EXPECT_EQ(test, refcount_read(&s->refs[i].slot->refs), refs);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_payload(f->refs, i, MC16_SAMEFILL), 0);
+ KUNIT_ASSERT_TRUE(test, mc16_get(token, &f->survivor[i]));
+ }
+ /* Normalize siblings first; drop the producer's head last. */
+ for (i = 2; i; ) {
+ i--;
+ token = f->survivor[i].token;
+ KUNIT_ASSERT_TRUE(test, mc16_get(token, &f->lookup));
+ dropped = mc16_owner_drop_folio_held(&f->lookup, s->owner->folio, 2, i);
+ if (dropped)
+ s->refs[i] = (struct memcomp_ref) { };
+ KUNIT_ASSERT_TRUE(test, dropped);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(f->survivor[i].slot->flags),
+ (unsigned int)MC16_SAMEFILL);
+ }
+ /* The record is gone; geometry must still work from the stable slots. */
+ for (i = 0; i < 2; i++) {
+ nr = mc16_entry_geometry(f->survivor[i].token, &base, &index);
+ KUNIT_EXPECT_EQ(test, nr, 2U);
+ KUNIT_EXPECT_EQ(test, base, (unsigned long)first);
+ KUNIT_EXPECT_EQ(test, index, (unsigned long)i);
+ }
+}
+
+static void memcomp_test_query_resident_busy(struct kunit *test)
+{
+ struct mc16_test_owner *f = mc16_test_owner_begin(test, 1);
+ struct mc16_test_storage *s;
+ struct memcomp_ref *ref;
+ struct mc16_view view;
+ unsigned int refs;
+ u16 flags;
+ int ret, folio_refs;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ ref = &s->refs[0];
+ refs = refcount_read(&ref->slot->refs);
+ folio_refs = folio_ref_count(s->owner->folio);
+ mutex_lock(memcomp_lock_of(ref->token));
+ ret = mc16_entry_resident_get(ref->token, &f->lookup);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ KUNIT_EXPECT_EQ(test, ret, -EAGAIN);
+ KUNIT_EXPECT_PTR_EQ(test, f->lookup.slot, NULL);
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), refs);
+ mc16_entry_resident_put(&f->lookup);
+
+ ret = mc16_entry_resident_get(ref->token, &f->lookup);
+ KUNIT_EXPECT_EQ(test, ret, 1);
+ KUNIT_EXPECT_PTR_EQ(test, f->lookup.slot, ref->slot);
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), refs + 1);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(s->owner->folio), folio_refs);
+ mc16_entry_resident_put(&f->lookup);
+ mc16_entry_resident_put(&f->lookup);
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), refs);
+
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ flags = *view.flags;
+ *view.flags |= BIT(15);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ ret = mc16_entry_resident_get(ref->token, &f->lookup);
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ *view.flags = flags;
+ mutex_unlock(memcomp_lock_of(ref->token));
+ KUNIT_EXPECT_EQ(test, ret, -EAGAIN);
+ KUNIT_EXPECT_PTR_EQ(test, f->lookup.slot, NULL);
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), refs);
+}
+
+static void memcomp_test_query_nonresident_guard(struct kunit *test)
+{
+ struct mc16_test_owner *f = mc16_test_owner_begin(test, 1);
+ struct mc16_test_storage *s;
+ u32 token;
+ int ret;
+ bool dropped;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ token = s->refs[0].token;
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_payload(f->refs, 0, MC16_SAMEFILL), 0);
+ ret = mc16_entry_resident_get(token, &f->survivor[0]);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ KUNIT_ASSERT_NOT_NULL(test, f->survivor[0].slot);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->survivor[0].slot->refs), 2U);
+ KUNIT_ASSERT_TRUE(test, mc16_get(token, &f->lookup));
+ dropped = mc16_owner_drop_folio_held(&f->lookup, s->owner->folio, 1, 0);
+ if (dropped)
+ s->refs[0] = (struct memcomp_ref) { };
+ KUNIT_ASSERT_TRUE(test, dropped);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->survivor[0].slot->refs), 1U);
+ KUNIT_EXPECT_TRUE(test, mc16_entry_present(token));
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(f->survivor[0].slot->flags),
+ (unsigned int)MC16_SAMEFILL);
+ mc16_entry_resident_put(&f->survivor[0]);
+ KUNIT_EXPECT_PTR_EQ(test, f->survivor[0].slot, NULL);
+ KUNIT_EXPECT_FALSE(test, mc16_entry_present(token));
+}
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_QUERY_TEST_H */
diff --git a/mm/memcomp/tests/refs_test.h b/mm/memcomp/tests/refs_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..a147f4155eba2c540ea3b469a076729798c1e48d
--- /dev/null
+++ b/mm/memcomp/tests/refs_test.h
@@ -0,0 +1,224 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_REFS_TEST_H
+#define _MM_MEMCOMP_TESTS_REFS_TEST_H
+#ifdef CONFIG_MEMCG
+struct mc16_test_refs {
+ struct mc16_test_storage *storage;
+ bool published[MC16_TEST_REFS];
+ s64 stored_before;
+ s64 compressed_before;
+ s64 pending_before;
+};
+
+static void mc16_test_refs_cleanup(void *data)
+{
+ struct mc16_test_refs *f = data;
+ struct mc16_test_storage *storage = f->storage;
+ unsigned int i;
+
+ for (i = 0; i < storage->nr; i++) {
+ if (f->published[i] && storage->refs[i].slot)
+ mc16_ref_put(&storage->refs[i]);
+ }
+ KUNIT_EXPECT_EQ(storage->test, atomic64_read(&memcomp_stored_bytes),
+ f->stored_before);
+ KUNIT_EXPECT_EQ(storage->test, atomic64_read(&memcomp_compressed_bytes),
+ f->compressed_before);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ KUNIT_EXPECT_EQ(storage->test, atomic64_read(&memcomp_pending_pages),
+ f->pending_before);
+#endif
+}
+
+static struct mc16_test_refs *mc16_test_refs_begin(struct kunit *test)
+{
+ struct mc16_test_refs *f;
+ int ret;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->storage = mc16_test_storage_begin(test);
+ if (!f->storage)
+ return NULL;
+ f->stored_before = atomic64_read(&memcomp_stored_bytes);
+ f->compressed_before = atomic64_read(&memcomp_compressed_bytes);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ f->pending_before = atomic64_read(&memcomp_pending_pages);
+#endif
+ ret = kunit_add_action_or_reset(test, mc16_test_refs_cleanup, f);
+ return ret ? NULL : f;
+}
+
+static int mc16_test_refs_reserve(struct mc16_test_refs *f, unsigned int nr)
+{
+ unsigned int start = f->storage->nr, i;
+ int ret;
+
+ ret = mc16_test_reserve(f->storage, nr);
+ if (ret)
+ return ret;
+ for (i = start; i < start + nr; i++) {
+ mc16_account_reservation(&f->storage->refs[i]);
+ f->published[i] = true;
+ }
+ return 0;
+}
+
+/* Publish a real payload while the slot remains physically record-backed. */
+static int mc16_test_refs_payload(struct mc16_test_refs *f, unsigned int i,
+ unsigned int state)
+{
+ struct memcomp_ref *ref = &f->storage->refs[i];
+ struct mc16_view view;
+ unsigned long handle = 0;
+ u32 bytes = state == MC16_STORED ? PAGE_SIZE : 0;
+ int ret;
+
+ if (bytes) {
+ ret = memcomp_zs_malloc(memcomp_pools[0], bytes, &handle, false);
+ if (ret)
+ return ret;
+ ret = memcomp_objcg_charge(&f->storage->owner->model.objcg,
+ GFP_KERNEL, bytes);
+ if (ret) {
+ memcomp_zs_free(0, handle);
+ return ret;
+ }
+ }
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ mc16_account_state(ref, false);
+ folio_put(view.member->owner_folio);
+ *view.flags = (*view.flags & MC16_ORDER_MASK) | state | MC16_F_OWNER;
+ *view.length = bytes ? bytes - 1 : 0;
+ *view.word = bytes ? handle : 0x9a9a9a9a9a9a9a9aULL;
+ view.member->payload_charge = bytes;
+ memcomp_debug_dec(&memcomp_pending_pages);
+ memcomp_compressed_bytes_add(bytes);
+ mc16_account_state(ref, true);
+ mc16_fifo_transition(ref, MC16_PENDING, state);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ return 0;
+}
+
+static void memcomp_test_put_pair(struct kunit *test)
+{
+ struct mc16_test_refs *f;
+ struct memcomp_ref guard;
+ spinlock_t ptl; /* Models a paired lookup and persistent PTE put. */
+ unsigned long irqflags;
+ u32 token;
+
+ f = mc16_test_refs_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f, 1), 0);
+ token = f->storage->refs[0].token;
+ KUNIT_ASSERT_TRUE(test, mc16_get(token, &guard));
+ f->storage->refs[0] = (struct memcomp_ref) { };
+ spin_lock_init(&ptl);
+ spin_lock_irqsave(&ptl, irqflags);
+ mc16_ref_put_count(&guard, 2);
+ spin_unlock_irqrestore(&ptl, irqflags);
+ KUNIT_EXPECT_PTR_EQ(test, guard.slot, NULL);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->storage->owner->folio), 1);
+ KUNIT_EXPECT_FALSE(test, mc16_get(token, &guard));
+}
+
+static void memcomp_test_fifo_eligibility(struct kunit *test)
+{
+ struct mc16_test_refs *f;
+ struct mc16_test_storage *storage;
+ struct memcomp_lru *fifo;
+ struct memcomp_ref guard;
+ unsigned int budget;
+ long before;
+
+ f = mc16_test_refs_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->storage;
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f, 2), 0);
+ fifo = mc16_fifo_of(&storage->refs[0]);
+ before = atomic_long_read(&fifo->nr_entries);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_payload(f, 0, MC16_STORED), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_payload(f, 1, MC16_SAMEFILL), 0);
+ KUNIT_EXPECT_EQ(test, storage->refs[0].cluster->eligible_slots, (u16)1);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&fifo->nr_entries), before + 1);
+ budget = 2;
+ KUNIT_ASSERT_TRUE(test, mc16_fifo_select(fifo, &budget, &guard));
+ KUNIT_EXPECT_EQ(test, guard.token, storage->refs[0].token);
+ mc16_ref_put(&guard);
+ mc16_ref_put(&storage->refs[0]);
+ KUNIT_EXPECT_EQ(test, storage->refs[1].cluster->eligible_slots, (u16)0);
+ KUNIT_EXPECT_TRUE(test, list_empty(&storage->refs[1].cluster->fifo));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&fifo->nr_entries), before);
+ /* SAMEFILL may still live but can no longer keep a cluster on FIFO. */
+ budget = 10;
+ KUNIT_EXPECT_FALSE(test, mc16_fifo_select(fifo, &budget, &guard));
+ KUNIT_EXPECT_EQ(test, budget, 10U);
+}
+
+static void memcomp_test_fifo_scan_budget(struct kunit *test)
+{
+ struct mc16_test_refs *f;
+ struct mc16_test_storage *storage;
+ struct memcomp_ref guard;
+ struct mc16_view view;
+ struct memcomp_lru *fifo;
+ unsigned int i, budget = 65, state, found = 0;
+
+ f = mc16_test_refs_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->storage;
+ for (i = 0; i < 65; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f, 1), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_payload(f, 64, MC16_STORED), 0);
+ fifo = mc16_fifo_of(&storage->refs[64]);
+ while (mc16_fifo_select(fifo, &budget, &guard)) {
+ mutex_lock(memcomp_lock_of(guard.token));
+ mc16_view_locked(&guard, &view);
+ state = mc16_flags_state(*view.flags);
+ mutex_unlock(memcomp_lock_of(guard.token));
+ if (state == MC16_STORED)
+ found++;
+ mc16_ref_put(&guard);
+ }
+ KUNIT_EXPECT_EQ(test, found, 1U);
+ KUNIT_EXPECT_EQ(test, budget, 0U);
+}
+
+static void memcomp_test_writeback_last_guard(struct kunit *test)
+{
+ struct mc16_test_refs *f;
+ struct mc16_test_storage *storage;
+ struct memcomp_ref writer;
+ struct mc16_view view;
+ struct memcomp_lru *fifo;
+ unsigned int budget = 1;
+ bool hot;
+
+ f = mc16_test_refs_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->storage;
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f, 1), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_payload(f, 0, MC16_STORED), 0);
+ fifo = mc16_fifo_of(&storage->refs[0]);
+ KUNIT_ASSERT_TRUE(test, mc16_fifo_select(fifo, &budget, &writer));
+ mutex_lock(memcomp_lock_of(writer.token));
+ mc16_view_locked(&writer, &view);
+ *view.flags |= MC16_F_REFERENCED;
+ hot = mc16_fifo_second_chance_locked(&writer);
+ KUNIT_EXPECT_TRUE(test, hot);
+ KUNIT_EXPECT_FALSE(test, mc16_fifo_second_chance_locked(&writer));
+ mc16_fifo_transition(&writer, MC16_STORED, MC16_WRITEBACK);
+ *view.flags = (*view.flags & ~MC16_STATE_MASK) | MC16_WRITEBACK;
+ mutex_unlock(memcomp_lock_of(writer.token));
+ /* PTE/owner release cannot destroy a payload still held by the writer. */
+ mc16_ref_put(&storage->refs[0]);
+ KUNIT_EXPECT_EQ(test, refcount_read(&writer.slot->refs), 1U);
+ KUNIT_EXPECT_TRUE(test, list_empty(&fifo->entries));
+ mc16_ref_put(&writer);
+ KUNIT_EXPECT_PTR_EQ(test, writer.slot, NULL);
+}
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_REFS_TEST_H */
diff --git a/mm/memcomp/tests/replacement_test.h b/mm/memcomp/tests/replacement_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..ee212cec936070398831df9b6c93a8999427ac5d
--- /dev/null
+++ b/mm/memcomp/tests/replacement_test.h
@@ -0,0 +1,360 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_REPLACEMENT_TEST_H
+#define _MM_MEMCOMP_TESTS_REPLACEMENT_TEST_H
+
+/*
+ * Author: nakixii
+ *
+ * Include after store_transaction_test.h in suite.h.
+ * Register the four cases below under MC16_REPLACEMENT_TEST_ENABLED.
+ * Inherits MEMCOMP_KUNIT_TEST (DEBUG_STATS), MEMCG, MMU, ADVISE_SYSCALLS.
+ *
+ * Old oracles: 6cc25de18927:mm/suite.h:
+ * refresh_rejects_mixed_state: -EAGAIN, one state event/two pages, unchanged
+ * PENDING+RESIDENT and SAMEFILL members, one pending page, balanced rollback.
+ * reserve_replaces_stored_span: a different reservation, one pending page,
+ * old token absent after its final reference, balanced reservation teardown.
+ * Extend replacement to order 1 and retain real old payloads with guards.
+ *
+ * Public reserve/store/load/release and real MM/PTE/rmap/codec/zs execute.
+ * A sibling SAMEFILL uses the existing real payload commit/finish fixture;
+ * the reverse mixed order uses an injected second-commit store failure.
+ * No fabricated entry state or mocked successful store is used.
+ * There is no public refresh entry point: call the production mc16 refresh
+ * transaction for its state-event oracle, and separately check public store's
+ * earlier mixed-span PREFLIGHT rejection. These counters are not interchangeable.
+ * Objcg/slot allocation and commit failure routing use the serial test models.
+ * No concurrent race, device IO or reclaim scheduling coverage is claimed.
+ * All actors release locks/leases before returning to any KUnit assertion.
+ */
+#ifdef MC16_STORE_TRANSACTION_TEST_ENABLED
+#define MC16_REPLACEMENT_TEST_ENABLED
+
+struct mc16_rct_member {
+ u64 word;
+ u16 length;
+ u16 flags;
+ unsigned int refs;
+ bool valid;
+};
+
+/* Only scalar effective state crosses T; the caller owns the lookup guard. */
+static struct mc16_rct_member mc16_rct_member(struct memcomp_ref *ref)
+{
+ struct mc16_rct_member s = { };
+ struct mc16_view view;
+
+ mutex_lock(memcomp_lock_of(ref->token));
+ if (mc16_view_locked(ref, &view)) {
+ s.word = *view.word;
+ s.length = *view.length;
+ s.flags = *view.flags;
+ s.refs = refcount_read(&ref->slot->refs);
+ s.valid = mc16_flags_valid(s.flags);
+ }
+ mutex_unlock(memcomp_lock_of(ref->token));
+ return s;
+}
+
+/* Drop the temporary public lookup before returning or asserting. */
+static struct mc16_rct_member mc16_rct_lookup(unsigned long token)
+{
+ struct mc16_rct_member s = { };
+ struct memcomp_ref ref = { };
+
+ if (memcomp_entry_get_ref(token, &ref)) {
+ s = mc16_rct_member(&ref);
+ /* Report references excluding this temporary lookup. */
+ if (s.refs)
+ s.refs--;
+ memcomp_entry_put_ref(&ref);
+ }
+ return s;
+}
+
+static void mc16_rct_expect_unchanged(struct mc16_flt *f,
+ struct mc16_rct_member before[2])
+{
+ struct mc16_rct_member after;
+ unsigned int i;
+
+ for (i = 0; i < f->nr; i++) {
+ after = mc16_rct_member(&f->guards[i]);
+ KUNIT_EXPECT_TRUE(f->test, after.valid);
+ KUNIT_EXPECT_EQ(f->test, after.word, before[i].word);
+ KUNIT_EXPECT_EQ(f->test, after.length, before[i].length);
+ KUNIT_EXPECT_EQ(f->test, after.flags, before[i].flags);
+ KUNIT_EXPECT_EQ(f->test, after.refs, before[i].refs);
+ }
+}
+
+/* Unlike public store, the refresh transaction retains the caller's ctx. */
+static int mc16_rct_refresh(struct mc16_flt *f, const void *src)
+{
+ struct memcomp_reclaim_ctx *ctx;
+ struct memcomp_store_ctx store = { };
+ unsigned int flags;
+ int ret = -EBUSY;
+
+ flags = memcomp_test_reclaimer_enter();
+ ctx = memcomp_reclaim_ctx_get(f->nr, MEMCOMP_RECLAIM_KSWAPD);
+ if (!ctx)
+ goto out;
+ folio_lock(f->folio);
+ ret = mc16_anon_refresh(&ctx, src, f->nr, &store, f->guards);
+ folio_unlock(f->folio);
+ if (ctx)
+ memcomp_reclaim_ctx_put(ctx);
+ memcomp_direct_staging_put(store.staging);
+out:
+ memcomp_test_reclaimer_exit(flags);
+ return ret;
+}
+
+static void mc16_rct_mixed(struct kunit *test, bool sibling_samefill)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 1);
+ struct mc16_rct_member members[2];
+ struct mc16_sct_stats stats;
+ s64 pending;
+ long charged;
+ u8 *src, *dst;
+ unsigned int i, stored = sibling_samefill ? 1 : 0;
+ int pins, ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ src = kunit_kmalloc(test, 2 * PAGE_SIZE, GFP_KERNEL);
+ dst = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ KUNIT_ASSERT_NOT_NULL(test, dst);
+ mc16_sct_pattern(src, 2, false);
+ if (sibling_samefill)
+ memset(src + PAGE_SIZE, 0x5a, PAGE_SIZE);
+ mc16_sct_copy(f, src);
+ pending = atomic64_read(&memcomp_pending_pages);
+ pins = atomic_read(&f->anon_vma->refcount);
+ KUNIT_ASSERT_TRUE(test, mc16_sct_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ for (i = 0; i < f->nr; i++)
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, i), 0);
+ stats = mc16_sct_stats_read();
+ if (sibling_samefill) {
+ /* Actual SAMEFILL commit matches the source bytes above. */
+ KUNIT_ASSERT_EQ(test, mc16_flt_commit(f, 1), 0);
+ mc16_sct_stats_expect(test, stats, (struct mc16_sct_stats) { });
+ } else {
+ /* Real codec/zs prefix, then public store fails commit 2. */
+ mc16_sct_run(f, (struct mc16_sct_attempt) {
+ .fail = MEMCOMP_KUNIT_FAIL_REFRESH_COMMIT, .nth = 2,
+ }, false);
+ mc16_sct_stats_expect(test, stats, (struct mc16_sct_stats) {
+ .state = 1, .state_pages = 2, .bg_fail = 2,
+ });
+ }
+ for (i = 0; i < f->nr; i++) {
+ members[i] = mc16_rct_member(&f->guards[i]);
+ KUNIT_ASSERT_TRUE(test, members[i].valid);
+ KUNIT_EXPECT_TRUE(test, members[i].flags & MC16_F_OWNER);
+ KUNIT_EXPECT_FALSE(test, members[i].flags & MC16_F_REFRESH_PENDING);
+ KUNIT_EXPECT_EQ(test, members[i].refs, 3U);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(members[i].flags),
+ i == stored ? (sibling_samefill ? MC16_SAMEFILL :
+ MC16_STORED) : MC16_PENDING);
+ KUNIT_EXPECT_EQ(test, !!(members[i].flags & MC16_F_RESIDENT),
+ i != stored);
+ }
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_pending_pages), pending + 1);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), 2);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->anon_vma->refcount), pins + 1);
+ charged = atomic_long_read(&f->base->model.charged);
+ stats = mc16_sct_stats_read();
+ ret = mc16_rct_refresh(f, src);
+ KUNIT_EXPECT_EQ(test, ret, -EAGAIN);
+ mc16_sct_stats_expect(test, stats, (struct mc16_sct_stats) {
+ .state = 1, .state_pages = 2,
+ });
+ mc16_rct_expect_unchanged(f, members);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->base->model.charged), charged);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_pending_pages), pending + 1);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), 2);
+ stats = mc16_sct_stats_read();
+ /* Public wrapper refuses the mixed span before refresh validation. */
+ mc16_sct_run(f, (struct mc16_sct_attempt) { }, false);
+ mc16_sct_stats_expect(test, stats, (struct mc16_sct_stats) {
+ .preflight = 2, .bg_fail = 2,
+ });
+ mc16_rct_expect_unchanged(f, members);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->base->model.charged), charged);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_pending_pages), pending + 1);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->anon_vma->refcount), pins + 1);
+ mc16_sct_rollback(f);
+ for (i = 0; i < f->nr; i++) {
+ mc16_flt_expect_present(f, i, f->addr + i * PAGE_SIZE);
+ KUNIT_EXPECT_FALSE(test, mc16_flt_flags(&f->guards[i], false) &
+ MC16_F_OWNER);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guards[i].slot->refs), 1U);
+ }
+ /* Normalized committed member remains loadable through its guard. */
+ mc16_sct_expect_load(f, stored, src + stored * PAGE_SIZE, dst);
+ for (i = 0; i < f->nr; i++)
+ memcomp_entry_put_ref(&f->guards[i]);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_pending_pages), pending);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->anon_vma->refcount), pins);
+}
+
+static void memcomp_test_replacement_mixed_sibling(struct kunit *test)
+{
+ mc16_rct_mixed(test, true);
+}
+
+static void memcomp_test_replacement_mixed_head(struct kunit *test)
+{
+ mc16_rct_mixed(test, false);
+}
+
+static void mc16_rct_load(struct mc16_flt *f, unsigned long token,
+ const void *src, void *dst)
+{
+ struct memcomp_ref ref = { };
+ void *bytes;
+ int ret = -ENOENT;
+
+ if (memcomp_load_folio_pin(token, &ref)) {
+ folio_lock(f->base->dst);
+ ret = memcomp_load_folio(&ref, f->base->dst);
+ if (!ret) {
+ bytes = kmap_local_folio(f->base->dst, 0);
+ memcpy(dst, bytes, PAGE_SIZE);
+ kunmap_local(bytes);
+ }
+ folio_unlock(f->base->dst);
+ memcomp_load_folio_abort(&ref);
+ }
+ KUNIT_EXPECT_EQ(f->test, ret, 0);
+ if (!ret)
+ KUNIT_EXPECT_EQ(f->test, memcmp(src, dst, PAGE_SIZE), 0);
+}
+
+static void mc16_rct_release_guards(void *data)
+{
+ struct memcomp_ref *refs = data;
+ unsigned int i;
+
+ for (i = 0; i < 2; i++)
+ memcomp_entry_put_ref(&refs[i]);
+}
+
+static void mc16_rct_replace(struct kunit *test, unsigned int order)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, order);
+ struct mc16_rct_member old, fresh;
+ struct memcomp_ref *new_guards;
+ struct mc16_sct_stats stats;
+ unsigned long token, old_tokens[2];
+ u8 *src, *next, *dst;
+ s64 pending;
+ unsigned int i;
+ int pins;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ new_guards = kunit_kcalloc(test, 2, sizeof(*new_guards), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, new_guards);
+ KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test,
+ mc16_rct_release_guards, new_guards), 0);
+ src = kunit_kmalloc(test, f->nr * PAGE_SIZE, GFP_KERNEL);
+ next = kunit_kmalloc(test, f->nr * PAGE_SIZE, GFP_KERNEL);
+ dst = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ KUNIT_ASSERT_NOT_NULL(test, next);
+ KUNIT_ASSERT_NOT_NULL(test, dst);
+ mc16_sct_pattern(src, f->nr, false);
+ for (i = 0; i < f->nr * PAGE_SIZE; i++)
+ next[i] = src[i] ^ 0x73;
+ mc16_sct_copy(f, src);
+ pending = atomic64_read(&memcomp_pending_pages);
+ pins = atomic_read(&f->anon_vma->refcount);
+ KUNIT_ASSERT_TRUE(test, mc16_sct_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ for (i = 0; i < f->nr; i++) {
+ old_tokens[i] = f->guards[i].token;
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, i), 0);
+ }
+ stats = mc16_sct_stats_read();
+ mc16_sct_run(f, (struct mc16_sct_attempt) { }, true);
+ mc16_sct_stats_expect(test, stats, (struct mc16_sct_stats) {
+ .bg_ok = f->nr,
+ });
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_pending_pages), pending);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->anon_vma->refcount), pins);
+ for (i = 0; i < f->nr; i++) {
+ old = mc16_rct_member(&f->guards[i]);
+ KUNIT_ASSERT_TRUE(test, old.valid);
+ KUNIT_ASSERT_EQ(test, mc16_flags_state(old.flags), MC16_STORED);
+ /* Remove actual old PTE ownership, retaining OWNER + guard. */
+ KUNIT_ASSERT_EQ(test, mc16_flt_zap(f, f->addr + i * PAGE_SIZE), 0);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guards[i].slot->refs), 2U);
+ }
+ KUNIT_ASSERT_TRUE(test, mc16_sct_reserve(f));
+ token = swp_offset(f->folio->swap);
+ KUNIT_EXPECT_NE(test, token, old_tokens[0]);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), 1 + f->nr);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_pending_pages),
+ pending + f->nr);
+ /* The unmapped source takes no new folio_get_anon_vma() pin. */
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->anon_vma->refcount), pins);
+ for (i = 0; i < f->nr; i++) {
+ old = mc16_rct_member(&f->guards[i]);
+ fresh = mc16_rct_lookup(token + i);
+ KUNIT_EXPECT_TRUE(test, old.valid);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(old.flags), MC16_STORED);
+ KUNIT_EXPECT_FALSE(test, old.flags & MC16_F_OWNER);
+ KUNIT_EXPECT_EQ(test, old.refs, 1U);
+ KUNIT_EXPECT_TRUE(test, memcomp_entry_present(old_tokens[i]));
+ KUNIT_EXPECT_TRUE(test, fresh.valid);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(fresh.flags), MC16_PENDING);
+ KUNIT_EXPECT_TRUE(test, fresh.flags & MC16_F_OWNER);
+ KUNIT_EXPECT_TRUE(test, fresh.flags & MC16_F_RESIDENT);
+ KUNIT_EXPECT_FALSE(test, fresh.flags & MC16_F_REFRESH_PENDING);
+ KUNIT_EXPECT_EQ(test, fresh.refs, 1U);
+ mc16_sct_expect_load(f, i, src + i * PAGE_SIZE, dst);
+ }
+ /* New stores change bytes without touching the guard-held old payloads. */
+ mc16_sct_copy(f, next);
+ stats = mc16_sct_stats_read();
+ mc16_sct_run(f, (struct mc16_sct_attempt) { }, true);
+ mc16_sct_stats_expect(test, stats, (struct mc16_sct_stats) {
+ .bg_ok = f->nr,
+ });
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_pending_pages), pending);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->anon_vma->refcount), pins);
+ /* A fresh span without PTE owners needs guards across real source exit. */
+ for (i = 0; i < f->nr; i++)
+ KUNIT_ASSERT_TRUE(test, memcomp_entry_get_ref(token + i,
+ &new_guards[i]));
+ KUNIT_ASSERT_TRUE(test, mc16_sct_release(f));
+ for (i = 0; i < f->nr; i++) {
+ mc16_sct_expect_load(f, i, src + i * PAGE_SIZE, dst);
+ mc16_rct_load(f, token + i, next + i * PAGE_SIZE, dst);
+ memcomp_entry_put_ref(&f->guards[i]);
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_present(old_tokens[i]));
+ KUNIT_EXPECT_TRUE(test, memcomp_entry_present(token + i));
+ }
+ for (i = 0; i < f->nr; i++)
+ memcomp_entry_put_ref(&new_guards[i]);
+ for (i = 0; i < f->nr; i++)
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_present(token + i));
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), 1);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_pending_pages), pending);
+}
+
+static void memcomp_test_replacement_stored_order0(struct kunit *test)
+{
+ mc16_rct_replace(test, 0);
+}
+
+static void memcomp_test_replacement_stored_order1(struct kunit *test)
+{
+ mc16_rct_replace(test, 1);
+}
+#endif /* MC16_STORE_TRANSACTION_TEST_ENABLED */
+#endif /* _MM_MEMCOMP_TESTS_REPLACEMENT_TEST_H */
diff --git a/mm/memcomp/tests/resident_test.h b/mm/memcomp/tests/resident_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..3a9e45ef8326fbfa96b21a652772f240af2121bf
--- /dev/null
+++ b/mm/memcomp/tests/resident_test.h
@@ -0,0 +1,796 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_RESIDENT_TEST_H
+#define _MM_MEMCOMP_TESTS_RESIDENT_TEST_H
+
+#include <kunit/resource.h>
+#include <kunit/static_stub.h>
+#include <linux/completion.h>
+#include <linux/kthread.h>
+#include <linux/mman.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task.h>
+#include "../resident.h"
+
+/* Include after consumers_test.h. Real MM/folio/rmap, modeled metadata objcg. */
+#if defined(CONFIG_MEMCG) && defined(CONFIG_ADVISE_SYSCALLS)
+#define MC16_RESIDENT_TEST_ENABLED
+enum mc16_rl_race {
+ MC16_RL_NONE,
+ MC16_RL_STORED,
+ MC16_RL_FOLIO,
+ MC16_RL_RESIDENT,
+};
+
+struct mc16_rl {
+ struct mc16_consumer_test *base;
+ struct mm_struct *mm;
+ unsigned long addr;
+ struct folio *folio, *other;
+ struct memcomp_ref owners[2], guard;
+ unsigned int nr, target;
+ struct anon_vma *anon_vma; /* Borrowed from the producer's record hold. */
+ vm_flags_t saved_flags;
+ pgprot_t saved_prot;
+ bool saved_vma;
+ pte_t orig;
+ s64 swapents, anonpages;
+ enum mc16_rl_race race;
+ int hook_error, lookup;
+ bool switched, cleared, handled;
+ unsigned int fault_flags;
+ pte_t *fault_pte;
+ struct task_struct *worker;
+ struct completion locked, go;
+};
+
+static struct mc16_rl *mc16_rl_current;
+
+static s64 mc16_rl_rss(struct mc16_rl *f, int member)
+{
+ return percpu_counter_sum(&f->mm->rss_stat[member]);
+}
+
+/* The PTE owns one existing slot ref; removing it requires no new lookup. */
+static void mc16_rl_zap_token(struct mc16_rl *f)
+{
+ struct memcomp_ref removed;
+ spinlock_t *ptl; /* Serializes the real PTE and its owner reference. */
+ pte_t *ptep, pte;
+ pmd_t *pmd;
+
+ if (!f->owners[f->target].slot)
+ return;
+ mmap_read_lock(f->mm);
+ pmd = mm_find_pmd(f->mm, f->addr);
+ if (!pmd)
+ goto out;
+ ptep = pte_offset_map_lock(f->mm, pmd, f->addr, &ptl);
+ if (!ptep)
+ goto out;
+ pte = ptep_get(ptep);
+ if (is_swap_pte(pte) && is_memcomp_entry(pte_to_swp_entry(pte)) &&
+ swp_offset(pte_to_swp_entry(pte)) == f->owners[f->target].token) {
+ ptep_get_and_clear(f->mm, f->addr, ptep);
+ add_mm_counter(f->mm, MM_SWAPENTS, -1);
+ removed = f->owners[f->target];
+ mc16_ref_put(&removed);
+ }
+ pte_unmap_unlock(ptep, ptl);
+out:
+ mmap_read_unlock(f->mm);
+}
+
+/* Undo only deliberately injected invalid identity/flags before final puts. */
+static void mc16_rl_restore_race(struct mc16_rl *f)
+{
+ struct mc16_view view;
+
+ if (!f->switched && !f->cleared)
+ return;
+ folio_lock(f->folio);
+ mutex_lock(memcomp_lock_of(f->owners[f->target].token));
+ if (mc16_view_locked(&f->owners[f->target], &view)) {
+ if (f->switched) {
+ folio_get(f->folio);
+ view.member->owner_folio = f->folio;
+ folio_put(f->other);
+ }
+ if (f->cleared)
+ *view.flags |= MC16_F_RESIDENT;
+ }
+ mutex_unlock(memcomp_lock_of(f->owners[f->target].token));
+ folio_unlock(f->folio);
+ f->switched = false;
+ f->cleared = false;
+}
+
+static void mc16_rl_join(struct mc16_rl *f)
+{
+ if (f->worker) {
+ complete_all(&f->go);
+ kthread_stop_put(f->worker);
+ f->worker = NULL;
+ }
+}
+
+static void mc16_rl_cleanup(void *data)
+{
+ struct mc16_rl *f = data;
+ struct vm_area_struct *vma;
+ struct anon_vma *anon_vma = NULL;
+ unsigned int i;
+ int ret;
+
+ mc16_rl_join(f);
+ mc16_rl_current = NULL;
+ mc16_rl_restore_race(f);
+ if (f->guard.slot)
+ mc16_ref_put(&f->guard);
+ mc16_rl_zap_token(f);
+ if (f->nr) {
+ folio_lock(f->folio);
+ mutex_lock(memcomp_lock_of(f->owners[0].token));
+ anon_vma = mc16_span_take_anon_vma_locked(&f->owners[0],
+ f->folio, f->nr);
+ mutex_unlock(memcomp_lock_of(f->owners[0].token));
+ /* Producer siblings leave first; head/anon_vma bookkeeping last. */
+ for (i = f->nr; i > 0; i--)
+ mc16_ref_put(&f->owners[i - 1]);
+ folio_unlock(f->folio);
+ }
+ KUNIT_EXPECT_PTR_EQ(f->base->test, anon_vma, f->anon_vma);
+ if (anon_vma)
+ put_anon_vma(anon_vma);
+ f->anon_vma = NULL;
+ mmap_write_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (vma && f->saved_vma) {
+ vma_start_write(vma);
+ vm_flags_reset(vma, f->saved_flags);
+ vma->vm_page_prot = f->saved_prot;
+ }
+ mmap_write_unlock(f->mm);
+ ret = mc16_test_dontneed(f->mm, f->addr, PAGE_SIZE);
+ KUNIT_EXPECT_EQ(f->base->test, ret, 0);
+ KUNIT_EXPECT_EQ(f->base->test, mc16_rl_rss(f, MM_SWAPENTS), f->swapents);
+ KUNIT_EXPECT_EQ(f->base->test, mc16_rl_rss(f, MM_ANONPAGES), f->anonpages);
+ if (f->folio) {
+ KUNIT_EXPECT_FALSE(f->base->test, folio_test_locked(f->folio));
+ KUNIT_EXPECT_EQ(f->base->test, folio_mapcount(f->folio), 0);
+ folio_put(f->folio);
+ }
+ if (f->other)
+ folio_put(f->other);
+ f->base->model.folio = f->base->account;
+ mmput(f->mm);
+}
+
+static int mc16_rl_reserve(struct mc16_rl *f, unsigned int nr)
+{
+ struct mc16_record *record;
+ struct anon_vma *anon_vma;
+ unsigned int i;
+ int ret;
+
+ folio_lock(f->folio);
+ record = mc16_record_alloc(f->folio, nr, false);
+ if (IS_ERR(record)) {
+ ret = PTR_ERR(record);
+ goto unlock;
+ }
+ anon_vma = folio_get_anon_vma(f->folio);
+ if (!anon_vma) {
+ mc16_record_abort(record);
+ ret = -EINVAL;
+ goto unlock;
+ }
+ record->anon_vma = anon_vma;
+ record->record_flags |= MC16_RECORD_ANON_VMA;
+ ret = mc16_reserve(record, nr, MC16_KEY_ANON, f->owners);
+ if (ret) {
+ record->anon_vma = NULL;
+ record->record_flags &= ~MC16_RECORD_ANON_VMA;
+ mc16_record_abort(record);
+ put_anon_vma(anon_vma);
+ goto unlock;
+ }
+ f->nr = nr;
+ f->anon_vma = anon_vma;
+ for (i = 0; i < nr; i++)
+ mc16_account_reservation(&f->owners[i]);
+unlock:
+ folio_unlock(f->folio);
+ return ret;
+}
+
+/* Replace an actual anonymous PTE, retaining the folio's existing anon rmap. */
+static int mc16_rl_install(struct mc16_rl *f)
+{
+ struct vm_area_struct *vma;
+ struct memcomp_ref pte_ref;
+ spinlock_t *ptl; /* Serializes the real PTE and its owner reference. */
+ pte_t *ptep, old;
+ void *data;
+ int ret = -EINVAL;
+
+ mmap_read_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (!vma)
+ goto out;
+ folio_lock(f->folio);
+ data = kmap_local_folio(f->folio, 0);
+ memset(data, 0x6b, PAGE_SIZE);
+ kunmap_local(data);
+ flush_dcache_folio(f->folio);
+ ptep = get_locked_pte(f->mm, f->addr, &ptl);
+ if (!ptep) {
+ ret = -ENOMEM;
+ goto unlock_folio;
+ }
+ __acquire(RCU);
+ old = ptep_get(ptep);
+ if (!pte_present(old) || pte_page(old) != &f->folio->page ||
+ !mc16_get(f->owners[f->target].token, &pte_ref))
+ goto unlock_pte;
+ ptep_clear_flush(vma, f->addr, ptep);
+ folio_remove_rmap_pte(f->folio, &f->folio->page, vma);
+ add_mm_counter(f->mm, MM_ANONPAGES, -1);
+ add_mm_counter(f->mm, MM_SWAPENTS, 1);
+ set_pte_at(f->mm, f->addr, ptep, f->orig);
+ /* The get is now the token PTE's; the old mapping drops its folio ref. */
+ folio_put(f->folio);
+ ret = 0;
+unlock_pte:
+ pte_unmap_unlock(ptep, ptl);
+unlock_folio:
+ folio_unlock(f->folio);
+out:
+ mmap_read_unlock(f->mm);
+ return ret;
+}
+
+static struct mc16_rl *mc16_rl_begin(struct kunit *test, unsigned int nr,
+ unsigned int target)
+{
+ struct mc16_rl *f;
+ struct vm_area_struct *vma;
+ spinlock_t *ptl; /* Serializes the real PTE and its owner reference. */
+ pte_t *ptep, pte;
+ vm_fault_t fault;
+ int ret;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->base = mc16_consumer_test_begin(test);
+ if (!f->base)
+ return NULL;
+ f->addr = kunit_vm_mmap(test, NULL, 0, PAGE_SIZE,
+ PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0);
+ if (!f->addr || IS_ERR_VALUE(f->addr))
+ return NULL;
+ f->mm = current->mm;
+ mmget(f->mm);
+ f->target = target;
+ f->swapents = mc16_rl_rss(f, MM_SWAPENTS);
+ f->anonpages = mc16_rl_rss(f, MM_ANONPAGES);
+ init_completion(&f->locked);
+ init_completion(&f->go);
+ if (kunit_add_action_or_reset(test, mc16_rl_cleanup, f))
+ return NULL;
+ f->other = folio_alloc(GFP_KERNEL, 0);
+ if (!f->other)
+ return NULL;
+ mmap_write_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (vma) {
+ f->saved_flags = vma->vm_flags;
+ f->saved_prot = vma->vm_page_prot;
+ f->saved_vma = true;
+ vma_start_write(vma);
+ vm_flags_set(vma, VM_NOHUGEPAGE);
+ }
+ mmap_write_unlock(f->mm);
+ if (!vma)
+ return NULL;
+ mmap_read_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (!vma) {
+ ret = -ENOENT;
+ goto out;
+ }
+ fault = handle_mm_fault(vma, f->addr, FAULT_FLAG_WRITE, NULL);
+ ret = fault & VM_FAULT_ERROR ? -EFAULT : -ENOMEM;
+ if (fault & VM_FAULT_ERROR)
+ goto out;
+ ptep = get_locked_pte(f->mm, f->addr, &ptl);
+ if (!ptep)
+ goto out;
+ __acquire(RCU);
+ pte = ptep_get(ptep);
+ if (pte_present(pte) && !pte_special(pte)) {
+ f->folio = page_folio(pte_page(pte));
+ folio_get(f->folio);
+ ret = folio_test_large(f->folio) ? -EINVAL : 0;
+ }
+ pte_unmap_unlock(ptep, ptl);
+out:
+ mmap_read_unlock(f->mm);
+ if (ret)
+ return NULL;
+ f->base->model.folio = f->folio;
+ if (mc16_rl_reserve(f, nr))
+ return NULL;
+ f->orig = swp_entry_to_pte(make_memcomp_entry(f->owners[target].token));
+ f->orig = pte_swp_mkexclusive(f->orig);
+ if (mc16_rl_install(f))
+ return NULL;
+ /* Remove pagevec's temporary folio ref before ownership assertions. */
+ lru_add_drain_all();
+ return f;
+}
+
+static pte_t mc16_rl_read(struct mc16_rl *f)
+{
+ spinlock_t *ptl; /* Serializes the real PTE and its owner reference. */
+ pte_t *ptep, pte = __pte(0);
+ pmd_t *pmd;
+
+ mmap_read_lock(f->mm);
+ pmd = mm_find_pmd(f->mm, f->addr);
+ if (pmd) {
+ ptep = pte_offset_map_lock(f->mm, pmd, f->addr, &ptl);
+ if (ptep) {
+ pte = ptep_get(ptep);
+ pte_unmap_unlock(ptep, ptl);
+ }
+ }
+ mmap_read_unlock(f->mm);
+ return pte;
+}
+
+static int mc16_rl_change_pte(struct mc16_rl *f, pte_t pte)
+{
+ spinlock_t *ptl; /* Serializes the real PTE and its owner reference. */
+ pte_t *ptep;
+ int ret = 0;
+
+ mmap_read_lock(f->mm);
+ ptep = get_locked_pte(f->mm, f->addr, &ptl);
+ if (!ptep) {
+ ret = -ENOMEM;
+ } else {
+ __acquire(RCU);
+ if (!pte_same(ptep_get(ptep), f->orig))
+ ret = -ESTALE;
+ else
+ set_pte_at(f->mm, f->addr, ptep, pte);
+ pte_unmap_unlock(ptep, ptl);
+ }
+ mmap_read_unlock(f->mm);
+ return ret;
+}
+
+static void mc16_rl_policy(struct mc16_rl *f, vm_flags_t set, vm_flags_t clear)
+{
+ struct vm_area_struct *vma;
+
+ mmap_write_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (vma) {
+ vma_start_write(vma);
+ vm_flags_reset(vma, (vma->vm_flags | set) & ~clear);
+ vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+ }
+ mmap_write_unlock(f->mm);
+ KUNIT_EXPECT_NOT_NULL(f->base->test, vma);
+}
+
+/* A genuine producer commit removes RESIDENT and its folio hold. */
+static int mc16_rl_store(struct mc16_rl *f)
+{
+ struct mc16_payload payload = { .word = 0x123456789abcdef0ULL };
+ struct mc16_payload_tail tail = { };
+ struct memcomp_ref *ref = &f->owners[0];
+ int ret;
+
+ folio_lock(f->folio);
+ mutex_lock(memcomp_lock_of(ref->token));
+ ret = mc16_payload_commit_locked(ref, f->folio, 1, 0, &payload, &tail);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ if (!ret)
+ mc16_payload_finish(ref, &tail);
+ folio_unlock(f->folio);
+ return ret;
+}
+
+static vm_fault_t mc16_rl_lock(struct folio *folio, struct vm_fault *vmf)
+{
+ struct mc16_rl *f = mc16_rl_current;
+ struct memcomp_ref *ref = &f->owners[f->target];
+ struct mc16_view view;
+
+ if (f->race == MC16_RL_STORED) {
+ f->hook_error = mc16_rl_store(f);
+ } else {
+ folio_lock(folio);
+ mutex_lock(memcomp_lock_of(ref->token));
+ if (!mc16_view_locked(ref, &view)) {
+ f->hook_error = -EIO;
+ } else if (f->race == MC16_RL_FOLIO) {
+ folio_get(f->other);
+ view.member->owner_folio = f->other;
+ folio_put(folio);
+ f->switched = true;
+ } else if (f->race == MC16_RL_RESIDENT) {
+ *view.flags &= ~MC16_F_RESIDENT;
+ f->cleared = true;
+ }
+ mutex_unlock(memcomp_lock_of(ref->token));
+ folio_unlock(folio);
+ }
+ return folio_lock_or_retry(folio, vmf);
+}
+
+/* No fatal assertions, resource allocations or returned locks in this helper. */
+static vm_fault_t mc16_rl_fault(struct mc16_rl *f, unsigned int flags,
+ bool hold_folio)
+{
+ struct vm_area_struct *vma;
+ vm_fault_t ret = VM_FAULT_SIGBUS;
+
+ f->lookup = mc16_entry_resident_get(f->owners[f->target].token, &f->guard);
+ if (f->lookup < 0 || !f->guard.slot)
+ return ret;
+ if (f->race != MC16_RL_NONE) {
+ mc16_rl_current = f;
+ kunit_activate_static_stub(f->base->test, mc16_resident_lock_or_retry,
+ mc16_rl_lock);
+ }
+ mmap_read_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (!vma)
+ goto unlock_mm;
+ if (hold_folio)
+ folio_lock(f->folio);
+ if (f->worker)
+ complete(&f->go);
+ {
+ struct vm_fault vmf = {
+ .vma = vma, .address = f->addr, .real_address = f->addr,
+ .flags = flags | FAULT_FLAG_ORIG_PTE_VALID,
+ .pmd = mm_find_pmd(f->mm, f->addr), .orig_pte = f->orig,
+ .gfp_mask = GFP_HIGHUSER_MOVABLE,
+ };
+
+ ret = mc16_load_resident(&vmf, &f->guard, &f->handled);
+ f->fault_pte = vmf.pte;
+ f->fault_flags = vmf.flags;
+ }
+ if (hold_folio)
+ folio_unlock(f->folio);
+ /* NOWAIT retains the read lock; ordinary first RETRY has dropped it. */
+ if ((ret & VM_FAULT_RETRY) && !(flags & FAULT_FLAG_RETRY_NOWAIT))
+ goto unlocked_mm;
+unlock_mm:
+ mmap_read_unlock(f->mm);
+unlocked_mm:
+ if (f->race != MC16_RL_NONE) {
+ kunit_deactivate_static_stub(f->base->test, mc16_resident_lock_or_retry);
+ mc16_rl_current = NULL;
+ }
+ mc16_rl_join(f);
+ mc16_rl_restore_race(f);
+ return ret;
+}
+
+static void mc16_rl_counts(struct mc16_rl *f, unsigned int ptes,
+ unsigned int present, unsigned int refs)
+{
+ struct mc16_view view;
+ bool owner = false, anon = false;
+
+ KUNIT_EXPECT_EQ(f->base->test, mc16_rl_rss(f, MM_SWAPENTS), f->swapents + ptes);
+ KUNIT_EXPECT_EQ(f->base->test, mc16_rl_rss(f, MM_ANONPAGES), f->anonpages + present);
+ KUNIT_EXPECT_EQ(f->base->test, refcount_read(&f->owners[f->target].slot->refs),
+ refs);
+ KUNIT_EXPECT_EQ(f->base->test, folio_mapcount(f->folio), (int)present);
+ KUNIT_EXPECT_FALSE(f->base->test, folio_test_locked(f->folio));
+ KUNIT_EXPECT_PTR_EQ(f->base->test, f->fault_pte, NULL);
+ mutex_lock(memcomp_lock_of(f->owners[0].token));
+ if (mc16_view_locked(&f->owners[0], &view)) {
+ owner = *view.flags & MC16_F_OWNER;
+ anon = view.record && view.record->anon_vma == f->anon_vma &&
+ (view.record->record_flags & MC16_RECORD_ANON_VMA);
+ }
+ mutex_unlock(memcomp_lock_of(f->owners[0].token));
+ KUNIT_EXPECT_TRUE(f->base->test, owner);
+ KUNIT_EXPECT_TRUE(f->base->test, anon);
+}
+
+static void mc16_rl_present(struct mc16_rl *f, bool write, bool dirty)
+{
+ pte_t pte = mc16_rl_read(f);
+ void *data;
+ bool same;
+
+ KUNIT_ASSERT_TRUE(f->base->test, pte_present(pte));
+ KUNIT_EXPECT_PTR_EQ(f->base->test, pte_page(pte), &f->folio->page);
+ KUNIT_EXPECT_EQ(f->base->test, !!pte_write(pte), write);
+ if (dirty)
+ KUNIT_EXPECT_TRUE(f->base->test, pte_dirty(pte));
+ else if (!write)
+ KUNIT_EXPECT_FALSE(f->base->test, pte_dirty(pte));
+ KUNIT_EXPECT_TRUE(f->base->test, folio_test_anon(f->folio));
+ KUNIT_EXPECT_TRUE(f->base->test, folio_test_swapbacked(f->folio));
+ KUNIT_EXPECT_TRUE(f->base->test, folio_test_uptodate(f->folio));
+ data = kmap_local_folio(f->folio, 0);
+ same = !memchr_inv(data, 0x6b, PAGE_SIZE);
+ kunmap_local(data);
+ KUNIT_EXPECT_TRUE(f->base->test, same);
+ mc16_rl_counts(f, 0, 1, 1);
+}
+
+static void memcomp_test_resident_write(struct kunit *test)
+{
+ struct mc16_rl *f = mc16_rl_begin(test, 1, 0);
+ pte_t orig;
+ vm_fault_t ret;
+ int refs;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ refs = folio_ref_count(f->folio);
+ orig = pte_swp_mksoft_dirty(f->orig);
+ KUNIT_ASSERT_EQ(test, mc16_rl_change_pte(f, orig), 0);
+ f->orig = orig;
+ ret = mc16_rl_fault(f, FAULT_FLAG_WRITE, false);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_NOPAGE);
+ KUNIT_EXPECT_TRUE(test, f->handled);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_FALSE(test, f->fault_flags & FAULT_FLAG_WRITE);
+ mc16_rl_present(f, true, true);
+ KUNIT_EXPECT_EQ(test, !!pte_soft_dirty(mc16_rl_read(f)), !!pte_swp_soft_dirty(orig));
+ KUNIT_EXPECT_TRUE(test, PageAnonExclusive(&f->folio->page));
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), refs + 1);
+ /* The still-resident producer cannot make a stale second fault win. */
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)mc16_rl_fault(f, 0, false), 0U);
+ KUNIT_EXPECT_TRUE(test, f->handled);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), refs + 1);
+ mc16_rl_counts(f, 0, 1, 1);
+}
+
+static void memcomp_test_resident_readonly(struct kunit *test)
+{
+ struct mc16_rl *f = mc16_rl_begin(test, 1, 0);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_rl_policy(f, 0, VM_WRITE);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)mc16_rl_fault(f, 0, false),
+ (__force unsigned int)VM_FAULT_NOPAGE);
+ KUNIT_EXPECT_TRUE(test, f->handled);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ mc16_rl_present(f, false, false);
+}
+
+static void memcomp_test_resident_shared(struct kunit *test)
+{
+ struct mc16_rl *f = mc16_rl_begin(test, 1, 0);
+ pte_t orig;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ orig = pte_swp_clear_exclusive(f->orig);
+ KUNIT_ASSERT_EQ(test, mc16_rl_change_pte(f, orig), 0);
+ f->orig = orig;
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)mc16_rl_fault(f, FAULT_FLAG_WRITE, false),
+ (__force unsigned int)VM_FAULT_NOPAGE);
+ KUNIT_EXPECT_TRUE(test, f->fault_flags & FAULT_FLAG_WRITE);
+ mc16_rl_present(f, false, false);
+}
+
+static void memcomp_test_resident_softdirty_wp(struct kunit *test)
+{
+ struct mc16_rl *f;
+
+ if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY)) {
+ kunit_skip(test, "soft-dirty tracking is disabled");
+ return;
+ }
+ f = mc16_rl_begin(test, 1, 0);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_rl_policy(f, 0, VM_SOFTDIRTY);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)mc16_rl_fault(f, 0, false),
+ (__force unsigned int)VM_FAULT_NOPAGE);
+ mc16_rl_present(f, false, false);
+ KUNIT_EXPECT_FALSE(test, pte_soft_dirty(mc16_rl_read(f)));
+}
+
+static void memcomp_test_resident_uffd_wp(struct kunit *test)
+{
+ struct mc16_rl *f;
+ pte_t orig = pte_swp_mkuffd_wp(swp_entry_to_pte(make_memcomp_entry(1)));
+
+ if (!IS_ENABLED(CONFIG_USERFAULTFD) || !pte_swp_uffd_wp(orig)) {
+ kunit_skip(test, "architecture has no enabled swap UFFD-WP bit");
+ return;
+ }
+ f = mc16_rl_begin(test, 1, 0);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_rl_policy(f, VM_UFFD_WP, 0);
+ orig = pte_swp_mkuffd_wp(f->orig);
+ KUNIT_ASSERT_EQ(test, mc16_rl_change_pte(f, orig), 0);
+ f->orig = orig;
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)mc16_rl_fault(f, 0, false),
+ (__force unsigned int)VM_FAULT_NOPAGE);
+ mc16_rl_present(f, false, false);
+ KUNIT_EXPECT_TRUE(test, pte_uffd_wp(mc16_rl_read(f)));
+}
+
+static void memcomp_test_resident_changed_pte(struct kunit *test)
+{
+ struct mc16_rl *f = mc16_rl_begin(test, 1, 0);
+ pte_t changed;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ changed = pte_swp_mksoft_dirty(f->orig);
+ if (pte_same(changed, f->orig))
+ changed = pte_swp_clear_exclusive(f->orig);
+ KUNIT_ASSERT_FALSE(test, pte_same(changed, f->orig));
+ KUNIT_ASSERT_EQ(test, mc16_rl_change_pte(f, changed), 0);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)mc16_rl_fault(f, 0, false), 0U);
+ KUNIT_EXPECT_TRUE(test, f->handled);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_rl_read(f), changed));
+ mc16_rl_counts(f, 1, 0, 2);
+}
+
+static void memcomp_test_resident_zapped(struct kunit *test)
+{
+ struct mc16_rl *f = mc16_rl_begin(test, 1, 0);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_rl_zap_token(f);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)mc16_rl_fault(f, 0, false), 0U);
+ KUNIT_EXPECT_TRUE(test, f->handled);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_TRUE(test, pte_none(mc16_rl_read(f)));
+ mc16_rl_counts(f, 0, 0, 1);
+}
+
+static void memcomp_test_resident_unhandled(struct kunit *test)
+{
+ struct mc16_rl *f = mc16_rl_begin(test, 1, 0);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_rl_store(f), 0);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)mc16_rl_fault(f, 0, false), 0U);
+ KUNIT_EXPECT_EQ(test, f->lookup, 0);
+ KUNIT_EXPECT_FALSE(test, f->handled);
+ KUNIT_EXPECT_NOT_NULL(test, f->guard.slot);
+ mc16_rl_counts(f, 1, 0, 3);
+ mc16_ref_put(&f->guard);
+}
+
+static void mc16_rl_race_case(struct kunit *test, enum mc16_rl_race race)
+{
+ struct mc16_rl *f = mc16_rl_begin(test, 1, 0);
+ int refs;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ refs = folio_ref_count(f->folio);
+ f->race = race;
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)mc16_rl_fault(f, 0, false), 0U);
+ KUNIT_EXPECT_EQ(test, f->hook_error, 0);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), refs - (race == MC16_RL_STORED));
+ KUNIT_EXPECT_TRUE(test, f->handled);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_rl_read(f), f->orig));
+ mc16_rl_counts(f, 1, 0, 2);
+}
+
+static void memcomp_test_resident_state_race(struct kunit *test)
+{
+ mc16_rl_race_case(test, MC16_RL_STORED);
+}
+
+static void memcomp_test_resident_folio_race(struct kunit *test)
+{
+ mc16_rl_race_case(test, MC16_RL_FOLIO);
+}
+
+static void memcomp_test_resident_flag_race(struct kunit *test)
+{
+ mc16_rl_race_case(test, MC16_RL_RESIDENT);
+}
+
+static void memcomp_test_resident_bad_index(struct kunit *test)
+{
+ /* A real two-member record on an order-0 folio models corrupt geometry. */
+ struct mc16_rl *f = mc16_rl_begin(test, 2, 1);
+ pte_t changed;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)mc16_rl_fault(f, 0, false),
+ (__force unsigned int)VM_FAULT_SIGBUS);
+ KUNIT_EXPECT_TRUE(test, f->handled);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_rl_read(f), f->orig));
+ mc16_rl_counts(f, 1, 0, 2);
+ /* Corruption must not SIGBUS a PTE that has already changed identity. */
+ changed = pte_swp_clear_exclusive(f->orig);
+ KUNIT_ASSERT_EQ(test, mc16_rl_change_pte(f, changed), 0);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)mc16_rl_fault(f, 0, false), 0U);
+ KUNIT_EXPECT_TRUE(test, f->handled);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_rl_read(f), changed));
+ mc16_rl_counts(f, 1, 0, 2);
+}
+
+static void memcomp_test_resident_retry_nowait(struct kunit *test)
+{
+ struct mc16_rl *f = mc16_rl_begin(test, 1, 0);
+ vm_fault_t ret;
+ int refs;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ refs = folio_ref_count(f->folio);
+ ret = mc16_rl_fault(f, FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT, true);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_RETRY);
+ KUNIT_EXPECT_TRUE(test, f->handled);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), refs);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_rl_read(f), f->orig));
+ mc16_rl_counts(f, 1, 0, 2);
+}
+
+static int mc16_rl_retry_writer(void *data)
+{
+ struct mc16_rl *f = data;
+
+ folio_lock(f->folio);
+ complete(&f->locked);
+ wait_for_completion(&f->go);
+ /* The ordinary retry must drop mmap_read_lock before waiting on us. */
+ mmap_write_lock(f->mm);
+ folio_unlock(f->folio);
+ mmap_write_unlock(f->mm);
+ return 0;
+}
+
+static void memcomp_test_resident_retry(struct kunit *test)
+{
+ struct mc16_rl *f = mc16_rl_begin(test, 1, 0);
+ unsigned long ready;
+ vm_fault_t ret;
+ int refs;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ refs = folio_ref_count(f->folio);
+ f->worker = kthread_run(mc16_rl_retry_writer, f, "mc16-resident-retry");
+ if (IS_ERR(f->worker)) {
+ f->worker = NULL;
+ KUNIT_FAIL(test, "could not start resident writer");
+ return;
+ }
+ get_task_struct(f->worker);
+ ready = wait_for_completion_timeout(&f->locked, msecs_to_jiffies(5000));
+ if (!ready) {
+ mc16_rl_join(f);
+ KUNIT_FAIL(test, "resident writer did not lock folio");
+ return;
+ }
+ ret = mc16_rl_fault(f, FAULT_FLAG_ALLOW_RETRY, false);
+ /* No test thread owns another task's folio or mmap lock on assertion. */
+ mc16_rl_join(f);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_RETRY);
+ KUNIT_EXPECT_TRUE(test, f->handled);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), refs);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_rl_read(f), f->orig));
+ mc16_rl_counts(f, 1, 0, 2);
+}
+#endif /* CONFIG_MEMCG && CONFIG_ADVISE_SYSCALLS */
+#endif /* _MM_MEMCOMP_TESTS_RESIDENT_TEST_H */
diff --git a/mm/memcomp/tests/rollback_test.h b/mm/memcomp/tests/rollback_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..f74c5a06961b45eb834f691986a0c5e73659c162
--- /dev/null
+++ b/mm/memcomp/tests/rollback_test.h
@@ -0,0 +1,163 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_ROLLBACK_TEST_H
+#define _MM_MEMCOMP_TESTS_ROLLBACK_TEST_H
+
+/* Real VM/PTE fixture; only allocation ownership uses the existing objcg model. */
+#if defined(MC16_FOLIO_TEST_ENABLED) && defined(CONFIG_MEMCOMP_DEBUG_STATS)
+#define MC16_ROLLBACK_TEST_ENABLED
+
+struct mc16_rb_counts {
+ s64 spans;
+ s64 ptes;
+ s64 rewalks;
+ s64 missed;
+ s64 pending;
+};
+
+static struct mc16_rb_counts mc16_rb_counts(void)
+{
+ return (struct mc16_rb_counts) {
+ .spans = atomic64_read(&memcomp_restore_spans),
+ .ptes = atomic64_read(&memcomp_restore_ptes),
+ .rewalks = atomic64_read(&memcomp_restore_rewalks),
+ .missed = atomic64_read(&memcomp_restore_missed_pages),
+ .pending = atomic64_read(&memcomp_pending_pages),
+ };
+}
+
+static bool mc16_rb_reserve(struct mc16_flt *f)
+{
+ bool reserved;
+
+ folio_lock(f->folio);
+ reserved = memcomp_reserve(f->folio);
+ folio_unlock(f->folio);
+ return reserved;
+}
+
+static void mc16_rb_rollback(struct mc16_flt *f)
+{
+ mmap_read_lock(f->mm);
+ folio_lock(f->folio);
+ memcomp_rollback_folio(f->folio);
+ folio_unlock(f->folio);
+ mmap_read_unlock(f->mm);
+}
+
+static void mc16_rb_expect_bytes(struct mc16_flt *f)
+{
+ unsigned int i;
+ void *bytes;
+
+ for (i = 0; i < f->nr; i++) {
+ bytes = kmap_local_folio(f->folio, i * PAGE_SIZE);
+ KUNIT_EXPECT_PTR_EQ(f->test, memchr_inv(bytes, 0x6b, PAGE_SIZE), NULL);
+ kunmap_local(bytes);
+ }
+}
+
+/* A fully restored span without a lookup guard must not start another walk. */
+static void memcomp_test_public_rollback_restored(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 0);
+ struct mc16_rb_counts before, after;
+ unsigned long token;
+ unsigned int first, second;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_rb_reserve(f));
+ token = swp_offset(f->folio->swap);
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, 0), 0);
+ mmap_read_lock(f->mm);
+ folio_lock(f->folio);
+ first = mc16_folio_restore_span(f->folio, token, false);
+ second = mc16_folio_restore_span(f->folio, token, false);
+ folio_unlock(f->folio);
+ mmap_read_unlock(f->mm);
+ KUNIT_ASSERT_EQ(test, first, 1U);
+ KUNIT_ASSERT_EQ(test, second, 0U);
+ before = mc16_rb_counts();
+ mc16_rb_rollback(f);
+ after = mc16_rb_counts();
+ KUNIT_EXPECT_EQ(test, after.spans, before.spans);
+ KUNIT_EXPECT_EQ(test, after.ptes, before.ptes);
+ KUNIT_EXPECT_EQ(test, after.rewalks, before.rewalks);
+ KUNIT_EXPECT_EQ(test, after.missed, before.missed);
+ KUNIT_EXPECT_EQ(test, after.pending, before.pending - 1);
+ KUNIT_EXPECT_EQ(test, f->folio->swap.val, 0UL);
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_present(token));
+ mc16_flt_expect_present(f, 0, f->addr);
+ mc16_rb_expect_bytes(f);
+}
+
+/* Temporary guards may request a bounded rewalk, but are not missed PTEs. */
+static void memcomp_test_public_rollback_lookup_only(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 1);
+ struct mc16_rb_counts before, after;
+ unsigned long token;
+ unsigned int i;
+ int pins;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ pins = atomic_read(&f->anon_vma->refcount);
+ KUNIT_ASSERT_TRUE(test, mc16_rb_reserve(f));
+ token = swp_offset(f->folio->swap);
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ before = mc16_rb_counts();
+ mc16_rb_rollback(f);
+ after = mc16_rb_counts();
+ KUNIT_EXPECT_EQ(test, after.spans, before.spans + 1);
+ KUNIT_EXPECT_EQ(test, after.rewalks, before.rewalks + 1);
+ KUNIT_EXPECT_EQ(test, after.ptes, before.ptes);
+ KUNIT_EXPECT_EQ(test, after.missed, before.missed);
+ /* Pending slots remain live while the two lookup guards are held. */
+ KUNIT_EXPECT_EQ(test, after.pending, before.pending);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->anon_vma->refcount), pins);
+ KUNIT_EXPECT_EQ(test, f->folio->swap.val, 0UL);
+ for (i = 0; i < f->nr; i++) {
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guards[i].slot->refs), 1U);
+ KUNIT_EXPECT_FALSE(test, mc16_flt_flags(&f->guards[i], false) & MC16_F_OWNER);
+ memcomp_entry_put_ref(&f->guards[i]);
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_present(token + i));
+ }
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_pending_pages), before.pending - 2);
+ mc16_rb_expect_bytes(f);
+}
+
+/* Both PTEs restore on the first pass; held guards cause exactly one rewalk. */
+static void memcomp_test_public_rollback_rewalk(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 1);
+ struct mc16_rb_counts before, after;
+ unsigned long token;
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_rb_reserve(f));
+ token = swp_offset(f->folio->swap);
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ for (i = 0; i < f->nr; i++)
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, i), 0);
+ before = mc16_rb_counts();
+ mc16_rb_rollback(f);
+ after = mc16_rb_counts();
+ KUNIT_EXPECT_EQ(test, after.spans, before.spans + 1);
+ KUNIT_EXPECT_EQ(test, after.ptes, before.ptes + 2);
+ KUNIT_EXPECT_EQ(test, after.rewalks, before.rewalks + 1);
+ KUNIT_EXPECT_EQ(test, after.missed, before.missed);
+ /* Pending slots remain live while the two lookup guards are held. */
+ KUNIT_EXPECT_EQ(test, after.pending, before.pending);
+ KUNIT_EXPECT_EQ(test, mc16_flt_rss(f, MM_SWAPENTS), f->swapents);
+ KUNIT_EXPECT_EQ(test, mc16_flt_rss(f, MM_ANONPAGES), f->anonpages + 2);
+ for (i = 0; i < f->nr; i++) {
+ mc16_flt_expect_present(f, i, f->addr + i * PAGE_SIZE);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guards[i].slot->refs), 1U);
+ memcomp_entry_put_ref(&f->guards[i]);
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_present(token + i));
+ }
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_pending_pages), before.pending - 2);
+ mc16_rb_expect_bytes(f);
+}
+#endif /* MC16_FOLIO_TEST_ENABLED && CONFIG_MEMCOMP_DEBUG_STATS */
+#endif /* _MM_MEMCOMP_TESTS_ROLLBACK_TEST_H */
diff --git a/mm/memcomp/tests/scan_test.h b/mm/memcomp/tests/scan_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..cf283f414d527005fdf63006e7638a0fa7dff79a
--- /dev/null
+++ b/mm/memcomp/tests/scan_test.h
@@ -0,0 +1,422 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_SCAN_TEST_H
+#define _MM_MEMCOMP_TESTS_SCAN_TEST_H
+
+/* Include after storage/refs fixtures and cache_store.h; register in main. */
+#include "../scan.h"
+
+#ifdef CONFIG_MEMCG
+struct mc16_test_scan {
+ struct mc16_test_refs *refs;
+ struct memcomp_ref guard;
+ struct memcomp_ref cache;
+ u32 markers[2];
+ bool active;
+};
+
+static void mc16_test_scan_cleanup(void *data)
+{
+ struct mc16_test_scan *f = data;
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(f->markers); i++)
+ if (f->markers[i])
+ xa_erase_irq(&mc16_clusters, f->markers[i]);
+ if (f->guard.slot)
+ mc16_ref_put(&f->guard);
+ if (f->cache.slot) {
+ if (f->active) {
+ mc16_ref_put(&f->cache);
+ } else {
+ mc16_release_resources(&f->cache);
+ mc16_retire_slot(&f->cache);
+ }
+ }
+}
+
+static struct mc16_test_scan *mc16_test_scan_begin(struct kunit *test)
+{
+ struct mc16_test_scan *f;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->refs = mc16_test_refs_begin(test);
+ if (!f->refs || kunit_add_action_or_reset(test,
+ mc16_test_scan_cleanup, f))
+ return NULL;
+ return f;
+}
+
+/* The real reservation transaction still publishes/charges the cluster. */
+static int mc16_test_scan_last_id(u32 *id)
+{
+ return __xa_alloc(&mc16_clusters, id, NULL,
+ XA_LIMIT(MC16_MAX_CLUSTER_ID, MC16_MAX_CLUSTER_ID),
+ GFP_NOWAIT | __GFP_NOWARN);
+}
+
+/* Reserve a real XArray zero entry, not a published refs-zero cache slot. */
+static int mc16_test_scan_marker(struct mc16_test_scan *f,
+ unsigned int which, u32 id)
+{
+ unsigned long flags;
+ u32 reserved;
+ int ret;
+
+ xa_lock_irqsave(&mc16_clusters, flags);
+ ret = __xa_alloc(&mc16_clusters, &reserved, NULL, XA_LIMIT(id, id),
+ GFP_NOWAIT | __GFP_NOWARN);
+ if (!ret)
+ f->markers[which] = reserved;
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ return ret;
+}
+
+static int mc16_test_scan_private(struct mc16_test_scan *f)
+{
+ const struct mc16_initial_payload initial = { .word = 0x5a5a };
+ struct mc16_test_storage *s = f->refs->storage;
+
+ return mc16_reserve_cache(&s->owner->model.objcg,
+ folio_nid(s->owner->folio), &initial, 1, &f->cache);
+}
+
+static void memcomp_test_scan_empty_budget(struct kunit *test)
+{
+ struct mc16_test_scan *f = mc16_test_scan_begin(test);
+ unsigned int scanned = 99;
+ u64 cursor = 0;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_EXPECT_EQ(test, mc16_scan_next(&cursor, 0, &f->guard,
+ &scanned), MC16_SCAN_BUDGET);
+ KUNIT_EXPECT_EQ(test, cursor, 0ULL);
+ KUNIT_EXPECT_EQ(test, scanned, 0U);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_END);
+ KUNIT_EXPECT_EQ(test, cursor, MC16_SCAN_END_CURSOR);
+ KUNIT_EXPECT_EQ(test, scanned, 0U);
+ cursor = U64_MAX;
+ KUNIT_EXPECT_EQ(test, mc16_scan_next(&cursor, 0, &f->guard,
+ &scanned), MC16_SCAN_END);
+ KUNIT_EXPECT_EQ(test, cursor, MC16_SCAN_END_CURSOR);
+}
+
+static void memcomp_test_scan_reserved_markers(struct kunit *test)
+{
+ struct mc16_test_scan *f = mc16_test_scan_begin(test);
+ unsigned int scanned, i;
+ u64 cursor = 0;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_scan_marker(f, 0, 1), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_scan_marker(f, 1, 2), 0);
+ /* A real later cluster distinguishes yield from hidden-marker skip. */
+ kunit_activate_static_stub(test, mc16_private_id_alloc,
+ mc16_test_scan_last_id);
+ ret = mc16_test_scan_private(f);
+ kunit_deactivate_static_stub(test, mc16_private_id_alloc);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ mc16_cache_publish(&f->cache, 0);
+ f->active = true;
+ for (i = 0; i < ARRAY_SIZE(f->markers); i++) {
+ XA_STATE(xas, &mc16_clusters, f->markers[i]);
+ void *entry;
+
+ rcu_read_lock();
+ entry = xas_load(&xas);
+ rcu_read_unlock();
+ KUNIT_ASSERT_TRUE(test, xa_is_zero(entry));
+ KUNIT_ASSERT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_BUDGET);
+ KUNIT_EXPECT_EQ(test, scanned, 1U);
+ KUNIT_EXPECT_EQ(test, cursor,
+ ((u64)f->markers[i] + 1) << MC16_CLUSTER_SHIFT);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_NULL(test, f->guard.cluster);
+ }
+ KUNIT_ASSERT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_GUARD);
+ KUNIT_EXPECT_EQ(test, scanned, 1U);
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.slot, f->cache.slot);
+ mc16_ref_put(&f->guard);
+ KUNIT_EXPECT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_END);
+ KUNIT_EXPECT_EQ(test, scanned, 0U);
+}
+
+static void memcomp_test_scan_reserved_last(struct kunit *test)
+{
+ struct mc16_test_scan *f = mc16_test_scan_begin(test);
+ unsigned int scanned;
+ u64 cursor = U32_MAX;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_scan_marker(f, 0,
+ MC16_MAX_CLUSTER_ID), 0);
+ KUNIT_EXPECT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_END);
+ KUNIT_EXPECT_EQ(test, scanned, 1U);
+ KUNIT_EXPECT_EQ(test, cursor, MC16_SCAN_END_CURSOR);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+}
+
+static void memcomp_test_scan_first(struct kunit *test)
+{
+ struct mc16_test_scan *f = mc16_test_scan_begin(test);
+ struct memcomp_ref *owner;
+ enum mc16_scan_result result;
+ unsigned int scanned;
+ u64 cursor = 0;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f->refs, 1), 0);
+ owner = &f->refs->storage->refs[0];
+ KUNIT_ASSERT_EQ(test, owner->token, (u32)MC16_CLUSTER_SLOTS);
+ /* Already holding T checks that scanning does not reacquire it. */
+ mutex_lock(memcomp_lock_of(owner->token));
+ result = mc16_scan_next(&cursor, 1, &f->guard, &scanned);
+ mutex_unlock(memcomp_lock_of(owner->token));
+ KUNIT_ASSERT_EQ(test, result, MC16_SCAN_GUARD);
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.slot, owner->slot);
+ KUNIT_EXPECT_EQ(test, scanned, 1U);
+ KUNIT_EXPECT_EQ(test, cursor, (u64)owner->token + 1);
+ KUNIT_EXPECT_EQ(test, refcount_read(&owner->slot->refs), 2U);
+ mc16_ref_put(&f->guard);
+ KUNIT_EXPECT_EQ(test, refcount_read(&owner->slot->refs), 1U);
+ KUNIT_EXPECT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_END);
+ KUNIT_EXPECT_EQ(test, scanned, 0U);
+}
+
+static void memcomp_test_scan_last_guard(struct kunit *test)
+{
+ struct mc16_test_scan *f = mc16_test_scan_begin(test);
+ unsigned int scanned;
+ u64 cursor = 0;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f->refs, 1), 0);
+ KUNIT_ASSERT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_GUARD);
+ mc16_ref_put(&f->refs->storage->refs[0]);
+ KUNIT_EXPECT_FALSE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guard.slot->refs), 1U);
+ mc16_ref_put(&f->guard);
+ /* The guard is gone, but the appendable current header is retained. */
+ KUNIT_EXPECT_FALSE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_END);
+ KUNIT_EXPECT_EQ(test, scanned, 0U);
+}
+
+static void memcomp_test_scan_alignment(struct kunit *test)
+{
+ struct mc16_test_scan *f = mc16_test_scan_begin(test);
+ struct mc16_test_storage *s;
+ unsigned int scanned;
+ u64 cursor, hole;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f->refs, 1), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f->refs, 2), 0);
+ hole = (u64)s->refs[0].token + 1;
+ cursor = hole;
+ KUNIT_EXPECT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_BUDGET);
+ KUNIT_EXPECT_EQ(test, scanned, 1U);
+ KUNIT_EXPECT_EQ(test, cursor, hole + 1);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_EXPECT_EQ(test, mc16_scan_next(&cursor, 0, &f->guard,
+ &scanned), MC16_SCAN_BUDGET);
+ KUNIT_EXPECT_EQ(test, scanned, 0U);
+ KUNIT_EXPECT_EQ(test, cursor, hole + 1);
+ KUNIT_ASSERT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_GUARD);
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.slot, s->refs[1].slot);
+ KUNIT_EXPECT_EQ(test, scanned, 1U);
+}
+
+static void memcomp_test_scan_refs_zero(struct kunit *test)
+{
+ struct mc16_test_scan *f = mc16_test_scan_begin(test);
+ struct mc16_test_storage *s;
+ unsigned int scanned;
+ u64 cursor;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f->refs, 2), 0);
+ cursor = s->refs[0].token;
+ mc16_ref_put(&s->refs[0]);
+ KUNIT_EXPECT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_BUDGET);
+ KUNIT_EXPECT_EQ(test, scanned, 1U);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ KUNIT_ASSERT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_GUARD);
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.slot, s->refs[1].slot);
+}
+
+static void memcomp_test_scan_segment_retired(struct kunit *test)
+{
+ struct mc16_test_scan *f = mc16_test_scan_begin(test);
+ struct mc16_test_storage *s;
+ unsigned int scanned, i;
+ u64 cursor, base;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ for (i = 0; i <= MC16_SEGMENT_SLOTS; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f->refs, 1), 0);
+ base = s->refs[0].token;
+ s->probe.hold_segments = true;
+ for (i = 0; i < MC16_SEGMENT_SLOTS; i++)
+ mc16_ref_put(&s->refs[i]);
+ KUNIT_ASSERT_NULL(test,
+ s->refs[MC16_SEGMENT_SLOTS].cluster->segments[0]);
+ cursor = base;
+ KUNIT_EXPECT_EQ(test, mc16_scan_next(&cursor, 3, &f->guard,
+ &scanned), MC16_SCAN_BUDGET);
+ KUNIT_EXPECT_EQ(test, scanned, 3U);
+ KUNIT_EXPECT_EQ(test, cursor, base + 3);
+ cursor = base + MC16_SEGMENT_SLOTS - 1;
+ KUNIT_ASSERT_EQ(test, mc16_scan_next(&cursor, 2, &f->guard,
+ &scanned), MC16_SCAN_GUARD);
+ KUNIT_EXPECT_EQ(test, scanned, 2U);
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.slot,
+ s->refs[MC16_SEGMENT_SLOTS].slot);
+ mc16_ref_put(&f->guard);
+ mc16_ref_put(&s->refs[MC16_SEGMENT_SLOTS]);
+ cursor = base;
+ /* The empty appendable header still consumes the scan's ID budget. */
+ KUNIT_EXPECT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_BUDGET);
+ KUNIT_EXPECT_EQ(test, scanned, 1U);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ mc16_test_reclaim_retained();
+ cursor = base;
+ KUNIT_EXPECT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_END);
+ KUNIT_EXPECT_EQ(test, scanned, 0U);
+ /* Cleanup drains both real RCU retirement paths. */
+}
+
+static void memcomp_test_scan_private(struct kunit *test)
+{
+ struct mc16_test_scan *f = mc16_test_scan_begin(test);
+ unsigned int scanned;
+ u64 cursor;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_scan_private(f), 0);
+ cursor = f->cache.token;
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->cache.slot->refs), 0U);
+ KUNIT_EXPECT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_BUDGET);
+ KUNIT_EXPECT_EQ(test, scanned, 1U);
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ mc16_cache_publish(&f->cache, 0);
+ f->active = true;
+ cursor = f->cache.token;
+ KUNIT_ASSERT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_GUARD);
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.slot, f->cache.slot);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->cache.slot->refs), 2U);
+}
+
+static void memcomp_test_scan_sparse(struct kunit *test)
+{
+ struct mc16_test_scan *f = mc16_test_scan_begin(test);
+ unsigned int scanned;
+ u64 cursor;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f->refs, 1), 0);
+ kunit_activate_static_stub(test, mc16_private_id_alloc,
+ mc16_test_scan_last_id);
+ ret = mc16_test_scan_private(f);
+ kunit_deactivate_static_stub(test, mc16_private_id_alloc);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ mc16_cache_publish(&f->cache, 0);
+ f->active = true;
+ KUNIT_ASSERT_EQ(test, f->cache.cluster->id, (u32)MC16_MAX_CLUSTER_ID);
+ cursor = (u64)f->refs->storage->refs[0].token + 1;
+ KUNIT_ASSERT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_GUARD);
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.slot, f->cache.slot);
+ KUNIT_EXPECT_EQ(test, scanned, 1U);
+}
+
+static void memcomp_test_scan_u32_max(struct kunit *test)
+{
+ struct mc16_test_scan *f = mc16_test_scan_begin(test);
+ struct mc16_test_storage *s;
+ unsigned int scanned, i;
+ u64 cursor = U32_MAX;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ kunit_activate_static_stub(test, mc16_private_id_alloc,
+ mc16_test_scan_last_id);
+ ret = mc16_test_refs_reserve(f->refs, 1);
+ kunit_deactivate_static_stub(test, mc16_private_id_alloc);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ for (i = 1; i < MC16_CLUSTER_SLOTS; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f->refs, 1), 0);
+ KUNIT_ASSERT_EQ(test, s->refs[MC16_CLUSTER_SLOTS - 1].token, U32_MAX);
+ KUNIT_ASSERT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_GUARD);
+ KUNIT_EXPECT_EQ(test, scanned, 1U);
+ KUNIT_EXPECT_EQ(test, cursor, MC16_SCAN_END_CURSOR);
+ KUNIT_EXPECT_EQ(test, f->guard.token, U32_MAX);
+ mc16_ref_put(&f->guard);
+ KUNIT_EXPECT_EQ(test, mc16_scan_next(&cursor, 1, &f->guard,
+ &scanned), MC16_SCAN_END);
+ KUNIT_EXPECT_EQ(test, scanned, 0U);
+}
+
+static void memcomp_test_scan_w_relookup(struct kunit *test)
+{
+ struct mc16_test_scan *f = mc16_test_scan_begin(test);
+ struct mc16_test_storage *s;
+ enum mc16_scan_result result;
+ unsigned int scanned;
+ u64 cursor = 0;
+ u32 token;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f->refs, 2), 0);
+ token = s->refs[0].token;
+ mutex_lock(&memcomp_writeback_lock);
+ result = mc16_scan_next(&cursor, 1, &f->guard, &scanned);
+ if (f->guard.slot)
+ mc16_ref_put(&f->guard);
+ mutex_unlock(&memcomp_writeback_lock);
+ KUNIT_ASSERT_EQ(test, result, MC16_SCAN_GUARD);
+ KUNIT_ASSERT_EQ(test, cursor, (u64)token + 1);
+ mc16_ref_put(&s->refs[0]);
+ mc16_ref_put(&s->refs[1]);
+ /* Explicit pressure enables numeric ID reuse for this relookup case. */
+ KUNIT_EXPECT_FALSE(test, xa_empty(&mc16_clusters));
+ mc16_test_reclaim_retained();
+ KUNIT_ASSERT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f->refs, 2), 0);
+ KUNIT_ASSERT_EQ(test, s->refs[2].token, token);
+ mutex_lock(&memcomp_writeback_lock);
+ result = mc16_scan_next(&cursor, 1, &f->guard, &scanned);
+ mutex_unlock(&memcomp_writeback_lock);
+ KUNIT_ASSERT_EQ(test, result, MC16_SCAN_GUARD);
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.slot, s->refs[3].slot);
+ KUNIT_EXPECT_EQ(test, scanned, 1U);
+}
+
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_SCAN_TEST_H */
diff --git a/mm/memcomp/tests/shared_test.h b/mm/memcomp/tests/shared_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..5b68a9af57b39139274fac75a074e924da5a29d2
--- /dev/null
+++ b/mm/memcomp/tests/shared_test.h
@@ -0,0 +1,3477 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_SHARED_TEST_H
+#define _MM_MEMCOMP_TESTS_SHARED_TEST_H
+
+#include <linux/sched/mm.h>
+
+/* Cleanup runs in a kthread with no mm; madvise needs the target mm. */
+#ifdef CONFIG_ADVISE_SYSCALLS
+static int mc16_test_dontneed(struct mm_struct *mm, unsigned long start,
+ size_t len)
+{
+ bool attach = current->mm != mm;
+ int ret;
+
+ if (attach)
+ kthread_use_mm(mm);
+ ret = do_madvise(mm, start, len, MADV_DONTNEED);
+ if (attach)
+ kthread_unuse_mm(mm);
+ return ret;
+}
+#endif
+
+/* Test-only shared services. No entry80 registry, object or fallback. */
+#define MEMCOMP_TEST_HUF_CACHE_PAGES 130
+#define MEMCOMP_TEST_USER_PATTERN 0x6d656d636f6d7072UL
+
+struct memcomp_test_real_page_record {
+ u64 seq;
+ u64 jiffies_snapshot;
+ unsigned long totalram;
+ unsigned int online_cpus;
+ u32 mix;
+ char comm[TASK_COMM_LEN];
+ struct new_utsname uts;
+ char banner[128];
+};
+
+struct memcomp_test_zstd_page_case {
+ const char *name;
+ void (*fill)(u8 *buf);
+ unsigned int max_length;
+ bool allow_incompressible;
+};
+
+/* Defined in storage_test.h; async teardown is included before that header. */
+static void mc16_test_reclaim_retained(void);
+
+#ifdef CONFIG_MEMCG
+/* Identity-only objcg model. Payload/storage allocations remain real.
+ * Descendant slot actions release all slots before this parent action.
+ */
+struct memcomp_test_objcg_fixture {
+ struct memcomp_kunit_objcg model;
+ struct kunit *test;
+ struct folio *folio;
+ unsigned int nr_pages;
+};
+
+static void memcomp_test_objcg_cleanup(void *data)
+{
+ struct memcomp_test_objcg_fixture *f = data;
+
+ /* All descendant slots are gone; reclaim before unregistering objcg. */
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_EQ(f->test, atomic_long_read(&f->model.charged), 0L);
+ KUNIT_EXPECT_EQ(f->test, atomic_read(&f->model.put_calls),
+ atomic_read(&f->model.get_calls));
+ if (rcu_access_pointer(memcomp_kunit_objcg) == &f->model) {
+ rcu_assign_pointer(memcomp_kunit_objcg, NULL);
+ synchronize_rcu();
+ }
+ if (f->folio) {
+ folio_unlock(f->folio);
+ folio_put(f->folio);
+ f->folio = NULL;
+ }
+}
+
+static struct memcomp_test_objcg_fixture *
+memcomp_test_objcg_begin(struct kunit *test, unsigned int nr_pages)
+{
+ struct memcomp_test_objcg_fixture *f;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return NULL;
+ }
+ rcu_barrier();
+ if (rcu_access_pointer(memcomp_kunit_objcg)) {
+ KUNIT_FAIL(test, "objcg model already registered");
+ return NULL;
+ }
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->test = test;
+ f->nr_pages = nr_pages;
+ f->model.task = current;
+ f->model.error = -ENOMEM;
+ if (kunit_add_action_or_reset(test, memcomp_test_objcg_cleanup, f))
+ return NULL;
+ f->folio = folio_alloc(GFP_KERNEL, nr_pages == 2 ? 1 : 0);
+ if (!f->folio)
+ return NULL;
+ folio_lock(f->folio);
+ f->model.folio = f->folio;
+ rcu_assign_pointer(memcomp_kunit_objcg, &f->model);
+ return f;
+}
+#endif /* CONFIG_MEMCG */
+
+/* Guard owns the slot; only T permits inspecting its current record. */
+static bool memcomp_test_head_holds_anon_vma(unsigned long token)
+{
+ struct memcomp_ref ref = { };
+ struct mc16_view view;
+ bool held = false;
+
+ if (!memcomp_entry_get_ref(token, &ref))
+ return false;
+ mutex_lock(memcomp_lock_of(ref.token));
+ if (mc16_view_locked(&ref, &view) && view.record)
+ held = view.record->record_flags & MC16_RECORD_ANON_VMA;
+ mutex_unlock(memcomp_lock_of(ref.token));
+ memcomp_entry_put_ref(&ref);
+ return held;
+}
+
+static bool memcomp_test_store_after_unmap(struct folio *folio)
+{
+ struct memcomp_reclaim_ctx *ctx;
+ bool ret;
+
+ ctx = memcomp_get_comp_ctx(MEMCOMP_PAYLOAD_MUST_SUCCEED, NULL,
+ MEMCOMP_RECLAIM_DIRECT);
+ if (!ctx)
+ return false;
+ ret = memcomp_store_after_unmap(folio, &ctx);
+ WARN_ON_ONCE(ctx);
+ return ret;
+}
+
+static void memcomp_test_initial_compression_placement(struct kunit *test)
+{
+ const enum memcomp_placement_stat compress_stat =
+ MEMCOMP_PLACEMENT_STAT_COMPRESS;
+ const enum memcomp_placement_stat fallback_stat =
+ MEMCOMP_PLACEMENT_STAT_NORMAL_SWAP_FALLBACK;
+ u64 compress;
+ u64 fallback;
+ u64 observed;
+
+ compress = memcomp_placement_stat_read(compress_stat);
+ fallback = memcomp_placement_stat_read(fallback_stat);
+
+ memcomp_count_initial_compress();
+ memcomp_count_normal_swap_fallback();
+
+ observed = memcomp_placement_stat_read(compress_stat);
+ KUNIT_EXPECT_EQ(test, observed, compress + 1);
+ observed = memcomp_placement_stat_read(fallback_stat);
+ KUNIT_EXPECT_EQ(test, observed, fallback + 1);
+}
+
+static void memcomp_test_writeback_pressure_scope(struct kunit *test)
+{
+ const int nid = 0;
+ int scopes;
+ int observed;
+
+ scopes = atomic_read(&memcomp_writeback_scopes[nid]);
+ memcomp_reclaim_writeback_begin(-1, 0);
+ memcomp_reclaim_writeback_end(-1);
+ observed = atomic_read(&memcomp_writeback_scopes[nid]);
+ KUNIT_EXPECT_EQ(test, observed, scopes);
+
+ memcomp_reclaim_writeback_begin(nid, 0);
+ observed = atomic_read(&memcomp_writeback_scopes[nid]);
+ KUNIT_EXPECT_EQ(test, observed, scopes + 1);
+ KUNIT_EXPECT_TRUE(test, memcomp_writeback_scope_active(nid));
+
+ memcomp_reclaim_writeback_begin(nid, 0);
+ observed = atomic_read(&memcomp_writeback_scopes[nid]);
+ KUNIT_EXPECT_EQ(test, observed, scopes + 2);
+ memcomp_reclaim_writeback_end(nid);
+ observed = atomic_read(&memcomp_writeback_scopes[nid]);
+ KUNIT_EXPECT_EQ(test, observed, scopes + 1);
+ memcomp_reclaim_writeback_end(nid);
+ observed = atomic_read(&memcomp_writeback_scopes[nid]);
+ KUNIT_EXPECT_EQ(test, observed, scopes);
+}
+
+static void memcomp_test_writeback_drain_recovery_applied(struct kunit *test)
+{
+ const int nid = 0;
+ unsigned int before;
+ unsigned int seeded;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+ /*
+ * The decay only fires for a node that has climbed back above its low
+ * watermark. A test host under real pressure would take the other
+ * branch, so check rather than assume.
+ */
+ if (memcomp_writeback_node_below_low(nid, 0)) {
+ kunit_skip(test, "node 0 is below its low watermark");
+ return;
+ }
+ KUNIT_ASSERT_FALSE(test, memcomp_writeback_scope_active(nid));
+
+ before = READ_ONCE(memcomp_writeback_drain[nid]);
+ seeded = MEMCOMP_WRITEBACK_DRAIN_MAX;
+
+ memcomp_reclaim_writeback_begin(nid, 0);
+ WRITE_ONCE(memcomp_writeback_drain[nid], seeded);
+ WRITE_ONCE(memcomp_writeback_feedback_valid[nid], true);
+ memcomp_reclaim_writeback_end(nid);
+
+ /* The last scope closing is what applies it, and exactly once. */
+ KUNIT_EXPECT_EQ(test, READ_ONCE(memcomp_writeback_drain[nid]),
+ memcomp_writeback_drain_recover(seeded));
+ /* A decayed share is not a measured one, so the sample is dropped. */
+ KUNIT_EXPECT_FALSE(test,
+ READ_ONCE(memcomp_writeback_feedback_valid[nid]));
+
+ /*
+ * And a nested scope leaves it alone until the outermost one ends.
+ * Seeded inside the outer scope: its grant runs on a pool below the
+ * level gate, which drops the share by design.
+ */
+ memcomp_reclaim_writeback_begin(nid, 0);
+ WRITE_ONCE(memcomp_writeback_drain[nid], seeded);
+ memcomp_reclaim_writeback_begin(nid, 0);
+ memcomp_reclaim_writeback_end(nid);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(memcomp_writeback_drain[nid]),
+ seeded);
+ memcomp_reclaim_writeback_end(nid);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(memcomp_writeback_drain[nid]),
+ memcomp_writeback_drain_recover(seeded));
+
+ WRITE_ONCE(memcomp_writeback_drain[nid], before);
+ WRITE_ONCE(memcomp_writeback_feedback_valid[nid], false);
+}
+
+static void memcomp_test_writeback_feedback(struct kunit *test)
+{
+ enum memcomp_writeback_feedback feedback;
+ const u64 sample = MEMCOMP_WRITEBACK_FEEDBACK_MIN_BYTES;
+ const u64 half_sample = sample / 2;
+
+ /* No source sample is never converted into a backing-I/O request. */
+ feedback = memcomp_writeback_feedback_values(0, 0, 0, PAGE_SIZE);
+ KUNIT_EXPECT_EQ(test, feedback, MEMCOMP_WRITEBACK_PENDING);
+
+ feedback = memcomp_writeback_feedback_values(0, 0, PAGE_SIZE, 0);
+ KUNIT_EXPECT_EQ(test, feedback, MEMCOMP_WRITEBACK_PENDING);
+
+ feedback = memcomp_writeback_feedback_values(0, 0, sample, 0);
+ KUNIT_EXPECT_EQ(test, feedback, MEMCOMP_WRITEBACK_EFFECTIVE);
+
+ feedback = memcomp_writeback_feedback_values(0, 0, sample, half_sample);
+ KUNIT_EXPECT_EQ(test, feedback, MEMCOMP_WRITEBACK_EFFECTIVE);
+
+ feedback = memcomp_writeback_feedback_values(0, 0, sample, sample);
+ KUNIT_EXPECT_EQ(test, feedback, MEMCOMP_WRITEBACK_INEFFICIENT);
+}
+
+static void memcomp_test_writeback_budget_values(struct kunit *test)
+{
+ const unsigned int over = MEMCOMP_WRITEBACK_DRAIN_DEADBAND +
+ MEMCOMP_WRITEBACK_FRAC_ONE / 64;
+ const unsigned long cold = 4096UL;
+
+ /* No share, no writeback, however many payloads are cold. */
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_budget_values(cold, 0), 0UL);
+
+ /*
+ * Nothing is granted until the share clears the deadband, so the few
+ * scopes a boot or a short allocation burst opens write nothing back.
+ */
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_budget_values(cold,
+ MEMCOMP_WRITEBACK_DRAIN_DEADBAND),
+ 0UL);
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_budget_values(cold,
+ MEMCOMP_WRITEBACK_DRAIN_DEADBAND - 1),
+ 0UL);
+ KUNIT_EXPECT_GT(test,
+ memcomp_writeback_budget_values(cold,
+ MEMCOMP_WRITEBACK_DRAIN_DEADBAND + 1),
+ 0UL);
+
+ /*
+ * The ceiling sits a deadband above half so the top of the ramp still
+ * grants half of the cold payloads.
+ */
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_budget_values(cold,
+ MEMCOMP_WRITEBACK_DRAIN_MAX),
+ cold / 2);
+
+ /* Above the deadband the share is a fraction of the cold payloads. */
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_budget_values(cold, over),
+ cold / 64);
+
+ /*
+ * Past the deadband a small share must not round down to "never write
+ * back" while a whole batch of cold payloads is still in the pool.
+ */
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_budget_values(MEMCOMP_WRITEBACK_BATCH,
+ over),
+ 1UL);
+
+ /* Below one batch there is nothing worth draining. */
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_budget_values(MEMCOMP_WRITEBACK_BATCH - 1,
+ over),
+ 0UL);
+}
+
+static void memcomp_test_writeback_level_floor(struct kunit *test)
+{
+ const u64 total = 15ULL << 30;
+ const u64 target = total * MEMCOMP_WRITEBACK_LEVEL_PERMILLE / 1000;
+ const unsigned long cold = 4096UL;
+ unsigned int floor;
+ u64 pool;
+ int i;
+
+ /* At or under the level, the pool keeps everything. */
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_level_floor_values(target, total),
+ 0U);
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_level_floor_values(target / 2, total),
+ 0U);
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_level_floor_values(0, total), 0U);
+
+ /* Just over the line asks for a little, not for half. */
+ floor = memcomp_writeback_level_floor_values(target + target / 64,
+ total);
+ KUNIT_EXPECT_GT(test, memcomp_writeback_budget_values(cold, floor),
+ 0UL);
+ KUNIT_EXPECT_LT(test, memcomp_writeback_budget_values(cold, floor),
+ cold / 16);
+
+ /* Far over the line is capped at half, so one scope cannot empty it. */
+ floor = memcomp_writeback_level_floor_values(target * 100, total);
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_budget_values(cold, floor),
+ cold / 2);
+
+ /*
+ * Monotone in the footprint below the cap. The excess fraction
+ * reaches half at twice the target, so anything from there up is
+ * saturated and equal by design.
+ */
+ KUNIT_EXPECT_LT(test,
+ memcomp_writeback_level_floor_values(target + target / 8,
+ total),
+ memcomp_writeback_level_floor_values(target + target / 2,
+ total));
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_level_floor_values(target * 2, total),
+ memcomp_writeback_level_floor_values(target * 4, total));
+
+ /*
+ * The error closes. Draining the asked-for share removes about that
+ * share of the footprint, so iterating has to converge on the target
+ * and then stop - this is the property whose absence drained 99% of
+ * the pool into swap.
+ */
+ pool = target * 8;
+ for (i = 0; i < 64; i++) {
+ floor = memcomp_writeback_level_floor_values(pool, total);
+ if (!floor)
+ break;
+ pool -= div64_u64(pool * (floor - MEMCOMP_WRITEBACK_DRAIN_DEADBAND),
+ MEMCOMP_WRITEBACK_FRAC_ONE);
+ }
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_level_floor_values(pool, total), 0U);
+ KUNIT_EXPECT_GT(test, pool, target / 2);
+ KUNIT_EXPECT_LE(test, pool, target);
+
+ /* Unknown total RAM leaves the loop in sole charge rather than guessing. */
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_level_floor_values(target * 4, 0),
+ 0U);
+
+ /* Scale-free: the same ratio gives the same share on any machine. */
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_level_floor_values(target * 4, total),
+ memcomp_writeback_level_floor_values(target, total / 4));
+}
+
+static void memcomp_test_writeback_gate_values(struct kunit *test)
+{
+ const unsigned long total = (15UL << 30) >> PAGE_SHIFT;
+ const unsigned long start = total * MEMCOMP_WRITEBACK_GATE_START_PERMILLE / 1000;
+ const unsigned long stop = total * MEMCOMP_WRITEBACK_GATE_STOP_PERMILLE / 1000;
+ const u64 total_bytes = (u64)total << PAGE_SHIFT;
+ const u64 fp_start = total_bytes * MEMCOMP_WRITEBACK_GATE_FP_START_PERMILLE / 1000;
+ const u64 fp_stop = total_bytes * MEMCOMP_WRITEBACK_GATE_FP_STOP_PERMILLE / 1000;
+ /* A 3.5:1 pool, whose footprint never reaches the backstop. */
+ u64 fp = ((u64)start << PAGE_SHIFT) * 2 / 7;
+ bool open = false;
+ unsigned long excess;
+
+ KUNIT_ASSERT_LT(test, fp, fp_stop);
+
+ /* Just under the start point writes nothing, however often asked. */
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_gate_values(&open, start - 1,
+ fp, total), 0UL);
+ KUNIT_EXPECT_FALSE(test, open);
+
+ /* At the start point it opens, and owes what is above the close point. */
+ excess = memcomp_writeback_gate_values(&open, start, fp, total);
+ KUNIT_EXPECT_TRUE(test, open);
+ KUNIT_EXPECT_EQ(test, excess, start - stop);
+
+ /* Inside the band it stays open: that is the hysteresis. */
+ excess = memcomp_writeback_gate_values(&open, stop + 1, fp, total);
+ KUNIT_EXPECT_TRUE(test, open);
+ KUNIT_EXPECT_EQ(test, excess, 1UL);
+
+ /* At the close point it closes and owes nothing. */
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_gate_values(&open, stop, fp,
+ total), 0UL);
+ KUNIT_EXPECT_FALSE(test, open);
+
+ /* And a closed gate inside the band stays closed. */
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_gate_values(&open, start - 1,
+ fp, total), 0UL);
+ KUNIT_EXPECT_FALSE(test, open);
+
+ /*
+ * Incompressible data: the footprint reaches a quarter of RAM long
+ * before the entries reach two thirds, and the backstop opens it.
+ * What it owes is the footprint's excess share of the entries.
+ */
+ excess = memcomp_writeback_gate_values(&open, stop / 2, fp_start,
+ total);
+ KUNIT_EXPECT_TRUE(test, open);
+ KUNIT_EXPECT_EQ(test, excess,
+ (unsigned long)div64_u64((u64)(stop / 2) *
+ (fp_start - fp_stop), fp_start));
+ /* It closes once both terms are back under their close points. */
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_gate_values(&open, stop / 2,
+ fp_stop, total), 0UL);
+ KUNIT_EXPECT_FALSE(test, open);
+
+ /* No RAM and no pool are both no reason to write anything. */
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_gate_values(&open, start, fp,
+ 0), 0UL);
+ KUNIT_EXPECT_FALSE(test, open);
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_gate_values(&open, 0,
+ fp_start, total), 0UL);
+ KUNIT_EXPECT_FALSE(test, open);
+
+ /* Scale-free: the same fractions on any machine. */
+ open = false;
+ excess = memcomp_writeback_gate_values(&open, start / 4 - 1, fp / 4,
+ total / 4);
+ KUNIT_EXPECT_EQ(test, excess, 0UL);
+ KUNIT_EXPECT_FALSE(test, open);
+}
+
+static void memcomp_test_writeback_drain_loop(struct kunit *test)
+{
+ unsigned int drain;
+ unsigned int prev;
+ unsigned int i;
+
+ /*
+ * A pool that keeps growing under pressure raises the share one step
+ * per scope, monotonically, until it reaches the ceiling and stops.
+ */
+ drain = 0;
+ for (i = 0; i < 256; i++) {
+ prev = drain;
+ drain = memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_INEFFICIENT,
+ false);
+ KUNIT_EXPECT_GE(test, drain, prev);
+ KUNIT_EXPECT_LE(test, drain, MEMCOMP_WRITEBACK_DRAIN_MAX);
+ if (i == 0)
+ KUNIT_EXPECT_EQ(test, drain,
+ MEMCOMP_WRITEBACK_DRAIN_STEP);
+ }
+ KUNIT_EXPECT_EQ(test, drain, MEMCOMP_WRITEBACK_DRAIN_MAX);
+
+ /* Nothing raises it past the ceiling. */
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_drain_next(MEMCOMP_WRITEBACK_DRAIN_MAX,
+ MEMCOMP_WRITEBACK_INEFFICIENT,
+ false),
+ MEMCOMP_WRITEBACK_DRAIN_MAX);
+
+ /*
+ * A shrinking pool is the loop's evidence that the share was enough,
+ * and gives a quarter of it back - reaching zero, so a pool that stays
+ * under control stops being drained entirely.
+ */
+ drain = MEMCOMP_WRITEBACK_DRAIN_MAX;
+ for (i = 0; i < 128 && drain; i++) {
+ prev = drain;
+ drain = memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_PENDING,
+ true);
+ KUNIT_EXPECT_LT(test, drain, prev);
+ }
+ KUNIT_EXPECT_EQ(test, drain, 0U);
+
+ /* Having shrunk the pool gives back a quarter, not a half. */
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_drain_next(MEMCOMP_WRITEBACK_DRAIN_MAX,
+ MEMCOMP_WRITEBACK_PENDING,
+ true),
+ MEMCOMP_WRITEBACK_DRAIN_MAX -
+ MEMCOMP_WRITEBACK_DRAIN_MAX / 4);
+
+ /* The pool yielding is what lowers the share, whatever the sample. */
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_drain_next(MEMCOMP_WRITEBACK_DRAIN_MAX,
+ MEMCOMP_WRITEBACK_EFFECTIVE,
+ true),
+ memcomp_writeback_drain_next(MEMCOMP_WRITEBACK_DRAIN_MAX,
+ MEMCOMP_WRITEBACK_PENDING,
+ true));
+
+ /*
+ * Net-positive compression only slows the rise. It reports so at any
+ * ratio better than 2:1 - the normal case for a compressed pool - so
+ * letting it decay the share pinned the loop at zero for exactly the
+ * workloads that grow the pool.
+ */
+ drain = MEMCOMP_WRITEBACK_DRAIN_STEP;
+ KUNIT_EXPECT_GT(test,
+ memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_EFFECTIVE,
+ false),
+ drain);
+ KUNIT_EXPECT_LT(test,
+ memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_EFFECTIVE,
+ false),
+ memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_INEFFICIENT,
+ false));
+
+ /*
+ * A workload storing pages at a healthy ratio while the pool grows is
+ * the case that used to pin the share at zero. It must wind up.
+ */
+ drain = 0;
+ for (i = 0; i < 16; i++)
+ drain = memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_EFFECTIVE,
+ false);
+ KUNIT_EXPECT_GT(test, drain, 4 * MEMCOMP_WRITEBACK_DRAIN_STEP);
+ KUNIT_EXPECT_GT(test,
+ memcomp_writeback_budget_values(4096UL, drain), 0UL);
+
+ /*
+ * The loop cannot latch at zero: while the pool keeps growing under
+ * pressure, every scope raises the share again. This is what keeps
+ * "conservative" from becoming "never writes back". One scope still
+ * grants nothing - it has not cleared the deadband - but sustained
+ * pressure does.
+ */
+ drain = 0;
+ drain = memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_INEFFICIENT,
+ false);
+ KUNIT_EXPECT_GT(test, drain, 0U);
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_budget_values(4096UL, drain), 0UL);
+ for (i = 0; i < 8; i++)
+ drain = memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_INEFFICIENT,
+ false);
+ KUNIT_EXPECT_GT(test,
+ memcomp_writeback_budget_values(4096UL, drain), 0UL);
+
+ /*
+ * And it cannot run away either: a pool that yields to every share it
+ * is given settles into a bounded cycle instead of climbing to the
+ * ceiling. Rising by a step and giving a quarter back puts that
+ * steady state at about four steps - twice where halving left it,
+ * which was low enough to barely drain a pool that keeps refilling.
+ */
+ drain = 0;
+ for (i = 0; i < 128; i++) {
+ drain = memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_INEFFICIENT,
+ false);
+ drain = memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_PENDING,
+ true);
+ }
+ KUNIT_EXPECT_GT(test, drain, 2 * MEMCOMP_WRITEBACK_DRAIN_STEP);
+ KUNIT_EXPECT_LT(test, drain, 5 * MEMCOMP_WRITEBACK_DRAIN_STEP);
+ KUNIT_EXPECT_LT(test, drain, MEMCOMP_WRITEBACK_DRAIN_MAX / 2);
+
+ /*
+ * A pool that merely holds steady is not evidence the share was
+ * enough - the node is below its watermark and the pool is not
+ * yielding - so the share rises. One scope of draining then shrinks
+ * it and the decay takes over.
+ */
+ drain = MEMCOMP_WRITEBACK_DRAIN_STEP;
+ KUNIT_EXPECT_GT(test,
+ memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_PENDING,
+ false),
+ drain);
+ KUNIT_EXPECT_LT(test,
+ memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_PENDING,
+ true),
+ drain);
+}
+
+static void memcomp_test_writeback_drain_transient(struct kunit *test)
+{
+ unsigned int settled[11] = {};
+ unsigned int burst;
+
+ for (burst = 1; burst <= 10; burst++) {
+ unsigned int drain = 0;
+ unsigned int cycle;
+
+ for (cycle = 0; cycle < 64; cycle++) {
+ unsigned int i;
+
+ /* A burst of scopes while the pool is growing... */
+ for (i = 0; i < burst; i++)
+ drain = memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_EFFECTIVE,
+ false);
+ /* ...then the node climbs back on its own. */
+ drain = memcomp_writeback_drain_recover(drain);
+ }
+ settled[burst] = drain;
+
+ /* Never anywhere near the ceiling, however long the uptime. */
+ KUNIT_EXPECT_LT(test, drain,
+ MEMCOMP_WRITEBACK_DRAIN_MAX / 4);
+ /* Monotone in burst length, so the signal means something. */
+ if (burst > 1)
+ KUNIT_EXPECT_GT(test, settled[burst],
+ settled[burst - 1]);
+ }
+
+ /* The shortest bursts - a boot, a single spike - grant nothing. */
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_budget_values(1UL << 20,
+ settled[1]),
+ 0UL);
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_budget_values(1UL << 20,
+ settled[2]),
+ 0UL);
+
+ /*
+ * Pressure that is never relieved is the case that must still reach
+ * the ceiling, and grant half of the cold payloads there.
+ */
+ {
+ unsigned int drain = 0;
+ unsigned int i;
+
+ for (i = 0; i < 256; i++)
+ drain = memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_EFFECTIVE,
+ false);
+ KUNIT_EXPECT_EQ(test, drain, MEMCOMP_WRITEBACK_DRAIN_MAX);
+ KUNIT_EXPECT_EQ(test,
+ memcomp_writeback_budget_values(4096UL, drain),
+ 4096UL / 2);
+ }
+}
+
+static void memcomp_test_writeback_drain_handoff(struct kunit *test)
+{
+ unsigned int drain;
+ unsigned int i;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+
+ /* Wind the loop up as a sustained pressure period would. */
+ drain = 0;
+ for (i = 0; i < 256; i++)
+ drain = memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_INEFFICIENT,
+ false);
+ KUNIT_ASSERT_EQ(test, drain, MEMCOMP_WRITEBACK_DRAIN_MAX);
+
+ /*
+ * A grant with no sample - the first scope of a period that follows a
+ * recovery - carries the share over instead of restarting from zero,
+ * because the drain has been running the same loop against it.
+ */
+ memcomp_writeback_drain[0] = drain;
+ WRITE_ONCE(memcomp_writeback_feedback_valid[0], false);
+ memcomp_writeback_budget_grant_locked(0, 0, 0, ULONG_MAX);
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_drain[0], drain);
+
+ /*
+ * Unless the gate is closed: then the grant is nothing and the share
+ * is dropped, so a later opening does not inherit it.
+ */
+ memcomp_writeback_budget_grant_locked(0, 0, 0, 0);
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_drain[0], 0U);
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_budget[0], 0UL);
+ memcomp_writeback_drain[0] = drain;
+
+ /*
+ * The drain applies the same decay the scope path does, so a pool
+ * that yields is worked down to nothing off the reclaim path - it
+ * does not sit frozen waiting for the next workload.
+ */
+ for (i = 0; i < 128 && drain; i++)
+ drain = memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_PENDING,
+ true);
+ KUNIT_EXPECT_EQ(test, drain, 0U);
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_budget_values(4096UL, drain),
+ 0UL);
+
+ /*
+ * The handoff itself. Everything above is the loop's arithmetic; none
+ * of it reaches memcomp_reclaim_writeback_end(), which is what
+ * hands a period's remaining share to the off-reclaim drain. Making
+ * that call never fire leaves every case above green, so drive a real
+ * scope and require the quota to appear.
+ */
+ {
+ struct memcomp_lru *lru = memcomp_lru_nid(0);
+ long saved_total = total_swap_pages;
+ long saved_entries;
+ unsigned long entries;
+ unsigned long expected;
+ unsigned long total;
+ unsigned long stop;
+
+ KUNIT_ASSERT_NOT_NULL(test, lru);
+ /*
+ * The drain needs somewhere to write to, and a UML test host
+ * has no swap device. Without this the arm below is skipped
+ * and the case is green whether or not the handoff exists.
+ */
+ total_swap_pages = 1024;
+ WRITE_ONCE(memcomp_writeback_carry[0], 0);
+ memcomp_writeback_drain[0] = MEMCOMP_WRITEBACK_DRAIN_MAX;
+
+ /*
+ * And cold payloads for the share to be a share of, enough
+ * of them to open the level gate. With an empty list the
+ * quota is zero and the schedule returns before it stores
+ * anything, so the assertion below would hold whether or not
+ * the handoff ran; below the gate it is zero by design.
+ */
+ saved_entries = atomic_long_read(&lru->nr_entries);
+ total = totalram_pages();
+ stop = (unsigned long)div64_u64((u64)total *
+ MEMCOMP_WRITEBACK_GATE_STOP_PERMILLE, 1000);
+ entries = (unsigned long)div64_u64((u64)total *
+ MEMCOMP_WRITEBACK_GATE_START_PERMILLE, 1000) + 1;
+ atomic_long_set(&lru->nr_entries, entries);
+ /*
+ * The share asks for half of the cold payloads, but the gate
+ * lets only what is above its close point go.
+ */
+ expected = memcomp_writeback_budget_values(entries,
+ MEMCOMP_WRITEBACK_DRAIN_MAX);
+ KUNIT_ASSERT_GT(test, expected, entries - stop);
+ expected = entries - stop;
+
+ memcomp_reclaim_writeback_begin(0, 0);
+ KUNIT_EXPECT_EQ(test,
+ atomic_read(&memcomp_writeback_gate_open), 1);
+ KUNIT_EXPECT_LE(test, READ_ONCE(memcomp_writeback_budget[0]),
+ expected);
+ memcomp_writeback_drain[0] = MEMCOMP_WRITEBACK_DRAIN_MAX;
+ memcomp_reclaim_writeback_end(0);
+
+ /*
+ * The quota is what the share asks of the cold payloads, and
+ * it is settled once - a second period must not add to it.
+ */
+ KUNIT_EXPECT_EQ(test, READ_ONCE(memcomp_writeback_carry[0]),
+ expected);
+ if (expected) {
+ /* And one pass of it stays inside the slice bound. */
+ KUNIT_EXPECT_LE(test,
+ memcomp_writeback_drain_slice(expected),
+ (unsigned long)MEMCOMP_WRITEBACK_DRAIN_SLICE_MAX);
+ }
+
+ /*
+ * Back at the close point the gate shuts, and the period hands
+ * nothing over - not even what an earlier period left behind,
+ * however large a share the loop still holds.
+ */
+ atomic_long_set(&lru->nr_entries, stop);
+ WRITE_ONCE(memcomp_writeback_carry[0], 77);
+ memcomp_reclaim_writeback_begin(0, 0);
+ KUNIT_EXPECT_EQ(test,
+ atomic_read(&memcomp_writeback_gate_open), 0);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(memcomp_writeback_budget[0]),
+ 0UL);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(memcomp_writeback_drain[0]),
+ 0U);
+ memcomp_writeback_drain[0] = MEMCOMP_WRITEBACK_DRAIN_MAX;
+ memcomp_reclaim_writeback_end(0);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(memcomp_writeback_carry[0]),
+ 0UL);
+
+ WRITE_ONCE(memcomp_writeback_carry[0], 0);
+ atomic_long_set(&lru->nr_entries, saved_entries);
+ /* Leave the latch as the real pool has it. */
+ memcomp_writeback_gate_excess(0, memcomp_pool_total_size());
+ total_swap_pages = saved_total;
+ }
+
+ memcomp_writeback_drain[0] = 0;
+ memcomp_writeback_budget[0] = 0;
+ WRITE_ONCE(memcomp_writeback_feedback_valid[0], false);
+}
+
+static void memcomp_test_writeback_drain_slice(struct kunit *test)
+{
+ static const unsigned long quotas[] = {
+ 1, 32, 1000, 100000, 10000000,
+ };
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(quotas); i++) {
+ unsigned long quota = quotas[i];
+ unsigned long limit = memcomp_writeback_drain_slice(quota);
+
+ /* Never more than the cap, and never more than is owed. */
+ KUNIT_EXPECT_LE(test, limit,
+ (unsigned long)MEMCOMP_WRITEBACK_DRAIN_SLICE_MAX);
+ KUNIT_EXPECT_LE(test, limit, quota);
+ /* And always forward progress, so the quota cannot stall. */
+ KUNIT_EXPECT_GT(test, limit, 0UL);
+ }
+
+ /*
+ * A huge quota must not scale the slice with it. Asserting the cap is
+ * reached, rather than only that it is not exceeded, is what catches a
+ * slice that regressed to a plain fraction of the quota.
+ */
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_drain_slice(10000000UL),
+ (unsigned long)MEMCOMP_WRITEBACK_DRAIN_SLICE_MAX);
+
+ /*
+ * Below the cap the slice still has to grow with the quota, or the
+ * drain would crawl through a large backlog a batch at a time.
+ */
+ KUNIT_EXPECT_GT(test,
+ memcomp_writeback_drain_slice(MEMCOMP_WRITEBACK_DRAIN_SLICE_MAX *
+ MEMCOMP_WRITEBACK_DRAIN_SLICES / 2),
+ (unsigned long)MEMCOMP_WRITEBACK_BATCH);
+}
+
+static void memcomp_test_writeback_pass_bounded(struct kunit *test)
+{
+ static const unsigned long budgets[] = {
+ 1, 32, MEMCOMP_WRITEBACK_BATCH, 4096, 500000, 10000000,
+ };
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(budgets); i++) {
+ unsigned long budget = budgets[i];
+ unsigned long pass = memcomp_writeback_drain_slice(budget);
+
+ /* One pass is bounded however large the budget grew. */
+ KUNIT_EXPECT_LE(test, pass,
+ (unsigned long)MEMCOMP_WRITEBACK_DRAIN_SLICE_MAX);
+ KUNIT_EXPECT_LE(test, pass, budget);
+ KUNIT_EXPECT_GT(test, pass, 0UL);
+
+ /*
+ * And what a pass does not spend survives: the consume step
+ * subtracts only what was scanned, so the scope keeps asking
+ * until the whole budget is gone.
+ */
+ if (pass < budget) {
+ unsigned long left = budget - pass;
+
+ KUNIT_EXPECT_GT(test, left, 0UL);
+ KUNIT_EXPECT_GT(test,
+ memcomp_writeback_drain_slice(left),
+ 0UL);
+ }
+ }
+
+ /* A saturated budget must not become a single pass. */
+ KUNIT_EXPECT_LT(test, memcomp_writeback_drain_slice(500000UL),
+ 500000UL);
+
+ /*
+ * And the reclaim path takes its limit from the same place. This
+ * covers the policy, not the call: nothing here can invoke
+ * memcomp_shrinker_scan(), so a future change that read the raw
+ * budget again would still have to be caught by review. Sharing the
+ * helper is what makes that change visible.
+ */
+ {
+ unsigned long saved = READ_ONCE(memcomp_writeback_budget[0]);
+
+ WRITE_ONCE(memcomp_writeback_budget[0], 500000UL);
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_pass_limit(0),
+ memcomp_writeback_drain_slice(500000UL));
+ KUNIT_EXPECT_LE(test, memcomp_writeback_pass_limit(0),
+ (unsigned long)MEMCOMP_WRITEBACK_DRAIN_SLICE_MAX);
+ WRITE_ONCE(memcomp_writeback_budget[0], saved);
+ }
+ /* An invalid node asks for nothing rather than reading past the array. */
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_pass_limit(-1), 0UL);
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_pass_limit(MAX_NUMNODES), 0UL);
+}
+
+static unsigned long
+memcomp_test_writeback_budget_legacy(unsigned long cold_entries,
+ enum memcomp_writeback_feedback feedback,
+ unsigned int streak)
+{
+ unsigned int shift = 6;
+
+ if (streak < shift - 2)
+ shift -= streak;
+ else
+ shift = 2;
+ if (feedback == MEMCOMP_WRITEBACK_EFFECTIVE)
+ shift++;
+ else if (feedback == MEMCOMP_WRITEBACK_INEFFICIENT && shift > 2)
+ shift--;
+
+ return cold_entries >> shift;
+}
+
+static void memcomp_test_writeback_budget_vs_legacy(struct kunit *test)
+{
+ const unsigned long cold = 4096UL;
+ unsigned int drain;
+ unsigned int i;
+
+ /*
+ * The old share depended only on how many scopes had passed, so a
+ * first scope drained 1/64 of the pool whether or not the pool was in
+ * any trouble, and four consecutive scopes reached its 1/4 ceiling
+ * whether or not the previous three had achieved anything.
+ */
+ KUNIT_EXPECT_EQ(test,
+ memcomp_test_writeback_budget_legacy(cold,
+ MEMCOMP_WRITEBACK_PENDING,
+ 0),
+ cold / 64);
+ KUNIT_EXPECT_EQ(test,
+ memcomp_test_writeback_budget_legacy(cold,
+ MEMCOMP_WRITEBACK_PENDING,
+ 100),
+ cold / 4);
+
+ /*
+ * The loop starts from nothing instead, and a pool that yields to the
+ * share it was given goes back to nothing - the state a device spends
+ * most of its life in, where the old policy still drained.
+ */
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_budget_values(cold, 0), 0UL);
+ drain = MEMCOMP_WRITEBACK_DRAIN_MAX;
+ for (i = 0; i < 128; i++)
+ drain = memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_PENDING,
+ true);
+ KUNIT_EXPECT_EQ(test, memcomp_writeback_budget_values(cold, drain),
+ 0UL);
+ KUNIT_EXPECT_GT(test,
+ memcomp_test_writeback_budget_legacy(cold,
+ MEMCOMP_WRITEBACK_EFFECTIVE,
+ 0),
+ 0UL);
+
+ /*
+ * Sustained resistance does eventually drain more than the old
+ * ceiling allowed - the loop is not merely a gentler constant.
+ */
+ drain = 0;
+ for (i = 0; i < 256; i++)
+ drain = memcomp_writeback_drain_next(drain,
+ MEMCOMP_WRITEBACK_INEFFICIENT,
+ false);
+ KUNIT_EXPECT_GT(test, memcomp_writeback_budget_values(cold, drain),
+ memcomp_test_writeback_budget_legacy(cold,
+ MEMCOMP_WRITEBACK_INEFFICIENT,
+ 100));
+}
+
+static void memcomp_test_comp_ctx_bitmap(struct kunit *test)
+{
+ unsigned long all_mask = BIT(0) | BIT(1);
+ unsigned long free_mask = BIT(2) | BIT(5);
+ enum memcomp_comp_ctx_miss miss;
+ s64 all_busy_pages;
+ s64 borrow_limit_pages;
+ s64 borrow_waiter_pages;
+ s64 budget_pages;
+ s64 busy_pages;
+ s64 reserved_free_pages;
+ int idx;
+
+ idx = memcomp_comp_ctx_claim(&free_mask, BIT(5));
+ KUNIT_EXPECT_EQ(test, idx, 5);
+ KUNIT_EXPECT_EQ(test, free_mask, BIT(2));
+ idx = memcomp_comp_ctx_claim(&free_mask, ~0UL);
+ KUNIT_EXPECT_EQ(test, idx, 2);
+ KUNIT_EXPECT_EQ(test, free_mask, 0UL);
+ KUNIT_EXPECT_EQ(test,
+ memcomp_comp_ctx_claim(&free_mask, ~0UL), -EBUSY);
+ KUNIT_EXPECT_TRUE(test, memcomp_comp_ctx_release(&free_mask, 2));
+ KUNIT_EXPECT_EQ(test, free_mask, BIT(2));
+ KUNIT_EXPECT_FALSE(test, memcomp_comp_ctx_release(&free_mask, 2));
+
+ miss = memcomp_comp_ctx_classify(BIT(0), BIT(1), BIT(0) | BIT(1));
+ KUNIT_EXPECT_EQ(test, miss, MEMCOMP_COMP_CTX_RESERVED_FREE);
+ miss = memcomp_comp_ctx_classify(0, BIT(1), BIT(0) | BIT(1));
+ KUNIT_EXPECT_EQ(test, miss, MEMCOMP_COMP_CTX_ALL_BUSY);
+ miss = memcomp_comp_ctx_classify(BIT(1), BIT(1), BIT(0) | BIT(1));
+ KUNIT_EXPECT_EQ(test, miss, MEMCOMP_COMP_CTX_AVAILABLE);
+ miss = memcomp_comp_ctx_borrow_decision(0, 1, 0);
+ KUNIT_EXPECT_EQ(test, miss, MEMCOMP_COMP_CTX_AVAILABLE);
+ miss = memcomp_comp_ctx_borrow_decision(1, 1, 0);
+ KUNIT_EXPECT_EQ(test, miss, MEMCOMP_COMP_CTX_BORROW_LIMIT);
+ miss = memcomp_comp_ctx_borrow_decision(0, 1, 1);
+ KUNIT_EXPECT_EQ(test, miss,
+ MEMCOMP_COMP_CTX_BACKGROUND_WAITERS);
+ miss = memcomp_comp_ctx_borrow_decision(1, 1, 1);
+ KUNIT_EXPECT_EQ(test, miss,
+ MEMCOMP_COMP_CTX_BACKGROUND_WAITERS);
+ miss = memcomp_comp_ctx_foreground_status(BIT(1), BIT(1),
+ all_mask, 1, 1, 1);
+ KUNIT_EXPECT_EQ(test, miss, MEMCOMP_COMP_CTX_AVAILABLE);
+ miss = memcomp_comp_ctx_foreground_status(BIT(0), BIT(1),
+ all_mask, 0, 1, 0);
+ KUNIT_EXPECT_EQ(test, miss, MEMCOMP_COMP_CTX_AVAILABLE);
+ miss = memcomp_comp_ctx_foreground_status(BIT(0), BIT(1),
+ all_mask, 1, 1, 0);
+ KUNIT_EXPECT_EQ(test, miss, MEMCOMP_COMP_CTX_BORROW_LIMIT);
+ miss = memcomp_comp_ctx_foreground_status(BIT(0), BIT(1),
+ all_mask, 0, 1, 1);
+ KUNIT_EXPECT_EQ(test, miss,
+ MEMCOMP_COMP_CTX_BACKGROUND_WAITERS);
+ miss = memcomp_comp_ctx_foreground_status(0, BIT(1), all_mask,
+ 0, 1, 0);
+ KUNIT_EXPECT_EQ(test, miss, MEMCOMP_COMP_CTX_ALL_BUSY);
+
+ reserved_free_pages =
+ atomic64_read(&memcomp_reclaim_ctx_busy_reserved_free_pages);
+ all_busy_pages =
+ atomic64_read(&memcomp_reclaim_ctx_busy_all_busy_pages);
+ borrow_limit_pages =
+ atomic64_read(&memcomp_reclaim_ctx_borrow_limit_pages);
+ borrow_waiter_pages =
+ atomic64_read(&memcomp_reclaim_ctx_borrow_waiter_pages);
+ budget_pages = atomic64_read(&memcomp_reclaim_ctx_budget_pages);
+ busy_pages = reserved_free_pages + all_busy_pages;
+ memcomp_account_reclaim_ctx_miss(MEMCOMP_COMP_CTX_RESERVED_FREE,
+ 2);
+ memcomp_account_reclaim_ctx_miss(MEMCOMP_COMP_CTX_ALL_BUSY, 3);
+ memcomp_account_reclaim_ctx_miss(MEMCOMP_COMP_CTX_BORROW_LIMIT,
+ 5);
+ miss = MEMCOMP_COMP_CTX_BACKGROUND_WAITERS;
+ memcomp_account_reclaim_ctx_miss(miss, 7);
+ memcomp_account_reclaim_ctx_miss(MEMCOMP_COMP_CTX_BUDGET, 11);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_reclaim_ctx_busy_reserved_free_pages) +
+ atomic64_read(&memcomp_reclaim_ctx_busy_all_busy_pages),
+ busy_pages + 17);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_reclaim_ctx_busy_reserved_free_pages),
+ reserved_free_pages + 14);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_reclaim_ctx_busy_all_busy_pages),
+ all_busy_pages + 3);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_reclaim_ctx_borrow_limit_pages),
+ borrow_limit_pages + 5);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_reclaim_ctx_borrow_waiter_pages),
+ borrow_waiter_pages + 7);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_reclaim_ctx_budget_pages),
+ budget_pages + 11);
+}
+
+static void memcomp_test_comp_budget(struct kunit *test)
+{
+ struct memcomp_comp_budget budget = {};
+ enum memcomp_payload_mode mode;
+ enum memcomp_reclaim_source source;
+ bool granted;
+ unsigned int i;
+
+ KUNIT_EXPECT_EQ(test, memcomp_comp_budget_limit(0, 0), 0U);
+ KUNIT_EXPECT_EQ(test, memcomp_comp_budget_limit(1, 32), 1U);
+ KUNIT_EXPECT_EQ(test, memcomp_comp_budget_limit(2, 32), 1U);
+ KUNIT_EXPECT_EQ(test, memcomp_comp_budget_limit(8, 32), 7U);
+ KUNIT_EXPECT_EQ(test, memcomp_comp_budget_limit(8, 4), 4U);
+ mode = MEMCOMP_PAYLOAD_BEST_EFFORT;
+ source = MEMCOMP_RECLAIM_KSWAPD;
+ granted = memcomp_comp_ctx_background(mode, source, true);
+ KUNIT_EXPECT_FALSE(test, granted);
+ mode = MEMCOMP_PAYLOAD_MUST_SUCCEED;
+ source = MEMCOMP_RECLAIM_DIRECT;
+ granted = memcomp_comp_ctx_background(mode, source, true);
+ KUNIT_EXPECT_FALSE(test, granted);
+ source = MEMCOMP_RECLAIM_KSWAPD;
+ granted = memcomp_comp_ctx_background(mode, source, true);
+ KUNIT_EXPECT_TRUE(test, granted);
+ granted = memcomp_comp_ctx_background(mode, source, false);
+ KUNIT_EXPECT_FALSE(test, granted);
+
+ for (i = 0; i < 7; i++) {
+ granted = memcomp_comp_budget_try_acquire(&budget, 7);
+ KUNIT_ASSERT_TRUE(test, granted);
+ }
+ KUNIT_EXPECT_EQ(test, budget.active, 7U);
+ KUNIT_EXPECT_EQ(test, budget.background_active, 7U);
+ KUNIT_EXPECT_EQ(test, budget.peak, 7U);
+ KUNIT_EXPECT_EQ(test, budget.background_peak, 7U);
+ granted = memcomp_comp_budget_try_acquire(&budget, 7);
+ KUNIT_EXPECT_FALSE(test, granted);
+ granted = memcomp_comp_budget_available(&budget, 4);
+ KUNIT_EXPECT_FALSE(test, granted);
+ KUNIT_EXPECT_EQ(test, budget.active, 7U);
+
+ for (i = 0; i < 4; i++)
+ KUNIT_ASSERT_TRUE(test,
+ memcomp_comp_budget_release(&budget));
+ KUNIT_EXPECT_TRUE(test,
+ memcomp_comp_budget_available(&budget, 4));
+ for (; i < 7; i++)
+ KUNIT_ASSERT_TRUE(test,
+ memcomp_comp_budget_release(&budget));
+ KUNIT_EXPECT_EQ(test, budget.active, 0U);
+ KUNIT_EXPECT_EQ(test, budget.background_active, 0U);
+ KUNIT_EXPECT_FALSE(test,
+ memcomp_comp_budget_release(&budget));
+}
+
+static void memcomp_test_direct_staging_policy(struct kunit *test)
+{
+ static const struct {
+ enum memcomp_payload_mode mode;
+ enum memcomp_reclaim_source source;
+ unsigned int nr_pages;
+ const char *driver;
+ bool eligible;
+ } cases[] = {
+ { MEMCOMP_PAYLOAD_MUST_SUCCEED,
+ MEMCOMP_RECLAIM_DIRECT, 1, "zstd-generic", true },
+ { MEMCOMP_PAYLOAD_MUST_SUCCEED,
+ MEMCOMP_RECLAIM_DIRECT, 2, "zstd-generic", false },
+ { MEMCOMP_PAYLOAD_BEST_EFFORT,
+ MEMCOMP_RECLAIM_DIRECT, 1, "zstd-generic", false },
+ { MEMCOMP_PAYLOAD_MUST_SUCCEED,
+ MEMCOMP_RECLAIM_KSWAPD, 1, "zstd-generic", false },
+ { MEMCOMP_PAYLOAD_MUST_SUCCEED,
+ MEMCOMP_RECLAIM_DIRECT, 1, "lz4-generic", false },
+ };
+ struct memcomp_reclaim_ctx *ctx;
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(cases); i++) {
+ const typeof(cases[0]) *c = &cases[i];
+ bool actual;
+ bool expected = c->eligible &&
+ !IS_ENABLED(CONFIG_PREEMPT_RT);
+
+ actual = memcomp_staging_policy(c->mode, c->source,
+ c->nr_pages, c->driver);
+ KUNIT_EXPECT_EQ(test, actual, expected);
+ }
+
+ if (!memcomp_is_enabled())
+ return;
+ ctx = memcomp_reclaim_ctx_get(2, MEMCOMP_RECLAIM_DIRECT);
+ KUNIT_EXPECT_NOT_NULL(test, ctx);
+ if (ctx) {
+ KUNIT_EXPECT_FALSE(test, ctx->direct_staging);
+ memcomp_reclaim_ctx_put(ctx);
+ }
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_DIRECT);
+ KUNIT_EXPECT_NOT_NULL(test, ctx);
+ if (ctx) {
+ const char *driver =
+ crypto_tfm_alg_driver_name(crypto_comp_tfm(ctx->tfm));
+
+ KUNIT_EXPECT_EQ(test, ctx->direct_staging,
+ !IS_ENABLED(CONFIG_PREEMPT_RT) &&
+ memcomp_driver_is_zstd(driver));
+ memcomp_reclaim_ctx_put(ctx);
+ }
+ ctx = memcomp_get_comp_ctx(MEMCOMP_PAYLOAD_BEST_EFFORT,
+ NULL, MEMCOMP_RECLAIM_DIRECT);
+ KUNIT_EXPECT_NOT_NULL(test, ctx);
+ if (ctx) {
+ KUNIT_EXPECT_FALSE(test, ctx->direct_staging);
+ memcomp_put_comp_ctx(ctx);
+ }
+}
+
+static void memcomp_test_direct_staging_pool(struct kunit *test)
+{
+ struct memcomp_direct_staging *claimed[MEMCOMP_MAX_COMP_CTXS] = {};
+ struct memcomp_direct_staging *extra;
+ unsigned int nr_staging = READ_ONCE(memcomp_nr_direct_staging);
+ unsigned int claimed_count = 0;
+ unsigned long flags;
+ unsigned int active;
+ unsigned int free;
+ unsigned int i;
+ unsigned int j;
+ unsigned int peak;
+
+ if (!nr_staging) {
+ kunit_skip(test, "no direct staging buffers");
+ return;
+ }
+ spin_lock_irqsave(&memcomp_direct_staging_lock, flags);
+ active = memcomp_direct_staging_active;
+ free = bitmap_weight(&memcomp_direct_staging_free, nr_staging);
+ spin_unlock_irqrestore(&memcomp_direct_staging_lock, flags);
+ if (active || free != nr_staging) {
+ kunit_skip(test, "direct staging pool is active");
+ return;
+ }
+
+ for (i = 0; i < nr_staging; i++) {
+ claimed[i] = memcomp_direct_staging_try_get();
+ if (!claimed[i])
+ goto out_skip;
+ claimed_count++;
+ }
+ for (i = 0; i < claimed_count; i++) {
+ for (j = 0; j < i; j++)
+ KUNIT_EXPECT_PTR_NE(test, claimed[i], claimed[j]);
+ }
+ extra = memcomp_direct_staging_try_get();
+ KUNIT_EXPECT_PTR_EQ(test, extra, NULL);
+ if (extra)
+ memcomp_direct_staging_put(extra);
+ spin_lock_irqsave(&memcomp_direct_staging_lock, flags);
+ active = memcomp_direct_staging_active;
+ peak = memcomp_direct_staging_peak;
+ spin_unlock_irqrestore(&memcomp_direct_staging_lock, flags);
+ KUNIT_EXPECT_EQ(test, active, claimed_count);
+ KUNIT_EXPECT_GE(test, peak, claimed_count);
+
+ for (i = 0; i < claimed_count; i++)
+ memcomp_direct_staging_put(claimed[i]);
+ spin_lock_irqsave(&memcomp_direct_staging_lock, flags);
+ active = memcomp_direct_staging_active;
+ free = bitmap_weight(&memcomp_direct_staging_free, nr_staging);
+ spin_unlock_irqrestore(&memcomp_direct_staging_lock, flags);
+ KUNIT_EXPECT_EQ(test, active, 0U);
+ KUNIT_EXPECT_EQ(test, free, nr_staging);
+ return;
+
+out_skip:
+ for (i = 0; i < claimed_count; i++)
+ memcomp_direct_staging_put(claimed[i]);
+ kunit_skip(test, "direct staging pool became active");
+}
+
+static void
+memcomp_test_release_prepared(struct memcomp_reclaim_ctx *ctx,
+ struct memcomp_store_ctx *store,
+ unsigned int length,
+ unsigned long handle, u8 pool_idx)
+{
+ if (length)
+ memcomp_zs_free(pool_idx, handle);
+ if (ctx)
+ memcomp_reclaim_ctx_put(ctx);
+ if (store->staging) {
+ memcomp_ctx_timing_account(&store->staging->timing);
+ memcomp_direct_staging_put(store->staging);
+ store->staging = NULL;
+ }
+}
+
+static void memcomp_test_fill_real_memory_page(u8 *buf)
+{
+ struct memcomp_test_real_page_record record = {};
+ size_t pos = 0;
+ u32 mix;
+
+ memcpy(&record.uts, init_utsname(), sizeof(record.uts));
+ strscpy(record.banner, linux_banner, sizeof(record.banner));
+ strscpy(record.comm, current->comm, sizeof(record.comm));
+ record.totalram = totalram_pages();
+ record.online_cpus = num_online_cpus();
+ mix = (u32)jiffies ^ (u32)record.totalram;
+
+ while (pos < PAGE_SIZE) {
+ size_t copy = min(sizeof(record), PAGE_SIZE - pos);
+
+ record.seq++;
+ record.jiffies_snapshot = get_jiffies_64();
+ record.mix = mix;
+ memcpy(buf + pos, &record, copy);
+ mix = mix * 1664525U + 1013904223U + record.seq;
+ pos += copy;
+ }
+}
+
+static void memcomp_test_fill_zstd_ratio_page(u8 *buf)
+{
+ u32 state = 0x9e3779b9;
+ size_t pos;
+
+ for (pos = 0; pos < PAGE_SIZE; pos += 64) {
+ u8 *dst = buf + pos;
+ unsigned int i;
+
+ memset(dst, 0, 49);
+ memcpy(dst, "memcomp-zstd-page",
+ sizeof("memcomp-zstd-page"));
+ for (i = 49; i < 64; i++) {
+ state = state * 1664525U + 1013904223U;
+ dst[i] = (state >> 16) & 0x7f;
+ }
+ }
+}
+
+static void memcomp_test_fill_zstd_shift_page(u8 *buf, unsigned int page)
+{
+ u32 state = 0x9e3779b9 ^ (page * 2654435761U);
+ size_t pos;
+
+ if (page < MEMCOMP_TEST_HUF_CACHE_PAGES / 2) {
+ memcomp_test_fill_zstd_ratio_page(buf);
+ buf[(page * 131U) % PAGE_SIZE] = page & 0x7f;
+ return;
+ }
+
+ for (pos = 0; pos < PAGE_SIZE; pos += 64) {
+ u8 *dst = buf + pos;
+ unsigned int i;
+
+ /*
+ * A different restricted alphabet, not a wider one. Uniform
+ * bytes would make zstd give up on the literals entirely
+ * (HUF_compress_internal() returns 0 below its largest-symbol
+ * threshold), emit a set_basic block, and code no table at all
+ * - which exercises neither the encoder cache nor the decoder
+ * one. Staying compressible with a different distribution is
+ * what produces a second Huffman table.
+ */
+ memset(dst, 0xa0, 49);
+ memcpy(dst, "\xb1\xb2\xb3-shift-page", sizeof("\xb1\xb2\xb3-shift-page"));
+ for (i = 49; i < 64; i++) {
+ state = state * 1664525U + 1013904223U;
+ dst[i] = 0x80 | ((state >> 16) & 0x3f);
+ }
+ }
+}
+
+static void memcomp_test_fill_zstd_cache_page(u8 *buf,
+ unsigned int page)
+{
+ memcomp_test_fill_zstd_ratio_page(buf);
+ buf[(page * 131U) % PAGE_SIZE] = page & 0x7f;
+
+ /* Exercise checked table misses before and after a cache refresh. */
+ if (page == 0 || page == MEMCOMP_TEST_HUF_CACHE_PAGES - 1)
+ buf[PAGE_SIZE - 1] = 0x81;
+ if (page == 63)
+ buf[PAGE_SIZE - 2] = 0xff;
+}
+
+static void
+memcomp_test_zstd_page_roundtrip(struct kunit *test,
+ const struct memcomp_test_zstd_page_case *page_case)
+{
+ struct memcomp_store_ctx store = {};
+ struct memcomp_reclaim_ctx *ctx;
+ const char *driver;
+ void *mapped;
+ unsigned int length = 0;
+ unsigned long handle = 0;
+ u8 pool_idx = 0;
+ u8 *dst;
+ u8 *src;
+ int ret;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+
+ src = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ dst = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, dst);
+ page_case->fill(src);
+
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_DIRECT);
+ KUNIT_ASSERT_NOT_NULL(test, ctx);
+ driver = crypto_tfm_alg_driver_name(crypto_comp_tfm(ctx->tfm));
+ if (!memcomp_driver_is_zstd(driver)) {
+ memcomp_reclaim_ctx_put(ctx);
+ kunit_skip(test, "test requires zstd-generic");
+ return;
+ }
+ ret = memcomp_prepare_payload_locked(&ctx, src, PAGE_SIZE,
+ &length, &handle, &pool_idx,
+ false, &store, NULL);
+ if (ret == -E2BIG && page_case->allow_incompressible) {
+ memcomp_test_release_prepared(ctx, &store, length, handle,
+ pool_idx);
+ kunit_info(test, "%s was rejected as incompressible",
+ page_case->name);
+ return;
+ }
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ if (ret)
+ goto out;
+
+ KUNIT_EXPECT_GT(test, length, 0U);
+ KUNIT_EXPECT_LE(test, length, PAGE_SIZE);
+ KUNIT_EXPECT_LE(test, length, page_case->max_length);
+ kunit_info(test, "%s compressed to %u bytes", page_case->name, length);
+ mapped = zs_map_object(memcomp_pools[pool_idx], handle, ZS_MM_RO);
+ if (!mapped) {
+ KUNIT_FAIL(test, "payload map failed");
+ goto out;
+ }
+ ret = memcomp_decompress_payload(mapped, length, dst);
+ zs_unmap_object(memcomp_pools[pool_idx], handle);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ KUNIT_EXPECT_MEMEQ(test, src, dst, PAGE_SIZE);
+out:
+ memcomp_test_release_prepared(ctx, &store, length, handle, pool_idx);
+}
+
+static void memcomp_test_zstd_real_page_roundtrip(struct kunit *test)
+{
+ static const struct memcomp_test_zstd_page_case page_case = {
+ .name = "real memory page",
+ .fill = memcomp_test_fill_real_memory_page,
+ .max_length = PAGE_SIZE * 28 / 100,
+ .allow_incompressible = true,
+ };
+
+ memcomp_test_zstd_page_roundtrip(test, &page_case);
+}
+
+static void memcomp_test_zstd_page_ratio_bench(struct kunit *test)
+{
+ static const struct memcomp_test_zstd_page_case page_case = {
+ .name = "near-budget page",
+ .fill = memcomp_test_fill_zstd_ratio_page,
+ .max_length = PAGE_SIZE * 28 / 100,
+ };
+
+ memcomp_test_zstd_page_roundtrip(test, &page_case);
+}
+
+static void memcomp_test_zstd_huf_cache_frames(struct kunit *test)
+{
+ struct memcomp_reclaim_ctx *ctx;
+ const char *driver;
+ u8 *src;
+ u8 *compressed;
+ u8 *dst;
+ unsigned int page;
+ unsigned int worst = 0;
+ u64 total = 0;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+
+ src = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ compressed = kunit_kmalloc(test, MEMCOMP_MAX_COMPRESSED_SIZE,
+ GFP_KERNEL);
+ dst = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ KUNIT_ASSERT_NOT_NULL(test, compressed);
+ KUNIT_ASSERT_NOT_NULL(test, dst);
+
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_DIRECT);
+ KUNIT_ASSERT_NOT_NULL(test, ctx);
+ driver = crypto_tfm_alg_driver_name(crypto_comp_tfm(ctx->tfm));
+ if (!memcomp_driver_is_zstd(driver)) {
+ memcomp_reclaim_ctx_put(ctx);
+ kunit_skip(test, "test requires zstd-generic");
+ return;
+ }
+
+ for (page = 0; page < MEMCOMP_TEST_HUF_CACHE_PAGES; ++page) {
+ struct crypto_comp *decomp;
+ unsigned int compressed_len = MEMCOMP_MAX_COMPRESSED_SIZE;
+ unsigned int decompressed_len = PAGE_SIZE;
+ int ret;
+
+ memcomp_test_fill_zstd_cache_page(src, page);
+ ret = crypto_comp_compress(ctx->tfm, src, PAGE_SIZE, compressed,
+ &compressed_len);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ if (ret)
+ goto out;
+ KUNIT_EXPECT_LT(test, compressed_len, PAGE_SIZE);
+ if (compressed_len >= PAGE_SIZE)
+ goto out;
+
+ /* A fresh decoder proves this frame carries its own Huffman table. */
+ decomp = crypto_alloc_comp("zstd-generic", 0, 0);
+ KUNIT_EXPECT_FALSE(test, IS_ERR(decomp));
+ if (IS_ERR(decomp))
+ goto out;
+ ret = crypto_comp_decompress(decomp, compressed, compressed_len,
+ dst, &decompressed_len);
+ crypto_free_comp(decomp);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ if (ret)
+ goto out;
+ KUNIT_EXPECT_EQ(test, decompressed_len, PAGE_SIZE);
+ KUNIT_EXPECT_MEMEQ(test, src, dst, PAGE_SIZE);
+ total += compressed_len;
+ worst = max(worst, compressed_len);
+ }
+
+ /*
+ * Report what the cache costs in ratio across its whole life, not just
+ * that each frame decoded. The selector decides per block whether the
+ * standing table is still worth using, and a block that keeps a table
+ * it should have rebuilt shows up here and nowhere else - the
+ * per-frame check above only requires beating PAGE_SIZE. The bound is
+ * the huge-class boundary, since a payload past it costs a whole page
+ * of backing store.
+ */
+ kunit_info(test, "%u pages through one cache: %llu bytes total, %u worst, %llu mean",
+ MEMCOMP_TEST_HUF_CACHE_PAGES, total, worst,
+ total / MEMCOMP_TEST_HUF_CACHE_PAGES);
+ KUNIT_EXPECT_LT(test, worst, (unsigned int)PAGE_SIZE);
+
+out:
+ memcomp_reclaim_ctx_put(ctx);
+}
+
+static void memcomp_test_zstd_huf_cache_shift(struct kunit *test)
+{
+ struct memcomp_reclaim_ctx *ctx;
+ const char *driver;
+ struct crypto_comp *shared = NULL;
+ unsigned int worst = 0;
+ unsigned int page;
+ u8 *compressed;
+ u8 *dst;
+ u8 *src;
+ u64 total = 0;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+
+ src = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ compressed = kunit_kmalloc(test, MEMCOMP_MAX_COMPRESSED_SIZE,
+ GFP_KERNEL);
+ dst = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ KUNIT_ASSERT_NOT_NULL(test, compressed);
+ KUNIT_ASSERT_NOT_NULL(test, dst);
+
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_DIRECT);
+ KUNIT_ASSERT_NOT_NULL(test, ctx);
+ driver = crypto_tfm_alg_driver_name(crypto_comp_tfm(ctx->tfm));
+ if (!memcomp_driver_is_zstd(driver)) {
+ memcomp_reclaim_ctx_put(ctx);
+ kunit_skip(test, "test requires zstd-generic");
+ return;
+ }
+
+ /*
+ * One decoder for every frame, which is what the fault path uses: the
+ * decode-side HUF_DTableCache only ever hits on a DCtx that has seen a
+ * frame before. A fresh decoder per frame - what the sibling case
+ * does, to prove frames are self-contained - leaves that cache empty on
+ * every lookup, so nothing there exercises it.
+ *
+ * The shift is what makes this a test of the cache rather than of the
+ * table: the two halves carry different Huffman descriptions, so a
+ * lookup that skipped HUF_dtableCacheLookup()'s memcmp would serve the
+ * first half's table to the second half and hand back wrong bytes.
+ */
+ shared = crypto_alloc_comp("zstd-generic", 0, 0);
+ KUNIT_ASSERT_FALSE(test, IS_ERR(shared));
+
+ for (page = 0; page < MEMCOMP_TEST_HUF_CACHE_PAGES; ++page) {
+ unsigned int compressed_len = MEMCOMP_MAX_COMPRESSED_SIZE;
+ unsigned int decompressed_len = PAGE_SIZE;
+ unsigned int shared_len = PAGE_SIZE;
+ struct crypto_comp *decomp;
+ int ret;
+
+ memcomp_test_fill_zstd_shift_page(src, page);
+ ret = crypto_comp_compress(ctx->tfm, src, PAGE_SIZE, compressed,
+ &compressed_len);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ if (ret)
+ goto out_shift;
+
+ /* Every frame stays independently decodable across the shift. */
+ decomp = crypto_alloc_comp("zstd-generic", 0, 0);
+ KUNIT_EXPECT_FALSE(test, IS_ERR(decomp));
+ if (IS_ERR(decomp))
+ goto out_shift;
+ ret = crypto_comp_decompress(decomp, compressed, compressed_len,
+ dst, &decompressed_len);
+ crypto_free_comp(decomp);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ if (ret)
+ goto out_shift;
+ KUNIT_EXPECT_EQ(test, decompressed_len, PAGE_SIZE);
+ KUNIT_EXPECT_MEMEQ(test, src, dst, PAGE_SIZE);
+
+ /* And the same bytes back through the decoder that remembers. */
+ memset(dst, 0, PAGE_SIZE);
+ ret = crypto_comp_decompress(shared, compressed, compressed_len,
+ dst, &shared_len);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ if (ret)
+ goto out_shift;
+ KUNIT_EXPECT_EQ(test, shared_len, PAGE_SIZE);
+ KUNIT_EXPECT_MEMEQ(test, src, dst, PAGE_SIZE);
+
+ total += compressed_len;
+ worst = max(worst, compressed_len);
+ }
+
+ kunit_info(test,
+ "%u pages across a distribution shift: %llu bytes total, %u worst, %llu mean",
+ MEMCOMP_TEST_HUF_CACHE_PAGES, total, worst,
+ total / MEMCOMP_TEST_HUF_CACHE_PAGES);
+ KUNIT_EXPECT_LT(test, worst, (unsigned int)PAGE_SIZE);
+
+ /*
+ * A truncated frame must be rejected, not read past, by a decoder that
+ * is holding a remembered description.
+ *
+ * This does not reach HUF_dtableCacheLookup()'s descSize > srcSize
+ * guard: frame and block header validation rejects a truncated buffer
+ * before any literal block is decoded, so the lookup never sees a
+ * short cSrcSize. Covering that guard needs a hand-built frame
+ * declaring a litCSize smaller than the cached description, which
+ * zstd's own encoder cannot emit - fuzzer work, not a unit test.
+ * What this does cover is that a remembered table does not make the
+ * ordinary truncation paths misbehave, which is worth having under
+ * KASAN.
+ */
+ {
+ unsigned int last_len = 0;
+ unsigned int cut;
+
+ memcomp_test_fill_zstd_shift_page(src,
+ MEMCOMP_TEST_HUF_CACHE_PAGES - 1);
+ last_len = MEMCOMP_MAX_COMPRESSED_SIZE;
+ if (crypto_comp_compress(ctx->tfm, src, PAGE_SIZE, compressed,
+ &last_len))
+ goto out_shift;
+
+ for (cut = 1; cut < last_len && cut <= 96; cut++) {
+ u8 *cropped = kunit_kmalloc(test, cut, GFP_KERNEL);
+ unsigned int out_len = PAGE_SIZE;
+
+ KUNIT_ASSERT_NOT_NULL(test, cropped);
+ memcpy(cropped, compressed, cut);
+ /* Any answer but a read past the end is acceptable. */
+ (void)crypto_comp_decompress(shared, cropped, cut,
+ dst, &out_len);
+ kunit_kfree(test, cropped);
+ }
+ }
+
+out_shift:
+ if (!IS_ERR_OR_NULL(shared))
+ crypto_free_comp(shared);
+ memcomp_reclaim_ctx_put(ctx);
+}
+
+static void memcomp_test_direct_staging_execution(struct kunit *test)
+{
+ struct memcomp_direct_staging *held[MEMCOMP_MAX_COMP_CTXS] = {};
+ struct memcomp_reclaim_ctx *ctx;
+ struct memcomp_store_ctx store = {};
+ unsigned int nr_staging = READ_ONCE(memcomp_nr_direct_staging);
+ unsigned int length = 0;
+ unsigned int held_count = 0;
+ unsigned long handle = 0;
+ unsigned int expected;
+ unsigned int i;
+ u64 lease_end;
+ u64 lease_start;
+ void *ctx_dst;
+ int before;
+ u8 pool_idx = 0;
+ u8 *src;
+ int ret;
+
+ if (!memcomp_is_enabled() || !nr_staging ||
+ IS_ENABLED(CONFIG_PREEMPT_RT)) {
+ kunit_skip(test, "direct staging is unavailable");
+ return;
+ }
+ src = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ for (i = 0; i < PAGE_SIZE; i++)
+ src[i] = (i >> 5) ^ (i >> 9);
+
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_DIRECT);
+ KUNIT_ASSERT_NOT_NULL(test, ctx);
+ if (!ctx->direct_staging) {
+ memcomp_reclaim_ctx_put(ctx);
+ kunit_skip(test, "current compressor is not staging-safe");
+ return;
+ }
+ before = preempt_count();
+ ret = memcomp_prepare_payload_locked(&ctx, src, PAGE_SIZE,
+ &length, &handle, &pool_idx,
+ false, &store, NULL);
+ expected = before + PREEMPT_DISABLE_OFFSET;
+ if (!store.staging) {
+ memcomp_test_release_prepared(ctx, &store, length, handle,
+ pool_idx);
+ kunit_skip(test, "direct staging slot was unavailable");
+ return;
+ }
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ KUNIT_EXPECT_PTR_EQ(test, ctx, NULL);
+ KUNIT_EXPECT_NOT_NULL(test, store.staging);
+ KUNIT_EXPECT_EQ(test, preempt_count(), before);
+ if (store.staging) {
+ KUNIT_EXPECT_EQ(test,
+ store.staging->kunit_compress_preempt_count,
+ (int)expected);
+ KUNIT_EXPECT_EQ(test,
+ store.staging->kunit_detach_preempt_count,
+ (int)expected);
+ KUNIT_EXPECT_PTR_EQ(test,
+ store.staging->kunit_compress_dst,
+ store.staging->dst);
+ lease_end = store.staging->timing.timing_lease_end_ns;
+ lease_start = store.staging->timing.timing_start_ns;
+ if (store.staging->timing.timing_active)
+ KUNIT_EXPECT_GT(test, lease_end, lease_start);
+ }
+ memcomp_test_release_prepared(ctx, &store, length, handle,
+ pool_idx);
+
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_DIRECT);
+ KUNIT_ASSERT_NOT_NULL(test, ctx);
+ length = 0;
+ handle = 0;
+ ctx->kunit_compress_error = -EIO;
+ before = preempt_count();
+ ret = memcomp_prepare_payload_locked(&ctx, src, PAGE_SIZE,
+ &length, &handle, &pool_idx,
+ false, &store, NULL);
+ if (!store.staging) {
+ memcomp_test_release_prepared(ctx, &store, length, handle,
+ pool_idx);
+ kunit_skip(test, "direct staging slot was unavailable");
+ return;
+ }
+ KUNIT_EXPECT_EQ(test, ret, -EIO);
+ KUNIT_EXPECT_PTR_EQ(test, ctx, NULL);
+ KUNIT_EXPECT_NOT_NULL(test, store.staging);
+ KUNIT_EXPECT_EQ(test, preempt_count(), before);
+ memcomp_test_release_prepared(ctx, &store, length, handle,
+ pool_idx);
+
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_DIRECT);
+ KUNIT_ASSERT_NOT_NULL(test, ctx);
+ length = 0;
+ handle = 0;
+ store.kunit_zs_failures = 2;
+ before = preempt_count();
+ ret = memcomp_prepare_payload_locked(&ctx, src, PAGE_SIZE,
+ &length, &handle, &pool_idx,
+ false, &store, NULL);
+ store.kunit_zs_failures = 0;
+ if (!store.staging) {
+ memcomp_test_release_prepared(ctx, &store, length, handle,
+ pool_idx);
+ kunit_skip(test, "direct staging slot was unavailable");
+ return;
+ }
+ KUNIT_EXPECT_LT(test, ret, 0);
+ KUNIT_EXPECT_PTR_EQ(test, ctx, NULL);
+ KUNIT_EXPECT_NOT_NULL(test, store.staging);
+ KUNIT_EXPECT_EQ(test, preempt_count(), before);
+ memcomp_test_release_prepared(ctx, &store, length, handle,
+ pool_idx);
+
+ for (i = 0; i < nr_staging; i++) {
+ held[i] = memcomp_direct_staging_try_get();
+ if (!held[i])
+ break;
+ held_count++;
+ }
+ if (held_count != nr_staging)
+ goto out_held;
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_DIRECT);
+ KUNIT_EXPECT_NOT_NULL(test, ctx);
+ if (!ctx)
+ goto out_held;
+ ctx_dst = ctx->dst;
+ length = 0;
+ handle = 0;
+ before = preempt_count();
+ ret = memcomp_prepare_payload_locked(&ctx, src, PAGE_SIZE,
+ &length, &handle, &pool_idx,
+ false, &store, NULL);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ KUNIT_EXPECT_NOT_NULL(test, ctx);
+ KUNIT_EXPECT_PTR_EQ(test, store.staging, NULL);
+ KUNIT_EXPECT_EQ(test, preempt_count(), before);
+ KUNIT_EXPECT_EQ(test, ctx->kunit_compress_preempt_count, before);
+ KUNIT_EXPECT_PTR_EQ(test, ctx->kunit_compress_dst, ctx_dst);
+ memcomp_test_release_prepared(ctx, &store, length, handle,
+ pool_idx);
+out_held:
+ for (i = 0; i < held_count; i++)
+ memcomp_direct_staging_put(held[i]);
+}
+
+static void memcomp_test_store_consumes_context(struct kunit *test)
+{
+ struct memcomp_reclaim_ctx *ctx;
+ unsigned int i;
+ bool detached = false;
+ bool success;
+ void *dst;
+ struct folio *folio;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+
+ folio = folio_alloc(GFP_KERNEL, 0);
+ KUNIT_ASSERT_NOT_NULL(test, folio);
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_DIRECT);
+ KUNIT_ASSERT_NOT_NULL(test, ctx);
+ KUNIT_EXPECT_FALSE(test,
+ memcomp_store_after_unmap(folio, &ctx));
+ KUNIT_EXPECT_PTR_EQ(test, ctx, NULL);
+ if (ctx)
+ memcomp_reclaim_ctx_put(ctx);
+ folio_put(folio);
+
+ folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0);
+ KUNIT_ASSERT_NOT_NULL(test, folio);
+ folio_lock(folio);
+ folio->mapping = (struct address_space *)PAGE_MAPPING_ANON;
+ __folio_set_swapbacked(folio);
+ KUNIT_EXPECT_TRUE(test, memcomp_reserve(folio));
+ if (!is_memcomp_entry(folio->swap))
+ goto out_success_folio;
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_DIRECT);
+ KUNIT_EXPECT_NOT_NULL(test, ctx);
+ if (!ctx)
+ goto out_success_rollback;
+ success = memcomp_store_after_unmap(folio, &ctx);
+ KUNIT_EXPECT_TRUE(test, success);
+ KUNIT_EXPECT_PTR_EQ(test, ctx, NULL);
+ if (ctx)
+ memcomp_reclaim_ctx_put(ctx);
+ if (success)
+ memcomp_invalidate_folio(folio);
+ else
+ memcomp_rollback_folio(folio);
+ goto out_success_folio;
+
+out_success_rollback:
+ memcomp_rollback_folio(folio);
+out_success_folio:
+ folio_unlock(folio);
+ folio_put(folio);
+
+ if (IS_ENABLED(CONFIG_PREEMPT_RT) ||
+ !READ_ONCE(memcomp_nr_direct_staging))
+ return;
+ folio = folio_alloc(GFP_KERNEL, 0);
+ KUNIT_ASSERT_NOT_NULL(test, folio);
+ folio_lock(folio);
+ folio->mapping = (struct address_space *)PAGE_MAPPING_ANON;
+ __folio_set_swapbacked(folio);
+ dst = kmap_local_folio(folio, 0);
+ for (i = 0; i < PAGE_SIZE; i++)
+ ((u8 *)dst)[i] = (i >> 5) ^ (i >> 9);
+ kunmap_local(dst);
+ KUNIT_EXPECT_TRUE(test, memcomp_reserve(folio));
+ if (!is_memcomp_entry(folio->swap))
+ goto out_failure_folio;
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_DIRECT);
+ KUNIT_EXPECT_NOT_NULL(test, ctx);
+ if (!ctx)
+ goto out_failure_rollback;
+ ctx->kunit_refresh_commit_error = -EIO;
+ ctx->kunit_detached = &detached;
+ success = memcomp_store_after_unmap(folio, &ctx);
+ if (!detached) {
+ if (ctx)
+ memcomp_reclaim_ctx_put(ctx);
+ memcomp_rollback_folio(folio);
+ kunit_skip(test, "direct staging slot was unavailable");
+ goto out_failure_folio;
+ }
+ KUNIT_EXPECT_FALSE(test, success);
+ KUNIT_EXPECT_PTR_EQ(test, ctx, NULL);
+ if (ctx)
+ memcomp_reclaim_ctx_put(ctx);
+ memcomp_rollback_folio(folio);
+ goto out_failure_folio;
+
+out_failure_rollback:
+ memcomp_rollback_folio(folio);
+out_failure_folio:
+ folio_unlock(folio);
+ folio_put(folio);
+}
+
+static unsigned int memcomp_test_reclaimer_enter(void)
+{
+ unsigned int flags = current->flags & (PF_MEMALLOC | PF_KSWAPD);
+
+ current->flags |= PF_MEMALLOC | PF_KSWAPD;
+ return flags;
+}
+
+static void memcomp_test_reclaimer_exit(unsigned int flags)
+{
+ current_restore_flags(flags, PF_MEMALLOC | PF_KSWAPD);
+}
+
+static void memcomp_test_async_pool(struct kunit *test)
+{
+ struct memcomp_async_pool *pool;
+ struct memcomp_async *a[3], *got[3];
+ unsigned int i;
+
+ pool = kunit_kzalloc(test, sizeof(*pool), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, pool);
+ spin_lock_init(&pool->lock);
+ for (i = 0; i < ARRAY_SIZE(a); i++) {
+ a[i] = kunit_kzalloc(test, sizeof(*a[i]), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, a[i]);
+ init_completion(&a[i]->done);
+ a[i]->id = i;
+ pool->batches[i] = a[i];
+ }
+ pool->free = GENMASK(ARRAY_SIZE(a) - 1, 0);
+ pool->nr = ARRAY_SIZE(a);
+
+ for (i = 0; i < ARRAY_SIZE(got) - 1; i++) {
+ got[i] = memcomp_async_take(pool, 1);
+ KUNIT_ASSERT_PTR_EQ(test, got[i], a[i]);
+ }
+ KUNIT_EXPECT_NULL(test, memcomp_async_take(pool, 1));
+ got[i] = memcomp_async_take(pool, 0);
+ KUNIT_ASSERT_PTR_EQ(test, got[i], a[i]);
+ KUNIT_EXPECT_NULL(test, memcomp_async_take(pool, 0));
+ for (i = 0; i < ARRAY_SIZE(got); i++)
+ memcomp_async_give(pool, got[i]);
+ KUNIT_EXPECT_EQ(test, pool->free, GENMASK(ARRAY_SIZE(a) - 1, 0));
+}
+
+static void memcomp_test_same_filled_plausible(struct kunit *test)
+{
+ const unsigned long e = 0x5a5a12345a5a1234UL;
+ const unsigned int last = PAGE_SIZE / sizeof(unsigned long) - 1;
+ unsigned long *w;
+
+ w = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, w);
+ memset_l(w, e, last + 1);
+
+ KUNIT_EXPECT_TRUE(test, memcomp_same_filled_plausible(w, PAGE_SIZE, e));
+ KUNIT_EXPECT_FALSE(test,
+ memcomp_same_filled_plausible(w, PAGE_SIZE / 2, e));
+
+ w[last / 2] ^= 1;
+ KUNIT_EXPECT_FALSE(test, memcomp_same_filled_plausible(w, PAGE_SIZE, e));
+ w[last / 2] ^= 1;
+
+ w[last] ^= 1;
+ KUNIT_EXPECT_FALSE(test, memcomp_same_filled_plausible(w, PAGE_SIZE, e));
+ w[last] ^= 1;
+ KUNIT_EXPECT_FALSE(test,
+ memcomp_same_filled_plausible(w, PAGE_SIZE, ~e));
+}
+
+static void memcomp_test_ctx_timing_helpers(struct kunit *test)
+{
+ struct memcomp_ctx_timing_record detached = {
+ .timing_start_ns = 10,
+ .timing_lease_end_ns = 110,
+ .timing_pages = 1,
+ .timing_source = MEMCOMP_RECLAIM_DIRECT,
+ .timing_active = true,
+ };
+ struct memcomp_ctx_timing_stats stats = {};
+ struct memcomp_ctx_timing_stats *direct_stats;
+ struct memcomp_ctx_timing_record bucket = {
+ .timing_store_start_ns = 1,
+ .timing_compress_ns = 300,
+ .timing_compress_cpu_ns = 240,
+ .timing_compress_call_cpu_max_ns = 120,
+ .timing_pages = 3,
+ .timing_store_success = true,
+ };
+ struct memcomp_reclaim_ctx sampled = {
+ .timing = {
+ .timing_compress_call_cpu_max_ns = 200,
+ },
+ };
+ struct memcomp_compress_timing_stamp delta = {
+ .wall_ns = 100,
+ .cpu_ns = 80,
+ };
+ struct memcomp_ctx_timing_record ctx = {
+ .timing_active = true,
+ };
+ struct memcomp_ctx_wait_stats *wait_stats;
+ atomic64_t maximum = ATOMIC64_INIT(5);
+ enum memcomp_ctx_timing_kind kind;
+ u64 global_wait_events;
+ s64 lease_ns;
+ s64 samples;
+ u64 source_wait_events;
+ u64 wait_start_ns;
+ unsigned int outcome = MEMCOMP_COMPRESS_TIMING_RAW_HUGE;
+ u32 seq = MEMCOMP_CTX_TIMING_SAMPLE_MASK - 1;
+ u32 seed;
+
+ KUNIT_EXPECT_FALSE(test, memcomp_ctx_timing_should_sample(&seq));
+ KUNIT_EXPECT_TRUE(test, memcomp_ctx_timing_should_sample(&seq));
+ KUNIT_EXPECT_EQ(test, seq,
+ (u32)MEMCOMP_CTX_TIMING_SAMPLE_INTERVAL);
+ seed = memcomp_ctx_timing_seed(1);
+ KUNIT_EXPECT_LT(test, seed,
+ (u32)MEMCOMP_CTX_TIMING_SAMPLE_INTERVAL);
+ KUNIT_EXPECT_NE(test, seed, memcomp_ctx_timing_seed(2));
+ KUNIT_EXPECT_EQ(test,
+ memcomp_ctx_timing_remainder(100, 50), 50ULL);
+ KUNIT_EXPECT_EQ(test,
+ memcomp_ctx_timing_remainder(40, 50), 0ULL);
+ KUNIT_EXPECT_EQ(test,
+ memcomp_ctx_timing_remainder(20, 30), 0ULL);
+ kind = memcomp_ctx_timing_aggregate_kind(MEMCOMP_RECLAIM_DIRECT);
+ KUNIT_EXPECT_EQ(test, kind, MEMCOMP_CTX_TIMING_DIRECT);
+ kind = memcomp_ctx_timing_aggregate_kind(MEMCOMP_RECLAIM_KSWAPD);
+ KUNIT_EXPECT_EQ(test, kind, MEMCOMP_CTX_TIMING_BACKGROUND);
+
+ sampled.timing_seq[MEMCOMP_RECLAIM_KSWAPD] =
+ MEMCOMP_CTX_TIMING_SAMPLE_MASK;
+ memcomp_ctx_timing_start(&sampled, 3,
+ MEMCOMP_RECLAIM_KSWAPD);
+ KUNIT_EXPECT_TRUE(test, sampled.timing.timing_active);
+ KUNIT_EXPECT_EQ(test, sampled.timing.timing_pages, 3U);
+ KUNIT_EXPECT_EQ(test, sampled.timing.timing_source,
+ MEMCOMP_RECLAIM_KSWAPD);
+ KUNIT_EXPECT_EQ(test,
+ sampled.timing.timing_compress_call_cpu_max_ns, 0ULL);
+ KUNIT_EXPECT_EQ(test,
+ sampled.timing_seq[MEMCOMP_RECLAIM_KSWAPD],
+ (u32)MEMCOMP_CTX_TIMING_SAMPLE_INTERVAL);
+ KUNIT_EXPECT_EQ(test,
+ sampled.timing_seq[MEMCOMP_RECLAIM_DIRECT], 0U);
+ sampled.timing.timing_active = false;
+
+ memcomp_atomic64_update_max(&maximum, 3);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&maximum), 5LL);
+ memcomp_atomic64_update_max(&maximum, 9);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&maximum), 9LL);
+ direct_stats =
+ &memcomp_ctx_timing[MEMCOMP_CTX_TIMING_DIRECT];
+ lease_ns = atomic64_read(&direct_stats->lease_ns);
+ samples = atomic64_read(&direct_stats->samples);
+ memcomp_ctx_timing_account(&detached);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&direct_stats->lease_ns),
+ lease_ns + 100);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&direct_stats->samples),
+ samples + 1);
+ KUNIT_EXPECT_FALSE(test, detached.timing_active);
+
+ memcomp_compress_timing_account(&ctx, delta, outcome);
+ KUNIT_EXPECT_EQ(test,
+ ctx.timing_compress_outcome_calls[outcome], 1U);
+ KUNIT_EXPECT_EQ(test,
+ ctx.timing_compress_outcome_ns[outcome], 100ULL);
+ KUNIT_EXPECT_EQ(test,
+ ctx.timing_compress_outcome_cpu_ns[outcome], 80ULL);
+ KUNIT_EXPECT_EQ(test, ctx.timing_compress_call_cpu_max_ns, 80ULL);
+ delta.cpu_ns = 60;
+ memcomp_compress_timing_account(&ctx, delta, outcome);
+ KUNIT_EXPECT_EQ(test, ctx.timing_compress_call_cpu_max_ns, 80ULL);
+ delta.cpu_ns = 120;
+ memcomp_compress_timing_account(&ctx, delta, outcome);
+ delta.cpu_ns = 90;
+ memcomp_compress_timing_account(&ctx, delta, outcome);
+ KUNIT_EXPECT_EQ(test, ctx.timing_compress_call_cpu_max_ns, 120ULL);
+ ctx.timing_active = false;
+ outcome = MEMCOMP_COMPRESS_TIMING_ERROR;
+ memcomp_compress_timing_account(&ctx, delta, outcome);
+ KUNIT_EXPECT_EQ(test,
+ ctx.timing_compress_outcome_calls[outcome], 0U);
+
+ global_wait_events = atomic64_read(&memcomp_comp_ctx_waits);
+ wait_stats =
+ &memcomp_ctx_source_wait[MEMCOMP_RECLAIM_KSWAPD];
+ source_wait_events = atomic64_read(&wait_stats->events);
+ memcomp_ctx_timing_wait_start(MEMCOMP_RECLAIM_KSWAPD);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_comp_ctx_waits),
+ global_wait_events + 1);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&wait_stats->events),
+ source_wait_events + 1);
+ wait_start_ns = ktime_get_ns();
+ memcomp_ctx_timing_wait_end(wait_start_ns,
+ MEMCOMP_RECLAIM_KSWAPD);
+
+ outcome = MEMCOMP_COMPRESS_TIMING_COMPRESSIBLE;
+ bucket.timing_compress_outcome_calls[outcome] = 2;
+ bucket.timing_compress_outcome_ns[outcome] = 300;
+ bucket.timing_compress_outcome_cpu_ns[outcome] = 240;
+ memcomp_ctx_timing_account_bucket(&bucket, &stats, 1000, 100, 50);
+ bucket.timing_store_success = false;
+ bucket.timing_compress_ns = 70;
+ bucket.timing_compress_cpu_ns = 50;
+ bucket.timing_compress_call_cpu_max_ns = 50;
+ bucket.timing_compress_outcome_calls[outcome] = 1;
+ bucket.timing_compress_outcome_ns[outcome] = 70;
+ bucket.timing_compress_outcome_cpu_ns[outcome] = 50;
+ memcomp_ctx_timing_account_bucket(&bucket, &stats, 500, 80, 30);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&stats.samples), 2LL);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&stats.pages), 6LL);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&stats.store_samples), 2LL);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&stats.store_success), 1LL);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&stats.compress_ns), 370LL);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&stats.compress_cpu_ns), 290LL);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&stats.compress_call_cpu_max_ns), 120LL);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&stats.compress_store_success_ns), 300LL);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&stats.compress_store_success_cpu_ns), 240LL);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&stats.compress_store_fail_ns), 70LL);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&stats.compress_store_fail_cpu_ns), 50LL);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&stats.compress_outcome[outcome].calls), 3LL);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&stats.compress_outcome[outcome].wall_ns),
+ 370LL);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&stats.compress_outcome[outcome].cpu_ns),
+ 290LL);
+}
+
+static void memcomp_test_comp_ctx_configuration(struct kunit *test)
+{
+ unsigned int i;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+
+ KUNIT_ASSERT_GT(test, memcomp_nr_comp_ctxs, 0U);
+ KUNIT_EXPECT_LE(test, memcomp_nr_comp_ctxs,
+ MEMCOMP_MAX_COMP_CTXS);
+ KUNIT_EXPECT_LT(test, memcomp_nr_reserved_comp_ctxs,
+ memcomp_nr_comp_ctxs);
+ KUNIT_EXPECT_LE(test, memcomp_nr_borrowable_comp_ctxs,
+ memcomp_nr_reserved_comp_ctxs);
+ if (memcomp_nr_reserved_comp_ctxs)
+ KUNIT_EXPECT_LT(test, memcomp_nr_borrowable_comp_ctxs,
+ memcomp_nr_reserved_comp_ctxs);
+ KUNIT_EXPECT_EQ(test, hweight_long(memcomp_comp_ctx_all),
+ (unsigned long)memcomp_nr_comp_ctxs);
+ KUNIT_EXPECT_EQ(test, hweight_long(memcomp_comp_ctx_foreground),
+ (unsigned long)(memcomp_nr_comp_ctxs -
+ memcomp_nr_reserved_comp_ctxs));
+ for (i = 0; i < memcomp_nr_comp_ctxs; i++) {
+ KUNIT_EXPECT_NOT_NULL(test, memcomp_comp_ctxs[i].tfm);
+ KUNIT_EXPECT_NOT_NULL(test, memcomp_comp_ctxs[i].dst);
+ }
+}
+
+static void memcomp_test_stock_transforms(struct kunit *test)
+{
+ struct crypto_comp *tfm;
+ const char *driver;
+ unsigned int i;
+ int cpu;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+
+ KUNIT_ASSERT_GT(test, memcomp_nr_comp_ctxs, 0U);
+ tfm = crypto_alloc_comp(memcomp_compressor, 0, 0);
+ KUNIT_ASSERT_FALSE(test, IS_ERR(tfm));
+ driver = crypto_tfm_alg_driver_name(crypto_comp_tfm(tfm));
+ for (i = 0; i < memcomp_nr_comp_ctxs; i++) {
+ struct crypto_comp *comp = memcomp_comp_ctxs[i].tfm;
+
+ KUNIT_EXPECT_NOT_NULL(test, comp);
+ if (comp)
+ KUNIT_EXPECT_STREQ(test,
+ crypto_tfm_alg_driver_name(crypto_comp_tfm(comp)),
+ driver);
+ }
+ for_each_possible_cpu(cpu) {
+ struct crypto_comp *decomp =
+ per_cpu(memcomp_decomp_ctxs, cpu).tfm;
+
+ KUNIT_EXPECT_NOT_NULL(test, decomp);
+ if (decomp)
+ KUNIT_EXPECT_STREQ(test,
+ crypto_tfm_alg_driver_name(crypto_comp_tfm(decomp)),
+ driver);
+ }
+ crypto_free_comp(tfm);
+}
+
+static void memcomp_test_zs_gfp(struct kunit *test)
+{
+ gfp_t fast = memcomp_zs_gfp(true);
+ gfp_t slow = memcomp_zs_gfp(false);
+
+ KUNIT_EXPECT_FALSE(test, fast & __GFP_DIRECT_RECLAIM);
+ KUNIT_EXPECT_TRUE(test, fast & __GFP_KSWAPD_RECLAIM);
+ KUNIT_EXPECT_FALSE(test, fast & (__GFP_IO | __GFP_FS));
+ KUNIT_EXPECT_TRUE(test, slow & __GFP_DIRECT_RECLAIM);
+ KUNIT_EXPECT_TRUE(test, slow & __GFP_KSWAPD_RECLAIM);
+ KUNIT_EXPECT_FALSE(test, slow & (__GFP_IO | __GFP_FS));
+}
+
+static void memcomp_test_reclaim_ctx_admission(struct kunit *test)
+{
+ struct memcomp_reclaim_ctx *ctx;
+ s64 busy_pages;
+ s64 all_busy_pages;
+ s64 ctx_busy;
+ s64 reserved_free_pages;
+ s64 store_fail;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+
+ ctx_busy = atomic64_read(&memcomp_ctx_busy);
+ reserved_free_pages =
+ atomic64_read(&memcomp_reclaim_ctx_busy_reserved_free_pages);
+ all_busy_pages =
+ atomic64_read(&memcomp_reclaim_ctx_busy_all_busy_pages);
+ busy_pages = reserved_free_pages + all_busy_pages;
+ store_fail = atomic64_read(&memcomp_bg_store_fail) +
+ atomic64_read(&memcomp_direct_store_fail) +
+ atomic64_read(&memcomp_pageout_store_fail);
+ atomic_set(&memcomp_kunit_ctx_get_fail, 1);
+ ctx = memcomp_reclaim_ctx_get(3, MEMCOMP_RECLAIM_DIRECT);
+ atomic_set(&memcomp_kunit_ctx_get_fail, 0);
+ KUNIT_EXPECT_PTR_EQ(test, ctx, NULL);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_ctx_busy),
+ ctx_busy + 1);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_reclaim_ctx_busy_reserved_free_pages) +
+ atomic64_read(&memcomp_reclaim_ctx_busy_all_busy_pages),
+ busy_pages + 3);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_reclaim_ctx_busy_reserved_free_pages),
+ reserved_free_pages);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_reclaim_ctx_busy_all_busy_pages),
+ all_busy_pages + 3);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_bg_store_fail) +
+ atomic64_read(&memcomp_direct_store_fail) +
+ atomic64_read(&memcomp_pageout_store_fail),
+ store_fail + 3);
+}
+
+static void memcomp_test_reclaim_ctx_precheck(struct kunit *test)
+{
+ struct reclaim_state reclaim_state = {};
+ struct reclaim_state *old_reclaim_state;
+ enum memcomp_comp_ctx_miss miss =
+ MEMCOMP_COMP_CTX_BORROW_LIMIT;
+ s64 all_busy_pages;
+ s64 borrow_limit_pages;
+ s64 borrow_waiter_pages;
+ s64 bg_fail;
+ s64 ctx_busy;
+ s64 direct_fail;
+ s64 early_reject;
+ s64 early_reject_pages;
+ s64 pageout_fail;
+ s64 reserved_free_pages;
+ bool admitted;
+
+ ctx_busy = atomic64_read(&memcomp_ctx_busy);
+ reserved_free_pages =
+ atomic64_read(&memcomp_reclaim_ctx_busy_reserved_free_pages);
+ all_busy_pages =
+ atomic64_read(&memcomp_reclaim_ctx_busy_all_busy_pages);
+ borrow_limit_pages =
+ atomic64_read(&memcomp_reclaim_ctx_borrow_limit_pages);
+ borrow_waiter_pages =
+ atomic64_read(&memcomp_reclaim_ctx_borrow_waiter_pages);
+ bg_fail = atomic64_read(&memcomp_bg_store_fail);
+ direct_fail = atomic64_read(&memcomp_direct_store_fail);
+ pageout_fail = atomic64_read(&memcomp_pageout_store_fail);
+ early_reject = atomic64_read(&memcomp_reclaim_ctx_early_reject);
+ early_reject_pages =
+ atomic64_read(&memcomp_reclaim_ctx_early_reject_pages);
+
+ admitted = memcomp_reclaim_ctx_precheck(3,
+ MEMCOMP_RECLAIM_KSWAPD);
+ KUNIT_EXPECT_TRUE(test, admitted);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_ctx_busy), ctx_busy);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_reclaim_ctx_early_reject),
+ early_reject);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_reclaim_ctx_early_reject_pages),
+ early_reject_pages);
+
+ old_reclaim_state = current->reclaim_state;
+ current->reclaim_state = &reclaim_state;
+ admitted = memcomp_reclaim_ctx_precheck_result(3, miss);
+ current->reclaim_state = old_reclaim_state;
+ KUNIT_EXPECT_FALSE(test, admitted);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_ctx_busy),
+ ctx_busy + 1);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_reclaim_ctx_busy_reserved_free_pages),
+ reserved_free_pages + 3);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_reclaim_ctx_busy_all_busy_pages),
+ all_busy_pages);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_reclaim_ctx_borrow_limit_pages),
+ borrow_limit_pages + 3);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_reclaim_ctx_borrow_waiter_pages),
+ borrow_waiter_pages);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_bg_store_fail),
+ bg_fail);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_direct_store_fail),
+ direct_fail + 3);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_pageout_store_fail),
+ pageout_fail);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_reclaim_ctx_early_reject),
+ early_reject + 1);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_reclaim_ctx_early_reject_pages),
+ early_reject_pages + 3);
+}
+
+static void memcomp_test_reserve_eligibility(struct kunit *test)
+{
+ struct folio *folio;
+
+ folio = folio_alloc(GFP_KERNEL, 0);
+ KUNIT_ASSERT_NOT_NULL(test, folio);
+ KUNIT_EXPECT_FALSE(test, memcomp_reserve_eligible(folio));
+ folio_lock(folio);
+ KUNIT_EXPECT_FALSE(test, memcomp_reserve_eligible(folio));
+ folio->mapping = (struct address_space *)PAGE_MAPPING_ANON;
+ __folio_set_swapbacked(folio);
+ KUNIT_EXPECT_FALSE(test, folio_test_ksm(folio));
+ KUNIT_EXPECT_TRUE(test, memcomp_reserve_eligible(folio));
+#ifdef CONFIG_KSM
+ /*
+ * Eligibility only reads the mapping flags. No stable-tree node or
+ * real KSM mapping is needed to check that KSM folios are rejected.
+ * Restore mapping before releasing the synthetic folio below.
+ */
+ folio->mapping = (void *)PAGE_MAPPING_KSM;
+ KUNIT_EXPECT_TRUE(test, folio_test_ksm(folio));
+ KUNIT_EXPECT_FALSE(test, memcomp_reserve_eligible(folio));
+#endif
+ folio->mapping = NULL;
+ folio_clear_swapbacked(folio);
+ folio_unlock(folio);
+ folio_put(folio);
+}
+
+static void memcomp_test_admission_rollback_unescaped(struct kunit *test)
+{
+ struct memcomp_reclaim_ctx *ctx;
+ struct folio *folio;
+ unsigned long token;
+ s64 pending;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+
+ pending = atomic64_read(&memcomp_pending_pages);
+ folio = folio_alloc(GFP_KERNEL, 0);
+ KUNIT_ASSERT_NOT_NULL(test, folio);
+ folio_lock(folio);
+ folio->mapping = (struct address_space *)PAGE_MAPPING_ANON;
+ __folio_set_swapbacked(folio);
+ KUNIT_ASSERT_TRUE(test, memcomp_reserve(folio));
+ token = swp_offset(folio->swap);
+ KUNIT_ASSERT_NE(test, token, 0UL);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_pending_pages),
+ pending + 1);
+
+ atomic_set(&memcomp_kunit_ctx_get_fail, 1);
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_DIRECT);
+ atomic_set(&memcomp_kunit_ctx_get_fail, 0);
+ KUNIT_EXPECT_PTR_EQ(test, ctx, NULL);
+ memcomp_rollback_folio(folio);
+ KUNIT_EXPECT_EQ(test, folio->swap.val, 0UL);
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_present(token));
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_pending_pages),
+ pending);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1);
+
+ folio->mapping = NULL;
+ folio_clear_swapbacked(folio);
+ folio_unlock(folio);
+ folio_put(folio);
+}
+
+static void memcomp_test_samefill(struct kunit *test)
+{
+ unsigned long *page;
+ unsigned long element = 0;
+ unsigned int i;
+
+ page = kunit_kzalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, page);
+ KUNIT_EXPECT_TRUE(test, page_same_filled(page, &element));
+ KUNIT_EXPECT_EQ(test, element, 0UL);
+
+ for (i = 0; i < PAGE_SIZE / sizeof(*page); i++)
+ page[i] = 0xa5a5a5a5a5a5a5a5UL;
+ KUNIT_EXPECT_TRUE(test, page_same_filled(page, &element));
+ KUNIT_EXPECT_EQ(test, element, 0xa5a5a5a5a5a5a5a5UL);
+
+ page[PAGE_SIZE / sizeof(*page) - 1] ^= 1;
+ KUNIT_EXPECT_FALSE(test, page_same_filled(page, &element));
+
+ /*
+ * A single differing word must be found wherever it sits: at either
+ * probe position, at the midpoint probe, and at offsets the probes do
+ * not cover so the full scan still has to run.
+ */
+ {
+ const unsigned int words = PAGE_SIZE / sizeof(*page);
+ static const unsigned int frac[] = { 1, 2, 3, 4, 5, 7 };
+ unsigned int f;
+
+ for (i = 0; i < words; i++)
+ page[i] = 0xa5a5a5a5a5a5a5a5UL;
+ page[1] ^= 1;
+ KUNIT_EXPECT_FALSE(test, page_same_filled(page, &element));
+ page[1] ^= 1;
+
+ page[(words - 1) / 2] ^= 1;
+ KUNIT_EXPECT_FALSE(test, page_same_filled(page, &element));
+ page[(words - 1) / 2] ^= 1;
+ KUNIT_EXPECT_TRUE(test, page_same_filled(page, &element));
+
+ for (f = 0; f < ARRAY_SIZE(frac); f++) {
+ unsigned int at = words / frac[f];
+
+ if (at >= words)
+ at = words - 1;
+ page[at] ^= 1;
+ KUNIT_EXPECT_FALSE(test, page_same_filled(page,
+ &element));
+ page[at] ^= 1;
+ }
+ KUNIT_EXPECT_TRUE(test, page_same_filled(page, &element));
+ KUNIT_EXPECT_EQ(test, element, 0xa5a5a5a5a5a5a5a5UL);
+
+ /*
+ * A long leading run followed by a mismatch is the case the
+ * probe exists for; it must still be rejected.
+ */
+ page[words - 2] ^= 1;
+ KUNIT_EXPECT_FALSE(test, page_same_filled(page, &element));
+ }
+}
+
+static void memcomp_test_raw_payload_threshold(struct kunit *test)
+{
+ enum memcomp_raw_reason reason;
+
+ reason = memcomp_payload_raw_reason(2048, PAGE_SIZE, 3264);
+ KUNIT_EXPECT_EQ(test, reason, MEMCOMP_RAW_NONE);
+ reason = memcomp_payload_raw_reason(3263, PAGE_SIZE, 3264);
+ KUNIT_EXPECT_EQ(test, reason, MEMCOMP_RAW_NONE);
+ reason = memcomp_payload_raw_reason(3264, PAGE_SIZE, 3264);
+ KUNIT_EXPECT_EQ(test, reason, MEMCOMP_RAW_HUGE_CLASS);
+ reason = memcomp_payload_raw_reason(PAGE_SIZE - 1, PAGE_SIZE, 3264);
+ KUNIT_EXPECT_EQ(test, reason, MEMCOMP_RAW_HUGE_CLASS);
+ reason = memcomp_payload_raw_reason(PAGE_SIZE, PAGE_SIZE, 3264);
+ KUNIT_EXPECT_EQ(test, reason, MEMCOMP_RAW_GE_PAGE);
+ reason = memcomp_payload_raw_reason(PAGE_SIZE - 1, PAGE_SIZE, 0);
+ KUNIT_EXPECT_EQ(test, reason, MEMCOMP_RAW_NONE);
+}
+
+static void memcomp_test_compressor_e2big(struct kunit *test)
+{
+ struct memcomp_reclaim_ctx *ctx;
+ struct memcomp_store_ctx store = {};
+ unsigned int reclaim_flags;
+ unsigned int length;
+ unsigned long handle;
+ s64 reject_ge, fail;
+ u8 pool_idx;
+ const u8 *payload;
+ u8 *src;
+ int ret;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+ src = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ memcomp_test_fill_zstd_ratio_page(src);
+
+ reclaim_flags = memcomp_test_reclaimer_enter();
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_KSWAPD);
+ KUNIT_ASSERT_NOT_NULL(test, ctx);
+ reject_ge = atomic64_read(&memcomp_raw_reject_ge_page);
+ fail = atomic64_read(&memcomp_compress_fail);
+ length = 0;
+ handle = 0;
+ pool_idx = 0;
+ ctx->kunit_compress_error = -E2BIG;
+ ret = memcomp_prepare_payload_locked(&ctx, src, PAGE_SIZE,
+ &length, &handle, &pool_idx,
+ false, NULL, NULL);
+ KUNIT_EXPECT_EQ(test, ret, -E2BIG);
+ KUNIT_EXPECT_EQ(test, length, 0U);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_raw_reject_ge_page),
+ reject_ge + 1);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_compress_fail), fail);
+
+ ctx->kunit_compress_error = -E2BIG;
+ ret = memcomp_prepare_payload_locked(&ctx, src, PAGE_SIZE,
+ &length, &handle, &pool_idx,
+ true, NULL, NULL);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ KUNIT_EXPECT_EQ(test, length, (unsigned int)PAGE_SIZE);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_compress_fail), fail);
+ if (!ret && length == PAGE_SIZE) {
+ struct zs_pool *pool = memcomp_pools[pool_idx];
+
+ payload = zs_map_object(pool, handle, ZS_MM_RO);
+ KUNIT_EXPECT_NOT_NULL(test, payload);
+ if (payload) {
+ KUNIT_EXPECT_MEMEQ(test, payload, src, PAGE_SIZE);
+ zs_unmap_object(pool, handle);
+ }
+ }
+ memcomp_test_release_prepared(ctx, &store, ret ? 0 : length,
+ handle, pool_idx);
+ memcomp_test_reclaimer_exit(reclaim_flags);
+}
+
+static void memcomp_test_mixed_raw_net_reclaim(struct kunit *test)
+{
+ unsigned int lengths[MEMCOMP_MAX_COMPRESS_PAGES];
+ bool reclaimable;
+ unsigned int i;
+
+ lengths[0] = PAGE_SIZE;
+ KUNIT_EXPECT_FALSE(test, memcomp_payloads_net_reclaimable(lengths, 1));
+
+ lengths[0] = PAGE_SIZE;
+ lengths[1] = 0;
+ KUNIT_EXPECT_TRUE(test, memcomp_payloads_net_reclaimable(lengths, 2));
+
+ lengths[0] = PAGE_SIZE;
+ lengths[1] = PAGE_SIZE;
+ KUNIT_EXPECT_FALSE(test, memcomp_payloads_net_reclaimable(lengths, 2));
+
+ lengths[0] = PAGE_SIZE;
+ lengths[1] = PAGE_SIZE;
+ lengths[2] = 1024;
+ lengths[3] = 1024;
+ KUNIT_EXPECT_TRUE(test, memcomp_payloads_net_reclaimable(lengths, 4));
+
+ lengths[0] = PAGE_SIZE;
+ lengths[1] = PAGE_SIZE;
+ lengths[2] = PAGE_SIZE;
+ lengths[3] = 0;
+ KUNIT_EXPECT_TRUE(test, memcomp_payloads_net_reclaimable(lengths, 4));
+
+ lengths[0] = PAGE_SIZE;
+ lengths[1] = PAGE_SIZE;
+ lengths[2] = PAGE_SIZE;
+ lengths[3] = 1;
+ KUNIT_EXPECT_FALSE(test, memcomp_payloads_net_reclaimable(lengths, 4));
+
+ lengths[0] = 3263;
+ KUNIT_EXPECT_TRUE(test, memcomp_payloads_net_reclaimable(lengths, 1));
+ lengths[0] = PAGE_SIZE + 1;
+ KUNIT_EXPECT_FALSE(test, memcomp_payloads_net_reclaimable(lengths, 1));
+
+ for (i = 0; i < MEMCOMP_MAX_COMPRESS_PAGES; i++)
+ lengths[i] = 1024;
+ lengths[0] = PAGE_SIZE;
+ reclaimable = memcomp_payloads_net_reclaimable(lengths,
+ ARRAY_SIZE(lengths));
+ KUNIT_EXPECT_TRUE(test, reclaimable);
+
+ for (i = 0; i < MEMCOMP_MAX_COMPRESS_PAGES; i++)
+ lengths[i] = PAGE_SIZE;
+ reclaimable = memcomp_payloads_net_reclaimable(lengths,
+ ARRAY_SIZE(lengths));
+ KUNIT_EXPECT_FALSE(test, reclaimable);
+}
+
+static void memcomp_test_state_signed_sum(struct kunit *test)
+{
+ const struct memcomp_state_stats samples[] = {
+ {
+ .folio_owner_pages = 7,
+ .samefill_pages = -2,
+ .raw_pages = 2,
+ .shmem_pages = 6,
+ },
+ {
+ .folio_owner_pages = -4,
+ .samefill_pages = 1,
+ .raw_pages = -3,
+ .shmem_pages = -1,
+ },
+ };
+ struct memcomp_state_stats snapshot = {};
+ int i;
+
+#ifndef CONFIG_MEMCOMP_DEBUG_STATS
+ kunit_skip(test, "state gauges require CONFIG_MEMCOMP_DEBUG_STATS");
+ return;
+#else
+ for (i = 0; i < ARRAY_SIZE(samples); i++)
+ memcomp_state_add(&snapshot, &samples[i]);
+ /* Aggregate signed deltas first, then clamp once, not per CPU. */
+ KUNIT_EXPECT_EQ(test, snapshot.folio_owner_pages, (s64)3);
+ KUNIT_EXPECT_EQ(test, snapshot.samefill_pages, (s64)-1);
+ KUNIT_EXPECT_EQ(test, snapshot.raw_pages, (s64)-1);
+ KUNIT_EXPECT_EQ(test, snapshot.shmem_pages, (s64)5);
+ memcomp_state_clamp(&snapshot);
+ KUNIT_EXPECT_EQ(test, snapshot.folio_owner_pages, (s64)3);
+ KUNIT_EXPECT_EQ(test, snapshot.samefill_pages, (s64)0);
+ KUNIT_EXPECT_EQ(test, snapshot.raw_pages, (s64)0);
+ KUNIT_EXPECT_EQ(test, snapshot.shmem_pages, (s64)5);
+#endif
+}
+
+static void memcomp_test_fill_incompressible(void *buf,
+ unsigned int nr_pages)
+{
+ u32 *words = buf;
+ u32 value = 0x9e3779b9;
+ unsigned int i;
+
+ for (i = 0; i < nr_pages * PAGE_SIZE / sizeof(*words); i++) {
+ value ^= value << 13;
+ value ^= value >> 17;
+ value ^= value << 5;
+ words[i] = value;
+ }
+}
+
+static void memcomp_test_incompressible_mark_lifetime(struct kunit *test)
+{
+ struct folio *folio, *dst;
+ s64 changed;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+ changed = atomic64_read(&memcomp_incompressible_changed_pages);
+ folio = folio_alloc(GFP_KERNEL, 0);
+ KUNIT_ASSERT_NOT_NULL(test, folio);
+ dst = folio_alloc(GFP_KERNEL, 0);
+ if (!dst)
+ folio_put(folio);
+ KUNIT_ASSERT_NOT_NULL(test, dst);
+ memcomp_test_fill_incompressible(folio_address(folio), 1);
+ folio_lock(folio);
+ folio_lock(dst);
+ folio->mapping = (struct address_space *)PAGE_MAPPING_ANON;
+ __folio_set_swapbacked(folio);
+ dst->mapping = (struct address_space *)PAGE_MAPPING_ANON;
+ __folio_set_swapbacked(dst);
+
+ folio_set_incompressible(folio, memcomp_folio_hash(folio));
+ KUNIT_EXPECT_TRUE(test, folio_test_incompressible(folio));
+ KUNIT_EXPECT_FALSE(test, folio_memcomp_reserved(folio));
+ KUNIT_EXPECT_FALSE(test, memcomp_release_folio(folio));
+ memcomp_invalidate_folio(folio);
+ memcomp_rollback_folio(folio);
+ KUNIT_EXPECT_TRUE(test, folio_test_incompressible(folio));
+ KUNIT_EXPECT_TRUE(test, memcomp_folio_incompressible(folio));
+ KUNIT_EXPECT_TRUE(test, folio_test_incompressible(folio));
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_incompressible_changed_pages),
+ changed);
+
+ KUNIT_EXPECT_EQ(test, memcomp_migrate_folio(dst, folio), 0);
+ KUNIT_EXPECT_EQ(test, dst->swap.val, folio->swap.val);
+ dst->swap.val = 0;
+
+ /* Written to since. */
+ ((u8 *)folio_address(folio))[PAGE_SIZE - 1] ^= 0x80;
+ KUNIT_EXPECT_FALSE(test, memcomp_folio_incompressible(folio));
+ KUNIT_EXPECT_FALSE(test, folio_test_incompressible(folio));
+ KUNIT_EXPECT_EQ(test, folio->swap.val, 0UL);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_incompressible_changed_pages),
+ changed + 1);
+ KUNIT_EXPECT_EQ(test, memcomp_migrate_folio(dst, folio), 0);
+ KUNIT_EXPECT_EQ(test, dst->swap.val, 0UL);
+
+ /* Reserved again. */
+ folio_set_incompressible(folio, memcomp_folio_hash(folio));
+ KUNIT_ASSERT_TRUE(test, memcomp_reserve(folio));
+ KUNIT_EXPECT_TRUE(test, folio_memcomp_reserved(folio));
+ KUNIT_EXPECT_FALSE(test, folio_test_incompressible(folio));
+ memcomp_rollback_folio(folio);
+ KUNIT_EXPECT_EQ(test, folio->swap.val, 0UL);
+
+ dst->mapping = NULL;
+ folio_clear_swapbacked(dst);
+ folio_unlock(dst);
+ folio_put(dst);
+ folio->mapping = NULL;
+ folio_clear_swapbacked(folio);
+ folio_unlock(folio);
+ folio_put(folio);
+}
+
+static struct folio *memcomp_test_user_folio(struct kunit *test,
+ unsigned long *addr)
+{
+ unsigned long *buf;
+ struct folio *folio;
+ struct page *page;
+ unsigned int i;
+
+ *addr = kunit_vm_mmap(test, NULL, 0, PAGE_SIZE, PROT_READ | PROT_WRITE,
+ MAP_ANONYMOUS | MAP_PRIVATE, 0);
+ if (IS_ERR_VALUE(*addr) || !*addr)
+ return NULL;
+
+ buf = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ if (!buf)
+ return NULL;
+ for (i = 0; i < PAGE_SIZE / sizeof(*buf); i++)
+ buf[i] = MEMCOMP_TEST_USER_PATTERN + i;
+ if (copy_to_user((void __user *)*addr, buf, PAGE_SIZE))
+ return NULL;
+
+ if (get_user_pages_fast(*addr, 1, FOLL_WRITE, &page) != 1)
+ return NULL;
+ folio = page_folio(page);
+ /* No per-CPU batch may hold a reference the store would count. */
+ lru_add_drain();
+ folio_lock(folio);
+ return folio;
+}
+
+static bool memcomp_test_user_intact(unsigned long addr)
+{
+ unsigned long val;
+ unsigned int i;
+
+ for (i = 0; i < PAGE_SIZE / sizeof(val); i++) {
+ if (get_user(val, (unsigned long __user *)addr + i))
+ return false;
+ if (val != MEMCOMP_TEST_USER_PATTERN + i)
+ return false;
+ }
+ return true;
+}
+
+#if (defined(CONFIG_ADVISE_SYSCALLS))
+static void memcomp_test_lazyfree_mark_discarded(struct kunit *test)
+{
+ struct folio *folio;
+ unsigned long addr;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+ folio = memcomp_test_user_folio(test, &addr);
+ KUNIT_ASSERT_NOT_NULL(test, folio);
+ folio_set_incompressible(folio, memcomp_folio_hash(folio));
+
+ /* MADV_FREE locks a dirty folio to clean it. */
+ folio_unlock(folio);
+ KUNIT_ASSERT_EQ(test, do_madvise(current->mm, addr, PAGE_SIZE,
+ MADV_FREE), 0);
+ lru_add_drain();
+ folio_lock(folio);
+ KUNIT_ASSERT_FALSE(test, folio_test_swapbacked(folio));
+ KUNIT_ASSERT_FALSE(test, folio_test_dirty(folio));
+ KUNIT_ASSERT_TRUE(test, folio_test_incompressible(folio));
+
+ try_to_unmap(folio, 0);
+ KUNIT_EXPECT_FALSE(test, folio_mapped(folio));
+ KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio));
+ folio_unlock(folio);
+ folio_put(folio);
+}
+#endif
+
+static void memcomp_test_rollback_restores_ptes(struct kunit *test)
+{
+ s64 spans, ptes, missed, pending;
+ struct anon_vma *anon_vma;
+ struct folio *folio;
+ unsigned long token;
+ unsigned long addr;
+ int refs;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+ folio = memcomp_test_user_folio(test, &addr);
+ KUNIT_ASSERT_NOT_NULL(test, folio);
+ anon_vma = folio_anon_vma(folio);
+ KUNIT_ASSERT_NOT_NULL(test, anon_vma);
+ refs = atomic_read(&anon_vma->refcount);
+ spans = atomic64_read(&memcomp_restore_spans);
+ ptes = atomic64_read(&memcomp_restore_ptes);
+ missed = atomic64_read(&memcomp_restore_missed_pages);
+ pending = atomic64_read(&memcomp_pending_pages);
+
+ KUNIT_ASSERT_TRUE(test, PageAnonExclusive(&folio->page));
+ KUNIT_ASSERT_TRUE(test, memcomp_reserve(folio));
+ token = swp_offset(folio->swap);
+ KUNIT_EXPECT_TRUE(test, memcomp_test_head_holds_anon_vma(token));
+ KUNIT_EXPECT_EQ(test, atomic_read(&anon_vma->refcount), refs + 1);
+
+ try_to_unmap(folio, 0);
+ KUNIT_ASSERT_FALSE(test, folio_mapped(folio));
+ KUNIT_EXPECT_FALSE(test, PageAnonExclusive(&folio->page));
+ KUNIT_EXPECT_TRUE(test, memcomp_entry_present(token));
+
+ memcomp_rollback_folio(folio);
+ KUNIT_EXPECT_TRUE(test, folio_mapped(folio));
+ /* Exclusive again, as the token PTE said; a write would not COW. */
+ KUNIT_EXPECT_TRUE(test, PageAnonExclusive(&folio->page));
+ KUNIT_EXPECT_EQ(test, folio->swap.val, 0UL);
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_present(token));
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_restore_spans),
+ spans + 1);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_restore_ptes),
+ ptes + 1);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_restore_missed_pages),
+ missed);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_pending_pages),
+ pending);
+ KUNIT_EXPECT_EQ(test, atomic_read(&anon_vma->refcount), refs);
+ /* The mapping's reference and ours. */
+ KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 2);
+ folio_unlock(folio);
+ folio_put(folio);
+
+ KUNIT_EXPECT_TRUE(test, memcomp_test_user_intact(addr));
+}
+
+static void memcomp_test_store_drops_anon_vma(struct kunit *test)
+{
+ struct anon_vma *anon_vma;
+ struct folio *folio;
+ unsigned long token;
+ unsigned long addr;
+ int refs;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+ folio = memcomp_test_user_folio(test, &addr);
+ KUNIT_ASSERT_NOT_NULL(test, folio);
+ anon_vma = folio_anon_vma(folio);
+ KUNIT_ASSERT_NOT_NULL(test, anon_vma);
+ refs = atomic_read(&anon_vma->refcount);
+
+ KUNIT_ASSERT_TRUE(test, memcomp_reserve(folio));
+ token = swp_offset(folio->swap);
+ KUNIT_EXPECT_EQ(test, atomic_read(&anon_vma->refcount), refs + 1);
+ try_to_unmap(folio, 0);
+ KUNIT_ASSERT_FALSE(test, folio_mapped(folio));
+
+ if (!memcomp_test_store_after_unmap(folio)) {
+ memcomp_rollback_folio(folio);
+ folio_unlock(folio);
+ folio_put(folio);
+ KUNIT_FAIL(test, "failed to store the user page");
+ return;
+ }
+ KUNIT_EXPECT_FALSE(test, memcomp_test_head_holds_anon_vma(token));
+ KUNIT_EXPECT_EQ(test, atomic_read(&anon_vma->refcount), refs);
+
+ /* Released the way reclaim frees a stored folio. */
+ memcomp_invalidate_folio(folio);
+ KUNIT_EXPECT_EQ(test, atomic_read(&anon_vma->refcount), refs);
+ folio_unlock(folio);
+ folio_put(folio);
+
+ KUNIT_EXPECT_TRUE(test, memcomp_test_user_intact(addr));
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_present(token));
+}
+
+static void memcomp_test_invalidate_drops_anon_vma(struct kunit *test)
+{
+ struct anon_vma *anon_vma;
+ struct folio *folio;
+ unsigned long token;
+ unsigned long addr;
+ int refs;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+ folio = memcomp_test_user_folio(test, &addr);
+ KUNIT_ASSERT_NOT_NULL(test, folio);
+ anon_vma = folio_anon_vma(folio);
+ KUNIT_ASSERT_NOT_NULL(test, anon_vma);
+ refs = atomic_read(&anon_vma->refcount);
+
+ KUNIT_ASSERT_TRUE(test, memcomp_reserve(folio));
+ token = swp_offset(folio->swap);
+ KUNIT_EXPECT_EQ(test, atomic_read(&anon_vma->refcount), refs + 1);
+ memcomp_invalidate_folio(folio);
+ KUNIT_EXPECT_EQ(test, folio->swap.val, 0UL);
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_present(token));
+ KUNIT_EXPECT_EQ(test, atomic_read(&anon_vma->refcount), refs);
+ folio_unlock(folio);
+ folio_put(folio);
+}
+
+static void memcomp_test_store_cache_rejects_mapped(struct kunit *test)
+{
+ struct folio *folio;
+ unsigned long token = 0;
+ s64 eligibility;
+ s64 fail;
+ s64 payload;
+ s64 arch;
+
+ folio = folio_alloc(GFP_KERNEL, 0);
+ KUNIT_ASSERT_NOT_NULL(test, folio);
+ folio_lock(folio);
+ eligibility = atomic64_read(&memcomp_shmem_eligibility_fail_pages);
+ payload = atomic64_read(&memcomp_shmem_payload_fail_pages);
+ arch = atomic64_read(&memcomp_shmem_arch_fail_pages);
+ fail = atomic64_read(&memcomp_shmem_store_fail_pages);
+
+ /* An initial -1 mapcount represents no mappings; zero represents one. */
+ atomic_set(&folio->_mapcount, 0);
+ KUNIT_EXPECT_FALSE(test, memcomp_store_cache(folio, &token));
+ KUNIT_EXPECT_EQ(test, token, 0UL);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_shmem_eligibility_fail_pages),
+ eligibility + 1);
+ KUNIT_EXPECT_EQ(test,
+ atomic64_read(&memcomp_shmem_payload_fail_pages),
+ payload);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_shmem_arch_fail_pages),
+ arch);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_shmem_store_fail_pages),
+ fail + 1);
+
+ atomic_set(&folio->_mapcount, -1);
+ folio_unlock(folio);
+ folio_put(folio);
+}
+
+#if (defined(CONFIG_SWAP))
+static void memcomp_test_writeback_pack_fits(struct kunit *test)
+{
+ const unsigned int max = MEMCOMP_WRITEBACK_PACK_MAX;
+
+ KUNIT_EXPECT_TRUE(test, memcomp_writeback_pack_fits(0, 0, 1));
+ KUNIT_EXPECT_TRUE(test, memcomp_writeback_pack_fits(0, 0,
+ PAGE_SIZE - 1));
+ KUNIT_EXPECT_FALSE(test, memcomp_writeback_pack_fits(0, 0,
+ PAGE_SIZE));
+ /* Exactly filling the page is a fit; one byte more is not. */
+ KUNIT_EXPECT_TRUE(test, memcomp_writeback_pack_fits(1, 1000,
+ PAGE_SIZE - 1000));
+ KUNIT_EXPECT_FALSE(test, memcomp_writeback_pack_fits(1, 1000,
+ PAGE_SIZE - 999));
+ /* The payload count caps a page of tiny payloads. */
+ KUNIT_EXPECT_TRUE(test, memcomp_writeback_pack_fits(max - 1,
+ (max - 1) * 8,
+ 8));
+ KUNIT_EXPECT_FALSE(test, memcomp_writeback_pack_fits(max, max * 8,
+ 8));
+ /* And the batch always leaves room for a close plus a write. */
+ KUNIT_EXPECT_LT(test, (unsigned long)MEMCOMP_WRITEBACK_PACK_MAX,
+ (unsigned long)SWAP_MAP_MAX);
+}
+#endif
+
+#if (defined(CONFIG_SWAP))
+static u8 *memcomp_test_packed_slot(struct kunit *test, u8 **pages,
+ unsigned int nr, unsigned int *offs,
+ unsigned int *lens)
+{
+ static const char text[] = "memcomp packed slot ";
+ unsigned int off = 0;
+ unsigned int i, j;
+ u8 *slot;
+
+ slot = kunit_kzalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, slot);
+ for (i = 0; i < nr; i++) {
+ unsigned long handle;
+ struct zs_pool *pool;
+ u8 pool_idx = 0;
+ const u8 *src;
+ int ret;
+
+ pages[i] = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, pages[i]);
+ /* Text-like and distinct per page, so each compresses well. */
+ for (j = 0; j < PAGE_SIZE; j++)
+ pages[i][j] = text[(j + i * 7) % (sizeof(text) - 1)] ^
+ (j / 512 == i ? 0x20 : 0);
+ ret = memcomp_prepare_payload(pages[i], PAGE_SIZE, &lens[i],
+ &handle, &pool_idx,
+ MEMCOMP_PAYLOAD_MUST_SUCCEED,
+ false);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ KUNIT_ASSERT_GT(test, lens[i], 0U);
+ KUNIT_ASSERT_LT(test, lens[i], (unsigned int)PAGE_SIZE / 4);
+ pool = memcomp_pools[pool_idx];
+ src = zs_map_object(pool, handle, ZS_MM_RO);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ memcpy(slot + off, src, lens[i]);
+ zs_unmap_object(pool, handle);
+ memcomp_zs_free(pool_idx, handle);
+ offs[i] = off;
+ off += lens[i];
+ }
+ return slot;
+}
+#endif
+
+#if (defined(CONFIG_SWAP))
+static void memcomp_test_writeback_decode_slot(struct kunit *test)
+{
+ unsigned int offs[3], lens[3];
+ u8 *pages[3];
+ unsigned int i;
+ u8 *slot, *dst;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+ slot = memcomp_test_packed_slot(test, pages, 3, offs, lens);
+ dst = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, dst);
+
+ for (i = 0; i < 3; i++) {
+ memset(dst, 0xa5, PAGE_SIZE);
+ KUNIT_EXPECT_EQ(test, memcomp_decode_slot(slot, true, offs[i],
+ lens[i], dst), 0);
+ KUNIT_EXPECT_MEMEQ(test, dst, pages[i], PAGE_SIZE);
+ }
+
+ /* Raw: the slot is the page, whatever the entry's length says. */
+ KUNIT_EXPECT_EQ(test, memcomp_decode_slot(pages[1], false, 0,
+ lens[1], dst), 0);
+ KUNIT_EXPECT_MEMEQ(test, dst, pages[1], PAGE_SIZE);
+
+ /* Out of the slot, empty, or a whole page: corrupt, never read. */
+ KUNIT_EXPECT_EQ(test, memcomp_decode_slot(slot, true,
+ PAGE_SIZE - 10, 11, dst),
+ -EIO);
+ KUNIT_EXPECT_EQ(test, memcomp_decode_slot(slot, true, 0, 0, dst),
+ -EIO);
+ KUNIT_EXPECT_EQ(test, memcomp_decode_slot(slot, true, 0,
+ PAGE_SIZE, dst), -EIO);
+ /* The wrong offset is a decompression failure, not someone else's page. */
+ KUNIT_EXPECT_NE(test, memcomp_decode_slot(slot, true, offs[1] + 1,
+ lens[1], dst), 0);
+}
+#endif
+
+#if (defined(CONFIG_SWAP))
+static void memcomp_test_writeback_defrag_wanted(struct kunit *test)
+{
+ const long total = 1L << 20;
+ const long short_free = (total >> MEMCOMP_WRITEBACK_DEFRAG_FREE_SHIFT) - 1;
+ const long enough = total >> MEMCOMP_WRITEBACK_DEFRAG_FREE_SHIFT;
+
+ KUNIT_EXPECT_TRUE(test, memcomp_writeback_defrag_wanted(false, false,
+ short_free,
+ total));
+ KUNIT_EXPECT_TRUE(test, memcomp_writeback_defrag_wanted(false, false,
+ 0, total));
+ /* An eighth free is enough; so is anything more. */
+ KUNIT_EXPECT_FALSE(test, memcomp_writeback_defrag_wanted(false, false,
+ enough,
+ total));
+ /* Never while writeback may want the device, or reclaim is running. */
+ KUNIT_EXPECT_FALSE(test, memcomp_writeback_defrag_wanted(true, false,
+ 0, total));
+ KUNIT_EXPECT_FALSE(test, memcomp_writeback_defrag_wanted(false, true,
+ 0, total));
+ KUNIT_EXPECT_FALSE(test, memcomp_writeback_defrag_wanted(false, false,
+ 0, 0));
+}
+#endif
+
+#if (defined(CONFIG_SWAP))
+static bool memcomp_test_sysfs_has_u64(const char *buf, const char *key)
+{
+ const char *at = buf;
+ char number[32];
+ u64 value;
+ size_t len;
+
+ while ((at = strstr(at, key))) {
+ const char *start = at + strlen(key);
+
+ if ((at != buf && !strchr(" \t\n", at[-1])) ||
+ !*start || !strchr(" \t=:", *start)) {
+ at = start;
+ continue;
+ }
+ while (*start && strchr(" \t=:", *start))
+ start++;
+ len = strcspn(start, " \t\n");
+ if (!len || len >= sizeof(number))
+ return false;
+ memcpy(number, start, len);
+ number[len] = '\0';
+ return !kstrtou64(number, 10, &value);
+ }
+ return false;
+}
+#endif
+
+#if (defined(CONFIG_SWAP))
+static void memcomp_test_sysfs_attrs_render(struct kunit *test)
+{
+ static const char * const state_keys[] = {
+ "folio_owner_pages", "samefill_pages", "raw_pages",
+ "shmem_pages",
+ };
+ bool state_seen[ARRAY_SIZE(state_keys)] = {};
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ static const char * const packed_keys[] = {
+ "writeback_zs_reclaimed",
+ "writeback_zs_no_progress",
+ };
+ bool packed_seen[ARRAY_SIZE(packed_keys)] = {};
+ bool packed_events_seen = false;
+#endif
+ unsigned int huge_class_size;
+ unsigned int i, j;
+ char *buf;
+
+ if (!memcomp_is_enabled()) {
+ kunit_skip(test, "memcomp initialization failed");
+ return;
+ }
+
+ KUNIT_ASSERT_NOT_NULL(test, memcomp_pools[0]);
+ huge_class_size = zs_huge_class_size(memcomp_pools[0]);
+ KUNIT_EXPECT_GT(test, huge_class_size, 0U);
+ kunit_info(test, "zsmalloc zs_huge_class_size=%u bytes", huge_class_size);
+
+ buf = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, buf);
+
+ for (i = 0; memcomp_attrs[i]; i++) {
+ struct kobj_attribute *attr =
+ container_of(memcomp_attrs[i],
+ struct kobj_attribute, attr);
+ ssize_t len;
+
+ KUNIT_ASSERT_NOT_NULL(test, attr->show);
+ memset(buf, 0xff, PAGE_SIZE);
+ len = attr->show(NULL, attr, buf);
+
+ KUNIT_EXPECT_GT_MSG(test, len, 0, "%s emitted nothing",
+ memcomp_attrs[i]->name);
+ KUNIT_EXPECT_LT_MSG(test, len, (ssize_t)PAGE_SIZE,
+ "%s filled the page - output truncated",
+ memcomp_attrs[i]->name);
+ if (len <= 0 || len >= (ssize_t)PAGE_SIZE)
+ continue;
+ KUNIT_EXPECT_EQ_MSG(test, buf[len], '\0',
+ "%s left the buffer unterminated",
+ memcomp_attrs[i]->name);
+ KUNIT_EXPECT_EQ_MSG(test, buf[len - 1], '\n',
+ "%s did not end in a newline",
+ memcomp_attrs[i]->name);
+ /* Do not parse beyond the buffer on a broken emitter. */
+ if (buf[len] != '\0')
+ continue;
+ for (j = 0; j < ARRAY_SIZE(state_keys); j++) {
+ u64 value;
+
+ if (!strcmp(attr->attr.name, state_keys[j]))
+ state_seen[j] |= !kstrtou64(buf, 10, &value);
+ else
+ state_seen[j] |=
+ memcomp_test_sysfs_has_u64(buf, state_keys[j]);
+ }
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ if (!strcmp(attr->attr.name, "packed_swap_events")) {
+ packed_events_seen = true;
+ for (j = 0; j < ARRAY_SIZE(packed_keys); j++)
+ packed_seen[j] =
+ memcomp_test_sysfs_has_u64(buf, packed_keys[j]);
+ }
+#endif
+ }
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ for (j = 0; j < ARRAY_SIZE(state_keys); j++)
+ KUNIT_EXPECT_TRUE_MSG(test, state_seen[j],
+ "missing or nonnumeric state key: %s",
+ state_keys[j]);
+#endif
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ KUNIT_EXPECT_TRUE_MSG(test, packed_events_seen,
+ "missing packed_swap_events attribute");
+ if (packed_events_seen)
+ for (j = 0; j < ARRAY_SIZE(packed_keys); j++)
+ KUNIT_EXPECT_TRUE_MSG(test, packed_seen[j],
+ "missing or nonnumeric packed key: %s",
+ packed_keys[j]);
+#endif
+ KUNIT_EXPECT_GT(test, i, 0U);
+ kunit_info(test, "rendered %u sysfs attributes", i);
+}
+#endif
+
+#if (defined(CONFIG_MEMCG))
+static void memcomp_test_objcg_source_identity(struct kunit *test)
+{
+ struct memcomp_test_objcg_fixture *f;
+ struct obj_cgroup *actual, *expected;
+ struct folio *other;
+
+ f = memcomp_test_objcg_begin(test, 1);
+ if (!f)
+ return;
+ other = folio_alloc(GFP_KERNEL, 0);
+ KUNIT_ASSERT_NOT_NULL(test, other);
+ /* An unrelated folio in the same task must use the real owner API. */
+ expected = get_obj_cgroup_from_folio(other);
+ actual = memcomp_objcg_get(other);
+ KUNIT_EXPECT_PTR_EQ(test, actual, expected);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->model.get_calls), 0);
+ if (actual)
+ memcomp_objcg_put(actual);
+ if (expected)
+ obj_cgroup_put(expected);
+ folio_put(other);
+
+ actual = memcomp_objcg_get(f->folio);
+ KUNIT_EXPECT_PTR_EQ(test, actual, &f->model.objcg);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->model.get_calls), 1);
+ if (actual)
+ memcomp_objcg_put(actual);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->model.put_calls), 1);
+ kunit_release_action(test, memcomp_test_objcg_cleanup, f);
+}
+#endif
+
+struct memcomp_test_acomp_tfm {
+ struct crypto_comp *sync;
+ struct workqueue_struct *wq;
+ void *src;
+ void *dst;
+};
+
+struct memcomp_test_acomp_req {
+ struct work_struct work;
+ struct acomp_req *req;
+ struct completion started;
+ struct completion release;
+ struct completion finished;
+ atomic_t submitted;
+ atomic_t completed;
+ bool backlog;
+ bool decompress;
+ int error;
+};
+
+#ifdef CONFIG_MEMCG
+static void memcomp_test_acomp_work(struct work_struct *work)
+{
+ struct memcomp_test_acomp_req *r;
+ struct memcomp_test_acomp_tfm *ctx;
+ struct acomp_req *req;
+ unsigned int len = MEMCOMP_MAX_COMPRESSED_SIZE;
+ int ret;
+
+ r = container_of(work, struct memcomp_test_acomp_req, work);
+ req = r->req;
+ ctx = acomp_tfm_ctx(crypto_acomp_reqtfm(req));
+ if (r->backlog)
+ acomp_request_complete(req, -EINPROGRESS);
+ complete(&r->started);
+ /* Cleanup opens this gate even when an assertion aborts the test. */
+ wait_for_completion(&r->release);
+ ret = r->error;
+ if (ret)
+ goto complete;
+ if (sg_copy_to_buffer(req->src, sg_nents(req->src), ctx->src,
+ req->slen) != req->slen) {
+ ret = -EINVAL;
+ goto complete;
+ }
+ if (r->decompress)
+ ret = crypto_comp_decompress(ctx->sync, ctx->src, req->slen,
+ ctx->dst, &len);
+ else
+ ret = crypto_comp_compress(ctx->sync, ctx->src, req->slen,
+ ctx->dst, &len);
+ if (!ret && len > req->dlen)
+ ret = -E2BIG;
+ if (!ret && sg_copy_from_buffer(req->dst, sg_nents(req->dst),
+ ctx->dst, len) != len)
+ ret = -EINVAL;
+ req->dlen = len;
+complete:
+ atomic_inc(&r->completed);
+ acomp_request_complete(req, ret);
+ /* Report callback return separately from its pending accounting. */
+ complete(&r->finished);
+}
+#endif /* CONFIG_MEMCG */
+
+static int memcomp_test_acomp_submit(struct acomp_req *req, bool decompress)
+{
+ struct memcomp_test_acomp_tfm *ctx;
+ struct memcomp_test_acomp_req *r = acomp_request_ctx(req);
+
+ if (!req->src || !req->dst || req->slen > PAGE_SIZE)
+ return -EINVAL;
+ if (r->backlog && !(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+ return -EBUSY;
+ ctx = acomp_tfm_ctx(crypto_acomp_reqtfm(req));
+ r->req = req;
+ r->decompress = decompress;
+ atomic_inc(&r->submitted);
+ queue_work(ctx->wq, &r->work);
+ return r->backlog ? -EBUSY : -EINPROGRESS;
+}
+
+static int memcomp_test_acomp_compress(struct acomp_req *req)
+{
+ return memcomp_test_acomp_submit(req, false);
+}
+
+static int memcomp_test_acomp_decompress(struct acomp_req *req)
+{
+ return memcomp_test_acomp_submit(req, true);
+}
+
+static atomic_t memcomp_test_acomp_inits = ATOMIC_INIT(0);
+static atomic_t memcomp_test_acomp_exits = ATOMIC_INIT(0);
+
+static int memcomp_test_acomp_init(struct crypto_acomp *tfm)
+{
+ struct memcomp_test_acomp_tfm *ctx = acomp_tfm_ctx(tfm);
+ int ret = -ENOMEM;
+
+ ctx->sync = crypto_alloc_comp(memcomp_compressor, 0, 0);
+ if (IS_ERR(ctx->sync))
+ return PTR_ERR(ctx->sync);
+ ctx->src = kmalloc(PAGE_SIZE, GFP_KERNEL);
+ ctx->dst = kmalloc(MEMCOMP_MAX_COMPRESSED_SIZE, GFP_KERNEL);
+ /* One scratch context requires global ordering, including across NUMA nodes. */
+ ctx->wq = alloc_ordered_workqueue("memcomp-kunit-acomp", 0);
+ if (ctx->src && ctx->dst && ctx->wq) {
+ atomic_inc(&memcomp_test_acomp_inits);
+ return 0;
+ }
+ if (ctx->wq)
+ destroy_workqueue(ctx->wq);
+ kfree(ctx->dst);
+ kfree(ctx->src);
+ crypto_free_comp(ctx->sync);
+ return ret;
+}
+
+static void memcomp_test_acomp_exit(struct crypto_acomp *tfm)
+{
+ struct memcomp_test_acomp_tfm *ctx = acomp_tfm_ctx(tfm);
+
+ destroy_workqueue(ctx->wq);
+ kfree(ctx->dst);
+ kfree(ctx->src);
+ crypto_free_comp(ctx->sync);
+ atomic_inc(&memcomp_test_acomp_exits);
+}
+
+static struct acomp_alg memcomp_test_acomp_alg = {
+ .compress = memcomp_test_acomp_compress,
+ .decompress = memcomp_test_acomp_decompress,
+ .init = memcomp_test_acomp_init,
+ .exit = memcomp_test_acomp_exit,
+ .reqsize = sizeof(struct memcomp_test_acomp_req),
+ .base = {
+ .cra_name = "memcomp-kunit-acomp",
+ .cra_driver_name = "memcomp-kunit-acomp-async",
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct memcomp_test_acomp_tfm),
+ .cra_module = THIS_MODULE,
+ },
+};
+
+#include "late_async_test.h"
+
+#endif /* _MM_MEMCOMP_TESTS_SHARED_TEST_H */
diff --git a/mm/memcomp/tests/shmem_retry_test.h b/mm/memcomp/tests/shmem_retry_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..6db5eb613ae5566ad2f9c749fbc76873f38c4284
--- /dev/null
+++ b/mm/memcomp/tests/shmem_retry_test.h
@@ -0,0 +1,340 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_SHMEM_RETRY_TEST_H
+#define _MM_MEMCOMP_TESTS_SHMEM_RETRY_TEST_H
+
+/*
+ * Include after cache_folio_test.h in suite.h. CONFIG_SHMEM gates the cases.
+ * The CONFIG_MEMCOMP_KUNIT_TEST entry seam in mc16_cache_load() observes
+ * the real consumer before its publication-window check.
+ *
+ * A real shmem file is populated and evicted through its writepage operation.
+ * The existing eviction stub computes a native shadow but delays visibility
+ * by leaving the PUBLISHING marker. The load observer calls the original
+ * consumer, records its actual -EAGAIN, then completes native publication.
+ * Only shmem's real outer loop can retry; no retry/error translation is
+ * implemented here. This is a scheduled window, not a concurrent stress test.
+ * A direct preflight checks the seam before entering a potentially repeating
+ * outer call, so forgetting the new seam fails rather than hangs the suite.
+ */
+#ifdef CONFIG_SHMEM
+#define MC16_SHMEM_RETRY_TEST_ENABLED
+
+#include <kunit/static_stub.h>
+#include <linux/file.h>
+#include <linux/shmem_fs.h>
+#include <linux/writeback.h>
+
+struct mc16_sr_test {
+ struct kunit *test;
+ struct mc16_cf_test *cache;
+ struct file *file;
+ struct folio *probe;
+ struct folio *result;
+ void *entry;
+ void *shadow;
+ unsigned int evictions;
+ unsigned int calls;
+ unsigned int reads;
+ unsigned int refs[2];
+ int errors[2];
+ int read_error;
+ bool isolated;
+ bool preflight;
+ bool published;
+ bool contract;
+ bool result_locked;
+ bool result_uptodate;
+ bool result_mapping;
+ bool result_bytes;
+};
+
+static struct mc16_sr_test *mc16_sr_current;
+
+static u64 mc16_sr_shadow(struct memcomp_ref *ref)
+{
+ u64 word;
+
+ mutex_lock(memcomp_lock_of(ref->token));
+ word = *mc16_shadow_ptr(ref);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ return word;
+}
+
+static void mc16_sr_cleanup(void *data)
+{
+ struct mc16_sr_test *f = data;
+
+ mc16_sr_current = NULL;
+ mc16_cf_actor = NULL;
+ f->cache->probe.task = current;
+ f->cache->payload.task = current;
+#ifdef CONFIG_MEMCG
+ f->cache->model.task = current;
+#endif
+ if (f->result) {
+ if (folio_test_locked(f->result))
+ folio_unlock(f->result);
+ folio_put(f->result);
+ }
+ if (f->probe)
+ folio_put(f->probe);
+ if (folio_test_locked(f->cache->src))
+ folio_unlock(f->cache->src);
+ if (f->isolated)
+ folio_putback_lru(f->cache->src);
+ if (f->file) {
+ /* Synchronous truncate drops the cache owner before cf cleanup. */
+ shmem_truncate_range(file_inode(f->file), 0, (loff_t)-1);
+ fput(f->file);
+ }
+ f->cache->src->swap.val = 0;
+}
+
+static void *mc16_sr_eviction(struct folio *folio,
+ struct mem_cgroup *target_memcg)
+{
+ struct mc16_sr_test *f = mc16_sr_current;
+
+ f->evictions++;
+ f->contract &= folio == f->cache->src && folio_test_locked(folio) &&
+ !folio_test_lru(folio);
+ kunit_deactivate_static_stub(f->test, mc16_store_eviction);
+ f->shadow = mc16_store_eviction(folio, target_memcg);
+ /* Retain the actual publication gate until a reader observes it. */
+ return (void *)(unsigned long)MC16_SHADOW_PUBLISHING;
+}
+
+static int mc16_sr_read(struct memcomp_ref *ref, void *dst)
+{
+ struct mc16_sr_test *f = mc16_sr_current;
+ int ret;
+
+ f->reads++;
+ if (f->read_error && f->reads == 1)
+ return f->read_error;
+ kunit_deactivate_static_stub(f->test, mc16_load_page);
+ ret = mc16_load_page(ref, dst);
+ kunit_activate_static_stub(f->test, mc16_load_page, mc16_sr_read);
+ return ret;
+}
+
+static int mc16_sr_load(struct memcomp_ref *ref, struct folio *folio)
+{
+ struct mc16_sr_test *f = mc16_sr_current;
+ unsigned int call;
+ u64 before, after;
+ int ret;
+
+ if (f->preflight)
+ return -EINTR;
+ call = f->calls++;
+ before = mc16_sr_shadow(ref);
+ f->contract &= ref->token == f->cache->token && folio_test_locked(folio);
+ rcu_read_lock();
+ f->contract &= xa_load(&f->file->f_mapping->i_pages, 0) == f->entry;
+ rcu_read_unlock();
+ if (call < ARRAY_SIZE(f->refs))
+ f->refs[call] = refcount_read(&ref->slot->refs);
+ kunit_deactivate_static_stub(f->test, mc16_cache_load);
+ ret = mc16_cache_load(ref, folio);
+ kunit_activate_static_stub(f->test, mc16_cache_load, mc16_sr_load);
+ after = mc16_sr_shadow(ref);
+ if (call < ARRAY_SIZE(f->errors))
+ f->errors[call] = ret;
+ if (!call) {
+ f->contract &= before == MC16_SHADOW_PUBLISHING &&
+ after == MC16_SHADOW_PUBLISHING && !f->reads;
+ if (ret == -EAGAIN) {
+ /* Finish the paused publisher; do not synthesize its shadow. */
+ mc16_store_publish(ref, 1, f->shadow);
+ f->published = true;
+ }
+ } else {
+ f->contract &= before == (u64)(unsigned long)f->shadow && !after;
+ }
+ return ret;
+}
+
+static struct mc16_sr_test *mc16_sr_begin(struct kunit *test)
+{
+ struct mc16_sr_test *f;
+ struct mc16_cf_test *cache = mc16_cf_begin(test);
+
+ if (!cache)
+ return NULL;
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->test = test;
+ f->cache = cache;
+ f->contract = true;
+ if (kunit_add_action_or_reset(test, mc16_sr_cleanup, f))
+ return NULL;
+ f->file = shmem_kernel_file_setup("memcomp-shmem-retry", PAGE_SIZE,
+ VM_NORESERVE);
+ if (IS_ERR(f->file)) {
+ f->file = NULL;
+ return NULL;
+ }
+ f->probe = folio_alloc(GFP_KERNEL, 0);
+ if (!f->probe)
+ return NULL;
+ return f;
+}
+
+/* All locks and the isolate reference are released before caller assertions. */
+static int mc16_sr_pageout(struct mc16_sr_test *f)
+{
+ struct address_space *mapping = f->file->f_mapping;
+ struct writeback_control wbc = { .for_reclaim = 1 };
+ struct folio *folio = NULL;
+ int ret;
+
+ ret = shmem_get_folio(file_inode(f->file), 0, 0, &folio, SGP_CACHE);
+ if (ret)
+ return ret;
+ if (!folio || folio_test_large(folio)) {
+ if (folio) {
+ folio_unlock(folio);
+ folio_put(folio);
+ }
+ return -EINVAL;
+ }
+ folio_put(f->cache->src);
+ f->cache->src = folio;
+ f->cache->fifo_entries = atomic_long_read(&memcomp_lrus[folio_nid(folio)].nr_entries);
+#ifdef CONFIG_MEMCG
+ f->cache->model.folio = folio;
+#endif
+ mc16_cf_fill(f->cache, MC16_CF_BYTES);
+ folio_mark_uptodate(folio);
+ folio_mark_dirty(folio);
+ lru_add_drain_all();
+ if (!folio_isolate_lru(folio)) {
+ folio_unlock(folio);
+ return -EBUSY;
+ }
+ /* writepage is normally called with an inactive reclaim source. */
+ folio_clear_active(folio);
+ f->isolated = true;
+ mc16_cf_actor = f->cache;
+ ret = mapping->a_ops->writepage(&folio->page, &wbc);
+ mc16_cf_actor = NULL;
+ if (folio_test_locked(folio))
+ folio_unlock(folio);
+ if (folio->mapping)
+ folio_putback_lru(folio);
+ else
+ folio_put(folio);
+ f->isolated = false;
+ return ret;
+}
+
+static int mc16_sr_outer(struct mc16_sr_test *f)
+{
+ struct folio *folio;
+ void *src, *dst;
+ int ret;
+
+ ret = shmem_get_folio(file_inode(f->file), 0, 0, &f->result, SGP_CACHE);
+ if (ret || !f->result)
+ return ret;
+ folio = f->result;
+ f->result_locked = folio_test_locked(folio);
+ f->result_uptodate = folio_test_uptodate(folio);
+ f->result_mapping = folio->mapping == f->file->f_mapping;
+ src = kmap_local_folio(f->cache->src, 0);
+ dst = kmap_local_folio(folio, 0);
+ f->result_bytes = !memcmp(src, dst, PAGE_SIZE);
+ kunmap_local(dst);
+ kunmap_local(src);
+ if (f->result_locked)
+ folio_unlock(folio);
+ return ret;
+}
+
+static void mc16_sr_run(struct kunit *test, int error)
+{
+ struct mc16_sr_test *f = mc16_sr_begin(test);
+ struct shmem_inode_info *info;
+ struct address_space *mapping;
+ blkcnt_t blocks;
+ void *entry;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_sr_current = f;
+ kunit_activate_static_stub(test, mc16_store_eviction, mc16_sr_eviction);
+ KUNIT_ASSERT_EQ(test, mc16_sr_pageout(f), 0);
+ mapping = f->file->f_mapping;
+ info = SHMEM_I(file_inode(f->file));
+ rcu_read_lock();
+ f->entry = xa_load(&mapping->i_pages, 0);
+ rcu_read_unlock();
+ KUNIT_ASSERT_TRUE(test, xa_is_value(f->entry));
+ KUNIT_ASSERT_TRUE(test, is_memcomp_entry(radix_to_swp_entry(f->entry)));
+ f->cache->token = swp_offset(radix_to_swp_entry(f->entry));
+ KUNIT_ASSERT_NE(test, f->cache->token, 0UL);
+ KUNIT_ASSERT_TRUE(test, mc16_get(f->cache->token, &f->cache->guard));
+ KUNIT_ASSERT_EQ(test, f->evictions, 1U);
+ KUNIT_ASSERT_TRUE(test, xa_is_value(f->shadow));
+ KUNIT_ASSERT_NE(test, (u64)(unsigned long)f->shadow, MC16_SHADOW_PUBLISHING);
+ KUNIT_ASSERT_EQ(test, mc16_sr_shadow(&f->cache->guard), MC16_SHADOW_PUBLISHING);
+ kunit_activate_static_stub(test, mc16_cache_load, mc16_sr_load);
+ /* Without the prerequisite seam this returns real -EAGAIN, not -EINTR. */
+ f->preflight = true;
+ folio_lock(f->probe);
+ ret = mc16_cache_load(&f->cache->guard, f->probe);
+ folio_unlock(f->probe);
+ f->preflight = false;
+ KUNIT_ASSERT_EQ(test, ret, -EINTR);
+ f->read_error = error;
+ kunit_activate_static_stub(test, mc16_load_page, mc16_sr_read);
+ blocks = file_inode(f->file)->i_blocks;
+ ret = mc16_sr_outer(f);
+ KUNIT_EXPECT_EQ(test, ret, error);
+ KUNIT_EXPECT_EQ(test, f->calls, 2U);
+ KUNIT_EXPECT_EQ(test, f->errors[0], -EAGAIN);
+ KUNIT_EXPECT_EQ(test, f->errors[1], error);
+ KUNIT_EXPECT_EQ(test, f->reads, 1U);
+ KUNIT_EXPECT_EQ(test, f->refs[0], 3U);
+ KUNIT_EXPECT_EQ(test, f->refs[1], 3U);
+ KUNIT_EXPECT_TRUE(test, f->contract);
+ KUNIT_EXPECT_TRUE(test, f->published);
+ KUNIT_EXPECT_EQ(test, mc16_sr_shadow(&f->cache->guard), 0ULL);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->cache->guard.slot->refs), error ? 2U : 1U);
+ KUNIT_EXPECT_EQ(test, file_inode(f->file)->i_blocks, blocks);
+ KUNIT_EXPECT_EQ(test, info->alloced, 1UL);
+ KUNIT_EXPECT_EQ(test, info->swapped, error ? 1UL : 0UL);
+ KUNIT_EXPECT_EQ(test, mapping->nrpages, error ? 0UL : 1UL);
+ if (error) {
+ KUNIT_EXPECT_PTR_EQ(test, f->result, NULL);
+ rcu_read_lock();
+ entry = xa_load(&mapping->i_pages, 0);
+ rcu_read_unlock();
+ KUNIT_EXPECT_PTR_EQ(test, entry, f->entry);
+ } else {
+ KUNIT_EXPECT_NOT_NULL(test, f->result);
+ KUNIT_EXPECT_TRUE(test, f->result_locked);
+ KUNIT_EXPECT_TRUE(test, f->result_uptodate);
+ KUNIT_EXPECT_TRUE(test, f->result_mapping);
+ KUNIT_EXPECT_TRUE(test, f->result_bytes);
+ }
+}
+
+static void memcomp_test_shmem_publish_retry(struct kunit *test)
+{
+ mc16_sr_run(test, 0);
+}
+
+static void memcomp_test_shmem_retry_eio(struct kunit *test)
+{
+ mc16_sr_run(test, -EIO);
+}
+
+static void memcomp_test_shmem_retry_enomem(struct kunit *test)
+{
+ mc16_sr_run(test, -ENOMEM);
+}
+#endif /* CONFIG_SHMEM */
+#endif /* _MM_MEMCOMP_TESTS_SHMEM_RETRY_TEST_H */
diff --git a/mm/memcomp/tests/shmem_zero_fault_test.h b/mm/memcomp/tests/shmem_zero_fault_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..b943d0596877406de4adbdb36d82ab632bbd338b
--- /dev/null
+++ b/mm/memcomp/tests/shmem_zero_fault_test.h
@@ -0,0 +1,121 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_SHMEM_ZERO_FAULT_TEST_H
+#define _MM_MEMCOMP_TESTS_SHMEM_ZERO_FAULT_TEST_H
+
+/* Include after shmem_retry_test.h and fault_test.h in suite.h. */
+#if defined(MC16_SHMEM_RETRY_TEST_ENABLED) && \
+ defined(MC16_FAULT_MM_TEST_ENABLED)
+#define MC16_SHMEM_ZERO_FAULT_TEST_ENABLED
+
+#include <linux/pagemap.h>
+
+/* Same real writepage actor as the retry fixture, with a zero source. */
+static int mc16_sz_pageout(struct mc16_sr_test *f)
+{
+ struct address_space *mapping = f->file->f_mapping;
+ struct writeback_control wbc = { .for_reclaim = 1 };
+ struct folio *folio = NULL;
+ int ret;
+
+ ret = shmem_get_folio(file_inode(f->file), 0, 0, &folio, SGP_CACHE);
+ if (ret)
+ return ret;
+ if (!folio || folio_test_large(folio)) {
+ if (folio) {
+ folio_unlock(folio);
+ folio_put(folio);
+ }
+ return -EINVAL;
+ }
+ folio_put(f->cache->src);
+ f->cache->src = folio;
+ f->cache->fifo_entries =
+ atomic_long_read(&memcomp_lrus[folio_nid(folio)].nr_entries);
+#ifdef CONFIG_MEMCG
+ f->cache->model.folio = folio;
+#endif
+ mc16_cf_fill(f->cache, MC16_CF_ZERO);
+ folio_mark_uptodate(folio);
+ folio_mark_dirty(folio);
+ lru_add_drain_all();
+ if (!folio_isolate_lru(folio)) {
+ folio_unlock(folio);
+ return -EBUSY;
+ }
+ folio_clear_active(folio);
+ f->isolated = true;
+ mc16_cf_actor = f->cache;
+ ret = mapping->a_ops->writepage(&folio->page, &wbc);
+ mc16_cf_actor = NULL;
+ if (folio_test_locked(folio))
+ folio_unlock(folio);
+ if (folio->mapping)
+ folio_putback_lru(folio);
+ else
+ folio_put(folio);
+ f->isolated = false;
+ return ret;
+}
+
+static void memcomp_test_shmem_token0_not_major(struct kunit *test)
+{
+ struct mc16_sr_test *f = mc16_sr_begin(test);
+ struct vm_area_struct *vma;
+ struct address_space *mapping;
+ struct folio *folio;
+ unsigned long addr;
+ vm_fault_t ret = VM_FAULT_SIGBUS;
+ void *entry, *data;
+ bool mapped = false, zero;
+ bool uptodate, correct_mapping;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mapping = f->file->f_mapping;
+ addr = kunit_vm_mmap(test, f->file, 0, PAGE_SIZE,
+ PROT_READ | PROT_WRITE, MAP_SHARED, 0);
+ KUNIT_ASSERT_NE(test, addr, 0UL);
+ KUNIT_ASSERT_FALSE(test, IS_ERR_VALUE(addr));
+ KUNIT_ASSERT_EQ(test, mc16_sz_pageout(f), 0);
+ rcu_read_lock();
+ entry = xa_load(&mapping->i_pages, 0);
+ rcu_read_unlock();
+ KUNIT_ASSERT_TRUE(test, xa_is_value(entry));
+ KUNIT_ASSERT_TRUE(test, is_memcomp_entry(radix_to_swp_entry(entry)));
+ KUNIT_ASSERT_EQ(test, swp_offset(radix_to_swp_entry(entry)), 0UL);
+ KUNIT_ASSERT_EQ(test, mapping->nrpages, 0UL);
+
+ /* No ALLOW_RETRY: this call retains the caller's mmap read lock. */
+ mmap_read_lock(current->mm);
+ vma = vma_lookup(current->mm, addr);
+ if (vma && vma->vm_file == f->file) {
+ mapped = true;
+ ret = handle_mm_fault(vma, addr, FAULT_FLAG_WRITE, NULL);
+ }
+ mmap_read_unlock(current->mm);
+ KUNIT_ASSERT_TRUE(test, mapped);
+ kunit_info(test, "shmem token0 native fault: ret=%#x",
+ (__force unsigned int)ret);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)(ret & VM_FAULT_MAJOR), 0U);
+ KUNIT_EXPECT_EQ(test,
+ (__force unsigned int)(ret & (VM_FAULT_ERROR | VM_FAULT_RETRY)),
+ 0U);
+
+ /* Lookup only: do not allocate a replacement folio after a bad fault. */
+ folio = filemap_get_folio(mapping, 0);
+ KUNIT_ASSERT_FALSE(test, IS_ERR(folio));
+ f->result = folio;
+ folio_lock(folio);
+ uptodate = folio_test_uptodate(folio);
+ correct_mapping = folio->mapping == mapping;
+ data = kmap_local_folio(folio, 0);
+ zero = !memchr_inv(data, 0, PAGE_SIZE);
+ kunmap_local(data);
+ folio_unlock(folio);
+ KUNIT_EXPECT_TRUE(test, uptodate);
+ KUNIT_EXPECT_TRUE(test, correct_mapping);
+ KUNIT_EXPECT_TRUE(test, zero);
+ KUNIT_EXPECT_EQ(test, SHMEM_I(file_inode(f->file))->swapped, 0UL);
+ KUNIT_EXPECT_EQ(test, mapping->nrpages, 1UL);
+}
+#endif
+#endif /* _MM_MEMCOMP_TESTS_SHMEM_ZERO_FAULT_TEST_H */
diff --git a/mm/memcomp/tests/span_test.h b/mm/memcomp/tests/span_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..f062387cd5ecb894a846355b9a70e6f587472e85
--- /dev/null
+++ b/mm/memcomp/tests/span_test.h
@@ -0,0 +1,128 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_SPAN_TEST_H
+#define _MM_MEMCOMP_TESTS_SPAN_TEST_H
+#ifdef CONFIG_MEMCG
+static void memcomp_test_span_geometry(struct kunit *test)
+{
+ struct mc16_test_refs *f = mc16_test_refs_begin(test);
+ struct mc16_test_storage *storage;
+ unsigned int nr, first, i, index;
+ u32 base;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->storage;
+ for (nr = 1; nr <= MEMCOMP_MAX_COMPRESS_PAGES; nr <<= 1) {
+ first = storage->nr;
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f, nr), 0);
+ for (i = 0; i < nr; i++) {
+ KUNIT_EXPECT_EQ(test, mc16_ref_geometry(&storage->refs[first + i],
+ &base, &index), nr);
+ KUNIT_EXPECT_EQ(test, base, storage->refs[first].token);
+ KUNIT_EXPECT_EQ(test, index, i);
+ }
+ }
+}
+
+static void memcomp_test_span_identity(struct kunit *test)
+{
+ struct mc16_test_refs *f = mc16_test_refs_begin(test);
+ struct memcomp_ref refs[2];
+ struct mc16_test_storage *storage;
+ unsigned int i;
+ u32 token;
+ bool found;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->storage;
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f, 2), 0);
+ token = storage->refs[0].token;
+ found = mc16_span_get(token, storage->owner->folio, 2, true, refs);
+ KUNIT_ASSERT_TRUE(test, found);
+ for (i = 0; i < 2; i++) {
+ KUNIT_EXPECT_EQ(test, refcount_read(&refs[i].slot->refs), 2U);
+ mc16_ref_put(&refs[i]);
+ }
+ KUNIT_EXPECT_FALSE(test, mc16_span_get(token, NULL, 2, true, refs));
+ KUNIT_EXPECT_FALSE(test, mc16_span_get(token, storage->owner->folio,
+ 1, true, refs));
+ for (i = 0; i < 2; i++) {
+ KUNIT_EXPECT_PTR_EQ(test, refs[i].slot, NULL);
+ KUNIT_EXPECT_EQ(test, refcount_read(&storage->refs[i].slot->refs), 1U);
+ }
+ /* A changed suffix must consume both temporary lookups on rejection. */
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_payload(f, 1, MC16_SAMEFILL), 0);
+ KUNIT_EXPECT_FALSE(test, mc16_span_get(token, storage->owner->folio,
+ 2, true, refs));
+ for (i = 0; i < 2; i++) {
+ KUNIT_EXPECT_PTR_EQ(test, refs[i].slot, NULL);
+ KUNIT_EXPECT_EQ(test, refcount_read(&storage->refs[i].slot->refs), 1U);
+ }
+ /* Valid geometry with an invalid token also clears poisoned outputs. */
+ memset(refs, 0xa5, sizeof(refs));
+ KUNIT_EXPECT_FALSE(test, mc16_span_get(0, storage->owner->folio,
+ 2, true, refs));
+ for (i = 0; i < 2; i++)
+ KUNIT_EXPECT_PTR_EQ(test, refs[i].slot, NULL);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_payload(f, 0, MC16_SAMEFILL), 0);
+ found = mc16_span_get(token, storage->owner->folio, 2, false, refs);
+ KUNIT_ASSERT_TRUE(test, found);
+ for (i = 0; i < 2; i++)
+ mc16_ref_put(&refs[i]);
+
+ /* Remove a real suffix slot; the successful lookup prefix must unwind. */
+ KUNIT_ASSERT_TRUE(test, mc16_get(token + 1, &refs[1]));
+ found = mc16_owner_drop_folio_held(&refs[1], storage->owner->folio, 2, 1);
+ if (found)
+ storage->refs[1] = (struct memcomp_ref) { };
+ KUNIT_ASSERT_TRUE(test, found);
+ KUNIT_EXPECT_FALSE(test, mc16_span_get(token, storage->owner->folio,
+ 2, false, refs));
+ for (i = 0; i < 2; i++)
+ KUNIT_EXPECT_PTR_EQ(test, refs[i].slot, NULL);
+ KUNIT_EXPECT_EQ(test, refcount_read(&storage->refs[0].slot->refs), 1U);
+}
+
+static void memcomp_test_span_anon_vma(struct kunit *test)
+{
+ struct mc16_test_refs *f = mc16_test_refs_begin(test);
+ struct mc16_test_storage *storage;
+ struct mc16_record *record;
+ struct anon_vma *anon_vma, *taken, *again;
+ struct folio *mapped;
+ unsigned long addr;
+ struct memcomp_ref *head;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ storage = f->storage;
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f, 2), 0);
+ head = &storage->refs[0];
+ record = (void *)(unsigned long)head->slot->word;
+ mapped = memcomp_test_user_folio(test, &addr);
+ KUNIT_ASSERT_NOT_NULL(test, mapped);
+ anon_vma = folio_get_anon_vma(mapped);
+ folio_unlock(mapped);
+ folio_put(mapped);
+ KUNIT_ASSERT_NOT_NULL(test, anon_vma);
+ /* Own one real reference; the helper must transfer it without a put. */
+ record->anon_vma = anon_vma;
+ record->record_flags |= MC16_RECORD_ANON_VMA;
+ mutex_lock(memcomp_lock_of(head->token));
+ taken = mc16_span_take_anon_vma_locked(head, NULL, 2);
+ mutex_unlock(memcomp_lock_of(head->token));
+ KUNIT_EXPECT_PTR_EQ(test, taken, NULL);
+ KUNIT_EXPECT_PTR_EQ(test, record->anon_vma, anon_vma);
+ mutex_lock(memcomp_lock_of(head->token));
+ taken = mc16_span_take_anon_vma_locked(head, storage->owner->folio, 2);
+ again = mc16_span_take_anon_vma_locked(head, storage->owner->folio, 2);
+ mutex_unlock(memcomp_lock_of(head->token));
+ KUNIT_EXPECT_PTR_EQ(test, taken, anon_vma);
+ KUNIT_EXPECT_PTR_EQ(test, again, NULL);
+ KUNIT_EXPECT_PTR_EQ(test, record->anon_vma, NULL);
+ KUNIT_EXPECT_FALSE(test, record->record_flags & MC16_RECORD_ANON_VMA);
+ /* Tests own the transferred reference even if a helper assertion fails. */
+ record->anon_vma = NULL;
+ record->record_flags &= ~MC16_RECORD_ANON_VMA;
+ put_anon_vma(anon_vma);
+}
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_SPAN_TEST_H */
diff --git a/mm/memcomp/tests/storage_fixture.h b/mm/memcomp/tests/storage_fixture.h
new file mode 100644
index 0000000000000000000000000000000000000000..c132682aadf402813b6dd6ffb0cd1701f8ff618a
--- /dev/null
+++ b/mm/memcomp/tests/storage_fixture.h
@@ -0,0 +1,84 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_STORAGE_FIXTURE_H
+#define _MM_MEMCOMP_TESTS_STORAGE_FIXTURE_H
+
+/* Fault/ordering injection only. No operation or XA budget accounting. */
+struct mc16_storage_fixture {
+ struct task_struct *task;
+ unsigned int allocations;
+ unsigned int fail_at;
+ bool fail_descriptor;
+ bool hold_segments;
+ struct rcu_head *delayed_segments;
+ void (*allocation_window)(void *data);
+ void *window_data;
+};
+
+static struct mc16_storage_fixture __rcu *mc16_storage_fixture;
+
+static void mc16_kunit_queue_rcu(struct rcu_head *head, rcu_callback_t func)
+{
+ struct mc16_storage_fixture *probe;
+ bool held = false;
+
+ rcu_read_lock();
+ probe = rcu_dereference(mc16_storage_fixture);
+ if (probe && probe->task == current && probe->hold_segments &&
+ func == mc16_segment_rcu_free) {
+ head->next = probe->delayed_segments;
+ probe->delayed_segments = head;
+ held = true;
+ }
+ rcu_read_unlock();
+ if (!held)
+ call_rcu(head, func);
+}
+
+static void mc16_test_drain_segments(struct mc16_storage_fixture *probe)
+{
+ struct rcu_head *head;
+
+ probe->hold_segments = false;
+ while ((head = probe->delayed_segments)) {
+ probe->delayed_segments = head->next;
+ call_rcu(head, mc16_segment_rcu_free);
+ }
+}
+
+static void *mc16_kunit_storage_alloc(size_t bytes, gfp_t gfp,
+ struct obj_cgroup *objcg)
+{
+ struct mc16_storage_fixture *probe;
+ void (*window)(void *data) = NULL;
+ void *data = NULL, *ptr;
+ bool modeled = false, fail = false;
+#ifdef CONFIG_MEMCG
+ struct memcomp_kunit_objcg *model;
+#endif
+
+ rcu_read_lock();
+#ifdef CONFIG_MEMCG
+ model = rcu_dereference(memcomp_kunit_objcg);
+ modeled = model && objcg == &model->objcg;
+#endif
+ probe = rcu_dereference(mc16_storage_fixture);
+ if (probe && probe->task == current) {
+ probe->allocations++;
+ fail = probe->allocations == probe->fail_at ||
+ (probe->fail_descriptor &&
+ bytes == sizeof(struct mc16_segment_retire));
+ window = probe->allocation_window;
+ data = probe->window_data;
+ probe->allocation_window = NULL;
+ }
+ rcu_read_unlock();
+ if (fail)
+ return NULL;
+ /* Synthetic objcg has no css: explicitly model only the allocator. */
+ ptr = modeled ? kmalloc(bytes, gfp & ~__GFP_ACCOUNT) :
+ mc16_kmalloc_owner(bytes, gfp, objcg);
+ if (window)
+ window(data);
+ return ptr;
+}
+#endif /* _MM_MEMCOMP_TESTS_STORAGE_FIXTURE_H */
diff --git a/mm/memcomp/tests/storage_test.h b/mm/memcomp/tests/storage_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..a3a03336d57c61af0c91fff79cec9b6328b26d9c
--- /dev/null
+++ b/mm/memcomp/tests/storage_test.h
@@ -0,0 +1,468 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_STORAGE_TEST_H
+#define _MM_MEMCOMP_TESTS_STORAGE_TEST_H
+
+/* Pressure cleanup is explicit; normal retirement keeps appendable headers. */
+static void mc16_test_reclaim_retained(void)
+{
+ struct shrink_control sc = { .nr_to_scan = 64 };
+ struct mc16_cluster *cluster;
+ unsigned long flags, id;
+ bool found;
+
+ cancel_delayed_work_sync(&mc16_retained_aging);
+ while (READ_ONCE(mc16_retained_empty_headers)) {
+ found = false;
+ xa_lock_irqsave(&mc16_clusters, flags);
+ xa_for_each(&mc16_clusters, id, cluster) {
+ if (xa_is_value(cluster) || xa_is_internal(cluster))
+ continue;
+ if (cluster->key->current_cluster != cluster ||
+ cluster->live_slots || !cluster->next ||
+ cluster->next >= MC16_CLUSTER_SLOTS ||
+ cluster->eligible_slots || !list_empty(&cluster->fifo))
+ continue;
+ mc16_meta_shrinker_cursor = id;
+ found = true;
+ break;
+ }
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ /* An inconsistent count must not turn a failed test into a hang. */
+ if (WARN_ON_ONCE(!found))
+ break;
+ if (WARN_ON_ONCE(!mc16_meta_shrinker_scan(NULL, &sc)))
+ break;
+ }
+ rcu_barrier();
+}
+
+#ifdef CONFIG_MEMCG
+#define MC16_TEST_REFS 514
+struct mc16_test_storage {
+ struct kunit *test;
+ struct memcomp_test_objcg_fixture *owner;
+ struct mc16_storage_fixture probe;
+ struct memcomp_kunit_async_probe payload;
+ struct memcomp_ref refs[MC16_TEST_REFS];
+ unsigned int nr;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ struct mc16_storage_debug baseline;
+#endif
+};
+
+/* Foundation reservations still hold their initial resident OWNER member. */
+static void mc16_test_cancel_pending(struct memcomp_ref *ref)
+{
+ if (!ref->slot)
+ return;
+ if (WARN_ON_ONCE(!refcount_dec_and_test(&ref->slot->refs)))
+ return;
+ mc16_release_resources(ref);
+ mc16_retire_slot(ref);
+}
+
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+static void mc16_test_metadata_snapshot(struct mc16_storage_debug *sum)
+{
+ struct mc16_storage_debug *debug;
+ unsigned int type;
+ int cpu;
+
+ memset(sum, 0, sizeof(*sum));
+ for_each_possible_cpu(cpu) {
+ debug = per_cpu_ptr(&mc16_storage_debug, cpu);
+ for (type = 0; type < MC16_META_CLASSES; type++) {
+ sum->allocated[type] += READ_ONCE(debug->allocated[type]);
+ sum->freed[type] += READ_ONCE(debug->freed[type]);
+ sum->allocated_bytes[type] += READ_ONCE(debug->allocated_bytes[type]);
+ sum->freed_bytes[type] += READ_ONCE(debug->freed_bytes[type]);
+ }
+ }
+}
+#endif
+
+static void mc16_test_storage_cleanup(void *data)
+{
+ struct mc16_test_storage *f = data;
+ unsigned int i;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ struct mc16_storage_debug after;
+#endif
+
+ f->probe.fail_at = 0;
+ f->probe.fail_descriptor = false;
+ f->probe.allocation_window = NULL;
+ for (i = 0; i < f->nr; i++)
+ mc16_test_cancel_pending(&f->refs[i]);
+ mc16_test_drain_segments(&f->probe);
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_EQ(f->test, READ_ONCE(mc16_retained_empty_headers), 0UL);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ mc16_test_metadata_snapshot(&after);
+ for (i = 0; i < MC16_META_CLASSES; i++) {
+ KUNIT_EXPECT_EQ(f->test, after.allocated[i] - f->baseline.allocated[i],
+ after.freed[i] - f->baseline.freed[i]);
+ KUNIT_EXPECT_EQ(f->test,
+ after.allocated_bytes[i] - f->baseline.allocated_bytes[i],
+ after.freed_bytes[i] - f->baseline.freed_bytes[i]);
+ }
+#endif
+ if (rcu_access_pointer(memcomp_kunit_async_probe) == &f->payload) {
+ rcu_assign_pointer(memcomp_kunit_async_probe, NULL);
+ synchronize_rcu();
+ }
+ KUNIT_EXPECT_TRUE(f->test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(f->test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_EQ(f->test, atomic_long_read(&f->owner->model.charged), 0L);
+ if (rcu_access_pointer(mc16_storage_fixture) == &f->probe) {
+ rcu_assign_pointer(mc16_storage_fixture, NULL);
+ synchronize_rcu();
+ }
+}
+
+static struct mc16_test_storage *
+mc16_test_storage_begin_pages(struct kunit *test, unsigned int nr_pages)
+{
+ struct mc16_test_storage *f;
+ int ret;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->test = test;
+ f->owner = memcomp_test_objcg_begin(test, nr_pages);
+ if (!f->owner)
+ return NULL;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ mc16_test_metadata_snapshot(&f->baseline);
+#endif
+ f->probe.task = current;
+ ret = kunit_add_action_or_reset(test, mc16_test_storage_cleanup, f);
+ if (ret)
+ return NULL;
+ rcu_assign_pointer(mc16_storage_fixture, &f->probe);
+ return f;
+}
+
+static struct mc16_test_storage *mc16_test_storage_begin(struct kunit *test)
+{
+ return mc16_test_storage_begin_pages(test, 2);
+}
+
+static int mc16_test_reserve(struct mc16_test_storage *f, unsigned int nr)
+{
+ struct mc16_record *record;
+ int ret;
+
+ if (f->nr + nr > MC16_TEST_REFS)
+ return -ENOSPC;
+ record = mc16_record_alloc(f->owner->folio, nr, false);
+ if (IS_ERR(record))
+ return PTR_ERR(record);
+ ret = mc16_reserve(record, nr, MC16_KEY_ANON, &f->refs[f->nr]);
+ if (ret) {
+ mc16_record_abort(record);
+ return ret;
+ }
+ f->nr += nr;
+ return 0;
+}
+
+static void memcomp_test_reserve_boundaries(struct kunit *test)
+{
+ struct mc16_test_storage *f;
+ struct memcomp_ref guard;
+ unsigned int i;
+ u32 base;
+
+ f = mc16_test_storage_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (i = 0; i < 66; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(f, 1), 0);
+ base = f->refs[0].token;
+ KUNIT_EXPECT_EQ(test, base & (MC16_CLUSTER_SLOTS - 1), 0U);
+ for (i = 0; i < 66; i++) {
+ KUNIT_EXPECT_EQ(test, f->refs[i].token, base + i);
+ KUNIT_EXPECT_PTR_EQ(test, f->refs[i].cluster, f->refs[0].cluster);
+ KUNIT_ASSERT_TRUE(test, mc16_get(base + i, &guard));
+ KUNIT_EXPECT_PTR_EQ(test, guard.slot, f->refs[i].slot);
+ KUNIT_EXPECT_FALSE(test, refcount_dec_and_test(&guard.slot->refs));
+ }
+ KUNIT_EXPECT_FALSE(test, mc16_get(base + 66, &guard));
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->owner->folio), 67);
+}
+
+static void memcomp_test_reserve_holes(struct kunit *test)
+{
+ struct mc16_test_storage *f;
+ struct memcomp_ref guard;
+ struct mc16_slot *old;
+ u32 base;
+
+ f = mc16_test_storage_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(f, 1), 0);
+ base = f->refs[0].token;
+ old = f->refs[0].slot;
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(f, 2), 0);
+ KUNIT_EXPECT_EQ(test, f->refs[1].token, base + 2);
+ KUNIT_EXPECT_FALSE(test, mc16_get(base + 1, &guard));
+ mc16_test_cancel_pending(&f->refs[0]);
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(f, 1), 0);
+ KUNIT_EXPECT_EQ(test, f->refs[3].token, base + 4);
+ KUNIT_EXPECT_PTR_NE(test, f->refs[3].slot, old);
+ KUNIT_EXPECT_FALSE(test, mc16_get(base, &guard));
+}
+
+static void mc16_test_segment_retire(struct kunit *test, bool fail_descriptor)
+{
+ struct mc16_test_storage *f;
+ struct mc16_cluster *cluster;
+ struct mc16_slot *old, *published;
+ unsigned int i, bad_refs = 0, live;
+
+ f = mc16_test_storage_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (i = 0; i < 65; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(f, 1), 0);
+ cluster = f->refs[64].cluster;
+ old = cluster->segments[0];
+ f->probe.fail_descriptor = fail_descriptor;
+ rcu_read_lock();
+ for (i = 0; i < 64; i++)
+ mc16_test_cancel_pending(&f->refs[i]);
+ live = cluster->live_slots;
+ published = cluster->segments[0];
+ for (i = 0; i < 64; i++)
+ bad_refs += !!refcount_read(&old[i].refs);
+ rcu_read_unlock();
+ KUNIT_EXPECT_EQ(test, live, 1U);
+ KUNIT_EXPECT_PTR_EQ(test, published, fail_descriptor ? old : NULL);
+ KUNIT_EXPECT_EQ(test, bad_refs, 0U);
+ rcu_barrier();
+ mc16_test_cancel_pending(&f->refs[64]);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 1UL);
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->owner->model.charged), 0L);
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+}
+
+static void memcomp_test_segment_retire(struct kunit *test)
+{
+ mc16_test_segment_retire(test, false);
+}
+
+static void memcomp_test_segment_retire_nomem(struct kunit *test)
+{
+ mc16_test_segment_retire(test, true);
+}
+
+static void memcomp_test_reserve_nomem(struct kunit *test)
+{
+ struct mc16_test_storage *f;
+ unsigned int at;
+
+ f = mc16_test_storage_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (at = 1; at <= 2; at++) {
+ f->probe.fail_at = f->probe.allocations + at;
+ KUNIT_EXPECT_EQ(test, mc16_test_reserve(f, 1), -ENOMEM);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->owner->folio), 1);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->owner->model.charged), 0L);
+ }
+ f->probe.fail_at = 0;
+ /* Record charge succeeds; the independent key charge fails. */
+ f->owner->model.fail_at = atomic_read(&f->owner->model.charge_calls) + 2;
+ KUNIT_EXPECT_EQ(test, mc16_test_reserve(f, 1), -ENOMEM);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->owner->model.charged), 0L);
+ f->owner->model.fail_at = 0;
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(f, 1), 0);
+ KUNIT_EXPECT_EQ(test, f->refs[0].token,
+ (u32)BIT(MC16_CLUSTER_SHIFT));
+}
+
+struct mc16_test_rival {
+ struct mc16_test_storage *f;
+ int result;
+};
+
+static void mc16_test_rival_reserve(void *data)
+{
+ struct mc16_test_rival *rival = data;
+
+ rival->result = mc16_test_reserve(rival->f, 1);
+}
+
+static void memcomp_test_reserve_race(struct kunit *test)
+{
+ struct mc16_test_storage *f;
+ struct mc16_test_rival rival;
+
+ f = mc16_test_storage_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ rival = (struct mc16_test_rival) { .f = f, .result = -EINPROGRESS };
+ f->probe.allocation_window = mc16_test_rival_reserve;
+ f->probe.window_data = &rival;
+ KUNIT_EXPECT_EQ(test, mc16_test_reserve(f, 1), -EAGAIN);
+ KUNIT_EXPECT_EQ(test, rival.result, 0);
+ KUNIT_EXPECT_EQ(test, f->nr, 1U);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->owner->folio), 2);
+}
+
+static void memcomp_test_cluster_key_lifetime(struct kunit *test)
+{
+ struct mc16_test_storage *f;
+ struct mc16_key *key;
+ unsigned int i;
+
+ f = mc16_test_storage_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (i = 0; i < 513; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(f, 1), 0);
+ KUNIT_EXPECT_PTR_NE(test, f->refs[0].cluster, f->refs[512].cluster);
+ key = f->refs[512].cluster->key;
+ KUNIT_EXPECT_EQ(test, key->clusters, 2U);
+ rcu_read_lock();
+ for (i = 0; i < 512; i++)
+ mc16_test_cancel_pending(&f->refs[i]);
+ /* The queued old header still owns its key through the callback. */
+ KUNIT_EXPECT_EQ(test, key->clusters, 2U);
+ KUNIT_EXPECT_PTR_EQ(test, key->current_cluster, f->refs[512].cluster);
+ rcu_read_unlock();
+ rcu_barrier();
+ KUNIT_EXPECT_EQ(test, key->clusters, 1U);
+ mc16_test_cancel_pending(&f->refs[512]);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 1UL);
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->owner->model.get_calls), 514);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->owner->model.put_calls), 514);
+}
+
+static void memcomp_test_segment_reserve_race(struct kunit *test)
+{
+ struct mc16_test_storage *f;
+ struct mc16_test_rival rival;
+ unsigned int i;
+
+ f = mc16_test_storage_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (i = 0; i < 64; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(f, 1), 0);
+ rival = (struct mc16_test_rival) { .f = f, .result = -EINPROGRESS };
+ f->probe.allocation_window = mc16_test_rival_reserve;
+ f->probe.window_data = &rival;
+ KUNIT_EXPECT_EQ(test, mc16_test_reserve(f, 1), -EAGAIN);
+ KUNIT_EXPECT_EQ(test, rival.result, 0);
+ KUNIT_EXPECT_EQ(test, f->nr, 65U);
+}
+
+static void mc16_test_retire_and_reuse(void *data)
+{
+ struct mc16_test_rival *rival = data;
+ struct mc16_test_storage *f = rival->f;
+ unsigned int i;
+
+ for (i = 0; i < f->nr; i++)
+ mc16_test_cancel_pending(&f->refs[i]);
+ rcu_barrier(); /* old key/header really disappear during preallocation */
+ f->nr = 0;
+ rival->result = mc16_test_reserve(f, 1);
+}
+
+static void memcomp_test_rollover_retire_race(struct kunit *test)
+{
+ struct mc16_test_storage *f;
+ struct mc16_test_rival rival;
+ unsigned int i;
+ u32 old_id;
+
+ f = mc16_test_storage_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (i = 0; i < 512; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(f, 1), 0);
+ old_id = f->refs[0].cluster->id;
+ rival = (struct mc16_test_rival) { .f = f, .result = -EINPROGRESS };
+ f->probe.allocation_window = mc16_test_retire_and_reuse;
+ f->probe.window_data = &rival;
+ KUNIT_EXPECT_EQ(test, mc16_test_reserve(f, 1), -EAGAIN);
+ KUNIT_EXPECT_EQ(test, rival.result, 0);
+ KUNIT_EXPECT_EQ(test, f->nr, 1U);
+ KUNIT_EXPECT_EQ(test, f->refs[0].cluster->id, old_id);
+ KUNIT_EXPECT_EQ(test, f->refs[0].cluster->next, (u16)1);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->owner->folio), 2);
+}
+
+static void memcomp_test_header_before_segment_callback(struct kunit *test)
+{
+ struct mc16_test_storage *f;
+ struct mc16_slot *old;
+ unsigned int i;
+
+ f = mc16_test_storage_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (i = 0; i < 64; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(f, 1), 0);
+ old = f->refs[0].cluster->segments[0];
+ f->probe.hold_segments = true;
+ for (i = 0; i < 64; i++)
+ mc16_test_cancel_pending(&f->refs[i]);
+ /* Pressure queues the retained header; segment callbacks stay delayed. */
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 1UL);
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_NOT_NULL(test, f->probe.delayed_segments);
+ for (i = 0; i < 64; i++)
+ KUNIT_EXPECT_EQ(test, refcount_read(&old[i].refs), 0U);
+ /* Actual callback and real grace, now after both header and key free. */
+ mc16_test_drain_segments(&f->probe);
+ rcu_barrier();
+ KUNIT_EXPECT_PTR_EQ(test, f->probe.delayed_segments, NULL);
+}
+
+static int mc16_test_id_alloc_fail(u32 *id)
+{
+ return -ENOMEM;
+}
+
+static struct mc16_key *mc16_test_key_zalloc_fail(void)
+{
+ return NULL;
+}
+
+static void memcomp_test_reserve_id_failure(struct kunit *test)
+{
+ struct mc16_test_storage *f;
+
+ f = mc16_test_storage_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ kunit_activate_static_stub(test, mc16_private_id_alloc, mc16_test_id_alloc_fail);
+ KUNIT_EXPECT_EQ(test, mc16_test_reserve(f, 1), -ENOMEM);
+ kunit_deactivate_static_stub(test, mc16_private_id_alloc);
+ KUNIT_EXPECT_EQ(test, f->probe.allocations, 0U);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->owner->model.charged), 0L);
+}
+
+static void memcomp_test_reserve_key_failure(struct kunit *test)
+{
+ struct mc16_test_storage *f;
+
+ f = mc16_test_storage_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ kunit_activate_static_stub(test, mc16_key_zalloc, mc16_test_key_zalloc_fail);
+ KUNIT_EXPECT_EQ(test, mc16_test_reserve(f, 1), -ENOMEM);
+ kunit_deactivate_static_stub(test, mc16_key_zalloc);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->owner->model.charged), 0L);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->owner->folio), 1);
+}
+
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_STORAGE_TEST_H */
diff --git a/mm/memcomp/tests/store_transaction_test.h b/mm/memcomp/tests/store_transaction_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..5dc421a4c5c96cd5a5e6cad5f19fb3094cbfdb3e
--- /dev/null
+++ b/mm/memcomp/tests/store_transaction_test.h
@@ -0,0 +1,587 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_STORE_TRANSACTION_TEST_H
+#define _MM_MEMCOMP_TESTS_STORE_TRANSACTION_TEST_H
+
+/*
+ * Author: nakixii
+ *
+ * Include after folio_test.h in suite.h. Register the
+ * eight cases below under MC16_STORE_TRANSACTION_TEST_ENABLED. The enclosing
+ * CONFIG_MEMCOMP_KUNIT_TEST suite supplies the codec and failure hooks.
+ *
+ * Transaction coverage:
+ * success and failure consume the public context lease;
+ * unlocked folios and missing spans fail preflight without changing owners;
+ * failure accounting records exclusive reasons and whole-span outcomes;
+ * payload failures record one event and whole-span pages, without a state event;
+ * only real -E2BIG marks incompressible folios, and rollback installs the hash;
+ * successful stores release the producer pin;
+ * committed payloads and surviving guards remain live through owner detach,
+ * with resource balance checked by the shared fixture;
+ * rollback restores real PTEs; exact rewalk and lookup-only checks belong to
+ * rollback_test.h.
+ *
+ * Real charged anonymous folios, PTEs, rmap, codec, zs payloads and public
+ * reserve/store/release/rollback execute. objcg and slot allocation use the
+ * existing consumer model. Compressor/arch/commit errors and task-context
+ * routing are serial models; no reclaim scheduling, device or async engine
+ * execution is claimed. No fabricated entries or successful stores.
+ */
+#ifdef MC16_FOLIO_TEST_ENABLED
+#define MC16_STORE_TRANSACTION_TEST_ENABLED
+
+struct mc16_sct_stats {
+ s64 preflight;
+ s64 busy;
+ s64 arch;
+ s64 payload;
+ s64 payload_pages;
+ s64 state;
+ s64 state_pages;
+ s64 bg_ok;
+ s64 bg_fail;
+ s64 direct_ok;
+ s64 direct_fail;
+ s64 pageout_ok;
+ s64 pageout_fail;
+};
+
+static struct mc16_sct_stats mc16_sct_stats_read(void)
+{
+ struct mc16_sct_stats s = { };
+
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ s.preflight = atomic64_read(&memcomp_refresh_preflight_fail_pages);
+ s.busy = atomic64_read(&memcomp_refresh_folio_busy_fail_pages);
+ s.arch = atomic64_read(&memcomp_refresh_arch_fail_pages);
+ s.payload = atomic64_read(&memcomp_refresh_payload_fail);
+ s.payload_pages = atomic64_read(&memcomp_refresh_payload_fail_pages);
+ s.state = atomic64_read(&memcomp_refresh_state_fail);
+ s.state_pages = atomic64_read(&memcomp_refresh_state_fail_pages);
+ s.bg_ok = atomic64_read(&memcomp_bg_store_ok);
+ s.bg_fail = atomic64_read(&memcomp_bg_store_fail);
+ s.direct_ok = atomic64_read(&memcomp_direct_store_ok);
+ s.direct_fail = atomic64_read(&memcomp_direct_store_fail);
+ s.pageout_ok = atomic64_read(&memcomp_pageout_store_ok);
+ s.pageout_fail = atomic64_read(&memcomp_pageout_store_fail);
+#endif
+ return s;
+}
+
+/* Every unspecified counter must remain unchanged. */
+static void mc16_sct_stats_expect(struct kunit *test,
+ struct mc16_sct_stats before,
+ struct mc16_sct_stats delta)
+{
+ struct mc16_sct_stats after = mc16_sct_stats_read();
+
+ if (!IS_ENABLED(CONFIG_MEMCOMP_DEBUG_STATS))
+ return;
+ KUNIT_EXPECT_EQ_MSG(test, after.preflight - before.preflight,
+ delta.preflight, "preflight");
+ KUNIT_EXPECT_EQ_MSG(test, after.busy - before.busy,
+ delta.busy, "busy");
+ KUNIT_EXPECT_EQ_MSG(test, after.arch - before.arch,
+ delta.arch, "arch");
+ KUNIT_EXPECT_EQ_MSG(test, after.payload - before.payload,
+ delta.payload, "payload");
+ KUNIT_EXPECT_EQ_MSG(test, after.payload_pages - before.payload_pages,
+ delta.payload_pages, "payload_pages");
+ KUNIT_EXPECT_EQ_MSG(test, after.state - before.state,
+ delta.state, "state");
+ KUNIT_EXPECT_EQ_MSG(test, after.state_pages - before.state_pages,
+ delta.state_pages, "state_pages");
+ KUNIT_EXPECT_EQ_MSG(test, after.bg_ok - before.bg_ok,
+ delta.bg_ok, "bg_ok");
+ KUNIT_EXPECT_EQ_MSG(test, after.bg_fail - before.bg_fail,
+ delta.bg_fail, "bg_fail");
+ KUNIT_EXPECT_EQ_MSG(test, after.direct_ok - before.direct_ok,
+ delta.direct_ok, "direct_ok");
+ KUNIT_EXPECT_EQ_MSG(test, after.direct_fail - before.direct_fail,
+ delta.direct_fail, "direct_fail");
+ KUNIT_EXPECT_EQ_MSG(test, after.pageout_ok - before.pageout_ok,
+ delta.pageout_ok, "pageout_ok");
+ KUNIT_EXPECT_EQ_MSG(test, after.pageout_fail - before.pageout_fail,
+ delta.pageout_fail, "pageout_fail");
+}
+
+enum mc16_sct_route {
+ MC16_SCT_BACKGROUND,
+ MC16_SCT_DIRECT,
+ MC16_SCT_PAGEOUT,
+};
+
+struct mc16_sct_attempt {
+ bool unlocked;
+ bool extra_ref;
+ bool bad_token;
+ int compress_error;
+ enum memcomp_kunit_fail_stage fail;
+ unsigned int nth;
+ enum mc16_sct_route route;
+};
+
+struct mc16_sct_result {
+ int error;
+ bool success;
+ bool consumed;
+};
+
+/* All leases, flags, markers and locks are restored before any assertion. */
+static struct mc16_sct_result
+mc16_sct_store(struct mc16_flt *f, struct mc16_sct_attempt a)
+{
+ struct mc16_sct_result r = { .error = -EBUSY };
+ struct reclaim_state direct = { };
+ struct reclaim_state *saved_reclaim = current->reclaim_state;
+ struct memcomp_reclaim_ctx *ctx;
+ swp_entry_t saved_swap;
+ unsigned int flags;
+
+ flags = memcomp_test_reclaimer_enter();
+ ctx = memcomp_reclaim_ctx_get(f->nr, MEMCOMP_RECLAIM_KSWAPD);
+ if (!ctx)
+ goto out_flags;
+ ctx->kunit_compress_error = a.compress_error;
+ if (a.route != MC16_SCT_BACKGROUND) {
+ current->flags &= ~PF_KSWAPD;
+ current->reclaim_state = a.route == MC16_SCT_DIRECT ? &direct : NULL;
+ }
+ if (!a.unlocked)
+ folio_lock(f->folio);
+ saved_swap = f->folio->swap;
+ if (a.bad_token)
+ f->folio->swap = make_memcomp_entry(U32_MAX - 1);
+ if (a.extra_ref)
+ folio_get(f->folio);
+ if (a.fail)
+ memcomp_kunit_fail_next(a.fail, a.nth,
+ a.fail == MEMCOMP_KUNIT_FAIL_REFRESH_COMMIT ?
+ -EAGAIN : -EIO);
+ r.success = memcomp_store_after_unmap(f->folio, &ctx);
+ r.consumed = !ctx;
+ /* Defensive put also makes a failed consumption oracle safe to report. */
+ if (ctx)
+ memcomp_reclaim_ctx_put(ctx);
+ memcomp_kunit_fail_clear();
+ if (a.extra_ref)
+ folio_put(f->folio);
+ if (a.bad_token)
+ f->folio->swap = saved_swap;
+ if (!a.unlocked)
+ folio_unlock(f->folio);
+ r.error = 0;
+out_flags:
+ current->reclaim_state = saved_reclaim;
+ memcomp_test_reclaimer_exit(flags);
+ return r;
+}
+
+static bool mc16_sct_reserve(struct mc16_flt *f)
+{
+ bool ret;
+
+ folio_lock(f->folio);
+ ret = memcomp_reserve(f->folio);
+ folio_unlock(f->folio);
+ return ret;
+}
+
+static void mc16_sct_rollback(struct mc16_flt *f)
+{
+ mmap_read_lock(f->mm);
+ folio_lock(f->folio);
+ memcomp_rollback_folio(f->folio);
+ folio_unlock(f->folio);
+ mmap_read_unlock(f->mm);
+}
+
+static bool mc16_sct_release(struct mc16_flt *f)
+{
+ bool ret;
+
+ /* Match the real reclaim actor's isolated-source precondition. */
+ lru_add_drain_all();
+ if (folio_test_lru(f->folio)) {
+ if (!folio_isolate_lru(f->folio))
+ return false;
+ /* Model reclaim isolation, not migration: the source is inactive. */
+ folio_clear_active(f->folio);
+ folio_put(f->folio);
+ }
+ folio_lock(f->folio);
+ ret = memcomp_release_folio(f->folio);
+ if (ret)
+ folio_ref_unfreeze(f->folio, 1);
+ folio_unlock(f->folio);
+ return ret;
+}
+
+static void mc16_sct_copy(struct mc16_flt *f, const void *src)
+{
+ unsigned int i;
+ void *dst;
+
+ folio_lock(f->folio);
+ for (i = 0; i < f->nr; i++) {
+ dst = kmap_local_folio(f->folio, i * PAGE_SIZE);
+ memcpy(dst, src + i * PAGE_SIZE, PAGE_SIZE);
+ kunmap_local(dst);
+ }
+ flush_dcache_folio(f->folio);
+ folio_unlock(f->folio);
+}
+
+/* Nonuniform, highly compressible bytes; page 0 can instead be nonzero SAMEFILL. */
+static void mc16_sct_pattern(u8 *src, unsigned int nr, bool samefill)
+{
+ unsigned int i;
+
+ for (i = 0; i < nr * PAGE_SIZE; i++)
+ src[i] = (i % 251) ^ ((i / PAGE_SIZE) * 0x39);
+ if (samefill)
+ memset(src, 0x5a, PAGE_SIZE);
+}
+
+static void mc16_sct_run(struct mc16_flt *f, struct mc16_sct_attempt a,
+ bool success)
+{
+ struct mc16_sct_result r = mc16_sct_store(f, a);
+
+ KUNIT_EXPECT_EQ(f->test, r.error, 0);
+ KUNIT_EXPECT_TRUE(f->test, r.consumed);
+ KUNIT_EXPECT_EQ(f->test, r.success, success);
+}
+
+static void mc16_sct_expect_pending(struct mc16_flt *f)
+{
+ unsigned int i, flags;
+
+ for (i = 0; i < f->nr; i++) {
+ flags = mc16_flt_flags(&f->guards[i], false);
+ KUNIT_EXPECT_TRUE(f->test, flags & MC16_F_OWNER);
+ KUNIT_EXPECT_EQ(f->test, mc16_flags_state(flags), MC16_PENDING);
+ KUNIT_EXPECT_TRUE(f->test, flags & MC16_F_RESIDENT);
+ KUNIT_EXPECT_FALSE(f->test, flags & MC16_F_REFRESH_PENDING);
+ KUNIT_EXPECT_FALSE(f->test, flags & MC16_F_INCOMPRESSIBLE);
+ KUNIT_EXPECT_EQ(f->test,
+ refcount_read(&f->guards[i].slot->refs), 3U);
+ }
+ KUNIT_EXPECT_EQ(f->test, folio_ref_count(f->folio), 1 + f->nr);
+}
+
+static void mc16_sct_expect_load(struct mc16_flt *f, unsigned int i,
+ const void *src, void *dst)
+{
+ struct memcomp_ref load = { };
+ void *bytes;
+ int ret = -ENOENT;
+
+ if (memcomp_load_folio_pin(f->guards[i].token, &load)) {
+ folio_lock(f->base->dst);
+ ret = memcomp_load_folio(&load, f->base->dst);
+ if (!ret) {
+ bytes = kmap_local_folio(f->base->dst, 0);
+ memcpy(dst, bytes, PAGE_SIZE);
+ kunmap_local(bytes);
+ }
+ folio_unlock(f->base->dst);
+ memcomp_load_folio_abort(&load);
+ }
+ KUNIT_EXPECT_EQ(f->test, ret, 0);
+ if (!ret)
+ KUNIT_EXPECT_EQ(f->test, memcmp(src, dst, PAGE_SIZE), 0);
+}
+
+static void memcomp_test_store_transaction_success(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 1);
+ struct mc16_sct_stats before;
+ u8 *src, *dst;
+ unsigned int i;
+ int anon_refs;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ src = kunit_kmalloc(test, 2 * PAGE_SIZE, GFP_KERNEL);
+ dst = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ KUNIT_ASSERT_NOT_NULL(test, dst);
+ mc16_sct_pattern(src, 2, true);
+ mc16_sct_copy(f, src);
+ anon_refs = atomic_read(&f->anon_vma->refcount);
+ KUNIT_ASSERT_TRUE(test, mc16_sct_reserve(f));
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->anon_vma->refcount), anon_refs + 1);
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ for (i = 0; i < f->nr; i++)
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, i), 0);
+ before = mc16_sct_stats_read();
+ mc16_sct_run(f,
+ (struct mc16_sct_attempt) { }, true);
+ mc16_sct_stats_expect(test, before,
+ (struct mc16_sct_stats) { .bg_ok = 2 });
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->anon_vma->refcount), anon_refs);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), 1);
+ KUNIT_EXPECT_FALSE(test, folio_mapped(f->folio));
+ KUNIT_EXPECT_EQ(test, mc16_flt_flags(&f->guards[0], false) &
+ MC16_STATE_MASK, (unsigned int)MC16_SAMEFILL);
+ KUNIT_EXPECT_EQ(test, mc16_flt_flags(&f->guards[1], false) &
+ MC16_STATE_MASK, (unsigned int)MC16_STORED);
+ KUNIT_ASSERT_TRUE(test, mc16_sct_release(f));
+ for (i = 0; i < f->nr; i++) {
+ KUNIT_EXPECT_FALSE(test, mc16_flt_flags(&f->guards[i], false) &
+ MC16_F_OWNER);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guards[i].slot->refs), 2U);
+ mc16_sct_expect_load(f, i, src + i * PAGE_SIZE, dst);
+ }
+ /* Shared cleanup zaps real PTE refs, then puts guards and checks balances. */
+}
+
+static void memcomp_test_store_transaction_preflight_lookup(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 1);
+ struct memcomp_ref absent = { };
+ struct mc16_sct_stats before;
+ bool found;
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_sct_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ for (i = 0; i < f->nr; i++)
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, i), 0);
+ found = memcomp_entry_get_ref(U32_MAX - 1, &absent);
+ if (found)
+ memcomp_entry_put_ref(&absent);
+ KUNIT_ASSERT_FALSE(test, found);
+ /* Each failed preflight preserves pending owners and consumes its context. */
+ before = mc16_sct_stats_read();
+ mc16_sct_run(f,
+ (struct mc16_sct_attempt) { .unlocked = true }, false);
+ mc16_sct_stats_expect(test, before,
+ (struct mc16_sct_stats) { .preflight = 2, .bg_fail = 2 });
+ mc16_sct_expect_pending(f);
+ before = mc16_sct_stats_read();
+ mc16_sct_run(f,
+ (struct mc16_sct_attempt) { .bad_token = true }, false);
+ mc16_sct_stats_expect(test, before,
+ (struct mc16_sct_stats) { .preflight = 2, .bg_fail = 2 });
+ mc16_sct_expect_pending(f);
+ mc16_sct_rollback(f);
+ mc16_flt_expect_present(f, 0, f->addr);
+}
+
+static void memcomp_test_store_transaction_busy(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 1);
+ struct mc16_sct_stats before;
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_sct_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ before = mc16_sct_stats_read();
+ /* Real mappings refuse store before the span lookup/codec path. */
+ mc16_sct_run(f,
+ (struct mc16_sct_attempt) { }, false);
+ for (i = 0; i < f->nr; i++)
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, i), 0);
+ mc16_sct_run(f,
+ (struct mc16_sct_attempt) { .extra_ref = true }, false);
+ mc16_sct_stats_expect(test, before,
+ (struct mc16_sct_stats) { .busy = 4, .bg_fail = 4 });
+ mc16_sct_expect_pending(f);
+ mc16_sct_rollback(f);
+}
+
+static void memcomp_test_store_transaction_arch_failure(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 1);
+ struct mc16_sct_stats before;
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_sct_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ for (i = 0; i < f->nr; i++)
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, i), 0);
+ before = mc16_sct_stats_read();
+ mc16_sct_run(f,
+ (struct mc16_sct_attempt) {
+ .fail = MEMCOMP_KUNIT_FAIL_ARCH_PREPARE, .nth = 1,
+ }, false);
+ mc16_sct_stats_expect(test, before,
+ (struct mc16_sct_stats) { .arch = 2, .bg_fail = 2 });
+ mc16_sct_expect_pending(f);
+ mc16_sct_rollback(f);
+ mc16_flt_expect_present(f, 0, f->addr);
+}
+
+static void memcomp_test_store_transaction_payload_pages(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 1);
+ struct mc16_sct_stats before;
+ long charged;
+ u8 *src;
+ unsigned int i, page;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ s64 compress_fail;
+#endif
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ src = kunit_kmalloc(test, 2 * PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ KUNIT_ASSERT_TRUE(test, mc16_sct_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ for (i = 0; i < f->nr; i++)
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, i), 0);
+ charged = atomic_long_read(&f->base->model.charged);
+ for (page = 0; page < f->nr; page++) {
+ /* SAMEFILL bypasses compression, so -EIO reaches page 1. */
+ mc16_sct_pattern(src, 2, page == 1);
+ mc16_sct_copy(f, src);
+ before = mc16_sct_stats_read();
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ compress_fail = atomic64_read(&memcomp_compress_fail);
+#endif
+ mc16_sct_run(f,
+ (struct mc16_sct_attempt) { .compress_error = -EIO }, false);
+ mc16_sct_stats_expect(test, before,
+ (struct mc16_sct_stats) {
+ .payload = 1, .payload_pages = 2, .bg_fail = 2,
+ });
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_compress_fail),
+ compress_fail + 1);
+#endif
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->base->model.charged),
+ charged);
+ mc16_sct_expect_pending(f);
+ }
+ /* The prepared page-0 prefix in the second attempt is SAMEFILL, not zs. */
+ mc16_sct_rollback(f);
+}
+
+static void memcomp_test_store_transaction_partial_commit(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 1);
+ struct mc16_sct_stats before;
+ u8 *src, *dst;
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ src = kunit_kmalloc(test, 2 * PAGE_SIZE, GFP_KERNEL);
+ dst = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ KUNIT_ASSERT_NOT_NULL(test, dst);
+ mc16_sct_pattern(src, 2, false);
+ mc16_sct_copy(f, src);
+ KUNIT_ASSERT_TRUE(test, mc16_sct_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ for (i = 0; i < f->nr; i++)
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, i), 0);
+ before = mc16_sct_stats_read();
+ mc16_sct_run(f,
+ (struct mc16_sct_attempt) {
+ .fail = MEMCOMP_KUNIT_FAIL_REFRESH_COMMIT, .nth = 2,
+ }, false);
+ mc16_sct_stats_expect(test, before,
+ (struct mc16_sct_stats) {
+ .state = 1, .state_pages = 2, .bg_fail = 2,
+ });
+ KUNIT_EXPECT_EQ(test, mc16_flt_flags(&f->guards[0], false) &
+ MC16_STATE_MASK, (unsigned int)MC16_STORED);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(mc16_flt_flags(&f->guards[1], false)),
+ MC16_PENDING);
+ KUNIT_EXPECT_TRUE(test, mc16_flt_flags(&f->guards[1], false) &
+ MC16_F_RESIDENT);
+ KUNIT_EXPECT_FALSE(test, mc16_flt_flags(&f->guards[1], false) &
+ MC16_F_REFRESH_PENDING);
+ KUNIT_EXPECT_FALSE(test, mc16_flt_flags(&f->guards[0], false) &
+ MC16_F_INCOMPRESSIBLE);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), 2);
+ mc16_sct_rollback(f);
+ KUNIT_EXPECT_FALSE(test, mc16_flt_flags(&f->guards[0], false) &
+ MC16_F_OWNER);
+ mc16_sct_expect_load(f, 0, src, dst);
+}
+
+static void memcomp_test_store_transaction_incompressible(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 0);
+ struct mc16_sct_stats before;
+ u8 *src;
+ bool skip;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ s64 rejected, compress_fail;
+#endif
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ src = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ memcomp_test_fill_incompressible(src, 1);
+ mc16_sct_copy(f, src);
+ KUNIT_ASSERT_TRUE(test, mc16_sct_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, 0), 0);
+ before = mc16_sct_stats_read();
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ rejected = atomic64_read(&memcomp_raw_reject_huge) +
+ atomic64_read(&memcomp_raw_reject_ge_page);
+ compress_fail = atomic64_read(&memcomp_compress_fail);
+#endif
+ /* Real codec output must exceed the accepted single-page payload size. */
+ mc16_sct_run(f,
+ (struct mc16_sct_attempt) { }, false);
+ mc16_sct_stats_expect(test, before,
+ (struct mc16_sct_stats) {
+ .payload = 1, .payload_pages = 1, .bg_fail = 1,
+ });
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_raw_reject_huge) +
+ atomic64_read(&memcomp_raw_reject_ge_page), rejected + 1);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_compress_fail), compress_fail);
+#endif
+ KUNIT_EXPECT_TRUE(test, mc16_flt_flags(&f->guards[0], false) &
+ MC16_F_INCOMPRESSIBLE);
+ mc16_sct_rollback(f);
+ mc16_flt_expect_present(f, 0, f->addr);
+ folio_lock(f->folio);
+ skip = memcomp_folio_incompressible(f->folio);
+ folio_unlock(f->folio);
+ KUNIT_EXPECT_TRUE(test, skip);
+ /* Skip predicate executes; vmscan's skip counter increment is not modeled. */
+ memcomp_entry_put_ref(&f->guards[0]);
+}
+
+static void memcomp_test_store_transaction_outcomes(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 1);
+ struct mc16_sct_stats before, delta;
+ unsigned int i;
+ enum mc16_sct_route route;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_sct_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ for (i = 0; i < f->nr; i++)
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, i), 0);
+ for (route = MC16_SCT_BACKGROUND; route <= MC16_SCT_PAGEOUT; route++) {
+ before = mc16_sct_stats_read();
+ mc16_sct_run(f,
+ (struct mc16_sct_attempt) {
+ .unlocked = true, .route = route,
+ }, false);
+ delta = (struct mc16_sct_stats) { .preflight = 2 };
+ if (route == MC16_SCT_BACKGROUND)
+ delta.bg_fail = 2;
+ else if (route == MC16_SCT_DIRECT)
+ delta.direct_fail = 2;
+ else
+ delta.pageout_fail = 2;
+ mc16_sct_stats_expect(test, before, delta);
+ }
+ mc16_sct_expect_pending(f);
+ mc16_sct_rollback(f);
+}
+#endif /* MC16_FOLIO_TEST_ENABLED */
+#endif /* _MM_MEMCOMP_TESTS_STORE_TRANSACTION_TEST_H */
diff --git a/mm/memcomp/tests/suite.h b/mm/memcomp/tests/suite.h
new file mode 100644
index 0000000000000000000000000000000000000000..42c5099cdcbfc5f772ef3caaf578816dfe4f92b6
--- /dev/null
+++ b/mm/memcomp/tests/suite.h
@@ -0,0 +1,693 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _MM_MEMCOMP_TESTS_SUITE_H
+#define _MM_MEMCOMP_TESTS_SUITE_H
+
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+
+#include <kunit/test.h>
+#include <crypto/internal/acompress.h>
+#include <linux/module.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/jiffies.h>
+#include <linux/kthread.h>
+#include <linux/mman.h>
+#include <linux/printk.h>
+#include <linux/sched.h>
+#include <linux/uaccess.h>
+#include <linux/utsname.h>
+
+#include "shared_test.h"
+#include "async_test.h"
+
+#include "format_test.h"
+#include "core_test.h"
+#include "storage_test.h"
+#include "workingset_storage_test.h"
+#include "cache_reserve_test.h"
+#include "access_test.h"
+#include "refs_test.h"
+#include "payload_test.h"
+#include "cache_store_test.h"
+#include "owner_test.h"
+#include "span_test.h"
+#include "load_test.h"
+#include "swapoff_test.h"
+#include "writeback_test.h"
+#include "zero_test.h"
+#include "query_test.h"
+#include "anon_store_test.h"
+#include "scan_test.h"
+#include "zero_load_test.h"
+#include "consumers_test.h"
+#include "cache_folio_test.h"
+#include "shmem_retry_test.h"
+#include "cache_xarray_race_test.h"
+#include "migrate_test.h"
+#include "writeback_batch_test.h"
+#include "drain_test.h"
+#include "swapoff_walk_test.h"
+#include "defrag_test.h"
+#include "resident_test.h"
+#include "folio_test.h"
+#include "store_transaction_test.h"
+#include "workingset_store_test.h"
+#include "replacement_test.h"
+#include "budget_test.h"
+#include "fault_test.h"
+#include "workingset_load_test.h"
+#include "workingset_mthp_error_test.h"
+#include "shmem_zero_fault_test.h"
+#include "budget_swapped_test.h"
+#include "budget_packed_test.h"
+#include "zero_convert_test.h"
+#include "zero_carrier_test.h"
+#include "zero_callers_test.h"
+#include "zero_hint_test.h"
+#include "aging_test.h"
+#include "enospc_test.h"
+#include "fault_alloc_test.h"
+#include "mincore_test.h"
+#include "transport_test.h"
+
+#include "public_api_test.h"
+#include "public_driver_test.h"
+#include "rollback_test.h"
+
+/* Registered first so it runs after every fixture-specific cleanup action. */
+static void memcomp_test_case_cleanup(void *data)
+{
+ struct kunit *test = data;
+
+ mc16_test_reclaim_retained();
+ mc16_test_aging_reset();
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 0UL);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+}
+
+static int memcomp_test_case_init(struct kunit *test)
+{
+ mc16_test_aging_reset();
+ atomic_set(&mc16_test_aging_schedules, 0);
+ kunit_activate_static_stub(test, mc16_schedule_aging,
+ mc16_test_schedule_aging);
+ return kunit_add_action_or_reset(test, memcomp_test_case_cleanup, test);
+}
+
+static struct kunit_case memcomp_test_cases[] = {
+#ifdef MC16_SHMEM_RETRY_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_shmem_publish_retry),
+ KUNIT_CASE(memcomp_test_shmem_retry_eio),
+ KUNIT_CASE(memcomp_test_shmem_retry_enomem),
+ KUNIT_CASE(memcomp_test_cache_xarray_race_no_restore),
+#ifdef MC16_SHMEM_ZERO_FAULT_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_shmem_token0_not_major),
+#endif
+#endif
+#ifdef MC16_WORKINGSET_LOAD_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_workingset_load_charge_failure),
+ KUNIT_CASE(memcomp_test_workingset_load_take_once),
+ KUNIT_CASE(memcomp_test_workingset_load_ineligible),
+ KUNIT_CASE(memcomp_test_workingset_load_read_error),
+ KUNIT_CASE(memcomp_test_workingset_load_pte_race),
+#endif
+#ifdef MC16_WORKINGSET_MTHP_TEST_ENABLED
+ KUNIT_CASE_PARAM(memcomp_test_workingset_load_mthp,
+ mc16_wl_batch_gen_params),
+ KUNIT_CASE_PARAM(memcomp_test_workingset_load_mthp_read_error,
+ mc16_wl_mthp_error_gen_params),
+#endif
+#ifdef MC16_WORKINGSET_MAJOR_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_workingset_load_major_shared),
+ KUNIT_CASE(memcomp_test_workingset_load_zero_not_major),
+#endif
+#ifdef MC16_WORKINGSET_RESIDENT_MAJOR_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_workingset_load_resident_not_major),
+#endif
+#ifdef MC16_WORKINGSET_SCOPE_TEST_ENABLED
+ KUNIT_CASE_PARAM(memcomp_test_workingset_load_scope,
+ mc16_wl_scope_gen_params),
+ KUNIT_CASE_PARAM(memcomp_test_workingset_load_cache_scope,
+ mc16_wl_cache_gen_params),
+ KUNIT_CASE(memcomp_test_workingset_load_scope_cold),
+ KUNIT_CASE(memcomp_test_workingset_load_scope_hot),
+#endif
+#ifdef MC16_WORKINGSET_TRANSPORT_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_workingset_load_transport),
+#endif
+
+#ifdef CONFIG_SWAP
+ KUNIT_CASE(memcomp_test_transport_bitmap_lifecycle),
+ KUNIT_CASE(memcomp_test_transport_bitmap_range),
+ KUNIT_CASE(memcomp_test_transport_bitmap_devices),
+ KUNIT_CASE(memcomp_test_transport_bitmap_budget),
+#endif
+
+ KUNIT_CASE(memcomp_test_workingset_storage_marker),
+#ifdef CONFIG_MEMCG
+ KUNIT_CASE(memcomp_test_workingset_storage_words),
+ KUNIT_CASE(memcomp_test_workingset_storage_segments),
+ KUNIT_CASE(memcomp_test_workingset_storage_nomem),
+ KUNIT_CASE(memcomp_test_workingset_storage_retire),
+ KUNIT_CASE(memcomp_test_workingset_storage_retire_nomem),
+ KUNIT_CASE(memcomp_test_workingset_storage_key_identity),
+ KUNIT_CASE(memcomp_test_workingset_storage_key_ref),
+ KUNIT_CASE(memcomp_test_workingset_storage_key_ref_failure),
+ KUNIT_CASE(memcomp_test_workingset_storage_source_id_failure),
+ KUNIT_CASE(memcomp_test_workingset_storage_source_id),
+ KUNIT_CASE(memcomp_test_workingset_storage_root_source),
+ KUNIT_CASE(memcomp_test_workingset_storage_charge_fallback),
+#endif
+#ifdef MC16_WORKINGSET_STORE_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_workingset_store_single),
+ KUNIT_CASE(memcomp_test_workingset_store_mthp),
+ KUNIT_CASE(memcomp_test_workingset_store_freeze_failure),
+ KUNIT_CASE(memcomp_test_workingset_store_bad_sibling),
+ KUNIT_CASE(memcomp_test_workingset_store_gate_race),
+ KUNIT_CASE(memcomp_test_workingset_store_prepared),
+ KUNIT_CASE(memcomp_test_workingset_store_cancel_before_commit),
+ KUNIT_CASE(memcomp_test_workingset_store_cache),
+ KUNIT_CASE(memcomp_test_workingset_store_cache_abort),
+ KUNIT_CASE(memcomp_test_workingset_store_legacy),
+ KUNIT_CASE(memcomp_test_workingset_store_zero_excluded),
+ KUNIT_CASE(memcomp_test_workingset_store_partial),
+#endif
+ KUNIT_CASE(memcomp_test_mincore_zero),
+#ifdef MC16_MINCORE_PENDING_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_mincore_pending),
+#endif
+#ifdef MC16_MINCORE_SWAP_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_mincore_pool),
+ KUNIT_CASE(memcomp_test_mincore_swapped_absent),
+ KUNIT_CASE(memcomp_test_mincore_swapped_cached),
+ KUNIT_CASE(memcomp_test_mincore_swapped_incomplete),
+#endif
+#ifdef MC16_MINCORE_SHMEM_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_mincore_shmem),
+ KUNIT_CASE(memcomp_test_mincore_shmem_replaced),
+#endif
+#ifdef MC16_MINCORE_MM_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_mincore_mthp),
+ KUNIT_CASE(memcomp_test_mincore_pte_replaced),
+#endif
+
+#ifdef MC16_ZERO_CARRIER_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_zero_carrier_ready),
+ KUNIT_CASE(memcomp_test_zero_carrier_commit_failure),
+ KUNIT_CASE(memcomp_test_zero_carrier_refresh_preflight),
+ KUNIT_CASE(memcomp_test_zero_carrier_store_preflight),
+ KUNIT_CASE(memcomp_test_zero_carrier_nonzero),
+ KUNIT_CASE(memcomp_test_zero_carrier_mthp),
+ KUNIT_CASE(memcomp_test_zero_carrier_token_mismatch),
+ KUNIT_CASE(memcomp_test_zero_carrier_invalid),
+ KUNIT_CASE(memcomp_test_zero_carrier_borrow_owner),
+ KUNIT_CASE(memcomp_test_zero_carrier_lifecycle_refault1),
+ KUNIT_CASE(memcomp_test_zero_carrier_lifecycle_exit),
+#endif
+#ifdef CONFIG_MEMCG
+ KUNIT_CASE(memcomp_test_aging_batch_bounds),
+ KUNIT_CASE(memcomp_test_aging_deadlines),
+ KUNIT_CASE(memcomp_test_aging_mutation),
+ KUNIT_CASE(memcomp_test_aging_eligibility),
+ KUNIT_CASE(memcomp_test_aging_requests),
+ KUNIT_CASE(memcomp_test_aging_rcu_ownership),
+ KUNIT_CASE(memcomp_test_aging_teardown),
+ KUNIT_CASE(memcomp_test_aging_reuse),
+ KUNIT_CASE(memcomp_test_aging_stops),
+ KUNIT_CASE(memcomp_test_aging_shrinker),
+ KUNIT_CASE(memcomp_test_aging_real_work),
+ KUNIT_CASE(memcomp_test_retained_x1),
+ KUNIT_CASE(memcomp_test_retained_x2_segment),
+ KUNIT_CASE(memcomp_test_retained_x2_replace),
+ KUNIT_CASE(memcomp_test_meta_shrinker_exact),
+ KUNIT_CASE(memcomp_test_meta_shrinker_wrap),
+ KUNIT_CASE(memcomp_test_meta_shrinker_progress),
+ KUNIT_CASE(memcomp_test_meta_shrinker_drained),
+#endif
+#ifdef MC16_ZERO_HINT_DENSITY_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_zero_hint_ordinary_refault),
+ KUNIT_CASE(memcomp_test_zero_hint_serial533),
+ KUNIT_CASE(memcomp_test_zero_hint_route512),
+ KUNIT_CASE(memcomp_test_zero_hint_density65536),
+#endif
+#ifdef MC16_FOLIO_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_budget_pageout_stored),
+ KUNIT_CASE(memcomp_test_budget_pageout_samefill),
+ KUNIT_CASE(memcomp_test_budget_refault1_stored),
+ KUNIT_CASE(memcomp_test_budget_refault1_samefill),
+ KUNIT_CASE(memcomp_test_budget_fork_stored),
+ KUNIT_CASE(memcomp_test_budget_fork_samefill),
+ KUNIT_CASE(memcomp_test_budget_exit_stored),
+ KUNIT_CASE(memcomp_test_budget_exit_samefill),
+ KUNIT_CASE(memcomp_test_budget_refault4_stored),
+ KUNIT_CASE(memcomp_test_budget_refault4_samefill),
+ KUNIT_CASE(memcomp_test_budget_fixed512),
+#endif
+#ifdef MC16_REPLACEMENT_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_replacement_mixed_sibling),
+ KUNIT_CASE(memcomp_test_replacement_mixed_head),
+ KUNIT_CASE(memcomp_test_replacement_stored_order0),
+ KUNIT_CASE(memcomp_test_replacement_stored_order1),
+#endif
+#ifdef MC16_STORE_TRANSACTION_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_store_transaction_success),
+ KUNIT_CASE(memcomp_test_store_transaction_preflight_lookup),
+ KUNIT_CASE(memcomp_test_store_transaction_busy),
+ KUNIT_CASE(memcomp_test_store_transaction_arch_failure),
+ KUNIT_CASE(memcomp_test_store_transaction_payload_pages),
+ KUNIT_CASE(memcomp_test_store_transaction_partial_commit),
+ KUNIT_CASE(memcomp_test_store_transaction_incompressible),
+ KUNIT_CASE(memcomp_test_store_transaction_outcomes),
+#endif
+#ifdef MC16_DRIVER_RESIDENT_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_public_resident_restore),
+ KUNIT_CASE(memcomp_test_public_restore_error),
+#endif
+#ifdef MC16_DRIVER_SWAPOFF_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_public_swapoff_type_miss),
+ KUNIT_CASE(memcomp_test_public_swapoff_error),
+#endif
+#ifdef CONFIG_MEMCG
+ KUNIT_CASE(memcomp_test_async_registered_submit),
+ KUNIT_CASE(memcomp_test_async_registered_backlog),
+ KUNIT_CASE(memcomp_test_async_registered_error),
+ KUNIT_CASE(memcomp_test_async_registered_finish_fail),
+ KUNIT_CASE(memcomp_test_async_software_samefill),
+ KUNIT_CASE(memcomp_test_async_lease_identity),
+ KUNIT_CASE(memcomp_test_async_precomp_offset),
+ KUNIT_CASE(memcomp_test_async_precomp_mixed),
+#endif
+#ifdef MC16_ROLLBACK_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_public_rollback_restored),
+ KUNIT_CASE(memcomp_test_public_rollback_lookup_only),
+ KUNIT_CASE(memcomp_test_public_rollback_rewalk),
+#endif
+#ifdef MC16_FAULT_MM_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_public_madv_free),
+ KUNIT_CASE(memcomp_test_public_madv_free_partial),
+ KUNIT_CASE(memcomp_test_public_madv_free_shared),
+ KUNIT_CASE(memcomp_test_public_fault),
+#endif
+#ifdef CONFIG_MEMCG
+ KUNIT_CASE(memcomp_test_public_cache_transaction),
+ KUNIT_CASE(memcomp_test_public_ref_domains),
+ KUNIT_CASE(memcomp_test_public_query_busy),
+ KUNIT_CASE(memcomp_test_public_refresh_retry),
+ KUNIT_CASE(memcomp_test_public_cache_charge_failure),
+ KUNIT_CASE(memcomp_test_public_invalid_refs),
+#endif
+ KUNIT_CASE(memcomp_test_cache_folio_compressed),
+ KUNIT_CASE(memcomp_test_cache_folio_samefill),
+ KUNIT_CASE(memcomp_test_cache_folio_last_byte),
+ KUNIT_CASE(memcomp_test_cache_folio_zero_abort),
+ KUNIT_CASE(memcomp_test_cache_folio_zero_publish),
+ KUNIT_CASE(memcomp_test_cache_folio_arch_failure),
+ KUNIT_CASE(memcomp_test_cache_folio_arch_seam),
+ KUNIT_CASE(memcomp_test_cache_folio_samefill_arch_failure),
+ KUNIT_CASE(memcomp_test_cache_folio_disabled),
+ KUNIT_CASE(memcomp_test_cache_folio_preflight),
+ KUNIT_CASE(memcomp_test_cache_folio_large),
+ KUNIT_CASE(memcomp_test_cache_folio_metadata_failure),
+#ifdef CONFIG_MEMCG
+ KUNIT_CASE(memcomp_test_cache_folio_charge_failure),
+ KUNIT_CASE(memcomp_test_cache_folio_key_charge_failure),
+#endif
+#if defined(CONFIG_ARM64) && defined(CONFIG_ARM64_MTE)
+ KUNIT_CASE(memcomp_test_cache_folio_tagged_zero),
+#endif
+#ifdef MC16_FOLIO_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_folio_reserve),
+ KUNIT_CASE(memcomp_test_folio_reserve_failure),
+ KUNIT_CASE(memcomp_test_folio_rollback_rmap),
+ KUNIT_CASE(memcomp_test_folio_partial_rollback),
+ KUNIT_CASE(memcomp_test_folio_partial_invalidate),
+ KUNIT_CASE(memcomp_test_folio_release),
+ KUNIT_CASE(memcomp_test_folio_release_bad_head),
+ KUNIT_CASE(memcomp_test_folio_release_bad_sibling),
+ KUNIT_CASE(memcomp_test_folio_transport),
+ KUNIT_CASE(memcomp_test_folio_missed_rmap),
+#endif
+#ifdef CONFIG_MEMCG
+ KUNIT_CASE(memcomp_test_migrate_stored),
+ KUNIT_CASE(memcomp_test_migrate_samefill),
+ KUNIT_CASE(memcomp_test_migrate_swapped),
+ KUNIT_CASE(memcomp_test_migrate_rollback),
+ KUNIT_CASE(memcomp_test_migrate_partial),
+ KUNIT_CASE(memcomp_test_migrate_revalidate),
+ KUNIT_CASE(memcomp_test_migrate_pending),
+ KUNIT_CASE(memcomp_test_migrate_geometry),
+ KUNIT_CASE(memcomp_test_migrate_owner),
+ KUNIT_CASE(memcomp_test_migrate_missing),
+ KUNIT_CASE(memcomp_test_migrate_locks),
+ KUNIT_CASE(memcomp_test_migrate_rollback_missing),
+ KUNIT_CASE(memcomp_test_migrate_rollback_invalid),
+ KUNIT_CASE(memcomp_test_migrate_incompressible),
+ KUNIT_CASE(memcomp_test_migrate_single),
+ KUNIT_CASE(memcomp_test_migrate_dst_geometry),
+ KUNIT_CASE(memcomp_test_migrate_range),
+ KUNIT_CASE(memcomp_test_migrate_record_geometry),
+#endif
+ KUNIT_CASE(memcomp_test_fault_alloc_single_page),
+ KUNIT_CASE(memcomp_test_fault_alloc_defrag_policy),
+ KUNIT_CASE(memcomp_test_fault_alloc_single_fallback),
+ KUNIT_CASE(memcomp_test_format_layout),
+ KUNIT_CASE(memcomp_test_zero_page),
+ KUNIT_CASE(memcomp_test_zero_owners),
+ KUNIT_CASE(memcomp_test_zero_batch_bounds),
+ KUNIT_CASE(memcomp_test_zero_stat),
+ KUNIT_CASE(memcomp_test_zero_cross_cpu),
+#ifdef MC16_ZERO_CALLERS_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_zero_callers_fork_get),
+ KUNIT_CASE(memcomp_test_zero_callers_put_domains),
+ KUNIT_CASE(memcomp_test_zero_callers_put_bounds),
+ KUNIT_CASE(memcomp_test_zero_callers_query),
+ KUNIT_CASE(memcomp_test_zero_callers_stat),
+ KUNIT_CASE(memcomp_test_zero_callers_batch_zero),
+ KUNIT_CASE(memcomp_test_zero_callers_batch_positive),
+ KUNIT_CASE(memcomp_test_zero_callers_batch_exclusive),
+ KUNIT_CASE(memcomp_test_zero_callers_batch_soft_dirty),
+ KUNIT_CASE(memcomp_test_zero_callers_batch_uffd_wp),
+ KUNIT_CASE(memcomp_test_zero_callers_cache_publish),
+ KUNIT_CASE(memcomp_test_zero_callers_cache_load),
+ KUNIT_CASE(memcomp_test_zero_callers_cache_invalid),
+ KUNIT_CASE(memcomp_test_zero_callers_release),
+ KUNIT_CASE(memcomp_test_zero_callers_release_bounds),
+ KUNIT_CASE(memcomp_test_zero_callers_load_read),
+ KUNIT_CASE(memcomp_test_zero_callers_load_write),
+ KUNIT_CASE(memcomp_test_zero_callers_load_oom),
+#endif
+#ifdef MC16_ZERO_LOAD_MM_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_zero_load_write),
+ KUNIT_CASE(memcomp_test_zero_load_softdirty),
+ KUNIT_CASE(memcomp_test_zero_load_softdirty_wp),
+ KUNIT_CASE(memcomp_test_zero_load_uffd_wp),
+ KUNIT_CASE(memcomp_test_zero_load_readonly),
+ KUNIT_CASE(memcomp_test_zero_load_changed),
+ KUNIT_CASE(memcomp_test_zero_load_zapped),
+ KUNIT_CASE(memcomp_test_zero_load_alloc_oom),
+ KUNIT_CASE(memcomp_test_zero_load_charge_oom),
+ KUNIT_CASE(memcomp_test_zero_load_retry),
+#endif
+ KUNIT_CASE(memcomp_test_format_states_orders),
+ KUNIT_CASE(memcomp_test_format_flags_reject),
+ KUNIT_CASE(memcomp_test_format_length_bounds),
+ KUNIT_CASE(memcomp_test_format_payload_states),
+ KUNIT_CASE(memcomp_test_format_full_words),
+ KUNIT_CASE(memcomp_test_format_swap_raw),
+ KUNIT_CASE(memcomp_test_format_swap_packed),
+ KUNIT_CASE(memcomp_test_format_swap_encode_reject),
+ KUNIT_CASE(memcomp_test_format_swap_decode_reject),
+ KUNIT_CASE(memcomp_test_format_pending_record),
+ KUNIT_CASE(memcomp_test_format_member_index_orders),
+ KUNIT_CASE(memcomp_test_format_member_index_reject),
+ KUNIT_CASE(memcomp_test_format_null_outputs),
+#ifdef CONFIG_SWAP
+#ifdef CONFIG_XARRAY_MULTI
+#endif
+#endif
+ KUNIT_CASE(memcomp_test_initial_compression_placement),
+ KUNIT_CASE(memcomp_test_writeback_pressure_scope),
+ KUNIT_CASE(memcomp_test_writeback_feedback),
+ KUNIT_CASE(memcomp_test_writeback_budget_values),
+ KUNIT_CASE(memcomp_test_writeback_level_floor),
+ KUNIT_CASE(memcomp_test_writeback_gate_values),
+ KUNIT_CASE(memcomp_test_writeback_drain_loop),
+ KUNIT_CASE(memcomp_test_writeback_drain_transient),
+ KUNIT_CASE(memcomp_test_writeback_drain_recovery_applied),
+ KUNIT_CASE(memcomp_test_writeback_drain_handoff),
+ KUNIT_CASE(memcomp_test_writeback_drain_slice),
+ KUNIT_CASE(memcomp_test_writeback_pass_bounded),
+#ifdef CONFIG_SWAP
+#endif
+ KUNIT_CASE(memcomp_test_writeback_budget_vs_legacy),
+ KUNIT_CASE(memcomp_test_comp_ctx_bitmap),
+ KUNIT_CASE(memcomp_test_comp_budget),
+ KUNIT_CASE(memcomp_test_direct_staging_policy),
+ KUNIT_CASE(memcomp_test_direct_staging_pool),
+ KUNIT_CASE(memcomp_test_zstd_real_page_roundtrip),
+ KUNIT_CASE(memcomp_test_zstd_page_ratio_bench),
+ KUNIT_CASE(memcomp_test_zstd_huf_cache_frames),
+ KUNIT_CASE(memcomp_test_zstd_huf_cache_shift),
+ KUNIT_CASE(memcomp_test_direct_staging_execution),
+ KUNIT_CASE(memcomp_test_store_consumes_context),
+ KUNIT_CASE(memcomp_test_async_pool),
+ KUNIT_CASE(memcomp_test_async_late_registration),
+ KUNIT_CASE(memcomp_test_same_filled_plausible),
+ KUNIT_CASE(memcomp_test_ctx_timing_helpers),
+ KUNIT_CASE(memcomp_test_comp_ctx_configuration),
+ KUNIT_CASE(memcomp_test_stock_transforms),
+ KUNIT_CASE(memcomp_test_zs_gfp),
+ KUNIT_CASE(memcomp_test_reclaim_ctx_admission),
+ KUNIT_CASE(memcomp_test_reclaim_ctx_precheck),
+ KUNIT_CASE(memcomp_test_reserve_eligibility),
+ KUNIT_CASE(memcomp_test_admission_rollback_unescaped),
+ KUNIT_CASE(memcomp_test_samefill),
+ KUNIT_CASE(memcomp_test_raw_payload_threshold),
+ KUNIT_CASE(memcomp_test_compressor_e2big),
+ KUNIT_CASE(memcomp_test_mixed_raw_net_reclaim),
+ KUNIT_CASE(memcomp_test_state_signed_sum),
+ KUNIT_CASE(memcomp_test_incompressible_mark_lifetime),
+#ifdef CONFIG_ADVISE_SYSCALLS
+ KUNIT_CASE(memcomp_test_lazyfree_mark_discarded),
+#endif
+ KUNIT_CASE(memcomp_test_rollback_restores_ptes),
+ KUNIT_CASE(memcomp_test_store_drops_anon_vma),
+ KUNIT_CASE(memcomp_test_invalidate_drops_anon_vma),
+#ifdef CONFIG_MEMCG
+ KUNIT_CASE(memcomp_test_record_members),
+ KUNIT_CASE(memcomp_test_record_abort),
+ KUNIT_CASE(memcomp_test_record_charge_failure),
+ KUNIT_CASE(memcomp_test_reserve_boundaries),
+ KUNIT_CASE(memcomp_test_reserve_holes),
+ KUNIT_CASE(memcomp_test_segment_retire),
+ KUNIT_CASE(memcomp_test_segment_retire_nomem),
+ KUNIT_CASE(memcomp_test_reserve_nomem),
+ KUNIT_CASE(memcomp_test_reserve_race),
+ KUNIT_CASE(memcomp_test_cluster_key_lifetime),
+ KUNIT_CASE(memcomp_test_pending_final_guard),
+ KUNIT_CASE(memcomp_test_samefill_normalize),
+ KUNIT_CASE(memcomp_test_payload_final_release),
+ KUNIT_CASE(memcomp_test_segment_reserve_race),
+ KUNIT_CASE(memcomp_test_rollover_retire_race),
+ KUNIT_CASE(memcomp_test_record_outlives_key),
+ KUNIT_CASE(memcomp_test_header_before_segment_callback),
+ KUNIT_CASE(memcomp_test_reserve_id_failure),
+ KUNIT_CASE(memcomp_test_reserve_key_failure),
+ KUNIT_CASE(memcomp_test_put_pair),
+ KUNIT_CASE(memcomp_test_payload_samefill),
+ KUNIT_CASE(memcomp_test_payload_replace),
+ KUNIT_CASE(memcomp_test_payload_stale),
+ KUNIT_CASE(memcomp_test_payload_charge_failure),
+ KUNIT_CASE(memcomp_test_payload_writeback),
+ KUNIT_CASE(memcomp_test_load_pending),
+ KUNIT_CASE(memcomp_test_load_samefill),
+ KUNIT_CASE(memcomp_test_load_raw),
+ KUNIT_CASE(memcomp_test_load_compressed),
+ KUNIT_CASE(memcomp_test_load_writeback),
+#ifdef CONFIG_SWAP
+ KUNIT_CASE(memcomp_test_swapoff_raw),
+ KUNIT_CASE(memcomp_test_swapoff_packed),
+ KUNIT_CASE(memcomp_test_swapoff_samefill),
+ KUNIT_CASE(memcomp_test_swapoff_owner_race),
+ KUNIT_CASE(memcomp_test_swapoff_failures),
+ KUNIT_CASE(memcomp_test_swapoff_changed_backing),
+ KUNIT_CASE(memcomp_test_swapoff_invalid_commit),
+ KUNIT_CASE(memcomp_test_swapoff_raw_commit),
+ KUNIT_CASE(memcomp_test_swapoff_deferred_tail),
+ KUNIT_CASE(memcomp_test_swapoff_changed_length),
+ KUNIT_CASE(memcomp_test_swapoff_changed_offset),
+ KUNIT_CASE(memcomp_test_swapoff_changed_packed),
+ KUNIT_CASE(memcomp_test_swapoff_bad_member_flags),
+ KUNIT_CASE(memcomp_test_swapoff_bad_stable_flags),
+ KUNIT_CASE(memcomp_test_writeback_cache_raw),
+ KUNIT_CASE(memcomp_test_writeback_owner_packed),
+ KUNIT_CASE(memcomp_test_writeback_revert),
+ KUNIT_CASE(memcomp_test_writeback_mismatch),
+ KUNIT_CASE(memcomp_test_writeback_encoding),
+ KUNIT_CASE(memcomp_test_writeback_last_pte),
+#endif
+ KUNIT_CASE(memcomp_test_owner_stored_normalize),
+ KUNIT_CASE(memcomp_test_owner_samefill_normalize),
+ KUNIT_CASE(memcomp_test_owner_writeback_guard),
+ KUNIT_CASE(memcomp_test_owner_resident_final_guard),
+ KUNIT_CASE(memcomp_test_owner_last_pair),
+ KUNIT_CASE(memcomp_test_owner_stale),
+ KUNIT_CASE(memcomp_test_span_geometry),
+ KUNIT_CASE(memcomp_test_span_identity),
+ KUNIT_CASE(memcomp_test_span_anon_vma),
+ KUNIT_CASE(memcomp_test_cache_reserve_payloads),
+ KUNIT_CASE(memcomp_test_cache_reserve_boundaries),
+ KUNIT_CASE(memcomp_test_cache_reserve_invalid),
+ KUNIT_CASE(memcomp_test_cache_reserve_nomem),
+ KUNIT_CASE(memcomp_test_cache_reserve_race),
+ KUNIT_CASE(memcomp_test_cache_reserve_retire),
+ KUNIT_CASE(memcomp_test_cache_reserve_root),
+ KUNIT_CASE(memcomp_test_cache_store_samefill),
+ KUNIT_CASE(memcomp_test_cache_store_compressed),
+ KUNIT_CASE(memcomp_test_cache_publish_window),
+ KUNIT_CASE(memcomp_test_zero_no_storage),
+ KUNIT_CASE(memcomp_test_scan_empty_budget),
+ KUNIT_CASE(memcomp_test_scan_reserved_markers),
+ KUNIT_CASE(memcomp_test_scan_reserved_last),
+ KUNIT_CASE(memcomp_test_scan_first),
+ KUNIT_CASE(memcomp_test_scan_last_guard),
+ KUNIT_CASE(memcomp_test_scan_alignment),
+ KUNIT_CASE(memcomp_test_scan_refs_zero),
+ KUNIT_CASE(memcomp_test_scan_segment_retired),
+ KUNIT_CASE(memcomp_test_scan_private),
+ KUNIT_CASE(memcomp_test_scan_sparse),
+ KUNIT_CASE(memcomp_test_scan_u32_max),
+ KUNIT_CASE(memcomp_test_scan_w_relookup),
+ KUNIT_CASE(memcomp_test_query_present),
+ KUNIT_CASE(memcomp_test_query_absent),
+ KUNIT_CASE(memcomp_test_query_zero),
+ KUNIT_CASE(memcomp_test_query_stat_ptl),
+ KUNIT_CASE(memcomp_test_query_stat_busy_invalid),
+ KUNIT_CASE(memcomp_test_query_stat_compressed),
+ KUNIT_CASE(memcomp_test_query_pending_owner_drop),
+ KUNIT_CASE(memcomp_test_query_geometry),
+ KUNIT_CASE(memcomp_test_query_resident_busy),
+ KUNIT_CASE(memcomp_test_query_nonresident_guard),
+ KUNIT_CASE(memcomp_test_anon_store_samefill),
+ KUNIT_CASE(memcomp_test_anon_store_compressed),
+ KUNIT_CASE(memcomp_test_anon_store_mixed_raw),
+ KUNIT_CASE(memcomp_test_anon_store_reserve_failure),
+ KUNIT_CASE(memcomp_test_anon_store_charge_failure),
+ KUNIT_CASE(memcomp_test_anon_store_partial_commit),
+ KUNIT_CASE(memcomp_test_anon_store_invalid_refs),
+ KUNIT_CASE(memcomp_test_anon_store_precomp),
+#if defined(CONFIG_MEMCG) && defined(CONFIG_MEMCOMP_DEBUG_STATS)
+ KUNIT_CASE(memcomp_test_anon_store_cross_node_refresh),
+#endif
+ KUNIT_CASE(memcomp_test_wbb_raw),
+ KUNIT_CASE(memcomp_test_wbb_unpacked_compressed),
+ KUNIT_CASE(memcomp_test_wbb_packed),
+ KUNIT_CASE(memcomp_test_wbb_owner_detach),
+ KUNIT_CASE(memcomp_test_wbb_alloc_failure),
+ KUNIT_CASE(memcomp_test_wbb_dup_failure),
+ KUNIT_CASE(memcomp_test_wbb_dirty_io),
+ KUNIT_CASE(memcomp_test_wbb_settle_reject),
+ KUNIT_CASE(memcomp_test_swapoff_walk_empty),
+ KUNIT_CASE(memcomp_test_swapoff_walk_31),
+ KUNIT_CASE(memcomp_test_swapoff_walk_32),
+ KUNIT_CASE(memcomp_test_swapoff_walk_33),
+ KUNIT_CASE(memcomp_test_swapoff_walk_64),
+ KUNIT_CASE(memcomp_test_swapoff_walk_non_target),
+ KUNIT_CASE(memcomp_test_swapoff_walk_hole),
+ KUNIT_CASE(memcomp_test_swapoff_walk_retired),
+ KUNIT_CASE(memcomp_test_swapoff_walk_segment_hole),
+ KUNIT_CASE(memcomp_test_swapoff_walk_private),
+ KUNIT_CASE(memcomp_test_swapoff_walk_markers),
+ KUNIT_CASE(memcomp_test_swapoff_walk_reflush),
+ KUNIT_CASE(memcomp_test_swapoff_walk_u32_max),
+ KUNIT_CASE(memcomp_test_swapoff_walk_exits),
+ KUNIT_CASE(memcomp_test_defrag_roundtrip),
+ KUNIT_CASE(memcomp_test_defrag_noncandidates),
+ KUNIT_CASE(memcomp_test_defrag_buffer_failure),
+ KUNIT_CASE(memcomp_test_defrag_read_failure),
+ KUNIT_CASE(memcomp_test_defrag_alloc_failure),
+ KUNIT_CASE(memcomp_test_defrag_charge_failure),
+ KUNIT_CASE(memcomp_test_defrag_stale_word),
+ KUNIT_CASE(memcomp_test_defrag_stale_length),
+ KUNIT_CASE(memcomp_test_defrag_owner_normalize),
+ KUNIT_CASE(memcomp_test_defrag_scan_budget),
+ KUNIT_CASE(memcomp_test_defrag_scan_moves),
+ KUNIT_CASE(memcomp_test_defrag_scan_error),
+ KUNIT_CASE(memcomp_test_drain_cold_packed),
+ KUNIT_CASE(memcomp_test_drain_cold_raw),
+ KUNIT_CASE(memcomp_test_drain_hot_second_chance),
+ KUNIT_CASE(memcomp_test_drain_zero_limit),
+ KUNIT_CASE(memcomp_test_drain_empty_fifo),
+ KUNIT_CASE(memcomp_test_drain_ineligible_budget),
+ KUNIT_CASE(memcomp_test_drain_alloc_failure),
+ KUNIT_CASE(memcomp_test_drain_stage_alloc_failure),
+ KUNIT_CASE(memcomp_test_drain_dirty_io),
+ KUNIT_CASE(memcomp_test_drain_enospc),
+ KUNIT_CASE(memcomp_test_drain_stage_enospc),
+ KUNIT_CASE(memcomp_test_drain_zero_limit_flush),
+ KUNIT_CASE(memcomp_test_drain_partial_flush_failure),
+ KUNIT_CASE(memcomp_test_drain_first_error),
+#ifdef MC16_RESIDENT_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_resident_write),
+ KUNIT_CASE(memcomp_test_resident_readonly),
+ KUNIT_CASE(memcomp_test_resident_shared),
+ KUNIT_CASE(memcomp_test_resident_softdirty_wp),
+ KUNIT_CASE(memcomp_test_resident_uffd_wp),
+ KUNIT_CASE(memcomp_test_resident_changed_pte),
+ KUNIT_CASE(memcomp_test_resident_zapped),
+ KUNIT_CASE(memcomp_test_resident_unhandled),
+ KUNIT_CASE(memcomp_test_resident_state_race),
+ KUNIT_CASE(memcomp_test_resident_folio_race),
+ KUNIT_CASE(memcomp_test_resident_flag_race),
+ KUNIT_CASE(memcomp_test_resident_bad_index),
+ KUNIT_CASE(memcomp_test_resident_retry_nowait),
+ KUNIT_CASE(memcomp_test_resident_retry),
+#endif
+#ifdef MC16_FAULT_MM_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_fault_single),
+ KUNIT_CASE(memcomp_test_fault_raw),
+ KUNIT_CASE(memcomp_test_fault_attached),
+ KUNIT_CASE(memcomp_test_fault_shared),
+ KUNIT_CASE(memcomp_test_fault_softdirty),
+ KUNIT_CASE(memcomp_test_fault_softdirty_wp),
+ KUNIT_CASE(memcomp_test_fault_uffd_wp),
+ KUNIT_CASE(memcomp_test_fault_readonly),
+ KUNIT_CASE(memcomp_test_fault_changed),
+ KUNIT_CASE(memcomp_test_fault_alloc_oom),
+ KUNIT_CASE(memcomp_test_fault_charge_oom),
+ KUNIT_CASE(memcomp_test_fault_payload_retry),
+ KUNIT_CASE(memcomp_test_fault_payload_invalid),
+ KUNIT_CASE(memcomp_test_fault_thp),
+ KUNIT_CASE(memcomp_test_fault_thp_alloc_fallback),
+ KUNIT_CASE(memcomp_test_fault_thp_charge_fallback),
+ KUNIT_CASE(memcomp_test_fault_thp_pte_fallback),
+ KUNIT_CASE(memcomp_test_fault_thp_sibling_gone),
+ KUNIT_CASE(memcomp_test_fault_zero),
+#endif
+#ifdef MC16_ZERO_CONVERT_TEST_ENABLED
+ KUNIT_CASE(memcomp_test_zero_convert_flags),
+ KUNIT_CASE(memcomp_test_zero_convert_stale),
+ KUNIT_CASE(memcomp_test_zero_convert_rmap),
+ KUNIT_CASE(memcomp_test_zero_convert_move_miss),
+#endif
+ KUNIT_CASE(memcomp_test_consumer_pte_shared),
+ KUNIT_CASE(memcomp_test_consumer_pte_bounds),
+ KUNIT_CASE(memcomp_test_consumer_cache_abort),
+ KUNIT_CASE(memcomp_test_consumer_cache_samefill),
+ KUNIT_CASE(memcomp_test_consumer_cache_commit),
+ KUNIT_CASE(memcomp_test_consumer_cache_retry),
+ KUNIT_CASE(memcomp_test_consumer_zero_domains),
+ KUNIT_CASE(memcomp_test_consumer_load_bounds),
+ KUNIT_CASE(memcomp_test_cache_store_charge_failure),
+ KUNIT_CASE(memcomp_test_cache_store_reserve_failure),
+ KUNIT_CASE(memcomp_test_fifo_eligibility),
+ KUNIT_CASE(memcomp_test_fifo_scan_budget),
+ KUNIT_CASE(memcomp_test_writeback_last_guard),
+ KUNIT_CASE(memcomp_test_enospc_result),
+ KUNIT_CASE(memcomp_test_enospc_sync),
+ KUNIT_CASE(memcomp_test_enospc_precomp),
+ KUNIT_CASE(memcomp_test_enospc_async_events),
+ KUNIT_CASE(memcomp_test_objcg_source_identity),
+#endif
+ KUNIT_CASE(memcomp_test_store_cache_rejects_mapped),
+#ifdef CONFIG_SWAP
+ KUNIT_CASE(memcomp_test_writeback_pack_fits),
+ KUNIT_CASE(memcomp_test_writeback_decode_slot),
+ KUNIT_CASE(memcomp_test_writeback_defrag_wanted),
+ KUNIT_CASE(memcomp_test_sysfs_attrs_render),
+#endif
+ {}
+};
+
+static struct kunit_suite memcomp_test_suite = {
+ .name = "memcomp",
+ .init = memcomp_test_case_init,
+ .test_cases = memcomp_test_cases,
+};
+
+kunit_test_suite(memcomp_test_suite);
+
+#endif /* CONFIG_MEMCOMP_KUNIT_TEST */
+
+#endif /* _MM_MEMCOMP_TESTS_SUITE_H */
diff --git a/mm/memcomp/tests/swapoff_test.h b/mm/memcomp/tests/swapoff_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..b320dbdb3f3ff7d1d1ef4d51d03ae48e1560220a
--- /dev/null
+++ b/mm/memcomp/tests/swapoff_test.h
@@ -0,0 +1,558 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_SWAPOFF_TEST_H
+#define _MM_MEMCOMP_TESTS_SWAPOFF_TEST_H
+#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP)
+
+struct mc16_test_swap_restore {
+ struct mc16_test_refs *refs;
+ struct memcomp_ref guard;
+ struct mc16_swapoff_tail tail;
+ u8 *page;
+ u8 *original;
+ int read_error;
+ int alloc_error;
+ unsigned int reads;
+ unsigned int allocations;
+ unsigned int frees;
+ bool interrupted;
+ bool fail_charge;
+ unsigned int race_kind;
+ bool drop_owner;
+};
+
+static struct mc16_test_swap_restore *mc16_test_swap_current;
+
+static void mc16_test_swap_restore_cleanup(void *data)
+{
+ struct mc16_test_swap_restore *f = data;
+ bool deferred = !!f->tail.swap.val;
+ int puts = atomic_read(&memcomp_kunit_slot_puts);
+
+ f->refs->storage->owner->model.fail_at = 0;
+ mc16_test_swap_current = NULL;
+ /* A fatal test assertion may interrupt commit-before-finish. */
+ mc16_swapoff_finish(&f->tail);
+ if (deferred)
+ KUNIT_EXPECT_EQ(f->refs->storage->test,
+ atomic_read(&memcomp_kunit_slot_puts) - puts, 1);
+ if (f->guard.slot)
+ mc16_ref_put(&f->guard);
+}
+
+static int mc16_test_swap_read(swp_entry_t swap, void *dst)
+{
+ struct mc16_test_swap_restore *f = mc16_test_swap_current;
+
+ f->reads++;
+ if (f->read_error)
+ return f->read_error;
+ memcpy(dst, f->page, PAGE_SIZE);
+ return 0;
+}
+
+static int mc16_test_swap_allocate(const void *src, unsigned int *bytes,
+ unsigned long *handle, u8 *pool, void *data)
+{
+ struct mc16_test_swap_restore *f = data;
+ struct mc16_test_storage *s = f->refs->storage;
+ struct mc16_view view;
+ int ret;
+
+ f->allocations++;
+ if (f->alloc_error)
+ return f->alloc_error;
+ *pool = 0;
+ ret = memcomp_prepare_payload(src, PAGE_SIZE, bytes, handle, pool,
+ MEMCOMP_PAYLOAD_MUST_SUCCEED, false);
+ if (ret)
+ return ret;
+ if (f->fail_charge)
+ s->owner->model.fail_at = atomic_read(&s->owner->model.charge_calls) + 1;
+ if (f->race_kind) {
+ struct mc16_swap_fields old;
+ u64 word;
+ u16 length;
+
+ mutex_lock(memcomp_lock_of(f->guard.token));
+ mc16_view_locked(&f->guard, &view);
+ mc16_swap_decode(*view.word, *view.length, &old);
+ if (f->race_kind == 1) {
+ memcomp_slot_put(swp_entry(old.type, old.swap_offset));
+ old.swap_offset++;
+ } else if (f->race_kind == 2) {
+ old.length++;
+ memcomp_compressed_bytes_add(1);
+ } else if (f->race_kind == 3) {
+ old.byte_offset++;
+ } else {
+ old.packed = false;
+ }
+ mc16_swap_encode(old.type, old.swap_offset, old.byte_offset,
+ old.length, old.packed, &word, &length);
+ *view.word = word;
+ *view.length = length;
+ mutex_unlock(memcomp_lock_of(f->guard.token));
+ }
+ if (f->drop_owner) {
+ struct memcomp_ref lookup;
+
+ if (!mc16_get(f->guard.token, &lookup))
+ return -EIO;
+ if (!mc16_owner_drop_folio_held(&lookup, s->owner->folio, 1, 0))
+ return -EIO;
+ /* The callback consumed the initial OWNER, not the restore guard. */
+ s->refs[0] = (struct memcomp_ref) { };
+ }
+ return 0;
+}
+
+static void mc16_test_swap_free(u8 pool, unsigned long handle, void *data)
+{
+ struct mc16_test_swap_restore *f = data;
+
+ f->frees++;
+ memcomp_zs_free(pool, handle);
+}
+
+static bool mc16_test_swap_interrupted(void *data)
+{
+ return ((struct mc16_test_swap_restore *)data)->interrupted;
+}
+
+static struct memcomp_swapoff_ops
+mc16_test_swap_ops(struct mc16_test_swap_restore *f)
+{
+ return (struct memcomp_swapoff_ops) {
+ .read_page = mc16_test_swap_read,
+ .alloc_payload = mc16_test_swap_allocate,
+ .free_payload = mc16_test_swap_free,
+ .interrupted = mc16_test_swap_interrupted, .ctx = f,
+ };
+}
+
+static struct mc16_test_swap_restore *
+mc16_test_swap_restore_begin(struct kunit *test, bool packed, bool samefill)
+{
+ struct mc16_test_swap_restore *f;
+ struct mc16_test_storage *s;
+ struct memcomp_ref *ref;
+ struct mc16_view view;
+ unsigned long handle = 0;
+ unsigned int bytes = PAGE_SIZE, i;
+ u64 word;
+ u16 length;
+ u8 pool = 0;
+ void *mapped;
+ int ret;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->refs = mc16_test_refs_begin(test);
+ if (!f->refs || mc16_test_refs_reserve(f->refs, 1))
+ return NULL;
+ if (kunit_add_action_or_reset(test, mc16_test_swap_restore_cleanup, f))
+ return NULL;
+ s = f->refs->storage;
+ ref = &s->refs[0];
+ f->page = kunit_kzalloc(test, PAGE_SIZE, GFP_KERNEL);
+ f->original = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ if (!f->page || !f->original)
+ return NULL;
+ for (i = 0; i < PAGE_SIZE; i++)
+ f->original[i] = samefill ? 0xab : i % 53;
+ if (packed) {
+ ret = memcomp_prepare_payload(f->original, PAGE_SIZE, &bytes,
+ &handle, &pool,
+ MEMCOMP_PAYLOAD_MUST_SUCCEED, false);
+ if (ret)
+ return NULL;
+ if (!bytes || bytes >= PAGE_SIZE - 37) {
+ if (bytes)
+ memcomp_zs_free(pool, handle);
+ return NULL;
+ }
+ mapped = zs_map_object(memcomp_pools[0], handle, ZS_MM_RO);
+ memcpy(f->page + 37, mapped, bytes);
+ zs_unmap_object(memcomp_pools[0], handle);
+ memcomp_zs_free(pool, handle);
+ } else {
+ memcpy(f->page, f->original, PAGE_SIZE);
+ }
+ if (!mc16_swap_encode(0, 0x567, packed ? 37 : 0, bytes,
+ packed, &word, &length))
+ return NULL;
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ mc16_account_state(ref, false);
+ folio_put(view.member->owner_folio);
+ *view.word = word;
+ *view.length = length;
+ *view.flags = MC16_SWAPPED | MC16_F_OWNER |
+ MC16_F_REFERENCED | MC16_F_INCOMPRESSIBLE;
+ memcomp_debug_dec(&memcomp_pending_pages);
+ memcomp_debug_long_inc(&memcomp_swapped_entries);
+ memcomp_compressed_bytes_add(bytes);
+ mc16_account_state(ref, true);
+ /* A pure pool restore must not consume the user shadow. */
+ *mc16_shadow_ptr(ref) = (unsigned long)xa_mk_value(123);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ if (!mc16_get(ref->token, &f->guard))
+ return NULL;
+ mc16_test_swap_current = f;
+ return f;
+}
+
+static void mc16_test_swap_restore_roundtrip(struct kunit *test, bool packed,
+ bool samefill, bool drop_owner)
+{
+ struct mc16_test_swap_restore *f;
+ struct memcomp_swapoff_ops ops;
+ struct mc16_view view;
+ struct mc16_key *key;
+ u32 token;
+ unsigned short owner_id;
+ u64 shadow;
+ u8 *dst;
+ int puts, dups;
+
+ f = mc16_test_swap_restore_begin(test, packed, samefill);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ ops = mc16_test_swap_ops(f);
+ f->drop_owner = drop_owner;
+ puts = atomic_read(&memcomp_kunit_slot_puts);
+ dups = atomic_read(&memcomp_kunit_slot_dups);
+ key = f->guard.cluster->key;
+ owner_id = key->owner_id;
+ token = f->guard.token;
+ mutex_lock(memcomp_lock_of(token));
+ shadow = *mc16_shadow_ptr(&f->guard);
+ mutex_unlock(memcomp_lock_of(token));
+ KUNIT_ASSERT_EQ(test, mc16_swapoff_entry(&f->guard, 0, &ops), 1);
+ mutex_lock(memcomp_lock_of(token));
+ KUNIT_EXPECT_EQ(test, *mc16_shadow_ptr(&f->guard), shadow);
+ KUNIT_EXPECT_EQ(test, f->guard.token, token);
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.cluster->key, key);
+ KUNIT_EXPECT_EQ(test, f->guard.cluster->key->owner_id, owner_id);
+ mutex_unlock(memcomp_lock_of(token));
+ KUNIT_EXPECT_EQ(test, f->reads, 1U);
+ KUNIT_EXPECT_EQ(test, f->allocations, 1U);
+ KUNIT_EXPECT_EQ(test, f->frees, 0U);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_puts) - puts, 2);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_dups) - dups, 1);
+ dst = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, dst);
+ KUNIT_ASSERT_EQ(test, mc16_load_page(&f->guard, dst), 0);
+ KUNIT_EXPECT_EQ(test, memcmp(dst, f->original, PAGE_SIZE), 0);
+ mutex_lock(memcomp_lock_of(f->guard.token));
+ if (mc16_view_locked(&f->guard, &view)) {
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(*view.flags),
+ (unsigned int)(samefill ? MC16_SAMEFILL : MC16_STORED));
+ KUNIT_EXPECT_EQ(test, !!view.record, !drop_owner);
+ KUNIT_EXPECT_EQ(test, *view.flags & (MC16_F_REFERENCED |
+ MC16_F_INCOMPRESSIBLE),
+ (u16)(MC16_F_REFERENCED | MC16_F_INCOMPRESSIBLE));
+ }
+ mutex_unlock(memcomp_lock_of(f->guard.token));
+}
+
+static void memcomp_test_swapoff_raw(struct kunit *test)
+{
+ mc16_test_swap_restore_roundtrip(test, false, false, false);
+}
+
+static void memcomp_test_swapoff_packed(struct kunit *test)
+{
+ mc16_test_swap_restore_roundtrip(test, true, false, false);
+}
+
+static void memcomp_test_swapoff_samefill(struct kunit *test)
+{
+ mc16_test_swap_restore_roundtrip(test, false, true, false);
+}
+
+static void memcomp_test_swapoff_owner_race(struct kunit *test)
+{
+ mc16_test_swap_restore_roundtrip(test, true, false, true);
+}
+
+static void memcomp_test_swapoff_failures(struct kunit *test)
+{
+ struct mc16_test_swap_restore *f = mc16_test_swap_restore_begin(test, false, false);
+ struct memcomp_swapoff_ops ops;
+ struct mc16_view view;
+ u64 word;
+ u16 length;
+ int puts, dups;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ ops = mc16_test_swap_ops(f);
+ puts = atomic_read(&memcomp_kunit_slot_puts);
+ dups = atomic_read(&memcomp_kunit_slot_dups);
+ f->interrupted = true;
+ KUNIT_EXPECT_EQ(test, mc16_swapoff_entry(&f->guard, 0, &ops), -EINTR);
+ f->interrupted = false;
+ f->read_error = -EIO;
+ KUNIT_EXPECT_EQ(test, mc16_swapoff_entry(&f->guard, 0, &ops), -EIO);
+ f->read_error = 0;
+ f->alloc_error = -ENOMEM;
+ KUNIT_EXPECT_EQ(test, mc16_swapoff_entry(&f->guard, 0, &ops), -ENOMEM);
+ f->alloc_error = 0;
+ KUNIT_EXPECT_EQ(test, mc16_swapoff_entry(&f->guard, 1, &ops), 0);
+ f->fail_charge = true;
+ KUNIT_EXPECT_EQ(test, mc16_swapoff_entry(&f->guard, 0, &ops), -ENOMEM);
+ f->fail_charge = false;
+ f->refs->storage->owner->model.fail_at = 0;
+ KUNIT_EXPECT_EQ(test, f->frees, 1U);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_dups) - dups, 5);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_puts) - puts, 5);
+ mutex_lock(memcomp_lock_of(f->guard.token));
+ mc16_view_locked(&f->guard, &view);
+ word = *view.word;
+ length = *view.length;
+ mutex_unlock(memcomp_lock_of(f->guard.token));
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(f->guard.slot->flags),
+ (unsigned int)MC16_PENDING);
+ KUNIT_EXPECT_TRUE(test, mc16_swap_decode(word, length,
+ &(struct mc16_swap_fields) { }));
+ KUNIT_EXPECT_EQ(test, f->guard.cluster->eligible_slots, (u16)0);
+}
+
+static void mc16_test_swapoff_backing_race(struct kunit *test, unsigned int kind)
+{
+ struct mc16_test_swap_restore *f = mc16_test_swap_restore_begin(test, true, false);
+ struct memcomp_swapoff_ops ops;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ ops = mc16_test_swap_ops(f);
+ f->race_kind = kind;
+ if (kind == 4) {
+ struct mc16_view view;
+ struct mc16_swap_fields fields;
+
+ mutex_lock(memcomp_lock_of(f->guard.token));
+ mc16_view_locked(&f->guard, &view);
+ mc16_swap_decode(*view.word, *view.length, &fields);
+ memmove(f->page, f->page + fields.byte_offset, fields.length);
+ mc16_swap_encode(fields.type, fields.swap_offset, 0, fields.length,
+ true, view.word, view.length);
+ mutex_unlock(memcomp_lock_of(f->guard.token));
+ }
+ KUNIT_EXPECT_EQ(test, mc16_swapoff_entry(&f->guard, 0, &ops), 0);
+ KUNIT_EXPECT_EQ(test, f->frees, 1U);
+ KUNIT_EXPECT_EQ(test, f->guard.cluster->eligible_slots, (u16)0);
+ KUNIT_EXPECT_EQ(test, f->refs->storage->owner->model.fail_at, 0);
+}
+
+static void memcomp_test_swapoff_changed_backing(struct kunit *test)
+{
+ mc16_test_swapoff_backing_race(test, 1);
+}
+
+static void memcomp_test_swapoff_changed_length(struct kunit *test)
+{
+ mc16_test_swapoff_backing_race(test, 2);
+}
+
+static void memcomp_test_swapoff_changed_offset(struct kunit *test)
+{
+ mc16_test_swapoff_backing_race(test, 3);
+}
+
+static void memcomp_test_swapoff_changed_packed(struct kunit *test)
+{
+ mc16_test_swapoff_backing_race(test, 4);
+}
+
+static void memcomp_test_swapoff_invalid_commit(struct kunit *test)
+{
+ struct mc16_test_swap_restore *f = mc16_test_swap_restore_begin(test, false, false);
+ struct mc16_swapoff_tail tail = { .swap = { .val = 123 } };
+ struct mc16_payload payload = { .word = 1, .bytes = PAGE_SIZE + 1 };
+ struct mc16_view view;
+ u64 word;
+ u16 length;
+ int charges;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mutex_lock(memcomp_lock_of(f->guard.token));
+ mc16_view_locked(&f->guard, &view);
+ word = *view.word;
+ length = *view.length;
+ mutex_unlock(memcomp_lock_of(f->guard.token));
+ charges = atomic_read(&f->refs->storage->owner->model.charge_calls);
+ KUNIT_EXPECT_EQ(test, mc16_swapoff_commit(&f->guard, word, length,
+ &payload, GFP_KERNEL, &tail), -EINVAL);
+ payload.bytes = 1;
+ payload.word = 0;
+ KUNIT_EXPECT_EQ(test, mc16_swapoff_commit(&f->guard, word, length,
+ &payload, GFP_KERNEL, &tail), -EINVAL);
+ payload.word = 1;
+ payload.charge = 1;
+ KUNIT_EXPECT_EQ(test, mc16_swapoff_commit(&f->guard, word, length,
+ &payload, GFP_KERNEL, &tail), -EINVAL);
+ payload.charge = 0;
+ KUNIT_EXPECT_EQ(test, mc16_swapoff_commit(&f->guard, 0, length,
+ &payload, GFP_KERNEL, &tail), -EINVAL);
+ KUNIT_EXPECT_EQ(test, mc16_swapoff_commit(&f->guard, word, U16_MAX,
+ &payload, GFP_KERNEL, &tail), -EINVAL);
+ KUNIT_EXPECT_EQ(test, tail.swap.val, 123UL);
+ KUNIT_EXPECT_EQ(test, atomic_read(&f->refs->storage->owner->model.charge_calls),
+ charges);
+}
+
+static void memcomp_test_swapoff_raw_commit(struct kunit *test)
+{
+ struct mc16_test_swap_restore *f = mc16_test_swap_restore_begin(test, true, false);
+ struct mc16_swapoff_tail *tail;
+ struct mc16_payload payload = { };
+ struct mc16_view view;
+ unsigned long handle;
+ unsigned int bytes;
+ u64 word;
+ u16 length;
+ int ret, puts;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ tail = &f->tail;
+ KUNIT_ASSERT_EQ(test, memcomp_alloc_raw_payload(0, f->original, PAGE_SIZE,
+ &bytes, &handle, false, NULL), 0);
+ payload = (struct mc16_payload) { .word = handle, .bytes = bytes };
+ mutex_lock(memcomp_lock_of(f->guard.token));
+ mc16_view_locked(&f->guard, &view);
+ word = *view.word;
+ length = *view.length;
+ mutex_unlock(memcomp_lock_of(f->guard.token));
+ ret = mc16_swapoff_commit(&f->guard, word, length, &payload,
+ GFP_NOIO | __GFP_NOWARN, tail);
+ if (ret != 1) {
+ memcomp_zs_free(0, handle);
+ KUNIT_FAIL(test, "raw commit failed: %d", ret);
+ return;
+ }
+ KUNIT_EXPECT_EQ(test, payload.word, 0ULL);
+ {
+ struct memcomp_ref lookup;
+ bool dropped;
+
+ KUNIT_ASSERT_TRUE(test, mc16_get(f->guard.token, &lookup));
+ dropped = mc16_owner_drop_folio_held(&lookup,
+ f->refs->storage->owner->folio, 1, 0);
+ KUNIT_ASSERT_TRUE(test, dropped);
+ f->refs->storage->refs[0] = (struct memcomp_ref) { };
+ }
+ puts = atomic_read(&memcomp_kunit_slot_puts);
+ mc16_swapoff_finish(tail);
+ mc16_swapoff_finish(tail);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_puts) - puts, 1);
+ KUNIT_EXPECT_EQ(test, tail->swap.val, 0UL);
+ KUNIT_EXPECT_EQ(test, f->guard.cluster->eligible_slots, (u16)1);
+ KUNIT_ASSERT_EQ(test, mc16_load_page(&f->guard, f->page), 0);
+ KUNIT_EXPECT_EQ(test, memcmp(f->page, f->original, PAGE_SIZE), 0);
+}
+
+static void mc16_test_swapoff_bad_flags(struct kunit *test, bool stable)
+{
+ struct mc16_test_swap_restore *f = mc16_test_swap_restore_begin(test, false, false);
+ struct mc16_swapoff_tail tail = { };
+ struct mc16_payload payload;
+ struct memcomp_swapoff_ops ops;
+ struct mc16_view view;
+ unsigned long handle;
+ unsigned int bytes, i;
+ u64 word;
+ u16 length, saved;
+ long charged;
+ int ret, dups, puts;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ ops = mc16_test_swap_ops(f);
+ if (stable) {
+ struct memcomp_ref lookup;
+ bool dropped;
+
+ KUNIT_ASSERT_TRUE(test, mc16_get(f->guard.token, &lookup));
+ dropped = mc16_owner_drop_folio_held(&lookup,
+ f->refs->storage->owner->folio, 1, 0);
+ KUNIT_ASSERT_TRUE(test, dropped);
+ f->refs->storage->refs[0] = (struct memcomp_ref) { };
+ }
+ KUNIT_ASSERT_EQ(test, memcomp_alloc_raw_payload(0, f->original, PAGE_SIZE,
+ &bytes, &handle, false, NULL), 0);
+ payload = (struct mc16_payload) { .word = handle, .bytes = bytes };
+ charged = atomic_long_read(&f->refs->storage->owner->model.charged);
+ dups = atomic_read(&memcomp_kunit_slot_dups);
+ puts = atomic_read(&memcomp_kunit_slot_puts);
+ for (i = 0; i < 2; i++) {
+ mutex_lock(memcomp_lock_of(f->guard.token));
+ mc16_view_locked(&f->guard, &view);
+ word = *view.word;
+ length = *view.length;
+ saved = *view.flags;
+ *view.flags = i ? (saved & ~MC16_ORDER_MASK) | MC16_ORDER_MASK :
+ saved | BIT(15);
+ mutex_unlock(memcomp_lock_of(f->guard.token));
+ KUNIT_EXPECT_EQ(test, mc16_swapoff_entry(&f->guard, 0, &ops), -EIO);
+ ret = mc16_swapoff_commit(&f->guard, word, length, &payload,
+ GFP_KERNEL, &tail);
+ KUNIT_EXPECT_EQ(test, ret, -EIO);
+ KUNIT_EXPECT_EQ(test, payload.word, (u64)handle);
+ KUNIT_EXPECT_EQ(test, tail.swap.val, 0UL);
+ KUNIT_EXPECT_EQ(test,
+ atomic_long_read(&f->refs->storage->owner->model.charged), charged);
+ mutex_lock(memcomp_lock_of(f->guard.token));
+ mc16_view_locked(&f->guard, &view);
+ *view.flags = saved;
+ mutex_unlock(memcomp_lock_of(f->guard.token));
+ }
+ memcomp_zs_free(0, handle);
+ KUNIT_EXPECT_EQ(test, f->reads, 0U);
+ KUNIT_EXPECT_EQ(test, f->allocations, 0U);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_dups), dups);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_puts), puts);
+ KUNIT_EXPECT_EQ(test, f->guard.cluster->eligible_slots, (u16)0);
+}
+
+static void memcomp_test_swapoff_bad_member_flags(struct kunit *test)
+{
+ mc16_test_swapoff_bad_flags(test, false);
+}
+
+static void memcomp_test_swapoff_bad_stable_flags(struct kunit *test)
+{
+ mc16_test_swapoff_bad_flags(test, true);
+}
+
+/* Return without finishing: cleanup must settle tail before dropping guards. */
+static void memcomp_test_swapoff_deferred_tail(struct kunit *test)
+{
+ struct mc16_test_swap_restore *f = mc16_test_swap_restore_begin(test, true, false);
+ struct mc16_payload payload;
+ struct mc16_view view;
+ unsigned long handle;
+ unsigned int bytes;
+ u64 word;
+ u16 length;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, memcomp_alloc_raw_payload(0, f->original, PAGE_SIZE,
+ &bytes, &handle, false, NULL), 0);
+ payload = (struct mc16_payload) { .word = handle, .bytes = bytes };
+ mutex_lock(memcomp_lock_of(f->guard.token));
+ mc16_view_locked(&f->guard, &view);
+ word = *view.word;
+ length = *view.length;
+ mutex_unlock(memcomp_lock_of(f->guard.token));
+ ret = mc16_swapoff_commit(&f->guard, word, length, &payload,
+ GFP_KERNEL, &f->tail);
+ if (ret != 1) {
+ memcomp_zs_free(0, handle);
+ KUNIT_FAIL(test, "deferred commit failed: %d", ret);
+ return;
+ }
+ KUNIT_EXPECT_NE(test, f->tail.swap.val, 0UL);
+ /* Exactly the same cleanup path as a fatal assertion after commit. */
+}
+#endif /* CONFIG_MEMCG && CONFIG_SWAP */
+#endif /* _MM_MEMCOMP_TESTS_SWAPOFF_TEST_H */
diff --git a/mm/memcomp/tests/swapoff_walk_test.h b/mm/memcomp/tests/swapoff_walk_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..d306616d607105e420b881db75cdfa37873eac5d
--- /dev/null
+++ b/mm/memcomp/tests/swapoff_walk_test.h
@@ -0,0 +1,562 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_SWAPOFF_WALK_TEST_H
+#define _MM_MEMCOMP_TESTS_SWAPOFF_WALK_TEST_H
+
+/*
+ * Include after storage_test.h, refs_test.h and cache_store.h. The production
+ * walker includes scan.h; no scan_test.h or legacy swapoff hooks are needed.
+ * Register the cases in the main suite under CONFIG_MEMCG && CONFIG_SWAP.
+ */
+#include "../swapoff_walk.h"
+
+#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP)
+#define MC16_WALK_TEST_FLAGS (PF_MEMALLOC_NOFS | PF_MEMALLOC)
+
+enum mc16_test_walk_stop {
+ MC16_WALK_STOP_NONE,
+ MC16_WALK_STOP_BEFORE_SCAN,
+ MC16_WALK_STOP_AFTER_SCAN,
+ MC16_WALK_STOP_AFTER_RESTORE,
+ MC16_WALK_STOP_UNLOCKED,
+ MC16_WALK_STOP_EOF,
+};
+
+struct mc16_test_walk {
+ struct mc16_test_refs *refs;
+ struct mc16_swapoff_walk_ops ops;
+ struct memcomp_ref private;
+ u32 markers[MC16_SWAPOFF_SCAN_BATCH];
+ unsigned int flushes, restores, yields, probes, slice, scans;
+ unsigned int fail_flush, fail_restore, settled;
+ u64 last_cursor, last_token;
+ enum mc16_test_walk_stop stop;
+ int actor_result, writer_error;
+ bool interrupted, bad_scope, bad_ref, bad_budget, dirty_scan;
+ bool private_active, staging, replace, retained, appended;
+};
+
+static void mc16_test_walk_cleanup(void *data)
+{
+ struct mc16_test_walk *f = data;
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(f->markers); i++)
+ if (f->markers[i])
+ xa_erase_irq(&mc16_clusters, f->markers[i]);
+ if (f->private.slot) {
+ if (f->private_active) {
+ mc16_ref_put(&f->private);
+ } else {
+ mc16_release_resources(&f->private);
+ mc16_retire_slot(&f->private);
+ }
+ }
+}
+
+static void mc16_test_walk_scope(struct mc16_test_walk *f)
+{
+ if (!mutex_is_locked(f->ops.lock) ||
+ (current->flags & MC16_WALK_TEST_FLAGS) != MC16_WALK_TEST_FLAGS)
+ f->bad_scope = true;
+}
+
+/* Fixture owner refs are independent of the one borrowed by the walker. */
+static void mc16_test_walk_refs(struct mc16_test_walk *f,
+ const struct memcomp_ref *borrowed)
+{
+ struct mc16_test_storage *s = f->refs->storage;
+ unsigned int i;
+
+ for (i = 0; i < s->nr; i++) {
+ struct memcomp_ref *owner = &s->refs[i];
+ unsigned int expected = 1;
+
+ if (!owner->slot)
+ continue;
+ if (borrowed && borrowed->slot == owner->slot)
+ expected++;
+ if (refcount_read(&owner->slot->refs) != expected)
+ f->bad_ref = true;
+ }
+ if (f->private.slot) {
+ unsigned int expected = f->private_active ? 1 : 0;
+
+ if (borrowed && borrowed->slot == f->private.slot)
+ expected++;
+ if (refcount_read(&f->private.slot->refs) != expected)
+ f->bad_ref = true;
+ }
+}
+
+static int mc16_test_walk_private(struct mc16_test_walk *f)
+{
+ const struct mc16_initial_payload initial = { .word = 0x5a5a };
+ struct mc16_test_storage *s = f->refs->storage;
+
+ return mc16_reserve_cache(&s->owner->model.objcg,
+ folio_nid(s->owner->folio), &initial, 1, &f->private);
+}
+
+static int mc16_test_walk_flush(void *data)
+{
+ struct mc16_test_walk *f = data;
+
+ mc16_test_walk_scope(f);
+ mc16_test_walk_refs(f, NULL);
+ f->flushes++;
+ if (f->flushes == f->fail_flush)
+ return -EIO;
+ /* Real private refs0 storage becomes scannable only at this flush. */
+ if (f->staging) {
+ mc16_cache_publish(&f->private, 0);
+ f->private_active = true;
+ f->staging = false;
+ f->settled++;
+ }
+ return 0;
+}
+
+static int mc16_test_walk_restore(struct memcomp_ref *ref,
+ unsigned int type, void *data)
+{
+ struct mc16_test_walk *f = data;
+
+ mc16_test_walk_scope(f);
+ mc16_test_walk_refs(f, ref);
+ f->restores++;
+ if (type != 7 || !ref->slot || !ref->cluster ||
+ (u64)ref->token <= f->last_token)
+ f->bad_ref = true;
+ f->last_token = ref->token;
+ if (f->staging)
+ f->dirty_scan = true;
+ if (f->stop == MC16_WALK_STOP_AFTER_RESTORE)
+ f->interrupted = true;
+ if (f->restores == f->fail_restore)
+ return -ENOMEM;
+ /* Zero models the actor rejecting a non-target, positive a restore. */
+ return f->actor_result;
+}
+
+static bool mc16_test_walk_interrupted(void *data)
+{
+ struct mc16_test_walk *f = data;
+
+ mc16_test_walk_scope(f);
+ return f->interrupted;
+}
+
+static void mc16_test_walk_scanned(u64 cursor, unsigned int used,
+ enum mc16_scan_result result,
+ const struct memcomp_ref *ref, void *data)
+{
+ struct mc16_test_walk *f = data;
+
+ mc16_test_walk_scope(f);
+ mc16_test_walk_refs(f, ref);
+ f->scans++;
+ f->probes += used;
+ f->slice += used;
+ if (f->slice > MC16_SWAPOFF_SCAN_BATCH || cursor < f->last_cursor ||
+ (result == MC16_SCAN_GUARD && (!ref->slot || !used)) ||
+ (result != MC16_SCAN_GUARD && (ref->slot || ref->cluster)))
+ f->bad_budget = true;
+ f->last_cursor = cursor;
+ if (f->staging)
+ f->dirty_scan = true;
+ if ((used && f->stop == MC16_WALK_STOP_AFTER_SCAN) ||
+ (result == MC16_SCAN_END && !used && f->stop == MC16_WALK_STOP_EOF))
+ f->interrupted = true;
+}
+
+static void mc16_test_walk_unlocked(void *data)
+{
+ struct mc16_test_walk *f = data;
+ struct mc16_test_storage *s = f->refs->storage;
+ unsigned int i, nr = s->nr;
+ u32 base = nr ? s->refs[0].token : 0;
+
+ f->yields++;
+ if (f->slice != MC16_SWAPOFF_SCAN_BATCH)
+ f->bad_budget = true;
+ f->slice = 0;
+ if ((current->flags & MC16_WALK_TEST_FLAGS) != MC16_WALK_TEST_FLAGS)
+ f->bad_scope = true;
+ /* Enter the real W as a writer at the actual unlock boundary. */
+ if (!mutex_trylock(f->ops.lock)) {
+ f->bad_scope = true;
+ return;
+ }
+ mc16_test_walk_refs(f, NULL);
+ if (f->replace && f->yields == 1) {
+ /* Retire the old slots; append without pressure or token reuse. */
+ for (i = 0; i < nr; i++)
+ mc16_ref_put(&s->refs[i]);
+ f->retained = !xa_empty(&mc16_clusters);
+ for (i = 0; i < nr; i++) {
+ f->writer_error = mc16_test_refs_reserve(f->refs, 1);
+ if (f->writer_error)
+ goto out_unlock;
+ }
+ f->appended = s->refs[nr].token == base + nr;
+ f->writer_error = mc16_test_walk_private(f);
+ if (!f->writer_error)
+ f->staging = true;
+ }
+out_unlock:
+ mutex_unlock(f->ops.lock);
+ if (f->stop == MC16_WALK_STOP_UNLOCKED)
+ f->interrupted = true;
+}
+
+static struct mc16_test_walk *mc16_test_walk_begin(struct kunit *test)
+{
+ struct mc16_test_walk *f;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->refs = mc16_test_refs_begin(test);
+ if (!f->refs || kunit_add_action_or_reset(test,
+ mc16_test_walk_cleanup, f))
+ return NULL;
+ f->actor_result = 1;
+ f->ops = (struct mc16_swapoff_walk_ops) {
+ .lock = &memcomp_writeback_lock,
+ .flush = mc16_test_walk_flush,
+ .restore = mc16_test_walk_restore,
+ .interrupted = mc16_test_walk_interrupted,
+ .scanned = mc16_test_walk_scanned,
+ .unlocked = mc16_test_walk_unlocked,
+ .ctx = f,
+ };
+ return f;
+}
+
+static int mc16_test_walk_reserve(struct mc16_test_walk *f, unsigned int nr)
+{
+ unsigned int i;
+ int ret;
+
+ for (i = 0; i < nr; i++) {
+ ret = mc16_test_refs_reserve(f->refs, 1);
+ if (ret)
+ return ret;
+ }
+ return 0;
+}
+
+/* No fatal assertions while W or a modified task allocation scope is held. */
+static int mc16_test_walk_run(struct kunit *test, struct mc16_test_walk *f,
+ unsigned int nested)
+{
+ unsigned int original = current->flags & MC16_WALK_TEST_FLAGS;
+ unsigned int nofs = 0, noreclaim = 0, before, after;
+ bool unlocked;
+ int ret;
+
+ if (nested & PF_MEMALLOC_NOFS)
+ nofs = memalloc_nofs_save();
+ if (nested & PF_MEMALLOC)
+ noreclaim = memalloc_noreclaim_save();
+ before = current->flags & MC16_WALK_TEST_FLAGS;
+ ret = mc16_swapoff_walk(7, &f->ops);
+ after = current->flags & MC16_WALK_TEST_FLAGS;
+ if (nested & PF_MEMALLOC)
+ memalloc_noreclaim_restore(noreclaim);
+ if (nested & PF_MEMALLOC_NOFS)
+ memalloc_nofs_restore(nofs);
+ unlocked = mutex_trylock(f->ops.lock);
+ if (unlocked) {
+ mc16_test_walk_refs(f, NULL);
+ mutex_unlock(f->ops.lock);
+ }
+ KUNIT_EXPECT_EQ(test, after, before);
+ KUNIT_EXPECT_EQ(test, current->flags & MC16_WALK_TEST_FLAGS, original);
+ KUNIT_EXPECT_TRUE(test, unlocked);
+ KUNIT_EXPECT_FALSE(test, f->bad_scope);
+ KUNIT_EXPECT_FALSE(test, f->bad_ref);
+ KUNIT_EXPECT_FALSE(test, f->bad_budget);
+ KUNIT_EXPECT_FALSE(test, f->dirty_scan);
+ return ret;
+}
+
+static void mc16_test_walk_boundary(struct kunit *test, unsigned int nr,
+ unsigned int yields)
+{
+ struct mc16_test_walk *f = mc16_test_walk_begin(test);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_walk_reserve(f, nr), 0);
+ KUNIT_EXPECT_EQ(test, mc16_test_walk_run(test, f, 0), 0);
+ KUNIT_EXPECT_EQ(test, f->probes, nr);
+ KUNIT_EXPECT_EQ(test, f->restores, nr);
+ KUNIT_EXPECT_EQ(test, f->yields, yields);
+ KUNIT_EXPECT_EQ(test, f->flushes, yields + 1);
+ KUNIT_EXPECT_EQ(test, f->last_cursor, MC16_SCAN_END_CURSOR);
+}
+
+static void memcomp_test_swapoff_walk_empty(struct kunit *test)
+{
+ mc16_test_walk_boundary(test, 0, 0);
+}
+
+static void memcomp_test_swapoff_walk_31(struct kunit *test)
+{
+ mc16_test_walk_boundary(test, 31, 0);
+}
+
+static void memcomp_test_swapoff_walk_32(struct kunit *test)
+{
+ mc16_test_walk_boundary(test, 32, 1);
+}
+
+static void memcomp_test_swapoff_walk_33(struct kunit *test)
+{
+ mc16_test_walk_boundary(test, 33, 1);
+}
+
+static void memcomp_test_swapoff_walk_64(struct kunit *test)
+{
+ mc16_test_walk_boundary(test, 64, 2);
+}
+
+static void memcomp_test_swapoff_walk_non_target(struct kunit *test)
+{
+ struct mc16_test_walk *f = mc16_test_walk_begin(test);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_walk_reserve(f, 65), 0);
+ f->actor_result = 0;
+ KUNIT_EXPECT_EQ(test, mc16_test_walk_run(test, f, 0), 0);
+ KUNIT_EXPECT_EQ(test, f->probes, 65U);
+ KUNIT_EXPECT_EQ(test, f->restores, 65U);
+ KUNIT_EXPECT_EQ(test, f->yields, 2U);
+ KUNIT_EXPECT_EQ(test, f->flushes, 3U);
+}
+
+static void memcomp_test_swapoff_walk_hole(struct kunit *test)
+{
+ struct mc16_test_walk *f = mc16_test_walk_begin(test);
+ struct mc16_test_storage *s;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f->refs, 1), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f->refs, 2), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_walk_reserve(f, 29), 0);
+ KUNIT_ASSERT_EQ(test, s->refs[1].token, s->refs[0].token + 2);
+ KUNIT_EXPECT_EQ(test, mc16_test_walk_run(test, f, 0), 0);
+ KUNIT_EXPECT_EQ(test, f->probes, 33U);
+ KUNIT_EXPECT_EQ(test, f->restores, 32U);
+ KUNIT_EXPECT_EQ(test, f->yields, 1U);
+ KUNIT_EXPECT_EQ(test, f->flushes, 2U);
+}
+
+static void memcomp_test_swapoff_walk_retired(struct kunit *test)
+{
+ struct mc16_test_walk *f = mc16_test_walk_begin(test);
+ struct mc16_test_storage *s;
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ KUNIT_ASSERT_EQ(test, mc16_test_walk_reserve(f, 33), 0);
+ for (i = 0; i < 32; i++)
+ mc16_ref_put(&s->refs[i]);
+ KUNIT_EXPECT_EQ(test, mc16_test_walk_run(test, f, 0), 0);
+ KUNIT_EXPECT_EQ(test, f->probes, 33U);
+ KUNIT_EXPECT_EQ(test, f->restores, 1U);
+ KUNIT_EXPECT_EQ(test, f->yields, 1U);
+ KUNIT_EXPECT_EQ(test, f->flushes, 2U);
+}
+
+static void memcomp_test_swapoff_walk_segment_hole(struct kunit *test)
+{
+ struct mc16_test_walk *f = mc16_test_walk_begin(test);
+ struct mc16_test_storage *s;
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ s->probe.hold_segments = true;
+ KUNIT_ASSERT_EQ(test, mc16_test_walk_reserve(f, 65), 0);
+ for (i = 0; i < 64; i++)
+ mc16_ref_put(&s->refs[i]);
+ KUNIT_ASSERT_NULL(test, s->refs[64].cluster->segments[0]);
+ KUNIT_EXPECT_EQ(test, mc16_test_walk_run(test, f, 0), 0);
+ KUNIT_EXPECT_EQ(test, f->probes, 65U);
+ KUNIT_EXPECT_EQ(test, f->restores, 1U);
+ KUNIT_EXPECT_EQ(test, f->yields, 2U);
+ KUNIT_EXPECT_EQ(test, f->flushes, 3U);
+}
+
+static void memcomp_test_swapoff_walk_private(struct kunit *test)
+{
+ struct mc16_test_walk *f = mc16_test_walk_begin(test);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_walk_reserve(f, 31), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_walk_private(f), 0);
+ KUNIT_ASSERT_EQ(test, refcount_read(&f->private.slot->refs), 0U);
+ KUNIT_EXPECT_EQ(test, mc16_test_walk_run(test, f, 0), 0);
+ KUNIT_EXPECT_EQ(test, f->probes, 32U);
+ KUNIT_EXPECT_EQ(test, f->restores, 31U);
+ KUNIT_EXPECT_EQ(test, f->yields, 1U);
+ KUNIT_EXPECT_EQ(test, f->flushes, 2U);
+ KUNIT_EXPECT_FALSE(test, f->private_active);
+}
+
+static void memcomp_test_swapoff_walk_markers(struct kunit *test)
+{
+ struct mc16_test_walk *f = mc16_test_walk_begin(test);
+ unsigned long flags;
+ unsigned int i;
+ u32 reserved;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (i = 0; i < ARRAY_SIZE(f->markers); i++) {
+ XA_STATE(xas, &mc16_clusters, i + 1);
+ void *entry;
+
+ xa_lock_irqsave(&mc16_clusters, flags);
+ ret = __xa_alloc(&mc16_clusters, &reserved, NULL,
+ XA_LIMIT(i + 1, i + 1), GFP_NOWAIT | __GFP_NOWARN);
+ if (!ret)
+ f->markers[i] = reserved;
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ rcu_read_lock();
+ entry = xas_load(&xas);
+ rcu_read_unlock();
+ KUNIT_ASSERT_TRUE(test, xa_is_zero(entry));
+ }
+ KUNIT_ASSERT_EQ(test, mc16_test_walk_reserve(f, 1), 0);
+ KUNIT_EXPECT_EQ(test, mc16_test_walk_run(test, f, 0), 0);
+ KUNIT_EXPECT_EQ(test, f->probes, 33U);
+ KUNIT_EXPECT_EQ(test, f->restores, 1U);
+ KUNIT_EXPECT_EQ(test, f->yields, 1U);
+ KUNIT_EXPECT_EQ(test, f->flushes, 2U);
+}
+
+static void memcomp_test_swapoff_walk_reflush(struct kunit *test)
+{
+ struct mc16_test_walk *f = mc16_test_walk_begin(test);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_walk_reserve(f, 33), 0);
+ f->replace = true;
+ KUNIT_EXPECT_EQ(test, mc16_test_walk_run(test, f, 0), 0);
+ KUNIT_EXPECT_EQ(test, f->writer_error, 0);
+ KUNIT_EXPECT_TRUE(test, f->retained);
+ KUNIT_EXPECT_TRUE(test, f->appended);
+ KUNIT_EXPECT_TRUE(test, f->private_active);
+ KUNIT_EXPECT_EQ(test, f->settled, 1U);
+ /* 32 old guards, one retired position, 33 appended, one private. */
+ KUNIT_EXPECT_EQ(test, f->probes, 67U);
+ KUNIT_EXPECT_EQ(test, f->restores, 66U);
+ KUNIT_EXPECT_EQ(test, f->yields, 2U);
+ KUNIT_EXPECT_EQ(test, f->flushes, 3U);
+}
+
+/* Force the real allocator to publish a cluster at the last numeric ID. */
+static int mc16_test_walk_last_id(u32 *id)
+{
+ return __xa_alloc(&mc16_clusters, id, NULL,
+ XA_LIMIT(MC16_MAX_CLUSTER_ID, MC16_MAX_CLUSTER_ID),
+ GFP_NOWAIT | __GFP_NOWARN);
+}
+
+static void memcomp_test_swapoff_walk_u32_max(struct kunit *test)
+{
+ struct mc16_test_walk *f = mc16_test_walk_begin(test);
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ kunit_activate_static_stub(test, mc16_private_id_alloc,
+ mc16_test_walk_last_id);
+ ret = mc16_test_walk_reserve(f, MC16_CLUSTER_SLOTS);
+ kunit_deactivate_static_stub(test, mc16_private_id_alloc);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ KUNIT_ASSERT_EQ(test,
+ f->refs->storage->refs[MC16_CLUSTER_SLOTS - 1].token, U32_MAX);
+ KUNIT_EXPECT_EQ(test, mc16_test_walk_run(test, f, 0), 0);
+ KUNIT_EXPECT_EQ(test, f->last_cursor, MC16_SCAN_END_CURSOR);
+ KUNIT_EXPECT_EQ(test, f->last_token, (u64)U32_MAX);
+ KUNIT_EXPECT_EQ(test, f->probes, (unsigned int)MC16_CLUSTER_SLOTS);
+ KUNIT_EXPECT_EQ(test, f->restores, (unsigned int)MC16_CLUSTER_SLOTS);
+ /* Known EOF at the final token does not reflush an empty batch. */
+ KUNIT_EXPECT_EQ(test, f->yields,
+ (unsigned int)MC16_CLUSTER_SLOTS / MC16_SWAPOFF_SCAN_BATCH - 1);
+ KUNIT_EXPECT_EQ(test, f->flushes, f->yields + 1);
+}
+
+/* Reuse the real slots, resetting observers only, for each exit/flag pair. */
+static void mc16_test_walk_reset(struct mc16_test_walk *f)
+{
+ f->flushes = 0;
+ f->restores = 0;
+ f->yields = 0;
+ f->probes = 0;
+ f->slice = 0;
+ f->scans = 0;
+ f->last_cursor = 0;
+ f->last_token = 0;
+ f->bad_scope = false;
+ f->bad_ref = false;
+ f->bad_budget = false;
+ f->dirty_scan = false;
+ f->interrupted = false;
+ f->stop = MC16_WALK_STOP_NONE;
+ f->fail_flush = 0;
+ f->fail_restore = 0;
+}
+
+static void memcomp_test_swapoff_walk_exits(struct kunit *test)
+{
+ static const struct {
+ unsigned int flush, restore;
+ enum mc16_test_walk_stop stop;
+ int result;
+ unsigned int probes, restores, flushes, yields;
+ } cases[] = {
+ { 1, 0, MC16_WALK_STOP_NONE, -EIO, 0, 0, 1, 0 },
+ { 2, 0, MC16_WALK_STOP_NONE, -EIO, 32, 32, 2, 1 },
+ { 0, 3, MC16_WALK_STOP_NONE, -ENOMEM, 3, 3, 1, 0 },
+ { 1, 0, MC16_WALK_STOP_BEFORE_SCAN, -EIO, 0, 0, 1, 0 },
+ { 0, 1, MC16_WALK_STOP_AFTER_RESTORE, -ENOMEM, 1, 1, 1, 0 },
+ { 0, 0, MC16_WALK_STOP_BEFORE_SCAN, -EINTR, 0, 0, 1, 0 },
+ { 0, 0, MC16_WALK_STOP_AFTER_SCAN, -EINTR, 1, 0, 1, 0 },
+ { 0, 0, MC16_WALK_STOP_AFTER_RESTORE, -EINTR, 1, 1, 1, 0 },
+ { 0, 0, MC16_WALK_STOP_UNLOCKED, -EINTR, 32, 32, 2, 1 },
+ { 0, 0, MC16_WALK_STOP_NONE, 0, 33, 33, 2, 1 },
+ { 0, 0, MC16_WALK_STOP_EOF, 0, 33, 33, 2, 1 },
+ };
+ static const unsigned int nested[] = {
+ 0, PF_MEMALLOC_NOFS, PF_MEMALLOC, MC16_WALK_TEST_FLAGS,
+ };
+ struct mc16_test_walk *f = mc16_test_walk_begin(test);
+ unsigned int i, j;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_walk_reserve(f, 33), 0);
+ for (i = 0; i < ARRAY_SIZE(nested); i++) {
+ for (j = 0; j < ARRAY_SIZE(cases); j++) {
+ mc16_test_walk_reset(f);
+ f->fail_flush = cases[j].flush;
+ f->fail_restore = cases[j].restore;
+ f->stop = cases[j].stop;
+ f->interrupted = f->stop == MC16_WALK_STOP_BEFORE_SCAN;
+ ret = mc16_test_walk_run(test, f, nested[i]);
+ KUNIT_EXPECT_EQ(test, ret, cases[j].result);
+ KUNIT_EXPECT_EQ(test, f->probes, cases[j].probes);
+ KUNIT_EXPECT_EQ(test, f->restores, cases[j].restores);
+ KUNIT_EXPECT_EQ(test, f->flushes, cases[j].flushes);
+ KUNIT_EXPECT_EQ(test, f->yields, cases[j].yields);
+ }
+ }
+}
+#endif /* CONFIG_MEMCG && CONFIG_SWAP */
+#endif /* _MM_MEMCOMP_TESTS_SWAPOFF_WALK_TEST_H */
diff --git a/mm/memcomp/tests/swapoff_yield_test.h b/mm/memcomp/tests/swapoff_yield_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..ea5f714e5036bc74eefead6d4406559a208b512e
--- /dev/null
+++ b/mm/memcomp/tests/swapoff_yield_test.h
@@ -0,0 +1,532 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_SWAPOFF_YIELD_TEST_H
+#define _MM_MEMCOMP_TESTS_SWAPOFF_YIELD_TEST_H
+
+#if defined(CONFIG_SWAP) && defined(CONFIG_MEMCOMP_KUNIT_TEST)
+#include <kunit/resource.h>
+#include <kunit/test.h>
+#include <linux/completion.h>
+#include <linux/kthread.h>
+#include <linux/rcupdate.h>
+#include <linux/sched/signal.h>
+
+/*
+ * Include after memcomp_swapoff(), independently of suite.h. Exercise
+ * the production walk with real XA lookups, temporary krefs and mutexes. Only
+ * the device flush/restore edges use fixture callbacks. STORED misses still
+ * run memcomp_swapoff_entry(); no synthetic backing reaches physical IO.
+ *
+ * The writer actually acquires W. It checks refs before replacing future and
+ * already-scanned objects, then leaves both staging and plugged IO pending.
+ * Every subsequent lookup must observe a flush. Disabling the unlock, skipping
+ * the flush, resetting the cursor, counting only conversions, or retaining a
+ * temporary reference must fail these assertions, independently of a counter
+ * that merely claims that a yield happened.
+ */
+#define MEMCOMP_YIELD_TEST_OBJECTS 72
+#define MEMCOMP_YIELD_TEST_FLAGS (PF_MEMALLOC_NOFS | PF_MEMALLOC)
+
+struct memcomp_yield_test {
+ struct xarray tree;
+ /* Serializes the tree walker with fixture object updates. */
+ struct mutex lock;
+ struct memcomp_entry objects[MEMCOMP_YIELD_TEST_OBJECTS];
+ unsigned long keys[MEMCOMP_YIELD_TEST_OBJECTS];
+ unsigned int visits[MEMCOMP_YIELD_TEST_OBJECTS];
+ unsigned int nr;
+ struct memcomp_swapoff_yield_hooks hooks;
+ struct completion writer_start;
+ struct completion writer_attempting;
+ struct completion writer_done;
+ struct completion walker_done;
+ struct task_struct *writer;
+ struct task_struct *walker;
+ unsigned int scans;
+ unsigned int restores;
+ unsigned int yields;
+ unsigned int flushes;
+ unsigned int settlements;
+ unsigned int writer_at_scan;
+ unsigned int flags_before;
+ unsigned int flags_after;
+ unsigned int fail_flush;
+ unsigned int fail_restore;
+ unsigned int signal_flush;
+ unsigned int signal_scan;
+ unsigned long last_key;
+ int restore_result;
+ int result;
+ int writer_error;
+ bool want_writer;
+ bool dying;
+ bool sparse;
+ bool nested;
+ bool signal_writer;
+ bool cancel;
+ bool timed_out;
+ bool bad_ref;
+ bool bad_scope;
+ bool dirty_scan;
+ bool staging;
+ bool plugged;
+ bool writer_acquired;
+};
+
+static void memcomp_yield_test_destroy(void *data)
+{
+ struct memcomp_yield_test *f = data;
+
+ xa_destroy(&f->tree);
+}
+
+static struct memcomp_yield_test *
+memcomp_yield_test_alloc(struct kunit *test, unsigned int nr, bool dying)
+{
+ struct memcomp_yield_test *f;
+ unsigned int i;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ xa_init(&f->tree);
+ mutex_init(&f->lock);
+ init_completion(&f->writer_start);
+ init_completion(&f->writer_attempting);
+ init_completion(&f->writer_done);
+ init_completion(&f->walker_done);
+ if (kunit_add_action_or_reset(test, memcomp_yield_test_destroy, f))
+ return NULL;
+ f->nr = nr;
+ f->dying = dying;
+ /* The initial acquisition also has to flush, even on an empty index. */
+ f->staging = true;
+ f->plugged = true;
+ for (i = 0; i < ARRAY_SIZE(f->objects); i++) {
+ struct memcomp_entry *entry = &f->objects[i];
+
+ f->keys[i] = 1 + 3 * i;
+ entry->token = f->keys[i];
+ entry->state = MC_STORED;
+ kref_init(&entry->refcount);
+ if (dying)
+ refcount_set(&entry->refcount.refcount, 0);
+ }
+ for (i = 0; i < nr; i++) {
+ if (xa_err(xa_store(&f->tree, f->keys[i], &f->objects[i],
+ GFP_KERNEL)))
+ return NULL;
+ }
+ return f;
+}
+
+static int memcomp_yield_test_flush(void *data)
+{
+ struct memcomp_yield_test *f = data;
+
+ lockdep_assert_held(&f->lock);
+ f->flushes++;
+ if (f->staging || f->plugged)
+ f->settlements++;
+ f->staging = false;
+ f->plugged = false;
+ if (f->signal_flush == f->flushes)
+ send_sig(SIGUSR1, current, 0);
+ return f->fail_flush == f->flushes ? -EIO : 0;
+}
+
+static int memcomp_yield_test_restore(struct memcomp_entry *entry,
+ unsigned int type, void *data)
+{
+ struct memcomp_yield_test *f = data;
+ unsigned int object = entry - f->objects;
+ int ret;
+
+ lockdep_assert_held(&f->lock);
+ f->restores++;
+ if (object >= ARRAY_SIZE(f->objects))
+ return -EINVAL;
+ f->visits[object]++;
+ if (f->fail_restore == f->restores)
+ return -ENOMEM;
+ /* All-miss cases execute the actual entry restore's state rejection. */
+ ret = memcomp_swapoff_entry(entry, type, &memcomp_swapoff_ops);
+ return ret ? ret : f->restore_result;
+}
+
+static void memcomp_yield_test_scanned(struct memcomp_entry *entry,
+ unsigned long key, void *data)
+{
+ struct memcomp_yield_test *f = data;
+
+ lockdep_assert_held(&f->lock);
+ WRITE_ONCE(f->scans, f->scans + 1);
+ f->last_key = key;
+ if (f->staging || f->plugged)
+ f->dirty_scan = true;
+ if ((current->flags & MEMCOMP_YIELD_TEST_FLAGS) !=
+ MEMCOMP_YIELD_TEST_FLAGS)
+ f->bad_scope = true;
+ if (entry && kref_read(&entry->refcount) != 2)
+ f->bad_ref = true;
+ if (f->signal_scan == f->scans)
+ send_sig(SIGUSR1, current, 0);
+ if (f->want_writer && f->scans == 32) {
+ complete(&f->writer_start);
+ /* The writer must try W while the scanner still owns it. */
+ if (!wait_for_completion_timeout(&f->writer_attempting, 2 * HZ)) {
+ f->timed_out = true;
+ send_sig(SIGUSR1, current, 0);
+ }
+ }
+}
+
+static void memcomp_yield_test_unlocked(void *data)
+{
+ struct memcomp_yield_test *f = data;
+
+ f->yields++;
+ if ((current->flags & MEMCOMP_YIELD_TEST_FLAGS) !=
+ MEMCOMP_YIELD_TEST_FLAGS)
+ f->bad_scope = true;
+ /* Observe real writer progress before the scanner reacquires W. */
+ if (f->want_writer && f->yields == 1 &&
+ !wait_for_completion_timeout(&f->writer_done, 2 * HZ)) {
+ f->timed_out = true;
+ send_sig(SIGUSR1, current, 0);
+ }
+}
+
+static int memcomp_yield_test_writer(void *data)
+{
+ struct memcomp_yield_test *f = data;
+ unsigned int i;
+
+ wait_for_completion(&f->writer_start);
+ if (READ_ONCE(f->cancel))
+ goto done;
+ complete(&f->writer_attempting);
+ mutex_lock(&f->lock);
+ f->writer_acquired = true;
+ f->writer_at_scan = READ_ONCE(f->scans);
+ for (i = 0; i < f->nr; i++) {
+ if (kref_read(&f->objects[i].refcount) != !f->dying)
+ f->bad_ref = true;
+ }
+ if (f->sparse) {
+ /* Reuse a visited key, replace a future object, remove the tail. */
+ f->writer_error = xa_err(xa_store(&f->tree, 16,
+ &f->objects[34], GFP_KERNEL));
+ if (!f->writer_error)
+ f->writer_error = xa_err(xa_store(&f->tree, 4096,
+ &f->objects[35], GFP_KERNEL));
+ xa_erase(&f->tree, f->keys[31]);
+ }
+ f->staging = true;
+ f->plugged = true;
+ if (f->signal_writer)
+ send_sig(SIGUSR1, f->walker, 0);
+ mutex_unlock(&f->lock);
+done:
+ complete(&f->writer_done);
+ return 0;
+}
+
+static int memcomp_yield_test_walker(void *data)
+{
+ struct memcomp_yield_test *f = data;
+ unsigned int nofs = 0, noreclaim = 0;
+
+ allow_signal(SIGUSR1);
+ if (f->nested) {
+ nofs = memalloc_nofs_save();
+ noreclaim = memalloc_noreclaim_save();
+ }
+ f->flags_before = current->flags & MEMCOMP_YIELD_TEST_FLAGS;
+ f->hooks.task = current;
+ rcu_assign_pointer(memcomp_swapoff_yield_active, &f->hooks);
+ f->result = memcomp_swapoff(0);
+ rcu_assign_pointer(memcomp_swapoff_yield_active, NULL);
+ f->flags_after = current->flags & MEMCOMP_YIELD_TEST_FLAGS;
+ if (f->nested) {
+ memalloc_noreclaim_restore(noreclaim);
+ memalloc_nofs_restore(nofs);
+ }
+ flush_signals(current);
+ /* Retire readers that fetched active before the task identity check. */
+ synchronize_rcu();
+ complete(&f->walker_done);
+ return 0;
+}
+
+/* No assertion may abort this helper while either thread still owns f. */
+static void memcomp_yield_test_run(struct kunit *test,
+ struct memcomp_yield_test *f)
+{
+ unsigned int i;
+
+ KUNIT_ASSERT_PTR_EQ(test,
+ rcu_access_pointer(memcomp_swapoff_yield_active), NULL);
+ f->hooks = (struct memcomp_swapoff_yield_hooks) {
+ .entries = &f->tree,
+ .lock = &f->lock,
+ .flush = memcomp_yield_test_flush,
+ .restore = memcomp_yield_test_restore,
+ .scanned = memcomp_yield_test_scanned,
+ .unlocked = memcomp_yield_test_unlocked,
+ .ctx = f,
+ };
+ if (f->want_writer) {
+ f->writer = kthread_create(memcomp_yield_test_writer, f,
+ "mc-swapoff-writer");
+ if (IS_ERR(f->writer)) {
+ KUNIT_FAIL(test, "cannot create writer: %ld",
+ PTR_ERR(f->writer));
+ return;
+ }
+ get_task_struct(f->writer);
+ wake_up_process(f->writer);
+ }
+ f->walker = kthread_create(memcomp_yield_test_walker, f,
+ "mc-swapoff-walker");
+ if (IS_ERR(f->walker)) {
+ KUNIT_FAIL(test, "cannot create walker: %ld", PTR_ERR(f->walker));
+ goto stop_writer;
+ }
+ get_task_struct(f->walker);
+ wake_up_process(f->walker);
+ if (!wait_for_completion_timeout(&f->walker_done, 6 * HZ)) {
+ f->timed_out = true;
+ send_sig(SIGUSR1, f->walker, 0);
+ }
+ kthread_stop(f->walker);
+ put_task_struct(f->walker);
+stop_writer:
+ WRITE_ONCE(f->cancel, true);
+ complete_all(&f->writer_start);
+ if (f->want_writer) {
+ kthread_stop(f->writer);
+ put_task_struct(f->writer);
+ }
+ KUNIT_EXPECT_PTR_EQ(test,
+ rcu_access_pointer(memcomp_swapoff_yield_active), NULL);
+ KUNIT_EXPECT_FALSE(test, f->timed_out);
+ KUNIT_EXPECT_FALSE(test, f->bad_ref);
+ KUNIT_EXPECT_FALSE(test, f->bad_scope);
+ KUNIT_EXPECT_FALSE(test, f->dirty_scan);
+ KUNIT_EXPECT_EQ(test, f->writer_error, 0);
+ KUNIT_EXPECT_EQ(test, f->flags_before, f->flags_after);
+ KUNIT_EXPECT_FALSE(test, mutex_is_locked(&f->lock));
+ for (i = 0; i < ARRAY_SIZE(f->objects); i++)
+ KUNIT_EXPECT_EQ_MSG(test, kref_read(&f->objects[i].refcount),
+ (unsigned int)!f->dying, "object %u", i);
+}
+
+static void memcomp_test_swapoff_yield_all_miss(struct kunit *test)
+{
+ struct memcomp_yield_test *f = memcomp_yield_test_alloc(test, 65, false);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->want_writer = true;
+ memcomp_yield_test_run(test, f);
+ KUNIT_EXPECT_EQ(test, f->result, 0);
+ KUNIT_EXPECT_EQ(test, f->scans, 65U);
+ KUNIT_EXPECT_EQ(test, f->restores, 65U);
+ KUNIT_EXPECT_EQ(test, f->yields, 2U);
+ KUNIT_EXPECT_EQ(test, f->flushes, 3U);
+ KUNIT_EXPECT_EQ(test, f->settlements, 2U);
+ KUNIT_EXPECT_TRUE(test, f->writer_acquired);
+ KUNIT_EXPECT_EQ(test, f->writer_at_scan, 32U);
+}
+
+static void memcomp_test_swapoff_yield_dying(struct kunit *test)
+{
+ struct memcomp_yield_test *f = memcomp_yield_test_alloc(test, 65, true);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->want_writer = true;
+ memcomp_yield_test_run(test, f);
+ KUNIT_EXPECT_EQ(test, f->result, 0);
+ KUNIT_EXPECT_EQ(test, f->scans, 65U);
+ KUNIT_EXPECT_EQ(test, f->restores, 0U);
+ KUNIT_EXPECT_EQ(test, f->yields, 2U);
+ KUNIT_EXPECT_EQ(test, f->flushes, 3U);
+ KUNIT_EXPECT_TRUE(test, f->writer_acquired);
+ KUNIT_EXPECT_EQ(test, f->writer_at_scan, 32U);
+}
+
+static void memcomp_test_swapoff_yield_cursor(struct kunit *test)
+{
+ struct memcomp_yield_test *f = memcomp_yield_test_alloc(test, 34, false);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ xa_erase(&f->tree, f->keys[32]);
+ xa_erase(&f->tree, f->keys[33]);
+ f->objects[32].token = 4096;
+ f->keys[32] = 4096;
+ f->objects[33].token = UINT_MAX;
+ f->keys[33] = UINT_MAX;
+ f->objects[34].token = 16;
+ f->objects[35].token = 4096;
+ KUNIT_ASSERT_EQ(test, xa_err(xa_store(&f->tree, 4096,
+ &f->objects[32], GFP_KERNEL)), 0);
+ KUNIT_ASSERT_EQ(test, xa_err(xa_store(&f->tree, UINT_MAX,
+ &f->objects[33], GFP_KERNEL)), 0);
+ f->want_writer = true;
+ f->sparse = true;
+ memcomp_yield_test_run(test, f);
+ KUNIT_EXPECT_EQ(test, f->result, 0);
+ KUNIT_EXPECT_EQ(test, f->scans, 34U);
+ KUNIT_EXPECT_EQ(test, f->yields, 1U);
+ KUNIT_EXPECT_EQ(test, f->last_key, (unsigned long)UINT_MAX);
+ KUNIT_EXPECT_EQ(test, f->visits[5], 1U);
+ KUNIT_EXPECT_EQ(test, f->visits[32], 0U);
+ KUNIT_EXPECT_EQ(test, f->visits[33], 1U);
+ KUNIT_EXPECT_EQ(test, f->visits[34], 0U);
+ KUNIT_EXPECT_EQ(test, f->visits[35], 1U);
+ KUNIT_EXPECT_TRUE(test, f->writer_acquired);
+ KUNIT_EXPECT_EQ(test, f->writer_at_scan, 32U);
+}
+
+static void memcomp_test_swapoff_yield_flush_error(struct kunit *test)
+{
+ unsigned int flush;
+
+ for (flush = 1; flush <= 2; flush++) {
+ struct memcomp_yield_test *f =
+ memcomp_yield_test_alloc(test, 65, false);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->fail_flush = flush;
+ f->want_writer = flush == 2;
+ f->nested = flush == 2;
+ memcomp_yield_test_run(test, f);
+ KUNIT_EXPECT_EQ(test, f->result, -EIO);
+ KUNIT_EXPECT_EQ(test, f->flushes, flush);
+ KUNIT_EXPECT_EQ(test, f->scans, flush == 1 ? 0U : 32U);
+ KUNIT_EXPECT_EQ(test, f->yields, flush - 1);
+ KUNIT_EXPECT_FALSE(test, f->staging);
+ KUNIT_EXPECT_FALSE(test, f->plugged);
+ }
+}
+
+static void memcomp_test_swapoff_yield_signal(struct kunit *test)
+{
+ unsigned int phase;
+
+ for (phase = 0; phase < 4; phase++) {
+ struct memcomp_yield_test *f =
+ memcomp_yield_test_alloc(test, phase == 3 ? 0 : 65, false);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ if (phase == 0 || phase == 3) {
+ f->signal_flush = 1;
+ } else if (phase == 1) {
+ f->signal_scan = 1;
+ } else {
+ f->want_writer = true;
+ f->signal_writer = true;
+ }
+ memcomp_yield_test_run(test, f);
+ KUNIT_EXPECT_EQ(test, f->result, -EINTR);
+ KUNIT_EXPECT_EQ(test, f->scans,
+ phase == 1 ? 1U : phase == 2 ? 32U : 0U);
+ if (phase == 2) {
+ KUNIT_EXPECT_TRUE(test, f->writer_acquired);
+ KUNIT_EXPECT_EQ(test, f->flushes, 2U);
+ KUNIT_EXPECT_FALSE(test, f->staging);
+ KUNIT_EXPECT_FALSE(test, f->plugged);
+ }
+ }
+}
+
+static void memcomp_test_swapoff_yield_positive_status(struct kunit *test)
+{
+ struct memcomp_yield_test *f = memcomp_yield_test_alloc(test, 33, false);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->restore_result = 1;
+ memcomp_yield_test_run(test, f);
+ KUNIT_EXPECT_EQ(test, f->result, 0);
+ KUNIT_EXPECT_EQ(test, f->restores, 33U);
+ KUNIT_EXPECT_EQ(test, f->yields, 1U);
+}
+
+static void memcomp_test_swapoff_yield_restore_error(struct kunit *test)
+{
+ struct memcomp_yield_test *f = memcomp_yield_test_alloc(test, 33, false);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->fail_restore = 12;
+ f->restore_result = 1;
+ memcomp_yield_test_run(test, f);
+ KUNIT_EXPECT_EQ(test, f->result, -ENOMEM);
+ KUNIT_EXPECT_EQ(test, f->scans, 12U);
+ KUNIT_EXPECT_EQ(test, f->restores, 12U);
+ KUNIT_EXPECT_EQ(test, f->yields, 0U);
+}
+
+static void memcomp_test_swapoff_yield_budget(struct kunit *test)
+{
+ static const unsigned int counts[] = { 0, 31, 32, 33 };
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(counts); i++) {
+ struct memcomp_yield_test *f =
+ memcomp_yield_test_alloc(test, counts[i], false);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->nested = i & 1;
+ memcomp_yield_test_run(test, f);
+ KUNIT_EXPECT_EQ(test, f->result, 0);
+ KUNIT_EXPECT_EQ(test, f->scans, counts[i]);
+ KUNIT_EXPECT_EQ(test, f->yields, counts[i] / 32);
+ KUNIT_EXPECT_EQ(test, f->flushes, 1 + counts[i] / 32);
+ }
+}
+
+static void memcomp_test_swapoff_yield_uint_max(struct kunit *test)
+{
+ unsigned int signal;
+
+ for (signal = 0; signal <= 1; signal++) {
+ struct memcomp_yield_test *f =
+ memcomp_yield_test_alloc(test, 32, false);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ xa_erase(&f->tree, f->keys[31]);
+ f->objects[31].token = UINT_MAX;
+ f->keys[31] = UINT_MAX;
+ KUNIT_ASSERT_EQ(test, xa_err(xa_store(&f->tree, UINT_MAX,
+ &f->objects[31], GFP_KERNEL)), 0);
+ f->restore_result = 1;
+ f->signal_scan = signal ? 32 : 0;
+ memcomp_yield_test_run(test, f);
+ KUNIT_EXPECT_EQ(test, f->result, signal ? -EINTR : 0);
+ KUNIT_EXPECT_EQ(test, f->scans, 32U);
+ KUNIT_EXPECT_EQ(test, f->last_key, (unsigned long)UINT_MAX);
+ KUNIT_EXPECT_EQ(test, f->yields, 0U);
+ KUNIT_EXPECT_EQ(test, f->flushes, 1U);
+ }
+}
+
+static struct kunit_case memcomp_swapoff_yield_test_cases[] = {
+ KUNIT_CASE(memcomp_test_swapoff_yield_all_miss),
+ KUNIT_CASE(memcomp_test_swapoff_yield_dying),
+ KUNIT_CASE(memcomp_test_swapoff_yield_cursor),
+ KUNIT_CASE(memcomp_test_swapoff_yield_flush_error),
+ KUNIT_CASE(memcomp_test_swapoff_yield_signal),
+ KUNIT_CASE(memcomp_test_swapoff_yield_positive_status),
+ KUNIT_CASE(memcomp_test_swapoff_yield_restore_error),
+ KUNIT_CASE(memcomp_test_swapoff_yield_budget),
+ KUNIT_CASE(memcomp_test_swapoff_yield_uint_max),
+ {}
+};
+
+static struct kunit_suite memcomp_swapoff_yield_test_suite = {
+ .name = "memcomp-swapoff-yield",
+ .test_cases = memcomp_swapoff_yield_test_cases,
+};
+
+kunit_test_suite(memcomp_swapoff_yield_test_suite);
+#endif /* CONFIG_SWAP && CONFIG_MEMCOMP_KUNIT_TEST */
+#endif /* _MM_MEMCOMP_TESTS_SWAPOFF_YIELD_TEST_H */
diff --git a/mm/memcomp/tests/test_api.h b/mm/memcomp/tests/test_api.h
new file mode 100644
index 0000000000000000000000000000000000000000..712e1e3c9de497b78ca786da120643fa9cb19b98
--- /dev/null
+++ b/mm/memcomp/tests/test_api.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_TEST_API_H
+#define _MM_MEMCOMP_TESTS_TEST_API_H
+
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+struct memcomp_record_info {
+ u64 allocated_bytes;
+ u64 charged_bytes;
+ unsigned int members;
+};
+
+static inline void *
+memcomp_test_record_alloc(struct folio *folio, unsigned int nr_pages)
+{
+ return mc16_record_alloc(folio, nr_pages, false);
+}
+
+static inline struct memcomp_record_info
+memcomp_test_record_info(void *handle)
+{
+ struct mc16_record *record = handle;
+
+ return (struct memcomp_record_info) {
+ .allocated_bytes = record->alloc_bytes,
+ .charged_bytes = record->charge_bytes,
+ .members = refcount_read(&record->members),
+ };
+}
+
+static inline void memcomp_test_record_put(void *handle)
+{
+ mc16_record_member_put(handle);
+}
+
+static inline void memcomp_test_record_handle_abort(void *handle)
+{
+ mc16_record_abort(handle);
+}
+#endif
+
+#endif /* _MM_MEMCOMP_TESTS_TEST_API_H */
diff --git a/mm/memcomp/tests/transport_reclaim_test.h b/mm/memcomp/tests/transport_reclaim_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..b9cba4161552a53cf57b4e86ea3601b0459918a4
--- /dev/null
+++ b/mm/memcomp/tests/transport_reclaim_test.h
@@ -0,0 +1,298 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Included once inside workingset_load_test.h's existing swap/scope guard. */
+struct mc16_tr {
+ struct kunit *test;
+ struct folio *folio[2]; /* Allocation refs only; [1] is the keeper. */
+ struct swap_info_struct *si[2];
+ swp_entry_t swap[2];
+ struct xa_node *leaf; /* Compare only; dereference current cursors under XA. */
+ bool reserved[2], held[2], handed;
+ bool shadow, empty;
+};
+
+static void mc16_tr_cleanup(void *data)
+{
+ struct mc16_tr *f = data;
+ unsigned char map[2] = { };
+ bool checked[2] = { }, marked[2] = { };
+ unsigned int i;
+
+ if (!f->swap[0].val && !f->swap[1].val)
+ return;
+ lru_add_drain_all();
+ disable_swap_slots_cache_lock();
+ /* Target first: keep the keeper and its slot alive through the oracle. */
+ for (i = 0; i < 2; i++) {
+ struct folio *folio = f->folio[i];
+ swp_entry_t entry = f->swap[i];
+
+ f->folio[i] = NULL;
+ /* Reclaim consumed the old isolation ref, even on putback. */
+ if (!folio && !i && f->handed)
+ folio = swap_cache_get_folio(entry, NULL, 0);
+ if (folio) {
+ folio_lock(folio);
+ folio_wait_writeback(folio);
+ if (folio_test_swapcache(folio))
+ delete_from_swap_cache(folio);
+ folio_unlock(folio);
+ folio_put(folio);
+ }
+ if (f->reserved[i])
+ swapcache_free_entries(&entry, 1);
+ if (f->held[i])
+ swap_free(f->swap[i]);
+ if (f->si[i]) {
+ map[i] = READ_ONCE(f->si[i]->swap_map[swp_offset(f->swap[i])]);
+ marked[i] = swap_memcomp_transport_test(f->si[i],
+ swp_offset(f->swap[i]));
+ checked[i] = true;
+ put_swap_device(f->si[i]);
+ f->si[i] = NULL;
+ }
+ f->reserved[i] = false;
+ f->held[i] = false;
+ f->swap[i].val = 0;
+ }
+ f->handed = false;
+ reenable_swap_slots_cache_unlock();
+ for (i = 0; i < 2; i++) {
+ if (!checked[i])
+ continue;
+ kunit_info(f->test, "transport reclaim cleanup: index=%u map=%u marked=%d",
+ i, map[i], marked[i]);
+ KUNIT_EXPECT_EQ(f->test, map[i], (unsigned char)0);
+ KUNIT_EXPECT_FALSE(f->test, marked[i]);
+ }
+}
+
+static int mc16_tr_alloc(struct mc16_tr *f, bool transport)
+{
+ unsigned int i;
+ int nr, ret = 0;
+
+ if (!READ_ONCE(mc16_kunit_real_swap))
+ return -EINVAL;
+ if (transport) {
+ mutex_lock(&memcomp_writeback_lock);
+ if (memcomp_writeback_slots_free) {
+ mutex_unlock(&memcomp_writeback_lock);
+ return -EBUSY;
+ }
+ for (i = 0; i < 2; i++) {
+ ret = memcomp_swap_folio_alloc(&f->folio[i], &f->swap[i]);
+ if (ret)
+ break;
+ f->held[i] = true;
+ }
+ memcomp_writeback_slots_return();
+ mutex_unlock(&memcomp_writeback_lock);
+ for (i = 0; i < 2; i++)
+ if (f->folio[i])
+ folio_unlock(f->folio[i]);
+ if (ret)
+ return ret;
+ } else {
+ nr = get_swap_pages(2, f->swap, 0);
+ for (i = 0; i < nr; i++)
+ f->reserved[i] = true;
+ if (nr != 2)
+ return -ENOSPC;
+ }
+ for (i = 0; i < 2; i++) {
+ f->si[i] = get_swap_device(f->swap[i]);
+ if (!f->si[i])
+ return -ENODEV;
+ if (transport)
+ continue;
+ f->folio[i] = folio_alloc(GFP_HIGHUSER_MOVABLE, 0);
+ if (!f->folio[i])
+ return -ENOMEM;
+ folio_lock(f->folio[i]);
+ __folio_set_swapbacked(f->folio[i]);
+ ret = add_to_swap_cache(f->folio[i], f->swap[i], GFP_KERNEL, NULL);
+ if (!ret) {
+ f->reserved[i] = false;
+ ret = swap_duplicate(f->swap[i]);
+ if (!ret)
+ f->held[i] = true;
+ }
+ folio_unlock(f->folio[i]);
+ if (ret)
+ return ret;
+ }
+ return 0;
+}
+
+static int mc16_tr_prepare(struct mc16_tr *f, const void *src, unsigned int i)
+{
+ struct folio *folio = f->folio[i];
+ struct writeback_control wbc = { .sync_mode = WB_SYNC_ALL };
+ void *dst;
+ int ret;
+
+ folio_lock(folio);
+ /* Balance the initial uncharged cache stats before charging. */
+ delete_from_swap_cache(folio); /* The held slot ref stays live. */
+ ret = mem_cgroup_charge(folio, &init_mm, GFP_KERNEL);
+ if (ret)
+ goto unlock;
+ if (!mem_cgroup_disabled() && folio_memcg(folio) != root_mem_cgroup) {
+ ret = -EINVAL;
+ goto unlock;
+ }
+ ret = mem_cgroup_try_charge_swap(folio, f->swap[i]);
+ if (ret)
+ goto unlock;
+ ret = swapcache_prepare(f->swap[i], 1);
+ if (ret)
+ goto unlock;
+ ret = add_to_swap_cache(folio, f->swap[i], GFP_KERNEL, NULL);
+ if (ret) {
+ put_swap_folio(folio, f->swap[i]);
+ goto unlock;
+ }
+ dst = kmap_local_folio(folio, 0);
+ memcpy(dst, src, PAGE_SIZE);
+ kunmap_local(dst);
+ __folio_mark_uptodate(folio);
+ if (i)
+ goto unlock;
+ ret = swap_writepage(&folio->page, &wbc);
+ if (ret) {
+ if (folio_test_locked(folio))
+ folio_unlock(folio);
+ return ret;
+ }
+ folio_lock(folio);
+ folio_wait_writeback(folio);
+ if (folio_test_dirty(folio) || !folio_test_swapcache(folio) ||
+ !folio_test_uptodate(folio))
+ ret = -EIO;
+unlock:
+ folio_unlock(folio);
+ return ret;
+}
+
+/* Actual cursors, not just adjacent offsets; never write XA internals. */
+static bool mc16_tr_leaf(struct mc16_tr *f, bool after)
+{
+ struct address_space *mapping = swap_address_space(f->swap[0]);
+ XA_STATE(target, &mapping->i_pages, swap_cache_index(f->swap[0]));
+ XA_STATE(keeper, &mapping->i_pages, swap_cache_index(f->swap[1]));
+ void *entry, *kept;
+ bool same;
+
+ if (mapping != swap_address_space(f->swap[1]))
+ return false;
+ xa_lock_irq(&mapping->i_pages);
+ entry = xas_load(&target);
+ kept = xas_load(&keeper);
+ same = xas_is_node(&target) && xas_is_node(&keeper) &&
+ target.xa_node == keeper.xa_node && !target.xa_node->shift &&
+ kept == f->folio[1] &&
+ target.xa_node->count > target.xa_node->nr_values &&
+ list_empty(&target.xa_node->private_list) && !mapping_exiting(mapping);
+ if (after) {
+ same = same && target.xa_node == f->leaf;
+ f->shadow = xa_is_value(entry);
+ f->empty = !entry;
+ } else {
+ same = same && entry == f->folio[0];
+ if (same)
+ f->leaf = target.xa_node;
+ }
+ xa_unlock_irq(&mapping->i_pages);
+ return same;
+}
+
+static void mc16_tr_one(struct kunit *test, const void *src, bool transport)
+{
+ struct mc16_tr *f;
+ struct folio *folio;
+ unsigned long reclaimed = 0;
+ unsigned int i;
+ unsigned char target_map = 0;
+ int ret, refs = 0;
+ bool before = false, after = false, marked;
+ LIST_HEAD(list);
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->test = test;
+ KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, mc16_tr_cleanup, f), 0);
+ ret = mc16_tr_alloc(f, transport);
+ if (ret)
+ goto out;
+ ret = mc16_tr_prepare(f, src, 0);
+ if (!ret)
+ ret = mc16_tr_prepare(f, src, 1);
+ if (ret)
+ goto out;
+ for (i = 0; i < 2; i++) {
+ if (READ_ONCE(f->si[i]->swap_map[swp_offset(f->swap[i])]) !=
+ (SWAP_HAS_CACHE | 1)) {
+ ret = -EINVAL;
+ goto out;
+ }
+ }
+ marked = swap_memcomp_transport_test(f->si[0], swp_offset(f->swap[0]));
+ if (marked != transport) {
+ ret = -EINVAL;
+ goto out;
+ }
+ before = mc16_tr_leaf(f, false);
+ kunit_info(test,
+ "transport reclaim pair: transport=%d target=%#lx keeper=%#lx leaf=%p same=%d",
+ transport, f->swap[0].val, f->swap[1].val, f->leaf, before);
+ kunit_info(test, "transport reclaim indexes: target=%lu keeper=%lu marked=%d",
+ swap_cache_index(f->swap[0]), swap_cache_index(f->swap[1]), marked);
+ if (!before) {
+ ret = -EXDEV;
+ goto out;
+ }
+ folio = f->folio[0];
+ folio_add_lru(folio);
+ lru_add_drain_all();
+ if (!folio_isolate_lru(folio)) {
+ ret = -EBUSY;
+ goto out;
+ }
+ /* Allocation + cache + isolation. No extra target pin may survive. */
+ refs = folio_ref_count(folio);
+ if (refs != 3) {
+ folio_putback_lru(folio);
+ ret = -EBUSY;
+ goto out;
+ }
+ f->folio[0] = NULL;
+ f->handed = true;
+ list_add(&folio->lru, &list);
+ folio_put(folio); /* Drop only allocation; list owns isolation. */
+ folio = NULL;
+ reclaimed = reclaim_pages(&list);
+ /* Neither success nor putback leaves a fixture-owned target pointer. */
+ after = mc16_tr_leaf(f, true);
+ target_map = READ_ONCE(f->si[0]->swap_map[swp_offset(f->swap[0])]);
+ if (reclaimed != 1 || !after || target_map != 1) {
+ ret = -ESTALE;
+ goto out;
+ }
+ kunit_info(test, "transport reclaim transfer: refs=%d leaf=%d map=%u",
+ refs, after, target_map);
+ kunit_info(test,
+ "transport reclaim oracle: transport=%d reclaimed=%lu shadow=%d empty=%d",
+ transport, reclaimed, f->shadow, f->empty);
+ if (transport) {
+ KUNIT_EXPECT_FALSE(test, f->shadow);
+ KUNIT_EXPECT_TRUE(test, f->empty);
+ } else {
+ KUNIT_EXPECT_TRUE(test, f->shadow);
+ KUNIT_EXPECT_FALSE(test, f->empty);
+ }
+out:
+ kunit_info(test, "transport reclaim result: transport=%d ret=%d reclaimed=%lu",
+ transport, ret, reclaimed);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ kunit_release_action(test, mc16_tr_cleanup, f);
+}
diff --git a/mm/memcomp/tests/transport_test.h b/mm/memcomp/tests/transport_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..e2c7a7172386900872d48950fad047d17c6dd0cf
--- /dev/null
+++ b/mm/memcomp/tests/transport_test.h
@@ -0,0 +1,114 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_TRANSPORT_TEST_H
+#define _MM_MEMCOMP_TESTS_TRANSPORT_TEST_H
+
+#include <linux/bitmap.h>
+
+#ifdef CONFIG_SWAP
+static void mc16_test_transport_cleanup(void *data)
+{
+ swap_memcomp_transport_free(data);
+}
+
+/* Exercise the same allocation and bit operations as swapon/reclaim/free. */
+static struct swap_info_struct *
+mc16_test_transport_info(struct kunit *test, unsigned int max)
+{
+ struct swap_info_struct *si;
+
+ si = kunit_kzalloc(test, sizeof(*si), GFP_KERNEL);
+ if (!si)
+ return NULL;
+ si->max = max;
+ if (kunit_add_action_or_reset(test, mc16_test_transport_cleanup, si))
+ return NULL;
+ if (swap_memcomp_transport_alloc(si))
+ return NULL;
+ return si;
+}
+
+static void memcomp_test_transport_bitmap_lifecycle(struct kunit *test)
+{
+ struct swap_info_struct *si;
+ unsigned long last = BITS_PER_LONG + 2;
+
+ si = mc16_test_transport_info(test, last + 1);
+ KUNIT_ASSERT_NOT_NULL(test, si);
+ KUNIT_EXPECT_TRUE(test, bitmap_empty(si->memcomp_transport, si->max));
+ swap_memcomp_transport_set(si, 1);
+ swap_memcomp_transport_set(si, last);
+ KUNIT_EXPECT_TRUE(test, swap_memcomp_transport_test(si, 1));
+ KUNIT_EXPECT_TRUE(test, swap_memcomp_transport_test(si, last));
+ KUNIT_EXPECT_FALSE(test, swap_memcomp_transport_test(si, 2));
+
+ swap_memcomp_transport_clear(si, 1, 1);
+ KUNIT_EXPECT_FALSE(test, swap_memcomp_transport_test(si, 1));
+ KUNIT_EXPECT_TRUE(test, swap_memcomp_transport_test(si, last));
+ /* A slot returned unused must not mark its next, ordinary owner. */
+ swap_memcomp_transport_set(si, 1);
+ swap_memcomp_transport_clear(si, 1, 1);
+ KUNIT_EXPECT_FALSE(test, swap_memcomp_transport_test(si, 1));
+
+ /* Teardown and reactivation of the same device start with an empty map. */
+ swap_memcomp_transport_free(si);
+ KUNIT_EXPECT_PTR_EQ(test, si->memcomp_transport, NULL);
+ KUNIT_ASSERT_EQ(test, swap_memcomp_transport_alloc(si), 0);
+ KUNIT_EXPECT_TRUE(test, bitmap_empty(si->memcomp_transport, si->max));
+}
+
+static void memcomp_test_transport_bitmap_range(struct kunit *test)
+{
+ struct swap_info_struct *si;
+ unsigned long first = BITS_PER_LONG - 1;
+ unsigned int i;
+
+ si = mc16_test_transport_info(test, 2 * BITS_PER_LONG + 5);
+ KUNIT_ASSERT_NOT_NULL(test, si);
+ swap_memcomp_transport_set(si, first - 1);
+ for (i = 0; i < 4; i++)
+ swap_memcomp_transport_set(si, first + i);
+ swap_memcomp_transport_set(si, first + 4);
+ swap_memcomp_transport_clear(si, first, 0);
+ KUNIT_EXPECT_TRUE(test, swap_memcomp_transport_test(si, first));
+ swap_memcomp_transport_clear(si, first, 4);
+ for (i = 0; i < 4; i++)
+ KUNIT_EXPECT_FALSE(test, swap_memcomp_transport_test(si, first + i));
+ KUNIT_EXPECT_TRUE(test, swap_memcomp_transport_test(si, first - 1));
+ KUNIT_EXPECT_TRUE(test, swap_memcomp_transport_test(si, first + 4));
+}
+
+static void memcomp_test_transport_bitmap_devices(struct kunit *test)
+{
+ struct swap_info_struct *a, *b;
+
+ a = mc16_test_transport_info(test, BITS_PER_LONG + 1);
+ KUNIT_ASSERT_NOT_NULL(test, a);
+ b = mc16_test_transport_info(test, BITS_PER_LONG + 1);
+ KUNIT_ASSERT_NOT_NULL(test, b);
+ swap_memcomp_transport_set(a, BITS_PER_LONG);
+ KUNIT_EXPECT_TRUE(test, swap_memcomp_transport_test(a, BITS_PER_LONG));
+ KUNIT_EXPECT_FALSE(test, swap_memcomp_transport_test(b, BITS_PER_LONG));
+ swap_memcomp_transport_set(b, BITS_PER_LONG);
+ swap_memcomp_transport_clear(a, BITS_PER_LONG, 1);
+ KUNIT_EXPECT_FALSE(test, swap_memcomp_transport_test(a, BITS_PER_LONG));
+ KUNIT_EXPECT_TRUE(test, swap_memcomp_transport_test(b, BITS_PER_LONG));
+}
+
+static void memcomp_test_transport_bitmap_budget(struct kunit *test)
+{
+ struct swap_info_struct *si;
+ unsigned long bytes;
+
+ /* 1 GiB / 4 KiB is 262144 offsets, independent of this kernel's PAGE_SIZE. */
+ si = mc16_test_transport_info(test, 1U << 18);
+ KUNIT_ASSERT_NOT_NULL(test, si);
+ bytes = BITS_TO_LONGS((unsigned long)si->max) * sizeof(unsigned long);
+ KUNIT_EXPECT_EQ(test, bytes, 32UL * 1024);
+ swap_memcomp_transport_set(si, si->max - 1);
+ KUNIT_EXPECT_TRUE(test, swap_memcomp_transport_test(si, si->max - 1));
+ KUNIT_EXPECT_FALSE(test, swap_memcomp_transport_test(si, si->max - 2));
+ swap_memcomp_transport_clear(si, si->max - 1, 1);
+ KUNIT_EXPECT_TRUE(test, bitmap_empty(si->memcomp_transport, si->max));
+}
+#endif /* CONFIG_SWAP */
+#endif /* _MM_MEMCOMP_TESTS_TRANSPORT_TEST_H */
diff --git a/mm/memcomp/tests/workingset_load_test.h b/mm/memcomp/tests/workingset_load_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..70d178f6727d56cbee4ae9b76eaa97b9a6f75fd4
--- /dev/null
+++ b/mm/memcomp/tests/workingset_load_test.h
@@ -0,0 +1,1095 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_WORKINGSET_LOAD_TEST_H
+#define _MM_MEMCOMP_TESTS_WORKINGSET_LOAD_TEST_H
+
+/*
+ * Author: nakixii
+ *
+ * Include after fault_test.h. Real MM, payload, charge and stock workingset.
+ * Required observation seam: at the start of mc16_load_page(), after local
+ * declarations, under CONFIG_MEMCOMP_KUNIT_TEST:
+ * KUNIT_STATIC_STUB_REDIRECT(mc16_load_page, ref, dst);
+ * The observer calls the original function, not a second implementation.
+ * Its hit count is asserted; omitting this seam cannot silently pass.
+ */
+#ifdef MC16_FAULT_MM_TEST_ENABLED
+#define MC16_WORKINGSET_LOAD_TEST_ENABLED
+
+#include <kunit/static_stub.h>
+#include <linux/cgroup.h>
+#include <linux/mm_inline.h>
+#include <linux/swap_slots.h>
+#include <linux/vmstat.h>
+#include "../../memcontrol-v1.h"
+
+#if defined(CONFIG_PSI) && defined(CONFIG_TASK_DELAY_ACCT)
+#define MC16_WORKINGSET_SCOPE_TEST_ENABLED
+#endif
+
+struct mc16_wl {
+ struct kunit *test;
+ struct mc16_ft *fault;
+ struct memcomp_ref refs[MC16_FT_PAGES];
+ struct folio *dst[2];
+ unsigned int nr;
+ unsigned int reads;
+ int read_error;
+ bool observe_scope;
+ unsigned int scope_reads;
+};
+
+/* Test-task-local static stub; KUnit never redirects another task's reads. */
+static struct mc16_wl *mc16_wl_current;
+
+static void mc16_wl_cleanup(void *data)
+{
+ struct mc16_wl *f = data;
+ unsigned int i;
+
+ mc16_wl_current = NULL;
+ for (i = 0; i < ARRAY_SIZE(f->dst); i++) {
+ if (!f->dst[i])
+ continue;
+ if (folio_test_locked(f->dst[i]))
+ folio_unlock(f->dst[i]);
+ folio_clear_active(f->dst[i]);
+ folio_put(f->dst[i]);
+ }
+ for (i = 0; i < f->nr; i++)
+ mc16_ref_put(&f->refs[i]);
+}
+
+static struct mc16_wl *mc16_wl_begin(struct kunit *test, unsigned int nr,
+ bool attached)
+{
+ struct mc16_wl *f;
+ unsigned int i;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->test = test;
+ f->fault = mc16_ft_begin(test, nr, false, !attached);
+ if (!f->fault)
+ return NULL;
+ if (kunit_add_action_or_reset(test, mc16_wl_cleanup, f))
+ return NULL;
+ for (i = 0; i < nr; i++) {
+ if (!mc16_get(f->fault->tokens[i], &f->refs[i]))
+ return NULL;
+ f->nr++;
+ }
+ return f;
+}
+
+static struct folio *mc16_wl_destination(struct mc16_wl *f, unsigned int index,
+ unsigned int order)
+{
+ struct folio *folio;
+ int nid = folio_nid(f->fault->refs->storage->owner->folio);
+
+ folio = __folio_alloc_node(GFP_KERNEL, order, nid);
+ if (!folio)
+ return NULL;
+ f->dst[index] = folio;
+ folio_lock(folio);
+ if (mc16_user_charge(&f->refs[0], folio, f->fault->mm, GFP_KERNEL))
+ return NULL;
+ __folio_set_swapbacked(folio);
+ return folio;
+}
+
+/* Produce a native shadow: never encode generation, memcg or distance bits. */
+static unsigned long mc16_wl_shadow(struct mc16_wl *f, bool hot)
+{
+ struct folio *source;
+ struct mem_cgroup *memcg;
+ unsigned long shadow = 0;
+ int nid = folio_nid(f->fault->refs->storage->owner->folio);
+
+ source = __folio_alloc_node(GFP_KERNEL, 0, nid);
+ if (!source)
+ return 0;
+ folio_lock(source);
+ if (mc16_user_charge(&f->refs[0], source, f->fault->mm, GFP_KERNEL))
+ goto out;
+ __folio_set_swapbacked(source);
+ if (hot) {
+ folio_set_workingset(source);
+#ifdef CONFIG_LRU_GEN
+ /* Model a frequently accessed source using stock's flag format. */
+ source->flags |= LRU_REFS_MASK;
+#endif
+ }
+ memcg = folio_memcg(source);
+ if (!memcg && !mem_cgroup_disabled())
+ memcg = root_mem_cgroup;
+ shadow = (unsigned long)workingset_memcomp_eviction(source, memcg);
+out:
+ folio_clear_workingset(source);
+#ifdef CONFIG_LRU_GEN
+ source->flags &= ~LRU_REFS_MASK;
+#endif
+ folio_unlock(source);
+ folio_put(source);
+ return shadow;
+}
+
+static void mc16_wl_set_shadow(struct memcomp_ref *ref, unsigned long shadow)
+{
+ mutex_lock(memcomp_lock_of(ref->token));
+ *mc16_shadow_ptr(ref) = shadow;
+ mutex_unlock(memcomp_lock_of(ref->token));
+}
+
+static u64 mc16_wl_get_shadow(struct memcomp_ref *ref)
+{
+ u64 shadow;
+
+ mutex_lock(memcomp_lock_of(ref->token));
+ shadow = *mc16_shadow_ptr(ref);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ return shadow;
+}
+
+/*
+ * Include pending per-CPU node deltas, not just the periodically folded
+ * counter. These cases run serially in the isolated KUnit guest.
+ */
+static long mc16_wl_refaults(void)
+{
+ long count = atomic_long_read(&vm_node_stat[WORKINGSET_REFAULT_ANON]);
+#ifdef CONFIG_SMP
+ int nid, cpu;
+
+ for_each_online_node(nid) {
+ struct pglist_data *pgdat = NODE_DATA(nid);
+
+ for_each_online_cpu(cpu)
+ count += READ_ONCE(per_cpu_ptr(pgdat->per_cpu_nodestats,
+ cpu)->vm_node_stat_diff[WORKINGSET_REFAULT_ANON]);
+ }
+#endif
+ return count;
+}
+
+/* Match handle_mm_fault()'s context without replacing stock's policy. */
+static vm_fault_t mc16_wl_fault(struct mc16_ft *f, unsigned int index)
+{
+ vm_fault_t ret;
+#ifdef CONFIG_LRU_GEN
+ bool saved = current->in_lru_fault;
+
+ current->in_lru_fault = true;
+#endif
+ ret = mc16_ft_fault(f->test, f->mm, f->addr + index * PAGE_SIZE,
+ mc16_ft_orig(f, index), FAULT_FLAG_WRITE);
+#ifdef CONFIG_LRU_GEN
+ current->in_lru_fault = saved;
+#endif
+ return ret;
+}
+
+#ifdef CONFIG_VM_EVENT_COUNTERS
+struct mc16_wl_major {
+ unsigned long global;
+ unsigned long memcg;
+ bool has_memcg;
+};
+
+static struct mc16_wl_major mc16_wl_major_read(struct mm_struct *mm)
+{
+ struct mc16_wl_major count = { };
+ struct mem_cgroup *memcg;
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ count.global += READ_ONCE(per_cpu(vm_event_states, cpu).event[PGMAJFAULT]);
+ if (mem_cgroup_disabled())
+ return count;
+ rcu_read_lock();
+ memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
+ if (memcg)
+ css_get(&memcg->css);
+ rcu_read_unlock();
+ if (memcg) {
+ /* The public rate-limited flush need not flush a single event. */
+ cgroup_rstat_flush(memcg->css.cgroup);
+ count.memcg = memcg_events(memcg, PGMAJFAULT);
+ count.has_memcg = true;
+ css_put(&memcg->css);
+ }
+ return count;
+}
+
+static void mc16_wl_major_expect(struct kunit *test, struct mm_struct *mm,
+ struct mc16_wl_major before, unsigned int nr)
+{
+ struct mc16_wl_major after = mc16_wl_major_read(mm);
+
+ KUNIT_EXPECT_EQ(test, after.global - before.global, (unsigned long)nr);
+ KUNIT_EXPECT_EQ(test, after.has_memcg, before.has_memcg);
+ if (before.has_memcg)
+ KUNIT_EXPECT_EQ(test, after.memcg - before.memcg, (unsigned long)nr);
+}
+#endif
+
+static int mc16_wl_read_page(struct memcomp_ref *ref, void *dst)
+{
+ struct mc16_wl *f = mc16_wl_current;
+ int ret;
+
+ f->reads++;
+#ifdef MC16_WORKINGSET_SCOPE_TEST_ENABLED
+ if (f->observe_scope) {
+ struct folio *folio = page_folio(kmap_to_page(dst));
+ bool workingset = folio_test_workingset(folio);
+
+ /* Match the native refault result, not the source's hot hint. */
+ KUNIT_EXPECT_EQ(f->test, (bool)current->in_memstall, workingset);
+ KUNIT_EXPECT_EQ(f->test, (bool)current->in_thrashing, workingset);
+ KUNIT_EXPECT_GT(f->test, current->delays->swapin_start, 0ULL);
+ if (workingset) {
+ KUNIT_EXPECT_GT(f->test, current->delays->thrashing_start, 0ULL);
+ f->scope_reads++;
+ }
+ if (lru_gen_enabled())
+ KUNIT_EXPECT_TRUE(f->test, workingset);
+ }
+#endif
+ kunit_deactivate_static_stub(f->test, mc16_load_page);
+ ret = mc16_load_page(ref, dst);
+ kunit_activate_static_stub(f->test, mc16_load_page, mc16_wl_read_page);
+ /* Make the real delayacct_end() positive even for a same-clock read. */
+ udelay(1);
+ return ret ? ret : f->read_error;
+}
+
+static void mc16_wl_observe(struct mc16_wl *f, int error, bool scope)
+{
+ f->read_error = error;
+ f->observe_scope = scope;
+ mc16_wl_current = f;
+ kunit_activate_static_stub(f->test, mc16_load_page, mc16_wl_read_page);
+}
+
+static void memcomp_test_workingset_load_charge_failure(struct kunit *test)
+{
+ struct mc16_wl *f = mc16_wl_begin(test, 1, false);
+ unsigned long shadow;
+ long before;
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ shadow = mc16_wl_shadow(f, true);
+ KUNIT_ASSERT_TRUE(test, xa_is_value((void *)shadow));
+ mc16_wl_set_shadow(&f->refs[0], shadow);
+ before = mc16_wl_refaults();
+ mc16_ft_probe_start(f->fault);
+ mc16_fault_probe.fail_small_charge = true;
+ ret = mc16_wl_fault(f->fault, 0);
+ WRITE_ONCE(mc16_fault_probe.task, NULL);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_OOM);
+ KUNIT_EXPECT_EQ(test, mc16_wl_get_shadow(&f->refs[0]), (u64)shadow);
+ KUNIT_EXPECT_EQ(test, mc16_wl_refaults() - before, 0L);
+ KUNIT_EXPECT_EQ(test, mc16_fault_probe.small_charges, 1U);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_ft_read(f->fault, 0),
+ mc16_ft_orig(f->fault, 0)));
+}
+
+static void memcomp_test_workingset_load_take_once(struct kunit *test)
+{
+ struct mc16_wl *f = mc16_wl_begin(test, 1, false);
+ struct folio *first, *second;
+ unsigned long shadow;
+ long before;
+ int a, b;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ first = mc16_wl_destination(f, 0, 0);
+ second = mc16_wl_destination(f, 1, 0);
+ KUNIT_ASSERT_NOT_NULL(test, first);
+ KUNIT_ASSERT_NOT_NULL(test, second);
+ shadow = mc16_wl_shadow(f, true);
+ KUNIT_ASSERT_TRUE(test, xa_is_value((void *)shadow));
+ mc16_wl_set_shadow(&f->refs[0], shadow);
+ before = mc16_wl_refaults();
+ a = mc16_user_refault(f->refs, 1, first);
+ KUNIT_EXPECT_EQ(test, a, 0);
+ KUNIT_EXPECT_EQ(test, mc16_wl_get_shadow(&f->refs[0]), 0ULL);
+ KUNIT_EXPECT_EQ(test, mc16_wl_refaults() - before, 1L);
+ b = mc16_user_refault(f->refs, 1, second);
+ KUNIT_EXPECT_EQ(test, b, 0);
+ KUNIT_EXPECT_EQ(test, mc16_wl_refaults() - before, 1L);
+ KUNIT_EXPECT_FALSE(test, folio_test_workingset(second));
+ KUNIT_EXPECT_FALSE(test, folio_test_active(second));
+ /* The counter above makes removing the real stock call observable. */
+ if (!mem_cgroup_disabled()) {
+ KUNIT_EXPECT_NE(test, f->refs[0].cluster->key->owner_id,
+ (unsigned short)0);
+ KUNIT_EXPECT_EQ(test, f->refs[0].cluster->key->owner_id,
+ mem_cgroup_id(root_mem_cgroup));
+ KUNIT_EXPECT_PTR_EQ(test, folio_memcg(first), root_mem_cgroup);
+ }
+}
+
+static void memcomp_test_workingset_load_ineligible(struct kunit *test)
+{
+ struct mc16_wl *f = mc16_wl_begin(test, MC16_FT_PAGES, true);
+ struct folio *dst;
+ struct mc16_view view;
+ unsigned long shadow;
+ unsigned int kind, i;
+ struct memcomp_ref *last;
+ u16 saved;
+ u16 tested;
+ long before;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ dst = mc16_wl_destination(f, 0, 2);
+ KUNIT_ASSERT_NOT_NULL(test, dst);
+ shadow = mc16_wl_shadow(f, true);
+ KUNIT_ASSERT_TRUE(test, xa_is_value((void *)shadow));
+ last = &f->refs[f->nr - 1];
+ for (kind = 0; kind < 4; kind++) {
+ int ret;
+ u64 expected = kind == 2 ? MC16_SHADOW_PUBLISHING : shadow;
+
+ for (i = 0; i < f->nr; i++)
+ mc16_wl_set_shadow(&f->refs[i], shadow);
+ mutex_lock(memcomp_lock_of(last->token));
+ mc16_view_locked(last, &view);
+ saved = *view.flags;
+ if (!kind)
+ *view.flags = (saved & ~MC16_STATE_MASK) | MC16_PENDING |
+ MC16_F_RESIDENT;
+ else if (kind == 1)
+ *view.flags |= MC16_F_RESIDENT;
+ else if (kind == 3)
+ *view.flags = (saved & ~(MC16_STATE_MASK |
+ MC16_F_RESIDENT)) | MC16_PENDING;
+ tested = *view.flags;
+ *mc16_shadow_ptr(last) = expected;
+ mutex_unlock(memcomp_lock_of(last->token));
+ before = mc16_wl_refaults();
+ ret = mc16_user_refault(f->refs, f->nr, dst);
+ mutex_lock(memcomp_lock_of(last->token));
+ mc16_view_locked(last, &view);
+ *view.flags = saved;
+ mutex_unlock(memcomp_lock_of(last->token));
+ if (kind == 3) {
+ kunit_info(test, "ineligible: PENDING without RESIDENT");
+ KUNIT_EXPECT_TRUE(test, mc16_flags_valid(tested));
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(tested),
+ (unsigned int)MC16_PENDING);
+ KUNIT_EXPECT_FALSE(test, tested & MC16_F_RESIDENT);
+ KUNIT_EXPECT_NE(test, expected,
+ (u64)MC16_SHADOW_PUBLISHING);
+ }
+ KUNIT_EXPECT_EQ(test, ret, -EAGAIN);
+ KUNIT_EXPECT_EQ(test, mc16_wl_get_shadow(last), expected);
+ for (i = 0; i + 1 < f->nr; i++)
+ KUNIT_EXPECT_EQ(test, mc16_wl_get_shadow(&f->refs[i]),
+ (u64)shadow);
+ KUNIT_EXPECT_EQ(test, mc16_wl_refaults() - before, 0L);
+ }
+ for (i = 0; i < f->nr; i++)
+ mc16_wl_set_shadow(&f->refs[i], 0);
+}
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+#define MC16_WORKINGSET_MTHP_TEST_ENABLED
+static const unsigned int mc16_wl_batch_cases[] = { 0, 1, 2, 3 };
+
+static void mc16_wl_batch_desc(const unsigned int *kind, char *desc)
+{
+ static const char * const names[] = {
+ "same", "empty-first", "different", "part-consumed",
+ };
+
+ strscpy(desc, names[*kind], KUNIT_PARAM_DESC_SIZE);
+}
+
+KUNIT_ARRAY_PARAM(mc16_wl_batch, mc16_wl_batch_cases, mc16_wl_batch_desc);
+
+static void memcomp_test_workingset_load_mthp(struct kunit *test)
+{
+ unsigned int kind = *(const unsigned int *)test->param_value;
+ struct mc16_wl *f = mc16_wl_begin(test, MC16_FT_PAGES, true);
+ unsigned long hot, cold;
+ unsigned int i;
+ long before;
+ vm_fault_t ret;
+#ifdef CONFIG_VM_EVENT_COUNTERS
+ struct mc16_wl_major major;
+#endif
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ /*
+ * Unlike the old convenience helper, this case must not silently skip
+ * a configured order-2 guest. Keep restoration in the existing fixture.
+ */
+ f->fault->orders = READ_ONCE(huge_anon_orders_always);
+ f->fault->saved_orders = true;
+ WRITE_ONCE(huge_anon_orders_always, f->fault->orders | BIT(2));
+ mc16_ft_policy(f->fault, VM_HUGEPAGE, VM_NOHUGEPAGE);
+ KUNIT_ASSERT_NE(test, THP_ORDERS_ALL_ANON & BIT(2), 0UL);
+ hot = mc16_wl_shadow(f, true);
+ cold = mc16_wl_shadow(f, false);
+ KUNIT_ASSERT_TRUE(test, xa_is_value((void *)hot));
+ KUNIT_ASSERT_TRUE(test, xa_is_value((void *)cold));
+ KUNIT_ASSERT_NE(test, hot, cold);
+ for (i = 0; i < f->nr; i++) {
+ unsigned long shadow = hot;
+
+ if (kind == 1 && !i)
+ shadow = 0;
+ if (kind == 2 && i)
+ shadow = cold;
+ if (kind == 3 && (i & 1))
+ shadow = 0;
+ mc16_wl_set_shadow(&f->refs[i], shadow);
+ }
+ before = mc16_wl_refaults();
+#ifdef CONFIG_VM_EVENT_COUNTERS
+ major = mc16_wl_major_read(f->fault->mm);
+#endif
+ mc16_ft_probe_start(f->fault);
+ /* Middle PTE: only the span's first shadow is authoritative. */
+ ret = mc16_wl_fault(f->fault, 2);
+ WRITE_ONCE(mc16_fault_probe.task, NULL);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)(VM_FAULT_NOPAGE | VM_FAULT_MAJOR));
+ KUNIT_EXPECT_EQ(test, mc16_fault_probe.large_allocs, 1U);
+ KUNIT_EXPECT_EQ(test, mc16_fault_probe.small_allocs, 0U);
+ KUNIT_EXPECT_EQ(test, mc16_wl_refaults() - before,
+ kind == 1 ? 0L : (long)MC16_FT_PAGES);
+ for (i = 0; i < f->nr; i++) {
+ KUNIT_EXPECT_EQ(test, mc16_wl_get_shadow(&f->refs[i]), 0ULL);
+ mc16_ft_present(f->fault, i, 2, MC16_FT_PAGES);
+ }
+ mc16_ft_counts(f->fault, 0, MC16_FT_PAGES);
+#ifdef CONFIG_VM_EVENT_COUNTERS
+ mc16_wl_major_expect(test, f->fault->mm, major, 1);
+#endif
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+
+static void memcomp_test_workingset_load_read_error(struct kunit *test)
+{
+ struct mc16_wl *f = mc16_wl_begin(test, 1, false);
+ unsigned long shadow;
+ long before;
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ shadow = mc16_wl_shadow(f, true);
+ KUNIT_ASSERT_TRUE(test, xa_is_value((void *)shadow));
+ mc16_wl_set_shadow(&f->refs[0], shadow);
+ mc16_wl_observe(f, -EIO, false);
+ before = mc16_wl_refaults();
+ ret = mc16_wl_fault(f->fault, 0);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_SIGBUS);
+ KUNIT_EXPECT_EQ(test, f->reads, 1U);
+ KUNIT_EXPECT_EQ(test, mc16_wl_get_shadow(&f->refs[0]), 0ULL);
+ KUNIT_EXPECT_EQ(test, mc16_wl_refaults() - before, 1L);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_ft_read(f->fault, 0),
+ mc16_ft_orig(f->fault, 0)));
+ /* A retry succeeds but must not recreate or consume another shadow. */
+ f->read_error = 0;
+ ret = mc16_wl_fault(f->fault, 0);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)(VM_FAULT_NOPAGE | VM_FAULT_MAJOR));
+ KUNIT_EXPECT_EQ(test, f->reads, 2U);
+ KUNIT_EXPECT_EQ(test, mc16_wl_refaults() - before, 1L);
+}
+
+static void memcomp_test_workingset_load_pte_race(struct kunit *test)
+{
+ struct mc16_wl *f = mc16_wl_begin(test, 1, false);
+ unsigned long shadow;
+ long before;
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ shadow = mc16_wl_shadow(f, true);
+ KUNIT_ASSERT_TRUE(test, xa_is_value((void *)shadow));
+ mc16_wl_set_shadow(&f->refs[0], shadow);
+ before = mc16_wl_refaults();
+ mc16_ft_probe_start(f->fault);
+ mc16_fault_probe.before_install = mc16_ft_race;
+ ret = mc16_wl_fault(f->fault, 0);
+ WRITE_ONCE(mc16_fault_probe.task, NULL);
+ KUNIT_EXPECT_TRUE(test, f->fault->actor_ran);
+ KUNIT_EXPECT_EQ(test, f->fault->actor_error, 0);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_MAJOR);
+ KUNIT_EXPECT_EQ(test, mc16_wl_get_shadow(&f->refs[0]), 0ULL);
+ KUNIT_EXPECT_EQ(test, mc16_wl_refaults() - before, 1L);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_ft_read(f->fault, 0),
+ pte_swp_mkexclusive(mc16_ft_orig(f->fault, 0))));
+}
+
+#ifdef CONFIG_VM_EVENT_COUNTERS
+#define MC16_WORKINGSET_MAJOR_TEST_ENABLED
+static void memcomp_test_workingset_load_major_shared(struct kunit *test)
+{
+ struct mc16_wl *f = mc16_wl_begin(test, 1, false);
+ struct mc16_wl_major major;
+ unsigned long shadow;
+ long before;
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_ft_install(f->fault, 1,
+ mc16_ft_orig(f->fault, 0)), 0);
+ shadow = mc16_wl_shadow(f, true);
+ KUNIT_ASSERT_TRUE(test, xa_is_value((void *)shadow));
+ mc16_wl_set_shadow(&f->refs[0], shadow);
+ major = mc16_wl_major_read(f->fault->mm);
+ before = mc16_wl_refaults();
+ ret = mc16_wl_fault(f->fault, 0);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)(VM_FAULT_NOPAGE | VM_FAULT_MAJOR));
+ mc16_wl_major_expect(test, f->fault->mm, major, 1);
+ /*
+ * Index 1 owns the same token, not tokens[1]. The second actual
+ * materialization is another major, but not another stock refault.
+ */
+ ret = mc16_ft_fault(test, f->fault->mm, f->fault->addr + PAGE_SIZE,
+ mc16_ft_orig(f->fault, 0), FAULT_FLAG_WRITE);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)(VM_FAULT_NOPAGE | VM_FAULT_MAJOR));
+ mc16_wl_major_expect(test, f->fault->mm, major, 2);
+ KUNIT_EXPECT_EQ(test, mc16_wl_refaults() - before, 1L);
+ KUNIT_EXPECT_EQ(test, mc16_wl_get_shadow(&f->refs[0]), 0ULL);
+ mc16_ft_counts(f->fault, 0, 2);
+}
+
+static void memcomp_test_workingset_load_zero_not_major(struct kunit *test)
+{
+ struct mc16_zl_mm *f = mc16_zl_begin(test);
+ struct mc16_wl_major major;
+ pte_t orig = mc16_zl_zero_pte();
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_zl_install(f, orig), 0);
+ major = mc16_wl_major_read(f->mm);
+ ret = mc16_ft_fault(test, f->mm, f->addr, orig, FAULT_FLAG_WRITE);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_NOPAGE);
+ mc16_wl_major_expect(test, f->mm, major, 0);
+ mc16_zl_present(f, mc16_zl_read(f));
+}
+
+#ifdef MC16_RESIDENT_TEST_ENABLED
+#define MC16_WORKINGSET_RESIDENT_MAJOR_TEST_ENABLED
+static void memcomp_test_workingset_load_resident_not_major(struct kunit *test)
+{
+ struct mc16_rl *f = mc16_rl_begin(test, 1, 0);
+ struct mc16_wl_major major;
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ major = mc16_wl_major_read(f->mm);
+ /* Use the complete dispatcher, not the resident helper in isolation. */
+ ret = mc16_ft_fault(test, f->mm, f->addr, f->orig, FAULT_FLAG_WRITE);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_NOPAGE);
+ mc16_wl_major_expect(test, f->mm, major, 0);
+ KUNIT_EXPECT_TRUE(test, pte_present(mc16_rl_read(f)));
+}
+#endif
+#endif /* CONFIG_VM_EVENT_COUNTERS */
+
+#ifdef MC16_WORKINGSET_SCOPE_TEST_ENABLED
+struct mc16_wl_delays {
+ u32 swapin;
+ u32 thrashing;
+};
+
+static struct mc16_wl_delays mc16_wl_delays_read(void)
+{
+ return (struct mc16_wl_delays) {
+ .swapin = current->delays->swapin_count,
+ .thrashing = current->delays->thrashing_count,
+ };
+}
+
+static bool mc16_wl_scope_ready(struct kunit *test)
+{
+ bool ready = !static_branch_unlikely(&psi_disabled) &&
+ static_branch_unlikely(&delayacct_key) &&
+ current->delays && !current->in_memstall &&
+ !current->in_thrashing;
+
+ /* Missing boot options are a failed precondition, not a skipped case. */
+ KUNIT_EXPECT_TRUE_MSG(test, ready, "requires PSI and boot delayacct");
+ return ready;
+}
+
+static void mc16_wl_scope_expect(struct kunit *test,
+ struct mc16_wl_delays before, unsigned int nr,
+ unsigned int thrashing)
+{
+ unsigned long pflags = 0;
+ bool in_thrashing = false;
+
+ KUNIT_EXPECT_FALSE(test, current->in_memstall);
+ KUNIT_EXPECT_FALSE(test, current->in_thrashing);
+ KUNIT_EXPECT_EQ(test, current->delays->swapin_count - before.swapin, nr);
+ KUNIT_EXPECT_EQ(test, current->delays->thrashing_count - before.thrashing,
+ thrashing);
+ /* Do not let a failed end-scope mutation affect later cases. */
+ if (current->in_memstall)
+ psi_memstall_leave(&pflags);
+ if (current->in_thrashing)
+ delayacct_thrashing_end(&in_thrashing);
+}
+
+static const unsigned int mc16_wl_scope_cases[] = { 0, 1, 2 };
+
+static void mc16_wl_scope_desc(const unsigned int *kind, char *desc)
+{
+ static const char * const names[] = { "success", "read-error", "pte-race" };
+
+ strscpy(desc, names[*kind], KUNIT_PARAM_DESC_SIZE);
+}
+
+KUNIT_ARRAY_PARAM(mc16_wl_scope, mc16_wl_scope_cases, mc16_wl_scope_desc);
+
+static void memcomp_test_workingset_load_scope(struct kunit *test)
+{
+ unsigned int kind = *(const unsigned int *)test->param_value;
+ struct mc16_wl *f;
+ struct mc16_wl_delays before;
+ unsigned long shadow;
+ vm_fault_t ret;
+#ifdef CONFIG_VM_EVENT_COUNTERS
+ struct mc16_wl_major major;
+#endif
+
+ if (!mc16_wl_scope_ready(test))
+ return;
+ f = mc16_wl_begin(test, 1, false);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ shadow = mc16_wl_shadow(f, true);
+ KUNIT_ASSERT_TRUE(test, xa_is_value((void *)shadow));
+ mc16_wl_set_shadow(&f->refs[0], shadow);
+ mc16_wl_observe(f, kind == 1 ? -EIO : 0, true);
+ if (kind == 2) {
+ mc16_ft_probe_start(f->fault);
+ mc16_fault_probe.before_install = mc16_ft_race;
+ }
+ before = mc16_wl_delays_read();
+#ifdef CONFIG_VM_EVENT_COUNTERS
+ major = mc16_wl_major_read(f->fault->mm);
+#endif
+ ret = mc16_wl_fault(f->fault, 0);
+ WRITE_ONCE(mc16_fault_probe.task, NULL);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)(kind == 1 ? VM_FAULT_SIGBUS :
+ kind == 2 ? VM_FAULT_MAJOR :
+ VM_FAULT_NOPAGE | VM_FAULT_MAJOR));
+ KUNIT_EXPECT_EQ(test, f->reads, 1U);
+ mc16_wl_scope_expect(test, before, 1, f->scope_reads);
+ KUNIT_EXPECT_EQ(test, mc16_wl_get_shadow(&f->refs[0]), 0ULL);
+#ifdef CONFIG_VM_EVENT_COUNTERS
+ mc16_wl_major_expect(test, f->fault->mm, major, kind == 1 ? 0 : 1);
+#endif
+}
+
+static const unsigned int mc16_wl_cache_cases[] = { 0, 1 };
+
+KUNIT_ARRAY_PARAM(mc16_wl_cache, mc16_wl_cache_cases, mc16_wl_scope_desc);
+
+static void memcomp_test_workingset_load_cache_scope(struct kunit *test)
+{
+ unsigned int error = *(const unsigned int *)test->param_value;
+ struct memcomp_ref guard = { };
+ struct mc16_wl *f;
+ struct mc16_wl_delays before;
+ struct folio *folio;
+ unsigned long shadow;
+ long refaults;
+ int ret;
+#ifdef CONFIG_LRU_GEN
+ bool saved_in_fault;
+#endif
+#ifdef CONFIG_VM_EVENT_COUNTERS
+ struct mc16_wl_major major;
+#endif
+
+ if (!mc16_wl_scope_ready(test))
+ return;
+ f = mc16_wl_begin(test, 1, false);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ folio = mc16_wl_destination(f, 0, 0);
+ KUNIT_ASSERT_NOT_NULL(test, folio);
+ shadow = mc16_wl_shadow(f, true);
+ KUNIT_ASSERT_TRUE(test, xa_is_value((void *)shadow));
+ mc16_wl_set_shadow(&f->refs[0], shadow);
+ mc16_wl_observe(f, error ? -EIO : 0, true);
+ KUNIT_ASSERT_TRUE(test, mc16_get(f->refs[0].token, &guard));
+ before = mc16_wl_delays_read();
+ refaults = mc16_wl_refaults();
+#ifdef CONFIG_VM_EVENT_COUNTERS
+ major = mc16_wl_major_read(f->fault->mm);
+#endif
+#ifdef CONFIG_LRU_GEN
+ saved_in_fault = current->in_lru_fault;
+ current->in_lru_fault = true;
+#endif
+ ret = mc16_cache_load(&guard, folio);
+#ifdef CONFIG_LRU_GEN
+ current->in_lru_fault = saved_in_fault;
+#endif
+ /* Model losing the cache replacement using the real guard abort. */
+ mc16_cache_abort(&guard);
+ KUNIT_EXPECT_EQ(test, ret, error ? -EIO : 0);
+ KUNIT_EXPECT_EQ(test, f->reads, 1U);
+ KUNIT_EXPECT_EQ(test, folio_test_uptodate(folio), !error);
+ KUNIT_EXPECT_EQ(test, mc16_wl_refaults() - refaults, 1L);
+ KUNIT_EXPECT_EQ(test, mc16_wl_get_shadow(&f->refs[0]), 0ULL);
+ mc16_wl_scope_expect(test, before, 1, f->scope_reads);
+ KUNIT_EXPECT_PTR_EQ(test, guard.slot, NULL);
+#ifdef CONFIG_VM_EVENT_COUNTERS
+ mc16_wl_major_expect(test, f->fault->mm, major, 0);
+#endif
+}
+
+static void memcomp_test_workingset_load_scope_cold(struct kunit *test)
+{
+ struct mc16_wl *f;
+ struct folio *folio;
+ struct mc16_user_read scope;
+ struct mc16_wl_delays before;
+
+ if (!mc16_wl_scope_ready(test))
+ return;
+ f = mc16_wl_begin(test, 1, false);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ folio = mc16_wl_destination(f, 0, 0);
+ KUNIT_ASSERT_NOT_NULL(test, folio);
+ KUNIT_ASSERT_FALSE(test, folio_test_workingset(folio));
+ before = mc16_wl_delays_read();
+ mc16_user_read_start(&scope, folio);
+ KUNIT_EXPECT_FALSE(test, scope.workingset);
+ KUNIT_EXPECT_FALSE(test, current->in_memstall);
+ KUNIT_EXPECT_FALSE(test, current->in_thrashing);
+ udelay(1);
+ mc16_user_read_end(&scope);
+ KUNIT_EXPECT_FALSE(test, current->in_memstall);
+ KUNIT_EXPECT_FALSE(test, current->in_thrashing);
+ KUNIT_EXPECT_EQ(test, current->delays->swapin_count - before.swapin, 1U);
+ KUNIT_EXPECT_EQ(test, current->delays->thrashing_count - before.thrashing, 0U);
+}
+
+/* Deterministic scope-on control, independent of native recency policy. */
+static void memcomp_test_workingset_load_scope_hot(struct kunit *test)
+{
+ struct mc16_wl *f;
+ struct folio *folio;
+ struct mc16_user_read scope;
+ struct mc16_wl_delays before;
+ u64 previous_swapin_start, this_swapin_start;
+
+ if (!mc16_wl_scope_ready(test))
+ return;
+ f = mc16_wl_begin(test, 1, false);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ folio = mc16_wl_destination(f, 0, 0);
+ KUNIT_ASSERT_NOT_NULL(test, folio);
+ folio_set_workingset(folio);
+ /*
+ * Leave a real, nonzero old timestamp on this CPU. The tested start
+ * must replace it; a later end can increment count even without start.
+ * Keep the warmup outside the existing count snapshot.
+ */
+ preempt_disable();
+ delayacct_swapin_start();
+ previous_swapin_start = current->delays->swapin_start;
+ udelay(1);
+ delayacct_swapin_end();
+ before = mc16_wl_delays_read();
+ mc16_user_read_start(&scope, folio);
+ this_swapin_start = current->delays->swapin_start;
+ preempt_enable();
+ KUNIT_EXPECT_GT(test, previous_swapin_start, 0ULL);
+ KUNIT_EXPECT_GT(test, this_swapin_start, previous_swapin_start);
+ KUNIT_EXPECT_TRUE(test, scope.workingset);
+ KUNIT_EXPECT_TRUE(test, current->in_memstall);
+ KUNIT_EXPECT_TRUE(test, current->in_thrashing);
+ KUNIT_EXPECT_GT(test, current->delays->swapin_start, 0ULL);
+ KUNIT_EXPECT_GT(test, current->delays->thrashing_start, 0ULL);
+ /*
+ * Repair a missing PSI entry only after the start expectations.
+ * Keep the real end path paired and its exit expectations intact.
+ */
+ if (scope.workingset && !current->in_memstall)
+ psi_memstall_enter(&scope.pflags);
+ udelay(1);
+ mc16_user_read_end(&scope);
+ mc16_wl_scope_expect(test, before, 1, 1);
+}
+
+#ifdef CONFIG_SWAP
+#define MC16_WORKINGSET_TRANSPORT_TEST_ENABLED
+struct mc16_wl_transport {
+ struct folio *folio;
+ swp_entry_t swap;
+ bool real;
+};
+
+static void mc16_wl_transport_cleanup(void *data)
+{
+ struct mc16_wl_transport *t = data;
+
+ if (t->folio) {
+ folio_lock(t->folio);
+ memcomp_drop_swapcache_folio(t->folio, false);
+ }
+ if (t->swap.val)
+ memcomp_slot_put(t->swap);
+ if (t->real)
+ WRITE_ONCE(mc16_kunit_real_swap, false);
+}
+
+#include "transport_reclaim_test.h"
+
+static void memcomp_test_workingset_load_transport(struct kunit *test)
+{
+ struct mc16_wl_transport *t;
+ struct mc16_wl *f;
+ struct folio *user;
+ struct mc16_user_read scope;
+ struct mc16_wl_delays before;
+ struct writeback_control wbc = { .sync_mode = WB_SYNC_ALL };
+ struct {
+ swp_entry_t entry;
+ bool marked;
+ struct swap_info_struct *si;
+ unsigned char reserved_map, returned_map;
+ bool checked, returned_marked;
+ } prefetch[MEMCOMP_WRITEBACK_BATCH];
+ unsigned int prefetch_nr, i;
+ unsigned long shadow;
+ long refaults;
+ void *src, *dst;
+ bool no_swap;
+ int ret;
+#ifdef CONFIG_VM_EVENT_COUNTERS
+ struct mc16_wl_major major;
+#endif
+
+ if (!mc16_wl_scope_ready(test))
+ return;
+ f = mc16_wl_begin(test, 1, false);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ user = mc16_wl_destination(f, 0, 0);
+ KUNIT_ASSERT_NOT_NULL(test, user);
+ t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, t);
+ KUNIT_ASSERT_FALSE(test, READ_ONCE(mc16_kunit_real_swap));
+ KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test,
+ mc16_wl_transport_cleanup, t), 0);
+ mutex_lock(&memcomp_writeback_lock);
+ if (memcomp_writeback_slots_free) {
+ mutex_unlock(&memcomp_writeback_lock);
+ KUNIT_FAIL(test, "transport test requires an idle slot pool");
+ return;
+ }
+ t->real = true;
+ WRITE_ONCE(mc16_kunit_real_swap, true);
+ no_swap = get_nr_swap_pages() <= 0;
+ ret = memcomp_swap_folio_alloc(&t->folio, &t->swap);
+ /* W keeps every unused reservation live until the native return. */
+ prefetch_nr = memcomp_writeback_slots_free;
+ for (i = 0; i < prefetch_nr && i < ARRAY_SIZE(prefetch); i++) {
+ swp_entry_t entry = memcomp_writeback_slot_pool[i];
+
+ prefetch[i].entry = entry;
+ prefetch[i].marked =
+ swap_memcomp_transport_test(swp_swap_info(entry), swp_offset(entry));
+ /* Keep the bitmap/map alive across this reservation's return. */
+ prefetch[i].si = get_swap_device(entry);
+ prefetch[i].checked = prefetch[i].si;
+ prefetch[i].reserved_map = prefetch[i].si ?
+ READ_ONCE(prefetch[i].si->swap_map[swp_offset(entry)]) : 0;
+ prefetch[i].returned_map = SWAP_HAS_CACHE;
+ prefetch[i].returned_marked = true;
+ }
+ memcomp_writeback_slots_return();
+ /* W excludes another memcomp take until these snapshots finish. */
+ for (i = 0; i < prefetch_nr && i < ARRAY_SIZE(prefetch); i++) {
+ struct swap_info_struct *si = prefetch[i].si;
+ unsigned long offset = swp_offset(prefetch[i].entry);
+
+ if (!si)
+ continue;
+ spin_lock(&si->lock);
+ prefetch[i].returned_map = READ_ONCE(si->swap_map[offset]);
+ prefetch[i].returned_marked =
+ swap_memcomp_transport_test(si, offset);
+ spin_unlock(&si->lock);
+ put_swap_device(si);
+ prefetch[i].si = NULL;
+ }
+ mutex_unlock(&memcomp_writeback_lock);
+ /* alloc returns locked; cleanup always takes its own lock. */
+ if (t->folio)
+ folio_unlock(t->folio);
+ if (no_swap) {
+ KUNIT_EXPECT_EQ(test, ret, -ENOSPC);
+ KUNIT_EXPECT_PTR_EQ(test, t->folio, NULL);
+ KUNIT_EXPECT_EQ(test, t->swap.val, 0UL);
+ kunit_info(test, "ENOSPC checked; rerun with swap for native read");
+ return;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ KUNIT_ASSERT_NOT_NULL(test, t->folio);
+ /* Only copied entries/bits are read after W and the unused slots go. */
+ kunit_info(test, "transport prefetch: taken=%#lx batch=%u unused=%u",
+ t->swap.val, prefetch_nr + 1, prefetch_nr);
+ KUNIT_EXPECT_GT(test, prefetch_nr, 0U);
+ KUNIT_EXPECT_LT(test, prefetch_nr,
+ (unsigned int)ARRAY_SIZE(prefetch));
+ for (i = 0; i < prefetch_nr && i < ARRAY_SIZE(prefetch); i++) {
+ swp_entry_t entry = prefetch[i].entry;
+
+ kunit_info(test,
+ "transport prefetch: index=%u entry=%#lx type=%u offset=%lu marked=%d",
+ i, entry.val, swp_type(entry), swp_offset(entry),
+ prefetch[i].marked);
+ KUNIT_EXPECT_TRUE(test, prefetch[i].marked);
+ kunit_info(test,
+ "transport unused return: index=%u entry=%#lx pinned=%d map=%u->%u marked=%d->%d",
+ i, entry.val, prefetch[i].checked,
+ prefetch[i].reserved_map, prefetch[i].returned_map,
+ prefetch[i].marked, prefetch[i].returned_marked);
+ KUNIT_EXPECT_TRUE(test, prefetch[i].checked);
+ KUNIT_EXPECT_EQ(test, prefetch[i].reserved_map,
+ (unsigned char)SWAP_HAS_CACHE);
+ KUNIT_EXPECT_EQ(test, prefetch[i].returned_map, (unsigned char)0);
+ KUNIT_EXPECT_FALSE(test, prefetch[i].returned_marked);
+ }
+ folio_lock(t->folio);
+ {
+ bool marked = swap_memcomp_transport_test(swp_swap_info(t->swap),
+ swp_offset(t->swap));
+
+ kunit_info(test, "transport bitmap: allocated entry=%#lx marked=%d",
+ t->swap.val, marked);
+ KUNIT_EXPECT_TRUE(test, marked);
+ }
+ src = kmap_local_folio(t->folio, 0);
+ memcpy(src, f->fault->src, PAGE_SIZE);
+ kunmap_local(src);
+ __folio_mark_uptodate(t->folio);
+ /* Normal stock write also handles zeromap and zswap consistently. */
+ ret = swap_writepage(&t->folio->page, &wbc);
+ if (ret && folio_test_locked(t->folio))
+ folio_unlock(t->folio);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ folio_lock(t->folio);
+ folio_wait_writeback(t->folio);
+ KUNIT_EXPECT_FALSE(test, folio_test_dirty(t->folio));
+ memcomp_drop_swapcache_folio(t->folio, false);
+ t->folio = NULL;
+ shadow = mc16_wl_shadow(f, true);
+ KUNIT_ASSERT_TRUE(test, xa_is_value((void *)shadow));
+ mc16_wl_set_shadow(&f->refs[0], shadow);
+ KUNIT_ASSERT_EQ(test, mc16_user_refault(f->refs, 1, user), 0);
+ KUNIT_ASSERT_TRUE(test, folio_test_workingset(user));
+ before = mc16_wl_delays_read();
+ refaults = mc16_wl_refaults();
+#ifdef CONFIG_VM_EVENT_COUNTERS
+ major = mc16_wl_major_read(f->fault->mm);
+#endif
+ mc16_user_read_start(&scope, user);
+ dst = kmap_local_folio(user, 0);
+ ret = memcomp_read_swapped_page(t->swap, false, 0, 0, dst, GFP_KERNEL);
+ if (!ret)
+ KUNIT_EXPECT_EQ(test, memcmp(dst, f->fault->src, PAGE_SIZE), 0);
+ kunmap_local(dst);
+ udelay(1);
+ mc16_user_read_end(&scope);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ mc16_wl_scope_expect(test, before, 1, 1);
+ /* Native transport adds neither user refault nor major nor delay scope. */
+ KUNIT_EXPECT_EQ(test, mc16_wl_refaults() - refaults, 0L);
+#ifdef CONFIG_VM_EVENT_COUNTERS
+ mc16_wl_major_expect(test, f->fault->mm, major, 0);
+#endif
+ {
+ struct swap_info_struct *si;
+ struct folio *cached;
+ swp_entry_t released = t->swap, ordinary = { };
+ unsigned long offset = swp_offset(released);
+ unsigned char before_map = 0, after_map = SWAP_HAS_CACHE;
+ unsigned char ordinary_map = 0;
+ bool after_marked = true, ordinary_marked = false;
+ bool same_device = false;
+ int nr = 0;
+
+ /*
+ * A single-slot put can leave SWAP_HAS_CACHE in a return cache.
+ * Use the native drain/disable pair so the final put reaches
+ * swap_range_free(). Never clear the transport bit here.
+ */
+ disable_swap_slots_cache_lock();
+ si = get_swap_device(released);
+ if (si) {
+ cached = swap_cache_get_folio(released, NULL, 0);
+ if (cached) {
+ folio_lock(cached);
+ memcomp_drop_swapcache_folio(cached, false);
+ }
+ before_map = READ_ONCE(si->swap_map[offset]);
+ memcomp_slot_put(released);
+ t->swap.val = 0;
+ after_map = READ_ONCE(si->swap_map[offset]);
+ after_marked = swap_memcomp_transport_test(si, offset);
+
+ /* Ordinary reservation, with no memcomp tagging or cursor edits. */
+ nr = get_swap_pages(1, &ordinary, 0);
+ if (nr) {
+ same_device = swp_type(ordinary) == si->type;
+ if (same_device) {
+ ordinary_map =
+ READ_ONCE(si->swap_map[swp_offset(ordinary)]);
+ ordinary_marked =
+ swap_memcomp_transport_test(si, swp_offset(ordinary));
+ }
+ swapcache_free_entries(&ordinary, 1);
+ }
+ put_swap_device(si);
+ }
+ reenable_swap_slots_cache_unlock();
+ /* No assertion or early return while holding the cache/device pins. */
+ KUNIT_ASSERT_NOT_NULL(test, si);
+ kunit_info(test,
+ "transport bitmap: freed=%#lx map=%u->%u marked=%d",
+ released.val, before_map, after_map, after_marked);
+ KUNIT_EXPECT_EQ(test, before_map, (unsigned char)1);
+ KUNIT_EXPECT_EQ(test, after_map, (unsigned char)0);
+ KUNIT_EXPECT_FALSE(test, after_marked);
+ kunit_info(test,
+ "transport bitmap: ordinary=%#lx same_slot=%d map=%u marked=%d",
+ ordinary.val, nr == 1 && ordinary.val == released.val,
+ ordinary_map, ordinary_marked);
+ KUNIT_EXPECT_EQ(test, nr, 1);
+ KUNIT_EXPECT_TRUE(test, same_device);
+ KUNIT_EXPECT_EQ(test, ordinary_map, (unsigned char)SWAP_HAS_CACHE);
+ KUNIT_EXPECT_FALSE(test, ordinary_marked);
+ }
+ mc16_tr_one(test, f->fault->src, true);
+ mc16_tr_one(test, f->fault->src, false);
+}
+#endif /* CONFIG_SWAP */
+#endif /* MC16_WORKINGSET_SCOPE_TEST_ENABLED */
+#endif /* MC16_FAULT_MM_TEST_ENABLED */
+#endif /* _MM_MEMCOMP_TESTS_WORKINGSET_LOAD_TEST_H */
diff --git a/mm/memcomp/tests/workingset_mthp_error_test.h b/mm/memcomp/tests/workingset_mthp_error_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..e3045e7de1760348bf6a350f001f0a2b2d5c6953
--- /dev/null
+++ b/mm/memcomp/tests/workingset_mthp_error_test.h
@@ -0,0 +1,164 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_WORKINGSET_MTHP_ERROR_TEST_H
+#define _MM_MEMCOMP_TESTS_WORKINGSET_MTHP_ERROR_TEST_H
+
+/* Include after workingset_load_test.h; reuse its real MM/load fixture. */
+#ifdef MC16_WORKINGSET_MTHP_TEST_ENABLED
+struct mc16_wl_mthp_error_read {
+ unsigned long token;
+ unsigned int order, pages;
+ int result;
+ bool locked, swapbacked, workingset, cleared, contents;
+};
+
+struct mc16_wl_mthp_error {
+ struct mc16_wl *wl;
+ struct mc16_wl_mthp_error_read read[1 + MC16_FT_PAGES];
+ unsigned int nr;
+ bool fail;
+};
+
+static int mc16_wl_mthp_error_read_page(struct memcomp_ref *ref, void *dst)
+{
+ struct mc16_wl_mthp_error *o = mc16_fault_probe.data;
+ struct mc16_wl *f = o->wl;
+ struct mc16_wl_mthp_error_read *r = NULL;
+ struct folio *folio = page_folio(kmap_to_page(dst));
+ unsigned int n = o->nr++, i;
+ int ret;
+
+ /* Snapshot while the actual destination is still locked and alive. */
+ if (n < ARRAY_SIZE(o->read)) {
+ r = &o->read[n];
+ r->token = ref->token;
+ r->order = folio_order(folio);
+ r->pages = folio_nr_pages(folio);
+ r->locked = folio_test_locked(folio);
+ r->swapbacked = folio_test_swapbacked(folio);
+ r->workingset = folio_test_workingset(folio);
+ r->cleared = true;
+ for (i = 0; i < f->nr; i++)
+ r->cleared &= !mc16_wl_get_shadow(&f->refs[i]);
+ }
+ /* The existing observer calls the original mc16_load_page(). */
+ ret = mc16_wl_read_page(ref, dst);
+ if (r) {
+ r->result = ret;
+ for (i = 0; i < f->nr; i++) {
+ void *src = f->fault->src + i * PAGE_SIZE;
+
+ if (ref->token == f->fault->tokens[i])
+ r->contents = !memcmp(dst, src, PAGE_SIZE);
+ }
+ }
+ /* The old observer reinstalls itself; retain this observer on retry. */
+ kunit_activate_static_stub(f->test, mc16_load_page,
+ mc16_wl_mthp_error_read_page);
+ return ret ? ret : (o->fail ? -EIO : 0);
+}
+
+static const bool mc16_wl_mthp_error_cases[] = { false, true };
+
+static void mc16_wl_mthp_error_desc(const bool *empty, char *desc)
+{
+ strscpy(desc, *empty ? "empty-first-read-error" : "first-only-read-error",
+ KUNIT_PARAM_DESC_SIZE);
+}
+
+KUNIT_ARRAY_PARAM(mc16_wl_mthp_error, mc16_wl_mthp_error_cases,
+ mc16_wl_mthp_error_desc);
+
+static void memcomp_test_workingset_load_mthp_read_error(struct kunit *test)
+{
+ bool empty = *(const bool *)test->param_value;
+ struct mc16_wl *f = mc16_wl_begin(test, MC16_FT_PAGES, true);
+ struct mc16_wl_mthp_error *o;
+ unsigned long shadow;
+ unsigned int i;
+ vm_fault_t ret;
+ long before;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ o = kunit_kzalloc(test, sizeof(*o), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, o);
+ o->wl = f;
+ o->fail = true;
+ f->fault->orders = READ_ONCE(huge_anon_orders_always);
+ f->fault->saved_orders = true;
+ WRITE_ONCE(huge_anon_orders_always, f->fault->orders | BIT(2));
+ mc16_ft_policy(f->fault, VM_HUGEPAGE, VM_NOHUGEPAGE);
+ KUNIT_ASSERT_NE(test, THP_ORDERS_ALL_ANON & BIT(2), 0UL);
+ shadow = mc16_wl_shadow(f, true);
+ KUNIT_ASSERT_TRUE(test, xa_is_value((void *)shadow));
+ for (i = 0; i < f->nr; i++) {
+ unsigned long value = (empty ? i != 0 : i == 0) ? shadow : 0;
+
+ mc16_wl_set_shadow(&f->refs[i], value);
+ }
+ /* Inject after a real successful read; native refault is untouched. */
+ mc16_wl_observe(f, 0, false);
+ kunit_activate_static_stub(test, mc16_load_page,
+ mc16_wl_mthp_error_read_page);
+ before = mc16_wl_refaults();
+ mc16_ft_probe_start(f->fault);
+ mc16_fault_probe.data = o;
+ ret = mc16_wl_fault(f->fault, 2);
+ WRITE_ONCE(mc16_fault_probe.task, NULL);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_SIGBUS);
+ KUNIT_EXPECT_EQ(test, mc16_fault_probe.large_allocs, 1U);
+ KUNIT_EXPECT_EQ(test, mc16_fault_probe.small_allocs, 0U);
+ KUNIT_EXPECT_EQ(test, f->reads, 1U);
+ /* Stock sees only the span's first shadow, even if the read fails. */
+ KUNIT_EXPECT_EQ(test, mc16_wl_refaults() - before,
+ empty ? 0L : (long)MC16_FT_PAGES);
+ KUNIT_ASSERT_EQ(test, o->nr, 1U);
+ for (i = 0; i < f->nr; i++) {
+ KUNIT_EXPECT_EQ(test, mc16_wl_get_shadow(&f->refs[i]), 0ULL);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_ft_read(f->fault, i),
+ mc16_ft_orig(f->fault, i)));
+ }
+ mc16_ft_counts(f->fault, MC16_FT_PAGES, 0);
+ /* Retry the original middle token PTE, without replenishing shadow. */
+ o->fail = false;
+ mc16_ft_probe_start(f->fault);
+ mc16_fault_probe.data = o;
+ ret = mc16_wl_fault(f->fault, 2);
+ WRITE_ONCE(mc16_fault_probe.task, NULL);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)(VM_FAULT_NOPAGE | VM_FAULT_MAJOR));
+ KUNIT_EXPECT_EQ(test, mc16_fault_probe.large_allocs, 1U);
+ KUNIT_EXPECT_EQ(test, mc16_fault_probe.small_allocs, 0U);
+ KUNIT_EXPECT_EQ(test, f->reads, 1U + MC16_FT_PAGES);
+ /* Every shadow is already cleared, so the retry adds no refault. */
+ KUNIT_EXPECT_EQ(test, mc16_wl_refaults() - before,
+ empty ? 0L : (long)MC16_FT_PAGES);
+ KUNIT_ASSERT_EQ(test, o->nr, 1U + MC16_FT_PAGES);
+ for (i = 0; i < o->nr; i++) {
+ struct mc16_wl_mthp_error_read *a = &o->read[i];
+
+ KUNIT_EXPECT_EQ(test, a->order, 2U);
+ KUNIT_EXPECT_EQ(test, a->pages, (unsigned int)MC16_FT_PAGES);
+ KUNIT_EXPECT_EQ(test, a->token, f->fault->tokens[i ? i - 1 : 0]);
+ KUNIT_EXPECT_EQ(test, a->result, 0);
+ KUNIT_EXPECT_TRUE(test, a->contents);
+ KUNIT_EXPECT_TRUE(test, a->locked);
+ KUNIT_EXPECT_TRUE(test, a->swapbacked);
+ KUNIT_EXPECT_TRUE(test, a->cleared);
+ /*
+ * Classic recency needs a refault distance the fixture lacks;
+ * MGLRU sets it in-fault if lruvec and generation still match.
+ */
+ if (i || empty)
+ KUNIT_EXPECT_FALSE(test, a->workingset);
+ else if (lru_gen_enabled())
+ KUNIT_EXPECT_TRUE(test, a->workingset);
+ }
+ for (i = 0; i < f->nr; i++) {
+ KUNIT_EXPECT_EQ(test, mc16_wl_get_shadow(&f->refs[i]), 0ULL);
+ mc16_ft_present(f->fault, i, 2, MC16_FT_PAGES);
+ }
+ mc16_ft_counts(f->fault, 0, MC16_FT_PAGES);
+}
+#endif /* MC16_WORKINGSET_MTHP_TEST_ENABLED */
+#endif /* _MM_MEMCOMP_TESTS_WORKINGSET_MTHP_ERROR_TEST_H */
diff --git a/mm/memcomp/tests/workingset_storage_test.h b/mm/memcomp/tests/workingset_storage_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..fdc7a035359523d82c9fb0590cde29289c5ec631
--- /dev/null
+++ b/mm/memcomp/tests/workingset_storage_test.h
@@ -0,0 +1,442 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_WORKINGSET_STORAGE_TEST_H
+#define _MM_MEMCOMP_TESTS_WORKINGSET_STORAGE_TEST_H
+
+/* Include after storage_test.h. Registration belongs to tests/suite.h. */
+static void memcomp_test_workingset_storage_marker(struct kunit *test)
+{
+ KUNIT_EXPECT_NE(test, MC16_SHADOW_PUBLISHING, 0ULL);
+ KUNIT_EXPECT_FALSE(test, xa_is_value((void *)(unsigned long)
+ MC16_SHADOW_PUBLISHING));
+ KUNIT_EXPECT_EQ(test, MC16_SEGMENT_SLOTS * sizeof(u64), (size_t)512);
+ KUNIT_EXPECT_EQ(test, sizeof(struct mc16_slot), (size_t)16);
+ KUNIT_EXPECT_EQ(test, sizeof(struct mc16_record), (size_t)64);
+}
+
+#ifdef CONFIG_MEMCG
+static void memcomp_test_workingset_storage_words(struct kunit *test)
+{
+ struct mc16_test_storage *f = mc16_test_storage_begin(test);
+ struct memcomp_ref guard = { };
+ u64 *word, saved = (unsigned long)xa_mk_value(123);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(f, 2), 0);
+ KUNIT_ASSERT_TRUE(test, mc16_get(f->refs[0].token, &guard));
+ mutex_lock(memcomp_lock_of(guard.token));
+ word = mc16_shadow_ptr(&guard);
+ KUNIT_EXPECT_NOT_NULL(test, word);
+ if (word) {
+ KUNIT_EXPECT_EQ(test, *word, 0ULL);
+ *word = MC16_SHADOW_PUBLISHING;
+ KUNIT_EXPECT_EQ(test, *mc16_shadow_ptr(&guard),
+ MC16_SHADOW_PUBLISHING);
+ *word = saved;
+ KUNIT_EXPECT_EQ(test, *mc16_shadow_ptr(&guard), saved);
+ *word = 0;
+ }
+ mutex_unlock(memcomp_lock_of(guard.token));
+ mc16_ref_put(&guard);
+
+ mutex_lock(memcomp_lock_of(f->refs[1].token));
+ word = mc16_shadow_ptr(&f->refs[1]);
+ KUNIT_EXPECT_NOT_NULL(test, word);
+ if (word)
+ KUNIT_EXPECT_EQ(test, *word, 0ULL);
+ mutex_unlock(memcomp_lock_of(f->refs[1].token));
+}
+
+static void memcomp_test_workingset_storage_segments(struct kunit *test)
+{
+ struct mc16_test_storage *f = mc16_test_storage_begin(test);
+ u64 *words[2], value[2] = {
+ (unsigned long)xa_mk_value(11),
+ (unsigned long)xa_mk_value(22),
+ };
+ unsigned int i, index;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (i = 0; i < 65; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(f, 1), 0);
+ KUNIT_EXPECT_PTR_NE(test, f->refs[0].cluster->shadows[0],
+ f->refs[0].cluster->shadows[1]);
+ for (i = 0; i < 2; i++) {
+ index = 63 + i;
+ mutex_lock(memcomp_lock_of(f->refs[index].token));
+ words[i] = mc16_shadow_ptr(&f->refs[index]);
+ KUNIT_EXPECT_NOT_NULL(test, words[i]);
+ if (words[i]) {
+ KUNIT_EXPECT_EQ(test, *words[i], 0ULL);
+ *words[i] = value[i];
+ }
+ mutex_unlock(memcomp_lock_of(f->refs[index].token));
+ }
+ for (i = 0; i < 2; i++) {
+ index = 63 + i;
+ mutex_lock(memcomp_lock_of(f->refs[index].token));
+ if (words[i])
+ KUNIT_EXPECT_EQ(test, *mc16_shadow_ptr(&f->refs[index]),
+ value[i]);
+ mutex_unlock(memcomp_lock_of(f->refs[index].token));
+ }
+}
+
+static void memcomp_test_workingset_storage_nomem(struct kunit *test)
+{
+ struct mc16_test_storage *f = mc16_test_storage_begin(test);
+ unsigned int at;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ struct mc16_storage_debug before, after;
+#endif
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (at = 1; at <= 3; at++) {
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ mc16_test_metadata_snapshot(&before);
+#endif
+ /* Segment, shadow, then cluster allocation, all before publication. */
+ f->probe.fail_at = f->probe.allocations + at;
+ KUNIT_EXPECT_EQ(test, mc16_test_reserve(f, 1), -ENOMEM);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->owner->folio), 1);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->owner->model.charged), 0L);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ mc16_test_metadata_snapshot(&after);
+ KUNIT_EXPECT_EQ(test,
+ after.allocated[MC16_META_SHADOW] -
+ before.allocated[MC16_META_SHADOW],
+ after.freed[MC16_META_SHADOW] -
+ before.freed[MC16_META_SHADOW]);
+#endif
+ }
+ f->probe.fail_at = 0;
+ KUNIT_EXPECT_EQ(test, mc16_test_reserve(f, 1), 0);
+}
+
+static void mc16_ws_test_retire(struct kunit *test, bool fail_descriptor)
+{
+ struct mc16_test_storage *f = mc16_test_storage_begin(test);
+ struct mc16_cluster *cluster;
+ struct mc16_segment_retire *retired;
+ u64 *shadows;
+ u64 survivor;
+ unsigned short owner_id;
+ unsigned int i;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ struct mc16_storage_debug before, after;
+#endif
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (i = 0; i < 65; i++)
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(f, 1), 0);
+ cluster = f->refs[64].cluster;
+ owner_id = mc16_owner_id(&f->refs[64]);
+ /* Storage-only opaque words; never submit them to stock refault. */
+ for (i = 0; i < 65; i++) {
+ mutex_lock(memcomp_lock_of(f->refs[i].token));
+ *mc16_shadow_ptr(&f->refs[i]) = (unsigned long)xa_mk_value(i + 1);
+ mutex_unlock(memcomp_lock_of(f->refs[i].token));
+ }
+ shadows = cluster->shadows[0];
+ f->probe.hold_segments = true;
+ f->probe.fail_descriptor = fail_descriptor;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ mc16_test_metadata_snapshot(&before);
+#endif
+ for (i = 0; i < 64; i++)
+ mc16_test_cancel_pending(&f->refs[i]);
+ KUNIT_EXPECT_EQ(test, cluster->live_slots, 1U);
+ mutex_lock(memcomp_lock_of(f->refs[64].token));
+ survivor = *mc16_shadow_ptr(&f->refs[64]);
+ mutex_unlock(memcomp_lock_of(f->refs[64].token));
+ KUNIT_EXPECT_EQ(test, survivor, (u64)(unsigned long)xa_mk_value(65));
+ KUNIT_EXPECT_EQ(test, mc16_owner_id(&f->refs[64]), owner_id);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->refs[64].slot->refs), 1U);
+ if (fail_descriptor) {
+ KUNIT_EXPECT_PTR_EQ(test, cluster->shadows[0], shadows);
+ KUNIT_EXPECT_NOT_NULL(test, cluster->segments[0]);
+ KUNIT_EXPECT_PTR_EQ(test, f->probe.delayed_segments, NULL);
+ } else {
+ KUNIT_EXPECT_PTR_EQ(test, cluster->shadows[0], NULL);
+ KUNIT_EXPECT_PTR_EQ(test, cluster->segments[0], NULL);
+ KUNIT_ASSERT_NOT_NULL(test, f->probe.delayed_segments);
+ retired = container_of(f->probe.delayed_segments,
+ struct mc16_segment_retire, rcu);
+ KUNIT_EXPECT_PTR_EQ(test, retired->shadows, shadows);
+ }
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ mc16_test_metadata_snapshot(&after);
+ KUNIT_EXPECT_EQ(test, after.freed[MC16_META_SHADOW],
+ before.freed[MC16_META_SHADOW]);
+#endif
+ f->probe.fail_descriptor = false;
+ mc16_test_drain_segments(&f->probe);
+ rcu_barrier();
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ mc16_test_metadata_snapshot(&after);
+ KUNIT_EXPECT_EQ(test, after.freed[MC16_META_SHADOW] -
+ before.freed[MC16_META_SHADOW],
+ fail_descriptor ? 0ULL : 1ULL);
+#endif
+ mc16_test_cancel_pending(&f->refs[64]);
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+}
+
+static void memcomp_test_workingset_storage_retire(struct kunit *test)
+{
+ mc16_ws_test_retire(test, false);
+}
+
+static void memcomp_test_workingset_storage_retire_nomem(struct kunit *test)
+{
+ mc16_ws_test_retire(test, true);
+}
+
+static void memcomp_test_workingset_storage_key_identity(struct kunit *test)
+{
+ struct mc16_key a = { .owner_id = 7 }, b = { .owner_id = 8 };
+ struct mc16_key *found_a, *found_b, *missing;
+ unsigned long flags;
+
+ /* Same owner pointer and geometry, only the immutable source ID differs. */
+ xa_lock_irqsave(&mc16_clusters, flags);
+ hash_add(mc16_keys, &a.node, mc16_key_tag_id(NULL, 0, 0, false, 7));
+ hash_add(mc16_keys, &b.node, mc16_key_tag_id(NULL, 0, 0, false, 8));
+ found_a = mc16_key_find_id(NULL, 0, 0, false, 7);
+ found_b = mc16_key_find_id(NULL, 0, 0, false, 8);
+ missing = mc16_key_find_id(NULL, 0, 0, false, 9);
+ hash_del(&a.node);
+ hash_del(&b.node);
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ KUNIT_EXPECT_PTR_EQ(test, found_a, &a);
+ KUNIT_EXPECT_PTR_EQ(test, found_b, &b);
+ KUNIT_EXPECT_PTR_EQ(test, missing, NULL);
+}
+
+/* Only direct alloc/free is modeled; no synthetic ID enters an RCU callback. */
+static unsigned int mc16_ws_id_get_calls;
+static unsigned int mc16_ws_id_put_calls;
+
+static bool mc16_ws_id_get_ref(unsigned short id)
+{
+ if (!id)
+ return true;
+ mc16_ws_id_get_calls++;
+ return id == 7;
+}
+
+static void mc16_ws_id_put(unsigned short id)
+{
+ if (id)
+ mc16_ws_id_put_calls++;
+}
+
+struct mc16_ws_key_hold {
+ struct mc16_key *key;
+};
+
+static void mc16_ws_key_cleanup(void *data)
+{
+ struct mc16_ws_key_hold *hold = data;
+
+ mc16_key_free(hold->key);
+ hold->key = NULL;
+}
+
+static void memcomp_test_workingset_storage_key_ref(struct kunit *test)
+{
+ struct mc16_test_storage *f = mc16_test_storage_begin(test);
+ struct mc16_ws_key_hold *hold;
+ struct mc16_key *key;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ hold = kunit_kzalloc(test, sizeof(*hold), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, hold);
+ mc16_ws_id_get_calls = 0;
+ mc16_ws_id_put_calls = 0;
+ kunit_activate_static_stub(test, mc16_owner_id_get_ref, mc16_ws_id_get_ref);
+ kunit_activate_static_stub(test, mc16_owner_id_put, mc16_ws_id_put);
+ /* Registered after the stubs: even an assertion exit frees while modeled. */
+ KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, mc16_ws_key_cleanup,
+ hold), 0);
+ key = mc16_key_alloc_id(&f->owner->model.objcg, 0, MC16_KEY_ANON, false, 7);
+ KUNIT_ASSERT_FALSE(test, IS_ERR(key));
+ hold->key = key;
+ KUNIT_EXPECT_EQ(test, key->owner_id, (unsigned short)7);
+ KUNIT_EXPECT_EQ(test, mc16_ws_id_get_calls, 1U);
+ KUNIT_EXPECT_EQ(test, mc16_ws_id_put_calls, 0U);
+ mc16_ws_key_cleanup(hold);
+ KUNIT_EXPECT_EQ(test, mc16_ws_id_put_calls, 1U);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->owner->model.charged), 0L);
+}
+
+static void memcomp_test_workingset_storage_key_ref_failure(struct kunit *test)
+{
+ struct mc16_test_storage *f = mc16_test_storage_begin(test);
+ const struct mc16_initial_payload payload = { .word = 0x55 };
+ struct memcomp_ref ref = { };
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_ws_id_get_calls = 0;
+ kunit_activate_static_stub(test, mc16_owner_id_get_ref, mc16_ws_id_get_ref);
+ /* ID 8 cannot acquire a key reference; nothing may become visible. */
+ ret = mc16_reserve_cache_id(&f->owner->model.objcg, 8, 0, &payload, 1,
+ &ref);
+ KUNIT_EXPECT_EQ(test, ret, -ENOENT);
+ KUNIT_EXPECT_PTR_EQ(test, ref.slot, NULL);
+ KUNIT_EXPECT_EQ(test, mc16_ws_id_get_calls, 1U);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->owner->model.charged), 0L);
+}
+
+static int mc16_ws_source_id_failure(struct folio *folio, unsigned short *id)
+{
+ *id = 0;
+ return -EAGAIN;
+}
+
+static void memcomp_test_workingset_storage_source_id_failure(struct kunit *test)
+{
+ struct mc16_test_storage *f = mc16_test_storage_begin(test);
+ unsigned int allocations;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ allocations = f->probe.allocations;
+ kunit_activate_static_stub(test, mc16_source_id_get,
+ mc16_ws_source_id_failure);
+ KUNIT_EXPECT_EQ(test, mc16_test_reserve(f, 1), -EAGAIN);
+ KUNIT_EXPECT_EQ(test, f->nr, 0U);
+ KUNIT_EXPECT_EQ(test, f->probe.allocations, allocations);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->owner->folio), 1);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->owner->model.charged), 0L);
+}
+
+struct mc16_ws_charge_hold {
+ struct folio *folio;
+ unsigned short id;
+};
+
+static void mc16_ws_charge_cleanup(void *data)
+{
+ struct mc16_ws_charge_hold *hold = data;
+
+ mem_cgroup_memcomp_id_put(hold->id);
+ if (hold->folio) {
+ folio_unlock(hold->folio);
+ folio_put(hold->folio);
+ }
+}
+
+static void memcomp_test_workingset_storage_source_id(struct kunit *test)
+{
+ struct mc16_ws_charge_hold *hold;
+ struct mem_cgroup *memcg;
+ unsigned short expected_id;
+ bool got;
+ int ret;
+
+ hold = kunit_kzalloc(test, sizeof(*hold), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, hold);
+ KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, mc16_ws_charge_cleanup,
+ hold), 0);
+ hold->folio = folio_alloc(GFP_KERNEL, 0);
+ KUNIT_ASSERT_NOT_NULL(test, hold->folio);
+ folio_lock(hold->folio);
+ ret = mem_cgroup_memcomp_charge(hold->folio, &init_mm, GFP_KERNEL, 0);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ if (mem_cgroup_disabled()) {
+ KUNIT_EXPECT_EQ(test, mem_cgroup_memcomp_id_get(hold->folio),
+ (unsigned short)0);
+ return;
+ }
+ memcg = folio_memcg(hold->folio);
+ KUNIT_ASSERT_NOT_NULL(test, memcg);
+ expected_id = mem_cgroup_id(memcg);
+ KUNIT_EXPECT_NE(test, expected_id, (unsigned short)0);
+ KUNIT_EXPECT_PTR_EQ(test, memcg, root_mem_cgroup);
+ hold->id = mem_cgroup_memcomp_id_get(hold->folio);
+ KUNIT_EXPECT_EQ(test, hold->id, expected_id);
+ got = mem_cgroup_memcomp_id_get_ref(hold->id);
+ KUNIT_EXPECT_TRUE(test, got);
+ if (got)
+ mem_cgroup_memcomp_id_put(hold->id);
+ if (hold->id) {
+ rcu_read_lock();
+ KUNIT_EXPECT_PTR_EQ(test, mem_cgroup_from_id(hold->id), memcg);
+ rcu_read_unlock();
+ }
+}
+
+static void memcomp_test_workingset_storage_root_source(struct kunit *test)
+{
+ struct mc16_ws_charge_hold *hold;
+ unsigned short root_id;
+ bool got;
+
+ hold = kunit_kzalloc(test, sizeof(*hold), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, hold);
+ KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, mc16_ws_charge_cleanup,
+ hold), 0);
+ hold->folio = folio_alloc(GFP_KERNEL, 0);
+ KUNIT_ASSERT_NOT_NULL(test, hold->folio);
+ folio_lock(hold->folio);
+ /* An uncharged source still has root identity, not the later reader's. */
+ KUNIT_EXPECT_PTR_EQ(test, folio_memcg(hold->folio), NULL);
+ KUNIT_ASSERT_EQ(test, mem_cgroup_memcomp_id_get_checked(hold->folio,
+ &hold->id), 0);
+ if (mem_cgroup_disabled()) {
+ KUNIT_EXPECT_EQ(test, hold->id, (unsigned short)0);
+ return;
+ }
+ root_id = mem_cgroup_id(root_mem_cgroup);
+ KUNIT_EXPECT_NE(test, root_id, (unsigned short)0);
+ KUNIT_EXPECT_EQ(test, hold->id, root_id);
+ rcu_read_lock();
+ KUNIT_EXPECT_PTR_EQ(test, mem_cgroup_from_id(hold->id), root_mem_cgroup);
+ rcu_read_unlock();
+ got = mem_cgroup_memcomp_id_get_ref(hold->id);
+ KUNIT_EXPECT_TRUE(test, got);
+ if (got)
+ mem_cgroup_memcomp_id_put(hold->id);
+ KUNIT_ASSERT_EQ(test, mem_cgroup_memcomp_charge(hold->folio, &init_mm,
+ GFP_KERNEL, hold->id), 0);
+ KUNIT_EXPECT_PTR_EQ(test, folio_memcg(hold->folio), root_mem_cgroup);
+}
+
+static void memcomp_test_workingset_storage_charge_fallback(struct kunit *test)
+{
+ struct mc16_ws_charge_hold *hold;
+ unsigned int id;
+
+ hold = kunit_kzalloc(test, sizeof(*hold), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, hold);
+ KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, mc16_ws_charge_cleanup,
+ hold), 0);
+ hold->folio = folio_alloc(GFP_KERNEL, 0);
+ KUNIT_ASSERT_NOT_NULL(test, hold->folio);
+ folio_lock(hold->folio);
+ rcu_read_lock();
+ for (id = 1; id <= U16_MAX; id++) {
+ if (!mem_cgroup_from_id(id))
+ break;
+ }
+ rcu_read_unlock();
+ if (id > U16_MAX) {
+ kunit_skip(test, "no unassigned memcg ID for fallback check");
+ return;
+ }
+ KUNIT_EXPECT_FALSE(test, mem_cgroup_memcomp_id_get_ref(id));
+ KUNIT_ASSERT_EQ(test, mem_cgroup_memcomp_charge(hold->folio, &init_mm,
+ GFP_KERNEL, id), 0);
+ if (!mem_cgroup_disabled())
+ KUNIT_EXPECT_PTR_EQ(test, folio_memcg(hold->folio), root_mem_cgroup);
+}
+#endif /* CONFIG_MEMCG */
+#endif /* _MM_MEMCOMP_TESTS_WORKINGSET_STORAGE_TEST_H */
diff --git a/mm/memcomp/tests/workingset_store_test.h b/mm/memcomp/tests/workingset_store_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..ea6d1d49c272293188b04702b987ebe0a78866c2
--- /dev/null
+++ b/mm/memcomp/tests/workingset_store_test.h
@@ -0,0 +1,572 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_WORKINGSET_STORE_TEST_H
+#define _MM_MEMCOMP_TESTS_WORKINGSET_STORE_TEST_H
+
+/*
+ * Author: nakixii
+ *
+ * Include after cache_folio_test.h, folio_test.h and store_transaction_test.h.
+ * Register the cases under MC16_WORKINGSET_STORE_TEST_ENABLED. Sources, guards,
+ * payloads, LRU isolation, eviction and PTE removal use the real implementation.
+ * The eviction observation stub calls the real stock-policy entry; it does
+ * not fabricate a shadow or bypass the release transaction.
+ *
+ * The race case runs a separate reader/writeback/PTE-zap task while the
+ * producer is stopped at its irrevocable eviction point with no T/PTL/XA.
+ * No worker outlives the synchronous release actor or its fixture.
+ */
+#ifdef MC16_FOLIO_TEST_ENABLED
+#define MC16_WORKINGSET_STORE_TEST_ENABLED
+
+#include <linux/stop_machine.h>
+
+struct mc16_ws_test {
+ struct kunit *test;
+ struct mc16_flt *anon;
+ struct mem_cgroup *target;
+ struct mem_cgroup *seen_target;
+ struct folio *source;
+ struct completion start;
+ struct completion done;
+ struct task_struct *worker;
+ unsigned int calls;
+ unsigned int pages;
+ unsigned long native_before;
+ unsigned long native_after;
+ bool native_mglru;
+ bool locked;
+ bool isolated;
+ bool token_locks_free;
+ bool worker_done;
+ int source_refs;
+ int load[2];
+ bool writeback[2];
+ int zap[2];
+ u8 *buffer;
+ void *shadow;
+};
+
+/* Only the synchronous test-task eviction stub reads this pointer. */
+static struct mc16_ws_test *mc16_ws_actor;
+
+static u64 mc16_ws_word(struct memcomp_ref *ref)
+{
+ u64 word;
+
+ mutex_lock(memcomp_lock_of(ref->token));
+ word = *mc16_shadow_ptr(ref);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ return word;
+}
+
+static void mc16_ws_clear(struct memcomp_ref *ref)
+{
+ mutex_lock(memcomp_lock_of(ref->token));
+ *mc16_shadow_ptr(ref) = 0;
+ mutex_unlock(memcomp_lock_of(ref->token));
+}
+
+static int mc16_ws_competitors(void *data)
+{
+ struct mc16_ws_test *p = data;
+ struct mc16_flt *f = p->anon;
+ unsigned int i;
+
+ wait_for_completion(&p->start);
+ if (kthread_should_stop())
+ goto out;
+ for (i = 0; i < f->nr; i++) {
+ struct mc16_writeback snapshot;
+
+ p->load[i] = mc16_load_page(&f->guards[i], p->buffer);
+ mutex_lock(memcomp_lock_of(f->guards[i].token));
+ p->writeback[i] =
+ mc16_writeback_claim_locked(&f->guards[i], &snapshot);
+ /* Repair a failed gate assertion before release can continue. */
+ if (p->writeback[i])
+ mc16_writeback_revert_locked(&f->guards[i]);
+ mutex_unlock(memcomp_lock_of(f->guards[i].token));
+ p->zap[i] = mc16_flt_zap(f, f->addr + i * PAGE_SIZE);
+ }
+out:
+ complete(&p->done);
+ return 0;
+}
+
+/*
+ * No other CPU or interrupt can reclaim or advance an eviction bucket while
+ * this callback samples the native entry. Assertions run after CPUs resume.
+ * Do not reset native counters or substitute the producer call counters.
+ */
+static int mc16_ws_native_eviction(void *data)
+{
+ struct mc16_ws_test *p = data;
+ struct folio *folio = p->source;
+ struct lruvec *lruvec;
+ atomic_long_t *counter;
+
+ lruvec = mem_cgroup_lruvec(p->seen_target, folio_pgdat(folio));
+ counter = &lruvec->nonresident_age;
+#ifdef CONFIG_LRU_GEN
+ if (lru_gen_enabled()) {
+ int type = folio_is_file_lru(folio);
+ int tier = lru_tier_from_refs(folio_lru_refs(folio));
+ unsigned long seq;
+
+ lruvec = mem_cgroup_lruvec(folio_memcg(folio), folio_pgdat(folio));
+ seq = READ_ONCE(lruvec->lrugen.min_seq[type]);
+ counter = &lruvec->lrugen.evicted[lru_hist_from_seq(seq)][type][tier];
+ p->native_mglru = true;
+ }
+#endif
+ p->native_before = atomic_long_read(counter);
+ p->shadow = workingset_memcomp_eviction(folio, p->seen_target);
+ p->native_after = atomic_long_read(counter);
+ return 0;
+}
+
+static void *mc16_ws_eviction(struct folio *folio,
+ struct mem_cgroup *target_memcg)
+{
+ struct mc16_ws_test *p = mc16_ws_actor;
+ unsigned int i;
+ int ret;
+
+ p->calls++;
+ p->source = folio;
+ p->seen_target = target_memcg;
+ p->pages = folio_nr_pages(folio);
+ p->locked = folio_test_locked(folio);
+ p->isolated = !folio_test_lru(folio);
+ p->source_refs = folio_ref_count(folio);
+ p->token_locks_free = true;
+ if (p->anon) {
+ for (i = 0; i < p->anon->nr; i++)
+ if (mutex_is_locked(memcomp_lock_of(p->anon->guards[i].token)))
+ p->token_locks_free = false;
+ }
+ if (p->worker && p->token_locks_free) {
+ complete(&p->start);
+ p->worker_done = wait_for_completion_timeout(&p->done, 5 * HZ);
+ }
+ ret = stop_machine(mc16_ws_native_eviction, p, NULL);
+ KUNIT_EXPECT_EQ(p->test, ret, 0);
+ KUNIT_EXPECT_EQ(p->test, p->native_after - p->native_before,
+ (unsigned long)p->pages);
+ kunit_info(p->test, "native eviction: lru=%s before=%lu after=%lu pages=%u",
+ p->native_mglru ? "mglru" : "classic",
+ p->native_before, p->native_after, p->pages);
+ return p->shadow;
+}
+
+static void mc16_ws_cleanup(void *data)
+{
+ struct mc16_ws_test *p = data;
+
+ if (p->worker) {
+ complete(&p->start);
+ kthread_stop_put(p->worker);
+ }
+ if (mc16_ws_actor == p)
+ mc16_ws_actor = NULL;
+}
+
+static struct mc16_ws_test *mc16_ws_begin(struct kunit *test)
+{
+ struct mc16_ws_test *p = kunit_kzalloc(test, sizeof(*p), GFP_KERNEL);
+
+ if (!p)
+ return NULL;
+ p->test = test;
+ p->buffer = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ if (!p->buffer)
+ return NULL;
+ init_completion(&p->start);
+ init_completion(&p->done);
+ if (kunit_add_action_or_reset(test, mc16_ws_cleanup, p))
+ return NULL;
+ kunit_activate_static_stub(test, mc16_store_eviction, mc16_ws_eviction);
+ mc16_ws_actor = p;
+ return p;
+}
+
+/* Drop exactly the extra isolation reference, preserving the fixture's one. */
+static int mc16_ws_isolate(struct folio *folio)
+{
+ lru_add_drain_all();
+ if (!folio_test_lru(folio) || !folio_isolate_lru(folio))
+ return -EBUSY;
+ /* Model reclaim isolation, not migration: the source is inactive. */
+ folio_clear_active(folio);
+ folio_put(folio);
+ return 0;
+}
+
+static int mc16_ws_prepare(struct mc16_flt *f)
+{
+ unsigned int i;
+
+ if (!mc16_flt_reserve(f) || mc16_flt_hold(f))
+ return -EINVAL;
+ for (i = 0; i < f->nr; i++) {
+ if (mc16_flt_escape(f, i) || mc16_flt_commit(f, i))
+ return -EINVAL;
+ }
+ return mc16_ws_isolate(f->folio);
+}
+
+static bool mc16_ws_release(struct mc16_ws_test *p)
+{
+ bool released;
+
+ folio_lock(p->anon->folio);
+ released = mc16_release_folio_reclaim(p->anon->folio, p->target);
+ if (released)
+ folio_ref_unfreeze(p->anon->folio, 1);
+ folio_unlock(p->anon->folio);
+ return released;
+}
+
+static void mc16_ws_expect_exit(struct kunit *test, struct mc16_ws_test *p)
+{
+ struct mc16_flt *f = p->anon;
+ unsigned int i;
+
+ KUNIT_EXPECT_EQ(test, p->calls, 1U);
+ KUNIT_EXPECT_EQ(test, p->pages, f->nr);
+ KUNIT_EXPECT_PTR_EQ(test, p->seen_target, p->target);
+ KUNIT_EXPECT_PTR_EQ(test, p->source, f->folio);
+ KUNIT_EXPECT_TRUE(test, p->locked);
+ KUNIT_EXPECT_TRUE(test, p->isolated);
+ KUNIT_EXPECT_TRUE(test, p->token_locks_free);
+ KUNIT_EXPECT_EQ(test, p->source_refs, 0);
+ KUNIT_EXPECT_TRUE(test, xa_is_value(p->shadow));
+ KUNIT_EXPECT_EQ(test, f->folio->swap.val, 0UL);
+ for (i = 0; i < f->nr; i++) {
+ KUNIT_EXPECT_EQ(test, mc16_ws_word(&f->guards[i]),
+ (u64)(unsigned long)p->shadow);
+ KUNIT_EXPECT_FALSE(test, mc16_flt_flags(&f->guards[i], false) &
+ MC16_F_OWNER);
+ KUNIT_EXPECT_NE(test, refcount_read(&f->guards[i].slot->refs), 0U);
+ }
+}
+
+static void memcomp_test_workingset_store_single(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 0);
+ struct mc16_ws_test *p;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_ws_prepare(f), 0);
+ p = mc16_ws_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, p);
+ p->anon = f;
+ p->target = root_mem_cgroup;
+ KUNIT_ASSERT_TRUE(test, mc16_ws_release(p));
+ mc16_ws_expect_exit(test, p);
+ /* The compatibility API must not create a second eviction epoch. */
+ mc16_ws_clear(&f->guards[0]);
+ KUNIT_EXPECT_FALSE(test, mc16_flt_release(f->folio));
+ KUNIT_EXPECT_EQ(test, p->calls, 1U);
+ KUNIT_EXPECT_EQ(test, mc16_ws_word(&f->guards[0]), 0ULL);
+}
+
+static void memcomp_test_workingset_store_mthp(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 1);
+ struct mc16_ws_test *p;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_ws_prepare(f), 0);
+ p = mc16_ws_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, p);
+ p->anon = f;
+ KUNIT_ASSERT_TRUE(test, mc16_ws_release(p));
+ /* One real stock call for the whole folio, not one call per slot. */
+ mc16_ws_expect_exit(test, p);
+}
+
+static void memcomp_test_workingset_store_freeze_failure(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 1);
+ struct mc16_ws_test *p;
+ bool released;
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_ws_prepare(f), 0);
+ p = mc16_ws_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, p);
+ p->anon = f;
+ folio_get(f->folio);
+ released = mc16_ws_release(p);
+ folio_put(f->folio);
+ KUNIT_EXPECT_FALSE(test, released);
+ KUNIT_EXPECT_EQ(test, p->calls, 0U);
+ for (i = 0; i < f->nr; i++) {
+ KUNIT_EXPECT_EQ(test, mc16_ws_word(&f->guards[i]), 0ULL);
+ KUNIT_EXPECT_TRUE(test, mc16_flt_flags(&f->guards[i], false) &
+ MC16_F_OWNER);
+ }
+ KUNIT_ASSERT_TRUE(test, mc16_ws_release(p));
+ mc16_ws_expect_exit(test, p);
+}
+
+static void memcomp_test_workingset_store_bad_sibling(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 1);
+ struct mc16_ws_test *p;
+ struct mc16_view view;
+ u16 length;
+ bool released;
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_ws_prepare(f), 0);
+ p = mc16_ws_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, p);
+ p->anon = f;
+ mutex_lock(memcomp_lock_of(f->guards[1].token));
+ mc16_view_locked(&f->guards[1], &view);
+ length = *view.length;
+ *view.length = U16_MAX;
+ mutex_unlock(memcomp_lock_of(f->guards[1].token));
+ released = mc16_ws_release(p);
+ mutex_lock(memcomp_lock_of(f->guards[1].token));
+ mc16_view_locked(&f->guards[1], &view);
+ *view.length = length;
+ mutex_unlock(memcomp_lock_of(f->guards[1].token));
+ KUNIT_EXPECT_FALSE(test, released);
+ KUNIT_EXPECT_EQ(test, p->calls, 0U);
+ for (i = 0; i < f->nr; i++) {
+ KUNIT_EXPECT_EQ(test, mc16_ws_word(&f->guards[i]), 0ULL);
+ KUNIT_EXPECT_TRUE(test, mc16_flt_flags(&f->guards[i], false) &
+ MC16_F_OWNER);
+ }
+ KUNIT_ASSERT_TRUE(test, mc16_ws_release(p));
+ mc16_ws_expect_exit(test, p);
+}
+
+static void memcomp_test_workingset_store_gate_race(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 1);
+ struct mc16_ws_test *p;
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_ws_prepare(f), 0);
+ p = mc16_ws_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, p);
+ p->anon = f;
+ p->worker = kthread_run(mc16_ws_competitors, p, "mc16-store-gate");
+ if (IS_ERR(p->worker)) {
+ p->worker = NULL;
+ KUNIT_FAIL(test, "cannot start store-gate actor");
+ return;
+ }
+ get_task_struct(p->worker);
+ KUNIT_ASSERT_TRUE(test, mc16_ws_release(p));
+ KUNIT_EXPECT_TRUE(test, p->worker_done);
+ /* Join before inspecting results or letting fixture cleanup run. */
+ kthread_stop_put(p->worker);
+ p->worker = NULL;
+ mc16_ws_expect_exit(test, p);
+ for (i = 0; i < f->nr; i++) {
+ KUNIT_EXPECT_EQ(test, p->load[i], -EAGAIN);
+ KUNIT_EXPECT_FALSE(test, p->writeback[i]);
+ KUNIT_EXPECT_EQ(test, p->zap[i], 0);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guards[i].slot->refs), 1U);
+ }
+ KUNIT_EXPECT_EQ(test, mc16_flt_rss(f, MM_SWAPENTS), f->swapents);
+}
+
+static void memcomp_test_workingset_store_prepared(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 0);
+ struct mc16_sct_result result;
+ struct mc16_ws_test *p;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_flt_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, 0), 0);
+ p = mc16_ws_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, p);
+ p->anon = f;
+ result = mc16_sct_store(f, (struct mc16_sct_attempt) { });
+ KUNIT_ASSERT_EQ(test, result.error, 0);
+ KUNIT_ASSERT_TRUE(test, result.success);
+ KUNIT_EXPECT_EQ(test, p->calls, 0U);
+ KUNIT_EXPECT_EQ(test, mc16_ws_word(&f->guards[0]), MC16_SHADOW_PUBLISHING);
+ KUNIT_EXPECT_EQ(test, mc16_load_page(&f->guards[0], p->buffer), -EAGAIN);
+ KUNIT_ASSERT_EQ(test, mc16_ws_isolate(f->folio), 0);
+ KUNIT_ASSERT_TRUE(test, mc16_ws_release(p));
+ mc16_ws_expect_exit(test, p);
+}
+
+/* Cancel a real begun transaction before any payload commit, not after store. */
+static void
+memcomp_test_workingset_store_cancel_before_commit(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 0);
+ struct mc16_sct_result result;
+ struct mc16_ws_test *p;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_flt_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, 0), 0);
+ p = mc16_ws_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, p);
+ /* Cancel after the real gate opens, before a payload commit can win. */
+ folio_lock(f->folio);
+ mutex_lock(memcomp_lock_of(f->guards[0].token));
+ result.error = mc16_store_begin_locked(&f->guards[0], f->folio, f->nr, 0);
+ result.success = !result.error;
+ mutex_unlock(memcomp_lock_of(f->guards[0].token));
+ folio_unlock(f->folio);
+ KUNIT_ASSERT_EQ(test, result.error, 0);
+ KUNIT_ASSERT_TRUE(test, result.success);
+ KUNIT_EXPECT_EQ(test, mc16_ws_word(&f->guards[0]), MC16_SHADOW_PUBLISHING);
+ mc16_flt_drop(f, true);
+ KUNIT_EXPECT_EQ(test, p->calls, 0U);
+ KUNIT_EXPECT_EQ(test, mc16_ws_word(&f->guards[0]), 0ULL);
+ KUNIT_EXPECT_FALSE(test, mc16_flt_flags(&f->guards[0], false) & MC16_F_OWNER);
+ mc16_flt_expect_present(f, 0, f->addr);
+}
+
+static void memcomp_test_workingset_store_cache(struct kunit *test)
+{
+ struct mc16_cf_test *f = mc16_cf_begin(test);
+ struct mc16_ws_test *p;
+ u64 shadow;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ p = mc16_ws_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, p);
+ folio_set_swapbacked(f->src);
+ mc16_cf_fill(f, MC16_CF_BYTES);
+ KUNIT_ASSERT_TRUE(test, mc16_cf_store(f));
+ KUNIT_ASSERT_NE(test, f->token, 0UL);
+ KUNIT_ASSERT_TRUE(test, mc16_get(f->token, &f->guard));
+ KUNIT_EXPECT_EQ(test, p->calls, 0U);
+ KUNIT_EXPECT_EQ(test, mc16_ws_word(&f->guard), MC16_SHADOW_PUBLISHING);
+ KUNIT_EXPECT_EQ(test, mc16_load_page(&f->guard, p->buffer), -EAGAIN);
+ /*
+ * This models the actual winning XA replacement's post-unlock tail.
+ * Removing its owner concurrently is legal: our prior guard survives.
+ */
+ mc16_cache_drop(f->token);
+ f->cache_owner = false;
+ folio_lock(f->src);
+ mc16_publish_cache_eviction(f->src, &f->guard);
+ folio_unlock(f->src);
+ shadow = mc16_ws_word(&f->guard);
+ KUNIT_EXPECT_EQ(test, p->calls, 1U);
+ KUNIT_EXPECT_EQ(test, p->pages, 1U);
+ KUNIT_EXPECT_EQ(test, p->source_refs, 1);
+ KUNIT_EXPECT_TRUE(test, p->locked);
+ KUNIT_EXPECT_TRUE(test, p->isolated);
+ KUNIT_EXPECT_TRUE(test, xa_is_value((void *)(unsigned long)shadow));
+ KUNIT_EXPECT_EQ(test, mc16_load_page(&f->guard, p->buffer), 0);
+ {
+ void *src = kmap_local_folio(f->src, 0);
+
+ KUNIT_EXPECT_EQ(test, memcmp(src, p->buffer, PAGE_SIZE), 0);
+ kunmap_local(src);
+ }
+ mc16_ws_clear(&f->guard);
+ folio_lock(f->src);
+ mc16_publish_cache_eviction(f->src, &f->guard);
+ folio_unlock(f->src);
+ KUNIT_EXPECT_EQ(test, p->calls, 1U);
+ KUNIT_EXPECT_EQ(test, mc16_ws_word(&f->guard), 0ULL);
+}
+
+static void memcomp_test_workingset_store_cache_abort(struct kunit *test)
+{
+ struct mc16_cf_test *f = mc16_cf_begin(test);
+ struct mc16_ws_test *p;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ p = mc16_ws_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, p);
+ folio_set_swapbacked(f->src);
+ mc16_cf_fill(f, MC16_CF_FILL);
+ KUNIT_ASSERT_TRUE(test, mc16_cf_store(f));
+ KUNIT_ASSERT_NE(test, f->token, 0UL);
+ KUNIT_ASSERT_TRUE(test, mc16_get(f->token, &f->guard));
+ KUNIT_EXPECT_EQ(test, mc16_ws_word(&f->guard), MC16_SHADOW_PUBLISHING);
+ mc16_cache_drop(f->token); /* Losing replacement: no eviction tail. */
+ f->cache_owner = false;
+ mc16_ref_put(&f->guard);
+ KUNIT_EXPECT_EQ(test, p->calls, 0U);
+ KUNIT_EXPECT_FALSE(test, mc16_entry_present(f->token));
+}
+
+static void memcomp_test_workingset_store_legacy(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 0);
+ struct mc16_ws_test *p;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_ws_prepare(f), 0);
+ p = mc16_ws_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, p);
+ p->anon = f;
+ /* Successful old API executes the same real eviction, with NULL target. */
+ KUNIT_ASSERT_TRUE(test, mc16_flt_release(f->folio));
+ mc16_ws_expect_exit(test, p);
+}
+
+static void memcomp_test_workingset_store_zero_excluded(struct kunit *test)
+{
+ struct mc16_consumer_test *f = mc16_consumer_test_begin(test);
+ struct mc16_ws_test *p;
+ bool released;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ p = mc16_ws_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, p);
+ f->account->swap = make_memcomp_entry(0);
+ released = mc16_flt_release(f->account);
+ f->account->swap.val = 0;
+ KUNIT_EXPECT_TRUE(test, released);
+ KUNIT_EXPECT_EQ(test, p->calls, 0U);
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+}
+
+static void memcomp_test_workingset_store_partial(struct kunit *test)
+{
+ struct mc16_flt *f = mc16_flt_begin(test, 1);
+ struct mc16_sct_result result;
+ struct mc16_ws_test *p;
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_flt_reserve(f));
+ KUNIT_ASSERT_EQ(test, mc16_flt_hold(f), 0);
+ for (i = 0; i < f->nr; i++)
+ KUNIT_ASSERT_EQ(test, mc16_flt_escape(f, i), 0);
+ p = mc16_ws_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, p);
+ result = mc16_sct_store(f, (struct mc16_sct_attempt) {
+ .fail = MEMCOMP_KUNIT_FAIL_REFRESH_COMMIT, .nth = 2,
+ });
+ KUNIT_ASSERT_EQ(test, result.error, 0);
+ KUNIT_EXPECT_FALSE(test, result.success);
+ KUNIT_EXPECT_EQ(test, mc16_ws_word(&f->guards[0]), MC16_SHADOW_PUBLISHING);
+ KUNIT_EXPECT_EQ(test, mc16_ws_word(&f->guards[1]), 0ULL);
+ KUNIT_EXPECT_EQ(test, p->calls, 0U);
+ mc16_flt_drop(f, true);
+ for (i = 0; i < f->nr; i++) {
+ KUNIT_EXPECT_EQ(test, mc16_ws_word(&f->guards[i]), 0ULL);
+ KUNIT_EXPECT_FALSE(test, mc16_flt_flags(&f->guards[i], false) &
+ MC16_F_OWNER);
+ mc16_flt_expect_present(f, i, f->addr + i * PAGE_SIZE);
+ }
+ KUNIT_EXPECT_EQ(test, p->calls, 0U);
+}
+
+#endif /* MC16_FOLIO_TEST_ENABLED */
+#endif /* _MM_MEMCOMP_TESTS_WORKINGSET_STORE_TEST_H */
diff --git a/mm/memcomp/tests/writeback_batch_test.h b/mm/memcomp/tests/writeback_batch_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..9db16b1592546cdd2627fff132bbf13a85b05e4a
--- /dev/null
+++ b/mm/memcomp/tests/writeback_batch_test.h
@@ -0,0 +1,688 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_WRITEBACK_BATCH_TEST_H
+#define _MM_MEMCOMP_TESTS_WRITEBACK_BATCH_TEST_H
+
+#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP)
+#include <kunit/static_stub.h>
+
+/*
+ * Include after refs_test.h (and its storage/objcg fixtures),
+ * with writeback_batch.h already visible in memcomp.c.
+ * These are model-I/O tests: real zs payloads, copy/decompression, folio
+ * writeback completion and slot transitions, but no device or swapcache
+ * insertion. All fake backing operations use the existing slot tally model.
+ */
+struct mc16_test_wbb {
+ struct mc16_test_refs *refs;
+ struct mc16_storage_fixture *saved_probe;
+ struct mc16_payload prepared[2];
+ struct mc16_payload_tail tail[2];
+ struct memcomp_ref survivor[2];
+ struct memcomp_ref caller[2];
+ struct mc16_writeback_member staged[2];
+ swp_entry_t saved_pool[MEMCOMP_WRITEBACK_BATCH];
+ swp_entry_t used_swap;
+ u32 token[2];
+ u32 bytes[2];
+ unsigned int before_claim[2];
+ unsigned int after_claim[2];
+ unsigned int after_caller_put[2];
+ unsigned int after_detach[2];
+ unsigned int alloc_calls;
+ unsigned int dup_calls;
+ unsigned int submits;
+ unsigned int completed;
+ unsigned int seed_after_take;
+ unsigned int seed_after_flush;
+ unsigned int probe_frees;
+ unsigned long freed;
+ long charged_before;
+ int alloc_error;
+ int dup_error;
+ int flush_ret;
+ int dups;
+ int puts;
+ int reclaims;
+ bool dirty_io;
+ bool reject_first;
+ bool live_before_io;
+ bool live_at_submit;
+ bool live_after;
+ bool bad_lifetime;
+ bool submit_contract;
+ bool batch_empty;
+ bool pool_restored;
+ bool probe_restored;
+ u8 *src;
+ u8 *encoded[2];
+ u8 *images;
+ u8 *dst;
+};
+
+/* Set only inside the synchronous test task's W scope, restored before exit. */
+static struct mc16_test_wbb *mc16_test_wbb_current;
+
+/* The inherited objcg fixture normally keeps its folio locked across tests. */
+static void mc16_test_wbb_objcg_cleanup(void *data)
+{
+ struct memcomp_test_objcg_fixture *owner = data;
+
+ /* Acquire here, release in the same call; never inherit a task's lock. */
+ if (owner->folio)
+ folio_lock(owner->folio);
+ memcomp_test_objcg_cleanup(owner);
+}
+
+static void mc16_test_wbb_storage_cleanup(void *data)
+{
+ struct mc16_test_wbb *f = data;
+
+ mc16_test_storage_cleanup(f->refs->storage);
+ rcu_assign_pointer(mc16_storage_fixture, f->saved_probe);
+ synchronize_rcu();
+}
+
+static void mc16_test_wbb_cleanup(void *data)
+{
+ struct mc16_test_wbb *f = data;
+ struct mc16_test_storage *s = f->refs->storage;
+ unsigned int i;
+
+ /* The test-task transaction must already have drained its locked work. */
+ KUNIT_EXPECT_PTR_NE(s->test, mc16_test_wbb_current, f);
+ for (i = 0; i < ARRAY_SIZE(f->survivor); i++) {
+ KUNIT_EXPECT_PTR_EQ(s->test, f->staged[i].ref.slot, NULL);
+ KUNIT_EXPECT_PTR_EQ(s->test, f->caller[i].slot, NULL);
+ if (s->refs[i].slot) {
+ mc16_payload_finish(&s->refs[i], &f->tail[i]);
+ mc16_payload_abort(&s->refs[i], &f->prepared[i]);
+ }
+ if (f->survivor[i].slot)
+ mc16_ref_put(&f->survivor[i]);
+ }
+}
+
+/* Reuse shared cleanup/reserve logic, with a source folio unlocked on return. */
+static int mc16_test_wbb_fixtures(struct kunit *test, struct mc16_test_wbb *f)
+{
+ struct memcomp_test_objcg_fixture *owner;
+ struct mc16_test_storage *s;
+ int ret;
+
+ f->refs = kunit_kzalloc(test, sizeof(*f->refs), GFP_KERNEL);
+ s = kunit_kzalloc(test, sizeof(*s), GFP_KERNEL);
+ if (!f->refs || !s)
+ return -ENOMEM;
+ f->refs->storage = s;
+ owner = memcomp_test_objcg_begin(test, 2);
+ if (!owner)
+ return -ENOMEM;
+ folio_unlock(owner->folio);
+ kunit_remove_action(test, memcomp_test_objcg_cleanup, owner);
+ ret = kunit_add_action_or_reset(test, mc16_test_wbb_objcg_cleanup, owner);
+ if (ret)
+ return ret;
+ s->owner = owner;
+ s->test = test;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ mc16_test_metadata_snapshot(&s->baseline);
+#endif
+ f->saved_probe = rcu_access_pointer(mc16_storage_fixture);
+ s->probe.task = current;
+ ret = kunit_add_action_or_reset(test, mc16_test_wbb_storage_cleanup, f);
+ if (ret)
+ return ret;
+ rcu_assign_pointer(mc16_storage_fixture, &s->probe);
+ f->refs->stored_before = atomic64_read(&memcomp_stored_bytes);
+ f->refs->compressed_before = atomic64_read(&memcomp_compressed_bytes);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ f->refs->pending_before = atomic64_read(&memcomp_pending_pages);
+#endif
+ ret = kunit_add_action_or_reset(test, mc16_test_refs_cleanup, f->refs);
+ if (ret)
+ return ret;
+ ret = kunit_add_action_or_reset(test, mc16_test_wbb_cleanup, f);
+ if (ret)
+ return ret;
+ folio_lock(owner->folio);
+ ret = mc16_test_refs_reserve(f->refs, 2);
+ folio_unlock(owner->folio);
+ return ret;
+}
+
+static int mc16_test_wbb_prepare(struct mc16_test_wbb *f, unsigned int i,
+ bool raw)
+{
+ struct mc16_test_storage *s = f->refs->storage;
+ struct memcomp_ref *ref = &s->refs[i];
+ unsigned long handle = 0;
+ unsigned int bytes = 0;
+ void *payload;
+ u8 pool = 0;
+ int ret;
+
+ if (raw)
+ ret = memcomp_alloc_raw_payload(0, f->src + i * PAGE_SIZE,
+ PAGE_SIZE, &bytes, &handle, false, NULL);
+ else
+ ret = memcomp_prepare_payload(f->src + i * PAGE_SIZE, PAGE_SIZE,
+ &bytes, &handle, &pool,
+ MEMCOMP_PAYLOAD_MUST_SUCCEED, false);
+ if (ret)
+ return ret;
+ if (pool) {
+ memcomp_zs_free(pool, handle);
+ return -EINVAL;
+ }
+ f->prepared[i] = (struct mc16_payload) { .word = handle, .bytes = bytes };
+ if (!bytes || (!raw && bytes >= PAGE_SIZE))
+ return -E2BIG;
+ f->bytes[i] = bytes;
+ payload = zs_map_object(memcomp_pools[0], handle, ZS_MM_RO);
+ if (!payload)
+ return -EIO;
+ memcpy(f->encoded[i], payload, bytes);
+ zs_unmap_object(memcomp_pools[0], handle);
+ ret = mc16_payload_charge(ref, &f->prepared[i]);
+ if (ret)
+ return ret;
+ folio_lock(s->owner->folio);
+ mutex_lock(memcomp_lock_of(ref->token));
+ ret = mc16_payload_commit_locked(ref, s->owner->folio, 2, i,
+ &f->prepared[i], &f->tail[i]);
+ mutex_unlock(memcomp_lock_of(ref->token));
+ mc16_payload_finish(ref, &f->tail[i]);
+ folio_unlock(s->owner->folio);
+ if (ret)
+ return ret;
+ f->token[i] = ref->token;
+ return mc16_get(ref->token, &f->survivor[i]) ? 0 : -ENOENT;
+}
+
+/* Seeded reservations only: an empty pool must NEVER refill from a device. */
+static int mc16_test_wbb_alloc(struct folio **foliop, swp_entry_t *swapp)
+{
+ struct mc16_test_wbb *f = mc16_test_wbb_current;
+ struct folio *folio;
+ swp_entry_t swap;
+ void *dst;
+ int ret;
+
+ if (!f)
+ return -EINVAL;
+ f->alloc_calls++;
+ if (f->alloc_error)
+ return f->alloc_error;
+ if (!memcomp_writeback_slots_free)
+ return -ENOSPC;
+ folio = folio_alloc(GFP_HIGHUSER_MOVABLE | __GFP_NOWARN |
+ __GFP_NOMEMALLOC, 0);
+ if (!folio)
+ return -ENOMEM;
+ ret = memcomp_writeback_slot_take(&swap);
+ if (ret) {
+ folio_put(folio);
+ return ret;
+ }
+ ret = memcomp_slot_dup(swap);
+ if (ret) {
+ memcomp_writeback_slots_release(&swap, 1);
+ folio_put(folio);
+ return ret;
+ }
+ folio_lock(folio);
+ __folio_set_swapbacked(folio);
+ folio->swap = swap;
+ dst = kmap_local_folio(folio, 0);
+ memset(dst, 0xa7, PAGE_SIZE);
+ kunmap_local(dst);
+ f->seed_after_take = memcomp_writeback_slots_free;
+ f->used_swap = swap;
+ *foliop = folio;
+ *swapp = swap;
+ /* Locked by allocator contract; batch consumes it before any assertion. */
+ return 0;
+}
+
+/* Synchronous model completion; the actual batch builds every byte first. */
+static void mc16_test_wbb_submit(struct folio *folio,
+ struct writeback_control *wbc)
+{
+ struct mc16_test_wbb *f = mc16_test_wbb_current;
+ void *src;
+
+ if (f) {
+ f->submit_contract = folio_test_locked(folio) &&
+ folio_test_uptodate(folio) && wbc->sync_mode == WB_SYNC_NONE &&
+ wbc->swap_plug == &memcomp_writeback_plugged;
+ f->live_at_submit = f->refs->storage->payload.live;
+ if (f->submits < 2) {
+ src = kmap_local_folio(folio, 0);
+ memcpy(f->images + f->submits * PAGE_SIZE, src, PAGE_SIZE);
+ kunmap_local(src);
+ }
+ f->submits++;
+ /* Stale attempt identity; leave the real live payload untouched. */
+ if (f->reject_first)
+ mc16_writeback_members[0].snapshot.word ^= 1;
+ }
+ folio_start_writeback(folio);
+ if (f && f->dirty_io)
+ folio_mark_dirty(folio);
+ folio_unlock(folio);
+ folio_end_writeback(folio);
+ if (f)
+ f->completed++;
+}
+
+static int mc16_test_wbb_dup(swp_entry_t swap)
+{
+ struct mc16_test_wbb *f = mc16_test_wbb_current;
+
+ if (!f)
+ return -EINVAL;
+ f->dup_calls++;
+ if (f->dup_error)
+ return f->dup_error;
+ return memcomp_slot_dup(swap);
+}
+
+static struct mc16_test_wbb *mc16_test_wbb_begin(struct kunit *test, bool raw)
+{
+ struct mc16_test_wbb *f;
+ unsigned int i;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f || mc16_test_wbb_fixtures(test, f))
+ return NULL;
+ f->src = kunit_kmalloc(test, 2 * PAGE_SIZE, GFP_KERNEL);
+ f->images = kunit_kmalloc(test, 2 * PAGE_SIZE, GFP_KERNEL);
+ f->dst = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ for (i = 0; i < ARRAY_SIZE(f->encoded); i++)
+ f->encoded[i] = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ if (!f->src || !f->images || !f->dst || !f->encoded[0] || !f->encoded[1])
+ return NULL;
+ for (i = 0; i < PAGE_SIZE; i++) {
+ f->src[i] = raw ? i * 37 + (i >> 5) : i % 53;
+ f->src[PAGE_SIZE + i] = (i * 3) % 71;
+ }
+ for (i = 0; i < ARRAY_SIZE(f->survivor); i++) {
+ if (mc16_test_wbb_prepare(f, i, raw && !i))
+ return NULL;
+ }
+ kunit_activate_static_stub(test, mc16_writeback_folio_alloc,
+ mc16_test_wbb_alloc);
+ kunit_activate_static_stub(test, mc16_writeback_submit,
+ mc16_test_wbb_submit);
+ kunit_activate_static_stub(test, mc16_writeback_backing_dup,
+ mc16_test_wbb_dup);
+ return f;
+}
+
+/* No assertions here: every return restores W, T, folios, flags and globals. */
+static int mc16_test_wbb_run(struct mc16_test_wbb *f, unsigned int nr,
+ bool detach)
+{
+ struct mc16_test_storage *s = f->refs->storage;
+ struct memcomp_kunit_async_probe *saved_async;
+ struct mc16_test_wbb *saved_current;
+ unsigned int nofs, noreclaim, saved_free, i;
+ struct mc16_view view;
+ int ret = 0, dups, puts, reclaims;
+
+ nofs = memalloc_nofs_save();
+ noreclaim = memalloc_noreclaim_save();
+ mutex_lock(&memcomp_writeback_lock);
+ if (mc16_writeback_batch_used || mc16_writeback_members_used ||
+ memcomp_writeback_plugged || mc16_test_wbb_current || !nr || nr > 2) {
+ ret = -EBUSY;
+ goto unlock;
+ }
+ memcpy(f->saved_pool, memcomp_writeback_slot_pool, sizeof(f->saved_pool));
+ saved_free = memcomp_writeback_slots_free;
+ for (i = 0; i < 3; i++)
+ memcomp_writeback_slot_pool[i] = swp_entry(MAX_SWAPFILES - 1, 140 + i);
+ memcomp_writeback_slots_free = 3;
+ saved_current = mc16_test_wbb_current;
+ mc16_test_wbb_current = f;
+ saved_async = rcu_access_pointer(memcomp_kunit_async_probe);
+ s->payload.task = current;
+ s->payload.pool = memcomp_pools[0];
+ mutex_lock(memcomp_lock_of(f->token[0]));
+ if (!mc16_view_locked(&f->survivor[0], &view)) {
+ ret = -EIO;
+ } else {
+ s->payload.handle = *view.word;
+ s->payload.live = true;
+ }
+ mutex_unlock(memcomp_lock_of(f->token[0]));
+ rcu_assign_pointer(memcomp_kunit_async_probe, &s->payload);
+ dups = atomic_read(&memcomp_kunit_slot_dups);
+ puts = atomic_read(&memcomp_kunit_slot_puts);
+ reclaims = atomic_read(&memcomp_kunit_slot_reclaims);
+ f->charged_before = atomic_long_read(&s->owner->model.charged);
+ for (i = 0; !ret && i < nr; i++) {
+ if (!mc16_get(f->token[i], &f->caller[i])) {
+ ret = -ENOENT;
+ break;
+ }
+ f->before_claim[i] = refcount_read(&f->caller[i].slot->refs);
+ mutex_lock(memcomp_lock_of(f->caller[i].token));
+ ret = mc16_writeback_stage_locked(&f->caller[i], &f->staged[i]);
+ mutex_unlock(memcomp_lock_of(f->caller[i].token));
+ f->after_claim[i] = refcount_read(&f->caller[i].slot->refs);
+ mc16_ref_put(&f->caller[i]);
+ if (!ret)
+ f->after_caller_put[i] = refcount_read(&f->staged[i].ref.slot->refs);
+ }
+ if (!ret && detach) {
+ /* Drop siblings before head, then leave only each batch guard. */
+ folio_lock(s->owner->folio);
+ for (i = nr; i; ) {
+ struct memcomp_ref lookup;
+ bool dropped;
+
+ i--;
+ if (!mc16_get(f->token[i], &lookup)) {
+ ret = -ENOENT;
+ break;
+ }
+ dropped = mc16_owner_drop_folio_held(&lookup,
+ s->owner->folio, 2, i);
+ if (!dropped) {
+ ret = -ESTALE;
+ break;
+ }
+ s->refs[i] = (struct memcomp_ref) { };
+ }
+ folio_unlock(s->owner->folio);
+ if (!ret) {
+ for (i = 0; i < nr; i++) {
+ mc16_ref_put(&f->survivor[i]);
+ f->after_detach[i] = refcount_read(&f->staged[i].ref.slot->refs);
+ }
+ }
+ }
+ f->live_before_io = s->payload.live;
+ for (i = 0; !ret && i < nr; i++)
+ ret = mc16_writeback_stage(&f->staged[i]);
+ /* Include temporaries not consumed when an earlier member failed. */
+ for (i = 0; i < nr; i++) {
+ if (f->staged[i].ref.slot)
+ mc16_writeback_stage_abort(&f->staged[i]);
+ if (f->caller[i].slot)
+ mc16_ref_put(&f->caller[i]);
+ }
+ f->flush_ret = mc16_writeback_batch_flush(&f->freed);
+ if (!ret)
+ ret = f->flush_ret;
+ f->seed_after_flush = memcomp_writeback_slots_free;
+ f->dups = atomic_read(&memcomp_kunit_slot_dups) - dups;
+ f->puts = atomic_read(&memcomp_kunit_slot_puts) - puts;
+ f->reclaims = atomic_read(&memcomp_kunit_slot_reclaims) - reclaims;
+ f->probe_frees = s->payload.frees;
+ f->live_after = s->payload.live;
+ f->bad_lifetime = s->payload.bad_lifetime;
+ f->batch_empty = !mc16_writeback_batch_used &&
+ !mc16_writeback_members_used && !mc16_writeback_open_first &&
+ !mc16_writeback_open_bytes && !memcomp_writeback_plugged;
+ memcpy(memcomp_writeback_slot_pool, f->saved_pool, sizeof(f->saved_pool));
+ memcomp_writeback_slots_free = saved_free;
+ f->pool_restored = !memcmp(memcomp_writeback_slot_pool, f->saved_pool,
+ sizeof(f->saved_pool)) &&
+ memcomp_writeback_slots_free == saved_free;
+ rcu_assign_pointer(memcomp_kunit_async_probe, saved_async);
+ f->probe_restored = rcu_access_pointer(memcomp_kunit_async_probe) == saved_async;
+ mc16_test_wbb_current = saved_current;
+unlock:
+ mutex_unlock(&memcomp_writeback_lock);
+ memalloc_noreclaim_restore(noreclaim);
+ memalloc_nofs_restore(nofs);
+ synchronize_rcu();
+ return ret;
+}
+
+static void mc16_test_wbb_drained(struct kunit *test, struct mc16_test_wbb *f,
+ unsigned int nr)
+{
+ unsigned int i;
+
+ KUNIT_EXPECT_TRUE(test, f->batch_empty);
+ KUNIT_EXPECT_TRUE(test, f->pool_restored);
+ KUNIT_EXPECT_TRUE(test, f->probe_restored);
+ KUNIT_EXPECT_EQ(test, f->seed_after_flush, 0U);
+ KUNIT_EXPECT_FALSE(test, f->bad_lifetime);
+ KUNIT_EXPECT_TRUE(test, f->live_before_io);
+ KUNIT_EXPECT_EQ(test, f->submits, f->completed);
+ for (i = 0; i < nr; i++) {
+ KUNIT_EXPECT_EQ(test, f->after_claim[i], f->before_claim[i] + 1);
+ KUNIT_EXPECT_EQ(test, f->after_caller_put[i], f->before_claim[i]);
+ KUNIT_EXPECT_PTR_EQ(test, f->staged[i].ref.slot, NULL);
+ KUNIT_EXPECT_PTR_EQ(test, f->caller[i].slot, NULL);
+ }
+ if (f->submits) {
+ KUNIT_EXPECT_TRUE(test, f->submit_contract);
+ KUNIT_EXPECT_TRUE(test, f->live_at_submit);
+ KUNIT_EXPECT_EQ(test, f->seed_after_take, 2U);
+ }
+ KUNIT_EXPECT_FALSE(test, folio_test_locked(f->refs->storage->owner->folio));
+}
+
+static void mc16_test_wbb_state(struct kunit *test, struct mc16_test_wbb *f,
+ unsigned int i, unsigned int expected,
+ bool packed, unsigned int offset)
+{
+ struct memcomp_ref *ref = &f->survivor[i];
+ struct mc16_swap_fields swap = { };
+ struct mc16_view view;
+ unsigned int state = MC16_EMPTY;
+ bool valid, decoded = false;
+
+ KUNIT_ASSERT_NOT_NULL(test, ref->slot);
+ mutex_lock(memcomp_lock_of(ref->token));
+ valid = mc16_view_locked(ref, &view);
+ if (valid) {
+ state = mc16_flags_state(*view.flags);
+ if (state == MC16_SWAPPED)
+ decoded = mc16_swap_decode(*view.word, *view.length, &swap);
+ }
+ mutex_unlock(memcomp_lock_of(ref->token));
+ KUNIT_ASSERT_TRUE(test, valid);
+ KUNIT_EXPECT_EQ(test, state, expected);
+ KUNIT_EXPECT_EQ(test, refcount_read(&ref->slot->refs), 2U);
+ if (expected == MC16_SWAPPED) {
+ KUNIT_EXPECT_TRUE(test, decoded);
+ KUNIT_EXPECT_EQ(test, swap.type, (u8)swp_type(f->used_swap));
+ KUNIT_EXPECT_EQ(test, swap.swap_offset, (u64)swp_offset(f->used_swap));
+ KUNIT_EXPECT_EQ(test, swap.byte_offset, offset);
+ KUNIT_EXPECT_EQ(test, swap.length, f->bytes[i]);
+ KUNIT_EXPECT_EQ(test, swap.packed, packed);
+ } else {
+ KUNIT_ASSERT_EQ(test, mc16_load_page(ref, f->dst), 0);
+ KUNIT_EXPECT_EQ(test, memcmp(f->src + i * PAGE_SIZE, f->dst, PAGE_SIZE), 0);
+ }
+}
+
+static void mc16_test_wbb_single(struct kunit *test, bool raw)
+{
+ struct mc16_test_wbb *f = mc16_test_wbb_begin(test, raw);
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ ret = mc16_test_wbb_run(f, 1, false);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ mc16_test_wbb_drained(test, f, 1);
+ KUNIT_EXPECT_EQ(test, f->submits, 1U);
+ KUNIT_EXPECT_EQ(test, f->freed, 1UL);
+ KUNIT_EXPECT_EQ(test, memcmp(f->images, f->src, PAGE_SIZE), 0);
+ KUNIT_EXPECT_EQ(test, f->probe_frees, 1U);
+ KUNIT_EXPECT_FALSE(test, f->live_after);
+ KUNIT_EXPECT_EQ(test, f->dups, 1);
+ KUNIT_EXPECT_EQ(test, f->puts, 0);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->refs->storage->owner->model.charged),
+ f->charged_before - f->bytes[0]);
+ mc16_test_wbb_state(test, f, 0, MC16_SWAPPED, false, 0);
+}
+
+static void memcomp_test_wbb_raw(struct kunit *test)
+{
+ mc16_test_wbb_single(test, true);
+}
+
+static void memcomp_test_wbb_unpacked_compressed(struct kunit *test)
+{
+ mc16_test_wbb_single(test, false);
+}
+
+static void memcomp_test_wbb_packed(struct kunit *test)
+{
+ struct mc16_test_wbb *f = mc16_test_wbb_begin(test, false);
+ unsigned int off, i;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_LT(test, f->bytes[0] + f->bytes[1], (u32)PAGE_SIZE);
+ ret = mc16_test_wbb_run(f, 2, false);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ mc16_test_wbb_drained(test, f, 2);
+ KUNIT_EXPECT_EQ(test, f->submits, 1U);
+ KUNIT_EXPECT_EQ(test, f->freed, 2UL);
+ KUNIT_EXPECT_EQ(test, f->dups, 2);
+ KUNIT_EXPECT_EQ(test, f->puts, 0);
+ KUNIT_EXPECT_EQ(test, f->probe_frees, 1U);
+ KUNIT_EXPECT_FALSE(test, f->live_after);
+ off = 0;
+ for (i = 0; i < 2; i++) {
+ KUNIT_EXPECT_EQ(test, memcmp(f->images + off, f->encoded[i], f->bytes[i]), 0);
+ KUNIT_ASSERT_EQ(test,
+ memcomp_decode_slot(f->images, true, off, f->bytes[i], f->dst), 0);
+ KUNIT_EXPECT_EQ(test, memcmp(f->dst, f->src + i * PAGE_SIZE, PAGE_SIZE), 0);
+ mc16_test_wbb_state(test, f, i, MC16_SWAPPED, true, off);
+ off += f->bytes[i];
+ }
+ KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->images + off, 0, PAGE_SIZE - off), NULL);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->refs->storage->owner->model.charged),
+ f->charged_before - f->bytes[0] - f->bytes[1]);
+}
+
+static void memcomp_test_wbb_owner_detach(struct kunit *test)
+{
+ struct mc16_test_wbb *f = mc16_test_wbb_begin(test, false);
+ struct memcomp_ref gone;
+ unsigned int i;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ ret = mc16_test_wbb_run(f, 2, true);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ mc16_test_wbb_drained(test, f, 2);
+ KUNIT_EXPECT_EQ(test, f->freed, 2UL);
+ KUNIT_EXPECT_EQ(test, f->reclaims, 2);
+ KUNIT_EXPECT_EQ(test, f->puts, 0);
+ KUNIT_EXPECT_EQ(test, f->probe_frees, 1U);
+ KUNIT_EXPECT_FALSE(test, f->live_after);
+ for (i = 0; i < 2; i++) {
+ KUNIT_EXPECT_EQ(test, f->after_detach[i], 1U);
+ if (mc16_get(f->token[i], &gone)) {
+ mc16_ref_put(&gone);
+ KUNIT_FAIL(test, "batch final put did not retire member %u", i);
+ }
+ }
+ /* All batch guards are gone; pressure may now reclaim metadata. */
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->refs->storage->owner->model.charged), 0L);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->refs->storage->owner->folio), 1);
+}
+
+static void memcomp_test_wbb_alloc_failure(struct kunit *test)
+{
+ struct mc16_test_wbb *f = mc16_test_wbb_begin(test, false);
+ unsigned int i;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->alloc_error = -ENOMEM;
+ ret = mc16_test_wbb_run(f, 2, false);
+ KUNIT_ASSERT_EQ(test, ret, -ENOMEM);
+ mc16_test_wbb_drained(test, f, 2);
+ KUNIT_EXPECT_EQ(test, f->alloc_calls, 1U);
+ KUNIT_EXPECT_EQ(test, f->submits, 0U);
+ KUNIT_EXPECT_EQ(test, f->freed, 0UL);
+ KUNIT_EXPECT_EQ(test, f->dups, 0);
+ KUNIT_EXPECT_EQ(test, f->puts, 0);
+ KUNIT_EXPECT_EQ(test, f->probe_frees, 0U);
+ KUNIT_EXPECT_TRUE(test, f->live_after);
+ for (i = 0; i < 2; i++)
+ mc16_test_wbb_state(test, f, i, MC16_STORED, false, 0);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->refs->storage->owner->model.charged),
+ f->charged_before);
+}
+
+static void memcomp_test_wbb_dup_failure(struct kunit *test)
+{
+ struct mc16_test_wbb *f = mc16_test_wbb_begin(test, false);
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->dup_error = -ENOMEM;
+ ret = mc16_test_wbb_run(f, 2, false);
+ KUNIT_ASSERT_EQ(test, ret, -ENOMEM);
+ mc16_test_wbb_drained(test, f, 2);
+ KUNIT_EXPECT_EQ(test, f->freed, 1UL);
+ KUNIT_EXPECT_EQ(test, f->dup_calls, 1U);
+ KUNIT_EXPECT_EQ(test, f->dups, 1);
+ /* A failed dup owns nothing: do not refund a reference it never got. */
+ KUNIT_EXPECT_EQ(test, f->puts, 0);
+ KUNIT_EXPECT_EQ(test, f->probe_frees, 1U);
+ KUNIT_EXPECT_FALSE(test, f->live_after);
+ mc16_test_wbb_state(test, f, 0, MC16_SWAPPED, true, 0);
+ mc16_test_wbb_state(test, f, 1, MC16_STORED, false, 0);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->refs->storage->owner->model.charged),
+ f->charged_before - f->bytes[0]);
+}
+
+static void memcomp_test_wbb_dirty_io(struct kunit *test)
+{
+ struct mc16_test_wbb *f = mc16_test_wbb_begin(test, false);
+ unsigned int i;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->dirty_io = true;
+ ret = mc16_test_wbb_run(f, 2, false);
+ KUNIT_ASSERT_EQ(test, ret, -EIO);
+ mc16_test_wbb_drained(test, f, 2);
+ KUNIT_EXPECT_EQ(test, f->submits, 1U);
+ KUNIT_EXPECT_EQ(test, f->freed, 0UL);
+ KUNIT_EXPECT_EQ(test, f->dups, 1);
+ KUNIT_EXPECT_EQ(test, f->puts, 1);
+ KUNIT_EXPECT_EQ(test, f->probe_frees, 0U);
+ KUNIT_EXPECT_TRUE(test, f->live_after);
+ for (i = 0; i < 2; i++)
+ mc16_test_wbb_state(test, f, i, MC16_STORED, false, 0);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->refs->storage->owner->model.charged),
+ f->charged_before);
+}
+
+static void memcomp_test_wbb_settle_reject(struct kunit *test)
+{
+ struct mc16_test_wbb *f = mc16_test_wbb_begin(test, false);
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->reject_first = true;
+ ret = mc16_test_wbb_run(f, 2, false);
+ KUNIT_ASSERT_EQ(test, ret, -ESTALE);
+ mc16_test_wbb_drained(test, f, 2);
+ KUNIT_EXPECT_EQ(test, f->freed, 1UL);
+ KUNIT_EXPECT_EQ(test, f->dups, 2);
+ KUNIT_EXPECT_EQ(test, f->puts, 1);
+ KUNIT_EXPECT_EQ(test, f->probe_frees, 0U);
+ KUNIT_EXPECT_TRUE(test, f->live_after);
+ mc16_test_wbb_state(test, f, 0, MC16_STORED, false, 0);
+ mc16_test_wbb_state(test, f, 1, MC16_SWAPPED, true, f->bytes[0]);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&f->refs->storage->owner->model.charged),
+ f->charged_before - f->bytes[1]);
+}
+#endif /* CONFIG_MEMCG && CONFIG_SWAP */
+#endif /* _MM_MEMCOMP_TESTS_WRITEBACK_BATCH_TEST_H */
diff --git a/mm/memcomp/tests/writeback_test.h b/mm/memcomp/tests/writeback_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..8c8edc3b8ff18df1fd2e79d58755e72eb12a03ea
--- /dev/null
+++ b/mm/memcomp/tests/writeback_test.h
@@ -0,0 +1,540 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_WRITEBACK_TEST_H
+#define _MM_MEMCOMP_TESTS_WRITEBACK_TEST_H
+
+#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP)
+/*
+ * Include after refs/payload/cache_store test fixtures and the WB helpers.
+ * Payload allocation, compression, load and freeing are real zsmalloc paths.
+ * objcg uses the existing identity-only charge model (not memcg enforcement);
+ * backing ownership uses memcomp_slot_* KUnit tallies, never a swap device.
+ * Interleavings below are scheduled serially at their actual T boundaries.
+ */
+struct mc16_test_writeback {
+ struct mc16_test_refs *refs;
+ struct mc16_payload prepared;
+ struct mc16_payload_tail prepare_tail;
+ struct mc16_test_cache_producer *cache;
+ struct memcomp_ref writer;
+ struct memcomp_ref pte;
+ struct memcomp_ref lookup;
+ struct mc16_writeback snapshot;
+ struct mc16_writeback_tail tail;
+ swp_entry_t backing;
+ bool backing_owned;
+ u32 bytes;
+ u8 *src;
+};
+
+/* Registered last: finish the payload tail before consuming any guard. */
+static void mc16_test_writeback_cleanup(void *data)
+{
+ struct mc16_test_writeback *f = data;
+ struct memcomp_ref *entry = f->writer.slot ? &f->writer :
+ &f->refs->storage->refs[0];
+
+ if (entry->slot) {
+ mc16_payload_finish(entry, &f->prepare_tail);
+ mc16_payload_abort(entry, &f->prepared);
+ }
+ if (f->writer.slot)
+ mc16_writeback_finish(&f->writer, &f->tail);
+ if (f->lookup.slot)
+ mc16_ref_put(&f->lookup);
+ if (f->pte.slot)
+ mc16_ref_put(&f->pte);
+ if (f->writer.slot)
+ mc16_ref_put(&f->writer);
+ if (f->backing_owned)
+ memcomp_slot_put(f->backing);
+}
+
+static struct mc16_test_writeback *
+mc16_test_writeback_begin(struct kunit *test, bool attached)
+{
+ struct mc16_test_writeback *f;
+ struct mc16_test_storage *s;
+ struct memcomp_ref *entry;
+ struct mc16_view view;
+ unsigned long handle = 0;
+ unsigned int bytes = 0, i;
+ u8 pool = 0;
+ int ret;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ if (attached) {
+ f->refs = mc16_test_refs_begin(test);
+ if (!f->refs)
+ return NULL;
+ entry = &f->refs->storage->refs[0];
+ } else {
+ f->cache = mc16_test_cache_producer_begin(test);
+ if (!f->cache)
+ return NULL;
+ f->refs = f->cache->refs;
+ entry = &f->cache->cache;
+ }
+ if (kunit_add_action_or_reset(test, mc16_test_writeback_cleanup, f))
+ return NULL;
+ s = f->refs->storage;
+ if (attached && mc16_test_refs_reserve(f->refs, 1))
+ return NULL;
+ f->src = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ if (!f->src)
+ return NULL;
+ for (i = 0; i < PAGE_SIZE; i++)
+ f->src[i] = i % 53;
+ if (attached) {
+ ret = memcomp_prepare_payload(f->src, PAGE_SIZE, &bytes, &handle,
+ &pool, MEMCOMP_PAYLOAD_MUST_SUCCEED,
+ false);
+ if (ret)
+ return NULL;
+ f->prepared = (struct mc16_payload) {
+ .word = handle, .bytes = bytes,
+ };
+ if (pool || !bytes || bytes >= PAGE_SIZE ||
+ mc16_payload_charge(entry, &f->prepared))
+ return NULL;
+ mutex_lock(memcomp_lock_of(entry->token));
+ ret = mc16_payload_commit_locked(entry, s->owner->folio, 1, 0,
+ &f->prepared, &f->prepare_tail);
+ mutex_unlock(memcomp_lock_of(entry->token));
+ if (ret)
+ return NULL;
+ mc16_payload_finish(entry, &f->prepare_tail);
+ } else if (mc16_cache_store(f->src, s->owner->folio, entry)) {
+ return NULL;
+ }
+ if (!mc16_get(entry->token, &f->writer))
+ return NULL;
+ mutex_lock(memcomp_lock_of(entry->token));
+ ret = mc16_view_locked(entry, &view) &&
+ mc16_length_decode(*view.length, &f->bytes);
+ if (ret) {
+ s->payload.handle = *view.word;
+ s->payload.pool = memcomp_pools[0];
+ s->payload.task = current;
+ s->payload.live = true;
+ /* Opaque shadow contents must survive backing-only changes. */
+ *mc16_shadow_ptr(entry) = (unsigned long)xa_mk_value(123);
+ }
+ mutex_unlock(memcomp_lock_of(entry->token));
+ if (!ret || !f->bytes || f->bytes >= PAGE_SIZE)
+ return NULL;
+ rcu_assign_pointer(memcomp_kunit_async_probe, &s->payload);
+ return f;
+}
+
+static bool mc16_test_writeback_claim(struct mc16_test_writeback *f)
+{
+ bool claimed;
+
+ mutex_lock(memcomp_lock_of(f->writer.token));
+ claimed = mc16_writeback_claim_locked(&f->writer, &f->snapshot);
+ mutex_unlock(memcomp_lock_of(f->writer.token));
+ return claimed;
+}
+
+static bool mc16_test_writeback_revert(struct mc16_test_writeback *f)
+{
+ bool reverted;
+
+ mutex_lock(memcomp_lock_of(f->writer.token));
+ reverted = mc16_writeback_revert_locked(&f->writer);
+ mutex_unlock(memcomp_lock_of(f->writer.token));
+ return reverted;
+}
+
+static int
+mc16_test_writeback_settle(struct mc16_test_writeback *f,
+ const struct mc16_swap_fields *swap)
+{
+ struct kunit *test = f->refs->storage->test;
+ struct mc16_key *key = f->writer.cluster->key;
+ u32 token = f->writer.token;
+ unsigned short owner_id = key->owner_id;
+ u64 shadow;
+ int ret;
+
+ mutex_lock(memcomp_lock_of(f->writer.token));
+ shadow = *mc16_shadow_ptr(&f->writer);
+ ret = mc16_writeback_settle_locked(&f->writer, &f->snapshot, swap,
+ &f->tail);
+ KUNIT_EXPECT_EQ(test, *mc16_shadow_ptr(&f->writer), shadow);
+ KUNIT_EXPECT_EQ(test, f->writer.token, token);
+ KUNIT_EXPECT_PTR_EQ(test, f->writer.cluster->key, key);
+ KUNIT_EXPECT_EQ(test, f->writer.cluster->key->owner_id, owner_id);
+ mutex_unlock(memcomp_lock_of(f->writer.token));
+ if (!ret)
+ f->backing_owned = false;
+ return ret;
+}
+
+static struct mc16_swap_fields
+mc16_test_writeback_backing(struct mc16_test_writeback *f, bool packed)
+{
+ struct mc16_swap_fields swap = {
+ .type = MAX_SWAPFILES - 1,
+ .swap_offset = mc16_swap_offset_limit(),
+ .length = f->bytes,
+ .byte_offset = packed ? PAGE_SIZE - f->bytes : 0,
+ .packed = packed,
+ };
+
+ f->backing = swp_entry(swap.type, swap.swap_offset);
+ memcomp_slot_dup(f->backing);
+ f->backing_owned = true;
+ return swap;
+}
+
+static void mc16_test_writeback_load(struct kunit *test,
+ struct mc16_test_writeback *f)
+{
+ void *dst = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+
+ KUNIT_ASSERT_NOT_NULL(test, dst);
+ KUNIT_ASSERT_EQ(test, mc16_load_page(&f->writer, dst), 0);
+ KUNIT_EXPECT_EQ(test, memcmp(f->src, dst, PAGE_SIZE), 0);
+}
+
+static void mc16_test_writeback_swapped(struct kunit *test,
+ struct mc16_test_writeback *f,
+ const struct mc16_swap_fields *swap,
+ bool attached)
+{
+ struct mc16_swap_fields decoded = { };
+ struct mc16_view view;
+ bool valid, has_record, owner;
+ u32 charge;
+ unsigned int state;
+
+ mutex_lock(memcomp_lock_of(f->writer.token));
+ valid = mc16_view_locked(&f->writer, &view);
+ if (valid) {
+ state = mc16_flags_state(*view.flags);
+ valid = mc16_swap_decode(*view.word, *view.length, &decoded);
+ has_record = !!view.record;
+ owner = !!(*view.flags & MC16_F_OWNER);
+ charge = view.member ? view.member->payload_charge : 0;
+ } else {
+ state = MC16_EMPTY;
+ has_record = false;
+ owner = false;
+ charge = 0;
+ }
+ mutex_unlock(memcomp_lock_of(f->writer.token));
+ KUNIT_ASSERT_TRUE(test, valid);
+ KUNIT_EXPECT_EQ(test, state, (unsigned int)MC16_SWAPPED);
+ KUNIT_EXPECT_EQ(test, has_record, attached);
+ KUNIT_EXPECT_EQ(test, owner, attached);
+ KUNIT_EXPECT_EQ(test, charge, 0U);
+ KUNIT_EXPECT_EQ(test, decoded.type, swap->type);
+ KUNIT_EXPECT_EQ(test, decoded.swap_offset, swap->swap_offset);
+ KUNIT_EXPECT_EQ(test, decoded.byte_offset, swap->byte_offset);
+ KUNIT_EXPECT_EQ(test, decoded.length, swap->length);
+ KUNIT_EXPECT_EQ(test, decoded.packed, swap->packed);
+ KUNIT_EXPECT_EQ(test, f->writer.cluster->eligible_slots, (u16)0);
+}
+
+/* Exercise real cache compressed data, including reads while claimed. */
+static void memcomp_test_writeback_cache_raw(struct kunit *test)
+{
+ struct mc16_test_writeback *f = mc16_test_writeback_begin(test, false);
+ struct mc16_test_storage *s;
+ struct mc16_swap_fields swap;
+ long charged;
+ s64 compressed, stored;
+ int dups, puts, uncharges;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ long swapped;
+ s64 packed;
+#endif
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ compressed = atomic64_read(&memcomp_compressed_bytes);
+ stored = atomic64_read(&memcomp_stored_bytes);
+ charged = atomic_long_read(&s->owner->model.charged);
+ KUNIT_ASSERT_TRUE(test, mc16_test_writeback_claim(f));
+ KUNIT_EXPECT_EQ(test, f->writer.cluster->eligible_slots, (u16)0);
+ mc16_test_writeback_load(test, f);
+ swap = mc16_test_writeback_backing(f, false);
+ dups = atomic_read(&memcomp_kunit_slot_dups);
+ puts = atomic_read(&memcomp_kunit_slot_puts);
+ uncharges = atomic_read(&s->owner->model.uncharge_calls);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ swapped = atomic_long_read(&memcomp_swapped_entries);
+ packed = atomic64_read(&memcomp_writeback_packed_payloads);
+#endif
+ KUNIT_ASSERT_EQ(test, mc16_test_writeback_settle(f, &swap), 0);
+ KUNIT_EXPECT_FALSE(test, f->backing_owned);
+ mc16_test_writeback_swapped(test, f, &swap, false);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_dups), dups);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_puts), puts);
+ KUNIT_EXPECT_EQ(test, s->payload.frees, 0U);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged), charged);
+ mc16_writeback_finish(&f->writer, &f->tail);
+ mc16_writeback_finish(&f->writer, &f->tail);
+ KUNIT_EXPECT_EQ(test, s->payload.frees, 1U);
+ KUNIT_EXPECT_FALSE(test, s->payload.bad_lifetime);
+ KUNIT_EXPECT_EQ(test, atomic_read(&s->owner->model.uncharge_calls),
+ uncharges + 1);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged),
+ charged - f->bytes);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_compressed_bytes), compressed);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_stored_bytes), stored);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&memcomp_swapped_entries), swapped + 1);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_writeback_packed_payloads),
+ packed);
+#endif
+ mc16_ref_put(&f->cache->cache);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_puts), puts);
+ mc16_ref_put(&f->writer);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_puts), puts + 1);
+ /* Final guard release retains metadata until explicit pressure. */
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged), 0L);
+ KUNIT_EXPECT_EQ(test, s->payload.frees, 1U);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&memcomp_swapped_entries), swapped);
+#endif
+}
+
+static void memcomp_test_writeback_owner_packed(struct kunit *test)
+{
+ struct mc16_test_writeback *f = mc16_test_writeback_begin(test, true);
+ struct mc16_test_storage *s;
+ struct mc16_swap_fields swap;
+ struct mc16_view view;
+ u64 physical;
+ long charged;
+ int reclaims, uncharges;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ s64 packed;
+#endif
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ /* Remember only an opaque comparison value, never dereference after T. */
+ physical = f->writer.slot->word;
+ charged = atomic_long_read(&s->owner->model.charged);
+ KUNIT_ASSERT_TRUE(test, mc16_test_writeback_claim(f));
+ mc16_test_writeback_load(test, f);
+ swap = mc16_test_writeback_backing(f, true);
+ reclaims = atomic_read(&memcomp_kunit_slot_reclaims);
+ uncharges = atomic_read(&s->owner->model.uncharge_calls);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ packed = atomic64_read(&memcomp_writeback_packed_payloads);
+#endif
+ KUNIT_ASSERT_EQ(test, mc16_test_writeback_settle(f, &swap), 0);
+ KUNIT_EXPECT_EQ(test, f->writer.slot->word, physical);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(f->writer.slot->flags),
+ (unsigned int)MC16_PENDING);
+ mc16_test_writeback_swapped(test, f, &swap, true);
+ mc16_writeback_finish(&f->writer, &f->tail);
+ mc16_writeback_finish(&f->writer, &f->tail);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged),
+ charged - f->bytes);
+ KUNIT_EXPECT_EQ(test, atomic_read(&s->owner->model.uncharge_calls),
+ uncharges + 1);
+ KUNIT_EXPECT_EQ(test, s->payload.frees, 1U);
+ mutex_lock(memcomp_lock_of(f->writer.token));
+ mc16_view_locked(&f->writer, &view);
+ KUNIT_EXPECT_EQ(test, view.member->payload_charge, 0U);
+ mutex_unlock(memcomp_lock_of(f->writer.token));
+ mc16_ref_put(&f->writer);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_reclaims), reclaims);
+ mc16_ref_put(&s->refs[0]);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_reclaims), reclaims + 1);
+ KUNIT_EXPECT_EQ(test, s->payload.frees, 1U);
+ KUNIT_EXPECT_FALSE(test, s->payload.bad_lifetime);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_writeback_packed_payloads),
+ packed + 1);
+#endif
+}
+
+static void memcomp_test_writeback_revert(struct kunit *test)
+{
+ struct mc16_test_writeback *f = mc16_test_writeback_begin(test, false);
+ struct mc16_test_storage *s;
+ struct mc16_writeback snapshot;
+ struct memcomp_lru *fifo;
+ long charged, eligible;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ fifo = mc16_fifo_of(&f->writer);
+ eligible = atomic_long_read(&fifo->nr_entries);
+ charged = atomic_long_read(&s->owner->model.charged);
+ KUNIT_ASSERT_TRUE(test, mc16_test_writeback_claim(f));
+ snapshot = f->snapshot;
+ KUNIT_EXPECT_FALSE(test, mc16_test_writeback_claim(f));
+ KUNIT_EXPECT_PTR_EQ(test, f->snapshot.slot, snapshot.slot);
+ KUNIT_EXPECT_EQ(test, f->snapshot.word, snapshot.word);
+ KUNIT_EXPECT_EQ(test, f->snapshot.length, snapshot.length);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&fifo->nr_entries), eligible - 1);
+ KUNIT_EXPECT_TRUE(test, mc16_test_writeback_revert(f));
+ KUNIT_EXPECT_FALSE(test, mc16_test_writeback_revert(f));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&fifo->nr_entries), eligible);
+ KUNIT_EXPECT_EQ(test, f->writer.cluster->eligible_slots, (u16)1);
+ KUNIT_EXPECT_EQ(test, f->writer.slot->word, snapshot.word);
+ KUNIT_EXPECT_EQ(test, f->writer.slot->length, snapshot.length);
+ KUNIT_EXPECT_EQ(test, s->payload.frees, 0U);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged), charged);
+ mc16_test_writeback_load(test, f);
+}
+
+/* Mutate the staged identity, keeping the actual zs handle valid throughout. */
+static void memcomp_test_writeback_mismatch(struct kunit *test)
+{
+ struct mc16_test_writeback *f = mc16_test_writeback_begin(test, false);
+ struct mc16_test_storage *s;
+ struct mc16_swap_fields swap;
+ int puts, dups, uncharges;
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ swap = mc16_test_writeback_backing(f, true);
+ puts = atomic_read(&memcomp_kunit_slot_puts);
+ dups = atomic_read(&memcomp_kunit_slot_dups);
+ uncharges = atomic_read(&s->owner->model.uncharge_calls);
+ for (i = 0; i < 3; i++) {
+ KUNIT_ASSERT_TRUE(test, mc16_test_writeback_claim(f));
+ if (i == 0)
+ f->snapshot.word ^= 1;
+ else if (i == 1)
+ f->snapshot.length ^= 1;
+ else
+ KUNIT_ASSERT_TRUE(test, mc16_test_writeback_revert(f));
+ KUNIT_EXPECT_EQ(test, mc16_test_writeback_settle(f, &swap), -ESTALE);
+ KUNIT_EXPECT_TRUE(test, f->backing_owned);
+ KUNIT_EXPECT_EQ(test, f->tail.word, 0ULL);
+ KUNIT_EXPECT_EQ(test, f->tail.charge, 0U);
+ KUNIT_EXPECT_EQ(test, f->writer.cluster->eligible_slots, (u16)1);
+ mc16_test_writeback_load(test, f);
+ }
+ KUNIT_EXPECT_EQ(test, s->payload.frees, 0U);
+ KUNIT_EXPECT_EQ(test, atomic_read(&s->owner->model.uncharge_calls), uncharges);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_puts), puts);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_dups), dups);
+ /* Caller can still return exactly its own unconsumed reference. */
+ memcomp_slot_put(f->backing);
+ f->backing_owned = false;
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_puts), puts + 1);
+}
+
+static void memcomp_test_writeback_encoding(struct kunit *test)
+{
+ struct mc16_test_writeback *f = mc16_test_writeback_begin(test, false);
+ struct mc16_swap_fields swap, bad;
+ unsigned int i;
+ int puts;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_test_writeback_claim(f));
+ swap = mc16_test_writeback_backing(f, true);
+ puts = atomic_read(&memcomp_kunit_slot_puts);
+ for (i = 0; i < 8; i++) {
+ bad = swap;
+ switch (i) {
+ case 0:
+ bad.swap_offset++;
+ break;
+ case 1:
+ bad.type = MAX_SWAPFILES;
+ break;
+ case 2:
+ bad.byte_offset = PAGE_SIZE;
+ break;
+ case 3:
+ bad.byte_offset++; /* packed payload crosses backing page */
+ break;
+ case 4:
+ bad.packed = false; /* raw requires byte_offset == 0 */
+ break;
+ case 5:
+ bad.length = 0;
+ break;
+ case 6:
+ bad.length = PAGE_SIZE;
+ break;
+ default:
+ bad.type = 0;
+ bad.swap_offset = 0; /* zero backing is never transferable */
+ break;
+ }
+ KUNIT_EXPECT_EQ(test, mc16_test_writeback_settle(f, &bad), -EINVAL);
+ KUNIT_EXPECT_TRUE(test, f->backing_owned);
+ KUNIT_EXPECT_EQ(test, f->tail.word, 0ULL);
+ KUNIT_EXPECT_EQ(test, f->writer.slot->word, f->snapshot.word);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(f->writer.slot->flags),
+ (unsigned int)MC16_WRITEBACK);
+ }
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_puts), puts);
+ KUNIT_EXPECT_EQ(test, f->refs->storage->payload.frees, 0U);
+ /* Maximum type/offset and a payload ending exactly at PAGE_SIZE. */
+ KUNIT_ASSERT_EQ(test, mc16_test_writeback_settle(f, &swap), 0);
+ mc16_test_writeback_swapped(test, f, &swap, false);
+}
+
+/* OWNER normalization frees the record before WB settles using the guard. */
+static void memcomp_test_writeback_last_pte(struct kunit *test)
+{
+ struct mc16_test_writeback *f = mc16_test_writeback_begin(test, true);
+ struct mc16_test_storage *s;
+ struct mc16_swap_fields swap;
+ struct mc16_view view;
+ long charged;
+ u64 record_charge;
+ u32 token;
+ int reclaims, uncharges;
+ bool dropped;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ s = f->refs->storage;
+ token = f->writer.token;
+ KUNIT_ASSERT_TRUE(test, mc16_get(token, &f->pte));
+ KUNIT_ASSERT_TRUE(test, mc16_get(token, &f->lookup));
+ KUNIT_ASSERT_TRUE(test, mc16_test_writeback_claim(f));
+ mutex_lock(memcomp_lock_of(token));
+ mc16_view_locked(&f->writer, &view);
+ record_charge = view.record->charge_bytes;
+ mutex_unlock(memcomp_lock_of(token));
+ charged = atomic_long_read(&s->owner->model.charged);
+ dropped = mc16_owner_drop_folio_held(&f->lookup, s->owner->folio, 1, 0);
+ if (dropped)
+ s->refs[0] = (struct memcomp_ref) { };
+ KUNIT_ASSERT_TRUE(test, dropped);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged),
+ charged - record_charge);
+ /* This is the last persistent PTE reference; writer is still live. */
+ mc16_ref_put(&f->pte);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->writer.slot->refs), 1U);
+ KUNIT_EXPECT_EQ(test, f->writer.slot->word, f->snapshot.word);
+ KUNIT_EXPECT_EQ(test, s->payload.frees, 0U);
+ mc16_test_writeback_load(test, f);
+ swap = mc16_test_writeback_backing(f, true);
+ reclaims = atomic_read(&memcomp_kunit_slot_reclaims);
+ uncharges = atomic_read(&s->owner->model.uncharge_calls);
+ KUNIT_ASSERT_EQ(test, mc16_test_writeback_settle(f, &swap), 0);
+ mc16_test_writeback_swapped(test, f, &swap, false);
+ mc16_writeback_finish(&f->writer, &f->tail);
+ mc16_writeback_finish(&f->writer, &f->tail);
+ KUNIT_EXPECT_EQ(test, atomic_read(&s->owner->model.uncharge_calls),
+ uncharges + 1);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&s->owner->model.charged),
+ charged - record_charge - f->bytes);
+ mc16_ref_put(&f->writer);
+ KUNIT_EXPECT_EQ(test, atomic_read(&memcomp_kunit_slot_reclaims), reclaims + 1);
+ KUNIT_EXPECT_EQ(test, s->payload.frees, 1U);
+ KUNIT_EXPECT_FALSE(test, s->payload.bad_lifetime);
+ KUNIT_EXPECT_FALSE(test, mc16_get(token, &f->lookup));
+}
+#endif /* CONFIG_MEMCG && CONFIG_SWAP */
+#endif /* _MM_MEMCOMP_TESTS_WRITEBACK_TEST_H */
diff --git a/mm/memcomp/tests/zero_callers_test.h b/mm/memcomp/tests/zero_callers_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..fd2d5ecbc3b88f8ab30278d3cf562dc3f8ec4b87
--- /dev/null
+++ b/mm/memcomp/tests/zero_callers_test.h
@@ -0,0 +1,717 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_ZERO_CALLERS_TEST_H
+#define _MM_MEMCOMP_TESTS_ZERO_CALLERS_TEST_H
+
+#include <kunit/resource.h>
+#include <kunit/test.h>
+#include <linux/highmem.h>
+#include <linux/mman.h>
+#include <linux/sched/mm.h>
+
+/*
+ * Include in suite.h AFTER BOTH zero_load_test.h and
+ * writeback_batch_test.h. The first supplies real MM actors;
+ * under MEMCG the second supplies an unlocked, real zs/accounting sentinel.
+ * Public APIs and stat_show() must already be visible from memcomp.c.
+ *
+ * Register ALL cases under MC16_ZERO_CALLERS_TEST_ENABLED, outside any
+ * CONFIG_MEMCG-only block. It requires MEMCOMP_KUNIT_TEST, MMU, SWAP and
+ * ADVISE_SYSCALLS, and supports MEMCG=n/y. Optional PTE protection bits skip
+ * only their corresponding cases when the architecture/config lacks them.
+ *
+ * Requires zero-callers.patch plus the transport-release fix in live source.
+ * get(0) adds a PTE owner; get_ref(0)/cache pin(0) reject with an empty ref.
+ * Cache zero uses memcomp_load_folio_zero() and a later explicit cache drop;
+ * there is no sentinel guard. No fallback public dispatch is supplied here.
+ * Public get/put actors model fork/zap ownership under a PTL. Fault actors use
+ * real MM/PTE/rmap/memcg through memcomp_load(), not the internal zero loader.
+ */
+#if defined(MC16_ZERO_LOAD_MM_TEST_ENABLED) && defined(CONFIG_SWAP)
+#define MC16_ZERO_CALLERS_TEST_ENABLED
+
+struct mc16_test_zcall {
+ struct kunit *test;
+ struct mc16_zl_mm watch;
+ struct mc16_zero_owners owners;
+ struct memcomp_ref guard;
+ unsigned long unexpected_cache[2];
+ struct folio *transport;
+ struct folio *large;
+ char *stat;
+#ifdef CONFIG_MEMCG
+ struct mc16_test_wbb *wb;
+#endif
+};
+
+/*
+ * Tests own a private delta in a quiescent ledger. Unwind that actual delta
+ * after an early assertion, even if a broken public API failed to account.
+ * Negative deltas are failures too; repair them so later cases stay isolated.
+ * No production reset, global atomic, or inherited test-task lock is used.
+ */
+static void mc16_test_zcall_cleanup(void *data)
+{
+ struct mc16_test_zcall *f = data;
+ struct mc16_zero_owners after;
+ s64 pte, cache;
+ unsigned int i;
+
+ memcomp_load_folio_abort(&f->guard);
+ for (i = 0; i < ARRAY_SIZE(f->unexpected_cache); i++) {
+ if (f->unexpected_cache[i])
+ memcomp_invalidate_cache(f->unexpected_cache[i]);
+ }
+ mc16_zero_owners_read(&after);
+ pte = after.pte_pages - f->owners.pte_pages;
+ cache = after.cache_pages - f->owners.cache_pages;
+ KUNIT_EXPECT_GE(f->test, pte, 0LL);
+ KUNIT_EXPECT_GE(f->test, cache, 0LL);
+ if (pte > 0)
+ mc16_zero_pte_put(pte);
+ else if (pte < 0)
+ mc16_zero_pte_init(-pte);
+ if (cache > 0)
+ mc16_zero_cache_put(cache);
+ else if (cache < 0)
+ mc16_zero_cache_get(-cache);
+ mc16_zero_owners_read(&after);
+ KUNIT_EXPECT_EQ(f->test, after.pte_pages, f->owners.pte_pages);
+ KUNIT_EXPECT_EQ(f->test, after.cache_pages, f->owners.cache_pages);
+ mc16_zl_storage_unchanged(&f->watch);
+ if (f->large) {
+ KUNIT_EXPECT_FALSE(f->test, folio_test_locked(f->large));
+ f->large->swap.val = 0;
+ folio_put(f->large);
+ }
+ if (f->transport) {
+ KUNIT_EXPECT_FALSE(f->test, folio_test_locked(f->transport));
+ f->transport->swap.val = 0;
+ folio_put(f->transport);
+ }
+ if (f->stat)
+ free_page((unsigned long)f->stat);
+}
+
+static struct mc16_test_zcall *mc16_test_zcall_begin(struct kunit *test)
+{
+ struct mc16_test_zcall *f;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->test = test;
+ f->watch.test = test;
+#ifdef CONFIG_MEMCG
+ /* This fixture returns its source folio unlocked, including on error. */
+ f->wb = mc16_test_wbb_begin(test, false);
+ if (!f->wb)
+ return NULL;
+ f->watch.refs = f->wb->refs;
+#endif
+ mc16_zero_owners_read(&f->owners);
+ mc16_zl_storage_read(&f->watch, &f->watch.storage);
+ if (kunit_add_action_or_reset(test, mc16_test_zcall_cleanup, f))
+ return NULL;
+ return f;
+}
+
+static void mc16_test_zcall_counts(struct mc16_test_zcall *f, s64 pte, s64 cache)
+{
+ struct mc16_zero_owners after;
+
+ mc16_zero_owners_read(&after);
+ KUNIT_EXPECT_EQ(f->test, after.pte_pages, f->owners.pte_pages + pte);
+ KUNIT_EXPECT_EQ(f->test, after.cache_pages, f->owners.cache_pages + cache);
+ mc16_zl_storage_unchanged(&f->watch);
+}
+
+static void memcomp_test_zero_callers_fork_get(struct kunit *test)
+{
+ struct mc16_test_zcall *f = mc16_test_zcall_begin(test);
+ struct mc16_zero_diagnostics before, after;
+ spinlock_t ptl; /* Models parent/child publication, not a fork syscall. */
+ unsigned long flags;
+ bool first, second;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_zero_diagnostics_read(&before);
+ KUNIT_ASSERT_TRUE(test, mc16_zero_pte_init(1));
+ KUNIT_ASSERT_TRUE(test, mc16_zero_cache_get(2));
+ spin_lock_init(&ptl);
+ spin_lock_irqsave(&ptl, flags);
+ first = memcomp_entry_get(0);
+ second = memcomp_entry_get(0);
+ spin_unlock_irqrestore(&ptl, flags);
+ KUNIT_EXPECT_TRUE(test, first);
+ KUNIT_EXPECT_TRUE(test, second);
+ mc16_test_zcall_counts(f, 3, 2);
+ mc16_zero_diagnostics_read(&after);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ KUNIT_EXPECT_EQ(test, after.calls[MC16_ZERO_PTE_INIT] -
+ before.calls[MC16_ZERO_PTE_INIT], 1ULL);
+ KUNIT_EXPECT_EQ(test, after.pages[MC16_ZERO_PTE_GET] -
+ before.pages[MC16_ZERO_PTE_GET], 2ULL);
+ KUNIT_EXPECT_EQ(test, after.calls[MC16_ZERO_PTE_GET] -
+ before.calls[MC16_ZERO_PTE_GET], 2ULL);
+#endif
+ memcomp_invalidate_nr(0, 3);
+ memcomp_invalidate_cache(0);
+ memcomp_invalidate_cache(0);
+ mc16_test_zcall_counts(f, 0, 0);
+}
+
+static void memcomp_test_zero_callers_put_domains(struct kunit *test)
+{
+ struct mc16_test_zcall *f = mc16_test_zcall_begin(test);
+ struct mc16_zero_diagnostics before, after, noop;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_zero_pte_init(5));
+ KUNIT_ASSERT_TRUE(test, mc16_zero_cache_get(3));
+ mc16_zero_diagnostics_read(&before);
+ memcomp_invalidate(0);
+ mc16_test_zcall_counts(f, 4, 3);
+ memcomp_invalidate_nr(0, 3);
+ mc16_test_zcall_counts(f, 1, 3);
+ memcomp_invalidate_cache(0);
+ mc16_test_zcall_counts(f, 1, 2);
+ mc16_zero_diagnostics_read(&after);
+ memcomp_invalidate_nr(0, 0);
+ mc16_zero_diagnostics_read(&noop);
+ KUNIT_EXPECT_EQ(test, memcmp(&after, &noop, sizeof(after)), 0);
+ mc16_test_zcall_counts(f, 1, 2);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ KUNIT_EXPECT_EQ(test, after.calls[MC16_ZERO_PTE_PUT] -
+ before.calls[MC16_ZERO_PTE_PUT], 2ULL);
+ KUNIT_EXPECT_EQ(test, after.pages[MC16_ZERO_PTE_PUT] -
+ before.pages[MC16_ZERO_PTE_PUT], 4ULL);
+ KUNIT_EXPECT_EQ(test, after.pages[MC16_ZERO_CACHE_PUT] -
+ before.pages[MC16_ZERO_CACHE_PUT], 1ULL);
+#endif
+ memcomp_invalidate(0);
+ memcomp_invalidate_cache(0);
+ memcomp_invalidate_cache(0);
+ mc16_test_zcall_counts(f, 0, 0);
+}
+
+static void memcomp_test_zero_callers_put_bounds(struct kunit *test)
+{
+ static const unsigned int sizes[] = { 1, 2, 511, 512, 513, UINT_MAX };
+ struct mc16_test_zcall *f = mc16_test_zcall_begin(test);
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, mc16_zero_cache_get(1));
+ for (i = 0; i < ARRAY_SIZE(sizes); i++) {
+ unsigned int nr = sizes[i];
+
+ /* On 32-bit, UINT_MAX exceeds the zero ledger's signed range. */
+ if (!mc16_zero_nr_valid(nr))
+ continue;
+ KUNIT_ASSERT_TRUE(test, mc16_zero_pte_init(nr));
+ mc16_test_zcall_counts(f, nr, 1);
+ memcomp_invalidate_nr(0, nr);
+ mc16_test_zcall_counts(f, 0, 1);
+ }
+ memcomp_invalidate_cache(0);
+ mc16_test_zcall_counts(f, 0, 0);
+}
+
+static void memcomp_test_zero_callers_query(struct kunit *test)
+{
+ struct mc16_test_zcall *f = mc16_test_zcall_begin(test);
+ spinlock_t ptl; /* task_mmu/shmem-style query must not sleep under PTL. */
+ unsigned long flags, base = ULONG_MAX, compressed = PAGE_SIZE;
+ int mapcount = -1;
+ unsigned int nr;
+ bool present, stat;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ spin_lock_init(&ptl);
+ spin_lock_irqsave(&ptl, flags);
+ present = memcomp_entry_present(0);
+ stat = memcomp_entry_stat(0, NULL, &mapcount);
+ nr = memcomp_entry_nr_pages(0, &base);
+ spin_unlock_irqrestore(&ptl, flags);
+ KUNIT_EXPECT_TRUE(test, present);
+ KUNIT_EXPECT_TRUE(test, stat);
+ KUNIT_EXPECT_EQ(test, mapcount, 1);
+ KUNIT_EXPECT_EQ(test, nr, 1U);
+ KUNIT_EXPECT_EQ(test, base, 0UL);
+ KUNIT_EXPECT_EQ(test, memcomp_entry_nr_pages(0, NULL), 1U);
+ KUNIT_EXPECT_TRUE(test, memcomp_entry_stat(0, &compressed, &mapcount));
+ KUNIT_EXPECT_EQ(test, compressed, 0UL);
+ KUNIT_EXPECT_EQ(test, mapcount, 1);
+ KUNIT_EXPECT_TRUE(test, memcomp_entry_stat(0, NULL, NULL));
+ KUNIT_EXPECT_TRUE(test, memcomp_entry_stat(0, &compressed, NULL));
+ KUNIT_EXPECT_FALSE(test, memcomp_entry_get_ref(0, &f->guard));
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.slot, NULL);
+ KUNIT_EXPECT_FALSE(test, memcomp_load_folio_pin(0, &f->guard));
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.slot, NULL);
+ memcomp_load_folio_abort(&f->guard);
+ mc16_test_zcall_counts(f, 0, 0);
+}
+
+static void memcomp_test_zero_callers_stat(struct kunit *test)
+{
+ struct mc16_test_zcall *f = mc16_test_zcall_begin(test);
+ unsigned long long stored, compressed, pool;
+ u64 expected, pool_before;
+ ssize_t len;
+ int fields;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ /* sysfs_emit requires a page-aligned buffer. */
+ f->stat = (char *)get_zeroed_page(GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, f->stat);
+ KUNIT_ASSERT_TRUE(test, mc16_zero_pte_init(2));
+ KUNIT_ASSERT_TRUE(test, mc16_zero_cache_get(3));
+ expected = mc16_zero_stored_bytes(atomic64_read(&memcomp_stored_bytes));
+ pool_before = memcomp_pool_total_size();
+ len = stat_show(NULL, NULL, f->stat);
+ KUNIT_ASSERT_GT(test, len, (ssize_t)0);
+ fields = sscanf(f->stat, "%llu %llu %llu", &stored, &compressed, &pool);
+ KUNIT_ASSERT_EQ(test, fields, 3);
+ KUNIT_EXPECT_EQ(test, (u64)stored, expected);
+ KUNIT_EXPECT_EQ(test, (u64)compressed, (u64)f->watch.storage.compressed);
+ KUNIT_EXPECT_EQ(test, (u64)pool, pool_before);
+ mc16_test_zcall_counts(f, 2, 3);
+ memcomp_invalidate_nr(0, 2);
+ memcomp_invalidate_cache(0);
+ memcomp_invalidate_cache(0);
+ memcomp_invalidate_cache(0);
+ mc16_test_zcall_counts(f, 0, 0);
+}
+
+/* Public batch accepts an encoded PTE array; no mapping or owners are made. */
+static unsigned int mc16_test_zcall_batch(pte_t *ptes, unsigned int nr)
+{
+ spinlock_t ptl; /* Serializes reads of the encoded PTE array. */
+ unsigned long flags;
+ unsigned int result;
+
+ spin_lock_init(&ptl);
+ spin_lock_irqsave(&ptl, flags);
+ result = memcomp_pte_batch(ptes, nr);
+ spin_unlock_irqrestore(&ptl, flags);
+ return result;
+}
+
+static void memcomp_test_zero_callers_batch_zero(struct kunit *test)
+{
+ struct mc16_test_zcall *f = mc16_test_zcall_begin(test);
+ pte_t ptes[6];
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (i = 0; i < ARRAY_SIZE(ptes); i++)
+ ptes[i] = mc16_zl_zero_pte();
+ KUNIT_EXPECT_EQ(test, mc16_test_zcall_batch(NULL, 0), 0U);
+ KUNIT_EXPECT_EQ(test, mc16_test_zcall_batch(ptes, 6), 6U);
+ KUNIT_EXPECT_EQ(test, mc16_test_zcall_batch(ptes, 1), 1U);
+ KUNIT_EXPECT_EQ(test, mc16_test_zcall_batch(ptes, 3), 3U);
+ ptes[2] = swp_entry_to_pte(make_memcomp_entry(1));
+ KUNIT_EXPECT_EQ(test, mc16_test_zcall_batch(ptes, 6), 2U);
+ ptes[2] = swp_entry_to_pte(swp_entry(0, 7));
+ KUNIT_EXPECT_EQ(test, mc16_test_zcall_batch(ptes, 6), 2U);
+ ptes[2] = __pte(0);
+ KUNIT_EXPECT_EQ(test, mc16_test_zcall_batch(ptes, 6), 2U);
+ KUNIT_EXPECT_EQ(test, mc16_test_zcall_batch(&ptes[2], 1), 0U);
+ mc16_test_zcall_counts(f, 0, 0);
+}
+
+static void memcomp_test_zero_callers_batch_positive(struct kunit *test)
+{
+ struct mc16_test_zcall *f = mc16_test_zcall_begin(test);
+ pte_t ptes[4];
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (i = 0; i < ARRAY_SIZE(ptes); i++)
+ ptes[i] = swp_entry_to_pte(make_memcomp_entry(100 + i));
+ KUNIT_EXPECT_EQ(test, mc16_test_zcall_batch(ptes, 4), 4U);
+ ptes[2] = mc16_zl_zero_pte();
+ KUNIT_EXPECT_EQ(test, mc16_test_zcall_batch(ptes, 4), 2U);
+ ptes[1] = ptes[0];
+ KUNIT_EXPECT_EQ(test, mc16_test_zcall_batch(ptes, 4), 1U);
+ mc16_test_zcall_counts(f, 0, 0);
+}
+
+static void mc16_test_zcall_batch_protection(struct kunit *test, unsigned int bit)
+{
+ struct mc16_test_zcall *f;
+ pte_t plain = mc16_zl_zero_pte(), protected;
+ pte_t ptes[4];
+ unsigned int i;
+
+ if (!bit)
+ protected = pte_swp_mkexclusive(plain);
+ else if (bit == 1)
+ protected = pte_swp_mksoft_dirty(plain);
+ else
+ protected = pte_swp_mkuffd_wp(plain);
+ if (pte_same(plain, protected)) {
+ kunit_skip(test, "requested swap protection bit is disabled");
+ return;
+ }
+ f = mc16_test_zcall_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (i = 0; i < ARRAY_SIZE(ptes); i++)
+ ptes[i] = plain;
+ ptes[2] = protected;
+ KUNIT_EXPECT_EQ(test, mc16_test_zcall_batch(ptes, 4), 2U);
+ for (i = 0; i < ARRAY_SIZE(ptes); i++)
+ ptes[i] = protected;
+ KUNIT_EXPECT_EQ(test, mc16_test_zcall_batch(ptes, 4), 4U);
+ ptes[2] = plain;
+ KUNIT_EXPECT_EQ(test, mc16_test_zcall_batch(ptes, 4), 2U);
+ mc16_test_zcall_counts(f, 0, 0);
+}
+
+static void memcomp_test_zero_callers_batch_exclusive(struct kunit *test)
+{
+ mc16_test_zcall_batch_protection(test, 0);
+}
+
+static void memcomp_test_zero_callers_batch_soft_dirty(struct kunit *test)
+{
+ mc16_test_zcall_batch_protection(test, 1);
+}
+
+static void memcomp_test_zero_callers_batch_uffd_wp(struct kunit *test)
+{
+ mc16_test_zcall_batch_protection(test, 2);
+}
+
+static int mc16_test_zcall_cache_load(struct folio *folio, bool lock)
+{
+ int ret;
+
+ if (lock)
+ folio_lock(folio);
+ ret = memcomp_load_folio_zero(folio);
+ if (lock)
+ folio_unlock(folio);
+ return ret;
+}
+
+static bool mc16_test_zcall_bytes(struct folio *folio, u8 value)
+{
+ void *data = kmap_local_folio(folio, 0);
+ bool same = !memchr_inv(data, value, PAGE_SIZE);
+
+ kunmap_local(data);
+ return same;
+}
+
+static void mc16_test_zcall_fill(struct folio *folio)
+{
+ void *data = kmap_local_folio(folio, 0);
+
+ memset(data, 0xa7, PAGE_SIZE);
+ kunmap_local(data);
+ folio_clear_uptodate(folio);
+}
+
+/* Public preparation owns nothing; explicit publication owns one cache item. */
+static void memcomp_test_zero_callers_cache_publish(struct kunit *test)
+{
+ struct mc16_test_zcall *f = mc16_test_zcall_begin(test);
+ unsigned long token = ULONG_MAX;
+ unsigned int i;
+ bool prepared;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->transport = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0);
+ KUNIT_ASSERT_NOT_NULL(test, f->transport);
+ /* Two preparations model a discarded result and the eventual winner. */
+ for (i = 0; i < 2; i++) {
+ folio_lock(f->transport);
+ prepared = memcomp_store_cache(f->transport, &token);
+ folio_unlock(f->transport);
+ /* Register each actual owner before an assertion or overwrite. */
+ if (prepared && token)
+ f->unexpected_cache[i] = token;
+ KUNIT_ASSERT_TRUE(test, prepared);
+ KUNIT_EXPECT_EQ(test, token, 0UL);
+ mc16_test_zcall_counts(f, 0, 0);
+ }
+ /* The caller models publication; this is not a shmem XA-race test. */
+ memcomp_publish_cache_zero();
+ mc16_test_zcall_counts(f, 0, 1);
+ KUNIT_EXPECT_FALSE(test, memcomp_load_folio_pin(token, &f->guard));
+ KUNIT_EXPECT_NULL(test, f->guard.slot);
+ mc16_test_zcall_fill(f->transport);
+ KUNIT_ASSERT_EQ(test, mc16_test_zcall_cache_load(f->transport, true), 0);
+ KUNIT_EXPECT_TRUE(test, mc16_test_zcall_bytes(f->transport, 0));
+ mc16_test_zcall_counts(f, 0, 1);
+ memcomp_load_folio_abort(&f->guard);
+ mc16_test_zcall_counts(f, 0, 1);
+ memcomp_invalidate_cache(token);
+ if (token)
+ f->unexpected_cache[1] = 0;
+ mc16_test_zcall_counts(f, 0, 0);
+}
+
+static void memcomp_test_zero_callers_cache_load(struct kunit *test)
+{
+ struct mc16_test_zcall *f = mc16_test_zcall_begin(test);
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->transport = folio_alloc(GFP_KERNEL, 0);
+ KUNIT_ASSERT_NOT_NULL(test, f->transport);
+ f->transport->swap.val = 0;
+ KUNIT_ASSERT_TRUE(test, mc16_zero_cache_get(1));
+ KUNIT_ASSERT_TRUE(test, mc16_zero_pte_init(2));
+ KUNIT_EXPECT_FALSE(test, memcomp_load_folio_pin(0, &f->guard));
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.slot, NULL);
+ /* First xa replacement loses: retain the same logical cache owner. */
+ for (i = 0; i < 2; i++) {
+ mc16_test_zcall_fill(f->transport);
+ KUNIT_ASSERT_EQ(test, mc16_test_zcall_cache_load(f->transport, true), 0);
+ KUNIT_EXPECT_TRUE(test, folio_test_uptodate(f->transport));
+ KUNIT_EXPECT_TRUE(test, mc16_test_zcall_bytes(f->transport, 0));
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->transport), 1);
+ KUNIT_EXPECT_EQ(test, f->transport->swap.val, 0UL);
+ mc16_test_zcall_counts(f, 2, 1);
+ memcomp_load_folio_abort(&f->guard);
+ mc16_test_zcall_counts(f, 2, 1);
+ }
+ /* Model the winner's explicit cache drop only after replacing its xa. */
+ memcomp_invalidate_cache(0);
+ mc16_test_zcall_counts(f, 2, 0);
+ memcomp_invalidate_nr(0, 2);
+ mc16_test_zcall_counts(f, 0, 0);
+}
+
+static void memcomp_test_zero_callers_cache_invalid(struct kunit *test)
+{
+ struct mc16_test_zcall *f = mc16_test_zcall_begin(test);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->transport = folio_alloc(GFP_KERNEL, 0);
+ f->large = folio_alloc(GFP_KERNEL, 1);
+ KUNIT_ASSERT_NOT_NULL(test, f->transport);
+ KUNIT_ASSERT_NOT_NULL(test, f->large);
+ KUNIT_ASSERT_TRUE(test, mc16_zero_cache_get(1));
+ mc16_test_zcall_fill(f->transport);
+ KUNIT_EXPECT_EQ(test, mc16_test_zcall_cache_load(NULL, false), -EINVAL);
+ KUNIT_EXPECT_EQ(test, mc16_test_zcall_cache_load(f->transport, false), -EINVAL);
+ KUNIT_EXPECT_EQ(test, mc16_test_zcall_cache_load(f->large, true), -EINVAL);
+ KUNIT_EXPECT_FALSE(test, folio_test_uptodate(f->transport));
+ KUNIT_EXPECT_TRUE(test, mc16_test_zcall_bytes(f->transport, 0xa7));
+ mc16_test_zcall_counts(f, 0, 1);
+ memcomp_invalidate_cache(0);
+ mc16_test_zcall_counts(f, 0, 0);
+}
+
+/* Pair a successful freeze with unfreeze before returning to assertions. */
+static bool mc16_test_zcall_release(struct folio *folio, bool lock)
+{
+ bool released;
+
+ if (lock)
+ folio_lock(folio);
+ released = memcomp_release_folio(folio);
+ if (released)
+ folio_ref_unfreeze(folio, 1);
+ if (lock)
+ folio_unlock(folio);
+ return released;
+}
+
+static void memcomp_test_zero_callers_release(struct kunit *test)
+{
+ struct mc16_test_zcall *f = mc16_test_zcall_begin(test);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->transport = folio_alloc(GFP_KERNEL, 0);
+ KUNIT_ASSERT_NOT_NULL(test, f->transport);
+ KUNIT_ASSERT_TRUE(test, mc16_zero_pte_init(1));
+ KUNIT_ASSERT_TRUE(test, mc16_zero_cache_get(1));
+ f->transport->swap = make_memcomp_entry(0);
+ KUNIT_ASSERT_FALSE(test, folio_test_anon(f->transport));
+ KUNIT_ASSERT_PTR_EQ(test, f->transport->mapping, NULL);
+ KUNIT_ASSERT_TRUE(test, folio_memcomp_reserved(f->transport));
+ KUNIT_EXPECT_FALSE(test, mc16_test_zcall_release(f->transport, false));
+ KUNIT_EXPECT_TRUE(test, folio_memcomp_reserved(f->transport));
+ folio_get(f->transport);
+ KUNIT_EXPECT_FALSE(test, mc16_test_zcall_release(f->transport, true));
+ folio_put(f->transport);
+ KUNIT_EXPECT_TRUE(test, folio_memcomp_reserved(f->transport));
+ KUNIT_EXPECT_TRUE(test, mc16_test_zcall_release(f->transport, true));
+ KUNIT_EXPECT_EQ(test, f->transport->swap.val, 0UL);
+ KUNIT_EXPECT_EQ(test, folio_ref_count(f->transport), 1);
+ mc16_test_zcall_counts(f, 1, 1);
+ memcomp_invalidate(0);
+ memcomp_invalidate_cache(0);
+ mc16_test_zcall_counts(f, 0, 0);
+}
+
+/* Only the eligibility predicate sees this unlinked mapping, under the lock. */
+static bool mc16_test_zcall_release_mapped(struct folio *folio,
+ struct address_space *mapping)
+{
+ bool released;
+
+ folio_lock(folio);
+ folio->mapping = mapping;
+ released = memcomp_release_folio(folio);
+ if (released)
+ folio_ref_unfreeze(folio, 1);
+ folio->mapping = NULL;
+ folio_unlock(folio);
+ return released;
+}
+
+static void memcomp_test_zero_callers_release_bounds(struct kunit *test)
+{
+ struct mc16_test_zcall *f = mc16_test_zcall_begin(test);
+ struct address_space mapping = { };
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->transport = folio_alloc(GFP_KERNEL, 0);
+ f->large = folio_alloc(GFP_KERNEL, 1);
+ KUNIT_ASSERT_NOT_NULL(test, f->transport);
+ KUNIT_ASSERT_NOT_NULL(test, f->large);
+ f->transport->swap = make_memcomp_entry(0);
+ f->large->swap = make_memcomp_entry(0);
+ KUNIT_EXPECT_FALSE(test, mc16_test_zcall_release_mapped(f->transport, &mapping));
+ KUNIT_EXPECT_TRUE(test, folio_memcomp_reserved(f->transport));
+ KUNIT_EXPECT_FALSE(test, mc16_test_zcall_release(f->large, true));
+ KUNIT_EXPECT_TRUE(test, folio_memcomp_reserved(f->large));
+ /* No marker is left on a folio returned to the allocator by cleanup. */
+ f->large->swap.val = 0;
+ mc16_test_zcall_counts(f, 0, 0);
+}
+
+/* Use the existing MM actors with an independently unlocked WB sentinel. */
+static struct mc16_test_zcall *mc16_test_zcall_mm(struct kunit *test)
+{
+ struct mc16_test_zcall *f = mc16_test_zcall_begin(test);
+ struct mc16_zl_mm *m;
+ struct vm_area_struct *vma;
+
+ if (!f)
+ return NULL;
+ m = &f->watch;
+ m->addr = kunit_vm_mmap(test, NULL, 0, PAGE_SIZE, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, 0);
+ if (!m->addr || IS_ERR_VALUE(m->addr))
+ return NULL;
+ m->mm = current->mm;
+ mmget(m->mm);
+ mc16_zero_owners_read(&m->owners);
+ m->swapents = mc16_zl_rss(m->mm, MM_SWAPENTS);
+ m->anonpages = mc16_zl_rss(m->mm, MM_ANONPAGES);
+#ifdef CONFIG_PER_VMA_LOCK
+ init_completion(&m->writer_locked);
+ init_completion(&m->writer_release);
+#endif
+ if (kunit_add_action_or_reset(test, mc16_zl_cleanup, m))
+ return NULL;
+ mmap_read_lock(m->mm);
+ vma = vma_lookup(m->mm, m->addr);
+ if (vma) {
+ m->flags = vma->vm_flags;
+ m->prot = vma->vm_page_prot;
+ m->saved_vma = true;
+ }
+ mmap_read_unlock(m->mm);
+ return vma ? f : NULL;
+}
+
+static vm_fault_t mc16_test_zcall_fault(struct mc16_test_zcall *f, pte_t orig,
+ unsigned int flags, bool *locked, pte_t **left)
+{
+ struct mc16_zl_mm *m = &f->watch;
+ struct vm_area_struct *vma;
+ vm_fault_t ret = VM_FAULT_SIGBUS;
+
+ mmap_read_lock(m->mm);
+ vma = vma_lookup(m->mm, m->addr);
+ if (vma) {
+ struct vm_fault vmf = {
+ .vma = vma,
+ .address = m->addr,
+ .real_address = m->addr,
+ .flags = flags | FAULT_FLAG_ORIG_PTE_VALID,
+ .pmd = mm_find_pmd(m->mm, m->addr),
+ .orig_pte = orig,
+ .gfp_mask = GFP_HIGHUSER_MOVABLE,
+ };
+
+ ret = memcomp_load(&vmf, locked);
+ *left = vmf.pte;
+ }
+ mmap_read_unlock(m->mm);
+ return ret;
+}
+
+static void mc16_test_zcall_load(struct kunit *test, bool write)
+{
+ struct mc16_test_zcall *f = mc16_test_zcall_mm(test);
+ pte_t orig = pte_swp_mksoft_dirty(mc16_zl_zero_pte()), pte;
+ pte_t *left = NULL;
+ bool locked = true;
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_zl_install(&f->watch, orig), 0);
+ ret = mc16_test_zcall_fault(f, orig, write ? FAULT_FLAG_WRITE : 0, &locked, &left);
+ KUNIT_EXPECT_FALSE(test, locked);
+ KUNIT_EXPECT_PTR_EQ(test, left, NULL);
+ KUNIT_ASSERT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_NOPAGE);
+ pte = mc16_zl_read(&f->watch);
+ mc16_zl_present(&f->watch, pte);
+ KUNIT_EXPECT_TRUE(test, pte_write(pte));
+ /* A writable read PTE may encode hardware dirty on arm64. */
+ if (write)
+ KUNIT_EXPECT_TRUE(test, pte_dirty(pte));
+ KUNIT_EXPECT_EQ(test, !!pte_soft_dirty(pte), !!pte_swp_soft_dirty(orig));
+ /* A stale second public fault must not consume the same owner twice. */
+ locked = true;
+ ret = mc16_test_zcall_fault(f, orig, 0, &locked, &left);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret, 0U);
+ KUNIT_EXPECT_FALSE(test, locked);
+ KUNIT_EXPECT_PTR_EQ(test, left, NULL);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_zl_read(&f->watch), pte));
+ mc16_zl_counts(&f->watch, 0, 1);
+}
+
+static void memcomp_test_zero_callers_load_read(struct kunit *test)
+{
+ mc16_test_zcall_load(test, false);
+}
+
+static void memcomp_test_zero_callers_load_write(struct kunit *test)
+{
+ mc16_test_zcall_load(test, true);
+}
+
+static void memcomp_test_zero_callers_load_oom(struct kunit *test)
+{
+ struct mc16_test_zcall *f = mc16_test_zcall_mm(test);
+ pte_t orig = mc16_zl_zero_pte();
+ pte_t *left = NULL;
+ bool locked = true;
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_zl_install(&f->watch, orig), 0);
+ mc16_zero_fault_probe.fail_alloc = true;
+ WRITE_ONCE(mc16_zero_fault_probe.task, current);
+ ret = mc16_test_zcall_fault(f, orig, 0, &locked, &left);
+ WRITE_ONCE(mc16_zero_fault_probe.task, NULL);
+ mc16_zero_fault_probe.fail_alloc = false;
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_OOM);
+ KUNIT_EXPECT_FALSE(test, locked);
+ KUNIT_EXPECT_PTR_EQ(test, left, NULL);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_zl_read(&f->watch), orig));
+ mc16_zl_counts(&f->watch, 1, 0);
+}
+#endif /* MC16_ZERO_LOAD_MM_TEST_ENABLED && CONFIG_SWAP */
+#endif /* _MM_MEMCOMP_TESTS_ZERO_CALLERS_TEST_H */
diff --git a/mm/memcomp/tests/zero_carrier_test.h b/mm/memcomp/tests/zero_carrier_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..bebc27207c779c60045fe038f8c1baeced668a45
--- /dev/null
+++ b/mm/memcomp/tests/zero_carrier_test.h
@@ -0,0 +1,405 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_ZERO_CARRIER_TEST_H
+#define _MM_MEMCOMP_TESTS_ZERO_CARRIER_TEST_H
+
+/* Include after zero_convert_test.h and budget_test.h. */
+#if defined(MC16_ZERO_CONVERT_TEST_ENABLED) && defined(MC16_FOLIO_TEST_ENABLED)
+#define MC16_ZERO_CARRIER_TEST_ENABLED
+
+static void mc16_zcar_expect_invalid(struct kunit *test,
+ const struct mc16_zero_carrier *zero)
+{
+ KUNIT_EXPECT_FALSE(test, zero->valid);
+ KUNIT_EXPECT_FALSE(test, zero->ready);
+ KUNIT_EXPECT_EQ(test, zero->token, 0UL);
+ KUNIT_EXPECT_EQ(test, zero->ref.token, 0U);
+ KUNIT_EXPECT_PTR_EQ(test, zero->ref.slot, NULL);
+ KUNIT_EXPECT_PTR_EQ(test, zero->ref.cluster, NULL);
+}
+
+static unsigned int mc16_zcar_convert(struct mc16_test_zc *f,
+ const struct mc16_zero_carrier *zero)
+{
+ unsigned int converted;
+
+ mmap_read_lock(f->mm);
+ folio_lock(f->folio);
+ converted = mc16_zero_convert_folio(f->folio, zero);
+ folio_unlock(f->folio);
+ mmap_read_unlock(f->mm);
+ return converted;
+}
+
+/* Keep the old OWNER alive while reusing its real carrier on a fresh token. */
+static int mc16_zcar_reserve_pending(struct mc16_test_zc *f)
+{
+ struct mc16_test_storage *storage = f->refs->storage;
+ struct memcomp_ref guard = { };
+ unsigned int index = storage->nr;
+ int ret;
+
+ folio_lock(storage->owner->folio);
+ ret = mc16_test_refs_reserve(f->refs, 1);
+ if (!ret && !mc16_get(storage->refs[index].token, &guard))
+ ret = -ENOENT;
+ if (!ret) {
+ mc16_ref_put(&f->guard);
+ f->guard = guard;
+ f->tokens[0] = guard.token;
+ }
+ folio_unlock(storage->owner->folio);
+ return ret;
+}
+
+static int mc16_zcar_refresh(struct mc16_test_zc *f, unsigned int pattern)
+{
+ struct folio *folio = f->refs->storage->owner->folio;
+ u8 *src;
+ unsigned int i;
+ int ret;
+
+ folio_lock(folio);
+ src = kmap_local_folio(folio, 0);
+ for (i = 0; i < PAGE_SIZE; i++)
+ src[i] = pattern == 256 ? i % 53 : pattern;
+ kunmap_local(src);
+ ret = mc16_test_zc_refresh(folio, &f->guard, &f->store[0]);
+ folio_unlock(folio);
+ return ret;
+}
+
+static void memcomp_test_zero_carrier_ready(struct kunit *test)
+{
+ struct mc16_test_zc *f = mc16_test_zc_begin(test);
+ struct mc16_zero_carrier *zero;
+ struct mc16_view view;
+ unsigned int refs;
+ bool effective_zero;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_zcar_reserve_pending(f), 0);
+ refs = refcount_read(&f->guard.slot->refs);
+ ret = mc16_zcar_refresh(f, 0);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ zero = &f->store[0].zero;
+ KUNIT_EXPECT_TRUE(test, zero->valid);
+ KUNIT_EXPECT_TRUE(test, zero->ready);
+ KUNIT_EXPECT_EQ(test, zero->token, (unsigned long)f->tokens[0]);
+ KUNIT_EXPECT_EQ(test, zero->ref.token, f->tokens[0]);
+ KUNIT_EXPECT_PTR_EQ(test, zero->ref.slot, f->guard.slot);
+ KUNIT_EXPECT_PTR_EQ(test, zero->ref.cluster, f->guard.cluster);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guard.slot->refs), refs);
+ /* Only validation and commit T: carrier preparation adds no operation. */
+ /* Observe after measurement; never manufacture readiness in this T. */
+ mutex_lock(memcomp_lock_of(f->tokens[0]));
+ effective_zero = mc16_view_locked(&f->guard, &view) && view.record &&
+ mc16_flags_state(*view.flags) == MC16_SAMEFILL &&
+ !(*view.flags & (MC16_F_RESIDENT | MC16_F_REFRESH_PENDING)) &&
+ !*view.word && !*view.length;
+ mutex_unlock(memcomp_lock_of(f->tokens[0]));
+ KUNIT_EXPECT_TRUE(test, effective_zero);
+ KUNIT_EXPECT_EQ(test, mc16_flags_state(f->guard.slot->flags),
+ (unsigned int)MC16_PENDING);
+}
+
+static void memcomp_test_zero_carrier_commit_failure(struct kunit *test)
+{
+ struct mc16_test_zc *f = mc16_test_zc_begin(test);
+ unsigned int refs;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, f->store[0].zero.valid);
+ KUNIT_ASSERT_EQ(test, mc16_zcar_reserve_pending(f), 0);
+ refs = refcount_read(&f->guard.slot->refs);
+ f->store[0].kunit_refresh_commit_error = -EIO;
+ KUNIT_EXPECT_EQ(test, mc16_zcar_refresh(f, 0), -EIO);
+ KUNIT_EXPECT_EQ(test, f->store[0].kunit_refresh_commit_error, 0);
+ mc16_zcar_expect_invalid(test, &f->store[0].zero);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guard.slot->refs), refs);
+}
+
+static void memcomp_test_zero_carrier_refresh_preflight(struct kunit *test)
+{
+ struct mc16_test_zc *f = mc16_test_zc_begin(test);
+ struct folio *folio;
+ struct memcomp_reclaim_ctx *ctx;
+ unsigned int flags;
+ void *src;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, f->store[0].zero.valid);
+ folio = f->refs->storage->owner->folio;
+ flags = memcomp_test_reclaimer_enter();
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_KSWAPD);
+ if (!ctx) {
+ memcomp_test_reclaimer_exit(flags);
+ KUNIT_FAIL(test, "no codec context for preflight rejection");
+ return;
+ }
+ folio_lock(folio);
+ src = kmap_local_folio(folio, 0);
+ ret = mc16_anon_refresh(&ctx, src, 0, &f->store[0], &f->guard);
+ kunmap_local(src);
+ folio_unlock(folio);
+ if (ctx)
+ memcomp_reclaim_ctx_put(ctx);
+ memcomp_test_reclaimer_exit(flags);
+ KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+ mc16_zcar_expect_invalid(test, &f->store[0].zero);
+}
+
+static void memcomp_test_zero_carrier_store_preflight(struct kunit *test)
+{
+ struct mc16_test_zc *f = mc16_test_zc_begin(test);
+ struct memcomp_reclaim_ctx *ctx;
+ enum memcomp_anon_store_fail failure;
+ struct folio *folio;
+ unsigned int flags;
+ bool stored;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, f->store[0].zero.valid);
+ folio = f->refs->storage->owner->folio;
+ KUNIT_ASSERT_FALSE(test, folio_test_anon(folio));
+ flags = memcomp_test_reclaimer_enter();
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_KSWAPD);
+ if (!ctx) {
+ memcomp_test_reclaimer_exit(flags);
+ KUNIT_FAIL(test, "no codec context for store rejection");
+ return;
+ }
+ folio_lock(folio);
+ stored = mc16_anon_store_internal(folio, &ctx, &f->store[0], &failure);
+ folio_unlock(folio);
+ if (ctx)
+ memcomp_reclaim_ctx_put(ctx);
+ memcomp_test_reclaimer_exit(flags);
+ KUNIT_EXPECT_FALSE(test, stored);
+ KUNIT_EXPECT_EQ(test, failure, MEMCOMP_ANON_STORE_PREFLIGHT);
+ mc16_zcar_expect_invalid(test, &f->store[0].zero);
+}
+
+static void memcomp_test_zero_carrier_nonzero(struct kunit *test)
+{
+ struct mc16_test_zc *f = mc16_test_zc_begin(test);
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_TRUE(test, f->store[0].zero.ready);
+ /* Each payload is the first commit of its own real reservation. */
+ KUNIT_ASSERT_EQ(test, mc16_zcar_reserve_pending(f), 0);
+ KUNIT_ASSERT_EQ(test, mc16_zcar_refresh(f, 0x6b), 0);
+ mc16_zcar_expect_invalid(test, &f->store[0].zero);
+ KUNIT_ASSERT_EQ(test, mc16_zcar_reserve_pending(f), 0);
+ KUNIT_ASSERT_EQ(test, mc16_zcar_refresh(f, 0), 0);
+ KUNIT_ASSERT_TRUE(test, f->store[0].zero.ready);
+ /* Also replace a real ready result with genuinely compressed data. */
+ KUNIT_ASSERT_EQ(test, mc16_zcar_reserve_pending(f), 0);
+ KUNIT_ASSERT_EQ(test, mc16_zcar_refresh(f, 256), 0);
+ mc16_zcar_expect_invalid(test, &f->store[0].zero);
+}
+
+static void memcomp_test_zero_carrier_mthp(struct kunit *test)
+{
+ struct mc16_test_anon_store *f = mc16_test_anon_store_begin(test);
+ unsigned int i;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_reserve(f), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_anon_ctx_get(f), 0);
+ ret = mc16_test_anon_refresh(f);
+ mc16_test_anon_ctx_put(f);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ mc16_zcar_expect_invalid(test, &f->store.zero);
+ KUNIT_EXPECT_EQ(test, folio_nr_pages(f->refs->storage->owner->folio), 2L);
+ for (i = 0; i < ARRAY_SIZE(f->guards); i++) {
+ struct mc16_view view;
+ bool zero;
+
+ mutex_lock(memcomp_lock_of(f->guards[i].token));
+ zero = mc16_view_locked(&f->guards[i], &view) &&
+ mc16_flags_state(*view.flags) == MC16_SAMEFILL &&
+ !*view.word && !*view.length;
+ mutex_unlock(memcomp_lock_of(f->guards[i].token));
+ KUNIT_EXPECT_TRUE(test, zero);
+ }
+}
+
+static void memcomp_test_zero_carrier_token_mismatch(struct kunit *test)
+{
+ struct mc16_test_zc *f = mc16_test_zc_begin(test);
+ struct mc16_zero_carrier bad;
+ struct mc16_zero_diagnostics before;
+ unsigned int refs;
+ pte_t orig;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_zc_unmap_anon(f, 7), 0);
+ KUNIT_ASSERT_TRUE(test, f->store[0].zero.ready);
+ orig = mc16_test_zc_read(f, f->addr);
+ refs = refcount_read(&f->guard.slot->refs);
+ mc16_zero_diagnostics_read(&before);
+ bad = f->store[0].zero;
+ bad.token = f->tokens[1];
+ KUNIT_EXPECT_EQ(test, mc16_zcar_convert(f, &bad), 0U);
+ bad = f->store[0].zero;
+ bad.ref.token = f->tokens[1];
+ KUNIT_EXPECT_EQ(test, mc16_zcar_convert(f, &bad), 0U);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_test_zc_read(f, f->addr), orig));
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guard.slot->refs), refs);
+ mc16_test_zc_counts(f, 0, 1);
+ mc16_test_zc_events(f, &before, 0);
+}
+
+static void memcomp_test_zero_carrier_invalid(struct kunit *test)
+{
+ struct mc16_test_zc *f = mc16_test_zc_begin(test);
+ struct mc16_zero_carrier bad[6];
+ struct mc16_zero_diagnostics before;
+ unsigned int i, refs;
+ pte_t orig;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_zc_unmap_anon(f, 7), 0);
+ KUNIT_ASSERT_TRUE(test, f->store[0].zero.ready);
+ orig = mc16_test_zc_read(f, f->addr);
+ refs = refcount_read(&f->guard.slot->refs);
+ for (i = 0; i < ARRAY_SIZE(bad); i++)
+ bad[i] = f->store[0].zero;
+ bad[0].valid = false;
+ bad[1].ready = false;
+ bad[2].ref.slot = NULL;
+ bad[3].ref.cluster = NULL;
+ bad[4].token = 0;
+ bad[5].ref.token = 0;
+ mc16_zero_diagnostics_read(&before);
+ KUNIT_EXPECT_EQ(test, mc16_zcar_convert(f, NULL), 0U);
+ for (i = 0; i < ARRAY_SIZE(bad); i++)
+ KUNIT_EXPECT_EQ(test, mc16_zcar_convert(f, &bad[i]), 0U);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_test_zc_read(f, f->addr), orig));
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guard.slot->refs), refs);
+ mc16_test_zc_events(f, &before, 0);
+}
+
+static void memcomp_test_zero_carrier_borrow_owner(struct kunit *test)
+{
+ struct mc16_test_zc *f = mc16_test_zc_begin(test);
+ struct mc16_zero_diagnostics before;
+ struct mc16_slot *slot;
+ unsigned int converted, again;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_zc_unmap_anon(f, 7), 0);
+ KUNIT_ASSERT_TRUE(test, f->store[0].zero.ready);
+ slot = f->guard.slot;
+ mc16_ref_put(&f->guard);
+ KUNIT_ASSERT_EQ(test, refcount_read(&slot->refs), 2U);
+ mc16_zero_diagnostics_read(&before);
+ converted = mc16_test_zc_walk(f);
+ again = mc16_test_zc_walk(f);
+ KUNIT_EXPECT_EQ(test, converted, 1U);
+ KUNIT_EXPECT_EQ(test, again, 0U);
+ KUNIT_EXPECT_EQ(test, refcount_read(&slot->refs), 1U);
+ mc16_test_zc_counts(f, 1, 1);
+ mc16_test_zc_events(f, &before, 1);
+}
+
+static void mc16_zcar_expect_zero_page(struct kunit *test, struct mc16_flt *p)
+{
+ struct page *page = NULL;
+ /* Protects the mapped PTE while its contents are read. */
+ spinlock_t *ptl;
+ pte_t *ptep, pte = __pte(0);
+ pmd_t *pmd;
+ void *bytes;
+ bool zero;
+
+ mmap_read_lock(p->mm);
+ pmd = mm_find_pmd(p->mm, p->addr);
+ if (pmd) {
+ ptep = pte_offset_map_lock(p->mm, pmd, p->addr, &ptl);
+ if (ptep) {
+ pte = ptep_get(ptep);
+ if (pte_present(pte) && !pte_special(pte)) {
+ page = pte_page(pte);
+ get_page(page);
+ }
+ pte_unmap_unlock(ptep, ptl);
+ }
+ }
+ mmap_read_unlock(p->mm);
+ KUNIT_ASSERT_TRUE(test, pte_present(pte));
+ if (pte_special(pte)) {
+ KUNIT_EXPECT_TRUE(test, is_zero_pfn(pte_pfn(pte)));
+ return;
+ }
+ KUNIT_ASSERT_NOT_NULL(test, page);
+ bytes = kmap_local_page(page);
+ zero = !memchr_inv(bytes, 0, PAGE_SIZE);
+ kunmap_local(bytes);
+ put_page(page);
+ KUNIT_EXPECT_TRUE(test, zero);
+}
+
+static void mc16_zcar_lifecycle(struct kunit *test, bool refault)
+{
+ struct mc16_bm *f = mc16_bm_begin(test, 1, true);
+ struct mc16_zero_owners before, after;
+ pte_t canonical;
+ void *src;
+ bool unlocked = false;
+ vm_fault_t fault = 0;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ folio_lock(f->page[0].folio);
+ src = kmap_local_folio(f->page[0].folio, 0);
+ memset(src, 0, PAGE_SIZE);
+ kunmap_local(src);
+ folio_unlock(f->page[0].folio);
+ mc16_zero_owners_read(&before);
+ ret = mc16_bm_pageout(&f->page[0]);
+ if (ret) {
+ KUNIT_FAIL(test, "canonical-zero pageout failed: %d", ret);
+ return;
+ }
+ canonical = mc16_flt_read(f->seed, f->seed->addr);
+ /* This read observes PTE state only and adds no token lookup or T. */
+ if (!is_swap_pte(canonical) ||
+ pte_to_swp_entry(canonical).val != make_memcomp_entry(0).val) {
+ KUNIT_FAIL(test, "pageout did not canonically convert the zero PTE");
+ return;
+ }
+ if (refault) {
+ fault = mc16_bm_fault(&f->page[0], &unlocked);
+ ret = fault == VM_FAULT_NOPAGE ? 0 : -EIO;
+ } else {
+ ret = mc16_bm_pte(f, f->seed->addr, false);
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ mc16_zero_owners_read(&after);
+ KUNIT_EXPECT_EQ(test, after.pte_pages, before.pte_pages);
+ KUNIT_EXPECT_EQ(test, after.cache_pages, before.cache_pages);
+ KUNIT_EXPECT_EQ(test, mc16_flt_rss(f->seed, MM_SWAPENTS), f->seed->swapents);
+ if (refault) {
+ KUNIT_EXPECT_TRUE(test, unlocked);
+ mc16_zcar_expect_zero_page(test, &f->page[0]);
+ } else {
+ KUNIT_EXPECT_TRUE(test, pte_none(mc16_flt_read(f->seed, f->seed->addr)));
+ }
+}
+
+static void memcomp_test_zero_carrier_lifecycle_refault1(struct kunit *test)
+{
+ mc16_zcar_lifecycle(test, true);
+}
+
+static void memcomp_test_zero_carrier_lifecycle_exit(struct kunit *test)
+{
+ mc16_zcar_lifecycle(test, false);
+}
+
+#endif /* MC16_ZERO_CONVERT_TEST_ENABLED && MC16_FOLIO_TEST_ENABLED */
+#endif /* _MM_MEMCOMP_TESTS_ZERO_CARRIER_TEST_H */
diff --git a/mm/memcomp/tests/zero_convert_test.h b/mm/memcomp/tests/zero_convert_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..6cbca297720aac99bc03b964d00e183fc214a393
--- /dev/null
+++ b/mm/memcomp/tests/zero_convert_test.h
@@ -0,0 +1,694 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_ZERO_CONVERT_TEST_H
+#define _MM_MEMCOMP_TESTS_ZERO_CONVERT_TEST_H
+
+#include <kunit/resource.h>
+#include <linux/mman.h>
+#include <linux/rmap.h>
+#include <linux/sched/mm.h>
+
+/*
+ * Include after zero_convert.h and the storage/refs test fixtures. Register
+ * under the same guard below. Positive slots use real reserve/refresh/finish;
+ * page tables, anonymous faults, rmap and the zero ledger are not stubbed.
+ * The MOVE case models an interleaving with a PTL-protected PTE transfer; it
+ * does not invoke the mremap syscall or claim concurrent fork coverage.
+ */
+#if defined(CONFIG_MEMCG) && defined(CONFIG_MMU) && \
+ defined(CONFIG_ADVISE_SYSCALLS)
+#define MC16_ZERO_CONVERT_TEST_ENABLED
+
+struct mc16_test_zc {
+ struct kunit *test;
+ struct mc16_test_refs *refs;
+ struct mc16_storage_fixture *saved_probe;
+ struct memcomp_ref guard;
+ struct memcomp_store_ctx store[2];
+ struct mm_struct *mm;
+ unsigned long addr;
+ unsigned long mapping;
+ struct folio *folio;
+ struct anon_vma *anon_vma;
+ struct mc16_zero_owners owners;
+ s64 swapents;
+ s64 anonpages;
+ u32 tokens[2];
+};
+
+/* Acquire and release in this cleanup call, never inherit the test's lock. */
+static void mc16_test_zc_objcg_cleanup(void *data)
+{
+ struct memcomp_test_objcg_fixture *owner = data;
+
+ if (owner->folio)
+ folio_lock(owner->folio);
+ memcomp_test_objcg_cleanup(owner);
+}
+
+static void mc16_test_zc_storage_cleanup(void *data)
+{
+ struct mc16_test_zc *f = data;
+
+ mc16_test_storage_cleanup(f->refs->storage);
+ rcu_assign_pointer(mc16_storage_fixture, f->saved_probe);
+ synchronize_rcu();
+}
+
+static s64 mc16_test_zc_rss(struct mm_struct *mm, int member)
+{
+ return percpu_counter_sum(&mm->rss_stat[member]);
+}
+
+/* No owner survives an unmap into the still-active entry80 dispatch. */
+static int mc16_test_zc_zap(struct mc16_test_zc *f, unsigned long address)
+{
+ struct memcomp_ref removed = { };
+ spinlock_t *ptl; /* Protects the actual PTE ownership transaction. */
+ pte_t *ptep, pte;
+ pmd_t *pmd;
+ unsigned long token;
+ int ret = 0;
+
+ mmap_read_lock(f->mm);
+ pmd = mm_find_pmd(f->mm, address);
+ if (!pmd)
+ goto out;
+ ptep = pte_offset_map_lock(f->mm, pmd, address, &ptl);
+ if (!ptep)
+ goto out;
+ pte = ptep_get(ptep);
+ if (!is_swap_pte(pte) || !is_memcomp_entry(pte_to_swp_entry(pte)))
+ goto out_pte;
+ token = swp_offset(pte_to_swp_entry(pte));
+ /* Cleanup also works after the test explicitly retires its main guard. */
+ if (token && ((token != f->tokens[0] && token != f->tokens[1]) ||
+ !mc16_get(token, &removed)))
+ ret = -ENOENT;
+ ptep_get_and_clear(f->mm, address, ptep);
+ add_mm_counter(f->mm, MM_SWAPENTS, -1);
+ if (removed.slot)
+ mc16_ref_put_count(&removed, 2); /* Lookup guard plus PTE owner. */
+ else if (!token)
+ mc16_zero_pte_put(1);
+out_pte:
+ pte_unmap_unlock(ptep, ptl);
+out:
+ mmap_read_unlock(f->mm);
+ return ret;
+}
+
+static void mc16_test_zc_cleanup(void *data)
+{
+ struct mc16_test_zc *f = data;
+ struct mc16_zero_owners owners;
+ int ret;
+
+ ret = mc16_test_zc_zap(f, f->addr);
+ KUNIT_EXPECT_EQ(f->test, ret, 0);
+ ret = mc16_test_zc_zap(f, f->addr + PAGE_SIZE);
+ KUNIT_EXPECT_EQ(f->test, ret, 0);
+ /* Any present page left by an incomplete setup uses normal MM teardown. */
+ ret = mc16_test_dontneed(f->mm, f->mapping, 3 * PAGE_SIZE);
+ KUNIT_EXPECT_EQ(f->test, ret, 0);
+ if (f->anon_vma)
+ put_anon_vma(f->anon_vma);
+ if (f->guard.slot)
+ mc16_ref_put(&f->guard);
+ if (f->folio) {
+ struct memcomp_ref *owner = &f->refs->storage->refs[0];
+
+ folio_lock(f->folio);
+ if (owner->slot)
+ mc16_ref_put(owner);
+ folio_unlock(f->folio);
+ f->refs->storage->owner->model.folio = f->refs->storage->owner->folio;
+ folio_put(f->folio);
+ }
+ mc16_zero_owners_read(&owners);
+ KUNIT_EXPECT_EQ(f->test, owners.pte_pages, f->owners.pte_pages);
+ KUNIT_EXPECT_EQ(f->test, owners.cache_pages, f->owners.cache_pages);
+ KUNIT_EXPECT_EQ(f->test, mc16_test_zc_rss(f->mm, MM_SWAPENTS), f->swapents);
+ KUNIT_EXPECT_EQ(f->test, mc16_test_zc_rss(f->mm, MM_ANONPAGES), f->anonpages);
+ mmput(f->mm);
+}
+
+/* Carrier readiness comes only from the real refresh's existing commit T. */
+static int mc16_test_zc_refresh(struct folio *folio, struct memcomp_ref *ref,
+ struct memcomp_store_ctx *store)
+{
+ struct memcomp_reclaim_ctx *ctx;
+ unsigned int flags;
+ void *src;
+ int ret;
+
+ flags = memcomp_test_reclaimer_enter();
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_KSWAPD);
+ if (!ctx) {
+ memcomp_test_reclaimer_exit(flags);
+ return -EBUSY;
+ }
+ src = kmap_local_folio(folio, 0);
+ ret = mc16_anon_refresh(&ctx, src, 1, store, ref);
+ kunmap_local(src);
+ if (ctx)
+ memcomp_reclaim_ctx_put(ctx);
+ memcomp_direct_staging_put(store->staging);
+ store->staging = NULL;
+ memcomp_test_reclaimer_exit(flags);
+ return ret;
+}
+
+/* Shared reserve/cleanup logic, with no folio lock held on return. */
+static int mc16_test_zc_storage(struct mc16_test_zc *f)
+{
+ struct kunit *test = f->test;
+ struct memcomp_test_objcg_fixture *owner;
+ struct mc16_test_storage *s;
+ unsigned int i;
+ int ret;
+
+ f->refs = kunit_kzalloc(test, sizeof(*f->refs), GFP_KERNEL);
+ s = kunit_kzalloc(test, sizeof(*s), GFP_KERNEL);
+ if (!f->refs || !s)
+ return -ENOMEM;
+ f->refs->storage = s;
+ owner = memcomp_test_objcg_begin(test, 1);
+ if (!owner)
+ return -ENOMEM;
+ folio_unlock(owner->folio);
+ kunit_remove_action(test, memcomp_test_objcg_cleanup, owner);
+ ret = kunit_add_action_or_reset(test, mc16_test_zc_objcg_cleanup, owner);
+ if (ret)
+ return ret;
+ s->test = test;
+ s->owner = owner;
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ mc16_test_metadata_snapshot(&s->baseline);
+#endif
+ f->saved_probe = rcu_access_pointer(mc16_storage_fixture);
+ s->probe.task = current;
+ ret = kunit_add_action_or_reset(test, mc16_test_zc_storage_cleanup, f);
+ if (ret)
+ return ret;
+ rcu_assign_pointer(mc16_storage_fixture, &s->probe);
+ f->refs->stored_before = atomic64_read(&memcomp_stored_bytes);
+ f->refs->compressed_before = atomic64_read(&memcomp_compressed_bytes);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ f->refs->pending_before = atomic64_read(&memcomp_pending_pages);
+#endif
+ ret = kunit_add_action_or_reset(test, mc16_test_refs_cleanup, f->refs);
+ if (ret)
+ return ret;
+ /* Two real refreshes return separate borrowed carriers, without extra T. */
+ for (i = 0; i < ARRAY_SIZE(f->tokens); i++) {
+ struct memcomp_ref lookup = { };
+ void *src;
+
+ folio_lock(owner->folio);
+ src = kmap_local_folio(owner->folio, 0);
+ memset(src, 0, PAGE_SIZE);
+ kunmap_local(src);
+ ret = mc16_test_refs_reserve(f->refs, 1);
+ if (!ret) {
+ f->tokens[i] = s->refs[i].token;
+ if (!mc16_get(f->tokens[i], &lookup))
+ ret = -ENOENT;
+ else
+ ret = mc16_test_zc_refresh(owner->folio, &lookup,
+ &f->store[i]);
+ }
+ if (lookup.slot)
+ mc16_ref_put(&lookup);
+ folio_unlock(owner->folio);
+ if (ret)
+ return ret;
+ }
+ return 0;
+}
+
+static struct mc16_test_zc *mc16_test_zc_begin(struct kunit *test)
+{
+ struct mc16_test_zc *f;
+ struct vm_area_struct *vma;
+ int ret;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->test = test;
+ if (mc16_test_zc_storage(f))
+ return NULL;
+ f->mapping = kunit_vm_mmap(test, NULL, 0, 3 * PAGE_SIZE,
+ PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, 0);
+ if (!f->mapping || IS_ERR_VALUE(f->mapping))
+ return NULL;
+ /* Always choose two adjacent pages sharing one PTE-table lock. */
+ f->addr = f->mapping;
+ if ((f->addr >> PMD_SHIFT) != ((f->addr + PAGE_SIZE) >> PMD_SHIFT))
+ f->addr += PAGE_SIZE;
+ f->mm = current->mm;
+ mmget(f->mm);
+ mc16_zero_owners_read(&f->owners);
+ f->swapents = mc16_test_zc_rss(f->mm, MM_SWAPENTS);
+ f->anonpages = mc16_test_zc_rss(f->mm, MM_ANONPAGES);
+ ret = kunit_add_action_or_reset(test, mc16_test_zc_cleanup, f);
+ if (ret)
+ return NULL;
+ if (!mc16_get(f->tokens[0], &f->guard))
+ return NULL;
+ mmap_write_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (vma) {
+ vma_start_write(vma);
+ vm_flags_set(vma, VM_NOHUGEPAGE);
+ }
+ mmap_write_unlock(f->mm);
+ return vma ? f : NULL;
+}
+
+/* Construct expected flags independently of pte_move_swp_offset(). */
+static pte_t mc16_test_zc_pte(unsigned long token, unsigned int flags)
+{
+ pte_t pte = swp_entry_to_pte(make_memcomp_entry(token));
+
+ if (flags & 1)
+ pte = pte_swp_mksoft_dirty(pte);
+ if (flags & 2)
+ pte = pte_swp_mkuffd_wp(pte);
+ if (flags & 4)
+ pte = pte_swp_mkexclusive(pte);
+ return pte;
+}
+
+static int mc16_test_zc_install(struct mc16_test_zc *f,
+ unsigned long address, unsigned int index,
+ unsigned int flags)
+{
+ struct memcomp_ref owner = { };
+ spinlock_t *ptl; /* Transfers one actual positive-token PTE reference. */
+ pte_t *ptep;
+ int ret = 0;
+
+ if (!mc16_get(f->tokens[index], &owner))
+ return -ENOENT;
+ mmap_read_lock(f->mm);
+ ptep = get_locked_pte(f->mm, address, &ptl);
+ if (!ptep) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ __acquire(RCU);
+ if (!pte_none(ptep_get(ptep))) {
+ ret = -EEXIST;
+ } else {
+ set_pte_at(f->mm, address, ptep,
+ mc16_test_zc_pte(owner.token, flags));
+ add_mm_counter(f->mm, MM_SWAPENTS, 1);
+ owner = (struct memcomp_ref) { };
+ }
+ pte_unmap_unlock(ptep, ptl);
+out:
+ mmap_read_unlock(f->mm);
+ if (owner.slot)
+ mc16_ref_put(&owner);
+ return ret;
+}
+
+static pte_t mc16_test_zc_read(struct mc16_test_zc *f, unsigned long address)
+{
+ spinlock_t *ptl; /* Samples the real PTE without retaining its pointer. */
+ pte_t *ptep, pte = __pte(0);
+ pmd_t *pmd;
+
+ mmap_read_lock(f->mm);
+ pmd = mm_find_pmd(f->mm, address);
+ if (pmd) {
+ ptep = pte_offset_map_lock(f->mm, pmd, address, &ptl);
+ if (ptep) {
+ pte = ptep_get(ptep);
+ pte_unmap_unlock(ptep, ptl);
+ }
+ }
+ mmap_read_unlock(f->mm);
+ return pte;
+}
+
+/* Callback tests borrow the real refresh result; they do not prepare it. */
+static unsigned int mc16_test_zc_callback(struct mc16_test_zc *f)
+{
+ struct mc16_zero_walk walk = { .ref = f->store[0].zero.ref };
+ struct vm_area_struct *vma;
+
+ if (!f->store[0].zero.valid || !f->store[0].zero.ready)
+ return 0;
+ mmap_read_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (vma)
+ mc16_zero_convert_one(f->refs->storage->owner->folio, vma,
+ f->addr, &walk);
+ mmap_read_unlock(f->mm);
+ return walk.converted;
+}
+
+static void mc16_test_zc_counts(struct mc16_test_zc *f, s64 zeros, s64 swapped)
+{
+ struct mc16_zero_owners owners;
+
+ mc16_zero_owners_read(&owners);
+ KUNIT_EXPECT_EQ(f->test, owners.pte_pages, f->owners.pte_pages + zeros);
+ KUNIT_EXPECT_EQ(f->test, owners.cache_pages, f->owners.cache_pages);
+ KUNIT_EXPECT_EQ(f->test, mc16_test_zc_rss(f->mm, MM_SWAPENTS),
+ f->swapents + swapped);
+}
+
+/* Net owner totals alone would miss a spurious get/put pair. */
+static void mc16_test_zc_events(struct mc16_test_zc *f,
+ struct mc16_zero_diagnostics *before,
+ unsigned int inits)
+{
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ struct mc16_zero_diagnostics after;
+ unsigned int event;
+
+ mc16_zero_diagnostics_read(&after);
+ for (event = 0; event < MC16_ZERO_EVENTS; event++) {
+ u64 expected = event == MC16_ZERO_PTE_INIT ? inits : 0;
+
+ KUNIT_EXPECT_EQ(f->test, after.calls[event] - before->calls[event], expected);
+ KUNIT_EXPECT_EQ(f->test, after.pages[event] - before->pages[event], expected);
+ }
+#endif
+}
+
+/* Fault a genuine order-0 anonymous page, then pin before its last unmap. */
+static int mc16_test_zc_unmap_anon(struct mc16_test_zc *f, unsigned int flags)
+{
+ struct vm_area_struct *vma;
+ struct memcomp_ref owner = { };
+ spinlock_t *ptl; /* Revalidates and replaces the faulted present PTE. */
+ pte_t *ptep, orig;
+ vm_fault_t fault;
+ int ret = -EINVAL;
+
+ mmap_read_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (!vma)
+ goto out_mmap;
+ fault = handle_mm_fault(vma, f->addr, FAULT_FLAG_WRITE, NULL);
+ /* These returns mean the fault actor has already dropped mmap_lock. */
+ if (fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED))
+ return -EAGAIN;
+ if (fault & VM_FAULT_ERROR)
+ goto out_mmap;
+ ptep = get_locked_pte(f->mm, f->addr, &ptl);
+ if (!ptep) {
+ ret = -ENOMEM;
+ goto out_mmap;
+ }
+ __acquire(RCU);
+ orig = ptep_get(ptep);
+ if (!pte_present(orig) || pte_special(orig)) {
+ pte_unmap_unlock(ptep, ptl);
+ goto out_mmap;
+ }
+ f->folio = page_folio(pte_page(orig));
+ folio_get(f->folio);
+ pte_unmap_unlock(ptep, ptl);
+ folio_lock(f->folio);
+ if (!folio_test_anon(f->folio) || folio_test_large(f->folio) ||
+ !pages_identical(&f->folio->page, ZERO_PAGE(0)))
+ goto out_folio;
+ f->anon_vma = folio_get_anon_vma(f->folio);
+ if (!f->anon_vma)
+ goto out_folio;
+ /* Replace the synthetic primary with this real source's own OWNER. */
+ mc16_ref_put(&f->guard);
+ mc16_ref_put(&f->refs->storage->refs[0]);
+ f->refs->storage->owner->model.folio = f->folio;
+ {
+ struct mc16_record *record;
+ struct memcomp_ref *initial = &f->refs->storage->refs[0];
+
+ record = mc16_record_alloc(f->folio, 1, false);
+ if (IS_ERR(record)) {
+ ret = PTR_ERR(record);
+ goto out_folio;
+ }
+ ret = mc16_reserve_hint(record, 1, MC16_KEY_ANON,
+ mc16_folio_zero_hint(f->folio), initial);
+ if (ret) {
+ mc16_record_abort(record);
+ goto out_folio;
+ }
+ mc16_account_reservation(initial);
+ f->tokens[0] = initial->token;
+ }
+ if (!mc16_get(f->tokens[0], &f->guard) ||
+ !mc16_get(f->tokens[0], &owner)) {
+ ret = -ENOENT;
+ goto out_folio;
+ }
+ ptep = get_locked_pte(f->mm, f->addr, &ptl);
+ if (!ptep) {
+ ret = -ENOMEM;
+ goto out_folio;
+ }
+ __acquire(RCU);
+ if (!pte_same(ptep_get(ptep), orig)) {
+ ret = -ESTALE;
+ } else {
+ flush_cache_page(vma, f->addr, pte_pfn(orig));
+ ptep_clear_flush(vma, f->addr, ptep);
+ folio_remove_rmap_pte(f->folio, &f->folio->page, vma);
+ set_pte_at(f->mm, f->addr, ptep,
+ mc16_test_zc_pte(owner.token, flags));
+ add_mm_counter(f->mm, MM_ANONPAGES, -1);
+ add_mm_counter(f->mm, MM_SWAPENTS, 1);
+ owner = (struct memcomp_ref) { };
+ folio_put(f->folio); /* Retire the old present mapping's ref. */
+ ret = 0;
+ }
+ pte_unmap_unlock(ptep, ptl);
+ if (!ret)
+ ret = mc16_test_zc_refresh(f->folio, &f->guard, &f->store[0]);
+out_folio:
+ if (owner.slot)
+ mc16_ref_put(&owner);
+ folio_unlock(f->folio);
+out_mmap:
+ mmap_read_unlock(f->mm);
+ return ret;
+}
+
+/* The helper acquires/releases anon_vma's read lock; we retain the real pin. */
+static unsigned int mc16_test_zc_walk(struct mc16_test_zc *f)
+{
+ unsigned int converted;
+
+ mmap_read_lock(f->mm);
+ folio_lock(f->folio);
+ converted = mc16_zero_convert_folio(f->folio, &f->store[0].zero);
+ folio_unlock(f->folio);
+ mmap_read_unlock(f->mm);
+ return converted;
+}
+
+static void memcomp_test_zero_convert_flags(struct kunit *test)
+{
+ struct mc16_test_zc *f = mc16_test_zc_begin(test);
+ unsigned int flags;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ for (flags = 0; flags < 8; flags++) {
+ struct mc16_zero_diagnostics before;
+ unsigned int refs;
+ pte_t orig, actual;
+
+ KUNIT_ASSERT_EQ(test, mc16_test_zc_install(f, f->addr, 0, flags), 0);
+ orig = mc16_test_zc_read(f, f->addr);
+ refs = refcount_read(&f->guard.slot->refs);
+ mc16_zero_diagnostics_read(&before);
+ KUNIT_EXPECT_EQ(test, mc16_test_zc_callback(f), 1U);
+ actual = mc16_test_zc_read(f, f->addr);
+ KUNIT_EXPECT_TRUE(test, pte_same(actual,
+ mc16_test_zc_pte(0, flags)));
+ KUNIT_EXPECT_EQ(test, pte_swp_soft_dirty(actual), pte_swp_soft_dirty(orig));
+ KUNIT_EXPECT_EQ(test, pte_swp_uffd_wp(actual), pte_swp_uffd_wp(orig));
+ KUNIT_EXPECT_EQ(test, pte_swp_exclusive(actual), pte_swp_exclusive(orig));
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guard.slot->refs), refs - 1);
+ mc16_test_zc_counts(f, 1, 1);
+ /* A second PTL visit loses; no extra ledger init or positive put. */
+ KUNIT_EXPECT_EQ(test, mc16_test_zc_callback(f), 0U);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_test_zc_read(f, f->addr), actual));
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guard.slot->refs), refs - 1);
+ mc16_test_zc_counts(f, 1, 1);
+ mc16_test_zc_events(f, &before, 1);
+ KUNIT_ASSERT_EQ(test, mc16_test_zc_zap(f, f->addr), 0);
+ mc16_test_zc_counts(f, 0, 0);
+ }
+}
+
+static void memcomp_test_zero_convert_stale(struct kunit *test)
+{
+ struct mc16_test_zc *f = mc16_test_zc_begin(test);
+ struct memcomp_ref *other;
+ struct mc16_zero_diagnostics before;
+ unsigned int refs, other_refs;
+ pte_t orig;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_zero_diagnostics_read(&before);
+ KUNIT_EXPECT_EQ(test, mc16_test_zc_callback(f), 0U); /* No PTE table. */
+ KUNIT_ASSERT_EQ(test, mc16_test_zc_install(f, f->addr, 1, 7), 0);
+ orig = mc16_test_zc_read(f, f->addr);
+ other = &f->refs->storage->refs[1];
+ refs = refcount_read(&f->guard.slot->refs);
+ other_refs = refcount_read(&other->slot->refs);
+ KUNIT_EXPECT_EQ(test, mc16_test_zc_callback(f), 0U);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_test_zc_read(f, f->addr), orig));
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guard.slot->refs), refs);
+ KUNIT_EXPECT_EQ(test, refcount_read(&other->slot->refs), other_refs);
+ mc16_test_zc_counts(f, 0, 1);
+ mc16_test_zc_events(f, &before, 0);
+}
+
+static void memcomp_test_zero_convert_rmap(struct kunit *test)
+{
+ struct mc16_test_zc *f = mc16_test_zc_begin(test);
+ struct memcomp_ref probe = { };
+ struct mc16_zero_diagnostics before;
+ struct mc16_slot *slot;
+ unsigned int converted;
+ bool locked;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_zc_unmap_anon(f, 7), 0);
+ KUNIT_ASSERT_NOT_NULL(test, f->anon_vma);
+ KUNIT_EXPECT_FALSE(test, folio_mapped(f->folio));
+ KUNIT_EXPECT_FALSE(test, folio_test_locked(f->folio));
+ /* Source OWNER survives conversion, alongside the PTE and test guard. */
+ slot = f->guard.slot;
+ KUNIT_EXPECT_EQ(test, refcount_read(&slot->refs), 3U);
+ mc16_zero_diagnostics_read(&before);
+ converted = mc16_test_zc_walk(f);
+ KUNIT_EXPECT_EQ(test, converted, 1U);
+ KUNIT_EXPECT_PTR_EQ(test, f->guard.slot, slot);
+ KUNIT_EXPECT_EQ(test, refcount_read(&slot->refs), 2U);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_test_zc_read(f, f->addr),
+ mc16_test_zc_pte(0, 7)));
+ mc16_test_zc_counts(f, 1, 1);
+ KUNIT_EXPECT_EQ(test, mc16_test_zc_rss(f->mm, MM_ANONPAGES), f->anonpages);
+ KUNIT_EXPECT_EQ(test, mc16_test_zc_walk(f), 0U);
+ KUNIT_EXPECT_EQ(test, refcount_read(&slot->refs), 2U);
+ mc16_test_zc_events(f, &before, 1);
+ /* A writer can acquire the rmap lock after the walk has returned. */
+ locked = down_write_trylock(&f->anon_vma->root->rwsem);
+ if (locked)
+ up_write(&f->anon_vma->root->rwsem);
+ KUNIT_EXPECT_TRUE(test, locked);
+ mc16_ref_put(&f->guard);
+ folio_lock(f->folio);
+ mc16_ref_put(&f->refs->storage->refs[0]);
+ folio_unlock(f->folio);
+ /* No access to slot after final put. A fresh lookup must now miss. */
+ locked = mc16_get(f->tokens[0], &probe);
+ if (probe.slot)
+ mc16_ref_put(&probe);
+ KUNIT_EXPECT_FALSE(test, locked);
+}
+
+/* Move one swap PTE inside the same VMA, after folio->index was established. */
+static int mc16_test_zc_move(struct mc16_test_zc *f)
+{
+ spinlock_t *ptl; /* Both addresses share a prevalidated PTE-table lock. */
+ pte_t *src, *dst;
+ unsigned long destination = f->addr + PAGE_SIZE;
+ int ret = 0;
+
+ /* Materialize the destination table before taking either PTL. */
+ mmap_write_lock(f->mm);
+ dst = get_locked_pte(f->mm, destination, &ptl);
+ if (!dst) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ __acquire(RCU);
+ pte_unmap_unlock(dst, ptl);
+ /* Different PTE tables would require two locks and a different model. */
+ if ((f->addr >> PMD_SHIFT) != (destination >> PMD_SHIFT)) {
+ ret = -ERANGE;
+ goto out;
+ }
+ src = pte_offset_map_lock(f->mm, mm_find_pmd(f->mm, f->addr), f->addr, &ptl);
+ if (!src) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ dst = src + 1;
+ if (!is_swap_pte(ptep_get(src)) || !pte_none(ptep_get(dst))) {
+ ret = -EINVAL;
+ } else {
+ pte_t pte = ptep_get_and_clear(f->mm, f->addr, src);
+
+ set_pte_at(f->mm, destination, dst, pte);
+ }
+ pte_unmap_unlock(src, ptl);
+out:
+ mmap_write_unlock(f->mm);
+ return ret;
+}
+
+static void memcomp_test_zero_convert_move_miss(struct kunit *test)
+{
+ struct mc16_test_zc *f = mc16_test_zc_begin(test);
+ struct mc16_zero_diagnostics before;
+ unsigned long *dst;
+ unsigned int refs;
+ swp_entry_t saved_swap;
+ pte_t orig;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ dst = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, dst);
+ KUNIT_ASSERT_EQ(test, mc16_test_zc_unmap_anon(f, 7), 0);
+ orig = mc16_test_zc_read(f, f->addr);
+ ret = mc16_test_zc_move(f);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ refs = refcount_read(&f->guard.slot->refs);
+ mc16_zero_diagnostics_read(&before);
+ KUNIT_EXPECT_EQ(test, mc16_test_zc_walk(f), 0U);
+ KUNIT_EXPECT_TRUE(test, pte_none(mc16_test_zc_read(f, f->addr)));
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_test_zc_read(f, f->addr + PAGE_SIZE), orig));
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guard.slot->refs), refs);
+ mc16_test_zc_counts(f, 0, 1);
+ mc16_test_zc_events(f, &before, 0);
+ /* The checks above cover conversion miss before source exit. */
+ folio_lock(f->folio);
+ saved_swap = f->folio->swap;
+ /* The low-level reservation has not bound its token to the source. */
+ f->folio->swap = make_memcomp_entry(f->tokens[0]);
+ folio_unlock(f->folio);
+ ret = mc16_flt_release(f->folio);
+ if (ret) {
+ /* Release consumed OWNER, not this fixture's guard or moved PTE. */
+ f->refs->storage->refs[0] = (struct memcomp_ref) { };
+ } else {
+ /* Failed release leaves OWNER for the original cleanup actor. */
+ folio_lock(f->folio);
+ f->folio->swap = saved_swap;
+ folio_unlock(f->folio);
+ }
+ KUNIT_ASSERT_TRUE(test, ret);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guard.slot->refs), refs - 1);
+ /* The read phase starts after real publication and OWNER retirement. */
+ refs = refcount_read(&f->guard.slot->refs);
+ memset(dst, 0xa7, PAGE_SIZE);
+ ret = mc16_load_page(&f->guard, dst);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ KUNIT_EXPECT_PTR_EQ(test, memchr_inv(dst, 0, PAGE_SIZE), NULL);
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->guard.slot->refs), refs);
+}
+#endif /* CONFIG_MEMCG && CONFIG_MMU && CONFIG_ADVISE_SYSCALLS */
+#endif /* _MM_MEMCOMP_TESTS_ZERO_CONVERT_TEST_H */
diff --git a/mm/memcomp/tests/zero_hint_test.h b/mm/memcomp/tests/zero_hint_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..3a727f0b4ad264e61590b08cdf48215da0251b78
--- /dev/null
+++ b/mm/memcomp/tests/zero_hint_test.h
@@ -0,0 +1,860 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_ZERO_HINT_TEST_H
+#define _MM_MEMCOMP_TESTS_ZERO_HINT_TEST_H
+
+/* Include after budget_test.h and zero_convert_test.h. */
+#include <asm/sections.h>
+#ifdef CONFIG_MEMCG
+static int mc16_zh_reserve(struct mc16_test_storage *s, unsigned int nr,
+ bool hint)
+{
+ struct mc16_record *record;
+ int ret;
+
+ if (s->nr + nr > MC16_TEST_REFS)
+ return -ENOSPC;
+ record = mc16_record_alloc(s->owner->folio, nr, false);
+ if (IS_ERR(record))
+ return PTR_ERR(record);
+ ret = mc16_reserve_hint(record, nr, MC16_KEY_ANON, hint,
+ &s->refs[s->nr]);
+ if (ret)
+ mc16_record_abort(record);
+ else
+ s->nr += nr;
+ return ret;
+}
+
+static void memcomp_test_retained_x1(struct kunit *test)
+{
+ struct mc16_test_storage *s = mc16_test_storage_begin(test);
+ struct shrink_control sc = { };
+ unsigned int i;
+ u32 token = 0;
+
+ KUNIT_ASSERT_NOT_NULL(test, s);
+ for (i = 0; i < 32; i++) {
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_count(NULL, &sc),
+ i ? 1UL : SHRINK_EMPTY);
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(s, 1), 0);
+ if (!i)
+ token = s->refs[i].token;
+ KUNIT_EXPECT_EQ(test, s->refs[i].token, token + i);
+ KUNIT_EXPECT_EQ(test, s->refs[i].cluster->next, (u16)(i + 1));
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 0UL);
+ mc16_test_cancel_pending(&s->refs[i]);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 1UL);
+ }
+ /* Each immediate refault reuses the header and the unsealed segment. */
+}
+
+static void memcomp_test_retained_x2_segment(struct kunit *test)
+{
+ struct mc16_test_storage *s = mc16_test_storage_begin(test);
+ struct mc16_cluster *cluster;
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, s);
+ for (i = 0; i < MC16_SEGMENT_SLOTS; i++) {
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(s, 1), 0);
+ mc16_test_cancel_pending(&s->refs[i]);
+ }
+ cluster = xa_load(&mc16_clusters, 1);
+ KUNIT_ASSERT_NOT_NULL(test, cluster);
+ KUNIT_EXPECT_PTR_EQ(test, cluster->segments[0], NULL);
+ KUNIT_EXPECT_PTR_EQ(test, cluster->segments[1], NULL);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 1UL);
+ /* A failed private segment allocation must leave the retained count. */
+ s->probe.fail_at = s->probe.allocations + 1;
+ KUNIT_EXPECT_EQ(test, mc16_test_reserve(s, 1), -ENOMEM);
+ KUNIT_EXPECT_EQ(test, cluster->next, (u16)64);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 1UL);
+ s->probe.fail_at = 0;
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(s, 1), 0);
+ KUNIT_EXPECT_PTR_EQ(test, s->refs[64].cluster, cluster);
+ KUNIT_EXPECT_EQ(test, s->refs[64].token, (u32)(BIT(9) + 64));
+ KUNIT_EXPECT_NOT_NULL(test, cluster->segments[1]);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 0UL);
+ mc16_test_cancel_pending(&s->refs[64]);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 1UL);
+}
+
+static void memcomp_test_retained_x2_replace(struct kunit *test)
+{
+ struct mc16_test_storage *s = mc16_test_storage_begin(test);
+ unsigned int i;
+ u32 old_id = 0;
+
+ KUNIT_ASSERT_NOT_NULL(test, s);
+ for (i = 0; i < MC16_CLUSTER_SLOTS - 1; i++) {
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(s, 1), 0);
+ if (!i)
+ old_id = s->refs[0].cluster->id;
+ mc16_test_cancel_pending(&s->refs[i]);
+ }
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 1UL);
+ /* ALIGN(511, 2) cannot fit: replace an empty, appendable current. */
+ KUNIT_ASSERT_EQ(test, mc16_test_reserve(s, 2), 0);
+ KUNIT_EXPECT_NE(test, s->refs[511].cluster->id, old_id);
+ KUNIT_EXPECT_PTR_EQ(test, xa_load(&mc16_clusters, old_id), NULL);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 0UL);
+}
+
+static void memcomp_test_meta_shrinker_exact(struct kunit *test)
+{
+ struct mc16_test_storage *s = mc16_test_storage_begin(test);
+ struct shrink_control sc = { .nr_to_scan = 1 };
+ u32 ordinary, zero;
+
+ KUNIT_ASSERT_NOT_NULL(test, s);
+ KUNIT_ASSERT_EQ(test, mc16_zh_reserve(s, 1, false), 0);
+ KUNIT_ASSERT_EQ(test, mc16_zh_reserve(s, 1, true), 0);
+ ordinary = s->refs[0].cluster->id;
+ zero = s->refs[1].cluster->id;
+ KUNIT_ASSERT_NE(test, ordinary, zero);
+ KUNIT_EXPECT_FALSE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_count(NULL, &sc), SHRINK_EMPTY);
+ mc16_test_cancel_pending(&s->refs[0]);
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_count(NULL, &sc), 1UL);
+ mc16_test_cancel_pending(&s->refs[1]);
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_count(NULL, &sc), 2UL);
+ WRITE_ONCE(mc16_meta_shrinker_cursor, ordinary);
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_scan(NULL, &sc), 1UL);
+ KUNIT_EXPECT_EQ(test, sc.nr_scanned, 1UL);
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_count(NULL, &sc), 1UL);
+ KUNIT_EXPECT_NOT_NULL(test, xa_load(&mc16_clusters, zero));
+ WRITE_ONCE(mc16_meta_shrinker_cursor, zero);
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_scan(NULL, &sc), 1UL);
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_count(NULL, &sc), SHRINK_EMPTY);
+ rcu_barrier();
+ KUNIT_EXPECT_TRUE(test, xa_empty(&mc16_clusters));
+ KUNIT_EXPECT_TRUE(test, hash_empty(mc16_keys));
+}
+
+static int mc16_zh_high_id(u32 *id)
+{
+ return __xa_alloc(&mc16_clusters, id, NULL,
+ XA_LIMIT(MC16_MAX_CLUSTER_ID, MC16_MAX_CLUSTER_ID),
+ GFP_NOWAIT | __GFP_NOWARN);
+}
+
+static void memcomp_test_meta_shrinker_wrap(struct kunit *test)
+{
+ struct mc16_test_storage *s = mc16_test_storage_begin(test);
+ struct shrink_control sc = { .nr_to_scan = 4096 };
+ unsigned long start = MC16_MAX_CLUSTER_ID - 31;
+ u32 high, private;
+ int ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, s);
+ KUNIT_ASSERT_EQ(test, mc16_zh_reserve(s, 1, false), 0);
+ KUNIT_EXPECT_EQ(test, s->refs[0].cluster->id, 1U);
+ kunit_activate_static_stub(test, mc16_private_id_alloc, mc16_zh_high_id);
+ KUNIT_EXPECT_EQ(test, mc16_zh_reserve(s, 1, true), 0);
+ kunit_deactivate_static_stub(test, mc16_private_id_alloc);
+ KUNIT_ASSERT_EQ(test, s->nr, 2U);
+ high = s->refs[1].cluster->id;
+ KUNIT_ASSERT_EQ(test, high, (u32)MC16_MAX_CLUSTER_ID);
+ /* A private zero entry must not consume a published-entry budget. */
+ xa_lock_irq(&mc16_clusters);
+ ret = __xa_alloc(&mc16_clusters, &private, NULL,
+ XA_LIMIT(start, start), GFP_NOWAIT | __GFP_NOWARN);
+ xa_unlock_irq(&mc16_clusters);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+ WRITE_ONCE(mc16_meta_shrinker_cursor, start);
+ /* Keep both entries live: a full wrap checks each exactly once. */
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_scan(NULL, &sc), 0UL);
+ KUNIT_EXPECT_EQ(test, sc.nr_scanned, 2UL);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_meta_shrinker_cursor), 1UL);
+ mc16_test_cancel_pending(&s->refs[0]);
+ mc16_test_cancel_pending(&s->refs[1]);
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_scan(NULL, &sc), 2UL);
+ KUNIT_EXPECT_EQ(test, sc.nr_scanned, 2UL);
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_count(NULL, &sc), SHRINK_EMPTY);
+ sc.nr_to_scan = 0;
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_scan(NULL, &sc), 0UL);
+ KUNIT_EXPECT_EQ(test, sc.nr_scanned, 0UL);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_meta_shrinker_cursor), 1UL);
+ xa_lock_irq(&mc16_clusters);
+ __xa_erase(&mc16_clusters, private);
+ xa_unlock_irq(&mc16_clusters);
+ rcu_barrier();
+}
+
+static void memcomp_test_meta_shrinker_progress(struct kunit *test)
+{
+ struct mc16_test_storage *s = mc16_test_storage_begin(test);
+ struct shrink_control sc = { .nr_to_scan = 1 };
+ unsigned long detached = 0;
+ unsigned int calls;
+
+ KUNIT_ASSERT_NOT_NULL(test, s);
+ KUNIT_ASSERT_EQ(test, mc16_zh_reserve(s, 1, false), 0);
+ /* A live entry naturally advances the cursor past all used IDs. */
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_scan(NULL, &sc), 0UL);
+ KUNIT_EXPECT_EQ(test, sc.nr_scanned, 1UL);
+ KUNIT_ASSERT_GT(test, READ_ONCE(mc16_meta_shrinker_cursor),
+ (unsigned long)s->refs[0].cluster->id);
+ mc16_test_cancel_pending(&s->refs[0]);
+ sc.nr_to_scan = 64;
+ for (calls = 0; calls < 2 && !detached; calls++)
+ detached += mc16_meta_shrinker_scan(NULL, &sc);
+ KUNIT_EXPECT_EQ(test, detached, 1UL);
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_count(NULL, &sc), SHRINK_EMPTY);
+ rcu_barrier();
+}
+
+static void memcomp_test_meta_shrinker_drained(struct kunit *test)
+{
+ struct mc16_test_storage *s = mc16_test_storage_begin(test);
+ struct shrink_control sc = { .nr_to_scan = 1 };
+
+ KUNIT_ASSERT_NOT_NULL(test, s);
+ KUNIT_ASSERT_EQ(test, mc16_zh_reserve(s, 1, false), 0);
+ mc16_test_cancel_pending(&s->refs[0]);
+ KUNIT_ASSERT_EQ(test, mc16_meta_shrinker_count(NULL, &sc), 1UL);
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_scan(NULL, &sc), 1UL);
+ KUNIT_EXPECT_EQ(test, sc.nr_scanned, 1UL);
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_count(NULL, &sc), SHRINK_EMPTY);
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_scan(NULL, &sc), SHRINK_STOP);
+ KUNIT_EXPECT_EQ(test, sc.nr_scanned, 0UL);
+ rcu_barrier();
+}
+
+#endif /* CONFIG_MEMCG */
+
+#if defined(MC16_FOLIO_TEST_ENABLED) && \
+ defined(MC16_ZERO_CONVERT_TEST_ENABLED) && \
+ defined(CONFIG_MEMCOMP_DEBUG_STATS)
+#define MC16_ZERO_HINT_DENSITY_TEST_ENABLED
+
+struct mc16_zh_fixture {
+ bool age_between_zeros;
+ struct kunit *test;
+ struct mc16_test_storage *storage;
+ struct memcomp_ref *live;
+ void *loaded;
+ struct mm_struct *mm;
+ unsigned long address;
+ unsigned int nr, capacity;
+ unsigned int issued[2];
+ struct mc16_key *key[2];
+ u32 zero_token;
+ u32 failed_owner_token;
+ unsigned int route_mismatches;
+ unsigned int key_mismatches;
+ unsigned int index_mismatches;
+ s64 swapents;
+ struct mc16_zero_owners zeros;
+};
+
+struct mc16_zh_ledger {
+ u64 count[MC16_META_CLASSES];
+ u64 bytes[MC16_META_CLASSES];
+ u64 attached[MC16_META_CLASSES];
+ u64 attached_bytes;
+ u64 retained_bytes;
+ u64 record_bytes;
+ u64 descriptor_tail, segment_tail, shadow_tail;
+ unsigned int unpaired_segments;
+};
+
+static u64 mc16_zh_allocated(struct mc16_zh_fixture *f,
+ enum mc16_storage_class type)
+{
+ struct mc16_storage_debug now;
+
+ mc16_test_metadata_snapshot(&now);
+ return now.allocated[type] - f->storage->baseline.allocated[type];
+}
+
+/* One real PTE, recycled after every canonical-zero transaction. */
+static int mc16_zh_clear_pte(struct mc16_zh_fixture *f)
+{
+ struct memcomp_ref positive = { };
+ /* Protects the mapped PTE while its entry is cleared. */
+ spinlock_t *ptl;
+ pte_t *ptep, pte;
+ pmd_t *pmd;
+ int ret = 0;
+
+ mmap_read_lock(f->mm);
+ pmd = mm_find_pmd(f->mm, f->address);
+ if (!pmd)
+ goto out;
+ ptep = pte_offset_map_lock(f->mm, pmd, f->address, &ptl);
+ if (!ptep)
+ goto out;
+ pte = ptep_get(ptep);
+ if (is_swap_pte(pte) && is_memcomp_entry(pte_to_swp_entry(pte))) {
+ unsigned long token = swp_offset(pte_to_swp_entry(pte));
+
+ if (token && !mc16_get(token, &positive)) {
+ ret = -ENOENT;
+ } else {
+ ptep_get_and_clear(f->mm, f->address, ptep);
+ add_mm_counter(f->mm, MM_SWAPENTS, -1);
+ if (positive.slot)
+ mc16_ref_put_count(&positive, 2);
+ else
+ mc16_zero_pte_put(1);
+ }
+ }
+ pte_unmap_unlock(ptep, ptl);
+out:
+ mmap_read_unlock(f->mm);
+ return ret;
+}
+
+/* Recover only the existing initial OWNER, never manufacture a lookup. */
+static void mc16_zh_cleanup_failed_owner(struct mc16_zh_fixture *f)
+{
+ struct folio *folio = f->storage->owner->folio;
+ struct memcomp_ref owner = { };
+ struct mc16_cluster *cluster;
+ struct mc16_slot *segment;
+ struct mc16_view view;
+ u32 token = f->failed_owner_token;
+ unsigned int index = token & (MC16_CLUSTER_SLOTS - 1);
+ unsigned long flags;
+ bool valid = false;
+
+ if (!token)
+ return;
+ xa_lock_irqsave(&mc16_clusters, flags);
+ cluster = xa_load(&mc16_clusters, token >> MC16_CLUSTER_SHIFT);
+ if (cluster && !xa_is_value(cluster) && !xa_is_internal(cluster) &&
+ cluster->id == token >> MC16_CLUSTER_SHIFT && index < cluster->next) {
+ segment = cluster->segments[index >> MC16_SEGMENT_SHIFT];
+ if (segment &&
+ refcount_read(&segment[index & (MC16_SEGMENT_SLOTS - 1)].refs) == 1) {
+ owner = (struct memcomp_ref) {
+ .cluster = cluster,
+ .slot = &segment[index & (MC16_SEGMENT_SLOTS - 1)],
+ .token = token,
+ };
+ }
+ }
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ if (!owner.slot) {
+ KUNIT_FAIL(f->test, "failed lookup token %u has no sole OWNER", token);
+ return;
+ }
+ /* The outstanding initial OWNER keeps these objects alive after X. */
+ mutex_lock(memcomp_lock_of(token));
+ if (mc16_view_locked(&owner, &view) && view.record &&
+ mc16_flags_valid(*view.flags) &&
+ mc16_flags_state(*view.flags) == MC16_PENDING &&
+ (*view.flags & (MC16_F_OWNER | MC16_F_RESIDENT)) ==
+ (MC16_F_OWNER | MC16_F_RESIDENT) &&
+ view.member->owner_folio == folio && view.record->producer == folio &&
+ view.record->nr_pages == 1 && view.record->token_base == token &&
+ refcount_read(&owner.slot->refs) == 1)
+ valid = true;
+ mutex_unlock(memcomp_lock_of(token));
+ if (!valid) {
+ KUNIT_FAIL(f->test, "failed lookup token %u OWNER identity changed", token);
+ return;
+ }
+ f->failed_owner_token = 0;
+ /* anon_reserve already accounted this member; use the full final put. */
+ mc16_ref_put(&owner);
+}
+
+static void mc16_zh_cleanup(void *data)
+{
+ struct mc16_zh_fixture *f = data;
+ struct mc16_zero_owners after;
+ unsigned int i;
+
+ if (f->mm)
+ KUNIT_EXPECT_EQ(f->test, mc16_zh_clear_pte(f), 0);
+ mc16_zh_cleanup_failed_owner(f);
+ for (i = 0; i < f->nr; i++) {
+ if (f->live[i].slot)
+ mc16_ref_put(&f->live[i]);
+ }
+ mc16_test_drain_segments(&f->storage->probe);
+ mc16_test_reclaim_retained();
+ KUNIT_EXPECT_EQ(f->test, READ_ONCE(mc16_retained_empty_headers), 0UL);
+ if (f->mm) {
+ mc16_zero_owners_read(&after);
+ KUNIT_EXPECT_EQ(f->test, after.pte_pages, f->zeros.pte_pages);
+ KUNIT_EXPECT_EQ(f->test, after.cache_pages, f->zeros.cache_pages);
+ KUNIT_EXPECT_EQ(f->test, mc16_test_zc_rss(f->mm, MM_SWAPENTS),
+ f->swapents);
+ mmput(f->mm);
+ }
+ kvfree(f->live);
+}
+
+static struct mc16_zh_fixture *mc16_zh_begin(struct kunit *test,
+ unsigned int nr)
+{
+ struct mc16_zh_fixture *f;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->test = test;
+ f->storage = mc16_test_storage_begin_pages(test, 1);
+ if (!f->storage)
+ return NULL;
+ f->loaded = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+ if (!f->loaded)
+ return NULL;
+ f->capacity = nr;
+ f->live = kvcalloc(nr, sizeof(*f->live), GFP_KERNEL);
+ if (!f->live)
+ return NULL;
+ /* Register before activation; partial setup has the same cleanup. */
+ if (kunit_add_action_or_reset(test, mc16_zh_cleanup, f))
+ return NULL;
+ if (!xa_empty(&mc16_clusters) || !hash_empty(mc16_keys))
+ return NULL;
+ f->address = kunit_vm_mmap(test, NULL, 0, PAGE_SIZE,
+ PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, 0);
+ if (!f->address || IS_ERR_VALUE(f->address))
+ return NULL;
+ f->mm = current->mm;
+ mmget(f->mm);
+ f->swapents = mc16_test_zc_rss(f->mm, MM_SWAPENTS);
+ mc16_zero_owners_read(&f->zeros);
+ /* Run token cleanup before kunit_vm_mmap() removes the VMA. */
+ kunit_remove_action(test, mc16_zh_cleanup, f);
+ if (kunit_add_action_or_reset(test, mc16_zh_cleanup, f))
+ return NULL;
+ /* Retain real segment callbacks for the pre-drain ledger. */
+ f->storage->probe.hold_segments = true;
+ return f;
+}
+
+static bool mc16_zh_is_zero(unsigned int page)
+{
+ unsigned int i = page % 512;
+
+ /* Exactly 42 zeros/block; every unsegregated 64-slot segment has both. */
+ return ((i + 1) * 42 / 512) != (i * 42 / 512);
+}
+
+static int mc16_zh_convert(struct mc16_zh_fixture *f, struct memcomp_ref *ref)
+{
+ struct memcomp_ref pte_owner = { };
+ struct mc16_zero_walk walk = { .ref = *ref };
+ struct vm_area_struct *vma;
+ /* Protects the PTE while the memcomp entry is installed. */
+ spinlock_t *ptl;
+ pte_t *ptep;
+ int ret = 0;
+
+ if (!mc16_get(ref->token, &pte_owner))
+ return -ENOENT;
+ mmap_read_lock(f->mm);
+ ptep = get_locked_pte(f->mm, f->address, &ptl);
+ if (!ptep) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ __acquire(RCU);
+ if (!pte_none(ptep_get(ptep))) {
+ ret = -EEXIST;
+ } else {
+ set_pte_at(f->mm, f->address, ptep,
+ swp_entry_to_pte(make_memcomp_entry(ref->token)));
+ add_mm_counter(f->mm, MM_SWAPENTS, 1);
+ pte_owner = (struct memcomp_ref) { };
+ }
+ pte_unmap_unlock(ptep, ptl);
+ if (ret)
+ goto out;
+ vma = vma_lookup(f->mm, f->address);
+ if (vma)
+ mc16_zero_convert_one(f->storage->owner->folio, vma,
+ f->address, &walk);
+ if (walk.converted != 1)
+ ret = -EIO;
+out:
+ mmap_read_unlock(f->mm);
+ if (pte_owner.slot)
+ mc16_ref_put(&pte_owner);
+ if (!ret) {
+ pte_t pte = mc16_test_zc_read(&(struct mc16_test_zc) {
+ .mm = f->mm, .addr = f->address }, f->address);
+
+ if (!is_swap_pte(pte) ||
+ pte_to_swp_entry(pte).val != make_memcomp_entry(0).val)
+ ret = -EINVAL;
+ }
+ if (mc16_zh_clear_pte(f) && !ret)
+ ret = -EIO;
+ return ret;
+}
+
+/* Reserve scans the locked source; refresh uses the real codec and charge. */
+static int mc16_zh_page(struct mc16_zh_fixture *f, unsigned int page,
+ bool release_ordinary)
+{
+ struct folio *folio = f->storage->owner->folio;
+ struct memcomp_ref ref = { }, drop = { };
+ struct memcomp_reclaim_ctx *ctx = NULL;
+ unsigned long token;
+ unsigned int flags, i, route = mc16_zh_is_zero(page);
+ void *src;
+ int ret;
+
+ src = kmap_local_folio(folio, 0);
+ for (i = 0; i < PAGE_SIZE; i++)
+ ((u8 *)src)[i] = route ? 0 : i % 53;
+ kunmap_local(src);
+ if (mc16_folio_zero_hint(folio) != !!route)
+ return -EINVAL;
+ ret = mc16_anon_reserve(folio, 1, NULL, &token);
+ if (ret)
+ return ret;
+ /* Save a lookup before any operation which may fail. */
+ if (!mc16_get(token, &ref)) {
+ f->failed_owner_token = token;
+ return -ENOENT;
+ }
+ flags = memcomp_test_reclaimer_enter();
+ ctx = memcomp_reclaim_ctx_get(1, MEMCOMP_RECLAIM_KSWAPD);
+ if (!ctx) {
+ ret = -EBUSY;
+ goto out;
+ }
+ /* GFP_KERNEL order-0 folio: use its stable lowmem address while sleeping. */
+ src = page_address(&folio->page);
+ ret = mc16_anon_refresh(&ctx, src, 1, NULL, &ref);
+ memcomp_reclaim_ctx_put(ctx);
+out:
+ memcomp_test_reclaimer_exit(flags);
+ if (!ret) {
+ struct mc16_key *key = ref.cluster->key;
+ struct mc16_view view;
+ bool valid;
+ unsigned int state = MC16_EMPTY;
+
+ mutex_lock(memcomp_lock_of(token));
+ valid = mc16_view_locked(&ref, &view);
+ if (valid)
+ state = mc16_flags_state(*view.flags);
+ mutex_unlock(memcomp_lock_of(token));
+ KUNIT_EXPECT_TRUE(f->test, valid);
+ KUNIT_EXPECT_EQ(f->test, state,
+ route ? (unsigned int)MC16_SAMEFILL :
+ (unsigned int)MC16_STORED);
+ f->route_mismatches += key->zero_hint != (u8)route;
+ if (f->issued[route] % MC16_CLUSTER_SLOTS)
+ f->key_mismatches += key != f->key[route];
+ f->key[route] = key;
+ f->index_mismatches += (ref.token & (MC16_CLUSTER_SLOTS - 1)) !=
+ f->issued[route] % MC16_CLUSTER_SLOTS;
+ f->issued[route]++;
+ if (route) {
+ f->zero_token = ref.token;
+ ret = mc16_zh_convert(f, &ref);
+ }
+ }
+ /* Drop source OWNER with the same production normalization as release. */
+ if (mc16_get(token, &drop)) {
+ if (!mc16_owner_drop_folio_held(&drop, folio, 1, 0) && !ret)
+ ret = -EIO;
+ } else if (!ret) {
+ ret = -ENOENT;
+ }
+ if (!ret && !route && release_ordinary) {
+ ret = mc16_load_page(&ref, f->loaded);
+ if (!ret && memcmp(f->loaded, page_address(&folio->page), PAGE_SIZE))
+ ret = -EILSEQ;
+ }
+ if (ret || route || release_ordinary) {
+ mc16_ref_put(&ref);
+ } else if (f->nr < f->capacity) {
+ f->live[f->nr++] = ref;
+ } else {
+ mc16_ref_put(&ref);
+ ret = -ENOSPC;
+ }
+ return ret;
+}
+
+/* No constants stand in for slab/XA/infrastructure measurements. */
+static void mc16_zh_ledger(struct mc16_zh_fixture *f, const char *phase,
+ struct mc16_zh_ledger *l)
+{
+ struct mc16_storage_debug now;
+ struct mc16_storage_debug *base = &f->storage->baseline;
+ struct mc16_cluster *cluster;
+ struct mc16_key *key;
+ unsigned long id, flags;
+ unsigned int type, seg, bucket, i;
+ u64 key_bytes = 0;
+
+ memset(l, 0, sizeof(*l));
+ mc16_test_metadata_snapshot(&now);
+ for (type = 0; type < MC16_META_CLASSES; type++) {
+ l->count[type] = now.allocated[type] - base->allocated[type] -
+ (now.freed[type] - base->freed[type]);
+ l->bytes[type] = now.allocated_bytes[type] - base->allocated_bytes[type] -
+ (now.freed_bytes[type] - base->freed_bytes[type]);
+ }
+ /* Attached pointers only survive X while their real sizes are sampled. */
+ xa_lock_irqsave(&mc16_clusters, flags);
+ xa_for_each(&mc16_clusters, id, cluster) {
+ if (xa_is_value(cluster) || xa_is_internal(cluster))
+ continue;
+ l->attached[MC16_META_CLUSTER]++;
+ l->attached_bytes += ksize(cluster);
+ if (!cluster->live_slots && cluster->key->current_cluster == cluster)
+ l->retained_bytes += ksize(cluster);
+ for (seg = 0; seg < MC16_SEGMENTS; seg++) {
+ u64 bytes = 0;
+
+ l->unpaired_segments += !!cluster->segments[seg] !=
+ !!cluster->shadows[seg];
+ if (cluster->segments[seg]) {
+ l->attached[MC16_META_SEGMENT]++;
+ bytes += ksize(cluster->segments[seg]);
+ }
+ if (cluster->shadows[seg]) {
+ l->attached[MC16_META_SHADOW]++;
+ bytes += ksize(cluster->shadows[seg]);
+ }
+ l->attached_bytes += bytes;
+ if (!cluster->live_slots &&
+ cluster->key->current_cluster == cluster)
+ l->retained_bytes += bytes;
+ }
+ }
+ hash_for_each(mc16_keys, bucket, key, node) {
+ l->attached[MC16_META_KEY]++;
+ key_bytes += ksize(key);
+ }
+ xa_unlock_irqrestore(&mc16_clusters, flags);
+ KUNIT_EXPECT_EQ(f->test, l->unpaired_segments, 0U);
+ /* Order-0 live slots have disjoint records, if normalization failed. */
+ for (i = 0; i < f->nr; i++) {
+ struct mc16_view view;
+
+ if (!f->live[i].slot)
+ continue;
+ mutex_lock(memcomp_lock_of(f->live[i].token));
+ if (mc16_view_locked(&f->live[i], &view) && view.record)
+ l->record_bytes += ksize(view.record);
+ mutex_unlock(memcomp_lock_of(f->live[i].token));
+ }
+ l->descriptor_tail = l->count[MC16_META_DESCRIPTOR];
+ l->segment_tail = l->count[MC16_META_SEGMENT] -
+ l->attached[MC16_META_SEGMENT];
+ l->shadow_tail = l->count[MC16_META_SHADOW] -
+ l->attached[MC16_META_SHADOW];
+ kunit_info(f->test,
+ "%s allocated-live segment/header/key/descriptor/shadow=%llu/%llu/%llu/%llu/%llu bytes=%llu/%llu/%llu/%llu/%llu\n",
+ phase, l->count[0], l->count[1], l->count[2], l->count[3],
+ l->count[MC16_META_SHADOW],
+ l->bytes[0], l->bytes[1], l->bytes[2], l->bytes[3],
+ l->bytes[MC16_META_SHADOW]);
+ kunit_info(f->test,
+ "%s attached segment/header/key/shadow=%llu/%llu/%llu/%llu attached_bytes=%llu key_bytes=%llu retained_bytes=%llu\n",
+ phase, l->attached[0], l->attached[1], l->attached[2],
+ l->attached[MC16_META_SHADOW],
+ l->attached_bytes, key_bytes, l->retained_bytes);
+}
+
+static void mc16_zh_drain(struct mc16_zh_fixture *f)
+{
+ mc16_test_drain_segments(&f->storage->probe);
+ rcu_barrier();
+}
+
+static void mc16_zh_expect_drained(struct mc16_zh_fixture *f,
+ struct mc16_zh_ledger *l)
+{
+ unsigned int i;
+
+ KUNIT_EXPECT_EQ(f->test, l->record_bytes, 0ULL);
+ KUNIT_EXPECT_EQ(f->test, l->descriptor_tail, 0ULL);
+ KUNIT_EXPECT_EQ(f->test, l->segment_tail, 0ULL);
+ KUNIT_EXPECT_EQ(f->test, l->shadow_tail, 0ULL);
+ KUNIT_EXPECT_EQ(f->test, l->bytes[MC16_META_SEGMENT] +
+ l->bytes[MC16_META_CLUSTER] + l->bytes[MC16_META_SHADOW],
+ l->attached_bytes);
+ kunit_info(f->test,
+ "post-drain private residual segment/header/key/shadow=%lld/%lld/%lld/%lld bytes=%lld\n",
+ (s64)l->count[0] - (s64)l->attached[0],
+ (s64)l->count[1] - (s64)l->attached[1],
+ (s64)l->count[2] - (s64)l->attached[2],
+ (s64)l->count[MC16_META_SHADOW] -
+ (s64)l->attached[MC16_META_SHADOW],
+ (s64)(l->bytes[0] + l->bytes[1] + l->bytes[MC16_META_SHADOW]) -
+ (s64)l->attached_bytes);
+ /* No unexplained population is silently classified as freed/private. */
+ for (i = 0; i < MC16_META_CLASSES; i++)
+ KUNIT_EXPECT_EQ(f->test, l->count[i], l->attached[i]);
+}
+
+/* Keep per-page checks, but emit bounded diagnostics for routing mutants. */
+static void mc16_zh_expect_routes(struct mc16_zh_fixture *f)
+{
+ KUNIT_EXPECT_EQ(f->test, f->route_mismatches, 0U);
+ KUNIT_EXPECT_EQ(f->test, f->key_mismatches, 0U);
+ KUNIT_EXPECT_EQ(f->test, f->index_mismatches, 0U);
+}
+
+static void mc16_zh_fill(struct kunit *test, struct mc16_zh_fixture *f,
+ unsigned int pages, bool serial)
+{
+ unsigned int i;
+
+ for (i = 0; i < pages; i++) {
+ int ret = mc16_zh_page(f, i, serial);
+
+ if (ret) {
+ KUNIT_FAIL(test, "page %u/%u failed: %d", i, pages, ret);
+ return;
+ }
+ if (mc16_zh_is_zero(i)) {
+ struct memcomp_ref probe = { };
+ bool got = mc16_get(f->zero_token, &probe);
+
+ if (got)
+ mc16_ref_put(&probe);
+ KUNIT_EXPECT_FALSE(test, got);
+ if (f->age_between_zeros)
+ mc16_retained_age_work(&mc16_retained_aging.work);
+ }
+ cond_resched();
+ }
+ mc16_zh_expect_routes(f);
+}
+
+static void memcomp_test_zero_hint_ordinary_refault(struct kunit *test)
+{
+ struct mc16_zh_fixture *f = mc16_zh_begin(test, 128);
+ unsigned int i;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ /* Page 0 is ordinary; store, drop OWNER, decode, and final-put each time. */
+ for (i = 0; i < 128; i++)
+ KUNIT_ASSERT_EQ(test, mc16_zh_page(f, 0, true), 0);
+ mc16_zh_expect_routes(f);
+ KUNIT_EXPECT_EQ(test, f->issued[0], 128U);
+ KUNIT_EXPECT_EQ(test, f->issued[1], 0U);
+ KUNIT_EXPECT_EQ(test, mc16_zh_allocated(f, MC16_META_CLUSTER), 1ULL);
+ KUNIT_EXPECT_EQ(test, mc16_zh_allocated(f, MC16_META_SEGMENT), 2ULL);
+ KUNIT_EXPECT_EQ(test, mc16_zh_allocated(f, MC16_META_SHADOW), 2ULL);
+ KUNIT_EXPECT_EQ(test, mc16_zh_allocated(f, MC16_META_KEY), 1ULL);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 1UL);
+ KUNIT_EXPECT_EQ(test, f->key[0]->current_cluster->next, (u16)128);
+}
+
+static void memcomp_test_zero_hint_serial533(struct kunit *test)
+{
+ struct mc16_zh_fixture *f = mc16_zh_begin(test, 512);
+ struct mc16_zh_ledger l;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ f->age_between_zeros = true;
+ mc16_zh_fill(test, f, 512, true);
+ KUNIT_ASSERT_EQ(test, f->issued[0], 470U);
+ KUNIT_ASSERT_EQ(test, f->issued[1], 42U);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 2UL);
+ KUNIT_EXPECT_EQ(test, mc16_zh_allocated(f, MC16_META_SEGMENT), 9ULL);
+ KUNIT_EXPECT_EQ(test, mc16_zh_allocated(f, MC16_META_SHADOW), 9ULL);
+ KUNIT_EXPECT_EQ(test, mc16_zh_allocated(f, MC16_META_CLUSTER), 2ULL);
+ KUNIT_EXPECT_EQ(test, mc16_zh_allocated(f, MC16_META_KEY), 2ULL);
+ KUNIT_EXPECT_EQ(test, mc16_zh_allocated(f, MC16_META_DESCRIPTOR), 7ULL);
+ mc16_zh_ledger(f, "pre-drain", &l);
+ mc16_zh_drain(f);
+ mc16_zh_ledger(f, "post-drain", &l);
+ mc16_zh_expect_drained(f, &l);
+ KUNIT_EXPECT_EQ(test, l.attached[MC16_META_CLUSTER], 2ULL);
+ KUNIT_EXPECT_EQ(test, l.attached[MC16_META_SEGMENT], 2ULL);
+ KUNIT_EXPECT_EQ(test, l.attached[MC16_META_SHADOW], 2ULL);
+ /* A separate pressure checkpoint after the serial lifecycle checks. */
+ mc16_test_reclaim_retained();
+ mc16_zh_ledger(f, "pressure-cleanup", &l);
+ KUNIT_EXPECT_EQ(test, l.attached[MC16_META_CLUSTER], 0ULL);
+ KUNIT_EXPECT_EQ(test, l.attached[MC16_META_KEY], 0ULL);
+ KUNIT_EXPECT_EQ(test, l.attached[MC16_META_SHADOW], 0ULL);
+}
+
+static void memcomp_test_zero_hint_route512(struct kunit *test)
+{
+ struct mc16_zh_fixture *f = mc16_zh_begin(test, 512);
+ struct mc16_zh_ledger l;
+ struct shrink_control sc = { .nr_to_scan = 1 };
+ struct mc16_cluster *zero;
+ unsigned int i, ordinary_holes = 0;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_zh_fill(test, f, 512, false);
+ KUNIT_ASSERT_EQ(test, f->issued[0], 470U);
+ KUNIT_ASSERT_EQ(test, f->issued[1], 42U);
+ KUNIT_ASSERT_EQ(test, f->nr, 470U);
+ KUNIT_EXPECT_PTR_NE(test, f->key[0], f->key[1]);
+ KUNIT_EXPECT_EQ(test, f->key[0]->current_cluster->next, (u16)470);
+ zero = f->key[1]->current_cluster;
+ KUNIT_ASSERT_NOT_NULL(test, zero);
+ KUNIT_EXPECT_EQ(test, zero->next, (u16)42);
+ KUNIT_EXPECT_EQ(test, zero->live_slots, 0U);
+ /* A dense ordinary token sequence proves there are no zero holes. */
+ for (i = 0; i < f->nr; i++)
+ ordinary_holes += f->live[i].token != f->live[0].token + i;
+ KUNIT_EXPECT_EQ(test, ordinary_holes, 0U);
+ mc16_zh_ledger(f, "pre-drain", &l);
+ WRITE_ONCE(mc16_meta_shrinker_cursor, zero->id);
+ KUNIT_EXPECT_EQ(test, mc16_meta_shrinker_scan(NULL, &sc), 1UL);
+ mc16_zh_drain(f);
+ mc16_zh_ledger(f, "post-drain", &l);
+ mc16_zh_expect_drained(f, &l);
+ KUNIT_EXPECT_EQ(test, l.attached[MC16_META_SEGMENT], 8ULL);
+ KUNIT_EXPECT_EQ(test, l.attached[MC16_META_SHADOW], 8ULL);
+ KUNIT_EXPECT_EQ(test, l.attached[MC16_META_CLUSTER], 1ULL);
+ KUNIT_EXPECT_EQ(test, l.attached[MC16_META_KEY], 1ULL);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 0UL);
+ for (i = 0; i < f->nr; i++)
+ KUNIT_EXPECT_EQ(test, refcount_read(&f->live[i].slot->refs), 1U);
+ /* Routing gate only; 470 payload pages cannot meet 16.4 B/page. */
+}
+
+static void memcomp_test_zero_hint_density65536(struct kunit *test)
+{
+ struct mc16_zh_fixture *f = mc16_zh_begin(test, 65536);
+ struct mc16_zh_ledger l;
+ struct mc16_cluster *zero;
+ unsigned int seg;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_zh_fill(test, f, 65536, false);
+ KUNIT_ASSERT_EQ(test, f->issued[0], 60160U);
+ KUNIT_ASSERT_EQ(test, f->issued[1], 5376U);
+ KUNIT_ASSERT_EQ(test, f->nr, 60160U);
+ KUNIT_EXPECT_EQ(test, mc16_zh_allocated(f, MC16_META_CLUSTER), 129ULL);
+ KUNIT_ASSERT_NOT_NULL(test, memcomp_meta_shrinker);
+ zero = f->key[1]->current_cluster;
+ KUNIT_ASSERT_NOT_NULL(test, zero);
+ KUNIT_EXPECT_EQ(test, zero->next, (u16)256);
+ KUNIT_EXPECT_EQ(test, zero->live_slots, 0U);
+ /* 5376 is a multiple of 64: no unsealed zero tail is required. */
+ for (seg = 0; seg < MC16_SEGMENTS; seg++) {
+ KUNIT_EXPECT_PTR_EQ(test, zero->segments[seg], NULL);
+ KUNIT_EXPECT_PTR_EQ(test, zero->shadows[seg], NULL);
+ }
+ mc16_zh_ledger(f, "pre-drain", &l);
+ /* Callback drain only. Do not pressure-evict the final zero header. */
+ mc16_zh_drain(f);
+ mc16_zh_ledger(f, "post-drain", &l);
+ mc16_zh_expect_drained(f, &l);
+ KUNIT_EXPECT_EQ(test, l.attached[MC16_META_SEGMENT], 940ULL);
+ KUNIT_EXPECT_EQ(test, l.attached[MC16_META_SHADOW], 940ULL);
+ KUNIT_EXPECT_EQ(test, l.attached[MC16_META_CLUSTER], 119ULL);
+ KUNIT_EXPECT_EQ(test, l.attached[MC16_META_KEY], 2ULL);
+ KUNIT_EXPECT_EQ(test, READ_ONCE(mc16_retained_empty_headers), 1UL);
+ KUNIT_EXPECT_EQ(test, l.retained_bytes, (u64)ksize(zero));
+ /* Population coverage only: XA-inclusive density is not measured here. */
+}
+
+#endif /* density fixtures */
+#endif /* _MM_MEMCOMP_TESTS_ZERO_HINT_TEST_H */
diff --git a/mm/memcomp/tests/zero_load_test.h b/mm/memcomp/tests/zero_load_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..db252a0c8f94723a0a8ef21f29db630379ccd456
--- /dev/null
+++ b/mm/memcomp/tests/zero_load_test.h
@@ -0,0 +1,645 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_ZERO_LOAD_TEST_H
+#define _MM_MEMCOMP_TESTS_ZERO_LOAD_TEST_H
+
+#include <kunit/resource.h>
+#include <kunit/test.h>
+#include <linux/completion.h>
+#include <linux/kthread.h>
+#include <linux/mman.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task.h>
+
+/*
+ * Include after zero_load.h and the storage/refs test fixtures in suite.h.
+ * These are real mmap/PTE/folio/rmap actors calling mc16_load_zero() directly.
+ * No old entry fixture, old zero ledger or public dispatch is used. Under
+ * MEMCG an unrelated real STORED slot uses the existing modeled objcg fixture;
+ * the fault folio itself always uses the real MM allocator and memcg charge.
+ */
+#if defined(CONFIG_MEMCOMP_KUNIT_TEST) && defined(CONFIG_MMU) && \
+ defined(CONFIG_ADVISE_SYSCALLS)
+#define MC16_ZERO_LOAD_MM_TEST_ENABLED
+
+struct mc16_zl_storage {
+ s64 stored;
+ s64 compressed;
+#ifdef CONFIG_MEMCG
+ unsigned int allocations;
+ unsigned int refs;
+ int gets;
+ int puts;
+ int charges;
+ int uncharges;
+ int folio_refs;
+ long charged;
+ u64 word;
+ u16 flags;
+ u16 length;
+#endif
+};
+
+struct mc16_zl_mm {
+ struct kunit *test;
+ struct mm_struct *mm;
+ unsigned long addr;
+ vm_flags_t flags;
+ pgprot_t prot;
+ bool saved_vma;
+ struct mc16_zero_owners owners;
+ s64 swapents;
+ s64 anonpages;
+ /* Pin any sampled present folio under PTL until cleanup, even on ASSERT. */
+ struct folio *folio;
+ struct mc16_zl_storage storage;
+#ifdef CONFIG_MEMCG
+ struct mc16_test_refs *refs;
+#endif
+#ifdef CONFIG_PER_VMA_LOCK
+ struct completion writer_locked;
+ struct completion writer_release;
+ struct task_struct *writer;
+#endif
+};
+
+static s64 mc16_zl_rss(struct mm_struct *mm, int member)
+{
+ return percpu_counter_sum(&mm->rss_stat[member]);
+}
+
+static void mc16_zl_storage_read(struct mc16_zl_mm *f,
+ struct mc16_zl_storage *out)
+{
+#ifdef CONFIG_MEMCG
+ struct mc16_test_storage *s = f->refs->storage;
+ struct memcomp_ref *ref = &s->refs[0];
+ struct mc16_view view;
+
+ out->allocations = s->probe.allocations;
+ out->refs = refcount_read(&ref->slot->refs);
+ out->gets = atomic_read(&s->owner->model.get_calls);
+ out->puts = atomic_read(&s->owner->model.put_calls);
+ out->charges = atomic_read(&s->owner->model.charge_calls);
+ out->uncharges = atomic_read(&s->owner->model.uncharge_calls);
+ out->folio_refs = folio_ref_count(s->owner->folio);
+ out->charged = atomic_long_read(&s->owner->model.charged);
+ mutex_lock(memcomp_lock_of(ref->token));
+ mc16_view_locked(ref, &view);
+ out->word = *view.word;
+ out->flags = *view.flags;
+ out->length = *view.length;
+ mutex_unlock(memcomp_lock_of(ref->token));
+#endif
+ out->stored = atomic64_read(&memcomp_stored_bytes);
+ out->compressed = atomic64_read(&memcomp_compressed_bytes);
+}
+
+static void mc16_zl_storage_unchanged(struct mc16_zl_mm *f)
+{
+ struct mc16_zl_storage after;
+ struct kunit *test = f->test;
+#ifdef CONFIG_MEMCG
+#endif
+
+ mc16_zl_storage_read(f, &after);
+ KUNIT_EXPECT_EQ(test, after.stored, f->storage.stored);
+ KUNIT_EXPECT_EQ(test, after.compressed, f->storage.compressed);
+#ifdef CONFIG_MEMCG
+ KUNIT_EXPECT_EQ(test, after.allocations, f->storage.allocations);
+ KUNIT_EXPECT_EQ(test, after.refs, f->storage.refs);
+ KUNIT_EXPECT_EQ(test, after.gets, f->storage.gets);
+ KUNIT_EXPECT_EQ(test, after.puts, f->storage.puts);
+ KUNIT_EXPECT_EQ(test, after.charges, f->storage.charges);
+ KUNIT_EXPECT_EQ(test, after.uncharges, f->storage.uncharges);
+ KUNIT_EXPECT_EQ(test, after.folio_refs, f->storage.folio_refs);
+ KUNIT_EXPECT_EQ(test, after.charged, f->storage.charged);
+ KUNIT_EXPECT_EQ(test, after.word, f->storage.word);
+ KUNIT_EXPECT_EQ(test, after.flags, f->storage.flags);
+ KUNIT_EXPECT_EQ(test, after.length, f->storage.length);
+#endif
+}
+
+static void mc16_zl_counts(struct mc16_zl_mm *f, s64 ptes, s64 present)
+{
+ struct mc16_zero_owners owners;
+
+ mc16_zero_owners_read(&owners);
+ KUNIT_EXPECT_EQ(f->test, owners.pte_pages, f->owners.pte_pages + ptes);
+ KUNIT_EXPECT_EQ(f->test, owners.cache_pages, f->owners.cache_pages);
+ KUNIT_EXPECT_EQ(f->test, mc16_zl_rss(f->mm, MM_SWAPENTS),
+ f->swapents + ptes);
+ KUNIT_EXPECT_EQ(f->test, mc16_zl_rss(f->mm, MM_ANONPAGES),
+ f->anonpages + present);
+ mc16_zl_storage_unchanged(f);
+}
+
+static pte_t mc16_zl_zero_pte(void)
+{
+ return swp_entry_to_pte(make_memcomp_entry(0));
+}
+
+/* A real zap actor; old entry80 invalidation cannot retire this ledger. */
+static void mc16_zl_zap_zero(struct mc16_zl_mm *f)
+{
+ spinlock_t *ptl; /* Serializes the actual token0 PTE owner. */
+ pte_t *ptep, pte;
+ pmd_t *pmd;
+
+ mmap_read_lock(f->mm);
+ pmd = mm_find_pmd(f->mm, f->addr);
+ if (!pmd)
+ goto out;
+ ptep = pte_offset_map_lock(f->mm, pmd, f->addr, &ptl);
+ if (!ptep)
+ goto out;
+ pte = ptep_get(ptep);
+ if (is_swap_pte(pte) && is_memcomp_entry(pte_to_swp_entry(pte)) &&
+ !swp_offset(pte_to_swp_entry(pte))) {
+ ptep_get_and_clear(f->mm, f->addr, ptep);
+ add_mm_counter(f->mm, MM_SWAPENTS, -1);
+ mc16_zero_pte_put(1);
+ }
+ pte_unmap_unlock(ptep, ptl);
+out:
+ mmap_read_unlock(f->mm);
+}
+
+static void mc16_zl_cleanup(void *data)
+{
+ struct mc16_zl_mm *f = data;
+ struct vm_area_struct *vma;
+ int ret;
+
+ WRITE_ONCE(mc16_zero_fault_probe.task, NULL);
+ mc16_zero_fault_probe.fail_alloc = false;
+ mc16_zero_fault_probe.fail_charge = false;
+#ifdef CONFIG_PER_VMA_LOCK
+ if (f->writer) {
+ complete_all(&f->writer_release);
+ kthread_stop_put(f->writer);
+ f->writer = NULL;
+ }
+#endif
+ mc16_zl_zap_zero(f);
+ mmap_write_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (vma && f->saved_vma) {
+ vma_start_write(vma);
+ vm_flags_reset(vma, f->flags);
+ vma->vm_page_prot = f->prot;
+ }
+ mmap_write_unlock(f->mm);
+ /* Normal MM removes rmap/LRU and drops the mapping's real folio ref. */
+ ret = mc16_test_dontneed(f->mm, f->addr, PAGE_SIZE);
+ KUNIT_EXPECT_EQ(f->test, ret, 0);
+ if (f->folio) {
+ folio_put(f->folio);
+ f->folio = NULL;
+ }
+ mc16_zl_counts(f, 0, 0);
+ mmput(f->mm);
+}
+
+static struct mc16_zl_mm *mc16_zl_begin(struct kunit *test)
+{
+ struct mc16_zl_mm *f;
+ struct vm_area_struct *vma;
+
+ f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+ f->test = test;
+#ifdef CONFIG_MEMCG
+ f->refs = mc16_test_refs_begin(test);
+ if (!f->refs)
+ return NULL;
+ if (mc16_test_refs_reserve(f->refs, 1) ||
+ mc16_test_refs_payload(f->refs, 0, MC16_STORED))
+ return NULL;
+#endif
+ f->addr = kunit_vm_mmap(test, NULL, 0, PAGE_SIZE,
+ PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, 0);
+ if (!f->addr || IS_ERR_VALUE(f->addr))
+ return NULL;
+ f->mm = current->mm;
+ mmget(f->mm);
+ mc16_zero_owners_read(&f->owners);
+ f->swapents = mc16_zl_rss(f->mm, MM_SWAPENTS);
+ f->anonpages = mc16_zl_rss(f->mm, MM_ANONPAGES);
+ mc16_zl_storage_read(f, &f->storage);
+#ifdef CONFIG_PER_VMA_LOCK
+ init_completion(&f->writer_locked);
+ init_completion(&f->writer_release);
+#endif
+ /* Registered last: PTE/folio cleanup precedes the unrelated slot puts. */
+ if (kunit_add_action_or_reset(test, mc16_zl_cleanup, f))
+ return NULL;
+ mmap_read_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (vma) {
+ f->flags = vma->vm_flags;
+ f->prot = vma->vm_page_prot;
+ f->saved_vma = true;
+ }
+ mmap_read_unlock(f->mm);
+ /* Leave anon_vma unprepared for the genuine VMA-lock retry actor. */
+ return vma ? f : NULL;
+}
+
+static int mc16_zl_install(struct mc16_zl_mm *f, pte_t pte)
+{
+ spinlock_t *ptl; /* Publishes and accounts one real PTE atomically. */
+ pte_t *ptep;
+ int ret = 0;
+
+ mmap_read_lock(f->mm);
+ ptep = get_locked_pte(f->mm, f->addr, &ptl);
+ if (!ptep) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ __acquire(RCU);
+ if (!pte_none(ptep_get(ptep))) {
+ ret = -EEXIST;
+ } else {
+ set_pte_at(f->mm, f->addr, ptep, pte);
+ add_mm_counter(f->mm, MM_SWAPENTS, 1);
+ mc16_zero_pte_init(1);
+ }
+ pte_unmap_unlock(ptep, ptl);
+out:
+ mmap_read_unlock(f->mm);
+ return ret;
+}
+
+static pte_t mc16_zl_read(struct mc16_zl_mm *f)
+{
+ spinlock_t *ptl; /* Pins a sampled present page while it is still mapped. */
+ pte_t *ptep, pte = __pte(0);
+ pmd_t *pmd;
+
+ mmap_read_lock(f->mm);
+ pmd = mm_find_pmd(f->mm, f->addr);
+ if (!pmd)
+ goto out;
+ ptep = pte_offset_map_lock(f->mm, pmd, f->addr, &ptl);
+ if (!ptep)
+ goto out;
+ pte = ptep_get(ptep);
+ if (pte_present(pte) && !pte_special(pte) && !f->folio) {
+ f->folio = page_folio(pte_page(pte));
+ folio_get(f->folio);
+ }
+ pte_unmap_unlock(ptep, ptl);
+out:
+ mmap_read_unlock(f->mm);
+ return pte;
+}
+
+static vm_fault_t mc16_zl_fault(struct mc16_zl_mm *f, pte_t orig,
+ unsigned int flags)
+{
+ struct vm_area_struct *vma;
+ vm_fault_t ret;
+ bool locked = true;
+
+ mmap_read_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (!vma) {
+ mmap_read_unlock(f->mm);
+ KUNIT_FAIL(f->test, "zero fault lost VMA");
+ return VM_FAULT_SIGBUS;
+ }
+ {
+ struct vm_fault vmf = {
+ .vma = vma,
+ .address = f->addr,
+ .real_address = f->addr,
+ .flags = flags | FAULT_FLAG_ORIG_PTE_VALID,
+ .pmd = mm_find_pmd(f->mm, f->addr),
+ .orig_pte = orig,
+ .gfp_mask = GFP_HIGHUSER_MOVABLE,
+ };
+
+ ret = mc16_load_zero(&vmf, &locked);
+ KUNIT_EXPECT_PTR_EQ(f->test, vmf.pte, NULL);
+ }
+ mmap_read_unlock(f->mm);
+ KUNIT_EXPECT_FALSE(f->test, locked);
+ return ret;
+}
+
+static void mc16_zl_present(struct mc16_zl_mm *f, pte_t pte)
+{
+ struct kunit *test = f->test;
+ struct folio *folio = f->folio;
+ void *data;
+
+ KUNIT_ASSERT_TRUE(test, pte_present(pte));
+ KUNIT_ASSERT_NOT_NULL(test, folio);
+ KUNIT_ASSERT_PTR_EQ(test, pte_page(pte), &folio->page);
+ KUNIT_EXPECT_TRUE(test, folio_test_anon(folio));
+ KUNIT_EXPECT_TRUE(test, folio_test_swapbacked(folio));
+ KUNIT_EXPECT_TRUE(test, folio_test_uptodate(folio));
+ KUNIT_EXPECT_FALSE(test, folio_test_locked(folio));
+ KUNIT_EXPECT_TRUE(test, PageAnonExclusive(&folio->page));
+ KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 1);
+ data = kmap_local_folio(folio, 0);
+ KUNIT_EXPECT_PTR_EQ(test, memchr_inv(data, 0, PAGE_SIZE), NULL);
+ kunmap_local(data);
+ mc16_zl_counts(f, 0, 1);
+}
+
+static void memcomp_test_zero_load_write(struct kunit *test)
+{
+ struct mc16_zl_mm *f = mc16_zl_begin(test);
+ pte_t orig = pte_swp_mksoft_dirty(mc16_zl_zero_pte());
+ pte_t pte;
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_zl_install(f, orig), 0);
+ mc16_zl_counts(f, 1, 0);
+ ret = mc16_zl_fault(f, orig, FAULT_FLAG_WRITE);
+ KUNIT_ASSERT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_NOPAGE);
+ pte = mc16_zl_read(f);
+ mc16_zl_present(f, pte);
+ KUNIT_EXPECT_TRUE(test, pte_write(pte));
+ KUNIT_EXPECT_TRUE(test, pte_dirty(pte));
+ /* A stale second fault must not consume the same logical owner twice. */
+ ret = mc16_zl_fault(f, orig, FAULT_FLAG_WRITE);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret, 0U);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_zl_read(f), pte));
+ mc16_zl_counts(f, 0, 1);
+}
+
+static void memcomp_test_zero_load_softdirty(struct kunit *test)
+{
+ struct mc16_zl_mm *f;
+ pte_t orig = pte_swp_mksoft_dirty(mc16_zl_zero_pte());
+ pte_t pte;
+ vm_fault_t ret;
+
+ if (!pte_swp_soft_dirty(orig)) {
+ kunit_skip(test, "architecture has no swap soft-dirty bit");
+ return;
+ }
+ f = mc16_zl_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_zl_install(f, orig), 0);
+ ret = mc16_zl_fault(f, orig, 0);
+ KUNIT_ASSERT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_NOPAGE);
+ pte = mc16_zl_read(f);
+ mc16_zl_present(f, pte);
+ KUNIT_EXPECT_TRUE(test, pte_soft_dirty(pte));
+ /* Writable read PTEs can encode hardware dirty on arm64. */
+}
+
+/* Change only VMA policy; no replacement page or fake charge backend. */
+static void mc16_zl_policy(struct mc16_zl_mm *f, vm_flags_t set,
+ vm_flags_t clear)
+{
+ struct vm_area_struct *vma;
+
+ mmap_write_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (vma) {
+ vma_start_write(vma);
+ vm_flags_reset(vma, (vma->vm_flags | set) & ~clear);
+ vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+ }
+ mmap_write_unlock(f->mm);
+ KUNIT_EXPECT_NOT_NULL(f->test, vma);
+}
+
+static void memcomp_test_zero_load_softdirty_wp(struct kunit *test)
+{
+ struct mc16_zl_mm *f;
+ pte_t orig = mc16_zl_zero_pte();
+ pte_t pte;
+ vm_fault_t ret;
+
+ if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY)) {
+ kunit_skip(test, "soft-dirty tracking is disabled");
+ return;
+ }
+ f = mc16_zl_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_zl_policy(f, 0, VM_SOFTDIRTY);
+ KUNIT_ASSERT_EQ(test, mc16_zl_install(f, orig), 0);
+ ret = mc16_zl_fault(f, orig, 0);
+ KUNIT_ASSERT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_NOPAGE);
+ pte = mc16_zl_read(f);
+ mc16_zl_present(f, pte);
+ KUNIT_EXPECT_FALSE(test, pte_write(pte));
+ KUNIT_EXPECT_FALSE(test, pte_soft_dirty(pte));
+ KUNIT_EXPECT_FALSE(test, pte_dirty(pte));
+}
+
+static void memcomp_test_zero_load_uffd_wp(struct kunit *test)
+{
+ struct mc16_zl_mm *f;
+ pte_t orig = pte_swp_mkuffd_wp(mc16_zl_zero_pte());
+ pte_t pte;
+ vm_fault_t ret;
+
+ if (!IS_ENABLED(CONFIG_USERFAULTFD) || !pte_swp_uffd_wp(orig)) {
+ kunit_skip(test, "architecture has no enabled swap UFFD-WP bit");
+ return;
+ }
+ f = mc16_zl_begin(test);
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ /* Read fault needs the WP policy bit, not a registered userfaultfd. */
+ mc16_zl_policy(f, VM_UFFD_WP, 0);
+ KUNIT_ASSERT_EQ(test, mc16_zl_install(f, orig), 0);
+ ret = mc16_zl_fault(f, orig, 0);
+ KUNIT_ASSERT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_NOPAGE);
+ pte = mc16_zl_read(f);
+ mc16_zl_present(f, pte);
+ KUNIT_EXPECT_TRUE(test, pte_uffd_wp(pte));
+ KUNIT_EXPECT_FALSE(test, pte_write(pte));
+ KUNIT_EXPECT_FALSE(test, pte_dirty(pte));
+}
+
+static void memcomp_test_zero_load_readonly(struct kunit *test)
+{
+ struct mc16_zl_mm *f = mc16_zl_begin(test);
+ pte_t orig = pte_swp_mksoft_dirty(mc16_zl_zero_pte());
+ pte_t pte;
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ mc16_zl_policy(f, 0, VM_WRITE);
+ KUNIT_ASSERT_EQ(test, mc16_zl_install(f, orig), 0);
+ ret = mc16_zl_fault(f, orig, 0);
+ KUNIT_ASSERT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_NOPAGE);
+ pte = mc16_zl_read(f);
+ mc16_zl_present(f, pte);
+ KUNIT_EXPECT_FALSE(test, pte_write(pte));
+ KUNIT_EXPECT_FALSE(test, pte_dirty(pte));
+}
+
+static void memcomp_test_zero_load_changed(struct kunit *test)
+{
+ struct mc16_zl_mm *f = mc16_zl_begin(test);
+ pte_t orig = mc16_zl_zero_pte();
+ pte_t changed = pte_swp_mkexclusive(orig);
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_FALSE(test, pte_same(orig, changed));
+ /* Actor commits a newer complete PTE before the stale fault takes PTL. */
+ KUNIT_ASSERT_EQ(test, mc16_zl_install(f, changed), 0);
+ mc16_zl_counts(f, 1, 0);
+ ret = mc16_zl_fault(f, orig, 0);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret, 0U);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_zl_read(f), changed));
+ mc16_zl_counts(f, 1, 0);
+}
+
+static void memcomp_test_zero_load_zapped(struct kunit *test)
+{
+ struct mc16_zl_mm *f = mc16_zl_begin(test);
+ pte_t orig = mc16_zl_zero_pte();
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_zl_install(f, orig), 0);
+ mc16_zl_zap_zero(f);
+ mc16_zl_counts(f, 0, 0);
+ ret = mc16_zl_fault(f, orig, 0);
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret, 0U);
+ KUNIT_EXPECT_TRUE(test, pte_none(mc16_zl_read(f)));
+ mc16_zl_counts(f, 0, 0);
+}
+
+static void mc16_zl_oom(struct kunit *test, bool charge)
+{
+ struct mc16_zl_mm *f = mc16_zl_begin(test);
+ pte_t orig = mc16_zl_zero_pte();
+ vm_fault_t ret;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_zl_install(f, orig), 0);
+ mc16_zero_fault_probe.fail_alloc = !charge;
+ mc16_zero_fault_probe.fail_charge = charge;
+ WRITE_ONCE(mc16_zero_fault_probe.task, current);
+ ret = mc16_zl_fault(f, orig, 0);
+ WRITE_ONCE(mc16_zero_fault_probe.task, NULL);
+ mc16_zero_fault_probe.fail_alloc = false;
+ mc16_zero_fault_probe.fail_charge = false;
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_OOM);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_zl_read(f), orig));
+ mc16_zl_counts(f, 1, 0);
+ /* Normal return deliberately leaves this owner to registered cleanup. */
+}
+
+static void memcomp_test_zero_load_alloc_oom(struct kunit *test)
+{
+ mc16_zl_oom(test, false);
+}
+
+static void memcomp_test_zero_load_charge_oom(struct kunit *test)
+{
+ /* Injection is after real allocation and before the real charge call. */
+ mc16_zl_oom(test, true);
+}
+
+#ifdef CONFIG_PER_VMA_LOCK
+static int mc16_zl_mmap_writer(void *data)
+{
+ struct mc16_zl_mm *f = data;
+
+ /* Deliberately no VMA write lock: it is held for reading by the actor. */
+ mmap_write_lock(f->mm);
+ complete(&f->writer_locked);
+ wait_for_completion(&f->writer_release);
+ mmap_write_unlock(f->mm);
+ return 0;
+}
+#endif
+
+static void memcomp_test_zero_load_retry(struct kunit *test)
+{
+#ifdef CONFIG_PER_VMA_LOCK
+ struct mc16_zl_mm *f = mc16_zl_begin(test);
+ struct vm_area_struct *vma;
+ pmd_t *pmd = NULL;
+ pte_t *pte_after;
+ pte_t orig = mc16_zl_zero_pte();
+ unsigned long ready;
+ vm_fault_t ret;
+ bool locked = true, held = false;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_zl_install(f, orig), 0);
+ mmap_read_lock(f->mm);
+ vma = vma_lookup(f->mm, f->addr);
+ if (vma && !vma->anon_vma)
+ held = vma_start_read(vma);
+ if (held)
+ pmd = mm_find_pmd(f->mm, f->addr);
+ mmap_read_unlock(f->mm);
+ if (!held) {
+ kunit_skip(test, "fresh anon VMA read lock unavailable");
+ return;
+ }
+ /* No fatal assertion or KUnit resource allocation while VMA is locked. */
+ f->writer = kthread_run(mc16_zl_mmap_writer, f, "mc16-zero-retry");
+ if (IS_ERR(f->writer)) {
+ f->writer = NULL;
+ vma_end_read(vma);
+ KUNIT_FAIL(test, "could not start mmap writer");
+ return;
+ }
+ /* The worker may exit as soon as writer_release is completed. */
+ get_task_struct(f->writer);
+ ready = wait_for_completion_timeout(&f->writer_locked,
+ msecs_to_jiffies(5000));
+ if (!ready) {
+ vma_end_read(vma);
+ complete_all(&f->writer_release);
+ kthread_stop_put(f->writer);
+ f->writer = NULL;
+ KUNIT_FAIL(test, "mmap writer did not acquire its lock");
+ return;
+ }
+ {
+ struct vm_fault vmf = {
+ .vma = vma,
+ .address = f->addr,
+ .real_address = f->addr,
+ .flags = FAULT_FLAG_VMA_LOCK | FAULT_FLAG_ORIG_PTE_VALID,
+ .pmd = pmd,
+ .orig_pte = orig,
+ .gfp_mask = GFP_HIGHUSER_MOVABLE,
+ };
+
+ ret = mc16_load_zero(&vmf, &locked);
+ pte_after = vmf.pte;
+ }
+ /* vmf_anon_prepare() drops the VMA read lock only on RETRY. */
+ if (!(ret & VM_FAULT_RETRY))
+ vma_end_read(vma);
+ complete_all(&f->writer_release);
+ kthread_stop_put(f->writer);
+ f->writer = NULL;
+ KUNIT_EXPECT_EQ(test, (__force unsigned int)ret,
+ (__force unsigned int)VM_FAULT_RETRY);
+ KUNIT_EXPECT_FALSE(test, locked);
+ KUNIT_EXPECT_PTR_EQ(test, pte_after, NULL);
+ KUNIT_EXPECT_TRUE(test, pte_same(mc16_zl_read(f), orig));
+ mc16_zl_counts(f, 1, 0);
+#else
+ kunit_skip(test, "per-VMA locking is disabled");
+#endif
+}
+#endif /* MC16_ZERO_LOAD_MM_TEST_ENABLED */
+#endif /* _MM_MEMCOMP_TESTS_ZERO_LOAD_TEST_H */
diff --git a/mm/memcomp/tests/zero_test.h b/mm/memcomp/tests/zero_test.h
new file mode 100644
index 0000000000000000000000000000000000000000..845d6ac29c3f650b19c9d3f13bafe6100964eed6
--- /dev/null
+++ b/mm/memcomp/tests/zero_test.h
@@ -0,0 +1,313 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _MM_MEMCOMP_TESTS_ZERO_TEST_H
+#define _MM_MEMCOMP_TESTS_ZERO_TEST_H
+
+#include <kunit/test.h>
+#include <linux/cpu.h>
+#include <linux/smp.h>
+#include <linux/slab.h>
+
+#include "../zero.h"
+
+/*
+ * Registration belongs to the enclosing memcomp KUnit suite. The MEMCG case
+ * needs the existing storage/refs fixtures included first. No public wrapper,
+ * fault, fork or rmap wiring is supplied or validated by these helper tests.
+ */
+static void memcomp_test_zero_page(struct kunit *test)
+{
+ unsigned long pattern = 0x9876543210fedcbaUL;
+ u8 *allocation, *page;
+ unsigned int i;
+ static const unsigned int positions[] = {
+ 0, 1, sizeof(unsigned long) - 1, sizeof(unsigned long),
+ PAGE_SIZE / 2, PAGE_SIZE - 2, PAGE_SIZE - 1,
+ };
+
+ allocation = kunit_kzalloc(test, PAGE_SIZE + 2, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, allocation);
+ page = allocation + 1; /* Deliberately unaligned; preserve both guards. */
+ allocation[0] = 0xa5;
+ page[PAGE_SIZE] = 0x5a;
+ KUNIT_EXPECT_TRUE(test, mc16_zero_is_page(page, PAGE_SIZE));
+ KUNIT_EXPECT_FALSE(test, mc16_zero_is_page(NULL, PAGE_SIZE));
+ KUNIT_EXPECT_FALSE(test, mc16_zero_is_page(page, 0));
+ KUNIT_EXPECT_FALSE(test, mc16_zero_is_page(page, PAGE_SIZE - 1));
+ KUNIT_EXPECT_FALSE(test, mc16_zero_is_page(page, PAGE_SIZE + 1));
+ KUNIT_EXPECT_FALSE(test, mc16_zero_is_page(page, SIZE_MAX));
+ for (i = 0; i < ARRAY_SIZE(positions); i++) {
+ page[positions[i]] = 1;
+ KUNIT_EXPECT_FALSE(test, mc16_zero_is_page(page, PAGE_SIZE));
+ page[positions[i]] = 0;
+ }
+ /* A full repeated nonzero word is SAMEFILL, never slotless zero. */
+ for (i = 0; i < PAGE_SIZE; i += sizeof(pattern))
+ memcpy(page + i, &pattern, sizeof(pattern));
+ KUNIT_EXPECT_FALSE(test, mc16_zero_is_page(page, PAGE_SIZE));
+ memset(page, 0xff, PAGE_SIZE);
+ KUNIT_EXPECT_FALSE(test, mc16_zero_is_page(page, PAGE_SIZE));
+ memset(page, 0, PAGE_SIZE);
+ KUNIT_EXPECT_TRUE(test, mc16_zero_is_page(page, PAGE_SIZE));
+ KUNIT_EXPECT_EQ(test, allocation[0], (u8)0xa5);
+ KUNIT_EXPECT_EQ(test, page[PAGE_SIZE], (u8)0x5a);
+}
+
+static void memcomp_test_zero_owners(struct kunit *test)
+{
+ struct mc16_zero_owners before, after;
+ struct mc16_zero_diagnostics events, end;
+ u64 pages = mc16_zero_pages();
+
+ mc16_zero_owners_read(&before);
+ mc16_zero_diagnostics_read(&events);
+ KUNIT_EXPECT_TRUE(test, mc16_zero_pte_init(3));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_pte_get(2)); /* forked PTEs */
+ KUNIT_EXPECT_TRUE(test, mc16_zero_cache_get(4)); /* xa owners */
+ mc16_zero_owners_read(&after);
+ KUNIT_EXPECT_EQ(test, after.pte_pages, before.pte_pages + 5);
+ KUNIT_EXPECT_EQ(test, after.cache_pages, before.cache_pages + 4);
+ KUNIT_EXPECT_EQ(test, mc16_zero_pages(), pages + 9);
+ KUNIT_EXPECT_TRUE(test, mc16_zero_pte_put(2));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_cache_put(4));
+ KUNIT_EXPECT_EQ(test, mc16_zero_pages(), pages + 3);
+ KUNIT_EXPECT_TRUE(test, mc16_zero_pte_put(3));
+ mc16_zero_owners_read(&after);
+ KUNIT_EXPECT_EQ(test, after.pte_pages, before.pte_pages);
+ KUNIT_EXPECT_EQ(test, after.cache_pages, before.cache_pages);
+ mc16_zero_diagnostics_read(&end);
+#ifdef CONFIG_MEMCOMP_DEBUG_STATS
+ KUNIT_EXPECT_EQ(test, end.calls[MC16_ZERO_PTE_INIT] -
+ events.calls[MC16_ZERO_PTE_INIT], 1ULL);
+ KUNIT_EXPECT_EQ(test, end.pages[MC16_ZERO_PTE_INIT] -
+ events.pages[MC16_ZERO_PTE_INIT], 3ULL);
+ KUNIT_EXPECT_EQ(test, end.calls[MC16_ZERO_PTE_GET] -
+ events.calls[MC16_ZERO_PTE_GET], 1ULL);
+ KUNIT_EXPECT_EQ(test, end.pages[MC16_ZERO_PTE_GET] -
+ events.pages[MC16_ZERO_PTE_GET], 2ULL);
+ KUNIT_EXPECT_EQ(test, end.calls[MC16_ZERO_PTE_PUT] -
+ events.calls[MC16_ZERO_PTE_PUT], 2ULL);
+ KUNIT_EXPECT_EQ(test, end.pages[MC16_ZERO_PTE_PUT] -
+ events.pages[MC16_ZERO_PTE_PUT], 5ULL);
+ KUNIT_EXPECT_EQ(test, end.pages[MC16_ZERO_CACHE_GET] -
+ events.pages[MC16_ZERO_CACHE_GET], 4ULL);
+ KUNIT_EXPECT_EQ(test, end.pages[MC16_ZERO_CACHE_PUT] -
+ events.pages[MC16_ZERO_CACHE_PUT], 4ULL);
+#else
+ KUNIT_EXPECT_EQ(test, memcmp(&events, &end, sizeof(end)), 0);
+#endif
+}
+
+static void memcomp_test_zero_batch_bounds(struct kunit *test)
+{
+ static const unsigned long sizes[] = {
+ 1, 2, 511, 512, 513, UINT_MAX,
+#if BITS_PER_LONG > 32
+ 1UL << 32,
+#endif
+ };
+ struct mc16_zero_owners before, after;
+ struct mc16_zero_diagnostics events, end;
+ unsigned long invalid = (unsigned long)LONG_MAX + 1;
+ unsigned int i;
+
+ mc16_zero_owners_read(&before);
+ mc16_zero_diagnostics_read(&events);
+ KUNIT_EXPECT_TRUE(test, mc16_zero_nr_valid(LONG_MAX));
+ KUNIT_EXPECT_FALSE(test, mc16_zero_nr_valid(invalid));
+ KUNIT_EXPECT_FALSE(test, mc16_zero_nr_valid(ULONG_MAX));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_pte_init(0));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_pte_get(0));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_pte_put(0));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_cache_get(0));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_cache_put(0));
+ KUNIT_EXPECT_FALSE(test, mc16_zero_pte_init(invalid));
+ KUNIT_EXPECT_FALSE(test, mc16_zero_pte_get(invalid));
+ KUNIT_EXPECT_FALSE(test, mc16_zero_pte_put(invalid));
+ KUNIT_EXPECT_FALSE(test, mc16_zero_cache_get(invalid));
+ KUNIT_EXPECT_FALSE(test, mc16_zero_cache_put(invalid));
+ mc16_zero_diagnostics_read(&end);
+ KUNIT_EXPECT_EQ(test, memcmp(&events, &end, sizeof(end)), 0);
+ for (i = 0; i < ARRAY_SIZE(sizes); i++) {
+ unsigned long nr = sizes[i];
+
+ if (!mc16_zero_nr_valid(nr))
+ continue;
+ KUNIT_EXPECT_TRUE(test, mc16_zero_pte_init(nr));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_cache_get(nr));
+ mc16_zero_owners_read(&after);
+ KUNIT_EXPECT_EQ(test, after.pte_pages, before.pte_pages + (s64)nr);
+ KUNIT_EXPECT_EQ(test, after.cache_pages, before.cache_pages + (s64)nr);
+ KUNIT_EXPECT_TRUE(test, mc16_zero_pte_put(nr));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_cache_put(nr));
+ }
+ mc16_zero_owners_read(&after);
+ KUNIT_EXPECT_EQ(test, after.pte_pages, before.pte_pages);
+ KUNIT_EXPECT_EQ(test, after.cache_pages, before.cache_pages);
+}
+
+static void memcomp_test_zero_stat(struct kunit *test)
+{
+ unsigned long compressed = PAGE_SIZE;
+ int mapcount = 42;
+ u64 pages = mc16_zero_pages();
+ u64 base = 7 * (u64)PAGE_SIZE;
+ s64 stored = atomic64_read(&memcomp_stored_bytes);
+ s64 payload = atomic64_read(&memcomp_compressed_bytes);
+
+ KUNIT_EXPECT_TRUE(test, mc16_zero_entry_stat(0, &compressed, &mapcount));
+ KUNIT_EXPECT_EQ(test, compressed, 0UL);
+ KUNIT_EXPECT_EQ(test, mapcount, 1);
+ KUNIT_EXPECT_TRUE(test, mc16_zero_entry_stat(0, NULL, NULL));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_entry_stat(0, &compressed, NULL));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_entry_stat(0, NULL, &mapcount));
+ compressed = 123;
+ mapcount = 45;
+ KUNIT_EXPECT_FALSE(test, mc16_zero_entry_stat(1, &compressed, &mapcount));
+ KUNIT_EXPECT_FALSE(test, mc16_zero_entry_stat(ULONG_MAX, NULL, NULL));
+#if BITS_PER_LONG > 32
+ KUNIT_EXPECT_FALSE(test, mc16_zero_entry_stat(1UL << 32, NULL, NULL));
+#endif
+ KUNIT_EXPECT_EQ(test, compressed, 123UL);
+ KUNIT_EXPECT_EQ(test, mapcount, 45);
+ KUNIT_EXPECT_TRUE(test, mc16_zero_pte_init(2));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_cache_get(3));
+ KUNIT_EXPECT_EQ(test, mc16_zero_stored_bytes(base),
+ base + ((pages + 5) << PAGE_SHIFT));
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_stored_bytes), stored);
+ KUNIT_EXPECT_EQ(test, atomic64_read(&memcomp_compressed_bytes), payload);
+ KUNIT_EXPECT_TRUE(test, mc16_zero_pte_put(2));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_cache_put(3));
+ KUNIT_EXPECT_EQ(test, mc16_zero_stored_bytes(base),
+ base + (pages << PAGE_SHIFT));
+}
+
+static void mc16_test_zero_cpu_get(void *unused)
+{
+ mc16_zero_pte_get(3);
+ mc16_zero_cache_get(5);
+}
+
+static void mc16_test_zero_cpu_put(void *unused)
+{
+ mc16_zero_pte_put(3);
+ mc16_zero_cache_put(5);
+}
+
+static void memcomp_test_zero_cross_cpu(struct kunit *test)
+{
+ struct mc16_zero_owners before, after;
+ s64 get_before, put_before, cache_get_before, cache_put_before;
+ s64 get_after, put_after, cache_get_after, cache_put_after;
+ u64 pages;
+ unsigned int get_cpu, put_cpu;
+ int ret, put_ret;
+
+ cpus_read_lock();
+ get_cpu = cpumask_first(cpu_online_mask);
+ put_cpu = cpumask_next(get_cpu, cpu_online_mask);
+ if (put_cpu >= nr_cpu_ids) {
+ cpus_read_unlock();
+ kunit_skip(test, "needs two online CPUs");
+ return;
+ }
+ mc16_zero_owners_read(&before);
+ pages = mc16_zero_pages();
+ get_before = READ_ONCE(per_cpu(mc16_zero_owners, get_cpu).pte_pages);
+ put_before = READ_ONCE(per_cpu(mc16_zero_owners, put_cpu).pte_pages);
+ cache_get_before = READ_ONCE(per_cpu(mc16_zero_owners, get_cpu).cache_pages);
+ cache_put_before = READ_ONCE(per_cpu(mc16_zero_owners, put_cpu).cache_pages);
+ ret = smp_call_function_single(get_cpu, mc16_test_zero_cpu_get, NULL, 1);
+ KUNIT_EXPECT_EQ(test, ret, 0);
+ if (!ret) {
+ KUNIT_EXPECT_EQ(test, mc16_zero_pages(), pages + 8);
+ put_ret = smp_call_function_single(put_cpu, mc16_test_zero_cpu_put,
+ NULL, 1);
+ KUNIT_EXPECT_EQ(test, put_ret, 0);
+ if (put_ret) {
+ mc16_zero_pte_put(3);
+ mc16_zero_cache_put(5);
+ } else {
+ get_after = READ_ONCE(per_cpu(mc16_zero_owners, get_cpu).pte_pages);
+ put_after = READ_ONCE(per_cpu(mc16_zero_owners, put_cpu).pte_pages);
+ cache_get_after = READ_ONCE(per_cpu(mc16_zero_owners, get_cpu).cache_pages);
+ cache_put_after = READ_ONCE(per_cpu(mc16_zero_owners, put_cpu).cache_pages);
+ KUNIT_EXPECT_EQ(test, get_after, get_before + 3);
+ KUNIT_EXPECT_EQ(test, put_after, put_before - 3);
+ KUNIT_EXPECT_EQ(test, cache_get_after, cache_get_before + 5);
+ KUNIT_EXPECT_EQ(test, cache_put_after, cache_put_before - 5);
+ }
+ }
+ KUNIT_EXPECT_EQ(test, mc16_zero_pages(), pages);
+ mc16_zero_owners_read(&after);
+ cpus_read_unlock();
+ KUNIT_EXPECT_EQ(test, after.pte_pages, before.pte_pages);
+ KUNIT_EXPECT_EQ(test, after.cache_pages, before.cache_pages);
+}
+
+#ifdef CONFIG_MEMCG
+static void memcomp_test_zero_no_storage(struct kunit *test)
+{
+ struct mc16_test_refs *f = mc16_test_refs_begin(test);
+ struct mc16_test_storage *storage;
+ struct mc16_view view;
+ unsigned int allocations, refs, frees;
+ int gets, puts, charges, uncharges;
+ long charge;
+ u8 *src;
+ unsigned long compressed = PAGE_SIZE;
+ int mapcount = 0;
+
+ KUNIT_ASSERT_NOT_NULL(test, f);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_reserve(f, 1), 0);
+ KUNIT_ASSERT_EQ(test, mc16_test_refs_payload(f, 0, MC16_STORED), 0);
+ src = kunit_kzalloc(test, PAGE_SIZE, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, src);
+ storage = f->storage;
+ /* Observe the actual unrelated live payload, not an unbound counter. */
+ mutex_lock(memcomp_lock_of(storage->refs[0].token));
+ mc16_view_locked(&storage->refs[0], &view);
+ storage->payload.handle = *view.word;
+ mutex_unlock(memcomp_lock_of(storage->refs[0].token));
+ storage->payload.task = current;
+ storage->payload.pool = memcomp_pools[0];
+ storage->payload.live = true;
+ rcu_assign_pointer(memcomp_kunit_async_probe, &storage->payload);
+ allocations = storage->probe.allocations;
+ refs = refcount_read(&storage->refs[0].slot->refs);
+ frees = storage->payload.frees;
+ gets = atomic_read(&storage->owner->model.get_calls);
+ puts = atomic_read(&storage->owner->model.put_calls);
+ charges = atomic_read(&storage->owner->model.charge_calls);
+ uncharges = atomic_read(&storage->owner->model.uncharge_calls);
+ charge = atomic_long_read(&storage->owner->model.charged);
+ /* Actual zero actors, with an unrelated positive slot remaining live. */
+ KUNIT_EXPECT_TRUE(test, mc16_zero_is_page(src, PAGE_SIZE));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_entry_stat(0, &compressed, &mapcount));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_pte_init(512));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_pte_get(2));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_pte_put(514));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_cache_get(1));
+ KUNIT_EXPECT_TRUE(test, mc16_zero_cache_put(1));
+ KUNIT_EXPECT_EQ(test, compressed, 0UL);
+ KUNIT_EXPECT_EQ(test, mapcount, 1);
+ KUNIT_EXPECT_EQ(test, storage->probe.allocations, allocations);
+ KUNIT_EXPECT_EQ(test, refcount_read(&storage->refs[0].slot->refs), refs);
+ KUNIT_EXPECT_EQ(test, storage->payload.frees, frees);
+ KUNIT_EXPECT_TRUE(test, storage->payload.live);
+ KUNIT_EXPECT_FALSE(test, storage->payload.bad_lifetime);
+ KUNIT_EXPECT_EQ(test, atomic_read(&storage->owner->model.get_calls), gets);
+ KUNIT_EXPECT_EQ(test, atomic_read(&storage->owner->model.put_calls), puts);
+ KUNIT_EXPECT_EQ(test, atomic_read(&storage->owner->model.charge_calls), charges);
+ KUNIT_EXPECT_EQ(test, atomic_read(&storage->owner->model.uncharge_calls),
+ uncharges);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&storage->owner->model.charged), charge);
+ /* The same probe must observe the subsequent real positive-slot free. */
+ mc16_ref_put(&storage->refs[0]);
+ f->published[0] = false;
+ KUNIT_EXPECT_EQ(test, storage->payload.frees, frees + 1);
+ KUNIT_EXPECT_FALSE(test, storage->payload.live);
+ KUNIT_EXPECT_FALSE(test, storage->payload.bad_lifetime);
+}
+#endif /* CONFIG_MEMCG */
+
+#endif /* _MM_MEMCOMP_TESTS_ZERO_TEST_H */
diff --git a/mm/memcomp/workingset_store.h b/mm/memcomp/workingset_store.h
index 32629bb1e9ea1793f756ec8856371bd474e182d8..7943f4d43d675a2bb223912d75a6b88ac400b1db 100644
--- a/mm/memcomp/workingset_store.h
+++ b/mm/memcomp/workingset_store.h
@@ -10,7 +10,17 @@
* readers and background actors, not token reference removal by PTE zap.
* Only a source OWNER in its first PENDING epoch can open this gate.
*/
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+#include <kunit/static_stub.h>
+static void *mc16_store_eviction(struct folio *folio,
+ struct mem_cgroup *target_memcg)
+{
+ KUNIT_STATIC_STUB_REDIRECT(mc16_store_eviction, folio, target_memcg);
+ return workingset_memcomp_eviction(folio, target_memcg);
+}
+#else
#define mc16_store_eviction workingset_memcomp_eviction
+#endif
static inline int
mc16_store_begin_locked(struct memcomp_ref *ref, struct folio *folio,
diff --git a/mm/memcomp/writeback_batch.h b/mm/memcomp/writeback_batch.h
index 9fa6dc6473d2ed5f6b8aac1b3caaf83cc16271ca..3fbc532a7b024561c89f5ec6e7763d49d092a0f9 100644
--- a/mm/memcomp/writeback_batch.h
+++ b/mm/memcomp/writeback_batch.h
@@ -3,6 +3,9 @@
#define _MM_MEMCOMP_WRITEBACK_BATCH_H
#ifdef CONFIG_SWAP
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+#include <kunit/static_stub.h>
+#endif
/*
* Include in memcomp.c after the old writeback batch implementation. Uses
@@ -52,17 +55,26 @@ static unsigned int mc16_writeback_open_bytes;
/* Test seams preserve the production allocator, submit and backing APIs. */
static int mc16_writeback_folio_alloc(struct folio **folio, swp_entry_t *swap)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_writeback_folio_alloc, folio, swap);
+#endif
return memcomp_swap_folio_alloc(folio, swap);
}
static void mc16_writeback_submit(struct folio *folio,
struct writeback_control *wbc)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_writeback_submit, folio, wbc);
+#endif
__swap_writepage(folio, wbc);
}
static int mc16_writeback_backing_dup(swp_entry_t swap)
{
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ KUNIT_STATIC_STUB_REDIRECT(mc16_writeback_backing_dup, swap);
+#endif
return memcomp_slot_dup(swap);
}
diff --git a/mm/memcomp/zero_load.h b/mm/memcomp/zero_load.h
index 41f7af3f3699147907b57dd1f4c6d70e88312a67..43be6b5b83554069896c67b0535dd9bee9541767 100644
--- a/mm/memcomp/zero_load.h
+++ b/mm/memcomp/zero_load.h
@@ -5,6 +5,14 @@
#include <linux/highmem.h>
/* Include after zero.h and memcomp_pte_map_lock(), in memcomp.c. */
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+/* Current-task failure injection only; successful data always uses real MM. */
+static struct {
+ struct task_struct *task;
+ bool fail_alloc;
+ bool fail_charge;
+} mc16_zero_fault_probe;
+#endif
static inline vm_fault_t mc16_load_zero(struct vm_fault *vmf, bool *locked)
{
@@ -18,9 +26,21 @@ static inline vm_fault_t mc16_load_zero(struct vm_fault *vmf, bool *locked)
ret = vmf_anon_prepare(vmf);
if (ret)
return ret;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ if (READ_ONCE(mc16_zero_fault_probe.task) == current &&
+ mc16_zero_fault_probe.fail_alloc)
+ return VM_FAULT_OOM;
+#endif
folio = vma_alloc_zeroed_movable_folio(vma, vmf->address);
if (!folio)
return VM_FAULT_OOM;
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+ if (READ_ONCE(mc16_zero_fault_probe.task) == current &&
+ mc16_zero_fault_probe.fail_charge) {
+ folio_put(folio);
+ return VM_FAULT_OOM;
+ }
+#endif
if (mem_cgroup_charge(folio, vma->vm_mm, GFP_HIGHUSER_MOVABLE)) {
folio_put(folio);
return VM_FAULT_OOM;
diff --git a/mm/mincore.c b/mm/mincore.c
index 17e8d7a5b316edcfb13d72242767eea662a559dd..dddf42bd4f8e0ac6cf9fae801b47e9ae520884fe 100644
--- a/mm/mincore.c
+++ b/mm/mincore.c
@@ -281,6 +281,27 @@ static const struct mm_walk_ops mincore_walk_ops = {
.walk_lock = PGWALK_RDLOCK,
};
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+#define MC16_MINCORE_DECLARE_TEST_API
+#include "memcomp/tests/mincore_test.h"
+#undef MC16_MINCORE_DECLARE_TEST_API
+#endif
+
+#ifdef CONFIG_MEMCOMP_KUNIT_TEST
+/* Test-only entry points retain the real cache lookup and PTE walk. */
+unsigned char memcomp_test_mincore_page(struct address_space *mapping,
+ pgoff_t index)
+{
+ return mincore_page(mapping, index);
+}
+
+int memcomp_test_mincore_walk(struct mm_struct *mm, unsigned long start,
+ unsigned long end, unsigned char *vec)
+{
+ return walk_page_range(mm, start, end, &mincore_walk_ops, vec);
+}
+#endif
+
/*
* Do a chunk of "sys_mincore()". We've already checked
* all the arguments, we hold the mmap semaphore: we should
^ permalink raw reply related [flat|nested] 19+ messages in thread