* [PATCH v7 03/12] ima: Introduce per binary measurements list type ima_num_records counter
From: Roberto Sassu @ 2026-06-05 17:22 UTC (permalink / raw)
To: corbet, skhan, zohar, dmitry.kasatkin, eric.snowberg, paul,
jmorris, serge
Cc: linux-doc, linux-kernel, linux-integrity, linux-security-module,
gregorylumen, chenste, nramas, Roberto Sassu
In-Reply-To: <20260605172236.2042045-1-roberto.sassu@huaweicloud.com>
From: Roberto Sassu <roberto.sassu@huawei.com>
Make ima_num_records as an array, to have separate counters per binary
measurements list type. Currently, define the BINARY type for the existing
binary measurements list.
No functional change: the BINARY type is equivalent to the value without
the array.
Link: https://github.com/linux-integrity/linux/issues/1
Signed-off-by: Roberto Sassu <roberto.sassu@huawei.com>
Reviewed-by: Mimi Zohar <zohar@linux.ibm.com>
---
security/integrity/ima/ima.h | 9 ++++++++-
security/integrity/ima/ima_fs.c | 2 +-
security/integrity/ima/ima_kexec.c | 2 +-
security/integrity/ima/ima_queue.c | 6 ++++--
4 files changed, 14 insertions(+), 5 deletions(-)
diff --git a/security/integrity/ima/ima.h b/security/integrity/ima/ima.h
index 0e41c2113efd..8f457f2c7b79 100644
--- a/security/integrity/ima/ima.h
+++ b/security/integrity/ima/ima.h
@@ -28,6 +28,13 @@ enum ima_show_type { IMA_SHOW_BINARY, IMA_SHOW_BINARY_NO_FIELD_LEN,
IMA_SHOW_BINARY_OLD_STRING_FMT, IMA_SHOW_ASCII };
enum tpm_pcrs { TPM_PCR0 = 0, TPM_PCR8 = 8, TPM_PCR10 = 10 };
+/*
+ * BINARY: current binary measurements list
+ */
+enum binary_lists {
+ BINARY, BINARY__LAST
+};
+
/* digest size for IMA, fits SHA1 or MD5 */
#define IMA_DIGEST_SIZE SHA1_DIGEST_SIZE
#define IMA_EVENT_NAME_LEN_MAX 255
@@ -326,7 +333,7 @@ int ima_lsm_policy_change(struct notifier_block *nb, unsigned long event,
extern spinlock_t ima_queue_lock;
/* Total number of measurement list records since hard boot. */
-extern atomic_long_t ima_num_records;
+extern atomic_long_t ima_num_records[BINARY__LAST];
/* Total number of violations since hard boot. */
extern atomic_long_t ima_num_violations;
extern struct hlist_head __rcu *ima_htable;
diff --git a/security/integrity/ima/ima_fs.c b/security/integrity/ima/ima_fs.c
index 523d3e81f631..fcfcf7b6eae2 100644
--- a/security/integrity/ima/ima_fs.c
+++ b/security/integrity/ima/ima_fs.c
@@ -63,7 +63,7 @@ static ssize_t ima_show_measurements_count(struct file *filp,
char __user *buf,
size_t count, loff_t *ppos)
{
- return ima_show_counter(buf, count, ppos, &ima_num_records);
+ return ima_show_counter(buf, count, ppos, &ima_num_records[BINARY]);
}
static const struct file_operations ima_measurements_count_ops = {
diff --git a/security/integrity/ima/ima_kexec.c b/security/integrity/ima/ima_kexec.c
index 77ad370dbc37..1a0211a12ea4 100644
--- a/security/integrity/ima/ima_kexec.c
+++ b/security/integrity/ima/ima_kexec.c
@@ -43,7 +43,7 @@ void ima_measure_kexec_event(const char *event_name)
int n;
buf_size = ima_get_binary_runtime_size();
- len = atomic_long_read(&ima_num_records);
+ len = atomic_long_read(&ima_num_records[BINARY]);
n = scnprintf(ima_kexec_event, IMA_KEXEC_EVENT_LEN,
"kexec_segment_size=%lu;ima_binary_runtime_size=%lu;"
diff --git a/security/integrity/ima/ima_queue.c b/security/integrity/ima/ima_queue.c
index a31b75d9302b..012e725ed4fc 100644
--- a/security/integrity/ima/ima_queue.c
+++ b/security/integrity/ima/ima_queue.c
@@ -32,7 +32,9 @@ static unsigned long binary_runtime_size;
static unsigned long binary_runtime_size = ULONG_MAX;
#endif
-atomic_long_t ima_num_records = ATOMIC_LONG_INIT(0);
+atomic_long_t ima_num_records[BINARY__LAST] = {
+ [0 ... BINARY__LAST - 1] = ATOMIC_LONG_INIT(0)
+};
atomic_long_t ima_num_violations = ATOMIC_LONG_INIT(0);
/* key: inode (before secure-hashing a file) */
@@ -152,7 +154,7 @@ static int ima_add_digest_entry(struct ima_template_entry *entry,
htable = rcu_dereference_protected(ima_htable,
lockdep_is_held(&ima_extend_list_mutex));
- atomic_long_inc(&ima_num_records);
+ atomic_long_inc(&ima_num_records[BINARY]);
if (update_htable) {
key = ima_hash_key(entry->digests[ima_hash_algo_idx].digest);
hlist_add_head_rcu(&qe->hnext, &htable[key]);
--
2.43.0
^ permalink raw reply related
* [PATCH v7 02/12] ima: Replace static htable queue with dynamically allocated array
From: Roberto Sassu @ 2026-06-05 17:22 UTC (permalink / raw)
To: corbet, skhan, zohar, dmitry.kasatkin, eric.snowberg, paul,
jmorris, serge
Cc: linux-doc, linux-kernel, linux-integrity, linux-security-module,
gregorylumen, chenste, nramas, Roberto Sassu
In-Reply-To: <20260605172236.2042045-1-roberto.sassu@huaweicloud.com>
From: Roberto Sassu <roberto.sassu@huawei.com>
The IMA hash table is a fixed-size array of hlist_head buckets:
struct hlist_head ima_htable[IMA_MEASURE_HTABLE_SIZE];
IMA_MEASURE_HTABLE_SIZE is (1 << IMA_HASH_BITS) = 1024 buckets, each a
struct hlist_head (one pointer, 8 bytes on 64-bit). That is 8 KiB allocated
in BSS for every kernel, regardless of whether IMA is ever used, and
regardless of how many measurements are actually made.
Replace the fixed-size array with a RCU-protected pointer to a dynamically
allocated array that is initialized in ima_init_htable(), which is called
from ima_init() during early boot. ima_init_htable() calls the static
function ima_alloc_replace_htable() which, other than initializing the hash
table the first time, can also hot-swap the existing hash table with a
blank one.
The allocation in ima_alloc_replace_htable() uses kcalloc() so the buckets
are zero-initialised (equivalent to HLIST_HEAD_INIT { .first = NULL }).
Callers of ima_alloc_replace_htable() must call synchronize_rcu() and free
the returned hash table.
Finally, access the hash table with rcu_dereference() in
ima_lookup_digest_entry() (reader side) and with
rcu_dereference_protected() in ima_add_digest_entry() (writer side).
No functional change: bucket count, hash function, and all locking remain
identical.
Link: https://github.com/linux-integrity/linux/issues/1
Signed-off-by: Roberto Sassu <roberto.sassu@huawei.com>
Reviewed-by: Mimi Zohar <zohar@linux.ibm.com>
---
security/integrity/ima/ima.h | 3 +-
security/integrity/ima/ima_init.c | 5 ++++
security/integrity/ima/ima_queue.c | 48 ++++++++++++++++++++++++++----
3 files changed, 50 insertions(+), 6 deletions(-)
diff --git a/security/integrity/ima/ima.h b/security/integrity/ima/ima.h
index b3ad7eac6a1e..0e41c2113efd 100644
--- a/security/integrity/ima/ima.h
+++ b/security/integrity/ima/ima.h
@@ -311,6 +311,7 @@ bool ima_template_has_modsig(const struct ima_template_desc *ima_template);
int ima_restore_measurement_entry(struct ima_template_entry *entry);
int ima_restore_measurement_list(loff_t bufsize, void *buf);
int ima_measurements_show(struct seq_file *m, void *v);
+int __init ima_init_htable(void);
unsigned long ima_get_binary_runtime_size(void);
int ima_init_template(void);
void ima_init_template_list(void);
@@ -328,7 +329,7 @@ extern spinlock_t ima_queue_lock;
extern atomic_long_t ima_num_records;
/* Total number of violations since hard boot. */
extern atomic_long_t ima_num_violations;
-extern struct hlist_head ima_htable[IMA_MEASURE_HTABLE_SIZE];
+extern struct hlist_head __rcu *ima_htable;
static inline unsigned int ima_hash_key(u8 *digest)
{
diff --git a/security/integrity/ima/ima_init.c b/security/integrity/ima/ima_init.c
index a2f34f2d8ad7..7e0aa09a12e6 100644
--- a/security/integrity/ima/ima_init.c
+++ b/security/integrity/ima/ima_init.c
@@ -140,6 +140,11 @@ int __init ima_init(void)
rc = ima_init_digests();
if (rc != 0)
return rc;
+
+ rc = ima_init_htable();
+ if (rc != 0)
+ return rc;
+
rc = ima_add_boot_aggregate(); /* boot aggregate must be first entry */
if (rc != 0)
return rc;
diff --git a/security/integrity/ima/ima_queue.c b/security/integrity/ima/ima_queue.c
index 6bdaefc790c3..a31b75d9302b 100644
--- a/security/integrity/ima/ima_queue.c
+++ b/security/integrity/ima/ima_queue.c
@@ -36,9 +36,7 @@ atomic_long_t ima_num_records = ATOMIC_LONG_INIT(0);
atomic_long_t ima_num_violations = ATOMIC_LONG_INIT(0);
/* key: inode (before secure-hashing a file) */
-struct hlist_head ima_htable[IMA_MEASURE_HTABLE_SIZE] = {
- [0 ... IMA_MEASURE_HTABLE_SIZE - 1] = HLIST_HEAD_INIT
-};
+struct hlist_head __rcu *ima_htable;
/* mutex protects atomicity of extending measurement list
* and extending the TPM PCR aggregate. Since tpm_extend can take
@@ -52,17 +50,53 @@ static DEFINE_MUTEX(ima_extend_list_mutex);
*/
static bool ima_measurements_suspended;
+/* Callers must call synchronize_rcu() and free the hash table. */
+static struct hlist_head *ima_alloc_replace_htable(void)
+{
+ struct hlist_head *old_htable, *new_htable;
+
+ /* Initializing to zeros is equivalent to call HLIST_HEAD_INIT. */
+ new_htable = kcalloc(IMA_MEASURE_HTABLE_SIZE, sizeof(struct hlist_head),
+ GFP_KERNEL);
+ if (!new_htable)
+ return ERR_PTR(-ENOMEM);
+
+ old_htable = rcu_replace_pointer(ima_htable, new_htable,
+ lockdep_is_held(&ima_extend_list_mutex));
+
+ return old_htable;
+}
+
+int __init ima_init_htable(void)
+{
+ struct hlist_head *old_htable;
+
+ mutex_lock(&ima_extend_list_mutex);
+ old_htable = ima_alloc_replace_htable();
+ mutex_unlock(&ima_extend_list_mutex);
+
+ if (IS_ERR(old_htable))
+ return PTR_ERR(old_htable);
+
+ /* Synchronize_rcu() and kfree() not necessary, only for robustness. */
+ synchronize_rcu();
+ kfree(old_htable);
+ return 0;
+}
+
/* lookup up the digest value in the hash table, and return the entry */
static struct ima_queue_entry *ima_lookup_digest_entry(u8 *digest_value,
int pcr)
{
struct ima_queue_entry *qe, *ret = NULL;
+ struct hlist_head *htable;
unsigned int key;
int rc;
key = ima_hash_key(digest_value);
rcu_read_lock();
- hlist_for_each_entry_rcu(qe, &ima_htable[key], hnext) {
+ htable = rcu_dereference(ima_htable);
+ hlist_for_each_entry_rcu(qe, &htable[key], hnext) {
rc = memcmp(qe->entry->digests[ima_hash_algo_idx].digest,
digest_value, hash_digest_size[ima_hash_algo]);
if ((rc == 0) && (qe->entry->pcr == pcr)) {
@@ -102,6 +136,7 @@ static int ima_add_digest_entry(struct ima_template_entry *entry,
bool update_htable)
{
struct ima_queue_entry *qe;
+ struct hlist_head *htable;
unsigned int key;
qe = kmalloc_obj(*qe);
@@ -114,10 +149,13 @@ static int ima_add_digest_entry(struct ima_template_entry *entry,
INIT_LIST_HEAD(&qe->later);
list_add_tail_rcu(&qe->later, &ima_measurements);
+ htable = rcu_dereference_protected(ima_htable,
+ lockdep_is_held(&ima_extend_list_mutex));
+
atomic_long_inc(&ima_num_records);
if (update_htable) {
key = ima_hash_key(entry->digests[ima_hash_algo_idx].digest);
- hlist_add_head_rcu(&qe->hnext, &ima_htable[key]);
+ hlist_add_head_rcu(&qe->hnext, &htable[key]);
}
if (binary_runtime_size != ULONG_MAX) {
--
2.43.0
^ permalink raw reply related
* [PATCH v7 01/12] ima: Remove ima_h_table structure
From: Roberto Sassu @ 2026-06-05 17:22 UTC (permalink / raw)
To: corbet, skhan, zohar, dmitry.kasatkin, eric.snowberg, paul,
jmorris, serge
Cc: linux-doc, linux-kernel, linux-integrity, linux-security-module,
gregorylumen, chenste, nramas, Roberto Sassu
In-Reply-To: <20260605172236.2042045-1-roberto.sassu@huaweicloud.com>
From: Roberto Sassu <roberto.sassu@huawei.com>
The ima_h_table structure is a collection of IMA measurement list
metadata - number of records in the IMA measurement list, number of
integrity violations, and a hash table containing the IMA template data
hash, needed to prevent measurement list record duplication.
Removing records from the measurement list needs to be reflected in the
hash table. As a pre-req to removing records from the measurement list,
separate those counters from the hash table, remove the ima_h_table
structure, and just replace the hash table pointer.
Finally, rename ima_show_htable_value(), ima_show_htable_violations()
and ima_htable_violations_ops respectively to ima_show_counter(),
ima_show_num_violations() and ima_num_violations_ops.
Link: https://github.com/linux-integrity/linux/issues/1
Signed-off-by: Roberto Sassu <roberto.sassu@huawei.com>
---
security/integrity/ima/ima.h | 11 +++++------
security/integrity/ima/ima_api.c | 2 +-
security/integrity/ima/ima_fs.c | 20 +++++++++-----------
security/integrity/ima/ima_kexec.c | 2 +-
security/integrity/ima/ima_queue.c | 15 ++++++++-------
5 files changed, 24 insertions(+), 26 deletions(-)
diff --git a/security/integrity/ima/ima.h b/security/integrity/ima/ima.h
index 69e9bf0b82c6..b3ad7eac6a1e 100644
--- a/security/integrity/ima/ima.h
+++ b/security/integrity/ima/ima.h
@@ -324,12 +324,11 @@ int ima_lsm_policy_change(struct notifier_block *nb, unsigned long event,
*/
extern spinlock_t ima_queue_lock;
-struct ima_h_table {
- atomic_long_t len; /* number of stored measurements in the list */
- atomic_long_t violations;
- struct hlist_head queue[IMA_MEASURE_HTABLE_SIZE];
-};
-extern struct ima_h_table ima_htable;
+/* Total number of measurement list records since hard boot. */
+extern atomic_long_t ima_num_records;
+/* Total number of violations since hard boot. */
+extern atomic_long_t ima_num_violations;
+extern struct hlist_head ima_htable[IMA_MEASURE_HTABLE_SIZE];
static inline unsigned int ima_hash_key(u8 *digest)
{
diff --git a/security/integrity/ima/ima_api.c b/security/integrity/ima/ima_api.c
index 0916f24f005f..122d127e108d 100644
--- a/security/integrity/ima/ima_api.c
+++ b/security/integrity/ima/ima_api.c
@@ -146,7 +146,7 @@ void ima_add_violation(struct file *file, const unsigned char *filename,
int result;
/* can overflow, only indicator */
- atomic_long_inc(&ima_htable.violations);
+ atomic_long_inc(&ima_num_violations);
result = ima_alloc_init_template(&event_data, &entry, NULL);
if (result < 0) {
diff --git a/security/integrity/ima/ima_fs.c b/security/integrity/ima/ima_fs.c
index ca4931a95098..523d3e81f631 100644
--- a/security/integrity/ima/ima_fs.c
+++ b/security/integrity/ima/ima_fs.c
@@ -38,8 +38,8 @@ __setup("ima_canonical_fmt", default_canonical_fmt_setup);
static int valid_policy = 1;
-static ssize_t ima_show_htable_value(char __user *buf, size_t count,
- loff_t *ppos, atomic_long_t *val)
+static ssize_t ima_show_counter(char __user *buf, size_t count, loff_t *ppos,
+ atomic_long_t *val)
{
char tmpbuf[32]; /* greater than largest 'long' string value */
ssize_t len;
@@ -48,15 +48,14 @@ static ssize_t ima_show_htable_value(char __user *buf, size_t count,
return simple_read_from_buffer(buf, count, ppos, tmpbuf, len);
}
-static ssize_t ima_show_htable_violations(struct file *filp,
- char __user *buf,
- size_t count, loff_t *ppos)
+static ssize_t ima_show_num_violations(struct file *filp, char __user *buf,
+ size_t count, loff_t *ppos)
{
- return ima_show_htable_value(buf, count, ppos, &ima_htable.violations);
+ return ima_show_counter(buf, count, ppos, &ima_num_violations);
}
-static const struct file_operations ima_htable_violations_ops = {
- .read = ima_show_htable_violations,
+static const struct file_operations ima_num_violations_ops = {
+ .read = ima_show_num_violations,
.llseek = generic_file_llseek,
};
@@ -64,8 +63,7 @@ static ssize_t ima_show_measurements_count(struct file *filp,
char __user *buf,
size_t count, loff_t *ppos)
{
- return ima_show_htable_value(buf, count, ppos, &ima_htable.len);
-
+ return ima_show_counter(buf, count, ppos, &ima_num_records);
}
static const struct file_operations ima_measurements_count_ops = {
@@ -545,7 +543,7 @@ int __init ima_fs_init(void)
}
dentry = securityfs_create_file("violations", S_IRUSR | S_IRGRP,
- ima_dir, NULL, &ima_htable_violations_ops);
+ ima_dir, NULL, &ima_num_violations_ops);
if (IS_ERR(dentry)) {
ret = PTR_ERR(dentry);
goto out;
diff --git a/security/integrity/ima/ima_kexec.c b/security/integrity/ima/ima_kexec.c
index 36a34c54de58..77ad370dbc37 100644
--- a/security/integrity/ima/ima_kexec.c
+++ b/security/integrity/ima/ima_kexec.c
@@ -43,7 +43,7 @@ void ima_measure_kexec_event(const char *event_name)
int n;
buf_size = ima_get_binary_runtime_size();
- len = atomic_long_read(&ima_htable.len);
+ len = atomic_long_read(&ima_num_records);
n = scnprintf(ima_kexec_event, IMA_KEXEC_EVENT_LEN,
"kexec_segment_size=%lu;ima_binary_runtime_size=%lu;"
diff --git a/security/integrity/ima/ima_queue.c b/security/integrity/ima/ima_queue.c
index 319522450854..6bdaefc790c3 100644
--- a/security/integrity/ima/ima_queue.c
+++ b/security/integrity/ima/ima_queue.c
@@ -32,11 +32,12 @@ static unsigned long binary_runtime_size;
static unsigned long binary_runtime_size = ULONG_MAX;
#endif
+atomic_long_t ima_num_records = ATOMIC_LONG_INIT(0);
+atomic_long_t ima_num_violations = ATOMIC_LONG_INIT(0);
+
/* key: inode (before secure-hashing a file) */
-struct ima_h_table ima_htable = {
- .len = ATOMIC_LONG_INIT(0),
- .violations = ATOMIC_LONG_INIT(0),
- .queue[0 ... IMA_MEASURE_HTABLE_SIZE - 1] = HLIST_HEAD_INIT
+struct hlist_head ima_htable[IMA_MEASURE_HTABLE_SIZE] = {
+ [0 ... IMA_MEASURE_HTABLE_SIZE - 1] = HLIST_HEAD_INIT
};
/* mutex protects atomicity of extending measurement list
@@ -61,7 +62,7 @@ static struct ima_queue_entry *ima_lookup_digest_entry(u8 *digest_value,
key = ima_hash_key(digest_value);
rcu_read_lock();
- hlist_for_each_entry_rcu(qe, &ima_htable.queue[key], hnext) {
+ hlist_for_each_entry_rcu(qe, &ima_htable[key], hnext) {
rc = memcmp(qe->entry->digests[ima_hash_algo_idx].digest,
digest_value, hash_digest_size[ima_hash_algo]);
if ((rc == 0) && (qe->entry->pcr == pcr)) {
@@ -113,10 +114,10 @@ static int ima_add_digest_entry(struct ima_template_entry *entry,
INIT_LIST_HEAD(&qe->later);
list_add_tail_rcu(&qe->later, &ima_measurements);
- atomic_long_inc(&ima_htable.len);
+ atomic_long_inc(&ima_num_records);
if (update_htable) {
key = ima_hash_key(entry->digests[ima_hash_algo_idx].digest);
- hlist_add_head_rcu(&qe->hnext, &ima_htable.queue[key]);
+ hlist_add_head_rcu(&qe->hnext, &ima_htable[key]);
}
if (binary_runtime_size != ULONG_MAX) {
--
2.43.0
^ permalink raw reply related
* [PATCH v7 00/12] ima: Exporting and deleting IMA measurement records from kernel memory
From: Roberto Sassu @ 2026-06-05 17:22 UTC (permalink / raw)
To: corbet, skhan, zohar, dmitry.kasatkin, eric.snowberg, paul,
jmorris, serge
Cc: linux-doc, linux-kernel, linux-integrity, linux-security-module,
gregorylumen, chenste, nramas, Roberto Sassu
From: Roberto Sassu <roberto.sassu@huawei.com>
Introduction
============
The IMA measurements list is currently stored in the kernel memory.
Memory occupation grows linearly with the number of records, and can
become a problem especially in environments with reduced resources.
While there is an advantage in keeping the IMA measurements list in
kernel memory, so that it is always available for reading from the
securityfs interfaces, storing it elsewhere would make it possible to
free precious memory for other kernel usage.
The IMA measurements list needs to be retained and safely stored for new
attestation servers to validate it. Assuming the IMA measurements list
is properly saved, storing it outside the kernel does not introduce
security issues, since its integrity is anyway protected by the TPM.
Hence, the new IMA staging mechanism is introduced to export IMA
measurements to user space and delete them from kernel space.
Staging consists in atomically moving the current measurements list to a
temporary list, so that measurements can be deleted afterwards. The
staging operation locks the hot path (racing with addition of new
measurements) for a very short time, only for swapping the list
pointers. Deletion of the measurements instead is done locklessly, away
from the hot path.
There are two flavors of the staging mechanism. In the staging with
prompt, all current measurements are staged, read and deleted upon
confirmation. In the staging and deleting flavor, N measurements are
staged from the beginning of the current measurements list and
immediately deleted without confirmation.
Usage
=====
The IMA staging mechanism can be enabled from the kernel configuration
with the CONFIG_IMA_STAGING option. This option prevents inadvertently
removing the IMA measurement list on systems which do not properly save
it.
If the option is enabled, IMA duplicates the current securityfs
measurements interfaces (both binary and ASCII), by adding the _staged
file suffix. Both the original and the staging interfaces gain the write
permission for the root user and group, but require the process to have
CAP_SYS_ADMIN set.
The staging mechanism supports two flavors.
Staging with prompt
~~~~~~~~~~~~~~~~~~~
The current measurement list is moved to a temporary staging area,
allowing it to be saved to external storage, before being deleted upon
confirmation.
This staging process is achieved with the following steps.
1. echo A > <_staged interface>: the user requests IMA to stage the
entire measurements list;
2. cat <_staged interface>: the user reads the staged measurements;
3. echo D > <_staged interface>: the user requests IMA to delete
staged measurements.
Staging and deleting
~~~~~~~~~~~~~~~~~~~~
N measurements are staged to a temporary staging area, and immediately
deleted without further confirmation.
This staging process is achieved with the following steps.
1. cat <original interface>: the user reads the current measurements
list and determines what the value N for staging should be;
2. echo N > <original interface>: the user requests IMA to delete N
measurements from the current measurements list.
Management of Staged Measurements
=================================
Since with the staging mechanism measurement records are removed from
the kernel, the staged measurements need to be saved in a storage and
concatenated together, so that they can be presented during remote
attestation as if staging was never done. This task can be accomplished
by a remote attestation agent modified to support staging, or a system
service.
Patch set content
=================
Patches 1-8 are preparatory patches to quickly replace the hash table,
maintain separate counters for the different measurements list types,
mediate access to the measurements list interface, and simplify the staging
patches.
Patch 9 introduces the staging with prompt flavor. Patch 10 makes it
possible to flush the hash table when deleting all the staged measurements.
Patch 11 introduces the staging and deleting flavor. Patch 12 adds the
documentation of the staging mechanism.
Changelog
=========
v6:
- Make ima_extend_list_mutex as static since it is not needed anymore by
ima_dump_measurement_list() (suggested by Mimi)
- Export ima_flush_htable in patch 11 instead of 10 (suggested by Mimi)
- Add clarification in the documentation regarding a proactive remote
attestation agent, and storing all the measurements in the storage
(suggested by Mimi)
v5:
- Add motivation for the ima_flush_htable= kernel option (suggested by
Mimi)
- New documentation title and fixes (suggested by Mimi)
- Allow stage all command on the _staged interface instead of the original
- Set CONFIG_IMA_STAGING default to n (suggested by Mimi)
- Rename ima_num_entries to ima_num_records (suggested by Mimi)
- Comment for ima_num_records and ima_num_violations (suggested by Mimi)
- Add overflow check in ima_measure_lock()
- Allow a writer to open for write or read/write the other staging
interfaces
- Ignore ppos in _ima_measurements_write()
- Implement lockless kexec measurement lists dump by denying
staging/delete after measurement suspend (collapse patch 12 into 9 and
11)
- Refuse delete based on measurement suspend instead of using
ima_copied_flags (suggested by Mimi)
- Add staging/deleting functions documentation
v4:
- Add write permission to the original measurement interface, and move
the A and N staging commands to that interface
- Explain better the two staging flavors and highlight that the staging
and delete only stages measurements internally
- Rename ima_queue_staged_delete_partial() to ima_queue_delete_partial()
- Replace ima_staged_measurements_prepended with per measurements list
flag to avoid copying staged and active list measurements twice
- Optimize the staging and deleting flavor by locklessly determining the
cut position in the active list, and immediately deleting entries
without explicit staging and splicing (suggested by Steven Chen)
v3:
- Add Kconfig option to enable the staging mechanism (suggested by Mimi)
- Change the meaning of BINARY_STAGED to be just the staged measurements
- Separate the two staging flavors in two different functions:
ima_queue_staged_delete_all() for staging with prompt,
ima_queue_staged_delete_partial() for staging and deleting
- Delete N entries without staging first (suggested by Mimi)
- Avoid duplicate staged entries if there is contention between the
measurements list interfaces and kexec
v2:
- New patch to move measurements and violation counters outside the
ima_h_table structure
- New patch to quickly replace the hash table
- Forbid partial deletion when flushing hash table (suggested by Mimi)
- Ignore ima_flush_htable if CONFIG_IMA_DISABLE_HTABLE is enabled
- BINARY_SIZE_* renamed to BINARY_* for better clarity
- Removed ima_measurements_staged_exist and testing list empty instead
- ima_queue_stage_trim() and ima_queue_delete_staged_trimmed() renamed to
ima_queue_stage() and ima_queue_delete_staged()
- New delete interval [1, ULONG_MAX - 1]
- Rename ima_measure_lock to ima_measure_mutex
- Move seq_open() and seq_release() outside the ima_measure_mutex lock
- Drop ima_measurements_staged_read() and use seq_read() instead
- Optimize create_securityfs_measurement_lists() changes
- New file name format with _staged suffix at the end of the file name
- Use _rcu list variant in ima_dump_measurement_list()
- Remove support for direct trimming and splice the remaining entries to
the active list (suggested by Mimi)
- Hot swap the hash table if flushing is requested
v1:
- Support for direct trimming without staging
- Support unstaging on kexec (requested by Gregory Lumen)
Roberto Sassu (12):
ima: Remove ima_h_table structure
ima: Replace static htable queue with dynamically allocated array
ima: Introduce per binary measurements list type ima_num_records
counter
ima: Introduce per binary measurements list type binary_runtime_size
value
ima: Introduce _ima_measurements_start() and _ima_measurements_next()
ima: Mediate open/release method of the measurements list
ima: Use snprintf() in create_securityfs_measurement_lists
ima: Introduce ima_dump_measurement()
ima: Add support for staging measurements with prompt
ima: Add support for flushing the hash table when staging measurements
ima: Support staging and deleting N measurements records
doc: security: Add documentation of exporting and deleting IMA
measurements
.../admin-guide/kernel-parameters.txt | 6 +
Documentation/security/IMA-export-delete.rst | 203 ++++++++++
Documentation/security/index.rst | 1 +
MAINTAINERS | 2 +
security/integrity/ima/Kconfig | 15 +
security/integrity/ima/ima.h | 28 +-
security/integrity/ima/ima_api.c | 2 +-
security/integrity/ima/ima_fs.c | 346 ++++++++++++++++--
security/integrity/ima/ima_init.c | 5 +
security/integrity/ima/ima_kexec.c | 42 ++-
security/integrity/ima/ima_queue.c | 327 ++++++++++++++++-
11 files changed, 905 insertions(+), 72 deletions(-)
create mode 100644 Documentation/security/IMA-export-delete.rst
--
2.43.0
^ permalink raw reply
* Re: [RFC PATCH v2 1/6] kcov: add per-task dataflow tracking for function arguments/return values
From: Nicolas Schier @ 2026-06-03 19:25 UTC (permalink / raw)
To: Yunseong Kim
Cc: Ingo Molnar, Peter Zijlstra, Juri Lelli, Vincent Guittot,
Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, K Prateek Nayak, Dmitry Vyukov,
Andrey Konovalov, Andrew Morton, Nathan Chancellor,
Nick Desaulniers, Bill Wendling, Justin Stitt, Miguel Ojeda,
Boqun Feng, Gary Guo, Björn Roy Baron, Benno Lossin,
Andreas Hindborg, Alice Ryhl, Trevor Gross, Danilo Krummrich,
Jonathan Corbet, Shuah Khan, linux-kernel, kasan-dev, llvm,
linux-kbuild, rust-for-linux, workflows, linux-doc, Yunseong Kim
In-Reply-To: <20260603-kcov-dataflow-next-20260603-v2-1-fee0939de2c4@est.tech>
On Wed, Jun 03, 2026 at 07:43:28PM +0200, Yunseong Kim wrote:
> Add a new KCOV subsystem that captures function arguments at entry and
> return values at exit, with automatic struct field expansion using
> compiler-generated DebugInfo metadata.
>
> Key components:
> - CONFIG_KCOV_DATAFLOW_ARGS: enables argument capture
> - CONFIG_KCOV_DATAFLOW_RET: enables return value capture
> - /sys/kernel/debug/kcov_dataflow: separate device from legacy kcov
> - Ioctl namespace 'd' (KCOV_DF_INIT_TRACE, KCOV_DF_ENABLE, KCOV_DF_DISABLE)
> - Per-task buffer: task->kcov_df_area with atomic xadd reservation
> - Fault-tolerant: all reads via copy_from_kernel_nofault()
> - Recursion-safe: notrace __no_sanitize_coverage noinline
> - ERR_PTR aware: skips struct expansion for error pointers
>
> The callbacks (__sanitizer_cov_trace_args/ret) are inserted by the
> compiler when -fsanitize-coverage=dataflow-args,dataflow-ret is used.
> The Kconfig options depend on cc-option to verify compiler support.
>
> Buffer format (TLV records, all u64):
> area[0]: atomic word count
> [pos+0]: type_and_seq (0xE=entry, 0xF=return in upper 4 bits)
> [pos+1]: PC
> [pos+2]: meta (arg_idx | arg_size | ptr)
> [pos+3..N]: field values read via copy_from_kernel_nofault()
>
> This is completely independent from legacy /sys/kernel/debug/kcov.
> Existing users (syzkaller, oss-fuzz) are unaffected.
>
> Signed-off-by: Yunseong Kim <yunseong.kim@est.tech>
> ---
> include/linux/sched.h | 8 ++
> kernel/kcov.c | 291 ++++++++++++++++++++++++++++++++++++++++++++++++++
> lib/Kconfig.debug | 22 ++++
> 3 files changed, 321 insertions(+)
>
> diff --git a/include/linux/sched.h b/include/linux/sched.h
> index c4433c185ad8..03be4b495f70 100644
> --- a/include/linux/sched.h
> +++ b/include/linux/sched.h
> @@ -1533,6 +1533,14 @@ struct task_struct {
> /* KCOV sequence number: */
> int kcov_sequence;
>
> + /* KCOV dataflow per-task sequence counter for TLV records: */
> + u32 kcov_dataflow_seq;
> +
> + /* KCOV dataflow: separate buffer for trace-args/trace-ret */
> + unsigned int kcov_df_size;
> + void *kcov_df_area;
> + bool kcov_df_enabled;
> +
> /* Collect coverage from softirq context: */
> unsigned int kcov_softirq;
> #endif
> diff --git a/kernel/kcov.c b/kernel/kcov.c
> index 1df373fb562b..d3c9c0efe961 100644
> --- a/kernel/kcov.c
> +++ b/kernel/kcov.c
> @@ -353,6 +353,288 @@ void notrace __sanitizer_cov_trace_switch(kcov_u64 val, void *arg)
> EXPORT_SYMBOL(__sanitizer_cov_trace_switch);
> #endif /* ifdef CONFIG_KCOV_ENABLE_COMPARISONS */
>
> +#if defined(CONFIG_KCOV_DATAFLOW_ARGS) || defined(CONFIG_KCOV_DATAFLOW_RET)
> +/*
> + * KCOV Dataflow: /sys/kernel/debug/kcov_dataflow
> + *
> + * Completely separate from legacy /sys/kernel/debug/kcov.
> + * Own buffer, own ioctl, own mmap. No printk — buffer only.
> + *
> + * TLV record layout (all u64):
> + * area[0]: total u64 words written (atomic counter)
> + * [pos+0]: type_and_seq (0xE=entry|0xF=return in upper 4 bits, seq in lower 24)
> + * [pos+1]: PC
> + * [pos+2]: raw pointer | (arg_idx << 56) | (arg_size << 48) for entry
> + * [pos+3..N]: field values (or scalar value if num_fields=0)
> + */
> +#define KCOV_DF_TYPE_ENTRY 0xE0000000ULL
> +#define KCOV_DF_TYPE_RET 0xF0000000ULL
> +#define KCOV_DF_MAGIC_BAD 0xBADADD85ULL
> +#define KCOV_DF_IS_ERR(p) ((unsigned long)(p) >= (unsigned long)-4095UL)
> +
> +/* Ioctl commands for /sys/kernel/debug/kcov_dataflow */
> +#define KCOV_DF_INIT_TRACE _IOR('d', 1, unsigned long)
> +#define KCOV_DF_ENABLE _IO('d', 100)
> +#define KCOV_DF_DISABLE _IO('d', 101)
> +
> +struct kcov_dataflow {
> + refcount_t refcount;
> + spinlock_t lock;
> + unsigned int size; /* in u64 words */
> + void *area;
> + struct task_struct *t;
> +};
> +
> +static void kcov_df_put(struct kcov_dataflow *df)
> +{
> + if (refcount_dec_and_test(&df->refcount)) {
> + vfree(df->area);
> + kfree(df);
> + }
> +}
> +
> +/*
> + * Core write function — no printk, no locks, just atomic buffer write.
> + * Called from __sanitizer_cov_trace_args/ret in instrumented code.
> + */
> +static noinline notrace __no_sanitize_coverage void
> +kcov_df_write(u64 type_marker, u64 pc, u64 meta, void *ptr,
> + u64 *offsets, u32 num_fields)
> +{
> + struct task_struct *t = current;
> + u64 *area;
> + unsigned long pos, max_pos;
> + u32 record_len, seq, i;
> +
> + if (!t->kcov_df_enabled)
> + return;
> +
> + area = (u64 *)t->kcov_df_area;
> + if (!area)
> + return;
> +
> + max_pos = t->kcov_df_size;
> +
> + /* Record: header(1) + pc(1) + meta(1) + fields or scalar(max 1) */
> + record_len = 3 + (num_fields > 0 ? num_fields : 1);
> +
> + /* Atomic reservation */
> + pos = 1 + xadd((unsigned long *)&area[0], record_len);
> + if (unlikely(pos + record_len > max_pos)) {
> + xadd((unsigned long *)&area[0], -(long)record_len);
> + return;
> + }
> +
> + seq = ++t->kcov_dataflow_seq;
> + area[pos] = type_marker | (seq & 0x00FFFFFFULL);
> + area[pos + 1] = pc;
> + area[pos + 2] = meta;
> +
> + if (num_fields == 0) {
> + /* Scalar: read value from ptr using size from meta */
> + u64 val = 0;
> + u32 sz = (meta >> 48) & 0xFF;
> +
> + if (sz > sizeof(val))
> + sz = sizeof(val);
> + if (ptr && !KCOV_DF_IS_ERR(ptr))
> + copy_from_kernel_nofault(&val, ptr, sz);
> + area[pos + 3] = val;
> + } else {
> + /* Struct fields */
> + if (KCOV_DF_IS_ERR(ptr)) {
> + for (i = 0; i < num_fields; i++)
> + area[pos + 3 + i] = KCOV_DF_MAGIC_BAD;
> + return;
> + }
> + for (i = 0; i < num_fields; i++) {
> + u64 off, sz, val = KCOV_DF_MAGIC_BAD;
> + void *fa;
> +
> + if (copy_from_kernel_nofault(&off, &offsets[i * 2], sizeof(off)) ||
> + copy_from_kernel_nofault(&sz, &offsets[i * 2 + 1], sizeof(sz))) {
> + area[pos + 3 + i] = KCOV_DF_MAGIC_BAD;
> + continue;
> + }
> + fa = (void *)((unsigned long)ptr + off);
> + val = 0;
> + if (sz <= sizeof(val))
> + copy_from_kernel_nofault(&val, fa, sz);
> + else
> + copy_from_kernel_nofault(&val, fa, sizeof(val));
> + area[pos + 3 + i] = val;
> + }
> + }
> +}
> +
> +#ifdef CONFIG_KCOV_DATAFLOW_ARGS
> +noinline void notrace __no_sanitize_coverage
> +__sanitizer_cov_trace_args(u64 pc, u32 arg_idx, u32 arg_size, void *arg_ptr,
> + u64 *offsets, u32 num_fields);
> +
> +noinline void notrace __no_sanitize_coverage
> +__sanitizer_cov_trace_args(u64 pc, u32 arg_idx, u32 arg_size, void *arg_ptr,
> + u64 *offsets, u32 num_fields)
> +{
> + /* meta: [arg_idx(8) | arg_size(8) | ptr(48)] */
> + u64 meta = ((u64)arg_idx << 56) | ((u64)arg_size << 48) |
> + ((u64)(unsigned long)arg_ptr & 0xFFFFFFFFFFFFULL);
> + kcov_df_write(KCOV_DF_TYPE_ENTRY, pc, meta, arg_ptr,
> + offsets, num_fields);
> +}
> +EXPORT_SYMBOL(__sanitizer_cov_trace_args);
> +#endif
> +
> +#ifdef CONFIG_KCOV_DATAFLOW_RET
> +noinline void notrace __no_sanitize_coverage
> +__sanitizer_cov_trace_ret(u64 pc, u32 ret_size, void *ret_val,
> + u64 *offsets, u32 num_fields);
> +
> +noinline void notrace __no_sanitize_coverage
> +__sanitizer_cov_trace_ret(u64 pc, u32 ret_size, void *ret_val,
> + u64 *offsets, u32 num_fields)
> +{
> + u64 meta = ((u64)ret_size << 48) |
> + ((u64)(unsigned long)ret_val & 0xFFFFFFFFFFFFULL);
> + kcov_df_write(KCOV_DF_TYPE_RET, pc, meta, ret_val,
> + offsets, num_fields);
> +}
> +EXPORT_SYMBOL(__sanitizer_cov_trace_ret);
> +#endif
> +
> +/* --- /sys/kernel/debug/kcov_dataflow file operations --- */
> +
> +static int kcov_df_open(struct inode *inode, struct file *filep)
> +{
> + struct kcov_dataflow *df;
> +
> + df = kzalloc(sizeof(*df), GFP_KERNEL);
> + if (!df)
> + return -ENOMEM;
> + spin_lock_init(&df->lock);
> + refcount_set(&df->refcount, 1);
> + filep->private_data = df;
> + return nonseekable_open(inode, filep);
> +}
> +
> +static int kcov_df_close(struct inode *inode, struct file *filep)
> +{
> + struct kcov_dataflow *df = filep->private_data;
> + unsigned long flags;
> +
> + spin_lock_irqsave(&df->lock, flags);
> + if (df->t == current) {
> + current->kcov_df_enabled = false;
> + current->kcov_df_area = NULL;
> + current->kcov_df_size = 0;
> + df->t = NULL;
> + }
> + spin_unlock_irqrestore(&df->lock, flags);
> + kcov_df_put(df);
> + return 0;
> +}
> +
> +static int kcov_df_mmap(struct file *filep, struct vm_area_struct *vma)
> +{
> + struct kcov_dataflow *df = filep->private_data;
> + unsigned long size, off;
> + struct page *page;
> + unsigned long flags;
> + void *area;
> + int res = 0;
> +
> + spin_lock_irqsave(&df->lock, flags);
> + size = df->size * sizeof(u64);
> + if (!df->area || vma->vm_pgoff != 0 ||
> + vma->vm_end - vma->vm_start != size) {
> + res = -EINVAL;
> + goto out;
> + }
> + area = df->area;
> + spin_unlock_irqrestore(&df->lock, flags);
> +
> + vm_flags_set(vma, VM_DONTEXPAND);
> + for (off = 0; off < size; off += PAGE_SIZE) {
> + page = vmalloc_to_page(area + off);
> + res = vm_insert_page(vma, vma->vm_start + off, page);
> + if (res)
> + return res;
> + }
> + return 0;
> +out:
> + spin_unlock_irqrestore(&df->lock, flags);
> + return res;
> +}
> +
> +static long kcov_df_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
> +{
> + struct kcov_dataflow *df = filep->private_data;
> + unsigned long flags;
> + unsigned long size;
> + int res = 0;
> +
> + spin_lock_irqsave(&df->lock, flags);
> + switch (cmd) {
> + case KCOV_DF_INIT_TRACE:
> + if (df->area) {
> + res = -EBUSY;
> + break;
> + }
> + size = arg;
> + if (size < 2 || size > (128 << 20) / sizeof(u64)) {
> + res = -EINVAL;
> + break;
> + }
> + spin_unlock_irqrestore(&df->lock, flags);
> + df->area = vmalloc_user(size * sizeof(u64));
> + if (!df->area)
> + return -ENOMEM;
> + spin_lock_irqsave(&df->lock, flags);
> + df->size = size;
> + break;
> +
> + case KCOV_DF_ENABLE:
> + if (!df->area || df->t) {
> + res = -EINVAL;
> + break;
> + }
> + df->t = current;
> + current->kcov_df_area = df->area;
> + current->kcov_df_size = df->size;
> + current->kcov_dataflow_seq = 0;
> + /* Barrier before enabling */
> + barrier();
> + current->kcov_df_enabled = true;
> + break;
> +
> + case KCOV_DF_DISABLE:
> + if (df->t != current) {
> + res = -EINVAL;
> + break;
> + }
> + current->kcov_df_enabled = false;
> + barrier();
> + current->kcov_df_area = NULL;
> + current->kcov_df_size = 0;
> + df->t = NULL;
> + break;
> +
> + default:
> + res = -ENOTTY;
> + }
> + spin_unlock_irqrestore(&df->lock, flags);
> + return res;
> +}
> +
> +static const struct file_operations kcov_df_fops = {
> + .open = kcov_df_open,
> + .unlocked_ioctl = kcov_df_ioctl,
> + .compat_ioctl = kcov_df_ioctl,
> + .mmap = kcov_df_mmap,
> + .release = kcov_df_close,
> +};
> +#endif /* CONFIG_KCOV_DATAFLOW_ARGS || CONFIG_KCOV_DATAFLOW_RET */
> +
> static void kcov_start(struct task_struct *t, struct kcov *kcov,
> unsigned int size, void *area, enum kcov_mode mode,
> int sequence)
> @@ -1146,6 +1428,15 @@ static int __init kcov_init(void)
> */
> debugfs_create_file_unsafe("kcov", 0600, NULL, NULL, &kcov_fops);
>
> +#if defined(CONFIG_KCOV_DATAFLOW_ARGS) || defined(CONFIG_KCOV_DATAFLOW_RET)
> + /*
> + * Toggle verbose printk: echo 1 > /sys/kernel/debug/kcov_dataflow_verbose
> + * Default off — zero overhead when not debugging.
> + */
> + debugfs_create_file_unsafe("kcov_dataflow", 0600, NULL, NULL,
> + &kcov_df_fops);
> +#endif
> +
> #ifdef CONFIG_KCOV_SELFTEST
> selftest();
> #endif
> diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
> index e2f976c3301b..abd1a94589aa 100644
> --- a/lib/Kconfig.debug
> +++ b/lib/Kconfig.debug
> @@ -2261,6 +2261,28 @@ config KCOV_SELFTEST
> On test failure, causes the kernel to panic. Recommended to be
> enabled, ensuring critical functionality works as intended.
>
> +
> +config KCOV_DATAFLOW_ARGS
> + bool "Enable KCOV dataflow: function argument capture"
> + depends on KCOV
> + depends on $(cc-option,-fsanitize-coverage=dataflow-args)
> + help
> + Captures function arguments at entry via /sys/kernel/debug/kcov_dataflow.
> + Struct pointer arguments are auto-expanded using compiler DebugInfo
> + metadata, recording individual field values at runtime.
> + Enable per-module with: KCOV_DATAFLOW_file.o := y in the Makefile.
> + Requires clang with -fsanitize-coverage=dataflow-args support.
> +
> +config KCOV_DATAFLOW_RET
> + bool "Enable KCOV dataflow: return value capture"
> + depends on KCOV
> + depends on $(cc-option,-fsanitize-coverage=dataflow-ret)
> + help
> + Captures function return values via /sys/kernel/debug/kcov_dataflow.
> + Struct pointer returns are auto-expanded using compiler DebugInfo
> + metadata, recording individual field values at runtime.
> + Enable per-module with: KCOV_DATAFLOW_file.o := y in the Makefile.
> + Requires clang with -fsanitize-coverage=dataflow-ret support.
You might want to add an empty line here.
--
Nicolas
^ permalink raw reply
* Re: [PATCH] Documentation: bug-hunting.rst: fix grammar
From: Jonathan Corbet @ 2026-06-05 17:16 UTC (permalink / raw)
To: Manuel Ebner, Shuah Khan, open list:DOCUMENTATION, open list; +Cc: Manuel Ebner
In-Reply-To: <20260605170851.14096-2-manuelebner@mailbox.org>
Manuel Ebner <manuelebner@mailbox.org> writes:
> Fix two grammar issues to improve readability
>
> Signed-off-by: Manuel Ebner <manuelebner@mailbox.org>
> ---
> Documentation/admin-guide/bug-hunting.rst | 6 +++---
> 1 file changed, 3 insertions(+), 3 deletions(-)
>
> diff --git a/Documentation/admin-guide/bug-hunting.rst b/Documentation/admin-guide/bug-hunting.rst
> index 3901b43c96df..a74ffe2d8951 100644
> --- a/Documentation/admin-guide/bug-hunting.rst
> +++ b/Documentation/admin-guide/bug-hunting.rst
> @@ -63,8 +63,8 @@ Documentation/admin-guide/tainted-kernels.rst, "being loaded" is
> annotated with "+", and "being unloaded" is annotated with "-".
>
>
> -Where is the Oops message is located?
> --------------------------------------
> +Where is the Oops message located?
> +----------------------------------
This is fine
> Normally the Oops text is read from the kernel buffers by klogd and
> handed to ``syslogd`` which writes it to a syslog file, typically
> @@ -72,7 +72,7 @@ handed to ``syslogd`` which writes it to a syslog file, typically
> systemd, it may also be stored by the ``journald`` daemon, and accessed
> by running ``journalctl`` command.
>
> -Sometimes ``klogd`` dies, in which case you can run ``dmesg > file`` to
> +Sometimes ``klogd`` dies. In that case you can run ``dmesg > file`` to
This really just seems like churn?
Thanks,
jon
^ permalink raw reply
* [PATCH] Documentation: bug-hunting.rst: fix grammar
From: Manuel Ebner @ 2026-06-05 17:08 UTC (permalink / raw)
To: Jonathan Corbet, Shuah Khan, open list:DOCUMENTATION, open list
Cc: Manuel Ebner
Fix two grammar issues to improve readability
Signed-off-by: Manuel Ebner <manuelebner@mailbox.org>
---
Documentation/admin-guide/bug-hunting.rst | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/Documentation/admin-guide/bug-hunting.rst b/Documentation/admin-guide/bug-hunting.rst
index 3901b43c96df..a74ffe2d8951 100644
--- a/Documentation/admin-guide/bug-hunting.rst
+++ b/Documentation/admin-guide/bug-hunting.rst
@@ -63,8 +63,8 @@ Documentation/admin-guide/tainted-kernels.rst, "being loaded" is
annotated with "+", and "being unloaded" is annotated with "-".
-Where is the Oops message is located?
--------------------------------------
+Where is the Oops message located?
+----------------------------------
Normally the Oops text is read from the kernel buffers by klogd and
handed to ``syslogd`` which writes it to a syslog file, typically
@@ -72,7 +72,7 @@ handed to ``syslogd`` which writes it to a syslog file, typically
systemd, it may also be stored by the ``journald`` daemon, and accessed
by running ``journalctl`` command.
-Sometimes ``klogd`` dies, in which case you can run ``dmesg > file`` to
+Sometimes ``klogd`` dies. In that case you can run ``dmesg > file`` to
read the data from the kernel buffers and save it. Or you can
``cat /proc/kmsg > file``, however you have to break in to stop the transfer,
since ``kmsg`` is a "never ending file".
--
2.54.0
^ permalink raw reply related
* [RFC PATCH v2 10/10] selftests: kvm: Add guest_memfd_preservation_test
From: Tarun Sahu @ 2026-06-05 17:08 UTC (permalink / raw)
To: Jonathan Corbet, vannapurve, Tarun Sahu, fvdl, Pasha Tatashin,
Shuah Khan, sagis, aneesh.kumar, skhawaja, vipinsh, ackerleytng,
Pratyush Yadav, david, dmatlack, mark.rutland, Paolo Bonzini,
Mike Rapoport, Alexander Graf, seanjc, axelrasmussen
Cc: linux-kselftest, kexec, linux-kernel, linux-doc, kvm, linux-mm
In-Reply-To: <cover.1780676742.git.tarunsahu@google.com>
Add a new KVM selftest `guest_memfd_preservation_test` to verify that
guest memory backed by guest_memfd is preserved properly.
The test leverages the Live Update Orchestrator (LUO) infrastructure
to validate that memory folios and configuration layouts are
successfully saved and then restored during kernel live updates,
preventing any memory loss for the guest.
Here, I have used the kvm selftests framework by creating a new
vm and mapping two memory slots to it. One is the code that is executed
inside the vm and other is the guest_memfd whose memory is being
written by the guest code.
In Phase 1: Once data is written the vm exits and wait for the user
to trigger the kexec.
In Phase 2: A new vm is created with retrieved kvm and again two
memory slots are assigned. Once for guest code, and another is for
retrieved guest_memfd where guest_memfd memory is verified by the
executed guest code. If verification succeeds, The test passes.
Signed-off-by: Tarun Sahu <tarunsahu@google.com>
---
MAINTAINERS | 1 +
tools/testing/selftests/kvm/Makefile.kvm | 6 +-
.../kvm/guest_memfd_preservation_test.c | 230 ++++++++++++++++++
3 files changed, 236 insertions(+), 1 deletion(-)
create mode 100644 tools/testing/selftests/kvm/guest_memfd_preservation_test.c
diff --git a/MAINTAINERS b/MAINTAINERS
index ca459d032712..76e59620d2f1 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -14419,6 +14419,7 @@ L: kvm@vger.kernel.org
S: Maintained
T: git git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git
F: Documentation/liveupdate/vmm.rst
+F: tools/testing/selftests/kvm/guest_memfd_preservation_test.c
F: virt/kvm/guest_memfd_luo.c
F: virt/kvm/kvm_luo.c
diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
index 9118a5a51b89..68584d4ee1b0 100644
--- a/tools/testing/selftests/kvm/Makefile.kvm
+++ b/tools/testing/selftests/kvm/Makefile.kvm
@@ -161,6 +161,8 @@ TEST_GEN_PROGS_x86 += pre_fault_memory_test
# Compiled outputs used by test targets
TEST_GEN_PROGS_EXTENDED_x86 += x86/nx_huge_pages_test
+# Manual test that forks a persistent background daemon; skip auto CI run
+TEST_GEN_PROGS_EXTENDED_x86 += guest_memfd_preservation_test
TEST_GEN_PROGS_arm64 = $(TEST_GEN_PROGS_COMMON)
TEST_GEN_PROGS_arm64 += arm64/aarch32_id_regs
@@ -254,6 +256,7 @@ OVERRIDE_TARGETS = 1
# which causes the environment variable to override the makefile).
include ../lib.mk
include ../cgroup/lib/libcgroup.mk
+include ../liveupdate/lib/libliveupdate.mk
INSTALL_HDR_PATH = $(top_srcdir)/usr
LINUX_HDR_PATH = $(INSTALL_HDR_PATH)/include/
@@ -308,7 +311,8 @@ LIBKVM_S := $(filter %.S,$(LIBKVM))
LIBKVM_C_OBJ := $(patsubst %.c, $(OUTPUT)/%.o, $(LIBKVM_C))
LIBKVM_S_OBJ := $(patsubst %.S, $(OUTPUT)/%.o, $(LIBKVM_S))
LIBKVM_STRING_OBJ := $(patsubst %.c, $(OUTPUT)/%.o, $(LIBKVM_STRING))
-LIBKVM_OBJS = $(LIBKVM_C_OBJ) $(LIBKVM_S_OBJ) $(LIBKVM_STRING_OBJ) $(LIBCGROUP_O)
+LIBKVM_OBJS = $(LIBKVM_C_OBJ) $(LIBKVM_S_OBJ) $(LIBKVM_STRING_OBJ) \
+ $(LIBCGROUP_O) $(LIBLIVEUPDATE_O)
SPLIT_TEST_GEN_PROGS := $(patsubst %, $(OUTPUT)/%, $(SPLIT_TESTS))
SPLIT_TEST_GEN_OBJ := $(patsubst %, $(OUTPUT)/$(ARCH)/%.o, $(SPLIT_TESTS))
diff --git a/tools/testing/selftests/kvm/guest_memfd_preservation_test.c b/tools/testing/selftests/kvm/guest_memfd_preservation_test.c
new file mode 100644
index 000000000000..74f90c5c4bf5
--- /dev/null
+++ b/tools/testing/selftests/kvm/guest_memfd_preservation_test.c
@@ -0,0 +1,230 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026, Google LLC.
+ *
+ * Author: Tarun Sahu <tarunsahu@google.com>
+ *
+ * Test for VM and guest_memfd preservation across kexec (Live Update) via LUO.
+ *
+ * NOTE: This is a MANUAL test and is excluded from automated CI/testing
+ * frameworks because Phase 1 daemonizes into the background to pin resources
+ * and requires a human operator to manually trigger kexec before Phase 2
+ * is executed. Running Phase 1 automatically would leak the background daemon
+ * and cause CI runners to falsely interpret it as a passed test.
+ *
+ * Usage:
+ * Phase 1: ./guest_memfd_preservation_test
+ * Phase 2: ./guest_memfd_preservation_test --phase2
+ */
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <linux/sizes.h>
+#include <linux/falloc.h>
+
+#include "kvm_util.h"
+#include "processor.h"
+#include "test_util.h"
+#include "ucall_common.h"
+#include "../kselftest.h"
+#include "../kselftest_harness.h"
+
+#include <libliveupdate.h>
+
+#define SESSION_NAME "gmem_vm_preservation_session"
+#define VM_TOKEN 0x1001
+#define GMEM_TOKEN 0x1002
+
+#define GMEM_SIZE (16ULL * 1024 * 1024)
+#define DATA_SIZE (5ULL * 1024 * 1024)
+
+static size_t page_size;
+
+/* Deterministic byte pattern generation based on offset */
+static inline uint8_t get_pattern_byte(size_t offset)
+{
+ return (uint8_t)(offset ^ 0x5A);
+}
+
+static void guest_code_phase1(uint64_t gpa, uint64_t size, uint64_t data_size)
+{
+ uint8_t *mem = (uint8_t *)gpa;
+ size_t i;
+
+ for (i = 0; i < data_size; i++)
+ mem[i] = get_pattern_byte(i);
+
+ GUEST_DONE();
+}
+
+static void guest_code_phase2(uint64_t gpa, uint64_t size, uint64_t data_size)
+{
+ uint8_t *mem = (uint8_t *)gpa;
+ size_t i;
+
+ for (i = 0; i < data_size; i++) {
+ uint8_t val = get_pattern_byte(i);
+
+ __GUEST_ASSERT(mem[i] == val,
+ "Data mismatch at offset %lu! Expected 0x%x, got 0x%x",
+ i, val, mem[i]);
+ }
+
+ GUEST_DONE();
+}
+
+static void do_phase1(void)
+{
+ uint64_t flags = GUEST_MEMFD_FLAG_MMAP | GUEST_MEMFD_FLAG_INIT_SHARED;
+ int gmem_fd, dev_luo_fd, session_fd, ret;
+ const uint64_t gpa = SZ_4G;
+ struct kvm_vcpu *vcpu;
+ const int slot = 1;
+ struct kvm_vm *vm;
+
+ vm = __vm_create_shape_with_one_vcpu(VM_SHAPE_DEFAULT, &vcpu, 1,
+ guest_code_phase1);
+ gmem_fd = vm_create_guest_memfd(vm, GMEM_SIZE, flags);
+ vm_set_user_memory_region2(vm, slot, KVM_MEM_GUEST_MEMFD, gpa, GMEM_SIZE, NULL,
+ gmem_fd, 0);
+
+ for (size_t i = 0; i < GMEM_SIZE; i += page_size)
+ virt_pg_map(vm, gpa + i, gpa + i);
+
+ vcpu_args_set(vcpu, 3, gpa, GMEM_SIZE, DATA_SIZE);
+
+ vcpu_run(vcpu);
+ TEST_ASSERT_EQ(get_ucall(vcpu, NULL), UCALL_DONE);
+
+ dev_luo_fd = luo_open_device();
+ TEST_ASSERT(dev_luo_fd >= 0, "Failed to open /dev/liveupdate");
+
+ session_fd = luo_create_session(dev_luo_fd, SESSION_NAME);
+ TEST_ASSERT(session_fd >= 0, "Failed to create LUO session");
+
+ ret = luo_session_preserve_fd(session_fd, vm->fd, VM_TOKEN);
+ TEST_ASSERT(ret == 0, "Failed to preserve VM file descriptor");
+
+ ret = luo_session_preserve_fd(session_fd, gmem_fd, GMEM_TOKEN);
+ TEST_ASSERT(ret == 0, "Failed to preserve guest_memfd file descriptor");
+
+ printf("\n============================================================\n");
+ printf("Phase 1 Complete Successfully!\n");
+ printf("VM file and guest_memfd file have been preserved via LUO.\n");
+ printf("Tokens: VM_TOKEN=0x%x, GMEM_TOKEN=0x%x\n", VM_TOKEN, GMEM_TOKEN);
+ printf("Machine Size: %llu MB, Data Size: %llu MB\n", GMEM_SIZE / SZ_1M,
+ DATA_SIZE / SZ_1M);
+ printf("------------------------------------------------------------\n");
+
+ daemonize_and_wait();
+}
+
+static struct kvm_vm *vm_create_from_fd(int resurrected_vm_fd,
+ struct vm_shape shape)
+{
+ struct kvm_vm *vm;
+
+ vm = calloc(1, sizeof(*vm));
+ TEST_ASSERT(vm != NULL, "Insufficient Memory");
+
+ vm_init_fields(vm, shape);
+
+ vm->kvm_fd = open_path_or_exit(KVM_DEV_PATH, O_RDWR);
+ vm->fd = resurrected_vm_fd;
+
+ if (kvm_has_cap(KVM_CAP_BINARY_STATS_FD))
+ vm->stats.fd = vm_get_stats_fd(vm);
+ else
+ vm->stats.fd = -1;
+
+ vm_init_memory_properties(vm);
+
+ return vm;
+}
+
+static void do_phase2(void)
+{
+ int retrieved_vm_fd, retrieved_gmem_fd, dev_luo_fd, session_fd;
+ struct vm_shape shape = VM_SHAPE_DEFAULT;
+ const uint64_t gpa = SZ_4G;
+ struct kvm_vcpu *vcpu;
+ const int slot = 1;
+ struct kvm_vm *vm;
+
+ dev_luo_fd = luo_open_device();
+ TEST_ASSERT(dev_luo_fd >= 0, "Failed to open /dev/liveupdate");
+
+ session_fd = luo_retrieve_session(dev_luo_fd, SESSION_NAME);
+ TEST_ASSERT(session_fd >= 0, "Failed to retrieve LUO session");
+
+ retrieved_vm_fd = luo_session_retrieve_fd(session_fd, VM_TOKEN);
+ TEST_ASSERT(retrieved_vm_fd >= 0, "Failed to retrieve VM file descriptor");
+
+ retrieved_gmem_fd = luo_session_retrieve_fd(session_fd, GMEM_TOKEN);
+ TEST_ASSERT(retrieved_gmem_fd >= 0, "Failed to retrieve guest_memfd file descriptor");
+
+ vm = vm_create_from_fd(retrieved_vm_fd, shape);
+
+ u64 nr_pages = 2048; /* 8MB is plenty for slot0 pages */
+
+ vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, 0);
+ kvm_vm_elf_load(vm, program_invocation_name);
+
+ for (int i = 0; i < NR_MEM_REGIONS; i++)
+ vm->memslots[i] = 0;
+
+ struct userspace_mem_region *slot0 = memslot2region(vm, 0);
+
+ ucall_init(vm, slot0->region.guest_phys_addr + slot0->region.memory_size);
+
+ vm_set_user_memory_region2(vm, slot, KVM_MEM_GUEST_MEMFD, gpa, GMEM_SIZE, NULL,
+ retrieved_gmem_fd, 0);
+
+ for (size_t i = 0; i < GMEM_SIZE; i += page_size)
+ virt_pg_map(vm, gpa + i, gpa + i);
+
+ vcpu = vm_vcpu_add(vm, 0, guest_code_phase2);
+ kvm_arch_vm_finalize_vcpus(vm);
+
+ vcpu_args_set(vcpu, 3, gpa, GMEM_SIZE, DATA_SIZE);
+
+ printf("Resuming / Running VM in Phase 2...\n");
+ vcpu_run(vcpu);
+ TEST_ASSERT_EQ(get_ucall(vcpu, NULL), UCALL_DONE);
+
+ printf("\nSUCCESS: Phase 2 Complete! All 5MB complex data verified intact!\n");
+
+ luo_session_finish(session_fd);
+ close(session_fd);
+ close(dev_luo_fd);
+ /* This will also close the vm_fd */
+ kvm_vm_free(vm);
+ close(retrieved_gmem_fd);
+}
+
+int main(int argc, char *argv[])
+{
+ bool phase2 = false;
+
+ TEST_REQUIRE(kvm_has_cap(KVM_CAP_GUEST_MEMFD));
+ page_size = getpagesize();
+
+ for (int i = 1; i < argc; i++) {
+ if (strcmp(argv[i], "--phase2") == 0)
+ phase2 = true;
+ }
+
+ if (phase2)
+ do_phase2();
+ else
+ do_phase1();
+
+ return 0;
+}
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply related
* [RFC PATCH v2 09/10] selftests: kvm: Split ____vm_create() to expose init helpers
From: Tarun Sahu @ 2026-06-05 17:08 UTC (permalink / raw)
To: Jonathan Corbet, vannapurve, Tarun Sahu, fvdl, Pasha Tatashin,
Shuah Khan, sagis, aneesh.kumar, skhawaja, vipinsh, ackerleytng,
Pratyush Yadav, david, dmatlack, mark.rutland, Paolo Bonzini,
Mike Rapoport, Alexander Graf, seanjc, axelrasmussen
Cc: linux-kselftest, kexec, linux-kernel, linux-doc, kvm, linux-mm
In-Reply-To: <cover.1780676742.git.tarunsahu@google.com>
Refactor `____vm_create()` in the KVM selftest library to extract its
initialization steps into separate, reusable internal helpers.
Introduce `vm_init_fields()` and `vm_init_memory_properties()`. This
allows advanced test setups to perform targeted VM fields or memory
property initializations independently, which is required by upcoming
test cases that restore preserved VMs. No functional changes are
introduced for the existing tests.
Signed-off-by: Tarun Sahu <tarunsahu@google.com>
---
.../testing/selftests/kvm/include/kvm_util.h | 2 ++
tools/testing/selftests/kvm/lib/kvm_util.c | 26 +++++++++++++------
2 files changed, 20 insertions(+), 8 deletions(-)
diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h
index 2ecaaa0e9965..d10cd25d0658 100644
--- a/tools/testing/selftests/kvm/include/kvm_util.h
+++ b/tools/testing/selftests/kvm/include/kvm_util.h
@@ -471,6 +471,8 @@ const char *vm_guest_mode_string(u32 i);
void kvm_vm_free(struct kvm_vm *vmp);
void kvm_vm_restart(struct kvm_vm *vmp);
+void vm_init_fields(struct kvm_vm *vm, struct vm_shape shape);
+void vm_init_memory_properties(struct kvm_vm *vm);
void kvm_vm_release(struct kvm_vm *vmp);
void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename);
int kvm_memfd_alloc(size_t size, bool hugepages);
diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c
index e08967ef7b7b..d3e6508e9863 100644
--- a/tools/testing/selftests/kvm/lib/kvm_util.c
+++ b/tools/testing/selftests/kvm/lib/kvm_util.c
@@ -276,13 +276,8 @@ __weak void vm_populate_gva_bitmap(struct kvm_vm *vm)
(1ULL << (vm->va_bits - 1)) >> vm->page_shift);
}
-struct kvm_vm *____vm_create(struct vm_shape shape)
+void vm_init_fields(struct kvm_vm *vm, struct vm_shape shape)
{
- struct kvm_vm *vm;
-
- vm = calloc(1, sizeof(*vm));
- TEST_ASSERT(vm != NULL, "Insufficient Memory");
-
INIT_LIST_HEAD(&vm->vcpus);
vm->regions.gpa_tree = RB_ROOT;
vm->regions.hva_tree = RB_ROOT;
@@ -380,9 +375,10 @@ struct kvm_vm *____vm_create(struct vm_shape shape)
if (vm->pa_bits != 40)
vm->type = KVM_VM_TYPE_ARM_IPA_SIZE(vm->pa_bits);
#endif
+}
- vm_open(vm);
-
+void vm_init_memory_properties(struct kvm_vm *vm)
+{
/* Limit to VA-bit canonical virtual addresses. */
vm->vpages_valid = sparsebit_alloc();
vm_populate_gva_bitmap(vm);
@@ -392,6 +388,20 @@ struct kvm_vm *____vm_create(struct vm_shape shape)
/* Allocate and setup memory for guest. */
vm->vpages_mapped = sparsebit_alloc();
+}
+
+struct kvm_vm *____vm_create(struct vm_shape shape)
+{
+ struct kvm_vm *vm;
+
+ vm = calloc(1, sizeof(*vm));
+ TEST_ASSERT(vm != NULL, "Insufficient Memory");
+
+ vm_init_fields(vm, shape);
+
+ vm_open(vm);
+
+ vm_init_memory_properties(vm);
return vm;
}
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply related
* [RFC PATCH v2 08/10] docs: add documentation for guest_memfd preservation via LUO
From: Tarun Sahu @ 2026-06-05 17:08 UTC (permalink / raw)
To: Jonathan Corbet, vannapurve, Tarun Sahu, fvdl, Pasha Tatashin,
Shuah Khan, sagis, aneesh.kumar, skhawaja, vipinsh, ackerleytng,
Pratyush Yadav, david, dmatlack, mark.rutland, Paolo Bonzini,
Mike Rapoport, Alexander Graf, seanjc, axelrasmussen
Cc: linux-kselftest, kexec, linux-kernel, linux-doc, kvm, linux-mm
In-Reply-To: <cover.1780676742.git.tarunsahu@google.com>
Add the documentation under the "Preserving file descriptors" section
of LUO's documentation.
Signed-off-by: Tarun Sahu <tarunsahu@google.com>
---
Documentation/core-api/liveupdate.rst | 1 +
Documentation/liveupdate/vmm.rst | 103 ++++++++++++++++++++++++++
MAINTAINERS | 1 +
3 files changed, 105 insertions(+)
create mode 100644 Documentation/liveupdate/vmm.rst
diff --git a/Documentation/core-api/liveupdate.rst b/Documentation/core-api/liveupdate.rst
index 5a292d0f3706..bac58a363151 100644
--- a/Documentation/core-api/liveupdate.rst
+++ b/Documentation/core-api/liveupdate.rst
@@ -34,6 +34,7 @@ The following types of file descriptors can be preserved
:maxdepth: 1
../mm/memfd_preservation
+ ../liveupdate/vmm
Public API
==========
diff --git a/Documentation/liveupdate/vmm.rst b/Documentation/liveupdate/vmm.rst
new file mode 100644
index 000000000000..0cd487a0e1a6
--- /dev/null
+++ b/Documentation/liveupdate/vmm.rst
@@ -0,0 +1,103 @@
+.. SPDX-License-Identifier: GPL-2.0-or-later
+
+=============================
+VM & Guest_Memfd Preservation
+=============================
+
+.. kernel-doc:: virt/kvm/kvm_luo.c
+ :doc: KVM VM Preservation via LUO
+
+.. kernel-doc:: virt/kvm/guest_memfd_luo.c
+ :doc: Guest_Memfd Preservation via LUO
+
+VMM Instructions
+================
+
+This section describes the requirements, scope, conditions, and
+ordering constraints that a Virtual Machine Monitor (VMM) must adhere
+to for successful preservation and retrieval of guest_memfd files
+across a Live Update Orchestrator (LUO) sequence.
+
+Scope and Limitations
+---------------------
+
+At this stage, the scope of guest_memfd preservation is restricted to:
+
+1. **Fully Shared guest_memfd**:
+ This time only fully shared guest_memfd supported. Any system that
+ supports coco vm (which uses private guest_memfd), will not support
+ the preservation.
+
+2. **Standard Page Size**:
+ Only guest_memfd backed by standard page size (``PAGE_SIZE``,
+ order-0) pages is supported. Large/huge page backing (e.g.,
+ hugetlb guest_memfd) is not supported.
+
+Any Virtual Machine (VM) whose memory is fully backed by such
+guest_memfd files can be preserved across live update.
+
+VMM Actions and Conditions during Live Update
+---------------------------------------------
+
+During the live update sequence, the kernel introduces a *freezing*
+phase for the guest_memfd inode. Freezing prevents any modifications to
+the guest_memfd page cache. Specifically, once a guest_memfd mapping is
+frozen:
+
+- Any subsequent ``fallocate`` calls on the guest_memfd file descriptor
+ will fail and return ``-EPERM``.
+- Any new page faults (guest-side or host-userspace-side) that require
+ folio allocation will fail and return ``-EPERM``.
+
+To prevent vCPUs or VMM helper threads from failing due to these
+``-EPERM`` errors, the VMM must implement one of the following
+strategies:
+
+1. **Pause the VM (Recommended)**:
+ The VMM should pause/suspend all vCPUs before invoking the
+ preservation or freezing of the VM and guest_memfd files. This
+ ensures no new page faults or memory accesses can occur while the
+ guest_memfd is frozen.
+
+2. **Handle Fault Failures**:
+ If the VM is not paused, the VMM must be prepared to handle VM
+ exits or user page fault errors resulting from the ``-EPERM``
+ failures. The VMM must take appropriate action, such as
+ immediately pausing the VM, or aborting the live update sequence
+ (by tearing down or unpreserving the live update session).
+
+Preservation and Retrieval Ordering
+-----------------------------------
+
+Preservation Order
+~~~~~~~~~~~~~~~~~~
+
+There is no strict ordering requirement for initiating the
+preservation of the KVM VM file and the guest_memfd files; they are
+preserved independently. If kexec is triggered with guest_memfd
+preservation without preserving the vm file, kexec will fail.
+
+Retrieval Order
+~~~~~~~~~~~~~~~
+
+Similarly, there is no strict ordering required for retrieving the VM
+and guest_memfd files. Any file can be retrieved at any order.
+
+If guest_memfd file is retrieved and VM file is not retrieved, and
+luo_finish is called, then vm_file will be lost and guest_memfd file
+will be hanging around.
+
+VM & Guest_Memfd Preservation ABI
+=================================
+
+.. kernel-doc:: include/linux/kho/abi/kvm.h
+ :doc: DOC: guest_memfd Live Update ABI
+
+.. kernel-doc:: include/linux/kho/abi/kvm.h
+ :internal:
+
+See Also
+========
+
+- :doc:`/core-api/liveupdate`
+- :doc:`/userspace-api/liveupdate`
diff --git a/MAINTAINERS b/MAINTAINERS
index 16cba790a84d..ca459d032712 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -14418,6 +14418,7 @@ L: kexec@lists.infradead.org
L: kvm@vger.kernel.org
S: Maintained
T: git git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git
+F: Documentation/liveupdate/vmm.rst
F: virt/kvm/guest_memfd_luo.c
F: virt/kvm/kvm_luo.c
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply related
* [RFC PATCH v2 07/10] kvm: guest_memfd_luo: add support for guest_memfd preservation
From: Tarun Sahu @ 2026-06-05 17:08 UTC (permalink / raw)
To: Jonathan Corbet, vannapurve, Tarun Sahu, fvdl, Pasha Tatashin,
Shuah Khan, sagis, aneesh.kumar, skhawaja, vipinsh, ackerleytng,
Pratyush Yadav, david, dmatlack, mark.rutland, Paolo Bonzini,
Mike Rapoport, Alexander Graf, seanjc, axelrasmussen
Cc: linux-kselftest, kexec, linux-kernel, linux-doc, kvm, linux-mm
In-Reply-To: <cover.1780676742.git.tarunsahu@google.com>
This patch sets up the basic infrastructure to preserve the guest_memfd.
Currently this supports only fully shared guest_memfd and backed by
PAGE_SIZE pages.
It registers a new LUO file handler for guest_memfd files to serialize
and deserialize guest memory. This allows preserving guest memory backed
by guest_memfd across updates, ensuring that guest instances can be
resumed seamlessly without losing their memory contents.
Preservation is straight forward. It walks through the folios and
serialize them.
There is kvm_gmem_freeze call on preserve which freeze the guest_memfd
inode. It avoids any changes to inode mapping with fallocate calls or
any new fault allocation (fails) on or after preservation. No need to check
this during the page fault as preservation is only supported for
pre-faulted/pre-allocated guest_memfd.
While retrieving the guest_memfd, it requires the struct kvm to create
new guest_memfd. So it first get the vm_file from the same session using
the token passed during the preservation. And use it to get
vm_file->kvm.
This change also update the MAINTAINERS list.
Signed-off-by: Tarun Sahu <tarunsahu@google.com>
---
MAINTAINERS | 1 +
include/linux/kho/abi/kvm.h | 79 +++++-
virt/kvm/Makefile.kvm | 2 +-
virt/kvm/guest_memfd_luo.c | 485 ++++++++++++++++++++++++++++++++++++
virt/kvm/kvm_main.c | 7 +
virt/kvm/kvm_mm.h | 4 +
6 files changed, 571 insertions(+), 7 deletions(-)
create mode 100644 virt/kvm/guest_memfd_luo.c
diff --git a/MAINTAINERS b/MAINTAINERS
index 9bfc3c1f6676..16cba790a84d 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -14418,6 +14418,7 @@ L: kexec@lists.infradead.org
L: kvm@vger.kernel.org
S: Maintained
T: git git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git
+F: virt/kvm/guest_memfd_luo.c
F: virt/kvm/kvm_luo.c
KVM PARAVIRT (KVM/paravirt)
diff --git a/include/linux/kho/abi/kvm.h b/include/linux/kho/abi/kvm.h
index 718db68a541a..42074d76e04a 100644
--- a/include/linux/kho/abi/kvm.h
+++ b/include/linux/kho/abi/kvm.h
@@ -9,20 +9,23 @@
#define _LINUX_KHO_ABI_KVM_H
#include <linux/types.h>
+#include <linux/bits.h>
#include <linux/kho/abi/kexec_handover.h>
/**
- * DOC: KVM Live Update ABI
+ * DOC: KVM and guest_memfd Live Update ABI
*
- * KVM uses the ABI defined below for preserving its state
+ * KVM and guest_memfd use the ABI defined below for preserving their states
* across a kexec reboot using the LUO.
*
- * The state is serialized into a packed structure `struct kvm_luo_ser`
- * which is handed over to the next kernel via the KHO mechanism.
+ * The state is serialized into packed structures (struct kvm_luo_ser and
+ * struct guest_memfd_luo_ser) which are handed over to the next kernel via
+ * the KHO mechanism.
*
- * This interface is a contract. Any modification to the structure layout
+ * This interface is a contract. Any modification to the structure layouts
* constitutes a breaking change. Such changes require incrementing the
- * version number in the KVM_LUO_FH_COMPATIBLE compatibility string.
+ * version number in the KVM_LUO_FH_COMPATIBLE or
+ * GUEST_MEMFD_LUO_FH_COMPATIBLE compatibility strings.
*/
/**
@@ -36,4 +39,68 @@ struct kvm_luo_ser {
/* The compatibility string for KVM VM file handler */
#define KVM_LUO_FH_COMPATIBLE "kvm_vm_luo_v1"
+/**
+ * struct guest_memfd_luo_folio_ser - Serialization layout for a single folio in guest_memfd.
+ * @pfn: Page Frame Number of the folio.
+ * @index: Page offset of the folio within the file.
+ * @flags: State flags associated with the folio.
+ */
+struct guest_memfd_luo_folio_ser {
+ u64 pfn:52;
+ u64 flags:12;
+ u64 index;
+} __packed;
+
+/**
+ * GUEST_MEMFD_LUO_FOLIO_UPTODATE - The folio is up-to-date.
+ *
+ * This flag is per folio to check if the folio is uptodate.
+ */
+#define GUEST_MEMFD_LUO_FOLIO_UPTODATE BIT(0)
+
+
+/**
+ * GUEST_MEMFD_LUO_FLAG_MMAP - The guest_memfd supports mmap.
+ *
+ * This flag indicates that the guest_memfd supports host-side mmap.
+ */
+#define GUEST_MEMFD_LUO_FLAG_MMAP BIT(0)
+
+/**
+ * GUEST_MEMFD_LUO_FLAG_INIT_SHARED - Initialize memory as shared.
+ *
+ * This flag indicates that the guest_memfd has been initialized as shared
+ * memory.
+ */
+#define GUEST_MEMFD_LUO_FLAG_INIT_SHARED BIT(1)
+
+/**
+ * GUEST_MEMFD_LUO_SUPPORTED_FLAGS - Supported guest_memfd LUO flags mask.
+ *
+ * A mask of all guest_memfd preservation flags supported by this version
+ * of the KVM LUO ABI.
+ */
+#define GUEST_MEMFD_LUO_SUPPORTED_FLAGS (GUEST_MEMFD_LUO_FLAG_MMAP | \
+ GUEST_MEMFD_LUO_FLAG_INIT_SHARED)
+
+/**
+ * struct guest_memfd_luo_ser - Main serialization structure for guest_memfd.
+ * @size: The size of the file in bytes.
+ * @flags: File-level flags.
+ * @nr_folios: Number of folios in the folios array.
+ * @vm_token: Token of the associated KVM VM instance.
+ * @folios: KHO vmalloc descriptor pointing to the array of
+ * struct guest_memfd_luo_folio_ser.
+ */
+struct guest_memfd_luo_ser {
+ u64 size;
+ u64 flags;
+ u64 nr_folios;
+ u64 vm_token;
+ struct kho_vmalloc folios;
+} __packed;
+
+/* The compatibility string for GUEST_MEMFD file handler */
+#define GUEST_MEMFD_LUO_FH_COMPATIBLE "guest_memfd_luo_v1"
+
#endif /* _LINUX_KHO_ABI_KVM_H */
diff --git a/virt/kvm/Makefile.kvm b/virt/kvm/Makefile.kvm
index c1a962159264..d30fca094c42 100644
--- a/virt/kvm/Makefile.kvm
+++ b/virt/kvm/Makefile.kvm
@@ -13,4 +13,4 @@ kvm-$(CONFIG_HAVE_KVM_IRQ_ROUTING) += $(KVM)/irqchip.o
kvm-$(CONFIG_HAVE_KVM_DIRTY_RING) += $(KVM)/dirty_ring.o
kvm-$(CONFIG_HAVE_KVM_PFNCACHE) += $(KVM)/pfncache.o
kvm-$(CONFIG_KVM_GUEST_MEMFD) += $(KVM)/guest_memfd.o
-kvm-$(CONFIG_LIVEUPDATE_GUEST_MEMFD) += $(KVM)/kvm_luo.o
+kvm-$(CONFIG_LIVEUPDATE_GUEST_MEMFD) += $(KVM)/guest_memfd_luo.o $(KVM)/kvm_luo.o
diff --git a/virt/kvm/guest_memfd_luo.c b/virt/kvm/guest_memfd_luo.c
new file mode 100644
index 000000000000..d466f889c9aa
--- /dev/null
+++ b/virt/kvm/guest_memfd_luo.c
@@ -0,0 +1,485 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (c) 2026, Google LLC.
+ * Tarun Sahu <tarunsahu@google.com>
+ *
+ * Guestmemfd Preservation for Live Update Orchestrator (LUO)
+ */
+
+/**
+ * DOC: Guestmemfd Preservation via LUO
+ *
+ * Overview
+ * ========
+ *
+ * Guest memory file descriptors (guest_memfd) can be preserved over a kexec
+ * reboot using the Live Update Orchestrator (LUO) file preservation. This
+ * allows userspace to preserve VM memory across kexec reboots.
+ *
+ * The preservation is not intended to be transparent. Only select properties
+ * of the guest_memfd are preserved, while others are reset to default.
+ *
+ * Preserved Properties
+ * ====================
+ *
+ * The following properties of guest_memfd are preserved across kexec:
+ *
+ * File Size
+ * The size of the file is preserved.
+ *
+ * File Contents
+ * All folios present in the page cache are preserved.
+ *
+ * File-level Flags
+ * The file-level flags (such as MMAP support and INIT_SHARED default mapping)
+ * are preserved.
+ *
+ * Non-Preserved Properties
+ * ========================
+ *
+ * NUMA Memory Policy
+ * NUMA memory policies associated with the guest_memfd are not preserved.
+ */
+#include <linux/liveupdate.h>
+#include <linux/kvm_host.h>
+#include <linux/pagemap.h>
+#include <linux/file.h>
+#include <linux/err.h>
+#include <linux/anon_inodes.h>
+#include <linux/magic.h>
+#include <linux/kexec_handover.h>
+#include <linux/kho/abi/kexec_handover.h>
+#include <linux/kho/abi/kvm.h>
+#include "guest_memfd.h"
+
+static int kvm_gmem_luo_walk_folios(struct address_space *mapping,
+ pgoff_t end_index, struct guest_memfd_luo_folio_ser *folios_ser,
+ u64 *out_count)
+{
+ struct folio_batch fbatch;
+ pgoff_t index = 0;
+ u64 count = 0;
+ int err = 0;
+
+ folio_batch_init(&fbatch);
+ while (index < end_index) {
+ unsigned int nr, i;
+
+ nr = filemap_get_folios(mapping, &index, end_index - 1, &fbatch);
+ if (nr == 0)
+ break;
+
+ for (i = 0; i < nr; i++) {
+ struct folio *folio = fbatch.folios[i];
+
+ if (folios_ser) {
+ if (folio_test_hwpoison(folio)) {
+ err = -EHWPOISON;
+ folio_batch_release(&fbatch);
+ goto out;
+ }
+ err = kho_preserve_folio(folio);
+ if (err) {
+ folio_batch_release(&fbatch);
+ goto out;
+ }
+
+ folios_ser[count].pfn = folio_pfn(folio);
+ folios_ser[count].index = folio->index;
+ folios_ser[count].flags = folio_test_uptodate(folio) ?
+ GUEST_MEMFD_LUO_FOLIO_UPTODATE : 0;
+ }
+ count++;
+ }
+ folio_batch_release(&fbatch);
+ cond_resched();
+ }
+
+out:
+ *out_count = count;
+ return err;
+}
+
+static bool kvm_gmem_luo_can_preserve(struct liveupdate_file_handler *handler, struct file *file)
+{
+ struct inode *inode = file_inode(file);
+ struct gmem_file *gmem_file = file->private_data;
+ struct kvm *kvm = gmem_file->kvm;
+
+ if (inode->i_sb->s_magic != GUEST_MEMFD_MAGIC)
+ return 0;
+
+ if (kvm_arch_has_private_mem(kvm))
+ return 0;
+
+ if (mapping_large_folio_support(inode->i_mapping))
+ return 0;
+
+ return 1;
+}
+
+static int kvm_gmem_luo_preserve(struct liveupdate_file_op_args *args)
+{
+ struct guest_memfd_luo_folio_ser *folios_ser = NULL;
+ u64 count = 0, gmem_flags, abi_flags = 0;
+ struct guest_memfd_luo_ser *ser;
+ struct address_space *mapping;
+ struct gmem_file *gmem_file;
+ struct inode *inode;
+ pgoff_t end_index;
+ struct kvm *kvm;
+ int err = 0;
+ long size;
+
+ inode = file_inode(args->file);
+ kvm_gmem_freeze(inode, true);
+
+ mapping = inode->i_mapping;
+ size = i_size_read(inode);
+ if (!size) {
+ err = -EINVAL;
+ goto err_unfreeze_inode;
+ }
+
+ if (WARN_ON_ONCE(!PAGE_ALIGNED(size))) {
+ err = -EINVAL;
+ goto err_unfreeze_inode;
+ }
+
+ gmem_file = args->file->private_data;
+ kvm = gmem_file->kvm;
+
+ gmem_flags = READ_ONCE(GMEM_I(inode)->flags);
+ if (gmem_flags & ~(GUEST_MEMFD_FLAG_MMAP | GUEST_MEMFD_FLAG_INIT_SHARED
+ | GUEST_MEMFD_F_MAPPING_FROZEN)) {
+ err = -EOPNOTSUPP;
+ goto err_unfreeze_inode;
+ }
+
+ if (gmem_flags & GUEST_MEMFD_FLAG_MMAP)
+ abi_flags |= GUEST_MEMFD_LUO_FLAG_MMAP;
+ if (gmem_flags & GUEST_MEMFD_FLAG_INIT_SHARED)
+ abi_flags |= GUEST_MEMFD_LUO_FLAG_INIT_SHARED;
+
+ end_index = size >> PAGE_SHIFT;
+
+ ser = kho_alloc_preserve(sizeof(*ser));
+ if (IS_ERR(ser)) {
+ err = PTR_ERR(ser);
+ goto err_unfreeze_inode;
+ }
+
+ /* First pass: Count the folios present in the page cache */
+ err = kvm_gmem_luo_walk_folios(mapping, end_index, NULL, &count);
+ if (err)
+ goto err_free_ser;
+
+ ser->size = size;
+ ser->flags = abi_flags;
+ ser->nr_folios = count;
+ ser->vm_token = 0; // It will be set during the kvm_gmem_luo_freeze()
+
+ if (count > 0) {
+ folios_ser = vcalloc(count, sizeof(*folios_ser));
+ if (!folios_ser) {
+ err = -ENOMEM;
+ goto err_free_ser;
+ }
+
+ /* Second pass: Fill the metadata array and preserve folios */
+ err = kvm_gmem_luo_walk_folios(mapping, end_index, folios_ser, &count);
+ if (err)
+ goto err_unpreserve_unlocked;
+
+ if (WARN_ON_ONCE(count != ser->nr_folios)) {
+ err = -EINVAL;
+ goto err_unpreserve_unlocked;
+ }
+ }
+
+ if (count > 0) {
+ err = kho_preserve_vmalloc(folios_ser, &ser->folios);
+ if (err)
+ goto err_unpreserve_unlocked;
+ }
+
+ args->serialized_data = virt_to_phys(ser);
+ args->private_data = folios_ser;
+
+ return 0;
+
+err_unpreserve_unlocked:
+ for (long i = (long)count - 1; i >= 0; i--) {
+ struct folio *folio = pfn_folio(folios_ser[i].pfn);
+
+ kho_unpreserve_folio(folio);
+ }
+ vfree(folios_ser);
+err_free_ser:
+ kho_unpreserve_free(ser);
+err_unfreeze_inode:
+ kvm_gmem_freeze(inode, false);
+ return err;
+}
+
+static int kvm_gmem_luo_freeze(struct liveupdate_file_op_args *args)
+{
+ struct guest_memfd_luo_ser *ser;
+ struct gmem_file *gmem_file;
+ struct kvm *kvm;
+ struct file *kvm_file;
+ u64 vm_token;
+ int err;
+
+ if (WARN_ON_ONCE(!args->serialized_data))
+ return -EINVAL;
+
+ ser = phys_to_virt(args->serialized_data);
+
+ gmem_file = args->file->private_data;
+ kvm = gmem_file->kvm;
+
+ /*
+ * Obtain a strong reference to kvm->vm_file to prevent the SLAB_TYPESAFE_BY_RCU
+ * file memory from being reallocated while it is being processed.
+ */
+ kvm_file = get_file_active(&kvm->vm_file);
+ if (!kvm_file)
+ return -ENOENT;
+
+ err = liveupdate_get_token_outgoing(args->session, kvm_file, &vm_token);
+ fput(kvm_file);
+ if (err)
+ return err;
+
+ ser->vm_token = vm_token;
+ return 0;
+}
+
+static void kvm_gmem_luo_discard_folios(
+ const struct guest_memfd_luo_folio_ser *folios_ser,
+ u64 nr_folios, u64 start_idx)
+{
+ long i;
+
+ for (i = start_idx; i < nr_folios; i++) {
+ struct folio *folio;
+ phys_addr_t phys;
+
+ if (!folios_ser[i].pfn)
+ continue;
+
+ phys = PFN_PHYS(folios_ser[i].pfn);
+ folio = kho_restore_folio(phys);
+ if (folio)
+ folio_put(folio);
+ }
+}
+
+static void kvm_gmem_luo_unpreserve(struct liveupdate_file_op_args *args)
+{
+ struct guest_memfd_luo_folio_ser *folios_ser = args->private_data;
+ struct guest_memfd_luo_ser *ser;
+ long i;
+
+ if (WARN_ON_ONCE(!args->serialized_data))
+ return;
+
+ ser = phys_to_virt(args->serialized_data);
+ if (!ser)
+ return;
+
+ if (ser->nr_folios > 0)
+ kho_unpreserve_vmalloc(&ser->folios);
+ for (i = ser->nr_folios - 1; i >= 0; i--) {
+ struct folio *folio;
+
+ if (!folios_ser[i].pfn)
+ continue;
+
+ folio = pfn_folio(folios_ser[i].pfn);
+ kho_unpreserve_folio(folio);
+ }
+ vfree(folios_ser);
+
+ kho_unpreserve_free(ser);
+ kvm_gmem_freeze(file_inode(args->file), false);
+}
+
+static int kvm_gmem_luo_retrieve(struct liveupdate_file_op_args *args)
+{
+ struct guest_memfd_luo_folio_ser *folios_ser = NULL;
+ struct guest_memfd_luo_ser *ser;
+ struct kvm *kvm = NULL;
+ struct file *vm_file;
+ struct inode *inode;
+ struct file *file;
+ u64 gmem_flags = 0;
+ int err = 0;
+ long i = 0;
+
+ if (!args->serialized_data)
+ return -EINVAL;
+
+ ser = phys_to_virt(args->serialized_data);
+
+ if (ser->flags & ~GUEST_MEMFD_LUO_SUPPORTED_FLAGS) {
+ err = -EOPNOTSUPP;
+ goto err_free_ser;
+ }
+
+ if (ser->flags & GUEST_MEMFD_LUO_FLAG_MMAP)
+ gmem_flags |= GUEST_MEMFD_FLAG_MMAP;
+ if (ser->flags & GUEST_MEMFD_LUO_FLAG_INIT_SHARED)
+ gmem_flags |= GUEST_MEMFD_FLAG_INIT_SHARED;
+
+ err = liveupdate_get_file_incoming(args->session, ser->vm_token, &vm_file);
+ if (err) {
+ pr_warn("gmem: provided VM FD token (%llx) on preserve is incorrect\n",
+ ser->vm_token);
+ goto err_free_ser;
+ }
+
+ if (file_is_kvm(vm_file))
+ kvm = vm_file->private_data;
+
+ /*
+ * Release the temporary reference taken by the liveupdate_get_file_incoming
+ * call. LUO still holds a reference.
+ */
+ fput(vm_file);
+
+ if (!kvm) {
+ err = -EINVAL;
+ goto err_free_ser;
+ }
+
+ file = __kvm_gmem_create_file(kvm, ser->size, gmem_flags);
+ if (IS_ERR(file)) {
+ err = PTR_ERR(file);
+ goto err_free_ser;
+ }
+
+ inode = file_inode(file);
+
+ if (ser->nr_folios) {
+ folios_ser = kho_restore_vmalloc(&ser->folios);
+ if (!folios_ser) {
+ err = -EINVAL;
+ goto err_destroy_file;
+ }
+
+ for (i = 0; i < ser->nr_folios; i++) {
+ struct folio *folio;
+ phys_addr_t phys;
+
+ if (!folios_ser[i].pfn)
+ continue;
+
+ phys = PFN_PHYS(folios_ser[i].pfn);
+ folio = kho_restore_folio(phys);
+ if (!folio) {
+ pr_err("gmem: failed to restore folio at %llx\n", phys);
+ err = -EIO;
+ goto err_put_remaining_folios;
+ }
+
+ err = filemap_add_folio(inode->i_mapping, folio, folios_ser[i].index,
+ GFP_KERNEL);
+ if (err) {
+ pr_err("gmem: failed to add folio to page cache\n");
+ folio_put(folio);
+ goto err_put_remaining_folios;
+ }
+
+ if (folios_ser[i].flags & GUEST_MEMFD_LUO_FOLIO_UPTODATE)
+ folio_mark_uptodate(folio);
+ folio_unlock(folio);
+ folio_put(folio);
+ }
+ vfree(folios_ser);
+ }
+
+ args->file = file;
+ kho_restore_free(ser);
+ return 0;
+
+err_put_remaining_folios:
+ i++;
+err_destroy_file:
+ fput(file);
+err_free_ser:
+ if (ser->nr_folios) {
+ if (!folios_ser)
+ folios_ser = kho_restore_vmalloc(&ser->folios);
+ if (folios_ser) {
+ kvm_gmem_luo_discard_folios(folios_ser, ser->nr_folios, i);
+ vfree(folios_ser);
+ }
+ }
+ kho_restore_free(ser);
+ return err;
+}
+
+static void kvm_gmem_luo_finish(struct liveupdate_file_op_args *args)
+{
+ struct guest_memfd_luo_ser *ser;
+ struct guest_memfd_luo_folio_ser *folios_ser;
+
+ /* Nothing to be done here, if retrieve_status was successful or errored,
+ * Cleanup is taken care of in retrieval call.
+ */
+ if (args->retrieve_status)
+ return;
+
+ if (!args->serialized_data)
+ return;
+
+ ser = phys_to_virt(args->serialized_data);
+ if (!ser)
+ return;
+
+ if (ser->nr_folios) {
+ folios_ser = kho_restore_vmalloc(&ser->folios);
+ if (folios_ser) {
+ kvm_gmem_luo_discard_folios(folios_ser, ser->nr_folios, 0);
+ vfree(folios_ser);
+ }
+ }
+
+ kho_restore_free(ser);
+}
+
+static const struct liveupdate_file_ops kvm_gmem_luo_file_ops = {
+ .can_preserve = kvm_gmem_luo_can_preserve,
+ .preserve = kvm_gmem_luo_preserve,
+ .freeze = kvm_gmem_luo_freeze,
+ .retrieve = kvm_gmem_luo_retrieve,
+ .unpreserve = kvm_gmem_luo_unpreserve,
+ .finish = kvm_gmem_luo_finish,
+ .owner = THIS_MODULE,
+};
+
+static struct liveupdate_file_handler kvm_gmem_luo_handler = {
+ .ops = &kvm_gmem_luo_file_ops,
+ .compatible = GUEST_MEMFD_LUO_FH_COMPATIBLE,
+};
+
+int kvm_gmem_luo_init(void)
+{
+ int err = liveupdate_register_file_handler(&kvm_gmem_luo_handler);
+
+ if (err && err != -EOPNOTSUPP) {
+ pr_err("Could not register luo filesystem handler: %pe\n", ERR_PTR(err));
+ return err;
+ }
+
+ return 0;
+}
+
+void kvm_gmem_luo_exit(void)
+{
+ liveupdate_unregister_file_handler(&kvm_gmem_luo_handler);
+}
+
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index c70346906a89..501a5d048418 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -6580,6 +6580,10 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module)
if (r)
goto err_luo;
+ r = kvm_gmem_luo_init();
+ if (r)
+ goto err_gmem_luo;
+
/*
* Registration _must_ be the very last thing done, as this exposes
* /dev/kvm to userspace, i.e. all infrastructure must be setup!
@@ -6593,6 +6597,8 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module)
return 0;
err_register:
+ kvm_gmem_luo_exit();
+err_gmem_luo:
kvm_luo_exit();
err_luo:
kvm_uninit_virtualization();
@@ -6624,6 +6630,7 @@ void kvm_exit(void)
*/
misc_deregister(&kvm_dev);
+ kvm_gmem_luo_exit();
kvm_luo_exit();
kvm_uninit_virtualization();
diff --git a/virt/kvm/kvm_mm.h b/virt/kvm/kvm_mm.h
index 118edc47df83..d8ccb68e7e9b 100644
--- a/virt/kvm/kvm_mm.h
+++ b/virt/kvm/kvm_mm.h
@@ -100,9 +100,13 @@ static inline void kvm_gmem_unbind(struct kvm_memory_slot *slot)
#ifdef CONFIG_LIVEUPDATE_GUEST_MEMFD
int kvm_luo_init(void);
void kvm_luo_exit(void);
+int kvm_gmem_luo_init(void);
+void kvm_gmem_luo_exit(void);
#else
static inline int kvm_luo_init(void) { return 0; }
static inline void kvm_luo_exit(void) {}
+static inline int kvm_gmem_luo_init(void) { return 0; }
+static inline void kvm_gmem_luo_exit(void) {}
#endif /* CONFIG_LIVEUPDATE_GUEST_MEMFD */
#endif /* __KVM_MM_H__ */
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply related
* [RFC PATCH v2 06/10] kvm: guest_memfd: Add support for freezing and unfreezing mappings
From: Tarun Sahu @ 2026-06-05 17:08 UTC (permalink / raw)
To: Jonathan Corbet, vannapurve, Tarun Sahu, fvdl, Pasha Tatashin,
Shuah Khan, sagis, aneesh.kumar, skhawaja, vipinsh, ackerleytng,
Pratyush Yadav, david, dmatlack, mark.rutland, Paolo Bonzini,
Mike Rapoport, Alexander Graf, seanjc, axelrasmussen
Cc: linux-kselftest, kexec, linux-kernel, linux-doc, kvm, linux-mm
In-Reply-To: <cover.1780676742.git.tarunsahu@google.com>
This patch introduces the freeze on gmem_inode which prevents
the fallocate call and any new page fault allocation. This will avoid
gmem file modification when it is being preserved
Used srcu lock to synchronise the freeze call, where write blocks
until all the reads are free. And reads are re-entrant.
Incase fault fails, It return -EPERM and VM_EXIT to userspace. userspace
must handle this properly as every new fault will fail.
Signed-off-by: Tarun Sahu <tarunsahu@google.com>
---
virt/kvm/guest_memfd.c | 117 +++++++++++++++++++++++++++++++++++++----
virt/kvm/guest_memfd.h | 5 ++
2 files changed, 111 insertions(+), 11 deletions(-)
diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c
index 6740ae2bf948..b94639cdf312 100644
--- a/virt/kvm/guest_memfd.c
+++ b/virt/kvm/guest_memfd.c
@@ -7,11 +7,13 @@
#include <linux/mempolicy.h>
#include <linux/pseudo_fs.h>
#include <linux/pagemap.h>
+#include <linux/srcu.h>
#include "guest_memfd.h"
#include "kvm_mm.h"
static struct vfsmount *kvm_gmem_mnt;
+static struct srcu_struct kvm_gmem_freeze_srcu;
#define kvm_gmem_for_each_file(f, inode) \
@@ -96,6 +98,7 @@ static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index)
/* TODO: Support huge pages. */
struct mempolicy *policy;
struct folio *folio;
+ int idx;
/*
* Fast-path: See if folio is already present in mapping to avoid
@@ -105,12 +108,20 @@ static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index)
if (!IS_ERR(folio))
return folio;
+ idx = srcu_read_lock(&kvm_gmem_freeze_srcu);
+ if (kvm_gmem_is_frozen(inode)) {
+ srcu_read_unlock(&kvm_gmem_freeze_srcu, idx);
+ return ERR_PTR(-EPERM);
+ }
+
policy = mpol_shared_policy_lookup(&GMEM_I(inode)->policy, index);
folio = __filemap_get_folio_mpol(inode->i_mapping, index,
FGP_LOCK | FGP_CREAT,
mapping_gfp_mask(inode->i_mapping), policy);
mpol_cond_put(policy);
+ srcu_read_unlock(&kvm_gmem_freeze_srcu, idx);
+
/*
* External interfaces like kvm_gmem_get_pfn() support dealing
* with hugepages to a degree, but internally, guest_memfd currently
@@ -273,16 +284,30 @@ static long kvm_gmem_allocate(struct inode *inode, loff_t offset, loff_t len)
static long kvm_gmem_fallocate(struct file *file, int mode, loff_t offset,
loff_t len)
{
+ struct inode *inode = file_inode(file);
int ret;
+ int idx;
- if (!(mode & FALLOC_FL_KEEP_SIZE))
- return -EOPNOTSUPP;
+ idx = srcu_read_lock(&kvm_gmem_freeze_srcu);
+ if (kvm_gmem_is_frozen(inode)) {
+ srcu_read_unlock(&kvm_gmem_freeze_srcu, idx);
+ return -EPERM;
+ }
- if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
- return -EOPNOTSUPP;
+ if (!(mode & FALLOC_FL_KEEP_SIZE)) {
+ ret = -EOPNOTSUPP;
+ goto out;
+ }
- if (!PAGE_ALIGNED(offset) || !PAGE_ALIGNED(len))
- return -EINVAL;
+ if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) {
+ ret = -EOPNOTSUPP;
+ goto out;
+ }
+
+ if (!PAGE_ALIGNED(offset) || !PAGE_ALIGNED(len)) {
+ ret = -EINVAL;
+ goto out;
+ }
if (mode & FALLOC_FL_PUNCH_HOLE)
ret = kvm_gmem_punch_hole(file_inode(file), offset, len);
@@ -291,6 +316,9 @@ static long kvm_gmem_fallocate(struct file *file, int mode, loff_t offset,
if (!ret)
file_modified(file);
+
+out:
+ srcu_read_unlock(&kvm_gmem_freeze_srcu, idx);
return ret;
}
@@ -944,7 +972,9 @@ static void kvm_gmem_destroy_inode(struct inode *inode)
static void kvm_gmem_free_inode(struct inode *inode)
{
- kmem_cache_free(kvm_gmem_inode_cachep, GMEM_I(inode));
+ struct gmem_inode *gi = GMEM_I(inode);
+
+ kmem_cache_free(kvm_gmem_inode_cachep, gi);
}
static const struct super_operations kvm_gmem_super_operations = {
@@ -1001,12 +1031,21 @@ int kvm_gmem_init(struct module *module)
if (!kvm_gmem_inode_cachep)
return -ENOMEM;
+ ret = init_srcu_struct(&kvm_gmem_freeze_srcu);
+ if (ret)
+ goto err_cache;
+
ret = kvm_gmem_init_mount();
- if (ret) {
- kmem_cache_destroy(kvm_gmem_inode_cachep);
- return ret;
- }
+ if (ret)
+ goto err_srcu;
+
return 0;
+
+err_srcu:
+ cleanup_srcu_struct(&kvm_gmem_freeze_srcu);
+err_cache:
+ kmem_cache_destroy(kvm_gmem_inode_cachep);
+ return ret;
}
void kvm_gmem_exit(void)
@@ -1014,5 +1053,61 @@ void kvm_gmem_exit(void)
kern_unmount(kvm_gmem_mnt);
kvm_gmem_mnt = NULL;
rcu_barrier();
+ cleanup_srcu_struct(&kvm_gmem_freeze_srcu);
kmem_cache_destroy(kvm_gmem_inode_cachep);
}
+
+/**
+ * kvm_gmem_freeze - Freeze or unfreeze a guest_memfd inode mapping.
+ * @inode: The guest_memfd inode.
+ * @freeze: True to freeze, false to unfreeze.
+ *
+ * This API is used strictly during the live update / preservation transition
+ * window to prevent host userspace and guest-side faults from making any
+ * mapping modifications (such as fallocate or page fault allocation)
+ * to the guest_memfd page cache.
+ *
+ * Synchronization Strategy (Sleepable RCU):
+ * To avoid high-contention VFS locks (like inode_lock or
+ * filemap_invalidate_lock) on the vCPU page fault hot paths, this subsystem
+ * implements a lightweight, system-wide Sleepable RCU (SRCU) mechanism
+ * (`kvm_gmem_freeze_srcu`):
+ *
+ * Global vs. Per-Inode SRCU
+ * ======================
+ * A single system-wide global static `srcu_struct` is used instead of a
+ * per-inode SRCU structure to completely prevent unprivileged users from
+ * exhausting the host's per-CPU memory allocator. Because
+ * `init_srcu_struct()` allocates per-CPU memory via `alloc_percpu()`, which
+ * is not accounted by memory cgroups (memcg),
+ * a per-inode SRCU structure would allow a tenant to bypass cgroup limits and
+ * trigger a system-wide Out-of-Memory (OOM) crash simply by spawning a large
+ * number of guest_memfd file descriptors (bounded only by RLIMIT_NOFILE).
+ *
+ * Flag Modification Note:
+ * Since `GUEST_MEMFD_F_MAPPING_FROZEN` is the ONLY flag in
+ * `GMEM_I(inode)->flags` that is mutated dynamically at runtime (all other
+ * flags are creation-time flags which remain strictly read-only), there is
+ * no possibility of concurrent bit-modification races. Therefore, a standard
+ * `WRITE_ONCE` is fully safe and does not require complex `cmpxchg`
+ * synchronization loops.
+ */
+void kvm_gmem_freeze(struct inode *inode, bool freeze)
+{
+ u64 flags = READ_ONCE(GMEM_I(inode)->flags);
+
+ if (freeze)
+ flags |= GUEST_MEMFD_F_MAPPING_FROZEN;
+ else
+ flags &= ~GUEST_MEMFD_F_MAPPING_FROZEN;
+
+ WRITE_ONCE(GMEM_I(inode)->flags, flags);
+
+ if (freeze)
+ synchronize_srcu(&kvm_gmem_freeze_srcu);
+}
+
+bool kvm_gmem_is_frozen(struct inode *inode)
+{
+ return READ_ONCE(GMEM_I(inode)->flags) & GUEST_MEMFD_F_MAPPING_FROZEN;
+}
diff --git a/virt/kvm/guest_memfd.h b/virt/kvm/guest_memfd.h
index c528b046dd69..028c348a1023 100644
--- a/virt/kvm/guest_memfd.h
+++ b/virt/kvm/guest_memfd.h
@@ -29,11 +29,16 @@ struct gmem_inode {
u64 flags;
};
+/* Internal kernel-only flags (must not overlap with UAPI flags) */
+#define GUEST_MEMFD_F_MAPPING_FROZEN (1ULL << 63)
+
static inline struct gmem_inode *GMEM_I(struct inode *inode)
{
return container_of(inode, struct gmem_inode, vfs_inode);
}
struct file *__kvm_gmem_create_file(struct kvm *kvm, loff_t size, u64 flags);
+void kvm_gmem_freeze(struct inode *inode, bool freeze);
+bool kvm_gmem_is_frozen(struct inode *inode);
#endif /* __KVM_GUEST_MEMFD_H__ */
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply related
* [RFC PATCH v2 05/10] kvm: guest_memfd: Move internal definitions and helper to new header
From: Tarun Sahu @ 2026-06-05 17:08 UTC (permalink / raw)
To: Jonathan Corbet, vannapurve, Tarun Sahu, fvdl, Pasha Tatashin,
Shuah Khan, sagis, aneesh.kumar, skhawaja, vipinsh, ackerleytng,
Pratyush Yadav, david, dmatlack, mark.rutland, Paolo Bonzini,
Mike Rapoport, Alexander Graf, seanjc, axelrasmussen
Cc: linux-kselftest, kexec, linux-kernel, linux-doc, kvm, linux-mm
In-Reply-To: <cover.1780676742.git.tarunsahu@google.com>
To support guest_memfd memory preservation with LUO, guest_memfd luo
code needs to access guest_memfd internals and reconstruct guest_memfd
file instances from a preserved state.
Extract gmem_file, gmem_inode, and the GMEM_I() helper from guest_memfd.c
into a new internal header virt/kvm/guest_memfd.h.
Additionally, split __kvm_gmem_create() to expose a non-static
__kvm_gmem_create_file() helper. This helper returns a struct file
instead of a file descriptor, enabling file creation and initialization
without installing it into a file descriptor table.
Signed-off-by: Tarun Sahu <tarunsahu@google.com>
---
virt/kvm/guest_memfd.c | 68 +++++++++++++++++-------------------------
virt/kvm/guest_memfd.h | 39 ++++++++++++++++++++++++
2 files changed, 67 insertions(+), 40 deletions(-)
create mode 100644 virt/kvm/guest_memfd.h
diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c
index 69c9d6d546b2..6740ae2bf948 100644
--- a/virt/kvm/guest_memfd.c
+++ b/virt/kvm/guest_memfd.c
@@ -7,38 +7,12 @@
#include <linux/mempolicy.h>
#include <linux/pseudo_fs.h>
#include <linux/pagemap.h>
+#include "guest_memfd.h"
#include "kvm_mm.h"
static struct vfsmount *kvm_gmem_mnt;
-/*
- * A guest_memfd instance can be associated multiple VMs, each with its own
- * "view" of the underlying physical memory.
- *
- * The gmem's inode is effectively the raw underlying physical storage, and is
- * used to track properties of the physical memory, while each gmem file is
- * effectively a single VM's view of that storage, and is used to track assets
- * specific to its associated VM, e.g. memslots=>gmem bindings.
- */
-struct gmem_file {
- struct kvm *kvm;
- struct xarray bindings;
- struct list_head entry;
-};
-
-struct gmem_inode {
- struct shared_policy policy;
- struct inode vfs_inode;
- struct list_head gmem_file_list;
-
- u64 flags;
-};
-
-static __always_inline struct gmem_inode *GMEM_I(struct inode *inode)
-{
- return container_of(inode, struct gmem_inode, vfs_inode);
-}
#define kvm_gmem_for_each_file(f, inode) \
list_for_each_entry(f, &GMEM_I(inode)->gmem_file_list, entry)
@@ -556,23 +530,17 @@ bool __weak kvm_arch_supports_gmem_init_shared(struct kvm *kvm)
return true;
}
-static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
+struct file *__kvm_gmem_create_file(struct kvm *kvm, loff_t size, u64 flags)
{
static const char *name = "[kvm-gmem]";
struct gmem_file *f;
struct inode *inode;
struct file *file;
- int fd, err;
-
- fd = get_unused_fd_flags(0);
- if (fd < 0)
- return fd;
+ int err;
f = kzalloc_obj(*f);
- if (!f) {
- err = -ENOMEM;
- goto err_fd;
- }
+ if (!f)
+ return ERR_PTR(-ENOMEM);
/* __fput() will take care of fops_put(). */
if (!fops_get(&kvm_gmem_fops)) {
@@ -611,8 +579,7 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
xa_init(&f->bindings);
list_add(&f->entry, &GMEM_I(inode)->gmem_file_list);
- fd_install(fd, file);
- return fd;
+ return file;
err_inode:
iput(inode);
@@ -620,7 +587,28 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
fops_put(&kvm_gmem_fops);
err_gmem:
kfree(f);
-err_fd:
+ return ERR_PTR(err);
+}
+
+static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
+{
+ struct file *file;
+ int fd, err;
+
+ fd = get_unused_fd_flags(0);
+ if (fd < 0)
+ return fd;
+
+ file = __kvm_gmem_create_file(kvm, size, flags);
+ if (IS_ERR(file)) {
+ err = PTR_ERR(file);
+ goto err_put_fd;
+ }
+
+ fd_install(fd, file);
+ return fd;
+
+err_put_fd:
put_unused_fd(fd);
return err;
}
diff --git a/virt/kvm/guest_memfd.h b/virt/kvm/guest_memfd.h
new file mode 100644
index 000000000000..c528b046dd69
--- /dev/null
+++ b/virt/kvm/guest_memfd.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef __KVM_GUEST_MEMFD_H__
+#define __KVM_GUEST_MEMFD_H__ 1
+
+#include <linux/kvm_host.h>
+#include <linux/fs.h>
+#include <linux/mempolicy.h>
+
+/*
+ * A guest_memfd instance can be associated multiple VMs, each with its own
+ * "view" of the underlying physical memory.
+ *
+ * The gmem's inode is effectively the raw underlying physical storage, and is
+ * used to track properties of the physical memory, while each gmem file is
+ * effectively a single VM's view of that storage, and is used to track assets
+ * specific to its associated VM, e.g. memslots=>gmem bindings.
+ */
+struct gmem_file {
+ struct kvm *kvm;
+ struct xarray bindings;
+ struct list_head entry;
+};
+
+struct gmem_inode {
+ struct shared_policy policy;
+ struct inode vfs_inode;
+ struct list_head gmem_file_list;
+
+ u64 flags;
+};
+
+static inline struct gmem_inode *GMEM_I(struct inode *inode)
+{
+ return container_of(inode, struct gmem_inode, vfs_inode);
+}
+
+struct file *__kvm_gmem_create_file(struct kvm *kvm, loff_t size, u64 flags);
+
+#endif /* __KVM_GUEST_MEMFD_H__ */
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply related
* [RFC PATCH v2 04/10] kvm: kvm_luo: Allow kvm preservation with LUO
From: Tarun Sahu @ 2026-06-05 17:08 UTC (permalink / raw)
To: Jonathan Corbet, vannapurve, Tarun Sahu, fvdl, Pasha Tatashin,
Shuah Khan, sagis, aneesh.kumar, skhawaja, vipinsh, ackerleytng,
Pratyush Yadav, david, dmatlack, mark.rutland, Paolo Bonzini,
Mike Rapoport, Alexander Graf, seanjc, axelrasmussen
Cc: linux-kselftest, kexec, linux-kernel, linux-doc, kvm, linux-mm
In-Reply-To: <cover.1780676742.git.tarunsahu@google.com>
Introduce KVM VM preservation support for Live Update Orchestrator.
Register an LUO file handler for KVM files to serialize and
deserialize necessary VM state across live updates. Currently, this
preserves the VM type. This implementation provides the necessary
infrastructure and dependencies for the upcoming guest_memfd
preservation support. And it can be extended to preserve more vm
state in future.
Retrieve is simply creating the kvm and populate the retrieved data.
Only catch here is there is no way to know which fd is going to be
assigned to this kvm file hence I am using atomically incremented id
for the fdname.
This change also updates the MAINTAINERS list for kvm_luo.c.
Signed-off-by: Tarun Sahu <tarunsahu@google.com>
---
My only worry is if userspace strictly depends on the fdname, that it
needs to be consistent with vm_fd. Discussed more details in the
cover letter. Would really appreciates the alternatives/other approaches.
---
MAINTAINERS | 11 +++
include/linux/kho/abi/kvm.h | 39 ++++++++
virt/kvm/Makefile.kvm | 1 +
virt/kvm/kvm_luo.c | 190 ++++++++++++++++++++++++++++++++++++
virt/kvm/kvm_main.c | 8 ++
virt/kvm/kvm_mm.h | 8 ++
6 files changed, 257 insertions(+)
create mode 100644 include/linux/kho/abi/kvm.h
create mode 100644 virt/kvm/kvm_luo.c
diff --git a/MAINTAINERS b/MAINTAINERS
index 9ec290e38b44..9bfc3c1f6676 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -14409,6 +14409,17 @@ S: Maintained
F: Documentation/devicetree/bindings/leds/backlight/kinetic,ktz8866.yaml
F: drivers/video/backlight/ktz8866.c
+KVM LIVE UPDATE
+M: Pasha Tatashin <pasha.tatashin@soleen.com>
+M: Mike Rapoport <rppt@kernel.org>
+M: Pratyush Yadav <pratyush@kernel.org>
+R: Tarun Sahu <tarunsahu@google.com>
+L: kexec@lists.infradead.org
+L: kvm@vger.kernel.org
+S: Maintained
+T: git git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git
+F: virt/kvm/kvm_luo.c
+
KVM PARAVIRT (KVM/paravirt)
M: Paolo Bonzini <pbonzini@redhat.com>
R: Vitaly Kuznetsov <vkuznets@redhat.com>
diff --git a/include/linux/kho/abi/kvm.h b/include/linux/kho/abi/kvm.h
new file mode 100644
index 000000000000..718db68a541a
--- /dev/null
+++ b/include/linux/kho/abi/kvm.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026, Google LLC.
+ * Tarun Sahu <tarunsahu@google.com>
+ *
+ * KVM Preservation ABI for Live Update Orchestrator (LUO)
+ */
+#ifndef _LINUX_KHO_ABI_KVM_H
+#define _LINUX_KHO_ABI_KVM_H
+
+#include <linux/types.h>
+#include <linux/kho/abi/kexec_handover.h>
+
+/**
+ * DOC: KVM Live Update ABI
+ *
+ * KVM uses the ABI defined below for preserving its state
+ * across a kexec reboot using the LUO.
+ *
+ * The state is serialized into a packed structure `struct kvm_luo_ser`
+ * which is handed over to the next kernel via the KHO mechanism.
+ *
+ * This interface is a contract. Any modification to the structure layout
+ * constitutes a breaking change. Such changes require incrementing the
+ * version number in the KVM_LUO_FH_COMPATIBLE compatibility string.
+ */
+
+/**
+ * struct kvm_luo_ser - Main serialization structure for a KVM VM.
+ * @type: The type of VM.
+ */
+struct kvm_luo_ser {
+ u64 type;
+} __packed;
+
+/* The compatibility string for KVM VM file handler */
+#define KVM_LUO_FH_COMPATIBLE "kvm_vm_luo_v1"
+
+#endif /* _LINUX_KHO_ABI_KVM_H */
diff --git a/virt/kvm/Makefile.kvm b/virt/kvm/Makefile.kvm
index d047d4cf58c9..c1a962159264 100644
--- a/virt/kvm/Makefile.kvm
+++ b/virt/kvm/Makefile.kvm
@@ -13,3 +13,4 @@ kvm-$(CONFIG_HAVE_KVM_IRQ_ROUTING) += $(KVM)/irqchip.o
kvm-$(CONFIG_HAVE_KVM_DIRTY_RING) += $(KVM)/dirty_ring.o
kvm-$(CONFIG_HAVE_KVM_PFNCACHE) += $(KVM)/pfncache.o
kvm-$(CONFIG_KVM_GUEST_MEMFD) += $(KVM)/guest_memfd.o
+kvm-$(CONFIG_LIVEUPDATE_GUEST_MEMFD) += $(KVM)/kvm_luo.o
diff --git a/virt/kvm/kvm_luo.c b/virt/kvm/kvm_luo.c
new file mode 100644
index 000000000000..25619f94ace5
--- /dev/null
+++ b/virt/kvm/kvm_luo.c
@@ -0,0 +1,190 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (c) 2026, Google LLC.
+ * Tarun Sahu <tarunsahu@google.com>
+ *
+ * KVM VM Preservation for Live Update Orchestrator (LUO)
+ */
+
+/**
+ * DOC: KVM VM Preservation via LUO
+ *
+ * Overview
+ * ========
+ *
+ * KVM virtual machines (VMs) can be preserved over a kexec reboot using the
+ * Live Update Orchestrator (LUO) file preservation. This allows userspace
+ * to preserve KVM VM state across kexec reboots.
+ *
+ * The preservation is not intended to be fully transparent. Only specific
+ * VM configuration and state are preserved, while other aspects of the VM
+ * must be re-established or re-configured by userspace after retrieval.
+ *
+ * Preserved Properties
+ * ====================
+ *
+ * The following properties of the KVM VM are preserved across kexec:
+ *
+ * VM Type
+ * The VM type (e.g., on x86 architecture, the vm_type parameter) is
+ * preserved.
+ *
+ * Non-Preserved Properties
+ * ========================
+ *
+ * The preservation does not cover:
+ *
+ * - vCPUs and vCPU states
+ * - Memspots / Memory slot layout (memslots)
+ * - Interrupt controllers and IRQ routings
+ * - Coalesced MMIO zones
+ * - Device bindings (VFIO/Eventfds)
+ * - Active paging or guest registers state
+ * - etc
+ */
+#include <linux/liveupdate.h>
+#include <linux/kvm_host.h>
+#include <linux/pagemap.h>
+#include <linux/file.h>
+#include <linux/err.h>
+#include <linux/anon_inodes.h>
+#include <linux/magic.h>
+#include <linux/kexec_handover.h>
+#include <linux/kho/abi/kexec_handover.h>
+#include <linux/kho/abi/kvm.h>
+#include "kvm_mm.h"
+
+static bool kvm_luo_can_preserve(struct liveupdate_file_handler *handler,
+ struct file *file)
+{
+ return file_is_kvm(file);
+}
+
+static int kvm_luo_preserve(struct liveupdate_file_op_args *args)
+{
+ struct kvm *kvm = args->file->private_data;
+ struct kvm_luo_ser *ser;
+
+ if (kvm->vm_dead || kvm->vm_bugged)
+ return -EINVAL;
+
+ ser = kho_alloc_preserve(sizeof(*ser));
+ if (IS_ERR(ser))
+ return PTR_ERR(ser);
+
+#ifdef CONFIG_X86
+ ser->type = kvm->arch.vm_type;
+#else
+ ser->type = 0;
+#endif
+
+ args->serialized_data = virt_to_phys(ser);
+
+ return 0;
+}
+
+static atomic_t restored_vm_id = ATOMIC_INIT(0);
+
+static int kvm_luo_retrieve(struct liveupdate_file_op_args *args)
+{
+ char fdname[ITOA_MAX_LEN + 1];
+ struct kvm_luo_ser *ser;
+ struct file *file;
+ struct kvm *kvm;
+ int err = 0;
+
+ if (!args->serialized_data)
+ return -EINVAL;
+
+ ser = phys_to_virt(args->serialized_data);
+
+ snprintf(fdname, sizeof(fdname), "%d",
+ atomic_inc_return(&restored_vm_id));
+
+ file = kvm_create_vm_file(ser->type, fdname);
+ if (IS_ERR(file)) {
+ err = PTR_ERR(file);
+ goto err_free_ser;
+ }
+
+ kvm = file->private_data;
+
+ args->file = file;
+ kho_restore_free(ser);
+
+ kvm_uevent_notify_vm_create(kvm);
+ return 0;
+
+err_free_ser:
+ kho_restore_free(ser);
+ return err;
+}
+
+static void kvm_luo_unpreserve(struct liveupdate_file_op_args *args)
+{
+ struct kvm_luo_ser *ser;
+
+ /*
+ * in case preservation failed, args->serialized_data will
+ * be NULL and kvm_luo_preserve takes care of cleaning up.
+ * If preserve succeeds, this condition fails and unpreserve
+ * function takes care of cleaning up.
+ */
+ if (WARN_ON_ONCE(!args->serialized_data))
+ return;
+
+ ser = phys_to_virt(args->serialized_data);
+
+ kho_unpreserve_free(ser);
+}
+
+static void kvm_luo_finish(struct liveupdate_file_op_args *args)
+{
+ struct kvm_luo_ser *ser;
+
+ /*
+ * If retrieve_status is true or set to error, nothing to do here.
+ * Already cleaned up in kvm_luo_retrieve().
+ */
+ if (args->retrieve_status)
+ return;
+
+ if (!args->serialized_data)
+ return;
+
+ ser = phys_to_virt(args->serialized_data);
+ kho_restore_free(ser);
+}
+
+static const struct liveupdate_file_ops kvm_luo_file_ops = {
+ .can_preserve = kvm_luo_can_preserve,
+ .preserve = kvm_luo_preserve,
+ .retrieve = kvm_luo_retrieve,
+ .unpreserve = kvm_luo_unpreserve,
+ .finish = kvm_luo_finish,
+ .owner = THIS_MODULE,
+};
+
+static struct liveupdate_file_handler kvm_luo_handler = {
+ .ops = &kvm_luo_file_ops,
+ .compatible = KVM_LUO_FH_COMPATIBLE,
+};
+
+int kvm_luo_init(void)
+{
+ int err = liveupdate_register_file_handler(&kvm_luo_handler);
+
+ if (err && err != -EOPNOTSUPP) {
+ pr_err("Could not register kvm_vm_luo handler: %pe\n", ERR_PTR(err));
+ return err;
+ }
+
+ return 0;
+}
+
+void kvm_luo_exit(void)
+{
+ liveupdate_unregister_file_handler(&kvm_luo_handler);
+}
+
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 65f0c5fb353e..c70346906a89 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -6576,6 +6576,10 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module)
if (r)
goto err_virt;
+ r = kvm_luo_init();
+ if (r)
+ goto err_luo;
+
/*
* Registration _must_ be the very last thing done, as this exposes
* /dev/kvm to userspace, i.e. all infrastructure must be setup!
@@ -6589,6 +6593,8 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module)
return 0;
err_register:
+ kvm_luo_exit();
+err_luo:
kvm_uninit_virtualization();
err_virt:
kvm_gmem_exit();
@@ -6618,6 +6624,8 @@ void kvm_exit(void)
*/
misc_deregister(&kvm_dev);
+ kvm_luo_exit();
+
kvm_uninit_virtualization();
debugfs_remove_recursive(kvm_debugfs_dir);
diff --git a/virt/kvm/kvm_mm.h b/virt/kvm/kvm_mm.h
index 7aa1d65c3d46..118edc47df83 100644
--- a/virt/kvm/kvm_mm.h
+++ b/virt/kvm/kvm_mm.h
@@ -97,4 +97,12 @@ static inline void kvm_gmem_unbind(struct kvm_memory_slot *slot)
}
#endif /* CONFIG_KVM_GUEST_MEMFD */
+#ifdef CONFIG_LIVEUPDATE_GUEST_MEMFD
+int kvm_luo_init(void);
+void kvm_luo_exit(void);
+#else
+static inline int kvm_luo_init(void) { return 0; }
+static inline void kvm_luo_exit(void) {}
+#endif /* CONFIG_LIVEUPDATE_GUEST_MEMFD */
+
#endif /* __KVM_MM_H__ */
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply related
* [RFC PATCH v2 03/10] kvm: Prepare core VM structs and helpers for LUO support
From: Tarun Sahu @ 2026-06-05 17:08 UTC (permalink / raw)
To: Jonathan Corbet, vannapurve, Tarun Sahu, fvdl, Pasha Tatashin,
Shuah Khan, sagis, aneesh.kumar, skhawaja, vipinsh, ackerleytng,
Pratyush Yadav, david, dmatlack, mark.rutland, Paolo Bonzini,
Mike Rapoport, Alexander Graf, seanjc, axelrasmussen
Cc: linux-kselftest, kexec, linux-kernel, linux-doc, kvm, linux-mm
In-Reply-To: <cover.1780676742.git.tarunsahu@google.com>
Introduce core infrastructure to support VM preservation with LUO.
First two changes are just refactoring, no functional change, third
change introduces a new member in struct kvm.
- Move ITOA_MAX_LEN to kvm_mm.h for reuse by upcoming kvm_luo code.
- Add a public kvm_create_vm_file() helper wrapping kvm_create_vm()
and anon_inode_getfile() to provide a unified VM file creation API.
- Track a weak reference to the backing file in struct kvm under
CONFIG_LIVEUPDATE_GUEST_MEMFD to enable reverse file resolution
without circular lifetime dependencies.
Signed-off-by: Tarun Sahu <tarunsahu@google.com>
---
include/linux/kvm_host.h | 14 +++++++
virt/kvm/kvm_main.c | 79 +++++++++++++++++++++++++++++-----------
virt/kvm/kvm_mm.h | 3 ++
3 files changed, 75 insertions(+), 21 deletions(-)
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 4c14aee1fb06..9111a28637af 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -874,6 +874,18 @@ struct kvm {
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
/* Protected by slots_lock (for writes) and RCU (for reads) */
struct xarray mem_attr_array;
+#endif
+#ifdef CONFIG_LIVEUPDATE_GUEST_MEMFD
+ /*
+ * Weak reference to the VFS file backing this KVM instance. Stored
+ * without incrementing the file refcount to prevent a circular lifetime
+ * dependency (since file->private_data already pins this struct kvm).
+ * Used exclusively to resolve the file pointer back from struct kvm.
+ *
+ * Written/cleared via rcu_assign_pointer() and read locklessly under
+ * RCU (e.g. via get_file_active() to prevent ABA races).
+ */
+ struct file *vm_file;
#endif
char stats_id[KVM_STATS_NAME_SIZE];
};
@@ -1074,7 +1086,9 @@ void kvm_get_kvm(struct kvm *kvm);
bool kvm_get_kvm_safe(struct kvm *kvm);
void kvm_put_kvm(struct kvm *kvm);
bool file_is_kvm(struct file *file);
+struct file *kvm_create_vm_file(unsigned long type, const char *fdname);
void kvm_put_kvm_no_destroy(struct kvm *kvm);
+void kvm_uevent_notify_vm_create(struct kvm *kvm);
static inline struct kvm_memslots *__kvm_memslots(struct kvm *kvm, int as_id)
{
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 89489996fbc1..65f0c5fb353e 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -67,9 +67,6 @@
#include <linux/kvm_dirty_ring.h>
-/* Worst case buffer size needed for holding an integer. */
-#define ITOA_MAX_LEN 12
-
MODULE_AUTHOR("Qumranet");
MODULE_DESCRIPTION("Kernel-based Virtual Machine (KVM) Hypervisor");
MODULE_LICENSE("GPL");
@@ -1349,6 +1346,19 @@ static int kvm_vm_release(struct inode *inode, struct file *filp)
{
struct kvm *kvm = filp->private_data;
+#ifdef CONFIG_LIVEUPDATE_GUEST_MEMFD
+ /*
+ * Clear the weak reference of the vm file.
+ * In case vm file is closed by userspace, but kvm still has
+ * other users like vCPUs, clearing this pointer ensures
+ * that we don't have a dangling pointer to a closed file.
+ *
+ * Cleared via rcu_assign_pointer() to ensure proper memory visibility
+ * for concurrent lockless readers under RCU.
+ */
+ rcu_assign_pointer(kvm->vm_file, NULL);
+#endif
+
kvm_irqfd_release(kvm);
kvm_put_kvm(kvm);
@@ -5476,11 +5486,47 @@ bool file_is_kvm(struct file *file)
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(file_is_kvm);
+struct file *kvm_create_vm_file(unsigned long type, const char *fdname)
+{
+ struct kvm *kvm = kvm_create_vm(type, fdname);
+ struct file *file;
+
+ if (IS_ERR(kvm))
+ return ERR_CAST(kvm);
+
+ file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
+ if (IS_ERR(file)) {
+ kvm_put_kvm(kvm);
+ return file;
+ }
+
+#ifdef CONFIG_LIVEUPDATE_GUEST_MEMFD
+ /*
+ * Weak reference to the file (without get_file()) to prevent a circular
+ * dependency. Safe because the file's release path clears this pointer
+ * and drops its reference to the VM.
+ *
+ * Written via rcu_assign_pointer() because the pointer can be read
+ * locklessly under RCU (e.g., in kvm_gmem_luo_preserve() via
+ * get_file_active() to prevent lockless ABA races).
+ */
+ rcu_assign_pointer(kvm->vm_file, file);
+#endif
+
+ /*
+ * Don't call kvm_put_kvm anymore at this point; file->f_op is
+ * already set, with ->release() being kvm_vm_release(). In error
+ * cases it will be called by the final fput(file) and will take
+ * care of doing kvm_put_kvm(kvm).
+ */
+
+ return file;
+}
+
static int kvm_dev_ioctl_create_vm(unsigned long type)
{
char fdname[ITOA_MAX_LEN + 1];
int r, fd;
- struct kvm *kvm;
struct file *file;
fd = get_unused_fd_flags(O_CLOEXEC);
@@ -5489,31 +5535,17 @@ static int kvm_dev_ioctl_create_vm(unsigned long type)
snprintf(fdname, sizeof(fdname), "%d", fd);
- kvm = kvm_create_vm(type, fdname);
- if (IS_ERR(kvm)) {
- r = PTR_ERR(kvm);
- goto put_fd;
- }
-
- file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
+ file = kvm_create_vm_file(type, fdname);
if (IS_ERR(file)) {
r = PTR_ERR(file);
- goto put_kvm;
+ goto put_fd;
}
- /*
- * Don't call kvm_put_kvm anymore at this point; file->f_op is
- * already set, with ->release() being kvm_vm_release(). In error
- * cases it will be called by the final fput(file) and will take
- * care of doing kvm_put_kvm(kvm).
- */
- kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
+ kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, file->private_data);
fd_install(fd, file);
return fd;
-put_kvm:
- kvm_put_kvm(kvm);
put_fd:
put_unused_fd(fd);
return r;
@@ -6341,6 +6373,11 @@ static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm)
kfree(env);
}
+void kvm_uevent_notify_vm_create(struct kvm *kvm)
+{
+ kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
+}
+
static void kvm_init_debug(void)
{
const struct file_operations *fops;
diff --git a/virt/kvm/kvm_mm.h b/virt/kvm/kvm_mm.h
index 9fcc5d5b7f8d..7aa1d65c3d46 100644
--- a/virt/kvm/kvm_mm.h
+++ b/virt/kvm/kvm_mm.h
@@ -3,6 +3,9 @@
#ifndef __KVM_MM_H__
#define __KVM_MM_H__ 1
+/* Worst case buffer size needed for holding an integer as a string. */
+#define ITOA_MAX_LEN 12
+
/*
* Architectures can choose whether to use an rwlock or spinlock
* for the mmu_lock. These macros, for use in common code
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply related
* [RFC PATCH v2 02/10] liveupdate: Add LIVEUPDATE_GUEST_MEMFD config option
From: Tarun Sahu @ 2026-06-05 17:08 UTC (permalink / raw)
To: Jonathan Corbet, vannapurve, Tarun Sahu, fvdl, Pasha Tatashin,
Shuah Khan, sagis, aneesh.kumar, skhawaja, vipinsh, ackerleytng,
Pratyush Yadav, david, dmatlack, mark.rutland, Paolo Bonzini,
Mike Rapoport, Alexander Graf, seanjc, axelrasmussen
Cc: linux-kselftest, kexec, linux-kernel, linux-doc, kvm, linux-mm
In-Reply-To: <cover.1780676742.git.tarunsahu@google.com>
Introduce the LIVEUPDATE_GUEST_MEMFD Kconfig option. This option
enables live update support for KVM guest_memfd files, enabling
guest_memfd-backed memory preservation across kernel upgrades.
Currently this support only guest_memfd files that are full-shared
and pre-faulted.
Signed-off-by: Tarun Sahu <tarunsahu@google.com>
---
kernel/liveupdate/Kconfig | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/kernel/liveupdate/Kconfig b/kernel/liveupdate/Kconfig
index 1a8513f16ef7..0bbc4037192e 100644
--- a/kernel/liveupdate/Kconfig
+++ b/kernel/liveupdate/Kconfig
@@ -88,4 +88,19 @@ config LIVEUPDATE_MEMFD
If unsure, say N.
+config LIVEUPDATE_GUEST_MEMFD
+ bool "Live update support for guest_memfd"
+ depends on LIVEUPDATE
+ depends on KVM_GUEST_MEMFD
+ default LIVEUPDATE
+ help
+ Enable live update support for KVM guest_memfd files. This allows
+ preserving VM Memory backed by guest_memfd file across kernel live
+ updates.
+
+ This can only be used for the guest_memfd that are fully-shared
+ and pre-faulted.
+
+ If unsure, say N.
+
endmenu
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply related
* [RFC PATCH v2 01/10] liveupdate: luo_file: Add internal APIs for file preservation
From: Tarun Sahu @ 2026-06-05 17:08 UTC (permalink / raw)
To: Jonathan Corbet, vannapurve, Tarun Sahu, fvdl, Pasha Tatashin,
Shuah Khan, sagis, aneesh.kumar, skhawaja, vipinsh, ackerleytng,
Pratyush Yadav, david, dmatlack, mark.rutland, Paolo Bonzini,
Mike Rapoport, Alexander Graf, seanjc, axelrasmussen
Cc: linux-kselftest, kexec, linux-kernel, linux-doc, kvm, linux-mm
In-Reply-To: <cover.1780676742.git.tarunsahu@google.com>
From: Pasha Tatashin <pasha.tatashin@soleen.com>
The core liveupdate mechanism allows userspace to preserve file
descriptors. However, kernel subsystems often manage struct file
objects directly and need to participate in the preservation process
programmatically without relying solely on userspace interaction.
Signed-off-by: Pasha Tatashin <pasha.tatashin@soleen.com>
Signed-off-by: Samiullah Khawaja <skhawaja@google.com>
Signed-off-by: Tarun Sahu <tarunsahu@google.com>
---
include/linux/liveupdate.h | 21 ++++++++++
kernel/liveupdate/luo_file.c | 69 ++++++++++++++++++++++++++++++++
kernel/liveupdate/luo_internal.h | 17 ++++++++
3 files changed, 107 insertions(+)
diff --git a/include/linux/liveupdate.h b/include/linux/liveupdate.h
index 30c5a39ff9e9..de052438eaac 100644
--- a/include/linux/liveupdate.h
+++ b/include/linux/liveupdate.h
@@ -24,6 +24,7 @@ struct file;
/**
* struct liveupdate_file_op_args - Arguments for file operation callbacks.
* @handler: The file handler being called.
+ * @session: The session this file belongs to.
* @retrieve_status: The retrieve status for the 'can_finish / finish'
* operation. A value of 0 means the retrieve has not been
* attempted, a positive value means the retrieve was
@@ -44,6 +45,7 @@ struct file;
*/
struct liveupdate_file_op_args {
struct liveupdate_file_handler *handler;
+ struct liveupdate_session *session;
int retrieve_status;
struct file *file;
u64 serialized_data;
@@ -240,6 +242,13 @@ void liveupdate_unregister_flb(struct liveupdate_file_handler *fh,
int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp);
int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp);
+/* kernel can internally retrieve files */
+int liveupdate_get_file_incoming(struct liveupdate_session *s, u64 token,
+ struct file **filep);
+
+/* Get a token for an outgoing file, or -ENOENT if file is not preserved */
+int liveupdate_get_token_outgoing(struct liveupdate_session *s,
+ struct file *file, u64 *tokenp);
#else /* CONFIG_LIVEUPDATE */
@@ -285,5 +294,17 @@ static inline int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb,
return -EOPNOTSUPP;
}
+static inline int liveupdate_get_file_incoming(struct liveupdate_session *s,
+ u64 token, struct file **filep)
+{
+ return -EOPNOTSUPP;
+}
+
+static inline int liveupdate_get_token_outgoing(struct liveupdate_session *s,
+ struct file *file, u64 *tokenp)
+{
+ return -EOPNOTSUPP;
+}
+
#endif /* CONFIG_LIVEUPDATE */
#endif /* _LINUX_LIVEUPDATE_H */
diff --git a/kernel/liveupdate/luo_file.c b/kernel/liveupdate/luo_file.c
index a0a419085e28..0aa0b4e5339f 100644
--- a/kernel/liveupdate/luo_file.c
+++ b/kernel/liveupdate/luo_file.c
@@ -323,6 +323,7 @@ int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd)
mutex_init(&luo_file->mutex);
args.handler = fh;
+ args.session = luo_session_from_file_set(file_set);
args.file = file;
err = fh->ops->preserve(&args);
if (err)
@@ -380,6 +381,7 @@ void luo_file_unpreserve_files(struct luo_file_set *file_set)
struct luo_file, list);
args.handler = luo_file->fh;
+ args.session = luo_session_from_file_set(file_set);
args.file = luo_file->file;
args.serialized_data = luo_file->serialized_data;
args.private_data = luo_file->private_data;
@@ -411,6 +413,7 @@ static int luo_file_freeze_one(struct luo_file_set *file_set,
struct liveupdate_file_op_args args = {0};
args.handler = luo_file->fh;
+ args.session = luo_session_from_file_set(file_set);
args.file = luo_file->file;
args.serialized_data = luo_file->serialized_data;
args.private_data = luo_file->private_data;
@@ -432,6 +435,7 @@ static void luo_file_unfreeze_one(struct luo_file_set *file_set,
struct liveupdate_file_op_args args = {0};
args.handler = luo_file->fh;
+ args.session = luo_session_from_file_set(file_set);
args.file = luo_file->file;
args.serialized_data = luo_file->serialized_data;
args.private_data = luo_file->private_data;
@@ -621,6 +625,7 @@ int luo_retrieve_file(struct luo_file_set *file_set, u64 token,
}
args.handler = luo_file->fh;
+ args.session = luo_session_from_file_set(file_set);
args.serialized_data = luo_file->serialized_data;
err = luo_file->fh->ops->retrieve(&args);
if (err) {
@@ -654,6 +659,7 @@ static int luo_file_can_finish_one(struct luo_file_set *file_set,
struct liveupdate_file_op_args args = {0};
args.handler = luo_file->fh;
+ args.session = luo_session_from_file_set(file_set);
args.file = luo_file->file;
args.serialized_data = luo_file->serialized_data;
args.retrieve_status = luo_file->retrieve_status;
@@ -671,6 +677,7 @@ static void luo_file_finish_one(struct luo_file_set *file_set,
guard(mutex)(&luo_file->mutex);
args.handler = luo_file->fh;
+ args.session = luo_session_from_file_set(file_set);
args.file = luo_file->file;
args.serialized_data = luo_file->serialized_data;
args.retrieve_status = luo_file->retrieve_status;
@@ -924,3 +931,65 @@ void liveupdate_unregister_file_handler(struct liveupdate_file_handler *fh)
luo_flb_unregister_all(fh);
list_del(&ACCESS_PRIVATE(fh, list));
}
+EXPORT_SYMBOL_GPL(liveupdate_unregister_file_handler);
+
+/**
+ * liveupdate_get_token_outgoing - Get the token for a preserved file.
+ * @s: The outgoing liveupdate session.
+ * @file: The file object to search for.
+ * @tokenp: Output parameter for the found token.
+ *
+ * Searches the list of preserved files in an outgoing session for a matching
+ * file object. If found, the corresponding user-provided token is returned.
+ *
+ * This function is intended for in-kernel callers that need to correlate a
+ * file with its liveupdate token.
+ *
+ * Context: It must be called with session mutex acquired.
+ * Return: 0 on success, -ENOENT if the file is not preserved in this session.
+ */
+int liveupdate_get_token_outgoing(struct liveupdate_session *s,
+ struct file *file, u64 *tokenp)
+{
+ struct luo_file_set *file_set = luo_file_set_from_session_locked(s);
+ struct luo_file *luo_file;
+ int err = -ENOENT;
+
+ list_for_each_entry(luo_file, &file_set->files_list, list) {
+ if (luo_file->file == file) {
+ if (tokenp)
+ *tokenp = luo_file->token;
+ err = 0;
+ break;
+ }
+ }
+
+ return err;
+}
+
+/**
+ * liveupdate_get_file_incoming - Retrieves a preserved file for in-kernel use.
+ * @s: The incoming liveupdate session (restored from the previous kernel).
+ * @token: The unique token identifying the file to retrieve.
+ * @filep: On success, this will be populated with a pointer to the retrieved
+ * 'struct file'.
+ *
+ * Provides a kernel-internal API for other subsystems to retrieve their
+ * preserved files after a live update. This function is a simple wrapper
+ * around luo_retrieve_file(), allowing callers to find a file by its token.
+ *
+ * The caller receives a new reference to the file and must call fput() when it
+ * is no longer needed. The file's lifetime is managed by LUO and any userspace
+ * file descriptors. If the caller needs to hold a reference to the file beyond
+ * the immediate scope, it must call get_file() itself.
+ *
+ * Context: It must be called with session mutex acquired of a restored session.
+ * Return: 0 on success. Returns -ENOENT if no file with the matching token is
+ * found, or any other negative errno on failure.
+ */
+int liveupdate_get_file_incoming(struct liveupdate_session *s, u64 token,
+ struct file **filep)
+{
+ return luo_retrieve_file(luo_file_set_from_session_locked(s),
+ token, filep);
+}
diff --git a/kernel/liveupdate/luo_internal.h b/kernel/liveupdate/luo_internal.h
index 875844d7a41d..08b198802e7f 100644
--- a/kernel/liveupdate/luo_internal.h
+++ b/kernel/liveupdate/luo_internal.h
@@ -79,6 +79,23 @@ struct luo_session {
extern struct rw_semaphore luo_register_rwlock;
+static inline struct liveupdate_session *luo_session_from_file_set(struct luo_file_set *file_set)
+{
+ struct luo_session *session;
+
+ session = container_of(file_set, struct luo_session, file_set);
+
+ return (struct liveupdate_session *)session;
+}
+
+static inline struct luo_file_set *luo_file_set_from_session_locked(struct liveupdate_session *s)
+{
+ struct luo_session *session = (struct luo_session *)s;
+
+ lockdep_assert_held(&session->mutex);
+ return &session->file_set;
+}
+
int luo_session_create(const char *name, struct file **filep);
int luo_session_retrieve(const char *name, struct file **filep);
int __init luo_session_setup_outgoing(void *fdt);
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply related
* [RFC PATCH v1 0/10] liveupdate: kvm: Guest_memfd preservation
From: Tarun Sahu @ 2026-06-05 17:08 UTC (permalink / raw)
To: Jonathan Corbet, vannapurve, Tarun Sahu, fvdl, Pasha Tatashin,
Shuah Khan, sagis, aneesh.kumar, skhawaja, vipinsh, ackerleytng,
Pratyush Yadav, david, dmatlack, mark.rutland, Paolo Bonzini,
Mike Rapoport, Alexander Graf, seanjc, axelrasmussen
Cc: linux-kselftest, kexec, linux-kernel, linux-doc, kvm, linux-mm
Changes from V1:
1. Remove mem_attr_array preservation
2. Removed prefaulted guest_memfd condition
3. Updated the check for shared guest_memfd from INIT_SHARED to
kvm_arch_has_private_mem
4. Added the document liveupdate/vmm.rst
Hello,
I am proposing this series as RFC, to initiate the discussion for
supporting the guest_memfd preservation. This will setup basic arhitecture
for VM preservation during liveupdate. This Cover letter has three
sections (please feel free to skip the section you already know):
A. Guest_memfd introduction:
To make the audience familiar with guest_memfd
B. Liveupdate introduction:
To make the audience familiar with liveupdate
C. Actual Implementation Design and questions.
**A: GUEST MEMFD INTRODUCTION**
Initially, guest_memfd was created to support guest private memory in
confidential computing VMs (CoCo VMs). It was designed so that whenever
a guest wants to grant the host access to private memory, a series of
calls occurs: from the guest to KVM, KVM to the host userspace, host
userspace back to KVM, and finally a new page fault maps the memory into
a separate shared address space. Conversely, if the guest transitions the
memory back to private, the subsequent fault is handled by guest_memfd.
(Dual Mapping Architecture). In such a VM, all guest memory is initially
shared. On the fly, the guest may request to change pages to private; the
metadata indicating which parts of memory are private is stored in an
xarray inside struct kvm (mem_attr_array). This array serves as the source
of truth for the fault mechanism, determining whether a mapping should be
created from host-userspace-mapped pages or directly from the guest_memfd
file. For private memory, Fault also calls architecture-specific function
to set up private hardware access (e.g., on SEV-SNP or TDX). This type of
guest_memfd is fully-private where shared mapping comes from userspace
mapped address space.
Subsequently, support was added to allow the entire guest memory to be
backed by guest_memfd. This led to the implementation of the MMAP and
INIT_SHARED flags for the guest_memfd inode. When KVM_CREATE_GUEST_MEMFD
is called with these flags, the guest_memfd becomes mmap-able by host
userspace. The INIT_SHARED flag is used to make the guest_memfd completely
shared between the host and the guest. Consequently, page faults from both
host userspace and the guest resolve to the same guest_memfd page cache.
However, under this configuration, marking a portion of this memory as
private is not possible. This type of guest_memfd is fully-shared.
If guest_memfd is created with INIT_SHARED without MMAP, the host
can never access the guest_memfd. But the memory is still considered
shared.
Hence, At this point, Only use-case of guest_memfd is either fully-shared
or fully-private.
There is ongoing work to make shared and private mapping in-place backed
by guest_memfd. [1] There is also ongoing work to back guest_memfd by
hugetlb pages. [2]
**B: LIVEUPDATE INTRODUCTION (LIVEUPDATE ORCHESTRATOR - LUO)**
Livepdate support was added in kernel to update the host kernel by
minimizing the downtime to minimal. This is generally achieved by
preserving the current state of the system and retrieve after boot to
resume from where we left it.
Any subsystem that wants to preserve themselves, register their handler
with liveupdate system. This handler includes calls to the following
*can_preserve (file)*:
This tells the luo system about the eligibility of the file. When
preserve ioctl is called, it first loop through all the file handlers
and call can_preserve, the one which return true, luo uses this file
handler fh->preserve call to preserve the file.
*preserve(file)*:
This actually preserves the file.
*unpreserve(file)*:
This unpreserve the file incase userspace want to go back.
*retrieve(file)*:
On new kernel boot, this function retrieves the file.
*finish(file)*:
When userspace decides that all the files in the liveupdate session has
been retrieved, it can trigger this to do final work of cleaning up.
LUO preserve its memory using KHO (kexec-handover). All these APIs will
be implemented using KHO calls.
**C: GUEST MEMFD PRESERVATION**
SCOPE:
1. Fully Shared Guest_memfd
2. Guest_memfd backed by PAGE_SIZE pages
Any VM whose memory is backed by such guest_memfd can be preserved
across liveupdate.
The preservation call is straight forward. It walks through the page
cache, serialize the folios and preserve them.
On the retrieval path:
Currently, creating a guest_memfd requires an associated struct kvm
(derived from vm_file / vm_fd). Since there is no direct way to pass a
VM file descriptor via the LUO API.
I leverage a companion patch [3] (Also added as part of this series
PATCH[1]) that allows one file to retrieve another file from the same
LUO session. This enables the guest_memfd retrieval path to obtain the
preserved KVM file, use it during guest_memfd file creation, and
subsequently populate its preserved memory.
Preserving the KVM file allows us to preserve additional VM-specific
metadata, which will be crucial in the future for cleanly resuming the
VM. Currently, it preserves only the VM type.
On the retrieval path:
KVM normally requires a unique identifier (fdname) upon creation,
which KVM typically assigns based on the newly created file descriptor
number. However, in the LUO retrieval path, the retrieve call restores
the underlying file structure and delegates actual file descriptor
allocation to LUO (check luo_session_retrieve_fd). Currently, I used an
atomically incremented sequence number as the fdname. I would like to
discuss whether userspace services rely on specific naming conventions
here. Or if we can change underlying the retrieve call
(luo_retrieve_file) to pass fd?
This series also introduces the inode freeze call for guest_memfd inode.
Which fails any subseuquent fallocate calls or new page fault allocation.
VMM is supposed to take necessary measure when it is triggering the
liveupdate. VMM must:
1. Either pause the VM before preserving the VM/guest_memfd OR
2. Take action (vm_pause or unpreserve/destroy liveupdate sequence)
when a fault fails and VM_EXIT to VMM with -EPERM.
Preservation Order between VM and guest_memfd file:
There is no strict order, they are independent. Guest_memfd file needs
the kvm_file preserved token, which it update on freeze call as freeze
is called just before kexec jump. kexec fails incase freeze will be
unsuccessful, for this case, it will fail if vm_file token is not found.
Retrieval order for VM and guest_memfd file:
There is no strict order needed for retrieval.
1. If VM file is retrieve before guest_memfd: guest_memfd will be
retrieved and vm_file also retrieved and userspace hold reference to
both files.
2. If guest_memfd file is retrieved before vm_file: guest_memfd will be
retrieved and it will retrieve vm_file internally and userspace can
retrieve vm_file later. But userspace will not have reference to vm_file
and luo_finish() will drop vm_file final reference if userspace does not
retrieve vm_file before calling luo_finish(). This is valid case, as
guest_memfd can live without vm_file as in the case vm_file is closed
before guest_memfd file.
I have implemented the basic test, where it spawn a VM with guest_memfd
or 16MB and write data to its 5MB portion. After LUO preserve call, and
kexec, On retrieve, a new VM is spawn with the restored vm_file and
restored guest_memfd and the data is verified. It uses the liveupdate
test library [5].
Future Work:
1. Support private guest_memfd preservation.
2. Extend the support for guest_memfd with in-place conversion of
shared/private.
[1] https://lore.kernel.org/all/20260507-gmem-inplace-conversion-v6-0-91ab5a8b19a4@google.com/
[2] https://lore.kernel.org/all/cover.1747264138.git.ackerleytng@google.com/
[3] https://lore.kernel.org/all/20260427175633.1978233-2-skhawaja@google.com/
[4] https://lore.kernel.org/all/cover.1691446946.git.ackerleytng@google.com/
[5] https://lore.kernel.org/all/20260511201155.1488670-1-vipinsh@google.com/
Pasha Tatashin (1):
liveupdate: luo_file: Add internal APIs for file preservation
Tarun Sahu (8):
liveupdate: Add LIVEUPDATE_GUEST_MEMFD config option
kvm: Prepare core VM structs and helpers for LUO support
kvm: kvm_luo: Allow kvm preservation with LUO
kvm: guest_memfd: Move internal definitions and helper to new header
kvm: guest_memfd: Add support for freezing and unfreezing mappings
kvm: guest_memfd_luo: add support for guest_memfd preservation
selftests: kvm: Split ____vm_create() to expose init helpers
selftests: kvm: Add guest_memfd_preservation_test
MAINTAINERS | 13 +
include/linux/kho/abi/kvm.h | 106 ++++
include/linux/kvm_host.h | 14 +
include/linux/liveupdate.h | 21 +
kernel/liveupdate/Kconfig | 15 +
kernel/liveupdate/luo_file.c | 69 +++
kernel/liveupdate/luo_internal.h | 17 +
tools/testing/selftests/kvm/Makefile.kvm | 6 +-
.../kvm/guest_memfd_preservation_test.c | 230 ++++++++
.../testing/selftests/kvm/include/kvm_util.h | 2 +
tools/testing/selftests/kvm/lib/kvm_util.c | 26 +-
virt/kvm/Makefile.kvm | 1 +
virt/kvm/guest_memfd.c | 185 +++++--
virt/kvm/guest_memfd.h | 44 ++
virt/kvm/guest_memfd_luo.c | 489 ++++++++++++++++++
virt/kvm/kvm_luo.c | 190 +++++++
virt/kvm/kvm_main.c | 94 +++-
virt/kvm/kvm_mm.h | 15 +
18 files changed, 1456 insertions(+), 81 deletions(-)
create mode 100644 include/linux/kho/abi/kvm.h
create mode 100644 tools/testing/selftests/kvm/guest_memfd_preservation_test.c
create mode 100644 virt/kvm/guest_memfd.h
create mode 100644 virt/kvm/guest_memfd_luo.c
create mode 100644 virt/kvm/kvm_luo.c
base-commit: e43ffb69e0438cddd72aaa30898b4dc446f664f8
prerequisite-patch-id: 85705fb54d3065efe1d87ab4b69e828a9f3404e7
prerequisite-patch-id: 7bf85ca17e12b26a72d41ee35f2ec8fc5ce2e692
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply
* Re: [PATCH v6 06/11] x86/virt/tdx: Optimize tdx_pamt_get/put()
From: Dave Hansen @ 2026-06-05 16:23 UTC (permalink / raw)
To: Kiryl Shutsemau, Chao Gao
Cc: Edgecombe, Rick P, kvm@vger.kernel.org,
linux-coco@lists.linux.dev, Huang, Kai, Zhao, Yan Y,
seanjc@google.com, mingo@redhat.com, linux-kernel@vger.kernel.org,
pbonzini@redhat.com, nik.borisov@suse.com,
linux-doc@vger.kernel.org, hpa@zytor.com, tglx@kernel.org,
Annapurve, Vishal, bp@alien8.de, kirill.shutemov@linux.intel.com,
x86@kernel.org
In-Reply-To: <aiK1_q8beMcIEiwO@thinkstation>
On 6/5/26 04:42, Kiryl Shutsemau wrote:
>>> I don't see a reason why we can't keep the scoped_guard() on get side.
>> One additional reason to drop scoped_guard() is that it mixes cleanup helpers
>> with goto, which is discouraged. See [*]
>>
>> :Lastly, given that the benefit of cleanup helpers is removal of “goto”, and
>> :that the “goto” statement can jump between scopes, the expectation is that
>> :usage of “goto” and cleanup helpers is never mixed in the same function.
> Fair enough.
>
> But it can also be address if we free the PAMT page array with the guard
> too :P
How important is this patch? I see "Optimize" but I read "Optional".
If we're arguing about it, maybe we should just kick it out and focus on
the more important bits.
^ permalink raw reply
* Re: [RFC PATCH v2 0/6] kcov: per-task dataflow extraction at kernel function boundaries
From: Alexander Potapenko @ 2026-06-05 16:20 UTC (permalink / raw)
To: Yunseong Kim
Cc: Ingo Molnar, Peter Zijlstra, Juri Lelli, Vincent Guittot,
Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, K Prateek Nayak, Dmitry Vyukov,
Andrey Konovalov, Andrew Morton, Nathan Chancellor,
Nick Desaulniers, Bill Wendling, Justin Stitt, Nicolas Schier,
Miguel Ojeda, Boqun Feng, Gary Guo, Björn Roy Baron,
Benno Lossin, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Danilo Krummrich, Jonathan Corbet, Shuah Khan, linux-kernel,
kasan-dev, llvm, linux-kbuild, rust-for-linux, workflows,
linux-doc, Yunseong Kim
In-Reply-To: <20260603-kcov-dataflow-next-20260603-v2-0-fee0939de2c4@est.tech>
On Wed, Jun 3, 2026 at 7:43 PM Yunseong Kim <yunseong.kim@est.tech> wrote:
>
> Introduces a new KCOV exetened feature that captures function arguments and
> return values at kernel function boundaries, enabling per-process visibility
> into runtime dataflow.
Some high-level comments:
- Make sure your code can run on every platform supported by kcov (namely ARM64)
- Check out Sashiko findings:
https://sashiko.dev/#/patchset/20260603-kcov-dataflow-next-20260603-v2-0-fee0939de2c4%40est.tech,
at least some of them seem to make sense
- Please consolidate changes to the same file into a single patch
- There seem to be two tools (one in C and one in Python) with
overlapping functionality, can you keep only one?
- The test modules seem to be used only in manual testing. Can you
convert them to kselftests or remove them?
- At this point, long dashes in the kernel codebase are quite rare,
and I don't see a reason to add more.
^ permalink raw reply
* [PATCH mm-unstable v19 14/14] Documentation: mm: update the admin guide for mTHP collapse
From: Nico Pache @ 2026-06-05 16:14 UTC (permalink / raw)
To: linux-doc, linux-kernel, linux-mm, linux-trace-kernel
Cc: aarcange, akpm, anshuman.khandual, apopple, baohua, baolin.wang,
byungchul, catalin.marinas, cl, corbet, dave.hansen, david,
dev.jain, gourry, hannes, hughd, jack, jackmanb, jannh, jglisse,
joshua.hahnjy, kas, lance.yang, liam, ljs, mathieu.desnoyers,
matthew.brost, mhiramat, mhocko, npache, peterx, pfalcato,
rakie.kim, raquini, rdunlap, richard.weiyang, rientjes, rostedt,
rppt, ryan.roberts, shivankg, sunnanyong, surenb,
thomas.hellstrom, tiwai, usamaarif642, vbabka, vishal.moola,
wangkefeng.wang, will, willy, yang, ying.huang, ziy, zokeefe,
Bagas Sanjaya
In-Reply-To: <20260605161422.213817-1-npache@redhat.com>
Now that we can collapse to mTHPs lets update the admin guide to
reflect these changes and provide proper guidance on how to utilize it.
Reviewed-by: Lorenzo Stoakes <ljs@kernel.org>
Reviewed-by: Bagas Sanjaya <bagasdotme@gmail.com>
Signed-off-by: Nico Pache <npache@redhat.com>
---
Documentation/admin-guide/mm/transhuge.rst | 49 ++++++++++++++--------
1 file changed, 32 insertions(+), 17 deletions(-)
diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst
index b98e18c80185..23f8d13c2629 100644
--- a/Documentation/admin-guide/mm/transhuge.rst
+++ b/Documentation/admin-guide/mm/transhuge.rst
@@ -63,7 +63,8 @@ often.
THP can be enabled system wide or restricted to certain tasks or even
memory ranges inside task's address space. Unless THP is completely
disabled, there is ``khugepaged`` daemon that scans memory and
-collapses sequences of basic pages into PMD-sized huge pages.
+collapses sequences of basic pages into huge pages of either PMD size
+or mTHP sizes, if the system is configured to do so.
The THP behaviour is controlled via :ref:`sysfs <thp_sysfs>`
interface and using madvise(2) and prctl(2) system calls.
@@ -219,10 +220,10 @@ this behaviour by writing 0 to shrink_underused, and enable it by writing
echo 0 > /sys/kernel/mm/transparent_hugepage/shrink_underused
echo 1 > /sys/kernel/mm/transparent_hugepage/shrink_underused
-khugepaged will be automatically started when PMD-sized THP is enabled
+khugepaged will be automatically started when any THP size is enabled
(either of the per-size anon control or the top-level control are set
to "always" or "madvise"), and it'll be automatically shutdown when
-PMD-sized THP is disabled (when both the per-size anon control and the
+all THP sizes are disabled (when both the per-size anon control and the
top-level control are "never")
process THP controls
@@ -265,8 +266,8 @@ Khugepaged controls
-------------------
.. note::
- khugepaged currently only searches for opportunities to collapse to
- PMD-sized THP and no attempt is made to collapse to other THP
+ khugepaged currently only searches for opportunities to collapse file/shmem
+ to PMD-sized THP. Only anonymous memory will attempt to collapse to other THP
sizes.
khugepaged runs usually at low frequency so while one may not want to
@@ -296,11 +297,11 @@ allocation failure to throttle the next allocation attempt::
The khugepaged progress can be seen in the number of pages collapsed (note
that this counter may not be an exact count of the number of pages
collapsed, since "collapsed" could mean multiple things: (1) A PTE mapping
-being replaced by a PMD mapping, or (2) All 4K physical pages replaced by
-one 2M hugepage. Each may happen independently, or together, depending on
-the type of memory and the failures that occur. As such, this value should
-be interpreted roughly as a sign of progress, and counters in /proc/vmstat
-consulted for more accurate accounting)::
+being replaced by a PMD mapping, or (2) physical pages replaced by one
+hugepage of various sizes (PMD-sized or mTHP). Each may happen independently,
+or together, depending on the type of memory and the failures that occur.
+As such, this value should be interpreted roughly as a sign of progress,
+and counters in /proc/vmstat consulted for more accurate accounting)::
/sys/kernel/mm/transparent_hugepage/khugepaged/pages_collapsed
@@ -308,16 +309,21 @@ for each pass::
/sys/kernel/mm/transparent_hugepage/khugepaged/full_scans
-``max_ptes_none`` specifies how many extra small pages (that are
-not already mapped) can be allocated when collapsing a group
-of small pages into one large page::
+``max_ptes_none`` specifies how many empty (none/zero) pages are allowed
+when collapsing a group of small pages into one large page::
/sys/kernel/mm/transparent_hugepage/khugepaged/max_ptes_none
-A higher value leads to use additional memory for programs.
-A lower value leads to gain less thp performance. Value of
-max_ptes_none can waste cpu time very little, you can
-ignore it.
+For PMD-sized THP collapse, this directly limits the number of empty pages
+allowed in the 2MB region.
+
+For mTHP collapse, only 0 or (HPAGE_PMD_NR - 1) are supported. At
+HPAGE_PMD_NR - 1, we collapse to the highest possible order. Any intermediate
+value will emit a warning and mTHP collapse will default to max_ptes_none=0.
+
+A higher value allows more empty pages, potentially leading to more memory
+usage but better THP performance. A lower value is more conservative and
+may result in fewer THP collapses.
``max_ptes_swap`` specifies how many pages can be brought in from
swap when collapsing a group of pages into a transparent huge page::
@@ -337,6 +343,15 @@ that THP is shared. Exceeding the number would block the collapse::
A higher value may increase memory footprint for some workloads.
+.. note::
+ For mTHP collapse, khugepaged does not support collapsing regions that
+ contain shared or swapped out pages, as this could lead to continuous
+ promotion to higher orders. The collapse will fail if any shared or
+ swapped PTEs are encountered during the scan.
+
+ Currently, madvise_collapse only supports collapsing to PMD-sized THPs
+ and does not attempt mTHP collapses.
+
Boot parameters
===============
--
2.54.0
^ permalink raw reply related
* [PATCH mm-unstable v19 13/14] mm/khugepaged: run khugepaged for all orders
From: Nico Pache @ 2026-06-05 16:14 UTC (permalink / raw)
To: linux-doc, linux-kernel, linux-mm, linux-trace-kernel
Cc: aarcange, akpm, anshuman.khandual, apopple, baohua, baolin.wang,
byungchul, catalin.marinas, cl, corbet, dave.hansen, david,
dev.jain, gourry, hannes, hughd, jack, jackmanb, jannh, jglisse,
joshua.hahnjy, kas, lance.yang, liam, ljs, mathieu.desnoyers,
matthew.brost, mhiramat, mhocko, npache, peterx, pfalcato,
rakie.kim, raquini, rdunlap, richard.weiyang, rientjes, rostedt,
rppt, ryan.roberts, shivankg, sunnanyong, surenb,
thomas.hellstrom, tiwai, usamaarif642, vbabka, vishal.moola,
wangkefeng.wang, will, willy, yang, ying.huang, ziy, zokeefe,
Usama Arif
In-Reply-To: <20260605161422.213817-1-npache@redhat.com>
From: Baolin Wang <baolin.wang@linux.alibaba.com>
If any order (m)THP is enabled we should allow running khugepaged to
attempt scanning and collapsing mTHPs. In order for khugepaged to operate
when only mTHP sizes are specified in sysfs, we must modify the predicate
function that determines whether it ought to run to do so.
This function is currently called hugepage_pmd_enabled(), this patch
renames it to hugepage_enabled() and updates the logic to check to
determine whether any valid orders may exist which would justify
khugepaged running.
We must also update collapse_possible_orders() to check all orders if
the vma is anonymous and the collapse is khugepaged.
After this patch khugepaged mTHP collapse is fully enabled.
Reviewed-by: Lorenzo Stoakes <ljs@kernel.org>
Reviewed-by: Lance Yang <lance.yang@linux.dev>
Acked-by: Usama Arif <usama.arif@linux.dev>
Acked-by: David Hildenbrand (Arm) <david@kernel.org>
Signed-off-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Signed-off-by: Nico Pache <npache@redhat.com>
---
mm/khugepaged.c | 36 ++++++++++++++++++++----------------
1 file changed, 20 insertions(+), 16 deletions(-)
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 7de92b28dd30..996e014a03d3 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -503,23 +503,23 @@ static inline int collapse_test_exit_or_disable(struct mm_struct *mm)
mm_flags_test(MMF_DISABLE_THP_COMPLETELY, mm);
}
-static bool hugepage_pmd_enabled(void)
+static bool hugepage_enabled(void)
{
/*
* We cover the anon, shmem and the file-backed case here; file-backed
* hugepages, when configured in, are determined by the global control.
- * Anon pmd-sized hugepages are determined by the pmd-size control.
+ * Anon hugepages are determined by its per-size mTHP control.
* Shmem pmd-sized hugepages are also determined by its pmd-size control,
* except when the global shmem_huge is set to SHMEM_HUGE_DENY.
*/
if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) &&
hugepage_global_enabled())
return true;
- if (test_bit(PMD_ORDER, &huge_anon_orders_always))
+ if (READ_ONCE(huge_anon_orders_always))
return true;
- if (test_bit(PMD_ORDER, &huge_anon_orders_madvise))
+ if (READ_ONCE(huge_anon_orders_madvise))
return true;
- if (test_bit(PMD_ORDER, &huge_anon_orders_inherit) &&
+ if (READ_ONCE(huge_anon_orders_inherit) &&
hugepage_global_enabled())
return true;
if (IS_ENABLED(CONFIG_SHMEM) && shmem_hpage_pmd_enabled())
@@ -566,7 +566,13 @@ void __khugepaged_enter(struct mm_struct *mm)
static unsigned long collapse_possible_orders(struct vm_area_struct *vma,
vm_flags_t vm_flags, enum tva_type tva_flags)
{
- const unsigned long orders = BIT(HPAGE_PMD_ORDER);
+ unsigned long orders;
+
+ /* If khugepaged is scanning an anonymous vma, allow mTHP collapse */
+ if ((tva_flags == TVA_KHUGEPAGED) && vma_is_anonymous(vma))
+ orders = THP_ORDERS_ALL_ANON;
+ else
+ orders = BIT(HPAGE_PMD_ORDER);
return thp_vma_allowable_orders(vma, vm_flags, tva_flags, orders);
}
@@ -580,11 +586,9 @@ static bool collapse_possible(struct vm_area_struct *vma,
void khugepaged_enter_vma(struct vm_area_struct *vma,
vm_flags_t vm_flags)
{
- if (!mm_flags_test(MMF_VM_HUGEPAGE, vma->vm_mm) &&
- hugepage_pmd_enabled()) {
- if (collapse_possible(vma, vm_flags, TVA_KHUGEPAGED))
- __khugepaged_enter(vma->vm_mm);
- }
+ if (!mm_flags_test(MMF_VM_HUGEPAGE, vma->vm_mm) && hugepage_enabled()
+ && collapse_possible(vma, vm_flags, TVA_KHUGEPAGED))
+ __khugepaged_enter(vma->vm_mm);
}
void __khugepaged_exit(struct mm_struct *mm)
@@ -2936,7 +2940,7 @@ static void collapse_scan_mm_slot(unsigned int progress_max,
static int khugepaged_has_work(void)
{
- return !list_empty(&khugepaged_scan.mm_head) && hugepage_pmd_enabled();
+ return !list_empty(&khugepaged_scan.mm_head) && hugepage_enabled();
}
static int khugepaged_wait_event(void)
@@ -3009,7 +3013,7 @@ static void khugepaged_wait_work(void)
return;
}
- if (hugepage_pmd_enabled())
+ if (hugepage_enabled())
wait_event_freezable(khugepaged_wait, khugepaged_wait_event());
}
@@ -3040,7 +3044,7 @@ void set_recommended_min_free_kbytes(void)
int nr_zones = 0;
unsigned long recommended_min;
- if (!hugepage_pmd_enabled()) {
+ if (!hugepage_enabled()) {
calculate_min_free_kbytes();
goto update_wmarks;
}
@@ -3090,7 +3094,7 @@ int start_stop_khugepaged(void)
int err = 0;
mutex_lock(&khugepaged_mutex);
- if (hugepage_pmd_enabled()) {
+ if (hugepage_enabled()) {
if (!khugepaged_thread)
khugepaged_thread = kthread_run(khugepaged, NULL,
"khugepaged");
@@ -3116,7 +3120,7 @@ int start_stop_khugepaged(void)
void khugepaged_min_free_kbytes_update(void)
{
mutex_lock(&khugepaged_mutex);
- if (hugepage_pmd_enabled() && khugepaged_thread)
+ if (hugepage_enabled() && khugepaged_thread)
set_recommended_min_free_kbytes();
mutex_unlock(&khugepaged_mutex);
}
--
2.54.0
^ permalink raw reply related
* [PATCH mm-unstable v19 12/14] mm/khugepaged: avoid unnecessary mTHP collapse attempts
From: Nico Pache @ 2026-06-05 16:14 UTC (permalink / raw)
To: linux-doc, linux-kernel, linux-mm, linux-trace-kernel
Cc: aarcange, akpm, anshuman.khandual, apopple, baohua, baolin.wang,
byungchul, catalin.marinas, cl, corbet, dave.hansen, david,
dev.jain, gourry, hannes, hughd, jack, jackmanb, jannh, jglisse,
joshua.hahnjy, kas, lance.yang, liam, ljs, mathieu.desnoyers,
matthew.brost, mhiramat, mhocko, npache, peterx, pfalcato,
rakie.kim, raquini, rdunlap, richard.weiyang, rientjes, rostedt,
rppt, ryan.roberts, shivankg, sunnanyong, surenb,
thomas.hellstrom, tiwai, usamaarif642, vbabka, vishal.moola,
wangkefeng.wang, will, willy, yang, ying.huang, ziy, zokeefe,
Usama Arif
In-Reply-To: <20260605161422.213817-1-npache@redhat.com>
There are cases where, if an attempted collapse fails, all subsequent
orders are guaranteed to also fail. Avoid these collapse attempts by
bailing out early.
Reviewed-by: Lorenzo Stoakes <ljs@kernel.org>
Acked-by: Usama Arif <usama.arif@linux.dev>
Acked-by: David Hildenbrand (Arm) <david@kernel.org>
Signed-off-by: Nico Pache <npache@redhat.com>
---
mm/khugepaged.c | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 430047316f43..7de92b28dd30 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -1499,6 +1499,7 @@ static enum scan_result mthp_collapse(struct mm_struct *mm,
collapse_address = address + offset * PAGE_SIZE;
ret = collapse_huge_page(mm, collapse_address, referenced,
unmapped, cc, order);
+
switch (ret) {
/* Cases where we continue to next collapse candidate */
case SCAN_SUCCEED:
@@ -1509,6 +1510,18 @@ static enum scan_result mthp_collapse(struct mm_struct *mm,
/* Cases where lower orders might still succeed */
case SCAN_ALLOC_HUGE_PAGE_FAIL:
alloc_failed = true;
+ fallthrough;
+ case SCAN_LACK_REFERENCED_PAGE:
+ case SCAN_EXCEED_NONE_PTE:
+ case SCAN_EXCEED_SWAP_PTE:
+ case SCAN_EXCEED_SHARED_PTE:
+ case SCAN_PAGE_LOCK:
+ case SCAN_PAGE_COUNT:
+ case SCAN_PAGE_NULL:
+ case SCAN_DEL_PAGE_LRU:
+ case SCAN_PTE_NON_PRESENT:
+ case SCAN_PTE_UFFD_WP:
+ case SCAN_PAGE_LAZYFREE:
last_result = ret;
goto next_order;
/* Cases where no further collapse is possible */
--
2.54.0
^ permalink raw reply related
* [PATCH mm-unstable v19 11/14] mm/khugepaged: Introduce mTHP collapse support
From: Nico Pache @ 2026-06-05 16:14 UTC (permalink / raw)
To: linux-doc, linux-kernel, linux-mm, linux-trace-kernel
Cc: aarcange, akpm, anshuman.khandual, apopple, baohua, baolin.wang,
byungchul, catalin.marinas, cl, corbet, dave.hansen, david,
dev.jain, gourry, hannes, hughd, jack, jackmanb, jannh, jglisse,
joshua.hahnjy, kas, lance.yang, liam, ljs, mathieu.desnoyers,
matthew.brost, mhiramat, mhocko, npache, peterx, pfalcato,
rakie.kim, raquini, rdunlap, richard.weiyang, rientjes, rostedt,
rppt, ryan.roberts, shivankg, sunnanyong, surenb,
thomas.hellstrom, tiwai, usamaarif642, vbabka, vishal.moola,
wangkefeng.wang, will, willy, yang, ying.huang, ziy, zokeefe
In-Reply-To: <20260605161422.213817-1-npache@redhat.com>
Enable khugepaged to collapse to mTHP orders. This patch implements the
main scanning logic using a bitmap to track occupied pages and the
algorithm to find optimal collapse sizes.
Previous to this patch, PMD collapse had 3 main phases, a light weight
scanning phase (mmap_read_lock) that determines a potential PMD
collapse, an alloc phase (mmap unlocked), then finally heavier collapse
phase (mmap_write_lock).
To enabled mTHP collapse we make the following changes:
During PMD scan phase, track occupied pages in a bitmap. When mTHP
orders are enabled, we remove the restriction of max_ptes_none during the
scan phase to avoid missing potential mTHP collapse candidates. Once we
have scanned the full PMD range and updated the bitmap to track occupied
pages, we use the bitmap to find the optimal mTHP size.
Implement mthp_collapse() to walk forward through the bitmap and
determine the best eligible order for each naturally-aligned region. The
algorithm starts at the beginning of the PMD range and, for each offset,
tries the highest order that fits the alignment. If the number of
occupied PTEs in that region satisfies the max_ptes_none threshold for
that order, a collapse is attempted. On failure, the order is
decremented and the same offset is retried at the next smaller size. Once
the smallest enabled order is exhausted (or a collapse succeeds), the
offset advances past the region just processed, and the next attempt
starts at the highest order permitted by the new offset's natural
alignment.
The algorithm works as follows:
1) set offset=0 and order=HPAGE_PMD_ORDER
2) if the order is not enabled, go to step (5)
3) count occupied PTEs in the (offset, order) range using
bitmap_weight_from()
4) if the count satisfies the max_ptes_none threshold, attempt
collapse; on success, advance to step (6)
5) if a smaller enabled order exists, decrement order and retry
from step (2) at the same offset
6) advance offset past the current region and compute the next
order from the new offset's natural alignment via __ffs(offset),
capped at HPAGE_PMD_ORDER
7) repeat from step (2) until the full PMD range is covered
mTHP collapses reject regions containing swapped out or shared pages.
This is because adding new entries can lead to new none pages, and these
may lead to constant promotion into a higher order mTHP. A similar
issue can occur with "max_ptes_none > HPAGE_PMD_NR/2" due to a collapse
introducing at least 2x the number of pages, and on a future scan will
satisfy the promotion condition once again. This issue is prevented via
the collapse_max_ptes_none() function which imposes the max_ptes_none
restrictions above.
We currently only support mTHP collapse for max_ptes_none values of 0
and HPAGE_PMD_NR - 1. resulting in the following behavior:
- max_ptes_none=0: Never introduce new empty pages during collapse
- max_ptes_none=HPAGE_PMD_NR-1: Always try collapse to the highest
available mTHP order
Any other max_ptes_none value will emit a warning and default mTHP
collapse to max_ptes_none=0. There should be no behavior change for PMD
collapse.
Once we determine what mTHP sizes fits best in that PMD range a collapse
is attempted. A minimum collapse order of 2 is used as this is the lowest
order supported by anon memory as defined by THP_ORDERS_ALL_ANON.
Currently madv_collapse is not supported and will only attempt PMD
collapse.
We can also remove the check for is_khugepaged inside the PMD scan as
the collapse_max_ptes_none() function handles this logic now.
Signed-off-by: Nico Pache <npache@redhat.com>
---
mm/khugepaged.c | 146 +++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 138 insertions(+), 8 deletions(-)
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index ec886a031952..430047316f43 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -99,6 +99,8 @@ static DEFINE_READ_MOSTLY_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
static struct kmem_cache *mm_slot_cache __ro_after_init;
+#define KHUGEPAGED_MIN_MTHP_ORDER 2
+
struct collapse_control {
bool is_khugepaged;
@@ -110,6 +112,9 @@ struct collapse_control {
/* nodemask for allocation fallback */
nodemask_t alloc_nmask;
+
+ /* Each bit represents a single occupied (!none/zero) page. */
+ DECLARE_BITMAP(mthp_present_ptes, MAX_PTRS_PER_PTE);
};
/**
@@ -1440,20 +1445,130 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s
return result;
}
+/* Return the highest naturally aligned order that fits at @offset within a PMD. */
+static unsigned int max_order_from_offset(unsigned int offset)
+{
+ if (offset == 0)
+ return HPAGE_PMD_ORDER;
+
+ return min_t(unsigned int, __ffs(offset), HPAGE_PMD_ORDER);
+}
+
+/*
+ * mthp_collapse() consumes the bitmap that is generated during
+ * collapse_scan_pmd() to determine what regions and mTHP orders fit best.
+ *
+ * Each bit in cc->mthp_present_ptes represents a single occupied (!none/zero)
+ * page. We start at the PMD order and check if it is eligible for collapse;
+ * if not, we check the left and right halves of the PTE page table we are
+ * examining at a lower order.
+ *
+ * For each of these, we determine how many PTE entries are occupied in the
+ * range of PTE entries we propose to collapse, then we compare this to a
+ * threshold number of PTE entries which would need to be occupied for a
+ * collapse to be permitted at that order (accounting for max_ptes_none).
+ *
+ * If a collapse is permitted, we attempt to collapse the PTE range into a
+ * mTHP.
+ */
+static enum scan_result mthp_collapse(struct mm_struct *mm,
+ unsigned long address, int referenced, int unmapped,
+ struct collapse_control *cc, unsigned long enabled_orders)
+{
+ unsigned int nr_occupied_ptes, nr_ptes, max_ptes_none;
+ enum scan_result last_result = SCAN_FAIL;
+ int collapsed = 0;
+ bool alloc_failed = false;
+ unsigned long collapse_address;
+ unsigned int offset = 0;
+ unsigned int order = HPAGE_PMD_ORDER;
+
+ while (offset < HPAGE_PMD_NR) {
+ nr_ptes = 1UL << order;
+
+ if (!test_bit(order, &enabled_orders))
+ goto next_order;
+
+ max_ptes_none = collapse_max_ptes_none(cc, NULL, order);
+ nr_occupied_ptes = bitmap_weight_from(cc->mthp_present_ptes, offset,
+ offset + nr_ptes);
+
+ if (nr_occupied_ptes >= nr_ptes - max_ptes_none) {
+ enum scan_result ret;
+
+ collapse_address = address + offset * PAGE_SIZE;
+ ret = collapse_huge_page(mm, collapse_address, referenced,
+ unmapped, cc, order);
+ switch (ret) {
+ /* Cases where we continue to next collapse candidate */
+ case SCAN_SUCCEED:
+ collapsed += nr_ptes;
+ fallthrough;
+ case SCAN_PTE_MAPPED_HUGEPAGE:
+ goto next_offset;
+ /* Cases where lower orders might still succeed */
+ case SCAN_ALLOC_HUGE_PAGE_FAIL:
+ alloc_failed = true;
+ last_result = ret;
+ goto next_order;
+ /* Cases where no further collapse is possible */
+ case SCAN_PMD_MAPPED:
+ fallthrough;
+ default:
+ last_result = ret;
+ goto done;
+ }
+ }
+
+next_order:
+ /*
+ * Continue with the next smaller order if there is still
+ * any smaller order enabled. When at the smallest order
+ * we must always move to the next offset.
+ */
+ if (order > KHUGEPAGED_MIN_MTHP_ORDER &&
+ (enabled_orders & GENMASK(order - 1, 0))) {
+ order--;
+ continue;
+ }
+next_offset:
+ /*
+ * Advance past the region we just processed and determine the
+ * highest order we can attempt next. Since huge pages must be
+ * naturally aligned, the max order we can attempt next is
+ * limited by the alignment of the new offset.
+ * E.g. if we collapsed a order-2 mTHP at offset 0, offset
+ * becomes 4 and __ffs(4) == 2, so the next attempt starts at
+ * order 2.
+ */
+ offset += nr_ptes;
+ order = max_order_from_offset(offset);
+ }
+done:
+ if (collapsed)
+ return SCAN_SUCCEED;
+ if (alloc_failed)
+ return SCAN_ALLOC_HUGE_PAGE_FAIL;
+ return last_result;
+}
+
static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
struct vm_area_struct *vma, unsigned long start_addr,
bool *lock_dropped, struct collapse_control *cc)
{
- const unsigned int max_ptes_none = collapse_max_ptes_none(cc, vma, HPAGE_PMD_ORDER);
const unsigned int max_ptes_shared = collapse_max_ptes_shared(cc, HPAGE_PMD_ORDER);
const unsigned int max_ptes_swap = collapse_max_ptes_swap(cc, HPAGE_PMD_ORDER);
+ unsigned int max_ptes_none = collapse_max_ptes_none(cc, vma, HPAGE_PMD_ORDER);
+ enum tva_type tva_flags = cc->is_khugepaged ? TVA_KHUGEPAGED : TVA_FORCED_COLLAPSE;
pmd_t *pmd;
- pte_t *pte, *_pte;
+ pte_t *pte, *_pte, pteval;
+ int i;
int none_or_zero = 0, shared = 0, referenced = 0;
enum scan_result result = SCAN_FAIL;
struct page *page = NULL;
struct folio *folio = NULL;
unsigned long addr;
+ unsigned long enabled_orders;
spinlock_t *ptl;
int node = NUMA_NO_NODE, unmapped = 0;
@@ -1465,8 +1580,19 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
goto out;
}
+ bitmap_zero(cc->mthp_present_ptes, MAX_PTRS_PER_PTE);
memset(cc->node_load, 0, sizeof(cc->node_load));
nodes_clear(cc->alloc_nmask);
+
+ enabled_orders = collapse_possible_orders(vma, vma->vm_flags, tva_flags);
+
+ /*
+ * If PMD is the only enabled order, enforce max_ptes_none, otherwise
+ * scan all pages to populate the bitmap for mTHP collapse.
+ */
+ if (enabled_orders != BIT(HPAGE_PMD_ORDER))
+ max_ptes_none = KHUGEPAGED_MAX_PTES_LIMIT;
+
pte = pte_offset_map_lock(mm, pmd, start_addr, &ptl);
if (!pte) {
cc->progress++;
@@ -1474,11 +1600,13 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
goto out;
}
- for (addr = start_addr, _pte = pte; _pte < pte + HPAGE_PMD_NR;
- _pte++, addr += PAGE_SIZE) {
+ for (i = 0; i < HPAGE_PMD_NR; i++) {
+ _pte = pte + i;
+ addr = start_addr + i * PAGE_SIZE;
+ pteval = ptep_get(_pte);
+
cc->progress++;
- pte_t pteval = ptep_get(_pte);
if (pte_none_or_zero(pteval)) {
if (++none_or_zero > max_ptes_none) {
result = SCAN_EXCEED_NONE_PTE;
@@ -1558,6 +1686,8 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
}
}
+ /* Set bit for occupied pages */
+ __set_bit(i, cc->mthp_present_ptes);
/*
* Record which node the original page is from and save this
* information to cc->node_load[].
@@ -1616,9 +1746,9 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
if (result == SCAN_SUCCEED) {
/* collapse_huge_page expects the lock to be dropped before calling */
mmap_read_unlock(mm);
- result = collapse_huge_page(mm, start_addr, referenced,
- unmapped, cc, HPAGE_PMD_ORDER);
- /* collapse_huge_page will return with the mmap_lock released */
+ result = mthp_collapse(mm, start_addr, referenced,
+ unmapped, cc, enabled_orders);
+ /* mmap_lock was released above, set lock_dropped */
*lock_dropped = true;
}
out:
--
2.54.0
^ permalink raw reply related
* [PATCH mm-unstable v19 10/14] mm/khugepaged: introduce collapse_possible_orders helper functions
From: Nico Pache @ 2026-06-05 16:14 UTC (permalink / raw)
To: linux-doc, linux-kernel, linux-mm, linux-trace-kernel
Cc: aarcange, akpm, anshuman.khandual, apopple, baohua, baolin.wang,
byungchul, catalin.marinas, cl, corbet, dave.hansen, david,
dev.jain, gourry, hannes, hughd, jack, jackmanb, jannh, jglisse,
joshua.hahnjy, kas, lance.yang, liam, ljs, mathieu.desnoyers,
matthew.brost, mhiramat, mhocko, npache, peterx, pfalcato,
rakie.kim, raquini, rdunlap, richard.weiyang, rientjes, rostedt,
rppt, ryan.roberts, shivankg, sunnanyong, surenb,
thomas.hellstrom, tiwai, usamaarif642, vbabka, vishal.moola,
wangkefeng.wang, will, willy, yang, ying.huang, ziy, zokeefe
In-Reply-To: <20260605161422.213817-1-npache@redhat.com>
Add collapse_possible_orders() to generalize THP order eligibility. The
function determines which THP orders are permitted based on collapse
context (khugepaged vs madv_collapse). We also add collapse_possible()
as a thin wrapper around collapse_possible_orders() that returns a bool
rather than the whole bitmap.
This consolidates collapse configuration logic and provides a clean
interface for future mTHP collapse support where the orders may be
different.
Acked-by: David Hildenbrand (Arm) <david@kernel.org>
Reviewed-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Signed-off-by: Nico Pache <npache@redhat.com>
---
mm/khugepaged.c | 24 +++++++++++++++++++++---
1 file changed, 21 insertions(+), 3 deletions(-)
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 26c343a6fa3d..ec886a031952 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -554,12 +554,30 @@ void __khugepaged_enter(struct mm_struct *mm)
wake_up_interruptible(&khugepaged_wait);
}
+/*
+ * Check what orders are possible based on the vma and collapse type.
+ * This is used to determine if mTHP collapse is a viable option.
+ */
+static unsigned long collapse_possible_orders(struct vm_area_struct *vma,
+ vm_flags_t vm_flags, enum tva_type tva_flags)
+{
+ const unsigned long orders = BIT(HPAGE_PMD_ORDER);
+
+ return thp_vma_allowable_orders(vma, vm_flags, tva_flags, orders);
+}
+
+static bool collapse_possible(struct vm_area_struct *vma,
+ vm_flags_t vm_flags, enum tva_type tva_flags)
+{
+ return collapse_possible_orders(vma, vm_flags, tva_flags);
+}
+
void khugepaged_enter_vma(struct vm_area_struct *vma,
vm_flags_t vm_flags)
{
if (!mm_flags_test(MMF_VM_HUGEPAGE, vma->vm_mm) &&
hugepage_pmd_enabled()) {
- if (thp_vma_allowable_order(vma, vm_flags, TVA_KHUGEPAGED, PMD_ORDER))
+ if (collapse_possible(vma, vm_flags, TVA_KHUGEPAGED))
__khugepaged_enter(vma->vm_mm);
}
}
@@ -2700,7 +2718,7 @@ static void collapse_scan_mm_slot(unsigned int progress_max,
cc->progress++;
break;
}
- if (!thp_vma_allowable_order(vma, vma->vm_flags, TVA_KHUGEPAGED, PMD_ORDER)) {
+ if (!collapse_possible(vma, vma->vm_flags, TVA_KHUGEPAGED)) {
cc->progress++;
continue;
}
@@ -3010,7 +3028,7 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start,
BUG_ON(vma->vm_start > start);
BUG_ON(vma->vm_end < end);
- if (!thp_vma_allowable_order(vma, vma->vm_flags, TVA_FORCED_COLLAPSE, PMD_ORDER))
+ if (!collapse_possible(vma, vma->vm_flags, TVA_FORCED_COLLAPSE))
return -EINVAL;
cc = kmalloc_obj(*cc);
--
2.54.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox