[PATCH v1 11/12] KVM: arm64: Add hardware-accelerated dirty-ring cleaning routine

Leonardo Bras
2026-04-30T11:16:35Z

Implement arm64 version of kvm_arch_dirty_ring_clear() making use of
FEAT_HACDBS.

It works by transversing the dirty-ring and converting its entries into
HDBSS entries based on the slot offset.

The resulting HDBSS array is then fed to the HACDBS mechanism that walks
the pagetable marking writable-dirty pages as writable-clean.

Only successfully cleaned entries are set as invalid on the dirty-ring, so
in case of error, falling back to generic software cleaning will take care
of any remaining entry in the dirty-ring.

Signed-off-by: Leonardo Bras <leo.bras@arm.com>
---
arch/arm64/include/asm/kvm_dirty_bit.h | 13 +++++
arch/arm64/kvm/dirty_bit.c | 66 ++++++++++++++++++++++++++
2 files changed, 79 insertions(+)

diff --git a/arch/arm64/include/asm/kvm_dirty_bit.h b/arch/arm64/include/asm/kvm_dirty_bit.h
index 3d749f979c67..d76c109937d8 100644
--- a/arch/arm64/include/asm/kvm_dirty_bit.h
+++ b/arch/arm64/include/asm/kvm_dirty_bit.h
@@ -26,29 +26,42 @@ DECLARE_PER_CPU(struct hacdbs, hacdbs_pcp);
void __init kvm_hacdbs_init(void);
void kvm_hacdbs_cpu_up(void);
void kvm_hacdbs_cpu_down(void);

int __kvm_arch_dirty_log_clear(struct kvm *kvm,
struct kvm_memory_slot *memslot,
struct kvm_clear_dirty_log *log,
unsigned long *bitmap,
bool *flush);

+int __kvm_arch_dirty_ring_clear(struct kvm *kvm, struct kvm_dirty_ring *ring,
+ int *nr_entries_reset);
+
static inline bool kvm_arch_dirty_clear_enabled(struct kvm *kvm)
{
return this_cpu_read(hacdbs_pcp.status) == HACDBS_IDLE &&
(kvm->arch.mmu.pgt->flags & KVM_PGTABLE_S2_DBM);
}

static inline int kvm_arch_dirty_log_clear(struct kvm *kvm,
struct kvm_memory_slot *memslot,
struct kvm_clear_dirty_log *log,
unsigned long *bitmap,
bool *flush)
{
if (!kvm_arch_dirty_clear_enabled(kvm))
return -EPERM;

return __kvm_arch_dirty_log_clear(kvm, memslot, log, bitmap, flush);
}

+static inline int kvm_arch_dirty_ring_clear(struct kvm *kvm,
+ struct kvm_dirty_ring *ring,
+ int *nr_entries_reset)
+{
+ if (!kvm_arch_dirty_clear_enabled(kvm))
+ return -EPERM;
+
+ return __kvm_arch_dirty_ring_clear(kvm, ring, nr_entries_reset);
+}
+
#endif /* __ARM64_KVM_DIRTY_BIT_H__ */
diff --git a/arch/arm64/kvm/dirty_bit.c b/arch/arm64/kvm/dirty_bit.c
index 0b7dcb8467c0..b99808a36469 100644
--- a/arch/arm64/kvm/dirty_bit.c
+++ b/arch/arm64/kvm/dirty_bit.c
@@ -256,20 +256,86 @@ int __kvm_arch_dirty_log_clear(struct kvm *kvm,

ret = -EAGAIN;
}

write_unlock(&kvm->mmu_lock);
kfree(hw_entries);

return ret;
}

+int __kvm_arch_dirty_ring_clear(struct kvm *kvm, struct kvm_dirty_ring *ring,
+ int *nr_entries_reset)
+{
+ u64 *hw_entries;
+ u64 slot_offset = 0;
+ u64 ttwl;
+ int i, ret;
+ u32 slot = -1;
+
+ if (signal_pending(current))
+ return -EINTR;
+
+ ttwl = hdbss_get_ttwl(kvm->arch.mmu.split_page_chunk_size);
+
+ hw_entries = kmalloc(max(ring->size * sizeof(u64), PAGE_SIZE), GFP_KERNEL);
+ if (!hw_entries)
+ return -ENOMEM;
+
+ for (i = 0; i < ring->size; i++) {
+ struct kvm_dirty_gfn *entry;
+ gfn_t gfn;
+
+ entry = &ring->dirty_gfns[(ring->reset_index + i) &
+ (ring->size - 1)];
+
+ if (!kvm_dirty_gfn_harvested(entry))
+ break;
+
+ if (entry->slot != slot) {
+ struct kvm_memory_slot *memslot;
+
+ memslot = kvm_dirty_ring_get_memslot(kvm, entry->slot);
+ slot = entry->slot;
+ slot_offset = memslot->base_gfn;
+ }
+
+ gfn = slot_offset + entry->offset;
+
+ hw_entries[i] = (gfn_to_gpa(gfn) & HDBSS_ENTRY_IPA) |
+ ttwl | HDBSS_ENTRY_VALID;
+ }
+
+ ret = dirty_bit_clear(kvm, hw_entries, i);
+
+ /* Set as invalid all successfully cleaned entries */
+ for (int j = 0; j < ret; j++) {
+ struct kvm_dirty_gfn *entry;
+
+ entry = &ring->dirty_gfns[(ring->reset_index + j) &
+ (ring->size - 1)];
+
+ kvm_dirty_gfn_set_invalid(entry);
+ }
+
+ /* In case of error, try software cleaning from the faulting entry */
+ ring->reset_index += ret;
+ *nr_entries_reset += ret;
+
+ kfree(hw_entries);
+
+ if (ret < i)
+ return -EFAULT;
+
+ return ret;
+}
+
static irqreturn_t hacdbsirq_handler(int irq, void *pcpu)
{
u64 cons = read_sysreg_s(SYS_HACDBSCONS_EL2);
unsigned long err = FIELD_GET(HACDBSCONS_EL2_ERR_REASON, cons);

switch (err) {
case HACDBSCONS_EL2_ERR_REASON_NOF:
this_cpu_write(hacdbs_pcp.status, HACDBS_IDLE);
break;
case HACDBSCONS_EL2_ERR_REASON_IPAHACF:
--
2.54.0