* [PATCH v3 01/11] KVM: arm64: HDBSS bits
2026-07-29 10:45 [PATCH v3 00/11] KVM Dirty-bit cleaning hw accelerator (HACDBS) Leonardo Bras
@ 2026-07-29 10:45 ` Leonardo Bras
2026-07-29 10:45 ` [PATCH v3 02/11] arm64/cpufeature: Add system-wide FEAT_HACDBS detection Leonardo Bras
` (9 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Leonardo Bras @ 2026-07-29 10:45 UTC (permalink / raw)
To: Catalin Marinas, Will Deacon, Marc Zyngier, Oliver Upton,
Fuad Tabba, Joey Gouly, Steffen Eiden, Suzuki K Poulose,
Zenghui Yu, Paolo Bonzini, James Morse, Leonardo Bras, Yang Shi,
mrigendrachaubey, Thomas Huth, Zeng Heng, Ryan Roberts,
Mark Brown, Sascha Bischoff, Yicong Yang, Kevin Brodsky,
James Clark, Raghavendra Rao Ananta, Yeoreum Yun, Mark Rutland,
Anshuman Khandual, Tian Zheng
Cc: linux-arm-kernel, linux-kernel, kvmarm, kvm
All those bits should come from a future version of HDBSS patchset:
https://lore.kernel.org/all/20260709104026.2612599-1-zhengtian10@huawei.com/
I added them here in order to fulfill the dependencies and be able to
easily build and test this patchset, but this particular patch should *not*
be merged upstream.
It also sets a default nonzero value for chunksize, which will force eager
splitting, and allow this patchset to be properly tested. While
lazy-splitting should be possible in the next HDBSS patchset, it introduces
no new behaviours in how HACDBS shoud work, so having eager splitting
should be enough for testing.
Signed-off-by: Leonardo Bras <leo.bras@arm.com>
---
arch/arm64/include/asm/cpufeature.h | 5 +++++
arch/arm64/include/asm/kvm_dirty_bit.h | 12 ++++++++++++
arch/arm64/include/asm/kvm_pgtable.h | 3 +++
arch/arm64/kernel/cpufeature.c | 12 ++++++++++++
arch/arm64/kvm/dirty_bit.c | 16 ++++++++++++++++
arch/arm64/kvm/hyp/pgtable.c | 15 +++++++++++++--
arch/arm64/kvm/mmu.c | 8 ++++++--
arch/arm64/kvm/Makefile | 2 +-
arch/arm64/tools/cpucaps | 1 +
9 files changed, 69 insertions(+), 5 deletions(-)
create mode 100644 arch/arm64/include/asm/kvm_dirty_bit.h
create mode 100644 arch/arm64/kvm/dirty_bit.c
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index a57870fa96db..bdfab086fd94 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -856,20 +856,25 @@ static inline bool system_supports_poe(void)
static inline bool system_supports_gcs(void)
{
return alternative_has_cap_unlikely(ARM64_HAS_GCS);
}
static inline bool system_supports_haft(void)
{
return cpus_have_final_cap(ARM64_HAFT);
}
+static inline bool system_supports_hdbss(void)
+{
+ return cpus_have_final_cap(ARM64_HAS_HDBSS);
+}
+
static __always_inline bool system_supports_mpam(void)
{
return alternative_has_cap_unlikely(ARM64_MPAM);
}
static __always_inline bool system_supports_mpam_hcr(void)
{
return alternative_has_cap_unlikely(ARM64_MPAM_HCR);
}
diff --git a/arch/arm64/include/asm/kvm_dirty_bit.h b/arch/arm64/include/asm/kvm_dirty_bit.h
new file mode 100644
index 000000000000..dd16438f0651
--- /dev/null
+++ b/arch/arm64/include/asm/kvm_dirty_bit.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2026 ARM Ltd.
+ * Author: Leonardo Bras <leo.bras@arm.com>
+ */
+
+#ifndef __ARM64_KVM_DIRTY_BIT_H__
+#define __ARM64_KVM_DIRTY_BIT_H__
+
+#include <asm/kvm_pgtable.h>
+
+#endif /* __ARM64_KVM_DIRTY_BIT_H__ */
diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h
index 41a8687938eb..646ff88e0258 100644
--- a/arch/arm64/include/asm/kvm_pgtable.h
+++ b/arch/arm64/include/asm/kvm_pgtable.h
@@ -86,20 +86,22 @@ typedef u64 kvm_pte_t;
#define KVM_PTE_LEAF_ATTR_HI GENMASK(63, 50)
#define KVM_PTE_LEAF_ATTR_HI_SW GENMASK(58, 55)
#define KVM_PTE_LEAF_ATTR_HI_S1_XN BIT(54)
#define KVM_PTE_LEAF_ATTR_HI_S1_UXN BIT(54)
#define KVM_PTE_LEAF_ATTR_HI_S1_PXN BIT(53)
#define KVM_PTE_LEAF_ATTR_HI_S2_XN GENMASK(54, 53)
+#define KVM_PTE_LEAF_ATTR_HI_S2_DBM BIT(51)
+
#define KVM_PTE_LEAF_ATTR_HI_S1_GP BIT(50)
#define KVM_PTE_LEAF_ATTR_S2_PERMS (KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R | \
KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W | \
KVM_PTE_LEAF_ATTR_HI_S2_XN)
/* pKVM invalid pte encodings */
#define KVM_INVALID_PTE_TYPE_MASK GENMASK(63, 60)
#define KVM_INVALID_PTE_ANNOT_MASK ~(KVM_PTE_VALID | \
KVM_INVALID_PTE_TYPE_MASK)
@@ -246,20 +248,21 @@ struct kvm_pgtable_mm_ops {
};
/**
* enum kvm_pgtable_stage2_flags - Stage-2 page-table flags.
* @KVM_PGTABLE_S2_IDMAP: Only use identity mappings.
* @KVM_PGTABLE_S2_AS_S1: Final memory attributes are that of Stage-1.
*/
enum kvm_pgtable_stage2_flags {
KVM_PGTABLE_S2_IDMAP = BIT(0),
KVM_PGTABLE_S2_AS_S1 = BIT(1),
+ KVM_PGTABLE_S2_DBM = BIT(2),
};
/**
* enum kvm_pgtable_prot - Page-table permissions and attributes.
* @KVM_PGTABLE_PROT_UX: Unprivileged execute permission.
* @KVM_PGTABLE_PROT_PX: Privileged execute permission.
* @KVM_PGTABLE_PROT_X: Privileged and unprivileged execute permission.
* @KVM_PGTABLE_PROT_W: Write permission.
* @KVM_PGTABLE_PROT_R: Read permission.
* @KVM_PGTABLE_PROT_DEVICE: Device attributes.
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 9a22df0c5120..aa327eebaf1c 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -2124,20 +2124,25 @@ static bool has_nested_virt_support(const struct arm64_cpu_capabilities *cap,
return true;
}
static bool hvhe_possible(const struct arm64_cpu_capabilities *entry,
int __unused)
{
return arm64_test_sw_feature_override(ARM64_SW_FEATURE_OVERRIDE_HVHE);
}
+static bool has_vhe_hdbss(const struct arm64_cpu_capabilities *entry, int cope)
+{
+ return is_kernel_in_hyp_mode() && has_cpuid_feature(entry, cope);
+}
+
bool cpu_supports_bbml2_noabort(void)
{
/*
* We want to allow usage of BBML2 in as wide a range of kernel contexts
* as possible. This list is therefore an allow-list of known-good
* implementations that both support BBML2 and additionally, fulfill the
* extra constraint of never generating TLB conflict aborts when using
* the relaxed BBML2 semantics (such aborts make use of BBML2 in certain
* kernel contexts difficult to prove safe against recursive aborts).
*
@@ -2774,20 +2779,27 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
* cannot be emulated in software (no access fault will occur).
* Therefore this should be used only if it's supported system
* wide.
*/
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.capability = ARM64_HAFT,
.matches = has_cpuid_feature,
ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, HAFDBS, HAFT)
},
#endif
+ {
+ .desc = "Hardware Dirty state tracking structure (HDBSS)",
+ .type = ARM64_CPUCAP_SYSTEM_FEATURE,
+ .capability = ARM64_HAS_HDBSS,
+ .matches = has_vhe_hdbss,
+ ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, HAFDBS, HDBSS)
+ },
{
.desc = "CRC32 instructions",
.capability = ARM64_HAS_CRC32,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = has_cpuid_feature,
ARM64_CPUID_FIELDS(ID_AA64ISAR0_EL1, CRC32, IMP)
},
{
.desc = "Speculative Store Bypassing Safe (SSBS)",
.capability = ARM64_SSBS,
diff --git a/arch/arm64/kvm/dirty_bit.c b/arch/arm64/kvm/dirty_bit.c
new file mode 100644
index 000000000000..32fe938d6bf7
--- /dev/null
+++ b/arch/arm64/kvm/dirty_bit.c
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026 ARM Ltd.
+ * Author: Leonardo Bras <leo.bras@arm.com>
+ */
+
+#include <asm/kvm_dirty_bit.h>
+
+/* HDBSS entry field definitions */
+#define HDBSS_ENTRY_VALID BIT(0)
+#define HDBSS_ENTRY_TTWL_SHIFT (1)
+#define HDBSS_ENTRY_TTWL_MASK (GENMASK(3, 1))
+#define HDBSS_ENTRY_TTWL(x) \
+ (((x) << HDBSS_ENTRY_TTWL_SHIFT) & HDBSS_ENTRY_TTWL_MASK)
+#define HDBSS_ENTRY_TTWL_RESV HDBSS_ENTRY_TTWL(-4)
+#define HDBSS_ENTRY_IPA GENMASK_ULL(55, 12)
diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
index b74dd5ce1efd..0e2ab3fc6910 100644
--- a/arch/arm64/kvm/hyp/pgtable.c
+++ b/arch/arm64/kvm/hyp/pgtable.c
@@ -724,23 +724,27 @@ static int stage2_set_prot_attr(struct kvm_pgtable *pgt, enum kvm_pgtable_prot p
attr = KVM_S2_MEMATTR(pgt, NORMAL);
}
r = stage2_set_xn_attr(prot, &attr);
if (r)
return r;
if (prot & KVM_PGTABLE_PROT_R)
attr |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R;
- if (prot & KVM_PGTABLE_PROT_W)
+ if (prot & KVM_PGTABLE_PROT_W) {
attr |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W;
+ if (pgt->flags & KVM_PGTABLE_S2_DBM)
+ attr |= KVM_PTE_LEAF_ATTR_HI_S2_DBM;
+ }
+
if (!kvm_lpa2_is_enabled())
attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S2_SH, sh);
attr |= KVM_PTE_LEAF_ATTR_LO_S2_AF;
attr |= prot & KVM_PTE_LEAF_ATTR_HI_SW;
*ptep = attr;
return 0;
}
@@ -1360,23 +1364,27 @@ int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr,
kvm_pte_t xn = 0, set = 0, clr = 0;
s8 level;
int ret;
if (prot & KVM_PTE_LEAF_ATTR_HI_SW)
return -EINVAL;
if (prot & KVM_PGTABLE_PROT_R)
set |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R;
- if (prot & KVM_PGTABLE_PROT_W)
+ if (prot & KVM_PGTABLE_PROT_W) {
set |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W;
+ if (pgt->flags & KVM_PGTABLE_S2_DBM)
+ set |= KVM_PTE_LEAF_ATTR_HI_S2_DBM;
+ }
+
if (prot & KVM_PGTABLE_PROT_X) {
ret = stage2_set_xn_attr(prot, &xn);
if (ret)
return ret;
set |= xn & KVM_PTE_LEAF_ATTR_HI_S2_XN;
clr |= ~xn & KVM_PTE_LEAF_ATTR_HI_S2_XN;
}
ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level, flags);
@@ -1581,20 +1589,23 @@ int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,
u64 vtcr = mmu->vtcr;
u32 ia_bits = VTCR_EL2_IPA(vtcr);
u32 sl0 = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);
s8 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0;
pgd_sz = kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE;
pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_pages_exact(pgd_sz);
if (!pgt->pgd)
return -ENOMEM;
+ if (system_supports_hdbss())
+ flags |= KVM_PGTABLE_S2_DBM;
+
pgt->ia_bits = ia_bits;
pgt->start_level = start_level;
pgt->mm_ops = mm_ops;
pgt->mmu = mmu;
pgt->flags = flags;
pgt->force_pte_cb = force_pte_cb;
/* Ensure zeroed PGD pages are visible to the hardware walker */
dsb(ishst);
return 0;
diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index 6c941aaa10c6..e086c01a9325 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -1020,22 +1020,26 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t
mmu->last_vcpu_ran = alloc_percpu(typeof(*mmu->last_vcpu_ran));
if (!mmu->last_vcpu_ran) {
err = -ENOMEM;
goto out_destroy_pgtable;
}
for_each_possible_cpu(cpu)
*per_cpu_ptr(mmu->last_vcpu_ran, cpu) = -1;
- /* The eager page splitting is disabled by default */
- mmu->split_page_chunk_size = KVM_ARM_EAGER_SPLIT_CHUNK_SIZE_DEFAULT;
+ /* The eager page splitting is disabled by default if system has no HDBSS */
+ if (system_supports_hdbss())
+ mmu->split_page_chunk_size = 256 * 1024 * PAGE_SIZE;
+ else
+ mmu->split_page_chunk_size = KVM_ARM_EAGER_SPLIT_CHUNK_SIZE_DEFAULT;
+
mmu->split_page_cache.gfp_zero = __GFP_ZERO;
mmu->pgd_phys = __pa(pgt->pgd);
if (kvm_is_nested_s2_mmu(kvm, mmu))
kvm_init_nested_s2_mmu(mmu);
return 0;
out_destroy_pgtable:
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index 59612d2f277c..6faacd857346 100644
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -17,21 +17,21 @@ kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o \
inject_fault.o va_layout.o handle_exit.o config.o \
guest.o debug.o reset.o sys_regs.o stacktrace.o \
vgic-sys-reg-v3.o fpsimd.o pkvm.o \
arch_timer.o trng.o vmid.o emulate-nested.o nested.o at.o \
vgic/vgic.o vgic/vgic-init.o \
vgic/vgic-irqfd.o vgic/vgic-v2.o \
vgic/vgic-v3.o vgic/vgic-v4.o \
vgic/vgic-mmio.o vgic/vgic-mmio-v2.o \
vgic/vgic-mmio-v3.o vgic/vgic-kvm-device.o \
vgic/vgic-its.o vgic/vgic-debug.o vgic/vgic-v3-nested.o \
- vgic/vgic-v5.o
+ vgic/vgic-v5.o dirty_bit.o
kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o pmu.o
kvm-$(CONFIG_ARM64_PTR_AUTH) += pauth.o
kvm-$(CONFIG_PTDUMP_STAGE2_DEBUGFS) += ptdump.o
kvm-$(CONFIG_NVHE_EL2_TRACING) += hyp_trace.o
always-y := hyp_constants.h hyp-constants.s
define rule_gen_hyp_constants
diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
index 9b85a84f6fd4..a87706c9d160 100644
--- a/arch/arm64/tools/cpucaps
+++ b/arch/arm64/tools/cpucaps
@@ -62,20 +62,21 @@ HAS_RASV1P1_EXTN
HAS_RNG
HAS_SB
HAS_STAGE2_FWB
HAS_TCR2
HAS_TIDCP1
HAS_TLB_RANGE
HAS_VA52
HAS_VIRT_HOST_EXTN
HAS_WFXT
HAS_XNX
+HAS_HDBSS
HAFT
HW_DBM
KVM_HVHE
KVM_PROTECTED_MODE
MISMATCHED_CACHE_TYPE
MPAM
MPAM_HCR
MTE
MTE_ASYMM
MTE_FAR
--
2.55.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH v3 02/11] arm64/cpufeature: Add system-wide FEAT_HACDBS detection
2026-07-29 10:45 [PATCH v3 00/11] KVM Dirty-bit cleaning hw accelerator (HACDBS) Leonardo Bras
2026-07-29 10:45 ` [PATCH v3 01/11] KVM: arm64: HDBSS bits Leonardo Bras
@ 2026-07-29 10:45 ` Leonardo Bras
2026-07-29 10:45 ` [PATCH v3 03/11] arm64/sysreg: Add HACDBS consumer and base registers Leonardo Bras
` (8 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Leonardo Bras @ 2026-07-29 10:45 UTC (permalink / raw)
To: Catalin Marinas, Will Deacon, Marc Zyngier, Oliver Upton,
Fuad Tabba, Joey Gouly, Steffen Eiden, Suzuki K Poulose,
Zenghui Yu, Paolo Bonzini, James Morse, Leonardo Bras, Yang Shi,
mrigendrachaubey, Thomas Huth, Zeng Heng, Ryan Roberts,
Mark Brown, Sascha Bischoff, Yicong Yang, Kevin Brodsky,
James Clark, Raghavendra Rao Ananta, Yeoreum Yun, Mark Rutland,
Anshuman Khandual, Tian Zheng
Cc: linux-arm-kernel, linux-kernel, kvmarm, kvm
FEAT_HACDBS will only be used for dirty-bit cleaning if it is detected in
all running cpus.
Signed-off-by: Leonardo Bras <leo.bras@arm.com>
---
arch/arm64/include/asm/cpufeature.h | 5 +++++
arch/arm64/kernel/cpufeature.c | 8 ++++++++
arch/arm64/tools/cpucaps | 1 +
3 files changed, 14 insertions(+)
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index bdfab086fd94..620ae4cddb76 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -861,20 +861,25 @@ static inline bool system_supports_gcs(void)
static inline bool system_supports_haft(void)
{
return cpus_have_final_cap(ARM64_HAFT);
}
static inline bool system_supports_hdbss(void)
{
return cpus_have_final_cap(ARM64_HAS_HDBSS);
}
+static inline bool system_supports_hacdbs(void)
+{
+ return cpus_have_final_cap(ARM64_HACDBS);
+}
+
static __always_inline bool system_supports_mpam(void)
{
return alternative_has_cap_unlikely(ARM64_MPAM);
}
static __always_inline bool system_supports_mpam_hcr(void)
{
return alternative_has_cap_unlikely(ARM64_MPAM_HCR);
}
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index aa327eebaf1c..62f56bbd0a65 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -516,20 +516,21 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr3[] = {
FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_S1POE_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_S1PIE_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_SCTLRX_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_TCRX_SHIFT, 4, 0),
ARM64_FTR_END,
};
static const struct arm64_ftr_bits ftr_id_aa64mmfr4[] = {
S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR4_EL1_E2H0_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR4_EL1_NV_frac_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR4_EL1_HACDBS_SHIFT, 4, 0),
ARM64_FTR_END,
};
static const struct arm64_ftr_bits ftr_ctr[] = {
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 31, 1, 1), /* RES1 */
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_EL0_DIC_SHIFT, 1, 1),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_EL0_IDC_SHIFT, 1, 1),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_OR_ZERO_SAFE, CTR_EL0_CWG_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_OR_ZERO_SAFE, CTR_EL0_ERG_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_EL0_DminLine_SHIFT, 4, 1),
@@ -2764,20 +2765,27 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
{
.desc = "Hardware dirty bit management",
.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
.capability = ARM64_HW_DBM,
.matches = has_hw_dbm,
.cpu_enable = cpu_enable_hw_dbm,
.cpus = &dbm_cpus,
ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, HAFDBS, DBM)
},
#endif
+ {
+ .desc = "Hardware dirty bit Cleaning",
+ .type = ARM64_CPUCAP_SYSTEM_FEATURE,
+ .capability = ARM64_HACDBS,
+ .matches = has_cpuid_feature,
+ ARM64_CPUID_FIELDS(ID_AA64MMFR4_EL1, HACDBS, IMP)
+ },
#ifdef CONFIG_ARM64_HAFT
{
.desc = "Hardware managed Access Flag for Table Descriptors",
/*
* Contrary to the page/block access flag, the table access flag
* cannot be emulated in software (no access fault will occur).
* Therefore this should be used only if it's supported system
* wide.
*/
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
index a87706c9d160..bd2c0bb98da6 100644
--- a/arch/arm64/tools/cpucaps
+++ b/arch/arm64/tools/cpucaps
@@ -65,20 +65,21 @@ HAS_STAGE2_FWB
HAS_TCR2
HAS_TIDCP1
HAS_TLB_RANGE
HAS_VA52
HAS_VIRT_HOST_EXTN
HAS_WFXT
HAS_XNX
HAS_HDBSS
HAFT
HW_DBM
+HACDBS
KVM_HVHE
KVM_PROTECTED_MODE
MISMATCHED_CACHE_TYPE
MPAM
MPAM_HCR
MTE
MTE_ASYMM
MTE_FAR
MTE_STORE_ONLY
SME
--
2.55.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH v3 03/11] arm64/sysreg: Add HACDBS consumer and base registers
2026-07-29 10:45 [PATCH v3 00/11] KVM Dirty-bit cleaning hw accelerator (HACDBS) Leonardo Bras
2026-07-29 10:45 ` [PATCH v3 01/11] KVM: arm64: HDBSS bits Leonardo Bras
2026-07-29 10:45 ` [PATCH v3 02/11] arm64/cpufeature: Add system-wide FEAT_HACDBS detection Leonardo Bras
@ 2026-07-29 10:45 ` Leonardo Bras
2026-07-29 10:45 ` [PATCH v3 04/11] KVM: arm64: dirty_bit: Add base FEAT_HACDBS cleaning routine Leonardo Bras
` (7 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Leonardo Bras @ 2026-07-29 10:45 UTC (permalink / raw)
To: Catalin Marinas, Will Deacon, Marc Zyngier, Oliver Upton,
Fuad Tabba, Joey Gouly, Steffen Eiden, Suzuki K Poulose,
Zenghui Yu, Paolo Bonzini, James Morse, Leonardo Bras, Yang Shi,
mrigendrachaubey, Thomas Huth, Zeng Heng, Ryan Roberts,
Mark Brown, Sascha Bischoff, Yicong Yang, Kevin Brodsky,
James Clark, Raghavendra Rao Ananta, Yeoreum Yun, Mark Rutland,
Anshuman Khandual, Tian Zheng
Cc: linux-arm-kernel, linux-kernel, kvmarm, kvm
They will be used on a later commit to make use of the FEAT_HACDBS
mechanism if available.
Signed-off-by: Leonardo Bras <leo.bras@arm.com>
Reviewed-by: Mark Brown <broonie@kernel.org>
---
arch/arm64/tools/sysreg | 30 ++++++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 7cb61aca3797..50ec9751b44d 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -4627,20 +4627,50 @@ EndEnum
Enum 9:8 IRGN0
0b00 NC
0b01 WBWA
0b10 WT
0b11 WBnWA
EndEnum
Field 7:6 SL0
Field 5:0 T0SZ
EndSysreg
+Sysreg HACDBSBR_EL2 3 4 2 3 4
+Res0 63:56
+Field 55:12 BADDR
+Field 11 EN
+Res0 10:4
+UnsignedEnum 3:0 SZ
+ 0b0000 4K
+ 0b0001 8K
+ 0b0010 16K
+ 0b0011 32K
+ 0b0100 64K
+ 0b0101 128K
+ 0b0110 256K
+ 0b0111 512K
+ 0b1000 1M
+ 0b1001 2M
+EndEnum
+EndSysreg
+
+Sysreg HACDBSCONS_EL2 3 4 2 3 5
+UnsignedEnum 63:62 ERR_REASON
+ 0b00 NOF
+ 0b01 STRUCTF
+ 0b10 IPAF
+ 0b11 IPAHACF
+EndEnum
+Res0 61:19
+Field 18:0 INDEX
+EndSysreg
+
Sysreg GCSCR_EL2 3 4 2 5 0
Fields GCSCR_ELx
EndSysreg
Sysreg GCSPR_EL2 3 4 2 5 1
Fields GCSPR_ELx
EndSysreg
Sysreg HDBSSBR_EL2 3 4 2 3 2
Res0 63:56
--
2.55.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH v3 04/11] KVM: arm64: dirty_bit: Add base FEAT_HACDBS cleaning routine
2026-07-29 10:45 [PATCH v3 00/11] KVM Dirty-bit cleaning hw accelerator (HACDBS) Leonardo Bras
` (2 preceding siblings ...)
2026-07-29 10:45 ` [PATCH v3 03/11] arm64/sysreg: Add HACDBS consumer and base registers Leonardo Bras
@ 2026-07-29 10:45 ` Leonardo Bras
2026-07-29 10:45 ` [PATCH v3 05/11] kvm: Add arch-generic interface for hw-accelerated dirty-bitmap cleaning Leonardo Bras
` (6 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Leonardo Bras @ 2026-07-29 10:45 UTC (permalink / raw)
To: Catalin Marinas, Will Deacon, Marc Zyngier, Oliver Upton,
Fuad Tabba, Joey Gouly, Steffen Eiden, Suzuki K Poulose,
Zenghui Yu, Paolo Bonzini, James Morse, Leonardo Bras, Yang Shi,
mrigendrachaubey, Thomas Huth, Zeng Heng, Ryan Roberts,
Mark Brown, Sascha Bischoff, Yicong Yang, Kevin Brodsky,
James Clark, Raghavendra Rao Ananta, Yeoreum Yun, Mark Rutland,
Anshuman Khandual, Tian Zheng
Cc: linux-arm-kernel, linux-kernel, kvmarm, kvm
Introduce the basic cleaning routine that is going to be used for both
dirty-bitmap and dirty-ring routines.
It sets the required registers with the input buffer, and wait for
HACDBS to finish, which means either the task is done, or there was some
error during processing.
It is ran with preemption disabled, as a task being scheduled in could
change the translation registers used by HACDBS and end up corrupting the
current dirty-bit tracking and the sched-in task's S2 pagetables.
Signed-off-by: Leonardo Bras <leo.bras@arm.com>
---
arch/arm64/kvm/dirty_bit.c | 176 +++++++++++++++++++++++++++++++++++++
1 file changed, 176 insertions(+)
diff --git a/arch/arm64/kvm/dirty_bit.c b/arch/arm64/kvm/dirty_bit.c
index 32fe938d6bf7..33826bbd16d9 100644
--- a/arch/arm64/kvm/dirty_bit.c
+++ b/arch/arm64/kvm/dirty_bit.c
@@ -1,16 +1,192 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2026 ARM Ltd.
* Author: Leonardo Bras <leo.bras@arm.com>
*/
#include <asm/kvm_dirty_bit.h>
+#include <asm/kvm_mmu.h>
/* HDBSS entry field definitions */
#define HDBSS_ENTRY_VALID BIT(0)
#define HDBSS_ENTRY_TTWL_SHIFT (1)
#define HDBSS_ENTRY_TTWL_MASK (GENMASK(3, 1))
#define HDBSS_ENTRY_TTWL(x) \
(((x) << HDBSS_ENTRY_TTWL_SHIFT) & HDBSS_ENTRY_TTWL_MASK)
#define HDBSS_ENTRY_TTWL_RESV HDBSS_ENTRY_TTWL(-4)
#define HDBSS_ENTRY_IPA GENMASK_ULL(55, 12)
+
+struct hacdbs_save {
+ u64 hacdbsbr_el2;
+ u64 hacdbscons_el2;
+ u64 vttbr_el2;
+ u64 vtcr_el2;
+ bool hcr_vm_set;
+};
+
+struct hacdbs_sched_save {
+ struct preempt_notifier notifier;
+ struct hacdbs_save save;
+};
+
+static int hacdbs_last_entry(int size)
+{
+ u64 cons = read_sysreg_s(SYS_HACDBSCONS_EL2);
+ int index = FIELD_GET(HACDBSCONS_EL2_INDEX, cons);
+
+ switch (FIELD_GET(HACDBSCONS_EL2_ERR_REASON, cons)) {
+ case HACDBSCONS_EL2_ERR_REASON_NOF:
+ return size;
+ case HACDBSCONS_EL2_ERR_REASON_IPAHACF:
+ /* When size not a power of two >= 4k, exit with reserved TTLW */
+ if (index >= size)
+ return size;
+
+ fallthrough;
+ default:
+ /* In case of error, INDEX should point the faulty entry */
+ return index;
+ }
+}
+
+static void hacdbs_start(u64 *hw_entries, int size)
+{
+ u64 br;
+ /* Each entry is 8 bytes */
+ int size_b = size * sizeof(hw_entries[0]);
+ int size_p2 = max(roundup_pow_of_two(size_b), PAGE_SIZE);
+
+ /* If not using the full size of the array, put a stop entry at the end */
+ if (size_b < size_p2)
+ hw_entries[size] = HDBSS_ENTRY_VALID | HDBSS_ENTRY_TTWL_RESV;
+
+ sysreg_clear_set_s(SYS_HACDBSCONS_EL2,
+ HACDBSCONS_EL2_ERR_REASON | HACDBSCONS_EL2_INDEX, 0);
+
+ br = (virt_to_phys(hw_entries) & HACDBSBR_EL2_BADDR_MASK) |
+ FIELD_PREP(HACDBSBR_EL2_SZ, ilog2(size_p2) - 12) |
+ FIELD_PREP(HACDBSBR_EL2_EN, 1);
+
+ /* All writes to the array have to complete before starting */
+ dsb(ishst);
+
+ write_sysreg_s(br, SYS_HACDBSBR_EL2);
+ isb();
+}
+
+static int hacdbs_stop(int size)
+{
+ int idx;
+
+ write_sysreg_s(0, SYS_HACDBSBR_EL2);
+ isb();
+
+ idx = hacdbs_last_entry(size);
+
+ return idx;
+}
+
+static void hacdbs_sched_in(struct preempt_notifier *notifier, int cpu)
+{
+ struct hacdbs_sched_save *save = container_of(notifier,
+ struct hacdbs_sched_save,
+ notifier);
+
+ write_sysreg_s(save->save.vtcr_el2, SYS_VTCR_EL2);
+ write_sysreg_s(save->save.vttbr_el2, SYS_VTTBR_EL2);
+ isb();
+
+ if (save->save.hcr_vm_set) {
+ sysreg_clear_set_hcr(0, HCR_EL2_VM);
+ isb();
+ }
+
+ write_sysreg_s(save->save.hacdbscons_el2, SYS_HACDBSCONS_EL2);
+ write_sysreg_s(save->save.hacdbsbr_el2, SYS_HACDBSBR_EL2);
+ isb();
+}
+
+static void hacdbs_sched_out(struct preempt_notifier *notifier,
+ struct task_struct *next)
+{
+ struct hacdbs_sched_save *save = container_of(notifier,
+ struct hacdbs_sched_save,
+ notifier);
+
+ if (read_sysreg_s(SYS_HACDBSBR_EL2) & HACDBSBR_EL2_EN) {
+ save->save.hacdbsbr_el2 = HACDBSBR_EL2_EN;
+ sysreg_clear_set_s(SYS_HACDBSBR_EL2, HACDBSBR_EL2_EN, 0);
+ isb();
+ } else {
+ save->save.hacdbsbr_el2 = 0;
+ }
+
+ save->save.hacdbscons_el2 = read_sysreg_s(SYS_HACDBSCONS_EL2);
+ save->save.hacdbsbr_el2 |= read_sysreg_s(SYS_HACDBSBR_EL2);
+ save->save.vttbr_el2 = read_sysreg_s(SYS_VTTBR_EL2);
+ save->save.vtcr_el2 = read_sysreg_s(SYS_VTCR_EL2);
+
+ if (read_sysreg_s(SYS_HCR_EL2) & HCR_EL2_VM) {
+ sysreg_clear_set_hcr(HCR_EL2_VM, 0);
+ isb();
+ save->save.hcr_vm_set = true;
+ } else {
+ save->save.hcr_vm_set = false;
+ }
+}
+
+static struct preempt_ops hacdbs_preempt_ops = {
+ .sched_in = hacdbs_sched_in,
+ .sched_out = hacdbs_sched_out,
+};
+
+/*
+ * Clears dirty-bits for an array of pages (hw_entries) using HACDBS
+ * Returns the number of items cleaned from the array. If returns value < size,
+ * there was an error in the processing.
+ */
+static int dirty_bit_clear(struct kvm *kvm, u64 *hw_entries, int size)
+{
+ int ret;
+ u64 cons;
+ struct hacdbs_sched_save save;
+
+ preempt_notifier_init(&save.notifier, &hacdbs_preempt_ops);
+ preempt_disable();
+ preempt_notifier_register(&save.notifier);
+ preempt_enable();
+
+ __load_stage2(&kvm->arch.mmu);
+ sysreg_clear_set_hcr(0, HCR_EL2_VM);
+ isb();
+
+ hacdbs_start(hw_entries, size);
+
+ do {
+ cons = read_sysreg_s(SYS_HACDBSCONS_EL2);
+ if (FIELD_GET(HACDBSCONS_EL2_ERR_REASON, cons))
+ break;
+
+ if (FIELD_GET(HACDBSCONS_EL2_INDEX, cons) >= size)
+ break;
+
+ udelay(1);
+ } while (true);
+
+ ret = hacdbs_stop(size);
+
+ sysreg_clear_set_hcr(HCR_EL2_VM, 0);
+ isb();
+
+ /*
+ * No DSB is needed here, as kvm_flush_remote_tlbs_memslot() that happens
+ * later in generic dirty-cleaning code already performs a DSB before
+ * doing the TLBI.
+ */
+
+ preempt_disable();
+ preempt_notifier_unregister(&save.notifier);
+ preempt_enable();
+
+ return ret;
+}
--
2.55.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH v3 05/11] kvm: Add arch-generic interface for hw-accelerated dirty-bitmap cleaning
2026-07-29 10:45 [PATCH v3 00/11] KVM Dirty-bit cleaning hw accelerator (HACDBS) Leonardo Bras
` (3 preceding siblings ...)
2026-07-29 10:45 ` [PATCH v3 04/11] KVM: arm64: dirty_bit: Add base FEAT_HACDBS cleaning routine Leonardo Bras
@ 2026-07-29 10:45 ` Leonardo Bras
2026-07-29 10:45 ` [PATCH v3 06/11] KVM: arm64: Add hardware-accelerated dirty-bitmap cleaning routine Leonardo Bras
` (5 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Leonardo Bras @ 2026-07-29 10:45 UTC (permalink / raw)
To: Catalin Marinas, Will Deacon, Marc Zyngier, Oliver Upton,
Fuad Tabba, Joey Gouly, Steffen Eiden, Suzuki K Poulose,
Zenghui Yu, Paolo Bonzini, James Morse, Leonardo Bras, Yang Shi,
mrigendrachaubey, Thomas Huth, Zeng Heng, Ryan Roberts,
Mark Brown, Sascha Bischoff, Yicong Yang, Kevin Brodsky,
James Clark, Raghavendra Rao Ananta, Yeoreum Yun, Mark Rutland,
Anshuman Khandual, Tian Zheng
Cc: linux-arm-kernel, linux-kernel, kvmarm, kvm
Introduce kvm_arch_dirty_log_clear() that allow implementation of
arch-specific hardware-accelerated dirty-log routines.
A call to that is added on both kvm_get_dirty_log_protect() and
kvm_clear_dirty_log_protect() and will fall back to software version if
not implemented, or any error was detected in the arch-specific routine.
For an arch to implement this function, it's required to provide an
asm/kvm_dirty_bit.h and have CONFIG_HAVE_KVM_HW_DIRTY_BIT=y on building.
If the arch does not implement it, and thus lack above config, the
introduced snippet is expected to be compiled-out and have zero impact at
runtime.
Signed-off-by: Leonardo Bras <leo.bras@arm.com>
---
include/linux/kvm_dirty_bit.h | 27 +++++++++++++++++++++++++++
virt/kvm/kvm_main.c | 13 ++++++++++++-
virt/kvm/Kconfig | 3 +++
3 files changed, 42 insertions(+), 1 deletion(-)
create mode 100644 include/linux/kvm_dirty_bit.h
diff --git a/include/linux/kvm_dirty_bit.h b/include/linux/kvm_dirty_bit.h
new file mode 100644
index 000000000000..fa4f6b67b623
--- /dev/null
+++ b/include/linux/kvm_dirty_bit.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2026 ARM Ltd.
+ * Author: Leonardo Bras <leo.bras@arm.com>
+ */
+
+#ifndef __KVM_DIRTY_BIT_H__
+#define __KVM_DIRTY_BIT_H__
+
+#ifndef CONFIG_HAVE_KVM_HW_DIRTY_BIT
+
+static inline int kvm_arch_dirty_log_clear(struct kvm *kvm,
+ struct kvm_memory_slot *memslot,
+ struct kvm_clear_dirty_log *log,
+ unsigned long *bitmap,
+ bool *flush)
+{
+ return -ENXIO;
+}
+
+#else /* CONFIG_HAVE_KVM_HW_DIRTY_BIT */
+
+#include <asm/kvm_dirty_bit.h>
+
+#endif /* CONFIG_HAVE_KVM_HW_DIRTY_BIT */
+
+#endif /* __KVM_DIRTY_BIT_H__ */
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 45e784462ec6..7a13ef69b6e3 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -58,20 +58,21 @@
#include "async_pf.h"
#include "kvm_mm.h"
#include "vfio.h"
#include <trace/events/ipi.h>
#define CREATE_TRACE_POINTS
#include <trace/events/kvm.h>
#include <linux/kvm_dirty_ring.h>
+#include <linux/kvm_dirty_bit.h>
/* Worst case buffer size needed for holding an integer. */
#define ITOA_MAX_LEN 12
MODULE_AUTHOR("Qumranet");
MODULE_DESCRIPTION("Kernel-based Virtual Machine (KVM) Hypervisor");
MODULE_LICENSE("GPL");
/* Architectures should define their poll value according to the halt latency */
@@ -2255,39 +2256,44 @@ static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log)
* is some code duplication between this function and
* kvm_get_dirty_log, but hopefully all architecture
* transition to kvm_get_dirty_log_protect and kvm_get_dirty_log
* can be eliminated.
*/
dirty_bitmap_buffer = dirty_bitmap;
} else {
dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
memset(dirty_bitmap_buffer, 0, n);
+ if (kvm_arch_dirty_log_clear(kvm, memslot, NULL,
+ dirty_bitmap_buffer, &flush) >= 0)
+ goto out;
+
KVM_MMU_LOCK(kvm);
for (i = 0; i < n / sizeof(long); i++) {
unsigned long mask;
gfn_t offset;
if (!dirty_bitmap[i])
continue;
flush = true;
mask = xchg(&dirty_bitmap[i], 0);
dirty_bitmap_buffer[i] = mask;
offset = i * BITS_PER_LONG;
kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
offset, mask);
}
KVM_MMU_UNLOCK(kvm);
}
+out:
if (flush)
kvm_flush_remote_tlbs_memslot(kvm, memslot);
if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
return -EFAULT;
return 0;
}
/**
@@ -2366,45 +2372,50 @@ static int kvm_clear_dirty_log_protect(struct kvm *kvm,
(log->num_pages < memslot->npages - log->first_page && (log->num_pages & 63)))
return -EINVAL;
kvm_arch_sync_dirty_log(kvm, memslot);
flush = false;
dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n))
return -EFAULT;
+ if (kvm_arch_dirty_log_clear(kvm, memslot, log, dirty_bitmap_buffer,
+ &flush) >= 0)
+ goto out;
+
KVM_MMU_LOCK(kvm);
for (offset = log->first_page, i = offset / BITS_PER_LONG,
n = DIV_ROUND_UP(log->num_pages, BITS_PER_LONG); n--;
i++, offset += BITS_PER_LONG) {
unsigned long mask = *dirty_bitmap_buffer++;
atomic_long_t *p = (atomic_long_t *) &dirty_bitmap[i];
if (!mask)
continue;
mask &= atomic_long_fetch_andnot(mask, p);
/*
* mask contains the bits that really have been cleared. This
* never includes any bits beyond the length of the memslot (if
* the length is not aligned to 64 pages), therefore it is not
* a problem if userspace sets them in log->dirty_bitmap.
*/
if (mask) {
flush = true;
+
kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
offset, mask);
}
}
KVM_MMU_UNLOCK(kvm);
-
+out:
if (flush)
kvm_flush_remote_tlbs_memslot(kvm, memslot);
return 0;
}
static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm,
struct kvm_clear_dirty_log *log)
{
int r;
diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig
index 794976b88c6f..f8757b5b84b3 100644
--- a/virt/kvm/Kconfig
+++ b/virt/kvm/Kconfig
@@ -13,20 +13,23 @@ config HAVE_KVM_PFNCACHE
config HAVE_KVM_IRQCHIP
bool
config HAVE_KVM_IRQ_ROUTING
bool
config HAVE_KVM_DIRTY_RING
bool
+config HAVE_KVM_HW_DIRTY_BIT
+ bool
+
# Only strongly ordered architectures can select this, as it doesn't
# put any explicit constraint on userspace ordering. They can also
# select the _ACQ_REL version.
config HAVE_KVM_DIRTY_RING_TSO
bool
select HAVE_KVM_DIRTY_RING
depends on X86
# Weakly ordered architectures can only select this, advertising
# to userspace the additional ordering requirements.
--
2.55.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH v3 06/11] KVM: arm64: Add hardware-accelerated dirty-bitmap cleaning routine
2026-07-29 10:45 [PATCH v3 00/11] KVM Dirty-bit cleaning hw accelerator (HACDBS) Leonardo Bras
` (4 preceding siblings ...)
2026-07-29 10:45 ` [PATCH v3 05/11] kvm: Add arch-generic interface for hw-accelerated dirty-bitmap cleaning Leonardo Bras
@ 2026-07-29 10:45 ` Leonardo Bras
2026-07-29 10:45 ` [PATCH v3 07/11] KVM: arm64: Dirty-bitmap: avoid splitting previously split blocks Leonardo Bras
` (4 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Leonardo Bras @ 2026-07-29 10:45 UTC (permalink / raw)
To: Catalin Marinas, Will Deacon, Marc Zyngier, Oliver Upton,
Fuad Tabba, Joey Gouly, Steffen Eiden, Suzuki K Poulose,
Zenghui Yu, Paolo Bonzini, James Morse, Leonardo Bras, Yang Shi,
mrigendrachaubey, Thomas Huth, Zeng Heng, Ryan Roberts,
Mark Brown, Sascha Bischoff, Yicong Yang, Kevin Brodsky,
James Clark, Raghavendra Rao Ananta, Yeoreum Yun, Mark Rutland,
Anshuman Khandual, Tian Zheng
Cc: linux-arm-kernel, linux-kernel, kvmarm, kvm
Implement arm64 version of kvm_arch_dirty_log_clear() making use of
FEAT_HACDBS.
It works by transversing the dirty-bitmap and converting the set bits into
HDBSS entries in a 64-page blocks granularity.
The resulting HDBSS array is then fed to the HACDBS mechanism that walks
the pagetable marking writable-dirty pages as writable-clean.
In case of error, rewrite all unprocessed entries, including the faulting
one, to the dirty-bitmap and fall back to generic software cleaning.
In case of the options to "manual protect + init set" are enabled, do
the hugepage splitting in the same fashion as the generic software
cleaning, i.e. in 64-page blocks. For that, remove the static qualifier
from kvm_mmu_split_huge_pages() and make the function available on
kvm_host.h.
Signed-off-by: Leonardo Bras <leo.bras@arm.com>
---
arch/arm64/include/asm/kvm_dirty_bit.h | 24 ++++
include/linux/kvm_host.h | 3 +
arch/arm64/kvm/dirty_bit.c | 163 +++++++++++++++++++++++++
arch/arm64/kvm/mmu.c | 4 +-
4 files changed, 192 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_dirty_bit.h b/arch/arm64/include/asm/kvm_dirty_bit.h
index dd16438f0651..d368a23a84b3 100644
--- a/arch/arm64/include/asm/kvm_dirty_bit.h
+++ b/arch/arm64/include/asm/kvm_dirty_bit.h
@@ -2,11 +2,35 @@
/*
* Copyright (C) 2026 ARM Ltd.
* Author: Leonardo Bras <leo.bras@arm.com>
*/
#ifndef __ARM64_KVM_DIRTY_BIT_H__
#define __ARM64_KVM_DIRTY_BIT_H__
#include <asm/kvm_pgtable.h>
+int __kvm_arch_dirty_log_clear(struct kvm *kvm,
+ struct kvm_memory_slot *memslot,
+ struct kvm_clear_dirty_log *log,
+ unsigned long *bitmap,
+ bool *flush);
+
+static inline bool kvm_arch_dirty_clear_enabled(struct kvm *kvm)
+{
+ return system_supports_hacdbs() && kvm->arch.mmu.pgt &&
+ (kvm->arch.mmu.pgt->flags & KVM_PGTABLE_S2_DBM);
+}
+
+static inline int kvm_arch_dirty_log_clear(struct kvm *kvm,
+ struct kvm_memory_slot *memslot,
+ struct kvm_clear_dirty_log *log,
+ unsigned long *bitmap,
+ bool *flush)
+{
+ if (!kvm_arch_dirty_clear_enabled(kvm))
+ return -EPERM;
+
+ return __kvm_arch_dirty_log_clear(kvm, memslot, log, bitmap, flush);
+}
+
#endif /* __ARM64_KVM_DIRTY_BIT_H__ */
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index ab8cfaec82d3..7ea6ed7ce203 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -1662,20 +1662,23 @@ void kvm_arch_disable_virtualization_cpu(void);
bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu);
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu);
bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu);
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu);
bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu);
bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu);
bool kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu *vcpu);
void kvm_arch_pre_destroy_vm(struct kvm *kvm);
void kvm_arch_create_vm_debugfs(struct kvm *kvm);
+int kvm_mmu_split_huge_pages(struct kvm *kvm, phys_addr_t addr,
+ phys_addr_t end);
+
#ifndef __KVM_HAVE_ARCH_VM_ALLOC
/*
* All architectures that want to use vzalloc currently also
* need their own kvm_arch_alloc_vm implementation.
*/
static inline struct kvm *kvm_arch_alloc_vm(void)
{
return kzalloc_obj(struct kvm, GFP_KERNEL_ACCOUNT);
}
#endif
diff --git a/arch/arm64/kvm/dirty_bit.c b/arch/arm64/kvm/dirty_bit.c
index 33826bbd16d9..5a6266205295 100644
--- a/arch/arm64/kvm/dirty_bit.c
+++ b/arch/arm64/kvm/dirty_bit.c
@@ -183,10 +183,173 @@ static int dirty_bit_clear(struct kvm *kvm, u64 *hw_entries, int size)
* later in generic dirty-cleaning code already performs a DSB before
* doing the TLBI.
*/
preempt_disable();
preempt_notifier_unregister(&save.notifier);
preempt_enable();
return ret;
}
+
+static inline void hdbss_to_bitmap(u64 *hdbss_array, int start, int end,
+ unsigned long *dirty_bitmap,
+ unsigned long long offset)
+{
+ u64 w = (gpa_to_gfn(hdbss_array[start]) - offset) / BITS_PER_LONG;
+ u64 mask = 0;
+ int idx = start;
+
+ /*
+ * Entries should be in order, since they were extracted from
+ * the dirty-bitmap, so batching the atomic writes is efficient.
+ */
+
+ do {
+ u64 entry = (gpa_to_gfn(hdbss_array[idx]) - offset);
+
+ if (entry / BITS_PER_LONG == w) {
+ mask |= BIT(entry % BITS_PER_LONG);
+ } else {
+ atomic_long_or(mask, (atomic_long_t *)&dirty_bitmap[w]);
+ w = entry / BITS_PER_LONG;
+ mask = BIT(entry % BITS_PER_LONG);
+ }
+ } while (++idx < end);
+ atomic_long_or(mask, (atomic_long_t *)&dirty_bitmap[w]);
+}
+
+static inline int mask_to_hdbss(unsigned long *mask, u64 *hw_entries, const gfn_t offset,
+ u64 ttwl, int idx, int entries_sz)
+{
+ while (idx < entries_sz) {
+ int j = __ffs(*mask);
+ u64 a = gfn_to_gpa(offset + j);
+
+ hw_entries[idx++] = (a & HDBSS_ENTRY_IPA) |
+ ttwl |
+ HDBSS_ENTRY_VALID;
+
+ *mask &= ~BIT(j);
+ if (!*mask)
+ break;
+ }
+
+ return idx;
+}
+
+int __kvm_arch_dirty_log_clear(struct kvm *kvm,
+ struct kvm_memory_slot *memslot,
+ struct kvm_clear_dirty_log *log,
+ unsigned long *bitmap,
+ bool *flush)
+{
+ int ret = 0;
+ int idx = 0;
+ unsigned long *dirty_bitmap = memslot->dirty_bitmap;
+ u64 *hw_entries;
+ const int entries_sz = PAGE_SIZE / sizeof(*hw_entries);
+ u64 ttwl;
+ u64 start, end;
+ gfn_t base_gfn;
+
+ hw_entries = kmalloc_objs(u64, entries_sz, GFP_KERNEL);
+ if (!hw_entries)
+ return -ENOMEM;
+
+ ttwl = HDBSS_ENTRY_TTWL(KVM_PGTABLE_LAST_LEVEL);
+
+ if (log) {
+ start = log->first_page / BITS_PER_LONG;
+ end = start + DIV_ROUND_UP(log->num_pages, BITS_PER_LONG);
+ base_gfn = memslot->base_gfn + log->first_page % BITS_PER_LONG;
+ } else {
+ start = 0;
+ end = kvm_dirty_bitmap_bytes(memslot) / sizeof(long);
+ base_gfn = memslot->base_gfn;
+ }
+
+ write_lock(&kvm->mmu_lock);
+
+ for (unsigned long i = start; i < end; i++) {
+ unsigned long mask;
+ gfn_t offset;
+ atomic_long_t *p;
+
+ if (log) { /* Clean only what is in the input bitmap */
+ mask = bitmap[i - start];
+ if (!mask)
+ continue;
+
+ p = (atomic_long_t *)&dirty_bitmap[i];
+ mask &= atomic_long_fetch_andnot(mask, p);
+ } else { /* Clean everything, save to output bitmap */
+ if (!dirty_bitmap[i])
+ continue;
+
+ mask = xchg(&dirty_bitmap[i], 0);
+ bitmap[i] = mask;
+ }
+
+ if (!mask)
+ continue;
+
+ offset = base_gfn + i * BITS_PER_LONG;
+
+ if (kvm_dirty_log_manual_protect_and_init_set(kvm))
+ kvm_mmu_split_huge_pages(kvm,
+ gfn_to_gpa(offset + __ffs(mask)),
+ gfn_to_gpa(offset + __fls(mask) + 1));
+
+ do {
+ idx = mask_to_hdbss(&mask, hw_entries, offset, ttwl, idx, entries_sz);
+ if (idx >= entries_sz) {
+ ret = dirty_bit_clear(kvm, hw_entries, idx);
+ *flush = *flush || ret > 0;
+ if (ret != idx) {
+ /* Restore mask as before processing */
+ atomic_long_or(mask, (atomic_long_t *)&dirty_bitmap[i]);
+ end = i + 1;
+ goto out_err;
+ }
+ idx = 0;
+ }
+ } while (mask);
+ }
+
+ if (idx != 0) {
+ ret = dirty_bit_clear(kvm, hw_entries, idx);
+ *flush = *flush || ret > 0;
+ }
+out_err:
+ if (unlikely(idx > 0 && ret != idx)) {
+ /*
+ * In case there is an error and not all entries in HACDBS get
+ * cleaned, we have to mark the dirty bits back in the bitmap,
+ * as that will be used by the software routine.
+ */
+
+ if (ret < idx) {
+ if (!log) {
+ /*
+ * We have to restore everything, cleaned up to
+ * now, because software routine uses xchg, and
+ * something may have been dirtied in the meantime
+ */
+ for (int i = start; i < end; i++)
+ atomic_long_or(bitmap[i],
+ (atomic_long_t *)&dirty_bitmap[i]);
+ } else {
+ /* Restore only what have not been cleaned yet */
+ hdbss_to_bitmap(hw_entries, ret, idx, dirty_bitmap,
+ memslot->base_gfn);
+ }
+ }
+
+ ret = -EAGAIN;
+ }
+
+ write_unlock(&kvm->mmu_lock);
+ kfree(hw_entries);
+
+ return ret;
+}
diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index e086c01a9325..2f9d90c35668 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -110,22 +110,22 @@ static bool need_split_memcache_topup_or_resched(struct kvm *kvm)
if (need_resched() || rwlock_needbreak(&kvm->mmu_lock))
return true;
chunk_size = kvm->arch.mmu.split_page_chunk_size;
min = kvm_mmu_split_nr_page_tables(chunk_size);
cache = &kvm->arch.mmu.split_page_cache;
return kvm_mmu_memory_cache_nr_free_objects(cache) < min;
}
-static int kvm_mmu_split_huge_pages(struct kvm *kvm, phys_addr_t addr,
- phys_addr_t end)
+int kvm_mmu_split_huge_pages(struct kvm *kvm, phys_addr_t addr,
+ phys_addr_t end)
{
struct kvm_mmu_memory_cache *cache;
struct kvm_pgtable *pgt;
int ret, cache_capacity;
u64 next, chunk_size;
lockdep_assert_held_write(&kvm->mmu_lock);
chunk_size = kvm->arch.mmu.split_page_chunk_size;
cache_capacity = kvm_mmu_split_nr_page_tables(chunk_size);
--
2.55.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH v3 07/11] KVM: arm64: Dirty-bitmap: avoid splitting previously split blocks
2026-07-29 10:45 [PATCH v3 00/11] KVM Dirty-bit cleaning hw accelerator (HACDBS) Leonardo Bras
` (5 preceding siblings ...)
2026-07-29 10:45 ` [PATCH v3 06/11] KVM: arm64: Add hardware-accelerated dirty-bitmap cleaning routine Leonardo Bras
@ 2026-07-29 10:45 ` Leonardo Bras
2026-07-29 10:45 ` [PATCH v3 08/11] kvm/dirty_ring: Introduce get_memslot and move helpers to header Leonardo Bras
` (3 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Leonardo Bras @ 2026-07-29 10:45 UTC (permalink / raw)
To: Catalin Marinas, Will Deacon, Marc Zyngier, Oliver Upton,
Fuad Tabba, Joey Gouly, Steffen Eiden, Suzuki K Poulose,
Zenghui Yu, Paolo Bonzini, James Morse, Leonardo Bras, Yang Shi,
mrigendrachaubey, Thomas Huth, Zeng Heng, Ryan Roberts,
Mark Brown, Sascha Bischoff, Yicong Yang, Kevin Brodsky,
James Clark, Raghavendra Rao Ananta, Yeoreum Yun, Mark Rutland,
Anshuman Khandual, Tian Zheng
Cc: linux-arm-kernel, linux-kernel, kvmarm, kvm
If previous dirty-clean already split a block, then avoid calling the
split helper on that block again.
Signed-off-by: Leonardo Bras <leo.bras@arm.com>
---
arch/arm64/kvm/dirty_bit.c | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/kvm/dirty_bit.c b/arch/arm64/kvm/dirty_bit.c
index 5a6266205295..d7e624f9b630 100644
--- a/arch/arm64/kvm/dirty_bit.c
+++ b/arch/arm64/kvm/dirty_bit.c
@@ -244,20 +244,21 @@ int __kvm_arch_dirty_log_clear(struct kvm *kvm,
bool *flush)
{
int ret = 0;
int idx = 0;
unsigned long *dirty_bitmap = memslot->dirty_bitmap;
u64 *hw_entries;
const int entries_sz = PAGE_SIZE / sizeof(*hw_entries);
u64 ttwl;
u64 start, end;
gfn_t base_gfn;
+ gpa_t last_split = 0;
hw_entries = kmalloc_objs(u64, entries_sz, GFP_KERNEL);
if (!hw_entries)
return -ENOMEM;
ttwl = HDBSS_ENTRY_TTWL(KVM_PGTABLE_LAST_LEVEL);
if (log) {
start = log->first_page / BITS_PER_LONG;
end = start + DIV_ROUND_UP(log->num_pages, BITS_PER_LONG);
@@ -288,24 +289,29 @@ int __kvm_arch_dirty_log_clear(struct kvm *kvm,
mask = xchg(&dirty_bitmap[i], 0);
bitmap[i] = mask;
}
if (!mask)
continue;
offset = base_gfn + i * BITS_PER_LONG;
- if (kvm_dirty_log_manual_protect_and_init_set(kvm))
- kvm_mmu_split_huge_pages(kvm,
- gfn_to_gpa(offset + __ffs(mask)),
- gfn_to_gpa(offset + __fls(mask) + 1));
+ if (kvm_dirty_log_manual_protect_and_init_set(kvm)) {
+ gpa_t split_start = gfn_to_gpa(offset + __ffs(mask));
+ gpa_t split_end = gfn_to_gpa(offset + __fls(mask) + 1);
+
+ if (split_end > last_split) {
+ kvm_mmu_split_huge_pages(kvm, split_start, split_end);
+ last_split = ALIGN_DOWN(split_end - 1, PMD_SIZE) + PMD_SIZE;
+ }
+ }
do {
idx = mask_to_hdbss(&mask, hw_entries, offset, ttwl, idx, entries_sz);
if (idx >= entries_sz) {
ret = dirty_bit_clear(kvm, hw_entries, idx);
*flush = *flush || ret > 0;
if (ret != idx) {
/* Restore mask as before processing */
atomic_long_or(mask, (atomic_long_t *)&dirty_bitmap[i]);
end = i + 1;
--
2.55.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH v3 08/11] kvm/dirty_ring: Introduce get_memslot and move helpers to header
2026-07-29 10:45 [PATCH v3 00/11] KVM Dirty-bit cleaning hw accelerator (HACDBS) Leonardo Bras
` (6 preceding siblings ...)
2026-07-29 10:45 ` [PATCH v3 07/11] KVM: arm64: Dirty-bitmap: avoid splitting previously split blocks Leonardo Bras
@ 2026-07-29 10:45 ` Leonardo Bras
2026-07-29 10:45 ` [PATCH v3 09/11] kvm/dirty_ring: Add arch-generic interface for hw-accelerated dirty-ring cleaning Leonardo Bras
` (2 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Leonardo Bras @ 2026-07-29 10:45 UTC (permalink / raw)
To: Catalin Marinas, Will Deacon, Marc Zyngier, Oliver Upton,
Fuad Tabba, Joey Gouly, Steffen Eiden, Suzuki K Poulose,
Zenghui Yu, Paolo Bonzini, James Morse, Leonardo Bras, Yang Shi,
mrigendrachaubey, Thomas Huth, Zeng Heng, Ryan Roberts,
Mark Brown, Sascha Bischoff, Yicong Yang, Kevin Brodsky,
James Clark, Raghavendra Rao Ananta, Yeoreum Yun, Mark Rutland,
Anshuman Khandual, Tian Zheng
Cc: linux-arm-kernel, linux-kernel, kvmarm, kvm
Dirty-ring entry struct carries a slot number which is used to return
a struct memslot*. That struct carry important information such as
memory offset of that memory slot, which is used to calculate the guest
physical address of the page which needs to be marked clean.
In order to fetch that memslot information without duplicating code, split
that part of kvm_reset_dirty_gfn() into a new helper
kvm_dirty_ring_get_memslot().
Along with that, make it available on kvm_dirty_ring.h other helpers such
as kvm_dirty_gfn_harvested() and kvm_dirty_gfn_set_invalid() which will be
useful on implementing arch specific dirty-ring cleaning accelerators.
Signed-off-by: Leonardo Bras <leo.bras@arm.com>
---
include/linux/kvm_dirty_ring.h | 12 ++++++++++++
virt/kvm/dirty_ring.c | 30 ++++++++++++++++--------------
2 files changed, 28 insertions(+), 14 deletions(-)
diff --git a/include/linux/kvm_dirty_ring.h b/include/linux/kvm_dirty_ring.h
index eb10d87adf7d..190d97fce4a4 100644
--- a/include/linux/kvm_dirty_ring.h
+++ b/include/linux/kvm_dirty_ring.h
@@ -77,18 +77,30 @@ bool kvm_use_dirty_bitmap(struct kvm *kvm);
bool kvm_arch_allow_write_without_running_vcpu(struct kvm *kvm);
u32 kvm_dirty_ring_get_rsvd_entries(struct kvm *kvm);
int kvm_dirty_ring_alloc(struct kvm *kvm, struct kvm_dirty_ring *ring,
int index, u32 size);
int kvm_dirty_ring_reset(struct kvm *kvm, struct kvm_dirty_ring *ring,
int *nr_entries_reset);
void kvm_dirty_ring_push(struct kvm_vcpu *vcpu, u32 slot, u64 offset);
bool kvm_dirty_ring_check_request(struct kvm_vcpu *vcpu);
+static inline bool kvm_dirty_gfn_harvested(struct kvm_dirty_gfn *gfn)
+{
+ return smp_load_acquire(&gfn->flags) & KVM_DIRTY_GFN_F_RESET;
+}
+
+static inline void kvm_dirty_gfn_set_invalid(struct kvm_dirty_gfn *gfn)
+{
+ smp_store_release(&gfn->flags, 0);
+}
+
+struct kvm_memory_slot *kvm_dirty_ring_get_memslot(struct kvm *kvm, u32 slot);
+
/* for use in vm_operations_struct */
struct page *kvm_dirty_ring_get_page(struct kvm_dirty_ring *ring, u32 offset);
void kvm_dirty_ring_free(struct kvm_dirty_ring *ring);
#endif /* CONFIG_HAVE_KVM_DIRTY_RING */
#endif /* KVM_DIRTY_RING_H */
diff --git a/virt/kvm/dirty_ring.c b/virt/kvm/dirty_ring.c
index 572b854edf74..807c45fb1fc7 100644
--- a/virt/kvm/dirty_ring.c
+++ b/virt/kvm/dirty_ring.c
@@ -43,32 +43,44 @@ static u32 kvm_dirty_ring_used(struct kvm_dirty_ring *ring)
static bool kvm_dirty_ring_soft_full(struct kvm_dirty_ring *ring)
{
return kvm_dirty_ring_used(ring) >= ring->soft_limit;
}
static bool kvm_dirty_ring_full(struct kvm_dirty_ring *ring)
{
return kvm_dirty_ring_used(ring) >= ring->size;
}
-static void kvm_reset_dirty_gfn(struct kvm *kvm, u32 slot, u64 offset, u64 mask)
+static inline struct kvm_memory_slot *
+__kvm_dirty_ring_get_memslot(struct kvm *kvm, u32 slot)
{
- struct kvm_memory_slot *memslot;
int as_id, id;
as_id = slot >> 16;
id = (u16)slot;
if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
- return;
+ return NULL;
- memslot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
+ return id_to_memslot(__kvm_memslots(kvm, as_id), id);
+}
+
+struct kvm_memory_slot *kvm_dirty_ring_get_memslot(struct kvm *kvm, u32 slot)
+{
+ return __kvm_dirty_ring_get_memslot(kvm, slot);
+}
+
+static void kvm_reset_dirty_gfn(struct kvm *kvm, u32 slot, u64 offset, u64 mask)
+{
+ struct kvm_memory_slot *memslot;
+
+ memslot = __kvm_dirty_ring_get_memslot(kvm, slot);
if (!memslot || offset >= memslot->npages ||
offset + __fls(mask) >= memslot->npages)
return;
KVM_MMU_LOCK(kvm);
kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, offset, mask);
KVM_MMU_UNLOCK(kvm);
}
@@ -81,35 +93,25 @@ int kvm_dirty_ring_alloc(struct kvm *kvm, struct kvm_dirty_ring *ring,
ring->size = size / sizeof(struct kvm_dirty_gfn);
ring->soft_limit = ring->size - kvm_dirty_ring_get_rsvd_entries(kvm);
ring->dirty_index = 0;
ring->reset_index = 0;
ring->index = index;
return 0;
}
-static inline void kvm_dirty_gfn_set_invalid(struct kvm_dirty_gfn *gfn)
-{
- smp_store_release(&gfn->flags, 0);
-}
-
static inline void kvm_dirty_gfn_set_dirtied(struct kvm_dirty_gfn *gfn)
{
gfn->flags = KVM_DIRTY_GFN_F_DIRTY;
}
-static inline bool kvm_dirty_gfn_harvested(struct kvm_dirty_gfn *gfn)
-{
- return smp_load_acquire(&gfn->flags) & KVM_DIRTY_GFN_F_RESET;
-}
-
int kvm_dirty_ring_reset(struct kvm *kvm, struct kvm_dirty_ring *ring,
int *nr_entries_reset)
{
/*
* To minimize mmu_lock contention, batch resets for harvested entries
* whose gfns are in the same slot, and are within N frame numbers of
* each other, where N is the number of bits in an unsigned long. For
* simplicity, process the current set of entries when the next entry
* can't be included in the batch.
*
--
2.55.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH v3 09/11] kvm/dirty_ring: Add arch-generic interface for hw-accelerated dirty-ring cleaning
2026-07-29 10:45 [PATCH v3 00/11] KVM Dirty-bit cleaning hw accelerator (HACDBS) Leonardo Bras
` (7 preceding siblings ...)
2026-07-29 10:45 ` [PATCH v3 08/11] kvm/dirty_ring: Introduce get_memslot and move helpers to header Leonardo Bras
@ 2026-07-29 10:45 ` Leonardo Bras
2026-07-29 10:45 ` [PATCH v3 10/11] KVM: arm64: Add hardware-accelerated dirty-ring cleaning routine Leonardo Bras
2026-07-29 10:45 ` [PATCH v3 11/11] KVM: arm64: Enable KVM_HW_DIRTY_BIT Leonardo Bras
10 siblings, 0 replies; 12+ messages in thread
From: Leonardo Bras @ 2026-07-29 10:45 UTC (permalink / raw)
To: Catalin Marinas, Will Deacon, Marc Zyngier, Oliver Upton,
Fuad Tabba, Joey Gouly, Steffen Eiden, Suzuki K Poulose,
Zenghui Yu, Paolo Bonzini, James Morse, Leonardo Bras, Yang Shi,
mrigendrachaubey, Thomas Huth, Zeng Heng, Ryan Roberts,
Mark Brown, Sascha Bischoff, Yicong Yang, Kevin Brodsky,
James Clark, Raghavendra Rao Ananta, Yeoreum Yun, Mark Rutland,
Anshuman Khandual, Tian Zheng
Cc: linux-arm-kernel, linux-kernel, kvmarm, kvm
Introduce kvm_arch_dirty_ring_clear() that allow implementation of
arch-specific hardware-accelerated dirty-ring routines.
A call to that is added on kvm_dirty_ring_reset() and will fall back to
software version if not implemented, or any error was detected in the
arch-specific routine.
For an arch to implement this function, it's required to provide it in a
asm/kvm_dirty_bit.h and have CONFIG_HAVE_KVM_HW_DIRTY_BIT=y on building.
If the arch does not implement it, and thus lack above config, the
introduced snippet is expected to be compiled-out and have zero impact at
runtime.
Signed-off-by: Leonardo Bras <leo.bras@arm.com>
---
include/linux/kvm_dirty_bit.h | 7 +++++++
virt/kvm/dirty_ring.c | 5 +++++
2 files changed, 12 insertions(+)
diff --git a/include/linux/kvm_dirty_bit.h b/include/linux/kvm_dirty_bit.h
index fa4f6b67b623..8492979d694e 100644
--- a/include/linux/kvm_dirty_bit.h
+++ b/include/linux/kvm_dirty_bit.h
@@ -11,17 +11,24 @@
static inline int kvm_arch_dirty_log_clear(struct kvm *kvm,
struct kvm_memory_slot *memslot,
struct kvm_clear_dirty_log *log,
unsigned long *bitmap,
bool *flush)
{
return -ENXIO;
}
+static inline int kvm_arch_dirty_ring_clear(struct kvm *kvm,
+ struct kvm_dirty_ring *ring,
+ int *nr_entries_reset)
+{
+ return -ENXIO;
+}
+
#else /* CONFIG_HAVE_KVM_HW_DIRTY_BIT */
#include <asm/kvm_dirty_bit.h>
#endif /* CONFIG_HAVE_KVM_HW_DIRTY_BIT */
#endif /* __KVM_DIRTY_BIT_H__ */
diff --git a/virt/kvm/dirty_ring.c b/virt/kvm/dirty_ring.c
index 807c45fb1fc7..3fa62894f3d2 100644
--- a/virt/kvm/dirty_ring.c
+++ b/virt/kvm/dirty_ring.c
@@ -1,20 +1,21 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* KVM dirty ring implementation
*
* Copyright 2019 Red Hat, Inc.
*/
#include <linux/kvm_host.h>
#include <linux/kvm.h>
#include <linux/vmalloc.h>
#include <linux/kvm_dirty_ring.h>
+#include <linux/kvm_dirty_bit.h>
#include <trace/events/kvm.h>
#include "kvm_mm.h"
int __weak kvm_cpu_dirty_log_size(struct kvm *kvm)
{
return 0;
}
u32 kvm_dirty_ring_get_rsvd_entries(struct kvm *kvm)
{
@@ -126,20 +127,23 @@ int kvm_dirty_ring_reset(struct kvm *kvm, struct kvm_dirty_ring *ring,
struct kvm_dirty_gfn *entry;
/*
* Ensure concurrent calls to KVM_RESET_DIRTY_RINGS are serialized,
* e.g. so that KVM fully resets all entries processed by a given call
* before returning to userspace. Holding slots_lock also protects
* the various memslot accesses.
*/
lockdep_assert_held(&kvm->slots_lock);
+ if (kvm_arch_dirty_ring_clear(kvm, ring, nr_entries_reset) >= 0)
+ goto out;
+
while (likely((*nr_entries_reset) < INT_MAX)) {
if (signal_pending(current))
return -EINTR;
entry = &ring->dirty_gfns[ring->reset_index & (ring->size - 1)];
if (!kvm_dirty_gfn_harvested(entry))
break;
next_slot = READ_ONCE(entry->slot);
@@ -206,20 +210,21 @@ int kvm_dirty_ring_reset(struct kvm *kvm, struct kvm_dirty_ring *ring,
* always be left pending.
*/
if (mask)
kvm_reset_dirty_gfn(kvm, cur_slot, cur_offset, mask);
/*
* The request KVM_REQ_DIRTY_RING_SOFT_FULL will be cleared
* by the VCPU thread next time when it enters the guest.
*/
+out:
trace_kvm_dirty_ring_reset(ring);
return 0;
}
void kvm_dirty_ring_push(struct kvm_vcpu *vcpu, u32 slot, u64 offset)
{
struct kvm_dirty_ring *ring = &vcpu->dirty_ring;
struct kvm_dirty_gfn *entry;
--
2.55.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH v3 10/11] KVM: arm64: Add hardware-accelerated dirty-ring cleaning routine
2026-07-29 10:45 [PATCH v3 00/11] KVM Dirty-bit cleaning hw accelerator (HACDBS) Leonardo Bras
` (8 preceding siblings ...)
2026-07-29 10:45 ` [PATCH v3 09/11] kvm/dirty_ring: Add arch-generic interface for hw-accelerated dirty-ring cleaning Leonardo Bras
@ 2026-07-29 10:45 ` Leonardo Bras
2026-07-29 10:45 ` [PATCH v3 11/11] KVM: arm64: Enable KVM_HW_DIRTY_BIT Leonardo Bras
10 siblings, 0 replies; 12+ messages in thread
From: Leonardo Bras @ 2026-07-29 10:45 UTC (permalink / raw)
To: Catalin Marinas, Will Deacon, Marc Zyngier, Oliver Upton,
Fuad Tabba, Joey Gouly, Steffen Eiden, Suzuki K Poulose,
Zenghui Yu, Paolo Bonzini, James Morse, Leonardo Bras, Yang Shi,
mrigendrachaubey, Thomas Huth, Zeng Heng, Ryan Roberts,
Mark Brown, Sascha Bischoff, Yicong Yang, Kevin Brodsky,
James Clark, Raghavendra Rao Ananta, Yeoreum Yun, Mark Rutland,
Anshuman Khandual, Tian Zheng
Cc: linux-arm-kernel, linux-kernel, kvmarm, kvm
Implement arm64 version of kvm_arch_dirty_ring_clear() making use of
FEAT_HACDBS.
It works by transversing the dirty-ring and converting its entries into
HDBSS entries based on the slot offset.
The resulting HDBSS array is then fed to the HACDBS mechanism that walks
the pagetable marking writable-dirty pages as writable-clean.
Only successfully cleaned entries are set as invalid on the dirty-ring, so
in case of error, falling back to generic software cleaning will take care
of any remaining entry in the dirty-ring.
Signed-off-by: Leonardo Bras <leo.bras@arm.com>
---
arch/arm64/include/asm/kvm_dirty_bit.h | 13 +++++
arch/arm64/kvm/dirty_bit.c | 77 ++++++++++++++++++++++++++
2 files changed, 90 insertions(+)
diff --git a/arch/arm64/include/asm/kvm_dirty_bit.h b/arch/arm64/include/asm/kvm_dirty_bit.h
index d368a23a84b3..f785a7aad9e8 100644
--- a/arch/arm64/include/asm/kvm_dirty_bit.h
+++ b/arch/arm64/include/asm/kvm_dirty_bit.h
@@ -8,29 +8,42 @@
#define __ARM64_KVM_DIRTY_BIT_H__
#include <asm/kvm_pgtable.h>
int __kvm_arch_dirty_log_clear(struct kvm *kvm,
struct kvm_memory_slot *memslot,
struct kvm_clear_dirty_log *log,
unsigned long *bitmap,
bool *flush);
+int __kvm_arch_dirty_ring_clear(struct kvm *kvm, struct kvm_dirty_ring *ring,
+ int *nr_entries_reset);
+
static inline bool kvm_arch_dirty_clear_enabled(struct kvm *kvm)
{
return system_supports_hacdbs() && kvm->arch.mmu.pgt &&
(kvm->arch.mmu.pgt->flags & KVM_PGTABLE_S2_DBM);
}
static inline int kvm_arch_dirty_log_clear(struct kvm *kvm,
struct kvm_memory_slot *memslot,
struct kvm_clear_dirty_log *log,
unsigned long *bitmap,
bool *flush)
{
if (!kvm_arch_dirty_clear_enabled(kvm))
return -EPERM;
return __kvm_arch_dirty_log_clear(kvm, memslot, log, bitmap, flush);
}
+static inline int kvm_arch_dirty_ring_clear(struct kvm *kvm,
+ struct kvm_dirty_ring *ring,
+ int *nr_entries_reset)
+{
+ if (!kvm_arch_dirty_clear_enabled(kvm))
+ return -EPERM;
+
+ return __kvm_arch_dirty_ring_clear(kvm, ring, nr_entries_reset);
+}
+
#endif /* __ARM64_KVM_DIRTY_BIT_H__ */
diff --git a/arch/arm64/kvm/dirty_bit.c b/arch/arm64/kvm/dirty_bit.c
index d7e624f9b630..ca9cfefe0893 100644
--- a/arch/arm64/kvm/dirty_bit.c
+++ b/arch/arm64/kvm/dirty_bit.c
@@ -352,10 +352,87 @@ int __kvm_arch_dirty_log_clear(struct kvm *kvm,
}
ret = -EAGAIN;
}
write_unlock(&kvm->mmu_lock);
kfree(hw_entries);
return ret;
}
+
+int __kvm_arch_dirty_ring_clear(struct kvm *kvm, struct kvm_dirty_ring *ring,
+ int *nr_entries_reset)
+{
+ u64 *hw_entries __free(kfree) = NULL;
+ u64 ttwl;
+ s64 cur_slot = S64_MAX;
+ int i, ret;
+ struct kvm_memory_slot *memslot;
+
+ if (signal_pending(current))
+ return -EINTR;
+
+ ttwl = HDBSS_ENTRY_TTWL(KVM_PGTABLE_LAST_LEVEL);
+
+ hw_entries = kmalloc(max(ring->size * sizeof(u64), PAGE_SIZE), GFP_KERNEL);
+ if (!hw_entries)
+ return -ENOMEM;
+
+ for (i = 0; i < ring->size; i++) {
+ struct kvm_dirty_gfn *entry;
+ gfn_t gfn;
+ u32 entry_slot;
+ u64 entry_offset;
+
+ entry = &ring->dirty_gfns[(ring->reset_index + i) &
+ (ring->size - 1)];
+
+ if (!kvm_dirty_gfn_harvested(entry))
+ break;
+
+ entry_slot = READ_ONCE(entry->slot);
+ entry_offset = READ_ONCE(entry->offset);
+
+ if (entry_slot != cur_slot) {
+ memslot = kvm_dirty_ring_get_memslot(kvm, entry_slot);
+ if (!memslot)
+ return -EFAULT;
+
+ cur_slot = entry_slot;
+ }
+
+ if (entry_offset >= memslot->npages)
+ return -EFAULT;
+
+ gfn = memslot->base_gfn + entry_offset;
+
+ hw_entries[i] = (gfn_to_gpa(gfn) & HDBSS_ENTRY_IPA) |
+ ttwl | HDBSS_ENTRY_VALID;
+ }
+
+ if (i == 0)
+ return 0;
+
+ write_lock(&kvm->mmu_lock);
+ ret = dirty_bit_clear(kvm, hw_entries, i);
+ write_unlock(&kvm->mmu_lock);
+
+ /* Set as invalid all successfully cleaned entries */
+ for (int j = 0; j < ret; j++) {
+ struct kvm_dirty_gfn *entry;
+
+ entry = &ring->dirty_gfns[(ring->reset_index + j) &
+ (ring->size - 1)];
+
+ kvm_dirty_gfn_set_invalid(entry);
+ }
+
+ /* In case of error, try software cleaning from the faulting entry */
+ ring->reset_index += ret;
+ *nr_entries_reset += ret;
+
+ if (ret < i)
+ return -EAGAIN;
+
+ return ret;
+}
--
2.55.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH v3 11/11] KVM: arm64: Enable KVM_HW_DIRTY_BIT
2026-07-29 10:45 [PATCH v3 00/11] KVM Dirty-bit cleaning hw accelerator (HACDBS) Leonardo Bras
` (9 preceding siblings ...)
2026-07-29 10:45 ` [PATCH v3 10/11] KVM: arm64: Add hardware-accelerated dirty-ring cleaning routine Leonardo Bras
@ 2026-07-29 10:45 ` Leonardo Bras
10 siblings, 0 replies; 12+ messages in thread
From: Leonardo Bras @ 2026-07-29 10:45 UTC (permalink / raw)
To: Catalin Marinas, Will Deacon, Marc Zyngier, Oliver Upton,
Fuad Tabba, Joey Gouly, Steffen Eiden, Suzuki K Poulose,
Zenghui Yu, Paolo Bonzini, James Morse, Leonardo Bras, Yang Shi,
mrigendrachaubey, Thomas Huth, Zeng Heng, Ryan Roberts,
Mark Brown, Sascha Bischoff, Yicong Yang, Kevin Brodsky,
James Clark, Raghavendra Rao Ananta, Yeoreum Yun, Mark Rutland,
Anshuman Khandual, Tian Zheng
Cc: linux-arm-kernel, linux-kernel, kvmarm, kvm
Set the corresponding bit to enable hardware accelerated dirty-bitmap and
dirty-ring cleaning for arm64. Actually using acceleration depends on the
cpus enabling FEAT_HACDBS as well as the pre-requisite features for it,
such as FEAT_HDBSS and FEAT_HAFDBS.
Signed-off-by: Leonardo Bras <leo.bras@arm.com>
---
arch/arm64/kvm/Kconfig | 1 +
1 file changed, 1 insertion(+)
diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
index 449154f9a485..3264d8ca093e 100644
--- a/arch/arm64/kvm/Kconfig
+++ b/arch/arm64/kvm/Kconfig
@@ -27,20 +27,21 @@ menuconfig KVM
select VIRT_XFER_TO_GUEST_WORK
select KVM_VFIO
select HAVE_KVM_DIRTY_RING_ACQ_REL
select NEED_KVM_DIRTY_RING_WITH_BITMAP
select HAVE_KVM_MSI
select HAVE_KVM_IRQCHIP
select HAVE_KVM_IRQ_ROUTING
select HAVE_KVM_IRQ_BYPASS
select HAVE_KVM_READONLY_MEM
select HAVE_KVM_VCPU_RUN_PID_CHANGE
+ select HAVE_KVM_HW_DIRTY_BIT
select SCHED_INFO
select GUEST_PERF_EVENTS if PERF_EVENTS
select KVM_GUEST_MEMFD
help
Support hosting virtualized guest machines.
If unsure, say N.
if KVM
--
2.55.0
^ permalink raw reply related [flat|nested] 12+ messages in thread