[PATCH] RISC-V: KVM: Improve dirty log clearing by skipping zero bits in mask

[PATCH] RISC-V: KVM: Improve dirty log clearing by skipping zero bits in mask

[Wang Yechao]

The existing kvm_riscv_gstage_wp_range() walks the entire [start, end)
range to apply write protection for dirty log clearing, even when the
provided mask has zero bits (i.e., many GFNs do not need protection).
This leads to unnecessary page table walks when the mask is sparse.

Replace the range-based approach with a new function
kvm_riscv_gstage_wp_pt_masked() that iterates only over the set bits in
the mask. For each set bit, it looks up the leaf PTE and applies write
protection. Once a huge page is encountered, the entire huge-page range
is processed in one go, and the corresponding bits in the mask are cleared
using bitmap_clear().

Performance was measured with KVM selftests dirty_log_perf_test on a
Spacemit k3 host with the following configuration:
- vCPUs: 2 (-v 2)
- Memory: 1GB (-b 1G)
- Iterations: 3 (-i 3)
- Write percentage varied via -w parameter to simulate different
  dirty mask densities.

The following data shows the time spent in the clear-dirty-log phase
(i.e., the KVM_CLEAR_DIRTY_LOG ioctl) under each configuration.

+------------------+------------------+------------------+-------------+
| Write Percentage | Original (s)     | Patched (s)      | Improvement |
+------------------+------------------+------------------+-------------+
| 10%              | 0.012905         | 0.009213         | +28.6%      |
| 30%              | 0.014217         | 0.010287         | +27.6%      |
| 50%              | 0.014606         | 0.011772         | +19.4%      |
| 70%              | 0.014735         | 0.013199         | +10.4%      |
| 100%             | 0.014759         | 0.015294         | -3.6%       |
+------------------+------------------+------------------+-------------+

The performance improvement is most significant when the dirty mask is
sparse (low write percentage), which is common in real-world scenarios
with low to moderate memory write intensity. In the worst-case scenario
where the mask is fully set (100% write), the optimization introduces
a slight 3.6% overhead due to the additional bit operations, which is
acceptable given the substantial gains in common cases.

This change significantly reduces the number of page-table walks when
the dirty mask has many zero bits, improving the efficiency of
KVM_CLEAR_DIRTY_LOG and related ioctls.

Signed-off-by: Wang Yechao <wang.yechao255@zte.com.cn>
---
 arch/riscv/include/asm/kvm_gstage.h |  3 ++
 arch/riscv/kvm/gstage.c             | 59 +++++++++++++++++++++++++++++
 arch/riscv/kvm/mmu.c                |  2 +-
 3 files changed, 63 insertions(+), 1 deletion(-)

diff --git a/arch/riscv/include/asm/kvm_gstage.h b/arch/riscv/include/asm/kvm_gstage.h
index 21e2019df0cf..50c74527ddd2 100644
--- a/arch/riscv/include/asm/kvm_gstage.h
+++ b/arch/riscv/include/asm/kvm_gstage.h
@@ -82,6 +82,9 @@ bool kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage,
 
 bool kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end);
 
+bool kvm_riscv_gstage_wp_pt_masked(struct kvm_gstage *gstage, gfn_t base_gfn,
+				   unsigned long mask);
+
 void kvm_riscv_gstage_mode_detect(void);
 
 static inline unsigned long kvm_riscv_gstage_mode(unsigned long pgd_levels)
diff --git a/arch/riscv/kvm/gstage.c b/arch/riscv/kvm/gstage.c
index c4c3b79567f1..9af86b1c5e3c 100644
--- a/arch/riscv/kvm/gstage.c
+++ b/arch/riscv/kvm/gstage.c
@@ -472,6 +472,65 @@ bool kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end
 	return flush;
 }
 
+static inline void clear_huge_mask(unsigned long *mask, unsigned long page_size,
+				   gfn_t base_gfn, gpa_t addr)
+{
+	unsigned long start_index = 0;
+	unsigned long end_index = BITS_PER_LONG - 1;
+	unsigned long end_gfn = base_gfn + end_index;
+	unsigned long aligned_start_gfn = addr >> PAGE_SHIFT;
+	unsigned long aligned_end_gfn = aligned_start_gfn + (page_size >> PAGE_SHIFT) - 1;
+	unsigned int nbits = 0;
+
+	if (aligned_start_gfn > base_gfn)
+		start_index = aligned_start_gfn - base_gfn;
+
+	if (aligned_end_gfn < end_gfn)
+		end_index = aligned_end_gfn - base_gfn;
+
+	nbits = end_index - start_index + 1;
+	bitmap_clear(mask, start_index, nbits);
+}
+
+bool kvm_riscv_gstage_wp_pt_masked(struct kvm_gstage *gstage, gfn_t base_gfn,
+				   unsigned long mask)
+{
+	unsigned long page_size;
+	bool flush = false;
+	bool found_leaf;
+	u32 ptep_level;
+	pte_t *ptep;
+	gpa_t addr = 0;
+	int ret;
+
+	while (mask) {
+		addr = (base_gfn + __ffs(mask)) << PAGE_SHIFT;
+
+		found_leaf = kvm_riscv_gstage_get_leaf(gstage, addr, &ptep, &ptep_level);
+		ret = gstage_level_to_page_size(gstage, ptep_level, &page_size);
+		if (ret)
+			break;
+
+		if (found_leaf) {
+			if (ptep_level) {
+				addr = ALIGN_DOWN(addr, page_size);
+				clear_huge_mask(&mask, page_size, base_gfn, addr);
+			}
+
+			flush |= kvm_riscv_gstage_op_pte(gstage, addr, ptep,
+							 ptep_level, GSTAGE_OP_WP);
+
+			if (ptep_level)
+				continue;
+		}
+
+		/* clear the first set bit*/
+		mask &= mask - 1;
+	}
+
+	return flush;
+}
+
 void __init kvm_riscv_gstage_mode_detect(void)
 {
 #ifdef CONFIG_64BIT
diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c
index 082f9b261733..7eab58ff63f9 100644
--- a/arch/riscv/kvm/mmu.c
+++ b/arch/riscv/kvm/mmu.c
@@ -110,7 +110,7 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
 
 	kvm_riscv_gstage_init(&gstage, kvm);
 
-	flush = kvm_riscv_gstage_wp_range(&gstage, start, end);
+	flush = kvm_riscv_gstage_wp_pt_masked(&gstage, base_gfn, mask);
 	if (flush)
 		kvm_flush_remote_tlbs_range(kvm, start >> PAGE_SHIFT,
 					    (end - start) >> PAGE_SHIFT);
-- 
2.27.0


_______________________________________________
linux-riscv mailing list
linux-riscv@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-riscv

[PATCH] RISC-V: KVM: Improve dirty log clearing by skipping zero bits in mask

[Wang Yechao]

The existing kvm_riscv_gstage_wp_range() walks the entire [start, end)
range to apply write protection for dirty log clearing, even when the
provided mask has zero bits (i.e., many GFNs do not need protection).
This leads to unnecessary page table walks when the mask is sparse.

Replace the range-based approach with a new function
kvm_riscv_gstage_wp_pt_masked() that iterates only over the set bits in
the mask. For each set bit, it looks up the leaf PTE and applies write
protection. Once a huge page is encountered, the entire huge-page range
is processed in one go, and the corresponding bits in the mask are cleared
using bitmap_clear().

Performance was measured with KVM selftests dirty_log_perf_test on a
Spacemit k3 host with the following configuration:
- vCPUs: 2 (-v 2)
- Memory: 1GB (-b 1G)
- Iterations: 3 (-i 3)
- Write percentage varied via -w parameter to simulate different
  dirty mask densities.

The following data shows the time spent in the clear-dirty-log phase
(i.e., the KVM_CLEAR_DIRTY_LOG ioctl) under each configuration.

+------------------+------------------+------------------+-------------+
| Write Percentage | Original (s)     | Patched (s)      | Improvement |
+------------------+------------------+------------------+-------------+
| 10%              | 0.012905         | 0.009213         | +28.6%      |
| 30%              | 0.014217         | 0.010287         | +27.6%      |
| 50%              | 0.014606         | 0.011772         | +19.4%      |
| 70%              | 0.014735         | 0.013199         | +10.4%      |
| 100%             | 0.014759         | 0.015294         | -3.6%       |
+------------------+------------------+------------------+-------------+

The performance improvement is most significant when the dirty mask is
sparse (low write percentage), which is common in real-world scenarios
with low to moderate memory write intensity. In the worst-case scenario
where the mask is fully set (100% write), the optimization introduces
a slight 3.6% overhead due to the additional bit operations, which is
acceptable given the substantial gains in common cases.

This change significantly reduces the number of page-table walks when
the dirty mask has many zero bits, improving the efficiency of
KVM_CLEAR_DIRTY_LOG and related ioctls.

Signed-off-by: Wang Yechao <wang.yechao255@zte.com.cn>
---
 arch/riscv/include/asm/kvm_gstage.h |  3 ++
 arch/riscv/kvm/gstage.c             | 59 +++++++++++++++++++++++++++++
 arch/riscv/kvm/mmu.c                |  2 +-
 3 files changed, 63 insertions(+), 1 deletion(-)

diff --git a/arch/riscv/include/asm/kvm_gstage.h b/arch/riscv/include/asm/kvm_gstage.h
index 21e2019df0cf..50c74527ddd2 100644
--- a/arch/riscv/include/asm/kvm_gstage.h
+++ b/arch/riscv/include/asm/kvm_gstage.h
@@ -82,6 +82,9 @@ bool kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage,
 
 bool kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end);
 
+bool kvm_riscv_gstage_wp_pt_masked(struct kvm_gstage *gstage, gfn_t base_gfn,
+				   unsigned long mask);
+
 void kvm_riscv_gstage_mode_detect(void);
 
 static inline unsigned long kvm_riscv_gstage_mode(unsigned long pgd_levels)
diff --git a/arch/riscv/kvm/gstage.c b/arch/riscv/kvm/gstage.c
index c4c3b79567f1..9af86b1c5e3c 100644
--- a/arch/riscv/kvm/gstage.c
+++ b/arch/riscv/kvm/gstage.c
@@ -472,6 +472,65 @@ bool kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end
 	return flush;
 }
 
+static inline void clear_huge_mask(unsigned long *mask, unsigned long page_size,
+				   gfn_t base_gfn, gpa_t addr)
+{
+	unsigned long start_index = 0;
+	unsigned long end_index = BITS_PER_LONG - 1;
+	unsigned long end_gfn = base_gfn + end_index;
+	unsigned long aligned_start_gfn = addr >> PAGE_SHIFT;
+	unsigned long aligned_end_gfn = aligned_start_gfn + (page_size >> PAGE_SHIFT) - 1;
+	unsigned int nbits = 0;
+
+	if (aligned_start_gfn > base_gfn)
+		start_index = aligned_start_gfn - base_gfn;
+
+	if (aligned_end_gfn < end_gfn)
+		end_index = aligned_end_gfn - base_gfn;
+
+	nbits = end_index - start_index + 1;
+	bitmap_clear(mask, start_index, nbits);
+}
+
+bool kvm_riscv_gstage_wp_pt_masked(struct kvm_gstage *gstage, gfn_t base_gfn,
+				   unsigned long mask)
+{
+	unsigned long page_size;
+	bool flush = false;
+	bool found_leaf;
+	u32 ptep_level;
+	pte_t *ptep;
+	gpa_t addr = 0;
+	int ret;
+
+	while (mask) {
+		addr = (base_gfn + __ffs(mask)) << PAGE_SHIFT;
+
+		found_leaf = kvm_riscv_gstage_get_leaf(gstage, addr, &ptep, &ptep_level);
+		ret = gstage_level_to_page_size(gstage, ptep_level, &page_size);
+		if (ret)
+			break;
+
+		if (found_leaf) {
+			if (ptep_level) {
+				addr = ALIGN_DOWN(addr, page_size);
+				clear_huge_mask(&mask, page_size, base_gfn, addr);
+			}
+
+			flush |= kvm_riscv_gstage_op_pte(gstage, addr, ptep,
+							 ptep_level, GSTAGE_OP_WP);
+
+			if (ptep_level)
+				continue;
+		}
+
+		/* clear the first set bit*/
+		mask &= mask - 1;
+	}
+
+	return flush;
+}
+
 void __init kvm_riscv_gstage_mode_detect(void)
 {
 #ifdef CONFIG_64BIT
diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c
index 082f9b261733..7eab58ff63f9 100644
--- a/arch/riscv/kvm/mmu.c
+++ b/arch/riscv/kvm/mmu.c
@@ -110,7 +110,7 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
 
 	kvm_riscv_gstage_init(&gstage, kvm);
 
-	flush = kvm_riscv_gstage_wp_range(&gstage, start, end);
+	flush = kvm_riscv_gstage_wp_pt_masked(&gstage, base_gfn, mask);
 	if (flush)
 		kvm_flush_remote_tlbs_range(kvm, start >> PAGE_SHIFT,
 					    (end - start) >> PAGE_SHIFT);
-- 
2.27.0

[PATCH] RISC-V: KVM: Improve dirty log clearing by skipping zero bits in mask

[Wang Yechao]

The existing kvm_riscv_gstage_wp_range() walks the entire [start, end)
range to apply write protection for dirty log clearing, even when the
provided mask has zero bits (i.e., many GFNs do not need protection).
This leads to unnecessary page table walks when the mask is sparse.

Replace the range-based approach with a new function
kvm_riscv_gstage_wp_pt_masked() that iterates only over the set bits in
the mask. For each set bit, it looks up the leaf PTE and applies write
protection. Once a huge page is encountered, the entire huge-page range
is processed in one go, and the corresponding bits in the mask are cleared
using bitmap_clear().

Performance was measured with KVM selftests dirty_log_perf_test on a
Spacemit k3 host with the following configuration:
- vCPUs: 2 (-v 2)
- Memory: 1GB (-b 1G)
- Iterations: 3 (-i 3)
- Write percentage varied via -w parameter to simulate different
  dirty mask densities.

The following data shows the time spent in the clear-dirty-log phase
(i.e., the KVM_CLEAR_DIRTY_LOG ioctl) under each configuration.

+------------------+------------------+------------------+-------------+
| Write Percentage | Original (s)     | Patched (s)      | Improvement |
+------------------+------------------+------------------+-------------+
| 10%              | 0.012905         | 0.009213         | +28.6%      |
| 30%              | 0.014217         | 0.010287         | +27.6%      |
| 50%              | 0.014606         | 0.011772         | +19.4%      |
| 70%              | 0.014735         | 0.013199         | +10.4%      |
| 100%             | 0.014759         | 0.015294         | -3.6%       |
+------------------+------------------+------------------+-------------+

The performance improvement is most significant when the dirty mask is
sparse (low write percentage), which is common in real-world scenarios
with low to moderate memory write intensity. In the worst-case scenario
where the mask is fully set (100% write), the optimization introduces
a slight 3.6% overhead due to the additional bit operations, which is
acceptable given the substantial gains in common cases.

This change significantly reduces the number of page-table walks when
the dirty mask has many zero bits, improving the efficiency of
KVM_CLEAR_DIRTY_LOG and related ioctls.

Signed-off-by: Wang Yechao <wang.yechao255@zte.com.cn>
---
 arch/riscv/include/asm/kvm_gstage.h |  3 ++
 arch/riscv/kvm/gstage.c             | 59 +++++++++++++++++++++++++++++
 arch/riscv/kvm/mmu.c                |  2 +-
 3 files changed, 63 insertions(+), 1 deletion(-)

diff --git a/arch/riscv/include/asm/kvm_gstage.h b/arch/riscv/include/asm/kvm_gstage.h
index 21e2019df0cf..50c74527ddd2 100644
--- a/arch/riscv/include/asm/kvm_gstage.h
+++ b/arch/riscv/include/asm/kvm_gstage.h
@@ -82,6 +82,9 @@ bool kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage,
 
 bool kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end);
 
+bool kvm_riscv_gstage_wp_pt_masked(struct kvm_gstage *gstage, gfn_t base_gfn,
+				   unsigned long mask);
+
 void kvm_riscv_gstage_mode_detect(void);
 
 static inline unsigned long kvm_riscv_gstage_mode(unsigned long pgd_levels)
diff --git a/arch/riscv/kvm/gstage.c b/arch/riscv/kvm/gstage.c
index c4c3b79567f1..9af86b1c5e3c 100644
--- a/arch/riscv/kvm/gstage.c
+++ b/arch/riscv/kvm/gstage.c
@@ -472,6 +472,65 @@ bool kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end
 	return flush;
 }
 
+static inline void clear_huge_mask(unsigned long *mask, unsigned long page_size,
+				   gfn_t base_gfn, gpa_t addr)
+{
+	unsigned long start_index = 0;
+	unsigned long end_index = BITS_PER_LONG - 1;
+	unsigned long end_gfn = base_gfn + end_index;
+	unsigned long aligned_start_gfn = addr >> PAGE_SHIFT;
+	unsigned long aligned_end_gfn = aligned_start_gfn + (page_size >> PAGE_SHIFT) - 1;
+	unsigned int nbits = 0;
+
+	if (aligned_start_gfn > base_gfn)
+		start_index = aligned_start_gfn - base_gfn;
+
+	if (aligned_end_gfn < end_gfn)
+		end_index = aligned_end_gfn - base_gfn;
+
+	nbits = end_index - start_index + 1;
+	bitmap_clear(mask, start_index, nbits);
+}
+
+bool kvm_riscv_gstage_wp_pt_masked(struct kvm_gstage *gstage, gfn_t base_gfn,
+				   unsigned long mask)
+{
+	unsigned long page_size;
+	bool flush = false;
+	bool found_leaf;
+	u32 ptep_level;
+	pte_t *ptep;
+	gpa_t addr = 0;
+	int ret;
+
+	while (mask) {
+		addr = (base_gfn + __ffs(mask)) << PAGE_SHIFT;
+
+		found_leaf = kvm_riscv_gstage_get_leaf(gstage, addr, &ptep, &ptep_level);
+		ret = gstage_level_to_page_size(gstage, ptep_level, &page_size);
+		if (ret)
+			break;
+
+		if (found_leaf) {
+			if (ptep_level) {
+				addr = ALIGN_DOWN(addr, page_size);
+				clear_huge_mask(&mask, page_size, base_gfn, addr);
+			}
+
+			flush |= kvm_riscv_gstage_op_pte(gstage, addr, ptep,
+							 ptep_level, GSTAGE_OP_WP);
+
+			if (ptep_level)
+				continue;
+		}
+
+		/* clear the first set bit*/
+		mask &= mask - 1;
+	}
+
+	return flush;
+}
+
 void __init kvm_riscv_gstage_mode_detect(void)
 {
 #ifdef CONFIG_64BIT
diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c
index 082f9b261733..7eab58ff63f9 100644
--- a/arch/riscv/kvm/mmu.c
+++ b/arch/riscv/kvm/mmu.c
@@ -110,7 +110,7 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
 
 	kvm_riscv_gstage_init(&gstage, kvm);
 
-	flush = kvm_riscv_gstage_wp_range(&gstage, start, end);
+	flush = kvm_riscv_gstage_wp_pt_masked(&gstage, base_gfn, mask);
 	if (flush)
 		kvm_flush_remote_tlbs_range(kvm, start >> PAGE_SHIFT,
 					    (end - start) >> PAGE_SHIFT);
-- 
2.27.0


-- 
kvm-riscv mailing list
kvm-riscv@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/kvm-riscv

Re: [PATCH] RISC-V: KVM: Improve dirty log clearing by skipping zero bits in mask

[sashiko-bot]

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:

Pre-existing issues:
- [Critical] This problem wasn't introduced by this patch, but the non-atomic read-modify-write in `kvm_riscv_gstage_op_pte()` for `GSTAGE_OP_WP` overwrites and loses concurrent hardware A/D (Accessed/Dirty) bit updates. When write-protecting a page, it reads the PTE, clears `_PAGE_WRITE`, and writes it back using `set_pte()`. If the hardware page-table walker concurrently sets the Dirty or Accessed bit during this window, the update is lost. Atomic single-instruction operations (e.g., atomic AND or `cmpxchg`) must be used for updating hardware-shared PTEs.
--

--- Patch [1]: [PATCH] RISC-V: KVM: Improve dirty log clearing by skipping zero bits in mask ---
commit 69dff78a641691bca22a783a5f60188157d6a975
Author: Wang Yechao <wang.yechao255@zte.com.cn>

RISC-V: KVM: Improve dirty log clearing by skipping zero bits in mask

This commit optimizes the KVM dirty log clearing process on RISC-V by
iterating only over the set bits in the dirty mask instead of scanning
the entire range. It uses a new function to process leaf PTEs and entire
huge pages, reducing unnecessary page table walks for sparse masks.

> diff --git a/arch/riscv/kvm/gstage.c b/arch/riscv/kvm/gstage.c
> index c4c3b79567f10..9af86b1c5e3c3 100644
> --- a/arch/riscv/kvm/gstage.c
> +++ b/arch/riscv/kvm/gstage.c

[ ... ]

> @@ -472,6 +472,65 @@ bool kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end

[ ... ]

> +bool kvm_riscv_gstage_wp_pt_masked(struct kvm_gstage *gstage, gfn_t base_gfn,
> +				   unsigned long mask)
> +{

[ ... ]

> +		if (found_leaf) {
> +			if (ptep_level) {
> +				addr = ALIGN_DOWN(addr, page_size);
> +				clear_huge_mask(&mask, page_size, base_gfn, addr);
> +			}
> +
> +			flush |= kvm_riscv_gstage_op_pte(gstage, addr, ptep,
> +							 ptep_level, GSTAGE_OP_WP);

[Severity: Critical]
This is a pre-existing issue, but does the invocation of
kvm_riscv_gstage_op_pte() with GSTAGE_OP_WP race with hardware A/D bit
updates?

Looking at kvm_riscv_gstage_op_pte():

	else if (op == GSTAGE_OP_WP)
		set_pte(ptep, __pte(pte_val(ptep_get(ptep)) & ~_PAGE_WRITE));

This performs a non-atomic read-modify-write on the PTE. If the hardware
page-table walker concurrently sets the Dirty or Accessed bit during this
window, would the update be overwritten and lost?

When write-protecting a page for dirty log clearing, losing the
hardware-updated dirty bit could cause KVM to believe a dirtied page is
clean. During VM live migration, could this lead to the page not being
re-transferred, resulting in data corruption in the migrated VM?

Should hardware-shared SPTE updates use atomic single-instruction
operations (like atomic AND or cmpxchg) instead to preserve concurrent
hardware writes?

> +
> +			if (ptep_level)
> +				continue;
> +		}
> +
> +		/* clear the first set bit*/
> +		mask &= mask - 1;
> +	}
> +
> +	return flush;
> +}

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20260717080654.877151-1-wang.yechao255@zte.com.cn?part=1