Linux-ARM-Kernel Archive on lore.kernel.org
 help / color / mirror / Atom feed
From: Vincent Donnefort <vdonnefort@google.com>
To: maz@kernel.org, oupton@kernel.org, kvmarm@lists.linux.dev,
	 linux-arm-kernel@lists.infradead.org
Cc: joey.gouly@arm.com, seiden@linux.ibm.com, suzuki.poulose@arm.com,
	 yuzenghui@huawei.com, catalin.marinas@arm.com, will@kernel.org,
	 kernel-team@android.com, fuad.tabba@linux.dev,
	qperret@google.com,  weilin.chang@arm.com,
	Vincent Donnefort <vdonnefort@google.com>
Subject: [PATCH v4 23/23] KVM: arm64: Stage-2 huge mappings for protected VMs
Date: Tue, 17 Nov 2026 17:44:02 +0000	[thread overview]
Message-ID: <20261117174403.58762-24-vdonnefort@google.com> (raw)
In-Reply-To: <20261117174403.58762-1-vdonnefort@google.com>

Enable PMD-sized stage-2 block mappings for protected VMs. This is
possible whenever the stage-1 mapping allows it, that is, if it is
itself backed by THPs.

When a THP is found, an entire PMD_SIZE mapping is donated to the guest,
unless it overlaps an existing mapping. The new PMD_SIZE mapping can
only be broken down via the HVC __pkvm_host_split_guest() which the
hypervisor can request with PKVM_HYP_REQ_SPLIT.

Signed-off-by: Vincent Donnefort <vdonnefort@google.com>
---
 arch/arm64/kvm/mmu.c  | 128 +++++++++++++++++++++++++++---------------
 arch/arm64/kvm/pkvm.c |  34 +++++++----
 2 files changed, 104 insertions(+), 58 deletions(-)

diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index 1db10d8d6db7..df8ec7c798dd 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -1489,11 +1489,9 @@ static bool fault_supports_stage2_huge_mapping(const struct kvm_s2_fault_desc *s
 	if (map_size == PAGE_SIZE)
 		return true;
 
-	/* pKVM only supports PMD_SIZE huge-mappings for non-protected VMs */
-	if (is_protected_kvm_enabled()) {
-		if (vcpu_is_protected(s2fd->vcpu) || map_size != PMD_SIZE)
-			return false;
-	}
+	/* pKVM only supports PMD_SIZE huge-mappings */
+	if (is_protected_kvm_enabled() && map_size != PMD_SIZE)
+		return false;
 
 	size = memslot->npages * PAGE_SIZE;
 
@@ -1843,42 +1841,23 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd,
 	return ret;
 }
 
-static int protected_pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd)
+static int pkvm_pin_user_pages(const struct kvm_s2_fault_desc *s2fd,
+			       struct kvm_s2_fault_vma_info *s2vi)
 {
 	unsigned int flags = FOLL_HWPOISON | FOLL_LONGTERM | FOLL_WRITE;
-	struct kvm_s2_fault_vma_info s2vi = {};
-	struct kvm_vcpu *vcpu = s2fd->vcpu;
-	struct kvm_pgtable *pgt = s2fd->mmu->pgt;
-	struct mm_struct *mm = current->mm;
-	struct kvm *kvm = vcpu->kvm;
-	void *hyp_memcache;
-	int ret;
+	long ret;
 
-	hyp_memcache = get_mmu_memcache(vcpu);
-	ret = topup_mmu_memcache(s2fd->mmu, hyp_memcache);
-	if (ret)
-		return -ENOMEM;
-
-	ret = kvm_s2_fault_get_vma_info(s2fd, &s2vi);
+	ret = kvm_s2_fault_get_vma_info(s2fd, s2vi);
 	if (ret)
 		return ret;
 
-	ret = account_locked_vm(mm, 1, true);
-	if (ret)
-		return ret;
-
-	mmap_read_lock(mm);
-	ret = pin_user_pages(s2fd->hva, 1, flags, &s2vi.page);
-	mmap_read_unlock(mm);
-
+	ret = pin_user_pages_unlocked(s2fd->hva, 1, &s2vi->page, flags);
 	if (ret == -EHWPOISON) {
 		kvm_send_hwpoison_signal(s2fd->hva, PAGE_SHIFT);
-		ret = 0;
-		goto dec_account;
+		return ret;
 	} else if (ret != 1) {
-		ret = -EFAULT;
-		goto dec_account;
-	} else if (!folio_test_swapbacked(page_folio(s2vi.page))) {
+		return -EFAULT;
+	} else if (!folio_test_swapbacked(page_folio(s2vi->page))) {
 		/*
 		 * We really can't deal with page-cache pages returned by GUP
 		 * because (a) we may trigger writeback of a page for which we
@@ -1893,26 +1872,83 @@ static int protected_pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd)
 		 * pages backed by swap in the knowledge that the GUP pin will
 		 * prevent try_to_unmap() from succeeding.
 		 */
-		ret = -EIO;
-		goto unpin;
+		unpin_user_page(s2vi->page);
+		return -EIO;
 	}
 
-	write_lock(&kvm->mmu_lock);
-	ret = pkvm_pgtable_stage2_map(pgt, gfn_to_gpa(s2vi.gfn), PAGE_SIZE,
-				      page_to_phys(s2vi.page), KVM_PGTABLE_PROT_RWX,
-				      hyp_memcache, 0);
-	write_unlock(&kvm->mmu_lock);
-	if (ret) {
-		if (ret == -EAGAIN)
-			ret = 0;
-		goto unpin;
+	s2vi->pfn = page_to_pfn(s2vi->page);
+
+	ret = transparent_hugepage_adjust(s2fd, &s2vi->pfn, &s2vi->gfn);
+	if (ret < 0) {
+		unpin_user_page(s2vi->page);
+		return ret;
 	}
 
+	s2vi->vma_pagesize = ret;
+
 	return 0;
+}
+
+static int protected_pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd)
+{
+	struct kvm_s2_fault_vma_info s2vi = {};
+	struct kvm_vcpu *vcpu = s2fd->vcpu;
+	struct kvm_pgtable *pgt = s2fd->mmu->pgt;
+	struct mm_struct *mm = current->mm;
+	struct kvm *kvm = vcpu->kvm;
+	unsigned long unaccount;
+	void *hyp_memcache;
+	u64 size;
+	int ret;
+
+	hyp_memcache = get_mmu_memcache(vcpu);
+	ret = topup_mmu_memcache(s2fd->mmu, hyp_memcache);
+	if (ret)
+		return -ENOMEM;
+
+	ret = pkvm_pin_user_pages(s2fd, &s2vi);
+	if (ret == -EHWPOISON || ret == -EAGAIN)
+		return 0;
+	else if (ret)
+		return ret;
+
+	size = s2vi.vma_pagesize;
+
+	ret = account_locked_vm(mm, size / PAGE_SIZE, true);
+	if (ret)
+		goto unpin;
+
+	kvm_fault_lock(kvm);
+	if (mmu_invalidate_retry(kvm, s2vi.mmu_seq)) {
+		ret = -EAGAIN;
+	} else {
+		ret = pkvm_pgtable_stage2_map(pgt, gfn_to_gpa(s2vi.gfn), size,
+					      __pfn_to_phys(s2vi.pfn), KVM_PGTABLE_PROT_RWX,
+					      hyp_memcache, 0);
+
+		/* undo transparent_hugepage_adjust() and try again */
+		if (ret == -E2BIG) {
+			u64 ipa = ALIGN_DOWN(s2fd->fault_ipa, PAGE_SIZE);
+
+			size = PAGE_SIZE;
+			ret = pkvm_pgtable_stage2_map(pgt, ipa, size,
+						      page_to_phys(s2vi.page),
+						      KVM_PGTABLE_PROT_RWX,
+						      hyp_memcache, 0);
+		}
+	}
+	kvm_fault_unlock(kvm);
+
+	unaccount = (ret ? s2vi.vma_pagesize : s2vi.vma_pagesize - size) / PAGE_SIZE;
+	account_locked_vm(mm, unaccount, false);
+
 unpin:
-	unpin_user_page(s2vi.page);
-dec_account:
-	account_locked_vm(mm, 1, false);
+	if (ret) {
+		unpin_user_page(s2vi.page);
+		if (ret == -EAGAIN)
+			ret = 0;
+	}
+
 	return ret;
 }
 
diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c
index 58ca7ffa68e3..5aea9e9aef17 100644
--- a/arch/arm64/kvm/pkvm.c
+++ b/arch/arm64/kvm/pkvm.c
@@ -572,9 +572,8 @@ static int __pkvm_pgtable_stage2_reclaim(struct kvm_pgtable *pgt, u64 start, u64
 			continue;
 
 		page = pfn_to_page(mapping->pfn);
-		WARN_ON_ONCE(mapping->nr_pages != 1);
 		unpin_user_pages_dirty_lock(&page, 1, true);
-		account_locked_vm(kvm->mm, 1, false);
+		account_locked_vm(kvm->mm, mapping->nr_pages, false);
 		pkvm_mapping_remove(mapping, &pgt->pkvm_mappings);
 		kfree(mapping);
 	}
@@ -627,9 +626,17 @@ void pkvm_pgtable_stage2_destroy_pgd(struct kvm_pgtable *pgt)
 	WARN_ON_ONCE(!RB_EMPTY_ROOT(&pgt->pkvm_mappings.rb_root));
 }
 
+/*
+ * Return:
+ *  0		- Mapping successfully installed.
+ *  -EAGAIN	- Mapping already exists due to a concurrent vCPU fault race.
+ *  -E2BIG	- Mapping overlaps an existing smaller one.
+ *  -EFAULT	- Mapping overlaps an existing poisoned address.
+ *  < 0		- Other negative error code on invalid arguments or HVC failure.
+ */
 int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
-			   u64 phys, enum kvm_pgtable_prot prot,
-			   void *mc, enum kvm_pgtable_walk_flags flags)
+			    u64 phys, enum kvm_pgtable_prot prot,
+			    void *mc, enum kvm_pgtable_walk_flags flags)
 {
 	struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);
 	struct pkvm_mapping *mapping = NULL;
@@ -639,20 +646,22 @@ int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
 	u64 end = addr + size;
 	int ret;
 
-	if (WARN_ON_ONCE(!PAGE_ALIGNED(addr | size)))
+	if (WARN_ON_ONCE(!PAGE_ALIGNED(addr)))
+		return -EINVAL;
+
+	if (WARN_ON_ONCE(size != PAGE_SIZE && size != PMD_SIZE))
 		return -EINVAL;
 
 	lockdep_assert_held_write(&kvm->mmu_lock);
 	mapping = pkvm_mapping_iter_first(&pgt->pkvm_mappings, addr, end - 1);
 
 	if (kvm_vm_is_protected(kvm)) {
-		/* Protected VMs are mapped using RWX page-granular mappings */
-		if (WARN_ON_ONCE(size != PAGE_SIZE))
-			return -EINVAL;
-
 		if (WARN_ON_ONCE(prot != KVM_PGTABLE_PROT_RWX))
 			return -EINVAL;
 
+		if (mapping && mapping->nr_pages == 1 && size > PAGE_SIZE)
+			return -E2BIG;
+
 		/*
 		 * We either raced with another vCPU or the guest PTE
 		 * has been poisoned by an erroneous host access.
@@ -662,10 +671,11 @@ int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
 			return ret ? -EFAULT : -EAGAIN;
 		}
 
-		ret = kvm_call_hyp_nvhe(__pkvm_host_donate_guest, pfn, gfn, 1);
+		ret = kvm_call_hyp_nvhe(__pkvm_host_donate_guest, pfn, gfn, size / PAGE_SIZE);
+
+		/* Warn if -E2BIG is overloaded */
+		WARN_ON_ONCE(ret == -E2BIG);
 	} else {
-		if (WARN_ON_ONCE(size != PAGE_SIZE && size != PMD_SIZE))
-			return -EINVAL;
 
 		/*
 		 * We either raced with another vCPU or we're changing between
-- 
2.56.1.708.g77570a6a1d-goog



      parent reply	other threads:[~2026-11-17 17:45 UTC|newest]

Thread overview: 24+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-17 17:43 [PATCH v4 00/23] Huge mapping support for protected VMs Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 01/23] KVM: arm64: Prefault host stage-2 entries on block split Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 02/23] KVM: arm64: Propagate host stage-2 annotated " Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 03/23] KVM: arm64: Allow block-level stage-2 annotation Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 04/23] KVM: arm64: Use block-level annotations when setting up the host stage-2 Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 05/23] KVM: arm64: Make pKVM ownership selftest an HVC Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 06/23] KVM: arm64: Reject zero-sized ranges in pfn_range_is_valid() Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 07/23] KVM: arm64: Add a range to __pkvm_host_share/unshare_hyp() Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 08/23] KVM: arm64: Add a range to __pkvm_host_donate_guest() Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 09/23] KVM: arm64: Add a range to hyp_poison_page() Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 10/23] KVM: arm64: Add a range to __pkvm_host_reclaim_guest() Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 11/23] KVM: arm64: Add a range to __pkvm_guest_share_host() Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 12/23] KVM: arm64: Add a range to __pkvm_guest_unshare_host() Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 13/23] KVM: arm64: Handle huge mappings in __pkvm_host_force_reclaim_page_guest() Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 14/23] KVM: arm64: Handle huge mappings in __pkvm_vcpu_in_poison_fault() Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 15/23] KVM: arm64: Add a range to pKVM ownership selftest Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 16/23] KVM: arm64: Warn on pKVM guest stage-2 block collapse Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 17/23] KVM: arm64: Add vcpu conduit for pkvm_hyp_req Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 18/23] KVM: arm64: Introduce kvm_pgtable_stage2_table_install() Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 19/23] KVM: arm64: Add __pkvm_host_split_guest HVC Vincent Donnefort
2026-11-17 17:43 ` [PATCH v4 20/23] KVM: arm64: Extend pKVM page ownership selftests to cover guest block split Vincent Donnefort
2026-11-17 17:44 ` [PATCH v4 21/23] KVM: arm64: Add PKVM_HYP_REQ_SPLIT Vincent Donnefort
2026-11-17 17:44 ` [PATCH v4 22/23] KVM: arm64: Raise PKVM_HYP_REQ_SPLIT on guest to host sharing Vincent Donnefort
2026-11-17 17:44 ` Vincent Donnefort [this message]

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261117174403.58762-24-vdonnefort@google.com \
    --to=vdonnefort@google.com \
    --cc=catalin.marinas@arm.com \
    --cc=fuad.tabba@linux.dev \
    --cc=joey.gouly@arm.com \
    --cc=kernel-team@android.com \
    --cc=kvmarm@lists.linux.dev \
    --cc=linux-arm-kernel@lists.infradead.org \
    --cc=maz@kernel.org \
    --cc=oupton@kernel.org \
    --cc=qperret@google.com \
    --cc=seiden@linux.ibm.com \
    --cc=suzuki.poulose@arm.com \
    --cc=weilin.chang@arm.com \
    --cc=will@kernel.org \
    --cc=yuzenghui@huawei.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox