[Qemu-devel] [RFC PATCH 3/3] arm/arm64: KVM: implement KVM_MEM_UNCACHED

[Andrew Jones <drjones@redhat.com>]

When userspace tells us a memory region is uncached, then we
need to pin all its pages and set them all to be uncached.

Signed-off-by: Andrew Jones <drjones@redhat.com>
---
 arch/arm/include/asm/kvm_mmu.h    |  9 +++++
 arch/arm/include/uapi/asm/kvm.h   |  1 +
 arch/arm/kvm/mmu.c                | 71 +++++++++++++++++++++++++++++++++++++++
 arch/arm64/include/asm/kvm_mmu.h  |  9 +++++
 arch/arm64/include/uapi/asm/kvm.h |  1 +
 5 files changed, 91 insertions(+)

diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
index 37ca2a4c6f094..6802f6adc12bf 100644
--- a/arch/arm/include/asm/kvm_mmu.h
+++ b/arch/arm/include/asm/kvm_mmu.h
@@ -265,6 +265,15 @@ static inline void __kvm_flush_dcache_pud(pud_t pud)
 {
 }
 
+static inline void __set_page_uncached(pte_t *ptep)
+{
+	pte_t pte = *ptep;
+
+	pte = clear_pte_bit(pte, L_PTE_MT_MASK);
+	pte = set_pte_bit(pte, L_PTE_MT_UNCACHED);
+	set_pte_ext(ptep, pte, 0);
+}
+
 #define kvm_virt_to_phys(x)		virt_to_idmap((unsigned long)(x))
 
 void kvm_set_way_flush(struct kvm_vcpu *vcpu);
diff --git a/arch/arm/include/uapi/asm/kvm.h b/arch/arm/include/uapi/asm/kvm.h
index 9d6fc19acf8a2..cdd456f591882 100644
--- a/arch/arm/include/uapi/asm/kvm.h
+++ b/arch/arm/include/uapi/asm/kvm.h
@@ -109,6 +109,7 @@ struct kvm_sync_regs {
 };
 
 struct kvm_arch_memory_slot {
+	struct page **pages;
 };
 
 /* If you need to interpret the index values, here is the key: */
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 59af5ad779eb6..d4e47572d3a5d 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -1697,6 +1697,54 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
 		kvm_mmu_wp_memory_region(kvm, mem->slot);
 }
 
+static int set_page_uncached(pte_t *ptep, pgtable_t token,
+			     unsigned long addr, void *data)
+{
+	__set_page_uncached(ptep);
+	kvm_flush_dcache_pte(*ptep);
+	return 0;
+}
+
+static int vma_range_pin_and_set_uncached(struct vm_area_struct *vma,
+					  hva_t start, long nr_pages,
+					  struct page **pages)
+{
+	unsigned long size = nr_pages * PAGE_SIZE;
+	int ret;
+
+	down_read(&vma->vm_mm->mmap_sem);
+	ret = get_user_pages(NULL, vma->vm_mm, start, nr_pages,
+			     true, true, pages, NULL);
+	up_read(&vma->vm_mm->mmap_sem);
+
+	if (ret < 0)
+		return ret;
+
+	if (ret == nr_pages) {
+		ret = apply_to_page_range(vma->vm_mm, start, size,
+					  set_page_uncached, NULL);
+		flush_tlb_kernel_range(start, start + size);
+		return ret;
+	}
+
+	return -EFAULT;
+}
+
+static void unpin_pages(struct kvm_memory_slot *memslot)
+{
+	int i;
+
+	if (!memslot->arch.pages)
+		return;
+
+	for (i = 0; i < memslot->npages; ++i) {
+		if (memslot->arch.pages[i])
+			put_page(memslot->arch.pages[i]);
+	}
+	kfree(memslot->arch.pages);
+	memslot->arch.pages = NULL;
+}
+
 int kvm_arch_prepare_memory_region(struct kvm *kvm,
 				   struct kvm_memory_slot *memslot,
 				   struct kvm_userspace_memory_region *mem,
@@ -1705,6 +1753,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
 	hva_t hva = mem->userspace_addr;
 	hva_t reg_end = hva + mem->memory_size;
 	bool writable = !(mem->flags & KVM_MEM_READONLY);
+	struct page **pages = memslot->arch.pages;
 	int ret = 0;
 
 	if (change != KVM_MR_CREATE && change != KVM_MR_MOVE &&
@@ -1768,6 +1817,26 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
 						    writable);
 			if (ret)
 				break;
+		} else if ((change != KVM_MR_FLAGS_ONLY)
+				&& (memslot->flags & KVM_MEM_UNCACHED)) {
+
+			long nr_pages = (vm_end - vm_start)/PAGE_SIZE;
+
+			if (!pages) {
+				pages = kzalloc(memslot->npages *
+					sizeof(struct page *), GFP_KERNEL);
+				if (!pages)
+					return -ENOMEM;
+				memslot->arch.pages = pages;
+			}
+
+			ret = vma_range_pin_and_set_uncached(vma, vm_start,
+							nr_pages, pages);
+			if (ret) {
+				unpin_pages(memslot);
+				break;
+			}
+			pages += nr_pages;
 		}
 		hva = vm_end;
 	} while (hva < reg_end);
@@ -1787,6 +1856,8 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
 			   struct kvm_memory_slot *dont)
 {
+	if (free->flags & KVM_MEM_UNCACHED)
+		unpin_pages(free);
 }
 
 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 6458b53731421..f4c3c56587a9f 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -300,6 +300,15 @@ static inline void __kvm_flush_dcache_pud(pud_t pud)
 	kvm_flush_dcache_to_poc(page_address(page), PUD_SIZE);
 }
 
+static inline void __set_page_uncached(pte_t *ptep)
+{
+	pte_t pte = *ptep;
+
+	pte = clear_pte_bit(pte, PTE_ATTRINDX_MASK);
+	pte = set_pte_bit(pte, PTE_ATTRINDX(MT_DEVICE_nGnRnE));
+	set_pte(ptep, pte);
+}
+
 #define kvm_virt_to_phys(x)		__virt_to_phys((unsigned long)(x))
 
 void kvm_set_way_flush(struct kvm_vcpu *vcpu);
diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
index 5553d112e405b..22b4e4a6da950 100644
--- a/arch/arm64/include/uapi/asm/kvm.h
+++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -104,6 +104,7 @@ struct kvm_sync_regs {
 };
 
 struct kvm_arch_memory_slot {
+	struct page **pages;
 };
 
 /* If you need to interpret the index values, here is the key: */
-- 
1.8.3.1