[RFC PATCH v4 07/25] KVM: guest_memfd: Support private/shared conversion of device memory

["Aneesh Kumar K.V (Arm)" <aneesh.kumar@kernel.org>]

Allow device-backed guest_memfd ranges to be converted between shared
and private memory using file offsets. Use the common attribute conversion
path for inode locking, maple-tree allocation, guest invalidation and
attribute publication.

Reuse PFN attribute preparation to revoke host mappings before private
conversion. Resolve guest addresses through memslot bindings and obtain
physical addresses through the provider's get_pfn() callback.

Install private device mappings during attribute conversion because
device mapping validation needs them before the guest's request is
completed. That validation does not require a preceding guest PFN
lookup, so a prepare_pfn() callback cannot be the sole place that
installs mappings. Use the architecture device-map hook before
publishing private attributes, independently of guest faults. The VMM
validates guest MMIO requests against its BAR layout before setting the
corresponding range private.

Sharing also completes before publishing attributes. Generic
invalidation with KVM_FILTER_PRIVATE has already removed the protected
device mappings, so the device conversion callback needs no additional
shared operation. Shared host and guest mappings can be installed by
their normal fault paths.

Leave revoke() and populate() unset, so full revocation, hole punching
and preallocation remain unsupported at this point in the series.

Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Sean Christopherson <seanjc@google.com>
Cc: David Hildenbrand <david@kernel.org>
Cc: kvm@vger.kernel.org
Cc: patches@lists.linux.dev
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
 Documentation/virt/kvm/api.rst |   5 ++
 include/linux/kvm_host.h       |   2 +
 virt/kvm/guest_memfd.c         | 103 ++++++++++++++++++++++++++++++++-
 3 files changed, 107 insertions(+), 3 deletions(-)

diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst
index 8455a9c7c971..4482d34ec281 100644
--- a/Documentation/virt/kvm/api.rst
+++ b/Documentation/virt/kvm/api.rst
@@ -6540,6 +6540,11 @@ provider. PFN_MODE selects the PFN callbacks; providers that supply device
 operations additionally require INIT_SHARED and a VM supporting protected
 device mappings.
 
+KVM_SET_MEMORY_ATTRIBUTES2 converts device memory using guest_memfd offsets.
+For private conversion, KVM resolves the guest address through the memslot
+binding and obtains the physical address from the provider. Shared conversion
+removes the protected device mappings before publishing shared attributes.
+
 When the KVM MMU performs a PFN lookup to service a guest fault, the fault will
 always be consumed from guest_memfd, regardless of whether it is a shared or a
 private fault (unless in-place conversion is disabled and the backing
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index f2dd387ab0fa..c7831356b913 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -2618,6 +2618,8 @@ struct guest_memfd_device_binding;
 bool kvm_arch_gmem_device_supported(struct kvm *kvm);
 void kvm_arch_gmem_device_bind(struct kvm_memory_slot *slot,
 		const struct guest_memfd_device_binding *binding);
+int kvm_arch_gmem_device_map(struct kvm *kvm, const struct kvm_memory_slot *slot,
+			     u64 gpa, u64 size, u64 pa);
 bool kvm_gmem_is_private_gfn(struct kvm *kvm, gfn_t gfn);
 bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm);
 
diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c
index 397ead84bdb3..a06abb6dda82 100644
--- a/virt/kvm/guest_memfd.c
+++ b/virt/kvm/guest_memfd.c
@@ -99,6 +99,9 @@ static bool gmem_provider_is_device(struct gmem_inode *gi)
 	return gi->provider_ops && gi->provider_ops->device_memory;
 }
 
+static int kvm_gmem_device_convert_attributes(struct inode *inode, pgoff_t start,
+					      pgoff_t end, u64 attrs);
+
 static bool kvm_gmem_is_pfn_mode(struct gmem_inode *gi)
 {
 	return gi->flags & GUEST_MEMFD_FLAG_PFN_MODE;
@@ -1158,8 +1161,8 @@ static int kvm_gmem_folio_prepare_attributes(struct inode *inode, pgoff_t start,
 	return 0;
 }
 
-static int kvm_gmem_ram_pfn_prepare_attributes(struct inode *inode, pgoff_t start,
-		size_t nr_pages, u64 attrs, pgoff_t *err_index)
+static int kvm_gmem_pfn_prepare_attributes(struct inode *inode, pgoff_t start,
+					   size_t nr_pages, u64 attrs, pgoff_t *err_index)
 {
 	if (!(attrs & KVM_MEMORY_ATTRIBUTE_PRIVATE))
 		return 0;
@@ -1188,14 +1191,22 @@ static const struct kvm_gmem_backing_ops kvm_gmem_folio_backing_ops = {
 static const struct kvm_gmem_backing_ops kvm_gmem_ram_pfn_backing_ops = {
 	.populate = kvm_gmem_ram_pfn_populate,
 	.revoke = kvm_gmem_ram_pfn_revoke,
-	.prepare_attributes = kvm_gmem_ram_pfn_prepare_attributes,
+	.prepare_attributes = kvm_gmem_pfn_prepare_attributes,
 	.convert_attributes = kvm_gmem_ram_pfn_convert_attributes,
 	.prepare_pfn = kvm_gmem_ram_prepare_pfn,
 };
 
+static const struct kvm_gmem_backing_ops kvm_gmem_device_backing_ops = {
+	.prepare_attributes = kvm_gmem_pfn_prepare_attributes,
+	.convert_attributes = kvm_gmem_device_convert_attributes,
+};
+
 static const struct kvm_gmem_backing_ops *
 kvm_gmem_select_backing_ops(struct gmem_inode *gi)
 {
+	if (gmem_provider_is_device(gi))
+		return &kvm_gmem_device_backing_ops;
+
 	if (kvm_gmem_is_pfn_mode(gi))
 		return &kvm_gmem_ram_pfn_backing_ops;
 
@@ -1491,6 +1502,92 @@ void __weak kvm_arch_gmem_device_bind(struct kvm_memory_slot *slot,
 {
 }
 
+int __weak kvm_arch_gmem_device_map(struct kvm *kvm,
+				    const struct kvm_memory_slot *slot,
+				    u64 gpa, u64 size, u64 pa)
+{
+	return -EOPNOTSUPP;
+}
+
+/*
+ * Install protected device mappings for the provider PFNs when making a range
+ * private. Protected device-mapping validation needs these mappings in place
+ * before the guest's request is completed. That validation does not require a
+ * preceding kvm_gmem_get_pfn() call, so prepare_pfn() cannot be the sole place
+ * that installs them. Install them here with kvm_arch_gmem_device_map() before
+ * publishing private attributes, so validation can use the existing mappings.
+ *
+ * Sharing also completes before publishing attributes, but needs no additional
+ * operation here. kvm_gmem_invalidate_start() has already removed the private
+ * device mappings with KVM_FILTER_PRIVATE. Shared host and guest mappings can
+ * then be installed by their normal fault paths.
+ *
+ * Caller holds inode->i_mapping->invalidate_lock for write, which protects
+ * the attributes and offset-to-GPA bindings. The gmem_provider_lock()
+ * prevents device unlock from racing with conversion and publication of
+ * attributes.
+ */
+static int kvm_gmem_device_convert_attributes(struct inode *inode, pgoff_t start,
+					      pgoff_t end, u64 attrs)
+{
+	struct gmem_inode *gi = GMEM_I(inode);
+	struct gmem_file *f;
+	void *entry;
+
+	MA_STATE(mas, &gi->attributes, start, start);
+
+	if (!(attrs & KVM_MEMORY_ATTRIBUTE_PRIVATE))
+		return 0;
+
+	mas_for_each(&mas, entry, end - 1) {
+		pgoff_t first = max(start, mas.index);
+		pgoff_t last = min(end, mas.last + 1);
+
+		if (kvm_gmem_get_attributes(inode, entry) & KVM_MEMORY_ATTRIBUTE_PRIVATE)
+			continue;
+
+		while (first < last) {
+			struct kvm_memory_slot *slot = NULL;
+			kvm_pfn_t pfn, next_pfn;
+			pgoff_t high, next;
+			u64 gpa;
+			int ret;
+
+			kvm_gmem_for_each_file(f, inode) {
+				slot = xa_load(&f->bindings, first);
+				if (slot)
+					break;
+			}
+			if (!slot)
+				return -EINVAL;
+
+			high = min(last, slot->gmem.pgoff + slot->npages);
+			gpa = (slot->base_gfn + first - slot->gmem.pgoff) << PAGE_SHIFT;
+			ret = gmem_provider_get_pfn(gi, first, &pfn, NULL);
+			if (ret)
+				return ret;
+
+			/* Map a contiguous provider PFN run within this binding. */
+			for (next = first + 1; next < high; next++) {
+				ret = gmem_provider_get_pfn(gi, next, &next_pfn, NULL);
+				if (ret)
+					return ret;
+				if (next_pfn != pfn + next - first)
+					break;
+			}
+			high = next;
+			ret = kvm_arch_gmem_device_map(gi->device.kvm, slot, gpa,
+						       (u64)(high - first) << PAGE_SHIFT,
+						       PFN_PHYS(pfn));
+			if (ret)
+				return ret;
+
+			first = high;
+		}
+	}
+	return 0;
+}
+
 static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags, int resource_fd)
 {
 	static const char *name = "[kvm-gmem]";
-- 
2.43.0