[PATCH 6/6] drm/xe/pagefault: add CPU mirror VMA for SVA faults without PRI

[Nareshkumar Gollakoti <naresh.kumar.g@intel.com>]

On ATS/PASID platforms without PRI, xe cannot rely on the hardware
page request path to populate missing CPU mirror mappings for SVA faults.
This can leave xe_pagefault_service() without a VMA even when the faulting
address is valid in the process address space.

Create a CPU-address-mirror VMA on demand for such faults so they can
proceed through the existing SVM fault handling path.

Faults against unmapped addresses or unusable VMs continue to fail.

Signed-off-by: Nareshkumar Gollakoti <naresh.kumar.g@intel.com>
---
 drivers/gpu/drm/xe/xe_pagefault.c       | 49 +++++++++++++++++++++++++
 drivers/gpu/drm/xe/xe_pagefault_types.h |  6 +++
 2 files changed, 55 insertions(+)

diff --git a/drivers/gpu/drm/xe/xe_pagefault.c b/drivers/gpu/drm/xe/xe_pagefault.c
index 98cce0f12b12..c3acec28e115 100644
--- a/drivers/gpu/drm/xe/xe_pagefault.c
+++ b/drivers/gpu/drm/xe/xe_pagefault.c
@@ -17,6 +17,7 @@
 #include "xe_migrate.h"
 #include "xe_pagefault.h"
 #include "xe_pagefault_types.h"
+#include "xe_pci.h"
 #include "xe_pm.h"
 #include "xe_svm.h"
 #include "xe_trace_bo.h"
@@ -268,6 +269,7 @@ static int xe_pagefault_service(struct xe_pagefault *pf)
 {
 	struct xe_gt *gt = pf->gt;
 	struct xe_device *xe = gt_to_xe(gt);
+	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
 	struct xe_vm *vm;
 	struct xe_vma *vma = NULL;
 	int err;
@@ -298,6 +300,51 @@ static int xe_pagefault_service(struct xe_pagefault *pf)
 	}
 
 	vma = xe_vm_find_vma_by_addr(vm, xe_pagefault_addr(pf));
+
+	/*
+	 * ATS/PASID is negotiated but this hardware has no PRI, so there is
+	 * no IOMMU self-healing path to fault the page in - the driver has
+	 * to create the missing CPU-address-mirror binding itself and let
+	 * the SVM path below service the fault.
+	 */
+	if (!vma && vm->sva_handle && xe_pci_svm_needs_sw_iopf(pdev)) {
+		u64 page_size = vm->flags & XE_VM_FLAG_64K ? SZ_64K : SZ_4K;
+		u64 start = ALIGN_DOWN(xe_pagefault_addr(pf), page_size);
+		u64 end = start + page_size;
+		int alloc_err = 0;
+
+		up_read(&vm->lock);
+		down_write(&vm->lock);
+		/* Lock was dropped, re-check: a racing fault may have won */
+		vma = xe_vm_find_vma_by_addr(vm, xe_pagefault_addr(pf));
+		if (!vma) {
+			if (xe_vm_is_closed_or_banned(vm)) {
+				alloc_err = -ENOENT;
+			} else {
+				/* Merge with adjacent mirror VMAs instead of leaving singletons */
+				xe_vm_find_cpu_addr_mirror_vma_range(vm, &start, &end);
+				alloc_err = xe_vm_alloc_cpu_addr_mirror_vma(vm, start, end - start);
+			}
+			if (!alloc_err)
+				vma = xe_vm_find_vma_by_addr(vm, xe_pagefault_addr(pf));
+		}
+		downgrade_write(&vm->lock);
+
+		if (alloc_err || !vma || !xe_vma_is_cpu_addr_mirror(vma)) {
+			/*
+			 * err is what actually tells guc_ack_fault() to NACK;
+			 * distinguish a transient allocation failure from a
+			 * genuinely unmapped/closed address for diagnostics.
+			 */
+			err = alloc_err ? : -EINVAL;
+			if (alloc_err == -ENOMEM)
+				xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_ALLOC);
+			else
+				xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_NOT_FOUND);
+			goto unlock_vm;
+		}
+	}
+
 	if (!vma) {
 		err = -EINVAL;
 		xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_NOT_FOUND);
@@ -589,6 +636,8 @@ static const char *xe_pagefault_error_to_str(enum xe_pagefault_error error)
 		return "VM_CLOSED";
 	case XE_PAGEFAULT_ERROR_VMA_NOT_FOUND:
 		return "VMA_NOT_FOUND";
+	case XE_PAGEFAULT_ERROR_VMA_ALLOC:
+		return "VMA_ALLOC";
 	case XE_PAGEFAULT_ERROR_READ_ONLY_VIOLATION:
 		return "READ_ONLY_VIOLATION";
 	case XE_PAGEFAULT_ERROR_VMA_NEEDS_VRAM_CHECK:
diff --git a/drivers/gpu/drm/xe/xe_pagefault_types.h b/drivers/gpu/drm/xe/xe_pagefault_types.h
index 56defc05922e..d95d47ee0c0c 100644
--- a/drivers/gpu/drm/xe/xe_pagefault_types.h
+++ b/drivers/gpu/drm/xe/xe_pagefault_types.h
@@ -59,6 +59,12 @@ enum xe_pagefault_error {
 	XE_PAGEFAULT_ERROR_VM_CLOSED,
 	/** @XE_PAGEFAULT_ERROR_VMA_NOT_FOUND: No VMA covers the faulted address */
 	XE_PAGEFAULT_ERROR_VMA_NOT_FOUND,
+	/**
+	 * @XE_PAGEFAULT_ERROR_VMA_ALLOC: Failed to allocate a CPU-address-mirror
+	 * VMA to service a PRI-less SVA fault (e.g. -ENOMEM); distinct from a
+	 * genuinely unmapped address
+	 */
+	XE_PAGEFAULT_ERROR_VMA_ALLOC,
 	/**
 	 * @XE_PAGEFAULT_ERROR_READ_ONLY_VIOLATION: Write/atomic fault on a
 	 * read-only VMA
-- 
2.43.0