[PATCH v5 8/8] drm/amdgpu: integrate VRAM migration into SVM restore and realign paths

[Junhua Shen <Junhua.Shen@amd.com>]

Integrate the VRAM migration infrastructure into the SVM restore and
boundary realignment paths to ensure correct interaction between
migration and the existing SVM lifecycle framework.

Suppress restore for realigned ranges:
- Add suppress_restore flag to amdgpu_svm_range
- Set flag before svm_restore_realign_boundary() evicts a crossing range
- Prevent migration notifier from enqueueing restore work while
  realignment or VRAM migration is in progress
- Remove existing restore-list entry and drop its queue kref
- On realign failure, clear flag and re-enqueue restore for rollback
- Enqueue, worker retry, and put_if_dequeued paths respect the flag

Extend svm_restore_range() with migration support:
- Move amdgpu_svm_range_is_valid() check before map_ctx setup
- Set devmem_only = false since restore should not force VRAM placement
- Add device_private_page_owner for proper devmem identification

Signed-off-by: Junhua Shen <Junhua.Shen@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c |  4 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_userptr.c   | 76 +++++++++++++++----
 3 files changed, 65 insertions(+), 16 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
index 88923e6a0194..31229c4770b8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
@@ -728,11 +728,13 @@ amdgpu_svm_range_put_if_dequeued(struct amdgpu_svm *svm,
 		list_add_tail(&range->work_node, &svm->gc.list);
 		range->queue_state = AMDGPU_SVM_RANGE_IN_GC;
 		queue_gc = true;
-	} else if (RESTORE_WORK(range->pending_ops)) {
+	} else if (RESTORE_WORK(range->pending_ops) &&
+		   !READ_ONCE(range->suppress_restore)) {
 		list_add_tail(&range->work_node, &svm->restore.list);
 		range->queue_state = AMDGPU_SVM_RANGE_IN_RESTORE;
 		queue_restore = true;
 	} else {
+		range->pending_ops &= ~AMDGPU_SVM_RANGE_OP_RESTORE;
 		range->queue_state = AMDGPU_SVM_RANGE_NOT_QUEUED;
 		release_kref = true;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h
index 2d677f0965b4..4bf05c304564 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h
@@ -55,6 +55,7 @@ struct amdgpu_svm_range {
 	struct drm_gpusvm_range base;
 	struct list_head work_node;
 	bool gpu_mapped;
+	bool suppress_restore;
 	u8 queue_state;
 	u8 pending_ops;
 	unsigned long pending_start_page;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userptr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userptr.c
index 8bf00aed7f15..45a611479d1c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userptr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userptr.c
@@ -28,6 +28,7 @@
 #include "amdgpu_svm_attr.h"
 #include "amdgpu.h"
 #include "amdgpu_amdkfd.h"
+#include "amdgpu_migrate.h"
 
 #include <drm/drm_exec.h>
 #include <drm/drm_gpusvm.h>
@@ -151,6 +152,11 @@ svm_restore_enqueue_work(struct amdgpu_svm *svm,
 
 	spin_lock(&svm->work_lock);
 
+	if (!UNMAP_WORK(pending_ops) && READ_ONCE(range->suppress_restore)) {
+		spin_unlock(&svm->work_lock);
+		return;
+	}
+
 	/* Deny any work if range is unmapped */
 	if (UNMAP_WORK(range->pending_ops)) {
 		spin_unlock(&svm->work_lock);
@@ -232,7 +238,7 @@ svm_restore_range(struct amdgpu_svm *svm, struct amdgpu_svm_range *range)
 	unsigned long range_start_page;
 	int ret;
 	struct drm_gpusvm_ctx map_ctx;
-	bool devmem_possible, need_vram_migration;
+	bool devmem_possible;
 
 	amdgpu_svm_assert_locked(svm);
 
@@ -247,20 +253,20 @@ svm_restore_range(struct amdgpu_svm *svm, struct amdgpu_svm_range *range)
 	if (!attr_range || !amdgpu_svm_attr_has_access(attrs.access))
 		return 0;
 
+	if (amdgpu_svm_range_is_valid(svm, range, &attrs))
+		return 0;
+
 	devmem_possible = amdgpu_svm_devmem_possible(svm);
-	need_vram_migration = devmem_possible &&
-			      amdgpu_svm_attr_prefer_vram(&attrs);
 
 	map_ctx = (struct drm_gpusvm_ctx){
 		.read_only = !!(attrs.flags & AMDGPU_SVM_ATTR_BIT_GPU_RO),
 		.devmem_possible = devmem_possible,
-		.devmem_only = need_vram_migration,
+		.devmem_only = false,
 		.check_pages_threshold = devmem_possible ? SZ_64K : 0,
+		.device_private_page_owner = devmem_possible ?
+			AMDGPU_PGMAP_OWNER(svm->adev) : NULL,
 	};
 
-	if (amdgpu_svm_range_is_valid(svm, range, &attrs))
-		return 0;
-
 	AMDGPU_SVM_RANGE_DEBUG(range, "RESTORE - GET PAGES");
 
 	ret = amdgpu_svm_range_get_pages(svm, &range->base, &map_ctx);
@@ -289,6 +295,42 @@ static bool amdgpu_svm_nonretryable(int ret)
 	}
 }
 
+static void
+svm_restore_suppress_range_restore(struct amdgpu_svm *svm,
+				   struct amdgpu_svm_range *range)
+{
+	bool release_kref = false;
+
+	spin_lock(&svm->work_lock);
+
+	WRITE_ONCE(range->suppress_restore, true);
+	range->pending_ops &= ~AMDGPU_SVM_RANGE_OP_RESTORE;
+
+	if (range->queue_state == AMDGPU_SVM_RANGE_IN_RESTORE) {
+		list_del_init(&range->work_node);
+		range->queue_state = AMDGPU_SVM_RANGE_NOT_QUEUED;
+		range->pending_start_page = ULONG_MAX;
+		range->pending_last_page = 0;
+		release_kref = true;
+	}
+
+	spin_unlock(&svm->work_lock);
+
+	if (release_kref)
+		drm_gpusvm_range_put(&range->base);
+}
+
+static void
+svm_restore_unsuppress_range_restore(struct amdgpu_svm *svm,
+				     struct amdgpu_svm_range *range,
+				     unsigned long start_page,
+				     unsigned long last_page)
+{
+	WRITE_ONCE(range->suppress_restore, false);
+	svm_restore_enqueue_work(svm, range, start_page, last_page,
+			       AMDGPU_SVM_RANGE_OP_RESTORE);
+}
+
 static void amdgpu_svm_restore_worker(struct work_struct *w)
 {
 	struct delayed_work *dwork = to_delayed_work(w);
@@ -334,17 +376,19 @@ static void amdgpu_svm_restore_worker(struct work_struct *w)
 						 op_ctx.start_page,
 						 op_ctx.last_page, ret);
 				spin_lock(&svm->work_lock);
-				if (!UNMAP_WORK(op_ctx.range->pending_ops))
+				if (!UNMAP_WORK(op_ctx.range->pending_ops) &&
+				    !READ_ONCE(op_ctx.range->suppress_restore)) {
 					op_ctx.range->pending_ops |=
 						AMDGPU_SVM_RANGE_OP_RESTORE;
-				op_ctx.range->pending_start_page =
-					min(op_ctx.range->pending_start_page,
-					    op_ctx.start_page);
-				op_ctx.range->pending_last_page =
-					max(op_ctx.range->pending_last_page,
-					    op_ctx.last_page);
+					op_ctx.range->pending_start_page =
+						min(op_ctx.range->pending_start_page,
+						    op_ctx.start_page);
+					op_ctx.range->pending_last_page =
+						max(op_ctx.range->pending_last_page,
+						    op_ctx.last_page);
+					need_resched = true;
+				}
 				spin_unlock(&svm->work_lock);
-				need_resched = true;
 			} else {
 				AMDGPU_SVM_TRACE(
 					"restore work drop non retryable [0x%lx-0x%lx] ret=%d\n",
@@ -746,6 +790,7 @@ svm_restore_realign_boundary(struct amdgpu_svm *svm,
 		rs = drm_gpusvm_range_start(r) >> PAGE_SHIFT;
 		rl = (drm_gpusvm_range_end(r) >> PAGE_SHIFT) - 1;
 
+		svm_restore_suppress_range_restore(svm, svm_range);
 		amdgpu_svm_range_evict(svm, r);
 
 		if (svm_range->gpu_mapped) {
@@ -755,6 +800,7 @@ svm_restore_realign_boundary(struct amdgpu_svm *svm,
 			if (ret < 0) {
 				AMDGPU_SVM_TRACE("zap failed ret=%d [0x%lx-0x%lx]\n",
 						 ret, rs, rl);
+				svm_restore_unsuppress_range_restore(svm, svm_range, rs, rl);
 				drm_exec_fini(&exec);
 				return ret;
 			}
-- 
2.34.1