[PATCH 00/13] drm/amdgpu: fixes for the VM critical section and page fault rework

[PATCH 00/13] drm/amdgpu: fixes for the VM critical section and page fault rework

[Huang Yu]

Hi Christian,

patches 1-8 and 10 are your VM rework. The other four are fixes found
while testing it with KFD SVM on top:

1. patch 09: amdgpu_vm_pt_alloc() leaks the new PT BO when the
   allocation succeeds but the VM is evicting once the critical section
   is re-entered. Release the unpublished BO, keep the critical section
   owned by the caller.

2. patch 11: amdgpu_vm_pt_leaves() always updated cursor.parent->bo, so
   a walk stopping at an allocated PTB wrote a PDE in the parent instead
   of the PTEs in the leaf. Use the entry's own BO and level, and go
   through amdgpu_vm_pte_update_flags().

3. patch 12: both callers passed an inclusive last address, but
   amdgpu_vm_pt_leaves() works on a half-open range. That dropped the
   last page in amdgpu_vm_unmap_range(), and made amdgpu_vm_handle_fault()
   skip the loop entirely, so AMDGPU_VM_NORETRY_FLAGS were never written
   and the wave kept retrying.

4. patch 13: for a redirect to the dummy hierarchy above the PTE level
   the entry points to the next-level dummy PD/PT, but
   amdgpu_vm_pte_update_flags() treats everything above PTB as a leaf and
   applies AMDGPU_PDE_PTE_FLAG(). Encode it with
   amdgpu_gmc_get_pde_for_bo() instead.

One question on patch 13: it special cases the dummy PDE inside
amdgpu_vm_pt_leaves(), which adds a second update path next to
amdgpu_vm_pte_update_flags(). The alternatives are to let
amdgpu_vm_pte_update_flags() encode a directory entry when the level is
not AMDGPU_VM_PTB, or to pass already encoded dummy PDEs in dst[] from
amdgpu_vm_handle_fault(). Which one do you prefer?

Thanks,
Yu

Christian König (9):
  drm/amdgpu: start to move VM internals into amdgpu_vm_internal.h
  drm/amdgpu: remove unecessary parameters from
    trace_amdgpu_vm_update_ptes
  drm/amdgpu: allocate and fill dummy PDs/PTs
  drm/amdgpu: add amdgpu_vm_pt_leaves()
  drm/amdgpu: drop immediate updates from amdgpu_vm_update_range
  drm/amdgpu: split amdgpu_vm_update_range
  drm/amdgpu: nuke most amdgpu_vm_eviction_(try)lock uses
  drm/amdgpu: rework eviction lock handling into critical section v2
  drm/amdgpu: fix the HMM range handling for KFD SVM

Yu Huang (4):
  drm/amdgpu: free PT when VM critical section cannot resume
  drm/amdgpu: update the PTB itself in amdgpu_vm_pt_leaves()
  drm/amdgpu: pass an exclusive end to amdgpu_vm_pt_leaves()
  drm/amdgpu: use directory flags for dummy PDEs

-- 
2.53.0

[PATCH 01/13] drm/amdgpu: start to move VM internals into amdgpu_vm_internal.h

[Huang Yu]

From: Christian König <ckoenig.leichtzumerken@gmail.com>

Avoid exposing some defines outside of the VM code.

Co-developed by Claude Sonnet 4.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h     |   2 +
 .../gpu/drm/amd/amdgpu/amdgpu_trace_points.c  |   1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        |   1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h        | 116 -------------
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c    |   1 +
 .../gpu/drm/amd/amdgpu/amdgpu_vm_internal.h   | 155 ++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c     |   1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c   |   1 +
 8 files changed, 162 insertions(+), 116 deletions(-)
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index 5324030a13f5..784adf975b04 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -37,6 +37,8 @@
 #define AMDGPU_JOB_GET_TIMELINE_NAME(job) \
 	 job->base.s_fence->finished.ops->get_timeline_name(&job->base.s_fence->finished)
 
+struct amdgpu_vm_update_params;
+
 TRACE_EVENT(amdgpu_device_rreg,
 	    TP_PROTO(unsigned did, uint32_t reg, uint32_t value),
 	    TP_ARGS(did, reg, value),
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
index b96d885f6e33..fe61ed5b487e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
@@ -25,6 +25,7 @@
 #include <drm/amdgpu_drm.h>
 #include "amdgpu_cs.h"
 #include "amdgpu.h"
+#include "amdgpu_vm_internal.h"
 
 #define CREATE_TRACE_POINTS
 #include "amdgpu_trace.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 88249aa89ee3..517490946a9c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -37,6 +37,7 @@
 #include <drm/drm_exec.h>
 #include "amdgpu.h"
 #include "amdgpu_vm.h"
+#include "amdgpu_vm_internal.h"
 #include "amdgpu_trace.h"
 #include "amdgpu_amdkfd.h"
 #include "amdgpu_gmc.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index d32183cd9e0f..ac3b8e8525c1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -262,80 +262,6 @@ struct amdgpu_task_info {
 	struct kref	refcount;
 };
 
-/**
- * struct amdgpu_vm_update_params
- *
- * Encapsulate some VM table update parameters to reduce
- * the number of function parameters
- *
- */
-struct amdgpu_vm_update_params {
-
-	/**
-	 * @adev: amdgpu device we do this update for
-	 */
-	struct amdgpu_device *adev;
-
-	/**
-	 * @vm: optional amdgpu_vm we do this update for
-	 */
-	struct amdgpu_vm *vm;
-
-	/**
-	 * @immediate: if changes should be made immediately
-	 */
-	bool immediate;
-
-	/**
-	 * @unlocked: true if the root BO is not locked
-	 */
-	bool unlocked;
-
-	/**
-	 * @pages_addr:
-	 *
-	 * DMA addresses to use for mapping
-	 */
-	dma_addr_t *pages_addr;
-
-	/**
-	 * @job: job to used for hw submission
-	 */
-	struct amdgpu_job *job;
-
-	/**
-	 * @num_dw_left: number of dw left for the IB
-	 */
-	unsigned int num_dw_left;
-
-	/**
-	 * @needs_flush: true whenever we need to invalidate the TLB
-	 */
-	bool needs_flush;
-
-	/**
-	 * @override_pte: true for memory that is not uncached and gmc override function is
-	 * implemented to allow MTYPE to be overridden for NUMA local memory.
-	 */
-	bool override_pte;
-
-	/**
-	 * @tlb_flush_waitlist: temporary storage for BOs until tlb_flush
-	 */
-	struct list_head tlb_flush_waitlist;
-};
-
-struct amdgpu_vm_update_funcs {
-	int (*map_table)(struct amdgpu_bo_vm *bo);
-	int (*prepare)(struct amdgpu_vm_update_params *p,
-		       struct amdgpu_sync *sync, u64 k_job_id);
-	int (*update)(struct amdgpu_vm_update_params *p,
-		      struct amdgpu_bo_vm *bo, uint64_t pe, uint64_t addr,
-		      unsigned count, uint32_t incr, uint64_t flags);
-	int (*commit)(struct amdgpu_vm_update_params *p,
-		      struct dma_fence **fence);
-};
-
 struct amdgpu_vm_fault_info {
 	/* fault address */
 	uint64_t	addr;
@@ -600,22 +526,6 @@ void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
 void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
 			  struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM]);
 
-int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-		       struct amdgpu_bo_vm *vmbo, bool immediate);
-int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			int level, bool immediate, struct amdgpu_bo_vm **vmbo,
-			int32_t xcp_id);
-void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm);
-
-int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
-			 struct amdgpu_vm_bo_base *entry);
-int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
-			  uint64_t start, uint64_t end,
-			  uint64_t dst, uint64_t flags);
-void amdgpu_vm_pt_free_work(struct work_struct *work);
-void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
-			    struct amdgpu_vm_update_params *params);
-
 #if defined(CONFIG_DEBUG_FS)
 void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m);
 #endif
@@ -651,32 +561,6 @@ static inline uint64_t amdgpu_vm_tlb_seq(struct amdgpu_vm *vm)
 	return atomic64_read(&vm->tlb_seq);
 }
 
-/*
- * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS
- * happens while holding this lock anywhere to prevent deadlocks when
- * an MMU notifier runs in reclaim-FS context.
- */
-static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm)
-{
-	mutex_lock(&vm->eviction_lock);
-	vm->saved_flags = memalloc_noreclaim_save();
-}
-
-static inline bool amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm)
-{
-	if (mutex_trylock(&vm->eviction_lock)) {
-		vm->saved_flags = memalloc_noreclaim_save();
-		return true;
-	}
-	return false;
-}
-
-static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm)
-{
-	memalloc_noreclaim_restore(vm->saved_flags);
-	mutex_unlock(&vm->eviction_lock);
-}
-
 void amdgpu_vm_update_fault_cache(struct amdgpu_device *adev,
 				  unsigned int pasid,
 				  uint64_t addr,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
index b31ff6f56f0d..d03b9bbe20c7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
@@ -21,6 +21,7 @@
  */
 
 #include "amdgpu_vm.h"
+#include "amdgpu_vm_internal.h"
 #include "amdgpu.h"
 #include "amdgpu_reset.h"
 #include "amdgpu_object.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
new file mode 100644
index 000000000000..447195cc4593
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
@@ -0,0 +1,155 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Christian König
+ */
+#ifndef __AMDGPU_VM_INTERNAL_H__
+#define __AMDGPU_VM_INTERNAL_H__
+
+#include <linux/types.h>
+#include <linux/list.h>
+#include "amdgpu_vm.h"
+
+struct amdgpu_device;
+struct amdgpu_vm;
+struct amdgpu_job;
+struct amdgpu_sync;
+struct amdgpu_bo_vm;
+struct amdgpu_vm_bo_base;
+struct dma_fence;
+
+/**
+ * struct amdgpu_vm_update_params
+ *
+ * Encapsulate some VM table update parameters to reduce
+ * the number of function parameters
+ *
+ */
+struct amdgpu_vm_update_params {
+
+	/**
+	 * @adev: amdgpu device we do this update for
+	 */
+	struct amdgpu_device *adev;
+
+	/**
+	 * @vm: optional amdgpu_vm we do this update for
+	 */
+	struct amdgpu_vm *vm;
+
+	/**
+	 * @immediate: if changes should be made immediately
+	 */
+	bool immediate;
+
+	/**
+	 * @unlocked: true if the root BO is not locked
+	 */
+	bool unlocked;
+
+	/**
+	 * @pages_addr:
+	 *
+	 * DMA addresses to use for mapping
+	 */
+	dma_addr_t *pages_addr;
+
+	/**
+	 * @job: job to used for hw submission
+	 */
+	struct amdgpu_job *job;
+
+	/**
+	 * @num_dw_left: number of dw left for the IB
+	 */
+	unsigned int num_dw_left;
+
+	/**
+	 * @needs_flush: true whenever we need to invalidate the TLB
+	 */
+	bool needs_flush;
+
+	/**
+	 * @override_pte: true for memory that is not uncached and gmc override function is
+	 * implemented to allow MTYPE to be overridden for NUMA local memory.
+	 */
+	bool override_pte;
+
+	/**
+	 * @tlb_flush_waitlist: temporary storage for BOs until tlb_flush
+	 */
+	struct list_head tlb_flush_waitlist;
+};
+
+struct amdgpu_vm_update_funcs {
+	int (*map_table)(struct amdgpu_bo_vm *bo);
+	int (*prepare)(struct amdgpu_vm_update_params *p,
+		       struct amdgpu_sync *sync, u64 k_job_id);
+	int (*update)(struct amdgpu_vm_update_params *p,
+		      struct amdgpu_bo_vm *bo, uint64_t pe, uint64_t addr,
+		      unsigned count, uint32_t incr, uint64_t flags);
+	int (*commit)(struct amdgpu_vm_update_params *p,
+		      struct dma_fence **fence);
+};
+
+int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+		       struct amdgpu_bo_vm *vmbo, bool immediate);
+int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+			int level, bool immediate, struct amdgpu_bo_vm **vmbo,
+			int32_t xcp_id);
+void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm);
+int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
+			 struct amdgpu_vm_bo_base *entry);
+int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
+			  uint64_t start, uint64_t end,
+			  uint64_t dst, uint64_t flags);
+void amdgpu_vm_pt_free_work(struct work_struct *work);
+void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
+			    struct amdgpu_vm_update_params *params);
+int amdgpu_vm_pt_map_tables(struct amdgpu_device *adev, struct amdgpu_vm *vm);
+
+/*
+ * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS
+ * happens while holding this lock anywhere to prevent deadlocks when
+ * an MMU notifier runs in reclaim-FS context.
+ */
+static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm)
+{
+	mutex_lock(&vm->eviction_lock);
+	vm->saved_flags = memalloc_noreclaim_save();
+}
+
+static inline bool amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm)
+{
+	if (mutex_trylock(&vm->eviction_lock)) {
+		vm->saved_flags = memalloc_noreclaim_save();
+		return true;
+	}
+	return false;
+}
+
+static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm)
+{
+	memalloc_noreclaim_restore(vm->saved_flags);
+	mutex_unlock(&vm->eviction_lock);
+}
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index e43a60d09808..0d7fe32d0c7d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -26,6 +26,7 @@
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
 #include "amdgpu_vm.h"
+#include "amdgpu_vm_internal.h"
 #include "amdgpu_job.h"
 
 /*
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
index 430274237689..a6ae2f62f80d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
@@ -21,6 +21,7 @@
  */
 
 #include "amdgpu_vm.h"
+#include "amdgpu_vm_internal.h"
 #include "amdgpu_job.h"
 #include "amdgpu_object.h"
 #include "amdgpu_trace.h"
-- 
2.53.0

[PATCH 02/13] drm/amdgpu: remove unecessary parameters from trace_amdgpu_vm_update_ptes

[Huang Yu]

From: Christian König <ckoenig.leichtzumerken@gmail.com>

Those can be determined by the parameter structure as well.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h | 10 +++++-----
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c |  5 +----
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index 784adf975b04..599c1d0e512b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -331,9 +331,8 @@ TRACE_EVENT(amdgpu_vm_update_ptes,
 	    TP_PROTO(struct amdgpu_vm_update_params *p,
 		     uint64_t start, uint64_t end,
 		     unsigned int nptes, uint64_t dst,
-		     uint64_t incr, uint64_t flags,
-		     pid_t pid, uint64_t vm_ctx),
-	TP_ARGS(p, start, end, nptes, dst, incr, flags, pid, vm_ctx),
+		     uint64_t incr, uint64_t flags),
+	TP_ARGS(p, start, end, nptes, dst, incr, flags),
 	TP_STRUCT__entry(
 			 __field(u64, start)
 			 __field(u64, end)
@@ -353,8 +352,9 @@ TRACE_EVENT(amdgpu_vm_update_ptes,
 			__entry->flags = flags;
 			__entry->incr = incr;
 			__entry->nptes = nptes;
-			__entry->pid = pid;
-			__entry->vm_ctx = vm_ctx;
+			__entry->pid = p->vm->task_info ?
+				p->vm->task_info->tgid : 0;
+			__entry->vm_ctx = p->vm->immediate.fence_context;
 			for (i = 0; i < nptes; ++i) {
 				u64 addr = p->pages_addr ? amdgpu_vm_map_gart(
 					p->pages_addr, dst) : dst;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index 0d7fe32d0c7d..b85b70a1a428 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -882,7 +882,6 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 		entry_end = min(entry_end, end);
 
 		do {
-			struct amdgpu_vm *vm = params->vm;
 			uint64_t upd_end = min(entry_end, frag_end);
 			unsigned int nptes = (upd_end - frag_start) >> shift;
 			uint64_t upd_flags = flags | AMDGPU_PTE_FRAG(frag);
@@ -894,9 +893,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 
 			trace_amdgpu_vm_update_ptes(params, frag_start, upd_end,
 						    min(nptes, 32u), dst, incr,
-						    upd_flags,
-						    vm->task_info ? vm->task_info->tgid : 0,
-						    vm->immediate.fence_context);
+						    upd_flags);
 			amdgpu_vm_pte_update_flags(params, to_amdgpu_bo_vm(pt),
 						   cursor.level, pe_start, dst,
 						   nptes, incr, upd_flags);
-- 
2.53.0

[PATCH 03/13] drm/amdgpu: allocate and fill dummy PDs/PTs

[Huang Yu]

From: Christian König <ckoenig.leichtzumerken@gmail.com>

Allocate some PDs/PTs which just point to the dummy page.

Those can be used in page faults to redirect recoverable page faults
to the dummy page.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        | 11 +++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h        |  5 +-
 .../gpu/drm/amd/amdgpu/amdgpu_vm_internal.h   |  2 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c     | 57 +++++++++++++++++++
 4 files changed, 73 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 517490946a9c..5675925ca656 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -2843,8 +2843,10 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
  *
  * Initialize the VM manager structures
  */
-void amdgpu_vm_manager_init(struct amdgpu_device *adev)
+int amdgpu_vm_manager_init(struct amdgpu_device *adev)
 {
+	int r;
+
 	/* Concurrent flushes are only possible starting with Vega10 and
 	 * are broken on Navi10 and Navi14.
 	 */
@@ -2856,6 +2858,10 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
 	spin_lock_init(&adev->vm_manager.prt_lock);
 	atomic_set(&adev->vm_manager.num_prt_users, 0);
 
+	r = amdgpu_vm_pt_alloc_dummies(adev);
+	if (r)
+		return r;
+
 	/* If not overridden by the user, by default, only in large BAR systems
 	 * Compute VM tables will be updated by CPU
 	 */
@@ -2875,6 +2881,8 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
 #else
 	adev->vm_manager.vm_update_mode = 0;
 #endif
+
+	return 0;
 }
 
 /**
@@ -2886,6 +2894,7 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
  */
 void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
 {
+	amdgpu_vm_pt_free_dummies(adev);
 	amdgpu_vmid_mgr_fini(adev);
 	amdgpu_pasid_mgr_cleanup();
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index ac3b8e8525c1..826406522b24 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -409,6 +409,9 @@ struct amdgpu_vm_manager {
 
 	/* Global registration of recent page fault information */
 	struct amdgpu_vm_fault_info	fault_info;
+
+	struct amdgpu_bo		*dummy_pd[AMDGPU_VM_PTB + 1];
+	uint64_t			dummy_dst[AMDGPU_VM_PTB + 1];
 };
 
 struct amdgpu_bo_va_mapping;
@@ -420,7 +423,7 @@ struct amdgpu_bo_va_mapping;
 extern const struct amdgpu_vm_update_funcs amdgpu_vm_cpu_funcs;
 extern const struct amdgpu_vm_update_funcs amdgpu_vm_sdma_funcs;
 
-void amdgpu_vm_manager_init(struct amdgpu_device *adev);
+int amdgpu_vm_manager_init(struct amdgpu_device *adev);
 void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
 
 long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
index 447195cc4593..16cc0d5f5894 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
@@ -125,6 +125,8 @@ void amdgpu_vm_pt_free_work(struct work_struct *work);
 void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
 			    struct amdgpu_vm_update_params *params);
 int amdgpu_vm_pt_map_tables(struct amdgpu_device *adev, struct amdgpu_vm *vm);
+int amdgpu_vm_pt_alloc_dummies(struct amdgpu_device *adev);
+void amdgpu_vm_pt_free_dummies(struct amdgpu_device *adev);
 
 /*
  * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index b85b70a1a428..3a6449d60988 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -963,3 +963,60 @@ int amdgpu_vm_pt_map_tables(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 
 	return 0;
 }
+
+/* amdgpu_vm_pt_alloc_dummies - allocate dummy PDs/PTs
+ *
+ * @adev: the amdgpu device pointer
+ *
+ * Allocate some dummy PDs/PTs which can be used to redirect page faults to the
+ * dummy page.
+ */
+int amdgpu_vm_pt_alloc_dummies(struct amdgpu_device *adev)
+{
+	struct amdgpu_vm_manager *vm_mgr = &adev->vm_manager;
+	int r;
+
+	for (int level = AMDGPU_VM_PTB; level != vm_mgr->root_level; level--) {
+		size_t size =  amdgpu_vm_pt_size(adev, level);
+		uint64_t addr, flags;
+		void *ptr;
+
+		r = amdgpu_bo_create_kernel(adev, size, 0,
+					    AMDGPU_GEM_DOMAIN_VRAM,
+					    &vm_mgr->dummy_pd[level],
+					    &vm_mgr->dummy_dst[level],
+					    &ptr);
+		if (r)
+			return r;
+
+		if (level == AMDGPU_VM_PTB) {
+			addr = adev->dummy_page_addr;
+			/*
+			 * TODO: We want to have separate dummies for reads and
+			 * writes.
+			 */
+			flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED |
+				AMDGPU_PTE_SYSTEM | AMDGPU_PTE_EXECUTABLE |
+				AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE;
+		} else {
+			amdgpu_gmc_get_pde_for_bo(vm_mgr->dummy_pd[level + 1],
+						  level, &addr, &flags);
+		}
+
+		for (int i = 0; i < amdgpu_vm_pt_num_entries(adev, level); i++)
+			amdgpu_gmc_set_pte_pde(adev, ptr, i, addr, flags);
+	}
+
+	return 0;
+}
+
+void amdgpu_vm_pt_free_dummies(struct amdgpu_device *adev)
+{
+	struct amdgpu_vm_manager *vm_mgr = &adev->vm_manager;
+	void *ptr;
+
+	for (int level = AMDGPU_VM_PTB; level != vm_mgr->root_level; level--)
+		amdgpu_bo_free_kernel(&vm_mgr->dummy_pd[level],
+				      &vm_mgr->dummy_dst[level],
+				      &ptr);
+}
-- 
2.53.0

[PATCH 04/13] drm/amdgpu: add amdgpu_vm_pt_leaves()

[Huang Yu]

From: Christian König <ckoenig.leichtzumerken@gmail.com>

Add a new function amdgpu_vm_update_leaves() to avoid memory allocation
on page faults.

The idea is to only update the leave PDEs/PTEs to let them point to the
dummy page.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        | 55 +++++++++++-----
 .../gpu/drm/amd/amdgpu/amdgpu_vm_internal.h   |  3 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c     | 62 +++++++++++++++++++
 3 files changed, 105 insertions(+), 15 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 5675925ca656..e60143d4990c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -3013,10 +3013,12 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
 			    u32 vmid, u32 node_id, uint64_t addr,
 			    uint64_t ts, bool write_fault)
 {
+	struct amdgpu_vm_update_params params;
 	bool is_compute_context = false;
-	struct drm_exec exec;
-	uint64_t value, flags;
+	uint64_t *values, flags;
 	struct amdgpu_vm *vm;
+	struct drm_exec exec;
+	unsigned int idx;
 	int r;
 
 	drm_exec_init(&exec, 0, 1);
@@ -3060,24 +3062,21 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
 	}
 
 	addr /= AMDGPU_GPU_PAGE_SIZE;
-	flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED |
-		AMDGPU_PTE_SYSTEM;
-
 	if (is_compute_context) {
 		/* Intentionally setting invalid PTE flag
 		 * combination to force a no-retry-fault
 		 */
 		flags = AMDGPU_VM_NORETRY_FLAGS;
-		value = 0;
+		values = NULL;
 	} else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) {
 		/* Redirect the access to the dummy page */
-		value = adev->dummy_page_addr;
-		flags |= AMDGPU_PTE_EXECUTABLE | AMDGPU_PTE_READABLE |
-			AMDGPU_PTE_WRITEABLE;
-
+		flags = 0;
+		values = adev->vm_manager.dummy_dst;
 	} else {
 		/* Let the hw retry silently on the PTE */
-		value = 0;
+		flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED |
+			AMDGPU_PTE_SYSTEM;
+		values = NULL;
 	}
 
 	r = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1);
@@ -3086,12 +3085,38 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
 		goto error_unlock;
 	}
 
-	r = amdgpu_vm_update_range(adev, vm, true, false, false, false,
-				   NULL, addr, addr, flags, value, 0, NULL, NULL, NULL);
-	if (r)
+	if (!drm_dev_enter(adev_to_drm(adev), &idx)) {
+		r = -ENODEV;
 		goto error_unlock;
+	}
 
-	r = amdgpu_vm_update_pdes(adev, vm, true);
+	amdgpu_vm_eviction_lock(vm);
+	if (vm->evicting) {
+		r = -EBUSY;
+		goto error_dev_exit;
+	}
+
+	memset(&params, 0, sizeof(params));
+	params.adev = adev;
+	params.vm = vm;
+	params.immediate = true;
+	params.pages_addr = NULL;
+
+	r = vm->update_funcs->prepare(&params, NULL,
+				      AMDGPU_KERNEL_JOB_ID_VM_UPDATE_PDES);
+	if (r)
+		goto error_dev_exit;
+
+	amdgpu_vm_pt_leaves(&params, addr, addr, values, flags);
+
+	r = vm->update_funcs->commit(&params, &vm->last_update);
+	if (r)
+		goto error_dev_exit;
+
+	amdgpu_vm_eviction_unlock(vm);
+
+error_dev_exit:
+	drm_dev_exit(idx);
 
 error_unlock:
 	drm_exec_fini(&exec);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
index 16cc0d5f5894..7d3c0904e1a1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
@@ -121,6 +121,9 @@ int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
 int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 			  uint64_t start, uint64_t end,
 			  uint64_t dst, uint64_t flags);
+void amdgpu_vm_pt_leaves(struct amdgpu_vm_update_params *params,
+			 uint64_t start, uint64_t end,
+			 int64_t *dst, uint64_t flags);
 void amdgpu_vm_pt_free_work(struct work_struct *work);
 void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
 			    struct amdgpu_vm_update_params *params);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index 3a6449d60988..2f542c731447 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -936,6 +936,68 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 	return 0;
 }
 
+/**
+ * amdgpu_vm_pt_leaves - update leave PDEs/PTEs
+ *
+ * @params: see amdgpu_vm_update_params definition
+ * @start: start of GPU address range
+ * @end: end of GPU address range
+ * @dst: optional array with one dst addr per layer
+ * @flags: mapping flags
+ *
+ * Update the leave PDEs/PTEs in the range @start - @end without allocating or
+ * freeing page tables.
+ *
+ * Returns:
+ * 0 for success, negative error code for failure.
+ */
+void amdgpu_vm_pt_leaves(struct amdgpu_vm_update_params *params,
+			 uint64_t start, uint64_t end,
+			 int64_t *dst, uint64_t flags)
+{
+	struct amdgpu_device *adev = params->adev;
+	struct amdgpu_vm_pt_cursor cursor;
+
+	amdgpu_vm_pt_start(adev, params->vm, start, &cursor);
+	while (cursor.pfn < end) {
+		uint64_t entry_end, pe_start, val;
+		unsigned int shift, mask;
+		struct amdgpu_bo *pt;
+		unsigned int nptes;
+
+		/* Walk to the leave entries */
+		if (amdgpu_vm_pt_descendant(adev, &cursor))
+			continue;
+
+		pt = cursor.parent->bo;
+		shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1);
+		mask = amdgpu_vm_pt_entries_mask(adev, cursor.level - 1);
+
+		/* Looks good so far, calculate parameters for the update */
+		pe_start = ((cursor.pfn >> shift) & mask) * 8;
+
+		entry_end = ((uint64_t)mask + 1) << shift;
+		entry_end += cursor.pfn & ~(entry_end - 1);
+		entry_end = min(entry_end, end);
+
+		nptes = (entry_end - cursor.pfn) >> shift;
+		/*
+		 * This can happen when we set higher level PDEs to unmap and/or
+		 * silent to stop fault floods.
+		 */
+		nptes = max(nptes, 1u);
+
+		val = dst ? dst[cursor.level] : 0;
+		trace_amdgpu_vm_update_ptes(params, cursor.pfn, entry_end,
+					    min(nptes, 32u), val, 0, flags);
+
+		params->vm->update_funcs->update(params, to_amdgpu_bo_vm(pt),
+						 pe_start, val, nptes, 0,
+						 flags);
+		amdgpu_vm_pt_next(adev, &cursor);
+	}
+}
+
 /**
  * amdgpu_vm_pt_map_tables - have bo of root PD cpu accessible
  * @adev: amdgpu device structure
-- 
2.53.0

[PATCH 05/13] drm/amdgpu: drop immediate updates from amdgpu_vm_update_range

[Huang Yu]

From: Christian König <ckoenig.leichtzumerken@gmail.com>

That case is handled by amdgpu_vm_update_leaves now.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        | 21 ++++++++-----------
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h        | 11 +++++-----
 .../gpu/drm/amd/amdgpu/amdgpu_vm_internal.h   |  5 ++---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c     | 19 ++++++-----------
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c          |  4 ++--
 5 files changed, 24 insertions(+), 36 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index e60143d4990c..6a69e876e729 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -1109,7 +1109,6 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
  *
  * @adev: amdgpu_device pointer to use for commands
  * @vm: the VM to update the range
- * @immediate: immediate submission in a page fault
  * @unlocked: unlocked invalidation during MM callback
  * @flush_tlb: trigger tlb invalidation after update completed
  * @allow_override: change MTYPE for local NUMA nodes
@@ -1129,12 +1128,11 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
  * 0 for success, negative erro code for failure.
  */
 int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			   bool immediate, bool unlocked, bool flush_tlb,
-			   bool allow_override, struct amdgpu_sync *sync,
-			   uint64_t start, uint64_t last, uint64_t flags,
-			   uint64_t offset, uint64_t vram_base,
-			   struct ttm_resource *res, dma_addr_t *pages_addr,
-			   struct dma_fence **fence)
+			   bool unlocked, bool flush_tlb, bool allow_override,
+			   struct amdgpu_sync *sync, uint64_t start,
+			   uint64_t last, uint64_t flags, uint64_t offset,
+			   uint64_t vram_base, struct ttm_resource *res,
+			   dma_addr_t *pages_addr, struct dma_fence **fence)
 {
 	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
 	struct amdgpu_vm_update_params params;
@@ -1164,7 +1162,6 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	memset(&params, 0, sizeof(params));
 	params.adev = adev;
 	params.vm = vm;
-	params.immediate = immediate;
 	params.pages_addr = pages_addr;
 	params.unlocked = unlocked;
 	params.needs_flush = flush_tlb;
@@ -1390,7 +1387,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
 
 		trace_amdgpu_vm_bo_update(mapping);
 
-		r = amdgpu_vm_update_range(adev, vm, false, false, flush_tlb,
+		r = amdgpu_vm_update_range(adev, vm, false, flush_tlb,
 					   !uncached, &sync, mapping->start,
 					   mapping->last, update_flags,
 					   mapping->offset, vram_base, mem,
@@ -1601,7 +1598,7 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
 			struct amdgpu_bo_va_mapping, list);
 		list_del(&mapping->list);
 
-		r = amdgpu_vm_update_range(adev, vm, false, false, true, false,
+		r = amdgpu_vm_update_range(adev, vm, false, true, false,
 					   &sync, mapping->start, mapping->last,
 					   0, 0, 0, NULL, NULL, &f);
 		amdgpu_vm_free_mapping(adev, vm, mapping, f);
@@ -2644,7 +2641,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	vm->tlb_fence_context = dma_fence_context_alloc(1);
 
 	r = amdgpu_vm_pt_create(adev, vm, adev->vm_manager.root_level,
-				false, &root, xcp_id);
+				&root, xcp_id);
 	if (r)
 		goto error_free_delayed;
 
@@ -2660,7 +2657,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	if (r)
 		goto error_free_root;
 
-	r = amdgpu_vm_pt_clear(adev, vm, root, false);
+	r = amdgpu_vm_pt_clear(adev, vm, root);
 	if (r)
 		goto error_free_root;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 826406522b24..8ab20826ddf2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -458,12 +458,11 @@ int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev,
 void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
 			    struct amdgpu_vm *vm, struct amdgpu_bo *bo);
 int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			   bool immediate, bool unlocked, bool flush_tlb,
-			   bool allow_override, struct amdgpu_sync *sync,
-			   uint64_t start, uint64_t last, uint64_t flags,
-			   uint64_t offset, uint64_t vram_base,
-			   struct ttm_resource *res, dma_addr_t *pages_addr,
-			   struct dma_fence **fence);
+			   bool unlocked, bool flush_tlb, bool allow_override,
+			   struct amdgpu_sync *sync, uint64_t start,
+			   uint64_t last, uint64_t flags, uint64_t offset,
+			   uint64_t vram_base, struct ttm_resource *res,
+			   dma_addr_t *pages_addr, struct dma_fence **fence);
 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 			struct amdgpu_bo_va *bo_va,
 			bool clear);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
index 7d3c0904e1a1..1011f49f0478 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
@@ -111,10 +111,9 @@ struct amdgpu_vm_update_funcs {
 };
 
 int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-		       struct amdgpu_bo_vm *vmbo, bool immediate);
+		       struct amdgpu_bo_vm *vmbo);
 int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			int level, bool immediate, struct amdgpu_bo_vm **vmbo,
-			int32_t xcp_id);
+			int level, struct amdgpu_bo_vm **vmbo, int32_t xcp_id);
 void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
 			 struct amdgpu_vm_bo_base *entry);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index 2f542c731447..919e1967c5ba 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -352,7 +352,6 @@ static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev,
  * @adev: amdgpu_device pointer
  * @vm: VM to clear BO from
  * @vmbo: BO to clear
- * @immediate: use an immediate update
  *
  * Root PD needs to be reserved when calling this.
  *
@@ -360,7 +359,7 @@ static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev,
  * 0 on success, errno otherwise.
  */
 int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-		       struct amdgpu_bo_vm *vmbo, bool immediate)
+		       struct amdgpu_bo_vm *vmbo)
 {
 	unsigned int level = adev->vm_manager.root_level;
 	struct ttm_operation_ctx ctx = { true, false };
@@ -397,7 +396,6 @@ int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	memset(&params, 0, sizeof(params));
 	params.adev = adev;
 	params.vm = vm;
-	params.immediate = immediate;
 
 	r = vm->update_funcs->prepare(&params, NULL,
 				      AMDGPU_KERNEL_JOB_ID_VM_PT_CLEAR);
@@ -435,13 +433,11 @@ int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  * @adev: amdgpu_device pointer
  * @vm: requesting vm
  * @level: the page table level
- * @immediate: use a immediate update
  * @vmbo: pointer to the buffer object pointer
  * @xcp_id: GPU partition id
  */
 int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			int level, bool immediate, struct amdgpu_bo_vm **vmbo,
-			int32_t xcp_id)
+			int level, struct amdgpu_bo_vm **vmbo, int32_t xcp_id)
 {
 	struct amdgpu_bo_param bp;
 	unsigned int num_entries;
@@ -471,7 +467,6 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		bp.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 
 	bp.type = ttm_bo_type_kernel;
-	bp.no_wait_gpu = immediate;
 	bp.xcp_id_plus1 = xcp_id + 1;
 
 	if (vm->root.bo)
@@ -486,7 +481,6 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  * @adev: amdgpu_device pointer
  * @vm: VM to allocate page tables for
  * @cursor: Which page table to allocate
- * @immediate: use an immediate update
  *
  * Make sure a specific page table or directory is allocated.
  *
@@ -496,8 +490,7 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  */
 static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev,
 			      struct amdgpu_vm *vm,
-			      struct amdgpu_vm_pt_cursor *cursor,
-			      bool immediate)
+			      struct amdgpu_vm_pt_cursor *cursor)
 {
 	struct amdgpu_vm_bo_base *entry = cursor->entry;
 	struct amdgpu_bo *pt_bo;
@@ -508,7 +501,7 @@ static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev,
 		return 0;
 
 	amdgpu_vm_eviction_unlock(vm);
-	r = amdgpu_vm_pt_create(adev, vm, cursor->level, immediate, &pt,
+	r = amdgpu_vm_pt_create(adev, vm, cursor->level, &pt,
 				vm->root.bo->xcp_id);
 	amdgpu_vm_eviction_lock(vm);
 	if (r)
@@ -520,7 +513,7 @@ static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev,
 	pt_bo = &pt->bo;
 	pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo);
 	amdgpu_vm_bo_base_init(entry, vm, pt_bo);
-	r = amdgpu_vm_pt_clear(adev, vm, pt, immediate);
+	r = amdgpu_vm_pt_clear(adev, vm, pt);
 	if (r)
 		goto error_free_pt;
 
@@ -812,7 +805,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 			 * address range are actually allocated
 			 */
 			r = amdgpu_vm_pt_alloc(params->adev, params->vm,
-					       &cursor, params->immediate);
+					       &cursor);
 			if (r)
 				return r;
 		}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index fa4054d51f60..bc754f407f00 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -1406,7 +1406,7 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		return -EINVAL;
 	}
 
-	return amdgpu_vm_update_range(adev, vm, false, true, true, false, NULL, gpu_start,
+	return amdgpu_vm_update_range(adev, vm, true, true, false, NULL, gpu_start,
 				      gpu_end, init_pte_value, 0, 0, NULL, NULL,
 				      fence);
 }
@@ -1518,7 +1518,7 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
 			 (last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0,
 			 pte_flags);
 
-		r = amdgpu_vm_update_range(adev, vm, false, false, flush_tlb, true,
+		r = amdgpu_vm_update_range(adev, vm, false, flush_tlb, true,
 					   NULL, gpu_start, gpu_end,
 					   pte_flags,
 					   (last_start - prange->start) << PAGE_SHIFT,
-- 
2.53.0

[PATCH 06/13] drm/amdgpu: split amdgpu_vm_update_range

[Huang Yu]

From: Christian König <ckoenig.leichtzumerken@gmail.com>

Split amdgpu_vm_update_range into two functions.

amdgpu_vm_map_range() is for mapping PTEs into a range and updates
which can be done while holding the VM lock.

amdgpu_vm_unmap_range() is for unmapping PTEs without holding the VM
lock in MMU notifiers.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.h   |   3 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 112 ++++++++++++++++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h    |  14 ++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c |  35 ++-----
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c      |  16 ++--
 5 files changed, 120 insertions(+), 60 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
index 7e03bd2ff639..16832b0914fa 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
@@ -47,7 +47,7 @@ enum amdgpu_ib_pool_type;
 /* Internal kernel job ids. (decreasing values, starting from U64_MAX). */
 #define AMDGPU_KERNEL_JOB_ID_VM_UPDATE              (18446744073709551615ULL)
 #define AMDGPU_KERNEL_JOB_ID_VM_UPDATE_PDES         (18446744073709551614ULL)
-#define AMDGPU_KERNEL_JOB_ID_VM_UPDATE_RANGE        (18446744073709551613ULL)
+#define AMDGPU_KERNEL_JOB_ID_VM_MAP_RANGE           (18446744073709551613ULL)
 #define AMDGPU_KERNEL_JOB_ID_VM_PT_CLEAR            (18446744073709551612ULL)
 #define AMDGPU_KERNEL_JOB_ID_TTM_MAP_BUFFER         (18446744073709551611ULL)
 #define AMDGPU_KERNEL_JOB_ID_TTM_ACCESS_MEMORY_SDMA (18446744073709551610ULL)
@@ -63,6 +63,7 @@ enum amdgpu_ib_pool_type;
 #define AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST         (18446744073709551600ULL)
 #define AMDGPU_KERNEL_JOB_ID_VPE_RING_TEST          (18446744073709551599ULL)
 #define AMDGPU_KERNEL_JOB_ID_RUN_SHADER             (18446744073709551598ULL)
+#define AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE         (18446744073709551597ULL)
 
 struct amdgpu_job {
 	struct drm_sched_job    base;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 6a69e876e729..77437436292c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -1105,11 +1105,10 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
 }
 
 /**
- * amdgpu_vm_update_range - update a range in the vm page table
+ * amdgpu_vm_map_range - map something to a range in the vm page tables
  *
  * @adev: amdgpu_device pointer to use for commands
  * @vm: the VM to update the range
- * @unlocked: unlocked invalidation during MM callback
  * @flush_tlb: trigger tlb invalidation after update completed
  * @allow_override: change MTYPE for local NUMA nodes
  * @sync: fences we need to sync to
@@ -1122,23 +1121,26 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
  * @pages_addr: DMA addresses to use for mapping
  * @fence: optional resulting fence
  *
- * Fill in the page table entries between @start and @last.
+ * Fill in the page table entries between @start and @last. Allocate and free
+ * new page tables as needed. Can only be called while holding the VM lock.
  *
  * Returns:
  * 0 for success, negative erro code for failure.
  */
-int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			   bool unlocked, bool flush_tlb, bool allow_override,
-			   struct amdgpu_sync *sync, uint64_t start,
-			   uint64_t last, uint64_t flags, uint64_t offset,
-			   uint64_t vram_base, struct ttm_resource *res,
-			   dma_addr_t *pages_addr, struct dma_fence **fence)
+int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+			bool flush_tlb, bool allow_override,
+			struct amdgpu_sync *sync, uint64_t start,
+			uint64_t last, uint64_t flags, uint64_t offset,
+			uint64_t vram_base, struct ttm_resource *res,
+			dma_addr_t *pages_addr, struct dma_fence **fence)
 {
 	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
 	struct amdgpu_vm_update_params params;
 	struct amdgpu_res_cursor cursor;
 	int r, idx;
 
+	amdgpu_vm_assert_locked(vm);
+
 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
 		return -ENODEV;
 
@@ -1163,7 +1165,6 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	params.adev = adev;
 	params.vm = vm;
 	params.pages_addr = pages_addr;
-	params.unlocked = unlocked;
 	params.needs_flush = flush_tlb;
 	params.override_pte = allow_override && adev->gmc.override_pte;
 	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
@@ -1174,7 +1175,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		goto error_free;
 	}
 
-	if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) {
+	if (!dma_fence_is_signaled(vm->last_unlocked)) {
 		struct dma_fence *tmp = dma_fence_get_stub();
 
 		amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true);
@@ -1183,7 +1184,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	}
 
 	r = vm->update_funcs->prepare(&params, sync,
-				      AMDGPU_KERNEL_JOB_ID_VM_UPDATE_RANGE);
+				      AMDGPU_KERNEL_JOB_ID_VM_MAP_RANGE);
 	if (r)
 		goto error_free;
 
@@ -1259,6 +1260,77 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	return r;
 }
 
+/**
+ * amdgpu_vm_unmap_range - clear leave PTEs to unmap something
+ *
+ * @adev: amdgpu_device pointer to use for commands
+ * @vm: the VM to update the range
+ * @sync: fences we need to sync to
+ * @start: start of unmapped range
+ * @last: last unmapped entry
+ * @flags: flags for the entries
+ * @fence: optional resulting fence
+ *
+ * Fill in the page table entries between @start and @last with a fixed flags
+ * value without allocating or freeing page tables. Can be used without locking
+ * the VM.
+ *
+ * Returns:
+ * 0 for success, negative erro code for failure.
+ */
+int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+			   struct amdgpu_sync *sync, uint64_t start,
+			   uint64_t last, uint64_t flags,
+			   struct dma_fence **fence)
+{
+	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
+	struct amdgpu_vm_update_params params;
+	int r, idx;
+
+	if (!drm_dev_enter(adev_to_drm(adev), &idx))
+		return -ENODEV;
+
+	tlb_cb = kmalloc(sizeof(*tlb_cb), GFP_KERNEL);
+	if (!tlb_cb) {
+		drm_dev_exit(idx);
+		return -ENOMEM;
+	}
+
+	memset(&params, 0, sizeof(params));
+	params.adev = adev;
+	params.vm = vm;
+	params.needs_flush = true;
+	params.unlocked = true;
+	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
+
+	amdgpu_vm_eviction_lock(vm);
+	if (vm->evicting) {
+		r = -EBUSY;
+		goto error_free;
+	}
+
+	r = vm->update_funcs->prepare(&params, sync,
+				      AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE);
+	if (r)
+		goto error_free;
+
+	amdgpu_vm_pt_leaves(&params, start, last, NULL, flags);
+
+	r = vm->update_funcs->commit(&params, fence);
+	if (r)
+		goto error_free;
+
+	amdgpu_vm_tlb_flush(&params, fence, tlb_cb);
+	amdgpu_vm_pt_free_list(adev, &params);
+	tlb_cb = NULL;
+
+error_free:
+	kfree(tlb_cb);
+	amdgpu_vm_eviction_unlock(vm);
+	drm_dev_exit(idx);
+	return r;
+}
+
 void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
 			  struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
 {
@@ -1387,11 +1459,11 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
 
 		trace_amdgpu_vm_bo_update(mapping);
 
-		r = amdgpu_vm_update_range(adev, vm, false, flush_tlb,
-					   !uncached, &sync, mapping->start,
-					   mapping->last, update_flags,
-					   mapping->offset, vram_base, mem,
-					   pages_addr, last_update);
+		r = amdgpu_vm_map_range(adev, vm, flush_tlb, !uncached, &sync,
+					mapping->start, mapping->last,
+					update_flags, mapping->offset,
+					vram_base, mem, pages_addr,
+					last_update);
 		if (r)
 			goto error_free;
 	}
@@ -1598,9 +1670,9 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
 			struct amdgpu_bo_va_mapping, list);
 		list_del(&mapping->list);
 
-		r = amdgpu_vm_update_range(adev, vm, false, true, false,
-					   &sync, mapping->start, mapping->last,
-					   0, 0, 0, NULL, NULL, &f);
+		r = amdgpu_vm_map_range(adev, vm, true, false,
+					&sync, mapping->start, mapping->last,
+					0, 0, 0, NULL, NULL, &f);
 		amdgpu_vm_free_mapping(adev, vm, mapping, f);
 		if (r) {
 			dma_fence_put(f);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 8ab20826ddf2..e871a9e3e1a9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -457,12 +457,16 @@ int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev,
 				uint32_t xcc_mask);
 void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
 			    struct amdgpu_vm *vm, struct amdgpu_bo *bo);
-int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			   bool unlocked, bool flush_tlb, bool allow_override,
+int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+			bool flush_tlb, bool allow_override,
+			struct amdgpu_sync *sync, uint64_t start,
+			uint64_t last, uint64_t flags, uint64_t offset,
+			uint64_t vram_base, struct ttm_resource *res,
+			dma_addr_t *pages_addr, struct dma_fence **fence);
+int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			   struct amdgpu_sync *sync, uint64_t start,
-			   uint64_t last, uint64_t flags, uint64_t offset,
-			   uint64_t vram_base, struct ttm_resource *res,
-			   dma_addr_t *pages_addr, struct dma_fence **fence);
+			   uint64_t last, uint64_t flags,
+			   struct dma_fence **fence);
 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 			struct amdgpu_bo_va *bo_va,
 			bool clear);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index 919e1967c5ba..5bb1a3300aa4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -554,7 +554,6 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
 			    struct amdgpu_vm_update_params *params)
 {
 	struct amdgpu_vm_bo_base *entry, *next;
-	bool unlocked = params->unlocked;
 
 	if (list_empty(&params->tlb_flush_waitlist))
 		return;
@@ -562,7 +561,7 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
 	/*
 	 * unlocked unmap clear page table leaves, warning to free the page entry.
 	 */
-	WARN_ON(unlocked);
+	WARN_ON(params->unlocked);
 
 	list_for_each_entry_safe(entry, next, &params->tlb_flush_waitlist, vm_status)
 		amdgpu_vm_pt_free(entry);
@@ -800,24 +799,17 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 		uint64_t incr, entry_end, pe_start;
 		struct amdgpu_bo *pt;
 
-		if (!params->unlocked) {
-			/* make sure that the page tables covering the
-			 * address range are actually allocated
-			 */
-			r = amdgpu_vm_pt_alloc(params->adev, params->vm,
-					       &cursor);
-			if (r)
-				return r;
-		}
+		/* make sure that the page tables covering the
+		 * address range are actually allocated
+		 */
+		r = amdgpu_vm_pt_alloc(params->adev, params->vm, &cursor);
+		if (r)
+			return r;
 
 		shift = amdgpu_vm_pt_level_shift(adev, cursor.level);
 		parent_shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1);
-		if (params->unlocked) {
-			/* Unlocked updates are only allowed on the leaves */
-			if (amdgpu_vm_pt_descendant(adev, &cursor))
-				continue;
-		} else if (adev->asic_type < CHIP_VEGA10 &&
-			   (flags & AMDGPU_PTE_VALID)) {
+		if (adev->asic_type < CHIP_VEGA10 &&
+		    (flags & AMDGPU_PTE_VALID)) {
 			/* No huge page support before GMC v9 */
 			if (cursor.level != AMDGPU_VM_PTB) {
 				if (!amdgpu_vm_pt_descendant(adev, &cursor))
@@ -863,14 +855,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 		mask = amdgpu_vm_pt_entries_mask(adev, cursor.level);
 		pe_start = ((cursor.pfn >> shift) & mask) * 8;
 
-		if (cursor.level < AMDGPU_VM_PTB && params->unlocked)
-			/*
-			 * MMU notifier callback unlocked unmap huge page, leave is PDE entry,
-			 * only clear one entry. Next entry search again for PDE or PTE leave.
-			 */
-			entry_end = 1ULL << shift;
-		else
-			entry_end = ((uint64_t)mask + 1) << shift;
+		entry_end = ((uint64_t)mask + 1) << shift;
 		entry_end += cursor.pfn & ~(entry_end - 1);
 		entry_end = min(entry_end, end);
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index bc754f407f00..93e490f00c83 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -1406,9 +1406,8 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		return -EINVAL;
 	}
 
-	return amdgpu_vm_update_range(adev, vm, true, true, false, NULL, gpu_start,
-				      gpu_end, init_pte_value, 0, 0, NULL, NULL,
-				      fence);
+	return amdgpu_vm_unmap_range(adev, vm, NULL, gpu_start, gpu_end,
+				     init_pte_value, fence);
 }
 
 static int
@@ -1518,12 +1517,11 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
 			 (last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0,
 			 pte_flags);
 
-		r = amdgpu_vm_update_range(adev, vm, false, flush_tlb, true,
-					   NULL, gpu_start, gpu_end,
-					   pte_flags,
-					   (last_start - prange->start) << PAGE_SHIFT,
-					   bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
-					   NULL, dma_addr, &vm->last_update);
+		r = amdgpu_vm_map_range(adev, vm, flush_tlb, true, NULL,
+					gpu_start, gpu_end, pte_flags,
+					(last_start - prange->start) << PAGE_SHIFT,
+					bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
+					NULL, dma_addr, &vm->last_update);
 
 		for (j = last_start - prange->start; j <= i; j++)
 			dma_addr[j] |= last_domain;
-- 
2.53.0

[PATCH 07/13] drm/amdgpu: nuke most amdgpu_vm_eviction_(try)lock uses

[Huang Yu]

From: Christian König <ckoenig.leichtzumerken@gmail.com>

Use scoped_guard/scoped_cond_guard instead.

Saving and restoring the GFP flags is not necessary in those use case.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        | 27 +++++++++----------
 .../gpu/drm/amd/amdgpu/amdgpu_vm_internal.h   |  9 -------
 2 files changed, 12 insertions(+), 24 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 77437436292c..0930b3fdade7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -616,9 +616,8 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	 * As soon as all page tables are in place we can start updating them
 	 * again.
 	 */
-	amdgpu_vm_eviction_lock(vm);
-	vm->evicting = false;
-	amdgpu_vm_eviction_unlock(vm);
+	scoped_guard(mutex, &vm->eviction_lock)
+		vm->evicting = false;
 
 	list_for_each_entry_safe(bo_base, tmp, &vm->always_valid.evicted,
 				 vm_status) {
@@ -678,9 +677,8 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 
 	amdgpu_vm_assert_locked(vm);
 
-	amdgpu_vm_eviction_lock(vm);
-	ret = !vm->evicting;
-	amdgpu_vm_eviction_unlock(vm);
+	scoped_guard(mutex, &vm->eviction_lock)
+		ret = !vm->evicting;
 
 	ret &= list_empty(&vm->kernel.evicted);
 
@@ -2340,6 +2338,7 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev,
 bool amdgpu_vm_evictable(struct amdgpu_bo *bo)
 {
 	struct amdgpu_vm_bo_base *bo_base = bo->vm_bo;
+	struct amdgpu_vm *vm;
 
 	/* Page tables of a destroyed VM can go away immediately */
 	if (!bo_base || !bo_base->vm)
@@ -2350,17 +2349,15 @@ bool amdgpu_vm_evictable(struct amdgpu_bo *bo)
 		return false;
 
 	/* Try to block ongoing updates */
-	if (!amdgpu_vm_eviction_trylock(bo_base->vm))
-		return false;
+	vm = bo_base->vm;
+	scoped_cond_guard(mutex_try, return false, &vm->eviction_lock) {
 
-	/* Don't evict VM page tables while they are updated */
-	if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) {
-		amdgpu_vm_eviction_unlock(bo_base->vm);
-		return false;
-	}
+		/* Don't evict VM page tables while they are updated */
+		if (!dma_fence_is_signaled(vm->last_unlocked))
+			return false;
 
-	bo_base->vm->evicting = true;
-	amdgpu_vm_eviction_unlock(bo_base->vm);
+		vm->evicting = true;
+	}
 	return true;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
index 1011f49f0478..7f6442d0b647 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
@@ -141,15 +141,6 @@ static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm)
 	vm->saved_flags = memalloc_noreclaim_save();
 }
 
-static inline bool amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm)
-{
-	if (mutex_trylock(&vm->eviction_lock)) {
-		vm->saved_flags = memalloc_noreclaim_save();
-		return true;
-	}
-	return false;
-}
-
 static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm)
 {
 	memalloc_noreclaim_restore(vm->saved_flags);
-- 
2.53.0

[PATCH 08/13] drm/amdgpu: rework eviction lock handling into critical section v2

[Huang Yu]

From: Christian König <ckoenig.leichtzumerken@gmail.com>

Taking the eviction lock is actually just one step which we need to do
in the critical section handling.

Rename the functions to reflect that, use the update parameters instead of the
vm to save the GFP flags.

v2: rebased and reordered

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        | 36 ++++++----------
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h        |  1 -
 .../gpu/drm/amd/amdgpu/amdgpu_vm_internal.h   | 41 ++++++++++++++-----
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c     | 28 ++++++-------
 4 files changed, 58 insertions(+), 48 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 0930b3fdade7..a994141e9df5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -1167,11 +1167,9 @@ int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	params.override_pte = allow_override && adev->gmc.override_pte;
 	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
 
-	amdgpu_vm_eviction_lock(vm);
-	if (vm->evicting) {
-		r = -EBUSY;
+	r = amdgpu_vm_begin_critical(&params);
+	if (r)
 		goto error_free;
-	}
 
 	if (!dma_fence_is_signaled(vm->last_unlocked)) {
 		struct dma_fence *tmp = dma_fence_get_stub();
@@ -1253,7 +1251,7 @@ int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 
 error_free:
 	kfree(tlb_cb);
-	amdgpu_vm_eviction_unlock(vm);
+	amdgpu_vm_end_critical(&params);
 	drm_dev_exit(idx);
 	return r;
 }
@@ -1301,11 +1299,9 @@ int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	params.unlocked = true;
 	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
 
-	amdgpu_vm_eviction_lock(vm);
-	if (vm->evicting) {
-		r = -EBUSY;
+	r = amdgpu_vm_begin_critical(&params);
+	if (r)
 		goto error_free;
-	}
 
 	r = vm->update_funcs->prepare(&params, sync,
 				      AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE);
@@ -1324,7 +1320,7 @@ int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 
 error_free:
 	kfree(tlb_cb);
-	amdgpu_vm_eviction_unlock(vm);
+	amdgpu_vm_end_critical(&params);
 	drm_dev_exit(idx);
 	return r;
 }
@@ -3156,32 +3152,26 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
 		goto error_unlock;
 	}
 
-	amdgpu_vm_eviction_lock(vm);
-	if (vm->evicting) {
-		r = -EBUSY;
-		goto error_dev_exit;
-	}
-
 	memset(&params, 0, sizeof(params));
 	params.adev = adev;
 	params.vm = vm;
 	params.immediate = true;
 	params.pages_addr = NULL;
 
+	r = amdgpu_vm_begin_critical(&params);
+	if (r)
+		goto error_end_critical;
+
 	r = vm->update_funcs->prepare(&params, NULL,
 				      AMDGPU_KERNEL_JOB_ID_VM_UPDATE_PDES);
 	if (r)
-		goto error_dev_exit;
+		goto error_end_critical;
 
 	amdgpu_vm_pt_leaves(&params, addr, addr, values, flags);
-
 	r = vm->update_funcs->commit(&params, &vm->last_update);
-	if (r)
-		goto error_dev_exit;
-
-	amdgpu_vm_eviction_unlock(vm);
 
-error_dev_exit:
+error_end_critical:
+	amdgpu_vm_end_critical(&params);
 	drm_dev_exit(idx);
 
 error_unlock:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index e871a9e3e1a9..fbbd9e725799 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -287,7 +287,6 @@ struct amdgpu_vm {
 	 */
 	struct mutex		eviction_lock;
 	bool			evicting;
-	unsigned int		saved_flags;
 
 	/* Memory statistics for this vm, protected by stats_lock */
 	spinlock_t		stats_lock;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
index 7f6442d0b647..a3328a48fecd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
@@ -93,6 +93,11 @@ struct amdgpu_vm_update_params {
 	 */
 	bool override_pte;
 
+	/**
+	 * @saved_flags: Saved flags for GFP reduction.
+	 */
+	unsigned int saved_flags;
+
 	/**
 	 * @tlb_flush_waitlist: temporary storage for BOs until tlb_flush
 	 */
@@ -130,21 +135,37 @@ int amdgpu_vm_pt_map_tables(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 int amdgpu_vm_pt_alloc_dummies(struct amdgpu_device *adev);
 void amdgpu_vm_pt_free_dummies(struct amdgpu_device *adev);
 
-/*
- * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS
- * happens while holding this lock anywhere to prevent deadlocks when
- * an MMU notifier runs in reclaim-FS context.
+/**
+ * amdgpu_vm_begin_critical - start the critical section of the update
+ * @p: The update parameters
+ *
+ * Serialize all updates, check parameters and make sure that memory allocations
+ * don't enter the reclaim path so that we don't deadlock with MMU notifiers.
+ *
+ * Returns:
+ *
+ * 0 on success or a negative error code on failure.
+ * Even on error amdgpu_vm_end_critical() must still be called to clean up!
  */
-static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm)
+static inline int amdgpu_vm_begin_critical(struct amdgpu_vm_update_params *p)
 {
-	mutex_lock(&vm->eviction_lock);
-	vm->saved_flags = memalloc_noreclaim_save();
+	mutex_lock(&p->vm->eviction_lock);
+	p->saved_flags = memalloc_noreclaim_save();
+	if (p->vm->evicting)
+		return -EBUSY;
+	return 0;
 }
 
-static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm)
+/**
+ * amdgpu_vm_end_critical - end the critical section of the update
+ * @p: The update parameters
+ *
+ * Restore the GFP flags and drop the lock.
+ */
+static inline void amdgpu_vm_end_critical(struct amdgpu_vm_update_params *p)
 {
-	memalloc_noreclaim_restore(vm->saved_flags);
-	mutex_unlock(&vm->eviction_lock);
+	memalloc_noreclaim_restore(p->saved_flags);
+	mutex_unlock(&p->vm->eviction_lock);
 }
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index 5bb1a3300aa4..5385f662d0ac 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -478,42 +478,42 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 /**
  * amdgpu_vm_pt_alloc - Allocate a specific page table
  *
- * @adev: amdgpu_device pointer
- * @vm: VM to allocate page tables for
+ * @p: see amdgpu_vm_update_params definition
  * @cursor: Which page table to allocate
  *
  * Make sure a specific page table or directory is allocated.
  *
  * Returns:
- * 1 if page table needed to be allocated, 0 if page table was already
- * allocated, negative errno if an error occurred.
+ *
+ * 0 on success or a negative error code on failure.
  */
-static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev,
-			      struct amdgpu_vm *vm,
+static int amdgpu_vm_pt_alloc(struct amdgpu_vm_update_params *p,
 			      struct amdgpu_vm_pt_cursor *cursor)
 {
 	struct amdgpu_vm_bo_base *entry = cursor->entry;
 	struct amdgpu_bo *pt_bo;
 	struct amdgpu_bo_vm *pt;
-	int r;
+	int r, r2;
 
 	if (entry->bo)
 		return 0;
 
-	amdgpu_vm_eviction_unlock(vm);
-	r = amdgpu_vm_pt_create(adev, vm, cursor->level, &pt,
-				vm->root.bo->xcp_id);
-	amdgpu_vm_eviction_lock(vm);
+	amdgpu_vm_end_critical(p);
+	r = amdgpu_vm_pt_create(p->adev, p->vm, cursor->level, &pt,
+				p->vm->root.bo->xcp_id);
+	r2 = amdgpu_vm_begin_critical(p);
 	if (r)
 		return r;
+	if (r2)
+		return r2;
 
 	/* Keep a reference to the root directory to avoid
 	 * freeing them up in the wrong order.
 	 */
 	pt_bo = &pt->bo;
 	pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo);
-	amdgpu_vm_bo_base_init(entry, vm, pt_bo);
-	r = amdgpu_vm_pt_clear(adev, vm, pt);
+	amdgpu_vm_bo_base_init(entry, p->vm, pt_bo);
+	r = amdgpu_vm_pt_clear(p->adev, p->vm, pt);
 	if (r)
 		goto error_free_pt;
 
@@ -802,7 +802,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 		/* make sure that the page tables covering the
 		 * address range are actually allocated
 		 */
-		r = amdgpu_vm_pt_alloc(params->adev, params->vm, &cursor);
+		r = amdgpu_vm_pt_alloc(params, &cursor);
 		if (r)
 			return r;
 
-- 
2.53.0

[PATCH 09/13] drm/amdgpu: free PT when VM critical section cannot resume

[Huang Yu]

From: Yu Huang <Yu.Huang@amd.com>

amdgpu_vm_pt_alloc() drops the VM critical section while
allocating a page table. If allocation succeeds but re-entering the
critical section finds that the VM is evicting, the new page-table BO
is returned without being released.

Temporarily leave the failed critical section, release the
unpublished BO and re-enter it so that the caller still owns the
critical section on return and can run the normal cleanup path.

Fixes: 39061e38483a ("drm/amdgpu: rework eviction lock handling into critical section v2")
Signed-off-by: Yu Huang <Yu.Huang@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index 5385f662d0ac..ec7bdec1a3a7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -504,13 +504,17 @@ static int amdgpu_vm_pt_alloc(struct amdgpu_vm_update_params *p,
 	r2 = amdgpu_vm_begin_critical(p);
 	if (r)
 		return r;
-	if (r2)
+	pt_bo = &pt->bo;
+	if (r2) {
+		amdgpu_vm_end_critical(p);
+		amdgpu_bo_unref(&pt_bo);
+		(void)amdgpu_vm_begin_critical(p);
 		return r2;
+	}
 
 	/* Keep a reference to the root directory to avoid
 	 * freeing them up in the wrong order.
 	 */
-	pt_bo = &pt->bo;
 	pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo);
 	amdgpu_vm_bo_base_init(entry, p->vm, pt_bo);
 	r = amdgpu_vm_pt_clear(p->adev, p->vm, pt);
-- 
2.53.0

[PATCH 10/13] drm/amdgpu: fix the HMM range handling for KFD SVM

[Huang Yu]

From: Christian König <ckoenig.leichtzumerken@gmail.com>

It's mandatory that we have this check inside the VM handling or
otherwise page table allocation and filling PTEs doesn't work correctly.

This allows to remove the buggy SVM range lock, but that's not part of
this patch set.

Only compile tested!

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        | 10 ++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h        |  4 ++-
 .../gpu/drm/amd/amdgpu/amdgpu_vm_internal.h   | 10 ++++++
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c          | 32 ++++++++-----------
 4 files changed, 34 insertions(+), 22 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index a994141e9df5..9ea1a07577c2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -1117,6 +1117,7 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
  * @vram_base: base for vram mappings
  * @res: ttm_resource to map
  * @pages_addr: DMA addresses to use for mapping
+ * @hmm_range: to check validity of DMA addresses
  * @fence: optional resulting fence
  *
  * Fill in the page table entries between @start and @last. Allocate and free
@@ -1130,7 +1131,9 @@ int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			struct amdgpu_sync *sync, uint64_t start,
 			uint64_t last, uint64_t flags, uint64_t offset,
 			uint64_t vram_base, struct ttm_resource *res,
-			dma_addr_t *pages_addr, struct dma_fence **fence)
+			dma_addr_t *pages_addr,
+			struct amdgpu_hmm_range *hmm_range,
+			struct dma_fence **fence)
 {
 	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
 	struct amdgpu_vm_update_params params;
@@ -1163,6 +1166,7 @@ int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	params.adev = adev;
 	params.vm = vm;
 	params.pages_addr = pages_addr;
+	params.hmm_range = hmm_range;
 	params.needs_flush = flush_tlb;
 	params.override_pte = allow_override && adev->gmc.override_pte;
 	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
@@ -1456,7 +1460,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
 		r = amdgpu_vm_map_range(adev, vm, flush_tlb, !uncached, &sync,
 					mapping->start, mapping->last,
 					update_flags, mapping->offset,
-					vram_base, mem, pages_addr,
+					vram_base, mem, pages_addr, NULL,
 					last_update);
 		if (r)
 			goto error_free;
@@ -1666,7 +1670,7 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
 
 		r = amdgpu_vm_map_range(adev, vm, true, false,
 					&sync, mapping->start, mapping->last,
-					0, 0, 0, NULL, NULL, &f);
+					0, 0, 0, NULL, NULL, NULL, &f);
 		amdgpu_vm_free_mapping(adev, vm, mapping, f);
 		if (r) {
 			dma_fence_put(f);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index fbbd9e725799..2def386c4f95 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -461,7 +461,9 @@ int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			struct amdgpu_sync *sync, uint64_t start,
 			uint64_t last, uint64_t flags, uint64_t offset,
 			uint64_t vram_base, struct ttm_resource *res,
-			dma_addr_t *pages_addr, struct dma_fence **fence);
+			dma_addr_t *pages_addr,
+			struct amdgpu_hmm_range *hmm_range,
+			struct dma_fence **fence);
 int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			   struct amdgpu_sync *sync, uint64_t start,
 			   uint64_t last, uint64_t flags,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
index a3328a48fecd..3d4a80fd7661 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
@@ -26,6 +26,7 @@
 
 #include <linux/types.h>
 #include <linux/list.h>
+#include "amdgpu_hmm.h"
 #include "amdgpu_vm.h"
 
 struct amdgpu_device;
@@ -72,6 +73,13 @@ struct amdgpu_vm_update_params {
 	 */
 	dma_addr_t *pages_addr;
 
+	/**
+	 * @hmm_range:
+	 *
+	 * Used to check the validity of pages_addr.
+	 */
+	struct amdgpu_hmm_range *hmm_range;
+
 	/**
 	 * @job: job to used for hw submission
 	 */
@@ -153,6 +161,8 @@ static inline int amdgpu_vm_begin_critical(struct amdgpu_vm_update_params *p)
 	p->saved_flags = memalloc_noreclaim_save();
 	if (p->vm->evicting)
 		return -EBUSY;
+	if (p->hmm_range && !amdgpu_hmm_range_valid(p->hmm_range))
+		return -EAGAIN;
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 93e490f00c83..3a224dac7b1c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -1461,7 +1461,8 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
 static int
 svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
 		     unsigned long offset, unsigned long npages, bool readonly,
-		     dma_addr_t *dma_addr, struct amdgpu_device *bo_adev,
+		     dma_addr_t *dma_addr, struct amdgpu_hmm_range *hmm_range,
+		     struct amdgpu_device *bo_adev,
 		     struct dma_fence **fence, bool flush_tlb)
 {
 	struct amdgpu_device *adev = pdd->dev->adev;
@@ -1521,7 +1522,7 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
 					gpu_start, gpu_end, pte_flags,
 					(last_start - prange->start) << PAGE_SHIFT,
 					bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
-					NULL, dma_addr, &vm->last_update);
+					NULL, dma_addr, hmm_range, &vm->last_update);
 
 		for (j = last_start - prange->start; j <= i; j++)
 			dma_addr[j] |= last_domain;
@@ -1548,7 +1549,9 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
 }
 
 static int
-svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,
+svm_range_map_to_gpus(struct svm_range *prange,
+		      struct amdgpu_hmm_range *hmm_range,
+		      unsigned long offset,
 		      unsigned long npages, bool readonly,
 		      unsigned long *bitmap, bool wait, bool flush_tlb)
 {
@@ -1584,7 +1587,7 @@ svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,
 		set_bit(gpuidx, prange->bitmap_mapped);
 
 		r = svm_range_map_to_gpu(pdd, prange, offset, npages, readonly,
-					 prange->dma_addr[gpuidx],
+					 prange->dma_addr[gpuidx], hmm_range,
 					 bo_adev, wait ? &fence : NULL,
 					 flush_tlb);
 		if (r)
@@ -1855,18 +1858,6 @@ static int svm_range_validate_and_map(struct mm_struct *mm,
 
 		svm_range_lock(prange);
 
-		/* Free backing memory of hmm_range if it was initialized
-		 * Override return value to TRY AGAIN only if prior returns
-		 * were successful
-		 */
-		if (range && !amdgpu_hmm_range_valid(range) && !r) {
-			pr_debug("hmm update the range, need validate again\n");
-			r = -EAGAIN;
-		}
-
-		/* Free the hmm range */
-		amdgpu_hmm_range_free(range);
-
 		if (!r && !list_empty(&prange->child_list)) {
 			pr_debug("range split by unmap in parallel, validate again\n");
 			r = -EAGAIN;
@@ -1878,11 +1869,16 @@ static int svm_range_validate_and_map(struct mm_struct *mm,
 			if (map_start_vma <= map_last_vma) {
 				offset = map_start_vma - prange->start;
 				npages = map_last_vma - map_start_vma + 1;
-				r = svm_range_map_to_gpus(prange, offset, npages, readonly,
-							  ctx->bitmap, wait, flush_tlb);
+				r = svm_range_map_to_gpus(prange, range, offset,
+							  npages, readonly,
+							  ctx->bitmap, wait,
+							  flush_tlb);
 			}
 		}
 
+		/* Free the hmm range */
+		amdgpu_hmm_range_free(range);
+
 		if (!r && next == end)
 			prange->mapping_done = true;
 		else
-- 
2.53.0

[PATCH 11/13] drm/amdgpu: update the PTB itself in amdgpu_vm_pt_leaves()

[Huang Yu]

From: Yu Huang <Yu.Huang@amd.com>

When the walk stops at a PTB that has its own BO allocated, the update
was still aimed at cursor.parent->bo using the parent's shift and mask.
That writes a PDE in the parent table instead of the PTEs in the leaf
table the cursor actually reached, so the intended leaves are never
touched.

Target the entry's BO with its own level whenever the cursor sits on an
allocated AMDGPU_VM_PTB, and keep the parent for every other level.

Also go through amdgpu_vm_pte_update_flags() rather than calling
update_funcs->update() directly, so the flags are adjusted for the level
being written instead of being passed through verbatim.

Signed-off-by: Yu Huang <Yu.Huang@amd.com>
Change-Id: I5e3f4a1803c6a1bac10677f48f273a4ad711d760
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index ec7bdec1a3a7..d97e790396bf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -951,9 +951,16 @@ void amdgpu_vm_pt_leaves(struct amdgpu_vm_update_params *params,
 		if (amdgpu_vm_pt_descendant(adev, &cursor))
 			continue;
 
-		pt = cursor.parent->bo;
-		shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1);
-		mask = amdgpu_vm_pt_entries_mask(adev, cursor.level - 1);
+		if (cursor.level == AMDGPU_VM_PTB && cursor.entry &&
+		    cursor.entry->bo) {
+			pt = cursor.entry->bo;
+			shift = amdgpu_vm_pt_level_shift(adev, cursor.level);
+			mask = amdgpu_vm_pt_entries_mask(adev, cursor.level);
+		} else {
+			pt = cursor.parent->bo;
+			shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1);
+			mask = amdgpu_vm_pt_entries_mask(adev, cursor.level - 1);
+		}
 
 		/* Looks good so far, calculate parameters for the update */
 		pe_start = ((cursor.pfn >> shift) & mask) * 8;
@@ -973,9 +980,11 @@ void amdgpu_vm_pt_leaves(struct amdgpu_vm_update_params *params,
 		trace_amdgpu_vm_update_ptes(params, cursor.pfn, entry_end,
 					    min(nptes, 32u), val, 0, flags);
 
-		params->vm->update_funcs->update(params, to_amdgpu_bo_vm(pt),
-						 pe_start, val, nptes, 0,
-						 flags);
+		amdgpu_vm_pte_update_flags(params, to_amdgpu_bo_vm(pt),
+					   pt == cursor.parent->bo ?
+						   cursor.level - 1 :
+						   cursor.level,
+					   pe_start, val, nptes, 0, flags);
 		amdgpu_vm_pt_next(adev, &cursor);
 	}
 }
-- 
2.53.0

[PATCH 12/13] drm/amdgpu: pass an exclusive end to amdgpu_vm_pt_leaves()

[Huang Yu]

From: Yu Huang <Yu.Huang@amd.com>

amdgpu_vm_pt_leaves() treats @end as exclusive - it loops on
"cursor.pfn < end" and clamps with "entry_end = min(entry_end, end)" -
but both callers passed an inclusive last address.

In amdgpu_vm_unmap_range() that silently left the last page of every
range untouched.

In amdgpu_vm_handle_fault() it is worse: the call passes the same value
as start and end, so "addr < addr" is false and the loop body never runs
at all. That is the path which installs AMDGPU_VM_NORETRY_FLAGS to turn
an unservicable retry fault into a no-retry fault, so nothing was
written and the faulting wave kept retrying the same address forever.

Pass last + 1 and addr + 1 so both calls describe the half-open range the
helper expects.

Signed-off-by: Yu Huang <Yu.Huang@amd.com>
Change-Id: I0b89042774e1e3352829e8bf3b9d9953954bc870
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 9ea1a07577c2..4e9dd0485db0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -1312,7 +1312,7 @@ int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	if (r)
 		goto error_free;
 
-	amdgpu_vm_pt_leaves(&params, start, last, NULL, flags);
+	amdgpu_vm_pt_leaves(&params, start, last + 1, NULL, flags);
 
 	r = vm->update_funcs->commit(&params, fence);
 	if (r)
@@ -3171,7 +3171,7 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
 	if (r)
 		goto error_end_critical;
 
-	amdgpu_vm_pt_leaves(&params, addr, addr, values, flags);
+	amdgpu_vm_pt_leaves(&params, addr, addr + 1, values, flags);
 	r = vm->update_funcs->commit(&params, &vm->last_update);
 
 error_end_critical:
-- 
2.53.0

[PATCH 13/13] drm/amdgpu: use directory flags for dummy PDEs

[Huang Yu]

From: Yu Huang <Yu.Huang@amd.com>

amdgpu_vm_pt_leaves() can redirect a fault to the preallocated dummy
page-table hierarchy when the walk stops above the PTE level. The
entry written in this case must point to the next-level dummy PD/PT.

amdgpu_vm_pte_update_flags() treats entries above PTB as leaf mappings
and applies AMDGPU_PDE_PTE_FLAG() before the ASIC-specific conversion.
This is not appropriate for a pointer to another page table and can
produce an incorrectly encoded dummy PDE.

Build the dummy PDE address and flags with amdgpu_gmc_get_pde_for_bo()
and pass them directly to the update callback. Keep the existing PTE
flag handling for other updates, including unmaps with a NULL
destination.

Authored-by: Yu Huang <Yu.Huang@amd.com>
Signed-off-by: Yu Huang <Yu.Huang@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 39 ++++++++++++++++-------
 1 file changed, 27 insertions(+), 12 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index 06492358d85a..0c3ecd3eb740 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -950,19 +950,19 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
  * Returns:
  * 0 for success, negative error code for failure.
  */
-void amdgpu_vm_pt_leaves(struct amdgpu_vm_update_params *params,
-			 uint64_t start, uint64_t end,
-			 int64_t *dst, uint64_t flags)
+void amdgpu_vm_pt_leaves(struct amdgpu_vm_update_params *params, uint64_t start,
+			 uint64_t end, int64_t *dst, uint64_t flags)
 {
 	struct amdgpu_device *adev = params->adev;
 	struct amdgpu_vm_pt_cursor cursor;
 
 	amdgpu_vm_pt_start(adev, params->vm, start, &cursor);
 	while (cursor.pfn < end) {
-		uint64_t entry_end, pe_start, val;
-		unsigned int shift, mask;
+		uint64_t entry_end, pe_start, val, update_flags = flags;
+		unsigned int level, shift, mask;
 		struct amdgpu_bo *pt;
 		unsigned int nptes;
+		bool dummy_pde;
 
 		/* Walk to the leave entries */
 		if (amdgpu_vm_pt_descendant(adev, &cursor))
@@ -993,15 +993,30 @@ void amdgpu_vm_pt_leaves(struct amdgpu_vm_update_params *params,
 		 */
 		nptes = max(nptes, 1u);
 
+		level = pt == cursor.parent->bo ? cursor.level - 1 :
+						  cursor.level;
+		dummy_pde = dst && level != AMDGPU_VM_PTB;
+
 		val = dst ? dst[cursor.level] : 0;
+		if (dummy_pde)
+			amdgpu_gmc_get_pde_for_bo(
+				adev->vm_manager.dummy_pd[level + 1], level,
+				&val, &update_flags);
 		trace_amdgpu_vm_update_ptes(params, cursor.pfn, entry_end,
-					    min(nptes, 32u), val, 0, flags);
-
-		amdgpu_vm_pte_update_flags(params, to_amdgpu_bo_vm(pt),
-					   pt == cursor.parent->bo ?
-						   cursor.level - 1 :
-						   cursor.level,
-					   pe_start, val, nptes, 0, flags);
+					    min(nptes, 32u), val, 0,
+					    update_flags);
+		if (dummy_pde)
+			params->vm->update_funcs->update(params,
+							 to_amdgpu_bo_vm(pt),
+							 pe_start, val, nptes,
+							 0, update_flags);
+		else
+			amdgpu_vm_pte_update_flags(params, to_amdgpu_bo_vm(pt),
+						   pt == cursor.parent->bo ?
+							   cursor.level - 1 :
+							   cursor.level,
+						   pe_start, val, nptes, 0,
+						   flags);
 		amdgpu_vm_pt_next(adev, &cursor);
 	}
 }
-- 
2.53.0

Re: [PATCH 01/13] drm/amdgpu: start to move VM internals into amdgpu_vm_internal.h

[Timur Kristóf]

On Thursday, September 17, 2026 1:01:46 PM Central European Summer Time Huang 
Yu wrote:
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> 
> Avoid exposing some defines outside of the VM code.
> 
> Co-developed by Claude Sonnet 4.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>

Hi,

I believe I've already commented on this patch two times already, but it seems 
both comments were lost, so I'm going to repeat:

Please use the "Assisted-by" tag instead, see this page for details and an 
example: https://docs.kernel.org/process/coding-assistants.html

with that fixed, the patch is:
Reviewed-by: Timur Kristóf <timur.kristof@gmail.com>


> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h     |   2 +
>  .../gpu/drm/amd/amdgpu/amdgpu_trace_points.c  |   1 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        |   1 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h        | 116 -------------
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c    |   1 +
>  .../gpu/drm/amd/amdgpu/amdgpu_vm_internal.h   | 155 ++++++++++++++++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c     |   1 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c   |   1 +
>  8 files changed, 162 insertions(+), 116 deletions(-)
>  create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h index
> 5324030a13f5..784adf975b04 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
> @@ -37,6 +37,8 @@
>  #define AMDGPU_JOB_GET_TIMELINE_NAME(job) \
>  	
> job->base.s_fence->finished.ops->get_timeline_name(&job->base.s_fence->fini
> shed)
> 
> +struct amdgpu_vm_update_params;
> +
>  TRACE_EVENT(amdgpu_device_rreg,
>  	    TP_PROTO(unsigned did, uint32_t reg, uint32_t value),
>  	    TP_ARGS(did, reg, value),
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c index
> b96d885f6e33..fe61ed5b487e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
> @@ -25,6 +25,7 @@
>  #include <drm/amdgpu_drm.h>
>  #include "amdgpu_cs.h"
>  #include "amdgpu.h"
> +#include "amdgpu_vm_internal.h"
> 
>  #define CREATE_TRACE_POINTS
>  #include "amdgpu_trace.h"
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 88249aa89ee3..517490946a9c
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -37,6 +37,7 @@
>  #include <drm/drm_exec.h>
>  #include "amdgpu.h"
>  #include "amdgpu_vm.h"
> +#include "amdgpu_vm_internal.h"
>  #include "amdgpu_trace.h"
>  #include "amdgpu_amdkfd.h"
>  #include "amdgpu_gmc.h"
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index d32183cd9e0f..ac3b8e8525c1
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> @@ -262,80 +262,6 @@ struct amdgpu_task_info {
>  	struct kref	refcount;
>  };
> 
> -/**
> - * struct amdgpu_vm_update_params
> - *
> - * Encapsulate some VM table update parameters to reduce
> - * the number of function parameters
> - *
> - */
> -struct amdgpu_vm_update_params {
> -
> -	/**
> -	 * @adev: amdgpu device we do this update for
> -	 */
> -	struct amdgpu_device *adev;
> -
> -	/**
> -	 * @vm: optional amdgpu_vm we do this update for
> -	 */
> -	struct amdgpu_vm *vm;
> -
> -	/**
> -	 * @immediate: if changes should be made immediately
> -	 */
> -	bool immediate;
> -
> -	/**
> -	 * @unlocked: true if the root BO is not locked
> -	 */
> -	bool unlocked;
> -
> -	/**
> -	 * @pages_addr:
> -	 *
> -	 * DMA addresses to use for mapping
> -	 */
> -	dma_addr_t *pages_addr;
> -
> -	/**
> -	 * @job: job to used for hw submission
> -	 */
> -	struct amdgpu_job *job;
> -
> -	/**
> -	 * @num_dw_left: number of dw left for the IB
> -	 */
> -	unsigned int num_dw_left;
> -
> -	/**
> -	 * @needs_flush: true whenever we need to invalidate the TLB
> -	 */
> -	bool needs_flush;
> -
> -	/**
> -	 * @override_pte: true for memory that is not uncached and gmc 
override
> function is -	 * implemented to allow MTYPE to be overridden for 
NUMA local
> memory. -	 */
> -	bool override_pte;
> -
> -	/**
> -	 * @tlb_flush_waitlist: temporary storage for BOs until tlb_flush
> -	 */
> -	struct list_head tlb_flush_waitlist;
> -};
> -
> -struct amdgpu_vm_update_funcs {
> -	int (*map_table)(struct amdgpu_bo_vm *bo);
> -	int (*prepare)(struct amdgpu_vm_update_params *p,
> -		       struct amdgpu_sync *sync, u64 k_job_id);
> -	int (*update)(struct amdgpu_vm_update_params *p,
> -		      struct amdgpu_bo_vm *bo, uint64_t pe, uint64_t 
addr,
> -		      unsigned count, uint32_t incr, uint64_t flags);
> -	int (*commit)(struct amdgpu_vm_update_params *p,
> -		      struct dma_fence **fence);
> -};
> -
>  struct amdgpu_vm_fault_info {
>  	/* fault address */
>  	uint64_t	addr;
> @@ -600,22 +526,6 @@ void amdgpu_vm_move_to_lru_tail(struct amdgpu_device
> *adev, void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
>  			  struct amdgpu_mem_stats 
stats[__AMDGPU_PL_NUM]);
> 
> -int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> -		       struct amdgpu_bo_vm *vmbo, bool immediate);
> -int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> -			int level, bool immediate, struct 
amdgpu_bo_vm **vmbo,
> -			int32_t xcp_id);
> -void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm
> *vm); -
> -int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
> -			 struct amdgpu_vm_bo_base *entry);
> -int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
> -			  uint64_t start, uint64_t end,
> -			  uint64_t dst, uint64_t flags);
> -void amdgpu_vm_pt_free_work(struct work_struct *work);
> -void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
> -			    struct amdgpu_vm_update_params *params);
> -
>  #if defined(CONFIG_DEBUG_FS)
>  void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m);
>  #endif
> @@ -651,32 +561,6 @@ static inline uint64_t amdgpu_vm_tlb_seq(struct
> amdgpu_vm *vm) return atomic64_read(&vm->tlb_seq);
>  }
> 
> -/*
> - * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS
> - * happens while holding this lock anywhere to prevent deadlocks when
> - * an MMU notifier runs in reclaim-FS context.
> - */
> -static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm)
> -{
> -	mutex_lock(&vm->eviction_lock);
> -	vm->saved_flags = memalloc_noreclaim_save();
> -}
> -
> -static inline bool amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm)
> -{
> -	if (mutex_trylock(&vm->eviction_lock)) {
> -		vm->saved_flags = memalloc_noreclaim_save();
> -		return true;
> -	}
> -	return false;
> -}
> -
> -static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm)
> -{
> -	memalloc_noreclaim_restore(vm->saved_flags);
> -	mutex_unlock(&vm->eviction_lock);
> -}
> -
>  void amdgpu_vm_update_fault_cache(struct amdgpu_device *adev,
>  				  unsigned int pasid,
>  				  uint64_t addr,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c index
> b31ff6f56f0d..d03b9bbe20c7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
> @@ -21,6 +21,7 @@
>   */
> 
>  #include "amdgpu_vm.h"
> +#include "amdgpu_vm_internal.h"
>  #include "amdgpu.h"
>  #include "amdgpu_reset.h"
>  #include "amdgpu_object.h"
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h new file mode 100644
> index 000000000000..447195cc4593
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
> @@ -0,0 +1,155 @@
> +/*
> + * Copyright 2016 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the
> "Software"), + * to deal in the Software without restriction, including
> without limitation + * the rights to use, copy, modify, merge, publish,
> distribute, sublicense, + * and/or sell copies of the Software, and to
> permit persons to whom the + * Software is furnished to do so, subject to
> the following conditions: + *
> + * The above copyright notice and this permission notice shall be included
> in + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
> OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 
> IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR
> ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF
> CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH
> THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE.
> + *
> + * Authors: Christian König
> + */
> +#ifndef __AMDGPU_VM_INTERNAL_H__
> +#define __AMDGPU_VM_INTERNAL_H__
> +
> +#include <linux/types.h>
> +#include <linux/list.h>
> +#include "amdgpu_vm.h"
> +
> +struct amdgpu_device;
> +struct amdgpu_vm;
> +struct amdgpu_job;
> +struct amdgpu_sync;
> +struct amdgpu_bo_vm;
> +struct amdgpu_vm_bo_base;
> +struct dma_fence;
> +
> +/**
> + * struct amdgpu_vm_update_params
> + *
> + * Encapsulate some VM table update parameters to reduce
> + * the number of function parameters
> + *
> + */
> +struct amdgpu_vm_update_params {
> +
> +	/**
> +	 * @adev: amdgpu device we do this update for
> +	 */
> +	struct amdgpu_device *adev;
> +
> +	/**
> +	 * @vm: optional amdgpu_vm we do this update for
> +	 */
> +	struct amdgpu_vm *vm;
> +
> +	/**
> +	 * @immediate: if changes should be made immediately
> +	 */
> +	bool immediate;
> +
> +	/**
> +	 * @unlocked: true if the root BO is not locked
> +	 */
> +	bool unlocked;
> +
> +	/**
> +	 * @pages_addr:
> +	 *
> +	 * DMA addresses to use for mapping
> +	 */
> +	dma_addr_t *pages_addr;
> +
> +	/**
> +	 * @job: job to used for hw submission
> +	 */
> +	struct amdgpu_job *job;
> +
> +	/**
> +	 * @num_dw_left: number of dw left for the IB
> +	 */
> +	unsigned int num_dw_left;
> +
> +	/**
> +	 * @needs_flush: true whenever we need to invalidate the TLB
> +	 */
> +	bool needs_flush;
> +
> +	/**
> +	 * @override_pte: true for memory that is not uncached and gmc 
override
> function is +	 * implemented to allow MTYPE to be overridden for 
NUMA local
> memory. +	 */
> +	bool override_pte;
> +
> +	/**
> +	 * @tlb_flush_waitlist: temporary storage for BOs until tlb_flush
> +	 */
> +	struct list_head tlb_flush_waitlist;
> +};
> +
> +struct amdgpu_vm_update_funcs {
> +	int (*map_table)(struct amdgpu_bo_vm *bo);
> +	int (*prepare)(struct amdgpu_vm_update_params *p,
> +		       struct amdgpu_sync *sync, u64 k_job_id);
> +	int (*update)(struct amdgpu_vm_update_params *p,
> +		      struct amdgpu_bo_vm *bo, uint64_t pe, uint64_t 
addr,
> +		      unsigned count, uint32_t incr, uint64_t flags);
> +	int (*commit)(struct amdgpu_vm_update_params *p,
> +		      struct dma_fence **fence);
> +};
> +
> +int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> +		       struct amdgpu_bo_vm *vmbo, bool immediate);
> +int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> +			int level, bool immediate, struct 
amdgpu_bo_vm **vmbo,
> +			int32_t xcp_id);
> +void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm
> *vm); +int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
> +			 struct amdgpu_vm_bo_base *entry);
> +int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
> +			  uint64_t start, uint64_t end,
> +			  uint64_t dst, uint64_t flags);
> +void amdgpu_vm_pt_free_work(struct work_struct *work);
> +void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
> +			    struct amdgpu_vm_update_params *params);
> +int amdgpu_vm_pt_map_tables(struct amdgpu_device *adev, struct amdgpu_vm
> *vm); +
> +/*
> + * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS
> + * happens while holding this lock anywhere to prevent deadlocks when
> + * an MMU notifier runs in reclaim-FS context.
> + */
> +static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm)
> +{
> +	mutex_lock(&vm->eviction_lock);
> +	vm->saved_flags = memalloc_noreclaim_save();
> +}
> +
> +static inline bool amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm)
> +{
> +	if (mutex_trylock(&vm->eviction_lock)) {
> +		vm->saved_flags = memalloc_noreclaim_save();
> +		return true;
> +	}
> +	return false;
> +}
> +
> +static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm)
> +{
> +	memalloc_noreclaim_restore(vm->saved_flags);
> +	mutex_unlock(&vm->eviction_lock);
> +}
> +
> +#endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c index
> e43a60d09808..0d7fe32d0c7d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> @@ -26,6 +26,7 @@
>  #include "amdgpu.h"
>  #include "amdgpu_trace.h"
>  #include "amdgpu_vm.h"
> +#include "amdgpu_vm_internal.h"
>  #include "amdgpu_job.h"
> 
>  /*
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c index
> 430274237689..a6ae2f62f80d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
> @@ -21,6 +21,7 @@
>   */
> 
>  #include "amdgpu_vm.h"
> +#include "amdgpu_vm_internal.h"
>  #include "amdgpu_job.h"
>  #include "amdgpu_object.h"
>  #include "amdgpu_trace.h"

Re: [PATCH 02/13] drm/amdgpu: remove unecessary parameters from trace_amdgpu_vm_update_ptes

[Timur Kristóf]

On Thursday, September 17, 2026 1:01:47 PM Central European Summer Time Huang 
Yu wrote:
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> 
> Those can be determined by the parameter structure as well.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>

Nice cleanup!
Reviewed-by: Timur Kristóf <timur.kristof@gmail.com>

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h | 10 +++++-----
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c |  5 +----
>  2 files changed, 6 insertions(+), 9 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h index
> 784adf975b04..599c1d0e512b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
> @@ -331,9 +331,8 @@ TRACE_EVENT(amdgpu_vm_update_ptes,
>  	    TP_PROTO(struct amdgpu_vm_update_params *p,
>  		     uint64_t start, uint64_t end,
>  		     unsigned int nptes, uint64_t dst,
> -		     uint64_t incr, uint64_t flags,
> -		     pid_t pid, uint64_t vm_ctx),
> -	TP_ARGS(p, start, end, nptes, dst, incr, flags, pid, vm_ctx),
> +		     uint64_t incr, uint64_t flags),
> +	TP_ARGS(p, start, end, nptes, dst, incr, flags),
>  	TP_STRUCT__entry(
>  			 __field(u64, start)
>  			 __field(u64, end)
> @@ -353,8 +352,9 @@ TRACE_EVENT(amdgpu_vm_update_ptes,
>  			__entry->flags = flags;
>  			__entry->incr = incr;
>  			__entry->nptes = nptes;
> -			__entry->pid = pid;
> -			__entry->vm_ctx = vm_ctx;
> +			__entry->pid = p->vm->task_info ?
> +				p->vm->task_info->tgid : 0;
> +			__entry->vm_ctx = p->vm-
>immediate.fence_context;
>  			for (i = 0; i < nptes; ++i) {
>  				u64 addr = p->pages_addr ? 
amdgpu_vm_map_gart(
>  					p->pages_addr, dst) : 
dst;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c index
> 0d7fe32d0c7d..b85b70a1a428 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> @@ -882,7 +882,6 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params
> *params, entry_end = min(entry_end, end);
> 
>  		do {
> -			struct amdgpu_vm *vm = params->vm;
>  			uint64_t upd_end = min(entry_end, frag_end);
>  			unsigned int nptes = (upd_end - frag_start) 
>> shift;
>  			uint64_t upd_flags = flags | 
AMDGPU_PTE_FRAG(frag);
> @@ -894,9 +893,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params
> *params,
> 
>  			trace_amdgpu_vm_update_ptes(params, 
frag_start, upd_end,
>  						    
min(nptes, 32u), dst, incr,
> -						    
upd_flags,
> -						    vm-
>task_info ? vm->task_info->tgid : 0,
> -						    vm-
>immediate.fence_context);
> +						    
upd_flags);
>  			amdgpu_vm_pte_update_flags(params, 
to_amdgpu_bo_vm(pt),
>  						   
cursor.level, pe_start, dst,
>  						   nptes, 
incr, upd_flags);

Re: [PATCH 03/13] drm/amdgpu: allocate and fill dummy PDs/PTs

[Timur Kristóf]

On Thursday, September 17, 2026 1:01:48 PM Central European Summer Time Huang 
Yu wrote:
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> 
> Allocate some PDs/PTs which just point to the dummy page.
> 
> Those can be used in page faults to redirect recoverable page faults
> to the dummy page.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>

Makes good sense.

Reviewed-by: Timur Kristóf <timur.kristof@gmail.com>

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        | 11 +++-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h        |  5 +-
>  .../gpu/drm/amd/amdgpu/amdgpu_vm_internal.h   |  2 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c     | 57 +++++++++++++++++++
>  4 files changed, 73 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 517490946a9c..5675925ca656
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -2843,8 +2843,10 @@ void amdgpu_vm_fini(struct amdgpu_device *adev,
> struct amdgpu_vm *vm) *
>   * Initialize the VM manager structures
>   */
> -void amdgpu_vm_manager_init(struct amdgpu_device *adev)
> +int amdgpu_vm_manager_init(struct amdgpu_device *adev)
>  {
> +	int r;
> +
>  	/* Concurrent flushes are only possible starting with Vega10 and
>  	 * are broken on Navi10 and Navi14.
>  	 */
> @@ -2856,6 +2858,10 @@ void amdgpu_vm_manager_init(struct amdgpu_device
> *adev) spin_lock_init(&adev->vm_manager.prt_lock);
>  	atomic_set(&adev->vm_manager.num_prt_users, 0);
> 
> +	r = amdgpu_vm_pt_alloc_dummies(adev);
> +	if (r)
> +		return r;
> +
>  	/* If not overridden by the user, by default, only in large BAR 
systems
>  	 * Compute VM tables will be updated by CPU
>  	 */
> @@ -2875,6 +2881,8 @@ void amdgpu_vm_manager_init(struct amdgpu_device
> *adev) #else
>  	adev->vm_manager.vm_update_mode = 0;
>  #endif
> +
> +	return 0;
>  }
> 
>  /**
> @@ -2886,6 +2894,7 @@ void amdgpu_vm_manager_init(struct amdgpu_device
> *adev) */
>  void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
>  {
> +	amdgpu_vm_pt_free_dummies(adev);
>  	amdgpu_vmid_mgr_fini(adev);
>  	amdgpu_pasid_mgr_cleanup();
>  }
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index ac3b8e8525c1..826406522b24
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> @@ -409,6 +409,9 @@ struct amdgpu_vm_manager {
> 
>  	/* Global registration of recent page fault information */
>  	struct amdgpu_vm_fault_info	fault_info;
> +
> +	struct amdgpu_bo		*dummy_pd[AMDGPU_VM_PTB + 1];
> +	uint64_t			dummy_dst[AMDGPU_VM_PTB + 1];
>  };
> 
>  struct amdgpu_bo_va_mapping;
> @@ -420,7 +423,7 @@ struct amdgpu_bo_va_mapping;
>  extern const struct amdgpu_vm_update_funcs amdgpu_vm_cpu_funcs;
>  extern const struct amdgpu_vm_update_funcs amdgpu_vm_sdma_funcs;
> 
> -void amdgpu_vm_manager_init(struct amdgpu_device *adev);
> +int amdgpu_vm_manager_init(struct amdgpu_device *adev);
>  void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
> 
>  long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h index
> 447195cc4593..16cc0d5f5894 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
> @@ -125,6 +125,8 @@ void amdgpu_vm_pt_free_work(struct work_struct *work);
>  void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
>  			    struct amdgpu_vm_update_params *params);
>  int amdgpu_vm_pt_map_tables(struct amdgpu_device *adev, struct amdgpu_vm
> *vm); +int amdgpu_vm_pt_alloc_dummies(struct amdgpu_device *adev);
> +void amdgpu_vm_pt_free_dummies(struct amdgpu_device *adev);
> 
>  /*
>   * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c index
> b85b70a1a428..3a6449d60988 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> @@ -963,3 +963,60 @@ int amdgpu_vm_pt_map_tables(struct amdgpu_device *adev,
> struct amdgpu_vm *vm)
> 
>  	return 0;
>  }
> +
> +/* amdgpu_vm_pt_alloc_dummies - allocate dummy PDs/PTs
> + *
> + * @adev: the amdgpu device pointer
> + *
> + * Allocate some dummy PDs/PTs which can be used to redirect page faults to
> the + * dummy page.
> + */
> +int amdgpu_vm_pt_alloc_dummies(struct amdgpu_device *adev)
> +{
> +	struct amdgpu_vm_manager *vm_mgr = &adev->vm_manager;
> +	int r;
> +
> +	for (int level = AMDGPU_VM_PTB; level != vm_mgr->root_level; 
level--) {
> +		size_t size =  amdgpu_vm_pt_size(adev, level);
> +		uint64_t addr, flags;
> +		void *ptr;
> +
> +		r = amdgpu_bo_create_kernel(adev, size, 0,
> +					    
AMDGPU_GEM_DOMAIN_VRAM,
> +					    &vm_mgr-
>dummy_pd[level],
> +					    &vm_mgr-
>dummy_dst[level],
> +					    &ptr);
> +		if (r)
> +			return r;
> +
> +		if (level == AMDGPU_VM_PTB) {
> +			addr = adev->dummy_page_addr;
> +			/*
> +			 * TODO: We want to have separate dummies for 
reads and
> +			 * writes.
> +			 */
> +			flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED 
|
> +				AMDGPU_PTE_SYSTEM | 
AMDGPU_PTE_EXECUTABLE |
> +				AMDGPU_PTE_READABLE | 
AMDGPU_PTE_WRITEABLE;
> +		} else {
> +			amdgpu_gmc_get_pde_for_bo(vm_mgr-
>dummy_pd[level + 1],
> +						  level, 
&addr, &flags);
> +		}
> +
> +		for (int i = 0; i < amdgpu_vm_pt_num_entries(adev, 
level); i++)
> +			amdgpu_gmc_set_pte_pde(adev, ptr, i, addr, 
flags);
> +	}
> +
> +	return 0;
> +}
> +
> +void amdgpu_vm_pt_free_dummies(struct amdgpu_device *adev)
> +{
> +	struct amdgpu_vm_manager *vm_mgr = &adev->vm_manager;
> +	void *ptr;
> +
> +	for (int level = AMDGPU_VM_PTB; level != vm_mgr->root_level; 
level--)
> +		amdgpu_bo_free_kernel(&vm_mgr->dummy_pd[level],
> +				      &vm_mgr->dummy_dst[level],
> +				      &ptr);
> +}

Re: [PATCH 04/13] drm/amdgpu: add amdgpu_vm_pt_leaves()

[Timur Kristóf]

On Thursday, September 17, 2026 1:01:49 PM Central European Summer Time Huang 
Yu wrote:
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> 
> Add a new function amdgpu_vm_update_leaves() to avoid memory allocation
> on page faults.
> 
> The idea is to only update the leave PDEs/PTEs to let them point to the
> dummy page.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        | 55 +++++++++++-----
>  .../gpu/drm/amd/amdgpu/amdgpu_vm_internal.h   |  3 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c     | 62 +++++++++++++++++++
>  3 files changed, 105 insertions(+), 15 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 5675925ca656..e60143d4990c
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -3013,10 +3013,12 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device
> *adev, u32 pasid, u32 vmid, u32 node_id, uint64_t addr,
>  			    uint64_t ts, bool write_fault)
>  {
> +	struct amdgpu_vm_update_params params;
>  	bool is_compute_context = false;
> -	struct drm_exec exec;
> -	uint64_t value, flags;
> +	uint64_t *values, flags;
>  	struct amdgpu_vm *vm;
> +	struct drm_exec exec;
> +	unsigned int idx;
>  	int r;
> 
>  	drm_exec_init(&exec, 0, 1);
> @@ -3060,24 +3062,21 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device
> *adev, u32 pasid, }
> 
>  	addr /= AMDGPU_GPU_PAGE_SIZE;
> -	flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED |
> -		AMDGPU_PTE_SYSTEM;
> -
>  	if (is_compute_context) {
>  		/* Intentionally setting invalid PTE flag
>  		 * combination to force a no-retry-fault
>  		 */
>  		flags = AMDGPU_VM_NORETRY_FLAGS;
> -		value = 0;
> +		values = NULL;
>  	} else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) {
>  		/* Redirect the access to the dummy page */
> -		value = adev->dummy_page_addr;
> -		flags |= AMDGPU_PTE_EXECUTABLE | AMDGPU_PTE_READABLE |
> -			AMDGPU_PTE_WRITEABLE;
> -
> +		flags = 0;

Why do you set the flags to zero here?

> +		values = adev->vm_manager.dummy_dst;
>  	} else {
>  		/* Let the hw retry silently on the PTE */
> -		value = 0;
> +		flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED |
> +			AMDGPU_PTE_SYSTEM;
> +		values = NULL;
>  	}
> 
>  	r = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1);
> @@ -3086,12 +3085,38 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device
> *adev, u32 pasid, goto error_unlock;
>  	}
> 
> -	r = amdgpu_vm_update_range(adev, vm, true, false, false, false,
> -				   NULL, addr, addr, flags, value, 
0, NULL, NULL, NULL);
> -	if (r)
> +	if (!drm_dev_enter(adev_to_drm(adev), &idx)) {
> +		r = -ENODEV;
>  		goto error_unlock;
> +	}
> 
> -	r = amdgpu_vm_update_pdes(adev, vm, true);
> +	amdgpu_vm_eviction_lock(vm);
> +	if (vm->evicting) {
> +		r = -EBUSY;
> +		goto error_dev_exit;
> +	}
> +
> +	memset(&params, 0, sizeof(params));
> +	params.adev = adev;
> +	params.vm = vm;
> +	params.immediate = true;
> +	params.pages_addr = NULL;
> +
> +	r = vm->update_funcs->prepare(&params, NULL,
> +				      
AMDGPU_KERNEL_JOB_ID_VM_UPDATE_PDES);
> +	if (r)
> +		goto error_dev_exit;
> +
> +	amdgpu_vm_pt_leaves(&params, addr, addr, values, flags);
> +
> +	r = vm->update_funcs->commit(&params, &vm->last_update);


I think this shouldn't pass the last VM update.
Note that amdgpu_vm_sdma_commit() doesn't return a fence anyway when 
params.immediate is true, so I think it's OK to just pass NULL here.



> +	if (r)
> +		goto error_dev_exit;
> +
> +	amdgpu_vm_eviction_unlock(vm);
> +
> +error_dev_exit:
> +	drm_dev_exit(idx);
> 
>  error_unlock:
>  	drm_exec_fini(&exec);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h index
> 16cc0d5f5894..7d3c0904e1a1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
> @@ -121,6 +121,9 @@ int amdgpu_vm_pde_update(struct amdgpu_vm_update_params
> *params, int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
> uint64_t start, uint64_t end,
>  			  uint64_t dst, uint64_t flags);
> +void amdgpu_vm_pt_leaves(struct amdgpu_vm_update_params *params,
> +			 uint64_t start, uint64_t end,
> +			 int64_t *dst, uint64_t flags);
>  void amdgpu_vm_pt_free_work(struct work_struct *work);
>  void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
>  			    struct amdgpu_vm_update_params *params);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c index
> 3a6449d60988..2f542c731447 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> @@ -936,6 +936,68 @@ int amdgpu_vm_ptes_update(struct
> amdgpu_vm_update_params *params, return 0;
>  }
> 
> +/**
> + * amdgpu_vm_pt_leaves - update leave PDEs/PTEs

Typo: should be "leaf" instead of "leave".

> + *
> + * @params: see amdgpu_vm_update_params definition
> + * @start: start of GPU address range
> + * @end: end of GPU address range
> + * @dst: optional array with one dst addr per layer
> + * @flags: mapping flags
> + *
> + * Update the leave PDEs/PTEs in the range @start - @end without allocating
> or + * freeing page tables.
> + *
> + * Returns:
> + * 0 for success, negative error code for failure.
> + */
> +void amdgpu_vm_pt_leaves(struct amdgpu_vm_update_params *params,
> +			 uint64_t start, uint64_t end,
> +			 int64_t *dst, uint64_t flags)
> +{
> +	struct amdgpu_device *adev = params->adev;
> +	struct amdgpu_vm_pt_cursor cursor;
> +
> +	amdgpu_vm_pt_start(adev, params->vm, start, &cursor);
> +	while (cursor.pfn < end) {
> +		uint64_t entry_end, pe_start, val;
> +		unsigned int shift, mask;
> +		struct amdgpu_bo *pt;
> +		unsigned int nptes;
> +
> +		/* Walk to the leave entries */
> +		if (amdgpu_vm_pt_descendant(adev, &cursor))
> +			continue;
> +
> +		pt = cursor.parent->bo;
> +		shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 
1);
> +		mask = amdgpu_vm_pt_entries_mask(adev, cursor.level - 
1);
> +
> +		/* Looks good so far, calculate parameters for the 
update */
> +		pe_start = ((cursor.pfn >> shift) & mask) * 8;
> +
> +		entry_end = ((uint64_t)mask + 1) << shift;
> +		entry_end += cursor.pfn & ~(entry_end - 1);
> +		entry_end = min(entry_end, end);
> +
> +		nptes = (entry_end - cursor.pfn) >> shift;
> +		/*
> +		 * This can happen when we set higher level PDEs to 
unmap and/or
> +		 * silent to stop fault floods.
> +		 */
> +		nptes = max(nptes, 1u);
> +
> +		val = dst ? dst[cursor.level] : 0;
> +		trace_amdgpu_vm_update_ptes(params, cursor.pfn, 
entry_end,
> +					    min(nptes, 32u), 
val, 0, flags);
> +
> +		params->vm->update_funcs->update(params, 
to_amdgpu_bo_vm(pt),
> +						 pe_start, 
val, nptes, 0,
> +						 flags);
> +		amdgpu_vm_pt_next(adev, &cursor);
> +	}
> +}
> +
>  /**
>   * amdgpu_vm_pt_map_tables - have bo of root PD cpu accessible
>   * @adev: amdgpu device structure