(no subject)

[Christian König]

Hi everyone,

this patch set aims to fix a couple of problems we still have with the VM, page fault and HMM/SVM handling.

First I rework a bit what we have on headers/defines for the suballocator, especially that the right GFP flags are now used for GPU reset and graphics page faults is important to avoid hangs.

Then I rework the VM handling, especially splitting up the amdgpu_vm_update_range() into an amdgpu_vm_map_range() and unmap_range(). The later is only meant to be used for MMU notifier/HMM/SVM use, but I couldn't come up with a better name.

And finally it fixes the problem with KFD SVM that we can't properly allocate page tables under the SVM lock. The missing piece here is to remove the SVM lock and rely on the eviction lock of the VM for that.

Timur & Natalie I've put you on the patch set because I know you have page fault patches in the pipeline, it would be nice if you could apply this set to amd-staging-drm-next and rebase your stuff on top of it.

Honglei and Philip this is basically the pre-requisite work I have been talking about. Please integrate that into the new SVM patches and re-test.

I've only did some very brief testing, so something more extensional testing is clearly a good idea.

Thanks,
Christian.

[PATCH 01/13] drm/amdgpu: move suballoc defines into own header

[Christian König]

Just some code cleanup, while at it remove outdated comment.

No functional change.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h        | 32 +--------
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.h | 40 -----------
 drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h     | 77 ++++++++++++++++++++++
 3 files changed, 78 insertions(+), 71 deletions(-)
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 5d7bfa59424a..4b6c9d9e6773 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -112,6 +112,7 @@
 #include "amdgpu_userq.h"
 #include "amdgpu_eviction_fence.h"
 #include "amdgpu_ip.h"
+#include "amdgpu_sa.h"
 #if defined(CONFIG_DRM_AMD_ISP)
 #include "amdgpu_isp.h"
 #endif
@@ -386,37 +387,6 @@ struct amdgpu_clock {
 	uint32_t max_pixel_clock;
 };
 
-/* sub-allocation manager, it has to be protected by another lock.
- * By conception this is an helper for other part of the driver
- * like the indirect buffer or semaphore, which both have their
- * locking.
- *
- * Principe is simple, we keep a list of sub allocation in offset
- * order (first entry has offset == 0, last entry has the highest
- * offset).
- *
- * When allocating new object we first check if there is room at
- * the end total_size - (last_object_offset + last_object_size) >=
- * alloc_size. If so we allocate new object there.
- *
- * When there is not enough room at the end, we start waiting for
- * each sub object until we reach object_offset+object_size >=
- * alloc_size, this object then become the sub object we return.
- *
- * Alignment can't be bigger than page size.
- *
- * Hole are not considered for allocation to keep things simple.
- * Assumption is that there won't be hole (all object on same
- * alignment).
- */
-
-struct amdgpu_sa_manager {
-	struct drm_suballoc_manager	base;
-	struct amdgpu_bo		*bo;
-	uint64_t			gpu_addr;
-	void				*cpu_ptr;
-};
-
 /*
  * IRQS.
  */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 4d68732d6223..ff11a0903499 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -312,46 +312,6 @@ uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo);
 uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,
 					    uint32_t domain);
 
-/*
- * sub allocation
- */
-static inline struct amdgpu_sa_manager *
-to_amdgpu_sa_manager(struct drm_suballoc_manager *manager)
-{
-	return container_of(manager, struct amdgpu_sa_manager, base);
-}
-
-static inline uint64_t amdgpu_sa_bo_gpu_addr(struct drm_suballoc *sa_bo)
-{
-	return to_amdgpu_sa_manager(sa_bo->manager)->gpu_addr +
-		drm_suballoc_soffset(sa_bo);
-}
-
-static inline void *amdgpu_sa_bo_cpu_addr(struct drm_suballoc *sa_bo)
-{
-	return to_amdgpu_sa_manager(sa_bo->manager)->cpu_ptr +
-		drm_suballoc_soffset(sa_bo);
-}
-
-int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
-				     struct amdgpu_sa_manager *sa_manager,
-				     unsigned size, u32 align, u32 domain);
-void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
-				      struct amdgpu_sa_manager *sa_manager);
-int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev,
-				      struct amdgpu_sa_manager *sa_manager);
-int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
-		     struct drm_suballoc **sa_bo,
-		     unsigned int size);
-void amdgpu_sa_bo_free(struct drm_suballoc **sa_bo,
-		       struct dma_fence *fence);
-#if defined(CONFIG_DEBUG_FS)
-void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
-					 struct seq_file *m);
-u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m);
-#endif
-void amdgpu_debugfs_sa_init(struct amdgpu_device *adev);
-
 bool amdgpu_bo_support_uswc(u64 bo_flags);
 
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
new file mode 100644
index 000000000000..8c85c80fc119
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef AMDGPU_SA_H_
+#define AMDGPU_SA_H_
+
+#include <drm/drm_suballoc.h>
+
+struct amdgpu_device;
+struct amdgpu_bo;
+
+struct amdgpu_sa_manager {
+	struct drm_suballoc_manager	base;
+	struct amdgpu_bo		*bo;
+	uint64_t			gpu_addr;
+	void				*cpu_ptr;
+};
+
+static inline struct amdgpu_sa_manager *
+to_amdgpu_sa_manager(struct drm_suballoc_manager *manager)
+{
+	return container_of(manager, struct amdgpu_sa_manager, base);
+}
+
+static inline uint64_t amdgpu_sa_bo_gpu_addr(struct drm_suballoc *sa_bo)
+{
+	return to_amdgpu_sa_manager(sa_bo->manager)->gpu_addr +
+		drm_suballoc_soffset(sa_bo);
+}
+
+static inline void *amdgpu_sa_bo_cpu_addr(struct drm_suballoc *sa_bo)
+{
+	return to_amdgpu_sa_manager(sa_bo->manager)->cpu_ptr +
+		drm_suballoc_soffset(sa_bo);
+}
+
+int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
+			      struct amdgpu_sa_manager *sa_manager,
+			      unsigned size, u32 align, u32 domain);
+void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
+			       struct amdgpu_sa_manager *sa_manager);
+int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev,
+			       struct amdgpu_sa_manager *sa_manager);
+int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
+		     struct drm_suballoc **sa_bo,
+		     unsigned int size);
+void amdgpu_sa_bo_free(struct drm_suballoc **sa_bo,
+		       struct dma_fence *fence);
+#if defined(CONFIG_DEBUG_FS)
+void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
+				  struct seq_file *m);
+u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m);
+#endif
+void amdgpu_debugfs_sa_init(struct amdgpu_device *adev);
+
+#endif
-- 
2.43.0

[PATCH 02/13] drm/amdgpu: give different sizes for each SA pool type

[Christian König]

The IMMEDIATE (page fault) and DIRECT (reset) pool should be used only
very rarely and by a single thread.

Saves roughly 1.25MiB of memory and GART space for each amdgpu device.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c   | 8 +++++++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 2 --
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index f1ed4a436f5b..334f95f8f339 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -351,14 +351,20 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
  */
 int amdgpu_ib_pool_init(struct amdgpu_device *adev)
 {
+	const int sizes[AMDGPU_IB_POOL_MAX] = {
+		[AMDGPU_IB_POOL_DELAYED] = SZ_1M,
+		[AMDGPU_IB_POOL_IMMEDIATE] = SZ_128K,
+		[AMDGPU_IB_POOL_DIRECT] = SZ_512K
+	};
 	int r, i;
 
 	if (adev->ib_pool_ready)
 		return 0;
 
+
 	for (i = 0; i < AMDGPU_IB_POOL_MAX; i++) {
 		r = amdgpu_sa_bo_manager_init(adev, &adev->ib_pools[i],
-					      AMDGPU_IB_POOL_SIZE, 256,
+					      sizes[i], 256,
 					      AMDGPU_GEM_DOMAIN_GTT);
 		if (r)
 			goto error;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 8f28b3bd7010..1a063a0a4280 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -68,8 +68,6 @@ enum amdgpu_ring_priority_level {
 
 #define to_amdgpu_ring(s) container_of((s), struct amdgpu_ring, sched)
 
-#define AMDGPU_IB_POOL_SIZE	(1024 * 1024)
-
 enum amdgpu_ring_type {
 	AMDGPU_RING_TYPE_GFX		= AMDGPU_HW_IP_GFX,
 	AMDGPU_RING_TYPE_COMPUTE	= AMDGPU_HW_IP_COMPUTE,
-- 
2.43.0

[PATCH 03/13] drm/amdgpu: add gfp_flags to amdgpu_sa_manager

[Christian König]

Make sure that we use the emmergency reserves for unrecoverable page
faults and GPU resets.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 23 ++++++++++++++++++++---
 drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c | 18 ++++++++++++------
 drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h |  3 ++-
 3 files changed, 34 insertions(+), 10 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index 334f95f8f339..60e4c3985029 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -356,16 +356,33 @@ int amdgpu_ib_pool_init(struct amdgpu_device *adev)
 		[AMDGPU_IB_POOL_IMMEDIATE] = SZ_128K,
 		[AMDGPU_IB_POOL_DIRECT] = SZ_512K
 	};
+	const gfp_t gfp_flags[AMDGPU_IB_POOL_MAX] = {
+		/*
+		 * For normal page table updates and recoverable page faults,
+		 * further restricted by the VM eviction lock to not wait for
+		 * memory reclaim.
+		 */
+		[AMDGPU_IB_POOL_DELAYED] = GFP_KERNEL,
+		/*
+		 * For redirecting unrecoverable page faults to the dummy page
+		 * or set the PRT bits. dma_fence submissions might depend on
+		 * that so we need the emmergency resewrves.
+		 */
+		[AMDGPU_IB_POOL_IMMEDIATE] = GFP_ATOMIC,
+		/*
+		 * For IB tests during GPU resets. Only very small and temporary
+		 * allocation to make allow dma_fences to signal.
+		 */
+		[AMDGPU_IB_POOL_DIRECT] = GFP_ATOMIC
+	};
 	int r, i;
 
 	if (adev->ib_pool_ready)
 		return 0;
 
-
 	for (i = 0; i < AMDGPU_IB_POOL_MAX; i++) {
 		r = amdgpu_sa_bo_manager_init(adev, &adev->ib_pools[i],
-					      sizes[i], 256,
-					      AMDGPU_GEM_DOMAIN_GTT);
+					      sizes[i], gfp_flags[i]);
 		if (r)
 			goto error;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
index 39070b2a4c04..74124f80601e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
@@ -46,11 +46,13 @@
 
 int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
 			      struct amdgpu_sa_manager *sa_manager,
-			      unsigned int size, u32 suballoc_align, u32 domain)
+			      unsigned int size, gfp_t gfp_flags)
 {
 	int r;
 
-	r = amdgpu_bo_create_kernel(adev, size, AMDGPU_GPU_PAGE_SIZE, domain,
+	sa_manager->gfp_flags = gfp_flags;
+	r = amdgpu_bo_create_kernel(adev, size, AMDGPU_GPU_PAGE_SIZE,
+				    AMDGPU_GEM_DOMAIN_GTT,
 				    &sa_manager->bo, &sa_manager->gpu_addr,
 				    &sa_manager->cpu_ptr);
 	if (r) {
@@ -59,7 +61,8 @@ int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
 	}
 
 	memset(sa_manager->cpu_ptr, 0, size);
-	drm_suballoc_manager_init(&sa_manager->base, size, suballoc_align);
+	drm_suballoc_manager_init(&sa_manager->base, size, 256);
+
 	return r;
 }
 
@@ -73,7 +76,8 @@ void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
 
 	drm_suballoc_manager_fini(&sa_manager->base);
 
-	amdgpu_bo_free_kernel(&sa_manager->bo, &sa_manager->gpu_addr, &sa_manager->cpu_ptr);
+	amdgpu_bo_free_kernel(&sa_manager->bo, &sa_manager->gpu_addr,
+			      &sa_manager->cpu_ptr);
 }
 
 int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
@@ -81,7 +85,8 @@ int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
 		     unsigned int size)
 {
 	struct drm_suballoc *sa = drm_suballoc_new(&sa_manager->base, size,
-						   GFP_KERNEL, false, 0);
+						   sa_manager->gfp_flags,
+						   false, 0);
 
 	if (IS_ERR(sa)) {
 		*sa_bo = NULL;
@@ -110,6 +115,7 @@ void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
 {
 	struct drm_printer p = drm_seq_file_printer(m);
 
-	drm_suballoc_dump_debug_info(&sa_manager->base, &p, sa_manager->gpu_addr);
+	drm_suballoc_dump_debug_info(&sa_manager->base, &p,
+				     sa_manager->gpu_addr);
 }
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
index 8c85c80fc119..1d1c89348709 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
@@ -35,6 +35,7 @@ struct amdgpu_sa_manager {
 	struct amdgpu_bo		*bo;
 	uint64_t			gpu_addr;
 	void				*cpu_ptr;
+	gfp_t				gfp_flags;
 };
 
 static inline struct amdgpu_sa_manager *
@@ -57,7 +58,7 @@ static inline void *amdgpu_sa_bo_cpu_addr(struct drm_suballoc *sa_bo)
 
 int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
 			      struct amdgpu_sa_manager *sa_manager,
-			      unsigned size, u32 align, u32 domain);
+			      unsigned size, gfp_t gfp_flags);
 void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
 			       struct amdgpu_sa_manager *sa_manager);
 int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev,
-- 
2.43.0

[PATCH 04/13] drm/amdgpu: move job parameter to the end in amdgpu_job_alloc() and *_with_ib()

[Christian König]

Move the job output parameter to be the last parameter in the
amdgpu_job_alloc() and amdgpu_job_alloc_with_ib() function signature. This
aligns with the common kernel coding convention where output parameters
typically come last.

Co-developed by Claude Sonnet 4.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c  |  2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c      |  4 ++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c     |  5 +++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c     |  3 ++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.c     | 10 +++++-----
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.h     |  8 ++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c    |  5 +++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c     | 11 ++++++-----
 drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c     |  5 +++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c     |  8 +++++---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c     | 12 ++++++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c |  2 +-
 drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c       | 10 ++++++----
 drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c       | 10 ++++++----
 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c    |  4 ++--
 15 files changed, 57 insertions(+), 42 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 0017d502d169..44751d71b741 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -696,7 +696,7 @@ int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev,
 		goto err;
 	}
 
-	ret = amdgpu_job_alloc(adev, NULL, NULL, NULL, 1, &job, 0);
+	ret = amdgpu_job_alloc(adev, NULL, NULL, NULL, 1, 0, &job);
 	if (ret)
 		goto err;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 10d8dcc3a972..fdf01d824d66 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -274,8 +274,8 @@ static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p,
 
 	for (i = 0; i < p->gang_size; ++i) {
 		ret = amdgpu_job_alloc(p->adev, vm, p->entities[i], vm,
-				       num_ibs[i], &p->jobs[i],
-				       p->filp->client_id);
+				       num_ibs[i], p->filp->client_id,
+				       &p->jobs[i]);
 		if (ret)
 			goto free_all_kdata;
 		switch (p->adev->enforce_isolation[fpriv->xcp_id]) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 515cc4a2aeb4..54450ac49834 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -1669,8 +1669,9 @@ static int amdgpu_gfx_run_cleaner_shader_job(struct amdgpu_ring *ring)
 	owner = (void *)(unsigned long)atomic_inc_return(&counter);
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, &entity, owner,
-				     64, 0, &job,
-				     AMDGPU_KERNEL_JOB_ID_CLEANER_SHADER);
+				     64, 0,
+				     AMDGPU_KERNEL_JOB_ID_CLEANER_SHADER,
+				     &job);
 	if (r)
 		goto err;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 13bec8461cde..181d69770c40 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -761,7 +761,8 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 	r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.default_entity.base,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
-				     &job, AMDGPU_KERNEL_JOB_ID_FLUSH_GPU_TLB);
+				     AMDGPU_KERNEL_JOB_ID_FLUSH_GPU_TLB,
+				     &job);
 	if (r)
 		goto error_alloc;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 55172c2dcc35..71c1ba735a6b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -187,8 +187,8 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 
 int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		     struct drm_sched_entity *entity, void *owner,
-		     unsigned int num_ibs, struct amdgpu_job **job,
-		     u64 drm_client_id)
+		     unsigned int num_ibs, u64 drm_client_id,
+		     struct amdgpu_job **job)
 {
 	struct amdgpu_fence *af;
 	int r;
@@ -241,12 +241,12 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev,
 			     struct drm_sched_entity *entity, void *owner,
 			     size_t size, enum amdgpu_ib_pool_type pool_type,
-			     struct amdgpu_job **job, u64 k_job_id)
+			     u64 k_job_id, struct amdgpu_job **job)
 {
 	int r;
 
-	r = amdgpu_job_alloc(adev, NULL, entity, owner, 1, job,
-			     k_job_id);
+	r = amdgpu_job_alloc(adev, NULL, entity, owner, 1, k_job_id,
+			     job);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
index 56a88e14a044..6b7cf594714c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
@@ -112,13 +112,13 @@ static inline struct amdgpu_ring *amdgpu_job_ring(struct amdgpu_job *job)
 
 int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		     struct drm_sched_entity *entity, void *owner,
-		     unsigned int num_ibs, struct amdgpu_job **job,
-		     u64 drm_client_id);
+		     unsigned int num_ibs, u64 drm_client_id,
+		     struct amdgpu_job **job);
 int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev,
 			     struct drm_sched_entity *entity, void *owner,
 			     size_t size, enum amdgpu_ib_pool_type pool_type,
-			     struct amdgpu_job **job,
-			     u64 k_job_id);
+			     u64 k_job_id,
+			     struct amdgpu_job **job);
 void amdgpu_job_set_resources(struct amdgpu_job *job, struct amdgpu_bo *gds,
 			      struct amdgpu_bo *gws, struct amdgpu_bo *oa);
 void amdgpu_job_free_resources(struct amdgpu_job *job);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c
index 63ee6ba6a931..988ad86971a8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c
@@ -196,8 +196,9 @@ static int amdgpu_jpeg_dec_set_reg(struct amdgpu_ring *ring, uint32_t handle,
 	int i, r;
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, ib_size_dw * 4,
-				     AMDGPU_IB_POOL_DIRECT, &job,
-				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 6c6ab4dd6ea9..cf78d7020494 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -243,8 +243,9 @@ static int amdgpu_ttm_map_buffer(struct amdgpu_ttm_buffer_entity *entity,
 	r = amdgpu_job_alloc_with_ib(adev, &entity->base,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     num_dw * 4 + num_bytes,
-				     AMDGPU_IB_POOL_DELAYED, &job,
-				     AMDGPU_KERNEL_JOB_ID_TTM_MAP_BUFFER);
+				     AMDGPU_IB_POOL_DELAYED,
+				     AMDGPU_KERNEL_JOB_ID_TTM_MAP_BUFFER,
+				     &job);
 	if (r)
 		return r;
 
@@ -1573,8 +1574,8 @@ static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo,
 	r = amdgpu_job_alloc_with_ib(adev, &adev->mman.default_entity.base,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     num_dw * 4, AMDGPU_IB_POOL_DELAYED,
-				     &job,
-				     AMDGPU_KERNEL_JOB_ID_TTM_ACCESS_MEMORY_SDMA);
+				     AMDGPU_KERNEL_JOB_ID_TTM_ACCESS_MEMORY_SDMA,
+				     &job);
 	if (r)
 		goto out;
 
@@ -2428,7 +2429,7 @@ static int amdgpu_ttm_prepare_job(struct amdgpu_device *adev,
 	int r;
 	r = amdgpu_job_alloc_with_ib(adev, &entity->base,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
-				     num_dw * 4, pool, job, k_job_id);
+				     num_dw * 4, pool, k_job_id, job);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index 3a3bc0d370fa..7df839634098 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -1137,8 +1137,9 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 	r = amdgpu_job_alloc_with_ib(ring->adev, &adev->uvd.entity,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     64, direct ? AMDGPU_IB_POOL_DIRECT :
-				     AMDGPU_IB_POOL_DELAYED, &job,
-				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_IB_POOL_DELAYED,
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index efdebd9c0a1f..344a703d03be 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -473,7 +473,8 @@ static int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
 	r = amdgpu_job_alloc_with_ib(ring->adev, &ring->adev->vce.entity,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     ib_size_dw * 4, AMDGPU_IB_POOL_DIRECT,
-				     &job, AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
@@ -564,8 +565,9 @@ static int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     ib_size_dw * 4,
 				     direct ? AMDGPU_IB_POOL_DIRECT :
-				     AMDGPU_IB_POOL_DELAYED, &job,
-				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_IB_POOL_DELAYED,
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
index 616967519869..6b15415f989c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
@@ -631,7 +631,8 @@ static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring,
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL,
 				     64, AMDGPU_IB_POOL_DIRECT,
-				     &job, AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		goto err;
 
@@ -811,7 +812,8 @@ static int amdgpu_vcn_dec_sw_send_msg(struct amdgpu_ring *ring,
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL,
 				     ib_size_dw * 4, AMDGPU_IB_POOL_DIRECT,
-				     &job, AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		goto err;
 
@@ -941,7 +943,8 @@ static int amdgpu_vcn_enc_get_create_msg(struct amdgpu_ring *ring, uint32_t hand
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL,
 				     ib_size_dw * 4, AMDGPU_IB_POOL_DIRECT,
-				     &job, AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
@@ -1008,7 +1011,8 @@ static int amdgpu_vcn_enc_get_destroy_msg(struct amdgpu_ring *ring, uint32_t han
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL,
 				     ib_size_dw * 4, AMDGPU_IB_POOL_DIRECT,
-				     &job, AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
index 2eb64df6daa9..50cc0779c340 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
@@ -56,7 +56,7 @@ static int amdgpu_vm_sdma_alloc_job(struct amdgpu_vm_update_params *p,
 	ndw = min(ndw, AMDGPU_VM_SDMA_MAX_NUM_DW);
 
 	r = amdgpu_job_alloc_with_ib(p->adev, entity, AMDGPU_FENCE_OWNER_VM,
-				     ndw * 4, pool, &p->job, k_job_id);
+				     ndw * 4, pool, k_job_id, &p->job);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index ecd7ead7a60b..16137829fe84 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -217,8 +217,9 @@ static int uvd_v6_0_enc_get_create_msg(struct amdgpu_ring *ring, uint32_t handle
 	int i, r;
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, ib_size_dw * 4,
-				     AMDGPU_IB_POOL_DIRECT, &job,
-				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
@@ -282,8 +283,9 @@ static int uvd_v6_0_enc_get_destroy_msg(struct amdgpu_ring *ring,
 	int i, r;
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, ib_size_dw * 4,
-				     AMDGPU_IB_POOL_DIRECT, &job,
-				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
index df2c83348315..46222fc30be6 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
@@ -225,8 +225,9 @@ static int uvd_v7_0_enc_get_create_msg(struct amdgpu_ring *ring, u32 handle,
 	int i, r;
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, ib_size_dw * 4,
-				     AMDGPU_IB_POOL_DIRECT, &job,
-				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
@@ -289,8 +290,9 @@ static int uvd_v7_0_enc_get_destroy_msg(struct amdgpu_ring *ring, u32 handle,
 	int i, r;
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, ib_size_dw * 4,
-				     AMDGPU_IB_POOL_DIRECT, &job,
-				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 28dc6886c1ff..51ad46c25c5a 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -68,8 +68,8 @@ svm_migrate_gart_map(struct amdgpu_ring *ring,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     num_dw * 4 + num_bytes,
 				     AMDGPU_IB_POOL_DELAYED,
-				     &job,
-				     AMDGPU_KERNEL_JOB_ID_KFD_GART_MAP);
+				     AMDGPU_KERNEL_JOB_ID_KFD_GART_MAP,
+				     &job);
 	if (r)
 		return r;
 
-- 
2.43.0

[PATCH 05/13] drm/amdgpu: use correct gfp_t for job allocation

[Christian König]

For job allocation in GPU reset and page fault handling we must use
GFP_ATOMIC to guarantee that we don't cycle back and depend on a
dma_fence submission for the memory allocation.

Co-developed by Claude Sonnet 4.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c |  3 ++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c     |  2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c     | 11 +++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.c    |  9 +++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.h    |  2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h   |  2 ++
 6 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 44751d71b741..c425f960361c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -696,7 +696,8 @@ int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev,
 		goto err;
 	}
 
-	ret = amdgpu_job_alloc(adev, NULL, NULL, NULL, 1, 0, &job);
+	ret = amdgpu_job_alloc(adev, NULL, NULL, NULL, 1, 0, GFP_KERNEL,
+			       &job);
 	if (ret)
 		goto err;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index fdf01d824d66..3ed2cb9dfa38 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -275,7 +275,7 @@ static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p,
 	for (i = 0; i < p->gang_size; ++i) {
 		ret = amdgpu_job_alloc(p->adev, vm, p->entities[i], vm,
 				       num_ibs[i], p->filp->client_id,
-				       &p->jobs[i]);
+				       GFP_KERNEL, &p->jobs[i]);
 		if (ret)
 			goto free_all_kdata;
 		switch (p->adev->enforce_isolation[fpriv->xcp_id]) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index 60e4c3985029..83a802cf5837 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -416,6 +416,17 @@ void amdgpu_ib_pool_fini(struct amdgpu_device *adev)
 	adev->ib_pool_ready = false;
 }
 
+/**
+ * amdgpu_ib_pool_gfp_flags - Returns the gfp flags to use for each pool
+ * @adev: amdgpu device pointer
+ * @type: the IB pool type
+ */
+gfp_t amdgpu_ib_pool_gfp_flags(struct amdgpu_device *adev,
+			       enum amdgpu_ib_pool_type type)
+{
+	return adev->ib_pools[type].gfp_flags;
+}
+
 /**
  * amdgpu_ib_ring_tests - test IBs on the rings
  *
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 71c1ba735a6b..b4b424d84479 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -188,7 +188,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		     struct drm_sched_entity *entity, void *owner,
 		     unsigned int num_ibs, u64 drm_client_id,
-		     struct amdgpu_job **job)
+		     gfp_t gfp_flags, struct amdgpu_job **job)
 {
 	struct amdgpu_fence *af;
 	int r;
@@ -196,18 +196,18 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	if (num_ibs == 0)
 		return -EINVAL;
 
-	*job = kzalloc(struct_size(*job, ibs, num_ibs), GFP_KERNEL);
+	*job = kzalloc(struct_size(*job, ibs, num_ibs), gfp_flags);
 	if (!*job)
 		return -ENOMEM;
 
-	af = kzalloc(sizeof(struct amdgpu_fence), GFP_KERNEL);
+	af = kzalloc(sizeof(struct amdgpu_fence), gfp_flags);
 	if (!af) {
 		r = -ENOMEM;
 		goto err_job;
 	}
 	(*job)->hw_fence = af;
 
-	af = kzalloc(sizeof(struct amdgpu_fence), GFP_KERNEL);
+	af = kzalloc(sizeof(struct amdgpu_fence), gfp_flags);
 	if (!af) {
 		r = -ENOMEM;
 		goto err_fence;
@@ -246,6 +246,7 @@ int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev,
 	int r;
 
 	r = amdgpu_job_alloc(adev, NULL, entity, owner, 1, k_job_id,
+			     amdgpu_ib_pool_gfp_flags(adev, pool_type),
 			     job);
 	if (r)
 		return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
index 6b7cf594714c..44fe40f9e8df 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
@@ -113,7 +113,7 @@ static inline struct amdgpu_ring *amdgpu_job_ring(struct amdgpu_job *job)
 int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		     struct drm_sched_entity *entity, void *owner,
 		     unsigned int num_ibs, u64 drm_client_id,
-		     struct amdgpu_job **job);
+		     gfp_t gfp_flags, struct amdgpu_job **job);
 int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev,
 			     struct drm_sched_entity *entity, void *owner,
 			     size_t size, enum amdgpu_ib_pool_type pool_type,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 1a063a0a4280..4bf9734acc18 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -582,6 +582,8 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 		       struct dma_fence **f);
 int amdgpu_ib_pool_init(struct amdgpu_device *adev);
 void amdgpu_ib_pool_fini(struct amdgpu_device *adev);
+gfp_t amdgpu_ib_pool_gfp_flags(struct amdgpu_device *adev,
+			       enum amdgpu_ib_pool_type type);
 int amdgpu_ib_ring_tests(struct amdgpu_device *adev);
 bool amdgpu_ring_sched_ready(struct amdgpu_ring *ring);
 void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring,
-- 
2.43.0

[PATCH 06/13] drm/amdgpu: add amdgpu_vm_update_leaves()

[Christian König]

Add a new function amdgpu_vm_update_leaves() to avoid memory allocation
on page faults.

The idea is to only update the leave PTEs to insert a dummy PRT PTE.

TODO: HW older than GMC v9 needs a different solution.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 40 ++++++++++---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h    |  3 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 71 ++++++++++++++++++++++-
 3 files changed, 103 insertions(+), 11 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index e5588346a03f..94632a660b79 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -2984,10 +2984,12 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
 			    u32 vmid, u32 node_id, uint64_t addr,
 			    uint64_t ts, bool write_fault)
 {
-	bool is_compute_context = false;
+	struct amdgpu_vm_update_params params;
+	bool is_compute_context;
 	struct amdgpu_bo *root;
 	uint64_t value, flags;
 	struct amdgpu_vm *vm;
+	unsigned int idx;
 	int r;
 
 	vm = amdgpu_vm_lock_by_pasid(adev, &root, pasid);
@@ -3029,24 +3031,46 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
 		value = adev->dummy_page_addr;
 		flags |= AMDGPU_PTE_EXECUTABLE | AMDGPU_PTE_READABLE |
 			AMDGPU_PTE_WRITEABLE;
-
+		/* On +gfx9 we can use the PRT functionality instead */
+		if (!adev->gmc.gmc_funcs->set_prt) {
+			flags &= ~AMDGPU_PTE_VALID;
+			flags |= AMDGPU_PTE_PRT;
+		}
 	} else {
 		/* Let the hw retry silently on the PTE */
 		value = 0;
 	}
 
+	if (!drm_dev_enter(adev_to_drm(adev), &idx)) {
+		r = -ENODEV;
+		goto error_unlock;
+	}
+
+	amdgpu_vm_eviction_lock(vm);
+	if (vm->evicting) {
+		r = -EBUSY;
+		goto error_dev_exit;
+	}
+
+	memset(&params, 0, sizeof(params));
+	params.adev = adev;
+	params.vm = vm;
+	params.immediate = true;
+	params.pages_addr = NULL;
+
 	r = dma_resv_reserve_fences(root->tbo.base.resv, 1);
 	if (r) {
 		pr_debug("failed %d to reserve fence slot\n", r);
-		goto error_unlock;
+		goto error_eviction_lock;
 	}
 
-	r = amdgpu_vm_update_range(adev, vm, true, false, false, false,
-				   NULL, addr, addr, flags, value, 0, NULL, NULL, NULL);
-	if (r)
-		goto error_unlock;
+	amdgpu_vm_update_leaves(&params, addr, addr, value, flags);
 
-	r = amdgpu_vm_update_pdes(adev, vm, true);
+error_eviction_lock:
+	amdgpu_vm_eviction_unlock(vm);
+
+error_dev_exit:
+	drm_dev_exit(idx);
 
 error_unlock:
 	amdgpu_bo_unreserve(root);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index cc096c005e34..04b32accfa3f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -613,6 +613,9 @@ int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
 int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 			  uint64_t start, uint64_t end,
 			  uint64_t dst, uint64_t flags);
+void amdgpu_vm_update_leaves(struct amdgpu_vm_update_params *params,
+			     uint64_t start, uint64_t end,
+			     int64_t dst, uint64_t flags);
 void amdgpu_vm_pt_free_work(struct work_struct *work);
 void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
 			    struct amdgpu_vm_update_params *params);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index e43a60d09808..9766b6b9aecc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -790,6 +790,8 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 			  uint64_t dst, uint64_t flags)
 {
 	struct amdgpu_device *adev = params->adev;
+	struct amdgpu_vm *vm = params->vm;
+	pid_t tgid = vm->task_info ? vm->task_info->tgid : 0;
 	struct amdgpu_vm_pt_cursor cursor;
 	uint64_t frag_start = start, frag_end;
 	unsigned int frag;
@@ -881,7 +883,6 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 		entry_end = min(entry_end, end);
 
 		do {
-			struct amdgpu_vm *vm = params->vm;
 			uint64_t upd_end = min(entry_end, frag_end);
 			unsigned int nptes = (upd_end - frag_start) >> shift;
 			uint64_t upd_flags = flags | AMDGPU_PTE_FRAG(frag);
@@ -893,8 +894,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 
 			trace_amdgpu_vm_update_ptes(params, frag_start, upd_end,
 						    min(nptes, 32u), dst, incr,
-						    upd_flags,
-						    vm->task_info ? vm->task_info->tgid : 0,
+						    upd_flags, tgid,
 						    vm->immediate.fence_context);
 			amdgpu_vm_pte_update_flags(params, to_amdgpu_bo_vm(pt),
 						   cursor.level, pe_start, dst,
@@ -938,6 +938,71 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 	return 0;
 }
 
+/**
+ * amdgpu_vm_update_leaves - update leave PDEs/PTEs
+ *
+ * @params: see amdgpu_vm_update_params definition
+ * @start: start of GPU address range
+ * @end: end of GPU address range
+ * @dst: destination address to insert into the leave PDEs/PTEs
+ * @flags: mapping flags
+ *
+ * Update the leave PDEs/PTEs in the range @start - @end without allocating or
+ * freeing page tables.
+ *
+ * Returns:
+ * 0 for success, negative error code for failure.
+ */
+void amdgpu_vm_update_leaves(struct amdgpu_vm_update_params *params,
+			     uint64_t start, uint64_t end,
+			     int64_t dst, uint64_t flags)
+{
+	struct amdgpu_device *adev = params->adev;
+	struct amdgpu_vm *vm = params->vm;
+	pid_t tgid = vm->task_info ? vm->task_info->tgid : 0;
+	struct amdgpu_vm_pt_cursor cursor;
+
+	amdgpu_vm_pt_start(adev, params->vm, start, &cursor);
+	while (cursor.pfn < end) {
+		unsigned int shift, mask;
+		uint64_t entry_end, pe_start;
+		struct amdgpu_bo *pt;
+		unsigned int nptes;
+
+		/* Walk to the leave entries */
+		if (amdgpu_vm_pt_descendant(adev, &cursor))
+			continue;
+
+		pt = cursor.parent->bo;
+		shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1);
+		mask = amdgpu_vm_pt_entries_mask(adev, cursor.level - 1);
+
+		/* Looks good so far, calculate parameters for the update */
+		pe_start = ((cursor.pfn >> shift) & mask) * 8;
+
+		entry_end = ((uint64_t)mask + 1) << shift;
+		entry_end += cursor.pfn & ~(entry_end - 1);
+		entry_end = min(entry_end, end);
+
+		nptes = (entry_end - cursor.pfn) >> shift;
+		/*
+		 * This can happen when we set higher level PDEs to unmap and/or
+		 * silent to stop fault floods.
+		 */
+		nptes = max(nptes, 1u);
+
+		trace_amdgpu_vm_update_ptes(params, cursor.pfn, entry_end,
+					    min(nptes, 32u), dst, 0, flags,
+					    tgid,
+					    vm->immediate.fence_context);
+		amdgpu_vm_pte_update_flags(params, to_amdgpu_bo_vm(pt),
+					   cursor.level - 1, pe_start, dst,
+					   nptes, 0, flags);
+
+		amdgpu_vm_pt_next(adev, &cursor);
+	}
+}
+
 /**
  * amdgpu_vm_pt_map_tables - have bo of root PD cpu accessible
  * @adev: amdgpu device structure
-- 
2.43.0

[PATCH 07/13] drm/amdgpu: drop immediate updates from amdgpu_vm_update_range

[Christian König]

That case is handled by amdgpu_vm_update_leaves now.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 21 +++++++++------------
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h    | 16 +++++++---------
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 19 ++++++-------------
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c      |  4 ++--
 4 files changed, 24 insertions(+), 36 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 94632a660b79..edc8b1ca2d3e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -1084,7 +1084,6 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
  *
  * @adev: amdgpu_device pointer to use for commands
  * @vm: the VM to update the range
- * @immediate: immediate submission in a page fault
  * @unlocked: unlocked invalidation during MM callback
  * @flush_tlb: trigger tlb invalidation after update completed
  * @allow_override: change MTYPE for local NUMA nodes
@@ -1104,12 +1103,11 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
  * 0 for success, negative erro code for failure.
  */
 int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			   bool immediate, bool unlocked, bool flush_tlb,
-			   bool allow_override, struct amdgpu_sync *sync,
-			   uint64_t start, uint64_t last, uint64_t flags,
-			   uint64_t offset, uint64_t vram_base,
-			   struct ttm_resource *res, dma_addr_t *pages_addr,
-			   struct dma_fence **fence)
+			   bool unlocked, bool flush_tlb, bool allow_override,
+			   struct amdgpu_sync *sync, uint64_t start,
+			   uint64_t last, uint64_t flags, uint64_t offset,
+			   uint64_t vram_base, struct ttm_resource *res,
+			   dma_addr_t *pages_addr, struct dma_fence **fence)
 {
 	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
 	struct amdgpu_vm_update_params params;
@@ -1139,7 +1137,6 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	memset(&params, 0, sizeof(params));
 	params.adev = adev;
 	params.vm = vm;
-	params.immediate = immediate;
 	params.pages_addr = pages_addr;
 	params.unlocked = unlocked;
 	params.needs_flush = flush_tlb;
@@ -1365,7 +1362,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
 
 		trace_amdgpu_vm_bo_update(mapping);
 
-		r = amdgpu_vm_update_range(adev, vm, false, false, flush_tlb,
+		r = amdgpu_vm_update_range(adev, vm, false, flush_tlb,
 					   !uncached, &sync, mapping->start,
 					   mapping->last, update_flags,
 					   mapping->offset, vram_base, mem,
@@ -1568,7 +1565,7 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
 			struct amdgpu_bo_va_mapping, list);
 		list_del(&mapping->list);
 
-		r = amdgpu_vm_update_range(adev, vm, false, false, true, false,
+		r = amdgpu_vm_update_range(adev, vm, false, true, false,
 					   &sync, mapping->start, mapping->last,
 					   0, 0, 0, NULL, NULL, &f);
 		amdgpu_vm_free_mapping(adev, vm, mapping, f);
@@ -2617,7 +2614,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	vm->tlb_fence_context = dma_fence_context_alloc(1);
 
 	r = amdgpu_vm_pt_create(adev, vm, adev->vm_manager.root_level,
-				false, &root, xcp_id);
+				&root, xcp_id);
 	if (r)
 		goto error_free_delayed;
 
@@ -2633,7 +2630,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	if (r)
 		goto error_free_root;
 
-	r = amdgpu_vm_pt_clear(adev, vm, root, false);
+	r = amdgpu_vm_pt_clear(adev, vm, root);
 	if (r)
 		goto error_free_root;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 04b32accfa3f..3e86a2a470f0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -530,12 +530,11 @@ int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev,
 void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
 			    struct amdgpu_vm *vm, struct amdgpu_bo *bo);
 int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			   bool immediate, bool unlocked, bool flush_tlb,
-			   bool allow_override, struct amdgpu_sync *sync,
-			   uint64_t start, uint64_t last, uint64_t flags,
-			   uint64_t offset, uint64_t vram_base,
-			   struct ttm_resource *res, dma_addr_t *pages_addr,
-			   struct dma_fence **fence);
+			   bool unlocked, bool flush_tlb, bool allow_override,
+			   struct amdgpu_sync *sync, uint64_t start,
+			   uint64_t last, uint64_t flags, uint64_t offset,
+			   uint64_t vram_base, struct ttm_resource *res,
+			   dma_addr_t *pages_addr, struct dma_fence **fence);
 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 			struct amdgpu_bo_va *bo_va,
 			bool clear);
@@ -602,10 +601,9 @@ void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
 			  struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM]);
 
 int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-		       struct amdgpu_bo_vm *vmbo, bool immediate);
+		       struct amdgpu_bo_vm *vmbo);
 int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			int level, bool immediate, struct amdgpu_bo_vm **vmbo,
-			int32_t xcp_id);
+			int level, struct amdgpu_bo_vm **vmbo, int32_t xcp_id);
 void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 
 int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index 9766b6b9aecc..6f5415d5a1bc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -351,7 +351,6 @@ static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev,
  * @adev: amdgpu_device pointer
  * @vm: VM to clear BO from
  * @vmbo: BO to clear
- * @immediate: use an immediate update
  *
  * Root PD needs to be reserved when calling this.
  *
@@ -359,7 +358,7 @@ static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev,
  * 0 on success, errno otherwise.
  */
 int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-		       struct amdgpu_bo_vm *vmbo, bool immediate)
+		       struct amdgpu_bo_vm *vmbo)
 {
 	unsigned int level = adev->vm_manager.root_level;
 	struct ttm_operation_ctx ctx = { true, false };
@@ -396,7 +395,6 @@ int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	memset(&params, 0, sizeof(params));
 	params.adev = adev;
 	params.vm = vm;
-	params.immediate = immediate;
 
 	r = vm->update_funcs->prepare(&params, NULL,
 				      AMDGPU_KERNEL_JOB_ID_VM_PT_CLEAR);
@@ -434,13 +432,11 @@ int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  * @adev: amdgpu_device pointer
  * @vm: requesting vm
  * @level: the page table level
- * @immediate: use a immediate update
  * @vmbo: pointer to the buffer object pointer
  * @xcp_id: GPU partition id
  */
 int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			int level, bool immediate, struct amdgpu_bo_vm **vmbo,
-			int32_t xcp_id)
+			int level, struct amdgpu_bo_vm **vmbo, int32_t xcp_id)
 {
 	struct amdgpu_bo_param bp;
 	unsigned int num_entries;
@@ -470,7 +466,6 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		bp.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 
 	bp.type = ttm_bo_type_kernel;
-	bp.no_wait_gpu = immediate;
 	bp.xcp_id_plus1 = xcp_id + 1;
 
 	if (vm->root.bo)
@@ -485,7 +480,6 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  * @adev: amdgpu_device pointer
  * @vm: VM to allocate page tables for
  * @cursor: Which page table to allocate
- * @immediate: use an immediate update
  *
  * Make sure a specific page table or directory is allocated.
  *
@@ -495,8 +489,7 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  */
 static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev,
 			      struct amdgpu_vm *vm,
-			      struct amdgpu_vm_pt_cursor *cursor,
-			      bool immediate)
+			      struct amdgpu_vm_pt_cursor *cursor)
 {
 	struct amdgpu_vm_bo_base *entry = cursor->entry;
 	struct amdgpu_bo *pt_bo;
@@ -507,7 +500,7 @@ static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev,
 		return 0;
 
 	amdgpu_vm_eviction_unlock(vm);
-	r = amdgpu_vm_pt_create(adev, vm, cursor->level, immediate, &pt,
+	r = amdgpu_vm_pt_create(adev, vm, cursor->level, &pt,
 				vm->root.bo->xcp_id);
 	amdgpu_vm_eviction_lock(vm);
 	if (r)
@@ -519,7 +512,7 @@ static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev,
 	pt_bo = &pt->bo;
 	pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo);
 	amdgpu_vm_bo_base_init(entry, vm, pt_bo);
-	r = amdgpu_vm_pt_clear(adev, vm, pt, immediate);
+	r = amdgpu_vm_pt_clear(adev, vm, pt);
 	if (r)
 		goto error_free_pt;
 
@@ -813,7 +806,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 			 * address range are actually allocated
 			 */
 			r = amdgpu_vm_pt_alloc(params->adev, params->vm,
-					       &cursor, params->immediate);
+					       &cursor);
 			if (r)
 				return r;
 		}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 72cfb4a6ab3e..37b5166e9a14 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -1372,7 +1372,7 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		return -EINVAL;
 	}
 
-	return amdgpu_vm_update_range(adev, vm, false, true, true, false, NULL, gpu_start,
+	return amdgpu_vm_update_range(adev, vm, true, true, false, NULL, gpu_start,
 				      gpu_end, init_pte_value, 0, 0, NULL, NULL,
 				      fence);
 }
@@ -1489,7 +1489,7 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
 			 (last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0,
 			 pte_flags);
 
-		r = amdgpu_vm_update_range(adev, vm, false, false, flush_tlb, true,
+		r = amdgpu_vm_update_range(adev, vm, false, flush_tlb, true,
 					   NULL, gpu_start, gpu_end,
 					   pte_flags,
 					   (last_start - prange->start) << PAGE_SHIFT,
-- 
2.43.0

[PATCH 08/13] drm/amdgpu: split amdgpu_vm_update_range

[Christian König]

Split amdgpu_vm_update_range into two functions.

amdgpu_vm_map_range() is for mapping PTEs into a range and updates
which can be done while holding the VM lock.

amdgpu_vm_unmap_range() is for unmapping PTEs without holding the VM
lock in MMU notifiers.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.h   |   3 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 112 ++++++++++++++++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h    |  14 ++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c |  35 ++-----
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c      |  16 ++--
 5 files changed, 120 insertions(+), 60 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
index 44fe40f9e8df..653ffa9ca0f3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
@@ -47,7 +47,7 @@ enum amdgpu_ib_pool_type;
 /* Internal kernel job ids. (decreasing values, starting from U64_MAX). */
 #define AMDGPU_KERNEL_JOB_ID_VM_UPDATE              (18446744073709551615ULL)
 #define AMDGPU_KERNEL_JOB_ID_VM_UPDATE_PDES         (18446744073709551614ULL)
-#define AMDGPU_KERNEL_JOB_ID_VM_UPDATE_RANGE        (18446744073709551613ULL)
+#define AMDGPU_KERNEL_JOB_ID_VM_MAP_RANGE           (18446744073709551613ULL)
 #define AMDGPU_KERNEL_JOB_ID_VM_PT_CLEAR            (18446744073709551612ULL)
 #define AMDGPU_KERNEL_JOB_ID_TTM_MAP_BUFFER         (18446744073709551611ULL)
 #define AMDGPU_KERNEL_JOB_ID_TTM_ACCESS_MEMORY_SDMA (18446744073709551610ULL)
@@ -63,6 +63,7 @@ enum amdgpu_ib_pool_type;
 #define AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST         (18446744073709551600ULL)
 #define AMDGPU_KERNEL_JOB_ID_VPE_RING_TEST          (18446744073709551599ULL)
 #define AMDGPU_KERNEL_JOB_ID_RUN_SHADER             (18446744073709551598ULL)
+#define AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE         (18446744073709551597ULL)
 
 struct amdgpu_job {
 	struct drm_sched_job    base;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index edc8b1ca2d3e..b5adfcacc55a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -1080,11 +1080,10 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
 }
 
 /**
- * amdgpu_vm_update_range - update a range in the vm page table
+ * amdgpu_vm_map_range - map something to a range in the vm page tables
  *
  * @adev: amdgpu_device pointer to use for commands
  * @vm: the VM to update the range
- * @unlocked: unlocked invalidation during MM callback
  * @flush_tlb: trigger tlb invalidation after update completed
  * @allow_override: change MTYPE for local NUMA nodes
  * @sync: fences we need to sync to
@@ -1097,23 +1096,26 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
  * @pages_addr: DMA addresses to use for mapping
  * @fence: optional resulting fence
  *
- * Fill in the page table entries between @start and @last.
+ * Fill in the page table entries between @start and @last. Allocate and free
+ * new page tables as needed. Can only be called while holding the VM lock.
  *
  * Returns:
  * 0 for success, negative erro code for failure.
  */
-int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			   bool unlocked, bool flush_tlb, bool allow_override,
-			   struct amdgpu_sync *sync, uint64_t start,
-			   uint64_t last, uint64_t flags, uint64_t offset,
-			   uint64_t vram_base, struct ttm_resource *res,
-			   dma_addr_t *pages_addr, struct dma_fence **fence)
+int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+			bool flush_tlb, bool allow_override,
+			struct amdgpu_sync *sync, uint64_t start,
+			uint64_t last, uint64_t flags, uint64_t offset,
+			uint64_t vram_base, struct ttm_resource *res,
+			dma_addr_t *pages_addr, struct dma_fence **fence)
 {
 	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
 	struct amdgpu_vm_update_params params;
 	struct amdgpu_res_cursor cursor;
 	int r, idx;
 
+	amdgpu_vm_assert_locked(vm);
+
 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
 		return -ENODEV;
 
@@ -1138,7 +1140,6 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	params.adev = adev;
 	params.vm = vm;
 	params.pages_addr = pages_addr;
-	params.unlocked = unlocked;
 	params.needs_flush = flush_tlb;
 	params.override_pte = allow_override && adev->gmc.override_pte;
 	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
@@ -1149,7 +1150,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		goto error_free;
 	}
 
-	if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) {
+	if (!dma_fence_is_signaled(vm->last_unlocked)) {
 		struct dma_fence *tmp = dma_fence_get_stub();
 
 		amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true);
@@ -1158,7 +1159,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	}
 
 	r = vm->update_funcs->prepare(&params, sync,
-				      AMDGPU_KERNEL_JOB_ID_VM_UPDATE_RANGE);
+				      AMDGPU_KERNEL_JOB_ID_VM_MAP_RANGE);
 	if (r)
 		goto error_free;
 
@@ -1234,6 +1235,77 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	return r;
 }
 
+/**
+ * amdgpu_vm_unmap_range - clear leave PTEs to unmap something
+ *
+ * @adev: amdgpu_device pointer to use for commands
+ * @vm: the VM to update the range
+ * @sync: fences we need to sync to
+ * @start: start of unmapped range
+ * @last: last unmapped entry
+ * @flags: flags for the entries
+ * @fence: optional resulting fence
+ *
+ * Fill in the page table entries between @start and @last with a fixed flags
+ * value without allocating or freeing page tables. Can be used without locking
+ * the VM.
+ *
+ * Returns:
+ * 0 for success, negative erro code for failure.
+ */
+int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+			   struct amdgpu_sync *sync, uint64_t start,
+			   uint64_t last, uint64_t flags,
+			   struct dma_fence **fence)
+{
+	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
+	struct amdgpu_vm_update_params params;
+	int r, idx;
+
+	if (!drm_dev_enter(adev_to_drm(adev), &idx))
+		return -ENODEV;
+
+	tlb_cb = kmalloc(sizeof(*tlb_cb), GFP_KERNEL);
+	if (!tlb_cb) {
+		drm_dev_exit(idx);
+		return -ENOMEM;
+	}
+
+	memset(&params, 0, sizeof(params));
+	params.adev = adev;
+	params.vm = vm;
+	params.needs_flush = true;
+	params.unlocked = true;
+	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
+
+	amdgpu_vm_eviction_lock(vm);
+	if (vm->evicting) {
+		r = -EBUSY;
+		goto error_free;
+	}
+
+	r = vm->update_funcs->prepare(&params, sync,
+				      AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE);
+	if (r)
+		goto error_free;
+
+	amdgpu_vm_update_leaves(&params, start, last, 0, flags);
+
+	r = vm->update_funcs->commit(&params, fence);
+	if (r)
+		goto error_free;
+
+	amdgpu_vm_tlb_flush(&params, fence, tlb_cb);
+	amdgpu_vm_pt_free_list(adev, &params);
+	tlb_cb = NULL;
+
+error_free:
+	kfree(tlb_cb);
+	amdgpu_vm_eviction_unlock(vm);
+	drm_dev_exit(idx);
+	return r;
+}
+
 void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
 			  struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
 {
@@ -1362,11 +1434,11 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
 
 		trace_amdgpu_vm_bo_update(mapping);
 
-		r = amdgpu_vm_update_range(adev, vm, false, flush_tlb,
-					   !uncached, &sync, mapping->start,
-					   mapping->last, update_flags,
-					   mapping->offset, vram_base, mem,
-					   pages_addr, last_update);
+		r = amdgpu_vm_map_range(adev, vm, flush_tlb, !uncached, &sync,
+					mapping->start, mapping->last,
+					update_flags, mapping->offset,
+					vram_base, mem, pages_addr,
+					last_update);
 		if (r)
 			goto error_free;
 	}
@@ -1565,9 +1637,9 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
 			struct amdgpu_bo_va_mapping, list);
 		list_del(&mapping->list);
 
-		r = amdgpu_vm_update_range(adev, vm, false, true, false,
-					   &sync, mapping->start, mapping->last,
-					   0, 0, 0, NULL, NULL, &f);
+		r = amdgpu_vm_map_range(adev, vm, true, false,
+					&sync, mapping->start, mapping->last,
+					0, 0, 0, NULL, NULL, &f);
 		amdgpu_vm_free_mapping(adev, vm, mapping, f);
 		if (r) {
 			dma_fence_put(f);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 3e86a2a470f0..561f2873d2ec 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -529,12 +529,16 @@ int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev,
 				uint32_t xcc_mask);
 void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
 			    struct amdgpu_vm *vm, struct amdgpu_bo *bo);
-int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			   bool unlocked, bool flush_tlb, bool allow_override,
+int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+			bool flush_tlb, bool allow_override,
+			struct amdgpu_sync *sync, uint64_t start,
+			uint64_t last, uint64_t flags, uint64_t offset,
+			uint64_t vram_base, struct ttm_resource *res,
+			dma_addr_t *pages_addr, struct dma_fence **fence);
+int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			   struct amdgpu_sync *sync, uint64_t start,
-			   uint64_t last, uint64_t flags, uint64_t offset,
-			   uint64_t vram_base, struct ttm_resource *res,
-			   dma_addr_t *pages_addr, struct dma_fence **fence);
+			   uint64_t last, uint64_t flags,
+			   struct dma_fence **fence);
 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 			struct amdgpu_bo_va *bo_va,
 			bool clear);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index 6f5415d5a1bc..ac3f3e31e2e2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -553,7 +553,6 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
 			    struct amdgpu_vm_update_params *params)
 {
 	struct amdgpu_vm_bo_base *entry, *next;
-	bool unlocked = params->unlocked;
 
 	if (list_empty(&params->tlb_flush_waitlist))
 		return;
@@ -561,7 +560,7 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
 	/*
 	 * unlocked unmap clear page table leaves, warning to free the page entry.
 	 */
-	WARN_ON(unlocked);
+	WARN_ON(params->unlocked);
 
 	list_for_each_entry_safe(entry, next, &params->tlb_flush_waitlist, vm_status)
 		amdgpu_vm_pt_free(entry);
@@ -801,24 +800,17 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 		uint64_t incr, entry_end, pe_start;
 		struct amdgpu_bo *pt;
 
-		if (!params->unlocked) {
-			/* make sure that the page tables covering the
-			 * address range are actually allocated
-			 */
-			r = amdgpu_vm_pt_alloc(params->adev, params->vm,
-					       &cursor);
-			if (r)
-				return r;
-		}
+		/* make sure that the page tables covering the
+		 * address range are actually allocated
+		 */
+		r = amdgpu_vm_pt_alloc(params->adev, params->vm, &cursor);
+		if (r)
+			return r;
 
 		shift = amdgpu_vm_pt_level_shift(adev, cursor.level);
 		parent_shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1);
-		if (params->unlocked) {
-			/* Unlocked updates are only allowed on the leaves */
-			if (amdgpu_vm_pt_descendant(adev, &cursor))
-				continue;
-		} else if (adev->asic_type < CHIP_VEGA10 &&
-			   (flags & AMDGPU_PTE_VALID)) {
+		if (adev->asic_type < CHIP_VEGA10 &&
+		    (flags & AMDGPU_PTE_VALID)) {
 			/* No huge page support before GMC v9 */
 			if (cursor.level != AMDGPU_VM_PTB) {
 				if (!amdgpu_vm_pt_descendant(adev, &cursor))
@@ -864,14 +856,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 		mask = amdgpu_vm_pt_entries_mask(adev, cursor.level);
 		pe_start = ((cursor.pfn >> shift) & mask) * 8;
 
-		if (cursor.level < AMDGPU_VM_PTB && params->unlocked)
-			/*
-			 * MMU notifier callback unlocked unmap huge page, leave is PDE entry,
-			 * only clear one entry. Next entry search again for PDE or PTE leave.
-			 */
-			entry_end = 1ULL << shift;
-		else
-			entry_end = ((uint64_t)mask + 1) << shift;
+		entry_end = ((uint64_t)mask + 1) << shift;
 		entry_end += cursor.pfn & ~(entry_end - 1);
 		entry_end = min(entry_end, end);
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 37b5166e9a14..d0ea20dea3e1 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -1372,9 +1372,8 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		return -EINVAL;
 	}
 
-	return amdgpu_vm_update_range(adev, vm, true, true, false, NULL, gpu_start,
-				      gpu_end, init_pte_value, 0, 0, NULL, NULL,
-				      fence);
+	return amdgpu_vm_unmap_range(adev, vm, NULL, gpu_start, gpu_end,
+				     init_pte_value, fence);
 }
 
 static int
@@ -1489,12 +1488,11 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
 			 (last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0,
 			 pte_flags);
 
-		r = amdgpu_vm_update_range(adev, vm, false, flush_tlb, true,
-					   NULL, gpu_start, gpu_end,
-					   pte_flags,
-					   (last_start - prange->start) << PAGE_SHIFT,
-					   bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
-					   NULL, dma_addr, &vm->last_update);
+		r = amdgpu_vm_map_range(adev, vm, flush_tlb, true, NULL,
+					gpu_start, gpu_end, pte_flags,
+					(last_start - prange->start) << PAGE_SHIFT,
+					bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
+					NULL, dma_addr, &vm->last_update);
 
 		for (j = last_start - prange->start; j <= i; j++)
 			dma_addr[j] |= last_domain;
-- 
2.43.0

[PATCH 09/13] drm/amdgpu: start to move VM internals into amdgpu_vm_internal.h

[Christian König]

Avoid exposing some defines outside of the VM code.

Co-developed by Claude Sonnet 4.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h     |   2 +
 .../gpu/drm/amd/amdgpu/amdgpu_trace_points.c  |   1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        |   1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h        | 118 -------------
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c    |   1 +
 .../gpu/drm/amd/amdgpu/amdgpu_vm_internal.h   | 159 ++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c     |   1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c   |   1 +
 8 files changed, 166 insertions(+), 118 deletions(-)
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index d13e64a69e25..d6def9bec72c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -35,6 +35,8 @@
 #define AMDGPU_JOB_GET_TIMELINE_NAME(job) \
 	 job->base.s_fence->finished.ops->get_timeline_name(&job->base.s_fence->finished)
 
+struct amdgpu_vm_update_params;
+
 TRACE_EVENT(amdgpu_device_rreg,
 	    TP_PROTO(unsigned did, uint32_t reg, uint32_t value),
 	    TP_ARGS(did, reg, value),
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
index b96d885f6e33..fe61ed5b487e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
@@ -25,6 +25,7 @@
 #include <drm/amdgpu_drm.h>
 #include "amdgpu_cs.h"
 #include "amdgpu.h"
+#include "amdgpu_vm_internal.h"
 
 #define CREATE_TRACE_POINTS
 #include "amdgpu_trace.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index b5adfcacc55a..f81ddc6873a0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -37,6 +37,7 @@
 #include <drm/drm_exec.h>
 #include "amdgpu.h"
 #include "amdgpu_vm.h"
+#include "amdgpu_vm_internal.h"
 #include "amdgpu_trace.h"
 #include "amdgpu_amdkfd.h"
 #include "amdgpu_gmc.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 561f2873d2ec..21c78b18f4df 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -261,80 +261,6 @@ struct amdgpu_task_info {
 	struct kref	refcount;
 };
 
-/**
- * struct amdgpu_vm_update_params
- *
- * Encapsulate some VM table update parameters to reduce
- * the number of function parameters
- *
- */
-struct amdgpu_vm_update_params {
-
-	/**
-	 * @adev: amdgpu device we do this update for
-	 */
-	struct amdgpu_device *adev;
-
-	/**
-	 * @vm: optional amdgpu_vm we do this update for
-	 */
-	struct amdgpu_vm *vm;
-
-	/**
-	 * @immediate: if changes should be made immediately
-	 */
-	bool immediate;
-
-	/**
-	 * @unlocked: true if the root BO is not locked
-	 */
-	bool unlocked;
-
-	/**
-	 * @pages_addr:
-	 *
-	 * DMA addresses to use for mapping
-	 */
-	dma_addr_t *pages_addr;
-
-	/**
-	 * @job: job to used for hw submission
-	 */
-	struct amdgpu_job *job;
-
-	/**
-	 * @num_dw_left: number of dw left for the IB
-	 */
-	unsigned int num_dw_left;
-
-	/**
-	 * @needs_flush: true whenever we need to invalidate the TLB
-	 */
-	bool needs_flush;
-
-	/**
-	 * @override_pte: true for memory that is not uncached and gmc override function is
-	 * implemented to allow MTYPE to be overridden for NUMA local memory.
-	 */
-	bool override_pte;
-
-	/**
-	 * @tlb_flush_waitlist: temporary storage for BOs until tlb_flush
-	 */
-	struct list_head tlb_flush_waitlist;
-};
-
-struct amdgpu_vm_update_funcs {
-	int (*map_table)(struct amdgpu_bo_vm *bo);
-	int (*prepare)(struct amdgpu_vm_update_params *p,
-		       struct amdgpu_sync *sync, u64 k_job_id);
-	int (*update)(struct amdgpu_vm_update_params *p,
-		      struct amdgpu_bo_vm *bo, uint64_t pe, uint64_t addr,
-		      unsigned count, uint32_t incr, uint64_t flags);
-	int (*commit)(struct amdgpu_vm_update_params *p,
-		      struct dma_fence **fence);
-};
-
 struct amdgpu_vm_fault_info {
 	/* fault address */
 	uint64_t	addr;
@@ -604,24 +530,6 @@ void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
 void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
 			  struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM]);
 
-int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-		       struct amdgpu_bo_vm *vmbo);
-int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			int level, struct amdgpu_bo_vm **vmbo, int32_t xcp_id);
-void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm);
-
-int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
-			 struct amdgpu_vm_bo_base *entry);
-int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
-			  uint64_t start, uint64_t end,
-			  uint64_t dst, uint64_t flags);
-void amdgpu_vm_update_leaves(struct amdgpu_vm_update_params *params,
-			     uint64_t start, uint64_t end,
-			     int64_t dst, uint64_t flags);
-void amdgpu_vm_pt_free_work(struct work_struct *work);
-void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
-			    struct amdgpu_vm_update_params *params);
-
 #if defined(CONFIG_DEBUG_FS)
 void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m);
 #endif
@@ -657,32 +565,6 @@ static inline uint64_t amdgpu_vm_tlb_seq(struct amdgpu_vm *vm)
 	return atomic64_read(&vm->tlb_seq);
 }
 
-/*
- * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS
- * happens while holding this lock anywhere to prevent deadlocks when
- * an MMU notifier runs in reclaim-FS context.
- */
-static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm)
-{
-	mutex_lock(&vm->eviction_lock);
-	vm->saved_flags = memalloc_noreclaim_save();
-}
-
-static inline bool amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm)
-{
-	if (mutex_trylock(&vm->eviction_lock)) {
-		vm->saved_flags = memalloc_noreclaim_save();
-		return true;
-	}
-	return false;
-}
-
-static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm)
-{
-	memalloc_noreclaim_restore(vm->saved_flags);
-	mutex_unlock(&vm->eviction_lock);
-}
-
 void amdgpu_vm_update_fault_cache(struct amdgpu_device *adev,
 				  unsigned int pasid,
 				  uint64_t addr,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
index b31ff6f56f0d..d03b9bbe20c7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
@@ -21,6 +21,7 @@
  */
 
 #include "amdgpu_vm.h"
+#include "amdgpu_vm_internal.h"
 #include "amdgpu.h"
 #include "amdgpu_reset.h"
 #include "amdgpu_object.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
new file mode 100644
index 000000000000..16710017e8ca
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
@@ -0,0 +1,159 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Christian König
+ */
+#ifndef __AMDGPU_VM_INTERNAL_H__
+#define __AMDGPU_VM_INTERNAL_H__
+
+#include <linux/types.h>
+#include <linux/list.h>
+#include "amdgpu_vm.h"
+
+struct amdgpu_device;
+struct amdgpu_vm;
+struct amdgpu_job;
+struct amdgpu_sync;
+struct amdgpu_bo_vm;
+struct amdgpu_vm_bo_base;
+struct dma_fence;
+
+/**
+ * struct amdgpu_vm_update_params
+ *
+ * Encapsulate some VM table update parameters to reduce
+ * the number of function parameters
+ *
+ */
+struct amdgpu_vm_update_params {
+
+	/**
+	 * @adev: amdgpu device we do this update for
+	 */
+	struct amdgpu_device *adev;
+
+	/**
+	 * @vm: optional amdgpu_vm we do this update for
+	 */
+	struct amdgpu_vm *vm;
+
+	/**
+	 * @immediate: if changes should be made immediately
+	 */
+	bool immediate;
+
+	/**
+	 * @unlocked: true if the root BO is not locked
+	 */
+	bool unlocked;
+
+	/**
+	 * @pages_addr:
+	 *
+	 * DMA addresses to use for mapping
+	 */
+	dma_addr_t *pages_addr;
+
+	/**
+	 * @job: job to used for hw submission
+	 */
+	struct amdgpu_job *job;
+
+	/**
+	 * @num_dw_left: number of dw left for the IB
+	 */
+	unsigned int num_dw_left;
+
+	/**
+	 * @needs_flush: true whenever we need to invalidate the TLB
+	 */
+	bool needs_flush;
+
+	/**
+	 * @override_pte: true for memory that is not uncached and gmc override function is
+	 * implemented to allow MTYPE to be overridden for NUMA local memory.
+	 */
+	bool override_pte;
+
+	/**
+	 * @tlb_flush_waitlist: temporary storage for BOs until tlb_flush
+	 */
+	struct list_head tlb_flush_waitlist;
+};
+
+struct amdgpu_vm_update_funcs {
+	int (*map_table)(struct amdgpu_bo_vm *bo);
+	int (*prepare)(struct amdgpu_vm_update_params *p,
+		       struct amdgpu_sync *sync, u64 k_job_id);
+	int (*update)(struct amdgpu_vm_update_params *p,
+		      struct amdgpu_bo_vm *bo, uint64_t pe, uint64_t addr,
+		      unsigned count, uint32_t incr, uint64_t flags);
+	int (*commit)(struct amdgpu_vm_update_params *p,
+		      struct dma_fence **fence);
+};
+
+int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+		       struct amdgpu_bo_vm *vmbo);
+int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+			int level, struct amdgpu_bo_vm **vmbo,
+			int32_t xcp_id);
+void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm);
+
+int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
+			 struct amdgpu_vm_bo_base *entry);
+int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
+			  uint64_t start, uint64_t end,
+			  uint64_t dst, uint64_t flags);
+void amdgpu_vm_update_leaves(struct amdgpu_vm_update_params *params,
+			     uint64_t start, uint64_t end,
+			     int64_t dst, uint64_t flags);
+void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
+			    struct amdgpu_vm_update_params *params);
+
+int amdgpu_vm_pt_map_tables(struct amdgpu_device *adev, struct amdgpu_vm *vm);
+
+/*
+ * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS
+ * happens while holding this lock anywhere to prevent deadlocks when
+ * an MMU notifier runs in reclaim-FS context.
+ */
+static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm)
+{
+	mutex_lock(&vm->eviction_lock);
+	vm->saved_flags = memalloc_noreclaim_save();
+}
+
+static inline bool amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm)
+{
+	if (mutex_trylock(&vm->eviction_lock)) {
+		vm->saved_flags = memalloc_noreclaim_save();
+		return true;
+	}
+	return false;
+}
+
+static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm)
+{
+	memalloc_noreclaim_restore(vm->saved_flags);
+	mutex_unlock(&vm->eviction_lock);
+}
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index ac3f3e31e2e2..f9c93a71e153 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -26,6 +26,7 @@
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
 #include "amdgpu_vm.h"
+#include "amdgpu_vm_internal.h"
 #include "amdgpu_job.h"
 
 /*
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
index 50cc0779c340..6acb6f08ddbf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
@@ -21,6 +21,7 @@
  */
 
 #include "amdgpu_vm.h"
+#include "amdgpu_vm_internal.h"
 #include "amdgpu_job.h"
 #include "amdgpu_object.h"
 #include "amdgpu_trace.h"
-- 
2.43.0

[PATCH 10/13] drm/amdgpu: remove unecessary parameters from trace_amdgpu_vm_update_ptes

[Christian König]

Those can be determined by the parameter structure as well.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h | 10 +++++-----
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 11 ++---------
 2 files changed, 7 insertions(+), 14 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index d6def9bec72c..4580ab533ca9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -325,9 +325,8 @@ TRACE_EVENT(amdgpu_vm_update_ptes,
 	    TP_PROTO(struct amdgpu_vm_update_params *p,
 		     uint64_t start, uint64_t end,
 		     unsigned int nptes, uint64_t dst,
-		     uint64_t incr, uint64_t flags,
-		     pid_t pid, uint64_t vm_ctx),
-	TP_ARGS(p, start, end, nptes, dst, incr, flags, pid, vm_ctx),
+		     uint64_t incr, uint64_t flags),
+	TP_ARGS(p, start, end, nptes, dst, incr, flags),
 	TP_STRUCT__entry(
 			 __field(u64, start)
 			 __field(u64, end)
@@ -347,8 +346,9 @@ TRACE_EVENT(amdgpu_vm_update_ptes,
 			__entry->flags = flags;
 			__entry->incr = incr;
 			__entry->nptes = nptes;
-			__entry->pid = pid;
-			__entry->vm_ctx = vm_ctx;
+			__entry->pid = p->vm->task_info ?
+				p->vm->task_info->tgid : 0;
+			__entry->vm_ctx = p->vm->immediate.fence_context;
 			for (i = 0; i < nptes; ++i) {
 				u64 addr = p->pages_addr ? amdgpu_vm_map_gart(
 					p->pages_addr, dst) : dst;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index f9c93a71e153..6026d6f27842 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -783,8 +783,6 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 			  uint64_t dst, uint64_t flags)
 {
 	struct amdgpu_device *adev = params->adev;
-	struct amdgpu_vm *vm = params->vm;
-	pid_t tgid = vm->task_info ? vm->task_info->tgid : 0;
 	struct amdgpu_vm_pt_cursor cursor;
 	uint64_t frag_start = start, frag_end;
 	unsigned int frag;
@@ -873,8 +871,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 
 			trace_amdgpu_vm_update_ptes(params, frag_start, upd_end,
 						    min(nptes, 32u), dst, incr,
-						    upd_flags, tgid,
-						    vm->immediate.fence_context);
+						    upd_flags);
 			amdgpu_vm_pte_update_flags(params, to_amdgpu_bo_vm(pt),
 						   cursor.level, pe_start, dst,
 						   nptes, incr, upd_flags);
@@ -937,8 +934,6 @@ void amdgpu_vm_update_leaves(struct amdgpu_vm_update_params *params,
 			     int64_t dst, uint64_t flags)
 {
 	struct amdgpu_device *adev = params->adev;
-	struct amdgpu_vm *vm = params->vm;
-	pid_t tgid = vm->task_info ? vm->task_info->tgid : 0;
 	struct amdgpu_vm_pt_cursor cursor;
 
 	amdgpu_vm_pt_start(adev, params->vm, start, &cursor);
@@ -971,9 +966,7 @@ void amdgpu_vm_update_leaves(struct amdgpu_vm_update_params *params,
 		nptes = max(nptes, 1u);
 
 		trace_amdgpu_vm_update_ptes(params, cursor.pfn, entry_end,
-					    min(nptes, 32u), dst, 0, flags,
-					    tgid,
-					    vm->immediate.fence_context);
+					    min(nptes, 32u), dst, 0, flags);
 		amdgpu_vm_pte_update_flags(params, to_amdgpu_bo_vm(pt),
 					   cursor.level - 1, pe_start, dst,
 					   nptes, 0, flags);
-- 
2.43.0

[PATCH 11/13] drm/amdgpu: nuke most amdgpu_vm_eviction_(try)lock uses

[Christian König]

Use scoped_guard/scoped_cond_guard instead.

Saving and restoring the GFP flags is not necessary in those use case.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        | 29 +++++++++----------
 .../gpu/drm/amd/amdgpu/amdgpu_vm_internal.h   |  9 ------
 2 files changed, 13 insertions(+), 25 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index f81ddc6873a0..3ac083c9f77e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -616,9 +616,8 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	 * As soon as all page tables are in place we can start updating them
 	 * again.
 	 */
-	amdgpu_vm_eviction_lock(vm);
-	vm->evicting = false;
-	amdgpu_vm_eviction_unlock(vm);
+	scoped_guard(mutex, &vm->eviction_lock)
+		vm->evicting = false;
 
 	list_for_each_entry_safe(bo_base, tmp, &vm->always_valid.evicted,
 				 vm_status) {
@@ -676,9 +675,8 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 
 	amdgpu_vm_assert_locked(vm);
 
-	amdgpu_vm_eviction_lock(vm);
-	ret = !vm->evicting;
-	amdgpu_vm_eviction_unlock(vm);
+	scoped_guard(mutex, &vm->eviction_lock)
+		ret = !vm->evicting;
 
 	ret &= list_empty(&vm->kernel.evicted);
 
@@ -2308,9 +2306,10 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev,
 bool amdgpu_vm_evictable(struct amdgpu_bo *bo)
 {
 	struct amdgpu_vm_bo_base *bo_base = bo->vm_bo;
+	struct amdgpu_vm *vm;
 
 	/* Page tables of a destroyed VM can go away immediately */
-	if (!bo_base || !bo_base->vm)
+	if (bo_base || !bo_base->vm)
 		return true;
 
 	/* Don't evict VM page tables while they are busy */
@@ -2318,17 +2317,15 @@ bool amdgpu_vm_evictable(struct amdgpu_bo *bo)
 		return false;
 
 	/* Try to block ongoing updates */
-	if (!amdgpu_vm_eviction_trylock(bo_base->vm))
-		return false;
+	vm = bo_base->vm;
+	scoped_cond_guard(mutex_try, return false, &vm->eviction_lock) {
 
-	/* Don't evict VM page tables while they are updated */
-	if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) {
-		amdgpu_vm_eviction_unlock(bo_base->vm);
-		return false;
-	}
+		/* Don't evict VM page tables while they are updated */
+		if (!dma_fence_is_signaled(vm->last_unlocked))
+			return false;
 
-	bo_base->vm->evicting = true;
-	amdgpu_vm_eviction_unlock(bo_base->vm);
+		vm->evicting = true;
+	}
 	return true;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
index 16710017e8ca..7276d987e9a6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
@@ -141,15 +141,6 @@ static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm)
 	vm->saved_flags = memalloc_noreclaim_save();
 }
 
-static inline bool amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm)
-{
-	if (mutex_trylock(&vm->eviction_lock)) {
-		vm->saved_flags = memalloc_noreclaim_save();
-		return true;
-	}
-	return false;
-}
-
 static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm)
 {
 	memalloc_noreclaim_restore(vm->saved_flags);
-- 
2.43.0

[PATCH 12/13] drm/amdgpu: rework eviction lock handling into critical section

[Christian König]

Taking the eviction lock is actually just one step which we need to do
in the critical section handling.

Rename the functions to reflect that, use the update parameters instead of the
vm to save the GFP flags.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        | 34 ++++++---------
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h        |  1 -
 .../gpu/drm/amd/amdgpu/amdgpu_vm_internal.h   | 41 ++++++++++++++-----
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c     | 28 ++++++-------
 4 files changed, 58 insertions(+), 46 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 3ac083c9f77e..d4e207db83c9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -1143,11 +1143,9 @@ int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	params.override_pte = allow_override && adev->gmc.override_pte;
 	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
 
-	amdgpu_vm_eviction_lock(vm);
-	if (vm->evicting) {
-		r = -EBUSY;
+	r = amdgpu_vm_begin_critical(&params);
+	if (r)
 		goto error_free;
-	}
 
 	if (!dma_fence_is_signaled(vm->last_unlocked)) {
 		struct dma_fence *tmp = dma_fence_get_stub();
@@ -1229,7 +1227,7 @@ int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 
 error_free:
 	kfree(tlb_cb);
-	amdgpu_vm_eviction_unlock(vm);
+	amdgpu_vm_end_critical(&params);
 	drm_dev_exit(idx);
 	return r;
 }
@@ -1277,11 +1275,9 @@ int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	params.unlocked = true;
 	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
 
-	amdgpu_vm_eviction_lock(vm);
-	if (vm->evicting) {
-		r = -EBUSY;
+	r = amdgpu_vm_begin_critical(&params);
+	if (r)
 		goto error_free;
-	}
 
 	r = vm->update_funcs->prepare(&params, sync,
 				      AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE);
@@ -1300,7 +1296,7 @@ int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 
 error_free:
 	kfree(tlb_cb);
-	amdgpu_vm_eviction_unlock(vm);
+	amdgpu_vm_end_critical(&params);
 	drm_dev_exit(idx);
 	return r;
 }
@@ -3113,30 +3109,26 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
 		goto error_unlock;
 	}
 
-	amdgpu_vm_eviction_lock(vm);
-	if (vm->evicting) {
-		r = -EBUSY;
-		goto error_dev_exit;
-	}
-
 	memset(&params, 0, sizeof(params));
 	params.adev = adev;
 	params.vm = vm;
 	params.immediate = true;
 	params.pages_addr = NULL;
 
+	r = amdgpu_vm_begin_critical(&params);
+	if (r)
+		goto error_end_critical;
+
 	r = dma_resv_reserve_fences(root->tbo.base.resv, 1);
 	if (r) {
 		pr_debug("failed %d to reserve fence slot\n", r);
-		goto error_eviction_lock;
+		goto error_end_critical;
 	}
 
 	amdgpu_vm_update_leaves(&params, addr, addr, value, flags);
 
-error_eviction_lock:
-	amdgpu_vm_eviction_unlock(vm);
-
-error_dev_exit:
+error_end_critical:
+	amdgpu_vm_end_critical(&params);
 	drm_dev_exit(idx);
 
 error_unlock:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 21c78b18f4df..7db3c3577949 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -286,7 +286,6 @@ struct amdgpu_vm {
 	 */
 	struct mutex		eviction_lock;
 	bool			evicting;
-	unsigned int		saved_flags;
 
 	/* Memory statistics for this vm, protected by stats_lock */
 	spinlock_t		stats_lock;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
index 7276d987e9a6..29c74920f3b0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
@@ -93,6 +93,11 @@ struct amdgpu_vm_update_params {
 	 */
 	bool override_pte;
 
+	/**
+	 * @saved_flags: Saved flags for GFP reduction.
+	 */
+	unsigned int saved_flags;
+
 	/**
 	 * @tlb_flush_waitlist: temporary storage for BOs until tlb_flush
 	 */
@@ -130,21 +135,37 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
 
 int amdgpu_vm_pt_map_tables(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 
-/*
- * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS
- * happens while holding this lock anywhere to prevent deadlocks when
- * an MMU notifier runs in reclaim-FS context.
+/**
+ * amdgpu_vm_begin_critical - start the critical section of the update
+ * @p: The update parameters
+ *
+ * Serialize all updates, check parameters and make sure that memory allocations
+ * don't enter the reclaim path so that we don't deadlock with MMU notifiers.
+ *
+ * Returns:
+ *
+ * 0 on success or a negative error code on failure.
+ * Even on error amdgpu_vm_end_critical() must still be called to clean up!
  */
-static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm)
+static inline int amdgpu_vm_begin_critical(struct amdgpu_vm_update_params *p)
 {
-	mutex_lock(&vm->eviction_lock);
-	vm->saved_flags = memalloc_noreclaim_save();
+	mutex_lock(&p->vm->eviction_lock);
+	p->saved_flags = memalloc_noreclaim_save();
+	if (p->vm->evicting)
+		return -EBUSY;
+	return 0;
 }
 
-static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm)
+/**
+ * amdgpu_vm_end_critical - end the critical section of the update
+ * @p: The update parameters
+ *
+ * Restore the GFP flags and drop the lock.
+ */
+static inline void amdgpu_vm_end_critical(struct amdgpu_vm_update_params *p)
 {
-	memalloc_noreclaim_restore(vm->saved_flags);
-	mutex_unlock(&vm->eviction_lock);
+	memalloc_noreclaim_restore(p->saved_flags);
+	mutex_unlock(&p->vm->eviction_lock);
 }
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index 6026d6f27842..cf1697374416 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -478,42 +478,42 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 /**
  * amdgpu_vm_pt_alloc - Allocate a specific page table
  *
- * @adev: amdgpu_device pointer
- * @vm: VM to allocate page tables for
+ * @p: see amdgpu_vm_update_params definition
  * @cursor: Which page table to allocate
  *
  * Make sure a specific page table or directory is allocated.
  *
  * Returns:
- * 1 if page table needed to be allocated, 0 if page table was already
- * allocated, negative errno if an error occurred.
+ *
+ * 0 on success or a negative error code on failure.
  */
-static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev,
-			      struct amdgpu_vm *vm,
+static int amdgpu_vm_pt_alloc(struct amdgpu_vm_update_params *p,
 			      struct amdgpu_vm_pt_cursor *cursor)
 {
 	struct amdgpu_vm_bo_base *entry = cursor->entry;
 	struct amdgpu_bo *pt_bo;
 	struct amdgpu_bo_vm *pt;
-	int r;
+	int r, r2;
 
 	if (entry->bo)
 		return 0;
 
-	amdgpu_vm_eviction_unlock(vm);
-	r = amdgpu_vm_pt_create(adev, vm, cursor->level, &pt,
-				vm->root.bo->xcp_id);
-	amdgpu_vm_eviction_lock(vm);
+	amdgpu_vm_end_critical(p);
+	r = amdgpu_vm_pt_create(p->adev, p->vm, cursor->level, &pt,
+				p->vm->root.bo->xcp_id);
+	r2 = amdgpu_vm_begin_critical(p);
 	if (r)
 		return r;
+	if (r2)
+		return r2;
 
 	/* Keep a reference to the root directory to avoid
 	 * freeing them up in the wrong order.
 	 */
 	pt_bo = &pt->bo;
 	pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo);
-	amdgpu_vm_bo_base_init(entry, vm, pt_bo);
-	r = amdgpu_vm_pt_clear(adev, vm, pt);
+	amdgpu_vm_bo_base_init(entry, p->vm, pt_bo);
+	r = amdgpu_vm_pt_clear(p->adev, p->vm, pt);
 	if (r)
 		goto error_free_pt;
 
@@ -802,7 +802,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 		/* make sure that the page tables covering the
 		 * address range are actually allocated
 		 */
-		r = amdgpu_vm_pt_alloc(params->adev, params->vm, &cursor);
+		r = amdgpu_vm_pt_alloc(params, &cursor);
 		if (r)
 			return r;
 
-- 
2.43.0

[PATCH 13/13] drm/amdgpu: fix the HMM range handling for KFD SVM

[Christian König]

It's mandatory that we have this check inside the VM handling or
otherwise page table allocation and filling PTEs doesn't work correctly.

This allows to remove the buggy SVM range lock, but that's not part of
this patch set.

Only compile tested!

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        | 10 ++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h        |  4 ++-
 .../gpu/drm/amd/amdgpu/amdgpu_vm_internal.h   | 10 ++++++
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c          | 32 ++++++++-----------
 4 files changed, 34 insertions(+), 22 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index d4e207db83c9..a368327cc025 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -1093,6 +1093,7 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
  * @vram_base: base for vram mappings
  * @res: ttm_resource to map
  * @pages_addr: DMA addresses to use for mapping
+ * @hmm_range: to check validity of DMA addresses
  * @fence: optional resulting fence
  *
  * Fill in the page table entries between @start and @last. Allocate and free
@@ -1106,7 +1107,9 @@ int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			struct amdgpu_sync *sync, uint64_t start,
 			uint64_t last, uint64_t flags, uint64_t offset,
 			uint64_t vram_base, struct ttm_resource *res,
-			dma_addr_t *pages_addr, struct dma_fence **fence)
+			dma_addr_t *pages_addr,
+			struct amdgpu_hmm_range *hmm_range,
+			struct dma_fence **fence)
 {
 	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
 	struct amdgpu_vm_update_params params;
@@ -1139,6 +1142,7 @@ int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	params.adev = adev;
 	params.vm = vm;
 	params.pages_addr = pages_addr;
+	params.hmm_range = hmm_range;
 	params.needs_flush = flush_tlb;
 	params.override_pte = allow_override && adev->gmc.override_pte;
 	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
@@ -1432,7 +1436,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
 		r = amdgpu_vm_map_range(adev, vm, flush_tlb, !uncached, &sync,
 					mapping->start, mapping->last,
 					update_flags, mapping->offset,
-					vram_base, mem, pages_addr,
+					vram_base, mem, pages_addr, NULL,
 					last_update);
 		if (r)
 			goto error_free;
@@ -1634,7 +1638,7 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
 
 		r = amdgpu_vm_map_range(adev, vm, true, false,
 					&sync, mapping->start, mapping->last,
-					0, 0, 0, NULL, NULL, &f);
+					0, 0, 0, NULL, NULL, NULL, &f);
 		amdgpu_vm_free_mapping(adev, vm, mapping, f);
 		if (r) {
 			dma_fence_put(f);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 7db3c3577949..ec1196d390bb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -459,7 +459,9 @@ int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			struct amdgpu_sync *sync, uint64_t start,
 			uint64_t last, uint64_t flags, uint64_t offset,
 			uint64_t vram_base, struct ttm_resource *res,
-			dma_addr_t *pages_addr, struct dma_fence **fence);
+			dma_addr_t *pages_addr,
+			struct amdgpu_hmm_range *hmm_range,
+			struct dma_fence **fence);
 int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			   struct amdgpu_sync *sync, uint64_t start,
 			   uint64_t last, uint64_t flags,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
index 29c74920f3b0..d47951437f0d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
@@ -26,6 +26,7 @@
 
 #include <linux/types.h>
 #include <linux/list.h>
+#include "amdgpu_hmm.h"
 #include "amdgpu_vm.h"
 
 struct amdgpu_device;
@@ -72,6 +73,13 @@ struct amdgpu_vm_update_params {
 	 */
 	dma_addr_t *pages_addr;
 
+	/**
+	 * @hmm_range:
+	 *
+	 * Used to check the validity of pages_addr.
+	 */
+	struct amdgpu_hmm_range *hmm_range;
+
 	/**
 	 * @job: job to used for hw submission
 	 */
@@ -153,6 +161,8 @@ static inline int amdgpu_vm_begin_critical(struct amdgpu_vm_update_params *p)
 	p->saved_flags = memalloc_noreclaim_save();
 	if (p->vm->evicting)
 		return -EBUSY;
+	if (p->hmm_range && !amdgpu_hmm_range_valid(p->hmm_range))
+		return -EAGAIN;
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index d0ea20dea3e1..d506afff684e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -1432,7 +1432,8 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
 static int
 svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
 		     unsigned long offset, unsigned long npages, bool readonly,
-		     dma_addr_t *dma_addr, struct amdgpu_device *bo_adev,
+		     dma_addr_t *dma_addr, struct amdgpu_hmm_range *hmm_range,
+		     struct amdgpu_device *bo_adev,
 		     struct dma_fence **fence, bool flush_tlb)
 {
 	struct amdgpu_device *adev = pdd->dev->adev;
@@ -1492,7 +1493,7 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
 					gpu_start, gpu_end, pte_flags,
 					(last_start - prange->start) << PAGE_SHIFT,
 					bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
-					NULL, dma_addr, &vm->last_update);
+					NULL, dma_addr, hmm_range, &vm->last_update);
 
 		for (j = last_start - prange->start; j <= i; j++)
 			dma_addr[j] |= last_domain;
@@ -1519,7 +1520,9 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
 }
 
 static int
-svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,
+svm_range_map_to_gpus(struct svm_range *prange,
+		      struct amdgpu_hmm_range *hmm_range,
+		      unsigned long offset,
 		      unsigned long npages, bool readonly,
 		      unsigned long *bitmap, bool wait, bool flush_tlb)
 {
@@ -1553,7 +1556,7 @@ svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,
 		}
 
 		r = svm_range_map_to_gpu(pdd, prange, offset, npages, readonly,
-					 prange->dma_addr[gpuidx],
+					 prange->dma_addr[gpuidx], hmm_range,
 					 bo_adev, wait ? &fence : NULL,
 					 flush_tlb);
 		if (r)
@@ -1823,18 +1826,6 @@ static int svm_range_validate_and_map(struct mm_struct *mm,
 
 		svm_range_lock(prange);
 
-		/* Free backing memory of hmm_range if it was initialized
-		 * Override return value to TRY AGAIN only if prior returns
-		 * were successful
-		 */
-		if (range && !amdgpu_hmm_range_valid(range) && !r) {
-			pr_debug("hmm update the range, need validate again\n");
-			r = -EAGAIN;
-		}
-
-		/* Free the hmm range */
-		amdgpu_hmm_range_free(range);
-
 		if (!r && !list_empty(&prange->child_list)) {
 			pr_debug("range split by unmap in parallel, validate again\n");
 			r = -EAGAIN;
@@ -1846,11 +1837,16 @@ static int svm_range_validate_and_map(struct mm_struct *mm,
 			if (map_start_vma <= map_last_vma) {
 				offset = map_start_vma - prange->start;
 				npages = map_last_vma - map_start_vma + 1;
-				r = svm_range_map_to_gpus(prange, offset, npages, readonly,
-							  ctx->bitmap, wait, flush_tlb);
+				r = svm_range_map_to_gpus(prange, range, offset,
+							  npages, readonly,
+							  ctx->bitmap, wait,
+							  flush_tlb);
 			}
 		}
 
+		/* Free the hmm range */
+		amdgpu_hmm_range_free(range);
+
 		if (!r && next == end)
 			prange->mapped_to_gpu = true;
 
-- 
2.43.0

Re: VM reworks

[Natalie Vock]

On 5/29/26 13:24, Christian König wrote:
> Hi everyone,
> 
> this patch set aims to fix a couple of problems we still have with the VM, page fault and HMM/SVM handling.
> 
> First I rework a bit what we have on headers/defines for the suballocator, especially that the right GFP flags are now used for GPU reset and graphics page faults is important to avoid hangs.
> 
> Then I rework the VM handling, especially splitting up the amdgpu_vm_update_range() into an amdgpu_vm_map_range() and unmap_range(). The later is only meant to be used for MMU notifier/HMM/SVM use, but I couldn't come up with a better name.
> 
> And finally it fixes the problem with KFD SVM that we can't properly allocate page tables under the SVM lock. The missing piece here is to remove the SVM lock and rely on the eviction lock of the VM for that.
> 
> Timur & Natalie I've put you on the patch set because I know you have page fault patches in the pipeline, it would be nice if you could apply this set to amd-staging-drm-next and rebase your stuff on top of it.

Thanks! I just finished drafting up a patchset for explicit sync of VM 
unmaps, I'll also rebase it onto this patchset.

I also sent the current state of the patchset to the ML earlier 
today[1]. That patchset includes a non-trivial rework to add a "context" 
struct to VM maps, in order to track changed/freed VM mappings on a 
per-operation basis. Happy for feedback on that approach - if there's 
something to rework I can roll that into the rebase right away.

Thanks,
Natalie

[1] 
https://lore.kernel.org/amd-gfx/20260529-prt-explicit-sync-v1-0-9f1c60955cab@gmx.de/

> 
> Honglei and Philip this is basically the pre-requisite work I have been talking about. Please integrate that into the new SVM patches and re-test.
> 
> I've only did some very brief testing, so something more extensional testing is clearly a good idea.
> 
> Thanks,
> Christian.
> 

Re:

[Huang, Honglei1]

On 5/29/2026 7:24 PM, Christian König wrote:
> Hi everyone,
> 
> this patch set aims to fix a couple of problems we still have with the VM, page fault and HMM/SVM handling.
> 
> First I rework a bit what we have on headers/defines for the suballocator, especially that the right GFP flags are now used for GPU reset and graphics page faults is important to avoid hangs.
> 
> Then I rework the VM handling, especially splitting up the amdgpu_vm_update_range() into an amdgpu_vm_map_range() and unmap_range(). The later is only meant to be used for MMU notifier/HMM/SVM use, but I couldn't come up with a better name.
> 
> And finally it fixes the problem with KFD SVM that we can't properly allocate page tables under the SVM lock. The missing piece here is to remove the SVM lock and rely on the eviction lock of the VM for that.
> 
> Timur & Natalie I've put you on the patch set because I know you have page fault patches in the pipeline, it would be nice if you could apply this set to amd-staging-drm-next and rebase your stuff on top of it.
> 
> Honglei and Philip this is basically the pre-requisite work I have been talking about. Please integrate that into the new SVM patches and re-test.
> 
> I've only did some very brief testing, so something more extensional testing is clearly a good idea.
> 

Got it, will integrate into new SVM and retest.

Regards,
Honglei

> Thanks,
> Christian.
> 

Re: [PATCH 08/13] drm/amdgpu: split amdgpu_vm_update_range

[Pierre-Eric Pelloux-Prayer]

Le 29/05/2026 à 13:24, Christian König a écrit :
> Split amdgpu_vm_update_range into two functions.
> 
> amdgpu_vm_map_range() is for mapping PTEs into a range and updates
> which can be done while holding the VM lock.
> 
> amdgpu_vm_unmap_range() is for unmapping PTEs without holding the VM
> lock in MMU notifiers.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_job.h   |   3 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 112 ++++++++++++++++++----
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h    |  14 ++-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c |  35 ++-----
>   drivers/gpu/drm/amd/amdkfd/kfd_svm.c      |  16 ++--
>   5 files changed, 120 insertions(+), 60 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> index 44fe40f9e8df..653ffa9ca0f3 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> @@ -47,7 +47,7 @@ enum amdgpu_ib_pool_type;
>   /* Internal kernel job ids. (decreasing values, starting from U64_MAX). */
>   #define AMDGPU_KERNEL_JOB_ID_VM_UPDATE              (18446744073709551615ULL)
>   #define AMDGPU_KERNEL_JOB_ID_VM_UPDATE_PDES         (18446744073709551614ULL)
> -#define AMDGPU_KERNEL_JOB_ID_VM_UPDATE_RANGE        (18446744073709551613ULL)
> +#define AMDGPU_KERNEL_JOB_ID_VM_MAP_RANGE           (18446744073709551613ULL)

Not reusing the same ID would make it easier for umr, but it's not a blocker so you can keep the 
code as is if you prefer.

Pierre-Eric

>   #define AMDGPU_KERNEL_JOB_ID_VM_PT_CLEAR            (18446744073709551612ULL)
>   #define AMDGPU_KERNEL_JOB_ID_TTM_MAP_BUFFER         (18446744073709551611ULL)
>   #define AMDGPU_KERNEL_JOB_ID_TTM_ACCESS_MEMORY_SDMA (18446744073709551610ULL)
> @@ -63,6 +63,7 @@ enum amdgpu_ib_pool_type;
>   #define AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST         (18446744073709551600ULL)
>   #define AMDGPU_KERNEL_JOB_ID_VPE_RING_TEST          (18446744073709551599ULL)
>   #define AMDGPU_KERNEL_JOB_ID_RUN_SHADER             (18446744073709551598ULL)
> +#define AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE         (18446744073709551597ULL)
>   
>   struct amdgpu_job {
>   	struct drm_sched_job    base;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> index edc8b1ca2d3e..b5adfcacc55a 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -1080,11 +1080,10 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
>   }
>   
>   /**
> - * amdgpu_vm_update_range - update a range in the vm page table
> + * amdgpu_vm_map_range - map something to a range in the vm page tables
>    *
>    * @adev: amdgpu_device pointer to use for commands
>    * @vm: the VM to update the range
> - * @unlocked: unlocked invalidation during MM callback
>    * @flush_tlb: trigger tlb invalidation after update completed
>    * @allow_override: change MTYPE for local NUMA nodes
>    * @sync: fences we need to sync to
> @@ -1097,23 +1096,26 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
>    * @pages_addr: DMA addresses to use for mapping
>    * @fence: optional resulting fence
>    *
> - * Fill in the page table entries between @start and @last.
> + * Fill in the page table entries between @start and @last. Allocate and free
> + * new page tables as needed. Can only be called while holding the VM lock.
>    *
>    * Returns:
>    * 0 for success, negative erro code for failure.
>    */
> -int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> -			   bool unlocked, bool flush_tlb, bool allow_override,
> -			   struct amdgpu_sync *sync, uint64_t start,
> -			   uint64_t last, uint64_t flags, uint64_t offset,
> -			   uint64_t vram_base, struct ttm_resource *res,
> -			   dma_addr_t *pages_addr, struct dma_fence **fence)
> +int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> +			bool flush_tlb, bool allow_override,
> +			struct amdgpu_sync *sync, uint64_t start,
> +			uint64_t last, uint64_t flags, uint64_t offset,
> +			uint64_t vram_base, struct ttm_resource *res,
> +			dma_addr_t *pages_addr, struct dma_fence **fence)
>   {
>   	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
>   	struct amdgpu_vm_update_params params;
>   	struct amdgpu_res_cursor cursor;
>   	int r, idx;
>   
> +	amdgpu_vm_assert_locked(vm);
> +
>   	if (!drm_dev_enter(adev_to_drm(adev), &idx))
>   		return -ENODEV;
>   
> @@ -1138,7 +1140,6 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   	params.adev = adev;
>   	params.vm = vm;
>   	params.pages_addr = pages_addr;
> -	params.unlocked = unlocked;
>   	params.needs_flush = flush_tlb;
>   	params.override_pte = allow_override && adev->gmc.override_pte;
>   	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
> @@ -1149,7 +1150,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   		goto error_free;
>   	}
>   
> -	if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) {
> +	if (!dma_fence_is_signaled(vm->last_unlocked)) {
>   		struct dma_fence *tmp = dma_fence_get_stub();
>   
>   		amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true);
> @@ -1158,7 +1159,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   	}
>   
>   	r = vm->update_funcs->prepare(&params, sync,
> -				      AMDGPU_KERNEL_JOB_ID_VM_UPDATE_RANGE);
> +				      AMDGPU_KERNEL_JOB_ID_VM_MAP_RANGE);
>   	if (r)
>   		goto error_free;
>   
> @@ -1234,6 +1235,77 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   	return r;
>   }
>   
> +/**
> + * amdgpu_vm_unmap_range - clear leave PTEs to unmap something
> + *
> + * @adev: amdgpu_device pointer to use for commands
> + * @vm: the VM to update the range
> + * @sync: fences we need to sync to
> + * @start: start of unmapped range
> + * @last: last unmapped entry
> + * @flags: flags for the entries
> + * @fence: optional resulting fence
> + *
> + * Fill in the page table entries between @start and @last with a fixed flags
> + * value without allocating or freeing page tables. Can be used without locking
> + * the VM.
> + *
> + * Returns:
> + * 0 for success, negative erro code for failure.
> + */
> +int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> +			   struct amdgpu_sync *sync, uint64_t start,
> +			   uint64_t last, uint64_t flags,
> +			   struct dma_fence **fence)
> +{
> +	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
> +	struct amdgpu_vm_update_params params;
> +	int r, idx;
> +
> +	if (!drm_dev_enter(adev_to_drm(adev), &idx))
> +		return -ENODEV;
> +
> +	tlb_cb = kmalloc(sizeof(*tlb_cb), GFP_KERNEL);
> +	if (!tlb_cb) {
> +		drm_dev_exit(idx);
> +		return -ENOMEM;
> +	}
> +
> +	memset(&params, 0, sizeof(params));
> +	params.adev = adev;
> +	params.vm = vm;
> +	params.needs_flush = true;
> +	params.unlocked = true;
> +	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
> +
> +	amdgpu_vm_eviction_lock(vm);
> +	if (vm->evicting) {
> +		r = -EBUSY;
> +		goto error_free;
> +	}
> +
> +	r = vm->update_funcs->prepare(&params, sync,
> +				      AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE);
> +	if (r)
> +		goto error_free;
> +
> +	amdgpu_vm_update_leaves(&params, start, last, 0, flags);
> +
> +	r = vm->update_funcs->commit(&params, fence);
> +	if (r)
> +		goto error_free;
> +
> +	amdgpu_vm_tlb_flush(&params, fence, tlb_cb);
> +	amdgpu_vm_pt_free_list(adev, &params);
> +	tlb_cb = NULL;
> +
> +error_free:
> +	kfree(tlb_cb);
> +	amdgpu_vm_eviction_unlock(vm);
> +	drm_dev_exit(idx);
> +	return r;
> +}
> +
>   void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
>   			  struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
>   {
> @@ -1362,11 +1434,11 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
>   
>   		trace_amdgpu_vm_bo_update(mapping);
>   
> -		r = amdgpu_vm_update_range(adev, vm, false, flush_tlb,
> -					   !uncached, &sync, mapping->start,
> -					   mapping->last, update_flags,
> -					   mapping->offset, vram_base, mem,
> -					   pages_addr, last_update);
> +		r = amdgpu_vm_map_range(adev, vm, flush_tlb, !uncached, &sync,
> +					mapping->start, mapping->last,
> +					update_flags, mapping->offset,
> +					vram_base, mem, pages_addr,
> +					last_update);
>   		if (r)
>   			goto error_free;
>   	}
> @@ -1565,9 +1637,9 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
>   			struct amdgpu_bo_va_mapping, list);
>   		list_del(&mapping->list);
>   
> -		r = amdgpu_vm_update_range(adev, vm, false, true, false,
> -					   &sync, mapping->start, mapping->last,
> -					   0, 0, 0, NULL, NULL, &f);
> +		r = amdgpu_vm_map_range(adev, vm, true, false,
> +					&sync, mapping->start, mapping->last,
> +					0, 0, 0, NULL, NULL, &f);
>   		amdgpu_vm_free_mapping(adev, vm, mapping, f);
>   		if (r) {
>   			dma_fence_put(f);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> index 3e86a2a470f0..561f2873d2ec 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> @@ -529,12 +529,16 @@ int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev,
>   				uint32_t xcc_mask);
>   void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
>   			    struct amdgpu_vm *vm, struct amdgpu_bo *bo);
> -int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> -			   bool unlocked, bool flush_tlb, bool allow_override,
> +int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> +			bool flush_tlb, bool allow_override,
> +			struct amdgpu_sync *sync, uint64_t start,
> +			uint64_t last, uint64_t flags, uint64_t offset,
> +			uint64_t vram_base, struct ttm_resource *res,
> +			dma_addr_t *pages_addr, struct dma_fence **fence);
> +int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   			   struct amdgpu_sync *sync, uint64_t start,
> -			   uint64_t last, uint64_t flags, uint64_t offset,
> -			   uint64_t vram_base, struct ttm_resource *res,
> -			   dma_addr_t *pages_addr, struct dma_fence **fence);
> +			   uint64_t last, uint64_t flags,
> +			   struct dma_fence **fence);
>   int amdgpu_vm_bo_update(struct amdgpu_device *adev,
>   			struct amdgpu_bo_va *bo_va,
>   			bool clear);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> index 6f5415d5a1bc..ac3f3e31e2e2 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> @@ -553,7 +553,6 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
>   			    struct amdgpu_vm_update_params *params)
>   {
>   	struct amdgpu_vm_bo_base *entry, *next;
> -	bool unlocked = params->unlocked;
>   
>   	if (list_empty(&params->tlb_flush_waitlist))
>   		return;
> @@ -561,7 +560,7 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
>   	/*
>   	 * unlocked unmap clear page table leaves, warning to free the page entry.
>   	 */
> -	WARN_ON(unlocked);
> +	WARN_ON(params->unlocked);
>   
>   	list_for_each_entry_safe(entry, next, &params->tlb_flush_waitlist, vm_status)
>   		amdgpu_vm_pt_free(entry);
> @@ -801,24 +800,17 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
>   		uint64_t incr, entry_end, pe_start;
>   		struct amdgpu_bo *pt;
>   
> -		if (!params->unlocked) {
> -			/* make sure that the page tables covering the
> -			 * address range are actually allocated
> -			 */
> -			r = amdgpu_vm_pt_alloc(params->adev, params->vm,
> -					       &cursor);
> -			if (r)
> -				return r;
> -		}
> +		/* make sure that the page tables covering the
> +		 * address range are actually allocated
> +		 */
> +		r = amdgpu_vm_pt_alloc(params->adev, params->vm, &cursor);
> +		if (r)
> +			return r;
>   
>   		shift = amdgpu_vm_pt_level_shift(adev, cursor.level);
>   		parent_shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1);
> -		if (params->unlocked) {
> -			/* Unlocked updates are only allowed on the leaves */
> -			if (amdgpu_vm_pt_descendant(adev, &cursor))
> -				continue;
> -		} else if (adev->asic_type < CHIP_VEGA10 &&
> -			   (flags & AMDGPU_PTE_VALID)) {
> +		if (adev->asic_type < CHIP_VEGA10 &&
> +		    (flags & AMDGPU_PTE_VALID)) {
>   			/* No huge page support before GMC v9 */
>   			if (cursor.level != AMDGPU_VM_PTB) {
>   				if (!amdgpu_vm_pt_descendant(adev, &cursor))
> @@ -864,14 +856,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
>   		mask = amdgpu_vm_pt_entries_mask(adev, cursor.level);
>   		pe_start = ((cursor.pfn >> shift) & mask) * 8;
>   
> -		if (cursor.level < AMDGPU_VM_PTB && params->unlocked)
> -			/*
> -			 * MMU notifier callback unlocked unmap huge page, leave is PDE entry,
> -			 * only clear one entry. Next entry search again for PDE or PTE leave.
> -			 */
> -			entry_end = 1ULL << shift;
> -		else
> -			entry_end = ((uint64_t)mask + 1) << shift;
> +		entry_end = ((uint64_t)mask + 1) << shift;
>   		entry_end += cursor.pfn & ~(entry_end - 1);
>   		entry_end = min(entry_end, end);
>   
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> index 37b5166e9a14..d0ea20dea3e1 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> @@ -1372,9 +1372,8 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   		return -EINVAL;
>   	}
>   
> -	return amdgpu_vm_update_range(adev, vm, true, true, false, NULL, gpu_start,
> -				      gpu_end, init_pte_value, 0, 0, NULL, NULL,
> -				      fence);
> +	return amdgpu_vm_unmap_range(adev, vm, NULL, gpu_start, gpu_end,
> +				     init_pte_value, fence);
>   }
>   
>   static int
> @@ -1489,12 +1488,11 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
>   			 (last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0,
>   			 pte_flags);
>   
> -		r = amdgpu_vm_update_range(adev, vm, false, flush_tlb, true,
> -					   NULL, gpu_start, gpu_end,
> -					   pte_flags,
> -					   (last_start - prange->start) << PAGE_SHIFT,
> -					   bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
> -					   NULL, dma_addr, &vm->last_update);
> +		r = amdgpu_vm_map_range(adev, vm, flush_tlb, true, NULL,
> +					gpu_start, gpu_end, pte_flags,
> +					(last_start - prange->start) << PAGE_SHIFT,
> +					bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
> +					NULL, dma_addr, &vm->last_update);
>   
>   		for (j = last_start - prange->start; j <= i; j++)
>   			dma_addr[j] |= last_domain;

Re: [PATCH 08/13] drm/amdgpu: split amdgpu_vm_update_range

[Christian König]

On 6/1/26 15:51, Pierre-Eric Pelloux-Prayer wrote:
> 
> 
> Le 29/05/2026 à 13:24, Christian König a écrit :
>> Split amdgpu_vm_update_range into two functions.
>>
>> amdgpu_vm_map_range() is for mapping PTEs into a range and updates
>> which can be done while holding the VM lock.
>>
>> amdgpu_vm_unmap_range() is for unmapping PTEs without holding the VM
>> lock in MMU notifiers.
>>
>> Signed-off-by: Christian König <christian.koenig@amd.com>
>> ---
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_job.h   |   3 +-
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 112 ++++++++++++++++++----
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h    |  14 ++-
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c |  35 ++-----
>>   drivers/gpu/drm/amd/amdkfd/kfd_svm.c      |  16 ++--
>>   5 files changed, 120 insertions(+), 60 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
>> index 44fe40f9e8df..653ffa9ca0f3 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
>> @@ -47,7 +47,7 @@ enum amdgpu_ib_pool_type;
>>   /* Internal kernel job ids. (decreasing values, starting from U64_MAX). */
>>   #define AMDGPU_KERNEL_JOB_ID_VM_UPDATE              (18446744073709551615ULL)
>>   #define AMDGPU_KERNEL_JOB_ID_VM_UPDATE_PDES         (18446744073709551614ULL)
>> -#define AMDGPU_KERNEL_JOB_ID_VM_UPDATE_RANGE        (18446744073709551613ULL)
>> +#define AMDGPU_KERNEL_JOB_ID_VM_MAP_RANGE           (18446744073709551613ULL)
> 
> Not reusing the same ID would make it easier for umr, but it's not a blocker so you can keep the code as is if you prefer.

The update_range is replaced by the map_range in almost all cases. Only the unmap_range from an MMU notifier is specially handled now.

The term "map_range" is also kind of misleading since we also use it for unmapping during regular BO free, but I couldn't come up with a better name.

Maybe just keep the name update_range like it is and add unmap_range as special case? Idk.

Christian.

> 
> Pierre-Eric
> 
>>   #define AMDGPU_KERNEL_JOB_ID_VM_PT_CLEAR            (18446744073709551612ULL)
>>   #define AMDGPU_KERNEL_JOB_ID_TTM_MAP_BUFFER         (18446744073709551611ULL)
>>   #define AMDGPU_KERNEL_JOB_ID_TTM_ACCESS_MEMORY_SDMA (18446744073709551610ULL)
>> @@ -63,6 +63,7 @@ enum amdgpu_ib_pool_type;
>>   #define AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST         (18446744073709551600ULL)
>>   #define AMDGPU_KERNEL_JOB_ID_VPE_RING_TEST          (18446744073709551599ULL)
>>   #define AMDGPU_KERNEL_JOB_ID_RUN_SHADER             (18446744073709551598ULL)
>> +#define AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE         (18446744073709551597ULL)
>>     struct amdgpu_job {
>>       struct drm_sched_job    base;
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
>> index edc8b1ca2d3e..b5adfcacc55a 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
>> @@ -1080,11 +1080,10 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
>>   }
>>     /**
>> - * amdgpu_vm_update_range - update a range in the vm page table
>> + * amdgpu_vm_map_range - map something to a range in the vm page tables
>>    *
>>    * @adev: amdgpu_device pointer to use for commands
>>    * @vm: the VM to update the range
>> - * @unlocked: unlocked invalidation during MM callback
>>    * @flush_tlb: trigger tlb invalidation after update completed
>>    * @allow_override: change MTYPE for local NUMA nodes
>>    * @sync: fences we need to sync to
>> @@ -1097,23 +1096,26 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
>>    * @pages_addr: DMA addresses to use for mapping
>>    * @fence: optional resulting fence
>>    *
>> - * Fill in the page table entries between @start and @last.
>> + * Fill in the page table entries between @start and @last. Allocate and free
>> + * new page tables as needed. Can only be called while holding the VM lock.
>>    *
>>    * Returns:
>>    * 0 for success, negative erro code for failure.
>>    */
>> -int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>> -               bool unlocked, bool flush_tlb, bool allow_override,
>> -               struct amdgpu_sync *sync, uint64_t start,
>> -               uint64_t last, uint64_t flags, uint64_t offset,
>> -               uint64_t vram_base, struct ttm_resource *res,
>> -               dma_addr_t *pages_addr, struct dma_fence **fence)
>> +int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>> +            bool flush_tlb, bool allow_override,
>> +            struct amdgpu_sync *sync, uint64_t start,
>> +            uint64_t last, uint64_t flags, uint64_t offset,
>> +            uint64_t vram_base, struct ttm_resource *res,
>> +            dma_addr_t *pages_addr, struct dma_fence **fence)
>>   {
>>       struct amdgpu_vm_tlb_seq_struct *tlb_cb;
>>       struct amdgpu_vm_update_params params;
>>       struct amdgpu_res_cursor cursor;
>>       int r, idx;
>>   +    amdgpu_vm_assert_locked(vm);
>> +
>>       if (!drm_dev_enter(adev_to_drm(adev), &idx))
>>           return -ENODEV;
>>   @@ -1138,7 +1140,6 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>>       params.adev = adev;
>>       params.vm = vm;
>>       params.pages_addr = pages_addr;
>> -    params.unlocked = unlocked;
>>       params.needs_flush = flush_tlb;
>>       params.override_pte = allow_override && adev->gmc.override_pte;
>>       INIT_LIST_HEAD(&params.tlb_flush_waitlist);
>> @@ -1149,7 +1150,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>>           goto error_free;
>>       }
>>   -    if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) {
>> +    if (!dma_fence_is_signaled(vm->last_unlocked)) {
>>           struct dma_fence *tmp = dma_fence_get_stub();
>>             amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true);
>> @@ -1158,7 +1159,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>>       }
>>         r = vm->update_funcs->prepare(&params, sync,
>> -                      AMDGPU_KERNEL_JOB_ID_VM_UPDATE_RANGE);
>> +                      AMDGPU_KERNEL_JOB_ID_VM_MAP_RANGE);
>>       if (r)
>>           goto error_free;
>>   @@ -1234,6 +1235,77 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>>       return r;
>>   }
>>   +/**
>> + * amdgpu_vm_unmap_range - clear leave PTEs to unmap something
>> + *
>> + * @adev: amdgpu_device pointer to use for commands
>> + * @vm: the VM to update the range
>> + * @sync: fences we need to sync to
>> + * @start: start of unmapped range
>> + * @last: last unmapped entry
>> + * @flags: flags for the entries
>> + * @fence: optional resulting fence
>> + *
>> + * Fill in the page table entries between @start and @last with a fixed flags
>> + * value without allocating or freeing page tables. Can be used without locking
>> + * the VM.
>> + *
>> + * Returns:
>> + * 0 for success, negative erro code for failure.
>> + */
>> +int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>> +               struct amdgpu_sync *sync, uint64_t start,
>> +               uint64_t last, uint64_t flags,
>> +               struct dma_fence **fence)
>> +{
>> +    struct amdgpu_vm_tlb_seq_struct *tlb_cb;
>> +    struct amdgpu_vm_update_params params;
>> +    int r, idx;
>> +
>> +    if (!drm_dev_enter(adev_to_drm(adev), &idx))
>> +        return -ENODEV;
>> +
>> +    tlb_cb = kmalloc(sizeof(*tlb_cb), GFP_KERNEL);
>> +    if (!tlb_cb) {
>> +        drm_dev_exit(idx);
>> +        return -ENOMEM;
>> +    }
>> +
>> +    memset(&params, 0, sizeof(params));
>> +    params.adev = adev;
>> +    params.vm = vm;
>> +    params.needs_flush = true;
>> +    params.unlocked = true;
>> +    INIT_LIST_HEAD(&params.tlb_flush_waitlist);
>> +
>> +    amdgpu_vm_eviction_lock(vm);
>> +    if (vm->evicting) {
>> +        r = -EBUSY;
>> +        goto error_free;
>> +    }
>> +
>> +    r = vm->update_funcs->prepare(&params, sync,
>> +                      AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE);
>> +    if (r)
>> +        goto error_free;
>> +
>> +    amdgpu_vm_update_leaves(&params, start, last, 0, flags);
>> +
>> +    r = vm->update_funcs->commit(&params, fence);
>> +    if (r)
>> +        goto error_free;
>> +
>> +    amdgpu_vm_tlb_flush(&params, fence, tlb_cb);
>> +    amdgpu_vm_pt_free_list(adev, &params);
>> +    tlb_cb = NULL;
>> +
>> +error_free:
>> +    kfree(tlb_cb);
>> +    amdgpu_vm_eviction_unlock(vm);
>> +    drm_dev_exit(idx);
>> +    return r;
>> +}
>> +
>>   void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
>>                 struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
>>   {
>> @@ -1362,11 +1434,11 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
>>             trace_amdgpu_vm_bo_update(mapping);
>>   -        r = amdgpu_vm_update_range(adev, vm, false, flush_tlb,
>> -                       !uncached, &sync, mapping->start,
>> -                       mapping->last, update_flags,
>> -                       mapping->offset, vram_base, mem,
>> -                       pages_addr, last_update);
>> +        r = amdgpu_vm_map_range(adev, vm, flush_tlb, !uncached, &sync,
>> +                    mapping->start, mapping->last,
>> +                    update_flags, mapping->offset,
>> +                    vram_base, mem, pages_addr,
>> +                    last_update);
>>           if (r)
>>               goto error_free;
>>       }
>> @@ -1565,9 +1637,9 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
>>               struct amdgpu_bo_va_mapping, list);
>>           list_del(&mapping->list);
>>   -        r = amdgpu_vm_update_range(adev, vm, false, true, false,
>> -                       &sync, mapping->start, mapping->last,
>> -                       0, 0, 0, NULL, NULL, &f);
>> +        r = amdgpu_vm_map_range(adev, vm, true, false,
>> +                    &sync, mapping->start, mapping->last,
>> +                    0, 0, 0, NULL, NULL, &f);
>>           amdgpu_vm_free_mapping(adev, vm, mapping, f);
>>           if (r) {
>>               dma_fence_put(f);
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
>> index 3e86a2a470f0..561f2873d2ec 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
>> @@ -529,12 +529,16 @@ int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev,
>>                   uint32_t xcc_mask);
>>   void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
>>                   struct amdgpu_vm *vm, struct amdgpu_bo *bo);
>> -int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>> -               bool unlocked, bool flush_tlb, bool allow_override,
>> +int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>> +            bool flush_tlb, bool allow_override,
>> +            struct amdgpu_sync *sync, uint64_t start,
>> +            uint64_t last, uint64_t flags, uint64_t offset,
>> +            uint64_t vram_base, struct ttm_resource *res,
>> +            dma_addr_t *pages_addr, struct dma_fence **fence);
>> +int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>>                  struct amdgpu_sync *sync, uint64_t start,
>> -               uint64_t last, uint64_t flags, uint64_t offset,
>> -               uint64_t vram_base, struct ttm_resource *res,
>> -               dma_addr_t *pages_addr, struct dma_fence **fence);
>> +               uint64_t last, uint64_t flags,
>> +               struct dma_fence **fence);
>>   int amdgpu_vm_bo_update(struct amdgpu_device *adev,
>>               struct amdgpu_bo_va *bo_va,
>>               bool clear);
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
>> index 6f5415d5a1bc..ac3f3e31e2e2 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
>> @@ -553,7 +553,6 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
>>                   struct amdgpu_vm_update_params *params)
>>   {
>>       struct amdgpu_vm_bo_base *entry, *next;
>> -    bool unlocked = params->unlocked;
>>         if (list_empty(&params->tlb_flush_waitlist))
>>           return;
>> @@ -561,7 +560,7 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
>>       /*
>>        * unlocked unmap clear page table leaves, warning to free the page entry.
>>        */
>> -    WARN_ON(unlocked);
>> +    WARN_ON(params->unlocked);
>>         list_for_each_entry_safe(entry, next, &params->tlb_flush_waitlist, vm_status)
>>           amdgpu_vm_pt_free(entry);
>> @@ -801,24 +800,17 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
>>           uint64_t incr, entry_end, pe_start;
>>           struct amdgpu_bo *pt;
>>   -        if (!params->unlocked) {
>> -            /* make sure that the page tables covering the
>> -             * address range are actually allocated
>> -             */
>> -            r = amdgpu_vm_pt_alloc(params->adev, params->vm,
>> -                           &cursor);
>> -            if (r)
>> -                return r;
>> -        }
>> +        /* make sure that the page tables covering the
>> +         * address range are actually allocated
>> +         */
>> +        r = amdgpu_vm_pt_alloc(params->adev, params->vm, &cursor);
>> +        if (r)
>> +            return r;
>>             shift = amdgpu_vm_pt_level_shift(adev, cursor.level);
>>           parent_shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1);
>> -        if (params->unlocked) {
>> -            /* Unlocked updates are only allowed on the leaves */
>> -            if (amdgpu_vm_pt_descendant(adev, &cursor))
>> -                continue;
>> -        } else if (adev->asic_type < CHIP_VEGA10 &&
>> -               (flags & AMDGPU_PTE_VALID)) {
>> +        if (adev->asic_type < CHIP_VEGA10 &&
>> +            (flags & AMDGPU_PTE_VALID)) {
>>               /* No huge page support before GMC v9 */
>>               if (cursor.level != AMDGPU_VM_PTB) {
>>                   if (!amdgpu_vm_pt_descendant(adev, &cursor))
>> @@ -864,14 +856,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
>>           mask = amdgpu_vm_pt_entries_mask(adev, cursor.level);
>>           pe_start = ((cursor.pfn >> shift) & mask) * 8;
>>   -        if (cursor.level < AMDGPU_VM_PTB && params->unlocked)
>> -            /*
>> -             * MMU notifier callback unlocked unmap huge page, leave is PDE entry,
>> -             * only clear one entry. Next entry search again for PDE or PTE leave.
>> -             */
>> -            entry_end = 1ULL << shift;
>> -        else
>> -            entry_end = ((uint64_t)mask + 1) << shift;
>> +        entry_end = ((uint64_t)mask + 1) << shift;
>>           entry_end += cursor.pfn & ~(entry_end - 1);
>>           entry_end = min(entry_end, end);
>>   diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
>> index 37b5166e9a14..d0ea20dea3e1 100644
>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
>> @@ -1372,9 +1372,8 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>>           return -EINVAL;
>>       }
>>   -    return amdgpu_vm_update_range(adev, vm, true, true, false, NULL, gpu_start,
>> -                      gpu_end, init_pte_value, 0, 0, NULL, NULL,
>> -                      fence);
>> +    return amdgpu_vm_unmap_range(adev, vm, NULL, gpu_start, gpu_end,
>> +                     init_pte_value, fence);
>>   }
>>     static int
>> @@ -1489,12 +1488,11 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
>>                (last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0,
>>                pte_flags);
>>   -        r = amdgpu_vm_update_range(adev, vm, false, flush_tlb, true,
>> -                       NULL, gpu_start, gpu_end,
>> -                       pte_flags,
>> -                       (last_start - prange->start) << PAGE_SHIFT,
>> -                       bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
>> -                       NULL, dma_addr, &vm->last_update);
>> +        r = amdgpu_vm_map_range(adev, vm, flush_tlb, true, NULL,
>> +                    gpu_start, gpu_end, pte_flags,
>> +                    (last_start - prange->start) << PAGE_SHIFT,
>> +                    bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
>> +                    NULL, dma_addr, &vm->last_update);
>>             for (j = last_start - prange->start; j <= i; j++)
>>               dma_addr[j] |= last_domain;

Re: [PATCH 08/13] drm/amdgpu: split amdgpu_vm_update_range

[Kuehling, Felix]

On 2026-05-29 07:24, Christian König wrote:
> Split amdgpu_vm_update_range into two functions.
>
> amdgpu_vm_map_range() is for mapping PTEs into a range and updates
> which can be done while holding the VM lock.
>
> amdgpu_vm_unmap_range() is for unmapping PTEs without holding the VM
> lock in MMU notifiers.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_job.h   |   3 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 112 ++++++++++++++++++----
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h    |  14 ++-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c |  35 ++-----
>   drivers/gpu/drm/amd/amdkfd/kfd_svm.c      |  16 ++--
>   5 files changed, 120 insertions(+), 60 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> index 44fe40f9e8df..653ffa9ca0f3 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> @@ -47,7 +47,7 @@ enum amdgpu_ib_pool_type;
>   /* Internal kernel job ids. (decreasing values, starting from U64_MAX). */
>   #define AMDGPU_KERNEL_JOB_ID_VM_UPDATE              (18446744073709551615ULL)
>   #define AMDGPU_KERNEL_JOB_ID_VM_UPDATE_PDES         (18446744073709551614ULL)
> -#define AMDGPU_KERNEL_JOB_ID_VM_UPDATE_RANGE        (18446744073709551613ULL)
> +#define AMDGPU_KERNEL_JOB_ID_VM_MAP_RANGE           (18446744073709551613ULL)
>   #define AMDGPU_KERNEL_JOB_ID_VM_PT_CLEAR            (18446744073709551612ULL)
>   #define AMDGPU_KERNEL_JOB_ID_TTM_MAP_BUFFER         (18446744073709551611ULL)
>   #define AMDGPU_KERNEL_JOB_ID_TTM_ACCESS_MEMORY_SDMA (18446744073709551610ULL)
> @@ -63,6 +63,7 @@ enum amdgpu_ib_pool_type;
>   #define AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST         (18446744073709551600ULL)
>   #define AMDGPU_KERNEL_JOB_ID_VPE_RING_TEST          (18446744073709551599ULL)
>   #define AMDGPU_KERNEL_JOB_ID_RUN_SHADER             (18446744073709551598ULL)
> +#define AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE         (18446744073709551597ULL)
>   
>   struct amdgpu_job {
>   	struct drm_sched_job    base;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> index edc8b1ca2d3e..b5adfcacc55a 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -1080,11 +1080,10 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
>   }
>   
>   /**
> - * amdgpu_vm_update_range - update a range in the vm page table
> + * amdgpu_vm_map_range - map something to a range in the vm page tables
>    *
>    * @adev: amdgpu_device pointer to use for commands
>    * @vm: the VM to update the range
> - * @unlocked: unlocked invalidation during MM callback
>    * @flush_tlb: trigger tlb invalidation after update completed
>    * @allow_override: change MTYPE for local NUMA nodes
>    * @sync: fences we need to sync to
> @@ -1097,23 +1096,26 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
>    * @pages_addr: DMA addresses to use for mapping
>    * @fence: optional resulting fence
>    *
> - * Fill in the page table entries between @start and @last.
> + * Fill in the page table entries between @start and @last. Allocate and free
> + * new page tables as needed. Can only be called while holding the VM lock.
>    *
>    * Returns:
>    * 0 for success, negative erro code for failure.
>    */
> -int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> -			   bool unlocked, bool flush_tlb, bool allow_override,
> -			   struct amdgpu_sync *sync, uint64_t start,
> -			   uint64_t last, uint64_t flags, uint64_t offset,
> -			   uint64_t vram_base, struct ttm_resource *res,
> -			   dma_addr_t *pages_addr, struct dma_fence **fence)
> +int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> +			bool flush_tlb, bool allow_override,
> +			struct amdgpu_sync *sync, uint64_t start,
> +			uint64_t last, uint64_t flags, uint64_t offset,
> +			uint64_t vram_base, struct ttm_resource *res,
> +			dma_addr_t *pages_addr, struct dma_fence **fence)
>   {
>   	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
>   	struct amdgpu_vm_update_params params;
>   	struct amdgpu_res_cursor cursor;
>   	int r, idx;
>   
> +	amdgpu_vm_assert_locked(vm);
> +
>   	if (!drm_dev_enter(adev_to_drm(adev), &idx))
>   		return -ENODEV;
>   
> @@ -1138,7 +1140,6 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   	params.adev = adev;
>   	params.vm = vm;
>   	params.pages_addr = pages_addr;
> -	params.unlocked = unlocked;
>   	params.needs_flush = flush_tlb;
>   	params.override_pte = allow_override && adev->gmc.override_pte;
>   	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
> @@ -1149,7 +1150,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   		goto error_free;
>   	}
>   
> -	if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) {
> +	if (!dma_fence_is_signaled(vm->last_unlocked)) {
>   		struct dma_fence *tmp = dma_fence_get_stub();
>   
>   		amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true);
> @@ -1158,7 +1159,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   	}
>   
>   	r = vm->update_funcs->prepare(&params, sync,
> -				      AMDGPU_KERNEL_JOB_ID_VM_UPDATE_RANGE);
> +				      AMDGPU_KERNEL_JOB_ID_VM_MAP_RANGE);
>   	if (r)
>   		goto error_free;
>   
> @@ -1234,6 +1235,77 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   	return r;
>   }
>   
> +/**
> + * amdgpu_vm_unmap_range - clear leave PTEs to unmap something
> + *
> + * @adev: amdgpu_device pointer to use for commands
> + * @vm: the VM to update the range
> + * @sync: fences we need to sync to
> + * @start: start of unmapped range
> + * @last: last unmapped entry
> + * @flags: flags for the entries
> + * @fence: optional resulting fence
> + *
> + * Fill in the page table entries between @start and @last with a fixed flags
> + * value without allocating or freeing page tables. Can be used without locking
> + * the VM.
> + *
> + * Returns:
> + * 0 for success, negative erro code for failure.
> + */
> +int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> +			   struct amdgpu_sync *sync, uint64_t start,
> +			   uint64_t last, uint64_t flags,
> +			   struct dma_fence **fence)
> +{
> +	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
> +	struct amdgpu_vm_update_params params;
> +	int r, idx;
> +
> +	if (!drm_dev_enter(adev_to_drm(adev), &idx))
> +		return -ENODEV;
> +
> +	tlb_cb = kmalloc(sizeof(*tlb_cb), GFP_KERNEL);
> +	if (!tlb_cb) {
> +		drm_dev_exit(idx);
> +		return -ENOMEM;
> +	}
> +
> +	memset(&params, 0, sizeof(params));
> +	params.adev = adev;
> +	params.vm = vm;
> +	params.needs_flush = true;
> +	params.unlocked = true;
> +	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
> +
> +	amdgpu_vm_eviction_lock(vm);
> +	if (vm->evicting) {
> +		r = -EBUSY;
> +		goto error_free;
> +	}
> +
> +	r = vm->update_funcs->prepare(&params, sync,
> +				      AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE);
> +	if (r)
> +		goto error_free;
> +
> +	amdgpu_vm_update_leaves(&params, start, last, 0, flags);
> +
> +	r = vm->update_funcs->commit(&params, fence);
> +	if (r)
> +		goto error_free;

So this wraps amdgpu_vm_update_leaves with all the stuff that's 
necessary to actually execute the page table update. I don't understand 
how amdgpu_vm_update_leaves works without this when you call it directly 
from amdgpu_vm_handle_fault (in patch 6). Shouldn't you use 
amdgpu_vm_unmap_range there instead?

And if that's true, then maybe you can use the name 
amdgpu_vm_update_leaves for this instead.

Regards,
   Felix


> +
> +	amdgpu_vm_tlb_flush(&params, fence, tlb_cb);
> +	amdgpu_vm_pt_free_list(adev, &params);
> +	tlb_cb = NULL;
> +
> +error_free:
> +	kfree(tlb_cb);
> +	amdgpu_vm_eviction_unlock(vm);
> +	drm_dev_exit(idx);
> +	return r;
> +}
> +
>   void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
>   			  struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
>   {
> @@ -1362,11 +1434,11 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
>   
>   		trace_amdgpu_vm_bo_update(mapping);
>   
> -		r = amdgpu_vm_update_range(adev, vm, false, flush_tlb,
> -					   !uncached, &sync, mapping->start,
> -					   mapping->last, update_flags,
> -					   mapping->offset, vram_base, mem,
> -					   pages_addr, last_update);
> +		r = amdgpu_vm_map_range(adev, vm, flush_tlb, !uncached, &sync,
> +					mapping->start, mapping->last,
> +					update_flags, mapping->offset,
> +					vram_base, mem, pages_addr,
> +					last_update);
>   		if (r)
>   			goto error_free;
>   	}
> @@ -1565,9 +1637,9 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
>   			struct amdgpu_bo_va_mapping, list);
>   		list_del(&mapping->list);
>   
> -		r = amdgpu_vm_update_range(adev, vm, false, true, false,
> -					   &sync, mapping->start, mapping->last,
> -					   0, 0, 0, NULL, NULL, &f);
> +		r = amdgpu_vm_map_range(adev, vm, true, false,
> +					&sync, mapping->start, mapping->last,
> +					0, 0, 0, NULL, NULL, &f);
>   		amdgpu_vm_free_mapping(adev, vm, mapping, f);
>   		if (r) {
>   			dma_fence_put(f);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> index 3e86a2a470f0..561f2873d2ec 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> @@ -529,12 +529,16 @@ int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev,
>   				uint32_t xcc_mask);
>   void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
>   			    struct amdgpu_vm *vm, struct amdgpu_bo *bo);
> -int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> -			   bool unlocked, bool flush_tlb, bool allow_override,
> +int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> +			bool flush_tlb, bool allow_override,
> +			struct amdgpu_sync *sync, uint64_t start,
> +			uint64_t last, uint64_t flags, uint64_t offset,
> +			uint64_t vram_base, struct ttm_resource *res,
> +			dma_addr_t *pages_addr, struct dma_fence **fence);
> +int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   			   struct amdgpu_sync *sync, uint64_t start,
> -			   uint64_t last, uint64_t flags, uint64_t offset,
> -			   uint64_t vram_base, struct ttm_resource *res,
> -			   dma_addr_t *pages_addr, struct dma_fence **fence);
> +			   uint64_t last, uint64_t flags,
> +			   struct dma_fence **fence);
>   int amdgpu_vm_bo_update(struct amdgpu_device *adev,
>   			struct amdgpu_bo_va *bo_va,
>   			bool clear);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> index 6f5415d5a1bc..ac3f3e31e2e2 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> @@ -553,7 +553,6 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
>   			    struct amdgpu_vm_update_params *params)
>   {
>   	struct amdgpu_vm_bo_base *entry, *next;
> -	bool unlocked = params->unlocked;
>   
>   	if (list_empty(&params->tlb_flush_waitlist))
>   		return;
> @@ -561,7 +560,7 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
>   	/*
>   	 * unlocked unmap clear page table leaves, warning to free the page entry.
>   	 */
> -	WARN_ON(unlocked);
> +	WARN_ON(params->unlocked);
>   
>   	list_for_each_entry_safe(entry, next, &params->tlb_flush_waitlist, vm_status)
>   		amdgpu_vm_pt_free(entry);
> @@ -801,24 +800,17 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
>   		uint64_t incr, entry_end, pe_start;
>   		struct amdgpu_bo *pt;
>   
> -		if (!params->unlocked) {
> -			/* make sure that the page tables covering the
> -			 * address range are actually allocated
> -			 */
> -			r = amdgpu_vm_pt_alloc(params->adev, params->vm,
> -					       &cursor);
> -			if (r)
> -				return r;
> -		}
> +		/* make sure that the page tables covering the
> +		 * address range are actually allocated
> +		 */
> +		r = amdgpu_vm_pt_alloc(params->adev, params->vm, &cursor);
> +		if (r)
> +			return r;
>   
>   		shift = amdgpu_vm_pt_level_shift(adev, cursor.level);
>   		parent_shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1);
> -		if (params->unlocked) {
> -			/* Unlocked updates are only allowed on the leaves */
> -			if (amdgpu_vm_pt_descendant(adev, &cursor))
> -				continue;
> -		} else if (adev->asic_type < CHIP_VEGA10 &&
> -			   (flags & AMDGPU_PTE_VALID)) {
> +		if (adev->asic_type < CHIP_VEGA10 &&
> +		    (flags & AMDGPU_PTE_VALID)) {
>   			/* No huge page support before GMC v9 */
>   			if (cursor.level != AMDGPU_VM_PTB) {
>   				if (!amdgpu_vm_pt_descendant(adev, &cursor))
> @@ -864,14 +856,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
>   		mask = amdgpu_vm_pt_entries_mask(adev, cursor.level);
>   		pe_start = ((cursor.pfn >> shift) & mask) * 8;
>   
> -		if (cursor.level < AMDGPU_VM_PTB && params->unlocked)
> -			/*
> -			 * MMU notifier callback unlocked unmap huge page, leave is PDE entry,
> -			 * only clear one entry. Next entry search again for PDE or PTE leave.
> -			 */
> -			entry_end = 1ULL << shift;
> -		else
> -			entry_end = ((uint64_t)mask + 1) << shift;
> +		entry_end = ((uint64_t)mask + 1) << shift;
>   		entry_end += cursor.pfn & ~(entry_end - 1);
>   		entry_end = min(entry_end, end);
>   
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> index 37b5166e9a14..d0ea20dea3e1 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> @@ -1372,9 +1372,8 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   		return -EINVAL;
>   	}
>   
> -	return amdgpu_vm_update_range(adev, vm, true, true, false, NULL, gpu_start,
> -				      gpu_end, init_pte_value, 0, 0, NULL, NULL,
> -				      fence);
> +	return amdgpu_vm_unmap_range(adev, vm, NULL, gpu_start, gpu_end,
> +				     init_pte_value, fence);
>   }
>   
>   static int
> @@ -1489,12 +1488,11 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
>   			 (last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0,
>   			 pte_flags);
>   
> -		r = amdgpu_vm_update_range(adev, vm, false, flush_tlb, true,
> -					   NULL, gpu_start, gpu_end,
> -					   pte_flags,
> -					   (last_start - prange->start) << PAGE_SHIFT,
> -					   bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
> -					   NULL, dma_addr, &vm->last_update);
> +		r = amdgpu_vm_map_range(adev, vm, flush_tlb, true, NULL,
> +					gpu_start, gpu_end, pte_flags,
> +					(last_start - prange->start) << PAGE_SHIFT,
> +					bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
> +					NULL, dma_addr, &vm->last_update);
>   
>   		for (j = last_start - prange->start; j <= i; j++)
>   			dma_addr[j] |= last_domain;

Re: [PATCH 11/13] drm/amdgpu: nuke most amdgpu_vm_eviction_(try)lock uses

[Kuehling, Felix]

On 2026-05-29 07:24, Christian König wrote:
> Use scoped_guard/scoped_cond_guard instead.
>
> Saving and restoring the GFP flags is not necessary in those use case.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        | 29 +++++++++----------
>   .../gpu/drm/amd/amdgpu/amdgpu_vm_internal.h   |  9 ------
>   2 files changed, 13 insertions(+), 25 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> index f81ddc6873a0..3ac083c9f77e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -616,9 +616,8 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   	 * As soon as all page tables are in place we can start updating them
>   	 * again.
>   	 */
> -	amdgpu_vm_eviction_lock(vm);
> -	vm->evicting = false;
> -	amdgpu_vm_eviction_unlock(vm);
> +	scoped_guard(mutex, &vm->eviction_lock)
> +		vm->evicting = false;
>   
>   	list_for_each_entry_safe(bo_base, tmp, &vm->always_valid.evicted,
>   				 vm_status) {
> @@ -676,9 +675,8 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
>   
>   	amdgpu_vm_assert_locked(vm);
>   
> -	amdgpu_vm_eviction_lock(vm);
> -	ret = !vm->evicting;
> -	amdgpu_vm_eviction_unlock(vm);
> +	scoped_guard(mutex, &vm->eviction_lock)
> +		ret = !vm->evicting;
>   
>   	ret &= list_empty(&vm->kernel.evicted);
>   
> @@ -2308,9 +2306,10 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev,
>   bool amdgpu_vm_evictable(struct amdgpu_bo *bo)
>   {
>   	struct amdgpu_vm_bo_base *bo_base = bo->vm_bo;
> +	struct amdgpu_vm *vm;
>   
>   	/* Page tables of a destroyed VM can go away immediately */
> -	if (!bo_base || !bo_base->vm)
> +	if (bo_base || !bo_base->vm)

This is either a mistake or a deliberate logic fix that would justify a 
separate patch.

Regards,
   Felix


>   		return true;
>   
>   	/* Don't evict VM page tables while they are busy */
> @@ -2318,17 +2317,15 @@ bool amdgpu_vm_evictable(struct amdgpu_bo *bo)
>   		return false;
>   
>   	/* Try to block ongoing updates */
> -	if (!amdgpu_vm_eviction_trylock(bo_base->vm))
> -		return false;
> +	vm = bo_base->vm;
> +	scoped_cond_guard(mutex_try, return false, &vm->eviction_lock) {
>   
> -	/* Don't evict VM page tables while they are updated */
> -	if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) {
> -		amdgpu_vm_eviction_unlock(bo_base->vm);
> -		return false;
> -	}
> +		/* Don't evict VM page tables while they are updated */
> +		if (!dma_fence_is_signaled(vm->last_unlocked))
> +			return false;
>   
> -	bo_base->vm->evicting = true;
> -	amdgpu_vm_eviction_unlock(bo_base->vm);
> +		vm->evicting = true;
> +	}
>   	return true;
>   }
>   
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
> index 16710017e8ca..7276d987e9a6 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
> @@ -141,15 +141,6 @@ static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm)
>   	vm->saved_flags = memalloc_noreclaim_save();
>   }
>   
> -static inline bool amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm)
> -{
> -	if (mutex_trylock(&vm->eviction_lock)) {
> -		vm->saved_flags = memalloc_noreclaim_save();
> -		return true;
> -	}
> -	return false;
> -}
> -
>   static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm)
>   {
>   	memalloc_noreclaim_restore(vm->saved_flags);

Re: [PATCH 13/13] drm/amdgpu: fix the HMM range handling for KFD SVM

[Kuehling, Felix]

On 2026-05-29 07:24, Christian König wrote:
> It's mandatory that we have this check inside the VM handling or
> otherwise page table allocation and filling PTEs doesn't work correctly.
>
> This allows to remove the buggy SVM range lock, but that's not part of
> this patch set.

FWIW, It looks reasonable to me.

Regards,
   Felix


>
> Only compile tested!
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        | 10 ++++--
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h        |  4 ++-
>   .../gpu/drm/amd/amdgpu/amdgpu_vm_internal.h   | 10 ++++++
>   drivers/gpu/drm/amd/amdkfd/kfd_svm.c          | 32 ++++++++-----------
>   4 files changed, 34 insertions(+), 22 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> index d4e207db83c9..a368327cc025 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -1093,6 +1093,7 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
>    * @vram_base: base for vram mappings
>    * @res: ttm_resource to map
>    * @pages_addr: DMA addresses to use for mapping
> + * @hmm_range: to check validity of DMA addresses
>    * @fence: optional resulting fence
>    *
>    * Fill in the page table entries between @start and @last. Allocate and free
> @@ -1106,7 +1107,9 @@ int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   			struct amdgpu_sync *sync, uint64_t start,
>   			uint64_t last, uint64_t flags, uint64_t offset,
>   			uint64_t vram_base, struct ttm_resource *res,
> -			dma_addr_t *pages_addr, struct dma_fence **fence)
> +			dma_addr_t *pages_addr,
> +			struct amdgpu_hmm_range *hmm_range,
> +			struct dma_fence **fence)
>   {
>   	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
>   	struct amdgpu_vm_update_params params;
> @@ -1139,6 +1142,7 @@ int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   	params.adev = adev;
>   	params.vm = vm;
>   	params.pages_addr = pages_addr;
> +	params.hmm_range = hmm_range;
>   	params.needs_flush = flush_tlb;
>   	params.override_pte = allow_override && adev->gmc.override_pte;
>   	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
> @@ -1432,7 +1436,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
>   		r = amdgpu_vm_map_range(adev, vm, flush_tlb, !uncached, &sync,
>   					mapping->start, mapping->last,
>   					update_flags, mapping->offset,
> -					vram_base, mem, pages_addr,
> +					vram_base, mem, pages_addr, NULL,
>   					last_update);
>   		if (r)
>   			goto error_free;
> @@ -1634,7 +1638,7 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
>   
>   		r = amdgpu_vm_map_range(adev, vm, true, false,
>   					&sync, mapping->start, mapping->last,
> -					0, 0, 0, NULL, NULL, &f);
> +					0, 0, 0, NULL, NULL, NULL, &f);
>   		amdgpu_vm_free_mapping(adev, vm, mapping, f);
>   		if (r) {
>   			dma_fence_put(f);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> index 7db3c3577949..ec1196d390bb 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> @@ -459,7 +459,9 @@ int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   			struct amdgpu_sync *sync, uint64_t start,
>   			uint64_t last, uint64_t flags, uint64_t offset,
>   			uint64_t vram_base, struct ttm_resource *res,
> -			dma_addr_t *pages_addr, struct dma_fence **fence);
> +			dma_addr_t *pages_addr,
> +			struct amdgpu_hmm_range *hmm_range,
> +			struct dma_fence **fence);
>   int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   			   struct amdgpu_sync *sync, uint64_t start,
>   			   uint64_t last, uint64_t flags,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
> index 29c74920f3b0..d47951437f0d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
> @@ -26,6 +26,7 @@
>   
>   #include <linux/types.h>
>   #include <linux/list.h>
> +#include "amdgpu_hmm.h"
>   #include "amdgpu_vm.h"
>   
>   struct amdgpu_device;
> @@ -72,6 +73,13 @@ struct amdgpu_vm_update_params {
>   	 */
>   	dma_addr_t *pages_addr;
>   
> +	/**
> +	 * @hmm_range:
> +	 *
> +	 * Used to check the validity of pages_addr.
> +	 */
> +	struct amdgpu_hmm_range *hmm_range;
> +
>   	/**
>   	 * @job: job to used for hw submission
>   	 */
> @@ -153,6 +161,8 @@ static inline int amdgpu_vm_begin_critical(struct amdgpu_vm_update_params *p)
>   	p->saved_flags = memalloc_noreclaim_save();
>   	if (p->vm->evicting)
>   		return -EBUSY;
> +	if (p->hmm_range && !amdgpu_hmm_range_valid(p->hmm_range))
> +		return -EAGAIN;
>   	return 0;
>   }
>   
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> index d0ea20dea3e1..d506afff684e 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> @@ -1432,7 +1432,8 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
>   static int
>   svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
>   		     unsigned long offset, unsigned long npages, bool readonly,
> -		     dma_addr_t *dma_addr, struct amdgpu_device *bo_adev,
> +		     dma_addr_t *dma_addr, struct amdgpu_hmm_range *hmm_range,
> +		     struct amdgpu_device *bo_adev,
>   		     struct dma_fence **fence, bool flush_tlb)
>   {
>   	struct amdgpu_device *adev = pdd->dev->adev;
> @@ -1492,7 +1493,7 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
>   					gpu_start, gpu_end, pte_flags,
>   					(last_start - prange->start) << PAGE_SHIFT,
>   					bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
> -					NULL, dma_addr, &vm->last_update);
> +					NULL, dma_addr, hmm_range, &vm->last_update);
>   
>   		for (j = last_start - prange->start; j <= i; j++)
>   			dma_addr[j] |= last_domain;
> @@ -1519,7 +1520,9 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
>   }
>   
>   static int
> -svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,
> +svm_range_map_to_gpus(struct svm_range *prange,
> +		      struct amdgpu_hmm_range *hmm_range,
> +		      unsigned long offset,
>   		      unsigned long npages, bool readonly,
>   		      unsigned long *bitmap, bool wait, bool flush_tlb)
>   {
> @@ -1553,7 +1556,7 @@ svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,
>   		}
>   
>   		r = svm_range_map_to_gpu(pdd, prange, offset, npages, readonly,
> -					 prange->dma_addr[gpuidx],
> +					 prange->dma_addr[gpuidx], hmm_range,
>   					 bo_adev, wait ? &fence : NULL,
>   					 flush_tlb);
>   		if (r)
> @@ -1823,18 +1826,6 @@ static int svm_range_validate_and_map(struct mm_struct *mm,
>   
>   		svm_range_lock(prange);
>   
> -		/* Free backing memory of hmm_range if it was initialized
> -		 * Override return value to TRY AGAIN only if prior returns
> -		 * were successful
> -		 */
> -		if (range && !amdgpu_hmm_range_valid(range) && !r) {
> -			pr_debug("hmm update the range, need validate again\n");
> -			r = -EAGAIN;
> -		}
> -
> -		/* Free the hmm range */
> -		amdgpu_hmm_range_free(range);
> -
>   		if (!r && !list_empty(&prange->child_list)) {
>   			pr_debug("range split by unmap in parallel, validate again\n");
>   			r = -EAGAIN;
> @@ -1846,11 +1837,16 @@ static int svm_range_validate_and_map(struct mm_struct *mm,
>   			if (map_start_vma <= map_last_vma) {
>   				offset = map_start_vma - prange->start;
>   				npages = map_last_vma - map_start_vma + 1;
> -				r = svm_range_map_to_gpus(prange, offset, npages, readonly,
> -							  ctx->bitmap, wait, flush_tlb);
> +				r = svm_range_map_to_gpus(prange, range, offset,
> +							  npages, readonly,
> +							  ctx->bitmap, wait,
> +							  flush_tlb);
>   			}
>   		}
>   
> +		/* Free the hmm range */
> +		amdgpu_hmm_range_free(range);
> +
>   		if (!r && next == end)
>   			prange->mapped_to_gpu = true;
>   

Re: [PATCH 08/13] drm/amdgpu: split amdgpu_vm_update_range

[Huang, Honglei]

On 5/29/2026 7:24 PM, Christian König wrote:
> Split amdgpu_vm_update_range into two functions.
> 
> amdgpu_vm_map_range() is for mapping PTEs into a range and updates
> which can be done while holding the VM lock.
> 
> amdgpu_vm_unmap_range() is for unmapping PTEs without holding the VM
> lock in MMU notifiers.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_job.h   |   3 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 112 ++++++++++++++++++----
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h    |  14 ++-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c |  35 ++-----
>   drivers/gpu/drm/amd/amdkfd/kfd_svm.c      |  16 ++--
>   5 files changed, 120 insertions(+), 60 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> index 44fe40f9e8df..653ffa9ca0f3 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> @@ -47,7 +47,7 @@ enum amdgpu_ib_pool_type;
>   /* Internal kernel job ids. (decreasing values, starting from U64_MAX). */
>   #define AMDGPU_KERNEL_JOB_ID_VM_UPDATE              (18446744073709551615ULL)
>   #define AMDGPU_KERNEL_JOB_ID_VM_UPDATE_PDES         (18446744073709551614ULL)
> -#define AMDGPU_KERNEL_JOB_ID_VM_UPDATE_RANGE        (18446744073709551613ULL)
> +#define AMDGPU_KERNEL_JOB_ID_VM_MAP_RANGE           (18446744073709551613ULL)
>   #define AMDGPU_KERNEL_JOB_ID_VM_PT_CLEAR            (18446744073709551612ULL)
>   #define AMDGPU_KERNEL_JOB_ID_TTM_MAP_BUFFER         (18446744073709551611ULL)
>   #define AMDGPU_KERNEL_JOB_ID_TTM_ACCESS_MEMORY_SDMA (18446744073709551610ULL)
> @@ -63,6 +63,7 @@ enum amdgpu_ib_pool_type;
>   #define AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST         (18446744073709551600ULL)
>   #define AMDGPU_KERNEL_JOB_ID_VPE_RING_TEST          (18446744073709551599ULL)
>   #define AMDGPU_KERNEL_JOB_ID_RUN_SHADER             (18446744073709551598ULL)
> +#define AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE         (18446744073709551597ULL)
>   
>   struct amdgpu_job {
>   	struct drm_sched_job    base;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> index edc8b1ca2d3e..b5adfcacc55a 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -1080,11 +1080,10 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
>   }
>   
>   /**
> - * amdgpu_vm_update_range - update a range in the vm page table
> + * amdgpu_vm_map_range - map something to a range in the vm page tables
>    *
>    * @adev: amdgpu_device pointer to use for commands
>    * @vm: the VM to update the range
> - * @unlocked: unlocked invalidation during MM callback
>    * @flush_tlb: trigger tlb invalidation after update completed
>    * @allow_override: change MTYPE for local NUMA nodes
>    * @sync: fences we need to sync to
> @@ -1097,23 +1096,26 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
>    * @pages_addr: DMA addresses to use for mapping
>    * @fence: optional resulting fence
>    *
> - * Fill in the page table entries between @start and @last.
> + * Fill in the page table entries between @start and @last. Allocate and free
> + * new page tables as needed. Can only be called while holding the VM lock.
>    *
>    * Returns:
>    * 0 for success, negative erro code for failure.
>    */
> -int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> -			   bool unlocked, bool flush_tlb, bool allow_override,
> -			   struct amdgpu_sync *sync, uint64_t start,
> -			   uint64_t last, uint64_t flags, uint64_t offset,
> -			   uint64_t vram_base, struct ttm_resource *res,
> -			   dma_addr_t *pages_addr, struct dma_fence **fence)
> +int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> +			bool flush_tlb, bool allow_override,
> +			struct amdgpu_sync *sync, uint64_t start,
> +			uint64_t last, uint64_t flags, uint64_t offset,
> +			uint64_t vram_base, struct ttm_resource *res,
> +			dma_addr_t *pages_addr, struct dma_fence **fence)
>   {
>   	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
>   	struct amdgpu_vm_update_params params;
>   	struct amdgpu_res_cursor cursor;
>   	int r, idx;
>   
> +	amdgpu_vm_assert_locked(vm);
> +
>   	if (!drm_dev_enter(adev_to_drm(adev), &idx))
>   		return -ENODEV;
>   
> @@ -1138,7 +1140,6 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   	params.adev = adev;
>   	params.vm = vm;
>   	params.pages_addr = pages_addr;
> -	params.unlocked = unlocked;
>   	params.needs_flush = flush_tlb;
>   	params.override_pte = allow_override && adev->gmc.override_pte;
>   	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
> @@ -1149,7 +1150,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   		goto error_free;
>   	}
>   
> -	if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) {
> +	if (!dma_fence_is_signaled(vm->last_unlocked)) {
>   		struct dma_fence *tmp = dma_fence_get_stub();
>   
>   		amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true);
> @@ -1158,7 +1159,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   	}
>   
>   	r = vm->update_funcs->prepare(&params, sync,
> -				      AMDGPU_KERNEL_JOB_ID_VM_UPDATE_RANGE);
> +				      AMDGPU_KERNEL_JOB_ID_VM_MAP_RANGE);
>   	if (r)
>   		goto error_free;
>   
> @@ -1234,6 +1235,77 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   	return r;
>   }
>   
> +/**
> + * amdgpu_vm_unmap_range - clear leave PTEs to unmap something
> + *
> + * @adev: amdgpu_device pointer to use for commands
> + * @vm: the VM to update the range
> + * @sync: fences we need to sync to
> + * @start: start of unmapped range
> + * @last: last unmapped entry
> + * @flags: flags for the entries
> + * @fence: optional resulting fence
> + *
> + * Fill in the page table entries between @start and @last with a fixed flags
> + * value without allocating or freeing page tables. Can be used without locking
> + * the VM.
> + *
> + * Returns:
> + * 0 for success, negative erro code for failure.
> + */
> +int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> +			   struct amdgpu_sync *sync, uint64_t start,
> +			   uint64_t last, uint64_t flags,
> +			   struct dma_fence **fence)
> +{
> +	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
> +	struct amdgpu_vm_update_params params;
> +	int r, idx;
> +
> +	if (!drm_dev_enter(adev_to_drm(adev), &idx))
> +		return -ENODEV;
> +
> +	tlb_cb = kmalloc(sizeof(*tlb_cb), GFP_KERNEL);
> +	if (!tlb_cb) {
> +		drm_dev_exit(idx);
> +		return -ENOMEM;
> +	}
> +
> +	memset(&params, 0, sizeof(params));
> +	params.adev = adev;
> +	params.vm = vm;
> +	params.needs_flush = true;
> +	params.unlocked = true;
> +	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
> +
> +	amdgpu_vm_eviction_lock(vm);
> +	if (vm->evicting) {
> +		r = -EBUSY;
> +		goto error_free;
> +	}
> +
> +	r = vm->update_funcs->prepare(&params, sync,
> +				      AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE);
> +	if (r)
> +		goto error_free;
> +
> +	amdgpu_vm_update_leaves(&params, start, last, 0, flags);

A quick test update:
Since the amdgpu_vm_update_leaves update the PTEs to insert a dummy PRT 
PTE, use this func to unmap GPU mapping will cover entire PDB0 range, 2M 
in my local test, it will umap extra ranges, causing GPU fault errors on 
the xnack off platform. Blocks some basic features, many cases failed here.

and it seems like the parameters in amdgpu_vm_update_leaves is start and 
end, maybe need "last + 1" here.

Regards,
Honglei


> +
> +	r = vm->update_funcs->commit(&params, fence);
> +	if (r)
> +		goto error_free;
> +
> +	amdgpu_vm_tlb_flush(&params, fence, tlb_cb);
> +	amdgpu_vm_pt_free_list(adev, &params);
> +	tlb_cb = NULL;
> +
> +error_free:
> +	kfree(tlb_cb);
> +	amdgpu_vm_eviction_unlock(vm);
> +	drm_dev_exit(idx);
> +	return r;
> +}
> +
>   void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
>   			  struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
>   {
> @@ -1362,11 +1434,11 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
>   
>   		trace_amdgpu_vm_bo_update(mapping);
>   
> -		r = amdgpu_vm_update_range(adev, vm, false, flush_tlb,
> -					   !uncached, &sync, mapping->start,
> -					   mapping->last, update_flags,
> -					   mapping->offset, vram_base, mem,
> -					   pages_addr, last_update);
> +		r = amdgpu_vm_map_range(adev, vm, flush_tlb, !uncached, &sync,
> +					mapping->start, mapping->last,
> +					update_flags, mapping->offset,
> +					vram_base, mem, pages_addr,
> +					last_update);
>   		if (r)
>   			goto error_free;
>   	}
> @@ -1565,9 +1637,9 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
>   			struct amdgpu_bo_va_mapping, list);
>   		list_del(&mapping->list);
>   
> -		r = amdgpu_vm_update_range(adev, vm, false, true, false,
> -					   &sync, mapping->start, mapping->last,
> -					   0, 0, 0, NULL, NULL, &f);
> +		r = amdgpu_vm_map_range(adev, vm, true, false,
> +					&sync, mapping->start, mapping->last,
> +					0, 0, 0, NULL, NULL, &f);
>   		amdgpu_vm_free_mapping(adev, vm, mapping, f);
>   		if (r) {
>   			dma_fence_put(f);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> index 3e86a2a470f0..561f2873d2ec 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> @@ -529,12 +529,16 @@ int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev,
>   				uint32_t xcc_mask);
>   void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
>   			    struct amdgpu_vm *vm, struct amdgpu_bo *bo);
> -int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> -			   bool unlocked, bool flush_tlb, bool allow_override,
> +int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> +			bool flush_tlb, bool allow_override,
> +			struct amdgpu_sync *sync, uint64_t start,
> +			uint64_t last, uint64_t flags, uint64_t offset,
> +			uint64_t vram_base, struct ttm_resource *res,
> +			dma_addr_t *pages_addr, struct dma_fence **fence);
> +int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   			   struct amdgpu_sync *sync, uint64_t start,
> -			   uint64_t last, uint64_t flags, uint64_t offset,
> -			   uint64_t vram_base, struct ttm_resource *res,
> -			   dma_addr_t *pages_addr, struct dma_fence **fence);
> +			   uint64_t last, uint64_t flags,
> +			   struct dma_fence **fence);
>   int amdgpu_vm_bo_update(struct amdgpu_device *adev,
>   			struct amdgpu_bo_va *bo_va,
>   			bool clear);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> index 6f5415d5a1bc..ac3f3e31e2e2 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> @@ -553,7 +553,6 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
>   			    struct amdgpu_vm_update_params *params)
>   {
>   	struct amdgpu_vm_bo_base *entry, *next;
> -	bool unlocked = params->unlocked;
>   
>   	if (list_empty(&params->tlb_flush_waitlist))
>   		return;
> @@ -561,7 +560,7 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
>   	/*
>   	 * unlocked unmap clear page table leaves, warning to free the page entry.
>   	 */
> -	WARN_ON(unlocked);
> +	WARN_ON(params->unlocked);
>   
>   	list_for_each_entry_safe(entry, next, &params->tlb_flush_waitlist, vm_status)
>   		amdgpu_vm_pt_free(entry);
> @@ -801,24 +800,17 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
>   		uint64_t incr, entry_end, pe_start;
>   		struct amdgpu_bo *pt;
>   
> -		if (!params->unlocked) {
> -			/* make sure that the page tables covering the
> -			 * address range are actually allocated
> -			 */
> -			r = amdgpu_vm_pt_alloc(params->adev, params->vm,
> -					       &cursor);
> -			if (r)
> -				return r;
> -		}
> +		/* make sure that the page tables covering the
> +		 * address range are actually allocated
> +		 */
> +		r = amdgpu_vm_pt_alloc(params->adev, params->vm, &cursor);
> +		if (r)
> +			return r;
>   
>   		shift = amdgpu_vm_pt_level_shift(adev, cursor.level);
>   		parent_shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1);
> -		if (params->unlocked) {
> -			/* Unlocked updates are only allowed on the leaves */
> -			if (amdgpu_vm_pt_descendant(adev, &cursor))
> -				continue;
> -		} else if (adev->asic_type < CHIP_VEGA10 &&
> -			   (flags & AMDGPU_PTE_VALID)) {
> +		if (adev->asic_type < CHIP_VEGA10 &&
> +		    (flags & AMDGPU_PTE_VALID)) {
>   			/* No huge page support before GMC v9 */
>   			if (cursor.level != AMDGPU_VM_PTB) {
>   				if (!amdgpu_vm_pt_descendant(adev, &cursor))
> @@ -864,14 +856,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
>   		mask = amdgpu_vm_pt_entries_mask(adev, cursor.level);
>   		pe_start = ((cursor.pfn >> shift) & mask) * 8;
>   
> -		if (cursor.level < AMDGPU_VM_PTB && params->unlocked)
> -			/*
> -			 * MMU notifier callback unlocked unmap huge page, leave is PDE entry,
> -			 * only clear one entry. Next entry search again for PDE or PTE leave.
> -			 */
> -			entry_end = 1ULL << shift;
> -		else
> -			entry_end = ((uint64_t)mask + 1) << shift;
> +		entry_end = ((uint64_t)mask + 1) << shift;
>   		entry_end += cursor.pfn & ~(entry_end - 1);
>   		entry_end = min(entry_end, end);
>   
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> index 37b5166e9a14..d0ea20dea3e1 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> @@ -1372,9 +1372,8 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   		return -EINVAL;
>   	}
>   
> -	return amdgpu_vm_update_range(adev, vm, true, true, false, NULL, gpu_start,
> -				      gpu_end, init_pte_value, 0, 0, NULL, NULL,
> -				      fence);
> +	return amdgpu_vm_unmap_range(adev, vm, NULL, gpu_start, gpu_end,
> +				     init_pte_value, fence);
>   }
>   
>   static int
> @@ -1489,12 +1488,11 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
>   			 (last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0,
>   			 pte_flags);
>   
> -		r = amdgpu_vm_update_range(adev, vm, false, flush_tlb, true,
> -					   NULL, gpu_start, gpu_end,
> -					   pte_flags,
> -					   (last_start - prange->start) << PAGE_SHIFT,
> -					   bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
> -					   NULL, dma_addr, &vm->last_update);
> +		r = amdgpu_vm_map_range(adev, vm, flush_tlb, true, NULL,
> +					gpu_start, gpu_end, pte_flags,
> +					(last_start - prange->start) << PAGE_SHIFT,
> +					bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
> +					NULL, dma_addr, &vm->last_update);
>   
>   		for (j = last_start - prange->start; j <= i; j++)
>   			dma_addr[j] |= last_domain;

Re: [PATCH 08/13] drm/amdgpu: split amdgpu_vm_update_range

[Christian König]

On 6/3/26 19:54, Kuehling, Felix wrote:
...
>> +    r = vm->update_funcs->prepare(&params, sync,
>> +                      AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE);
>> +    if (r)
>> +        goto error_free;
>> +
>> +    amdgpu_vm_update_leaves(&params, start, last, 0, flags);
>> +
>> +    r = vm->update_funcs->commit(&params, fence);
>> +    if (r)
>> +        goto error_free;
> 
> So this wraps amdgpu_vm_update_leaves with all the stuff that's necessary to actually execute the page table update.

Yes, exactly that.

> I don't understand how amdgpu_vm_update_leaves works without this when you call it directly from amdgpu_vm_handle_fault (in patch 6).

It does mostly the same, but not 100%.

> Shouldn't you use amdgpu_vm_unmap_range there instead?

No, the main difference is that calling amdgpu_vm_update_leaves() from the fault handler needs to bypass the normal sequencial handling of VM updates.

Background is that this is a band aid for a dma_fence based submission to avoid a full GPU lockup and instead just continue even when the rendering is incorrect.

> And if that's true, then maybe you can use the name amdgpu_vm_update_leaves for this instead.

I'm certainly open for better naming.

Maybe we should call the function in amdgpu_vm_pt.c amdgpu_vm_pt_update_leaves() and the higher level function in amdgpu_vm.c just amdgpu_vm_update_leaves() ?

Thanks for the review,
Christian.


> 
> Regards,
>   Felix
> 
> 
>> +
>> +    amdgpu_vm_tlb_flush(&params, fence, tlb_cb);
>> +    amdgpu_vm_pt_free_list(adev, &params);
>> +    tlb_cb = NULL;
>> +
>> +error_free:
>> +    kfree(tlb_cb);
>> +    amdgpu_vm_eviction_unlock(vm);
>> +    drm_dev_exit(idx);
>> +    return r;
>> +}
>> +
>>   void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
>>                 struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
>>   {
>> @@ -1362,11 +1434,11 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
>>             trace_amdgpu_vm_bo_update(mapping);
>>   -        r = amdgpu_vm_update_range(adev, vm, false, flush_tlb,
>> -                       !uncached, &sync, mapping->start,
>> -                       mapping->last, update_flags,
>> -                       mapping->offset, vram_base, mem,
>> -                       pages_addr, last_update);
>> +        r = amdgpu_vm_map_range(adev, vm, flush_tlb, !uncached, &sync,
>> +                    mapping->start, mapping->last,
>> +                    update_flags, mapping->offset,
>> +                    vram_base, mem, pages_addr,
>> +                    last_update);
>>           if (r)
>>               goto error_free;
>>       }
>> @@ -1565,9 +1637,9 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
>>               struct amdgpu_bo_va_mapping, list);
>>           list_del(&mapping->list);
>>   -        r = amdgpu_vm_update_range(adev, vm, false, true, false,
>> -                       &sync, mapping->start, mapping->last,
>> -                       0, 0, 0, NULL, NULL, &f);
>> +        r = amdgpu_vm_map_range(adev, vm, true, false,
>> +                    &sync, mapping->start, mapping->last,
>> +                    0, 0, 0, NULL, NULL, &f);
>>           amdgpu_vm_free_mapping(adev, vm, mapping, f);
>>           if (r) {
>>               dma_fence_put(f);
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
>> index 3e86a2a470f0..561f2873d2ec 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
>> @@ -529,12 +529,16 @@ int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev,
>>                   uint32_t xcc_mask);
>>   void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
>>                   struct amdgpu_vm *vm, struct amdgpu_bo *bo);
>> -int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>> -               bool unlocked, bool flush_tlb, bool allow_override,
>> +int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>> +            bool flush_tlb, bool allow_override,
>> +            struct amdgpu_sync *sync, uint64_t start,
>> +            uint64_t last, uint64_t flags, uint64_t offset,
>> +            uint64_t vram_base, struct ttm_resource *res,
>> +            dma_addr_t *pages_addr, struct dma_fence **fence);
>> +int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>>                  struct amdgpu_sync *sync, uint64_t start,
>> -               uint64_t last, uint64_t flags, uint64_t offset,
>> -               uint64_t vram_base, struct ttm_resource *res,
>> -               dma_addr_t *pages_addr, struct dma_fence **fence);
>> +               uint64_t last, uint64_t flags,
>> +               struct dma_fence **fence);
>>   int amdgpu_vm_bo_update(struct amdgpu_device *adev,
>>               struct amdgpu_bo_va *bo_va,
>>               bool clear);
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
>> index 6f5415d5a1bc..ac3f3e31e2e2 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
>> @@ -553,7 +553,6 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
>>                   struct amdgpu_vm_update_params *params)
>>   {
>>       struct amdgpu_vm_bo_base *entry, *next;
>> -    bool unlocked = params->unlocked;
>>         if (list_empty(&params->tlb_flush_waitlist))
>>           return;
>> @@ -561,7 +560,7 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
>>       /*
>>        * unlocked unmap clear page table leaves, warning to free the page entry.
>>        */
>> -    WARN_ON(unlocked);
>> +    WARN_ON(params->unlocked);
>>         list_for_each_entry_safe(entry, next, &params->tlb_flush_waitlist, vm_status)
>>           amdgpu_vm_pt_free(entry);
>> @@ -801,24 +800,17 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
>>           uint64_t incr, entry_end, pe_start;
>>           struct amdgpu_bo *pt;
>>   -        if (!params->unlocked) {
>> -            /* make sure that the page tables covering the
>> -             * address range are actually allocated
>> -             */
>> -            r = amdgpu_vm_pt_alloc(params->adev, params->vm,
>> -                           &cursor);
>> -            if (r)
>> -                return r;
>> -        }
>> +        /* make sure that the page tables covering the
>> +         * address range are actually allocated
>> +         */
>> +        r = amdgpu_vm_pt_alloc(params->adev, params->vm, &cursor);
>> +        if (r)
>> +            return r;
>>             shift = amdgpu_vm_pt_level_shift(adev, cursor.level);
>>           parent_shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1);
>> -        if (params->unlocked) {
>> -            /* Unlocked updates are only allowed on the leaves */
>> -            if (amdgpu_vm_pt_descendant(adev, &cursor))
>> -                continue;
>> -        } else if (adev->asic_type < CHIP_VEGA10 &&
>> -               (flags & AMDGPU_PTE_VALID)) {
>> +        if (adev->asic_type < CHIP_VEGA10 &&
>> +            (flags & AMDGPU_PTE_VALID)) {
>>               /* No huge page support before GMC v9 */
>>               if (cursor.level != AMDGPU_VM_PTB) {
>>                   if (!amdgpu_vm_pt_descendant(adev, &cursor))
>> @@ -864,14 +856,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
>>           mask = amdgpu_vm_pt_entries_mask(adev, cursor.level);
>>           pe_start = ((cursor.pfn >> shift) & mask) * 8;
>>   -        if (cursor.level < AMDGPU_VM_PTB && params->unlocked)
>> -            /*
>> -             * MMU notifier callback unlocked unmap huge page, leave is PDE entry,
>> -             * only clear one entry. Next entry search again for PDE or PTE leave.
>> -             */
>> -            entry_end = 1ULL << shift;
>> -        else
>> -            entry_end = ((uint64_t)mask + 1) << shift;
>> +        entry_end = ((uint64_t)mask + 1) << shift;
>>           entry_end += cursor.pfn & ~(entry_end - 1);
>>           entry_end = min(entry_end, end);
>>   diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
>> index 37b5166e9a14..d0ea20dea3e1 100644
>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
>> @@ -1372,9 +1372,8 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>>           return -EINVAL;
>>       }
>>   -    return amdgpu_vm_update_range(adev, vm, true, true, false, NULL, gpu_start,
>> -                      gpu_end, init_pte_value, 0, 0, NULL, NULL,
>> -                      fence);
>> +    return amdgpu_vm_unmap_range(adev, vm, NULL, gpu_start, gpu_end,
>> +                     init_pte_value, fence);
>>   }
>>     static int
>> @@ -1489,12 +1488,11 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
>>                (last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0,
>>                pte_flags);
>>   -        r = amdgpu_vm_update_range(adev, vm, false, flush_tlb, true,
>> -                       NULL, gpu_start, gpu_end,
>> -                       pte_flags,
>> -                       (last_start - prange->start) << PAGE_SHIFT,
>> -                       bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
>> -                       NULL, dma_addr, &vm->last_update);
>> +        r = amdgpu_vm_map_range(adev, vm, flush_tlb, true, NULL,
>> +                    gpu_start, gpu_end, pte_flags,
>> +                    (last_start - prange->start) << PAGE_SHIFT,
>> +                    bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
>> +                    NULL, dma_addr, &vm->last_update);
>>             for (j = last_start - prange->start; j <= i; j++)
>>               dma_addr[j] |= last_domain;

Re: [PATCH 08/13] drm/amdgpu: split amdgpu_vm_update_range

[Kuehling, Felix]

On 2026-06-05 05:21, Christian König wrote:
> On 6/3/26 19:54, Kuehling, Felix wrote:
> ...
>>> +    r = vm->update_funcs->prepare(&params, sync,
>>> +                      AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE);
>>> +    if (r)
>>> +        goto error_free;
>>> +
>>> +    amdgpu_vm_update_leaves(&params, start, last, 0, flags);
>>> +
>>> +    r = vm->update_funcs->commit(&params, fence);
>>> +    if (r)
>>> +        goto error_free;
>> So this wraps amdgpu_vm_update_leaves with all the stuff that's necessary to actually execute the page table update.
> Yes, exactly that.
>
>> I don't understand how amdgpu_vm_update_leaves works without this when you call it directly from amdgpu_vm_handle_fault (in patch 6).
> It does mostly the same, but not 100%.
>
>> Shouldn't you use amdgpu_vm_unmap_range there instead?
> No, the main difference is that calling amdgpu_vm_update_leaves() from the fault handler needs to bypass the normal sequencial handling of VM updates.
>
> Background is that this is a band aid for a dma_fence based submission to avoid a full GPU lockup and instead just continue even when the rendering is incorrect.

I'm still missing something. amdgpu_vm_unmap_range calls 
vm->update_funcs->prepare, and vm->update_funcs->commit. 
amdgpu_vm_update_leaves doesn't. Neither does amdgpu_vm_handle_fault. So 
how do you e.g. create and submit an SDMA job for the page table update 
in the VM fault handler?

Or is this only meant to work with CPU page table updates? But then 
you're still missing some synchronization and HDP flushing if you never 
call prepare and commit.

Regards,
   Felix


>
>> And if that's true, then maybe you can use the name amdgpu_vm_update_leaves for this instead.
> I'm certainly open for better naming.
>
> Maybe we should call the function in amdgpu_vm_pt.c amdgpu_vm_pt_update_leaves() and the higher level function in amdgpu_vm.c just amdgpu_vm_update_leaves() ?
>
> Thanks for the review,
> Christian.
>
>
>> Regards,
>>    Felix
>>
>>
>>> +
>>> +    amdgpu_vm_tlb_flush(&params, fence, tlb_cb);
>>> +    amdgpu_vm_pt_free_list(adev, &params);
>>> +    tlb_cb = NULL;
>>> +
>>> +error_free:
>>> +    kfree(tlb_cb);
>>> +    amdgpu_vm_eviction_unlock(vm);
>>> +    drm_dev_exit(idx);
>>> +    return r;
>>> +}
>>> +
>>>    void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
>>>                  struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
>>>    {
>>> @@ -1362,11 +1434,11 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
>>>              trace_amdgpu_vm_bo_update(mapping);
>>>    -        r = amdgpu_vm_update_range(adev, vm, false, flush_tlb,
>>> -                       !uncached, &sync, mapping->start,
>>> -                       mapping->last, update_flags,
>>> -                       mapping->offset, vram_base, mem,
>>> -                       pages_addr, last_update);
>>> +        r = amdgpu_vm_map_range(adev, vm, flush_tlb, !uncached, &sync,
>>> +                    mapping->start, mapping->last,
>>> +                    update_flags, mapping->offset,
>>> +                    vram_base, mem, pages_addr,
>>> +                    last_update);
>>>            if (r)
>>>                goto error_free;
>>>        }
>>> @@ -1565,9 +1637,9 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
>>>                struct amdgpu_bo_va_mapping, list);
>>>            list_del(&mapping->list);
>>>    -        r = amdgpu_vm_update_range(adev, vm, false, true, false,
>>> -                       &sync, mapping->start, mapping->last,
>>> -                       0, 0, 0, NULL, NULL, &f);
>>> +        r = amdgpu_vm_map_range(adev, vm, true, false,
>>> +                    &sync, mapping->start, mapping->last,
>>> +                    0, 0, 0, NULL, NULL, &f);
>>>            amdgpu_vm_free_mapping(adev, vm, mapping, f);
>>>            if (r) {
>>>                dma_fence_put(f);
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
>>> index 3e86a2a470f0..561f2873d2ec 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
>>> @@ -529,12 +529,16 @@ int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev,
>>>                    uint32_t xcc_mask);
>>>    void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
>>>                    struct amdgpu_vm *vm, struct amdgpu_bo *bo);
>>> -int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>>> -               bool unlocked, bool flush_tlb, bool allow_override,
>>> +int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>>> +            bool flush_tlb, bool allow_override,
>>> +            struct amdgpu_sync *sync, uint64_t start,
>>> +            uint64_t last, uint64_t flags, uint64_t offset,
>>> +            uint64_t vram_base, struct ttm_resource *res,
>>> +            dma_addr_t *pages_addr, struct dma_fence **fence);
>>> +int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>>>                   struct amdgpu_sync *sync, uint64_t start,
>>> -               uint64_t last, uint64_t flags, uint64_t offset,
>>> -               uint64_t vram_base, struct ttm_resource *res,
>>> -               dma_addr_t *pages_addr, struct dma_fence **fence);
>>> +               uint64_t last, uint64_t flags,
>>> +               struct dma_fence **fence);
>>>    int amdgpu_vm_bo_update(struct amdgpu_device *adev,
>>>                struct amdgpu_bo_va *bo_va,
>>>                bool clear);
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
>>> index 6f5415d5a1bc..ac3f3e31e2e2 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
>>> @@ -553,7 +553,6 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
>>>                    struct amdgpu_vm_update_params *params)
>>>    {
>>>        struct amdgpu_vm_bo_base *entry, *next;
>>> -    bool unlocked = params->unlocked;
>>>          if (list_empty(&params->tlb_flush_waitlist))
>>>            return;
>>> @@ -561,7 +560,7 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
>>>        /*
>>>         * unlocked unmap clear page table leaves, warning to free the page entry.
>>>         */
>>> -    WARN_ON(unlocked);
>>> +    WARN_ON(params->unlocked);
>>>          list_for_each_entry_safe(entry, next, &params->tlb_flush_waitlist, vm_status)
>>>            amdgpu_vm_pt_free(entry);
>>> @@ -801,24 +800,17 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
>>>            uint64_t incr, entry_end, pe_start;
>>>            struct amdgpu_bo *pt;
>>>    -        if (!params->unlocked) {
>>> -            /* make sure that the page tables covering the
>>> -             * address range are actually allocated
>>> -             */
>>> -            r = amdgpu_vm_pt_alloc(params->adev, params->vm,
>>> -                           &cursor);
>>> -            if (r)
>>> -                return r;
>>> -        }
>>> +        /* make sure that the page tables covering the
>>> +         * address range are actually allocated
>>> +         */
>>> +        r = amdgpu_vm_pt_alloc(params->adev, params->vm, &cursor);
>>> +        if (r)
>>> +            return r;
>>>              shift = amdgpu_vm_pt_level_shift(adev, cursor.level);
>>>            parent_shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1);
>>> -        if (params->unlocked) {
>>> -            /* Unlocked updates are only allowed on the leaves */
>>> -            if (amdgpu_vm_pt_descendant(adev, &cursor))
>>> -                continue;
>>> -        } else if (adev->asic_type < CHIP_VEGA10 &&
>>> -               (flags & AMDGPU_PTE_VALID)) {
>>> +        if (adev->asic_type < CHIP_VEGA10 &&
>>> +            (flags & AMDGPU_PTE_VALID)) {
>>>                /* No huge page support before GMC v9 */
>>>                if (cursor.level != AMDGPU_VM_PTB) {
>>>                    if (!amdgpu_vm_pt_descendant(adev, &cursor))
>>> @@ -864,14 +856,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
>>>            mask = amdgpu_vm_pt_entries_mask(adev, cursor.level);
>>>            pe_start = ((cursor.pfn >> shift) & mask) * 8;
>>>    -        if (cursor.level < AMDGPU_VM_PTB && params->unlocked)
>>> -            /*
>>> -             * MMU notifier callback unlocked unmap huge page, leave is PDE entry,
>>> -             * only clear one entry. Next entry search again for PDE or PTE leave.
>>> -             */
>>> -            entry_end = 1ULL << shift;
>>> -        else
>>> -            entry_end = ((uint64_t)mask + 1) << shift;
>>> +        entry_end = ((uint64_t)mask + 1) << shift;
>>>            entry_end += cursor.pfn & ~(entry_end - 1);
>>>            entry_end = min(entry_end, end);
>>>    diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
>>> index 37b5166e9a14..d0ea20dea3e1 100644
>>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
>>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
>>> @@ -1372,9 +1372,8 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>>>            return -EINVAL;
>>>        }
>>>    -    return amdgpu_vm_update_range(adev, vm, true, true, false, NULL, gpu_start,
>>> -                      gpu_end, init_pte_value, 0, 0, NULL, NULL,
>>> -                      fence);
>>> +    return amdgpu_vm_unmap_range(adev, vm, NULL, gpu_start, gpu_end,
>>> +                     init_pte_value, fence);
>>>    }
>>>      static int
>>> @@ -1489,12 +1488,11 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
>>>                 (last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0,
>>>                 pte_flags);
>>>    -        r = amdgpu_vm_update_range(adev, vm, false, flush_tlb, true,
>>> -                       NULL, gpu_start, gpu_end,
>>> -                       pte_flags,
>>> -                       (last_start - prange->start) << PAGE_SHIFT,
>>> -                       bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
>>> -                       NULL, dma_addr, &vm->last_update);
>>> +        r = amdgpu_vm_map_range(adev, vm, flush_tlb, true, NULL,
>>> +                    gpu_start, gpu_end, pte_flags,
>>> +                    (last_start - prange->start) << PAGE_SHIFT,
>>> +                    bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
>>> +                    NULL, dma_addr, &vm->last_update);
>>>              for (j = last_start - prange->start; j <= i; j++)
>>>                dma_addr[j] |= last_domain;

Re: [PATCH 01/13] drm/amdgpu: move suballoc defines into own header

[Christian König]

Can anybody give me a quick rb or ack on this patch alone?

An interim is working on some minor amdgpu.h cleanup and that is blocking him so I want to get it pushed to amd-staging-drm-next.

Thanks,
Christian.

On 5/29/26 13:24, Christian König wrote:
> Just some code cleanup, while at it remove outdated comment.
> 
> No functional change.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu.h        | 32 +--------
>  drivers/gpu/drm/amd/amdgpu/amdgpu_object.h | 40 -----------
>  drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h     | 77 ++++++++++++++++++++++
>  3 files changed, 78 insertions(+), 71 deletions(-)
>  create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index 5d7bfa59424a..4b6c9d9e6773 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -112,6 +112,7 @@
>  #include "amdgpu_userq.h"
>  #include "amdgpu_eviction_fence.h"
>  #include "amdgpu_ip.h"
> +#include "amdgpu_sa.h"
>  #if defined(CONFIG_DRM_AMD_ISP)
>  #include "amdgpu_isp.h"
>  #endif
> @@ -386,37 +387,6 @@ struct amdgpu_clock {
>  	uint32_t max_pixel_clock;
>  };
>  
> -/* sub-allocation manager, it has to be protected by another lock.
> - * By conception this is an helper for other part of the driver
> - * like the indirect buffer or semaphore, which both have their
> - * locking.
> - *
> - * Principe is simple, we keep a list of sub allocation in offset
> - * order (first entry has offset == 0, last entry has the highest
> - * offset).
> - *
> - * When allocating new object we first check if there is room at
> - * the end total_size - (last_object_offset + last_object_size) >=
> - * alloc_size. If so we allocate new object there.
> - *
> - * When there is not enough room at the end, we start waiting for
> - * each sub object until we reach object_offset+object_size >=
> - * alloc_size, this object then become the sub object we return.
> - *
> - * Alignment can't be bigger than page size.
> - *
> - * Hole are not considered for allocation to keep things simple.
> - * Assumption is that there won't be hole (all object on same
> - * alignment).
> - */
> -
> -struct amdgpu_sa_manager {
> -	struct drm_suballoc_manager	base;
> -	struct amdgpu_bo		*bo;
> -	uint64_t			gpu_addr;
> -	void				*cpu_ptr;
> -};
> -
>  /*
>   * IRQS.
>   */
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
> index 4d68732d6223..ff11a0903499 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
> @@ -312,46 +312,6 @@ uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo);
>  uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,
>  					    uint32_t domain);
>  
> -/*
> - * sub allocation
> - */
> -static inline struct amdgpu_sa_manager *
> -to_amdgpu_sa_manager(struct drm_suballoc_manager *manager)
> -{
> -	return container_of(manager, struct amdgpu_sa_manager, base);
> -}
> -
> -static inline uint64_t amdgpu_sa_bo_gpu_addr(struct drm_suballoc *sa_bo)
> -{
> -	return to_amdgpu_sa_manager(sa_bo->manager)->gpu_addr +
> -		drm_suballoc_soffset(sa_bo);
> -}
> -
> -static inline void *amdgpu_sa_bo_cpu_addr(struct drm_suballoc *sa_bo)
> -{
> -	return to_amdgpu_sa_manager(sa_bo->manager)->cpu_ptr +
> -		drm_suballoc_soffset(sa_bo);
> -}
> -
> -int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
> -				     struct amdgpu_sa_manager *sa_manager,
> -				     unsigned size, u32 align, u32 domain);
> -void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
> -				      struct amdgpu_sa_manager *sa_manager);
> -int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev,
> -				      struct amdgpu_sa_manager *sa_manager);
> -int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
> -		     struct drm_suballoc **sa_bo,
> -		     unsigned int size);
> -void amdgpu_sa_bo_free(struct drm_suballoc **sa_bo,
> -		       struct dma_fence *fence);
> -#if defined(CONFIG_DEBUG_FS)
> -void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
> -					 struct seq_file *m);
> -u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m);
> -#endif
> -void amdgpu_debugfs_sa_init(struct amdgpu_device *adev);
> -
>  bool amdgpu_bo_support_uswc(u64 bo_flags);
>  
>  
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
> new file mode 100644
> index 000000000000..8c85c80fc119
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
> @@ -0,0 +1,77 @@
> +/* SPDX-License-Identifier: MIT */
> +/*
> + * Copyright 2026 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + *
> + */
> +
> +#ifndef AMDGPU_SA_H_
> +#define AMDGPU_SA_H_
> +
> +#include <drm/drm_suballoc.h>
> +
> +struct amdgpu_device;
> +struct amdgpu_bo;
> +
> +struct amdgpu_sa_manager {
> +	struct drm_suballoc_manager	base;
> +	struct amdgpu_bo		*bo;
> +	uint64_t			gpu_addr;
> +	void				*cpu_ptr;
> +};
> +
> +static inline struct amdgpu_sa_manager *
> +to_amdgpu_sa_manager(struct drm_suballoc_manager *manager)
> +{
> +	return container_of(manager, struct amdgpu_sa_manager, base);
> +}
> +
> +static inline uint64_t amdgpu_sa_bo_gpu_addr(struct drm_suballoc *sa_bo)
> +{
> +	return to_amdgpu_sa_manager(sa_bo->manager)->gpu_addr +
> +		drm_suballoc_soffset(sa_bo);
> +}
> +
> +static inline void *amdgpu_sa_bo_cpu_addr(struct drm_suballoc *sa_bo)
> +{
> +	return to_amdgpu_sa_manager(sa_bo->manager)->cpu_ptr +
> +		drm_suballoc_soffset(sa_bo);
> +}
> +
> +int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
> +			      struct amdgpu_sa_manager *sa_manager,
> +			      unsigned size, u32 align, u32 domain);
> +void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
> +			       struct amdgpu_sa_manager *sa_manager);
> +int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev,
> +			       struct amdgpu_sa_manager *sa_manager);
> +int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
> +		     struct drm_suballoc **sa_bo,
> +		     unsigned int size);
> +void amdgpu_sa_bo_free(struct drm_suballoc **sa_bo,
> +		       struct dma_fence *fence);
> +#if defined(CONFIG_DEBUG_FS)
> +void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
> +				  struct seq_file *m);
> +u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m);
> +#endif
> +void amdgpu_debugfs_sa_init(struct amdgpu_device *adev);
> +
> +#endif

Re: [PATCH 01/13] drm/amdgpu: move suballoc defines into own header

[Kuehling, Felix]

On 2026-05-29 07:24, Christian König wrote:
> Just some code cleanup, while at it remove outdated comment.
>
> No functional change.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>

Acked-by: Felix Kuehling <felix.kuehling@amd.com>


> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu.h        | 32 +--------
>   drivers/gpu/drm/amd/amdgpu/amdgpu_object.h | 40 -----------
>   drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h     | 77 ++++++++++++++++++++++
>   3 files changed, 78 insertions(+), 71 deletions(-)
>   create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index 5d7bfa59424a..4b6c9d9e6773 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -112,6 +112,7 @@
>   #include "amdgpu_userq.h"
>   #include "amdgpu_eviction_fence.h"
>   #include "amdgpu_ip.h"
> +#include "amdgpu_sa.h"
>   #if defined(CONFIG_DRM_AMD_ISP)
>   #include "amdgpu_isp.h"
>   #endif
> @@ -386,37 +387,6 @@ struct amdgpu_clock {
>   	uint32_t max_pixel_clock;
>   };
>   
> -/* sub-allocation manager, it has to be protected by another lock.
> - * By conception this is an helper for other part of the driver
> - * like the indirect buffer or semaphore, which both have their
> - * locking.
> - *
> - * Principe is simple, we keep a list of sub allocation in offset
> - * order (first entry has offset == 0, last entry has the highest
> - * offset).
> - *
> - * When allocating new object we first check if there is room at
> - * the end total_size - (last_object_offset + last_object_size) >=
> - * alloc_size. If so we allocate new object there.
> - *
> - * When there is not enough room at the end, we start waiting for
> - * each sub object until we reach object_offset+object_size >=
> - * alloc_size, this object then become the sub object we return.
> - *
> - * Alignment can't be bigger than page size.
> - *
> - * Hole are not considered for allocation to keep things simple.
> - * Assumption is that there won't be hole (all object on same
> - * alignment).
> - */
> -
> -struct amdgpu_sa_manager {
> -	struct drm_suballoc_manager	base;
> -	struct amdgpu_bo		*bo;
> -	uint64_t			gpu_addr;
> -	void				*cpu_ptr;
> -};
> -
>   /*
>    * IRQS.
>    */
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
> index 4d68732d6223..ff11a0903499 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
> @@ -312,46 +312,6 @@ uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo);
>   uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,
>   					    uint32_t domain);
>   
> -/*
> - * sub allocation
> - */
> -static inline struct amdgpu_sa_manager *
> -to_amdgpu_sa_manager(struct drm_suballoc_manager *manager)
> -{
> -	return container_of(manager, struct amdgpu_sa_manager, base);
> -}
> -
> -static inline uint64_t amdgpu_sa_bo_gpu_addr(struct drm_suballoc *sa_bo)
> -{
> -	return to_amdgpu_sa_manager(sa_bo->manager)->gpu_addr +
> -		drm_suballoc_soffset(sa_bo);
> -}
> -
> -static inline void *amdgpu_sa_bo_cpu_addr(struct drm_suballoc *sa_bo)
> -{
> -	return to_amdgpu_sa_manager(sa_bo->manager)->cpu_ptr +
> -		drm_suballoc_soffset(sa_bo);
> -}
> -
> -int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
> -				     struct amdgpu_sa_manager *sa_manager,
> -				     unsigned size, u32 align, u32 domain);
> -void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
> -				      struct amdgpu_sa_manager *sa_manager);
> -int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev,
> -				      struct amdgpu_sa_manager *sa_manager);
> -int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
> -		     struct drm_suballoc **sa_bo,
> -		     unsigned int size);
> -void amdgpu_sa_bo_free(struct drm_suballoc **sa_bo,
> -		       struct dma_fence *fence);
> -#if defined(CONFIG_DEBUG_FS)
> -void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
> -					 struct seq_file *m);
> -u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m);
> -#endif
> -void amdgpu_debugfs_sa_init(struct amdgpu_device *adev);
> -
>   bool amdgpu_bo_support_uswc(u64 bo_flags);
>   
>   
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
> new file mode 100644
> index 000000000000..8c85c80fc119
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
> @@ -0,0 +1,77 @@
> +/* SPDX-License-Identifier: MIT */
> +/*
> + * Copyright 2026 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + *
> + */
> +
> +#ifndef AMDGPU_SA_H_
> +#define AMDGPU_SA_H_
> +
> +#include <drm/drm_suballoc.h>
> +
> +struct amdgpu_device;
> +struct amdgpu_bo;
> +
> +struct amdgpu_sa_manager {
> +	struct drm_suballoc_manager	base;
> +	struct amdgpu_bo		*bo;
> +	uint64_t			gpu_addr;
> +	void				*cpu_ptr;
> +};
> +
> +static inline struct amdgpu_sa_manager *
> +to_amdgpu_sa_manager(struct drm_suballoc_manager *manager)
> +{
> +	return container_of(manager, struct amdgpu_sa_manager, base);
> +}
> +
> +static inline uint64_t amdgpu_sa_bo_gpu_addr(struct drm_suballoc *sa_bo)
> +{
> +	return to_amdgpu_sa_manager(sa_bo->manager)->gpu_addr +
> +		drm_suballoc_soffset(sa_bo);
> +}
> +
> +static inline void *amdgpu_sa_bo_cpu_addr(struct drm_suballoc *sa_bo)
> +{
> +	return to_amdgpu_sa_manager(sa_bo->manager)->cpu_ptr +
> +		drm_suballoc_soffset(sa_bo);
> +}
> +
> +int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
> +			      struct amdgpu_sa_manager *sa_manager,
> +			      unsigned size, u32 align, u32 domain);
> +void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
> +			       struct amdgpu_sa_manager *sa_manager);
> +int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev,
> +			       struct amdgpu_sa_manager *sa_manager);
> +int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
> +		     struct drm_suballoc **sa_bo,
> +		     unsigned int size);
> +void amdgpu_sa_bo_free(struct drm_suballoc **sa_bo,
> +		       struct dma_fence *fence);
> +#if defined(CONFIG_DEBUG_FS)
> +void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
> +				  struct seq_file *m);
> +u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m);
> +#endif
> +void amdgpu_debugfs_sa_init(struct amdgpu_device *adev);
> +
> +#endif

Re: [PATCH 01/13] drm/amdgpu: move suballoc defines into own header

[Timur Kristóf]

On 2026. május 29., péntek 13:24:03 közép-európai nyári idő Christian König 
wrote:
> Just some code cleanup, while at it remove outdated comment.
> 
> No functional change.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>

Looks good, makes good sense.

Reviewed-by: Timur Kristóf <timur.kristof@gmail.com>

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu.h        | 32 +--------
>  drivers/gpu/drm/amd/amdgpu/amdgpu_object.h | 40 -----------
>  drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h     | 77 ++++++++++++++++++++++
>  3 files changed, 78 insertions(+), 71 deletions(-)
>  create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 5d7bfa59424a..4b6c9d9e6773
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -112,6 +112,7 @@
>  #include "amdgpu_userq.h"
>  #include "amdgpu_eviction_fence.h"
>  #include "amdgpu_ip.h"
> +#include "amdgpu_sa.h"
>  #if defined(CONFIG_DRM_AMD_ISP)
>  #include "amdgpu_isp.h"
>  #endif
> @@ -386,37 +387,6 @@ struct amdgpu_clock {
>  	uint32_t max_pixel_clock;
>  };
> 
> -/* sub-allocation manager, it has to be protected by another lock.
> - * By conception this is an helper for other part of the driver
> - * like the indirect buffer or semaphore, which both have their
> - * locking.
> - *
> - * Principe is simple, we keep a list of sub allocation in offset
> - * order (first entry has offset == 0, last entry has the highest
> - * offset).
> - *
> - * When allocating new object we first check if there is room at
> - * the end total_size - (last_object_offset + last_object_size) >=
> - * alloc_size. If so we allocate new object there.
> - *
> - * When there is not enough room at the end, we start waiting for
> - * each sub object until we reach object_offset+object_size >=
> - * alloc_size, this object then become the sub object we return.
> - *
> - * Alignment can't be bigger than page size.
> - *
> - * Hole are not considered for allocation to keep things simple.
> - * Assumption is that there won't be hole (all object on same
> - * alignment).
> - */
> -
> -struct amdgpu_sa_manager {
> -	struct drm_suballoc_manager	base;
> -	struct amdgpu_bo		*bo;
> -	uint64_t			gpu_addr;
> -	void				*cpu_ptr;
> -};
> -
>  /*
>   * IRQS.
>   */
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h index
> 4d68732d6223..ff11a0903499 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
> @@ -312,46 +312,6 @@ uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo
> *bo); uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,
> uint32_t domain);
> 
> -/*
> - * sub allocation
> - */
> -static inline struct amdgpu_sa_manager *
> -to_amdgpu_sa_manager(struct drm_suballoc_manager *manager)
> -{
> -	return container_of(manager, struct amdgpu_sa_manager, base);
> -}
> -
> -static inline uint64_t amdgpu_sa_bo_gpu_addr(struct drm_suballoc *sa_bo)
> -{
> -	return to_amdgpu_sa_manager(sa_bo->manager)->gpu_addr +
> -		drm_suballoc_soffset(sa_bo);
> -}
> -
> -static inline void *amdgpu_sa_bo_cpu_addr(struct drm_suballoc *sa_bo)
> -{
> -	return to_amdgpu_sa_manager(sa_bo->manager)->cpu_ptr +
> -		drm_suballoc_soffset(sa_bo);
> -}
> -
> -int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
> -				     struct amdgpu_sa_manager 
*sa_manager,
> -				     unsigned size, u32 align, 
u32 domain);
> -void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
> -				      struct amdgpu_sa_manager 
*sa_manager);
> -int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev,
> -				      struct amdgpu_sa_manager 
*sa_manager);
> -int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
> -		     struct drm_suballoc **sa_bo,
> -		     unsigned int size);
> -void amdgpu_sa_bo_free(struct drm_suballoc **sa_bo,
> -		       struct dma_fence *fence);
> -#if defined(CONFIG_DEBUG_FS)
> -void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
> -					 struct seq_file *m);
> -u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m);
> -#endif
> -void amdgpu_debugfs_sa_init(struct amdgpu_device *adev);
> -
>  bool amdgpu_bo_support_uswc(u64 bo_flags);
> 
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h new file mode 100644
> index 000000000000..8c85c80fc119
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
> @@ -0,0 +1,77 @@
> +/* SPDX-License-Identifier: MIT */
> +/*
> + * Copyright 2026 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the
> "Software"), + * to deal in the Software without restriction, including
> without limitation + * the rights to use, copy, modify, merge, publish,
> distribute, sublicense, + * and/or sell copies of the Software, and to
> permit persons to whom the + * Software is furnished to do so, subject to
> the following conditions: + *
> + * The above copyright notice and this permission notice shall be included
> in + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
> OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 
> IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR
> ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF
> CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH
> THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE.
> + *
> + */
> +
> +#ifndef AMDGPU_SA_H_
> +#define AMDGPU_SA_H_
> +
> +#include <drm/drm_suballoc.h>
> +
> +struct amdgpu_device;
> +struct amdgpu_bo;
> +
> +struct amdgpu_sa_manager {
> +	struct drm_suballoc_manager	base;
> +	struct amdgpu_bo		*bo;
> +	uint64_t			gpu_addr;
> +	void				*cpu_ptr;
> +};
> +
> +static inline struct amdgpu_sa_manager *
> +to_amdgpu_sa_manager(struct drm_suballoc_manager *manager)
> +{
> +	return container_of(manager, struct amdgpu_sa_manager, base);
> +}
> +
> +static inline uint64_t amdgpu_sa_bo_gpu_addr(struct drm_suballoc *sa_bo)
> +{
> +	return to_amdgpu_sa_manager(sa_bo->manager)->gpu_addr +
> +		drm_suballoc_soffset(sa_bo);
> +}
> +
> +static inline void *amdgpu_sa_bo_cpu_addr(struct drm_suballoc *sa_bo)
> +{
> +	return to_amdgpu_sa_manager(sa_bo->manager)->cpu_ptr +
> +		drm_suballoc_soffset(sa_bo);
> +}
> +
> +int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
> +			      struct amdgpu_sa_manager *sa_manager,
> +			      unsigned size, u32 align, u32 domain);
> +void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
> +			       struct amdgpu_sa_manager *sa_manager);
> +int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev,
> +			       struct amdgpu_sa_manager *sa_manager);
> +int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
> +		     struct drm_suballoc **sa_bo,
> +		     unsigned int size);
> +void amdgpu_sa_bo_free(struct drm_suballoc **sa_bo,
> +		       struct dma_fence *fence);
> +#if defined(CONFIG_DEBUG_FS)
> +void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
> +				  struct seq_file *m);
> +u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m);
> +#endif
> +void amdgpu_debugfs_sa_init(struct amdgpu_device *adev);
> +
> +#endif

Re: [PATCH 02/13] drm/amdgpu: give different sizes for each SA pool type

[Timur Kristóf]

On 2026. május 29., péntek 13:24:04 közép-európai nyári idő Christian König 
wrote:
> The IMMEDIATE (page fault) and DIRECT (reset) pool should be used only
> very rarely and by a single thread.
> 
> Saves roughly 1.25MiB of memory and GART space for each amdgpu device.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c   | 8 +++++++-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 2 --
>  2 files changed, 7 insertions(+), 3 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index f1ed4a436f5b..334f95f8f339
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
> @@ -351,14 +351,20 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring,
> unsigned int num_ibs, */
>  int amdgpu_ib_pool_init(struct amdgpu_device *adev)
>  {
> +	const int sizes[AMDGPU_IB_POOL_MAX] = {
> +		[AMDGPU_IB_POOL_DELAYED] = SZ_1M,
> +		[AMDGPU_IB_POOL_IMMEDIATE] = SZ_128K,
> +		[AMDGPU_IB_POOL_DIRECT] = SZ_512K
> +	};
>  	int r, i;
> 
>  	if (adev->ib_pool_ready)
>  		return 0;
> 
> +

There is a spurious newline here.

Otherwise the patch makes good sense.
Reviewed-by: Timur Kristóf <timur.kristof@gmail.com>

>  	for (i = 0; i < AMDGPU_IB_POOL_MAX; i++) {
>  		r = amdgpu_sa_bo_manager_init(adev, &adev->ib_pools[i],
> -					      
AMDGPU_IB_POOL_SIZE, 256,
> +					      sizes[i], 256,
>  					      
AMDGPU_GEM_DOMAIN_GTT);
>  		if (r)
>  			goto error;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h index 8f28b3bd7010..1a063a0a4280
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> @@ -68,8 +68,6 @@ enum amdgpu_ring_priority_level {
> 
>  #define to_amdgpu_ring(s) container_of((s), struct amdgpu_ring, sched)
> 
> -#define AMDGPU_IB_POOL_SIZE	(1024 * 1024)
> -
>  enum amdgpu_ring_type {
>  	AMDGPU_RING_TYPE_GFX		= AMDGPU_HW_IP_GFX,
>  	AMDGPU_RING_TYPE_COMPUTE	= AMDGPU_HW_IP_COMPUTE,

Re: [PATCH 03/13] drm/amdgpu: add gfp_flags to amdgpu_sa_manager

[Timur Kristóf]

On 2026. május 29., péntek 13:24:05 közép-európai nyári idő Christian König 
wrote:
> Make sure that we use the emmergency reserves for unrecoverable page
> faults and GPU resets.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 23 ++++++++++++++++++++---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c | 18 ++++++++++++------
>  drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h |  3 ++-
>  3 files changed, 34 insertions(+), 10 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 334f95f8f339..60e4c3985029
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
> @@ -356,16 +356,33 @@ int amdgpu_ib_pool_init(struct amdgpu_device *adev)
>  		[AMDGPU_IB_POOL_IMMEDIATE] = SZ_128K,
>  		[AMDGPU_IB_POOL_DIRECT] = SZ_512K
>  	};
> +	const gfp_t gfp_flags[AMDGPU_IB_POOL_MAX] = {
> +		/*
> +		 * For normal page table updates and recoverable page 
faults,
> +		 * further restricted by the VM eviction lock to not 
wait for
> +		 * memory reclaim.
> +		 */
> +		[AMDGPU_IB_POOL_DELAYED] = GFP_KERNEL,
> +		/*
> +		 * For redirecting unrecoverable page faults to the 
dummy page
> +		 * or set the PRT bits. dma_fence submissions might 
depend on
> +		 * that so we need the emmergency resewrves.
> +		 */
> +		[AMDGPU_IB_POOL_IMMEDIATE] = GFP_ATOMIC,

I know that "retry faults" and "recoverable faults" are different, but both of 
these faults can be mitigated, so using the term "unrecoverable" here feels 
somewhat confusing to me.

Can you please clarify the comment above as well as the commit message that 
this is referring to retry faults?

> +		/*
> +		 * For IB tests during GPU resets. Only very small and 
temporary
> +		 * allocation to make allow dma_fences to signal.
> +		 */
> +		[AMDGPU_IB_POOL_DIRECT] = GFP_ATOMIC

Just a small grammatical nitpick but "make" is not necessary in this comment.

> +	};
>  	int r, i;
> 
>  	if (adev->ib_pool_ready)
>  		return 0;
> 
> -

Looks like this is just removing the spurious newline from the previous 
commit.

With that fixed and the comments + commit message clarified:
Reviewed-by: Timur Kristóf <timur.kristof@gmail.com>

>  	for (i = 0; i < AMDGPU_IB_POOL_MAX; i++) {
>  		r = amdgpu_sa_bo_manager_init(adev, &adev->ib_pools[i],
> -					      sizes[i], 256,
> -					      
AMDGPU_GEM_DOMAIN_GTT);
> +					      sizes[i], 
gfp_flags[i]);
>  		if (r)
>  			goto error;
>  	}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c index 39070b2a4c04..74124f80601e
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
> @@ -46,11 +46,13 @@
> 
>  int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
>  			      struct amdgpu_sa_manager *sa_manager,
> -			      unsigned int size, u32 suballoc_align, 
u32 domain)
> +			      unsigned int size, gfp_t gfp_flags)
>  {
>  	int r;
> 
> -	r = amdgpu_bo_create_kernel(adev, size, AMDGPU_GPU_PAGE_SIZE, 
domain,
> +	sa_manager->gfp_flags = gfp_flags;
> +	r = amdgpu_bo_create_kernel(adev, size, AMDGPU_GPU_PAGE_SIZE,
> +				    AMDGPU_GEM_DOMAIN_GTT,
>  				    &sa_manager->bo, &sa_manager-
>gpu_addr,
>  				    &sa_manager->cpu_ptr);
>  	if (r) {
> @@ -59,7 +61,8 @@ int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
>  	}
> 
>  	memset(sa_manager->cpu_ptr, 0, size);
> -	drm_suballoc_manager_init(&sa_manager->base, size, 
suballoc_align);
> +	drm_suballoc_manager_init(&sa_manager->base, size, 256);
> +
>  	return r;
>  }
> 
> @@ -73,7 +76,8 @@ void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
> 
>  	drm_suballoc_manager_fini(&sa_manager->base);
> 
> -	amdgpu_bo_free_kernel(&sa_manager->bo, &sa_manager->gpu_addr,
> &sa_manager->cpu_ptr); +	amdgpu_bo_free_kernel(&sa_manager->bo,
> &sa_manager->gpu_addr,
> +			      &sa_manager->cpu_ptr);
>  }
> 
>  int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
> @@ -81,7 +85,8 @@ int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
> unsigned int size)
>  {
>  	struct drm_suballoc *sa = drm_suballoc_new(&sa_manager->base, 
size,
> -						   
GFP_KERNEL, false, 0);
> +						   
sa_manager->gfp_flags,
> +						   false, 
0);
> 
>  	if (IS_ERR(sa)) {
>  		*sa_bo = NULL;
> @@ -110,6 +115,7 @@ void amdgpu_sa_bo_dump_debug_info(struct
> amdgpu_sa_manager *sa_manager, {
>  	struct drm_printer p = drm_seq_file_printer(m);
> 
> -	drm_suballoc_dump_debug_info(&sa_manager->base, &p, sa_manager-
>gpu_addr);
> +	drm_suballoc_dump_debug_info(&sa_manager->base, &p,
> +				     sa_manager->gpu_addr);
>  }
>  #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h index 8c85c80fc119..1d1c89348709
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
> @@ -35,6 +35,7 @@ struct amdgpu_sa_manager {
>  	struct amdgpu_bo		*bo;
>  	uint64_t			gpu_addr;
>  	void				*cpu_ptr;
> +	gfp_t				gfp_flags;
>  };
> 
>  static inline struct amdgpu_sa_manager *
> @@ -57,7 +58,7 @@ static inline void *amdgpu_sa_bo_cpu_addr(struct
> drm_suballoc *sa_bo)
> 
>  int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
>  			      struct amdgpu_sa_manager *sa_manager,
> -			      unsigned size, u32 align, u32 domain);
> +			      unsigned size, gfp_t gfp_flags);
>  void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
>  			       struct amdgpu_sa_manager 
*sa_manager);
>  int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev,

Re: [PATCH 04/13] drm/amdgpu: move job parameter to the end in amdgpu_job_alloc() and *_with_ib()

[Timur Kristóf]

On 2026. május 29., péntek 13:24:06 közép-európai nyári idő Christian König 
wrote:
> Move the job output parameter to be the last parameter in the
> amdgpu_job_alloc() and amdgpu_job_alloc_with_ib() function signature. This
> aligns with the common kernel coding convention where output parameters
> typically come last.
> 
> Co-developed by Claude Sonnet 4.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>

Looks good. Nice cleanup.

Reviewed-by: Timur Kristóf <timur.kristof@gmail.com>

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c  |  2 +-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c      |  4 ++--
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c     |  5 +++--
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c     |  3 ++-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_job.c     | 10 +++++-----
>  drivers/gpu/drm/amd/amdgpu/amdgpu_job.h     |  8 ++++----
>  drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c    |  5 +++--
>  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c     | 11 ++++++-----
>  drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c     |  5 +++--
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c     |  8 +++++---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c     | 12 ++++++++----
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c |  2 +-
>  drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c       | 10 ++++++----
>  drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c       | 10 ++++++----
>  drivers/gpu/drm/amd/amdkfd/kfd_migrate.c    |  4 ++--
>  15 files changed, 57 insertions(+), 42 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index
> 0017d502d169..44751d71b741 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> @@ -696,7 +696,7 @@ int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev,
>  		goto err;
>  	}
> 
> -	ret = amdgpu_job_alloc(adev, NULL, NULL, NULL, 1, &job, 0);
> +	ret = amdgpu_job_alloc(adev, NULL, NULL, NULL, 1, 0, &job);
>  	if (ret)
>  		goto err;
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 10d8dcc3a972..fdf01d824d66
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> @@ -274,8 +274,8 @@ static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p,
> 
>  	for (i = 0; i < p->gang_size; ++i) {
>  		ret = amdgpu_job_alloc(p->adev, vm, p->entities[i], vm,
> -				       num_ibs[i], &p->jobs[i],
> -				       p->filp->client_id);
> +				       num_ibs[i], p->filp-
>client_id,
> +				       &p->jobs[i]);
>  		if (ret)
>  			goto free_all_kdata;
>  		switch (p->adev->enforce_isolation[fpriv->xcp_id]) {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c index 515cc4a2aeb4..54450ac49834
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> @@ -1669,8 +1669,9 @@ static int amdgpu_gfx_run_cleaner_shader_job(struct
> amdgpu_ring *ring) owner = (void *)(unsigned
> long)atomic_inc_return(&counter);
> 
>  	r = amdgpu_job_alloc_with_ib(ring->adev, &entity, owner,
> -				     64, 0, &job,
> -				     
AMDGPU_KERNEL_JOB_ID_CLEANER_SHADER);
> +				     64, 0,
> +				     
AMDGPU_KERNEL_JOB_ID_CLEANER_SHADER,
> +				     &job);
>  	if (r)
>  		goto err;
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index 13bec8461cde..181d69770c40
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> @@ -761,7 +761,8 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device
> *adev, uint32_t vmid, r = amdgpu_job_alloc_with_ib(ring->adev,
> &adev->mman.default_entity.base, AMDGPU_FENCE_OWNER_UNDEFINED,
>  				     16 * 4, 
AMDGPU_IB_POOL_IMMEDIATE,
> -				     &job, 
AMDGPU_KERNEL_JOB_ID_FLUSH_GPU_TLB);
> +				     
AMDGPU_KERNEL_JOB_ID_FLUSH_GPU_TLB,
> +				     &job);
>  	if (r)
>  		goto error_alloc;
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index 55172c2dcc35..71c1ba735a6b
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> @@ -187,8 +187,8 @@ static enum drm_gpu_sched_stat
> amdgpu_job_timedout(struct drm_sched_job *s_job)
> 
>  int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>  		     struct drm_sched_entity *entity, void *owner,
> -		     unsigned int num_ibs, struct amdgpu_job **job,
> -		     u64 drm_client_id)
> +		     unsigned int num_ibs, u64 drm_client_id,
> +		     struct amdgpu_job **job)
>  {
>  	struct amdgpu_fence *af;
>  	int r;
> @@ -241,12 +241,12 @@ int amdgpu_job_alloc(struct amdgpu_device *adev,
> struct amdgpu_vm *vm, int amdgpu_job_alloc_with_ib(struct amdgpu_device
> *adev,
>  			     struct drm_sched_entity *entity, void 
*owner,
>  			     size_t size, enum amdgpu_ib_pool_type 
pool_type,
> -			     struct amdgpu_job **job, u64 k_job_id)
> +			     u64 k_job_id, struct amdgpu_job **job)
>  {
>  	int r;
> 
> -	r = amdgpu_job_alloc(adev, NULL, entity, owner, 1, job,
> -			     k_job_id);
> +	r = amdgpu_job_alloc(adev, NULL, entity, owner, 1, k_job_id,
> +			     job);
>  	if (r)
>  		return r;
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h index 56a88e14a044..6b7cf594714c
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> @@ -112,13 +112,13 @@ static inline struct amdgpu_ring
> *amdgpu_job_ring(struct amdgpu_job *job)
> 
>  int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>  		     struct drm_sched_entity *entity, void *owner,
> -		     unsigned int num_ibs, struct amdgpu_job **job,
> -		     u64 drm_client_id);
> +		     unsigned int num_ibs, u64 drm_client_id,
> +		     struct amdgpu_job **job);
>  int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev,
>  			     struct drm_sched_entity *entity, void 
*owner,
>  			     size_t size, enum amdgpu_ib_pool_type 
pool_type,
> -			     struct amdgpu_job **job,
> -			     u64 k_job_id);
> +			     u64 k_job_id,
> +			     struct amdgpu_job **job);
>  void amdgpu_job_set_resources(struct amdgpu_job *job, struct amdgpu_bo
> *gds, struct amdgpu_bo *gws, struct amdgpu_bo *oa);
>  void amdgpu_job_free_resources(struct amdgpu_job *job);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c index 63ee6ba6a931..988ad86971a8
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c
> @@ -196,8 +196,9 @@ static int amdgpu_jpeg_dec_set_reg(struct amdgpu_ring
> *ring, uint32_t handle, int i, r;
> 
>  	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, ib_size_dw * 
4,
> -				     AMDGPU_IB_POOL_DIRECT, &job,
> -				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
> +				     AMDGPU_IB_POOL_DIRECT,
> +				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
> +				     &job);
>  	if (r)
>  		return r;
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 6c6ab4dd6ea9..cf78d7020494
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> @@ -243,8 +243,9 @@ static int amdgpu_ttm_map_buffer(struct
> amdgpu_ttm_buffer_entity *entity, r = amdgpu_job_alloc_with_ib(adev,
> &entity->base,
>  				     
AMDGPU_FENCE_OWNER_UNDEFINED,
>  				     num_dw * 4 + num_bytes,
> -				     AMDGPU_IB_POOL_DELAYED, 
&job,
> -				     
AMDGPU_KERNEL_JOB_ID_TTM_MAP_BUFFER);
> +				     AMDGPU_IB_POOL_DELAYED,
> +				     
AMDGPU_KERNEL_JOB_ID_TTM_MAP_BUFFER,
> +				     &job);
>  	if (r)
>  		return r;
> 
> @@ -1573,8 +1574,8 @@ static int amdgpu_ttm_access_memory_sdma(struct
> ttm_buffer_object *bo, r = amdgpu_job_alloc_with_ib(adev,
> &adev->mman.default_entity.base, AMDGPU_FENCE_OWNER_UNDEFINED,
>  				     num_dw * 4, 
AMDGPU_IB_POOL_DELAYED,
> -				     &job,
> -				     
AMDGPU_KERNEL_JOB_ID_TTM_ACCESS_MEMORY_SDMA);
> +				     
AMDGPU_KERNEL_JOB_ID_TTM_ACCESS_MEMORY_SDMA,
> +				     &job);
>  	if (r)
>  		goto out;
> 
> @@ -2428,7 +2429,7 @@ static int amdgpu_ttm_prepare_job(struct amdgpu_device
> *adev, int r;
>  	r = amdgpu_job_alloc_with_ib(adev, &entity->base,
>  				     
AMDGPU_FENCE_OWNER_UNDEFINED,
> -				     num_dw * 4, pool, job, 
k_job_id);
> +				     num_dw * 4, pool, k_job_id, 
job);
>  	if (r)
>  		return r;
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index 3a3bc0d370fa..7df839634098
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
> @@ -1137,8 +1137,9 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring
> *ring, struct amdgpu_bo *bo, r = amdgpu_job_alloc_with_ib(ring->adev,
> &adev->uvd.entity,
>  				     
AMDGPU_FENCE_OWNER_UNDEFINED,
>  				     64, direct ? 
AMDGPU_IB_POOL_DIRECT :
> -				     AMDGPU_IB_POOL_DELAYED, 
&job,
> -				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
> +				     AMDGPU_IB_POOL_DELAYED,
> +				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
> +				     &job);
>  	if (r)
>  		return r;
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index efdebd9c0a1f..344a703d03be
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
> @@ -473,7 +473,8 @@ static int amdgpu_vce_get_create_msg(struct amdgpu_ring
> *ring, uint32_t handle, r = amdgpu_job_alloc_with_ib(ring->adev,
> &ring->adev->vce.entity, AMDGPU_FENCE_OWNER_UNDEFINED,
>  				     ib_size_dw * 4, 
AMDGPU_IB_POOL_DIRECT,
> -				     &job, 
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
> +				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
> +				     &job);
>  	if (r)
>  		return r;
> 
> @@ -564,8 +565,9 @@ static int amdgpu_vce_get_destroy_msg(struct amdgpu_ring
> *ring, uint32_t handle, AMDGPU_FENCE_OWNER_UNDEFINED,
>  				     ib_size_dw * 4,
>  				     direct ? 
AMDGPU_IB_POOL_DIRECT :
> -				     AMDGPU_IB_POOL_DELAYED, 
&job,
> -				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
> +				     AMDGPU_IB_POOL_DELAYED,
> +				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
> +				     &job);
>  	if (r)
>  		return r;
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c index 616967519869..6b15415f989c
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
> @@ -631,7 +631,8 @@ static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring
> *ring,
> 
>  	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL,
>  				     64, AMDGPU_IB_POOL_DIRECT,
> -				     &job, 
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
> +				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
> +				     &job);
>  	if (r)
>  		goto err;
> 
> @@ -811,7 +812,8 @@ static int amdgpu_vcn_dec_sw_send_msg(struct amdgpu_ring
> *ring,
> 
>  	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL,
>  				     ib_size_dw * 4, 
AMDGPU_IB_POOL_DIRECT,
> -				     &job, 
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
> +				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
> +				     &job);
>  	if (r)
>  		goto err;
> 
> @@ -941,7 +943,8 @@ static int amdgpu_vcn_enc_get_create_msg(struct
> amdgpu_ring *ring, uint32_t hand
> 
>  	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL,
>  				     ib_size_dw * 4, 
AMDGPU_IB_POOL_DIRECT,
> -				     &job, 
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
> +				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
> +				     &job);
>  	if (r)
>  		return r;
> 
> @@ -1008,7 +1011,8 @@ static int amdgpu_vcn_enc_get_destroy_msg(struct
> amdgpu_ring *ring, uint32_t han
> 
>  	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL,
>  				     ib_size_dw * 4, 
AMDGPU_IB_POOL_DIRECT,
> -				     &job, 
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
> +				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
> +				     &job);
>  	if (r)
>  		return r;
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c index
> 2eb64df6daa9..50cc0779c340 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
> @@ -56,7 +56,7 @@ static int amdgpu_vm_sdma_alloc_job(struct
> amdgpu_vm_update_params *p, ndw = min(ndw, AMDGPU_VM_SDMA_MAX_NUM_DW);
> 
>  	r = amdgpu_job_alloc_with_ib(p->adev, entity, 
AMDGPU_FENCE_OWNER_VM,
> -				     ndw * 4, pool, &p->job, 
k_job_id);
> +				     ndw * 4, pool, k_job_id, &p-
>job);
>  	if (r)
>  		return r;
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
> b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c index ecd7ead7a60b..16137829fe84
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
> @@ -217,8 +217,9 @@ static int uvd_v6_0_enc_get_create_msg(struct
> amdgpu_ring *ring, uint32_t handle int i, r;
> 
>  	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, ib_size_dw * 
4,
> -				     AMDGPU_IB_POOL_DIRECT, &job,
> -				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
> +				     AMDGPU_IB_POOL_DIRECT,
> +				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
> +				     &job);
>  	if (r)
>  		return r;
> 
> @@ -282,8 +283,9 @@ static int uvd_v6_0_enc_get_destroy_msg(struct
> amdgpu_ring *ring, int i, r;
> 
>  	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, ib_size_dw * 
4,
> -				     AMDGPU_IB_POOL_DIRECT, &job,
> -				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
> +				     AMDGPU_IB_POOL_DIRECT,
> +				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
> +				     &job);
>  	if (r)
>  		return r;
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
> b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c index df2c83348315..46222fc30be6
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
> @@ -225,8 +225,9 @@ static int uvd_v7_0_enc_get_create_msg(struct
> amdgpu_ring *ring, u32 handle, int i, r;
> 
>  	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, ib_size_dw * 
4,
> -				     AMDGPU_IB_POOL_DIRECT, &job,
> -				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
> +				     AMDGPU_IB_POOL_DIRECT,
> +				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
> +				     &job);
>  	if (r)
>  		return r;
> 
> @@ -289,8 +290,9 @@ static int uvd_v7_0_enc_get_destroy_msg(struct
> amdgpu_ring *ring, u32 handle, int i, r;
> 
>  	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, ib_size_dw * 
4,
> -				     AMDGPU_IB_POOL_DIRECT, &job,
> -				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
> +				     AMDGPU_IB_POOL_DIRECT,
> +				     
AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
> +				     &job);
>  	if (r)
>  		return r;
> 
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index 28dc6886c1ff..51ad46c25c5a
> 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> @@ -68,8 +68,8 @@ svm_migrate_gart_map(struct amdgpu_ring *ring,
>  				     
AMDGPU_FENCE_OWNER_UNDEFINED,
>  				     num_dw * 4 + num_bytes,
>  				     AMDGPU_IB_POOL_DELAYED,
> -				     &job,
> -				     
AMDGPU_KERNEL_JOB_ID_KFD_GART_MAP);
> +				     
AMDGPU_KERNEL_JOB_ID_KFD_GART_MAP,
> +				     &job);
>  	if (r)
>  		return r;

Re: [PATCH 05/13] drm/amdgpu: use correct gfp_t for job allocation

[Timur Kristóf]

On 2026. május 29., péntek 13:24:07 közép-európai nyári idő Christian König 
wrote:
> For job allocation in GPU reset and page fault handling we must use
> GFP_ATOMIC to guarantee that we don't cycle back and depend on a
> dma_fence submission for the memory allocation.

Can you please edit the commit message to elaborate  a little more?
For example, please add something like this:

Add gfp_flags argument to amdgpu_job_alloc() and expose the
gfp_flags of IB pools with amdgpu_ib_pool_gfp_flags() so that
we can use different flags when allocating jobs.

(Feel free to edit the text to improve it to your tastes.)

> 
> Co-developed by Claude Sonnet 4.

This tag seems inconsistent with the kernel guidelines and also inconsistent 
with commits from other AMD engineers. Can you please change this to use the
Assisted-by tag? Also in the other commits in this series.

Link to the guidelines:
https://docs.kernel.org/process/coding-assistants.html

Otherwise, the code looks fine.

Reviewed-by: Timur Kristóf <timur.kristof@gmail.com>

> 
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c |  3 ++-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c     |  2 +-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c     | 11 +++++++++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_job.c    |  9 +++++----
>  drivers/gpu/drm/amd/amdgpu/amdgpu_job.h    |  2 +-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h   |  2 ++
>  6 files changed, 22 insertions(+), 7 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index
> 44751d71b741..c425f960361c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> @@ -696,7 +696,8 @@ int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev,
>  		goto err;
>  	}
> 
> -	ret = amdgpu_job_alloc(adev, NULL, NULL, NULL, 1, 0, &job);
> +	ret = amdgpu_job_alloc(adev, NULL, NULL, NULL, 1, 0, GFP_KERNEL,
> +			       &job);
>  	if (ret)
>  		goto err;
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index fdf01d824d66..3ed2cb9dfa38
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> @@ -275,7 +275,7 @@ static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p,
>  	for (i = 0; i < p->gang_size; ++i) {
>  		ret = amdgpu_job_alloc(p->adev, vm, p->entities[i], vm,
>  				       num_ibs[i], p->filp-
>client_id,
> -				       &p->jobs[i]);
> +				       GFP_KERNEL, &p->jobs[i]);
>  		if (ret)
>  			goto free_all_kdata;
>  		switch (p->adev->enforce_isolation[fpriv->xcp_id]) {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 60e4c3985029..83a802cf5837
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
> @@ -416,6 +416,17 @@ void amdgpu_ib_pool_fini(struct amdgpu_device *adev)
>  	adev->ib_pool_ready = false;
>  }
> 
> +/**
> + * amdgpu_ib_pool_gfp_flags - Returns the gfp flags to use for each pool
> + * @adev: amdgpu device pointer
> + * @type: the IB pool type
> + */
> +gfp_t amdgpu_ib_pool_gfp_flags(struct amdgpu_device *adev,
> +			       enum amdgpu_ib_pool_type type)
> +{
> +	return adev->ib_pools[type].gfp_flags;
> +}
> +
>  /**
>   * amdgpu_ib_ring_tests - test IBs on the rings
>   *
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index 71c1ba735a6b..b4b424d84479
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> @@ -188,7 +188,7 @@ static enum drm_gpu_sched_stat
> amdgpu_job_timedout(struct drm_sched_job *s_job) int
> amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct
> drm_sched_entity *entity, void *owner,
>  		     unsigned int num_ibs, u64 drm_client_id,
> -		     struct amdgpu_job **job)
> +		     gfp_t gfp_flags, struct amdgpu_job **job)
>  {
>  	struct amdgpu_fence *af;
>  	int r;
> @@ -196,18 +196,18 @@ int amdgpu_job_alloc(struct amdgpu_device *adev,
> struct amdgpu_vm *vm, if (num_ibs == 0)
>  		return -EINVAL;
> 
> -	*job = kzalloc(struct_size(*job, ibs, num_ibs), GFP_KERNEL);
> +	*job = kzalloc(struct_size(*job, ibs, num_ibs), gfp_flags);
>  	if (!*job)
>  		return -ENOMEM;
> 
> -	af = kzalloc(sizeof(struct amdgpu_fence), GFP_KERNEL);
> +	af = kzalloc(sizeof(struct amdgpu_fence), gfp_flags);
>  	if (!af) {
>  		r = -ENOMEM;
>  		goto err_job;
>  	}
>  	(*job)->hw_fence = af;
> 
> -	af = kzalloc(sizeof(struct amdgpu_fence), GFP_KERNEL);
> +	af = kzalloc(sizeof(struct amdgpu_fence), gfp_flags);
>  	if (!af) {
>  		r = -ENOMEM;
>  		goto err_fence;
> @@ -246,6 +246,7 @@ int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev,
> int r;
> 
>  	r = amdgpu_job_alloc(adev, NULL, entity, owner, 1, k_job_id,
> +			     amdgpu_ib_pool_gfp_flags(adev, 
pool_type),
>  			     job);
>  	if (r)
>  		return r;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h index 6b7cf594714c..44fe40f9e8df
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> @@ -113,7 +113,7 @@ static inline struct amdgpu_ring *amdgpu_job_ring(struct
> amdgpu_job *job) int amdgpu_job_alloc(struct amdgpu_device *adev, struct
> amdgpu_vm *vm, struct drm_sched_entity *entity, void *owner,
>  		     unsigned int num_ibs, u64 drm_client_id,
> -		     struct amdgpu_job **job);
> +		     gfp_t gfp_flags, struct amdgpu_job **job);
>  int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev,
>  			     struct drm_sched_entity *entity, void 
*owner,
>  			     size_t size, enum amdgpu_ib_pool_type 
pool_type,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h index 1a063a0a4280..4bf9734acc18
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> @@ -582,6 +582,8 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring,
> unsigned num_ibs, struct dma_fence **f);
>  int amdgpu_ib_pool_init(struct amdgpu_device *adev);
>  void amdgpu_ib_pool_fini(struct amdgpu_device *adev);
> +gfp_t amdgpu_ib_pool_gfp_flags(struct amdgpu_device *adev,
> +			       enum amdgpu_ib_pool_type type);
>  int amdgpu_ib_ring_tests(struct amdgpu_device *adev);
>  bool amdgpu_ring_sched_ready(struct amdgpu_ring *ring);
>  void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring,

Re: [PATCH 06/13] drm/amdgpu: add amdgpu_vm_update_leaves()

[Timur Kristóf]

On 2026. május 29., péntek 13:24:08 közép-európai nyári idő Christian König 
wrote:
> Add a new function amdgpu_vm_update_leaves() to avoid memory allocation
> on page faults.
> 
> The idea is to only update the leave PTEs to insert a dummy PRT PTE.

We should only use PRT PTEs on GPUs that do not have the SMEM PRT bug,
in other words only GFX9 for now.

As-is, this basically regresses all the work that I did for retry faults.

Due to the SMEM PRT bug, using PRT PTEs is unacceptable because it would imply 
that we can only mitigate VM faults that come from VMEM instructions. That 
creates unnecessary inconsistency (inability to mitigate some VM faults) and 
poor user experience (GPU hangs).

> 
> TODO: HW older than GMC v9 needs a different solution.

Can you elaborate what different solution is needed?
As far as I know, GMC v6-v7-v8 do not use and do not need to call the 
amdgpu_vm_handle_fault() function, they can just mitigate page faults on their 
own.

> 
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 40 ++++++++++---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h    |  3 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 71 ++++++++++++++++++++++-
>  3 files changed, 103 insertions(+), 11 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index e5588346a03f..94632a660b79
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -2984,10 +2984,12 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device
> *adev, u32 pasid, u32 vmid, u32 node_id, uint64_t addr,
>  			    uint64_t ts, bool write_fault)
>  {
> -	bool is_compute_context = false;
> +	struct amdgpu_vm_update_params params;
> +	bool is_compute_context;
>  	struct amdgpu_bo *root;
>  	uint64_t value, flags;
>  	struct amdgpu_vm *vm;
> +	unsigned int idx;
>  	int r;
> 
>  	vm = amdgpu_vm_lock_by_pasid(adev, &root, pasid);
> @@ -3029,24 +3031,46 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device
> *adev, u32 pasid, value = adev->dummy_page_addr;
>  		flags |= AMDGPU_PTE_EXECUTABLE | AMDGPU_PTE_READABLE |
>  			AMDGPU_PTE_WRITEABLE;
> -
> +		/* On +gfx9 we can use the PRT functionality instead */
> +		if (!adev->gmc.gmc_funcs->set_prt) {
> +			flags &= ~AMDGPU_PTE_VALID;
> +			flags |= AMDGPU_PTE_PRT;
> +		}

As noted above, PRT should only be used on GFX9 (and potentially, future 
products where the SMEM PRT bug is fixed).

Also, checking the set_prt is not necessary here as this function only makes 
sense on GFX9 and newer.

Maybe we could help the confusion by documenting both amdgpu_vm_handle_fault() 
and amdgpu_gmc_handle_retry_fault() that they are only applicable to GFX9 and 
newer that support recoverable faults or retry faults. (In a separate commit 
from this one, though.)

>  	} else {
>  		/* Let the hw retry silently on the PTE */
>  		value = 0;
>  	}
> 
> +	if (!drm_dev_enter(adev_to_drm(adev), &idx)) {
> +		r = -ENODEV;
> +		goto error_unlock;
> +	}
> +
> +	amdgpu_vm_eviction_lock(vm);
> +	if (vm->evicting) {
> +		r = -EBUSY;
> +		goto error_dev_exit;
> +	}
> +
> +	memset(&params, 0, sizeof(params));
> +	params.adev = adev;
> +	params.vm = vm;
> +	params.immediate = true;
> +	params.pages_addr = NULL;
> +
>  	r = dma_resv_reserve_fences(root->tbo.base.resv, 1);
>  	if (r) {
>  		pr_debug("failed %d to reserve fence slot\n", r);
> -		goto error_unlock;
> +		goto error_eviction_lock;
>  	}
> 
> -	r = amdgpu_vm_update_range(adev, vm, true, false, false, false,
> -				   NULL, addr, addr, flags, value, 
0, NULL, NULL, NULL);
> -	if (r)
> -		goto error_unlock;
> +	amdgpu_vm_update_leaves(&params, addr, addr, value, flags);
> 
> -	r = amdgpu_vm_update_pdes(adev, vm, true);
> +error_eviction_lock:
> +	amdgpu_vm_eviction_unlock(vm);
> +
> +error_dev_exit:
> +	drm_dev_exit(idx);
> 
>  error_unlock:
>  	amdgpu_bo_unreserve(root);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index cc096c005e34..04b32accfa3f
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> @@ -613,6 +613,9 @@ int amdgpu_vm_pde_update(struct amdgpu_vm_update_params
> *params, int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
> uint64_t start, uint64_t end,
>  			  uint64_t dst, uint64_t flags);
> +void amdgpu_vm_update_leaves(struct amdgpu_vm_update_params *params,
> +			     uint64_t start, uint64_t end,
> +			     int64_t dst, uint64_t flags);
>  void amdgpu_vm_pt_free_work(struct work_struct *work);
>  void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
>  			    struct amdgpu_vm_update_params *params);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c index
> e43a60d09808..9766b6b9aecc 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> @@ -790,6 +790,8 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params
> *params, uint64_t dst, uint64_t flags)
>  {
>  	struct amdgpu_device *adev = params->adev;
> +	struct amdgpu_vm *vm = params->vm;
> +	pid_t tgid = vm->task_info ? vm->task_info->tgid : 0;

These spurious changes in amdgpu_vm_ptes_update() look unrelated the rest of 
the commit, so I think they should either be split off into a separate commit 
or explained in the commit message why they are necessary here.

>  	struct amdgpu_vm_pt_cursor cursor;
>  	uint64_t frag_start = start, frag_end;
>  	unsigned int frag;
> @@ -881,7 +883,6 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params
> *params, entry_end = min(entry_end, end);
> 
>  		do {
> -			struct amdgpu_vm *vm = params->vm;
>  			uint64_t upd_end = min(entry_end, frag_end);
>  			unsigned int nptes = (upd_end - frag_start) 
>> shift;
>  			uint64_t upd_flags = flags | 
AMDGPU_PTE_FRAG(frag);
> @@ -893,8 +894,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params
> *params,
> 
>  			trace_amdgpu_vm_update_ptes(params, 
frag_start, upd_end,
>  						    
min(nptes, 32u), dst, incr,
> -						    
upd_flags,
> -						    vm-
>task_info ? vm->task_info->tgid : 0,
> +						    
upd_flags, tgid,
>  						    vm-
>immediate.fence_context);
>  			amdgpu_vm_pte_update_flags(params, 
to_amdgpu_bo_vm(pt),
>  						   
cursor.level, pe_start, dst,
> @@ -938,6 +938,71 @@ int amdgpu_vm_ptes_update(struct
> amdgpu_vm_update_params *params, return 0;
>  }
> 
> +/**
> + * amdgpu_vm_update_leaves - update leave PDEs/PTEs
> + *
> + * @params: see amdgpu_vm_update_params definition
> + * @start: start of GPU address range
> + * @end: end of GPU address range
> + * @dst: destination address to insert into the leave PDEs/PTEs
> + * @flags: mapping flags
> + *
> + * Update the leave PDEs/PTEs in the range @start - @end without allocating
> or + * freeing page tables.
> + *
> + * Returns:
> + * 0 for success, negative error code for failure.
> + */
> +void amdgpu_vm_update_leaves(struct amdgpu_vm_update_params *params,
> +			     uint64_t start, uint64_t end,
> +			     int64_t dst, uint64_t flags)
> +{
> +	struct amdgpu_device *adev = params->adev;
> +	struct amdgpu_vm *vm = params->vm;
> +	pid_t tgid = vm->task_info ? vm->task_info->tgid : 0;
> +	struct amdgpu_vm_pt_cursor cursor;
> +
> +	amdgpu_vm_pt_start(adev, params->vm, start, &cursor);
> +	while (cursor.pfn < end) {
> +		unsigned int shift, mask;
> +		uint64_t entry_end, pe_start;
> +		struct amdgpu_bo *pt;
> +		unsigned int nptes;
> +
> +		/* Walk to the leave entries */
> +		if (amdgpu_vm_pt_descendant(adev, &cursor))
> +			continue;
> +
> +		pt = cursor.parent->bo;
> +		shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 
1);
> +		mask = amdgpu_vm_pt_entries_mask(adev, cursor.level - 
1);
> +
> +		/* Looks good so far, calculate parameters for the 
update */
> +		pe_start = ((cursor.pfn >> shift) & mask) * 8;
> +
> +		entry_end = ((uint64_t)mask + 1) << shift;
> +		entry_end += cursor.pfn & ~(entry_end - 1);
> +		entry_end = min(entry_end, end);
> +
> +		nptes = (entry_end - cursor.pfn) >> shift;
> +		/*
> +		 * This can happen when we set higher level PDEs to 
unmap and/or
> +		 * silent to stop fault floods.
> +		 */
> +		nptes = max(nptes, 1u);
> +
> +		trace_amdgpu_vm_update_ptes(params, cursor.pfn, 
entry_end,
> +					    min(nptes, 32u), 
dst, 0, flags,
> +					    tgid,
> +					    vm-
>immediate.fence_context);
> +		amdgpu_vm_pte_update_flags(params, to_amdgpu_bo_vm(pt),
> +					   cursor.level - 1, 
pe_start, dst,
> +					   nptes, 0, flags);
> +
> +		amdgpu_vm_pt_next(adev, &cursor);
> +	}
> +}
> +
>  /**
>   * amdgpu_vm_pt_map_tables - have bo of root PD cpu accessible
>   * @adev: amdgpu device structure

Re: [PATCH 07/13] drm/amdgpu: drop immediate updates from amdgpu_vm_update_range

[Timur Kristóf]

On 2026. május 29., péntek 13:24:09 közép-európai nyári idő Christian König 
wrote:
> That case is handled by amdgpu_vm_update_leaves now.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>

This commit seems to be doing more than suggested by the (very short) commit 
message. It removes immediate updates not only from amdgpu_vm_update_range() 
but also amdgpu_vm_pt_clear() and amdgpu_vm_pt_create().

Otherwise the code looks good.

Reviewed-by: Timur Kristóf <timur.kristof@gmail.com>


> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 21 +++++++++------------
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h    | 16 +++++++---------
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 19 ++++++-------------
>  drivers/gpu/drm/amd/amdkfd/kfd_svm.c      |  4 ++--
>  4 files changed, 24 insertions(+), 36 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 94632a660b79..edc8b1ca2d3e
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -1084,7 +1084,6 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params
> *params, *
>   * @adev: amdgpu_device pointer to use for commands
>   * @vm: the VM to update the range
> - * @immediate: immediate submission in a page fault
>   * @unlocked: unlocked invalidation during MM callback
>   * @flush_tlb: trigger tlb invalidation after update completed
>   * @allow_override: change MTYPE for local NUMA nodes
> @@ -1104,12 +1103,11 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params
> *params, * 0 for success, negative erro code for failure.
>   */
>  int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm
> *vm, -			   bool immediate, bool unlocked, bool 
flush_tlb,
> -			   bool allow_override, struct amdgpu_sync 
*sync,
> -			   uint64_t start, uint64_t last, uint64_t 
flags,
> -			   uint64_t offset, uint64_t vram_base,
> -			   struct ttm_resource *res, dma_addr_t 
*pages_addr,
> -			   struct dma_fence **fence)
> +			   bool unlocked, bool flush_tlb, bool 
allow_override,
> +			   struct amdgpu_sync *sync, uint64_t start,
> +			   uint64_t last, uint64_t flags, uint64_t 
offset,
> +			   uint64_t vram_base, struct ttm_resource 
*res,
> +			   dma_addr_t *pages_addr, struct dma_fence 
**fence)
>  {
>  	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
>  	struct amdgpu_vm_update_params params;
> @@ -1139,7 +1137,6 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev,
> struct amdgpu_vm *vm, memset(&params, 0, sizeof(params));
>  	params.adev = adev;
>  	params.vm = vm;
> -	params.immediate = immediate;
>  	params.pages_addr = pages_addr;
>  	params.unlocked = unlocked;
>  	params.needs_flush = flush_tlb;
> @@ -1365,7 +1362,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
> struct amdgpu_bo_va *bo_va,
> 
>  		trace_amdgpu_vm_bo_update(mapping);
> 
> -		r = amdgpu_vm_update_range(adev, vm, false, false, 
flush_tlb,
> +		r = amdgpu_vm_update_range(adev, vm, false, flush_tlb,
>  					   !uncached, &sync, 
mapping->start,
>  					   mapping->last, 
update_flags,
>  					   mapping->offset, 
vram_base, mem,
> @@ -1568,7 +1565,7 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
>  			struct amdgpu_bo_va_mapping, list);
>  		list_del(&mapping->list);
> 
> -		r = amdgpu_vm_update_range(adev, vm, false, false, 
true, false,
> +		r = amdgpu_vm_update_range(adev, vm, false, true, false,
>  					   &sync, mapping-
>start, mapping->last,
>  					   0, 0, 0, NULL, 
NULL, &f);
>  		amdgpu_vm_free_mapping(adev, vm, mapping, f);
> @@ -2617,7 +2614,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct
> amdgpu_vm *vm, vm->tlb_fence_context = dma_fence_context_alloc(1);
> 
>  	r = amdgpu_vm_pt_create(adev, vm, adev->vm_manager.root_level,
> -				false, &root, xcp_id);
> +				&root, xcp_id);
>  	if (r)
>  		goto error_free_delayed;
> 
> @@ -2633,7 +2630,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct
> amdgpu_vm *vm, if (r)
>  		goto error_free_root;
> 
> -	r = amdgpu_vm_pt_clear(adev, vm, root, false);
> +	r = amdgpu_vm_pt_clear(adev, vm, root);
>  	if (r)
>  		goto error_free_root;
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index 04b32accfa3f..3e86a2a470f0
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> @@ -530,12 +530,11 @@ int amdgpu_vm_flush_compute_tlb(struct amdgpu_device
> *adev, void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
>  			    struct amdgpu_vm *vm, struct amdgpu_bo 
*bo);
>  int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm
> *vm, -			   bool immediate, bool unlocked, bool 
flush_tlb,
> -			   bool allow_override, struct amdgpu_sync 
*sync,
> -			   uint64_t start, uint64_t last, uint64_t 
flags,
> -			   uint64_t offset, uint64_t vram_base,
> -			   struct ttm_resource *res, dma_addr_t 
*pages_addr,
> -			   struct dma_fence **fence);
> +			   bool unlocked, bool flush_tlb, bool 
allow_override,
> +			   struct amdgpu_sync *sync, uint64_t start,
> +			   uint64_t last, uint64_t flags, uint64_t 
offset,
> +			   uint64_t vram_base, struct ttm_resource 
*res,
> +			   dma_addr_t *pages_addr, struct dma_fence 
**fence);
>  int amdgpu_vm_bo_update(struct amdgpu_device *adev,
>  			struct amdgpu_bo_va *bo_va,
>  			bool clear);
> @@ -602,10 +601,9 @@ void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
>  			  struct amdgpu_mem_stats 
stats[__AMDGPU_PL_NUM]);
> 
>  int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> -		       struct amdgpu_bo_vm *vmbo, bool immediate);
> +		       struct amdgpu_bo_vm *vmbo);
>  int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> -			int level, bool immediate, struct 
amdgpu_bo_vm **vmbo,
> -			int32_t xcp_id);
> +			int level, struct amdgpu_bo_vm **vmbo, 
int32_t xcp_id);
>  void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm
> *vm);
> 
>  int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c index
> 9766b6b9aecc..6f5415d5a1bc 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> @@ -351,7 +351,6 @@ static void amdgpu_vm_pt_next_dfs(struct amdgpu_device
> *adev, * @adev: amdgpu_device pointer
>   * @vm: VM to clear BO from
>   * @vmbo: BO to clear
> - * @immediate: use an immediate update
>   *
>   * Root PD needs to be reserved when calling this.
>   *
> @@ -359,7 +358,7 @@ static void amdgpu_vm_pt_next_dfs(struct amdgpu_device
> *adev, * 0 on success, errno otherwise.
>   */
>  int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> -		       struct amdgpu_bo_vm *vmbo, bool immediate)
> +		       struct amdgpu_bo_vm *vmbo)
>  {
>  	unsigned int level = adev->vm_manager.root_level;
>  	struct ttm_operation_ctx ctx = { true, false };
> @@ -396,7 +395,6 @@ int amdgpu_vm_pt_clear(struct amdgpu_device *adev,
> struct amdgpu_vm *vm, memset(&params, 0, sizeof(params));
>  	params.adev = adev;
>  	params.vm = vm;
> -	params.immediate = immediate;
> 
>  	r = vm->update_funcs->prepare(&params, NULL,
>  				      
AMDGPU_KERNEL_JOB_ID_VM_PT_CLEAR);
> @@ -434,13 +432,11 @@ int amdgpu_vm_pt_clear(struct amdgpu_device *adev,
> struct amdgpu_vm *vm, * @adev: amdgpu_device pointer
>   * @vm: requesting vm
>   * @level: the page table level
> - * @immediate: use a immediate update
>   * @vmbo: pointer to the buffer object pointer
>   * @xcp_id: GPU partition id
>   */
>  int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> -			int level, bool immediate, struct 
amdgpu_bo_vm **vmbo,
> -			int32_t xcp_id)
> +			int level, struct amdgpu_bo_vm **vmbo, 
int32_t xcp_id)
>  {
>  	struct amdgpu_bo_param bp;
>  	unsigned int num_entries;
> @@ -470,7 +466,6 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev,
> struct amdgpu_vm *vm, bp.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
> 
>  	bp.type = ttm_bo_type_kernel;
> -	bp.no_wait_gpu = immediate;
>  	bp.xcp_id_plus1 = xcp_id + 1;
> 
>  	if (vm->root.bo)
> @@ -485,7 +480,6 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev,
> struct amdgpu_vm *vm, * @adev: amdgpu_device pointer
>   * @vm: VM to allocate page tables for
>   * @cursor: Which page table to allocate
> - * @immediate: use an immediate update
>   *
>   * Make sure a specific page table or directory is allocated.
>   *
> @@ -495,8 +489,7 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev,
> struct amdgpu_vm *vm, */
>  static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev,
>  			      struct amdgpu_vm *vm,
> -			      struct amdgpu_vm_pt_cursor *cursor,
> -			      bool immediate)
> +			      struct amdgpu_vm_pt_cursor *cursor)
>  {
>  	struct amdgpu_vm_bo_base *entry = cursor->entry;
>  	struct amdgpu_bo *pt_bo;
> @@ -507,7 +500,7 @@ static int amdgpu_vm_pt_alloc(struct amdgpu_device
> *adev, return 0;
> 
>  	amdgpu_vm_eviction_unlock(vm);
> -	r = amdgpu_vm_pt_create(adev, vm, cursor->level, immediate, &pt,
> +	r = amdgpu_vm_pt_create(adev, vm, cursor->level, &pt,
>  				vm->root.bo->xcp_id);
>  	amdgpu_vm_eviction_lock(vm);
>  	if (r)
> @@ -519,7 +512,7 @@ static int amdgpu_vm_pt_alloc(struct amdgpu_device
> *adev, pt_bo = &pt->bo;
>  	pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo);
>  	amdgpu_vm_bo_base_init(entry, vm, pt_bo);
> -	r = amdgpu_vm_pt_clear(adev, vm, pt, immediate);
> +	r = amdgpu_vm_pt_clear(adev, vm, pt);
>  	if (r)
>  		goto error_free_pt;
> 
> @@ -813,7 +806,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params
> *params, * address range are actually allocated
>  			 */
>  			r = amdgpu_vm_pt_alloc(params->adev, params-
>vm,
> -					       &cursor, 
params->immediate);
> +					       &cursor);
>  			if (r)
>  				return r;
>  		}
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 72cfb4a6ab3e..37b5166e9a14
> 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> @@ -1372,7 +1372,7 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev,
> struct amdgpu_vm *vm, return -EINVAL;
>  	}
> 
> -	return amdgpu_vm_update_range(adev, vm, false, true, true, false, 
NULL,
> gpu_start, +	return amdgpu_vm_update_range(adev, vm, true, true, 
false,
> NULL, gpu_start, gpu_end, init_pte_value, 0, 0, NULL, NULL,
>  				      fence);
>  }
> @@ -1489,7 +1489,7 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd,
> struct svm_range *prange, (last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0,
>  			 pte_flags);
> 
> -		r = amdgpu_vm_update_range(adev, vm, false, false, 
flush_tlb, true,
> +		r = amdgpu_vm_update_range(adev, vm, false, flush_tlb, 
true,
>  					   NULL, gpu_start, 
gpu_end,
>  					   pte_flags,
>  					   (last_start - 
prange->start) << PAGE_SHIFT,