[PATCH 1/9] drm/amdgpu: lock both VM and BO in amdgpu_gem_object_open

[PATCH 1/9] drm/amdgpu: lock both VM and BO in amdgpu_gem_object_open

[Christian König]

The VM was not locked in the past since we initially only cleared the
linked list element and not added it to any VM state.

But this has changed quite some time ago, we just never realized this
problem because the VM state lock was masking it.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c  | 19 +++++++++++-----
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c       | 22 ++++++++++++++-----
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c       | 10 +++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        |  2 ++
 4 files changed, 42 insertions(+), 11 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 768998c82b43..ec5130497743 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -878,6 +878,7 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
 	struct amdgpu_bo *bo[2] = {NULL, NULL};
 	struct amdgpu_bo_va *bo_va;
 	bool same_hive = false;
+	struct drm_exec exec;
 	int i, ret;
 
 	if (!va) {
@@ -958,19 +959,25 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
 			goto unwind;
 		}
 
-		/* Add BO to VM internal data structures */
-		ret = amdgpu_bo_reserve(bo[i], false);
-		if (ret) {
-			pr_debug("Unable to reserve BO during memory attach");
-			goto unwind;
+		drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
+		drm_exec_until_all_locked(&exec) {
+			ret = amdgpu_vm_lock_pd(vm, &exec, 0);
+			drm_exec_retry_on_contention(&exec);
+			if (unlikely(ret))
+				goto unwind;
+			ret = drm_exec_lock_obj(&exec, &bo[i]->tbo.base);
+			drm_exec_retry_on_contention(&exec);
+			if (unlikely(ret))
+				goto unwind;
 		}
+
 		bo_va = amdgpu_vm_bo_find(vm, bo[i]);
 		if (!bo_va)
 			bo_va = amdgpu_vm_bo_add(adev, vm, bo[i]);
 		else
 			++bo_va->ref_count;
 		attachment[i]->bo_va = bo_va;
-		amdgpu_bo_unreserve(bo[i]);
+		drm_exec_fini(&exec);
 		if (unlikely(!attachment[i]->bo_va)) {
 			ret = -ENOMEM;
 			pr_err("Failed to add BO object to VM. ret == %d\n",
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 5f9fa2140f09..5c90de58cc28 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -232,6 +232,7 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
 	struct amdgpu_vm *vm = &fpriv->vm;
 	struct amdgpu_bo_va *bo_va;
 	struct mm_struct *mm;
+	struct drm_exec exec;
 	int r;
 
 	mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm);
@@ -242,9 +243,18 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
 	    !amdgpu_vm_is_bo_always_valid(vm, abo))
 		return -EPERM;
 
-	r = amdgpu_bo_reserve(abo, false);
-	if (r)
-		return r;
+	drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
+	drm_exec_until_all_locked(&exec) {
+		r = drm_exec_prepare_obj(&exec, &abo->tbo.base, 1);
+		drm_exec_retry_on_contention(&exec);
+		if (unlikely(r))
+			goto out_unlock;
+
+		r = amdgpu_vm_lock_pd(vm, &exec, 0);
+		drm_exec_retry_on_contention(&exec);
+		if (unlikely(r))
+			goto out_unlock;
+	}
 
 	amdgpu_vm_bo_update_shared(abo);
 	bo_va = amdgpu_vm_bo_find(vm, abo);
@@ -260,8 +270,7 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
 		amdgpu_bo_unreserve(abo);
 		return r;
 	}
-
-	amdgpu_bo_unreserve(abo);
+	drm_exec_fini(&exec);
 
 	/* Validate and add eviction fence to DMABuf imports with dynamic
 	 * attachment in compute VMs. Re-validation will be done by
@@ -294,7 +303,10 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
 		}
 	}
 	mutex_unlock(&vm->process_info->lock);
+	return r;
 
+out_unlock:
+	drm_exec_fini(&exec);
 	return r;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 1878e0faa722..f69332eed051 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -1445,6 +1445,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 {
 	struct amdgpu_device *adev = drm_to_adev(dev);
 	struct amdgpu_fpriv *fpriv;
+	struct drm_exec exec;
 	int r, pasid;
 
 	/* Ensure IB tests are run on ring */
@@ -1484,7 +1485,16 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 	if (r)
 		goto error_pasid;
 
+	drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
+	drm_exec_until_all_locked(&exec) {
+		r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 0);
+		drm_exec_retry_on_contention(&exec);
+		if (unlikely(r))
+			goto error_vm;
+	}
+
 	fpriv->prt_va = amdgpu_vm_bo_add(adev, &fpriv->vm, NULL);
+	drm_exec_fini(&exec);
 	if (!fpriv->prt_va) {
 		r = -ENOMEM;
 		goto error_vm;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 6a2ea200d90c..b4bf1b7c214f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -1735,6 +1735,8 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
 {
 	struct amdgpu_bo_va *bo_va;
 
+	amdgpu_vm_assert_locked(vm);
+
 	bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
 	if (bo_va == NULL) {
 		return NULL;
-- 
2.43.0

[PATCH 2/9] drm/amdgpu: revert to old status lock handling v3

[Christian König]

This reverts commit 7e64d0e5979157ee5fc83e58ac61b4a36803c7f1.

The problems were caused by not holding the VM lock while adding new
BOs.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c |   8 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 146 ++++++++--------------
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h    |  15 ++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c |   4 -
 4 files changed, 68 insertions(+), 105 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index b700c2b91465..8feeaeea1c36 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -1058,12 +1058,12 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
 	struct amdgpu_bo *bo;
 	int ret;
 
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->invalidated_lock);
 	while (!list_empty(&vm->invalidated)) {
 		bo_va = list_first_entry(&vm->invalidated,
 					 struct amdgpu_bo_va,
 					 base.vm_status);
-		spin_unlock(&vm->status_lock);
+		spin_unlock(&vm->invalidated_lock);
 
 		bo = bo_va->base.bo;
 		ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2);
@@ -1080,9 +1080,9 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
 		if (ret)
 			return ret;
 
-		spin_lock(&vm->status_lock);
+		spin_lock(&vm->invalidated_lock);
 	}
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->invalidated_lock);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index b4bf1b7c214f..a8a4fffc5764 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -153,12 +153,10 @@ static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
 
 	vm_bo->moved = true;
 	amdgpu_vm_assert_locked(vm);
-	spin_lock(&vm_bo->vm->status_lock);
 	if (bo->tbo.type == ttm_bo_type_kernel)
 		list_move(&vm_bo->vm_status, &vm->evicted);
 	else
 		list_move_tail(&vm_bo->vm_status, &vm->evicted);
-	spin_unlock(&vm_bo->vm->status_lock);
 }
 /**
  * amdgpu_vm_bo_moved - vm_bo is moved
@@ -171,9 +169,7 @@ static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
 static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
 {
 	amdgpu_vm_assert_locked(vm_bo->vm);
-	spin_lock(&vm_bo->vm->status_lock);
 	list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
-	spin_unlock(&vm_bo->vm->status_lock);
 }
 
 /**
@@ -187,9 +183,7 @@ static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
 static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
 {
 	amdgpu_vm_assert_locked(vm_bo->vm);
-	spin_lock(&vm_bo->vm->status_lock);
 	list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
-	spin_unlock(&vm_bo->vm->status_lock);
 	vm_bo->moved = false;
 }
 
@@ -203,9 +197,9 @@ static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
  */
 static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
 {
-	spin_lock(&vm_bo->vm->status_lock);
+	spin_lock(&vm_bo->vm->invalidated_lock);
 	list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
-	spin_unlock(&vm_bo->vm->status_lock);
+	spin_unlock(&vm_bo->vm->invalidated_lock);
 }
 
 /**
@@ -218,10 +212,9 @@ static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
  */
 static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)
 {
+	amdgpu_vm_assert_locked(vm_bo->vm);
 	vm_bo->moved = true;
-	spin_lock(&vm_bo->vm->status_lock);
 	list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user);
-	spin_unlock(&vm_bo->vm->status_lock);
 }
 
 /**
@@ -235,13 +228,10 @@ static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)
 static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
 {
 	amdgpu_vm_assert_locked(vm_bo->vm);
-	if (vm_bo->bo->parent) {
-		spin_lock(&vm_bo->vm->status_lock);
+	if (vm_bo->bo->parent)
 		list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
-		spin_unlock(&vm_bo->vm->status_lock);
-	} else {
+	else
 		amdgpu_vm_bo_idle(vm_bo);
-	}
 }
 
 /**
@@ -255,9 +245,7 @@ static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
 static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
 {
 	amdgpu_vm_assert_locked(vm_bo->vm);
-	spin_lock(&vm_bo->vm->status_lock);
 	list_move(&vm_bo->vm_status, &vm_bo->vm->done);
-	spin_unlock(&vm_bo->vm->status_lock);
 }
 
 /**
@@ -271,13 +259,13 @@ static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
 {
 	struct amdgpu_vm_bo_base *vm_bo, *tmp;
 
-	amdgpu_vm_assert_locked(vm);
-
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->invalidated_lock);
 	list_splice_init(&vm->done, &vm->invalidated);
 	list_for_each_entry(vm_bo, &vm->invalidated, vm_status)
 		vm_bo->moved = true;
+	spin_unlock(&vm->invalidated_lock);
 
+	amdgpu_vm_assert_locked(vm);
 	list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) {
 		struct amdgpu_bo *bo = vm_bo->bo;
 
@@ -287,14 +275,13 @@ static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
 		else if (bo->parent)
 			list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
 	}
-	spin_unlock(&vm->status_lock);
 }
 
 /**
  * amdgpu_vm_update_shared - helper to update shared memory stat
  * @base: base structure for tracking BO usage in a VM
  *
- * Takes the vm status_lock and updates the shared memory stat. If the basic
+ * Takes the vm stats_lock and updates the shared memory stat. If the basic
  * stat changed (e.g. buffer was moved) amdgpu_vm_update_stats need to be called
  * as well.
  */
@@ -307,7 +294,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base)
 	bool shared;
 
 	dma_resv_assert_held(bo->tbo.base.resv);
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->stats_lock);
 	shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
 	if (base->shared != shared) {
 		base->shared = shared;
@@ -319,7 +306,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base)
 			vm->stats[bo_memtype].drm.private += size;
 		}
 	}
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->stats_lock);
 }
 
 /**
@@ -344,11 +331,11 @@ void amdgpu_vm_bo_update_shared(struct amdgpu_bo *bo)
  *        be bo->tbo.resource
  * @sign: if we should add (+1) or subtract (-1) from the stat
  *
- * Caller need to have the vm status_lock held. Useful for when multiple update
+ * Caller need to have the vm stats_lock held. Useful for when multiple update
  * need to happen at the same time.
  */
 static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
-			    struct ttm_resource *res, int sign)
+					  struct ttm_resource *res, int sign)
 {
 	struct amdgpu_vm *vm = base->vm;
 	struct amdgpu_bo *bo = base->bo;
@@ -372,7 +359,8 @@ static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
 		 */
 		if (bo->flags & AMDGPU_GEM_CREATE_DISCARDABLE)
 			vm->stats[res_memtype].drm.purgeable += size;
-		if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(res_memtype)))
+		if (!(bo->preferred_domains &
+		      amdgpu_mem_type_to_domain(res_memtype)))
 			vm->stats[bo_memtype].evicted += size;
 	}
 }
@@ -391,9 +379,9 @@ void amdgpu_vm_update_stats(struct amdgpu_vm_bo_base *base,
 {
 	struct amdgpu_vm *vm = base->vm;
 
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->stats_lock);
 	amdgpu_vm_update_stats_locked(base, res, sign);
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->stats_lock);
 }
 
 /**
@@ -419,10 +407,10 @@ void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
 	base->next = bo->vm_bo;
 	bo->vm_bo = base;
 
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->stats_lock);
 	base->shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
 	amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1);
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->stats_lock);
 
 	if (!amdgpu_vm_is_bo_always_valid(vm, bo))
 		return;
@@ -481,25 +469,25 @@ int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
 	int ret;
 
 	/* We can only trust prev->next while holding the lock */
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->invalidated_lock);
 	while (!list_is_head(prev->next, &vm->done)) {
 		bo_va = list_entry(prev->next, typeof(*bo_va), base.vm_status);
 
 		bo = bo_va->base.bo;
 		if (bo) {
 			amdgpu_bo_ref(bo);
-			spin_unlock(&vm->status_lock);
+			spin_unlock(&vm->invalidated_lock);
 
 			ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 1);
 			amdgpu_bo_unref(&bo);
 			if (unlikely(ret))
 				return ret;
 
-			spin_lock(&vm->status_lock);
+			spin_lock(&vm->invalidated_lock);
 		}
 		prev = prev->next;
 	}
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->invalidated_lock);
 
 	return 0;
 }
@@ -595,7 +583,7 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		       void *param)
 {
 	uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm);
-	struct amdgpu_vm_bo_base *bo_base;
+	struct amdgpu_vm_bo_base *bo_base, *tmp;
 	struct amdgpu_bo *bo;
 	int r;
 
@@ -608,13 +596,7 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			return r;
 	}
 
-	spin_lock(&vm->status_lock);
-	while (!list_empty(&vm->evicted)) {
-		bo_base = list_first_entry(&vm->evicted,
-					   struct amdgpu_vm_bo_base,
-					   vm_status);
-		spin_unlock(&vm->status_lock);
-
+	list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {
 		bo = bo_base->bo;
 
 		r = validate(param, bo);
@@ -627,26 +609,21 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
 			amdgpu_vm_bo_relocated(bo_base);
 		}
-		spin_lock(&vm->status_lock);
 	}
-	while (ticket && !list_empty(&vm->evicted_user)) {
-		bo_base = list_first_entry(&vm->evicted_user,
-					   struct amdgpu_vm_bo_base,
-					   vm_status);
-		spin_unlock(&vm->status_lock);
 
-		bo = bo_base->bo;
-		dma_resv_assert_held(bo->tbo.base.resv);
+	if (ticket) {
+		list_for_each_entry_safe(bo_base, tmp, &vm->evicted_user,
+					 vm_status) {
+			bo = bo_base->bo;
+			dma_resv_assert_held(bo->tbo.base.resv);
 
-		r = validate(param, bo);
-		if (r)
-			return r;
-
-		amdgpu_vm_bo_invalidated(bo_base);
+			r = validate(param, bo);
+			if (r)
+				return r;
 
-		spin_lock(&vm->status_lock);
+			amdgpu_vm_bo_invalidated(bo_base);
+		}
 	}
-	spin_unlock(&vm->status_lock);
 
 	amdgpu_vm_eviction_lock(vm);
 	vm->evicting = false;
@@ -675,9 +652,7 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 	ret = !vm->evicting;
 	amdgpu_vm_eviction_unlock(vm);
 
-	spin_lock(&vm->status_lock);
 	ret &= list_empty(&vm->evicted);
-	spin_unlock(&vm->status_lock);
 
 	spin_lock(&vm->immediate.lock);
 	ret &= !vm->immediate.stopped;
@@ -967,18 +942,13 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 			  struct amdgpu_vm *vm, bool immediate)
 {
 	struct amdgpu_vm_update_params params;
-	struct amdgpu_vm_bo_base *entry;
+	struct amdgpu_vm_bo_base *entry, *tmp;
 	bool flush_tlb_needed = false;
-	LIST_HEAD(relocated);
 	int r, idx;
 
 	amdgpu_vm_assert_locked(vm);
 
-	spin_lock(&vm->status_lock);
-	list_splice_init(&vm->relocated, &relocated);
-	spin_unlock(&vm->status_lock);
-
-	if (list_empty(&relocated))
+	if (list_empty(&vm->relocated))
 		return 0;
 
 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
@@ -994,7 +964,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 	if (r)
 		goto error;
 
-	list_for_each_entry(entry, &relocated, vm_status) {
+	list_for_each_entry(entry, &vm->relocated, vm_status) {
 		/* vm_flush_needed after updating moved PDEs */
 		flush_tlb_needed |= entry->moved;
 
@@ -1010,9 +980,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 	if (flush_tlb_needed)
 		atomic64_inc(&vm->tlb_seq);
 
-	while (!list_empty(&relocated)) {
-		entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base,
-					 vm_status);
+	list_for_each_entry_safe(entry, tmp, &vm->relocated, vm_status) {
 		amdgpu_vm_bo_idle(entry);
 	}
 
@@ -1239,9 +1207,9 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
 			  struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
 {
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->stats_lock);
 	memcpy(stats, vm->stats, sizeof(*stats) * __AMDGPU_PL_NUM);
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->stats_lock);
 }
 
 /**
@@ -1608,29 +1576,24 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
 			   struct amdgpu_vm *vm,
 			   struct ww_acquire_ctx *ticket)
 {
-	struct amdgpu_bo_va *bo_va;
+	struct amdgpu_bo_va *bo_va, *tmp;
 	struct dma_resv *resv;
 	bool clear, unlock;
 	int r;
 
-	spin_lock(&vm->status_lock);
-	while (!list_empty(&vm->moved)) {
-		bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va,
-					 base.vm_status);
-		spin_unlock(&vm->status_lock);
-
+	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
 		/* Per VM BOs never need to bo cleared in the page tables */
 		r = amdgpu_vm_bo_update(adev, bo_va, false);
 		if (r)
 			return r;
-		spin_lock(&vm->status_lock);
 	}
 
+	spin_lock(&vm->invalidated_lock);
 	while (!list_empty(&vm->invalidated)) {
 		bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
 					 base.vm_status);
 		resv = bo_va->base.bo->tbo.base.resv;
-		spin_unlock(&vm->status_lock);
+		spin_unlock(&vm->invalidated_lock);
 
 		/* Try to reserve the BO to avoid clearing its ptes */
 		if (!adev->debug_vm && dma_resv_trylock(resv)) {
@@ -1662,9 +1625,9 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
 		     bo_va->base.bo->tbo.resource->mem_type == TTM_PL_SYSTEM))
 			amdgpu_vm_bo_evicted_user(&bo_va->base);
 
-		spin_lock(&vm->status_lock);
+		spin_lock(&vm->invalidated_lock);
 	}
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->invalidated_lock);
 
 	return 0;
 }
@@ -2207,9 +2170,9 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev,
 		}
 	}
 
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->invalidated_lock);
 	list_del(&bo_va->base.vm_status);
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->invalidated_lock);
 
 	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
 		list_del(&mapping->list);
@@ -2317,10 +2280,10 @@ void amdgpu_vm_bo_move(struct amdgpu_bo *bo, struct ttm_resource *new_mem,
 	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
 		struct amdgpu_vm *vm = bo_base->vm;
 
-		spin_lock(&vm->status_lock);
+		spin_lock(&vm->stats_lock);
 		amdgpu_vm_update_stats_locked(bo_base, bo->tbo.resource, -1);
 		amdgpu_vm_update_stats_locked(bo_base, new_mem, +1);
-		spin_unlock(&vm->status_lock);
+		spin_unlock(&vm->stats_lock);
 	}
 
 	amdgpu_vm_bo_invalidate(bo, evicted);
@@ -2608,11 +2571,12 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	INIT_LIST_HEAD(&vm->relocated);
 	INIT_LIST_HEAD(&vm->moved);
 	INIT_LIST_HEAD(&vm->idle);
+	spin_lock_init(&vm->invalidated_lock);
 	INIT_LIST_HEAD(&vm->invalidated);
-	spin_lock_init(&vm->status_lock);
 	INIT_LIST_HEAD(&vm->freed);
 	INIT_LIST_HEAD(&vm->done);
 	INIT_KFIFO(vm->faults);
+	spin_lock_init(&vm->stats_lock);
 
 	r = amdgpu_vm_init_entities(adev, vm);
 	if (r)
@@ -3080,7 +3044,6 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
 
 	amdgpu_vm_assert_locked(vm);
 
-	spin_lock(&vm->status_lock);
 	seq_puts(m, "\tIdle BOs:\n");
 	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
 		if (!bo_va->base.bo)
@@ -3118,11 +3081,13 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
 	id = 0;
 
 	seq_puts(m, "\tInvalidated BOs:\n");
+	spin_lock(&vm->invalidated_lock);
 	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
 		if (!bo_va->base.bo)
 			continue;
 		total_invalidated += amdgpu_bo_print_info(id++,	bo_va->base.bo, m);
 	}
+	spin_unlock(&vm->invalidated_lock);
 	total_invalidated_objs = id;
 	id = 0;
 
@@ -3132,7 +3097,6 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
 			continue;
 		total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
 	}
-	spin_unlock(&vm->status_lock);
 	total_done_objs = id;
 
 	seq_printf(m, "\tTotal idle size:        %12lld\tobjs:\t%d\n", total_idle,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 139642eacdd0..db9664ec55a9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -205,11 +205,11 @@ struct amdgpu_vm_bo_base {
 	/* protected by bo being reserved */
 	struct amdgpu_vm_bo_base	*next;
 
-	/* protected by vm status_lock */
+	/* protected by vm reservation and invalidated_lock */
 	struct list_head		vm_status;
 
 	/* if the bo is counted as shared in mem stats
-	 * protected by vm status_lock */
+	 * protected by vm BO being reserved */
 	bool				shared;
 
 	/* protected by the BO being reserved */
@@ -345,10 +345,8 @@ struct amdgpu_vm {
 	bool			evicting;
 	unsigned int		saved_flags;
 
-	/* Lock to protect vm_bo add/del/move on all lists of vm */
-	spinlock_t		status_lock;
-
-	/* Memory statistics for this vm, protected by status_lock */
+	/* Memory statistics for this vm, protected by stats_lock */
+	spinlock_t		stats_lock;
 	struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM];
 
 	/*
@@ -356,6 +354,8 @@ struct amdgpu_vm {
 	 * PDs, PTs or per VM BOs. The state transits are:
 	 *
 	 * evicted -> relocated (PDs, PTs) or moved (per VM BOs) -> idle
+	 *
+	 * Lists are protected by the root PD dma_resv lock.
 	 */
 
 	/* Per-VM and PT BOs who needs a validation */
@@ -376,7 +376,10 @@ struct amdgpu_vm {
 	 * state transits are:
 	 *
 	 * evicted_user or invalidated -> done
+	 *
+	 * Lists are protected by the invalidated_lock.
 	 */
+	spinlock_t		invalidated_lock;
 
 	/* BOs for user mode queues that need a validation */
 	struct list_head	evicted_user;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index 31a437ce9570..7bdd664f0770 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -544,9 +544,7 @@ static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry)
 	entry->bo->vm_bo = NULL;
 	ttm_bo_set_bulk_move(&entry->bo->tbo, NULL);
 
-	spin_lock(&entry->vm->status_lock);
 	list_del(&entry->vm_status);
-	spin_unlock(&entry->vm->status_lock);
 	amdgpu_bo_unref(&entry->bo);
 }
 
@@ -590,7 +588,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params,
 	struct amdgpu_vm_pt_cursor seek;
 	struct amdgpu_vm_bo_base *entry;
 
-	spin_lock(&params->vm->status_lock);
 	for_each_amdgpu_vm_pt_dfs_safe(params->adev, params->vm, cursor, seek, entry) {
 		if (entry && entry->bo)
 			list_move(&entry->vm_status, &params->tlb_flush_waitlist);
@@ -598,7 +595,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params,
 
 	/* enter start node now */
 	list_move(&cursor->entry->vm_status, &params->tlb_flush_waitlist);
-	spin_unlock(&params->vm->status_lock);
 }
 
 /**
-- 
2.43.0

[PATCH 3/9] drm/amdgpu: fix amdgpu_userq_evict

[Christian König]

Canceling the delayed synchonized can deadlock!

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 8feeaeea1c36..af3922e9caea 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -1337,13 +1337,8 @@ amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr,
 	/* Signal current eviction fence */
 	amdgpu_eviction_fence_signal(evf_mgr, ev_fence);
 
-	if (evf_mgr->fd_closing) {
-		cancel_delayed_work_sync(&uq_mgr->resume_work);
-		return;
-	}
-
-	/* Schedule a resume work */
-	schedule_delayed_work(&uq_mgr->resume_work, 0);
+	if (!evf_mgr->fd_closing)
+		schedule_delayed_work(&uq_mgr->resume_work, 0);
 }
 
 int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,
-- 
2.43.0

[PATCH 4/9] drm/amdgpu: completely rework eviction fence handling

[Christian König]

Well that was broken on multiple levels.

First of all a lot of checks where placed at incorrect locations, especially if
the resume worker should run or not.

Then a bunch of code was just mid-layering because of incorrect assignment who
should do what.

And finally comments explaining what happens instead of why.

Just re-write it from scratch, that should at least fix some of the hangs we
are seeing.

Use RCU for the eviction fence pointer in the manager, the spinlock usage was
mostly incorrect as well. Then finally remove all the nonsense checks and
actually add them in the correct locations.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c       |   4 +-
 .../drm/amd/amdgpu/amdgpu_eviction_fence.c    | 213 ++++++------------
 .../drm/amd/amdgpu/amdgpu_eviction_fence.h    |  54 ++---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c       |  10 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c       |   5 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c     |  29 ++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h     |   2 +-
 7 files changed, 112 insertions(+), 205 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index d6d0a6e34c6b..02abe85624a6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -2975,9 +2975,9 @@ static int amdgpu_drm_release(struct inode *inode, struct file *filp)
 	int idx;
 
 	if (fpriv && drm_dev_enter(dev, &idx)) {
-		fpriv->evf_mgr.fd_closing = true;
-		amdgpu_eviction_fence_destroy(&fpriv->evf_mgr);
+		amdgpu_evf_mgr_shutdown(&fpriv->evf_mgr);
 		amdgpu_userq_mgr_fini(&fpriv->userq_mgr);
+		amdgpu_evf_mgr_fini(&fpriv->evf_mgr);
 		drm_dev_exit(idx);
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
index 23d7d0b0d625..8fe9f91f9551 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
@@ -25,9 +25,6 @@
 #include <drm/drm_exec.h>
 #include "amdgpu.h"
 
-#define work_to_evf_mgr(w, name) container_of(w, struct amdgpu_eviction_fence_mgr, name)
-#define evf_mgr_to_fpriv(e) container_of(e, struct amdgpu_fpriv, evf_mgr)
-
 static const char *
 amdgpu_eviction_fence_get_driver_name(struct dma_fence *fence)
 {
@@ -43,102 +40,14 @@ amdgpu_eviction_fence_get_timeline_name(struct dma_fence *f)
 	return ef->timeline_name;
 }
 
-int
-amdgpu_eviction_fence_replace_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
-				    struct drm_exec *exec)
-{
-	struct amdgpu_eviction_fence *old_ef, *new_ef;
-	struct drm_gem_object *obj;
-	unsigned long index;
-	int ret;
-
-	if (evf_mgr->ev_fence &&
-	    !dma_fence_is_signaled(&evf_mgr->ev_fence->base))
-		return 0;
-	/*
-	 * Steps to replace eviction fence:
-	 * * lock all objects in exec (caller)
-	 * * create a new eviction fence
-	 * * update new eviction fence in evf_mgr
-	 * * attach the new eviction fence to BOs
-	 * * release the old fence
-	 * * unlock the objects (caller)
-	 */
-	new_ef = amdgpu_eviction_fence_create(evf_mgr);
-	if (!new_ef) {
-		DRM_ERROR("Failed to create new eviction fence\n");
-		return -ENOMEM;
-	}
-
-	/* Update the eviction fence now */
-	spin_lock(&evf_mgr->ev_fence_lock);
-	old_ef = evf_mgr->ev_fence;
-	evf_mgr->ev_fence = new_ef;
-	spin_unlock(&evf_mgr->ev_fence_lock);
-
-	/* Attach the new fence */
-	drm_exec_for_each_locked_object(exec, index, obj) {
-		struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
-
-		if (!bo)
-			continue;
-		ret = amdgpu_eviction_fence_attach(evf_mgr, bo);
-		if (ret) {
-			DRM_ERROR("Failed to attch new eviction fence\n");
-			goto free_err;
-		}
-	}
-
-	/* Free old fence */
-	if (old_ef)
-		dma_fence_put(&old_ef->base);
-	return 0;
-
-free_err:
-	kfree(new_ef);
-	return ret;
-}
-
-static void
-amdgpu_eviction_fence_suspend_worker(struct work_struct *work)
-{
-	struct amdgpu_eviction_fence_mgr *evf_mgr = work_to_evf_mgr(work, suspend_work.work);
-	struct amdgpu_fpriv *fpriv = evf_mgr_to_fpriv(evf_mgr);
-	struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr;
-	struct amdgpu_eviction_fence *ev_fence;
-
-	mutex_lock(&uq_mgr->userq_mutex);
-	spin_lock(&evf_mgr->ev_fence_lock);
-	ev_fence = evf_mgr->ev_fence;
-	if (ev_fence)
-		dma_fence_get(&ev_fence->base);
-	else
-		goto unlock;
-	spin_unlock(&evf_mgr->ev_fence_lock);
-
-	amdgpu_userq_evict(uq_mgr, ev_fence);
-
-	mutex_unlock(&uq_mgr->userq_mutex);
-	dma_fence_put(&ev_fence->base);
-	return;
-
-unlock:
-	spin_unlock(&evf_mgr->ev_fence_lock);
-	mutex_unlock(&uq_mgr->userq_mutex);
-}
-
 static bool amdgpu_eviction_fence_enable_signaling(struct dma_fence *f)
 {
 	struct amdgpu_eviction_fence_mgr *evf_mgr;
 	struct amdgpu_eviction_fence *ev_fence;
 
-	if (!f)
-		return true;
-
 	ev_fence = to_ev_fence(f);
 	evf_mgr = ev_fence->evf_mgr;
-
-	schedule_delayed_work(&evf_mgr->suspend_work, 0);
+	schedule_work(&evf_mgr->suspend_work);
 	return true;
 }
 
@@ -148,22 +57,52 @@ static const struct dma_fence_ops amdgpu_eviction_fence_ops = {
 	.enable_signaling = amdgpu_eviction_fence_enable_signaling,
 };
 
-void amdgpu_eviction_fence_signal(struct amdgpu_eviction_fence_mgr *evf_mgr,
-				  struct amdgpu_eviction_fence *ev_fence)
+static void
+amdgpu_eviction_fence_suspend_worker(struct work_struct *work)
 {
-	spin_lock(&evf_mgr->ev_fence_lock);
-	dma_fence_signal(&ev_fence->base);
-	spin_unlock(&evf_mgr->ev_fence_lock);
+	struct amdgpu_eviction_fence_mgr *evf_mgr =
+		container_of(work, struct amdgpu_eviction_fence_mgr,
+			     suspend_work);
+	struct amdgpu_fpriv *fpriv =
+		container_of(evf_mgr, struct amdgpu_fpriv, evf_mgr);
+	struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr;
+	struct dma_fence *ev_fence;
+
+	mutex_lock(&uq_mgr->userq_mutex);
+	ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);
+	amdgpu_userq_evict(uq_mgr, !evf_mgr->shutdown);
+
+	/*
+	 * Signaling the eviction fence must be done while holding the
+	 * userq_mutex. Otherwise we won't resume the queues before issuing the
+	 * next fence.
+	 */
+	dma_fence_signal(ev_fence);
+	dma_fence_put(ev_fence);
+	mutex_unlock(&uq_mgr->userq_mutex);
+}
+
+void amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
+				 struct amdgpu_bo *bo)
+{
+	struct dma_fence *ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);
+	struct dma_resv *resv = bo->tbo.base.resv;
+
+	dma_resv_add_fence(resv, ev_fence, DMA_RESV_USAGE_BOOKKEEP);
+	dma_fence_put(ev_fence);
 }
 
-struct amdgpu_eviction_fence *
-amdgpu_eviction_fence_create(struct amdgpu_eviction_fence_mgr *evf_mgr)
+int amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr *evf_mgr,
+			 struct drm_exec *exec)
 {
 	struct amdgpu_eviction_fence *ev_fence;
+	struct drm_gem_object *obj;
+	unsigned long index;
 
+	/* Create and initialize a new eviction fence */
 	ev_fence = kzalloc(sizeof(*ev_fence), GFP_KERNEL);
 	if (!ev_fence)
-		return NULL;
+		return -ENOMEM;
 
 	ev_fence->evf_mgr = evf_mgr;
 	get_task_comm(ev_fence->timeline_name, current);
@@ -171,56 +110,22 @@ amdgpu_eviction_fence_create(struct amdgpu_eviction_fence_mgr *evf_mgr)
 	dma_fence_init64(&ev_fence->base, &amdgpu_eviction_fence_ops,
 			 &ev_fence->lock, evf_mgr->ev_fence_ctx,
 			 atomic_inc_return(&evf_mgr->ev_fence_seq));
-	return ev_fence;
-}
-
-void amdgpu_eviction_fence_destroy(struct amdgpu_eviction_fence_mgr *evf_mgr)
-{
-	struct amdgpu_eviction_fence *ev_fence;
-
-	/* Wait for any pending work to execute */
-	flush_delayed_work(&evf_mgr->suspend_work);
-
-	spin_lock(&evf_mgr->ev_fence_lock);
-	ev_fence = evf_mgr->ev_fence;
-	spin_unlock(&evf_mgr->ev_fence_lock);
-
-	if (!ev_fence)
-		return;
-
-	dma_fence_wait(&ev_fence->base, false);
 
-	/* Last unref of ev_fence */
-	dma_fence_put(&ev_fence->base);
-}
-
-int amdgpu_eviction_fence_attach(struct amdgpu_eviction_fence_mgr *evf_mgr,
-				 struct amdgpu_bo *bo)
-{
-	struct amdgpu_eviction_fence *ev_fence;
-	struct dma_resv *resv = bo->tbo.base.resv;
-	int ret;
+	/* Remember it for newly added BOs */
+	dma_fence_put(evf_mgr->ev_fence);
+	evf_mgr->ev_fence = &ev_fence->base;
 
-	if (!resv)
-		return 0;
+	/* And add it to all existing BOs */
+	drm_exec_for_each_locked_object(exec, index, obj) {
+		struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
 
-	ret = dma_resv_reserve_fences(resv, 1);
-	if (ret) {
-		DRM_DEBUG_DRIVER("Failed to resv fence space\n");
-		return ret;
+		amdgpu_evf_mgr_attach_fence(evf_mgr, bo);
 	}
-
-	spin_lock(&evf_mgr->ev_fence_lock);
-	ev_fence = evf_mgr->ev_fence;
-	if (ev_fence)
-		dma_resv_add_fence(resv, &ev_fence->base, DMA_RESV_USAGE_BOOKKEEP);
-	spin_unlock(&evf_mgr->ev_fence_lock);
-
 	return 0;
 }
 
-void amdgpu_eviction_fence_detach(struct amdgpu_eviction_fence_mgr *evf_mgr,
-				  struct amdgpu_bo *bo)
+void amdgpu_evf_mgr_detach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
+				 struct amdgpu_bo *bo)
 {
 	struct dma_fence *stub = dma_fence_get_stub();
 
@@ -229,13 +134,25 @@ void amdgpu_eviction_fence_detach(struct amdgpu_eviction_fence_mgr *evf_mgr,
 	dma_fence_put(stub);
 }
 
-int amdgpu_eviction_fence_init(struct amdgpu_eviction_fence_mgr *evf_mgr)
+void amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr *evf_mgr)
 {
-	/* This needs to be done one time per open */
 	atomic_set(&evf_mgr->ev_fence_seq, 0);
 	evf_mgr->ev_fence_ctx = dma_fence_context_alloc(1);
-	spin_lock_init(&evf_mgr->ev_fence_lock);
+	evf_mgr->ev_fence = dma_fence_get_stub();
 
-	INIT_DELAYED_WORK(&evf_mgr->suspend_work, amdgpu_eviction_fence_suspend_worker);
-	return 0;
+	INIT_WORK(&evf_mgr->suspend_work, amdgpu_eviction_fence_suspend_worker);
+}
+
+void amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr *evf_mgr)
+{
+	evf_mgr->shutdown = true;
+	flush_work(&evf_mgr->suspend_work);
+}
+
+void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr)
+{
+	dma_fence_wait(rcu_dereference_protected(evf_mgr->ev_fence, true),
+		       false);
+	flush_work(&evf_mgr->suspend_work);
+	dma_fence_put(evf_mgr->ev_fence);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
index fcd867b7147d..527de3a23583 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
@@ -25,6 +25,8 @@
 #ifndef AMDGPU_EV_FENCE_H_
 #define AMDGPU_EV_FENCE_H_
 
+#include <linux/dma-fence.h>
+
 struct amdgpu_eviction_fence {
 	struct dma_fence base;
 	spinlock_t	 lock;
@@ -35,35 +37,35 @@ struct amdgpu_eviction_fence {
 struct amdgpu_eviction_fence_mgr {
 	u64			ev_fence_ctx;
 	atomic_t		ev_fence_seq;
-	spinlock_t		ev_fence_lock;
-	struct amdgpu_eviction_fence *ev_fence;
-	struct delayed_work	suspend_work;
-	uint8_t fd_closing;
-};
-
-/* Eviction fence helper functions */
-struct amdgpu_eviction_fence *
-amdgpu_eviction_fence_create(struct amdgpu_eviction_fence_mgr *evf_mgr);
 
-void
-amdgpu_eviction_fence_destroy(struct amdgpu_eviction_fence_mgr *evf_mgr);
-
-int
-amdgpu_eviction_fence_attach(struct amdgpu_eviction_fence_mgr *evf_mgr,
-			     struct amdgpu_bo *bo);
+	/*
+	 * Only updated while holding the VM resv lock.
+	 * Only signaled while holding the userq mutex.
+	 */
+	struct dma_fence __rcu	*ev_fence;
+	struct work_struct	suspend_work;
+	bool			shutdown;
+};
 
-void
-amdgpu_eviction_fence_detach(struct amdgpu_eviction_fence_mgr *evf_mgr,
-			     struct amdgpu_bo *bo);
+static inline struct dma_fence *
+amdgpu_evf_mgr_get_fence(struct amdgpu_eviction_fence_mgr *evf_mgr)
+{
+	struct dma_fence *ev_fence;
 
-int
-amdgpu_eviction_fence_init(struct amdgpu_eviction_fence_mgr *evf_mgr);
+	rcu_read_lock();
+	ev_fence = dma_fence_get_rcu_safe(&evf_mgr->ev_fence);
+	rcu_read_unlock();
+	return ev_fence;
+}
 
-void
-amdgpu_eviction_fence_signal(struct amdgpu_eviction_fence_mgr *evf_mgr,
-			     struct amdgpu_eviction_fence *ev_fence);
+void amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
+				 struct amdgpu_bo *bo);
+int amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr *evf_mgr,
+			 struct drm_exec *exec);
+void amdgpu_evf_mgr_detach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
+				 struct amdgpu_bo *bo);
+void amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr *evf_mgr);
+void amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr *evf_mgr);
+void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr);
 
-int
-amdgpu_eviction_fence_replace_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
-				    struct drm_exec *exec);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 5c90de58cc28..e28abfd04867 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -263,13 +263,7 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
 	else
 		++bo_va->ref_count;
 
-	/* attach gfx eviction fence */
-	r = amdgpu_eviction_fence_attach(&fpriv->evf_mgr, abo);
-	if (r) {
-		DRM_DEBUG_DRIVER("Failed to attach eviction fence to BO\n");
-		amdgpu_bo_unreserve(abo);
-		return r;
-	}
+	amdgpu_evf_mgr_attach_fence(&fpriv->evf_mgr, abo);
 	drm_exec_fini(&exec);
 
 	/* Validate and add eviction fence to DMABuf imports with dynamic
@@ -337,7 +331,7 @@ static void amdgpu_gem_object_close(struct drm_gem_object *obj,
 	}
 
 	if (!amdgpu_vm_is_bo_always_valid(vm, bo))
-		amdgpu_eviction_fence_detach(&fpriv->evf_mgr, bo);
+		amdgpu_evf_mgr_detach_fence(&fpriv->evf_mgr, bo);
 
 	bo_va = amdgpu_vm_bo_find(vm, bo);
 	if (!bo_va || --bo_va->ref_count)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index f69332eed051..f512b6ec6c53 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -1522,10 +1522,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 			 "Failed to init usermode queue manager (%d), use legacy workload submission only\n",
 			 r);
 
-	r = amdgpu_eviction_fence_init(&fpriv->evf_mgr);
-	if (r)
-		goto error_vm;
-
+	amdgpu_evf_mgr_init(&fpriv->evf_mgr);
 	amdgpu_ctx_mgr_init(&fpriv->ctx_mgr, adev);
 
 	file_priv->driver_priv = fpriv;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index af3922e9caea..e007f10a6c1c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -480,17 +480,16 @@ void
 amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr,
 			     struct amdgpu_eviction_fence_mgr *evf_mgr)
 {
-	struct amdgpu_eviction_fence *ev_fence;
+	struct dma_fence *ev_fence;
 
 retry:
 	/* Flush any pending resume work to create ev_fence */
 	flush_delayed_work(&uq_mgr->resume_work);
 
 	mutex_lock(&uq_mgr->userq_mutex);
-	spin_lock(&evf_mgr->ev_fence_lock);
-	ev_fence = evf_mgr->ev_fence;
-	spin_unlock(&evf_mgr->ev_fence_lock);
-	if (!ev_fence || dma_fence_is_signaled(&ev_fence->base)) {
+	ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);
+	if (dma_fence_is_signaled(ev_fence)) {
+		dma_fence_put(ev_fence);
 		mutex_unlock(&uq_mgr->userq_mutex);
 		/*
 		 * Looks like there was no pending resume work,
@@ -499,6 +498,7 @@ amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr,
 		schedule_delayed_work(&uq_mgr->resume_work, 0);
 		goto retry;
 	}
+	dma_fence_put(ev_fence);
 }
 
 int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr,
@@ -1214,7 +1214,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
 		dma_fence_wait(bo_va->last_pt_update, false);
 	dma_fence_wait(vm->last_update, false);
 
-	ret = amdgpu_eviction_fence_replace_fence(&fpriv->evf_mgr, &exec);
+	ret = amdgpu_evf_mgr_rearm(&fpriv->evf_mgr, &exec);
 	if (ret)
 		drm_file_err(uq_mgr->file, "Failed to replace eviction fence\n");
 
@@ -1234,11 +1234,13 @@ static void amdgpu_userq_restore_worker(struct work_struct *work)
 {
 	struct amdgpu_userq_mgr *uq_mgr = work_to_uq_mgr(work, resume_work.work);
 	struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
+	struct dma_fence *ev_fence;
 	int ret;
 
-	flush_delayed_work(&fpriv->evf_mgr.suspend_work);
-
 	mutex_lock(&uq_mgr->userq_mutex);
+	ev_fence = amdgpu_evf_mgr_get_fence(&fpriv->evf_mgr);
+	if (!dma_fence_is_signaled(ev_fence))
+		goto unlock;
 
 	ret = amdgpu_userq_vm_validate(uq_mgr);
 	if (ret) {
@@ -1254,6 +1256,7 @@ static void amdgpu_userq_restore_worker(struct work_struct *work)
 
 unlock:
 	mutex_unlock(&uq_mgr->userq_mutex);
+	dma_fence_put(ev_fence);
 }
 
 static int
@@ -1317,11 +1320,8 @@ amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr)
 }
 
 void
-amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr,
-		   struct amdgpu_eviction_fence *ev_fence)
+amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr, bool schedule_resume)
 {
-	struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
-	struct amdgpu_eviction_fence_mgr *evf_mgr = &fpriv->evf_mgr;
 	struct amdgpu_device *adev = uq_mgr->adev;
 	int ret;
 
@@ -1334,10 +1334,7 @@ amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr,
 	if (ret)
 		dev_err(adev->dev, "Failed to evict userqueue\n");
 
-	/* Signal current eviction fence */
-	amdgpu_eviction_fence_signal(evf_mgr, ev_fence);
-
-	if (!evf_mgr->fd_closing)
+	if (schedule_resume)
 		schedule_delayed_work(&uq_mgr->resume_work, 0);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index 5845d8959034..095c09f3d96c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -127,7 +127,7 @@ void amdgpu_userq_destroy_object(struct amdgpu_userq_mgr *uq_mgr,
 				 struct amdgpu_userq_obj *userq_obj);
 
 void amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr,
-			struct amdgpu_eviction_fence *ev_fence);
+			bool schedule_resume);
 
 void amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *userq_mgr,
 				  struct amdgpu_eviction_fence_mgr *evf_mgr);
-- 
2.43.0

[PATCH 5/9] drm/amdgpu: fix eviction fence and userq manager shutdown

[Christian König]

That is a really complicated dance and wasn't implemented fully correct.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c            | 2 ++
 drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 8 +++++++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h | 1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c          | 7 +++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h          | 1 +
 5 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 02abe85624a6..9cd44be45861 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -2976,6 +2976,8 @@ static int amdgpu_drm_release(struct inode *inode, struct file *filp)
 
 	if (fpriv && drm_dev_enter(dev, &idx)) {
 		amdgpu_evf_mgr_shutdown(&fpriv->evf_mgr);
+		amdgpu_userq_mgr_cancel_resume(&fpriv->userq_mgr);
+		amdgpu_evf_mgr_flush_suspend(&fpriv->evf_mgr);
 		amdgpu_userq_mgr_fini(&fpriv->userq_mgr);
 		amdgpu_evf_mgr_fini(&fpriv->evf_mgr);
 		drm_dev_exit(idx);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
index 8fe9f91f9551..ef4da6f2e2a3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
@@ -146,13 +146,19 @@ void amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr *evf_mgr)
 void amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr *evf_mgr)
 {
 	evf_mgr->shutdown = true;
+	/* Make sure that the shutdown is visible to the suspend work */
 	flush_work(&evf_mgr->suspend_work);
 }
 
-void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr)
+void amdgpu_evf_mgr_flush_suspend(struct amdgpu_eviction_fence_mgr *evf_mgr)
 {
 	dma_fence_wait(rcu_dereference_protected(evf_mgr->ev_fence, true),
 		       false);
+	/* Make sure that we are done with the last suspend work */
 	flush_work(&evf_mgr->suspend_work);
+}
+
+void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr)
+{
 	dma_fence_put(evf_mgr->ev_fence);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
index 527de3a23583..132a13a5dc1c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
@@ -66,6 +66,7 @@ void amdgpu_evf_mgr_detach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
 				 struct amdgpu_bo *bo);
 void amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr *evf_mgr);
 void amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr *evf_mgr);
+void amdgpu_evf_mgr_flush_suspend(struct amdgpu_eviction_fence_mgr *evf_mgr);
 void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index e007f10a6c1c..60e8a993557a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -1350,13 +1350,16 @@ int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *f
 	return 0;
 }
 
+void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr)
+{
+	cancel_delayed_work_sync(&userq_mgr->resume_work);
+}
+
 void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr)
 {
 	struct amdgpu_usermode_queue *queue;
 	unsigned long queue_id;
 
-	cancel_delayed_work_sync(&userq_mgr->resume_work);
-
 	mutex_lock(&userq_mgr->userq_mutex);
 	amdgpu_userq_detect_and_reset_queues(userq_mgr);
 	xa_for_each(&userq_mgr->userq_xa, queue_id, queue) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index 095c09f3d96c..13f4e356728b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -117,6 +117,7 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp
 int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,
 			  struct amdgpu_device *adev);
 
+void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr);
 void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr);
 
 int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr,
-- 
2.43.0

[PATCH 6/9] drm/amdgpu: fix adding eviction fence

[Christian König]

We can't add the eviction fence without validating the BO.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 .../drm/amd/amdgpu/amdgpu_eviction_fence.c    | 19 ++++++++++++++++---
 .../drm/amd/amdgpu/amdgpu_eviction_fence.h    |  4 ++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c       |  9 ++++++---
 3 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
index ef4da6f2e2a3..6598823ec619 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
@@ -82,14 +82,27 @@ amdgpu_eviction_fence_suspend_worker(struct work_struct *work)
 	mutex_unlock(&uq_mgr->userq_mutex);
 }
 
-void amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
-				 struct amdgpu_bo *bo)
+int amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
+				struct amdgpu_bo *bo)
 {
 	struct dma_fence *ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);
+	struct ttm_operation_ctx ctx = { false, false };
 	struct dma_resv *resv = bo->tbo.base.resv;
+	int ret;
+
+	if (!dma_fence_is_signaled(ev_fence)) {
+
+		amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
+		ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+		if (!ret)
+			dma_resv_add_fence(resv, ev_fence,
+					   DMA_RESV_USAGE_BOOKKEEP);
+	} else {
+		ret = 0;
+	}
 
-	dma_resv_add_fence(resv, ev_fence, DMA_RESV_USAGE_BOOKKEEP);
 	dma_fence_put(ev_fence);
+	return ret;
 }
 
 int amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr *evf_mgr,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
index 132a13a5dc1c..2a750add4e7b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
@@ -58,8 +58,8 @@ amdgpu_evf_mgr_get_fence(struct amdgpu_eviction_fence_mgr *evf_mgr)
 	return ev_fence;
 }
 
-void amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
-				 struct amdgpu_bo *bo);
+int amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
+				struct amdgpu_bo *bo);
 int amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr *evf_mgr,
 			 struct drm_exec *exec);
 void amdgpu_evf_mgr_detach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index e28abfd04867..88a21400ae09 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -258,12 +258,15 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
 
 	amdgpu_vm_bo_update_shared(abo);
 	bo_va = amdgpu_vm_bo_find(vm, abo);
-	if (!bo_va)
+	if (!bo_va) {
 		bo_va = amdgpu_vm_bo_add(adev, vm, abo);
-	else
+		r = amdgpu_evf_mgr_attach_fence(&fpriv->evf_mgr, abo);
+		if (r)
+			goto out_unlock;
+	} else {
 		++bo_va->ref_count;
+	}
 
-	amdgpu_evf_mgr_attach_fence(&fpriv->evf_mgr, abo);
 	drm_exec_fini(&exec);
 
 	/* Validate and add eviction fence to DMABuf imports with dynamic
-- 
2.43.0

[PATCH 7/9] drm/amdgpu: rework amdgpu_userq_wait_ioctl v2

[Christian König]

Lockdep was complaining about a number of issues here. Especially lock
inversion between syncobj, dma_resv and copying things into userspace.

Rework the functionality. Split it up into multiple functions,
consistenly use memdup_array_user(), fix the lock inversions and a few
more bugs in error handling.

v2: drop the dma_fence leak fix, turned out that was actually correct,
    just not well documented. Apply some more cleanup suggestion from
    Tvrtko.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 614 +++++++++---------
 1 file changed, 319 insertions(+), 295 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
index 212056d4ddf0..da329d00843b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
@@ -645,368 +645,397 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 	return r;
 }
 
-int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
-			    struct drm_file *filp)
+/* Count the number of expected fences so userspace can alloc a buffer */
+static int
+amdgpu_userq_wait_count_fences(struct drm_file *filp,
+			       struct drm_amdgpu_userq_wait *wait_info,
+			       u32 *syncobj_handles, u32 *timeline_points,
+			       u32 *timeline_handles,
+			       struct drm_gem_object **gobj_write,
+			       struct drm_gem_object **gobj_read)
 {
-	u32 *syncobj_handles, *timeline_points, *timeline_handles, *bo_handles_read, *bo_handles_write;
-	u32 num_syncobj, num_read_bo_handles, num_write_bo_handles;
-	struct drm_amdgpu_userq_fence_info *fence_info = NULL;
-	struct drm_amdgpu_userq_wait *wait_info = data;
-	struct amdgpu_fpriv *fpriv = filp->driver_priv;
-	struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
-	struct amdgpu_usermode_queue *waitq;
-	struct drm_gem_object **gobj_write;
-	struct drm_gem_object **gobj_read;
-	struct dma_fence **fences = NULL;
-	u16 num_points, num_fences = 0;
-	int r, i, rentry, wentry, cnt;
+	int num_read_bo_handles, num_write_bo_handles;
+	struct dma_fence_unwrap iter;
+	struct dma_fence *fence, *f;
+	unsigned int num_fences = 0;
 	struct drm_exec exec;
+	int i, r;
+
+	/*
+	 * This needs to be outside of the lock provided by drm_exec for
+	 * DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT to work correctly.
+	 */
+
+	/* Count timeline fences */
+	for (i = 0; i < wait_info->num_syncobj_timeline_handles; i++) {
+		r = drm_syncobj_find_fence(filp, timeline_handles[i],
+					   timeline_points[i],
+					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
+					   &fence);
+		if (r)
+			return r;
+
+		dma_fence_unwrap_for_each(f, &iter, fence)
+			num_fences++;
 
-	if (!amdgpu_userq_enabled(dev))
-		return -ENOTSUPP;
+		dma_fence_put(fence);
+	}
 
-	num_read_bo_handles = wait_info->num_bo_read_handles;
-	bo_handles_read = memdup_user(u64_to_user_ptr(wait_info->bo_read_handles),
-				      size_mul(sizeof(u32), num_read_bo_handles));
-	if (IS_ERR(bo_handles_read))
-		return PTR_ERR(bo_handles_read);
+	/* Count boolean fences */
+	for (i = 0; i < wait_info->num_syncobj_handles; i++) {
+		r = drm_syncobj_find_fence(filp, syncobj_handles[i], 0,
+					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
+					   &fence);
+		if (r)
+			return r;
 
-	num_write_bo_handles = wait_info->num_bo_write_handles;
-	bo_handles_write = memdup_user(u64_to_user_ptr(wait_info->bo_write_handles),
-				       size_mul(sizeof(u32), num_write_bo_handles));
-	if (IS_ERR(bo_handles_write)) {
-		r = PTR_ERR(bo_handles_write);
-		goto free_bo_handles_read;
+		num_fences++;
+		dma_fence_put(fence);
 	}
 
-	num_syncobj = wait_info->num_syncobj_handles;
-	syncobj_handles = memdup_user(u64_to_user_ptr(wait_info->syncobj_handles),
-				      size_mul(sizeof(u32), num_syncobj));
-	if (IS_ERR(syncobj_handles)) {
-		r = PTR_ERR(syncobj_handles);
-		goto free_bo_handles_write;
-	}
+	/* Lock all the GEM objects */
+	/* TODO: It is actually not necessary to lock them */
+	num_read_bo_handles = wait_info->num_bo_read_handles;
+	num_write_bo_handles = wait_info->num_bo_write_handles;
+	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
+		      num_read_bo_handles + num_write_bo_handles);
 
-	num_points = wait_info->num_syncobj_timeline_handles;
-	timeline_handles = memdup_user(u64_to_user_ptr(wait_info->syncobj_timeline_handles),
-				       sizeof(u32) * num_points);
-	if (IS_ERR(timeline_handles)) {
-		r = PTR_ERR(timeline_handles);
-		goto free_syncobj_handles;
-	}
+	drm_exec_until_all_locked(&exec) {
+		r = drm_exec_prepare_array(&exec, gobj_read,
+					   num_read_bo_handles, 1);
+		drm_exec_retry_on_contention(&exec);
+		if (r)
+			goto error_unlock;
 
-	timeline_points = memdup_user(u64_to_user_ptr(wait_info->syncobj_timeline_points),
-				      sizeof(u32) * num_points);
-	if (IS_ERR(timeline_points)) {
-		r = PTR_ERR(timeline_points);
-		goto free_timeline_handles;
+		r = drm_exec_prepare_array(&exec, gobj_write,
+					   num_write_bo_handles, 1);
+		drm_exec_retry_on_contention(&exec);
+		if (r)
+			goto error_unlock;
 	}
 
-	gobj_read = kmalloc_array(num_read_bo_handles, sizeof(*gobj_read), GFP_KERNEL);
-	if (!gobj_read) {
-		r = -ENOMEM;
-		goto free_timeline_points;
-	}
+	/* Count read fences */
+	for (i = 0; i < num_read_bo_handles; i++) {
+		struct dma_resv_iter resv_cursor;
+		struct dma_fence *fence;
 
-	for (rentry = 0; rentry < num_read_bo_handles; rentry++) {
-		gobj_read[rentry] = drm_gem_object_lookup(filp, bo_handles_read[rentry]);
-		if (!gobj_read[rentry]) {
-			r = -ENOENT;
-			goto put_gobj_read;
-		}
+		dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
+					DMA_RESV_USAGE_READ, fence)
+			num_fences++;
 	}
 
-	gobj_write = kmalloc_array(num_write_bo_handles, sizeof(*gobj_write), GFP_KERNEL);
-	if (!gobj_write) {
-		r = -ENOMEM;
-		goto put_gobj_read;
-	}
+	/* Count write fences */
+	for (i = 0; i < num_write_bo_handles; i++) {
+		struct dma_resv_iter resv_cursor;
+		struct dma_fence *fence;
 
-	for (wentry = 0; wentry < num_write_bo_handles; wentry++) {
-		gobj_write[wentry] = drm_gem_object_lookup(filp, bo_handles_write[wentry]);
-		if (!gobj_write[wentry]) {
-			r = -ENOENT;
-			goto put_gobj_write;
-		}
+		dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
+					DMA_RESV_USAGE_WRITE, fence)
+			num_fences++;
 	}
 
-	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
-		      (num_read_bo_handles + num_write_bo_handles));
-
-	/* Lock all BOs with retry handling */
-	drm_exec_until_all_locked(&exec) {
-		r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1);
-		drm_exec_retry_on_contention(&exec);
-		if (r) {
-			drm_exec_fini(&exec);
-			goto put_gobj_write;
-		}
+	wait_info->num_fences = num_fences;
+	r = 0;
 
-		r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1);
-		drm_exec_retry_on_contention(&exec);
-		if (r) {
-			drm_exec_fini(&exec);
-			goto put_gobj_write;
-		}
-	}
+error_unlock:
+	/* Unlock all the GEM objects */
+	drm_exec_fini(&exec);
+	return r;
+}
 
-	if (!wait_info->num_fences) {
-		if (num_points) {
-			struct dma_fence_unwrap iter;
-			struct dma_fence *fence;
-			struct dma_fence *f;
-
-			for (i = 0; i < num_points; i++) {
-				r = drm_syncobj_find_fence(filp, timeline_handles[i],
-							   timeline_points[i],
-							   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
-							   &fence);
-				if (r)
-					goto exec_fini;
-
-				dma_fence_unwrap_for_each(f, &iter, fence)
-					num_fences++;
-
-				dma_fence_put(fence);
-			}
-		}
+static int
+amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
+				    struct drm_amdgpu_userq_wait *wait_info,
+				    u32 *syncobj_handles, u32 *timeline_points,
+				    u32 *timeline_handles,
+				    struct drm_gem_object **gobj_write,
+				    struct drm_gem_object **gobj_read)
+{
+	struct amdgpu_fpriv *fpriv = filp->driver_priv;
+	struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
+	struct drm_amdgpu_userq_fence_info *fence_info;
+	int num_read_bo_handles, num_write_bo_handles;
+	struct amdgpu_usermode_queue *waitq;
+	struct dma_fence **fences, *fence, *f;
+	struct dma_fence_unwrap iter;
+	int num_points, num_syncobj;
+	unsigned int num_fences = 0;
+	struct drm_exec exec;
+	int i, cnt, r;
 
-		/* Count syncobj's fence */
-		for (i = 0; i < num_syncobj; i++) {
-			struct dma_fence *fence;
+	fence_info = kmalloc_array(wait_info->num_fences, sizeof(*fence_info),
+				   GFP_KERNEL);
+	if (!fence_info)
+		return -ENOMEM;
 
-			r = drm_syncobj_find_fence(filp, syncobj_handles[i],
-						   0,
-						   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
-						   &fence);
-			if (r)
-				goto exec_fini;
+	fences = kmalloc_array(wait_info->num_fences, sizeof(*fences),
+			       GFP_KERNEL);
+	if (!fences) {
+		r = -ENOMEM;
+		goto free_fence_info;
+	}
 
-			num_fences++;
-			dma_fence_put(fence);
-		}
+	/* Retrieve timeline fences */
+	num_points = wait_info->num_syncobj_timeline_handles;
+	for (i = 0; i < num_points; i++) {
+		r = drm_syncobj_find_fence(filp, timeline_handles[i],
+					   timeline_points[i],
+					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
+					   &fence);
+		if (r)
+			goto free_fences;
 
-		/* Count GEM objects fence */
-		for (i = 0; i < num_read_bo_handles; i++) {
-			struct dma_resv_iter resv_cursor;
-			struct dma_fence *fence;
+		dma_fence_unwrap_for_each(f, &iter, fence) {
+			if (num_fences >= wait_info->num_fences) {
+				r = -EINVAL;
+				goto free_fences;
+			}
 
-			dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
-						DMA_RESV_USAGE_READ, fence)
-				num_fences++;
+			fences[num_fences++] = dma_fence_get(f);
 		}
 
-		for (i = 0; i < num_write_bo_handles; i++) {
-			struct dma_resv_iter resv_cursor;
-			struct dma_fence *fence;
+		dma_fence_put(fence);
+	}
 
-			dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
-						DMA_RESV_USAGE_WRITE, fence)
-				num_fences++;
-		}
+	/* Retrieve boolean fences */
+	num_syncobj = wait_info->num_syncobj_handles;
+	for (i = 0; i < num_syncobj; i++) {
+		struct dma_fence *fence;
 
-		/*
-		 * Passing num_fences = 0 means that userspace doesn't want to
-		 * retrieve userq_fence_info. If num_fences = 0 we skip filling
-		 * userq_fence_info and return the actual number of fences on
-		 * args->num_fences.
-		 */
-		wait_info->num_fences = num_fences;
-	} else {
-		/* Array of fence info */
-		fence_info = kmalloc_array(wait_info->num_fences, sizeof(*fence_info), GFP_KERNEL);
-		if (!fence_info) {
-			r = -ENOMEM;
-			goto exec_fini;
-		}
+		r = drm_syncobj_find_fence(filp, syncobj_handles[i], 0,
+					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
+					   &fence);
+		if (r)
+			goto free_fences;
 
-		/* Array of fences */
-		fences = kmalloc_array(wait_info->num_fences, sizeof(*fences), GFP_KERNEL);
-		if (!fences) {
-			r = -ENOMEM;
-			goto free_fence_info;
+		if (num_fences >= wait_info->num_fences) {
+			r = -EINVAL;
+			goto free_fences;
 		}
 
-		/* Retrieve GEM read objects fence */
-		for (i = 0; i < num_read_bo_handles; i++) {
-			struct dma_resv_iter resv_cursor;
-			struct dma_fence *fence;
+		/* Give the reference to the fence array */
+		fences[num_fences++] = fence;
+	}
 
-			dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
-						DMA_RESV_USAGE_READ, fence) {
-				if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) {
-					r = -EINVAL;
-					goto free_fences;
-				}
+	/* Lock all the GEM objects */
+	num_read_bo_handles = wait_info->num_bo_read_handles;
+	num_write_bo_handles = wait_info->num_bo_write_handles;
+	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
+		      num_read_bo_handles + num_write_bo_handles);
 
-				fences[num_fences++] = fence;
-				dma_fence_get(fence);
-			}
-		}
+	drm_exec_until_all_locked(&exec) {
+		r = drm_exec_prepare_array(&exec, gobj_read,
+					   num_read_bo_handles, 1);
+		drm_exec_retry_on_contention(&exec);
+		if (r)
+			goto error_unlock;
 
-		/* Retrieve GEM write objects fence */
-		for (i = 0; i < num_write_bo_handles; i++) {
-			struct dma_resv_iter resv_cursor;
-			struct dma_fence *fence;
+		r = drm_exec_prepare_array(&exec, gobj_write,
+					   num_write_bo_handles, 1);
+		drm_exec_retry_on_contention(&exec);
+		if (r)
+			goto error_unlock;
+	}
 
-			dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
-						DMA_RESV_USAGE_WRITE, fence) {
-				if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) {
-					r = -EINVAL;
-					goto free_fences;
-				}
+	/* Retrieve GEM read objects fence */
+	for (i = 0; i < num_read_bo_handles; i++) {
+		struct dma_resv_iter resv_cursor;
+		struct dma_fence *fence;
 
-				fences[num_fences++] = fence;
-				dma_fence_get(fence);
+		dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
+					DMA_RESV_USAGE_READ, fence) {
+			if (num_fences >= wait_info->num_fences) {
+				r = -EINVAL;
+				goto error_unlock;
 			}
-		}
 
-		if (num_points) {
-			struct dma_fence_unwrap iter;
-			struct dma_fence *fence;
-			struct dma_fence *f;
-
-			for (i = 0; i < num_points; i++) {
-				r = drm_syncobj_find_fence(filp, timeline_handles[i],
-							   timeline_points[i],
-							   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
-							   &fence);
-				if (r)
-					goto free_fences;
-
-				dma_fence_unwrap_for_each(f, &iter, fence) {
-					if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) {
-						r = -EINVAL;
-						goto free_fences;
-					}
-
-					dma_fence_get(f);
-					fences[num_fences++] = f;
-				}
-
-				dma_fence_put(fence);
-			}
+			fences[num_fences++] = dma_fence_get(fence);
 		}
+	}
 
-		/* Retrieve syncobj's fence */
-		for (i = 0; i < num_syncobj; i++) {
-			struct dma_fence *fence;
-
-			r = drm_syncobj_find_fence(filp, syncobj_handles[i],
-						   0,
-						   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
-						   &fence);
-			if (r)
-				goto free_fences;
+	/* Retrieve GEM write objects fence */
+	for (i = 0; i < num_write_bo_handles; i++) {
+		struct dma_resv_iter resv_cursor;
+		struct dma_fence *fence;
 
-			if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) {
+		dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
+					DMA_RESV_USAGE_WRITE, fence) {
+			if (num_fences >= wait_info->num_fences) {
 				r = -EINVAL;
-				goto free_fences;
+				goto error_unlock;
 			}
 
-			fences[num_fences++] = fence;
+			fences[num_fences++] = dma_fence_get(fence);
 		}
+	}
 
-		/*
-		 * Keep only the latest fences to reduce the number of values
-		 * given back to userspace.
-		 */
-		num_fences = dma_fence_dedup_array(fences, num_fences);
+	drm_exec_fini(&exec);
 
-		waitq = xa_load(&userq_mgr->userq_xa, wait_info->waitq_id);
-		if (!waitq) {
-			r = -EINVAL;
-			goto free_fences;
-		}
+	/*
+	 * Keep only the latest fences to reduce the number of values
+	 * given back to userspace.
+	 */
+	num_fences = dma_fence_dedup_array(fences, num_fences);
 
-		for (i = 0, cnt = 0; i < num_fences; i++) {
-			struct amdgpu_userq_fence_driver *fence_drv;
-			struct amdgpu_userq_fence *userq_fence;
-			u32 index;
-
-			userq_fence = to_amdgpu_userq_fence(fences[i]);
-			if (!userq_fence) {
-				/*
-				 * Just waiting on other driver fences should
-				 * be good for now
-				 */
-				r = dma_fence_wait(fences[i], true);
-				if (r) {
-					dma_fence_put(fences[i]);
-					goto free_fences;
-				}
+	waitq = xa_load(&userq_mgr->userq_xa, wait_info->waitq_id);
+	if (!waitq) {
+		r = -EINVAL;
+		goto free_fences;
+	}
 
-				dma_fence_put(fences[i]);
-				continue;
-			}
+	for (i = 0, cnt = 0; i < num_fences; i++) {
+		struct amdgpu_userq_fence_driver *fence_drv;
+		struct amdgpu_userq_fence *userq_fence;
+		u32 index;
 
-			fence_drv = userq_fence->fence_drv;
+		userq_fence = to_amdgpu_userq_fence(fences[i]);
+		if (!userq_fence) {
 			/*
-			 * We need to make sure the user queue release their reference
-			 * to the fence drivers at some point before queue destruction.
-			 * Otherwise, we would gather those references until we don't
-			 * have any more space left and crash.
+			 * Just waiting on other driver fences should
+			 * be good for now
 			 */
-			r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
-				     xa_limit_32b, GFP_KERNEL);
-			if (r)
+			r = dma_fence_wait(fences[i], true);
+			if (r) {
+				dma_fence_put(fences[i]);
 				goto free_fences;
-
-			amdgpu_userq_fence_driver_get(fence_drv);
-
-			/* Store drm syncobj's gpu va address and value */
-			fence_info[cnt].va = fence_drv->va;
-			fence_info[cnt].value = fences[i]->seqno;
+			}
 
 			dma_fence_put(fences[i]);
-			/* Increment the actual userq fence count */
-			cnt++;
+			continue;
 		}
 
-		wait_info->num_fences = cnt;
-		/* Copy userq fence info to user space */
-		if (copy_to_user(u64_to_user_ptr(wait_info->out_fences),
-				 fence_info, wait_info->num_fences * sizeof(*fence_info))) {
-			r = -EFAULT;
+		fence_drv = userq_fence->fence_drv;
+		/*
+		 * We need to make sure the user queue release their reference
+		 * to the fence drivers at some point before queue destruction.
+		 * Otherwise, we would gather those references until we don't
+		 * have any more space left and crash.
+		 */
+		r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
+			     xa_limit_32b, GFP_KERNEL);
+		if (r)
 			goto free_fences;
-		}
 
-		kfree(fences);
-		kfree(fence_info);
-	}
+		amdgpu_userq_fence_driver_get(fence_drv);
 
-	drm_exec_fini(&exec);
-	for (i = 0; i < num_read_bo_handles; i++)
-		drm_gem_object_put(gobj_read[i]);
-	kfree(gobj_read);
+		/* Store drm syncobj's gpu va address and value */
+		fence_info[cnt].va = fence_drv->va;
+		fence_info[cnt].value = fences[i]->seqno;
 
-	for (i = 0; i < num_write_bo_handles; i++)
-		drm_gem_object_put(gobj_write[i]);
-	kfree(gobj_write);
+		dma_fence_put(fences[i]);
+		/* Increment the actual userq fence count */
+		cnt++;
+	}
+	/* The refcount in the array is already decremented */
+	num_fences = 0;
 
-	kfree(timeline_points);
-	kfree(timeline_handles);
-	kfree(syncobj_handles);
-	kfree(bo_handles_write);
-	kfree(bo_handles_read);
+	wait_info->num_fences = cnt;
+	r = 0;
 
-	return 0;
+	/* Copy userq fence info to user space */
+	if (copy_to_user(u64_to_user_ptr(wait_info->out_fences),
+			 fence_info, cnt * sizeof(*fence_info))) {
+		r = -EFAULT;
+	}
 
 free_fences:
-	while (num_fences-- > 0)
+	while (num_fences--)
 		dma_fence_put(fences[num_fences]);
 	kfree(fences);
+
 free_fence_info:
 	kfree(fence_info);
-exec_fini:
+	return r;
+
+error_unlock:
 	drm_exec_fini(&exec);
-put_gobj_write:
-	while (wentry-- > 0)
-		drm_gem_object_put(gobj_write[wentry]);
+	goto free_fences;
+}
+
+
+int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
+			    struct drm_file *filp)
+{
+	int num_points, num_syncobj, num_read_bo_handles, num_write_bo_handles;
+	u32 *syncobj_handles, *timeline_points, *timeline_handles;
+	struct drm_amdgpu_userq_wait *wait_info = data;
+	struct drm_gem_object **gobj_write;
+	struct drm_gem_object **gobj_read;
+	void __user *ptr;
+	int r;
+
+	if (!amdgpu_userq_enabled(dev))
+		return -ENOTSUPP;
+
+	num_syncobj = wait_info->num_syncobj_handles;
+	ptr = u64_to_user_ptr(wait_info->syncobj_handles);
+	syncobj_handles = memdup_array_user(ptr, num_syncobj, sizeof(u32));
+	if (IS_ERR(syncobj_handles))
+		return PTR_ERR(syncobj_handles);
+
+	num_points = wait_info->num_syncobj_timeline_handles;
+	ptr = u64_to_user_ptr(wait_info->syncobj_timeline_handles);
+	timeline_handles = memdup_array_user(ptr, num_points, sizeof(u32));
+	if (IS_ERR(timeline_handles)) {
+		r = PTR_ERR(timeline_handles);
+		goto free_syncobj_handles;
+	}
+
+	ptr = u64_to_user_ptr(wait_info->syncobj_timeline_points);
+	timeline_points = memdup_array_user(ptr, num_points, sizeof(u32));
+	if (IS_ERR(timeline_points)) {
+		r = PTR_ERR(timeline_points);
+		goto free_timeline_handles;
+	}
+
+	gobj_read = kmalloc_array(num_read_bo_handles, sizeof(*gobj_read),
+				  GFP_KERNEL);
+	if (!gobj_read) {
+		r = -ENOMEM;
+		goto free_timeline_points;
+	}
+
+	ptr = u64_to_user_ptr(wait_info->bo_read_handles);
+	num_read_bo_handles = wait_info->num_bo_read_handles;
+	r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
+	if (r)
+		goto free_timeline_points;
+
+	ptr = u64_to_user_ptr(wait_info->bo_write_handles);
+	num_write_bo_handles = wait_info->num_bo_write_handles;
+	r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
+				   &gobj_write);
+	if (r)
+		goto put_gobj_read;
+
+	/*
+	 * Passing num_fences = 0 means that userspace doesn't want to
+	 * retrieve userq_fence_info. If num_fences = 0 we skip filling
+	 * userq_fence_info and return the actual number of fences on
+	 * args->num_fences.
+	 */
+	if (!wait_info->num_fences) {
+		r = amdgpu_userq_wait_count_fences(filp, wait_info,
+						   syncobj_handles,
+						   timeline_points,
+						   timeline_handles,
+						   gobj_write,
+						   gobj_read);
+	} else {
+		r = amdgpu_userq_wait_return_fence_info(filp, wait_info,
+							syncobj_handles,
+							timeline_points,
+							timeline_handles,
+							gobj_write,
+							gobj_read);
+	}
+
+	while (num_write_bo_handles--)
+		drm_gem_object_put(gobj_write[num_write_bo_handles]);
 	kfree(gobj_write);
 put_gobj_read:
-	while (rentry-- > 0)
-		drm_gem_object_put(gobj_read[rentry]);
+	while (num_read_bo_handles--)
+		drm_gem_object_put(gobj_read[num_read_bo_handles]);
 	kfree(gobj_read);
 free_timeline_points:
 	kfree(timeline_points);
@@ -1014,10 +1043,5 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
 	kfree(timeline_handles);
 free_syncobj_handles:
 	kfree(syncobj_handles);
-free_bo_handles_write:
-	kfree(bo_handles_write);
-free_bo_handles_read:
-	kfree(bo_handles_read);
-
 	return r;
 }
-- 
2.43.0

[PATCH 8/9] drm/amdgpu: make amdgpu_user_wait_ioctl more resilent v2

[Christian König]

When the memory allocated by userspace isn't sufficient for all the
fences then just wait on them instead of returning an error.

v2: use correct variable as pointed out by Sunil

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 52 +++++++++++--------
 1 file changed, 29 insertions(+), 23 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
index da329d00843b..8a1a0af59910 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
@@ -734,7 +734,7 @@ amdgpu_userq_wait_count_fences(struct drm_file *filp,
 			num_fences++;
 	}
 
-	wait_info->num_fences = num_fences;
+	wait_info->num_fences = min(num_fences, USHRT_MAX);
 	r = 0;
 
 error_unlock:
@@ -743,6 +743,19 @@ amdgpu_userq_wait_count_fences(struct drm_file *filp,
 	return r;
 }
 
+static int
+amdgpu_userq_wait_add_fence(struct drm_amdgpu_userq_wait *wait_info,
+			    struct dma_fence **fences, unsigned int *num_fences,
+			    struct dma_fence *fence)
+{
+	/* As fallback shouldn't userspace allocate enough space */
+	if (*num_fences >= wait_info->num_fences)
+		return dma_fence_wait(fence, true);
+
+	fences[(*num_fences)++] = dma_fence_get(fence);
+	return 0;
+}
+
 static int
 amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
 				    struct drm_amdgpu_userq_wait *wait_info,
@@ -786,12 +799,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
 			goto free_fences;
 
 		dma_fence_unwrap_for_each(f, &iter, fence) {
-			if (num_fences >= wait_info->num_fences) {
-				r = -EINVAL;
+			r = amdgpu_userq_wait_add_fence(wait_info, fences,
+							&num_fences, f);
+			if (r)
 				goto free_fences;
-			}
-
-			fences[num_fences++] = dma_fence_get(f);
 		}
 
 		dma_fence_put(fence);
@@ -808,13 +819,12 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
 		if (r)
 			goto free_fences;
 
-		if (num_fences >= wait_info->num_fences) {
-			r = -EINVAL;
+		r = amdgpu_userq_wait_add_fence(wait_info, fences,
+						&num_fences, fence);
+		if (r)
 			goto free_fences;
-		}
 
-		/* Give the reference to the fence array */
-		fences[num_fences++] = fence;
+		dma_fence_put(fence);
 	}
 
 	/* Lock all the GEM objects */
@@ -844,12 +854,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
 
 		dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
 					DMA_RESV_USAGE_READ, fence) {
-			if (num_fences >= wait_info->num_fences) {
-				r = -EINVAL;
-				goto error_unlock;
-			}
-
-			fences[num_fences++] = dma_fence_get(fence);
+			r = amdgpu_userq_wait_add_fence(wait_info, fences,
+							&num_fences, fence);
+			if (r)
+				goto free_fences;
 		}
 	}
 
@@ -860,12 +868,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
 
 		dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
 					DMA_RESV_USAGE_WRITE, fence) {
-			if (num_fences >= wait_info->num_fences) {
-				r = -EINVAL;
-				goto error_unlock;
-			}
-
-			fences[num_fences++] = dma_fence_get(fence);
+			r = amdgpu_userq_wait_add_fence(wait_info, fences,
+							&num_fences, fence);
+			if (r)
+				goto free_fences;
 		}
 	}
 
-- 
2.43.0

[PATCH 9/9] drm/amdgpu: annotate eviction fence signaling path

[Christian König]

Make sure lockdep sees the dependencies here.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
index 6598823ec619..e5aaa5156069 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
@@ -67,8 +67,17 @@ amdgpu_eviction_fence_suspend_worker(struct work_struct *work)
 		container_of(evf_mgr, struct amdgpu_fpriv, evf_mgr);
 	struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr;
 	struct dma_fence *ev_fence;
+	bool cookie;
 
 	mutex_lock(&uq_mgr->userq_mutex);
+
+	/*
+	 * This is intentionally after taking the userq_mutex since we do
+	 * allocate memory while holding this lock, but only after ensuring that
+	 * the eviction fence is signaled.
+	 */
+	cookie = dma_fence_begin_signalling();
+
 	ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);
 	amdgpu_userq_evict(uq_mgr, !evf_mgr->shutdown);
 
@@ -78,6 +87,7 @@ amdgpu_eviction_fence_suspend_worker(struct work_struct *work)
 	 * next fence.
 	 */
 	dma_fence_signal(ev_fence);
+	dma_fence_end_signalling(cookie);
 	dma_fence_put(ev_fence);
 	mutex_unlock(&uq_mgr->userq_mutex);
 }
-- 
2.43.0

Re: [PATCH 7/9] drm/amdgpu: rework amdgpu_userq_wait_ioctl v2

[Tvrtko Ursulin]

On 02/02/2026 12:51, Christian König wrote:
> Lockdep was complaining about a number of issues here. Especially lock
> inversion between syncobj, dma_resv and copying things into userspace.
> 
> Rework the functionality. Split it up into multiple functions,
> consistenly use memdup_array_user(), fix the lock inversions and a few
> more bugs in error handling.
> 
> v2: drop the dma_fence leak fix, turned out that was actually correct,
>      just not well documented. Apply some more cleanup suggestion from
>      Tvrtko.

Which dma_fence leak was that? The two I thought I found in
https://lore.kernel.org/amd-gfx/20251205134035.91551-2-tvrtko.ursulin@igalia.com/, 
ie:

	/* Retrieve timeline fences */
	num_points = wait_info->num_syncobj_timeline_handles;
	for (i = 0; i < num_points; i++) {
		r = drm_syncobj_find_fence(filp, timeline_handles[i],
					   timeline_points[i],
					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
					   &fence);
		if (r)
			goto free_fences;

		dma_fence_unwrap_for_each(f, &iter, fence) {
			r = amdgpu_userq_wait_add_fence(wait_info, fences,
							&num_fences, f);
			if (r)
				goto free_fences;

^^^ Error path leaks fence. It doesn't?

		dma_fence_put(fence);
	}

Regards,

Tvrtko

> 
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>   .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 614 +++++++++---------
>   1 file changed, 319 insertions(+), 295 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> index 212056d4ddf0..da329d00843b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> @@ -645,368 +645,397 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>   	return r;
>   }
>   
> -int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
> -			    struct drm_file *filp)
> +/* Count the number of expected fences so userspace can alloc a buffer */
> +static int
> +amdgpu_userq_wait_count_fences(struct drm_file *filp,
> +			       struct drm_amdgpu_userq_wait *wait_info,
> +			       u32 *syncobj_handles, u32 *timeline_points,
> +			       u32 *timeline_handles,
> +			       struct drm_gem_object **gobj_write,
> +			       struct drm_gem_object **gobj_read)
>   {
> -	u32 *syncobj_handles, *timeline_points, *timeline_handles, *bo_handles_read, *bo_handles_write;
> -	u32 num_syncobj, num_read_bo_handles, num_write_bo_handles;
> -	struct drm_amdgpu_userq_fence_info *fence_info = NULL;
> -	struct drm_amdgpu_userq_wait *wait_info = data;
> -	struct amdgpu_fpriv *fpriv = filp->driver_priv;
> -	struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
> -	struct amdgpu_usermode_queue *waitq;
> -	struct drm_gem_object **gobj_write;
> -	struct drm_gem_object **gobj_read;
> -	struct dma_fence **fences = NULL;
> -	u16 num_points, num_fences = 0;
> -	int r, i, rentry, wentry, cnt;
> +	int num_read_bo_handles, num_write_bo_handles;
> +	struct dma_fence_unwrap iter;
> +	struct dma_fence *fence, *f;
> +	unsigned int num_fences = 0;
>   	struct drm_exec exec;
> +	int i, r;
> +
> +	/*
> +	 * This needs to be outside of the lock provided by drm_exec for
> +	 * DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT to work correctly.
> +	 */
> +
> +	/* Count timeline fences */
> +	for (i = 0; i < wait_info->num_syncobj_timeline_handles; i++) {
> +		r = drm_syncobj_find_fence(filp, timeline_handles[i],
> +					   timeline_points[i],
> +					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> +					   &fence);
> +		if (r)
> +			return r;
> +
> +		dma_fence_unwrap_for_each(f, &iter, fence)
> +			num_fences++;
>   
> -	if (!amdgpu_userq_enabled(dev))
> -		return -ENOTSUPP;
> +		dma_fence_put(fence);
> +	}
>   
> -	num_read_bo_handles = wait_info->num_bo_read_handles;
> -	bo_handles_read = memdup_user(u64_to_user_ptr(wait_info->bo_read_handles),
> -				      size_mul(sizeof(u32), num_read_bo_handles));
> -	if (IS_ERR(bo_handles_read))
> -		return PTR_ERR(bo_handles_read);
> +	/* Count boolean fences */
> +	for (i = 0; i < wait_info->num_syncobj_handles; i++) {
> +		r = drm_syncobj_find_fence(filp, syncobj_handles[i], 0,
> +					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> +					   &fence);
> +		if (r)
> +			return r;
>   
> -	num_write_bo_handles = wait_info->num_bo_write_handles;
> -	bo_handles_write = memdup_user(u64_to_user_ptr(wait_info->bo_write_handles),
> -				       size_mul(sizeof(u32), num_write_bo_handles));
> -	if (IS_ERR(bo_handles_write)) {
> -		r = PTR_ERR(bo_handles_write);
> -		goto free_bo_handles_read;
> +		num_fences++;
> +		dma_fence_put(fence);
>   	}
>   
> -	num_syncobj = wait_info->num_syncobj_handles;
> -	syncobj_handles = memdup_user(u64_to_user_ptr(wait_info->syncobj_handles),
> -				      size_mul(sizeof(u32), num_syncobj));
> -	if (IS_ERR(syncobj_handles)) {
> -		r = PTR_ERR(syncobj_handles);
> -		goto free_bo_handles_write;
> -	}
> +	/* Lock all the GEM objects */
> +	/* TODO: It is actually not necessary to lock them */
> +	num_read_bo_handles = wait_info->num_bo_read_handles;
> +	num_write_bo_handles = wait_info->num_bo_write_handles;
> +	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
> +		      num_read_bo_handles + num_write_bo_handles);
>   
> -	num_points = wait_info->num_syncobj_timeline_handles;
> -	timeline_handles = memdup_user(u64_to_user_ptr(wait_info->syncobj_timeline_handles),
> -				       sizeof(u32) * num_points);
> -	if (IS_ERR(timeline_handles)) {
> -		r = PTR_ERR(timeline_handles);
> -		goto free_syncobj_handles;
> -	}
> +	drm_exec_until_all_locked(&exec) {
> +		r = drm_exec_prepare_array(&exec, gobj_read,
> +					   num_read_bo_handles, 1);
> +		drm_exec_retry_on_contention(&exec);
> +		if (r)
> +			goto error_unlock;
>   
> -	timeline_points = memdup_user(u64_to_user_ptr(wait_info->syncobj_timeline_points),
> -				      sizeof(u32) * num_points);
> -	if (IS_ERR(timeline_points)) {
> -		r = PTR_ERR(timeline_points);
> -		goto free_timeline_handles;
> +		r = drm_exec_prepare_array(&exec, gobj_write,
> +					   num_write_bo_handles, 1);
> +		drm_exec_retry_on_contention(&exec);
> +		if (r)
> +			goto error_unlock;
>   	}
>   
> -	gobj_read = kmalloc_array(num_read_bo_handles, sizeof(*gobj_read), GFP_KERNEL);
> -	if (!gobj_read) {
> -		r = -ENOMEM;
> -		goto free_timeline_points;
> -	}
> +	/* Count read fences */
> +	for (i = 0; i < num_read_bo_handles; i++) {
> +		struct dma_resv_iter resv_cursor;
> +		struct dma_fence *fence;
>   
> -	for (rentry = 0; rentry < num_read_bo_handles; rentry++) {
> -		gobj_read[rentry] = drm_gem_object_lookup(filp, bo_handles_read[rentry]);
> -		if (!gobj_read[rentry]) {
> -			r = -ENOENT;
> -			goto put_gobj_read;
> -		}
> +		dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
> +					DMA_RESV_USAGE_READ, fence)
> +			num_fences++;
>   	}
>   
> -	gobj_write = kmalloc_array(num_write_bo_handles, sizeof(*gobj_write), GFP_KERNEL);
> -	if (!gobj_write) {
> -		r = -ENOMEM;
> -		goto put_gobj_read;
> -	}
> +	/* Count write fences */
> +	for (i = 0; i < num_write_bo_handles; i++) {
> +		struct dma_resv_iter resv_cursor;
> +		struct dma_fence *fence;
>   
> -	for (wentry = 0; wentry < num_write_bo_handles; wentry++) {
> -		gobj_write[wentry] = drm_gem_object_lookup(filp, bo_handles_write[wentry]);
> -		if (!gobj_write[wentry]) {
> -			r = -ENOENT;
> -			goto put_gobj_write;
> -		}
> +		dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
> +					DMA_RESV_USAGE_WRITE, fence)
> +			num_fences++;
>   	}
>   
> -	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
> -		      (num_read_bo_handles + num_write_bo_handles));
> -
> -	/* Lock all BOs with retry handling */
> -	drm_exec_until_all_locked(&exec) {
> -		r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1);
> -		drm_exec_retry_on_contention(&exec);
> -		if (r) {
> -			drm_exec_fini(&exec);
> -			goto put_gobj_write;
> -		}
> +	wait_info->num_fences = num_fences;
> +	r = 0;
>   
> -		r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1);
> -		drm_exec_retry_on_contention(&exec);
> -		if (r) {
> -			drm_exec_fini(&exec);
> -			goto put_gobj_write;
> -		}
> -	}
> +error_unlock:
> +	/* Unlock all the GEM objects */
> +	drm_exec_fini(&exec);
> +	return r;
> +}
>   
> -	if (!wait_info->num_fences) {
> -		if (num_points) {
> -			struct dma_fence_unwrap iter;
> -			struct dma_fence *fence;
> -			struct dma_fence *f;
> -
> -			for (i = 0; i < num_points; i++) {
> -				r = drm_syncobj_find_fence(filp, timeline_handles[i],
> -							   timeline_points[i],
> -							   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> -							   &fence);
> -				if (r)
> -					goto exec_fini;
> -
> -				dma_fence_unwrap_for_each(f, &iter, fence)
> -					num_fences++;
> -
> -				dma_fence_put(fence);
> -			}
> -		}
> +static int
> +amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
> +				    struct drm_amdgpu_userq_wait *wait_info,
> +				    u32 *syncobj_handles, u32 *timeline_points,
> +				    u32 *timeline_handles,
> +				    struct drm_gem_object **gobj_write,
> +				    struct drm_gem_object **gobj_read)
> +{
> +	struct amdgpu_fpriv *fpriv = filp->driver_priv;
> +	struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
> +	struct drm_amdgpu_userq_fence_info *fence_info;
> +	int num_read_bo_handles, num_write_bo_handles;
> +	struct amdgpu_usermode_queue *waitq;
> +	struct dma_fence **fences, *fence, *f;
> +	struct dma_fence_unwrap iter;
> +	int num_points, num_syncobj;
> +	unsigned int num_fences = 0;
> +	struct drm_exec exec;
> +	int i, cnt, r;
>   
> -		/* Count syncobj's fence */
> -		for (i = 0; i < num_syncobj; i++) {
> -			struct dma_fence *fence;
> +	fence_info = kmalloc_array(wait_info->num_fences, sizeof(*fence_info),
> +				   GFP_KERNEL);
> +	if (!fence_info)
> +		return -ENOMEM;
>   
> -			r = drm_syncobj_find_fence(filp, syncobj_handles[i],
> -						   0,
> -						   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> -						   &fence);
> -			if (r)
> -				goto exec_fini;
> +	fences = kmalloc_array(wait_info->num_fences, sizeof(*fences),
> +			       GFP_KERNEL);
> +	if (!fences) {
> +		r = -ENOMEM;
> +		goto free_fence_info;
> +	}
>   
> -			num_fences++;
> -			dma_fence_put(fence);
> -		}
> +	/* Retrieve timeline fences */
> +	num_points = wait_info->num_syncobj_timeline_handles;
> +	for (i = 0; i < num_points; i++) {
> +		r = drm_syncobj_find_fence(filp, timeline_handles[i],
> +					   timeline_points[i],
> +					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> +					   &fence);
> +		if (r)
> +			goto free_fences;
>   
> -		/* Count GEM objects fence */
> -		for (i = 0; i < num_read_bo_handles; i++) {
> -			struct dma_resv_iter resv_cursor;
> -			struct dma_fence *fence;
> +		dma_fence_unwrap_for_each(f, &iter, fence) {
> +			if (num_fences >= wait_info->num_fences) {
> +				r = -EINVAL;
> +				goto free_fences;
> +			}
>   
> -			dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
> -						DMA_RESV_USAGE_READ, fence)
> -				num_fences++;
> +			fences[num_fences++] = dma_fence_get(f);
>   		}
>   
> -		for (i = 0; i < num_write_bo_handles; i++) {
> -			struct dma_resv_iter resv_cursor;
> -			struct dma_fence *fence;
> +		dma_fence_put(fence);
> +	}
>   
> -			dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
> -						DMA_RESV_USAGE_WRITE, fence)
> -				num_fences++;
> -		}
> +	/* Retrieve boolean fences */
> +	num_syncobj = wait_info->num_syncobj_handles;
> +	for (i = 0; i < num_syncobj; i++) {
> +		struct dma_fence *fence;
>   
> -		/*
> -		 * Passing num_fences = 0 means that userspace doesn't want to
> -		 * retrieve userq_fence_info. If num_fences = 0 we skip filling
> -		 * userq_fence_info and return the actual number of fences on
> -		 * args->num_fences.
> -		 */
> -		wait_info->num_fences = num_fences;
> -	} else {
> -		/* Array of fence info */
> -		fence_info = kmalloc_array(wait_info->num_fences, sizeof(*fence_info), GFP_KERNEL);
> -		if (!fence_info) {
> -			r = -ENOMEM;
> -			goto exec_fini;
> -		}
> +		r = drm_syncobj_find_fence(filp, syncobj_handles[i], 0,
> +					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> +					   &fence);
> +		if (r)
> +			goto free_fences;
>   
> -		/* Array of fences */
> -		fences = kmalloc_array(wait_info->num_fences, sizeof(*fences), GFP_KERNEL);
> -		if (!fences) {
> -			r = -ENOMEM;
> -			goto free_fence_info;
> +		if (num_fences >= wait_info->num_fences) {
> +			r = -EINVAL;
> +			goto free_fences;
>   		}
>   
> -		/* Retrieve GEM read objects fence */
> -		for (i = 0; i < num_read_bo_handles; i++) {
> -			struct dma_resv_iter resv_cursor;
> -			struct dma_fence *fence;
> +		/* Give the reference to the fence array */
> +		fences[num_fences++] = fence;
> +	}
>   
> -			dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
> -						DMA_RESV_USAGE_READ, fence) {
> -				if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) {
> -					r = -EINVAL;
> -					goto free_fences;
> -				}
> +	/* Lock all the GEM objects */
> +	num_read_bo_handles = wait_info->num_bo_read_handles;
> +	num_write_bo_handles = wait_info->num_bo_write_handles;
> +	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
> +		      num_read_bo_handles + num_write_bo_handles);
>   
> -				fences[num_fences++] = fence;
> -				dma_fence_get(fence);
> -			}
> -		}
> +	drm_exec_until_all_locked(&exec) {
> +		r = drm_exec_prepare_array(&exec, gobj_read,
> +					   num_read_bo_handles, 1);
> +		drm_exec_retry_on_contention(&exec);
> +		if (r)
> +			goto error_unlock;
>   
> -		/* Retrieve GEM write objects fence */
> -		for (i = 0; i < num_write_bo_handles; i++) {
> -			struct dma_resv_iter resv_cursor;
> -			struct dma_fence *fence;
> +		r = drm_exec_prepare_array(&exec, gobj_write,
> +					   num_write_bo_handles, 1);
> +		drm_exec_retry_on_contention(&exec);
> +		if (r)
> +			goto error_unlock;
> +	}
>   
> -			dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
> -						DMA_RESV_USAGE_WRITE, fence) {
> -				if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) {
> -					r = -EINVAL;
> -					goto free_fences;
> -				}
> +	/* Retrieve GEM read objects fence */
> +	for (i = 0; i < num_read_bo_handles; i++) {
> +		struct dma_resv_iter resv_cursor;
> +		struct dma_fence *fence;
>   
> -				fences[num_fences++] = fence;
> -				dma_fence_get(fence);
> +		dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
> +					DMA_RESV_USAGE_READ, fence) {
> +			if (num_fences >= wait_info->num_fences) {
> +				r = -EINVAL;
> +				goto error_unlock;
>   			}
> -		}
>   
> -		if (num_points) {
> -			struct dma_fence_unwrap iter;
> -			struct dma_fence *fence;
> -			struct dma_fence *f;
> -
> -			for (i = 0; i < num_points; i++) {
> -				r = drm_syncobj_find_fence(filp, timeline_handles[i],
> -							   timeline_points[i],
> -							   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> -							   &fence);
> -				if (r)
> -					goto free_fences;
> -
> -				dma_fence_unwrap_for_each(f, &iter, fence) {
> -					if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) {
> -						r = -EINVAL;
> -						goto free_fences;
> -					}
> -
> -					dma_fence_get(f);
> -					fences[num_fences++] = f;
> -				}
> -
> -				dma_fence_put(fence);
> -			}
> +			fences[num_fences++] = dma_fence_get(fence);
>   		}
> +	}
>   
> -		/* Retrieve syncobj's fence */
> -		for (i = 0; i < num_syncobj; i++) {
> -			struct dma_fence *fence;
> -
> -			r = drm_syncobj_find_fence(filp, syncobj_handles[i],
> -						   0,
> -						   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> -						   &fence);
> -			if (r)
> -				goto free_fences;
> +	/* Retrieve GEM write objects fence */
> +	for (i = 0; i < num_write_bo_handles; i++) {
> +		struct dma_resv_iter resv_cursor;
> +		struct dma_fence *fence;
>   
> -			if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) {
> +		dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
> +					DMA_RESV_USAGE_WRITE, fence) {
> +			if (num_fences >= wait_info->num_fences) {
>   				r = -EINVAL;
> -				goto free_fences;
> +				goto error_unlock;
>   			}
>   
> -			fences[num_fences++] = fence;
> +			fences[num_fences++] = dma_fence_get(fence);
>   		}
> +	}
>   
> -		/*
> -		 * Keep only the latest fences to reduce the number of values
> -		 * given back to userspace.
> -		 */
> -		num_fences = dma_fence_dedup_array(fences, num_fences);
> +	drm_exec_fini(&exec);
>   
> -		waitq = xa_load(&userq_mgr->userq_xa, wait_info->waitq_id);
> -		if (!waitq) {
> -			r = -EINVAL;
> -			goto free_fences;
> -		}
> +	/*
> +	 * Keep only the latest fences to reduce the number of values
> +	 * given back to userspace.
> +	 */
> +	num_fences = dma_fence_dedup_array(fences, num_fences);
>   
> -		for (i = 0, cnt = 0; i < num_fences; i++) {
> -			struct amdgpu_userq_fence_driver *fence_drv;
> -			struct amdgpu_userq_fence *userq_fence;
> -			u32 index;
> -
> -			userq_fence = to_amdgpu_userq_fence(fences[i]);
> -			if (!userq_fence) {
> -				/*
> -				 * Just waiting on other driver fences should
> -				 * be good for now
> -				 */
> -				r = dma_fence_wait(fences[i], true);
> -				if (r) {
> -					dma_fence_put(fences[i]);
> -					goto free_fences;
> -				}
> +	waitq = xa_load(&userq_mgr->userq_xa, wait_info->waitq_id);
> +	if (!waitq) {
> +		r = -EINVAL;
> +		goto free_fences;
> +	}
>   
> -				dma_fence_put(fences[i]);
> -				continue;
> -			}
> +	for (i = 0, cnt = 0; i < num_fences; i++) {
> +		struct amdgpu_userq_fence_driver *fence_drv;
> +		struct amdgpu_userq_fence *userq_fence;
> +		u32 index;
>   
> -			fence_drv = userq_fence->fence_drv;
> +		userq_fence = to_amdgpu_userq_fence(fences[i]);
> +		if (!userq_fence) {
>   			/*
> -			 * We need to make sure the user queue release their reference
> -			 * to the fence drivers at some point before queue destruction.
> -			 * Otherwise, we would gather those references until we don't
> -			 * have any more space left and crash.
> +			 * Just waiting on other driver fences should
> +			 * be good for now
>   			 */
> -			r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
> -				     xa_limit_32b, GFP_KERNEL);
> -			if (r)
> +			r = dma_fence_wait(fences[i], true);
> +			if (r) {
> +				dma_fence_put(fences[i]);
>   				goto free_fences;
> -
> -			amdgpu_userq_fence_driver_get(fence_drv);
> -
> -			/* Store drm syncobj's gpu va address and value */
> -			fence_info[cnt].va = fence_drv->va;
> -			fence_info[cnt].value = fences[i]->seqno;
> +			}
>   
>   			dma_fence_put(fences[i]);
> -			/* Increment the actual userq fence count */
> -			cnt++;
> +			continue;
>   		}
>   
> -		wait_info->num_fences = cnt;
> -		/* Copy userq fence info to user space */
> -		if (copy_to_user(u64_to_user_ptr(wait_info->out_fences),
> -				 fence_info, wait_info->num_fences * sizeof(*fence_info))) {
> -			r = -EFAULT;
> +		fence_drv = userq_fence->fence_drv;
> +		/*
> +		 * We need to make sure the user queue release their reference
> +		 * to the fence drivers at some point before queue destruction.
> +		 * Otherwise, we would gather those references until we don't
> +		 * have any more space left and crash.
> +		 */
> +		r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
> +			     xa_limit_32b, GFP_KERNEL);
> +		if (r)
>   			goto free_fences;
> -		}
>   
> -		kfree(fences);
> -		kfree(fence_info);
> -	}
> +		amdgpu_userq_fence_driver_get(fence_drv);
>   
> -	drm_exec_fini(&exec);
> -	for (i = 0; i < num_read_bo_handles; i++)
> -		drm_gem_object_put(gobj_read[i]);
> -	kfree(gobj_read);
> +		/* Store drm syncobj's gpu va address and value */
> +		fence_info[cnt].va = fence_drv->va;
> +		fence_info[cnt].value = fences[i]->seqno;
>   
> -	for (i = 0; i < num_write_bo_handles; i++)
> -		drm_gem_object_put(gobj_write[i]);
> -	kfree(gobj_write);
> +		dma_fence_put(fences[i]);
> +		/* Increment the actual userq fence count */
> +		cnt++;
> +	}
> +	/* The refcount in the array is already decremented */
> +	num_fences = 0;
>   
> -	kfree(timeline_points);
> -	kfree(timeline_handles);
> -	kfree(syncobj_handles);
> -	kfree(bo_handles_write);
> -	kfree(bo_handles_read);
> +	wait_info->num_fences = cnt;
> +	r = 0;
>   
> -	return 0;
> +	/* Copy userq fence info to user space */
> +	if (copy_to_user(u64_to_user_ptr(wait_info->out_fences),
> +			 fence_info, cnt * sizeof(*fence_info))) {
> +		r = -EFAULT;
> +	}
>   
>   free_fences:
> -	while (num_fences-- > 0)
> +	while (num_fences--)
>   		dma_fence_put(fences[num_fences]);
>   	kfree(fences);
> +
>   free_fence_info:
>   	kfree(fence_info);
> -exec_fini:
> +	return r;
> +
> +error_unlock:
>   	drm_exec_fini(&exec);
> -put_gobj_write:
> -	while (wentry-- > 0)
> -		drm_gem_object_put(gobj_write[wentry]);
> +	goto free_fences;
> +}
> +
> +
> +int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
> +			    struct drm_file *filp)
> +{
> +	int num_points, num_syncobj, num_read_bo_handles, num_write_bo_handles;
> +	u32 *syncobj_handles, *timeline_points, *timeline_handles;
> +	struct drm_amdgpu_userq_wait *wait_info = data;
> +	struct drm_gem_object **gobj_write;
> +	struct drm_gem_object **gobj_read;
> +	void __user *ptr;
> +	int r;
> +
> +	if (!amdgpu_userq_enabled(dev))
> +		return -ENOTSUPP;
> +
> +	num_syncobj = wait_info->num_syncobj_handles;
> +	ptr = u64_to_user_ptr(wait_info->syncobj_handles);
> +	syncobj_handles = memdup_array_user(ptr, num_syncobj, sizeof(u32));
> +	if (IS_ERR(syncobj_handles))
> +		return PTR_ERR(syncobj_handles);
> +
> +	num_points = wait_info->num_syncobj_timeline_handles;
> +	ptr = u64_to_user_ptr(wait_info->syncobj_timeline_handles);
> +	timeline_handles = memdup_array_user(ptr, num_points, sizeof(u32));
> +	if (IS_ERR(timeline_handles)) {
> +		r = PTR_ERR(timeline_handles);
> +		goto free_syncobj_handles;
> +	}
> +
> +	ptr = u64_to_user_ptr(wait_info->syncobj_timeline_points);
> +	timeline_points = memdup_array_user(ptr, num_points, sizeof(u32));
> +	if (IS_ERR(timeline_points)) {
> +		r = PTR_ERR(timeline_points);
> +		goto free_timeline_handles;
> +	}
> +
> +	gobj_read = kmalloc_array(num_read_bo_handles, sizeof(*gobj_read),
> +				  GFP_KERNEL);
> +	if (!gobj_read) {
> +		r = -ENOMEM;
> +		goto free_timeline_points;
> +	}
> +
> +	ptr = u64_to_user_ptr(wait_info->bo_read_handles);
> +	num_read_bo_handles = wait_info->num_bo_read_handles;
> +	r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
> +	if (r)
> +		goto free_timeline_points;
> +
> +	ptr = u64_to_user_ptr(wait_info->bo_write_handles);
> +	num_write_bo_handles = wait_info->num_bo_write_handles;
> +	r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
> +				   &gobj_write);
> +	if (r)
> +		goto put_gobj_read;
> +
> +	/*
> +	 * Passing num_fences = 0 means that userspace doesn't want to
> +	 * retrieve userq_fence_info. If num_fences = 0 we skip filling
> +	 * userq_fence_info and return the actual number of fences on
> +	 * args->num_fences.
> +	 */
> +	if (!wait_info->num_fences) {
> +		r = amdgpu_userq_wait_count_fences(filp, wait_info,
> +						   syncobj_handles,
> +						   timeline_points,
> +						   timeline_handles,
> +						   gobj_write,
> +						   gobj_read);
> +	} else {
> +		r = amdgpu_userq_wait_return_fence_info(filp, wait_info,
> +							syncobj_handles,
> +							timeline_points,
> +							timeline_handles,
> +							gobj_write,
> +							gobj_read);
> +	}
> +
> +	while (num_write_bo_handles--)
> +		drm_gem_object_put(gobj_write[num_write_bo_handles]);
>   	kfree(gobj_write);
>   put_gobj_read:
> -	while (rentry-- > 0)
> -		drm_gem_object_put(gobj_read[rentry]);
> +	while (num_read_bo_handles--)
> +		drm_gem_object_put(gobj_read[num_read_bo_handles]);
>   	kfree(gobj_read);
>   free_timeline_points:
>   	kfree(timeline_points);
> @@ -1014,10 +1043,5 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
>   	kfree(timeline_handles);
>   free_syncobj_handles:
>   	kfree(syncobj_handles);
> -free_bo_handles_write:
> -	kfree(bo_handles_write);
> -free_bo_handles_read:
> -	kfree(bo_handles_read);
> -
>   	return r;
>   }

Re: [PATCH 1/9] drm/amdgpu: lock both VM and BO in amdgpu_gem_object_open

[Alex Deucher]

On Mon, Feb 2, 2026 at 7:51 AM Christian König
<ckoenig.leichtzumerken@gmail.com> wrote:
>
> The VM was not locked in the past since we initially only cleared the
> linked list element and not added it to any VM state.
>
> But this has changed quite some time ago, we just never realized this
> problem because the VM state lock was masking it.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>

Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

> ---
>  .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c  | 19 +++++++++++-----
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c       | 22 ++++++++++++++-----
>  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c       | 10 +++++++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        |  2 ++
>  4 files changed, 42 insertions(+), 11 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> index 768998c82b43..ec5130497743 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> @@ -878,6 +878,7 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
>         struct amdgpu_bo *bo[2] = {NULL, NULL};
>         struct amdgpu_bo_va *bo_va;
>         bool same_hive = false;
> +       struct drm_exec exec;
>         int i, ret;
>
>         if (!va) {
> @@ -958,19 +959,25 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
>                         goto unwind;
>                 }
>
> -               /* Add BO to VM internal data structures */
> -               ret = amdgpu_bo_reserve(bo[i], false);
> -               if (ret) {
> -                       pr_debug("Unable to reserve BO during memory attach");
> -                       goto unwind;
> +               drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
> +               drm_exec_until_all_locked(&exec) {
> +                       ret = amdgpu_vm_lock_pd(vm, &exec, 0);
> +                       drm_exec_retry_on_contention(&exec);
> +                       if (unlikely(ret))
> +                               goto unwind;
> +                       ret = drm_exec_lock_obj(&exec, &bo[i]->tbo.base);
> +                       drm_exec_retry_on_contention(&exec);
> +                       if (unlikely(ret))
> +                               goto unwind;
>                 }
> +
>                 bo_va = amdgpu_vm_bo_find(vm, bo[i]);
>                 if (!bo_va)
>                         bo_va = amdgpu_vm_bo_add(adev, vm, bo[i]);
>                 else
>                         ++bo_va->ref_count;
>                 attachment[i]->bo_va = bo_va;
> -               amdgpu_bo_unreserve(bo[i]);
> +               drm_exec_fini(&exec);
>                 if (unlikely(!attachment[i]->bo_va)) {
>                         ret = -ENOMEM;
>                         pr_err("Failed to add BO object to VM. ret == %d\n",
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> index 5f9fa2140f09..5c90de58cc28 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> @@ -232,6 +232,7 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
>         struct amdgpu_vm *vm = &fpriv->vm;
>         struct amdgpu_bo_va *bo_va;
>         struct mm_struct *mm;
> +       struct drm_exec exec;
>         int r;
>
>         mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm);
> @@ -242,9 +243,18 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
>             !amdgpu_vm_is_bo_always_valid(vm, abo))
>                 return -EPERM;
>
> -       r = amdgpu_bo_reserve(abo, false);
> -       if (r)
> -               return r;
> +       drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
> +       drm_exec_until_all_locked(&exec) {
> +               r = drm_exec_prepare_obj(&exec, &abo->tbo.base, 1);
> +               drm_exec_retry_on_contention(&exec);
> +               if (unlikely(r))
> +                       goto out_unlock;
> +
> +               r = amdgpu_vm_lock_pd(vm, &exec, 0);
> +               drm_exec_retry_on_contention(&exec);
> +               if (unlikely(r))
> +                       goto out_unlock;
> +       }
>
>         amdgpu_vm_bo_update_shared(abo);
>         bo_va = amdgpu_vm_bo_find(vm, abo);
> @@ -260,8 +270,7 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
>                 amdgpu_bo_unreserve(abo);
>                 return r;
>         }
> -
> -       amdgpu_bo_unreserve(abo);
> +       drm_exec_fini(&exec);
>
>         /* Validate and add eviction fence to DMABuf imports with dynamic
>          * attachment in compute VMs. Re-validation will be done by
> @@ -294,7 +303,10 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
>                 }
>         }
>         mutex_unlock(&vm->process_info->lock);
> +       return r;
>
> +out_unlock:
> +       drm_exec_fini(&exec);
>         return r;
>  }
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index 1878e0faa722..f69332eed051 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -1445,6 +1445,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
>  {
>         struct amdgpu_device *adev = drm_to_adev(dev);
>         struct amdgpu_fpriv *fpriv;
> +       struct drm_exec exec;
>         int r, pasid;
>
>         /* Ensure IB tests are run on ring */
> @@ -1484,7 +1485,16 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
>         if (r)
>                 goto error_pasid;
>
> +       drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
> +       drm_exec_until_all_locked(&exec) {
> +               r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 0);
> +               drm_exec_retry_on_contention(&exec);
> +               if (unlikely(r))
> +                       goto error_vm;
> +       }
> +
>         fpriv->prt_va = amdgpu_vm_bo_add(adev, &fpriv->vm, NULL);
> +       drm_exec_fini(&exec);
>         if (!fpriv->prt_va) {
>                 r = -ENOMEM;
>                 goto error_vm;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> index 6a2ea200d90c..b4bf1b7c214f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -1735,6 +1735,8 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
>  {
>         struct amdgpu_bo_va *bo_va;
>
> +       amdgpu_vm_assert_locked(vm);
> +
>         bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
>         if (bo_va == NULL) {
>                 return NULL;
> --
> 2.43.0
>

Re: [PATCH 2/9] drm/amdgpu: revert to old status lock handling v3

[Alex Deucher]

On Mon, Feb 2, 2026 at 1:12 PM Christian König
<ckoenig.leichtzumerken@gmail.com> wrote:
>
> This reverts commit 7e64d0e5979157ee5fc83e58ac61b4a36803c7f1.
>
> The problems were caused by not holding the VM lock while adding new
> BOs.

I think it would be clearer to restate the original commit message as
this just reverts a partial revert of a revert and the original goal
of the change is hard to sort out without digging through the commit
history.  E.g., from the original change:

Re-add a separate stats_lock and lockdep assertions that the correct
reservation lock is held all over the place.

This not only allows for better checks if a state transition is properly
protected by a lock, but also switching back to using list macros to
iterate over the state of lists protected by the dma_resv lock of the
root PD.

With that cleaned up,

Reviewed-by: Alex Deucher <alexander.deucher@amd.com>


>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c |   8 +-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 146 ++++++++--------------
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h    |  15 ++-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c |   4 -
>  4 files changed, 68 insertions(+), 105 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index b700c2b91465..8feeaeea1c36 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -1058,12 +1058,12 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
>         struct amdgpu_bo *bo;
>         int ret;
>
> -       spin_lock(&vm->status_lock);
> +       spin_lock(&vm->invalidated_lock);
>         while (!list_empty(&vm->invalidated)) {
>                 bo_va = list_first_entry(&vm->invalidated,
>                                          struct amdgpu_bo_va,
>                                          base.vm_status);
> -               spin_unlock(&vm->status_lock);
> +               spin_unlock(&vm->invalidated_lock);
>
>                 bo = bo_va->base.bo;
>                 ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2);
> @@ -1080,9 +1080,9 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
>                 if (ret)
>                         return ret;
>
> -               spin_lock(&vm->status_lock);
> +               spin_lock(&vm->invalidated_lock);
>         }
> -       spin_unlock(&vm->status_lock);
> +       spin_unlock(&vm->invalidated_lock);
>
>         return 0;
>  }
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> index b4bf1b7c214f..a8a4fffc5764 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -153,12 +153,10 @@ static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
>
>         vm_bo->moved = true;
>         amdgpu_vm_assert_locked(vm);
> -       spin_lock(&vm_bo->vm->status_lock);
>         if (bo->tbo.type == ttm_bo_type_kernel)
>                 list_move(&vm_bo->vm_status, &vm->evicted);
>         else
>                 list_move_tail(&vm_bo->vm_status, &vm->evicted);
> -       spin_unlock(&vm_bo->vm->status_lock);
>  }
>  /**
>   * amdgpu_vm_bo_moved - vm_bo is moved
> @@ -171,9 +169,7 @@ static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
>  static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
>  {
>         amdgpu_vm_assert_locked(vm_bo->vm);
> -       spin_lock(&vm_bo->vm->status_lock);
>         list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
> -       spin_unlock(&vm_bo->vm->status_lock);
>  }
>
>  /**
> @@ -187,9 +183,7 @@ static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
>  static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
>  {
>         amdgpu_vm_assert_locked(vm_bo->vm);
> -       spin_lock(&vm_bo->vm->status_lock);
>         list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
> -       spin_unlock(&vm_bo->vm->status_lock);
>         vm_bo->moved = false;
>  }
>
> @@ -203,9 +197,9 @@ static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
>   */
>  static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
>  {
> -       spin_lock(&vm_bo->vm->status_lock);
> +       spin_lock(&vm_bo->vm->invalidated_lock);
>         list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
> -       spin_unlock(&vm_bo->vm->status_lock);
> +       spin_unlock(&vm_bo->vm->invalidated_lock);
>  }
>
>  /**
> @@ -218,10 +212,9 @@ static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
>   */
>  static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)
>  {
> +       amdgpu_vm_assert_locked(vm_bo->vm);
>         vm_bo->moved = true;
> -       spin_lock(&vm_bo->vm->status_lock);
>         list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user);
> -       spin_unlock(&vm_bo->vm->status_lock);
>  }
>
>  /**
> @@ -235,13 +228,10 @@ static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)
>  static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
>  {
>         amdgpu_vm_assert_locked(vm_bo->vm);
> -       if (vm_bo->bo->parent) {
> -               spin_lock(&vm_bo->vm->status_lock);
> +       if (vm_bo->bo->parent)
>                 list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
> -               spin_unlock(&vm_bo->vm->status_lock);
> -       } else {
> +       else
>                 amdgpu_vm_bo_idle(vm_bo);
> -       }
>  }
>
>  /**
> @@ -255,9 +245,7 @@ static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
>  static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
>  {
>         amdgpu_vm_assert_locked(vm_bo->vm);
> -       spin_lock(&vm_bo->vm->status_lock);
>         list_move(&vm_bo->vm_status, &vm_bo->vm->done);
> -       spin_unlock(&vm_bo->vm->status_lock);
>  }
>
>  /**
> @@ -271,13 +259,13 @@ static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
>  {
>         struct amdgpu_vm_bo_base *vm_bo, *tmp;
>
> -       amdgpu_vm_assert_locked(vm);
> -
> -       spin_lock(&vm->status_lock);
> +       spin_lock(&vm->invalidated_lock);
>         list_splice_init(&vm->done, &vm->invalidated);
>         list_for_each_entry(vm_bo, &vm->invalidated, vm_status)
>                 vm_bo->moved = true;
> +       spin_unlock(&vm->invalidated_lock);
>
> +       amdgpu_vm_assert_locked(vm);
>         list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) {
>                 struct amdgpu_bo *bo = vm_bo->bo;
>
> @@ -287,14 +275,13 @@ static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
>                 else if (bo->parent)
>                         list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
>         }
> -       spin_unlock(&vm->status_lock);
>  }
>
>  /**
>   * amdgpu_vm_update_shared - helper to update shared memory stat
>   * @base: base structure for tracking BO usage in a VM
>   *
> - * Takes the vm status_lock and updates the shared memory stat. If the basic
> + * Takes the vm stats_lock and updates the shared memory stat. If the basic
>   * stat changed (e.g. buffer was moved) amdgpu_vm_update_stats need to be called
>   * as well.
>   */
> @@ -307,7 +294,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base)
>         bool shared;
>
>         dma_resv_assert_held(bo->tbo.base.resv);
> -       spin_lock(&vm->status_lock);
> +       spin_lock(&vm->stats_lock);
>         shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
>         if (base->shared != shared) {
>                 base->shared = shared;
> @@ -319,7 +306,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base)
>                         vm->stats[bo_memtype].drm.private += size;
>                 }
>         }
> -       spin_unlock(&vm->status_lock);
> +       spin_unlock(&vm->stats_lock);
>  }
>
>  /**
> @@ -344,11 +331,11 @@ void amdgpu_vm_bo_update_shared(struct amdgpu_bo *bo)
>   *        be bo->tbo.resource
>   * @sign: if we should add (+1) or subtract (-1) from the stat
>   *
> - * Caller need to have the vm status_lock held. Useful for when multiple update
> + * Caller need to have the vm stats_lock held. Useful for when multiple update
>   * need to happen at the same time.
>   */
>  static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
> -                           struct ttm_resource *res, int sign)
> +                                         struct ttm_resource *res, int sign)
>  {
>         struct amdgpu_vm *vm = base->vm;
>         struct amdgpu_bo *bo = base->bo;
> @@ -372,7 +359,8 @@ static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
>                  */
>                 if (bo->flags & AMDGPU_GEM_CREATE_DISCARDABLE)
>                         vm->stats[res_memtype].drm.purgeable += size;
> -               if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(res_memtype)))
> +               if (!(bo->preferred_domains &
> +                     amdgpu_mem_type_to_domain(res_memtype)))
>                         vm->stats[bo_memtype].evicted += size;
>         }
>  }
> @@ -391,9 +379,9 @@ void amdgpu_vm_update_stats(struct amdgpu_vm_bo_base *base,
>  {
>         struct amdgpu_vm *vm = base->vm;
>
> -       spin_lock(&vm->status_lock);
> +       spin_lock(&vm->stats_lock);
>         amdgpu_vm_update_stats_locked(base, res, sign);
> -       spin_unlock(&vm->status_lock);
> +       spin_unlock(&vm->stats_lock);
>  }
>
>  /**
> @@ -419,10 +407,10 @@ void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
>         base->next = bo->vm_bo;
>         bo->vm_bo = base;
>
> -       spin_lock(&vm->status_lock);
> +       spin_lock(&vm->stats_lock);
>         base->shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
>         amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1);
> -       spin_unlock(&vm->status_lock);
> +       spin_unlock(&vm->stats_lock);
>
>         if (!amdgpu_vm_is_bo_always_valid(vm, bo))
>                 return;
> @@ -481,25 +469,25 @@ int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
>         int ret;
>
>         /* We can only trust prev->next while holding the lock */
> -       spin_lock(&vm->status_lock);
> +       spin_lock(&vm->invalidated_lock);
>         while (!list_is_head(prev->next, &vm->done)) {
>                 bo_va = list_entry(prev->next, typeof(*bo_va), base.vm_status);
>
>                 bo = bo_va->base.bo;
>                 if (bo) {
>                         amdgpu_bo_ref(bo);
> -                       spin_unlock(&vm->status_lock);
> +                       spin_unlock(&vm->invalidated_lock);
>
>                         ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 1);
>                         amdgpu_bo_unref(&bo);
>                         if (unlikely(ret))
>                                 return ret;
>
> -                       spin_lock(&vm->status_lock);
> +                       spin_lock(&vm->invalidated_lock);
>                 }
>                 prev = prev->next;
>         }
> -       spin_unlock(&vm->status_lock);
> +       spin_unlock(&vm->invalidated_lock);
>
>         return 0;
>  }
> @@ -595,7 +583,7 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>                        void *param)
>  {
>         uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm);
> -       struct amdgpu_vm_bo_base *bo_base;
> +       struct amdgpu_vm_bo_base *bo_base, *tmp;
>         struct amdgpu_bo *bo;
>         int r;
>
> @@ -608,13 +596,7 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>                         return r;
>         }
>
> -       spin_lock(&vm->status_lock);
> -       while (!list_empty(&vm->evicted)) {
> -               bo_base = list_first_entry(&vm->evicted,
> -                                          struct amdgpu_vm_bo_base,
> -                                          vm_status);
> -               spin_unlock(&vm->status_lock);
> -
> +       list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {
>                 bo = bo_base->bo;
>
>                 r = validate(param, bo);
> @@ -627,26 +609,21 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>                         vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
>                         amdgpu_vm_bo_relocated(bo_base);
>                 }
> -               spin_lock(&vm->status_lock);
>         }
> -       while (ticket && !list_empty(&vm->evicted_user)) {
> -               bo_base = list_first_entry(&vm->evicted_user,
> -                                          struct amdgpu_vm_bo_base,
> -                                          vm_status);
> -               spin_unlock(&vm->status_lock);
>
> -               bo = bo_base->bo;
> -               dma_resv_assert_held(bo->tbo.base.resv);
> +       if (ticket) {
> +               list_for_each_entry_safe(bo_base, tmp, &vm->evicted_user,
> +                                        vm_status) {
> +                       bo = bo_base->bo;
> +                       dma_resv_assert_held(bo->tbo.base.resv);
>
> -               r = validate(param, bo);
> -               if (r)
> -                       return r;
> -
> -               amdgpu_vm_bo_invalidated(bo_base);
> +                       r = validate(param, bo);
> +                       if (r)
> +                               return r;
>
> -               spin_lock(&vm->status_lock);
> +                       amdgpu_vm_bo_invalidated(bo_base);
> +               }
>         }
> -       spin_unlock(&vm->status_lock);
>
>         amdgpu_vm_eviction_lock(vm);
>         vm->evicting = false;
> @@ -675,9 +652,7 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
>         ret = !vm->evicting;
>         amdgpu_vm_eviction_unlock(vm);
>
> -       spin_lock(&vm->status_lock);
>         ret &= list_empty(&vm->evicted);
> -       spin_unlock(&vm->status_lock);
>
>         spin_lock(&vm->immediate.lock);
>         ret &= !vm->immediate.stopped;
> @@ -967,18 +942,13 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
>                           struct amdgpu_vm *vm, bool immediate)
>  {
>         struct amdgpu_vm_update_params params;
> -       struct amdgpu_vm_bo_base *entry;
> +       struct amdgpu_vm_bo_base *entry, *tmp;
>         bool flush_tlb_needed = false;
> -       LIST_HEAD(relocated);
>         int r, idx;
>
>         amdgpu_vm_assert_locked(vm);
>
> -       spin_lock(&vm->status_lock);
> -       list_splice_init(&vm->relocated, &relocated);
> -       spin_unlock(&vm->status_lock);
> -
> -       if (list_empty(&relocated))
> +       if (list_empty(&vm->relocated))
>                 return 0;
>
>         if (!drm_dev_enter(adev_to_drm(adev), &idx))
> @@ -994,7 +964,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
>         if (r)
>                 goto error;
>
> -       list_for_each_entry(entry, &relocated, vm_status) {
> +       list_for_each_entry(entry, &vm->relocated, vm_status) {
>                 /* vm_flush_needed after updating moved PDEs */
>                 flush_tlb_needed |= entry->moved;
>
> @@ -1010,9 +980,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
>         if (flush_tlb_needed)
>                 atomic64_inc(&vm->tlb_seq);
>
> -       while (!list_empty(&relocated)) {
> -               entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base,
> -                                        vm_status);
> +       list_for_each_entry_safe(entry, tmp, &vm->relocated, vm_status) {
>                 amdgpu_vm_bo_idle(entry);
>         }
>
> @@ -1239,9 +1207,9 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>  void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
>                           struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
>  {
> -       spin_lock(&vm->status_lock);
> +       spin_lock(&vm->stats_lock);
>         memcpy(stats, vm->stats, sizeof(*stats) * __AMDGPU_PL_NUM);
> -       spin_unlock(&vm->status_lock);
> +       spin_unlock(&vm->stats_lock);
>  }
>
>  /**
> @@ -1608,29 +1576,24 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
>                            struct amdgpu_vm *vm,
>                            struct ww_acquire_ctx *ticket)
>  {
> -       struct amdgpu_bo_va *bo_va;
> +       struct amdgpu_bo_va *bo_va, *tmp;
>         struct dma_resv *resv;
>         bool clear, unlock;
>         int r;
>
> -       spin_lock(&vm->status_lock);
> -       while (!list_empty(&vm->moved)) {
> -               bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va,
> -                                        base.vm_status);
> -               spin_unlock(&vm->status_lock);
> -
> +       list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
>                 /* Per VM BOs never need to bo cleared in the page tables */
>                 r = amdgpu_vm_bo_update(adev, bo_va, false);
>                 if (r)
>                         return r;
> -               spin_lock(&vm->status_lock);
>         }
>
> +       spin_lock(&vm->invalidated_lock);
>         while (!list_empty(&vm->invalidated)) {
>                 bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
>                                          base.vm_status);
>                 resv = bo_va->base.bo->tbo.base.resv;
> -               spin_unlock(&vm->status_lock);
> +               spin_unlock(&vm->invalidated_lock);
>
>                 /* Try to reserve the BO to avoid clearing its ptes */
>                 if (!adev->debug_vm && dma_resv_trylock(resv)) {
> @@ -1662,9 +1625,9 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
>                      bo_va->base.bo->tbo.resource->mem_type == TTM_PL_SYSTEM))
>                         amdgpu_vm_bo_evicted_user(&bo_va->base);
>
> -               spin_lock(&vm->status_lock);
> +               spin_lock(&vm->invalidated_lock);
>         }
> -       spin_unlock(&vm->status_lock);
> +       spin_unlock(&vm->invalidated_lock);
>
>         return 0;
>  }
> @@ -2207,9 +2170,9 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev,
>                 }
>         }
>
> -       spin_lock(&vm->status_lock);
> +       spin_lock(&vm->invalidated_lock);
>         list_del(&bo_va->base.vm_status);
> -       spin_unlock(&vm->status_lock);
> +       spin_unlock(&vm->invalidated_lock);
>
>         list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
>                 list_del(&mapping->list);
> @@ -2317,10 +2280,10 @@ void amdgpu_vm_bo_move(struct amdgpu_bo *bo, struct ttm_resource *new_mem,
>         for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
>                 struct amdgpu_vm *vm = bo_base->vm;
>
> -               spin_lock(&vm->status_lock);
> +               spin_lock(&vm->stats_lock);
>                 amdgpu_vm_update_stats_locked(bo_base, bo->tbo.resource, -1);
>                 amdgpu_vm_update_stats_locked(bo_base, new_mem, +1);
> -               spin_unlock(&vm->status_lock);
> +               spin_unlock(&vm->stats_lock);
>         }
>
>         amdgpu_vm_bo_invalidate(bo, evicted);
> @@ -2608,11 +2571,12 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>         INIT_LIST_HEAD(&vm->relocated);
>         INIT_LIST_HEAD(&vm->moved);
>         INIT_LIST_HEAD(&vm->idle);
> +       spin_lock_init(&vm->invalidated_lock);
>         INIT_LIST_HEAD(&vm->invalidated);
> -       spin_lock_init(&vm->status_lock);
>         INIT_LIST_HEAD(&vm->freed);
>         INIT_LIST_HEAD(&vm->done);
>         INIT_KFIFO(vm->faults);
> +       spin_lock_init(&vm->stats_lock);
>
>         r = amdgpu_vm_init_entities(adev, vm);
>         if (r)
> @@ -3080,7 +3044,6 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
>
>         amdgpu_vm_assert_locked(vm);
>
> -       spin_lock(&vm->status_lock);
>         seq_puts(m, "\tIdle BOs:\n");
>         list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
>                 if (!bo_va->base.bo)
> @@ -3118,11 +3081,13 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
>         id = 0;
>
>         seq_puts(m, "\tInvalidated BOs:\n");
> +       spin_lock(&vm->invalidated_lock);
>         list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
>                 if (!bo_va->base.bo)
>                         continue;
>                 total_invalidated += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
>         }
> +       spin_unlock(&vm->invalidated_lock);
>         total_invalidated_objs = id;
>         id = 0;
>
> @@ -3132,7 +3097,6 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
>                         continue;
>                 total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
>         }
> -       spin_unlock(&vm->status_lock);
>         total_done_objs = id;
>
>         seq_printf(m, "\tTotal idle size:        %12lld\tobjs:\t%d\n", total_idle,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> index 139642eacdd0..db9664ec55a9 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> @@ -205,11 +205,11 @@ struct amdgpu_vm_bo_base {
>         /* protected by bo being reserved */
>         struct amdgpu_vm_bo_base        *next;
>
> -       /* protected by vm status_lock */
> +       /* protected by vm reservation and invalidated_lock */
>         struct list_head                vm_status;
>
>         /* if the bo is counted as shared in mem stats
> -        * protected by vm status_lock */
> +        * protected by vm BO being reserved */
>         bool                            shared;
>
>         /* protected by the BO being reserved */
> @@ -345,10 +345,8 @@ struct amdgpu_vm {
>         bool                    evicting;
>         unsigned int            saved_flags;
>
> -       /* Lock to protect vm_bo add/del/move on all lists of vm */
> -       spinlock_t              status_lock;
> -
> -       /* Memory statistics for this vm, protected by status_lock */
> +       /* Memory statistics for this vm, protected by stats_lock */
> +       spinlock_t              stats_lock;
>         struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM];
>
>         /*
> @@ -356,6 +354,8 @@ struct amdgpu_vm {
>          * PDs, PTs or per VM BOs. The state transits are:
>          *
>          * evicted -> relocated (PDs, PTs) or moved (per VM BOs) -> idle
> +        *
> +        * Lists are protected by the root PD dma_resv lock.
>          */
>
>         /* Per-VM and PT BOs who needs a validation */
> @@ -376,7 +376,10 @@ struct amdgpu_vm {
>          * state transits are:
>          *
>          * evicted_user or invalidated -> done
> +        *
> +        * Lists are protected by the invalidated_lock.
>          */
> +       spinlock_t              invalidated_lock;
>
>         /* BOs for user mode queues that need a validation */
>         struct list_head        evicted_user;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> index 31a437ce9570..7bdd664f0770 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> @@ -544,9 +544,7 @@ static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry)
>         entry->bo->vm_bo = NULL;
>         ttm_bo_set_bulk_move(&entry->bo->tbo, NULL);
>
> -       spin_lock(&entry->vm->status_lock);
>         list_del(&entry->vm_status);
> -       spin_unlock(&entry->vm->status_lock);
>         amdgpu_bo_unref(&entry->bo);
>  }
>
> @@ -590,7 +588,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params,
>         struct amdgpu_vm_pt_cursor seek;
>         struct amdgpu_vm_bo_base *entry;
>
> -       spin_lock(&params->vm->status_lock);
>         for_each_amdgpu_vm_pt_dfs_safe(params->adev, params->vm, cursor, seek, entry) {
>                 if (entry && entry->bo)
>                         list_move(&entry->vm_status, &params->tlb_flush_waitlist);
> @@ -598,7 +595,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params,
>
>         /* enter start node now */
>         list_move(&cursor->entry->vm_status, &params->tlb_flush_waitlist);
> -       spin_unlock(&params->vm->status_lock);
>  }
>
>  /**
> --
> 2.43.0
>

Re: [PATCH 3/9] drm/amdgpu: fix amdgpu_userq_evict

[Alex Deucher]

On Mon, Feb 2, 2026 at 8:02 AM Christian König
<ckoenig.leichtzumerken@gmail.com> wrote:
>
> Canceling the delayed synchonized can deadlock!

Please clarify why it can deadlock.  E.g.,
Canceling the delayed work synchonized can deadlock because we already
hold the userq_mutex from the caller.

Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 9 ++-------
>  1 file changed, 2 insertions(+), 7 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index 8feeaeea1c36..af3922e9caea 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -1337,13 +1337,8 @@ amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr,
>         /* Signal current eviction fence */
>         amdgpu_eviction_fence_signal(evf_mgr, ev_fence);
>
> -       if (evf_mgr->fd_closing) {
> -               cancel_delayed_work_sync(&uq_mgr->resume_work);
> -               return;
> -       }
> -
> -       /* Schedule a resume work */
> -       schedule_delayed_work(&uq_mgr->resume_work, 0);
> +       if (!evf_mgr->fd_closing)
> +               schedule_delayed_work(&uq_mgr->resume_work, 0);
>  }
>
>  int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,
> --
> 2.43.0
>

Re: [PATCH 4/9] drm/amdgpu: completely rework eviction fence handling

[Alex Deucher]

On Mon, Feb 2, 2026 at 8:02 AM Christian König
<ckoenig.leichtzumerken@gmail.com> wrote:
>
> Well that was broken on multiple levels.
>
> First of all a lot of checks where placed at incorrect locations, especially if
> the resume worker should run or not.
>
> Then a bunch of code was just mid-layering because of incorrect assignment who
> should do what.
>
> And finally comments explaining what happens instead of why.
>
> Just re-write it from scratch, that should at least fix some of the hangs we
> are seeing.
>
> Use RCU for the eviction fence pointer in the manager, the spinlock usage was
> mostly incorrect as well. Then finally remove all the nonsense checks and
> actually add them in the correct locations.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>

The diff was a bit hard to parse, but it looks to be correct.
Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c       |   4 +-
>  .../drm/amd/amdgpu/amdgpu_eviction_fence.c    | 213 ++++++------------
>  .../drm/amd/amdgpu/amdgpu_eviction_fence.h    |  54 ++---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c       |  10 +-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c       |   5 +-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c     |  29 ++-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h     |   2 +-
>  7 files changed, 112 insertions(+), 205 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index d6d0a6e34c6b..02abe85624a6 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -2975,9 +2975,9 @@ static int amdgpu_drm_release(struct inode *inode, struct file *filp)
>         int idx;
>
>         if (fpriv && drm_dev_enter(dev, &idx)) {
> -               fpriv->evf_mgr.fd_closing = true;
> -               amdgpu_eviction_fence_destroy(&fpriv->evf_mgr);
> +               amdgpu_evf_mgr_shutdown(&fpriv->evf_mgr);
>                 amdgpu_userq_mgr_fini(&fpriv->userq_mgr);
> +               amdgpu_evf_mgr_fini(&fpriv->evf_mgr);
>                 drm_dev_exit(idx);
>         }
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
> index 23d7d0b0d625..8fe9f91f9551 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
> @@ -25,9 +25,6 @@
>  #include <drm/drm_exec.h>
>  #include "amdgpu.h"
>
> -#define work_to_evf_mgr(w, name) container_of(w, struct amdgpu_eviction_fence_mgr, name)
> -#define evf_mgr_to_fpriv(e) container_of(e, struct amdgpu_fpriv, evf_mgr)
> -
>  static const char *
>  amdgpu_eviction_fence_get_driver_name(struct dma_fence *fence)
>  {
> @@ -43,102 +40,14 @@ amdgpu_eviction_fence_get_timeline_name(struct dma_fence *f)
>         return ef->timeline_name;
>  }
>
> -int
> -amdgpu_eviction_fence_replace_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
> -                                   struct drm_exec *exec)
> -{
> -       struct amdgpu_eviction_fence *old_ef, *new_ef;
> -       struct drm_gem_object *obj;
> -       unsigned long index;
> -       int ret;
> -
> -       if (evf_mgr->ev_fence &&
> -           !dma_fence_is_signaled(&evf_mgr->ev_fence->base))
> -               return 0;
> -       /*
> -        * Steps to replace eviction fence:
> -        * * lock all objects in exec (caller)
> -        * * create a new eviction fence
> -        * * update new eviction fence in evf_mgr
> -        * * attach the new eviction fence to BOs
> -        * * release the old fence
> -        * * unlock the objects (caller)
> -        */
> -       new_ef = amdgpu_eviction_fence_create(evf_mgr);
> -       if (!new_ef) {
> -               DRM_ERROR("Failed to create new eviction fence\n");
> -               return -ENOMEM;
> -       }
> -
> -       /* Update the eviction fence now */
> -       spin_lock(&evf_mgr->ev_fence_lock);
> -       old_ef = evf_mgr->ev_fence;
> -       evf_mgr->ev_fence = new_ef;
> -       spin_unlock(&evf_mgr->ev_fence_lock);
> -
> -       /* Attach the new fence */
> -       drm_exec_for_each_locked_object(exec, index, obj) {
> -               struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
> -
> -               if (!bo)
> -                       continue;
> -               ret = amdgpu_eviction_fence_attach(evf_mgr, bo);
> -               if (ret) {
> -                       DRM_ERROR("Failed to attch new eviction fence\n");
> -                       goto free_err;
> -               }
> -       }
> -
> -       /* Free old fence */
> -       if (old_ef)
> -               dma_fence_put(&old_ef->base);
> -       return 0;
> -
> -free_err:
> -       kfree(new_ef);
> -       return ret;
> -}
> -
> -static void
> -amdgpu_eviction_fence_suspend_worker(struct work_struct *work)
> -{
> -       struct amdgpu_eviction_fence_mgr *evf_mgr = work_to_evf_mgr(work, suspend_work.work);
> -       struct amdgpu_fpriv *fpriv = evf_mgr_to_fpriv(evf_mgr);
> -       struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr;
> -       struct amdgpu_eviction_fence *ev_fence;
> -
> -       mutex_lock(&uq_mgr->userq_mutex);
> -       spin_lock(&evf_mgr->ev_fence_lock);
> -       ev_fence = evf_mgr->ev_fence;
> -       if (ev_fence)
> -               dma_fence_get(&ev_fence->base);
> -       else
> -               goto unlock;
> -       spin_unlock(&evf_mgr->ev_fence_lock);
> -
> -       amdgpu_userq_evict(uq_mgr, ev_fence);
> -
> -       mutex_unlock(&uq_mgr->userq_mutex);
> -       dma_fence_put(&ev_fence->base);
> -       return;
> -
> -unlock:
> -       spin_unlock(&evf_mgr->ev_fence_lock);
> -       mutex_unlock(&uq_mgr->userq_mutex);
> -}
> -
>  static bool amdgpu_eviction_fence_enable_signaling(struct dma_fence *f)
>  {
>         struct amdgpu_eviction_fence_mgr *evf_mgr;
>         struct amdgpu_eviction_fence *ev_fence;
>
> -       if (!f)
> -               return true;
> -
>         ev_fence = to_ev_fence(f);
>         evf_mgr = ev_fence->evf_mgr;
> -
> -       schedule_delayed_work(&evf_mgr->suspend_work, 0);
> +       schedule_work(&evf_mgr->suspend_work);
>         return true;
>  }
>
> @@ -148,22 +57,52 @@ static const struct dma_fence_ops amdgpu_eviction_fence_ops = {
>         .enable_signaling = amdgpu_eviction_fence_enable_signaling,
>  };
>
> -void amdgpu_eviction_fence_signal(struct amdgpu_eviction_fence_mgr *evf_mgr,
> -                                 struct amdgpu_eviction_fence *ev_fence)
> +static void
> +amdgpu_eviction_fence_suspend_worker(struct work_struct *work)
>  {
> -       spin_lock(&evf_mgr->ev_fence_lock);
> -       dma_fence_signal(&ev_fence->base);
> -       spin_unlock(&evf_mgr->ev_fence_lock);
> +       struct amdgpu_eviction_fence_mgr *evf_mgr =
> +               container_of(work, struct amdgpu_eviction_fence_mgr,
> +                            suspend_work);
> +       struct amdgpu_fpriv *fpriv =
> +               container_of(evf_mgr, struct amdgpu_fpriv, evf_mgr);
> +       struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr;
> +       struct dma_fence *ev_fence;
> +
> +       mutex_lock(&uq_mgr->userq_mutex);
> +       ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);
> +       amdgpu_userq_evict(uq_mgr, !evf_mgr->shutdown);
> +
> +       /*
> +        * Signaling the eviction fence must be done while holding the
> +        * userq_mutex. Otherwise we won't resume the queues before issuing the
> +        * next fence.
> +        */
> +       dma_fence_signal(ev_fence);
> +       dma_fence_put(ev_fence);
> +       mutex_unlock(&uq_mgr->userq_mutex);
> +}
> +
> +void amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
> +                                struct amdgpu_bo *bo)
> +{
> +       struct dma_fence *ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);
> +       struct dma_resv *resv = bo->tbo.base.resv;
> +
> +       dma_resv_add_fence(resv, ev_fence, DMA_RESV_USAGE_BOOKKEEP);
> +       dma_fence_put(ev_fence);
>  }
>
> -struct amdgpu_eviction_fence *
> -amdgpu_eviction_fence_create(struct amdgpu_eviction_fence_mgr *evf_mgr)
> +int amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr *evf_mgr,
> +                        struct drm_exec *exec)
>  {
>         struct amdgpu_eviction_fence *ev_fence;
> +       struct drm_gem_object *obj;
> +       unsigned long index;
>
> +       /* Create and initialize a new eviction fence */
>         ev_fence = kzalloc(sizeof(*ev_fence), GFP_KERNEL);
>         if (!ev_fence)
> -               return NULL;
> +               return -ENOMEM;
>
>         ev_fence->evf_mgr = evf_mgr;
>         get_task_comm(ev_fence->timeline_name, current);
> @@ -171,56 +110,22 @@ amdgpu_eviction_fence_create(struct amdgpu_eviction_fence_mgr *evf_mgr)
>         dma_fence_init64(&ev_fence->base, &amdgpu_eviction_fence_ops,
>                          &ev_fence->lock, evf_mgr->ev_fence_ctx,
>                          atomic_inc_return(&evf_mgr->ev_fence_seq));
> -       return ev_fence;
> -}
> -
> -void amdgpu_eviction_fence_destroy(struct amdgpu_eviction_fence_mgr *evf_mgr)
> -{
> -       struct amdgpu_eviction_fence *ev_fence;
> -
> -       /* Wait for any pending work to execute */
> -       flush_delayed_work(&evf_mgr->suspend_work);
> -
> -       spin_lock(&evf_mgr->ev_fence_lock);
> -       ev_fence = evf_mgr->ev_fence;
> -       spin_unlock(&evf_mgr->ev_fence_lock);
> -
> -       if (!ev_fence)
> -               return;
> -
> -       dma_fence_wait(&ev_fence->base, false);
>
> -       /* Last unref of ev_fence */
> -       dma_fence_put(&ev_fence->base);
> -}
> -
> -int amdgpu_eviction_fence_attach(struct amdgpu_eviction_fence_mgr *evf_mgr,
> -                                struct amdgpu_bo *bo)
> -{
> -       struct amdgpu_eviction_fence *ev_fence;
> -       struct dma_resv *resv = bo->tbo.base.resv;
> -       int ret;
> +       /* Remember it for newly added BOs */
> +       dma_fence_put(evf_mgr->ev_fence);
> +       evf_mgr->ev_fence = &ev_fence->base;
>
> -       if (!resv)
> -               return 0;
> +       /* And add it to all existing BOs */
> +       drm_exec_for_each_locked_object(exec, index, obj) {
> +               struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
>
> -       ret = dma_resv_reserve_fences(resv, 1);
> -       if (ret) {
> -               DRM_DEBUG_DRIVER("Failed to resv fence space\n");
> -               return ret;
> +               amdgpu_evf_mgr_attach_fence(evf_mgr, bo);
>         }
> -
> -       spin_lock(&evf_mgr->ev_fence_lock);
> -       ev_fence = evf_mgr->ev_fence;
> -       if (ev_fence)
> -               dma_resv_add_fence(resv, &ev_fence->base, DMA_RESV_USAGE_BOOKKEEP);
> -       spin_unlock(&evf_mgr->ev_fence_lock);
> -
>         return 0;
>  }
>
> -void amdgpu_eviction_fence_detach(struct amdgpu_eviction_fence_mgr *evf_mgr,
> -                                 struct amdgpu_bo *bo)
> +void amdgpu_evf_mgr_detach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
> +                                struct amdgpu_bo *bo)
>  {
>         struct dma_fence *stub = dma_fence_get_stub();
>
> @@ -229,13 +134,25 @@ void amdgpu_eviction_fence_detach(struct amdgpu_eviction_fence_mgr *evf_mgr,
>         dma_fence_put(stub);
>  }
>
> -int amdgpu_eviction_fence_init(struct amdgpu_eviction_fence_mgr *evf_mgr)
> +void amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr *evf_mgr)
>  {
> -       /* This needs to be done one time per open */
>         atomic_set(&evf_mgr->ev_fence_seq, 0);
>         evf_mgr->ev_fence_ctx = dma_fence_context_alloc(1);
> -       spin_lock_init(&evf_mgr->ev_fence_lock);
> +       evf_mgr->ev_fence = dma_fence_get_stub();
>
> -       INIT_DELAYED_WORK(&evf_mgr->suspend_work, amdgpu_eviction_fence_suspend_worker);
> -       return 0;
> +       INIT_WORK(&evf_mgr->suspend_work, amdgpu_eviction_fence_suspend_worker);
> +}
> +
> +void amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr *evf_mgr)
> +{
> +       evf_mgr->shutdown = true;
> +       flush_work(&evf_mgr->suspend_work);
> +}
> +
> +void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr)
> +{
> +       dma_fence_wait(rcu_dereference_protected(evf_mgr->ev_fence, true),
> +                      false);
> +       flush_work(&evf_mgr->suspend_work);
> +       dma_fence_put(evf_mgr->ev_fence);
>  }
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
> index fcd867b7147d..527de3a23583 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
> @@ -25,6 +25,8 @@
>  #ifndef AMDGPU_EV_FENCE_H_
>  #define AMDGPU_EV_FENCE_H_
>
> +#include <linux/dma-fence.h>
> +
>  struct amdgpu_eviction_fence {
>         struct dma_fence base;
>         spinlock_t       lock;
> @@ -35,35 +37,35 @@ struct amdgpu_eviction_fence {
>  struct amdgpu_eviction_fence_mgr {
>         u64                     ev_fence_ctx;
>         atomic_t                ev_fence_seq;
> -       spinlock_t              ev_fence_lock;
> -       struct amdgpu_eviction_fence *ev_fence;
> -       struct delayed_work     suspend_work;
> -       uint8_t fd_closing;
> -};
> -
> -/* Eviction fence helper functions */
> -struct amdgpu_eviction_fence *
> -amdgpu_eviction_fence_create(struct amdgpu_eviction_fence_mgr *evf_mgr);
>
> -void
> -amdgpu_eviction_fence_destroy(struct amdgpu_eviction_fence_mgr *evf_mgr);
> -
> -int
> -amdgpu_eviction_fence_attach(struct amdgpu_eviction_fence_mgr *evf_mgr,
> -                            struct amdgpu_bo *bo);
> +       /*
> +        * Only updated while holding the VM resv lock.
> +        * Only signaled while holding the userq mutex.
> +        */
> +       struct dma_fence __rcu  *ev_fence;
> +       struct work_struct      suspend_work;
> +       bool                    shutdown;
> +};
>
> -void
> -amdgpu_eviction_fence_detach(struct amdgpu_eviction_fence_mgr *evf_mgr,
> -                            struct amdgpu_bo *bo);
> +static inline struct dma_fence *
> +amdgpu_evf_mgr_get_fence(struct amdgpu_eviction_fence_mgr *evf_mgr)
> +{
> +       struct dma_fence *ev_fence;
>
> -int
> -amdgpu_eviction_fence_init(struct amdgpu_eviction_fence_mgr *evf_mgr);
> +       rcu_read_lock();
> +       ev_fence = dma_fence_get_rcu_safe(&evf_mgr->ev_fence);
> +       rcu_read_unlock();
> +       return ev_fence;
> +}
>
> -void
> -amdgpu_eviction_fence_signal(struct amdgpu_eviction_fence_mgr *evf_mgr,
> -                            struct amdgpu_eviction_fence *ev_fence);
> +void amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
> +                                struct amdgpu_bo *bo);
> +int amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr *evf_mgr,
> +                        struct drm_exec *exec);
> +void amdgpu_evf_mgr_detach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
> +                                struct amdgpu_bo *bo);
> +void amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr *evf_mgr);
> +void amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr *evf_mgr);
> +void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr);
>
> -int
> -amdgpu_eviction_fence_replace_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
> -                                   struct drm_exec *exec);
>  #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> index 5c90de58cc28..e28abfd04867 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> @@ -263,13 +263,7 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
>         else
>                 ++bo_va->ref_count;
>
> -       /* attach gfx eviction fence */
> -       r = amdgpu_eviction_fence_attach(&fpriv->evf_mgr, abo);
> -       if (r) {
> -               DRM_DEBUG_DRIVER("Failed to attach eviction fence to BO\n");
> -               amdgpu_bo_unreserve(abo);
> -               return r;
> -       }
> +       amdgpu_evf_mgr_attach_fence(&fpriv->evf_mgr, abo);
>         drm_exec_fini(&exec);
>
>         /* Validate and add eviction fence to DMABuf imports with dynamic
> @@ -337,7 +331,7 @@ static void amdgpu_gem_object_close(struct drm_gem_object *obj,
>         }
>
>         if (!amdgpu_vm_is_bo_always_valid(vm, bo))
> -               amdgpu_eviction_fence_detach(&fpriv->evf_mgr, bo);
> +               amdgpu_evf_mgr_detach_fence(&fpriv->evf_mgr, bo);
>
>         bo_va = amdgpu_vm_bo_find(vm, bo);
>         if (!bo_va || --bo_va->ref_count)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index f69332eed051..f512b6ec6c53 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -1522,10 +1522,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
>                          "Failed to init usermode queue manager (%d), use legacy workload submission only\n",
>                          r);
>
> -       r = amdgpu_eviction_fence_init(&fpriv->evf_mgr);
> -       if (r)
> -               goto error_vm;
> -
> +       amdgpu_evf_mgr_init(&fpriv->evf_mgr);
>         amdgpu_ctx_mgr_init(&fpriv->ctx_mgr, adev);
>
>         file_priv->driver_priv = fpriv;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index af3922e9caea..e007f10a6c1c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -480,17 +480,16 @@ void
>  amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr,
>                              struct amdgpu_eviction_fence_mgr *evf_mgr)
>  {
> -       struct amdgpu_eviction_fence *ev_fence;
> +       struct dma_fence *ev_fence;
>
>  retry:
>         /* Flush any pending resume work to create ev_fence */
>         flush_delayed_work(&uq_mgr->resume_work);
>
>         mutex_lock(&uq_mgr->userq_mutex);
> -       spin_lock(&evf_mgr->ev_fence_lock);
> -       ev_fence = evf_mgr->ev_fence;
> -       spin_unlock(&evf_mgr->ev_fence_lock);
> -       if (!ev_fence || dma_fence_is_signaled(&ev_fence->base)) {
> +       ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);
> +       if (dma_fence_is_signaled(ev_fence)) {
> +               dma_fence_put(ev_fence);
>                 mutex_unlock(&uq_mgr->userq_mutex);
>                 /*
>                  * Looks like there was no pending resume work,
> @@ -499,6 +498,7 @@ amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr,
>                 schedule_delayed_work(&uq_mgr->resume_work, 0);
>                 goto retry;
>         }
> +       dma_fence_put(ev_fence);
>  }
>
>  int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr,
> @@ -1214,7 +1214,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
>                 dma_fence_wait(bo_va->last_pt_update, false);
>         dma_fence_wait(vm->last_update, false);
>
> -       ret = amdgpu_eviction_fence_replace_fence(&fpriv->evf_mgr, &exec);
> +       ret = amdgpu_evf_mgr_rearm(&fpriv->evf_mgr, &exec);
>         if (ret)
>                 drm_file_err(uq_mgr->file, "Failed to replace eviction fence\n");
>
> @@ -1234,11 +1234,13 @@ static void amdgpu_userq_restore_worker(struct work_struct *work)
>  {
>         struct amdgpu_userq_mgr *uq_mgr = work_to_uq_mgr(work, resume_work.work);
>         struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
> +       struct dma_fence *ev_fence;
>         int ret;
>
> -       flush_delayed_work(&fpriv->evf_mgr.suspend_work);
> -
>         mutex_lock(&uq_mgr->userq_mutex);
> +       ev_fence = amdgpu_evf_mgr_get_fence(&fpriv->evf_mgr);
> +       if (!dma_fence_is_signaled(ev_fence))
> +               goto unlock;
>
>         ret = amdgpu_userq_vm_validate(uq_mgr);
>         if (ret) {
> @@ -1254,6 +1256,7 @@ static void amdgpu_userq_restore_worker(struct work_struct *work)
>
>  unlock:
>         mutex_unlock(&uq_mgr->userq_mutex);
> +       dma_fence_put(ev_fence);
>  }
>
>  static int
> @@ -1317,11 +1320,8 @@ amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr)
>  }
>
>  void
> -amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr,
> -                  struct amdgpu_eviction_fence *ev_fence)
> +amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr, bool schedule_resume)
>  {
> -       struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
> -       struct amdgpu_eviction_fence_mgr *evf_mgr = &fpriv->evf_mgr;
>         struct amdgpu_device *adev = uq_mgr->adev;
>         int ret;
>
> @@ -1334,10 +1334,7 @@ amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr,
>         if (ret)
>                 dev_err(adev->dev, "Failed to evict userqueue\n");
>
> -       /* Signal current eviction fence */
> -       amdgpu_eviction_fence_signal(evf_mgr, ev_fence);
> -
> -       if (!evf_mgr->fd_closing)
> +       if (schedule_resume)
>                 schedule_delayed_work(&uq_mgr->resume_work, 0);
>  }
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> index 5845d8959034..095c09f3d96c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> @@ -127,7 +127,7 @@ void amdgpu_userq_destroy_object(struct amdgpu_userq_mgr *uq_mgr,
>                                  struct amdgpu_userq_obj *userq_obj);
>
>  void amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr,
> -                       struct amdgpu_eviction_fence *ev_fence);
> +                       bool schedule_resume);
>
>  void amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *userq_mgr,
>                                   struct amdgpu_eviction_fence_mgr *evf_mgr);
> --
> 2.43.0
>

Re: [PATCH 5/9] drm/amdgpu: fix eviction fence and userq manager shutdown

[Alex Deucher]

On Mon, Feb 2, 2026 at 7:51 AM Christian König
<ckoenig.leichtzumerken@gmail.com> wrote:
>
> That is a really complicated dance and wasn't implemented fully correct.

I'd suggest elaborating on what was wrong with the previous flow.
It's not entirely clear from the patch.

>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c            | 2 ++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 8 +++++++-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h | 1 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c          | 7 +++++--
>  drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h          | 1 +
>  5 files changed, 16 insertions(+), 3 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index 02abe85624a6..9cd44be45861 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -2976,6 +2976,8 @@ static int amdgpu_drm_release(struct inode *inode, struct file *filp)
>
>         if (fpriv && drm_dev_enter(dev, &idx)) {
>                 amdgpu_evf_mgr_shutdown(&fpriv->evf_mgr);
> +               amdgpu_userq_mgr_cancel_resume(&fpriv->userq_mgr);
> +               amdgpu_evf_mgr_flush_suspend(&fpriv->evf_mgr);
>                 amdgpu_userq_mgr_fini(&fpriv->userq_mgr);
>                 amdgpu_evf_mgr_fini(&fpriv->evf_mgr);
>                 drm_dev_exit(idx);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
> index 8fe9f91f9551..ef4da6f2e2a3 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
> @@ -146,13 +146,19 @@ void amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr *evf_mgr)
>  void amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr *evf_mgr)
>  {
>         evf_mgr->shutdown = true;
> +       /* Make sure that the shutdown is visible to the suspend work */
>         flush_work(&evf_mgr->suspend_work);
>  }
>
> -void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr)
> +void amdgpu_evf_mgr_flush_suspend(struct amdgpu_eviction_fence_mgr *evf_mgr)
>  {
>         dma_fence_wait(rcu_dereference_protected(evf_mgr->ev_fence, true),
>                        false);
> +       /* Make sure that we are done with the last suspend work */
>         flush_work(&evf_mgr->suspend_work);

Is it intended to flush the work here and in amdgpu_evf_mgr_shutdown()?

Alex

> +}
> +
> +void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr)
> +{
>         dma_fence_put(evf_mgr->ev_fence);
>  }
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
> index 527de3a23583..132a13a5dc1c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
> @@ -66,6 +66,7 @@ void amdgpu_evf_mgr_detach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
>                                  struct amdgpu_bo *bo);
>  void amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr *evf_mgr);
>  void amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr *evf_mgr);
> +void amdgpu_evf_mgr_flush_suspend(struct amdgpu_eviction_fence_mgr *evf_mgr);
>  void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr);
>
>  #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index e007f10a6c1c..60e8a993557a 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -1350,13 +1350,16 @@ int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *f
>         return 0;
>  }
>
> +void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr)
> +{
> +       cancel_delayed_work_sync(&userq_mgr->resume_work);
> +}
> +
>  void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr)
>  {
>         struct amdgpu_usermode_queue *queue;
>         unsigned long queue_id;
>
> -       cancel_delayed_work_sync(&userq_mgr->resume_work);
> -
>         mutex_lock(&userq_mgr->userq_mutex);
>         amdgpu_userq_detect_and_reset_queues(userq_mgr);
>         xa_for_each(&userq_mgr->userq_xa, queue_id, queue) {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> index 095c09f3d96c..13f4e356728b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> @@ -117,6 +117,7 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp
>  int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,
>                           struct amdgpu_device *adev);
>
> +void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr);
>  void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr);
>
>  int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr,
> --
> 2.43.0
>

Re: [PATCH 6/9] drm/amdgpu: fix adding eviction fence

[Alex Deucher]

On Mon, Feb 2, 2026 at 8:37 AM Christian König
<ckoenig.leichtzumerken@gmail.com> wrote:
>
> We can't add the eviction fence without validating the BO.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>

Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

> ---
>  .../drm/amd/amdgpu/amdgpu_eviction_fence.c    | 19 ++++++++++++++++---
>  .../drm/amd/amdgpu/amdgpu_eviction_fence.h    |  4 ++--
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c       |  9 ++++++---
>  3 files changed, 24 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
> index ef4da6f2e2a3..6598823ec619 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
> @@ -82,14 +82,27 @@ amdgpu_eviction_fence_suspend_worker(struct work_struct *work)
>         mutex_unlock(&uq_mgr->userq_mutex);
>  }
>
> -void amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
> -                                struct amdgpu_bo *bo)
> +int amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
> +                               struct amdgpu_bo *bo)
>  {
>         struct dma_fence *ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);
> +       struct ttm_operation_ctx ctx = { false, false };
>         struct dma_resv *resv = bo->tbo.base.resv;
> +       int ret;
> +
> +       if (!dma_fence_is_signaled(ev_fence)) {
> +
> +               amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
> +               ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> +               if (!ret)
> +                       dma_resv_add_fence(resv, ev_fence,
> +                                          DMA_RESV_USAGE_BOOKKEEP);
> +       } else {
> +               ret = 0;
> +       }
>
> -       dma_resv_add_fence(resv, ev_fence, DMA_RESV_USAGE_BOOKKEEP);
>         dma_fence_put(ev_fence);
> +       return ret;
>  }
>
>  int amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr *evf_mgr,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
> index 132a13a5dc1c..2a750add4e7b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
> @@ -58,8 +58,8 @@ amdgpu_evf_mgr_get_fence(struct amdgpu_eviction_fence_mgr *evf_mgr)
>         return ev_fence;
>  }
>
> -void amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
> -                                struct amdgpu_bo *bo);
> +int amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
> +                               struct amdgpu_bo *bo);
>  int amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr *evf_mgr,
>                          struct drm_exec *exec);
>  void amdgpu_evf_mgr_detach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> index e28abfd04867..88a21400ae09 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> @@ -258,12 +258,15 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
>
>         amdgpu_vm_bo_update_shared(abo);
>         bo_va = amdgpu_vm_bo_find(vm, abo);
> -       if (!bo_va)
> +       if (!bo_va) {
>                 bo_va = amdgpu_vm_bo_add(adev, vm, abo);
> -       else
> +               r = amdgpu_evf_mgr_attach_fence(&fpriv->evf_mgr, abo);
> +               if (r)
> +                       goto out_unlock;
> +       } else {
>                 ++bo_va->ref_count;
> +       }
>
> -       amdgpu_evf_mgr_attach_fence(&fpriv->evf_mgr, abo);
>         drm_exec_fini(&exec);
>
>         /* Validate and add eviction fence to DMABuf imports with dynamic
> --
> 2.43.0
>

RE: [PATCH 7/9] drm/amdgpu: rework amdgpu_userq_wait_ioctl v2

[Liang, Prike]

[-- Attachment #1: Type: text/plain, Size: 32360 bytes --]

[Public]

Regards,
      Prike

> -----Original Message-----
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> Sent: Monday, February 2, 2026 8:52 PM
> To: tursulin@ursulin.net; Deucher, Alexander <Alexander.Deucher@amd.com>;
> Liang, Prike <Prike.Liang@amd.com>; Mohan Marimuthu, Yogesh
> <Yogesh.Mohanmarimuthu@amd.com>; SHANMUGAM, SRINIVASAN
> <SRINIVASAN.SHANMUGAM@amd.com>; Khatri, Sunil <Sunil.Khatri@amd.com>;
> amd-gfx@lists.freedesktop.org
> Subject: [PATCH 7/9] drm/amdgpu: rework amdgpu_userq_wait_ioctl v2
>
> Lockdep was complaining about a number of issues here. Especially lock inversion
> between syncobj, dma_resv and copying things into userspace.
>
> Rework the functionality. Split it up into multiple functions, consistenly use
> memdup_array_user(), fix the lock inversions and a few more bugs in error handling.
>
> v2: drop the dma_fence leak fix, turned out that was actually correct,
>     just not well documented. Apply some more cleanup suggestion from
>     Tvrtko.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>  .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 614 +++++++++---------
>  1 file changed, 319 insertions(+), 295 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> index 212056d4ddf0..da329d00843b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> @@ -645,368 +645,397 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev,
> void *data,
>       return r;
>  }
>
> -int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
> -                         struct drm_file *filp)
> +/* Count the number of expected fences so userspace can alloc a buffer
> +*/ static int amdgpu_userq_wait_count_fences(struct drm_file *filp,
> +                            struct drm_amdgpu_userq_wait *wait_info,
> +                            u32 *syncobj_handles, u32 *timeline_points,
> +                            u32 *timeline_handles,
> +                            struct drm_gem_object **gobj_write,
> +                            struct drm_gem_object **gobj_read)
>  {
> -     u32 *syncobj_handles, *timeline_points, *timeline_handles,
> *bo_handles_read, *bo_handles_write;
> -     u32 num_syncobj, num_read_bo_handles, num_write_bo_handles;
> -     struct drm_amdgpu_userq_fence_info *fence_info = NULL;
> -     struct drm_amdgpu_userq_wait *wait_info = data;
> -     struct amdgpu_fpriv *fpriv = filp->driver_priv;
> -     struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
> -     struct amdgpu_usermode_queue *waitq;
> -     struct drm_gem_object **gobj_write;
> -     struct drm_gem_object **gobj_read;
> -     struct dma_fence **fences = NULL;
> -     u16 num_points, num_fences = 0;
> -     int r, i, rentry, wentry, cnt;
> +     int num_read_bo_handles, num_write_bo_handles;
> +     struct dma_fence_unwrap iter;
> +     struct dma_fence *fence, *f;
> +     unsigned int num_fences = 0;
>       struct drm_exec exec;
> +     int i, r;
> +
> +     /*
> +      * This needs to be outside of the lock provided by drm_exec for
> +      * DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT to work correctly.
> +      */
> +
> +     /* Count timeline fences */
> +     for (i = 0; i < wait_info->num_syncobj_timeline_handles; i++) {
> +             r = drm_syncobj_find_fence(filp, timeline_handles[i],
> +                                        timeline_points[i],
> +
> DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> +                                        &fence);
> +             if (r)
> +                     return r;
> +
> +             dma_fence_unwrap_for_each(f, &iter, fence)
> +                     num_fences++;
>
> -     if (!amdgpu_userq_enabled(dev))
> -             return -ENOTSUPP;
> +             dma_fence_put(fence);
> +     }
>
> -     num_read_bo_handles = wait_info->num_bo_read_handles;
> -     bo_handles_read = memdup_user(u64_to_user_ptr(wait_info-
> >bo_read_handles),
> -                                   size_mul(sizeof(u32), num_read_bo_handles));
> -     if (IS_ERR(bo_handles_read))
> -             return PTR_ERR(bo_handles_read);
> +     /* Count boolean fences */
> +     for (i = 0; i < wait_info->num_syncobj_handles; i++) {
> +             r = drm_syncobj_find_fence(filp, syncobj_handles[i], 0,
> +
> DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> +                                        &fence);
> +             if (r)
> +                     return r;
>
> -     num_write_bo_handles = wait_info->num_bo_write_handles;
> -     bo_handles_write = memdup_user(u64_to_user_ptr(wait_info-
> >bo_write_handles),
> -                                    size_mul(sizeof(u32), num_write_bo_handles));
> -     if (IS_ERR(bo_handles_write)) {
> -             r = PTR_ERR(bo_handles_write);
> -             goto free_bo_handles_read;
> +             num_fences++;
> +             dma_fence_put(fence);
>       }
>
> -     num_syncobj = wait_info->num_syncobj_handles;
> -     syncobj_handles = memdup_user(u64_to_user_ptr(wait_info-
> >syncobj_handles),
> -                                   size_mul(sizeof(u32), num_syncobj));
> -     if (IS_ERR(syncobj_handles)) {
> -             r = PTR_ERR(syncobj_handles);
> -             goto free_bo_handles_write;
> -     }
> +     /* Lock all the GEM objects */
> +     /* TODO: It is actually not necessary to lock them */
> +     num_read_bo_handles = wait_info->num_bo_read_handles;
> +     num_write_bo_handles = wait_info->num_bo_write_handles;
> +     drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
> +                   num_read_bo_handles + num_write_bo_handles);
>
> -     num_points = wait_info->num_syncobj_timeline_handles;
> -     timeline_handles = memdup_user(u64_to_user_ptr(wait_info-
> >syncobj_timeline_handles),
> -                                    sizeof(u32) * num_points);
> -     if (IS_ERR(timeline_handles)) {
> -             r = PTR_ERR(timeline_handles);
> -             goto free_syncobj_handles;
> -     }
> +     drm_exec_until_all_locked(&exec) {
> +             r = drm_exec_prepare_array(&exec, gobj_read,
> +                                        num_read_bo_handles, 1);
> +             drm_exec_retry_on_contention(&exec);
> +             if (r)
> +                     goto error_unlock;
>
> -     timeline_points = memdup_user(u64_to_user_ptr(wait_info-
> >syncobj_timeline_points),
> -                                   sizeof(u32) * num_points);
> -     if (IS_ERR(timeline_points)) {
> -             r = PTR_ERR(timeline_points);
> -             goto free_timeline_handles;
> +             r = drm_exec_prepare_array(&exec, gobj_write,
> +                                        num_write_bo_handles, 1);
> +             drm_exec_retry_on_contention(&exec);
> +             if (r)
> +                     goto error_unlock;
>       }
>
> -     gobj_read = kmalloc_array(num_read_bo_handles, sizeof(*gobj_read),
> GFP_KERNEL);
> -     if (!gobj_read) {
> -             r = -ENOMEM;
> -             goto free_timeline_points;
> -     }
> +     /* Count read fences */
> +     for (i = 0; i < num_read_bo_handles; i++) {
> +             struct dma_resv_iter resv_cursor;
> +             struct dma_fence *fence;
>
> -     for (rentry = 0; rentry < num_read_bo_handles; rentry++) {
> -             gobj_read[rentry] = drm_gem_object_lookup(filp,
> bo_handles_read[rentry]);
> -             if (!gobj_read[rentry]) {
> -                     r = -ENOENT;
> -                     goto put_gobj_read;
> -             }
> +             dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
> +                                     DMA_RESV_USAGE_READ, fence)
> +                     num_fences++;
>       }
>
> -     gobj_write = kmalloc_array(num_write_bo_handles, sizeof(*gobj_write),
> GFP_KERNEL);
> -     if (!gobj_write) {
> -             r = -ENOMEM;
> -             goto put_gobj_read;
> -     }
> +     /* Count write fences */
> +     for (i = 0; i < num_write_bo_handles; i++) {
> +             struct dma_resv_iter resv_cursor;
> +             struct dma_fence *fence;
>
> -     for (wentry = 0; wentry < num_write_bo_handles; wentry++) {
> -             gobj_write[wentry] = drm_gem_object_lookup(filp,
> bo_handles_write[wentry]);
> -             if (!gobj_write[wentry]) {
> -                     r = -ENOENT;
> -                     goto put_gobj_write;
> -             }
> +             dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
> +                                     DMA_RESV_USAGE_WRITE, fence)
> +                     num_fences++;
>       }
>
> -     drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
> -                   (num_read_bo_handles + num_write_bo_handles));
> -
> -     /* Lock all BOs with retry handling */
> -     drm_exec_until_all_locked(&exec) {
> -             r = drm_exec_prepare_array(&exec, gobj_read,
> num_read_bo_handles, 1);
> -             drm_exec_retry_on_contention(&exec);
> -             if (r) {
> -                     drm_exec_fini(&exec);
> -                     goto put_gobj_write;
> -             }
> +     wait_info->num_fences = num_fences;
> +     r = 0;
>
> -             r = drm_exec_prepare_array(&exec, gobj_write,
> num_write_bo_handles, 1);
> -             drm_exec_retry_on_contention(&exec);
> -             if (r) {
> -                     drm_exec_fini(&exec);
> -                     goto put_gobj_write;
> -             }
> -     }
> +error_unlock:
> +     /* Unlock all the GEM objects */
> +     drm_exec_fini(&exec);
> +     return r;
> +}
>
> -     if (!wait_info->num_fences) {
> -             if (num_points) {
> -                     struct dma_fence_unwrap iter;
> -                     struct dma_fence *fence;
> -                     struct dma_fence *f;
> -
> -                     for (i = 0; i < num_points; i++) {
> -                             r = drm_syncobj_find_fence(filp, timeline_handles[i],
> -                                                        timeline_points[i],
> -
> DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> -                                                        &fence);
> -                             if (r)
> -                                     goto exec_fini;
> -
> -                             dma_fence_unwrap_for_each(f, &iter, fence)
> -                                     num_fences++;
> -
> -                             dma_fence_put(fence);
> -                     }
> -             }
> +static int
> +amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
> +                                 struct drm_amdgpu_userq_wait *wait_info,
> +                                 u32 *syncobj_handles, u32 *timeline_points,
> +                                 u32 *timeline_handles,
> +                                 struct drm_gem_object **gobj_write,
> +                                 struct drm_gem_object **gobj_read) {
> +     struct amdgpu_fpriv *fpriv = filp->driver_priv;
> +     struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
> +     struct drm_amdgpu_userq_fence_info *fence_info;
> +     int num_read_bo_handles, num_write_bo_handles;
> +     struct amdgpu_usermode_queue *waitq;
> +     struct dma_fence **fences, *fence, *f;
> +     struct dma_fence_unwrap iter;
> +     int num_points, num_syncobj;
> +     unsigned int num_fences = 0;
> +     struct drm_exec exec;
> +     int i, cnt, r;
>
> -             /* Count syncobj's fence */
> -             for (i = 0; i < num_syncobj; i++) {
> -                     struct dma_fence *fence;
> +     fence_info = kmalloc_array(wait_info->num_fences, sizeof(*fence_info),
> +                                GFP_KERNEL);
> +     if (!fence_info)
> +             return -ENOMEM;
>
> -                     r = drm_syncobj_find_fence(filp, syncobj_handles[i],
> -                                                0,
> -
> DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> -                                                &fence);
> -                     if (r)
> -                             goto exec_fini;
> +     fences = kmalloc_array(wait_info->num_fences, sizeof(*fences),
> +                            GFP_KERNEL);
> +     if (!fences) {
> +             r = -ENOMEM;
> +             goto free_fence_info;
> +     }
>
> -                     num_fences++;
> -                     dma_fence_put(fence);
> -             }
> +     /* Retrieve timeline fences */
> +     num_points = wait_info->num_syncobj_timeline_handles;
> +     for (i = 0; i < num_points; i++) {
> +             r = drm_syncobj_find_fence(filp, timeline_handles[i],
> +                                        timeline_points[i],
> +
> DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> +                                        &fence);
> +             if (r)
> +                     goto free_fences;
>
> -             /* Count GEM objects fence */
> -             for (i = 0; i < num_read_bo_handles; i++) {
> -                     struct dma_resv_iter resv_cursor;
> -                     struct dma_fence *fence;
> +             dma_fence_unwrap_for_each(f, &iter, fence) {
> +                     if (num_fences >= wait_info->num_fences) {
> +                             r = -EINVAL;
> +                             goto free_fences;
> +                     }
>
> -                     dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
> -                                             DMA_RESV_USAGE_READ, fence)
> -                             num_fences++;
> +                     fences[num_fences++] = dma_fence_get(f);
>               }
>
> -             for (i = 0; i < num_write_bo_handles; i++) {
> -                     struct dma_resv_iter resv_cursor;
> -                     struct dma_fence *fence;
> +             dma_fence_put(fence);
> +     }
>
> -                     dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
> -                                             DMA_RESV_USAGE_WRITE, fence)
> -                             num_fences++;
> -             }
> +     /* Retrieve boolean fences */
> +     num_syncobj = wait_info->num_syncobj_handles;
> +     for (i = 0; i < num_syncobj; i++) {
> +             struct dma_fence *fence;
>
> -             /*
> -              * Passing num_fences = 0 means that userspace doesn't want to
> -              * retrieve userq_fence_info. If num_fences = 0 we skip filling
> -              * userq_fence_info and return the actual number of fences on
> -              * args->num_fences.
> -              */
> -             wait_info->num_fences = num_fences;
> -     } else {
> -             /* Array of fence info */
> -             fence_info = kmalloc_array(wait_info->num_fences,
> sizeof(*fence_info), GFP_KERNEL);
> -             if (!fence_info) {
> -                     r = -ENOMEM;
> -                     goto exec_fini;
> -             }
> +             r = drm_syncobj_find_fence(filp, syncobj_handles[i], 0,
> +
> DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> +                                        &fence);
> +             if (r)
> +                     goto free_fences;
>
> -             /* Array of fences */
> -             fences = kmalloc_array(wait_info->num_fences, sizeof(*fences),
> GFP_KERNEL);
> -             if (!fences) {
> -                     r = -ENOMEM;
> -                     goto free_fence_info;
> +             if (num_fences >= wait_info->num_fences) {
> +                     r = -EINVAL;
> +                     goto free_fences;
>               }
>
> -             /* Retrieve GEM read objects fence */
> -             for (i = 0; i < num_read_bo_handles; i++) {
> -                     struct dma_resv_iter resv_cursor;
> -                     struct dma_fence *fence;
> +             /* Give the reference to the fence array */
> +             fences[num_fences++] = fence;
> +     }
>
> -                     dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
> -                                             DMA_RESV_USAGE_READ, fence) {
> -                             if (WARN_ON_ONCE(num_fences >= wait_info-
> >num_fences)) {
> -                                     r = -EINVAL;
> -                                     goto free_fences;
> -                             }
> +     /* Lock all the GEM objects */
> +     num_read_bo_handles = wait_info->num_bo_read_handles;
> +     num_write_bo_handles = wait_info->num_bo_write_handles;
> +     drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
> +                   num_read_bo_handles + num_write_bo_handles);
>
> -                             fences[num_fences++] = fence;
> -                             dma_fence_get(fence);
> -                     }
> -             }
> +     drm_exec_until_all_locked(&exec) {
> +             r = drm_exec_prepare_array(&exec, gobj_read,
> +                                        num_read_bo_handles, 1);
> +             drm_exec_retry_on_contention(&exec);
> +             if (r)
> +                     goto error_unlock;
>
> -             /* Retrieve GEM write objects fence */
> -             for (i = 0; i < num_write_bo_handles; i++) {
> -                     struct dma_resv_iter resv_cursor;
> -                     struct dma_fence *fence;
> +             r = drm_exec_prepare_array(&exec, gobj_write,
> +                                        num_write_bo_handles, 1);
> +             drm_exec_retry_on_contention(&exec);
> +             if (r)
> +                     goto error_unlock;
> +     }
>
> -                     dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
> -                                             DMA_RESV_USAGE_WRITE, fence) {
> -                             if (WARN_ON_ONCE(num_fences >= wait_info-
> >num_fences)) {
> -                                     r = -EINVAL;
> -                                     goto free_fences;
> -                             }
> +     /* Retrieve GEM read objects fence */
> +     for (i = 0; i < num_read_bo_handles; i++) {
> +             struct dma_resv_iter resv_cursor;
> +             struct dma_fence *fence;
>
> -                             fences[num_fences++] = fence;
> -                             dma_fence_get(fence);
> +             dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
> +                                     DMA_RESV_USAGE_READ, fence) {
> +                     if (num_fences >= wait_info->num_fences) {
> +                             r = -EINVAL;
> +                             goto error_unlock;
>                       }
> -             }
>
> -             if (num_points) {
> -                     struct dma_fence_unwrap iter;
> -                     struct dma_fence *fence;
> -                     struct dma_fence *f;
> -
> -                     for (i = 0; i < num_points; i++) {
> -                             r = drm_syncobj_find_fence(filp, timeline_handles[i],
> -                                                        timeline_points[i],
> -
> DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> -                                                        &fence);
> -                             if (r)
> -                                     goto free_fences;
> -
> -                             dma_fence_unwrap_for_each(f, &iter, fence) {
> -                                     if (WARN_ON_ONCE(num_fences >=
> wait_info->num_fences)) {
> -                                             r = -EINVAL;
> -                                             goto free_fences;
> -                                     }
> -
> -                                     dma_fence_get(f);
> -                                     fences[num_fences++] = f;
> -                             }
> -
> -                             dma_fence_put(fence);
> -                     }
> +                     fences[num_fences++] = dma_fence_get(fence);
>               }
> +     }
>
> -             /* Retrieve syncobj's fence */
> -             for (i = 0; i < num_syncobj; i++) {
> -                     struct dma_fence *fence;
> -
> -                     r = drm_syncobj_find_fence(filp, syncobj_handles[i],
> -                                                0,
> -
> DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> -                                                &fence);
> -                     if (r)
> -                             goto free_fences;
> +     /* Retrieve GEM write objects fence */
> +     for (i = 0; i < num_write_bo_handles; i++) {
> +             struct dma_resv_iter resv_cursor;
> +             struct dma_fence *fence;
>
> -                     if (WARN_ON_ONCE(num_fences >= wait_info-
> >num_fences)) {
> +             dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
> +                                     DMA_RESV_USAGE_WRITE, fence) {
> +                     if (num_fences >= wait_info->num_fences) {
>                               r = -EINVAL;
> -                             goto free_fences;
> +                             goto error_unlock;
>                       }
>
> -                     fences[num_fences++] = fence;
> +                     fences[num_fences++] = dma_fence_get(fence);
>               }
> +     }
>
> -             /*
> -              * Keep only the latest fences to reduce the number of values
> -              * given back to userspace.
> -              */
> -             num_fences = dma_fence_dedup_array(fences, num_fences);
> +     drm_exec_fini(&exec);
>
> -             waitq = xa_load(&userq_mgr->userq_xa, wait_info->waitq_id);
> -             if (!waitq) {
> -                     r = -EINVAL;
> -                     goto free_fences;
> -             }
> +     /*
> +      * Keep only the latest fences to reduce the number of values
> +      * given back to userspace.
> +      */
> +     num_fences = dma_fence_dedup_array(fences, num_fences);
>
> -             for (i = 0, cnt = 0; i < num_fences; i++) {
> -                     struct amdgpu_userq_fence_driver *fence_drv;
> -                     struct amdgpu_userq_fence *userq_fence;
> -                     u32 index;
> -
> -                     userq_fence = to_amdgpu_userq_fence(fences[i]);
> -                     if (!userq_fence) {
> -                             /*
> -                              * Just waiting on other driver fences should
> -                              * be good for now
> -                              */
> -                             r = dma_fence_wait(fences[i], true);
> -                             if (r) {
> -                                     dma_fence_put(fences[i]);
> -                                     goto free_fences;
> -                             }
> +     waitq = xa_load(&userq_mgr->userq_xa, wait_info->waitq_id);
> +     if (!waitq) {
> +             r = -EINVAL;
> +             goto free_fences;
> +     }
>
> -                             dma_fence_put(fences[i]);
> -                             continue;
> -                     }
> +     for (i = 0, cnt = 0; i < num_fences; i++) {
> +             struct amdgpu_userq_fence_driver *fence_drv;
> +             struct amdgpu_userq_fence *userq_fence;
> +             u32 index;
>
> -                     fence_drv = userq_fence->fence_drv;
> +             userq_fence = to_amdgpu_userq_fence(fences[i]);
> +             if (!userq_fence) {
>                       /*
> -                      * We need to make sure the user queue release their
> reference
> -                      * to the fence drivers at some point before queue destruction.
> -                      * Otherwise, we would gather those references until we don't
> -                      * have any more space left and crash.
> +                      * Just waiting on other driver fences should
> +                      * be good for now
>                        */
> -                     r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
> -                                  xa_limit_32b, GFP_KERNEL);
> -                     if (r)
> +                     r = dma_fence_wait(fences[i], true);
> +                     if (r) {
> +                             dma_fence_put(fences[i]);
>                               goto free_fences;
> -
> -                     amdgpu_userq_fence_driver_get(fence_drv);
> -
> -                     /* Store drm syncobj's gpu va address and value */
> -                     fence_info[cnt].va = fence_drv->va;
> -                     fence_info[cnt].value = fences[i]->seqno;
> +                     }
>
>                       dma_fence_put(fences[i]);
> -                     /* Increment the actual userq fence count */
> -                     cnt++;
> +                     continue;
>               }
>
> -             wait_info->num_fences = cnt;
> -             /* Copy userq fence info to user space */
> -             if (copy_to_user(u64_to_user_ptr(wait_info->out_fences),
> -                              fence_info, wait_info->num_fences *
> sizeof(*fence_info))) {
> -                     r = -EFAULT;
> +             fence_drv = userq_fence->fence_drv;
> +             /*
> +              * We need to make sure the user queue release their reference
> +              * to the fence drivers at some point before queue destruction.
> +              * Otherwise, we would gather those references until we don't
> +              * have any more space left and crash.
> +              */
> +             r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
> +                          xa_limit_32b, GFP_KERNEL);
> +             if (r)
>                       goto free_fences;
> -             }
>
> -             kfree(fences);
> -             kfree(fence_info);
> -     }
> +             amdgpu_userq_fence_driver_get(fence_drv);
>
> -     drm_exec_fini(&exec);
> -     for (i = 0; i < num_read_bo_handles; i++)
> -             drm_gem_object_put(gobj_read[i]);
> -     kfree(gobj_read);
> +             /* Store drm syncobj's gpu va address and value */
> +             fence_info[cnt].va = fence_drv->va;
> +             fence_info[cnt].value = fences[i]->seqno;
>
> -     for (i = 0; i < num_write_bo_handles; i++)
> -             drm_gem_object_put(gobj_write[i]);
> -     kfree(gobj_write);
> +             dma_fence_put(fences[i]);
> +             /* Increment the actual userq fence count */
> +             cnt++;
> +     }
> +     /* The refcount in the array is already decremented */
> +     num_fences = 0;
>
> -     kfree(timeline_points);
> -     kfree(timeline_handles);
> -     kfree(syncobj_handles);
> -     kfree(bo_handles_write);
> -     kfree(bo_handles_read);
> +     wait_info->num_fences = cnt;
> +     r = 0;
>
> -     return 0;
> +     /* Copy userq fence info to user space */
> +     if (copy_to_user(u64_to_user_ptr(wait_info->out_fences),
> +                      fence_info, cnt * sizeof(*fence_info))) {
> +             r = -EFAULT;
> +     }
>
>  free_fences:
> -     while (num_fences-- > 0)
> +     while (num_fences--)
>               dma_fence_put(fences[num_fences]);
>       kfree(fences);
> +
>  free_fence_info:
>       kfree(fence_info);
> -exec_fini:
> +     return r;
> +
> +error_unlock:
>       drm_exec_fini(&exec);
> -put_gobj_write:
> -     while (wentry-- > 0)
> -             drm_gem_object_put(gobj_write[wentry]);
> +     goto free_fences;
> +}
> +
> +
> +int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
> +                         struct drm_file *filp)
> +{
> +     int num_points, num_syncobj, num_read_bo_handles,
> num_write_bo_handles;
> +     u32 *syncobj_handles, *timeline_points, *timeline_handles;
> +     struct drm_amdgpu_userq_wait *wait_info = data;
> +     struct drm_gem_object **gobj_write;
> +     struct drm_gem_object **gobj_read;
> +     void __user *ptr;
> +     int r;
> +
> +     if (!amdgpu_userq_enabled(dev))
> +             return -ENOTSUPP;
> +
> +     num_syncobj = wait_info->num_syncobj_handles;
> +     ptr = u64_to_user_ptr(wait_info->syncobj_handles);
> +     syncobj_handles = memdup_array_user(ptr, num_syncobj, sizeof(u32));
> +     if (IS_ERR(syncobj_handles))
> +             return PTR_ERR(syncobj_handles);
> +
> +     num_points = wait_info->num_syncobj_timeline_handles;
> +     ptr = u64_to_user_ptr(wait_info->syncobj_timeline_handles);
> +     timeline_handles = memdup_array_user(ptr, num_points, sizeof(u32));
> +     if (IS_ERR(timeline_handles)) {
> +             r = PTR_ERR(timeline_handles);
> +             goto free_syncobj_handles;
> +     }
> +
> +     ptr = u64_to_user_ptr(wait_info->syncobj_timeline_points);
> +     timeline_points = memdup_array_user(ptr, num_points, sizeof(u32));
> +     if (IS_ERR(timeline_points)) {
> +             r = PTR_ERR(timeline_points);
> +             goto free_timeline_handles;
> +     }
> +
> +     gobj_read = kmalloc_array(num_read_bo_handles, sizeof(*gobj_read),
> +                               GFP_KERNEL);
> +     if (!gobj_read) {
> +             r = -ENOMEM;
> +             goto free_timeline_points;
> +     }
> +
> +     ptr = u64_to_user_ptr(wait_info->bo_read_handles);
> +     num_read_bo_handles = wait_info->num_bo_read_handles;
> +     r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
> +     if (r)
> +             goto free_timeline_points;
> +
> +     ptr = u64_to_user_ptr(wait_info->bo_write_handles);


The gobj_write pointer allocation is missing, and this part also has some error handling issues.
I’ve drafted an attached patch to address this, which can be added to the end of this series.

> +     num_write_bo_handles = wait_info->num_bo_write_handles;
> +     r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
> +                                &gobj_write);
> +     if (r)
> +             goto put_gobj_read;
> +
> +     /*
> +      * Passing num_fences = 0 means that userspace doesn't want to
> +      * retrieve userq_fence_info. If num_fences = 0 we skip filling
> +      * userq_fence_info and return the actual number of fences on
> +      * args->num_fences.
> +      */
> +     if (!wait_info->num_fences) {
> +             r = amdgpu_userq_wait_count_fences(filp, wait_info,
> +                                                syncobj_handles,
> +                                                timeline_points,
> +                                                timeline_handles,
> +                                                gobj_write,
> +                                                gobj_read);
> +     } else {
> +             r = amdgpu_userq_wait_return_fence_info(filp, wait_info,
> +                                                     syncobj_handles,
> +                                                     timeline_points,
> +                                                     timeline_handles,
> +                                                     gobj_write,
> +                                                     gobj_read);
> +     }
> +
> +     while (num_write_bo_handles--)
> +             drm_gem_object_put(gobj_write[num_write_bo_handles]);
>       kfree(gobj_write);
>  put_gobj_read:
> -     while (rentry-- > 0)
> -             drm_gem_object_put(gobj_read[rentry]);
> +     while (num_read_bo_handles--)
> +             drm_gem_object_put(gobj_read[num_read_bo_handles]);
>       kfree(gobj_read);
>  free_timeline_points:
>       kfree(timeline_points);
> @@ -1014,10 +1043,5 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev,
> void *data,
>       kfree(timeline_handles);
>  free_syncobj_handles:
>       kfree(syncobj_handles);
> -free_bo_handles_write:
> -     kfree(bo_handles_write);
> -free_bo_handles_read:
> -     kfree(bo_handles_read);
> -
>       return r;
>  }
> --
> 2.43.0


[-- Attachment #2: 0001-drm-amdgpu-Fix-userq-wait-fence-gathering-and-error-.patch --]
[-- Type: application/octet-stream, Size: 1767 bytes --]

From f28668dad5d4d1fda96306156eab7f103b92d672 Mon Sep 17 00:00:00 2001
From: Prike Liang <Prike.Liang@amd.com>
Date: Tue, 3 Feb 2026 10:50:08 +0800
Subject: [PATCH 1/2] drm/amdgpu: Fix userq wait-fence gathering and error
 handling

Fix the userq write-buffer fence gathering and error handler path.

Signed-off-by: Prike Liang <Prike.Liang@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
index 8a1a0af59910..e8833fdec2ba 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
@@ -1005,14 +1005,21 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
 	num_read_bo_handles = wait_info->num_bo_read_handles;
 	r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
 	if (r)
-		goto free_timeline_points;
+		goto put_gobj_read;
+
+	gobj_write = kmalloc_array(num_read_bo_handles, sizeof(*gobj_write),
+				  GFP_KERNEL);
+	if (!gobj_write) {
+		r = -ENOMEM;
+		goto put_gobj_read;
+	}
 
 	ptr = u64_to_user_ptr(wait_info->bo_write_handles);
 	num_write_bo_handles = wait_info->num_bo_write_handles;
 	r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
 				   &gobj_write);
 	if (r)
-		goto put_gobj_read;
+		goto put_gobj_write;
 
 	/*
 	 * Passing num_fences = 0 means that userspace doesn't want to
@@ -1035,7 +1042,7 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
 							gobj_write,
 							gobj_read);
 	}
-
+put_gobj_write:
 	while (num_write_bo_handles--)
 		drm_gem_object_put(gobj_write[num_write_bo_handles]);
 	kfree(gobj_write);
-- 
2.34.1

Re: [PATCH 5/9] drm/amdgpu: fix eviction fence and userq manager shutdown

[Christian König]

On 2/2/26 23:37, Alex Deucher wrote:
> On Mon, Feb 2, 2026 at 7:51 AM Christian König
> <ckoenig.leichtzumerken@gmail.com> wrote:
>>
>> That is a really complicated dance and wasn't implemented fully correct.
> 
> I'd suggest elaborating on what was wrong with the previous flow.
> It's not entirely clear from the patch.
> 
>>
>> Signed-off-by: Christian König <christian.koenig@amd.com>
>> ---
>>  drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c            | 2 ++
>>  drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 8 +++++++-
>>  drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h | 1 +
>>  drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c          | 7 +++++--
>>  drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h          | 1 +
>>  5 files changed, 16 insertions(+), 3 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>> index 02abe85624a6..9cd44be45861 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>> @@ -2976,6 +2976,8 @@ static int amdgpu_drm_release(struct inode *inode, struct file *filp)
>>
>>         if (fpriv && drm_dev_enter(dev, &idx)) {
>>                 amdgpu_evf_mgr_shutdown(&fpriv->evf_mgr);
>> +               amdgpu_userq_mgr_cancel_resume(&fpriv->userq_mgr);
>> +               amdgpu_evf_mgr_flush_suspend(&fpriv->evf_mgr);
>>                 amdgpu_userq_mgr_fini(&fpriv->userq_mgr);
>>                 amdgpu_evf_mgr_fini(&fpriv->evf_mgr);
>>                 drm_dev_exit(idx);
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
>> index 8fe9f91f9551..ef4da6f2e2a3 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
>> @@ -146,13 +146,19 @@ void amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr *evf_mgr)
>>  void amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr *evf_mgr)
>>  {
>>         evf_mgr->shutdown = true;
>> +       /* Make sure that the shutdown is visible to the suspend work */
>>         flush_work(&evf_mgr->suspend_work);
>>  }
>>
>> -void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr)
>> +void amdgpu_evf_mgr_flush_suspend(struct amdgpu_eviction_fence_mgr *evf_mgr)
>>  {
>>         dma_fence_wait(rcu_dereference_protected(evf_mgr->ev_fence, true),
>>                        false);
>> +       /* Make sure that we are done with the last suspend work */
>>         flush_work(&evf_mgr->suspend_work);
> 
> Is it intended to flush the work here and in amdgpu_evf_mgr_shutdown()?

Yes, we need both. That's why I added the comments to explain why we have each.

Going to improve the commit message.

Christian.

> 
> Alex
> 
>> +}
>> +
>> +void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr)
>> +{
>>         dma_fence_put(evf_mgr->ev_fence);
>>  }
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
>> index 527de3a23583..132a13a5dc1c 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
>> @@ -66,6 +66,7 @@ void amdgpu_evf_mgr_detach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
>>                                  struct amdgpu_bo *bo);
>>  void amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr *evf_mgr);
>>  void amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr *evf_mgr);
>> +void amdgpu_evf_mgr_flush_suspend(struct amdgpu_eviction_fence_mgr *evf_mgr);
>>  void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr);
>>
>>  #endif
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>> index e007f10a6c1c..60e8a993557a 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>> @@ -1350,13 +1350,16 @@ int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *f
>>         return 0;
>>  }
>>
>> +void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr)
>> +{
>> +       cancel_delayed_work_sync(&userq_mgr->resume_work);
>> +}
>> +
>>  void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr)
>>  {
>>         struct amdgpu_usermode_queue *queue;
>>         unsigned long queue_id;
>>
>> -       cancel_delayed_work_sync(&userq_mgr->resume_work);
>> -
>>         mutex_lock(&userq_mgr->userq_mutex);
>>         amdgpu_userq_detect_and_reset_queues(userq_mgr);
>>         xa_for_each(&userq_mgr->userq_xa, queue_id, queue) {
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
>> index 095c09f3d96c..13f4e356728b 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
>> @@ -117,6 +117,7 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp
>>  int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,
>>                           struct amdgpu_device *adev);
>>
>> +void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr);
>>  void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr);
>>
>>  int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr,
>> --
>> 2.43.0
>>

Re: [PATCH 7/9] drm/amdgpu: rework amdgpu_userq_wait_ioctl v2

[Christian König]

On 2/3/26 04:39, Liang, Prike wrote:
>> +     ptr = u64_to_user_ptr(wait_info->bo_read_handles);
>> +     num_read_bo_handles = wait_info->num_bo_read_handles;
>> +     r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
>> +     if (r)
>> +             goto free_timeline_points;
>> +
>> +     ptr = u64_to_user_ptr(wait_info->bo_write_handles);
> 
> 
> The gobj_write pointer allocation is missing, and this part also has some error handling issues.
> I’ve drafted an attached patch to address this, which can be added to the end of this series.
> 
>> +     num_write_bo_handles = wait_info->num_bo_write_handles;
>> +     r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
>> +                                &gobj_write);

The gobj_write pointer is allocated by this call here.

We currently need to clean that up, but drm_gem_objects_lookup() should potentially be changed to do that itself.

Regards,
Christian.

Re: [PATCH 1/9] drm/amdgpu: lock both VM and BO in amdgpu_gem_object_open

[Khatri, Sunil]

On 02-02-2026 06:21 pm, Christian König wrote:
> The VM was not locked in the past since we initially only cleared the
> linked list element and not added it to any VM state.
>
> But this has changed quite some time ago, we just never realized this
> problem because the VM state lock was masking it.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>   .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c  | 19 +++++++++++-----
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c       | 22 ++++++++++++++-----
>   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c       | 10 +++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        |  2 ++
>   4 files changed, 42 insertions(+), 11 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> index 768998c82b43..ec5130497743 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> @@ -878,6 +878,7 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
>   	struct amdgpu_bo *bo[2] = {NULL, NULL};
>   	struct amdgpu_bo_va *bo_va;
>   	bool same_hive = false;
> +	struct drm_exec exec;
>   	int i, ret;
>   
>   	if (!va) {
> @@ -958,19 +959,25 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
>   			goto unwind;
>   		}
>   
> -		/* Add BO to VM internal data structures */
> -		ret = amdgpu_bo_reserve(bo[i], false);
> -		if (ret) {
> -			pr_debug("Unable to reserve BO during memory attach");
> -			goto unwind;
> +		drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
> +		drm_exec_until_all_locked(&exec) {
> +			ret = amdgpu_vm_lock_pd(vm, &exec, 0);
> +			drm_exec_retry_on_contention(&exec);
> +			if (unlikely(ret))
> +				goto unwind;
> +			ret = drm_exec_lock_obj(&exec, &bo[i]->tbo.base);
> +			drm_exec_retry_on_contention(&exec);
> +			if (unlikely(ret))
> +				goto unwind;
>   		}
> +
>   		bo_va = amdgpu_vm_bo_find(vm, bo[i]);
>   		if (!bo_va)
>   			bo_va = amdgpu_vm_bo_add(adev, vm, bo[i]);
>   		else
>   			++bo_va->ref_count;
>   		attachment[i]->bo_va = bo_va;
> -		amdgpu_bo_unreserve(bo[i]);
> +		drm_exec_fini(&exec);
>   		if (unlikely(!attachment[i]->bo_va)) {
>   			ret = -ENOMEM;
>   			pr_err("Failed to add BO object to VM. ret == %d\n",
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> index 5f9fa2140f09..5c90de58cc28 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> @@ -232,6 +232,7 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
>   	struct amdgpu_vm *vm = &fpriv->vm;
>   	struct amdgpu_bo_va *bo_va;
>   	struct mm_struct *mm;
> +	struct drm_exec exec;
>   	int r;
>   
>   	mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm);
> @@ -242,9 +243,18 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
>   	    !amdgpu_vm_is_bo_always_valid(vm, abo))
>   		return -EPERM;
>   
> -	r = amdgpu_bo_reserve(abo, false);
> -	if (r)
> -		return r;
> +	drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
> +	drm_exec_until_all_locked(&exec) {
> +		r = drm_exec_prepare_obj(&exec, &abo->tbo.base, 1);
> +		drm_exec_retry_on_contention(&exec);
> +		if (unlikely(r))
> +			goto out_unlock;
> +
> +		r = amdgpu_vm_lock_pd(vm, &exec, 0);
> +		drm_exec_retry_on_contention(&exec);
> +		if (unlikely(r))
> +			goto out_unlock;
> +	}
>   
>   	amdgpu_vm_bo_update_shared(abo);
>   	bo_va = amdgpu_vm_bo_find(vm, abo);
> @@ -260,8 +270,7 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
>   		amdgpu_bo_unreserve(abo);
>   		return r;
>   	}
> -
> -	amdgpu_bo_unreserve(abo);
> +	drm_exec_fini(&exec);
>   
>   	/* Validate and add eviction fence to DMABuf imports with dynamic
>   	 * attachment in compute VMs. Re-validation will be done by
> @@ -294,7 +303,10 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
>   		}
>   	}
>   	mutex_unlock(&vm->process_info->lock);
> +	return r;
>   
> +out_unlock:
> +	drm_exec_fini(&exec);
>   	return r;
>   }
>   
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index 1878e0faa722..f69332eed051 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -1445,6 +1445,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
>   {
>   	struct amdgpu_device *adev = drm_to_adev(dev);
>   	struct amdgpu_fpriv *fpriv;
> +	struct drm_exec exec;
>   	int r, pasid;
>   
>   	/* Ensure IB tests are run on ring */
> @@ -1484,7 +1485,16 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
>   	if (r)
>   		goto error_pasid;
>   
> +	drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
> +	drm_exec_until_all_locked(&exec) {
> +		r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 0);
Do we really need to do this here? We are in the process of creating a 
new VM and new PD so at this time, no one would be able to use the VM 
and PD till it returns from the function.

Looking at it with that understanding it feels we should be fine without 
locking the pd here

Regards
Sunil Khatri

> +		drm_exec_retry_on_contention(&exec);
> +		if (unlikely(r))
> +			goto error_vm;
> +	}
> +
>   	fpriv->prt_va = amdgpu_vm_bo_add(adev, &fpriv->vm, NULL);
> +	drm_exec_fini(&exec);
>   	if (!fpriv->prt_va) {
>   		r = -ENOMEM;
>   		goto error_vm;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> index 6a2ea200d90c..b4bf1b7c214f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -1735,6 +1735,8 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
>   {
>   	struct amdgpu_bo_va *bo_va;
>   
> +	amdgpu_vm_assert_locked(vm);
> +
>   	bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
>   	if (bo_va == NULL) {
>   		return NULL;

Re: [PATCH 7/9] drm/amdgpu: rework amdgpu_userq_wait_ioctl v2

[Khatri, Sunil]

[-- Attachment #1: Type: text/plain, Size: 23376 bytes --]


On 02-02-2026 06:21 pm, Christian König wrote:
> Lockdep was complaining about a number of issues here. Especially lock
> inversion between syncobj, dma_resv and copying things into userspace.
>
> Rework the functionality. Split it up into multiple functions,
> consistenly use memdup_array_user(), fix the lock inversions and a few
> more bugs in error handling.
>
> v2: drop the dma_fence leak fix, turned out that was actually correct,
>      just not well documented. Apply some more cleanup suggestion from
>      Tvrtko.
>
> Signed-off-by: Christian König<christian.koenig@amd.com>
> ---
>   .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 614 +++++++++---------
>   1 file changed, 319 insertions(+), 295 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> index 212056d4ddf0..da329d00843b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> @@ -645,368 +645,397 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>   	return r;
>   }
>   
> -int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
> -			    struct drm_file *filp)
> +/* Count the number of expected fences so userspace can alloc a buffer */
> +static int
> +amdgpu_userq_wait_count_fences(struct drm_file *filp,
> +			       struct drm_amdgpu_userq_wait *wait_info,
> +			       u32 *syncobj_handles, u32 *timeline_points,
> +			       u32 *timeline_handles,
> +			       struct drm_gem_object **gobj_write,
> +			       struct drm_gem_object **gobj_read)
>   {
> -	u32 *syncobj_handles, *timeline_points, *timeline_handles, *bo_handles_read, *bo_handles_write;
> -	u32 num_syncobj, num_read_bo_handles, num_write_bo_handles;
> -	struct drm_amdgpu_userq_fence_info *fence_info = NULL;
> -	struct drm_amdgpu_userq_wait *wait_info = data;
> -	struct amdgpu_fpriv *fpriv = filp->driver_priv;
> -	struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
> -	struct amdgpu_usermode_queue *waitq;
> -	struct drm_gem_object **gobj_write;
> -	struct drm_gem_object **gobj_read;
> -	struct dma_fence **fences = NULL;
> -	u16 num_points, num_fences = 0;
> -	int r, i, rentry, wentry, cnt;
> +	int num_read_bo_handles, num_write_bo_handles;
> +	struct dma_fence_unwrap iter;
> +	struct dma_fence *fence, *f;
> +	unsigned int num_fences = 0;
>   	struct drm_exec exec;
> +	int i, r;
> +
> +	/*
> +	 * This needs to be outside of the lock provided by drm_exec for
> +	 * DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT to work correctly.
> +	 */
> +
> +	/* Count timeline fences */
> +	for (i = 0; i < wait_info->num_syncobj_timeline_handles; i++) {
> +		r = drm_syncobj_find_fence(filp, timeline_handles[i],
> +					   timeline_points[i],
> +					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> +					   &fence);
> +		if (r)
> +			return r;
> +
> +		dma_fence_unwrap_for_each(f, &iter, fence)
> +			num_fences++;
>   
> -	if (!amdgpu_userq_enabled(dev))
> -		return -ENOTSUPP;
> +		dma_fence_put(fence);
> +	}
>   
> -	num_read_bo_handles = wait_info->num_bo_read_handles;
> -	bo_handles_read = memdup_user(u64_to_user_ptr(wait_info->bo_read_handles),
> -				      size_mul(sizeof(u32), num_read_bo_handles));
> -	if (IS_ERR(bo_handles_read))
> -		return PTR_ERR(bo_handles_read);
> +	/* Count boolean fences */
> +	for (i = 0; i < wait_info->num_syncobj_handles; i++) {
> +		r = drm_syncobj_find_fence(filp, syncobj_handles[i], 0,
> +					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> +					   &fence);
> +		if (r)
> +			return r;
>   
> -	num_write_bo_handles = wait_info->num_bo_write_handles;
> -	bo_handles_write = memdup_user(u64_to_user_ptr(wait_info->bo_write_handles),
> -				       size_mul(sizeof(u32), num_write_bo_handles));
> -	if (IS_ERR(bo_handles_write)) {
> -		r = PTR_ERR(bo_handles_write);
> -		goto free_bo_handles_read;
> +		num_fences++;
> +		dma_fence_put(fence);
>   	}
>   
> -	num_syncobj = wait_info->num_syncobj_handles;
> -	syncobj_handles = memdup_user(u64_to_user_ptr(wait_info->syncobj_handles),
> -				      size_mul(sizeof(u32), num_syncobj));
> -	if (IS_ERR(syncobj_handles)) {
> -		r = PTR_ERR(syncobj_handles);
> -		goto free_bo_handles_write;
> -	}
> +	/* Lock all the GEM objects */
> +	/* TODO: It is actually not necessary to lock them */
> +	num_read_bo_handles = wait_info->num_bo_read_handles;
> +	num_write_bo_handles = wait_info->num_bo_write_handles;
> +	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
> +		      num_read_bo_handles + num_write_bo_handles);
>   
> -	num_points = wait_info->num_syncobj_timeline_handles;
> -	timeline_handles = memdup_user(u64_to_user_ptr(wait_info->syncobj_timeline_handles),
> -				       sizeof(u32) * num_points);
> -	if (IS_ERR(timeline_handles)) {
> -		r = PTR_ERR(timeline_handles);
> -		goto free_syncobj_handles;
> -	}
> +	drm_exec_until_all_locked(&exec) {
> +		r = drm_exec_prepare_array(&exec, gobj_read,
> +					   num_read_bo_handles, 1);
> +		drm_exec_retry_on_contention(&exec);
> +		if (r)
> +			goto error_unlock;
>   
> -	timeline_points = memdup_user(u64_to_user_ptr(wait_info->syncobj_timeline_points),
> -				      sizeof(u32) * num_points);
> -	if (IS_ERR(timeline_points)) {
> -		r = PTR_ERR(timeline_points);
> -		goto free_timeline_handles;
> +		r = drm_exec_prepare_array(&exec, gobj_write,
> +					   num_write_bo_handles, 1);
> +		drm_exec_retry_on_contention(&exec);
> +		if (r)
> +			goto error_unlock;
>   	}
>   
> -	gobj_read = kmalloc_array(num_read_bo_handles, sizeof(*gobj_read), GFP_KERNEL);
> -	if (!gobj_read) {
> -		r = -ENOMEM;
> -		goto free_timeline_points;
> -	}
> +	/* Count read fences */
> +	for (i = 0; i < num_read_bo_handles; i++) {
> +		struct dma_resv_iter resv_cursor;
> +		struct dma_fence *fence;
>   
> -	for (rentry = 0; rentry < num_read_bo_handles; rentry++) {
> -		gobj_read[rentry] = drm_gem_object_lookup(filp, bo_handles_read[rentry]);
> -		if (!gobj_read[rentry]) {
> -			r = -ENOENT;
> -			goto put_gobj_read;
> -		}
> +		dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
> +					DMA_RESV_USAGE_READ, fence)
> +			num_fences++;
>   	}
>   
> -	gobj_write = kmalloc_array(num_write_bo_handles, sizeof(*gobj_write), GFP_KERNEL);
> -	if (!gobj_write) {
> -		r = -ENOMEM;
> -		goto put_gobj_read;
> -	}
> +	/* Count write fences */
> +	for (i = 0; i < num_write_bo_handles; i++) {
> +		struct dma_resv_iter resv_cursor;
> +		struct dma_fence *fence;
>   
> -	for (wentry = 0; wentry < num_write_bo_handles; wentry++) {
> -		gobj_write[wentry] = drm_gem_object_lookup(filp, bo_handles_write[wentry]);
> -		if (!gobj_write[wentry]) {
> -			r = -ENOENT;
> -			goto put_gobj_write;
> -		}
> +		dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
> +					DMA_RESV_USAGE_WRITE, fence)
> +			num_fences++;
>   	}
>   
> -	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
> -		      (num_read_bo_handles + num_write_bo_handles));
> -
> -	/* Lock all BOs with retry handling */
> -	drm_exec_until_all_locked(&exec) {
> -		r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1);
> -		drm_exec_retry_on_contention(&exec);
> -		if (r) {
> -			drm_exec_fini(&exec);
> -			goto put_gobj_write;
> -		}
> +	wait_info->num_fences = num_fences;
> +	r = 0;
>   
> -		r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1);
> -		drm_exec_retry_on_contention(&exec);
> -		if (r) {
> -			drm_exec_fini(&exec);
> -			goto put_gobj_write;
> -		}
> -	}
> +error_unlock:
> +	/* Unlock all the GEM objects */
> +	drm_exec_fini(&exec);
> +	return r;
> +}
>   
> -	if (!wait_info->num_fences) {
> -		if (num_points) {
> -			struct dma_fence_unwrap iter;
> -			struct dma_fence *fence;
> -			struct dma_fence *f;
> -
> -			for (i = 0; i < num_points; i++) {
> -				r = drm_syncobj_find_fence(filp, timeline_handles[i],
> -							   timeline_points[i],
> -							   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> -							   &fence);
> -				if (r)
> -					goto exec_fini;
> -
> -				dma_fence_unwrap_for_each(f, &iter, fence)
> -					num_fences++;
> -
> -				dma_fence_put(fence);
> -			}
> -		}
> +static int
> +amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
> +				    struct drm_amdgpu_userq_wait *wait_info,
> +				    u32 *syncobj_handles, u32 *timeline_points,
> +				    u32 *timeline_handles,
> +				    struct drm_gem_object **gobj_write,
> +				    struct drm_gem_object **gobj_read)
> +{
> +	struct amdgpu_fpriv *fpriv = filp->driver_priv;
> +	struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
> +	struct drm_amdgpu_userq_fence_info *fence_info;
> +	int num_read_bo_handles, num_write_bo_handles;
> +	struct amdgpu_usermode_queue *waitq;
> +	struct dma_fence **fences, *fence, *f;
> +	struct dma_fence_unwrap iter;
> +	int num_points, num_syncobj;
> +	unsigned int num_fences = 0;
> +	struct drm_exec exec;
> +	int i, cnt, r;
>   
> -		/* Count syncobj's fence */
> -		for (i = 0; i < num_syncobj; i++) {
> -			struct dma_fence *fence;
> +	fence_info = kmalloc_array(wait_info->num_fences, sizeof(*fence_info),
> +				   GFP_KERNEL);
> +	if (!fence_info)
> +		return -ENOMEM;
>   
> -			r = drm_syncobj_find_fence(filp, syncobj_handles[i],
> -						   0,
> -						   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> -						   &fence);
> -			if (r)
> -				goto exec_fini;
> +	fences = kmalloc_array(wait_info->num_fences, sizeof(*fences),
> +			       GFP_KERNEL);
> +	if (!fences) {
> +		r = -ENOMEM;
> +		goto free_fence_info;
> +	}
>   
> -			num_fences++;
> -			dma_fence_put(fence);
> -		}
> +	/* Retrieve timeline fences */
> +	num_points = wait_info->num_syncobj_timeline_handles;
> +	for (i = 0; i < num_points; i++) {
> +		r = drm_syncobj_find_fence(filp, timeline_handles[i],
> +					   timeline_points[i],
> +					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> +					   &fence);
> +		if (r)
> +			goto free_fences;
>   
> -		/* Count GEM objects fence */
> -		for (i = 0; i < num_read_bo_handles; i++) {
> -			struct dma_resv_iter resv_cursor;
> -			struct dma_fence *fence;
> +		dma_fence_unwrap_for_each(f, &iter, fence) {
> +			if (num_fences >= wait_info->num_fences) {
> +				r = -EINVAL;
> +				goto free_fences;
> +			}
>   
> -			dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
> -						DMA_RESV_USAGE_READ, fence)
> -				num_fences++;
> +			fences[num_fences++] = dma_fence_get(f);
>   		}
>   
> -		for (i = 0; i < num_write_bo_handles; i++) {
> -			struct dma_resv_iter resv_cursor;
> -			struct dma_fence *fence;
> +		dma_fence_put(fence);
> +	}
>   
> -			dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
> -						DMA_RESV_USAGE_WRITE, fence)
> -				num_fences++;
> -		}
> +	/* Retrieve boolean fences */
> +	num_syncobj = wait_info->num_syncobj_handles;
> +	for (i = 0; i < num_syncobj; i++) {
> +		struct dma_fence *fence;
>   
> -		/*
> -		 * Passing num_fences = 0 means that userspace doesn't want to
> -		 * retrieve userq_fence_info. If num_fences = 0 we skip filling
> -		 * userq_fence_info and return the actual number of fences on
> -		 * args->num_fences.
> -		 */
> -		wait_info->num_fences = num_fences;
> -	} else {
> -		/* Array of fence info */
> -		fence_info = kmalloc_array(wait_info->num_fences, sizeof(*fence_info), GFP_KERNEL);
> -		if (!fence_info) {
> -			r = -ENOMEM;
> -			goto exec_fini;
> -		}
> +		r = drm_syncobj_find_fence(filp, syncobj_handles[i], 0,
> +					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> +					   &fence);
> +		if (r)
> +			goto free_fences;
>   
> -		/* Array of fences */
> -		fences = kmalloc_array(wait_info->num_fences, sizeof(*fences), GFP_KERNEL);
> -		if (!fences) {
> -			r = -ENOMEM;
> -			goto free_fence_info;
> +		if (num_fences >= wait_info->num_fences) {
> +			r = -EINVAL;
> +			goto free_fences;
>   		}
>   
> -		/* Retrieve GEM read objects fence */
> -		for (i = 0; i < num_read_bo_handles; i++) {
> -			struct dma_resv_iter resv_cursor;
> -			struct dma_fence *fence;
> +		/* Give the reference to the fence array */
> +		fences[num_fences++] = fence;
> +	}
>   
> -			dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
> -						DMA_RESV_USAGE_READ, fence) {
> -				if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) {
> -					r = -EINVAL;
> -					goto free_fences;
> -				}
> +	/* Lock all the GEM objects */
> +	num_read_bo_handles = wait_info->num_bo_read_handles;
> +	num_write_bo_handles = wait_info->num_bo_write_handles;
> +	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
> +		      num_read_bo_handles + num_write_bo_handles);
>   
> -				fences[num_fences++] = fence;
> -				dma_fence_get(fence);
> -			}
> -		}
> +	drm_exec_until_all_locked(&exec) {
> +		r = drm_exec_prepare_array(&exec, gobj_read,
> +					   num_read_bo_handles, 1);
> +		drm_exec_retry_on_contention(&exec);
> +		if (r)
> +			goto error_unlock;
>   
> -		/* Retrieve GEM write objects fence */
> -		for (i = 0; i < num_write_bo_handles; i++) {
> -			struct dma_resv_iter resv_cursor;
> -			struct dma_fence *fence;
> +		r = drm_exec_prepare_array(&exec, gobj_write,
> +					   num_write_bo_handles, 1);
> +		drm_exec_retry_on_contention(&exec);
> +		if (r)
> +			goto error_unlock;
> +	}
>   
> -			dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
> -						DMA_RESV_USAGE_WRITE, fence) {
> -				if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) {
> -					r = -EINVAL;
> -					goto free_fences;
> -				}
> +	/* Retrieve GEM read objects fence */
> +	for (i = 0; i < num_read_bo_handles; i++) {
> +		struct dma_resv_iter resv_cursor;
> +		struct dma_fence *fence;
>   
> -				fences[num_fences++] = fence;
> -				dma_fence_get(fence);
> +		dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
> +					DMA_RESV_USAGE_READ, fence) {
> +			if (num_fences >= wait_info->num_fences) {
> +				r = -EINVAL;
> +				goto error_unlock;
>   			}
> -		}
>   
> -		if (num_points) {
> -			struct dma_fence_unwrap iter;
> -			struct dma_fence *fence;
> -			struct dma_fence *f;
> -
> -			for (i = 0; i < num_points; i++) {
> -				r = drm_syncobj_find_fence(filp, timeline_handles[i],
> -							   timeline_points[i],
> -							   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> -							   &fence);
> -				if (r)
> -					goto free_fences;
> -
> -				dma_fence_unwrap_for_each(f, &iter, fence) {
> -					if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) {
> -						r = -EINVAL;
> -						goto free_fences;
> -					}
> -
> -					dma_fence_get(f);
> -					fences[num_fences++] = f;
> -				}
> -
> -				dma_fence_put(fence);
> -			}
> +			fences[num_fences++] = dma_fence_get(fence);
>   		}
> +	}
>   
> -		/* Retrieve syncobj's fence */
> -		for (i = 0; i < num_syncobj; i++) {
> -			struct dma_fence *fence;
> -
> -			r = drm_syncobj_find_fence(filp, syncobj_handles[i],
> -						   0,
> -						   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
> -						   &fence);
> -			if (r)
> -				goto free_fences;
> +	/* Retrieve GEM write objects fence */
> +	for (i = 0; i < num_write_bo_handles; i++) {
> +		struct dma_resv_iter resv_cursor;
> +		struct dma_fence *fence;
>   
> -			if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) {
> +		dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
> +					DMA_RESV_USAGE_WRITE, fence) {
> +			if (num_fences >= wait_info->num_fences) {
>   				r = -EINVAL;
> -				goto free_fences;
> +				goto error_unlock;
>   			}
>   
> -			fences[num_fences++] = fence;
> +			fences[num_fences++] = dma_fence_get(fence);
>   		}
> +	}
>   
> -		/*
> -		 * Keep only the latest fences to reduce the number of values
> -		 * given back to userspace.
> -		 */
> -		num_fences = dma_fence_dedup_array(fences, num_fences);
> +	drm_exec_fini(&exec);
>   
> -		waitq = xa_load(&userq_mgr->userq_xa, wait_info->waitq_id);
> -		if (!waitq) {
> -			r = -EINVAL;
> -			goto free_fences;
> -		}
> +	/*
> +	 * Keep only the latest fences to reduce the number of values
> +	 * given back to userspace.
> +	 */
> +	num_fences = dma_fence_dedup_array(fences, num_fences);
>   
> -		for (i = 0, cnt = 0; i < num_fences; i++) {
> -			struct amdgpu_userq_fence_driver *fence_drv;
> -			struct amdgpu_userq_fence *userq_fence;
> -			u32 index;
> -
> -			userq_fence = to_amdgpu_userq_fence(fences[i]);
> -			if (!userq_fence) {
> -				/*
> -				 * Just waiting on other driver fences should
> -				 * be good for now
> -				 */
> -				r = dma_fence_wait(fences[i], true);
> -				if (r) {
> -					dma_fence_put(fences[i]);
> -					goto free_fences;
> -				}
> +	waitq = xa_load(&userq_mgr->userq_xa, wait_info->waitq_id);
> +	if (!waitq) {
> +		r = -EINVAL;
> +		goto free_fences;
> +	}
>   
> -				dma_fence_put(fences[i]);
> -				continue;
> -			}
> +	for (i = 0, cnt = 0; i < num_fences; i++) {
> +		struct amdgpu_userq_fence_driver *fence_drv;
> +		struct amdgpu_userq_fence *userq_fence;
> +		u32 index;
>   
> -			fence_drv = userq_fence->fence_drv;
> +		userq_fence = to_amdgpu_userq_fence(fences[i]);
> +		if (!userq_fence) {
>   			/*
> -			 * We need to make sure the user queue release their reference
> -			 * to the fence drivers at some point before queue destruction.
> -			 * Otherwise, we would gather those references until we don't
> -			 * have any more space left and crash.
> +			 * Just waiting on other driver fences should
> +			 * be good for now
>   			 */
> -			r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
> -				     xa_limit_32b, GFP_KERNEL);
> -			if (r)
> +			r = dma_fence_wait(fences[i], true);
> +			if (r) {
> +				dma_fence_put(fences[i]);
>   				goto free_fences;
> -
> -			amdgpu_userq_fence_driver_get(fence_drv);
> -
> -			/* Store drm syncobj's gpu va address and value */
> -			fence_info[cnt].va = fence_drv->va;
> -			fence_info[cnt].value = fences[i]->seqno;
> +			}
>   
>   			dma_fence_put(fences[i]);
> -			/* Increment the actual userq fence count */
> -			cnt++;
> +			continue;
>   		}
>   
> -		wait_info->num_fences = cnt;
> -		/* Copy userq fence info to user space */
> -		if (copy_to_user(u64_to_user_ptr(wait_info->out_fences),
> -				 fence_info, wait_info->num_fences * sizeof(*fence_info))) {
> -			r = -EFAULT;
> +		fence_drv = userq_fence->fence_drv;
> +		/*
> +		 * We need to make sure the user queue release their reference
> +		 * to the fence drivers at some point before queue destruction.
> +		 * Otherwise, we would gather those references until we don't
> +		 * have any more space left and crash.
> +		 */
> +		r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
> +			     xa_limit_32b, GFP_KERNEL);
> +		if (r)
>   			goto free_fences;
> -		}
>   
> -		kfree(fences);
> -		kfree(fence_info);
> -	}
> +		amdgpu_userq_fence_driver_get(fence_drv);
>   
> -	drm_exec_fini(&exec);
> -	for (i = 0; i < num_read_bo_handles; i++)
> -		drm_gem_object_put(gobj_read[i]);
> -	kfree(gobj_read);
> +		/* Store drm syncobj's gpu va address and value */
> +		fence_info[cnt].va = fence_drv->va;
> +		fence_info[cnt].value = fences[i]->seqno;
>   
> -	for (i = 0; i < num_write_bo_handles; i++)
> -		drm_gem_object_put(gobj_write[i]);
> -	kfree(gobj_write);
> +		dma_fence_put(fences[i]);
> +		/* Increment the actual userq fence count */
> +		cnt++;
> +	}
> +	/* The refcount in the array is already decremented */
> +	num_fences = 0;
>   
> -	kfree(timeline_points);
> -	kfree(timeline_handles);
> -	kfree(syncobj_handles);
> -	kfree(bo_handles_write);
> -	kfree(bo_handles_read);
> +	wait_info->num_fences = cnt;
> +	r = 0;
>   
> -	return 0;
> +	/* Copy userq fence info to user space */
> +	if (copy_to_user(u64_to_user_ptr(wait_info->out_fences),
> +			 fence_info, cnt * sizeof(*fence_info))) {
> +		r = -EFAULT;
> +	}
>   
>   free_fences:
> -	while (num_fences-- > 0)
> +	while (num_fences--)
>   		dma_fence_put(fences[num_fences]);
>   	kfree(fences);
> +
>   free_fence_info:
>   	kfree(fence_info);
> -exec_fini:
> +	return r;
> +
> +error_unlock:
>   	drm_exec_fini(&exec);
> -put_gobj_write:
> -	while (wentry-- > 0)
> -		drm_gem_object_put(gobj_write[wentry]);
> +	goto free_fences;
> +}
> +
> +
> +int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
> +			    struct drm_file *filp)
> +{
> +	int num_points, num_syncobj, num_read_bo_handles, num_write_bo_handles;
> +	u32 *syncobj_handles, *timeline_points, *timeline_handles;
> +	struct drm_amdgpu_userq_wait *wait_info = data;
> +	struct drm_gem_object **gobj_write;
> +	struct drm_gem_object **gobj_read;
> +	void __user *ptr;
> +	int r;
> +
> +	if (!amdgpu_userq_enabled(dev))
> +		return -ENOTSUPP;
> +
> +	num_syncobj = wait_info->num_syncobj_handles;
> +	ptr = u64_to_user_ptr(wait_info->syncobj_handles);
> +	syncobj_handles = memdup_array_user(ptr, num_syncobj, sizeof(u32));
> +	if (IS_ERR(syncobj_handles))
> +		return PTR_ERR(syncobj_handles);
> +
> +	num_points = wait_info->num_syncobj_timeline_handles;
> +	ptr = u64_to_user_ptr(wait_info->syncobj_timeline_handles);
> +	timeline_handles = memdup_array_user(ptr, num_points, sizeof(u32));
> +	if (IS_ERR(timeline_handles)) {
> +		r = PTR_ERR(timeline_handles);
> +		goto free_syncobj_handles;
> +	}
> +
> +	ptr = u64_to_user_ptr(wait_info->syncobj_timeline_points);
> +	timeline_points = memdup_array_user(ptr, num_points, sizeof(u32));
> +	if (IS_ERR(timeline_points)) {
> +		r = PTR_ERR(timeline_points);
> +		goto free_timeline_handles;
> +	}
> +
> +	gobj_read = kmalloc_array(num_read_bo_handles, sizeof(*gobj_read),
> +				  GFP_KERNEL);
memory is allocated for gobj_read before initializing num_read_bo_handle 
which is done later.
> +	if (!gobj_read) {
> +		r = -ENOMEM;
> +		goto free_timeline_points;
> +	}
> +
> +	ptr = u64_to_user_ptr(wait_info->bo_read_handles);
> +	num_read_bo_handles = wait_info->num_bo_read_handles;
> +	r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
> +	if (r)
> +		goto free_timeline_points;
> +
> +	ptr = u64_to_user_ptr(wait_info->bo_write_handles);
> +	num_write_bo_handles = wait_info->num_bo_write_handles;

No memory allocated here for gobj_write before using that.

Regards
Sunil khatri

> +	r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
> +				   &gobj_write);
> +	if (r)
> +		goto put_gobj_read;
> +
> +	/*
> +	 * Passing num_fences = 0 means that userspace doesn't want to
> +	 * retrieve userq_fence_info. If num_fences = 0 we skip filling
> +	 * userq_fence_info and return the actual number of fences on
> +	 * args->num_fences.
> +	 */
> +	if (!wait_info->num_fences) {
> +		r = amdgpu_userq_wait_count_fences(filp, wait_info,
> +						   syncobj_handles,
> +						   timeline_points,
> +						   timeline_handles,
> +						   gobj_write,
> +						   gobj_read);
> +	} else {
> +		r = amdgpu_userq_wait_return_fence_info(filp, wait_info,
> +							syncobj_handles,
> +							timeline_points,
> +							timeline_handles,
> +							gobj_write,
> +							gobj_read);
> +	}
> +
> +	while (num_write_bo_handles--)
> +		drm_gem_object_put(gobj_write[num_write_bo_handles]);
>   	kfree(gobj_write);
>   put_gobj_read:
> -	while (rentry-- > 0)
> -		drm_gem_object_put(gobj_read[rentry]);
> +	while (num_read_bo_handles--)
> +		drm_gem_object_put(gobj_read[num_read_bo_handles]);
>   	kfree(gobj_read);
>   free_timeline_points:
>   	kfree(timeline_points);
> @@ -1014,10 +1043,5 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
>   	kfree(timeline_handles);
>   free_syncobj_handles:
>   	kfree(syncobj_handles);
> -free_bo_handles_write:
> -	kfree(bo_handles_write);
> -free_bo_handles_read:
> -	kfree(bo_handles_read);
> -
>   	return r;
>   }

[-- Attachment #2: Type: text/html, Size: 23219 bytes --]

Re: [PATCH 1/9] drm/amdgpu: lock both VM and BO in amdgpu_gem_object_open

[Christian König]

On 2/6/26 11:21, Khatri, Sunil wrote:
> 
> On 02-02-2026 06:21 pm, Christian König wrote:
>> The VM was not locked in the past since we initially only cleared the
>> linked list element and not added it to any VM state.
>>
>> But this has changed quite some time ago, we just never realized this
>> problem because the VM state lock was masking it.
>>
>> Signed-off-by: Christian König <christian.koenig@amd.com>
>> ---
>>   .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c  | 19 +++++++++++-----
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c       | 22 ++++++++++++++-----
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c       | 10 +++++++++
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        |  2 ++
>>   4 files changed, 42 insertions(+), 11 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
>> index 768998c82b43..ec5130497743 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
>> @@ -878,6 +878,7 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
>>       struct amdgpu_bo *bo[2] = {NULL, NULL};
>>       struct amdgpu_bo_va *bo_va;
>>       bool same_hive = false;
>> +    struct drm_exec exec;
>>       int i, ret;
>>         if (!va) {
>> @@ -958,19 +959,25 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
>>               goto unwind;
>>           }
>>   -        /* Add BO to VM internal data structures */
>> -        ret = amdgpu_bo_reserve(bo[i], false);
>> -        if (ret) {
>> -            pr_debug("Unable to reserve BO during memory attach");
>> -            goto unwind;
>> +        drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
>> +        drm_exec_until_all_locked(&exec) {
>> +            ret = amdgpu_vm_lock_pd(vm, &exec, 0);
>> +            drm_exec_retry_on_contention(&exec);
>> +            if (unlikely(ret))
>> +                goto unwind;
>> +            ret = drm_exec_lock_obj(&exec, &bo[i]->tbo.base);
>> +            drm_exec_retry_on_contention(&exec);
>> +            if (unlikely(ret))
>> +                goto unwind;
>>           }
>> +
>>           bo_va = amdgpu_vm_bo_find(vm, bo[i]);
>>           if (!bo_va)
>>               bo_va = amdgpu_vm_bo_add(adev, vm, bo[i]);
>>           else
>>               ++bo_va->ref_count;
>>           attachment[i]->bo_va = bo_va;
>> -        amdgpu_bo_unreserve(bo[i]);
>> +        drm_exec_fini(&exec);
>>           if (unlikely(!attachment[i]->bo_va)) {
>>               ret = -ENOMEM;
>>               pr_err("Failed to add BO object to VM. ret == %d\n",
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
>> index 5f9fa2140f09..5c90de58cc28 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
>> @@ -232,6 +232,7 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
>>       struct amdgpu_vm *vm = &fpriv->vm;
>>       struct amdgpu_bo_va *bo_va;
>>       struct mm_struct *mm;
>> +    struct drm_exec exec;
>>       int r;
>>         mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm);
>> @@ -242,9 +243,18 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
>>           !amdgpu_vm_is_bo_always_valid(vm, abo))
>>           return -EPERM;
>>   -    r = amdgpu_bo_reserve(abo, false);
>> -    if (r)
>> -        return r;
>> +    drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
>> +    drm_exec_until_all_locked(&exec) {
>> +        r = drm_exec_prepare_obj(&exec, &abo->tbo.base, 1);
>> +        drm_exec_retry_on_contention(&exec);
>> +        if (unlikely(r))
>> +            goto out_unlock;
>> +
>> +        r = amdgpu_vm_lock_pd(vm, &exec, 0);
>> +        drm_exec_retry_on_contention(&exec);
>> +        if (unlikely(r))
>> +            goto out_unlock;
>> +    }
>>         amdgpu_vm_bo_update_shared(abo);
>>       bo_va = amdgpu_vm_bo_find(vm, abo);
>> @@ -260,8 +270,7 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
>>           amdgpu_bo_unreserve(abo);
>>           return r;
>>       }
>> -
>> -    amdgpu_bo_unreserve(abo);
>> +    drm_exec_fini(&exec);
>>         /* Validate and add eviction fence to DMABuf imports with dynamic
>>        * attachment in compute VMs. Re-validation will be done by
>> @@ -294,7 +303,10 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
>>           }
>>       }
>>       mutex_unlock(&vm->process_info->lock);
>> +    return r;
>>   +out_unlock:
>> +    drm_exec_fini(&exec);
>>       return r;
>>   }
>>   diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>> index 1878e0faa722..f69332eed051 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>> @@ -1445,6 +1445,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
>>   {
>>       struct amdgpu_device *adev = drm_to_adev(dev);
>>       struct amdgpu_fpriv *fpriv;
>> +    struct drm_exec exec;
>>       int r, pasid;
>>         /* Ensure IB tests are run on ring */
>> @@ -1484,7 +1485,16 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
>>       if (r)
>>           goto error_pasid;
>>   +    drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
>> +    drm_exec_until_all_locked(&exec) {
>> +        r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 0);
> Do we really need to do this here? We are in the process of creating a new VM and new PD so at this time, no one would be able to use the VM and PD till it returns from the function.
> 
> Looking at it with that understanding it feels we should be fine without locking the pd here

The PD is already on TTM BO eviction list at that point and might be locked concurrently.

So yes, we absolutely need that.

Regards,
Christian.

> 
> Regards
> Sunil Khatri
> 
>> +        drm_exec_retry_on_contention(&exec);
>> +        if (unlikely(r))
>> +            goto error_vm;
>> +    }
>> +
>>       fpriv->prt_va = amdgpu_vm_bo_add(adev, &fpriv->vm, NULL);
>> +    drm_exec_fini(&exec);
>>       if (!fpriv->prt_va) {
>>           r = -ENOMEM;
>>           goto error_vm;
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
>> index 6a2ea200d90c..b4bf1b7c214f 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
>> @@ -1735,6 +1735,8 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
>>   {
>>       struct amdgpu_bo_va *bo_va;
>>   +    amdgpu_vm_assert_locked(vm);
>> +
>>       bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
>>       if (bo_va == NULL) {
>>           return NULL;

RE: [PATCH 1/9] drm/amdgpu: lock both VM and BO in amdgpu_gem_object_open

[Liang, Prike]

[Public]

Regards,
      Prike

> -----Original Message-----
> From: Alex Deucher <alexdeucher@gmail.com>
> Sent: Tuesday, February 3, 2026 5:45 AM
> To: Christian König <ckoenig.leichtzumerken@gmail.com>
> Cc: tursulin@ursulin.net; Deucher, Alexander <Alexander.Deucher@amd.com>;
> Liang, Prike <Prike.Liang@amd.com>; Mohan Marimuthu, Yogesh
> <Yogesh.Mohanmarimuthu@amd.com>; SHANMUGAM, SRINIVASAN
> <SRINIVASAN.SHANMUGAM@amd.com>; Khatri, Sunil <Sunil.Khatri@amd.com>;
> amd-gfx@lists.freedesktop.org
> Subject: Re: [PATCH 1/9] drm/amdgpu: lock both VM and BO in
> amdgpu_gem_object_open
>
> On Mon, Feb 2, 2026 at 7:51 AM Christian König
> <ckoenig.leichtzumerken@gmail.com> wrote:
> >
> > The VM was not locked in the past since we initially only cleared the
> > linked list element and not added it to any VM state.
> >
> > But this has changed quite some time ago, we just never realized this
> > problem because the VM state lock was masking it.
> >
> > Signed-off-by: Christian König <christian.koenig@amd.com>
>
> Reviewed-by: Alex Deucher <alexander.deucher@amd.com>
>
> > ---
> >  .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c  | 19 +++++++++++-----
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c       | 22 ++++++++++++++-----
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c       | 10 +++++++++
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        |  2 ++
> >  4 files changed, 42 insertions(+), 11 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> > index 768998c82b43..ec5130497743 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> > @@ -878,6 +878,7 @@ static int kfd_mem_attach(struct amdgpu_device *adev,
> struct kgd_mem *mem,
> >         struct amdgpu_bo *bo[2] = {NULL, NULL};
> >         struct amdgpu_bo_va *bo_va;
> >         bool same_hive = false;
> > +       struct drm_exec exec;
> >         int i, ret;
> >
> >         if (!va) {
> > @@ -958,19 +959,25 @@ static int kfd_mem_attach(struct amdgpu_device *adev,
> struct kgd_mem *mem,
> >                         goto unwind;
> >                 }
> >
> > -               /* Add BO to VM internal data structures */
> > -               ret = amdgpu_bo_reserve(bo[i], false);
> > -               if (ret) {
> > -                       pr_debug("Unable to reserve BO during memory attach");
> > -                       goto unwind;
> > +               drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
> > +               drm_exec_until_all_locked(&exec) {
> > +                       ret = amdgpu_vm_lock_pd(vm, &exec, 0);
> > +                       drm_exec_retry_on_contention(&exec);
> > +                       if (unlikely(ret))
> > +                               goto unwind;
Here we should handle the error return path properly and destroy the exec resource in the error handler.

> > +                       ret = drm_exec_lock_obj(&exec, &bo[i]->tbo.base);
> > +                       drm_exec_retry_on_contention(&exec);
> > +                       if (unlikely(ret))
> > +                               goto unwind;
> >                 }
> > +
> >                 bo_va = amdgpu_vm_bo_find(vm, bo[i]);
> >                 if (!bo_va)
> >                         bo_va = amdgpu_vm_bo_add(adev, vm, bo[i]);
> >                 else
> >                         ++bo_va->ref_count;
> >                 attachment[i]->bo_va = bo_va;
> > -               amdgpu_bo_unreserve(bo[i]);
> > +               drm_exec_fini(&exec);
> >                 if (unlikely(!attachment[i]->bo_va)) {
> >                         ret = -ENOMEM;
> >                         pr_err("Failed to add BO object to VM. ret ==
> > %d\n", diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> > index 5f9fa2140f09..5c90de58cc28 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> > @@ -232,6 +232,7 @@ static int amdgpu_gem_object_open(struct
> drm_gem_object *obj,
> >         struct amdgpu_vm *vm = &fpriv->vm;
> >         struct amdgpu_bo_va *bo_va;
> >         struct mm_struct *mm;
> > +       struct drm_exec exec;
> >         int r;
> >
> >         mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm);
> > @@ -242,9 +243,18 @@ static int amdgpu_gem_object_open(struct
> drm_gem_object *obj,
> >             !amdgpu_vm_is_bo_always_valid(vm, abo))
> >                 return -EPERM;
> >
> > -       r = amdgpu_bo_reserve(abo, false);
> > -       if (r)
> > -               return r;
> > +       drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
> > +       drm_exec_until_all_locked(&exec) {
> > +               r = drm_exec_prepare_obj(&exec, &abo->tbo.base, 1);
> > +               drm_exec_retry_on_contention(&exec);
> > +               if (unlikely(r))
> > +                       goto out_unlock;
> > +
> > +               r = amdgpu_vm_lock_pd(vm, &exec, 0);
> > +               drm_exec_retry_on_contention(&exec);
> > +               if (unlikely(r))
> > +                       goto out_unlock;
> > +       }
> >
> >         amdgpu_vm_bo_update_shared(abo);
> >         bo_va = amdgpu_vm_bo_find(vm, abo); @@ -260,8 +270,7 @@ static
> > int amdgpu_gem_object_open(struct drm_gem_object *obj,
> >                 amdgpu_bo_unreserve(abo);
> >                 return r;
> >         }
> > -
> > -       amdgpu_bo_unreserve(abo);
> > +       drm_exec_fini(&exec);
> >
> >         /* Validate and add eviction fence to DMABuf imports with dynamic
> >          * attachment in compute VMs. Re-validation will be done by @@
> > -294,7 +303,10 @@ static int amdgpu_gem_object_open(struct drm_gem_object
> *obj,
> >                 }
> >         }
> >         mutex_unlock(&vm->process_info->lock);
> > +       return r;
> >
> > +out_unlock:
> > +       drm_exec_fini(&exec);
> >         return r;
> >  }
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > index 1878e0faa722..f69332eed051 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > @@ -1445,6 +1445,7 @@ int amdgpu_driver_open_kms(struct drm_device
> > *dev, struct drm_file *file_priv)  {
> >         struct amdgpu_device *adev = drm_to_adev(dev);
> >         struct amdgpu_fpriv *fpriv;
> > +       struct drm_exec exec;
> >         int r, pasid;
> >
> >         /* Ensure IB tests are run on ring */ @@ -1484,7 +1485,16 @@
> > int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
> >         if (r)
> >                 goto error_pasid;
> >
> > +       drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
> > +       drm_exec_until_all_locked(&exec) {
> > +               r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 0);
> > +               drm_exec_retry_on_contention(&exec);
> > +               if (unlikely(r))
> > +                       goto error_vm;
> > +       }
> > +
Same here
> >         fpriv->prt_va = amdgpu_vm_bo_add(adev, &fpriv->vm, NULL);
> > +       drm_exec_fini(&exec);
> >         if (!fpriv->prt_va) {
> >                 r = -ENOMEM;
> >                 goto error_vm;
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> > index 6a2ea200d90c..b4bf1b7c214f 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> > @@ -1735,6 +1735,8 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct
> > amdgpu_device *adev,  {
> >         struct amdgpu_bo_va *bo_va;
> >
> > +       amdgpu_vm_assert_locked(vm);
> > +
> >         bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
> >         if (bo_va == NULL) {
> >                 return NULL;
> > --
> > 2.43.0
> >