[PATCH 01/11] drm/amdgpu: fix AMDGPU_INFO_READ_MMR_REG

[PATCH 01/11] drm/amdgpu: fix AMDGPU_INFO_READ_MMR_REG

[Christian König]

There were multiple issues in that code.

First of all the order between the reset semaphore and the mm_lock was
wrong (e.g. copy_to_user) was called while holding the lock.

Then we allocated memory while holding the reset semaphore which is also
a pretty big bug and can deadlock.

Then we used down_read_trylock() instead of waiting for the reset to
finish.

Signed-off-by: Christian König <christian.koenig@amd.com>
Fixes: 9e823f307074 ("drm/amdgpu: Block MMR_READ IOCTL in reset")
Reviewed-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 57 +++++++++++--------------
 1 file changed, 24 insertions(+), 33 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index d88e4994c8c1..24526e92f9b8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -873,68 +873,59 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 				    ? -EFAULT : 0;
 	}
 	case AMDGPU_INFO_READ_MMR_REG: {
-		int ret = 0;
-		unsigned int n, alloc_size;
-		uint32_t *regs;
 		unsigned int se_num = (info->read_mmr_reg.instance >>
 				   AMDGPU_INFO_MMR_SE_INDEX_SHIFT) &
 				  AMDGPU_INFO_MMR_SE_INDEX_MASK;
 		unsigned int sh_num = (info->read_mmr_reg.instance >>
 				   AMDGPU_INFO_MMR_SH_INDEX_SHIFT) &
 				  AMDGPU_INFO_MMR_SH_INDEX_MASK;
-
-		if (!down_read_trylock(&adev->reset_domain->sem))
-			return -ENOENT;
+		unsigned int alloc_size;
+		uint32_t *regs;
+		int ret;
 
 		/* set full masks if the userspace set all bits
 		 * in the bitfields
 		 */
-		if (se_num == AMDGPU_INFO_MMR_SE_INDEX_MASK) {
+		if (se_num == AMDGPU_INFO_MMR_SE_INDEX_MASK)
 			se_num = 0xffffffff;
-		} else if (se_num >= AMDGPU_GFX_MAX_SE) {
-			ret = -EINVAL;
-			goto out;
-		}
+		else if (se_num >= AMDGPU_GFX_MAX_SE)
+			return -EINVAL;
 
-		if (sh_num == AMDGPU_INFO_MMR_SH_INDEX_MASK) {
+		if (sh_num == AMDGPU_INFO_MMR_SH_INDEX_MASK)
 			sh_num = 0xffffffff;
-		} else if (sh_num >= AMDGPU_GFX_MAX_SH_PER_SE) {
-			ret = -EINVAL;
-			goto out;
-		}
+		else if (sh_num >= AMDGPU_GFX_MAX_SH_PER_SE)
+			return -EINVAL;
 
-		if (info->read_mmr_reg.count > 128) {
-			ret = -EINVAL;
-			goto out;
-		}
+		if (info->read_mmr_reg.count > 128)
+			return -EINVAL;
 
-		regs = kmalloc_array(info->read_mmr_reg.count, sizeof(*regs), GFP_KERNEL);
-		if (!regs) {
-			ret = -ENOMEM;
-			goto out;
-		}
+		regs = kmalloc_array(info->read_mmr_reg.count, sizeof(*regs),
+				     GFP_KERNEL);
+		if (!regs)
+			return -ENOMEM;
 
+		down_read(&adev->reset_domain->sem);
 		alloc_size = info->read_mmr_reg.count * sizeof(*regs);
-
 		amdgpu_gfx_off_ctrl(adev, false);
+		ret = 0;
 		for (i = 0; i < info->read_mmr_reg.count; i++) {
 			if (amdgpu_asic_read_register(adev, se_num, sh_num,
 						      info->read_mmr_reg.dword_offset + i,
 						      &regs[i])) {
 				DRM_DEBUG_KMS("unallowed offset %#x\n",
 					      info->read_mmr_reg.dword_offset + i);
-				kfree(regs);
-				amdgpu_gfx_off_ctrl(adev, true);
 				ret = -EFAULT;
-				goto out;
+				break;
 			}
 		}
 		amdgpu_gfx_off_ctrl(adev, true);
-		n = copy_to_user(out, regs, min(size, alloc_size));
-		kfree(regs);
-		ret = (n ? -EFAULT : 0);
-out:
 		up_read(&adev->reset_domain->sem);
+
+		if (!ret) {
+			ret = copy_to_user(out, regs, min(size, alloc_size))
+				? -EFAULT : 0;
+		}
+		kfree(regs);
 		return ret;
 	}
 	case AMDGPU_INFO_DEV_INFO: {
-- 
2.43.0

[PATCH 02/11] drm/amdgpu: remove deadlocks from amdgpu_userq_pre_reset

[Christian König]

The purpose of a GPU reset is to make sure that fence can be signaled
again and the signal and resume workers can make progress again.

So waiting for the resume worker or any fence in the GPU reset path is
just utterly nonsense.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 26 +++++++++++------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 8f48520cb822..b632bc3c952b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -1496,23 +1496,21 @@ void amdgpu_userq_pre_reset(struct amdgpu_device *adev)
 {
 	const struct amdgpu_userq_funcs *userq_funcs;
 	struct amdgpu_usermode_queue *queue;
-	struct amdgpu_userq_mgr *uqm;
 	unsigned long queue_id;
 
+	/* TODO: We probably need a new lock for the queue state */
 	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
-		uqm = queue->userq_mgr;
-		cancel_delayed_work_sync(&uqm->resume_work);
-		if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
-			amdgpu_userq_wait_for_last_fence(queue);
-			userq_funcs = adev->userq_funcs[queue->queue_type];
-			userq_funcs->unmap(queue);
-			/* just mark all queues as hung at this point.
-			 * if unmap succeeds, we could map again
-			 * in amdgpu_userq_post_reset() if vram is not lost
-			 */
-			queue->state = AMDGPU_USERQ_STATE_HUNG;
-			amdgpu_userq_fence_driver_force_completion(queue);
-		}
+		if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
+			continue;
+
+		userq_funcs = adev->userq_funcs[queue->queue_type];
+		userq_funcs->unmap(queue);
+		/* just mark all queues as hung at this point.
+		 * if unmap succeeds, we could map again
+		 * in amdgpu_userq_post_reset() if vram is not lost
+		 */
+		queue->state = AMDGPU_USERQ_STATE_HUNG;
+		amdgpu_userq_fence_driver_force_completion(queue);
 	}
 }
 
-- 
2.43.0

[PATCH 03/11] drm/amdgpu: nuke amdgpu_userq_fence_free

[Christian König]

As preparation for independent fences remove the function and do all of
it's cleanup directly after signaling.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c       | 13 +--
 .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 79 +++++++------------
 .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.h   |  3 -
 3 files changed, 31 insertions(+), 64 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index c6546a858597..1b15b51dc3f4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -3162,11 +3162,7 @@ static int __init amdgpu_init(void)
 
 	r = amdgpu_sync_init();
 	if (r)
-		goto error_sync;
-
-	r = amdgpu_userq_fence_slab_init();
-	if (r)
-		goto error_fence;
+		return r;
 
 	amdgpu_register_atpx_handler();
 	amdgpu_acpi_detect();
@@ -3182,12 +3178,6 @@ static int __init amdgpu_init(void)
 
 	/* let modprobe override vga console setting */
 	return pci_register_driver(&amdgpu_kms_pci_driver);
-
-error_fence:
-	amdgpu_sync_fini();
-
-error_sync:
-	return r;
 }
 
 static void __exit amdgpu_exit(void)
@@ -3197,7 +3187,6 @@ static void __exit amdgpu_exit(void)
 	amdgpu_unregister_atpx_handler();
 	amdgpu_acpi_release();
 	amdgpu_sync_fini();
-	amdgpu_userq_fence_slab_fini();
 	mmu_notifier_synchronize();
 	amdgpu_xcp_drv_release();
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
index a58342c2ac44..909bdccc2a92 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
@@ -32,29 +32,9 @@
 #include "amdgpu.h"
 #include "amdgpu_userq_fence.h"
 
-static const struct dma_fence_ops amdgpu_userq_fence_ops;
-static struct kmem_cache *amdgpu_userq_fence_slab;
-
 #define AMDGPU_USERQ_MAX_HANDLES	(1U << 16)
 
-int amdgpu_userq_fence_slab_init(void)
-{
-	amdgpu_userq_fence_slab = kmem_cache_create("amdgpu_userq_fence",
-						    sizeof(struct amdgpu_userq_fence),
-						    0,
-						    SLAB_HWCACHE_ALIGN,
-						    NULL);
-	if (!amdgpu_userq_fence_slab)
-		return -ENOMEM;
-
-	return 0;
-}
-
-void amdgpu_userq_fence_slab_fini(void)
-{
-	rcu_barrier();
-	kmem_cache_destroy(amdgpu_userq_fence_slab);
-}
+static const struct dma_fence_ops amdgpu_userq_fence_ops;
 
 static inline struct amdgpu_userq_fence *to_amdgpu_userq_fence(struct dma_fence *f)
 {
@@ -146,12 +126,18 @@ amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq)
 }
 
 static void
-amdgpu_userq_fence_put_fence_drv_array(struct amdgpu_userq_fence *userq_fence)
+amdgpu_userq_fence_put_fence_drv_refs(struct amdgpu_userq_fence *userq_fence)
 {
 	unsigned long i;
+
 	for (i = 0; i < userq_fence->fence_drv_array_count; i++)
 		amdgpu_userq_fence_driver_put(userq_fence->fence_drv_array[i]);
 	userq_fence->fence_drv_array_count = 0;
+	kfree(userq_fence->fence_drv_array);
+	userq_fence->fence_drv_array = NULL;
+
+	amdgpu_userq_fence_driver_put(userq_fence->fence_drv);
+	userq_fence->fence_drv = NULL;
 }
 
 void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
@@ -181,10 +167,11 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
 		fence = &userq_fence->base;
 		list_del_init(&userq_fence->link);
 		dma_fence_signal(fence);
-		/* Drop fence_drv_array outside fence_list_lock
+		/*
+		 * Drop fence_drv_array outside fence_list_lock
 		 * to avoid the recursion lock.
 		 */
-		amdgpu_userq_fence_put_fence_drv_array(userq_fence);
+		amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
 		dma_fence_put(fence);
 	}
 
@@ -231,7 +218,7 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
 
 static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
 {
-	*userq_fence = kmem_cache_alloc(amdgpu_userq_fence_slab, GFP_ATOMIC);
+	*userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
 	return *userq_fence ? 0 : -ENOMEM;
 }
 
@@ -299,7 +286,7 @@ static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
 	spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
 
 	if (signaled)
-		amdgpu_userq_fence_put_fence_drv_array(userq_fence);
+		amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
 
 	*f = fence;
 
@@ -333,29 +320,10 @@ static bool amdgpu_userq_fence_signaled(struct dma_fence *f)
 	return false;
 }
 
-static void amdgpu_userq_fence_free(struct rcu_head *rcu)
-{
-	struct dma_fence *fence = container_of(rcu, struct dma_fence, rcu);
-	struct amdgpu_userq_fence *userq_fence = to_amdgpu_userq_fence(fence);
-	struct amdgpu_userq_fence_driver *fence_drv = userq_fence->fence_drv;
-
-	/* Release the fence driver reference */
-	amdgpu_userq_fence_driver_put(fence_drv);
-
-	kvfree(userq_fence->fence_drv_array);
-	kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
-}
-
-static void amdgpu_userq_fence_release(struct dma_fence *f)
-{
-	call_rcu(&f->rcu, amdgpu_userq_fence_free);
-}
-
 static const struct dma_fence_ops amdgpu_userq_fence_ops = {
 	.get_driver_name = amdgpu_userq_fence_get_driver_name,
 	.get_timeline_name = amdgpu_userq_fence_get_timeline_name,
 	.signaled = amdgpu_userq_fence_signaled,
-	.release = amdgpu_userq_fence_release,
 };
 
 /**
@@ -546,7 +514,7 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 	r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
 	if (r) {
 		mutex_unlock(&userq_mgr->userq_mutex);
-		kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
+		kfree(userq_fence);
 		goto put_gobj_write;
 	}
 
@@ -871,6 +839,7 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
 	for (i = 0, cnt = 0; i < num_fences; i++) {
 		struct amdgpu_userq_fence_driver *fence_drv;
 		struct amdgpu_userq_fence *userq_fence;
+		unsigned long flags;
 		u32 index;
 
 		userq_fence = to_amdgpu_userq_fence(fences[i]);
@@ -886,7 +855,19 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
 			continue;
 		}
 
+		spin_lock_irqsave(userq_fence->base.lock, flags);
+		if (dma_fence_is_signaled_locked(&userq_fence->base)) {
+			/*
+			 * It is possible that fence is already signaled and the
+			 * fence_drv now NULL, just skip over such fences.
+			*/
+			spin_unlock_irqrestore(userq_fence->base.lock, flags);
+			continue;
+		}
 		fence_drv = userq_fence->fence_drv;
+		amdgpu_userq_fence_driver_get(fence_drv);
+		spin_unlock_irqrestore(userq_fence->base.lock, flags);
+
 		/*
 		 * We need to make sure the user queue release their reference
 		 * to the fence drivers at some point before queue destruction.
@@ -895,10 +876,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
 		 */
 		r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
 			     xa_limit_32b, GFP_KERNEL);
-		if (r)
+		if (r) {
+			amdgpu_userq_fence_driver_put(fence_drv);
 			goto put_waitq;
-
-		amdgpu_userq_fence_driver_get(fence_drv);
+		}
 
 		/* Store drm syncobj's gpu va address and value */
 		fence_info[cnt].va = fence_drv->va;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
index d56246ad8c26..d355a0eecc07 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
@@ -58,9 +58,6 @@ struct amdgpu_userq_fence_driver {
 	char timeline_name[TASK_COMM_LEN];
 };
 
-int amdgpu_userq_fence_slab_init(void);
-void amdgpu_userq_fence_slab_fini(void);
-
 void amdgpu_userq_fence_driver_get(struct amdgpu_userq_fence_driver *fence_drv);
 void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv);
 int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,
-- 
2.43.0

[PATCH 04/11] drm/amdgpu: rework amdgpu_userq_signal_ioctl

[Christian König]

This one was fortunately not looking so bad as the wait ioctl path, but
there were still a few things which could be fixed/improved:

1. Allocating with GFP_ATOMIC was quite unecessary, we can do that
   before taking the userq_lock.
2. Use a new mutex as protection for the fence_drv_xa so that we can do
   memory allocations while holding it.
3. Starting the reset timer is unecessary when the fence is already
   signaled when we create it.
4. Cleanup error handling, avoid trying to free the queue when we don't
   even got one.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c     |   1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h     |  12 +
 .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 220 ++++++++----------
 3 files changed, 111 insertions(+), 122 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index b632bc3c952b..174190a77005 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -793,6 +793,7 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 	}
 
 	queue->doorbell_index = index;
+	mutex_init(&queue->fence_drv_lock);
 	xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
 	r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
 	if (r) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index 675fe6395ac8..cb92789c1ed1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -66,6 +66,18 @@ struct amdgpu_usermode_queue {
 	struct amdgpu_userq_obj	db_obj;
 	struct amdgpu_userq_obj fw_obj;
 	struct amdgpu_userq_obj wptr_obj;
+
+	/**
+	 * @fence_drv_lock: Protecting @fence_drv_xa.
+	 */
+	struct mutex		fence_drv_lock;
+
+	/**
+	 * @fence_drv_xa:
+	 *
+	 * References to the external fence drivers returned by wait_ioctl.
+	 * Dropped on the next signaled dma_fence or queue destruction.
+	 */
 	struct xarray		fence_drv_xa;
 	struct amdgpu_userq_fence_driver *fence_drv;
 	struct dma_fence	*last_fence;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
index 909bdccc2a92..b0543fa257ed 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
@@ -121,6 +121,7 @@ amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq)
 	userq->last_fence = NULL;
 	amdgpu_userq_walk_and_drop_fence_drv(&userq->fence_drv_xa);
 	xa_destroy(&userq->fence_drv_xa);
+	mutex_destroy(&userq->fence_drv_lock);
 	/* Drop the queue's ownership reference to fence_drv explicitly */
 	amdgpu_userq_fence_driver_put(userq->fence_drv);
 }
@@ -216,81 +217,77 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
 	kref_put(&fence_drv->refcount, amdgpu_userq_fence_driver_destroy);
 }
 
-static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
+static int amdgpu_userq_fence_alloc(struct amdgpu_usermode_queue *userq,
+				    struct amdgpu_userq_fence **pfence)
 {
-	*userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
-	return *userq_fence ? 0 : -ENOMEM;
+	struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
+	struct amdgpu_userq_fence *userq_fence;
+	unsigned long count;
+
+	userq_fence = kmalloc(sizeof(*userq_fence), GFP_KERNEL);
+	if (!userq_fence)
+		return -ENOMEM;
+
+	/*
+	 * Get the next unused entry, since we fill from the start this can be
+	 * used as size to allocate the array.
+	 */
+	mutex_lock(&userq->fence_drv_lock);
+	xa_find(&userq->fence_drv_xa, &count, ULONG_MAX, XA_FREE_MARK);
+
+	userq_fence->fence_drv_array = kvmalloc_array(count, sizeof(fence_drv),
+						      GFP_KERNEL);
+	if (!userq_fence->fence_drv_array) {
+		mutex_unlock(&userq->fence_drv_lock);
+		kfree(userq_fence);
+		return -ENOMEM;
+	}
+
+	userq_fence->fence_drv_array_count = count;
+	xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array,
+		   0, ULONG_MAX, count, XA_PRESENT);
+	xa_destroy(&userq->fence_drv_xa);
+
+	mutex_unlock(&userq->fence_drv_lock);
+
+	userq_fence->fence_drv = fence_drv;
+	amdgpu_userq_fence_driver_get(fence_drv);
+
+	*pfence = userq_fence;
+	return 0;
 }
 
-static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
-				     struct amdgpu_userq_fence *userq_fence,
-				     u64 seq, struct dma_fence **f)
+static void amdgpu_userq_fence_init(struct amdgpu_usermode_queue *userq,
+				    struct amdgpu_userq_fence *fence,
+				    u64 seq)
 {
-	struct amdgpu_userq_fence_driver *fence_drv;
-	struct dma_fence *fence;
+	struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
 	unsigned long flags;
 	bool signaled = false;
 
-	fence_drv = userq->fence_drv;
-	if (!fence_drv)
-		return -EINVAL;
-
-	spin_lock_init(&userq_fence->lock);
-	INIT_LIST_HEAD(&userq_fence->link);
-	fence = &userq_fence->base;
-	userq_fence->fence_drv = fence_drv;
-
-	dma_fence_init64(fence, &amdgpu_userq_fence_ops, &userq_fence->lock,
+	spin_lock_init(&fence->lock);
+	dma_fence_init64(&fence->base, &amdgpu_userq_fence_ops, &fence->lock,
 			 fence_drv->context, seq);
 
-	amdgpu_userq_fence_driver_get(fence_drv);
-	dma_fence_get(fence);
-
-	if (!xa_empty(&userq->fence_drv_xa)) {
-		struct amdgpu_userq_fence_driver *stored_fence_drv;
-		unsigned long index, count = 0;
-		int i = 0;
-
-		xa_lock(&userq->fence_drv_xa);
-		xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv)
-			count++;
-
-		userq_fence->fence_drv_array =
-			kvmalloc_array(count,
-				       sizeof(struct amdgpu_userq_fence_driver *),
-				       GFP_ATOMIC);
-
-		if (userq_fence->fence_drv_array) {
-			xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv) {
-				userq_fence->fence_drv_array[i] = stored_fence_drv;
-				__xa_erase(&userq->fence_drv_xa, index);
-				i++;
-			}
-		}
-
-		userq_fence->fence_drv_array_count = i;
-		xa_unlock(&userq->fence_drv_xa);
-	} else {
-		userq_fence->fence_drv_array = NULL;
-		userq_fence->fence_drv_array_count = 0;
-	}
+	/* Make sure the fence is visible to the hang detect worker */
+	dma_fence_put(userq->last_fence);
+	userq->last_fence = dma_fence_get(&fence->base);
 
-	/* Check if hardware has already processed the job */
+	/* Check if hardware has already processed the fence */
 	spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
-	if (!dma_fence_is_signaled(fence)) {
-		list_add_tail(&userq_fence->link, &fence_drv->fences);
+	if (!dma_fence_is_signaled(&fence->base)) {
+		dma_fence_get(&fence->base);
+		list_add_tail(&fence->link, &fence_drv->fences);
 	} else {
+		INIT_LIST_HEAD(&fence->link);
 		signaled = true;
-		dma_fence_put(fence);
 	}
 	spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
 
 	if (signaled)
-		amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
-
-	*f = fence;
-
-	return 0;
+		amdgpu_userq_fence_put_fence_drv_refs(fence);
+	else
+		amdgpu_userq_start_hang_detect_work(userq);
 }
 
 static const char *amdgpu_userq_fence_get_driver_name(struct dma_fence *f)
@@ -392,11 +389,6 @@ static int amdgpu_userq_fence_read_wptr(struct amdgpu_device *adev,
 	return r;
 }
 
-static void amdgpu_userq_fence_cleanup(struct dma_fence *fence)
-{
-	dma_fence_put(fence);
-}
-
 static void
 amdgpu_userq_fence_driver_set_error(struct amdgpu_userq_fence *fence,
 				    int error)
@@ -440,13 +432,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 	const unsigned int num_read_bo_handles = args->num_bo_read_handles;
 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
 	struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
+
 	struct drm_gem_object **gobj_write, **gobj_read;
 	u32 *syncobj_handles, num_syncobj_handles;
-	struct amdgpu_userq_fence *userq_fence;
-	struct amdgpu_usermode_queue *queue = NULL;
-	struct drm_syncobj **syncobj = NULL;
-	struct dma_fence *fence;
+	struct amdgpu_usermode_queue *queue;
+	struct amdgpu_userq_fence *fence;
+	struct drm_syncobj **syncobj;
 	struct drm_exec exec;
+	void __user *ptr;
 	int r, i, entry;
 	u64 wptr;
 
@@ -458,13 +451,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 		return -EINVAL;
 
 	num_syncobj_handles = args->num_syncobj_handles;
-	syncobj_handles = memdup_array_user(u64_to_user_ptr(args->syncobj_handles),
-					    num_syncobj_handles, sizeof(u32));
+	ptr = u64_to_user_ptr(args->syncobj_handles);
+	syncobj_handles = memdup_array_user(ptr, num_syncobj_handles,
+					    sizeof(u32));
 	if (IS_ERR(syncobj_handles))
 		return PTR_ERR(syncobj_handles);
 
-	/* Array of pointers to the looked up syncobjs */
-	syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj), GFP_KERNEL);
+	syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj),
+				GFP_KERNEL);
 	if (!syncobj) {
 		r = -ENOMEM;
 		goto free_syncobj_handles;
@@ -478,21 +472,17 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 		}
 	}
 
-	r = drm_gem_objects_lookup(filp,
-				   u64_to_user_ptr(args->bo_read_handles),
-				   num_read_bo_handles,
-				   &gobj_read);
+	ptr = u64_to_user_ptr(args->bo_read_handles);
+	r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
 	if (r)
 		goto free_syncobj;
 
-	r = drm_gem_objects_lookup(filp,
-				   u64_to_user_ptr(args->bo_write_handles),
-				   num_write_bo_handles,
+	ptr = u64_to_user_ptr(args->bo_write_handles);
+	r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
 				   &gobj_write);
 	if (r)
 		goto put_gobj_read;
 
-	/* Retrieve the user queue */
 	queue = amdgpu_userq_get(userq_mgr, args->queue_id);
 	if (!queue) {
 		r = -ENOENT;
@@ -501,73 +491,61 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 
 	r = amdgpu_userq_fence_read_wptr(adev, queue, &wptr);
 	if (r)
-		goto put_gobj_write;
+		goto put_queue;
 
-	r = amdgpu_userq_fence_alloc(&userq_fence);
+	r = amdgpu_userq_fence_alloc(queue, &fence);
 	if (r)
-		goto put_gobj_write;
+		goto put_queue;
 
 	/* We are here means UQ is active, make sure the eviction fence is valid */
 	amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
 
-	/* Create a new fence */
-	r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
-	if (r) {
-		mutex_unlock(&userq_mgr->userq_mutex);
-		kfree(userq_fence);
-		goto put_gobj_write;
-	}
+	/* Create the new fence */
+	amdgpu_userq_fence_init(queue, fence, wptr);
 
-	dma_fence_put(queue->last_fence);
-	queue->last_fence = dma_fence_get(fence);
-	amdgpu_userq_start_hang_detect_work(queue);
 	mutex_unlock(&userq_mgr->userq_mutex);
 
+	/*
+	 * This needs to come after the fence is created since
+	 * amdgpu_userq_ensure_ev_fence() can't be called while holding the resv
+	 * locks.
+	 */
 	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
 		      (num_read_bo_handles + num_write_bo_handles));
 
-	/* Lock all BOs with retry handling */
 	drm_exec_until_all_locked(&exec) {
-		r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1);
+		r = drm_exec_prepare_array(&exec, gobj_read,
+					   num_read_bo_handles, 1);
 		drm_exec_retry_on_contention(&exec);
-		if (r) {
-			amdgpu_userq_fence_cleanup(fence);
+		if (r)
 			goto exec_fini;
-		}
 
-		r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1);
+		r = drm_exec_prepare_array(&exec, gobj_write,
+					   num_write_bo_handles, 1);
 		drm_exec_retry_on_contention(&exec);
-		if (r) {
-			amdgpu_userq_fence_cleanup(fence);
+		if (r)
 			goto exec_fini;
-		}
 	}
 
-	for (i = 0; i < num_read_bo_handles; i++) {
-		if (!gobj_read || !gobj_read[i]->resv)
-			continue;
-
-		dma_resv_add_fence(gobj_read[i]->resv, fence,
+	/* And publish the new fence in the BOs and syncobj */
+	for (i = 0; i < num_read_bo_handles; i++)
+		dma_resv_add_fence(gobj_read[i]->resv, &fence->base,
 				   DMA_RESV_USAGE_READ);
-	}
 
-	for (i = 0; i < num_write_bo_handles; i++) {
-		if (!gobj_write || !gobj_write[i]->resv)
-			continue;
-
-		dma_resv_add_fence(gobj_write[i]->resv, fence,
+	for (i = 0; i < num_write_bo_handles; i++)
+		dma_resv_add_fence(gobj_write[i]->resv, &fence->base,
 				   DMA_RESV_USAGE_WRITE);
-	}
 
-	/* Add the created fence to syncobj/BO's */
 	for (i = 0; i < num_syncobj_handles; i++)
-		drm_syncobj_replace_fence(syncobj[i], fence);
+		drm_syncobj_replace_fence(syncobj[i], &fence->base);
 
+exec_fini:
 	/* drop the reference acquired in fence creation function */
-	dma_fence_put(fence);
+	dma_fence_put(&fence->base);
 
-exec_fini:
 	drm_exec_fini(&exec);
+put_queue:
+	amdgpu_userq_put(queue);
 put_gobj_write:
 	for (i = 0; i < num_write_bo_handles; i++)
 		drm_gem_object_put(gobj_write[i]);
@@ -578,15 +556,11 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 	kvfree(gobj_read);
 free_syncobj:
 	while (entry-- > 0)
-		if (syncobj[entry])
-			drm_syncobj_put(syncobj[entry]);
+		drm_syncobj_put(syncobj[entry]);
 	kfree(syncobj);
 free_syncobj_handles:
 	kfree(syncobj_handles);
 
-	if (queue)
-		amdgpu_userq_put(queue);
-
 	return r;
 }
 
@@ -874,8 +848,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
 		 * Otherwise, we would gather those references until we don't
 		 * have any more space left and crash.
 		 */
+		mutex_lock(&waitq->fence_drv_lock);
 		r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
 			     xa_limit_32b, GFP_KERNEL);
+		mutex_unlock(&waitq->fence_drv_lock);
 		if (r) {
 			amdgpu_userq_fence_driver_put(fence_drv);
 			goto put_waitq;
-- 
2.43.0

[PATCH 05/11] drm/amdgpu: rework userq fence signal processing

[Christian König]

Move more code into a common userq function.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 13 +++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h |  1 +
 drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c    | 10 +---------
 drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c    | 10 +---------
 drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c    | 11 +----------
 drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c    | 11 +----------
 drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c    | 11 +----------
 7 files changed, 19 insertions(+), 48 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 174190a77005..8ce001481d42 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -205,6 +205,19 @@ void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
 		     msecs_to_jiffies(timeout_ms));
 }
 
+void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell)
+{
+	struct xarray *xa = &adev->userq_doorbell_xa;
+	struct amdgpu_usermode_queue *queue;
+	unsigned long flags;
+
+	xa_lock_irqsave(xa, flags);
+	queue = xa_load(xa, doorbell);
+	if (queue)
+		amdgpu_userq_fence_driver_process(queue->fence_drv);
+	xa_unlock_irqrestore(xa, flags);
+}
+
 static void amdgpu_userq_init_hang_detect_work(struct amdgpu_usermode_queue *queue)
 {
 	INIT_DELAYED_WORK(&queue->hang_detect_work, amdgpu_userq_hang_detect_work);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index cb92789c1ed1..843ea8ecc5d7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -168,6 +168,7 @@ void amdgpu_userq_reset_work(struct work_struct *work);
 void amdgpu_userq_pre_reset(struct amdgpu_device *adev);
 int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost);
 void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue);
+void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell);
 
 int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
 				   struct amdgpu_usermode_queue *queue,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
index 837d98947958..1ffbb5450f3a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
@@ -6528,15 +6528,7 @@ static int gfx_v11_0_eop_irq(struct amdgpu_device *adev,
 	DRM_DEBUG("IH: CP EOP\n");
 
 	if (adev->enable_mes && doorbell_offset) {
-		struct amdgpu_usermode_queue *queue;
-		struct xarray *xa = &adev->userq_doorbell_xa;
-		unsigned long flags;
-
-		xa_lock_irqsave(xa, flags);
-		queue = xa_load(xa, doorbell_offset);
-		if (queue)
-			amdgpu_userq_fence_driver_process(queue->fence_drv);
-		xa_unlock_irqrestore(xa, flags);
+		amdgpu_userq_process_fence_irq(adev, doorbell_offset);
 	} else {
 		me_id = (entry->ring_id & 0x0c) >> 2;
 		pipe_id = (entry->ring_id & 0x03) >> 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
index 60f0c7d6a7a3..6baac533a2e6 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
@@ -4859,15 +4859,7 @@ static int gfx_v12_0_eop_irq(struct amdgpu_device *adev,
 	DRM_DEBUG("IH: CP EOP\n");
 
 	if (adev->enable_mes && doorbell_offset) {
-		struct xarray *xa = &adev->userq_doorbell_xa;
-		struct amdgpu_usermode_queue *queue;
-		unsigned long flags;
-
-		xa_lock_irqsave(xa, flags);
-		queue = xa_load(xa, doorbell_offset);
-		if (queue)
-			amdgpu_userq_fence_driver_process(queue->fence_drv);
-		xa_unlock_irqrestore(xa, flags);
+		amdgpu_userq_process_fence_irq(adev, doorbell_offset);
 	} else {
 		me_id = (entry->ring_id & 0x0c) >> 2;
 		pipe_id = (entry->ring_id & 0x03) >> 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
index 948758b51b5c..ae65412109c2 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
@@ -3654,16 +3654,7 @@ static int gfx_v12_1_eop_irq(struct amdgpu_device *adev,
 	DRM_DEBUG("IH: CP EOP\n");
 
 	if (adev->enable_mes && doorbell_offset) {
-		struct xarray *xa = &adev->userq_doorbell_xa;
-		struct amdgpu_usermode_queue *queue;
-		unsigned long flags;
-
-		xa_lock_irqsave(xa, flags);
-		queue = xa_load(xa, doorbell_offset);
-		if (queue)
-			amdgpu_userq_fence_driver_process(queue->fence_drv);
-
-		xa_unlock_irqrestore(xa, flags);
+		amdgpu_userq_process_fence_irq(adev, doorbell_offset);
 	} else {
 		me_id = (entry->ring_id & 0x0c) >> 2;
 		pipe_id = (entry->ring_id & 0x03) >> 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
index de329b76a00c..bf09ac841a68 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
@@ -1662,17 +1662,8 @@ static int sdma_v6_0_process_fence_irq(struct amdgpu_device *adev,
 	u32 doorbell_offset = entry->src_data[0];
 
 	if (adev->enable_mes && doorbell_offset) {
-		struct amdgpu_usermode_queue *queue;
-		struct xarray *xa = &adev->userq_doorbell_xa;
-		unsigned long flags;
-
 		doorbell_offset >>= SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
-
-		xa_lock_irqsave(xa, flags);
-		queue = xa_load(xa, doorbell_offset);
-		if (queue)
-			amdgpu_userq_fence_driver_process(queue->fence_drv);
-		xa_unlock_irqrestore(xa, flags);
+		amdgpu_userq_process_fence_irq(adev, doorbell_offset);
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
index 85d98a0e1bff..f154b68dda70 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
@@ -1594,17 +1594,8 @@ static int sdma_v7_0_process_fence_irq(struct amdgpu_device *adev,
 	u32 doorbell_offset = entry->src_data[0];
 
 	if (adev->enable_mes && doorbell_offset) {
-		struct xarray *xa = &adev->userq_doorbell_xa;
-		struct amdgpu_usermode_queue *queue;
-		unsigned long flags;
-
 		doorbell_offset >>= SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
-
-		xa_lock_irqsave(xa, flags);
-		queue = xa_load(xa, doorbell_offset);
-		if (queue)
-			amdgpu_userq_fence_driver_process(queue->fence_drv);
-		xa_unlock_irqrestore(xa, flags);
+		amdgpu_userq_process_fence_irq(adev, doorbell_offset);
 	}
 
 	return 0;
-- 
2.43.0

[PATCH 06/11] drm/amdgpu: remove almost all calls to amdgpu_userq_detect_and_reset_queues

[Christian König]

Well the reset handling seems broken on multiple levels.

As first step of fixing this remove most calls to the hang detection.
That function should only be called after we run into a timeout! And *NOT*
as random check spread over the code in multiple places.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 38 +++++++++--------------
 1 file changed, 14 insertions(+), 24 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 8ce001481d42..5ccd53ad8efd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -345,23 +345,18 @@ static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue)
 	struct amdgpu_device *adev = uq_mgr->adev;
 	const struct amdgpu_userq_funcs *userq_funcs =
 		adev->userq_funcs[queue->queue_type];
-	bool found_hung_queue = false;
-	int r = 0;
+	int r;
 
 	if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
 		r = userq_funcs->preempt(queue);
 		if (r) {
 			queue->state = AMDGPU_USERQ_STATE_HUNG;
-			found_hung_queue = true;
+			return r;
 		} else {
 			queue->state = AMDGPU_USERQ_STATE_PREEMPTED;
 		}
 	}
-
-	if (found_hung_queue)
-		amdgpu_userq_detect_and_reset_queues(uq_mgr);
-
-	return r;
+	return 0;
 }
 
 static int amdgpu_userq_restore_helper(struct amdgpu_usermode_queue *queue)
@@ -390,24 +385,21 @@ static int amdgpu_userq_unmap_helper(struct amdgpu_usermode_queue *queue)
 	struct amdgpu_device *adev = uq_mgr->adev;
 	const struct amdgpu_userq_funcs *userq_funcs =
 		adev->userq_funcs[queue->queue_type];
-	bool found_hung_queue = false;
-	int r = 0;
+	int r;
 
 	if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) ||
-		(queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {
+	    (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {
+
 		r = userq_funcs->unmap(queue);
 		if (r) {
 			queue->state = AMDGPU_USERQ_STATE_HUNG;
-			found_hung_queue = true;
+			return r;
 		} else {
 			queue->state = AMDGPU_USERQ_STATE_UNMAPPED;
 		}
 	}
 
-	if (found_hung_queue)
-		amdgpu_userq_detect_and_reset_queues(uq_mgr);
-
-	return r;
+	return 0;
 }
 
 static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue)
@@ -416,19 +408,19 @@ static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue)
 	struct amdgpu_device *adev = uq_mgr->adev;
 	const struct amdgpu_userq_funcs *userq_funcs =
 		adev->userq_funcs[queue->queue_type];
-	int r = 0;
+	int r;
 
 	if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) {
 		r = userq_funcs->map(queue);
 		if (r) {
 			queue->state = AMDGPU_USERQ_STATE_HUNG;
-			amdgpu_userq_detect_and_reset_queues(uq_mgr);
+			return r;
 		} else {
 			queue->state = AMDGPU_USERQ_STATE_MAPPED;
 		}
 	}
 
-	return r;
+	return 0;
 }
 
 static void amdgpu_userq_wait_for_last_fence(struct amdgpu_usermode_queue *queue)
@@ -654,7 +646,6 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_que
 #if defined(CONFIG_DEBUG_FS)
 	debugfs_remove_recursive(queue->debugfs_queue);
 #endif
-	amdgpu_userq_detect_and_reset_queues(uq_mgr);
 	r = amdgpu_userq_unmap_helper(queue);
 	/*TODO: It requires a reset for userq hw unmap error*/
 	if (r) {
@@ -1268,7 +1259,6 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
 	unsigned long queue_id;
 	int ret = 0, r;
 
-	amdgpu_userq_detect_and_reset_queues(uq_mgr);
 	/* Try to unmap all the queues in this process ctx */
 	xa_for_each(&uq_mgr->userq_xa, queue_id, queue) {
 		r = amdgpu_userq_preempt_helper(queue);
@@ -1276,9 +1266,11 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
 			ret = r;
 	}
 
-	if (ret)
+	if (ret) {
 		drm_file_err(uq_mgr->file,
 			     "Couldn't unmap all the queues, eviction failed ret=%d\n", ret);
+		amdgpu_userq_detect_and_reset_queues(uq_mgr);
+	}
 	return ret;
 }
 
@@ -1378,7 +1370,6 @@ int amdgpu_userq_suspend(struct amdgpu_device *adev)
 		uqm = queue->userq_mgr;
 		cancel_delayed_work_sync(&uqm->resume_work);
 		guard(mutex)(&uqm->userq_mutex);
-		amdgpu_userq_detect_and_reset_queues(uqm);
 		if (adev->in_s0ix)
 			r = amdgpu_userq_preempt_helper(queue);
 		else
@@ -1437,7 +1428,6 @@ int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev,
 		if (((queue->queue_type == AMDGPU_HW_IP_GFX) ||
 		     (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) &&
 		    (queue->xcp_id == idx)) {
-			amdgpu_userq_detect_and_reset_queues(uqm);
 			r = amdgpu_userq_preempt_helper(queue);
 			if (r)
 				ret = r;
-- 
2.43.0

[PATCH 07/11] drm/amdgpu: fix userq hang detection and reset

[Christian König]

Fix lock inversions pointed out by Prike and Sunil. The hang detection
timeout *CAN'T* grab locks under which we wait for fences, especially
not the userq_mutex lock.

Then instead of this completely broken handling with the
hang_detect_fence just cancel the work when fences are processed and
re-start if necessary.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c     | 65 ++++++++-----------
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h     |  1 -
 .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 17 +++--
 .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.h   |  2 +-
 4 files changed, 40 insertions(+), 45 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 5ccd53ad8efd..0a4c39d83adc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -106,9 +106,6 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
 	int r = 0;
 	int i;
 
-	/* Warning if current process mutex is not held */
-	WARN_ON(!mutex_is_locked(&uq_mgr->userq_mutex));
-
 	if (unlikely(adev->debug_disable_gpu_ring_reset)) {
 		dev_err(adev->dev, "userq reset disabled by debug mask\n");
 		return 0;
@@ -127,9 +124,11 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
 	 */
 	for (i = 0; i < num_queue_types; i++) {
 		int ring_type = queue_types[i];
-		const struct amdgpu_userq_funcs *funcs = adev->userq_funcs[ring_type];
+		const struct amdgpu_userq_funcs *funcs =
+			adev->userq_funcs[ring_type];
 
-		if (!amdgpu_userq_is_reset_type_supported(adev, ring_type, AMDGPU_RESET_TYPE_PER_QUEUE))
+		if (!amdgpu_userq_is_reset_type_supported(adev, ring_type,
+							  AMDGPU_RESET_TYPE_PER_QUEUE))
 				continue;
 
 		if (atomic_read(&uq_mgr->userq_count[ring_type]) > 0 &&
@@ -150,38 +149,22 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
 
 static void amdgpu_userq_hang_detect_work(struct work_struct *work)
 {
-	struct amdgpu_usermode_queue *queue = container_of(work,
-							  struct amdgpu_usermode_queue,
-							  hang_detect_work.work);
-	struct dma_fence *fence;
-	struct amdgpu_userq_mgr *uq_mgr;
-
-	if (!queue->userq_mgr)
-		return;
-
-	uq_mgr = queue->userq_mgr;
-	fence = READ_ONCE(queue->hang_detect_fence);
-	/* Fence already signaled – no action needed */
-	if (!fence || dma_fence_is_signaled(fence))
-		return;
+	struct amdgpu_usermode_queue *queue =
+		container_of(work, struct amdgpu_usermode_queue,
+			     hang_detect_work.work);
 
-	mutex_lock(&uq_mgr->userq_mutex);
-	amdgpu_userq_detect_and_reset_queues(uq_mgr);
-	mutex_unlock(&uq_mgr->userq_mutex);
+	amdgpu_userq_detect_and_reset_queues(queue->userq_mgr);
 }
 
 /*
  * Start hang detection for a user queue fence. A delayed work will be scheduled
- * to check if the fence is still pending after the timeout period.
-*/
+ * to reset the queues when the fence doesn't signal in time.
+ */
 void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
 {
 	struct amdgpu_device *adev;
 	unsigned long timeout_ms;
 
-	if (!queue || !queue->userq_mgr || !queue->userq_mgr->adev)
-		return;
-
 	adev = queue->userq_mgr->adev;
 	/* Determine timeout based on queue type */
 	switch (queue->queue_type) {
@@ -199,8 +182,6 @@ void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
 		break;
 	}
 
-	/* Store the fence to monitor and schedule hang detection */
-	WRITE_ONCE(queue->hang_detect_fence, queue->last_fence);
 	schedule_delayed_work(&queue->hang_detect_work,
 		     msecs_to_jiffies(timeout_ms));
 }
@@ -210,18 +191,24 @@ void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell)
 	struct xarray *xa = &adev->userq_doorbell_xa;
 	struct amdgpu_usermode_queue *queue;
 	unsigned long flags;
+	int r;
 
 	xa_lock_irqsave(xa, flags);
 	queue = xa_load(xa, doorbell);
-	if (queue)
-		amdgpu_userq_fence_driver_process(queue->fence_drv);
-	xa_unlock_irqrestore(xa, flags);
-}
+	if (queue) {
+		r = amdgpu_userq_fence_driver_process(queue->fence_drv);
+		/*
+		 * We are in interrupt context here, this *can't* wait for
+		 * reset work to finish.
+		 */
+		if (r >= 0)
+			cancel_delayed_work(&queue->hang_detect_work);
 
-static void amdgpu_userq_init_hang_detect_work(struct amdgpu_usermode_queue *queue)
-{
-	INIT_DELAYED_WORK(&queue->hang_detect_work, amdgpu_userq_hang_detect_work);
-	queue->hang_detect_fence = NULL;
+		/* Restart the timer when there are still fences pending */
+		if (r == 1)
+			amdgpu_userq_start_hang_detect_work(queue);
+	}
+	xa_unlock_irqrestore(xa, flags);
 }
 
 static int amdgpu_userq_buffer_va_list_add(struct amdgpu_usermode_queue *queue,
@@ -640,7 +627,6 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_que
 	amdgpu_bo_unreserve(vm->root.bo);
 
 	mutex_lock(&uq_mgr->userq_mutex);
-	queue->hang_detect_fence = NULL;
 	amdgpu_userq_wait_for_last_fence(queue);
 
 #if defined(CONFIG_DEBUG_FS)
@@ -853,7 +839,8 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 	up_read(&adev->reset_domain->sem);
 
 	amdgpu_debugfs_userq_init(filp, queue, qid);
-	amdgpu_userq_init_hang_detect_work(queue);
+	INIT_DELAYED_WORK(&queue->hang_detect_work,
+			  amdgpu_userq_hang_detect_work);
 
 	args->out.queue_id = qid;
 	atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index 843ea8ecc5d7..85f460e7c31b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -85,7 +85,6 @@ struct amdgpu_usermode_queue {
 	int			priority;
 	struct dentry		*debugfs_queue;
 	struct delayed_work hang_detect_work;
-	struct dma_fence *hang_detect_fence;
 	struct kref		refcount;
 
 	struct list_head	userq_va_list;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
index b0543fa257ed..beb2a1f679b8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
@@ -141,7 +141,14 @@ amdgpu_userq_fence_put_fence_drv_refs(struct amdgpu_userq_fence *userq_fence)
 	userq_fence->fence_drv = NULL;
 }
 
-void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
+/*
+ * Returns:
+ * -ENOENT when no fences were processes
+ * 1 when more fences are pending
+ * 0 when no fences are pending any more
+ */
+int
+amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
 {
 	struct amdgpu_userq_fence *userq_fence, *tmp;
 	LIST_HEAD(to_be_signaled);
@@ -149,9 +156,6 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
 	unsigned long flags;
 	u64 rptr;
 
-	if (!fence_drv)
-		return;
-
 	spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
 	rptr = amdgpu_userq_fence_read(fence_drv);
 
@@ -164,6 +168,9 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
 				&userq_fence->link);
 	spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
 
+	if (list_empty(&to_be_signaled))
+		return -ENOENT;
+
 	list_for_each_entry_safe(userq_fence, tmp, &to_be_signaled, link) {
 		fence = &userq_fence->base;
 		list_del_init(&userq_fence->link);
@@ -176,6 +183,8 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
 		dma_fence_put(fence);
 	}
 
+	/* That doesn't need to be accurate so no locking */
+	return list_empty(&fence_drv->fences) ? 0 : 1;
 }
 
 void amdgpu_userq_fence_driver_destroy(struct kref *ref)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
index d355a0eecc07..0bd51616cef1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
@@ -63,7 +63,7 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv);
 int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,
 				    struct amdgpu_userq_fence_driver **fence_drv_req);
 void amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq);
-void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv);
+int amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv);
 void amdgpu_userq_fence_driver_force_completion(struct amdgpu_usermode_queue *userq);
 void amdgpu_userq_fence_driver_destroy(struct kref *ref);
 int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
-- 
2.43.0

[PATCH 08/11] drm/amdgpu: rework userq reset work handling

[Christian König]

It is illegal to schedule reset work from another reset work!

Fix this by scheduling the userq reset work directly on the work queue
of the reset domain.

Not fully tested, I leave that to the IGT test cases.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h        |  1 -
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  3 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c  | 84 +++++++++++-----------
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h  | 16 ++++-
 4 files changed, 60 insertions(+), 44 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 39894e38fee4..17341e384caf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -1191,7 +1191,6 @@ struct amdgpu_device {
 	bool                            apu_prefer_gtt;
 
 	bool                            userq_halt_for_enforce_isolation;
-	struct work_struct              userq_reset_work;
 	struct amdgpu_uid *uid_info;
 
 	struct amdgpu_uma_carveout_info uma_info;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index b11c4b5fa8fc..cf61be17e061 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -3786,7 +3786,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	}
 
 	INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
-	INIT_WORK(&adev->userq_reset_work, amdgpu_userq_reset_work);
 
 	amdgpu_coredump_init(adev);
 
@@ -5477,7 +5476,7 @@ static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
 	if (!amdgpu_sriov_vf(adev))
 		cancel_work(&adev->reset_work);
 #endif
-	cancel_work(&adev->userq_reset_work);
+	amdgpu_userq_mgr_cancel_reset_work(adev);
 
 	if (adev->kfd.dev)
 		cancel_work(&adev->kfd.reset_work);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 0a4c39d83adc..ad6dac17dd21 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -82,19 +82,11 @@ static bool amdgpu_userq_is_reset_type_supported(struct amdgpu_device *adev,
 	return false;
 }
 
-static void amdgpu_userq_gpu_reset(struct amdgpu_device *adev)
-{
-	if (amdgpu_device_should_recover_gpu(adev)) {
-		amdgpu_reset_domain_schedule(adev->reset_domain,
-					     &adev->userq_reset_work);
-		/* Wait for the reset job to complete */
-		flush_work(&adev->userq_reset_work);
-	}
-}
-
-static int
-amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
+static void amdgpu_userq_mgr_reset_work(struct work_struct *work)
 {
+	struct amdgpu_userq_mgr *uq_mgr =
+		container_of(work, struct amdgpu_userq_mgr,
+			     reset_work);
 	struct amdgpu_device *adev = uq_mgr->adev;
 	const int queue_types[] = {
 		AMDGPU_RING_TYPE_COMPUTE,
@@ -103,12 +95,11 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
 	};
 	const int num_queue_types = ARRAY_SIZE(queue_types);
 	bool gpu_reset = false;
-	int r = 0;
-	int i;
+	int i, r;
 
 	if (unlikely(adev->debug_disable_gpu_ring_reset)) {
 		dev_err(adev->dev, "userq reset disabled by debug mask\n");
-		return 0;
+		return;
 	}
 
 	/*
@@ -116,7 +107,7 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
 	 * skip all reset detection logic
 	 */
 	if (!amdgpu_gpu_recovery)
-		return 0;
+		return;
 
 	/*
 	 * Iterate through all queue types to detect and reset problematic queues
@@ -141,10 +132,19 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
 		}
 	}
 
-	if (gpu_reset)
-		amdgpu_userq_gpu_reset(adev);
+	if (gpu_reset) {
+		struct amdgpu_reset_context reset_context;
 
-	return r;
+		memset(&reset_context, 0, sizeof(reset_context));
+
+		reset_context.method = AMD_RESET_METHOD_NONE;
+		reset_context.reset_req_dev = adev;
+		reset_context.src = AMDGPU_RESET_SRC_USERQ;
+		set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
+		/*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/
+
+		amdgpu_device_gpu_recover(adev, NULL, &reset_context);
+	}
 }
 
 static void amdgpu_userq_hang_detect_work(struct work_struct *work)
@@ -153,7 +153,11 @@ static void amdgpu_userq_hang_detect_work(struct work_struct *work)
 		container_of(work, struct amdgpu_usermode_queue,
 			     hang_detect_work.work);
 
-	amdgpu_userq_detect_and_reset_queues(queue->userq_mgr);
+	/*
+	 * Don't schedule the work here! Scheduling or queue work from one reset
+	 * handler to another is illegal if you don't take extra precautions!
+	 */
+	amdgpu_userq_mgr_reset_work(&queue->userq_mgr->reset_work);
 }
 
 /*
@@ -182,8 +186,8 @@ void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
 		break;
 	}
 
-	schedule_delayed_work(&queue->hang_detect_work,
-		     msecs_to_jiffies(timeout_ms));
+	queue_delayed_work(adev->reset_domain->wq, &queue->hang_detect_work,
+			   msecs_to_jiffies(timeout_ms));
 }
 
 void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell)
@@ -1256,28 +1260,13 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
 	if (ret) {
 		drm_file_err(uq_mgr->file,
 			     "Couldn't unmap all the queues, eviction failed ret=%d\n", ret);
-		amdgpu_userq_detect_and_reset_queues(uq_mgr);
+		amdgpu_reset_domain_schedule(uq_mgr->adev->reset_domain,
+					     &uq_mgr->reset_work);
+		flush_work(&uq_mgr->reset_work);
 	}
 	return ret;
 }
 
-void amdgpu_userq_reset_work(struct work_struct *work)
-{
-	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
-						  userq_reset_work);
-	struct amdgpu_reset_context reset_context;
-
-	memset(&reset_context, 0, sizeof(reset_context));
-
-	reset_context.method = AMD_RESET_METHOD_NONE;
-	reset_context.reset_req_dev = adev;
-	reset_context.src = AMDGPU_RESET_SRC_USERQ;
-	set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
-	/*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/
-
-	amdgpu_device_gpu_recover(adev, NULL, &reset_context);
-}
-
 static void
 amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr)
 {
@@ -1311,9 +1300,24 @@ int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *f
 	userq_mgr->file = file_priv;
 
 	INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userq_restore_worker);
+	INIT_WORK(&userq_mgr->reset_work, amdgpu_userq_mgr_reset_work);
 	return 0;
 }
 
+void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev)
+{
+	struct xarray *xa = &adev->userq_doorbell_xa;
+	struct amdgpu_usermode_queue *queue;
+	unsigned long flags, queue_id;
+
+	xa_lock_irqsave(xa, flags);
+	xa_for_each(xa, queue_id, queue) {
+		cancel_delayed_work(&queue->hang_detect_work);
+		cancel_work(&queue->userq_mgr->reset_work);
+	}
+	xa_unlock_irqrestore(xa, flags);
+}
+
 void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr)
 {
 	cancel_delayed_work_sync(&userq_mgr->resume_work);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index 85f460e7c31b..49b33e2d6932 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -84,7 +84,13 @@ struct amdgpu_usermode_queue {
 	u32			xcp_id;
 	int			priority;
 	struct dentry		*debugfs_queue;
-	struct delayed_work hang_detect_work;
+
+	/**
+	 * @hang_detect_work:
+	 *
+	 * Delayed work which runs when userq_fences time out.
+	 */
+	struct delayed_work	hang_detect_work;
 	struct kref		refcount;
 
 	struct list_head	userq_va_list;
@@ -116,6 +122,13 @@ struct amdgpu_userq_mgr {
 	struct amdgpu_device		*adev;
 	struct delayed_work		resume_work;
 	struct drm_file			*file;
+
+	/**
+	 * @reset_work:
+	 *
+	 * Reset work which is used when eviction fails.
+	 */
+	struct work_struct		reset_work;
 	atomic_t                        userq_count[AMDGPU_RING_TYPE_MAX];
 };
 
@@ -134,6 +147,7 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp
 int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,
 			  struct amdgpu_device *adev);
 
+void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev);
 void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr);
 void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr);
 
-- 
2.43.0

[PATCH 09/11] drm/amdgpu: revert to old status lock handling v4

[Christian König]

It turned out that protecting the status of each bo_va with a
spinlock was just hiding problems instead of solving them.

Revert the whole approach, add a separate stats_lock and lockdep
assertions that the correct reservation lock is held all over the place.

This not only allows for better checks if a state transition is properly
protected by a lock, but also switching back to using list macros to
iterate over the state of lists protected by the dma_resv lock of the
root PD.

v2: re-add missing check
v3: split into two patches
v4: re-apply by fixing holding the VM lock at the right places.

Signed-off-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Alex Deucher <alexander.deucher@amd.com>
Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c |   8 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 146 ++++++++--------------
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h    |  15 ++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c |   4 -
 4 files changed, 68 insertions(+), 105 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index ad6dac17dd21..7fc733ba962e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -1048,12 +1048,12 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
 	struct amdgpu_bo *bo;
 	int ret;
 
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->invalidated_lock);
 	while (!list_empty(&vm->invalidated)) {
 		bo_va = list_first_entry(&vm->invalidated,
 					 struct amdgpu_bo_va,
 					 base.vm_status);
-		spin_unlock(&vm->status_lock);
+		spin_unlock(&vm->invalidated_lock);
 
 		bo = bo_va->base.bo;
 		ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2);
@@ -1070,9 +1070,9 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
 		if (ret)
 			return ret;
 
-		spin_lock(&vm->status_lock);
+		spin_lock(&vm->invalidated_lock);
 	}
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->invalidated_lock);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 63156289ae7f..e2a21a66b28f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -167,12 +167,10 @@ static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
 
 	vm_bo->moved = true;
 	amdgpu_vm_assert_locked(vm);
-	spin_lock(&vm_bo->vm->status_lock);
 	if (bo->tbo.type == ttm_bo_type_kernel)
 		list_move(&vm_bo->vm_status, &vm->evicted);
 	else
 		list_move_tail(&vm_bo->vm_status, &vm->evicted);
-	spin_unlock(&vm_bo->vm->status_lock);
 }
 /**
  * amdgpu_vm_bo_moved - vm_bo is moved
@@ -185,9 +183,7 @@ static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
 static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
 {
 	amdgpu_vm_assert_locked(vm_bo->vm);
-	spin_lock(&vm_bo->vm->status_lock);
 	list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
-	spin_unlock(&vm_bo->vm->status_lock);
 }
 
 /**
@@ -201,9 +197,7 @@ static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
 static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
 {
 	amdgpu_vm_assert_locked(vm_bo->vm);
-	spin_lock(&vm_bo->vm->status_lock);
 	list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
-	spin_unlock(&vm_bo->vm->status_lock);
 	vm_bo->moved = false;
 }
 
@@ -217,9 +211,9 @@ static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
  */
 static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
 {
-	spin_lock(&vm_bo->vm->status_lock);
+	spin_lock(&vm_bo->vm->invalidated_lock);
 	list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
-	spin_unlock(&vm_bo->vm->status_lock);
+	spin_unlock(&vm_bo->vm->invalidated_lock);
 }
 
 /**
@@ -232,10 +226,9 @@ static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
  */
 static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)
 {
+	amdgpu_vm_assert_locked(vm_bo->vm);
 	vm_bo->moved = true;
-	spin_lock(&vm_bo->vm->status_lock);
 	list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user);
-	spin_unlock(&vm_bo->vm->status_lock);
 }
 
 /**
@@ -249,13 +242,10 @@ static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)
 static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
 {
 	amdgpu_vm_assert_locked(vm_bo->vm);
-	if (vm_bo->bo->parent) {
-		spin_lock(&vm_bo->vm->status_lock);
+	if (vm_bo->bo->parent)
 		list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
-		spin_unlock(&vm_bo->vm->status_lock);
-	} else {
+	else
 		amdgpu_vm_bo_idle(vm_bo);
-	}
 }
 
 /**
@@ -269,9 +259,7 @@ static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
 static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
 {
 	amdgpu_vm_assert_locked(vm_bo->vm);
-	spin_lock(&vm_bo->vm->status_lock);
 	list_move(&vm_bo->vm_status, &vm_bo->vm->done);
-	spin_unlock(&vm_bo->vm->status_lock);
 }
 
 /**
@@ -285,13 +273,13 @@ static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
 {
 	struct amdgpu_vm_bo_base *vm_bo, *tmp;
 
-	amdgpu_vm_assert_locked(vm);
-
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->invalidated_lock);
 	list_splice_init(&vm->done, &vm->invalidated);
 	list_for_each_entry(vm_bo, &vm->invalidated, vm_status)
 		vm_bo->moved = true;
+	spin_unlock(&vm->invalidated_lock);
 
+	amdgpu_vm_assert_locked(vm);
 	list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) {
 		struct amdgpu_bo *bo = vm_bo->bo;
 
@@ -301,14 +289,13 @@ static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
 		else if (bo->parent)
 			list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
 	}
-	spin_unlock(&vm->status_lock);
 }
 
 /**
  * amdgpu_vm_update_shared - helper to update shared memory stat
  * @base: base structure for tracking BO usage in a VM
  *
- * Takes the vm status_lock and updates the shared memory stat. If the basic
+ * Takes the vm stats_lock and updates the shared memory stat. If the basic
  * stat changed (e.g. buffer was moved) amdgpu_vm_update_stats need to be called
  * as well.
  */
@@ -321,7 +308,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base)
 	bool shared;
 
 	dma_resv_assert_held(bo->tbo.base.resv);
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->stats_lock);
 	shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
 	if (base->shared != shared) {
 		base->shared = shared;
@@ -333,7 +320,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base)
 			vm->stats[bo_memtype].drm.private += size;
 		}
 	}
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->stats_lock);
 }
 
 /**
@@ -358,11 +345,11 @@ void amdgpu_vm_bo_update_shared(struct amdgpu_bo *bo)
  *        be bo->tbo.resource
  * @sign: if we should add (+1) or subtract (-1) from the stat
  *
- * Caller need to have the vm status_lock held. Useful for when multiple update
+ * Caller need to have the vm stats_lock held. Useful for when multiple update
  * need to happen at the same time.
  */
 static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
-			    struct ttm_resource *res, int sign)
+					  struct ttm_resource *res, int sign)
 {
 	struct amdgpu_vm *vm = base->vm;
 	struct amdgpu_bo *bo = base->bo;
@@ -386,7 +373,8 @@ static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
 		 */
 		if (bo->flags & AMDGPU_GEM_CREATE_DISCARDABLE)
 			vm->stats[res_memtype].drm.purgeable += size;
-		if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(res_memtype)))
+		if (!(bo->preferred_domains &
+		      amdgpu_mem_type_to_domain(res_memtype)))
 			vm->stats[bo_memtype].evicted += size;
 	}
 }
@@ -405,9 +393,9 @@ void amdgpu_vm_update_stats(struct amdgpu_vm_bo_base *base,
 {
 	struct amdgpu_vm *vm = base->vm;
 
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->stats_lock);
 	amdgpu_vm_update_stats_locked(base, res, sign);
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->stats_lock);
 }
 
 /**
@@ -433,10 +421,10 @@ void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
 	base->next = bo->vm_bo;
 	bo->vm_bo = base;
 
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->stats_lock);
 	base->shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
 	amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1);
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->stats_lock);
 
 	if (!amdgpu_vm_is_bo_always_valid(vm, bo))
 		return;
@@ -495,25 +483,25 @@ int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
 	int ret;
 
 	/* We can only trust prev->next while holding the lock */
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->invalidated_lock);
 	while (!list_is_head(prev->next, &vm->done)) {
 		bo_va = list_entry(prev->next, typeof(*bo_va), base.vm_status);
 
 		bo = bo_va->base.bo;
 		if (bo) {
 			amdgpu_bo_ref(bo);
-			spin_unlock(&vm->status_lock);
+			spin_unlock(&vm->invalidated_lock);
 
 			ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 1);
 			amdgpu_bo_unref(&bo);
 			if (unlikely(ret))
 				return ret;
 
-			spin_lock(&vm->status_lock);
+			spin_lock(&vm->invalidated_lock);
 		}
 		prev = prev->next;
 	}
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->invalidated_lock);
 
 	return 0;
 }
@@ -609,7 +597,7 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		       void *param)
 {
 	uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm);
-	struct amdgpu_vm_bo_base *bo_base;
+	struct amdgpu_vm_bo_base *bo_base, *tmp;
 	struct amdgpu_bo *bo;
 	int r;
 
@@ -622,13 +610,7 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			return r;
 	}
 
-	spin_lock(&vm->status_lock);
-	while (!list_empty(&vm->evicted)) {
-		bo_base = list_first_entry(&vm->evicted,
-					   struct amdgpu_vm_bo_base,
-					   vm_status);
-		spin_unlock(&vm->status_lock);
-
+	list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {
 		bo = bo_base->bo;
 
 		r = validate(param, bo);
@@ -641,26 +623,21 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
 			amdgpu_vm_bo_relocated(bo_base);
 		}
-		spin_lock(&vm->status_lock);
 	}
-	while (ticket && !list_empty(&vm->evicted_user)) {
-		bo_base = list_first_entry(&vm->evicted_user,
-					   struct amdgpu_vm_bo_base,
-					   vm_status);
-		spin_unlock(&vm->status_lock);
 
-		bo = bo_base->bo;
-		dma_resv_assert_held(bo->tbo.base.resv);
+	if (ticket) {
+		list_for_each_entry_safe(bo_base, tmp, &vm->evicted_user,
+					 vm_status) {
+			bo = bo_base->bo;
+			dma_resv_assert_held(bo->tbo.base.resv);
 
-		r = validate(param, bo);
-		if (r)
-			return r;
-
-		amdgpu_vm_bo_invalidated(bo_base);
+			r = validate(param, bo);
+			if (r)
+				return r;
 
-		spin_lock(&vm->status_lock);
+			amdgpu_vm_bo_invalidated(bo_base);
+		}
 	}
-	spin_unlock(&vm->status_lock);
 
 	amdgpu_vm_eviction_lock(vm);
 	vm->evicting = false;
@@ -689,9 +666,7 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 	ret = !vm->evicting;
 	amdgpu_vm_eviction_unlock(vm);
 
-	spin_lock(&vm->status_lock);
 	ret &= list_empty(&vm->evicted);
-	spin_unlock(&vm->status_lock);
 
 	spin_lock(&vm->immediate.lock);
 	ret &= !vm->immediate.stopped;
@@ -985,18 +960,13 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 			  struct amdgpu_vm *vm, bool immediate)
 {
 	struct amdgpu_vm_update_params params;
-	struct amdgpu_vm_bo_base *entry;
+	struct amdgpu_vm_bo_base *entry, *tmp;
 	bool flush_tlb_needed = false;
-	LIST_HEAD(relocated);
 	int r, idx;
 
 	amdgpu_vm_assert_locked(vm);
 
-	spin_lock(&vm->status_lock);
-	list_splice_init(&vm->relocated, &relocated);
-	spin_unlock(&vm->status_lock);
-
-	if (list_empty(&relocated))
+	if (list_empty(&vm->relocated))
 		return 0;
 
 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
@@ -1012,7 +982,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 	if (r)
 		goto error;
 
-	list_for_each_entry(entry, &relocated, vm_status) {
+	list_for_each_entry(entry, &vm->relocated, vm_status) {
 		/* vm_flush_needed after updating moved PDEs */
 		flush_tlb_needed |= entry->moved;
 
@@ -1028,9 +998,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 	if (flush_tlb_needed)
 		atomic64_inc(&vm->tlb_seq);
 
-	while (!list_empty(&relocated)) {
-		entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base,
-					 vm_status);
+	list_for_each_entry_safe(entry, tmp, &vm->relocated, vm_status) {
 		amdgpu_vm_bo_idle(entry);
 	}
 
@@ -1260,9 +1228,9 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
 			  struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
 {
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->stats_lock);
 	memcpy(stats, vm->stats, sizeof(*stats) * __AMDGPU_PL_NUM);
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->stats_lock);
 }
 
 /**
@@ -1629,29 +1597,24 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
 			   struct amdgpu_vm *vm,
 			   struct ww_acquire_ctx *ticket)
 {
-	struct amdgpu_bo_va *bo_va;
+	struct amdgpu_bo_va *bo_va, *tmp;
 	struct dma_resv *resv;
 	bool clear, unlock;
 	int r;
 
-	spin_lock(&vm->status_lock);
-	while (!list_empty(&vm->moved)) {
-		bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va,
-					 base.vm_status);
-		spin_unlock(&vm->status_lock);
-
+	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
 		/* Per VM BOs never need to bo cleared in the page tables */
 		r = amdgpu_vm_bo_update(adev, bo_va, false);
 		if (r)
 			return r;
-		spin_lock(&vm->status_lock);
 	}
 
+	spin_lock(&vm->invalidated_lock);
 	while (!list_empty(&vm->invalidated)) {
 		bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
 					 base.vm_status);
 		resv = bo_va->base.bo->tbo.base.resv;
-		spin_unlock(&vm->status_lock);
+		spin_unlock(&vm->invalidated_lock);
 
 		/* Try to reserve the BO to avoid clearing its ptes */
 		if (!adev->debug_vm && dma_resv_trylock(resv)) {
@@ -1683,9 +1646,9 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
 		     bo_va->base.bo->tbo.resource->mem_type == TTM_PL_SYSTEM))
 			amdgpu_vm_bo_evicted_user(&bo_va->base);
 
-		spin_lock(&vm->status_lock);
+		spin_lock(&vm->invalidated_lock);
 	}
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->invalidated_lock);
 
 	return 0;
 }
@@ -2223,9 +2186,9 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev,
 		}
 	}
 
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->invalidated_lock);
 	list_del(&bo_va->base.vm_status);
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->invalidated_lock);
 
 	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
 		list_del(&mapping->list);
@@ -2333,10 +2296,10 @@ void amdgpu_vm_bo_move(struct amdgpu_bo *bo, struct ttm_resource *new_mem,
 	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
 		struct amdgpu_vm *vm = bo_base->vm;
 
-		spin_lock(&vm->status_lock);
+		spin_lock(&vm->stats_lock);
 		amdgpu_vm_update_stats_locked(bo_base, bo->tbo.resource, -1);
 		amdgpu_vm_update_stats_locked(bo_base, new_mem, +1);
-		spin_unlock(&vm->status_lock);
+		spin_unlock(&vm->stats_lock);
 	}
 
 	amdgpu_vm_bo_invalidate(bo, evicted);
@@ -2608,11 +2571,12 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	INIT_LIST_HEAD(&vm->relocated);
 	INIT_LIST_HEAD(&vm->moved);
 	INIT_LIST_HEAD(&vm->idle);
+	spin_lock_init(&vm->invalidated_lock);
 	INIT_LIST_HEAD(&vm->invalidated);
-	spin_lock_init(&vm->status_lock);
 	INIT_LIST_HEAD(&vm->freed);
 	INIT_LIST_HEAD(&vm->done);
 	INIT_KFIFO(vm->faults);
+	spin_lock_init(&vm->stats_lock);
 
 	r = amdgpu_vm_init_entities(adev, vm);
 	if (r)
@@ -3105,7 +3069,6 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
 
 	amdgpu_vm_assert_locked(vm);
 
-	spin_lock(&vm->status_lock);
 	seq_puts(m, "\tIdle BOs:\n");
 	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
 		if (!bo_va->base.bo)
@@ -3143,11 +3106,13 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
 	id = 0;
 
 	seq_puts(m, "\tInvalidated BOs:\n");
+	spin_lock(&vm->invalidated_lock);
 	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
 		if (!bo_va->base.bo)
 			continue;
 		total_invalidated += amdgpu_bo_print_info(id++,	bo_va->base.bo, m);
 	}
+	spin_unlock(&vm->invalidated_lock);
 	total_invalidated_objs = id;
 	id = 0;
 
@@ -3157,7 +3122,6 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
 			continue;
 		total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
 	}
-	spin_unlock(&vm->status_lock);
 	total_done_objs = id;
 
 	seq_printf(m, "\tTotal idle size:        %12lld\tobjs:\t%d\n", total_idle,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index f33ea7f8509b..b5216bc1292f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -205,11 +205,11 @@ struct amdgpu_vm_bo_base {
 	/* protected by bo being reserved */
 	struct amdgpu_vm_bo_base	*next;
 
-	/* protected by vm status_lock */
+	/* protected by vm reservation and invalidated_lock */
 	struct list_head		vm_status;
 
 	/* if the bo is counted as shared in mem stats
-	 * protected by vm status_lock */
+	 * protected by vm BO being reserved */
 	bool				shared;
 
 	/* protected by the BO being reserved */
@@ -345,10 +345,8 @@ struct amdgpu_vm {
 	bool			evicting;
 	unsigned int		saved_flags;
 
-	/* Lock to protect vm_bo add/del/move on all lists of vm */
-	spinlock_t		status_lock;
-
-	/* Memory statistics for this vm, protected by status_lock */
+	/* Memory statistics for this vm, protected by stats_lock */
+	spinlock_t		stats_lock;
 	struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM];
 
 	/*
@@ -356,6 +354,8 @@ struct amdgpu_vm {
 	 * PDs, PTs or per VM BOs. The state transits are:
 	 *
 	 * evicted -> relocated (PDs, PTs) or moved (per VM BOs) -> idle
+	 *
+	 * Lists are protected by the root PD dma_resv lock.
 	 */
 
 	/* Per-VM and PT BOs who needs a validation */
@@ -376,7 +376,10 @@ struct amdgpu_vm {
 	 * state transits are:
 	 *
 	 * evicted_user or invalidated -> done
+	 *
+	 * Lists are protected by the invalidated_lock.
 	 */
+	spinlock_t		invalidated_lock;
 
 	/* BOs for user mode queues that need a validation */
 	struct list_head	evicted_user;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index 31a437ce9570..7bdd664f0770 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -544,9 +544,7 @@ static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry)
 	entry->bo->vm_bo = NULL;
 	ttm_bo_set_bulk_move(&entry->bo->tbo, NULL);
 
-	spin_lock(&entry->vm->status_lock);
 	list_del(&entry->vm_status);
-	spin_unlock(&entry->vm->status_lock);
 	amdgpu_bo_unref(&entry->bo);
 }
 
@@ -590,7 +588,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params,
 	struct amdgpu_vm_pt_cursor seek;
 	struct amdgpu_vm_bo_base *entry;
 
-	spin_lock(&params->vm->status_lock);
 	for_each_amdgpu_vm_pt_dfs_safe(params->adev, params->vm, cursor, seek, entry) {
 		if (entry && entry->bo)
 			list_move(&entry->vm_status, &params->tlb_flush_waitlist);
@@ -598,7 +595,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params,
 
 	/* enter start node now */
 	list_move(&cursor->entry->vm_status, &params->tlb_flush_waitlist);
-	spin_unlock(&params->vm->status_lock);
 }
 
 /**
-- 
2.43.0

[PATCH 10/11] drm/amdgpu: restructure VM state machine v2

[Christian König]

Instead of coming up with more sophisticated names for states a VM BO
can be in, group them by the type of BO first and then by the state.

So we end with BO type kernel, always_valid and individual and then states
evicted, moved and idle.

Not much functional change, except that evicted_user is moved back
together with the other BOs again which makes the handling in
amdgpu_vm_validate() a bit more complex.

Also fixes a problem with user queues and amdgpu_vm_ready(). We didn't
considered the VM ready when user BOs were not ideally placed, harmless
performance impact for kernel queues but a complete show stopper for
userqueues.

v2: fix a few typos in comments, rename the BO types to make them more
    descriptive, fix a couple of bugs found during testing

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c |  22 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 468 ++++++++++------------
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h    |  64 ++-
 3 files changed, 252 insertions(+), 302 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 7fc733ba962e..9d01449e1487 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -1048,12 +1048,12 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
 	struct amdgpu_bo *bo;
 	int ret;
 
-	spin_lock(&vm->invalidated_lock);
-	while (!list_empty(&vm->invalidated)) {
-		bo_va = list_first_entry(&vm->invalidated,
+	spin_lock(&vm->individual_lock);
+	while (!list_empty(&vm->always_valid.evicted)) {
+		bo_va = list_first_entry(&vm->always_valid.evicted,
 					 struct amdgpu_bo_va,
 					 base.vm_status);
-		spin_unlock(&vm->invalidated_lock);
+		spin_unlock(&vm->individual_lock);
 
 		bo = bo_va->base.bo;
 		ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2);
@@ -1065,14 +1065,14 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
 		if (ret)
 			return ret;
 
-		/* This moves the bo_va to the done list */
+		/* This moves the bo_va to the idle list */
 		ret = amdgpu_vm_bo_update(adev, bo_va, false);
 		if (ret)
 			return ret;
 
-		spin_lock(&vm->invalidated_lock);
+		spin_lock(&vm->individual_lock);
 	}
-	spin_unlock(&vm->invalidated_lock);
+	spin_unlock(&vm->individual_lock);
 
 	return 0;
 }
@@ -1104,7 +1104,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
 		if (unlikely(ret))
 			goto unlock_all;
 
-		ret = amdgpu_vm_lock_done_list(vm, &exec, 1);
+		ret = amdgpu_vm_lock_individual(vm, &exec, 1);
 		drm_exec_retry_on_contention(&exec);
 		if (unlikely(ret))
 			goto unlock_all;
@@ -1147,7 +1147,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
 
 	key = 0;
 	/* Validate User Ptr BOs */
-	list_for_each_entry(bo_va, &vm->done, base.vm_status) {
+	list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status) {
 		bo = bo_va->base.bo;
 		if (!bo)
 			continue;
@@ -1197,10 +1197,10 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
 
 	/*
 	 * We need to wait for all VM updates to finish before restarting the
-	 * queues. Using the done list like that is now ok since everything is
+	 * queues. Using the idle list like that is now ok since everything is
 	 * locked in place.
 	 */
-	list_for_each_entry(bo_va, &vm->done, base.vm_status)
+	list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status)
 		dma_fence_wait(bo_va->last_pt_update, false);
 	dma_fence_wait(vm->last_update, false);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index e2a21a66b28f..118761e2c409 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -138,6 +138,47 @@ static void amdgpu_vm_assert_locked(struct amdgpu_vm *vm)
 	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
 }
 
+/* Initialize the amdgpu_vm_bo_status object */
+static void amdgpu_vm_bo_status_init(struct amdgpu_vm_bo_status *lists)
+{
+	INIT_LIST_HEAD(&lists->evicted);
+	INIT_LIST_HEAD(&lists->moved);
+	INIT_LIST_HEAD(&lists->idle);
+}
+
+/*
+ * Make sure we have the lock to modify the vm_bo status and return the object
+ * with the status lists.
+ */
+static struct amdgpu_vm_bo_status *
+amdgpu_vm_bo_lock_lists(struct amdgpu_vm_bo_base *vm_bo)
+{
+	struct amdgpu_vm *vm = vm_bo->vm;
+	struct amdgpu_bo *bo = vm_bo->bo;
+
+	if (amdgpu_vm_is_bo_always_valid(vm, bo)) {
+		/* No extra locking needed, protected by the root PD resv lock */
+		amdgpu_vm_assert_locked(vm);
+
+		if (bo->tbo.type == ttm_bo_type_kernel)
+			return &vm->kernel;
+
+		return &vm->always_valid;
+	}
+
+	spin_lock(&vm_bo->vm->individual_lock);
+	return &vm->individual;
+}
+
+/* Eventually unlock the status list lock again */
+static void amdgpu_vm_bo_unlock_lists(struct amdgpu_vm_bo_base *vm_bo)
+{
+	if (amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
+		amdgpu_vm_assert_locked(vm_bo->vm);
+	else
+		spin_unlock(&vm_bo->vm->individual_lock);
+}
+
 /**
  * amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid
  *
@@ -157,33 +198,44 @@ bool amdgpu_vm_is_bo_always_valid(struct amdgpu_vm *vm, struct amdgpu_bo *bo)
  *
  * @vm_bo: vm_bo which is evicted
  *
- * State for PDs/PTs and per VM BOs which are not at the location they should
- * be.
+ * State for vm_bo objects meaning the underlying BO was evicted and need to
+ * move in place again.
  */
 static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
 {
-	struct amdgpu_vm *vm = vm_bo->vm;
-	struct amdgpu_bo *bo = vm_bo->bo;
+	struct amdgpu_vm_bo_status *lists;
 
+	lists = amdgpu_vm_bo_lock_lists(vm_bo);
 	vm_bo->moved = true;
-	amdgpu_vm_assert_locked(vm);
-	if (bo->tbo.type == ttm_bo_type_kernel)
-		list_move(&vm_bo->vm_status, &vm->evicted);
-	else
-		list_move_tail(&vm_bo->vm_status, &vm->evicted);
+	list_move(&vm_bo->vm_status, &lists->evicted);
+	amdgpu_vm_bo_unlock_lists(vm_bo);
 }
 /**
  * amdgpu_vm_bo_moved - vm_bo is moved
  *
  * @vm_bo: vm_bo which is moved
  *
- * State for per VM BOs which are moved, but that change is not yet reflected
- * in the page tables.
+ * State for vm_bo objects meaning the underlying BO was moved but the new
+ * location not yet reflected in the page tables.
  */
 static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
 {
-	amdgpu_vm_assert_locked(vm_bo->vm);
-	list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
+	struct amdgpu_vm_bo_status *lists;
+	struct amdgpu_bo *bo = vm_bo->bo;
+
+	/*
+	 * The root PD doesn't have a parent PDE and goes directly into the
+	 * idle state.
+	 */
+	lists = amdgpu_vm_bo_lock_lists(vm_bo);
+	if (bo && bo->tbo.type == ttm_bo_type_kernel && !bo->parent) {
+		vm_bo->moved = false;
+		list_move(&vm_bo->vm_status, &lists->idle);
+	} else {
+		vm_bo->moved = true;
+		list_move(&vm_bo->vm_status, &lists->moved);
+	}
+	amdgpu_vm_bo_unlock_lists(vm_bo);
 }
 
 /**
@@ -191,104 +243,36 @@ static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
  *
  * @vm_bo: vm_bo which is now idle
  *
- * State for PDs/PTs and per VM BOs which have gone through the state machine
- * and are now idle.
+ * State for vm_bo objects meaning we are done with the state machine and no
+ * further action is necessary.
  */
 static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
 {
-	amdgpu_vm_assert_locked(vm_bo->vm);
-	list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
-	vm_bo->moved = false;
-}
-
-/**
- * amdgpu_vm_bo_invalidated - vm_bo is invalidated
- *
- * @vm_bo: vm_bo which is now invalidated
- *
- * State for normal BOs which are invalidated and that change not yet reflected
- * in the PTs.
- */
-static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
-{
-	spin_lock(&vm_bo->vm->invalidated_lock);
-	list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
-	spin_unlock(&vm_bo->vm->invalidated_lock);
-}
-
-/**
- * amdgpu_vm_bo_evicted_user - vm_bo is evicted
- *
- * @vm_bo: vm_bo which is evicted
- *
- * State for BOs used by user mode queues which are not at the location they
- * should be.
- */
-static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)
-{
-	amdgpu_vm_assert_locked(vm_bo->vm);
-	vm_bo->moved = true;
-	list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user);
-}
+	struct amdgpu_vm_bo_status *lists;
 
-/**
- * amdgpu_vm_bo_relocated - vm_bo is reloacted
- *
- * @vm_bo: vm_bo which is relocated
- *
- * State for PDs/PTs which needs to update their parent PD.
- * For the root PD, just move to idle state.
- */
-static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
-{
-	amdgpu_vm_assert_locked(vm_bo->vm);
-	if (vm_bo->bo->parent)
-		list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
-	else
-		amdgpu_vm_bo_idle(vm_bo);
-}
-
-/**
- * amdgpu_vm_bo_done - vm_bo is done
- *
- * @vm_bo: vm_bo which is now done
- *
- * State for normal BOs which are invalidated and that change has been updated
- * in the PTs.
- */
-static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
-{
-	amdgpu_vm_assert_locked(vm_bo->vm);
-	list_move(&vm_bo->vm_status, &vm_bo->vm->done);
+	lists = amdgpu_vm_bo_lock_lists(vm_bo);
+	if (!amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
+		vm_bo->moved = false;
+	list_move(&vm_bo->vm_status, &lists->idle);
+	amdgpu_vm_bo_unlock_lists(vm_bo);
 }
 
 /**
  * amdgpu_vm_bo_reset_state_machine - reset the vm_bo state machine
  * @vm: the VM which state machine to reset
  *
- * Move all vm_bo object in the VM into a state where they will be updated
- * again during validation.
+ * Move all vm_bo object in the VM into a state where their location will be
+ * updated in the page tables again.
  */
 static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
 {
-	struct amdgpu_vm_bo_base *vm_bo, *tmp;
-
-	spin_lock(&vm->invalidated_lock);
-	list_splice_init(&vm->done, &vm->invalidated);
-	list_for_each_entry(vm_bo, &vm->invalidated, vm_status)
-		vm_bo->moved = true;
-	spin_unlock(&vm->invalidated_lock);
-
 	amdgpu_vm_assert_locked(vm);
-	list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) {
-		struct amdgpu_bo *bo = vm_bo->bo;
+	list_splice_init(&vm->kernel.idle, &vm->kernel.moved);
+	list_splice_init(&vm->always_valid.idle, &vm->always_valid.moved);
 
-		vm_bo->moved = true;
-		if (!bo || bo->tbo.type != ttm_bo_type_kernel)
-			list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
-		else if (bo->parent)
-			list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
-	}
+	spin_lock(&vm->individual_lock);
+	list_splice_init(&vm->individual.idle, &vm->individual.moved);
+	spin_unlock(&vm->individual_lock);
 }
 
 /**
@@ -416,8 +400,10 @@ void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
 	base->next = NULL;
 	INIT_LIST_HEAD(&base->vm_status);
 
+	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
 	if (!bo)
 		return;
+
 	base->next = bo->vm_bo;
 	bo->vm_bo = base;
 
@@ -426,27 +412,22 @@ void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
 	amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1);
 	spin_unlock(&vm->stats_lock);
 
-	if (!amdgpu_vm_is_bo_always_valid(vm, bo))
+	if (!amdgpu_vm_is_bo_always_valid(vm, bo)) {
+		amdgpu_vm_bo_idle(base);
 		return;
-
-	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
+	}
 
 	ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move);
-	if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
-		amdgpu_vm_bo_relocated(base);
-	else
-		amdgpu_vm_bo_idle(base);
 
+	/*
+	 * When a per VM isn't in the desired domain put it into the evicted
+	 * state to make sure that it gets validated on the next best occasion.
+	 */
 	if (bo->preferred_domains &
 	    amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type))
-		return;
-
-	/*
-	 * we checked all the prerequisites, but it looks like this per vm bo
-	 * is currently evicted. add the bo to the evicted list to make sure it
-	 * is validated on next vm use to avoid fault.
-	 * */
-	amdgpu_vm_bo_evicted(base);
+		amdgpu_vm_bo_moved(base);
+	else
+		amdgpu_vm_bo_evicted(base);
 }
 
 /**
@@ -467,41 +448,41 @@ int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec,
 }
 
 /**
- * amdgpu_vm_lock_done_list - lock all BOs on the done list
+ * amdgpu_vm_lock_individual - lock all BOs on the individual idle list
  * @vm: vm providing the BOs
  * @exec: drm execution context
  * @num_fences: number of extra fences to reserve
  *
- * Lock the BOs on the done list in the DRM execution context.
+ * Lock the BOs on the individual idle list in the DRM execution context.
  */
-int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
-			     unsigned int num_fences)
+int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
+			      unsigned int num_fences)
 {
-	struct list_head *prev = &vm->done;
+	struct list_head *prev = &vm->individual.idle;
 	struct amdgpu_bo_va *bo_va;
 	struct amdgpu_bo *bo;
 	int ret;
 
 	/* We can only trust prev->next while holding the lock */
-	spin_lock(&vm->invalidated_lock);
-	while (!list_is_head(prev->next, &vm->done)) {
+	spin_lock(&vm->individual_lock);
+	while (!list_is_head(prev->next, &vm->individual.idle)) {
 		bo_va = list_entry(prev->next, typeof(*bo_va), base.vm_status);
 
 		bo = bo_va->base.bo;
 		if (bo) {
 			amdgpu_bo_ref(bo);
-			spin_unlock(&vm->invalidated_lock);
+			spin_unlock(&vm->individual_lock);
 
 			ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 1);
 			amdgpu_bo_unref(&bo);
 			if (unlikely(ret))
 				return ret;
 
-			spin_lock(&vm->invalidated_lock);
+			spin_lock(&vm->individual_lock);
 		}
 		prev = prev->next;
 	}
-	spin_unlock(&vm->invalidated_lock);
+	spin_unlock(&vm->individual_lock);
 
 	return 0;
 }
@@ -598,9 +579,9 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 {
 	uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm);
 	struct amdgpu_vm_bo_base *bo_base, *tmp;
-	struct amdgpu_bo *bo;
 	int r;
 
+	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
 	if (vm->generation != new_vm_generation) {
 		vm->generation = new_vm_generation;
 		amdgpu_vm_bo_reset_state_machine(vm);
@@ -610,38 +591,59 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			return r;
 	}
 
-	list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {
-		bo = bo_base->bo;
-
-		r = validate(param, bo);
+	list_for_each_entry_safe(bo_base, tmp, &vm->kernel.evicted, vm_status) {
+		r = validate(param, bo_base->bo);
 		if (r)
 			return r;
 
-		if (bo->tbo.type != ttm_bo_type_kernel) {
-			amdgpu_vm_bo_moved(bo_base);
-		} else {
-			vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
-			amdgpu_vm_bo_relocated(bo_base);
-		}
+		vm->update_funcs->map_table(to_amdgpu_bo_vm(bo_base->bo));
+		amdgpu_vm_bo_moved(bo_base);
 	}
 
-	if (ticket) {
-		list_for_each_entry_safe(bo_base, tmp, &vm->evicted_user,
-					 vm_status) {
-			bo = bo_base->bo;
-			dma_resv_assert_held(bo->tbo.base.resv);
+	/*
+	 * As soon as all page tables are in place we can start updating them
+	 * again.
+	 */
+	amdgpu_vm_eviction_lock(vm);
+	vm->evicting = false;
+	amdgpu_vm_eviction_unlock(vm);
 
-			r = validate(param, bo);
-			if (r)
-				return r;
+	list_for_each_entry_safe(bo_base, tmp, &vm->always_valid.evicted,
+				 vm_status) {
+		r = validate(param, bo_base->bo);
+		if (r)
+			return r;
 
-			amdgpu_vm_bo_invalidated(bo_base);
-		}
+		amdgpu_vm_bo_moved(bo_base);
 	}
 
-	amdgpu_vm_eviction_lock(vm);
-	vm->evicting = false;
-	amdgpu_vm_eviction_unlock(vm);
+	if (!ticket)
+		return 0;
+
+	spin_lock(&vm->individual_lock);
+restart:
+	list_for_each_entry(bo_base, &vm->individual.evicted, vm_status) {
+		struct amdgpu_bo *bo = bo_base->bo;
+
+		if (dma_resv_locking_ctx(bo->tbo.base.resv) != ticket)
+			continue;
+
+		spin_unlock(&vm->individual_lock);
+
+		r = validate(param, bo);
+		if (r)
+			return r;
+
+		amdgpu_vm_bo_moved(bo_base);
+
+		/* It's a bit inefficient to always jump back to the start, but
+		 * we would need to re-structure the KFD for properly fixing
+		 * that.
+		 */
+		spin_lock(&vm->individual_lock);
+		goto restart;
+	}
+	spin_unlock(&vm->individual_lock);
 
 	return 0;
 }
@@ -666,7 +668,7 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 	ret = !vm->evicting;
 	amdgpu_vm_eviction_unlock(vm);
 
-	ret &= list_empty(&vm->evicted);
+	ret &= list_empty(&vm->kernel.evicted);
 
 	spin_lock(&vm->immediate.lock);
 	ret &= !vm->immediate.stopped;
@@ -966,7 +968,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 
 	amdgpu_vm_assert_locked(vm);
 
-	if (list_empty(&vm->relocated))
+	if (list_empty(&vm->kernel.moved))
 		return 0;
 
 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
@@ -982,7 +984,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 	if (r)
 		goto error;
 
-	list_for_each_entry(entry, &vm->relocated, vm_status) {
+	list_for_each_entry(entry, &vm->kernel.moved, vm_status) {
 		/* vm_flush_needed after updating moved PDEs */
 		flush_tlb_needed |= entry->moved;
 
@@ -998,9 +1000,8 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 	if (flush_tlb_needed)
 		atomic64_inc(&vm->tlb_seq);
 
-	list_for_each_entry_safe(entry, tmp, &vm->relocated, vm_status) {
+	list_for_each_entry_safe(entry, tmp, &vm->kernel.moved, vm_status)
 		amdgpu_vm_bo_idle(entry);
-	}
 
 error:
 	drm_dev_exit(idx);
@@ -1374,7 +1375,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
 		else
 			amdgpu_vm_bo_idle(&bo_va->base);
 	} else {
-		amdgpu_vm_bo_done(&bo_va->base);
+		amdgpu_vm_bo_idle(&bo_va->base);
 	}
 
 	list_splice_init(&bo_va->invalids, &bo_va->valids);
@@ -1602,19 +1603,20 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
 	bool clear, unlock;
 	int r;
 
-	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
+	list_for_each_entry_safe(bo_va, tmp, &vm->always_valid.moved,
+				 base.vm_status) {
 		/* Per VM BOs never need to bo cleared in the page tables */
 		r = amdgpu_vm_bo_update(adev, bo_va, false);
 		if (r)
 			return r;
 	}
 
-	spin_lock(&vm->invalidated_lock);
-	while (!list_empty(&vm->invalidated)) {
-		bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
-					 base.vm_status);
+	spin_lock(&vm->individual_lock);
+	while (!list_empty(&vm->individual.moved)) {
+		bo_va = list_first_entry(&vm->individual.moved,
+					 typeof(*bo_va), base.vm_status);
 		resv = bo_va->base.bo->tbo.base.resv;
-		spin_unlock(&vm->invalidated_lock);
+		spin_unlock(&vm->individual_lock);
 
 		/* Try to reserve the BO to avoid clearing its ptes */
 		if (!adev->debug_vm && dma_resv_trylock(resv)) {
@@ -1644,11 +1646,11 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
 		    drm_gem_is_imported(&bo_va->base.bo->tbo.base) &&
 		    (!bo_va->base.bo->tbo.resource ||
 		     bo_va->base.bo->tbo.resource->mem_type == TTM_PL_SYSTEM))
-			amdgpu_vm_bo_evicted_user(&bo_va->base);
+			amdgpu_vm_bo_evicted(&bo_va->base);
 
-		spin_lock(&vm->invalidated_lock);
+		spin_lock(&vm->individual_lock);
 	}
-	spin_unlock(&vm->invalidated_lock);
+	spin_unlock(&vm->individual_lock);
 
 	return 0;
 }
@@ -2186,9 +2188,9 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev,
 		}
 	}
 
-	spin_lock(&vm->invalidated_lock);
+	spin_lock(&vm->individual_lock);
 	list_del(&bo_va->base.vm_status);
-	spin_unlock(&vm->invalidated_lock);
+	spin_unlock(&vm->individual_lock);
 
 	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
 		list_del(&mapping->list);
@@ -2268,14 +2270,7 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_bo *bo, bool evicted)
 
 		if (bo_base->moved)
 			continue;
-		bo_base->moved = true;
-
-		if (bo->tbo.type == ttm_bo_type_kernel)
-			amdgpu_vm_bo_relocated(bo_base);
-		else if (amdgpu_vm_is_bo_always_valid(vm, bo))
-			amdgpu_vm_bo_moved(bo_base);
-		else
-			amdgpu_vm_bo_invalidated(bo_base);
+		amdgpu_vm_bo_moved(bo_base);
 	}
 }
 
@@ -2566,15 +2561,12 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	vm->va = RB_ROOT_CACHED;
 	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
 		vm->reserved_vmid[i] = NULL;
-	INIT_LIST_HEAD(&vm->evicted);
-	INIT_LIST_HEAD(&vm->evicted_user);
-	INIT_LIST_HEAD(&vm->relocated);
-	INIT_LIST_HEAD(&vm->moved);
-	INIT_LIST_HEAD(&vm->idle);
-	spin_lock_init(&vm->invalidated_lock);
-	INIT_LIST_HEAD(&vm->invalidated);
+
+	amdgpu_vm_bo_status_init(&vm->kernel);
+	amdgpu_vm_bo_status_init(&vm->always_valid);
+	spin_lock_init(&vm->individual_lock);
+	amdgpu_vm_bo_status_init(&vm->individual);
 	INIT_LIST_HEAD(&vm->freed);
-	INIT_LIST_HEAD(&vm->done);
 	INIT_KFIFO(vm->faults);
 	spin_lock_init(&vm->stats_lock);
 
@@ -3042,100 +3034,62 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
 }
 
 #if defined(CONFIG_DEBUG_FS)
-/**
- * amdgpu_debugfs_vm_bo_info  - print BO info for the VM
- *
- * @vm: Requested VM for printing BO info
- * @m: debugfs file
- *
- * Print BO information in debugfs file for the VM
- */
-void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
-{
-	struct amdgpu_bo_va *bo_va, *tmp;
-	u64 total_idle = 0;
-	u64 total_evicted = 0;
-	u64 total_relocated = 0;
-	u64 total_moved = 0;
-	u64 total_invalidated = 0;
-	u64 total_done = 0;
-	unsigned int total_idle_objs = 0;
-	unsigned int total_evicted_objs = 0;
-	unsigned int total_relocated_objs = 0;
-	unsigned int total_moved_objs = 0;
-	unsigned int total_invalidated_objs = 0;
-	unsigned int total_done_objs = 0;
-	unsigned int id = 0;
 
-	amdgpu_vm_assert_locked(vm);
+/* print the debug info for a specific set of status lists */
+static void amdgpu_debugfs_vm_bo_status_info(struct seq_file *m,
+					     struct amdgpu_vm_bo_status *lists)
+{
+	struct amdgpu_vm_bo_base *base;
+	unsigned int id;
 
-	seq_puts(m, "\tIdle BOs:\n");
-	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
-		if (!bo_va->base.bo)
-			continue;
-		total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
-	}
-	total_idle_objs = id;
 	id = 0;
-
 	seq_puts(m, "\tEvicted BOs:\n");
-	list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
-		if (!bo_va->base.bo)
+	list_for_each_entry(base, &lists->evicted, vm_status) {
+		if (!base->bo)
 			continue;
-		total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
-	}
-	total_evicted_objs = id;
-	id = 0;
 
-	seq_puts(m, "\tRelocated BOs:\n");
-	list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
-		if (!bo_va->base.bo)
-			continue;
-		total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
+		amdgpu_bo_print_info(id++, base->bo, m);
 	}
-	total_relocated_objs = id;
-	id = 0;
 
+	id = 0;
 	seq_puts(m, "\tMoved BOs:\n");
-	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
-		if (!bo_va->base.bo)
+	list_for_each_entry(base, &lists->moved, vm_status) {
+		if (!base->bo)
 			continue;
-		total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
+
+		amdgpu_bo_print_info(id++, base->bo, m);
 	}
-	total_moved_objs = id;
-	id = 0;
 
-	seq_puts(m, "\tInvalidated BOs:\n");
-	spin_lock(&vm->invalidated_lock);
-	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
-		if (!bo_va->base.bo)
+	id = 0;
+	seq_puts(m, "\tIdle BOs:\n");
+	list_for_each_entry(base, &lists->moved, vm_status) {
+		if (!base->bo)
 			continue;
-		total_invalidated += amdgpu_bo_print_info(id++,	bo_va->base.bo, m);
+
+		amdgpu_bo_print_info(id++, base->bo, m);
 	}
-	spin_unlock(&vm->invalidated_lock);
-	total_invalidated_objs = id;
-	id = 0;
+}
 
-	seq_puts(m, "\tDone BOs:\n");
-	list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
-		if (!bo_va->base.bo)
-			continue;
-		total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
-	}
-	total_done_objs = id;
-
-	seq_printf(m, "\tTotal idle size:        %12lld\tobjs:\t%d\n", total_idle,
-		   total_idle_objs);
-	seq_printf(m, "\tTotal evicted size:     %12lld\tobjs:\t%d\n", total_evicted,
-		   total_evicted_objs);
-	seq_printf(m, "\tTotal relocated size:   %12lld\tobjs:\t%d\n", total_relocated,
-		   total_relocated_objs);
-	seq_printf(m, "\tTotal moved size:       %12lld\tobjs:\t%d\n", total_moved,
-		   total_moved_objs);
-	seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated,
-		   total_invalidated_objs);
-	seq_printf(m, "\tTotal done size:        %12lld\tobjs:\t%d\n", total_done,
-		   total_done_objs);
+/**
+ * amdgpu_debugfs_vm_bo_info  - print BO info for the VM
+ *
+ * @vm: Requested VM for printing BO info
+ * @m: debugfs file
+ *
+ * Print BO information in debugfs file for the VM
+ */
+void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
+{
+	amdgpu_vm_assert_locked(vm);
+
+	seq_puts(m, "\tKernel PT/PDs:\n");
+	amdgpu_debugfs_vm_bo_status_info(m, &vm->kernel);
+
+	seq_puts(m, "\tPer VM BOs:\n");
+	amdgpu_debugfs_vm_bo_status_info(m, &vm->always_valid);
+
+	seq_puts(m, "\tIndividual BOs:\n");
+	amdgpu_debugfs_vm_bo_status_info(m, &vm->individual);
 }
 #endif
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index b5216bc1292f..cc96a3e6252f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -216,6 +216,23 @@ struct amdgpu_vm_bo_base {
 	bool				moved;
 };
 
+/*
+ * The following status lists contain amdgpu_vm_bo_base objects for
+ * either PD/PTs, per VM BOs or BOs with individual resv object.
+ *
+ * The state transits are: evicted -> moved -> idle
+ */
+struct amdgpu_vm_bo_status {
+	/* BOs evicted which need to move into place again */
+	struct list_head		evicted;
+
+	/* BOs which moved but new location hasn't been updated in the PDs/PTs */
+	struct list_head		moved;
+
+	/* BOs done with the state machine and need no further action */
+	struct list_head		idle;
+};
+
 /* provided by hw blocks that can write ptes, e.g., sdma */
 struct amdgpu_vm_pte_funcs {
 	/* number of dw to reserve per operation */
@@ -349,46 +366,25 @@ struct amdgpu_vm {
 	spinlock_t		stats_lock;
 	struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM];
 
+	/* BO's belonging to PD/PT which are internal to the kernel. */
+	struct amdgpu_vm_bo_status	kernel;
+
 	/*
-	 * The following lists contain amdgpu_vm_bo_base objects for either
-	 * PDs, PTs or per VM BOs. The state transits are:
-	 *
-	 * evicted -> relocated (PDs, PTs) or moved (per VM BOs) -> idle
-	 *
-	 * Lists are protected by the root PD dma_resv lock.
+	 * BOs allocated by userspace where the dma_resv is shared with the
+	 * root PD
 	 */
-
-	/* Per-VM and PT BOs who needs a validation */
-	struct list_head	evicted;
-
-	/* PT BOs which relocated and their parent need an update */
-	struct list_head	relocated;
-
-	/* per VM BOs moved, but not yet updated in the PT */
-	struct list_head	moved;
-
-	/* All BOs of this VM not currently in the state machine */
-	struct list_head	idle;
+	struct amdgpu_vm_bo_status	always_valid;
 
 	/*
 	 * The following lists contain amdgpu_vm_bo_base objects for BOs which
-	 * have their own dma_resv object and not depend on the root PD. Their
-	 * state transits are:
-	 *
-	 * evicted_user or invalidated -> done
+	 * have their own dma_resv object and not depend on the root PD.
 	 *
-	 * Lists are protected by the invalidated_lock.
+	 * Lists are protected by the individual_lock.
 	 */
-	spinlock_t		invalidated_lock;
-
-	/* BOs for user mode queues that need a validation */
-	struct list_head	evicted_user;
-
-	/* regular invalidated BOs, but not yet updated in the PT */
-	struct list_head	invalidated;
+	spinlock_t			individual_lock;
 
-	/* BOs which are invalidated, has been updated in the PTs */
-	struct list_head        done;
+	/* Userspace BOs with individual resv object */
+	struct amdgpu_vm_bo_status	individual;
 
 	/*
 	 * This list contains amdgpu_bo_va_mapping objects which have been freed
@@ -510,8 +506,8 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec,
 		      unsigned int num_fences);
-int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
-			     unsigned int num_fences);
+int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
+			      unsigned int num_fences);
 bool amdgpu_vm_ready(struct amdgpu_vm *vm);
 uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-- 
2.43.0

[PATCH 11/11] drm/amdgpu: WIP sync amdgpu_ttm_fill_mem only to kernel fences

[Christian König]

That's not even remotely correct, but should unblock testing for now.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 4c7d1917d9bb..5130f77b7543 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -2417,12 +2417,14 @@ static int amdgpu_ttm_prepare_job(struct amdgpu_device *adev,
 				  struct amdgpu_ttm_buffer_entity *entity,
 				  unsigned int num_dw,
 				  struct dma_resv *resv,
+				  enum dma_resv_usage usage,
 				  bool vm_needs_flush,
 				  struct amdgpu_job **job,
 				  u64 k_job_id)
 {
 	enum amdgpu_ib_pool_type pool = AMDGPU_IB_POOL_DELAYED;
 	int r;
+
 	r = amdgpu_job_alloc_with_ib(adev, &entity->base,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     num_dw * 4, pool, job, k_job_id);
@@ -2438,8 +2440,7 @@ static int amdgpu_ttm_prepare_job(struct amdgpu_device *adev,
 	if (!resv)
 		return 0;
 
-	return drm_sched_job_add_resv_dependencies(&(*job)->base, resv,
-						   DMA_RESV_USAGE_BOOKKEEP);
+	return drm_sched_job_add_resv_dependencies(&(*job)->base, resv, usage);
 }
 
 int amdgpu_copy_buffer(struct amdgpu_device *adev,
@@ -2468,9 +2469,9 @@ int amdgpu_copy_buffer(struct amdgpu_device *adev,
 	max_bytes = adev->mman.buffer_funcs->copy_max_bytes;
 	num_loops = DIV_ROUND_UP(byte_count, max_bytes);
 	num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->copy_num_dw, 8);
-	r = amdgpu_ttm_prepare_job(adev, entity, num_dw,
-				   resv, vm_needs_flush, &job,
-				   AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER);
+	r = amdgpu_ttm_prepare_job(adev, entity, num_dw, resv,
+				   DMA_RESV_USAGE_BOOKKEEP, vm_needs_flush,
+				   &job, AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER);
 	if (r)
 		goto error_free;
 
@@ -2513,6 +2514,7 @@ static int amdgpu_ttm_fill_mem(struct amdgpu_device *adev,
 	num_loops = DIV_ROUND_UP_ULL(byte_count, max_bytes);
 	num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->fill_num_dw, 8);
 	r = amdgpu_ttm_prepare_job(adev, entity, num_dw, resv,
+				   DMA_RESV_USAGE_KERNEL,
 				   vm_needs_flush, &job, k_job_id);
 	if (r)
 		return r;
-- 
2.43.0

Re: [PATCH 02/11] drm/amdgpu: remove deadlocks from amdgpu_userq_pre_reset

[Khatri, Sunil]

[-- Attachment #1: Type: text/plain, Size: 2674 bytes --]


On 21-04-2026 06:25 pm, Christian König wrote:
> The purpose of a GPU reset is to make sure that fence can be signaled
> again and the signal and resume workers can make progress again.
>
> So waiting for the resume worker or any fence in the GPU reset path is
> just utterly nonsense.
>
> Signed-off-by: Christian König<christian.koenig@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 26 +++++++++++------------
>   1 file changed, 12 insertions(+), 14 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index 8f48520cb822..b632bc3c952b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -1496,23 +1496,21 @@ void amdgpu_userq_pre_reset(struct amdgpu_device *adev)
>   {
>   	const struct amdgpu_userq_funcs *userq_funcs;
>   	struct amdgpu_usermode_queue *queue;
> -	struct amdgpu_userq_mgr *uqm;
>   	unsigned long queue_id;
>   
> +	/* TODO: We probably need a new lock for the queue state */
>   	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
> -		uqm = queue->userq_mgr;
> -		cancel_delayed_work_sync(&uqm->resume_work);
> -		if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
> -			amdgpu_userq_wait_for_last_fence(queue);
> -			userq_funcs = adev->userq_funcs[queue->queue_type];
> -			userq_funcs->unmap(queue);
> -			/* just mark all queues as hung at this point.
> -			 * if unmap succeeds, we could map again
> -			 * in amdgpu_userq_post_reset() if vram is not lost
> -			 */
> -			queue->state = AMDGPU_USERQ_STATE_HUNG;
> -			amdgpu_userq_fence_driver_force_completion(queue);
> -		}
> +		if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
> +			continue;

If the queue is in prempt state and if at that time we are in this 
function we should still be doing force completion for work in those 
queue else the waiters will keep waiting.

> +
> +		userq_funcs = adev->userq_funcs[queue->queue_type];
> +		userq_funcs->unmap(queue);
GPU is already hung if we are here and observation is we are unable to 
unmap as we have tried to reset via the fw and that failed to atleast 
thats what i have seen. Could we skip unmap ???
> +		/* just mark all queues as hung at this point.
> +		 * if unmap succeeds, we could map again
> +		 * in amdgpu_userq_post_reset() if vram is not lost
> +		 */
> +		queue->state = AMDGPU_USERQ_STATE_HUNG;
> +		amdgpu_userq_fence_driver_force_completion(queue);

we should be calling completion irrespective of queue state here. The 
GPU atleast the queue is hung and fw has failed to reset. We have to 
release the fences by foce completion.

Regards
Sunil Khatri

>   	}
>   }
>   

[-- Attachment #2: Type: text/html, Size: 3612 bytes --]

Re: [PATCH 02/11] drm/amdgpu: remove deadlocks from amdgpu_userq_pre_reset

[Christian König]

On 4/22/26 06:53, Khatri, Sunil wrote:
> On 21-04-2026 06:25 pm, Christian König wrote:
>> The purpose of a GPU reset is to make sure that fence can be signaled
>> again and the signal and resume workers can make progress again.
>>
>> So waiting for the resume worker or any fence in the GPU reset path is
>> just utterly nonsense.
>>
>> Signed-off-by: Christian König <christian.koenig@amd.com>
>> ---
>>  drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 26 +++++++++++------------
>>  1 file changed, 12 insertions(+), 14 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>> index 8f48520cb822..b632bc3c952b 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>> @@ -1496,23 +1496,21 @@ void amdgpu_userq_pre_reset(struct amdgpu_device *adev)
>>  {
>>  	const struct amdgpu_userq_funcs *userq_funcs;
>>  	struct amdgpu_usermode_queue *queue;
>> -	struct amdgpu_userq_mgr *uqm;
>>  	unsigned long queue_id;
>>  
>> +	/* TODO: We probably need a new lock for the queue state */
>>  	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
>> -		uqm = queue->userq_mgr;
>> -		cancel_delayed_work_sync(&uqm->resume_work);
>> -		if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
>> -			amdgpu_userq_wait_for_last_fence(queue);
>> -			userq_funcs = adev->userq_funcs[queue->queue_type];
>> -			userq_funcs->unmap(queue);
>> -			/* just mark all queues as hung at this point.
>> -			 * if unmap succeeds, we could map again
>> -			 * in amdgpu_userq_post_reset() if vram is not lost
>> -			 */
>> -			queue->state = AMDGPU_USERQ_STATE_HUNG;
>> -			amdgpu_userq_fence_driver_force_completion(queue);
>> -		}
>> +		if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
>> +			continue;
> 
> If the queue is in prempt state and if at that time we are in this function we should still be doing force completion for work in those queue else the waiters will keep waiting.
> 
>> +
>> +		userq_funcs = adev->userq_funcs[queue->queue_type];
>> +		userq_funcs->unmap(queue);
> GPU is already hung if we are here and observation is we are unable to unmap as we have tried to reset via the fw and that failed to atleast thats what i have seen. Could we skip unmap ???
>> +		/* just mark all queues as hung at this point.
>> +		 * if unmap succeeds, we could map again
>> +		 * in amdgpu_userq_post_reset() if vram is not lost
>> +		 */
>> +		queue->state = AMDGPU_USERQ_STATE_HUNG;
>> +		amdgpu_userq_fence_driver_force_completion(queue);
> 
> we should be calling completion irrespective of queue state here. The GPU atleast the queue is hung and fw has failed to reset. We have to release the fences by foce completion.

Yeah, I agree completely with those require comments I has similar thoughts while going over this.

I'm only driven by bugs here and my goal with the patch was to remove the obvious deadlock in the function.

Somebody needs to sit down and go over the whole handling for GPU resets and make sure it plays nicely with user queues.

Regards,
Christian.

> 
> Regards
> Sunil Khatri
> 
>>  	}
>>  }
>>  

Re: [PATCH 02/11] drm/amdgpu: remove deadlocks from amdgpu_userq_pre_reset

[Khatri, Sunil]

[-- Attachment #1: Type: text/plain, Size: 3459 bytes --]


On 22-04-2026 12:43 pm, Christian König wrote:
> On 4/22/26 06:53, Khatri, Sunil wrote:
>> On 21-04-2026 06:25 pm, Christian König wrote:
>>> The purpose of a GPU reset is to make sure that fence can be signaled
>>> again and the signal and resume workers can make progress again.
>>>
>>> So waiting for the resume worker or any fence in the GPU reset path is
>>> just utterly nonsense.
>>>
>>> Signed-off-by: Christian König<christian.koenig@amd.com>
>>> ---
>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 26 +++++++++++------------
>>>   1 file changed, 12 insertions(+), 14 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>>> index 8f48520cb822..b632bc3c952b 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>>> @@ -1496,23 +1496,21 @@ void amdgpu_userq_pre_reset(struct amdgpu_device *adev)
>>>   {
>>>   	const struct amdgpu_userq_funcs *userq_funcs;
>>>   	struct amdgpu_usermode_queue *queue;
>>> -	struct amdgpu_userq_mgr *uqm;
>>>   	unsigned long queue_id;
>>>   
>>> +	/* TODO: We probably need a new lock for the queue state */
>>>   	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
>>> -		uqm = queue->userq_mgr;
>>> -		cancel_delayed_work_sync(&uqm->resume_work);
>>> -		if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
>>> -			amdgpu_userq_wait_for_last_fence(queue);
>>> -			userq_funcs = adev->userq_funcs[queue->queue_type];
>>> -			userq_funcs->unmap(queue);
>>> -			/* just mark all queues as hung at this point.
>>> -			 * if unmap succeeds, we could map again
>>> -			 * in amdgpu_userq_post_reset() if vram is not lost
>>> -			 */
>>> -			queue->state = AMDGPU_USERQ_STATE_HUNG;
>>> -			amdgpu_userq_fence_driver_force_completion(queue);
>>> -		}
>>> +		if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
>>> +			continue;
>> If the queue is in prempt state and if at that time we are in this function we should still be doing force completion for work in those queue else the waiters will keep waiting.
>>
>>> +
>>> +		userq_funcs = adev->userq_funcs[queue->queue_type];
>>> +		userq_funcs->unmap(queue);
>> GPU is already hung if we are here and observation is we are unable to unmap as we have tried to reset via the fw and that failed to atleast thats what i have seen. Could we skip unmap ???
>>> +		/* just mark all queues as hung at this point.
>>> +		 * if unmap succeeds, we could map again
>>> +		 * in amdgpu_userq_post_reset() if vram is not lost
>>> +		 */
>>> +		queue->state = AMDGPU_USERQ_STATE_HUNG;
>>> +		amdgpu_userq_fence_driver_force_completion(queue);
>> we should be calling completion irrespective of queue state here. The GPU atleast the queue is hung and fw has failed to reset. We have to release the fences by foce completion.
> Yeah, I agree completely with those require comments I has similar thoughts while going over this.
>
> I'm only driven by bugs here and my goal with the patch was to remove the obvious deadlock in the function.
>
> Somebody needs to sit down and go over the whole handling for GPU resets and make sure it plays nicely with user queues.
>
> Regards,
> Christian.
For now this is what i think we could do.
In the loop: If a queue is mapped -> try unmap

Irrespective of the queue state:
Set queue to HUNG and call amdgpu_userq_fence_driver_force_completion

Regards
Sunil Khatri
>
>> Regards
>> Sunil Khatri
>>
>>>   	}
>>>   }
>>>   

[-- Attachment #2: Type: text/html, Size: 4870 bytes --]

Re: [PATCH 02/11] drm/amdgpu: remove deadlocks from amdgpu_userq_pre_reset

[Christian König]

On 4/22/26 09:19, Khatri, Sunil wrote:
> 
> On 22-04-2026 12:43 pm, Christian König wrote:
>> On 4/22/26 06:53, Khatri, Sunil wrote:
>>> On 21-04-2026 06:25 pm, Christian König wrote:
>>>> The purpose of a GPU reset is to make sure that fence can be signaled
>>>> again and the signal and resume workers can make progress again.
>>>>
>>>> So waiting for the resume worker or any fence in the GPU reset path is
>>>> just utterly nonsense.
>>>>
>>>> Signed-off-by: Christian König <christian.koenig@amd.com>
>>>> ---
>>>>  drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 26 +++++++++++------------
>>>>  1 file changed, 12 insertions(+), 14 deletions(-)
>>>>
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>>>> index 8f48520cb822..b632bc3c952b 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>>>> @@ -1496,23 +1496,21 @@ void amdgpu_userq_pre_reset(struct amdgpu_device *adev)
>>>>  {
>>>>  	const struct amdgpu_userq_funcs *userq_funcs;
>>>>  	struct amdgpu_usermode_queue *queue;
>>>> -	struct amdgpu_userq_mgr *uqm;
>>>>  	unsigned long queue_id;
>>>>  
>>>> +	/* TODO: We probably need a new lock for the queue state */
>>>>  	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
>>>> -		uqm = queue->userq_mgr;
>>>> -		cancel_delayed_work_sync(&uqm->resume_work);
>>>> -		if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
>>>> -			amdgpu_userq_wait_for_last_fence(queue);
>>>> -			userq_funcs = adev->userq_funcs[queue->queue_type];
>>>> -			userq_funcs->unmap(queue);
>>>> -			/* just mark all queues as hung at this point.
>>>> -			 * if unmap succeeds, we could map again
>>>> -			 * in amdgpu_userq_post_reset() if vram is not lost
>>>> -			 */
>>>> -			queue->state = AMDGPU_USERQ_STATE_HUNG;
>>>> -			amdgpu_userq_fence_driver_force_completion(queue);
>>>> -		}
>>>> +		if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
>>>> +			continue;
>>> If the queue is in prempt state and if at that time we are in this function we should still be doing force completion for work in those queue else the waiters will keep waiting.
>>>
>>>> +
>>>> +		userq_funcs = adev->userq_funcs[queue->queue_type];
>>>> +		userq_funcs->unmap(queue);
>>> GPU is already hung if we are here and observation is we are unable to unmap as we have tried to reset via the fw and that failed to atleast thats what i have seen. Could we skip unmap ???
>>>> +		/* just mark all queues as hung at this point.
>>>> +		 * if unmap succeeds, we could map again
>>>> +		 * in amdgpu_userq_post_reset() if vram is not lost
>>>> +		 */
>>>> +		queue->state = AMDGPU_USERQ_STATE_HUNG;
>>>> +		amdgpu_userq_fence_driver_force_completion(queue);
>>> we should be calling completion irrespective of queue state here. The GPU atleast the queue is hung and fw has failed to reset. We have to release the fences by foce completion.
>> Yeah, I agree completely with those require comments I has similar thoughts while going over this.
>>
>> I'm only driven by bugs here and my goal with the patch was to remove the obvious deadlock in the function.
>>
>> Somebody needs to sit down and go over the whole handling for GPU resets and make sure it plays nicely with user queues.
>>
>> Regards,
>> Christian.
> For now this is what i think we could do.
> In the loop: If a queue is mapped -> try unmap
> 
> Irrespective of the queue state:
> Set queue to HUNG and call amdgpu_userq_fence_driver_force_completion

How do we protect the queue state? We don't have a lock for that and in a GPU reset you can't grab things like the userq_lock.

Could we call unmap while holding the doorbell XA lock or would that clash with something?

Regards,
Christian.

> 
> Regards
> Sunil Khatri 
>>> Regards
>>> Sunil Khatri
>>>
>>>>  	}
>>>>  }
>>>>  

Re: [PATCH 02/11] drm/amdgpu: remove deadlocks from amdgpu_userq_pre_reset

[Khatri, Sunil]

On 22-04-2026 12:54 pm, Christian König wrote:
> On 4/22/26 09:19, Khatri, Sunil wrote:
>> On 22-04-2026 12:43 pm, Christian König wrote:
>>> On 4/22/26 06:53, Khatri, Sunil wrote:
>>>> On 21-04-2026 06:25 pm, Christian König wrote:
>>>>> The purpose of a GPU reset is to make sure that fence can be signaled
>>>>> again and the signal and resume workers can make progress again.
>>>>>
>>>>> So waiting for the resume worker or any fence in the GPU reset path is
>>>>> just utterly nonsense.
>>>>>
>>>>> Signed-off-by: Christian König <christian.koenig@amd.com>
>>>>> ---
>>>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 26 +++++++++++------------
>>>>>   1 file changed, 12 insertions(+), 14 deletions(-)
>>>>>
>>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>>>>> index 8f48520cb822..b632bc3c952b 100644
>>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>>>>> @@ -1496,23 +1496,21 @@ void amdgpu_userq_pre_reset(struct amdgpu_device *adev)
>>>>>   {
>>>>>   	const struct amdgpu_userq_funcs *userq_funcs;
>>>>>   	struct amdgpu_usermode_queue *queue;
>>>>> -	struct amdgpu_userq_mgr *uqm;
>>>>>   	unsigned long queue_id;
>>>>>   
>>>>> +	/* TODO: We probably need a new lock for the queue state */
>>>>>   	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
>>>>> -		uqm = queue->userq_mgr;
>>>>> -		cancel_delayed_work_sync(&uqm->resume_work);
>>>>> -		if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
>>>>> -			amdgpu_userq_wait_for_last_fence(queue);
>>>>> -			userq_funcs = adev->userq_funcs[queue->queue_type];
>>>>> -			userq_funcs->unmap(queue);
>>>>> -			/* just mark all queues as hung at this point.
>>>>> -			 * if unmap succeeds, we could map again
>>>>> -			 * in amdgpu_userq_post_reset() if vram is not lost
>>>>> -			 */
>>>>> -			queue->state = AMDGPU_USERQ_STATE_HUNG;
>>>>> -			amdgpu_userq_fence_driver_force_completion(queue);
>>>>> -		}
>>>>> +		if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
>>>>> +			continue;
>>>> If the queue is in prempt state and if at that time we are in this function we should still be doing force completion for work in those queue else the waiters will keep waiting.
>>>>
>>>>> +
>>>>> +		userq_funcs = adev->userq_funcs[queue->queue_type];
>>>>> +		userq_funcs->unmap(queue);
>>>> GPU is already hung if we are here and observation is we are unable to unmap as we have tried to reset via the fw and that failed to atleast thats what i have seen. Could we skip unmap ???
>>>>> +		/* just mark all queues as hung at this point.
>>>>> +		 * if unmap succeeds, we could map again
>>>>> +		 * in amdgpu_userq_post_reset() if vram is not lost
>>>>> +		 */
>>>>> +		queue->state = AMDGPU_USERQ_STATE_HUNG;
>>>>> +		amdgpu_userq_fence_driver_force_completion(queue);
>>>> we should be calling completion irrespective of queue state here. The GPU atleast the queue is hung and fw has failed to reset. We have to release the fences by foce completion.
>>> Yeah, I agree completely with those require comments I has similar thoughts while going over this.
>>>
>>> I'm only driven by bugs here and my goal with the patch was to remove the obvious deadlock in the function.
>>>
>>> Somebody needs to sit down and go over the whole handling for GPU resets and make sure it plays nicely with user queues.
>>>
>>> Regards,
>>> Christian.
>> For now this is what i think we could do.
>> In the loop: If a queue is mapped -> try unmap
>>
>> Irrespective of the queue state:
>> Set queue to HUNG and call amdgpu_userq_fence_driver_force_completion
> How do we protect the queue state? We don't have a lock for that and in a GPU reset you can't grab things like the userq_lock.

It would be best and i thought of this too earlier to have a per queue 
or per queue manager lock and which is for setting up the state of 
queue...That will help a lot too in the all the hw function like prempt 
resume etc that we call as many times lock is only needed for state 
change than anything else... If you want i can look into that later once 
these patches land up or you decide if you want to fix it now.

Regards
Sunil Khatri

>
> Could we call unmap while holding the doorbell XA lock or would that clash with something?
>
> Regards,
> Christian.
>
>> Regards
>> Sunil Khatri
>>>> Regards
>>>> Sunil Khatri
>>>>
>>>>>   	}
>>>>>   }
>>>>>   

Re: [PATCH 03/11] drm/amdgpu: nuke amdgpu_userq_fence_free

[Khatri, Sunil]

[-- Attachment #1: Type: text/plain, Size: 8467 bytes --]


On 21-04-2026 06:25 pm, Christian König wrote:
> As preparation for independent fences remove the function and do all of
> it's cleanup directly after signaling.
>
> Signed-off-by: Christian König<christian.koenig@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c       | 13 +--
>   .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 79 +++++++------------
>   .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.h   |  3 -
>   3 files changed, 31 insertions(+), 64 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index c6546a858597..1b15b51dc3f4 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -3162,11 +3162,7 @@ static int __init amdgpu_init(void)
>   
>   	r = amdgpu_sync_init();
>   	if (r)
> -		goto error_sync;
> -
> -	r = amdgpu_userq_fence_slab_init();
> -	if (r)
> -		goto error_fence;
> +		return r;
>   
>   	amdgpu_register_atpx_handler();
>   	amdgpu_acpi_detect();
> @@ -3182,12 +3178,6 @@ static int __init amdgpu_init(void)
>   
>   	/* let modprobe override vga console setting */
>   	return pci_register_driver(&amdgpu_kms_pci_driver);
> -
> -error_fence:
> -	amdgpu_sync_fini();
> -
> -error_sync:
> -	return r;
>   }
>   
>   static void __exit amdgpu_exit(void)
> @@ -3197,7 +3187,6 @@ static void __exit amdgpu_exit(void)
>   	amdgpu_unregister_atpx_handler();
>   	amdgpu_acpi_release();
>   	amdgpu_sync_fini();
> -	amdgpu_userq_fence_slab_fini();
>   	mmu_notifier_synchronize();
>   	amdgpu_xcp_drv_release();
>   }
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> index a58342c2ac44..909bdccc2a92 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> @@ -32,29 +32,9 @@
>   #include "amdgpu.h"
>   #include "amdgpu_userq_fence.h"
>   
> -static const struct dma_fence_ops amdgpu_userq_fence_ops;
> -static struct kmem_cache *amdgpu_userq_fence_slab;
> -
>   #define AMDGPU_USERQ_MAX_HANDLES	(1U << 16)
>   
> -int amdgpu_userq_fence_slab_init(void)
> -{
> -	amdgpu_userq_fence_slab = kmem_cache_create("amdgpu_userq_fence",
> -						    sizeof(struct amdgpu_userq_fence),
> -						    0,
> -						    SLAB_HWCACHE_ALIGN,
> -						    NULL);
Are we not having benefit enough to continue create a cache here ? If 
that is fine that LGTM,

Acked-by: Sunil Khatri <sunil.khatri@amd.com>

Regards
Sunil



> -	if (!amdgpu_userq_fence_slab)
> -		return -ENOMEM;
> -
> -	return 0;
> -}
> -
> -void amdgpu_userq_fence_slab_fini(void)
> -{
> -	rcu_barrier();
> -	kmem_cache_destroy(amdgpu_userq_fence_slab);
> -}
> +static const struct dma_fence_ops amdgpu_userq_fence_ops;
>   
>   static inline struct amdgpu_userq_fence *to_amdgpu_userq_fence(struct dma_fence *f)
>   {
> @@ -146,12 +126,18 @@ amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq)
>   }
>   
>   static void
> -amdgpu_userq_fence_put_fence_drv_array(struct amdgpu_userq_fence *userq_fence)
> +amdgpu_userq_fence_put_fence_drv_refs(struct amdgpu_userq_fence *userq_fence)
>   {
>   	unsigned long i;
> +
>   	for (i = 0; i < userq_fence->fence_drv_array_count; i++)
>   		amdgpu_userq_fence_driver_put(userq_fence->fence_drv_array[i]);
>   	userq_fence->fence_drv_array_count = 0;
> +	kfree(userq_fence->fence_drv_array);
> +	userq_fence->fence_drv_array = NULL;
> +
> +	amdgpu_userq_fence_driver_put(userq_fence->fence_drv);
> +	userq_fence->fence_drv = NULL;
>   }
>   
>   void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
> @@ -181,10 +167,11 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
>   		fence = &userq_fence->base;
>   		list_del_init(&userq_fence->link);
>   		dma_fence_signal(fence);
> -		/* Drop fence_drv_array outside fence_list_lock
> +		/*
> +		 * Drop fence_drv_array outside fence_list_lock
>   		 * to avoid the recursion lock.
>   		 */
> -		amdgpu_userq_fence_put_fence_drv_array(userq_fence);
> +		amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
>   		dma_fence_put(fence);
>   	}
>   
> @@ -231,7 +218,7 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
>   
>   static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
>   {
> -	*userq_fence = kmem_cache_alloc(amdgpu_userq_fence_slab, GFP_ATOMIC);
> +	*userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
>   	return *userq_fence ? 0 : -ENOMEM;
>   }
>   
> @@ -299,7 +286,7 @@ static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
>   	spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
>   
>   	if (signaled)
> -		amdgpu_userq_fence_put_fence_drv_array(userq_fence);
> +		amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
>   
>   	*f = fence;
>   
> @@ -333,29 +320,10 @@ static bool amdgpu_userq_fence_signaled(struct dma_fence *f)
>   	return false;
>   }
>   
> -static void amdgpu_userq_fence_free(struct rcu_head *rcu)
> -{
> -	struct dma_fence *fence = container_of(rcu, struct dma_fence, rcu);
> -	struct amdgpu_userq_fence *userq_fence = to_amdgpu_userq_fence(fence);
> -	struct amdgpu_userq_fence_driver *fence_drv = userq_fence->fence_drv;
> -
> -	/* Release the fence driver reference */
> -	amdgpu_userq_fence_driver_put(fence_drv);
> -
> -	kvfree(userq_fence->fence_drv_array);
> -	kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
> -}
> -
> -static void amdgpu_userq_fence_release(struct dma_fence *f)
> -{
> -	call_rcu(&f->rcu, amdgpu_userq_fence_free);
> -}
> -
>   static const struct dma_fence_ops amdgpu_userq_fence_ops = {
>   	.get_driver_name = amdgpu_userq_fence_get_driver_name,
>   	.get_timeline_name = amdgpu_userq_fence_get_timeline_name,
>   	.signaled = amdgpu_userq_fence_signaled,
> -	.release = amdgpu_userq_fence_release,
>   };
>   
>   /**
> @@ -546,7 +514,7 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>   	r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
>   	if (r) {
>   		mutex_unlock(&userq_mgr->userq_mutex);
> -		kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
> +		kfree(userq_fence);
>   		goto put_gobj_write;
>   	}
>   
> @@ -871,6 +839,7 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>   	for (i = 0, cnt = 0; i < num_fences; i++) {
>   		struct amdgpu_userq_fence_driver *fence_drv;
>   		struct amdgpu_userq_fence *userq_fence;
> +		unsigned long flags;
>   		u32 index;
>   
>   		userq_fence = to_amdgpu_userq_fence(fences[i]);
> @@ -886,7 +855,19 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>   			continue;
>   		}
>   
> +		spin_lock_irqsave(userq_fence->base.lock, flags);
> +		if (dma_fence_is_signaled_locked(&userq_fence->base)) {
> +			/*
> +			 * It is possible that fence is already signaled and the
> +			 * fence_drv now NULL, just skip over such fences.
> +			*/
> +			spin_unlock_irqrestore(userq_fence->base.lock, flags);
> +			continue;
> +		}
>   		fence_drv = userq_fence->fence_drv;
> +		amdgpu_userq_fence_driver_get(fence_drv);
> +		spin_unlock_irqrestore(userq_fence->base.lock, flags);
> +
>   		/*
>   		 * We need to make sure the user queue release their reference
>   		 * to the fence drivers at some point before queue destruction.
> @@ -895,10 +876,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>   		 */
>   		r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
>   			     xa_limit_32b, GFP_KERNEL);
> -		if (r)
> +		if (r) {
> +			amdgpu_userq_fence_driver_put(fence_drv);
>   			goto put_waitq;
> -
> -		amdgpu_userq_fence_driver_get(fence_drv);
> +		}
>   
>   		/* Store drm syncobj's gpu va address and value */
>   		fence_info[cnt].va = fence_drv->va;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> index d56246ad8c26..d355a0eecc07 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> @@ -58,9 +58,6 @@ struct amdgpu_userq_fence_driver {
>   	char timeline_name[TASK_COMM_LEN];
>   };
>   
> -int amdgpu_userq_fence_slab_init(void);
> -void amdgpu_userq_fence_slab_fini(void);
> -
>   void amdgpu_userq_fence_driver_get(struct amdgpu_userq_fence_driver *fence_drv);
>   void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv);
>   int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,

[-- Attachment #2: Type: text/html, Size: 8894 bytes --]

Re: [PATCH 03/11] drm/amdgpu: nuke amdgpu_userq_fence_free

[Christian König]

On 4/22/26 10:29, Khatri, Sunil wrote:
> 
> On 21-04-2026 06:25 pm, Christian König wrote:
>> As preparation for independent fences remove the function and do all of
>> it's cleanup directly after signaling.
>>
>> Signed-off-by: Christian König <christian.koenig@amd.com>
>> ---
>>  drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c       | 13 +--
>>  .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 79 +++++++------------
>>  .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.h   |  3 -
>>  3 files changed, 31 insertions(+), 64 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>> index c6546a858597..1b15b51dc3f4 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>> @@ -3162,11 +3162,7 @@ static int __init amdgpu_init(void)
>>  
>>  	r = amdgpu_sync_init();
>>  	if (r)
>> -		goto error_sync;
>> -
>> -	r = amdgpu_userq_fence_slab_init();
>> -	if (r)
>> -		goto error_fence;
>> +		return r;
>>  
>>  	amdgpu_register_atpx_handler();
>>  	amdgpu_acpi_detect();
>> @@ -3182,12 +3178,6 @@ static int __init amdgpu_init(void)
>>  
>>  	/* let modprobe override vga console setting */
>>  	return pci_register_driver(&amdgpu_kms_pci_driver);
>> -
>> -error_fence:
>> -	amdgpu_sync_fini();
>> -
>> -error_sync:
>> -	return r;
>>  }
>>  
>>  static void __exit amdgpu_exit(void)
>> @@ -3197,7 +3187,6 @@ static void __exit amdgpu_exit(void)
>>  	amdgpu_unregister_atpx_handler();
>>  	amdgpu_acpi_release();
>>  	amdgpu_sync_fini();
>> -	amdgpu_userq_fence_slab_fini();
>>  	mmu_notifier_synchronize();
>>  	amdgpu_xcp_drv_release();
>>  }
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>> index a58342c2ac44..909bdccc2a92 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>> @@ -32,29 +32,9 @@
>>  #include "amdgpu.h"
>>  #include "amdgpu_userq_fence.h"
>>  
>> -static const struct dma_fence_ops amdgpu_userq_fence_ops;
>> -static struct kmem_cache *amdgpu_userq_fence_slab;
>> -
>>  #define AMDGPU_USERQ_MAX_HANDLES	(1U << 16)
>>  
>> -int amdgpu_userq_fence_slab_init(void)
>> -{
>> -	amdgpu_userq_fence_slab = kmem_cache_create("amdgpu_userq_fence",
>> -						    sizeof(struct amdgpu_userq_fence),
>> -						    0,
>> -						    SLAB_HWCACHE_ALIGN,
>> -						    NULL);
> Are we not having benefit enough to continue create a cache here ? If that is fine that LGTM, 

Using all those kmem_cache instances was a bad idea to begin with.

See the idea of a kmem_cache is to reduce the number of CPU cache lines and memory you need for certain number of objects when the object size is not a power of two.

So for exampe two objects with 96 bytes only take 3 cache lines and 192 bytes instead of 256 bytes and 4 cache lines.

But that difference is so marginally for most use cases that you absolutely don't need it.

Regards,
Christian.

> 
> Acked-by: Sunil Khatri <sunil.khatri@amd.com>
> 
> Regards
> Sunil
> 
> 
> 
>> -	if (!amdgpu_userq_fence_slab)
>> -		return -ENOMEM;
>> -
>> -	return 0;
>> -}
>> -
>> -void amdgpu_userq_fence_slab_fini(void)
>> -{
>> -	rcu_barrier();
>> -	kmem_cache_destroy(amdgpu_userq_fence_slab);
>> -}
>> +static const struct dma_fence_ops amdgpu_userq_fence_ops;
>>  
>>  static inline struct amdgpu_userq_fence *to_amdgpu_userq_fence(struct dma_fence *f)
>>  {
>> @@ -146,12 +126,18 @@ amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq)
>>  }
>>  
>>  static void
>> -amdgpu_userq_fence_put_fence_drv_array(struct amdgpu_userq_fence *userq_fence)
>> +amdgpu_userq_fence_put_fence_drv_refs(struct amdgpu_userq_fence *userq_fence)
>>  {
>>  	unsigned long i;
>> +
>>  	for (i = 0; i < userq_fence->fence_drv_array_count; i++)
>>  		amdgpu_userq_fence_driver_put(userq_fence->fence_drv_array[i]);
>>  	userq_fence->fence_drv_array_count = 0;
>> +	kfree(userq_fence->fence_drv_array);
>> +	userq_fence->fence_drv_array = NULL;
>> +
>> +	amdgpu_userq_fence_driver_put(userq_fence->fence_drv);
>> +	userq_fence->fence_drv = NULL;
>>  }
>>  
>>  void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
>> @@ -181,10 +167,11 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
>>  		fence = &userq_fence->base;
>>  		list_del_init(&userq_fence->link);
>>  		dma_fence_signal(fence);
>> -		/* Drop fence_drv_array outside fence_list_lock
>> +		/*
>> +		 * Drop fence_drv_array outside fence_list_lock
>>  		 * to avoid the recursion lock.
>>  		 */
>> -		amdgpu_userq_fence_put_fence_drv_array(userq_fence);
>> +		amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
>>  		dma_fence_put(fence);
>>  	}
>>  
>> @@ -231,7 +218,7 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
>>  
>>  static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
>>  {
>> -	*userq_fence = kmem_cache_alloc(amdgpu_userq_fence_slab, GFP_ATOMIC);
>> +	*userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
>>  	return *userq_fence ? 0 : -ENOMEM;
>>  }
>>  
>> @@ -299,7 +286,7 @@ static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
>>  	spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
>>  
>>  	if (signaled)
>> -		amdgpu_userq_fence_put_fence_drv_array(userq_fence);
>> +		amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
>>  
>>  	*f = fence;
>>  
>> @@ -333,29 +320,10 @@ static bool amdgpu_userq_fence_signaled(struct dma_fence *f)
>>  	return false;
>>  }
>>  
>> -static void amdgpu_userq_fence_free(struct rcu_head *rcu)
>> -{
>> -	struct dma_fence *fence = container_of(rcu, struct dma_fence, rcu);
>> -	struct amdgpu_userq_fence *userq_fence = to_amdgpu_userq_fence(fence);
>> -	struct amdgpu_userq_fence_driver *fence_drv = userq_fence->fence_drv;
>> -
>> -	/* Release the fence driver reference */
>> -	amdgpu_userq_fence_driver_put(fence_drv);
>> -
>> -	kvfree(userq_fence->fence_drv_array);
>> -	kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
>> -}
>> -
>> -static void amdgpu_userq_fence_release(struct dma_fence *f)
>> -{
>> -	call_rcu(&f->rcu, amdgpu_userq_fence_free);
>> -}
>> -
>>  static const struct dma_fence_ops amdgpu_userq_fence_ops = {
>>  	.get_driver_name = amdgpu_userq_fence_get_driver_name,
>>  	.get_timeline_name = amdgpu_userq_fence_get_timeline_name,
>>  	.signaled = amdgpu_userq_fence_signaled,
>> -	.release = amdgpu_userq_fence_release,
>>  };
>>  
>>  /**
>> @@ -546,7 +514,7 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>>  	r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
>>  	if (r) {
>>  		mutex_unlock(&userq_mgr->userq_mutex);
>> -		kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
>> +		kfree(userq_fence);
>>  		goto put_gobj_write;
>>  	}
>>  
>> @@ -871,6 +839,7 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>>  	for (i = 0, cnt = 0; i < num_fences; i++) {
>>  		struct amdgpu_userq_fence_driver *fence_drv;
>>  		struct amdgpu_userq_fence *userq_fence;
>> +		unsigned long flags;
>>  		u32 index;
>>  
>>  		userq_fence = to_amdgpu_userq_fence(fences[i]);
>> @@ -886,7 +855,19 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>>  			continue;
>>  		}
>>  
>> +		spin_lock_irqsave(userq_fence->base.lock, flags);
>> +		if (dma_fence_is_signaled_locked(&userq_fence->base)) {
>> +			/*
>> +			 * It is possible that fence is already signaled and the
>> +			 * fence_drv now NULL, just skip over such fences.
>> +			*/
>> +			spin_unlock_irqrestore(userq_fence->base.lock, flags);
>> +			continue;
>> +		}
>>  		fence_drv = userq_fence->fence_drv;
>> +		amdgpu_userq_fence_driver_get(fence_drv);
>> +		spin_unlock_irqrestore(userq_fence->base.lock, flags);
>> +
>>  		/*
>>  		 * We need to make sure the user queue release their reference
>>  		 * to the fence drivers at some point before queue destruction.
>> @@ -895,10 +876,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>>  		 */
>>  		r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
>>  			     xa_limit_32b, GFP_KERNEL);
>> -		if (r)
>> +		if (r) {
>> +			amdgpu_userq_fence_driver_put(fence_drv);
>>  			goto put_waitq;
>> -
>> -		amdgpu_userq_fence_driver_get(fence_drv);
>> +		}
>>  
>>  		/* Store drm syncobj's gpu va address and value */
>>  		fence_info[cnt].va = fence_drv->va;
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
>> index d56246ad8c26..d355a0eecc07 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
>> @@ -58,9 +58,6 @@ struct amdgpu_userq_fence_driver {
>>  	char timeline_name[TASK_COMM_LEN];
>>  };
>>  
>> -int amdgpu_userq_fence_slab_init(void);
>> -void amdgpu_userq_fence_slab_fini(void);
>> -
>>  void amdgpu_userq_fence_driver_get(struct amdgpu_userq_fence_driver *fence_drv);
>>  void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv);
>>  int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,

Re: [PATCH 03/11] drm/amdgpu: nuke amdgpu_userq_fence_free

[Khatri, Sunil]

On 22-04-2026 02:56 pm, Christian König wrote:
> On 4/22/26 10:29, Khatri, Sunil wrote:
>> On 21-04-2026 06:25 pm, Christian König wrote:
>>> As preparation for independent fences remove the function and do all of
>>> it's cleanup directly after signaling.
>>>
>>> Signed-off-by: Christian König <christian.koenig@amd.com>
>>> ---
>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c       | 13 +--
>>>   .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 79 +++++++------------
>>>   .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.h   |  3 -
>>>   3 files changed, 31 insertions(+), 64 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>>> index c6546a858597..1b15b51dc3f4 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>>> @@ -3162,11 +3162,7 @@ static int __init amdgpu_init(void)
>>>   
>>>   	r = amdgpu_sync_init();
>>>   	if (r)
>>> -		goto error_sync;
>>> -
>>> -	r = amdgpu_userq_fence_slab_init();
>>> -	if (r)
>>> -		goto error_fence;
>>> +		return r;
>>>   
>>>   	amdgpu_register_atpx_handler();
>>>   	amdgpu_acpi_detect();
>>> @@ -3182,12 +3178,6 @@ static int __init amdgpu_init(void)
>>>   
>>>   	/* let modprobe override vga console setting */
>>>   	return pci_register_driver(&amdgpu_kms_pci_driver);
>>> -
>>> -error_fence:
>>> -	amdgpu_sync_fini();
>>> -
>>> -error_sync:
>>> -	return r;
>>>   }
>>>   
>>>   static void __exit amdgpu_exit(void)
>>> @@ -3197,7 +3187,6 @@ static void __exit amdgpu_exit(void)
>>>   	amdgpu_unregister_atpx_handler();
>>>   	amdgpu_acpi_release();
>>>   	amdgpu_sync_fini();
>>> -	amdgpu_userq_fence_slab_fini();
>>>   	mmu_notifier_synchronize();
>>>   	amdgpu_xcp_drv_release();
>>>   }
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>>> index a58342c2ac44..909bdccc2a92 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>>> @@ -32,29 +32,9 @@
>>>   #include "amdgpu.h"
>>>   #include "amdgpu_userq_fence.h"
>>>   
>>> -static const struct dma_fence_ops amdgpu_userq_fence_ops;
>>> -static struct kmem_cache *amdgpu_userq_fence_slab;
>>> -
>>>   #define AMDGPU_USERQ_MAX_HANDLES	(1U << 16)
>>>   
>>> -int amdgpu_userq_fence_slab_init(void)
>>> -{
>>> -	amdgpu_userq_fence_slab = kmem_cache_create("amdgpu_userq_fence",
>>> -						    sizeof(struct amdgpu_userq_fence),
>>> -						    0,
>>> -						    SLAB_HWCACHE_ALIGN,
>>> -						    NULL);
>> Are we not having benefit enough to continue create a cache here ? If that is fine that LGTM,
> Using all those kmem_cache instances was a bad idea to begin with.
>
> See the idea of a kmem_cache is to reduce the number of CPU cache lines and memory you need for certain number of objects when the object size is not a power of two.
>
> So for exampe two objects with 96 bytes only take 3 cache lines and 192 bytes instead of 256 bytes and 4 cache lines.
>
> But that difference is so marginally for most use cases that you absolutely don't need it.

Thanks for the explanation. Also if i am not wrong in cases where last 
no of such objects are often created and deleted kmem_cache helps in 
reuse and helps with internal fragmentation too.

Regards

Sunil Khatri

>
> Regards,
> Christian.
>
>> Acked-by: Sunil Khatri <sunil.khatri@amd.com>
>>
>> Regards
>> Sunil
>>
>>
>>
>>> -	if (!amdgpu_userq_fence_slab)
>>> -		return -ENOMEM;
>>> -
>>> -	return 0;
>>> -}
>>> -
>>> -void amdgpu_userq_fence_slab_fini(void)
>>> -{
>>> -	rcu_barrier();
>>> -	kmem_cache_destroy(amdgpu_userq_fence_slab);
>>> -}
>>> +static const struct dma_fence_ops amdgpu_userq_fence_ops;
>>>   
>>>   static inline struct amdgpu_userq_fence *to_amdgpu_userq_fence(struct dma_fence *f)
>>>   {
>>> @@ -146,12 +126,18 @@ amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq)
>>>   }
>>>   
>>>   static void
>>> -amdgpu_userq_fence_put_fence_drv_array(struct amdgpu_userq_fence *userq_fence)
>>> +amdgpu_userq_fence_put_fence_drv_refs(struct amdgpu_userq_fence *userq_fence)
>>>   {
>>>   	unsigned long i;
>>> +
>>>   	for (i = 0; i < userq_fence->fence_drv_array_count; i++)
>>>   		amdgpu_userq_fence_driver_put(userq_fence->fence_drv_array[i]);
>>>   	userq_fence->fence_drv_array_count = 0;
>>> +	kfree(userq_fence->fence_drv_array);
>>> +	userq_fence->fence_drv_array = NULL;
>>> +
>>> +	amdgpu_userq_fence_driver_put(userq_fence->fence_drv);
>>> +	userq_fence->fence_drv = NULL;
>>>   }
>>>   
>>>   void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
>>> @@ -181,10 +167,11 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
>>>   		fence = &userq_fence->base;
>>>   		list_del_init(&userq_fence->link);
>>>   		dma_fence_signal(fence);
>>> -		/* Drop fence_drv_array outside fence_list_lock
>>> +		/*
>>> +		 * Drop fence_drv_array outside fence_list_lock
>>>   		 * to avoid the recursion lock.
>>>   		 */
>>> -		amdgpu_userq_fence_put_fence_drv_array(userq_fence);
>>> +		amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
>>>   		dma_fence_put(fence);
>>>   	}
>>>   
>>> @@ -231,7 +218,7 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
>>>   
>>>   static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
>>>   {
>>> -	*userq_fence = kmem_cache_alloc(amdgpu_userq_fence_slab, GFP_ATOMIC);
>>> +	*userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
>>>   	return *userq_fence ? 0 : -ENOMEM;
>>>   }
>>>   
>>> @@ -299,7 +286,7 @@ static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
>>>   	spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
>>>   
>>>   	if (signaled)
>>> -		amdgpu_userq_fence_put_fence_drv_array(userq_fence);
>>> +		amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
>>>   
>>>   	*f = fence;
>>>   
>>> @@ -333,29 +320,10 @@ static bool amdgpu_userq_fence_signaled(struct dma_fence *f)
>>>   	return false;
>>>   }
>>>   
>>> -static void amdgpu_userq_fence_free(struct rcu_head *rcu)
>>> -{
>>> -	struct dma_fence *fence = container_of(rcu, struct dma_fence, rcu);
>>> -	struct amdgpu_userq_fence *userq_fence = to_amdgpu_userq_fence(fence);
>>> -	struct amdgpu_userq_fence_driver *fence_drv = userq_fence->fence_drv;
>>> -
>>> -	/* Release the fence driver reference */
>>> -	amdgpu_userq_fence_driver_put(fence_drv);
>>> -
>>> -	kvfree(userq_fence->fence_drv_array);
>>> -	kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
>>> -}
>>> -
>>> -static void amdgpu_userq_fence_release(struct dma_fence *f)
>>> -{
>>> -	call_rcu(&f->rcu, amdgpu_userq_fence_free);
>>> -}
>>> -
>>>   static const struct dma_fence_ops amdgpu_userq_fence_ops = {
>>>   	.get_driver_name = amdgpu_userq_fence_get_driver_name,
>>>   	.get_timeline_name = amdgpu_userq_fence_get_timeline_name,
>>>   	.signaled = amdgpu_userq_fence_signaled,
>>> -	.release = amdgpu_userq_fence_release,
>>>   };
>>>   
>>>   /**
>>> @@ -546,7 +514,7 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>>>   	r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
>>>   	if (r) {
>>>   		mutex_unlock(&userq_mgr->userq_mutex);
>>> -		kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
>>> +		kfree(userq_fence);
>>>   		goto put_gobj_write;
>>>   	}
>>>   
>>> @@ -871,6 +839,7 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>>>   	for (i = 0, cnt = 0; i < num_fences; i++) {
>>>   		struct amdgpu_userq_fence_driver *fence_drv;
>>>   		struct amdgpu_userq_fence *userq_fence;
>>> +		unsigned long flags;
>>>   		u32 index;
>>>   
>>>   		userq_fence = to_amdgpu_userq_fence(fences[i]);
>>> @@ -886,7 +855,19 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>>>   			continue;
>>>   		}
>>>   
>>> +		spin_lock_irqsave(userq_fence->base.lock, flags);
>>> +		if (dma_fence_is_signaled_locked(&userq_fence->base)) {
>>> +			/*
>>> +			 * It is possible that fence is already signaled and the
>>> +			 * fence_drv now NULL, just skip over such fences.
>>> +			*/
>>> +			spin_unlock_irqrestore(userq_fence->base.lock, flags);
>>> +			continue;
>>> +		}
>>>   		fence_drv = userq_fence->fence_drv;
>>> +		amdgpu_userq_fence_driver_get(fence_drv);
>>> +		spin_unlock_irqrestore(userq_fence->base.lock, flags);
>>> +
>>>   		/*
>>>   		 * We need to make sure the user queue release their reference
>>>   		 * to the fence drivers at some point before queue destruction.
>>> @@ -895,10 +876,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>>>   		 */
>>>   		r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
>>>   			     xa_limit_32b, GFP_KERNEL);
>>> -		if (r)
>>> +		if (r) {
>>> +			amdgpu_userq_fence_driver_put(fence_drv);
>>>   			goto put_waitq;
>>> -
>>> -		amdgpu_userq_fence_driver_get(fence_drv);
>>> +		}
>>>   
>>>   		/* Store drm syncobj's gpu va address and value */
>>>   		fence_info[cnt].va = fence_drv->va;
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
>>> index d56246ad8c26..d355a0eecc07 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
>>> @@ -58,9 +58,6 @@ struct amdgpu_userq_fence_driver {
>>>   	char timeline_name[TASK_COMM_LEN];
>>>   };
>>>   
>>> -int amdgpu_userq_fence_slab_init(void);
>>> -void amdgpu_userq_fence_slab_fini(void);
>>> -
>>>   void amdgpu_userq_fence_driver_get(struct amdgpu_userq_fence_driver *fence_drv);
>>>   void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv);
>>>   int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,

Re: [PATCH 04/11] drm/amdgpu: rework amdgpu_userq_signal_ioctl

[Khatri, Sunil]

On 21-04-2026 06:25 pm, Christian König wrote:
> This one was fortunately not looking so bad as the wait ioctl path, but
> there were still a few things which could be fixed/improved:
>
> 1. Allocating with GFP_ATOMIC was quite unecessary, we can do that
>     before taking the userq_lock.
> 2. Use a new mutex as protection for the fence_drv_xa so that we can do
>     memory allocations while holding it.
> 3. Starting the reset timer is unecessary when the fence is already
>     signaled when we create it.
> 4. Cleanup error handling, avoid trying to free the queue when we don't
>     even got one.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c     |   1 +
>   drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h     |  12 +
>   .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 220 ++++++++----------
>   3 files changed, 111 insertions(+), 122 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index b632bc3c952b..174190a77005 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -793,6 +793,7 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
>   	}
>   
>   	queue->doorbell_index = index;
> +	mutex_init(&queue->fence_drv_lock);
we do want to destroy the mutex in case queue creation fails ? RIght now 
amdgpu_userq_fence_driver_alloc fails we goto clean_mapping and not 
destroying the mutex which is done in amdgpu_userq_fence_driver_free.
goto needs to be modified i guess to handle it.

Apart from that LGTM. Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>

Regards
Sunil
>   	xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
>   	r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
>   	if (r) {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> index 675fe6395ac8..cb92789c1ed1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> @@ -66,6 +66,18 @@ struct amdgpu_usermode_queue {
>   	struct amdgpu_userq_obj	db_obj;
>   	struct amdgpu_userq_obj fw_obj;
>   	struct amdgpu_userq_obj wptr_obj;
> +
> +	/**
> +	 * @fence_drv_lock: Protecting @fence_drv_xa.
> +	 */
> +	struct mutex		fence_drv_lock;
> +
> +	/**
> +	 * @fence_drv_xa:
> +	 *
> +	 * References to the external fence drivers returned by wait_ioctl.
> +	 * Dropped on the next signaled dma_fence or queue destruction.
> +	 */
>   	struct xarray		fence_drv_xa;
>   	struct amdgpu_userq_fence_driver *fence_drv;
>   	struct dma_fence	*last_fence;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> index 909bdccc2a92..b0543fa257ed 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> @@ -121,6 +121,7 @@ amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq)
>   	userq->last_fence = NULL;
>   	amdgpu_userq_walk_and_drop_fence_drv(&userq->fence_drv_xa);
>   	xa_destroy(&userq->fence_drv_xa);
> +	mutex_destroy(&userq->fence_drv_lock);
>   	/* Drop the queue's ownership reference to fence_drv explicitly */
>   	amdgpu_userq_fence_driver_put(userq->fence_drv);
>   }
> @@ -216,81 +217,77 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
>   	kref_put(&fence_drv->refcount, amdgpu_userq_fence_driver_destroy);
>   }
>   
> -static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
> +static int amdgpu_userq_fence_alloc(struct amdgpu_usermode_queue *userq,
> +				    struct amdgpu_userq_fence **pfence)
>   {
> -	*userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
> -	return *userq_fence ? 0 : -ENOMEM;
> +	struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
> +	struct amdgpu_userq_fence *userq_fence;
> +	unsigned long count;
> +
> +	userq_fence = kmalloc(sizeof(*userq_fence), GFP_KERNEL);
> +	if (!userq_fence)
> +		return -ENOMEM;
> +
> +	/*
> +	 * Get the next unused entry, since we fill from the start this can be
> +	 * used as size to allocate the array.
> +	 */
> +	mutex_lock(&userq->fence_drv_lock);
> +	xa_find(&userq->fence_drv_xa, &count, ULONG_MAX, XA_FREE_MARK);
> +
> +	userq_fence->fence_drv_array = kvmalloc_array(count, sizeof(fence_drv),
> +						      GFP_KERNEL);
> +	if (!userq_fence->fence_drv_array) {
> +		mutex_unlock(&userq->fence_drv_lock);
> +		kfree(userq_fence);
> +		return -ENOMEM;
> +	}
> +
> +	userq_fence->fence_drv_array_count = count;
> +	xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array,
> +		   0, ULONG_MAX, count, XA_PRESENT);
> +	xa_destroy(&userq->fence_drv_xa);
> +
> +	mutex_unlock(&userq->fence_drv_lock);
> +
> +	userq_fence->fence_drv = fence_drv;
> +	amdgpu_userq_fence_driver_get(fence_drv);
> +
> +	*pfence = userq_fence;
> +	return 0;
>   }
>   
> -static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
> -				     struct amdgpu_userq_fence *userq_fence,
> -				     u64 seq, struct dma_fence **f)
> +static void amdgpu_userq_fence_init(struct amdgpu_usermode_queue *userq,
> +				    struct amdgpu_userq_fence *fence,
> +				    u64 seq)
>   {
> -	struct amdgpu_userq_fence_driver *fence_drv;
> -	struct dma_fence *fence;
> +	struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
>   	unsigned long flags;
>   	bool signaled = false;
>   
> -	fence_drv = userq->fence_drv;
> -	if (!fence_drv)
> -		return -EINVAL;
> -
> -	spin_lock_init(&userq_fence->lock);
> -	INIT_LIST_HEAD(&userq_fence->link);
> -	fence = &userq_fence->base;
> -	userq_fence->fence_drv = fence_drv;
> -
> -	dma_fence_init64(fence, &amdgpu_userq_fence_ops, &userq_fence->lock,
> +	spin_lock_init(&fence->lock);
> +	dma_fence_init64(&fence->base, &amdgpu_userq_fence_ops, &fence->lock,
>   			 fence_drv->context, seq);
>   
> -	amdgpu_userq_fence_driver_get(fence_drv);
> -	dma_fence_get(fence);
> -
> -	if (!xa_empty(&userq->fence_drv_xa)) {
> -		struct amdgpu_userq_fence_driver *stored_fence_drv;
> -		unsigned long index, count = 0;
> -		int i = 0;
> -
> -		xa_lock(&userq->fence_drv_xa);
> -		xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv)
> -			count++;
> -
> -		userq_fence->fence_drv_array =
> -			kvmalloc_array(count,
> -				       sizeof(struct amdgpu_userq_fence_driver *),
> -				       GFP_ATOMIC);
> -
> -		if (userq_fence->fence_drv_array) {
> -			xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv) {
> -				userq_fence->fence_drv_array[i] = stored_fence_drv;
> -				__xa_erase(&userq->fence_drv_xa, index);
> -				i++;
> -			}
> -		}
> -
> -		userq_fence->fence_drv_array_count = i;
> -		xa_unlock(&userq->fence_drv_xa);
> -	} else {
> -		userq_fence->fence_drv_array = NULL;
> -		userq_fence->fence_drv_array_count = 0;
> -	}
> +	/* Make sure the fence is visible to the hang detect worker */
> +	dma_fence_put(userq->last_fence);
> +	userq->last_fence = dma_fence_get(&fence->base);
>   
> -	/* Check if hardware has already processed the job */
> +	/* Check if hardware has already processed the fence */
>   	spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
> -	if (!dma_fence_is_signaled(fence)) {
> -		list_add_tail(&userq_fence->link, &fence_drv->fences);
> +	if (!dma_fence_is_signaled(&fence->base)) {
> +		dma_fence_get(&fence->base);
> +		list_add_tail(&fence->link, &fence_drv->fences);
>   	} else {
> +		INIT_LIST_HEAD(&fence->link);
>   		signaled = true;
> -		dma_fence_put(fence);
>   	}
>   	spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
>   
>   	if (signaled)
> -		amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
> -
> -	*f = fence;
> -
> -	return 0;
> +		amdgpu_userq_fence_put_fence_drv_refs(fence);
> +	else
> +		amdgpu_userq_start_hang_detect_work(userq);
>   }
>   
>   static const char *amdgpu_userq_fence_get_driver_name(struct dma_fence *f)
> @@ -392,11 +389,6 @@ static int amdgpu_userq_fence_read_wptr(struct amdgpu_device *adev,
>   	return r;
>   }
>   
> -static void amdgpu_userq_fence_cleanup(struct dma_fence *fence)
> -{
> -	dma_fence_put(fence);
> -}
> -
>   static void
>   amdgpu_userq_fence_driver_set_error(struct amdgpu_userq_fence *fence,
>   				    int error)
> @@ -440,13 +432,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>   	const unsigned int num_read_bo_handles = args->num_bo_read_handles;
>   	struct amdgpu_fpriv *fpriv = filp->driver_priv;
>   	struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
> +
>   	struct drm_gem_object **gobj_write, **gobj_read;
>   	u32 *syncobj_handles, num_syncobj_handles;
> -	struct amdgpu_userq_fence *userq_fence;
> -	struct amdgpu_usermode_queue *queue = NULL;
> -	struct drm_syncobj **syncobj = NULL;
> -	struct dma_fence *fence;
> +	struct amdgpu_usermode_queue *queue;
> +	struct amdgpu_userq_fence *fence;
> +	struct drm_syncobj **syncobj;
>   	struct drm_exec exec;
> +	void __user *ptr;
>   	int r, i, entry;
>   	u64 wptr;
>   
> @@ -458,13 +451,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>   		return -EINVAL;
>   
>   	num_syncobj_handles = args->num_syncobj_handles;
> -	syncobj_handles = memdup_array_user(u64_to_user_ptr(args->syncobj_handles),
> -					    num_syncobj_handles, sizeof(u32));
> +	ptr = u64_to_user_ptr(args->syncobj_handles);
> +	syncobj_handles = memdup_array_user(ptr, num_syncobj_handles,
> +					    sizeof(u32));
>   	if (IS_ERR(syncobj_handles))
>   		return PTR_ERR(syncobj_handles);
>   
> -	/* Array of pointers to the looked up syncobjs */
> -	syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj), GFP_KERNEL);
> +	syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj),
> +				GFP_KERNEL);
>   	if (!syncobj) {
>   		r = -ENOMEM;
>   		goto free_syncobj_handles;
> @@ -478,21 +472,17 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>   		}
>   	}
>   
> -	r = drm_gem_objects_lookup(filp,
> -				   u64_to_user_ptr(args->bo_read_handles),
> -				   num_read_bo_handles,
> -				   &gobj_read);
> +	ptr = u64_to_user_ptr(args->bo_read_handles);
> +	r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
>   	if (r)
>   		goto free_syncobj;
>   
> -	r = drm_gem_objects_lookup(filp,
> -				   u64_to_user_ptr(args->bo_write_handles),
> -				   num_write_bo_handles,
> +	ptr = u64_to_user_ptr(args->bo_write_handles);
> +	r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
>   				   &gobj_write);
>   	if (r)
>   		goto put_gobj_read;
>   
> -	/* Retrieve the user queue */
>   	queue = amdgpu_userq_get(userq_mgr, args->queue_id);
>   	if (!queue) {
>   		r = -ENOENT;
> @@ -501,73 +491,61 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>   
>   	r = amdgpu_userq_fence_read_wptr(adev, queue, &wptr);
>   	if (r)
> -		goto put_gobj_write;
> +		goto put_queue;
>   
> -	r = amdgpu_userq_fence_alloc(&userq_fence);
> +	r = amdgpu_userq_fence_alloc(queue, &fence);
>   	if (r)
> -		goto put_gobj_write;
> +		goto put_queue;
>   
>   	/* We are here means UQ is active, make sure the eviction fence is valid */
>   	amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
>   
> -	/* Create a new fence */
> -	r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
> -	if (r) {
> -		mutex_unlock(&userq_mgr->userq_mutex);
> -		kfree(userq_fence);
> -		goto put_gobj_write;
> -	}
> +	/* Create the new fence */
> +	amdgpu_userq_fence_init(queue, fence, wptr);
>   
> -	dma_fence_put(queue->last_fence);
> -	queue->last_fence = dma_fence_get(fence);
> -	amdgpu_userq_start_hang_detect_work(queue);
>   	mutex_unlock(&userq_mgr->userq_mutex);
>   
> +	/*
> +	 * This needs to come after the fence is created since
> +	 * amdgpu_userq_ensure_ev_fence() can't be called while holding the resv
> +	 * locks.
> +	 */
>   	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
>   		      (num_read_bo_handles + num_write_bo_handles));
>   
> -	/* Lock all BOs with retry handling */
>   	drm_exec_until_all_locked(&exec) {
> -		r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1);
> +		r = drm_exec_prepare_array(&exec, gobj_read,
> +					   num_read_bo_handles, 1);
>   		drm_exec_retry_on_contention(&exec);
> -		if (r) {
> -			amdgpu_userq_fence_cleanup(fence);
> +		if (r)
>   			goto exec_fini;
> -		}
>   
> -		r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1);
> +		r = drm_exec_prepare_array(&exec, gobj_write,
> +					   num_write_bo_handles, 1);
>   		drm_exec_retry_on_contention(&exec);
> -		if (r) {
> -			amdgpu_userq_fence_cleanup(fence);
> +		if (r)
>   			goto exec_fini;
> -		}
>   	}
>   
> -	for (i = 0; i < num_read_bo_handles; i++) {
> -		if (!gobj_read || !gobj_read[i]->resv)
> -			continue;
> -
> -		dma_resv_add_fence(gobj_read[i]->resv, fence,
> +	/* And publish the new fence in the BOs and syncobj */
> +	for (i = 0; i < num_read_bo_handles; i++)
> +		dma_resv_add_fence(gobj_read[i]->resv, &fence->base,
>   				   DMA_RESV_USAGE_READ);
> -	}
>   
> -	for (i = 0; i < num_write_bo_handles; i++) {
> -		if (!gobj_write || !gobj_write[i]->resv)
> -			continue;
> -
> -		dma_resv_add_fence(gobj_write[i]->resv, fence,
> +	for (i = 0; i < num_write_bo_handles; i++)
> +		dma_resv_add_fence(gobj_write[i]->resv, &fence->base,
>   				   DMA_RESV_USAGE_WRITE);
> -	}
>   
> -	/* Add the created fence to syncobj/BO's */
>   	for (i = 0; i < num_syncobj_handles; i++)
> -		drm_syncobj_replace_fence(syncobj[i], fence);
> +		drm_syncobj_replace_fence(syncobj[i], &fence->base);
>   
> +exec_fini:
>   	/* drop the reference acquired in fence creation function */
> -	dma_fence_put(fence);
> +	dma_fence_put(&fence->base);
>   
> -exec_fini:
>   	drm_exec_fini(&exec);
> +put_queue:
> +	amdgpu_userq_put(queue);
>   put_gobj_write:
>   	for (i = 0; i < num_write_bo_handles; i++)
>   		drm_gem_object_put(gobj_write[i]);
> @@ -578,15 +556,11 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>   	kvfree(gobj_read);
>   free_syncobj:
>   	while (entry-- > 0)
> -		if (syncobj[entry])
> -			drm_syncobj_put(syncobj[entry]);
> +		drm_syncobj_put(syncobj[entry]);
>   	kfree(syncobj);
>   free_syncobj_handles:
>   	kfree(syncobj_handles);
>   
> -	if (queue)
> -		amdgpu_userq_put(queue);
> -
>   	return r;
>   }
>   
> @@ -874,8 +848,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>   		 * Otherwise, we would gather those references until we don't
>   		 * have any more space left and crash.
>   		 */
> +		mutex_lock(&waitq->fence_drv_lock);
>   		r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
>   			     xa_limit_32b, GFP_KERNEL);
> +		mutex_unlock(&waitq->fence_drv_lock);
>   		if (r) {
>   			amdgpu_userq_fence_driver_put(fence_drv);
>   			goto put_waitq;

Re: [PATCH 03/11] drm/amdgpu: nuke amdgpu_userq_fence_free

[Christian König]

On 4/22/26 11:40, Khatri, Sunil wrote:
> 
> On 22-04-2026 02:56 pm, Christian König wrote:
>> On 4/22/26 10:29, Khatri, Sunil wrote:
>>> On 21-04-2026 06:25 pm, Christian König wrote:
>>>> As preparation for independent fences remove the function and do all of
>>>> it's cleanup directly after signaling.
>>>>
>>>> Signed-off-by: Christian König <christian.koenig@amd.com>
>>>> ---
>>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c       | 13 +--
>>>>   .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 79 +++++++------------
>>>>   .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.h   |  3 -
>>>>   3 files changed, 31 insertions(+), 64 deletions(-)
>>>>
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>>>> index c6546a858597..1b15b51dc3f4 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>>>> @@ -3162,11 +3162,7 @@ static int __init amdgpu_init(void)
>>>>         r = amdgpu_sync_init();
>>>>       if (r)
>>>> -        goto error_sync;
>>>> -
>>>> -    r = amdgpu_userq_fence_slab_init();
>>>> -    if (r)
>>>> -        goto error_fence;
>>>> +        return r;
>>>>         amdgpu_register_atpx_handler();
>>>>       amdgpu_acpi_detect();
>>>> @@ -3182,12 +3178,6 @@ static int __init amdgpu_init(void)
>>>>         /* let modprobe override vga console setting */
>>>>       return pci_register_driver(&amdgpu_kms_pci_driver);
>>>> -
>>>> -error_fence:
>>>> -    amdgpu_sync_fini();
>>>> -
>>>> -error_sync:
>>>> -    return r;
>>>>   }
>>>>     static void __exit amdgpu_exit(void)
>>>> @@ -3197,7 +3187,6 @@ static void __exit amdgpu_exit(void)
>>>>       amdgpu_unregister_atpx_handler();
>>>>       amdgpu_acpi_release();
>>>>       amdgpu_sync_fini();
>>>> -    amdgpu_userq_fence_slab_fini();
>>>>       mmu_notifier_synchronize();
>>>>       amdgpu_xcp_drv_release();
>>>>   }
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>>>> index a58342c2ac44..909bdccc2a92 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>>>> @@ -32,29 +32,9 @@
>>>>   #include "amdgpu.h"
>>>>   #include "amdgpu_userq_fence.h"
>>>>   -static const struct dma_fence_ops amdgpu_userq_fence_ops;
>>>> -static struct kmem_cache *amdgpu_userq_fence_slab;
>>>> -
>>>>   #define AMDGPU_USERQ_MAX_HANDLES    (1U << 16)
>>>>   -int amdgpu_userq_fence_slab_init(void)
>>>> -{
>>>> -    amdgpu_userq_fence_slab = kmem_cache_create("amdgpu_userq_fence",
>>>> -                            sizeof(struct amdgpu_userq_fence),
>>>> -                            0,
>>>> -                            SLAB_HWCACHE_ALIGN,
>>>> -                            NULL);
>>> Are we not having benefit enough to continue create a cache here ? If that is fine that LGTM,
>> Using all those kmem_cache instances was a bad idea to begin with.
>>
>> See the idea of a kmem_cache is to reduce the number of CPU cache lines and memory you need for certain number of objects when the object size is not a power of two.
>>
>> So for exampe two objects with 96 bytes only take 3 cache lines and 192 bytes instead of 256 bytes and 4 cache lines.
>>
>> But that difference is so marginally for most use cases that you absolutely don't need it.
> 
> Thanks for the explanation. Also if i am not wrong in cases where last no of such objects are often created and deleted kmem_cache helps in reuse and helps with internal fragmentation too.

Nope that's not correct. See kmalloc() and co also uses kmem_cache underneath, they just round up the size of the allocated object to the next power of two.

As long as you don't give special flags kmem_cache object storages are also shared among users, e.g. when one driver creates a kmem_cache for an 96 byte sized object and another driver does the same they will actually share their allocations which each other.

Regards,
Christian.

> 
> Regards
> 
> Sunil Khatri
> 
>>
>> Regards,
>> Christian.
>>
>>> Acked-by: Sunil Khatri <sunil.khatri@amd.com>
>>>
>>> Regards
>>> Sunil
>>>
>>>
>>>
>>>> -    if (!amdgpu_userq_fence_slab)
>>>> -        return -ENOMEM;
>>>> -
>>>> -    return 0;
>>>> -}
>>>> -
>>>> -void amdgpu_userq_fence_slab_fini(void)
>>>> -{
>>>> -    rcu_barrier();
>>>> -    kmem_cache_destroy(amdgpu_userq_fence_slab);
>>>> -}
>>>> +static const struct dma_fence_ops amdgpu_userq_fence_ops;
>>>>     static inline struct amdgpu_userq_fence *to_amdgpu_userq_fence(struct dma_fence *f)
>>>>   {
>>>> @@ -146,12 +126,18 @@ amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq)
>>>>   }
>>>>     static void
>>>> -amdgpu_userq_fence_put_fence_drv_array(struct amdgpu_userq_fence *userq_fence)
>>>> +amdgpu_userq_fence_put_fence_drv_refs(struct amdgpu_userq_fence *userq_fence)
>>>>   {
>>>>       unsigned long i;
>>>> +
>>>>       for (i = 0; i < userq_fence->fence_drv_array_count; i++)
>>>>           amdgpu_userq_fence_driver_put(userq_fence->fence_drv_array[i]);
>>>>       userq_fence->fence_drv_array_count = 0;
>>>> +    kfree(userq_fence->fence_drv_array);
>>>> +    userq_fence->fence_drv_array = NULL;
>>>> +
>>>> +    amdgpu_userq_fence_driver_put(userq_fence->fence_drv);
>>>> +    userq_fence->fence_drv = NULL;
>>>>   }
>>>>     void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
>>>> @@ -181,10 +167,11 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
>>>>           fence = &userq_fence->base;
>>>>           list_del_init(&userq_fence->link);
>>>>           dma_fence_signal(fence);
>>>> -        /* Drop fence_drv_array outside fence_list_lock
>>>> +        /*
>>>> +         * Drop fence_drv_array outside fence_list_lock
>>>>            * to avoid the recursion lock.
>>>>            */
>>>> -        amdgpu_userq_fence_put_fence_drv_array(userq_fence);
>>>> +        amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
>>>>           dma_fence_put(fence);
>>>>       }
>>>>   @@ -231,7 +218,7 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
>>>>     static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
>>>>   {
>>>> -    *userq_fence = kmem_cache_alloc(amdgpu_userq_fence_slab, GFP_ATOMIC);
>>>> +    *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
>>>>       return *userq_fence ? 0 : -ENOMEM;
>>>>   }
>>>>   @@ -299,7 +286,7 @@ static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
>>>>       spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
>>>>         if (signaled)
>>>> -        amdgpu_userq_fence_put_fence_drv_array(userq_fence);
>>>> +        amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
>>>>         *f = fence;
>>>>   @@ -333,29 +320,10 @@ static bool amdgpu_userq_fence_signaled(struct dma_fence *f)
>>>>       return false;
>>>>   }
>>>>   -static void amdgpu_userq_fence_free(struct rcu_head *rcu)
>>>> -{
>>>> -    struct dma_fence *fence = container_of(rcu, struct dma_fence, rcu);
>>>> -    struct amdgpu_userq_fence *userq_fence = to_amdgpu_userq_fence(fence);
>>>> -    struct amdgpu_userq_fence_driver *fence_drv = userq_fence->fence_drv;
>>>> -
>>>> -    /* Release the fence driver reference */
>>>> -    amdgpu_userq_fence_driver_put(fence_drv);
>>>> -
>>>> -    kvfree(userq_fence->fence_drv_array);
>>>> -    kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
>>>> -}
>>>> -
>>>> -static void amdgpu_userq_fence_release(struct dma_fence *f)
>>>> -{
>>>> -    call_rcu(&f->rcu, amdgpu_userq_fence_free);
>>>> -}
>>>> -
>>>>   static const struct dma_fence_ops amdgpu_userq_fence_ops = {
>>>>       .get_driver_name = amdgpu_userq_fence_get_driver_name,
>>>>       .get_timeline_name = amdgpu_userq_fence_get_timeline_name,
>>>>       .signaled = amdgpu_userq_fence_signaled,
>>>> -    .release = amdgpu_userq_fence_release,
>>>>   };
>>>>     /**
>>>> @@ -546,7 +514,7 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>>>>       r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
>>>>       if (r) {
>>>>           mutex_unlock(&userq_mgr->userq_mutex);
>>>> -        kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
>>>> +        kfree(userq_fence);
>>>>           goto put_gobj_write;
>>>>       }
>>>>   @@ -871,6 +839,7 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>>>>       for (i = 0, cnt = 0; i < num_fences; i++) {
>>>>           struct amdgpu_userq_fence_driver *fence_drv;
>>>>           struct amdgpu_userq_fence *userq_fence;
>>>> +        unsigned long flags;
>>>>           u32 index;
>>>>             userq_fence = to_amdgpu_userq_fence(fences[i]);
>>>> @@ -886,7 +855,19 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>>>>               continue;
>>>>           }
>>>>   +        spin_lock_irqsave(userq_fence->base.lock, flags);
>>>> +        if (dma_fence_is_signaled_locked(&userq_fence->base)) {
>>>> +            /*
>>>> +             * It is possible that fence is already signaled and the
>>>> +             * fence_drv now NULL, just skip over such fences.
>>>> +            */
>>>> +            spin_unlock_irqrestore(userq_fence->base.lock, flags);
>>>> +            continue;
>>>> +        }
>>>>           fence_drv = userq_fence->fence_drv;
>>>> +        amdgpu_userq_fence_driver_get(fence_drv);
>>>> +        spin_unlock_irqrestore(userq_fence->base.lock, flags);
>>>> +
>>>>           /*
>>>>            * We need to make sure the user queue release their reference
>>>>            * to the fence drivers at some point before queue destruction.
>>>> @@ -895,10 +876,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>>>>            */
>>>>           r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
>>>>                    xa_limit_32b, GFP_KERNEL);
>>>> -        if (r)
>>>> +        if (r) {
>>>> +            amdgpu_userq_fence_driver_put(fence_drv);
>>>>               goto put_waitq;
>>>> -
>>>> -        amdgpu_userq_fence_driver_get(fence_drv);
>>>> +        }
>>>>             /* Store drm syncobj's gpu va address and value */
>>>>           fence_info[cnt].va = fence_drv->va;
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
>>>> index d56246ad8c26..d355a0eecc07 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
>>>> @@ -58,9 +58,6 @@ struct amdgpu_userq_fence_driver {
>>>>       char timeline_name[TASK_COMM_LEN];
>>>>   };
>>>>   -int amdgpu_userq_fence_slab_init(void);
>>>> -void amdgpu_userq_fence_slab_fini(void);
>>>> -
>>>>   void amdgpu_userq_fence_driver_get(struct amdgpu_userq_fence_driver *fence_drv);
>>>>   void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv);
>>>>   int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,

Re: [PATCH 04/11] drm/amdgpu: rework amdgpu_userq_signal_ioctl

[Christian König]

On 4/22/26 12:08, Khatri, Sunil wrote:
> 
> On 21-04-2026 06:25 pm, Christian König wrote:
>> This one was fortunately not looking so bad as the wait ioctl path, but
>> there were still a few things which could be fixed/improved:
>>
>> 1. Allocating with GFP_ATOMIC was quite unecessary, we can do that
>>     before taking the userq_lock.
>> 2. Use a new mutex as protection for the fence_drv_xa so that we can do
>>     memory allocations while holding it.
>> 3. Starting the reset timer is unecessary when the fence is already
>>     signaled when we create it.
>> 4. Cleanup error handling, avoid trying to free the queue when we don't
>>     even got one.
>>
>> Signed-off-by: Christian König <christian.koenig@amd.com>
>> ---
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c     |   1 +
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h     |  12 +
>>   .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 220 ++++++++----------
>>   3 files changed, 111 insertions(+), 122 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>> index b632bc3c952b..174190a77005 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>> @@ -793,6 +793,7 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
>>       }
>>         queue->doorbell_index = index;
>> +    mutex_init(&queue->fence_drv_lock);
> we do want to destroy the mutex in case queue creation fails ? RIght now amdgpu_userq_fence_driver_alloc fails we goto clean_mapping and not destroying the mutex which is done in amdgpu_userq_fence_driver_free.
> goto needs to be modified i guess to handle it.

Good point, going to clean that up.

> Apart from that LGTM. Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>

Thanks,
Christian

> 
> Regards
> Sunil
>>       xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
>>       r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
>>       if (r) {
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
>> index 675fe6395ac8..cb92789c1ed1 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
>> @@ -66,6 +66,18 @@ struct amdgpu_usermode_queue {
>>       struct amdgpu_userq_obj    db_obj;
>>       struct amdgpu_userq_obj fw_obj;
>>       struct amdgpu_userq_obj wptr_obj;
>> +
>> +    /**
>> +     * @fence_drv_lock: Protecting @fence_drv_xa.
>> +     */
>> +    struct mutex        fence_drv_lock;
>> +
>> +    /**
>> +     * @fence_drv_xa:
>> +     *
>> +     * References to the external fence drivers returned by wait_ioctl.
>> +     * Dropped on the next signaled dma_fence or queue destruction.
>> +     */
>>       struct xarray        fence_drv_xa;
>>       struct amdgpu_userq_fence_driver *fence_drv;
>>       struct dma_fence    *last_fence;
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>> index 909bdccc2a92..b0543fa257ed 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>> @@ -121,6 +121,7 @@ amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq)
>>       userq->last_fence = NULL;
>>       amdgpu_userq_walk_and_drop_fence_drv(&userq->fence_drv_xa);
>>       xa_destroy(&userq->fence_drv_xa);
>> +    mutex_destroy(&userq->fence_drv_lock);
>>       /* Drop the queue's ownership reference to fence_drv explicitly */
>>       amdgpu_userq_fence_driver_put(userq->fence_drv);
>>   }
>> @@ -216,81 +217,77 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
>>       kref_put(&fence_drv->refcount, amdgpu_userq_fence_driver_destroy);
>>   }
>>   -static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
>> +static int amdgpu_userq_fence_alloc(struct amdgpu_usermode_queue *userq,
>> +                    struct amdgpu_userq_fence **pfence)
>>   {
>> -    *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
>> -    return *userq_fence ? 0 : -ENOMEM;
>> +    struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
>> +    struct amdgpu_userq_fence *userq_fence;
>> +    unsigned long count;
>> +
>> +    userq_fence = kmalloc(sizeof(*userq_fence), GFP_KERNEL);
>> +    if (!userq_fence)
>> +        return -ENOMEM;
>> +
>> +    /*
>> +     * Get the next unused entry, since we fill from the start this can be
>> +     * used as size to allocate the array.
>> +     */
>> +    mutex_lock(&userq->fence_drv_lock);
>> +    xa_find(&userq->fence_drv_xa, &count, ULONG_MAX, XA_FREE_MARK);
>> +
>> +    userq_fence->fence_drv_array = kvmalloc_array(count, sizeof(fence_drv),
>> +                              GFP_KERNEL);
>> +    if (!userq_fence->fence_drv_array) {
>> +        mutex_unlock(&userq->fence_drv_lock);
>> +        kfree(userq_fence);
>> +        return -ENOMEM;
>> +    }
>> +
>> +    userq_fence->fence_drv_array_count = count;
>> +    xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array,
>> +           0, ULONG_MAX, count, XA_PRESENT);
>> +    xa_destroy(&userq->fence_drv_xa);
>> +
>> +    mutex_unlock(&userq->fence_drv_lock);
>> +
>> +    userq_fence->fence_drv = fence_drv;
>> +    amdgpu_userq_fence_driver_get(fence_drv);
>> +
>> +    *pfence = userq_fence;
>> +    return 0;
>>   }
>>   -static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
>> -                     struct amdgpu_userq_fence *userq_fence,
>> -                     u64 seq, struct dma_fence **f)
>> +static void amdgpu_userq_fence_init(struct amdgpu_usermode_queue *userq,
>> +                    struct amdgpu_userq_fence *fence,
>> +                    u64 seq)
>>   {
>> -    struct amdgpu_userq_fence_driver *fence_drv;
>> -    struct dma_fence *fence;
>> +    struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
>>       unsigned long flags;
>>       bool signaled = false;
>>   -    fence_drv = userq->fence_drv;
>> -    if (!fence_drv)
>> -        return -EINVAL;
>> -
>> -    spin_lock_init(&userq_fence->lock);
>> -    INIT_LIST_HEAD(&userq_fence->link);
>> -    fence = &userq_fence->base;
>> -    userq_fence->fence_drv = fence_drv;
>> -
>> -    dma_fence_init64(fence, &amdgpu_userq_fence_ops, &userq_fence->lock,
>> +    spin_lock_init(&fence->lock);
>> +    dma_fence_init64(&fence->base, &amdgpu_userq_fence_ops, &fence->lock,
>>                fence_drv->context, seq);
>>   -    amdgpu_userq_fence_driver_get(fence_drv);
>> -    dma_fence_get(fence);
>> -
>> -    if (!xa_empty(&userq->fence_drv_xa)) {
>> -        struct amdgpu_userq_fence_driver *stored_fence_drv;
>> -        unsigned long index, count = 0;
>> -        int i = 0;
>> -
>> -        xa_lock(&userq->fence_drv_xa);
>> -        xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv)
>> -            count++;
>> -
>> -        userq_fence->fence_drv_array =
>> -            kvmalloc_array(count,
>> -                       sizeof(struct amdgpu_userq_fence_driver *),
>> -                       GFP_ATOMIC);
>> -
>> -        if (userq_fence->fence_drv_array) {
>> -            xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv) {
>> -                userq_fence->fence_drv_array[i] = stored_fence_drv;
>> -                __xa_erase(&userq->fence_drv_xa, index);
>> -                i++;
>> -            }
>> -        }
>> -
>> -        userq_fence->fence_drv_array_count = i;
>> -        xa_unlock(&userq->fence_drv_xa);
>> -    } else {
>> -        userq_fence->fence_drv_array = NULL;
>> -        userq_fence->fence_drv_array_count = 0;
>> -    }
>> +    /* Make sure the fence is visible to the hang detect worker */
>> +    dma_fence_put(userq->last_fence);
>> +    userq->last_fence = dma_fence_get(&fence->base);
>>   -    /* Check if hardware has already processed the job */
>> +    /* Check if hardware has already processed the fence */
>>       spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
>> -    if (!dma_fence_is_signaled(fence)) {
>> -        list_add_tail(&userq_fence->link, &fence_drv->fences);
>> +    if (!dma_fence_is_signaled(&fence->base)) {
>> +        dma_fence_get(&fence->base);
>> +        list_add_tail(&fence->link, &fence_drv->fences);
>>       } else {
>> +        INIT_LIST_HEAD(&fence->link);
>>           signaled = true;
>> -        dma_fence_put(fence);
>>       }
>>       spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
>>         if (signaled)
>> -        amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
>> -
>> -    *f = fence;
>> -
>> -    return 0;
>> +        amdgpu_userq_fence_put_fence_drv_refs(fence);
>> +    else
>> +        amdgpu_userq_start_hang_detect_work(userq);
>>   }
>>     static const char *amdgpu_userq_fence_get_driver_name(struct dma_fence *f)
>> @@ -392,11 +389,6 @@ static int amdgpu_userq_fence_read_wptr(struct amdgpu_device *adev,
>>       return r;
>>   }
>>   -static void amdgpu_userq_fence_cleanup(struct dma_fence *fence)
>> -{
>> -    dma_fence_put(fence);
>> -}
>> -
>>   static void
>>   amdgpu_userq_fence_driver_set_error(struct amdgpu_userq_fence *fence,
>>                       int error)
>> @@ -440,13 +432,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>>       const unsigned int num_read_bo_handles = args->num_bo_read_handles;
>>       struct amdgpu_fpriv *fpriv = filp->driver_priv;
>>       struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
>> +
>>       struct drm_gem_object **gobj_write, **gobj_read;
>>       u32 *syncobj_handles, num_syncobj_handles;
>> -    struct amdgpu_userq_fence *userq_fence;
>> -    struct amdgpu_usermode_queue *queue = NULL;
>> -    struct drm_syncobj **syncobj = NULL;
>> -    struct dma_fence *fence;
>> +    struct amdgpu_usermode_queue *queue;
>> +    struct amdgpu_userq_fence *fence;
>> +    struct drm_syncobj **syncobj;
>>       struct drm_exec exec;
>> +    void __user *ptr;
>>       int r, i, entry;
>>       u64 wptr;
>>   @@ -458,13 +451,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>>           return -EINVAL;
>>         num_syncobj_handles = args->num_syncobj_handles;
>> -    syncobj_handles = memdup_array_user(u64_to_user_ptr(args->syncobj_handles),
>> -                        num_syncobj_handles, sizeof(u32));
>> +    ptr = u64_to_user_ptr(args->syncobj_handles);
>> +    syncobj_handles = memdup_array_user(ptr, num_syncobj_handles,
>> +                        sizeof(u32));
>>       if (IS_ERR(syncobj_handles))
>>           return PTR_ERR(syncobj_handles);
>>   -    /* Array of pointers to the looked up syncobjs */
>> -    syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj), GFP_KERNEL);
>> +    syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj),
>> +                GFP_KERNEL);
>>       if (!syncobj) {
>>           r = -ENOMEM;
>>           goto free_syncobj_handles;
>> @@ -478,21 +472,17 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>>           }
>>       }
>>   -    r = drm_gem_objects_lookup(filp,
>> -                   u64_to_user_ptr(args->bo_read_handles),
>> -                   num_read_bo_handles,
>> -                   &gobj_read);
>> +    ptr = u64_to_user_ptr(args->bo_read_handles);
>> +    r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
>>       if (r)
>>           goto free_syncobj;
>>   -    r = drm_gem_objects_lookup(filp,
>> -                   u64_to_user_ptr(args->bo_write_handles),
>> -                   num_write_bo_handles,
>> +    ptr = u64_to_user_ptr(args->bo_write_handles);
>> +    r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
>>                      &gobj_write);
>>       if (r)
>>           goto put_gobj_read;
>>   -    /* Retrieve the user queue */
>>       queue = amdgpu_userq_get(userq_mgr, args->queue_id);
>>       if (!queue) {
>>           r = -ENOENT;
>> @@ -501,73 +491,61 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>>         r = amdgpu_userq_fence_read_wptr(adev, queue, &wptr);
>>       if (r)
>> -        goto put_gobj_write;
>> +        goto put_queue;
>>   -    r = amdgpu_userq_fence_alloc(&userq_fence);
>> +    r = amdgpu_userq_fence_alloc(queue, &fence);
>>       if (r)
>> -        goto put_gobj_write;
>> +        goto put_queue;
>>         /* We are here means UQ is active, make sure the eviction fence is valid */
>>       amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
>>   -    /* Create a new fence */
>> -    r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
>> -    if (r) {
>> -        mutex_unlock(&userq_mgr->userq_mutex);
>> -        kfree(userq_fence);
>> -        goto put_gobj_write;
>> -    }
>> +    /* Create the new fence */
>> +    amdgpu_userq_fence_init(queue, fence, wptr);
>>   -    dma_fence_put(queue->last_fence);
>> -    queue->last_fence = dma_fence_get(fence);
>> -    amdgpu_userq_start_hang_detect_work(queue);
>>       mutex_unlock(&userq_mgr->userq_mutex);
>>   +    /*
>> +     * This needs to come after the fence is created since
>> +     * amdgpu_userq_ensure_ev_fence() can't be called while holding the resv
>> +     * locks.
>> +     */
>>       drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
>>                 (num_read_bo_handles + num_write_bo_handles));
>>   -    /* Lock all BOs with retry handling */
>>       drm_exec_until_all_locked(&exec) {
>> -        r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1);
>> +        r = drm_exec_prepare_array(&exec, gobj_read,
>> +                       num_read_bo_handles, 1);
>>           drm_exec_retry_on_contention(&exec);
>> -        if (r) {
>> -            amdgpu_userq_fence_cleanup(fence);
>> +        if (r)
>>               goto exec_fini;
>> -        }
>>   -        r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1);
>> +        r = drm_exec_prepare_array(&exec, gobj_write,
>> +                       num_write_bo_handles, 1);
>>           drm_exec_retry_on_contention(&exec);
>> -        if (r) {
>> -            amdgpu_userq_fence_cleanup(fence);
>> +        if (r)
>>               goto exec_fini;
>> -        }
>>       }
>>   -    for (i = 0; i < num_read_bo_handles; i++) {
>> -        if (!gobj_read || !gobj_read[i]->resv)
>> -            continue;
>> -
>> -        dma_resv_add_fence(gobj_read[i]->resv, fence,
>> +    /* And publish the new fence in the BOs and syncobj */
>> +    for (i = 0; i < num_read_bo_handles; i++)
>> +        dma_resv_add_fence(gobj_read[i]->resv, &fence->base,
>>                      DMA_RESV_USAGE_READ);
>> -    }
>>   -    for (i = 0; i < num_write_bo_handles; i++) {
>> -        if (!gobj_write || !gobj_write[i]->resv)
>> -            continue;
>> -
>> -        dma_resv_add_fence(gobj_write[i]->resv, fence,
>> +    for (i = 0; i < num_write_bo_handles; i++)
>> +        dma_resv_add_fence(gobj_write[i]->resv, &fence->base,
>>                      DMA_RESV_USAGE_WRITE);
>> -    }
>>   -    /* Add the created fence to syncobj/BO's */
>>       for (i = 0; i < num_syncobj_handles; i++)
>> -        drm_syncobj_replace_fence(syncobj[i], fence);
>> +        drm_syncobj_replace_fence(syncobj[i], &fence->base);
>>   +exec_fini:
>>       /* drop the reference acquired in fence creation function */
>> -    dma_fence_put(fence);
>> +    dma_fence_put(&fence->base);
>>   -exec_fini:
>>       drm_exec_fini(&exec);
>> +put_queue:
>> +    amdgpu_userq_put(queue);
>>   put_gobj_write:
>>       for (i = 0; i < num_write_bo_handles; i++)
>>           drm_gem_object_put(gobj_write[i]);
>> @@ -578,15 +556,11 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>>       kvfree(gobj_read);
>>   free_syncobj:
>>       while (entry-- > 0)
>> -        if (syncobj[entry])
>> -            drm_syncobj_put(syncobj[entry]);
>> +        drm_syncobj_put(syncobj[entry]);
>>       kfree(syncobj);
>>   free_syncobj_handles:
>>       kfree(syncobj_handles);
>>   -    if (queue)
>> -        amdgpu_userq_put(queue);
>> -
>>       return r;
>>   }
>>   @@ -874,8 +848,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>>            * Otherwise, we would gather those references until we don't
>>            * have any more space left and crash.
>>            */
>> +        mutex_lock(&waitq->fence_drv_lock);
>>           r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
>>                    xa_limit_32b, GFP_KERNEL);
>> +        mutex_unlock(&waitq->fence_drv_lock);
>>           if (r) {
>>               amdgpu_userq_fence_driver_put(fence_drv);
>>               goto put_waitq;

Re: [PATCH 05/11] drm/amdgpu: rework userq fence signal processing

[Khatri, Sunil]

Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>

On 21-04-2026 06:25 pm, Christian König wrote:
> Move more code into a common userq function.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 13 +++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h |  1 +
>   drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c    | 10 +---------
>   drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c    | 10 +---------
>   drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c    | 11 +----------
>   drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c    | 11 +----------
>   drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c    | 11 +----------
>   7 files changed, 19 insertions(+), 48 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index 174190a77005..8ce001481d42 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -205,6 +205,19 @@ void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
>   		     msecs_to_jiffies(timeout_ms));
>   }
>   
> +void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell)
> +{
> +	struct xarray *xa = &adev->userq_doorbell_xa;
> +	struct amdgpu_usermode_queue *queue;
> +	unsigned long flags;
> +
> +	xa_lock_irqsave(xa, flags);
> +	queue = xa_load(xa, doorbell);
> +	if (queue)
> +		amdgpu_userq_fence_driver_process(queue->fence_drv);
> +	xa_unlock_irqrestore(xa, flags);
> +}
> +
>   static void amdgpu_userq_init_hang_detect_work(struct amdgpu_usermode_queue *queue)
>   {
>   	INIT_DELAYED_WORK(&queue->hang_detect_work, amdgpu_userq_hang_detect_work);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> index cb92789c1ed1..843ea8ecc5d7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> @@ -168,6 +168,7 @@ void amdgpu_userq_reset_work(struct work_struct *work);
>   void amdgpu_userq_pre_reset(struct amdgpu_device *adev);
>   int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost);
>   void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue);
> +void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell);
>   
>   int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
>   				   struct amdgpu_usermode_queue *queue,
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
> index 837d98947958..1ffbb5450f3a 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
> @@ -6528,15 +6528,7 @@ static int gfx_v11_0_eop_irq(struct amdgpu_device *adev,
>   	DRM_DEBUG("IH: CP EOP\n");
>   
>   	if (adev->enable_mes && doorbell_offset) {
> -		struct amdgpu_usermode_queue *queue;
> -		struct xarray *xa = &adev->userq_doorbell_xa;
> -		unsigned long flags;
> -
> -		xa_lock_irqsave(xa, flags);
> -		queue = xa_load(xa, doorbell_offset);
> -		if (queue)
> -			amdgpu_userq_fence_driver_process(queue->fence_drv);
> -		xa_unlock_irqrestore(xa, flags);
> +		amdgpu_userq_process_fence_irq(adev, doorbell_offset);
>   	} else {
>   		me_id = (entry->ring_id & 0x0c) >> 2;
>   		pipe_id = (entry->ring_id & 0x03) >> 0;
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
> index 60f0c7d6a7a3..6baac533a2e6 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
> @@ -4859,15 +4859,7 @@ static int gfx_v12_0_eop_irq(struct amdgpu_device *adev,
>   	DRM_DEBUG("IH: CP EOP\n");
>   
>   	if (adev->enable_mes && doorbell_offset) {
> -		struct xarray *xa = &adev->userq_doorbell_xa;
> -		struct amdgpu_usermode_queue *queue;
> -		unsigned long flags;
> -
> -		xa_lock_irqsave(xa, flags);
> -		queue = xa_load(xa, doorbell_offset);
> -		if (queue)
> -			amdgpu_userq_fence_driver_process(queue->fence_drv);
> -		xa_unlock_irqrestore(xa, flags);
> +		amdgpu_userq_process_fence_irq(adev, doorbell_offset);
>   	} else {
>   		me_id = (entry->ring_id & 0x0c) >> 2;
>   		pipe_id = (entry->ring_id & 0x03) >> 0;
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
> index 948758b51b5c..ae65412109c2 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
> @@ -3654,16 +3654,7 @@ static int gfx_v12_1_eop_irq(struct amdgpu_device *adev,
>   	DRM_DEBUG("IH: CP EOP\n");
>   
>   	if (adev->enable_mes && doorbell_offset) {
> -		struct xarray *xa = &adev->userq_doorbell_xa;
> -		struct amdgpu_usermode_queue *queue;
> -		unsigned long flags;
> -
> -		xa_lock_irqsave(xa, flags);
> -		queue = xa_load(xa, doorbell_offset);
> -		if (queue)
> -			amdgpu_userq_fence_driver_process(queue->fence_drv);
> -
> -		xa_unlock_irqrestore(xa, flags);
> +		amdgpu_userq_process_fence_irq(adev, doorbell_offset);
>   	} else {
>   		me_id = (entry->ring_id & 0x0c) >> 2;
>   		pipe_id = (entry->ring_id & 0x03) >> 0;
> diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
> index de329b76a00c..bf09ac841a68 100644
> --- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
> @@ -1662,17 +1662,8 @@ static int sdma_v6_0_process_fence_irq(struct amdgpu_device *adev,
>   	u32 doorbell_offset = entry->src_data[0];
>   
>   	if (adev->enable_mes && doorbell_offset) {
> -		struct amdgpu_usermode_queue *queue;
> -		struct xarray *xa = &adev->userq_doorbell_xa;
> -		unsigned long flags;
> -
>   		doorbell_offset >>= SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
> -
> -		xa_lock_irqsave(xa, flags);
> -		queue = xa_load(xa, doorbell_offset);
> -		if (queue)
> -			amdgpu_userq_fence_driver_process(queue->fence_drv);
> -		xa_unlock_irqrestore(xa, flags);
> +		amdgpu_userq_process_fence_irq(adev, doorbell_offset);
>   	}
>   
>   	return 0;
> diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
> index 85d98a0e1bff..f154b68dda70 100644
> --- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
> @@ -1594,17 +1594,8 @@ static int sdma_v7_0_process_fence_irq(struct amdgpu_device *adev,
>   	u32 doorbell_offset = entry->src_data[0];
>   
>   	if (adev->enable_mes && doorbell_offset) {
> -		struct xarray *xa = &adev->userq_doorbell_xa;
> -		struct amdgpu_usermode_queue *queue;
> -		unsigned long flags;
> -
>   		doorbell_offset >>= SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
> -
> -		xa_lock_irqsave(xa, flags);
> -		queue = xa_load(xa, doorbell_offset);
> -		if (queue)
> -			amdgpu_userq_fence_driver_process(queue->fence_drv);
> -		xa_unlock_irqrestore(xa, flags);
> +		amdgpu_userq_process_fence_irq(adev, doorbell_offset);
>   	}
>   
>   	return 0;

Re: [PATCH 06/11] drm/amdgpu: remove almost all calls to amdgpu_userq_detect_and_reset_queues

[Khatri, Sunil]

Now this is exactly how i nearly disabled the reset login in my 
validation setup.
Looks clean and as per the expectations.

Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>

On 21-04-2026 06:25 pm, Christian König wrote:
> Well the reset handling seems broken on multiple levels.
>
> As first step of fixing this remove most calls to the hang detection.
> That function should only be called after we run into a timeout! And *NOT*
> as random check spread over the code in multiple places.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 38 +++++++++--------------
>   1 file changed, 14 insertions(+), 24 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index 8ce001481d42..5ccd53ad8efd 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -345,23 +345,18 @@ static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue)
>   	struct amdgpu_device *adev = uq_mgr->adev;
>   	const struct amdgpu_userq_funcs *userq_funcs =
>   		adev->userq_funcs[queue->queue_type];
> -	bool found_hung_queue = false;
> -	int r = 0;
> +	int r;
>   
>   	if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
>   		r = userq_funcs->preempt(queue);
>   		if (r) {
>   			queue->state = AMDGPU_USERQ_STATE_HUNG;
> -			found_hung_queue = true;
> +			return r;
>   		} else {
>   			queue->state = AMDGPU_USERQ_STATE_PREEMPTED;
>   		}
>   	}
> -
> -	if (found_hung_queue)
> -		amdgpu_userq_detect_and_reset_queues(uq_mgr);
> -
> -	return r;
> +	return 0;
>   }
>   
>   static int amdgpu_userq_restore_helper(struct amdgpu_usermode_queue *queue)
> @@ -390,24 +385,21 @@ static int amdgpu_userq_unmap_helper(struct amdgpu_usermode_queue *queue)
>   	struct amdgpu_device *adev = uq_mgr->adev;
>   	const struct amdgpu_userq_funcs *userq_funcs =
>   		adev->userq_funcs[queue->queue_type];
> -	bool found_hung_queue = false;
> -	int r = 0;
> +	int r;
>   
>   	if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) ||
> -		(queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {
> +	    (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {
> +
>   		r = userq_funcs->unmap(queue);
>   		if (r) {
>   			queue->state = AMDGPU_USERQ_STATE_HUNG;
> -			found_hung_queue = true;
> +			return r;
>   		} else {
>   			queue->state = AMDGPU_USERQ_STATE_UNMAPPED;
>   		}
>   	}
>   
> -	if (found_hung_queue)
> -		amdgpu_userq_detect_and_reset_queues(uq_mgr);
> -
> -	return r;
> +	return 0;
>   }
>   
>   static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue)
> @@ -416,19 +408,19 @@ static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue)
>   	struct amdgpu_device *adev = uq_mgr->adev;
>   	const struct amdgpu_userq_funcs *userq_funcs =
>   		adev->userq_funcs[queue->queue_type];
> -	int r = 0;
> +	int r;
>   
>   	if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) {
>   		r = userq_funcs->map(queue);
>   		if (r) {
>   			queue->state = AMDGPU_USERQ_STATE_HUNG;
> -			amdgpu_userq_detect_and_reset_queues(uq_mgr);
> +			return r;
>   		} else {
>   			queue->state = AMDGPU_USERQ_STATE_MAPPED;
>   		}
>   	}
>   
> -	return r;
> +	return 0;
>   }
>   
>   static void amdgpu_userq_wait_for_last_fence(struct amdgpu_usermode_queue *queue)
> @@ -654,7 +646,6 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_que
>   #if defined(CONFIG_DEBUG_FS)
>   	debugfs_remove_recursive(queue->debugfs_queue);
>   #endif
> -	amdgpu_userq_detect_and_reset_queues(uq_mgr);
>   	r = amdgpu_userq_unmap_helper(queue);
>   	/*TODO: It requires a reset for userq hw unmap error*/
>   	if (r) {
> @@ -1268,7 +1259,6 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
>   	unsigned long queue_id;
>   	int ret = 0, r;
>   
> -	amdgpu_userq_detect_and_reset_queues(uq_mgr);
>   	/* Try to unmap all the queues in this process ctx */
>   	xa_for_each(&uq_mgr->userq_xa, queue_id, queue) {
>   		r = amdgpu_userq_preempt_helper(queue);
> @@ -1276,9 +1266,11 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
>   			ret = r;
>   	}
>   
> -	if (ret)
> +	if (ret) {
>   		drm_file_err(uq_mgr->file,
>   			     "Couldn't unmap all the queues, eviction failed ret=%d\n", ret);
> +		amdgpu_userq_detect_and_reset_queues(uq_mgr);
> +	}
>   	return ret;
>   }
>   
> @@ -1378,7 +1370,6 @@ int amdgpu_userq_suspend(struct amdgpu_device *adev)
>   		uqm = queue->userq_mgr;
>   		cancel_delayed_work_sync(&uqm->resume_work);
>   		guard(mutex)(&uqm->userq_mutex);
> -		amdgpu_userq_detect_and_reset_queues(uqm);
>   		if (adev->in_s0ix)
>   			r = amdgpu_userq_preempt_helper(queue);
>   		else
> @@ -1437,7 +1428,6 @@ int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev,
>   		if (((queue->queue_type == AMDGPU_HW_IP_GFX) ||
>   		     (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) &&
>   		    (queue->xcp_id == idx)) {
> -			amdgpu_userq_detect_and_reset_queues(uqm);
>   			r = amdgpu_userq_preempt_helper(queue);
>   			if (r)
>   				ret = r;

Re: [PATCH 07/11] drm/amdgpu: fix userq hang detection and reset

[Khatri, Sunil]

Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>

On 21-04-2026 06:25 pm, Christian König wrote:
> Fix lock inversions pointed out by Prike and Sunil. The hang detection
> timeout *CAN'T* grab locks under which we wait for fences, especially
> not the userq_mutex lock.
>
> Then instead of this completely broken handling with the
> hang_detect_fence just cancel the work when fences are processed and
> re-start if necessary.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c     | 65 ++++++++-----------
>   drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h     |  1 -
>   .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 17 +++--
>   .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.h   |  2 +-
>   4 files changed, 40 insertions(+), 45 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index 5ccd53ad8efd..0a4c39d83adc 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -106,9 +106,6 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
>   	int r = 0;
>   	int i;
>   
> -	/* Warning if current process mutex is not held */
> -	WARN_ON(!mutex_is_locked(&uq_mgr->userq_mutex));
> -
>   	if (unlikely(adev->debug_disable_gpu_ring_reset)) {
>   		dev_err(adev->dev, "userq reset disabled by debug mask\n");
>   		return 0;
> @@ -127,9 +124,11 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
>   	 */
>   	for (i = 0; i < num_queue_types; i++) {
>   		int ring_type = queue_types[i];
> -		const struct amdgpu_userq_funcs *funcs = adev->userq_funcs[ring_type];
> +		const struct amdgpu_userq_funcs *funcs =
> +			adev->userq_funcs[ring_type];
>   
> -		if (!amdgpu_userq_is_reset_type_supported(adev, ring_type, AMDGPU_RESET_TYPE_PER_QUEUE))
> +		if (!amdgpu_userq_is_reset_type_supported(adev, ring_type,
> +							  AMDGPU_RESET_TYPE_PER_QUEUE))
>   				continue;
>   
>   		if (atomic_read(&uq_mgr->userq_count[ring_type]) > 0 &&
> @@ -150,38 +149,22 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
>   
>   static void amdgpu_userq_hang_detect_work(struct work_struct *work)
>   {
> -	struct amdgpu_usermode_queue *queue = container_of(work,
> -							  struct amdgpu_usermode_queue,
> -							  hang_detect_work.work);
> -	struct dma_fence *fence;
> -	struct amdgpu_userq_mgr *uq_mgr;
> -
> -	if (!queue->userq_mgr)
> -		return;
> -
> -	uq_mgr = queue->userq_mgr;
> -	fence = READ_ONCE(queue->hang_detect_fence);
> -	/* Fence already signaled – no action needed */
> -	if (!fence || dma_fence_is_signaled(fence))
> -		return;
> +	struct amdgpu_usermode_queue *queue =
> +		container_of(work, struct amdgpu_usermode_queue,
> +			     hang_detect_work.work);
>   
> -	mutex_lock(&uq_mgr->userq_mutex);
> -	amdgpu_userq_detect_and_reset_queues(uq_mgr);
> -	mutex_unlock(&uq_mgr->userq_mutex);
> +	amdgpu_userq_detect_and_reset_queues(queue->userq_mgr);
>   }
>   
>   /*
>    * Start hang detection for a user queue fence. A delayed work will be scheduled
> - * to check if the fence is still pending after the timeout period.
> -*/
> + * to reset the queues when the fence doesn't signal in time.
> + */
>   void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
>   {
>   	struct amdgpu_device *adev;
>   	unsigned long timeout_ms;
>   
> -	if (!queue || !queue->userq_mgr || !queue->userq_mgr->adev)
> -		return;
> -
>   	adev = queue->userq_mgr->adev;
>   	/* Determine timeout based on queue type */
>   	switch (queue->queue_type) {
> @@ -199,8 +182,6 @@ void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
>   		break;
>   	}
>   
> -	/* Store the fence to monitor and schedule hang detection */
> -	WRITE_ONCE(queue->hang_detect_fence, queue->last_fence);
>   	schedule_delayed_work(&queue->hang_detect_work,
>   		     msecs_to_jiffies(timeout_ms));
>   }
> @@ -210,18 +191,24 @@ void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell)
>   	struct xarray *xa = &adev->userq_doorbell_xa;
>   	struct amdgpu_usermode_queue *queue;
>   	unsigned long flags;
> +	int r;
>   
>   	xa_lock_irqsave(xa, flags);
>   	queue = xa_load(xa, doorbell);
> -	if (queue)
> -		amdgpu_userq_fence_driver_process(queue->fence_drv);
> -	xa_unlock_irqrestore(xa, flags);
> -}
> +	if (queue) {
> +		r = amdgpu_userq_fence_driver_process(queue->fence_drv);
> +		/*
> +		 * We are in interrupt context here, this *can't* wait for
> +		 * reset work to finish.
> +		 */
> +		if (r >= 0)
> +			cancel_delayed_work(&queue->hang_detect_work);
>   
> -static void amdgpu_userq_init_hang_detect_work(struct amdgpu_usermode_queue *queue)
> -{
> -	INIT_DELAYED_WORK(&queue->hang_detect_work, amdgpu_userq_hang_detect_work);
> -	queue->hang_detect_fence = NULL;
> +		/* Restart the timer when there are still fences pending */
> +		if (r == 1)
> +			amdgpu_userq_start_hang_detect_work(queue);
> +	}
> +	xa_unlock_irqrestore(xa, flags);
>   }
>   
>   static int amdgpu_userq_buffer_va_list_add(struct amdgpu_usermode_queue *queue,
> @@ -640,7 +627,6 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_que
>   	amdgpu_bo_unreserve(vm->root.bo);
>   
>   	mutex_lock(&uq_mgr->userq_mutex);
> -	queue->hang_detect_fence = NULL;
>   	amdgpu_userq_wait_for_last_fence(queue);
>   
>   #if defined(CONFIG_DEBUG_FS)
> @@ -853,7 +839,8 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
>   	up_read(&adev->reset_domain->sem);
>   
>   	amdgpu_debugfs_userq_init(filp, queue, qid);
> -	amdgpu_userq_init_hang_detect_work(queue);
> +	INIT_DELAYED_WORK(&queue->hang_detect_work,
> +			  amdgpu_userq_hang_detect_work);
>   
>   	args->out.queue_id = qid;
>   	atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> index 843ea8ecc5d7..85f460e7c31b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> @@ -85,7 +85,6 @@ struct amdgpu_usermode_queue {
>   	int			priority;
>   	struct dentry		*debugfs_queue;
>   	struct delayed_work hang_detect_work;
> -	struct dma_fence *hang_detect_fence;
>   	struct kref		refcount;
>   
>   	struct list_head	userq_va_list;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> index b0543fa257ed..beb2a1f679b8 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> @@ -141,7 +141,14 @@ amdgpu_userq_fence_put_fence_drv_refs(struct amdgpu_userq_fence *userq_fence)
>   	userq_fence->fence_drv = NULL;
>   }
>   
> -void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
> +/*
> + * Returns:
> + * -ENOENT when no fences were processes
> + * 1 when more fences are pending
> + * 0 when no fences are pending any more
> + */
> +int
> +amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
>   {
>   	struct amdgpu_userq_fence *userq_fence, *tmp;
>   	LIST_HEAD(to_be_signaled);
> @@ -149,9 +156,6 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
>   	unsigned long flags;
>   	u64 rptr;
>   
> -	if (!fence_drv)
> -		return;
> -
>   	spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
>   	rptr = amdgpu_userq_fence_read(fence_drv);
>   
> @@ -164,6 +168,9 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
>   				&userq_fence->link);
>   	spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
>   
> +	if (list_empty(&to_be_signaled))
> +		return -ENOENT;
> +
>   	list_for_each_entry_safe(userq_fence, tmp, &to_be_signaled, link) {
>   		fence = &userq_fence->base;
>   		list_del_init(&userq_fence->link);
> @@ -176,6 +183,8 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
>   		dma_fence_put(fence);
>   	}
>   
> +	/* That doesn't need to be accurate so no locking */
> +	return list_empty(&fence_drv->fences) ? 0 : 1;
>   }
>   
>   void amdgpu_userq_fence_driver_destroy(struct kref *ref)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> index d355a0eecc07..0bd51616cef1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> @@ -63,7 +63,7 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv);
>   int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,
>   				    struct amdgpu_userq_fence_driver **fence_drv_req);
>   void amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq);
> -void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv);
> +int amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv);
>   void amdgpu_userq_fence_driver_force_completion(struct amdgpu_usermode_queue *userq);
>   void amdgpu_userq_fence_driver_destroy(struct kref *ref);
>   int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,

Re: [PATCH 03/11] drm/amdgpu: nuke amdgpu_userq_fence_free

[Khatri, Sunil]

On 22-04-2026 03:42 pm, Christian König wrote:
> On 4/22/26 11:40, Khatri, Sunil wrote:
>> On 22-04-2026 02:56 pm, Christian König wrote:
>>> On 4/22/26 10:29, Khatri, Sunil wrote:
>>>> On 21-04-2026 06:25 pm, Christian König wrote:
>>>>> As preparation for independent fences remove the function and do all of
>>>>> it's cleanup directly after signaling.
>>>>>
>>>>> Signed-off-by: Christian König <christian.koenig@amd.com>
>>>>> ---
>>>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c       | 13 +--
>>>>>    .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 79 +++++++------------
>>>>>    .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.h   |  3 -
>>>>>    3 files changed, 31 insertions(+), 64 deletions(-)
>>>>>
>>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>>>>> index c6546a858597..1b15b51dc3f4 100644
>>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>>>>> @@ -3162,11 +3162,7 @@ static int __init amdgpu_init(void)
>>>>>          r = amdgpu_sync_init();
>>>>>        if (r)
>>>>> -        goto error_sync;
>>>>> -
>>>>> -    r = amdgpu_userq_fence_slab_init();
>>>>> -    if (r)
>>>>> -        goto error_fence;
>>>>> +        return r;
>>>>>          amdgpu_register_atpx_handler();
>>>>>        amdgpu_acpi_detect();
>>>>> @@ -3182,12 +3178,6 @@ static int __init amdgpu_init(void)
>>>>>          /* let modprobe override vga console setting */
>>>>>        return pci_register_driver(&amdgpu_kms_pci_driver);
>>>>> -
>>>>> -error_fence:
>>>>> -    amdgpu_sync_fini();
>>>>> -
>>>>> -error_sync:
>>>>> -    return r;
>>>>>    }
>>>>>      static void __exit amdgpu_exit(void)
>>>>> @@ -3197,7 +3187,6 @@ static void __exit amdgpu_exit(void)
>>>>>        amdgpu_unregister_atpx_handler();
>>>>>        amdgpu_acpi_release();
>>>>>        amdgpu_sync_fini();
>>>>> -    amdgpu_userq_fence_slab_fini();
>>>>>        mmu_notifier_synchronize();
>>>>>        amdgpu_xcp_drv_release();
>>>>>    }
>>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>>>>> index a58342c2ac44..909bdccc2a92 100644
>>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>>>>> @@ -32,29 +32,9 @@
>>>>>    #include "amdgpu.h"
>>>>>    #include "amdgpu_userq_fence.h"
>>>>>    -static const struct dma_fence_ops amdgpu_userq_fence_ops;
>>>>> -static struct kmem_cache *amdgpu_userq_fence_slab;
>>>>> -
>>>>>    #define AMDGPU_USERQ_MAX_HANDLES    (1U << 16)
>>>>>    -int amdgpu_userq_fence_slab_init(void)
>>>>> -{
>>>>> -    amdgpu_userq_fence_slab = kmem_cache_create("amdgpu_userq_fence",
>>>>> -                            sizeof(struct amdgpu_userq_fence),
>>>>> -                            0,
>>>>> -                            SLAB_HWCACHE_ALIGN,
>>>>> -                            NULL);
>>>> Are we not having benefit enough to continue create a cache here ? If that is fine that LGTM,
>>> Using all those kmem_cache instances was a bad idea to begin with.
>>>
>>> See the idea of a kmem_cache is to reduce the number of CPU cache lines and memory you need for certain number of objects when the object size is not a power of two.
>>>
>>> So for exampe two objects with 96 bytes only take 3 cache lines and 192 bytes instead of 256 bytes and 4 cache lines.
>>>
>>> But that difference is so marginally for most use cases that you absolutely don't need it.
>> Thanks for the explanation. Also if i am not wrong in cases where last no of such objects are often created and deleted kmem_cache helps in reuse and helps with internal fragmentation too.
> Nope that's not correct. See kmalloc() and co also uses kmem_cache underneath, they just round up the size of the allocated object to the next power of two.
>
> As long as you don't give special flags kmem_cache object storages are also shared among users, e.g. when one driver creates a kmem_cache for an 96 byte sized object and another driver does the same they will actually share their allocations which each other.
>
> Regards,
> Christian.
>
>> Regards
>>
>> Sunil Khatri
>>
>>> Regards,
>>> Christian.
>>>
>>>> Acked-by: Sunil Khatri <sunil.khatri@amd.com>
>>>>
>>>> Regards
>>>> Sunil
>>>>
>>>>
>>>>
>>>>> -    if (!amdgpu_userq_fence_slab)
>>>>> -        return -ENOMEM;
>>>>> -
>>>>> -    return 0;
>>>>> -}
>>>>> -
>>>>> -void amdgpu_userq_fence_slab_fini(void)
>>>>> -{
>>>>> -    rcu_barrier();
>>>>> -    kmem_cache_destroy(amdgpu_userq_fence_slab);
>>>>> -}
>>>>> +static const struct dma_fence_ops amdgpu_userq_fence_ops;
>>>>>      static inline struct amdgpu_userq_fence *to_amdgpu_userq_fence(struct dma_fence *f)
>>>>>    {
>>>>> @@ -146,12 +126,18 @@ amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq)
>>>>>    }
>>>>>      static void
>>>>> -amdgpu_userq_fence_put_fence_drv_array(struct amdgpu_userq_fence *userq_fence)
>>>>> +amdgpu_userq_fence_put_fence_drv_refs(struct amdgpu_userq_fence *userq_fence)
>>>>>    {
>>>>>        unsigned long i;
>>>>> +
>>>>>        for (i = 0; i < userq_fence->fence_drv_array_count; i++)
>>>>>            amdgpu_userq_fence_driver_put(userq_fence->fence_drv_array[i]);
>>>>>        userq_fence->fence_drv_array_count = 0;
>>>>> +    kfree(userq_fence->fence_drv_array);
>>>>> +    userq_fence->fence_drv_array = NULL;
>>>>> +
>>>>> +    amdgpu_userq_fence_driver_put(userq_fence->fence_drv);
>>>>> +    userq_fence->fence_drv = NULL;

Setting this to NULL is hitting a NULL dereference in, so either we add 
a NULL check in the amdgpu_userq_fence_driver_process or lets not set 
the NULL.

Regards
Sunil Khatri

>>>>>    }
>>>>>      void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
>>>>> @@ -181,10 +167,11 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
>>>>>            fence = &userq_fence->base;
>>>>>            list_del_init(&userq_fence->link);
>>>>>            dma_fence_signal(fence);
>>>>> -        /* Drop fence_drv_array outside fence_list_lock
>>>>> +        /*
>>>>> +         * Drop fence_drv_array outside fence_list_lock
>>>>>             * to avoid the recursion lock.
>>>>>             */
>>>>> -        amdgpu_userq_fence_put_fence_drv_array(userq_fence);
>>>>> +        amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
>>>>>            dma_fence_put(fence);
>>>>>        }
>>>>>    @@ -231,7 +218,7 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
>>>>>      static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
>>>>>    {
>>>>> -    *userq_fence = kmem_cache_alloc(amdgpu_userq_fence_slab, GFP_ATOMIC);
>>>>> +    *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
>>>>>        return *userq_fence ? 0 : -ENOMEM;
>>>>>    }
>>>>>    @@ -299,7 +286,7 @@ static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
>>>>>        spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
>>>>>          if (signaled)
>>>>> -        amdgpu_userq_fence_put_fence_drv_array(userq_fence);
>>>>> +        amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
>>>>>          *f = fence;
>>>>>    @@ -333,29 +320,10 @@ static bool amdgpu_userq_fence_signaled(struct dma_fence *f)
>>>>>        return false;
>>>>>    }
>>>>>    -static void amdgpu_userq_fence_free(struct rcu_head *rcu)
>>>>> -{
>>>>> -    struct dma_fence *fence = container_of(rcu, struct dma_fence, rcu);
>>>>> -    struct amdgpu_userq_fence *userq_fence = to_amdgpu_userq_fence(fence);
>>>>> -    struct amdgpu_userq_fence_driver *fence_drv = userq_fence->fence_drv;
>>>>> -
>>>>> -    /* Release the fence driver reference */
>>>>> -    amdgpu_userq_fence_driver_put(fence_drv);
>>>>> -
>>>>> -    kvfree(userq_fence->fence_drv_array);
>>>>> -    kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
>>>>> -}
>>>>> -
>>>>> -static void amdgpu_userq_fence_release(struct dma_fence *f)
>>>>> -{
>>>>> -    call_rcu(&f->rcu, amdgpu_userq_fence_free);
>>>>> -}
>>>>> -
>>>>>    static const struct dma_fence_ops amdgpu_userq_fence_ops = {
>>>>>        .get_driver_name = amdgpu_userq_fence_get_driver_name,
>>>>>        .get_timeline_name = amdgpu_userq_fence_get_timeline_name,
>>>>>        .signaled = amdgpu_userq_fence_signaled,
>>>>> -    .release = amdgpu_userq_fence_release,
>>>>>    };
>>>>>      /**
>>>>> @@ -546,7 +514,7 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>>>>>        r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
>>>>>        if (r) {
>>>>>            mutex_unlock(&userq_mgr->userq_mutex);
>>>>> -        kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
>>>>> +        kfree(userq_fence);
>>>>>            goto put_gobj_write;
>>>>>        }
>>>>>    @@ -871,6 +839,7 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>>>>>        for (i = 0, cnt = 0; i < num_fences; i++) {
>>>>>            struct amdgpu_userq_fence_driver *fence_drv;
>>>>>            struct amdgpu_userq_fence *userq_fence;
>>>>> +        unsigned long flags;
>>>>>            u32 index;
>>>>>              userq_fence = to_amdgpu_userq_fence(fences[i]);
>>>>> @@ -886,7 +855,19 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>>>>>                continue;
>>>>>            }
>>>>>    +        spin_lock_irqsave(userq_fence->base.lock, flags);
>>>>> +        if (dma_fence_is_signaled_locked(&userq_fence->base)) {
>>>>> +            /*
>>>>> +             * It is possible that fence is already signaled and the
>>>>> +             * fence_drv now NULL, just skip over such fences.
>>>>> +            */
>>>>> +            spin_unlock_irqrestore(userq_fence->base.lock, flags);
>>>>> +            continue;
>>>>> +        }
>>>>>            fence_drv = userq_fence->fence_drv;
>>>>> +        amdgpu_userq_fence_driver_get(fence_drv);
>>>>> +        spin_unlock_irqrestore(userq_fence->base.lock, flags);
>>>>> +
>>>>>            /*
>>>>>             * We need to make sure the user queue release their reference
>>>>>             * to the fence drivers at some point before queue destruction.
>>>>> @@ -895,10 +876,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>>>>>             */
>>>>>            r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
>>>>>                     xa_limit_32b, GFP_KERNEL);
>>>>> -        if (r)
>>>>> +        if (r) {
>>>>> +            amdgpu_userq_fence_driver_put(fence_drv);
>>>>>                goto put_waitq;
>>>>> -
>>>>> -        amdgpu_userq_fence_driver_get(fence_drv);
>>>>> +        }
>>>>>              /* Store drm syncobj's gpu va address and value */
>>>>>            fence_info[cnt].va = fence_drv->va;
>>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
>>>>> index d56246ad8c26..d355a0eecc07 100644
>>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
>>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
>>>>> @@ -58,9 +58,6 @@ struct amdgpu_userq_fence_driver {
>>>>>        char timeline_name[TASK_COMM_LEN];
>>>>>    };
>>>>>    -int amdgpu_userq_fence_slab_init(void);
>>>>> -void amdgpu_userq_fence_slab_fini(void);
>>>>> -
>>>>>    void amdgpu_userq_fence_driver_get(struct amdgpu_userq_fence_driver *fence_drv);
>>>>>    void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv);
>>>>>    int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,

Re: [PATCH 04/11] drm/amdgpu: rework amdgpu_userq_signal_ioctl

[Khatri, Sunil]

On 22-04-2026 03:44 pm, Christian König wrote:
> On 4/22/26 12:08, Khatri, Sunil wrote:
>> On 21-04-2026 06:25 pm, Christian König wrote:
>>> This one was fortunately not looking so bad as the wait ioctl path, but
>>> there were still a few things which could be fixed/improved:
>>>
>>> 1. Allocating with GFP_ATOMIC was quite unecessary, we can do that
>>>      before taking the userq_lock.
>>> 2. Use a new mutex as protection for the fence_drv_xa so that we can do
>>>      memory allocations while holding it.
>>> 3. Starting the reset timer is unecessary when the fence is already
>>>      signaled when we create it.
>>> 4. Cleanup error handling, avoid trying to free the queue when we don't
>>>      even got one.
>>>
>>> Signed-off-by: Christian König <christian.koenig@amd.com>
>>> ---
>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c     |   1 +
>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h     |  12 +
>>>    .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 220 ++++++++----------
>>>    3 files changed, 111 insertions(+), 122 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>>> index b632bc3c952b..174190a77005 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>>> @@ -793,6 +793,7 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
>>>        }
>>>          queue->doorbell_index = index;
>>> +    mutex_init(&queue->fence_drv_lock);
>> we do want to destroy the mutex in case queue creation fails ? RIght now amdgpu_userq_fence_driver_alloc fails we goto clean_mapping and not destroying the mutex which is done in amdgpu_userq_fence_driver_free.
>> goto needs to be modified i guess to handle it.
> Good point, going to clean that up.
>
>> Apart from that LGTM. Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
> Thanks,
> Christian
>
>> Regards
>> Sunil
>>>        xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
>>>        r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
>>>        if (r) {
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
>>> index 675fe6395ac8..cb92789c1ed1 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
>>> @@ -66,6 +66,18 @@ struct amdgpu_usermode_queue {
>>>        struct amdgpu_userq_obj    db_obj;
>>>        struct amdgpu_userq_obj fw_obj;
>>>        struct amdgpu_userq_obj wptr_obj;
>>> +
>>> +    /**
>>> +     * @fence_drv_lock: Protecting @fence_drv_xa.
>>> +     */
>>> +    struct mutex        fence_drv_lock;
>>> +
>>> +    /**
>>> +     * @fence_drv_xa:
>>> +     *
>>> +     * References to the external fence drivers returned by wait_ioctl.
>>> +     * Dropped on the next signaled dma_fence or queue destruction.
>>> +     */
>>>        struct xarray        fence_drv_xa;
>>>        struct amdgpu_userq_fence_driver *fence_drv;
>>>        struct dma_fence    *last_fence;
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>>> index 909bdccc2a92..b0543fa257ed 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
>>> @@ -121,6 +121,7 @@ amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq)
>>>        userq->last_fence = NULL;
>>>        amdgpu_userq_walk_and_drop_fence_drv(&userq->fence_drv_xa);
>>>        xa_destroy(&userq->fence_drv_xa);
>>> +    mutex_destroy(&userq->fence_drv_lock);
>>>        /* Drop the queue's ownership reference to fence_drv explicitly */
>>>        amdgpu_userq_fence_driver_put(userq->fence_drv);
>>>    }
>>> @@ -216,81 +217,77 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
>>>        kref_put(&fence_drv->refcount, amdgpu_userq_fence_driver_destroy);
>>>    }
>>>    -static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
>>> +static int amdgpu_userq_fence_alloc(struct amdgpu_usermode_queue *userq,
>>> +                    struct amdgpu_userq_fence **pfence)
>>>    {
>>> -    *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
>>> -    return *userq_fence ? 0 : -ENOMEM;
>>> +    struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
>>> +    struct amdgpu_userq_fence *userq_fence;
>>> +    unsigned long count;

we need to set the initial value of count to 0 to have a valid count 
value else it goes to some huge random garbage value. eventually leading 
to memory allocation failure and app crash.

regards
Sunil Khatri

>>> +
>>> +    userq_fence = kmalloc(sizeof(*userq_fence), GFP_KERNEL);
>>> +    if (!userq_fence)
>>> +        return -ENOMEM;
>>> +
>>> +    /*
>>> +     * Get the next unused entry, since we fill from the start this can be
>>> +     * used as size to allocate the array.
>>> +     */
>>> +    mutex_lock(&userq->fence_drv_lock);
>>> +    xa_find(&userq->fence_drv_xa, &count, ULONG_MAX, XA_FREE_MARK);
We are using that count value in below kvmalloc_Array and that fails and 
we return ENOMEM.
>>> +
>>> +    userq_fence->fence_drv_array = kvmalloc_array(count, sizeof(fence_drv),
>>> +                              GFP_KERNEL);
>>> +    if (!userq_fence->fence_drv_array) {
>>> +        mutex_unlock(&userq->fence_drv_lock);
>>> +        kfree(userq_fence);
>>> +        return -ENOMEM;
>>> +    }
>>> +
>>> +    userq_fence->fence_drv_array_count = count;
>>> +    xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array,
>>> +           0, ULONG_MAX, count, XA_PRESENT);
>>> +    xa_destroy(&userq->fence_drv_xa);
>>> +
>>> +    mutex_unlock(&userq->fence_drv_lock);
>>> +
>>> +    userq_fence->fence_drv = fence_drv;
>>> +    amdgpu_userq_fence_driver_get(fence_drv);
>>> +
>>> +    *pfence = userq_fence;
>>> +    return 0;
>>>    }
>>>    -static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
>>> -                     struct amdgpu_userq_fence *userq_fence,
>>> -                     u64 seq, struct dma_fence **f)
>>> +static void amdgpu_userq_fence_init(struct amdgpu_usermode_queue *userq,
>>> +                    struct amdgpu_userq_fence *fence,
>>> +                    u64 seq)
>>>    {
>>> -    struct amdgpu_userq_fence_driver *fence_drv;
>>> -    struct dma_fence *fence;
>>> +    struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
>>>        unsigned long flags;
>>>        bool signaled = false;
>>>    -    fence_drv = userq->fence_drv;
>>> -    if (!fence_drv)
>>> -        return -EINVAL;
>>> -
>>> -    spin_lock_init(&userq_fence->lock);
>>> -    INIT_LIST_HEAD(&userq_fence->link);
>>> -    fence = &userq_fence->base;
>>> -    userq_fence->fence_drv = fence_drv;
>>> -
>>> -    dma_fence_init64(fence, &amdgpu_userq_fence_ops, &userq_fence->lock,
>>> +    spin_lock_init(&fence->lock);
>>> +    dma_fence_init64(&fence->base, &amdgpu_userq_fence_ops, &fence->lock,
>>>                 fence_drv->context, seq);
>>>    -    amdgpu_userq_fence_driver_get(fence_drv);
>>> -    dma_fence_get(fence);
>>> -
>>> -    if (!xa_empty(&userq->fence_drv_xa)) {
>>> -        struct amdgpu_userq_fence_driver *stored_fence_drv;
>>> -        unsigned long index, count = 0;
>>> -        int i = 0;
>>> -
>>> -        xa_lock(&userq->fence_drv_xa);
>>> -        xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv)
>>> -            count++;
>>> -
>>> -        userq_fence->fence_drv_array =
>>> -            kvmalloc_array(count,
>>> -                       sizeof(struct amdgpu_userq_fence_driver *),
>>> -                       GFP_ATOMIC);
>>> -
>>> -        if (userq_fence->fence_drv_array) {
>>> -            xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv) {
>>> -                userq_fence->fence_drv_array[i] = stored_fence_drv;
>>> -                __xa_erase(&userq->fence_drv_xa, index);
>>> -                i++;
>>> -            }
>>> -        }
>>> -
>>> -        userq_fence->fence_drv_array_count = i;
>>> -        xa_unlock(&userq->fence_drv_xa);
>>> -    } else {
>>> -        userq_fence->fence_drv_array = NULL;
>>> -        userq_fence->fence_drv_array_count = 0;
>>> -    }
>>> +    /* Make sure the fence is visible to the hang detect worker */
>>> +    dma_fence_put(userq->last_fence);
>>> +    userq->last_fence = dma_fence_get(&fence->base);
>>>    -    /* Check if hardware has already processed the job */
>>> +    /* Check if hardware has already processed the fence */
>>>        spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
>>> -    if (!dma_fence_is_signaled(fence)) {
>>> -        list_add_tail(&userq_fence->link, &fence_drv->fences);
>>> +    if (!dma_fence_is_signaled(&fence->base)) {
>>> +        dma_fence_get(&fence->base);
>>> +        list_add_tail(&fence->link, &fence_drv->fences);
>>>        } else {
>>> +        INIT_LIST_HEAD(&fence->link);
>>>            signaled = true;
>>> -        dma_fence_put(fence);
>>>        }
>>>        spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
>>>          if (signaled)
>>> -        amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
>>> -
>>> -    *f = fence;
>>> -
>>> -    return 0;
>>> +        amdgpu_userq_fence_put_fence_drv_refs(fence);
>>> +    else
>>> +        amdgpu_userq_start_hang_detect_work(userq);
>>>    }
>>>      static const char *amdgpu_userq_fence_get_driver_name(struct dma_fence *f)
>>> @@ -392,11 +389,6 @@ static int amdgpu_userq_fence_read_wptr(struct amdgpu_device *adev,
>>>        return r;
>>>    }
>>>    -static void amdgpu_userq_fence_cleanup(struct dma_fence *fence)
>>> -{
>>> -    dma_fence_put(fence);
>>> -}
>>> -
>>>    static void
>>>    amdgpu_userq_fence_driver_set_error(struct amdgpu_userq_fence *fence,
>>>                        int error)
>>> @@ -440,13 +432,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>>>        const unsigned int num_read_bo_handles = args->num_bo_read_handles;
>>>        struct amdgpu_fpriv *fpriv = filp->driver_priv;
>>>        struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
>>> +
>>>        struct drm_gem_object **gobj_write, **gobj_read;
>>>        u32 *syncobj_handles, num_syncobj_handles;
>>> -    struct amdgpu_userq_fence *userq_fence;
>>> -    struct amdgpu_usermode_queue *queue = NULL;
>>> -    struct drm_syncobj **syncobj = NULL;
>>> -    struct dma_fence *fence;
>>> +    struct amdgpu_usermode_queue *queue;
>>> +    struct amdgpu_userq_fence *fence;
>>> +    struct drm_syncobj **syncobj;
>>>        struct drm_exec exec;
>>> +    void __user *ptr;
>>>        int r, i, entry;
>>>        u64 wptr;
>>>    @@ -458,13 +451,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>>>            return -EINVAL;
>>>          num_syncobj_handles = args->num_syncobj_handles;
>>> -    syncobj_handles = memdup_array_user(u64_to_user_ptr(args->syncobj_handles),
>>> -                        num_syncobj_handles, sizeof(u32));
>>> +    ptr = u64_to_user_ptr(args->syncobj_handles);
>>> +    syncobj_handles = memdup_array_user(ptr, num_syncobj_handles,
>>> +                        sizeof(u32));
>>>        if (IS_ERR(syncobj_handles))
>>>            return PTR_ERR(syncobj_handles);
>>>    -    /* Array of pointers to the looked up syncobjs */
>>> -    syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj), GFP_KERNEL);
>>> +    syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj),
>>> +                GFP_KERNEL);
>>>        if (!syncobj) {
>>>            r = -ENOMEM;
>>>            goto free_syncobj_handles;
>>> @@ -478,21 +472,17 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>>>            }
>>>        }
>>>    -    r = drm_gem_objects_lookup(filp,
>>> -                   u64_to_user_ptr(args->bo_read_handles),
>>> -                   num_read_bo_handles,
>>> -                   &gobj_read);
>>> +    ptr = u64_to_user_ptr(args->bo_read_handles);
>>> +    r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
>>>        if (r)
>>>            goto free_syncobj;
>>>    -    r = drm_gem_objects_lookup(filp,
>>> -                   u64_to_user_ptr(args->bo_write_handles),
>>> -                   num_write_bo_handles,
>>> +    ptr = u64_to_user_ptr(args->bo_write_handles);
>>> +    r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
>>>                       &gobj_write);
>>>        if (r)
>>>            goto put_gobj_read;
>>>    -    /* Retrieve the user queue */
>>>        queue = amdgpu_userq_get(userq_mgr, args->queue_id);
>>>        if (!queue) {
>>>            r = -ENOENT;
>>> @@ -501,73 +491,61 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>>>          r = amdgpu_userq_fence_read_wptr(adev, queue, &wptr);
>>>        if (r)
>>> -        goto put_gobj_write;
>>> +        goto put_queue;
>>>    -    r = amdgpu_userq_fence_alloc(&userq_fence);
>>> +    r = amdgpu_userq_fence_alloc(queue, &fence);
>>>        if (r)
>>> -        goto put_gobj_write;
>>> +        goto put_queue;
>>>          /* We are here means UQ is active, make sure the eviction fence is valid */
>>>        amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
>>>    -    /* Create a new fence */
>>> -    r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
>>> -    if (r) {
>>> -        mutex_unlock(&userq_mgr->userq_mutex);
>>> -        kfree(userq_fence);
>>> -        goto put_gobj_write;
>>> -    }
>>> +    /* Create the new fence */
>>> +    amdgpu_userq_fence_init(queue, fence, wptr);
>>>    -    dma_fence_put(queue->last_fence);
>>> -    queue->last_fence = dma_fence_get(fence);
>>> -    amdgpu_userq_start_hang_detect_work(queue);
>>>        mutex_unlock(&userq_mgr->userq_mutex);
>>>    +    /*
>>> +     * This needs to come after the fence is created since
>>> +     * amdgpu_userq_ensure_ev_fence() can't be called while holding the resv
>>> +     * locks.
>>> +     */
>>>        drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
>>>                  (num_read_bo_handles + num_write_bo_handles));
>>>    -    /* Lock all BOs with retry handling */
>>>        drm_exec_until_all_locked(&exec) {
>>> -        r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1);
>>> +        r = drm_exec_prepare_array(&exec, gobj_read,
>>> +                       num_read_bo_handles, 1);
>>>            drm_exec_retry_on_contention(&exec);
>>> -        if (r) {
>>> -            amdgpu_userq_fence_cleanup(fence);
>>> +        if (r)
>>>                goto exec_fini;
>>> -        }
>>>    -        r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1);
>>> +        r = drm_exec_prepare_array(&exec, gobj_write,
>>> +                       num_write_bo_handles, 1);
>>>            drm_exec_retry_on_contention(&exec);
>>> -        if (r) {
>>> -            amdgpu_userq_fence_cleanup(fence);
>>> +        if (r)
>>>                goto exec_fini;
>>> -        }
>>>        }
>>>    -    for (i = 0; i < num_read_bo_handles; i++) {
>>> -        if (!gobj_read || !gobj_read[i]->resv)
>>> -            continue;
>>> -
>>> -        dma_resv_add_fence(gobj_read[i]->resv, fence,
>>> +    /* And publish the new fence in the BOs and syncobj */
>>> +    for (i = 0; i < num_read_bo_handles; i++)
>>> +        dma_resv_add_fence(gobj_read[i]->resv, &fence->base,
>>>                       DMA_RESV_USAGE_READ);
>>> -    }
>>>    -    for (i = 0; i < num_write_bo_handles; i++) {
>>> -        if (!gobj_write || !gobj_write[i]->resv)
>>> -            continue;
>>> -
>>> -        dma_resv_add_fence(gobj_write[i]->resv, fence,
>>> +    for (i = 0; i < num_write_bo_handles; i++)
>>> +        dma_resv_add_fence(gobj_write[i]->resv, &fence->base,
>>>                       DMA_RESV_USAGE_WRITE);
>>> -    }
>>>    -    /* Add the created fence to syncobj/BO's */
>>>        for (i = 0; i < num_syncobj_handles; i++)
>>> -        drm_syncobj_replace_fence(syncobj[i], fence);
>>> +        drm_syncobj_replace_fence(syncobj[i], &fence->base);
>>>    +exec_fini:
>>>        /* drop the reference acquired in fence creation function */
>>> -    dma_fence_put(fence);
>>> +    dma_fence_put(&fence->base);
>>>    -exec_fini:
>>>        drm_exec_fini(&exec);
>>> +put_queue:
>>> +    amdgpu_userq_put(queue);
>>>    put_gobj_write:
>>>        for (i = 0; i < num_write_bo_handles; i++)
>>>            drm_gem_object_put(gobj_write[i]);
>>> @@ -578,15 +556,11 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>>>        kvfree(gobj_read);
>>>    free_syncobj:
>>>        while (entry-- > 0)
>>> -        if (syncobj[entry])
>>> -            drm_syncobj_put(syncobj[entry]);
>>> +        drm_syncobj_put(syncobj[entry]);
>>>        kfree(syncobj);
>>>    free_syncobj_handles:
>>>        kfree(syncobj_handles);
>>>    -    if (queue)
>>> -        amdgpu_userq_put(queue);
>>> -
>>>        return r;
>>>    }
>>>    @@ -874,8 +848,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>>>             * Otherwise, we would gather those references until we don't
>>>             * have any more space left and crash.
>>>             */
>>> +        mutex_lock(&waitq->fence_drv_lock);
>>>            r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
>>>                     xa_limit_32b, GFP_KERNEL);
>>> +        mutex_unlock(&waitq->fence_drv_lock);
>>>            if (r) {
>>>                amdgpu_userq_fence_driver_put(fence_drv);
>>>                goto put_waitq;

RE: [PATCH 04/11] drm/amdgpu: rework amdgpu_userq_signal_ioctl

[Liang, Prike]

[Public]

Regards,
      Prike

> -----Original Message-----
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> Sent: Tuesday, April 21, 2026 8:55 PM
> To: Deucher, Alexander <Alexander.Deucher@amd.com>; Liang, Prike
> <Prike.Liang@amd.com>; Khatri, Sunil <Sunil.Khatri@amd.com>; amd-
> gfx@lists.freedesktop.org
> Cc: Koenig, Christian <Christian.Koenig@amd.com>
> Subject: [PATCH 04/11] drm/amdgpu: rework amdgpu_userq_signal_ioctl
>
> This one was fortunately not looking so bad as the wait ioctl path, but there were still
> a few things which could be fixed/improved:
>
> 1. Allocating with GFP_ATOMIC was quite unecessary, we can do that
>    before taking the userq_lock.
> 2. Use a new mutex as protection for the fence_drv_xa so that we can do
>    memory allocations while holding it.
> 3. Starting the reset timer is unecessary when the fence is already
>    signaled when we create it.
> 4. Cleanup error handling, avoid trying to free the queue when we don't
>    even got one.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c     |   1 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h     |  12 +
>  .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 220 ++++++++----------
>  3 files changed, 111 insertions(+), 122 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index b632bc3c952b..174190a77005 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -793,6 +793,7 @@ amdgpu_userq_create(struct drm_file *filp, union
> drm_amdgpu_userq *args)
>       }
>
>       queue->doorbell_index = index;
> +     mutex_init(&queue->fence_drv_lock);
>       xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
>       r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
>       if (r) {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> index 675fe6395ac8..cb92789c1ed1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> @@ -66,6 +66,18 @@ struct amdgpu_usermode_queue {
>       struct amdgpu_userq_obj db_obj;
>       struct amdgpu_userq_obj fw_obj;
>       struct amdgpu_userq_obj wptr_obj;
> +
> +     /**
> +      * @fence_drv_lock: Protecting @fence_drv_xa.
> +      */
> +     struct mutex            fence_drv_lock;
> +
> +     /**
> +      * @fence_drv_xa:
> +      *
> +      * References to the external fence drivers returned by wait_ioctl.
> +      * Dropped on the next signaled dma_fence or queue destruction.
> +      */
>       struct xarray           fence_drv_xa;
>       struct amdgpu_userq_fence_driver *fence_drv;
>       struct dma_fence        *last_fence;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> index 909bdccc2a92..b0543fa257ed 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> @@ -121,6 +121,7 @@ amdgpu_userq_fence_driver_free(struct
> amdgpu_usermode_queue *userq)
>       userq->last_fence = NULL;
>       amdgpu_userq_walk_and_drop_fence_drv(&userq->fence_drv_xa);
>       xa_destroy(&userq->fence_drv_xa);
> +     mutex_destroy(&userq->fence_drv_lock);
>       /* Drop the queue's ownership reference to fence_drv explicitly */
>       amdgpu_userq_fence_driver_put(userq->fence_drv);
>  }
> @@ -216,81 +217,77 @@ void amdgpu_userq_fence_driver_put(struct
> amdgpu_userq_fence_driver *fence_drv)
>       kref_put(&fence_drv->refcount, amdgpu_userq_fence_driver_destroy);
>  }
>
> -static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
> +static int amdgpu_userq_fence_alloc(struct amdgpu_usermode_queue *userq,
> +                                 struct amdgpu_userq_fence **pfence)
>  {
> -     *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
> -     return *userq_fence ? 0 : -ENOMEM;
> +     struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
> +     struct amdgpu_userq_fence *userq_fence;
> +     unsigned long count;
We must initialize count; otherwise, it may contain a garbage value, which can cause amdgpu_userq_fence_alloc() to fail and,
in turn, make userq fence emission fail.

> +     userq_fence = kmalloc(sizeof(*userq_fence), GFP_KERNEL);
> +     if (!userq_fence)
> +             return -ENOMEM;
> +
> +     /*
> +      * Get the next unused entry, since we fill from the start this can be
> +      * used as size to allocate the array.
> +      */
> +     mutex_lock(&userq->fence_drv_lock);
> +     xa_find(&userq->fence_drv_xa, &count, ULONG_MAX, XA_FREE_MARK);
> +
> +     userq_fence->fence_drv_array = kvmalloc_array(count, sizeof(fence_drv),
> +                                                   GFP_KERNEL);
> +     if (!userq_fence->fence_drv_array) {
> +             mutex_unlock(&userq->fence_drv_lock);
> +             kfree(userq_fence);
> +             return -ENOMEM;
> +     }
> +
> +     userq_fence->fence_drv_array_count = count;
> +     xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array,
> +                0, ULONG_MAX, count, XA_PRESENT);
We may need to assign the userq_fence->fence_drv_array_count the exact copied number from the xa_extract().

> +     xa_destroy(&userq->fence_drv_xa);
> +
> +     mutex_unlock(&userq->fence_drv_lock);
> +
> +     userq_fence->fence_drv = fence_drv;
> +     amdgpu_userq_fence_driver_get(fence_drv);
> +
> +     *pfence = userq_fence;
> +     return 0;
>  }
>
> -static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
> -                                  struct amdgpu_userq_fence *userq_fence,
> -                                  u64 seq, struct dma_fence **f)
> +static void amdgpu_userq_fence_init(struct amdgpu_usermode_queue *userq,
> +                                 struct amdgpu_userq_fence *fence,
> +                                 u64 seq)
>  {
> -     struct amdgpu_userq_fence_driver *fence_drv;
> -     struct dma_fence *fence;
> +     struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
>       unsigned long flags;
>       bool signaled = false;
>
> -     fence_drv = userq->fence_drv;
> -     if (!fence_drv)
> -             return -EINVAL;
> -
> -     spin_lock_init(&userq_fence->lock);
> -     INIT_LIST_HEAD(&userq_fence->link);
> -     fence = &userq_fence->base;
> -     userq_fence->fence_drv = fence_drv;
> -
> -     dma_fence_init64(fence, &amdgpu_userq_fence_ops, &userq_fence->lock,
> +     spin_lock_init(&fence->lock);
> +     dma_fence_init64(&fence->base, &amdgpu_userq_fence_ops, &fence->lock,
>                        fence_drv->context, seq);
>
> -     amdgpu_userq_fence_driver_get(fence_drv);
> -     dma_fence_get(fence);
> -
> -     if (!xa_empty(&userq->fence_drv_xa)) {
> -             struct amdgpu_userq_fence_driver *stored_fence_drv;
> -             unsigned long index, count = 0;
> -             int i = 0;
> -
> -             xa_lock(&userq->fence_drv_xa);
> -             xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv)
> -                     count++;
> -
> -             userq_fence->fence_drv_array =
> -                     kvmalloc_array(count,
> -                                    sizeof(struct amdgpu_userq_fence_driver *),
> -                                    GFP_ATOMIC);
> -
> -             if (userq_fence->fence_drv_array) {
> -                     xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv)
> {
> -                             userq_fence->fence_drv_array[i] = stored_fence_drv;
> -                             __xa_erase(&userq->fence_drv_xa, index);
> -                             i++;
> -                     }
> -             }
> -
> -             userq_fence->fence_drv_array_count = i;
> -             xa_unlock(&userq->fence_drv_xa);
> -     } else {
> -             userq_fence->fence_drv_array = NULL;
> -             userq_fence->fence_drv_array_count = 0;
> -     }
> +     /* Make sure the fence is visible to the hang detect worker */
> +     dma_fence_put(userq->last_fence);
> +     userq->last_fence = dma_fence_get(&fence->base);
>
> -     /* Check if hardware has already processed the job */
> +     /* Check if hardware has already processed the fence */
>       spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
> -     if (!dma_fence_is_signaled(fence)) {
> -             list_add_tail(&userq_fence->link, &fence_drv->fences);
> +     if (!dma_fence_is_signaled(&fence->base)) {
> +             dma_fence_get(&fence->base);
> +             list_add_tail(&fence->link, &fence_drv->fences);
>       } else {
> +             INIT_LIST_HEAD(&fence->link);
>               signaled = true;
> -             dma_fence_put(fence);
>       }
>       spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
>
>       if (signaled)
> -             amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
> -
> -     *f = fence;
> -
> -     return 0;
> +             amdgpu_userq_fence_put_fence_drv_refs(fence);
> +     else
> +             amdgpu_userq_start_hang_detect_work(userq);
>  }
>
>  static const char *amdgpu_userq_fence_get_driver_name(struct dma_fence *f) @@
> -392,11 +389,6 @@ static int amdgpu_userq_fence_read_wptr(struct
> amdgpu_device *adev,
>       return r;
>  }
>
> -static void amdgpu_userq_fence_cleanup(struct dma_fence *fence) -{
> -     dma_fence_put(fence);
> -}
> -
>  static void
>  amdgpu_userq_fence_driver_set_error(struct amdgpu_userq_fence *fence,
>                                   int error)
> @@ -440,13 +432,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev,
> void *data,
>       const unsigned int num_read_bo_handles = args->num_bo_read_handles;
>       struct amdgpu_fpriv *fpriv = filp->driver_priv;
>       struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
> +
>       struct drm_gem_object **gobj_write, **gobj_read;
>       u32 *syncobj_handles, num_syncobj_handles;
> -     struct amdgpu_userq_fence *userq_fence;
> -     struct amdgpu_usermode_queue *queue = NULL;
> -     struct drm_syncobj **syncobj = NULL;
> -     struct dma_fence *fence;
> +     struct amdgpu_usermode_queue *queue;
> +     struct amdgpu_userq_fence *fence;
> +     struct drm_syncobj **syncobj;
>       struct drm_exec exec;
> +     void __user *ptr;
>       int r, i, entry;
>       u64 wptr;
>
> @@ -458,13 +451,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev,
> void *data,
>               return -EINVAL;
>
>       num_syncobj_handles = args->num_syncobj_handles;
> -     syncobj_handles = memdup_array_user(u64_to_user_ptr(args-
> >syncobj_handles),
> -                                         num_syncobj_handles, sizeof(u32));
> +     ptr = u64_to_user_ptr(args->syncobj_handles);
> +     syncobj_handles = memdup_array_user(ptr, num_syncobj_handles,
> +                                         sizeof(u32));
>       if (IS_ERR(syncobj_handles))
>               return PTR_ERR(syncobj_handles);
>
> -     /* Array of pointers to the looked up syncobjs */
> -     syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj),
> GFP_KERNEL);
> +     syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj),
> +                             GFP_KERNEL);
>       if (!syncobj) {
>               r = -ENOMEM;
>               goto free_syncobj_handles;
> @@ -478,21 +472,17 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev,
> void *data,
>               }
>       }
>
> -     r = drm_gem_objects_lookup(filp,
> -                                u64_to_user_ptr(args->bo_read_handles),
> -                                num_read_bo_handles,
> -                                &gobj_read);
> +     ptr = u64_to_user_ptr(args->bo_read_handles);
> +     r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles,
> +&gobj_read);
>       if (r)
>               goto free_syncobj;
>
> -     r = drm_gem_objects_lookup(filp,
> -                                u64_to_user_ptr(args->bo_write_handles),
> -                                num_write_bo_handles,
> +     ptr = u64_to_user_ptr(args->bo_write_handles);
> +     r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
>                                  &gobj_write);
>       if (r)
>               goto put_gobj_read;
>
> -     /* Retrieve the user queue */
>       queue = amdgpu_userq_get(userq_mgr, args->queue_id);
>       if (!queue) {
>               r = -ENOENT;
> @@ -501,73 +491,61 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev,
> void *data,
>
>       r = amdgpu_userq_fence_read_wptr(adev, queue, &wptr);
>       if (r)
> -             goto put_gobj_write;
> +             goto put_queue;
>
> -     r = amdgpu_userq_fence_alloc(&userq_fence);
> +     r = amdgpu_userq_fence_alloc(queue, &fence);
>       if (r)
> -             goto put_gobj_write;
> +             goto put_queue;
>
>       /* We are here means UQ is active, make sure the eviction fence is valid */
>       amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
>
> -     /* Create a new fence */
> -     r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
> -     if (r) {
> -             mutex_unlock(&userq_mgr->userq_mutex);
> -             kfree(userq_fence);
> -             goto put_gobj_write;
> -     }
> +     /* Create the new fence */
> +     amdgpu_userq_fence_init(queue, fence, wptr);
>
> -     dma_fence_put(queue->last_fence);
> -     queue->last_fence = dma_fence_get(fence);
> -     amdgpu_userq_start_hang_detect_work(queue);
>       mutex_unlock(&userq_mgr->userq_mutex);
>
> +     /*
> +      * This needs to come after the fence is created since
> +      * amdgpu_userq_ensure_ev_fence() can't be called while holding the resv
> +      * locks.
> +      */
>       drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
>                     (num_read_bo_handles + num_write_bo_handles));
>
> -     /* Lock all BOs with retry handling */
>       drm_exec_until_all_locked(&exec) {
> -             r = drm_exec_prepare_array(&exec, gobj_read,
> num_read_bo_handles, 1);
> +             r = drm_exec_prepare_array(&exec, gobj_read,
> +                                        num_read_bo_handles, 1);
>               drm_exec_retry_on_contention(&exec);
> -             if (r) {
> -                     amdgpu_userq_fence_cleanup(fence);
> +             if (r)
>                       goto exec_fini;
> -             }
>
> -             r = drm_exec_prepare_array(&exec, gobj_write,
> num_write_bo_handles, 1);
> +             r = drm_exec_prepare_array(&exec, gobj_write,
> +                                        num_write_bo_handles, 1);
>               drm_exec_retry_on_contention(&exec);
> -             if (r) {
> -                     amdgpu_userq_fence_cleanup(fence);
> +             if (r)
>                       goto exec_fini;
> -             }
>       }
>
> -     for (i = 0; i < num_read_bo_handles; i++) {
> -             if (!gobj_read || !gobj_read[i]->resv)
> -                     continue;
> -
> -             dma_resv_add_fence(gobj_read[i]->resv, fence,
> +     /* And publish the new fence in the BOs and syncobj */
> +     for (i = 0; i < num_read_bo_handles; i++)
> +             dma_resv_add_fence(gobj_read[i]->resv, &fence->base,
>                                  DMA_RESV_USAGE_READ);
> -     }
>
> -     for (i = 0; i < num_write_bo_handles; i++) {
> -             if (!gobj_write || !gobj_write[i]->resv)
> -                     continue;
> -
> -             dma_resv_add_fence(gobj_write[i]->resv, fence,
> +     for (i = 0; i < num_write_bo_handles; i++)
> +             dma_resv_add_fence(gobj_write[i]->resv, &fence->base,
>                                  DMA_RESV_USAGE_WRITE);
> -     }
>
> -     /* Add the created fence to syncobj/BO's */
>       for (i = 0; i < num_syncobj_handles; i++)
> -             drm_syncobj_replace_fence(syncobj[i], fence);
> +             drm_syncobj_replace_fence(syncobj[i], &fence->base);
>
> +exec_fini:
>       /* drop the reference acquired in fence creation function */
> -     dma_fence_put(fence);
> +     dma_fence_put(&fence->base);
>
> -exec_fini:
>       drm_exec_fini(&exec);
> +put_queue:
> +     amdgpu_userq_put(queue);
>  put_gobj_write:
>       for (i = 0; i < num_write_bo_handles; i++)
>               drm_gem_object_put(gobj_write[i]);
> @@ -578,15 +556,11 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev,
> void *data,
>       kvfree(gobj_read);
>  free_syncobj:
>       while (entry-- > 0)
> -             if (syncobj[entry])
> -                     drm_syncobj_put(syncobj[entry]);
> +             drm_syncobj_put(syncobj[entry]);
>       kfree(syncobj);
>  free_syncobj_handles:
>       kfree(syncobj_handles);
>
> -     if (queue)
> -             amdgpu_userq_put(queue);
> -
>       return r;
>  }
>
> @@ -874,8 +848,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file
> *filp,
>                * Otherwise, we would gather those references until we don't
>                * have any more space left and crash.
>                */
> +             mutex_lock(&waitq->fence_drv_lock);
>               r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
>                            xa_limit_32b, GFP_KERNEL);
> +             mutex_unlock(&waitq->fence_drv_lock);
>               if (r) {
>                       amdgpu_userq_fence_driver_put(fence_drv);
>                       goto put_waitq;
> --
> 2.43.0

Re: [PATCH 08/11] drm/amdgpu: rework userq reset work handling

[Khatri, Sunil]

[-- Attachment #1: Type: text/plain, Size: 9526 bytes --]


On 21-04-2026 06:25 pm, Christian König wrote:
> It is illegal to schedule reset work from another reset work!
>
> Fix this by scheduling the userq reset work directly on the work queue
> of the reset domain.
>
> Not fully tested, I leave that to the IGT test cases.
>
> Signed-off-by: Christian König<christian.koenig@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu.h        |  1 -
>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  3 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c  | 84 +++++++++++-----------
>   drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h  | 16 ++++-
>   4 files changed, 60 insertions(+), 44 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index 39894e38fee4..17341e384caf 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -1191,7 +1191,6 @@ struct amdgpu_device {
>   	bool                            apu_prefer_gtt;
>   
>   	bool                            userq_halt_for_enforce_isolation;
> -	struct work_struct              userq_reset_work;
>   	struct amdgpu_uid *uid_info;
>   
>   	struct amdgpu_uma_carveout_info uma_info;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index b11c4b5fa8fc..cf61be17e061 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -3786,7 +3786,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
>   	}
>   
>   	INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
> -	INIT_WORK(&adev->userq_reset_work, amdgpu_userq_reset_work);
>   
>   	amdgpu_coredump_init(adev);
>   
> @@ -5477,7 +5476,7 @@ static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
>   	if (!amdgpu_sriov_vf(adev))
>   		cancel_work(&adev->reset_work);
>   #endif
> -	cancel_work(&adev->userq_reset_work);
> +	amdgpu_userq_mgr_cancel_reset_work(adev);
>   
>   	if (adev->kfd.dev)
>   		cancel_work(&adev->kfd.reset_work);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index 0a4c39d83adc..ad6dac17dd21 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -82,19 +82,11 @@ static bool amdgpu_userq_is_reset_type_supported(struct amdgpu_device *adev,
>   	return false;
>   }
>   
> -static void amdgpu_userq_gpu_reset(struct amdgpu_device *adev)
> -{
> -	if (amdgpu_device_should_recover_gpu(adev)) {
> -		amdgpu_reset_domain_schedule(adev->reset_domain,
> -					     &adev->userq_reset_work);
> -		/* Wait for the reset job to complete */
> -		flush_work(&adev->userq_reset_work);
> -	}
> -}
> -
> -static int
> -amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
> +static void amdgpu_userq_mgr_reset_work(struct work_struct *work)
>   {
> +	struct amdgpu_userq_mgr *uq_mgr =
> +		container_of(work, struct amdgpu_userq_mgr,
> +			     reset_work);
>   	struct amdgpu_device *adev = uq_mgr->adev;
>   	const int queue_types[] = {
>   		AMDGPU_RING_TYPE_COMPUTE,
> @@ -103,12 +95,11 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
>   	};
>   	const int num_queue_types = ARRAY_SIZE(queue_types);
>   	bool gpu_reset = false;
> -	int r = 0;
> -	int i;
> +	int i, r;
>   
>   	if (unlikely(adev->debug_disable_gpu_ring_reset)) {
>   		dev_err(adev->dev, "userq reset disabled by debug mask\n");
> -		return 0;
> +		return;
>   	}
>   
>   	/*
> @@ -116,7 +107,7 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
>   	 * skip all reset detection logic
>   	 */
>   	if (!amdgpu_gpu_recovery)
> -		return 0;
> +		return;
>   
>   	/*
>   	 * Iterate through all queue types to detect and reset problematic queues
> @@ -141,10 +132,19 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
>   		}
>   	}
>   
> -	if (gpu_reset)
> -		amdgpu_userq_gpu_reset(adev);
> +	if (gpu_reset) {
> +		struct amdgpu_reset_context reset_context;
>   
> -	return r;
> +		memset(&reset_context, 0, sizeof(reset_context));
> +
> +		reset_context.method = AMD_RESET_METHOD_NONE;
> +		reset_context.reset_req_dev = adev;
> +		reset_context.src = AMDGPU_RESET_SRC_USERQ;
> +		set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
> +		/*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/
> +
> +		amdgpu_device_gpu_recover(adev, NULL, &reset_context);
> +	}
>   }
>   
>   static void amdgpu_userq_hang_detect_work(struct work_struct *work)
The function and the work handler for are using the same name and it 
causes confusion to understand.
queue_delayed_work(adev->reset_domain->wq, &queue->hang_detect_work,
                            msecs_to_jiffies(timeout_ms)); The queued 
item here call the work item where the function name is same , so its 
better if we can keep a different name

Regards
Sunil Khatri

> @@ -153,7 +153,11 @@ static void amdgpu_userq_hang_detect_work(struct work_struct *work)
>   		container_of(work, struct amdgpu_usermode_queue,
>   			     hang_detect_work.work);
>   
> -	amdgpu_userq_detect_and_reset_queues(queue->userq_mgr);
> +	/*
> +	 * Don't schedule the work here! Scheduling or queue work from one reset
> +	 * handler to another is illegal if you don't take extra precautions!
> +	 */
> +	amdgpu_userq_mgr_reset_work(&queue->userq_mgr->reset_work);
>   }
>   
>   /*
> @@ -182,8 +186,8 @@ void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
>   		break;
>   	}
>   
> -	schedule_delayed_work(&queue->hang_detect_work,
> -		     msecs_to_jiffies(timeout_ms));
> +	queue_delayed_work(adev->reset_domain->wq, &queue->hang_detect_work,
> +			   msecs_to_jiffies(timeout_ms));
>   }
>   
>   void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell)
> @@ -1256,28 +1260,13 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
>   	if (ret) {
>   		drm_file_err(uq_mgr->file,
>   			     "Couldn't unmap all the queues, eviction failed ret=%d\n", ret);
> -		amdgpu_userq_detect_and_reset_queues(uq_mgr);
> +		amdgpu_reset_domain_schedule(uq_mgr->adev->reset_domain,
> +					     &uq_mgr->reset_work);
> +		flush_work(&uq_mgr->reset_work);
Flush work is called here with userq_mutex held? Is it ok to run for 
that long time and not sure about it but the flush_work might try to 
take the userq_mutex again, that was problem initially during reset.
>   	}
>   	return ret;
>   }
>   
> -void amdgpu_userq_reset_work(struct work_struct *work)
> -{
> -	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
> -						  userq_reset_work);
> -	struct amdgpu_reset_context reset_context;
> -
> -	memset(&reset_context, 0, sizeof(reset_context));
> -
> -	reset_context.method = AMD_RESET_METHOD_NONE;
> -	reset_context.reset_req_dev = adev;
> -	reset_context.src = AMDGPU_RESET_SRC_USERQ;
> -	set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
> -	/*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/
> -
> -	amdgpu_device_gpu_recover(adev, NULL, &reset_context);
> -}
> -
>   static void
>   amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr)
>   {
> @@ -1311,9 +1300,24 @@ int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *f
>   	userq_mgr->file = file_priv;
>   
>   	INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userq_restore_worker);
> +	INIT_WORK(&userq_mgr->reset_work, amdgpu_userq_mgr_reset_work);
>   	return 0;
>   }
>   
> +void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev)
> +{
> +	struct xarray *xa = &adev->userq_doorbell_xa;
> +	struct amdgpu_usermode_queue *queue;
> +	unsigned long flags, queue_id;
> +
> +	xa_lock_irqsave(xa, flags);
> +	xa_for_each(xa, queue_id, queue) {
> +		cancel_delayed_work(&queue->hang_detect_work);
> +		cancel_work(&queue->userq_mgr->reset_work);
> +	}
> +	xa_unlock_irqrestore(xa, flags);
> +}
> +
>   void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr)
>   {
>   	cancel_delayed_work_sync(&userq_mgr->resume_work);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> index 85f460e7c31b..49b33e2d6932 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> @@ -84,7 +84,13 @@ struct amdgpu_usermode_queue {
>   	u32			xcp_id;
>   	int			priority;
>   	struct dentry		*debugfs_queue;
> -	struct delayed_work hang_detect_work;
> +
> +	/**
> +	 * @hang_detect_work:
> +	 *
> +	 * Delayed work which runs when userq_fences time out.
> +	 */
> +	struct delayed_work	hang_detect_work;
>   	struct kref		refcount;
>   
>   	struct list_head	userq_va_list;
> @@ -116,6 +122,13 @@ struct amdgpu_userq_mgr {
>   	struct amdgpu_device		*adev;
>   	struct delayed_work		resume_work;
>   	struct drm_file			*file;
> +
> +	/**
> +	 * @reset_work:
> +	 *
> +	 * Reset work which is used when eviction fails.
> +	 */
> +	struct work_struct		reset_work;
>   	atomic_t                        userq_count[AMDGPU_RING_TYPE_MAX];
>   };
>   
> @@ -134,6 +147,7 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp
>   int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,
>   			  struct amdgpu_device *adev);
>   
> +void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev);
>   void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr);
>   void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr);
>   

[-- Attachment #2: Type: text/html, Size: 10220 bytes --]

Re: [PATCH 09/11] drm/amdgpu: revert to old status lock handling v4

[Khatri, Sunil]

Not sure if anything changes in this version if not its already reviewed.

regards
Sunil Khatri

On 21-04-2026 06:25 pm, Christian König wrote:
> It turned out that protecting the status of each bo_va with a
> spinlock was just hiding problems instead of solving them.
>
> Revert the whole approach, add a separate stats_lock and lockdep
> assertions that the correct reservation lock is held all over the place.
>
> This not only allows for better checks if a state transition is properly
> protected by a lock, but also switching back to using list macros to
> iterate over the state of lists protected by the dma_resv lock of the
> root PD.
>
> v2: re-add missing check
> v3: split into two patches
> v4: re-apply by fixing holding the VM lock at the right places.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> Reviewed-by: Alex Deucher <alexander.deucher@amd.com>
> Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c |   8 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 146 ++++++++--------------
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h    |  15 ++-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c |   4 -
>   4 files changed, 68 insertions(+), 105 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index ad6dac17dd21..7fc733ba962e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -1048,12 +1048,12 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
>   	struct amdgpu_bo *bo;
>   	int ret;
>   
> -	spin_lock(&vm->status_lock);
> +	spin_lock(&vm->invalidated_lock);
>   	while (!list_empty(&vm->invalidated)) {
>   		bo_va = list_first_entry(&vm->invalidated,
>   					 struct amdgpu_bo_va,
>   					 base.vm_status);
> -		spin_unlock(&vm->status_lock);
> +		spin_unlock(&vm->invalidated_lock);
>   
>   		bo = bo_va->base.bo;
>   		ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2);
> @@ -1070,9 +1070,9 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
>   		if (ret)
>   			return ret;
>   
> -		spin_lock(&vm->status_lock);
> +		spin_lock(&vm->invalidated_lock);
>   	}
> -	spin_unlock(&vm->status_lock);
> +	spin_unlock(&vm->invalidated_lock);
>   
>   	return 0;
>   }
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> index 63156289ae7f..e2a21a66b28f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -167,12 +167,10 @@ static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
>   
>   	vm_bo->moved = true;
>   	amdgpu_vm_assert_locked(vm);
> -	spin_lock(&vm_bo->vm->status_lock);
>   	if (bo->tbo.type == ttm_bo_type_kernel)
>   		list_move(&vm_bo->vm_status, &vm->evicted);
>   	else
>   		list_move_tail(&vm_bo->vm_status, &vm->evicted);
> -	spin_unlock(&vm_bo->vm->status_lock);
>   }
>   /**
>    * amdgpu_vm_bo_moved - vm_bo is moved
> @@ -185,9 +183,7 @@ static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
>   static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
>   {
>   	amdgpu_vm_assert_locked(vm_bo->vm);
> -	spin_lock(&vm_bo->vm->status_lock);
>   	list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
> -	spin_unlock(&vm_bo->vm->status_lock);
>   }
>   
>   /**
> @@ -201,9 +197,7 @@ static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
>   static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
>   {
>   	amdgpu_vm_assert_locked(vm_bo->vm);
> -	spin_lock(&vm_bo->vm->status_lock);
>   	list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
> -	spin_unlock(&vm_bo->vm->status_lock);
>   	vm_bo->moved = false;
>   }
>   
> @@ -217,9 +211,9 @@ static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
>    */
>   static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
>   {
> -	spin_lock(&vm_bo->vm->status_lock);
> +	spin_lock(&vm_bo->vm->invalidated_lock);
>   	list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
> -	spin_unlock(&vm_bo->vm->status_lock);
> +	spin_unlock(&vm_bo->vm->invalidated_lock);
>   }
>   
>   /**
> @@ -232,10 +226,9 @@ static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
>    */
>   static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)
>   {
> +	amdgpu_vm_assert_locked(vm_bo->vm);
>   	vm_bo->moved = true;
> -	spin_lock(&vm_bo->vm->status_lock);
>   	list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user);
> -	spin_unlock(&vm_bo->vm->status_lock);
>   }
>   
>   /**
> @@ -249,13 +242,10 @@ static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)
>   static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
>   {
>   	amdgpu_vm_assert_locked(vm_bo->vm);
> -	if (vm_bo->bo->parent) {
> -		spin_lock(&vm_bo->vm->status_lock);
> +	if (vm_bo->bo->parent)
>   		list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
> -		spin_unlock(&vm_bo->vm->status_lock);
> -	} else {
> +	else
>   		amdgpu_vm_bo_idle(vm_bo);
> -	}
>   }
>   
>   /**
> @@ -269,9 +259,7 @@ static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
>   static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
>   {
>   	amdgpu_vm_assert_locked(vm_bo->vm);
> -	spin_lock(&vm_bo->vm->status_lock);
>   	list_move(&vm_bo->vm_status, &vm_bo->vm->done);
> -	spin_unlock(&vm_bo->vm->status_lock);
>   }
>   
>   /**
> @@ -285,13 +273,13 @@ static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
>   {
>   	struct amdgpu_vm_bo_base *vm_bo, *tmp;
>   
> -	amdgpu_vm_assert_locked(vm);
> -
> -	spin_lock(&vm->status_lock);
> +	spin_lock(&vm->invalidated_lock);
>   	list_splice_init(&vm->done, &vm->invalidated);
>   	list_for_each_entry(vm_bo, &vm->invalidated, vm_status)
>   		vm_bo->moved = true;
> +	spin_unlock(&vm->invalidated_lock);
>   
> +	amdgpu_vm_assert_locked(vm);
>   	list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) {
>   		struct amdgpu_bo *bo = vm_bo->bo;
>   
> @@ -301,14 +289,13 @@ static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
>   		else if (bo->parent)
>   			list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
>   	}
> -	spin_unlock(&vm->status_lock);
>   }
>   
>   /**
>    * amdgpu_vm_update_shared - helper to update shared memory stat
>    * @base: base structure for tracking BO usage in a VM
>    *
> - * Takes the vm status_lock and updates the shared memory stat. If the basic
> + * Takes the vm stats_lock and updates the shared memory stat. If the basic
>    * stat changed (e.g. buffer was moved) amdgpu_vm_update_stats need to be called
>    * as well.
>    */
> @@ -321,7 +308,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base)
>   	bool shared;
>   
>   	dma_resv_assert_held(bo->tbo.base.resv);
> -	spin_lock(&vm->status_lock);
> +	spin_lock(&vm->stats_lock);
>   	shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
>   	if (base->shared != shared) {
>   		base->shared = shared;
> @@ -333,7 +320,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base)
>   			vm->stats[bo_memtype].drm.private += size;
>   		}
>   	}
> -	spin_unlock(&vm->status_lock);
> +	spin_unlock(&vm->stats_lock);
>   }
>   
>   /**
> @@ -358,11 +345,11 @@ void amdgpu_vm_bo_update_shared(struct amdgpu_bo *bo)
>    *        be bo->tbo.resource
>    * @sign: if we should add (+1) or subtract (-1) from the stat
>    *
> - * Caller need to have the vm status_lock held. Useful for when multiple update
> + * Caller need to have the vm stats_lock held. Useful for when multiple update
>    * need to happen at the same time.
>    */
>   static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
> -			    struct ttm_resource *res, int sign)
> +					  struct ttm_resource *res, int sign)
>   {
>   	struct amdgpu_vm *vm = base->vm;
>   	struct amdgpu_bo *bo = base->bo;
> @@ -386,7 +373,8 @@ static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
>   		 */
>   		if (bo->flags & AMDGPU_GEM_CREATE_DISCARDABLE)
>   			vm->stats[res_memtype].drm.purgeable += size;
> -		if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(res_memtype)))
> +		if (!(bo->preferred_domains &
> +		      amdgpu_mem_type_to_domain(res_memtype)))
>   			vm->stats[bo_memtype].evicted += size;
>   	}
>   }
> @@ -405,9 +393,9 @@ void amdgpu_vm_update_stats(struct amdgpu_vm_bo_base *base,
>   {
>   	struct amdgpu_vm *vm = base->vm;
>   
> -	spin_lock(&vm->status_lock);
> +	spin_lock(&vm->stats_lock);
>   	amdgpu_vm_update_stats_locked(base, res, sign);
> -	spin_unlock(&vm->status_lock);
> +	spin_unlock(&vm->stats_lock);
>   }
>   
>   /**
> @@ -433,10 +421,10 @@ void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
>   	base->next = bo->vm_bo;
>   	bo->vm_bo = base;
>   
> -	spin_lock(&vm->status_lock);
> +	spin_lock(&vm->stats_lock);
>   	base->shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
>   	amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1);
> -	spin_unlock(&vm->status_lock);
> +	spin_unlock(&vm->stats_lock);
>   
>   	if (!amdgpu_vm_is_bo_always_valid(vm, bo))
>   		return;
> @@ -495,25 +483,25 @@ int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
>   	int ret;
>   
>   	/* We can only trust prev->next while holding the lock */
> -	spin_lock(&vm->status_lock);
> +	spin_lock(&vm->invalidated_lock);
>   	while (!list_is_head(prev->next, &vm->done)) {
>   		bo_va = list_entry(prev->next, typeof(*bo_va), base.vm_status);
>   
>   		bo = bo_va->base.bo;
>   		if (bo) {
>   			amdgpu_bo_ref(bo);
> -			spin_unlock(&vm->status_lock);
> +			spin_unlock(&vm->invalidated_lock);
>   
>   			ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 1);
>   			amdgpu_bo_unref(&bo);
>   			if (unlikely(ret))
>   				return ret;
>   
> -			spin_lock(&vm->status_lock);
> +			spin_lock(&vm->invalidated_lock);
>   		}
>   		prev = prev->next;
>   	}
> -	spin_unlock(&vm->status_lock);
> +	spin_unlock(&vm->invalidated_lock);
>   
>   	return 0;
>   }
> @@ -609,7 +597,7 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   		       void *param)
>   {
>   	uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm);
> -	struct amdgpu_vm_bo_base *bo_base;
> +	struct amdgpu_vm_bo_base *bo_base, *tmp;
>   	struct amdgpu_bo *bo;
>   	int r;
>   
> @@ -622,13 +610,7 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   			return r;
>   	}
>   
> -	spin_lock(&vm->status_lock);
> -	while (!list_empty(&vm->evicted)) {
> -		bo_base = list_first_entry(&vm->evicted,
> -					   struct amdgpu_vm_bo_base,
> -					   vm_status);
> -		spin_unlock(&vm->status_lock);
> -
> +	list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {
>   		bo = bo_base->bo;
>   
>   		r = validate(param, bo);
> @@ -641,26 +623,21 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   			vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
>   			amdgpu_vm_bo_relocated(bo_base);
>   		}
> -		spin_lock(&vm->status_lock);
>   	}
> -	while (ticket && !list_empty(&vm->evicted_user)) {
> -		bo_base = list_first_entry(&vm->evicted_user,
> -					   struct amdgpu_vm_bo_base,
> -					   vm_status);
> -		spin_unlock(&vm->status_lock);
>   
> -		bo = bo_base->bo;
> -		dma_resv_assert_held(bo->tbo.base.resv);
> +	if (ticket) {
> +		list_for_each_entry_safe(bo_base, tmp, &vm->evicted_user,
> +					 vm_status) {
> +			bo = bo_base->bo;
> +			dma_resv_assert_held(bo->tbo.base.resv);
>   
> -		r = validate(param, bo);
> -		if (r)
> -			return r;
> -
> -		amdgpu_vm_bo_invalidated(bo_base);
> +			r = validate(param, bo);
> +			if (r)
> +				return r;
>   
> -		spin_lock(&vm->status_lock);
> +			amdgpu_vm_bo_invalidated(bo_base);
> +		}
>   	}
> -	spin_unlock(&vm->status_lock);
>   
>   	amdgpu_vm_eviction_lock(vm);
>   	vm->evicting = false;
> @@ -689,9 +666,7 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
>   	ret = !vm->evicting;
>   	amdgpu_vm_eviction_unlock(vm);
>   
> -	spin_lock(&vm->status_lock);
>   	ret &= list_empty(&vm->evicted);
> -	spin_unlock(&vm->status_lock);
>   
>   	spin_lock(&vm->immediate.lock);
>   	ret &= !vm->immediate.stopped;
> @@ -985,18 +960,13 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
>   			  struct amdgpu_vm *vm, bool immediate)
>   {
>   	struct amdgpu_vm_update_params params;
> -	struct amdgpu_vm_bo_base *entry;
> +	struct amdgpu_vm_bo_base *entry, *tmp;
>   	bool flush_tlb_needed = false;
> -	LIST_HEAD(relocated);
>   	int r, idx;
>   
>   	amdgpu_vm_assert_locked(vm);
>   
> -	spin_lock(&vm->status_lock);
> -	list_splice_init(&vm->relocated, &relocated);
> -	spin_unlock(&vm->status_lock);
> -
> -	if (list_empty(&relocated))
> +	if (list_empty(&vm->relocated))
>   		return 0;
>   
>   	if (!drm_dev_enter(adev_to_drm(adev), &idx))
> @@ -1012,7 +982,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
>   	if (r)
>   		goto error;
>   
> -	list_for_each_entry(entry, &relocated, vm_status) {
> +	list_for_each_entry(entry, &vm->relocated, vm_status) {
>   		/* vm_flush_needed after updating moved PDEs */
>   		flush_tlb_needed |= entry->moved;
>   
> @@ -1028,9 +998,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
>   	if (flush_tlb_needed)
>   		atomic64_inc(&vm->tlb_seq);
>   
> -	while (!list_empty(&relocated)) {
> -		entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base,
> -					 vm_status);
> +	list_for_each_entry_safe(entry, tmp, &vm->relocated, vm_status) {
>   		amdgpu_vm_bo_idle(entry);
>   	}
>   
> @@ -1260,9 +1228,9 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
>   			  struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
>   {
> -	spin_lock(&vm->status_lock);
> +	spin_lock(&vm->stats_lock);
>   	memcpy(stats, vm->stats, sizeof(*stats) * __AMDGPU_PL_NUM);
> -	spin_unlock(&vm->status_lock);
> +	spin_unlock(&vm->stats_lock);
>   }
>   
>   /**
> @@ -1629,29 +1597,24 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
>   			   struct amdgpu_vm *vm,
>   			   struct ww_acquire_ctx *ticket)
>   {
> -	struct amdgpu_bo_va *bo_va;
> +	struct amdgpu_bo_va *bo_va, *tmp;
>   	struct dma_resv *resv;
>   	bool clear, unlock;
>   	int r;
>   
> -	spin_lock(&vm->status_lock);
> -	while (!list_empty(&vm->moved)) {
> -		bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va,
> -					 base.vm_status);
> -		spin_unlock(&vm->status_lock);
> -
> +	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
>   		/* Per VM BOs never need to bo cleared in the page tables */
>   		r = amdgpu_vm_bo_update(adev, bo_va, false);
>   		if (r)
>   			return r;
> -		spin_lock(&vm->status_lock);
>   	}
>   
> +	spin_lock(&vm->invalidated_lock);
>   	while (!list_empty(&vm->invalidated)) {
>   		bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
>   					 base.vm_status);
>   		resv = bo_va->base.bo->tbo.base.resv;
> -		spin_unlock(&vm->status_lock);
> +		spin_unlock(&vm->invalidated_lock);
>   
>   		/* Try to reserve the BO to avoid clearing its ptes */
>   		if (!adev->debug_vm && dma_resv_trylock(resv)) {
> @@ -1683,9 +1646,9 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
>   		     bo_va->base.bo->tbo.resource->mem_type == TTM_PL_SYSTEM))
>   			amdgpu_vm_bo_evicted_user(&bo_va->base);
>   
> -		spin_lock(&vm->status_lock);
> +		spin_lock(&vm->invalidated_lock);
>   	}
> -	spin_unlock(&vm->status_lock);
> +	spin_unlock(&vm->invalidated_lock);
>   
>   	return 0;
>   }
> @@ -2223,9 +2186,9 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev,
>   		}
>   	}
>   
> -	spin_lock(&vm->status_lock);
> +	spin_lock(&vm->invalidated_lock);
>   	list_del(&bo_va->base.vm_status);
> -	spin_unlock(&vm->status_lock);
> +	spin_unlock(&vm->invalidated_lock);
>   
>   	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
>   		list_del(&mapping->list);
> @@ -2333,10 +2296,10 @@ void amdgpu_vm_bo_move(struct amdgpu_bo *bo, struct ttm_resource *new_mem,
>   	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
>   		struct amdgpu_vm *vm = bo_base->vm;
>   
> -		spin_lock(&vm->status_lock);
> +		spin_lock(&vm->stats_lock);
>   		amdgpu_vm_update_stats_locked(bo_base, bo->tbo.resource, -1);
>   		amdgpu_vm_update_stats_locked(bo_base, new_mem, +1);
> -		spin_unlock(&vm->status_lock);
> +		spin_unlock(&vm->stats_lock);
>   	}
>   
>   	amdgpu_vm_bo_invalidate(bo, evicted);
> @@ -2608,11 +2571,12 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   	INIT_LIST_HEAD(&vm->relocated);
>   	INIT_LIST_HEAD(&vm->moved);
>   	INIT_LIST_HEAD(&vm->idle);
> +	spin_lock_init(&vm->invalidated_lock);
>   	INIT_LIST_HEAD(&vm->invalidated);
> -	spin_lock_init(&vm->status_lock);
>   	INIT_LIST_HEAD(&vm->freed);
>   	INIT_LIST_HEAD(&vm->done);
>   	INIT_KFIFO(vm->faults);
> +	spin_lock_init(&vm->stats_lock);
>   
>   	r = amdgpu_vm_init_entities(adev, vm);
>   	if (r)
> @@ -3105,7 +3069,6 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
>   
>   	amdgpu_vm_assert_locked(vm);
>   
> -	spin_lock(&vm->status_lock);
>   	seq_puts(m, "\tIdle BOs:\n");
>   	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
>   		if (!bo_va->base.bo)
> @@ -3143,11 +3106,13 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
>   	id = 0;
>   
>   	seq_puts(m, "\tInvalidated BOs:\n");
> +	spin_lock(&vm->invalidated_lock);
>   	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
>   		if (!bo_va->base.bo)
>   			continue;
>   		total_invalidated += amdgpu_bo_print_info(id++,	bo_va->base.bo, m);
>   	}
> +	spin_unlock(&vm->invalidated_lock);
>   	total_invalidated_objs = id;
>   	id = 0;
>   
> @@ -3157,7 +3122,6 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
>   			continue;
>   		total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
>   	}
> -	spin_unlock(&vm->status_lock);
>   	total_done_objs = id;
>   
>   	seq_printf(m, "\tTotal idle size:        %12lld\tobjs:\t%d\n", total_idle,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> index f33ea7f8509b..b5216bc1292f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> @@ -205,11 +205,11 @@ struct amdgpu_vm_bo_base {
>   	/* protected by bo being reserved */
>   	struct amdgpu_vm_bo_base	*next;
>   
> -	/* protected by vm status_lock */
> +	/* protected by vm reservation and invalidated_lock */
>   	struct list_head		vm_status;
>   
>   	/* if the bo is counted as shared in mem stats
> -	 * protected by vm status_lock */
> +	 * protected by vm BO being reserved */
>   	bool				shared;
>   
>   	/* protected by the BO being reserved */
> @@ -345,10 +345,8 @@ struct amdgpu_vm {
>   	bool			evicting;
>   	unsigned int		saved_flags;
>   
> -	/* Lock to protect vm_bo add/del/move on all lists of vm */
> -	spinlock_t		status_lock;
> -
> -	/* Memory statistics for this vm, protected by status_lock */
> +	/* Memory statistics for this vm, protected by stats_lock */
> +	spinlock_t		stats_lock;
>   	struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM];
>   
>   	/*
> @@ -356,6 +354,8 @@ struct amdgpu_vm {
>   	 * PDs, PTs or per VM BOs. The state transits are:
>   	 *
>   	 * evicted -> relocated (PDs, PTs) or moved (per VM BOs) -> idle
> +	 *
> +	 * Lists are protected by the root PD dma_resv lock.
>   	 */
>   
>   	/* Per-VM and PT BOs who needs a validation */
> @@ -376,7 +376,10 @@ struct amdgpu_vm {
>   	 * state transits are:
>   	 *
>   	 * evicted_user or invalidated -> done
> +	 *
> +	 * Lists are protected by the invalidated_lock.
>   	 */
> +	spinlock_t		invalidated_lock;
>   
>   	/* BOs for user mode queues that need a validation */
>   	struct list_head	evicted_user;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> index 31a437ce9570..7bdd664f0770 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> @@ -544,9 +544,7 @@ static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry)
>   	entry->bo->vm_bo = NULL;
>   	ttm_bo_set_bulk_move(&entry->bo->tbo, NULL);
>   
> -	spin_lock(&entry->vm->status_lock);
>   	list_del(&entry->vm_status);
> -	spin_unlock(&entry->vm->status_lock);
>   	amdgpu_bo_unref(&entry->bo);
>   }
>   
> @@ -590,7 +588,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params,
>   	struct amdgpu_vm_pt_cursor seek;
>   	struct amdgpu_vm_bo_base *entry;
>   
> -	spin_lock(&params->vm->status_lock);
>   	for_each_amdgpu_vm_pt_dfs_safe(params->adev, params->vm, cursor, seek, entry) {
>   		if (entry && entry->bo)
>   			list_move(&entry->vm_status, &params->tlb_flush_waitlist);
> @@ -598,7 +595,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params,
>   
>   	/* enter start node now */
>   	list_move(&cursor->entry->vm_status, &params->tlb_flush_waitlist);
> -	spin_unlock(&params->vm->status_lock);
>   }
>   
>   /**

Re: [PATCH 10/11] drm/amdgpu: restructure VM state machine v2

[Khatri, Sunil]

I think this was already reviewed earlier too with just some additional 
comment.
Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>

Regards
Sunil Khatri

On 21-04-2026 06:25 pm, Christian König wrote:
> Instead of coming up with more sophisticated names for states a VM BO
> can be in, group them by the type of BO first and then by the state.
>
> So we end with BO type kernel, always_valid and individual and then states
> evicted, moved and idle.
>
> Not much functional change, except that evicted_user is moved back
> together with the other BOs again which makes the handling in
> amdgpu_vm_validate() a bit more complex.
>
> Also fixes a problem with user queues and amdgpu_vm_ready(). We didn't
> considered the VM ready when user BOs were not ideally placed, harmless
> performance impact for kernel queues but a complete show stopper for
> userqueues.
>
> v2: fix a few typos in comments, rename the BO types to make them more
>      descriptive, fix a couple of bugs found during testing
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c |  22 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 468 ++++++++++------------
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h    |  64 ++-
>   3 files changed, 252 insertions(+), 302 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index 7fc733ba962e..9d01449e1487 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -1048,12 +1048,12 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
>   	struct amdgpu_bo *bo;
>   	int ret;
>   
> -	spin_lock(&vm->invalidated_lock);
> -	while (!list_empty(&vm->invalidated)) {
> -		bo_va = list_first_entry(&vm->invalidated,
> +	spin_lock(&vm->individual_lock);
> +	while (!list_empty(&vm->always_valid.evicted)) {
> +		bo_va = list_first_entry(&vm->always_valid.evicted,
>   					 struct amdgpu_bo_va,
>   					 base.vm_status);
> -		spin_unlock(&vm->invalidated_lock);
> +		spin_unlock(&vm->individual_lock);
>   
>   		bo = bo_va->base.bo;
>   		ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2);
> @@ -1065,14 +1065,14 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
>   		if (ret)
>   			return ret;
>   
> -		/* This moves the bo_va to the done list */
> +		/* This moves the bo_va to the idle list */
>   		ret = amdgpu_vm_bo_update(adev, bo_va, false);
>   		if (ret)
>   			return ret;
>   
> -		spin_lock(&vm->invalidated_lock);
> +		spin_lock(&vm->individual_lock);
>   	}
> -	spin_unlock(&vm->invalidated_lock);
> +	spin_unlock(&vm->individual_lock);
>   
>   	return 0;
>   }
> @@ -1104,7 +1104,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
>   		if (unlikely(ret))
>   			goto unlock_all;
>   
> -		ret = amdgpu_vm_lock_done_list(vm, &exec, 1);
> +		ret = amdgpu_vm_lock_individual(vm, &exec, 1);
>   		drm_exec_retry_on_contention(&exec);
>   		if (unlikely(ret))
>   			goto unlock_all;
> @@ -1147,7 +1147,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
>   
>   	key = 0;
>   	/* Validate User Ptr BOs */
> -	list_for_each_entry(bo_va, &vm->done, base.vm_status) {
> +	list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status) {
>   		bo = bo_va->base.bo;
>   		if (!bo)
>   			continue;
> @@ -1197,10 +1197,10 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
>   
>   	/*
>   	 * We need to wait for all VM updates to finish before restarting the
> -	 * queues. Using the done list like that is now ok since everything is
> +	 * queues. Using the idle list like that is now ok since everything is
>   	 * locked in place.
>   	 */
> -	list_for_each_entry(bo_va, &vm->done, base.vm_status)
> +	list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status)
>   		dma_fence_wait(bo_va->last_pt_update, false);
>   	dma_fence_wait(vm->last_update, false);
>   
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> index e2a21a66b28f..118761e2c409 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -138,6 +138,47 @@ static void amdgpu_vm_assert_locked(struct amdgpu_vm *vm)
>   	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
>   }
>   
> +/* Initialize the amdgpu_vm_bo_status object */
> +static void amdgpu_vm_bo_status_init(struct amdgpu_vm_bo_status *lists)
> +{
> +	INIT_LIST_HEAD(&lists->evicted);
> +	INIT_LIST_HEAD(&lists->moved);
> +	INIT_LIST_HEAD(&lists->idle);
> +}
> +
> +/*
> + * Make sure we have the lock to modify the vm_bo status and return the object
> + * with the status lists.
> + */
> +static struct amdgpu_vm_bo_status *
> +amdgpu_vm_bo_lock_lists(struct amdgpu_vm_bo_base *vm_bo)
> +{
> +	struct amdgpu_vm *vm = vm_bo->vm;
> +	struct amdgpu_bo *bo = vm_bo->bo;
> +
> +	if (amdgpu_vm_is_bo_always_valid(vm, bo)) {
> +		/* No extra locking needed, protected by the root PD resv lock */
> +		amdgpu_vm_assert_locked(vm);
> +
> +		if (bo->tbo.type == ttm_bo_type_kernel)
> +			return &vm->kernel;
> +
> +		return &vm->always_valid;
> +	}
> +
> +	spin_lock(&vm_bo->vm->individual_lock);
> +	return &vm->individual;
> +}
> +
> +/* Eventually unlock the status list lock again */
> +static void amdgpu_vm_bo_unlock_lists(struct amdgpu_vm_bo_base *vm_bo)
> +{
> +	if (amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
> +		amdgpu_vm_assert_locked(vm_bo->vm);
> +	else
> +		spin_unlock(&vm_bo->vm->individual_lock);
> +}
> +
>   /**
>    * amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid
>    *
> @@ -157,33 +198,44 @@ bool amdgpu_vm_is_bo_always_valid(struct amdgpu_vm *vm, struct amdgpu_bo *bo)
>    *
>    * @vm_bo: vm_bo which is evicted
>    *
> - * State for PDs/PTs and per VM BOs which are not at the location they should
> - * be.
> + * State for vm_bo objects meaning the underlying BO was evicted and need to
> + * move in place again.
>    */
>   static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
>   {
> -	struct amdgpu_vm *vm = vm_bo->vm;
> -	struct amdgpu_bo *bo = vm_bo->bo;
> +	struct amdgpu_vm_bo_status *lists;
>   
> +	lists = amdgpu_vm_bo_lock_lists(vm_bo);
>   	vm_bo->moved = true;
> -	amdgpu_vm_assert_locked(vm);
> -	if (bo->tbo.type == ttm_bo_type_kernel)
> -		list_move(&vm_bo->vm_status, &vm->evicted);
> -	else
> -		list_move_tail(&vm_bo->vm_status, &vm->evicted);
> +	list_move(&vm_bo->vm_status, &lists->evicted);
> +	amdgpu_vm_bo_unlock_lists(vm_bo);
>   }
>   /**
>    * amdgpu_vm_bo_moved - vm_bo is moved
>    *
>    * @vm_bo: vm_bo which is moved
>    *
> - * State for per VM BOs which are moved, but that change is not yet reflected
> - * in the page tables.
> + * State for vm_bo objects meaning the underlying BO was moved but the new
> + * location not yet reflected in the page tables.
>    */
>   static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
>   {
> -	amdgpu_vm_assert_locked(vm_bo->vm);
> -	list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
> +	struct amdgpu_vm_bo_status *lists;
> +	struct amdgpu_bo *bo = vm_bo->bo;
> +
> +	/*
> +	 * The root PD doesn't have a parent PDE and goes directly into the
> +	 * idle state.
> +	 */
> +	lists = amdgpu_vm_bo_lock_lists(vm_bo);
> +	if (bo && bo->tbo.type == ttm_bo_type_kernel && !bo->parent) {
> +		vm_bo->moved = false;
> +		list_move(&vm_bo->vm_status, &lists->idle);
> +	} else {
> +		vm_bo->moved = true;
> +		list_move(&vm_bo->vm_status, &lists->moved);
> +	}
> +	amdgpu_vm_bo_unlock_lists(vm_bo);
>   }
>   
>   /**
> @@ -191,104 +243,36 @@ static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
>    *
>    * @vm_bo: vm_bo which is now idle
>    *
> - * State for PDs/PTs and per VM BOs which have gone through the state machine
> - * and are now idle.
> + * State for vm_bo objects meaning we are done with the state machine and no
> + * further action is necessary.
>    */
>   static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
>   {
> -	amdgpu_vm_assert_locked(vm_bo->vm);
> -	list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
> -	vm_bo->moved = false;
> -}
> -
> -/**
> - * amdgpu_vm_bo_invalidated - vm_bo is invalidated
> - *
> - * @vm_bo: vm_bo which is now invalidated
> - *
> - * State for normal BOs which are invalidated and that change not yet reflected
> - * in the PTs.
> - */
> -static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
> -{
> -	spin_lock(&vm_bo->vm->invalidated_lock);
> -	list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
> -	spin_unlock(&vm_bo->vm->invalidated_lock);
> -}
> -
> -/**
> - * amdgpu_vm_bo_evicted_user - vm_bo is evicted
> - *
> - * @vm_bo: vm_bo which is evicted
> - *
> - * State for BOs used by user mode queues which are not at the location they
> - * should be.
> - */
> -static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)
> -{
> -	amdgpu_vm_assert_locked(vm_bo->vm);
> -	vm_bo->moved = true;
> -	list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user);
> -}
> +	struct amdgpu_vm_bo_status *lists;
>   
> -/**
> - * amdgpu_vm_bo_relocated - vm_bo is reloacted
> - *
> - * @vm_bo: vm_bo which is relocated
> - *
> - * State for PDs/PTs which needs to update their parent PD.
> - * For the root PD, just move to idle state.
> - */
> -static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
> -{
> -	amdgpu_vm_assert_locked(vm_bo->vm);
> -	if (vm_bo->bo->parent)
> -		list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
> -	else
> -		amdgpu_vm_bo_idle(vm_bo);
> -}
> -
> -/**
> - * amdgpu_vm_bo_done - vm_bo is done
> - *
> - * @vm_bo: vm_bo which is now done
> - *
> - * State for normal BOs which are invalidated and that change has been updated
> - * in the PTs.
> - */
> -static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
> -{
> -	amdgpu_vm_assert_locked(vm_bo->vm);
> -	list_move(&vm_bo->vm_status, &vm_bo->vm->done);
> +	lists = amdgpu_vm_bo_lock_lists(vm_bo);
> +	if (!amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
> +		vm_bo->moved = false;
> +	list_move(&vm_bo->vm_status, &lists->idle);
> +	amdgpu_vm_bo_unlock_lists(vm_bo);
>   }
>   
>   /**
>    * amdgpu_vm_bo_reset_state_machine - reset the vm_bo state machine
>    * @vm: the VM which state machine to reset
>    *
> - * Move all vm_bo object in the VM into a state where they will be updated
> - * again during validation.
> + * Move all vm_bo object in the VM into a state where their location will be
> + * updated in the page tables again.
>    */
>   static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
>   {
> -	struct amdgpu_vm_bo_base *vm_bo, *tmp;
> -
> -	spin_lock(&vm->invalidated_lock);
> -	list_splice_init(&vm->done, &vm->invalidated);
> -	list_for_each_entry(vm_bo, &vm->invalidated, vm_status)
> -		vm_bo->moved = true;
> -	spin_unlock(&vm->invalidated_lock);
> -
>   	amdgpu_vm_assert_locked(vm);
> -	list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) {
> -		struct amdgpu_bo *bo = vm_bo->bo;
> +	list_splice_init(&vm->kernel.idle, &vm->kernel.moved);
> +	list_splice_init(&vm->always_valid.idle, &vm->always_valid.moved);
>   
> -		vm_bo->moved = true;
> -		if (!bo || bo->tbo.type != ttm_bo_type_kernel)
> -			list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
> -		else if (bo->parent)
> -			list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
> -	}
> +	spin_lock(&vm->individual_lock);
> +	list_splice_init(&vm->individual.idle, &vm->individual.moved);
> +	spin_unlock(&vm->individual_lock);
>   }
>   
>   /**
> @@ -416,8 +400,10 @@ void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
>   	base->next = NULL;
>   	INIT_LIST_HEAD(&base->vm_status);
>   
> +	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
>   	if (!bo)
>   		return;
> +
>   	base->next = bo->vm_bo;
>   	bo->vm_bo = base;
>   
> @@ -426,27 +412,22 @@ void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
>   	amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1);
>   	spin_unlock(&vm->stats_lock);
>   
> -	if (!amdgpu_vm_is_bo_always_valid(vm, bo))
> +	if (!amdgpu_vm_is_bo_always_valid(vm, bo)) {
> +		amdgpu_vm_bo_idle(base);
>   		return;
> -
> -	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
> +	}
>   
>   	ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move);
> -	if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
> -		amdgpu_vm_bo_relocated(base);
> -	else
> -		amdgpu_vm_bo_idle(base);
>   
> +	/*
> +	 * When a per VM isn't in the desired domain put it into the evicted
> +	 * state to make sure that it gets validated on the next best occasion.
> +	 */
>   	if (bo->preferred_domains &
>   	    amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type))
> -		return;
> -
> -	/*
> -	 * we checked all the prerequisites, but it looks like this per vm bo
> -	 * is currently evicted. add the bo to the evicted list to make sure it
> -	 * is validated on next vm use to avoid fault.
> -	 * */
> -	amdgpu_vm_bo_evicted(base);
> +		amdgpu_vm_bo_moved(base);
> +	else
> +		amdgpu_vm_bo_evicted(base);
>   }
>   
>   /**
> @@ -467,41 +448,41 @@ int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec,
>   }
>   
>   /**
> - * amdgpu_vm_lock_done_list - lock all BOs on the done list
> + * amdgpu_vm_lock_individual - lock all BOs on the individual idle list
>    * @vm: vm providing the BOs
>    * @exec: drm execution context
>    * @num_fences: number of extra fences to reserve
>    *
> - * Lock the BOs on the done list in the DRM execution context.
> + * Lock the BOs on the individual idle list in the DRM execution context.
>    */
> -int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
> -			     unsigned int num_fences)
> +int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
> +			      unsigned int num_fences)
>   {
> -	struct list_head *prev = &vm->done;
> +	struct list_head *prev = &vm->individual.idle;
>   	struct amdgpu_bo_va *bo_va;
>   	struct amdgpu_bo *bo;
>   	int ret;
>   
>   	/* We can only trust prev->next while holding the lock */
> -	spin_lock(&vm->invalidated_lock);
> -	while (!list_is_head(prev->next, &vm->done)) {
> +	spin_lock(&vm->individual_lock);
> +	while (!list_is_head(prev->next, &vm->individual.idle)) {
>   		bo_va = list_entry(prev->next, typeof(*bo_va), base.vm_status);
>   
>   		bo = bo_va->base.bo;
>   		if (bo) {
>   			amdgpu_bo_ref(bo);
> -			spin_unlock(&vm->invalidated_lock);
> +			spin_unlock(&vm->individual_lock);
>   
>   			ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 1);
>   			amdgpu_bo_unref(&bo);
>   			if (unlikely(ret))
>   				return ret;
>   
> -			spin_lock(&vm->invalidated_lock);
> +			spin_lock(&vm->individual_lock);
>   		}
>   		prev = prev->next;
>   	}
> -	spin_unlock(&vm->invalidated_lock);
> +	spin_unlock(&vm->individual_lock);
>   
>   	return 0;
>   }
> @@ -598,9 +579,9 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   {
>   	uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm);
>   	struct amdgpu_vm_bo_base *bo_base, *tmp;
> -	struct amdgpu_bo *bo;
>   	int r;
>   
> +	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
>   	if (vm->generation != new_vm_generation) {
>   		vm->generation = new_vm_generation;
>   		amdgpu_vm_bo_reset_state_machine(vm);
> @@ -610,38 +591,59 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   			return r;
>   	}
>   
> -	list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {
> -		bo = bo_base->bo;
> -
> -		r = validate(param, bo);
> +	list_for_each_entry_safe(bo_base, tmp, &vm->kernel.evicted, vm_status) {
> +		r = validate(param, bo_base->bo);
>   		if (r)
>   			return r;
>   
> -		if (bo->tbo.type != ttm_bo_type_kernel) {
> -			amdgpu_vm_bo_moved(bo_base);
> -		} else {
> -			vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
> -			amdgpu_vm_bo_relocated(bo_base);
> -		}
> +		vm->update_funcs->map_table(to_amdgpu_bo_vm(bo_base->bo));
> +		amdgpu_vm_bo_moved(bo_base);
>   	}
>   
> -	if (ticket) {
> -		list_for_each_entry_safe(bo_base, tmp, &vm->evicted_user,
> -					 vm_status) {
> -			bo = bo_base->bo;
> -			dma_resv_assert_held(bo->tbo.base.resv);
> +	/*
> +	 * As soon as all page tables are in place we can start updating them
> +	 * again.
> +	 */
> +	amdgpu_vm_eviction_lock(vm);
> +	vm->evicting = false;
> +	amdgpu_vm_eviction_unlock(vm);
>   
> -			r = validate(param, bo);
> -			if (r)
> -				return r;
> +	list_for_each_entry_safe(bo_base, tmp, &vm->always_valid.evicted,
> +				 vm_status) {
> +		r = validate(param, bo_base->bo);
> +		if (r)
> +			return r;
>   
> -			amdgpu_vm_bo_invalidated(bo_base);
> -		}
> +		amdgpu_vm_bo_moved(bo_base);
>   	}
>   
> -	amdgpu_vm_eviction_lock(vm);
> -	vm->evicting = false;
> -	amdgpu_vm_eviction_unlock(vm);
> +	if (!ticket)
> +		return 0;
> +
> +	spin_lock(&vm->individual_lock);
> +restart:
> +	list_for_each_entry(bo_base, &vm->individual.evicted, vm_status) {
> +		struct amdgpu_bo *bo = bo_base->bo;
> +
> +		if (dma_resv_locking_ctx(bo->tbo.base.resv) != ticket)
> +			continue;
> +
> +		spin_unlock(&vm->individual_lock);
> +
> +		r = validate(param, bo);
> +		if (r)
> +			return r;
> +
> +		amdgpu_vm_bo_moved(bo_base);
> +
> +		/* It's a bit inefficient to always jump back to the start, but
> +		 * we would need to re-structure the KFD for properly fixing
> +		 * that.
> +		 */
> +		spin_lock(&vm->individual_lock);
> +		goto restart;
> +	}
> +	spin_unlock(&vm->individual_lock);
>   
>   	return 0;
>   }
> @@ -666,7 +668,7 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
>   	ret = !vm->evicting;
>   	amdgpu_vm_eviction_unlock(vm);
>   
> -	ret &= list_empty(&vm->evicted);
> +	ret &= list_empty(&vm->kernel.evicted);
>   
>   	spin_lock(&vm->immediate.lock);
>   	ret &= !vm->immediate.stopped;
> @@ -966,7 +968,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
>   
>   	amdgpu_vm_assert_locked(vm);
>   
> -	if (list_empty(&vm->relocated))
> +	if (list_empty(&vm->kernel.moved))
>   		return 0;
>   
>   	if (!drm_dev_enter(adev_to_drm(adev), &idx))
> @@ -982,7 +984,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
>   	if (r)
>   		goto error;
>   
> -	list_for_each_entry(entry, &vm->relocated, vm_status) {
> +	list_for_each_entry(entry, &vm->kernel.moved, vm_status) {
>   		/* vm_flush_needed after updating moved PDEs */
>   		flush_tlb_needed |= entry->moved;
>   
> @@ -998,9 +1000,8 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
>   	if (flush_tlb_needed)
>   		atomic64_inc(&vm->tlb_seq);
>   
> -	list_for_each_entry_safe(entry, tmp, &vm->relocated, vm_status) {
> +	list_for_each_entry_safe(entry, tmp, &vm->kernel.moved, vm_status)
>   		amdgpu_vm_bo_idle(entry);
> -	}
>   
>   error:
>   	drm_dev_exit(idx);
> @@ -1374,7 +1375,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
>   		else
>   			amdgpu_vm_bo_idle(&bo_va->base);
>   	} else {
> -		amdgpu_vm_bo_done(&bo_va->base);
> +		amdgpu_vm_bo_idle(&bo_va->base);
>   	}
>   
>   	list_splice_init(&bo_va->invalids, &bo_va->valids);
> @@ -1602,19 +1603,20 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
>   	bool clear, unlock;
>   	int r;
>   
> -	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
> +	list_for_each_entry_safe(bo_va, tmp, &vm->always_valid.moved,
> +				 base.vm_status) {
>   		/* Per VM BOs never need to bo cleared in the page tables */
>   		r = amdgpu_vm_bo_update(adev, bo_va, false);
>   		if (r)
>   			return r;
>   	}
>   
> -	spin_lock(&vm->invalidated_lock);
> -	while (!list_empty(&vm->invalidated)) {
> -		bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
> -					 base.vm_status);
> +	spin_lock(&vm->individual_lock);
> +	while (!list_empty(&vm->individual.moved)) {
> +		bo_va = list_first_entry(&vm->individual.moved,
> +					 typeof(*bo_va), base.vm_status);
>   		resv = bo_va->base.bo->tbo.base.resv;
> -		spin_unlock(&vm->invalidated_lock);
> +		spin_unlock(&vm->individual_lock);
>   
>   		/* Try to reserve the BO to avoid clearing its ptes */
>   		if (!adev->debug_vm && dma_resv_trylock(resv)) {
> @@ -1644,11 +1646,11 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
>   		    drm_gem_is_imported(&bo_va->base.bo->tbo.base) &&
>   		    (!bo_va->base.bo->tbo.resource ||
>   		     bo_va->base.bo->tbo.resource->mem_type == TTM_PL_SYSTEM))
> -			amdgpu_vm_bo_evicted_user(&bo_va->base);
> +			amdgpu_vm_bo_evicted(&bo_va->base);
>   
> -		spin_lock(&vm->invalidated_lock);
> +		spin_lock(&vm->individual_lock);
>   	}
> -	spin_unlock(&vm->invalidated_lock);
> +	spin_unlock(&vm->individual_lock);
>   
>   	return 0;
>   }
> @@ -2186,9 +2188,9 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev,
>   		}
>   	}
>   
> -	spin_lock(&vm->invalidated_lock);
> +	spin_lock(&vm->individual_lock);
>   	list_del(&bo_va->base.vm_status);
> -	spin_unlock(&vm->invalidated_lock);
> +	spin_unlock(&vm->individual_lock);
>   
>   	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
>   		list_del(&mapping->list);
> @@ -2268,14 +2270,7 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_bo *bo, bool evicted)
>   
>   		if (bo_base->moved)
>   			continue;
> -		bo_base->moved = true;
> -
> -		if (bo->tbo.type == ttm_bo_type_kernel)
> -			amdgpu_vm_bo_relocated(bo_base);
> -		else if (amdgpu_vm_is_bo_always_valid(vm, bo))
> -			amdgpu_vm_bo_moved(bo_base);
> -		else
> -			amdgpu_vm_bo_invalidated(bo_base);
> +		amdgpu_vm_bo_moved(bo_base);
>   	}
>   }
>   
> @@ -2566,15 +2561,12 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>   	vm->va = RB_ROOT_CACHED;
>   	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
>   		vm->reserved_vmid[i] = NULL;
> -	INIT_LIST_HEAD(&vm->evicted);
> -	INIT_LIST_HEAD(&vm->evicted_user);
> -	INIT_LIST_HEAD(&vm->relocated);
> -	INIT_LIST_HEAD(&vm->moved);
> -	INIT_LIST_HEAD(&vm->idle);
> -	spin_lock_init(&vm->invalidated_lock);
> -	INIT_LIST_HEAD(&vm->invalidated);
> +
> +	amdgpu_vm_bo_status_init(&vm->kernel);
> +	amdgpu_vm_bo_status_init(&vm->always_valid);
> +	spin_lock_init(&vm->individual_lock);
> +	amdgpu_vm_bo_status_init(&vm->individual);
>   	INIT_LIST_HEAD(&vm->freed);
> -	INIT_LIST_HEAD(&vm->done);
>   	INIT_KFIFO(vm->faults);
>   	spin_lock_init(&vm->stats_lock);
>   
> @@ -3042,100 +3034,62 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
>   }
>   
>   #if defined(CONFIG_DEBUG_FS)
> -/**
> - * amdgpu_debugfs_vm_bo_info  - print BO info for the VM
> - *
> - * @vm: Requested VM for printing BO info
> - * @m: debugfs file
> - *
> - * Print BO information in debugfs file for the VM
> - */
> -void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
> -{
> -	struct amdgpu_bo_va *bo_va, *tmp;
> -	u64 total_idle = 0;
> -	u64 total_evicted = 0;
> -	u64 total_relocated = 0;
> -	u64 total_moved = 0;
> -	u64 total_invalidated = 0;
> -	u64 total_done = 0;
> -	unsigned int total_idle_objs = 0;
> -	unsigned int total_evicted_objs = 0;
> -	unsigned int total_relocated_objs = 0;
> -	unsigned int total_moved_objs = 0;
> -	unsigned int total_invalidated_objs = 0;
> -	unsigned int total_done_objs = 0;
> -	unsigned int id = 0;
>   
> -	amdgpu_vm_assert_locked(vm);
> +/* print the debug info for a specific set of status lists */
> +static void amdgpu_debugfs_vm_bo_status_info(struct seq_file *m,
> +					     struct amdgpu_vm_bo_status *lists)
> +{
> +	struct amdgpu_vm_bo_base *base;
> +	unsigned int id;
>   
> -	seq_puts(m, "\tIdle BOs:\n");
> -	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
> -		if (!bo_va->base.bo)
> -			continue;
> -		total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
> -	}
> -	total_idle_objs = id;
>   	id = 0;
> -
>   	seq_puts(m, "\tEvicted BOs:\n");
> -	list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
> -		if (!bo_va->base.bo)
> +	list_for_each_entry(base, &lists->evicted, vm_status) {
> +		if (!base->bo)
>   			continue;
> -		total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
> -	}
> -	total_evicted_objs = id;
> -	id = 0;
>   
> -	seq_puts(m, "\tRelocated BOs:\n");
> -	list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
> -		if (!bo_va->base.bo)
> -			continue;
> -		total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
> +		amdgpu_bo_print_info(id++, base->bo, m);
>   	}
> -	total_relocated_objs = id;
> -	id = 0;
>   
> +	id = 0;
>   	seq_puts(m, "\tMoved BOs:\n");
> -	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
> -		if (!bo_va->base.bo)
> +	list_for_each_entry(base, &lists->moved, vm_status) {
> +		if (!base->bo)
>   			continue;
> -		total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
> +
> +		amdgpu_bo_print_info(id++, base->bo, m);
>   	}
> -	total_moved_objs = id;
> -	id = 0;
>   
> -	seq_puts(m, "\tInvalidated BOs:\n");
> -	spin_lock(&vm->invalidated_lock);
> -	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
> -		if (!bo_va->base.bo)
> +	id = 0;
> +	seq_puts(m, "\tIdle BOs:\n");
> +	list_for_each_entry(base, &lists->moved, vm_status) {
> +		if (!base->bo)
>   			continue;
> -		total_invalidated += amdgpu_bo_print_info(id++,	bo_va->base.bo, m);
> +
> +		amdgpu_bo_print_info(id++, base->bo, m);
>   	}
> -	spin_unlock(&vm->invalidated_lock);
> -	total_invalidated_objs = id;
> -	id = 0;
> +}
>   
> -	seq_puts(m, "\tDone BOs:\n");
> -	list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
> -		if (!bo_va->base.bo)
> -			continue;
> -		total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
> -	}
> -	total_done_objs = id;
> -
> -	seq_printf(m, "\tTotal idle size:        %12lld\tobjs:\t%d\n", total_idle,
> -		   total_idle_objs);
> -	seq_printf(m, "\tTotal evicted size:     %12lld\tobjs:\t%d\n", total_evicted,
> -		   total_evicted_objs);
> -	seq_printf(m, "\tTotal relocated size:   %12lld\tobjs:\t%d\n", total_relocated,
> -		   total_relocated_objs);
> -	seq_printf(m, "\tTotal moved size:       %12lld\tobjs:\t%d\n", total_moved,
> -		   total_moved_objs);
> -	seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated,
> -		   total_invalidated_objs);
> -	seq_printf(m, "\tTotal done size:        %12lld\tobjs:\t%d\n", total_done,
> -		   total_done_objs);
> +/**
> + * amdgpu_debugfs_vm_bo_info  - print BO info for the VM
> + *
> + * @vm: Requested VM for printing BO info
> + * @m: debugfs file
> + *
> + * Print BO information in debugfs file for the VM
> + */
> +void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
> +{
> +	amdgpu_vm_assert_locked(vm);
> +
> +	seq_puts(m, "\tKernel PT/PDs:\n");
> +	amdgpu_debugfs_vm_bo_status_info(m, &vm->kernel);
> +
> +	seq_puts(m, "\tPer VM BOs:\n");
> +	amdgpu_debugfs_vm_bo_status_info(m, &vm->always_valid);
> +
> +	seq_puts(m, "\tIndividual BOs:\n");
> +	amdgpu_debugfs_vm_bo_status_info(m, &vm->individual);
>   }
>   #endif
>   
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> index b5216bc1292f..cc96a3e6252f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> @@ -216,6 +216,23 @@ struct amdgpu_vm_bo_base {
>   	bool				moved;
>   };
>   
> +/*
> + * The following status lists contain amdgpu_vm_bo_base objects for
> + * either PD/PTs, per VM BOs or BOs with individual resv object.
> + *
> + * The state transits are: evicted -> moved -> idle
> + */
> +struct amdgpu_vm_bo_status {
> +	/* BOs evicted which need to move into place again */
> +	struct list_head		evicted;
> +
> +	/* BOs which moved but new location hasn't been updated in the PDs/PTs */
> +	struct list_head		moved;
> +
> +	/* BOs done with the state machine and need no further action */
> +	struct list_head		idle;
> +};
> +
>   /* provided by hw blocks that can write ptes, e.g., sdma */
>   struct amdgpu_vm_pte_funcs {
>   	/* number of dw to reserve per operation */
> @@ -349,46 +366,25 @@ struct amdgpu_vm {
>   	spinlock_t		stats_lock;
>   	struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM];
>   
> +	/* BO's belonging to PD/PT which are internal to the kernel. */
> +	struct amdgpu_vm_bo_status	kernel;
> +
>   	/*
> -	 * The following lists contain amdgpu_vm_bo_base objects for either
> -	 * PDs, PTs or per VM BOs. The state transits are:
> -	 *
> -	 * evicted -> relocated (PDs, PTs) or moved (per VM BOs) -> idle
> -	 *
> -	 * Lists are protected by the root PD dma_resv lock.
> +	 * BOs allocated by userspace where the dma_resv is shared with the
> +	 * root PD
>   	 */
> -
> -	/* Per-VM and PT BOs who needs a validation */
> -	struct list_head	evicted;
> -
> -	/* PT BOs which relocated and their parent need an update */
> -	struct list_head	relocated;
> -
> -	/* per VM BOs moved, but not yet updated in the PT */
> -	struct list_head	moved;
> -
> -	/* All BOs of this VM not currently in the state machine */
> -	struct list_head	idle;
> +	struct amdgpu_vm_bo_status	always_valid;
>   
>   	/*
>   	 * The following lists contain amdgpu_vm_bo_base objects for BOs which
> -	 * have their own dma_resv object and not depend on the root PD. Their
> -	 * state transits are:
> -	 *
> -	 * evicted_user or invalidated -> done
> +	 * have their own dma_resv object and not depend on the root PD.
>   	 *
> -	 * Lists are protected by the invalidated_lock.
> +	 * Lists are protected by the individual_lock.
>   	 */
> -	spinlock_t		invalidated_lock;
> -
> -	/* BOs for user mode queues that need a validation */
> -	struct list_head	evicted_user;
> -
> -	/* regular invalidated BOs, but not yet updated in the PT */
> -	struct list_head	invalidated;
> +	spinlock_t			individual_lock;
>   
> -	/* BOs which are invalidated, has been updated in the PTs */
> -	struct list_head        done;
> +	/* Userspace BOs with individual resv object */
> +	struct amdgpu_vm_bo_status	individual;
>   
>   	/*
>   	 * This list contains amdgpu_bo_va_mapping objects which have been freed
> @@ -510,8 +506,8 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm);
>   void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
>   int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec,
>   		      unsigned int num_fences);
> -int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
> -			     unsigned int num_fences);
> +int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
> +			      unsigned int num_fences);
>   bool amdgpu_vm_ready(struct amdgpu_vm *vm);
>   uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm);
>   int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,

Re: [PATCH 04/11] drm/amdgpu: rework amdgpu_userq_signal_ioctl

[Christian König]

Hi guys,

On 4/23/26 11:58, Liang, Prike wrote:
...
>> -static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
>> +static int amdgpu_userq_fence_alloc(struct amdgpu_usermode_queue *userq,
>> +                                 struct amdgpu_userq_fence **pfence)
>>  {
>> -     *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
>> -     return *userq_fence ? 0 : -ENOMEM;
>> +     struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
>> +     struct amdgpu_userq_fence *userq_fence;
>> +     unsigned long count;
> We must initialize count; otherwise, it may contain a garbage value, which can cause amdgpu_userq_fence_alloc() to fail and,
> in turn, make userq fence emission fail.

I've got the same comment from both Sunil and Prike but as far as I can see  and that is actually incorrect.

> 
>> +     userq_fence = kmalloc(sizeof(*userq_fence), GFP_KERNEL);
>> +     if (!userq_fence)
>> +             return -ENOMEM;
>> +
>> +     /*
>> +      * Get the next unused entry, since we fill from the start this can be
>> +      * used as size to allocate the array.
>> +      */
>> +     mutex_lock(&userq->fence_drv_lock);
>> +     xa_find(&userq->fence_drv_xa, &count, ULONG_MAX, XA_FREE_MARK);

The count should be initialized here. But could be that this doesn't work.

Did you guys got a KASAN warning or something like that?

>> +
>> +     userq_fence->fence_drv_array = kvmalloc_array(count, sizeof(fence_drv),
>> +                                                   GFP_KERNEL);
>> +     if (!userq_fence->fence_drv_array) {
>> +             mutex_unlock(&userq->fence_drv_lock);
>> +             kfree(userq_fence);
>> +             return -ENOMEM;
>> +     }
>> +
>> +     userq_fence->fence_drv_array_count = count;
>> +     xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array,
>> +                0, ULONG_MAX, count, XA_PRESENT);
> We may need to assign the userq_fence->fence_drv_array_count the exact copied number from the xa_extract().

Interresting point. Why could that differ ?

Thanks for the comments,
Christian.

Re: [PATCH 11/11] drm/amdgpu: WIP sync amdgpu_ttm_fill_mem only to kernel fences

[Khatri, Sunil]

Acked-by: Sunil Khatri <sunil.khatri@amd.com>

Regards
Sunil Khatri

On 21-04-2026 06:25 pm, Christian König wrote:
> That's not even remotely correct, but should unblock testing for now.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 12 +++++++-----
>   1 file changed, 7 insertions(+), 5 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> index 4c7d1917d9bb..5130f77b7543 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> @@ -2417,12 +2417,14 @@ static int amdgpu_ttm_prepare_job(struct amdgpu_device *adev,
>   				  struct amdgpu_ttm_buffer_entity *entity,
>   				  unsigned int num_dw,
>   				  struct dma_resv *resv,
> +				  enum dma_resv_usage usage,
>   				  bool vm_needs_flush,
>   				  struct amdgpu_job **job,
>   				  u64 k_job_id)
>   {
>   	enum amdgpu_ib_pool_type pool = AMDGPU_IB_POOL_DELAYED;
>   	int r;
> +
>   	r = amdgpu_job_alloc_with_ib(adev, &entity->base,
>   				     AMDGPU_FENCE_OWNER_UNDEFINED,
>   				     num_dw * 4, pool, job, k_job_id);
> @@ -2438,8 +2440,7 @@ static int amdgpu_ttm_prepare_job(struct amdgpu_device *adev,
>   	if (!resv)
>   		return 0;
>   
> -	return drm_sched_job_add_resv_dependencies(&(*job)->base, resv,
> -						   DMA_RESV_USAGE_BOOKKEEP);
> +	return drm_sched_job_add_resv_dependencies(&(*job)->base, resv, usage);
>   }
>   
>   int amdgpu_copy_buffer(struct amdgpu_device *adev,
> @@ -2468,9 +2469,9 @@ int amdgpu_copy_buffer(struct amdgpu_device *adev,
>   	max_bytes = adev->mman.buffer_funcs->copy_max_bytes;
>   	num_loops = DIV_ROUND_UP(byte_count, max_bytes);
>   	num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->copy_num_dw, 8);
> -	r = amdgpu_ttm_prepare_job(adev, entity, num_dw,
> -				   resv, vm_needs_flush, &job,
> -				   AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER);
> +	r = amdgpu_ttm_prepare_job(adev, entity, num_dw, resv,
> +				   DMA_RESV_USAGE_BOOKKEEP, vm_needs_flush,
> +				   &job, AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER);
>   	if (r)
>   		goto error_free;
>   
> @@ -2513,6 +2514,7 @@ static int amdgpu_ttm_fill_mem(struct amdgpu_device *adev,
>   	num_loops = DIV_ROUND_UP_ULL(byte_count, max_bytes);
>   	num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->fill_num_dw, 8);
>   	r = amdgpu_ttm_prepare_job(adev, entity, num_dw, resv,
> +				   DMA_RESV_USAGE_KERNEL,
>   				   vm_needs_flush, &job, k_job_id);
>   	if (r)
>   		return r;

Re: [PATCH 04/11] drm/amdgpu: rework amdgpu_userq_signal_ioctl

[Khatri, Sunil]

On 23-04-2026 04:17 pm, Christian König wrote:
> Hi guys,
>
> On 4/23/26 11:58, Liang, Prike wrote:
> ...
>>> -static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
>>> +static int amdgpu_userq_fence_alloc(struct amdgpu_usermode_queue *userq,
>>> +                                 struct amdgpu_userq_fence **pfence)
>>>   {
>>> -     *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
>>> -     return *userq_fence ? 0 : -ENOMEM;
>>> +     struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
>>> +     struct amdgpu_userq_fence *userq_fence;
>>> +     unsigned long count;
>> We must initialize count; otherwise, it may contain a garbage value, which can cause amdgpu_userq_fence_alloc() to fail and,
>> in turn, make userq fence emission fail.
> I've got the same comment from both Sunil and Prike but as far as I can see  and that is actually incorrect.
>
>>> +     userq_fence = kmalloc(sizeof(*userq_fence), GFP_KERNEL);
>>> +     if (!userq_fence)
>>> +             return -ENOMEM;
>>> +
>>> +     /*
>>> +      * Get the next unused entry, since we fill from the start this can be
>>> +      * used as size to allocate the array.
>>> +      */
>>> +     mutex_lock(&userq->fence_drv_lock);
>>> +     xa_find(&userq->fence_drv_xa, &count, ULONG_MAX, XA_FREE_MARK);
> The count should be initialized here. But could be that this doesn't work.
>
> Did you guys got a KASAN warning or something like that?
I got application crash recieving -ENOMEM. The reason that count isnt 
initialized and as per information i gather xa_find need the starting 
index i.e pointed by count else it will start with whatever garbage 
value it have.

Regards
Sunil Khatri
>
>>> +
>>> +     userq_fence->fence_drv_array = kvmalloc_array(count, sizeof(fence_drv),
>>> +                                                   GFP_KERNEL);
>>> +     if (!userq_fence->fence_drv_array) {
>>> +             mutex_unlock(&userq->fence_drv_lock);
>>> +             kfree(userq_fence);
>>> +             return -ENOMEM;
>>> +     }
>>> +
>>> +     userq_fence->fence_drv_array_count = count;
>>> +     xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array,
>>> +                0, ULONG_MAX, count, XA_PRESENT);
>> We may need to assign the userq_fence->fence_drv_array_count the exact copied number from the xa_extract().
> Interresting point. Why could that differ ?
>
> Thanks for the comments,
> Christian.

RE: [PATCH 04/11] drm/amdgpu: rework amdgpu_userq_signal_ioctl

[Liang, Prike]

[Public]

Regards,
      Prike

> -----Original Message-----
> From: Koenig, Christian <Christian.Koenig@amd.com>
> Sent: Thursday, April 23, 2026 6:48 PM
> To: Liang, Prike <Prike.Liang@amd.com>; Khatri, Sunil <Sunil.Khatri@amd.com>
> Cc: Koenig, Christian <Christian.Koenig@amd.com>; Deucher, Alexander
> <Alexander.Deucher@amd.com>; amd-gfx@lists.freedesktop.org
> Subject: Re: [PATCH 04/11] drm/amdgpu: rework amdgpu_userq_signal_ioctl
>
> Hi guys,
>
> On 4/23/26 11:58, Liang, Prike wrote:
> ...
> >> -static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence
> >> **userq_fence)
> >> +static int amdgpu_userq_fence_alloc(struct amdgpu_usermode_queue *userq,
> >> +                                 struct amdgpu_userq_fence **pfence)
> >>  {
> >> -     *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
> >> -     return *userq_fence ? 0 : -ENOMEM;
> >> +     struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
> >> +     struct amdgpu_userq_fence *userq_fence;
> >> +     unsigned long count;
> > We must initialize count; otherwise, it may contain a garbage value,
> > which can cause amdgpu_userq_fence_alloc() to fail and, in turn, make userq
> fence emission fail.
>
> I've got the same comment from both Sunil and Prike but as far as I can see  and
> that is actually incorrect.
This patch breaks the userq fence emit path, causing desktop boot to fail. Initializing count only works around the amdgpu_userq_fence_alloc() failure, and it doesn't address the root cause, which is that xa_find() cannot initialize count when fence_drv_xa itself hasn't been set up yet. Instead of just initializing count, we may need to check the return value of xa_find(), and if no wait fences are pending, skip retrieving the wait fence array entirely.

> >
> >> +     userq_fence = kmalloc(sizeof(*userq_fence), GFP_KERNEL);
> >> +     if (!userq_fence)
> >> +             return -ENOMEM;
> >> +
> >> +     /*
> >> +      * Get the next unused entry, since we fill from the start this can be
> >> +      * used as size to allocate the array.
> >> +      */
> >> +     mutex_lock(&userq->fence_drv_lock);
> >> +     xa_find(&userq->fence_drv_xa, &count, ULONG_MAX, XA_FREE_MARK);
>
> The count should be initialized here. But could be that this doesn't work.
>
> Did you guys got a KASAN warning or something like that?
I didn't see the KASAN warning. However, the underlying problem is that when fence_drv_xa hasn't been set up, count remains uninitialized (garbage), which eventually causes kvmalloc_array() to fail when allocating fence_drv_array.

> >> +
> >> +     userq_fence->fence_drv_array = kvmalloc_array(count, sizeof(fence_drv),
> >> +                                                   GFP_KERNEL);
> >> +     if (!userq_fence->fence_drv_array) {
> >> +             mutex_unlock(&userq->fence_drv_lock);
> >> +             kfree(userq_fence);
> >> +             return -ENOMEM;
> >> +     }
> >> +
> >> +     userq_fence->fence_drv_array_count = count;
> >> +     xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array,
> >> +                0, ULONG_MAX, count, XA_PRESENT);
> > We may need to assign the userq_fence->fence_drv_array_count the exact copied
> number from the xa_extract().
>
> Interresting point. Why could that differ ?
Generally, xa_extract() should return the same number as count, but when there's a retry entry, the actual number of copied entries may differ from the wait fence array capacity indicated by count.

> Thanks for the comments,
> Christian.

Re: [PATCH 04/11] drm/amdgpu: rework amdgpu_userq_signal_ioctl

[Christian König]

Hi Prike,

On 4/24/26 10:01, Liang, Prike wrote:
>> -----Original Message-----
>> From: Koenig, Christian <Christian.Koenig@amd.com>
>> Sent: Thursday, April 23, 2026 6:48 PM
>> To: Liang, Prike <Prike.Liang@amd.com>; Khatri, Sunil <Sunil.Khatri@amd.com>
>> Cc: Koenig, Christian <Christian.Koenig@amd.com>; Deucher, Alexander
>> <Alexander.Deucher@amd.com>; amd-gfx@lists.freedesktop.org
>> Subject: Re: [PATCH 04/11] drm/amdgpu: rework amdgpu_userq_signal_ioctl
>>
>> Hi guys,
>>
>> On 4/23/26 11:58, Liang, Prike wrote:
>> ...
>>>> -static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence
>>>> **userq_fence)
>>>> +static int amdgpu_userq_fence_alloc(struct amdgpu_usermode_queue *userq,
>>>> +                                 struct amdgpu_userq_fence **pfence)
>>>>  {
>>>> -     *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
>>>> -     return *userq_fence ? 0 : -ENOMEM;
>>>> +     struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
>>>> +     struct amdgpu_userq_fence *userq_fence;
>>>> +     unsigned long count;
>>> We must initialize count; otherwise, it may contain a garbage value,
>>> which can cause amdgpu_userq_fence_alloc() to fail and, in turn, make userq
>> fence emission fail.
>>
>> I've got the same comment from both Sunil and Prike but as far as I can see  and
>> that is actually incorrect.
> This patch breaks the userq fence emit path, causing desktop boot to fail. Initializing count only works around the amdgpu_userq_fence_alloc() failure, and it doesn't address the root cause, which is that xa_find() cannot initialize count when fence_drv_xa itself hasn't been set up yet. Instead of just initializing count, we may need to check the return value of xa_find(), and if no wait fences are pending, skip retrieving the wait fence array entirely.

Yeah Sunil and I figured out what was wrong here.

I was looking at the xas_find() function and thought that xa_find() would be just a wrapper around that.

But that doesn't work like that. So I not only need to initialize count, but use the xas_fine function directly.

Thanks for pointing that out,
Christian.

> 
>>>
>>>> +     userq_fence = kmalloc(sizeof(*userq_fence), GFP_KERNEL);
>>>> +     if (!userq_fence)
>>>> +             return -ENOMEM;
>>>> +
>>>> +     /*
>>>> +      * Get the next unused entry, since we fill from the start this can be
>>>> +      * used as size to allocate the array.
>>>> +      */
>>>> +     mutex_lock(&userq->fence_drv_lock);
>>>> +     xa_find(&userq->fence_drv_xa, &count, ULONG_MAX, XA_FREE_MARK);
>>
>> The count should be initialized here. But could be that this doesn't work.
>>
>> Did you guys got a KASAN warning or something like that?
> I didn't see the KASAN warning. However, the underlying problem is that when fence_drv_xa hasn't been set up, count remains uninitialized (garbage), which eventually causes kvmalloc_array() to fail when allocating fence_drv_array.
> 
>>>> +
>>>> +     userq_fence->fence_drv_array = kvmalloc_array(count, sizeof(fence_drv),
>>>> +                                                   GFP_KERNEL);
>>>> +     if (!userq_fence->fence_drv_array) {
>>>> +             mutex_unlock(&userq->fence_drv_lock);
>>>> +             kfree(userq_fence);
>>>> +             return -ENOMEM;
>>>> +     }
>>>> +
>>>> +     userq_fence->fence_drv_array_count = count;
>>>> +     xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array,
>>>> +                0, ULONG_MAX, count, XA_PRESENT);
>>> We may need to assign the userq_fence->fence_drv_array_count the exact copied
>> number from the xa_extract().
>>
>> Interresting point. Why could that differ ?
> Generally, xa_extract() should return the same number as count, but when there's a retry entry, the actual number of copied entries may differ from the wait fence array capacity indicated by count.
> 
>> Thanks for the comments,
>> Christian.

RE: [PATCH 03/11] drm/amdgpu: nuke amdgpu_userq_fence_free

[Liang, Prike]

Public

Regards,
      Prike

> -----Original Message-----
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> Sent: Tuesday, April 21, 2026 8:55 PM
> To: Deucher, Alexander <Alexander.Deucher@amd.com>; Liang, Prike
> <Prike.Liang@amd.com>; Khatri, Sunil <Sunil.Khatri@amd.com>; amd-
> gfx@lists.freedesktop.org
> Cc: Koenig, Christian <Christian.Koenig@amd.com>
> Subject: [PATCH 03/11] drm/amdgpu: nuke amdgpu_userq_fence_free
>
> As preparation for independent fences remove the function and do all of it's cleanup
> directly after signaling.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c       | 13 +--
>  .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 79 +++++++------------
>  .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.h   |  3 -
>  3 files changed, 31 insertions(+), 64 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index c6546a858597..1b15b51dc3f4 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -3162,11 +3162,7 @@ static int __init amdgpu_init(void)
>
>       r = amdgpu_sync_init();
>       if (r)
> -             goto error_sync;
> -
> -     r = amdgpu_userq_fence_slab_init();
> -     if (r)
> -             goto error_fence;
> +             return r;
We need to clean up the error_fence error handler for the following amdgpu_amdkfd_init() error path in the amdgpu_init () as well.
Except that, the patch is Reviewed-by: Prike Liang <Prike.Liang@amd.com>

>       amdgpu_register_atpx_handler();
>       amdgpu_acpi_detect();
> @@ -3182,12 +3178,6 @@ static int __init amdgpu_init(void)
>
>       /* let modprobe override vga console setting */
>       return pci_register_driver(&amdgpu_kms_pci_driver);
> -
> -error_fence:
> -     amdgpu_sync_fini();
> -
> -error_sync:
> -     return r;
>  }
>
>  static void __exit amdgpu_exit(void)
> @@ -3197,7 +3187,6 @@ static void __exit amdgpu_exit(void)
>       amdgpu_unregister_atpx_handler();
>       amdgpu_acpi_release();
>       amdgpu_sync_fini();
> -     amdgpu_userq_fence_slab_fini();
>       mmu_notifier_synchronize();
>       amdgpu_xcp_drv_release();
>  }
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> index a58342c2ac44..909bdccc2a92 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> @@ -32,29 +32,9 @@
>  #include "amdgpu.h"
>  #include "amdgpu_userq_fence.h"
>
> -static const struct dma_fence_ops amdgpu_userq_fence_ops; -static struct
> kmem_cache *amdgpu_userq_fence_slab;
> -
>  #define AMDGPU_USERQ_MAX_HANDLES     (1U << 16)
>
> -int amdgpu_userq_fence_slab_init(void)
> -{
> -     amdgpu_userq_fence_slab = kmem_cache_create("amdgpu_userq_fence",
> -                                                 sizeof(struct amdgpu_userq_fence),
> -                                                 0,
> -                                                 SLAB_HWCACHE_ALIGN,
> -                                                 NULL);
> -     if (!amdgpu_userq_fence_slab)
> -             return -ENOMEM;
> -
> -     return 0;
> -}
> -
> -void amdgpu_userq_fence_slab_fini(void)
> -{
> -     rcu_barrier();
> -     kmem_cache_destroy(amdgpu_userq_fence_slab);
> -}
> +static const struct dma_fence_ops amdgpu_userq_fence_ops;
>
>  static inline struct amdgpu_userq_fence *to_amdgpu_userq_fence(struct
> dma_fence *f)  { @@ -146,12 +126,18 @@ amdgpu_userq_fence_driver_free(struct
> amdgpu_usermode_queue *userq)  }
>
>  static void
> -amdgpu_userq_fence_put_fence_drv_array(struct amdgpu_userq_fence
> *userq_fence)
> +amdgpu_userq_fence_put_fence_drv_refs(struct amdgpu_userq_fence
> +*userq_fence)
>  {
>       unsigned long i;
> +
>       for (i = 0; i < userq_fence->fence_drv_array_count; i++)
>               amdgpu_userq_fence_driver_put(userq_fence->fence_drv_array[i]);
>       userq_fence->fence_drv_array_count = 0;
> +     kfree(userq_fence->fence_drv_array);
> +     userq_fence->fence_drv_array = NULL;
> +
> +     amdgpu_userq_fence_driver_put(userq_fence->fence_drv);
> +     userq_fence->fence_drv = NULL;
>  }
>
>  void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver
> *fence_drv) @@ -181,10 +167,11 @@ void
> amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
>               fence = &userq_fence->base;
>               list_del_init(&userq_fence->link);
>               dma_fence_signal(fence);
> -             /* Drop fence_drv_array outside fence_list_lock
> +             /*
> +              * Drop fence_drv_array outside fence_list_lock
>                * to avoid the recursion lock.
>                */
> -             amdgpu_userq_fence_put_fence_drv_array(userq_fence);
> +             amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
>               dma_fence_put(fence);
>       }
>
> @@ -231,7 +218,7 @@ void amdgpu_userq_fence_driver_put(struct
> amdgpu_userq_fence_driver *fence_drv)
>
>  static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)  {
> -     *userq_fence = kmem_cache_alloc(amdgpu_userq_fence_slab,
> GFP_ATOMIC);
> +     *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
>       return *userq_fence ? 0 : -ENOMEM;
>  }
>
> @@ -299,7 +286,7 @@ static int amdgpu_userq_fence_create(struct
> amdgpu_usermode_queue *userq,
>       spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
>
>       if (signaled)
> -             amdgpu_userq_fence_put_fence_drv_array(userq_fence);
> +             amdgpu_userq_fence_put_fence_drv_refs(userq_fence);
>
>       *f = fence;
>
> @@ -333,29 +320,10 @@ static bool amdgpu_userq_fence_signaled(struct
> dma_fence *f)
>       return false;
>  }
>
> -static void amdgpu_userq_fence_free(struct rcu_head *rcu) -{
> -     struct dma_fence *fence = container_of(rcu, struct dma_fence, rcu);
> -     struct amdgpu_userq_fence *userq_fence = to_amdgpu_userq_fence(fence);
> -     struct amdgpu_userq_fence_driver *fence_drv = userq_fence->fence_drv;
> -
> -     /* Release the fence driver reference */
> -     amdgpu_userq_fence_driver_put(fence_drv);
> -
> -     kvfree(userq_fence->fence_drv_array);
> -     kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
> -}
> -
> -static void amdgpu_userq_fence_release(struct dma_fence *f) -{
> -     call_rcu(&f->rcu, amdgpu_userq_fence_free);
> -}
> -
>  static const struct dma_fence_ops amdgpu_userq_fence_ops = {
>       .get_driver_name = amdgpu_userq_fence_get_driver_name,
>       .get_timeline_name = amdgpu_userq_fence_get_timeline_name,
>       .signaled = amdgpu_userq_fence_signaled,
> -     .release = amdgpu_userq_fence_release,
>  };
>
>  /**
> @@ -546,7 +514,7 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void
> *data,
>       r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
>       if (r) {
>               mutex_unlock(&userq_mgr->userq_mutex);
> -             kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
> +             kfree(userq_fence);
>               goto put_gobj_write;
>       }
>
> @@ -871,6 +839,7 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>       for (i = 0, cnt = 0; i < num_fences; i++) {
>               struct amdgpu_userq_fence_driver *fence_drv;
>               struct amdgpu_userq_fence *userq_fence;
> +             unsigned long flags;
>               u32 index;
>
>               userq_fence = to_amdgpu_userq_fence(fences[i]); @@ -886,7
> +855,19 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>                       continue;
>               }
>
> +             spin_lock_irqsave(userq_fence->base.lock, flags);
> +             if (dma_fence_is_signaled_locked(&userq_fence->base)) {
> +                     /*
> +                      * It is possible that fence is already signaled and the
> +                      * fence_drv now NULL, just skip over such fences.
> +                     */
> +                     spin_unlock_irqrestore(userq_fence->base.lock, flags);
> +                     continue;
> +             }
>               fence_drv = userq_fence->fence_drv;
> +             amdgpu_userq_fence_driver_get(fence_drv);
> +             spin_unlock_irqrestore(userq_fence->base.lock, flags);
> +
>               /*
>                * We need to make sure the user queue release their reference
>                * to the fence drivers at some point before queue destruction.
> @@ -895,10 +876,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file
> *filp,
>                */
>               r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
>                            xa_limit_32b, GFP_KERNEL);
> -             if (r)
> +             if (r) {
> +                     amdgpu_userq_fence_driver_put(fence_drv);
>                       goto put_waitq;
> -
> -             amdgpu_userq_fence_driver_get(fence_drv);
> +             }
>
>               /* Store drm syncobj's gpu va address and value */
>               fence_info[cnt].va = fence_drv->va;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> index d56246ad8c26..d355a0eecc07 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> @@ -58,9 +58,6 @@ struct amdgpu_userq_fence_driver {
>       char timeline_name[TASK_COMM_LEN];
>  };
>
> -int amdgpu_userq_fence_slab_init(void);
> -void amdgpu_userq_fence_slab_fini(void);
> -
>  void amdgpu_userq_fence_driver_get(struct amdgpu_userq_fence_driver
> *fence_drv);  void amdgpu_userq_fence_driver_put(struct
> amdgpu_userq_fence_driver *fence_drv);  int
> amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,
> --
> 2.43.0

RE: [PATCH 02/11] drm/amdgpu: remove deadlocks from amdgpu_userq_pre_reset

[Liang, Prike]

Public

Reviewed-by: Prike Liang <Prike.Liang@amd.com>

Regards,
      Prike

> -----Original Message-----
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> Sent: Tuesday, April 21, 2026 8:55 PM
> To: Deucher, Alexander <Alexander.Deucher@amd.com>; Liang, Prike
> <Prike.Liang@amd.com>; Khatri, Sunil <Sunil.Khatri@amd.com>; amd-
> gfx@lists.freedesktop.org
> Cc: Koenig, Christian <Christian.Koenig@amd.com>
> Subject: [PATCH 02/11] drm/amdgpu: remove deadlocks from
> amdgpu_userq_pre_reset
>
> The purpose of a GPU reset is to make sure that fence can be signaled again and
> the signal and resume workers can make progress again.
>
> So waiting for the resume worker or any fence in the GPU reset path is just utterly
> nonsense.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 26 +++++++++++------------
>  1 file changed, 12 insertions(+), 14 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index 8f48520cb822..b632bc3c952b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -1496,23 +1496,21 @@ void amdgpu_userq_pre_reset(struct amdgpu_device
> *adev)  {
>       const struct amdgpu_userq_funcs *userq_funcs;
>       struct amdgpu_usermode_queue *queue;
> -     struct amdgpu_userq_mgr *uqm;
>       unsigned long queue_id;
>
> +     /* TODO: We probably need a new lock for the queue state */
>       xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
> -             uqm = queue->userq_mgr;
> -             cancel_delayed_work_sync(&uqm->resume_work);
> -             if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
> -                     amdgpu_userq_wait_for_last_fence(queue);
> -                     userq_funcs = adev->userq_funcs[queue->queue_type];
> -                     userq_funcs->unmap(queue);
> -                     /* just mark all queues as hung at this point.
> -                      * if unmap succeeds, we could map again
> -                      * in amdgpu_userq_post_reset() if vram is not lost
> -                      */
> -                     queue->state = AMDGPU_USERQ_STATE_HUNG;
> -                     amdgpu_userq_fence_driver_force_completion(queue);
> -             }
> +             if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
> +                     continue;
> +
> +             userq_funcs = adev->userq_funcs[queue->queue_type];
> +             userq_funcs->unmap(queue);
> +             /* just mark all queues as hung at this point.
> +              * if unmap succeeds, we could map again
> +              * in amdgpu_userq_post_reset() if vram is not lost
> +              */
> +             queue->state = AMDGPU_USERQ_STATE_HUNG;
> +             amdgpu_userq_fence_driver_force_completion(queue);
The userq hang detection and fence completion may have already been handled by detect_and_reset() in amdgpu_userq_mgr_reset_work(). In that case, we could validate the queue and fence state first and skip the redundant operation. That's an option improvement, and the patch is fine with it or not, so the patch is Reviewed-by: Prike Liang <Prike.Liang@amd.com>.

>       }
>  }
>
> --
> 2.43.0