* [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab
@ 2026-04-27 18:27 Christian König
2026-04-27 18:27 ` [PATCH 2/8] drm/amdgpu: remove deadlocks from amdgpu_userq_pre_reset Christian König
` (8 more replies)
0 siblings, 9 replies; 30+ messages in thread
From: Christian König @ 2026-04-27 18:27 UTC (permalink / raw)
To: alexander.deucher, Prike.Liang, sukhatri, jesse.zhang; +Cc: amd-gfx
As preparation for independent fences remove the extra slab, kmalloc
should do just fine.
Signed-off-by: Christian König <christian.koenig@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 17 ++++-------
.../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c | 28 +++----------------
.../gpu/drm/amd/amdgpu/amdgpu_userq_fence.h | 3 --
3 files changed, 9 insertions(+), 39 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 8f66f197f645..2b268cdf076b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -3162,11 +3162,7 @@ static int __init amdgpu_init(void)
r = amdgpu_sync_init();
if (r)
- goto error_sync;
-
- r = amdgpu_userq_fence_slab_init();
- if (r)
- goto error_fence;
+ return r;
amdgpu_register_atpx_handler();
amdgpu_acpi_detect();
@@ -3174,7 +3170,7 @@ static int __init amdgpu_init(void)
/* Ignore KFD init failures when CONFIG_HSA_AMD is not set. */
r = amdgpu_amdkfd_init();
if (r && r != -ENOENT)
- goto error_fence;
+ goto error_fini_sync;
if (amdgpu_pp_feature_mask & PP_OVERDRIVE_MASK) {
add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
@@ -3185,11 +3181,9 @@ static int __init amdgpu_init(void)
/* let modprobe override vga console setting */
return pci_register_driver(&amdgpu_kms_pci_driver);
-error_fence:
- amdgpu_sync_fini();
-
-error_sync:
- return r;
+error_fini_sync:
+ amdgpu_sync_fini();
+ return r;
}
static void __exit amdgpu_exit(void)
@@ -3199,7 +3193,6 @@ static void __exit amdgpu_exit(void)
amdgpu_unregister_atpx_handler();
amdgpu_acpi_release();
amdgpu_sync_fini();
- amdgpu_userq_fence_slab_fini();
mmu_notifier_synchronize();
amdgpu_xcp_drv_release();
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
index a58342c2ac44..f3d0595e3e61 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
@@ -32,29 +32,9 @@
#include "amdgpu.h"
#include "amdgpu_userq_fence.h"
-static const struct dma_fence_ops amdgpu_userq_fence_ops;
-static struct kmem_cache *amdgpu_userq_fence_slab;
-
#define AMDGPU_USERQ_MAX_HANDLES (1U << 16)
-int amdgpu_userq_fence_slab_init(void)
-{
- amdgpu_userq_fence_slab = kmem_cache_create("amdgpu_userq_fence",
- sizeof(struct amdgpu_userq_fence),
- 0,
- SLAB_HWCACHE_ALIGN,
- NULL);
- if (!amdgpu_userq_fence_slab)
- return -ENOMEM;
-
- return 0;
-}
-
-void amdgpu_userq_fence_slab_fini(void)
-{
- rcu_barrier();
- kmem_cache_destroy(amdgpu_userq_fence_slab);
-}
+static const struct dma_fence_ops amdgpu_userq_fence_ops;
static inline struct amdgpu_userq_fence *to_amdgpu_userq_fence(struct dma_fence *f)
{
@@ -231,7 +211,7 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
{
- *userq_fence = kmem_cache_alloc(amdgpu_userq_fence_slab, GFP_ATOMIC);
+ *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
return *userq_fence ? 0 : -ENOMEM;
}
@@ -343,7 +323,7 @@ static void amdgpu_userq_fence_free(struct rcu_head *rcu)
amdgpu_userq_fence_driver_put(fence_drv);
kvfree(userq_fence->fence_drv_array);
- kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
+ kfree(userq_fence);
}
static void amdgpu_userq_fence_release(struct dma_fence *f)
@@ -546,7 +526,7 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
if (r) {
mutex_unlock(&userq_mgr->userq_mutex);
- kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
+ kfree(userq_fence);
goto put_gobj_write;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
index d56246ad8c26..d355a0eecc07 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
@@ -58,9 +58,6 @@ struct amdgpu_userq_fence_driver {
char timeline_name[TASK_COMM_LEN];
};
-int amdgpu_userq_fence_slab_init(void);
-void amdgpu_userq_fence_slab_fini(void);
-
void amdgpu_userq_fence_driver_get(struct amdgpu_userq_fence_driver *fence_drv);
void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv);
int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,
--
2.43.0
^ permalink raw reply related [flat|nested] 30+ messages in thread
* [PATCH 2/8] drm/amdgpu: remove deadlocks from amdgpu_userq_pre_reset
2026-04-27 18:27 [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab Christian König
@ 2026-04-27 18:27 ` Christian König
2026-04-28 6:51 ` Liang, Prike
2026-04-27 18:27 ` [PATCH 3/8] drm/amdgpu: rework amdgpu_userq_signal_ioctl v2 Christian König
` (7 subsequent siblings)
8 siblings, 1 reply; 30+ messages in thread
From: Christian König @ 2026-04-27 18:27 UTC (permalink / raw)
To: alexander.deucher, Prike.Liang, sukhatri, jesse.zhang; +Cc: amd-gfx
The purpose of a GPU reset is to make sure that fence can be signaled
again and the signal and resume workers can make progress again.
So waiting for the resume worker or any fence in the GPU reset path is
just utterly nonsense.
Signed-off-by: Christian König <christian.koenig@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 26 +++++++++++------------
1 file changed, 12 insertions(+), 14 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 58b9477c0346..a34b1df513d3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -1510,23 +1510,21 @@ void amdgpu_userq_pre_reset(struct amdgpu_device *adev)
{
const struct amdgpu_userq_funcs *userq_funcs;
struct amdgpu_usermode_queue *queue;
- struct amdgpu_userq_mgr *uqm;
unsigned long queue_id;
+ /* TODO: We probably need a new lock for the queue state */
xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
- uqm = queue->userq_mgr;
- cancel_delayed_work_sync(&uqm->resume_work);
- if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
- amdgpu_userq_wait_for_last_fence(queue);
- userq_funcs = adev->userq_funcs[queue->queue_type];
- userq_funcs->unmap(queue);
- /* just mark all queues as hung at this point.
- * if unmap succeeds, we could map again
- * in amdgpu_userq_post_reset() if vram is not lost
- */
- queue->state = AMDGPU_USERQ_STATE_HUNG;
- amdgpu_userq_fence_driver_force_completion(queue);
- }
+ if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
+ continue;
+
+ userq_funcs = adev->userq_funcs[queue->queue_type];
+ userq_funcs->unmap(queue);
+ /* just mark all queues as hung at this point.
+ * if unmap succeeds, we could map again
+ * in amdgpu_userq_post_reset() if vram is not lost
+ */
+ queue->state = AMDGPU_USERQ_STATE_HUNG;
+ amdgpu_userq_fence_driver_force_completion(queue);
}
}
--
2.43.0
^ permalink raw reply related [flat|nested] 30+ messages in thread
* [PATCH 3/8] drm/amdgpu: rework amdgpu_userq_signal_ioctl v2
2026-04-27 18:27 [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab Christian König
2026-04-27 18:27 ` [PATCH 2/8] drm/amdgpu: remove deadlocks from amdgpu_userq_pre_reset Christian König
@ 2026-04-27 18:27 ` Christian König
2026-04-28 8:49 ` Liang, Prike
2026-04-28 9:35 ` Khatri, Sunil
2026-04-27 18:27 ` [PATCH 4/8] drm/amdgpu: remove almost all calls to amdgpu_userq_detect_and_reset_queues Christian König
` (6 subsequent siblings)
8 siblings, 2 replies; 30+ messages in thread
From: Christian König @ 2026-04-27 18:27 UTC (permalink / raw)
To: alexander.deucher, Prike.Liang, sukhatri, jesse.zhang; +Cc: amd-gfx
This one was fortunately not looking so bad as the wait ioctl path, but
there were still a few things which could be fixed/improved:
1. Allocating with GFP_ATOMIC was quite unecessary, we can do that
before taking the userq_lock.
2. Use a new mutex as protection for the fence_drv_xa so that we can do
memory allocations while holding it.
3. Starting the reset timer is unecessary when the fence is already
signaled when we create it.
4. Cleanup error handling, avoid trying to free the queue when we don't
even got one.
v2: fix incorrect usage of xa_find, destroy the new mutex on error
Signed-off-by: Christian König <christian.koenig@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 2 +
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 12 +
.../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c | 227 ++++++++----------
3 files changed, 119 insertions(+), 122 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index a34b1df513d3..62d155d6e4ed 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -806,6 +806,7 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
}
queue->doorbell_index = index;
+ mutex_init(&queue->fence_drv_lock);
xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
if (r) {
@@ -879,6 +880,7 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
amdgpu_bo_reserve(fpriv->vm.root.bo, true);
amdgpu_userq_buffer_vas_list_cleanup(adev, queue);
amdgpu_bo_unreserve(fpriv->vm.root.bo);
+ mutex_destroy(&queue->fence_drv_lock);
free_queue:
kfree(queue);
err_pm_runtime:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index 8b8f345b60b6..843ea8ecc5d7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -66,6 +66,18 @@ struct amdgpu_usermode_queue {
struct amdgpu_userq_obj db_obj;
struct amdgpu_userq_obj fw_obj;
struct amdgpu_userq_obj wptr_obj;
+
+ /**
+ * @fence_drv_lock: Protecting @fence_drv_xa.
+ */
+ struct mutex fence_drv_lock;
+
+ /**
+ * @fence_drv_xa:
+ *
+ * References to the external fence drivers returned by wait_ioctl.
+ * Dropped on the next signaled dma_fence or queue destruction.
+ */
struct xarray fence_drv_xa;
struct amdgpu_userq_fence_driver *fence_drv;
struct dma_fence *last_fence;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
index f3d0595e3e61..a2792f05b670 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
@@ -121,6 +121,7 @@ amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq)
userq->last_fence = NULL;
amdgpu_userq_walk_and_drop_fence_drv(&userq->fence_drv_xa);
xa_destroy(&userq->fence_drv_xa);
+ mutex_destroy(&userq->fence_drv_lock);
/* Drop the queue's ownership reference to fence_drv explicitly */
amdgpu_userq_fence_driver_put(userq->fence_drv);
}
@@ -209,81 +210,84 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
kref_put(&fence_drv->refcount, amdgpu_userq_fence_driver_destroy);
}
-static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
+static int amdgpu_userq_fence_alloc(struct amdgpu_usermode_queue *userq,
+ struct amdgpu_userq_fence **pfence)
{
- *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
- return *userq_fence ? 0 : -ENOMEM;
+ struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
+ struct amdgpu_userq_fence *userq_fence;
+ void * entry;
+
+ userq_fence = kmalloc(sizeof(*userq_fence), GFP_KERNEL);
+ if (!userq_fence)
+ return -ENOMEM;
+
+ /*
+ * Get the next unused entry, since we fill from the start this can be
+ * used as size to allocate the array.
+ */
+ mutex_lock(&userq->fence_drv_lock);
+ XA_STATE(xas, &userq->fence_drv_xa, 0);
+
+ rcu_read_lock();
+ do {
+ entry = xas_find_marked(&xas, ULONG_MAX, XA_FREE_MARK);
+ } while (xas_retry(&xas, entry));
+ rcu_read_unlock();
+
+ userq_fence->fence_drv_array = kvmalloc_array(xas.xa_index,
+ sizeof(fence_drv),
+ GFP_KERNEL);
+ if (!userq_fence->fence_drv_array) {
+ mutex_unlock(&userq->fence_drv_lock);
+ kfree(userq_fence);
+ return -ENOMEM;
+ }
+
+ userq_fence->fence_drv_array_count = xas.xa_index;
+ xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array,
+ 0, ULONG_MAX, xas.xa_index, XA_PRESENT);
+ xa_destroy(&userq->fence_drv_xa);
+
+ mutex_unlock(&userq->fence_drv_lock);
+
+ userq_fence->fence_drv = fence_drv;
+ amdgpu_userq_fence_driver_get(fence_drv);
+
+ *pfence = userq_fence;
+ return 0;
}
-static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
- struct amdgpu_userq_fence *userq_fence,
- u64 seq, struct dma_fence **f)
+static void amdgpu_userq_fence_init(struct amdgpu_usermode_queue *userq,
+ struct amdgpu_userq_fence *fence,
+ u64 seq)
{
- struct amdgpu_userq_fence_driver *fence_drv;
- struct dma_fence *fence;
+ struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
unsigned long flags;
bool signaled = false;
- fence_drv = userq->fence_drv;
- if (!fence_drv)
- return -EINVAL;
-
- spin_lock_init(&userq_fence->lock);
- INIT_LIST_HEAD(&userq_fence->link);
- fence = &userq_fence->base;
- userq_fence->fence_drv = fence_drv;
-
- dma_fence_init64(fence, &amdgpu_userq_fence_ops, &userq_fence->lock,
+ spin_lock_init(&fence->lock);
+ dma_fence_init64(&fence->base, &amdgpu_userq_fence_ops, &fence->lock,
fence_drv->context, seq);
- amdgpu_userq_fence_driver_get(fence_drv);
- dma_fence_get(fence);
-
- if (!xa_empty(&userq->fence_drv_xa)) {
- struct amdgpu_userq_fence_driver *stored_fence_drv;
- unsigned long index, count = 0;
- int i = 0;
-
- xa_lock(&userq->fence_drv_xa);
- xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv)
- count++;
-
- userq_fence->fence_drv_array =
- kvmalloc_array(count,
- sizeof(struct amdgpu_userq_fence_driver *),
- GFP_ATOMIC);
-
- if (userq_fence->fence_drv_array) {
- xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv) {
- userq_fence->fence_drv_array[i] = stored_fence_drv;
- __xa_erase(&userq->fence_drv_xa, index);
- i++;
- }
- }
-
- userq_fence->fence_drv_array_count = i;
- xa_unlock(&userq->fence_drv_xa);
- } else {
- userq_fence->fence_drv_array = NULL;
- userq_fence->fence_drv_array_count = 0;
- }
+ /* Make sure the fence is visible to the hang detect worker */
+ dma_fence_put(userq->last_fence);
+ userq->last_fence = dma_fence_get(&fence->base);
- /* Check if hardware has already processed the job */
+ /* Check if hardware has already processed the fence */
spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
- if (!dma_fence_is_signaled(fence)) {
- list_add_tail(&userq_fence->link, &fence_drv->fences);
+ if (!dma_fence_is_signaled(&fence->base)) {
+ dma_fence_get(&fence->base);
+ list_add_tail(&fence->link, &fence_drv->fences);
} else {
+ INIT_LIST_HEAD(&fence->link);
signaled = true;
- dma_fence_put(fence);
}
spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
if (signaled)
- amdgpu_userq_fence_put_fence_drv_array(userq_fence);
-
- *f = fence;
-
- return 0;
+ amdgpu_userq_fence_put_fence_drv_array(fence);
+ else
+ amdgpu_userq_start_hang_detect_work(userq);
}
static const char *amdgpu_userq_fence_get_driver_name(struct dma_fence *f)
@@ -404,11 +408,6 @@ static int amdgpu_userq_fence_read_wptr(struct amdgpu_device *adev,
return r;
}
-static void amdgpu_userq_fence_cleanup(struct dma_fence *fence)
-{
- dma_fence_put(fence);
-}
-
static void
amdgpu_userq_fence_driver_set_error(struct amdgpu_userq_fence *fence,
int error)
@@ -452,13 +451,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
const unsigned int num_read_bo_handles = args->num_bo_read_handles;
struct amdgpu_fpriv *fpriv = filp->driver_priv;
struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
+
struct drm_gem_object **gobj_write, **gobj_read;
u32 *syncobj_handles, num_syncobj_handles;
- struct amdgpu_userq_fence *userq_fence;
- struct amdgpu_usermode_queue *queue = NULL;
- struct drm_syncobj **syncobj = NULL;
- struct dma_fence *fence;
+ struct amdgpu_usermode_queue *queue;
+ struct amdgpu_userq_fence *fence;
+ struct drm_syncobj **syncobj;
struct drm_exec exec;
+ void __user *ptr;
int r, i, entry;
u64 wptr;
@@ -470,13 +470,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
return -EINVAL;
num_syncobj_handles = args->num_syncobj_handles;
- syncobj_handles = memdup_array_user(u64_to_user_ptr(args->syncobj_handles),
- num_syncobj_handles, sizeof(u32));
+ ptr = u64_to_user_ptr(args->syncobj_handles);
+ syncobj_handles = memdup_array_user(ptr, num_syncobj_handles,
+ sizeof(u32));
if (IS_ERR(syncobj_handles))
return PTR_ERR(syncobj_handles);
- /* Array of pointers to the looked up syncobjs */
- syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj), GFP_KERNEL);
+ syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj),
+ GFP_KERNEL);
if (!syncobj) {
r = -ENOMEM;
goto free_syncobj_handles;
@@ -490,21 +491,17 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
}
}
- r = drm_gem_objects_lookup(filp,
- u64_to_user_ptr(args->bo_read_handles),
- num_read_bo_handles,
- &gobj_read);
+ ptr = u64_to_user_ptr(args->bo_read_handles);
+ r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
if (r)
goto free_syncobj;
- r = drm_gem_objects_lookup(filp,
- u64_to_user_ptr(args->bo_write_handles),
- num_write_bo_handles,
+ ptr = u64_to_user_ptr(args->bo_write_handles);
+ r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
&gobj_write);
if (r)
goto put_gobj_read;
- /* Retrieve the user queue */
queue = amdgpu_userq_get(userq_mgr, args->queue_id);
if (!queue) {
r = -ENOENT;
@@ -513,73 +510,61 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
r = amdgpu_userq_fence_read_wptr(adev, queue, &wptr);
if (r)
- goto put_gobj_write;
+ goto put_queue;
- r = amdgpu_userq_fence_alloc(&userq_fence);
+ r = amdgpu_userq_fence_alloc(queue, &fence);
if (r)
- goto put_gobj_write;
+ goto put_queue;
/* We are here means UQ is active, make sure the eviction fence is valid */
amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
- /* Create a new fence */
- r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
- if (r) {
- mutex_unlock(&userq_mgr->userq_mutex);
- kfree(userq_fence);
- goto put_gobj_write;
- }
+ /* Create the new fence */
+ amdgpu_userq_fence_init(queue, fence, wptr);
- dma_fence_put(queue->last_fence);
- queue->last_fence = dma_fence_get(fence);
- amdgpu_userq_start_hang_detect_work(queue);
mutex_unlock(&userq_mgr->userq_mutex);
+ /*
+ * This needs to come after the fence is created since
+ * amdgpu_userq_ensure_ev_fence() can't be called while holding the resv
+ * locks.
+ */
drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
(num_read_bo_handles + num_write_bo_handles));
- /* Lock all BOs with retry handling */
drm_exec_until_all_locked(&exec) {
- r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1);
+ r = drm_exec_prepare_array(&exec, gobj_read,
+ num_read_bo_handles, 1);
drm_exec_retry_on_contention(&exec);
- if (r) {
- amdgpu_userq_fence_cleanup(fence);
+ if (r)
goto exec_fini;
- }
- r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1);
+ r = drm_exec_prepare_array(&exec, gobj_write,
+ num_write_bo_handles, 1);
drm_exec_retry_on_contention(&exec);
- if (r) {
- amdgpu_userq_fence_cleanup(fence);
+ if (r)
goto exec_fini;
- }
}
- for (i = 0; i < num_read_bo_handles; i++) {
- if (!gobj_read || !gobj_read[i]->resv)
- continue;
-
- dma_resv_add_fence(gobj_read[i]->resv, fence,
+ /* And publish the new fence in the BOs and syncobj */
+ for (i = 0; i < num_read_bo_handles; i++)
+ dma_resv_add_fence(gobj_read[i]->resv, &fence->base,
DMA_RESV_USAGE_READ);
- }
- for (i = 0; i < num_write_bo_handles; i++) {
- if (!gobj_write || !gobj_write[i]->resv)
- continue;
-
- dma_resv_add_fence(gobj_write[i]->resv, fence,
+ for (i = 0; i < num_write_bo_handles; i++)
+ dma_resv_add_fence(gobj_write[i]->resv, &fence->base,
DMA_RESV_USAGE_WRITE);
- }
- /* Add the created fence to syncobj/BO's */
for (i = 0; i < num_syncobj_handles; i++)
- drm_syncobj_replace_fence(syncobj[i], fence);
+ drm_syncobj_replace_fence(syncobj[i], &fence->base);
+exec_fini:
/* drop the reference acquired in fence creation function */
- dma_fence_put(fence);
+ dma_fence_put(&fence->base);
-exec_fini:
drm_exec_fini(&exec);
+put_queue:
+ amdgpu_userq_put(queue);
put_gobj_write:
for (i = 0; i < num_write_bo_handles; i++)
drm_gem_object_put(gobj_write[i]);
@@ -590,15 +575,11 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
kvfree(gobj_read);
free_syncobj:
while (entry-- > 0)
- if (syncobj[entry])
- drm_syncobj_put(syncobj[entry]);
+ drm_syncobj_put(syncobj[entry]);
kfree(syncobj);
free_syncobj_handles:
kfree(syncobj_handles);
- if (queue)
- amdgpu_userq_put(queue);
-
return r;
}
@@ -873,8 +854,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
* Otherwise, we would gather those references until we don't
* have any more space left and crash.
*/
+ mutex_lock(&waitq->fence_drv_lock);
r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
xa_limit_32b, GFP_KERNEL);
+ mutex_unlock(&waitq->fence_drv_lock);
if (r)
goto put_waitq;
--
2.43.0
^ permalink raw reply related [flat|nested] 30+ messages in thread
* [PATCH 4/8] drm/amdgpu: remove almost all calls to amdgpu_userq_detect_and_reset_queues
2026-04-27 18:27 [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab Christian König
2026-04-27 18:27 ` [PATCH 2/8] drm/amdgpu: remove deadlocks from amdgpu_userq_pre_reset Christian König
2026-04-27 18:27 ` [PATCH 3/8] drm/amdgpu: rework amdgpu_userq_signal_ioctl v2 Christian König
@ 2026-04-27 18:27 ` Christian König
2026-04-28 9:37 ` Khatri, Sunil
2026-04-29 3:16 ` Liang, Prike
2026-04-27 18:27 ` [PATCH 5/8] drm/amdgpu: fix userq hang detection and reset Christian König
` (5 subsequent siblings)
8 siblings, 2 replies; 30+ messages in thread
From: Christian König @ 2026-04-27 18:27 UTC (permalink / raw)
To: alexander.deucher, Prike.Liang, sukhatri, jesse.zhang; +Cc: amd-gfx
Well the reset handling seems broken on multiple levels.
As first step of fixing this remove most calls to the hang detection.
That function should only be called after we run into a timeout! And *NOT*
as random check spread over the code in multiple places.
Signed-off-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 38 +++++++++--------------
1 file changed, 14 insertions(+), 24 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 62d155d6e4ed..fb3c0425cf22 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -345,23 +345,18 @@ static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue)
struct amdgpu_device *adev = uq_mgr->adev;
const struct amdgpu_userq_funcs *userq_funcs =
adev->userq_funcs[queue->queue_type];
- bool found_hung_queue = false;
- int r = 0;
+ int r;
if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
r = userq_funcs->preempt(queue);
if (r) {
queue->state = AMDGPU_USERQ_STATE_HUNG;
- found_hung_queue = true;
+ return r;
} else {
queue->state = AMDGPU_USERQ_STATE_PREEMPTED;
}
}
-
- if (found_hung_queue)
- amdgpu_userq_detect_and_reset_queues(uq_mgr);
-
- return r;
+ return 0;
}
static int amdgpu_userq_restore_helper(struct amdgpu_usermode_queue *queue)
@@ -390,24 +385,21 @@ static int amdgpu_userq_unmap_helper(struct amdgpu_usermode_queue *queue)
struct amdgpu_device *adev = uq_mgr->adev;
const struct amdgpu_userq_funcs *userq_funcs =
adev->userq_funcs[queue->queue_type];
- bool found_hung_queue = false;
- int r = 0;
+ int r;
if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) ||
- (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {
+ (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {
+
r = userq_funcs->unmap(queue);
if (r) {
queue->state = AMDGPU_USERQ_STATE_HUNG;
- found_hung_queue = true;
+ return r;
} else {
queue->state = AMDGPU_USERQ_STATE_UNMAPPED;
}
}
- if (found_hung_queue)
- amdgpu_userq_detect_and_reset_queues(uq_mgr);
-
- return r;
+ return 0;
}
static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue)
@@ -416,19 +408,19 @@ static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue)
struct amdgpu_device *adev = uq_mgr->adev;
const struct amdgpu_userq_funcs *userq_funcs =
adev->userq_funcs[queue->queue_type];
- int r = 0;
+ int r;
if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) {
r = userq_funcs->map(queue);
if (r) {
queue->state = AMDGPU_USERQ_STATE_HUNG;
- amdgpu_userq_detect_and_reset_queues(uq_mgr);
+ return r;
} else {
queue->state = AMDGPU_USERQ_STATE_MAPPED;
}
}
- return r;
+ return 0;
}
static void amdgpu_userq_wait_for_last_fence(struct amdgpu_usermode_queue *queue)
@@ -654,7 +646,6 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_que
#if defined(CONFIG_DEBUG_FS)
debugfs_remove_recursive(queue->debugfs_queue);
#endif
- amdgpu_userq_detect_and_reset_queues(uq_mgr);
r = amdgpu_userq_unmap_helper(queue);
/*TODO: It requires a reset for userq hw unmap error*/
if (r) {
@@ -1270,7 +1261,6 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
unsigned long queue_id;
int ret = 0, r;
- amdgpu_userq_detect_and_reset_queues(uq_mgr);
/* Try to unmap all the queues in this process ctx */
xa_for_each(&uq_mgr->userq_xa, queue_id, queue) {
r = amdgpu_userq_preempt_helper(queue);
@@ -1278,9 +1268,11 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
ret = r;
}
- if (ret)
+ if (ret) {
drm_file_err(uq_mgr->file,
"Couldn't unmap all the queues, eviction failed ret=%d\n", ret);
+ amdgpu_userq_detect_and_reset_queues(uq_mgr);
+ }
return ret;
}
@@ -1380,7 +1372,6 @@ int amdgpu_userq_suspend(struct amdgpu_device *adev)
uqm = queue->userq_mgr;
cancel_delayed_work_sync(&uqm->resume_work);
guard(mutex)(&uqm->userq_mutex);
- amdgpu_userq_detect_and_reset_queues(uqm);
if (adev->in_s0ix)
r = amdgpu_userq_preempt_helper(queue);
else
@@ -1439,7 +1430,6 @@ int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev,
if (((queue->queue_type == AMDGPU_HW_IP_GFX) ||
(queue->queue_type == AMDGPU_HW_IP_COMPUTE)) &&
(queue->xcp_id == idx)) {
- amdgpu_userq_detect_and_reset_queues(uqm);
r = amdgpu_userq_preempt_helper(queue);
if (r)
ret = r;
--
2.43.0
^ permalink raw reply related [flat|nested] 30+ messages in thread
* [PATCH 5/8] drm/amdgpu: fix userq hang detection and reset
2026-04-27 18:27 [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab Christian König
` (2 preceding siblings ...)
2026-04-27 18:27 ` [PATCH 4/8] drm/amdgpu: remove almost all calls to amdgpu_userq_detect_and_reset_queues Christian König
@ 2026-04-27 18:27 ` Christian König
2026-04-29 8:30 ` Liang, Prike
2026-04-27 18:27 ` [PATCH 6/8] drm/amdgpu: rework userq reset work handling Christian König
` (4 subsequent siblings)
8 siblings, 1 reply; 30+ messages in thread
From: Christian König @ 2026-04-27 18:27 UTC (permalink / raw)
To: alexander.deucher, Prike.Liang, sukhatri, jesse.zhang; +Cc: amd-gfx
Fix lock inversions pointed out by Prike and Sunil. The hang detection
timeout *CAN'T* grab locks under which we wait for fences, especially
not the userq_mutex lock.
Then instead of this completely broken handling with the
hang_detect_fence just cancel the work when fences are processed and
re-start if necessary.
Signed-off-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 65 ++++++++-----------
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 1 -
.../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c | 17 +++--
.../gpu/drm/amd/amdgpu/amdgpu_userq_fence.h | 2 +-
4 files changed, 40 insertions(+), 45 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index fb3c0425cf22..b2c64a05d03a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -106,9 +106,6 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
int r = 0;
int i;
- /* Warning if current process mutex is not held */
- WARN_ON(!mutex_is_locked(&uq_mgr->userq_mutex));
-
if (unlikely(adev->debug_disable_gpu_ring_reset)) {
dev_err(adev->dev, "userq reset disabled by debug mask\n");
return 0;
@@ -127,9 +124,11 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
*/
for (i = 0; i < num_queue_types; i++) {
int ring_type = queue_types[i];
- const struct amdgpu_userq_funcs *funcs = adev->userq_funcs[ring_type];
+ const struct amdgpu_userq_funcs *funcs =
+ adev->userq_funcs[ring_type];
- if (!amdgpu_userq_is_reset_type_supported(adev, ring_type, AMDGPU_RESET_TYPE_PER_QUEUE))
+ if (!amdgpu_userq_is_reset_type_supported(adev, ring_type,
+ AMDGPU_RESET_TYPE_PER_QUEUE))
continue;
if (atomic_read(&uq_mgr->userq_count[ring_type]) > 0 &&
@@ -150,38 +149,22 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
static void amdgpu_userq_hang_detect_work(struct work_struct *work)
{
- struct amdgpu_usermode_queue *queue = container_of(work,
- struct amdgpu_usermode_queue,
- hang_detect_work.work);
- struct dma_fence *fence;
- struct amdgpu_userq_mgr *uq_mgr;
-
- if (!queue->userq_mgr)
- return;
-
- uq_mgr = queue->userq_mgr;
- fence = READ_ONCE(queue->hang_detect_fence);
- /* Fence already signaled – no action needed */
- if (!fence || dma_fence_is_signaled(fence))
- return;
+ struct amdgpu_usermode_queue *queue =
+ container_of(work, struct amdgpu_usermode_queue,
+ hang_detect_work.work);
- mutex_lock(&uq_mgr->userq_mutex);
- amdgpu_userq_detect_and_reset_queues(uq_mgr);
- mutex_unlock(&uq_mgr->userq_mutex);
+ amdgpu_userq_detect_and_reset_queues(queue->userq_mgr);
}
/*
* Start hang detection for a user queue fence. A delayed work will be scheduled
- * to check if the fence is still pending after the timeout period.
-*/
+ * to reset the queues when the fence doesn't signal in time.
+ */
void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
{
struct amdgpu_device *adev;
unsigned long timeout_ms;
- if (!queue || !queue->userq_mgr || !queue->userq_mgr->adev)
- return;
-
adev = queue->userq_mgr->adev;
/* Determine timeout based on queue type */
switch (queue->queue_type) {
@@ -199,8 +182,6 @@ void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
break;
}
- /* Store the fence to monitor and schedule hang detection */
- WRITE_ONCE(queue->hang_detect_fence, queue->last_fence);
schedule_delayed_work(&queue->hang_detect_work,
msecs_to_jiffies(timeout_ms));
}
@@ -210,18 +191,24 @@ void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell)
struct xarray *xa = &adev->userq_doorbell_xa;
struct amdgpu_usermode_queue *queue;
unsigned long flags;
+ int r;
xa_lock_irqsave(xa, flags);
queue = xa_load(xa, doorbell);
- if (queue)
- amdgpu_userq_fence_driver_process(queue->fence_drv);
- xa_unlock_irqrestore(xa, flags);
-}
+ if (queue) {
+ r = amdgpu_userq_fence_driver_process(queue->fence_drv);
+ /*
+ * We are in interrupt context here, this *can't* wait for
+ * reset work to finish.
+ */
+ if (r >= 0)
+ cancel_delayed_work(&queue->hang_detect_work);
-static void amdgpu_userq_init_hang_detect_work(struct amdgpu_usermode_queue *queue)
-{
- INIT_DELAYED_WORK(&queue->hang_detect_work, amdgpu_userq_hang_detect_work);
- queue->hang_detect_fence = NULL;
+ /* Restart the timer when there are still fences pending */
+ if (r == 1)
+ amdgpu_userq_start_hang_detect_work(queue);
+ }
+ xa_unlock_irqrestore(xa, flags);
}
static int amdgpu_userq_buffer_va_list_add(struct amdgpu_usermode_queue *queue,
@@ -640,7 +627,6 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_que
amdgpu_bo_unreserve(vm->root.bo);
mutex_lock(&uq_mgr->userq_mutex);
- queue->hang_detect_fence = NULL;
amdgpu_userq_wait_for_last_fence(queue);
#if defined(CONFIG_DEBUG_FS)
@@ -853,7 +839,8 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
up_read(&adev->reset_domain->sem);
amdgpu_debugfs_userq_init(filp, queue, qid);
- amdgpu_userq_init_hang_detect_work(queue);
+ INIT_DELAYED_WORK(&queue->hang_detect_work,
+ amdgpu_userq_hang_detect_work);
args->out.queue_id = qid;
atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index 843ea8ecc5d7..85f460e7c31b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -85,7 +85,6 @@ struct amdgpu_usermode_queue {
int priority;
struct dentry *debugfs_queue;
struct delayed_work hang_detect_work;
- struct dma_fence *hang_detect_fence;
struct kref refcount;
struct list_head userq_va_list;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
index a2792f05b670..bf452c5aacbe 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
@@ -135,7 +135,14 @@ amdgpu_userq_fence_put_fence_drv_array(struct amdgpu_userq_fence *userq_fence)
userq_fence->fence_drv_array_count = 0;
}
-void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
+/*
+ * Returns:
+ * -ENOENT when no fences were processes
+ * 1 when more fences are pending
+ * 0 when no fences are pending any more
+ */
+int
+amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
{
struct amdgpu_userq_fence *userq_fence, *tmp;
LIST_HEAD(to_be_signaled);
@@ -143,9 +150,6 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
unsigned long flags;
u64 rptr;
- if (!fence_drv)
- return;
-
spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
rptr = amdgpu_userq_fence_read(fence_drv);
@@ -158,6 +162,9 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
&userq_fence->link);
spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
+ if (list_empty(&to_be_signaled))
+ return -ENOENT;
+
list_for_each_entry_safe(userq_fence, tmp, &to_be_signaled, link) {
fence = &userq_fence->base;
list_del_init(&userq_fence->link);
@@ -169,6 +176,8 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
dma_fence_put(fence);
}
+ /* That doesn't need to be accurate so no locking */
+ return list_empty(&fence_drv->fences) ? 0 : 1;
}
void amdgpu_userq_fence_driver_destroy(struct kref *ref)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
index d355a0eecc07..0bd51616cef1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
@@ -63,7 +63,7 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv);
int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,
struct amdgpu_userq_fence_driver **fence_drv_req);
void amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq);
-void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv);
+int amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv);
void amdgpu_userq_fence_driver_force_completion(struct amdgpu_usermode_queue *userq);
void amdgpu_userq_fence_driver_destroy(struct kref *ref);
int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
--
2.43.0
^ permalink raw reply related [flat|nested] 30+ messages in thread
* [PATCH 6/8] drm/amdgpu: rework userq reset work handling
2026-04-27 18:27 [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab Christian König
` (3 preceding siblings ...)
2026-04-27 18:27 ` [PATCH 5/8] drm/amdgpu: fix userq hang detection and reset Christian König
@ 2026-04-27 18:27 ` Christian König
2026-04-30 6:37 ` Liang, Prike
2026-04-27 18:27 ` [PATCH 7/8] drm/amdgpu: fix handling in amdgpu_userq_create Christian König
` (3 subsequent siblings)
8 siblings, 1 reply; 30+ messages in thread
From: Christian König @ 2026-04-27 18:27 UTC (permalink / raw)
To: alexander.deucher, Prike.Liang, sukhatri, jesse.zhang; +Cc: amd-gfx
It is illegal to schedule reset work from another reset work!
Fix this by scheduling the userq reset work directly on the work queue
of the reset domain.
Not fully tested, I leave that to the IGT test cases.
Signed-off-by: Christian König <christian.koenig@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 -
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 3 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 84 +++++++++++-----------
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 16 ++++-
4 files changed, 60 insertions(+), 44 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 39894e38fee4..17341e384caf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -1191,7 +1191,6 @@ struct amdgpu_device {
bool apu_prefer_gtt;
bool userq_halt_for_enforce_isolation;
- struct work_struct userq_reset_work;
struct amdgpu_uid *uid_info;
struct amdgpu_uma_carveout_info uma_info;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 4bb8c34d2dd9..c400b4c5dda3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -3790,7 +3790,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
}
INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
- INIT_WORK(&adev->userq_reset_work, amdgpu_userq_reset_work);
amdgpu_coredump_init(adev);
@@ -5481,7 +5480,7 @@ static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
if (!amdgpu_sriov_vf(adev))
cancel_work(&adev->reset_work);
#endif
- cancel_work(&adev->userq_reset_work);
+ amdgpu_userq_mgr_cancel_reset_work(adev);
if (adev->kfd.dev)
cancel_work(&adev->kfd.reset_work);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index b2c64a05d03a..0374a70b631c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -82,19 +82,11 @@ static bool amdgpu_userq_is_reset_type_supported(struct amdgpu_device *adev,
return false;
}
-static void amdgpu_userq_gpu_reset(struct amdgpu_device *adev)
-{
- if (amdgpu_device_should_recover_gpu(adev)) {
- amdgpu_reset_domain_schedule(adev->reset_domain,
- &adev->userq_reset_work);
- /* Wait for the reset job to complete */
- flush_work(&adev->userq_reset_work);
- }
-}
-
-static int
-amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
+static void amdgpu_userq_mgr_reset_work(struct work_struct *work)
{
+ struct amdgpu_userq_mgr *uq_mgr =
+ container_of(work, struct amdgpu_userq_mgr,
+ reset_work);
struct amdgpu_device *adev = uq_mgr->adev;
const int queue_types[] = {
AMDGPU_RING_TYPE_COMPUTE,
@@ -103,12 +95,11 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
};
const int num_queue_types = ARRAY_SIZE(queue_types);
bool gpu_reset = false;
- int r = 0;
- int i;
+ int i, r;
if (unlikely(adev->debug_disable_gpu_ring_reset)) {
dev_err(adev->dev, "userq reset disabled by debug mask\n");
- return 0;
+ return;
}
/*
@@ -116,7 +107,7 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
* skip all reset detection logic
*/
if (!amdgpu_gpu_recovery)
- return 0;
+ return;
/*
* Iterate through all queue types to detect and reset problematic queues
@@ -141,10 +132,19 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
}
}
- if (gpu_reset)
- amdgpu_userq_gpu_reset(adev);
+ if (gpu_reset) {
+ struct amdgpu_reset_context reset_context;
- return r;
+ memset(&reset_context, 0, sizeof(reset_context));
+
+ reset_context.method = AMD_RESET_METHOD_NONE;
+ reset_context.reset_req_dev = adev;
+ reset_context.src = AMDGPU_RESET_SRC_USERQ;
+ set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
+ /*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/
+
+ amdgpu_device_gpu_recover(adev, NULL, &reset_context);
+ }
}
static void amdgpu_userq_hang_detect_work(struct work_struct *work)
@@ -153,7 +153,11 @@ static void amdgpu_userq_hang_detect_work(struct work_struct *work)
container_of(work, struct amdgpu_usermode_queue,
hang_detect_work.work);
- amdgpu_userq_detect_and_reset_queues(queue->userq_mgr);
+ /*
+ * Don't schedule the work here! Scheduling or queue work from one reset
+ * handler to another is illegal if you don't take extra precautions!
+ */
+ amdgpu_userq_mgr_reset_work(&queue->userq_mgr->reset_work);
}
/*
@@ -182,8 +186,8 @@ void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
break;
}
- schedule_delayed_work(&queue->hang_detect_work,
- msecs_to_jiffies(timeout_ms));
+ queue_delayed_work(adev->reset_domain->wq, &queue->hang_detect_work,
+ msecs_to_jiffies(timeout_ms));
}
void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell)
@@ -1258,28 +1262,13 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
if (ret) {
drm_file_err(uq_mgr->file,
"Couldn't unmap all the queues, eviction failed ret=%d\n", ret);
- amdgpu_userq_detect_and_reset_queues(uq_mgr);
+ amdgpu_reset_domain_schedule(uq_mgr->adev->reset_domain,
+ &uq_mgr->reset_work);
+ flush_work(&uq_mgr->reset_work);
}
return ret;
}
-void amdgpu_userq_reset_work(struct work_struct *work)
-{
- struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
- userq_reset_work);
- struct amdgpu_reset_context reset_context;
-
- memset(&reset_context, 0, sizeof(reset_context));
-
- reset_context.method = AMD_RESET_METHOD_NONE;
- reset_context.reset_req_dev = adev;
- reset_context.src = AMDGPU_RESET_SRC_USERQ;
- set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
- /*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/
-
- amdgpu_device_gpu_recover(adev, NULL, &reset_context);
-}
-
static void
amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr)
{
@@ -1313,9 +1302,24 @@ int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *f
userq_mgr->file = file_priv;
INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userq_restore_worker);
+ INIT_WORK(&userq_mgr->reset_work, amdgpu_userq_mgr_reset_work);
return 0;
}
+void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev)
+{
+ struct xarray *xa = &adev->userq_doorbell_xa;
+ struct amdgpu_usermode_queue *queue;
+ unsigned long flags, queue_id;
+
+ xa_lock_irqsave(xa, flags);
+ xa_for_each(xa, queue_id, queue) {
+ cancel_delayed_work(&queue->hang_detect_work);
+ cancel_work(&queue->userq_mgr->reset_work);
+ }
+ xa_unlock_irqrestore(xa, flags);
+}
+
void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr)
{
cancel_delayed_work_sync(&userq_mgr->resume_work);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index 85f460e7c31b..49b33e2d6932 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -84,7 +84,13 @@ struct amdgpu_usermode_queue {
u32 xcp_id;
int priority;
struct dentry *debugfs_queue;
- struct delayed_work hang_detect_work;
+
+ /**
+ * @hang_detect_work:
+ *
+ * Delayed work which runs when userq_fences time out.
+ */
+ struct delayed_work hang_detect_work;
struct kref refcount;
struct list_head userq_va_list;
@@ -116,6 +122,13 @@ struct amdgpu_userq_mgr {
struct amdgpu_device *adev;
struct delayed_work resume_work;
struct drm_file *file;
+
+ /**
+ * @reset_work:
+ *
+ * Reset work which is used when eviction fails.
+ */
+ struct work_struct reset_work;
atomic_t userq_count[AMDGPU_RING_TYPE_MAX];
};
@@ -134,6 +147,7 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp
int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,
struct amdgpu_device *adev);
+void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev);
void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr);
void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr);
--
2.43.0
^ permalink raw reply related [flat|nested] 30+ messages in thread
* [PATCH 7/8] drm/amdgpu: fix handling in amdgpu_userq_create
2026-04-27 18:27 [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab Christian König
` (4 preceding siblings ...)
2026-04-27 18:27 ` [PATCH 6/8] drm/amdgpu: rework userq reset work handling Christian König
@ 2026-04-27 18:27 ` Christian König
2026-04-28 11:26 ` Khatri, Sunil
2026-05-07 6:13 ` Liang, Prike
2026-04-27 18:27 ` [PATCH 8/8] drm/amdgpu: restructure VM state machine v2 Christian König
` (2 subsequent siblings)
8 siblings, 2 replies; 30+ messages in thread
From: Christian König @ 2026-04-27 18:27 UTC (permalink / raw)
To: alexander.deucher, Prike.Liang, sukhatri, jesse.zhang; +Cc: amd-gfx
Well mostly the same issues the other code had as well:
1. Memory allocation while holding the userq_mutex lock is forbidden!
2. Things were created/started/published in the wrong order.
3. Error messages on invalid input parameters can spam the logs.
4. Error messages on memory allocation failures are usually superflous
as well.
Signed-off-by: Christian König <christian.koenig@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 118 ++++++++++------------
1 file changed, 52 insertions(+), 66 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 0374a70b631c..28e0695b53e7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -718,14 +718,14 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
const struct amdgpu_userq_funcs *uq_funcs;
struct amdgpu_usermode_queue *queue;
struct amdgpu_db_info db_info;
- bool skip_map_queue;
- u32 qid;
uint64_t index;
- int r = 0;
- int priority =
- (args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK) >>
- AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT;
+ int priority;
+ u32 qid;
+ int r;
+ priority =
+ (args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK)
+ >> AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT;
r = amdgpu_userq_priority_permit(filp, priority);
if (r)
return r;
@@ -738,40 +738,43 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
uq_funcs = adev->userq_funcs[args->in.ip_type];
if (!uq_funcs) {
- drm_file_err(uq_mgr->file, "Usermode queue is not supported for this IP (%u)\n",
- args->in.ip_type);
r = -EINVAL;
goto err_pm_runtime;
}
queue = kzalloc(sizeof(struct amdgpu_usermode_queue), GFP_KERNEL);
if (!queue) {
- drm_file_err(uq_mgr->file, "Failed to allocate memory for queue\n");
r = -ENOMEM;
goto err_pm_runtime;
}
+ kref_init(&queue->refcount);
INIT_LIST_HEAD(&queue->userq_va_list);
queue->doorbell_handle = args->in.doorbell_handle;
queue->queue_type = args->in.ip_type;
queue->vm = &fpriv->vm;
queue->priority = priority;
-
- db_info.queue_type = queue->queue_type;
- db_info.doorbell_handle = queue->doorbell_handle;
- db_info.db_obj = &queue->db_obj;
- db_info.doorbell_offset = args->in.doorbell_offset;
-
queue->userq_mgr = uq_mgr;
+ INIT_DELAYED_WORK(&queue->hang_detect_work,
+ amdgpu_userq_hang_detect_work);
- /* Validate the userq virtual address.*/
- r = amdgpu_bo_reserve(fpriv->vm.root.bo, false);
+ mutex_init(&queue->fence_drv_lock);
+ xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
+ r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
if (r)
goto free_queue;
- if (amdgpu_userq_input_va_validate(adev, queue, args->in.queue_va, args->in.queue_size) ||
- amdgpu_userq_input_va_validate(adev, queue, args->in.rptr_va, AMDGPU_GPU_PAGE_SIZE) ||
- amdgpu_userq_input_va_validate(adev, queue, args->in.wptr_va, AMDGPU_GPU_PAGE_SIZE)) {
+ /* Make sure the queue can actually run with those virtual addresses. */
+ r = amdgpu_bo_reserve(fpriv->vm.root.bo, false);
+ if (r)
+ goto free_fence_drv;
+
+ if (amdgpu_userq_input_va_validate(adev, queue, args->in.queue_va,
+ args->in.queue_size) ||
+ amdgpu_userq_input_va_validate(adev, queue, args->in.rptr_va,
+ AMDGPU_GPU_PAGE_SIZE) ||
+ amdgpu_userq_input_va_validate(adev, queue, args->in.wptr_va,
+ AMDGPU_GPU_PAGE_SIZE)) {
r = -EINVAL;
amdgpu_bo_unreserve(fpriv->vm.root.bo);
goto clean_mapping;
@@ -779,6 +782,10 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
amdgpu_bo_unreserve(fpriv->vm.root.bo);
/* Convert relative doorbell offset into absolute doorbell index */
+ db_info.queue_type = queue->queue_type;
+ db_info.doorbell_handle = queue->doorbell_handle;
+ db_info.db_obj = &queue->db_obj;
+ db_info.doorbell_offset = args->in.doorbell_offset;
index = amdgpu_userq_get_doorbell_index(uq_mgr, &db_info, filp);
if (index == (uint64_t)-EINVAL) {
drm_file_err(uq_mgr->file, "Failed to get doorbell for queue\n");
@@ -787,82 +794,61 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
}
queue->doorbell_index = index;
- mutex_init(&queue->fence_drv_lock);
- xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
- r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
- if (r) {
- drm_file_err(uq_mgr->file, "Failed to alloc fence driver\n");
- goto clean_mapping;
- }
-
r = uq_funcs->mqd_create(queue, &args->in);
if (r) {
drm_file_err(uq_mgr->file, "Failed to create Queue\n");
- goto clean_fence_driver;
+ goto clean_mapping;
}
+ r = xa_err(xa_store_irq(&adev->userq_doorbell_xa, index, queue,
+ GFP_KERNEL));
+ if (r)
+ goto clean_mqd;
+
amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
/* don't map the queue if scheduling is halted */
- if (adev->userq_halt_for_enforce_isolation &&
- ((queue->queue_type == AMDGPU_HW_IP_GFX) ||
- (queue->queue_type == AMDGPU_HW_IP_COMPUTE)))
- skip_map_queue = true;
- else
- skip_map_queue = false;
- if (!skip_map_queue) {
+ if (!adev->userq_halt_for_enforce_isolation ||
+ ((queue->queue_type != AMDGPU_HW_IP_GFX) &&
+ (queue->queue_type != AMDGPU_HW_IP_COMPUTE))) {
r = amdgpu_userq_map_helper(queue);
if (r) {
drm_file_err(uq_mgr->file, "Failed to map Queue\n");
- goto clean_mqd;
+ mutex_unlock(&uq_mgr->userq_mutex);
+ goto clean_doorbell;
}
}
- /* drop this refcount during queue destroy */
- kref_init(&queue->refcount);
-
- /* Wait for mode-1 reset to complete */
- down_read(&adev->reset_domain->sem);
+ atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
+ mutex_unlock(&uq_mgr->userq_mutex);
r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
- XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT), GFP_KERNEL);
- if (r) {
- if (!skip_map_queue)
- amdgpu_userq_unmap_helper(queue);
- r = -ENOMEM;
- goto clean_reset_domain;
- }
-
- r = xa_err(xa_store_irq(&adev->userq_doorbell_xa, index, queue, GFP_KERNEL));
+ XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
+ GFP_KERNEL);
if (r) {
- xa_erase(&uq_mgr->userq_xa, qid);
- if (!skip_map_queue)
- amdgpu_userq_unmap_helper(queue);
- goto clean_reset_domain;
+ /*
+ * This drops the last reference which should take care of
+ * all cleanup.
+ */
+ amdgpu_userq_put(queue);
+ return r;
}
- up_read(&adev->reset_domain->sem);
amdgpu_debugfs_userq_init(filp, queue, qid);
- INIT_DELAYED_WORK(&queue->hang_detect_work,
- amdgpu_userq_hang_detect_work);
-
args->out.queue_id = qid;
- atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
- mutex_unlock(&uq_mgr->userq_mutex);
return 0;
-clean_reset_domain:
- up_read(&adev->reset_domain->sem);
+clean_doorbell:
+ xa_erase_irq(&adev->userq_doorbell_xa, index);
clean_mqd:
- mutex_unlock(&uq_mgr->userq_mutex);
uq_funcs->mqd_destroy(queue);
-clean_fence_driver:
- amdgpu_userq_fence_driver_free(queue);
clean_mapping:
amdgpu_bo_reserve(fpriv->vm.root.bo, true);
amdgpu_userq_buffer_vas_list_cleanup(adev, queue);
amdgpu_bo_unreserve(fpriv->vm.root.bo);
mutex_destroy(&queue->fence_drv_lock);
+free_fence_drv:
+ amdgpu_userq_fence_driver_free(queue);
free_queue:
kfree(queue);
err_pm_runtime:
--
2.43.0
^ permalink raw reply related [flat|nested] 30+ messages in thread
* [PATCH 8/8] drm/amdgpu: restructure VM state machine v2
2026-04-27 18:27 [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab Christian König
` (5 preceding siblings ...)
2026-04-27 18:27 ` [PATCH 7/8] drm/amdgpu: fix handling in amdgpu_userq_create Christian König
@ 2026-04-27 18:27 ` Christian König
2026-04-28 12:20 ` Khatri, Sunil
` (2 more replies)
2026-04-28 6:15 ` [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab Liang, Prike
2026-04-28 7:57 ` Khatri, Sunil
8 siblings, 3 replies; 30+ messages in thread
From: Christian König @ 2026-04-27 18:27 UTC (permalink / raw)
To: alexander.deucher, Prike.Liang, sukhatri, jesse.zhang; +Cc: amd-gfx
Instead of coming up with more sophisticated names for states a VM BO
can be in, group them by the type of BO first and then by the state.
So we end with BO type kernel, always_valid and individual and then states
evicted, moved and idle.
Not much functional change, except that evicted_user is moved back
together with the other BOs again which makes the handling in
amdgpu_vm_validate() a bit more complex.
Also fixes a problem with user queues and amdgpu_vm_ready(). We didn't
considered the VM ready when user BOs were not ideally placed, harmless
performance impact for kernel queues but a complete show stopper for
userqueues.
v2: fix a few typos in comments, rename the BO types to make them more
descriptive, fix a couple of bugs found during testing
v3: squashed together with revert to old status lock handling, looks
like the first patch still had some bug which this one here should fix.
Fix a missing lock around debugfs printing.
Signed-off-by: Christian König <christian.koenig@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 65 +--
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 534 +++++++++-------------
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 69 ++-
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 4 -
4 files changed, 298 insertions(+), 374 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 28e0695b53e7..07e30acbdc3a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -805,41 +805,50 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
if (r)
goto clean_mqd;
- amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
+ /* Make sure a concurrent close operation doesn't kill our userq */
+ kref_get(&queue->refcount);
+ r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
+ XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
+ GFP_KERNEL);
+ if (r) {
+ /*
+ * This drops the extra and last reference which should take
+ * care of all cleanup.
+ */
+ amdgpu_userq_put(queue);
+ amdgpu_userq_put(queue);
+ return r;
+ }
/* don't map the queue if scheduling is halted */
+ /* TODO: That is racy! */
if (!adev->userq_halt_for_enforce_isolation ||
((queue->queue_type != AMDGPU_HW_IP_GFX) &&
(queue->queue_type != AMDGPU_HW_IP_COMPUTE))) {
+
+ /* TODO: We could skip mapping if ev fence is signaled */
+ amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr,
+ &fpriv->evf_mgr);
r = amdgpu_userq_map_helper(queue);
if (r) {
drm_file_err(uq_mgr->file, "Failed to map Queue\n");
mutex_unlock(&uq_mgr->userq_mutex);
- goto clean_doorbell;
+ /* Prevent racing with close */
+ if (xa_erase(&uq_mgr->userq_xa, qid) == queue)
+ amdgpu_userq_put(queue);
+ amdgpu_userq_put(queue);
+ return r;
}
+ mutex_unlock(&uq_mgr->userq_mutex);
}
atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
- mutex_unlock(&uq_mgr->userq_mutex);
-
- r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
- XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
- GFP_KERNEL);
- if (r) {
- /*
- * This drops the last reference which should take care of
- * all cleanup.
- */
- amdgpu_userq_put(queue);
- return r;
- }
amdgpu_debugfs_userq_init(filp, queue, qid);
+ amdgpu_userq_put(queue);
args->out.queue_id = qid;
return 0;
-clean_doorbell:
- xa_erase_irq(&adev->userq_doorbell_xa, index);
clean_mqd:
uq_funcs->mqd_destroy(queue);
clean_mapping:
@@ -1035,12 +1044,12 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
struct amdgpu_bo *bo;
int ret;
- spin_lock(&vm->status_lock);
- while (!list_empty(&vm->invalidated)) {
- bo_va = list_first_entry(&vm->invalidated,
+ spin_lock(&vm->individual_lock);
+ while (!list_empty(&vm->always_valid.evicted)) {
+ bo_va = list_first_entry(&vm->always_valid.evicted,
struct amdgpu_bo_va,
base.vm_status);
- spin_unlock(&vm->status_lock);
+ spin_unlock(&vm->individual_lock);
bo = bo_va->base.bo;
ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2);
@@ -1052,14 +1061,14 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
if (ret)
return ret;
- /* This moves the bo_va to the done list */
+ /* This moves the bo_va to the idle list */
ret = amdgpu_vm_bo_update(adev, bo_va, false);
if (ret)
return ret;
- spin_lock(&vm->status_lock);
+ spin_lock(&vm->individual_lock);
}
- spin_unlock(&vm->status_lock);
+ spin_unlock(&vm->individual_lock);
return 0;
}
@@ -1091,7 +1100,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
if (unlikely(ret))
goto unlock_all;
- ret = amdgpu_vm_lock_done_list(vm, &exec, 1);
+ ret = amdgpu_vm_lock_individual(vm, &exec, 1);
drm_exec_retry_on_contention(&exec);
if (unlikely(ret))
goto unlock_all;
@@ -1134,7 +1143,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
key = 0;
/* Validate User Ptr BOs */
- list_for_each_entry(bo_va, &vm->done, base.vm_status) {
+ list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status) {
bo = bo_va->base.bo;
if (!bo)
continue;
@@ -1184,10 +1193,10 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
/*
* We need to wait for all VM updates to finish before restarting the
- * queues. Using the done list like that is now ok since everything is
+ * queues. Using the idle list like that is now ok since everything is
* locked in place.
*/
- list_for_each_entry(bo_va, &vm->done, base.vm_status)
+ list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status)
dma_fence_wait(bo_va->last_pt_update, false);
dma_fence_wait(vm->last_update, false);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 6b58b0ecfa06..9f71a146f1f6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -138,6 +138,47 @@ static void amdgpu_vm_assert_locked(struct amdgpu_vm *vm)
dma_resv_assert_held(vm->root.bo->tbo.base.resv);
}
+/* Initialize the amdgpu_vm_bo_status object */
+static void amdgpu_vm_bo_status_init(struct amdgpu_vm_bo_status *lists)
+{
+ INIT_LIST_HEAD(&lists->evicted);
+ INIT_LIST_HEAD(&lists->moved);
+ INIT_LIST_HEAD(&lists->idle);
+}
+
+/*
+ * Make sure we have the lock to modify the vm_bo status and return the object
+ * with the status lists.
+ */
+static struct amdgpu_vm_bo_status *
+amdgpu_vm_bo_lock_lists(struct amdgpu_vm_bo_base *vm_bo)
+{
+ struct amdgpu_vm *vm = vm_bo->vm;
+ struct amdgpu_bo *bo = vm_bo->bo;
+
+ if (amdgpu_vm_is_bo_always_valid(vm, bo)) {
+ /* No extra locking needed, protected by the root PD resv lock */
+ amdgpu_vm_assert_locked(vm);
+
+ if (bo->tbo.type == ttm_bo_type_kernel)
+ return &vm->kernel;
+
+ return &vm->always_valid;
+ }
+
+ spin_lock(&vm_bo->vm->individual_lock);
+ return &vm->individual;
+}
+
+/* Eventually unlock the status list lock again */
+static void amdgpu_vm_bo_unlock_lists(struct amdgpu_vm_bo_base *vm_bo)
+{
+ if (amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
+ amdgpu_vm_assert_locked(vm_bo->vm);
+ else
+ spin_unlock(&vm_bo->vm->individual_lock);
+}
+
/**
* amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid
*
@@ -157,37 +198,44 @@ bool amdgpu_vm_is_bo_always_valid(struct amdgpu_vm *vm, struct amdgpu_bo *bo)
*
* @vm_bo: vm_bo which is evicted
*
- * State for PDs/PTs and per VM BOs which are not at the location they should
- * be.
+ * State for vm_bo objects meaning the underlying BO was evicted and need to
+ * move in place again.
*/
static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
{
- struct amdgpu_vm *vm = vm_bo->vm;
- struct amdgpu_bo *bo = vm_bo->bo;
+ struct amdgpu_vm_bo_status *lists;
+ lists = amdgpu_vm_bo_lock_lists(vm_bo);
vm_bo->moved = true;
- amdgpu_vm_assert_locked(vm);
- spin_lock(&vm_bo->vm->status_lock);
- if (bo->tbo.type == ttm_bo_type_kernel)
- list_move(&vm_bo->vm_status, &vm->evicted);
- else
- list_move_tail(&vm_bo->vm_status, &vm->evicted);
- spin_unlock(&vm_bo->vm->status_lock);
+ list_move(&vm_bo->vm_status, &lists->evicted);
+ amdgpu_vm_bo_unlock_lists(vm_bo);
}
/**
* amdgpu_vm_bo_moved - vm_bo is moved
*
* @vm_bo: vm_bo which is moved
*
- * State for per VM BOs which are moved, but that change is not yet reflected
- * in the page tables.
+ * State for vm_bo objects meaning the underlying BO was moved but the new
+ * location not yet reflected in the page tables.
*/
static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
{
- amdgpu_vm_assert_locked(vm_bo->vm);
- spin_lock(&vm_bo->vm->status_lock);
- list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
- spin_unlock(&vm_bo->vm->status_lock);
+ struct amdgpu_vm_bo_status *lists;
+ struct amdgpu_bo *bo = vm_bo->bo;
+
+ /*
+ * The root PD doesn't have a parent PDE and goes directly into the
+ * idle state.
+ */
+ lists = amdgpu_vm_bo_lock_lists(vm_bo);
+ if (bo && bo->tbo.type == ttm_bo_type_kernel && !bo->parent) {
+ vm_bo->moved = false;
+ list_move(&vm_bo->vm_status, &lists->idle);
+ } else {
+ vm_bo->moved = true;
+ list_move(&vm_bo->vm_status, &lists->moved);
+ }
+ amdgpu_vm_bo_unlock_lists(vm_bo);
}
/**
@@ -195,120 +243,43 @@ static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
*
* @vm_bo: vm_bo which is now idle
*
- * State for PDs/PTs and per VM BOs which have gone through the state machine
- * and are now idle.
+ * State for vm_bo objects meaning we are done with the state machine and no
+ * further action is necessary.
*/
static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
{
- amdgpu_vm_assert_locked(vm_bo->vm);
- spin_lock(&vm_bo->vm->status_lock);
- list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
- spin_unlock(&vm_bo->vm->status_lock);
- vm_bo->moved = false;
-}
+ struct amdgpu_vm_bo_status *lists;
-/**
- * amdgpu_vm_bo_invalidated - vm_bo is invalidated
- *
- * @vm_bo: vm_bo which is now invalidated
- *
- * State for normal BOs which are invalidated and that change not yet reflected
- * in the PTs.
- */
-static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
-{
- spin_lock(&vm_bo->vm->status_lock);
- list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
- spin_unlock(&vm_bo->vm->status_lock);
-}
-
-/**
- * amdgpu_vm_bo_evicted_user - vm_bo is evicted
- *
- * @vm_bo: vm_bo which is evicted
- *
- * State for BOs used by user mode queues which are not at the location they
- * should be.
- */
-static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)
-{
- vm_bo->moved = true;
- spin_lock(&vm_bo->vm->status_lock);
- list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user);
- spin_unlock(&vm_bo->vm->status_lock);
-}
-
-/**
- * amdgpu_vm_bo_relocated - vm_bo is reloacted
- *
- * @vm_bo: vm_bo which is relocated
- *
- * State for PDs/PTs which needs to update their parent PD.
- * For the root PD, just move to idle state.
- */
-static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
-{
- amdgpu_vm_assert_locked(vm_bo->vm);
- if (vm_bo->bo->parent) {
- spin_lock(&vm_bo->vm->status_lock);
- list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
- spin_unlock(&vm_bo->vm->status_lock);
- } else {
- amdgpu_vm_bo_idle(vm_bo);
- }
-}
-
-/**
- * amdgpu_vm_bo_done - vm_bo is done
- *
- * @vm_bo: vm_bo which is now done
- *
- * State for normal BOs which are invalidated and that change has been updated
- * in the PTs.
- */
-static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
-{
- amdgpu_vm_assert_locked(vm_bo->vm);
- spin_lock(&vm_bo->vm->status_lock);
- list_move(&vm_bo->vm_status, &vm_bo->vm->done);
- spin_unlock(&vm_bo->vm->status_lock);
+ lists = amdgpu_vm_bo_lock_lists(vm_bo);
+ if (!amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
+ vm_bo->moved = false;
+ list_move(&vm_bo->vm_status, &lists->idle);
+ amdgpu_vm_bo_unlock_lists(vm_bo);
}
/**
* amdgpu_vm_bo_reset_state_machine - reset the vm_bo state machine
* @vm: the VM which state machine to reset
*
- * Move all vm_bo object in the VM into a state where they will be updated
- * again during validation.
+ * Move all vm_bo object in the VM into a state where their location will be
+ * updated in the page tables again.
*/
static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
{
- struct amdgpu_vm_bo_base *vm_bo, *tmp;
-
amdgpu_vm_assert_locked(vm);
+ list_splice_init(&vm->kernel.idle, &vm->kernel.moved);
+ list_splice_init(&vm->always_valid.idle, &vm->always_valid.moved);
- spin_lock(&vm->status_lock);
- list_splice_init(&vm->done, &vm->invalidated);
- list_for_each_entry(vm_bo, &vm->invalidated, vm_status)
- vm_bo->moved = true;
-
- list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) {
- struct amdgpu_bo *bo = vm_bo->bo;
-
- vm_bo->moved = true;
- if (!bo || bo->tbo.type != ttm_bo_type_kernel)
- list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
- else if (bo->parent)
- list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
- }
- spin_unlock(&vm->status_lock);
+ spin_lock(&vm->individual_lock);
+ list_splice_init(&vm->individual.idle, &vm->individual.moved);
+ spin_unlock(&vm->individual_lock);
}
/**
* amdgpu_vm_update_shared - helper to update shared memory stat
* @base: base structure for tracking BO usage in a VM
*
- * Takes the vm status_lock and updates the shared memory stat. If the basic
+ * Takes the vm stats_lock and updates the shared memory stat. If the basic
* stat changed (e.g. buffer was moved) amdgpu_vm_update_stats need to be called
* as well.
*/
@@ -321,7 +292,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base)
bool shared;
dma_resv_assert_held(bo->tbo.base.resv);
- spin_lock(&vm->status_lock);
+ spin_lock(&vm->stats_lock);
shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
if (base->shared != shared) {
base->shared = shared;
@@ -333,7 +304,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base)
vm->stats[bo_memtype].drm.private += size;
}
}
- spin_unlock(&vm->status_lock);
+ spin_unlock(&vm->stats_lock);
}
/**
@@ -358,11 +329,11 @@ void amdgpu_vm_bo_update_shared(struct amdgpu_bo *bo)
* be bo->tbo.resource
* @sign: if we should add (+1) or subtract (-1) from the stat
*
- * Caller need to have the vm status_lock held. Useful for when multiple update
+ * Caller need to have the vm stats_lock held. Useful for when multiple update
* need to happen at the same time.
*/
static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
- struct ttm_resource *res, int sign)
+ struct ttm_resource *res, int sign)
{
struct amdgpu_vm *vm = base->vm;
struct amdgpu_bo *bo = base->bo;
@@ -386,7 +357,8 @@ static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
*/
if (bo->flags & AMDGPU_GEM_CREATE_DISCARDABLE)
vm->stats[res_memtype].drm.purgeable += size;
- if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(res_memtype)))
+ if (!(bo->preferred_domains &
+ amdgpu_mem_type_to_domain(res_memtype)))
vm->stats[bo_memtype].evicted += size;
}
}
@@ -405,9 +377,9 @@ void amdgpu_vm_update_stats(struct amdgpu_vm_bo_base *base,
{
struct amdgpu_vm *vm = base->vm;
- spin_lock(&vm->status_lock);
+ spin_lock(&vm->stats_lock);
amdgpu_vm_update_stats_locked(base, res, sign);
- spin_unlock(&vm->status_lock);
+ spin_unlock(&vm->stats_lock);
}
/**
@@ -428,37 +400,34 @@ void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
base->next = NULL;
INIT_LIST_HEAD(&base->vm_status);
+ dma_resv_assert_held(vm->root.bo->tbo.base.resv);
if (!bo)
return;
+
base->next = bo->vm_bo;
bo->vm_bo = base;
- spin_lock(&vm->status_lock);
+ spin_lock(&vm->stats_lock);
base->shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1);
- spin_unlock(&vm->status_lock);
+ spin_unlock(&vm->stats_lock);
- if (!amdgpu_vm_is_bo_always_valid(vm, bo))
+ if (!amdgpu_vm_is_bo_always_valid(vm, bo)) {
+ amdgpu_vm_bo_idle(base);
return;
-
- dma_resv_assert_held(vm->root.bo->tbo.base.resv);
+ }
ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move);
- if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
- amdgpu_vm_bo_relocated(base);
- else
- amdgpu_vm_bo_idle(base);
+ /*
+ * When a per VM isn't in the desired domain put it into the evicted
+ * state to make sure that it gets validated on the next best occasion.
+ */
if (bo->preferred_domains &
amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type))
- return;
-
- /*
- * we checked all the prerequisites, but it looks like this per vm bo
- * is currently evicted. add the bo to the evicted list to make sure it
- * is validated on next vm use to avoid fault.
- * */
- amdgpu_vm_bo_evicted(base);
+ amdgpu_vm_bo_moved(base);
+ else
+ amdgpu_vm_bo_evicted(base);
}
/**
@@ -479,41 +448,41 @@ int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec,
}
/**
- * amdgpu_vm_lock_done_list - lock all BOs on the done list
+ * amdgpu_vm_lock_individual - lock all BOs on the individual idle list
* @vm: vm providing the BOs
* @exec: drm execution context
* @num_fences: number of extra fences to reserve
*
- * Lock the BOs on the done list in the DRM execution context.
+ * Lock the BOs on the individual idle list in the DRM execution context.
*/
-int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
- unsigned int num_fences)
+int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
+ unsigned int num_fences)
{
- struct list_head *prev = &vm->done;
+ struct list_head *prev = &vm->individual.idle;
struct amdgpu_bo_va *bo_va;
struct amdgpu_bo *bo;
int ret;
/* We can only trust prev->next while holding the lock */
- spin_lock(&vm->status_lock);
- while (!list_is_head(prev->next, &vm->done)) {
+ spin_lock(&vm->individual_lock);
+ while (!list_is_head(prev->next, &vm->individual.idle)) {
bo_va = list_entry(prev->next, typeof(*bo_va), base.vm_status);
bo = bo_va->base.bo;
if (bo) {
amdgpu_bo_ref(bo);
- spin_unlock(&vm->status_lock);
+ spin_unlock(&vm->individual_lock);
ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 1);
amdgpu_bo_unref(&bo);
if (unlikely(ret))
return ret;
- spin_lock(&vm->status_lock);
+ spin_lock(&vm->individual_lock);
}
prev = prev->next;
}
- spin_unlock(&vm->status_lock);
+ spin_unlock(&vm->individual_lock);
return 0;
}
@@ -609,10 +578,10 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
void *param)
{
uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm);
- struct amdgpu_vm_bo_base *bo_base;
- struct amdgpu_bo *bo;
+ struct amdgpu_vm_bo_base *bo_base, *tmp;
int r;
+ dma_resv_assert_held(vm->root.bo->tbo.base.resv);
if (vm->generation != new_vm_generation) {
vm->generation = new_vm_generation;
amdgpu_vm_bo_reset_state_machine(vm);
@@ -622,49 +591,59 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
return r;
}
- spin_lock(&vm->status_lock);
- while (!list_empty(&vm->evicted)) {
- bo_base = list_first_entry(&vm->evicted,
- struct amdgpu_vm_bo_base,
- vm_status);
- spin_unlock(&vm->status_lock);
+ list_for_each_entry_safe(bo_base, tmp, &vm->kernel.evicted, vm_status) {
+ r = validate(param, bo_base->bo);
+ if (r)
+ return r;
- bo = bo_base->bo;
+ vm->update_funcs->map_table(to_amdgpu_bo_vm(bo_base->bo));
+ amdgpu_vm_bo_moved(bo_base);
+ }
- r = validate(param, bo);
+ /*
+ * As soon as all page tables are in place we can start updating them
+ * again.
+ */
+ amdgpu_vm_eviction_lock(vm);
+ vm->evicting = false;
+ amdgpu_vm_eviction_unlock(vm);
+
+ list_for_each_entry_safe(bo_base, tmp, &vm->always_valid.evicted,
+ vm_status) {
+ r = validate(param, bo_base->bo);
if (r)
return r;
- if (bo->tbo.type != ttm_bo_type_kernel) {
- amdgpu_vm_bo_moved(bo_base);
- } else {
- vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
- amdgpu_vm_bo_relocated(bo_base);
- }
- spin_lock(&vm->status_lock);
+ amdgpu_vm_bo_moved(bo_base);
}
- while (ticket && !list_empty(&vm->evicted_user)) {
- bo_base = list_first_entry(&vm->evicted_user,
- struct amdgpu_vm_bo_base,
- vm_status);
- spin_unlock(&vm->status_lock);
- bo = bo_base->bo;
- dma_resv_assert_held(bo->tbo.base.resv);
+ if (!ticket)
+ return 0;
+
+ spin_lock(&vm->individual_lock);
+restart:
+ list_for_each_entry(bo_base, &vm->individual.evicted, vm_status) {
+ struct amdgpu_bo *bo = bo_base->bo;
+
+ if (dma_resv_locking_ctx(bo->tbo.base.resv) != ticket)
+ continue;
+
+ spin_unlock(&vm->individual_lock);
r = validate(param, bo);
if (r)
return r;
- amdgpu_vm_bo_invalidated(bo_base);
+ amdgpu_vm_bo_moved(bo_base);
- spin_lock(&vm->status_lock);
+ /* It's a bit inefficient to always jump back to the start, but
+ * we would need to re-structure the KFD for properly fixing
+ * that.
+ */
+ spin_lock(&vm->individual_lock);
+ goto restart;
}
- spin_unlock(&vm->status_lock);
-
- amdgpu_vm_eviction_lock(vm);
- vm->evicting = false;
- amdgpu_vm_eviction_unlock(vm);
+ spin_unlock(&vm->individual_lock);
return 0;
}
@@ -689,9 +668,7 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
ret = !vm->evicting;
amdgpu_vm_eviction_unlock(vm);
- spin_lock(&vm->status_lock);
- ret &= list_empty(&vm->evicted);
- spin_unlock(&vm->status_lock);
+ ret &= list_empty(&vm->kernel.evicted);
spin_lock(&vm->immediate.lock);
ret &= !vm->immediate.stopped;
@@ -985,18 +962,13 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
struct amdgpu_vm *vm, bool immediate)
{
struct amdgpu_vm_update_params params;
- struct amdgpu_vm_bo_base *entry;
+ struct amdgpu_vm_bo_base *entry, *tmp;
bool flush_tlb_needed = false;
- LIST_HEAD(relocated);
int r, idx;
amdgpu_vm_assert_locked(vm);
- spin_lock(&vm->status_lock);
- list_splice_init(&vm->relocated, &relocated);
- spin_unlock(&vm->status_lock);
-
- if (list_empty(&relocated))
+ if (list_empty(&vm->kernel.moved))
return 0;
if (!drm_dev_enter(adev_to_drm(adev), &idx))
@@ -1012,7 +984,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
if (r)
goto error;
- list_for_each_entry(entry, &relocated, vm_status) {
+ list_for_each_entry(entry, &vm->kernel.moved, vm_status) {
/* vm_flush_needed after updating moved PDEs */
flush_tlb_needed |= entry->moved;
@@ -1028,11 +1000,8 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
if (flush_tlb_needed)
atomic64_inc(&vm->tlb_seq);
- while (!list_empty(&relocated)) {
- entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base,
- vm_status);
+ list_for_each_entry_safe(entry, tmp, &vm->kernel.moved, vm_status)
amdgpu_vm_bo_idle(entry);
- }
error:
drm_dev_exit(idx);
@@ -1260,9 +1229,9 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
{
- spin_lock(&vm->status_lock);
+ spin_lock(&vm->stats_lock);
memcpy(stats, vm->stats, sizeof(*stats) * __AMDGPU_PL_NUM);
- spin_unlock(&vm->status_lock);
+ spin_unlock(&vm->stats_lock);
}
/**
@@ -1406,7 +1375,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
else
amdgpu_vm_bo_idle(&bo_va->base);
} else {
- amdgpu_vm_bo_done(&bo_va->base);
+ amdgpu_vm_bo_idle(&bo_va->base);
}
list_splice_init(&bo_va->invalids, &bo_va->valids);
@@ -1629,29 +1598,25 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
struct amdgpu_vm *vm,
struct ww_acquire_ctx *ticket)
{
- struct amdgpu_bo_va *bo_va;
+ struct amdgpu_bo_va *bo_va, *tmp;
struct dma_resv *resv;
bool clear, unlock;
int r;
- spin_lock(&vm->status_lock);
- while (!list_empty(&vm->moved)) {
- bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va,
- base.vm_status);
- spin_unlock(&vm->status_lock);
-
+ list_for_each_entry_safe(bo_va, tmp, &vm->always_valid.moved,
+ base.vm_status) {
/* Per VM BOs never need to bo cleared in the page tables */
r = amdgpu_vm_bo_update(adev, bo_va, false);
if (r)
return r;
- spin_lock(&vm->status_lock);
}
- while (!list_empty(&vm->invalidated)) {
- bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
- base.vm_status);
+ spin_lock(&vm->individual_lock);
+ while (!list_empty(&vm->individual.moved)) {
+ bo_va = list_first_entry(&vm->individual.moved,
+ typeof(*bo_va), base.vm_status);
resv = bo_va->base.bo->tbo.base.resv;
- spin_unlock(&vm->status_lock);
+ spin_unlock(&vm->individual_lock);
/* Try to reserve the BO to avoid clearing its ptes */
if (!adev->debug_vm && dma_resv_trylock(resv)) {
@@ -1681,11 +1646,11 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
drm_gem_is_imported(&bo_va->base.bo->tbo.base) &&
(!bo_va->base.bo->tbo.resource ||
bo_va->base.bo->tbo.resource->mem_type == TTM_PL_SYSTEM))
- amdgpu_vm_bo_evicted_user(&bo_va->base);
+ amdgpu_vm_bo_evicted(&bo_va->base);
- spin_lock(&vm->status_lock);
+ spin_lock(&vm->individual_lock);
}
- spin_unlock(&vm->status_lock);
+ spin_unlock(&vm->individual_lock);
return 0;
}
@@ -2223,9 +2188,9 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev,
}
}
- spin_lock(&vm->status_lock);
+ spin_lock(&vm->individual_lock);
list_del(&bo_va->base.vm_status);
- spin_unlock(&vm->status_lock);
+ spin_unlock(&vm->individual_lock);
list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
list_del(&mapping->list);
@@ -2305,14 +2270,7 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_bo *bo, bool evicted)
if (bo_base->moved)
continue;
- bo_base->moved = true;
-
- if (bo->tbo.type == ttm_bo_type_kernel)
- amdgpu_vm_bo_relocated(bo_base);
- else if (amdgpu_vm_is_bo_always_valid(vm, bo))
- amdgpu_vm_bo_moved(bo_base);
- else
- amdgpu_vm_bo_invalidated(bo_base);
+ amdgpu_vm_bo_moved(bo_base);
}
}
@@ -2333,10 +2291,10 @@ void amdgpu_vm_bo_move(struct amdgpu_bo *bo, struct ttm_resource *new_mem,
for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
struct amdgpu_vm *vm = bo_base->vm;
- spin_lock(&vm->status_lock);
+ spin_lock(&vm->stats_lock);
amdgpu_vm_update_stats_locked(bo_base, bo->tbo.resource, -1);
amdgpu_vm_update_stats_locked(bo_base, new_mem, +1);
- spin_unlock(&vm->status_lock);
+ spin_unlock(&vm->stats_lock);
}
amdgpu_vm_bo_invalidate(bo, evicted);
@@ -2603,16 +2561,14 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
vm->va = RB_ROOT_CACHED;
for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
vm->reserved_vmid[i] = NULL;
- INIT_LIST_HEAD(&vm->evicted);
- INIT_LIST_HEAD(&vm->evicted_user);
- INIT_LIST_HEAD(&vm->relocated);
- INIT_LIST_HEAD(&vm->moved);
- INIT_LIST_HEAD(&vm->idle);
- INIT_LIST_HEAD(&vm->invalidated);
- spin_lock_init(&vm->status_lock);
+
+ amdgpu_vm_bo_status_init(&vm->kernel);
+ amdgpu_vm_bo_status_init(&vm->always_valid);
+ spin_lock_init(&vm->individual_lock);
+ amdgpu_vm_bo_status_init(&vm->individual);
INIT_LIST_HEAD(&vm->freed);
- INIT_LIST_HEAD(&vm->done);
INIT_KFIFO(vm->faults);
+ spin_lock_init(&vm->stats_lock);
r = amdgpu_vm_init_entities(adev, vm);
if (r)
@@ -3089,100 +3045,64 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
}
#if defined(CONFIG_DEBUG_FS)
-/**
- * amdgpu_debugfs_vm_bo_info - print BO info for the VM
- *
- * @vm: Requested VM for printing BO info
- * @m: debugfs file
- *
- * Print BO information in debugfs file for the VM
- */
-void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
-{
- struct amdgpu_bo_va *bo_va, *tmp;
- u64 total_idle = 0;
- u64 total_evicted = 0;
- u64 total_relocated = 0;
- u64 total_moved = 0;
- u64 total_invalidated = 0;
- u64 total_done = 0;
- unsigned int total_idle_objs = 0;
- unsigned int total_evicted_objs = 0;
- unsigned int total_relocated_objs = 0;
- unsigned int total_moved_objs = 0;
- unsigned int total_invalidated_objs = 0;
- unsigned int total_done_objs = 0;
- unsigned int id = 0;
- amdgpu_vm_assert_locked(vm);
+/* print the debug info for a specific set of status lists */
+static void amdgpu_debugfs_vm_bo_status_info(struct seq_file *m,
+ struct amdgpu_vm_bo_status *lists)
+{
+ struct amdgpu_vm_bo_base *base;
+ unsigned int id;
- spin_lock(&vm->status_lock);
- seq_puts(m, "\tIdle BOs:\n");
- list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
- if (!bo_va->base.bo)
- continue;
- total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
- }
- total_idle_objs = id;
id = 0;
-
seq_puts(m, "\tEvicted BOs:\n");
- list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
- if (!bo_va->base.bo)
+ list_for_each_entry(base, &lists->evicted, vm_status) {
+ if (!base->bo)
continue;
- total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
- }
- total_evicted_objs = id;
- id = 0;
- seq_puts(m, "\tRelocated BOs:\n");
- list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
- if (!bo_va->base.bo)
- continue;
- total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
+ amdgpu_bo_print_info(id++, base->bo, m);
}
- total_relocated_objs = id;
- id = 0;
+ id = 0;
seq_puts(m, "\tMoved BOs:\n");
- list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
- if (!bo_va->base.bo)
+ list_for_each_entry(base, &lists->moved, vm_status) {
+ if (!base->bo)
continue;
- total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
+
+ amdgpu_bo_print_info(id++, base->bo, m);
}
- total_moved_objs = id;
- id = 0;
- seq_puts(m, "\tInvalidated BOs:\n");
- list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
- if (!bo_va->base.bo)
+ id = 0;
+ seq_puts(m, "\tIdle BOs:\n");
+ list_for_each_entry(base, &lists->moved, vm_status) {
+ if (!base->bo)
continue;
- total_invalidated += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
+
+ amdgpu_bo_print_info(id++, base->bo, m);
}
- total_invalidated_objs = id;
- id = 0;
+}
- seq_puts(m, "\tDone BOs:\n");
- list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
- if (!bo_va->base.bo)
- continue;
- total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
- }
- spin_unlock(&vm->status_lock);
- total_done_objs = id;
-
- seq_printf(m, "\tTotal idle size: %12lld\tobjs:\t%d\n", total_idle,
- total_idle_objs);
- seq_printf(m, "\tTotal evicted size: %12lld\tobjs:\t%d\n", total_evicted,
- total_evicted_objs);
- seq_printf(m, "\tTotal relocated size: %12lld\tobjs:\t%d\n", total_relocated,
- total_relocated_objs);
- seq_printf(m, "\tTotal moved size: %12lld\tobjs:\t%d\n", total_moved,
- total_moved_objs);
- seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated,
- total_invalidated_objs);
- seq_printf(m, "\tTotal done size: %12lld\tobjs:\t%d\n", total_done,
- total_done_objs);
+/**
+ * amdgpu_debugfs_vm_bo_info - print BO info for the VM
+ *
+ * @vm: Requested VM for printing BO info
+ * @m: debugfs file
+ *
+ * Print BO information in debugfs file for the VM
+ */
+void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
+{
+ amdgpu_vm_assert_locked(vm);
+
+ seq_puts(m, "\tKernel PT/PDs:\n");
+ amdgpu_debugfs_vm_bo_status_info(m, &vm->kernel);
+
+ seq_puts(m, "\tPer VM BOs:\n");
+ amdgpu_debugfs_vm_bo_status_info(m, &vm->always_valid);
+
+ seq_puts(m, "\tIndividual BOs:\n");
+ spin_lock(&vm->individual_lock);
+ amdgpu_debugfs_vm_bo_status_info(m, &vm->individual);
+ spin_unlock(&vm->individual_lock);
}
#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index f33ea7f8509b..cc96a3e6252f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -205,17 +205,34 @@ struct amdgpu_vm_bo_base {
/* protected by bo being reserved */
struct amdgpu_vm_bo_base *next;
- /* protected by vm status_lock */
+ /* protected by vm reservation and invalidated_lock */
struct list_head vm_status;
/* if the bo is counted as shared in mem stats
- * protected by vm status_lock */
+ * protected by vm BO being reserved */
bool shared;
/* protected by the BO being reserved */
bool moved;
};
+/*
+ * The following status lists contain amdgpu_vm_bo_base objects for
+ * either PD/PTs, per VM BOs or BOs with individual resv object.
+ *
+ * The state transits are: evicted -> moved -> idle
+ */
+struct amdgpu_vm_bo_status {
+ /* BOs evicted which need to move into place again */
+ struct list_head evicted;
+
+ /* BOs which moved but new location hasn't been updated in the PDs/PTs */
+ struct list_head moved;
+
+ /* BOs done with the state machine and need no further action */
+ struct list_head idle;
+};
+
/* provided by hw blocks that can write ptes, e.g., sdma */
struct amdgpu_vm_pte_funcs {
/* number of dw to reserve per operation */
@@ -345,47 +362,29 @@ struct amdgpu_vm {
bool evicting;
unsigned int saved_flags;
- /* Lock to protect vm_bo add/del/move on all lists of vm */
- spinlock_t status_lock;
-
- /* Memory statistics for this vm, protected by status_lock */
+ /* Memory statistics for this vm, protected by stats_lock */
+ spinlock_t stats_lock;
struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM];
+ /* BO's belonging to PD/PT which are internal to the kernel. */
+ struct amdgpu_vm_bo_status kernel;
+
/*
- * The following lists contain amdgpu_vm_bo_base objects for either
- * PDs, PTs or per VM BOs. The state transits are:
- *
- * evicted -> relocated (PDs, PTs) or moved (per VM BOs) -> idle
+ * BOs allocated by userspace where the dma_resv is shared with the
+ * root PD
*/
-
- /* Per-VM and PT BOs who needs a validation */
- struct list_head evicted;
-
- /* PT BOs which relocated and their parent need an update */
- struct list_head relocated;
-
- /* per VM BOs moved, but not yet updated in the PT */
- struct list_head moved;
-
- /* All BOs of this VM not currently in the state machine */
- struct list_head idle;
+ struct amdgpu_vm_bo_status always_valid;
/*
* The following lists contain amdgpu_vm_bo_base objects for BOs which
- * have their own dma_resv object and not depend on the root PD. Their
- * state transits are:
+ * have their own dma_resv object and not depend on the root PD.
*
- * evicted_user or invalidated -> done
+ * Lists are protected by the individual_lock.
*/
+ spinlock_t individual_lock;
- /* BOs for user mode queues that need a validation */
- struct list_head evicted_user;
-
- /* regular invalidated BOs, but not yet updated in the PT */
- struct list_head invalidated;
-
- /* BOs which are invalidated, has been updated in the PTs */
- struct list_head done;
+ /* Userspace BOs with individual resv object */
+ struct amdgpu_vm_bo_status individual;
/*
* This list contains amdgpu_bo_va_mapping objects which have been freed
@@ -507,8 +506,8 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm);
void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec,
unsigned int num_fences);
-int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
- unsigned int num_fences);
+int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
+ unsigned int num_fences);
bool amdgpu_vm_ready(struct amdgpu_vm *vm);
uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm);
int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index a930f1522f96..ff334f4dcc90 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -544,9 +544,7 @@ static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry)
entry->bo->vm_bo = NULL;
ttm_bo_set_bulk_move(&entry->bo->tbo, NULL);
- spin_lock(&entry->vm->status_lock);
list_del(&entry->vm_status);
- spin_unlock(&entry->vm->status_lock);
amdgpu_bo_unref(&entry->bo);
}
@@ -590,7 +588,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params,
struct amdgpu_vm_pt_cursor seek;
struct amdgpu_vm_bo_base *entry;
- spin_lock(¶ms->vm->status_lock);
for_each_amdgpu_vm_pt_dfs_safe(params->adev, params->vm, cursor, seek, entry) {
if (entry && entry->bo)
list_move(&entry->vm_status, ¶ms->tlb_flush_waitlist);
@@ -598,7 +595,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params,
/* enter start node now */
list_move(&cursor->entry->vm_status, ¶ms->tlb_flush_waitlist);
- spin_unlock(¶ms->vm->status_lock);
}
/**
--
2.43.0
^ permalink raw reply related [flat|nested] 30+ messages in thread
* RE: [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab
2026-04-27 18:27 [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab Christian König
` (6 preceding siblings ...)
2026-04-27 18:27 ` [PATCH 8/8] drm/amdgpu: restructure VM state machine v2 Christian König
@ 2026-04-28 6:15 ` Liang, Prike
2026-04-28 8:06 ` Christian König
2026-04-28 7:57 ` Khatri, Sunil
8 siblings, 1 reply; 30+ messages in thread
From: Liang, Prike @ 2026-04-28 6:15 UTC (permalink / raw)
To: Koenig, Christian, Deucher, Alexander, Khatri, Sunil,
Zhang, Jesse(Jie)
Cc: amd-gfx@lists.freedesktop.org
Public
Regards,
Prike
> -----Original Message-----
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> Sent: Tuesday, April 28, 2026 2:27 AM
> To: Deucher, Alexander <Alexander.Deucher@amd.com>; Liang, Prike
> <Prike.Liang@amd.com>; Khatri, Sunil <Sunil.Khatri@amd.com>; Zhang,
> Jesse(Jie) <Jesse.Zhang@amd.com>
> Cc: amd-gfx@lists.freedesktop.org
> Subject: [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab
>
> As preparation for independent fences remove the extra slab, kmalloc should do just
> fine.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 17 ++++-------
> .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c | 28 +++----------------
> .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.h | 3 --
> 3 files changed, 9 insertions(+), 39 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index 8f66f197f645..2b268cdf076b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -3162,11 +3162,7 @@ static int __init amdgpu_init(void)
>
> r = amdgpu_sync_init();
> if (r)
> - goto error_sync;
> -
> - r = amdgpu_userq_fence_slab_init();
> - if (r)
> - goto error_fence;
> + return r;
>
> amdgpu_register_atpx_handler();
> amdgpu_acpi_detect();
> @@ -3174,7 +3170,7 @@ static int __init amdgpu_init(void)
> /* Ignore KFD init failures when CONFIG_HSA_AMD is not set. */
> r = amdgpu_amdkfd_init();
> if (r && r != -ENOENT)
> - goto error_fence;
> + goto error_fini_sync;
>
> if (amdgpu_pp_feature_mask & PP_OVERDRIVE_MASK) {
> add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
> @@ -3185,11 +3181,9 @@ static int __init amdgpu_init(void)
> /* let modprobe override vga console setting */
> return pci_register_driver(&amdgpu_kms_pci_driver);
>
> -error_fence:
> - amdgpu_sync_fini();
> -
> -error_sync:
> - return r;
> +error_fini_sync:
> + amdgpu_sync_fini();
> + return r;
> }
>
> static void __exit amdgpu_exit(void)
> @@ -3199,7 +3193,6 @@ static void __exit amdgpu_exit(void)
> amdgpu_unregister_atpx_handler();
> amdgpu_acpi_release();
> amdgpu_sync_fini();
> - amdgpu_userq_fence_slab_fini();
> mmu_notifier_synchronize();
> amdgpu_xcp_drv_release();
> }
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> index a58342c2ac44..f3d0595e3e61 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> @@ -32,29 +32,9 @@
> #include "amdgpu.h"
> #include "amdgpu_userq_fence.h"
>
> -static const struct dma_fence_ops amdgpu_userq_fence_ops; -static struct
> kmem_cache *amdgpu_userq_fence_slab;
> -
> #define AMDGPU_USERQ_MAX_HANDLES (1U << 16)
>
> -int amdgpu_userq_fence_slab_init(void)
> -{
> - amdgpu_userq_fence_slab = kmem_cache_create("amdgpu_userq_fence",
> - sizeof(struct amdgpu_userq_fence),
> - 0,
> - SLAB_HWCACHE_ALIGN,
> - NULL);
> - if (!amdgpu_userq_fence_slab)
> - return -ENOMEM;
> -
> - return 0;
> -}
> -
> -void amdgpu_userq_fence_slab_fini(void)
> -{
> - rcu_barrier();
> - kmem_cache_destroy(amdgpu_userq_fence_slab);
> -}
> +static const struct dma_fence_ops amdgpu_userq_fence_ops;
>
> static inline struct amdgpu_userq_fence *to_amdgpu_userq_fence(struct
> dma_fence *f) { @@ -231,7 +211,7 @@ void
> amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
>
> static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence) {
> - *userq_fence = kmem_cache_alloc(amdgpu_userq_fence_slab,
> GFP_ATOMIC);
> + *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
We might need the GFP_KERNEL allocation?
Since the userq fence is a small, fixed-size object, this change may not have a significant performance impact. However, fence object allocation and deallocation should be profiled in scenarios where userq fences are emitted frequently, as a follow-up to confirm.
But now, the patch is Reviewed-by: Prike Liang <Prike.Liang@amd.com>
> return *userq_fence ? 0 : -ENOMEM;
> }
>
> @@ -343,7 +323,7 @@ static void amdgpu_userq_fence_free(struct rcu_head *rcu)
> amdgpu_userq_fence_driver_put(fence_drv);
>
> kvfree(userq_fence->fence_drv_array);
> - kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
> + kfree(userq_fence);
> }
>
> static void amdgpu_userq_fence_release(struct dma_fence *f) @@ -546,7 +526,7
> @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
> r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
> if (r) {
> mutex_unlock(&userq_mgr->userq_mutex);
> - kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
> + kfree(userq_fence);
> goto put_gobj_write;
> }
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> index d56246ad8c26..d355a0eecc07 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> @@ -58,9 +58,6 @@ struct amdgpu_userq_fence_driver {
> char timeline_name[TASK_COMM_LEN];
> };
>
> -int amdgpu_userq_fence_slab_init(void);
> -void amdgpu_userq_fence_slab_fini(void);
> -
> void amdgpu_userq_fence_driver_get(struct amdgpu_userq_fence_driver
> *fence_drv); void amdgpu_userq_fence_driver_put(struct
> amdgpu_userq_fence_driver *fence_drv); int
> amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,
> --
> 2.43.0
^ permalink raw reply [flat|nested] 30+ messages in thread
* RE: [PATCH 2/8] drm/amdgpu: remove deadlocks from amdgpu_userq_pre_reset
2026-04-27 18:27 ` [PATCH 2/8] drm/amdgpu: remove deadlocks from amdgpu_userq_pre_reset Christian König
@ 2026-04-28 6:51 ` Liang, Prike
2026-04-28 8:15 ` Khatri, Sunil
0 siblings, 1 reply; 30+ messages in thread
From: Liang, Prike @ 2026-04-28 6:51 UTC (permalink / raw)
To: Koenig, Christian, Deucher, Alexander, Khatri, Sunil,
Zhang, Jesse(Jie)
Cc: amd-gfx@lists.freedesktop.org
Public
Regards,
Prike
> -----Original Message-----
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> Sent: Tuesday, April 28, 2026 2:27 AM
> To: Deucher, Alexander <Alexander.Deucher@amd.com>; Liang, Prike
> <Prike.Liang@amd.com>; Khatri, Sunil <Sunil.Khatri@amd.com>; Zhang,
> Jesse(Jie) <Jesse.Zhang@amd.com>
> Cc: amd-gfx@lists.freedesktop.org
> Subject: [PATCH 2/8] drm/amdgpu: remove deadlocks from
> amdgpu_userq_pre_reset
>
> The purpose of a GPU reset is to make sure that fence can be signaled again and
> the signal and resume workers can make progress again.
>
> So waiting for the resume worker or any fence in the GPU reset path is just utterly
> nonsense.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 26 +++++++++++------------
> 1 file changed, 12 insertions(+), 14 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index 58b9477c0346..a34b1df513d3 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -1510,23 +1510,21 @@ void amdgpu_userq_pre_reset(struct amdgpu_device
> *adev) {
> const struct amdgpu_userq_funcs *userq_funcs;
> struct amdgpu_usermode_queue *queue;
> - struct amdgpu_userq_mgr *uqm;
> unsigned long queue_id;
>
> + /* TODO: We probably need a new lock for the queue state */
It seems a new per-queue lock may can help locking the queue precisely than the process wide lock userq_mutex,
so here prefer to create a new per-queue lock than reusing userq_mutex?
To resolve the deadlock in the userq fence wait nesting, this patch is Reviewed-by: Prike Liang <Prike.Liang@amd.com>
> xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
> - uqm = queue->userq_mgr;
> - cancel_delayed_work_sync(&uqm->resume_work);
> - if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
> - amdgpu_userq_wait_for_last_fence(queue);
> - userq_funcs = adev->userq_funcs[queue->queue_type];
> - userq_funcs->unmap(queue);
> - /* just mark all queues as hung at this point.
> - * if unmap succeeds, we could map again
> - * in amdgpu_userq_post_reset() if vram is not lost
> - */
> - queue->state = AMDGPU_USERQ_STATE_HUNG;
> - amdgpu_userq_fence_driver_force_completion(queue);
> - }
> + if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
> + continue;
> +
> + userq_funcs = adev->userq_funcs[queue->queue_type];
> + userq_funcs->unmap(queue);
> + /* just mark all queues as hung at this point.
> + * if unmap succeeds, we could map again
> + * in amdgpu_userq_post_reset() if vram is not lost
> + */
> + queue->state = AMDGPU_USERQ_STATE_HUNG;
> + amdgpu_userq_fence_driver_force_completion(queue);
> }
> }
>
> --
> 2.43.0
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab
2026-04-27 18:27 [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab Christian König
` (7 preceding siblings ...)
2026-04-28 6:15 ` [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab Liang, Prike
@ 2026-04-28 7:57 ` Khatri, Sunil
8 siblings, 0 replies; 30+ messages in thread
From: Khatri, Sunil @ 2026-04-28 7:57 UTC (permalink / raw)
To: christian.koenig, alexander.deucher, Prike.Liang, jesse.zhang; +Cc: amd-gfx
Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
On 27-04-2026 11:57 pm, Christian König wrote:
> As preparation for independent fences remove the extra slab, kmalloc
> should do just fine.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 17 ++++-------
> .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c | 28 +++----------------
> .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.h | 3 --
> 3 files changed, 9 insertions(+), 39 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index 8f66f197f645..2b268cdf076b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -3162,11 +3162,7 @@ static int __init amdgpu_init(void)
>
> r = amdgpu_sync_init();
> if (r)
> - goto error_sync;
> -
> - r = amdgpu_userq_fence_slab_init();
> - if (r)
> - goto error_fence;
> + return r;
>
> amdgpu_register_atpx_handler();
> amdgpu_acpi_detect();
> @@ -3174,7 +3170,7 @@ static int __init amdgpu_init(void)
> /* Ignore KFD init failures when CONFIG_HSA_AMD is not set. */
> r = amdgpu_amdkfd_init();
> if (r && r != -ENOENT)
> - goto error_fence;
> + goto error_fini_sync;
>
> if (amdgpu_pp_feature_mask & PP_OVERDRIVE_MASK) {
> add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
> @@ -3185,11 +3181,9 @@ static int __init amdgpu_init(void)
> /* let modprobe override vga console setting */
> return pci_register_driver(&amdgpu_kms_pci_driver);
>
> -error_fence:
> - amdgpu_sync_fini();
> -
> -error_sync:
> - return r;
> +error_fini_sync:
> + amdgpu_sync_fini();
> + return r;
> }
>
> static void __exit amdgpu_exit(void)
> @@ -3199,7 +3193,6 @@ static void __exit amdgpu_exit(void)
> amdgpu_unregister_atpx_handler();
> amdgpu_acpi_release();
> amdgpu_sync_fini();
> - amdgpu_userq_fence_slab_fini();
> mmu_notifier_synchronize();
> amdgpu_xcp_drv_release();
> }
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> index a58342c2ac44..f3d0595e3e61 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> @@ -32,29 +32,9 @@
> #include "amdgpu.h"
> #include "amdgpu_userq_fence.h"
>
> -static const struct dma_fence_ops amdgpu_userq_fence_ops;
> -static struct kmem_cache *amdgpu_userq_fence_slab;
> -
> #define AMDGPU_USERQ_MAX_HANDLES (1U << 16)
>
> -int amdgpu_userq_fence_slab_init(void)
> -{
> - amdgpu_userq_fence_slab = kmem_cache_create("amdgpu_userq_fence",
> - sizeof(struct amdgpu_userq_fence),
> - 0,
> - SLAB_HWCACHE_ALIGN,
> - NULL);
> - if (!amdgpu_userq_fence_slab)
> - return -ENOMEM;
> -
> - return 0;
> -}
> -
> -void amdgpu_userq_fence_slab_fini(void)
> -{
> - rcu_barrier();
> - kmem_cache_destroy(amdgpu_userq_fence_slab);
> -}
> +static const struct dma_fence_ops amdgpu_userq_fence_ops;
>
> static inline struct amdgpu_userq_fence *to_amdgpu_userq_fence(struct dma_fence *f)
> {
> @@ -231,7 +211,7 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
>
> static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
> {
> - *userq_fence = kmem_cache_alloc(amdgpu_userq_fence_slab, GFP_ATOMIC);
> + *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
> return *userq_fence ? 0 : -ENOMEM;
> }
>
> @@ -343,7 +323,7 @@ static void amdgpu_userq_fence_free(struct rcu_head *rcu)
> amdgpu_userq_fence_driver_put(fence_drv);
>
> kvfree(userq_fence->fence_drv_array);
> - kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
> + kfree(userq_fence);
> }
>
> static void amdgpu_userq_fence_release(struct dma_fence *f)
> @@ -546,7 +526,7 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
> r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
> if (r) {
> mutex_unlock(&userq_mgr->userq_mutex);
> - kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
> + kfree(userq_fence);
> goto put_gobj_write;
> }
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> index d56246ad8c26..d355a0eecc07 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> @@ -58,9 +58,6 @@ struct amdgpu_userq_fence_driver {
> char timeline_name[TASK_COMM_LEN];
> };
>
> -int amdgpu_userq_fence_slab_init(void);
> -void amdgpu_userq_fence_slab_fini(void);
> -
> void amdgpu_userq_fence_driver_get(struct amdgpu_userq_fence_driver *fence_drv);
> void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv);
> int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab
2026-04-28 6:15 ` [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab Liang, Prike
@ 2026-04-28 8:06 ` Christian König
0 siblings, 0 replies; 30+ messages in thread
From: Christian König @ 2026-04-28 8:06 UTC (permalink / raw)
To: Liang, Prike, Deucher, Alexander, Khatri, Sunil,
Zhang, Jesse(Jie)
Cc: amd-gfx@lists.freedesktop.org
On 4/28/26 08:15, Liang, Prike wrote:
...
>>
>> static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence) {
>> - *userq_fence = kmem_cache_alloc(amdgpu_userq_fence_slab,
>> GFP_ATOMIC);
>> + *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
>
> We might need the GFP_KERNEL allocation?
Ah, yes good point. I just incorrectly copy&pasted that here.
>
> Since the userq fence is a small, fixed-size object, this change may not have a significant performance impact. However, fence object allocation and deallocation should be profiled in scenarios where userq fences are emitted frequently, as a follow-up to confirm.
> But now, the patch is Reviewed-by: Prike Liang <Prike.Liang@amd.com>
Thanks,
Christian.
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH 2/8] drm/amdgpu: remove deadlocks from amdgpu_userq_pre_reset
2026-04-28 6:51 ` Liang, Prike
@ 2026-04-28 8:15 ` Khatri, Sunil
0 siblings, 0 replies; 30+ messages in thread
From: Khatri, Sunil @ 2026-04-28 8:15 UTC (permalink / raw)
To: Liang, Prike, Koenig, Christian, Deucher, Alexander,
Khatri, Sunil, Zhang, Jesse(Jie)
Cc: amd-gfx@lists.freedesktop.org
On 28-04-2026 12:21 pm, Liang, Prike wrote:
> Public
>
> Regards,
> Prike
>
>> -----Original Message-----
>> From: Christian König <ckoenig.leichtzumerken@gmail.com>
>> Sent: Tuesday, April 28, 2026 2:27 AM
>> To: Deucher, Alexander <Alexander.Deucher@amd.com>; Liang, Prike
>> <Prike.Liang@amd.com>; Khatri, Sunil <Sunil.Khatri@amd.com>; Zhang,
>> Jesse(Jie) <Jesse.Zhang@amd.com>
>> Cc: amd-gfx@lists.freedesktop.org
>> Subject: [PATCH 2/8] drm/amdgpu: remove deadlocks from
>> amdgpu_userq_pre_reset
>>
>> The purpose of a GPU reset is to make sure that fence can be signaled again and
>> the signal and resume workers can make progress again.
>>
>> So waiting for the resume worker or any fence in the GPU reset path is just utterly
>> nonsense.
>>
>> Signed-off-by: Christian König <christian.koenig@amd.com>
>> ---
>> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 26 +++++++++++------------
>> 1 file changed, 12 insertions(+), 14 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>> index 58b9477c0346..a34b1df513d3 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>> @@ -1510,23 +1510,21 @@ void amdgpu_userq_pre_reset(struct amdgpu_device
>> *adev) {
>> const struct amdgpu_userq_funcs *userq_funcs;
>> struct amdgpu_usermode_queue *queue;
>> - struct amdgpu_userq_mgr *uqm;
>> unsigned long queue_id;
>>
>> + /* TODO: We probably need a new lock for the queue state */
> It seems a new per-queue lock may can help locking the queue precisely than the process wide lock userq_mutex,
> so here prefer to create a new per-queue lock than reusing userq_mutex?
>
> To resolve the deadlock in the userq fence wait nesting, this patch is Reviewed-by: Prike Liang <Prike.Liang@amd.com>
>
>> xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
>> - uqm = queue->userq_mgr;
>> - cancel_delayed_work_sync(&uqm->resume_work);
>> - if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
>> - amdgpu_userq_wait_for_last_fence(queue);
>> - userq_funcs = adev->userq_funcs[queue->queue_type];
>> - userq_funcs->unmap(queue);
>> - /* just mark all queues as hung at this point.
>> - * if unmap succeeds, we could map again
>> - * in amdgpu_userq_post_reset() if vram is not lost
>> - */
>> - queue->state = AMDGPU_USERQ_STATE_HUNG;
>> - amdgpu_userq_fence_driver_force_completion(queue);
>> - }
>> + if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
>> + continue;
>> +
>> + userq_funcs = adev->userq_funcs[queue->queue_type];
>> + userq_funcs->unmap(queue);
>> + /* just mark all queues as hung at this point.
>> + * if unmap succeeds, we could map again
>> + * in amdgpu_userq_post_reset() if vram is not lost
>> + */
>> + queue->state = AMDGPU_USERQ_STATE_HUNG;
>> + amdgpu_userq_fence_driver_force_completion(queue);
>> }
My understanding here is: We are in this function after we tried and
failed in per queue reset where we send request to fw to reset it. Now
we have fallen to who GPU reset. Considering the scenario i shared is
true, we better just call amdgpu_userq_fence_driver_force_completion and
avoid calling unmap as it might be stuck again and makes no difference.
So setting the state to HUNG and calling force completion sounds better.
Other than that the patch is fine and good to push
Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Regards
Sunil Khatri
>> }
>>
>> --
>> 2.43.0
^ permalink raw reply [flat|nested] 30+ messages in thread
* RE: [PATCH 3/8] drm/amdgpu: rework amdgpu_userq_signal_ioctl v2
2026-04-27 18:27 ` [PATCH 3/8] drm/amdgpu: rework amdgpu_userq_signal_ioctl v2 Christian König
@ 2026-04-28 8:49 ` Liang, Prike
2026-05-04 12:33 ` Christian König
2026-04-28 9:35 ` Khatri, Sunil
1 sibling, 1 reply; 30+ messages in thread
From: Liang, Prike @ 2026-04-28 8:49 UTC (permalink / raw)
To: Koenig, Christian, Deucher, Alexander, Khatri, Sunil,
Zhang, Jesse(Jie)
Cc: amd-gfx@lists.freedesktop.org
Public
Regards,
Prike
> -----Original Message-----
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> Sent: Tuesday, April 28, 2026 2:27 AM
> To: Deucher, Alexander <Alexander.Deucher@amd.com>; Liang, Prike
> <Prike.Liang@amd.com>; Khatri, Sunil <Sunil.Khatri@amd.com>; Zhang,
> Jesse(Jie) <Jesse.Zhang@amd.com>
> Cc: amd-gfx@lists.freedesktop.org
> Subject: [PATCH 3/8] drm/amdgpu: rework amdgpu_userq_signal_ioctl v2
>
> This one was fortunately not looking so bad as the wait ioctl path, but there were still
> a few things which could be fixed/improved:
>
> 1. Allocating with GFP_ATOMIC was quite unecessary, we can do that
> before taking the userq_lock.
> 2. Use a new mutex as protection for the fence_drv_xa so that we can do
> memory allocations while holding it.
> 3. Starting the reset timer is unecessary when the fence is already
> signaled when we create it.
> 4. Cleanup error handling, avoid trying to free the queue when we don't
> even got one.
>
> v2: fix incorrect usage of xa_find, destroy the new mutex on error
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 2 +
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 12 +
> .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c | 227 ++++++++----------
> 3 files changed, 119 insertions(+), 122 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index a34b1df513d3..62d155d6e4ed 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -806,6 +806,7 @@ amdgpu_userq_create(struct drm_file *filp, union
> drm_amdgpu_userq *args)
> }
>
> queue->doorbell_index = index;
> + mutex_init(&queue->fence_drv_lock);
> xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
> r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
> if (r) {
> @@ -879,6 +880,7 @@ amdgpu_userq_create(struct drm_file *filp, union
> drm_amdgpu_userq *args)
> amdgpu_bo_reserve(fpriv->vm.root.bo, true);
> amdgpu_userq_buffer_vas_list_cleanup(adev, queue);
> amdgpu_bo_unreserve(fpriv->vm.root.bo);
> + mutex_destroy(&queue->fence_drv_lock);
> free_queue:
> kfree(queue);
> err_pm_runtime:
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> index 8b8f345b60b6..843ea8ecc5d7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> @@ -66,6 +66,18 @@ struct amdgpu_usermode_queue {
> struct amdgpu_userq_obj db_obj;
> struct amdgpu_userq_obj fw_obj;
> struct amdgpu_userq_obj wptr_obj;
> +
> + /**
> + * @fence_drv_lock: Protecting @fence_drv_xa.
> + */
> + struct mutex fence_drv_lock;
> +
> + /**
> + * @fence_drv_xa:
> + *
> + * References to the external fence drivers returned by wait_ioctl.
> + * Dropped on the next signaled dma_fence or queue destruction.
> + */
> struct xarray fence_drv_xa;
> struct amdgpu_userq_fence_driver *fence_drv;
> struct dma_fence *last_fence;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> index f3d0595e3e61..a2792f05b670 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> @@ -121,6 +121,7 @@ amdgpu_userq_fence_driver_free(struct
> amdgpu_usermode_queue *userq)
> userq->last_fence = NULL;
> amdgpu_userq_walk_and_drop_fence_drv(&userq->fence_drv_xa);
> xa_destroy(&userq->fence_drv_xa);
> + mutex_destroy(&userq->fence_drv_lock);
> /* Drop the queue's ownership reference to fence_drv explicitly */
> amdgpu_userq_fence_driver_put(userq->fence_drv);
> }
> @@ -209,81 +210,84 @@ void amdgpu_userq_fence_driver_put(struct
> amdgpu_userq_fence_driver *fence_drv)
> kref_put(&fence_drv->refcount, amdgpu_userq_fence_driver_destroy);
> }
>
> -static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
> +static int amdgpu_userq_fence_alloc(struct amdgpu_usermode_queue *userq,
> + struct amdgpu_userq_fence **pfence)
> {
> - *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
> - return *userq_fence ? 0 : -ENOMEM;
> + struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
> + struct amdgpu_userq_fence *userq_fence;
> + void * entry;
> +
> + userq_fence = kmalloc(sizeof(*userq_fence), GFP_KERNEL);
> + if (!userq_fence)
> + return -ENOMEM;
> +
> + /*
> + * Get the next unused entry, since we fill from the start this can be
> + * used as size to allocate the array.
> + */
> + mutex_lock(&userq->fence_drv_lock);
> + XA_STATE(xas, &userq->fence_drv_xa, 0);
> +
> + rcu_read_lock();
> + do {
> + entry = xas_find_marked(&xas, ULONG_MAX, XA_FREE_MARK);
> + } while (xas_retry(&xas, entry));
Do we need to handle the xas.xa_index = 0 case separately rather than use the invalid wait fence?
> + rcu_read_unlock();
> +
> + userq_fence->fence_drv_array = kvmalloc_array(xas.xa_index,
> + sizeof(fence_drv),
> + GFP_KERNEL);
The kvmalloc_array() should use the obj size of sizeof(*userq_fence->fence_drv_array).
> + if (!userq_fence->fence_drv_array) {
> + mutex_unlock(&userq->fence_drv_lock);
> + kfree(userq_fence);
> + return -ENOMEM;
> + }
> +
> + userq_fence->fence_drv_array_count = xas.xa_index;
> + xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array,
> + 0, ULONG_MAX, xas.xa_index, XA_PRESENT);
> + xa_destroy(&userq->fence_drv_xa);
> +
> + mutex_unlock(&userq->fence_drv_lock);
> +
> + userq_fence->fence_drv = fence_drv;
> + amdgpu_userq_fence_driver_get(fence_drv);
It will be better to acquire the fence_drv reference first before publishing it.
> + *pfence = userq_fence;
> + return 0;
> }
>
> -static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
> - struct amdgpu_userq_fence *userq_fence,
> - u64 seq, struct dma_fence **f)
> +static void amdgpu_userq_fence_init(struct amdgpu_usermode_queue *userq,
> + struct amdgpu_userq_fence *fence,
> + u64 seq)
> {
> - struct amdgpu_userq_fence_driver *fence_drv;
> - struct dma_fence *fence;
> + struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
> unsigned long flags;
> bool signaled = false;
>
> - fence_drv = userq->fence_drv;
> - if (!fence_drv)
> - return -EINVAL;
> -
> - spin_lock_init(&userq_fence->lock);
> - INIT_LIST_HEAD(&userq_fence->link);
> - fence = &userq_fence->base;
> - userq_fence->fence_drv = fence_drv;
> -
> - dma_fence_init64(fence, &amdgpu_userq_fence_ops, &userq_fence->lock,
> + spin_lock_init(&fence->lock);
> + dma_fence_init64(&fence->base, &amdgpu_userq_fence_ops, &fence->lock,
> fence_drv->context, seq);
>
> - amdgpu_userq_fence_driver_get(fence_drv);
> - dma_fence_get(fence);
> -
> - if (!xa_empty(&userq->fence_drv_xa)) {
> - struct amdgpu_userq_fence_driver *stored_fence_drv;
> - unsigned long index, count = 0;
> - int i = 0;
> -
> - xa_lock(&userq->fence_drv_xa);
> - xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv)
> - count++;
> -
> - userq_fence->fence_drv_array =
> - kvmalloc_array(count,
> - sizeof(struct amdgpu_userq_fence_driver *),
> - GFP_ATOMIC);
> -
> - if (userq_fence->fence_drv_array) {
> - xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv)
> {
> - userq_fence->fence_drv_array[i] = stored_fence_drv;
> - __xa_erase(&userq->fence_drv_xa, index);
> - i++;
> - }
> - }
> -
> - userq_fence->fence_drv_array_count = i;
> - xa_unlock(&userq->fence_drv_xa);
> - } else {
> - userq_fence->fence_drv_array = NULL;
> - userq_fence->fence_drv_array_count = 0;
> - }
> + /* Make sure the fence is visible to the hang detect worker */
> + dma_fence_put(userq->last_fence);
> + userq->last_fence = dma_fence_get(&fence->base);
>
> - /* Check if hardware has already processed the job */
> + /* Check if hardware has already processed the fence */
> spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
> - if (!dma_fence_is_signaled(fence)) {
> - list_add_tail(&userq_fence->link, &fence_drv->fences);
> + if (!dma_fence_is_signaled(&fence->base)) {
> + dma_fence_get(&fence->base);
> + list_add_tail(&fence->link, &fence_drv->fences);
> } else {
> + INIT_LIST_HEAD(&fence->link);
> signaled = true;
> - dma_fence_put(fence);
> }
> spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
>
> if (signaled)
> - amdgpu_userq_fence_put_fence_drv_array(userq_fence);
> -
> - *f = fence;
> -
> - return 0;
> + amdgpu_userq_fence_put_fence_drv_array(fence);
> + else
> + amdgpu_userq_start_hang_detect_work(userq);
> }
>
> static const char *amdgpu_userq_fence_get_driver_name(struct dma_fence *f) @@
> -404,11 +408,6 @@ static int amdgpu_userq_fence_read_wptr(struct
> amdgpu_device *adev,
> return r;
> }
>
> -static void amdgpu_userq_fence_cleanup(struct dma_fence *fence) -{
> - dma_fence_put(fence);
> -}
> -
> static void
> amdgpu_userq_fence_driver_set_error(struct amdgpu_userq_fence *fence,
> int error)
> @@ -452,13 +451,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev,
> void *data,
> const unsigned int num_read_bo_handles = args->num_bo_read_handles;
> struct amdgpu_fpriv *fpriv = filp->driver_priv;
> struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
> +
> struct drm_gem_object **gobj_write, **gobj_read;
> u32 *syncobj_handles, num_syncobj_handles;
> - struct amdgpu_userq_fence *userq_fence;
> - struct amdgpu_usermode_queue *queue = NULL;
> - struct drm_syncobj **syncobj = NULL;
> - struct dma_fence *fence;
> + struct amdgpu_usermode_queue *queue;
> + struct amdgpu_userq_fence *fence;
> + struct drm_syncobj **syncobj;
> struct drm_exec exec;
> + void __user *ptr;
> int r, i, entry;
> u64 wptr;
>
> @@ -470,13 +470,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev,
> void *data,
> return -EINVAL;
>
> num_syncobj_handles = args->num_syncobj_handles;
> - syncobj_handles = memdup_array_user(u64_to_user_ptr(args-
> >syncobj_handles),
> - num_syncobj_handles, sizeof(u32));
> + ptr = u64_to_user_ptr(args->syncobj_handles);
> + syncobj_handles = memdup_array_user(ptr, num_syncobj_handles,
> + sizeof(u32));
> if (IS_ERR(syncobj_handles))
> return PTR_ERR(syncobj_handles);
>
> - /* Array of pointers to the looked up syncobjs */
> - syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj),
> GFP_KERNEL);
> + syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj),
> + GFP_KERNEL);
> if (!syncobj) {
> r = -ENOMEM;
> goto free_syncobj_handles;
> @@ -490,21 +491,17 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev,
> void *data,
> }
> }
>
> - r = drm_gem_objects_lookup(filp,
> - u64_to_user_ptr(args->bo_read_handles),
> - num_read_bo_handles,
> - &gobj_read);
> + ptr = u64_to_user_ptr(args->bo_read_handles);
> + r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles,
> +&gobj_read);
> if (r)
> goto free_syncobj;
>
> - r = drm_gem_objects_lookup(filp,
> - u64_to_user_ptr(args->bo_write_handles),
> - num_write_bo_handles,
> + ptr = u64_to_user_ptr(args->bo_write_handles);
> + r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
> &gobj_write);
> if (r)
> goto put_gobj_read;
>
> - /* Retrieve the user queue */
> queue = amdgpu_userq_get(userq_mgr, args->queue_id);
> if (!queue) {
> r = -ENOENT;
> @@ -513,73 +510,61 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev,
> void *data,
>
> r = amdgpu_userq_fence_read_wptr(adev, queue, &wptr);
> if (r)
> - goto put_gobj_write;
> + goto put_queue;
>
> - r = amdgpu_userq_fence_alloc(&userq_fence);
> + r = amdgpu_userq_fence_alloc(queue, &fence);
> if (r)
> - goto put_gobj_write;
> + goto put_queue;
>
> /* We are here means UQ is active, make sure the eviction fence is valid */
> amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
>
> - /* Create a new fence */
> - r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
> - if (r) {
> - mutex_unlock(&userq_mgr->userq_mutex);
> - kfree(userq_fence);
> - goto put_gobj_write;
> - }
> + /* Create the new fence */
> + amdgpu_userq_fence_init(queue, fence, wptr);
>
> - dma_fence_put(queue->last_fence);
> - queue->last_fence = dma_fence_get(fence);
> - amdgpu_userq_start_hang_detect_work(queue);
> mutex_unlock(&userq_mgr->userq_mutex);
>
> + /*
> + * This needs to come after the fence is created since
> + * amdgpu_userq_ensure_ev_fence() can't be called while holding the resv
> + * locks.
> + */
> drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
> (num_read_bo_handles + num_write_bo_handles));
>
> - /* Lock all BOs with retry handling */
> drm_exec_until_all_locked(&exec) {
> - r = drm_exec_prepare_array(&exec, gobj_read,
> num_read_bo_handles, 1);
> + r = drm_exec_prepare_array(&exec, gobj_read,
> + num_read_bo_handles, 1);
> drm_exec_retry_on_contention(&exec);
> - if (r) {
> - amdgpu_userq_fence_cleanup(fence);
> + if (r)
> goto exec_fini;
> - }
>
> - r = drm_exec_prepare_array(&exec, gobj_write,
> num_write_bo_handles, 1);
> + r = drm_exec_prepare_array(&exec, gobj_write,
> + num_write_bo_handles, 1);
> drm_exec_retry_on_contention(&exec);
> - if (r) {
> - amdgpu_userq_fence_cleanup(fence);
> + if (r)
> goto exec_fini;
> - }
> }
>
> - for (i = 0; i < num_read_bo_handles; i++) {
> - if (!gobj_read || !gobj_read[i]->resv)
> - continue;
> -
> - dma_resv_add_fence(gobj_read[i]->resv, fence,
> + /* And publish the new fence in the BOs and syncobj */
> + for (i = 0; i < num_read_bo_handles; i++)
> + dma_resv_add_fence(gobj_read[i]->resv, &fence->base,
> DMA_RESV_USAGE_READ);
> - }
>
> - for (i = 0; i < num_write_bo_handles; i++) {
> - if (!gobj_write || !gobj_write[i]->resv)
> - continue;
> -
> - dma_resv_add_fence(gobj_write[i]->resv, fence,
> + for (i = 0; i < num_write_bo_handles; i++)
> + dma_resv_add_fence(gobj_write[i]->resv, &fence->base,
> DMA_RESV_USAGE_WRITE);
> - }
>
> - /* Add the created fence to syncobj/BO's */
> for (i = 0; i < num_syncobj_handles; i++)
> - drm_syncobj_replace_fence(syncobj[i], fence);
> + drm_syncobj_replace_fence(syncobj[i], &fence->base);
>
> +exec_fini:
> /* drop the reference acquired in fence creation function */
> - dma_fence_put(fence);
> + dma_fence_put(&fence->base);
>
> -exec_fini:
> drm_exec_fini(&exec);
> +put_queue:
> + amdgpu_userq_put(queue);
> put_gobj_write:
> for (i = 0; i < num_write_bo_handles; i++)
> drm_gem_object_put(gobj_write[i]);
> @@ -590,15 +575,11 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev,
> void *data,
> kvfree(gobj_read);
> free_syncobj:
> while (entry-- > 0)
> - if (syncobj[entry])
> - drm_syncobj_put(syncobj[entry]);
> + drm_syncobj_put(syncobj[entry]);
> kfree(syncobj);
> free_syncobj_handles:
> kfree(syncobj_handles);
>
> - if (queue)
> - amdgpu_userq_put(queue);
> -
> return r;
> }
>
> @@ -873,8 +854,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file
> *filp,
> * Otherwise, we would gather those references until we don't
> * have any more space left and crash.
> */
> + mutex_lock(&waitq->fence_drv_lock);
> r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
> xa_limit_32b, GFP_KERNEL);
> + mutex_unlock(&waitq->fence_drv_lock);
> if (r)
> goto put_waitq;
>
> --
> 2.43.0
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH 3/8] drm/amdgpu: rework amdgpu_userq_signal_ioctl v2
2026-04-27 18:27 ` [PATCH 3/8] drm/amdgpu: rework amdgpu_userq_signal_ioctl v2 Christian König
2026-04-28 8:49 ` Liang, Prike
@ 2026-04-28 9:35 ` Khatri, Sunil
2026-05-04 12:37 ` Christian König
1 sibling, 1 reply; 30+ messages in thread
From: Khatri, Sunil @ 2026-04-28 9:35 UTC (permalink / raw)
To: christian.koenig, alexander.deucher, Prike.Liang, jesse.zhang; +Cc: amd-gfx
On 27-04-2026 11:57 pm, Christian König wrote:
> This one was fortunately not looking so bad as the wait ioctl path, but
> there were still a few things which could be fixed/improved:
>
> 1. Allocating with GFP_ATOMIC was quite unecessary, we can do that
> before taking the userq_lock.
> 2. Use a new mutex as protection for the fence_drv_xa so that we can do
> memory allocations while holding it.
> 3. Starting the reset timer is unecessary when the fence is already
> signaled when we create it.
> 4. Cleanup error handling, avoid trying to free the queue when we don't
> even got one.
>
> v2: fix incorrect usage of xa_find, destroy the new mutex on error
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 2 +
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 12 +
> .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c | 227 ++++++++----------
> 3 files changed, 119 insertions(+), 122 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index a34b1df513d3..62d155d6e4ed 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -806,6 +806,7 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
> }
>
> queue->doorbell_index = index;
> + mutex_init(&queue->fence_drv_lock);
> xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
> r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
> if (r) {
> @@ -879,6 +880,7 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
> amdgpu_bo_reserve(fpriv->vm.root.bo, true);
> amdgpu_userq_buffer_vas_list_cleanup(adev, queue);
> amdgpu_bo_unreserve(fpriv->vm.root.bo);
> + mutex_destroy(&queue->fence_drv_lock);
> free_queue:
> kfree(queue);
> err_pm_runtime:
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> index 8b8f345b60b6..843ea8ecc5d7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> @@ -66,6 +66,18 @@ struct amdgpu_usermode_queue {
> struct amdgpu_userq_obj db_obj;
> struct amdgpu_userq_obj fw_obj;
> struct amdgpu_userq_obj wptr_obj;
> +
> + /**
> + * @fence_drv_lock: Protecting @fence_drv_xa.
> + */
> + struct mutex fence_drv_lock;
> +
> + /**
> + * @fence_drv_xa:
> + *
> + * References to the external fence drivers returned by wait_ioctl.
> + * Dropped on the next signaled dma_fence or queue destruction.
> + */
> struct xarray fence_drv_xa;
> struct amdgpu_userq_fence_driver *fence_drv;
> struct dma_fence *last_fence;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> index f3d0595e3e61..a2792f05b670 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> @@ -121,6 +121,7 @@ amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq)
> userq->last_fence = NULL;
> amdgpu_userq_walk_and_drop_fence_drv(&userq->fence_drv_xa);
> xa_destroy(&userq->fence_drv_xa);
> + mutex_destroy(&userq->fence_drv_lock);
> /* Drop the queue's ownership reference to fence_drv explicitly */
> amdgpu_userq_fence_driver_put(userq->fence_drv);
> }
> @@ -209,81 +210,84 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
> kref_put(&fence_drv->refcount, amdgpu_userq_fence_driver_destroy);
> }
>
> -static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
> +static int amdgpu_userq_fence_alloc(struct amdgpu_usermode_queue *userq,
> + struct amdgpu_userq_fence **pfence)
> {
> - *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
> - return *userq_fence ? 0 : -ENOMEM;
> + struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
> + struct amdgpu_userq_fence *userq_fence;
> + void * entry;
> +
> + userq_fence = kmalloc(sizeof(*userq_fence), GFP_KERNEL);
> + if (!userq_fence)
> + return -ENOMEM;
> +
> + /*
> + * Get the next unused entry, since we fill from the start this can be
> + * used as size to allocate the array.
> + */
> + mutex_lock(&userq->fence_drv_lock);
> + XA_STATE(xas, &userq->fence_drv_xa, 0);
> +
> + rcu_read_lock();
> + do {
> + entry = xas_find_marked(&xas, ULONG_MAX, XA_FREE_MARK);
> + } while (xas_retry(&xas, entry));
> + rcu_read_unlock();
Functionally its correct but Using xa_for_each to count the no of
entries should suffice for what we need? why doing more work or this
approach is fixing something which xa_for_each does not ?
> +
> + userq_fence->fence_drv_array = kvmalloc_array(xas.xa_index,
> + sizeof(fence_drv),
> + GFP_KERNEL);
> + if (!userq_fence->fence_drv_array) {
> + mutex_unlock(&userq->fence_drv_lock);
> + kfree(userq_fence);
> + return -ENOMEM;
> + }
> +
> + userq_fence->fence_drv_array_count = xas.xa_index;
> + xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array,
> + 0, ULONG_MAX, xas.xa_index, XA_PRESENT);
same count could be used here also to copy, just open questions but the
current code is good to go.
Irrespective of the change the code is Reviewed-by: Sunil Khatri
<sunil.khatri@amd.com>
Regards
Sunil Khatri
> + xa_destroy(&userq->fence_drv_xa);
> +
> + mutex_unlock(&userq->fence_drv_lock);
> +
> + userq_fence->fence_drv = fence_drv;
> + amdgpu_userq_fence_driver_get(fence_drv);
> +
> + *pfence = userq_fence;
> + return 0;
> }
>
> -static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
> - struct amdgpu_userq_fence *userq_fence,
> - u64 seq, struct dma_fence **f)
> +static void amdgpu_userq_fence_init(struct amdgpu_usermode_queue *userq,
> + struct amdgpu_userq_fence *fence,
> + u64 seq)
> {
> - struct amdgpu_userq_fence_driver *fence_drv;
> - struct dma_fence *fence;
> + struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
> unsigned long flags;
> bool signaled = false;
>
> - fence_drv = userq->fence_drv;
> - if (!fence_drv)
> - return -EINVAL;
> -
> - spin_lock_init(&userq_fence->lock);
> - INIT_LIST_HEAD(&userq_fence->link);
> - fence = &userq_fence->base;
> - userq_fence->fence_drv = fence_drv;
> -
> - dma_fence_init64(fence, &amdgpu_userq_fence_ops, &userq_fence->lock,
> + spin_lock_init(&fence->lock);
> + dma_fence_init64(&fence->base, &amdgpu_userq_fence_ops, &fence->lock,
> fence_drv->context, seq);
>
> - amdgpu_userq_fence_driver_get(fence_drv);
> - dma_fence_get(fence);
> -
> - if (!xa_empty(&userq->fence_drv_xa)) {
> - struct amdgpu_userq_fence_driver *stored_fence_drv;
> - unsigned long index, count = 0;
> - int i = 0;
> -
> - xa_lock(&userq->fence_drv_xa);
> - xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv)
> - count++;
> -
> - userq_fence->fence_drv_array =
> - kvmalloc_array(count,
> - sizeof(struct amdgpu_userq_fence_driver *),
> - GFP_ATOMIC);
> -
> - if (userq_fence->fence_drv_array) {
> - xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv) {
> - userq_fence->fence_drv_array[i] = stored_fence_drv;
> - __xa_erase(&userq->fence_drv_xa, index);
> - i++;
> - }
> - }
> -
> - userq_fence->fence_drv_array_count = i;
> - xa_unlock(&userq->fence_drv_xa);
> - } else {
> - userq_fence->fence_drv_array = NULL;
> - userq_fence->fence_drv_array_count = 0;
> - }
> + /* Make sure the fence is visible to the hang detect worker */
> + dma_fence_put(userq->last_fence);
> + userq->last_fence = dma_fence_get(&fence->base);
>
> - /* Check if hardware has already processed the job */
> + /* Check if hardware has already processed the fence */
> spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
> - if (!dma_fence_is_signaled(fence)) {
> - list_add_tail(&userq_fence->link, &fence_drv->fences);
> + if (!dma_fence_is_signaled(&fence->base)) {
> + dma_fence_get(&fence->base);
> + list_add_tail(&fence->link, &fence_drv->fences);
> } else {
> + INIT_LIST_HEAD(&fence->link);
> signaled = true;
> - dma_fence_put(fence);
> }
> spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
>
> if (signaled)
> - amdgpu_userq_fence_put_fence_drv_array(userq_fence);
> -
> - *f = fence;
> -
> - return 0;
> + amdgpu_userq_fence_put_fence_drv_array(fence);
> + else
> + amdgpu_userq_start_hang_detect_work(userq);
> }
>
> static const char *amdgpu_userq_fence_get_driver_name(struct dma_fence *f)
> @@ -404,11 +408,6 @@ static int amdgpu_userq_fence_read_wptr(struct amdgpu_device *adev,
> return r;
> }
>
> -static void amdgpu_userq_fence_cleanup(struct dma_fence *fence)
> -{
> - dma_fence_put(fence);
> -}
> -
> static void
> amdgpu_userq_fence_driver_set_error(struct amdgpu_userq_fence *fence,
> int error)
> @@ -452,13 +451,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
> const unsigned int num_read_bo_handles = args->num_bo_read_handles;
> struct amdgpu_fpriv *fpriv = filp->driver_priv;
> struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
> +
> struct drm_gem_object **gobj_write, **gobj_read;
> u32 *syncobj_handles, num_syncobj_handles;
> - struct amdgpu_userq_fence *userq_fence;
> - struct amdgpu_usermode_queue *queue = NULL;
> - struct drm_syncobj **syncobj = NULL;
> - struct dma_fence *fence;
> + struct amdgpu_usermode_queue *queue;
> + struct amdgpu_userq_fence *fence;
> + struct drm_syncobj **syncobj;
> struct drm_exec exec;
> + void __user *ptr;
> int r, i, entry;
> u64 wptr;
>
> @@ -470,13 +470,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
> return -EINVAL;
>
> num_syncobj_handles = args->num_syncobj_handles;
> - syncobj_handles = memdup_array_user(u64_to_user_ptr(args->syncobj_handles),
> - num_syncobj_handles, sizeof(u32));
> + ptr = u64_to_user_ptr(args->syncobj_handles);
> + syncobj_handles = memdup_array_user(ptr, num_syncobj_handles,
> + sizeof(u32));
> if (IS_ERR(syncobj_handles))
> return PTR_ERR(syncobj_handles);
>
> - /* Array of pointers to the looked up syncobjs */
> - syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj), GFP_KERNEL);
> + syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj),
> + GFP_KERNEL);
> if (!syncobj) {
> r = -ENOMEM;
> goto free_syncobj_handles;
> @@ -490,21 +491,17 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
> }
> }
>
> - r = drm_gem_objects_lookup(filp,
> - u64_to_user_ptr(args->bo_read_handles),
> - num_read_bo_handles,
> - &gobj_read);
> + ptr = u64_to_user_ptr(args->bo_read_handles);
> + r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
> if (r)
> goto free_syncobj;
>
> - r = drm_gem_objects_lookup(filp,
> - u64_to_user_ptr(args->bo_write_handles),
> - num_write_bo_handles,
> + ptr = u64_to_user_ptr(args->bo_write_handles);
> + r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
> &gobj_write);
> if (r)
> goto put_gobj_read;
>
> - /* Retrieve the user queue */
> queue = amdgpu_userq_get(userq_mgr, args->queue_id);
> if (!queue) {
> r = -ENOENT;
> @@ -513,73 +510,61 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>
> r = amdgpu_userq_fence_read_wptr(adev, queue, &wptr);
> if (r)
> - goto put_gobj_write;
> + goto put_queue;
>
> - r = amdgpu_userq_fence_alloc(&userq_fence);
> + r = amdgpu_userq_fence_alloc(queue, &fence);
> if (r)
> - goto put_gobj_write;
> + goto put_queue;
>
> /* We are here means UQ is active, make sure the eviction fence is valid */
> amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
>
> - /* Create a new fence */
> - r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
> - if (r) {
> - mutex_unlock(&userq_mgr->userq_mutex);
> - kfree(userq_fence);
> - goto put_gobj_write;
> - }
> + /* Create the new fence */
> + amdgpu_userq_fence_init(queue, fence, wptr);
>
> - dma_fence_put(queue->last_fence);
> - queue->last_fence = dma_fence_get(fence);
> - amdgpu_userq_start_hang_detect_work(queue);
> mutex_unlock(&userq_mgr->userq_mutex);
>
> + /*
> + * This needs to come after the fence is created since
> + * amdgpu_userq_ensure_ev_fence() can't be called while holding the resv
> + * locks.
> + */
> drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
> (num_read_bo_handles + num_write_bo_handles));
>
> - /* Lock all BOs with retry handling */
> drm_exec_until_all_locked(&exec) {
> - r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1);
> + r = drm_exec_prepare_array(&exec, gobj_read,
> + num_read_bo_handles, 1);
> drm_exec_retry_on_contention(&exec);
> - if (r) {
> - amdgpu_userq_fence_cleanup(fence);
> + if (r)
> goto exec_fini;
> - }
>
> - r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1);
> + r = drm_exec_prepare_array(&exec, gobj_write,
> + num_write_bo_handles, 1);
> drm_exec_retry_on_contention(&exec);
> - if (r) {
> - amdgpu_userq_fence_cleanup(fence);
> + if (r)
> goto exec_fini;
> - }
> }
>
> - for (i = 0; i < num_read_bo_handles; i++) {
> - if (!gobj_read || !gobj_read[i]->resv)
> - continue;
> -
> - dma_resv_add_fence(gobj_read[i]->resv, fence,
> + /* And publish the new fence in the BOs and syncobj */
> + for (i = 0; i < num_read_bo_handles; i++)
> + dma_resv_add_fence(gobj_read[i]->resv, &fence->base,
> DMA_RESV_USAGE_READ);
> - }
>
> - for (i = 0; i < num_write_bo_handles; i++) {
> - if (!gobj_write || !gobj_write[i]->resv)
> - continue;
> -
> - dma_resv_add_fence(gobj_write[i]->resv, fence,
> + for (i = 0; i < num_write_bo_handles; i++)
> + dma_resv_add_fence(gobj_write[i]->resv, &fence->base,
> DMA_RESV_USAGE_WRITE);
> - }
>
> - /* Add the created fence to syncobj/BO's */
> for (i = 0; i < num_syncobj_handles; i++)
> - drm_syncobj_replace_fence(syncobj[i], fence);
> + drm_syncobj_replace_fence(syncobj[i], &fence->base);
>
> +exec_fini:
> /* drop the reference acquired in fence creation function */
> - dma_fence_put(fence);
> + dma_fence_put(&fence->base);
>
> -exec_fini:
> drm_exec_fini(&exec);
> +put_queue:
> + amdgpu_userq_put(queue);
> put_gobj_write:
> for (i = 0; i < num_write_bo_handles; i++)
> drm_gem_object_put(gobj_write[i]);
> @@ -590,15 +575,11 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
> kvfree(gobj_read);
> free_syncobj:
> while (entry-- > 0)
> - if (syncobj[entry])
> - drm_syncobj_put(syncobj[entry]);
> + drm_syncobj_put(syncobj[entry]);
> kfree(syncobj);
> free_syncobj_handles:
> kfree(syncobj_handles);
>
> - if (queue)
> - amdgpu_userq_put(queue);
> -
> return r;
> }
>
> @@ -873,8 +854,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
> * Otherwise, we would gather those references until we don't
> * have any more space left and crash.
> */
> + mutex_lock(&waitq->fence_drv_lock);
> r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
> xa_limit_32b, GFP_KERNEL);
> + mutex_unlock(&waitq->fence_drv_lock);
> if (r)
> goto put_waitq;
>
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH 4/8] drm/amdgpu: remove almost all calls to amdgpu_userq_detect_and_reset_queues
2026-04-27 18:27 ` [PATCH 4/8] drm/amdgpu: remove almost all calls to amdgpu_userq_detect_and_reset_queues Christian König
@ 2026-04-28 9:37 ` Khatri, Sunil
2026-04-29 3:16 ` Liang, Prike
1 sibling, 0 replies; 30+ messages in thread
From: Khatri, Sunil @ 2026-04-28 9:37 UTC (permalink / raw)
To: christian.koenig, alexander.deucher, Prike.Liang, jesse.zhang; +Cc: amd-gfx
Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
On 27-04-2026 11:57 pm, Christian König wrote:
> Well the reset handling seems broken on multiple levels.
>
> As first step of fixing this remove most calls to the hang detection.
> That function should only be called after we run into a timeout! And *NOT*
> as random check spread over the code in multiple places.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 38 +++++++++--------------
> 1 file changed, 14 insertions(+), 24 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index 62d155d6e4ed..fb3c0425cf22 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -345,23 +345,18 @@ static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue)
> struct amdgpu_device *adev = uq_mgr->adev;
> const struct amdgpu_userq_funcs *userq_funcs =
> adev->userq_funcs[queue->queue_type];
> - bool found_hung_queue = false;
> - int r = 0;
> + int r;
>
> if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
> r = userq_funcs->preempt(queue);
> if (r) {
> queue->state = AMDGPU_USERQ_STATE_HUNG;
> - found_hung_queue = true;
> + return r;
> } else {
> queue->state = AMDGPU_USERQ_STATE_PREEMPTED;
> }
> }
> -
> - if (found_hung_queue)
> - amdgpu_userq_detect_and_reset_queues(uq_mgr);
> -
> - return r;
> + return 0;
> }
>
> static int amdgpu_userq_restore_helper(struct amdgpu_usermode_queue *queue)
> @@ -390,24 +385,21 @@ static int amdgpu_userq_unmap_helper(struct amdgpu_usermode_queue *queue)
> struct amdgpu_device *adev = uq_mgr->adev;
> const struct amdgpu_userq_funcs *userq_funcs =
> adev->userq_funcs[queue->queue_type];
> - bool found_hung_queue = false;
> - int r = 0;
> + int r;
>
> if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) ||
> - (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {
> + (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {
> +
> r = userq_funcs->unmap(queue);
> if (r) {
> queue->state = AMDGPU_USERQ_STATE_HUNG;
> - found_hung_queue = true;
> + return r;
> } else {
> queue->state = AMDGPU_USERQ_STATE_UNMAPPED;
> }
> }
>
> - if (found_hung_queue)
> - amdgpu_userq_detect_and_reset_queues(uq_mgr);
> -
> - return r;
> + return 0;
> }
>
> static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue)
> @@ -416,19 +408,19 @@ static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue)
> struct amdgpu_device *adev = uq_mgr->adev;
> const struct amdgpu_userq_funcs *userq_funcs =
> adev->userq_funcs[queue->queue_type];
> - int r = 0;
> + int r;
>
> if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) {
> r = userq_funcs->map(queue);
> if (r) {
> queue->state = AMDGPU_USERQ_STATE_HUNG;
> - amdgpu_userq_detect_and_reset_queues(uq_mgr);
> + return r;
> } else {
> queue->state = AMDGPU_USERQ_STATE_MAPPED;
> }
> }
>
> - return r;
> + return 0;
> }
>
> static void amdgpu_userq_wait_for_last_fence(struct amdgpu_usermode_queue *queue)
> @@ -654,7 +646,6 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_que
> #if defined(CONFIG_DEBUG_FS)
> debugfs_remove_recursive(queue->debugfs_queue);
> #endif
> - amdgpu_userq_detect_and_reset_queues(uq_mgr);
> r = amdgpu_userq_unmap_helper(queue);
> /*TODO: It requires a reset for userq hw unmap error*/
> if (r) {
> @@ -1270,7 +1261,6 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
> unsigned long queue_id;
> int ret = 0, r;
>
> - amdgpu_userq_detect_and_reset_queues(uq_mgr);
> /* Try to unmap all the queues in this process ctx */
> xa_for_each(&uq_mgr->userq_xa, queue_id, queue) {
> r = amdgpu_userq_preempt_helper(queue);
> @@ -1278,9 +1268,11 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
> ret = r;
> }
>
> - if (ret)
> + if (ret) {
> drm_file_err(uq_mgr->file,
> "Couldn't unmap all the queues, eviction failed ret=%d\n", ret);
> + amdgpu_userq_detect_and_reset_queues(uq_mgr);
> + }
> return ret;
> }
>
> @@ -1380,7 +1372,6 @@ int amdgpu_userq_suspend(struct amdgpu_device *adev)
> uqm = queue->userq_mgr;
> cancel_delayed_work_sync(&uqm->resume_work);
> guard(mutex)(&uqm->userq_mutex);
> - amdgpu_userq_detect_and_reset_queues(uqm);
> if (adev->in_s0ix)
> r = amdgpu_userq_preempt_helper(queue);
> else
> @@ -1439,7 +1430,6 @@ int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev,
> if (((queue->queue_type == AMDGPU_HW_IP_GFX) ||
> (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) &&
> (queue->xcp_id == idx)) {
> - amdgpu_userq_detect_and_reset_queues(uqm);
> r = amdgpu_userq_preempt_helper(queue);
> if (r)
> ret = r;
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH 7/8] drm/amdgpu: fix handling in amdgpu_userq_create
2026-04-27 18:27 ` [PATCH 7/8] drm/amdgpu: fix handling in amdgpu_userq_create Christian König
@ 2026-04-28 11:26 ` Khatri, Sunil
2026-05-07 6:13 ` Liang, Prike
1 sibling, 0 replies; 30+ messages in thread
From: Khatri, Sunil @ 2026-04-28 11:26 UTC (permalink / raw)
To: christian.koenig, alexander.deucher, Prike.Liang, jesse.zhang; +Cc: amd-gfx
On 27-04-2026 11:57 pm, Christian König wrote:
> Well mostly the same issues the other code had as well:
>
> 1. Memory allocation while holding the userq_mutex lock is forbidden!
> 2. Things were created/started/published in the wrong order.
> 3. Error messages on invalid input parameters can spam the logs.
> 4. Error messages on memory allocation failures are usually superflous
> as well.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 118 ++++++++++------------
> 1 file changed, 52 insertions(+), 66 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index 0374a70b631c..28e0695b53e7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -718,14 +718,14 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
> const struct amdgpu_userq_funcs *uq_funcs;
> struct amdgpu_usermode_queue *queue;
> struct amdgpu_db_info db_info;
> - bool skip_map_queue;
> - u32 qid;
> uint64_t index;
> - int r = 0;
> - int priority =
> - (args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK) >>
> - AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT;
> + int priority;
> + u32 qid;
> + int r;
>
> + priority =
> + (args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK)
> + >> AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT;
> r = amdgpu_userq_priority_permit(filp, priority);
> if (r)
> return r;
> @@ -738,40 +738,43 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
>
> uq_funcs = adev->userq_funcs[args->in.ip_type];
> if (!uq_funcs) {
> - drm_file_err(uq_mgr->file, "Usermode queue is not supported for this IP (%u)\n",
> - args->in.ip_type);
> r = -EINVAL;
> goto err_pm_runtime;
> }
>
> queue = kzalloc(sizeof(struct amdgpu_usermode_queue), GFP_KERNEL);
> if (!queue) {
> - drm_file_err(uq_mgr->file, "Failed to allocate memory for queue\n");
> r = -ENOMEM;
> goto err_pm_runtime;
> }
>
> + kref_init(&queue->refcount);
> INIT_LIST_HEAD(&queue->userq_va_list);
> queue->doorbell_handle = args->in.doorbell_handle;
> queue->queue_type = args->in.ip_type;
> queue->vm = &fpriv->vm;
> queue->priority = priority;
> -
> - db_info.queue_type = queue->queue_type;
> - db_info.doorbell_handle = queue->doorbell_handle;
> - db_info.db_obj = &queue->db_obj;
> - db_info.doorbell_offset = args->in.doorbell_offset;
> -
> queue->userq_mgr = uq_mgr;
> + INIT_DELAYED_WORK(&queue->hang_detect_work,
> + amdgpu_userq_hang_detect_work);
>
> - /* Validate the userq virtual address.*/
> - r = amdgpu_bo_reserve(fpriv->vm.root.bo, false);
> + mutex_init(&queue->fence_drv_lock);
> + xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
> + r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
> if (r)
> goto free_queue;
>
> - if (amdgpu_userq_input_va_validate(adev, queue, args->in.queue_va, args->in.queue_size) ||
> - amdgpu_userq_input_va_validate(adev, queue, args->in.rptr_va, AMDGPU_GPU_PAGE_SIZE) ||
> - amdgpu_userq_input_va_validate(adev, queue, args->in.wptr_va, AMDGPU_GPU_PAGE_SIZE)) {
> + /* Make sure the queue can actually run with those virtual addresses. */
> + r = amdgpu_bo_reserve(fpriv->vm.root.bo, false);
> + if (r)
> + goto free_fence_drv;
> +
> + if (amdgpu_userq_input_va_validate(adev, queue, args->in.queue_va,
> + args->in.queue_size) ||
> + amdgpu_userq_input_va_validate(adev, queue, args->in.rptr_va,
> + AMDGPU_GPU_PAGE_SIZE) ||
> + amdgpu_userq_input_va_validate(adev, queue, args->in.wptr_va,
> + AMDGPU_GPU_PAGE_SIZE)) {
> r = -EINVAL;
> amdgpu_bo_unreserve(fpriv->vm.root.bo);
> goto clean_mapping;
> @@ -779,6 +782,10 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
> amdgpu_bo_unreserve(fpriv->vm.root.bo);
>
> /* Convert relative doorbell offset into absolute doorbell index */
> + db_info.queue_type = queue->queue_type;
> + db_info.doorbell_handle = queue->doorbell_handle;
> + db_info.db_obj = &queue->db_obj;
> + db_info.doorbell_offset = args->in.doorbell_offset;
> index = amdgpu_userq_get_doorbell_index(uq_mgr, &db_info, filp);
> if (index == (uint64_t)-EINVAL) {
> drm_file_err(uq_mgr->file, "Failed to get doorbell for queue\n");
> @@ -787,82 +794,61 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
> }
>
> queue->doorbell_index = index;
> - mutex_init(&queue->fence_drv_lock);
> - xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
> - r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
> - if (r) {
> - drm_file_err(uq_mgr->file, "Failed to alloc fence driver\n");
> - goto clean_mapping;
> - }
> -
> r = uq_funcs->mqd_create(queue, &args->in);
> if (r) {
> drm_file_err(uq_mgr->file, "Failed to create Queue\n");
> - goto clean_fence_driver;
> + goto clean_mapping;
> }
>
> + r = xa_err(xa_store_irq(&adev->userq_doorbell_xa, index, queue,
> + GFP_KERNEL));
> + if (r)
> + goto clean_mqd;
> +
> amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
>
> /* don't map the queue if scheduling is halted */
> - if (adev->userq_halt_for_enforce_isolation &&
> - ((queue->queue_type == AMDGPU_HW_IP_GFX) ||
> - (queue->queue_type == AMDGPU_HW_IP_COMPUTE)))
> - skip_map_queue = true;
> - else
> - skip_map_queue = false;
> - if (!skip_map_queue) {
> + if (!adev->userq_halt_for_enforce_isolation ||
> + ((queue->queue_type != AMDGPU_HW_IP_GFX) &&
> + (queue->queue_type != AMDGPU_HW_IP_COMPUTE))) {
> r = amdgpu_userq_map_helper(queue);
> if (r) {
> drm_file_err(uq_mgr->file, "Failed to map Queue\n");
> - goto clean_mqd;
> + mutex_unlock(&uq_mgr->userq_mutex);
> + goto clean_doorbell;
> }
> }
>
> - /* drop this refcount during queue destroy */
> - kref_init(&queue->refcount);
> -
> - /* Wait for mode-1 reset to complete */
> - down_read(&adev->reset_domain->sem);
We dont need this anymore? If not the patch looks good, Reviewed-by:
Sunil Khatri <sunil.khatri@amd.com>
regards
Sunil
> + atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
> + mutex_unlock(&uq_mgr->userq_mutex);
>
> r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
> - XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT), GFP_KERNEL);
> - if (r) {
> - if (!skip_map_queue)
> - amdgpu_userq_unmap_helper(queue);
> - r = -ENOMEM;
> - goto clean_reset_domain;
> - }
> -
> - r = xa_err(xa_store_irq(&adev->userq_doorbell_xa, index, queue, GFP_KERNEL));
> + XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
> + GFP_KERNEL);
> if (r) {
> - xa_erase(&uq_mgr->userq_xa, qid);
> - if (!skip_map_queue)
> - amdgpu_userq_unmap_helper(queue);
> - goto clean_reset_domain;
> + /*
> + * This drops the last reference which should take care of
> + * all cleanup.
> + */
> + amdgpu_userq_put(queue);
> + return r;
> }
> - up_read(&adev->reset_domain->sem);
>
> amdgpu_debugfs_userq_init(filp, queue, qid);
> - INIT_DELAYED_WORK(&queue->hang_detect_work,
> - amdgpu_userq_hang_detect_work);
> -
> args->out.queue_id = qid;
> - atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
> - mutex_unlock(&uq_mgr->userq_mutex);
> return 0;
>
> -clean_reset_domain:
> - up_read(&adev->reset_domain->sem);
> +clean_doorbell:
> + xa_erase_irq(&adev->userq_doorbell_xa, index);
> clean_mqd:
> - mutex_unlock(&uq_mgr->userq_mutex);
> uq_funcs->mqd_destroy(queue);
> -clean_fence_driver:
> - amdgpu_userq_fence_driver_free(queue);
> clean_mapping:
> amdgpu_bo_reserve(fpriv->vm.root.bo, true);
> amdgpu_userq_buffer_vas_list_cleanup(adev, queue);
> amdgpu_bo_unreserve(fpriv->vm.root.bo);
> mutex_destroy(&queue->fence_drv_lock);
> +free_fence_drv:
> + amdgpu_userq_fence_driver_free(queue);
> free_queue:
> kfree(queue);
> err_pm_runtime:
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH 8/8] drm/amdgpu: restructure VM state machine v2
2026-04-27 18:27 ` [PATCH 8/8] drm/amdgpu: restructure VM state machine v2 Christian König
@ 2026-04-28 12:20 ` Khatri, Sunil
2026-05-08 2:24 ` Liang, Prike
2026-05-25 23:29 ` Timur Kristóf
2 siblings, 0 replies; 30+ messages in thread
From: Khatri, Sunil @ 2026-04-28 12:20 UTC (permalink / raw)
To: christian.koenig, alexander.deucher, Prike.Liang, jesse.zhang; +Cc: amd-gfx
On 27-04-2026 11:57 pm, Christian König wrote:
> Instead of coming up with more sophisticated names for states a VM BO
> can be in, group them by the type of BO first and then by the state.
>
> So we end with BO type kernel, always_valid and individual and then states
> evicted, moved and idle.
>
> Not much functional change, except that evicted_user is moved back
> together with the other BOs again which makes the handling in
> amdgpu_vm_validate() a bit more complex.
>
> Also fixes a problem with user queues and amdgpu_vm_ready(). We didn't
> considered the VM ready when user BOs were not ideally placed, harmless
> performance impact for kernel queues but a complete show stopper for
> userqueues.
>
> v2: fix a few typos in comments, rename the BO types to make them more
> descriptive, fix a couple of bugs found during testing
> v3: squashed together with revert to old status lock handling, looks
> like the first patch still had some bug which this one here should fix.
> Fix a missing lock around debugfs printing.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 65 +--
> drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 534 +++++++++-------------
> drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 69 ++-
> drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 4 -
> 4 files changed, 298 insertions(+), 374 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index 28e0695b53e7..07e30acbdc3a 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -805,41 +805,50 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
> if (r)
> goto clean_mqd;
>
> - amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
> + /* Make sure a concurrent close operation doesn't kill our userq */
> + kref_get(&queue->refcount);
> + r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
> + XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
> + GFP_KERNEL);
> + if (r) {
> + /*
> + * This drops the extra and last reference which should take
> + * care of all cleanup.
> + */
> + amdgpu_userq_put(queue);
> + amdgpu_userq_put(queue);
cleaning up here sounds more clean to be that do an extra put. We need
to make sure that after put the destroy clean up is doing same or some
extra clean up is needed here.
> + return r;
> + }
>
> /* don't map the queue if scheduling is halted */
> + /* TODO: That is racy! */
> if (!adev->userq_halt_for_enforce_isolation ||
> ((queue->queue_type != AMDGPU_HW_IP_GFX) &&
> (queue->queue_type != AMDGPU_HW_IP_COMPUTE))) {
> +
> + /* TODO: We could skip mapping if ev fence is signaled */
> + amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr,
> + &fpriv->evf_mgr);
I really dont understand this part? Earlier the condition was different
and so the combination of them...
Regards
Sunil Khatri
> r = amdgpu_userq_map_helper(queue);
> if (r) {
> drm_file_err(uq_mgr->file, "Failed to map Queue\n");
> mutex_unlock(&uq_mgr->userq_mutex);
> - goto clean_doorbell;
> + /* Prevent racing with close */
> + if (xa_erase(&uq_mgr->userq_xa, qid) == queue)
> + amdgpu_userq_put(queue);
> + amdgpu_userq_put(queue);
> + return r;
> }
> + mutex_unlock(&uq_mgr->userq_mutex);
> }
>
> atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
> - mutex_unlock(&uq_mgr->userq_mutex);
> -
> - r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
> - XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
> - GFP_KERNEL);
> - if (r) {
> - /*
> - * This drops the last reference which should take care of
> - * all cleanup.
> - */
> - amdgpu_userq_put(queue);
> - return r;
> - }
>
> amdgpu_debugfs_userq_init(filp, queue, qid);
> + amdgpu_userq_put(queue);
> args->out.queue_id = qid;
> return 0;
>
> -clean_doorbell:
> - xa_erase_irq(&adev->userq_doorbell_xa, index);
> clean_mqd:
> uq_funcs->mqd_destroy(queue);
> clean_mapping:
> @@ -1035,12 +1044,12 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
> struct amdgpu_bo *bo;
> int ret;
>
> - spin_lock(&vm->status_lock);
> - while (!list_empty(&vm->invalidated)) {
> - bo_va = list_first_entry(&vm->invalidated,
> + spin_lock(&vm->individual_lock);
> + while (!list_empty(&vm->always_valid.evicted)) {
> + bo_va = list_first_entry(&vm->always_valid.evicted,
> struct amdgpu_bo_va,
> base.vm_status);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> bo = bo_va->base.bo;
> ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2);
> @@ -1052,14 +1061,14 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
> if (ret)
> return ret;
>
> - /* This moves the bo_va to the done list */
> + /* This moves the bo_va to the idle list */
> ret = amdgpu_vm_bo_update(adev, bo_va, false);
> if (ret)
> return ret;
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->individual_lock);
> }
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> return 0;
> }
> @@ -1091,7 +1100,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
> if (unlikely(ret))
> goto unlock_all;
>
> - ret = amdgpu_vm_lock_done_list(vm, &exec, 1);
> + ret = amdgpu_vm_lock_individual(vm, &exec, 1);
> drm_exec_retry_on_contention(&exec);
> if (unlikely(ret))
> goto unlock_all;
> @@ -1134,7 +1143,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
>
> key = 0;
> /* Validate User Ptr BOs */
> - list_for_each_entry(bo_va, &vm->done, base.vm_status) {
> + list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status) {
> bo = bo_va->base.bo;
> if (!bo)
> continue;
> @@ -1184,10 +1193,10 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
>
> /*
> * We need to wait for all VM updates to finish before restarting the
> - * queues. Using the done list like that is now ok since everything is
> + * queues. Using the idle list like that is now ok since everything is
> * locked in place.
> */
> - list_for_each_entry(bo_va, &vm->done, base.vm_status)
> + list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status)
> dma_fence_wait(bo_va->last_pt_update, false);
> dma_fence_wait(vm->last_update, false);
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> index 6b58b0ecfa06..9f71a146f1f6 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -138,6 +138,47 @@ static void amdgpu_vm_assert_locked(struct amdgpu_vm *vm)
> dma_resv_assert_held(vm->root.bo->tbo.base.resv);
> }
>
> +/* Initialize the amdgpu_vm_bo_status object */
> +static void amdgpu_vm_bo_status_init(struct amdgpu_vm_bo_status *lists)
> +{
> + INIT_LIST_HEAD(&lists->evicted);
> + INIT_LIST_HEAD(&lists->moved);
> + INIT_LIST_HEAD(&lists->idle);
> +}
> +
> +/*
> + * Make sure we have the lock to modify the vm_bo status and return the object
> + * with the status lists.
> + */
> +static struct amdgpu_vm_bo_status *
> +amdgpu_vm_bo_lock_lists(struct amdgpu_vm_bo_base *vm_bo)
> +{
> + struct amdgpu_vm *vm = vm_bo->vm;
> + struct amdgpu_bo *bo = vm_bo->bo;
> +
> + if (amdgpu_vm_is_bo_always_valid(vm, bo)) {
> + /* No extra locking needed, protected by the root PD resv lock */
> + amdgpu_vm_assert_locked(vm);
> +
> + if (bo->tbo.type == ttm_bo_type_kernel)
> + return &vm->kernel;
> +
> + return &vm->always_valid;
> + }
> +
> + spin_lock(&vm_bo->vm->individual_lock);
> + return &vm->individual;
> +}
> +
> +/* Eventually unlock the status list lock again */
> +static void amdgpu_vm_bo_unlock_lists(struct amdgpu_vm_bo_base *vm_bo)
> +{
> + if (amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
> + amdgpu_vm_assert_locked(vm_bo->vm);
> + else
> + spin_unlock(&vm_bo->vm->individual_lock);
> +}
> +
> /**
> * amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid
> *
> @@ -157,37 +198,44 @@ bool amdgpu_vm_is_bo_always_valid(struct amdgpu_vm *vm, struct amdgpu_bo *bo)
> *
> * @vm_bo: vm_bo which is evicted
> *
> - * State for PDs/PTs and per VM BOs which are not at the location they should
> - * be.
> + * State for vm_bo objects meaning the underlying BO was evicted and need to
> + * move in place again.
> */
> static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
> {
> - struct amdgpu_vm *vm = vm_bo->vm;
> - struct amdgpu_bo *bo = vm_bo->bo;
> + struct amdgpu_vm_bo_status *lists;
>
> + lists = amdgpu_vm_bo_lock_lists(vm_bo);
> vm_bo->moved = true;
> - amdgpu_vm_assert_locked(vm);
> - spin_lock(&vm_bo->vm->status_lock);
> - if (bo->tbo.type == ttm_bo_type_kernel)
> - list_move(&vm_bo->vm_status, &vm->evicted);
> - else
> - list_move_tail(&vm_bo->vm_status, &vm->evicted);
> - spin_unlock(&vm_bo->vm->status_lock);
> + list_move(&vm_bo->vm_status, &lists->evicted);
> + amdgpu_vm_bo_unlock_lists(vm_bo);
> }
> /**
> * amdgpu_vm_bo_moved - vm_bo is moved
> *
> * @vm_bo: vm_bo which is moved
> *
> - * State for per VM BOs which are moved, but that change is not yet reflected
> - * in the page tables.
> + * State for vm_bo objects meaning the underlying BO was moved but the new
> + * location not yet reflected in the page tables.
> */
> static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
> {
> - amdgpu_vm_assert_locked(vm_bo->vm);
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
> - spin_unlock(&vm_bo->vm->status_lock);
> + struct amdgpu_vm_bo_status *lists;
> + struct amdgpu_bo *bo = vm_bo->bo;
> +
> + /*
> + * The root PD doesn't have a parent PDE and goes directly into the
> + * idle state.
> + */
> + lists = amdgpu_vm_bo_lock_lists(vm_bo);
> + if (bo && bo->tbo.type == ttm_bo_type_kernel && !bo->parent) {
> + vm_bo->moved = false;
> + list_move(&vm_bo->vm_status, &lists->idle);
> + } else {
> + vm_bo->moved = true;
> + list_move(&vm_bo->vm_status, &lists->moved);
> + }
> + amdgpu_vm_bo_unlock_lists(vm_bo);
> }
>
> /**
> @@ -195,120 +243,43 @@ static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
> *
> * @vm_bo: vm_bo which is now idle
> *
> - * State for PDs/PTs and per VM BOs which have gone through the state machine
> - * and are now idle.
> + * State for vm_bo objects meaning we are done with the state machine and no
> + * further action is necessary.
> */
> static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
> {
> - amdgpu_vm_assert_locked(vm_bo->vm);
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
> - spin_unlock(&vm_bo->vm->status_lock);
> - vm_bo->moved = false;
> -}
> + struct amdgpu_vm_bo_status *lists;
>
> -/**
> - * amdgpu_vm_bo_invalidated - vm_bo is invalidated
> - *
> - * @vm_bo: vm_bo which is now invalidated
> - *
> - * State for normal BOs which are invalidated and that change not yet reflected
> - * in the PTs.
> - */
> -static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
> -{
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
> - spin_unlock(&vm_bo->vm->status_lock);
> -}
> -
> -/**
> - * amdgpu_vm_bo_evicted_user - vm_bo is evicted
> - *
> - * @vm_bo: vm_bo which is evicted
> - *
> - * State for BOs used by user mode queues which are not at the location they
> - * should be.
> - */
> -static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)
> -{
> - vm_bo->moved = true;
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user);
> - spin_unlock(&vm_bo->vm->status_lock);
> -}
> -
> -/**
> - * amdgpu_vm_bo_relocated - vm_bo is reloacted
> - *
> - * @vm_bo: vm_bo which is relocated
> - *
> - * State for PDs/PTs which needs to update their parent PD.
> - * For the root PD, just move to idle state.
> - */
> -static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
> -{
> - amdgpu_vm_assert_locked(vm_bo->vm);
> - if (vm_bo->bo->parent) {
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
> - spin_unlock(&vm_bo->vm->status_lock);
> - } else {
> - amdgpu_vm_bo_idle(vm_bo);
> - }
> -}
> -
> -/**
> - * amdgpu_vm_bo_done - vm_bo is done
> - *
> - * @vm_bo: vm_bo which is now done
> - *
> - * State for normal BOs which are invalidated and that change has been updated
> - * in the PTs.
> - */
> -static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
> -{
> - amdgpu_vm_assert_locked(vm_bo->vm);
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->done);
> - spin_unlock(&vm_bo->vm->status_lock);
> + lists = amdgpu_vm_bo_lock_lists(vm_bo);
> + if (!amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
> + vm_bo->moved = false;
> + list_move(&vm_bo->vm_status, &lists->idle);
> + amdgpu_vm_bo_unlock_lists(vm_bo);
> }
>
> /**
> * amdgpu_vm_bo_reset_state_machine - reset the vm_bo state machine
> * @vm: the VM which state machine to reset
> *
> - * Move all vm_bo object in the VM into a state where they will be updated
> - * again during validation.
> + * Move all vm_bo object in the VM into a state where their location will be
> + * updated in the page tables again.
> */
> static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
> {
> - struct amdgpu_vm_bo_base *vm_bo, *tmp;
> -
> amdgpu_vm_assert_locked(vm);
> + list_splice_init(&vm->kernel.idle, &vm->kernel.moved);
> + list_splice_init(&vm->always_valid.idle, &vm->always_valid.moved);
>
> - spin_lock(&vm->status_lock);
> - list_splice_init(&vm->done, &vm->invalidated);
> - list_for_each_entry(vm_bo, &vm->invalidated, vm_status)
> - vm_bo->moved = true;
> -
> - list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) {
> - struct amdgpu_bo *bo = vm_bo->bo;
> -
> - vm_bo->moved = true;
> - if (!bo || bo->tbo.type != ttm_bo_type_kernel)
> - list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
> - else if (bo->parent)
> - list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
> - }
> - spin_unlock(&vm->status_lock);
> + spin_lock(&vm->individual_lock);
> + list_splice_init(&vm->individual.idle, &vm->individual.moved);
> + spin_unlock(&vm->individual_lock);
> }
>
> /**
> * amdgpu_vm_update_shared - helper to update shared memory stat
> * @base: base structure for tracking BO usage in a VM
> *
> - * Takes the vm status_lock and updates the shared memory stat. If the basic
> + * Takes the vm stats_lock and updates the shared memory stat. If the basic
> * stat changed (e.g. buffer was moved) amdgpu_vm_update_stats need to be called
> * as well.
> */
> @@ -321,7 +292,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base)
> bool shared;
>
> dma_resv_assert_held(bo->tbo.base.resv);
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->stats_lock);
> shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
> if (base->shared != shared) {
> base->shared = shared;
> @@ -333,7 +304,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base)
> vm->stats[bo_memtype].drm.private += size;
> }
> }
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->stats_lock);
> }
>
> /**
> @@ -358,11 +329,11 @@ void amdgpu_vm_bo_update_shared(struct amdgpu_bo *bo)
> * be bo->tbo.resource
> * @sign: if we should add (+1) or subtract (-1) from the stat
> *
> - * Caller need to have the vm status_lock held. Useful for when multiple update
> + * Caller need to have the vm stats_lock held. Useful for when multiple update
> * need to happen at the same time.
> */
> static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
> - struct ttm_resource *res, int sign)
> + struct ttm_resource *res, int sign)
> {
> struct amdgpu_vm *vm = base->vm;
> struct amdgpu_bo *bo = base->bo;
> @@ -386,7 +357,8 @@ static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
> */
> if (bo->flags & AMDGPU_GEM_CREATE_DISCARDABLE)
> vm->stats[res_memtype].drm.purgeable += size;
> - if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(res_memtype)))
> + if (!(bo->preferred_domains &
> + amdgpu_mem_type_to_domain(res_memtype)))
> vm->stats[bo_memtype].evicted += size;
> }
> }
> @@ -405,9 +377,9 @@ void amdgpu_vm_update_stats(struct amdgpu_vm_bo_base *base,
> {
> struct amdgpu_vm *vm = base->vm;
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->stats_lock);
> amdgpu_vm_update_stats_locked(base, res, sign);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->stats_lock);
> }
>
> /**
> @@ -428,37 +400,34 @@ void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
> base->next = NULL;
> INIT_LIST_HEAD(&base->vm_status);
>
> + dma_resv_assert_held(vm->root.bo->tbo.base.resv);
> if (!bo)
> return;
> +
> base->next = bo->vm_bo;
> bo->vm_bo = base;
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->stats_lock);
> base->shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
> amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->stats_lock);
>
> - if (!amdgpu_vm_is_bo_always_valid(vm, bo))
> + if (!amdgpu_vm_is_bo_always_valid(vm, bo)) {
> + amdgpu_vm_bo_idle(base);
> return;
> -
> - dma_resv_assert_held(vm->root.bo->tbo.base.resv);
> + }
>
> ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move);
> - if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
> - amdgpu_vm_bo_relocated(base);
> - else
> - amdgpu_vm_bo_idle(base);
>
> + /*
> + * When a per VM isn't in the desired domain put it into the evicted
> + * state to make sure that it gets validated on the next best occasion.
> + */
> if (bo->preferred_domains &
> amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type))
> - return;
> -
> - /*
> - * we checked all the prerequisites, but it looks like this per vm bo
> - * is currently evicted. add the bo to the evicted list to make sure it
> - * is validated on next vm use to avoid fault.
> - * */
> - amdgpu_vm_bo_evicted(base);
> + amdgpu_vm_bo_moved(base);
> + else
> + amdgpu_vm_bo_evicted(base);
> }
>
> /**
> @@ -479,41 +448,41 @@ int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec,
> }
>
> /**
> - * amdgpu_vm_lock_done_list - lock all BOs on the done list
> + * amdgpu_vm_lock_individual - lock all BOs on the individual idle list
> * @vm: vm providing the BOs
> * @exec: drm execution context
> * @num_fences: number of extra fences to reserve
> *
> - * Lock the BOs on the done list in the DRM execution context.
> + * Lock the BOs on the individual idle list in the DRM execution context.
> */
> -int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
> - unsigned int num_fences)
> +int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
> + unsigned int num_fences)
> {
> - struct list_head *prev = &vm->done;
> + struct list_head *prev = &vm->individual.idle;
> struct amdgpu_bo_va *bo_va;
> struct amdgpu_bo *bo;
> int ret;
>
> /* We can only trust prev->next while holding the lock */
> - spin_lock(&vm->status_lock);
> - while (!list_is_head(prev->next, &vm->done)) {
> + spin_lock(&vm->individual_lock);
> + while (!list_is_head(prev->next, &vm->individual.idle)) {
> bo_va = list_entry(prev->next, typeof(*bo_va), base.vm_status);
>
> bo = bo_va->base.bo;
> if (bo) {
> amdgpu_bo_ref(bo);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 1);
> amdgpu_bo_unref(&bo);
> if (unlikely(ret))
> return ret;
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->individual_lock);
> }
> prev = prev->next;
> }
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> return 0;
> }
> @@ -609,10 +578,10 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> void *param)
> {
> uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm);
> - struct amdgpu_vm_bo_base *bo_base;
> - struct amdgpu_bo *bo;
> + struct amdgpu_vm_bo_base *bo_base, *tmp;
> int r;
>
> + dma_resv_assert_held(vm->root.bo->tbo.base.resv);
> if (vm->generation != new_vm_generation) {
> vm->generation = new_vm_generation;
> amdgpu_vm_bo_reset_state_machine(vm);
> @@ -622,49 +591,59 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> return r;
> }
>
> - spin_lock(&vm->status_lock);
> - while (!list_empty(&vm->evicted)) {
> - bo_base = list_first_entry(&vm->evicted,
> - struct amdgpu_vm_bo_base,
> - vm_status);
> - spin_unlock(&vm->status_lock);
> + list_for_each_entry_safe(bo_base, tmp, &vm->kernel.evicted, vm_status) {
> + r = validate(param, bo_base->bo);
> + if (r)
> + return r;
>
> - bo = bo_base->bo;
> + vm->update_funcs->map_table(to_amdgpu_bo_vm(bo_base->bo));
> + amdgpu_vm_bo_moved(bo_base);
> + }
>
> - r = validate(param, bo);
> + /*
> + * As soon as all page tables are in place we can start updating them
> + * again.
> + */
> + amdgpu_vm_eviction_lock(vm);
> + vm->evicting = false;
> + amdgpu_vm_eviction_unlock(vm);
> +
> + list_for_each_entry_safe(bo_base, tmp, &vm->always_valid.evicted,
> + vm_status) {
> + r = validate(param, bo_base->bo);
> if (r)
> return r;
>
> - if (bo->tbo.type != ttm_bo_type_kernel) {
> - amdgpu_vm_bo_moved(bo_base);
> - } else {
> - vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
> - amdgpu_vm_bo_relocated(bo_base);
> - }
> - spin_lock(&vm->status_lock);
> + amdgpu_vm_bo_moved(bo_base);
> }
> - while (ticket && !list_empty(&vm->evicted_user)) {
> - bo_base = list_first_entry(&vm->evicted_user,
> - struct amdgpu_vm_bo_base,
> - vm_status);
> - spin_unlock(&vm->status_lock);
>
> - bo = bo_base->bo;
> - dma_resv_assert_held(bo->tbo.base.resv);
> + if (!ticket)
> + return 0;
> +
> + spin_lock(&vm->individual_lock);
> +restart:
> + list_for_each_entry(bo_base, &vm->individual.evicted, vm_status) {
> + struct amdgpu_bo *bo = bo_base->bo;
> +
> + if (dma_resv_locking_ctx(bo->tbo.base.resv) != ticket)
> + continue;
> +
> + spin_unlock(&vm->individual_lock);
>
> r = validate(param, bo);
> if (r)
> return r;
>
> - amdgpu_vm_bo_invalidated(bo_base);
> + amdgpu_vm_bo_moved(bo_base);
>
> - spin_lock(&vm->status_lock);
> + /* It's a bit inefficient to always jump back to the start, but
> + * we would need to re-structure the KFD for properly fixing
> + * that.
> + */
> + spin_lock(&vm->individual_lock);
> + goto restart;
> }
> - spin_unlock(&vm->status_lock);
> -
> - amdgpu_vm_eviction_lock(vm);
> - vm->evicting = false;
> - amdgpu_vm_eviction_unlock(vm);
> + spin_unlock(&vm->individual_lock);
>
> return 0;
> }
> @@ -689,9 +668,7 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
> ret = !vm->evicting;
> amdgpu_vm_eviction_unlock(vm);
>
> - spin_lock(&vm->status_lock);
> - ret &= list_empty(&vm->evicted);
> - spin_unlock(&vm->status_lock);
> + ret &= list_empty(&vm->kernel.evicted);
>
> spin_lock(&vm->immediate.lock);
> ret &= !vm->immediate.stopped;
> @@ -985,18 +962,13 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
> struct amdgpu_vm *vm, bool immediate)
> {
> struct amdgpu_vm_update_params params;
> - struct amdgpu_vm_bo_base *entry;
> + struct amdgpu_vm_bo_base *entry, *tmp;
> bool flush_tlb_needed = false;
> - LIST_HEAD(relocated);
> int r, idx;
>
> amdgpu_vm_assert_locked(vm);
>
> - spin_lock(&vm->status_lock);
> - list_splice_init(&vm->relocated, &relocated);
> - spin_unlock(&vm->status_lock);
> -
> - if (list_empty(&relocated))
> + if (list_empty(&vm->kernel.moved))
> return 0;
>
> if (!drm_dev_enter(adev_to_drm(adev), &idx))
> @@ -1012,7 +984,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
> if (r)
> goto error;
>
> - list_for_each_entry(entry, &relocated, vm_status) {
> + list_for_each_entry(entry, &vm->kernel.moved, vm_status) {
> /* vm_flush_needed after updating moved PDEs */
> flush_tlb_needed |= entry->moved;
>
> @@ -1028,11 +1000,8 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
> if (flush_tlb_needed)
> atomic64_inc(&vm->tlb_seq);
>
> - while (!list_empty(&relocated)) {
> - entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base,
> - vm_status);
> + list_for_each_entry_safe(entry, tmp, &vm->kernel.moved, vm_status)
> amdgpu_vm_bo_idle(entry);
> - }
>
> error:
> drm_dev_exit(idx);
> @@ -1260,9 +1229,9 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
> struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
> {
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->stats_lock);
> memcpy(stats, vm->stats, sizeof(*stats) * __AMDGPU_PL_NUM);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->stats_lock);
> }
>
> /**
> @@ -1406,7 +1375,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
> else
> amdgpu_vm_bo_idle(&bo_va->base);
> } else {
> - amdgpu_vm_bo_done(&bo_va->base);
> + amdgpu_vm_bo_idle(&bo_va->base);
> }
>
> list_splice_init(&bo_va->invalids, &bo_va->valids);
> @@ -1629,29 +1598,25 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
> struct amdgpu_vm *vm,
> struct ww_acquire_ctx *ticket)
> {
> - struct amdgpu_bo_va *bo_va;
> + struct amdgpu_bo_va *bo_va, *tmp;
> struct dma_resv *resv;
> bool clear, unlock;
> int r;
>
> - spin_lock(&vm->status_lock);
> - while (!list_empty(&vm->moved)) {
> - bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va,
> - base.vm_status);
> - spin_unlock(&vm->status_lock);
> -
> + list_for_each_entry_safe(bo_va, tmp, &vm->always_valid.moved,
> + base.vm_status) {
> /* Per VM BOs never need to bo cleared in the page tables */
> r = amdgpu_vm_bo_update(adev, bo_va, false);
> if (r)
> return r;
> - spin_lock(&vm->status_lock);
> }
>
> - while (!list_empty(&vm->invalidated)) {
> - bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
> - base.vm_status);
> + spin_lock(&vm->individual_lock);
> + while (!list_empty(&vm->individual.moved)) {
> + bo_va = list_first_entry(&vm->individual.moved,
> + typeof(*bo_va), base.vm_status);
> resv = bo_va->base.bo->tbo.base.resv;
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> /* Try to reserve the BO to avoid clearing its ptes */
> if (!adev->debug_vm && dma_resv_trylock(resv)) {
> @@ -1681,11 +1646,11 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
> drm_gem_is_imported(&bo_va->base.bo->tbo.base) &&
> (!bo_va->base.bo->tbo.resource ||
> bo_va->base.bo->tbo.resource->mem_type == TTM_PL_SYSTEM))
> - amdgpu_vm_bo_evicted_user(&bo_va->base);
> + amdgpu_vm_bo_evicted(&bo_va->base);
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->individual_lock);
> }
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> return 0;
> }
> @@ -2223,9 +2188,9 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev,
> }
> }
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->individual_lock);
> list_del(&bo_va->base.vm_status);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
> list_del(&mapping->list);
> @@ -2305,14 +2270,7 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_bo *bo, bool evicted)
>
> if (bo_base->moved)
> continue;
> - bo_base->moved = true;
> -
> - if (bo->tbo.type == ttm_bo_type_kernel)
> - amdgpu_vm_bo_relocated(bo_base);
> - else if (amdgpu_vm_is_bo_always_valid(vm, bo))
> - amdgpu_vm_bo_moved(bo_base);
> - else
> - amdgpu_vm_bo_invalidated(bo_base);
> + amdgpu_vm_bo_moved(bo_base);
> }
> }
>
> @@ -2333,10 +2291,10 @@ void amdgpu_vm_bo_move(struct amdgpu_bo *bo, struct ttm_resource *new_mem,
> for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
> struct amdgpu_vm *vm = bo_base->vm;
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->stats_lock);
> amdgpu_vm_update_stats_locked(bo_base, bo->tbo.resource, -1);
> amdgpu_vm_update_stats_locked(bo_base, new_mem, +1);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->stats_lock);
> }
>
> amdgpu_vm_bo_invalidate(bo, evicted);
> @@ -2603,16 +2561,14 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> vm->va = RB_ROOT_CACHED;
> for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
> vm->reserved_vmid[i] = NULL;
> - INIT_LIST_HEAD(&vm->evicted);
> - INIT_LIST_HEAD(&vm->evicted_user);
> - INIT_LIST_HEAD(&vm->relocated);
> - INIT_LIST_HEAD(&vm->moved);
> - INIT_LIST_HEAD(&vm->idle);
> - INIT_LIST_HEAD(&vm->invalidated);
> - spin_lock_init(&vm->status_lock);
> +
> + amdgpu_vm_bo_status_init(&vm->kernel);
> + amdgpu_vm_bo_status_init(&vm->always_valid);
> + spin_lock_init(&vm->individual_lock);
> + amdgpu_vm_bo_status_init(&vm->individual);
> INIT_LIST_HEAD(&vm->freed);
> - INIT_LIST_HEAD(&vm->done);
> INIT_KFIFO(vm->faults);
> + spin_lock_init(&vm->stats_lock);
>
> r = amdgpu_vm_init_entities(adev, vm);
> if (r)
> @@ -3089,100 +3045,64 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
> }
>
> #if defined(CONFIG_DEBUG_FS)
> -/**
> - * amdgpu_debugfs_vm_bo_info - print BO info for the VM
> - *
> - * @vm: Requested VM for printing BO info
> - * @m: debugfs file
> - *
> - * Print BO information in debugfs file for the VM
> - */
> -void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
> -{
> - struct amdgpu_bo_va *bo_va, *tmp;
> - u64 total_idle = 0;
> - u64 total_evicted = 0;
> - u64 total_relocated = 0;
> - u64 total_moved = 0;
> - u64 total_invalidated = 0;
> - u64 total_done = 0;
> - unsigned int total_idle_objs = 0;
> - unsigned int total_evicted_objs = 0;
> - unsigned int total_relocated_objs = 0;
> - unsigned int total_moved_objs = 0;
> - unsigned int total_invalidated_objs = 0;
> - unsigned int total_done_objs = 0;
> - unsigned int id = 0;
>
> - amdgpu_vm_assert_locked(vm);
> +/* print the debug info for a specific set of status lists */
> +static void amdgpu_debugfs_vm_bo_status_info(struct seq_file *m,
> + struct amdgpu_vm_bo_status *lists)
> +{
> + struct amdgpu_vm_bo_base *base;
> + unsigned int id;
>
> - spin_lock(&vm->status_lock);
> - seq_puts(m, "\tIdle BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
> - if (!bo_va->base.bo)
> - continue;
> - total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
> - }
> - total_idle_objs = id;
> id = 0;
> -
> seq_puts(m, "\tEvicted BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
> - if (!bo_va->base.bo)
> + list_for_each_entry(base, &lists->evicted, vm_status) {
> + if (!base->bo)
> continue;
> - total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
> - }
> - total_evicted_objs = id;
> - id = 0;
>
> - seq_puts(m, "\tRelocated BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
> - if (!bo_va->base.bo)
> - continue;
> - total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
> + amdgpu_bo_print_info(id++, base->bo, m);
> }
> - total_relocated_objs = id;
> - id = 0;
>
> + id = 0;
> seq_puts(m, "\tMoved BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
> - if (!bo_va->base.bo)
> + list_for_each_entry(base, &lists->moved, vm_status) {
> + if (!base->bo)
> continue;
> - total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
> +
> + amdgpu_bo_print_info(id++, base->bo, m);
> }
> - total_moved_objs = id;
> - id = 0;
>
> - seq_puts(m, "\tInvalidated BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
> - if (!bo_va->base.bo)
> + id = 0;
> + seq_puts(m, "\tIdle BOs:\n");
> + list_for_each_entry(base, &lists->moved, vm_status) {
> + if (!base->bo)
> continue;
> - total_invalidated += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
> +
> + amdgpu_bo_print_info(id++, base->bo, m);
> }
> - total_invalidated_objs = id;
> - id = 0;
> +}
>
> - seq_puts(m, "\tDone BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
> - if (!bo_va->base.bo)
> - continue;
> - total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
> - }
> - spin_unlock(&vm->status_lock);
> - total_done_objs = id;
> -
> - seq_printf(m, "\tTotal idle size: %12lld\tobjs:\t%d\n", total_idle,
> - total_idle_objs);
> - seq_printf(m, "\tTotal evicted size: %12lld\tobjs:\t%d\n", total_evicted,
> - total_evicted_objs);
> - seq_printf(m, "\tTotal relocated size: %12lld\tobjs:\t%d\n", total_relocated,
> - total_relocated_objs);
> - seq_printf(m, "\tTotal moved size: %12lld\tobjs:\t%d\n", total_moved,
> - total_moved_objs);
> - seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated,
> - total_invalidated_objs);
> - seq_printf(m, "\tTotal done size: %12lld\tobjs:\t%d\n", total_done,
> - total_done_objs);
> +/**
> + * amdgpu_debugfs_vm_bo_info - print BO info for the VM
> + *
> + * @vm: Requested VM for printing BO info
> + * @m: debugfs file
> + *
> + * Print BO information in debugfs file for the VM
> + */
> +void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
> +{
> + amdgpu_vm_assert_locked(vm);
> +
> + seq_puts(m, "\tKernel PT/PDs:\n");
> + amdgpu_debugfs_vm_bo_status_info(m, &vm->kernel);
> +
> + seq_puts(m, "\tPer VM BOs:\n");
> + amdgpu_debugfs_vm_bo_status_info(m, &vm->always_valid);
> +
> + seq_puts(m, "\tIndividual BOs:\n");
> + spin_lock(&vm->individual_lock);
> + amdgpu_debugfs_vm_bo_status_info(m, &vm->individual);
> + spin_unlock(&vm->individual_lock);
> }
> #endif
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> index f33ea7f8509b..cc96a3e6252f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> @@ -205,17 +205,34 @@ struct amdgpu_vm_bo_base {
> /* protected by bo being reserved */
> struct amdgpu_vm_bo_base *next;
>
> - /* protected by vm status_lock */
> + /* protected by vm reservation and invalidated_lock */
> struct list_head vm_status;
>
> /* if the bo is counted as shared in mem stats
> - * protected by vm status_lock */
> + * protected by vm BO being reserved */
> bool shared;
>
> /* protected by the BO being reserved */
> bool moved;
> };
>
> +/*
> + * The following status lists contain amdgpu_vm_bo_base objects for
> + * either PD/PTs, per VM BOs or BOs with individual resv object.
> + *
> + * The state transits are: evicted -> moved -> idle
> + */
> +struct amdgpu_vm_bo_status {
> + /* BOs evicted which need to move into place again */
> + struct list_head evicted;
> +
> + /* BOs which moved but new location hasn't been updated in the PDs/PTs */
> + struct list_head moved;
> +
> + /* BOs done with the state machine and need no further action */
> + struct list_head idle;
> +};
> +
> /* provided by hw blocks that can write ptes, e.g., sdma */
> struct amdgpu_vm_pte_funcs {
> /* number of dw to reserve per operation */
> @@ -345,47 +362,29 @@ struct amdgpu_vm {
> bool evicting;
> unsigned int saved_flags;
>
> - /* Lock to protect vm_bo add/del/move on all lists of vm */
> - spinlock_t status_lock;
> -
> - /* Memory statistics for this vm, protected by status_lock */
> + /* Memory statistics for this vm, protected by stats_lock */
> + spinlock_t stats_lock;
> struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM];
>
> + /* BO's belonging to PD/PT which are internal to the kernel. */
> + struct amdgpu_vm_bo_status kernel;
> +
> /*
> - * The following lists contain amdgpu_vm_bo_base objects for either
> - * PDs, PTs or per VM BOs. The state transits are:
> - *
> - * evicted -> relocated (PDs, PTs) or moved (per VM BOs) -> idle
> + * BOs allocated by userspace where the dma_resv is shared with the
> + * root PD
> */
> -
> - /* Per-VM and PT BOs who needs a validation */
> - struct list_head evicted;
> -
> - /* PT BOs which relocated and their parent need an update */
> - struct list_head relocated;
> -
> - /* per VM BOs moved, but not yet updated in the PT */
> - struct list_head moved;
> -
> - /* All BOs of this VM not currently in the state machine */
> - struct list_head idle;
> + struct amdgpu_vm_bo_status always_valid;
>
> /*
> * The following lists contain amdgpu_vm_bo_base objects for BOs which
> - * have their own dma_resv object and not depend on the root PD. Their
> - * state transits are:
> + * have their own dma_resv object and not depend on the root PD.
> *
> - * evicted_user or invalidated -> done
> + * Lists are protected by the individual_lock.
> */
> + spinlock_t individual_lock;
>
> - /* BOs for user mode queues that need a validation */
> - struct list_head evicted_user;
> -
> - /* regular invalidated BOs, but not yet updated in the PT */
> - struct list_head invalidated;
> -
> - /* BOs which are invalidated, has been updated in the PTs */
> - struct list_head done;
> + /* Userspace BOs with individual resv object */
> + struct amdgpu_vm_bo_status individual;
>
> /*
> * This list contains amdgpu_bo_va_mapping objects which have been freed
> @@ -507,8 +506,8 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm);
> void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
> int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec,
> unsigned int num_fences);
> -int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
> - unsigned int num_fences);
> +int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
> + unsigned int num_fences);
> bool amdgpu_vm_ready(struct amdgpu_vm *vm);
> uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm);
> int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> index a930f1522f96..ff334f4dcc90 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> @@ -544,9 +544,7 @@ static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry)
> entry->bo->vm_bo = NULL;
> ttm_bo_set_bulk_move(&entry->bo->tbo, NULL);
>
> - spin_lock(&entry->vm->status_lock);
> list_del(&entry->vm_status);
> - spin_unlock(&entry->vm->status_lock);
> amdgpu_bo_unref(&entry->bo);
> }
>
> @@ -590,7 +588,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params,
> struct amdgpu_vm_pt_cursor seek;
> struct amdgpu_vm_bo_base *entry;
>
> - spin_lock(¶ms->vm->status_lock);
> for_each_amdgpu_vm_pt_dfs_safe(params->adev, params->vm, cursor, seek, entry) {
> if (entry && entry->bo)
> list_move(&entry->vm_status, ¶ms->tlb_flush_waitlist);
> @@ -598,7 +595,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params,
>
> /* enter start node now */
> list_move(&cursor->entry->vm_status, ¶ms->tlb_flush_waitlist);
> - spin_unlock(¶ms->vm->status_lock);
> }
>
> /**
^ permalink raw reply [flat|nested] 30+ messages in thread
* RE: [PATCH 4/8] drm/amdgpu: remove almost all calls to amdgpu_userq_detect_and_reset_queues
2026-04-27 18:27 ` [PATCH 4/8] drm/amdgpu: remove almost all calls to amdgpu_userq_detect_and_reset_queues Christian König
2026-04-28 9:37 ` Khatri, Sunil
@ 2026-04-29 3:16 ` Liang, Prike
1 sibling, 0 replies; 30+ messages in thread
From: Liang, Prike @ 2026-04-29 3:16 UTC (permalink / raw)
To: Koenig, Christian, Deucher, Alexander, Khatri, Sunil,
Zhang, Jesse(Jie)
Cc: amd-gfx@lists.freedesktop.org
AMD General
Regards,
Prike
> -----Original Message-----
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> Sent: Tuesday, April 28, 2026 2:27 AM
> To: Deucher, Alexander <Alexander.Deucher@amd.com>; Liang, Prike
> <Prike.Liang@amd.com>; Khatri, Sunil <Sunil.Khatri@amd.com>; Zhang,
> Jesse(Jie) <Jesse.Zhang@amd.com>
> Cc: amd-gfx@lists.freedesktop.org
> Subject: [PATCH 4/8] drm/amdgpu: remove almost all calls to
> amdgpu_userq_detect_and_reset_queues
>
> Well the reset handling seems broken on multiple levels.
>
> As first step of fixing this remove most calls to the hang detection.
> That function should only be called after we run into a timeout! And *NOT* as
> random check spread over the code in multiple places.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 38 +++++++++--------------
> 1 file changed, 14 insertions(+), 24 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index 62d155d6e4ed..fb3c0425cf22 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -345,23 +345,18 @@ static int amdgpu_userq_preempt_helper(struct
> amdgpu_usermode_queue *queue)
> struct amdgpu_device *adev = uq_mgr->adev;
> const struct amdgpu_userq_funcs *userq_funcs =
> adev->userq_funcs[queue->queue_type];
> - bool found_hung_queue = false;
> - int r = 0;
> + int r;
>
> if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
> r = userq_funcs->preempt(queue);
> if (r) {
> queue->state = AMDGPU_USERQ_STATE_HUNG;
> - found_hung_queue = true;
We may still need to reset the hang queue, but instead of resetting the queue directly rather than walk over all the queues.
Here may need to add a todo list which implements a per queue reset once the per-queue lock introduced, otherwise there may miss reset the queue state which request from the kernel directly case without tracking the queue by the reset worker.
For now, the patch is Reviewed-by: Prike Liang <Prike.Liang@amd.com>
> + return r;
> } else {
> queue->state = AMDGPU_USERQ_STATE_PREEMPTED;
> }
> }
> -
> - if (found_hung_queue)
> - amdgpu_userq_detect_and_reset_queues(uq_mgr);
> -
> - return r;
> + return 0;
> }
>
> static int amdgpu_userq_restore_helper(struct amdgpu_usermode_queue *queue)
> @@ -390,24 +385,21 @@ static int amdgpu_userq_unmap_helper(struct
> amdgpu_usermode_queue *queue)
> struct amdgpu_device *adev = uq_mgr->adev;
> const struct amdgpu_userq_funcs *userq_funcs =
> adev->userq_funcs[queue->queue_type];
> - bool found_hung_queue = false;
> - int r = 0;
> + int r;
>
> if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) ||
> - (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {
> + (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {
> +
> r = userq_funcs->unmap(queue);
> if (r) {
> queue->state = AMDGPU_USERQ_STATE_HUNG;
> - found_hung_queue = true;
> + return r;
> } else {
> queue->state = AMDGPU_USERQ_STATE_UNMAPPED;
> }
> }
>
> - if (found_hung_queue)
> - amdgpu_userq_detect_and_reset_queues(uq_mgr);
> -
> - return r;
> + return 0;
> }
>
> static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue)
> @@ -416,19 +408,19 @@ static int amdgpu_userq_map_helper(struct
> amdgpu_usermode_queue *queue)
> struct amdgpu_device *adev = uq_mgr->adev;
> const struct amdgpu_userq_funcs *userq_funcs =
> adev->userq_funcs[queue->queue_type];
> - int r = 0;
> + int r;
>
> if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) {
> r = userq_funcs->map(queue);
> if (r) {
> queue->state = AMDGPU_USERQ_STATE_HUNG;
> - amdgpu_userq_detect_and_reset_queues(uq_mgr);
> + return r;
> } else {
> queue->state = AMDGPU_USERQ_STATE_MAPPED;
> }
> }
>
> - return r;
> + return 0;
> }
>
> static void amdgpu_userq_wait_for_last_fence(struct amdgpu_usermode_queue
> *queue) @@ -654,7 +646,6 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr
> *uq_mgr, struct amdgpu_usermode_que #if defined(CONFIG_DEBUG_FS)
> debugfs_remove_recursive(queue->debugfs_queue);
> #endif
> - amdgpu_userq_detect_and_reset_queues(uq_mgr);
> r = amdgpu_userq_unmap_helper(queue);
> /*TODO: It requires a reset for userq hw unmap error*/
> if (r) {
> @@ -1270,7 +1261,6 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr
> *uq_mgr)
> unsigned long queue_id;
> int ret = 0, r;
>
> - amdgpu_userq_detect_and_reset_queues(uq_mgr);
> /* Try to unmap all the queues in this process ctx */
> xa_for_each(&uq_mgr->userq_xa, queue_id, queue) {
> r = amdgpu_userq_preempt_helper(queue);
> @@ -1278,9 +1268,11 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr
> *uq_mgr)
> ret = r;
> }
>
> - if (ret)
> + if (ret) {
> drm_file_err(uq_mgr->file,
> "Couldn't unmap all the queues, eviction failed ret=%d\n",
> ret);
> + amdgpu_userq_detect_and_reset_queues(uq_mgr);
> + }
> return ret;
> }
>
> @@ -1380,7 +1372,6 @@ int amdgpu_userq_suspend(struct amdgpu_device
> *adev)
> uqm = queue->userq_mgr;
> cancel_delayed_work_sync(&uqm->resume_work);
> guard(mutex)(&uqm->userq_mutex);
> - amdgpu_userq_detect_and_reset_queues(uqm);
> if (adev->in_s0ix)
> r = amdgpu_userq_preempt_helper(queue);
> else
> @@ -1439,7 +1430,6 @@ int
> amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev,
> if (((queue->queue_type == AMDGPU_HW_IP_GFX) ||
> (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) &&
> (queue->xcp_id == idx)) {
> - amdgpu_userq_detect_and_reset_queues(uqm);
> r = amdgpu_userq_preempt_helper(queue);
> if (r)
> ret = r;
> --
> 2.43.0
^ permalink raw reply [flat|nested] 30+ messages in thread
* RE: [PATCH 5/8] drm/amdgpu: fix userq hang detection and reset
2026-04-27 18:27 ` [PATCH 5/8] drm/amdgpu: fix userq hang detection and reset Christian König
@ 2026-04-29 8:30 ` Liang, Prike
0 siblings, 0 replies; 30+ messages in thread
From: Liang, Prike @ 2026-04-29 8:30 UTC (permalink / raw)
To: Koenig, Christian, Deucher, Alexander, Khatri, Sunil,
Zhang, Jesse(Jie)
Cc: amd-gfx@lists.freedesktop.org
AMD General
Regards,
Prike
> -----Original Message-----
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> Sent: Tuesday, April 28, 2026 2:27 AM
> To: Deucher, Alexander <Alexander.Deucher@amd.com>; Liang, Prike
> <Prike.Liang@amd.com>; Khatri, Sunil <Sunil.Khatri@amd.com>; Zhang,
> Jesse(Jie) <Jesse.Zhang@amd.com>
> Cc: amd-gfx@lists.freedesktop.org
> Subject: [PATCH 5/8] drm/amdgpu: fix userq hang detection and reset
>
> Fix lock inversions pointed out by Prike and Sunil. The hang detection timeout
> *CAN'T* grab locks under which we wait for fences, especially not the userq_mutex
> lock.
>
> Then instead of this completely broken handling with the hang_detect_fence just
> cancel the work when fences are processed and re-start if necessary.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 65 ++++++++-----------
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 1 -
> .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c | 17 +++--
> .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.h | 2 +-
> 4 files changed, 40 insertions(+), 45 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index fb3c0425cf22..b2c64a05d03a 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -106,9 +106,6 @@ amdgpu_userq_detect_and_reset_queues(struct
> amdgpu_userq_mgr *uq_mgr)
> int r = 0;
> int i;
>
> - /* Warning if current process mutex is not held */
> - WARN_ON(!mutex_is_locked(&uq_mgr->userq_mutex));
> -
> if (unlikely(adev->debug_disable_gpu_ring_reset)) {
> dev_err(adev->dev, "userq reset disabled by debug mask\n");
> return 0;
> @@ -127,9 +124,11 @@ amdgpu_userq_detect_and_reset_queues(struct
> amdgpu_userq_mgr *uq_mgr)
> */
> for (i = 0; i < num_queue_types; i++) {
> int ring_type = queue_types[i];
> - const struct amdgpu_userq_funcs *funcs = adev-
> >userq_funcs[ring_type];
> + const struct amdgpu_userq_funcs *funcs =
> + adev->userq_funcs[ring_type];
>
> - if (!amdgpu_userq_is_reset_type_supported(adev, ring_type,
> AMDGPU_RESET_TYPE_PER_QUEUE))
> + if (!amdgpu_userq_is_reset_type_supported(adev, ring_type,
> +
> AMDGPU_RESET_TYPE_PER_QUEUE))
> continue;
>
> if (atomic_read(&uq_mgr->userq_count[ring_type]) > 0 && @@ -
> 150,38 +149,22 @@ amdgpu_userq_detect_and_reset_queues(struct
> amdgpu_userq_mgr *uq_mgr)
>
> static void amdgpu_userq_hang_detect_work(struct work_struct *work) {
> - struct amdgpu_usermode_queue *queue = container_of(work,
> - struct
> amdgpu_usermode_queue,
> - hang_detect_work.work);
> - struct dma_fence *fence;
> - struct amdgpu_userq_mgr *uq_mgr;
> -
> - if (!queue->userq_mgr)
> - return;
> -
> - uq_mgr = queue->userq_mgr;
> - fence = READ_ONCE(queue->hang_detect_fence);
> - /* Fence already signaled – no action needed */
> - if (!fence || dma_fence_is_signaled(fence))
> - return;
> + struct amdgpu_usermode_queue *queue =
> + container_of(work, struct amdgpu_usermode_queue,
> + hang_detect_work.work);
>
> - mutex_lock(&uq_mgr->userq_mutex);
> - amdgpu_userq_detect_and_reset_queues(uq_mgr);
> - mutex_unlock(&uq_mgr->userq_mutex);
As this will walk over all the types of queues, do we need to add the reset domain lock for preventing the race for walking over the queue even though there's a MES lock in the mes_userq_detect_and_reset()? In the long run we may need to reset the queue directly rather than trying to detect and reset all the queues.
But for resolving the userq rest lock problem about the userq eviction wait hang, the patch is Reviewed-by: Prike Liang <Prike.Liang@amd.com>
> + amdgpu_userq_detect_and_reset_queues(queue->userq_mgr);
> }
>
> /*
> * Start hang detection for a user queue fence. A delayed work will be scheduled
> - * to check if the fence is still pending after the timeout period.
> -*/
> + * to reset the queues when the fence doesn't signal in time.
> + */
> void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue
> *queue) {
> struct amdgpu_device *adev;
> unsigned long timeout_ms;
>
> - if (!queue || !queue->userq_mgr || !queue->userq_mgr->adev)
> - return;
> -
> adev = queue->userq_mgr->adev;
> /* Determine timeout based on queue type */
> switch (queue->queue_type) {
> @@ -199,8 +182,6 @@ void amdgpu_userq_start_hang_detect_work(struct
> amdgpu_usermode_queue *queue)
> break;
> }
>
> - /* Store the fence to monitor and schedule hang detection */
> - WRITE_ONCE(queue->hang_detect_fence, queue->last_fence);
> schedule_delayed_work(&queue->hang_detect_work,
> msecs_to_jiffies(timeout_ms));
> }
> @@ -210,18 +191,24 @@ void amdgpu_userq_process_fence_irq(struct
> amdgpu_device *adev, u32 doorbell)
> struct xarray *xa = &adev->userq_doorbell_xa;
> struct amdgpu_usermode_queue *queue;
> unsigned long flags;
> + int r;
>
> xa_lock_irqsave(xa, flags);
> queue = xa_load(xa, doorbell);
> - if (queue)
> - amdgpu_userq_fence_driver_process(queue->fence_drv);
> - xa_unlock_irqrestore(xa, flags);
> -}
> + if (queue) {
> + r = amdgpu_userq_fence_driver_process(queue->fence_drv);
> + /*
> + * We are in interrupt context here, this *can't* wait for
> + * reset work to finish.
> + */
> + if (r >= 0)
> + cancel_delayed_work(&queue->hang_detect_work);
>
> -static void amdgpu_userq_init_hang_detect_work(struct amdgpu_usermode_queue
> *queue) -{
> - INIT_DELAYED_WORK(&queue->hang_detect_work,
> amdgpu_userq_hang_detect_work);
> - queue->hang_detect_fence = NULL;
> + /* Restart the timer when there are still fences pending */
> + if (r == 1)
> + amdgpu_userq_start_hang_detect_work(queue);
> + }
> + xa_unlock_irqrestore(xa, flags);
> }
>
> static int amdgpu_userq_buffer_va_list_add(struct amdgpu_usermode_queue
> *queue, @@ -640,7 +627,6 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr
> *uq_mgr, struct amdgpu_usermode_que
> amdgpu_bo_unreserve(vm->root.bo);
>
> mutex_lock(&uq_mgr->userq_mutex);
> - queue->hang_detect_fence = NULL;
> amdgpu_userq_wait_for_last_fence(queue);
>
> #if defined(CONFIG_DEBUG_FS)
> @@ -853,7 +839,8 @@ amdgpu_userq_create(struct drm_file *filp, union
> drm_amdgpu_userq *args)
> up_read(&adev->reset_domain->sem);
>
> amdgpu_debugfs_userq_init(filp, queue, qid);
> - amdgpu_userq_init_hang_detect_work(queue);
> + INIT_DELAYED_WORK(&queue->hang_detect_work,
> + amdgpu_userq_hang_detect_work);
>
> args->out.queue_id = qid;
> atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> index 843ea8ecc5d7..85f460e7c31b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> @@ -85,7 +85,6 @@ struct amdgpu_usermode_queue {
> int priority;
> struct dentry *debugfs_queue;
> struct delayed_work hang_detect_work;
> - struct dma_fence *hang_detect_fence;
> struct kref refcount;
>
> struct list_head userq_va_list;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> index a2792f05b670..bf452c5aacbe 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
> @@ -135,7 +135,14 @@ amdgpu_userq_fence_put_fence_drv_array(struct
> amdgpu_userq_fence *userq_fence)
> userq_fence->fence_drv_array_count = 0; }
>
> -void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver
> *fence_drv)
> +/*
> + * Returns:
> + * -ENOENT when no fences were processes
> + * 1 when more fences are pending
> + * 0 when no fences are pending any more */ int
> +amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver
> +*fence_drv)
> {
> struct amdgpu_userq_fence *userq_fence, *tmp;
> LIST_HEAD(to_be_signaled);
> @@ -143,9 +150,6 @@ void amdgpu_userq_fence_driver_process(struct
> amdgpu_userq_fence_driver *fence_d
> unsigned long flags;
> u64 rptr;
>
> - if (!fence_drv)
> - return;
> -
> spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
> rptr = amdgpu_userq_fence_read(fence_drv);
>
> @@ -158,6 +162,9 @@ void amdgpu_userq_fence_driver_process(struct
> amdgpu_userq_fence_driver *fence_d
> &userq_fence->link);
> spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
>
> + if (list_empty(&to_be_signaled))
> + return -ENOENT;
> +
> list_for_each_entry_safe(userq_fence, tmp, &to_be_signaled, link) {
> fence = &userq_fence->base;
> list_del_init(&userq_fence->link);
> @@ -169,6 +176,8 @@ void amdgpu_userq_fence_driver_process(struct
> amdgpu_userq_fence_driver *fence_d
> dma_fence_put(fence);
> }
>
> + /* That doesn't need to be accurate so no locking */
> + return list_empty(&fence_drv->fences) ? 0 : 1;
> }
>
> void amdgpu_userq_fence_driver_destroy(struct kref *ref) diff --git
> a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> index d355a0eecc07..0bd51616cef1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
> @@ -63,7 +63,7 @@ void amdgpu_userq_fence_driver_put(struct
> amdgpu_userq_fence_driver *fence_drv); int
> amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,
> struct amdgpu_userq_fence_driver
> **fence_drv_req); void amdgpu_userq_fence_driver_free(struct
> amdgpu_usermode_queue *userq); -void
> amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver
> *fence_drv);
> +int amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver
> +*fence_drv);
> void amdgpu_userq_fence_driver_force_completion(struct
> amdgpu_usermode_queue *userq); void
> amdgpu_userq_fence_driver_destroy(struct kref *ref); int
> amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
> --
> 2.43.0
^ permalink raw reply [flat|nested] 30+ messages in thread
* RE: [PATCH 6/8] drm/amdgpu: rework userq reset work handling
2026-04-27 18:27 ` [PATCH 6/8] drm/amdgpu: rework userq reset work handling Christian König
@ 2026-04-30 6:37 ` Liang, Prike
0 siblings, 0 replies; 30+ messages in thread
From: Liang, Prike @ 2026-04-30 6:37 UTC (permalink / raw)
To: Koenig, Christian, Deucher, Alexander, Khatri, Sunil,
Zhang, Jesse(Jie)
Cc: amd-gfx@lists.freedesktop.org
AMD General
Regards,
Prike
> -----Original Message-----
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> Sent: Tuesday, April 28, 2026 2:27 AM
> To: Deucher, Alexander <Alexander.Deucher@amd.com>; Liang, Prike
> <Prike.Liang@amd.com>; Khatri, Sunil <Sunil.Khatri@amd.com>; Zhang,
> Jesse(Jie) <Jesse.Zhang@amd.com>
> Cc: amd-gfx@lists.freedesktop.org
> Subject: [PATCH 6/8] drm/amdgpu: rework userq reset work handling
>
> It is illegal to schedule reset work from another reset work!
>
> Fix this by scheduling the userq reset work directly on the work queue of the reset
> domain.
>
> Not fully tested, I leave that to the IGT test cases.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 -
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 3 +-
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 84 +++++++++++-----------
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 16 ++++-
> 4 files changed, 60 insertions(+), 44 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index 39894e38fee4..17341e384caf 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -1191,7 +1191,6 @@ struct amdgpu_device {
> bool apu_prefer_gtt;
>
> bool userq_halt_for_enforce_isolation;
> - struct work_struct userq_reset_work;
> struct amdgpu_uid *uid_info;
>
> struct amdgpu_uma_carveout_info uma_info; diff --git
> a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index 4bb8c34d2dd9..c400b4c5dda3 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -3790,7 +3790,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
> }
>
> INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
> - INIT_WORK(&adev->userq_reset_work, amdgpu_userq_reset_work);
>
> amdgpu_coredump_init(adev);
>
> @@ -5481,7 +5480,7 @@ static inline void
> amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
> if (!amdgpu_sriov_vf(adev))
> cancel_work(&adev->reset_work);
> #endif
> - cancel_work(&adev->userq_reset_work);
> + amdgpu_userq_mgr_cancel_reset_work(adev);
>
> if (adev->kfd.dev)
> cancel_work(&adev->kfd.reset_work);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index b2c64a05d03a..0374a70b631c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -82,19 +82,11 @@ static bool amdgpu_userq_is_reset_type_supported(struct
> amdgpu_device *adev,
> return false;
> }
>
> -static void amdgpu_userq_gpu_reset(struct amdgpu_device *adev) -{
> - if (amdgpu_device_should_recover_gpu(adev)) {
> - amdgpu_reset_domain_schedule(adev->reset_domain,
> - &adev->userq_reset_work);
> - /* Wait for the reset job to complete */
> - flush_work(&adev->userq_reset_work);
> - }
> -}
> -
> -static int
> -amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
> +static void amdgpu_userq_mgr_reset_work(struct work_struct *work)
> {
> + struct amdgpu_userq_mgr *uq_mgr =
> + container_of(work, struct amdgpu_userq_mgr,
> + reset_work);
> struct amdgpu_device *adev = uq_mgr->adev;
> const int queue_types[] = {
> AMDGPU_RING_TYPE_COMPUTE,
> @@ -103,12 +95,11 @@ amdgpu_userq_detect_and_reset_queues(struct
> amdgpu_userq_mgr *uq_mgr)
> };
> const int num_queue_types = ARRAY_SIZE(queue_types);
> bool gpu_reset = false;
> - int r = 0;
> - int i;
> + int i, r;
>
> if (unlikely(adev->debug_disable_gpu_ring_reset)) {
> dev_err(adev->dev, "userq reset disabled by debug mask\n");
> - return 0;
> + return;
> }
>
> /*
> @@ -116,7 +107,7 @@ amdgpu_userq_detect_and_reset_queues(struct
> amdgpu_userq_mgr *uq_mgr)
> * skip all reset detection logic
> */
> if (!amdgpu_gpu_recovery)
> - return 0;
> + return;
>
> /*
> * Iterate through all queue types to detect and reset problematic queues
> @@ -141,10 +132,19 @@ amdgpu_userq_detect_and_reset_queues(struct
> amdgpu_userq_mgr *uq_mgr)
> }
> }
>
> - if (gpu_reset)
> - amdgpu_userq_gpu_reset(adev);
> + if (gpu_reset) {
> + struct amdgpu_reset_context reset_context;
>
> - return r;
> + memset(&reset_context, 0, sizeof(reset_context));
> +
> + reset_context.method = AMD_RESET_METHOD_NONE;
> + reset_context.reset_req_dev = adev;
> + reset_context.src = AMDGPU_RESET_SRC_USERQ;
> + set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
> + /*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/
> +
> + amdgpu_device_gpu_recover(adev, NULL, &reset_context);
> + }
> }
>
> static void amdgpu_userq_hang_detect_work(struct work_struct *work) @@ -153,7
> +153,11 @@ static void amdgpu_userq_hang_detect_work(struct work_struct
> *work)
> container_of(work, struct amdgpu_usermode_queue,
> hang_detect_work.work);
>
> - amdgpu_userq_detect_and_reset_queues(queue->userq_mgr);
> + /*
> + * Don't schedule the work here! Scheduling or queue work from one reset
> + * handler to another is illegal if you don't take extra precautions!
> + */
It might be clearer to add a comment noting that the hang detection worker and reset worker are scheduled on
the same ordered workqueue, which can lead to a deadlock if one worker schedules and waits for the other
within a globally serialized queue.
With it or not, the patch is Reviewed-by: Prike Liang <Prike.Liang@amd.com>
> + amdgpu_userq_mgr_reset_work(&queue->userq_mgr->reset_work);
> }
>
> /*
> @@ -182,8 +186,8 @@ void amdgpu_userq_start_hang_detect_work(struct
> amdgpu_usermode_queue *queue)
> break;
> }
>
> - schedule_delayed_work(&queue->hang_detect_work,
> - msecs_to_jiffies(timeout_ms));
> + queue_delayed_work(adev->reset_domain->wq, &queue-
> >hang_detect_work,
> + msecs_to_jiffies(timeout_ms));
> }
>
> void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell)
> @@ -1258,28 +1262,13 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr
> *uq_mgr)
> if (ret) {
> drm_file_err(uq_mgr->file,
> "Couldn't unmap all the queues, eviction failed ret=%d\n",
> ret);
> - amdgpu_userq_detect_and_reset_queues(uq_mgr);
> + amdgpu_reset_domain_schedule(uq_mgr->adev->reset_domain,
> + &uq_mgr->reset_work);
> + flush_work(&uq_mgr->reset_work);
> }
> return ret;
> }
>
> -void amdgpu_userq_reset_work(struct work_struct *work) -{
> - struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
> - userq_reset_work);
> - struct amdgpu_reset_context reset_context;
> -
> - memset(&reset_context, 0, sizeof(reset_context));
> -
> - reset_context.method = AMD_RESET_METHOD_NONE;
> - reset_context.reset_req_dev = adev;
> - reset_context.src = AMDGPU_RESET_SRC_USERQ;
> - set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
> - /*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/
> -
> - amdgpu_device_gpu_recover(adev, NULL, &reset_context);
> -}
> -
> static void
> amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr) { @@ -1313,9
> +1302,24 @@ int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr,
> struct drm_file *f
> userq_mgr->file = file_priv;
>
> INIT_DELAYED_WORK(&userq_mgr->resume_work,
> amdgpu_userq_restore_worker);
> + INIT_WORK(&userq_mgr->reset_work, amdgpu_userq_mgr_reset_work);
> return 0;
> }
>
> +void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev) {
> + struct xarray *xa = &adev->userq_doorbell_xa;
> + struct amdgpu_usermode_queue *queue;
> + unsigned long flags, queue_id;
> +
> + xa_lock_irqsave(xa, flags);
> + xa_for_each(xa, queue_id, queue) {
> + cancel_delayed_work(&queue->hang_detect_work);
> + cancel_work(&queue->userq_mgr->reset_work);
> + }
> + xa_unlock_irqrestore(xa, flags);
> +}
> +
> void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr) {
> cancel_delayed_work_sync(&userq_mgr->resume_work);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> index 85f460e7c31b..49b33e2d6932 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> @@ -84,7 +84,13 @@ struct amdgpu_usermode_queue {
> u32 xcp_id;
> int priority;
> struct dentry *debugfs_queue;
> - struct delayed_work hang_detect_work;
> +
> + /**
> + * @hang_detect_work:
> + *
> + * Delayed work which runs when userq_fences time out.
> + */
> + struct delayed_work hang_detect_work;
> struct kref refcount;
>
> struct list_head userq_va_list;
> @@ -116,6 +122,13 @@ struct amdgpu_userq_mgr {
> struct amdgpu_device *adev;
> struct delayed_work resume_work;
> struct drm_file *file;
> +
> + /**
> + * @reset_work:
> + *
> + * Reset work which is used when eviction fails.
> + */
> + struct work_struct reset_work;
> atomic_t userq_count[AMDGPU_RING_TYPE_MAX];
> };
>
> @@ -134,6 +147,7 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data,
> struct drm_file *filp int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr
> *userq_mgr, struct drm_file *file_priv,
> struct amdgpu_device *adev);
>
> +void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev);
> void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr);
> void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr);
>
> --
> 2.43.0
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH 3/8] drm/amdgpu: rework amdgpu_userq_signal_ioctl v2
2026-04-28 8:49 ` Liang, Prike
@ 2026-05-04 12:33 ` Christian König
0 siblings, 0 replies; 30+ messages in thread
From: Christian König @ 2026-05-04 12:33 UTC (permalink / raw)
To: Liang, Prike, Deucher, Alexander, Khatri, Sunil,
Zhang, Jesse(Jie)
Cc: amd-gfx@lists.freedesktop.org
On 4/28/26 10:49, Liang, Prike wrote:
...
>> + /*
>> + * Get the next unused entry, since we fill from the start this can be
>> + * used as size to allocate the array.
>> + */
>> + mutex_lock(&userq->fence_drv_lock);
>> + XA_STATE(xas, &userq->fence_drv_xa, 0);
>> +
>> + rcu_read_lock();
>> + do {
>> + entry = xas_find_marked(&xas, ULONG_MAX, XA_FREE_MARK);
>> + } while (xas_retry(&xas, entry));
>
> Do we need to handle the xas.xa_index = 0 case separately rather than use the invalid wait fence?
No, zero sized array allocations are perfectly valid.
>
>> + rcu_read_unlock();
>> +
>> + userq_fence->fence_drv_array = kvmalloc_array(xas.xa_index,
>> + sizeof(fence_drv),
>> + GFP_KERNEL);
>
>
> The kvmalloc_array() should use the obj size of sizeof(*userq_fence->fence_drv_array).
As far as I can see that should be the same?
>
>> + if (!userq_fence->fence_drv_array) {
>> + mutex_unlock(&userq->fence_drv_lock);
>> + kfree(userq_fence);
>> + return -ENOMEM;
>> + }
>> +
>> + userq_fence->fence_drv_array_count = xas.xa_index;
>> + xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array,
>> + 0, ULONG_MAX, xas.xa_index, XA_PRESENT);
>> + xa_destroy(&userq->fence_drv_xa);
>> +
>> + mutex_unlock(&userq->fence_drv_lock);
>> +
>> + userq_fence->fence_drv = fence_drv;
>> + amdgpu_userq_fence_driver_get(fence_drv);
>
> It will be better to acquire the fence_drv reference first before publishing it.
Good point, going to clean that up.
Regards,
Christian.
>
>> + *pfence = userq_fence;
>> + return 0;
>> }
>>
>> -static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
>> - struct amdgpu_userq_fence *userq_fence,
>> - u64 seq, struct dma_fence **f)
>> +static void amdgpu_userq_fence_init(struct amdgpu_usermode_queue *userq,
>> + struct amdgpu_userq_fence *fence,
>> + u64 seq)
>> {
>> - struct amdgpu_userq_fence_driver *fence_drv;
>> - struct dma_fence *fence;
>> + struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
>> unsigned long flags;
>> bool signaled = false;
>>
>> - fence_drv = userq->fence_drv;
>> - if (!fence_drv)
>> - return -EINVAL;
>> -
>> - spin_lock_init(&userq_fence->lock);
>> - INIT_LIST_HEAD(&userq_fence->link);
>> - fence = &userq_fence->base;
>> - userq_fence->fence_drv = fence_drv;
>> -
>> - dma_fence_init64(fence, &amdgpu_userq_fence_ops, &userq_fence->lock,
>> + spin_lock_init(&fence->lock);
>> + dma_fence_init64(&fence->base, &amdgpu_userq_fence_ops, &fence->lock,
>> fence_drv->context, seq);
>>
>> - amdgpu_userq_fence_driver_get(fence_drv);
>> - dma_fence_get(fence);
>> -
>> - if (!xa_empty(&userq->fence_drv_xa)) {
>> - struct amdgpu_userq_fence_driver *stored_fence_drv;
>> - unsigned long index, count = 0;
>> - int i = 0;
>> -
>> - xa_lock(&userq->fence_drv_xa);
>> - xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv)
>> - count++;
>> -
>> - userq_fence->fence_drv_array =
>> - kvmalloc_array(count,
>> - sizeof(struct amdgpu_userq_fence_driver *),
>> - GFP_ATOMIC);
>> -
>> - if (userq_fence->fence_drv_array) {
>> - xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv)
>> {
>> - userq_fence->fence_drv_array[i] = stored_fence_drv;
>> - __xa_erase(&userq->fence_drv_xa, index);
>> - i++;
>> - }
>> - }
>> -
>> - userq_fence->fence_drv_array_count = i;
>> - xa_unlock(&userq->fence_drv_xa);
>> - } else {
>> - userq_fence->fence_drv_array = NULL;
>> - userq_fence->fence_drv_array_count = 0;
>> - }
>> + /* Make sure the fence is visible to the hang detect worker */
>> + dma_fence_put(userq->last_fence);
>> + userq->last_fence = dma_fence_get(&fence->base);
>>
>> - /* Check if hardware has already processed the job */
>> + /* Check if hardware has already processed the fence */
>> spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
>> - if (!dma_fence_is_signaled(fence)) {
>> - list_add_tail(&userq_fence->link, &fence_drv->fences);
>> + if (!dma_fence_is_signaled(&fence->base)) {
>> + dma_fence_get(&fence->base);
>> + list_add_tail(&fence->link, &fence_drv->fences);
>> } else {
>> + INIT_LIST_HEAD(&fence->link);
>> signaled = true;
>> - dma_fence_put(fence);
>> }
>> spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
>>
>> if (signaled)
>> - amdgpu_userq_fence_put_fence_drv_array(userq_fence);
>> -
>> - *f = fence;
>> -
>> - return 0;
>> + amdgpu_userq_fence_put_fence_drv_array(fence);
>> + else
>> + amdgpu_userq_start_hang_detect_work(userq);
>> }
>>
>> static const char *amdgpu_userq_fence_get_driver_name(struct dma_fence *f) @@
>> -404,11 +408,6 @@ static int amdgpu_userq_fence_read_wptr(struct
>> amdgpu_device *adev,
>> return r;
>> }
>>
>> -static void amdgpu_userq_fence_cleanup(struct dma_fence *fence) -{
>> - dma_fence_put(fence);
>> -}
>> -
>> static void
>> amdgpu_userq_fence_driver_set_error(struct amdgpu_userq_fence *fence,
>> int error)
>> @@ -452,13 +451,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev,
>> void *data,
>> const unsigned int num_read_bo_handles = args->num_bo_read_handles;
>> struct amdgpu_fpriv *fpriv = filp->driver_priv;
>> struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
>> +
>> struct drm_gem_object **gobj_write, **gobj_read;
>> u32 *syncobj_handles, num_syncobj_handles;
>> - struct amdgpu_userq_fence *userq_fence;
>> - struct amdgpu_usermode_queue *queue = NULL;
>> - struct drm_syncobj **syncobj = NULL;
>> - struct dma_fence *fence;
>> + struct amdgpu_usermode_queue *queue;
>> + struct amdgpu_userq_fence *fence;
>> + struct drm_syncobj **syncobj;
>> struct drm_exec exec;
>> + void __user *ptr;
>> int r, i, entry;
>> u64 wptr;
>>
>> @@ -470,13 +470,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev,
>> void *data,
>> return -EINVAL;
>>
>> num_syncobj_handles = args->num_syncobj_handles;
>> - syncobj_handles = memdup_array_user(u64_to_user_ptr(args-
>>> syncobj_handles),
>> - num_syncobj_handles, sizeof(u32));
>> + ptr = u64_to_user_ptr(args->syncobj_handles);
>> + syncobj_handles = memdup_array_user(ptr, num_syncobj_handles,
>> + sizeof(u32));
>> if (IS_ERR(syncobj_handles))
>> return PTR_ERR(syncobj_handles);
>>
>> - /* Array of pointers to the looked up syncobjs */
>> - syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj),
>> GFP_KERNEL);
>> + syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj),
>> + GFP_KERNEL);
>> if (!syncobj) {
>> r = -ENOMEM;
>> goto free_syncobj_handles;
>> @@ -490,21 +491,17 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev,
>> void *data,
>> }
>> }
>>
>> - r = drm_gem_objects_lookup(filp,
>> - u64_to_user_ptr(args->bo_read_handles),
>> - num_read_bo_handles,
>> - &gobj_read);
>> + ptr = u64_to_user_ptr(args->bo_read_handles);
>> + r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles,
>> +&gobj_read);
>> if (r)
>> goto free_syncobj;
>>
>> - r = drm_gem_objects_lookup(filp,
>> - u64_to_user_ptr(args->bo_write_handles),
>> - num_write_bo_handles,
>> + ptr = u64_to_user_ptr(args->bo_write_handles);
>> + r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
>> &gobj_write);
>> if (r)
>> goto put_gobj_read;
>>
>> - /* Retrieve the user queue */
>> queue = amdgpu_userq_get(userq_mgr, args->queue_id);
>> if (!queue) {
>> r = -ENOENT;
>> @@ -513,73 +510,61 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev,
>> void *data,
>>
>> r = amdgpu_userq_fence_read_wptr(adev, queue, &wptr);
>> if (r)
>> - goto put_gobj_write;
>> + goto put_queue;
>>
>> - r = amdgpu_userq_fence_alloc(&userq_fence);
>> + r = amdgpu_userq_fence_alloc(queue, &fence);
>> if (r)
>> - goto put_gobj_write;
>> + goto put_queue;
>>
>> /* We are here means UQ is active, make sure the eviction fence is valid */
>> amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
>>
>> - /* Create a new fence */
>> - r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
>> - if (r) {
>> - mutex_unlock(&userq_mgr->userq_mutex);
>> - kfree(userq_fence);
>> - goto put_gobj_write;
>> - }
>> + /* Create the new fence */
>> + amdgpu_userq_fence_init(queue, fence, wptr);
>>
>> - dma_fence_put(queue->last_fence);
>> - queue->last_fence = dma_fence_get(fence);
>> - amdgpu_userq_start_hang_detect_work(queue);
>> mutex_unlock(&userq_mgr->userq_mutex);
>>
>> + /*
>> + * This needs to come after the fence is created since
>> + * amdgpu_userq_ensure_ev_fence() can't be called while holding the resv
>> + * locks.
>> + */
>> drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
>> (num_read_bo_handles + num_write_bo_handles));
>>
>> - /* Lock all BOs with retry handling */
>> drm_exec_until_all_locked(&exec) {
>> - r = drm_exec_prepare_array(&exec, gobj_read,
>> num_read_bo_handles, 1);
>> + r = drm_exec_prepare_array(&exec, gobj_read,
>> + num_read_bo_handles, 1);
>> drm_exec_retry_on_contention(&exec);
>> - if (r) {
>> - amdgpu_userq_fence_cleanup(fence);
>> + if (r)
>> goto exec_fini;
>> - }
>>
>> - r = drm_exec_prepare_array(&exec, gobj_write,
>> num_write_bo_handles, 1);
>> + r = drm_exec_prepare_array(&exec, gobj_write,
>> + num_write_bo_handles, 1);
>> drm_exec_retry_on_contention(&exec);
>> - if (r) {
>> - amdgpu_userq_fence_cleanup(fence);
>> + if (r)
>> goto exec_fini;
>> - }
>> }
>>
>> - for (i = 0; i < num_read_bo_handles; i++) {
>> - if (!gobj_read || !gobj_read[i]->resv)
>> - continue;
>> -
>> - dma_resv_add_fence(gobj_read[i]->resv, fence,
>> + /* And publish the new fence in the BOs and syncobj */
>> + for (i = 0; i < num_read_bo_handles; i++)
>> + dma_resv_add_fence(gobj_read[i]->resv, &fence->base,
>> DMA_RESV_USAGE_READ);
>> - }
>>
>> - for (i = 0; i < num_write_bo_handles; i++) {
>> - if (!gobj_write || !gobj_write[i]->resv)
>> - continue;
>> -
>> - dma_resv_add_fence(gobj_write[i]->resv, fence,
>> + for (i = 0; i < num_write_bo_handles; i++)
>> + dma_resv_add_fence(gobj_write[i]->resv, &fence->base,
>> DMA_RESV_USAGE_WRITE);
>> - }
>>
>> - /* Add the created fence to syncobj/BO's */
>> for (i = 0; i < num_syncobj_handles; i++)
>> - drm_syncobj_replace_fence(syncobj[i], fence);
>> + drm_syncobj_replace_fence(syncobj[i], &fence->base);
>>
>> +exec_fini:
>> /* drop the reference acquired in fence creation function */
>> - dma_fence_put(fence);
>> + dma_fence_put(&fence->base);
>>
>> -exec_fini:
>> drm_exec_fini(&exec);
>> +put_queue:
>> + amdgpu_userq_put(queue);
>> put_gobj_write:
>> for (i = 0; i < num_write_bo_handles; i++)
>> drm_gem_object_put(gobj_write[i]);
>> @@ -590,15 +575,11 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev,
>> void *data,
>> kvfree(gobj_read);
>> free_syncobj:
>> while (entry-- > 0)
>> - if (syncobj[entry])
>> - drm_syncobj_put(syncobj[entry]);
>> + drm_syncobj_put(syncobj[entry]);
>> kfree(syncobj);
>> free_syncobj_handles:
>> kfree(syncobj_handles);
>>
>> - if (queue)
>> - amdgpu_userq_put(queue);
>> -
>> return r;
>> }
>>
>> @@ -873,8 +854,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file
>> *filp,
>> * Otherwise, we would gather those references until we don't
>> * have any more space left and crash.
>> */
>> + mutex_lock(&waitq->fence_drv_lock);
>> r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
>> xa_limit_32b, GFP_KERNEL);
>> + mutex_unlock(&waitq->fence_drv_lock);
>> if (r)
>> goto put_waitq;
>>
>> --
>> 2.43.0
>
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH 3/8] drm/amdgpu: rework amdgpu_userq_signal_ioctl v2
2026-04-28 9:35 ` Khatri, Sunil
@ 2026-05-04 12:37 ` Christian König
0 siblings, 0 replies; 30+ messages in thread
From: Christian König @ 2026-05-04 12:37 UTC (permalink / raw)
To: Khatri, Sunil, alexander.deucher, Prike.Liang, jesse.zhang; +Cc: amd-gfx
On 4/28/26 11:35, Khatri, Sunil wrote:
>
> On 27-04-2026 11:57 pm, Christian König wrote:
...
>> -static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
>> +static int amdgpu_userq_fence_alloc(struct amdgpu_usermode_queue *userq,
>> + struct amdgpu_userq_fence **pfence)
>> {
>> - *userq_fence = kmalloc(sizeof(**userq_fence), GFP_ATOMIC);
>> - return *userq_fence ? 0 : -ENOMEM;
>> + struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
>> + struct amdgpu_userq_fence *userq_fence;
>> + void * entry;
>> +
>> + userq_fence = kmalloc(sizeof(*userq_fence), GFP_KERNEL);
>> + if (!userq_fence)
>> + return -ENOMEM;
>> +
>> + /*
>> + * Get the next unused entry, since we fill from the start this can be
>> + * used as size to allocate the array.
>> + */
>> + mutex_lock(&userq->fence_drv_lock);
>> + XA_STATE(xas, &userq->fence_drv_xa, 0);
>> +
>> + rcu_read_lock();
>> + do {
>> + entry = xas_find_marked(&xas, ULONG_MAX, XA_FREE_MARK);
>> + } while (xas_retry(&xas, entry));
>> + rcu_read_unlock();
> Functionally its correct but Using xa_for_each to count the no of entries should suffice for what we need? why doing more work or this approach is fixing something which xa_for_each does not ?
The xa is a radix tree, using xa_for_each to count the number of entries is just horrible inefficient.
>> +
>> + userq_fence->fence_drv_array = kvmalloc_array(xas.xa_index,
>> + sizeof(fence_drv),
>> + GFP_KERNEL);
>> + if (!userq_fence->fence_drv_array) {
>> + mutex_unlock(&userq->fence_drv_lock);
>> + kfree(userq_fence);
>> + return -ENOMEM;
>> + }
>> +
>> + userq_fence->fence_drv_array_count = xas.xa_index;
>> + xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array,
>> + 0, ULONG_MAX, xas.xa_index, XA_PRESENT);
> same count could be used here also to copy, just open questions but the current code is good to go.
> Irrespective of the change the code is Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Thanks,
Christian.
>
> Regards
> Sunil Khatri
>> + xa_destroy(&userq->fence_drv_xa);
>> +
>> + mutex_unlock(&userq->fence_drv_lock);
>> +
>> + userq_fence->fence_drv = fence_drv;
>> + amdgpu_userq_fence_driver_get(fence_drv);
>> +
>> + *pfence = userq_fence;
>> + return 0;
>> }
>> -static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
>> - struct amdgpu_userq_fence *userq_fence,
>> - u64 seq, struct dma_fence **f)
>> +static void amdgpu_userq_fence_init(struct amdgpu_usermode_queue *userq,
>> + struct amdgpu_userq_fence *fence,
>> + u64 seq)
>> {
>> - struct amdgpu_userq_fence_driver *fence_drv;
>> - struct dma_fence *fence;
>> + struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
>> unsigned long flags;
>> bool signaled = false;
>> - fence_drv = userq->fence_drv;
>> - if (!fence_drv)
>> - return -EINVAL;
>> -
>> - spin_lock_init(&userq_fence->lock);
>> - INIT_LIST_HEAD(&userq_fence->link);
>> - fence = &userq_fence->base;
>> - userq_fence->fence_drv = fence_drv;
>> -
>> - dma_fence_init64(fence, &amdgpu_userq_fence_ops, &userq_fence->lock,
>> + spin_lock_init(&fence->lock);
>> + dma_fence_init64(&fence->base, &amdgpu_userq_fence_ops, &fence->lock,
>> fence_drv->context, seq);
>> - amdgpu_userq_fence_driver_get(fence_drv);
>> - dma_fence_get(fence);
>> -
>> - if (!xa_empty(&userq->fence_drv_xa)) {
>> - struct amdgpu_userq_fence_driver *stored_fence_drv;
>> - unsigned long index, count = 0;
>> - int i = 0;
>> -
>> - xa_lock(&userq->fence_drv_xa);
>> - xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv)
>> - count++;
>> -
>> - userq_fence->fence_drv_array =
>> - kvmalloc_array(count,
>> - sizeof(struct amdgpu_userq_fence_driver *),
>> - GFP_ATOMIC);
>> -
>> - if (userq_fence->fence_drv_array) {
>> - xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv) {
>> - userq_fence->fence_drv_array[i] = stored_fence_drv;
>> - __xa_erase(&userq->fence_drv_xa, index);
>> - i++;
>> - }
>> - }
>> -
>> - userq_fence->fence_drv_array_count = i;
>> - xa_unlock(&userq->fence_drv_xa);
>> - } else {
>> - userq_fence->fence_drv_array = NULL;
>> - userq_fence->fence_drv_array_count = 0;
>> - }
>> + /* Make sure the fence is visible to the hang detect worker */
>> + dma_fence_put(userq->last_fence);
>> + userq->last_fence = dma_fence_get(&fence->base);
>> - /* Check if hardware has already processed the job */
>> + /* Check if hardware has already processed the fence */
>> spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
>> - if (!dma_fence_is_signaled(fence)) {
>> - list_add_tail(&userq_fence->link, &fence_drv->fences);
>> + if (!dma_fence_is_signaled(&fence->base)) {
>> + dma_fence_get(&fence->base);
>> + list_add_tail(&fence->link, &fence_drv->fences);
>> } else {
>> + INIT_LIST_HEAD(&fence->link);
>> signaled = true;
>> - dma_fence_put(fence);
>> }
>> spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
>> if (signaled)
>> - amdgpu_userq_fence_put_fence_drv_array(userq_fence);
>> -
>> - *f = fence;
>> -
>> - return 0;
>> + amdgpu_userq_fence_put_fence_drv_array(fence);
>> + else
>> + amdgpu_userq_start_hang_detect_work(userq);
>> }
>> static const char *amdgpu_userq_fence_get_driver_name(struct dma_fence *f)
>> @@ -404,11 +408,6 @@ static int amdgpu_userq_fence_read_wptr(struct amdgpu_device *adev,
>> return r;
>> }
>> -static void amdgpu_userq_fence_cleanup(struct dma_fence *fence)
>> -{
>> - dma_fence_put(fence);
>> -}
>> -
>> static void
>> amdgpu_userq_fence_driver_set_error(struct amdgpu_userq_fence *fence,
>> int error)
>> @@ -452,13 +451,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>> const unsigned int num_read_bo_handles = args->num_bo_read_handles;
>> struct amdgpu_fpriv *fpriv = filp->driver_priv;
>> struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
>> +
>> struct drm_gem_object **gobj_write, **gobj_read;
>> u32 *syncobj_handles, num_syncobj_handles;
>> - struct amdgpu_userq_fence *userq_fence;
>> - struct amdgpu_usermode_queue *queue = NULL;
>> - struct drm_syncobj **syncobj = NULL;
>> - struct dma_fence *fence;
>> + struct amdgpu_usermode_queue *queue;
>> + struct amdgpu_userq_fence *fence;
>> + struct drm_syncobj **syncobj;
>> struct drm_exec exec;
>> + void __user *ptr;
>> int r, i, entry;
>> u64 wptr;
>> @@ -470,13 +470,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>> return -EINVAL;
>> num_syncobj_handles = args->num_syncobj_handles;
>> - syncobj_handles = memdup_array_user(u64_to_user_ptr(args->syncobj_handles),
>> - num_syncobj_handles, sizeof(u32));
>> + ptr = u64_to_user_ptr(args->syncobj_handles);
>> + syncobj_handles = memdup_array_user(ptr, num_syncobj_handles,
>> + sizeof(u32));
>> if (IS_ERR(syncobj_handles))
>> return PTR_ERR(syncobj_handles);
>> - /* Array of pointers to the looked up syncobjs */
>> - syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj), GFP_KERNEL);
>> + syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj),
>> + GFP_KERNEL);
>> if (!syncobj) {
>> r = -ENOMEM;
>> goto free_syncobj_handles;
>> @@ -490,21 +491,17 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>> }
>> }
>> - r = drm_gem_objects_lookup(filp,
>> - u64_to_user_ptr(args->bo_read_handles),
>> - num_read_bo_handles,
>> - &gobj_read);
>> + ptr = u64_to_user_ptr(args->bo_read_handles);
>> + r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
>> if (r)
>> goto free_syncobj;
>> - r = drm_gem_objects_lookup(filp,
>> - u64_to_user_ptr(args->bo_write_handles),
>> - num_write_bo_handles,
>> + ptr = u64_to_user_ptr(args->bo_write_handles);
>> + r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
>> &gobj_write);
>> if (r)
>> goto put_gobj_read;
>> - /* Retrieve the user queue */
>> queue = amdgpu_userq_get(userq_mgr, args->queue_id);
>> if (!queue) {
>> r = -ENOENT;
>> @@ -513,73 +510,61 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>> r = amdgpu_userq_fence_read_wptr(adev, queue, &wptr);
>> if (r)
>> - goto put_gobj_write;
>> + goto put_queue;
>> - r = amdgpu_userq_fence_alloc(&userq_fence);
>> + r = amdgpu_userq_fence_alloc(queue, &fence);
>> if (r)
>> - goto put_gobj_write;
>> + goto put_queue;
>> /* We are here means UQ is active, make sure the eviction fence is valid */
>> amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
>> - /* Create a new fence */
>> - r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
>> - if (r) {
>> - mutex_unlock(&userq_mgr->userq_mutex);
>> - kfree(userq_fence);
>> - goto put_gobj_write;
>> - }
>> + /* Create the new fence */
>> + amdgpu_userq_fence_init(queue, fence, wptr);
>> - dma_fence_put(queue->last_fence);
>> - queue->last_fence = dma_fence_get(fence);
>> - amdgpu_userq_start_hang_detect_work(queue);
>> mutex_unlock(&userq_mgr->userq_mutex);
>> + /*
>> + * This needs to come after the fence is created since
>> + * amdgpu_userq_ensure_ev_fence() can't be called while holding the resv
>> + * locks.
>> + */
>> drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
>> (num_read_bo_handles + num_write_bo_handles));
>> - /* Lock all BOs with retry handling */
>> drm_exec_until_all_locked(&exec) {
>> - r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1);
>> + r = drm_exec_prepare_array(&exec, gobj_read,
>> + num_read_bo_handles, 1);
>> drm_exec_retry_on_contention(&exec);
>> - if (r) {
>> - amdgpu_userq_fence_cleanup(fence);
>> + if (r)
>> goto exec_fini;
>> - }
>> - r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1);
>> + r = drm_exec_prepare_array(&exec, gobj_write,
>> + num_write_bo_handles, 1);
>> drm_exec_retry_on_contention(&exec);
>> - if (r) {
>> - amdgpu_userq_fence_cleanup(fence);
>> + if (r)
>> goto exec_fini;
>> - }
>> }
>> - for (i = 0; i < num_read_bo_handles; i++) {
>> - if (!gobj_read || !gobj_read[i]->resv)
>> - continue;
>> -
>> - dma_resv_add_fence(gobj_read[i]->resv, fence,
>> + /* And publish the new fence in the BOs and syncobj */
>> + for (i = 0; i < num_read_bo_handles; i++)
>> + dma_resv_add_fence(gobj_read[i]->resv, &fence->base,
>> DMA_RESV_USAGE_READ);
>> - }
>> - for (i = 0; i < num_write_bo_handles; i++) {
>> - if (!gobj_write || !gobj_write[i]->resv)
>> - continue;
>> -
>> - dma_resv_add_fence(gobj_write[i]->resv, fence,
>> + for (i = 0; i < num_write_bo_handles; i++)
>> + dma_resv_add_fence(gobj_write[i]->resv, &fence->base,
>> DMA_RESV_USAGE_WRITE);
>> - }
>> - /* Add the created fence to syncobj/BO's */
>> for (i = 0; i < num_syncobj_handles; i++)
>> - drm_syncobj_replace_fence(syncobj[i], fence);
>> + drm_syncobj_replace_fence(syncobj[i], &fence->base);
>> +exec_fini:
>> /* drop the reference acquired in fence creation function */
>> - dma_fence_put(fence);
>> + dma_fence_put(&fence->base);
>> -exec_fini:
>> drm_exec_fini(&exec);
>> +put_queue:
>> + amdgpu_userq_put(queue);
>> put_gobj_write:
>> for (i = 0; i < num_write_bo_handles; i++)
>> drm_gem_object_put(gobj_write[i]);
>> @@ -590,15 +575,11 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
>> kvfree(gobj_read);
>> free_syncobj:
>> while (entry-- > 0)
>> - if (syncobj[entry])
>> - drm_syncobj_put(syncobj[entry]);
>> + drm_syncobj_put(syncobj[entry]);
>> kfree(syncobj);
>> free_syncobj_handles:
>> kfree(syncobj_handles);
>> - if (queue)
>> - amdgpu_userq_put(queue);
>> -
>> return r;
>> }
>> @@ -873,8 +854,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
>> * Otherwise, we would gather those references until we don't
>> * have any more space left and crash.
>> */
>> + mutex_lock(&waitq->fence_drv_lock);
>> r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
>> xa_limit_32b, GFP_KERNEL);
>> + mutex_unlock(&waitq->fence_drv_lock);
>> if (r)
>> goto put_waitq;
>>
^ permalink raw reply [flat|nested] 30+ messages in thread
* RE: [PATCH 7/8] drm/amdgpu: fix handling in amdgpu_userq_create
2026-04-27 18:27 ` [PATCH 7/8] drm/amdgpu: fix handling in amdgpu_userq_create Christian König
2026-04-28 11:26 ` Khatri, Sunil
@ 2026-05-07 6:13 ` Liang, Prike
1 sibling, 0 replies; 30+ messages in thread
From: Liang, Prike @ 2026-05-07 6:13 UTC (permalink / raw)
To: Koenig, Christian, Deucher, Alexander, Khatri, Sunil,
Zhang, Jesse(Jie)
Cc: amd-gfx@lists.freedesktop.org
AMD General
Reviewed-by: Prike Liang <Prike.Liang@amd.com>
Regards,
Prike
> -----Original Message-----
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> Sent: Tuesday, April 28, 2026 2:27 AM
> To: Deucher, Alexander <Alexander.Deucher@amd.com>; Liang, Prike
> <Prike.Liang@amd.com>; Khatri, Sunil <Sunil.Khatri@amd.com>; Zhang,
> Jesse(Jie) <Jesse.Zhang@amd.com>
> Cc: amd-gfx@lists.freedesktop.org
> Subject: [PATCH 7/8] drm/amdgpu: fix handling in amdgpu_userq_create
>
> Well mostly the same issues the other code had as well:
>
> 1. Memory allocation while holding the userq_mutex lock is forbidden!
> 2. Things were created/started/published in the wrong order.
> 3. Error messages on invalid input parameters can spam the logs.
> 4. Error messages on memory allocation failures are usually superflous
> as well.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 118 ++++++++++------------
> 1 file changed, 52 insertions(+), 66 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index 0374a70b631c..28e0695b53e7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -718,14 +718,14 @@ amdgpu_userq_create(struct drm_file *filp, union
> drm_amdgpu_userq *args)
> const struct amdgpu_userq_funcs *uq_funcs;
> struct amdgpu_usermode_queue *queue;
> struct amdgpu_db_info db_info;
> - bool skip_map_queue;
> - u32 qid;
> uint64_t index;
> - int r = 0;
> - int priority =
> - (args->in.flags &
> AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK) >>
> - AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT;
> + int priority;
> + u32 qid;
> + int r;
>
> + priority =
> + (args->in.flags &
> AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK)
> + >>
> AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT;
> r = amdgpu_userq_priority_permit(filp, priority);
> if (r)
> return r;
> @@ -738,40 +738,43 @@ amdgpu_userq_create(struct drm_file *filp, union
> drm_amdgpu_userq *args)
>
> uq_funcs = adev->userq_funcs[args->in.ip_type];
> if (!uq_funcs) {
> - drm_file_err(uq_mgr->file, "Usermode queue is not supported for this
> IP (%u)\n",
> - args->in.ip_type);
> r = -EINVAL;
> goto err_pm_runtime;
> }
>
> queue = kzalloc(sizeof(struct amdgpu_usermode_queue), GFP_KERNEL);
> if (!queue) {
> - drm_file_err(uq_mgr->file, "Failed to allocate memory for queue\n");
> r = -ENOMEM;
> goto err_pm_runtime;
> }
>
> + kref_init(&queue->refcount);
> INIT_LIST_HEAD(&queue->userq_va_list);
> queue->doorbell_handle = args->in.doorbell_handle;
> queue->queue_type = args->in.ip_type;
> queue->vm = &fpriv->vm;
> queue->priority = priority;
> -
> - db_info.queue_type = queue->queue_type;
> - db_info.doorbell_handle = queue->doorbell_handle;
> - db_info.db_obj = &queue->db_obj;
> - db_info.doorbell_offset = args->in.doorbell_offset;
> -
> queue->userq_mgr = uq_mgr;
> + INIT_DELAYED_WORK(&queue->hang_detect_work,
> + amdgpu_userq_hang_detect_work);
>
> - /* Validate the userq virtual address.*/
> - r = amdgpu_bo_reserve(fpriv->vm.root.bo, false);
> + mutex_init(&queue->fence_drv_lock);
> + xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
> + r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
> if (r)
> goto free_queue;
>
> - if (amdgpu_userq_input_va_validate(adev, queue, args->in.queue_va, args-
> >in.queue_size) ||
> - amdgpu_userq_input_va_validate(adev, queue, args->in.rptr_va,
> AMDGPU_GPU_PAGE_SIZE) ||
> - amdgpu_userq_input_va_validate(adev, queue, args->in.wptr_va,
> AMDGPU_GPU_PAGE_SIZE)) {
> + /* Make sure the queue can actually run with those virtual addresses. */
> + r = amdgpu_bo_reserve(fpriv->vm.root.bo, false);
> + if (r)
> + goto free_fence_drv;
> +
> + if (amdgpu_userq_input_va_validate(adev, queue, args->in.queue_va,
> + args->in.queue_size) ||
> + amdgpu_userq_input_va_validate(adev, queue, args->in.rptr_va,
> + AMDGPU_GPU_PAGE_SIZE) ||
> + amdgpu_userq_input_va_validate(adev, queue, args->in.wptr_va,
> + AMDGPU_GPU_PAGE_SIZE)) {
> r = -EINVAL;
> amdgpu_bo_unreserve(fpriv->vm.root.bo);
> goto clean_mapping;
> @@ -779,6 +782,10 @@ amdgpu_userq_create(struct drm_file *filp, union
> drm_amdgpu_userq *args)
> amdgpu_bo_unreserve(fpriv->vm.root.bo);
>
> /* Convert relative doorbell offset into absolute doorbell index */
> + db_info.queue_type = queue->queue_type;
> + db_info.doorbell_handle = queue->doorbell_handle;
> + db_info.db_obj = &queue->db_obj;
> + db_info.doorbell_offset = args->in.doorbell_offset;
> index = amdgpu_userq_get_doorbell_index(uq_mgr, &db_info, filp);
> if (index == (uint64_t)-EINVAL) {
> drm_file_err(uq_mgr->file, "Failed to get doorbell for queue\n"); @@ -
> 787,82 +794,61 @@ amdgpu_userq_create(struct drm_file *filp, union
> drm_amdgpu_userq *args)
> }
>
> queue->doorbell_index = index;
> - mutex_init(&queue->fence_drv_lock);
> - xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
> - r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
> - if (r) {
> - drm_file_err(uq_mgr->file, "Failed to alloc fence driver\n");
> - goto clean_mapping;
> - }
> -
> r = uq_funcs->mqd_create(queue, &args->in);
> if (r) {
> drm_file_err(uq_mgr->file, "Failed to create Queue\n");
> - goto clean_fence_driver;
> + goto clean_mapping;
> }
>
> + r = xa_err(xa_store_irq(&adev->userq_doorbell_xa, index, queue,
> + GFP_KERNEL));
> + if (r)
> + goto clean_mqd;
> +
> amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
>
> /* don't map the queue if scheduling is halted */
> - if (adev->userq_halt_for_enforce_isolation &&
> - ((queue->queue_type == AMDGPU_HW_IP_GFX) ||
> - (queue->queue_type == AMDGPU_HW_IP_COMPUTE)))
> - skip_map_queue = true;
> - else
> - skip_map_queue = false;
> - if (!skip_map_queue) {
> + if (!adev->userq_halt_for_enforce_isolation ||
> + ((queue->queue_type != AMDGPU_HW_IP_GFX) &&
> + (queue->queue_type != AMDGPU_HW_IP_COMPUTE))) {
> r = amdgpu_userq_map_helper(queue);
> if (r) {
> drm_file_err(uq_mgr->file, "Failed to map Queue\n");
> - goto clean_mqd;
> + mutex_unlock(&uq_mgr->userq_mutex);
> + goto clean_doorbell;
> }
> }
>
> - /* drop this refcount during queue destroy */
> - kref_init(&queue->refcount);
> -
> - /* Wait for mode-1 reset to complete */
> - down_read(&adev->reset_domain->sem);
> + atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
> + mutex_unlock(&uq_mgr->userq_mutex);
>
> r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
> - XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
> GFP_KERNEL);
> - if (r) {
> - if (!skip_map_queue)
> - amdgpu_userq_unmap_helper(queue);
> - r = -ENOMEM;
> - goto clean_reset_domain;
> - }
> -
> - r = xa_err(xa_store_irq(&adev->userq_doorbell_xa, index, queue,
> GFP_KERNEL));
> + XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
> + GFP_KERNEL);
> if (r) {
> - xa_erase(&uq_mgr->userq_xa, qid);
> - if (!skip_map_queue)
> - amdgpu_userq_unmap_helper(queue);
> - goto clean_reset_domain;
> + /*
> + * This drops the last reference which should take care of
> + * all cleanup.
> + */
> + amdgpu_userq_put(queue);
> + return r;
> }
> - up_read(&adev->reset_domain->sem);
>
> amdgpu_debugfs_userq_init(filp, queue, qid);
> - INIT_DELAYED_WORK(&queue->hang_detect_work,
> - amdgpu_userq_hang_detect_work);
> -
> args->out.queue_id = qid;
> - atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
> - mutex_unlock(&uq_mgr->userq_mutex);
> return 0;
>
> -clean_reset_domain:
> - up_read(&adev->reset_domain->sem);
> +clean_doorbell:
> + xa_erase_irq(&adev->userq_doorbell_xa, index);
> clean_mqd:
> - mutex_unlock(&uq_mgr->userq_mutex);
> uq_funcs->mqd_destroy(queue);
> -clean_fence_driver:
> - amdgpu_userq_fence_driver_free(queue);
> clean_mapping:
> amdgpu_bo_reserve(fpriv->vm.root.bo, true);
> amdgpu_userq_buffer_vas_list_cleanup(adev, queue);
> amdgpu_bo_unreserve(fpriv->vm.root.bo);
> mutex_destroy(&queue->fence_drv_lock);
> +free_fence_drv:
> + amdgpu_userq_fence_driver_free(queue);
> free_queue:
> kfree(queue);
> err_pm_runtime:
> --
> 2.43.0
^ permalink raw reply [flat|nested] 30+ messages in thread
* RE: [PATCH 8/8] drm/amdgpu: restructure VM state machine v2
2026-04-27 18:27 ` [PATCH 8/8] drm/amdgpu: restructure VM state machine v2 Christian König
2026-04-28 12:20 ` Khatri, Sunil
@ 2026-05-08 2:24 ` Liang, Prike
2026-05-08 8:21 ` Christian König
2026-05-25 23:29 ` Timur Kristóf
2 siblings, 1 reply; 30+ messages in thread
From: Liang, Prike @ 2026-05-08 2:24 UTC (permalink / raw)
To: Koenig, Christian, Deucher, Alexander, Khatri, Sunil,
Zhang, Jesse(Jie)
Cc: amd-gfx@lists.freedesktop.org
AMD General
Regards,
Prike
> -----Original Message-----
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> Sent: Tuesday, April 28, 2026 2:27 AM
> To: Deucher, Alexander <Alexander.Deucher@amd.com>; Liang, Prike
> <Prike.Liang@amd.com>; Khatri, Sunil <Sunil.Khatri@amd.com>; Zhang,
> Jesse(Jie) <Jesse.Zhang@amd.com>
> Cc: amd-gfx@lists.freedesktop.org
> Subject: [PATCH 8/8] drm/amdgpu: restructure VM state machine v2
>
> Instead of coming up with more sophisticated names for states a VM BO can be in,
> group them by the type of BO first and then by the state.
>
> So we end with BO type kernel, always_valid and individual and then states evicted,
> moved and idle.
>
> Not much functional change, except that evicted_user is moved back together with
> the other BOs again which makes the handling in
> amdgpu_vm_validate() a bit more complex.
>
> Also fixes a problem with user queues and amdgpu_vm_ready(). We didn't
> considered the VM ready when user BOs were not ideally placed, harmless
> performance impact for kernel queues but a complete show stopper for userqueues.
>
> v2: fix a few typos in comments, rename the BO types to make them more
> descriptive, fix a couple of bugs found during testing
> v3: squashed together with revert to old status lock handling, looks
> like the first patch still had some bug which this one here should fix.
> Fix a missing lock around debugfs printing.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 65 +--
> drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 534 +++++++++-------------
> drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 69 ++-
> drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 4 -
> 4 files changed, 298 insertions(+), 374 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index 28e0695b53e7..07e30acbdc3a 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -805,41 +805,50 @@ amdgpu_userq_create(struct drm_file *filp, union
> drm_amdgpu_userq *args)
> if (r)
> goto clean_mqd;
>
> - amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
> + /* Make sure a concurrent close operation doesn't kill our userq */
> + kref_get(&queue->refcount);
> + r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
> + XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
> + GFP_KERNEL);
> + if (r) {
> + /*
> + * This drops the extra and last reference which should take
> + * care of all cleanup.
> + */
> + amdgpu_userq_put(queue);
> + amdgpu_userq_put(queue);
> + return r;
> + }
>
> /* don't map the queue if scheduling is halted */
> + /* TODO: That is racy! */
> if (!adev->userq_halt_for_enforce_isolation ||
> ((queue->queue_type != AMDGPU_HW_IP_GFX) &&
> (queue->queue_type != AMDGPU_HW_IP_COMPUTE))) {
> +
> + /* TODO: We could skip mapping if ev fence is signaled */
> + amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr,
> + &fpriv->evf_mgr);
> r = amdgpu_userq_map_helper(queue);
> if (r) {
> drm_file_err(uq_mgr->file, "Failed to map Queue\n");
> mutex_unlock(&uq_mgr->userq_mutex);
> - goto clean_doorbell;
> + /* Prevent racing with close */
> + if (xa_erase(&uq_mgr->userq_xa, qid) == queue)
> + amdgpu_userq_put(queue);
> + amdgpu_userq_put(queue);
> + return r;
> }
> + mutex_unlock(&uq_mgr->userq_mutex);
> }
>
> atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
> - mutex_unlock(&uq_mgr->userq_mutex);
> -
> - r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
> - XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
> - GFP_KERNEL);
> - if (r) {
> - /*
> - * This drops the last reference which should take care of
> - * all cleanup.
> - */
> - amdgpu_userq_put(queue);
> - return r;
> - }
>
> amdgpu_debugfs_userq_init(filp, queue, qid);
> + amdgpu_userq_put(queue);
> args->out.queue_id = qid;
> return 0;
Should this userq creation get/put and error handler changes be splashed into the patch#7?
> -clean_doorbell:
> - xa_erase_irq(&adev->userq_doorbell_xa, index);
> clean_mqd:
> uq_funcs->mqd_destroy(queue);
> clean_mapping:
> @@ -1035,12 +1044,12 @@ amdgpu_userq_bo_validate(struct amdgpu_device
> *adev, struct drm_exec *exec,
> struct amdgpu_bo *bo;
> int ret;
>
> - spin_lock(&vm->status_lock);
> - while (!list_empty(&vm->invalidated)) {
> - bo_va = list_first_entry(&vm->invalidated,
> + spin_lock(&vm->individual_lock);
> + while (!list_empty(&vm->always_valid.evicted)) {
> + bo_va = list_first_entry(&vm->always_valid.evicted,
> struct amdgpu_bo_va,
> base.vm_status);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> bo = bo_va->base.bo;
> ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2); @@ -1052,14
> +1061,14 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct
> drm_exec *exec,
> if (ret)
> return ret;
>
> - /* This moves the bo_va to the done list */
> + /* This moves the bo_va to the idle list */
> ret = amdgpu_vm_bo_update(adev, bo_va, false);
> if (ret)
> return ret;
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->individual_lock);
> }
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> return 0;
> }
> @@ -1091,7 +1100,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr
> *uq_mgr)
> if (unlikely(ret))
> goto unlock_all;
>
> - ret = amdgpu_vm_lock_done_list(vm, &exec, 1);
> + ret = amdgpu_vm_lock_individual(vm, &exec, 1);
In the following ttm_bo_validate() may consumer all the reservation fence slots by the reservation internal pipeline eviction fences or move fences, then this will
result in the userq eviction fences attached failed due to the out of reservation fence slots. How about set the amdgpu_vm_lock_individual() reservation fence
number as TTM_NUM_MOVE_FENCES + 1 (validate() move path fence slots + userq eviction fence)?
> drm_exec_retry_on_contention(&exec);
> if (unlikely(ret))
> goto unlock_all;
> @@ -1134,7 +1143,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr
> *uq_mgr)
>
> key = 0;
> /* Validate User Ptr BOs */
> - list_for_each_entry(bo_va, &vm->done, base.vm_status) {
> + list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status) {
> bo = bo_va->base.bo;
> if (!bo)
> continue;
> @@ -1184,10 +1193,10 @@ amdgpu_userq_vm_validate(struct
> amdgpu_userq_mgr *uq_mgr)
>
> /*
> * We need to wait for all VM updates to finish before restarting the
> - * queues. Using the done list like that is now ok since everything is
> + * queues. Using the idle list like that is now ok since everything is
> * locked in place.
> */
> - list_for_each_entry(bo_va, &vm->done, base.vm_status)
> + list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status)
> dma_fence_wait(bo_va->last_pt_update, false);
> dma_fence_wait(vm->last_update, false);
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> index 6b58b0ecfa06..9f71a146f1f6 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -138,6 +138,47 @@ static void amdgpu_vm_assert_locked(struct amdgpu_vm
> *vm)
> dma_resv_assert_held(vm->root.bo->tbo.base.resv);
> }
>
> +/* Initialize the amdgpu_vm_bo_status object */ static void
> +amdgpu_vm_bo_status_init(struct amdgpu_vm_bo_status *lists) {
> + INIT_LIST_HEAD(&lists->evicted);
> + INIT_LIST_HEAD(&lists->moved);
> + INIT_LIST_HEAD(&lists->idle);
> +}
> +
> +/*
> + * Make sure we have the lock to modify the vm_bo status and return the
> +object
> + * with the status lists.
> + */
> +static struct amdgpu_vm_bo_status *
> +amdgpu_vm_bo_lock_lists(struct amdgpu_vm_bo_base *vm_bo) {
> + struct amdgpu_vm *vm = vm_bo->vm;
> + struct amdgpu_bo *bo = vm_bo->bo;
> +
> + if (amdgpu_vm_is_bo_always_valid(vm, bo)) {
> + /* No extra locking needed, protected by the root PD resv lock */
> + amdgpu_vm_assert_locked(vm);
> +
> + if (bo->tbo.type == ttm_bo_type_kernel)
> + return &vm->kernel;
> +
> + return &vm->always_valid;
> + }
> +
> + spin_lock(&vm_bo->vm->individual_lock);
> + return &vm->individual;
> +}
> +
> +/* Eventually unlock the status list lock again */ static void
> +amdgpu_vm_bo_unlock_lists(struct amdgpu_vm_bo_base *vm_bo) {
> + if (amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
> + amdgpu_vm_assert_locked(vm_bo->vm);
> + else
> + spin_unlock(&vm_bo->vm->individual_lock);
> +}
> +
> /**
> * amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid
> *
> @@ -157,37 +198,44 @@ bool amdgpu_vm_is_bo_always_valid(struct
> amdgpu_vm *vm, struct amdgpu_bo *bo)
> *
> * @vm_bo: vm_bo which is evicted
> *
> - * State for PDs/PTs and per VM BOs which are not at the location they should
> - * be.
> + * State for vm_bo objects meaning the underlying BO was evicted and
> + need to
> + * move in place again.
> */
> static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo) {
> - struct amdgpu_vm *vm = vm_bo->vm;
> - struct amdgpu_bo *bo = vm_bo->bo;
> + struct amdgpu_vm_bo_status *lists;
>
> + lists = amdgpu_vm_bo_lock_lists(vm_bo);
> vm_bo->moved = true;
> - amdgpu_vm_assert_locked(vm);
> - spin_lock(&vm_bo->vm->status_lock);
> - if (bo->tbo.type == ttm_bo_type_kernel)
> - list_move(&vm_bo->vm_status, &vm->evicted);
> - else
> - list_move_tail(&vm_bo->vm_status, &vm->evicted);
> - spin_unlock(&vm_bo->vm->status_lock);
> + list_move(&vm_bo->vm_status, &lists->evicted);
> + amdgpu_vm_bo_unlock_lists(vm_bo);
> }
> /**
> * amdgpu_vm_bo_moved - vm_bo is moved
> *
> * @vm_bo: vm_bo which is moved
> *
> - * State for per VM BOs which are moved, but that change is not yet reflected
> - * in the page tables.
> + * State for vm_bo objects meaning the underlying BO was moved but the
> + new
> + * location not yet reflected in the page tables.
> */
> static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo) {
> - amdgpu_vm_assert_locked(vm_bo->vm);
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
> - spin_unlock(&vm_bo->vm->status_lock);
> + struct amdgpu_vm_bo_status *lists;
> + struct amdgpu_bo *bo = vm_bo->bo;
> +
> + /*
> + * The root PD doesn't have a parent PDE and goes directly into the
> + * idle state.
> + */
> + lists = amdgpu_vm_bo_lock_lists(vm_bo);
> + if (bo && bo->tbo.type == ttm_bo_type_kernel && !bo->parent) {
> + vm_bo->moved = false;
> + list_move(&vm_bo->vm_status, &lists->idle);
> + } else {
> + vm_bo->moved = true;
> + list_move(&vm_bo->vm_status, &lists->moved);
> + }
> + amdgpu_vm_bo_unlock_lists(vm_bo);
> }
>
> /**
> @@ -195,120 +243,43 @@ static void amdgpu_vm_bo_moved(struct
> amdgpu_vm_bo_base *vm_bo)
> *
> * @vm_bo: vm_bo which is now idle
> *
> - * State for PDs/PTs and per VM BOs which have gone through the state machine
> - * and are now idle.
> + * State for vm_bo objects meaning we are done with the state machine
> + and no
> + * further action is necessary.
> */
> static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo) {
> - amdgpu_vm_assert_locked(vm_bo->vm);
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
> - spin_unlock(&vm_bo->vm->status_lock);
> - vm_bo->moved = false;
> -}
> + struct amdgpu_vm_bo_status *lists;
>
> -/**
> - * amdgpu_vm_bo_invalidated - vm_bo is invalidated
> - *
> - * @vm_bo: vm_bo which is now invalidated
> - *
> - * State for normal BOs which are invalidated and that change not yet reflected
> - * in the PTs.
> - */
> -static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo) -{
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
> - spin_unlock(&vm_bo->vm->status_lock);
> -}
> -
> -/**
> - * amdgpu_vm_bo_evicted_user - vm_bo is evicted
> - *
> - * @vm_bo: vm_bo which is evicted
> - *
> - * State for BOs used by user mode queues which are not at the location they
> - * should be.
> - */
> -static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo) -{
> - vm_bo->moved = true;
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user);
> - spin_unlock(&vm_bo->vm->status_lock);
> -}
> -
> -/**
> - * amdgpu_vm_bo_relocated - vm_bo is reloacted
> - *
> - * @vm_bo: vm_bo which is relocated
> - *
> - * State for PDs/PTs which needs to update their parent PD.
> - * For the root PD, just move to idle state.
> - */
> -static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo) -{
> - amdgpu_vm_assert_locked(vm_bo->vm);
> - if (vm_bo->bo->parent) {
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
> - spin_unlock(&vm_bo->vm->status_lock);
> - } else {
> - amdgpu_vm_bo_idle(vm_bo);
> - }
> -}
> -
> -/**
> - * amdgpu_vm_bo_done - vm_bo is done
> - *
> - * @vm_bo: vm_bo which is now done
> - *
> - * State for normal BOs which are invalidated and that change has been updated
> - * in the PTs.
> - */
> -static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo) -{
> - amdgpu_vm_assert_locked(vm_bo->vm);
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->done);
> - spin_unlock(&vm_bo->vm->status_lock);
> + lists = amdgpu_vm_bo_lock_lists(vm_bo);
> + if (!amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
> + vm_bo->moved = false;
> + list_move(&vm_bo->vm_status, &lists->idle);
> + amdgpu_vm_bo_unlock_lists(vm_bo);
> }
>
> /**
> * amdgpu_vm_bo_reset_state_machine - reset the vm_bo state machine
> * @vm: the VM which state machine to reset
> *
> - * Move all vm_bo object in the VM into a state where they will be updated
> - * again during validation.
> + * Move all vm_bo object in the VM into a state where their location
> + will be
> + * updated in the page tables again.
> */
> static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm) {
> - struct amdgpu_vm_bo_base *vm_bo, *tmp;
> -
> amdgpu_vm_assert_locked(vm);
> + list_splice_init(&vm->kernel.idle, &vm->kernel.moved);
> + list_splice_init(&vm->always_valid.idle, &vm->always_valid.moved);
>
> - spin_lock(&vm->status_lock);
> - list_splice_init(&vm->done, &vm->invalidated);
> - list_for_each_entry(vm_bo, &vm->invalidated, vm_status)
> - vm_bo->moved = true;
> -
> - list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) {
> - struct amdgpu_bo *bo = vm_bo->bo;
> -
> - vm_bo->moved = true;
> - if (!bo || bo->tbo.type != ttm_bo_type_kernel)
> - list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
> - else if (bo->parent)
> - list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
> - }
> - spin_unlock(&vm->status_lock);
> + spin_lock(&vm->individual_lock);
> + list_splice_init(&vm->individual.idle, &vm->individual.moved);
> + spin_unlock(&vm->individual_lock);
> }
>
> /**
> * amdgpu_vm_update_shared - helper to update shared memory stat
> * @base: base structure for tracking BO usage in a VM
> *
> - * Takes the vm status_lock and updates the shared memory stat. If the basic
> + * Takes the vm stats_lock and updates the shared memory stat. If the
> + basic
> * stat changed (e.g. buffer was moved) amdgpu_vm_update_stats need to be
> called
> * as well.
> */
> @@ -321,7 +292,7 @@ static void amdgpu_vm_update_shared(struct
> amdgpu_vm_bo_base *base)
> bool shared;
>
> dma_resv_assert_held(bo->tbo.base.resv);
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->stats_lock);
> shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
> if (base->shared != shared) {
> base->shared = shared;
> @@ -333,7 +304,7 @@ static void amdgpu_vm_update_shared(struct
> amdgpu_vm_bo_base *base)
> vm->stats[bo_memtype].drm.private += size;
> }
> }
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->stats_lock);
> }
>
> /**
> @@ -358,11 +329,11 @@ void amdgpu_vm_bo_update_shared(struct amdgpu_bo
> *bo)
> * be bo->tbo.resource
> * @sign: if we should add (+1) or subtract (-1) from the stat
> *
> - * Caller need to have the vm status_lock held. Useful for when multiple update
> + * Caller need to have the vm stats_lock held. Useful for when multiple
> + update
> * need to happen at the same time.
> */
> static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
> - struct ttm_resource *res, int sign)
> + struct ttm_resource *res, int sign)
> {
> struct amdgpu_vm *vm = base->vm;
> struct amdgpu_bo *bo = base->bo;
> @@ -386,7 +357,8 @@ static void amdgpu_vm_update_stats_locked(struct
> amdgpu_vm_bo_base *base,
> */
> if (bo->flags & AMDGPU_GEM_CREATE_DISCARDABLE)
> vm->stats[res_memtype].drm.purgeable += size;
> - if (!(bo->preferred_domains &
> amdgpu_mem_type_to_domain(res_memtype)))
> + if (!(bo->preferred_domains &
> + amdgpu_mem_type_to_domain(res_memtype)))
> vm->stats[bo_memtype].evicted += size;
> }
> }
> @@ -405,9 +377,9 @@ void amdgpu_vm_update_stats(struct
> amdgpu_vm_bo_base *base, {
> struct amdgpu_vm *vm = base->vm;
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->stats_lock);
> amdgpu_vm_update_stats_locked(base, res, sign);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->stats_lock);
> }
>
> /**
> @@ -428,37 +400,34 @@ void amdgpu_vm_bo_base_init(struct
> amdgpu_vm_bo_base *base,
> base->next = NULL;
> INIT_LIST_HEAD(&base->vm_status);
>
> + dma_resv_assert_held(vm->root.bo->tbo.base.resv);
> if (!bo)
> return;
> +
> base->next = bo->vm_bo;
> bo->vm_bo = base;
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->stats_lock);
> base->shared = drm_gem_object_is_shared_for_memory_stats(&bo-
> >tbo.base);
> amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->stats_lock);
>
> - if (!amdgpu_vm_is_bo_always_valid(vm, bo))
> + if (!amdgpu_vm_is_bo_always_valid(vm, bo)) {
> + amdgpu_vm_bo_idle(base);
> return;
> -
> - dma_resv_assert_held(vm->root.bo->tbo.base.resv);
> + }
>
> ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move);
> - if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
> - amdgpu_vm_bo_relocated(base);
> - else
> - amdgpu_vm_bo_idle(base);
>
> + /*
> + * When a per VM isn't in the desired domain put it into the evicted
> + * state to make sure that it gets validated on the next best occasion.
> + */
> if (bo->preferred_domains &
> amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type))
> - return;
> -
> - /*
> - * we checked all the prerequisites, but it looks like this per vm bo
> - * is currently evicted. add the bo to the evicted list to make sure it
> - * is validated on next vm use to avoid fault.
> - * */
> - amdgpu_vm_bo_evicted(base);
> + amdgpu_vm_bo_moved(base);
> + else
> + amdgpu_vm_bo_evicted(base);
> }
>
> /**
> @@ -479,41 +448,41 @@ int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct
> drm_exec *exec, }
>
> /**
> - * amdgpu_vm_lock_done_list - lock all BOs on the done list
> + * amdgpu_vm_lock_individual - lock all BOs on the individual idle list
> * @vm: vm providing the BOs
> * @exec: drm execution context
> * @num_fences: number of extra fences to reserve
> *
> - * Lock the BOs on the done list in the DRM execution context.
> + * Lock the BOs on the individual idle list in the DRM execution context.
> */
> -int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
> - unsigned int num_fences)
> +int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
> + unsigned int num_fences)
> {
> - struct list_head *prev = &vm->done;
> + struct list_head *prev = &vm->individual.idle;
> struct amdgpu_bo_va *bo_va;
> struct amdgpu_bo *bo;
> int ret;
>
> /* We can only trust prev->next while holding the lock */
> - spin_lock(&vm->status_lock);
> - while (!list_is_head(prev->next, &vm->done)) {
> + spin_lock(&vm->individual_lock);
> + while (!list_is_head(prev->next, &vm->individual.idle)) {
> bo_va = list_entry(prev->next, typeof(*bo_va), base.vm_status);
>
> bo = bo_va->base.bo;
> if (bo) {
> amdgpu_bo_ref(bo);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 1);
> amdgpu_bo_unref(&bo);
> if (unlikely(ret))
> return ret;
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->individual_lock);
> }
> prev = prev->next;
> }
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> return 0;
> }
> @@ -609,10 +578,10 @@ int amdgpu_vm_validate(struct amdgpu_device *adev,
> struct amdgpu_vm *vm,
> void *param)
> {
> uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm);
> - struct amdgpu_vm_bo_base *bo_base;
> - struct amdgpu_bo *bo;
> + struct amdgpu_vm_bo_base *bo_base, *tmp;
> int r;
>
> + dma_resv_assert_held(vm->root.bo->tbo.base.resv);
> if (vm->generation != new_vm_generation) {
> vm->generation = new_vm_generation;
> amdgpu_vm_bo_reset_state_machine(vm);
> @@ -622,49 +591,59 @@ int amdgpu_vm_validate(struct amdgpu_device *adev,
> struct amdgpu_vm *vm,
> return r;
> }
>
> - spin_lock(&vm->status_lock);
> - while (!list_empty(&vm->evicted)) {
> - bo_base = list_first_entry(&vm->evicted,
> - struct amdgpu_vm_bo_base,
> - vm_status);
> - spin_unlock(&vm->status_lock);
> + list_for_each_entry_safe(bo_base, tmp, &vm->kernel.evicted, vm_status) {
> + r = validate(param, bo_base->bo);
> + if (r)
> + return r;
>
> - bo = bo_base->bo;
> + vm->update_funcs->map_table(to_amdgpu_bo_vm(bo_base->bo));
> + amdgpu_vm_bo_moved(bo_base);
> + }
>
> - r = validate(param, bo);
> + /*
> + * As soon as all page tables are in place we can start updating them
> + * again.
> + */
> + amdgpu_vm_eviction_lock(vm);
> + vm->evicting = false;
> + amdgpu_vm_eviction_unlock(vm);
> +
> + list_for_each_entry_safe(bo_base, tmp, &vm->always_valid.evicted,
> + vm_status) {
> + r = validate(param, bo_base->bo);
> if (r)
> return r;
>
> - if (bo->tbo.type != ttm_bo_type_kernel) {
> - amdgpu_vm_bo_moved(bo_base);
> - } else {
> - vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
> - amdgpu_vm_bo_relocated(bo_base);
> - }
> - spin_lock(&vm->status_lock);
> + amdgpu_vm_bo_moved(bo_base);
> }
> - while (ticket && !list_empty(&vm->evicted_user)) {
> - bo_base = list_first_entry(&vm->evicted_user,
> - struct amdgpu_vm_bo_base,
> - vm_status);
> - spin_unlock(&vm->status_lock);
>
> - bo = bo_base->bo;
> - dma_resv_assert_held(bo->tbo.base.resv);
> + if (!ticket)
> + return 0;
> +
> + spin_lock(&vm->individual_lock);
> +restart:
> + list_for_each_entry(bo_base, &vm->individual.evicted, vm_status) {
> + struct amdgpu_bo *bo = bo_base->bo;
> +
> + if (dma_resv_locking_ctx(bo->tbo.base.resv) != ticket)
> + continue;
> +
> + spin_unlock(&vm->individual_lock);
>
> r = validate(param, bo);
> if (r)
> return r;
>
> - amdgpu_vm_bo_invalidated(bo_base);
> + amdgpu_vm_bo_moved(bo_base);
>
> - spin_lock(&vm->status_lock);
> + /* It's a bit inefficient to always jump back to the start, but
> + * we would need to re-structure the KFD for properly fixing
> + * that.
> + */
> + spin_lock(&vm->individual_lock);
> + goto restart;
> }
> - spin_unlock(&vm->status_lock);
> -
> - amdgpu_vm_eviction_lock(vm);
> - vm->evicting = false;
> - amdgpu_vm_eviction_unlock(vm);
> + spin_unlock(&vm->individual_lock);
>
> return 0;
> }
> @@ -689,9 +668,7 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
> ret = !vm->evicting;
> amdgpu_vm_eviction_unlock(vm);
>
> - spin_lock(&vm->status_lock);
> - ret &= list_empty(&vm->evicted);
> - spin_unlock(&vm->status_lock);
> + ret &= list_empty(&vm->kernel.evicted);
>
> spin_lock(&vm->immediate.lock);
> ret &= !vm->immediate.stopped;
> @@ -985,18 +962,13 @@ int amdgpu_vm_update_pdes(struct amdgpu_device
> *adev,
> struct amdgpu_vm *vm, bool immediate) {
> struct amdgpu_vm_update_params params;
> - struct amdgpu_vm_bo_base *entry;
> + struct amdgpu_vm_bo_base *entry, *tmp;
> bool flush_tlb_needed = false;
> - LIST_HEAD(relocated);
> int r, idx;
>
> amdgpu_vm_assert_locked(vm);
>
> - spin_lock(&vm->status_lock);
> - list_splice_init(&vm->relocated, &relocated);
> - spin_unlock(&vm->status_lock);
> -
> - if (list_empty(&relocated))
> + if (list_empty(&vm->kernel.moved))
> return 0;
>
> if (!drm_dev_enter(adev_to_drm(adev), &idx)) @@ -1012,7 +984,7 @@ int
> amdgpu_vm_update_pdes(struct amdgpu_device *adev,
> if (r)
> goto error;
>
> - list_for_each_entry(entry, &relocated, vm_status) {
> + list_for_each_entry(entry, &vm->kernel.moved, vm_status) {
> /* vm_flush_needed after updating moved PDEs */
> flush_tlb_needed |= entry->moved;
>
> @@ -1028,11 +1000,8 @@ int amdgpu_vm_update_pdes(struct amdgpu_device
> *adev,
> if (flush_tlb_needed)
> atomic64_inc(&vm->tlb_seq);
>
> - while (!list_empty(&relocated)) {
> - entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base,
> - vm_status);
> + list_for_each_entry_safe(entry, tmp, &vm->kernel.moved, vm_status)
> amdgpu_vm_bo_idle(entry);
> - }
>
> error:
> drm_dev_exit(idx);
> @@ -1260,9 +1229,9 @@ int amdgpu_vm_update_range(struct amdgpu_device
> *adev, struct amdgpu_vm *vm, void amdgpu_vm_get_memory(struct amdgpu_vm
> *vm,
> struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM]) {
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->stats_lock);
> memcpy(stats, vm->stats, sizeof(*stats) * __AMDGPU_PL_NUM);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->stats_lock);
> }
>
> /**
> @@ -1406,7 +1375,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
> struct amdgpu_bo_va *bo_va,
> else
> amdgpu_vm_bo_idle(&bo_va->base);
> } else {
> - amdgpu_vm_bo_done(&bo_va->base);
> + amdgpu_vm_bo_idle(&bo_va->base);
> }
>
> list_splice_init(&bo_va->invalids, &bo_va->valids); @@ -1629,29 +1598,25
> @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
> struct amdgpu_vm *vm,
> struct ww_acquire_ctx *ticket)
> {
> - struct amdgpu_bo_va *bo_va;
> + struct amdgpu_bo_va *bo_va, *tmp;
> struct dma_resv *resv;
> bool clear, unlock;
> int r;
>
> - spin_lock(&vm->status_lock);
> - while (!list_empty(&vm->moved)) {
> - bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va,
> - base.vm_status);
> - spin_unlock(&vm->status_lock);
> -
> + list_for_each_entry_safe(bo_va, tmp, &vm->always_valid.moved,
> + base.vm_status) {
> /* Per VM BOs never need to bo cleared in the page tables */
> r = amdgpu_vm_bo_update(adev, bo_va, false);
> if (r)
> return r;
> - spin_lock(&vm->status_lock);
> }
>
> - while (!list_empty(&vm->invalidated)) {
> - bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
> - base.vm_status);
> + spin_lock(&vm->individual_lock);
> + while (!list_empty(&vm->individual.moved)) {
> + bo_va = list_first_entry(&vm->individual.moved,
> + typeof(*bo_va), base.vm_status);
> resv = bo_va->base.bo->tbo.base.resv;
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> /* Try to reserve the BO to avoid clearing its ptes */
> if (!adev->debug_vm && dma_resv_trylock(resv)) { @@ -1681,11
> +1646,11 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
> drm_gem_is_imported(&bo_va->base.bo->tbo.base) &&
> (!bo_va->base.bo->tbo.resource ||
> bo_va->base.bo->tbo.resource->mem_type ==
> TTM_PL_SYSTEM))
> - amdgpu_vm_bo_evicted_user(&bo_va->base);
> + amdgpu_vm_bo_evicted(&bo_va->base);
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->individual_lock);
> }
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> return 0;
> }
> @@ -2223,9 +2188,9 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev,
> }
> }
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->individual_lock);
> list_del(&bo_va->base.vm_status);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
> list_del(&mapping->list);
> @@ -2305,14 +2270,7 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_bo *bo,
> bool evicted)
>
> if (bo_base->moved)
> continue;
> - bo_base->moved = true;
> -
> - if (bo->tbo.type == ttm_bo_type_kernel)
> - amdgpu_vm_bo_relocated(bo_base);
> - else if (amdgpu_vm_is_bo_always_valid(vm, bo))
> - amdgpu_vm_bo_moved(bo_base);
> - else
> - amdgpu_vm_bo_invalidated(bo_base);
> + amdgpu_vm_bo_moved(bo_base);
> }
> }
>
> @@ -2333,10 +2291,10 @@ void amdgpu_vm_bo_move(struct amdgpu_bo *bo,
> struct ttm_resource *new_mem,
> for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
> struct amdgpu_vm *vm = bo_base->vm;
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->stats_lock);
> amdgpu_vm_update_stats_locked(bo_base, bo->tbo.resource, -1);
> amdgpu_vm_update_stats_locked(bo_base, new_mem, +1);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->stats_lock);
> }
>
> amdgpu_vm_bo_invalidate(bo, evicted);
> @@ -2603,16 +2561,14 @@ int amdgpu_vm_init(struct amdgpu_device *adev,
> struct amdgpu_vm *vm,
> vm->va = RB_ROOT_CACHED;
> for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
> vm->reserved_vmid[i] = NULL;
> - INIT_LIST_HEAD(&vm->evicted);
> - INIT_LIST_HEAD(&vm->evicted_user);
> - INIT_LIST_HEAD(&vm->relocated);
> - INIT_LIST_HEAD(&vm->moved);
> - INIT_LIST_HEAD(&vm->idle);
> - INIT_LIST_HEAD(&vm->invalidated);
> - spin_lock_init(&vm->status_lock);
> +
> + amdgpu_vm_bo_status_init(&vm->kernel);
> + amdgpu_vm_bo_status_init(&vm->always_valid);
> + spin_lock_init(&vm->individual_lock);
> + amdgpu_vm_bo_status_init(&vm->individual);
> INIT_LIST_HEAD(&vm->freed);
> - INIT_LIST_HEAD(&vm->done);
> INIT_KFIFO(vm->faults);
> + spin_lock_init(&vm->stats_lock);
>
> r = amdgpu_vm_init_entities(adev, vm);
> if (r)
> @@ -3089,100 +3045,64 @@ bool amdgpu_vm_handle_fault(struct
> amdgpu_device *adev, u32 pasid, }
>
> #if defined(CONFIG_DEBUG_FS)
> -/**
> - * amdgpu_debugfs_vm_bo_info - print BO info for the VM
> - *
> - * @vm: Requested VM for printing BO info
> - * @m: debugfs file
> - *
> - * Print BO information in debugfs file for the VM
> - */
> -void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m) -{
> - struct amdgpu_bo_va *bo_va, *tmp;
> - u64 total_idle = 0;
> - u64 total_evicted = 0;
> - u64 total_relocated = 0;
> - u64 total_moved = 0;
> - u64 total_invalidated = 0;
> - u64 total_done = 0;
> - unsigned int total_idle_objs = 0;
> - unsigned int total_evicted_objs = 0;
> - unsigned int total_relocated_objs = 0;
> - unsigned int total_moved_objs = 0;
> - unsigned int total_invalidated_objs = 0;
> - unsigned int total_done_objs = 0;
> - unsigned int id = 0;
>
> - amdgpu_vm_assert_locked(vm);
> +/* print the debug info for a specific set of status lists */ static
> +void amdgpu_debugfs_vm_bo_status_info(struct seq_file *m,
> + struct amdgpu_vm_bo_status *lists) {
> + struct amdgpu_vm_bo_base *base;
> + unsigned int id;
>
> - spin_lock(&vm->status_lock);
> - seq_puts(m, "\tIdle BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
> - if (!bo_va->base.bo)
> - continue;
> - total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
> - }
> - total_idle_objs = id;
> id = 0;
> -
> seq_puts(m, "\tEvicted BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
> - if (!bo_va->base.bo)
> + list_for_each_entry(base, &lists->evicted, vm_status) {
> + if (!base->bo)
> continue;
> - total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
> - }
> - total_evicted_objs = id;
> - id = 0;
>
> - seq_puts(m, "\tRelocated BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
> - if (!bo_va->base.bo)
> - continue;
> - total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
> + amdgpu_bo_print_info(id++, base->bo, m);
> }
> - total_relocated_objs = id;
> - id = 0;
>
> + id = 0;
> seq_puts(m, "\tMoved BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
> - if (!bo_va->base.bo)
> + list_for_each_entry(base, &lists->moved, vm_status) {
> + if (!base->bo)
> continue;
> - total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
> +
> + amdgpu_bo_print_info(id++, base->bo, m);
> }
> - total_moved_objs = id;
> - id = 0;
>
> - seq_puts(m, "\tInvalidated BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
> - if (!bo_va->base.bo)
> + id = 0;
> + seq_puts(m, "\tIdle BOs:\n");
> + list_for_each_entry(base, &lists->moved, vm_status) {
> + if (!base->bo)
> continue;
> - total_invalidated += amdgpu_bo_print_info(id++, bo_va->base.bo,
> m);
> +
> + amdgpu_bo_print_info(id++, base->bo, m);
> }
> - total_invalidated_objs = id;
> - id = 0;
> +}
>
> - seq_puts(m, "\tDone BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
> - if (!bo_va->base.bo)
> - continue;
> - total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
> - }
> - spin_unlock(&vm->status_lock);
> - total_done_objs = id;
> -
> - seq_printf(m, "\tTotal idle size: %12lld\tobjs:\t%d\n", total_idle,
> - total_idle_objs);
> - seq_printf(m, "\tTotal evicted size: %12lld\tobjs:\t%d\n", total_evicted,
> - total_evicted_objs);
> - seq_printf(m, "\tTotal relocated size: %12lld\tobjs:\t%d\n", total_relocated,
> - total_relocated_objs);
> - seq_printf(m, "\tTotal moved size: %12lld\tobjs:\t%d\n", total_moved,
> - total_moved_objs);
> - seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated,
> - total_invalidated_objs);
> - seq_printf(m, "\tTotal done size: %12lld\tobjs:\t%d\n", total_done,
> - total_done_objs);
> +/**
> + * amdgpu_debugfs_vm_bo_info - print BO info for the VM
> + *
> + * @vm: Requested VM for printing BO info
> + * @m: debugfs file
> + *
> + * Print BO information in debugfs file for the VM */ void
> +amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m) {
> + amdgpu_vm_assert_locked(vm);
> +
> + seq_puts(m, "\tKernel PT/PDs:\n");
> + amdgpu_debugfs_vm_bo_status_info(m, &vm->kernel);
> +
> + seq_puts(m, "\tPer VM BOs:\n");
> + amdgpu_debugfs_vm_bo_status_info(m, &vm->always_valid);
> +
> + seq_puts(m, "\tIndividual BOs:\n");
> + spin_lock(&vm->individual_lock);
> + amdgpu_debugfs_vm_bo_status_info(m, &vm->individual);
> + spin_unlock(&vm->individual_lock);
> }
> #endif
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> index f33ea7f8509b..cc96a3e6252f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> @@ -205,17 +205,34 @@ struct amdgpu_vm_bo_base {
> /* protected by bo being reserved */
> struct amdgpu_vm_bo_base *next;
>
> - /* protected by vm status_lock */
> + /* protected by vm reservation and invalidated_lock */
> struct list_head vm_status;
>
> /* if the bo is counted as shared in mem stats
> - * protected by vm status_lock */
> + * protected by vm BO being reserved */
> bool shared;
>
> /* protected by the BO being reserved */
> bool moved;
> };
>
> +/*
> + * The following status lists contain amdgpu_vm_bo_base objects for
> + * either PD/PTs, per VM BOs or BOs with individual resv object.
> + *
> + * The state transits are: evicted -> moved -> idle */ struct
> +amdgpu_vm_bo_status {
> + /* BOs evicted which need to move into place again */
> + struct list_head evicted;
> +
> + /* BOs which moved but new location hasn't been updated in the PDs/PTs */
> + struct list_head moved;
> +
> + /* BOs done with the state machine and need no further action */
> + struct list_head idle;
> +};
> +
> /* provided by hw blocks that can write ptes, e.g., sdma */ struct
> amdgpu_vm_pte_funcs {
> /* number of dw to reserve per operation */ @@ -345,47 +362,29 @@ struct
> amdgpu_vm {
> bool evicting;
> unsigned int saved_flags;
>
> - /* Lock to protect vm_bo add/del/move on all lists of vm */
> - spinlock_t status_lock;
> -
> - /* Memory statistics for this vm, protected by status_lock */
> + /* Memory statistics for this vm, protected by stats_lock */
> + spinlock_t stats_lock;
> struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM];
>
> + /* BO's belonging to PD/PT which are internal to the kernel. */
> + struct amdgpu_vm_bo_status kernel;
> +
> /*
> - * The following lists contain amdgpu_vm_bo_base objects for either
> - * PDs, PTs or per VM BOs. The state transits are:
> - *
> - * evicted -> relocated (PDs, PTs) or moved (per VM BOs) -> idle
> + * BOs allocated by userspace where the dma_resv is shared with the
> + * root PD
> */
> -
> - /* Per-VM and PT BOs who needs a validation */
> - struct list_head evicted;
> -
> - /* PT BOs which relocated and their parent need an update */
> - struct list_head relocated;
> -
> - /* per VM BOs moved, but not yet updated in the PT */
> - struct list_head moved;
> -
> - /* All BOs of this VM not currently in the state machine */
> - struct list_head idle;
> + struct amdgpu_vm_bo_status always_valid;
>
> /*
> * The following lists contain amdgpu_vm_bo_base objects for BOs which
> - * have their own dma_resv object and not depend on the root PD. Their
> - * state transits are:
> + * have their own dma_resv object and not depend on the root PD.
> *
> - * evicted_user or invalidated -> done
> + * Lists are protected by the individual_lock.
> */
> + spinlock_t individual_lock;
>
> - /* BOs for user mode queues that need a validation */
> - struct list_head evicted_user;
> -
> - /* regular invalidated BOs, but not yet updated in the PT */
> - struct list_head invalidated;
> -
> - /* BOs which are invalidated, has been updated in the PTs */
> - struct list_head done;
> + /* Userspace BOs with individual resv object */
> + struct amdgpu_vm_bo_status individual;
>
> /*
> * This list contains amdgpu_bo_va_mapping objects which have been freed
> @@ -507,8 +506,8 @@ int amdgpu_vm_make_compute(struct amdgpu_device
> *adev, struct amdgpu_vm *vm); void amdgpu_vm_fini(struct amdgpu_device *adev,
> struct amdgpu_vm *vm); int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct
> drm_exec *exec,
> unsigned int num_fences);
> -int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
> - unsigned int num_fences);
> +int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
> + unsigned int num_fences);
> bool amdgpu_vm_ready(struct amdgpu_vm *vm); uint64_t
> amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm); int
> amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm, diff --git
> a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> index a930f1522f96..ff334f4dcc90 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> @@ -544,9 +544,7 @@ static void amdgpu_vm_pt_free(struct
> amdgpu_vm_bo_base *entry)
> entry->bo->vm_bo = NULL;
> ttm_bo_set_bulk_move(&entry->bo->tbo, NULL);
>
> - spin_lock(&entry->vm->status_lock);
> list_del(&entry->vm_status);
> - spin_unlock(&entry->vm->status_lock);
> amdgpu_bo_unref(&entry->bo);
> }
>
> @@ -590,7 +588,6 @@ static void amdgpu_vm_pt_add_list(struct
> amdgpu_vm_update_params *params,
> struct amdgpu_vm_pt_cursor seek;
> struct amdgpu_vm_bo_base *entry;
>
> - spin_lock(¶ms->vm->status_lock);
> for_each_amdgpu_vm_pt_dfs_safe(params->adev, params->vm, cursor,
> seek, entry) {
> if (entry && entry->bo)
> list_move(&entry->vm_status, ¶ms->tlb_flush_waitlist);
> @@ -598,7 +595,6 @@ static void amdgpu_vm_pt_add_list(struct
> amdgpu_vm_update_params *params,
>
> /* enter start node now */
> list_move(&cursor->entry->vm_status, ¶ms->tlb_flush_waitlist);
> - spin_unlock(¶ms->vm->status_lock);
> }
>
> /**
> --
> 2.43.0
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH 8/8] drm/amdgpu: restructure VM state machine v2
2026-05-08 2:24 ` Liang, Prike
@ 2026-05-08 8:21 ` Christian König
0 siblings, 0 replies; 30+ messages in thread
From: Christian König @ 2026-05-08 8:21 UTC (permalink / raw)
To: Liang, Prike, Deucher, Alexander, Khatri, Sunil,
Zhang, Jesse(Jie)
Cc: amd-gfx@lists.freedesktop.org
On 5/8/26 04:24, Liang, Prike wrote:
>> amdgpu_debugfs_userq_init(filp, queue, qid);
>> + amdgpu_userq_put(queue);
>> args->out.queue_id = qid;
>> return 0;
>
> Should this userq creation get/put and error handler changes be splashed into the patch#7?
Oh, good point! That looks like my branch somehow got messed up during rebase. This is reverting some of the changes from patch #7.
Going to fix that, thanks.
>> @@ -1091,7 +1100,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr
>> *uq_mgr)
>> if (unlikely(ret))
>> goto unlock_all;
>>
>> - ret = amdgpu_vm_lock_done_list(vm, &exec, 1);
>> + ret = amdgpu_vm_lock_individual(vm, &exec, 1);
>
>
> In the following ttm_bo_validate() may consumer all the reservation fence slots by the reservation internal pipeline eviction fences or move fences, then this will
> result in the userq eviction fences attached failed due to the out of reservation fence slots. How about set the amdgpu_vm_lock_individual() reservation fence
> number as TTM_NUM_MOVE_FENCES + 1 (validate() move path fence slots + userq eviction fence)?
Good point as well.
This code was written before Pierre-Erics patch to introduce TTM_NUM_MOVE_FENCES was merged and I've forgotten to adjust it.
Thanks for the comments,
Christian.
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH 8/8] drm/amdgpu: restructure VM state machine v2
2026-04-27 18:27 ` [PATCH 8/8] drm/amdgpu: restructure VM state machine v2 Christian König
2026-04-28 12:20 ` Khatri, Sunil
2026-05-08 2:24 ` Liang, Prike
@ 2026-05-25 23:29 ` Timur Kristóf
2026-05-26 8:51 ` Christian König
2 siblings, 1 reply; 30+ messages in thread
From: Timur Kristóf @ 2026-05-25 23:29 UTC (permalink / raw)
To: alexander.deucher, Prike.Liang, sukhatri, jesse.zhang, amd-gfx
Cc: amd-gfx, christian.koenig, Vock Natalie
On 2026. április 27., hétfő 20:27:23 közép-európai nyári idő Christian König
wrote:
> Instead of coming up with more sophisticated names for states a VM BO
> can be in, group them by the type of BO first and then by the state.
>
> So we end with BO type kernel, always_valid and individual and then states
> evicted, moved and idle.
>
> Not much functional change, except that evicted_user is moved back
> together with the other BOs again which makes the handling in
> amdgpu_vm_validate() a bit more complex.
>
> Also fixes a problem with user queues and amdgpu_vm_ready(). We didn't
> considered the VM ready when user BOs were not ideally placed, harmless
> performance impact for kernel queues but a complete show stopper for
> userqueues.
>
> v2: fix a few typos in comments, rename the BO types to make them more
> descriptive, fix a couple of bugs found during testing
> v3: squashed together with revert to old status lock handling, looks
> like the first patch still had some bug which this one here should fix.
> Fix a missing lock around debugfs printing.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
Hi Christian,
This patch regresses GPU recovery.
(Tested on SI, haven't had time to test other GPUs yet.)
It hits the following warning:
WARNING: drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c:635 at amdgpu_vm_pde_update
And then this error:
[drm:amdgpu_gem_va_update_vm [amdgpu]] *ERROR* Couldn't update BO_VA (-22)
Please fix.
Thanks & best regards,
Timur
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 65 +--
> drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 534 +++++++++-------------
> drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 69 ++-
> drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 4 -
> 4 files changed, 298 insertions(+), 374 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c index
> 28e0695b53e7..07e30acbdc3a 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -805,41 +805,50 @@ amdgpu_userq_create(struct drm_file *filp, union
> drm_amdgpu_userq *args) if (r)
> goto clean_mqd;
>
> - amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
> + /* Make sure a concurrent close operation doesn't kill our userq */
> + kref_get(&queue->refcount);
> + r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
> + XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
> + GFP_KERNEL);
> + if (r) {
> + /*
> + * This drops the extra and last reference which should
take
> + * care of all cleanup.
> + */
> + amdgpu_userq_put(queue);
> + amdgpu_userq_put(queue);
> + return r;
> + }
>
> /* don't map the queue if scheduling is halted */
> + /* TODO: That is racy! */
> if (!adev->userq_halt_for_enforce_isolation ||
> ((queue->queue_type != AMDGPU_HW_IP_GFX) &&
> (queue->queue_type != AMDGPU_HW_IP_COMPUTE))) {
> +
> + /* TODO: We could skip mapping if ev fence is signaled
*/
> + amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr,
> + &fpriv->evf_mgr);
> r = amdgpu_userq_map_helper(queue);
> if (r) {
> drm_file_err(uq_mgr->file, "Failed to map
Queue\n");
> mutex_unlock(&uq_mgr->userq_mutex);
> - goto clean_doorbell;
> + /* Prevent racing with close */
> + if (xa_erase(&uq_mgr->userq_xa, qid) ==
queue)
> + amdgpu_userq_put(queue);
> + amdgpu_userq_put(queue);
> + return r;
> }
> + mutex_unlock(&uq_mgr->userq_mutex);
> }
>
> atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
> - mutex_unlock(&uq_mgr->userq_mutex);
> -
> - r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
> - XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
> - GFP_KERNEL);
> - if (r) {
> - /*
> - * This drops the last reference which should take care
of
> - * all cleanup.
> - */
> - amdgpu_userq_put(queue);
> - return r;
> - }
>
> amdgpu_debugfs_userq_init(filp, queue, qid);
> + amdgpu_userq_put(queue);
> args->out.queue_id = qid;
> return 0;
>
> -clean_doorbell:
> - xa_erase_irq(&adev->userq_doorbell_xa, index);
> clean_mqd:
> uq_funcs->mqd_destroy(queue);
> clean_mapping:
> @@ -1035,12 +1044,12 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev,
> struct drm_exec *exec, struct amdgpu_bo *bo;
> int ret;
>
> - spin_lock(&vm->status_lock);
> - while (!list_empty(&vm->invalidated)) {
> - bo_va = list_first_entry(&vm->invalidated,
> + spin_lock(&vm->individual_lock);
> + while (!list_empty(&vm->always_valid.evicted)) {
> + bo_va = list_first_entry(&vm->always_valid.evicted,
> struct amdgpu_bo_va,
> base.vm_status);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> bo = bo_va->base.bo;
> ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2);
> @@ -1052,14 +1061,14 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev,
> struct drm_exec *exec, if (ret)
> return ret;
>
> - /* This moves the bo_va to the done list */
> + /* This moves the bo_va to the idle list */
> ret = amdgpu_vm_bo_update(adev, bo_va, false);
> if (ret)
> return ret;
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->individual_lock);
> }
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> return 0;
> }
> @@ -1091,7 +1100,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr
> *uq_mgr) if (unlikely(ret))
> goto unlock_all;
>
> - ret = amdgpu_vm_lock_done_list(vm, &exec, 1);
> + ret = amdgpu_vm_lock_individual(vm, &exec, 1);
> drm_exec_retry_on_contention(&exec);
> if (unlikely(ret))
> goto unlock_all;
> @@ -1134,7 +1143,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr
> *uq_mgr)
>
> key = 0;
> /* Validate User Ptr BOs */
> - list_for_each_entry(bo_va, &vm->done, base.vm_status) {
> + list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status)
{
> bo = bo_va->base.bo;
> if (!bo)
> continue;
> @@ -1184,10 +1193,10 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr
> *uq_mgr)
>
> /*
> * We need to wait for all VM updates to finish before restarting
the
> - * queues. Using the done list like that is now ok since
everything is
> + * queues. Using the idle list like that is now ok since everything
is
> * locked in place.
> */
> - list_for_each_entry(bo_va, &vm->done, base.vm_status)
> + list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status)
> dma_fence_wait(bo_va->last_pt_update, false);
> dma_fence_wait(vm->last_update, false);
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 6b58b0ecfa06..9f71a146f1f6
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -138,6 +138,47 @@ static void amdgpu_vm_assert_locked(struct amdgpu_vm
> *vm) dma_resv_assert_held(vm->root.bo->tbo.base.resv);
> }
>
> +/* Initialize the amdgpu_vm_bo_status object */
> +static void amdgpu_vm_bo_status_init(struct amdgpu_vm_bo_status *lists)
> +{
> + INIT_LIST_HEAD(&lists->evicted);
> + INIT_LIST_HEAD(&lists->moved);
> + INIT_LIST_HEAD(&lists->idle);
> +}
> +
> +/*
> + * Make sure we have the lock to modify the vm_bo status and return the
> object + * with the status lists.
> + */
> +static struct amdgpu_vm_bo_status *
> +amdgpu_vm_bo_lock_lists(struct amdgpu_vm_bo_base *vm_bo)
> +{
> + struct amdgpu_vm *vm = vm_bo->vm;
> + struct amdgpu_bo *bo = vm_bo->bo;
> +
> + if (amdgpu_vm_is_bo_always_valid(vm, bo)) {
> + /* No extra locking needed, protected by the root PD
resv lock */
> + amdgpu_vm_assert_locked(vm);
> +
> + if (bo->tbo.type == ttm_bo_type_kernel)
> + return &vm->kernel;
> +
> + return &vm->always_valid;
> + }
> +
> + spin_lock(&vm_bo->vm->individual_lock);
> + return &vm->individual;
> +}
> +
> +/* Eventually unlock the status list lock again */
> +static void amdgpu_vm_bo_unlock_lists(struct amdgpu_vm_bo_base *vm_bo)
> +{
> + if (amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
> + amdgpu_vm_assert_locked(vm_bo->vm);
> + else
> + spin_unlock(&vm_bo->vm->individual_lock);
> +}
> +
> /**
> * amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid
> *
> @@ -157,37 +198,44 @@ bool amdgpu_vm_is_bo_always_valid(struct amdgpu_vm
> *vm, struct amdgpu_bo *bo) *
> * @vm_bo: vm_bo which is evicted
> *
> - * State for PDs/PTs and per VM BOs which are not at the location they
> should - * be.
> + * State for vm_bo objects meaning the underlying BO was evicted and need
> to + * move in place again.
> */
> static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
> {
> - struct amdgpu_vm *vm = vm_bo->vm;
> - struct amdgpu_bo *bo = vm_bo->bo;
> + struct amdgpu_vm_bo_status *lists;
>
> + lists = amdgpu_vm_bo_lock_lists(vm_bo);
> vm_bo->moved = true;
> - amdgpu_vm_assert_locked(vm);
> - spin_lock(&vm_bo->vm->status_lock);
> - if (bo->tbo.type == ttm_bo_type_kernel)
> - list_move(&vm_bo->vm_status, &vm->evicted);
> - else
> - list_move_tail(&vm_bo->vm_status, &vm->evicted);
> - spin_unlock(&vm_bo->vm->status_lock);
> + list_move(&vm_bo->vm_status, &lists->evicted);
> + amdgpu_vm_bo_unlock_lists(vm_bo);
> }
> /**
> * amdgpu_vm_bo_moved - vm_bo is moved
> *
> * @vm_bo: vm_bo which is moved
> *
> - * State for per VM BOs which are moved, but that change is not yet
> reflected - * in the page tables.
> + * State for vm_bo objects meaning the underlying BO was moved but the new
> + * location not yet reflected in the page tables.
> */
> static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
> {
> - amdgpu_vm_assert_locked(vm_bo->vm);
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
> - spin_unlock(&vm_bo->vm->status_lock);
> + struct amdgpu_vm_bo_status *lists;
> + struct amdgpu_bo *bo = vm_bo->bo;
> +
> + /*
> + * The root PD doesn't have a parent PDE and goes directly into the
> + * idle state.
> + */
> + lists = amdgpu_vm_bo_lock_lists(vm_bo);
> + if (bo && bo->tbo.type == ttm_bo_type_kernel && !bo->parent) {
> + vm_bo->moved = false;
> + list_move(&vm_bo->vm_status, &lists->idle);
> + } else {
> + vm_bo->moved = true;
> + list_move(&vm_bo->vm_status, &lists->moved);
> + }
> + amdgpu_vm_bo_unlock_lists(vm_bo);
> }
>
> /**
> @@ -195,120 +243,43 @@ static void amdgpu_vm_bo_moved(struct
> amdgpu_vm_bo_base *vm_bo) *
> * @vm_bo: vm_bo which is now idle
> *
> - * State for PDs/PTs and per VM BOs which have gone through the state
> machine - * and are now idle.
> + * State for vm_bo objects meaning we are done with the state machine and
> no + * further action is necessary.
> */
> static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
> {
> - amdgpu_vm_assert_locked(vm_bo->vm);
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
> - spin_unlock(&vm_bo->vm->status_lock);
> - vm_bo->moved = false;
> -}
> + struct amdgpu_vm_bo_status *lists;
>
> -/**
> - * amdgpu_vm_bo_invalidated - vm_bo is invalidated
> - *
> - * @vm_bo: vm_bo which is now invalidated
> - *
> - * State for normal BOs which are invalidated and that change not yet
> reflected - * in the PTs.
> - */
> -static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
> -{
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
> - spin_unlock(&vm_bo->vm->status_lock);
> -}
> -
> -/**
> - * amdgpu_vm_bo_evicted_user - vm_bo is evicted
> - *
> - * @vm_bo: vm_bo which is evicted
> - *
> - * State for BOs used by user mode queues which are not at the location
> they - * should be.
> - */
> -static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)
> -{
> - vm_bo->moved = true;
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user);
> - spin_unlock(&vm_bo->vm->status_lock);
> -}
> -
> -/**
> - * amdgpu_vm_bo_relocated - vm_bo is reloacted
> - *
> - * @vm_bo: vm_bo which is relocated
> - *
> - * State for PDs/PTs which needs to update their parent PD.
> - * For the root PD, just move to idle state.
> - */
> -static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
> -{
> - amdgpu_vm_assert_locked(vm_bo->vm);
> - if (vm_bo->bo->parent) {
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
> - spin_unlock(&vm_bo->vm->status_lock);
> - } else {
> - amdgpu_vm_bo_idle(vm_bo);
> - }
> -}
> -
> -/**
> - * amdgpu_vm_bo_done - vm_bo is done
> - *
> - * @vm_bo: vm_bo which is now done
> - *
> - * State for normal BOs which are invalidated and that change has been
> updated - * in the PTs.
> - */
> -static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
> -{
> - amdgpu_vm_assert_locked(vm_bo->vm);
> - spin_lock(&vm_bo->vm->status_lock);
> - list_move(&vm_bo->vm_status, &vm_bo->vm->done);
> - spin_unlock(&vm_bo->vm->status_lock);
> + lists = amdgpu_vm_bo_lock_lists(vm_bo);
> + if (!amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
> + vm_bo->moved = false;
> + list_move(&vm_bo->vm_status, &lists->idle);
> + amdgpu_vm_bo_unlock_lists(vm_bo);
> }
>
> /**
> * amdgpu_vm_bo_reset_state_machine - reset the vm_bo state machine
> * @vm: the VM which state machine to reset
> *
> - * Move all vm_bo object in the VM into a state where they will be updated
> - * again during validation.
> + * Move all vm_bo object in the VM into a state where their location will
> be + * updated in the page tables again.
> */
> static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
> {
> - struct amdgpu_vm_bo_base *vm_bo, *tmp;
> -
> amdgpu_vm_assert_locked(vm);
> + list_splice_init(&vm->kernel.idle, &vm->kernel.moved);
> + list_splice_init(&vm->always_valid.idle, &vm->always_valid.moved);
>
> - spin_lock(&vm->status_lock);
> - list_splice_init(&vm->done, &vm->invalidated);
> - list_for_each_entry(vm_bo, &vm->invalidated, vm_status)
> - vm_bo->moved = true;
> -
> - list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) {
> - struct amdgpu_bo *bo = vm_bo->bo;
> -
> - vm_bo->moved = true;
> - if (!bo || bo->tbo.type != ttm_bo_type_kernel)
> - list_move(&vm_bo->vm_status, &vm_bo->vm-
>moved);
> - else if (bo->parent)
> - list_move(&vm_bo->vm_status, &vm_bo->vm-
>relocated);
> - }
> - spin_unlock(&vm->status_lock);
> + spin_lock(&vm->individual_lock);
> + list_splice_init(&vm->individual.idle, &vm->individual.moved);
> + spin_unlock(&vm->individual_lock);
> }
>
> /**
> * amdgpu_vm_update_shared - helper to update shared memory stat
> * @base: base structure for tracking BO usage in a VM
> *
> - * Takes the vm status_lock and updates the shared memory stat. If the
> basic + * Takes the vm stats_lock and updates the shared memory stat. If
> the basic * stat changed (e.g. buffer was moved) amdgpu_vm_update_stats
> need to be called * as well.
> */
> @@ -321,7 +292,7 @@ static void amdgpu_vm_update_shared(struct
> amdgpu_vm_bo_base *base) bool shared;
>
> dma_resv_assert_held(bo->tbo.base.resv);
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->stats_lock);
> shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
> if (base->shared != shared) {
> base->shared = shared;
> @@ -333,7 +304,7 @@ static void amdgpu_vm_update_shared(struct
> amdgpu_vm_bo_base *base) vm->stats[bo_memtype].drm.private += size;
> }
> }
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->stats_lock);
> }
>
> /**
> @@ -358,11 +329,11 @@ void amdgpu_vm_bo_update_shared(struct amdgpu_bo *bo)
> * be bo->tbo.resource
> * @sign: if we should add (+1) or subtract (-1) from the stat
> *
> - * Caller need to have the vm status_lock held. Useful for when multiple
> update + * Caller need to have the vm stats_lock held. Useful for when
> multiple update * need to happen at the same time.
> */
> static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
> - struct ttm_resource *res, int sign)
> + struct ttm_resource
*res, int sign)
> {
> struct amdgpu_vm *vm = base->vm;
> struct amdgpu_bo *bo = base->bo;
> @@ -386,7 +357,8 @@ static void amdgpu_vm_update_stats_locked(struct
> amdgpu_vm_bo_base *base, */
> if (bo->flags & AMDGPU_GEM_CREATE_DISCARDABLE)
> vm->stats[res_memtype].drm.purgeable +=
size;
> - if (!(bo->preferred_domains &
amdgpu_mem_type_to_domain(res_memtype)))
> + if (!(bo->preferred_domains &
> + amdgpu_mem_type_to_domain(res_memtype)))
> vm->stats[bo_memtype].evicted += size;
> }
> }
> @@ -405,9 +377,9 @@ void amdgpu_vm_update_stats(struct amdgpu_vm_bo_base
> *base, {
> struct amdgpu_vm *vm = base->vm;
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->stats_lock);
> amdgpu_vm_update_stats_locked(base, res, sign);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->stats_lock);
> }
>
> /**
> @@ -428,37 +400,34 @@ void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base
> *base, base->next = NULL;
> INIT_LIST_HEAD(&base->vm_status);
>
> + dma_resv_assert_held(vm->root.bo->tbo.base.resv);
> if (!bo)
> return;
> +
> base->next = bo->vm_bo;
> bo->vm_bo = base;
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->stats_lock);
> base->shared = drm_gem_object_is_shared_for_memory_stats(&bo-
>tbo.base);
> amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->stats_lock);
>
> - if (!amdgpu_vm_is_bo_always_valid(vm, bo))
> + if (!amdgpu_vm_is_bo_always_valid(vm, bo)) {
> + amdgpu_vm_bo_idle(base);
> return;
> -
> - dma_resv_assert_held(vm->root.bo->tbo.base.resv);
> + }
>
> ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move);
> - if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
> - amdgpu_vm_bo_relocated(base);
> - else
> - amdgpu_vm_bo_idle(base);
>
> + /*
> + * When a per VM isn't in the desired domain put it into the
evicted
> + * state to make sure that it gets validated on the next best
occasion.
> + */
> if (bo->preferred_domains &
> amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type))
> - return;
> -
> - /*
> - * we checked all the prerequisites, but it looks like this per vm
bo
> - * is currently evicted. add the bo to the evicted list to make
sure it
> - * is validated on next vm use to avoid fault.
> - * */
> - amdgpu_vm_bo_evicted(base);
> + amdgpu_vm_bo_moved(base);
> + else
> + amdgpu_vm_bo_evicted(base);
> }
>
> /**
> @@ -479,41 +448,41 @@ int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct
> drm_exec *exec, }
>
> /**
> - * amdgpu_vm_lock_done_list - lock all BOs on the done list
> + * amdgpu_vm_lock_individual - lock all BOs on the individual idle list
> * @vm: vm providing the BOs
> * @exec: drm execution context
> * @num_fences: number of extra fences to reserve
> *
> - * Lock the BOs on the done list in the DRM execution context.
> + * Lock the BOs on the individual idle list in the DRM execution context.
> */
> -int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
> - unsigned int num_fences)
> +int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
> + unsigned int num_fences)
> {
> - struct list_head *prev = &vm->done;
> + struct list_head *prev = &vm->individual.idle;
> struct amdgpu_bo_va *bo_va;
> struct amdgpu_bo *bo;
> int ret;
>
> /* We can only trust prev->next while holding the lock */
> - spin_lock(&vm->status_lock);
> - while (!list_is_head(prev->next, &vm->done)) {
> + spin_lock(&vm->individual_lock);
> + while (!list_is_head(prev->next, &vm->individual.idle)) {
> bo_va = list_entry(prev->next, typeof(*bo_va),
base.vm_status);
>
> bo = bo_va->base.bo;
> if (bo) {
> amdgpu_bo_ref(bo);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> ret = drm_exec_prepare_obj(exec, &bo-
>tbo.base, 1);
> amdgpu_bo_unref(&bo);
> if (unlikely(ret))
> return ret;
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->individual_lock);
> }
> prev = prev->next;
> }
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> return 0;
> }
> @@ -609,10 +578,10 @@ int amdgpu_vm_validate(struct amdgpu_device *adev,
> struct amdgpu_vm *vm, void *param)
> {
> uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm);
> - struct amdgpu_vm_bo_base *bo_base;
> - struct amdgpu_bo *bo;
> + struct amdgpu_vm_bo_base *bo_base, *tmp;
> int r;
>
> + dma_resv_assert_held(vm->root.bo->tbo.base.resv);
> if (vm->generation != new_vm_generation) {
> vm->generation = new_vm_generation;
> amdgpu_vm_bo_reset_state_machine(vm);
> @@ -622,49 +591,59 @@ int amdgpu_vm_validate(struct amdgpu_device *adev,
> struct amdgpu_vm *vm, return r;
> }
>
> - spin_lock(&vm->status_lock);
> - while (!list_empty(&vm->evicted)) {
> - bo_base = list_first_entry(&vm->evicted,
> - struct
amdgpu_vm_bo_base,
> - vm_status);
> - spin_unlock(&vm->status_lock);
> + list_for_each_entry_safe(bo_base, tmp, &vm->kernel.evicted,
vm_status) {
> + r = validate(param, bo_base->bo);
> + if (r)
> + return r;
>
> - bo = bo_base->bo;
> + vm->update_funcs->map_table(to_amdgpu_bo_vm(bo_base-
>bo));
> + amdgpu_vm_bo_moved(bo_base);
> + }
>
> - r = validate(param, bo);
> + /*
> + * As soon as all page tables are in place we can start updating
them
> + * again.
> + */
> + amdgpu_vm_eviction_lock(vm);
> + vm->evicting = false;
> + amdgpu_vm_eviction_unlock(vm);
> +
> + list_for_each_entry_safe(bo_base, tmp, &vm->always_valid.evicted,
> + vm_status) {
> + r = validate(param, bo_base->bo);
> if (r)
> return r;
>
> - if (bo->tbo.type != ttm_bo_type_kernel) {
> - amdgpu_vm_bo_moved(bo_base);
> - } else {
> - vm->update_funcs-
>map_table(to_amdgpu_bo_vm(bo));
> - amdgpu_vm_bo_relocated(bo_base);
> - }
> - spin_lock(&vm->status_lock);
> + amdgpu_vm_bo_moved(bo_base);
> }
> - while (ticket && !list_empty(&vm->evicted_user)) {
> - bo_base = list_first_entry(&vm->evicted_user,
> - struct
amdgpu_vm_bo_base,
> - vm_status);
> - spin_unlock(&vm->status_lock);
>
> - bo = bo_base->bo;
> - dma_resv_assert_held(bo->tbo.base.resv);
> + if (!ticket)
> + return 0;
> +
> + spin_lock(&vm->individual_lock);
> +restart:
> + list_for_each_entry(bo_base, &vm->individual.evicted, vm_status) {
> + struct amdgpu_bo *bo = bo_base->bo;
> +
> + if (dma_resv_locking_ctx(bo->tbo.base.resv) != ticket)
> + continue;
> +
> + spin_unlock(&vm->individual_lock);
>
> r = validate(param, bo);
> if (r)
> return r;
>
> - amdgpu_vm_bo_invalidated(bo_base);
> + amdgpu_vm_bo_moved(bo_base);
>
> - spin_lock(&vm->status_lock);
> + /* It's a bit inefficient to always jump back to the
start, but
> + * we would need to re-structure the KFD for properly
fixing
> + * that.
> + */
> + spin_lock(&vm->individual_lock);
> + goto restart;
> }
> - spin_unlock(&vm->status_lock);
> -
> - amdgpu_vm_eviction_lock(vm);
> - vm->evicting = false;
> - amdgpu_vm_eviction_unlock(vm);
> + spin_unlock(&vm->individual_lock);
>
> return 0;
> }
> @@ -689,9 +668,7 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
> ret = !vm->evicting;
> amdgpu_vm_eviction_unlock(vm);
>
> - spin_lock(&vm->status_lock);
> - ret &= list_empty(&vm->evicted);
> - spin_unlock(&vm->status_lock);
> + ret &= list_empty(&vm->kernel.evicted);
>
> spin_lock(&vm->immediate.lock);
> ret &= !vm->immediate.stopped;
> @@ -985,18 +962,13 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
> struct amdgpu_vm *vm, bool immediate)
> {
> struct amdgpu_vm_update_params params;
> - struct amdgpu_vm_bo_base *entry;
> + struct amdgpu_vm_bo_base *entry, *tmp;
> bool flush_tlb_needed = false;
> - LIST_HEAD(relocated);
> int r, idx;
>
> amdgpu_vm_assert_locked(vm);
>
> - spin_lock(&vm->status_lock);
> - list_splice_init(&vm->relocated, &relocated);
> - spin_unlock(&vm->status_lock);
> -
> - if (list_empty(&relocated))
> + if (list_empty(&vm->kernel.moved))
> return 0;
>
> if (!drm_dev_enter(adev_to_drm(adev), &idx))
> @@ -1012,7 +984,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
> if (r)
> goto error;
>
> - list_for_each_entry(entry, &relocated, vm_status) {
> + list_for_each_entry(entry, &vm->kernel.moved, vm_status) {
> /* vm_flush_needed after updating moved PDEs */
> flush_tlb_needed |= entry->moved;
>
> @@ -1028,11 +1000,8 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
> if (flush_tlb_needed)
> atomic64_inc(&vm->tlb_seq);
>
> - while (!list_empty(&relocated)) {
> - entry = list_first_entry(&relocated, struct
amdgpu_vm_bo_base,
> - vm_status);
> + list_for_each_entry_safe(entry, tmp, &vm->kernel.moved, vm_status)
> amdgpu_vm_bo_idle(entry);
> - }
>
> error:
> drm_dev_exit(idx);
> @@ -1260,9 +1229,9 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev,
> struct amdgpu_vm *vm, void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
> struct amdgpu_mem_stats
stats[__AMDGPU_PL_NUM])
> {
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->stats_lock);
> memcpy(stats, vm->stats, sizeof(*stats) * __AMDGPU_PL_NUM);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->stats_lock);
> }
>
> /**
> @@ -1406,7 +1375,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
> struct amdgpu_bo_va *bo_va, else
> amdgpu_vm_bo_idle(&bo_va->base);
> } else {
> - amdgpu_vm_bo_done(&bo_va->base);
> + amdgpu_vm_bo_idle(&bo_va->base);
> }
>
> list_splice_init(&bo_va->invalids, &bo_va->valids);
> @@ -1629,29 +1598,25 @@ int amdgpu_vm_handle_moved(struct amdgpu_device
> *adev, struct amdgpu_vm *vm,
> struct ww_acquire_ctx *ticket)
> {
> - struct amdgpu_bo_va *bo_va;
> + struct amdgpu_bo_va *bo_va, *tmp;
> struct dma_resv *resv;
> bool clear, unlock;
> int r;
>
> - spin_lock(&vm->status_lock);
> - while (!list_empty(&vm->moved)) {
> - bo_va = list_first_entry(&vm->moved, struct
amdgpu_bo_va,
> - base.vm_status);
> - spin_unlock(&vm->status_lock);
> -
> + list_for_each_entry_safe(bo_va, tmp, &vm->always_valid.moved,
> + base.vm_status) {
> /* Per VM BOs never need to bo cleared in the page
tables */
> r = amdgpu_vm_bo_update(adev, bo_va, false);
> if (r)
> return r;
> - spin_lock(&vm->status_lock);
> }
>
> - while (!list_empty(&vm->invalidated)) {
> - bo_va = list_first_entry(&vm->invalidated, struct
amdgpu_bo_va,
> - base.vm_status);
> + spin_lock(&vm->individual_lock);
> + while (!list_empty(&vm->individual.moved)) {
> + bo_va = list_first_entry(&vm->individual.moved,
> + typeof(*bo_va),
base.vm_status);
> resv = bo_va->base.bo->tbo.base.resv;
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> /* Try to reserve the BO to avoid clearing its ptes */
> if (!adev->debug_vm && dma_resv_trylock(resv)) {
> @@ -1681,11 +1646,11 @@ int amdgpu_vm_handle_moved(struct amdgpu_device
> *adev, drm_gem_is_imported(&bo_va->base.bo->tbo.base) &&
> (!bo_va->base.bo->tbo.resource ||
> bo_va->base.bo->tbo.resource->mem_type ==
TTM_PL_SYSTEM))
> - amdgpu_vm_bo_evicted_user(&bo_va->base);
> + amdgpu_vm_bo_evicted(&bo_va->base);
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->individual_lock);
> }
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> return 0;
> }
> @@ -2223,9 +2188,9 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev,
> }
> }
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->individual_lock);
> list_del(&bo_va->base.vm_status);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->individual_lock);
>
> list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
> list_del(&mapping->list);
> @@ -2305,14 +2270,7 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_bo *bo,
> bool evicted)
>
> if (bo_base->moved)
> continue;
> - bo_base->moved = true;
> -
> - if (bo->tbo.type == ttm_bo_type_kernel)
> - amdgpu_vm_bo_relocated(bo_base);
> - else if (amdgpu_vm_is_bo_always_valid(vm, bo))
> - amdgpu_vm_bo_moved(bo_base);
> - else
> - amdgpu_vm_bo_invalidated(bo_base);
> + amdgpu_vm_bo_moved(bo_base);
> }
> }
>
> @@ -2333,10 +2291,10 @@ void amdgpu_vm_bo_move(struct amdgpu_bo *bo, struct
> ttm_resource *new_mem, for (bo_base = bo->vm_bo; bo_base; bo_base =
> bo_base->next) {
> struct amdgpu_vm *vm = bo_base->vm;
>
> - spin_lock(&vm->status_lock);
> + spin_lock(&vm->stats_lock);
> amdgpu_vm_update_stats_locked(bo_base, bo-
>tbo.resource, -1);
> amdgpu_vm_update_stats_locked(bo_base, new_mem, +1);
> - spin_unlock(&vm->status_lock);
> + spin_unlock(&vm->stats_lock);
> }
>
> amdgpu_vm_bo_invalidate(bo, evicted);
> @@ -2603,16 +2561,14 @@ int amdgpu_vm_init(struct amdgpu_device *adev,
> struct amdgpu_vm *vm, vm->va = RB_ROOT_CACHED;
> for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
> vm->reserved_vmid[i] = NULL;
> - INIT_LIST_HEAD(&vm->evicted);
> - INIT_LIST_HEAD(&vm->evicted_user);
> - INIT_LIST_HEAD(&vm->relocated);
> - INIT_LIST_HEAD(&vm->moved);
> - INIT_LIST_HEAD(&vm->idle);
> - INIT_LIST_HEAD(&vm->invalidated);
> - spin_lock_init(&vm->status_lock);
> +
> + amdgpu_vm_bo_status_init(&vm->kernel);
> + amdgpu_vm_bo_status_init(&vm->always_valid);
> + spin_lock_init(&vm->individual_lock);
> + amdgpu_vm_bo_status_init(&vm->individual);
> INIT_LIST_HEAD(&vm->freed);
> - INIT_LIST_HEAD(&vm->done);
> INIT_KFIFO(vm->faults);
> + spin_lock_init(&vm->stats_lock);
>
> r = amdgpu_vm_init_entities(adev, vm);
> if (r)
> @@ -3089,100 +3045,64 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device
> *adev, u32 pasid, }
>
> #if defined(CONFIG_DEBUG_FS)
> -/**
> - * amdgpu_debugfs_vm_bo_info - print BO info for the VM
> - *
> - * @vm: Requested VM for printing BO info
> - * @m: debugfs file
> - *
> - * Print BO information in debugfs file for the VM
> - */
> -void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
> -{
> - struct amdgpu_bo_va *bo_va, *tmp;
> - u64 total_idle = 0;
> - u64 total_evicted = 0;
> - u64 total_relocated = 0;
> - u64 total_moved = 0;
> - u64 total_invalidated = 0;
> - u64 total_done = 0;
> - unsigned int total_idle_objs = 0;
> - unsigned int total_evicted_objs = 0;
> - unsigned int total_relocated_objs = 0;
> - unsigned int total_moved_objs = 0;
> - unsigned int total_invalidated_objs = 0;
> - unsigned int total_done_objs = 0;
> - unsigned int id = 0;
>
> - amdgpu_vm_assert_locked(vm);
> +/* print the debug info for a specific set of status lists */
> +static void amdgpu_debugfs_vm_bo_status_info(struct seq_file *m,
> + struct
amdgpu_vm_bo_status *lists)
> +{
> + struct amdgpu_vm_bo_base *base;
> + unsigned int id;
>
> - spin_lock(&vm->status_lock);
> - seq_puts(m, "\tIdle BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
> - if (!bo_va->base.bo)
> - continue;
> - total_idle += amdgpu_bo_print_info(id++, bo_va-
>base.bo, m);
> - }
> - total_idle_objs = id;
> id = 0;
> -
> seq_puts(m, "\tEvicted BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status)
{
> - if (!bo_va->base.bo)
> + list_for_each_entry(base, &lists->evicted, vm_status) {
> + if (!base->bo)
> continue;
> - total_evicted += amdgpu_bo_print_info(id++, bo_va-
>base.bo, m);
> - }
> - total_evicted_objs = id;
> - id = 0;
>
> - seq_puts(m, "\tRelocated BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->relocated,
base.vm_status) {
> - if (!bo_va->base.bo)
> - continue;
> - total_relocated += amdgpu_bo_print_info(id++, bo_va-
>base.bo, m);
> + amdgpu_bo_print_info(id++, base->bo, m);
> }
> - total_relocated_objs = id;
> - id = 0;
>
> + id = 0;
> seq_puts(m, "\tMoved BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
> - if (!bo_va->base.bo)
> + list_for_each_entry(base, &lists->moved, vm_status) {
> + if (!base->bo)
> continue;
> - total_moved += amdgpu_bo_print_info(id++, bo_va-
>base.bo, m);
> +
> + amdgpu_bo_print_info(id++, base->bo, m);
> }
> - total_moved_objs = id;
> - id = 0;
>
> - seq_puts(m, "\tInvalidated BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->invalidated,
base.vm_status) {
> - if (!bo_va->base.bo)
> + id = 0;
> + seq_puts(m, "\tIdle BOs:\n");
> + list_for_each_entry(base, &lists->moved, vm_status) {
> + if (!base->bo)
> continue;
> - total_invalidated += amdgpu_bo_print_info(id++,
bo_va->base.bo, m);
> +
> + amdgpu_bo_print_info(id++, base->bo, m);
> }
> - total_invalidated_objs = id;
> - id = 0;
> +}
>
> - seq_puts(m, "\tDone BOs:\n");
> - list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
> - if (!bo_va->base.bo)
> - continue;
> - total_done += amdgpu_bo_print_info(id++, bo_va-
>base.bo, m);
> - }
> - spin_unlock(&vm->status_lock);
> - total_done_objs = id;
> -
> - seq_printf(m, "\tTotal idle size: %12lld\tobjs:\t%d\n",
total_idle,
> - total_idle_objs);
> - seq_printf(m, "\tTotal evicted size: %12lld\tobjs:\t%d\n",
> total_evicted, - total_evicted_objs);
> - seq_printf(m, "\tTotal relocated size: %12lld\tobjs:\t%d\n",
> total_relocated, - total_relocated_objs);
> - seq_printf(m, "\tTotal moved size: %12lld\tobjs:\t%d\n",
> total_moved, - total_moved_objs);
> - seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n",
> total_invalidated, - total_invalidated_objs);
> - seq_printf(m, "\tTotal done size: %12lld\tobjs:\t%d\n",
total_done,
> - total_done_objs);
> +/**
> + * amdgpu_debugfs_vm_bo_info - print BO info for the VM
> + *
> + * @vm: Requested VM for printing BO info
> + * @m: debugfs file
> + *
> + * Print BO information in debugfs file for the VM
> + */
> +void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
> +{
> + amdgpu_vm_assert_locked(vm);
> +
> + seq_puts(m, "\tKernel PT/PDs:\n");
> + amdgpu_debugfs_vm_bo_status_info(m, &vm->kernel);
> +
> + seq_puts(m, "\tPer VM BOs:\n");
> + amdgpu_debugfs_vm_bo_status_info(m, &vm->always_valid);
> +
> + seq_puts(m, "\tIndividual BOs:\n");
> + spin_lock(&vm->individual_lock);
> + amdgpu_debugfs_vm_bo_status_info(m, &vm->individual);
> + spin_unlock(&vm->individual_lock);
> }
> #endif
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index f33ea7f8509b..cc96a3e6252f
> 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> @@ -205,17 +205,34 @@ struct amdgpu_vm_bo_base {
> /* protected by bo being reserved */
> struct amdgpu_vm_bo_base *next;
>
> - /* protected by vm status_lock */
> + /* protected by vm reservation and invalidated_lock */
> struct list_head vm_status;
>
> /* if the bo is counted as shared in mem stats
> - * protected by vm status_lock */
> + * protected by vm BO being reserved */
> bool shared;
>
> /* protected by the BO being reserved */
> bool moved;
> };
>
> +/*
> + * The following status lists contain amdgpu_vm_bo_base objects for
> + * either PD/PTs, per VM BOs or BOs with individual resv object.
> + *
> + * The state transits are: evicted -> moved -> idle
> + */
> +struct amdgpu_vm_bo_status {
> + /* BOs evicted which need to move into place again */
> + struct list_head evicted;
> +
> + /* BOs which moved but new location hasn't been updated in the PDs/
PTs */
> + struct list_head moved;
> +
> + /* BOs done with the state machine and need no further action */
> + struct list_head idle;
> +};
> +
> /* provided by hw blocks that can write ptes, e.g., sdma */
> struct amdgpu_vm_pte_funcs {
> /* number of dw to reserve per operation */
> @@ -345,47 +362,29 @@ struct amdgpu_vm {
> bool evicting;
> unsigned int saved_flags;
>
> - /* Lock to protect vm_bo add/del/move on all lists of vm */
> - spinlock_t status_lock;
> -
> - /* Memory statistics for this vm, protected by status_lock */
> + /* Memory statistics for this vm, protected by stats_lock */
> + spinlock_t stats_lock;
> struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM];
>
> + /* BO's belonging to PD/PT which are internal to the kernel. */
> + struct amdgpu_vm_bo_status kernel;
> +
> /*
> - * The following lists contain amdgpu_vm_bo_base objects for
either
> - * PDs, PTs or per VM BOs. The state transits are:
> - *
> - * evicted -> relocated (PDs, PTs) or moved (per VM BOs) -> idle
> + * BOs allocated by userspace where the dma_resv is shared with the
> + * root PD
> */
> -
> - /* Per-VM and PT BOs who needs a validation */
> - struct list_head evicted;
> -
> - /* PT BOs which relocated and their parent need an update */
> - struct list_head relocated;
> -
> - /* per VM BOs moved, but not yet updated in the PT */
> - struct list_head moved;
> -
> - /* All BOs of this VM not currently in the state machine */
> - struct list_head idle;
> + struct amdgpu_vm_bo_status always_valid;
>
> /*
> * The following lists contain amdgpu_vm_bo_base objects for BOs
which
> - * have their own dma_resv object and not depend on the root PD.
Their
> - * state transits are:
> + * have their own dma_resv object and not depend on the root PD.
> *
> - * evicted_user or invalidated -> done
> + * Lists are protected by the individual_lock.
> */
> + spinlock_t individual_lock;
>
> - /* BOs for user mode queues that need a validation */
> - struct list_head evicted_user;
> -
> - /* regular invalidated BOs, but not yet updated in the PT */
> - struct list_head invalidated;
> -
> - /* BOs which are invalidated, has been updated in the PTs */
> - struct list_head done;
> + /* Userspace BOs with individual resv object */
> + struct amdgpu_vm_bo_status individual;
>
> /*
> * This list contains amdgpu_bo_va_mapping objects which have been
freed
> @@ -507,8 +506,8 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev,
> struct amdgpu_vm *vm); void amdgpu_vm_fini(struct amdgpu_device *adev,
> struct amdgpu_vm *vm); int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct
> drm_exec *exec, unsigned int num_fences);
> -int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
> - unsigned int num_fences);
> +int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
> + unsigned int num_fences);
> bool amdgpu_vm_ready(struct amdgpu_vm *vm);
> uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm
> *vm); int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm
> *vm, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c index
> a930f1522f96..ff334f4dcc90 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> @@ -544,9 +544,7 @@ static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base
> *entry) entry->bo->vm_bo = NULL;
> ttm_bo_set_bulk_move(&entry->bo->tbo, NULL);
>
> - spin_lock(&entry->vm->status_lock);
> list_del(&entry->vm_status);
> - spin_unlock(&entry->vm->status_lock);
> amdgpu_bo_unref(&entry->bo);
> }
>
> @@ -590,7 +588,6 @@ static void amdgpu_vm_pt_add_list(struct
> amdgpu_vm_update_params *params, struct amdgpu_vm_pt_cursor seek;
> struct amdgpu_vm_bo_base *entry;
>
> - spin_lock(¶ms->vm->status_lock);
> for_each_amdgpu_vm_pt_dfs_safe(params->adev, params->vm, cursor,
seek,
> entry) { if (entry && entry->bo)
> list_move(&entry->vm_status, ¶ms-
>tlb_flush_waitlist);
> @@ -598,7 +595,6 @@ static void amdgpu_vm_pt_add_list(struct
> amdgpu_vm_update_params *params,
>
> /* enter start node now */
> list_move(&cursor->entry->vm_status, ¶ms->tlb_flush_waitlist);
> - spin_unlock(¶ms->vm->status_lock);
> }
>
> /**
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH 8/8] drm/amdgpu: restructure VM state machine v2
2026-05-25 23:29 ` Timur Kristóf
@ 2026-05-26 8:51 ` Christian König
2026-05-26 10:11 ` Timur Kristóf
0 siblings, 1 reply; 30+ messages in thread
From: Christian König @ 2026-05-26 8:51 UTC (permalink / raw)
To: Timur Kristóf, alexander.deucher, Prike.Liang, sukhatri,
jesse.zhang, amd-gfx
Cc: Vock Natalie
On 5/26/26 01:29, Timur Kristóf wrote:
> On 2026. április 27., hétfő 20:27:23 közép-európai nyári idő Christian König
> wrote:
>> Instead of coming up with more sophisticated names for states a VM BO
>> can be in, group them by the type of BO first and then by the state.
>>
>> So we end with BO type kernel, always_valid and individual and then states
>> evicted, moved and idle.
>>
>> Not much functional change, except that evicted_user is moved back
>> together with the other BOs again which makes the handling in
>> amdgpu_vm_validate() a bit more complex.
>>
>> Also fixes a problem with user queues and amdgpu_vm_ready(). We didn't
>> considered the VM ready when user BOs were not ideally placed, harmless
>> performance impact for kernel queues but a complete show stopper for
>> userqueues.
>>
>> v2: fix a few typos in comments, rename the BO types to make them more
>> descriptive, fix a couple of bugs found during testing
>> v3: squashed together with revert to old status lock handling, looks
>> like the first patch still had some bug which this one here should fix.
>> Fix a missing lock around debugfs printing.
>>
>> Signed-off-by: Christian König <christian.koenig@amd.com>
>
> Hi Christian,
>
> This patch regresses GPU recovery.
> (Tested on SI, haven't had time to test other GPUs yet.)
>
> It hits the following warning:
> WARNING: drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c:635 at amdgpu_vm_pde_update
> And then this error:
> [drm:amdgpu_gem_va_update_vm [amdgpu]] *ERROR* Couldn't update BO_VA (-22)
>
> Please fix.
Yeah that's a known issue. Fix is already pushed to amd-staging-drm-next.
Thanks,
Christian.
>
> Thanks & best regards,
> Timur
>
>> ---
>> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 65 +--
>> drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 534 +++++++++-------------
>> drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 69 ++-
>> drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 4 -
>> 4 files changed, 298 insertions(+), 374 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c index
>> 28e0695b53e7..07e30acbdc3a 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
>> @@ -805,41 +805,50 @@ amdgpu_userq_create(struct drm_file *filp, union
>> drm_amdgpu_userq *args) if (r)
>> goto clean_mqd;
>>
>> - amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
>> + /* Make sure a concurrent close operation doesn't kill our userq */
>> + kref_get(&queue->refcount);
>> + r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
>> + XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
>> + GFP_KERNEL);
>> + if (r) {
>> + /*
>> + * This drops the extra and last reference which should
> take
>> + * care of all cleanup.
>> + */
>> + amdgpu_userq_put(queue);
>> + amdgpu_userq_put(queue);
>> + return r;
>> + }
>>
>> /* don't map the queue if scheduling is halted */
>> + /* TODO: That is racy! */
>> if (!adev->userq_halt_for_enforce_isolation ||
>> ((queue->queue_type != AMDGPU_HW_IP_GFX) &&
>> (queue->queue_type != AMDGPU_HW_IP_COMPUTE))) {
>> +
>> + /* TODO: We could skip mapping if ev fence is signaled
> */
>> + amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr,
>> + &fpriv->evf_mgr);
>> r = amdgpu_userq_map_helper(queue);
>> if (r) {
>> drm_file_err(uq_mgr->file, "Failed to map
> Queue\n");
>> mutex_unlock(&uq_mgr->userq_mutex);
>> - goto clean_doorbell;
>> + /* Prevent racing with close */
>> + if (xa_erase(&uq_mgr->userq_xa, qid) ==
> queue)
>> + amdgpu_userq_put(queue);
>> + amdgpu_userq_put(queue);
>> + return r;
>> }
>> + mutex_unlock(&uq_mgr->userq_mutex);
>> }
>>
>> atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
>> - mutex_unlock(&uq_mgr->userq_mutex);
>> -
>> - r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
>> - XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
>> - GFP_KERNEL);
>> - if (r) {
>> - /*
>> - * This drops the last reference which should take care
> of
>> - * all cleanup.
>> - */
>> - amdgpu_userq_put(queue);
>> - return r;
>> - }
>>
>> amdgpu_debugfs_userq_init(filp, queue, qid);
>> + amdgpu_userq_put(queue);
>> args->out.queue_id = qid;
>> return 0;
>>
>> -clean_doorbell:
>> - xa_erase_irq(&adev->userq_doorbell_xa, index);
>> clean_mqd:
>> uq_funcs->mqd_destroy(queue);
>> clean_mapping:
>> @@ -1035,12 +1044,12 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev,
>> struct drm_exec *exec, struct amdgpu_bo *bo;
>> int ret;
>>
>> - spin_lock(&vm->status_lock);
>> - while (!list_empty(&vm->invalidated)) {
>> - bo_va = list_first_entry(&vm->invalidated,
>> + spin_lock(&vm->individual_lock);
>> + while (!list_empty(&vm->always_valid.evicted)) {
>> + bo_va = list_first_entry(&vm->always_valid.evicted,
>> struct amdgpu_bo_va,
>> base.vm_status);
>> - spin_unlock(&vm->status_lock);
>> + spin_unlock(&vm->individual_lock);
>>
>> bo = bo_va->base.bo;
>> ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2);
>> @@ -1052,14 +1061,14 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev,
>> struct drm_exec *exec, if (ret)
>> return ret;
>>
>> - /* This moves the bo_va to the done list */
>> + /* This moves the bo_va to the idle list */
>> ret = amdgpu_vm_bo_update(adev, bo_va, false);
>> if (ret)
>> return ret;
>>
>> - spin_lock(&vm->status_lock);
>> + spin_lock(&vm->individual_lock);
>> }
>> - spin_unlock(&vm->status_lock);
>> + spin_unlock(&vm->individual_lock);
>>
>> return 0;
>> }
>> @@ -1091,7 +1100,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr
>> *uq_mgr) if (unlikely(ret))
>> goto unlock_all;
>>
>> - ret = amdgpu_vm_lock_done_list(vm, &exec, 1);
>> + ret = amdgpu_vm_lock_individual(vm, &exec, 1);
>> drm_exec_retry_on_contention(&exec);
>> if (unlikely(ret))
>> goto unlock_all;
>> @@ -1134,7 +1143,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr
>> *uq_mgr)
>>
>> key = 0;
>> /* Validate User Ptr BOs */
>> - list_for_each_entry(bo_va, &vm->done, base.vm_status) {
>> + list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status)
> {
>> bo = bo_va->base.bo;
>> if (!bo)
>> continue;
>> @@ -1184,10 +1193,10 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr
>> *uq_mgr)
>>
>> /*
>> * We need to wait for all VM updates to finish before restarting
> the
>> - * queues. Using the done list like that is now ok since
> everything is
>> + * queues. Using the idle list like that is now ok since everything
> is
>> * locked in place.
>> */
>> - list_for_each_entry(bo_va, &vm->done, base.vm_status)
>> + list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status)
>> dma_fence_wait(bo_va->last_pt_update, false);
>> dma_fence_wait(vm->last_update, false);
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 6b58b0ecfa06..9f71a146f1f6
>> 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
>> @@ -138,6 +138,47 @@ static void amdgpu_vm_assert_locked(struct amdgpu_vm
>> *vm) dma_resv_assert_held(vm->root.bo->tbo.base.resv);
>> }
>>
>> +/* Initialize the amdgpu_vm_bo_status object */
>> +static void amdgpu_vm_bo_status_init(struct amdgpu_vm_bo_status *lists)
>> +{
>> + INIT_LIST_HEAD(&lists->evicted);
>> + INIT_LIST_HEAD(&lists->moved);
>> + INIT_LIST_HEAD(&lists->idle);
>> +}
>> +
>> +/*
>> + * Make sure we have the lock to modify the vm_bo status and return the
>> object + * with the status lists.
>> + */
>> +static struct amdgpu_vm_bo_status *
>> +amdgpu_vm_bo_lock_lists(struct amdgpu_vm_bo_base *vm_bo)
>> +{
>> + struct amdgpu_vm *vm = vm_bo->vm;
>> + struct amdgpu_bo *bo = vm_bo->bo;
>> +
>> + if (amdgpu_vm_is_bo_always_valid(vm, bo)) {
>> + /* No extra locking needed, protected by the root PD
> resv lock */
>> + amdgpu_vm_assert_locked(vm);
>> +
>> + if (bo->tbo.type == ttm_bo_type_kernel)
>> + return &vm->kernel;
>> +
>> + return &vm->always_valid;
>> + }
>> +
>> + spin_lock(&vm_bo->vm->individual_lock);
>> + return &vm->individual;
>> +}
>> +
>> +/* Eventually unlock the status list lock again */
>> +static void amdgpu_vm_bo_unlock_lists(struct amdgpu_vm_bo_base *vm_bo)
>> +{
>> + if (amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
>> + amdgpu_vm_assert_locked(vm_bo->vm);
>> + else
>> + spin_unlock(&vm_bo->vm->individual_lock);
>> +}
>> +
>> /**
>> * amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid
>> *
>> @@ -157,37 +198,44 @@ bool amdgpu_vm_is_bo_always_valid(struct amdgpu_vm
>> *vm, struct amdgpu_bo *bo) *
>> * @vm_bo: vm_bo which is evicted
>> *
>> - * State for PDs/PTs and per VM BOs which are not at the location they
>> should - * be.
>> + * State for vm_bo objects meaning the underlying BO was evicted and need
>> to + * move in place again.
>> */
>> static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
>> {
>> - struct amdgpu_vm *vm = vm_bo->vm;
>> - struct amdgpu_bo *bo = vm_bo->bo;
>> + struct amdgpu_vm_bo_status *lists;
>>
>> + lists = amdgpu_vm_bo_lock_lists(vm_bo);
>> vm_bo->moved = true;
>> - amdgpu_vm_assert_locked(vm);
>> - spin_lock(&vm_bo->vm->status_lock);
>> - if (bo->tbo.type == ttm_bo_type_kernel)
>> - list_move(&vm_bo->vm_status, &vm->evicted);
>> - else
>> - list_move_tail(&vm_bo->vm_status, &vm->evicted);
>> - spin_unlock(&vm_bo->vm->status_lock);
>> + list_move(&vm_bo->vm_status, &lists->evicted);
>> + amdgpu_vm_bo_unlock_lists(vm_bo);
>> }
>> /**
>> * amdgpu_vm_bo_moved - vm_bo is moved
>> *
>> * @vm_bo: vm_bo which is moved
>> *
>> - * State for per VM BOs which are moved, but that change is not yet
>> reflected - * in the page tables.
>> + * State for vm_bo objects meaning the underlying BO was moved but the new
>> + * location not yet reflected in the page tables.
>> */
>> static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
>> {
>> - amdgpu_vm_assert_locked(vm_bo->vm);
>> - spin_lock(&vm_bo->vm->status_lock);
>> - list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
>> - spin_unlock(&vm_bo->vm->status_lock);
>> + struct amdgpu_vm_bo_status *lists;
>> + struct amdgpu_bo *bo = vm_bo->bo;
>> +
>> + /*
>> + * The root PD doesn't have a parent PDE and goes directly into the
>> + * idle state.
>> + */
>> + lists = amdgpu_vm_bo_lock_lists(vm_bo);
>> + if (bo && bo->tbo.type == ttm_bo_type_kernel && !bo->parent) {
>> + vm_bo->moved = false;
>> + list_move(&vm_bo->vm_status, &lists->idle);
>> + } else {
>> + vm_bo->moved = true;
>> + list_move(&vm_bo->vm_status, &lists->moved);
>> + }
>> + amdgpu_vm_bo_unlock_lists(vm_bo);
>> }
>>
>> /**
>> @@ -195,120 +243,43 @@ static void amdgpu_vm_bo_moved(struct
>> amdgpu_vm_bo_base *vm_bo) *
>> * @vm_bo: vm_bo which is now idle
>> *
>> - * State for PDs/PTs and per VM BOs which have gone through the state
>> machine - * and are now idle.
>> + * State for vm_bo objects meaning we are done with the state machine and
>> no + * further action is necessary.
>> */
>> static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
>> {
>> - amdgpu_vm_assert_locked(vm_bo->vm);
>> - spin_lock(&vm_bo->vm->status_lock);
>> - list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
>> - spin_unlock(&vm_bo->vm->status_lock);
>> - vm_bo->moved = false;
>> -}
>> + struct amdgpu_vm_bo_status *lists;
>>
>> -/**
>> - * amdgpu_vm_bo_invalidated - vm_bo is invalidated
>> - *
>> - * @vm_bo: vm_bo which is now invalidated
>> - *
>> - * State for normal BOs which are invalidated and that change not yet
>> reflected - * in the PTs.
>> - */
>> -static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
>> -{
>> - spin_lock(&vm_bo->vm->status_lock);
>> - list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
>> - spin_unlock(&vm_bo->vm->status_lock);
>> -}
>> -
>> -/**
>> - * amdgpu_vm_bo_evicted_user - vm_bo is evicted
>> - *
>> - * @vm_bo: vm_bo which is evicted
>> - *
>> - * State for BOs used by user mode queues which are not at the location
>> they - * should be.
>> - */
>> -static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)
>> -{
>> - vm_bo->moved = true;
>> - spin_lock(&vm_bo->vm->status_lock);
>> - list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user);
>> - spin_unlock(&vm_bo->vm->status_lock);
>> -}
>> -
>> -/**
>> - * amdgpu_vm_bo_relocated - vm_bo is reloacted
>> - *
>> - * @vm_bo: vm_bo which is relocated
>> - *
>> - * State for PDs/PTs which needs to update their parent PD.
>> - * For the root PD, just move to idle state.
>> - */
>> -static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
>> -{
>> - amdgpu_vm_assert_locked(vm_bo->vm);
>> - if (vm_bo->bo->parent) {
>> - spin_lock(&vm_bo->vm->status_lock);
>> - list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
>> - spin_unlock(&vm_bo->vm->status_lock);
>> - } else {
>> - amdgpu_vm_bo_idle(vm_bo);
>> - }
>> -}
>> -
>> -/**
>> - * amdgpu_vm_bo_done - vm_bo is done
>> - *
>> - * @vm_bo: vm_bo which is now done
>> - *
>> - * State for normal BOs which are invalidated and that change has been
>> updated - * in the PTs.
>> - */
>> -static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
>> -{
>> - amdgpu_vm_assert_locked(vm_bo->vm);
>> - spin_lock(&vm_bo->vm->status_lock);
>> - list_move(&vm_bo->vm_status, &vm_bo->vm->done);
>> - spin_unlock(&vm_bo->vm->status_lock);
>> + lists = amdgpu_vm_bo_lock_lists(vm_bo);
>> + if (!amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
>> + vm_bo->moved = false;
>> + list_move(&vm_bo->vm_status, &lists->idle);
>> + amdgpu_vm_bo_unlock_lists(vm_bo);
>> }
>>
>> /**
>> * amdgpu_vm_bo_reset_state_machine - reset the vm_bo state machine
>> * @vm: the VM which state machine to reset
>> *
>> - * Move all vm_bo object in the VM into a state where they will be updated
>> - * again during validation.
>> + * Move all vm_bo object in the VM into a state where their location will
>> be + * updated in the page tables again.
>> */
>> static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
>> {
>> - struct amdgpu_vm_bo_base *vm_bo, *tmp;
>> -
>> amdgpu_vm_assert_locked(vm);
>> + list_splice_init(&vm->kernel.idle, &vm->kernel.moved);
>> + list_splice_init(&vm->always_valid.idle, &vm->always_valid.moved);
>>
>> - spin_lock(&vm->status_lock);
>> - list_splice_init(&vm->done, &vm->invalidated);
>> - list_for_each_entry(vm_bo, &vm->invalidated, vm_status)
>> - vm_bo->moved = true;
>> -
>> - list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) {
>> - struct amdgpu_bo *bo = vm_bo->bo;
>> -
>> - vm_bo->moved = true;
>> - if (!bo || bo->tbo.type != ttm_bo_type_kernel)
>> - list_move(&vm_bo->vm_status, &vm_bo->vm-
>> moved);
>> - else if (bo->parent)
>> - list_move(&vm_bo->vm_status, &vm_bo->vm-
>> relocated);
>> - }
>> - spin_unlock(&vm->status_lock);
>> + spin_lock(&vm->individual_lock);
>> + list_splice_init(&vm->individual.idle, &vm->individual.moved);
>> + spin_unlock(&vm->individual_lock);
>> }
>>
>> /**
>> * amdgpu_vm_update_shared - helper to update shared memory stat
>> * @base: base structure for tracking BO usage in a VM
>> *
>> - * Takes the vm status_lock and updates the shared memory stat. If the
>> basic + * Takes the vm stats_lock and updates the shared memory stat. If
>> the basic * stat changed (e.g. buffer was moved) amdgpu_vm_update_stats
>> need to be called * as well.
>> */
>> @@ -321,7 +292,7 @@ static void amdgpu_vm_update_shared(struct
>> amdgpu_vm_bo_base *base) bool shared;
>>
>> dma_resv_assert_held(bo->tbo.base.resv);
>> - spin_lock(&vm->status_lock);
>> + spin_lock(&vm->stats_lock);
>> shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
>> if (base->shared != shared) {
>> base->shared = shared;
>> @@ -333,7 +304,7 @@ static void amdgpu_vm_update_shared(struct
>> amdgpu_vm_bo_base *base) vm->stats[bo_memtype].drm.private += size;
>> }
>> }
>> - spin_unlock(&vm->status_lock);
>> + spin_unlock(&vm->stats_lock);
>> }
>>
>> /**
>> @@ -358,11 +329,11 @@ void amdgpu_vm_bo_update_shared(struct amdgpu_bo *bo)
>> * be bo->tbo.resource
>> * @sign: if we should add (+1) or subtract (-1) from the stat
>> *
>> - * Caller need to have the vm status_lock held. Useful for when multiple
>> update + * Caller need to have the vm stats_lock held. Useful for when
>> multiple update * need to happen at the same time.
>> */
>> static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
>> - struct ttm_resource *res, int sign)
>> + struct ttm_resource
> *res, int sign)
>> {
>> struct amdgpu_vm *vm = base->vm;
>> struct amdgpu_bo *bo = base->bo;
>> @@ -386,7 +357,8 @@ static void amdgpu_vm_update_stats_locked(struct
>> amdgpu_vm_bo_base *base, */
>> if (bo->flags & AMDGPU_GEM_CREATE_DISCARDABLE)
>> vm->stats[res_memtype].drm.purgeable +=
> size;
>> - if (!(bo->preferred_domains &
> amdgpu_mem_type_to_domain(res_memtype)))
>> + if (!(bo->preferred_domains &
>> + amdgpu_mem_type_to_domain(res_memtype)))
>> vm->stats[bo_memtype].evicted += size;
>> }
>> }
>> @@ -405,9 +377,9 @@ void amdgpu_vm_update_stats(struct amdgpu_vm_bo_base
>> *base, {
>> struct amdgpu_vm *vm = base->vm;
>>
>> - spin_lock(&vm->status_lock);
>> + spin_lock(&vm->stats_lock);
>> amdgpu_vm_update_stats_locked(base, res, sign);
>> - spin_unlock(&vm->status_lock);
>> + spin_unlock(&vm->stats_lock);
>> }
>>
>> /**
>> @@ -428,37 +400,34 @@ void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base
>> *base, base->next = NULL;
>> INIT_LIST_HEAD(&base->vm_status);
>>
>> + dma_resv_assert_held(vm->root.bo->tbo.base.resv);
>> if (!bo)
>> return;
>> +
>> base->next = bo->vm_bo;
>> bo->vm_bo = base;
>>
>> - spin_lock(&vm->status_lock);
>> + spin_lock(&vm->stats_lock);
>> base->shared = drm_gem_object_is_shared_for_memory_stats(&bo-
>> tbo.base);
>> amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1);
>> - spin_unlock(&vm->status_lock);
>> + spin_unlock(&vm->stats_lock);
>>
>> - if (!amdgpu_vm_is_bo_always_valid(vm, bo))
>> + if (!amdgpu_vm_is_bo_always_valid(vm, bo)) {
>> + amdgpu_vm_bo_idle(base);
>> return;
>> -
>> - dma_resv_assert_held(vm->root.bo->tbo.base.resv);
>> + }
>>
>> ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move);
>> - if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
>> - amdgpu_vm_bo_relocated(base);
>> - else
>> - amdgpu_vm_bo_idle(base);
>>
>> + /*
>> + * When a per VM isn't in the desired domain put it into the
> evicted
>> + * state to make sure that it gets validated on the next best
> occasion.
>> + */
>> if (bo->preferred_domains &
>> amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type))
>> - return;
>> -
>> - /*
>> - * we checked all the prerequisites, but it looks like this per vm
> bo
>> - * is currently evicted. add the bo to the evicted list to make
> sure it
>> - * is validated on next vm use to avoid fault.
>> - * */
>> - amdgpu_vm_bo_evicted(base);
>> + amdgpu_vm_bo_moved(base);
>> + else
>> + amdgpu_vm_bo_evicted(base);
>> }
>>
>> /**
>> @@ -479,41 +448,41 @@ int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct
>> drm_exec *exec, }
>>
>> /**
>> - * amdgpu_vm_lock_done_list - lock all BOs on the done list
>> + * amdgpu_vm_lock_individual - lock all BOs on the individual idle list
>> * @vm: vm providing the BOs
>> * @exec: drm execution context
>> * @num_fences: number of extra fences to reserve
>> *
>> - * Lock the BOs on the done list in the DRM execution context.
>> + * Lock the BOs on the individual idle list in the DRM execution context.
>> */
>> -int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
>> - unsigned int num_fences)
>> +int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
>> + unsigned int num_fences)
>> {
>> - struct list_head *prev = &vm->done;
>> + struct list_head *prev = &vm->individual.idle;
>> struct amdgpu_bo_va *bo_va;
>> struct amdgpu_bo *bo;
>> int ret;
>>
>> /* We can only trust prev->next while holding the lock */
>> - spin_lock(&vm->status_lock);
>> - while (!list_is_head(prev->next, &vm->done)) {
>> + spin_lock(&vm->individual_lock);
>> + while (!list_is_head(prev->next, &vm->individual.idle)) {
>> bo_va = list_entry(prev->next, typeof(*bo_va),
> base.vm_status);
>>
>> bo = bo_va->base.bo;
>> if (bo) {
>> amdgpu_bo_ref(bo);
>> - spin_unlock(&vm->status_lock);
>> + spin_unlock(&vm->individual_lock);
>>
>> ret = drm_exec_prepare_obj(exec, &bo-
>> tbo.base, 1);
>> amdgpu_bo_unref(&bo);
>> if (unlikely(ret))
>> return ret;
>>
>> - spin_lock(&vm->status_lock);
>> + spin_lock(&vm->individual_lock);
>> }
>> prev = prev->next;
>> }
>> - spin_unlock(&vm->status_lock);
>> + spin_unlock(&vm->individual_lock);
>>
>> return 0;
>> }
>> @@ -609,10 +578,10 @@ int amdgpu_vm_validate(struct amdgpu_device *adev,
>> struct amdgpu_vm *vm, void *param)
>> {
>> uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm);
>> - struct amdgpu_vm_bo_base *bo_base;
>> - struct amdgpu_bo *bo;
>> + struct amdgpu_vm_bo_base *bo_base, *tmp;
>> int r;
>>
>> + dma_resv_assert_held(vm->root.bo->tbo.base.resv);
>> if (vm->generation != new_vm_generation) {
>> vm->generation = new_vm_generation;
>> amdgpu_vm_bo_reset_state_machine(vm);
>> @@ -622,49 +591,59 @@ int amdgpu_vm_validate(struct amdgpu_device *adev,
>> struct amdgpu_vm *vm, return r;
>> }
>>
>> - spin_lock(&vm->status_lock);
>> - while (!list_empty(&vm->evicted)) {
>> - bo_base = list_first_entry(&vm->evicted,
>> - struct
> amdgpu_vm_bo_base,
>> - vm_status);
>> - spin_unlock(&vm->status_lock);
>> + list_for_each_entry_safe(bo_base, tmp, &vm->kernel.evicted,
> vm_status) {
>> + r = validate(param, bo_base->bo);
>> + if (r)
>> + return r;
>>
>> - bo = bo_base->bo;
>> + vm->update_funcs->map_table(to_amdgpu_bo_vm(bo_base-
>> bo));
>> + amdgpu_vm_bo_moved(bo_base);
>> + }
>>
>> - r = validate(param, bo);
>> + /*
>> + * As soon as all page tables are in place we can start updating
> them
>> + * again.
>> + */
>> + amdgpu_vm_eviction_lock(vm);
>> + vm->evicting = false;
>> + amdgpu_vm_eviction_unlock(vm);
>> +
>> + list_for_each_entry_safe(bo_base, tmp, &vm->always_valid.evicted,
>> + vm_status) {
>> + r = validate(param, bo_base->bo);
>> if (r)
>> return r;
>>
>> - if (bo->tbo.type != ttm_bo_type_kernel) {
>> - amdgpu_vm_bo_moved(bo_base);
>> - } else {
>> - vm->update_funcs-
>> map_table(to_amdgpu_bo_vm(bo));
>> - amdgpu_vm_bo_relocated(bo_base);
>> - }
>> - spin_lock(&vm->status_lock);
>> + amdgpu_vm_bo_moved(bo_base);
>> }
>> - while (ticket && !list_empty(&vm->evicted_user)) {
>> - bo_base = list_first_entry(&vm->evicted_user,
>> - struct
> amdgpu_vm_bo_base,
>> - vm_status);
>> - spin_unlock(&vm->status_lock);
>>
>> - bo = bo_base->bo;
>> - dma_resv_assert_held(bo->tbo.base.resv);
>> + if (!ticket)
>> + return 0;
>> +
>> + spin_lock(&vm->individual_lock);
>> +restart:
>> + list_for_each_entry(bo_base, &vm->individual.evicted, vm_status) {
>> + struct amdgpu_bo *bo = bo_base->bo;
>> +
>> + if (dma_resv_locking_ctx(bo->tbo.base.resv) != ticket)
>> + continue;
>> +
>> + spin_unlock(&vm->individual_lock);
>>
>> r = validate(param, bo);
>> if (r)
>> return r;
>>
>> - amdgpu_vm_bo_invalidated(bo_base);
>> + amdgpu_vm_bo_moved(bo_base);
>>
>> - spin_lock(&vm->status_lock);
>> + /* It's a bit inefficient to always jump back to the
> start, but
>> + * we would need to re-structure the KFD for properly
> fixing
>> + * that.
>> + */
>> + spin_lock(&vm->individual_lock);
>> + goto restart;
>> }
>> - spin_unlock(&vm->status_lock);
>> -
>> - amdgpu_vm_eviction_lock(vm);
>> - vm->evicting = false;
>> - amdgpu_vm_eviction_unlock(vm);
>> + spin_unlock(&vm->individual_lock);
>>
>> return 0;
>> }
>> @@ -689,9 +668,7 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
>> ret = !vm->evicting;
>> amdgpu_vm_eviction_unlock(vm);
>>
>> - spin_lock(&vm->status_lock);
>> - ret &= list_empty(&vm->evicted);
>> - spin_unlock(&vm->status_lock);
>> + ret &= list_empty(&vm->kernel.evicted);
>>
>> spin_lock(&vm->immediate.lock);
>> ret &= !vm->immediate.stopped;
>> @@ -985,18 +962,13 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
>> struct amdgpu_vm *vm, bool immediate)
>> {
>> struct amdgpu_vm_update_params params;
>> - struct amdgpu_vm_bo_base *entry;
>> + struct amdgpu_vm_bo_base *entry, *tmp;
>> bool flush_tlb_needed = false;
>> - LIST_HEAD(relocated);
>> int r, idx;
>>
>> amdgpu_vm_assert_locked(vm);
>>
>> - spin_lock(&vm->status_lock);
>> - list_splice_init(&vm->relocated, &relocated);
>> - spin_unlock(&vm->status_lock);
>> -
>> - if (list_empty(&relocated))
>> + if (list_empty(&vm->kernel.moved))
>> return 0;
>>
>> if (!drm_dev_enter(adev_to_drm(adev), &idx))
>> @@ -1012,7 +984,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
>> if (r)
>> goto error;
>>
>> - list_for_each_entry(entry, &relocated, vm_status) {
>> + list_for_each_entry(entry, &vm->kernel.moved, vm_status) {
>> /* vm_flush_needed after updating moved PDEs */
>> flush_tlb_needed |= entry->moved;
>>
>> @@ -1028,11 +1000,8 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
>> if (flush_tlb_needed)
>> atomic64_inc(&vm->tlb_seq);
>>
>> - while (!list_empty(&relocated)) {
>> - entry = list_first_entry(&relocated, struct
> amdgpu_vm_bo_base,
>> - vm_status);
>> + list_for_each_entry_safe(entry, tmp, &vm->kernel.moved, vm_status)
>> amdgpu_vm_bo_idle(entry);
>> - }
>>
>> error:
>> drm_dev_exit(idx);
>> @@ -1260,9 +1229,9 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev,
>> struct amdgpu_vm *vm, void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
>> struct amdgpu_mem_stats
> stats[__AMDGPU_PL_NUM])
>> {
>> - spin_lock(&vm->status_lock);
>> + spin_lock(&vm->stats_lock);
>> memcpy(stats, vm->stats, sizeof(*stats) * __AMDGPU_PL_NUM);
>> - spin_unlock(&vm->status_lock);
>> + spin_unlock(&vm->stats_lock);
>> }
>>
>> /**
>> @@ -1406,7 +1375,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
>> struct amdgpu_bo_va *bo_va, else
>> amdgpu_vm_bo_idle(&bo_va->base);
>> } else {
>> - amdgpu_vm_bo_done(&bo_va->base);
>> + amdgpu_vm_bo_idle(&bo_va->base);
>> }
>>
>> list_splice_init(&bo_va->invalids, &bo_va->valids);
>> @@ -1629,29 +1598,25 @@ int amdgpu_vm_handle_moved(struct amdgpu_device
>> *adev, struct amdgpu_vm *vm,
>> struct ww_acquire_ctx *ticket)
>> {
>> - struct amdgpu_bo_va *bo_va;
>> + struct amdgpu_bo_va *bo_va, *tmp;
>> struct dma_resv *resv;
>> bool clear, unlock;
>> int r;
>>
>> - spin_lock(&vm->status_lock);
>> - while (!list_empty(&vm->moved)) {
>> - bo_va = list_first_entry(&vm->moved, struct
> amdgpu_bo_va,
>> - base.vm_status);
>> - spin_unlock(&vm->status_lock);
>> -
>> + list_for_each_entry_safe(bo_va, tmp, &vm->always_valid.moved,
>> + base.vm_status) {
>> /* Per VM BOs never need to bo cleared in the page
> tables */
>> r = amdgpu_vm_bo_update(adev, bo_va, false);
>> if (r)
>> return r;
>> - spin_lock(&vm->status_lock);
>> }
>>
>> - while (!list_empty(&vm->invalidated)) {
>> - bo_va = list_first_entry(&vm->invalidated, struct
> amdgpu_bo_va,
>> - base.vm_status);
>> + spin_lock(&vm->individual_lock);
>> + while (!list_empty(&vm->individual.moved)) {
>> + bo_va = list_first_entry(&vm->individual.moved,
>> + typeof(*bo_va),
> base.vm_status);
>> resv = bo_va->base.bo->tbo.base.resv;
>> - spin_unlock(&vm->status_lock);
>> + spin_unlock(&vm->individual_lock);
>>
>> /* Try to reserve the BO to avoid clearing its ptes */
>> if (!adev->debug_vm && dma_resv_trylock(resv)) {
>> @@ -1681,11 +1646,11 @@ int amdgpu_vm_handle_moved(struct amdgpu_device
>> *adev, drm_gem_is_imported(&bo_va->base.bo->tbo.base) &&
>> (!bo_va->base.bo->tbo.resource ||
>> bo_va->base.bo->tbo.resource->mem_type ==
> TTM_PL_SYSTEM))
>> - amdgpu_vm_bo_evicted_user(&bo_va->base);
>> + amdgpu_vm_bo_evicted(&bo_va->base);
>>
>> - spin_lock(&vm->status_lock);
>> + spin_lock(&vm->individual_lock);
>> }
>> - spin_unlock(&vm->status_lock);
>> + spin_unlock(&vm->individual_lock);
>>
>> return 0;
>> }
>> @@ -2223,9 +2188,9 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev,
>> }
>> }
>>
>> - spin_lock(&vm->status_lock);
>> + spin_lock(&vm->individual_lock);
>> list_del(&bo_va->base.vm_status);
>> - spin_unlock(&vm->status_lock);
>> + spin_unlock(&vm->individual_lock);
>>
>> list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
>> list_del(&mapping->list);
>> @@ -2305,14 +2270,7 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_bo *bo,
>> bool evicted)
>>
>> if (bo_base->moved)
>> continue;
>> - bo_base->moved = true;
>> -
>> - if (bo->tbo.type == ttm_bo_type_kernel)
>> - amdgpu_vm_bo_relocated(bo_base);
>> - else if (amdgpu_vm_is_bo_always_valid(vm, bo))
>> - amdgpu_vm_bo_moved(bo_base);
>> - else
>> - amdgpu_vm_bo_invalidated(bo_base);
>> + amdgpu_vm_bo_moved(bo_base);
>> }
>> }
>>
>> @@ -2333,10 +2291,10 @@ void amdgpu_vm_bo_move(struct amdgpu_bo *bo, struct
>> ttm_resource *new_mem, for (bo_base = bo->vm_bo; bo_base; bo_base =
>> bo_base->next) {
>> struct amdgpu_vm *vm = bo_base->vm;
>>
>> - spin_lock(&vm->status_lock);
>> + spin_lock(&vm->stats_lock);
>> amdgpu_vm_update_stats_locked(bo_base, bo-
>> tbo.resource, -1);
>> amdgpu_vm_update_stats_locked(bo_base, new_mem, +1);
>> - spin_unlock(&vm->status_lock);
>> + spin_unlock(&vm->stats_lock);
>> }
>>
>> amdgpu_vm_bo_invalidate(bo, evicted);
>> @@ -2603,16 +2561,14 @@ int amdgpu_vm_init(struct amdgpu_device *adev,
>> struct amdgpu_vm *vm, vm->va = RB_ROOT_CACHED;
>> for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
>> vm->reserved_vmid[i] = NULL;
>> - INIT_LIST_HEAD(&vm->evicted);
>> - INIT_LIST_HEAD(&vm->evicted_user);
>> - INIT_LIST_HEAD(&vm->relocated);
>> - INIT_LIST_HEAD(&vm->moved);
>> - INIT_LIST_HEAD(&vm->idle);
>> - INIT_LIST_HEAD(&vm->invalidated);
>> - spin_lock_init(&vm->status_lock);
>> +
>> + amdgpu_vm_bo_status_init(&vm->kernel);
>> + amdgpu_vm_bo_status_init(&vm->always_valid);
>> + spin_lock_init(&vm->individual_lock);
>> + amdgpu_vm_bo_status_init(&vm->individual);
>> INIT_LIST_HEAD(&vm->freed);
>> - INIT_LIST_HEAD(&vm->done);
>> INIT_KFIFO(vm->faults);
>> + spin_lock_init(&vm->stats_lock);
>>
>> r = amdgpu_vm_init_entities(adev, vm);
>> if (r)
>> @@ -3089,100 +3045,64 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device
>> *adev, u32 pasid, }
>>
>> #if defined(CONFIG_DEBUG_FS)
>> -/**
>> - * amdgpu_debugfs_vm_bo_info - print BO info for the VM
>> - *
>> - * @vm: Requested VM for printing BO info
>> - * @m: debugfs file
>> - *
>> - * Print BO information in debugfs file for the VM
>> - */
>> -void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
>> -{
>> - struct amdgpu_bo_va *bo_va, *tmp;
>> - u64 total_idle = 0;
>> - u64 total_evicted = 0;
>> - u64 total_relocated = 0;
>> - u64 total_moved = 0;
>> - u64 total_invalidated = 0;
>> - u64 total_done = 0;
>> - unsigned int total_idle_objs = 0;
>> - unsigned int total_evicted_objs = 0;
>> - unsigned int total_relocated_objs = 0;
>> - unsigned int total_moved_objs = 0;
>> - unsigned int total_invalidated_objs = 0;
>> - unsigned int total_done_objs = 0;
>> - unsigned int id = 0;
>>
>> - amdgpu_vm_assert_locked(vm);
>> +/* print the debug info for a specific set of status lists */
>> +static void amdgpu_debugfs_vm_bo_status_info(struct seq_file *m,
>> + struct
> amdgpu_vm_bo_status *lists)
>> +{
>> + struct amdgpu_vm_bo_base *base;
>> + unsigned int id;
>>
>> - spin_lock(&vm->status_lock);
>> - seq_puts(m, "\tIdle BOs:\n");
>> - list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
>> - if (!bo_va->base.bo)
>> - continue;
>> - total_idle += amdgpu_bo_print_info(id++, bo_va-
>> base.bo, m);
>> - }
>> - total_idle_objs = id;
>> id = 0;
>> -
>> seq_puts(m, "\tEvicted BOs:\n");
>> - list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status)
> {
>> - if (!bo_va->base.bo)
>> + list_for_each_entry(base, &lists->evicted, vm_status) {
>> + if (!base->bo)
>> continue;
>> - total_evicted += amdgpu_bo_print_info(id++, bo_va-
>> base.bo, m);
>> - }
>> - total_evicted_objs = id;
>> - id = 0;
>>
>> - seq_puts(m, "\tRelocated BOs:\n");
>> - list_for_each_entry_safe(bo_va, tmp, &vm->relocated,
> base.vm_status) {
>> - if (!bo_va->base.bo)
>> - continue;
>> - total_relocated += amdgpu_bo_print_info(id++, bo_va-
>> base.bo, m);
>> + amdgpu_bo_print_info(id++, base->bo, m);
>> }
>> - total_relocated_objs = id;
>> - id = 0;
>>
>> + id = 0;
>> seq_puts(m, "\tMoved BOs:\n");
>> - list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
>> - if (!bo_va->base.bo)
>> + list_for_each_entry(base, &lists->moved, vm_status) {
>> + if (!base->bo)
>> continue;
>> - total_moved += amdgpu_bo_print_info(id++, bo_va-
>> base.bo, m);
>> +
>> + amdgpu_bo_print_info(id++, base->bo, m);
>> }
>> - total_moved_objs = id;
>> - id = 0;
>>
>> - seq_puts(m, "\tInvalidated BOs:\n");
>> - list_for_each_entry_safe(bo_va, tmp, &vm->invalidated,
> base.vm_status) {
>> - if (!bo_va->base.bo)
>> + id = 0;
>> + seq_puts(m, "\tIdle BOs:\n");
>> + list_for_each_entry(base, &lists->moved, vm_status) {
>> + if (!base->bo)
>> continue;
>> - total_invalidated += amdgpu_bo_print_info(id++,
> bo_va->base.bo, m);
>> +
>> + amdgpu_bo_print_info(id++, base->bo, m);
>> }
>> - total_invalidated_objs = id;
>> - id = 0;
>> +}
>>
>> - seq_puts(m, "\tDone BOs:\n");
>> - list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
>> - if (!bo_va->base.bo)
>> - continue;
>> - total_done += amdgpu_bo_print_info(id++, bo_va-
>> base.bo, m);
>> - }
>> - spin_unlock(&vm->status_lock);
>> - total_done_objs = id;
>> -
>> - seq_printf(m, "\tTotal idle size: %12lld\tobjs:\t%d\n",
> total_idle,
>> - total_idle_objs);
>> - seq_printf(m, "\tTotal evicted size: %12lld\tobjs:\t%d\n",
>> total_evicted, - total_evicted_objs);
>> - seq_printf(m, "\tTotal relocated size: %12lld\tobjs:\t%d\n",
>> total_relocated, - total_relocated_objs);
>> - seq_printf(m, "\tTotal moved size: %12lld\tobjs:\t%d\n",
>> total_moved, - total_moved_objs);
>> - seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n",
>> total_invalidated, - total_invalidated_objs);
>> - seq_printf(m, "\tTotal done size: %12lld\tobjs:\t%d\n",
> total_done,
>> - total_done_objs);
>> +/**
>> + * amdgpu_debugfs_vm_bo_info - print BO info for the VM
>> + *
>> + * @vm: Requested VM for printing BO info
>> + * @m: debugfs file
>> + *
>> + * Print BO information in debugfs file for the VM
>> + */
>> +void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
>> +{
>> + amdgpu_vm_assert_locked(vm);
>> +
>> + seq_puts(m, "\tKernel PT/PDs:\n");
>> + amdgpu_debugfs_vm_bo_status_info(m, &vm->kernel);
>> +
>> + seq_puts(m, "\tPer VM BOs:\n");
>> + amdgpu_debugfs_vm_bo_status_info(m, &vm->always_valid);
>> +
>> + seq_puts(m, "\tIndividual BOs:\n");
>> + spin_lock(&vm->individual_lock);
>> + amdgpu_debugfs_vm_bo_status_info(m, &vm->individual);
>> + spin_unlock(&vm->individual_lock);
>> }
>> #endif
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index f33ea7f8509b..cc96a3e6252f
>> 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
>> @@ -205,17 +205,34 @@ struct amdgpu_vm_bo_base {
>> /* protected by bo being reserved */
>> struct amdgpu_vm_bo_base *next;
>>
>> - /* protected by vm status_lock */
>> + /* protected by vm reservation and invalidated_lock */
>> struct list_head vm_status;
>>
>> /* if the bo is counted as shared in mem stats
>> - * protected by vm status_lock */
>> + * protected by vm BO being reserved */
>> bool shared;
>>
>> /* protected by the BO being reserved */
>> bool moved;
>> };
>>
>> +/*
>> + * The following status lists contain amdgpu_vm_bo_base objects for
>> + * either PD/PTs, per VM BOs or BOs with individual resv object.
>> + *
>> + * The state transits are: evicted -> moved -> idle
>> + */
>> +struct amdgpu_vm_bo_status {
>> + /* BOs evicted which need to move into place again */
>> + struct list_head evicted;
>> +
>> + /* BOs which moved but new location hasn't been updated in the PDs/
> PTs */
>> + struct list_head moved;
>> +
>> + /* BOs done with the state machine and need no further action */
>> + struct list_head idle;
>> +};
>> +
>> /* provided by hw blocks that can write ptes, e.g., sdma */
>> struct amdgpu_vm_pte_funcs {
>> /* number of dw to reserve per operation */
>> @@ -345,47 +362,29 @@ struct amdgpu_vm {
>> bool evicting;
>> unsigned int saved_flags;
>>
>> - /* Lock to protect vm_bo add/del/move on all lists of vm */
>> - spinlock_t status_lock;
>> -
>> - /* Memory statistics for this vm, protected by status_lock */
>> + /* Memory statistics for this vm, protected by stats_lock */
>> + spinlock_t stats_lock;
>> struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM];
>>
>> + /* BO's belonging to PD/PT which are internal to the kernel. */
>> + struct amdgpu_vm_bo_status kernel;
>> +
>> /*
>> - * The following lists contain amdgpu_vm_bo_base objects for
> either
>> - * PDs, PTs or per VM BOs. The state transits are:
>> - *
>> - * evicted -> relocated (PDs, PTs) or moved (per VM BOs) -> idle
>> + * BOs allocated by userspace where the dma_resv is shared with the
>> + * root PD
>> */
>> -
>> - /* Per-VM and PT BOs who needs a validation */
>> - struct list_head evicted;
>> -
>> - /* PT BOs which relocated and their parent need an update */
>> - struct list_head relocated;
>> -
>> - /* per VM BOs moved, but not yet updated in the PT */
>> - struct list_head moved;
>> -
>> - /* All BOs of this VM not currently in the state machine */
>> - struct list_head idle;
>> + struct amdgpu_vm_bo_status always_valid;
>>
>> /*
>> * The following lists contain amdgpu_vm_bo_base objects for BOs
> which
>> - * have their own dma_resv object and not depend on the root PD.
> Their
>> - * state transits are:
>> + * have their own dma_resv object and not depend on the root PD.
>> *
>> - * evicted_user or invalidated -> done
>> + * Lists are protected by the individual_lock.
>> */
>> + spinlock_t individual_lock;
>>
>> - /* BOs for user mode queues that need a validation */
>> - struct list_head evicted_user;
>> -
>> - /* regular invalidated BOs, but not yet updated in the PT */
>> - struct list_head invalidated;
>> -
>> - /* BOs which are invalidated, has been updated in the PTs */
>> - struct list_head done;
>> + /* Userspace BOs with individual resv object */
>> + struct amdgpu_vm_bo_status individual;
>>
>> /*
>> * This list contains amdgpu_bo_va_mapping objects which have been
> freed
>> @@ -507,8 +506,8 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev,
>> struct amdgpu_vm *vm); void amdgpu_vm_fini(struct amdgpu_device *adev,
>> struct amdgpu_vm *vm); int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct
>> drm_exec *exec, unsigned int num_fences);
>> -int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
>> - unsigned int num_fences);
>> +int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
>> + unsigned int num_fences);
>> bool amdgpu_vm_ready(struct amdgpu_vm *vm);
>> uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm
>> *vm); int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm
>> *vm, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c index
>> a930f1522f96..ff334f4dcc90 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
>> @@ -544,9 +544,7 @@ static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base
>> *entry) entry->bo->vm_bo = NULL;
>> ttm_bo_set_bulk_move(&entry->bo->tbo, NULL);
>>
>> - spin_lock(&entry->vm->status_lock);
>> list_del(&entry->vm_status);
>> - spin_unlock(&entry->vm->status_lock);
>> amdgpu_bo_unref(&entry->bo);
>> }
>>
>> @@ -590,7 +588,6 @@ static void amdgpu_vm_pt_add_list(struct
>> amdgpu_vm_update_params *params, struct amdgpu_vm_pt_cursor seek;
>> struct amdgpu_vm_bo_base *entry;
>>
>> - spin_lock(¶ms->vm->status_lock);
>> for_each_amdgpu_vm_pt_dfs_safe(params->adev, params->vm, cursor,
> seek,
>> entry) { if (entry && entry->bo)
>> list_move(&entry->vm_status, ¶ms-
>> tlb_flush_waitlist);
>> @@ -598,7 +595,6 @@ static void amdgpu_vm_pt_add_list(struct
>> amdgpu_vm_update_params *params,
>>
>> /* enter start node now */
>> list_move(&cursor->entry->vm_status, ¶ms->tlb_flush_waitlist);
>> - spin_unlock(¶ms->vm->status_lock);
>> }
>>
>> /**
>
>
>
>
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH 8/8] drm/amdgpu: restructure VM state machine v2
2026-05-26 8:51 ` Christian König
@ 2026-05-26 10:11 ` Timur Kristóf
2026-05-26 10:44 ` Christian König
0 siblings, 1 reply; 30+ messages in thread
From: Timur Kristóf @ 2026-05-26 10:11 UTC (permalink / raw)
To: alexander.deucher, Prike.Liang, sukhatri, jesse.zhang, amd-gfx,
Christian König
Cc: Vock Natalie
On 2026. május 26., kedd 10:51:38 közép-európai nyári idő Christian König
wrote:
> On 5/26/26 01:29, Timur Kristóf wrote:
> > On 2026. április 27., hétfő 20:27:23 közép-európai nyári idő Christian
> > König>
> > wrote:
> >> Instead of coming up with more sophisticated names for states a VM BO
> >> can be in, group them by the type of BO first and then by the state.
> >>
> >> So we end with BO type kernel, always_valid and individual and then
> >> states
> >> evicted, moved and idle.
> >>
> >> Not much functional change, except that evicted_user is moved back
> >> together with the other BOs again which makes the handling in
> >> amdgpu_vm_validate() a bit more complex.
> >>
> >> Also fixes a problem with user queues and amdgpu_vm_ready(). We didn't
> >> considered the VM ready when user BOs were not ideally placed, harmless
> >> performance impact for kernel queues but a complete show stopper for
> >> userqueues.
> >>
> >> v2: fix a few typos in comments, rename the BO types to make them more
> >>
> >> descriptive, fix a couple of bugs found during testing
> >>
> >> v3: squashed together with revert to old status lock handling, looks
> >>
> >> like the first patch still had some bug which this one here should
> >> fix.
> >> Fix a missing lock around debugfs printing.
> >>
> >> Signed-off-by: Christian König <christian.koenig@amd.com>
> >
> > Hi Christian,
> >
> > This patch regresses GPU recovery.
> > (Tested on SI, haven't had time to test other GPUs yet.)
> >
> > It hits the following warning:
> > WARNING: drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c:635 at
> > amdgpu_vm_pde_update And then this error:
> > [drm:amdgpu_gem_va_update_vm [amdgpu]] *ERROR* Couldn't update BO_VA (-22)
> >
> > Please fix.
>
> Yeah that's a known issue. Fix is already pushed to amd-staging-drm-next.
Hi Christian,
Where is this fix? As far as I see "restructure VM state machine v2" is the top
commit on amd-staging-drm-next right now.
Timur
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH 8/8] drm/amdgpu: restructure VM state machine v2
2026-05-26 10:11 ` Timur Kristóf
@ 2026-05-26 10:44 ` Christian König
0 siblings, 0 replies; 30+ messages in thread
From: Christian König @ 2026-05-26 10:44 UTC (permalink / raw)
To: Timur Kristóf, alexander.deucher, Prike.Liang, sukhatri,
jesse.zhang, amd-gfx
Cc: Vock Natalie
On 5/26/26 12:11, Timur Kristóf wrote:
> On 2026. május 26., kedd 10:51:38 közép-európai nyári idő Christian König
> wrote:
>> On 5/26/26 01:29, Timur Kristóf wrote:
>>> On 2026. április 27., hétfő 20:27:23 közép-európai nyári idő Christian
>>> König>
>>> wrote:
>>>> Instead of coming up with more sophisticated names for states a VM BO
>>>> can be in, group them by the type of BO first and then by the state.
>>>>
>>>> So we end with BO type kernel, always_valid and individual and then
>>>> states
>>>> evicted, moved and idle.
>>>>
>>>> Not much functional change, except that evicted_user is moved back
>>>> together with the other BOs again which makes the handling in
>>>> amdgpu_vm_validate() a bit more complex.
>>>>
>>>> Also fixes a problem with user queues and amdgpu_vm_ready(). We didn't
>>>> considered the VM ready when user BOs were not ideally placed, harmless
>>>> performance impact for kernel queues but a complete show stopper for
>>>> userqueues.
>>>>
>>>> v2: fix a few typos in comments, rename the BO types to make them more
>>>>
>>>> descriptive, fix a couple of bugs found during testing
>>>>
>>>> v3: squashed together with revert to old status lock handling, looks
>>>>
>>>> like the first patch still had some bug which this one here should
>>>> fix.
>>>> Fix a missing lock around debugfs printing.
>>>>
>>>> Signed-off-by: Christian König <christian.koenig@amd.com>
>>>
>>> Hi Christian,
>>>
>>> This patch regresses GPU recovery.
>>> (Tested on SI, haven't had time to test other GPUs yet.)
>>>
>>> It hits the following warning:
>>> WARNING: drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c:635 at
>>> amdgpu_vm_pde_update And then this error:
>>> [drm:amdgpu_gem_va_update_vm [amdgpu]] *ERROR* Couldn't update BO_VA (-22)
>>>
>>> Please fix.
>>
>> Yeah that's a known issue. Fix is already pushed to amd-staging-drm-next.
>
> Hi Christian,
>
> Where is this fix? As far as I see "restructure VM state machine v2" is the top
> commit on amd-staging-drm-next right now.
The patch is called "drm/amdgpu: fix amdgpu_vm_bo_reset_state_machine" and should be on the mailing list.
Regards,
Christian.
>
> Timur
>
>
>
^ permalink raw reply [flat|nested] 30+ messages in thread
end of thread, other threads:[~2026-05-26 10:44 UTC | newest]
Thread overview: 30+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-04-27 18:27 [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab Christian König
2026-04-27 18:27 ` [PATCH 2/8] drm/amdgpu: remove deadlocks from amdgpu_userq_pre_reset Christian König
2026-04-28 6:51 ` Liang, Prike
2026-04-28 8:15 ` Khatri, Sunil
2026-04-27 18:27 ` [PATCH 3/8] drm/amdgpu: rework amdgpu_userq_signal_ioctl v2 Christian König
2026-04-28 8:49 ` Liang, Prike
2026-05-04 12:33 ` Christian König
2026-04-28 9:35 ` Khatri, Sunil
2026-05-04 12:37 ` Christian König
2026-04-27 18:27 ` [PATCH 4/8] drm/amdgpu: remove almost all calls to amdgpu_userq_detect_and_reset_queues Christian König
2026-04-28 9:37 ` Khatri, Sunil
2026-04-29 3:16 ` Liang, Prike
2026-04-27 18:27 ` [PATCH 5/8] drm/amdgpu: fix userq hang detection and reset Christian König
2026-04-29 8:30 ` Liang, Prike
2026-04-27 18:27 ` [PATCH 6/8] drm/amdgpu: rework userq reset work handling Christian König
2026-04-30 6:37 ` Liang, Prike
2026-04-27 18:27 ` [PATCH 7/8] drm/amdgpu: fix handling in amdgpu_userq_create Christian König
2026-04-28 11:26 ` Khatri, Sunil
2026-05-07 6:13 ` Liang, Prike
2026-04-27 18:27 ` [PATCH 8/8] drm/amdgpu: restructure VM state machine v2 Christian König
2026-04-28 12:20 ` Khatri, Sunil
2026-05-08 2:24 ` Liang, Prike
2026-05-08 8:21 ` Christian König
2026-05-25 23:29 ` Timur Kristóf
2026-05-26 8:51 ` Christian König
2026-05-26 10:11 ` Timur Kristóf
2026-05-26 10:44 ` Christian König
2026-04-28 6:15 ` [PATCH 1/8] drm/amdgpu: nuke amdgpu_userq_fence_slab Liang, Prike
2026-04-28 8:06 ` Christian König
2026-04-28 7:57 ` Khatri, Sunil
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.