* [PATCH 1/2] drm/amdkfd: Evict SVM BOs synchronously from TTM eviction
@ 2026-07-27 20:16 Philip Yang
2026-07-27 20:16 ` [PATCH 2/2] drm/amdkfd: Remove svm_bo eviction fence Philip Yang
0 siblings, 1 reply; 2+ messages in thread
From: Philip Yang @ 2026-07-27 20:16 UTC (permalink / raw)
To: amd-gfx, Felix.Kuehling, Christian.Koenig; +Cc: Philip Yang, Felix Kuehling
svm_range_evict_svm_bo_worker() migrated an SVM BO's pages back to system
memory from a work item that took mmap_read_lock. When an mmap writer was
pending, that read lock blocked behind the writer while the thread
allocating a new migration VRAM BO waited on this BO's eviction fence - a
circular wait that hung the SVM workers.
Evict the SVM BO synchronously from the TTM eviction path
(amdgpu_ttm_bo_eviction_valuable) instead of deferring to a work item.
The BO is already reserved and the lock order is mmap_lock -> BO
reservation, so only trylock the owning process's mmap lock; on
contention return -EBUSY so TTM skips this BO. This removes the eviction
work item and the enable_signaling path, so no worker can block on
mmap_read_lock.
The SVM BO uses AMDGPU_GEM_CREATE_DISCARDABLE, so ttm_bo_evict takes the
pipeline_gutting path and skips allocating a system memory placement.
That would be wasted work, since svm_migrate_vram_to_ram allocates the
system pages and copies the data back itself.
Eviction now migrates ranges directly, so it must serialize with the
owning process: it trylocks migrate_mutex under svm_bo->list_lock before
unlinking the range, and svm_range_free() unlinks the range then waits on
migrate_mutex, so a concurrent eviction cannot free a range under it.
Drop the mm reference with mmput_async so exit_mmap() does not run under
the BO reservation.
Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <felix.kuehling@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 22 +++++
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c | 3 -
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 16 ++++
drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 88 ++++++++++++-------
drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 10 +--
6 files changed, 100 insertions(+), 40 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 121282dd30c1..be764b6802b5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -39,6 +39,7 @@
#if IS_ENABLED(CONFIG_HSA_AMD)
#include "kfd_priv.h"
#endif
+#include "kfd_svm.h"
/* Total memory size in system memory and all GPU VRAM. Used to
* estimate worst case amount of memory to reserve for page tables
@@ -972,3 +973,24 @@ int amdgpu_amdkfd_reset_mes_queue(struct amdgpu_device *adev,
return kgd2kfd_reset_mes_queue(adev->kfd.dev, node_id, queue_type,
pipe, queue, db);
}
+
+int amdgpu_amdkfd_evict_svm_bo(struct amdgpu_bo *bo)
+{
+ struct dma_resv_iter cursor;
+ struct dma_fence *fence;
+ int r = 0;
+
+ dma_resv_iter_begin(&cursor, bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP);
+ dma_resv_for_each_fence_unlocked(&cursor, fence) {
+ struct amdgpu_amdkfd_fence *f = to_amdgpu_amdkfd_fence(fence);
+
+ if (f && f->svm_bo) {
+ r = svm_range_evict_svm_bo(f->svm_bo);
+ if (r)
+ break;
+ }
+ }
+ dma_resv_iter_end(&cursor);
+
+ return r;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 338412a750ed..b4840ee36f2b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -286,6 +286,7 @@ int amdgpu_amdkfd_reset_mes_queue(struct amdgpu_device *adev,
int queue_type,
int pipe, int queue,
unsigned int db);
+int amdgpu_amdkfd_evict_svm_bo(struct amdgpu_bo *bo);
/* Read user wptr from a specified user address space with page fault
* disabled. The memory must be pinned and mapped to the hardware when
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c
index b0299d861903..553d26c2744e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c
@@ -135,9 +135,6 @@ static bool amdkfd_fence_enable_signaling(struct dma_fence *f)
if (!fence->svm_bo) {
if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, fence->context_id, f))
return true;
- } else {
- if (!svm_range_schedule_evict_svm_bo(fence))
- return true;
}
return false;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 03c1e5e3580c..746290216aae 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1488,6 +1488,7 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
const struct ttm_place *place)
{
struct dma_resv_iter resv_cursor;
+ struct amdgpu_bo *abo;
struct dma_fence *f;
if (!amdgpu_bo_is_amdgpu_bo(bo))
@@ -1497,6 +1498,21 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
if (bo->resource->mem_type == TTM_PL_SYSTEM)
return true;
+ abo = ttm_to_amdgpu_bo(bo);
+ if (abo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) {
+ /*
+ * SVM BOs are migrated to system memory synchronously in this
+ * TTM eviction context. The migration needs the owning
+ * process's mmap lock, but the normal lock order is
+ * mmap_lock -> BO reservation and the BO is already reserved
+ * here. svm_range_evict_svm_bo() only trylocks the mmap lock;
+ * if the eviction fails for any reason, we return false so TTM
+ * skips this BO instead of risking a deadlock.
+ */
+ if (amdgpu_amdkfd_evict_svm_bo(abo) < 0)
+ return false;
+ }
+
if (bo->type == ttm_bo_type_kernel &&
!amdgpu_vm_evictable(ttm_to_amdgpu_bo(bo)))
return false;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 30ad10bbd47e..4c6700c6e88d 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -69,7 +69,6 @@ struct criu_svm_metadata {
struct kfd_criu_svm_range_priv_data data;
};
-static void svm_range_evict_svm_bo_worker(struct work_struct *work);
static bool
svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni,
const struct mmu_notifier_range *range,
@@ -97,7 +96,7 @@ static void svm_range_unlink(struct svm_range *prange)
if (prange->svm_bo) {
spin_lock(&prange->svm_bo->list_lock);
- list_del(&prange->svm_bo_list);
+ list_del_init(&prange->svm_bo_list);
spin_unlock(&prange->svm_bo->list_lock);
}
@@ -286,6 +285,17 @@ static void svm_range_free(struct svm_range *prange, bool do_unmap)
pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, prange,
prange->start, prange->last);
+ /* Unlink from range_list; no-op if already unlinked. */
+ if (prange->svm_bo) {
+ spin_lock(&prange->svm_bo->list_lock);
+ list_del_init(&prange->svm_bo_list);
+ spin_unlock(&prange->svm_bo->list_lock);
+ }
+
+ /* Wait for any in-flight eviction of this range to finish. */
+ mutex_lock(&prange->migrate_mutex);
+ mutex_unlock(&prange->migrate_mutex);
+
svm_range_vram_node_free(prange);
if (do_unmap)
svm_range_dma_unmap(prange);
@@ -588,7 +598,6 @@ svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
mm,
svm_bo, p->context_id);
mmput(mm);
- INIT_WORK(&svm_bo->eviction_work, svm_range_evict_svm_bo_worker);
svm_bo->evicting = 0;
memset(&bp, 0, sizeof(bp));
bp.size = prange->npages * PAGE_SIZE;
@@ -3631,39 +3640,36 @@ svm_range_trigger_migration(struct mm_struct *mm, struct svm_range *prange,
return 0;
}
-int svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence *fence)
-{
- /* Dereferencing fence->svm_bo is safe here because the fence hasn't
- * signaled yet and we're under the protection of the fence->lock.
- * After the fence is signaled in svm_range_bo_release, we cannot get
- * here any more.
- *
- * Reference is dropped in svm_range_evict_svm_bo_worker.
- */
- if (svm_bo_ref_unless_zero(fence->svm_bo)) {
- WRITE_ONCE(fence->svm_bo->evicting, 1);
- schedule_work(&fence->svm_bo->eviction_work);
- }
-
- return 0;
-}
-
-static void svm_range_evict_svm_bo_worker(struct work_struct *work)
+int svm_range_evict_svm_bo(struct svm_range_bo *svm_bo)
{
- struct svm_range_bo *svm_bo;
struct mm_struct *mm;
int r = 0;
- svm_bo = container_of(work, struct svm_range_bo, eviction_work);
+ if (!svm_bo_ref_unless_zero(svm_bo))
+ return 0;
- if (mmget_not_zero(svm_bo->eviction_fence->mm)) {
- mm = svm_bo->eviction_fence->mm;
- } else {
+ if (!mmget_not_zero(svm_bo->eviction_fence->mm)) {
svm_range_bo_unref(svm_bo);
- return;
+ return 0;
}
+ mm = svm_bo->eviction_fence->mm;
+
+ /*
+ * Called with the BO reserved; lock order is mmap_lock -> BO
+ * reservation. Only trylock mmap to invert that order safely: a
+ * trylock never blocks, so it cannot deadlock against the reservation
+ * and lockdep records no reverse dependency. On contention return
+ * -EBUSY so TTM skips this BO.
+ */
+ if (!mmap_read_trylock(mm)) {
+ pr_debug("skip eviction, contended to take mmap_read lock\n");
+ mmput_async(mm);
+ svm_range_bo_unref(svm_bo);
+ return -EBUSY;
+ }
+
+ WRITE_ONCE(svm_bo->evicting, 1);
- mmap_read_lock(mm);
spin_lock(&svm_bo->list_lock);
while (!list_empty(&svm_bo->range_list) && !r) {
struct svm_range *prange =
@@ -3671,13 +3677,24 @@ static void svm_range_evict_svm_bo_worker(struct work_struct *work)
struct svm_range, svm_bo_list);
int retries = 3;
+ /*
+ * Trylock migrate_mutex under list_lock, before unlinking the
+ * range, so svm_range_free() cannot free it under us. On
+ * contention the owner is migrating this range; skip the BO.
+ */
+ if (!mutex_trylock(&prange->migrate_mutex)) {
+ pr_debug("skip eviction, contended migrate_mutex\n");
+ /* Clear evicting so the BO keeps being reused. */
+ WRITE_ONCE(svm_bo->evicting, 0);
+ r = -EBUSY;
+ break;
+ }
list_del_init(&prange->svm_bo_list);
spin_unlock(&svm_bo->list_lock);
pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms,
prange->start, prange->last);
- mutex_lock(&prange->migrate_mutex);
do {
/* migrate all vram pages in this prange to sys ram
* after that prange->actual_loc should be zero
@@ -3701,15 +3718,24 @@ static void svm_range_evict_svm_bo_worker(struct work_struct *work)
}
spin_unlock(&svm_bo->list_lock);
mmap_read_unlock(mm);
- mmput(mm);
+ /* Defer mmput: exit_mmap() must not run under the BO reservation. */
+ mmput_async(mm);
- dma_fence_signal(&svm_bo->eviction_fence->base);
+ /*
+ * Only signal the eviction fence once the ranges have been processed.
+ * On -EBUSY we bailed out without migrating; leave the BO in VRAM and
+ * let TTM retry later.
+ */
+ if (r != -EBUSY)
+ dma_fence_signal(&svm_bo->eviction_fence->base);
/* This is the last reference to svm_bo, after svm_range_vram_node_free
* has been called in svm_migrate_vram_to_ram
*/
WARN_ONCE(!r && kref_read(&svm_bo->kref) != 1, "This was not the last reference\n");
svm_range_bo_unref(svm_bo);
+
+ return r;
}
static int
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
index a63dfc95b602..4232c47422e6 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
@@ -44,7 +44,6 @@ struct svm_range_bo {
struct list_head range_list; /* all svm ranges shared this bo */
spinlock_t list_lock;
struct amdgpu_amdkfd_fence *eviction_fence;
- struct work_struct eviction_work;
uint32_t evicting;
struct work_struct release_work;
struct kfd_node *node;
@@ -175,7 +174,8 @@ void svm_range_vram_node_free(struct svm_range *prange);
int svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
uint32_t vmid, uint32_t node_id, uint64_t addr, uint64_t ts,
bool write_fault);
-int svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence *fence);
+int svm_range_evict_svm_bo(struct svm_range_bo *svm_bo);
+
void svm_range_add_list_work(struct svm_range_list *svms,
struct svm_range *prange, struct mm_struct *mm,
enum svm_work_list_ops op);
@@ -229,11 +229,9 @@ static inline int svm_range_restore_pages(struct amdgpu_device *adev,
return -EFAULT;
}
-static inline int svm_range_schedule_evict_svm_bo(
- struct amdgpu_amdkfd_fence *fence)
+static inline int svm_range_evict_svm_bo(struct svm_range_bo *svm_bo)
{
- WARN_ONCE(1, "SVM eviction fence triggered, but SVM is disabled");
- return -EINVAL;
+ return 0;
}
static inline void svm_range_get_info(struct kfd_process *p,
--
2.50.1
^ permalink raw reply related [flat|nested] 2+ messages in thread
* [PATCH 2/2] drm/amdkfd: Remove svm_bo eviction fence
2026-07-27 20:16 [PATCH 1/2] drm/amdkfd: Evict SVM BOs synchronously from TTM eviction Philip Yang
@ 2026-07-27 20:16 ` Philip Yang
0 siblings, 0 replies; 2+ messages in thread
From: Philip Yang @ 2026-07-27 20:16 UTC (permalink / raw)
To: amd-gfx, Felix.Kuehling, Christian.Koenig; +Cc: Philip Yang, Felix Kuehling
SVM BOs are now migrated back to system memory synchronously from the TTM
eviction path (svm_range_evict_svm_bo), so the per-svm_bo eviction fence
is no longer used.
Remove the eviction fence from svm_range_bo, drop the
amdgpu_amdkfd_fence->svm_bo back pointer and the
amdgpu_amdkfd_evict_svm_bo() helper, and stop special-casing svm_bo
fences in the KFD fence enable_signaling and check_mm paths. Embed struct
amdgpu_bo directly in svm_range_bo with a dedicated svm_range_bo_destroy()
callback, and keep the owning mm via mmgrab()/mmdrop() instead of through
the fence.
Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <felix.kuehling@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 21 ----
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 3 -
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c | 15 +--
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 4 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 4 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 6 +-
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 4 +-
drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 109 +++++++-----------
drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 13 ++-
9 files changed, 64 insertions(+), 115 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index be764b6802b5..816d8817f0b2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -973,24 +973,3 @@ int amdgpu_amdkfd_reset_mes_queue(struct amdgpu_device *adev,
return kgd2kfd_reset_mes_queue(adev->kfd.dev, node_id, queue_type,
pipe, queue, db);
}
-
-int amdgpu_amdkfd_evict_svm_bo(struct amdgpu_bo *bo)
-{
- struct dma_resv_iter cursor;
- struct dma_fence *fence;
- int r = 0;
-
- dma_resv_iter_begin(&cursor, bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP);
- dma_resv_for_each_fence_unlocked(&cursor, fence) {
- struct amdgpu_amdkfd_fence *f = to_amdgpu_amdkfd_fence(fence);
-
- if (f && f->svm_bo) {
- r = svm_range_evict_svm_bo(f->svm_bo);
- if (r)
- break;
- }
- }
- dma_resv_iter_end(&cursor);
-
- return r;
-}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index b4840ee36f2b..1b7dc0d3963b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -99,7 +99,6 @@ struct amdgpu_amdkfd_fence {
struct mm_struct *mm;
spinlock_t lock;
char timeline_name[TASK_COMM_LEN];
- struct svm_range_bo *svm_bo;
uint16_t context_id;
};
@@ -194,7 +193,6 @@ int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev,
struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
struct mm_struct *mm,
- struct svm_range_bo *svm_bo,
u16 context_id);
int amdgpu_amdkfd_drm_client_create(struct amdgpu_device *adev);
@@ -286,7 +284,6 @@ int amdgpu_amdkfd_reset_mes_queue(struct amdgpu_device *adev,
int queue_type,
int pipe, int queue,
unsigned int db);
-int amdgpu_amdkfd_evict_svm_bo(struct amdgpu_bo *bo);
/* Read user wptr from a specified user address space with page fault
* disabled. The memory must be pinned and mapped to the hardware when
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c
index 553d26c2744e..9b10d015671c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c
@@ -62,7 +62,6 @@ static atomic_t fence_seq = ATOMIC_INIT(0);
struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
struct mm_struct *mm,
- struct svm_range_bo *svm_bo,
u16 context_id)
{
struct amdgpu_amdkfd_fence *fence;
@@ -76,7 +75,6 @@ struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
fence->mm = mm;
get_task_comm(fence->timeline_name, current);
spin_lock_init(&fence->lock);
- fence->svm_bo = svm_bo;
fence->context_id = context_id;
dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock,
context, atomic_inc_return(&fence_seq));
@@ -128,14 +126,8 @@ static bool amdkfd_fence_enable_signaling(struct dma_fence *f)
if (dma_fence_is_signaled(f))
return true;
- /* if fence->svm_bo is NULL, means this fence is created through
- * init_kfd_vm() or amdgpu_amdkfd_gpuvm_restore_process_bos().
- * Therefore, this fence is amdgpu_amdkfd_fence->eviction_fence.
- */
- if (!fence->svm_bo) {
- if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, fence->context_id, f))
- return true;
- }
+ if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, fence->context_id, f))
+ return true;
return false;
}
@@ -169,7 +161,6 @@ static void amdkfd_fence_release(struct dma_fence *f)
*
* Check if @mm is same as that of the fence @f, if same return TRUE else
* return FALSE.
- * For svm bo, which support vram overcommitment, always return FALSE.
*/
bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
{
@@ -177,7 +168,7 @@ bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
if (!fence)
return false;
- else if (fence->mm == mm && !fence->svm_bo)
+ else if (fence->mm == mm)
return true;
return false;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 451d7f7ef5c1..1e71829e0fc6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1423,7 +1423,7 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
info->eviction_fence =
amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
current->mm,
- NULL, process->context_id);
+ process->context_id);
if (!info->eviction_fence) {
pr_err("Failed to create eviction fence\n");
ret = -ENOMEM;
@@ -3093,7 +3093,7 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu *
amdgpu_amdkfd_fence_create(
process_info->eviction_fence->base.context,
process_info->eviction_fence->mm,
- NULL, process_info->context_id);
+ process_info->context_id);
if (!new_fence) {
pr_err("Failed to create eviction fence\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index d4a9d5e8fb42..5d9d137209b6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -43,6 +43,7 @@
#include "amdgpu_vram_mgr.h"
#include "amdgpu_vm.h"
#include "amdgpu_dma_buf.h"
+#include "kfd_svm.h"
/**
* DOC: amdgpu_object
@@ -93,7 +94,8 @@ static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo)
bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
{
if (bo->destroy == &amdgpu_bo_destroy ||
- bo->destroy == &amdgpu_bo_user_destroy)
+ bo->destroy == &amdgpu_bo_user_destroy ||
+ bo->destroy == &svm_range_bo_destroy)
return true;
return false;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 746290216aae..0e7e586b217a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -60,6 +60,7 @@
#include "amdgpu_atomfirmware.h"
#include "amdgpu_res_cursor.h"
#include "bif/bif_4_1_d.h"
+#include "kfd_svm.h"
MODULE_IMPORT_NS("DMA_BUF");
@@ -1499,7 +1500,8 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
return true;
abo = ttm_to_amdgpu_bo(bo);
- if (abo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) {
+ if ((abo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) &&
+ bo->destroy == &svm_range_bo_destroy) {
/*
* SVM BOs are migrated to system memory synchronously in this
* TTM eviction context. The migration needs the owning
@@ -1509,7 +1511,7 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
* if the eviction fails for any reason, we return false so TTM
* skips this BO instead of risking a deadlock.
*/
- if (amdgpu_amdkfd_evict_svm_bo(abo) < 0)
+ if (svm_range_evict_svm_bo(abo) < 0)
return false;
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index ed3649a81332..f5af1dd3b70e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -954,12 +954,12 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
pr_debug("failed get device page at addr 0x%lx\n", addr);
return VM_FAULT_SIGBUS;
}
- if (!mmget_not_zero(svm_bo->eviction_fence->mm)) {
+ if (!mmget_not_zero(svm_bo->mm)) {
pr_debug("addr 0x%lx of process mm is destroyed\n", addr);
return VM_FAULT_SIGBUS;
}
- mm = svm_bo->eviction_fence->mm;
+ mm = svm_bo->mm;
if (mm != vmf->vma->vm_mm)
pr_debug("addr 0x%lx is COW mapping in child process\n", addr);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 4c6700c6e88d..18abef8918ce 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -390,8 +390,10 @@ static bool svm_bo_ref_unless_zero(struct svm_range_bo *svm_bo)
static void svm_range_bo_release(struct kref *kref)
{
struct svm_range_bo *svm_bo;
+ struct amdgpu_bo *bo;
svm_bo = container_of(kref, struct svm_range_bo, kref);
+ bo = &svm_bo->bo;
pr_debug("svm_bo 0x%p\n", svm_bo);
spin_lock(&svm_bo->list_lock);
@@ -417,12 +419,12 @@ static void svm_range_bo_release(struct kref *kref)
}
spin_unlock(&svm_bo->list_lock);
- if (mmget_not_zero(svm_bo->eviction_fence->mm)) {
+ if (mmget_not_zero(svm_bo->mm)) {
struct kfd_process_device *pdd;
struct kfd_process *p;
struct mm_struct *mm;
- mm = svm_bo->eviction_fence->mm;
+ mm = svm_bo->mm;
/*
* The forked child process takes svm_bo device pages ref, svm_bo could be
* released after parent process is gone.
@@ -431,18 +433,13 @@ static void svm_range_bo_release(struct kref *kref)
if (p) {
pdd = kfd_get_process_device_data(svm_bo->node, p);
if (pdd)
- atomic64_sub(amdgpu_bo_size(svm_bo->bo), &pdd->vram_usage);
+ atomic64_sub(amdgpu_bo_size(bo), &pdd->vram_usage);
kfd_unref_process(p);
}
mmput(mm);
}
- if (!dma_fence_is_signaled(&svm_bo->eviction_fence->base))
- /* We're not in the eviction worker. Signal the fence. */
- dma_fence_signal(&svm_bo->eviction_fence->base);
- dma_fence_put(&svm_bo->eviction_fence->base);
- amdgpu_bo_unref(&svm_bo->bo);
- kfree(svm_bo);
+ amdgpu_bo_unref(&bo);
}
static void svm_range_bo_wq_release(struct work_struct *work)
@@ -504,20 +501,11 @@ svm_range_validate_svm_bo(struct kfd_node *node, struct svm_range *prange)
return false;
}
if (READ_ONCE(prange->svm_bo->evicting)) {
- struct dma_fence *f;
- struct svm_range_bo *svm_bo;
/* The BO is getting evicted,
* we need to get a new one
*/
mutex_unlock(&prange->lock);
- svm_bo = prange->svm_bo;
- f = dma_fence_get(&svm_bo->eviction_fence->base);
svm_range_bo_unref(prange->svm_bo);
- /* wait for the fence to avoid long spin-loop
- * at list_empty_careful
- */
- dma_fence_wait(f, false);
- dma_fence_put(f);
} else {
/* The BO was still around and we got
* a new reference to it
@@ -526,7 +514,7 @@ svm_range_validate_svm_bo(struct kfd_node *node, struct svm_range *prange)
pr_debug("reuse old bo svms 0x%p [0x%lx 0x%lx]\n",
prange->svms, prange->start, prange->last);
- prange->ttm_res = prange->svm_bo->bo->tbo.resource;
+ prange->ttm_res = prange->svm_bo->bo.tbo.resource;
return true;
}
@@ -545,19 +533,22 @@ svm_range_validate_svm_bo(struct kfd_node *node, struct svm_range *prange)
return false;
}
-static struct svm_range_bo *svm_range_bo_new(void)
-{
- struct svm_range_bo *svm_bo;
+#define to_svm_range_bo(bo) container_of((bo), struct svm_range_bo, bo)
- svm_bo = kzalloc_obj(*svm_bo);
- if (!svm_bo)
- return NULL;
-
- kref_init(&svm_bo->kref);
- INIT_LIST_HEAD(&svm_bo->range_list);
- spin_lock_init(&svm_bo->list_lock);
+void svm_range_bo_destroy(struct ttm_buffer_object *tbo)
+{
+ struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
+ struct svm_range_bo *svm_bo = to_svm_range_bo(bo);
- return svm_bo;
+ drm_gem_object_release(&bo->tbo.base);
+ /*
+ * svm_bo->mm is only set once the BO is fully created. If
+ * ttm_bo_init_reserved() fails (e.g. no VRAM could be evicted), it
+ * calls this destroy callback with mm still NULL, so guard the drop.
+ */
+ if (svm_bo->mm)
+ mmdrop(svm_bo->mm);
+ kvfree(svm_bo);
}
int
@@ -567,7 +558,6 @@ svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
struct kfd_process_device *pdd;
struct amdgpu_bo_param bp;
struct svm_range_bo *svm_bo;
- struct amdgpu_bo_user *ubo;
struct amdgpu_bo *bo;
struct kfd_process *p;
struct mm_struct *mm;
@@ -581,26 +571,16 @@ svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
if (svm_range_validate_svm_bo(node, prange))
return 0;
- svm_bo = svm_range_bo_new();
- if (!svm_bo) {
- pr_debug("failed to alloc svm bo\n");
- return -ENOMEM;
- }
mm = get_task_mm(p->lead_thread);
if (!mm) {
pr_debug("failed to get mm\n");
- kfree(svm_bo);
return -ESRCH;
}
- svm_bo->node = node;
- svm_bo->eviction_fence =
- amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
- mm,
- svm_bo, p->context_id);
- mmput(mm);
- svm_bo->evicting = 0;
+
memset(&bp, 0, sizeof(bp));
bp.size = prange->npages * PAGE_SIZE;
+ bp.bo_ptr_size = sizeof(struct svm_range_bo);
+ bp.destroy = svm_range_bo_destroy;
bp.byte_align = PAGE_SIZE;
bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
@@ -611,12 +591,23 @@ svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
if (node->xcp)
bp.xcp_id_plus1 = node->xcp->id + 1;
- r = amdgpu_bo_create_user(node->adev, &bp, &ubo);
+ r = amdgpu_bo_create(node->adev, &bp, &bo);
if (r) {
pr_debug("failed %d to create bo\n", r);
+ mmput(mm);
goto create_bo_failed;
}
- bo = &ubo->bo;
+
+ svm_bo = to_svm_range_bo(bo);
+ svm_bo->evicting = 0;
+ kref_init(&svm_bo->kref);
+ INIT_LIST_HEAD(&svm_bo->range_list);
+ spin_lock_init(&svm_bo->list_lock);
+
+ svm_bo->node = node;
+ svm_bo->mm = mm;
+ mmgrab(svm_bo->mm);
+ mmput(mm);
pr_debug("alloc bo at offset 0x%lx size 0x%lx on partition %d\n",
bo->tbo.resource->start << PAGE_SHIFT, bp.size,
@@ -637,16 +628,8 @@ svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
}
}
- r = dma_resv_reserve_fences(bo->tbo.base.resv, TTM_NUM_MOVE_FENCES);
- if (r) {
- amdgpu_bo_unreserve(bo);
- goto reserve_bo_failed;
- }
- amdgpu_bo_fence(bo, &svm_bo->eviction_fence->base, true);
-
amdgpu_bo_unreserve(bo);
- svm_bo->bo = bo;
prange->svm_bo = svm_bo;
prange->ttm_res = bo->tbo.resource;
prange->offset = 0;
@@ -664,9 +647,6 @@ svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
reserve_bo_failed:
amdgpu_bo_unref(&bo);
create_bo_failed:
- dma_fence_put(&svm_bo->eviction_fence->base);
- kfree(svm_bo);
- prange->ttm_res = NULL;
return r;
}
@@ -3640,19 +3620,20 @@ svm_range_trigger_migration(struct mm_struct *mm, struct svm_range *prange,
return 0;
}
-int svm_range_evict_svm_bo(struct svm_range_bo *svm_bo)
+int svm_range_evict_svm_bo(struct amdgpu_bo *bo)
{
+ struct svm_range_bo *svm_bo = to_svm_range_bo(bo);
struct mm_struct *mm;
int r = 0;
if (!svm_bo_ref_unless_zero(svm_bo))
return 0;
- if (!mmget_not_zero(svm_bo->eviction_fence->mm)) {
+ if (!mmget_not_zero(svm_bo->mm)) {
svm_range_bo_unref(svm_bo);
return 0;
}
- mm = svm_bo->eviction_fence->mm;
+ mm = svm_bo->mm;
/*
* Called with the BO reserved; lock order is mmap_lock -> BO
@@ -3721,14 +3702,6 @@ int svm_range_evict_svm_bo(struct svm_range_bo *svm_bo)
/* Defer mmput: exit_mmap() must not run under the BO reservation. */
mmput_async(mm);
- /*
- * Only signal the eviction fence once the ranges have been processed.
- * On -EBUSY we bailed out without migrating; leave the BO in VRAM and
- * let TTM retry later.
- */
- if (r != -EBUSY)
- dma_fence_signal(&svm_bo->eviction_fence->base);
-
/* This is the last reference to svm_bo, after svm_range_vram_node_free
* has been called in svm_migrate_vram_to_ram
*/
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
index 4232c47422e6..284c4cd5aa0f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
@@ -39,11 +39,11 @@
((adev)->hive ? (void *)(adev)->hive : (void *)(adev))
struct svm_range_bo {
- struct amdgpu_bo *bo;
+ struct amdgpu_bo bo;
struct kref kref;
struct list_head range_list; /* all svm ranges shared this bo */
spinlock_t list_lock;
- struct amdgpu_amdkfd_fence *eviction_fence;
+ struct mm_struct *mm;
uint32_t evicting;
struct work_struct release_work;
struct kfd_node *node;
@@ -168,13 +168,14 @@ struct svm_range *svm_range_from_addr(struct svm_range_list *svms,
struct svm_range **parent);
struct kfd_node *svm_range_get_node_by_id(struct svm_range *prange,
uint32_t gpu_id);
+void svm_range_bo_destroy(struct ttm_buffer_object *tbo);
int svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
bool clear);
void svm_range_vram_node_free(struct svm_range *prange);
int svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
uint32_t vmid, uint32_t node_id, uint64_t addr, uint64_t ts,
bool write_fault);
-int svm_range_evict_svm_bo(struct svm_range_bo *svm_bo);
+int svm_range_evict_svm_bo(struct amdgpu_bo *bo);
void svm_range_add_list_work(struct svm_range_list *svms,
struct svm_range *prange, struct mm_struct *mm,
@@ -229,7 +230,11 @@ static inline int svm_range_restore_pages(struct amdgpu_device *adev,
return -EFAULT;
}
-static inline int svm_range_evict_svm_bo(struct svm_range_bo *svm_bo)
+static inline void svm_range_bo_destroy(struct ttm_buffer_object *tbo)
+{
+}
+
+static inline int svm_range_evict_svm_bo(struct amdgpu_bo *bo)
{
return 0;
}
--
2.50.1
^ permalink raw reply related [flat|nested] 2+ messages in thread
end of thread, other threads:[~2026-07-27 20:17 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-07-27 20:16 [PATCH 1/2] drm/amdkfd: Evict SVM BOs synchronously from TTM eviction Philip Yang
2026-07-27 20:16 ` [PATCH 2/2] drm/amdkfd: Remove svm_bo eviction fence Philip Yang
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.