* [Intel-xe] [CI 02/10] drm/gpuvm: add common dma-resv per struct drm_gpuvm
2023-10-25 12:55 [Intel-xe] [CI 01/10] drm/gpuvm: convert WARN() to drm_WARN() variants Thomas Hellström
@ 2023-10-25 12:55 ` Thomas Hellström
2023-10-25 12:55 ` [Intel-xe] [CI 03/10] drm/xe: Adjust to "drm/gpuvm: add common dma-resv per struct drm_gpuvm" Thomas Hellström
` (8 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Thomas Hellström @ 2023-10-25 12:55 UTC (permalink / raw)
To: intel-xe
From: Danilo Krummrich <dakr@redhat.com>
Provide a common dma-resv for GEM objects not being used outside of this
GPU-VM. This is used in a subsequent patch to generalize dma-resv,
external and evicted object handling and GEM validation.
Reviewed-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
Signed-off-by: Danilo Krummrich <dakr@redhat.com>
---
drivers/gpu/drm/drm_gpuvm.c | 53 ++++++++++++++++++++++++++
drivers/gpu/drm/nouveau/nouveau_uvmm.c | 13 ++++++-
include/drm/drm_gpuvm.h | 33 ++++++++++++++++
3 files changed, 97 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c
index b378554b81ac..d4907aace831 100644
--- a/drivers/gpu/drm/drm_gpuvm.c
+++ b/drivers/gpu/drm/drm_gpuvm.c
@@ -61,6 +61,15 @@
* contained within struct drm_gpuva already. Hence, for inserting &drm_gpuva
* entries from within dma-fence signalling critical sections it is enough to
* pre-allocate the &drm_gpuva structures.
+ *
+ * &drm_gem_objects which are private to a single VM can share a common
+ * &dma_resv in order to improve locking efficiency (e.g. with &drm_exec).
+ * For this purpose drivers must pass a &drm_gem_object to drm_gpuvm_init(), in
+ * the following called 'resv object', which serves as the container of the
+ * GPUVM's shared &dma_resv. This resv object can be a driver specific
+ * &drm_gem_object, such as the &drm_gem_object containing the root page table,
+ * but it can also be a 'dummy' object, which can be allocated with
+ * drm_gpuvm_resv_object_alloc().
*/
/**
@@ -652,11 +661,49 @@ drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm,
!drm_gpuvm_in_kernel_node(gpuvm, addr, range);
}
+static void
+drm_gpuvm_gem_object_free(struct drm_gem_object *obj)
+{
+ drm_gem_object_release(obj);
+ kfree(obj);
+}
+
+static const struct drm_gem_object_funcs drm_gpuvm_object_funcs = {
+ .free = drm_gpuvm_gem_object_free,
+};
+
+/**
+ * drm_gpuvm_resv_object_alloc() - allocate a dummy &drm_gem_object
+ * @drm: the drivers &drm_device
+ *
+ * Allocates a dummy &drm_gem_object which can be passed to drm_gpuvm_init() in
+ * order to serve as root GEM object providing the &drm_resv shared across
+ * &drm_gem_objects local to a single GPUVM.
+ *
+ * Returns: the &drm_gem_object on success, NULL on failure
+ */
+struct drm_gem_object *
+drm_gpuvm_resv_object_alloc(struct drm_device *drm)
+{
+ struct drm_gem_object *obj;
+
+ obj = kzalloc(sizeof(*obj), GFP_KERNEL);
+ if (!obj)
+ return NULL;
+
+ obj->funcs = &drm_gpuvm_object_funcs;
+ drm_gem_private_object_init(drm, obj, 0);
+
+ return obj;
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_resv_object_alloc);
+
/**
* drm_gpuvm_init() - initialize a &drm_gpuvm
* @gpuvm: pointer to the &drm_gpuvm to initialize
* @name: the name of the GPU VA space
* @drm: the &drm_device this VM resides in
+ * @r_obj: the resv &drm_gem_object providing the GPUVM's common &dma_resv
* @start_offset: the start offset of the GPU VA space
* @range: the size of the GPU VA space
* @reserve_offset: the start of the kernel reserved GPU VA area
@@ -671,6 +718,7 @@ drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm,
void
drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
struct drm_device *drm,
+ struct drm_gem_object *r_obj,
u64 start_offset, u64 range,
u64 reserve_offset, u64 reserve_range,
const struct drm_gpuvm_ops *ops)
@@ -681,6 +729,9 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
gpuvm->name = name ? name : "unknown";
gpuvm->ops = ops;
gpuvm->drm = drm;
+ gpuvm->r_obj = r_obj;
+
+ drm_gem_object_get(r_obj);
drm_gpuvm_check_overflow(gpuvm, start_offset, range);
gpuvm->mm_start = start_offset;
@@ -715,6 +766,8 @@ drm_gpuvm_destroy(struct drm_gpuvm *gpuvm)
drm_WARN(gpuvm->drm, !RB_EMPTY_ROOT(&gpuvm->rb.tree.rb_root),
"GPUVA tree is not empty, potentially leaking memory.\n");
+
+ drm_gem_object_put(gpuvm->r_obj);
}
EXPORT_SYMBOL_GPL(drm_gpuvm_destroy);
diff --git a/drivers/gpu/drm/nouveau/nouveau_uvmm.c b/drivers/gpu/drm/nouveau/nouveau_uvmm.c
index aaf5d28bd587..b4e7d662961a 100644
--- a/drivers/gpu/drm/nouveau/nouveau_uvmm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_uvmm.c
@@ -1809,8 +1809,9 @@ nouveau_uvmm_init(struct nouveau_uvmm *uvmm, struct nouveau_cli *cli,
u64 kernel_managed_addr, u64 kernel_managed_size)
{
struct drm_device *drm = cli->drm->dev;
- int ret;
+ struct drm_gem_object *r_obj;
u64 kernel_managed_end = kernel_managed_addr + kernel_managed_size;
+ int ret;
mutex_init(&uvmm->mutex);
dma_resv_init(&uvmm->resv);
@@ -1834,14 +1835,22 @@ nouveau_uvmm_init(struct nouveau_uvmm *uvmm, struct nouveau_cli *cli,
goto out_unlock;
}
+ r_obj = drm_gpuvm_resv_object_alloc(drm);
+ if (!r_obj) {
+ ret = -ENOMEM;
+ goto out_unlock;
+ }
+
uvmm->kernel_managed_addr = kernel_managed_addr;
uvmm->kernel_managed_size = kernel_managed_size;
- drm_gpuvm_init(&uvmm->base, cli->name, drm,
+ drm_gpuvm_init(&uvmm->base, cli->name, drm, r_obj,
NOUVEAU_VA_SPACE_START,
NOUVEAU_VA_SPACE_END,
kernel_managed_addr, kernel_managed_size,
NULL);
+ /* GPUVM takes care from here on. */
+ drm_gem_object_put(r_obj);
ret = nvif_vmm_ctor(&cli->mmu, "uvmm",
cli->vmm.vmm.object.oclass, RAW,
diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h
index baa1a96c038c..54dc195a68ea 100644
--- a/include/drm/drm_gpuvm.h
+++ b/include/drm/drm_gpuvm.h
@@ -244,10 +244,16 @@ struct drm_gpuvm {
* @ops: &drm_gpuvm_ops providing the split/merge steps to drivers
*/
const struct drm_gpuvm_ops *ops;
+
+ /**
+ * @r_obj: Resv GEM object; representing the GPUVM's common &dma_resv.
+ */
+ struct drm_gem_object *r_obj;
};
void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
struct drm_device *drm,
+ struct drm_gem_object *r_obj,
u64 start_offset, u64 range,
u64 reserve_offset, u64 reserve_range,
const struct drm_gpuvm_ops *ops);
@@ -255,6 +261,33 @@ void drm_gpuvm_destroy(struct drm_gpuvm *gpuvm);
bool drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range);
+struct drm_gem_object *
+drm_gpuvm_resv_object_alloc(struct drm_device *drm);
+
+/**
+ * drm_gpuvm_resv() - returns the &drm_gpuvm's &dma_resv
+ * @gpuvm__: the &drm_gpuvm
+ *
+ * Returns: a pointer to the &drm_gpuvm's shared &dma_resv
+ */
+#define drm_gpuvm_resv(gpuvm__) ((gpuvm__)->r_obj->resv)
+
+/**
+ * drm_gpuvm_resv_obj() - returns the &drm_gem_object holding the &drm_gpuvm's
+ * &dma_resv
+ * @gpuvm__: the &drm_gpuvm
+ *
+ * Returns: a pointer to the &drm_gem_object holding the &drm_gpuvm's shared
+ * &dma_resv
+ */
+#define drm_gpuvm_resv_obj(gpuvm__) ((gpuvm__)->r_obj)
+
+#define drm_gpuvm_resv_held(gpuvm__) \
+ dma_resv_held(drm_gpuvm_resv(gpuvm__))
+
+#define drm_gpuvm_resv_assert_held(gpuvm__) \
+ dma_resv_assert_held(drm_gpuvm_resv(gpuvm__))
+
static inline struct drm_gpuva *
__drm_gpuva_next(struct drm_gpuva *va)
{
--
2.41.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [Intel-xe] [CI 03/10] drm/xe: Adjust to "drm/gpuvm: add common dma-resv per struct drm_gpuvm"
2023-10-25 12:55 [Intel-xe] [CI 01/10] drm/gpuvm: convert WARN() to drm_WARN() variants Thomas Hellström
2023-10-25 12:55 ` [Intel-xe] [CI 02/10] drm/gpuvm: add common dma-resv per struct drm_gpuvm Thomas Hellström
@ 2023-10-25 12:55 ` Thomas Hellström
2023-10-25 12:55 ` [Intel-xe] [CI 04/10] drm/gpuvm: add drm_gpuvm_flags to drm_gpuvm Thomas Hellström
` (7 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Thomas Hellström @ 2023-10-25 12:55 UTC (permalink / raw)
To: intel-xe
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/xe/xe_bo.c | 10 ++---
drivers/gpu/drm/xe/xe_bo.h | 2 +-
drivers/gpu/drm/xe/xe_exec.c | 4 +-
drivers/gpu/drm/xe/xe_migrate.c | 4 +-
drivers/gpu/drm/xe/xe_pt.c | 6 +--
drivers/gpu/drm/xe/xe_vm.c | 72 ++++++++++++++++----------------
drivers/gpu/drm/xe/xe_vm.h | 4 --
drivers/gpu/drm/xe/xe_vm_types.h | 12 +++---
8 files changed, 57 insertions(+), 57 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 61789c0e88fb..772ffcd94101 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -501,9 +501,9 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
* that we indeed have it locked, put the vma an the
* vm's notifier.rebind_list instead and scoop later.
*/
- if (dma_resv_trylock(&vm->resv))
+ if (dma_resv_trylock(xe_vm_resv(vm)))
vm_resv_locked = true;
- else if (ctx->resv != &vm->resv) {
+ else if (ctx->resv != xe_vm_resv(vm)) {
spin_lock(&vm->notifier.list_lock);
if (!(vma->gpuva.flags & XE_VMA_DESTROYED))
list_move_tail(&vma->notifier.rebind_link,
@@ -520,7 +520,7 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
&vm->rebind_list);
if (vm_resv_locked)
- dma_resv_unlock(&vm->resv);
+ dma_resv_unlock(xe_vm_resv(vm));
}
}
@@ -1340,7 +1340,7 @@ xe_bo_create_locked_range(struct xe_device *xe,
}
}
- bo = __xe_bo_create_locked(xe, bo, tile, vm ? &vm->resv : NULL,
+ bo = __xe_bo_create_locked(xe, bo, tile, vm ? xe_vm_resv(vm) : NULL,
vm && !xe_vm_in_fault_mode(vm) &&
flags & XE_BO_CREATE_USER_BIT ?
&vm->lru_bulk_move : NULL, size,
@@ -1656,7 +1656,7 @@ int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict)
xe_vm_assert_held(vm);
ctx.allow_res_evict = allow_res_evict;
- ctx.resv = &vm->resv;
+ ctx.resv = xe_vm_resv(vm);
}
return ttm_bo_validate(&bo->ttm, &bo->placement, &ctx);
diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h
index 4e74201cc986..d013112fa4f8 100644
--- a/drivers/gpu/drm/xe/xe_bo.h
+++ b/drivers/gpu/drm/xe/xe_bo.h
@@ -160,7 +160,7 @@ void xe_bo_unlock(struct xe_bo *bo);
static inline void xe_bo_unlock_vm_held(struct xe_bo *bo)
{
if (bo) {
- XE_WARN_ON(bo->vm && bo->ttm.base.resv != &bo->vm->resv);
+ XE_WARN_ON(bo->vm && bo->ttm.base.resv != xe_vm_resv(bo->vm));
if (bo->vm)
xe_vm_assert_held(bo->vm);
else
diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c
index 28e84a0bbeb0..85a8a793f527 100644
--- a/drivers/gpu/drm/xe/xe_exec.c
+++ b/drivers/gpu/drm/xe/xe_exec.c
@@ -281,7 +281,7 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
/* Wait behind munmap style rebinds */
if (!xe_vm_no_dma_fences(vm)) {
err = drm_sched_job_add_resv_dependencies(&job->drm,
- &vm->resv,
+ xe_vm_resv(vm),
DMA_RESV_USAGE_KERNEL);
if (err)
goto err_put_job;
@@ -309,7 +309,7 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
xe_sched_job_arm(job);
if (!xe_vm_no_dma_fences(vm)) {
/* Block userptr invalidations / BO eviction */
- dma_resv_add_fence(&vm->resv,
+ dma_resv_add_fence(xe_vm_resv(vm),
&job->drm.s_fence->finished,
DMA_RESV_USAGE_BOOKKEEP);
diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c
index b4baecde60e6..d9397c0d996a 100644
--- a/drivers/gpu/drm/xe/xe_migrate.c
+++ b/drivers/gpu/drm/xe/xe_migrate.c
@@ -1132,7 +1132,7 @@ xe_migrate_update_pgtables_cpu(struct xe_migrate *m,
DMA_RESV_USAGE_KERNEL))
return ERR_PTR(-ETIME);
- if (wait_vm && !dma_resv_test_signaled(&vm->resv,
+ if (wait_vm && !dma_resv_test_signaled(xe_vm_resv(vm),
DMA_RESV_USAGE_BOOKKEEP))
return ERR_PTR(-ETIME);
@@ -1346,7 +1346,7 @@ xe_migrate_update_pgtables(struct xe_migrate *m,
* trigger preempts before moving forward
*/
if (first_munmap_rebind) {
- err = job_add_deps(job, &vm->resv,
+ err = job_add_deps(job, xe_vm_resv(vm),
DMA_RESV_USAGE_BOOKKEEP);
if (err)
goto err_job;
diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index 31afab617b4e..ab08e4644529 100644
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -873,7 +873,7 @@ static void xe_pt_commit_locks_assert(struct xe_vma *vma)
else if (!xe_vma_is_null(vma))
dma_resv_assert_held(xe_vma_bo(vma)->ttm.base.resv);
- dma_resv_assert_held(&vm->resv);
+ xe_vm_assert_held(vm);
}
static void xe_pt_commit_bind(struct xe_vma *vma,
@@ -1335,7 +1335,7 @@ __xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue
}
/* add shared fence now for pagetable delayed destroy */
- dma_resv_add_fence(&vm->resv, fence, !rebind &&
+ dma_resv_add_fence(xe_vm_resv(vm), fence, !rebind &&
last_munmap_rebind ?
DMA_RESV_USAGE_KERNEL :
DMA_RESV_USAGE_BOOKKEEP);
@@ -1672,7 +1672,7 @@ __xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queu
fence = &ifence->base.base;
/* add shared fence now for pagetable delayed destroy */
- dma_resv_add_fence(&vm->resv, fence,
+ dma_resv_add_fence(xe_vm_resv(vm), fence,
DMA_RESV_USAGE_BOOKKEEP);
/* This fence will be installed by caller when doing eviction */
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 498c0b3e1d73..89f81f199d94 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -37,6 +37,11 @@
#define TEST_VM_ASYNC_OPS_ERROR
+static struct drm_gem_object *xe_vm_obj(struct xe_vm *vm)
+{
+ return vm->gpuvm.r_obj;
+}
+
/**
* xe_vma_userptr_check_repin() - Advisory check for repin needed
* @vma: The userptr vma
@@ -321,7 +326,7 @@ static void resume_and_reinstall_preempt_fences(struct xe_vm *vm)
list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) {
q->ops->resume(q);
- dma_resv_add_fence(&vm->resv, q->compute.pfence,
+ dma_resv_add_fence(xe_vm_resv(vm), q->compute.pfence,
DMA_RESV_USAGE_BOOKKEEP);
xe_vm_fence_all_extobjs(vm, q->compute.pfence,
DMA_RESV_USAGE_BOOKKEEP);
@@ -359,7 +364,7 @@ int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
down_read(&vm->userptr.notifier_lock);
- dma_resv_add_fence(&vm->resv, pfence,
+ dma_resv_add_fence(xe_vm_resv(vm), pfence,
DMA_RESV_USAGE_BOOKKEEP);
xe_vm_fence_all_extobjs(vm, pfence, DMA_RESV_USAGE_BOOKKEEP);
@@ -445,8 +450,7 @@ int xe_vm_lock_dma_resv(struct xe_vm *vm, struct drm_exec *exec,
lockdep_assert_held(&vm->lock);
if (lock_vm) {
- err = drm_exec_prepare_obj(exec, &xe_vm_ttm_bo(vm)->base,
- num_shared);
+ err = drm_exec_prepare_obj(exec, xe_vm_obj(vm), num_shared);
if (err)
return err;
}
@@ -542,7 +546,7 @@ static int xe_preempt_work_begin(struct drm_exec *exec, struct xe_vm *vm,
* 1 fence for each preempt fence plus a fence for each tile from a
* possible rebind
*/
- err = drm_exec_prepare_obj(exec, &xe_vm_ttm_bo(vm)->base,
+ err = drm_exec_prepare_obj(exec, xe_vm_obj(vm),
vm->preempt.num_exec_queues +
vm->xe->info.tile_count);
if (err)
@@ -641,7 +645,7 @@ static void preempt_rebind_work_func(struct work_struct *w)
}
/* Wait on munmap style VM unbinds */
- wait = dma_resv_wait_timeout(&vm->resv,
+ wait = dma_resv_wait_timeout(xe_vm_resv(vm),
DMA_RESV_USAGE_KERNEL,
false, MAX_SCHEDULE_TIMEOUT);
if (wait <= 0) {
@@ -736,13 +740,13 @@ static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni,
* unbinds to complete, and those are attached as BOOKMARK fences
* to the vm.
*/
- dma_resv_iter_begin(&cursor, &vm->resv,
+ dma_resv_iter_begin(&cursor, xe_vm_resv(vm),
DMA_RESV_USAGE_BOOKKEEP);
dma_resv_for_each_fence_unlocked(&cursor, fence)
dma_fence_enable_sw_signaling(fence);
dma_resv_iter_end(&cursor);
- err = dma_resv_wait_timeout(&vm->resv,
+ err = dma_resv_wait_timeout(xe_vm_resv(vm),
DMA_RESV_USAGE_BOOKKEEP,
false, MAX_SCHEDULE_TIMEOUT);
XE_WARN_ON(err <= 0);
@@ -791,14 +795,14 @@ int xe_vm_userptr_pin(struct xe_vm *vm)
}
/* Take lock and move to rebind_list for rebinding. */
- err = dma_resv_lock_interruptible(&vm->resv, NULL);
+ err = dma_resv_lock_interruptible(xe_vm_resv(vm), NULL);
if (err)
goto out_err;
list_for_each_entry_safe(vma, next, &tmp_evict, combined_links.userptr)
list_move_tail(&vma->combined_links.rebind, &vm->rebind_list);
- dma_resv_unlock(&vm->resv);
+ dma_resv_unlock(xe_vm_resv(vm));
return 0;
@@ -1116,7 +1120,7 @@ int xe_vm_prepare_vma(struct drm_exec *exec, struct xe_vma *vma,
int err;
XE_WARN_ON(!vm);
- err = drm_exec_prepare_obj(exec, &xe_vm_ttm_bo(vm)->base, num_shared);
+ err = drm_exec_prepare_obj(exec, xe_vm_obj(vm), num_shared);
if (!err && bo && !bo->vm)
err = drm_exec_prepare_obj(exec, &bo->ttm.base, num_shared);
@@ -1331,6 +1335,7 @@ static void vm_destroy_work_func(struct work_struct *w);
struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
{
+ struct drm_gem_object *vm_resv_obj;
struct xe_vm *vm;
int err, i = 0, number_tiles = 0;
struct xe_tile *tile;
@@ -1342,7 +1347,6 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
vm->xe = xe;
kref_init(&vm->refcount);
- dma_resv_init(&vm->resv);
vm->size = 1ull << xe->info.va_bits;
@@ -1375,12 +1379,21 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
if (!(flags & XE_VM_FLAG_MIGRATION))
xe_device_mem_access_get(xe);
- err = dma_resv_lock_interruptible(&vm->resv, NULL);
+ vm_resv_obj = drm_gpuvm_resv_object_alloc(&xe->drm);
+ if (!vm_resv_obj) {
+ err = -ENOMEM;
+ goto err_no_resv;
+ }
+
+ drm_gpuvm_init(&vm->gpuvm, "Xe VM", &xe->drm, vm_resv_obj, 0, vm->size,
+ 0, 0, &gpuvm_ops);
+
+ drm_gem_object_put(vm_resv_obj);
+
+ err = dma_resv_lock_interruptible(xe_vm_resv(vm), NULL);
if (err)
goto err_put;
- drm_gpuvm_init(&vm->gpuvm, "Xe VM", 0, vm->size, 0, 0,
- &gpuvm_ops);
if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)
vm->flags |= XE_VM_FLAG_64K;
@@ -1422,7 +1435,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
xe_pt_populate_empty(tile, vm, vm->pt_root[id]);
}
- dma_resv_unlock(&vm->resv);
+ dma_resv_unlock(xe_vm_resv(vm));
/* Kernel migration VM shouldn't have a circular loop.. */
if (!(flags & XE_VM_FLAG_MIGRATION)) {
@@ -1483,10 +1496,10 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
if (vm->pt_root[id])
xe_pt_destroy(vm->pt_root[id], vm->flags, NULL);
}
- dma_resv_unlock(&vm->resv);
- drm_gpuvm_destroy(&vm->gpuvm);
+ dma_resv_unlock(xe_vm_resv(vm));
err_put:
- dma_resv_fini(&vm->resv);
+ drm_gpuvm_destroy(&vm->gpuvm);
+err_no_resv:
for_each_tile(tile, xe, id)
xe_range_fence_tree_fini(&vm->rftree[id]);
kfree(vm);
@@ -1590,8 +1603,6 @@ void xe_vm_close_and_put(struct xe_vm *vm)
xe_assert(xe, list_empty(&vm->extobj.list));
up_write(&vm->lock);
- drm_gpuvm_destroy(&vm->gpuvm);
-
mutex_lock(&xe->usm.lock);
if (vm->flags & XE_VM_FLAG_FAULT_MODE)
xe->usm.num_vm_in_fault_mode--;
@@ -1644,7 +1655,7 @@ static void vm_destroy_work_func(struct work_struct *w)
trace_xe_vm_free(vm);
dma_fence_put(vm->rebind_fence);
- dma_resv_fini(&vm->resv);
+ drm_gpuvm_destroy(&vm->gpuvm);
kfree(vm);
}
@@ -2091,15 +2102,6 @@ static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
}
}
-struct ttm_buffer_object *xe_vm_ttm_bo(struct xe_vm *vm)
-{
- int idx = vm->flags & XE_VM_FLAG_MIGRATION ?
- XE_VM_FLAG_TILE_ID(vm->flags) : 0;
-
- /* Safe to use index 0 as all BO in the VM share a single dma-resv lock */
- return &vm->pt_root[idx]->bo->ttm;
-}
-
static void prep_vma_destroy(struct xe_vm *vm, struct xe_vma *vma,
bool post_commit)
{
@@ -3178,9 +3180,9 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
int xe_vm_lock(struct xe_vm *vm, bool intr)
{
if (intr)
- return dma_resv_lock_interruptible(&vm->resv, NULL);
+ return dma_resv_lock_interruptible(xe_vm_resv(vm), NULL);
- return dma_resv_lock(&vm->resv, NULL);
+ return dma_resv_lock(xe_vm_resv(vm), NULL);
}
/**
@@ -3191,7 +3193,7 @@ int xe_vm_lock(struct xe_vm *vm, bool intr)
*/
void xe_vm_unlock(struct xe_vm *vm)
{
- dma_resv_unlock(&vm->resv);
+ dma_resv_unlock(xe_vm_resv(vm));
}
/**
@@ -3223,7 +3225,7 @@ int xe_vm_invalidate_vma(struct xe_vma *vma)
WARN_ON_ONCE(!mmu_interval_check_retry
(&vma->userptr.notifier,
vma->userptr.notifier_seq));
- WARN_ON_ONCE(!dma_resv_test_signaled(&xe_vma_vm(vma)->resv,
+ WARN_ON_ONCE(!dma_resv_test_signaled(xe_vm_resv(xe_vma_vm(vma)),
DMA_RESV_USAGE_BOOKKEEP));
} else {
diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h
index b08c75fbd8a1..88077214261f 100644
--- a/drivers/gpu/drm/xe/xe_vm.h
+++ b/drivers/gpu/drm/xe/xe_vm.h
@@ -139,8 +139,6 @@ static inline bool xe_vma_is_userptr(struct xe_vma *vma)
return xe_vma_has_no_bo(vma) && !xe_vma_is_null(vma);
}
-#define xe_vm_assert_held(vm) dma_resv_assert_held(&(vm)->resv)
-
u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile);
int xe_vm_create_ioctl(struct drm_device *dev, void *data,
@@ -182,8 +180,6 @@ int xe_vm_invalidate_vma(struct xe_vma *vma);
extern struct ttm_device_funcs xe_ttm_funcs;
-struct ttm_buffer_object *xe_vm_ttm_bo(struct xe_vm *vm);
-
static inline void xe_vm_queue_rebind_worker(struct xe_vm *vm)
{
xe_assert(vm->xe, xe_vm_in_compute_mode(vm));
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index aaf0c7101019..f11ac81cf010 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -131,8 +131,6 @@ struct xe_vma {
struct xe_device;
-#define xe_vm_assert_held(vm) dma_resv_assert_held(&(vm)->resv)
-
struct xe_vm {
/** @gpuvm: base GPUVM used to track VMAs */
struct drm_gpuvm gpuvm;
@@ -144,9 +142,6 @@ struct xe_vm {
/* exec queue used for (un)binding vma's */
struct xe_exec_queue *q[XE_MAX_TILES_PER_DEVICE];
- /** Protects @rebind_list and the page-table structures */
- struct dma_resv resv;
-
/** @lru_bulk_move: Bulk LRU move list for this VM's BOs */
struct ttm_lru_bulk_move lru_bulk_move;
@@ -420,4 +415,11 @@ struct xe_vma_op {
};
};
+static inline struct dma_resv *xe_vm_resv(struct xe_vm *vm)
+{
+ return drm_gpuvm_resv(&vm->gpuvm);
+}
+
+#define xe_vm_assert_held(vm) dma_resv_assert_held(xe_vm_resv(vm))
+
#endif
--
2.41.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [Intel-xe] [CI 04/10] drm/gpuvm: add drm_gpuvm_flags to drm_gpuvm
2023-10-25 12:55 [Intel-xe] [CI 01/10] drm/gpuvm: convert WARN() to drm_WARN() variants Thomas Hellström
2023-10-25 12:55 ` [Intel-xe] [CI 02/10] drm/gpuvm: add common dma-resv per struct drm_gpuvm Thomas Hellström
2023-10-25 12:55 ` [Intel-xe] [CI 03/10] drm/xe: Adjust to "drm/gpuvm: add common dma-resv per struct drm_gpuvm" Thomas Hellström
@ 2023-10-25 12:55 ` Thomas Hellström
2023-10-25 12:55 ` [Intel-xe] [CI 05/10] drm/xe: Adjust to commit "drm/gpuvm: add drm_gpuvm_flags to drm_gpuvm" Thomas Hellström
` (6 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Thomas Hellström @ 2023-10-25 12:55 UTC (permalink / raw)
To: intel-xe
From: Danilo Krummrich <dakr@redhat.com>
Introduce flags for struct drm_gpuvm, this required by subsequent
commits.
Reviewed-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
Signed-off-by: Danilo Krummrich <dakr@redhat.com>
---
drivers/gpu/drm/drm_gpuvm.c | 3 +++
drivers/gpu/drm/nouveau/nouveau_uvmm.c | 2 +-
include/drm/drm_gpuvm.h | 16 ++++++++++++++++
3 files changed, 20 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c
index d4907aace831..3bb3d083b32e 100644
--- a/drivers/gpu/drm/drm_gpuvm.c
+++ b/drivers/gpu/drm/drm_gpuvm.c
@@ -702,6 +702,7 @@ EXPORT_SYMBOL_GPL(drm_gpuvm_resv_object_alloc);
* drm_gpuvm_init() - initialize a &drm_gpuvm
* @gpuvm: pointer to the &drm_gpuvm to initialize
* @name: the name of the GPU VA space
+ * @flags: the &drm_gpuvm_flags for this GPUVM
* @drm: the &drm_device this VM resides in
* @r_obj: the resv &drm_gem_object providing the GPUVM's common &dma_resv
* @start_offset: the start offset of the GPU VA space
@@ -717,6 +718,7 @@ EXPORT_SYMBOL_GPL(drm_gpuvm_resv_object_alloc);
*/
void
drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
+ enum drm_gpuvm_flags flags,
struct drm_device *drm,
struct drm_gem_object *r_obj,
u64 start_offset, u64 range,
@@ -727,6 +729,7 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
INIT_LIST_HEAD(&gpuvm->rb.list);
gpuvm->name = name ? name : "unknown";
+ gpuvm->flags = flags;
gpuvm->ops = ops;
gpuvm->drm = drm;
gpuvm->r_obj = r_obj;
diff --git a/drivers/gpu/drm/nouveau/nouveau_uvmm.c b/drivers/gpu/drm/nouveau/nouveau_uvmm.c
index b4e7d662961a..ed439bf4032f 100644
--- a/drivers/gpu/drm/nouveau/nouveau_uvmm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_uvmm.c
@@ -1844,7 +1844,7 @@ nouveau_uvmm_init(struct nouveau_uvmm *uvmm, struct nouveau_cli *cli,
uvmm->kernel_managed_addr = kernel_managed_addr;
uvmm->kernel_managed_size = kernel_managed_size;
- drm_gpuvm_init(&uvmm->base, cli->name, drm, r_obj,
+ drm_gpuvm_init(&uvmm->base, cli->name, 0, drm, r_obj,
NOUVEAU_VA_SPACE_START,
NOUVEAU_VA_SPACE_END,
kernel_managed_addr, kernel_managed_size,
diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h
index 54dc195a68ea..d3d3dd33ab7c 100644
--- a/include/drm/drm_gpuvm.h
+++ b/include/drm/drm_gpuvm.h
@@ -184,6 +184,16 @@ static inline bool drm_gpuva_invalidated(struct drm_gpuva *va)
return va->flags & DRM_GPUVA_INVALIDATED;
}
+/**
+ * enum drm_gpuvm_flags - flags for struct drm_gpuvm
+ */
+enum drm_gpuvm_flags {
+ /**
+ * @DRM_GPUVM_USERBITS: user defined bits
+ */
+ DRM_GPUVM_USERBITS = BIT(0),
+};
+
/**
* struct drm_gpuvm - DRM GPU VA Manager
*
@@ -202,6 +212,11 @@ struct drm_gpuvm {
*/
const char *name;
+ /**
+ * @flags: the &drm_gpuvm_flags of this GPUVM
+ */
+ enum drm_gpuvm_flags flags;
+
/**
* @drm: the &drm_device this VM lives in
*/
@@ -252,6 +267,7 @@ struct drm_gpuvm {
};
void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
+ enum drm_gpuvm_flags flags,
struct drm_device *drm,
struct drm_gem_object *r_obj,
u64 start_offset, u64 range,
--
2.41.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [Intel-xe] [CI 05/10] drm/xe: Adjust to commit "drm/gpuvm: add drm_gpuvm_flags to drm_gpuvm"
2023-10-25 12:55 [Intel-xe] [CI 01/10] drm/gpuvm: convert WARN() to drm_WARN() variants Thomas Hellström
` (2 preceding siblings ...)
2023-10-25 12:55 ` [Intel-xe] [CI 04/10] drm/gpuvm: add drm_gpuvm_flags to drm_gpuvm Thomas Hellström
@ 2023-10-25 12:55 ` Thomas Hellström
2023-10-25 12:55 ` [Intel-xe] [CI 06/10] drm/gpuvm: add an abstraction for a VM / BO combination Thomas Hellström
` (5 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Thomas Hellström @ 2023-10-25 12:55 UTC (permalink / raw)
To: intel-xe
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/xe/xe_vm.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 89f81f199d94..16ac77b67fe1 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -1385,8 +1385,8 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
goto err_no_resv;
}
- drm_gpuvm_init(&vm->gpuvm, "Xe VM", &xe->drm, vm_resv_obj, 0, vm->size,
- 0, 0, &gpuvm_ops);
+ drm_gpuvm_init(&vm->gpuvm, "Xe VM", 0, &xe->drm, vm_resv_obj,
+ 0, vm->size, 0, 0, &gpuvm_ops);
drm_gem_object_put(vm_resv_obj);
--
2.41.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [Intel-xe] [CI 06/10] drm/gpuvm: add an abstraction for a VM / BO combination
2023-10-25 12:55 [Intel-xe] [CI 01/10] drm/gpuvm: convert WARN() to drm_WARN() variants Thomas Hellström
` (3 preceding siblings ...)
2023-10-25 12:55 ` [Intel-xe] [CI 05/10] drm/xe: Adjust to commit "drm/gpuvm: add drm_gpuvm_flags to drm_gpuvm" Thomas Hellström
@ 2023-10-25 12:55 ` Thomas Hellström
2023-10-25 12:55 ` [Intel-xe] [CI 07/10] drm/xe: Adjust to commit "drm/gpuvm: add an abstraction for a VM/BO combination" Thomas Hellström
` (4 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Thomas Hellström @ 2023-10-25 12:55 UTC (permalink / raw)
To: intel-xe
From: Danilo Krummrich <dakr@redhat.com>
Add an abstraction layer between the drm_gpuva mappings of a particular
drm_gem_object and this GEM object itself. The abstraction represents a
combination of a drm_gem_object and drm_gpuvm. The drm_gem_object holds
a list of drm_gpuvm_bo structures (the structure representing this
abstraction), while each drm_gpuvm_bo contains list of mappings of this
GEM object.
This has multiple advantages:
1) We can use the drm_gpuvm_bo structure to attach it to various lists
of the drm_gpuvm. This is useful for tracking external and evicted
objects per VM, which is introduced in subsequent patches.
2) Finding mappings of a certain drm_gem_object mapped in a certain
drm_gpuvm becomes much cheaper.
3) Drivers can derive and extend the structure to easily represent
driver specific states of a BO for a certain GPUVM.
The idea of this abstraction was taken from amdgpu, hence the credit for
this idea goes to the developers of amdgpu.
Cc: Christian König <christian.koenig@amd.com>
Signed-off-by: Danilo Krummrich <dakr@redhat.com>
---
drivers/gpu/drm/drm_gpuvm.c | 335 +++++++++++++++++++++----
drivers/gpu/drm/nouveau/nouveau_uvmm.c | 64 +++--
include/drm/drm_gem.h | 32 +--
include/drm/drm_gpuvm.h | 188 +++++++++++++-
4 files changed, 533 insertions(+), 86 deletions(-)
diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c
index 3bb3d083b32e..f7876450766d 100644
--- a/drivers/gpu/drm/drm_gpuvm.c
+++ b/drivers/gpu/drm/drm_gpuvm.c
@@ -70,6 +70,18 @@
* &drm_gem_object, such as the &drm_gem_object containing the root page table,
* but it can also be a 'dummy' object, which can be allocated with
* drm_gpuvm_resv_object_alloc().
+ *
+ * In order to connect a struct drm_gpuva its backing &drm_gem_object each
+ * &drm_gem_object maintains a list of &drm_gpuvm_bo structures, and each
+ * &drm_gpuvm_bo contains a list of &drm_gpuva structures.
+ *
+ * A &drm_gpuvm_bo is an abstraction that represents a combination of a
+ * &drm_gpuvm and a &drm_gem_object. Every such combination should be unique.
+ * This is ensured by the API through drm_gpuvm_bo_obtain() and
+ * drm_gpuvm_bo_obtain_prealloc() which first look into the corresponding
+ * &drm_gem_object list of &drm_gpuvm_bos for an existing instance of this
+ * particular combination. If not existent a new instance is created and linked
+ * to the &drm_gem_object.
*/
/**
@@ -395,21 +407,28 @@
/**
* DOC: Locking
*
- * Generally, the GPU VA manager does not take care of locking itself, it is
- * the drivers responsibility to take care about locking. Drivers might want to
- * protect the following operations: inserting, removing and iterating
- * &drm_gpuva objects as well as generating all kinds of operations, such as
- * split / merge or prefetch.
- *
- * The GPU VA manager also does not take care of the locking of the backing
- * &drm_gem_object buffers GPU VA lists by itself; drivers are responsible to
- * enforce mutual exclusion using either the GEMs dma_resv lock or alternatively
- * a driver specific external lock. For the latter see also
- * drm_gem_gpuva_set_lock().
- *
- * However, the GPU VA manager contains lockdep checks to ensure callers of its
- * API hold the corresponding lock whenever the &drm_gem_objects GPU VA list is
- * accessed by functions such as drm_gpuva_link() or drm_gpuva_unlink().
+ * In terms of managing &drm_gpuva entries DRM GPUVM does not take care of
+ * locking itself, it is the drivers responsibility to take care about locking.
+ * Drivers might want to protect the following operations: inserting, removing
+ * and iterating &drm_gpuva objects as well as generating all kinds of
+ * operations, such as split / merge or prefetch.
+ *
+ * DRM GPUVM also does not take care of the locking of the backing
+ * &drm_gem_object buffers GPU VA lists and &drm_gpuvm_bo abstractions by
+ * itself; drivers are responsible to enforce mutual exclusion using either the
+ * GEMs dma_resv lock or alternatively a driver specific external lock. For the
+ * latter see also drm_gem_gpuva_set_lock().
+ *
+ * However, DRM GPUVM contains lockdep checks to ensure callers of its API hold
+ * the corresponding lock whenever the &drm_gem_objects GPU VA list is accessed
+ * by functions such as drm_gpuva_link() or drm_gpuva_unlink(), but also
+ * drm_gpuvm_bo_obtain() and drm_gpuvm_bo_put().
+ *
+ * The latter is required since on creation and destruction of a &drm_gpuvm_bo
+ * the &drm_gpuvm_bo is attached / removed from the &drm_gem_objects gpuva list.
+ * Subsequent calls to drm_gpuvm_bo_obtain() for the same &drm_gpuvm and
+ * &drm_gem_object must be able to observe previous creations and destructions
+ * of &drm_gpuvm_bos in order to keep instances unique.
*/
/**
@@ -439,6 +458,7 @@
* {
* struct drm_gpuva_ops *ops;
* struct drm_gpuva_op *op
+ * struct drm_gpuvm_bo *vm_bo;
*
* driver_lock_va_space();
* ops = drm_gpuvm_sm_map_ops_create(gpuvm, addr, range,
@@ -446,6 +466,10 @@
* if (IS_ERR(ops))
* return PTR_ERR(ops);
*
+ * vm_bo = drm_gpuvm_bo_obtain(gpuvm, obj);
+ * if (IS_ERR(vm_bo))
+ * return PTR_ERR(vm_bo);
+ *
* drm_gpuva_for_each_op(op, ops) {
* struct drm_gpuva *va;
*
@@ -458,7 +482,7 @@
*
* driver_vm_map();
* drm_gpuva_map(gpuvm, va, &op->map);
- * drm_gpuva_link(va);
+ * drm_gpuva_link(va, vm_bo);
*
* break;
* case DRM_GPUVA_OP_REMAP: {
@@ -485,11 +509,11 @@
* driver_vm_remap();
* drm_gpuva_remap(prev, next, &op->remap);
*
- * drm_gpuva_unlink(va);
* if (prev)
- * drm_gpuva_link(prev);
+ * drm_gpuva_link(prev, va->vm_bo);
* if (next)
- * drm_gpuva_link(next);
+ * drm_gpuva_link(next, va->vm_bo);
+ * drm_gpuva_unlink(va);
*
* break;
* }
@@ -505,6 +529,7 @@
* break;
* }
* }
+ * drm_gpuvm_bo_put(vm_bo);
* driver_unlock_va_space();
*
* return 0;
@@ -514,6 +539,7 @@
*
* struct driver_context {
* struct drm_gpuvm *gpuvm;
+ * struct drm_gpuvm_bo *vm_bo;
* struct drm_gpuva *new_va;
* struct drm_gpuva *prev_va;
* struct drm_gpuva *next_va;
@@ -534,6 +560,7 @@
* struct drm_gem_object *obj, u64 offset)
* {
* struct driver_context ctx;
+ * struct drm_gpuvm_bo *vm_bo;
* struct drm_gpuva_ops *ops;
* struct drm_gpuva_op *op;
* int ret = 0;
@@ -543,16 +570,23 @@
* ctx.new_va = kzalloc(sizeof(*ctx.new_va), GFP_KERNEL);
* ctx.prev_va = kzalloc(sizeof(*ctx.prev_va), GFP_KERNEL);
* ctx.next_va = kzalloc(sizeof(*ctx.next_va), GFP_KERNEL);
- * if (!ctx.new_va || !ctx.prev_va || !ctx.next_va) {
+ * ctx.vm_bo = drm_gpuvm_bo_create(gpuvm, obj);
+ * if (!ctx.new_va || !ctx.prev_va || !ctx.next_va || !vm_bo) {
* ret = -ENOMEM;
* goto out;
* }
*
+ * // Typically protected with a driver specific GEM gpuva lock
+ * // used in the fence signaling path for drm_gpuva_link() and
+ * // drm_gpuva_unlink(), hence pre-allocate.
+ * ctx.vm_bo = drm_gpuvm_bo_obtain_prealloc(ctx.vm_bo);
+ *
* driver_lock_va_space();
* ret = drm_gpuvm_sm_map(gpuvm, &ctx, addr, range, obj, offset);
* driver_unlock_va_space();
*
* out:
+ * drm_gpuvm_bo_put(ctx.vm_bo);
* kfree(ctx.new_va);
* kfree(ctx.prev_va);
* kfree(ctx.next_va);
@@ -565,7 +599,7 @@
*
* drm_gpuva_map(ctx->vm, ctx->new_va, &op->map);
*
- * drm_gpuva_link(ctx->new_va);
+ * drm_gpuva_link(ctx->new_va, ctx->vm_bo);
*
* // prevent the new GPUVA from being freed in
* // driver_mapping_create()
@@ -577,22 +611,23 @@
* int driver_gpuva_remap(struct drm_gpuva_op *op, void *__ctx)
* {
* struct driver_context *ctx = __ctx;
+ * struct drm_gpuva *va = op->remap.unmap->va;
*
* drm_gpuva_remap(ctx->prev_va, ctx->next_va, &op->remap);
*
- * drm_gpuva_unlink(op->remap.unmap->va);
- * kfree(op->remap.unmap->va);
- *
* if (op->remap.prev) {
- * drm_gpuva_link(ctx->prev_va);
+ * drm_gpuva_link(ctx->prev_va, va->vm_bo);
* ctx->prev_va = NULL;
* }
*
* if (op->remap.next) {
- * drm_gpuva_link(ctx->next_va);
+ * drm_gpuva_link(ctx->next_va, va->vm_bo);
* ctx->next_va = NULL;
* }
*
+ * drm_gpuva_unlink(va);
+ * kfree(va);
+ *
* return 0;
* }
*
@@ -774,6 +809,194 @@ drm_gpuvm_destroy(struct drm_gpuvm *gpuvm)
}
EXPORT_SYMBOL_GPL(drm_gpuvm_destroy);
+/**
+ * drm_gpuvm_bo_create() - create a new instance of struct drm_gpuvm_bo
+ * @gpuvm: The &drm_gpuvm the @obj is mapped in.
+ * @obj: The &drm_gem_object being mapped in the @gpuvm.
+ *
+ * If provided by the driver, this function uses the &drm_gpuvm_ops
+ * vm_bo_alloc() callback to allocate.
+ *
+ * Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure
+ */
+struct drm_gpuvm_bo *
+drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm,
+ struct drm_gem_object *obj)
+{
+ const struct drm_gpuvm_ops *ops = gpuvm->ops;
+ struct drm_gpuvm_bo *vm_bo;
+
+ if (ops && ops->vm_bo_alloc)
+ vm_bo = ops->vm_bo_alloc();
+ else
+ vm_bo = kzalloc(sizeof(*vm_bo), GFP_KERNEL);
+
+ if (unlikely(!vm_bo))
+ return NULL;
+
+ vm_bo->vm = gpuvm;
+ vm_bo->obj = obj;
+ drm_gem_object_get(obj);
+
+ kref_init(&vm_bo->kref);
+ INIT_LIST_HEAD(&vm_bo->list.gpuva);
+ INIT_LIST_HEAD(&vm_bo->list.entry.gem);
+
+ return vm_bo;
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_bo_create);
+
+static void
+drm_gpuvm_bo_destroy(struct kref *kref)
+{
+ struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo,
+ kref);
+ struct drm_gpuvm *gpuvm = vm_bo->vm;
+ const struct drm_gpuvm_ops *ops = gpuvm->ops;
+ struct drm_gem_object *obj = vm_bo->obj;
+ bool lock = !drm_gpuvm_resv_protected(gpuvm);
+
+ if (!lock)
+ drm_gpuvm_resv_assert_held(gpuvm);
+
+ drm_gem_gpuva_assert_lock_held(obj);
+ list_del(&vm_bo->list.entry.gem);
+
+ if (ops && ops->vm_bo_free)
+ ops->vm_bo_free(vm_bo);
+ else
+ kfree(vm_bo);
+
+ drm_gem_object_put(obj);
+}
+
+/**
+ * drm_gpuvm_bo_put() - drop a struct drm_gpuvm_bo reference
+ * @vm_bo: the &drm_gpuvm_bo to release the reference of
+ *
+ * This releases a reference to @vm_bo.
+ *
+ * If the reference count drops to zero, the &gpuvm_bo is destroyed, which
+ * includes removing it from the GEMs gpuva list. Hence, if a call to this
+ * function can potentially let the reference count to zero the caller must
+ * hold the dma-resv or driver specific GEM gpuva lock.
+ */
+void
+drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo)
+{
+ if (vm_bo)
+ kref_put(&vm_bo->kref, drm_gpuvm_bo_destroy);
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_bo_put);
+
+static struct drm_gpuvm_bo *
+__drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,
+ struct drm_gem_object *obj)
+{
+ struct drm_gpuvm_bo *vm_bo;
+
+ drm_gem_gpuva_assert_lock_held(obj);
+ drm_gem_for_each_gpuvm_bo(vm_bo, obj)
+ if (vm_bo->vm == gpuvm)
+ return vm_bo;
+
+ return NULL;
+}
+
+/**
+ * drm_gpuvm_bo_find() - find the &drm_gpuvm_bo for the given
+ * &drm_gpuvm and &drm_gem_object
+ * @gpuvm: The &drm_gpuvm the @obj is mapped in.
+ * @obj: The &drm_gem_object being mapped in the @gpuvm.
+ *
+ * Find the &drm_gpuvm_bo representing the combination of the given
+ * &drm_gpuvm and &drm_gem_object. If found, increases the reference
+ * count of the &drm_gpuvm_bo accordingly.
+ *
+ * Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure
+ */
+struct drm_gpuvm_bo *
+drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,
+ struct drm_gem_object *obj)
+{
+ struct drm_gpuvm_bo *vm_bo = __drm_gpuvm_bo_find(gpuvm, obj);
+
+ return vm_bo ? drm_gpuvm_bo_get(vm_bo) : NULL;
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_bo_find);
+
+/**
+ * drm_gpuvm_bo_obtain() - obtains and instance of the &drm_gpuvm_bo for the
+ * given &drm_gpuvm and &drm_gem_object
+ * @gpuvm: The &drm_gpuvm the @obj is mapped in.
+ * @obj: The &drm_gem_object being mapped in the @gpuvm.
+ *
+ * Find the &drm_gpuvm_bo representing the combination of the given
+ * &drm_gpuvm and &drm_gem_object. If found, increases the reference
+ * count of the &drm_gpuvm_bo accordingly. If not found, allocates a new
+ * &drm_gpuvm_bo.
+ *
+ * A new &drm_gpuvm_bo is added to the GEMs gpuva list.
+ *
+ * Returns: a pointer to the &drm_gpuvm_bo on success, an ERR_PTR on failure
+ */
+struct drm_gpuvm_bo *
+drm_gpuvm_bo_obtain(struct drm_gpuvm *gpuvm,
+ struct drm_gem_object *obj)
+{
+ struct drm_gpuvm_bo *vm_bo;
+
+ vm_bo = drm_gpuvm_bo_find(gpuvm, obj);
+ if (vm_bo)
+ return vm_bo;
+
+ vm_bo = drm_gpuvm_bo_create(gpuvm, obj);
+ if (!vm_bo)
+ return ERR_PTR(-ENOMEM);
+
+ drm_gem_gpuva_assert_lock_held(obj);
+ list_add_tail(&vm_bo->list.entry.gem, &obj->gpuva.list);
+
+ return vm_bo;
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain);
+
+/**
+ * drm_gpuvm_bo_obtain_prealloc() - obtains and instance of the &drm_gpuvm_bo
+ * for the given &drm_gpuvm and &drm_gem_object
+ * @__vm_bo: A pre-allocated struct drm_gpuvm_bo.
+ *
+ * Find the &drm_gpuvm_bo representing the combination of the given
+ * &drm_gpuvm and &drm_gem_object. If found, increases the reference
+ * count of the found &drm_gpuvm_bo accordingly, while the @__vm_bo reference
+ * count is decreased. If not found @__vm_bo is returned without further
+ * increase of the reference count.
+ *
+ * A new &drm_gpuvm_bo is added to the GEMs gpuva list.
+ *
+ * Returns: a pointer to the found &drm_gpuvm_bo or @__vm_bo if no existing
+ * &drm_gpuvm_bo was found
+ */
+struct drm_gpuvm_bo *
+drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *__vm_bo)
+{
+ struct drm_gpuvm *gpuvm = __vm_bo->vm;
+ struct drm_gem_object *obj = __vm_bo->obj;
+ struct drm_gpuvm_bo *vm_bo;
+
+ vm_bo = drm_gpuvm_bo_find(gpuvm, obj);
+ if (vm_bo) {
+ drm_gpuvm_bo_put(__vm_bo);
+ return vm_bo;
+ }
+
+ drm_gem_gpuva_assert_lock_held(obj);
+ list_add_tail(&__vm_bo->list.entry.gem, &obj->gpuva.list);
+
+ return __vm_bo;
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain_prealloc);
+
static int
__drm_gpuva_insert(struct drm_gpuvm *gpuvm,
struct drm_gpuva *va)
@@ -864,24 +1087,33 @@ EXPORT_SYMBOL_GPL(drm_gpuva_remove);
/**
* drm_gpuva_link() - link a &drm_gpuva
* @va: the &drm_gpuva to link
+ * @vm_bo: the &drm_gpuvm_bo to add the &drm_gpuva to
*
- * This adds the given &va to the GPU VA list of the &drm_gem_object it is
- * associated with.
+ * This adds the given &va to the GPU VA list of the &drm_gpuvm_bo and the
+ * &drm_gpuvm_bo to the &drm_gem_object it is associated with.
+ *
+ * For every &drm_gpuva entry added to the &drm_gpuvm_bo an additional
+ * reference of the latter is taken.
*
* This function expects the caller to protect the GEM's GPUVA list against
- * concurrent access using the GEMs dma_resv lock.
+ * concurrent access using either the GEMs dma_resv lock or a driver specific
+ * lock set through drm_gem_gpuva_set_lock().
*/
void
-drm_gpuva_link(struct drm_gpuva *va)
+drm_gpuva_link(struct drm_gpuva *va, struct drm_gpuvm_bo *vm_bo)
{
struct drm_gem_object *obj = va->gem.obj;
+ struct drm_gpuvm *gpuvm = va->vm;
if (unlikely(!obj))
return;
- drm_gem_gpuva_assert_lock_held(obj);
+ drm_WARN_ON(gpuvm->drm, obj != vm_bo->obj);
- list_add_tail(&va->gem.entry, &obj->gpuva.list);
+ va->vm_bo = drm_gpuvm_bo_get(vm_bo);
+
+ drm_gem_gpuva_assert_lock_held(obj);
+ list_add_tail(&va->gem.entry, &vm_bo->list.gpuva);
}
EXPORT_SYMBOL_GPL(drm_gpuva_link);
@@ -892,20 +1124,31 @@ EXPORT_SYMBOL_GPL(drm_gpuva_link);
* This removes the given &va from the GPU VA list of the &drm_gem_object it is
* associated with.
*
+ * This removes the given &va from the GPU VA list of the &drm_gpuvm_bo and
+ * the &drm_gpuvm_bo from the &drm_gem_object it is associated with in case
+ * this call unlinks the last &drm_gpuva from the &drm_gpuvm_bo.
+ *
+ * For every &drm_gpuva entry removed from the &drm_gpuvm_bo a reference of
+ * the latter is dropped.
+ *
* This function expects the caller to protect the GEM's GPUVA list against
- * concurrent access using the GEMs dma_resv lock.
+ * concurrent access using either the GEMs dma_resv lock or a driver specific
+ * lock set through drm_gem_gpuva_set_lock().
*/
void
drm_gpuva_unlink(struct drm_gpuva *va)
{
struct drm_gem_object *obj = va->gem.obj;
+ struct drm_gpuvm_bo *vm_bo = va->vm_bo;
if (unlikely(!obj))
return;
drm_gem_gpuva_assert_lock_held(obj);
-
list_del_init(&va->gem.entry);
+
+ va->vm_bo = NULL;
+ drm_gpuvm_bo_put(vm_bo);
}
EXPORT_SYMBOL_GPL(drm_gpuva_unlink);
@@ -1050,10 +1293,10 @@ drm_gpuva_remap(struct drm_gpuva *prev,
struct drm_gpuva *next,
struct drm_gpuva_op_remap *op)
{
- struct drm_gpuva *curr = op->unmap->va;
- struct drm_gpuvm *gpuvm = curr->vm;
+ struct drm_gpuva *va = op->unmap->va;
+ struct drm_gpuvm *gpuvm = va->vm;
- drm_gpuva_remove(curr);
+ drm_gpuva_remove(va);
if (op->prev) {
drm_gpuva_init_from_op(prev, op->prev);
@@ -1695,9 +1938,8 @@ drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm,
EXPORT_SYMBOL_GPL(drm_gpuvm_prefetch_ops_create);
/**
- * drm_gpuvm_gem_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM
- * @gpuvm: the &drm_gpuvm representing the GPU VA space
- * @obj: the &drm_gem_object to unmap
+ * drm_gpuvm_bo_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM
+ * @vm_bo: the &drm_gpuvm_bo abstraction
*
* This function creates a list of operations to perform unmapping for every
* GPUVA attached to a GEM.
@@ -1714,15 +1956,14 @@ EXPORT_SYMBOL_GPL(drm_gpuvm_prefetch_ops_create);
* Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
*/
struct drm_gpuva_ops *
-drm_gpuvm_gem_unmap_ops_create(struct drm_gpuvm *gpuvm,
- struct drm_gem_object *obj)
+drm_gpuvm_bo_unmap_ops_create(struct drm_gpuvm_bo *vm_bo)
{
struct drm_gpuva_ops *ops;
struct drm_gpuva_op *op;
struct drm_gpuva *va;
int ret;
- drm_gem_gpuva_assert_lock_held(obj);
+ drm_gem_gpuva_assert_lock_held(vm_bo->obj);
ops = kzalloc(sizeof(*ops), GFP_KERNEL);
if (!ops)
@@ -1730,8 +1971,8 @@ drm_gpuvm_gem_unmap_ops_create(struct drm_gpuvm *gpuvm,
INIT_LIST_HEAD(&ops->list);
- drm_gem_for_each_gpuva(va, obj) {
- op = gpuva_op_alloc(gpuvm);
+ drm_gpuvm_bo_for_each_va(va, vm_bo) {
+ op = gpuva_op_alloc(vm_bo->vm);
if (!op) {
ret = -ENOMEM;
goto err_free_ops;
@@ -1745,10 +1986,10 @@ drm_gpuvm_gem_unmap_ops_create(struct drm_gpuvm *gpuvm,
return ops;
err_free_ops:
- drm_gpuva_ops_free(gpuvm, ops);
+ drm_gpuva_ops_free(vm_bo->vm, ops);
return ERR_PTR(ret);
}
-EXPORT_SYMBOL_GPL(drm_gpuvm_gem_unmap_ops_create);
+EXPORT_SYMBOL_GPL(drm_gpuvm_bo_unmap_ops_create);
/**
* drm_gpuva_ops_free() - free the given &drm_gpuva_ops
diff --git a/drivers/gpu/drm/nouveau/nouveau_uvmm.c b/drivers/gpu/drm/nouveau/nouveau_uvmm.c
index ed439bf4032f..1e95b0a1b047 100644
--- a/drivers/gpu/drm/nouveau/nouveau_uvmm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_uvmm.c
@@ -62,6 +62,8 @@ struct bind_job_op {
enum vm_bind_op op;
u32 flags;
+ struct drm_gpuvm_bo *vm_bo;
+
struct {
u64 addr;
u64 range;
@@ -1113,22 +1115,28 @@ bind_validate_region(struct nouveau_job *job)
}
static void
-bind_link_gpuvas(struct drm_gpuva_ops *ops, struct nouveau_uvma_prealloc *new)
+bind_link_gpuvas(struct bind_job_op *bop)
{
+ struct nouveau_uvma_prealloc *new = &bop->new;
+ struct drm_gpuvm_bo *vm_bo = bop->vm_bo;
+ struct drm_gpuva_ops *ops = bop->ops;
struct drm_gpuva_op *op;
drm_gpuva_for_each_op(op, ops) {
switch (op->op) {
case DRM_GPUVA_OP_MAP:
- drm_gpuva_link(&new->map->va);
+ drm_gpuva_link(&new->map->va, vm_bo);
break;
- case DRM_GPUVA_OP_REMAP:
+ case DRM_GPUVA_OP_REMAP: {
+ struct drm_gpuva *va = op->remap.unmap->va;
+
if (op->remap.prev)
- drm_gpuva_link(&new->prev->va);
+ drm_gpuva_link(&new->prev->va, va->vm_bo);
if (op->remap.next)
- drm_gpuva_link(&new->next->va);
- drm_gpuva_unlink(op->remap.unmap->va);
+ drm_gpuva_link(&new->next->va, va->vm_bo);
+ drm_gpuva_unlink(va);
break;
+ }
case DRM_GPUVA_OP_UNMAP:
drm_gpuva_unlink(op->unmap.va);
break;
@@ -1150,10 +1158,18 @@ nouveau_uvmm_bind_job_submit(struct nouveau_job *job)
list_for_each_op(op, &bind_job->ops) {
if (op->op == OP_MAP) {
- op->gem.obj = drm_gem_object_lookup(job->file_priv,
- op->gem.handle);
- if (!op->gem.obj)
+ struct drm_gem_object *obj;
+
+ obj = drm_gem_object_lookup(job->file_priv,
+ op->gem.handle);
+ if (!(op->gem.obj = obj))
return -ENOENT;
+
+ dma_resv_lock(obj->resv, NULL);
+ op->vm_bo = drm_gpuvm_bo_obtain(&uvmm->base, obj);
+ dma_resv_unlock(obj->resv);
+ if (IS_ERR(op->vm_bo))
+ return PTR_ERR(op->vm_bo);
}
ret = bind_validate_op(job, op);
@@ -1364,7 +1380,7 @@ nouveau_uvmm_bind_job_submit(struct nouveau_job *job)
case OP_UNMAP_SPARSE:
case OP_MAP:
case OP_UNMAP:
- bind_link_gpuvas(op->ops, &op->new);
+ bind_link_gpuvas(op);
break;
default:
break;
@@ -1511,6 +1527,12 @@ nouveau_uvmm_bind_job_free_work_fn(struct work_struct *work)
if (!IS_ERR_OR_NULL(op->ops))
drm_gpuva_ops_free(&uvmm->base, op->ops);
+ if (!IS_ERR_OR_NULL(op->vm_bo)) {
+ dma_resv_lock(obj->resv, NULL);
+ drm_gpuvm_bo_put(op->vm_bo);
+ dma_resv_unlock(obj->resv);
+ }
+
if (obj)
drm_gem_object_put(obj);
}
@@ -1776,15 +1798,18 @@ void
nouveau_uvmm_bo_map_all(struct nouveau_bo *nvbo, struct nouveau_mem *mem)
{
struct drm_gem_object *obj = &nvbo->bo.base;
+ struct drm_gpuvm_bo *vm_bo;
struct drm_gpuva *va;
dma_resv_assert_held(obj->resv);
- drm_gem_for_each_gpuva(va, obj) {
- struct nouveau_uvma *uvma = uvma_from_va(va);
+ drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
+ drm_gpuvm_bo_for_each_va(va, vm_bo) {
+ struct nouveau_uvma *uvma = uvma_from_va(va);
- nouveau_uvma_map(uvma, mem);
- drm_gpuva_invalidate(va, false);
+ nouveau_uvma_map(uvma, mem);
+ drm_gpuva_invalidate(va, false);
+ }
}
}
@@ -1792,15 +1817,18 @@ void
nouveau_uvmm_bo_unmap_all(struct nouveau_bo *nvbo)
{
struct drm_gem_object *obj = &nvbo->bo.base;
+ struct drm_gpuvm_bo *vm_bo;
struct drm_gpuva *va;
dma_resv_assert_held(obj->resv);
- drm_gem_for_each_gpuva(va, obj) {
- struct nouveau_uvma *uvma = uvma_from_va(va);
+ drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
+ drm_gpuvm_bo_for_each_va(va, vm_bo) {
+ struct nouveau_uvma *uvma = uvma_from_va(va);
- nouveau_uvma_unmap(uvma);
- drm_gpuva_invalidate(va, true);
+ nouveau_uvma_unmap(uvma);
+ drm_gpuva_invalidate(va, true);
+ }
}
}
diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h
index bc9f6aa2f3fe..7147978d82d8 100644
--- a/include/drm/drm_gem.h
+++ b/include/drm/drm_gem.h
@@ -571,7 +571,7 @@ int drm_gem_evict(struct drm_gem_object *obj);
* drm_gem_gpuva_init() - initialize the gpuva list of a GEM object
* @obj: the &drm_gem_object
*
- * This initializes the &drm_gem_object's &drm_gpuva list.
+ * This initializes the &drm_gem_object's &drm_gpuvm_bo list.
*
* Calling this function is only necessary for drivers intending to support the
* &drm_driver_feature DRIVER_GEM_GPUVA.
@@ -584,28 +584,28 @@ static inline void drm_gem_gpuva_init(struct drm_gem_object *obj)
}
/**
- * drm_gem_for_each_gpuva() - iternator to walk over a list of gpuvas
- * @entry__: &drm_gpuva structure to assign to in each iteration step
- * @obj__: the &drm_gem_object the &drm_gpuvas to walk are associated with
+ * drm_gem_for_each_gpuvm_bo() - iterator to walk over a list of &drm_gpuvm_bo
+ * @entry__: &drm_gpuvm_bo structure to assign to in each iteration step
+ * @obj__: the &drm_gem_object the &drm_gpuvm_bo to walk are associated with
*
- * This iterator walks over all &drm_gpuva structures associated with the
- * &drm_gpuva_manager.
+ * This iterator walks over all &drm_gpuvm_bo structures associated with the
+ * &drm_gem_object.
*/
-#define drm_gem_for_each_gpuva(entry__, obj__) \
- list_for_each_entry(entry__, &(obj__)->gpuva.list, gem.entry)
+#define drm_gem_for_each_gpuvm_bo(entry__, obj__) \
+ list_for_each_entry(entry__, &(obj__)->gpuva.list, list.entry.gem)
/**
- * drm_gem_for_each_gpuva_safe() - iternator to safely walk over a list of
- * gpuvas
- * @entry__: &drm_gpuva structure to assign to in each iteration step
- * @next__: &next &drm_gpuva to store the next step
- * @obj__: the &drm_gem_object the &drm_gpuvas to walk are associated with
+ * drm_gem_for_each_gpuvm_bo_safe() - iterator to safely walk over a list of
+ * &drm_gpuvm_bo
+ * @entry__: &drm_gpuvm_bostructure to assign to in each iteration step
+ * @next__: &next &drm_gpuvm_bo to store the next step
+ * @obj__: the &drm_gem_object the &drm_gpuvm_bo to walk are associated with
*
- * This iterator walks over all &drm_gpuva structures associated with the
+ * This iterator walks over all &drm_gpuvm_bo structures associated with the
* &drm_gem_object. It is implemented with list_for_each_entry_safe(), hence
* it is save against removal of elements.
*/
-#define drm_gem_for_each_gpuva_safe(entry__, next__, obj__) \
- list_for_each_entry_safe(entry__, next__, &(obj__)->gpuva.list, gem.entry)
+#define drm_gem_for_each_gpuvm_bo_safe(entry__, next__, obj__) \
+ list_for_each_entry_safe(entry__, next__, &(obj__)->gpuva.list, list.entry.gem)
#endif /* __DRM_GEM_H__ */
diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h
index d3d3dd33ab7c..6e2893fe3808 100644
--- a/include/drm/drm_gpuvm.h
+++ b/include/drm/drm_gpuvm.h
@@ -25,6 +25,7 @@
* OTHER DEALINGS IN THE SOFTWARE.
*/
+#include <linux/dma-resv.h>
#include <linux/list.h>
#include <linux/rbtree.h>
#include <linux/types.h>
@@ -33,6 +34,7 @@
#include <drm/drm_gem.h>
struct drm_gpuvm;
+struct drm_gpuvm_bo;
struct drm_gpuvm_ops;
/**
@@ -73,6 +75,12 @@ struct drm_gpuva {
*/
struct drm_gpuvm *vm;
+ /**
+ * @vm_bo: the &drm_gpuvm_bo abstraction for the mapped
+ * &drm_gem_object
+ */
+ struct drm_gpuvm_bo *vm_bo;
+
/**
* @flags: the &drm_gpuva_flags for this mapping
*/
@@ -108,7 +116,7 @@ struct drm_gpuva {
struct drm_gem_object *obj;
/**
- * @entry: the &list_head to attach this object to a &drm_gem_object
+ * @entry: the &list_head to attach this object to a &drm_gpuvm_bo
*/
struct list_head entry;
} gem;
@@ -141,7 +149,7 @@ struct drm_gpuva {
int drm_gpuva_insert(struct drm_gpuvm *gpuvm, struct drm_gpuva *va);
void drm_gpuva_remove(struct drm_gpuva *va);
-void drm_gpuva_link(struct drm_gpuva *va);
+void drm_gpuva_link(struct drm_gpuva *va, struct drm_gpuvm_bo *vm_bo);
void drm_gpuva_unlink(struct drm_gpuva *va);
struct drm_gpuva *drm_gpuva_find(struct drm_gpuvm *gpuvm,
@@ -188,10 +196,16 @@ static inline bool drm_gpuva_invalidated(struct drm_gpuva *va)
* enum drm_gpuvm_flags - flags for struct drm_gpuvm
*/
enum drm_gpuvm_flags {
+ /**
+ * @DRM_GPUVM_RESV_PROTECTED: GPUVM is protected externally by the
+ * GPUVM's &dma_resv lock
+ */
+ DRM_GPUVM_RESV_PROTECTED = BIT(0),
+
/**
* @DRM_GPUVM_USERBITS: user defined bits
*/
- DRM_GPUVM_USERBITS = BIT(0),
+ DRM_GPUVM_USERBITS = BIT(1),
};
/**
@@ -280,6 +294,19 @@ bool drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range);
struct drm_gem_object *
drm_gpuvm_resv_object_alloc(struct drm_device *drm);
+/**
+ * drm_gpuvm_resv_protected() - indicates whether &DRM_GPUVM_RESV_PROTECTED is
+ * set
+ * @gpuvm: the &drm_gpuvm
+ *
+ * Returns: true if &DRM_GPUVM_RESV_PROTECTED is set, false otherwise.
+ */
+static inline bool
+drm_gpuvm_resv_protected(struct drm_gpuvm *gpuvm)
+{
+ return gpuvm->flags & DRM_GPUVM_RESV_PROTECTED;
+}
+
/**
* drm_gpuvm_resv() - returns the &drm_gpuvm's &dma_resv
* @gpuvm__: the &drm_gpuvm
@@ -298,6 +325,12 @@ drm_gpuvm_resv_object_alloc(struct drm_device *drm);
*/
#define drm_gpuvm_resv_obj(gpuvm__) ((gpuvm__)->r_obj)
+#define drm_gpuvm_resv_held(gpuvm__) \
+ dma_resv_held(drm_gpuvm_resv(gpuvm__))
+
+#define drm_gpuvm_resv_assert_held(gpuvm__) \
+ dma_resv_assert_held(drm_gpuvm_resv(gpuvm__))
+
#define drm_gpuvm_resv_held(gpuvm__) \
dma_resv_held(drm_gpuvm_resv(gpuvm__))
@@ -382,6 +415,128 @@ __drm_gpuva_next(struct drm_gpuva *va)
#define drm_gpuvm_for_each_va_safe(va__, next__, gpuvm__) \
list_for_each_entry_safe(va__, next__, &(gpuvm__)->rb.list, rb.entry)
+/**
+ * struct drm_gpuvm_bo - structure representing a &drm_gpuvm and
+ * &drm_gem_object combination
+ *
+ * This structure is an abstraction representing a &drm_gpuvm and
+ * &drm_gem_object combination. It serves as an indirection to accelerate
+ * iterating all &drm_gpuvas within a &drm_gpuvm backed by the same
+ * &drm_gem_object.
+ *
+ * Furthermore it is used cache evicted GEM objects for a certain GPU-VM to
+ * accelerate validation.
+ *
+ * Typically, drivers want to create an instance of a struct drm_gpuvm_bo once
+ * a GEM object is mapped first in a GPU-VM and release the instance once the
+ * last mapping of the GEM object in this GPU-VM is unmapped.
+ */
+struct drm_gpuvm_bo {
+ /**
+ * @vm: The &drm_gpuvm the @obj is mapped in. This pointer is not
+ * reference counted.
+ *
+ * A struct drm_gpuvm_bo is not allowed to out-live its &drm_gpuvm
+ * context. Implicitly, this is ensured by the fact that the driver is
+ * responsible to ensure the VM doesn't contain mappings once it's
+ * freed, since a struct drm_gpuvm_bo should be freed once the last
+ * mapping being backed by the corresponding buffer object is unmapped.
+ */
+ struct drm_gpuvm *vm;
+
+ /**
+ * @obj: The &drm_gem_object being mapped in @vm. This is a reference
+ * counted pointer.
+ */
+ struct drm_gem_object *obj;
+
+ /**
+ * @kref: The reference count for this &drm_gpuvm_bo.
+ */
+ struct kref kref;
+
+ /**
+ * @list: Structure containing all &list_heads.
+ */
+ struct {
+ /**
+ * @gpuva: The list of linked &drm_gpuvas.
+ */
+ struct list_head gpuva;
+
+ /**
+ * @entry: Structure containing all &list_heads serving as
+ * entry.
+ */
+ struct {
+ /**
+ * @gem: List entry to attach to the &drm_gem_objects
+ * gpuva list.
+ */
+ struct list_head gem;
+ } entry;
+ } list;
+};
+
+struct drm_gpuvm_bo *
+drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm,
+ struct drm_gem_object *obj);
+
+struct drm_gpuvm_bo *
+drm_gpuvm_bo_obtain(struct drm_gpuvm *gpuvm,
+ struct drm_gem_object *obj);
+struct drm_gpuvm_bo *
+drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *vm_bo);
+
+/**
+ * drm_gpuvm_bo_get() - acquire a struct drm_gpuvm_bo reference
+ * @vm_bo: the &drm_gpuvm_bo to acquire the reference of
+ *
+ * This function acquires an additional reference to @vm_bo. It is illegal to
+ * call this without already holding a reference. No locks required.
+ */
+static inline struct drm_gpuvm_bo *
+drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo)
+{
+ kref_get(&vm_bo->kref);
+ return vm_bo;
+}
+
+void drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo);
+
+struct drm_gpuvm_bo *
+drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,
+ struct drm_gem_object *obj);
+
+/**
+ * drm_gpuvm_bo_for_each_va() - iterator to walk over a list of &drm_gpuva
+ * @va__: &drm_gpuva structure to assign to in each iteration step
+ * @vm_bo__: the &drm_gpuvm_bo the &drm_gpuva to walk are associated with
+ *
+ * This iterator walks over all &drm_gpuva structures associated with the
+ * &drm_gpuvm_bo.
+ *
+ * The caller must hold the GEM's gpuva lock.
+ */
+#define drm_gpuvm_bo_for_each_va(va__, vm_bo__) \
+ list_for_each_entry(va__, &(vm_bo)->list.gpuva, gem.entry)
+
+/**
+ * drm_gpuvm_bo_for_each_va_safe() - iterator to safely walk over a list of
+ * &drm_gpuva
+ * @va__: &drm_gpuva structure to assign to in each iteration step
+ * @next__: &next &drm_gpuva to store the next step
+ * @vm_bo__: the &drm_gpuvm_bo the &drm_gpuva to walk are associated with
+ *
+ * This iterator walks over all &drm_gpuva structures associated with the
+ * &drm_gpuvm_bo. It is implemented with list_for_each_entry_safe(), hence
+ * it is save against removal of elements.
+ *
+ * The caller must hold the GEM's gpuva lock.
+ */
+#define drm_gpuvm_bo_for_each_va_safe(va__, next__, vm_bo__) \
+ list_for_each_entry_safe(va__, next__, &(vm_bo)->list.gpuva, gem.entry)
+
/**
* enum drm_gpuva_op_type - GPU VA operation type
*
@@ -661,8 +816,7 @@ drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm,
u64 addr, u64 range);
struct drm_gpuva_ops *
-drm_gpuvm_gem_unmap_ops_create(struct drm_gpuvm *gpuvm,
- struct drm_gem_object *obj);
+drm_gpuvm_bo_unmap_ops_create(struct drm_gpuvm_bo *vm_bo);
void drm_gpuva_ops_free(struct drm_gpuvm *gpuvm,
struct drm_gpuva_ops *ops);
@@ -706,6 +860,30 @@ struct drm_gpuvm_ops {
*/
void (*op_free)(struct drm_gpuva_op *op);
+ /**
+ * @vm_bo_alloc: called when the &drm_gpuvm allocates
+ * a struct drm_gpuvm_bo
+ *
+ * Some drivers may want to embed struct drm_gpuvm_bo into driver
+ * specific structures. By implementing this callback drivers can
+ * allocate memory accordingly.
+ *
+ * This callback is optional.
+ */
+ struct drm_gpuvm_bo *(*vm_bo_alloc)(void);
+
+ /**
+ * @vm_bo_free: called when the &drm_gpuvm frees a
+ * struct drm_gpuvm_bo
+ *
+ * Some drivers may want to embed struct drm_gpuvm_bo into driver
+ * specific structures. By implementing this callback drivers can
+ * free the previously allocated memory accordingly.
+ *
+ * This callback is optional.
+ */
+ void (*vm_bo_free)(struct drm_gpuvm_bo *vm_bo);
+
/**
* @sm_step_map: called from &drm_gpuvm_sm_map to finally insert the
* mapping once all previous steps were completed
--
2.41.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [Intel-xe] [CI 07/10] drm/xe: Adjust to commit "drm/gpuvm: add an abstraction for a VM/BO combination"
2023-10-25 12:55 [Intel-xe] [CI 01/10] drm/gpuvm: convert WARN() to drm_WARN() variants Thomas Hellström
` (4 preceding siblings ...)
2023-10-25 12:55 ` [Intel-xe] [CI 06/10] drm/gpuvm: add an abstraction for a VM / BO combination Thomas Hellström
@ 2023-10-25 12:55 ` Thomas Hellström
2023-10-25 12:55 ` [Intel-xe] [CI 08/10] drm/gpuvm: track/lock/validate external/evicted objects Thomas Hellström
` (3 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Thomas Hellström @ 2023-10-25 12:55 UTC (permalink / raw)
To: intel-xe
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/xe/xe_bo.c | 11 +++++++----
drivers/gpu/drm/xe/xe_vm.c | 32 +++++++++++++++++++++++++-------
2 files changed, 32 insertions(+), 11 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 772ffcd94101..82381de4876b 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -452,6 +452,7 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
struct dma_fence *fence;
struct drm_gpuva *gpuva;
struct drm_gem_object *obj = &bo->ttm.base;
+ struct drm_gpuvm_bo *vm_bo;
int ret = 0;
dma_resv_assert_held(bo->ttm.base.resv);
@@ -464,11 +465,12 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
dma_resv_iter_end(&cursor);
}
- drm_gem_for_each_gpuva(gpuva, obj) {
- struct xe_vma *vma = gpuva_to_vma(gpuva);
- struct xe_vm *vm = xe_vma_vm(vma);
+ drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
+ drm_gpuvm_bo_for_each_va(gpuva, vm_bo) {
+ struct xe_vma *vma = gpuva_to_vma(gpuva);
+ struct xe_vm *vm = xe_vma_vm(vma);
- trace_xe_vma_evict(vma);
+ trace_xe_vma_evict(vma);
if (xe_vm_in_fault_mode(vm)) {
/* Wait for pending binds / unbinds. */
@@ -522,6 +524,7 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
if (vm_resv_locked)
dma_resv_unlock(xe_vm_resv(vm));
}
+ }
}
return ret;
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 16ac77b67fe1..1b94736ef430 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -911,12 +911,21 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm,
vma->gpuva.flags |= XE_VMA_ATOMIC_PTE_BIT;
if (bo) {
+ struct drm_gpuvm_bo *vm_bo;
+
xe_bo_assert_held(bo);
+ vm_bo = drm_gpuvm_bo_obtain(vma->gpuva.vm, &bo->ttm.base);
+ if (IS_ERR(vm_bo)) {
+ kfree(vma);
+ return ERR_CAST(vm_bo);
+ }
+
drm_gem_object_get(&bo->ttm.base);
vma->gpuva.gem.obj = &bo->ttm.base;
vma->gpuva.gem.offset = bo_offset_or_userptr;
- drm_gpuva_link(&vma->gpuva);
+ drm_gpuva_link(&vma->gpuva, vm_bo);
+ drm_gpuvm_bo_put(vm_bo);
} else /* userptr or null */ {
if (!is_null) {
u64 size = end - start + 1;
@@ -998,16 +1007,19 @@ static struct xe_vma *
bo_has_vm_references_locked(struct xe_bo *bo, struct xe_vm *vm,
struct xe_vma *ignore)
{
- struct drm_gpuva *gpuva;
+ struct drm_gpuvm_bo *vm_bo;
+ struct drm_gpuva *va;
struct drm_gem_object *obj = &bo->ttm.base;
xe_bo_assert_held(bo);
- drm_gem_for_each_gpuva(gpuva, obj) {
- struct xe_vma *vma = gpuva_to_vma(gpuva);
+ drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
+ drm_gpuvm_bo_for_each_va(va, vm_bo) {
+ struct xe_vma *vma = gpuva_to_vma(va);
- if (vma != ignore && xe_vma_vm(vma) == vm)
- return vma;
+ if (vma != ignore && xe_vma_vm(vma) == vm)
+ return vma;
+ }
}
return NULL;
@@ -2175,6 +2187,7 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo,
struct drm_gpuva_ops *ops;
struct drm_gpuva_op *__op;
struct xe_vma_op *op;
+ struct drm_gpuvm_bo *vm_bo;
int err;
lockdep_assert_held_write(&vm->lock);
@@ -2232,7 +2245,12 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo,
err = xe_bo_lock(bo, true);
if (err)
return ERR_PTR(err);
- ops = drm_gpuvm_gem_unmap_ops_create(&vm->gpuvm, obj);
+
+ vm_bo = drm_gpuvm_bo_find(&vm->gpuvm, obj);
+ if (!vm_bo)
+ break;
+
+ ops = drm_gpuvm_bo_unmap_ops_create(vm_bo);
xe_bo_unlock(bo);
if (IS_ERR(ops))
return ops;
--
2.41.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [Intel-xe] [CI 08/10] drm/gpuvm: track/lock/validate external/evicted objects
2023-10-25 12:55 [Intel-xe] [CI 01/10] drm/gpuvm: convert WARN() to drm_WARN() variants Thomas Hellström
` (5 preceding siblings ...)
2023-10-25 12:55 ` [Intel-xe] [CI 07/10] drm/xe: Adjust to commit "drm/gpuvm: add an abstraction for a VM/BO combination" Thomas Hellström
@ 2023-10-25 12:55 ` Thomas Hellström
2023-10-25 12:55 ` [Intel-xe] [CI 09/10] drm/xe: Adapt to GPUVM tracking of external / evicted objects Thomas Hellström
` (2 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Thomas Hellström @ 2023-10-25 12:55 UTC (permalink / raw)
To: intel-xe
From: Danilo Krummrich <dakr@redhat.com>
Currently the DRM GPUVM offers common infrastructure to track GPU VA
allocations and mappings, generically connect GPU VA mappings to their
backing buffers and perform more complex mapping operations on the GPU VA
space.
However, there are more design patterns commonly used by drivers, which
can potentially be generalized in order to make the DRM GPUVM represent
a basis for GPU-VM implementations. In this context, this patch aims
at generalizing the following elements.
1) Provide a common dma-resv for GEM objects not being used outside of
this GPU-VM.
2) Provide tracking of external GEM objects (GEM objects which are
shared with other GPU-VMs).
3) Provide functions to efficiently lock all GEM objects dma-resv the
GPU-VM contains mappings of.
4) Provide tracking of evicted GEM objects the GPU-VM contains mappings
of, such that validation of evicted GEM objects is accelerated.
5) Provide some convinience functions for common patterns.
Big thanks to Boris Brezillon for his help to figure out locking for
drivers updating the GPU VA space within the fence signalling path.
Suggested-by: Matthew Brost <matthew.brost@intel.com>
Signed-off-by: Danilo Krummrich <dakr@redhat.com>
---
drivers/gpu/drm/drm_gpuvm.c | 633 ++++++++++++++++++++++++++++++++++++
include/drm/drm_gpuvm.h | 250 ++++++++++++++
2 files changed, 883 insertions(+)
diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c
index f7876450766d..be2531e633f2 100644
--- a/drivers/gpu/drm/drm_gpuvm.c
+++ b/drivers/gpu/drm/drm_gpuvm.c
@@ -82,6 +82,21 @@
* &drm_gem_object list of &drm_gpuvm_bos for an existing instance of this
* particular combination. If not existent a new instance is created and linked
* to the &drm_gem_object.
+ *
+ * &drm_gpuvm_bo structures, since unique for a given &drm_gpuvm, are also used
+ * as entry for the &drm_gpuvm's lists of external and evicted objects. Those
+ * lists are maintained in order to accelerate locking of dma-resv locks and
+ * validation of evicted objects bound in a &drm_gpuvm. For instance, all
+ * &drm_gem_object's &dma_resv of a given &drm_gpuvm can be locked by calling
+ * drm_gpuvm_exec_lock(). Once locked drivers can call drm_gpuvm_validate() in
+ * order to validate all evicted &drm_gem_objects. It is also possible to lock
+ * additional &drm_gem_objects by providing the corresponding parameters to
+ * drm_gpuvm_exec_lock() as well as open code the &drm_exec loop while making
+ * use of helper functions such as drm_gpuvm_prepare_range() or
+ * drm_gpuvm_prepare_objects().
+ *
+ * Every bound &drm_gem_object is treated as external object when its &dma_resv
+ * structure is different than the &drm_gpuvm's common &dma_resv structure.
*/
/**
@@ -429,6 +444,20 @@
* Subsequent calls to drm_gpuvm_bo_obtain() for the same &drm_gpuvm and
* &drm_gem_object must be able to observe previous creations and destructions
* of &drm_gpuvm_bos in order to keep instances unique.
+ *
+ * The &drm_gpuvm's lists for keeping track of external and evicted objects are
+ * protected against concurrent insertion / removal and iteration internally.
+ *
+ * However, drivers still need ensure to protect concurrent calls to functions
+ * iterating those lists, namely drm_gpuvm_prepare_objects() and
+ * drm_gpuvm_validate().
+ *
+ * Alternatively, drivers can set the &DRM_GPUVM_RESV_PROTECTED flag to indicate
+ * that the corresponding &dma_resv locks are held in order to protect the
+ * lists. If &DRM_GPUVM_RESV_PROTECTED is set, internal locking is disabled and
+ * the corresponding lockdep checks are enabled. This is an optimization for
+ * drivers which are capable of taking the corresponding &dma_resv locks and
+ * hence do not require internal locking.
*/
/**
@@ -641,6 +670,201 @@
* }
*/
+/**
+ * get_next_vm_bo_from_list() - get the next vm_bo element
+ * @__gpuvm: the &drm_gpuvm
+ * @__list_name: the name of the list we're iterating on
+ * @__local_list: a pointer to the local list used to store already iterated items
+ * @__prev_vm_bo: the previous element we got from get_next_vm_bo_from_list()
+ *
+ * This helper is here to provide lockless list iteration. Lockless as in, the
+ * iterator releases the lock immediately after picking the first element from
+ * the list, so list insertion deletion can happen concurrently.
+ *
+ * Elements popped from the original list are kept in a local list, so removal
+ * and is_empty checks can still happen while we're iterating the list.
+ */
+#define get_next_vm_bo_from_list(__gpuvm, __list_name, __local_list, __prev_vm_bo) \
+ ({ \
+ struct drm_gpuvm_bo *__vm_bo = NULL; \
+ \
+ drm_gpuvm_bo_put(__prev_vm_bo); \
+ \
+ spin_lock(&(__gpuvm)->__list_name.lock); \
+ if (!(__gpuvm)->__list_name.local_list) \
+ (__gpuvm)->__list_name.local_list = __local_list; \
+ else \
+ drm_WARN_ON((__gpuvm)->drm, \
+ (__gpuvm)->__list_name.local_list != __local_list); \
+ \
+ while (!list_empty(&(__gpuvm)->__list_name.list)) { \
+ __vm_bo = list_first_entry(&(__gpuvm)->__list_name.list, \
+ struct drm_gpuvm_bo, \
+ list.entry.__list_name); \
+ if (kref_get_unless_zero(&__vm_bo->kref)) { \
+ list_move_tail(&(__vm_bo)->list.entry.__list_name, \
+ __local_list); \
+ break; \
+ } else { \
+ list_del_init(&(__vm_bo)->list.entry.__list_name); \
+ __vm_bo = NULL; \
+ } \
+ } \
+ spin_unlock(&(__gpuvm)->__list_name.lock); \
+ \
+ __vm_bo; \
+ })
+
+/**
+ * for_each_vm_bo_in_list() - internal vm_bo list iterator
+ * @__gpuvm: the &drm_gpuvm
+ * @__list_name: the name of the list we're iterating on
+ * @__local_list: a pointer to the local list used to store already iterated items
+ * @__vm_bo: the struct drm_gpuvm_bo to assign in each iteration step
+ *
+ * This helper is here to provide lockless list iteration. Lockless as in, the
+ * iterator releases the lock immediately after picking the first element from the
+ * list, hence list insertion and deletion can happen concurrently.
+ *
+ * It is not allowed to re-assign the vm_bo pointer from inside this loop.
+ *
+ * Typical use:
+ *
+ * struct drm_gpuvm_bo *vm_bo;
+ * LIST_HEAD(my_local_list);
+ *
+ * ret = 0;
+ * for_each_vm_bo_in_list(gpuvm, <list_name>, &my_local_list, vm_bo) {
+ * ret = do_something_with_vm_bo(..., vm_bo);
+ * if (ret)
+ * break;
+ * }
+ * // Drop ref in case we break out of the loop.
+ * drm_gpuvm_bo_put(vm_bo);
+ * restore_vm_bo_list(gpuvm, <list_name>, &my_local_list);
+ *
+ *
+ * Only used for internal list iterations, not meant to be exposed to the outside
+ * world.
+ */
+#define for_each_vm_bo_in_list(__gpuvm, __list_name, __local_list, __vm_bo) \
+ for (__vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name, \
+ __local_list, NULL); \
+ __vm_bo; \
+ __vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name, \
+ __local_list, __vm_bo))
+
+static void
+__restore_vm_bo_list(struct drm_gpuvm *gpuvm, spinlock_t *lock,
+ struct list_head *list, struct list_head **local_list)
+{
+ /* Merge back the two lists, moving local list elements to the
+ * head to preserve previous ordering, in case it matters.
+ */
+ spin_lock(lock);
+ if (*local_list) {
+ list_splice(*local_list, list);
+ *local_list = NULL;
+ }
+ spin_unlock(lock);
+}
+
+/**
+ * restore_vm_bo_list() - move vm_bo elements back to their original list
+ * @__gpuvm: the &drm_gpuvm
+ * @__list_name: the name of the list we're iterating on
+ *
+ * When we're done iterating a vm_bo list, we should call restore_vm_bo_list()
+ * to restore the original state and let new iterations take place.
+ */
+#define restore_vm_bo_list(__gpuvm, __list_name) \
+ __restore_vm_bo_list((__gpuvm), &(__gpuvm)->__list_name.lock, \
+ &(__gpuvm)->__list_name.list, \
+ &(__gpuvm)->__list_name.local_list)
+
+static void
+cond_spin_lock(spinlock_t *lock, bool cond)
+{
+ if (cond)
+ spin_lock(lock);
+}
+
+static void
+cond_spin_unlock(spinlock_t *lock, bool cond)
+{
+ if (cond)
+ spin_unlock(lock);
+}
+
+static void
+__drm_gpuvm_bo_list_add(struct drm_gpuvm *gpuvm, spinlock_t *lock,
+ struct list_head *entry, struct list_head *list)
+{
+ cond_spin_lock(lock, !!lock);
+ if (list_empty(entry))
+ list_add_tail(entry, list);
+ cond_spin_unlock(lock, !!lock);
+}
+
+/**
+ * drm_gpuvm_bo_list_add() - insert a vm_bo into the given list
+ * @__vm_bo: the &drm_gpuvm_bo
+ * @__list_name: the name of the list to insert into
+ * @__lock: whether to lock with the internal spinlock
+ *
+ * Inserts the given @__vm_bo into the list specified by @__list_name.
+ */
+#define drm_gpuvm_bo_list_add(__vm_bo, __list_name, __lock) \
+ __drm_gpuvm_bo_list_add((__vm_bo)->vm, \
+ __lock ? &(__vm_bo)->vm->__list_name.lock : \
+ NULL, \
+ &(__vm_bo)->list.entry.__list_name, \
+ &(__vm_bo)->vm->__list_name.list)
+
+static void
+__drm_gpuvm_bo_list_del(struct drm_gpuvm *gpuvm, spinlock_t *lock,
+ struct list_head *entry, bool init)
+{
+ cond_spin_lock(lock, !!lock);
+ if (init) {
+ if (!list_empty(entry))
+ list_del_init(entry);
+ } else {
+ list_del(entry);
+ }
+ cond_spin_unlock(lock, !!lock);
+}
+
+/**
+ * drm_gpuvm_bo_list_del_init() - remove a vm_bo from the given list
+ * @__vm_bo: the &drm_gpuvm_bo
+ * @__list_name: the name of the list to insert into
+ * @__lock: whether to lock with the internal spinlock
+ *
+ * Removes the given @__vm_bo from the list specified by @__list_name.
+ */
+#define drm_gpuvm_bo_list_del_init(__vm_bo, __list_name, __lock) \
+ __drm_gpuvm_bo_list_del((__vm_bo)->vm, \
+ __lock ? &(__vm_bo)->vm->__list_name.lock : \
+ NULL, \
+ &(__vm_bo)->list.entry.__list_name, \
+ true)
+
+/**
+ * drm_gpuvm_bo_list_del() - remove a vm_bo from the given list
+ * @__vm_bo: the &drm_gpuvm_bo
+ * @__list_name: the name of the list to insert into
+ * @__lock: whether to lock with the internal spinlock
+ *
+ * Removes the given @__vm_bo from the list specified by @__list_name.
+ */
+#define drm_gpuvm_bo_list_del(__vm_bo, __list_name, __lock) \
+ __drm_gpuvm_bo_list_del((__vm_bo)->vm, \
+ __lock ? &(__vm_bo)->vm->__list_name.lock : \
+ NULL, \
+ &(__vm_bo)->list.entry.__list_name, \
+ false)
+
#define to_drm_gpuva(__node) container_of((__node), struct drm_gpuva, rb.node)
#define GPUVA_START(node) ((node)->va.addr)
@@ -763,6 +987,12 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
gpuvm->rb.tree = RB_ROOT_CACHED;
INIT_LIST_HEAD(&gpuvm->rb.list);
+ INIT_LIST_HEAD(&gpuvm->extobj.list);
+ spin_lock_init(&gpuvm->extobj.lock);
+
+ INIT_LIST_HEAD(&gpuvm->evict.list);
+ spin_lock_init(&gpuvm->evict.lock);
+
gpuvm->name = name ? name : "unknown";
gpuvm->flags = flags;
gpuvm->ops = ops;
@@ -805,10 +1035,352 @@ drm_gpuvm_destroy(struct drm_gpuvm *gpuvm)
drm_WARN(gpuvm->drm, !RB_EMPTY_ROOT(&gpuvm->rb.tree.rb_root),
"GPUVA tree is not empty, potentially leaking memory.\n");
+ drm_WARN(gpuvm->drm, !list_empty(&gpuvm->extobj.list),
+ "Extobj list should be empty.\n");
+ drm_WARN(gpuvm->drm, !list_empty(&gpuvm->evict.list),
+ "Evict list should be empty.\n");
+
drm_gem_object_put(gpuvm->r_obj);
}
EXPORT_SYMBOL_GPL(drm_gpuvm_destroy);
+static int
+__drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm,
+ struct drm_exec *exec,
+ unsigned int num_fences)
+{
+ struct drm_gpuvm_bo *vm_bo;
+ LIST_HEAD(extobjs);
+ int ret = 0;
+
+ for_each_vm_bo_in_list(gpuvm, extobj, &extobjs, vm_bo) {
+ ret = drm_exec_prepare_obj(exec, vm_bo->obj, num_fences);
+ if (ret)
+ break;
+ }
+ /* Drop ref in case we break out of the loop. */
+ drm_gpuvm_bo_put(vm_bo);
+ restore_vm_bo_list(gpuvm, extobj);
+
+ return ret;
+}
+
+static int
+drm_gpuvm_prepare_objects_locked(struct drm_gpuvm *gpuvm,
+ struct drm_exec *exec,
+ unsigned int num_fences)
+{
+ struct drm_gpuvm_bo *vm_bo;
+ int ret = 0;
+
+ drm_gpuvm_resv_assert_held(gpuvm);
+ list_for_each_entry(vm_bo, &gpuvm->extobj.list, list.entry.extobj) {
+ ret = drm_exec_prepare_obj(exec, vm_bo->obj, num_fences);
+ if (ret)
+ break;
+
+ if (vm_bo->evicted)
+ drm_gpuvm_bo_list_add(vm_bo, evict, false);
+ }
+
+ return ret;
+}
+
+/**
+ * drm_gpuvm_prepare_objects() - prepare all assoiciated BOs
+ * @gpuvm: the &drm_gpuvm
+ * @exec: the &drm_exec locking context
+ * @num_fences: the amount of &dma_fences to reserve
+ *
+ * Calls drm_exec_prepare_obj() for all &drm_gem_objects the given
+ * &drm_gpuvm contains mappings of.
+ *
+ * Using this function directly, it is the drivers responsibility to call
+ * drm_exec_init() and drm_exec_fini() accordingly.
+ *
+ * Note: This function is safe against concurrent insertion and removal of
+ * external objects, however it is not safe against concurrent usage itself.
+ *
+ * Drivers need to make sure to protect this case with either an outer VM lock
+ * or by calling drm_gpuvm_prepare_vm() before this function within the
+ * drm_exec_until_all_locked() loop, such that the GPUVM's dma-resv lock ensures
+ * mutual exclusion.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+int
+drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm,
+ struct drm_exec *exec,
+ unsigned int num_fences)
+{
+ if (drm_gpuvm_resv_protected(gpuvm))
+ return drm_gpuvm_prepare_objects_locked(gpuvm, exec,
+ num_fences);
+ else
+ return __drm_gpuvm_prepare_objects(gpuvm, exec, num_fences);
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_objects);
+
+/**
+ * drm_gpuvm_prepare_range() - prepare all BOs mapped within a given range
+ * @gpuvm: the &drm_gpuvm
+ * @exec: the &drm_exec locking context
+ * @addr: the start address within the VA space
+ * @range: the range to iterate within the VA space
+ * @num_fences: the amount of &dma_fences to reserve
+ *
+ * Calls drm_exec_prepare_obj() for all &drm_gem_objects mapped between @addr
+ * and @addr + @range.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+int
+drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, struct drm_exec *exec,
+ u64 addr, u64 range, unsigned int num_fences)
+{
+ struct drm_gpuva *va;
+ u64 end = addr + range;
+ int ret;
+
+ drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) {
+ struct drm_gem_object *obj = va->gem.obj;
+
+ ret = drm_exec_prepare_obj(exec, obj, num_fences);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_range);
+
+/**
+ * drm_gpuvm_exec_lock() - lock all dma-resv of all assoiciated BOs
+ * @vm_exec: the &drm_gpuvm_exec wrapper
+ *
+ * Acquires all dma-resv locks of all &drm_gem_objects the given
+ * &drm_gpuvm contains mappings of.
+ *
+ * Addionally, when calling this function with struct drm_gpuvm_exec::extra
+ * being set the driver receives the given @fn callback to lock additional
+ * dma-resv in the context of the &drm_gpuvm_exec instance. Typically, drivers
+ * would call drm_exec_prepare_obj() from within this callback.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+int
+drm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec)
+{
+ struct drm_gpuvm *gpuvm = vm_exec->vm;
+ struct drm_exec *exec = &vm_exec->exec;
+ unsigned int num_fences = vm_exec->num_fences;
+ int ret;
+
+ drm_exec_init(exec, vm_exec->flags);
+
+ drm_exec_until_all_locked(exec) {
+ ret = drm_gpuvm_prepare_vm(gpuvm, exec, num_fences);
+ drm_exec_retry_on_contention(exec);
+ if (ret)
+ goto err;
+
+ ret = drm_gpuvm_prepare_objects(gpuvm, exec, num_fences);
+ drm_exec_retry_on_contention(exec);
+ if (ret)
+ goto err;
+
+ if (vm_exec->extra.fn) {
+ ret = vm_exec->extra.fn(vm_exec);
+ drm_exec_retry_on_contention(exec);
+ if (ret)
+ goto err;
+ }
+ }
+
+ return 0;
+
+err:
+ drm_exec_fini(exec);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock);
+
+static int
+fn_lock_array(struct drm_gpuvm_exec *vm_exec)
+{
+ struct {
+ struct drm_gem_object **objs;
+ unsigned int num_objs;
+ } *args = vm_exec->extra.priv;
+
+ return drm_exec_prepare_array(&vm_exec->exec, args->objs,
+ args->num_objs, vm_exec->num_fences);
+}
+
+/**
+ * drm_gpuvm_exec_lock_array() - lock all dma-resv of all assoiciated BOs
+ * @vm_exec: the &drm_gpuvm_exec wrapper
+ * @objs: additional &drm_gem_objects to lock
+ * @num_objs: the number of additional &drm_gem_objects to lock
+ *
+ * Acquires all dma-resv locks of all &drm_gem_objects the given &drm_gpuvm
+ * contains mappings of, plus the ones given through @objs.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+int
+drm_gpuvm_exec_lock_array(struct drm_gpuvm_exec *vm_exec,
+ struct drm_gem_object **objs,
+ unsigned int num_objs)
+{
+ struct {
+ struct drm_gem_object **objs;
+ unsigned int num_objs;
+ } args;
+
+ args.objs = objs;
+ args.num_objs = num_objs;
+
+ vm_exec->extra.fn = fn_lock_array;
+ vm_exec->extra.priv = &args;
+
+ return drm_gpuvm_exec_lock(vm_exec);
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_array);
+
+/**
+ * drm_gpuvm_exec_lock_range() - prepare all BOs mapped within a given range
+ * @vm_exec: the &drm_gpuvm_exec wrapper
+ * @addr: the start address within the VA space
+ * @range: the range to iterate within the VA space
+ *
+ * Acquires all dma-resv locks of all &drm_gem_objects mapped between @addr and
+ * @addr + @range.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+int
+drm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec,
+ u64 addr, u64 range)
+{
+ struct drm_gpuvm *gpuvm = vm_exec->vm;
+ struct drm_exec *exec = &vm_exec->exec;
+ int ret;
+
+ drm_exec_init(exec, vm_exec->flags);
+
+ drm_exec_until_all_locked(exec) {
+ ret = drm_gpuvm_prepare_range(gpuvm, exec, addr, range,
+ vm_exec->num_fences);
+ drm_exec_retry_on_contention(exec);
+ if (ret)
+ goto err;
+ }
+
+ return ret;
+
+err:
+ drm_exec_fini(exec);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_range);
+
+static int
+__drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec)
+{
+ const struct drm_gpuvm_ops *ops = gpuvm->ops;
+ struct drm_gpuvm_bo *vm_bo;
+ LIST_HEAD(evict);
+ int ret = 0;
+
+ for_each_vm_bo_in_list(gpuvm, evict, &evict, vm_bo) {
+ ret = ops->vm_bo_validate(vm_bo, exec);
+ if (ret)
+ break;
+ }
+ /* Drop ref in case we break out of the loop. */
+ drm_gpuvm_bo_put(vm_bo);
+ restore_vm_bo_list(gpuvm, evict);
+
+ return ret;
+}
+
+static int
+drm_gpuvm_validate_locked(struct drm_gpuvm *gpuvm, struct drm_exec *exec)
+{
+ const struct drm_gpuvm_ops *ops = gpuvm->ops;
+ struct drm_gpuvm_bo *vm_bo, *next;
+ int ret = 0;
+
+ drm_gpuvm_resv_assert_held(gpuvm);
+
+ list_for_each_entry_safe(vm_bo, next, &gpuvm->evict.list,
+ list.entry.evict) {
+ ret = ops->vm_bo_validate(vm_bo, exec);
+ if (ret)
+ break;
+
+ dma_resv_assert_held(vm_bo->obj->resv);
+ if (!vm_bo->evicted)
+ drm_gpuvm_bo_list_del_init(vm_bo, evict, false);
+ }
+
+ return ret;
+}
+
+/**
+ * drm_gpuvm_validate() - validate all BOs marked as evicted
+ * @gpuvm: the &drm_gpuvm to validate evicted BOs
+ * @exec: the &drm_exec instance used for locking the GPUVM
+ *
+ * Calls the &drm_gpuvm_ops::vm_bo_validate callback for all evicted buffer
+ * objects being mapped in the given &drm_gpuvm.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+int
+drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec)
+{
+ const struct drm_gpuvm_ops *ops = gpuvm->ops;
+
+ if (unlikely(!ops || !ops->vm_bo_validate))
+ return -ENOTSUPP;
+
+ if (drm_gpuvm_resv_protected(gpuvm))
+ return drm_gpuvm_validate_locked(gpuvm, exec);
+ else
+ return __drm_gpuvm_validate(gpuvm, exec);
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_validate);
+
+/**
+ * drm_gpuvm_resv_add_fence - add fence to private and all extobj
+ * dma-resv
+ * @gpuvm: the &drm_gpuvm to add a fence to
+ * @exec: the &drm_exec locking context
+ * @fence: fence to add
+ * @private_usage: private dma-resv usage
+ * @extobj_usage: extobj dma-resv usage
+ */
+void
+drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm,
+ struct drm_exec *exec,
+ struct dma_fence *fence,
+ enum dma_resv_usage private_usage,
+ enum dma_resv_usage extobj_usage)
+{
+ struct drm_gem_object *obj;
+ unsigned long index;
+
+ drm_exec_for_each_locked_object(exec, index, obj) {
+ dma_resv_assert_held(obj->resv);
+ dma_resv_add_fence(obj->resv, fence,
+ drm_gpuvm_is_extobj(gpuvm, obj) ?
+ extobj_usage : private_usage);
+ }
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_resv_add_fence);
+
/**
* drm_gpuvm_bo_create() - create a new instance of struct drm_gpuvm_bo
* @gpuvm: The &drm_gpuvm the @obj is mapped in.
@@ -842,6 +1414,9 @@ drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm,
INIT_LIST_HEAD(&vm_bo->list.gpuva);
INIT_LIST_HEAD(&vm_bo->list.entry.gem);
+ INIT_LIST_HEAD(&vm_bo->list.entry.extobj);
+ INIT_LIST_HEAD(&vm_bo->list.entry.evict);
+
return vm_bo;
}
EXPORT_SYMBOL_GPL(drm_gpuvm_bo_create);
@@ -859,6 +1434,9 @@ drm_gpuvm_bo_destroy(struct kref *kref)
if (!lock)
drm_gpuvm_resv_assert_held(gpuvm);
+ drm_gpuvm_bo_list_del(vm_bo, extobj, lock);
+ drm_gpuvm_bo_list_del(vm_bo, evict, lock);
+
drm_gem_gpuva_assert_lock_held(obj);
list_del(&vm_bo->list.entry.gem);
@@ -997,6 +1575,61 @@ drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *__vm_bo)
}
EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain_prealloc);
+/**
+ * drm_gpuvm_bo_extobj_add() - adds the &drm_gpuvm_bo to its &drm_gpuvm's
+ * extobj list
+ * @vm_bo: The &drm_gpuvm_bo to add to its &drm_gpuvm's the extobj list.
+ *
+ * Adds the given @vm_bo to its &drm_gpuvm's extobj list if not on the list
+ * already and if the corresponding &drm_gem_object is an external object,
+ * actually.
+ */
+void
+drm_gpuvm_bo_extobj_add(struct drm_gpuvm_bo *vm_bo)
+{
+ struct drm_gpuvm *gpuvm = vm_bo->vm;
+ bool lock = !drm_gpuvm_resv_protected(gpuvm);
+
+ if (!lock)
+ drm_gpuvm_resv_assert_held(gpuvm);
+
+ if (drm_gpuvm_is_extobj(gpuvm, vm_bo->obj))
+ drm_gpuvm_bo_list_add(vm_bo, extobj, lock);
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_bo_extobj_add);
+
+/**
+ * drm_gpuvm_bo_evict() - add / remove a &drm_gpuvm_bo to / from the &drm_gpuvms
+ * evicted list
+ * @vm_bo: the &drm_gpuvm_bo to add or remove
+ * @evict: indicates whether the object is evicted
+ *
+ * Adds a &drm_gpuvm_bo to or removes it from the &drm_gpuvms evicted list.
+ */
+void
+drm_gpuvm_bo_evict(struct drm_gpuvm_bo *vm_bo, bool evict)
+{
+ struct drm_gpuvm *gpuvm = vm_bo->vm;
+ struct drm_gem_object *obj = vm_bo->obj;
+ bool lock = !drm_gpuvm_resv_protected(gpuvm);
+
+ dma_resv_assert_held(obj->resv);
+ vm_bo->evicted = evict;
+
+ /* Can't add external objects to the evicted list directly if not using
+ * internal spinlocks, since in this case the evicted list is protected
+ * with the VM's common dma-resv lock.
+ */
+ if (drm_gpuvm_is_extobj(gpuvm, obj) && !lock)
+ return;
+
+ if (evict)
+ drm_gpuvm_bo_list_add(vm_bo, evict, lock);
+ else
+ drm_gpuvm_bo_list_del_init(vm_bo, evict, lock);
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_bo_evict);
+
static int
__drm_gpuva_insert(struct drm_gpuvm *gpuvm,
struct drm_gpuva *va)
diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h
index 6e2893fe3808..bff5daa9962d 100644
--- a/include/drm/drm_gpuvm.h
+++ b/include/drm/drm_gpuvm.h
@@ -32,6 +32,7 @@
#include <drm/drm_device.h>
#include <drm/drm_gem.h>
+#include <drm/drm_exec.h>
struct drm_gpuvm;
struct drm_gpuvm_bo;
@@ -278,6 +279,50 @@ struct drm_gpuvm {
* @r_obj: Resv GEM object; representing the GPUVM's common &dma_resv.
*/
struct drm_gem_object *r_obj;
+
+ /**
+ * @extobj: structure holding the extobj list
+ */
+ struct {
+ /**
+ * @list: &list_head storing &drm_gpuvm_bos serving as
+ * external object
+ */
+ struct list_head list;
+
+ /**
+ * @local_list: pointer to the local list temporarily storing
+ * entries from the external object list
+ */
+ struct list_head *local_list;
+
+ /**
+ * @lock: spinlock to protect the extobj list
+ */
+ spinlock_t lock;
+ } extobj;
+
+ /**
+ * @evict: structure holding the evict list and evict list lock
+ */
+ struct {
+ /**
+ * @list: &list_head storing &drm_gpuvm_bos currently being
+ * evicted
+ */
+ struct list_head list;
+
+ /**
+ * @local_list: pointer to the local list temporarily storing
+ * entries from the evicted object list
+ */
+ struct list_head *local_list;
+
+ /**
+ * @lock: spinlock to protect the evict list
+ */
+ spinlock_t lock;
+ } evict;
};
void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
@@ -337,6 +382,22 @@ drm_gpuvm_resv_protected(struct drm_gpuvm *gpuvm)
#define drm_gpuvm_resv_assert_held(gpuvm__) \
dma_resv_assert_held(drm_gpuvm_resv(gpuvm__))
+/**
+ * drm_gpuvm_is_extobj() - indicates whether the given &drm_gem_object is an
+ * external object
+ * @gpuvm: the &drm_gpuvm to check
+ * @obj: the &drm_gem_object to check
+ *
+ * Returns: true if the &drm_gem_object &dma_resv differs from the
+ * &drm_gpuvms &dma_resv, false otherwise
+ */
+static inline bool
+drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm,
+ struct drm_gem_object *obj)
+{
+ return obj && obj->resv != drm_gpuvm_resv(gpuvm);
+}
+
static inline struct drm_gpuva *
__drm_gpuva_next(struct drm_gpuva *va)
{
@@ -415,6 +476,144 @@ __drm_gpuva_next(struct drm_gpuva *va)
#define drm_gpuvm_for_each_va_safe(va__, next__, gpuvm__) \
list_for_each_entry_safe(va__, next__, &(gpuvm__)->rb.list, rb.entry)
+/**
+ * struct drm_gpuvm_exec - &drm_gpuvm abstraction of &drm_exec
+ *
+ * This structure should be created on the stack as &drm_exec should be.
+ *
+ * Optionally, @extra can be set in order to lock additional &drm_gem_objects.
+ */
+struct drm_gpuvm_exec {
+ /**
+ * @exec: the &drm_exec structure
+ */
+ struct drm_exec exec;
+
+ /**
+ * @flags: the flags for the struct drm_exec
+ */
+ uint32_t flags;
+
+ /**
+ * @vm: the &drm_gpuvm to lock its DMA reservations
+ */
+ struct drm_gpuvm *vm;
+
+ /**
+ * @num_fences: the number of fences to reserve for the &dma_resv of the
+ * locked &drm_gem_objects
+ */
+ unsigned int num_fences;
+
+ /**
+ * @extra: Callback and corresponding private data for the driver to
+ * lock arbitrary additional &drm_gem_objects.
+ */
+ struct {
+ /**
+ * @fn: The driver callback to lock additional &drm_gem_objects.
+ */
+ int (*fn)(struct drm_gpuvm_exec *vm_exec);
+
+ /**
+ * @priv: driver private data for the @fn callback
+ */
+ void *priv;
+ } extra;
+};
+
+/**
+ * drm_gpuvm_prepare_vm() - prepare the GPUVMs common dma-resv
+ * @gpuvm: the &drm_gpuvm
+ * @exec: the &drm_exec context
+ * @num_fences: the amount of &dma_fences to reserve
+ *
+ * Calls drm_exec_prepare_obj() for the GPUVMs dummy &drm_gem_object.
+ *
+ * Using this function directly, it is the drivers responsibility to call
+ * drm_exec_init() and drm_exec_fini() accordingly.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+static inline int
+drm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm,
+ struct drm_exec *exec,
+ unsigned int num_fences)
+{
+ return drm_exec_prepare_obj(exec, gpuvm->r_obj, num_fences);
+}
+
+int drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm,
+ struct drm_exec *exec,
+ unsigned int num_fences);
+
+int drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm,
+ struct drm_exec *exec,
+ u64 addr, u64 range,
+ unsigned int num_fences);
+
+int drm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec);
+
+int drm_gpuvm_exec_lock_array(struct drm_gpuvm_exec *vm_exec,
+ struct drm_gem_object **objs,
+ unsigned int num_objs);
+
+int drm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec,
+ u64 addr, u64 range);
+
+/**
+ * drm_gpuvm_exec_unlock() - lock all dma-resv of all assoiciated BOs
+ * @vm_exec: the &drm_gpuvm_exec wrapper
+ *
+ * Releases all dma-resv locks of all &drm_gem_objects previously acquired
+ * through drm_gpuvm_exec_lock() or its variants.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+static inline void
+drm_gpuvm_exec_unlock(struct drm_gpuvm_exec *vm_exec)
+{
+ drm_exec_fini(&vm_exec->exec);
+}
+
+int drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec);
+void drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm,
+ struct drm_exec *exec,
+ struct dma_fence *fence,
+ enum dma_resv_usage private_usage,
+ enum dma_resv_usage extobj_usage);
+
+/**
+ * drm_gpuvm_exec_resv_add_fence()
+ * @vm_exec: the &drm_gpuvm_exec wrapper
+ * @fence: fence to add
+ * @private_usage: private dma-resv usage
+ * @extobj_usage: extobj dma-resv usage
+ *
+ * See drm_gpuvm_resv_add_fence().
+ */
+static inline void
+drm_gpuvm_exec_resv_add_fence(struct drm_gpuvm_exec *vm_exec,
+ struct dma_fence *fence,
+ enum dma_resv_usage private_usage,
+ enum dma_resv_usage extobj_usage)
+{
+ drm_gpuvm_resv_add_fence(vm_exec->vm, &vm_exec->exec, fence,
+ private_usage, extobj_usage);
+}
+
+/**
+ * drm_gpuvm_exec_validate()
+ * @vm_exec: the &drm_gpuvm_exec wrapper
+ *
+ * See drm_gpuvm_validate().
+ */
+static inline int
+drm_gpuvm_exec_validate(struct drm_gpuvm_exec *vm_exec)
+{
+ return drm_gpuvm_validate(vm_exec->vm, &vm_exec->exec);
+}
+
/**
* struct drm_gpuvm_bo - structure representing a &drm_gpuvm and
* &drm_gem_object combination
@@ -450,6 +649,12 @@ struct drm_gpuvm_bo {
*/
struct drm_gem_object *obj;
+ /**
+ * @evicted: Indicates whether the &drm_gem_object is evicted; field
+ * protected by the &drm_gem_object's dma-resv lock.
+ */
+ bool evicted;
+
/**
* @kref: The reference count for this &drm_gpuvm_bo.
*/
@@ -474,6 +679,18 @@ struct drm_gpuvm_bo {
* gpuva list.
*/
struct list_head gem;
+
+ /**
+ * @evict: List entry to attach to the &drm_gpuvms
+ * extobj list.
+ */
+ struct list_head extobj;
+
+ /**
+ * @evict: List entry to attach to the &drm_gpuvms evict
+ * list.
+ */
+ struct list_head evict;
} entry;
} list;
};
@@ -508,6 +725,27 @@ struct drm_gpuvm_bo *
drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,
struct drm_gem_object *obj);
+void drm_gpuvm_bo_evict(struct drm_gpuvm_bo *vm_bo, bool evict);
+
+/**
+ * drm_gpuvm_bo_gem_evict()
+ * @obj: the &drm_gem_object
+ * @evict: indicates whether @obj is evicted
+ *
+ * See drm_gpuvm_bo_evict().
+ */
+static inline void
+drm_gpuvm_bo_gem_evict(struct drm_gem_object *obj, bool evict)
+{
+ struct drm_gpuvm_bo *vm_bo;
+
+ drm_gem_gpuva_assert_lock_held(obj);
+ drm_gem_for_each_gpuvm_bo(vm_bo, obj)
+ drm_gpuvm_bo_evict(vm_bo, evict);
+}
+
+void drm_gpuvm_bo_extobj_add(struct drm_gpuvm_bo *vm_bo);
+
/**
* drm_gpuvm_bo_for_each_va() - iterator to walk over a list of &drm_gpuva
* @va__: &drm_gpuva structure to assign to in each iteration step
@@ -884,6 +1122,18 @@ struct drm_gpuvm_ops {
*/
void (*vm_bo_free)(struct drm_gpuvm_bo *vm_bo);
+ /**
+ * @vm_bo_validate: called from drm_gpuvm_validate()
+ *
+ * Drivers receive this callback for every evicted &drm_gem_object being
+ * mapped in the corresponding &drm_gpuvm.
+ *
+ * Typically, drivers would call their driver specific variant of
+ * ttm_bo_validate() from within this callback.
+ */
+ int (*vm_bo_validate)(struct drm_gpuvm_bo *vm_bo,
+ struct drm_exec *exec);
+
/**
* @sm_step_map: called from &drm_gpuvm_sm_map to finally insert the
* mapping once all previous steps were completed
--
2.41.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [Intel-xe] [CI 09/10] drm/xe: Adapt to GPUVM tracking of external / evicted objects.
2023-10-25 12:55 [Intel-xe] [CI 01/10] drm/gpuvm: convert WARN() to drm_WARN() variants Thomas Hellström
` (6 preceding siblings ...)
2023-10-25 12:55 ` [Intel-xe] [CI 08/10] drm/gpuvm: track/lock/validate external/evicted objects Thomas Hellström
@ 2023-10-25 12:55 ` Thomas Hellström
2023-10-25 12:55 ` [Intel-xe] [CI 10/10] drm/xe: Use DRM_GPUVM_RESV_PROTECTED for gpuvm Thomas Hellström
2023-10-25 14:14 ` [Intel-xe] ✗ CI.Patch_applied: failure for series starting with [CI,01/10] drm/gpuvm: convert WARN() to drm_WARN() variants Patchwork
9 siblings, 0 replies; 13+ messages in thread
From: Thomas Hellström @ 2023-10-25 12:55 UTC (permalink / raw)
To: intel-xe
Using fine-grained locking for now.
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/xe/xe_bo.c | 63 +++-----
drivers/gpu/drm/xe/xe_exec.c | 72 ++-------
drivers/gpu/drm/xe/xe_vm.c | 266 +++++++------------------------
drivers/gpu/drm/xe/xe_vm.h | 13 +-
drivers/gpu/drm/xe/xe_vm_types.h | 36 -----
5 files changed, 103 insertions(+), 347 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 82381de4876b..cc9984e827b9 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -450,9 +450,9 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
{
struct dma_resv_iter cursor;
struct dma_fence *fence;
- struct drm_gpuva *gpuva;
struct drm_gem_object *obj = &bo->ttm.base;
struct drm_gpuvm_bo *vm_bo;
+ bool idle = false;
int ret = 0;
dma_resv_assert_held(bo->ttm.base.resv);
@@ -466,14 +466,15 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
}
drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
- drm_gpuvm_bo_for_each_va(gpuva, vm_bo) {
- struct xe_vma *vma = gpuva_to_vma(gpuva);
- struct xe_vm *vm = xe_vma_vm(vma);
+ struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm);
+ struct drm_gpuva *gpuva;
- trace_xe_vma_evict(vma);
+ if (!xe_vm_in_fault_mode(vm)) {
+ drm_gpuvm_bo_evict(vm_bo, true);
+ continue;
+ }
- if (xe_vm_in_fault_mode(vm)) {
- /* Wait for pending binds / unbinds. */
+ if (!idle) {
long timeout;
if (ctx->no_wait_gpu &&
@@ -485,45 +486,21 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
DMA_RESV_USAGE_BOOKKEEP,
ctx->interruptible,
MAX_SCHEDULE_TIMEOUT);
- if (timeout > 0) {
- ret = xe_vm_invalidate_vma(vma);
- XE_WARN_ON(ret);
- } else if (!timeout) {
- ret = -ETIME;
- } else {
- ret = timeout;
- }
-
- } else {
- bool vm_resv_locked = false;
+ if (!timeout)
+ return -ETIME;
+ if (timeout < 0)
+ return timeout;
- /*
- * We need to put the vma on the vm's rebind_list,
- * but need the vm resv to do so. If we can't verify
- * that we indeed have it locked, put the vma an the
- * vm's notifier.rebind_list instead and scoop later.
- */
- if (dma_resv_trylock(xe_vm_resv(vm)))
- vm_resv_locked = true;
- else if (ctx->resv != xe_vm_resv(vm)) {
- spin_lock(&vm->notifier.list_lock);
- if (!(vma->gpuva.flags & XE_VMA_DESTROYED))
- list_move_tail(&vma->notifier.rebind_link,
- &vm->notifier.rebind_list);
- spin_unlock(&vm->notifier.list_lock);
- continue;
- }
+ idle = true;
+ }
- xe_vm_assert_held(vm);
- if (vma->tile_present &&
- !(vma->gpuva.flags & XE_VMA_DESTROYED) &&
- list_empty(&vma->combined_links.rebind))
- list_add_tail(&vma->combined_links.rebind,
- &vm->rebind_list);
+ drm_gpuvm_bo_for_each_va(gpuva, vm_bo) {
+ struct xe_vma *vma = gpuva_to_vma(gpuva);
- if (vm_resv_locked)
- dma_resv_unlock(xe_vm_resv(vm));
- }
+ trace_xe_vma_evict(vma);
+ ret = xe_vm_invalidate_vma(vma);
+ if (XE_WARN_ON(ret))
+ return ret;
}
}
diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c
index 85a8a793f527..cd5d2d2257f5 100644
--- a/drivers/gpu/drm/xe/xe_exec.c
+++ b/drivers/gpu/drm/xe/xe_exec.c
@@ -94,40 +94,9 @@
* Unlock all
*/
-static int xe_exec_begin(struct drm_exec *exec, struct xe_vm *vm)
+static int xe_exec_fn(struct drm_gpuvm_exec *vm_exec)
{
- struct xe_vma *vma;
- LIST_HEAD(dups);
- int err = 0;
-
- if (xe_vm_no_dma_fences(vm))
- return 0;
-
- /*
- * 1 fence for job from exec plus a fence for each tile from a possible
- * rebind
- */
- err = xe_vm_lock_dma_resv(vm, exec, 1 + vm->xe->info.tile_count, true);
- if (err)
- return err;
-
- /*
- * Validate BOs that have been evicted (i.e. make sure the
- * BOs have valid placements possibly moving an evicted BO back
- * to a location where the GPU can access it).
- */
- list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) {
- xe_assert(vm->xe, !xe_vma_is_null(vma));
-
- if (xe_vma_is_userptr(vma))
- continue;
-
- err = xe_bo_validate(xe_vma_bo(vma), vm, false);
- if (err)
- break;
- }
-
- return err;
+ return drm_gpuvm_validate(vm_exec->vm, &vm_exec->exec);
}
int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
@@ -140,7 +109,8 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
struct xe_exec_queue *q;
struct xe_sync_entry *syncs = NULL;
u64 addresses[XE_HW_ENGINE_MAX_INSTANCE];
- struct drm_exec exec;
+ struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn};
+ struct drm_exec *exec = &vm_exec.exec;
u32 i, num_syncs = 0;
struct xe_sched_job *job;
struct dma_fence *rebind_fence;
@@ -216,16 +186,14 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
goto err_unlock_list;
}
- drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
- drm_exec_until_all_locked(&exec) {
- err = xe_exec_begin(&exec, vm);
- drm_exec_retry_on_contention(&exec);
- if (err && xe_vm_validate_should_retry(&exec, err, &end)) {
+ vm_exec.vm = &vm->gpuvm;
+ vm_exec.num_fences = 1 + vm->xe->info.tile_count;
+ vm_exec.flags = DRM_EXEC_INTERRUPTIBLE_WAIT;
+ err = drm_gpuvm_exec_lock(&vm_exec);
+ if (err) {
+ if (xe_vm_validate_should_retry(exec, err, &end))
err = -EAGAIN;
- goto err_unlock_list;
- }
- if (err)
- goto err_exec;
+ goto err_unlock_list;
}
if (xe_vm_is_closed_or_banned(q->vm)) {
@@ -307,19 +275,9 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
* the job and let the DRM scheduler / backend clean up the job.
*/
xe_sched_job_arm(job);
- if (!xe_vm_no_dma_fences(vm)) {
- /* Block userptr invalidations / BO eviction */
- dma_resv_add_fence(xe_vm_resv(vm),
- &job->drm.s_fence->finished,
- DMA_RESV_USAGE_BOOKKEEP);
-
- /*
- * Make implicit sync work across drivers, assuming all external
- * BOs are written as we don't pass in a read / write list.
- */
- xe_vm_fence_all_extobjs(vm, &job->drm.s_fence->finished,
- DMA_RESV_USAGE_WRITE);
- }
+ if (!xe_vm_no_dma_fences(vm))
+ drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, &job->drm.s_fence->finished,
+ DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_WRITE);
for (i = 0; i < num_syncs; i++)
xe_sync_entry_signal(&syncs[i], job,
@@ -343,7 +301,7 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
if (err)
xe_sched_job_put(job);
err_exec:
- drm_exec_fini(&exec);
+ drm_exec_fini(exec);
err_unlock_list:
if (write_locked)
up_write(&vm->lock);
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 1b94736ef430..fa721822199a 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -297,26 +297,8 @@ static int add_preempt_fences(struct xe_vm *vm, struct xe_bo *bo)
return err;
}
-/**
- * xe_vm_fence_all_extobjs() - Add a fence to vm's external objects' resv
- * @vm: The vm.
- * @fence: The fence to add.
- * @usage: The resv usage for the fence.
- *
- * Loops over all of the vm's external object bindings and adds a @fence
- * with the given @usage to all of the external object's reservation
- * objects.
- */
-void xe_vm_fence_all_extobjs(struct xe_vm *vm, struct dma_fence *fence,
- enum dma_resv_usage usage)
-{
- struct xe_vma *vma;
-
- list_for_each_entry(vma, &vm->extobj.list, extobj.link)
- dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, usage);
-}
-
-static void resume_and_reinstall_preempt_fences(struct xe_vm *vm)
+static void resume_and_reinstall_preempt_fences(struct xe_vm *vm,
+ struct drm_exec *exec)
{
struct xe_exec_queue *q;
@@ -326,16 +308,19 @@ static void resume_and_reinstall_preempt_fences(struct xe_vm *vm)
list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) {
q->ops->resume(q);
- dma_resv_add_fence(xe_vm_resv(vm), q->compute.pfence,
- DMA_RESV_USAGE_BOOKKEEP);
- xe_vm_fence_all_extobjs(vm, q->compute.pfence,
- DMA_RESV_USAGE_BOOKKEEP);
+ drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, q->compute.pfence,
+ DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP);
}
}
int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
{
- struct drm_exec exec;
+ struct drm_gpuvm_exec vm_exec = {
+ .vm = &vm->gpuvm,
+ .flags = DRM_EXEC_INTERRUPTIBLE_WAIT,
+ .num_fences = 1,
+ };
+ struct drm_exec *exec = &vm_exec.exec;
struct dma_fence *pfence;
int err;
bool wait;
@@ -343,13 +328,9 @@ int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
xe_assert(vm->xe, xe_vm_in_compute_mode(vm));
down_write(&vm->lock);
- drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
- drm_exec_until_all_locked(&exec) {
- err = xe_vm_lock_dma_resv(vm, &exec, 1, true);
- drm_exec_retry_on_contention(&exec);
- if (err)
- goto out_unlock;
- }
+ err = drm_gpuvm_exec_lock(&vm_exec);
+ if (err)
+ return err;
pfence = xe_preempt_fence_create(q, q->compute.context,
++q->compute.seqno);
@@ -364,10 +345,8 @@ int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
down_read(&vm->userptr.notifier_lock);
- dma_resv_add_fence(xe_vm_resv(vm), pfence,
- DMA_RESV_USAGE_BOOKKEEP);
-
- xe_vm_fence_all_extobjs(vm, pfence, DMA_RESV_USAGE_BOOKKEEP);
+ drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, pfence,
+ DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP);
/*
* Check to see if a preemption on VM is in flight or userptr
@@ -381,7 +360,7 @@ int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
up_read(&vm->userptr.notifier_lock);
out_unlock:
- drm_exec_fini(&exec);
+ drm_exec_fini(exec);
up_write(&vm->lock);
return err;
@@ -427,55 +406,6 @@ int __xe_vm_userptr_needs_repin(struct xe_vm *vm)
list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN;
}
-/**
- * xe_vm_lock_dma_resv() - Lock the vm dma_resv object and the dma_resv
- * objects of the vm's external buffer objects.
- * @vm: The vm.
- * @exec: Pointer to a struct drm_exec locking context.
- * @num_shared: Number of dma-fence slots to reserve in the locked objects.
- * @lock_vm: Lock also the vm's dma_resv.
- *
- * Locks the vm dma-resv objects and all the dma-resv objects of the
- * buffer objects on the vm external object list.
- *
- * Return: 0 on success, Negative error code on error. In particular if
- * @intr is set to true, -EINTR or -ERESTARTSYS may be returned.
- */
-int xe_vm_lock_dma_resv(struct xe_vm *vm, struct drm_exec *exec,
- unsigned int num_shared, bool lock_vm)
-{
- struct xe_vma *vma, *next;
- int err = 0;
-
- lockdep_assert_held(&vm->lock);
-
- if (lock_vm) {
- err = drm_exec_prepare_obj(exec, xe_vm_obj(vm), num_shared);
- if (err)
- return err;
- }
-
- list_for_each_entry(vma, &vm->extobj.list, extobj.link) {
- err = drm_exec_prepare_obj(exec, &xe_vma_bo(vma)->ttm.base, num_shared);
- if (err)
- return err;
- }
-
- spin_lock(&vm->notifier.list_lock);
- list_for_each_entry_safe(vma, next, &vm->notifier.rebind_list,
- notifier.rebind_link) {
- xe_bo_assert_held(xe_vma_bo(vma));
-
- list_del_init(&vma->notifier.rebind_link);
- if (vma->tile_present && !(vma->gpuva.flags & XE_VMA_DESTROYED))
- list_move_tail(&vma->combined_links.rebind,
- &vm->rebind_list);
- }
- spin_unlock(&vm->notifier.list_lock);
-
- return 0;
-}
-
#define XE_VM_REBIND_RETRY_TIMEOUT_MS 1000
static void xe_vm_kill(struct xe_vm *vm)
@@ -524,30 +454,37 @@ bool xe_vm_validate_should_retry(struct drm_exec *exec, int err, ktime_t *end)
if (!ktime_before(cur, *end))
return false;
- /*
- * We would like to keep the ticket here with
- * drm_exec_unlock_all(), but WW mutex asserts currently
- * stop us from that. In any case this function could go away
- * with proper TTM -EDEADLK handling.
- */
- drm_exec_fini(exec);
-
msleep(20);
return true;
}
+static int xe_gpuvm_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec)
+{
+ struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm);
+ struct drm_gpuva *gpuva;
+ int ret;
+
+ drm_gpuvm_bo_for_each_va(gpuva, vm_bo)
+ list_move_tail(&gpuva_to_vma(gpuva)->combined_links.rebind,
+ &vm->rebind_list);
+
+ ret = xe_bo_validate(gem_to_xe_bo(vm_bo->obj), vm, false);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
static int xe_preempt_work_begin(struct drm_exec *exec, struct xe_vm *vm,
bool *done)
{
- struct xe_vma *vma;
int err;
/*
* 1 fence for each preempt fence plus a fence for each tile from a
* possible rebind
*/
- err = drm_exec_prepare_obj(exec, xe_vm_obj(vm),
- vm->preempt.num_exec_queues +
+ err = drm_gpuvm_prepare_vm(&vm->gpuvm, exec, vm->preempt.num_exec_queues +
vm->xe->info.tile_count);
if (err)
return err;
@@ -563,7 +500,7 @@ static int xe_preempt_work_begin(struct drm_exec *exec, struct xe_vm *vm,
return 0;
}
- err = xe_vm_lock_dma_resv(vm, exec, vm->preempt.num_exec_queues, false);
+ err = drm_gpuvm_prepare_objects(&vm->gpuvm, exec, vm->preempt.num_exec_queues);
if (err)
return err;
@@ -571,17 +508,7 @@ static int xe_preempt_work_begin(struct drm_exec *exec, struct xe_vm *vm,
if (err)
return err;
- list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) {
- if (xe_vma_has_no_bo(vma) ||
- vma->gpuva.flags & XE_VMA_DESTROYED)
- continue;
-
- err = xe_bo_validate(xe_vma_bo(vma), vm, false);
- if (err)
- break;
- }
-
- return err;
+ return drm_gpuvm_validate(&vm->gpuvm, exec);
}
static void preempt_rebind_work_func(struct work_struct *w)
@@ -621,12 +548,13 @@ static void preempt_rebind_work_func(struct work_struct *w)
err = xe_preempt_work_begin(&exec, vm, &done);
drm_exec_retry_on_contention(&exec);
- if (err && xe_vm_validate_should_retry(&exec, err, &end)) {
- err = -EAGAIN;
+ if (err || done) {
+ drm_exec_fini(&exec);
+ if (err && xe_vm_validate_should_retry(&exec, err, &end))
+ err = -EAGAIN;
+
goto out_unlock_outer;
}
- if (err || done)
- goto out_unlock;
}
err = alloc_preempt_fences(vm, &preempt_fences, &fence_count);
@@ -673,7 +601,7 @@ static void preempt_rebind_work_func(struct work_struct *w)
/* Point of no return. */
arm_preempt_fences(vm, &preempt_fences);
- resume_and_reinstall_preempt_fences(vm);
+ resume_and_reinstall_preempt_fences(vm, &exec);
up_read(&vm->userptr.notifier_lock);
out_unlock:
@@ -888,8 +816,6 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm,
}
INIT_LIST_HEAD(&vma->combined_links.rebind);
- INIT_LIST_HEAD(&vma->notifier.rebind_link);
- INIT_LIST_HEAD(&vma->extobj.link);
INIT_LIST_HEAD(&vma->gpuva.gem.entry);
vma->gpuva.vm = &vm->gpuvm;
@@ -921,6 +847,7 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm,
return ERR_CAST(vm_bo);
}
+ drm_gpuvm_bo_extobj_add(vm_bo);
drm_gem_object_get(&bo->ttm.base);
vma->gpuva.gem.obj = &bo->ttm.base;
vma->gpuva.gem.offset = bo_offset_or_userptr;
@@ -953,16 +880,6 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm,
return vma;
}
-static bool vm_remove_extobj(struct xe_vma *vma)
-{
- if (!list_empty(&vma->extobj.link)) {
- xe_vma_vm(vma)->extobj.entries--;
- list_del_init(&vma->extobj.link);
- return true;
- }
- return false;
-}
-
static void xe_vma_destroy_late(struct xe_vma *vma)
{
struct xe_vm *vm = xe_vma_vm(vma);
@@ -1003,60 +920,6 @@ static void vma_destroy_work_func(struct work_struct *w)
xe_vma_destroy_late(vma);
}
-static struct xe_vma *
-bo_has_vm_references_locked(struct xe_bo *bo, struct xe_vm *vm,
- struct xe_vma *ignore)
-{
- struct drm_gpuvm_bo *vm_bo;
- struct drm_gpuva *va;
- struct drm_gem_object *obj = &bo->ttm.base;
-
- xe_bo_assert_held(bo);
-
- drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
- drm_gpuvm_bo_for_each_va(va, vm_bo) {
- struct xe_vma *vma = gpuva_to_vma(va);
-
- if (vma != ignore && xe_vma_vm(vma) == vm)
- return vma;
- }
- }
-
- return NULL;
-}
-
-static bool bo_has_vm_references(struct xe_bo *bo, struct xe_vm *vm,
- struct xe_vma *ignore)
-{
- bool ret;
-
- xe_bo_lock(bo, false);
- ret = !!bo_has_vm_references_locked(bo, vm, ignore);
- xe_bo_unlock(bo);
-
- return ret;
-}
-
-static void __vm_insert_extobj(struct xe_vm *vm, struct xe_vma *vma)
-{
- lockdep_assert_held_write(&vm->lock);
-
- list_add(&vma->extobj.link, &vm->extobj.list);
- vm->extobj.entries++;
-}
-
-static void vm_insert_extobj(struct xe_vm *vm, struct xe_vma *vma)
-{
- struct xe_bo *bo = xe_vma_bo(vma);
-
- lockdep_assert_held_write(&vm->lock);
-
- if (bo_has_vm_references(bo, vm, vma))
- return;
-
- __vm_insert_extobj(vm, vma);
-}
-
static void vma_destroy_cb(struct dma_fence *fence,
struct dma_fence_cb *cb)
{
@@ -1082,20 +945,7 @@ static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence)
} else if (!xe_vma_is_null(vma)) {
xe_bo_assert_held(xe_vma_bo(vma));
- spin_lock(&vm->notifier.list_lock);
- list_del(&vma->notifier.rebind_link);
- spin_unlock(&vm->notifier.list_lock);
-
drm_gpuva_unlink(&vma->gpuva);
-
- if (!xe_vma_bo(vma)->vm && vm_remove_extobj(vma)) {
- struct xe_vma *other;
-
- other = bo_has_vm_references_locked(xe_vma_bo(vma), vm, NULL);
-
- if (other)
- __vm_insert_extobj(vm, other);
- }
}
xe_vm_assert_held(vm);
@@ -1211,6 +1061,7 @@ static struct drm_gpuva_op *xe_vm_op_alloc(void)
static struct drm_gpuvm_ops gpuvm_ops = {
.op_alloc = xe_vm_op_alloc,
+ .vm_bo_validate = xe_gpuvm_validate,
};
static u64 pde_encode_pat_index(struct xe_device *xe, u16 pat_index)
@@ -1373,9 +1224,6 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
init_rwsem(&vm->userptr.notifier_lock);
spin_lock_init(&vm->userptr.invalidated_lock);
- INIT_LIST_HEAD(&vm->notifier.rebind_list);
- spin_lock_init(&vm->notifier.list_lock);
-
INIT_WORK(&vm->destroy_work, vm_destroy_work_func);
INIT_LIST_HEAD(&vm->preempt.exec_queues);
@@ -1384,8 +1232,6 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
for_each_tile(tile, xe, id)
xe_range_fence_tree_init(&vm->rftree[id]);
- INIT_LIST_HEAD(&vm->extobj.list);
-
vm->pt_ops = &xelp_pt_ops;
if (!(flags & XE_VM_FLAG_MIGRATION))
@@ -1612,7 +1458,6 @@ void xe_vm_close_and_put(struct xe_vm *vm)
xe_vma_destroy_unlocked(vma);
}
- xe_assert(xe, list_empty(&vm->extobj.list));
up_write(&vm->lock);
mutex_lock(&xe->usm.lock);
@@ -2286,22 +2131,36 @@ static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op,
u8 tile_mask, bool read_only, bool is_null)
{
struct xe_bo *bo = op->gem.obj ? gem_to_xe_bo(op->gem.obj) : NULL;
+ struct drm_exec exec;
struct xe_vma *vma;
int err;
lockdep_assert_held_write(&vm->lock);
if (bo) {
- err = xe_bo_lock(bo, true);
- if (err)
- return ERR_PTR(err);
+ drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
+ drm_exec_until_all_locked(&exec) {
+ err = 0;
+ if (!bo->vm) {
+ err = drm_exec_lock_obj(&exec, xe_vm_obj(vm));
+ drm_exec_retry_on_contention(&exec);
+ }
+ if (!err) {
+ err = drm_exec_lock_obj(&exec, &bo->ttm.base);
+ drm_exec_retry_on_contention(&exec);
+ }
+ if (err) {
+ drm_exec_fini(&exec);
+ return ERR_PTR(err);
+ }
+ }
}
vma = xe_vma_create(vm, bo, op->gem.offset,
op->va.addr, op->va.addr +
op->va.range - 1, read_only, is_null,
tile_mask);
if (bo)
- xe_bo_unlock(bo);
+ drm_exec_fini(&exec);
if (xe_vma_is_userptr(vma)) {
err = xe_vma_userptr_pin_pages(vma);
@@ -2311,7 +2170,6 @@ static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op,
return ERR_PTR(err);
}
} else if (!xe_vma_has_no_bo(vma) && !bo->vm) {
- vm_insert_extobj(vm, vma);
err = add_preempt_fences(vm, bo);
if (err) {
prep_vma_destroy(vm, vma, false);
diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h
index 88077214261f..20b320aaeaa0 100644
--- a/drivers/gpu/drm/xe/xe_vm.h
+++ b/drivers/gpu/drm/xe/xe_vm.h
@@ -64,9 +64,14 @@ static inline bool xe_vm_is_closed_or_banned(struct xe_vm *vm)
struct xe_vma *
xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range);
+static inline struct xe_vm *gpuvm_to_vm(struct drm_gpuvm *gpuvm)
+{
+ return container_of(gpuvm, struct xe_vm, gpuvm);
+}
+
static inline struct xe_vm *gpuva_to_vm(struct drm_gpuva *gpuva)
{
- return container_of(gpuva->vm, struct xe_vm, gpuvm);
+ return gpuvm_to_vm(gpuva->vm);
}
static inline struct xe_vma *gpuva_to_vma(struct drm_gpuva *gpuva)
@@ -209,12 +214,6 @@ int xe_vma_userptr_check_repin(struct xe_vma *vma);
bool xe_vm_validate_should_retry(struct drm_exec *exec, int err, ktime_t *end);
-int xe_vm_lock_dma_resv(struct xe_vm *vm, struct drm_exec *exec,
- unsigned int num_shared, bool lock_vm);
-
-void xe_vm_fence_all_extobjs(struct xe_vm *vm, struct dma_fence *fence,
- enum dma_resv_usage usage);
-
int xe_analyze_vm(struct drm_printer *p, struct xe_vm *vm, int gt_id);
int xe_vm_prepare_vma(struct drm_exec *exec, struct xe_vma *vma,
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index f11ac81cf010..bff193920d25 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -110,18 +110,6 @@ struct xe_vma {
*/
u8 tile_present;
- struct {
- struct list_head rebind_link;
- } notifier;
-
- struct {
- /**
- * @extobj.link: Link into vm's external object list.
- * protected by the vm lock.
- */
- struct list_head link;
- } extobj;
-
/**
* @userptr: user pointer state, only allocated for VMAs that are
* user pointers
@@ -200,14 +188,6 @@ struct xe_vm {
*/
struct xe_range_fence_tree rftree[XE_MAX_TILES_PER_DEVICE];
- /** @extobj: bookkeeping for external objects. Protected by the vm lock */
- struct {
- /** @enties: number of external BOs attached this VM */
- u32 entries;
- /** @list: list of vmas with external bos attached */
- struct list_head list;
- } extobj;
-
/** @async_ops: async VM operations (bind / unbinds) */
struct {
/** @list: list of pending async VM ops */
@@ -297,22 +277,6 @@ struct xe_vm {
struct xe_vma *last_fault_vma;
} usm;
- /**
- * @notifier: Lists and locks for temporary usage within notifiers where
- * we either can't grab the vm lock or the vm resv.
- */
- struct {
- /** @notifier.list_lock: lock protecting @rebind_list */
- spinlock_t list_lock;
- /**
- * @notifier.rebind_list: list of vmas that we want to put on the
- * main @rebind_list. This list is protected for writing by both
- * notifier.list_lock, and the resv of the bo the vma points to,
- * and for reading by the notifier.list_lock only.
- */
- struct list_head rebind_list;
- } notifier;
-
/** @error_capture: allow to track errors */
struct {
/** @capture_once: capture only one error per VM */
--
2.41.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [Intel-xe] [CI 10/10] drm/xe: Use DRM_GPUVM_RESV_PROTECTED for gpuvm
2023-10-25 12:55 [Intel-xe] [CI 01/10] drm/gpuvm: convert WARN() to drm_WARN() variants Thomas Hellström
` (7 preceding siblings ...)
2023-10-25 12:55 ` [Intel-xe] [CI 09/10] drm/xe: Adapt to GPUVM tracking of external / evicted objects Thomas Hellström
@ 2023-10-25 12:55 ` Thomas Hellström
2023-10-25 14:14 ` [Intel-xe] ✗ CI.Patch_applied: failure for series starting with [CI,01/10] drm/gpuvm: convert WARN() to drm_WARN() variants Patchwork
9 siblings, 0 replies; 13+ messages in thread
From: Thomas Hellström @ 2023-10-25 12:55 UTC (permalink / raw)
To: intel-xe
Use DRM_GPUVM_RESV_PROTECTED to relax locking.
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/xe/xe_vm.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index fa721822199a..37e98d347539 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -1243,8 +1243,8 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
goto err_no_resv;
}
- drm_gpuvm_init(&vm->gpuvm, "Xe VM", 0, &xe->drm, vm_resv_obj,
- 0, vm->size, 0, 0, &gpuvm_ops);
+ drm_gpuvm_init(&vm->gpuvm, "Xe VM", DRM_GPUVM_RESV_PROTECTED, &xe->drm,
+ vm_resv_obj, 0, vm->size, 0, 0, &gpuvm_ops);
drm_gem_object_put(vm_resv_obj);
--
2.41.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [Intel-xe] ✗ CI.Patch_applied: failure for series starting with [CI,01/10] drm/gpuvm: convert WARN() to drm_WARN() variants
2023-10-25 12:55 [Intel-xe] [CI 01/10] drm/gpuvm: convert WARN() to drm_WARN() variants Thomas Hellström
` (8 preceding siblings ...)
2023-10-25 12:55 ` [Intel-xe] [CI 10/10] drm/xe: Use DRM_GPUVM_RESV_PROTECTED for gpuvm Thomas Hellström
@ 2023-10-25 14:14 ` Patchwork
9 siblings, 0 replies; 13+ messages in thread
From: Patchwork @ 2023-10-25 14:14 UTC (permalink / raw)
To: Thomas Hellström; +Cc: intel-xe
== Series Details ==
Series: series starting with [CI,01/10] drm/gpuvm: convert WARN() to drm_WARN() variants
URL : https://patchwork.freedesktop.org/series/125572/
State : failure
== Summary ==
=== Applying kernel patches on branch 'drm-xe-next' with base: ===
Base commit: bbcc0af8b drm/xe: Add performance tuning settings for MTL and Xe2
=== git am output follows ===
Applying: drm/gpuvm: convert WARN() to drm_WARN() variants
Applying: drm/gpuvm: add common dma-resv per struct drm_gpuvm
Applying: drm/xe: Adjust to "drm/gpuvm: add common dma-resv per struct drm_gpuvm"
Applying: drm/gpuvm: add drm_gpuvm_flags to drm_gpuvm
Applying: drm/xe: Adjust to commit "drm/gpuvm: add drm_gpuvm_flags to drm_gpuvm"
Applying: drm/gpuvm: add an abstraction for a VM / BO combination
Applying: drm/xe: Adjust to commit "drm/gpuvm: add an abstraction for a VM/BO combination"
Applying: drm/gpuvm: track/lock/validate external/evicted objects
Applying: drm/xe: Adapt to GPUVM tracking of external / evicted objects.
Applying: drm/xe: Use DRM_GPUVM_RESV_PROTECTED for gpuvm
^ permalink raw reply [flat|nested] 13+ messages in thread
* [Intel-xe] [CI 09/10] drm/xe: Adapt to GPUVM tracking of external / evicted objects.
2023-10-26 10:15 [Intel-xe] [CI 01/10] " Thomas Hellström
@ 2023-10-26 10:15 ` Thomas Hellström
0 siblings, 0 replies; 13+ messages in thread
From: Thomas Hellström @ 2023-10-26 10:15 UTC (permalink / raw)
To: intel-xe
Using fine-grained locking for now.
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/xe/xe_bo.c | 63 +++-----
drivers/gpu/drm/xe/xe_exec.c | 72 ++-------
drivers/gpu/drm/xe/xe_vm.c | 266 +++++++------------------------
drivers/gpu/drm/xe/xe_vm.h | 13 +-
drivers/gpu/drm/xe/xe_vm_types.h | 36 -----
5 files changed, 103 insertions(+), 347 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 82381de4876b..cc9984e827b9 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -450,9 +450,9 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
{
struct dma_resv_iter cursor;
struct dma_fence *fence;
- struct drm_gpuva *gpuva;
struct drm_gem_object *obj = &bo->ttm.base;
struct drm_gpuvm_bo *vm_bo;
+ bool idle = false;
int ret = 0;
dma_resv_assert_held(bo->ttm.base.resv);
@@ -466,14 +466,15 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
}
drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
- drm_gpuvm_bo_for_each_va(gpuva, vm_bo) {
- struct xe_vma *vma = gpuva_to_vma(gpuva);
- struct xe_vm *vm = xe_vma_vm(vma);
+ struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm);
+ struct drm_gpuva *gpuva;
- trace_xe_vma_evict(vma);
+ if (!xe_vm_in_fault_mode(vm)) {
+ drm_gpuvm_bo_evict(vm_bo, true);
+ continue;
+ }
- if (xe_vm_in_fault_mode(vm)) {
- /* Wait for pending binds / unbinds. */
+ if (!idle) {
long timeout;
if (ctx->no_wait_gpu &&
@@ -485,45 +486,21 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
DMA_RESV_USAGE_BOOKKEEP,
ctx->interruptible,
MAX_SCHEDULE_TIMEOUT);
- if (timeout > 0) {
- ret = xe_vm_invalidate_vma(vma);
- XE_WARN_ON(ret);
- } else if (!timeout) {
- ret = -ETIME;
- } else {
- ret = timeout;
- }
-
- } else {
- bool vm_resv_locked = false;
+ if (!timeout)
+ return -ETIME;
+ if (timeout < 0)
+ return timeout;
- /*
- * We need to put the vma on the vm's rebind_list,
- * but need the vm resv to do so. If we can't verify
- * that we indeed have it locked, put the vma an the
- * vm's notifier.rebind_list instead and scoop later.
- */
- if (dma_resv_trylock(xe_vm_resv(vm)))
- vm_resv_locked = true;
- else if (ctx->resv != xe_vm_resv(vm)) {
- spin_lock(&vm->notifier.list_lock);
- if (!(vma->gpuva.flags & XE_VMA_DESTROYED))
- list_move_tail(&vma->notifier.rebind_link,
- &vm->notifier.rebind_list);
- spin_unlock(&vm->notifier.list_lock);
- continue;
- }
+ idle = true;
+ }
- xe_vm_assert_held(vm);
- if (vma->tile_present &&
- !(vma->gpuva.flags & XE_VMA_DESTROYED) &&
- list_empty(&vma->combined_links.rebind))
- list_add_tail(&vma->combined_links.rebind,
- &vm->rebind_list);
+ drm_gpuvm_bo_for_each_va(gpuva, vm_bo) {
+ struct xe_vma *vma = gpuva_to_vma(gpuva);
- if (vm_resv_locked)
- dma_resv_unlock(xe_vm_resv(vm));
- }
+ trace_xe_vma_evict(vma);
+ ret = xe_vm_invalidate_vma(vma);
+ if (XE_WARN_ON(ret))
+ return ret;
}
}
diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c
index 85a8a793f527..cd5d2d2257f5 100644
--- a/drivers/gpu/drm/xe/xe_exec.c
+++ b/drivers/gpu/drm/xe/xe_exec.c
@@ -94,40 +94,9 @@
* Unlock all
*/
-static int xe_exec_begin(struct drm_exec *exec, struct xe_vm *vm)
+static int xe_exec_fn(struct drm_gpuvm_exec *vm_exec)
{
- struct xe_vma *vma;
- LIST_HEAD(dups);
- int err = 0;
-
- if (xe_vm_no_dma_fences(vm))
- return 0;
-
- /*
- * 1 fence for job from exec plus a fence for each tile from a possible
- * rebind
- */
- err = xe_vm_lock_dma_resv(vm, exec, 1 + vm->xe->info.tile_count, true);
- if (err)
- return err;
-
- /*
- * Validate BOs that have been evicted (i.e. make sure the
- * BOs have valid placements possibly moving an evicted BO back
- * to a location where the GPU can access it).
- */
- list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) {
- xe_assert(vm->xe, !xe_vma_is_null(vma));
-
- if (xe_vma_is_userptr(vma))
- continue;
-
- err = xe_bo_validate(xe_vma_bo(vma), vm, false);
- if (err)
- break;
- }
-
- return err;
+ return drm_gpuvm_validate(vm_exec->vm, &vm_exec->exec);
}
int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
@@ -140,7 +109,8 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
struct xe_exec_queue *q;
struct xe_sync_entry *syncs = NULL;
u64 addresses[XE_HW_ENGINE_MAX_INSTANCE];
- struct drm_exec exec;
+ struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn};
+ struct drm_exec *exec = &vm_exec.exec;
u32 i, num_syncs = 0;
struct xe_sched_job *job;
struct dma_fence *rebind_fence;
@@ -216,16 +186,14 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
goto err_unlock_list;
}
- drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
- drm_exec_until_all_locked(&exec) {
- err = xe_exec_begin(&exec, vm);
- drm_exec_retry_on_contention(&exec);
- if (err && xe_vm_validate_should_retry(&exec, err, &end)) {
+ vm_exec.vm = &vm->gpuvm;
+ vm_exec.num_fences = 1 + vm->xe->info.tile_count;
+ vm_exec.flags = DRM_EXEC_INTERRUPTIBLE_WAIT;
+ err = drm_gpuvm_exec_lock(&vm_exec);
+ if (err) {
+ if (xe_vm_validate_should_retry(exec, err, &end))
err = -EAGAIN;
- goto err_unlock_list;
- }
- if (err)
- goto err_exec;
+ goto err_unlock_list;
}
if (xe_vm_is_closed_or_banned(q->vm)) {
@@ -307,19 +275,9 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
* the job and let the DRM scheduler / backend clean up the job.
*/
xe_sched_job_arm(job);
- if (!xe_vm_no_dma_fences(vm)) {
- /* Block userptr invalidations / BO eviction */
- dma_resv_add_fence(xe_vm_resv(vm),
- &job->drm.s_fence->finished,
- DMA_RESV_USAGE_BOOKKEEP);
-
- /*
- * Make implicit sync work across drivers, assuming all external
- * BOs are written as we don't pass in a read / write list.
- */
- xe_vm_fence_all_extobjs(vm, &job->drm.s_fence->finished,
- DMA_RESV_USAGE_WRITE);
- }
+ if (!xe_vm_no_dma_fences(vm))
+ drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, &job->drm.s_fence->finished,
+ DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_WRITE);
for (i = 0; i < num_syncs; i++)
xe_sync_entry_signal(&syncs[i], job,
@@ -343,7 +301,7 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
if (err)
xe_sched_job_put(job);
err_exec:
- drm_exec_fini(&exec);
+ drm_exec_fini(exec);
err_unlock_list:
if (write_locked)
up_write(&vm->lock);
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 1b94736ef430..fa721822199a 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -297,26 +297,8 @@ static int add_preempt_fences(struct xe_vm *vm, struct xe_bo *bo)
return err;
}
-/**
- * xe_vm_fence_all_extobjs() - Add a fence to vm's external objects' resv
- * @vm: The vm.
- * @fence: The fence to add.
- * @usage: The resv usage for the fence.
- *
- * Loops over all of the vm's external object bindings and adds a @fence
- * with the given @usage to all of the external object's reservation
- * objects.
- */
-void xe_vm_fence_all_extobjs(struct xe_vm *vm, struct dma_fence *fence,
- enum dma_resv_usage usage)
-{
- struct xe_vma *vma;
-
- list_for_each_entry(vma, &vm->extobj.list, extobj.link)
- dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, usage);
-}
-
-static void resume_and_reinstall_preempt_fences(struct xe_vm *vm)
+static void resume_and_reinstall_preempt_fences(struct xe_vm *vm,
+ struct drm_exec *exec)
{
struct xe_exec_queue *q;
@@ -326,16 +308,19 @@ static void resume_and_reinstall_preempt_fences(struct xe_vm *vm)
list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) {
q->ops->resume(q);
- dma_resv_add_fence(xe_vm_resv(vm), q->compute.pfence,
- DMA_RESV_USAGE_BOOKKEEP);
- xe_vm_fence_all_extobjs(vm, q->compute.pfence,
- DMA_RESV_USAGE_BOOKKEEP);
+ drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, q->compute.pfence,
+ DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP);
}
}
int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
{
- struct drm_exec exec;
+ struct drm_gpuvm_exec vm_exec = {
+ .vm = &vm->gpuvm,
+ .flags = DRM_EXEC_INTERRUPTIBLE_WAIT,
+ .num_fences = 1,
+ };
+ struct drm_exec *exec = &vm_exec.exec;
struct dma_fence *pfence;
int err;
bool wait;
@@ -343,13 +328,9 @@ int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
xe_assert(vm->xe, xe_vm_in_compute_mode(vm));
down_write(&vm->lock);
- drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
- drm_exec_until_all_locked(&exec) {
- err = xe_vm_lock_dma_resv(vm, &exec, 1, true);
- drm_exec_retry_on_contention(&exec);
- if (err)
- goto out_unlock;
- }
+ err = drm_gpuvm_exec_lock(&vm_exec);
+ if (err)
+ return err;
pfence = xe_preempt_fence_create(q, q->compute.context,
++q->compute.seqno);
@@ -364,10 +345,8 @@ int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
down_read(&vm->userptr.notifier_lock);
- dma_resv_add_fence(xe_vm_resv(vm), pfence,
- DMA_RESV_USAGE_BOOKKEEP);
-
- xe_vm_fence_all_extobjs(vm, pfence, DMA_RESV_USAGE_BOOKKEEP);
+ drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, pfence,
+ DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP);
/*
* Check to see if a preemption on VM is in flight or userptr
@@ -381,7 +360,7 @@ int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
up_read(&vm->userptr.notifier_lock);
out_unlock:
- drm_exec_fini(&exec);
+ drm_exec_fini(exec);
up_write(&vm->lock);
return err;
@@ -427,55 +406,6 @@ int __xe_vm_userptr_needs_repin(struct xe_vm *vm)
list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN;
}
-/**
- * xe_vm_lock_dma_resv() - Lock the vm dma_resv object and the dma_resv
- * objects of the vm's external buffer objects.
- * @vm: The vm.
- * @exec: Pointer to a struct drm_exec locking context.
- * @num_shared: Number of dma-fence slots to reserve in the locked objects.
- * @lock_vm: Lock also the vm's dma_resv.
- *
- * Locks the vm dma-resv objects and all the dma-resv objects of the
- * buffer objects on the vm external object list.
- *
- * Return: 0 on success, Negative error code on error. In particular if
- * @intr is set to true, -EINTR or -ERESTARTSYS may be returned.
- */
-int xe_vm_lock_dma_resv(struct xe_vm *vm, struct drm_exec *exec,
- unsigned int num_shared, bool lock_vm)
-{
- struct xe_vma *vma, *next;
- int err = 0;
-
- lockdep_assert_held(&vm->lock);
-
- if (lock_vm) {
- err = drm_exec_prepare_obj(exec, xe_vm_obj(vm), num_shared);
- if (err)
- return err;
- }
-
- list_for_each_entry(vma, &vm->extobj.list, extobj.link) {
- err = drm_exec_prepare_obj(exec, &xe_vma_bo(vma)->ttm.base, num_shared);
- if (err)
- return err;
- }
-
- spin_lock(&vm->notifier.list_lock);
- list_for_each_entry_safe(vma, next, &vm->notifier.rebind_list,
- notifier.rebind_link) {
- xe_bo_assert_held(xe_vma_bo(vma));
-
- list_del_init(&vma->notifier.rebind_link);
- if (vma->tile_present && !(vma->gpuva.flags & XE_VMA_DESTROYED))
- list_move_tail(&vma->combined_links.rebind,
- &vm->rebind_list);
- }
- spin_unlock(&vm->notifier.list_lock);
-
- return 0;
-}
-
#define XE_VM_REBIND_RETRY_TIMEOUT_MS 1000
static void xe_vm_kill(struct xe_vm *vm)
@@ -524,30 +454,37 @@ bool xe_vm_validate_should_retry(struct drm_exec *exec, int err, ktime_t *end)
if (!ktime_before(cur, *end))
return false;
- /*
- * We would like to keep the ticket here with
- * drm_exec_unlock_all(), but WW mutex asserts currently
- * stop us from that. In any case this function could go away
- * with proper TTM -EDEADLK handling.
- */
- drm_exec_fini(exec);
-
msleep(20);
return true;
}
+static int xe_gpuvm_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec)
+{
+ struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm);
+ struct drm_gpuva *gpuva;
+ int ret;
+
+ drm_gpuvm_bo_for_each_va(gpuva, vm_bo)
+ list_move_tail(&gpuva_to_vma(gpuva)->combined_links.rebind,
+ &vm->rebind_list);
+
+ ret = xe_bo_validate(gem_to_xe_bo(vm_bo->obj), vm, false);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
static int xe_preempt_work_begin(struct drm_exec *exec, struct xe_vm *vm,
bool *done)
{
- struct xe_vma *vma;
int err;
/*
* 1 fence for each preempt fence plus a fence for each tile from a
* possible rebind
*/
- err = drm_exec_prepare_obj(exec, xe_vm_obj(vm),
- vm->preempt.num_exec_queues +
+ err = drm_gpuvm_prepare_vm(&vm->gpuvm, exec, vm->preempt.num_exec_queues +
vm->xe->info.tile_count);
if (err)
return err;
@@ -563,7 +500,7 @@ static int xe_preempt_work_begin(struct drm_exec *exec, struct xe_vm *vm,
return 0;
}
- err = xe_vm_lock_dma_resv(vm, exec, vm->preempt.num_exec_queues, false);
+ err = drm_gpuvm_prepare_objects(&vm->gpuvm, exec, vm->preempt.num_exec_queues);
if (err)
return err;
@@ -571,17 +508,7 @@ static int xe_preempt_work_begin(struct drm_exec *exec, struct xe_vm *vm,
if (err)
return err;
- list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) {
- if (xe_vma_has_no_bo(vma) ||
- vma->gpuva.flags & XE_VMA_DESTROYED)
- continue;
-
- err = xe_bo_validate(xe_vma_bo(vma), vm, false);
- if (err)
- break;
- }
-
- return err;
+ return drm_gpuvm_validate(&vm->gpuvm, exec);
}
static void preempt_rebind_work_func(struct work_struct *w)
@@ -621,12 +548,13 @@ static void preempt_rebind_work_func(struct work_struct *w)
err = xe_preempt_work_begin(&exec, vm, &done);
drm_exec_retry_on_contention(&exec);
- if (err && xe_vm_validate_should_retry(&exec, err, &end)) {
- err = -EAGAIN;
+ if (err || done) {
+ drm_exec_fini(&exec);
+ if (err && xe_vm_validate_should_retry(&exec, err, &end))
+ err = -EAGAIN;
+
goto out_unlock_outer;
}
- if (err || done)
- goto out_unlock;
}
err = alloc_preempt_fences(vm, &preempt_fences, &fence_count);
@@ -673,7 +601,7 @@ static void preempt_rebind_work_func(struct work_struct *w)
/* Point of no return. */
arm_preempt_fences(vm, &preempt_fences);
- resume_and_reinstall_preempt_fences(vm);
+ resume_and_reinstall_preempt_fences(vm, &exec);
up_read(&vm->userptr.notifier_lock);
out_unlock:
@@ -888,8 +816,6 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm,
}
INIT_LIST_HEAD(&vma->combined_links.rebind);
- INIT_LIST_HEAD(&vma->notifier.rebind_link);
- INIT_LIST_HEAD(&vma->extobj.link);
INIT_LIST_HEAD(&vma->gpuva.gem.entry);
vma->gpuva.vm = &vm->gpuvm;
@@ -921,6 +847,7 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm,
return ERR_CAST(vm_bo);
}
+ drm_gpuvm_bo_extobj_add(vm_bo);
drm_gem_object_get(&bo->ttm.base);
vma->gpuva.gem.obj = &bo->ttm.base;
vma->gpuva.gem.offset = bo_offset_or_userptr;
@@ -953,16 +880,6 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm,
return vma;
}
-static bool vm_remove_extobj(struct xe_vma *vma)
-{
- if (!list_empty(&vma->extobj.link)) {
- xe_vma_vm(vma)->extobj.entries--;
- list_del_init(&vma->extobj.link);
- return true;
- }
- return false;
-}
-
static void xe_vma_destroy_late(struct xe_vma *vma)
{
struct xe_vm *vm = xe_vma_vm(vma);
@@ -1003,60 +920,6 @@ static void vma_destroy_work_func(struct work_struct *w)
xe_vma_destroy_late(vma);
}
-static struct xe_vma *
-bo_has_vm_references_locked(struct xe_bo *bo, struct xe_vm *vm,
- struct xe_vma *ignore)
-{
- struct drm_gpuvm_bo *vm_bo;
- struct drm_gpuva *va;
- struct drm_gem_object *obj = &bo->ttm.base;
-
- xe_bo_assert_held(bo);
-
- drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
- drm_gpuvm_bo_for_each_va(va, vm_bo) {
- struct xe_vma *vma = gpuva_to_vma(va);
-
- if (vma != ignore && xe_vma_vm(vma) == vm)
- return vma;
- }
- }
-
- return NULL;
-}
-
-static bool bo_has_vm_references(struct xe_bo *bo, struct xe_vm *vm,
- struct xe_vma *ignore)
-{
- bool ret;
-
- xe_bo_lock(bo, false);
- ret = !!bo_has_vm_references_locked(bo, vm, ignore);
- xe_bo_unlock(bo);
-
- return ret;
-}
-
-static void __vm_insert_extobj(struct xe_vm *vm, struct xe_vma *vma)
-{
- lockdep_assert_held_write(&vm->lock);
-
- list_add(&vma->extobj.link, &vm->extobj.list);
- vm->extobj.entries++;
-}
-
-static void vm_insert_extobj(struct xe_vm *vm, struct xe_vma *vma)
-{
- struct xe_bo *bo = xe_vma_bo(vma);
-
- lockdep_assert_held_write(&vm->lock);
-
- if (bo_has_vm_references(bo, vm, vma))
- return;
-
- __vm_insert_extobj(vm, vma);
-}
-
static void vma_destroy_cb(struct dma_fence *fence,
struct dma_fence_cb *cb)
{
@@ -1082,20 +945,7 @@ static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence)
} else if (!xe_vma_is_null(vma)) {
xe_bo_assert_held(xe_vma_bo(vma));
- spin_lock(&vm->notifier.list_lock);
- list_del(&vma->notifier.rebind_link);
- spin_unlock(&vm->notifier.list_lock);
-
drm_gpuva_unlink(&vma->gpuva);
-
- if (!xe_vma_bo(vma)->vm && vm_remove_extobj(vma)) {
- struct xe_vma *other;
-
- other = bo_has_vm_references_locked(xe_vma_bo(vma), vm, NULL);
-
- if (other)
- __vm_insert_extobj(vm, other);
- }
}
xe_vm_assert_held(vm);
@@ -1211,6 +1061,7 @@ static struct drm_gpuva_op *xe_vm_op_alloc(void)
static struct drm_gpuvm_ops gpuvm_ops = {
.op_alloc = xe_vm_op_alloc,
+ .vm_bo_validate = xe_gpuvm_validate,
};
static u64 pde_encode_pat_index(struct xe_device *xe, u16 pat_index)
@@ -1373,9 +1224,6 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
init_rwsem(&vm->userptr.notifier_lock);
spin_lock_init(&vm->userptr.invalidated_lock);
- INIT_LIST_HEAD(&vm->notifier.rebind_list);
- spin_lock_init(&vm->notifier.list_lock);
-
INIT_WORK(&vm->destroy_work, vm_destroy_work_func);
INIT_LIST_HEAD(&vm->preempt.exec_queues);
@@ -1384,8 +1232,6 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
for_each_tile(tile, xe, id)
xe_range_fence_tree_init(&vm->rftree[id]);
- INIT_LIST_HEAD(&vm->extobj.list);
-
vm->pt_ops = &xelp_pt_ops;
if (!(flags & XE_VM_FLAG_MIGRATION))
@@ -1612,7 +1458,6 @@ void xe_vm_close_and_put(struct xe_vm *vm)
xe_vma_destroy_unlocked(vma);
}
- xe_assert(xe, list_empty(&vm->extobj.list));
up_write(&vm->lock);
mutex_lock(&xe->usm.lock);
@@ -2286,22 +2131,36 @@ static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op,
u8 tile_mask, bool read_only, bool is_null)
{
struct xe_bo *bo = op->gem.obj ? gem_to_xe_bo(op->gem.obj) : NULL;
+ struct drm_exec exec;
struct xe_vma *vma;
int err;
lockdep_assert_held_write(&vm->lock);
if (bo) {
- err = xe_bo_lock(bo, true);
- if (err)
- return ERR_PTR(err);
+ drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
+ drm_exec_until_all_locked(&exec) {
+ err = 0;
+ if (!bo->vm) {
+ err = drm_exec_lock_obj(&exec, xe_vm_obj(vm));
+ drm_exec_retry_on_contention(&exec);
+ }
+ if (!err) {
+ err = drm_exec_lock_obj(&exec, &bo->ttm.base);
+ drm_exec_retry_on_contention(&exec);
+ }
+ if (err) {
+ drm_exec_fini(&exec);
+ return ERR_PTR(err);
+ }
+ }
}
vma = xe_vma_create(vm, bo, op->gem.offset,
op->va.addr, op->va.addr +
op->va.range - 1, read_only, is_null,
tile_mask);
if (bo)
- xe_bo_unlock(bo);
+ drm_exec_fini(&exec);
if (xe_vma_is_userptr(vma)) {
err = xe_vma_userptr_pin_pages(vma);
@@ -2311,7 +2170,6 @@ static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op,
return ERR_PTR(err);
}
} else if (!xe_vma_has_no_bo(vma) && !bo->vm) {
- vm_insert_extobj(vm, vma);
err = add_preempt_fences(vm, bo);
if (err) {
prep_vma_destroy(vm, vma, false);
diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h
index 88077214261f..20b320aaeaa0 100644
--- a/drivers/gpu/drm/xe/xe_vm.h
+++ b/drivers/gpu/drm/xe/xe_vm.h
@@ -64,9 +64,14 @@ static inline bool xe_vm_is_closed_or_banned(struct xe_vm *vm)
struct xe_vma *
xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range);
+static inline struct xe_vm *gpuvm_to_vm(struct drm_gpuvm *gpuvm)
+{
+ return container_of(gpuvm, struct xe_vm, gpuvm);
+}
+
static inline struct xe_vm *gpuva_to_vm(struct drm_gpuva *gpuva)
{
- return container_of(gpuva->vm, struct xe_vm, gpuvm);
+ return gpuvm_to_vm(gpuva->vm);
}
static inline struct xe_vma *gpuva_to_vma(struct drm_gpuva *gpuva)
@@ -209,12 +214,6 @@ int xe_vma_userptr_check_repin(struct xe_vma *vma);
bool xe_vm_validate_should_retry(struct drm_exec *exec, int err, ktime_t *end);
-int xe_vm_lock_dma_resv(struct xe_vm *vm, struct drm_exec *exec,
- unsigned int num_shared, bool lock_vm);
-
-void xe_vm_fence_all_extobjs(struct xe_vm *vm, struct dma_fence *fence,
- enum dma_resv_usage usage);
-
int xe_analyze_vm(struct drm_printer *p, struct xe_vm *vm, int gt_id);
int xe_vm_prepare_vma(struct drm_exec *exec, struct xe_vma *vma,
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index f11ac81cf010..bff193920d25 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -110,18 +110,6 @@ struct xe_vma {
*/
u8 tile_present;
- struct {
- struct list_head rebind_link;
- } notifier;
-
- struct {
- /**
- * @extobj.link: Link into vm's external object list.
- * protected by the vm lock.
- */
- struct list_head link;
- } extobj;
-
/**
* @userptr: user pointer state, only allocated for VMAs that are
* user pointers
@@ -200,14 +188,6 @@ struct xe_vm {
*/
struct xe_range_fence_tree rftree[XE_MAX_TILES_PER_DEVICE];
- /** @extobj: bookkeeping for external objects. Protected by the vm lock */
- struct {
- /** @enties: number of external BOs attached this VM */
- u32 entries;
- /** @list: list of vmas with external bos attached */
- struct list_head list;
- } extobj;
-
/** @async_ops: async VM operations (bind / unbinds) */
struct {
/** @list: list of pending async VM ops */
@@ -297,22 +277,6 @@ struct xe_vm {
struct xe_vma *last_fault_vma;
} usm;
- /**
- * @notifier: Lists and locks for temporary usage within notifiers where
- * we either can't grab the vm lock or the vm resv.
- */
- struct {
- /** @notifier.list_lock: lock protecting @rebind_list */
- spinlock_t list_lock;
- /**
- * @notifier.rebind_list: list of vmas that we want to put on the
- * main @rebind_list. This list is protected for writing by both
- * notifier.list_lock, and the resv of the bo the vma points to,
- * and for reading by the notifier.list_lock only.
- */
- struct list_head rebind_list;
- } notifier;
-
/** @error_capture: allow to track errors */
struct {
/** @capture_once: capture only one error per VM */
--
2.41.0
^ permalink raw reply related [flat|nested] 13+ messages in thread
* [Intel-xe] [CI 09/10] drm/xe: Adapt to GPUVM tracking of external / evicted objects.
2023-10-27 8:15 [Intel-xe] [CI 01/10] drm/gpuvm: convert WARN() to drm_WARN() variants Thomas Hellström
@ 2023-10-27 8:15 ` Thomas Hellström
0 siblings, 0 replies; 13+ messages in thread
From: Thomas Hellström @ 2023-10-27 8:15 UTC (permalink / raw)
To: intel-xe
Using fine-grained locking for now.
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/xe/xe_bo.c | 63 +++----
drivers/gpu/drm/xe/xe_exec.c | 72 ++------
drivers/gpu/drm/xe/xe_vm.c | 292 +++++++------------------------
drivers/gpu/drm/xe/xe_vm.h | 13 +-
drivers/gpu/drm/xe/xe_vm_types.h | 67 ++-----
5 files changed, 120 insertions(+), 387 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 76ca4f3a3645..0dd6f79c28b6 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -450,9 +450,9 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
{
struct dma_resv_iter cursor;
struct dma_fence *fence;
- struct drm_gpuva *gpuva;
struct drm_gem_object *obj = &bo->ttm.base;
struct drm_gpuvm_bo *vm_bo;
+ bool idle = false;
int ret = 0;
dma_resv_assert_held(bo->ttm.base.resv);
@@ -466,14 +466,15 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
}
drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
- drm_gpuvm_bo_for_each_va(gpuva, vm_bo) {
- struct xe_vma *vma = gpuva_to_vma(gpuva);
- struct xe_vm *vm = xe_vma_vm(vma);
+ struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm);
+ struct drm_gpuva *gpuva;
- trace_xe_vma_evict(vma);
+ if (!xe_vm_in_fault_mode(vm)) {
+ drm_gpuvm_bo_evict(vm_bo, true);
+ continue;
+ }
- if (xe_vm_in_fault_mode(vm)) {
- /* Wait for pending binds / unbinds. */
+ if (!idle) {
long timeout;
if (ctx->no_wait_gpu &&
@@ -485,45 +486,21 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
DMA_RESV_USAGE_BOOKKEEP,
ctx->interruptible,
MAX_SCHEDULE_TIMEOUT);
- if (timeout > 0) {
- ret = xe_vm_invalidate_vma(vma);
- XE_WARN_ON(ret);
- } else if (!timeout) {
- ret = -ETIME;
- } else {
- ret = timeout;
- }
-
- } else {
- bool vm_resv_locked = false;
+ if (!timeout)
+ return -ETIME;
+ if (timeout < 0)
+ return timeout;
- /*
- * We need to put the vma on the vm's rebind_list,
- * but need the vm resv to do so. If we can't verify
- * that we indeed have it locked, put the vma an the
- * vm's notifier.rebind_list instead and scoop later.
- */
- if (dma_resv_trylock(xe_vm_resv(vm)))
- vm_resv_locked = true;
- else if (ctx->resv != xe_vm_resv(vm)) {
- spin_lock(&vm->notifier.list_lock);
- if (!(vma->gpuva.flags & XE_VMA_DESTROYED))
- list_move_tail(&vma->notifier.rebind_link,
- &vm->notifier.rebind_list);
- spin_unlock(&vm->notifier.list_lock);
- continue;
- }
+ idle = true;
+ }
- xe_vm_assert_held(vm);
- if (vma->tile_present &&
- !(vma->gpuva.flags & XE_VMA_DESTROYED) &&
- list_empty(&vma->combined_links.rebind))
- list_add_tail(&vma->combined_links.rebind,
- &vm->rebind_list);
+ drm_gpuvm_bo_for_each_va(gpuva, vm_bo) {
+ struct xe_vma *vma = gpuva_to_vma(gpuva);
- if (vm_resv_locked)
- dma_resv_unlock(xe_vm_resv(vm));
- }
+ trace_xe_vma_evict(vma);
+ ret = xe_vm_invalidate_vma(vma);
+ if (XE_WARN_ON(ret))
+ return ret;
}
}
diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c
index 85a8a793f527..cd5d2d2257f5 100644
--- a/drivers/gpu/drm/xe/xe_exec.c
+++ b/drivers/gpu/drm/xe/xe_exec.c
@@ -94,40 +94,9 @@
* Unlock all
*/
-static int xe_exec_begin(struct drm_exec *exec, struct xe_vm *vm)
+static int xe_exec_fn(struct drm_gpuvm_exec *vm_exec)
{
- struct xe_vma *vma;
- LIST_HEAD(dups);
- int err = 0;
-
- if (xe_vm_no_dma_fences(vm))
- return 0;
-
- /*
- * 1 fence for job from exec plus a fence for each tile from a possible
- * rebind
- */
- err = xe_vm_lock_dma_resv(vm, exec, 1 + vm->xe->info.tile_count, true);
- if (err)
- return err;
-
- /*
- * Validate BOs that have been evicted (i.e. make sure the
- * BOs have valid placements possibly moving an evicted BO back
- * to a location where the GPU can access it).
- */
- list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) {
- xe_assert(vm->xe, !xe_vma_is_null(vma));
-
- if (xe_vma_is_userptr(vma))
- continue;
-
- err = xe_bo_validate(xe_vma_bo(vma), vm, false);
- if (err)
- break;
- }
-
- return err;
+ return drm_gpuvm_validate(vm_exec->vm, &vm_exec->exec);
}
int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
@@ -140,7 +109,8 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
struct xe_exec_queue *q;
struct xe_sync_entry *syncs = NULL;
u64 addresses[XE_HW_ENGINE_MAX_INSTANCE];
- struct drm_exec exec;
+ struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn};
+ struct drm_exec *exec = &vm_exec.exec;
u32 i, num_syncs = 0;
struct xe_sched_job *job;
struct dma_fence *rebind_fence;
@@ -216,16 +186,14 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
goto err_unlock_list;
}
- drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
- drm_exec_until_all_locked(&exec) {
- err = xe_exec_begin(&exec, vm);
- drm_exec_retry_on_contention(&exec);
- if (err && xe_vm_validate_should_retry(&exec, err, &end)) {
+ vm_exec.vm = &vm->gpuvm;
+ vm_exec.num_fences = 1 + vm->xe->info.tile_count;
+ vm_exec.flags = DRM_EXEC_INTERRUPTIBLE_WAIT;
+ err = drm_gpuvm_exec_lock(&vm_exec);
+ if (err) {
+ if (xe_vm_validate_should_retry(exec, err, &end))
err = -EAGAIN;
- goto err_unlock_list;
- }
- if (err)
- goto err_exec;
+ goto err_unlock_list;
}
if (xe_vm_is_closed_or_banned(q->vm)) {
@@ -307,19 +275,9 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
* the job and let the DRM scheduler / backend clean up the job.
*/
xe_sched_job_arm(job);
- if (!xe_vm_no_dma_fences(vm)) {
- /* Block userptr invalidations / BO eviction */
- dma_resv_add_fence(xe_vm_resv(vm),
- &job->drm.s_fence->finished,
- DMA_RESV_USAGE_BOOKKEEP);
-
- /*
- * Make implicit sync work across drivers, assuming all external
- * BOs are written as we don't pass in a read / write list.
- */
- xe_vm_fence_all_extobjs(vm, &job->drm.s_fence->finished,
- DMA_RESV_USAGE_WRITE);
- }
+ if (!xe_vm_no_dma_fences(vm))
+ drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, &job->drm.s_fence->finished,
+ DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_WRITE);
for (i = 0; i < num_syncs; i++)
xe_sync_entry_signal(&syncs[i], job,
@@ -343,7 +301,7 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
if (err)
xe_sched_job_put(job);
err_exec:
- drm_exec_fini(&exec);
+ drm_exec_fini(exec);
err_unlock_list:
if (write_locked)
up_write(&vm->lock);
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 1b94736ef430..5f3bdb4e48ad 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -297,26 +297,8 @@ static int add_preempt_fences(struct xe_vm *vm, struct xe_bo *bo)
return err;
}
-/**
- * xe_vm_fence_all_extobjs() - Add a fence to vm's external objects' resv
- * @vm: The vm.
- * @fence: The fence to add.
- * @usage: The resv usage for the fence.
- *
- * Loops over all of the vm's external object bindings and adds a @fence
- * with the given @usage to all of the external object's reservation
- * objects.
- */
-void xe_vm_fence_all_extobjs(struct xe_vm *vm, struct dma_fence *fence,
- enum dma_resv_usage usage)
-{
- struct xe_vma *vma;
-
- list_for_each_entry(vma, &vm->extobj.list, extobj.link)
- dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, usage);
-}
-
-static void resume_and_reinstall_preempt_fences(struct xe_vm *vm)
+static void resume_and_reinstall_preempt_fences(struct xe_vm *vm,
+ struct drm_exec *exec)
{
struct xe_exec_queue *q;
@@ -326,16 +308,19 @@ static void resume_and_reinstall_preempt_fences(struct xe_vm *vm)
list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) {
q->ops->resume(q);
- dma_resv_add_fence(xe_vm_resv(vm), q->compute.pfence,
- DMA_RESV_USAGE_BOOKKEEP);
- xe_vm_fence_all_extobjs(vm, q->compute.pfence,
- DMA_RESV_USAGE_BOOKKEEP);
+ drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, q->compute.pfence,
+ DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP);
}
}
int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
{
- struct drm_exec exec;
+ struct drm_gpuvm_exec vm_exec = {
+ .vm = &vm->gpuvm,
+ .flags = DRM_EXEC_INTERRUPTIBLE_WAIT,
+ .num_fences = 1,
+ };
+ struct drm_exec *exec = &vm_exec.exec;
struct dma_fence *pfence;
int err;
bool wait;
@@ -343,13 +328,9 @@ int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
xe_assert(vm->xe, xe_vm_in_compute_mode(vm));
down_write(&vm->lock);
- drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
- drm_exec_until_all_locked(&exec) {
- err = xe_vm_lock_dma_resv(vm, &exec, 1, true);
- drm_exec_retry_on_contention(&exec);
- if (err)
- goto out_unlock;
- }
+ err = drm_gpuvm_exec_lock(&vm_exec);
+ if (err)
+ return err;
pfence = xe_preempt_fence_create(q, q->compute.context,
++q->compute.seqno);
@@ -364,10 +345,8 @@ int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
down_read(&vm->userptr.notifier_lock);
- dma_resv_add_fence(xe_vm_resv(vm), pfence,
- DMA_RESV_USAGE_BOOKKEEP);
-
- xe_vm_fence_all_extobjs(vm, pfence, DMA_RESV_USAGE_BOOKKEEP);
+ drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, pfence,
+ DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP);
/*
* Check to see if a preemption on VM is in flight or userptr
@@ -381,7 +360,7 @@ int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
up_read(&vm->userptr.notifier_lock);
out_unlock:
- drm_exec_fini(&exec);
+ drm_exec_fini(exec);
up_write(&vm->lock);
return err;
@@ -427,55 +406,6 @@ int __xe_vm_userptr_needs_repin(struct xe_vm *vm)
list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN;
}
-/**
- * xe_vm_lock_dma_resv() - Lock the vm dma_resv object and the dma_resv
- * objects of the vm's external buffer objects.
- * @vm: The vm.
- * @exec: Pointer to a struct drm_exec locking context.
- * @num_shared: Number of dma-fence slots to reserve in the locked objects.
- * @lock_vm: Lock also the vm's dma_resv.
- *
- * Locks the vm dma-resv objects and all the dma-resv objects of the
- * buffer objects on the vm external object list.
- *
- * Return: 0 on success, Negative error code on error. In particular if
- * @intr is set to true, -EINTR or -ERESTARTSYS may be returned.
- */
-int xe_vm_lock_dma_resv(struct xe_vm *vm, struct drm_exec *exec,
- unsigned int num_shared, bool lock_vm)
-{
- struct xe_vma *vma, *next;
- int err = 0;
-
- lockdep_assert_held(&vm->lock);
-
- if (lock_vm) {
- err = drm_exec_prepare_obj(exec, xe_vm_obj(vm), num_shared);
- if (err)
- return err;
- }
-
- list_for_each_entry(vma, &vm->extobj.list, extobj.link) {
- err = drm_exec_prepare_obj(exec, &xe_vma_bo(vma)->ttm.base, num_shared);
- if (err)
- return err;
- }
-
- spin_lock(&vm->notifier.list_lock);
- list_for_each_entry_safe(vma, next, &vm->notifier.rebind_list,
- notifier.rebind_link) {
- xe_bo_assert_held(xe_vma_bo(vma));
-
- list_del_init(&vma->notifier.rebind_link);
- if (vma->tile_present && !(vma->gpuva.flags & XE_VMA_DESTROYED))
- list_move_tail(&vma->combined_links.rebind,
- &vm->rebind_list);
- }
- spin_unlock(&vm->notifier.list_lock);
-
- return 0;
-}
-
#define XE_VM_REBIND_RETRY_TIMEOUT_MS 1000
static void xe_vm_kill(struct xe_vm *vm)
@@ -524,30 +454,39 @@ bool xe_vm_validate_should_retry(struct drm_exec *exec, int err, ktime_t *end)
if (!ktime_before(cur, *end))
return false;
- /*
- * We would like to keep the ticket here with
- * drm_exec_unlock_all(), but WW mutex asserts currently
- * stop us from that. In any case this function could go away
- * with proper TTM -EDEADLK handling.
- */
- drm_exec_fini(exec);
-
msleep(20);
return true;
}
+static int xe_gpuvm_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec)
+{
+ struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm);
+ struct drm_gpuva *gpuva;
+ int ret;
+
+ lockdep_assert_held(&vm->lock);
+ drm_gpuvm_bo_for_each_va(gpuva, vm_bo)
+ list_move_tail(&gpuva_to_vma(gpuva)->combined_links.rebind,
+ &vm->rebind_list);
+
+ ret = xe_bo_validate(gem_to_xe_bo(vm_bo->obj), vm, false);
+ if (ret)
+ return ret;
+
+ vm_bo->evicted = false;
+ return 0;
+}
+
static int xe_preempt_work_begin(struct drm_exec *exec, struct xe_vm *vm,
bool *done)
{
- struct xe_vma *vma;
int err;
/*
* 1 fence for each preempt fence plus a fence for each tile from a
* possible rebind
*/
- err = drm_exec_prepare_obj(exec, xe_vm_obj(vm),
- vm->preempt.num_exec_queues +
+ err = drm_gpuvm_prepare_vm(&vm->gpuvm, exec, vm->preempt.num_exec_queues +
vm->xe->info.tile_count);
if (err)
return err;
@@ -563,7 +502,7 @@ static int xe_preempt_work_begin(struct drm_exec *exec, struct xe_vm *vm,
return 0;
}
- err = xe_vm_lock_dma_resv(vm, exec, vm->preempt.num_exec_queues, false);
+ err = drm_gpuvm_prepare_objects(&vm->gpuvm, exec, vm->preempt.num_exec_queues);
if (err)
return err;
@@ -571,17 +510,7 @@ static int xe_preempt_work_begin(struct drm_exec *exec, struct xe_vm *vm,
if (err)
return err;
- list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) {
- if (xe_vma_has_no_bo(vma) ||
- vma->gpuva.flags & XE_VMA_DESTROYED)
- continue;
-
- err = xe_bo_validate(xe_vma_bo(vma), vm, false);
- if (err)
- break;
- }
-
- return err;
+ return drm_gpuvm_validate(&vm->gpuvm, exec);
}
static void preempt_rebind_work_func(struct work_struct *w)
@@ -621,12 +550,13 @@ static void preempt_rebind_work_func(struct work_struct *w)
err = xe_preempt_work_begin(&exec, vm, &done);
drm_exec_retry_on_contention(&exec);
- if (err && xe_vm_validate_should_retry(&exec, err, &end)) {
- err = -EAGAIN;
+ if (err || done) {
+ drm_exec_fini(&exec);
+ if (err && xe_vm_validate_should_retry(&exec, err, &end))
+ err = -EAGAIN;
+
goto out_unlock_outer;
}
- if (err || done)
- goto out_unlock;
}
err = alloc_preempt_fences(vm, &preempt_fences, &fence_count);
@@ -673,7 +603,7 @@ static void preempt_rebind_work_func(struct work_struct *w)
/* Point of no return. */
arm_preempt_fences(vm, &preempt_fences);
- resume_and_reinstall_preempt_fences(vm);
+ resume_and_reinstall_preempt_fences(vm, &exec);
up_read(&vm->userptr.notifier_lock);
out_unlock:
@@ -778,9 +708,8 @@ int xe_vm_userptr_pin(struct xe_vm *vm)
list_for_each_entry_safe(vma, next, &vm->userptr.invalidated,
userptr.invalidate_link) {
list_del_init(&vma->userptr.invalidate_link);
- if (list_empty(&vma->combined_links.userptr))
- list_move_tail(&vma->combined_links.userptr,
- &vm->userptr.repin_list);
+ list_move_tail(&vma->combined_links.userptr,
+ &vm->userptr.repin_list);
}
spin_unlock(&vm->userptr.invalidated_lock);
@@ -789,27 +718,12 @@ int xe_vm_userptr_pin(struct xe_vm *vm)
combined_links.userptr) {
err = xe_vma_userptr_pin_pages(vma);
if (err < 0)
- goto out_err;
+ return err;
- list_move_tail(&vma->combined_links.userptr, &tmp_evict);
+ list_move_tail(&vma->combined_links.userptr, &vm->rebind_list);
}
- /* Take lock and move to rebind_list for rebinding. */
- err = dma_resv_lock_interruptible(xe_vm_resv(vm), NULL);
- if (err)
- goto out_err;
-
- list_for_each_entry_safe(vma, next, &tmp_evict, combined_links.userptr)
- list_move_tail(&vma->combined_links.rebind, &vm->rebind_list);
-
- dma_resv_unlock(xe_vm_resv(vm));
-
return 0;
-
-out_err:
- list_splice_tail(&tmp_evict, &vm->userptr.repin_list);
-
- return err;
}
/**
@@ -888,8 +802,6 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm,
}
INIT_LIST_HEAD(&vma->combined_links.rebind);
- INIT_LIST_HEAD(&vma->notifier.rebind_link);
- INIT_LIST_HEAD(&vma->extobj.link);
INIT_LIST_HEAD(&vma->gpuva.gem.entry);
vma->gpuva.vm = &vm->gpuvm;
@@ -921,6 +833,7 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm,
return ERR_CAST(vm_bo);
}
+ drm_gpuvm_bo_extobj_add(vm_bo);
drm_gem_object_get(&bo->ttm.base);
vma->gpuva.gem.obj = &bo->ttm.base;
vma->gpuva.gem.offset = bo_offset_or_userptr;
@@ -953,16 +866,6 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm,
return vma;
}
-static bool vm_remove_extobj(struct xe_vma *vma)
-{
- if (!list_empty(&vma->extobj.link)) {
- xe_vma_vm(vma)->extobj.entries--;
- list_del_init(&vma->extobj.link);
- return true;
- }
- return false;
-}
-
static void xe_vma_destroy_late(struct xe_vma *vma)
{
struct xe_vm *vm = xe_vma_vm(vma);
@@ -1003,60 +906,6 @@ static void vma_destroy_work_func(struct work_struct *w)
xe_vma_destroy_late(vma);
}
-static struct xe_vma *
-bo_has_vm_references_locked(struct xe_bo *bo, struct xe_vm *vm,
- struct xe_vma *ignore)
-{
- struct drm_gpuvm_bo *vm_bo;
- struct drm_gpuva *va;
- struct drm_gem_object *obj = &bo->ttm.base;
-
- xe_bo_assert_held(bo);
-
- drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
- drm_gpuvm_bo_for_each_va(va, vm_bo) {
- struct xe_vma *vma = gpuva_to_vma(va);
-
- if (vma != ignore && xe_vma_vm(vma) == vm)
- return vma;
- }
- }
-
- return NULL;
-}
-
-static bool bo_has_vm_references(struct xe_bo *bo, struct xe_vm *vm,
- struct xe_vma *ignore)
-{
- bool ret;
-
- xe_bo_lock(bo, false);
- ret = !!bo_has_vm_references_locked(bo, vm, ignore);
- xe_bo_unlock(bo);
-
- return ret;
-}
-
-static void __vm_insert_extobj(struct xe_vm *vm, struct xe_vma *vma)
-{
- lockdep_assert_held_write(&vm->lock);
-
- list_add(&vma->extobj.link, &vm->extobj.list);
- vm->extobj.entries++;
-}
-
-static void vm_insert_extobj(struct xe_vm *vm, struct xe_vma *vma)
-{
- struct xe_bo *bo = xe_vma_bo(vma);
-
- lockdep_assert_held_write(&vm->lock);
-
- if (bo_has_vm_references(bo, vm, vma))
- return;
-
- __vm_insert_extobj(vm, vma);
-}
-
static void vma_destroy_cb(struct dma_fence *fence,
struct dma_fence_cb *cb)
{
@@ -1082,20 +931,7 @@ static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence)
} else if (!xe_vma_is_null(vma)) {
xe_bo_assert_held(xe_vma_bo(vma));
- spin_lock(&vm->notifier.list_lock);
- list_del(&vma->notifier.rebind_link);
- spin_unlock(&vm->notifier.list_lock);
-
drm_gpuva_unlink(&vma->gpuva);
-
- if (!xe_vma_bo(vma)->vm && vm_remove_extobj(vma)) {
- struct xe_vma *other;
-
- other = bo_has_vm_references_locked(xe_vma_bo(vma), vm, NULL);
-
- if (other)
- __vm_insert_extobj(vm, other);
- }
}
xe_vm_assert_held(vm);
@@ -1211,6 +1047,7 @@ static struct drm_gpuva_op *xe_vm_op_alloc(void)
static struct drm_gpuvm_ops gpuvm_ops = {
.op_alloc = xe_vm_op_alloc,
+ .vm_bo_validate = xe_gpuvm_validate,
};
static u64 pde_encode_pat_index(struct xe_device *xe, u16 pat_index)
@@ -1373,9 +1210,6 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
init_rwsem(&vm->userptr.notifier_lock);
spin_lock_init(&vm->userptr.invalidated_lock);
- INIT_LIST_HEAD(&vm->notifier.rebind_list);
- spin_lock_init(&vm->notifier.list_lock);
-
INIT_WORK(&vm->destroy_work, vm_destroy_work_func);
INIT_LIST_HEAD(&vm->preempt.exec_queues);
@@ -1384,8 +1218,6 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
for_each_tile(tile, xe, id)
xe_range_fence_tree_init(&vm->rftree[id]);
- INIT_LIST_HEAD(&vm->extobj.list);
-
vm->pt_ops = &xelp_pt_ops;
if (!(flags & XE_VM_FLAG_MIGRATION))
@@ -1612,7 +1444,6 @@ void xe_vm_close_and_put(struct xe_vm *vm)
xe_vma_destroy_unlocked(vma);
}
- xe_assert(xe, list_empty(&vm->extobj.list));
up_write(&vm->lock);
mutex_lock(&xe->usm.lock);
@@ -2286,22 +2117,36 @@ static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op,
u8 tile_mask, bool read_only, bool is_null)
{
struct xe_bo *bo = op->gem.obj ? gem_to_xe_bo(op->gem.obj) : NULL;
+ struct drm_exec exec;
struct xe_vma *vma;
int err;
lockdep_assert_held_write(&vm->lock);
if (bo) {
- err = xe_bo_lock(bo, true);
- if (err)
- return ERR_PTR(err);
+ drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
+ drm_exec_until_all_locked(&exec) {
+ err = 0;
+ if (!bo->vm) {
+ err = drm_exec_lock_obj(&exec, xe_vm_obj(vm));
+ drm_exec_retry_on_contention(&exec);
+ }
+ if (!err) {
+ err = drm_exec_lock_obj(&exec, &bo->ttm.base);
+ drm_exec_retry_on_contention(&exec);
+ }
+ if (err) {
+ drm_exec_fini(&exec);
+ return ERR_PTR(err);
+ }
+ }
}
vma = xe_vma_create(vm, bo, op->gem.offset,
op->va.addr, op->va.addr +
op->va.range - 1, read_only, is_null,
tile_mask);
if (bo)
- xe_bo_unlock(bo);
+ drm_exec_fini(&exec);
if (xe_vma_is_userptr(vma)) {
err = xe_vma_userptr_pin_pages(vma);
@@ -2311,7 +2156,6 @@ static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op,
return ERR_PTR(err);
}
} else if (!xe_vma_has_no_bo(vma) && !bo->vm) {
- vm_insert_extobj(vm, vma);
err = add_preempt_fences(vm, bo);
if (err) {
prep_vma_destroy(vm, vma, false);
diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h
index 88077214261f..20b320aaeaa0 100644
--- a/drivers/gpu/drm/xe/xe_vm.h
+++ b/drivers/gpu/drm/xe/xe_vm.h
@@ -64,9 +64,14 @@ static inline bool xe_vm_is_closed_or_banned(struct xe_vm *vm)
struct xe_vma *
xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range);
+static inline struct xe_vm *gpuvm_to_vm(struct drm_gpuvm *gpuvm)
+{
+ return container_of(gpuvm, struct xe_vm, gpuvm);
+}
+
static inline struct xe_vm *gpuva_to_vm(struct drm_gpuva *gpuva)
{
- return container_of(gpuva->vm, struct xe_vm, gpuvm);
+ return gpuvm_to_vm(gpuva->vm);
}
static inline struct xe_vma *gpuva_to_vma(struct drm_gpuva *gpuva)
@@ -209,12 +214,6 @@ int xe_vma_userptr_check_repin(struct xe_vma *vma);
bool xe_vm_validate_should_retry(struct drm_exec *exec, int err, ktime_t *end);
-int xe_vm_lock_dma_resv(struct xe_vm *vm, struct drm_exec *exec,
- unsigned int num_shared, bool lock_vm);
-
-void xe_vm_fence_all_extobjs(struct xe_vm *vm, struct dma_fence *fence,
- enum dma_resv_usage usage);
-
int xe_analyze_vm(struct drm_printer *p, struct xe_vm *vm, int gt_id);
int xe_vm_prepare_vma(struct drm_exec *exec, struct xe_vma *vma,
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index f11ac81cf010..a0c95e343129 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -62,26 +62,17 @@ struct xe_vma {
/** @gpuva: Base GPUVA object */
struct drm_gpuva gpuva;
- /** @combined_links: links into lists which are mutually exclusive */
+ /**
+ * @combined_links: links into lists which are mutually exclusive.
+ * Locking: vm lock in write mode OR vm lock in read mode and the vm's
+ * resv.
+ */
union {
- /**
- * @userptr: link into VM repin list if userptr. Protected by
- * vm->lock in write mode.
- */
+ /** @userptr: link into VM repin list if userptr. */
struct list_head userptr;
- /**
- * @rebind: link into VM if this VMA needs rebinding, and
- * if it's a bo (not userptr) needs validation after a possible
- * eviction. Protected by the vm's resv lock and typically
- * vm->lock is also held in write mode. The only place where
- * vm->lock isn't held is the BO eviction path which has
- * mutually exclusive execution with userptr.
- */
+ /** @rebind: link into VM if this VMA needs rebinding. */
struct list_head rebind;
- /**
- * @destroy: link to contested list when VM is being closed.
- * Protected by vm->lock in write mode and vm's resv lock.
- */
+ /** @destroy: link to contested list when VM is being closed. */
struct list_head destroy;
} combined_links;
@@ -110,18 +101,6 @@ struct xe_vma {
*/
u8 tile_present;
- struct {
- struct list_head rebind_link;
- } notifier;
-
- struct {
- /**
- * @extobj.link: Link into vm's external object list.
- * protected by the vm lock.
- */
- struct list_head link;
- } extobj;
-
/**
* @userptr: user pointer state, only allocated for VMAs that are
* user pointers
@@ -178,9 +157,9 @@ struct xe_vm {
struct rw_semaphore lock;
/**
- * @rebind_list: list of VMAs that need rebinding, and if they are
- * bos (not userptr), need validation after a possible eviction. The
- * list is protected by @resv.
+ * @rebind_list: list of VMAs that need rebinding. Protected by the
+ * vm->lock in write mode, OR (the vm->lock in read mode and the
+ * vm resv).
*/
struct list_head rebind_list;
@@ -200,14 +179,6 @@ struct xe_vm {
*/
struct xe_range_fence_tree rftree[XE_MAX_TILES_PER_DEVICE];
- /** @extobj: bookkeeping for external objects. Protected by the vm lock */
- struct {
- /** @enties: number of external BOs attached this VM */
- u32 entries;
- /** @list: list of vmas with external bos attached */
- struct list_head list;
- } extobj;
-
/** @async_ops: async VM operations (bind / unbinds) */
struct {
/** @list: list of pending async VM ops */
@@ -297,22 +268,6 @@ struct xe_vm {
struct xe_vma *last_fault_vma;
} usm;
- /**
- * @notifier: Lists and locks for temporary usage within notifiers where
- * we either can't grab the vm lock or the vm resv.
- */
- struct {
- /** @notifier.list_lock: lock protecting @rebind_list */
- spinlock_t list_lock;
- /**
- * @notifier.rebind_list: list of vmas that we want to put on the
- * main @rebind_list. This list is protected for writing by both
- * notifier.list_lock, and the resv of the bo the vma points to,
- * and for reading by the notifier.list_lock only.
- */
- struct list_head rebind_list;
- } notifier;
-
/** @error_capture: allow to track errors */
struct {
/** @capture_once: capture only one error per VM */
--
2.41.0
^ permalink raw reply related [flat|nested] 13+ messages in thread