* [PATCH] drm/xe: Rework GPU page fault handling
@ 2024-05-14 21:55 Matthew Brost
2024-06-06 11:36 ` Nirmoy Das
0 siblings, 1 reply; 4+ messages in thread
From: Matthew Brost @ 2024-05-14 21:55 UTC (permalink / raw)
To: intel-xe; +Cc: Matthew Brost
Add helper function to implement VMA (user binding) page faults, remove
unnecessary userptr.invalidate_link list del operation, retry on memory
pressure, remove unnecessary xe_vma_userptr_check_repin after rebinding,
remove unnecessary TLB invalidation, and always use vm->lock in write
mode. Changes help facilitate SVM page faults.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_gt_pagefault.c | 142 ++++++++++++---------------
1 file changed, 62 insertions(+), 80 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c
index 040dd142c49c..eaf68f0135c1 100644
--- a/drivers/gpu/drm/xe/xe_gt_pagefault.c
+++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c
@@ -125,126 +125,108 @@ static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma,
return 0;
}
-static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
+static int handle_vma_pagefault(struct xe_tile *tile, struct pagefault *pf,
+ struct xe_vma *vma)
{
- struct xe_device *xe = gt_to_xe(gt);
- struct xe_tile *tile = gt_to_tile(gt);
+ struct xe_vm *vm = xe_vma_vm(vma);
struct drm_exec exec;
- struct xe_vm *vm;
- struct xe_vma *vma = NULL;
struct dma_fence *fence;
- bool write_locked;
- int ret = 0;
+ ktime_t end = 0;
+ int err;
bool atomic;
- /* SW isn't expected to handle TRTT faults */
- if (pf->trva_fault)
- return -EFAULT;
-
- /* ASID to VM */
- mutex_lock(&xe->usm.lock);
- vm = xa_load(&xe->usm.asid_to_vm, pf->asid);
- if (vm && xe_vm_in_fault_mode(vm))
- xe_vm_get(vm);
- else
- vm = NULL;
- mutex_unlock(&xe->usm.lock);
- if (!vm)
- return -EINVAL;
-
-retry_userptr:
- /*
- * TODO: Avoid exclusive lock if VM doesn't have userptrs, or
- * start out read-locked?
- */
- down_write(&vm->lock);
- write_locked = true;
- vma = lookup_vma(vm, pf->page_addr);
- if (!vma) {
- ret = -EINVAL;
- goto unlock_vm;
- }
-
- if (!xe_vma_is_userptr(vma) ||
- !xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
- downgrade_write(&vm->lock);
- write_locked = false;
- }
-
trace_xe_vma_pagefault(vma);
-
atomic = access_is_atomic(pf->access_type);
/* Check if VMA is valid */
if (vma_is_valid(tile, vma) && !atomic)
- goto unlock_vm;
-
- /* TODO: Validate fault */
+ return 0;
- if (xe_vma_is_userptr(vma) && write_locked) {
+retry_userptr:
+ if (xe_vma_is_userptr(vma) &&
+ xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
struct xe_userptr_vma *uvma = to_userptr_vma(vma);
- spin_lock(&vm->userptr.invalidated_lock);
- list_del_init(&uvma->userptr.invalidate_link);
- spin_unlock(&vm->userptr.invalidated_lock);
-
- ret = xe_vma_userptr_pin_pages(uvma);
- if (ret)
- goto unlock_vm;
-
- downgrade_write(&vm->lock);
- write_locked = false;
+ err = xe_vma_userptr_pin_pages(uvma);
+ if (err)
+ return err;
}
/* Lock VM and BOs dma-resv */
drm_exec_init(&exec, 0, 0);
drm_exec_until_all_locked(&exec) {
- ret = xe_pf_begin(&exec, vma, atomic, tile->id);
+ err = xe_pf_begin(&exec, vma, atomic, tile->id);
drm_exec_retry_on_contention(&exec);
- if (ret)
+ if (xe_vm_validate_should_retry(&exec, err, &end))
+ err = -EAGAIN;
+ if (err)
goto unlock_dma_resv;
/* Bind VMA only to the GT that has faulted */
trace_xe_vma_pf_bind(vma);
fence = xe_vma_rebind(vm, vma, BIT(tile->id));
if (IS_ERR(fence)) {
- ret = PTR_ERR(fence);
+ err = PTR_ERR(fence);
+ if (xe_vm_validate_should_retry(&exec, err, &end))
+ err = -EAGAIN;
goto unlock_dma_resv;
}
}
- /*
- * XXX: Should we drop the lock before waiting? This only helps if doing
- * GPU binds which is currently only done if we have to wait for more
- * than 10ms on a move.
- */
dma_fence_wait(fence, false);
dma_fence_put(fence);
-
- if (xe_vma_is_userptr(vma))
- ret = xe_vma_userptr_check_repin(to_userptr_vma(vma));
vma->tile_invalidated &= ~BIT(tile->id);
unlock_dma_resv:
drm_exec_fini(&exec);
-unlock_vm:
- if (!ret)
- vm->usm.last_fault_vma = vma;
- if (write_locked)
- up_write(&vm->lock);
- else
- up_read(&vm->lock);
- if (ret == -EAGAIN)
+ if (err == -EAGAIN)
goto retry_userptr;
- if (!ret) {
- ret = xe_gt_tlb_invalidation_vma(gt, NULL, vma);
- if (ret >= 0)
- ret = 0;
+ return err;
+}
+
+static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
+{
+ struct xe_device *xe = gt_to_xe(gt);
+ struct xe_tile *tile = gt_to_tile(gt);
+ struct xe_vm *vm;
+ struct xe_vma *vma = NULL;
+ int err;
+
+ /* SW isn't expected to handle TRTT faults */
+ if (pf->trva_fault)
+ return -EFAULT;
+
+ /* ASID to VM */
+ mutex_lock(&xe->usm.lock);
+ vm = xa_load(&xe->usm.asid_to_vm, pf->asid);
+ if (vm && xe_vm_in_fault_mode(vm))
+ xe_vm_get(vm);
+ else
+ vm = NULL;
+ mutex_unlock(&xe->usm.lock);
+ if (!vm)
+ return -EINVAL;
+
+ /*
+ * TODO: Change to read lock? Using write lock for simplicity.
+ */
+ down_write(&vm->lock);
+ vma = lookup_vma(vm, pf->page_addr);
+ if (!vma) {
+ err = -EINVAL;
+ goto unlock_vm;
}
+
+ err = handle_vma_pagefault(tile, pf, vma);
+
+unlock_vm:
+ if (!err)
+ vm->usm.last_fault_vma = vma;
+ up_write(&vm->lock);
xe_vm_put(vm);
- return ret;
+ return err;
}
static int send_pagefault_reply(struct xe_guc *guc,
--
2.34.1
^ permalink raw reply related [flat|nested] 4+ messages in thread
* Re: [PATCH] drm/xe: Rework GPU page fault handling
2024-05-14 21:55 [PATCH] drm/xe: Rework GPU page fault handling Matthew Brost
@ 2024-06-06 11:36 ` Nirmoy Das
0 siblings, 0 replies; 4+ messages in thread
From: Nirmoy Das @ 2024-06-06 11:36 UTC (permalink / raw)
To: Matthew Brost, intel-xe
On 5/14/2024 11:55 PM, Matthew Brost wrote:
> Add helper function to implement VMA (user binding) page faults, remove
> unnecessary userptr.invalidate_link list del operation, retry on memory
> pressure, remove unnecessary xe_vma_userptr_check_repin after rebinding,
> remove unnecessary TLB invalidation, and always use vm->lock in write
> mode. Changes help facilitate SVM page faults.
>
> Signed-off-by: Matthew Brost <matthew.brost@intel.com>
Reviewed-by: Nirmoy Das <nirmoy.das@intel.com>
> ---
> drivers/gpu/drm/xe/xe_gt_pagefault.c | 142 ++++++++++++---------------
> 1 file changed, 62 insertions(+), 80 deletions(-)
>
> diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c
> index 040dd142c49c..eaf68f0135c1 100644
> --- a/drivers/gpu/drm/xe/xe_gt_pagefault.c
> +++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c
> @@ -125,126 +125,108 @@ static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma,
> return 0;
> }
>
> -static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
> +static int handle_vma_pagefault(struct xe_tile *tile, struct pagefault *pf,
> + struct xe_vma *vma)
> {
> - struct xe_device *xe = gt_to_xe(gt);
> - struct xe_tile *tile = gt_to_tile(gt);
> + struct xe_vm *vm = xe_vma_vm(vma);
> struct drm_exec exec;
> - struct xe_vm *vm;
> - struct xe_vma *vma = NULL;
> struct dma_fence *fence;
> - bool write_locked;
> - int ret = 0;
> + ktime_t end = 0;
> + int err;
> bool atomic;
>
> - /* SW isn't expected to handle TRTT faults */
> - if (pf->trva_fault)
> - return -EFAULT;
> -
> - /* ASID to VM */
> - mutex_lock(&xe->usm.lock);
> - vm = xa_load(&xe->usm.asid_to_vm, pf->asid);
> - if (vm && xe_vm_in_fault_mode(vm))
> - xe_vm_get(vm);
> - else
> - vm = NULL;
> - mutex_unlock(&xe->usm.lock);
> - if (!vm)
> - return -EINVAL;
> -
> -retry_userptr:
> - /*
> - * TODO: Avoid exclusive lock if VM doesn't have userptrs, or
> - * start out read-locked?
> - */
> - down_write(&vm->lock);
> - write_locked = true;
> - vma = lookup_vma(vm, pf->page_addr);
> - if (!vma) {
> - ret = -EINVAL;
> - goto unlock_vm;
> - }
> -
> - if (!xe_vma_is_userptr(vma) ||
> - !xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
> - downgrade_write(&vm->lock);
> - write_locked = false;
> - }
> -
> trace_xe_vma_pagefault(vma);
> -
> atomic = access_is_atomic(pf->access_type);
>
> /* Check if VMA is valid */
> if (vma_is_valid(tile, vma) && !atomic)
> - goto unlock_vm;
> -
> - /* TODO: Validate fault */
> + return 0;
>
> - if (xe_vma_is_userptr(vma) && write_locked) {
> +retry_userptr:
> + if (xe_vma_is_userptr(vma) &&
> + xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
> struct xe_userptr_vma *uvma = to_userptr_vma(vma);
>
> - spin_lock(&vm->userptr.invalidated_lock);
> - list_del_init(&uvma->userptr.invalidate_link);
> - spin_unlock(&vm->userptr.invalidated_lock);
> -
> - ret = xe_vma_userptr_pin_pages(uvma);
> - if (ret)
> - goto unlock_vm;
> -
> - downgrade_write(&vm->lock);
> - write_locked = false;
> + err = xe_vma_userptr_pin_pages(uvma);
> + if (err)
> + return err;
> }
>
> /* Lock VM and BOs dma-resv */
> drm_exec_init(&exec, 0, 0);
> drm_exec_until_all_locked(&exec) {
> - ret = xe_pf_begin(&exec, vma, atomic, tile->id);
> + err = xe_pf_begin(&exec, vma, atomic, tile->id);
> drm_exec_retry_on_contention(&exec);
> - if (ret)
> + if (xe_vm_validate_should_retry(&exec, err, &end))
> + err = -EAGAIN;
> + if (err)
> goto unlock_dma_resv;
>
> /* Bind VMA only to the GT that has faulted */
> trace_xe_vma_pf_bind(vma);
> fence = xe_vma_rebind(vm, vma, BIT(tile->id));
> if (IS_ERR(fence)) {
> - ret = PTR_ERR(fence);
> + err = PTR_ERR(fence);
> + if (xe_vm_validate_should_retry(&exec, err, &end))
> + err = -EAGAIN;
> goto unlock_dma_resv;
> }
> }
>
> - /*
> - * XXX: Should we drop the lock before waiting? This only helps if doing
> - * GPU binds which is currently only done if we have to wait for more
> - * than 10ms on a move.
> - */
> dma_fence_wait(fence, false);
> dma_fence_put(fence);
> -
> - if (xe_vma_is_userptr(vma))
> - ret = xe_vma_userptr_check_repin(to_userptr_vma(vma));
> vma->tile_invalidated &= ~BIT(tile->id);
>
> unlock_dma_resv:
> drm_exec_fini(&exec);
> -unlock_vm:
> - if (!ret)
> - vm->usm.last_fault_vma = vma;
> - if (write_locked)
> - up_write(&vm->lock);
> - else
> - up_read(&vm->lock);
> - if (ret == -EAGAIN)
> + if (err == -EAGAIN)
> goto retry_userptr;
>
> - if (!ret) {
> - ret = xe_gt_tlb_invalidation_vma(gt, NULL, vma);
> - if (ret >= 0)
> - ret = 0;
> + return err;
> +}
> +
> +static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
> +{
> + struct xe_device *xe = gt_to_xe(gt);
> + struct xe_tile *tile = gt_to_tile(gt);
> + struct xe_vm *vm;
> + struct xe_vma *vma = NULL;
> + int err;
> +
> + /* SW isn't expected to handle TRTT faults */
> + if (pf->trva_fault)
> + return -EFAULT;
> +
> + /* ASID to VM */
> + mutex_lock(&xe->usm.lock);
> + vm = xa_load(&xe->usm.asid_to_vm, pf->asid);
> + if (vm && xe_vm_in_fault_mode(vm))
> + xe_vm_get(vm);
> + else
> + vm = NULL;
> + mutex_unlock(&xe->usm.lock);
> + if (!vm)
> + return -EINVAL;
> +
> + /*
> + * TODO: Change to read lock? Using write lock for simplicity.
> + */
> + down_write(&vm->lock);
> + vma = lookup_vma(vm, pf->page_addr);
> + if (!vma) {
> + err = -EINVAL;
> + goto unlock_vm;
> }
> +
> + err = handle_vma_pagefault(tile, pf, vma);
> +
> +unlock_vm:
> + if (!err)
> + vm->usm.last_fault_vma = vma;
> + up_write(&vm->lock);
> xe_vm_put(vm);
>
> - return ret;
> + return err;
> }
>
> static int send_pagefault_reply(struct xe_guc *guc,
^ permalink raw reply [flat|nested] 4+ messages in thread
* [PATCH] drm/xe: Rework GPU page fault handling
@ 2024-06-07 1:52 Matthew Brost
2024-06-07 2:25 ` Randhawa, Jagmeet
0 siblings, 1 reply; 4+ messages in thread
From: Matthew Brost @ 2024-06-07 1:52 UTC (permalink / raw)
To: intel-xe
Add helper function to implement VMA (user binding) page faults, remove
unnecessary userptr.invalidate_link list del operation, retry on memory
pressure, remove unnecessary xe_vma_userptr_check_repin after rebinding,
remove unnecessary TLB invalidation, and always use vm->lock in write
mode. Changes help facilitate SVM page faults.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
Reviewed-by: Nirmoy Das <nirmoy.das@intel.com>
---
drivers/gpu/drm/xe/xe_gt_pagefault.c | 142 ++++++++++++---------------
1 file changed, 62 insertions(+), 80 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c
index 040dd142c49c..eaf68f0135c1 100644
--- a/drivers/gpu/drm/xe/xe_gt_pagefault.c
+++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c
@@ -125,126 +125,108 @@ static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma,
return 0;
}
-static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
+static int handle_vma_pagefault(struct xe_tile *tile, struct pagefault *pf,
+ struct xe_vma *vma)
{
- struct xe_device *xe = gt_to_xe(gt);
- struct xe_tile *tile = gt_to_tile(gt);
+ struct xe_vm *vm = xe_vma_vm(vma);
struct drm_exec exec;
- struct xe_vm *vm;
- struct xe_vma *vma = NULL;
struct dma_fence *fence;
- bool write_locked;
- int ret = 0;
+ ktime_t end = 0;
+ int err;
bool atomic;
- /* SW isn't expected to handle TRTT faults */
- if (pf->trva_fault)
- return -EFAULT;
-
- /* ASID to VM */
- mutex_lock(&xe->usm.lock);
- vm = xa_load(&xe->usm.asid_to_vm, pf->asid);
- if (vm && xe_vm_in_fault_mode(vm))
- xe_vm_get(vm);
- else
- vm = NULL;
- mutex_unlock(&xe->usm.lock);
- if (!vm)
- return -EINVAL;
-
-retry_userptr:
- /*
- * TODO: Avoid exclusive lock if VM doesn't have userptrs, or
- * start out read-locked?
- */
- down_write(&vm->lock);
- write_locked = true;
- vma = lookup_vma(vm, pf->page_addr);
- if (!vma) {
- ret = -EINVAL;
- goto unlock_vm;
- }
-
- if (!xe_vma_is_userptr(vma) ||
- !xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
- downgrade_write(&vm->lock);
- write_locked = false;
- }
-
trace_xe_vma_pagefault(vma);
-
atomic = access_is_atomic(pf->access_type);
/* Check if VMA is valid */
if (vma_is_valid(tile, vma) && !atomic)
- goto unlock_vm;
-
- /* TODO: Validate fault */
+ return 0;
- if (xe_vma_is_userptr(vma) && write_locked) {
+retry_userptr:
+ if (xe_vma_is_userptr(vma) &&
+ xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
struct xe_userptr_vma *uvma = to_userptr_vma(vma);
- spin_lock(&vm->userptr.invalidated_lock);
- list_del_init(&uvma->userptr.invalidate_link);
- spin_unlock(&vm->userptr.invalidated_lock);
-
- ret = xe_vma_userptr_pin_pages(uvma);
- if (ret)
- goto unlock_vm;
-
- downgrade_write(&vm->lock);
- write_locked = false;
+ err = xe_vma_userptr_pin_pages(uvma);
+ if (err)
+ return err;
}
/* Lock VM and BOs dma-resv */
drm_exec_init(&exec, 0, 0);
drm_exec_until_all_locked(&exec) {
- ret = xe_pf_begin(&exec, vma, atomic, tile->id);
+ err = xe_pf_begin(&exec, vma, atomic, tile->id);
drm_exec_retry_on_contention(&exec);
- if (ret)
+ if (xe_vm_validate_should_retry(&exec, err, &end))
+ err = -EAGAIN;
+ if (err)
goto unlock_dma_resv;
/* Bind VMA only to the GT that has faulted */
trace_xe_vma_pf_bind(vma);
fence = xe_vma_rebind(vm, vma, BIT(tile->id));
if (IS_ERR(fence)) {
- ret = PTR_ERR(fence);
+ err = PTR_ERR(fence);
+ if (xe_vm_validate_should_retry(&exec, err, &end))
+ err = -EAGAIN;
goto unlock_dma_resv;
}
}
- /*
- * XXX: Should we drop the lock before waiting? This only helps if doing
- * GPU binds which is currently only done if we have to wait for more
- * than 10ms on a move.
- */
dma_fence_wait(fence, false);
dma_fence_put(fence);
-
- if (xe_vma_is_userptr(vma))
- ret = xe_vma_userptr_check_repin(to_userptr_vma(vma));
vma->tile_invalidated &= ~BIT(tile->id);
unlock_dma_resv:
drm_exec_fini(&exec);
-unlock_vm:
- if (!ret)
- vm->usm.last_fault_vma = vma;
- if (write_locked)
- up_write(&vm->lock);
- else
- up_read(&vm->lock);
- if (ret == -EAGAIN)
+ if (err == -EAGAIN)
goto retry_userptr;
- if (!ret) {
- ret = xe_gt_tlb_invalidation_vma(gt, NULL, vma);
- if (ret >= 0)
- ret = 0;
+ return err;
+}
+
+static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
+{
+ struct xe_device *xe = gt_to_xe(gt);
+ struct xe_tile *tile = gt_to_tile(gt);
+ struct xe_vm *vm;
+ struct xe_vma *vma = NULL;
+ int err;
+
+ /* SW isn't expected to handle TRTT faults */
+ if (pf->trva_fault)
+ return -EFAULT;
+
+ /* ASID to VM */
+ mutex_lock(&xe->usm.lock);
+ vm = xa_load(&xe->usm.asid_to_vm, pf->asid);
+ if (vm && xe_vm_in_fault_mode(vm))
+ xe_vm_get(vm);
+ else
+ vm = NULL;
+ mutex_unlock(&xe->usm.lock);
+ if (!vm)
+ return -EINVAL;
+
+ /*
+ * TODO: Change to read lock? Using write lock for simplicity.
+ */
+ down_write(&vm->lock);
+ vma = lookup_vma(vm, pf->page_addr);
+ if (!vma) {
+ err = -EINVAL;
+ goto unlock_vm;
}
+
+ err = handle_vma_pagefault(tile, pf, vma);
+
+unlock_vm:
+ if (!err)
+ vm->usm.last_fault_vma = vma;
+ up_write(&vm->lock);
xe_vm_put(vm);
- return ret;
+ return err;
}
static int send_pagefault_reply(struct xe_guc *guc,
--
2.34.1
^ permalink raw reply related [flat|nested] 4+ messages in thread
* Re: [PATCH] drm/xe: Rework GPU page fault handling
2024-06-07 1:52 Matthew Brost
@ 2024-06-07 2:25 ` Randhawa, Jagmeet
0 siblings, 0 replies; 4+ messages in thread
From: Randhawa, Jagmeet @ 2024-06-07 2:25 UTC (permalink / raw)
To: Matthew Brost, intel-xe
On 6/6/2024 6:52 PM, Matthew Brost wrote:
> Add helper function to implement VMA (user binding) page faults, remove
> unnecessary userptr.invalidate_link list del operation, retry on memory
> pressure, remove unnecessary xe_vma_userptr_check_repin after rebinding,
> remove unnecessary TLB invalidation, and always use vm->lock in write
> mode. Changes help facilitate SVM page faults.
>
> Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> Reviewed-by: Nirmoy Das <nirmoy.das@intel.com>
> ---
> drivers/gpu/drm/xe/xe_gt_pagefault.c | 142 ++++++++++++---------------
> 1 file changed, 62 insertions(+), 80 deletions(-)
>
> diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c
> index 040dd142c49c..eaf68f0135c1 100644
> --- a/drivers/gpu/drm/xe/xe_gt_pagefault.c
> +++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c
> @@ -125,126 +125,108 @@ static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma,
> return 0;
> }
>
> -static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
> +static int handle_vma_pagefault(struct xe_tile *tile, struct pagefault *pf,
> + struct xe_vma *vma)
> {
> - struct xe_device *xe = gt_to_xe(gt);
> - struct xe_tile *tile = gt_to_tile(gt);
> + struct xe_vm *vm = xe_vma_vm(vma);
> struct drm_exec exec;
> - struct xe_vm *vm;
> - struct xe_vma *vma = NULL;
> struct dma_fence *fence;
> - bool write_locked;
> - int ret = 0;
> + ktime_t end = 0;
> + int err;
> bool atomic;
>
> - /* SW isn't expected to handle TRTT faults */
> - if (pf->trva_fault)
> - return -EFAULT;
> -
> - /* ASID to VM */
> - mutex_lock(&xe->usm.lock);
> - vm = xa_load(&xe->usm.asid_to_vm, pf->asid);
> - if (vm && xe_vm_in_fault_mode(vm))
> - xe_vm_get(vm);
> - else
> - vm = NULL;
> - mutex_unlock(&xe->usm.lock);
> - if (!vm)
> - return -EINVAL;
> -
> -retry_userptr:
> - /*
> - * TODO: Avoid exclusive lock if VM doesn't have userptrs, or
> - * start out read-locked?
> - */
> - down_write(&vm->lock);
> - write_locked = true;
> - vma = lookup_vma(vm, pf->page_addr);
> - if (!vma) {
> - ret = -EINVAL;
> - goto unlock_vm;
> - }
> -
> - if (!xe_vma_is_userptr(vma) ||
> - !xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
> - downgrade_write(&vm->lock);
> - write_locked = false;
> - }
> -
> trace_xe_vma_pagefault(vma);
> -
> atomic = access_is_atomic(pf->access_type);
>
> /* Check if VMA is valid */
> if (vma_is_valid(tile, vma) && !atomic)
> - goto unlock_vm;
> -
> - /* TODO: Validate fault */
> + return 0;
>
> - if (xe_vma_is_userptr(vma) && write_locked) {
> +retry_userptr:
> + if (xe_vma_is_userptr(vma) &&
> + xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
> struct xe_userptr_vma *uvma = to_userptr_vma(vma);
>
> - spin_lock(&vm->userptr.invalidated_lock);
> - list_del_init(&uvma->userptr.invalidate_link);
> - spin_unlock(&vm->userptr.invalidated_lock);
> -
> - ret = xe_vma_userptr_pin_pages(uvma);
> - if (ret)
> - goto unlock_vm;
> -
> - downgrade_write(&vm->lock);
> - write_locked = false;
> + err = xe_vma_userptr_pin_pages(uvma);
> + if (err)
> + return err;
> }
>
> /* Lock VM and BOs dma-resv */
> drm_exec_init(&exec, 0, 0);
> drm_exec_until_all_locked(&exec) {
> - ret = xe_pf_begin(&exec, vma, atomic, tile->id);
> + err = xe_pf_begin(&exec, vma, atomic, tile->id);
> drm_exec_retry_on_contention(&exec);
> - if (ret)
> + if (xe_vm_validate_should_retry(&exec, err, &end))
> + err = -EAGAIN;
> + if (err)
> goto unlock_dma_resv;
>
> /* Bind VMA only to the GT that has faulted */
> trace_xe_vma_pf_bind(vma);
> fence = xe_vma_rebind(vm, vma, BIT(tile->id));
> if (IS_ERR(fence)) {
> - ret = PTR_ERR(fence);
> + err = PTR_ERR(fence);
> + if (xe_vm_validate_should_retry(&exec, err, &end))
> + err = -EAGAIN;
> goto unlock_dma_resv;
> }
> }
>
> - /*
> - * XXX: Should we drop the lock before waiting? This only helps if doing
> - * GPU binds which is currently only done if we have to wait for more
> - * than 10ms on a move.
> - */
> dma_fence_wait(fence, false);
> dma_fence_put(fence);
> -
> - if (xe_vma_is_userptr(vma))
> - ret = xe_vma_userptr_check_repin(to_userptr_vma(vma));
> vma->tile_invalidated &= ~BIT(tile->id);
>
> unlock_dma_resv:
> drm_exec_fini(&exec);
> -unlock_vm:
> - if (!ret)
> - vm->usm.last_fault_vma = vma;
> - if (write_locked)
> - up_write(&vm->lock);
> - else
> - up_read(&vm->lock);
> - if (ret == -EAGAIN)
> + if (err == -EAGAIN)
> goto retry_userptr;
>
> - if (!ret) {
> - ret = xe_gt_tlb_invalidation_vma(gt, NULL, vma);
> - if (ret >= 0)
> - ret = 0;
> + return err;
> +}
> +
> +static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
> +{
> + struct xe_device *xe = gt_to_xe(gt);
> + struct xe_tile *tile = gt_to_tile(gt);
> + struct xe_vm *vm;
> + struct xe_vma *vma = NULL;
> + int err;
> +
> + /* SW isn't expected to handle TRTT faults */
> + if (pf->trva_fault)
> + return -EFAULT;
> +
> + /* ASID to VM */
> + mutex_lock(&xe->usm.lock);
> + vm = xa_load(&xe->usm.asid_to_vm, pf->asid);
> + if (vm && xe_vm_in_fault_mode(vm))
> + xe_vm_get(vm);
> + else
> + vm = NULL;
> + mutex_unlock(&xe->usm.lock);
> + if (!vm)
> + return -EINVAL;
> +
> + /*
> + * TODO: Change to read lock? Using write lock for simplicity.
> + */
> + down_write(&vm->lock);
> + vma = lookup_vma(vm, pf->page_addr);
> + if (!vma) {
> + err = -EINVAL;
> + goto unlock_vm;
> }
> +
> + err = handle_vma_pagefault(tile, pf, vma);
> +
> +unlock_vm:
> + if (!err)
> + vm->usm.last_fault_vma = vma;
> + up_write(&vm->lock);
> xe_vm_put(vm);
>
> - return ret;
> + return err;
> }
>
> static int send_pagefault_reply(struct xe_guc *guc,
Reviewed-by: Jagmeet Randhawa <jagmeet.randhawa@intel.com>
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2024-06-07 2:26 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2024-05-14 21:55 [PATCH] drm/xe: Rework GPU page fault handling Matthew Brost
2024-06-06 11:36 ` Nirmoy Das
-- strict thread matches above, loose matches on Subject: below --
2024-06-07 1:52 Matthew Brost
2024-06-07 2:25 ` Randhawa, Jagmeet
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).