* [Intel-xe] [PATCH v2 01/27] drm/xe: Allow num_binds == 0 in VM bind IOCTL
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-08 10:53 ` Dafna Hirschfeld
` (3 more replies)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 02/27] drm/xe: Allow num_batch_buffer == 0 in exec IOCTL Matthew Brost
` (29 subsequent siblings)
30 siblings, 4 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
The idea being out-syncs can signal indicating all previous operations
on the bind queue are complete. An example use case of this would be
support for implementing vkQueueWaitForIdle easily.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_vm.c | 30 ++++++++++++++++++------------
1 file changed, 18 insertions(+), 12 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index d26c90f0d702..403444ff3856 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -2850,7 +2850,6 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe,
int i;
if (XE_IOCTL_DBG(xe, args->extensions) ||
- XE_IOCTL_DBG(xe, !args->num_binds) ||
XE_IOCTL_DBG(xe, args->num_binds > MAX_BINDS))
return -EINVAL;
@@ -2977,7 +2976,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
goto put_exec_queue;
}
- if (XE_IOCTL_DBG(xe, async !=
+ if (XE_IOCTL_DBG(xe, args->num_binds && async !=
!!(q->flags & EXEC_QUEUE_FLAG_VM_ASYNC))) {
err = -EINVAL;
goto put_exec_queue;
@@ -2991,7 +2990,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
}
if (!args->exec_queue_id) {
- if (XE_IOCTL_DBG(xe, async !=
+ if (XE_IOCTL_DBG(xe, args->num_binds && async !=
!!(vm->flags & XE_VM_FLAG_ASYNC_DEFAULT))) {
err = -EINVAL;
goto put_vm;
@@ -3028,16 +3027,18 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
}
}
- bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL);
- if (!bos) {
- err = -ENOMEM;
- goto release_vm_lock;
- }
+ if (args->num_binds) {
+ bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL);
+ if (!bos) {
+ err = -ENOMEM;
+ goto release_vm_lock;
+ }
- ops = kzalloc(sizeof(*ops) * args->num_binds, GFP_KERNEL);
- if (!ops) {
- err = -ENOMEM;
- goto release_vm_lock;
+ ops = kzalloc(sizeof(*ops) * args->num_binds, GFP_KERNEL);
+ if (!ops) {
+ err = -ENOMEM;
+ goto release_vm_lock;
+ }
}
for (i = 0; i < args->num_binds; ++i) {
@@ -3092,6 +3093,11 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
goto free_syncs;
}
+ if (!args->num_binds) {
+ err = -ENODATA;
+ goto free_syncs;
+ }
+
for (i = 0; i < args->num_binds; ++i) {
u64 range = bind_ops[i].range;
u64 addr = bind_ops[i].addr;
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* Re: [Intel-xe] [PATCH v2 01/27] drm/xe: Allow num_binds == 0 in VM bind IOCTL
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 01/27] drm/xe: Allow num_binds == 0 in VM bind IOCTL Matthew Brost
@ 2023-11-08 10:53 ` Dafna Hirschfeld
2023-11-08 10:55 ` Dafna Hirschfeld
` (2 subsequent siblings)
3 siblings, 0 replies; 43+ messages in thread
From: Dafna Hirschfeld @ 2023-11-08 10:53 UTC (permalink / raw)
To: Matthew Brost; +Cc: intel-xe
On 06.11.2023 21:25, Matthew Brost wrote:
>The idea being out-syncs can signal indicating all previous operations
>on the bind queue are complete. An example use case of this would be
>support for implementing vkQueueWaitForIdle easily.
>
>Signed-off-by: Matthew Brost <matthew.brost@intel.com>
>---
> drivers/gpu/drm/xe/xe_vm.c | 30 ++++++++++++++++++------------
> 1 file changed, 18 insertions(+), 12 deletions(-)
>
>diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
>index d26c90f0d702..403444ff3856 100644
>--- a/drivers/gpu/drm/xe/xe_vm.c
>+++ b/drivers/gpu/drm/xe/xe_vm.c
>@@ -2850,7 +2850,6 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe,
> int i;
>
> if (XE_IOCTL_DBG(xe, args->extensions) ||
>- XE_IOCTL_DBG(xe, !args->num_binds) ||
> XE_IOCTL_DBG(xe, args->num_binds > MAX_BINDS))
> return -EINVAL;
>
>@@ -2977,7 +2976,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> goto put_exec_queue;
> }
>
>- if (XE_IOCTL_DBG(xe, async !=
>+ if (XE_IOCTL_DBG(xe, args->num_binds && async !=
> !!(q->flags & EXEC_QUEUE_FLAG_VM_ASYNC))) {
> err = -EINVAL;
> goto put_exec_queue;
>@@ -2991,7 +2990,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> }
>
> if (!args->exec_queue_id) {
>- if (XE_IOCTL_DBG(xe, async !=
>+ if (XE_IOCTL_DBG(xe, args->num_binds && async !=
> !!(vm->flags & XE_VM_FLAG_ASYNC_DEFAULT))) {
> err = -EINVAL;
> goto put_vm;
>@@ -3028,16 +3027,18 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> }
> }
>
>- bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL);
>- if (!bos) {
>- err = -ENOMEM;
>- goto release_vm_lock;
>- }
>+ if (args->num_binds) {
>+ bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL);
>+ if (!bos) {
>+ err = -ENOMEM;
>+ goto release_vm_lock;
>+ }
>
>- ops = kzalloc(sizeof(*ops) * args->num_binds, GFP_KERNEL);
>- if (!ops) {
>- err = -ENOMEM;
>- goto release_vm_lock;
>+ ops = kzalloc(sizeof(*ops) * args->num_binds, GFP_KERNEL);
>+ if (!ops) {
>+ err = -ENOMEM;
>+ goto release_vm_lock;
>+ }
> }
>
> for (i = 0; i < args->num_binds; ++i) {
>@@ -3092,6 +3093,11 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> goto free_syncs;
> }
>
>+ if (!args->num_binds) {
>+ err = -ENODATA;
>+ goto free_syncs;
>+ }
>+
> for (i = 0; i < args->num_binds; ++i) {
> u64 range = bind_ops[i].range;
> u64 addr = bind_ops[i].addr;
>--
>2.34.1
>
^ permalink raw reply [flat|nested] 43+ messages in thread* Re: [Intel-xe] [PATCH v2 01/27] drm/xe: Allow num_binds == 0 in VM bind IOCTL
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 01/27] drm/xe: Allow num_binds == 0 in VM bind IOCTL Matthew Brost
2023-11-08 10:53 ` Dafna Hirschfeld
@ 2023-11-08 10:55 ` Dafna Hirschfeld
2023-11-09 14:06 ` Matthew Brost
2023-11-08 11:30 ` Lionel Landwerlin
2023-11-10 10:35 ` Thomas Hellström
3 siblings, 1 reply; 43+ messages in thread
From: Dafna Hirschfeld @ 2023-11-08 10:55 UTC (permalink / raw)
To: Matthew Brost; +Cc: intel-xe
On 06.11.2023 21:25, Matthew Brost wrote:
>The idea being out-syncs can signal indicating all previous operations
>on the bind queue are complete. An example use case of this would be
>support for implementing vkQueueWaitForIdle easily.
>
>Signed-off-by: Matthew Brost <matthew.brost@intel.com>
>---
> drivers/gpu/drm/xe/xe_vm.c | 30 ++++++++++++++++++------------
> 1 file changed, 18 insertions(+), 12 deletions(-)
>
>diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
>index d26c90f0d702..403444ff3856 100644
>--- a/drivers/gpu/drm/xe/xe_vm.c
>+++ b/drivers/gpu/drm/xe/xe_vm.c
>@@ -2850,7 +2850,6 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe,
> int i;
>
> if (XE_IOCTL_DBG(xe, args->extensions) ||
>- XE_IOCTL_DBG(xe, !args->num_binds) ||
> XE_IOCTL_DBG(xe, args->num_binds > MAX_BINDS))
> return -EINVAL;
>
>@@ -2977,7 +2976,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> goto put_exec_queue;
> }
>
>- if (XE_IOCTL_DBG(xe, async !=
>+ if (XE_IOCTL_DBG(xe, args->num_binds && async !=
> !!(q->flags & EXEC_QUEUE_FLAG_VM_ASYNC))) {
> err = -EINVAL;
> goto put_exec_queue;
>@@ -2991,7 +2990,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> }
>
> if (!args->exec_queue_id) {
>- if (XE_IOCTL_DBG(xe, async !=
>+ if (XE_IOCTL_DBG(xe, args->num_binds && async !=
> !!(vm->flags & XE_VM_FLAG_ASYNC_DEFAULT))) {
> err = -EINVAL;
> goto put_vm;
>@@ -3028,16 +3027,18 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> }
> }
>
>- bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL);
>- if (!bos) {
>- err = -ENOMEM;
>- goto release_vm_lock;
>- }
>+ if (args->num_binds) {
>+ bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL);
>+ if (!bos) {
>+ err = -ENOMEM;
>+ goto release_vm_lock;
>+ }
>
>- ops = kzalloc(sizeof(*ops) * args->num_binds, GFP_KERNEL);
>- if (!ops) {
>- err = -ENOMEM;
>- goto release_vm_lock;
>+ ops = kzalloc(sizeof(*ops) * args->num_binds, GFP_KERNEL);
>+ if (!ops) {
>+ err = -ENOMEM;
>+ goto release_vm_lock;
>+ }
> }
>
> for (i = 0; i < args->num_binds; ++i) {
>@@ -3092,6 +3093,11 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> goto free_syncs;
> }
>
>+ if (!args->num_binds) {
>+ err = -ENODATA;
>+ goto free_syncs;
if args->num_binds = 0 is valid, then it might be confusing to indicate it with err = -ENODATA,
Also , it is invalid if both args->num_binds and args->num_syncs are 0 right? if so maybe
add XE_IOCTL_DBG for that,
Thanks,
Dafna
>+ }
>+
> for (i = 0; i < args->num_binds; ++i) {
> u64 range = bind_ops[i].range;
> u64 addr = bind_ops[i].addr;
>--
>2.34.1
>
^ permalink raw reply [flat|nested] 43+ messages in thread* Re: [Intel-xe] [PATCH v2 01/27] drm/xe: Allow num_binds == 0 in VM bind IOCTL
2023-11-08 10:55 ` Dafna Hirschfeld
@ 2023-11-09 14:06 ` Matthew Brost
0 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-09 14:06 UTC (permalink / raw)
To: Dafna Hirschfeld; +Cc: intel-xe
On Wed, Nov 08, 2023 at 12:55:35PM +0200, Dafna Hirschfeld wrote:
> On 06.11.2023 21:25, Matthew Brost wrote:
> > The idea being out-syncs can signal indicating all previous operations
> > on the bind queue are complete. An example use case of this would be
> > support for implementing vkQueueWaitForIdle easily.
> >
> > Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> > ---
> > drivers/gpu/drm/xe/xe_vm.c | 30 ++++++++++++++++++------------
> > 1 file changed, 18 insertions(+), 12 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
> > index d26c90f0d702..403444ff3856 100644
> > --- a/drivers/gpu/drm/xe/xe_vm.c
> > +++ b/drivers/gpu/drm/xe/xe_vm.c
> > @@ -2850,7 +2850,6 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe,
> > int i;
> >
> > if (XE_IOCTL_DBG(xe, args->extensions) ||
> > - XE_IOCTL_DBG(xe, !args->num_binds) ||
> > XE_IOCTL_DBG(xe, args->num_binds > MAX_BINDS))
> > return -EINVAL;
> >
> > @@ -2977,7 +2976,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> > goto put_exec_queue;
> > }
> >
> > - if (XE_IOCTL_DBG(xe, async !=
> > + if (XE_IOCTL_DBG(xe, args->num_binds && async !=
> > !!(q->flags & EXEC_QUEUE_FLAG_VM_ASYNC))) {
> > err = -EINVAL;
> > goto put_exec_queue;
> > @@ -2991,7 +2990,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> > }
> >
> > if (!args->exec_queue_id) {
> > - if (XE_IOCTL_DBG(xe, async !=
> > + if (XE_IOCTL_DBG(xe, args->num_binds && async !=
> > !!(vm->flags & XE_VM_FLAG_ASYNC_DEFAULT))) {
> > err = -EINVAL;
> > goto put_vm;
> > @@ -3028,16 +3027,18 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> > }
> > }
> >
> > - bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL);
> > - if (!bos) {
> > - err = -ENOMEM;
> > - goto release_vm_lock;
> > - }
> > + if (args->num_binds) {
> > + bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL);
> > + if (!bos) {
> > + err = -ENOMEM;
> > + goto release_vm_lock;
> > + }
> >
> > - ops = kzalloc(sizeof(*ops) * args->num_binds, GFP_KERNEL);
> > - if (!ops) {
> > - err = -ENOMEM;
> > - goto release_vm_lock;
> > + ops = kzalloc(sizeof(*ops) * args->num_binds, GFP_KERNEL);
> > + if (!ops) {
> > + err = -ENOMEM;
> > + goto release_vm_lock;
> > + }
> > }
> >
> > for (i = 0; i < args->num_binds; ++i) {
> > @@ -3092,6 +3093,11 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> > goto free_syncs;
> > }
> >
> > + if (!args->num_binds) {
> > + err = -ENODATA;
> > + goto free_syncs;
> if args->num_binds = 0 is valid, then it might be confusing to indicate it with err = -ENODATA,
-ENODATA gets concerted to 0 at the end of xe_vm_bind_ioctl. Internally
-ENODATA indicates to signal out-fences.
> Also , it is invalid if both args->num_binds and args->num_syncs are 0 right? if so maybe
> add XE_IOCTL_DBG for that,
It is not invalid rather just a odd use case.
Matt
>
> Thanks,
> Dafna
>
> > + }
> > +
> > for (i = 0; i < args->num_binds; ++i) {
> > u64 range = bind_ops[i].range;
> > u64 addr = bind_ops[i].addr;
> > --
> > 2.34.1
> >
^ permalink raw reply [flat|nested] 43+ messages in thread
* Re: [Intel-xe] [PATCH v2 01/27] drm/xe: Allow num_binds == 0 in VM bind IOCTL
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 01/27] drm/xe: Allow num_binds == 0 in VM bind IOCTL Matthew Brost
2023-11-08 10:53 ` Dafna Hirschfeld
2023-11-08 10:55 ` Dafna Hirschfeld
@ 2023-11-08 11:30 ` Lionel Landwerlin
2023-11-09 13:59 ` Matthew Brost
2023-11-10 10:35 ` Thomas Hellström
3 siblings, 1 reply; 43+ messages in thread
From: Lionel Landwerlin @ 2023-11-08 11:30 UTC (permalink / raw)
To: Matthew Brost, intel-xe
On 07/11/2023 07:25, Matthew Brost wrote:
> The idea being out-syncs can signal indicating all previous operations
> on the bind queue are complete. An example use case of this would be
> support for implementing vkQueueWaitForIdle easily.
>
> Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> ---
> drivers/gpu/drm/xe/xe_vm.c | 30 ++++++++++++++++++------------
> 1 file changed, 18 insertions(+), 12 deletions(-)
>
> diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
> index d26c90f0d702..403444ff3856 100644
> --- a/drivers/gpu/drm/xe/xe_vm.c
> +++ b/drivers/gpu/drm/xe/xe_vm.c
> @@ -2850,7 +2850,6 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe,
> int i;
>
> if (XE_IOCTL_DBG(xe, args->extensions) ||
> - XE_IOCTL_DBG(xe, !args->num_binds) ||
> XE_IOCTL_DBG(xe, args->num_binds > MAX_BINDS))
Just randomly noticed this, but is there a uAPI for the MAX_BINDS?
Or do we need to hardcode that in the UMD and hope it never changes?
-Lionel
> return -EINVAL;
>
> @@ -2977,7 +2976,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> goto put_exec_queue;
> }
>
> - if (XE_IOCTL_DBG(xe, async !=
> + if (XE_IOCTL_DBG(xe, args->num_binds && async !=
> !!(q->flags & EXEC_QUEUE_FLAG_VM_ASYNC))) {
> err = -EINVAL;
> goto put_exec_queue;
> @@ -2991,7 +2990,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> }
>
> if (!args->exec_queue_id) {
> - if (XE_IOCTL_DBG(xe, async !=
> + if (XE_IOCTL_DBG(xe, args->num_binds && async !=
> !!(vm->flags & XE_VM_FLAG_ASYNC_DEFAULT))) {
> err = -EINVAL;
> goto put_vm;
> @@ -3028,16 +3027,18 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> }
> }
>
> - bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL);
> - if (!bos) {
> - err = -ENOMEM;
> - goto release_vm_lock;
> - }
> + if (args->num_binds) {
> + bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL);
> + if (!bos) {
> + err = -ENOMEM;
> + goto release_vm_lock;
> + }
>
> - ops = kzalloc(sizeof(*ops) * args->num_binds, GFP_KERNEL);
> - if (!ops) {
> - err = -ENOMEM;
> - goto release_vm_lock;
> + ops = kzalloc(sizeof(*ops) * args->num_binds, GFP_KERNEL);
> + if (!ops) {
> + err = -ENOMEM;
> + goto release_vm_lock;
> + }
> }
>
> for (i = 0; i < args->num_binds; ++i) {
> @@ -3092,6 +3093,11 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> goto free_syncs;
> }
>
> + if (!args->num_binds) {
> + err = -ENODATA;
> + goto free_syncs;
> + }
> +
> for (i = 0; i < args->num_binds; ++i) {
> u64 range = bind_ops[i].range;
> u64 addr = bind_ops[i].addr;
^ permalink raw reply [flat|nested] 43+ messages in thread* Re: [Intel-xe] [PATCH v2 01/27] drm/xe: Allow num_binds == 0 in VM bind IOCTL
2023-11-08 11:30 ` Lionel Landwerlin
@ 2023-11-09 13:59 ` Matthew Brost
0 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-09 13:59 UTC (permalink / raw)
To: Lionel Landwerlin; +Cc: intel-xe
On Wed, Nov 08, 2023 at 01:30:56PM +0200, Lionel Landwerlin wrote:
> On 07/11/2023 07:25, Matthew Brost wrote:
> > The idea being out-syncs can signal indicating all previous operations
> > on the bind queue are complete. An example use case of this would be
> > support for implementing vkQueueWaitForIdle easily.
> >
> > Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> > ---
> > drivers/gpu/drm/xe/xe_vm.c | 30 ++++++++++++++++++------------
> > 1 file changed, 18 insertions(+), 12 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
> > index d26c90f0d702..403444ff3856 100644
> > --- a/drivers/gpu/drm/xe/xe_vm.c
> > +++ b/drivers/gpu/drm/xe/xe_vm.c
> > @@ -2850,7 +2850,6 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe,
> > int i;
> > if (XE_IOCTL_DBG(xe, args->extensions) ||
> > - XE_IOCTL_DBG(xe, !args->num_binds) ||
> > XE_IOCTL_DBG(xe, args->num_binds > MAX_BINDS))
>
>
> Just randomly noticed this, but is there a uAPI for the MAX_BINDS?
>
> Or do we need to hardcode that in the UMD and hope it never changes?
>
We should probably report this somewhere. Let me look into this.
Thanks - Matt
>
> -Lionel
>
>
> > return -EINVAL;
> > @@ -2977,7 +2976,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> > goto put_exec_queue;
> > }
> > - if (XE_IOCTL_DBG(xe, async !=
> > + if (XE_IOCTL_DBG(xe, args->num_binds && async !=
> > !!(q->flags & EXEC_QUEUE_FLAG_VM_ASYNC))) {
> > err = -EINVAL;
> > goto put_exec_queue;
> > @@ -2991,7 +2990,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> > }
> > if (!args->exec_queue_id) {
> > - if (XE_IOCTL_DBG(xe, async !=
> > + if (XE_IOCTL_DBG(xe, args->num_binds && async !=
> > !!(vm->flags & XE_VM_FLAG_ASYNC_DEFAULT))) {
> > err = -EINVAL;
> > goto put_vm;
> > @@ -3028,16 +3027,18 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> > }
> > }
> > - bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL);
> > - if (!bos) {
> > - err = -ENOMEM;
> > - goto release_vm_lock;
> > - }
> > + if (args->num_binds) {
> > + bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL);
> > + if (!bos) {
> > + err = -ENOMEM;
> > + goto release_vm_lock;
> > + }
> > - ops = kzalloc(sizeof(*ops) * args->num_binds, GFP_KERNEL);
> > - if (!ops) {
> > - err = -ENOMEM;
> > - goto release_vm_lock;
> > + ops = kzalloc(sizeof(*ops) * args->num_binds, GFP_KERNEL);
> > + if (!ops) {
> > + err = -ENOMEM;
> > + goto release_vm_lock;
> > + }
> > }
> > for (i = 0; i < args->num_binds; ++i) {
> > @@ -3092,6 +3093,11 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> > goto free_syncs;
> > }
> > + if (!args->num_binds) {
> > + err = -ENODATA;
> > + goto free_syncs;
> > + }
> > +
> > for (i = 0; i < args->num_binds; ++i) {
> > u64 range = bind_ops[i].range;
> > u64 addr = bind_ops[i].addr;
>
>
^ permalink raw reply [flat|nested] 43+ messages in thread
* Re: [Intel-xe] [PATCH v2 01/27] drm/xe: Allow num_binds == 0 in VM bind IOCTL
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 01/27] drm/xe: Allow num_binds == 0 in VM bind IOCTL Matthew Brost
` (2 preceding siblings ...)
2023-11-08 11:30 ` Lionel Landwerlin
@ 2023-11-10 10:35 ` Thomas Hellström
3 siblings, 0 replies; 43+ messages in thread
From: Thomas Hellström @ 2023-11-10 10:35 UTC (permalink / raw)
To: Matthew Brost, intel-xe
[-- Attachment #1: Type: text/plain, Size: 2708 bytes --]
On 11/7/23 06:25, Matthew Brost wrote:
> The idea being out-syncs can signal indicating all previous operations
> on the bind queue are complete. An example use case of this would be
> support for implementing vkQueueWaitForIdle easily.
>
> Signed-off-by: Matthew Brost<matthew.brost@intel.com>
Reviewed-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
> ---
> drivers/gpu/drm/xe/xe_vm.c | 30 ++++++++++++++++++------------
> 1 file changed, 18 insertions(+), 12 deletions(-)
>
> diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
> index d26c90f0d702..403444ff3856 100644
> --- a/drivers/gpu/drm/xe/xe_vm.c
> +++ b/drivers/gpu/drm/xe/xe_vm.c
> @@ -2850,7 +2850,6 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe,
> int i;
>
> if (XE_IOCTL_DBG(xe, args->extensions) ||
> - XE_IOCTL_DBG(xe, !args->num_binds) ||
> XE_IOCTL_DBG(xe, args->num_binds > MAX_BINDS))
> return -EINVAL;
>
> @@ -2977,7 +2976,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> goto put_exec_queue;
> }
>
> - if (XE_IOCTL_DBG(xe, async !=
> + if (XE_IOCTL_DBG(xe, args->num_binds && async !=
> !!(q->flags & EXEC_QUEUE_FLAG_VM_ASYNC))) {
> err = -EINVAL;
> goto put_exec_queue;
> @@ -2991,7 +2990,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> }
>
> if (!args->exec_queue_id) {
> - if (XE_IOCTL_DBG(xe, async !=
> + if (XE_IOCTL_DBG(xe, args->num_binds && async !=
> !!(vm->flags & XE_VM_FLAG_ASYNC_DEFAULT))) {
> err = -EINVAL;
> goto put_vm;
> @@ -3028,16 +3027,18 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> }
> }
>
> - bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL);
> - if (!bos) {
> - err = -ENOMEM;
> - goto release_vm_lock;
> - }
> + if (args->num_binds) {
> + bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL);
> + if (!bos) {
> + err = -ENOMEM;
> + goto release_vm_lock;
> + }
>
> - ops = kzalloc(sizeof(*ops) * args->num_binds, GFP_KERNEL);
> - if (!ops) {
> - err = -ENOMEM;
> - goto release_vm_lock;
> + ops = kzalloc(sizeof(*ops) * args->num_binds, GFP_KERNEL);
> + if (!ops) {
> + err = -ENOMEM;
> + goto release_vm_lock;
> + }
> }
>
> for (i = 0; i < args->num_binds; ++i) {
> @@ -3092,6 +3093,11 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> goto free_syncs;
> }
>
> + if (!args->num_binds) {
> + err = -ENODATA;
> + goto free_syncs;
> + }
> +
> for (i = 0; i < args->num_binds; ++i) {
> u64 range = bind_ops[i].range;
> u64 addr = bind_ops[i].addr;
[-- Attachment #2: Type: text/html, Size: 3370 bytes --]
^ permalink raw reply [flat|nested] 43+ messages in thread
* [Intel-xe] [PATCH v2 02/27] drm/xe: Allow num_batch_buffer == 0 in exec IOCTL
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 01/27] drm/xe: Allow num_binds == 0 in VM bind IOCTL Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-10 11:11 ` Thomas Hellström
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 03/27] drm/xe: Take in-syncs into account when num_execs or num_binds == 0 Matthew Brost
` (28 subsequent siblings)
30 siblings, 1 reply; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
The idea being out-syncs can signal indicating all previous operations
on the exec queue are complete. An example use case of this would be
support for implementing vkQueueWaitForIdle easily.
v2: Don't add last_fence for VM's that do not support dma fences
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_exec.c | 22 +++++++++++++++++++---
drivers/gpu/drm/xe/xe_exec_queue.c | 5 ++++-
drivers/gpu/drm/xe/xe_exec_queue_types.h | 5 +++--
drivers/gpu/drm/xe/xe_sync.c | 5 ++++-
drivers/gpu/drm/xe/xe_sync.h | 2 +-
drivers/gpu/drm/xe/xe_vm.c | 2 +-
6 files changed, 32 insertions(+), 9 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c
index 28e84a0bbeb0..4666f5b145f7 100644
--- a/drivers/gpu/drm/xe/xe_exec.c
+++ b/drivers/gpu/drm/xe/xe_exec.c
@@ -161,7 +161,8 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM))
return -EINVAL;
- if (XE_IOCTL_DBG(xe, q->width != args->num_batch_buffer))
+ if (XE_IOCTL_DBG(xe, args->num_batch_buffer &&
+ q->width != args->num_batch_buffer))
return -EINVAL;
if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_BANNED)) {
@@ -182,12 +183,13 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
for (i = 0; i < args->num_syncs; i++) {
err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs++],
&syncs_user[i], true,
- xe_vm_no_dma_fences(vm));
+ xe_vm_no_dma_fences(vm),
+ !args->num_batch_buffer);
if (err)
goto err_syncs;
}
- if (xe_exec_queue_is_parallel(q)) {
+ if (args->num_batch_buffer && xe_exec_queue_is_parallel(q)) {
err = __copy_from_user(addresses, addresses_user, sizeof(u64) *
q->width);
if (err) {
@@ -234,6 +236,18 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
goto err_exec;
}
+ if (!args->num_batch_buffer) {
+ if (!xe_vm_no_dma_fences(vm)) {
+ struct dma_fence *fence =
+ xe_exec_queue_last_fence_get(q, vm);
+
+ for (i = 0; i < num_syncs; i++)
+ xe_sync_entry_signal(&syncs[i], NULL, fence);
+ }
+
+ goto err_exec;
+ }
+
if (xe_exec_queue_is_lr(q) && xe_exec_queue_ring_full(q)) {
err = -EWOULDBLOCK;
goto err_exec;
@@ -327,6 +341,8 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
if (xe_exec_queue_is_lr(q))
q->ring_ops->emit_job(job);
+ if (!xe_vm_no_dma_fences(vm))
+ xe_exec_queue_last_fence_set(q, vm, &job->drm.s_fence->finished);
xe_sched_job_push(job);
xe_vm_reactivate_rebind(vm);
diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c
index 4fd44a9203e4..35710b66e5de 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue.c
+++ b/drivers/gpu/drm/xe/xe_exec_queue.c
@@ -924,7 +924,10 @@ int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data,
static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q,
struct xe_vm *vm)
{
- lockdep_assert_held_write(&vm->lock);
+ if (q->flags & EXEC_QUEUE_FLAG_VM)
+ lockdep_assert_held_write(&vm->lock);
+ else
+ xe_vm_assert_held(vm);
}
/**
diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h
index ecd761177567..35ffe7c55f25 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue_types.h
+++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h
@@ -53,8 +53,9 @@ struct xe_exec_queue {
struct xe_hw_fence_irq *fence_irq;
/**
- * @last_fence: last fence on engine, protected by vm->lock in write
- * mode if bind engine
+ * @last_fence: last fence on exec queue, protected by vm->lock in write
+ * mode if bind exec queue, protected by dma resv lock if non-bind exec
+ * queue
*/
struct dma_fence *last_fence;
diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c
index 73ef259aa387..2461e7d4814c 100644
--- a/drivers/gpu/drm/xe/xe_sync.c
+++ b/drivers/gpu/drm/xe/xe_sync.c
@@ -100,7 +100,7 @@ static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef,
struct xe_sync_entry *sync,
struct drm_xe_sync __user *sync_user,
- bool exec, bool no_dma_fences)
+ bool exec, bool no_dma_fences, bool exec_nop)
{
struct drm_xe_sync sync_in;
int err;
@@ -171,6 +171,9 @@ int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef,
break;
case DRM_XE_SYNC_USER_FENCE:
+ if (XE_IOCTL_DBG(xe, exec_nop))
+ return -EOPNOTSUPP;
+
if (XE_IOCTL_DBG(xe, !signal))
return -EOPNOTSUPP;
diff --git a/drivers/gpu/drm/xe/xe_sync.h b/drivers/gpu/drm/xe/xe_sync.h
index 30958ddc4cdc..98f02bb34637 100644
--- a/drivers/gpu/drm/xe/xe_sync.h
+++ b/drivers/gpu/drm/xe/xe_sync.h
@@ -15,7 +15,7 @@ struct xe_sched_job;
int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef,
struct xe_sync_entry *sync,
struct drm_xe_sync __user *sync_user,
- bool exec, bool compute_mode);
+ bool exec, bool compute_mode, bool exec_nop);
int xe_sync_entry_wait(struct xe_sync_entry *sync);
int xe_sync_entry_add_deps(struct xe_sync_entry *sync,
struct xe_sched_job *job);
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 403444ff3856..2f212939d2b5 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -3088,7 +3088,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) {
err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs],
&syncs_user[num_syncs], false,
- xe_vm_no_dma_fences(vm));
+ xe_vm_no_dma_fences(vm), false);
if (err)
goto free_syncs;
}
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* Re: [Intel-xe] [PATCH v2 02/27] drm/xe: Allow num_batch_buffer == 0 in exec IOCTL
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 02/27] drm/xe: Allow num_batch_buffer == 0 in exec IOCTL Matthew Brost
@ 2023-11-10 11:11 ` Thomas Hellström
2023-11-10 9:03 ` Matthew Brost
0 siblings, 1 reply; 43+ messages in thread
From: Thomas Hellström @ 2023-11-10 11:11 UTC (permalink / raw)
To: Matthew Brost, intel-xe
On Mon, 2023-11-06 at 21:25 -0800, Matthew Brost wrote:
> The idea being out-syncs can signal indicating all previous
> operations
> on the exec queue are complete. An example use case of this would be
> support for implementing vkQueueWaitForIdle easily.
>
> v2: Don't add last_fence for VM's that do not support dma fences
>
> Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> ---
> drivers/gpu/drm/xe/xe_exec.c | 22 +++++++++++++++++++---
> drivers/gpu/drm/xe/xe_exec_queue.c | 5 ++++-
> drivers/gpu/drm/xe/xe_exec_queue_types.h | 5 +++--
> drivers/gpu/drm/xe/xe_sync.c | 5 ++++-
> drivers/gpu/drm/xe/xe_sync.h | 2 +-
> drivers/gpu/drm/xe/xe_vm.c | 2 +-
> 6 files changed, 32 insertions(+), 9 deletions(-)
>
> diff --git a/drivers/gpu/drm/xe/xe_exec.c
> b/drivers/gpu/drm/xe/xe_exec.c
> index 28e84a0bbeb0..4666f5b145f7 100644
> --- a/drivers/gpu/drm/xe/xe_exec.c
> +++ b/drivers/gpu/drm/xe/xe_exec.c
> @@ -161,7 +161,8 @@ int xe_exec_ioctl(struct drm_device *dev, void
> *data, struct drm_file *file)
> if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM))
> return -EINVAL;
>
> - if (XE_IOCTL_DBG(xe, q->width != args->num_batch_buffer))
> + if (XE_IOCTL_DBG(xe, args->num_batch_buffer &&
> + q->width != args->num_batch_buffer))
> return -EINVAL;
>
> if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_BANNED)) {
> @@ -182,12 +183,13 @@ int xe_exec_ioctl(struct drm_device *dev, void
> *data, struct drm_file *file)
> for (i = 0; i < args->num_syncs; i++) {
> err = xe_sync_entry_parse(xe, xef,
> &syncs[num_syncs++],
> &syncs_user[i], true,
> - xe_vm_no_dma_fences(vm));
> + xe_vm_no_dma_fences(vm),
> + !args->num_batch_buffer);
> if (err)
> goto err_syncs;
> }
>
> - if (xe_exec_queue_is_parallel(q)) {
> + if (args->num_batch_buffer && xe_exec_queue_is_parallel(q)) {
> err = __copy_from_user(addresses, addresses_user,
> sizeof(u64) *
> q->width);
> if (err) {
> @@ -234,6 +236,18 @@ int xe_exec_ioctl(struct drm_device *dev, void
> *data, struct drm_file *file)
> goto err_exec;
> }
>
> + if (!args->num_batch_buffer) {
> + if (!xe_vm_no_dma_fences(vm)) {
> + struct dma_fence *fence =
> + xe_exec_queue_last_fence_get(q, vm);
> +
> + for (i = 0; i < num_syncs; i++)
> + xe_sync_entry_signal(&syncs[i], NULL,
> fence);
> + }
> +
> + goto err_exec;
> + }
> +
> if (xe_exec_queue_is_lr(q) && xe_exec_queue_ring_full(q)) {
> err = -EWOULDBLOCK;
> goto err_exec;
> @@ -327,6 +341,8 @@ int xe_exec_ioctl(struct drm_device *dev, void
> *data, struct drm_file *file)
>
> if (xe_exec_queue_is_lr(q))
> q->ring_ops->emit_job(job);
> + if (!xe_vm_no_dma_fences(vm))
> + xe_exec_queue_last_fence_set(q, vm, &job-
> >drm.s_fence->finished);
> xe_sched_job_push(job);
> xe_vm_reactivate_rebind(vm);
>
> diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c
> b/drivers/gpu/drm/xe/xe_exec_queue.c
> index 4fd44a9203e4..35710b66e5de 100644
> --- a/drivers/gpu/drm/xe/xe_exec_queue.c
> +++ b/drivers/gpu/drm/xe/xe_exec_queue.c
> @@ -924,7 +924,10 @@ int xe_exec_queue_set_property_ioctl(struct
> drm_device *dev, void *data,
> static void xe_exec_queue_last_fence_lockdep_assert(struct
> xe_exec_queue *q,
> struct xe_vm *vm)
> {
> - lockdep_assert_held_write(&vm->lock);
> + if (q->flags & EXEC_QUEUE_FLAG_VM)
> + lockdep_assert_held_write(&vm->lock);
> + else
> + xe_vm_assert_held(vm);
> }
>
> /**
> diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h
> b/drivers/gpu/drm/xe/xe_exec_queue_types.h
> index ecd761177567..35ffe7c55f25 100644
> --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h
> +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h
> @@ -53,8 +53,9 @@ struct xe_exec_queue {
> struct xe_hw_fence_irq *fence_irq;
>
> /**
> - * @last_fence: last fence on engine, protected by vm->lock
> in write
> - * mode if bind engine
> + * @last_fence: last fence on exec queue, protected by vm-
> >lock in write
> + * mode if bind exec queue, protected by dma resv lock if
> non-bind exec
> + * queue
> */
> struct dma_fence *last_fence;
>
> diff --git a/drivers/gpu/drm/xe/xe_sync.c
> b/drivers/gpu/drm/xe/xe_sync.c
> index 73ef259aa387..2461e7d4814c 100644
> --- a/drivers/gpu/drm/xe/xe_sync.c
> +++ b/drivers/gpu/drm/xe/xe_sync.c
> @@ -100,7 +100,7 @@ static void user_fence_cb(struct dma_fence
> *fence, struct dma_fence_cb *cb)
> int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef,
> struct xe_sync_entry *sync,
> struct drm_xe_sync __user *sync_user,
> - bool exec, bool no_dma_fences)
> + bool exec, bool no_dma_fences, bool exec_nop)
Here we have a number of bools indicating the context of a potential
caller. That makes the code hard to read. When reading the caller code
you'd both need to remember what each argument position means and
exactly what xe_sync_entry_parse does for a certain caller, for example
exec_nop.
One would have a set of flags instead
exec_nop becomes FLAG_DISALLOW_USER_FENCE
Otherwise LGTM.
^ permalink raw reply [flat|nested] 43+ messages in thread* Re: [Intel-xe] [PATCH v2 02/27] drm/xe: Allow num_batch_buffer == 0 in exec IOCTL
2023-11-10 11:11 ` Thomas Hellström
@ 2023-11-10 9:03 ` Matthew Brost
0 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-10 9:03 UTC (permalink / raw)
To: Thomas Hellström; +Cc: intel-xe
On Fri, Nov 10, 2023 at 12:11:19PM +0100, Thomas Hellström wrote:
> On Mon, 2023-11-06 at 21:25 -0800, Matthew Brost wrote:
> > The idea being out-syncs can signal indicating all previous
> > operations
> > on the exec queue are complete. An example use case of this would be
> > support for implementing vkQueueWaitForIdle easily.
> >
> > v2: Don't add last_fence for VM's that do not support dma fences
> >
> > Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> > ---
> > drivers/gpu/drm/xe/xe_exec.c | 22 +++++++++++++++++++---
> > drivers/gpu/drm/xe/xe_exec_queue.c | 5 ++++-
> > drivers/gpu/drm/xe/xe_exec_queue_types.h | 5 +++--
> > drivers/gpu/drm/xe/xe_sync.c | 5 ++++-
> > drivers/gpu/drm/xe/xe_sync.h | 2 +-
> > drivers/gpu/drm/xe/xe_vm.c | 2 +-
> > 6 files changed, 32 insertions(+), 9 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/xe/xe_exec.c
> > b/drivers/gpu/drm/xe/xe_exec.c
> > index 28e84a0bbeb0..4666f5b145f7 100644
> > --- a/drivers/gpu/drm/xe/xe_exec.c
> > +++ b/drivers/gpu/drm/xe/xe_exec.c
> > @@ -161,7 +161,8 @@ int xe_exec_ioctl(struct drm_device *dev, void
> > *data, struct drm_file *file)
> > if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM))
> > return -EINVAL;
> >
> > - if (XE_IOCTL_DBG(xe, q->width != args->num_batch_buffer))
> > + if (XE_IOCTL_DBG(xe, args->num_batch_buffer &&
> > + q->width != args->num_batch_buffer))
> > return -EINVAL;
> >
> > if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_BANNED)) {
> > @@ -182,12 +183,13 @@ int xe_exec_ioctl(struct drm_device *dev, void
> > *data, struct drm_file *file)
> > for (i = 0; i < args->num_syncs; i++) {
> > err = xe_sync_entry_parse(xe, xef,
> > &syncs[num_syncs++],
> > &syncs_user[i], true,
> > - xe_vm_no_dma_fences(vm));
> > + xe_vm_no_dma_fences(vm),
> > + !args->num_batch_buffer);
> > if (err)
> > goto err_syncs;
> > }
> >
> > - if (xe_exec_queue_is_parallel(q)) {
> > + if (args->num_batch_buffer && xe_exec_queue_is_parallel(q)) {
> > err = __copy_from_user(addresses, addresses_user,
> > sizeof(u64) *
> > q->width);
> > if (err) {
> > @@ -234,6 +236,18 @@ int xe_exec_ioctl(struct drm_device *dev, void
> > *data, struct drm_file *file)
> > goto err_exec;
> > }
> >
> > + if (!args->num_batch_buffer) {
> > + if (!xe_vm_no_dma_fences(vm)) {
> > + struct dma_fence *fence =
> > + xe_exec_queue_last_fence_get(q, vm);
> > +
> > + for (i = 0; i < num_syncs; i++)
> > + xe_sync_entry_signal(&syncs[i], NULL,
> > fence);
> > + }
> > +
> > + goto err_exec;
> > + }
> > +
> > if (xe_exec_queue_is_lr(q) && xe_exec_queue_ring_full(q)) {
> > err = -EWOULDBLOCK;
> > goto err_exec;
> > @@ -327,6 +341,8 @@ int xe_exec_ioctl(struct drm_device *dev, void
> > *data, struct drm_file *file)
> >
> > if (xe_exec_queue_is_lr(q))
> > q->ring_ops->emit_job(job);
> > + if (!xe_vm_no_dma_fences(vm))
> > + xe_exec_queue_last_fence_set(q, vm, &job-
> > >drm.s_fence->finished);
> > xe_sched_job_push(job);
> > xe_vm_reactivate_rebind(vm);
> >
> > diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c
> > b/drivers/gpu/drm/xe/xe_exec_queue.c
> > index 4fd44a9203e4..35710b66e5de 100644
> > --- a/drivers/gpu/drm/xe/xe_exec_queue.c
> > +++ b/drivers/gpu/drm/xe/xe_exec_queue.c
> > @@ -924,7 +924,10 @@ int xe_exec_queue_set_property_ioctl(struct
> > drm_device *dev, void *data,
> > static void xe_exec_queue_last_fence_lockdep_assert(struct
> > xe_exec_queue *q,
> > struct xe_vm *vm)
> > {
> > - lockdep_assert_held_write(&vm->lock);
> > + if (q->flags & EXEC_QUEUE_FLAG_VM)
> > + lockdep_assert_held_write(&vm->lock);
> > + else
> > + xe_vm_assert_held(vm);
> > }
> >
> > /**
> > diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h
> > b/drivers/gpu/drm/xe/xe_exec_queue_types.h
> > index ecd761177567..35ffe7c55f25 100644
> > --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h
> > +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h
> > @@ -53,8 +53,9 @@ struct xe_exec_queue {
> > struct xe_hw_fence_irq *fence_irq;
> >
> > /**
> > - * @last_fence: last fence on engine, protected by vm->lock
> > in write
> > - * mode if bind engine
> > + * @last_fence: last fence on exec queue, protected by vm-
> > >lock in write
> > + * mode if bind exec queue, protected by dma resv lock if
> > non-bind exec
> > + * queue
> > */
> > struct dma_fence *last_fence;
> >
> > diff --git a/drivers/gpu/drm/xe/xe_sync.c
> > b/drivers/gpu/drm/xe/xe_sync.c
> > index 73ef259aa387..2461e7d4814c 100644
> > --- a/drivers/gpu/drm/xe/xe_sync.c
> > +++ b/drivers/gpu/drm/xe/xe_sync.c
> > @@ -100,7 +100,7 @@ static void user_fence_cb(struct dma_fence
> > *fence, struct dma_fence_cb *cb)
> > int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef,
> > struct xe_sync_entry *sync,
> > struct drm_xe_sync __user *sync_user,
> > - bool exec, bool no_dma_fences)
> > + bool exec, bool no_dma_fences, bool exec_nop)
>
> Here we have a number of bools indicating the context of a potential
> caller. That makes the code hard to read. When reading the caller code
> you'd both need to remember what each argument position means and
> exactly what xe_sync_entry_parse does for a certain caller, for example
> exec_nop.
> One would have a set of flags instead
> exec_nop becomes FLAG_DISALLOW_USER_FENCE
>
Agree, let me clean this up.
Matt
> Otherwise LGTM.
>
^ permalink raw reply [flat|nested] 43+ messages in thread
* [Intel-xe] [PATCH v2 03/27] drm/xe: Take in-syncs into account when num_execs or num_binds == 0
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 01/27] drm/xe: Allow num_binds == 0 in VM bind IOCTL Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 02/27] drm/xe: Allow num_batch_buffer == 0 in exec IOCTL Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-13 12:50 ` Thomas Hellström
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 04/27] drm/xe: Lock all gpuva ops during VM bind IOCTL Matthew Brost
` (27 subsequent siblings)
30 siblings, 1 reply; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
Wait on in-syncs before signaling out-syncs if num_execs or num_binds ==
0 in execbuf IOCTL or VM bind IOCTL respectfully.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_exec.c | 10 ++++-
drivers/gpu/drm/xe/xe_sync.c | 75 ++++++++++++++++++++++++++++++++++++
drivers/gpu/drm/xe/xe_sync.h | 5 +++
drivers/gpu/drm/xe/xe_vm.c | 24 ++++++++++--
4 files changed, 108 insertions(+), 6 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c
index 4666f5b145f7..80ee6d8fcf68 100644
--- a/drivers/gpu/drm/xe/xe_exec.c
+++ b/drivers/gpu/drm/xe/xe_exec.c
@@ -238,11 +238,17 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
if (!args->num_batch_buffer) {
if (!xe_vm_no_dma_fences(vm)) {
- struct dma_fence *fence =
- xe_exec_queue_last_fence_get(q, vm);
+ struct dma_fence *fence;
+ fence = xe_sync_in_fence_get(syncs, num_syncs, q, vm);
+ if (IS_ERR(fence)) {
+ err = PTR_ERR(fence);
+ goto err_exec;
+ }
for (i = 0; i < num_syncs; i++)
xe_sync_entry_signal(&syncs[i], NULL, fence);
+ xe_exec_queue_last_fence_set(q, vm, fence);
+ dma_fence_put(fence);
}
goto err_exec;
diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c
index 2461e7d4814c..6b38c74a1de1 100644
--- a/drivers/gpu/drm/xe/xe_sync.c
+++ b/drivers/gpu/drm/xe/xe_sync.c
@@ -5,6 +5,7 @@
#include "xe_sync.h"
+#include <linux/dma-fence-array.h>
#include <linux/kthread.h>
#include <linux/sched/mm.h>
#include <linux/uaccess.h>
@@ -14,6 +15,7 @@
#include <drm/xe_drm.h>
#include "xe_device_types.h"
+#include "xe_exec_queue.h"
#include "xe_macros.h"
#include "xe_sched_job_types.h"
@@ -274,3 +276,76 @@ void xe_sync_entry_cleanup(struct xe_sync_entry *sync)
if (sync->ufence)
user_fence_put(sync->ufence);
}
+
+/**
+ * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM
+ * @sync: input syncs
+ * @num_sync: number of syncs
+ * @q: exec queue
+ * @vm: VM
+ *
+ * Get a fence from syncs, exec queue, and VM. If syncs contain more than 1
+ * in-fence create and return a composite fence of all in-fences, if syncs
+ * contain 1 in-fence return in-fence, if no in-fences return last fence on
+ * input exec queue. Caller must drop reference to returned fence.
+ *
+ * Return: fence on success, ERR_PTR(-ENOMEM) on failure
+ */
+struct dma_fence *
+xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
+ struct xe_exec_queue *q, struct xe_vm *vm)
+{
+ struct dma_fence **fences = NULL;
+ struct dma_fence_array *cf = NULL;
+ struct dma_fence *fence;
+ int i, num_in_fence = 0, current_fence = 0;
+
+ lockdep_assert_held(&vm->lock);
+
+ /* Count in-fences */
+ for (i = 0; i < num_sync; ++i) {
+ if (sync[i].fence) {
+ ++num_in_fence;
+ fence = sync[i].fence;
+ }
+ }
+
+ /* Easy cases... */
+ if (!num_in_fence) {
+ fence = xe_exec_queue_last_fence_get(q, vm);
+ dma_fence_get(fence);
+ return fence;
+ } else if (num_in_fence == 1) {
+ dma_fence_get(fence);
+ return fence;
+ }
+
+ /* Create composite fence */
+ fences = kmalloc_array(num_in_fence, sizeof(*fences), GFP_KERNEL);
+ if (!fences)
+ return ERR_PTR(-ENOMEM);
+ for (i = 0; i < num_sync; ++i) {
+ if (sync[i].fence) {
+ dma_fence_get(sync[i].fence);
+ fences[current_fence++] = sync[i].fence;
+ }
+ }
+ cf = dma_fence_array_create(num_in_fence, fences,
+ vm->composite_fence_ctx,
+ vm->composite_fence_seqno++,
+ false);
+ if (!cf) {
+ --vm->composite_fence_seqno;
+ goto err_out;
+ }
+
+ return &cf->base;
+
+err_out:
+ while (current_fence)
+ dma_fence_put(fences[--current_fence]);
+ kfree(fences);
+ kfree(cf);
+
+ return ERR_PTR(-ENOMEM);
+}
diff --git a/drivers/gpu/drm/xe/xe_sync.h b/drivers/gpu/drm/xe/xe_sync.h
index 98f02bb34637..c0c8ddac805d 100644
--- a/drivers/gpu/drm/xe/xe_sync.h
+++ b/drivers/gpu/drm/xe/xe_sync.h
@@ -9,8 +9,10 @@
#include "xe_sync_types.h"
struct xe_device;
+struct xe_exec_queue;
struct xe_file;
struct xe_sched_job;
+struct xe_vm;
int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef,
struct xe_sync_entry *sync,
@@ -23,5 +25,8 @@ void xe_sync_entry_signal(struct xe_sync_entry *sync,
struct xe_sched_job *job,
struct dma_fence *fence);
void xe_sync_entry_cleanup(struct xe_sync_entry *sync);
+struct dma_fence *
+xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
+ struct xe_exec_queue *q, struct xe_vm *vm);
#endif
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 2f212939d2b5..2a7fa8e2058e 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -3155,12 +3155,28 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
unwind_ops:
vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds);
free_syncs:
- for (i = 0; err == -ENODATA && i < num_syncs; i++) {
- struct dma_fence *fence =
- xe_exec_queue_last_fence_get(to_wait_exec_queue(vm, q), vm);
+ if (err == -ENODATA) {
+ struct dma_fence *fence;
- xe_sync_entry_signal(&syncs[i], NULL, fence);
+ fence = xe_sync_in_fence_get(syncs, num_syncs,
+ to_wait_exec_queue(vm, q), vm);
+ if (IS_ERR(fence)) {
+ err = PTR_ERR(fence);
+ goto cleanup_syncs;
+ }
+ for (i = 0; i < num_syncs; i++)
+ xe_sync_entry_signal(&syncs[i], NULL, fence);
+ if (!async) {
+ long timeout = dma_fence_wait(fence, true);
+
+ if (timeout < 0)
+ err = -EINTR;
+ }
+ xe_exec_queue_last_fence_set(to_wait_exec_queue(vm, q), vm,
+ fence);
+ dma_fence_put(fence);
}
+cleanup_syncs:
while (num_syncs--)
xe_sync_entry_cleanup(&syncs[num_syncs]);
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* Re: [Intel-xe] [PATCH v2 03/27] drm/xe: Take in-syncs into account when num_execs or num_binds == 0
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 03/27] drm/xe: Take in-syncs into account when num_execs or num_binds == 0 Matthew Brost
@ 2023-11-13 12:50 ` Thomas Hellström
2023-11-13 8:28 ` Matthew Brost
0 siblings, 1 reply; 43+ messages in thread
From: Thomas Hellström @ 2023-11-13 12:50 UTC (permalink / raw)
To: Matthew Brost, intel-xe
Hi,
On 11/7/23 06:25, Matthew Brost wrote:
> Wait on in-syncs before signaling out-syncs if num_execs or num_binds ==
> 0 in execbuf IOCTL or VM bind IOCTL respectfully.
>
> Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> ---
> drivers/gpu/drm/xe/xe_exec.c | 10 ++++-
> drivers/gpu/drm/xe/xe_sync.c | 75 ++++++++++++++++++++++++++++++++++++
> drivers/gpu/drm/xe/xe_sync.h | 5 +++
> drivers/gpu/drm/xe/xe_vm.c | 24 ++++++++++--
> 4 files changed, 108 insertions(+), 6 deletions(-)
>
> diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c
> index 4666f5b145f7..80ee6d8fcf68 100644
> --- a/drivers/gpu/drm/xe/xe_exec.c
> +++ b/drivers/gpu/drm/xe/xe_exec.c
> @@ -238,11 +238,17 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
>
> if (!args->num_batch_buffer) {
> if (!xe_vm_no_dma_fences(vm)) {
> - struct dma_fence *fence =
> - xe_exec_queue_last_fence_get(q, vm);
> + struct dma_fence *fence;
>
> + fence = xe_sync_in_fence_get(syncs, num_syncs, q, vm);
> + if (IS_ERR(fence)) {
> + err = PTR_ERR(fence);
> + goto err_exec;
> + }
> for (i = 0; i < num_syncs; i++)
> xe_sync_entry_signal(&syncs[i], NULL, fence);
> + xe_exec_queue_last_fence_set(q, vm, fence);
> + dma_fence_put(fence);
> }
>
> goto err_exec;
> diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c
> index 2461e7d4814c..6b38c74a1de1 100644
> --- a/drivers/gpu/drm/xe/xe_sync.c
> +++ b/drivers/gpu/drm/xe/xe_sync.c
> @@ -5,6 +5,7 @@
>
> #include "xe_sync.h"
>
> +#include <linux/dma-fence-array.h>
> #include <linux/kthread.h>
> #include <linux/sched/mm.h>
> #include <linux/uaccess.h>
> @@ -14,6 +15,7 @@
> #include <drm/xe_drm.h>
>
> #include "xe_device_types.h"
> +#include "xe_exec_queue.h"
> #include "xe_macros.h"
> #include "xe_sched_job_types.h"
>
> @@ -274,3 +276,76 @@ void xe_sync_entry_cleanup(struct xe_sync_entry *sync)
> if (sync->ufence)
> user_fence_put(sync->ufence);
> }
> +
> +/**
> + * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM
> + * @sync: input syncs
> + * @num_sync: number of syncs
> + * @q: exec queue
> + * @vm: VM
> + *
> + * Get a fence from syncs, exec queue, and VM. If syncs contain more than 1
> + * in-fence create and return a composite fence of all in-fences, if syncs
> + * contain 1 in-fence return in-fence, if no in-fences return last fence on
> + * input exec queue. Caller must drop reference to returned fence.
> + *
> + * Return: fence on success, ERR_PTR(-ENOMEM) on failure
> + */
> +struct dma_fence *
> +xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
> + struct xe_exec_queue *q, struct xe_vm *vm)
> +{
> + struct dma_fence **fences = NULL;
> + struct dma_fence_array *cf = NULL;
> + struct dma_fence *fence;
> + int i, num_in_fence = 0, current_fence = 0;
> +
> + lockdep_assert_held(&vm->lock);
> +
> + /* Count in-fences */
> + for (i = 0; i < num_sync; ++i) {
> + if (sync[i].fence) {
> + ++num_in_fence;
> + fence = sync[i].fence;
> + }
> + }
> +
> + /* Easy cases... */
> + if (!num_in_fence) {
> + fence = xe_exec_queue_last_fence_get(q, vm);
> + dma_fence_get(fence);
> + return fence;
> + } else if (num_in_fence == 1) {
Don't we need to also wait on the exec_queue last fence in this case,
and the multiple-in-fences below?
Otherwise we only wait for the in-fences but not on currently executing
jobs?
Did you investigate just to forward a non-existing batchbuffer in the
exec case and create a NOP binding job in the bind case?
/Thomas
> + dma_fence_get(fence);
> + return fence;
> + }
> +
> + /* Create composite fence */
> + fences = kmalloc_array(num_in_fence, sizeof(*fences), GFP_KERNEL);
> + if (!fences)
> + return ERR_PTR(-ENOMEM);
> + for (i = 0; i < num_sync; ++i) {
> + if (sync[i].fence) {
> + dma_fence_get(sync[i].fence);
> + fences[current_fence++] = sync[i].fence;
> + }
> + }
> + cf = dma_fence_array_create(num_in_fence, fences,
> + vm->composite_fence_ctx,
> + vm->composite_fence_seqno++,
> + false);
> + if (!cf) {
> + --vm->composite_fence_seqno;
> + goto err_out;
> + }
> +
> + return &cf->base;
> +
> +err_out:
> + while (current_fence)
> + dma_fence_put(fences[--current_fence]);
> + kfree(fences);
> + kfree(cf);
> +
> + return ERR_PTR(-ENOMEM);
> +}
> diff --git a/drivers/gpu/drm/xe/xe_sync.h b/drivers/gpu/drm/xe/xe_sync.h
> index 98f02bb34637..c0c8ddac805d 100644
> --- a/drivers/gpu/drm/xe/xe_sync.h
> +++ b/drivers/gpu/drm/xe/xe_sync.h
> @@ -9,8 +9,10 @@
> #include "xe_sync_types.h"
>
> struct xe_device;
> +struct xe_exec_queue;
> struct xe_file;
> struct xe_sched_job;
> +struct xe_vm;
>
> int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef,
> struct xe_sync_entry *sync,
> @@ -23,5 +25,8 @@ void xe_sync_entry_signal(struct xe_sync_entry *sync,
> struct xe_sched_job *job,
> struct dma_fence *fence);
> void xe_sync_entry_cleanup(struct xe_sync_entry *sync);
> +struct dma_fence *
> +xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
> + struct xe_exec_queue *q, struct xe_vm *vm);
>
> #endif
> diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
> index 2f212939d2b5..2a7fa8e2058e 100644
> --- a/drivers/gpu/drm/xe/xe_vm.c
> +++ b/drivers/gpu/drm/xe/xe_vm.c
> @@ -3155,12 +3155,28 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> unwind_ops:
> vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds);
> free_syncs:
> - for (i = 0; err == -ENODATA && i < num_syncs; i++) {
> - struct dma_fence *fence =
> - xe_exec_queue_last_fence_get(to_wait_exec_queue(vm, q), vm);
> + if (err == -ENODATA) {
> + struct dma_fence *fence;
>
> - xe_sync_entry_signal(&syncs[i], NULL, fence);
> + fence = xe_sync_in_fence_get(syncs, num_syncs,
> + to_wait_exec_queue(vm, q), vm);
> + if (IS_ERR(fence)) {
> + err = PTR_ERR(fence);
> + goto cleanup_syncs;
> + }
> + for (i = 0; i < num_syncs; i++)
> + xe_sync_entry_signal(&syncs[i], NULL, fence);
> + if (!async) {
> + long timeout = dma_fence_wait(fence, true);
> +
> + if (timeout < 0)
> + err = -EINTR;
> + }
> + xe_exec_queue_last_fence_set(to_wait_exec_queue(vm, q), vm,
> + fence);
> + dma_fence_put(fence);
> }
> +cleanup_syncs:
> while (num_syncs--)
> xe_sync_entry_cleanup(&syncs[num_syncs]);
>
^ permalink raw reply [flat|nested] 43+ messages in thread* Re: [Intel-xe] [PATCH v2 03/27] drm/xe: Take in-syncs into account when num_execs or num_binds == 0
2023-11-13 12:50 ` Thomas Hellström
@ 2023-11-13 8:28 ` Matthew Brost
0 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-13 8:28 UTC (permalink / raw)
To: Thomas Hellström; +Cc: intel-xe
On Mon, Nov 13, 2023 at 01:50:41PM +0100, Thomas Hellström wrote:
> Hi,
>
> On 11/7/23 06:25, Matthew Brost wrote:
> > Wait on in-syncs before signaling out-syncs if num_execs or num_binds ==
> > 0 in execbuf IOCTL or VM bind IOCTL respectfully.
> >
> > Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> > ---
> > drivers/gpu/drm/xe/xe_exec.c | 10 ++++-
> > drivers/gpu/drm/xe/xe_sync.c | 75 ++++++++++++++++++++++++++++++++++++
> > drivers/gpu/drm/xe/xe_sync.h | 5 +++
> > drivers/gpu/drm/xe/xe_vm.c | 24 ++++++++++--
> > 4 files changed, 108 insertions(+), 6 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c
> > index 4666f5b145f7..80ee6d8fcf68 100644
> > --- a/drivers/gpu/drm/xe/xe_exec.c
> > +++ b/drivers/gpu/drm/xe/xe_exec.c
> > @@ -238,11 +238,17 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> > if (!args->num_batch_buffer) {
> > if (!xe_vm_no_dma_fences(vm)) {
> > - struct dma_fence *fence =
> > - xe_exec_queue_last_fence_get(q, vm);
> > + struct dma_fence *fence;
> > + fence = xe_sync_in_fence_get(syncs, num_syncs, q, vm);
> > + if (IS_ERR(fence)) {
> > + err = PTR_ERR(fence);
> > + goto err_exec;
> > + }
> > for (i = 0; i < num_syncs; i++)
> > xe_sync_entry_signal(&syncs[i], NULL, fence);
> > + xe_exec_queue_last_fence_set(q, vm, fence);
> > + dma_fence_put(fence);
> > }
> > goto err_exec;
> > diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c
> > index 2461e7d4814c..6b38c74a1de1 100644
> > --- a/drivers/gpu/drm/xe/xe_sync.c
> > +++ b/drivers/gpu/drm/xe/xe_sync.c
> > @@ -5,6 +5,7 @@
> > #include "xe_sync.h"
> > +#include <linux/dma-fence-array.h>
> > #include <linux/kthread.h>
> > #include <linux/sched/mm.h>
> > #include <linux/uaccess.h>
> > @@ -14,6 +15,7 @@
> > #include <drm/xe_drm.h>
> > #include "xe_device_types.h"
> > +#include "xe_exec_queue.h"
> > #include "xe_macros.h"
> > #include "xe_sched_job_types.h"
> > @@ -274,3 +276,76 @@ void xe_sync_entry_cleanup(struct xe_sync_entry *sync)
> > if (sync->ufence)
> > user_fence_put(sync->ufence);
> > }
> > +
> > +/**
> > + * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM
> > + * @sync: input syncs
> > + * @num_sync: number of syncs
> > + * @q: exec queue
> > + * @vm: VM
> > + *
> > + * Get a fence from syncs, exec queue, and VM. If syncs contain more than 1
> > + * in-fence create and return a composite fence of all in-fences, if syncs
> > + * contain 1 in-fence return in-fence, if no in-fences return last fence on
> > + * input exec queue. Caller must drop reference to returned fence.
> > + *
> > + * Return: fence on success, ERR_PTR(-ENOMEM) on failure
> > + */
> > +struct dma_fence *
> > +xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
> > + struct xe_exec_queue *q, struct xe_vm *vm)
> > +{
> > + struct dma_fence **fences = NULL;
> > + struct dma_fence_array *cf = NULL;
> > + struct dma_fence *fence;
> > + int i, num_in_fence = 0, current_fence = 0;
> > +
> > + lockdep_assert_held(&vm->lock);
> > +
> > + /* Count in-fences */
> > + for (i = 0; i < num_sync; ++i) {
> > + if (sync[i].fence) {
> > + ++num_in_fence;
> > + fence = sync[i].fence;
> > + }
> > + }
> > +
> > + /* Easy cases... */
> > + if (!num_in_fence) {
> > + fence = xe_exec_queue_last_fence_get(q, vm);
> > + dma_fence_get(fence);
> > + return fence;
> > + } else if (num_in_fence == 1) {
>
>
> Don't we need to also wait on the exec_queue last fence in this case, and
> the multiple-in-fences below?
> Otherwise we only wait for the in-fences but not on currently executing
> jobs?
>
That is right. I think if num_in_fence > 0 we need to wait on the last fence too.
> Did you investigate just to forward a non-existing batchbuffer in the exec
> case and create a NOP binding job in the bind case?
>
Hmm, let me look at that as an option.
Matt
> /Thomas
>
>
>
> > + dma_fence_get(fence);
> > + return fence;
> > + }
> > +
> > + /* Create composite fence */
> > + fences = kmalloc_array(num_in_fence, sizeof(*fences), GFP_KERNEL);
> > + if (!fences)
> > + return ERR_PTR(-ENOMEM);
> > + for (i = 0; i < num_sync; ++i) {
> > + if (sync[i].fence) {
> > + dma_fence_get(sync[i].fence);
> > + fences[current_fence++] = sync[i].fence;
> > + }
> > + }
> > + cf = dma_fence_array_create(num_in_fence, fences,
> > + vm->composite_fence_ctx,
> > + vm->composite_fence_seqno++,
> > + false);
> > + if (!cf) {
> > + --vm->composite_fence_seqno;
> > + goto err_out;
> > + }
> > +
> > + return &cf->base;
> > +
> > +err_out:
> > + while (current_fence)
> > + dma_fence_put(fences[--current_fence]);
> > + kfree(fences);
> > + kfree(cf);
> > +
> > + return ERR_PTR(-ENOMEM);
> > +}
> > diff --git a/drivers/gpu/drm/xe/xe_sync.h b/drivers/gpu/drm/xe/xe_sync.h
> > index 98f02bb34637..c0c8ddac805d 100644
> > --- a/drivers/gpu/drm/xe/xe_sync.h
> > +++ b/drivers/gpu/drm/xe/xe_sync.h
> > @@ -9,8 +9,10 @@
> > #include "xe_sync_types.h"
> > struct xe_device;
> > +struct xe_exec_queue;
> > struct xe_file;
> > struct xe_sched_job;
> > +struct xe_vm;
> > int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef,
> > struct xe_sync_entry *sync,
> > @@ -23,5 +25,8 @@ void xe_sync_entry_signal(struct xe_sync_entry *sync,
> > struct xe_sched_job *job,
> > struct dma_fence *fence);
> > void xe_sync_entry_cleanup(struct xe_sync_entry *sync);
> > +struct dma_fence *
> > +xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
> > + struct xe_exec_queue *q, struct xe_vm *vm);
> > #endif
> > diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
> > index 2f212939d2b5..2a7fa8e2058e 100644
> > --- a/drivers/gpu/drm/xe/xe_vm.c
> > +++ b/drivers/gpu/drm/xe/xe_vm.c
> > @@ -3155,12 +3155,28 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> > unwind_ops:
> > vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds);
> > free_syncs:
> > - for (i = 0; err == -ENODATA && i < num_syncs; i++) {
> > - struct dma_fence *fence =
> > - xe_exec_queue_last_fence_get(to_wait_exec_queue(vm, q), vm);
> > + if (err == -ENODATA) {
> > + struct dma_fence *fence;
> > - xe_sync_entry_signal(&syncs[i], NULL, fence);
> > + fence = xe_sync_in_fence_get(syncs, num_syncs,
> > + to_wait_exec_queue(vm, q), vm);
> > + if (IS_ERR(fence)) {
> > + err = PTR_ERR(fence);
> > + goto cleanup_syncs;
> > + }
> > + for (i = 0; i < num_syncs; i++)
> > + xe_sync_entry_signal(&syncs[i], NULL, fence);
> > + if (!async) {
> > + long timeout = dma_fence_wait(fence, true);
> > +
> > + if (timeout < 0)
> > + err = -EINTR;
> > + }
> > + xe_exec_queue_last_fence_set(to_wait_exec_queue(vm, q), vm,
> > + fence);
> > + dma_fence_put(fence);
> > }
> > +cleanup_syncs:
> > while (num_syncs--)
> > xe_sync_entry_cleanup(&syncs[num_syncs]);
^ permalink raw reply [flat|nested] 43+ messages in thread
* [Intel-xe] [PATCH v2 04/27] drm/xe: Lock all gpuva ops during VM bind IOCTL
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (2 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 03/27] drm/xe: Take in-syncs into account when num_execs or num_binds == 0 Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 05/27] drm/xe: Add ops_execute function which returns a fence Matthew Brost
` (26 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
Lock all gpuva ops and validate all BOs in a single step durin the VM
bind IOCTL. This help with the transition to making all gpuva ops in a
VM bind IOCTL a single atomic job.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_vm.c | 126 +++++++++++++++++++++++++++----------
1 file changed, 94 insertions(+), 32 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 2a7fa8e2058e..f5c7540767a4 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -474,19 +474,23 @@ int xe_vm_lock_dma_resv(struct xe_vm *vm, struct drm_exec *exec,
#define XE_VM_REBIND_RETRY_TIMEOUT_MS 1000
-static void xe_vm_kill(struct xe_vm *vm)
+static void xe_vm_kill(struct xe_vm *vm, bool unlocked)
{
struct xe_exec_queue *q;
lockdep_assert_held(&vm->lock);
- xe_vm_lock(vm, false);
+ if (unlocked)
+ xe_vm_lock(vm, false);
+
vm->flags |= XE_VM_FLAG_BANNED;
trace_xe_vm_kill(vm);
list_for_each_entry(q, &vm->preempt.exec_queues, compute.link)
q->ops->kill(q);
- xe_vm_unlock(vm);
+
+ if (unlocked)
+ xe_vm_unlock(vm);
/* TODO: Inform user the VM is banned */
}
@@ -682,7 +686,7 @@ static void preempt_rebind_work_func(struct work_struct *w)
if (err) {
drm_warn(&vm->xe->drm, "VM worker error: %d\n", err);
- xe_vm_kill(vm);
+ xe_vm_kill(vm, true);
}
up_write(&vm->lock);
@@ -1879,17 +1883,9 @@ static int xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue
u32 num_syncs, bool immediate, bool first_op,
bool last_op)
{
- int err;
-
xe_vm_assert_held(vm);
xe_bo_assert_held(bo);
- if (bo && immediate) {
- err = xe_bo_validate(bo, vm, true);
- if (err)
- return err;
- }
-
return __xe_vm_bind(vm, vma, q, syncs, num_syncs, immediate, first_op,
last_op);
}
@@ -2538,17 +2534,12 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
return 0;
}
-static int op_execute(struct drm_exec *exec, struct xe_vm *vm,
- struct xe_vma *vma, struct xe_vma_op *op)
+static int op_execute(struct xe_vm *vm, struct xe_vma *vma,
+ struct xe_vma_op *op)
{
int err;
lockdep_assert_held_write(&vm->lock);
-
- err = xe_vm_prepare_vma(exec, vma, 1);
- if (err)
- return err;
-
xe_vm_assert_held(vm);
xe_bo_assert_held(xe_vma_bo(vma));
@@ -2629,19 +2620,10 @@ static int op_execute(struct drm_exec *exec, struct xe_vm *vm,
static int __xe_vma_op_execute(struct xe_vm *vm, struct xe_vma *vma,
struct xe_vma_op *op)
{
- struct drm_exec exec;
int err;
retry_userptr:
- drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
- drm_exec_until_all_locked(&exec) {
- err = op_execute(&exec, vm, vma, op);
- drm_exec_retry_on_contention(&exec);
- if (err)
- break;
- }
- drm_exec_fini(&exec);
-
+ err = op_execute(vm, vma, op);
if (err == -EAGAIN && xe_vma_is_userptr(vma)) {
lockdep_assert_held_write(&vm->lock);
err = xe_vma_userptr_pin_pages(vma);
@@ -2801,14 +2783,91 @@ static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm,
}
}
+static int vma_lock(struct drm_exec *exec, struct xe_vma *vma, bool validate)
+{
+ struct xe_bo *bo = xe_vma_bo(vma);
+ int err = 0;
+
+ if (bo) {
+ if (!bo->vm)
+ err = drm_exec_prepare_obj(exec, &bo->ttm.base, 1);
+ if (!err && validate)
+ err = xe_bo_validate(bo, xe_vma_vm(vma), true);
+ }
+
+ return err;
+}
+
+static int op_lock(struct drm_exec *exec, struct xe_vm *vm,
+ struct xe_vma_op *op)
+{
+ int err = 0;
+
+ switch (op->base.op) {
+ case DRM_GPUVA_OP_MAP:
+ err = vma_lock(exec, op->map.vma,
+ op->map.immediate || !xe_vm_in_fault_mode(vm));
+ break;
+ case DRM_GPUVA_OP_REMAP:
+ err = vma_lock(exec, gpuva_to_vma(op->base.remap.unmap->va),
+ false);
+ if (!err && op->remap.prev)
+ err = vma_lock(exec, op->remap.prev, true);
+ if (!err && op->remap.next)
+ err = vma_lock(exec, op->remap.next, true);
+ break;
+ case DRM_GPUVA_OP_UNMAP:
+ err = vma_lock(exec, gpuva_to_vma(op->base.unmap.va), false);
+ break;
+ case DRM_GPUVA_OP_PREFETCH:
+ err = vma_lock(exec, gpuva_to_vma(op->base.prefetch.va), true);
+ break;
+ default:
+ drm_warn(&vm->xe->drm, "NOT POSSIBLE");
+ }
+
+ return err;
+}
+
+static int vm_bind_ioctl_ops_lock(struct drm_exec *exec,
+ struct xe_vm *vm,
+ struct list_head *ops_list)
+{
+ struct xe_vma_op *op;
+ int err;
+
+ drm_exec_init(exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
+ drm_exec_until_all_locked(exec) {
+ err = drm_exec_prepare_obj(exec, &xe_vm_ttm_bo(vm)->base, 1);
+ drm_exec_retry_on_contention(exec);
+ if (err)
+ goto out;
+
+ list_for_each_entry(op, ops_list, link) {
+ err = op_lock(exec, vm, op);
+ drm_exec_retry_on_contention(exec);
+ if (err)
+ goto out;
+ }
+ }
+
+out:
+ return err;
+}
+
static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
struct list_head *ops_list)
{
+ struct drm_exec exec;
struct xe_vma_op *op, *next;
int err;
lockdep_assert_held_write(&vm->lock);
+ err = vm_bind_ioctl_ops_lock(&exec, vm, ops_list);
+ if (err)
+ return err;
+
list_for_each_entry_safe(op, next, ops_list, link) {
err = xe_vma_op_execute(vm, op);
if (err) {
@@ -2817,13 +2876,16 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
/*
* FIXME: Killing VM rather than proper error handling
*/
- xe_vm_kill(vm);
- return -ENOSPC;
+ xe_vm_kill(vm, false);
+ err = -ENOSPC;
+ goto unlock;
}
xe_vma_op_cleanup(vm, op);
}
- return 0;
+unlock:
+ drm_exec_fini(&exec);
+ return err;
}
#ifdef TEST_VM_ASYNC_OPS_ERROR
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 05/27] drm/xe: Add ops_execute function which returns a fence
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (3 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 04/27] drm/xe: Lock all gpuva ops during VM bind IOCTL Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 06/27] drm/xe: Move migrate to prefetch to op_lock funtion Matthew Brost
` (25 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
Add ops_execute function which returns a fence. This be helpful to
initiate all binds (VM bind IOCTL, rebinds in exec IOCTL, rebinds in
preempt rebind worker, and rebinds in pagefaults) via a gpuva ops list.
Returning a fence is needed in various paths.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_vm.c | 210 +++++++++++++++++++------------------
1 file changed, 109 insertions(+), 101 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index f5c7540767a4..73202e1909e5 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -1842,21 +1842,22 @@ static bool xe_vm_sync_mode(struct xe_vm *vm, struct xe_exec_queue *q)
!(vm->flags & XE_VM_FLAG_ASYNC_DEFAULT);
}
-static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma,
- struct xe_exec_queue *q, struct xe_sync_entry *syncs,
- u32 num_syncs, bool immediate, bool first_op,
- bool last_op)
+static struct dma_fence *
+xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q,
+ struct xe_bo *bo, struct xe_sync_entry *syncs, u32 num_syncs,
+ bool immediate, bool first_op, bool last_op)
{
struct dma_fence *fence;
struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
xe_vm_assert_held(vm);
+ xe_bo_assert_held(bo);
if (immediate) {
fence = xe_vm_bind_vma(vma, q, syncs, num_syncs, first_op,
last_op);
if (IS_ERR(fence))
- return PTR_ERR(fence);
+ return fence;
} else {
int i;
@@ -1873,26 +1874,14 @@ static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma,
xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence);
if (last_op && xe_vm_sync_mode(vm, q))
dma_fence_wait(fence, true);
- dma_fence_put(fence);
- return 0;
-}
-
-static int xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q,
- struct xe_bo *bo, struct xe_sync_entry *syncs,
- u32 num_syncs, bool immediate, bool first_op,
- bool last_op)
-{
- xe_vm_assert_held(vm);
- xe_bo_assert_held(bo);
-
- return __xe_vm_bind(vm, vma, q, syncs, num_syncs, immediate, first_op,
- last_op);
+ return fence;
}
-static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma,
- struct xe_exec_queue *q, struct xe_sync_entry *syncs,
- u32 num_syncs, bool first_op, bool last_op)
+static struct dma_fence *
+xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma,
+ struct xe_exec_queue *q, struct xe_sync_entry *syncs,
+ u32 num_syncs, bool first_op, bool last_op)
{
struct dma_fence *fence;
struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
@@ -1902,16 +1891,15 @@ static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma,
fence = xe_vm_unbind_vma(vma, q, syncs, num_syncs, first_op, last_op);
if (IS_ERR(fence))
- return PTR_ERR(fence);
+ return fence;
xe_vma_destroy(vma, fence);
if (last_op)
xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence);
if (last_op && xe_vm_sync_mode(vm, q))
dma_fence_wait(fence, true);
- dma_fence_put(fence);
- return 0;
+ return fence;
}
#define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_SCRATCH_PAGE | \
@@ -2051,10 +2039,11 @@ static const u32 region_to_mem_type[] = {
XE_PL_VRAM1,
};
-static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
- struct xe_exec_queue *q, u32 region,
- struct xe_sync_entry *syncs, u32 num_syncs,
- bool first_op, bool last_op)
+static struct dma_fence *
+xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
+ struct xe_exec_queue *q, u32 region,
+ struct xe_sync_entry *syncs, u32 num_syncs,
+ bool first_op, bool last_op)
{
struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
int err;
@@ -2064,26 +2053,24 @@ static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
if (!xe_vma_has_no_bo(vma)) {
err = xe_bo_migrate(xe_vma_bo(vma), region_to_mem_type[region]);
if (err)
- return err;
+ return ERR_PTR(err);
}
if (vma->tile_mask != (vma->tile_present & ~vma->usm.tile_invalidated)) {
return xe_vm_bind(vm, vma, q, xe_vma_bo(vma), syncs, num_syncs,
true, first_op, last_op);
} else {
+ struct dma_fence *fence =
+ xe_exec_queue_last_fence_get(wait_exec_queue, vm);
int i;
/* Nothing to do, signal fences now */
if (last_op) {
- for (i = 0; i < num_syncs; i++) {
- struct dma_fence *fence =
- xe_exec_queue_last_fence_get(wait_exec_queue, vm);
-
+ for (i = 0; i < num_syncs; i++)
xe_sync_entry_signal(&syncs[i], NULL, fence);
- }
}
- return 0;
+ return fence;
}
}
@@ -2534,10 +2521,10 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
return 0;
}
-static int op_execute(struct xe_vm *vm, struct xe_vma *vma,
- struct xe_vma_op *op)
+static struct dma_fence *op_execute(struct xe_vm *vm, struct xe_vma *vma,
+ struct xe_vma_op *op)
{
- int err;
+ struct dma_fence *fence;
lockdep_assert_held_write(&vm->lock);
xe_vm_assert_held(vm);
@@ -2545,11 +2532,12 @@ static int op_execute(struct xe_vm *vm, struct xe_vma *vma,
switch (op->base.op) {
case DRM_GPUVA_OP_MAP:
- err = xe_vm_bind(vm, vma, op->q, xe_vma_bo(vma),
- op->syncs, op->num_syncs,
- op->map.immediate || !xe_vm_in_fault_mode(vm),
- op->flags & XE_VMA_OP_FIRST,
- op->flags & XE_VMA_OP_LAST);
+ fence = xe_vm_bind(vm, vma, op->q, xe_vma_bo(vma),
+ op->syncs, op->num_syncs,
+ op->map.immediate ||
+ !xe_vm_in_fault_mode(vm),
+ op->flags & XE_VMA_OP_FIRST,
+ op->flags & XE_VMA_OP_LAST);
break;
case DRM_GPUVA_OP_REMAP:
{
@@ -2559,37 +2547,37 @@ static int op_execute(struct xe_vm *vm, struct xe_vma *vma,
if (!op->remap.unmap_done) {
if (prev || next)
vma->gpuva.flags |= XE_VMA_FIRST_REBIND;
- err = xe_vm_unbind(vm, vma, op->q, op->syncs,
- op->num_syncs,
- op->flags & XE_VMA_OP_FIRST,
- op->flags & XE_VMA_OP_LAST &&
- !prev && !next);
- if (err)
+ fence = xe_vm_unbind(vm, vma, op->q, op->syncs,
+ op->num_syncs,
+ op->flags & XE_VMA_OP_FIRST,
+ op->flags & XE_VMA_OP_LAST &&
+ !prev && !next);
+ if (IS_ERR(fence))
break;
op->remap.unmap_done = true;
}
if (prev) {
op->remap.prev->gpuva.flags |= XE_VMA_LAST_REBIND;
- err = xe_vm_bind(vm, op->remap.prev, op->q,
- xe_vma_bo(op->remap.prev), op->syncs,
- op->num_syncs, true, false,
- op->flags & XE_VMA_OP_LAST && !next);
+ fence = xe_vm_bind(vm, op->remap.prev, op->q,
+ xe_vma_bo(op->remap.prev), op->syncs,
+ op->num_syncs, true, false,
+ op->flags & XE_VMA_OP_LAST && !next);
op->remap.prev->gpuva.flags &= ~XE_VMA_LAST_REBIND;
- if (err)
+ if (IS_ERR(fence))
break;
op->remap.prev = NULL;
}
if (next) {
op->remap.next->gpuva.flags |= XE_VMA_LAST_REBIND;
- err = xe_vm_bind(vm, op->remap.next, op->q,
- xe_vma_bo(op->remap.next),
- op->syncs, op->num_syncs,
- true, false,
- op->flags & XE_VMA_OP_LAST);
+ fence = xe_vm_bind(vm, op->remap.next, op->q,
+ xe_vma_bo(op->remap.next),
+ op->syncs, op->num_syncs,
+ true, false,
+ op->flags & XE_VMA_OP_LAST);
op->remap.next->gpuva.flags &= ~XE_VMA_LAST_REBIND;
- if (err)
+ if (IS_ERR(fence))
break;
op->remap.next = NULL;
}
@@ -2597,61 +2585,66 @@ static int op_execute(struct xe_vm *vm, struct xe_vma *vma,
break;
}
case DRM_GPUVA_OP_UNMAP:
- err = xe_vm_unbind(vm, vma, op->q, op->syncs,
- op->num_syncs, op->flags & XE_VMA_OP_FIRST,
- op->flags & XE_VMA_OP_LAST);
+ fence = xe_vm_unbind(vm, vma, op->q, op->syncs,
+ op->num_syncs, op->flags & XE_VMA_OP_FIRST,
+ op->flags & XE_VMA_OP_LAST);
break;
case DRM_GPUVA_OP_PREFETCH:
- err = xe_vm_prefetch(vm, vma, op->q, op->prefetch.region,
- op->syncs, op->num_syncs,
- op->flags & XE_VMA_OP_FIRST,
- op->flags & XE_VMA_OP_LAST);
+ fence = xe_vm_prefetch(vm, vma, op->q, op->prefetch.region,
+ op->syncs, op->num_syncs,
+ op->flags & XE_VMA_OP_FIRST,
+ op->flags & XE_VMA_OP_LAST);
break;
default:
drm_warn(&vm->xe->drm, "NOT POSSIBLE");
}
- if (err)
+ if (IS_ERR(fence))
trace_xe_vma_fail(vma);
- return err;
+ return fence;
}
-static int __xe_vma_op_execute(struct xe_vm *vm, struct xe_vma *vma,
- struct xe_vma_op *op)
+static struct dma_fence *
+__xe_vma_op_execute(struct xe_vm *vm, struct xe_vma *vma,
+ struct xe_vma_op *op)
{
+ struct dma_fence *fence;
int err;
retry_userptr:
- err = op_execute(vm, vma, op);
- if (err == -EAGAIN && xe_vma_is_userptr(vma)) {
+ fence = op_execute(vm, vma, op);
+ if (IS_ERR(fence) && PTR_ERR(fence) == -EAGAIN &&
+ xe_vma_is_userptr(vma)) {
lockdep_assert_held_write(&vm->lock);
err = xe_vma_userptr_pin_pages(vma);
if (!err)
goto retry_userptr;
+ fence = ERR_PTR(err);
trace_xe_vma_fail(vma);
}
- return err;
+ return fence;
}
-static int xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op)
+static struct dma_fence *
+xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op)
{
- int ret = 0;
+ struct dma_fence *fence = ERR_PTR(-ENOMEM);
lockdep_assert_held_write(&vm->lock);
#ifdef TEST_VM_ASYNC_OPS_ERROR
if (op->inject_error) {
op->inject_error = false;
- return -ENOMEM;
+ return fence;
}
#endif
switch (op->base.op) {
case DRM_GPUVA_OP_MAP:
- ret = __xe_vma_op_execute(vm, op->map.vma, op);
+ fence = __xe_vma_op_execute(vm, op->map.vma, op);
break;
case DRM_GPUVA_OP_REMAP:
{
@@ -2664,23 +2657,23 @@ static int xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op)
else
vma = op->remap.next;
- ret = __xe_vma_op_execute(vm, vma, op);
+ fence = __xe_vma_op_execute(vm, vma, op);
break;
}
case DRM_GPUVA_OP_UNMAP:
- ret = __xe_vma_op_execute(vm, gpuva_to_vma(op->base.unmap.va),
- op);
+ fence = __xe_vma_op_execute(vm, gpuva_to_vma(op->base.unmap.va),
+ op);
break;
case DRM_GPUVA_OP_PREFETCH:
- ret = __xe_vma_op_execute(vm,
- gpuva_to_vma(op->base.prefetch.va),
- op);
+ fence = __xe_vma_op_execute(vm,
+ gpuva_to_vma(op->base.prefetch.va),
+ op);
break;
default:
drm_warn(&vm->xe->drm, "NOT POSSIBLE");
}
- return ret;
+ return fence;
}
static void xe_vma_op_cleanup(struct xe_vm *vm, struct xe_vma_op *op)
@@ -2855,11 +2848,32 @@ static int vm_bind_ioctl_ops_lock(struct drm_exec *exec,
return err;
}
+static struct dma_fence *ops_execute(struct xe_vm *vm,
+ struct list_head *ops_list,
+ bool cleanup)
+{
+ struct xe_vma_op *op, *next;
+ struct dma_fence *fence = NULL;
+
+ list_for_each_entry_safe(op, next, ops_list, link) {
+ fence = xe_vma_op_execute(vm, op);
+ if (IS_ERR(fence)) {
+ drm_warn(&vm->xe->drm, "VM op(%d) failed with %ld",
+ op->base.op, PTR_ERR(fence));
+ return ERR_PTR(-ENOSPC);
+ }
+ if (cleanup)
+ xe_vma_op_cleanup(vm, op);
+ }
+
+ return fence;
+}
+
static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
struct list_head *ops_list)
{
struct drm_exec exec;
- struct xe_vma_op *op, *next;
+ struct dma_fence *fence;
int err;
lockdep_assert_held_write(&vm->lock);
@@ -2868,23 +2882,17 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
if (err)
return err;
- list_for_each_entry_safe(op, next, ops_list, link) {
- err = xe_vma_op_execute(vm, op);
- if (err) {
- drm_warn(&vm->xe->drm, "VM op(%d) failed with %d",
- op->base.op, err);
- /*
- * FIXME: Killing VM rather than proper error handling
- */
- xe_vm_kill(vm, false);
- err = -ENOSPC;
- goto unlock;
- }
- xe_vma_op_cleanup(vm, op);
+ fence = ops_execute(vm, ops_list, true);
+ if (IS_ERR(fence)) {
+ err = PTR_ERR(fence);
+ /* FIXME: Killing VM rather than proper error handling */
+ xe_vm_kill(vm, false);
+ } else {
+ dma_fence_put(fence);
}
-unlock:
drm_exec_fini(&exec);
+
return err;
}
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 06/27] drm/xe: Move migrate to prefetch to op_lock funtion
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (4 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 05/27] drm/xe: Add ops_execute function which returns a fence Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 07/27] drm/xe: Add struct xe_vma_ops abstraction Matthew Brost
` (24 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
Migrates need to be done under drm exec to make lockdep happy, move
the migrate done for prefetches under the op_lock function.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_vm.c | 30 ++++++++++++++----------------
1 file changed, 14 insertions(+), 16 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 73202e1909e5..7c1491cdd0c3 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -2041,20 +2041,10 @@ static const u32 region_to_mem_type[] = {
static struct dma_fence *
xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
- struct xe_exec_queue *q, u32 region,
- struct xe_sync_entry *syncs, u32 num_syncs,
- bool first_op, bool last_op)
+ struct xe_exec_queue *q, struct xe_sync_entry *syncs,
+ u32 num_syncs, bool first_op, bool last_op)
{
struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
- int err;
-
- xe_assert(vm->xe, region <= ARRAY_SIZE(region_to_mem_type));
-
- if (!xe_vma_has_no_bo(vma)) {
- err = xe_bo_migrate(xe_vma_bo(vma), region_to_mem_type[region]);
- if (err)
- return ERR_PTR(err);
- }
if (vma->tile_mask != (vma->tile_present & ~vma->usm.tile_invalidated)) {
return xe_vm_bind(vm, vma, q, xe_vma_bo(vma), syncs, num_syncs,
@@ -2070,7 +2060,7 @@ xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
xe_sync_entry_signal(&syncs[i], NULL, fence);
}
- return fence;
+ return dma_fence_get(fence);
}
}
@@ -2590,8 +2580,7 @@ static struct dma_fence *op_execute(struct xe_vm *vm, struct xe_vma *vma,
op->flags & XE_VMA_OP_LAST);
break;
case DRM_GPUVA_OP_PREFETCH:
- fence = xe_vm_prefetch(vm, vma, op->q, op->prefetch.region,
- op->syncs, op->num_syncs,
+ fence = xe_vm_prefetch(vm, vma, op->q, op->syncs, op->num_syncs,
op->flags & XE_VMA_OP_FIRST,
op->flags & XE_VMA_OP_LAST);
break;
@@ -2813,8 +2802,17 @@ static int op_lock(struct drm_exec *exec, struct xe_vm *vm,
err = vma_lock(exec, gpuva_to_vma(op->base.unmap.va), false);
break;
case DRM_GPUVA_OP_PREFETCH:
- err = vma_lock(exec, gpuva_to_vma(op->base.prefetch.va), true);
+ {
+ struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va);
+ u32 region = op->prefetch.region;
+
+ xe_assert(vm->xe, region <= ARRAY_SIZE(region_to_mem_type));
+
+ err = vma_lock(exec, vma, false);
+ if (!err && !xe_vma_has_no_bo(vma))
+ err = xe_bo_migrate(xe_vma_bo(vma), region);
break;
+ }
default:
drm_warn(&vm->xe->drm, "NOT POSSIBLE");
}
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 07/27] drm/xe: Add struct xe_vma_ops abstraction
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (5 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 06/27] drm/xe: Move migrate to prefetch to op_lock funtion Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 08/27] drm/xe: Update xe_vm_rebind to use dummy VMA operations Matthew Brost
` (23 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
This will help with upcoming change to 1 job for list VMA operations.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_vm.c | 37 ++++++++++++++++++--------------
drivers/gpu/drm/xe/xe_vm_types.h | 5 +++++
2 files changed, 26 insertions(+), 16 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 7c1491cdd0c3..afb229cf2149 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -2367,7 +2367,7 @@ static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op)
static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
struct drm_gpuva_ops *ops,
struct xe_sync_entry *syncs, u32 num_syncs,
- struct list_head *ops_list, bool last,
+ struct xe_vma_ops *vops, bool last,
bool async)
{
struct xe_vma_op *last_op = NULL;
@@ -2378,10 +2378,10 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
drm_gpuva_for_each_op(__op, ops) {
struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
- bool first = list_empty(ops_list);
+ bool first = list_empty(&vops->list);
INIT_LIST_HEAD(&op->link);
- list_add_tail(&op->link, ops_list);
+ list_add_tail(&op->link, &vops->list);
if (first) {
op->flags |= XE_VMA_OP_FIRST;
@@ -2496,7 +2496,7 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
}
/* FIXME: Unhandled corner case */
- XE_WARN_ON(!last_op && last && !list_empty(ops_list));
+ XE_WARN_ON(!last_op && last && !list_empty(&vops->list));
if (!last_op)
return 0;
@@ -2822,7 +2822,7 @@ static int op_lock(struct drm_exec *exec, struct xe_vm *vm,
static int vm_bind_ioctl_ops_lock(struct drm_exec *exec,
struct xe_vm *vm,
- struct list_head *ops_list)
+ struct xe_vma_ops *vops)
{
struct xe_vma_op *op;
int err;
@@ -2834,7 +2834,7 @@ static int vm_bind_ioctl_ops_lock(struct drm_exec *exec,
if (err)
goto out;
- list_for_each_entry(op, ops_list, link) {
+ list_for_each_entry(op, &vops->list, link) {
err = op_lock(exec, vm, op);
drm_exec_retry_on_contention(exec);
if (err)
@@ -2847,13 +2847,13 @@ static int vm_bind_ioctl_ops_lock(struct drm_exec *exec,
}
static struct dma_fence *ops_execute(struct xe_vm *vm,
- struct list_head *ops_list,
+ struct xe_vma_ops *vops,
bool cleanup)
{
struct xe_vma_op *op, *next;
struct dma_fence *fence = NULL;
- list_for_each_entry_safe(op, next, ops_list, link) {
+ list_for_each_entry_safe(op, next, &vops->list, link) {
fence = xe_vma_op_execute(vm, op);
if (IS_ERR(fence)) {
drm_warn(&vm->xe->drm, "VM op(%d) failed with %ld",
@@ -2868,7 +2868,7 @@ static struct dma_fence *ops_execute(struct xe_vm *vm,
}
static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
- struct list_head *ops_list)
+ struct xe_vma_ops *vops)
{
struct drm_exec exec;
struct dma_fence *fence;
@@ -2876,11 +2876,11 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
lockdep_assert_held_write(&vm->lock);
- err = vm_bind_ioctl_ops_lock(&exec, vm, ops_list);
+ err = vm_bind_ioctl_ops_lock(&exec, vm, vops);
if (err)
return err;
- fence = ops_execute(vm, ops_list, true);
+ fence = ops_execute(vm, vops, true);
if (IS_ERR(fence)) {
err = PTR_ERR(fence);
/* FIXME: Killing VM rather than proper error handling */
@@ -3010,6 +3010,11 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe,
return err;
}
+static void xe_vma_ops_init(struct xe_vma_ops *vops)
+{
+ INIT_LIST_HEAD(&vops->list);
+}
+
int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
struct xe_device *xe = to_xe_device(dev);
@@ -3023,7 +3028,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
u32 num_syncs;
struct xe_sync_entry *syncs = NULL;
struct drm_xe_vm_bind_op *bind_ops;
- LIST_HEAD(ops_list);
+ struct xe_vma_ops vops;
bool async;
int err;
int i;
@@ -3166,6 +3171,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
goto free_syncs;
}
+ xe_vma_ops_init(&vops);
for (i = 0; i < args->num_binds; ++i) {
u64 range = bind_ops[i].range;
u64 addr = bind_ops[i].addr;
@@ -3185,15 +3191,14 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
}
err = vm_bind_ioctl_ops_parse(vm, q, ops[i], syncs, num_syncs,
- &ops_list,
- i == args->num_binds - 1,
+ &vops, i == args->num_binds - 1,
async);
if (err)
goto unwind_ops;
}
/* Nothing to do */
- if (list_empty(&ops_list)) {
+ if (list_empty(&vops.list)) {
err = -ENODATA;
goto unwind_ops;
}
@@ -3202,7 +3207,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
if (q)
xe_exec_queue_get(q);
- err = vm_bind_ioctl_ops_execute(vm, &ops_list);
+ err = vm_bind_ioctl_ops_execute(vm, &vops);
up_write(&vm->lock);
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index aaf0c7101019..eb06a59b6d2f 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -420,4 +420,9 @@ struct xe_vma_op {
};
};
+/** struct xe_vma_ops - VMA operations */
+struct xe_vma_ops {
+ struct list_head list;
+};
+
#endif
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 08/27] drm/xe: Update xe_vm_rebind to use dummy VMA operations
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (6 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 07/27] drm/xe: Add struct xe_vma_ops abstraction Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 09/27] drm/xe: Move drm exec loop to vm_bind_ioctl_ops_execute Matthew Brost
` (22 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
All bind interfaces are transitioning to use ops.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_vm.c | 43 +++++--
drivers/gpu/drm/xe/xe_vm_types.h | 199 ++++++++++++++++---------------
2 files changed, 134 insertions(+), 108 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index afb229cf2149..8dc448bf6222 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -829,10 +829,23 @@ int xe_vm_userptr_check_repin(struct xe_vm *vm)
list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN;
}
-static struct dma_fence *
-xe_vm_bind_vma(struct xe_vma *vma, struct xe_exec_queue *q,
- struct xe_sync_entry *syncs, u32 num_syncs,
- bool first_op, bool last_op);
+static void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma)
+{
+ vm->dummy_ops.op.base.op = DRM_GPUVA_OP_MAP;
+ vm->dummy_ops.op.base.map.va.addr = vma->gpuva.va.addr;
+ vm->dummy_ops.op.base.map.va.range = vma->gpuva.va.range;
+ vm->dummy_ops.op.base.map.gem.obj = vma->gpuva.gem.obj;
+ vm->dummy_ops.op.base.map.gem.offset = vma->gpuva.gem.offset;
+ vm->dummy_ops.op.tile_mask = vma->tile_present;
+ vm->dummy_ops.op.map.vma = vma;
+ vm->dummy_ops.op.map.immediate = true;
+ vm->dummy_ops.op.map.read_only = xe_vma_read_only(vma);
+ vm->dummy_ops.op.map.is_null = xe_vma_is_null(vma);
+}
+
+static struct dma_fence *ops_execute(struct xe_vm *vm,
+ struct xe_vma_ops *vops,
+ bool cleanup);
struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
{
@@ -854,7 +867,9 @@ struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
trace_xe_vma_rebind_worker(vma);
else
trace_xe_vma_rebind_exec(vma);
- fence = xe_vm_bind_vma(vma, NULL, NULL, 0, false, false);
+
+ xe_vm_populate_dummy_rebind(vm, vma);
+ fence = ops_execute(vm, &vm->dummy_ops.vops, false);
if (IS_ERR(fence))
return fence;
}
@@ -1333,6 +1348,11 @@ static const struct xe_pt_ops xelp_pt_ops = {
static void vm_destroy_work_func(struct work_struct *w);
+static void xe_vma_ops_init(struct xe_vma_ops *vops)
+{
+ INIT_LIST_HEAD(&vops->list);
+}
+
struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
{
struct xe_vm *vm;
@@ -1354,6 +1374,10 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
init_rwsem(&vm->lock);
+ xe_vma_ops_init(&vm->dummy_ops.vops);
+ INIT_LIST_HEAD(&vm->dummy_ops.op.link);
+ list_add(&vm->dummy_ops.op.link, &vm->dummy_ops.vops.list);
+
INIT_LIST_HEAD(&vm->rebind_list);
INIT_LIST_HEAD(&vm->userptr.repin_list);
@@ -2516,7 +2540,7 @@ static struct dma_fence *op_execute(struct xe_vm *vm, struct xe_vma *vma,
{
struct dma_fence *fence;
- lockdep_assert_held_write(&vm->lock);
+ lockdep_assert_held(&vm->lock);
xe_vm_assert_held(vm);
xe_bo_assert_held(xe_vma_bo(vma));
@@ -2622,7 +2646,7 @@ xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op)
{
struct dma_fence *fence = ERR_PTR(-ENOMEM);
- lockdep_assert_held_write(&vm->lock);
+ lockdep_assert_held(&vm->lock);
#ifdef TEST_VM_ASYNC_OPS_ERROR
if (op->inject_error) {
@@ -3010,11 +3034,6 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe,
return err;
}
-static void xe_vma_ops_init(struct xe_vma_ops *vops)
-{
- INIT_LIST_HEAD(&vops->list);
-}
-
int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
struct xe_device *xe = to_xe_device(dev);
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index eb06a59b6d2f..65cbed7bbdda 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -18,6 +18,7 @@
#include "xe_range_fence.h"
struct xe_bo;
+struct xe_device;
struct xe_sync_entry;
struct xe_vm;
@@ -129,7 +130,100 @@ struct xe_vma {
struct xe_userptr userptr;
};
-struct xe_device;
+/** struct xe_vma_op_map - VMA map operation */
+struct xe_vma_op_map {
+ /** @vma: VMA to map */
+ struct xe_vma *vma;
+ /** @immediate: Immediate bind */
+ bool immediate;
+ /** @read_only: Read only */
+ bool read_only;
+ /** @is_null: is NULL binding */
+ bool is_null;
+};
+
+/** struct xe_vma_op_remap - VMA remap operation */
+struct xe_vma_op_remap {
+ /** @prev: VMA preceding part of a split mapping */
+ struct xe_vma *prev;
+ /** @next: VMA subsequent part of a split mapping */
+ struct xe_vma *next;
+ /** @start: start of the VMA unmap */
+ u64 start;
+ /** @range: range of the VMA unmap */
+ u64 range;
+ /** @skip_prev: skip prev rebind */
+ bool skip_prev;
+ /** @skip_next: skip next rebind */
+ bool skip_next;
+ /** @unmap_done: unmap operation in done */
+ bool unmap_done;
+};
+
+/** struct xe_vma_op_prefetch - VMA prefetch operation */
+struct xe_vma_op_prefetch {
+ /** @region: memory region to prefetch to */
+ u32 region;
+};
+
+/** enum xe_vma_op_flags - flags for VMA operation */
+enum xe_vma_op_flags {
+ /** @XE_VMA_OP_FIRST: first VMA operation for a set of syncs */
+ XE_VMA_OP_FIRST = BIT(0),
+ /** @XE_VMA_OP_LAST: last VMA operation for a set of syncs */
+ XE_VMA_OP_LAST = BIT(1),
+ /** @XE_VMA_OP_COMMITTED: VMA operation committed */
+ XE_VMA_OP_COMMITTED = BIT(2),
+ /** @XE_VMA_OP_PREV_COMMITTED: Previous VMA operation committed */
+ XE_VMA_OP_PREV_COMMITTED = BIT(3),
+ /** @XE_VMA_OP_NEXT_COMMITTED: Next VMA operation committed */
+ XE_VMA_OP_NEXT_COMMITTED = BIT(4),
+};
+
+/** struct xe_vma_op - VMA operation */
+struct xe_vma_op {
+ /** @base: GPUVA base operation */
+ struct drm_gpuva_op base;
+ /**
+ * @ops: GPUVA ops, when set call drm_gpuva_ops_free after this
+ * operations is processed
+ */
+ struct drm_gpuva_ops *ops;
+ /** @q: exec queue for this operation */
+ struct xe_exec_queue *q;
+ /**
+ * @syncs: syncs for this operation, only used on first and last
+ * operation
+ */
+ struct xe_sync_entry *syncs;
+ /** @num_syncs: number of syncs */
+ u32 num_syncs;
+ /** @link: async operation link */
+ struct list_head link;
+ /** @tile_mask: gt mask for this operation */
+ u8 tile_mask;
+ /** @flags: operation flags */
+ enum xe_vma_op_flags flags;
+
+#ifdef TEST_VM_ASYNC_OPS_ERROR
+ /** @inject_error: inject error to test async op error handling */
+ bool inject_error;
+#endif
+
+ union {
+ /** @map: VMA map operation specific data */
+ struct xe_vma_op_map map;
+ /** @remap: VMA remap operation specific data */
+ struct xe_vma_op_remap remap;
+ /** @prefetch: VMA prefetch operation specific data */
+ struct xe_vma_op_prefetch prefetch;
+ };
+};
+
+/** struct xe_vma_ops - VMA operations */
+struct xe_vma_ops {
+ struct list_head list;
+};
#define xe_vm_assert_held(vm) dma_resv_assert_held(&(vm)->resv)
@@ -324,105 +418,18 @@ struct xe_vm {
bool capture_once;
} error_capture;
+ /** @dummy_ops: dummy VMA ops to issue rebinds */
+ struct {
+ /** @ops: dummy VMA ops */
+ struct xe_vma_ops vops;
+ /** @op: dummy VMA op */
+ struct xe_vma_op op;
+ } dummy_ops;
+
/** @batch_invalidate_tlb: Always invalidate TLB before batch start */
bool batch_invalidate_tlb;
/** @xef: XE file handle for tracking this VM's drm client */
struct xe_file *xef;
};
-/** struct xe_vma_op_map - VMA map operation */
-struct xe_vma_op_map {
- /** @vma: VMA to map */
- struct xe_vma *vma;
- /** @immediate: Immediate bind */
- bool immediate;
- /** @read_only: Read only */
- bool read_only;
- /** @is_null: is NULL binding */
- bool is_null;
-};
-
-/** struct xe_vma_op_remap - VMA remap operation */
-struct xe_vma_op_remap {
- /** @prev: VMA preceding part of a split mapping */
- struct xe_vma *prev;
- /** @next: VMA subsequent part of a split mapping */
- struct xe_vma *next;
- /** @start: start of the VMA unmap */
- u64 start;
- /** @range: range of the VMA unmap */
- u64 range;
- /** @skip_prev: skip prev rebind */
- bool skip_prev;
- /** @skip_next: skip next rebind */
- bool skip_next;
- /** @unmap_done: unmap operation in done */
- bool unmap_done;
-};
-
-/** struct xe_vma_op_prefetch - VMA prefetch operation */
-struct xe_vma_op_prefetch {
- /** @region: memory region to prefetch to */
- u32 region;
-};
-
-/** enum xe_vma_op_flags - flags for VMA operation */
-enum xe_vma_op_flags {
- /** @XE_VMA_OP_FIRST: first VMA operation for a set of syncs */
- XE_VMA_OP_FIRST = BIT(0),
- /** @XE_VMA_OP_LAST: last VMA operation for a set of syncs */
- XE_VMA_OP_LAST = BIT(1),
- /** @XE_VMA_OP_COMMITTED: VMA operation committed */
- XE_VMA_OP_COMMITTED = BIT(2),
- /** @XE_VMA_OP_PREV_COMMITTED: Previous VMA operation committed */
- XE_VMA_OP_PREV_COMMITTED = BIT(3),
- /** @XE_VMA_OP_NEXT_COMMITTED: Next VMA operation committed */
- XE_VMA_OP_NEXT_COMMITTED = BIT(4),
-};
-
-/** struct xe_vma_op - VMA operation */
-struct xe_vma_op {
- /** @base: GPUVA base operation */
- struct drm_gpuva_op base;
- /**
- * @ops: GPUVA ops, when set call drm_gpuva_ops_free after this
- * operations is processed
- */
- struct drm_gpuva_ops *ops;
- /** @q: exec queue for this operation */
- struct xe_exec_queue *q;
- /**
- * @syncs: syncs for this operation, only used on first and last
- * operation
- */
- struct xe_sync_entry *syncs;
- /** @num_syncs: number of syncs */
- u32 num_syncs;
- /** @link: async operation link */
- struct list_head link;
- /** @tile_mask: gt mask for this operation */
- u8 tile_mask;
- /** @flags: operation flags */
- enum xe_vma_op_flags flags;
-
-#ifdef TEST_VM_ASYNC_OPS_ERROR
- /** @inject_error: inject error to test async op error handling */
- bool inject_error;
-#endif
-
- union {
- /** @map: VMA map operation specific data */
- struct xe_vma_op_map map;
- /** @remap: VMA remap operation specific data */
- struct xe_vma_op_remap remap;
- /** @prefetch: VMA prefetch operation specific data */
- struct xe_vma_op_prefetch prefetch;
- };
-};
-
-/** struct xe_vma_ops - VMA operations */
-struct xe_vma_ops {
- struct list_head list;
-};
-
#endif
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 09/27] drm/xe: Move drm exec loop to vm_bind_ioctl_ops_execute
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (7 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 08/27] drm/xe: Update xe_vm_rebind to use dummy VMA operations Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-12 14:26 ` Dafna Hirschfeld
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 10/27] drm/xe: Fixup error handling / ref counting in VM bind IOCTL Matthew Brost
` (21 subsequent siblings)
30 siblings, 1 reply; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
The execution of the ops list can allocate memory in TTM which can
trigger an eviction. Lockdep is not happy if the drm exec is closed when
this happens.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_vm.c | 46 ++++++++++++++++++--------------------
1 file changed, 22 insertions(+), 24 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 8dc448bf6222..485252f69ed7 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -2851,23 +2851,17 @@ static int vm_bind_ioctl_ops_lock(struct drm_exec *exec,
struct xe_vma_op *op;
int err;
- drm_exec_init(exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
- drm_exec_until_all_locked(exec) {
- err = drm_exec_prepare_obj(exec, &xe_vm_ttm_bo(vm)->base, 1);
- drm_exec_retry_on_contention(exec);
- if (err)
- goto out;
+ err = drm_exec_prepare_obj(exec, &xe_vm_ttm_bo(vm)->base, 1);
+ if (err)
+ return err;
- list_for_each_entry(op, &vops->list, link) {
- err = op_lock(exec, vm, op);
- drm_exec_retry_on_contention(exec);
- if (err)
- goto out;
- }
+ list_for_each_entry(op, &vops->list, link) {
+ err = op_lock(exec, vm, op);
+ if (err)
+ return err;
}
-out:
- return err;
+ return 0;
}
static struct dma_fence *ops_execute(struct xe_vm *vm,
@@ -2900,17 +2894,21 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
lockdep_assert_held_write(&vm->lock);
- err = vm_bind_ioctl_ops_lock(&exec, vm, vops);
- if (err)
- return err;
+ drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
+ drm_exec_until_all_locked(&exec) {
+ err = vm_bind_ioctl_ops_lock(&exec, vm, vops);
+ drm_exec_retry_on_contention(&exec);
+ if (err)
+ return err;
- fence = ops_execute(vm, vops, true);
- if (IS_ERR(fence)) {
- err = PTR_ERR(fence);
- /* FIXME: Killing VM rather than proper error handling */
- xe_vm_kill(vm, false);
- } else {
- dma_fence_put(fence);
+ fence = ops_execute(vm, vops, true);
+ if (IS_ERR(fence)) {
+ err = PTR_ERR(fence);
+ /* FIXME: Killing VM rather than proper error handling */
+ xe_vm_kill(vm, false);
+ } else {
+ dma_fence_put(fence);
+ }
}
drm_exec_fini(&exec);
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* Re: [Intel-xe] [PATCH v2 09/27] drm/xe: Move drm exec loop to vm_bind_ioctl_ops_execute
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 09/27] drm/xe: Move drm exec loop to vm_bind_ioctl_ops_execute Matthew Brost
@ 2023-11-12 14:26 ` Dafna Hirschfeld
0 siblings, 0 replies; 43+ messages in thread
From: Dafna Hirschfeld @ 2023-11-12 14:26 UTC (permalink / raw)
To: Matthew Brost; +Cc: intel-xe
On 06.11.2023 21:25, Matthew Brost wrote:
>The execution of the ops list can allocate memory in TTM which can
>trigger an eviction. Lockdep is not happy if the drm exec is closed when
>this happens.
>
>Signed-off-by: Matthew Brost <matthew.brost@intel.com>
>---
> drivers/gpu/drm/xe/xe_vm.c | 46 ++++++++++++++++++--------------------
> 1 file changed, 22 insertions(+), 24 deletions(-)
>
>diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
>index 8dc448bf6222..485252f69ed7 100644
>--- a/drivers/gpu/drm/xe/xe_vm.c
>+++ b/drivers/gpu/drm/xe/xe_vm.c
>@@ -2851,23 +2851,17 @@ static int vm_bind_ioctl_ops_lock(struct drm_exec *exec,
> struct xe_vma_op *op;
> int err;
>
>- drm_exec_init(exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
>- drm_exec_until_all_locked(exec) {
>- err = drm_exec_prepare_obj(exec, &xe_vm_ttm_bo(vm)->base, 1);
>- drm_exec_retry_on_contention(exec);
>- if (err)
>- goto out;
>+ err = drm_exec_prepare_obj(exec, &xe_vm_ttm_bo(vm)->base, 1);
>+ if (err)
>+ return err;
>
>- list_for_each_entry(op, &vops->list, link) {
>- err = op_lock(exec, vm, op);
>- drm_exec_retry_on_contention(exec);
>- if (err)
>- goto out;
>- }
>+ list_for_each_entry(op, &vops->list, link) {
>+ err = op_lock(exec, vm, op);
>+ if (err)
>+ return err;
> }
>
>-out:
>- return err;
>+ return 0;
> }
>
> static struct dma_fence *ops_execute(struct xe_vm *vm,
>@@ -2900,17 +2894,21 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
>
> lockdep_assert_held_write(&vm->lock);
>
>- err = vm_bind_ioctl_ops_lock(&exec, vm, vops);
>- if (err)
>- return err;
>+ drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
>+ drm_exec_until_all_locked(&exec) {
>+ err = vm_bind_ioctl_ops_lock(&exec, vm, vops);
>+ drm_exec_retry_on_contention(&exec);
>+ if (err)
>+ return err;
I think here you miss drm_exec_fini
thanks,
Dafna
>
>- fence = ops_execute(vm, vops, true);
>- if (IS_ERR(fence)) {
>- err = PTR_ERR(fence);
>- /* FIXME: Killing VM rather than proper error handling */
>- xe_vm_kill(vm, false);
>- } else {
>- dma_fence_put(fence);
>+ fence = ops_execute(vm, vops, true);
>+ if (IS_ERR(fence)) {
>+ err = PTR_ERR(fence);
>+ /* FIXME: Killing VM rather than proper error handling */
>+ xe_vm_kill(vm, false);
>+ } else {
>+ dma_fence_put(fence);
>+ }
> }
>
> drm_exec_fini(&exec);
>--
>2.34.1
>
^ permalink raw reply [flat|nested] 43+ messages in thread
* [Intel-xe] [PATCH v2 10/27] drm/xe: Fixup error handling / ref counting in VM bind IOCTL
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (8 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 09/27] drm/xe: Move drm exec loop to vm_bind_ioctl_ops_execute Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 11/27] drm/xe: Convert pagefault rebind to use ops interface Matthew Brost
` (20 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
Part of the rework for the error handling.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_vm.c | 62 ++++++--------------------------
drivers/gpu/drm/xe/xe_vm_types.h | 9 ++---
2 files changed, 12 insertions(+), 59 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 485252f69ed7..20866d7ddb9b 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -844,8 +844,7 @@ static void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma)
}
static struct dma_fence *ops_execute(struct xe_vm *vm,
- struct xe_vma_ops *vops,
- bool cleanup);
+ struct xe_vma_ops *vops);
struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
{
@@ -869,7 +868,7 @@ struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
trace_xe_vma_rebind_exec(vma);
xe_vm_populate_dummy_rebind(vm, vma);
- fence = ops_execute(vm, &vm->dummy_ops.vops, false);
+ fence = ops_execute(vm, &vm->dummy_ops.vops);
if (IS_ERR(fence))
return fence;
}
@@ -2525,7 +2524,6 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
if (!last_op)
return 0;
- last_op->ops = ops;
if (last) {
last_op->flags |= XE_VMA_OP_LAST;
last_op->num_syncs = num_syncs;
@@ -2689,25 +2687,6 @@ xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op)
return fence;
}
-static void xe_vma_op_cleanup(struct xe_vm *vm, struct xe_vma_op *op)
-{
- bool last = op->flags & XE_VMA_OP_LAST;
-
- if (last) {
- while (op->num_syncs--)
- xe_sync_entry_cleanup(&op->syncs[op->num_syncs]);
- kfree(op->syncs);
- if (op->q)
- xe_exec_queue_put(op->q);
- }
- if (!list_empty(&op->link))
- list_del(&op->link);
- if (op->ops)
- drm_gpuva_ops_free(&vm->gpuvm, op->ops);
- if (last)
- xe_vm_put(vm);
-}
-
static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op,
bool post_commit, bool prev_post_commit,
bool next_post_commit)
@@ -2769,7 +2748,7 @@ static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm,
{
int i;
- for (i = num_ops_list - 1; i; ++i) {
+ for (i = num_ops_list - 1; i >= 0; --i) {
struct drm_gpuva_ops *__ops = ops[i];
struct drm_gpuva_op *__op;
@@ -2784,8 +2763,6 @@ static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm,
op->flags & XE_VMA_OP_PREV_COMMITTED,
op->flags & XE_VMA_OP_NEXT_COMMITTED);
}
-
- drm_gpuva_ops_free(&vm->gpuvm, __ops);
}
}
@@ -2865,8 +2842,7 @@ static int vm_bind_ioctl_ops_lock(struct drm_exec *exec,
}
static struct dma_fence *ops_execute(struct xe_vm *vm,
- struct xe_vma_ops *vops,
- bool cleanup)
+ struct xe_vma_ops *vops)
{
struct xe_vma_op *op, *next;
struct dma_fence *fence = NULL;
@@ -2878,8 +2854,6 @@ static struct dma_fence *ops_execute(struct xe_vm *vm,
op->base.op, PTR_ERR(fence));
return ERR_PTR(-ENOSPC);
}
- if (cleanup)
- xe_vma_op_cleanup(vm, op);
}
return fence;
@@ -2901,7 +2875,7 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
if (err)
return err;
- fence = ops_execute(vm, vops, true);
+ fence = ops_execute(vm, vops);
if (IS_ERR(fence)) {
err = PTR_ERR(fence);
/* FIXME: Killing VM rather than proper error handling */
@@ -3220,30 +3194,14 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
goto unwind_ops;
}
- xe_vm_get(vm);
- if (q)
- xe_exec_queue_get(q);
-
err = vm_bind_ioctl_ops_execute(vm, &vops);
- up_write(&vm->lock);
-
- if (q)
- xe_exec_queue_put(q);
- xe_vm_put(vm);
-
- for (i = 0; bos && i < args->num_binds; ++i)
- xe_bo_put(bos[i]);
-
- kfree(bos);
- kfree(ops);
- if (args->num_binds > 1)
- kfree(bind_ops);
-
- return err;
-
unwind_ops:
- vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds);
+ if (err && err != -ENODATA)
+ vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds);
+ for (i = args->num_binds - 1; i >= 0; --i)
+ if (ops[i])
+ drm_gpuva_ops_free(&vm->gpuvm, ops[i]);
free_syncs:
if (err == -ENODATA) {
struct dma_fence *fence;
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index 65cbed7bbdda..6424f75eb2df 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -184,11 +184,6 @@ enum xe_vma_op_flags {
struct xe_vma_op {
/** @base: GPUVA base operation */
struct drm_gpuva_op base;
- /**
- * @ops: GPUVA ops, when set call drm_gpuva_ops_free after this
- * operations is processed
- */
- struct drm_gpuva_ops *ops;
/** @q: exec queue for this operation */
struct xe_exec_queue *q;
/**
@@ -198,9 +193,9 @@ struct xe_vma_op {
struct xe_sync_entry *syncs;
/** @num_syncs: number of syncs */
u32 num_syncs;
- /** @link: async operation link */
+ /** @link: operation link */
struct list_head link;
- /** @tile_mask: gt mask for this operation */
+ /** @tile_mask: tile mask for this operation */
u8 tile_mask;
/** @flags: operation flags */
enum xe_vma_op_flags flags;
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 11/27] drm/xe: Convert pagefault rebind to use ops interface
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (9 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 10/27] drm/xe: Fixup error handling / ref counting in VM bind IOCTL Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 12/27] drm/xe: Add some members to xe_vma_ops Matthew Brost
` (19 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
Calling into the VM layer is correct not the PT layer. Also all bind
moving towards VMA operations interfaces.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_gt_pagefault.c | 7 ++++---
drivers/gpu/drm/xe/xe_vm.c | 12 ++++--------
drivers/gpu/drm/xe/xe_vm.h | 3 +++
3 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c
index 5b3585e2c125..27845a6b0b23 100644
--- a/drivers/gpu/drm/xe/xe_gt_pagefault.c
+++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c
@@ -18,7 +18,6 @@
#include "xe_guc.h"
#include "xe_guc_ct.h"
#include "xe_migrate.h"
-#include "xe_pt.h"
#include "xe_trace.h"
#include "xe_vm.h"
@@ -204,8 +203,10 @@ static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
/* Bind VMA only to the GT that has faulted */
trace_xe_vma_pf_bind(vma);
- fence = __xe_pt_bind_vma(tile, vma, xe_tile_migrate_engine(tile), NULL, 0,
- vma->tile_present & BIT(tile->id));
+ xe_vm_populate_dummy_rebind(vm, vma);
+ vm->dummy_ops.op.tile_mask = BIT(tile->id);
+ vm->dummy_ops.op.q = xe_tile_migrate_engine(tile);
+ fence = xe_vm_ops_execute(vm, &vm->dummy_ops.vops);
if (IS_ERR(fence)) {
ret = PTR_ERR(fence);
goto unlock_dma_resv;
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 20866d7ddb9b..085acd083ec2 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -829,7 +829,7 @@ int xe_vm_userptr_check_repin(struct xe_vm *vm)
list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN;
}
-static void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma)
+void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma)
{
vm->dummy_ops.op.base.op = DRM_GPUVA_OP_MAP;
vm->dummy_ops.op.base.map.va.addr = vma->gpuva.va.addr;
@@ -843,9 +843,6 @@ static void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma)
vm->dummy_ops.op.map.is_null = xe_vma_is_null(vma);
}
-static struct dma_fence *ops_execute(struct xe_vm *vm,
- struct xe_vma_ops *vops);
-
struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
{
struct dma_fence *fence = NULL;
@@ -868,7 +865,7 @@ struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
trace_xe_vma_rebind_exec(vma);
xe_vm_populate_dummy_rebind(vm, vma);
- fence = ops_execute(vm, &vm->dummy_ops.vops);
+ fence = xe_vm_ops_execute(vm, &vm->dummy_ops.vops);
if (IS_ERR(fence))
return fence;
}
@@ -2841,8 +2838,7 @@ static int vm_bind_ioctl_ops_lock(struct drm_exec *exec,
return 0;
}
-static struct dma_fence *ops_execute(struct xe_vm *vm,
- struct xe_vma_ops *vops)
+struct dma_fence *xe_vm_ops_execute(struct xe_vm *vm, struct xe_vma_ops *vops)
{
struct xe_vma_op *op, *next;
struct dma_fence *fence = NULL;
@@ -2875,7 +2871,7 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
if (err)
return err;
- fence = ops_execute(vm, vops);
+ fence = xe_vm_ops_execute(vm, vops);
if (IS_ERR(fence)) {
err = PTR_ERR(fence);
/* FIXME: Killing VM rather than proper error handling */
diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h
index b08c75fbd8a1..7cbb512bcb10 100644
--- a/drivers/gpu/drm/xe/xe_vm.h
+++ b/drivers/gpu/drm/xe/xe_vm.h
@@ -224,6 +224,9 @@ int xe_analyze_vm(struct drm_printer *p, struct xe_vm *vm, int gt_id);
int xe_vm_prepare_vma(struct drm_exec *exec, struct xe_vma *vma,
unsigned int num_shared);
+void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma);
+struct dma_fence *xe_vm_ops_execute(struct xe_vm *vm, struct xe_vma_ops *vops);
+
#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
#define vm_dbg drm_dbg
#else
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 12/27] drm/xe: Add some members to xe_vma_ops
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (10 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 11/27] drm/xe: Convert pagefault rebind to use ops interface Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 13/27] drm/xe: Add vm_bind_ioctl_ops_install_fences helper Matthew Brost
` (18 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
This will help with moving to single jobs for many bind operations.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_vm.c | 13 ++++++++++---
drivers/gpu/drm/xe/xe_vm_types.h | 9 +++++++++
2 files changed, 19 insertions(+), 3 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 085acd083ec2..6e9a3284238e 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -1344,9 +1344,16 @@ static const struct xe_pt_ops xelp_pt_ops = {
static void vm_destroy_work_func(struct work_struct *w);
-static void xe_vma_ops_init(struct xe_vma_ops *vops)
+static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm,
+ struct xe_exec_queue *q,
+ struct xe_sync_entry *syncs, u32 num_syncs)
{
+ memset(vops, 0, sizeof(*vops));
INIT_LIST_HEAD(&vops->list);
+ vops->vm = vm;
+ vops->q = q;
+ vops->syncs = syncs;
+ vops->num_syncs = num_syncs;
}
struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
@@ -1370,7 +1377,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
init_rwsem(&vm->lock);
- xe_vma_ops_init(&vm->dummy_ops.vops);
+ xe_vma_ops_init(&vm->dummy_ops.vops, vm, NULL, NULL, 0);
INIT_LIST_HEAD(&vm->dummy_ops.op.link);
list_add(&vm->dummy_ops.op.link, &vm->dummy_ops.vops.list);
@@ -3158,7 +3165,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
goto free_syncs;
}
- xe_vma_ops_init(&vops);
+ xe_vma_ops_init(&vops, vm, q, syncs, num_syncs);
for (i = 0; i < args->num_binds; ++i) {
u64 range = bind_ops[i].range;
u64 addr = bind_ops[i].addr;
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index 6424f75eb2df..df85975b43fb 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -217,7 +217,16 @@ struct xe_vma_op {
/** struct xe_vma_ops - VMA operations */
struct xe_vma_ops {
+ /** @list: list of VMA operations */
struct list_head list;
+ /** @vm: VM */
+ struct xe_vm *vm;
+ /** @q: exec queue these operations */
+ struct xe_exec_queue *q;
+ /** @syncs: syncs these operation */
+ struct xe_sync_entry *syncs;
+ /** @num_syncs: number of syncs */
+ u32 num_syncs;
};
#define xe_vm_assert_held(vm) dma_resv_assert_held(&(vm)->resv)
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 13/27] drm/xe: Add vm_bind_ioctl_ops_install_fences helper
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (11 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 12/27] drm/xe: Add some members to xe_vma_ops Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 14/27] drm/xe: Drop rebind argument from xe_pt_prepare_bind Matthew Brost
` (17 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
Will help with transition single job for many bind ops. Also fix some
reference counting errors.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_exec_queue.c | 8 +++--
drivers/gpu/drm/xe/xe_sync.c | 4 +--
drivers/gpu/drm/xe/xe_vm.c | 57 ++++++++++++------------------
3 files changed, 30 insertions(+), 39 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c
index 35710b66e5de..6d142fb75dfc 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue.c
+++ b/drivers/gpu/drm/xe/xe_exec_queue.c
@@ -964,20 +964,24 @@ void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q)
* @q: The exec queue
* @vm: The VM the engine does a bind or exec for
*
- * Get last fence, does not take a ref
+ * Get last fence, takes a ref
*
* Returns: last fence if not signaled, dma fence stub if signaled
*/
struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q,
struct xe_vm *vm)
{
+ struct dma_fence *fence;
+
xe_exec_queue_last_fence_lockdep_assert(q, vm);
if (q->last_fence &&
test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags))
xe_exec_queue_last_fence_put(q, vm);
- return q->last_fence ? q->last_fence : dma_fence_get_stub();
+ fence = q->last_fence ? q->last_fence : dma_fence_get_stub();
+ dma_fence_get(fence);
+ return fence;
}
/**
diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c
index 6b38c74a1de1..2e08957aec22 100644
--- a/drivers/gpu/drm/xe/xe_sync.c
+++ b/drivers/gpu/drm/xe/xe_sync.c
@@ -312,9 +312,7 @@ xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
/* Easy cases... */
if (!num_in_fence) {
- fence = xe_exec_queue_last_fence_get(q, vm);
- dma_fence_get(fence);
- return fence;
+ return xe_exec_queue_last_fence_get(q, vm);
} else if (num_in_fence == 1) {
dma_fence_get(fence);
return fence;
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 6e9a3284238e..f708c08633a7 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -1723,7 +1723,7 @@ xe_vm_unbind_vma(struct xe_vma *vma, struct xe_exec_queue *q,
struct dma_fence *fence = NULL;
struct dma_fence **fences = NULL;
struct dma_fence_array *cf = NULL;
- int cur_fence = 0, i;
+ int cur_fence = 0;
int number_tiles = hweight8(vma->tile_present);
int err;
u8 id;
@@ -1769,12 +1769,6 @@ xe_vm_unbind_vma(struct xe_vma *vma, struct xe_exec_queue *q,
}
}
- if (last_op) {
- for (i = 0; i < num_syncs; i++)
- xe_sync_entry_signal(&syncs[i], NULL,
- cf ? &cf->base : fence);
- }
-
return cf ? &cf->base : !fence ?
xe_exec_queue_last_fence_get(wait_exec_queue, vm) : fence;
@@ -1798,7 +1792,7 @@ xe_vm_bind_vma(struct xe_vma *vma, struct xe_exec_queue *q,
struct dma_fence **fences = NULL;
struct dma_fence_array *cf = NULL;
struct xe_vm *vm = xe_vma_vm(vma);
- int cur_fence = 0, i;
+ int cur_fence = 0;
int number_tiles = hweight8(vma->tile_mask);
int err;
u8 id;
@@ -1845,12 +1839,6 @@ xe_vm_bind_vma(struct xe_vma *vma, struct xe_exec_queue *q,
}
}
- if (last_op) {
- for (i = 0; i < num_syncs; i++)
- xe_sync_entry_signal(&syncs[i], NULL,
- cf ? &cf->base : fence);
- }
-
return cf ? &cf->base : fence;
err_fences:
@@ -1886,15 +1874,8 @@ xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q,
if (IS_ERR(fence))
return fence;
} else {
- int i;
-
xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
-
fence = xe_exec_queue_last_fence_get(wait_exec_queue, vm);
- if (last_op) {
- for (i = 0; i < num_syncs; i++)
- xe_sync_entry_signal(&syncs[i], NULL, fence);
- }
}
if (last_op)
@@ -1920,7 +1901,6 @@ xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma,
if (IS_ERR(fence))
return fence;
- xe_vma_destroy(vma, fence);
if (last_op)
xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence);
if (last_op && xe_vm_sync_mode(vm, q))
@@ -2077,17 +2057,7 @@ xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
return xe_vm_bind(vm, vma, q, xe_vma_bo(vma), syncs, num_syncs,
true, first_op, last_op);
} else {
- struct dma_fence *fence =
- xe_exec_queue_last_fence_get(wait_exec_queue, vm);
- int i;
-
- /* Nothing to do, signal fences now */
- if (last_op) {
- for (i = 0; i < num_syncs; i++)
- xe_sync_entry_signal(&syncs[i], NULL, fence);
- }
-
- return dma_fence_get(fence);
+ return xe_exec_queue_last_fence_get(wait_exec_queue, vm);
}
}
@@ -2862,6 +2832,25 @@ struct dma_fence *xe_vm_ops_execute(struct xe_vm *vm, struct xe_vma_ops *vops)
return fence;
}
+static void vm_bind_ioctl_ops_install_fences(struct xe_vm *vm,
+ struct xe_vma_ops *vops,
+ struct dma_fence *fence)
+{
+ struct xe_vma_op *op;
+ int i;
+
+ list_for_each_entry(op, &vops->list, link) {
+ if (op->base.op == DRM_GPUVA_OP_UNMAP)
+ xe_vma_destroy(gpuva_to_vma(op->base.unmap.va), fence);
+ else if (op->base.op == DRM_GPUVA_OP_REMAP)
+ xe_vma_destroy(gpuva_to_vma(op->base.remap.unmap->va),
+ fence);
+ }
+ for (i = 0; i < vops->num_syncs; i++)
+ xe_sync_entry_signal(vops->syncs + i, NULL, fence);
+ dma_fence_put(fence);
+}
+
static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
struct xe_vma_ops *vops)
{
@@ -2884,7 +2873,7 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
/* FIXME: Killing VM rather than proper error handling */
xe_vm_kill(vm, false);
} else {
- dma_fence_put(fence);
+ vm_bind_ioctl_ops_install_fences(vm, vops, fence);
}
}
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 14/27] drm/xe: Drop rebind argument from xe_pt_prepare_bind
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (12 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 13/27] drm/xe: Add vm_bind_ioctl_ops_install_fences helper Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 15/27] drm/xe: Add xe_vm_pgtable_update_op to xe_vma_ops Matthew Brost
` (16 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
This is unused, drop it.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_pt.c | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index 31afab617b4e..f06ceaea98cf 100644
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -914,8 +914,7 @@ static void xe_pt_commit_bind(struct xe_vma *vma,
static int
xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma,
- struct xe_vm_pgtable_update *entries, u32 *num_entries,
- bool rebind)
+ struct xe_vm_pgtable_update *entries, u32 *num_entries)
{
int err;
@@ -1271,7 +1270,7 @@ __xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue
"Preparing bind, with range [%llx...%llx) engine %p.\n",
xe_vma_start(vma), xe_vma_end(vma), q);
- err = xe_pt_prepare_bind(tile, vma, entries, &num_entries, rebind);
+ err = xe_pt_prepare_bind(tile, vma, entries, &num_entries);
if (err)
goto err;
xe_tile_assert(tile, num_entries <= ARRAY_SIZE(entries));
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 15/27] drm/xe: Add xe_vm_pgtable_update_op to xe_vma_ops
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (13 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 14/27] drm/xe: Drop rebind argument from xe_pt_prepare_bind Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 16/27] drm/xe: Move setting last fence and sync wait to vm_bind_ioctl_ops_install_fences Matthew Brost
` (15 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
Will help with the converstion to 1 job per VM bind IOCTL. Allocation
only implemented in this patch.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_pt_types.h | 12 ++++++
drivers/gpu/drm/xe/xe_vm.c | 64 ++++++++++++++++++++++++++++++--
drivers/gpu/drm/xe/xe_vm_types.h | 8 ++++
3 files changed, 80 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_pt_types.h b/drivers/gpu/drm/xe/xe_pt_types.h
index 82cbf1ef8e57..33b2df72aad7 100644
--- a/drivers/gpu/drm/xe/xe_pt_types.h
+++ b/drivers/gpu/drm/xe/xe_pt_types.h
@@ -73,4 +73,16 @@ struct xe_vm_pgtable_update {
u32 flags;
};
+/** struct xe_vm_pgtable_update_op - Page table update operation */
+struct xe_vm_pgtable_update_op {
+ /** @entries: entries to update for this operation */
+ struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1];
+ /** @num_entries: number of entries for this update operation */
+ u32 num_entries;
+ /** @bind: is a bind */
+ bool bind;
+ /** @rebind: is a rebind */
+ bool rebind;
+};
+
#endif
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index f708c08633a7..7ae12e4441b0 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -1356,6 +1356,42 @@ static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm,
vops->num_syncs = num_syncs;
}
+static int xe_vma_ops_alloc(struct xe_vma_ops *vops)
+{
+ int i;
+
+ for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) {
+ if (!vops->pt_update_ops[i].num_ops)
+ continue;
+
+ vops->pt_update_ops[i].ops =
+ kmalloc_array(vops->pt_update_ops[i].num_ops,
+ sizeof(*vops->pt_update_ops[i].ops),
+ GFP_KERNEL);
+ if (!vops->pt_update_ops[i].ops)
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void xe_vma_ops_fini(struct xe_vma_ops *vops)
+{
+ int i;
+
+ for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
+ kfree(vops->pt_update_ops[i].ops);
+}
+
+static void xe_vma_ops_incr_pt_update_ops(struct xe_vma_ops *vops, u8 tile_mask)
+{
+ int i;
+
+ for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
+ if (BIT(i) & tile_mask)
+ ++vops->pt_update_ops[i].num_ops;
+}
+
struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
{
struct xe_vm *vm;
@@ -1380,6 +1416,11 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
xe_vma_ops_init(&vm->dummy_ops.vops, vm, NULL, NULL, 0);
INIT_LIST_HEAD(&vm->dummy_ops.op.link);
list_add(&vm->dummy_ops.op.link, &vm->dummy_ops.vops.list);
+ for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
+ vm->dummy_ops.vops.pt_update_ops[i].num_ops = 1;
+ err = xe_vma_ops_alloc(&vm->dummy_ops.vops);
+ if (err)
+ goto err_free;
INIT_LIST_HEAD(&vm->rebind_list);
@@ -1517,12 +1558,14 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
dma_resv_unlock(&vm->resv);
drm_gpuvm_destroy(&vm->gpuvm);
err_put:
+ if (!(flags & XE_VM_FLAG_MIGRATION))
+ xe_device_mem_access_put(xe);
dma_resv_fini(&vm->resv);
for_each_tile(tile, xe, id)
xe_range_fence_tree_fini(&vm->rftree[id]);
+err_free:
+ xe_vma_ops_fini(&vm->dummy_ops.vops);
kfree(vm);
- if (!(flags & XE_VM_FLAG_MIGRATION))
- xe_device_mem_access_put(xe);
return ERR_PTR(err);
}
@@ -1676,6 +1719,7 @@ static void vm_destroy_work_func(struct work_struct *w)
trace_xe_vm_free(vm);
dma_fence_put(vm->rebind_fence);
dma_resv_fini(&vm->resv);
+ xe_vma_ops_fini(&vm->dummy_ops.vops);
kfree(vm);
}
@@ -2360,7 +2404,6 @@ static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op)
return err;
}
-
static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
struct drm_gpuva_ops *ops,
struct xe_sync_entry *syncs, u32 num_syncs,
@@ -2400,6 +2443,9 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
return PTR_ERR(vma);
op->map.vma = vma;
+ if (op->map.immediate || !xe_vm_in_fault_mode(vm))
+ xe_vma_ops_incr_pt_update_ops(vops,
+ op->tile_mask);
break;
}
case DRM_GPUVA_OP_REMAP:
@@ -2440,6 +2486,8 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
xe_vma_end(vma) -
xe_vma_start(old);
op->remap.start = xe_vma_end(vma);
+ } else {
+ xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask);
}
}
@@ -2473,13 +2521,16 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
op->remap.range -=
xe_vma_end(old) -
xe_vma_start(vma);
+ } else {
+ xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask);
}
}
+ xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask);
break;
}
case DRM_GPUVA_OP_UNMAP:
case DRM_GPUVA_OP_PREFETCH:
- /* Nothing to do */
+ xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask);
break;
default:
drm_warn(&vm->xe->drm, "NOT POSSIBLE");
@@ -3186,11 +3237,16 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
goto unwind_ops;
}
+ err = xe_vma_ops_alloc(&vops);
+ if (err)
+ goto unwind_ops;
+
err = vm_bind_ioctl_ops_execute(vm, &vops);
unwind_ops:
if (err && err != -ENODATA)
vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds);
+ xe_vma_ops_fini(&vops);
for (i = args->num_binds - 1; i >= 0; --i)
if (ops[i])
drm_gpuva_ops_free(&vm->gpuvm, ops[i]);
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index df85975b43fb..aa72330f824f 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -21,6 +21,7 @@ struct xe_bo;
struct xe_device;
struct xe_sync_entry;
struct xe_vm;
+struct xe_vm_pgtable_update_op;
#define TEST_VM_ASYNC_OPS_ERROR
#define FORCE_ASYNC_OP_ERROR BIT(31)
@@ -227,6 +228,13 @@ struct xe_vma_ops {
struct xe_sync_entry *syncs;
/** @num_syncs: number of syncs */
u32 num_syncs;
+ /** @pt_update_ops: page table update operations */
+ struct {
+ /** @ops: operations */
+ struct xe_vm_pgtable_update_op *ops;
+ /** @num_ops: number of operations */
+ u32 num_ops;
+ } pt_update_ops[XE_MAX_TILES_PER_DEVICE];
};
#define xe_vm_assert_held(vm) dma_resv_assert_held(&(vm)->resv)
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 16/27] drm/xe: Move setting last fence and sync wait to vm_bind_ioctl_ops_install_fences
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (14 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 15/27] drm/xe: Add xe_vm_pgtable_update_op to xe_vma_ops Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 17/27] drm/xe: Fix vma_is_valid to use tile argument Matthew Brost
` (14 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
This moves setting of the last fence and sync wait to a single location.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_vm.c | 15 ++++-----------
1 file changed, 4 insertions(+), 11 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 7ae12e4441b0..c2bca468a2d0 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -1922,11 +1922,6 @@ xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q,
fence = xe_exec_queue_last_fence_get(wait_exec_queue, vm);
}
- if (last_op)
- xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence);
- if (last_op && xe_vm_sync_mode(vm, q))
- dma_fence_wait(fence, true);
-
return fence;
}
@@ -1936,7 +1931,6 @@ xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma,
u32 num_syncs, bool first_op, bool last_op)
{
struct dma_fence *fence;
- struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
xe_vm_assert_held(vm);
xe_bo_assert_held(xe_vma_bo(vma));
@@ -1945,11 +1939,6 @@ xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma,
if (IS_ERR(fence))
return fence;
- if (last_op)
- xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence);
- if (last_op && xe_vm_sync_mode(vm, q))
- dma_fence_wait(fence, true);
-
return fence;
}
@@ -2887,6 +2876,7 @@ static void vm_bind_ioctl_ops_install_fences(struct xe_vm *vm,
struct xe_vma_ops *vops,
struct dma_fence *fence)
{
+ struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, vops->q);
struct xe_vma_op *op;
int i;
@@ -2899,6 +2889,9 @@ static void vm_bind_ioctl_ops_install_fences(struct xe_vm *vm,
}
for (i = 0; i < vops->num_syncs; i++)
xe_sync_entry_signal(vops->syncs + i, NULL, fence);
+ xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence);
+ if (xe_vm_sync_mode(vm, vops->q))
+ dma_fence_wait(fence, true);
dma_fence_put(fence);
}
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 17/27] drm/xe: Fix vma_is_valid to use tile argument
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (15 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 16/27] drm/xe: Move setting last fence and sync wait to vm_bind_ioctl_ops_install_fences Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 18/27] drm/xe: Adjust tile mask in operations create Matthew Brost
` (13 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
We convert gt to tile in vma_is_valid. Rather than so this just pass in
the tile.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_gt_pagefault.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c
index 27845a6b0b23..6125456373cb 100644
--- a/drivers/gpu/drm/xe/xe_gt_pagefault.c
+++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c
@@ -69,10 +69,10 @@ static bool access_is_atomic(enum access_type access_type)
return access_type == ACCESS_TYPE_ATOMIC;
}
-static bool vma_is_valid(struct xe_gt *gt, struct xe_vma *vma)
+static bool vma_is_valid(struct xe_tile *tile, struct xe_vma *vma)
{
- return BIT(gt_to_tile(gt)->id) & vma->tile_present &&
- !(BIT(gt->info.id) & vma->usm.tile_invalidated);
+ return BIT(tile->id) & vma->tile_present &&
+ !(BIT(tile->id) & vma->usm.tile_invalidated);
}
static bool vma_matches(struct xe_vma *vma, u64 page_addr)
@@ -174,7 +174,7 @@ static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
atomic = access_is_atomic(pf->access_type);
/* Check if VMA is valid */
- if (vma_is_valid(gt, vma) && !atomic)
+ if (vma_is_valid(gt_to_tile(gt), vma) && !atomic)
goto unlock_vm;
/* TODO: Validate fault */
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 18/27] drm/xe: Adjust tile mask in operations create
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (16 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 17/27] drm/xe: Fix vma_is_valid to use tile argument Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 19/27] drm/xe: Add xe_gt_tlb_invalidation_range and convert PT layer to use this Matthew Brost
` (12 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
This way the operations, not just created VMA, will have the correct
tile mask.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_vm.c | 25 ++++++++++++++++---------
1 file changed, 16 insertions(+), 9 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index c2bca468a2d0..071424a9d283 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -882,8 +882,6 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm,
u8 tile_mask)
{
struct xe_vma *vma;
- struct xe_tile *tile;
- u8 id;
xe_assert(vm->xe, start < end);
xe_assert(vm->xe, end < vm->size);
@@ -910,13 +908,7 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm,
vma->gpuva.flags |= XE_VMA_READ_ONLY;
if (is_null)
vma->gpuva.flags |= DRM_GPUVA_SPARSE;
-
- if (tile_mask) {
- vma->tile_mask = tile_mask;
- } else {
- for_each_tile(tile, vm->xe, id)
- vma->tile_mask |= 0x1 << id;
- }
+ vma->tile_mask = tile_mask;
if (GRAPHICS_VER(vm->xe) >= 20 || vm->xe->info.platform == XE_PVC)
vma->gpuva.flags |= XE_VMA_ATOMIC_PTE_BIT;
@@ -2163,6 +2155,19 @@ static void print_op(struct xe_device *xe, struct drm_gpuva_op *op)
}
#endif
+static u8 adj_tile_mask(struct xe_device *xe, u8 tile_mask)
+{
+ struct xe_tile *tile;
+ u8 id;
+
+ if (!tile_mask) {
+ for_each_tile(tile, xe, id)
+ tile_mask |= 0x1 << id;
+ }
+
+ return tile_mask;
+}
+
/*
* Create operations list from IOCTL arguments, setup operations fields so parse
* and commit steps are decoupled from IOCTL arguments. This step can fail.
@@ -2185,6 +2190,8 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo,
operation, (ULL)addr, (ULL)range,
(ULL)bo_offset_or_userptr);
+ tile_mask = adj_tile_mask(vm->xe, tile_mask);
+
switch (operation) {
case XE_VM_BIND_OP_MAP:
case XE_VM_BIND_OP_MAP_USERPTR:
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 19/27] drm/xe: Add xe_gt_tlb_invalidation_range and convert PT layer to use this
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (17 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 18/27] drm/xe: Adjust tile mask in operations create Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 20/27] drm/xe: s/xe_tile_migrate_engine/xe_tile_migrate_exec_queue Matthew Brost
` (11 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
xe_gt_tlb_invalidation_range accepts a start and end address rather than
a VMA. This will enable multiple VMAs to be invalidated in a single
invalidation. Update the PT layer to use this new function.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c | 60 +++++++++++++++------
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h | 3 ++
drivers/gpu/drm/xe/xe_pt.c | 25 ++++++---
3 files changed, 66 insertions(+), 22 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c
index b5c39c55e1fa..9a6408d3f0fe 100644
--- a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c
+++ b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c
@@ -227,11 +227,15 @@ int xe_gt_tlb_invalidation_guc(struct xe_gt *gt)
}
/**
- * xe_gt_tlb_invalidation_vma - Issue a TLB invalidation on this GT for a VMA
+ * xe_gt_tlb_invalidation_range - Issue a TLB invalidation on this GT for an
+ * address range
+ *
* @gt: graphics tile
* @fence: invalidation fence which will be signal on TLB invalidation
* completion, can be NULL
- * @vma: VMA to invalidate
+ * @start: start address
+ * @end: end address
+ * @asid: address space id
*
* Issue a range based TLB invalidation if supported, if not fallback to a full
* TLB invalidation. Completion of TLB is asynchronous and caller can either use
@@ -241,24 +245,22 @@ int xe_gt_tlb_invalidation_guc(struct xe_gt *gt)
* Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success,
* negative error code on error.
*/
-int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
- struct xe_gt_tlb_invalidation_fence *fence,
- struct xe_vma *vma)
+int xe_gt_tlb_invalidation_range(struct xe_gt *gt,
+ struct xe_gt_tlb_invalidation_fence *fence,
+ u64 start, u64 end, u32 asid)
{
struct xe_device *xe = gt_to_xe(gt);
#define MAX_TLB_INVALIDATION_LEN 7
u32 action[MAX_TLB_INVALIDATION_LEN];
int len = 0;
- xe_gt_assert(gt, vma);
-
action[len++] = XE_GUC_ACTION_TLB_INVALIDATION;
action[len++] = 0; /* seqno, replaced in send_tlb_invalidation */
if (!xe->info.has_range_tlb_invalidation) {
action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL);
} else {
- u64 start = xe_vma_start(vma);
- u64 length = xe_vma_size(vma);
+ u64 orig_start = start;
+ u64 length = end - start;
u64 align, end;
if (length < SZ_4K)
@@ -271,12 +273,12 @@ int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
* address mask covering the required range.
*/
align = roundup_pow_of_two(length);
- start = ALIGN_DOWN(xe_vma_start(vma), align);
- end = ALIGN(xe_vma_end(vma), align);
+ start = ALIGN_DOWN(start, align);
+ end = ALIGN(end, align);
length = align;
while (start + length < end) {
length <<= 1;
- start = ALIGN_DOWN(xe_vma_start(vma), length);
+ start = ALIGN_DOWN(orig_start, length);
}
/*
@@ -285,16 +287,17 @@ int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
*/
if (length >= SZ_2M) {
length = max_t(u64, SZ_16M, length);
- start = ALIGN_DOWN(xe_vma_start(vma), length);
+ start = ALIGN_DOWN(orig_start, length);
}
xe_gt_assert(gt, length >= SZ_4K);
xe_gt_assert(gt, is_power_of_2(length));
- xe_gt_assert(gt, !(length & GENMASK(ilog2(SZ_16M) - 1, ilog2(SZ_2M) + 1)));
+ xe_gt_assert(gt, !(length & GENMASK(ilog2(SZ_16M) - 1,
+ ilog2(SZ_2M) + 1)));
xe_gt_assert(gt, IS_ALIGNED(start, length));
action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_PAGE_SELECTIVE);
- action[len++] = xe_vma_vm(vma)->usm.asid;
+ action[len++] = asid;
action[len++] = lower_32_bits(start);
action[len++] = upper_32_bits(start);
action[len++] = ilog2(length) - ilog2(SZ_4K);
@@ -303,6 +306,33 @@ int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN);
return send_tlb_invalidation(>->uc.guc, fence, action, len);
+
+}
+
+/**
+ * xe_gt_tlb_invalidation_vma - Issue a TLB invalidation on this GT for a VMA
+ * @gt: graphics tile
+ * @fence: invalidation fence which will be signal on TLB invalidation
+ * completion, can be NULL
+ * @vma: VMA to invalidate
+ *
+ * Issue a range based TLB invalidation if supported, if not fallback to a full
+ * TLB invalidation. Completion of TLB is asynchronous and caller can either use
+ * the invalidation fence or seqno + xe_gt_tlb_invalidation_wait to wait for
+ * completion.
+ *
+ * Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success,
+ * negative error code on error.
+ */
+int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
+ struct xe_gt_tlb_invalidation_fence *fence,
+ struct xe_vma *vma)
+{
+ xe_gt_assert(gt, vma);
+
+ return xe_gt_tlb_invalidation_range(gt, fence, xe_vma_start(vma),
+ xe_vma_end(vma),
+ xe_vma_vm(vma)->usm.asid);
}
/**
diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h
index b333c1709397..5bb09885aa0f 100644
--- a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h
+++ b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h
@@ -20,6 +20,9 @@ int xe_gt_tlb_invalidation_guc(struct xe_gt *gt);
int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
struct xe_gt_tlb_invalidation_fence *fence,
struct xe_vma *vma);
+int xe_gt_tlb_invalidation_range(struct xe_gt *gt,
+ struct xe_gt_tlb_invalidation_fence *fence,
+ u64 start, u64 end, u32 asid);
int xe_gt_tlb_invalidation_wait(struct xe_gt *gt, int seqno);
int xe_guc_tlb_invalidation_done_handler(struct xe_guc *guc, u32 *msg, u32 len);
diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index f06ceaea98cf..0050ab6aad05 100644
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -1111,10 +1111,12 @@ static const struct xe_migrate_pt_update_ops userptr_bind_ops = {
struct invalidation_fence {
struct xe_gt_tlb_invalidation_fence base;
struct xe_gt *gt;
- struct xe_vma *vma;
struct dma_fence *fence;
struct dma_fence_cb cb;
struct work_struct work;
+ u64 start;
+ u64 end;
+ u32 asid;
};
static const char *
@@ -1157,13 +1159,14 @@ static void invalidation_fence_work_func(struct work_struct *w)
container_of(w, struct invalidation_fence, work);
trace_xe_gt_tlb_invalidation_fence_work_func(&ifence->base);
- xe_gt_tlb_invalidation_vma(ifence->gt, &ifence->base, ifence->vma);
+ xe_gt_tlb_invalidation_range(ifence->gt, &ifence->base, ifence->start,
+ ifence->end, ifence->asid);
}
static int invalidation_fence_init(struct xe_gt *gt,
struct invalidation_fence *ifence,
struct dma_fence *fence,
- struct xe_vma *vma)
+ u64 start, u64 end, u32 asid)
{
int ret;
@@ -1181,7 +1184,9 @@ static int invalidation_fence_init(struct xe_gt *gt,
dma_fence_get(&ifence->base.base); /* Ref for caller */
ifence->fence = fence;
ifence->gt = gt;
- ifence->vma = vma;
+ ifence->start = start;
+ ifence->end = end;
+ ifence->asid = asid;
INIT_WORK(&ifence->work, invalidation_fence_work_func);
ret = dma_fence_add_callback(fence, &ifence->cb, invalidation_fence_cb);
@@ -1323,8 +1328,11 @@ __xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue
/* TLB invalidation must be done before signaling rebind */
if (ifence) {
- int err = invalidation_fence_init(tile->primary_gt, ifence, fence,
- vma);
+ int err = invalidation_fence_init(tile->primary_gt,
+ ifence, fence,
+ xe_vma_start(vma),
+ xe_vma_end(vma),
+ xe_vma_vm(vma)->usm.asid);
if (err) {
dma_fence_put(fence);
kfree(ifence);
@@ -1662,7 +1670,10 @@ __xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queu
dma_fence_wait(fence, false);
/* TLB invalidation must be done before signaling unbind */
- err = invalidation_fence_init(tile->primary_gt, ifence, fence, vma);
+ err = invalidation_fence_init(tile->primary_gt, ifence, fence,
+ xe_vma_start(vma),
+ xe_vma_end(vma),
+ xe_vma_vm(vma)->usm.asid);
if (err) {
dma_fence_put(fence);
kfree(ifence);
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 20/27] drm/xe: s/xe_tile_migrate_engine/xe_tile_migrate_exec_queue
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (18 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 19/27] drm/xe: Add xe_gt_tlb_invalidation_range and convert PT layer to use this Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 21/27] drm/xe: Convert multiple bind ops into single job Matthew Brost
` (10 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
xe_engine is now xe_exec_queue, adjust this function's name to reflect.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_gt_pagefault.c | 2 +-
drivers/gpu/drm/xe/xe_migrate.c | 8 ++++----
drivers/gpu/drm/xe/xe_migrate.h | 3 ++-
3 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c
index 6125456373cb..fd9abe4f9c3c 100644
--- a/drivers/gpu/drm/xe/xe_gt_pagefault.c
+++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c
@@ -205,7 +205,7 @@ static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
trace_xe_vma_pf_bind(vma);
xe_vm_populate_dummy_rebind(vm, vma);
vm->dummy_ops.op.tile_mask = BIT(tile->id);
- vm->dummy_ops.op.q = xe_tile_migrate_engine(tile);
+ vm->dummy_ops.op.q = xe_tile_migrate_exec_queue(tile);
fence = xe_vm_ops_execute(vm, &vm->dummy_ops.vops);
if (IS_ERR(fence)) {
ret = PTR_ERR(fence);
diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c
index e6a31f90ebdb..33c4f8caa38f 100644
--- a/drivers/gpu/drm/xe/xe_migrate.c
+++ b/drivers/gpu/drm/xe/xe_migrate.c
@@ -74,15 +74,15 @@ struct xe_migrate {
#define NUM_PT_PER_BLIT (MAX_PREEMPTDISABLE_TRANSFER / SZ_2M)
/**
- * xe_tile_migrate_engine() - Get this tile's migrate engine.
+ * xe_tile_migrate_exec_queue() - Get this tile's migrate exec queue.
* @tile: The tile.
*
- * Returns the default migrate engine of this tile.
+ * Returns the default migrate exec queue of this tile.
* TODO: Perhaps this function is slightly misplaced, and even unneeded?
*
- * Return: The default migrate engine
+ * Return: The default migrate exec queue
*/
-struct xe_exec_queue *xe_tile_migrate_engine(struct xe_tile *tile)
+struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile)
{
return tile->migrate->q;
}
diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h
index c729241776ad..8b8eedce8499 100644
--- a/drivers/gpu/drm/xe/xe_migrate.h
+++ b/drivers/gpu/drm/xe/xe_migrate.h
@@ -105,5 +105,6 @@ xe_migrate_update_pgtables(struct xe_migrate *m,
void xe_migrate_wait(struct xe_migrate *m);
-struct xe_exec_queue *xe_tile_migrate_engine(struct xe_tile *tile);
+struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile);
+
#endif
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 21/27] drm/xe: Convert multiple bind ops into single job
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (19 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 20/27] drm/xe: s/xe_tile_migrate_engine/xe_tile_migrate_exec_queue Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 22/27] drm/xe: Update clear / populate arguments Matthew Brost
` (9 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
This aligns with the uAPI of an array of binds or single bind that
results in multiple GPUVA ops to be considered a single atomic
operations.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_bo_types.h | 2 +
drivers/gpu/drm/xe/xe_exec.c | 25 +-
drivers/gpu/drm/xe/xe_gt_pagefault.c | 6 +-
drivers/gpu/drm/xe/xe_migrate.c | 287 ++++----
drivers/gpu/drm/xe/xe_migrate.h | 32 +-
drivers/gpu/drm/xe/xe_pt.c | 957 ++++++++++++++++-----------
drivers/gpu/drm/xe/xe_pt.h | 7 +
drivers/gpu/drm/xe/xe_pt_types.h | 34 +
drivers/gpu/drm/xe/xe_vm.c | 516 ++++-----------
drivers/gpu/drm/xe/xe_vm.h | 5 +-
drivers/gpu/drm/xe/xe_vm_types.h | 29 +-
11 files changed, 899 insertions(+), 1001 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h
index 051fe990c133..1222b8bbbc6b 100644
--- a/drivers/gpu/drm/xe/xe_bo_types.h
+++ b/drivers/gpu/drm/xe/xe_bo_types.h
@@ -74,6 +74,8 @@ struct xe_bo {
} props;
/** @freed: List node for delayed put. */
struct llist_node freed;
+ /** @update_index: Update index if PT BO */
+ int update_index;
/** @created: Whether the bo has passed initial creation */
bool created;
};
diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c
index 80ee6d8fcf68..a1771e59eccf 100644
--- a/drivers/gpu/drm/xe/xe_exec.c
+++ b/drivers/gpu/drm/xe/xe_exec.c
@@ -275,30 +275,9 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
err = PTR_ERR(rebind_fence);
goto err_put_job;
}
+ dma_fence_put(rebind_fence);
- /*
- * We store the rebind_fence in the VM so subsequent execs don't get
- * scheduled before the rebinds of userptrs / evicted BOs is complete.
- */
- if (rebind_fence) {
- dma_fence_put(vm->rebind_fence);
- vm->rebind_fence = rebind_fence;
- }
- if (vm->rebind_fence) {
- if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
- &vm->rebind_fence->flags)) {
- dma_fence_put(vm->rebind_fence);
- vm->rebind_fence = NULL;
- } else {
- dma_fence_get(vm->rebind_fence);
- err = drm_sched_job_add_dependency(&job->drm,
- vm->rebind_fence);
- if (err)
- goto err_put_job;
- }
- }
-
- /* Wait behind munmap style rebinds */
+ /* Wait for rebinds */
if (!xe_vm_no_dma_fences(vm)) {
err = drm_sched_job_add_resv_dependencies(&job->drm,
&vm->resv,
diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c
index fd9abe4f9c3c..d383b5a3eaa2 100644
--- a/drivers/gpu/drm/xe/xe_gt_pagefault.c
+++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c
@@ -203,9 +203,9 @@ static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
/* Bind VMA only to the GT that has faulted */
trace_xe_vma_pf_bind(vma);
- xe_vm_populate_dummy_rebind(vm, vma);
- vm->dummy_ops.op.tile_mask = BIT(tile->id);
- vm->dummy_ops.op.q = xe_tile_migrate_exec_queue(tile);
+ xe_vm_populate_dummy_rebind(vm, vma, BIT(tile->id));
+ vm->dummy_ops.vops.pt_update_ops[tile->id].q =
+ xe_tile_migrate_exec_queue(tile);
fence = xe_vm_ops_execute(vm, &vm->dummy_ops.vops);
if (IS_ERR(fence)) {
ret = PTR_ERR(fence);
diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c
index 33c4f8caa38f..3a01c6de1ec6 100644
--- a/drivers/gpu/drm/xe/xe_migrate.c
+++ b/drivers/gpu/drm/xe/xe_migrate.c
@@ -1053,6 +1053,7 @@ struct dma_fence *xe_migrate_clear(struct xe_migrate *m,
}
static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs,
+ const struct xe_vm_pgtable_update_op *pt_op,
const struct xe_vm_pgtable_update *update,
struct xe_migrate_pt_update *pt_update)
{
@@ -1087,8 +1088,12 @@ static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs,
bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk);
bb->cs[bb->len++] = lower_32_bits(addr);
bb->cs[bb->len++] = upper_32_bits(addr);
- ops->populate(pt_update, tile, NULL, bb->cs + bb->len, ofs, chunk,
- update);
+ if (pt_op->bind)
+ ops->populate(pt_update, tile, NULL, bb->cs + bb->len,
+ ofs, chunk, update);
+ else
+ ops->clear(pt_update, tile, NULL, bb->cs + bb->len,
+ ofs, chunk, update);
bb->len += chunk * 2;
ofs += chunk;
@@ -1113,104 +1118,58 @@ struct migrate_test_params {
static struct dma_fence *
xe_migrate_update_pgtables_cpu(struct xe_migrate *m,
- struct xe_vm *vm, struct xe_bo *bo,
- const struct xe_vm_pgtable_update *updates,
- u32 num_updates, bool wait_vm,
struct xe_migrate_pt_update *pt_update)
{
XE_TEST_DECLARE(struct migrate_test_params *test =
to_migrate_test_params
(xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));)
const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
- struct dma_fence *fence;
+ struct xe_vm *vm = pt_update->vops->vm;
+ struct xe_vm_pgtable_update_ops *pt_update_ops =
+ &pt_update->vops->pt_update_ops[pt_update->tile_id];
int err;
- u32 i;
+ u32 j, i;
if (XE_TEST_ONLY(test && test->force_gpu))
return ERR_PTR(-ETIME);
- if (bo && !dma_resv_test_signaled(bo->ttm.base.resv,
- DMA_RESV_USAGE_KERNEL))
- return ERR_PTR(-ETIME);
-
- if (wait_vm && !dma_resv_test_signaled(&vm->resv,
- DMA_RESV_USAGE_BOOKKEEP))
- return ERR_PTR(-ETIME);
-
if (ops->pre_commit) {
pt_update->job = NULL;
err = ops->pre_commit(pt_update);
if (err)
return ERR_PTR(err);
}
- for (i = 0; i < num_updates; i++) {
- const struct xe_vm_pgtable_update *update = &updates[i];
- ops->populate(pt_update, m->tile, &update->pt_bo->vmap, NULL,
- update->ofs, update->qwords, update);
- }
-
- if (vm) {
- trace_xe_vm_cpu_bind(vm);
- xe_device_wmb(vm->xe);
+ for (j = 0; j < pt_update_ops->num_ops; ++j) {
+ const struct xe_vm_pgtable_update_op *pt_op =
+ &pt_update_ops->ops[j];
+
+ for (i = 0; i < pt_op->num_entries; i++) {
+ const struct xe_vm_pgtable_update *update =
+ &pt_op->entries[i];
+
+ if (pt_op->bind)
+ ops->populate(pt_update, m->tile,
+ &update->pt_bo->vmap, NULL,
+ update->ofs, update->qwords,
+ update);
+ else
+ ops->clear(pt_update, m->tile,
+ &update->pt_bo->vmap, NULL,
+ update->ofs, update->qwords, update);
+ }
}
- fence = dma_fence_get_stub();
-
- return fence;
-}
-
-static bool no_in_syncs(struct xe_sync_entry *syncs, u32 num_syncs)
-{
- int i;
-
- for (i = 0; i < num_syncs; i++) {
- struct dma_fence *fence = syncs[i].fence;
+ trace_xe_vm_cpu_bind(vm);
+ xe_device_wmb(vm->xe);
- if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
- &fence->flags))
- return false;
- }
-
- return true;
+ return dma_fence_get_stub();
}
-/**
- * xe_migrate_update_pgtables() - Pipelined page-table update
- * @m: The migrate context.
- * @vm: The vm we'll be updating.
- * @bo: The bo whose dma-resv we will await before updating, or NULL if userptr.
- * @q: The exec queue to be used for the update or NULL if the default
- * migration engine is to be used.
- * @updates: An array of update descriptors.
- * @num_updates: Number of descriptors in @updates.
- * @syncs: Array of xe_sync_entry to await before updating. Note that waits
- * will block the engine timeline.
- * @num_syncs: Number of entries in @syncs.
- * @pt_update: Pointer to a struct xe_migrate_pt_update, which contains
- * pointers to callback functions and, if subclassed, private arguments to
- * those.
- *
- * Perform a pipelined page-table update. The update descriptors are typically
- * built under the same lock critical section as a call to this function. If
- * using the default engine for the updates, they will be performed in the
- * order they grab the job_mutex. If different engines are used, external
- * synchronization is needed for overlapping updates to maintain page-table
- * consistency. Note that the meaing of "overlapping" is that the updates
- * touch the same page-table, which might be a higher-level page-directory.
- * If no pipelining is needed, then updates may be performed by the cpu.
- *
- * Return: A dma_fence that, when signaled, indicates the update completion.
- */
-struct dma_fence *
-xe_migrate_update_pgtables(struct xe_migrate *m,
- struct xe_vm *vm,
- struct xe_bo *bo,
- struct xe_exec_queue *q,
- const struct xe_vm_pgtable_update *updates,
- u32 num_updates,
- struct xe_sync_entry *syncs, u32 num_syncs,
- struct xe_migrate_pt_update *pt_update)
+static struct dma_fence *
+__xe_migrate_update_pgtables(struct xe_migrate *m,
+ struct xe_migrate_pt_update *pt_update,
+ struct xe_vm_pgtable_update_ops *pt_update_ops)
{
const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
struct xe_tile *tile = m->tile;
@@ -1219,59 +1178,45 @@ xe_migrate_update_pgtables(struct xe_migrate *m,
struct xe_sched_job *job;
struct dma_fence *fence;
struct drm_suballoc *sa_bo = NULL;
- struct xe_vma *vma = pt_update->vma;
struct xe_bb *bb;
- u32 i, batch_size, ppgtt_ofs, update_idx, page_ofs = 0;
+ u32 i, j, batch_size = 0, ppgtt_ofs, update_idx, page_ofs = 0;
+ u32 num_updates = 0, current_update = 0;
u64 addr;
int err = 0;
- bool usm = !q && xe->info.supports_usm;
- bool first_munmap_rebind = vma &&
- vma->gpuva.flags & XE_VMA_FIRST_REBIND;
- struct xe_exec_queue *q_override = !q ? m->q : q;
- u16 pat_index = xe->pat.idx[XE_CACHE_WB];
+ bool is_migrate = pt_update_ops->q == m->q;
+ bool usm = is_migrate && xe->info.supports_usm;
+
+ for (i = 0; i < pt_update_ops->num_ops; ++i) {
+ struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
+ struct xe_vm_pgtable_update *updates = pt_op->entries;
+
+ num_updates += pt_op->num_entries;
+ for (j = 0; j < pt_op->num_entries; ++j) {
+ u32 num_cmds = DIV_ROUND_UP(updates[j].qwords, 0x1ff);
- /* Use the CPU if no in syncs and engine is idle */
- if (no_in_syncs(syncs, num_syncs) && xe_exec_queue_is_idle(q_override)) {
- fence = xe_migrate_update_pgtables_cpu(m, vm, bo, updates,
- num_updates,
- first_munmap_rebind,
- pt_update);
- if (!IS_ERR(fence) || fence == ERR_PTR(-EAGAIN))
- return fence;
+ /* align noop + MI_STORE_DATA_IMM cmd prefix */
+ batch_size += 4 * num_cmds + updates[j].qwords * 2;
+ }
}
/* fixed + PTE entries */
if (IS_DGFX(xe))
- batch_size = 2;
+ batch_size += 2;
else
- batch_size = 6 + num_updates * 2;
-
- for (i = 0; i < num_updates; i++) {
- u32 num_cmds = DIV_ROUND_UP(updates[i].qwords, 0x1ff);
-
- /* align noop + MI_STORE_DATA_IMM cmd prefix */
- batch_size += 4 * num_cmds + updates[i].qwords * 2;
- }
-
- /*
- * XXX: Create temp bo to copy from, if batch_size becomes too big?
- *
- * Worst case: Sum(2 * (each lower level page size) + (top level page size))
- * Should be reasonably bound..
- */
- xe_tile_assert(tile, batch_size < SZ_128K);
+ batch_size += 6 + num_updates * 2;
- bb = xe_bb_new(gt, batch_size, !q && xe->info.supports_usm);
+ bb = xe_bb_new(gt, batch_size, usm);
if (IS_ERR(bb))
return ERR_CAST(bb);
/* For sysmem PTE's, need to map them in our hole.. */
if (!IS_DGFX(xe)) {
ppgtt_ofs = NUM_KERNEL_PDE - 1;
- if (q) {
- xe_tile_assert(tile, num_updates <= NUM_VMUSA_WRITES_PER_UNIT);
+ if (!is_migrate) {
+ u32 num_units = DIV_ROUND_UP(num_updates,
+ NUM_VMUSA_WRITES_PER_UNIT);
- sa_bo = drm_suballoc_new(&m->vm_update_sa, 1,
+ sa_bo = drm_suballoc_new(&m->vm_update_sa, num_units,
GFP_KERNEL, true, 0);
if (IS_ERR(sa_bo)) {
err = PTR_ERR(sa_bo);
@@ -1291,14 +1236,26 @@ xe_migrate_update_pgtables(struct xe_migrate *m,
bb->cs[bb->len++] = ppgtt_ofs * XE_PAGE_SIZE + page_ofs;
bb->cs[bb->len++] = 0; /* upper_32_bits */
- for (i = 0; i < num_updates; i++) {
- struct xe_bo *pt_bo = updates[i].pt_bo;
+ for (i = 0; i < pt_update_ops->num_ops; ++i) {
+ struct xe_vm_pgtable_update_op *pt_op =
+ &pt_update_ops->ops[i];
+ struct xe_vm_pgtable_update *updates = pt_op->entries;
- xe_tile_assert(tile, pt_bo->size == SZ_4K);
+ for (j = 0; j < pt_op->num_entries; ++j, ++current_update) {
+ struct xe_vm *vm = pt_update->vops->vm;
+ struct xe_bo *pt_bo = updates[j].pt_bo;
- addr = vm->pt_ops->pte_encode_bo(pt_bo, 0, pat_index, 0);
- bb->cs[bb->len++] = lower_32_bits(addr);
- bb->cs[bb->len++] = upper_32_bits(addr);
+ xe_tile_assert(tile, pt_bo->size == SZ_4K);
+
+ /* Map a PT at most once */
+ if (pt_bo->update_index < 0)
+ pt_bo->update_index = current_update;
+
+ addr = vm->pt_ops->pte_encode_bo(pt_bo, 0,
+ XE_CACHE_WB, 0);
+ bb->cs[bb->len++] = lower_32_bits(addr);
+ bb->cs[bb->len++] = upper_32_bits(addr);
+ }
}
bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
@@ -1306,22 +1263,39 @@ xe_migrate_update_pgtables(struct xe_migrate *m,
addr = xe_migrate_vm_addr(ppgtt_ofs, 0) +
(page_ofs / sizeof(u64)) * XE_PAGE_SIZE;
- for (i = 0; i < num_updates; i++)
- write_pgtable(tile, bb, addr + i * XE_PAGE_SIZE,
- &updates[i], pt_update);
+ for (i = 0; i < pt_update_ops->num_ops; ++i) {
+ struct xe_vm_pgtable_update_op *pt_op =
+ &pt_update_ops->ops[i];
+ struct xe_vm_pgtable_update *updates = pt_op->entries;
+
+ for (j = 0; j < pt_op->num_entries; ++j) {
+ struct xe_bo *pt_bo = updates[j].pt_bo;
+
+ write_pgtable(tile, bb, addr +
+ pt_bo->update_index * XE_PAGE_SIZE,
+ pt_op, &updates[j], pt_update);
+ }
+ }
} else {
/* phys pages, no preamble required */
bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
update_idx = bb->len;
- for (i = 0; i < num_updates; i++)
- write_pgtable(tile, bb, 0, &updates[i], pt_update);
+ for (i = 0; i < pt_update_ops->num_ops; ++i) {
+ struct xe_vm_pgtable_update_op *pt_op =
+ &pt_update_ops->ops[i];
+ struct xe_vm_pgtable_update *updates = pt_op->entries;
+
+ for (j = 0; j < pt_op->num_entries; ++j)
+ write_pgtable(tile, bb, 0, pt_op, &updates[j],
+ pt_update);
+ }
}
- if (!q)
+ if (is_migrate)
mutex_lock(&m->job_mutex);
- job = xe_bb_create_migration_job(q ?: m->q, bb,
+ job = xe_bb_create_migration_job(pt_update_ops->q, bb,
xe_migrate_batch_base(m, usm),
update_idx);
if (IS_ERR(job)) {
@@ -1329,31 +1303,6 @@ xe_migrate_update_pgtables(struct xe_migrate *m,
goto err_bb;
}
- /* Wait on BO move */
- if (bo) {
- err = job_add_deps(job, bo->ttm.base.resv,
- DMA_RESV_USAGE_KERNEL);
- if (err)
- goto err_job;
- }
-
- /*
- * Munmap style VM unbind, need to wait for all jobs to be complete /
- * trigger preempts before moving forward
- */
- if (first_munmap_rebind) {
- err = job_add_deps(job, &vm->resv,
- DMA_RESV_USAGE_BOOKKEEP);
- if (err)
- goto err_job;
- }
-
- for (i = 0; !err && i < num_syncs; i++)
- err = xe_sync_entry_add_deps(&syncs[i], job);
-
- if (err)
- goto err_job;
-
if (ops->pre_commit) {
pt_update->job = job;
err = ops->pre_commit(pt_update);
@@ -1364,7 +1313,7 @@ xe_migrate_update_pgtables(struct xe_migrate *m,
fence = dma_fence_get(&job->drm.s_fence->finished);
xe_sched_job_push(job);
- if (!q)
+ if (is_migrate)
mutex_unlock(&m->job_mutex);
xe_bb_free(bb, fence);
@@ -1375,7 +1324,7 @@ xe_migrate_update_pgtables(struct xe_migrate *m,
err_job:
xe_sched_job_put(job);
err_bb:
- if (!q)
+ if (is_migrate)
mutex_unlock(&m->job_mutex);
xe_bb_free(bb, NULL);
err:
@@ -1383,6 +1332,38 @@ xe_migrate_update_pgtables(struct xe_migrate *m,
return ERR_PTR(err);
}
+/**
+ * xe_migrate_update_pgtables() - Pipelined page-table update
+ * @m: The migrate context.
+ * @pt_update: PT update arguments
+ *
+ * Perform a pipelined page-table update. The update descriptors are typically
+ * built under the same lock critical section as a call to this function. If
+ * using the default engine for the updates, they will be performed in the
+ * order they grab the job_mutex. If different engines are used, external
+ * synchronization is needed for overlapping updates to maintain page-table
+ * consistency. Note that the meaing of "overlapping" is that the updates
+ * touch the same page-table, which might be a higher-level page-directory.
+ * If no pipelining is needed, then updates may be performed by the cpu.
+ *
+ * Return: A dma_fence that, when signaled, indicates the update completion.
+ */
+struct dma_fence *
+xe_migrate_update_pgtables(struct xe_migrate *m,
+ struct xe_migrate_pt_update *pt_update)
+
+{
+ struct xe_vm_pgtable_update_ops *pt_update_ops =
+ &pt_update->vops->pt_update_ops[pt_update->tile_id];
+ struct dma_fence *fence;
+
+ fence = xe_migrate_update_pgtables_cpu(m, pt_update);
+ if (!IS_ERR(fence))
+ return fence;
+
+ return __xe_migrate_update_pgtables(m, pt_update, pt_update_ops);
+}
+
/**
* xe_migrate_wait() - Complete all operations using the xe_migrate context
* @m: Migrate context to wait for.
diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h
index 8b8eedce8499..7fbc21bd265e 100644
--- a/drivers/gpu/drm/xe/xe_migrate.h
+++ b/drivers/gpu/drm/xe/xe_migrate.h
@@ -47,6 +47,24 @@ struct xe_migrate_pt_update_ops {
struct xe_tile *tile, struct iosys_map *map,
void *pos, u32 ofs, u32 num_qwords,
const struct xe_vm_pgtable_update *update);
+ /**
+ * @clear: Clear a command buffer or page-table with ptes.
+ * @pt_update: Embeddable callback argument.
+ * @tile: The tile for the current operation.
+ * @map: struct iosys_map into the memory to be populated.
+ * @pos: If @map is NULL, map into the memory to be populated.
+ * @ofs: qword offset into @map, unused if @map is NULL.
+ * @num_qwords: Number of qwords to write.
+ * @update: Information about the PTEs to be inserted.
+ *
+ * This interface is intended to be used as a callback into the
+ * page-table system to populate command buffers or shared
+ * page-tables with PTEs.
+ */
+ void (*clear)(struct xe_migrate_pt_update *pt_update,
+ struct xe_tile *tile, struct iosys_map *map,
+ void *pos, u32 ofs, u32 num_qwords,
+ const struct xe_vm_pgtable_update *update);
/**
* @pre_commit: Callback to be called just before arming the
@@ -67,14 +85,10 @@ struct xe_migrate_pt_update_ops {
struct xe_migrate_pt_update {
/** @ops: Pointer to the struct xe_migrate_pt_update_ops callbacks */
const struct xe_migrate_pt_update_ops *ops;
- /** @vma: The vma we're updating the pagetable for. */
- struct xe_vma *vma;
+ /** @vops: VMA operations */
+ struct xe_vma_ops *vops;
/** @job: The job if a GPU page-table update. NULL otherwise */
struct xe_sched_job *job;
- /** @start: Start of update for the range fence */
- u64 start;
- /** @last: Last of update for the range fence */
- u64 last;
/** @tile_id: Tile ID of the update */
u8 tile_id;
};
@@ -95,12 +109,6 @@ struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m);
struct dma_fence *
xe_migrate_update_pgtables(struct xe_migrate *m,
- struct xe_vm *vm,
- struct xe_bo *bo,
- struct xe_exec_queue *q,
- const struct xe_vm_pgtable_update *updates,
- u32 num_updates,
- struct xe_sync_entry *syncs, u32 num_syncs,
struct xe_migrate_pt_update *pt_update);
void xe_migrate_wait(struct xe_migrate *m);
diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index 0050ab6aad05..73199b34820f 100644
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -8,12 +8,14 @@
#include "xe_bo.h"
#include "xe_device.h"
#include "xe_drm_client.h"
+#include "xe_exec_queue.h"
#include "xe_gt.h"
#include "xe_gt_tlb_invalidation.h"
#include "xe_migrate.h"
#include "xe_pt_types.h"
#include "xe_pt_walk.h"
#include "xe_res_cursor.h"
+#include "xe_sync.h"
#include "xe_trace.h"
#include "xe_ttm_stolen_mgr.h"
#include "xe_vm.h"
@@ -364,6 +366,7 @@ xe_pt_new_shared(struct xe_walk_update *wupd, struct xe_pt *parent,
entry->pt = parent;
entry->flags = 0;
entry->qwords = 0;
+ entry->pt_bo->update_index = -1;
if (alloc_entries) {
entry->pt_entries = kmalloc_array(XE_PDES,
@@ -868,9 +871,7 @@ static void xe_pt_commit_locks_assert(struct xe_vma *vma)
lockdep_assert_held(&vm->lock);
- if (xe_vma_is_userptr(vma))
- lockdep_assert_held_read(&vm->userptr.notifier_lock);
- else if (!xe_vma_is_null(vma))
+ if (!xe_vma_is_userptr(vma) && !xe_vma_is_null(vma))
dma_resv_assert_held(xe_vma_bo(vma)->ttm.base.resv);
dma_resv_assert_held(&vm->resv);
@@ -892,10 +893,8 @@ static void xe_pt_commit_bind(struct xe_vma *vma,
if (!rebind)
pt->num_live += entries[i].qwords;
- if (!pt->level) {
- kfree(entries[i].pt_entries);
+ if (!pt->level)
continue;
- }
pt_dir = as_xe_pt_dir(pt);
for (j = 0; j < entries[i].qwords; j++) {
@@ -908,10 +907,18 @@ static void xe_pt_commit_bind(struct xe_vma *vma,
pt_dir->dir.entries[j_] = &newpte->base;
}
- kfree(entries[i].pt_entries);
}
}
+static void xe_pt_free_bind(struct xe_vm_pgtable_update *entries,
+ u32 num_entries)
+{
+ u32 i;
+
+ for (i = 0; i < num_entries; i++)
+ kfree(entries[i].pt_entries);
+}
+
static int
xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma,
struct xe_vm_pgtable_update *entries, u32 *num_entries)
@@ -956,66 +963,122 @@ static void xe_vm_dbg_print_entries(struct xe_device *xe,
{}
#endif
-#ifdef CONFIG_DRM_XE_USERPTR_INVAL_INJECT
+static int job_add_deps(struct xe_sched_job *job, struct dma_resv *resv,
+ enum dma_resv_usage usage)
+{
+ return drm_sched_job_add_resv_dependencies(&job->drm, resv, usage);
+}
-static int xe_pt_userptr_inject_eagain(struct xe_vma *vma)
+static bool no_in_syncs(struct xe_sync_entry *syncs, u32 num_syncs)
{
- u32 divisor = vma->userptr.divisor ? vma->userptr.divisor : 2;
- static u32 count;
+ int i;
- if (count++ % divisor == divisor - 1) {
- struct xe_vm *vm = xe_vma_vm(vma);
+ for (i = 0; i < num_syncs; i++) {
+ struct dma_fence *fence = syncs[i].fence;
- vma->userptr.divisor = divisor << 1;
- spin_lock(&vm->userptr.invalidated_lock);
- list_move_tail(&vma->userptr.invalidate_link,
- &vm->userptr.invalidated);
- spin_unlock(&vm->userptr.invalidated_lock);
- return true;
+ if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
+ &fence->flags))
+ return false;
}
- return false;
+ return true;
}
-#else
-
-static bool xe_pt_userptr_inject_eagain(struct xe_vma *vma)
+static int vma_add_deps(struct xe_vma *vma, struct xe_sched_job *job)
{
- return false;
+ struct xe_bo *bo = xe_vma_bo(vma);
+
+ xe_bo_assert_held(bo);
+
+ if (bo && !bo->vm) {
+ if (!job) {
+ if (!dma_resv_test_signaled(bo->ttm.base.resv,
+ DMA_RESV_USAGE_KERNEL))
+ return -ETIME;
+ } else {
+ return job_add_deps(job, bo->ttm.base.resv,
+ DMA_RESV_USAGE_KERNEL);
+ }
+ }
+
+ return 0;
}
-#endif
+static int op_add_deps(struct xe_vm *vm, struct xe_vma_op *op,
+ struct xe_sched_job *job)
+{
+ int err = 0;
-/**
- * struct xe_pt_migrate_pt_update - Callback argument for pre-commit callbacks
- * @base: Base we derive from.
- * @bind: Whether this is a bind or an unbind operation. A bind operation
- * makes the pre-commit callback error with -EAGAIN if it detects a
- * pending invalidation.
- * @locked: Whether the pre-commit callback locked the userptr notifier lock
- * and it needs unlocking.
- */
-struct xe_pt_migrate_pt_update {
- struct xe_migrate_pt_update base;
- bool bind;
- bool locked;
-};
+ switch (op->base.op) {
+ case DRM_GPUVA_OP_MAP:
+ if (!op->map.immediate && xe_vm_in_fault_mode(vm))
+ break;
+
+ err = vma_add_deps(op->map.vma, job);
+ break;
+ case DRM_GPUVA_OP_REMAP:
+ if (op->remap.prev)
+ err = vma_add_deps(op->remap.prev, job);
+ if (!err && op->remap.next)
+ err = vma_add_deps(op->remap.next, job);
+ break;
+ case DRM_GPUVA_OP_UNMAP:
+ break;
+ case DRM_GPUVA_OP_PREFETCH:
+ err = vma_add_deps(gpuva_to_vma(op->base.prefetch.va), job);
+ break;
+ default:
+ drm_warn(&vm->xe->drm, "NOT POSSIBLE");
+ }
+
+ return err;
+}
-/*
- * This function adds the needed dependencies to a page-table update job
- * to make sure racing jobs for separate bind engines don't race writing
- * to the same page-table range, wreaking havoc. Initially use a single
- * fence for the entire VM. An optimization would use smaller granularity.
- */
static int xe_pt_vm_dependencies(struct xe_sched_job *job,
- struct xe_range_fence_tree *rftree,
- u64 start, u64 last)
+ struct xe_vm *vm,
+ struct xe_vma_ops *vops,
+ struct xe_vm_pgtable_update_ops *pt_update_ops,
+ struct xe_range_fence_tree *rftree)
{
struct xe_range_fence *rtfence;
struct dma_fence *fence;
- int err;
+ struct xe_vma_op *op;
+ int err = 0, i;
+
+ xe_vm_assert_held(vm);
+
+ if (!job && !no_in_syncs(vops->syncs, vops->num_syncs))
+ return -ETIME;
+
+ if (!job && !xe_exec_queue_is_idle(pt_update_ops->q))
+ return -ETIME;
+
+ if (pt_update_ops->wait_vm_bookkeep) {
+ if (!job) {
+ if (!dma_resv_test_signaled(&vm->resv,
+ DMA_RESV_USAGE_BOOKKEEP))
+ return -ETIME;
+ } else {
+ err = job_add_deps(job, &vm->resv,
+ DMA_RESV_USAGE_BOOKKEEP);
+ if (err)
+ return err;
+ }
+ } else if (pt_update_ops->wait_vm_kernel) {
+ if (!job) {
+ if (!dma_resv_test_signaled(&vm->resv,
+ DMA_RESV_USAGE_KERNEL))
+ return -ETIME;
+ } else {
+ err = job_add_deps(job, &vm->resv,
+ DMA_RESV_USAGE_KERNEL);
+ if (err)
+ return err;
+ }
+ }
- rtfence = xe_range_fence_tree_first(rftree, start, last);
+ rtfence = xe_range_fence_tree_first(rftree, pt_update_ops->start,
+ pt_update_ops->last);
while (rtfence) {
fence = rtfence->fence;
@@ -1033,80 +1096,124 @@ static int xe_pt_vm_dependencies(struct xe_sched_job *job,
return err;
}
- rtfence = xe_range_fence_tree_next(rtfence, start, last);
+ rtfence = xe_range_fence_tree_next(rtfence,
+ pt_update_ops->start,
+ pt_update_ops->last);
}
- return 0;
+ list_for_each_entry(op, &vops->list, link) {
+ err = op_add_deps(vm, op, job);
+ if (err)
+ return err;
+ }
+
+ for (i = 0; job && !err && i < vops->num_syncs; i++)
+ err = xe_sync_entry_add_deps(&vops->syncs[i], job);
+
+ return err;
}
static int xe_pt_pre_commit(struct xe_migrate_pt_update *pt_update)
{
- struct xe_range_fence_tree *rftree =
- &xe_vma_vm(pt_update->vma)->rftree[pt_update->tile_id];
+ struct xe_vma_ops *vops = pt_update->vops;
+ struct xe_vm *vm = vops->vm;
+ struct xe_range_fence_tree *rftree = &vm->rftree[pt_update->tile_id];
+ struct xe_vm_pgtable_update_ops *pt_update_ops =
+ &vops->pt_update_ops[pt_update->tile_id];
+
+ return xe_pt_vm_dependencies(pt_update->job, vm, pt_update->vops,
+ pt_update_ops, rftree);
+}
- return xe_pt_vm_dependencies(pt_update->job, rftree,
- pt_update->start, pt_update->last);
+#ifdef CONFIG_DRM_XE_USERPTR_INVAL_INJECT
+
+static int xe_pt_userptr_inject_eagain(struct xe_vma *vma)
+{
+ u32 divisor = vma->userptr.divisor ? vma->userptr.divisor : 2;
+ static u32 count;
+
+ if (count++ % divisor == divisor - 1) {
+ vma->userptr.divisor = divisor << 1;
+ return true;
+ }
+
+ return false;
}
-static int xe_pt_userptr_pre_commit(struct xe_migrate_pt_update *pt_update)
+#else
+
+static bool xe_pt_userptr_inject_eagain(struct xe_vma *vma)
+{
+ return false;
+}
+
+#endif
+
+static void vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma)
{
- struct xe_pt_migrate_pt_update *userptr_update =
- container_of(pt_update, typeof(*userptr_update), base);
- struct xe_vma *vma = pt_update->vma;
unsigned long notifier_seq = vma->userptr.notifier_seq;
- struct xe_vm *vm = xe_vma_vm(vma);
- int err = xe_pt_vm_dependencies(pt_update->job,
- &vm->rftree[pt_update->tile_id],
- pt_update->start,
- pt_update->last);
- if (err)
- return err;
+ lockdep_assert_held_read(&vm->userptr.notifier_lock);
- userptr_update->locked = false;
+ if (vma->userptr.initial_bind || xe_vm_in_fault_mode(vm))
+ return;
- /*
- * Wait until nobody is running the invalidation notifier, and
- * since we're exiting the loop holding the notifier lock,
- * nobody can proceed invalidating either.
- *
- * Note that we don't update the vma->userptr.notifier_seq since
- * we don't update the userptr pages.
- */
- do {
- down_read(&vm->userptr.notifier_lock);
- if (!mmu_interval_read_retry(&vma->userptr.notifier,
- notifier_seq))
- break;
+ if (!mmu_interval_read_retry(&vma->userptr.notifier,
+ notifier_seq) &&
+ !xe_pt_userptr_inject_eagain(vma))
+ return;
- up_read(&vm->userptr.notifier_lock);
+ spin_lock(&vm->userptr.invalidated_lock);
+ list_move_tail(&vma->userptr.invalidate_link,
+ &vm->userptr.invalidated);
+ spin_unlock(&vm->userptr.invalidated_lock);
+}
- if (userptr_update->bind)
- return -EAGAIN;
+static void op_check_userptr(struct xe_vm *vm, struct xe_vma_op *op)
+{
+ lockdep_assert_held_read(&vm->userptr.notifier_lock);
- notifier_seq = mmu_interval_read_begin(&vma->userptr.notifier);
- } while (true);
+ switch (op->base.op) {
+ case DRM_GPUVA_OP_MAP:
+ if (!op->map.immediate && xe_vm_in_fault_mode(vm))
+ break;
- /* Inject errors to test_whether they are handled correctly */
- if (userptr_update->bind && xe_pt_userptr_inject_eagain(vma)) {
- up_read(&vm->userptr.notifier_lock);
- return -EAGAIN;
+ vma_check_userptr(vm, op->map.vma);
+ break;
+ case DRM_GPUVA_OP_REMAP:
+ if (op->remap.prev)
+ vma_check_userptr(vm, op->remap.prev);
+ if (op->remap.next)
+ vma_check_userptr(vm, op->remap.next);
+ break;
+ case DRM_GPUVA_OP_UNMAP:
+ break;
+ case DRM_GPUVA_OP_PREFETCH:
+ vma_check_userptr(vm, gpuva_to_vma(op->base.prefetch.va));
+ break;
+ default:
+ drm_warn(&vm->xe->drm, "NOT POSSIBLE");
}
+}
- userptr_update->locked = true;
+static int xe_pt_userptr_pre_commit(struct xe_migrate_pt_update *pt_update)
+{
+ struct xe_vm *vm = pt_update->vops->vm;
+ struct xe_vma_ops *vops = pt_update->vops;
+ struct xe_vma_op *op;
+ int err;
- return 0;
-}
+ err = xe_pt_pre_commit(pt_update);
+ if (err)
+ return err;
-static const struct xe_migrate_pt_update_ops bind_ops = {
- .populate = xe_vm_populate_pgtable,
- .pre_commit = xe_pt_pre_commit,
-};
+ down_read(&vm->userptr.notifier_lock);
-static const struct xe_migrate_pt_update_ops userptr_bind_ops = {
- .populate = xe_vm_populate_pgtable,
- .pre_commit = xe_pt_userptr_pre_commit,
-};
+ list_for_each_entry(op, &vops->list, link)
+ op_check_userptr(vm, op);
+
+ return 0;
+}
struct invalidation_fence {
struct xe_gt_tlb_invalidation_fence base;
@@ -1203,181 +1310,6 @@ static int invalidation_fence_init(struct xe_gt *gt,
return ret && ret != -ENOENT ? ret : 0;
}
-static void xe_pt_calc_rfence_interval(struct xe_vma *vma,
- struct xe_pt_migrate_pt_update *update,
- struct xe_vm_pgtable_update *entries,
- u32 num_entries)
-{
- int i, level = 0;
-
- for (i = 0; i < num_entries; i++) {
- const struct xe_vm_pgtable_update *entry = &entries[i];
-
- if (entry->pt->level > level)
- level = entry->pt->level;
- }
-
- /* Greedy (non-optimal) calculation but simple */
- update->base.start = ALIGN_DOWN(xe_vma_start(vma),
- 0x1ull << xe_pt_shift(level));
- update->base.last = ALIGN(xe_vma_end(vma),
- 0x1ull << xe_pt_shift(level)) - 1;
-}
-
-/**
- * __xe_pt_bind_vma() - Build and connect a page-table tree for the vma
- * address range.
- * @tile: The tile to bind for.
- * @vma: The vma to bind.
- * @q: The exec_queue with which to do pipelined page-table updates.
- * @syncs: Entries to sync on before binding the built tree to the live vm tree.
- * @num_syncs: Number of @sync entries.
- * @rebind: Whether we're rebinding this vma to the same address range without
- * an unbind in-between.
- *
- * This function builds a page-table tree (see xe_pt_stage_bind() for more
- * information on page-table building), and the xe_vm_pgtable_update entries
- * abstracting the operations needed to attach it to the main vm tree. It
- * then takes the relevant locks and updates the metadata side of the main
- * vm tree and submits the operations for pipelined attachment of the
- * gpu page-table to the vm main tree, (which can be done either by the
- * cpu and the GPU).
- *
- * Return: A valid dma-fence representing the pipelined attachment operation
- * on success, an error pointer on error.
- */
-struct dma_fence *
-__xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q,
- struct xe_sync_entry *syncs, u32 num_syncs,
- bool rebind)
-{
- struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1];
- struct xe_pt_migrate_pt_update bind_pt_update = {
- .base = {
- .ops = xe_vma_is_userptr(vma) ? &userptr_bind_ops : &bind_ops,
- .vma = vma,
- .tile_id = tile->id,
- },
- .bind = true,
- };
- struct xe_vm *vm = xe_vma_vm(vma);
- u32 num_entries;
- struct dma_fence *fence;
- struct invalidation_fence *ifence = NULL;
- struct xe_range_fence *rfence;
- int err;
-
- bind_pt_update.locked = false;
- xe_bo_assert_held(xe_vma_bo(vma));
- xe_vm_assert_held(vm);
-
- vm_dbg(&xe_vma_vm(vma)->xe->drm,
- "Preparing bind, with range [%llx...%llx) engine %p.\n",
- xe_vma_start(vma), xe_vma_end(vma), q);
-
- err = xe_pt_prepare_bind(tile, vma, entries, &num_entries);
- if (err)
- goto err;
- xe_tile_assert(tile, num_entries <= ARRAY_SIZE(entries));
-
- xe_vm_dbg_print_entries(tile_to_xe(tile), entries, num_entries);
- xe_pt_calc_rfence_interval(vma, &bind_pt_update, entries,
- num_entries);
-
- /*
- * If rebind, we have to invalidate TLB on !LR vms to invalidate
- * cached PTEs point to freed memory. on LR vms this is done
- * automatically when the context is re-enabled by the rebind worker,
- * or in fault mode it was invalidated on PTE zapping.
- *
- * If !rebind, and scratch enabled VMs, there is a chance the scratch
- * PTE is already cached in the TLB so it needs to be invalidated.
- * on !LR VMs this is done in the ring ops preceding a batch, but on
- * non-faulting LR, in particular on user-space batch buffer chaining,
- * it needs to be done here.
- */
- if ((rebind && !xe_vm_no_dma_fences(vm) && !vm->batch_invalidate_tlb) ||
- (!rebind && vm->scratch_bo[tile->id] && xe_vm_in_compute_mode(vm))) {
- ifence = kzalloc(sizeof(*ifence), GFP_KERNEL);
- if (!ifence)
- return ERR_PTR(-ENOMEM);
- }
-
- rfence = kzalloc(sizeof(*rfence), GFP_KERNEL);
- if (!rfence) {
- kfree(ifence);
- return ERR_PTR(-ENOMEM);
- }
-
- fence = xe_migrate_update_pgtables(tile->migrate,
- vm, xe_vma_bo(vma), q,
- entries, num_entries,
- syncs, num_syncs,
- &bind_pt_update.base);
- if (!IS_ERR(fence)) {
- bool last_munmap_rebind = vma->gpuva.flags & XE_VMA_LAST_REBIND;
- LLIST_HEAD(deferred);
- int err;
-
- err = xe_range_fence_insert(&vm->rftree[tile->id], rfence,
- &xe_range_fence_kfree_ops,
- bind_pt_update.base.start,
- bind_pt_update.base.last, fence);
- if (err)
- dma_fence_wait(fence, false);
-
- /* TLB invalidation must be done before signaling rebind */
- if (ifence) {
- int err = invalidation_fence_init(tile->primary_gt,
- ifence, fence,
- xe_vma_start(vma),
- xe_vma_end(vma),
- xe_vma_vm(vma)->usm.asid);
- if (err) {
- dma_fence_put(fence);
- kfree(ifence);
- return ERR_PTR(err);
- }
- fence = &ifence->base.base;
- }
-
- /* add shared fence now for pagetable delayed destroy */
- dma_resv_add_fence(&vm->resv, fence, !rebind &&
- last_munmap_rebind ?
- DMA_RESV_USAGE_KERNEL :
- DMA_RESV_USAGE_BOOKKEEP);
-
- if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
- dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
- DMA_RESV_USAGE_BOOKKEEP);
- xe_pt_commit_bind(vma, entries, num_entries, rebind,
- bind_pt_update.locked ? &deferred : NULL);
-
- /* This vma is live (again?) now */
- vma->tile_present |= BIT(tile->id);
-
- if (bind_pt_update.locked) {
- vma->userptr.initial_bind = true;
- up_read(&vm->userptr.notifier_lock);
- xe_bo_put_commit(&deferred);
- }
- if (!rebind && last_munmap_rebind &&
- xe_vm_in_compute_mode(vm))
- xe_vm_queue_rebind_worker(vm);
- } else {
- kfree(rfence);
- kfree(ifence);
- if (bind_pt_update.locked)
- up_read(&vm->userptr.notifier_lock);
- xe_pt_abort_bind(vma, entries, num_entries);
- }
-
- return fence;
-
-err:
- return ERR_PTR(err);
-}
-
struct xe_pt_stage_unbind_walk {
/** @base: The pagewalk base-class. */
struct xe_pt_walk base;
@@ -1528,8 +1460,8 @@ xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update,
void *ptr, u32 qword_ofs, u32 num_qwords,
const struct xe_vm_pgtable_update *update)
{
- struct xe_vma *vma = pt_update->vma;
- u64 empty = __xe_pt_empty_pte(tile, xe_vma_vm(vma), update->pt->level);
+ struct xe_vm *vm = pt_update->vops->vm;
+ u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level);
int i;
if (map && map->is_iomem)
@@ -1573,144 +1505,393 @@ xe_pt_commit_unbind(struct xe_vma *vma,
}
}
-static const struct xe_migrate_pt_update_ops unbind_ops = {
- .populate = xe_migrate_clear_pgtable_callback,
- .pre_commit = xe_pt_pre_commit,
-};
+static void
+xe_pt_update_ops_rfence_interval(struct xe_vm_pgtable_update_ops *pt_update_ops,
+ struct xe_vma *vma)
+{
+ u32 current_op = pt_update_ops->current_op;
+ struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
+ int i, level = 0;
+ u64 start, last;
-static const struct xe_migrate_pt_update_ops userptr_unbind_ops = {
- .populate = xe_migrate_clear_pgtable_callback,
- .pre_commit = xe_pt_userptr_pre_commit,
-};
+ for (i = 0; i < pt_op->num_entries; i++) {
+ const struct xe_vm_pgtable_update *entry = &pt_op->entries[i];
-/**
- * __xe_pt_unbind_vma() - Disconnect and free a page-table tree for the vma
- * address range.
- * @tile: The tile to unbind for.
- * @vma: The vma to unbind.
- * @q: The exec_queue with which to do pipelined page-table updates.
- * @syncs: Entries to sync on before disconnecting the tree to be destroyed.
- * @num_syncs: Number of @sync entries.
- *
- * This function builds a the xe_vm_pgtable_update entries abstracting the
- * operations needed to detach the page-table tree to be destroyed from the
- * man vm tree.
- * It then takes the relevant locks and submits the operations for
- * pipelined detachment of the gpu page-table from the vm main tree,
- * (which can be done either by the cpu and the GPU), Finally it frees the
- * detached page-table tree.
- *
- * Return: A valid dma-fence representing the pipelined detachment operation
- * on success, an error pointer on error.
- */
-struct dma_fence *
-__xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q,
- struct xe_sync_entry *syncs, u32 num_syncs)
+ if (entry->pt->level > level)
+ level = entry->pt->level;
+ }
+
+ /* Greedy (non-optimal) calculation but simple */
+ start = ALIGN_DOWN(xe_vma_start(vma), 0x1ull << xe_pt_shift(level));
+ last = ALIGN(xe_vma_end(vma), 0x1ull << xe_pt_shift(level)) - 1;
+
+ if (start < pt_update_ops->start)
+ pt_update_ops->start = start;
+ if (last > pt_update_ops->last)
+ pt_update_ops->last = last;
+}
+
+static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,
+ struct xe_vm_pgtable_update_ops *pt_update_ops,
+ struct xe_vma *vma)
{
- struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1];
- struct xe_pt_migrate_pt_update unbind_pt_update = {
- .base = {
- .ops = xe_vma_is_userptr(vma) ? &userptr_unbind_ops :
- &unbind_ops,
- .vma = vma,
- .tile_id = tile->id,
- },
- };
- struct xe_vm *vm = xe_vma_vm(vma);
- u32 num_entries;
- struct dma_fence *fence = NULL;
- struct invalidation_fence *ifence;
- struct xe_range_fence *rfence;
+ u32 current_op = pt_update_ops->current_op;
+ struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
+ struct llist_head *deferred = &pt_update_ops->deferred;
+ int err;
- LLIST_HEAD(deferred);
+ xe_bo_assert_held(xe_vma_bo(vma));
+
+ vm_dbg(&xe_vma_vm(vma)->xe->drm,
+ "Preparing bind, with range [%llx...%llx)\n",
+ xe_vma_start(vma), xe_vma_end(vma) - 1);
+
+ pt_op->bind = true;
+ pt_op->rebind = BIT(tile->id) & vma->tile_present;
+
+ err = xe_pt_prepare_bind(tile, vma, pt_op->entries,
+ &pt_op->num_entries);
+ if (!err) {
+ xe_tile_assert(tile, pt_op->num_entries <=
+ ARRAY_SIZE(pt_op->entries));
+ xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,
+ pt_op->num_entries);
+
+ xe_pt_update_ops_rfence_interval(pt_update_ops, vma);
+ ++pt_update_ops->current_op;
+ pt_update_ops->needs_userptr_lock |= xe_vma_is_userptr(vma);
+
+ /*
+ * If rebind, we have to invalidate TLB on !LR vms to invalidate
+ * cached PTEs point to freed memory. on LR vms this is done
+ * automatically when the context is re-enabled by the rebind
+ * worker, or in fault mode it was invalidated on PTE zapping.
+ *
+ * If !rebind, and scratch enabled VMs, there is a chance the
+ * scratch PTE is already cached in the TLB so it needs to be
+ * invalidated. on !LR VMs this is done in the ring ops
+ * preceding a batch, but on non-faulting LR, in particular on
+ * user-space batch buffer chaining, it needs to be done here.
+ */
+ pt_update_ops->needs_invalidation |=
+ (pt_op->rebind && !xe_vm_no_dma_fences(vm) &&
+ !vm->batch_invalidate_tlb) ||
+ (!pt_op->rebind && vm->scratch_bo[tile->id] &&
+ xe_vm_in_compute_mode(vm));
+
+ /* FIXME: Don't commit right away */
+ xe_pt_commit_bind(vma, pt_op->entries, pt_op->num_entries,
+ pt_op->rebind, deferred);
+ }
+
+ return err;
+}
+
+static int unbind_op_prepare(struct xe_tile *tile,
+ struct xe_vm_pgtable_update_ops *pt_update_ops,
+ struct xe_vma *vma)
+{
+ u32 current_op = pt_update_ops->current_op;
+ struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
+ struct llist_head *deferred = &pt_update_ops->deferred;
xe_bo_assert_held(xe_vma_bo(vma));
- xe_vm_assert_held(vm);
vm_dbg(&xe_vma_vm(vma)->xe->drm,
- "Preparing unbind, with range [%llx...%llx) engine %p.\n",
- xe_vma_start(vma), xe_vma_end(vma), q);
+ "Preparing unbind, with range [%llx...%llx)\n",
+ xe_vma_start(vma), xe_vma_end(vma) - 1);
- num_entries = xe_pt_stage_unbind(tile, vma, entries);
- xe_tile_assert(tile, num_entries <= ARRAY_SIZE(entries));
+ pt_op->bind = false;
+ pt_op->rebind = false;
- xe_vm_dbg_print_entries(tile_to_xe(tile), entries, num_entries);
- xe_pt_calc_rfence_interval(vma, &unbind_pt_update, entries,
- num_entries);
+ pt_op->num_entries = xe_pt_stage_unbind(tile, vma, pt_op->entries);
- ifence = kzalloc(sizeof(*ifence), GFP_KERNEL);
- if (!ifence)
- return ERR_PTR(-ENOMEM);
+ xe_pt_update_ops_rfence_interval(pt_update_ops, vma);
+ ++pt_update_ops->current_op;
+ pt_update_ops->needs_userptr_lock |= xe_vma_is_userptr(vma);
+ pt_update_ops->needs_invalidation = true;
- rfence = kzalloc(sizeof(*rfence), GFP_KERNEL);
- if (!rfence) {
- kfree(ifence);
- return ERR_PTR(-ENOMEM);
+ /* FIXME: Don't commit right away */
+ xe_pt_commit_unbind(vma, pt_op->entries, pt_op->num_entries,
+ deferred);
+
+ return 0;
+}
+
+static int op_prepare(struct xe_vm *vm,
+ struct xe_tile *tile,
+ struct xe_vm_pgtable_update_ops *pt_update_ops,
+ struct xe_vma_op *op)
+{
+ int err = 0;
+
+ xe_vm_assert_held(vm);
+
+ switch (op->base.op) {
+ case DRM_GPUVA_OP_MAP:
+ if (!op->map.immediate && xe_vm_in_fault_mode(vm))
+ break;
+
+ err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma);
+ pt_update_ops->wait_vm_kernel = true;
+ break;
+ case DRM_GPUVA_OP_REMAP:
+ err = unbind_op_prepare(tile, pt_update_ops,
+ gpuva_to_vma(op->base.remap.unmap->va));
+
+ if (!err && op->remap.prev) {
+ err = bind_op_prepare(vm, tile, pt_update_ops,
+ op->remap.prev);
+ pt_update_ops->wait_vm_bookkeep = true;
+ }
+ if (!err && op->remap.next) {
+ err = bind_op_prepare(vm, tile, pt_update_ops,
+ op->remap.next);
+ pt_update_ops->wait_vm_bookkeep = true;
+ }
+ break;
+ case DRM_GPUVA_OP_UNMAP:
+ err = unbind_op_prepare(tile, pt_update_ops,
+ gpuva_to_vma(op->base.unmap.va));
+ break;
+ case DRM_GPUVA_OP_PREFETCH:
+ err = bind_op_prepare(vm, tile, pt_update_ops,
+ gpuva_to_vma(op->base.prefetch.va));
+ pt_update_ops->wait_vm_kernel = true;
+ break;
+ default:
+ drm_warn(&vm->xe->drm, "NOT POSSIBLE");
}
- /*
- * Even if we were already evicted and unbind to destroy, we need to
- * clear again here. The eviction may have updated pagetables at a
- * lower level, because it needs to be more conservative.
- */
- fence = xe_migrate_update_pgtables(tile->migrate,
- vm, NULL, q ? q :
- vm->q[tile->id],
- entries, num_entries,
- syncs, num_syncs,
- &unbind_pt_update.base);
- if (!IS_ERR(fence)) {
- int err;
-
- err = xe_range_fence_insert(&vm->rftree[tile->id], rfence,
- &xe_range_fence_kfree_ops,
- unbind_pt_update.base.start,
- unbind_pt_update.base.last, fence);
+ return err;
+}
+
+static void
+xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops)
+{
+ init_llist_head(&pt_update_ops->deferred);
+ pt_update_ops->start = ~0x0ull;
+ pt_update_ops->last = 0x0ull;
+}
+
+int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops)
+{
+ struct xe_vm_pgtable_update_ops *pt_update_ops =
+ &vops->pt_update_ops[tile->id];
+ struct xe_vma_op *op;
+ int err;
+
+ lockdep_assert_held(&vops->vm->lock);
+ xe_vm_assert_held(vops->vm);
+
+ xe_pt_update_ops_init(pt_update_ops);
+
+ list_for_each_entry(op, &vops->list, link) {
+ err = op_prepare(vops->vm, tile, pt_update_ops, op);
if (err)
- dma_fence_wait(fence, false);
+ return err;
+ }
- /* TLB invalidation must be done before signaling unbind */
- err = invalidation_fence_init(tile->primary_gt, ifence, fence,
- xe_vma_start(vma),
- xe_vma_end(vma),
- xe_vma_vm(vma)->usm.asid);
- if (err) {
- dma_fence_put(fence);
- kfree(ifence);
- return ERR_PTR(err);
- }
- fence = &ifence->base.base;
- /* add shared fence now for pagetable delayed destroy */
- dma_resv_add_fence(&vm->resv, fence,
- DMA_RESV_USAGE_BOOKKEEP);
+ xe_tile_assert(tile, pt_update_ops->current_op ==
+ pt_update_ops->num_ops);
- /* This fence will be installed by caller when doing eviction */
- if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
- dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
- DMA_RESV_USAGE_BOOKKEEP);
- xe_pt_commit_unbind(vma, entries, num_entries,
- unbind_pt_update.locked ? &deferred : NULL);
- vma->tile_present &= ~BIT(tile->id);
- } else {
- kfree(rfence);
- kfree(ifence);
+ return 0;
+}
+
+static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile,
+ struct xe_vm_pgtable_update_ops *pt_update_ops,
+ struct xe_vma *vma, struct dma_fence *fence)
+{
+ if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
+ dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
+ DMA_RESV_USAGE_BOOKKEEP);
+ vma->tile_present |= BIT(tile->id);
+ if (xe_vma_is_userptr(vma)) {
+ lockdep_assert_held_read(&vm->userptr.notifier_lock);
+ vma->userptr.initial_bind = true;
}
+ if (pt_update_ops->wait_vm_bookkeep && xe_vm_in_compute_mode(vm))
+ xe_vm_queue_rebind_worker(vm);
+}
- if (!vma->tile_present)
+static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile,
+ struct xe_vma *vma, struct dma_fence *fence)
+{
+ if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
+ dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
+ DMA_RESV_USAGE_BOOKKEEP);
+ vma->tile_present &= ~BIT(tile->id);
+ if (!vma->tile_present) {
list_del_init(&vma->combined_links.rebind);
+ if (xe_vma_is_userptr(vma)) {
+ lockdep_assert_held_read(&vm->userptr.notifier_lock);
- if (unbind_pt_update.locked) {
- xe_tile_assert(tile, xe_vma_is_userptr(vma));
-
- if (!vma->tile_present) {
spin_lock(&vm->userptr.invalidated_lock);
list_del_init(&vma->userptr.invalidate_link);
spin_unlock(&vm->userptr.invalidated_lock);
}
- up_read(&vm->userptr.notifier_lock);
- xe_bo_put_commit(&deferred);
+ }
+}
+
+static void op_commit(struct xe_vm *vm,
+ struct xe_tile *tile,
+ struct xe_vm_pgtable_update_ops *pt_update_ops,
+ struct xe_vma_op *op, struct dma_fence *fence)
+{
+ xe_vm_assert_held(vm);
+
+ switch (op->base.op) {
+ case DRM_GPUVA_OP_MAP:
+ if (!op->map.immediate && xe_vm_in_fault_mode(vm))
+ break;
+
+ bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence);
+ break;
+ case DRM_GPUVA_OP_REMAP:
+ unbind_op_commit(vm, tile,
+ gpuva_to_vma(op->base.remap.unmap->va), fence);
+
+ if (op->remap.prev)
+ bind_op_commit(vm, tile, pt_update_ops, op->remap.prev,
+ fence);
+ if (op->remap.next)
+ bind_op_commit(vm, tile, pt_update_ops, op->remap.next,
+ fence);
+ break;
+ case DRM_GPUVA_OP_UNMAP:
+ unbind_op_commit(vm, tile, gpuva_to_vma(op->base.unmap.va),
+ fence);
+ break;
+ case DRM_GPUVA_OP_PREFETCH:
+ bind_op_commit(vm, tile, pt_update_ops,
+ gpuva_to_vma(op->base.prefetch.va), fence);
+ break;
+ default:
+ drm_warn(&vm->xe->drm, "NOT POSSIBLE");
+ }
+}
+
+static const struct xe_migrate_pt_update_ops migrate_ops = {
+ .populate = xe_vm_populate_pgtable,
+ .clear = xe_migrate_clear_pgtable_callback,
+ .pre_commit = xe_pt_pre_commit,
+};
+
+static const struct xe_migrate_pt_update_ops userptr_migrate_ops = {
+ .populate = xe_vm_populate_pgtable,
+ .clear = xe_migrate_clear_pgtable_callback,
+ .pre_commit = xe_pt_userptr_pre_commit,
+};
+
+struct dma_fence *
+xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
+{
+ struct xe_vm *vm = vops->vm;
+ struct xe_vm_pgtable_update_ops *pt_update_ops =
+ &vops->pt_update_ops[tile->id];
+ struct dma_fence *fence;
+ struct invalidation_fence *ifence = NULL;
+ struct xe_range_fence *rfence;
+ struct xe_vma_op *op;
+ int err = 0;
+ struct xe_migrate_pt_update update = {
+ .ops = pt_update_ops->needs_userptr_lock ?
+ &userptr_migrate_ops :
+ &migrate_ops,
+ .vops = vops,
+ .tile_id = tile->id
+ };
+
+ lockdep_assert_held(&vm->lock);
+ xe_vm_assert_held(vm);
+
+ if (pt_update_ops->needs_invalidation) {
+ ifence = kzalloc(sizeof(*ifence), GFP_KERNEL);
+ if (!ifence)
+ return ERR_PTR(-ENOMEM);
+ }
+
+ rfence = kzalloc(sizeof(*rfence), GFP_KERNEL);
+ if (!rfence) {
+ err = -ENOMEM;
+ goto free_ifence;
}
+ fence = xe_migrate_update_pgtables(tile->migrate, &update);
+ if (IS_ERR(fence)) {
+ err = PTR_ERR(fence);
+ goto free_rfence;
+ }
+
+ err = xe_range_fence_insert(&vm->rftree[tile->id], rfence,
+ &xe_range_fence_kfree_ops,
+ pt_update_ops->start,
+ pt_update_ops->last, fence);
+ if (err)
+ dma_fence_wait(fence, false);
+
+ /* TLB invalidation must be done before signaling rebind */
+ if (ifence) {
+ err = invalidation_fence_init(tile->primary_gt, ifence, fence,
+ pt_update_ops->start,
+ pt_update_ops->last,
+ vm->usm.asid);
+ if (err)
+ goto put_fence;
+ fence = &ifence->base.base;
+ }
+
+ dma_resv_add_fence(&vm->resv, fence,
+ pt_update_ops->wait_vm_bookkeep ?
+ DMA_RESV_USAGE_KERNEL :
+ DMA_RESV_USAGE_BOOKKEEP);
+
+ list_for_each_entry(op, &vops->list, link)
+ op_commit(vops->vm, tile, pt_update_ops, op, fence);
+
+ if (pt_update_ops->needs_userptr_lock)
+ up_read(&vm->userptr.notifier_lock);
+
return fence;
+
+put_fence:
+ if (pt_update_ops->needs_userptr_lock)
+ up_read(&vm->userptr.notifier_lock);
+ dma_fence_put(fence);
+free_rfence:
+ kfree(rfence);
+free_ifence:
+ kfree(ifence);
+
+ return ERR_PTR(err);
+}
+
+void xe_pt_update_ops_commit(struct xe_tile *tile, struct xe_vma_ops *vops)
+{
+ struct xe_vm_pgtable_update_ops *pt_update_ops =
+ &vops->pt_update_ops[tile->id];
+ int i;
+
+ lockdep_assert_held(&vops->vm->lock);
+ xe_vm_assert_held(vops->vm);
+
+ /* FIXME: Not 100% correct */
+ for (i = 0; i < pt_update_ops->num_ops; ++i) {
+ struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
+
+ if (pt_op->bind)
+ xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
+ }
+ xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
+}
+
+void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops)
+{
+ lockdep_assert_held(&vops->vm->lock);
+ xe_vm_assert_held(vops->vm);
+
+ /* FIXME: Just kill VM for now + cleanup PTs */
+ xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
+ xe_vm_kill(vops->vm, false);
}
diff --git a/drivers/gpu/drm/xe/xe_pt.h b/drivers/gpu/drm/xe/xe_pt.h
index d5460e58dbbf..547d0624685b 100644
--- a/drivers/gpu/drm/xe/xe_pt.h
+++ b/drivers/gpu/drm/xe/xe_pt.h
@@ -17,6 +17,7 @@ struct xe_sync_entry;
struct xe_tile;
struct xe_vm;
struct xe_vma;
+struct xe_vma_ops;
#define xe_pt_write(xe, map, idx, data) \
xe_map_wr(xe, map, (idx) * sizeof(u64), u64, data)
@@ -34,6 +35,12 @@ void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm,
void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred);
+int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops);
+struct dma_fence *xe_pt_update_ops_run(struct xe_tile *tile,
+ struct xe_vma_ops *vops);
+void xe_pt_update_ops_commit(struct xe_tile *tile, struct xe_vma_ops *vops);
+void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops);
+
struct dma_fence *
__xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q,
struct xe_sync_entry *syncs, u32 num_syncs,
diff --git a/drivers/gpu/drm/xe/xe_pt_types.h b/drivers/gpu/drm/xe/xe_pt_types.h
index 33b2df72aad7..f4d308ce7985 100644
--- a/drivers/gpu/drm/xe/xe_pt_types.h
+++ b/drivers/gpu/drm/xe/xe_pt_types.h
@@ -85,4 +85,38 @@ struct xe_vm_pgtable_update_op {
bool rebind;
};
+/** struct xe_vm_pgtable_update_ops: page table update operations */
+struct xe_vm_pgtable_update_ops {
+ /** @ops: operations */
+ struct xe_vm_pgtable_update_op *ops;
+ /** @deferred: deferred list to destroy PT entries */
+ struct llist_head deferred;
+ /** @q: exec queue for PT operations */
+ struct xe_exec_queue *q;
+ /** @start: start address of ops */
+ u64 start;
+ /** @last: last address of ops */
+ u64 last;
+ /** @num_ops: number of operations */
+ u32 num_ops;
+ /** @current_op: current operations */
+ u32 current_op;
+ /** @needs_userptr_lock: Needs userptr lock */
+ bool needs_userptr_lock;
+ /** @needs_invalidation: Needs invalidation */
+ bool needs_invalidation;
+ /**
+ * @wait_vm_bookkeep: PT operations need to wait until VM is idle
+ * (bookkeep dma-resv slots are idle) and stage all future VM activity
+ * behind these operations (install PT operations into VM kernel
+ * dma-resv slot).
+ */
+ bool wait_vm_bookkeep;
+ /**
+ * @wait_vm_kernel: PT operations need to wait until VM kernel dma-resv
+ * slots are idle.
+ */
+ bool wait_vm_kernel;
+};
+
#endif
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 071424a9d283..7d53b266a116 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -474,7 +474,7 @@ int xe_vm_lock_dma_resv(struct xe_vm *vm, struct drm_exec *exec,
#define XE_VM_REBIND_RETRY_TIMEOUT_MS 1000
-static void xe_vm_kill(struct xe_vm *vm, bool unlocked)
+void xe_vm_kill(struct xe_vm *vm, bool unlocked)
{
struct xe_exec_queue *q;
@@ -638,13 +638,9 @@ static void preempt_rebind_work_func(struct work_struct *w)
err = PTR_ERR(rebind_fence);
goto out_unlock;
}
+ dma_fence_put(rebind_fence);
- if (rebind_fence) {
- dma_fence_wait(rebind_fence, false);
- dma_fence_put(rebind_fence);
- }
-
- /* Wait on munmap style VM unbinds */
+ /* Wait on rebinds */
wait = dma_resv_wait_timeout(&vm->resv,
DMA_RESV_USAGE_KERNEL,
false, MAX_SCHEDULE_TIMEOUT);
@@ -829,8 +825,31 @@ int xe_vm_userptr_check_repin(struct xe_vm *vm)
list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN;
}
-void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma)
+void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma,
+ u8 tile_mask)
{
+ int i;
+
+ for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) {
+ if (BIT(i) & tile_mask) {
+ struct xe_vm_pgtable_update_op *pt_op =
+ vm->dummy_ops.vops.pt_update_ops[i].ops;
+
+ memset(&vm->dummy_ops.vops.pt_update_ops[i], 0,
+ sizeof(vm->dummy_ops.vops.pt_update_ops[i]));
+ vm->dummy_ops.vops.pt_update_ops[i].ops = pt_op;
+ vm->dummy_ops.vops.pt_update_ops[i].num_ops = 1;
+
+ /*
+ * Wait for VM to be idle / schedule execs + resume
+ * behind rebinds
+ */
+ vm->dummy_ops.vops.pt_update_ops[i].wait_vm_bookkeep =
+ true;
+ } else {
+ vm->dummy_ops.vops.pt_update_ops[i].num_ops = 0;
+ }
+ }
vm->dummy_ops.op.base.op = DRM_GPUVA_OP_MAP;
vm->dummy_ops.op.base.map.va.addr = vma->gpuva.va.addr;
vm->dummy_ops.op.base.map.va.range = vma->gpuva.va.range;
@@ -864,7 +883,7 @@ struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
else
trace_xe_vma_rebind_exec(vma);
- xe_vm_populate_dummy_rebind(vm, vma);
+ xe_vm_populate_dummy_rebind(vm, vma, vma->tile_present);
fence = xe_vm_ops_execute(vm, &vm->dummy_ops.vops);
if (IS_ERR(fence))
return fence;
@@ -1709,7 +1728,6 @@ static void vm_destroy_work_func(struct work_struct *w)
xe_vm_unlock(vm);
trace_xe_vm_free(vm);
- dma_fence_put(vm->rebind_fence);
dma_resv_fini(&vm->resv);
xe_vma_ops_fini(&vm->dummy_ops.vops);
kfree(vm);
@@ -1748,192 +1766,12 @@ to_wait_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
return q ? q : vm->q[0];
}
-static struct dma_fence *
-xe_vm_unbind_vma(struct xe_vma *vma, struct xe_exec_queue *q,
- struct xe_sync_entry *syncs, u32 num_syncs,
- bool first_op, bool last_op)
-{
- struct xe_vm *vm = xe_vma_vm(vma);
- struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
- struct xe_tile *tile;
- struct dma_fence *fence = NULL;
- struct dma_fence **fences = NULL;
- struct dma_fence_array *cf = NULL;
- int cur_fence = 0;
- int number_tiles = hweight8(vma->tile_present);
- int err;
- u8 id;
-
- trace_xe_vma_unbind(vma);
-
- if (number_tiles > 1) {
- fences = kmalloc_array(number_tiles, sizeof(*fences),
- GFP_KERNEL);
- if (!fences)
- return ERR_PTR(-ENOMEM);
- }
-
- for_each_tile(tile, vm->xe, id) {
- if (!(vma->tile_present & BIT(id)))
- goto next;
-
- fence = __xe_pt_unbind_vma(tile, vma, q ? q : vm->q[id],
- first_op ? syncs : NULL,
- first_op ? num_syncs : 0);
- if (IS_ERR(fence)) {
- err = PTR_ERR(fence);
- goto err_fences;
- }
-
- if (fences)
- fences[cur_fence++] = fence;
-
-next:
- if (q && vm->pt_root[id] && !list_empty(&q->multi_gt_list))
- q = list_next_entry(q, multi_gt_list);
- }
-
- if (fences) {
- cf = dma_fence_array_create(number_tiles, fences,
- vm->composite_fence_ctx,
- vm->composite_fence_seqno++,
- false);
- if (!cf) {
- --vm->composite_fence_seqno;
- err = -ENOMEM;
- goto err_fences;
- }
- }
-
- return cf ? &cf->base : !fence ?
- xe_exec_queue_last_fence_get(wait_exec_queue, vm) : fence;
-
-err_fences:
- if (fences) {
- while (cur_fence)
- dma_fence_put(fences[--cur_fence]);
- kfree(fences);
- }
-
- return ERR_PTR(err);
-}
-
-static struct dma_fence *
-xe_vm_bind_vma(struct xe_vma *vma, struct xe_exec_queue *q,
- struct xe_sync_entry *syncs, u32 num_syncs,
- bool first_op, bool last_op)
-{
- struct xe_tile *tile;
- struct dma_fence *fence;
- struct dma_fence **fences = NULL;
- struct dma_fence_array *cf = NULL;
- struct xe_vm *vm = xe_vma_vm(vma);
- int cur_fence = 0;
- int number_tiles = hweight8(vma->tile_mask);
- int err;
- u8 id;
-
- trace_xe_vma_bind(vma);
-
- if (number_tiles > 1) {
- fences = kmalloc_array(number_tiles, sizeof(*fences),
- GFP_KERNEL);
- if (!fences)
- return ERR_PTR(-ENOMEM);
- }
-
- for_each_tile(tile, vm->xe, id) {
- if (!(vma->tile_mask & BIT(id)))
- goto next;
-
- fence = __xe_pt_bind_vma(tile, vma, q ? q : vm->q[id],
- first_op ? syncs : NULL,
- first_op ? num_syncs : 0,
- vma->tile_present & BIT(id));
- if (IS_ERR(fence)) {
- err = PTR_ERR(fence);
- goto err_fences;
- }
-
- if (fences)
- fences[cur_fence++] = fence;
-
-next:
- if (q && vm->pt_root[id] && !list_empty(&q->multi_gt_list))
- q = list_next_entry(q, multi_gt_list);
- }
-
- if (fences) {
- cf = dma_fence_array_create(number_tiles, fences,
- vm->composite_fence_ctx,
- vm->composite_fence_seqno++,
- false);
- if (!cf) {
- --vm->composite_fence_seqno;
- err = -ENOMEM;
- goto err_fences;
- }
- }
-
- return cf ? &cf->base : fence;
-
-err_fences:
- if (fences) {
- while (cur_fence)
- dma_fence_put(fences[--cur_fence]);
- kfree(fences);
- }
-
- return ERR_PTR(err);
-}
-
static bool xe_vm_sync_mode(struct xe_vm *vm, struct xe_exec_queue *q)
{
return q ? !(q->flags & EXEC_QUEUE_FLAG_VM_ASYNC) :
!(vm->flags & XE_VM_FLAG_ASYNC_DEFAULT);
}
-static struct dma_fence *
-xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q,
- struct xe_bo *bo, struct xe_sync_entry *syncs, u32 num_syncs,
- bool immediate, bool first_op, bool last_op)
-{
- struct dma_fence *fence;
- struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
-
- xe_vm_assert_held(vm);
- xe_bo_assert_held(bo);
-
- if (immediate) {
- fence = xe_vm_bind_vma(vma, q, syncs, num_syncs, first_op,
- last_op);
- if (IS_ERR(fence))
- return fence;
- } else {
- xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
- fence = xe_exec_queue_last_fence_get(wait_exec_queue, vm);
- }
-
- return fence;
-}
-
-static struct dma_fence *
-xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma,
- struct xe_exec_queue *q, struct xe_sync_entry *syncs,
- u32 num_syncs, bool first_op, bool last_op)
-{
- struct dma_fence *fence;
-
- xe_vm_assert_held(vm);
- xe_bo_assert_held(xe_vma_bo(vma));
-
- fence = xe_vm_unbind_vma(vma, q, syncs, num_syncs, first_op, last_op);
- if (IS_ERR(fence))
- return fence;
-
- return fence;
-}
-
#define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_SCRATCH_PAGE | \
DRM_XE_VM_CREATE_COMPUTE_MODE | \
DRM_XE_VM_CREATE_ASYNC_DEFAULT | \
@@ -2071,21 +1909,6 @@ static const u32 region_to_mem_type[] = {
XE_PL_VRAM1,
};
-static struct dma_fence *
-xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
- struct xe_exec_queue *q, struct xe_sync_entry *syncs,
- u32 num_syncs, bool first_op, bool last_op)
-{
- struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
-
- if (vma->tile_mask != (vma->tile_present & ~vma->usm.tile_invalidated)) {
- return xe_vm_bind(vm, vma, q, xe_vma_bo(vma), syncs, num_syncs,
- true, first_op, last_op);
- } else {
- return xe_exec_queue_last_fence_get(wait_exec_queue, vm);
- }
-}
-
struct ttm_buffer_object *xe_vm_ttm_bo(struct xe_vm *vm)
{
int idx = vm->flags & XE_VM_FLAG_MIGRATION ?
@@ -2406,7 +2229,6 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
struct xe_vma_ops *vops, bool last,
bool async)
{
- struct xe_vma_op *last_op = NULL;
struct drm_gpuva_op *__op;
int err = 0;
@@ -2414,19 +2236,10 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
drm_gpuva_for_each_op(__op, ops) {
struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
- bool first = list_empty(&vops->list);
INIT_LIST_HEAD(&op->link);
list_add_tail(&op->link, &vops->list);
- if (first) {
- op->flags |= XE_VMA_OP_FIRST;
- op->num_syncs = num_syncs;
- op->syncs = syncs;
- }
-
- op->q = q;
-
switch (op->base.op) {
case DRM_GPUVA_OP_MAP:
{
@@ -2526,188 +2339,21 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
}
case DRM_GPUVA_OP_UNMAP:
case DRM_GPUVA_OP_PREFETCH:
+ /* FIXME: Need to skip some prefetch ops */
xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask);
break;
default:
drm_warn(&vm->xe->drm, "NOT POSSIBLE");
}
- last_op = op;
-
err = xe_vma_op_commit(vm, op);
if (err)
return err;
}
- /* FIXME: Unhandled corner case */
- XE_WARN_ON(!last_op && last && !list_empty(&vops->list));
-
- if (!last_op)
- return 0;
-
- if (last) {
- last_op->flags |= XE_VMA_OP_LAST;
- last_op->num_syncs = num_syncs;
- last_op->syncs = syncs;
- }
-
return 0;
}
-static struct dma_fence *op_execute(struct xe_vm *vm, struct xe_vma *vma,
- struct xe_vma_op *op)
-{
- struct dma_fence *fence;
-
- lockdep_assert_held(&vm->lock);
- xe_vm_assert_held(vm);
- xe_bo_assert_held(xe_vma_bo(vma));
-
- switch (op->base.op) {
- case DRM_GPUVA_OP_MAP:
- fence = xe_vm_bind(vm, vma, op->q, xe_vma_bo(vma),
- op->syncs, op->num_syncs,
- op->map.immediate ||
- !xe_vm_in_fault_mode(vm),
- op->flags & XE_VMA_OP_FIRST,
- op->flags & XE_VMA_OP_LAST);
- break;
- case DRM_GPUVA_OP_REMAP:
- {
- bool prev = !!op->remap.prev;
- bool next = !!op->remap.next;
-
- if (!op->remap.unmap_done) {
- if (prev || next)
- vma->gpuva.flags |= XE_VMA_FIRST_REBIND;
- fence = xe_vm_unbind(vm, vma, op->q, op->syncs,
- op->num_syncs,
- op->flags & XE_VMA_OP_FIRST,
- op->flags & XE_VMA_OP_LAST &&
- !prev && !next);
- if (IS_ERR(fence))
- break;
- op->remap.unmap_done = true;
- }
-
- if (prev) {
- op->remap.prev->gpuva.flags |= XE_VMA_LAST_REBIND;
- fence = xe_vm_bind(vm, op->remap.prev, op->q,
- xe_vma_bo(op->remap.prev), op->syncs,
- op->num_syncs, true, false,
- op->flags & XE_VMA_OP_LAST && !next);
- op->remap.prev->gpuva.flags &= ~XE_VMA_LAST_REBIND;
- if (IS_ERR(fence))
- break;
- op->remap.prev = NULL;
- }
-
- if (next) {
- op->remap.next->gpuva.flags |= XE_VMA_LAST_REBIND;
- fence = xe_vm_bind(vm, op->remap.next, op->q,
- xe_vma_bo(op->remap.next),
- op->syncs, op->num_syncs,
- true, false,
- op->flags & XE_VMA_OP_LAST);
- op->remap.next->gpuva.flags &= ~XE_VMA_LAST_REBIND;
- if (IS_ERR(fence))
- break;
- op->remap.next = NULL;
- }
-
- break;
- }
- case DRM_GPUVA_OP_UNMAP:
- fence = xe_vm_unbind(vm, vma, op->q, op->syncs,
- op->num_syncs, op->flags & XE_VMA_OP_FIRST,
- op->flags & XE_VMA_OP_LAST);
- break;
- case DRM_GPUVA_OP_PREFETCH:
- fence = xe_vm_prefetch(vm, vma, op->q, op->syncs, op->num_syncs,
- op->flags & XE_VMA_OP_FIRST,
- op->flags & XE_VMA_OP_LAST);
- break;
- default:
- drm_warn(&vm->xe->drm, "NOT POSSIBLE");
- }
-
- if (IS_ERR(fence))
- trace_xe_vma_fail(vma);
-
- return fence;
-}
-
-static struct dma_fence *
-__xe_vma_op_execute(struct xe_vm *vm, struct xe_vma *vma,
- struct xe_vma_op *op)
-{
- struct dma_fence *fence;
- int err;
-
-retry_userptr:
- fence = op_execute(vm, vma, op);
- if (IS_ERR(fence) && PTR_ERR(fence) == -EAGAIN &&
- xe_vma_is_userptr(vma)) {
- lockdep_assert_held_write(&vm->lock);
- err = xe_vma_userptr_pin_pages(vma);
- if (!err)
- goto retry_userptr;
-
- fence = ERR_PTR(err);
- trace_xe_vma_fail(vma);
- }
-
- return fence;
-}
-
-static struct dma_fence *
-xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op)
-{
- struct dma_fence *fence = ERR_PTR(-ENOMEM);
-
- lockdep_assert_held(&vm->lock);
-
-#ifdef TEST_VM_ASYNC_OPS_ERROR
- if (op->inject_error) {
- op->inject_error = false;
- return fence;
- }
-#endif
-
- switch (op->base.op) {
- case DRM_GPUVA_OP_MAP:
- fence = __xe_vma_op_execute(vm, op->map.vma, op);
- break;
- case DRM_GPUVA_OP_REMAP:
- {
- struct xe_vma *vma;
-
- if (!op->remap.unmap_done)
- vma = gpuva_to_vma(op->base.remap.unmap->va);
- else if (op->remap.prev)
- vma = op->remap.prev;
- else
- vma = op->remap.next;
-
- fence = __xe_vma_op_execute(vm, vma, op);
- break;
- }
- case DRM_GPUVA_OP_UNMAP:
- fence = __xe_vma_op_execute(vm, gpuva_to_vma(op->base.unmap.va),
- op);
- break;
- case DRM_GPUVA_OP_PREFETCH:
- fence = __xe_vma_op_execute(vm,
- gpuva_to_vma(op->base.prefetch.va),
- op);
- break;
- default:
- drm_warn(&vm->xe->drm, "NOT POSSIBLE");
- }
-
- return fence;
-}
-
static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op,
bool post_commit, bool prev_post_commit,
bool next_post_commit)
@@ -2864,18 +2510,97 @@ static int vm_bind_ioctl_ops_lock(struct drm_exec *exec,
struct dma_fence *xe_vm_ops_execute(struct xe_vm *vm, struct xe_vma_ops *vops)
{
- struct xe_vma_op *op, *next;
+ struct xe_exec_queue *q = vops->q;
+ struct xe_tile *tile;
struct dma_fence *fence = NULL;
+ struct dma_fence **fences = NULL;
+ struct dma_fence_array *cf = NULL;
+ int number_tiles = 0, current_fence = 0, err;
+ u8 id;
- list_for_each_entry_safe(op, next, &vops->list, link) {
- fence = xe_vma_op_execute(vm, op);
- if (IS_ERR(fence)) {
- drm_warn(&vm->xe->drm, "VM op(%d) failed with %ld",
- op->base.op, PTR_ERR(fence));
- return ERR_PTR(-ENOSPC);
+ for_each_tile(tile, vm->xe, id) {
+ if (vops->pt_update_ops[id].num_ops)
+ ++number_tiles;
+
+ if (vops->pt_update_ops[id].q)
+ continue;
+
+ if (q) {
+ vops->pt_update_ops[id].q = q;
+ if (vm->pt_root[id] && !list_empty(&q->multi_gt_list))
+ q = list_next_entry(q, multi_gt_list);
+ } else {
+ vops->pt_update_ops[id].q = vm->q[id];
+ }
+ }
+
+ if (number_tiles == 0)
+ return ERR_PTR(-ENODATA);
+
+ if (number_tiles > 1) {
+ fences = kmalloc_array(number_tiles, sizeof(*fences),
+ GFP_KERNEL);
+ if (!fences)
+ return ERR_PTR(-ENOMEM);
+ }
+
+ for_each_tile(tile, vm->xe, id) {
+ if (!vops->pt_update_ops[id].num_ops)
+ continue;
+
+ err = xe_pt_update_ops_prepare(tile, vops);
+ if (err) {
+ fence = ERR_PTR(err);
+ goto err_out;
}
}
+ for_each_tile(tile, vm->xe, id) {
+ if (!vops->pt_update_ops[id].num_ops)
+ continue;
+
+ fence = xe_pt_update_ops_run(tile, vops);
+ if (IS_ERR(fence))
+ goto err_out;
+
+ if (fences)
+ fences[current_fence++] = fence;
+ }
+
+ if (fences) {
+ cf = dma_fence_array_create(number_tiles, fences,
+ vm->composite_fence_ctx,
+ vm->composite_fence_seqno++,
+ false);
+ if (!cf) {
+ --vm->composite_fence_seqno;
+ fence = ERR_PTR(-ENOMEM);
+ goto err_out;
+ }
+ fence = &cf->base;
+ }
+
+ for_each_tile(tile, vm->xe, id) {
+ if (!vops->pt_update_ops[id].num_ops)
+ continue;
+
+ xe_pt_update_ops_commit(tile, vops);
+ }
+
+ return fence;
+
+err_out:
+ for_each_tile(tile, vm->xe, id) {
+ if (!vops->pt_update_ops[id].num_ops)
+ continue;
+
+ xe_pt_update_ops_abort(tile, vops);
+ }
+ while (current_fence)
+ dma_fence_put(fences[--current_fence]);
+ kfree(fences);
+ kfree(cf);
+
return fence;
}
@@ -2919,13 +2644,10 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
return err;
fence = xe_vm_ops_execute(vm, vops);
- if (IS_ERR(fence)) {
- err = PTR_ERR(fence);
- /* FIXME: Killing VM rather than proper error handling */
- xe_vm_kill(vm, false);
- } else {
- vm_bind_ioctl_ops_install_fences(vm, vops, fence);
- }
+ if (IS_ERR(fence))
+ return PTR_ERR(fence);
+
+ vm_bind_ioctl_ops_install_fences(vm, vops, fence);
}
drm_exec_fini(&exec);
diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h
index 7cbb512bcb10..34850f04d966 100644
--- a/drivers/gpu/drm/xe/xe_vm.h
+++ b/drivers/gpu/drm/xe/xe_vm.h
@@ -224,9 +224,12 @@ int xe_analyze_vm(struct drm_printer *p, struct xe_vm *vm, int gt_id);
int xe_vm_prepare_vma(struct drm_exec *exec, struct xe_vma *vma,
unsigned int num_shared);
-void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma);
+void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma,
+ u8 tile_mask);
struct dma_fence *xe_vm_ops_execute(struct xe_vm *vm, struct xe_vma_ops *vops);
+void xe_vm_kill(struct xe_vm *vm, bool unlocked);
+
#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
#define vm_dbg drm_dbg
#else
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index aa72330f824f..adbd8199aa8b 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -169,29 +169,18 @@ struct xe_vma_op_prefetch {
/** enum xe_vma_op_flags - flags for VMA operation */
enum xe_vma_op_flags {
- /** @XE_VMA_OP_FIRST: first VMA operation for a set of syncs */
- XE_VMA_OP_FIRST = BIT(0),
- /** @XE_VMA_OP_LAST: last VMA operation for a set of syncs */
- XE_VMA_OP_LAST = BIT(1),
/** @XE_VMA_OP_COMMITTED: VMA operation committed */
- XE_VMA_OP_COMMITTED = BIT(2),
+ XE_VMA_OP_COMMITTED = BIT(0),
/** @XE_VMA_OP_PREV_COMMITTED: Previous VMA operation committed */
- XE_VMA_OP_PREV_COMMITTED = BIT(3),
+ XE_VMA_OP_PREV_COMMITTED = BIT(1),
/** @XE_VMA_OP_NEXT_COMMITTED: Next VMA operation committed */
- XE_VMA_OP_NEXT_COMMITTED = BIT(4),
+ XE_VMA_OP_NEXT_COMMITTED = BIT(2),
};
/** struct xe_vma_op - VMA operation */
struct xe_vma_op {
/** @base: GPUVA base operation */
struct drm_gpuva_op base;
- /** @q: exec queue for this operation */
- struct xe_exec_queue *q;
- /**
- * @syncs: syncs for this operation, only used on first and last
- * operation
- */
- struct xe_sync_entry *syncs;
/** @num_syncs: number of syncs */
u32 num_syncs;
/** @link: operation link */
@@ -222,19 +211,14 @@ struct xe_vma_ops {
struct list_head list;
/** @vm: VM */
struct xe_vm *vm;
- /** @q: exec queue these operations */
+ /** @q: exec queue for VMA operations */
struct xe_exec_queue *q;
/** @syncs: syncs these operation */
struct xe_sync_entry *syncs;
/** @num_syncs: number of syncs */
u32 num_syncs;
/** @pt_update_ops: page table update operations */
- struct {
- /** @ops: operations */
- struct xe_vm_pgtable_update_op *ops;
- /** @num_ops: number of operations */
- u32 num_ops;
- } pt_update_ops[XE_MAX_TILES_PER_DEVICE];
+ struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE];
};
#define xe_vm_assert_held(vm) dma_resv_assert_held(&(vm)->resv)
@@ -295,9 +279,6 @@ struct xe_vm {
*/
struct list_head rebind_list;
- /** @rebind_fence: rebind fence from execbuf */
- struct dma_fence *rebind_fence;
-
/**
* @destroy_work: worker to destroy VM, needed as a dma_fence signaling
* from an irq context can be last put and the destroy needs to be able
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 22/27] drm/xe: Update clear / populate arguments
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (20 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 21/27] drm/xe: Convert multiple bind ops into single job Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 23/27] drm/xe: Add __xe_migrate_update_pgtables_cpu helper Matthew Brost
` (8 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
This will help implement CPU binds in run_job() as 'struct
xe_migrate_pt_update' is not available at the time of run_job().
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_migrate.c | 9 +++++----
drivers/gpu/drm/xe/xe_migrate.h | 12 +++++-------
drivers/gpu/drm/xe/xe_pt.c | 12 +++++-------
3 files changed, 15 insertions(+), 18 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c
index 3a01c6de1ec6..5d7c3dc080a3 100644
--- a/drivers/gpu/drm/xe/xe_migrate.c
+++ b/drivers/gpu/drm/xe/xe_migrate.c
@@ -1058,6 +1058,7 @@ static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs,
struct xe_migrate_pt_update *pt_update)
{
const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
+ struct xe_vm *vm = pt_update->vops->vm;
u32 chunk;
u32 ofs = update->ofs, size = update->qwords;
@@ -1089,10 +1090,10 @@ static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs,
bb->cs[bb->len++] = lower_32_bits(addr);
bb->cs[bb->len++] = upper_32_bits(addr);
if (pt_op->bind)
- ops->populate(pt_update, tile, NULL, bb->cs + bb->len,
+ ops->populate(tile, NULL, bb->cs + bb->len,
ofs, chunk, update);
else
- ops->clear(pt_update, tile, NULL, bb->cs + bb->len,
+ ops->clear(vm, tile, NULL, bb->cs + bb->len,
ofs, chunk, update);
bb->len += chunk * 2;
@@ -1149,12 +1150,12 @@ xe_migrate_update_pgtables_cpu(struct xe_migrate *m,
&pt_op->entries[i];
if (pt_op->bind)
- ops->populate(pt_update, m->tile,
+ ops->populate(m->tile,
&update->pt_bo->vmap, NULL,
update->ofs, update->qwords,
update);
else
- ops->clear(pt_update, m->tile,
+ ops->clear(vm, m->tile,
&update->pt_bo->vmap, NULL,
update->ofs, update->qwords, update);
}
diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h
index 7fbc21bd265e..805c780d3ab6 100644
--- a/drivers/gpu/drm/xe/xe_migrate.h
+++ b/drivers/gpu/drm/xe/xe_migrate.h
@@ -31,7 +31,6 @@ struct xe_vma;
struct xe_migrate_pt_update_ops {
/**
* @populate: Populate a command buffer or page-table with ptes.
- * @pt_update: Embeddable callback argument.
* @tile: The tile for the current operation.
* @map: struct iosys_map into the memory to be populated.
* @pos: If @map is NULL, map into the memory to be populated.
@@ -43,13 +42,12 @@ struct xe_migrate_pt_update_ops {
* page-table system to populate command buffers or shared
* page-tables with PTEs.
*/
- void (*populate)(struct xe_migrate_pt_update *pt_update,
- struct xe_tile *tile, struct iosys_map *map,
+ void (*populate)(struct xe_tile *tile, struct iosys_map *map,
void *pos, u32 ofs, u32 num_qwords,
const struct xe_vm_pgtable_update *update);
/**
* @clear: Clear a command buffer or page-table with ptes.
- * @pt_update: Embeddable callback argument.
+ * @vm: VM being updated
* @tile: The tile for the current operation.
* @map: struct iosys_map into the memory to be populated.
* @pos: If @map is NULL, map into the memory to be populated.
@@ -61,9 +59,9 @@ struct xe_migrate_pt_update_ops {
* page-table system to populate command buffers or shared
* page-tables with PTEs.
*/
- void (*clear)(struct xe_migrate_pt_update *pt_update,
- struct xe_tile *tile, struct iosys_map *map,
- void *pos, u32 ofs, u32 num_qwords,
+ void (*clear)(struct xe_vm *vm, struct xe_tile *tile,
+ struct iosys_map *map, void *pos, u32 ofs,
+ u32 num_qwords,
const struct xe_vm_pgtable_update *update);
/**
diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index 73199b34820f..7d9bd7d1dabb 100644
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -831,9 +831,8 @@ bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma)
}
static void
-xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *tile,
- struct iosys_map *map, void *data,
- u32 qword_ofs, u32 num_qwords,
+xe_vm_populate_pgtable(struct xe_tile *tile, struct iosys_map *map,
+ void *data, u32 qword_ofs, u32 num_qwords,
const struct xe_vm_pgtable_update *update)
{
struct xe_pt_entry *ptes = update->pt_entries;
@@ -1455,12 +1454,11 @@ static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, struct xe_vma *vma,
}
static void
-xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update,
- struct xe_tile *tile, struct iosys_map *map,
- void *ptr, u32 qword_ofs, u32 num_qwords,
+xe_migrate_clear_pgtable_callback(struct xe_vm *vm, struct xe_tile *tile,
+ struct iosys_map *map, void *ptr,
+ u32 qword_ofs, u32 num_qwords,
const struct xe_vm_pgtable_update *update)
{
- struct xe_vm *vm = pt_update->vops->vm;
u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level);
int i;
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 23/27] drm/xe: Add __xe_migrate_update_pgtables_cpu helper
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (21 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 22/27] drm/xe: Update clear / populate arguments Matthew Brost
@ 2023-11-07 5:25 ` Matthew Brost
2023-11-07 5:26 ` [Intel-xe] [PATCH v2 24/27] drm/xe: Add xe_hw_fence_signal helper Matthew Brost
` (7 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:25 UTC (permalink / raw)
To: intel-xe
This will help implement CPU binds as the submision backend can call
this helper when a bind jobs dependencies are resolved.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_migrate.c | 54 +++++++++++++++++++--------------
1 file changed, 31 insertions(+), 23 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c
index 5d7c3dc080a3..d1022d5544b2 100644
--- a/drivers/gpu/drm/xe/xe_migrate.c
+++ b/drivers/gpu/drm/xe/xe_migrate.c
@@ -1117,6 +1117,34 @@ struct migrate_test_params {
container_of(_priv, struct migrate_test_params, base)
#endif
+static void
+__xe_migrate_update_pgtables_cpu(struct xe_vm *vm, struct xe_tile *tile,
+ const struct xe_migrate_pt_update_ops *ops,
+ struct xe_vm_pgtable_update_op *pt_op,
+ int num_ops)
+{
+ u32 j, i;
+
+ for (j = 0; j < num_ops; ++j, ++pt_op) {
+ for (i = 0; i < pt_op->num_entries; i++) {
+ const struct xe_vm_pgtable_update *update =
+ &pt_op->entries[i];
+
+ if (pt_op->bind)
+ ops->populate(tile, &update->pt_bo->vmap,
+ NULL, update->ofs, update->qwords,
+ update);
+ else
+ ops->clear(vm, tile, &update->pt_bo->vmap,
+ NULL, update->ofs, update->qwords,
+ update);
+ }
+ }
+
+ trace_xe_vm_cpu_bind(vm);
+ xe_device_wmb(vm->xe);
+}
+
static struct dma_fence *
xe_migrate_update_pgtables_cpu(struct xe_migrate *m,
struct xe_migrate_pt_update *pt_update)
@@ -1129,7 +1157,6 @@ xe_migrate_update_pgtables_cpu(struct xe_migrate *m,
struct xe_vm_pgtable_update_ops *pt_update_ops =
&pt_update->vops->pt_update_ops[pt_update->tile_id];
int err;
- u32 j, i;
if (XE_TEST_ONLY(test && test->force_gpu))
return ERR_PTR(-ETIME);
@@ -1141,28 +1168,9 @@ xe_migrate_update_pgtables_cpu(struct xe_migrate *m,
return ERR_PTR(err);
}
- for (j = 0; j < pt_update_ops->num_ops; ++j) {
- const struct xe_vm_pgtable_update_op *pt_op =
- &pt_update_ops->ops[j];
-
- for (i = 0; i < pt_op->num_entries; i++) {
- const struct xe_vm_pgtable_update *update =
- &pt_op->entries[i];
-
- if (pt_op->bind)
- ops->populate(m->tile,
- &update->pt_bo->vmap, NULL,
- update->ofs, update->qwords,
- update);
- else
- ops->clear(vm, m->tile,
- &update->pt_bo->vmap, NULL,
- update->ofs, update->qwords, update);
- }
- }
-
- trace_xe_vm_cpu_bind(vm);
- xe_device_wmb(vm->xe);
+ __xe_migrate_update_pgtables_cpu(vm, m->tile, ops,
+ pt_update_ops->ops,
+ pt_update_ops->num_ops);
return dma_fence_get_stub();
}
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 24/27] drm/xe: Add xe_hw_fence_signal helper
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (22 preceding siblings ...)
2023-11-07 5:25 ` [Intel-xe] [PATCH v2 23/27] drm/xe: Add __xe_migrate_update_pgtables_cpu helper Matthew Brost
@ 2023-11-07 5:26 ` Matthew Brost
2023-11-07 5:26 ` [Intel-xe] [PATCH v2 25/27] drm/xe: CPU binds for jobs Matthew Brost
` (6 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:26 UTC (permalink / raw)
To: intel-xe; +Cc: Matthew Brost
This help implement cpu binds in run_job by having a helper to signal
the cpu bind job's fence.
Signed-off-by: Matthew Brost <matthw.brost@intel.com>
---
drivers/gpu/drm/xe/xe_hw_fence.c | 8 ++++++++
drivers/gpu/drm/xe/xe_hw_fence.h | 1 +
2 files changed, 9 insertions(+)
diff --git a/drivers/gpu/drm/xe/xe_hw_fence.c b/drivers/gpu/drm/xe/xe_hw_fence.c
index a6094c81f2ad..d53124587076 100644
--- a/drivers/gpu/drm/xe/xe_hw_fence.c
+++ b/drivers/gpu/drm/xe/xe_hw_fence.c
@@ -208,6 +208,14 @@ static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence)
return container_of(fence, struct xe_hw_fence, dma);
}
+void xe_hw_fence_signal(struct dma_fence *dma_fence)
+{
+ struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
+ struct xe_device *xe = gt_to_xe(fence->ctx->gt);
+
+ xe_map_wr(xe, &fence->seqno_map, 0, u32, dma_fence->seqno);
+}
+
struct xe_hw_fence *xe_hw_fence_create(struct xe_hw_fence_ctx *ctx,
struct iosys_map seqno_map)
{
diff --git a/drivers/gpu/drm/xe/xe_hw_fence.h b/drivers/gpu/drm/xe/xe_hw_fence.h
index cfe5fd603787..faa0d7fb9ce9 100644
--- a/drivers/gpu/drm/xe/xe_hw_fence.h
+++ b/drivers/gpu/drm/xe/xe_hw_fence.h
@@ -26,5 +26,6 @@ void xe_hw_fence_ctx_finish(struct xe_hw_fence_ctx *ctx);
struct xe_hw_fence *xe_hw_fence_create(struct xe_hw_fence_ctx *ctx,
struct iosys_map seqno_map);
+void xe_hw_fence_signal(struct dma_fence *fence);
#endif
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 25/27] drm/xe: CPU binds for jobs
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (23 preceding siblings ...)
2023-11-07 5:26 ` [Intel-xe] [PATCH v2 24/27] drm/xe: Add xe_hw_fence_signal helper Matthew Brost
@ 2023-11-07 5:26 ` Matthew Brost
2023-11-07 5:26 ` [Intel-xe] [PATCH v2 26/27] drm/xe: Don't use migrate exec queue for page fault binds Matthew Brost
` (5 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:26 UTC (permalink / raw)
To: intel-xe
No reason to use the GPU for binds, rather in run_job use the CPU to do
binds once the bind job dependencies are resolved.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_bo.c | 7 +-
drivers/gpu/drm/xe/xe_bo.h | 6 +-
drivers/gpu/drm/xe/xe_device.c | 23 +++
drivers/gpu/drm/xe/xe_device.h | 7 +
drivers/gpu/drm/xe/xe_device_types.h | 4 +
drivers/gpu/drm/xe/xe_gt_pagefault.c | 5 +-
drivers/gpu/drm/xe/xe_guc_submit.c | 43 ++++-
drivers/gpu/drm/xe/xe_migrate.c | 201 +++---------------------
drivers/gpu/drm/xe/xe_migrate.h | 6 +
drivers/gpu/drm/xe/xe_pt.c | 31 ++--
drivers/gpu/drm/xe/xe_pt.h | 1 +
drivers/gpu/drm/xe/xe_pt_types.h | 5 +
drivers/gpu/drm/xe/xe_sched_job.c | 4 +-
drivers/gpu/drm/xe/xe_sched_job_types.h | 31 +++-
drivers/gpu/drm/xe/xe_vm.c | 97 ++++++------
drivers/gpu/drm/xe/xe_vm.h | 5 +-
16 files changed, 223 insertions(+), 253 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 1ae0543882a0..d63620008e96 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -2097,16 +2097,16 @@ void __xe_bo_release_dummy(struct kref *kref)
/**
* xe_bo_put_commit() - Put bos whose put was deferred by xe_bo_put_deferred().
+ ( @xe: Xe device
* @deferred: The lockless list used for the call to xe_bo_put_deferred().
*
* Puts all bos whose put was deferred by xe_bo_put_deferred().
* The @deferred list can be either an onstack local list or a global
* shared list used by a workqueue.
*/
-void xe_bo_put_commit(struct llist_head *deferred)
+void xe_bo_put_commit(struct xe_device *xe, struct llist_head *deferred)
{
struct llist_node *freed;
- struct xe_bo *bo, *next;
if (!deferred)
return;
@@ -2115,8 +2115,7 @@ void xe_bo_put_commit(struct llist_head *deferred)
if (!freed)
return;
- llist_for_each_entry_safe(bo, next, freed, freed)
- drm_gem_object_free(&bo->ttm.base.refcount);
+ xe_device_put_deferred(xe, freed);
}
/**
diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h
index 98ed35cf39a3..4bdf4875b2e9 100644
--- a/drivers/gpu/drm/xe/xe_bo.h
+++ b/drivers/gpu/drm/xe/xe_bo.h
@@ -10,7 +10,7 @@
#include "xe_bo_types.h"
#include "xe_macros.h"
-#include "xe_vm_types.h"
+#include "xe_vm.h"
#define XE_DEFAULT_GTT_SIZE_MB 3072ULL /* 3GB by default */
@@ -285,10 +285,12 @@ xe_bo_put_deferred(struct xe_bo *bo, struct llist_head *deferred)
if (!kref_put(&bo->ttm.base.refcount, __xe_bo_release_dummy))
return false;
+ xe_vm_get(bo->vm);
+
return llist_add(&bo->freed, deferred);
}
-void xe_bo_put_commit(struct llist_head *deferred);
+void xe_bo_put_commit(struct xe_device *xe, struct llist_head *deferred);
struct sg_table *xe_bo_get_sg(struct xe_bo *bo);
diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
index 515cdf599fab..35662c0e9cd1 100644
--- a/drivers/gpu/drm/xe/xe_device.c
+++ b/drivers/gpu/drm/xe/xe_device.c
@@ -188,6 +188,8 @@ static void xe_device_destroy(struct drm_device *dev, void *dummy)
{
struct xe_device *xe = to_xe_device(dev);
+ flush_work(&xe->mem.deferred_work);
+
if (xe->ordered_wq)
destroy_workqueue(xe->ordered_wq);
@@ -197,6 +199,24 @@ static void xe_device_destroy(struct drm_device *dev, void *dummy)
ttm_device_fini(&xe->ttm);
}
+static void deferred_work(struct work_struct *w)
+{
+ struct xe_device *xe = container_of(w, struct xe_device,
+ mem.deferred_work);
+ struct llist_node *freed = llist_del_all(&xe->mem.deferred);
+ struct xe_bo *bo, *next;
+
+ if (!freed)
+ return;
+
+ llist_for_each_entry_safe(bo, next, freed, freed) {
+ struct xe_vm *vm = bo->vm;
+
+ drm_gem_object_free(&bo->ttm.base.refcount);
+ xe_vm_put(vm);
+ }
+}
+
struct xe_device *xe_device_create(struct pci_dev *pdev,
const struct pci_device_id *ent)
{
@@ -250,6 +270,9 @@ struct xe_device *xe_device_create(struct pci_dev *pdev,
goto err_put;
}
+ init_llist_head(&xe->mem.deferred);
+ INIT_WORK(&xe->mem.deferred_work, deferred_work);
+
err = xe_display_create(xe);
if (WARN_ON(err))
goto err_put;
diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h
index c4232de40ae0..dfc3e6876008 100644
--- a/drivers/gpu/drm/xe/xe_device.h
+++ b/drivers/gpu/drm/xe/xe_device.h
@@ -160,4 +160,11 @@ static inline bool xe_device_has_flat_ccs(struct xe_device *xe)
u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size);
+static inline void xe_device_put_deferred(struct xe_device *xe,
+ struct llist_node *deferred)
+{
+ llist_add(deferred, &xe->mem.deferred);
+ queue_work(system_wq, &xe->mem.deferred_work);
+}
+
#endif
diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h
index 4119ef03fb7e..ce6bbfc92c56 100644
--- a/drivers/gpu/drm/xe/xe_device_types.h
+++ b/drivers/gpu/drm/xe/xe_device_types.h
@@ -295,6 +295,10 @@ struct xe_device {
struct xe_mem_region vram;
/** @sys_mgr: system TTM manager */
struct ttm_resource_manager sys_mgr;
+ /** @deferred: deferred list to destroy PT entries */
+ struct llist_head deferred;
+ /** @deferred_work: worker to destroy PT entries */
+ struct work_struct deferred_work;
} mem;
/** @usm: unified memory state */
diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c
index d383b5a3eaa2..3a3b06d6a4b0 100644
--- a/drivers/gpu/drm/xe/xe_gt_pagefault.c
+++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c
@@ -203,10 +203,13 @@ static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
/* Bind VMA only to the GT that has faulted */
trace_xe_vma_pf_bind(vma);
- xe_vm_populate_dummy_rebind(vm, vma, BIT(tile->id));
+ ret = xe_vm_populate_dummy_rebind(vm, vma, BIT(tile->id));
+ if (ret)
+ goto unlock_dma_resv;
vm->dummy_ops.vops.pt_update_ops[tile->id].q =
xe_tile_migrate_exec_queue(tile);
fence = xe_vm_ops_execute(vm, &vm->dummy_ops.vops);
+ xe_vma_ops_free(&vm->dummy_ops.vops);
if (IS_ERR(fence)) {
ret = PTR_ERR(fence);
goto unlock_dma_resv;
diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c
index 870dc5c532fa..2790ed0c4ffb 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit.c
+++ b/drivers/gpu/drm/xe/xe_guc_submit.c
@@ -15,6 +15,7 @@
#include "regs/xe_lrc_layout.h"
#include "xe_assert.h"
+#include "xe_bo.h"
#include "xe_devcoredump.h"
#include "xe_device.h"
#include "xe_exec_queue.h"
@@ -29,7 +30,9 @@
#include "xe_lrc.h"
#include "xe_macros.h"
#include "xe_map.h"
+#include "xe_migrate.h"
#include "xe_mocs.h"
+#include "xe_pt.h"
#include "xe_ring_ops_types.h"
#include "xe_sched_job.h"
#include "xe_trace.h"
@@ -668,6 +671,30 @@ static void submit_exec_queue(struct xe_exec_queue *q)
}
}
+static bool is_pt_job(struct xe_sched_job *job)
+{
+ return test_bit(JOB_FLAG_PT, &job->fence->flags);
+}
+
+static void cleanup_pt_job(struct xe_device *xe, struct xe_sched_job *job)
+{
+ xe_hw_fence_signal(job->fence);
+ xe_pt_update_ops_free(job->pt_update[0].pt_op,
+ job->pt_update[0].num_ops);
+ xe_bo_put_commit(xe, &job->pt_update[0].deferred);
+ kfree(job->pt_update[0].pt_op);
+}
+
+static void run_pt_job(struct xe_device *xe, struct xe_sched_job *job)
+{
+ __xe_migrate_update_pgtables_cpu(job->pt_update[0].vm,
+ job->pt_update[0].tile,
+ job->pt_update[0].ops,
+ job->pt_update[0].pt_op,
+ job->pt_update[0].num_ops);
+ cleanup_pt_job(xe, job);
+}
+
static struct dma_fence *
guc_exec_queue_run_job(struct drm_sched_job *drm_job)
{
@@ -683,11 +710,17 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job)
trace_xe_sched_job_run(job);
if (!exec_queue_killed_or_banned(q) && !xe_sched_job_is_error(job)) {
- if (!exec_queue_registered(q))
- register_engine(q);
- if (!lr) /* LR jobs are emitted in the exec IOCTL */
- q->ring_ops->emit_job(job);
- submit_exec_queue(q);
+ if (is_pt_job(job)) {
+ run_pt_job(xe, job);
+ } else {
+ if (!exec_queue_registered(q))
+ register_engine(q);
+ if (!lr) /* LR jobs are emitted in the exec IOCTL */
+ q->ring_ops->emit_job(job);
+ submit_exec_queue(q);
+ }
+ } else if (is_pt_job(job)) {
+ cleanup_pt_job(xe, job);
}
if (lr) {
diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c
index d1022d5544b2..d56448dbf135 100644
--- a/drivers/gpu/drm/xe/xe_migrate.c
+++ b/drivers/gpu/drm/xe/xe_migrate.c
@@ -1052,56 +1052,6 @@ struct dma_fence *xe_migrate_clear(struct xe_migrate *m,
return fence;
}
-static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs,
- const struct xe_vm_pgtable_update_op *pt_op,
- const struct xe_vm_pgtable_update *update,
- struct xe_migrate_pt_update *pt_update)
-{
- const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
- struct xe_vm *vm = pt_update->vops->vm;
- u32 chunk;
- u32 ofs = update->ofs, size = update->qwords;
-
- /*
- * If we have 512 entries (max), we would populate it ourselves,
- * and update the PDE above it to the new pointer.
- * The only time this can only happen if we have to update the top
- * PDE. This requires a BO that is almost vm->size big.
- *
- * This shouldn't be possible in practice.. might change when 16K
- * pages are used. Hence the assert.
- */
- xe_tile_assert(tile, update->qwords <= 0x1ff);
- if (!ppgtt_ofs)
- ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile),
- xe_bo_addr(update->pt_bo, 0,
- XE_PAGE_SIZE));
-
- do {
- u64 addr = ppgtt_ofs + ofs * 8;
-
- chunk = min(update->qwords, 0x1ffU);
-
- /* Ensure populatefn can do memset64 by aligning bb->cs */
- if (!(bb->len & 1))
- bb->cs[bb->len++] = MI_NOOP;
-
- bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk);
- bb->cs[bb->len++] = lower_32_bits(addr);
- bb->cs[bb->len++] = upper_32_bits(addr);
- if (pt_op->bind)
- ops->populate(tile, NULL, bb->cs + bb->len,
- ofs, chunk, update);
- else
- ops->clear(vm, tile, NULL, bb->cs + bb->len,
- ofs, chunk, update);
-
- bb->len += chunk * 2;
- ofs += chunk;
- size -= chunk;
- } while (size);
-}
-
struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m)
{
return xe_vm_get(m->q->vm);
@@ -1117,11 +1067,10 @@ struct migrate_test_params {
container_of(_priv, struct migrate_test_params, base)
#endif
-static void
-__xe_migrate_update_pgtables_cpu(struct xe_vm *vm, struct xe_tile *tile,
- const struct xe_migrate_pt_update_ops *ops,
- struct xe_vm_pgtable_update_op *pt_op,
- int num_ops)
+void __xe_migrate_update_pgtables_cpu(struct xe_vm *vm, struct xe_tile *tile,
+ const struct xe_migrate_pt_update_ops *ops,
+ struct xe_vm_pgtable_update_op *pt_op,
+ int num_ops)
{
u32 j, i;
@@ -1182,131 +1131,16 @@ __xe_migrate_update_pgtables(struct xe_migrate *m,
{
const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
struct xe_tile *tile = m->tile;
- struct xe_gt *gt = tile->primary_gt;
- struct xe_device *xe = tile_to_xe(tile);
struct xe_sched_job *job;
struct dma_fence *fence;
- struct drm_suballoc *sa_bo = NULL;
- struct xe_bb *bb;
- u32 i, j, batch_size = 0, ppgtt_ofs, update_idx, page_ofs = 0;
- u32 num_updates = 0, current_update = 0;
- u64 addr;
- int err = 0;
bool is_migrate = pt_update_ops->q == m->q;
- bool usm = is_migrate && xe->info.supports_usm;
-
- for (i = 0; i < pt_update_ops->num_ops; ++i) {
- struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
- struct xe_vm_pgtable_update *updates = pt_op->entries;
-
- num_updates += pt_op->num_entries;
- for (j = 0; j < pt_op->num_entries; ++j) {
- u32 num_cmds = DIV_ROUND_UP(updates[j].qwords, 0x1ff);
-
- /* align noop + MI_STORE_DATA_IMM cmd prefix */
- batch_size += 4 * num_cmds + updates[j].qwords * 2;
- }
- }
-
- /* fixed + PTE entries */
- if (IS_DGFX(xe))
- batch_size += 2;
- else
- batch_size += 6 + num_updates * 2;
-
- bb = xe_bb_new(gt, batch_size, usm);
- if (IS_ERR(bb))
- return ERR_CAST(bb);
-
- /* For sysmem PTE's, need to map them in our hole.. */
- if (!IS_DGFX(xe)) {
- ppgtt_ofs = NUM_KERNEL_PDE - 1;
- if (!is_migrate) {
- u32 num_units = DIV_ROUND_UP(num_updates,
- NUM_VMUSA_WRITES_PER_UNIT);
-
- sa_bo = drm_suballoc_new(&m->vm_update_sa, num_units,
- GFP_KERNEL, true, 0);
- if (IS_ERR(sa_bo)) {
- err = PTR_ERR(sa_bo);
- goto err;
- }
-
- ppgtt_ofs = NUM_KERNEL_PDE +
- (drm_suballoc_soffset(sa_bo) /
- NUM_VMUSA_UNIT_PER_PAGE);
- page_ofs = (drm_suballoc_soffset(sa_bo) %
- NUM_VMUSA_UNIT_PER_PAGE) *
- VM_SA_UPDATE_UNIT_SIZE;
- }
-
- /* Map our PT's to gtt */
- bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(num_updates);
- bb->cs[bb->len++] = ppgtt_ofs * XE_PAGE_SIZE + page_ofs;
- bb->cs[bb->len++] = 0; /* upper_32_bits */
-
- for (i = 0; i < pt_update_ops->num_ops; ++i) {
- struct xe_vm_pgtable_update_op *pt_op =
- &pt_update_ops->ops[i];
- struct xe_vm_pgtable_update *updates = pt_op->entries;
-
- for (j = 0; j < pt_op->num_entries; ++j, ++current_update) {
- struct xe_vm *vm = pt_update->vops->vm;
- struct xe_bo *pt_bo = updates[j].pt_bo;
-
- xe_tile_assert(tile, pt_bo->size == SZ_4K);
-
- /* Map a PT at most once */
- if (pt_bo->update_index < 0)
- pt_bo->update_index = current_update;
-
- addr = vm->pt_ops->pte_encode_bo(pt_bo, 0,
- XE_CACHE_WB, 0);
- bb->cs[bb->len++] = lower_32_bits(addr);
- bb->cs[bb->len++] = upper_32_bits(addr);
- }
- }
-
- bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
- update_idx = bb->len;
-
- addr = xe_migrate_vm_addr(ppgtt_ofs, 0) +
- (page_ofs / sizeof(u64)) * XE_PAGE_SIZE;
- for (i = 0; i < pt_update_ops->num_ops; ++i) {
- struct xe_vm_pgtable_update_op *pt_op =
- &pt_update_ops->ops[i];
- struct xe_vm_pgtable_update *updates = pt_op->entries;
-
- for (j = 0; j < pt_op->num_entries; ++j) {
- struct xe_bo *pt_bo = updates[j].pt_bo;
-
- write_pgtable(tile, bb, addr +
- pt_bo->update_index * XE_PAGE_SIZE,
- pt_op, &updates[j], pt_update);
- }
- }
- } else {
- /* phys pages, no preamble required */
- bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
- update_idx = bb->len;
-
- for (i = 0; i < pt_update_ops->num_ops; ++i) {
- struct xe_vm_pgtable_update_op *pt_op =
- &pt_update_ops->ops[i];
- struct xe_vm_pgtable_update *updates = pt_op->entries;
-
- for (j = 0; j < pt_op->num_entries; ++j)
- write_pgtable(tile, bb, 0, pt_op, &updates[j],
- pt_update);
- }
- }
+ u64 batch_addr[2] = {0, 0};
+ int err;
if (is_migrate)
mutex_lock(&m->job_mutex);
- job = xe_bb_create_migration_job(pt_update_ops->q, bb,
- xe_migrate_batch_base(m, usm),
- update_idx);
+ job = xe_sched_job_create(pt_update_ops->q, batch_addr);
if (IS_ERR(job)) {
err = PTR_ERR(job);
goto err_bb;
@@ -1318,6 +1152,21 @@ __xe_migrate_update_pgtables(struct xe_migrate *m,
if (err)
goto err_job;
}
+
+
+ set_bit(JOB_FLAG_PT, &job->fence->flags);
+ job->pt_update[0].vm = pt_update->vops->vm;
+ job->pt_update[0].tile = tile;
+ job->pt_update[0].ops = ops;
+ job->pt_update[0].pt_op = pt_update_ops->ops;
+ job->pt_update[0].num_ops = pt_update_ops->num_ops;
+ job->pt_update[0].deferred = pt_update_ops->deferred;
+
+ /* Submission backend now owns freeing of pt_update_ops->ops */
+ init_llist_head(&pt_update_ops->deferred);
+ pt_update_ops->skip_free = true;
+ pt_update_ops->ops = NULL;
+
xe_sched_job_arm(job);
fence = dma_fence_get(&job->drm.s_fence->finished);
xe_sched_job_push(job);
@@ -1325,9 +1174,6 @@ __xe_migrate_update_pgtables(struct xe_migrate *m,
if (is_migrate)
mutex_unlock(&m->job_mutex);
- xe_bb_free(bb, fence);
- drm_suballoc_free(sa_bo, fence);
-
return fence;
err_job:
@@ -1335,9 +1181,6 @@ __xe_migrate_update_pgtables(struct xe_migrate *m,
err_bb:
if (is_migrate)
mutex_unlock(&m->job_mutex);
- xe_bb_free(bb, NULL);
-err:
- drm_suballoc_free(sa_bo, NULL);
return ERR_PTR(err);
}
diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h
index 805c780d3ab6..0b8f307d3970 100644
--- a/drivers/gpu/drm/xe/xe_migrate.h
+++ b/drivers/gpu/drm/xe/xe_migrate.h
@@ -22,6 +22,7 @@ struct xe_pt;
struct xe_tile;
struct xe_vm;
struct xe_vm_pgtable_update;
+struct xe_vm_pgtable_update_op;
struct xe_vma;
/**
@@ -105,6 +106,11 @@ struct dma_fence *xe_migrate_clear(struct xe_migrate *m,
struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m);
+void __xe_migrate_update_pgtables_cpu(struct xe_vm *vm, struct xe_tile *tile,
+ const struct xe_migrate_pt_update_ops *ops,
+ struct xe_vm_pgtable_update_op *pt_op,
+ int num_ops);
+
struct dma_fence *
xe_migrate_update_pgtables(struct xe_migrate *m,
struct xe_migrate_pt_update *pt_update);
diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index 7d9bd7d1dabb..fdb307e81cd6 100644
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -366,6 +366,7 @@ xe_pt_new_shared(struct xe_walk_update *wupd, struct xe_pt *parent,
entry->pt = parent;
entry->flags = 0;
entry->qwords = 0;
+ entry->level = parent->level;
entry->pt_bo->update_index = -1;
if (alloc_entries) {
@@ -1459,7 +1460,7 @@ xe_migrate_clear_pgtable_callback(struct xe_vm *vm, struct xe_tile *tile,
u32 qword_ofs, u32 num_qwords,
const struct xe_vm_pgtable_update *update)
{
- u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level);
+ u64 empty = __xe_pt_empty_pte(tile, vm, update->level);
int i;
if (map && map->is_iomem)
@@ -1865,31 +1866,43 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
return ERR_PTR(err);
}
+void xe_pt_update_ops_free(struct xe_vm_pgtable_update_op *pt_op, u32 num_ops)
+{
+ u32 i;
+
+ for (i = 0; i < num_ops; ++i, ++pt_op)
+ if (pt_op->bind)
+ xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
+}
+
void xe_pt_update_ops_commit(struct xe_tile *tile, struct xe_vma_ops *vops)
{
struct xe_vm_pgtable_update_ops *pt_update_ops =
&vops->pt_update_ops[tile->id];
- int i;
lockdep_assert_held(&vops->vm->lock);
xe_vm_assert_held(vops->vm);
/* FIXME: Not 100% correct */
- for (i = 0; i < pt_update_ops->num_ops; ++i) {
- struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
- if (pt_op->bind)
- xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
- }
- xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
+ xe_bo_put_commit(tile_to_xe(tile), &pt_update_ops->deferred);
+ if (!pt_update_ops->skip_free)
+ xe_pt_update_ops_free(pt_update_ops->ops,
+ pt_update_ops->num_ops);
}
void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops)
{
+ struct xe_vm_pgtable_update_ops *pt_update_ops =
+ &vops->pt_update_ops[tile->id];
+
lockdep_assert_held(&vops->vm->lock);
xe_vm_assert_held(vops->vm);
/* FIXME: Just kill VM for now + cleanup PTs */
- xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
+ xe_bo_put_commit(tile_to_xe(tile), &pt_update_ops->deferred);
+ if (!pt_update_ops->skip_free)
+ xe_pt_update_ops_free(pt_update_ops->ops,
+ pt_update_ops->num_ops);
xe_vm_kill(vops->vm, false);
}
diff --git a/drivers/gpu/drm/xe/xe_pt.h b/drivers/gpu/drm/xe/xe_pt.h
index 547d0624685b..4f474b0bd2aa 100644
--- a/drivers/gpu/drm/xe/xe_pt.h
+++ b/drivers/gpu/drm/xe/xe_pt.h
@@ -40,6 +40,7 @@ struct dma_fence *xe_pt_update_ops_run(struct xe_tile *tile,
struct xe_vma_ops *vops);
void xe_pt_update_ops_commit(struct xe_tile *tile, struct xe_vma_ops *vops);
void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops);
+void xe_pt_update_ops_free(struct xe_vm_pgtable_update_op *pt_op, u32 num_ops);
struct dma_fence *
__xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q,
diff --git a/drivers/gpu/drm/xe/xe_pt_types.h b/drivers/gpu/drm/xe/xe_pt_types.h
index f4d308ce7985..46fbb4325400 100644
--- a/drivers/gpu/drm/xe/xe_pt_types.h
+++ b/drivers/gpu/drm/xe/xe_pt_types.h
@@ -69,6 +69,9 @@ struct xe_vm_pgtable_update {
/** @pt_entries: Newly added pagetable entries */
struct xe_pt_entry *pt_entries;
+ /** @level: level of update */
+ unsigned int level;
+
/** @flags: Target flags */
u32 flags;
};
@@ -117,6 +120,8 @@ struct xe_vm_pgtable_update_ops {
* slots are idle.
*/
bool wait_vm_kernel;
+ /** @skip_free: Free @ops in submission backend rather than in IOCTL */
+ bool skip_free;
};
#endif
diff --git a/drivers/gpu/drm/xe/xe_sched_job.c b/drivers/gpu/drm/xe/xe_sched_job.c
index adbd82f8744e..0e28532c5048 100644
--- a/drivers/gpu/drm/xe/xe_sched_job.c
+++ b/drivers/gpu/drm/xe/xe_sched_job.c
@@ -34,8 +34,10 @@ int __init xe_sched_job_module_init(void)
xe_sched_job_parallel_slab =
kmem_cache_create("xe_sched_job_parallel",
sizeof(struct xe_sched_job) +
+ max_t(size_t,
sizeof(u64) *
- XE_HW_ENGINE_MAX_INSTANCE, 0,
+ XE_HW_ENGINE_MAX_INSTANCE,
+ sizeof(struct pt_update_args)), 0,
SLAB_HWCACHE_ALIGN, NULL);
if (!xe_sched_job_parallel_slab) {
kmem_cache_destroy(xe_sched_job_slab);
diff --git a/drivers/gpu/drm/xe/xe_sched_job_types.h b/drivers/gpu/drm/xe/xe_sched_job_types.h
index 71213ba9735b..b2d986e87ef3 100644
--- a/drivers/gpu/drm/xe/xe_sched_job_types.h
+++ b/drivers/gpu/drm/xe/xe_sched_job_types.h
@@ -11,6 +11,28 @@
#include <drm/gpu_scheduler.h>
struct xe_exec_queue;
+struct xe_migrate_pt_update_ops;
+struct xe_tile;
+struct xe_vm;
+struct xe_vm_pgtable_update_op;
+
+/**
+ * struct pt_update_args - PT update arguments
+ */
+struct pt_update_args {
+ /** @vm: VM */
+ struct xe_vm *vm;
+ /** @tile: Tile */
+ struct xe_tile *tile;
+ /** @ops: Migrate PT update ops */
+ const struct xe_migrate_pt_update_ops *ops;
+ /** @pt_op: PT update ops */
+ struct xe_vm_pgtable_update_op *pt_op;
+ /** @deferred: deferred list to destroy PT entries */
+ struct llist_head deferred;
+ /** @num_ops: number of PT update ops */
+ int num_ops;
+};
/**
* struct xe_sched_job - XE schedule job (batch buffer tracking)
@@ -27,6 +49,7 @@ struct xe_sched_job {
* can safely reference fence, fence cannot safely reference job.
*/
#define JOB_FLAG_SUBMIT DMA_FENCE_FLAG_USER_BITS
+#define JOB_FLAG_PT (DMA_FENCE_FLAG_USER_BITS << 1)
struct dma_fence *fence;
/** @user_fence: write back value when BB is complete */
struct {
@@ -39,8 +62,12 @@ struct xe_sched_job {
} user_fence;
/** @migrate_flush_flags: Additional flush flags for migration jobs */
u32 migrate_flush_flags;
- /** @batch_addr: batch buffer address of job */
- u64 batch_addr[];
+ union {
+ /** @batch_addr: batch buffer address of job */
+ u64 batch_addr[0];
+ /** @pt_update: PT update arguments */
+ struct pt_update_args pt_update[0];
+ };
};
#endif
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 7d53b266a116..52f563580fc3 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -825,7 +825,46 @@ int xe_vm_userptr_check_repin(struct xe_vm *vm)
list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN;
}
-void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma,
+static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm,
+ struct xe_exec_queue *q,
+ struct xe_sync_entry *syncs, u32 num_syncs)
+{
+ memset(vops, 0, sizeof(*vops));
+ INIT_LIST_HEAD(&vops->list);
+ vops->vm = vm;
+ vops->q = q;
+ vops->syncs = syncs;
+ vops->num_syncs = num_syncs;
+}
+
+static int xe_vma_ops_alloc(struct xe_vma_ops *vops)
+{
+ int i;
+
+ for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) {
+ if (!vops->pt_update_ops[i].num_ops)
+ continue;
+
+ vops->pt_update_ops[i].ops =
+ kmalloc_array(vops->pt_update_ops[i].num_ops,
+ sizeof(*vops->pt_update_ops[i].ops),
+ GFP_KERNEL);
+ if (!vops->pt_update_ops[i].ops)
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+void xe_vma_ops_free(struct xe_vma_ops *vops)
+{
+ int i;
+
+ for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
+ kfree(vops->pt_update_ops[i].ops);
+}
+
+int xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma,
u8 tile_mask)
{
int i;
@@ -860,12 +899,15 @@ void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma,
vm->dummy_ops.op.map.immediate = true;
vm->dummy_ops.op.map.read_only = xe_vma_read_only(vma);
vm->dummy_ops.op.map.is_null = xe_vma_is_null(vma);
+
+ return xe_vma_ops_alloc(&vm->dummy_ops.vops);
}
struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
{
struct dma_fence *fence = NULL;
struct xe_vma *vma, *next;
+ int err;
lockdep_assert_held(&vm->lock);
if (xe_vm_no_dma_fences(vm) && !rebind_worker)
@@ -883,8 +925,12 @@ struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
else
trace_xe_vma_rebind_exec(vma);
- xe_vm_populate_dummy_rebind(vm, vma, vma->tile_present);
+ err = xe_vm_populate_dummy_rebind(vm, vma, vma->tile_present);
+ if (err)
+ return ERR_PTR(err);
+
fence = xe_vm_ops_execute(vm, &vm->dummy_ops.vops);
+ xe_vma_ops_free(&vm->dummy_ops.vops);
if (IS_ERR(fence))
return fence;
}
@@ -1355,45 +1401,6 @@ static const struct xe_pt_ops xelp_pt_ops = {
static void vm_destroy_work_func(struct work_struct *w);
-static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm,
- struct xe_exec_queue *q,
- struct xe_sync_entry *syncs, u32 num_syncs)
-{
- memset(vops, 0, sizeof(*vops));
- INIT_LIST_HEAD(&vops->list);
- vops->vm = vm;
- vops->q = q;
- vops->syncs = syncs;
- vops->num_syncs = num_syncs;
-}
-
-static int xe_vma_ops_alloc(struct xe_vma_ops *vops)
-{
- int i;
-
- for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) {
- if (!vops->pt_update_ops[i].num_ops)
- continue;
-
- vops->pt_update_ops[i].ops =
- kmalloc_array(vops->pt_update_ops[i].num_ops,
- sizeof(*vops->pt_update_ops[i].ops),
- GFP_KERNEL);
- if (!vops->pt_update_ops[i].ops)
- return -ENOMEM;
- }
-
- return 0;
-}
-
-static void xe_vma_ops_fini(struct xe_vma_ops *vops)
-{
- int i;
-
- for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
- kfree(vops->pt_update_ops[i].ops);
-}
-
static void xe_vma_ops_incr_pt_update_ops(struct xe_vma_ops *vops, u8 tile_mask)
{
int i;
@@ -1429,9 +1436,6 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
list_add(&vm->dummy_ops.op.link, &vm->dummy_ops.vops.list);
for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
vm->dummy_ops.vops.pt_update_ops[i].num_ops = 1;
- err = xe_vma_ops_alloc(&vm->dummy_ops.vops);
- if (err)
- goto err_free;
INIT_LIST_HEAD(&vm->rebind_list);
@@ -1574,8 +1578,6 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
dma_resv_fini(&vm->resv);
for_each_tile(tile, xe, id)
xe_range_fence_tree_fini(&vm->rftree[id]);
-err_free:
- xe_vma_ops_fini(&vm->dummy_ops.vops);
kfree(vm);
return ERR_PTR(err);
}
@@ -1729,7 +1731,6 @@ static void vm_destroy_work_func(struct work_struct *w)
trace_xe_vm_free(vm);
dma_resv_fini(&vm->resv);
- xe_vma_ops_fini(&vm->dummy_ops.vops);
kfree(vm);
}
@@ -2968,7 +2969,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
unwind_ops:
if (err && err != -ENODATA)
vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds);
- xe_vma_ops_fini(&vops);
+ xe_vma_ops_free(&vops);
for (i = args->num_binds - 1; i >= 0; --i)
if (ops[i])
drm_gpuva_ops_free(&vm->gpuvm, ops[i]);
diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h
index 34850f04d966..a61e23ea5b1a 100644
--- a/drivers/gpu/drm/xe/xe_vm.h
+++ b/drivers/gpu/drm/xe/xe_vm.h
@@ -224,8 +224,9 @@ int xe_analyze_vm(struct drm_printer *p, struct xe_vm *vm, int gt_id);
int xe_vm_prepare_vma(struct drm_exec *exec, struct xe_vma *vma,
unsigned int num_shared);
-void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma,
- u8 tile_mask);
+int xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma,
+ u8 tile_mask);
+void xe_vma_ops_free(struct xe_vma_ops *vops);
struct dma_fence *xe_vm_ops_execute(struct xe_vm *vm, struct xe_vma_ops *vops);
void xe_vm_kill(struct xe_vm *vm, bool unlocked);
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 26/27] drm/xe: Don't use migrate exec queue for page fault binds
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (24 preceding siblings ...)
2023-11-07 5:26 ` [Intel-xe] [PATCH v2 25/27] drm/xe: CPU binds for jobs Matthew Brost
@ 2023-11-07 5:26 ` Matthew Brost
2023-11-07 5:26 ` [Intel-xe] [PATCH v2 27/27] drm/xe/uapi: Make sync vs async VM bind operations per IOCTL rather than queue Matthew Brost
` (4 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:26 UTC (permalink / raw)
To: intel-xe
Now that the CPU is always used for binds even in jobs, CPU bind jobs
can pass GPU jobs in the same exec queue resulting dma-fences signaling
out-of-order. Use a dedicated exec queue for binds issued from page
faults to avoid ordering issues.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_gt_pagefault.c | 2 +-
drivers/gpu/drm/xe/xe_migrate.c | 22 +++++++++++++++++++++-
drivers/gpu/drm/xe/xe_migrate.h | 1 +
3 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c
index 3a3b06d6a4b0..b97d5b962152 100644
--- a/drivers/gpu/drm/xe/xe_gt_pagefault.c
+++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c
@@ -207,7 +207,7 @@ static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
if (ret)
goto unlock_dma_resv;
vm->dummy_ops.vops.pt_update_ops[tile->id].q =
- xe_tile_migrate_exec_queue(tile);
+ xe_tile_migrate_bind_exec_queue(tile);
fence = xe_vm_ops_execute(vm, &vm->dummy_ops.vops);
xe_vma_ops_free(&vm->dummy_ops.vops);
if (IS_ERR(fence)) {
diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c
index d56448dbf135..59cae1181d31 100644
--- a/drivers/gpu/drm/xe/xe_migrate.c
+++ b/drivers/gpu/drm/xe/xe_migrate.c
@@ -40,6 +40,8 @@
struct xe_migrate {
/** @q: Default exec queue used for migration */
struct xe_exec_queue *q;
+ /** @bind_q: Default exec queue used for binds */
+ struct xe_exec_queue *bind_q;
/** @tile: Backpointer to the tile this struct xe_migrate belongs to. */
struct xe_tile *tile;
/** @job_mutex: Timeline mutex for @eng. */
@@ -87,6 +89,11 @@ struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile)
return tile->migrate->q;
}
+struct xe_exec_queue *xe_tile_migrate_bind_exec_queue(struct xe_tile *tile)
+{
+ return tile->migrate->bind_q;
+}
+
static void xe_migrate_fini(struct drm_device *dev, void *arg)
{
struct xe_migrate *m = arg;
@@ -105,6 +112,8 @@ static void xe_migrate_fini(struct drm_device *dev, void *arg)
mutex_destroy(&m->job_mutex);
xe_vm_close_and_put(m->q->vm);
xe_exec_queue_put(m->q);
+ if (m->bind_q)
+ xe_exec_queue_put(m->bind_q);
}
static u64 xe_migrate_vm_addr(u64 slot, u32 level)
@@ -381,6 +390,15 @@ struct xe_migrate *xe_migrate_init(struct xe_tile *tile)
if (!hwe || !logical_mask)
return ERR_PTR(-EINVAL);
+ m->bind_q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe,
+ EXEC_QUEUE_FLAG_KERNEL |
+ EXEC_QUEUE_FLAG_PERMANENT);
+ if (IS_ERR(m->bind_q)) {
+ xe_vm_close_and_put(vm);
+ return ERR_CAST(m->bind_q);
+ }
+ m->bind_q->entity->priority = DRM_SCHED_PRIORITY_KERNEL;
+
m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe,
EXEC_QUEUE_FLAG_KERNEL |
EXEC_QUEUE_FLAG_PERMANENT);
@@ -391,6 +409,8 @@ struct xe_migrate *xe_migrate_init(struct xe_tile *tile)
EXEC_QUEUE_FLAG_PERMANENT);
}
if (IS_ERR(m->q)) {
+ if (m->bind_q)
+ xe_exec_queue_put(m->bind_q);
xe_vm_close_and_put(vm);
return ERR_CAST(m->q);
}
@@ -1133,7 +1153,7 @@ __xe_migrate_update_pgtables(struct xe_migrate *m,
struct xe_tile *tile = m->tile;
struct xe_sched_job *job;
struct dma_fence *fence;
- bool is_migrate = pt_update_ops->q == m->q;
+ bool is_migrate = pt_update_ops->q == m->bind_q;
u64 batch_addr[2] = {0, 0};
int err;
diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h
index 0b8f307d3970..8e5ce03cdf1f 100644
--- a/drivers/gpu/drm/xe/xe_migrate.h
+++ b/drivers/gpu/drm/xe/xe_migrate.h
@@ -118,5 +118,6 @@ xe_migrate_update_pgtables(struct xe_migrate *m,
void xe_migrate_wait(struct xe_migrate *m);
struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile);
+struct xe_exec_queue *xe_tile_migrate_bind_exec_queue(struct xe_tile *tile);
#endif
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] [PATCH v2 27/27] drm/xe/uapi: Make sync vs async VM bind operations per IOCTL rather than queue
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (25 preceding siblings ...)
2023-11-07 5:26 ` [Intel-xe] [PATCH v2 26/27] drm/xe: Don't use migrate exec queue for page fault binds Matthew Brost
@ 2023-11-07 5:26 ` Matthew Brost
2023-11-07 5:29 ` [Intel-xe] ✓ CI.Patch_applied: success for Refactor VM bind code (rev2) Patchwork
` (3 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Matthew Brost @ 2023-11-07 5:26 UTC (permalink / raw)
To: intel-xe
Making sync vs async per IOCTL simplifies the uAPI while also making it
more flexible. Adjust the uAPI accordingly.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
drivers/gpu/drm/xe/xe_exec_queue.c | 7 +---
drivers/gpu/drm/xe/xe_exec_queue_types.h | 2 -
drivers/gpu/drm/xe/xe_vm.c | 53 ++++++++----------------
drivers/gpu/drm/xe/xe_vm_types.h | 15 +++----
include/uapi/drm/xe_drm.h | 6 +--
5 files changed, 28 insertions(+), 55 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c
index 6d142fb75dfc..57a4b688505f 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue.c
+++ b/drivers/gpu/drm/xe/xe_exec_queue.c
@@ -625,10 +625,7 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
if (XE_IOCTL_DBG(xe, eci[0].gt_id >= xe->info.gt_count))
return -EINVAL;
- if (eci[0].engine_class >= DRM_XE_ENGINE_CLASS_VM_BIND_ASYNC) {
- bool sync = eci[0].engine_class ==
- DRM_XE_ENGINE_CLASS_VM_BIND_SYNC;
-
+ if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) {
for_each_gt(gt, xe, id) {
struct xe_exec_queue *new;
@@ -654,8 +651,6 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
args->width, hwe,
EXEC_QUEUE_FLAG_PERSISTENT |
EXEC_QUEUE_FLAG_VM |
- (sync ? 0 :
- EXEC_QUEUE_FLAG_VM_ASYNC) |
(id ?
EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD :
0));
diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h
index 35ffe7c55f25..91c1f243aad4 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue_types.h
+++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h
@@ -71,8 +71,6 @@ struct xe_exec_queue {
#define EXEC_QUEUE_FLAG_VM BIT(4)
/* child of VM queue for multi-tile VM jobs */
#define EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD BIT(5)
-/* VM jobs for this queue are asynchronous */
-#define EXEC_QUEUE_FLAG_VM_ASYNC BIT(6)
/**
* @flags: flags for this exec queue, should statically setup aside from ban
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 52f563580fc3..9ec0896e5ac6 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -827,7 +827,8 @@ int xe_vm_userptr_check_repin(struct xe_vm *vm)
static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm,
struct xe_exec_queue *q,
- struct xe_sync_entry *syncs, u32 num_syncs)
+ struct xe_sync_entry *syncs, u32 num_syncs,
+ bool async)
{
memset(vops, 0, sizeof(*vops));
INIT_LIST_HEAD(&vops->list);
@@ -835,6 +836,7 @@ static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm,
vops->q = q;
vops->syncs = syncs;
vops->num_syncs = num_syncs;
+ vops->async = async;
}
static int xe_vma_ops_alloc(struct xe_vma_ops *vops)
@@ -1431,7 +1433,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
init_rwsem(&vm->lock);
- xe_vma_ops_init(&vm->dummy_ops.vops, vm, NULL, NULL, 0);
+ xe_vma_ops_init(&vm->dummy_ops.vops, vm, NULL, NULL, 0, false);
INIT_LIST_HEAD(&vm->dummy_ops.op.link);
list_add(&vm->dummy_ops.op.link, &vm->dummy_ops.vops.list);
for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
@@ -1517,9 +1519,6 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
struct xe_gt *gt = tile->primary_gt;
struct xe_vm *migrate_vm;
struct xe_exec_queue *q;
- u32 create_flags = EXEC_QUEUE_FLAG_VM |
- ((flags & XE_VM_FLAG_ASYNC_DEFAULT) ?
- EXEC_QUEUE_FLAG_VM_ASYNC : 0);
if (!vm->pt_root[id])
continue;
@@ -1527,7 +1526,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
migrate_vm = xe_migrate_get_vm(tile->migrate);
q = xe_exec_queue_create_class(xe, gt, migrate_vm,
XE_ENGINE_CLASS_COPY,
- create_flags);
+ EXEC_QUEUE_FLAG_VM);
xe_vm_put(migrate_vm);
if (IS_ERR(q)) {
xe_vm_close_and_put(vm);
@@ -1767,15 +1766,8 @@ to_wait_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
return q ? q : vm->q[0];
}
-static bool xe_vm_sync_mode(struct xe_vm *vm, struct xe_exec_queue *q)
-{
- return q ? !(q->flags & EXEC_QUEUE_FLAG_VM_ASYNC) :
- !(vm->flags & XE_VM_FLAG_ASYNC_DEFAULT);
-}
-
#define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_SCRATCH_PAGE | \
DRM_XE_VM_CREATE_COMPUTE_MODE | \
- DRM_XE_VM_CREATE_ASYNC_DEFAULT | \
DRM_XE_VM_CREATE_FAULT_MODE)
int xe_vm_create_ioctl(struct drm_device *dev, void *data,
@@ -1829,8 +1821,6 @@ int xe_vm_create_ioctl(struct drm_device *dev, void *data,
flags |= XE_VM_FLAG_SCRATCH_PAGE;
if (args->flags & DRM_XE_VM_CREATE_COMPUTE_MODE)
flags |= XE_VM_FLAG_COMPUTE_MODE;
- if (args->flags & DRM_XE_VM_CREATE_ASYNC_DEFAULT)
- flags |= XE_VM_FLAG_ASYNC_DEFAULT;
if (args->flags & DRM_XE_VM_CREATE_FAULT_MODE)
flags |= XE_VM_FLAG_FAULT_MODE;
@@ -2605,12 +2595,13 @@ struct dma_fence *xe_vm_ops_execute(struct xe_vm *vm, struct xe_vma_ops *vops)
return fence;
}
-static void vm_bind_ioctl_ops_install_fences(struct xe_vm *vm,
- struct xe_vma_ops *vops,
- struct dma_fence *fence)
+static int vm_bind_ioctl_ops_install_fences(struct xe_vm *vm,
+ struct xe_vma_ops *vops,
+ struct dma_fence *fence)
{
struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, vops->q);
struct xe_vma_op *op;
+ long timeout = 0;
int i;
list_for_each_entry(op, &vops->list, link) {
@@ -2623,9 +2614,13 @@ static void vm_bind_ioctl_ops_install_fences(struct xe_vm *vm,
for (i = 0; i < vops->num_syncs; i++)
xe_sync_entry_signal(vops->syncs + i, NULL, fence);
xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence);
- if (xe_vm_sync_mode(vm, vops->q))
- dma_fence_wait(fence, true);
+ if (!vops->async)
+ timeout = dma_fence_wait(fence, true);
dma_fence_put(fence);
+
+ if (timeout < 0)
+ return -EINTR;
+ return 0;
}
static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
@@ -2648,7 +2643,7 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
if (IS_ERR(fence))
return PTR_ERR(fence);
- vm_bind_ioctl_ops_install_fences(vm, vops, fence);
+ err = vm_bind_ioctl_ops_install_fences(vm, vops, fence);
}
drm_exec_fini(&exec);
@@ -2805,12 +2800,6 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
err = -EINVAL;
goto put_exec_queue;
}
-
- if (XE_IOCTL_DBG(xe, args->num_binds && async !=
- !!(q->flags & EXEC_QUEUE_FLAG_VM_ASYNC))) {
- err = -EINVAL;
- goto put_exec_queue;
- }
}
vm = xe_vm_lookup(xef, args->vm_id);
@@ -2819,14 +2808,6 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
goto put_exec_queue;
}
- if (!args->exec_queue_id) {
- if (XE_IOCTL_DBG(xe, args->num_binds && async !=
- !!(vm->flags & XE_VM_FLAG_ASYNC_DEFAULT))) {
- err = -EINVAL;
- goto put_vm;
- }
- }
-
err = down_write_killable(&vm->lock);
if (err)
goto put_vm;
@@ -2928,7 +2909,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
goto free_syncs;
}
- xe_vma_ops_init(&vops, vm, q, syncs, num_syncs);
+ xe_vma_ops_init(&vops, vm, q, syncs, num_syncs, async);
for (i = 0; i < args->num_binds; ++i) {
u64 range = bind_ops[i].range;
u64 addr = bind_ops[i].addr;
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index adbd8199aa8b..2ff904edb583 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -217,6 +217,8 @@ struct xe_vma_ops {
struct xe_sync_entry *syncs;
/** @num_syncs: number of syncs */
u32 num_syncs;
+ /** @async: VMA operations are asynchronous */
+ bool async;
/** @pt_update_ops: page table update operations */
struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE];
};
@@ -252,13 +254,12 @@ struct xe_vm {
*/
#define XE_VM_FLAG_64K BIT(0)
#define XE_VM_FLAG_COMPUTE_MODE BIT(1)
-#define XE_VM_FLAG_ASYNC_DEFAULT BIT(2)
-#define XE_VM_FLAG_MIGRATION BIT(3)
-#define XE_VM_FLAG_SCRATCH_PAGE BIT(4)
-#define XE_VM_FLAG_FAULT_MODE BIT(5)
-#define XE_VM_FLAG_BANNED BIT(6)
-#define XE_VM_FLAG_TILE_ID(flags) FIELD_GET(GENMASK(8, 7), flags)
-#define XE_VM_FLAG_SET_TILE_ID(tile) FIELD_PREP(GENMASK(8, 7), (tile)->id)
+#define XE_VM_FLAG_MIGRATION BIT(2)
+#define XE_VM_FLAG_SCRATCH_PAGE BIT(3)
+#define XE_VM_FLAG_FAULT_MODE BIT(4)
+#define XE_VM_FLAG_BANNED BIT(5)
+#define XE_VM_FLAG_TILE_ID(flags) FIELD_GET(GENMASK(7, 6), flags)
+#define XE_VM_FLAG_SET_TILE_ID(tile) FIELD_PREP(GENMASK(7, 6), (tile)->id)
unsigned long flags;
/** @composite_fence_ctx: context composite fence */
diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h
index 9bd7092a7ea4..78a346960240 100644
--- a/include/uapi/drm/xe_drm.h
+++ b/include/uapi/drm/xe_drm.h
@@ -137,8 +137,7 @@ struct drm_xe_engine_class_instance {
* Kernel only classes (not actual hardware engine class). Used for
* creating ordered queues of VM bind operations.
*/
-#define DRM_XE_ENGINE_CLASS_VM_BIND_ASYNC 5
-#define DRM_XE_ENGINE_CLASS_VM_BIND_SYNC 6
+#define DRM_XE_ENGINE_CLASS_VM_BIND 5
__u16 engine_class;
__u16 engine_instance;
@@ -597,8 +596,7 @@ struct drm_xe_vm_create {
#define DRM_XE_VM_CREATE_SCRATCH_PAGE (0x1 << 0)
#define DRM_XE_VM_CREATE_COMPUTE_MODE (0x1 << 1)
-#define DRM_XE_VM_CREATE_ASYNC_DEFAULT (0x1 << 2)
-#define DRM_XE_VM_CREATE_FAULT_MODE (0x1 << 3)
+#define DRM_XE_VM_CREATE_FAULT_MODE (0x1 << 2)
/** @flags: Flags */
__u32 flags;
--
2.34.1
^ permalink raw reply related [flat|nested] 43+ messages in thread* [Intel-xe] ✓ CI.Patch_applied: success for Refactor VM bind code (rev2)
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (26 preceding siblings ...)
2023-11-07 5:26 ` [Intel-xe] [PATCH v2 27/27] drm/xe/uapi: Make sync vs async VM bind operations per IOCTL rather than queue Matthew Brost
@ 2023-11-07 5:29 ` Patchwork
2023-11-07 5:30 ` [Intel-xe] ✗ CI.checkpatch: warning " Patchwork
` (2 subsequent siblings)
30 siblings, 0 replies; 43+ messages in thread
From: Patchwork @ 2023-11-07 5:29 UTC (permalink / raw)
To: Matthew Brost; +Cc: intel-xe
== Series Details ==
Series: Refactor VM bind code (rev2)
URL : https://patchwork.freedesktop.org/series/125608/
State : success
== Summary ==
=== Applying kernel patches on branch 'drm-xe-next' with base: ===
Base commit: 5a3a6fdda drm/xe: Fix pagefault and access counter worker functions
=== git am output follows ===
Applying: drm/xe: Allow num_binds == 0 in VM bind IOCTL
Applying: drm/xe: Allow num_batch_buffer == 0 in exec IOCTL
Applying: drm/xe: Take in-syncs into account when num_execs or num_binds == 0
Applying: drm/xe: Lock all gpuva ops during VM bind IOCTL
Applying: drm/xe: Add ops_execute function which returns a fence
Applying: drm/xe: Move migrate to prefetch to op_lock funtion
Applying: drm/xe: Add struct xe_vma_ops abstraction
Applying: drm/xe: Update xe_vm_rebind to use dummy VMA operations
Applying: drm/xe: Move drm exec loop to vm_bind_ioctl_ops_execute
Applying: drm/xe: Fixup error handling / ref counting in VM bind IOCTL
Applying: drm/xe: Convert pagefault rebind to use ops interface
Applying: drm/xe: Add some members to xe_vma_ops
Applying: drm/xe: Add vm_bind_ioctl_ops_install_fences helper
Applying: drm/xe: Drop rebind argument from xe_pt_prepare_bind
Applying: drm/xe: Add xe_vm_pgtable_update_op to xe_vma_ops
Applying: drm/xe: Move setting last fence and sync wait to vm_bind_ioctl_ops_install_fences
Applying: drm/xe: Fix vma_is_valid to use tile argument
Applying: drm/xe: Adjust tile mask in operations create
Applying: drm/xe: Add xe_gt_tlb_invalidation_range and convert PT layer to use this
Applying: drm/xe: s/xe_tile_migrate_engine/xe_tile_migrate_exec_queue
Applying: drm/xe: Convert multiple bind ops into single job
Applying: drm/xe: Update clear / populate arguments
Applying: drm/xe: Add __xe_migrate_update_pgtables_cpu helper
Applying: drm/xe: Add xe_hw_fence_signal helper
Applying: drm/xe: CPU binds for jobs
Applying: drm/xe: Don't use migrate exec queue for page fault binds
Applying: drm/xe/uapi: Make sync vs async VM bind operations per IOCTL rather than queue
^ permalink raw reply [flat|nested] 43+ messages in thread* [Intel-xe] ✗ CI.checkpatch: warning for Refactor VM bind code (rev2)
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (27 preceding siblings ...)
2023-11-07 5:29 ` [Intel-xe] ✓ CI.Patch_applied: success for Refactor VM bind code (rev2) Patchwork
@ 2023-11-07 5:30 ` Patchwork
2023-11-07 5:30 ` [Intel-xe] ✗ CI.KUnit: failure " Patchwork
2023-11-08 13:55 ` [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Thomas Hellström
30 siblings, 0 replies; 43+ messages in thread
From: Patchwork @ 2023-11-07 5:30 UTC (permalink / raw)
To: Matthew Brost; +Cc: intel-xe
== Series Details ==
Series: Refactor VM bind code (rev2)
URL : https://patchwork.freedesktop.org/series/125608/
State : warning
== Summary ==
+ KERNEL=/kernel
+ git clone https://gitlab.freedesktop.org/drm/maintainer-tools mt
Cloning into 'mt'...
warning: redirecting to https://gitlab.freedesktop.org/drm/maintainer-tools.git/
+ git -C mt rev-list -n1 origin/master
63c2b6b160bca2df6efc7bc4cea6f442097d7854
+ cd /kernel
+ git config --global --add safe.directory /kernel
+ git log -n1
commit 6f6401348a096a2ac1926c9de123cb085082eb3f
Author: Matthew Brost <matthew.brost@intel.com>
Date: Mon Nov 6 21:26:03 2023 -0800
drm/xe/uapi: Make sync vs async VM bind operations per IOCTL rather than queue
Making sync vs async per IOCTL simplifies the uAPI while also making it
more flexible. Adjust the uAPI accordingly.
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
+ /mt/dim checkpatch 5a3a6fdda31fc8fa60db6d2fa30bd4db2509eb8b drm-intel
cdad5aaa2 drm/xe: Allow num_binds == 0 in VM bind IOCTL
-:52: WARNING:ALLOC_WITH_MULTIPLY: Prefer kcalloc over kzalloc with multiply
#52: FILE: drivers/gpu/drm/xe/xe_vm.c:3031:
+ bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL);
total: 0 errors, 1 warnings, 0 checks, 61 lines checked
14563afdc drm/xe: Allow num_batch_buffer == 0 in exec IOCTL
1294f1135 drm/xe: Take in-syncs into account when num_execs or num_binds == 0
0295ad2c2 drm/xe: Lock all gpuva ops during VM bind IOCTL
878ff3224 drm/xe: Add ops_execute function which returns a fence
3ffe5d6f8 drm/xe: Move migrate to prefetch to op_lock funtion
-:4: WARNING:TYPO_SPELLING: 'funtion' may be misspelled - perhaps 'function'?
#4:
Subject: [PATCH] drm/xe: Move migrate to prefetch to op_lock funtion
^^^^^^^
total: 0 errors, 1 warnings, 0 checks, 57 lines checked
30616d336 drm/xe: Add struct xe_vma_ops abstraction
8d5e31aa3 drm/xe: Update xe_vm_rebind to use dummy VMA operations
956e9da16 drm/xe: Move drm exec loop to vm_bind_ioctl_ops_execute
44bfdfac6 drm/xe: Fixup error handling / ref counting in VM bind IOCTL
e981e197a drm/xe: Convert pagefault rebind to use ops interface
ec09ce4f3 drm/xe: Add some members to xe_vma_ops
ddab35b82 drm/xe: Add vm_bind_ioctl_ops_install_fences helper
52fec586d drm/xe: Drop rebind argument from xe_pt_prepare_bind
e35e30fc0 drm/xe: Add xe_vm_pgtable_update_op to xe_vma_ops
d093f039e drm/xe: Move setting last fence and sync wait to vm_bind_ioctl_ops_install_fences
670bd09fa drm/xe: Fix vma_is_valid to use tile argument
cacd81368 drm/xe: Adjust tile mask in operations create
16927068f drm/xe: Add xe_gt_tlb_invalidation_range and convert PT layer to use this
-:107: CHECK:BRACES: Blank lines aren't necessary before a close brace '}'
#107: FILE: drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c:310:
+
+}
total: 0 errors, 0 warnings, 1 checks, 186 lines checked
8268a260c drm/xe: s/xe_tile_migrate_engine/xe_tile_migrate_exec_queue
d47de46a8 drm/xe: Convert multiple bind ops into single job
8c824b6ff drm/xe: Update clear / populate arguments
8f3dfa745 drm/xe: Add __xe_migrate_update_pgtables_cpu helper
797a05132 drm/xe: Add xe_hw_fence_signal helper
-:40: WARNING:FROM_SIGN_OFF_MISMATCH: From:/Signed-off-by: email address mismatch: 'From: Matthew Brost <matthew.brost@intel.com>' != 'Signed-off-by: Matthew Brost <matthw.brost@intel.com>'
total: 0 errors, 1 warnings, 0 checks, 20 lines checked
1bea95d58 drm/xe: CPU binds for jobs
-:462: CHECK:LINE_SPACING: Please don't use multiple blank lines
#462: FILE: drivers/gpu/drm/xe/xe_migrate.c:1156:
+
+
-:640: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
#640: FILE: drivers/gpu/drm/xe/xe_sched_job.c:38:
+ max_t(size_t,
sizeof(u64) *
-:699: ERROR:FLEXIBLE_ARRAY: Use C99 flexible arrays - see https://docs.kernel.org/process/deprecated.html#zero-length-and-one-element-arrays
#699: FILE: drivers/gpu/drm/xe/xe_sched_job_types.h:70:
+ struct pt_update_args pt_update[0];
+ };
-:752: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
#752: FILE: drivers/gpu/drm/xe/xe_vm.c:868:
+int xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma,
u8 tile_mask)
total: 1 errors, 0 warnings, 3 checks, 767 lines checked
5e80b6708 drm/xe: Don't use migrate exec queue for page fault binds
6f6401348 drm/xe/uapi: Make sync vs async VM bind operations per IOCTL rather than queue
^ permalink raw reply [flat|nested] 43+ messages in thread* [Intel-xe] ✗ CI.KUnit: failure for Refactor VM bind code (rev2)
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (28 preceding siblings ...)
2023-11-07 5:30 ` [Intel-xe] ✗ CI.checkpatch: warning " Patchwork
@ 2023-11-07 5:30 ` Patchwork
2023-11-08 13:55 ` [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Thomas Hellström
30 siblings, 0 replies; 43+ messages in thread
From: Patchwork @ 2023-11-07 5:30 UTC (permalink / raw)
To: Matthew Brost; +Cc: intel-xe
== Series Details ==
Series: Refactor VM bind code (rev2)
URL : https://patchwork.freedesktop.org/series/125608/
State : failure
== Summary ==
+ trap cleanup EXIT
+ /kernel/tools/testing/kunit/kunit.py run --kunitconfig /kernel/drivers/gpu/drm/xe/.kunitconfig
ERROR:root:In file included from ../drivers/gpu/drm/xe/xe_migrate.c:1254:
../drivers/gpu/drm/xe/tests/xe_migrate.c:91:14: error: initialization of ‘void (*)(struct xe_tile *, struct iosys_map *, void *, u32, u32, const struct xe_vm_pgtable_update *)’ {aka ‘void (*)(struct xe_tile *, struct iosys_map *, void *, unsigned int, unsigned int, const struct xe_vm_pgtable_update *)’} from incompatible pointer type ‘void (*)(struct xe_migrate_pt_update *, struct xe_tile *, struct iosys_map *, void *, u32, u32, const struct xe_vm_pgtable_update *)’ {aka ‘void (*)(struct xe_migrate_pt_update *, struct xe_tile *, struct iosys_map *, void *, unsigned int, unsigned int, const struct xe_vm_pgtable_update *)’} [-Werror=incompatible-pointer-types]
91 | .populate = sanity_populate_cb,
| ^~~~~~~~~~~~~~~~~~
../drivers/gpu/drm/xe/tests/xe_migrate.c:91:14: note: (near initialization for ‘sanity_ops.populate’)
../drivers/gpu/drm/xe/tests/xe_migrate.c: In function ‘test_pt_update’:
../drivers/gpu/drm/xe/tests/xe_migrate.c:239:10: error: too many arguments to function ‘xe_migrate_update_pgtables’
239 | fence = xe_migrate_update_pgtables(m, NULL, NULL, m->q, &update, 1,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~
../drivers/gpu/drm/xe/xe_migrate.c:1224:1: note: declared here
1224 | xe_migrate_update_pgtables(struct xe_migrate *m,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~
cc1: some warnings being treated as errors
make[7]: *** [../scripts/Makefile.build:243: drivers/gpu/drm/xe/xe_migrate.o] Error 1
make[7]: *** Waiting for unfinished jobs....
make[6]: *** [../scripts/Makefile.build:480: drivers/gpu/drm/xe] Error 2
make[5]: *** [../scripts/Makefile.build:480: drivers/gpu/drm] Error 2
make[4]: *** [../scripts/Makefile.build:480: drivers/gpu] Error 2
make[3]: *** [../scripts/Makefile.build:480: drivers] Error 2
make[3]: *** Waiting for unfinished jobs....
make[2]: *** [/kernel/Makefile:1913: .] Error 2
make[1]: *** [/kernel/Makefile:234: __sub-make] Error 2
make: *** [Makefile:234: __sub-make] Error 2
[05:30:09] Configuring KUnit Kernel ...
Generating .config ...
Populating config with:
$ make ARCH=um O=.kunit olddefconfig
[05:30:13] Building KUnit Kernel ...
Populating config with:
$ make ARCH=um O=.kunit olddefconfig
Building with:
$ make ARCH=um O=.kunit --jobs=48
+ cleanup
++ stat -c %u:%g /kernel
+ chown -R 1003:1003 /kernel
^ permalink raw reply [flat|nested] 43+ messages in thread* Re: [Intel-xe] [PATCH v2 00/27] Refactor VM bind code
2023-11-07 5:25 [Intel-xe] [PATCH v2 00/27] Refactor VM bind code Matthew Brost
` (29 preceding siblings ...)
2023-11-07 5:30 ` [Intel-xe] ✗ CI.KUnit: failure " Patchwork
@ 2023-11-08 13:55 ` Thomas Hellström
30 siblings, 0 replies; 43+ messages in thread
From: Thomas Hellström @ 2023-11-08 13:55 UTC (permalink / raw)
To: Matthew Brost, intel-xe
On Mon, 2023-11-06 at 21:25 -0800, Matthew Brost wrote:
> WIP on refactoring VM bind to final error handling solution.
>
> Major changes are:
> 1. Allow bind / exec IOCTLs with zero binds / batches
> 2. Convert multiple binds from IOCTL into single job
> 3. CPU binds in jobs
> 4. Sync vs async bind per IOCTL rather than per queue
>
> Error handling still not 100% correct but this this series structures
> the code to make it possible.
>
> This series shouldn't break any uAPI compatibility until the last
> patch
> in the series.
>
> v2: Rebase, Take in-syncs into account when num_execs or num_binds ==
> 0
I see a compliation failure in tests/xe_migrate.c. Looks like the Xe
kunit tests were not in the config?
/Thomas
^ permalink raw reply [flat|nested] 43+ messages in thread