Re: [PATCH 1/1] drm/amdgpu: disable gpu_sched load balancer for vcn jobs

[Andrey Grodzovsky <Andrey.Grodzovsky@amd.com>]

On 3/11/20 4:32 PM, Nirmoy wrote:
>
> On 3/11/20 9:02 PM, Andrey Grodzovsky wrote:
>>
>> On 3/11/20 4:00 PM, Andrey Grodzovsky wrote:
>>>
>>> On 3/11/20 4:00 PM, Nirmoy Das wrote:
>>>> VCN HW  doesn't support dynamic load balance on multiple
>>>> instances for a context. This patch modifies entity's
>>>> sched_list to a sched_list consist of only one drm scheduler.
>>>>
>>>> Signed-off-by: Nirmoy Das <nirmoy.das@amd.com>
>>>> ---
>>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
>>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c  | 13 +++++++++++++
>>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h  |  1 +
>>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_job.c  |  3 +++
>>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  1 +
>>>>   drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c    |  2 ++
>>>>   drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c    |  2 ++
>>>>   drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c    |  2 ++
>>>>   8 files changed, 28 insertions(+)
>>>>
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c 
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
>>>> index 8304d0c87899..db0eef19c636 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
>>>> @@ -1203,6 +1203,7 @@ static int amdgpu_cs_submit(struct 
>>>> amdgpu_cs_parser *p,
>>>>                   union drm_amdgpu_cs *cs)
>>>>   {
>>>>       struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
>>>> +    struct amdgpu_ring *ring = to_amdgpu_ring(p->entity->rq->sched);
>>>>       struct drm_sched_entity *entity = p->entity;
>>>>       enum drm_sched_priority priority;
>>>>       struct amdgpu_bo_list_entry *e;
>>>> @@ -1257,6 +1258,9 @@ static int amdgpu_cs_submit(struct 
>>>> amdgpu_cs_parser *p,
>>>>       priority = job->base.s_priority;
>>>>       drm_sched_entity_push_job(&job->base, entity);
>>>>   +    if (ring->funcs->no_gpu_sched_loadbalance)
>>>> +        amdgpu_ctx_disable_gpu_sched_load_balance(entity);
>>>> +
>>>
>>>
>>> Why this needs to be done each time a job is submitted and not once 
>>> in drm_sched_entity_init (same foramdgpu_job_submit bellow ?)
>>>
>>> Andrey
>>
>>
>> My bad - not in drm_sched_entity_init but in relevant amdgpu code.
>
>
> Hi Andrey,
>
> Do you mean drm_sched_job_init() or after creating VCN entities?
>
>
> Nirmoy


I guess after creating the VCN entities (has to be amdgpu specific code) 
- I just don't get why it needs to be done each time job is submitted, I 
mean - since you set .no_gpu_sched_loadbalance = true anyway this is 
always true and so shouldn't you just initialize the VCN entity with a 
schedulers list consisting of one scheduler and that it ?

Andrey


>
>>
>> Andrey
>>
>>
>>>
>>>
>>>> amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm);
>>>>         ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, 
>>>> p->fence);
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c 
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
>>>> index fa575bdc03c8..1127e8f77721 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
>>>> @@ -559,6 +559,19 @@ void amdgpu_ctx_priority_override(struct 
>>>> amdgpu_ctx *ctx,
>>>>       }
>>>>   }
>>>>   +/**
>>>> + * amdgpu_ctx_disable_gpu_sched_load_balance - disable gpu_sched's 
>>>> load balancer
>>>> + * @entity: entity on which load balancer will be disabled
>>>> + */
>>>> +void amdgpu_ctx_disable_gpu_sched_load_balance(struct 
>>>> drm_sched_entity *entity)
>>>> +{
>>>> +    struct drm_gpu_scheduler **scheds = &entity->rq->sched;
>>>> +
>>>> +    /* disable gpu_sched's job load balancer by assigning only one */
>>>> +    /* drm scheduler to the entity */
>>>> +    drm_sched_entity_modify_sched(entity, scheds, 1);
>>>> +}
>>>> +
>>>>   int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
>>>>                      struct drm_sched_entity *entity)
>>>>   {
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h 
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
>>>> index de490f183af2..3a2f900b8000 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
>>>> @@ -90,5 +90,6 @@ void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr 
>>>> *mgr);
>>>>     void amdgpu_ctx_init_sched(struct amdgpu_device *adev);
>>>>   +void amdgpu_ctx_disable_gpu_sched_load_balance(struct 
>>>> drm_sched_entity *entity);
>>>>     #endif
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c 
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>>>> index 4981e443a884..64dad7ba74da 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>>>> @@ -140,6 +140,7 @@ void amdgpu_job_free(struct amdgpu_job *job)
>>>>   int amdgpu_job_submit(struct amdgpu_job *job, struct 
>>>> drm_sched_entity *entity,
>>>>                 void *owner, struct dma_fence **f)
>>>>   {
>>>> +    struct amdgpu_ring *ring = to_amdgpu_ring(entity->rq->sched);
>>>>       enum drm_sched_priority priority;
>>>>       int r;
>>>>   @@ -154,6 +155,8 @@ int amdgpu_job_submit(struct amdgpu_job *job, 
>>>> struct drm_sched_entity *entity,
>>>>       amdgpu_job_free_resources(job);
>>>>       priority = job->base.s_priority;
>>>>       drm_sched_entity_push_job(&job->base, entity);
>>>> +    if (ring->funcs->no_gpu_sched_loadbalance)
>>>> +        amdgpu_ctx_disable_gpu_sched_load_balance(entity);
>>>>         return 0;
>>>>   }
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h 
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
>>>> index 448c76cbf3ed..f78fe1a6912b 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
>>>> @@ -115,6 +115,7 @@ struct amdgpu_ring_funcs {
>>>>       u32            nop;
>>>>       bool            support_64bit_ptrs;
>>>>       bool            no_user_fence;
>>>> +    bool            no_gpu_sched_loadbalance;
>>>>       unsigned        vmhub;
>>>>       unsigned        extra_dw;
>>>>   diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c 
>>>> b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
>>>> index 71f61afdc655..749ccdb5fbfb 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
>>>> @@ -1871,6 +1871,7 @@ static const struct amdgpu_ring_funcs 
>>>> vcn_v1_0_dec_ring_vm_funcs = {
>>>>       .align_mask = 0xf,
>>>>       .support_64bit_ptrs = false,
>>>>       .no_user_fence = true,
>>>> +    .no_gpu_sched_loadbalance = true,
>>>>       .vmhub = AMDGPU_MMHUB_0,
>>>>       .get_rptr = vcn_v1_0_dec_ring_get_rptr,
>>>>       .get_wptr = vcn_v1_0_dec_ring_get_wptr,
>>>> @@ -1905,6 +1906,7 @@ static const struct amdgpu_ring_funcs 
>>>> vcn_v1_0_enc_ring_vm_funcs = {
>>>>       .nop = VCN_ENC_CMD_NO_OP,
>>>>       .support_64bit_ptrs = false,
>>>>       .no_user_fence = true,
>>>> +    .no_gpu_sched_loadbalance = true,
>>>>       .vmhub = AMDGPU_MMHUB_0,
>>>>       .get_rptr = vcn_v1_0_enc_ring_get_rptr,
>>>>       .get_wptr = vcn_v1_0_enc_ring_get_wptr,
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c 
>>>> b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
>>>> index f2745fd1ddb3..c48423b54bc5 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
>>>> @@ -1954,6 +1954,7 @@ static const struct amd_ip_funcs 
>>>> vcn_v2_0_ip_funcs = {
>>>>   static const struct amdgpu_ring_funcs vcn_v2_0_dec_ring_vm_funcs = {
>>>>       .type = AMDGPU_RING_TYPE_VCN_DEC,
>>>>       .align_mask = 0xf,
>>>> +    .no_gpu_sched_loadbalance = true,
>>>>       .vmhub = AMDGPU_MMHUB_0,
>>>>       .get_rptr = vcn_v2_0_dec_ring_get_rptr,
>>>>       .get_wptr = vcn_v2_0_dec_ring_get_wptr,
>>>> @@ -1984,6 +1985,7 @@ static const struct amdgpu_ring_funcs 
>>>> vcn_v2_0_dec_ring_vm_funcs = {
>>>>   static const struct amdgpu_ring_funcs vcn_v2_0_enc_ring_vm_funcs = {
>>>>       .type = AMDGPU_RING_TYPE_VCN_ENC,
>>>>       .align_mask = 0x3f,
>>>> +    .no_gpu_sched_loadbalance = true,
>>>>       .nop = VCN_ENC_CMD_NO_OP,
>>>>       .vmhub = AMDGPU_MMHUB_0,
>>>>       .get_rptr = vcn_v2_0_enc_ring_get_rptr,
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c 
>>>> b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
>>>> index 9b22e2b55132..1cc8e1420fc8 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
>>>> @@ -1478,6 +1478,7 @@ static void vcn_v2_5_dec_ring_set_wptr(struct 
>>>> amdgpu_ring *ring)
>>>>   static const struct amdgpu_ring_funcs vcn_v2_5_dec_ring_vm_funcs = {
>>>>       .type = AMDGPU_RING_TYPE_VCN_DEC,
>>>>       .align_mask = 0xf,
>>>> +    .no_gpu_sched_loadbalance = true,
>>>>       .vmhub = AMDGPU_MMHUB_1,
>>>>       .get_rptr = vcn_v2_5_dec_ring_get_rptr,
>>>>       .get_wptr = vcn_v2_5_dec_ring_get_wptr,
>>>> @@ -1577,6 +1578,7 @@ static void vcn_v2_5_enc_ring_set_wptr(struct 
>>>> amdgpu_ring *ring)
>>>>   static const struct amdgpu_ring_funcs vcn_v2_5_enc_ring_vm_funcs = {
>>>>       .type = AMDGPU_RING_TYPE_VCN_ENC,
>>>>       .align_mask = 0x3f,
>>>> +    .no_gpu_sched_loadbalance = true,
>>>>       .nop = VCN_ENC_CMD_NO_OP,
>>>>       .vmhub = AMDGPU_MMHUB_1,
>>>>       .get_rptr = vcn_v2_5_enc_ring_get_rptr,
>>> _______________________________________________
>>> amd-gfx mailing list
>>> amd-gfx@lists.freedesktop.org
>>> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&amp;data=02%7C01%7Candrey.grodzovsky%40amd.com%7Cc6958b3a536448ecd9bd08d7c5f6cfc9%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637195536148567145&amp;sdata=OCuOocxzDKPVCUKDUzqqbL3lKj4lYFyDR8Ly%2FtG3Gi0%3D&amp;reserved=0 
>>>
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx