Re: [RFC 6/7] drm/amdgpu: Map userqueue into HW

[Shashank Sharma <shashank.sharma@amd.com>]

On 29/12/2022 18:51, Alex Deucher wrote:
> On Fri, Dec 23, 2022 at 2:37 PM Shashank Sharma <shashank.sharma@amd.com> wrote:
>> This patch add the function to map/unmap the usermode queue into the HW,
>> using the prepared MQD and other objects. After this mapping, the queue
>> will be ready to accept the workload.
> This should also be a callback into IP specific code.  It will be
> different for each IP type and version.
>
> Alex

Noted, so far we have two IP specific functions, .init_mqd() and .map()

- Shashank

>> Cc: Alex Deucher <alexander.deucher@amd.com>
>> Cc: Christian Koenig <christian.koenig@amd.com>
>>
>> Signed-off-by: Shashank Sharma <shashank.sharma@amd.com>
>> ---
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c | 71 +++++++++++++++++++
>>   1 file changed, 71 insertions(+)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c
>> index 2a854a5e2f70..b164e24247ca 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c
>> @@ -50,6 +50,67 @@ amdgpu_userqueue_remove_index(struct amdgpu_device *adev, struct amdgpu_usermode
>>       ida_simple_remove(&uqg->ida, queue->queue_id);
>>   }
>>
>> +static int amdgpu_userqueue_map(struct amdgpu_device *adev,
>> +                    struct amdgpu_usermode_queue *queue)
>> +{
>> +    int r;
>> +    struct mes_add_queue_input queue_input;
>> +
>> +    memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input));
>> +
>> +    queue_input.process_va_start = 0;
>> +    queue_input.process_va_end = adev->vm_manager.max_pfn - 1;
>> +    queue_input.process_quantum = 100000; /* 10ms */
>> +    queue_input.gang_quantum = 10000; /* 1ms */
>> +    queue_input.paging = false;
>> +
>> +    queue_input.gang_context_addr = queue->gang_ctx.gpu_addr;
>> +    queue_input.process_context_addr = queue->proc_ctx.gpu_addr;
>> +    queue_input.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
>> +    queue_input.gang_global_priority_level = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
>> +
>> +    queue_input.process_id = queue->pasid;
>> +    queue_input.queue_type = queue->queue_type;
>> +    queue_input.mqd_addr = queue->mqd_gpu_addr;
>> +    queue_input.wptr_addr = queue->wptr_gpu_addr;
>> +    queue_input.queue_size = queue->queue_size >> 2;
>> +    queue_input.doorbell_offset = queue->doorbell_index;
>> +    queue_input.page_table_base_addr =  queue->vm->pd_phys_addr;
>> +
>> +    amdgpu_mes_lock(&adev->mes);
>> +    r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
>> +    amdgpu_mes_unlock(&adev->mes);
>> +    if (r) {
>> +        DRM_ERROR("Failed to map queue in HW, err (%d)\n", r);
>> +        return r;
>> +    }
>> +
>> +    DRM_DEBUG_DRIVER("Queue %d mapped successfully\n", queue->queue_id);
>> +    return 0;
>> +}
>> +
>> +static void amdgpu_userqueue_unmap(struct amdgpu_device *adev,
>> +                    struct amdgpu_usermode_queue *queue)
>> +{
>> +    int r;
>> +    struct mes_remove_queue_input queue_input;
>> +
>> +    memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
>> +    queue_input.doorbell_offset = queue->doorbell_index;
>> +    queue_input.gang_context_addr = queue->gang_ctx.gpu_addr;
>> +
>> +    amdgpu_mes_lock(&adev->mes);
>> +    r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);
>> +    amdgpu_mes_unlock(&adev->mes);
>> +
>> +    if (r) {
>> +        DRM_ERROR("Failed to unmap usermode queue %d\n", queue->queue_id);
>> +        return;
>> +    }
>> +
>> +    DRM_DEBUG_DRIVER("Usermode queue %d unmapped\n", queue->queue_id);
>> +}
>> +
>>   static int
>>   amdgpu_userqueue_get_doorbell(struct amdgpu_device *adev,
>>                       struct amdgpu_usermode_queue *queue)
>> @@ -338,12 +399,21 @@ int amdgpu_userqueue_create(struct amdgpu_device *adev, struct drm_file *filp,
>>           goto free_mqd;
>>       }
>>
>> +    r = amdgpu_userqueue_map(adev, queue);
>> +    if (r < 0) {
>> +        DRM_ERROR("Failed to map queue\n");
>> +        goto free_ctx;
>> +    }
>> +
>>       ctx->userq = queue;
>>       args->out.q_id = queue->queue_id;
>>       args->out.flags = 0;
>>       mutex_unlock(&adev->userq.userq_mutex);
>>       return 0;
>>
>> +free_ctx:
>> +    amdgpu_userqueue_free_context(adev, queue);
>> +
>>   free_mqd:
>>       amdgpu_userqueue_destroy_mqd(queue);
>>
>> @@ -362,6 +432,7 @@ void amdgpu_userqueue_destroy(struct amdgpu_device *adev, struct drm_file *filp,
>>       struct amdgpu_usermode_queue *queue = ctx->userq;
>>
>>       mutex_lock(&adev->userq.userq_mutex);
>> +    amdgpu_userqueue_unmap(adev, queue);
>>       amdgpu_userqueue_free_context(adev, queue);
>>       amdgpu_userqueue_destroy_mqd(queue);
>>       amdgpu_userqueue_remove_index(adev, queue);
>> --
>> 2.34.1
>>