Re: [PATCH v4 04/10] drm/amdgpu: create GFX-gen11 MQD for userqueue

[Shashank Sharma <shashank.sharma@amd.com>]

On 19/05/2023 23:19, Alex Deucher wrote:
> On Mon, Apr 24, 2023 at 1:39 PM Shashank Sharma <shashank.sharma@amd.com> wrote:
>> A Memory queue descriptor (MQD) of a userqueue defines it in
>> the hw's context. As MQD format can vary between different
>> graphics IPs, we need gfx GEN specific handlers to create MQDs.
>>
>> This patch:
>> - Introduces MQD handler functions for the usermode queues.
>> - Adds new functions to create and destroy userqueue MQD for
>>    GFX-GEN-11 IP
>>
>> V1: Worked on review comments from Alex:
>>      - Make MQD functions GEN and IP specific
>>
>> V2: Worked on review comments from Alex:
>>      - Reuse the existing adev->mqd[ip] for MQD creation
>>      - Formatting and arrangement of code
>>
>> V3:
>>      - Integration with doorbell manager
>>
>> V4: Review comments addressed:
>>      - Do not create a new file for userq, reuse gfx_v11_0.c (Alex)
>>      - Align name of structure members (Luben)
>>      - Don't break up the Cc tag list and the Sob tag list in commit
>>        message (Luben)
>>
>> Cc: Alex Deucher <alexander.deucher@amd.com>
>> Cc: Christian Koenig <christian.koenig@amd.com>
>> Signed-off-by: Shashank Sharma <shashank.sharma@amd.com>
>> Signed-off-by: Arvind Yadav <arvind.yadav@amd.com>
>> ---
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c | 25 ++++++++
>>   drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c        | 57 +++++++++++++++++++
>>   .../gpu/drm/amd/include/amdgpu_userqueue.h    |  7 +++
>>   3 files changed, 89 insertions(+)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c
>> index 333f31efbe7b..e95fb35b0cb5 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c
>> @@ -81,6 +81,14 @@ static int amdgpu_userqueue_create_gfx(struct drm_file *filp, union drm_amdgpu_u
>>                  goto free_queue;
>>          }
>>
>> +       if (uq_mgr->userq_funcs[queue->queue_type]->mqd_create) {
>> +               r = uq_mgr->userq_funcs[queue->queue_type]->mqd_create(uq_mgr, queue);
>> +               if (r) {
>> +                       DRM_ERROR("Failed to create/map userqueue MQD\n");
>> +                       goto free_queue;
>> +               }
>> +       }
>> +
>>          args->out.queue_id = queue->queue_id;
>>          args->out.flags = 0;
>>          mutex_unlock(&uq_mgr->userq_mutex);
>> @@ -119,6 +127,8 @@ static void amdgpu_userqueue_destroy(struct drm_file *filp, int queue_id)
>>          }
>>
>>          mutex_lock(&uq_mgr->userq_mutex);
>> +       if (uq_mgr->userq_funcs[queue->queue_type]->mqd_destroy)
>> +               uq_mgr->userq_funcs[queue->queue_type]->mqd_destroy(uq_mgr, queue);
>>          amdgpu_userqueue_free_index(uq_mgr, queue->queue_id);
>>          mutex_unlock(&uq_mgr->userq_mutex);
>>          kfree(queue);
>> @@ -149,6 +159,20 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data,
>>          return r;
>>   }
>>
>> +extern const struct amdgpu_userq_funcs userq_gfx_v11_funcs;
>> +
>> +static void
>> +amdgpu_userqueue_setup_ip_funcs(struct amdgpu_userq_mgr *uq_mgr)
>> +{
>> +       int maj;
>> +       struct amdgpu_device *adev = uq_mgr->adev;
>> +       uint32_t version = adev->ip_versions[GC_HWIP][0];
>> +
>> +       /* We support usermode queue only for GFX IP as of now */
>> +       maj = IP_VERSION_MAJ(version);
>> +       if (maj == 11)
>> +               uq_mgr->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_gfx_v11_funcs;
> Do we need to keep function pointers in uq_mgr?  It would be nice to
> keep the logic in the IPs directly.  E.g, in gfx_v11_0.c, we could set
> adev->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_gfx_v11_funcs;
> then the userq code can just check
>
>         if (adev->userq_funcs[queue->queue_type]->mqd_destroy)
>                 adev->userq_funcs[queue->queue_type]->mqd_destroy(adev, queue);
>
> etc.

We have discussed about this here as well: 
https://patchwork.freedesktop.org/patch/529513/?series=113675&rev=2 
(which we probably could not follow up)

The problem is, in the discussion from first patch set we decided not to 
use adev for this, instead moved this in driver private from fptr.

>> +}
>>
>>   int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct amdgpu_device *adev)
>>   {
>> @@ -156,6 +180,7 @@ int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct amdgpu_devi
>>          idr_init_base(&userq_mgr->userq_idr, 1);
>>          userq_mgr->adev = adev;
>>
>> +       amdgpu_userqueue_setup_ip_funcs(userq_mgr);
>>          return 0;
>>   }
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
>> index a56c6e106d00..9f7b14966ac8 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
>> @@ -30,6 +30,7 @@
>>   #include "amdgpu_psp.h"
>>   #include "amdgpu_smu.h"
>>   #include "amdgpu_atomfirmware.h"
>> +#include "amdgpu_userqueue.h"
>>   #include "imu_v11_0.h"
>>   #include "soc21.h"
>>   #include "nvd.h"
>> @@ -6404,3 +6405,59 @@ const struct amdgpu_ip_block_version gfx_v11_0_ip_block =
>>          .rev = 0,
>>          .funcs = &gfx_v11_0_ip_funcs,
>>   };
>> +
>> +static int
>> +gfx_v11_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_queue *queue)
>> +{
>> +       struct amdgpu_device *adev = uq_mgr->adev;
>> +       struct amdgpu_userq_ctx_space *mqd = &queue->mqd;
>> +       struct amdgpu_mqd *gfx_v11_mqd = &adev->mqds[queue->queue_type];
>> +       int size = gfx_v11_mqd->mqd_size;
>> +       int r;
>> +
>> +       r = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE,
>> +                                   AMDGPU_GEM_DOMAIN_GTT,
>> +                                   &mqd->obj,
>> +                                   &mqd->gpu_addr,
>> +                                   &mqd->cpu_ptr);
>> +       if (r) {
>> +               DRM_ERROR("Failed to allocate bo for userqueue (%d)", r);
>> +               return r;
>> +       }
>> +
>> +       memset(mqd->cpu_ptr, 0, size);
>> +       r = amdgpu_bo_reserve(mqd->obj, false);
>> +       if (unlikely(r != 0)) {
>> +               DRM_ERROR("Failed to reserve mqd for userqueue (%d)", r);
>> +               goto free_mqd;
>> +       }
>> +
>> +       queue->userq_prop.use_doorbell = true;
>> +       queue->userq_prop.mqd_gpu_addr = mqd->gpu_addr;
>> +       r = gfx_v11_mqd->init_mqd(adev, (void *)mqd->cpu_ptr, &queue->userq_prop);
>> +       amdgpu_bo_unreserve(mqd->obj);
>> +       if (r) {
>> +               DRM_ERROR("Failed to init MQD for queue\n");
>> +               goto free_mqd;
>> +       }
>> +
>> +       DRM_DEBUG_DRIVER("MQD for queue %d created\n", queue->queue_id);
>> +       return 0;
>> +
>> +free_mqd:
>> +       amdgpu_bo_free_kernel(&mqd->obj, &mqd->gpu_addr, &mqd->cpu_ptr);
>> +       return r;
>> +}
>> +
>> +static void
>> +gfx_v11_userq_mqd_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_queue *queue)
>> +{
>> +       struct amdgpu_userq_ctx_space *mqd = &queue->mqd;
>> +
>> +       amdgpu_bo_free_kernel(&mqd->obj, &mqd->gpu_addr, &mqd->cpu_ptr);
>> +}
>> +
>> +const struct amdgpu_userq_funcs userq_gfx_v11_funcs = {
>> +       .mqd_create = gfx_v11_userq_mqd_create,
>> +       .mqd_destroy = gfx_v11_userq_mqd_destroy,
>> +};
> These functions and structures should start with gfx_v11_0_ for consistency.

Noted,

- Shashank

>
>> diff --git a/drivers/gpu/drm/amd/include/amdgpu_userqueue.h b/drivers/gpu/drm/amd/include/amdgpu_userqueue.h
>> index 8d8f6b3bcda5..e7da27918bd2 100644
>> --- a/drivers/gpu/drm/amd/include/amdgpu_userqueue.h
>> +++ b/drivers/gpu/drm/amd/include/amdgpu_userqueue.h
>> @@ -28,6 +28,12 @@
>>   #include "amdgpu.h"
>>   #define AMDGPU_MAX_USERQ 512
>>
>> +struct amdgpu_userq_ctx_space {
>> +       uint64_t         gpu_addr;
>> +       void             *cpu_ptr;
>> +       struct amdgpu_bo *obj;
>> +};
>> +
>>   struct amdgpu_usermode_queue {
>>          int                     queue_id;
>>          int                     queue_type;
>> @@ -36,6 +42,7 @@ struct amdgpu_usermode_queue {
>>          struct amdgpu_mqd_prop  userq_prop;
>>          struct amdgpu_userq_mgr *userq_mgr;
>>          struct amdgpu_vm        *vm;
>> +       struct amdgpu_userq_ctx_space mqd;
>>   };
>>
>>   struct amdgpu_userq_funcs {
>> --
>> 2.40.0
>>