Re: [PATCH 09/16] drm/amdgpu: create kernel doorbell page

[Luben Tuikov <luben.tuikov@amd.com>]

On 2023-03-30 10:49, Christian König wrote:
> Am 30.03.23 um 16:40 schrieb Shashank Sharma:
>>
>> On 30/03/2023 16:24, Luben Tuikov wrote:
>>> On 2023-03-29 11:47, Shashank Sharma wrote:
>>>> From: Shashank Sharma <contactshashanksharma@gmail.com>
>>>>
>>>> This patch:
>>>> - creates a doorbell page for graphics driver usages.
>>>> - removes the adev->doorbell.ptr variable, replaces it with
>>>>    kernel-doorbell-bo's cpu address.
>>>>
>>>> Cc: Alex Deucher <alexander.deucher@amd.com>
>>>> Cc: Christian Koenig <christian.koenig@amd.com>
>>>> Signed-off-by: Shashank Sharma <shashank.sharma@amd.com>
>>>> ---
>>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h  | 16 ++++++-
>>>>   .../gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c  | 44 
>>>> +++++++++++++++----
>>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c       |  7 +++
>>>>   3 files changed, 57 insertions(+), 10 deletions(-)
>>>>
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h 
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
>>>> index 6581b78fe438..10a9bb10e974 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
>>>> @@ -49,10 +49,13 @@ struct amdgpu_doorbell {
>>>>       /* doorbell mmio */
>>>>       resource_size_t        base;
>>>>       resource_size_t        size;
>>>> -    u32 __iomem        *ptr;
>>>> +    u32    __iomem        *ptr;
>>>>         /* Number of doorbells reserved for amdgpu kernel driver */
>>>>       u32 num_kernel_doorbells;
>>>> +
>>>> +    /* For kernel doorbell pages */
>>>> +    struct amdgpu_doorbell_obj kernel_doorbells;
>>>>   };
>>> Here's an example where it could be confusing what the difference is
>>> between "struct amdgpu_doorbell" and "struct amdgpu_doobell_obj".
>>> As the comment to the struct doorbell_obj declarations says
>>> in patch 7,
>>>> +/* Structure to hold doorbell pages from PCI doorbell BAR */
>>>> +struct amdgpu_doorbell_obj {
>>
>> What is the confusion ? This is the object which is holding doorbell 
>> page. There could be 2 type of
>>
>> doorbell pages, kernel and process, this is a kernel one.
>>
>>> Perhaps we should call it "struct amdgpu_doorbell_bo", since
>>> it does contain amdgpu_bo's, which are doorbell's bos.
>>
>> This is not a buffer object (memory), this is doorbell object, so 
>> calling it bo would be wrong.
> 
> I think what Luben means is that in object orient programming this here 
> would be the class. The object is then the actual instantiation of that.

Yes, absolutely exactly what Christian said.

Regards,
Luben

> 
> But I have some real doubts that this is the right approach in the first 
> place.
> 
> Regards,
> Christian.
> 
>>
>> - Shashank
>>
>>>
>>> Regards,
>>> Luben
>>>
>>>>     /* Reserved doorbells for amdgpu (including multimedia).
>>>> @@ -369,6 +372,17 @@ void amdgpu_doorbell_free_page(struct 
>>>> amdgpu_device *adev,
>>>>   int amdgpu_doorbell_alloc_page(struct amdgpu_device *adev,
>>>>                   struct amdgpu_doorbell_obj *db_obj);
>>>>   +/**
>>>> + * amdgpu_doorbell_create_kernel_doorbells - Create kernel 
>>>> doorbells for graphics
>>>> + *
>>>> + * @adev: amdgpu_device pointer
>>>> + *
>>>> + * Creates doorbells for graphics driver
>>>> + *
>>>> + * returns 0 on success, error otherwise.
>>>> + */
>>>> +int amdgpu_doorbell_create_kernel_doorbells(struct amdgpu_device 
>>>> *adev);
>>>> +
>>>>   #define RDOORBELL32(index) amdgpu_mm_rdoorbell(adev, (index))
>>>>   #define WDOORBELL32(index, v) amdgpu_mm_wdoorbell(adev, (index), (v))
>>>>   #define RDOORBELL64(index) amdgpu_mm_rdoorbell64(adev, (index))
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c 
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
>>>> index 8be15b82b545..b46fe8b1378d 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
>>>> @@ -160,6 +160,38 @@ int amdgpu_doorbell_alloc_page(struct 
>>>> amdgpu_device *adev,
>>>>       return 0;
>>>>   }
>>>>   +/**
>>>> + * amdgpu_doorbell_create_kernel_doorbells - Create kernel 
>>>> doorbells for graphics
>>>> + *
>>>> + * @adev: amdgpu_device pointer
>>>> + *
>>>> + * Creates doorbells for graphics driver
>>>> + *
>>>> + * returns 0 on success, error otherwise.
>>>> + */
>>>> +int amdgpu_doorbell_create_kernel_doorbells(struct amdgpu_device 
>>>> *adev)
>>>> +{
>>>> +    int r;
>>>> +    struct amdgpu_doorbell_obj *kernel_doorbells = 
>>>> &adev->doorbell.kernel_doorbells;
>>>> +
>>>> +    kernel_doorbells->doorbell_bitmap = 
>>>> bitmap_zalloc(adev->doorbell.num_kernel_doorbells,
>>>> +                              GFP_KERNEL);
>>>> +    if (!kernel_doorbells->doorbell_bitmap) {
>>>> +        DRM_ERROR("Failed to create kernel doorbell bitmap\n");
>>>> +        return -ENOMEM;
>>>> +    }
>>>> +
>>>> +    kernel_doorbells->size = adev->doorbell.num_kernel_doorbells * 
>>>> sizeof(u32);
>>>> +    r = amdgpu_doorbell_alloc_page(adev, kernel_doorbells);
>>>> +    if (r) {
>>>> +        bitmap_free(kernel_doorbells->doorbell_bitmap);
>>>> +        DRM_ERROR("Failed to allocate kernel doorbells, err=%d\n", r);
>>>> +        return r;
>>>> +    }
>>>> +
>>>> +    return 0;
>>>> +}
>>>> +
>>>>   /*
>>>>    * GPU doorbell aperture helpers function.
>>>>    */
>>>> @@ -179,7 +211,6 @@ int amdgpu_device_doorbell_init(struct 
>>>> amdgpu_device *adev)
>>>>           adev->doorbell.base = 0;
>>>>           adev->doorbell.size = 0;
>>>>           adev->doorbell.num_kernel_doorbells = 0;
>>>> -        adev->doorbell.ptr = NULL;
>>>>           return 0;
>>>>       }
>>>>   @@ -208,12 +239,7 @@ int amdgpu_device_doorbell_init(struct 
>>>> amdgpu_device *adev)
>>>>       if (adev->asic_type >= CHIP_VEGA10)
>>>>           adev->doorbell.num_kernel_doorbells += 0x400;
>>>>   -    adev->doorbell.ptr = ioremap(adev->doorbell.base,
>>>> -                     adev->doorbell.num_kernel_doorbells *
>>>> -                     sizeof(u32));
>>>> -    if (adev->doorbell.ptr == NULL)
>>>> -        return -ENOMEM;
>>>> -
>>>> +    adev->doorbell.ptr = ioremap(adev->doorbell.base, 
>>>> adev->doorbell.size);
>>>>       return 0;
>>>>   }
>>>>   @@ -226,6 +252,6 @@ int amdgpu_device_doorbell_init(struct 
>>>> amdgpu_device *adev)
>>>>    */
>>>>   void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
>>>>   {
>>>> -    iounmap(adev->doorbell.ptr);
>>>> -    adev->doorbell.ptr = NULL;
>>>> + bitmap_free(adev->doorbell.kernel_doorbells.doorbell_bitmap);
>>>> +    amdgpu_doorbell_free_page(adev, &adev->doorbell.kernel_doorbells);
>>>>   }
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c 
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>>> index 203d77a20507..75c6852845c4 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>>> @@ -1866,6 +1866,13 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
>>>>           return r;
>>>>       }
>>>>   +    /* Create a boorbell page for kernel usages */
>>>> +    r = amdgpu_doorbell_create_kernel_doorbells(adev);
>>>> +    if (r) {
>>>> +        DRM_ERROR("Failed to initialize kernel doorbells. \n");
>>>> +        return r;
>>>> +    }
>>>> +
>>>>       /* Initialize preemptible memory pool */
>>>>       r = amdgpu_preempt_mgr_init(adev);
>>>>       if (r) {
>