Re: [PATCH v2 01/12] drm/amdgpu: create a new file for doorbell manager

["Christian König" <christian.koenig@amd.com>]

Am 13.04.23 um 12:48 schrieb Christian König:
> Am 12.04.23 um 18:25 schrieb Shashank Sharma:
>> This patch:
>> - creates a new file for doorbell management.
>> - moves doorbell code from amdgpu_device.c to this file.
>>
>> V2:
>>   - remove doc from function declaration (Christian)
>>   - remove 'device' from function names to make it consistent (Alex)
>>   - add SPDX license identifier (Luben)
>>
>> Cc: Alex Deucher <alexander.deucher@amd.com>
>> Cc: Christian Koenig <christian.koenig@amd.com>
>> Signed-off-by: Shashank Sharma <shashank.sharma@amd.com>
>> ---
>>   drivers/gpu/drm/amd/amdgpu/Makefile           |   2 +-
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c    | 174 +----------------
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h  |   6 +
>>   .../gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c  | 183 ++++++++++++++++++
>>   4 files changed, 195 insertions(+), 170 deletions(-)
>>   create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile 
>> b/drivers/gpu/drm/amd/amdgpu/Makefile
>> index 798d0e9a60b7..204665f20319 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
>> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
>> @@ -41,7 +41,7 @@ ccflags-y := -I$(FULL_AMD_PATH)/include/asic_reg \
>>   amdgpu-y := amdgpu_drv.o
>>     # add KMS driver
>> -amdgpu-y += amdgpu_device.o amdgpu_kms.o \
>> +amdgpu-y += amdgpu_device.o amdgpu_doorbell_mgr.o amdgpu_kms.o \
>>       amdgpu_atombios.o atombios_crtc.o amdgpu_connectors.o \
>>       atom.o amdgpu_fence.o amdgpu_ttm.o amdgpu_object.o amdgpu_gart.o \
>>       amdgpu_encoders.o amdgpu_display.o amdgpu_i2c.o \
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c 
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> index 57ee1c4a81e9..88c75f7a8d66 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> @@ -579,94 +579,6 @@ void amdgpu_mm_wreg_mmio_rlc(struct 
>> amdgpu_device *adev,
>>       }
>>   }
>>   -/**
>> - * amdgpu_mm_rdoorbell - read a doorbell dword
>> - *
>> - * @adev: amdgpu_device pointer
>> - * @index: doorbell index
>> - *
>> - * Returns the value in the doorbell aperture at the
>> - * requested doorbell index (CIK).
>> - */
>> -u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
>> -{
>> -    if (amdgpu_device_skip_hw_access(adev))
>> -        return 0;
>> -
>> -    if (index < adev->doorbell.num_kernel_doorbells) {
>> -        return readl(adev->doorbell.ptr + index);
>> -    } else {
>> -        DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", 
>> index);
>> -        return 0;
>> -    }
>> -}
>> -
>> -/**
>> - * amdgpu_mm_wdoorbell - write a doorbell dword
>> - *
>> - * @adev: amdgpu_device pointer
>> - * @index: doorbell index
>> - * @v: value to write
>> - *
>> - * Writes @v to the doorbell aperture at the
>> - * requested doorbell index (CIK).
>> - */
>> -void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
>> -{
>> -    if (amdgpu_device_skip_hw_access(adev))
>> -        return;
>> -
>> -    if (index < adev->doorbell.num_kernel_doorbells) {
>> -        writel(v, adev->doorbell.ptr + index);
>> -    } else {
>> -        DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", 
>> index);
>> -    }
>> -}
>> -
>> -/**
>> - * amdgpu_mm_rdoorbell64 - read a doorbell Qword
>> - *
>> - * @adev: amdgpu_device pointer
>> - * @index: doorbell index
>> - *
>> - * Returns the value in the doorbell aperture at the
>> - * requested doorbell index (VEGA10+).
>> - */
>> -u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
>> -{
>> -    if (amdgpu_device_skip_hw_access(adev))
>> -        return 0;
>> -
>> -    if (index < adev->doorbell.num_kernel_doorbells) {
>> -        return atomic64_read((atomic64_t *)(adev->doorbell.ptr + 
>> index));
>> -    } else {
>> -        DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", 
>> index);
>> -        return 0;
>> -    }
>> -}
>> -
>> -/**
>> - * amdgpu_mm_wdoorbell64 - write a doorbell Qword
>> - *
>> - * @adev: amdgpu_device pointer
>> - * @index: doorbell index
>> - * @v: value to write
>> - *
>> - * Writes @v to the doorbell aperture at the
>> - * requested doorbell index (VEGA10+).
>> - */
>> -void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, 
>> u64 v)
>> -{
>> -    if (amdgpu_device_skip_hw_access(adev))
>> -        return;
>> -
>> -    if (index < adev->doorbell.num_kernel_doorbells) {
>> -        atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
>> -    } else {
>> -        DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", 
>> index);
>> -    }
>> -}
>> -
>>   /**
>>    * amdgpu_device_indirect_rreg - read an indirect register
>>    *
>> @@ -1016,82 +928,6 @@ int amdgpu_device_pci_reset(struct 
>> amdgpu_device *adev)
>>       return pci_reset_function(adev->pdev);
>>   }
>>   -/*
>> - * GPU doorbell aperture helpers function.
>> - */
>> -/**
>> - * amdgpu_device_doorbell_init - Init doorbell driver information.
>> - *
>> - * @adev: amdgpu_device pointer
>> - *
>> - * Init doorbell driver information (CIK)
>> - * Returns 0 on success, error on failure.
>> - */
>> -static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
>> -{
>> -
>> -    /* No doorbell on SI hardware generation */
>> -    if (adev->asic_type < CHIP_BONAIRE) {
>> -        adev->doorbell.base = 0;
>> -        adev->doorbell.size = 0;
>> -        adev->doorbell.num_kernel_doorbells = 0;
>> -        adev->doorbell.ptr = NULL;
>> -        return 0;
>> -    }
>> -
>> -    if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
>> -        return -EINVAL;
>> -
>> -    amdgpu_asic_init_doorbell_index(adev);
>> -
>> -    /* doorbell bar mapping */
>> -    adev->doorbell.base = pci_resource_start(adev->pdev, 2);
>> -    adev->doorbell.size = pci_resource_len(adev->pdev, 2);
>> -
>> -    if (adev->enable_mes) {
>> -        adev->doorbell.num_kernel_doorbells =
>> -            adev->doorbell.size / sizeof(u32);
>> -    } else {
>> -        adev->doorbell.num_kernel_doorbells =
>> -            min_t(u32, adev->doorbell.size / sizeof(u32),
>> -                  adev->doorbell_index.max_assignment+1);
>> -        if (adev->doorbell.num_kernel_doorbells == 0)
>> -            return -EINVAL;
>> -
>> -        /* For Vega, reserve and map two pages on doorbell BAR since 
>> SDMA
>> -         * paging queue doorbell use the second page. The
>> -         * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
>> -         * doorbells are in the first page. So with paging queue 
>> enabled,
>> -         * the max num_kernel_doorbells should + 1 page (0x400 in 
>> dword)
>> -         */
>> -        if (adev->asic_type >= CHIP_VEGA10)
>> -            adev->doorbell.num_kernel_doorbells += 0x400;
>> -    }
>> -
>> -    adev->doorbell.ptr = ioremap(adev->doorbell.base,
>> -                     adev->doorbell.num_kernel_doorbells *
>> -                     sizeof(u32));
>> -    if (adev->doorbell.ptr == NULL)
>> -        return -ENOMEM;
>> -
>> -    return 0;
>> -}
>> -
>> -/**
>> - * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
>> - *
>> - * @adev: amdgpu_device pointer
>> - *
>> - * Tear down doorbell driver information (CIK)
>> - */
>> -static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
>> -{
>> -    iounmap(adev->doorbell.ptr);
>> -    adev->doorbell.ptr = NULL;
>> -}
>> -
>> -
>> -
>>   /*
>>    * amdgpu_device_wb_*()
>>    * Writeback is the method by which the GPU updates special pages 
>> in memory
>> @@ -1239,7 +1075,7 @@ int amdgpu_device_resize_fb_bar(struct 
>> amdgpu_device *adev)
>>                     cmd & ~PCI_COMMAND_MEMORY);
>>         /* Free the VRAM and doorbell BAR, we most likely need to 
>> move both. */
>> -    amdgpu_device_doorbell_fini(adev);
>> +    amdgpu_doorbell_fini(adev);
>>       if (adev->asic_type >= CHIP_BONAIRE)
>>           pci_release_resource(adev->pdev, 2);
>>   @@ -1256,7 +1092,7 @@ int amdgpu_device_resize_fb_bar(struct 
>> amdgpu_device *adev)
>>       /* When the doorbell or fb BAR isn't available we have no 
>> chance of
>>        * using the device.
>>        */
>> -    r = amdgpu_device_doorbell_init(adev);
>> +    r = amdgpu_doorbell_init(adev);
>>       if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
>>           return -ENODEV;
>>   @@ -3712,7 +3548,7 @@ int amdgpu_device_init(struct amdgpu_device 
>> *adev,
>>           dev_info(adev->dev, "PCIE atomic ops is not supported\n");
>>         /* doorbell bar mapping and doorbell index init*/
>> -    amdgpu_device_doorbell_init(adev);
>> +    amdgpu_doorbell_init(adev);
>>         if (amdgpu_emu_mode == 1) {
>>           /* post the asic on emulation mode */
>> @@ -3942,7 +3778,7 @@ static void amdgpu_device_unmap_mmio(struct 
>> amdgpu_device *adev)
>> unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
>>         /* Unmap all mapped bars - Doorbell, registers and VRAM */
>> -    amdgpu_device_doorbell_fini(adev);
>> +    amdgpu_doorbell_fini(adev);
>>         iounmap(adev->rmmio);
>>       adev->rmmio = NULL;
>> @@ -4051,7 +3887,7 @@ void amdgpu_device_fini_sw(struct amdgpu_device 
>> *adev)
>>             iounmap(adev->rmmio);
>>           adev->rmmio = NULL;
>> -        amdgpu_device_doorbell_fini(adev);
>> +        amdgpu_doorbell_fini(adev);
>>           drm_dev_exit(idx);
>>       }
>>   diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h 
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
>> index 908433ac6cb3..3e77a6b4bd69 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
>> @@ -306,6 +306,12 @@ void amdgpu_mm_wdoorbell(struct amdgpu_device 
>> *adev, u32 index, u32 v);
>>   u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index);
>>   void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, 
>> u64 v);
>>   +/*
>> + * GPU doorbell aperture helpers function.
>> + */
>> +int amdgpu_doorbell_init(struct amdgpu_device *adev);
>> +void amdgpu_doorbell_fini(struct amdgpu_device *adev);
>> +
>>   #define RDOORBELL32(index) amdgpu_mm_rdoorbell(adev, (index))
>>   #define WDOORBELL32(index, v) amdgpu_mm_wdoorbell(adev, (index), (v))
>>   #define RDOORBELL64(index) amdgpu_mm_rdoorbell64(adev, (index))
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c 
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
>> new file mode 100644
>> index 000000000000..64fec954815d
>> --- /dev/null
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
>> @@ -0,0 +1,183 @@
>> +// SPDX-License-Identifier: GPL-2.0+
>
> That should probably be MIT. Except if Alex thinks otherwise.
>
>> +/*
>> + * Copyright 2022 Advanced Micro Devices, Inc.
>> + *
>> + * Permission is hereby granted, free of charge, to any person 
>> obtaining a
>> + * copy of this software and associated documentation files (the 
>> "Software"),
>> + * to deal in the Software without restriction, including without 
>> limitation
>> + * the rights to use, copy, modify, merge, publish, distribute, 
>> sublicense,
>> + * and/or sell copies of the Software, and to permit persons to whom 
>> the
>> + * Software is furnished to do so, subject to the following conditions:
>> + *
>> + * The above copyright notice and this permission notice shall be 
>> included in
>> + * all copies or substantial portions of the Software.
>> + *
>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
>> EXPRESS OR
>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
>> MERCHANTABILITY,
>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO 
>> EVENT SHALL
>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
>> DAMAGES OR
>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
>> OTHERWISE,
>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 
>> USE OR
>> + * OTHER DEALINGS IN THE SOFTWARE.
>> + *
>> + */
>> +
>> +#include "amdgpu.h"
>> +
>> +/**
>> + * amdgpu_mm_rdoorbell - read a doorbell dword
>> + *
>> + * @adev: amdgpu_device pointer
>> + * @index: doorbell index
>> + *
>> + * Returns the value in the doorbell aperture at the
>> + * requested doorbell index (CIK).
>> + */
>> +u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
>> +{
>> +    if (amdgpu_device_skip_hw_access(adev))
>> +        return 0;
>> +
>> +    if (index < adev->doorbell.num_kernel_doorbells)
>> +        return readl(adev->doorbell.ptr + index);
>> +
>> +    DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
>> +    return 0;
>> +}
>> +
>> +/**
>> + * amdgpu_mm_wdoorbell - write a doorbell dword
>> + *
>> + * @adev: amdgpu_device pointer
>> + * @index: doorbell index
>> + * @v: value to write
>> + *
>> + * Writes @v to the doorbell aperture at the
>> + * requested doorbell index (CIK).
>> + */
>> +void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
>> +{
>> +    if (amdgpu_device_skip_hw_access(adev))
>> +        return;
>> +
>> +    if (index < adev->doorbell.num_kernel_doorbells)
>> +        writel(v, adev->doorbell.ptr + index);
>> +    else
>> +        DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", 
>> index);
>> +}
>> +
>> +/**
>> + * amdgpu_mm_rdoorbell64 - read a doorbell Qword
>> + *
>> + * @adev: amdgpu_device pointer
>> + * @index: doorbell index
>> + *
>> + * Returns the value in the doorbell aperture at the
>> + * requested doorbell index (VEGA10+).
>> + */
>> +u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
>> +{
>> +    if (amdgpu_device_skip_hw_access(adev))
>> +        return 0;
>> +
>> +    if (index < adev->doorbell.num_kernel_doorbells)
>> +        return atomic64_read((atomic64_t *)(adev->doorbell.ptr + 
>> index));
>> +
>> +    DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
>> +    return 0;
>> +}
>> +
>> +/**
>> + * amdgpu_mm_wdoorbell64 - write a doorbell Qword
>> + *
>> + * @adev: amdgpu_device pointer
>> + * @index: doorbell index
>> + * @v: value to write
>> + *
>> + * Writes @v to the doorbell aperture at the
>> + * requested doorbell index (VEGA10+).
>> + */
>> +void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, 
>> u64 v)
>> +{
>> +    if (amdgpu_device_skip_hw_access(adev))
>> +        return;
>> +
>> +    if (index < adev->doorbell.num_kernel_doorbells)
>> +        atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
>> +    else
>> +        DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", 
>> index);
>> +}
>> +
>> +/*
>> + * GPU doorbell aperture helpers function.
>> + */
>> +/**
>> + * amdgpu_doorbell_init - Init doorbell driver information.
>> + *
>> + * @adev: amdgpu_device pointer
>> + *
>> + * Init doorbell driver information (CIK)
>> + * Returns 0 on success, error on failure.
>> + */
>> +int amdgpu_doorbell_init(struct amdgpu_device *adev)
>> +{
>> +
>> +    /* No doorbell on SI hardware generation */
>> +    if (adev->asic_type < CHIP_BONAIRE) {
>> +        adev->doorbell.base = 0;
>> +        adev->doorbell.size = 0;
>> +        adev->doorbell.num_kernel_doorbells = 0;
>> +        adev->doorbell.ptr = NULL;
>> +        return 0;
>> +    }
>> +
>> +    if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
>> +        return -EINVAL;
>> +
>> +    amdgpu_asic_init_doorbell_index(adev);
>> +
>> +    /* doorbell bar mapping */
>> +    adev->doorbell.base = pci_resource_start(adev->pdev, 2);
>> +    adev->doorbell.size = pci_resource_len(adev->pdev, 2);
>> +
>> +    if (adev->enable_mes) {
>> +        adev->doorbell.num_kernel_doorbells =
>> +            adev->doorbell.size / sizeof(u32);
>> +    } else {
>> +        adev->doorbell.num_kernel_doorbells =
>> +            min_t(u32, adev->doorbell.size / sizeof(u32),
>> +                  adev->doorbell_index.max_assignment+1);
>> +        if (adev->doorbell.num_kernel_doorbells == 0)
>> +            return -EINVAL;
>> +
>> +        /* For Vega, reserve and map two pages on doorbell BAR since 
>> SDMA
>> +         * paging queue doorbell use the second page. The
>> +         * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
>> +         * doorbells are in the first page. So with paging queue 
>> enabled,
>> +         * the max num_kernel_doorbells should + 1 page (0x400 in 
>> dword)
>> +         */
>> +        if (adev->asic_type >= CHIP_VEGA10)
>> +            adev->doorbell.num_kernel_doorbells += 0x400;
>> +    }
>> +
>> +    adev->doorbell.ptr = ioremap(adev->doorbell.base,
>> +                     adev->doorbell.num_kernel_doorbells *
>> +                     sizeof(u32));
>
> It might be a good idea to drop this. Or do we really have any use 
> case where we need to access doorbells outside of the kernel reserved 
> ones? (If yes, forget what I've said this looks good then).

Forget what I've wrote here. Just seen than you clean this up in patch #6.

With the license question solved feel free to add Reviewed-by: Christian 
König <christian.koenig@amd.com> to the patch.

Regards,
Christian.

>
> Christian.
>
>> +    if (adev->doorbell.ptr == NULL)
>> +        return -ENOMEM;
>> +
>> +    return 0;
>> +}
>> +
>> +/**
>> + * amdgpu_doorbell_fini - Tear down doorbell driver information.
>> + *
>> + * @adev: amdgpu_device pointer
>> + *
>> + * Tear down doorbell driver information (CIK)
>> + */
>> +void amdgpu_doorbell_fini(struct amdgpu_device *adev)
>> +{
>> +    iounmap(adev->doorbell.ptr);
>> +    adev->doorbell.ptr = NULL;
>> +}
>