Re: [PATCH 07/16] drm/amdgpu: add helper to create doorbell pages

["Christian König" <ckoenig.leichtzumerken@gmail.com>]

Am 29.03.23 um 17:47 schrieb Shashank Sharma:
> From: Shashank Sharma <contactshashanksharma@gmail.com>
>
> This patch adds helper functions to create and free doorbell
> pages for kernel objects.
>
> Cc: Alex Deucher <alexander.deucher@amd.com>
> Cc: Christian Koenig <christian.koenig@amd.com>
> Signed-off-by: Shashank Sharma <shashank.sharma@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h  | 41 ++++++++++++++++
>   .../gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c  | 49 +++++++++++++++++++
>   2 files changed, 90 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
> index f9c3b77bf65d..6581b78fe438 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
> @@ -27,6 +27,24 @@
>   /*
>    * GPU doorbell structures, functions & helpers
>    */
> +
> +/* Structure to hold doorbell pages from PCI doorbell BAR */
> +struct amdgpu_doorbell_obj {
> +	struct amdgpu_bo *bo;
> +	uint64_t gpu_addr;

Do we need to map the doorbells into the GART for something?

On the other hand we probably don't have the check to not do it in 
amdgpu_bo_create_reserved yet.

> +	uint32_t *cpu_addr;
> +	uint32_t size;
> +
> +	/* First index in this object */
> +	uint32_t start;
> +
> +	/* Last index in this object */
> +	uint32_t end;
> +
> +	/* bitmap for dynamic doorbell allocation from this object */
> +	unsigned long *doorbell_bitmap;
> +};
> +

Do we really need a separate structure for all this? As far as I can see 
this is just part of the doorbell handling.

>   struct amdgpu_doorbell {
>   	/* doorbell mmio */
>   	resource_size_t		base;
> @@ -328,6 +346,29 @@ int amdgpu_device_doorbell_init(struct amdgpu_device *adev);
>    */
>   void amdgpu_device_doorbell_fini(struct amdgpu_device *adev);
>   
> +/**
> + * amdgpu_doorbell_free_page - Free a doorbell page
> + *
> + * @adev: amdgpu_device pointer
> + *
> + * @db_age: previously allocated doobell page details
> + *
> + */
> +void amdgpu_doorbell_free_page(struct amdgpu_device *adev,
> +				struct amdgpu_doorbell_obj *db_obj);
> +
> +/**
> + * amdgpu_doorbell_alloc_page - create a page from doorbell pool
> + *
> + * @adev: amdgpu_device pointer
> + *
> + * @db_age: doobell page structure to fill details with
> + *
> + * returns 0 on success, else error number
> + */
> +int amdgpu_doorbell_alloc_page(struct amdgpu_device *adev,
> +				struct amdgpu_doorbell_obj *db_obj);
> +
>   #define RDOORBELL32(index) amdgpu_mm_rdoorbell(adev, (index))
>   #define WDOORBELL32(index, v) amdgpu_mm_wdoorbell(adev, (index), (v))
>   #define RDOORBELL64(index) amdgpu_mm_rdoorbell64(adev, (index))
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
> index 1aea92363fd3..8be15b82b545 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
> @@ -111,6 +111,55 @@ void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
>   	}
>   }
>   
> +/**
> + * amdgpu_doorbell_free_page - Free a doorbell page
> + *
> + * @adev: amdgpu_device pointer
> + *
> + * @db_age: previously allocated doobell page details
> + *
> + */
> +void amdgpu_doorbell_free_page(struct amdgpu_device *adev,
> +					struct amdgpu_doorbell_obj *db_obj)
> +{
> +	amdgpu_bo_free_kernel(&db_obj->bo,
> +			      &db_obj->gpu_addr,
> +			      (void **)&db_obj->cpu_addr);
> +
> +}
> +
> +/**
> + * amdgpu_doorbell_alloc_page - create a page from doorbell pool
> + *
> + * @adev: amdgpu_device pointer
> + *
> + * @db_age: doobell page structure to fill details with
> + *
> + * returns 0 on success, else error number
> + */
> +int amdgpu_doorbell_alloc_page(struct amdgpu_device *adev,
> +				struct amdgpu_doorbell_obj *db_obj)
> +{
> +	int r;
> +
> +	db_obj->size = ALIGN(db_obj->size, PAGE_SIZE);
> +
> +	r = amdgpu_bo_create_kernel(adev,
> +				    db_obj->size,
> +				    PAGE_SIZE,
> +				    AMDGPU_GEM_DOMAIN_DOORBELL,
> +				    &db_obj->bo,
> +				    &db_obj->gpu_addr,
> +				    (void **)&db_obj->cpu_addr);
> +
> +	if (r) {
> +		DRM_ERROR("Failed to create doorbell BO, err=%d\n", r);
> +		return r;
> +	}
> +
> +	return 0;
> +}
> +
>   /*
>    * GPU doorbell aperture helpers function.
>    */