Re: [PATCH 3/3] drm/amdgpu/userq: create the same workaround of oversubscription timer

[Mario Limonciello <mario.limonciello@amd.com>]

On 8/19/26 15:14, Eric Huang wrote:
> removing MES oversubscription timer will affect both amdgpu/amdkfd, so
> add the similar timer for amdgpu userq as well.
> 
> Signed-off-by: Eric Huang <jinhuieric.huang@amd.com>
Reviewed-by: Mario Limonciello (AMD) <superm1@kernel.org>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c    | 70 ++++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h    |  7 +++
>   drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 10 +++-
>   3 files changed, 85 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> index 421d937c1188..b2c64ec8e0af 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> @@ -101,6 +101,8 @@ static inline u32 amdgpu_mes_get_hqd_mask(u32 num_pipe,
>   	return (total_hqd_mask & ~reserved_hqd_mask);
>   }
>   
> +static void amdgpu_mes_userq_notify_unmap_work_handler(struct work_struct *work);
> +
>   int amdgpu_mes_init(struct amdgpu_device *adev)
>   {
>   	int i, r, num_pipes, num_queues = 0;
> @@ -123,6 +125,9 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
>   		spin_lock_init(&adev->mes.ring_lock[i]);
>   
>   	adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;
> +	atomic_set(&adev->mes.userq_hw_queue_count, 0);
> +	INIT_DELAYED_WORK(&adev->mes.userq_notify_unmap_work,
> +			  amdgpu_mes_userq_notify_unmap_work_handler);
>   	total_vmid_mask = (u32)((1UL << 16) - 1);
>   	reserved_vmid_mask = (u32)((1UL << adev->vm_manager.first_kfd_vmid) - 1);
>   
> @@ -288,6 +293,8 @@ void amdgpu_mes_fini(struct amdgpu_device *adev)
>   	int i;
>   	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
>   
> +	cancel_delayed_work_sync(&adev->mes.userq_notify_unmap_work);
> +
>   	kfree(adev->gfx.mec.mes_hung_db_array);
>   
>   	amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj,
> @@ -1163,6 +1170,69 @@ int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev)
>   	return r;
>   }
>   
> +/* Interval for notifying MES of work on unmapped queues during oversubscription */
> +#define AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_MS 50
> +
> +static unsigned int amdgpu_mes_userq_hw_queue_num(struct amdgpu_device *adev)
> +{
> +	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
> +	unsigned int n = bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
> +	int i;
> +
> +	for (i = 0; i < num_xcc; i++)
> +		n += bitmap_weight(adev->gfx.mec_bitmap[i].queue_bitmap,
> +				    AMDGPU_MAX_COMPUTE_QUEUES);
> +
> +	return n;
> +}
> +
> +static void amdgpu_mes_userq_notify_unmap_work_handler(struct work_struct *work)
> +{
> +	struct amdgpu_mes *mes = container_of(work, struct amdgpu_mes,
> +					       userq_notify_unmap_work.work);
> +	struct amdgpu_device *adev = mes->adev;
> +
> +	amdgpu_mes_notify_unmap_queue(adev);
> +
> +	/* Re-arm if still oversubscribed */
> +	if (atomic_read(&mes->userq_hw_queue_count) >
> +	    amdgpu_mes_userq_hw_queue_num(adev))
> +		queue_delayed_work(system_wq, &mes->userq_notify_unmap_work,
> +				   msecs_to_jiffies(AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_MS));
> +}
> +
> +/*
> + * Called after a GFX11 usermode queue is successfully mapped to MES.
> + * Starts the periodic unmap-notify timer if this pushed the device into
> + * HW queue oversubscription.
> + */
> +void amdgpu_mes_userq_queue_mapped(struct amdgpu_device *adev)
> +{
> +	if (!(amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) &&
> +	      amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)))
> +		return;
> +
> +	if (atomic_inc_return(&adev->mes.userq_hw_queue_count) >
> +	    amdgpu_mes_userq_hw_queue_num(adev))
> +		queue_delayed_work(system_wq, &adev->mes.userq_notify_unmap_work,
> +				   msecs_to_jiffies(AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_MS));
> +}
> +
> +/*
> + * Called after a GFX11 usermode queue is unmapped from MES. Stops the
> + * periodic unmap-notify timer once oversubscription clears.
> + */
> +void amdgpu_mes_userq_queue_unmapped(struct amdgpu_device *adev)
> +{
> +	if (!(amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) &&
> +	      amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)))
> +		return;
> +
> +	if (atomic_dec_return(&adev->mes.userq_hw_queue_count) <=
> +	    amdgpu_mes_userq_hw_queue_num(adev))
> +		cancel_delayed_work(&adev->mes.userq_notify_unmap_work);
> +}
> +
>   #if defined(CONFIG_DEBUG_FS)
>   
>   static int amdgpu_debugfs_mes_event_log_show(struct seq_file *m, void *unused)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> index 0291ae14a1e7..5160b227943b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> @@ -29,6 +29,7 @@
>   #include "amdgpu_gfx.h"
>   #include "amdgpu_doorbell.h"
>   #include <linux/sched/mm.h>
> +#include <linux/workqueue.h>
>   
>   #define AMDGPU_MES_MAX_COMPUTE_PIPES        8
>   #define AMDGPU_MES_MAX_GFX_PIPES            2
> @@ -90,6 +91,10 @@ struct amdgpu_mes {
>   	uint32_t                        total_max_queue;
>   	uint32_t                        max_doorbell_slices;
>   
> +	/* GFX11 usermode queue oversubscription notify timer */
> +	atomic_t                        userq_hw_queue_count;
> +	struct delayed_work             userq_notify_unmap_work;
> +
>   	uint64_t                        default_process_quantum;
>   	uint64_t                        default_gang_quantum;
>   
> @@ -646,5 +651,7 @@ void amdgpu_mes_free_gang_ctx_index(struct amdgpu_mes *mes,
>   				    uint32_t index);
>   
>   int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev);
> +void amdgpu_mes_userq_queue_mapped(struct amdgpu_device *adev);
> +void amdgpu_mes_userq_queue_unmapped(struct amdgpu_device *adev);
>   
>   #endif /* __AMDGPU_MES_H__ */
> diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> index fae709f134bb..5a10af16e0c1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> +++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> @@ -171,6 +171,8 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
>   		return r;
>   	}
>   
> +	amdgpu_mes_userq_queue_mapped(adev);
> +
>   	DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index);
>   	return 0;
>   }
> @@ -195,9 +197,13 @@ static int mes_userq_unmap(struct amdgpu_usermode_queue *queue)
>   	amdgpu_mes_unlock(&adev->mes);
>   	if (mes->use_rs64mem)
>   		amdgpu_mes_free_gang_ctx_index(mes, queue->gang_ctx_array_index);
> -	if (r)
> +	if (r) {
>   		DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r);
> -	return r;
> +		return r;
> +	}
> +
> +	amdgpu_mes_userq_queue_unmapped(adev);
> +	return 0;
>   }
>   
>   int mes_userq_reset(struct amdgpu_usermode_queue *queue)