[PATCH 1/3] drm/amdgpu: add new mes misc api for gfx11

[PATCH 1/3] drm/amdgpu: add new mes misc api for gfx11

[Eric Huang]

new api will be used to workaround a HW scheduler issue
for 100% usage when gpu has no workload.

Signed-off-by: Eric Huang <jinhuieric.huang@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 23 +++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h |  4 ++++
 drivers/gpu/drm/amd/amdgpu/mes_v11_0.c  | 10 +++++++++-
 3 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index b96f94e5169f..421d937c1188 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -1140,6 +1140,29 @@ void amdgpu_mes_free_gang_ctx_index(struct amdgpu_mes *mes,
 	amdgpu_mes_unlock(mes);
 }
 
+int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev)
+{
+	struct mes_misc_op_input op_input = {0};
+	int r;
+
+	op_input.op = MES_MISC_NOTIFY_WORK_ON_UNMAPPED_QUEUE;
+
+	if (!adev->mes.funcs->misc_op) {
+		dev_err(adev->dev, "mes notify unmap queue is not supported!\n");
+		r = -EINVAL;
+		goto error;
+	}
+
+	amdgpu_mes_lock(&adev->mes);
+	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
+	amdgpu_mes_unlock(&adev->mes);
+	if (r)
+		dev_err(adev->dev, "failed to notify unmap queue.\n");
+
+error:
+	return r;
+}
+
 #if defined(CONFIG_DEBUG_FS)
 
 static int amdgpu_debugfs_mes_event_log_show(struct seq_file *m, void *unused)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
index 977c057dcce8..0291ae14a1e7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
@@ -366,6 +366,7 @@ enum mes_misc_opcode {
 	MES_MISC_OP_WRM_REG_WR_WAIT,
 	MES_MISC_OP_SET_SHADER_DEBUGGER,
 	MES_MISC_OP_CHANGE_CONFIG,
+	MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE
 };
 
 struct mes_misc_op_input {
@@ -643,4 +644,7 @@ int amdgpu_mes_alloc_gang_ctx_index(struct amdgpu_mes *mes,
 				    uint32_t *index);
 void amdgpu_mes_free_gang_ctx_index(struct amdgpu_mes *mes,
 				    uint32_t index);
+
+int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev);
+
 #endif /* __AMDGPU_MES_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
index c57bcb9a98b1..df71e9447b35 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
@@ -957,7 +957,15 @@ static int mes_v11_0_misc_op(struct amdgpu_mes *mes,
 		misc_pkt.change_config.option.bits.limit_single_process =
 				input->change_config.option.limit_single_process;
 		break;
-
+	case MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE:
+		if ((mes->adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) < 0x70) {
+			dev_warn_once(mes->adev->dev,
+				      "MES FW version must be larger than 0x70 to support notify work on unmapped queue.\n");
+			return 0;
+		}
+		misc_pkt.opcode = MESAPI_MISC__NOTIFY_WORK_ON_UNMAPPED_QUEUE;
+		misc_pkt.queue_sch_level = AMD_PRIORITY_LEVEL_NORMAL;
+		break;
 	default:
 		drm_err(adev_to_drm(mes->adev), "unsupported misc op (%d)\n", input->op);
 		return -EINVAL;
-- 
2.34.1

[PATCH 2/3] drm/amdkfd: workaround 100% gpu usage issue for gfx11

[Eric Huang]

the issue only happens with oversubscription when gpu has no
workload, the root cause is mes oversubscription timer, so
disable mes timer and make a similar timer in kfd to resolve
the issue.

Signed-off-by: Eric Huang <jinhuieric.huang@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/mes_v11_0.c        |  1 -
 .../drm/amd/amdkfd/kfd_device_queue_manager.c | 39 ++++++++++++++++++-
 .../drm/amd/amdkfd/kfd_device_queue_manager.h |  1 +
 3 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
index df71e9447b35..33ff1afd7c4c 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
@@ -1024,7 +1024,6 @@ static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
 	mes_set_hw_res_pkt.use_different_vmid_compute = 1;
 	mes_set_hw_res_pkt.enable_reg_active_poll = 1;
 	mes_set_hw_res_pkt.enable_level_process_quantum_check = 1;
-	mes_set_hw_res_pkt.oversubscription_timer = 50;
 	if (adev->mes.use_rs64mem)
 		mes_set_hw_res_pkt.use_rs64mem_for_proc_gang_ctx = 1;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index a23384571193..45e039c9f31b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -47,6 +47,9 @@
 /* See unmap_queues_cpsch() */
 #define USE_DEFAULT_GRACE_PERIOD 0xffffffff
 
+/* Interval for notifying MES of work on unmapped queues during oversubscription */
+#define DQM_MES_UNMAP_NOTIFY_DELAY_MS 50
+
 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
 				  u32 pasid, unsigned int vmid);
 
@@ -276,8 +279,16 @@ static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
 			q->properties.doorbell_off);
 		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
 		kfd_hws_hang(dqm);
+		return r;
 	}
 
+	/* GFX11: start notify timer only once when oversubscription begins */
+	if (KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) &&
+	    KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) &&
+	    dqm->active_cp_queue_count > get_cp_queues_num(dqm))
+		queue_delayed_work(system_wq, &dqm->notify_unmap_work,
+				   msecs_to_jiffies(DQM_MES_UNMAP_NOTIFY_DELAY_MS));
+
 	return r;
 }
 
@@ -354,7 +365,16 @@ static void set_perfcount(struct device_queue_manager *dqm, int enable)
 static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
 			    struct qcm_process_device *qpd)
 {
-	return remove_queue_mes_on_reset_option(dqm, q, qpd, false, false);
+	int r = remove_queue_mes_on_reset_option(dqm, q, qpd, false, false);
+
+	/* GFX11: stop notify timer when oversubscription clears */
+	if (!r &&
+	    KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) &&
+	    KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) &&
+	    dqm->active_cp_queue_count <= get_cp_queues_num(dqm))
+		cancel_delayed_work(&dqm->notify_unmap_work);
+
+	return r;
 }
 
 static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm)
@@ -3194,6 +3214,20 @@ static void deallocate_hiq_sdma_mqd(struct kfd_node *dev,
 	amdgpu_amdkfd_free_kernel_mem(dev->adev, &mqd->mem);
 }
 
+static void mes_notify_unmap_work_handler(struct work_struct *work)
+{
+	struct device_queue_manager *dqm =
+		container_of(work, struct device_queue_manager,
+			     notify_unmap_work.work);
+
+	amdgpu_mes_notify_unmap_queue((struct amdgpu_device *)dqm->dev->adev);
+
+	/* Re-arm if still oversubscribed */
+	if (READ_ONCE(dqm->active_cp_queue_count) > get_cp_queues_num(dqm))
+		queue_delayed_work(system_wq, &dqm->notify_unmap_work,
+				   msecs_to_jiffies(DQM_MES_UNMAP_NOTIFY_DELAY_MS));
+}
+
 struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
 {
 	struct device_queue_manager *dqm;
@@ -3319,6 +3353,8 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
 
 	if (!dqm->ops.initialize(dqm)) {
 		init_waitqueue_head(&dqm->destroy_wait);
+		INIT_DELAYED_WORK(&dqm->notify_unmap_work,
+				  mes_notify_unmap_work_handler);
 		return dqm;
 	}
 
@@ -3335,6 +3371,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
 
 void device_queue_manager_uninit(struct device_queue_manager *dqm)
 {
+	cancel_delayed_work_sync(&dqm->notify_unmap_work);
 	dqm->ops.stop(dqm);
 	dqm->ops.uninitialize(dqm);
 	if (!dqm->dev->kfd->shared_resources.enable_mes)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index c9f9f7a87111..21cf3c16f3f9 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -281,6 +281,7 @@ struct device_queue_manager {
 	uint32_t		wait_times;
 
 	wait_queue_head_t	destroy_wait;
+	struct delayed_work	notify_unmap_work;
 
 	/* for per-queue reset support */
 	struct dqm_detect_hang_info *detect_hang_info;
-- 
2.34.1

[PATCH 3/3] drm/amdgpu/userq: create the same workaround of oversubscription timer

[Eric Huang]

removing MES oversubscription timer will affect both amdgpu/amdkfd, so
add the similar timer for amdgpu userq as well.

Signed-off-by: Eric Huang <jinhuieric.huang@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c    | 70 ++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h    |  7 +++
 drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 10 +++-
 3 files changed, 85 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index 421d937c1188..b2c64ec8e0af 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -101,6 +101,8 @@ static inline u32 amdgpu_mes_get_hqd_mask(u32 num_pipe,
 	return (total_hqd_mask & ~reserved_hqd_mask);
 }
 
+static void amdgpu_mes_userq_notify_unmap_work_handler(struct work_struct *work);
+
 int amdgpu_mes_init(struct amdgpu_device *adev)
 {
 	int i, r, num_pipes, num_queues = 0;
@@ -123,6 +125,9 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 		spin_lock_init(&adev->mes.ring_lock[i]);
 
 	adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;
+	atomic_set(&adev->mes.userq_hw_queue_count, 0);
+	INIT_DELAYED_WORK(&adev->mes.userq_notify_unmap_work,
+			  amdgpu_mes_userq_notify_unmap_work_handler);
 	total_vmid_mask = (u32)((1UL << 16) - 1);
 	reserved_vmid_mask = (u32)((1UL << adev->vm_manager.first_kfd_vmid) - 1);
 
@@ -288,6 +293,8 @@ void amdgpu_mes_fini(struct amdgpu_device *adev)
 	int i;
 	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
 
+	cancel_delayed_work_sync(&adev->mes.userq_notify_unmap_work);
+
 	kfree(adev->gfx.mec.mes_hung_db_array);
 
 	amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj,
@@ -1163,6 +1170,69 @@ int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev)
 	return r;
 }
 
+/* Interval for notifying MES of work on unmapped queues during oversubscription */
+#define AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_MS 50
+
+static unsigned int amdgpu_mes_userq_hw_queue_num(struct amdgpu_device *adev)
+{
+	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
+	unsigned int n = bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
+	int i;
+
+	for (i = 0; i < num_xcc; i++)
+		n += bitmap_weight(adev->gfx.mec_bitmap[i].queue_bitmap,
+				    AMDGPU_MAX_COMPUTE_QUEUES);
+
+	return n;
+}
+
+static void amdgpu_mes_userq_notify_unmap_work_handler(struct work_struct *work)
+{
+	struct amdgpu_mes *mes = container_of(work, struct amdgpu_mes,
+					       userq_notify_unmap_work.work);
+	struct amdgpu_device *adev = mes->adev;
+
+	amdgpu_mes_notify_unmap_queue(adev);
+
+	/* Re-arm if still oversubscribed */
+	if (atomic_read(&mes->userq_hw_queue_count) >
+	    amdgpu_mes_userq_hw_queue_num(adev))
+		queue_delayed_work(system_wq, &mes->userq_notify_unmap_work,
+				   msecs_to_jiffies(AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_MS));
+}
+
+/*
+ * Called after a GFX11 usermode queue is successfully mapped to MES.
+ * Starts the periodic unmap-notify timer if this pushed the device into
+ * HW queue oversubscription.
+ */
+void amdgpu_mes_userq_queue_mapped(struct amdgpu_device *adev)
+{
+	if (!(amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) &&
+	      amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)))
+		return;
+
+	if (atomic_inc_return(&adev->mes.userq_hw_queue_count) >
+	    amdgpu_mes_userq_hw_queue_num(adev))
+		queue_delayed_work(system_wq, &adev->mes.userq_notify_unmap_work,
+				   msecs_to_jiffies(AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_MS));
+}
+
+/*
+ * Called after a GFX11 usermode queue is unmapped from MES. Stops the
+ * periodic unmap-notify timer once oversubscription clears.
+ */
+void amdgpu_mes_userq_queue_unmapped(struct amdgpu_device *adev)
+{
+	if (!(amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) &&
+	      amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)))
+		return;
+
+	if (atomic_dec_return(&adev->mes.userq_hw_queue_count) <=
+	    amdgpu_mes_userq_hw_queue_num(adev))
+		cancel_delayed_work(&adev->mes.userq_notify_unmap_work);
+}
+
 #if defined(CONFIG_DEBUG_FS)
 
 static int amdgpu_debugfs_mes_event_log_show(struct seq_file *m, void *unused)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
index 0291ae14a1e7..5160b227943b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
@@ -29,6 +29,7 @@
 #include "amdgpu_gfx.h"
 #include "amdgpu_doorbell.h"
 #include <linux/sched/mm.h>
+#include <linux/workqueue.h>
 
 #define AMDGPU_MES_MAX_COMPUTE_PIPES        8
 #define AMDGPU_MES_MAX_GFX_PIPES            2
@@ -90,6 +91,10 @@ struct amdgpu_mes {
 	uint32_t                        total_max_queue;
 	uint32_t                        max_doorbell_slices;
 
+	/* GFX11 usermode queue oversubscription notify timer */
+	atomic_t                        userq_hw_queue_count;
+	struct delayed_work             userq_notify_unmap_work;
+
 	uint64_t                        default_process_quantum;
 	uint64_t                        default_gang_quantum;
 
@@ -646,5 +651,7 @@ void amdgpu_mes_free_gang_ctx_index(struct amdgpu_mes *mes,
 				    uint32_t index);
 
 int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev);
+void amdgpu_mes_userq_queue_mapped(struct amdgpu_device *adev);
+void amdgpu_mes_userq_queue_unmapped(struct amdgpu_device *adev);
 
 #endif /* __AMDGPU_MES_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
index fae709f134bb..5a10af16e0c1 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
@@ -171,6 +171,8 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
 		return r;
 	}
 
+	amdgpu_mes_userq_queue_mapped(adev);
+
 	DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index);
 	return 0;
 }
@@ -195,9 +197,13 @@ static int mes_userq_unmap(struct amdgpu_usermode_queue *queue)
 	amdgpu_mes_unlock(&adev->mes);
 	if (mes->use_rs64mem)
 		amdgpu_mes_free_gang_ctx_index(mes, queue->gang_ctx_array_index);
-	if (r)
+	if (r) {
 		DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r);
-	return r;
+		return r;
+	}
+
+	amdgpu_mes_userq_queue_unmapped(adev);
+	return 0;
 }
 
 int mes_userq_reset(struct amdgpu_usermode_queue *queue)
-- 
2.34.1

Re: [PATCH 1/3] drm/amdgpu: add new mes misc api for gfx11

[Mario Limonciello]

On 8/19/26 15:14, Eric Huang wrote:
> new api will be used to workaround a HW scheduler issue
> for 100% usage when gpu has no workload.
> 
> Signed-off-by: Eric Huang <jinhuieric.huang@amd.com>
Reviewed-by: Mario Limonciello (AMD) <superm1@kernel.org>> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 23 +++++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h |  4 ++++
>   drivers/gpu/drm/amd/amdgpu/mes_v11_0.c  | 10 +++++++++-
>   3 files changed, 36 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> index b96f94e5169f..421d937c1188 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> @@ -1140,6 +1140,29 @@ void amdgpu_mes_free_gang_ctx_index(struct amdgpu_mes *mes,
>   	amdgpu_mes_unlock(mes);
>   }
>   
> +int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev)
> +{
> +	struct mes_misc_op_input op_input = {0};
> +	int r;
> +
> +	op_input.op = MES_MISC_NOTIFY_WORK_ON_UNMAPPED_QUEUE;
> +
> +	if (!adev->mes.funcs->misc_op) {
> +		dev_err(adev->dev, "mes notify unmap queue is not supported!\n");
> +		r = -EINVAL;
> +		goto error;
> +	}
> +
> +	amdgpu_mes_lock(&adev->mes);
> +	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
> +	amdgpu_mes_unlock(&adev->mes);
> +	if (r)
> +		dev_err(adev->dev, "failed to notify unmap queue.\n");
> +
> +error:
> +	return r;
> +}
> +
>   #if defined(CONFIG_DEBUG_FS)
>   
>   static int amdgpu_debugfs_mes_event_log_show(struct seq_file *m, void *unused)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> index 977c057dcce8..0291ae14a1e7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> @@ -366,6 +366,7 @@ enum mes_misc_opcode {
>   	MES_MISC_OP_WRM_REG_WR_WAIT,
>   	MES_MISC_OP_SET_SHADER_DEBUGGER,
>   	MES_MISC_OP_CHANGE_CONFIG,
> +	MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE
>   };
>   
>   struct mes_misc_op_input {
> @@ -643,4 +644,7 @@ int amdgpu_mes_alloc_gang_ctx_index(struct amdgpu_mes *mes,
>   				    uint32_t *index);
>   void amdgpu_mes_free_gang_ctx_index(struct amdgpu_mes *mes,
>   				    uint32_t index);
> +
> +int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev);
> +
>   #endif /* __AMDGPU_MES_H__ */
> diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
> index c57bcb9a98b1..df71e9447b35 100644
> --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
> @@ -957,7 +957,15 @@ static int mes_v11_0_misc_op(struct amdgpu_mes *mes,
>   		misc_pkt.change_config.option.bits.limit_single_process =
>   				input->change_config.option.limit_single_process;
>   		break;
> -
> +	case MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE:
> +		if ((mes->adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) < 0x70) {
> +			dev_warn_once(mes->adev->dev,
> +				      "MES FW version must be larger than 0x70 to support notify work on unmapped queue.\n");
> +			return 0;
> +		}
> +		misc_pkt.opcode = MESAPI_MISC__NOTIFY_WORK_ON_UNMAPPED_QUEUE;
> +		misc_pkt.queue_sch_level = AMD_PRIORITY_LEVEL_NORMAL;
> +		break;
>   	default:
>   		drm_err(adev_to_drm(mes->adev), "unsupported misc op (%d)\n", input->op);
>   		return -EINVAL;

Re: [PATCH 2/3] drm/amdkfd: workaround 100% gpu usage issue for gfx11

[Mario Limonciello]

On 8/19/26 15:14, Eric Huang wrote:
> the issue only happens with oversubscription when gpu has no
> workload, the root cause is mes oversubscription timer, so
> disable mes timer and make a similar timer in kfd to resolve
> the issue.
> 
> Signed-off-by: Eric Huang <jinhuieric.huang@amd.com>
Reviewed-by: Mario Limonciello (AMD) <superm1@kernel.org>
> ---
>   drivers/gpu/drm/amd/amdgpu/mes_v11_0.c        |  1 -
>   .../drm/amd/amdkfd/kfd_device_queue_manager.c | 39 ++++++++++++++++++-
>   .../drm/amd/amdkfd/kfd_device_queue_manager.h |  1 +
>   3 files changed, 39 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
> index df71e9447b35..33ff1afd7c4c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
> @@ -1024,7 +1024,6 @@ static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
>   	mes_set_hw_res_pkt.use_different_vmid_compute = 1;
>   	mes_set_hw_res_pkt.enable_reg_active_poll = 1;
>   	mes_set_hw_res_pkt.enable_level_process_quantum_check = 1;
> -	mes_set_hw_res_pkt.oversubscription_timer = 50;
>   	if (adev->mes.use_rs64mem)
>   		mes_set_hw_res_pkt.use_rs64mem_for_proc_gang_ctx = 1;
>   
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> index a23384571193..45e039c9f31b 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> @@ -47,6 +47,9 @@
>   /* See unmap_queues_cpsch() */
>   #define USE_DEFAULT_GRACE_PERIOD 0xffffffff
>   
> +/* Interval for notifying MES of work on unmapped queues during oversubscription */
> +#define DQM_MES_UNMAP_NOTIFY_DELAY_MS 50
> +
>   static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
>   				  u32 pasid, unsigned int vmid);
>   
> @@ -276,8 +279,16 @@ static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
>   			q->properties.doorbell_off);
>   		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
>   		kfd_hws_hang(dqm);
> +		return r;
>   	}
>   
> +	/* GFX11: start notify timer only once when oversubscription begins */
> +	if (KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) &&
> +	    KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) &&
> +	    dqm->active_cp_queue_count > get_cp_queues_num(dqm))
> +		queue_delayed_work(system_wq, &dqm->notify_unmap_work,
> +				   msecs_to_jiffies(DQM_MES_UNMAP_NOTIFY_DELAY_MS));
> +
>   	return r;
>   }
>   
> @@ -354,7 +365,16 @@ static void set_perfcount(struct device_queue_manager *dqm, int enable)
>   static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
>   			    struct qcm_process_device *qpd)
>   {
> -	return remove_queue_mes_on_reset_option(dqm, q, qpd, false, false);
> +	int r = remove_queue_mes_on_reset_option(dqm, q, qpd, false, false);
> +
> +	/* GFX11: stop notify timer when oversubscription clears */
> +	if (!r &&
> +	    KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) &&
> +	    KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) &&
> +	    dqm->active_cp_queue_count <= get_cp_queues_num(dqm))
> +		cancel_delayed_work(&dqm->notify_unmap_work);
> +
> +	return r;
>   }
>   
>   static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm)
> @@ -3194,6 +3214,20 @@ static void deallocate_hiq_sdma_mqd(struct kfd_node *dev,
>   	amdgpu_amdkfd_free_kernel_mem(dev->adev, &mqd->mem);
>   }
>   
> +static void mes_notify_unmap_work_handler(struct work_struct *work)
> +{
> +	struct device_queue_manager *dqm =
> +		container_of(work, struct device_queue_manager,
> +			     notify_unmap_work.work);
> +
> +	amdgpu_mes_notify_unmap_queue((struct amdgpu_device *)dqm->dev->adev);
> +
> +	/* Re-arm if still oversubscribed */
> +	if (READ_ONCE(dqm->active_cp_queue_count) > get_cp_queues_num(dqm))
> +		queue_delayed_work(system_wq, &dqm->notify_unmap_work,
> +				   msecs_to_jiffies(DQM_MES_UNMAP_NOTIFY_DELAY_MS));
> +}
> +
>   struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
>   {
>   	struct device_queue_manager *dqm;
> @@ -3319,6 +3353,8 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
>   
>   	if (!dqm->ops.initialize(dqm)) {
>   		init_waitqueue_head(&dqm->destroy_wait);
> +		INIT_DELAYED_WORK(&dqm->notify_unmap_work,
> +				  mes_notify_unmap_work_handler);
>   		return dqm;
>   	}
>   
> @@ -3335,6 +3371,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
>   
>   void device_queue_manager_uninit(struct device_queue_manager *dqm)
>   {
> +	cancel_delayed_work_sync(&dqm->notify_unmap_work);
>   	dqm->ops.stop(dqm);
>   	dqm->ops.uninitialize(dqm);
>   	if (!dqm->dev->kfd->shared_resources.enable_mes)
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
> index c9f9f7a87111..21cf3c16f3f9 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
> @@ -281,6 +281,7 @@ struct device_queue_manager {
>   	uint32_t		wait_times;
>   
>   	wait_queue_head_t	destroy_wait;
> +	struct delayed_work	notify_unmap_work;
>   
>   	/* for per-queue reset support */
>   	struct dqm_detect_hang_info *detect_hang_info;

Re: [PATCH 3/3] drm/amdgpu/userq: create the same workaround of oversubscription timer

[Mario Limonciello]

On 8/19/26 15:14, Eric Huang wrote:
> removing MES oversubscription timer will affect both amdgpu/amdkfd, so
> add the similar timer for amdgpu userq as well.
> 
> Signed-off-by: Eric Huang <jinhuieric.huang@amd.com>
Reviewed-by: Mario Limonciello (AMD) <superm1@kernel.org>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c    | 70 ++++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h    |  7 +++
>   drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 10 +++-
>   3 files changed, 85 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> index 421d937c1188..b2c64ec8e0af 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> @@ -101,6 +101,8 @@ static inline u32 amdgpu_mes_get_hqd_mask(u32 num_pipe,
>   	return (total_hqd_mask & ~reserved_hqd_mask);
>   }
>   
> +static void amdgpu_mes_userq_notify_unmap_work_handler(struct work_struct *work);
> +
>   int amdgpu_mes_init(struct amdgpu_device *adev)
>   {
>   	int i, r, num_pipes, num_queues = 0;
> @@ -123,6 +125,9 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
>   		spin_lock_init(&adev->mes.ring_lock[i]);
>   
>   	adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;
> +	atomic_set(&adev->mes.userq_hw_queue_count, 0);
> +	INIT_DELAYED_WORK(&adev->mes.userq_notify_unmap_work,
> +			  amdgpu_mes_userq_notify_unmap_work_handler);
>   	total_vmid_mask = (u32)((1UL << 16) - 1);
>   	reserved_vmid_mask = (u32)((1UL << adev->vm_manager.first_kfd_vmid) - 1);
>   
> @@ -288,6 +293,8 @@ void amdgpu_mes_fini(struct amdgpu_device *adev)
>   	int i;
>   	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
>   
> +	cancel_delayed_work_sync(&adev->mes.userq_notify_unmap_work);
> +
>   	kfree(adev->gfx.mec.mes_hung_db_array);
>   
>   	amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj,
> @@ -1163,6 +1170,69 @@ int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev)
>   	return r;
>   }
>   
> +/* Interval for notifying MES of work on unmapped queues during oversubscription */
> +#define AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_MS 50
> +
> +static unsigned int amdgpu_mes_userq_hw_queue_num(struct amdgpu_device *adev)
> +{
> +	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
> +	unsigned int n = bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
> +	int i;
> +
> +	for (i = 0; i < num_xcc; i++)
> +		n += bitmap_weight(adev->gfx.mec_bitmap[i].queue_bitmap,
> +				    AMDGPU_MAX_COMPUTE_QUEUES);
> +
> +	return n;
> +}
> +
> +static void amdgpu_mes_userq_notify_unmap_work_handler(struct work_struct *work)
> +{
> +	struct amdgpu_mes *mes = container_of(work, struct amdgpu_mes,
> +					       userq_notify_unmap_work.work);
> +	struct amdgpu_device *adev = mes->adev;
> +
> +	amdgpu_mes_notify_unmap_queue(adev);
> +
> +	/* Re-arm if still oversubscribed */
> +	if (atomic_read(&mes->userq_hw_queue_count) >
> +	    amdgpu_mes_userq_hw_queue_num(adev))
> +		queue_delayed_work(system_wq, &mes->userq_notify_unmap_work,
> +				   msecs_to_jiffies(AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_MS));
> +}
> +
> +/*
> + * Called after a GFX11 usermode queue is successfully mapped to MES.
> + * Starts the periodic unmap-notify timer if this pushed the device into
> + * HW queue oversubscription.
> + */
> +void amdgpu_mes_userq_queue_mapped(struct amdgpu_device *adev)
> +{
> +	if (!(amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) &&
> +	      amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)))
> +		return;
> +
> +	if (atomic_inc_return(&adev->mes.userq_hw_queue_count) >
> +	    amdgpu_mes_userq_hw_queue_num(adev))
> +		queue_delayed_work(system_wq, &adev->mes.userq_notify_unmap_work,
> +				   msecs_to_jiffies(AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_MS));
> +}
> +
> +/*
> + * Called after a GFX11 usermode queue is unmapped from MES. Stops the
> + * periodic unmap-notify timer once oversubscription clears.
> + */
> +void amdgpu_mes_userq_queue_unmapped(struct amdgpu_device *adev)
> +{
> +	if (!(amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) &&
> +	      amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)))
> +		return;
> +
> +	if (atomic_dec_return(&adev->mes.userq_hw_queue_count) <=
> +	    amdgpu_mes_userq_hw_queue_num(adev))
> +		cancel_delayed_work(&adev->mes.userq_notify_unmap_work);
> +}
> +
>   #if defined(CONFIG_DEBUG_FS)
>   
>   static int amdgpu_debugfs_mes_event_log_show(struct seq_file *m, void *unused)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> index 0291ae14a1e7..5160b227943b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> @@ -29,6 +29,7 @@
>   #include "amdgpu_gfx.h"
>   #include "amdgpu_doorbell.h"
>   #include <linux/sched/mm.h>
> +#include <linux/workqueue.h>
>   
>   #define AMDGPU_MES_MAX_COMPUTE_PIPES        8
>   #define AMDGPU_MES_MAX_GFX_PIPES            2
> @@ -90,6 +91,10 @@ struct amdgpu_mes {
>   	uint32_t                        total_max_queue;
>   	uint32_t                        max_doorbell_slices;
>   
> +	/* GFX11 usermode queue oversubscription notify timer */
> +	atomic_t                        userq_hw_queue_count;
> +	struct delayed_work             userq_notify_unmap_work;
> +
>   	uint64_t                        default_process_quantum;
>   	uint64_t                        default_gang_quantum;
>   
> @@ -646,5 +651,7 @@ void amdgpu_mes_free_gang_ctx_index(struct amdgpu_mes *mes,
>   				    uint32_t index);
>   
>   int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev);
> +void amdgpu_mes_userq_queue_mapped(struct amdgpu_device *adev);
> +void amdgpu_mes_userq_queue_unmapped(struct amdgpu_device *adev);
>   
>   #endif /* __AMDGPU_MES_H__ */
> diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> index fae709f134bb..5a10af16e0c1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> +++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> @@ -171,6 +171,8 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
>   		return r;
>   	}
>   
> +	amdgpu_mes_userq_queue_mapped(adev);
> +
>   	DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index);
>   	return 0;
>   }
> @@ -195,9 +197,13 @@ static int mes_userq_unmap(struct amdgpu_usermode_queue *queue)
>   	amdgpu_mes_unlock(&adev->mes);
>   	if (mes->use_rs64mem)
>   		amdgpu_mes_free_gang_ctx_index(mes, queue->gang_ctx_array_index);
> -	if (r)
> +	if (r) {
>   		DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r);
> -	return r;
> +		return r;
> +	}
> +
> +	amdgpu_mes_userq_queue_unmapped(adev);
> +	return 0;
>   }
>   
>   int mes_userq_reset(struct amdgpu_usermode_queue *queue)