* [PATCH 2/3] drm/amdkfd: workaround 100% gpu usage issue for gfx11
2026-08-19 20:14 [PATCH 1/3] drm/amdgpu: add new mes misc api for gfx11 Eric Huang
@ 2026-08-19 20:14 ` Eric Huang
2026-08-21 21:31 ` Mario Limonciello
2026-08-19 20:14 ` [PATCH 3/3] drm/amdgpu/userq: create the same workaround of oversubscription timer Eric Huang
2026-08-21 21:30 ` [PATCH 1/3] drm/amdgpu: add new mes misc api for gfx11 Mario Limonciello
2 siblings, 1 reply; 6+ messages in thread
From: Eric Huang @ 2026-08-19 20:14 UTC (permalink / raw)
To: amd-gfx; +Cc: Tishko.Araz, Alexander.Deucher, Eric Huang
the issue only happens with oversubscription when gpu has no
workload, the root cause is mes oversubscription timer, so
disable mes timer and make a similar timer in kfd to resolve
the issue.
Signed-off-by: Eric Huang <jinhuieric.huang@amd.com>
---
drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 1 -
.../drm/amd/amdkfd/kfd_device_queue_manager.c | 39 ++++++++++++++++++-
.../drm/amd/amdkfd/kfd_device_queue_manager.h | 1 +
3 files changed, 39 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
index df71e9447b35..33ff1afd7c4c 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
@@ -1024,7 +1024,6 @@ static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
mes_set_hw_res_pkt.use_different_vmid_compute = 1;
mes_set_hw_res_pkt.enable_reg_active_poll = 1;
mes_set_hw_res_pkt.enable_level_process_quantum_check = 1;
- mes_set_hw_res_pkt.oversubscription_timer = 50;
if (adev->mes.use_rs64mem)
mes_set_hw_res_pkt.use_rs64mem_for_proc_gang_ctx = 1;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index a23384571193..45e039c9f31b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -47,6 +47,9 @@
/* See unmap_queues_cpsch() */
#define USE_DEFAULT_GRACE_PERIOD 0xffffffff
+/* Interval for notifying MES of work on unmapped queues during oversubscription */
+#define DQM_MES_UNMAP_NOTIFY_DELAY_MS 50
+
static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
u32 pasid, unsigned int vmid);
@@ -276,8 +279,16 @@ static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
q->properties.doorbell_off);
dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
kfd_hws_hang(dqm);
+ return r;
}
+ /* GFX11: start notify timer only once when oversubscription begins */
+ if (KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) &&
+ KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) &&
+ dqm->active_cp_queue_count > get_cp_queues_num(dqm))
+ queue_delayed_work(system_wq, &dqm->notify_unmap_work,
+ msecs_to_jiffies(DQM_MES_UNMAP_NOTIFY_DELAY_MS));
+
return r;
}
@@ -354,7 +365,16 @@ static void set_perfcount(struct device_queue_manager *dqm, int enable)
static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
struct qcm_process_device *qpd)
{
- return remove_queue_mes_on_reset_option(dqm, q, qpd, false, false);
+ int r = remove_queue_mes_on_reset_option(dqm, q, qpd, false, false);
+
+ /* GFX11: stop notify timer when oversubscription clears */
+ if (!r &&
+ KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) &&
+ KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) &&
+ dqm->active_cp_queue_count <= get_cp_queues_num(dqm))
+ cancel_delayed_work(&dqm->notify_unmap_work);
+
+ return r;
}
static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm)
@@ -3194,6 +3214,20 @@ static void deallocate_hiq_sdma_mqd(struct kfd_node *dev,
amdgpu_amdkfd_free_kernel_mem(dev->adev, &mqd->mem);
}
+static void mes_notify_unmap_work_handler(struct work_struct *work)
+{
+ struct device_queue_manager *dqm =
+ container_of(work, struct device_queue_manager,
+ notify_unmap_work.work);
+
+ amdgpu_mes_notify_unmap_queue((struct amdgpu_device *)dqm->dev->adev);
+
+ /* Re-arm if still oversubscribed */
+ if (READ_ONCE(dqm->active_cp_queue_count) > get_cp_queues_num(dqm))
+ queue_delayed_work(system_wq, &dqm->notify_unmap_work,
+ msecs_to_jiffies(DQM_MES_UNMAP_NOTIFY_DELAY_MS));
+}
+
struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
{
struct device_queue_manager *dqm;
@@ -3319,6 +3353,8 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
if (!dqm->ops.initialize(dqm)) {
init_waitqueue_head(&dqm->destroy_wait);
+ INIT_DELAYED_WORK(&dqm->notify_unmap_work,
+ mes_notify_unmap_work_handler);
return dqm;
}
@@ -3335,6 +3371,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
void device_queue_manager_uninit(struct device_queue_manager *dqm)
{
+ cancel_delayed_work_sync(&dqm->notify_unmap_work);
dqm->ops.stop(dqm);
dqm->ops.uninitialize(dqm);
if (!dqm->dev->kfd->shared_resources.enable_mes)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index c9f9f7a87111..21cf3c16f3f9 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -281,6 +281,7 @@ struct device_queue_manager {
uint32_t wait_times;
wait_queue_head_t destroy_wait;
+ struct delayed_work notify_unmap_work;
/* for per-queue reset support */
struct dqm_detect_hang_info *detect_hang_info;
--
2.34.1
^ permalink raw reply related [flat|nested] 6+ messages in thread* Re: [PATCH 2/3] drm/amdkfd: workaround 100% gpu usage issue for gfx11
2026-08-19 20:14 ` [PATCH 2/3] drm/amdkfd: workaround 100% gpu usage issue " Eric Huang
@ 2026-08-21 21:31 ` Mario Limonciello
0 siblings, 0 replies; 6+ messages in thread
From: Mario Limonciello @ 2026-08-21 21:31 UTC (permalink / raw)
To: Eric Huang, amd-gfx; +Cc: Tishko.Araz, Alexander.Deucher
On 8/19/26 15:14, Eric Huang wrote:
> the issue only happens with oversubscription when gpu has no
> workload, the root cause is mes oversubscription timer, so
> disable mes timer and make a similar timer in kfd to resolve
> the issue.
>
> Signed-off-by: Eric Huang <jinhuieric.huang@amd.com>
Reviewed-by: Mario Limonciello (AMD) <superm1@kernel.org>
> ---
> drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 1 -
> .../drm/amd/amdkfd/kfd_device_queue_manager.c | 39 ++++++++++++++++++-
> .../drm/amd/amdkfd/kfd_device_queue_manager.h | 1 +
> 3 files changed, 39 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
> index df71e9447b35..33ff1afd7c4c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
> @@ -1024,7 +1024,6 @@ static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
> mes_set_hw_res_pkt.use_different_vmid_compute = 1;
> mes_set_hw_res_pkt.enable_reg_active_poll = 1;
> mes_set_hw_res_pkt.enable_level_process_quantum_check = 1;
> - mes_set_hw_res_pkt.oversubscription_timer = 50;
> if (adev->mes.use_rs64mem)
> mes_set_hw_res_pkt.use_rs64mem_for_proc_gang_ctx = 1;
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> index a23384571193..45e039c9f31b 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> @@ -47,6 +47,9 @@
> /* See unmap_queues_cpsch() */
> #define USE_DEFAULT_GRACE_PERIOD 0xffffffff
>
> +/* Interval for notifying MES of work on unmapped queues during oversubscription */
> +#define DQM_MES_UNMAP_NOTIFY_DELAY_MS 50
> +
> static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
> u32 pasid, unsigned int vmid);
>
> @@ -276,8 +279,16 @@ static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
> q->properties.doorbell_off);
> dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
> kfd_hws_hang(dqm);
> + return r;
> }
>
> + /* GFX11: start notify timer only once when oversubscription begins */
> + if (KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) &&
> + KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) &&
> + dqm->active_cp_queue_count > get_cp_queues_num(dqm))
> + queue_delayed_work(system_wq, &dqm->notify_unmap_work,
> + msecs_to_jiffies(DQM_MES_UNMAP_NOTIFY_DELAY_MS));
> +
> return r;
> }
>
> @@ -354,7 +365,16 @@ static void set_perfcount(struct device_queue_manager *dqm, int enable)
> static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
> struct qcm_process_device *qpd)
> {
> - return remove_queue_mes_on_reset_option(dqm, q, qpd, false, false);
> + int r = remove_queue_mes_on_reset_option(dqm, q, qpd, false, false);
> +
> + /* GFX11: stop notify timer when oversubscription clears */
> + if (!r &&
> + KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) &&
> + KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) &&
> + dqm->active_cp_queue_count <= get_cp_queues_num(dqm))
> + cancel_delayed_work(&dqm->notify_unmap_work);
> +
> + return r;
> }
>
> static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm)
> @@ -3194,6 +3214,20 @@ static void deallocate_hiq_sdma_mqd(struct kfd_node *dev,
> amdgpu_amdkfd_free_kernel_mem(dev->adev, &mqd->mem);
> }
>
> +static void mes_notify_unmap_work_handler(struct work_struct *work)
> +{
> + struct device_queue_manager *dqm =
> + container_of(work, struct device_queue_manager,
> + notify_unmap_work.work);
> +
> + amdgpu_mes_notify_unmap_queue((struct amdgpu_device *)dqm->dev->adev);
> +
> + /* Re-arm if still oversubscribed */
> + if (READ_ONCE(dqm->active_cp_queue_count) > get_cp_queues_num(dqm))
> + queue_delayed_work(system_wq, &dqm->notify_unmap_work,
> + msecs_to_jiffies(DQM_MES_UNMAP_NOTIFY_DELAY_MS));
> +}
> +
> struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
> {
> struct device_queue_manager *dqm;
> @@ -3319,6 +3353,8 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
>
> if (!dqm->ops.initialize(dqm)) {
> init_waitqueue_head(&dqm->destroy_wait);
> + INIT_DELAYED_WORK(&dqm->notify_unmap_work,
> + mes_notify_unmap_work_handler);
> return dqm;
> }
>
> @@ -3335,6 +3371,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
>
> void device_queue_manager_uninit(struct device_queue_manager *dqm)
> {
> + cancel_delayed_work_sync(&dqm->notify_unmap_work);
> dqm->ops.stop(dqm);
> dqm->ops.uninitialize(dqm);
> if (!dqm->dev->kfd->shared_resources.enable_mes)
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
> index c9f9f7a87111..21cf3c16f3f9 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
> @@ -281,6 +281,7 @@ struct device_queue_manager {
> uint32_t wait_times;
>
> wait_queue_head_t destroy_wait;
> + struct delayed_work notify_unmap_work;
>
> /* for per-queue reset support */
> struct dqm_detect_hang_info *detect_hang_info;
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH 3/3] drm/amdgpu/userq: create the same workaround of oversubscription timer
2026-08-19 20:14 [PATCH 1/3] drm/amdgpu: add new mes misc api for gfx11 Eric Huang
2026-08-19 20:14 ` [PATCH 2/3] drm/amdkfd: workaround 100% gpu usage issue " Eric Huang
@ 2026-08-19 20:14 ` Eric Huang
2026-08-21 21:32 ` Mario Limonciello
2026-08-21 21:30 ` [PATCH 1/3] drm/amdgpu: add new mes misc api for gfx11 Mario Limonciello
2 siblings, 1 reply; 6+ messages in thread
From: Eric Huang @ 2026-08-19 20:14 UTC (permalink / raw)
To: amd-gfx; +Cc: Tishko.Araz, Alexander.Deucher, Eric Huang
removing MES oversubscription timer will affect both amdgpu/amdkfd, so
add the similar timer for amdgpu userq as well.
Signed-off-by: Eric Huang <jinhuieric.huang@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 70 ++++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h | 7 +++
drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 10 +++-
3 files changed, 85 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index 421d937c1188..b2c64ec8e0af 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -101,6 +101,8 @@ static inline u32 amdgpu_mes_get_hqd_mask(u32 num_pipe,
return (total_hqd_mask & ~reserved_hqd_mask);
}
+static void amdgpu_mes_userq_notify_unmap_work_handler(struct work_struct *work);
+
int amdgpu_mes_init(struct amdgpu_device *adev)
{
int i, r, num_pipes, num_queues = 0;
@@ -123,6 +125,9 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
spin_lock_init(&adev->mes.ring_lock[i]);
adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;
+ atomic_set(&adev->mes.userq_hw_queue_count, 0);
+ INIT_DELAYED_WORK(&adev->mes.userq_notify_unmap_work,
+ amdgpu_mes_userq_notify_unmap_work_handler);
total_vmid_mask = (u32)((1UL << 16) - 1);
reserved_vmid_mask = (u32)((1UL << adev->vm_manager.first_kfd_vmid) - 1);
@@ -288,6 +293,8 @@ void amdgpu_mes_fini(struct amdgpu_device *adev)
int i;
int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
+ cancel_delayed_work_sync(&adev->mes.userq_notify_unmap_work);
+
kfree(adev->gfx.mec.mes_hung_db_array);
amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj,
@@ -1163,6 +1170,69 @@ int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev)
return r;
}
+/* Interval for notifying MES of work on unmapped queues during oversubscription */
+#define AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_MS 50
+
+static unsigned int amdgpu_mes_userq_hw_queue_num(struct amdgpu_device *adev)
+{
+ int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
+ unsigned int n = bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
+ int i;
+
+ for (i = 0; i < num_xcc; i++)
+ n += bitmap_weight(adev->gfx.mec_bitmap[i].queue_bitmap,
+ AMDGPU_MAX_COMPUTE_QUEUES);
+
+ return n;
+}
+
+static void amdgpu_mes_userq_notify_unmap_work_handler(struct work_struct *work)
+{
+ struct amdgpu_mes *mes = container_of(work, struct amdgpu_mes,
+ userq_notify_unmap_work.work);
+ struct amdgpu_device *adev = mes->adev;
+
+ amdgpu_mes_notify_unmap_queue(adev);
+
+ /* Re-arm if still oversubscribed */
+ if (atomic_read(&mes->userq_hw_queue_count) >
+ amdgpu_mes_userq_hw_queue_num(adev))
+ queue_delayed_work(system_wq, &mes->userq_notify_unmap_work,
+ msecs_to_jiffies(AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_MS));
+}
+
+/*
+ * Called after a GFX11 usermode queue is successfully mapped to MES.
+ * Starts the periodic unmap-notify timer if this pushed the device into
+ * HW queue oversubscription.
+ */
+void amdgpu_mes_userq_queue_mapped(struct amdgpu_device *adev)
+{
+ if (!(amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) &&
+ amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)))
+ return;
+
+ if (atomic_inc_return(&adev->mes.userq_hw_queue_count) >
+ amdgpu_mes_userq_hw_queue_num(adev))
+ queue_delayed_work(system_wq, &adev->mes.userq_notify_unmap_work,
+ msecs_to_jiffies(AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_MS));
+}
+
+/*
+ * Called after a GFX11 usermode queue is unmapped from MES. Stops the
+ * periodic unmap-notify timer once oversubscription clears.
+ */
+void amdgpu_mes_userq_queue_unmapped(struct amdgpu_device *adev)
+{
+ if (!(amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) &&
+ amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)))
+ return;
+
+ if (atomic_dec_return(&adev->mes.userq_hw_queue_count) <=
+ amdgpu_mes_userq_hw_queue_num(adev))
+ cancel_delayed_work(&adev->mes.userq_notify_unmap_work);
+}
+
#if defined(CONFIG_DEBUG_FS)
static int amdgpu_debugfs_mes_event_log_show(struct seq_file *m, void *unused)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
index 0291ae14a1e7..5160b227943b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
@@ -29,6 +29,7 @@
#include "amdgpu_gfx.h"
#include "amdgpu_doorbell.h"
#include <linux/sched/mm.h>
+#include <linux/workqueue.h>
#define AMDGPU_MES_MAX_COMPUTE_PIPES 8
#define AMDGPU_MES_MAX_GFX_PIPES 2
@@ -90,6 +91,10 @@ struct amdgpu_mes {
uint32_t total_max_queue;
uint32_t max_doorbell_slices;
+ /* GFX11 usermode queue oversubscription notify timer */
+ atomic_t userq_hw_queue_count;
+ struct delayed_work userq_notify_unmap_work;
+
uint64_t default_process_quantum;
uint64_t default_gang_quantum;
@@ -646,5 +651,7 @@ void amdgpu_mes_free_gang_ctx_index(struct amdgpu_mes *mes,
uint32_t index);
int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev);
+void amdgpu_mes_userq_queue_mapped(struct amdgpu_device *adev);
+void amdgpu_mes_userq_queue_unmapped(struct amdgpu_device *adev);
#endif /* __AMDGPU_MES_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
index fae709f134bb..5a10af16e0c1 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
@@ -171,6 +171,8 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
return r;
}
+ amdgpu_mes_userq_queue_mapped(adev);
+
DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index);
return 0;
}
@@ -195,9 +197,13 @@ static int mes_userq_unmap(struct amdgpu_usermode_queue *queue)
amdgpu_mes_unlock(&adev->mes);
if (mes->use_rs64mem)
amdgpu_mes_free_gang_ctx_index(mes, queue->gang_ctx_array_index);
- if (r)
+ if (r) {
DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r);
- return r;
+ return r;
+ }
+
+ amdgpu_mes_userq_queue_unmapped(adev);
+ return 0;
}
int mes_userq_reset(struct amdgpu_usermode_queue *queue)
--
2.34.1
^ permalink raw reply related [flat|nested] 6+ messages in thread* Re: [PATCH 3/3] drm/amdgpu/userq: create the same workaround of oversubscription timer
2026-08-19 20:14 ` [PATCH 3/3] drm/amdgpu/userq: create the same workaround of oversubscription timer Eric Huang
@ 2026-08-21 21:32 ` Mario Limonciello
0 siblings, 0 replies; 6+ messages in thread
From: Mario Limonciello @ 2026-08-21 21:32 UTC (permalink / raw)
To: Eric Huang, amd-gfx; +Cc: Tishko.Araz, Alexander.Deucher
On 8/19/26 15:14, Eric Huang wrote:
> removing MES oversubscription timer will affect both amdgpu/amdkfd, so
> add the similar timer for amdgpu userq as well.
>
> Signed-off-by: Eric Huang <jinhuieric.huang@amd.com>
Reviewed-by: Mario Limonciello (AMD) <superm1@kernel.org>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 70 ++++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h | 7 +++
> drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 10 +++-
> 3 files changed, 85 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> index 421d937c1188..b2c64ec8e0af 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> @@ -101,6 +101,8 @@ static inline u32 amdgpu_mes_get_hqd_mask(u32 num_pipe,
> return (total_hqd_mask & ~reserved_hqd_mask);
> }
>
> +static void amdgpu_mes_userq_notify_unmap_work_handler(struct work_struct *work);
> +
> int amdgpu_mes_init(struct amdgpu_device *adev)
> {
> int i, r, num_pipes, num_queues = 0;
> @@ -123,6 +125,9 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
> spin_lock_init(&adev->mes.ring_lock[i]);
>
> adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;
> + atomic_set(&adev->mes.userq_hw_queue_count, 0);
> + INIT_DELAYED_WORK(&adev->mes.userq_notify_unmap_work,
> + amdgpu_mes_userq_notify_unmap_work_handler);
> total_vmid_mask = (u32)((1UL << 16) - 1);
> reserved_vmid_mask = (u32)((1UL << adev->vm_manager.first_kfd_vmid) - 1);
>
> @@ -288,6 +293,8 @@ void amdgpu_mes_fini(struct amdgpu_device *adev)
> int i;
> int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
>
> + cancel_delayed_work_sync(&adev->mes.userq_notify_unmap_work);
> +
> kfree(adev->gfx.mec.mes_hung_db_array);
>
> amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj,
> @@ -1163,6 +1170,69 @@ int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev)
> return r;
> }
>
> +/* Interval for notifying MES of work on unmapped queues during oversubscription */
> +#define AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_MS 50
> +
> +static unsigned int amdgpu_mes_userq_hw_queue_num(struct amdgpu_device *adev)
> +{
> + int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
> + unsigned int n = bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
> + int i;
> +
> + for (i = 0; i < num_xcc; i++)
> + n += bitmap_weight(adev->gfx.mec_bitmap[i].queue_bitmap,
> + AMDGPU_MAX_COMPUTE_QUEUES);
> +
> + return n;
> +}
> +
> +static void amdgpu_mes_userq_notify_unmap_work_handler(struct work_struct *work)
> +{
> + struct amdgpu_mes *mes = container_of(work, struct amdgpu_mes,
> + userq_notify_unmap_work.work);
> + struct amdgpu_device *adev = mes->adev;
> +
> + amdgpu_mes_notify_unmap_queue(adev);
> +
> + /* Re-arm if still oversubscribed */
> + if (atomic_read(&mes->userq_hw_queue_count) >
> + amdgpu_mes_userq_hw_queue_num(adev))
> + queue_delayed_work(system_wq, &mes->userq_notify_unmap_work,
> + msecs_to_jiffies(AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_MS));
> +}
> +
> +/*
> + * Called after a GFX11 usermode queue is successfully mapped to MES.
> + * Starts the periodic unmap-notify timer if this pushed the device into
> + * HW queue oversubscription.
> + */
> +void amdgpu_mes_userq_queue_mapped(struct amdgpu_device *adev)
> +{
> + if (!(amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) &&
> + amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)))
> + return;
> +
> + if (atomic_inc_return(&adev->mes.userq_hw_queue_count) >
> + amdgpu_mes_userq_hw_queue_num(adev))
> + queue_delayed_work(system_wq, &adev->mes.userq_notify_unmap_work,
> + msecs_to_jiffies(AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_MS));
> +}
> +
> +/*
> + * Called after a GFX11 usermode queue is unmapped from MES. Stops the
> + * periodic unmap-notify timer once oversubscription clears.
> + */
> +void amdgpu_mes_userq_queue_unmapped(struct amdgpu_device *adev)
> +{
> + if (!(amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) &&
> + amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)))
> + return;
> +
> + if (atomic_dec_return(&adev->mes.userq_hw_queue_count) <=
> + amdgpu_mes_userq_hw_queue_num(adev))
> + cancel_delayed_work(&adev->mes.userq_notify_unmap_work);
> +}
> +
> #if defined(CONFIG_DEBUG_FS)
>
> static int amdgpu_debugfs_mes_event_log_show(struct seq_file *m, void *unused)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> index 0291ae14a1e7..5160b227943b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> @@ -29,6 +29,7 @@
> #include "amdgpu_gfx.h"
> #include "amdgpu_doorbell.h"
> #include <linux/sched/mm.h>
> +#include <linux/workqueue.h>
>
> #define AMDGPU_MES_MAX_COMPUTE_PIPES 8
> #define AMDGPU_MES_MAX_GFX_PIPES 2
> @@ -90,6 +91,10 @@ struct amdgpu_mes {
> uint32_t total_max_queue;
> uint32_t max_doorbell_slices;
>
> + /* GFX11 usermode queue oversubscription notify timer */
> + atomic_t userq_hw_queue_count;
> + struct delayed_work userq_notify_unmap_work;
> +
> uint64_t default_process_quantum;
> uint64_t default_gang_quantum;
>
> @@ -646,5 +651,7 @@ void amdgpu_mes_free_gang_ctx_index(struct amdgpu_mes *mes,
> uint32_t index);
>
> int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev);
> +void amdgpu_mes_userq_queue_mapped(struct amdgpu_device *adev);
> +void amdgpu_mes_userq_queue_unmapped(struct amdgpu_device *adev);
>
> #endif /* __AMDGPU_MES_H__ */
> diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> index fae709f134bb..5a10af16e0c1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> +++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> @@ -171,6 +171,8 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
> return r;
> }
>
> + amdgpu_mes_userq_queue_mapped(adev);
> +
> DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index);
> return 0;
> }
> @@ -195,9 +197,13 @@ static int mes_userq_unmap(struct amdgpu_usermode_queue *queue)
> amdgpu_mes_unlock(&adev->mes);
> if (mes->use_rs64mem)
> amdgpu_mes_free_gang_ctx_index(mes, queue->gang_ctx_array_index);
> - if (r)
> + if (r) {
> DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r);
> - return r;
> + return r;
> + }
> +
> + amdgpu_mes_userq_queue_unmapped(adev);
> + return 0;
> }
>
> int mes_userq_reset(struct amdgpu_usermode_queue *queue)
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [PATCH 1/3] drm/amdgpu: add new mes misc api for gfx11
2026-08-19 20:14 [PATCH 1/3] drm/amdgpu: add new mes misc api for gfx11 Eric Huang
2026-08-19 20:14 ` [PATCH 2/3] drm/amdkfd: workaround 100% gpu usage issue " Eric Huang
2026-08-19 20:14 ` [PATCH 3/3] drm/amdgpu/userq: create the same workaround of oversubscription timer Eric Huang
@ 2026-08-21 21:30 ` Mario Limonciello
2 siblings, 0 replies; 6+ messages in thread
From: Mario Limonciello @ 2026-08-21 21:30 UTC (permalink / raw)
To: Eric Huang, amd-gfx; +Cc: Tishko.Araz, Alexander.Deucher
On 8/19/26 15:14, Eric Huang wrote:
> new api will be used to workaround a HW scheduler issue
> for 100% usage when gpu has no workload.
>
> Signed-off-by: Eric Huang <jinhuieric.huang@amd.com>
Reviewed-by: Mario Limonciello (AMD) <superm1@kernel.org>> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 23 +++++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h | 4 ++++
> drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 10 +++++++++-
> 3 files changed, 36 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> index b96f94e5169f..421d937c1188 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> @@ -1140,6 +1140,29 @@ void amdgpu_mes_free_gang_ctx_index(struct amdgpu_mes *mes,
> amdgpu_mes_unlock(mes);
> }
>
> +int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev)
> +{
> + struct mes_misc_op_input op_input = {0};
> + int r;
> +
> + op_input.op = MES_MISC_NOTIFY_WORK_ON_UNMAPPED_QUEUE;
> +
> + if (!adev->mes.funcs->misc_op) {
> + dev_err(adev->dev, "mes notify unmap queue is not supported!\n");
> + r = -EINVAL;
> + goto error;
> + }
> +
> + amdgpu_mes_lock(&adev->mes);
> + r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
> + amdgpu_mes_unlock(&adev->mes);
> + if (r)
> + dev_err(adev->dev, "failed to notify unmap queue.\n");
> +
> +error:
> + return r;
> +}
> +
> #if defined(CONFIG_DEBUG_FS)
>
> static int amdgpu_debugfs_mes_event_log_show(struct seq_file *m, void *unused)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> index 977c057dcce8..0291ae14a1e7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> @@ -366,6 +366,7 @@ enum mes_misc_opcode {
> MES_MISC_OP_WRM_REG_WR_WAIT,
> MES_MISC_OP_SET_SHADER_DEBUGGER,
> MES_MISC_OP_CHANGE_CONFIG,
> + MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE
> };
>
> struct mes_misc_op_input {
> @@ -643,4 +644,7 @@ int amdgpu_mes_alloc_gang_ctx_index(struct amdgpu_mes *mes,
> uint32_t *index);
> void amdgpu_mes_free_gang_ctx_index(struct amdgpu_mes *mes,
> uint32_t index);
> +
> +int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev);
> +
> #endif /* __AMDGPU_MES_H__ */
> diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
> index c57bcb9a98b1..df71e9447b35 100644
> --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
> @@ -957,7 +957,15 @@ static int mes_v11_0_misc_op(struct amdgpu_mes *mes,
> misc_pkt.change_config.option.bits.limit_single_process =
> input->change_config.option.limit_single_process;
> break;
> -
> + case MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE:
> + if ((mes->adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) < 0x70) {
> + dev_warn_once(mes->adev->dev,
> + "MES FW version must be larger than 0x70 to support notify work on unmapped queue.\n");
> + return 0;
> + }
> + misc_pkt.opcode = MESAPI_MISC__NOTIFY_WORK_ON_UNMAPPED_QUEUE;
> + misc_pkt.queue_sch_level = AMD_PRIORITY_LEVEL_NORMAL;
> + break;
> default:
> drm_err(adev_to_drm(mes->adev), "unsupported misc op (%d)\n", input->op);
> return -EINVAL;
^ permalink raw reply [flat|nested] 6+ messages in thread