[PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq

[PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq

[Alex Deucher]

Add proper checks for disable_kq functionality in
gfx helper functions.  Add special logic for families
that require the clear state setup.

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 92 +++++++++++++++++--------
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h |  2 +
 2 files changed, 67 insertions(+), 27 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index a194bf3347cbc..af3f8b62f6fd5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -371,6 +371,18 @@ int amdgpu_gfx_kiq_init(struct amdgpu_device *adev,
 	return 0;
 }
 
+static bool amdgpu_gfx_disable_gfx_kq(struct amdgpu_device *adev)
+{
+	if (adev->gfx.disable_kq) {
+		/* GFX11 needs the GFX ring for clear buffer */
+		if (amdgpu_ip_version(adev, GC_HWIP, 0) <= IP_VERSION(12, 0, 0))
+			return false;
+		else
+			return true;
+	}
+	return false;
+}
+
 /* create MQD for each compute/gfx queue */
 int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
 			   unsigned int mqd_size, int xcc_id)
@@ -379,6 +391,7 @@ int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 	struct amdgpu_ring *ring = &kiq->ring;
 	u32 domain = AMDGPU_GEM_DOMAIN_GTT;
+	bool disable_kq_gfx = amdgpu_gfx_disable_gfx_kq(adev);
 
 #if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
 	/* Only enable on gfx10 and 11 for now to avoid changing behavior on older chips */
@@ -413,7 +426,8 @@ int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
 		}
 	}
 
-	if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
+	if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring &&
+	    !disable_kq_gfx) {
 		/* create MQD for each KGQ */
 		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 			ring = &adev->gfx.gfx_ring[i];
@@ -437,25 +451,28 @@ int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
 		}
 	}
 
-	/* create MQD for each KCQ */
-	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
-		j = i + xcc_id * adev->gfx.num_compute_rings;
-		ring = &adev->gfx.compute_ring[j];
-		if (!ring->mqd_obj) {
-			r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
-						    domain, &ring->mqd_obj,
-						    &ring->mqd_gpu_addr, &ring->mqd_ptr);
-			if (r) {
-				dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
-				return r;
-			}
+	if (!adev->gfx.disable_kq) {
+		/* create MQD for each KCQ */
+		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+			j = i + xcc_id * adev->gfx.num_compute_rings;
+			ring = &adev->gfx.compute_ring[j];
+			if (!ring->mqd_obj) {
+				r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
+							    domain, &ring->mqd_obj,
+							    &ring->mqd_gpu_addr, &ring->mqd_ptr);
+				if (r) {
+					dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
+					return r;
+				}
 
-			ring->mqd_size = mqd_size;
-			/* prepare MQD backup */
-			adev->gfx.mec.mqd_backup[j] = kzalloc(mqd_size, GFP_KERNEL);
-			if (!adev->gfx.mec.mqd_backup[j]) {
-				dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
-				return -ENOMEM;
+				ring->mqd_size = mqd_size;
+				/* prepare MQD backup */
+				adev->gfx.mec.mqd_backup[j] = kzalloc(mqd_size, GFP_KERNEL);
+				if (!adev->gfx.mec.mqd_backup[j]) {
+					dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n",
+						 ring->name);
+					return -ENOMEM;
+				}
 			}
 		}
 	}
@@ -468,8 +485,10 @@ void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev, int xcc_id)
 	struct amdgpu_ring *ring = NULL;
 	int i, j;
 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
+	bool disable_kq_gfx = amdgpu_gfx_disable_gfx_kq(adev);
 
-	if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
+	if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring &&
+	    !disable_kq_gfx) {
 		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 			ring = &adev->gfx.gfx_ring[i];
 			kfree(adev->gfx.me.mqd_backup[i]);
@@ -479,13 +498,15 @@ void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev, int xcc_id)
 		}
 	}
 
-	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
-		j = i + xcc_id * adev->gfx.num_compute_rings;
-		ring = &adev->gfx.compute_ring[j];
-		kfree(adev->gfx.mec.mqd_backup[j]);
-		amdgpu_bo_free_kernel(&ring->mqd_obj,
-				      &ring->mqd_gpu_addr,
-				      &ring->mqd_ptr);
+	if (!adev->gfx.disable_kq) {
+		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+			j = i + xcc_id * adev->gfx.num_compute_rings;
+			ring = &adev->gfx.compute_ring[j];
+			kfree(adev->gfx.mec.mqd_backup[j]);
+			amdgpu_bo_free_kernel(&ring->mqd_obj,
+					      &ring->mqd_gpu_addr,
+					      &ring->mqd_ptr);
+		}
 	}
 
 	ring = &kiq->ring;
@@ -502,6 +523,9 @@ int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev, int xcc_id)
 	int i, r = 0;
 	int j;
 
+	if (adev->gfx.disable_kq)
+		return 0;
+
 	if (adev->enable_mes) {
 		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 			j = i + xcc_id * adev->gfx.num_compute_rings;
@@ -547,11 +571,15 @@ int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev, int xcc_id)
 
 int amdgpu_gfx_disable_kgq(struct amdgpu_device *adev, int xcc_id)
 {
+	bool disable_kq_gfx = amdgpu_gfx_disable_gfx_kq(adev);
 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 	struct amdgpu_ring *kiq_ring = &kiq->ring;
 	int i, r = 0;
 	int j;
 
+	if (disable_kq_gfx)
+		return 0;
+
 	if (adev->enable_mes) {
 		if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) {
 			for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
@@ -657,6 +685,9 @@ int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev, int xcc_id)
 	uint64_t queue_mask = 0;
 	int r, i, j;
 
+	if (adev->gfx.disable_kq)
+		return 0;
+
 	if (adev->mes.enable_legacy_queue_map)
 		return amdgpu_gfx_mes_enable_kcq(adev, xcc_id);
 
@@ -716,10 +747,14 @@ int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev, int xcc_id)
 
 int amdgpu_gfx_enable_kgq(struct amdgpu_device *adev, int xcc_id)
 {
+	bool disable_kq_gfx = amdgpu_gfx_disable_gfx_kq(adev);
 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 	struct amdgpu_ring *kiq_ring = &kiq->ring;
 	int r, i, j;
 
+	if (disable_kq_gfx)
+		return 0;
+
 	if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
 		return -EINVAL;
 
@@ -1544,6 +1579,9 @@ static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device *dev,
 	if (adev->in_suspend && !adev->in_runpm)
 		return -EPERM;
 
+	if (adev->gfx.disable_kq)
+		return -ENOTSUPP;
+
 	ret = kstrtol(buf, 0, &value);
 
 	if (ret)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index ddf4533614bac..8fa68a4ac34f1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -483,6 +483,8 @@ struct amdgpu_gfx {
 
 	atomic_t			total_submission_cnt;
 	struct delayed_work		idle_work;
+
+	bool				disable_kq;
 };
 
 struct amdgpu_gfx_ras_reg_entry {
-- 
2.48.1

Re: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq

[Felix Kuehling]

On 2025-03-05 15:47, Alex Deucher wrote:
> Add proper checks for disable_kq functionality in
> gfx helper functions.  Add special logic for families
> that require the clear state setup.
>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 92 +++++++++++++++++--------
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h |  2 +
>   2 files changed, 67 insertions(+), 27 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> index a194bf3347cbc..af3f8b62f6fd5 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> @@ -371,6 +371,18 @@ int amdgpu_gfx_kiq_init(struct amdgpu_device *adev,
>   	return 0;
>   }
>   
> +static bool amdgpu_gfx_disable_gfx_kq(struct amdgpu_device *adev)
> +{
> +	if (adev->gfx.disable_kq) {
> +		/* GFX11 needs the GFX ring for clear buffer */
> +		if (amdgpu_ip_version(adev, GC_HWIP, 0) <= IP_VERSION(12, 0, 0))

Should this be < instead of <=?

Regards,
   Felix

> +			return false;
> +		else
> +			return true;
> +	}
> +	return false;
> +}
> +
>   /* create MQD for each compute/gfx queue */
>   int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
>   			   unsigned int mqd_size, int xcc_id)
> @@ -379,6 +391,7 @@ int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
>   	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
>   	struct amdgpu_ring *ring = &kiq->ring;
>   	u32 domain = AMDGPU_GEM_DOMAIN_GTT;
> +	bool disable_kq_gfx = amdgpu_gfx_disable_gfx_kq(adev);
>   
>   #if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
>   	/* Only enable on gfx10 and 11 for now to avoid changing behavior on older chips */
> @@ -413,7 +426,8 @@ int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
>   		}
>   	}
>   
> -	if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
> +	if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring &&
> +	    !disable_kq_gfx) {
>   		/* create MQD for each KGQ */
>   		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
>   			ring = &adev->gfx.gfx_ring[i];
> @@ -437,25 +451,28 @@ int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
>   		}
>   	}
>   
> -	/* create MQD for each KCQ */
> -	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
> -		j = i + xcc_id * adev->gfx.num_compute_rings;
> -		ring = &adev->gfx.compute_ring[j];
> -		if (!ring->mqd_obj) {
> -			r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
> -						    domain, &ring->mqd_obj,
> -						    &ring->mqd_gpu_addr, &ring->mqd_ptr);
> -			if (r) {
> -				dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
> -				return r;
> -			}
> +	if (!adev->gfx.disable_kq) {

Maybe just set adev->gfx.num_compute_rings to 0 somewhere, then you 
don't need this condition.


> +		/* create MQD for each KCQ */
> +		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
> +			j = i + xcc_id * adev->gfx.num_compute_rings;
> +			ring = &adev->gfx.compute_ring[j];
> +			if (!ring->mqd_obj) {
> +				r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
> +							    domain, &ring->mqd_obj,
> +							    &ring->mqd_gpu_addr, &ring->mqd_ptr);
> +				if (r) {
> +					dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
> +					return r;
> +				}
>   
> -			ring->mqd_size = mqd_size;
> -			/* prepare MQD backup */
> -			adev->gfx.mec.mqd_backup[j] = kzalloc(mqd_size, GFP_KERNEL);
> -			if (!adev->gfx.mec.mqd_backup[j]) {
> -				dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
> -				return -ENOMEM;
> +				ring->mqd_size = mqd_size;
> +				/* prepare MQD backup */
> +				adev->gfx.mec.mqd_backup[j] = kzalloc(mqd_size, GFP_KERNEL);
> +				if (!adev->gfx.mec.mqd_backup[j]) {
> +					dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n",
> +						 ring->name);
> +					return -ENOMEM;
> +				}
>   			}
>   		}
>   	}
> @@ -468,8 +485,10 @@ void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev, int xcc_id)
>   	struct amdgpu_ring *ring = NULL;
>   	int i, j;
>   	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
> +	bool disable_kq_gfx = amdgpu_gfx_disable_gfx_kq(adev);
>   
> -	if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
> +	if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring &&
> +	    !disable_kq_gfx) {
>   		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
>   			ring = &adev->gfx.gfx_ring[i];
>   			kfree(adev->gfx.me.mqd_backup[i]);
> @@ -479,13 +498,15 @@ void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev, int xcc_id)
>   		}
>   	}
>   
> -	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
> -		j = i + xcc_id * adev->gfx.num_compute_rings;
> -		ring = &adev->gfx.compute_ring[j];
> -		kfree(adev->gfx.mec.mqd_backup[j]);
> -		amdgpu_bo_free_kernel(&ring->mqd_obj,
> -				      &ring->mqd_gpu_addr,
> -				      &ring->mqd_ptr);
> +	if (!adev->gfx.disable_kq) {

Same as above.


> +		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
> +			j = i + xcc_id * adev->gfx.num_compute_rings;
> +			ring = &adev->gfx.compute_ring[j];
> +			kfree(adev->gfx.mec.mqd_backup[j]);
> +			amdgpu_bo_free_kernel(&ring->mqd_obj,
> +					      &ring->mqd_gpu_addr,
> +					      &ring->mqd_ptr);
> +		}
>   	}
>   
>   	ring = &kiq->ring;
> @@ -502,6 +523,9 @@ int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev, int xcc_id)
>   	int i, r = 0;
>   	int j;
>   
> +	if (adev->gfx.disable_kq)

Same as above.


> +		return 0;
> +
>   	if (adev->enable_mes) {
>   		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
>   			j = i + xcc_id * adev->gfx.num_compute_rings;
> @@ -547,11 +571,15 @@ int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev, int xcc_id)
>   
>   int amdgpu_gfx_disable_kgq(struct amdgpu_device *adev, int xcc_id)
>   {
> +	bool disable_kq_gfx = amdgpu_gfx_disable_gfx_kq(adev);
>   	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
>   	struct amdgpu_ring *kiq_ring = &kiq->ring;
>   	int i, r = 0;
>   	int j;
>   
> +	if (disable_kq_gfx)
> +		return 0;
Maybe just set adev->gfx.num_gfx_rings to 0 somewhere, then you don't 
need this condition.

Regards,
   Felix


> +
>   	if (adev->enable_mes) {
>   		if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) {
>   			for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
> @@ -657,6 +685,9 @@ int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev, int xcc_id)
>   	uint64_t queue_mask = 0;
>   	int r, i, j;
>   
> +	if (adev->gfx.disable_kq)
> +		return 0;
> +
>   	if (adev->mes.enable_legacy_queue_map)
>   		return amdgpu_gfx_mes_enable_kcq(adev, xcc_id);
>   
> @@ -716,10 +747,14 @@ int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev, int xcc_id)
>   
>   int amdgpu_gfx_enable_kgq(struct amdgpu_device *adev, int xcc_id)
>   {
> +	bool disable_kq_gfx = amdgpu_gfx_disable_gfx_kq(adev);
>   	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
>   	struct amdgpu_ring *kiq_ring = &kiq->ring;
>   	int r, i, j;
>   
> +	if (disable_kq_gfx)
> +		return 0;
> +
>   	if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
>   		return -EINVAL;
>   
> @@ -1544,6 +1579,9 @@ static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device *dev,
>   	if (adev->in_suspend && !adev->in_runpm)
>   		return -EPERM;
>   
> +	if (adev->gfx.disable_kq)
> +		return -ENOTSUPP;
> +
>   	ret = kstrtol(buf, 0, &value);
>   
>   	if (ret)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> index ddf4533614bac..8fa68a4ac34f1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> @@ -483,6 +483,8 @@ struct amdgpu_gfx {
>   
>   	atomic_t			total_submission_cnt;
>   	struct delayed_work		idle_work;
> +
> +	bool				disable_kq;
>   };
>   
>   struct amdgpu_gfx_ras_reg_entry {

Re: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq

[Khatri, Sunil]

On 3/6/2025 6:36 AM, Felix Kuehling wrote:
>
> On 2025-03-05 15:47, Alex Deucher wrote:
>> Add proper checks for disable_kq functionality in
>> gfx helper functions.  Add special logic for families
>> that require the clear state setup.
>>
>> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
>> ---
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 92 +++++++++++++++++--------
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h |  2 +
>>   2 files changed, 67 insertions(+), 27 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c 
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
>> index a194bf3347cbc..af3f8b62f6fd5 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
>> @@ -371,6 +371,18 @@ int amdgpu_gfx_kiq_init(struct amdgpu_device *adev,
>>       return 0;
>>   }
>>   +static bool amdgpu_gfx_disable_gfx_kq(struct amdgpu_device *adev)
>> +{
>> +    if (adev->gfx.disable_kq) {
>> +        /* GFX11 needs the GFX ring for clear buffer */
>> +        if (amdgpu_ip_version(adev, GC_HWIP, 0) <= IP_VERSION(12, 0, 
>> 0))

Yes the check has to be  < as gfx12 do not need the clear buffer based 
on our discussions.

Regards
Sunil

>
> Should this be < instead of <=?

>
> Regards,
>   Felix
>
>> +            return false;
>> +        else
>> +            return true;
>> +    }
>> +    return false;
>> +}
>> +
>>   /* create MQD for each compute/gfx queue */
>>   int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
>>                  unsigned int mqd_size, int xcc_id)
>> @@ -379,6 +391,7 @@ int amdgpu_gfx_mqd_sw_init(struct amdgpu_device 
>> *adev,
>>       struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
>>       struct amdgpu_ring *ring = &kiq->ring;
>>       u32 domain = AMDGPU_GEM_DOMAIN_GTT;
>> +    bool disable_kq_gfx = amdgpu_gfx_disable_gfx_kq(adev);

name of variable and function could be in sync. disable_gfx_kq and 
amdgpu_gfx_disable_gfx_kq or change function name according to variable.

Also another suggestion here is better to have one more variable in the 
gfx struct or ring and read this amdgpu_gfx_disable_gfx_kq once and use 
it in all the places. It does looks confusing
so many similar sounding names.

Regards
Sunil
>>     #if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
>>       /* Only enable on gfx10 and 11 for now to avoid changing 
>> behavior on older chips */
>> @@ -413,7 +426,8 @@ int amdgpu_gfx_mqd_sw_init(struct amdgpu_device 
>> *adev,
>>           }
>>       }
>>   -    if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
>> +    if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring &&
>> +        !disable_kq_gfx) {
>>           /* create MQD for each KGQ */
>>           for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
>>               ring = &adev->gfx.gfx_ring[i];
>> @@ -437,25 +451,28 @@ int amdgpu_gfx_mqd_sw_init(struct amdgpu_device 
>> *adev,
>>           }
>>       }
>>   -    /* create MQD for each KCQ */
>> -    for (i = 0; i < adev->gfx.num_compute_rings; i++) {
>> -        j = i + xcc_id * adev->gfx.num_compute_rings;
>> -        ring = &adev->gfx.compute_ring[j];
>> -        if (!ring->mqd_obj) {
>> -            r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
>> -                            domain, &ring->mqd_obj,
>> -                            &ring->mqd_gpu_addr, &ring->mqd_ptr);
>> -            if (r) {
>> -                dev_warn(adev->dev, "failed to create ring mqd bo 
>> (%d)", r);
>> -                return r;
>> -            }
>> +    if (!adev->gfx.disable_kq) {
>
> Maybe just set adev->gfx.num_compute_rings to 0 somewhere, then you 
> don't need this condition.
>
>
>> +        /* create MQD for each KCQ */
>> +        for (i = 0; i < adev->gfx.num_compute_rings; i++) {
>> +            j = i + xcc_id * adev->gfx.num_compute_rings;
>> +            ring = &adev->gfx.compute_ring[j];
>> +            if (!ring->mqd_obj) {
>> +                r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
>> +                                domain, &ring->mqd_obj,
>> +                                &ring->mqd_gpu_addr, &ring->mqd_ptr);
>> +                if (r) {
>> +                    dev_warn(adev->dev, "failed to create ring mqd 
>> bo (%d)", r);
>> +                    return r;
>> +                }
>>   -            ring->mqd_size = mqd_size;
>> -            /* prepare MQD backup */
>> -            adev->gfx.mec.mqd_backup[j] = kzalloc(mqd_size, 
>> GFP_KERNEL);
>> -            if (!adev->gfx.mec.mqd_backup[j]) {
>> -                dev_warn(adev->dev, "no memory to create MQD backup 
>> for ring %s\n", ring->name);
>> -                return -ENOMEM;
>> +                ring->mqd_size = mqd_size;
>> +                /* prepare MQD backup */
>> +                adev->gfx.mec.mqd_backup[j] = kzalloc(mqd_size, 
>> GFP_KERNEL);
>> +                if (!adev->gfx.mec.mqd_backup[j]) {
>> +                    dev_warn(adev->dev, "no memory to create MQD 
>> backup for ring %s\n",
>> +                         ring->name);
>> +                    return -ENOMEM;
>> +                }
>>               }
>>           }
>>       }
>> @@ -468,8 +485,10 @@ void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device 
>> *adev, int xcc_id)
>>       struct amdgpu_ring *ring = NULL;
>>       int i, j;
>>       struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
>> +    bool disable_kq_gfx = amdgpu_gfx_disable_gfx_kq(adev);
>>   -    if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
>> +    if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring &&
>> +        !disable_kq_gfx) {
>>           for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
>>               ring = &adev->gfx.gfx_ring[i];
>>               kfree(adev->gfx.me.mqd_backup[i]);
>> @@ -479,13 +498,15 @@ void amdgpu_gfx_mqd_sw_fini(struct 
>> amdgpu_device *adev, int xcc_id)
>>           }
>>       }
>>   -    for (i = 0; i < adev->gfx.num_compute_rings; i++) {
>> -        j = i + xcc_id * adev->gfx.num_compute_rings;
>> -        ring = &adev->gfx.compute_ring[j];
>> -        kfree(adev->gfx.mec.mqd_backup[j]);
>> -        amdgpu_bo_free_kernel(&ring->mqd_obj,
>> -                      &ring->mqd_gpu_addr,
>> -                      &ring->mqd_ptr);
>> +    if (!adev->gfx.disable_kq) {
>
> Same as above.
>
>
>> +        for (i = 0; i < adev->gfx.num_compute_rings; i++) {
>> +            j = i + xcc_id * adev->gfx.num_compute_rings;
>> +            ring = &adev->gfx.compute_ring[j];
>> +            kfree(adev->gfx.mec.mqd_backup[j]);
>> +            amdgpu_bo_free_kernel(&ring->mqd_obj,
>> +                          &ring->mqd_gpu_addr,
>> +                          &ring->mqd_ptr);
>> +        }
>>       }
>>         ring = &kiq->ring;
>> @@ -502,6 +523,9 @@ int amdgpu_gfx_disable_kcq(struct amdgpu_device 
>> *adev, int xcc_id)
>>       int i, r = 0;
>>       int j;
>>   +    if (adev->gfx.disable_kq)
>
> Same as above.
>
>
>> +        return 0;
>> +
>>       if (adev->enable_mes) {
>>           for (i = 0; i < adev->gfx.num_compute_rings; i++) {
>>               j = i + xcc_id * adev->gfx.num_compute_rings;
>> @@ -547,11 +571,15 @@ int amdgpu_gfx_disable_kcq(struct amdgpu_device 
>> *adev, int xcc_id)
>>     int amdgpu_gfx_disable_kgq(struct amdgpu_device *adev, int xcc_id)
>>   {
>> +    bool disable_kq_gfx = amdgpu_gfx_disable_gfx_kq(adev);
>>       struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
>>       struct amdgpu_ring *kiq_ring = &kiq->ring;
>>       int i, r = 0;
>>       int j;
>>   +    if (disable_kq_gfx)
>> +        return 0;
> Maybe just set adev->gfx.num_gfx_rings to 0 somewhere, then you don't 
> need this condition.
>
> Regards,
>   Felix
>
>
>> +
>>       if (adev->enable_mes) {
>>           if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) {
>>               for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
>> @@ -657,6 +685,9 @@ int amdgpu_gfx_enable_kcq(struct amdgpu_device 
>> *adev, int xcc_id)
>>       uint64_t queue_mask = 0;
>>       int r, i, j;
>>   +    if (adev->gfx.disable_kq)
>> +        return 0;
>> +
>>       if (adev->mes.enable_legacy_queue_map)
>>           return amdgpu_gfx_mes_enable_kcq(adev, xcc_id);
>>   @@ -716,10 +747,14 @@ int amdgpu_gfx_enable_kcq(struct 
>> amdgpu_device *adev, int xcc_id)
>>     int amdgpu_gfx_enable_kgq(struct amdgpu_device *adev, int xcc_id)
>>   {
>> +    bool disable_kq_gfx = amdgpu_gfx_disable_gfx_kq(adev);
>>       struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
>>       struct amdgpu_ring *kiq_ring = &kiq->ring;
>>       int r, i, j;
>>   +    if (disable_kq_gfx)
>> +        return 0;
>> +
>>       if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
>>           return -EINVAL;
>>   @@ -1544,6 +1579,9 @@ static ssize_t 
>> amdgpu_gfx_set_run_cleaner_shader(struct device *dev,
>>       if (adev->in_suspend && !adev->in_runpm)
>>           return -EPERM;
>>   +    if (adev->gfx.disable_kq)
>> +        return -ENOTSUPP;
>> +
>>       ret = kstrtol(buf, 0, &value);
>>         if (ret)
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h 
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
>> index ddf4533614bac..8fa68a4ac34f1 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
>> @@ -483,6 +483,8 @@ struct amdgpu_gfx {
>>         atomic_t            total_submission_cnt;
>>       struct delayed_work        idle_work;
>> +
>> +    bool                disable_kq;
>>   };
>>     struct amdgpu_gfx_ras_reg_entry {

[PATCH V2 00/11] Add disable kernel queue support

[Alex Deucher]

To better evaluate user queues, add a module parameter
to disable kernel queues.  With this set kernel queues
are disabled and only user queues are available.  This
frees up hardware resources for use in user queues which
would otherwise be used by kernel queues and provides
a way to validate user queues without the presence
of kernel queues.

v2: use num_gfx_rings and num_compute_rings per
    Felix suggestion

Alex Deucher (11):
  drm/amdgpu: add parameter to disable kernel queues
  drm/amdgpu: add ring flag for no user submissions
  drm/amdgpu/gfx: add generic handling for disable_kq
  drm/amdgpu/mes: centralize gfx_hqd mask management
  drm/amdgpu/mes: update hqd masks when disable_kq is set
  drm/amdgpu/mes: make more vmids available when disable_kq=1
  drm/amdgpu/gfx11: add support for disable_kq
  drm/amdgpu/gfx12: add support for disable_kq
  drm/amdgpu/sdma: add flag for tracking disable_kq
  drm/amdgpu/sdma6: add support for disable_kq
  drm/amdgpu/sdma7: add support for disable_kq

 drivers/gpu/drm/amd/amdgpu/amdgpu.h      |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c  |  9 +++
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c  |  3 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h  |  2 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++---
 drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c  | 26 ++++++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h |  1 +
 drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c   | 99 ++++++++++++++++--------
 drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c   | 96 ++++++++++++++---------
 drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c   |  2 +-
 drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c   |  2 +-
 drivers/gpu/drm/amd/amdgpu/mes_v11_0.c   | 16 +---
 drivers/gpu/drm/amd/amdgpu/mes_v12_0.c   | 15 +---
 drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c   |  4 +
 drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c   |  4 +
 17 files changed, 204 insertions(+), 112 deletions(-)

-- 
2.48.1

[PATCH 01/11] drm/amdgpu: add parameter to disable kernel queues

[Alex Deucher]

On chips that support user queues, setting this option
will disable kernel queues to be used to validate
user queues without kernel queues.

Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h     | 1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 9 +++++++++
 2 files changed, 10 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 87062c1adcdf7..45437a8f29d3b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -270,6 +270,7 @@ extern int amdgpu_user_partt_mode;
 extern int amdgpu_agp;
 
 extern int amdgpu_wbrf;
+extern int amdgpu_disable_kq;
 
 #define AMDGPU_VM_MAX_NUM_CTX			4096
 #define AMDGPU_SG_THRESHOLD			(256*1024*1024)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index b161daa900198..42a7619592ab9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -237,6 +237,7 @@ int amdgpu_agp = -1; /* auto */
 int amdgpu_wbrf = -1;
 int amdgpu_damage_clips = -1; /* auto */
 int amdgpu_umsch_mm_fwlog;
+int amdgpu_disable_kq = -1;
 
 DECLARE_DYNDBG_CLASSMAP(drm_debug_classes, DD_CLASS_TYPE_DISJOINT_BITS, 0,
 			"DRM_UT_CORE",
@@ -1083,6 +1084,14 @@ MODULE_PARM_DESC(wbrf,
 	"Enable Wifi RFI interference mitigation (0 = disabled, 1 = enabled, -1 = auto(default)");
 module_param_named(wbrf, amdgpu_wbrf, int, 0444);
 
+/**
+ * DOC: disable_kq (int)
+ * Disable kernel queues on systems that support user queues.
+ * (0 = kernel queues enabled, 1 = kernel queues disabled, -1 = auto (default setting))
+ */
+MODULE_PARM_DESC(disable_kq, "Disable kernel queues (-1 = auto (default), 0 = enable KQ, 1 = disable KQ)");
+module_param_named(disable_kq, amdgpu_disable_kq, int, 0444);
+
 /* These devices are not supported by amdgpu.
  * They are supported by the mach64, r128, radeon drivers
  */
-- 
2.48.1

[PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Alex Deucher]

This would be set by IPs which only accept submissions
from the kernel, not userspace, such as when kernel
queues are disabled. Don't expose the rings to userspace
and reject any submissions in the CS IOCTL.

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
 3 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 5df21529b3b13..5cc18034b75df 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,
 	ring = amdgpu_job_ring(job);
 	ib = &job->ibs[job->num_ibs++];
 
+	/* submissions to kernel queus are disabled */
+	if (ring->no_user_submission)
+		return -EINVAL;
+
 	/* MM engine doesn't support user fences */
 	if (p->uf_bo && ring->funcs->no_user_fence)
 		return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_GFX:
 		type = AMD_IP_BLOCK_TYPE_GFX;
 		for (i = 0; i < adev->gfx.num_gfx_rings; i++)
-			if (adev->gfx.gfx_ring[i].sched.ready)
+			if (adev->gfx.gfx_ring[i].sched.ready &&
+			    !adev->gfx.gfx_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 32;
 		ib_size_alignment = 32;
@@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_COMPUTE:
 		type = AMD_IP_BLOCK_TYPE_GFX;
 		for (i = 0; i < adev->gfx.num_compute_rings; i++)
-			if (adev->gfx.compute_ring[i].sched.ready)
+			if (adev->gfx.compute_ring[i].sched.ready &&
+			    !adev->gfx.compute_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 32;
 		ib_size_alignment = 32;
@@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_DMA:
 		type = AMD_IP_BLOCK_TYPE_SDMA;
 		for (i = 0; i < adev->sdma.num_instances; i++)
-			if (adev->sdma.instance[i].ring.sched.ready)
+			if (adev->sdma.instance[i].ring.sched.ready &&
+			    !adev->gfx.gfx_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
@@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 			if (adev->uvd.harvest_config & (1 << i))
 				continue;
 
-			if (adev->uvd.inst[i].ring.sched.ready)
+			if (adev->uvd.inst[i].ring.sched.ready &&
+			    !adev->uvd.inst[i].ring.no_user_submission)
 				++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_VCE:
 		type = AMD_IP_BLOCK_TYPE_VCE;
 		for (i = 0; i < adev->vce.num_rings; i++)
-			if (adev->vce.ring[i].sched.ready)
+			if (adev->vce.ring[i].sched.ready &&
+			    !adev->vce.ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
@@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->uvd.num_enc_rings; j++)
-				if (adev->uvd.inst[i].ring_enc[j].sched.ready)
+				if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
+				    !adev->uvd.inst[i].ring_enc[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 			if (adev->vcn.harvest_config & (1 << i))
 				continue;
 
-			if (adev->vcn.inst[i].ring_dec.sched.ready)
+			if (adev->vcn.inst[i].ring_dec.sched.ready &&
+			    !adev->vcn.inst[i].ring_dec.no_user_submission)
 				++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
-				if (adev->vcn.inst[i].ring_enc[j].sched.ready)
+				if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
+				    !adev->vcn.inst[i].ring_enc[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
-				if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
+				if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
+				    !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 		break;
 	case AMDGPU_HW_IP_VPE:
 		type = AMD_IP_BLOCK_TYPE_VPE;
-		if (adev->vpe.ring.sched.ready)
+		if (adev->vpe.ring.sched.ready &&
+		    !adev->vpe.ring.no_user_submission)
 			++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index b4fd1e17205e9..4a97afcb38b78 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -297,6 +297,7 @@ struct amdgpu_ring {
 	struct dma_fence	*vmid_wait;
 	bool			has_compute_vm_bug;
 	bool			no_scheduler;
+	bool			no_user_submission;
 	int			hw_prio;
 	unsigned 		num_hw_submission;
 	atomic_t		*sched_score;
@@ -310,7 +311,6 @@ struct amdgpu_ring {
 	unsigned int    entry_index;
 	/* store the cached rptr to restore after reset */
 	uint64_t cached_rptr;
-
 };
 
 #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
-- 
2.48.1

[PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq

[Alex Deucher]

Add proper checks for disable_kq functionality in
gfx helper functions.  Add special logic for families
that require the clear state setup.

v2: use ring count as per Felix suggestion

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 3 +++
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index a194bf3347cbc..4083b4d0595a2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -1544,6 +1544,9 @@ static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device *dev,
 	if (adev->in_suspend && !adev->in_runpm)
 		return -EPERM;
 
+	if (adev->gfx.disable_kq)
+		return -ENOTSUPP;
+
 	ret = kstrtol(buf, 0, &value);
 
 	if (ret)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index ddf4533614bac..8fa68a4ac34f1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -483,6 +483,8 @@ struct amdgpu_gfx {
 
 	atomic_t			total_submission_cnt;
 	struct delayed_work		idle_work;
+
+	bool				disable_kq;
 };
 
 struct amdgpu_gfx_ras_reg_entry {
-- 
2.48.1

[PATCH 04/11] drm/amdgpu/mes: centralize gfx_hqd mask management

[Alex Deucher]

Move it to amdgpu_mes to align with the compute and
sdma hqd masks. No functional change.

v2: rebase on new changes

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 22 ++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/mes_v11_0.c  | 16 +++-------------
 drivers/gpu/drm/amd/amdgpu/mes_v12_0.c  | 15 +++------------
 3 files changed, 28 insertions(+), 25 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index ca076306adba4..5913c5ba85ed0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -144,6 +144,28 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 	adev->mes.vmid_mask_mmhub = 0xffffff00;
 	adev->mes.vmid_mask_gfxhub = 0xffffff00;
 
+	for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++) {
+		/* use only 1st ME pipe */
+		if (i >= adev->gfx.me.num_pipe_per_me)
+			continue;
+		if (amdgpu_ip_version(adev, GC_HWIP, 0) >=
+		    IP_VERSION(12, 0, 0))
+			/*
+			 * GFX V12 has only one GFX pipe, but 8 queues in it.
+			 * GFX pipe 0 queue 0 is being used by Kernel queue.
+			 * Set GFX pipe 0 queue 1-7 for MES scheduling
+			 * mask = 1111 1110b
+			 */
+			adev->mes.gfx_hqd_mask[i] = 0xFE;
+		else
+			/*
+			 * GFX pipe 0 queue 0 is being used by Kernel queue.
+			 * Set GFX pipe 0 queue 1 for MES scheduling
+			 * mask = 10b
+			 */
+			adev->mes.gfx_hqd_mask[i] = 0x2;
+	}
+
 	for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) {
 		/* use only 1st MEC pipes */
 		if (i >= adev->gfx.mec.num_pipe_per_mec)
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
index a569d09a1a748..39b45d8b5f049 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
@@ -669,18 +669,6 @@ static int mes_v11_0_misc_op(struct amdgpu_mes *mes,
 			offsetof(union MESAPI__MISC, api_status));
 }
 
-static void mes_v11_0_set_gfx_hqd_mask(union MESAPI_SET_HW_RESOURCES *pkt)
-{
-	/*
-	 * GFX pipe 0 queue 0 is being used by Kernel queue.
-	 * Set GFX pipe 0 queue 1 for MES scheduling
-	 * mask = 10b
-	 * GFX pipe 1 can't be used for MES due to HW limitation.
-	 */
-	pkt->gfx_hqd_mask[0] = 0x2;
-	pkt->gfx_hqd_mask[1] = 0;
-}
-
 static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
 {
 	int i;
@@ -705,7 +693,9 @@ static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
 		mes_set_hw_res_pkt.compute_hqd_mask[i] =
 			mes->compute_hqd_mask[i];
 
-	mes_v11_0_set_gfx_hqd_mask(&mes_set_hw_res_pkt);
+	for (i = 0; i < MAX_GFX_PIPES; i++)
+		mes_set_hw_res_pkt.gfx_hqd_mask[i] =
+			mes->gfx_hqd_mask[i];
 
 	for (i = 0; i < MAX_SDMA_PIPES; i++)
 		mes_set_hw_res_pkt.sdma_hqd_mask[i] = mes->sdma_hqd_mask[i];
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
index 96336652d14c5..519f054bec60d 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
@@ -694,17 +694,6 @@ static int mes_v12_0_set_hw_resources_1(struct amdgpu_mes *mes, int pipe)
 			offsetof(union MESAPI_SET_HW_RESOURCES_1, api_status));
 }
 
-static void mes_v12_0_set_gfx_hqd_mask(union MESAPI_SET_HW_RESOURCES *pkt)
-{
-	/*
-	 * GFX V12 has only one GFX pipe, but 8 queues in it.
-	 * GFX pipe 0 queue 0 is being used by Kernel queue.
-	 * Set GFX pipe 0 queue 1-7 for MES scheduling
-	 * mask = 1111 1110b
-	 */
-	pkt->gfx_hqd_mask[0] = 0xFE;
-}
-
 static int mes_v12_0_set_hw_resources(struct amdgpu_mes *mes, int pipe)
 {
 	int i;
@@ -727,7 +716,9 @@ static int mes_v12_0_set_hw_resources(struct amdgpu_mes *mes, int pipe)
 			mes_set_hw_res_pkt.compute_hqd_mask[i] =
 				mes->compute_hqd_mask[i];
 
-		mes_v12_0_set_gfx_hqd_mask(&mes_set_hw_res_pkt);
+		for (i = 0; i < MAX_GFX_PIPES; i++)
+			mes_set_hw_res_pkt.gfx_hqd_mask[i] =
+				mes->gfx_hqd_mask[i];
 
 		for (i = 0; i < MAX_SDMA_PIPES; i++)
 			mes_set_hw_res_pkt.sdma_hqd_mask[i] =
-- 
2.48.1

[PATCH 05/11] drm/amdgpu/mes: update hqd masks when disable_kq is set

[Alex Deucher]

Make all resources available to user queues.

Suggested-by: Sunil Khatri <sunil.khatri@amd.com>
Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index 5913c5ba85ed0..e585e8690edf0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -156,21 +156,21 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 			 * Set GFX pipe 0 queue 1-7 for MES scheduling
 			 * mask = 1111 1110b
 			 */
-			adev->mes.gfx_hqd_mask[i] = 0xFE;
+			adev->mes.gfx_hqd_mask[i] = adev->gfx.disable_kq ? 0xFF : 0xFE;
 		else
 			/*
 			 * GFX pipe 0 queue 0 is being used by Kernel queue.
 			 * Set GFX pipe 0 queue 1 for MES scheduling
 			 * mask = 10b
 			 */
-			adev->mes.gfx_hqd_mask[i] = 0x2;
+			adev->mes.gfx_hqd_mask[i] = adev->gfx.disable_kq ? 0x3 : 0x2;
 	}
 
 	for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) {
 		/* use only 1st MEC pipes */
 		if (i >= adev->gfx.mec.num_pipe_per_mec)
 			continue;
-		adev->mes.compute_hqd_mask[i] = 0xc;
+		adev->mes.compute_hqd_mask[i] = adev->gfx.disable_kq ? 0xF : 0xC;
 	}
 
 	for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) {
-- 
2.48.1

[PATCH 06/11] drm/amdgpu/mes: make more vmids available when disable_kq=1

[Alex Deucher]

If we don't have kernel queues, the vmids can be used by
the MES for user queues.

Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 2 +-
 drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c  | 2 +-
 drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index e585e8690edf0..d7cdd2895889a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -142,7 +142,7 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 
 	adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;
 	adev->mes.vmid_mask_mmhub = 0xffffff00;
-	adev->mes.vmid_mask_gfxhub = 0xffffff00;
+	adev->mes.vmid_mask_gfxhub = adev->gfx.disable_kq ? 0xfffffffe : 0xffffff00;
 
 	for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++) {
 		/* use only 1st ME pipe */
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
index 95d894a231fcf..19a5f196829f3 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
@@ -900,7 +900,7 @@ static int gmc_v10_0_sw_init(struct amdgpu_ip_block *ip_block)
 	 * amdgpu graphics/compute will use VMIDs 1-7
 	 * amdkfd will use VMIDs 8-15
 	 */
-	adev->vm_manager.first_kfd_vmid = 8;
+	adev->vm_manager.first_kfd_vmid = adev->gfx.disable_kq ? 1 : 8;
 
 	amdgpu_vm_manager_init(adev);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
index ea7c32d8380ba..598324e736092 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
@@ -837,7 +837,7 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 	 * amdgpu graphics/compute will use VMIDs 1-7
 	 * amdkfd will use VMIDs 8-15
 	 */
-	adev->vm_manager.first_kfd_vmid = 8;
+	adev->vm_manager.first_kfd_vmid = adev->gfx.disable_kq ? 1 : 8;
 
 	amdgpu_vm_manager_init(adev);
 
-- 
2.48.1

[PATCH 07/11] drm/amdgpu/gfx11: add support for disable_kq

[Alex Deucher]

Plumb in support for disabling kernel queues in
GFX11.  We have to bring up a GFX queue briefly in
order to initialize the clear state.  After that
we can disable it.

v2: use ring counts per Felix' suggestion

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 99 +++++++++++++++++---------
 1 file changed, 65 insertions(+), 34 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
index 95eefd9a40d28..b20624f8cbbbd 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
@@ -1145,6 +1145,10 @@ static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
 
 	ring->ring_obj = NULL;
 	ring->use_doorbell = true;
+	if (adev->gfx.disable_kq) {
+		ring->no_scheduler = true;
+		ring->no_user_submission = true;
+	}
 
 	if (!ring_id)
 		ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
@@ -1577,7 +1581,7 @@ static void gfx_v11_0_alloc_ip_dump(struct amdgpu_device *adev)
 
 static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
 {
-	int i, j, k, r, ring_id = 0;
+	int i, j, k, r, ring_id;
 	int xcc_id = 0;
 	struct amdgpu_device *adev = ip_block->adev;
 
@@ -1710,37 +1714,42 @@ static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
 		return r;
 	}
 
-	/* set up the gfx ring */
-	for (i = 0; i < adev->gfx.me.num_me; i++) {
-		for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
-			for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
-				if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
-					continue;
-
-				r = gfx_v11_0_gfx_ring_init(adev, ring_id,
-							    i, k, j);
-				if (r)
-					return r;
-				ring_id++;
+	if (adev->gfx.num_gfx_rings) {
+		ring_id = 0;
+		/* set up the gfx ring */
+		for (i = 0; i < adev->gfx.me.num_me; i++) {
+			for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
+				for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
+					if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
+						continue;
+
+					r = gfx_v11_0_gfx_ring_init(adev, ring_id,
+								    i, k, j);
+					if (r)
+						return r;
+					ring_id++;
+				}
 			}
 		}
 	}
 
-	ring_id = 0;
-	/* set up the compute queues - allocate horizontally across pipes */
-	for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
-		for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
-			for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
-				if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
-								     k, j))
-					continue;
+	if (adev->gfx.num_compute_rings) {
+		ring_id = 0;
+		/* set up the compute queues - allocate horizontally across pipes */
+		for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
+			for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
+				for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
+					if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
+									     k, j))
+						continue;
 
-				r = gfx_v11_0_compute_ring_init(adev, ring_id,
-								i, k, j);
-				if (r)
-					return r;
+					r = gfx_v11_0_compute_ring_init(adev, ring_id,
+									i, k, j);
+					if (r)
+						return r;
 
-				ring_id++;
+					ring_id++;
+				}
 			}
 		}
 	}
@@ -4578,11 +4587,22 @@ static int gfx_v11_0_cp_resume(struct amdgpu_device *adev)
 			return r;
 	}
 
-	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
-		ring = &adev->gfx.gfx_ring[i];
-		r = amdgpu_ring_test_helper(ring);
-		if (r)
-			return r;
+	if (adev->gfx.disable_kq) {
+		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
+			ring = &adev->gfx.gfx_ring[i];
+			/* we don't want to set ring->ready */
+			r = amdgpu_ring_test_ring(ring);
+			if (r)
+				return r;
+		}
+		amdgpu_gfx_disable_kgq(adev, 0);
+	} else {
+		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
+			ring = &adev->gfx.gfx_ring[i];
+			r = amdgpu_ring_test_helper(ring);
+			if (r)
+				return r;
+		}
 	}
 
 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
@@ -5128,11 +5148,22 @@ static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
+	if (amdgpu_disable_kq == 1)
+		adev->gfx.disable_kq = true;
+
 	adev->gfx.funcs = &gfx_v11_0_gfx_funcs;
 
-	adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS;
-	adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
-					  AMDGPU_MAX_COMPUTE_RINGS);
+	if (adev->gfx.disable_kq) {
+		/* We need one GFX ring temporarily to set up
+		 * the clear state.
+		 */
+		adev->gfx.num_gfx_rings = 1;
+		adev->gfx.num_compute_rings = 0;
+	} else {
+		adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS;
+		adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
+						  AMDGPU_MAX_COMPUTE_RINGS);
+	}
 
 	gfx_v11_0_set_kiq_pm4_funcs(adev);
 	gfx_v11_0_set_ring_funcs(adev);
-- 
2.48.1

[PATCH 08/11] drm/amdgpu/gfx12: add support for disable_kq

[Alex Deucher]

Plumb in support for disabling kernel queues.

v2: use ring counts per Felix' suggestion

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c | 96 ++++++++++++++++----------
 1 file changed, 58 insertions(+), 38 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
index 34cf187e72d9f..23ee4651cbffb 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
@@ -1421,11 +1421,13 @@ static int gfx_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 		break;
 	}
 
-	/* recalculate compute rings to use based on hardware configuration */
-	num_compute_rings = (adev->gfx.mec.num_pipe_per_mec *
-			     adev->gfx.mec.num_queue_per_pipe) / 2;
-	adev->gfx.num_compute_rings = min(adev->gfx.num_compute_rings,
-					  num_compute_rings);
+	if (adev->gfx.num_compute_rings) {
+		/* recalculate compute rings to use based on hardware configuration */
+		num_compute_rings = (adev->gfx.mec.num_pipe_per_mec *
+				     adev->gfx.mec.num_queue_per_pipe) / 2;
+		adev->gfx.num_compute_rings = min(adev->gfx.num_compute_rings,
+						  num_compute_rings);
+	}
 
 	/* EOP Event */
 	r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
@@ -1471,37 +1473,41 @@ static int gfx_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 		return r;
 	}
 
-	/* set up the gfx ring */
-	for (i = 0; i < adev->gfx.me.num_me; i++) {
-		for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
-			for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
-				if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
-					continue;
-
-				r = gfx_v12_0_gfx_ring_init(adev, ring_id,
-							    i, k, j);
-				if (r)
-					return r;
-				ring_id++;
+	if (adev->gfx.num_gfx_rings) {
+		/* set up the gfx ring */
+		for (i = 0; i < adev->gfx.me.num_me; i++) {
+			for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
+				for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
+					if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
+						continue;
+
+					r = gfx_v12_0_gfx_ring_init(adev, ring_id,
+								    i, k, j);
+					if (r)
+						return r;
+					ring_id++;
+				}
 			}
 		}
 	}
 
-	ring_id = 0;
-	/* set up the compute queues - allocate horizontally across pipes */
-	for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
-		for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
-			for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
-				if (!amdgpu_gfx_is_mec_queue_enabled(adev,
-								0, i, k, j))
-					continue;
+	if (adev->gfx.num_compute_rings) {
+		ring_id = 0;
+		/* set up the compute queues - allocate horizontally across pipes */
+		for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
+			for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
+				for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
+					if (!amdgpu_gfx_is_mec_queue_enabled(adev,
+									     0, i, k, j))
+						continue;
 
-				r = gfx_v12_0_compute_ring_init(adev, ring_id,
-								i, k, j);
-				if (r)
-					return r;
+					r = gfx_v12_0_compute_ring_init(adev, ring_id,
+									i, k, j);
+					if (r)
+						return r;
 
-				ring_id++;
+					ring_id++;
+				}
 			}
 		}
 	}
@@ -3495,12 +3501,18 @@ static int gfx_v12_0_cp_resume(struct amdgpu_device *adev)
 	if (r)
 		return r;
 
-	if (!amdgpu_async_gfx_ring) {
-		r = gfx_v12_0_cp_gfx_resume(adev);
-		if (r)
-			return r;
+	if (adev->gfx.num_gfx_rings) {
+		if (!amdgpu_async_gfx_ring) {
+			r = gfx_v12_0_cp_gfx_resume(adev);
+			if (r)
+				return r;
+		} else {
+			r = gfx_v12_0_cp_async_gfx_ring_resume(adev);
+			if (r)
+				return r;
+		}
 	} else {
-		r = gfx_v12_0_cp_async_gfx_ring_resume(adev);
+		r = gfx_v12_0_cp_gfx_start(adev);
 		if (r)
 			return r;
 	}
@@ -3809,11 +3821,19 @@ static int gfx_v12_0_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
+	if (amdgpu_disable_kq == 1)
+		adev->gfx.disable_kq = true;
+
 	adev->gfx.funcs = &gfx_v12_0_gfx_funcs;
 
-	adev->gfx.num_gfx_rings = GFX12_NUM_GFX_RINGS;
-	adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
-					  AMDGPU_MAX_COMPUTE_RINGS);
+	if (adev->gfx.disable_kq) {
+		adev->gfx.num_gfx_rings = 0;
+		adev->gfx.num_compute_rings = 0;
+	} else {
+		adev->gfx.num_gfx_rings = GFX12_NUM_GFX_RINGS;
+		adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
+						  AMDGPU_MAX_COMPUTE_RINGS);
+	}
 
 	gfx_v12_0_set_kiq_pm4_funcs(adev);
 	gfx_v12_0_set_ring_funcs(adev);
-- 
2.48.1

[PATCH 09/11] drm/amdgpu/sdma: add flag for tracking disable_kq

[Alex Deucher]

For SDMA, we still need kernel queues for paging so
they need to be initialized, but we no not want to
accept submissions from userspace when disable_kq
is set.

Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
index 9651693200655..edc856e10337a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
@@ -129,6 +129,7 @@ struct amdgpu_sdma {
 	/* track guilty state of GFX and PAGE queues */
 	bool gfx_guilty;
 	bool page_guilty;
+	bool			no_user_submission;
 };
 
 /*
-- 
2.48.1

[PATCH 10/11] drm/amdgpu/sdma6: add support for disable_kq

[Alex Deucher]

When the parameter is set, disable user submissions
to kernel queues.

Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
index 3aa4fec4d9e4a..bcc72737f8084 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
@@ -1304,6 +1304,9 @@ static int sdma_v6_0_early_init(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	int r;
 
+	if (amdgpu_disable_kq == 1)
+		adev->sdma.no_user_submission = true;
+
 	r = amdgpu_sdma_init_microcode(adev, 0, true);
 	if (r)
 		return r;
@@ -1338,6 +1341,7 @@ static int sdma_v6_0_sw_init(struct amdgpu_ip_block *ip_block)
 		ring->ring_obj = NULL;
 		ring->use_doorbell = true;
 		ring->me = i;
+		ring->no_user_submission = adev->sdma.no_user_submission;
 
 		DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
 				ring->use_doorbell?"true":"false");
-- 
2.48.1

[PATCH 11/11] drm/amdgpu/sdma7: add support for disable_kq

[Alex Deucher]

When the parameter is set, disable user submissions
to kernel queues.

Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
index 92a79296708ae..40d45f738c0a8 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
@@ -1316,6 +1316,9 @@ static int sdma_v7_0_early_init(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	int r;
 
+	if (amdgpu_disable_kq == 1)
+		adev->sdma.no_user_submission = true;
+
 	r = amdgpu_sdma_init_microcode(adev, 0, true);
 	if (r) {
 		DRM_ERROR("Failed to init sdma firmware!\n");
@@ -1351,6 +1354,7 @@ static int sdma_v7_0_sw_init(struct amdgpu_ip_block *ip_block)
 		ring->ring_obj = NULL;
 		ring->use_doorbell = true;
 		ring->me = i;
+		ring->no_user_submission = adev->sdma.no_user_submission;
 
 		DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
 				ring->use_doorbell?"true":"false");
-- 
2.48.1

[PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq

[Alex Deucher]

Add proper checks for disable_kq functionality in
gfx helper functions.  Add special logic for families
that require the clear state setup.

v2: use ring count as per Felix suggestion
v3: fix num_gfx_rings handling in amdgpu_gfx_graphics_queue_acquire()

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 8 ++++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 2 ++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index a194bf3347cbc..2c78185a33218 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -258,8 +258,9 @@ void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev)
 	}
 
 	/* update the number of active graphics rings */
-	adev->gfx.num_gfx_rings =
-		bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
+	if (adev->gfx.num_gfx_rings)
+		adev->gfx.num_gfx_rings =
+			bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
 }
 
 static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev,
@@ -1544,6 +1545,9 @@ static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device *dev,
 	if (adev->in_suspend && !adev->in_runpm)
 		return -EPERM;
 
+	if (adev->gfx.disable_kq)
+		return -ENOTSUPP;
+
 	ret = kstrtol(buf, 0, &value);
 
 	if (ret)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index ddf4533614bac..8fa68a4ac34f1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -483,6 +483,8 @@ struct amdgpu_gfx {
 
 	atomic_t			total_submission_cnt;
 	struct delayed_work		idle_work;
+
+	bool				disable_kq;
 };
 
 struct amdgpu_gfx_ras_reg_entry {
-- 
2.48.1

RE: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Liang, Prike]

[Public]

> From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Alex
> Deucher
> Sent: Friday, March 7, 2025 2:46 AM
> To: amd-gfx@lists.freedesktop.org
> Cc: Deucher, Alexander <Alexander.Deucher@amd.com>
> Subject: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions
>
> This would be set by IPs which only accept submissions from the kernel, not
> userspace, such as when kernel queues are disabled. Don't expose the rings to
> userspace and reject any submissions in the CS IOCTL.
>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
> drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
>  3 files changed, 25 insertions(+), 11 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> index 5df21529b3b13..5cc18034b75df 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> @@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser
> *p,
>       ring = amdgpu_job_ring(job);
>       ib = &job->ibs[job->num_ibs++];
>
> +     /* submissions to kernel queus are disabled */
> +     if (ring->no_user_submission)
> +             return -EINVAL;
> +

Do we need reject the kernel queue submission at the beginning placement of amdgpu_cs_ioctl()?

Thanks,
Prike
>       /* MM engine doesn't support user fences */
>       if (p->uf_bo && ring->funcs->no_user_fence)
>               return -EINVAL;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>       case AMDGPU_HW_IP_GFX:
>               type = AMD_IP_BLOCK_TYPE_GFX;
>               for (i = 0; i < adev->gfx.num_gfx_rings; i++)
> -                     if (adev->gfx.gfx_ring[i].sched.ready)
> +                     if (adev->gfx.gfx_ring[i].sched.ready &&
> +                         !adev->gfx.gfx_ring[i].no_user_submission)
>                               ++num_rings;
>               ib_start_alignment = 32;
>               ib_size_alignment = 32;
> @@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>       case AMDGPU_HW_IP_COMPUTE:
>               type = AMD_IP_BLOCK_TYPE_GFX;
>               for (i = 0; i < adev->gfx.num_compute_rings; i++)
> -                     if (adev->gfx.compute_ring[i].sched.ready)
> +                     if (adev->gfx.compute_ring[i].sched.ready &&
> +                         !adev->gfx.compute_ring[i].no_user_submission)
>                               ++num_rings;
>               ib_start_alignment = 32;
>               ib_size_alignment = 32;
> @@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>       case AMDGPU_HW_IP_DMA:
>               type = AMD_IP_BLOCK_TYPE_SDMA;
>               for (i = 0; i < adev->sdma.num_instances; i++)
> -                     if (adev->sdma.instance[i].ring.sched.ready)
> +                     if (adev->sdma.instance[i].ring.sched.ready &&
> +                         !adev->gfx.gfx_ring[i].no_user_submission)
>                               ++num_rings;
>               ib_start_alignment = 256;
>               ib_size_alignment = 4;
> @@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>                       if (adev->uvd.harvest_config & (1 << i))
>                               continue;
>
> -                     if (adev->uvd.inst[i].ring.sched.ready)
> +                     if (adev->uvd.inst[i].ring.sched.ready &&
> +                         !adev->uvd.inst[i].ring.no_user_submission)
>                               ++num_rings;
>               }
>               ib_start_alignment = 256;
> @@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>       case AMDGPU_HW_IP_VCE:
>               type = AMD_IP_BLOCK_TYPE_VCE;
>               for (i = 0; i < adev->vce.num_rings; i++)
> -                     if (adev->vce.ring[i].sched.ready)
> +                     if (adev->vce.ring[i].sched.ready &&
> +                         !adev->vce.ring[i].no_user_submission)
>                               ++num_rings;
>               ib_start_alignment = 256;
>               ib_size_alignment = 4;
> @@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>                               continue;
>
>                       for (j = 0; j < adev->uvd.num_enc_rings; j++)
> -                             if (adev->uvd.inst[i].ring_enc[j].sched.ready)
> +                             if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
> +                                 !adev->uvd.inst[i].ring_enc[j].no_user_submission)
>                                       ++num_rings;
>               }
>               ib_start_alignment = 256;
> @@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>                       if (adev->vcn.harvest_config & (1 << i))
>                               continue;
>
> -                     if (adev->vcn.inst[i].ring_dec.sched.ready)
> +                     if (adev->vcn.inst[i].ring_dec.sched.ready &&
> +                         !adev->vcn.inst[i].ring_dec.no_user_submission)
>                               ++num_rings;
>               }
>               ib_start_alignment = 256;
> @@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>                               continue;
>
>                       for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
> -                             if (adev->vcn.inst[i].ring_enc[j].sched.ready)
> +                             if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
> +                                 !adev->vcn.inst[i].ring_enc[j].no_user_submission)
>                                       ++num_rings;
>               }
>               ib_start_alignment = 256;
> @@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>                               continue;
>
>                       for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
> -                             if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
> +                             if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
> +                                 !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
>                                       ++num_rings;
>               }
>               ib_start_alignment = 256;
> @@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>               break;
>       case AMDGPU_HW_IP_VPE:
>               type = AMD_IP_BLOCK_TYPE_VPE;
> -             if (adev->vpe.ring.sched.ready)
> +             if (adev->vpe.ring.sched.ready &&
> +                 !adev->vpe.ring.no_user_submission)
>                       ++num_rings;
>               ib_start_alignment = 256;
>               ib_size_alignment = 4;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> index b4fd1e17205e9..4a97afcb38b78 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> @@ -297,6 +297,7 @@ struct amdgpu_ring {
>       struct dma_fence        *vmid_wait;
>       bool                    has_compute_vm_bug;
>       bool                    no_scheduler;
> +     bool                    no_user_submission;
>       int                     hw_prio;
>       unsigned                num_hw_submission;
>       atomic_t                *sched_score;
> @@ -310,7 +311,6 @@ struct amdgpu_ring {
>       unsigned int    entry_index;
>       /* store the cached rptr to restore after reset */
>       uint64_t cached_rptr;
> -
>  };
>
>  #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
> --
> 2.48.1

RE: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq

[Liang, Prike]

[Public]

> From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Alex
> Deucher
> Sent: Friday, March 7, 2025 11:16 PM
> To: amd-gfx@lists.freedesktop.org
> Cc: Deucher, Alexander <Alexander.Deucher@amd.com>
> Subject: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq
>
> Add proper checks for disable_kq functionality in gfx helper functions.  Add special
> logic for families that require the clear state setup.
>
> v2: use ring count as per Felix suggestion
> v3: fix num_gfx_rings handling in amdgpu_gfx_graphics_queue_acquire()
>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 8 ++++++--
> drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 2 ++
>  2 files changed, 8 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> index a194bf3347cbc..2c78185a33218 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> @@ -258,8 +258,9 @@ void amdgpu_gfx_graphics_queue_acquire(struct
> amdgpu_device *adev)
>       }
>
>       /* update the number of active graphics rings */
> -     adev->gfx.num_gfx_rings =
> -             bitmap_weight(adev->gfx.me.queue_bitmap,
> AMDGPU_MAX_GFX_QUEUES);
> +     if (adev->gfx.num_gfx_rings)
> +             adev->gfx.num_gfx_rings =
> +                     bitmap_weight(adev->gfx.me.queue_bitmap,
> AMDGPU_MAX_GFX_QUEUES);
>  }
>
>  static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev, @@ -1544,6
> +1545,9 @@ static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device
> *dev,
>       if (adev->in_suspend && !adev->in_runpm)
>               return -EPERM;
>
> +     if (adev->gfx.disable_kq)
> +             return -ENOTSUPP;
> +
Maybe here need to disable the flag enable_cleaner_shader as well?

Thanks,
Prike
>       ret = kstrtol(buf, 0, &value);
>
>       if (ret)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> index ddf4533614bac..8fa68a4ac34f1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> @@ -483,6 +483,8 @@ struct amdgpu_gfx {
>
>       atomic_t                        total_submission_cnt;
>       struct delayed_work             idle_work;
> +
> +     bool                            disable_kq;
>  };
>
>  struct amdgpu_gfx_ras_reg_entry {
> --
> 2.48.1

Re: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Alex Deucher]

On Tue, Mar 11, 2025 at 9:13 AM Liang, Prike <Prike.Liang@amd.com> wrote:
>
> [Public]
>
> > From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Alex
> > Deucher
> > Sent: Friday, March 7, 2025 2:46 AM
> > To: amd-gfx@lists.freedesktop.org
> > Cc: Deucher, Alexander <Alexander.Deucher@amd.com>
> > Subject: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions
> >
> > This would be set by IPs which only accept submissions from the kernel, not
> > userspace, such as when kernel queues are disabled. Don't expose the rings to
> > userspace and reject any submissions in the CS IOCTL.
> >
> > Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> > ---
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
> > drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
> >  3 files changed, 25 insertions(+), 11 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > index 5df21529b3b13..5cc18034b75df 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > @@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser
> > *p,
> >       ring = amdgpu_job_ring(job);
> >       ib = &job->ibs[job->num_ibs++];
> >
> > +     /* submissions to kernel queus are disabled */
> > +     if (ring->no_user_submission)
> > +             return -EINVAL;
> > +
>
> Do we need reject the kernel queue submission at the beginning placement of amdgpu_cs_ioctl()?

I think we only need to reject it if it's targeted at a kernel ring
and this is the first place we can determine that.

Alex

>
> Thanks,
> Prike
> >       /* MM engine doesn't support user fences */
> >       if (p->uf_bo && ring->funcs->no_user_fence)
> >               return -EINVAL;
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > @@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >       case AMDGPU_HW_IP_GFX:
> >               type = AMD_IP_BLOCK_TYPE_GFX;
> >               for (i = 0; i < adev->gfx.num_gfx_rings; i++)
> > -                     if (adev->gfx.gfx_ring[i].sched.ready)
> > +                     if (adev->gfx.gfx_ring[i].sched.ready &&
> > +                         !adev->gfx.gfx_ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 32;
> >               ib_size_alignment = 32;
> > @@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >       case AMDGPU_HW_IP_COMPUTE:
> >               type = AMD_IP_BLOCK_TYPE_GFX;
> >               for (i = 0; i < adev->gfx.num_compute_rings; i++)
> > -                     if (adev->gfx.compute_ring[i].sched.ready)
> > +                     if (adev->gfx.compute_ring[i].sched.ready &&
> > +                         !adev->gfx.compute_ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 32;
> >               ib_size_alignment = 32;
> > @@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >       case AMDGPU_HW_IP_DMA:
> >               type = AMD_IP_BLOCK_TYPE_SDMA;
> >               for (i = 0; i < adev->sdma.num_instances; i++)
> > -                     if (adev->sdma.instance[i].ring.sched.ready)
> > +                     if (adev->sdma.instance[i].ring.sched.ready &&
> > +                         !adev->gfx.gfx_ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 256;
> >               ib_size_alignment = 4;
> > @@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >                       if (adev->uvd.harvest_config & (1 << i))
> >                               continue;
> >
> > -                     if (adev->uvd.inst[i].ring.sched.ready)
> > +                     if (adev->uvd.inst[i].ring.sched.ready &&
> > +                         !adev->uvd.inst[i].ring.no_user_submission)
> >                               ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >       case AMDGPU_HW_IP_VCE:
> >               type = AMD_IP_BLOCK_TYPE_VCE;
> >               for (i = 0; i < adev->vce.num_rings; i++)
> > -                     if (adev->vce.ring[i].sched.ready)
> > +                     if (adev->vce.ring[i].sched.ready &&
> > +                         !adev->vce.ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 256;
> >               ib_size_alignment = 4;
> > @@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >                               continue;
> >
> >                       for (j = 0; j < adev->uvd.num_enc_rings; j++)
> > -                             if (adev->uvd.inst[i].ring_enc[j].sched.ready)
> > +                             if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
> > +                                 !adev->uvd.inst[i].ring_enc[j].no_user_submission)
> >                                       ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >                       if (adev->vcn.harvest_config & (1 << i))
> >                               continue;
> >
> > -                     if (adev->vcn.inst[i].ring_dec.sched.ready)
> > +                     if (adev->vcn.inst[i].ring_dec.sched.ready &&
> > +                         !adev->vcn.inst[i].ring_dec.no_user_submission)
> >                               ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >                               continue;
> >
> >                       for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
> > -                             if (adev->vcn.inst[i].ring_enc[j].sched.ready)
> > +                             if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
> > +                                 !adev->vcn.inst[i].ring_enc[j].no_user_submission)
> >                                       ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >                               continue;
> >
> >                       for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
> > -                             if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
> > +                             if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
> > +                                 !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
> >                                       ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >               break;
> >       case AMDGPU_HW_IP_VPE:
> >               type = AMD_IP_BLOCK_TYPE_VPE;
> > -             if (adev->vpe.ring.sched.ready)
> > +             if (adev->vpe.ring.sched.ready &&
> > +                 !adev->vpe.ring.no_user_submission)
> >                       ++num_rings;
> >               ib_start_alignment = 256;
> >               ib_size_alignment = 4;
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > index b4fd1e17205e9..4a97afcb38b78 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > @@ -297,6 +297,7 @@ struct amdgpu_ring {
> >       struct dma_fence        *vmid_wait;
> >       bool                    has_compute_vm_bug;
> >       bool                    no_scheduler;
> > +     bool                    no_user_submission;
> >       int                     hw_prio;
> >       unsigned                num_hw_submission;
> >       atomic_t                *sched_score;
> > @@ -310,7 +311,6 @@ struct amdgpu_ring {
> >       unsigned int    entry_index;
> >       /* store the cached rptr to restore after reset */
> >       uint64_t cached_rptr;
> > -
> >  };
> >
> >  #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
> > --
> > 2.48.1
>

Re: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq

[Alex Deucher]

On Tue, Mar 11, 2025 at 10:18 AM Liang, Prike <Prike.Liang@amd.com> wrote:
>
> [Public]
>
> > From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Alex
> > Deucher
> > Sent: Friday, March 7, 2025 11:16 PM
> > To: amd-gfx@lists.freedesktop.org
> > Cc: Deucher, Alexander <Alexander.Deucher@amd.com>
> > Subject: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq
> >
> > Add proper checks for disable_kq functionality in gfx helper functions.  Add special
> > logic for families that require the clear state setup.
> >
> > v2: use ring count as per Felix suggestion
> > v3: fix num_gfx_rings handling in amdgpu_gfx_graphics_queue_acquire()
> >
> > Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> > ---
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 8 ++++++--
> > drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 2 ++
> >  2 files changed, 8 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> > index a194bf3347cbc..2c78185a33218 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> > @@ -258,8 +258,9 @@ void amdgpu_gfx_graphics_queue_acquire(struct
> > amdgpu_device *adev)
> >       }
> >
> >       /* update the number of active graphics rings */
> > -     adev->gfx.num_gfx_rings =
> > -             bitmap_weight(adev->gfx.me.queue_bitmap,
> > AMDGPU_MAX_GFX_QUEUES);
> > +     if (adev->gfx.num_gfx_rings)
> > +             adev->gfx.num_gfx_rings =
> > +                     bitmap_weight(adev->gfx.me.queue_bitmap,
> > AMDGPU_MAX_GFX_QUEUES);
> >  }
> >
> >  static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev, @@ -1544,6
> > +1545,9 @@ static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device
> > *dev,
> >       if (adev->in_suspend && !adev->in_runpm)
> >               return -EPERM;
> >
> > +     if (adev->gfx.disable_kq)
> > +             return -ENOTSUPP;
> > +
> Maybe here need to disable the flag enable_cleaner_shader as well?

We still need it, but the MES runs it when switching queues rather
than the driver in the userq case.

Alex

>
> Thanks,
> Prike
> >       ret = kstrtol(buf, 0, &value);
> >
> >       if (ret)
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> > index ddf4533614bac..8fa68a4ac34f1 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> > @@ -483,6 +483,8 @@ struct amdgpu_gfx {
> >
> >       atomic_t                        total_submission_cnt;
> >       struct delayed_work             idle_work;
> > +
> > +     bool                            disable_kq;
> >  };
> >
> >  struct amdgpu_gfx_ras_reg_entry {
> > --
> > 2.48.1
>

[PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq

[Alex Deucher]

Add proper checks for disable_kq functionality in
gfx helper functions.  Add special logic for families
that require the clear state setup.

v2: use ring count as per Felix suggestion
v3: fix num_gfx_rings handling in amdgpu_gfx_graphics_queue_acquire()

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 8 ++++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 2 ++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 984e6ff6e4632..a08243dd0798e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -258,8 +258,9 @@ void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev)
 	}
 
 	/* update the number of active graphics rings */
-	adev->gfx.num_gfx_rings =
-		bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
+	if (adev->gfx.num_gfx_rings)
+		adev->gfx.num_gfx_rings =
+			bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
 }
 
 static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev,
@@ -1544,6 +1545,9 @@ static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device *dev,
 	if (adev->in_suspend && !adev->in_runpm)
 		return -EPERM;
 
+	if (adev->gfx.disable_kq)
+		return -ENOTSUPP;
+
 	ret = kstrtol(buf, 0, &value);
 
 	if (ret)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index ddf4533614bac..8fa68a4ac34f1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -483,6 +483,8 @@ struct amdgpu_gfx {
 
 	atomic_t			total_submission_cnt;
 	struct delayed_work		idle_work;
+
+	bool				disable_kq;
 };
 
 struct amdgpu_gfx_ras_reg_entry {
-- 
2.48.1

[PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq

[Alex Deucher]

Add proper checks for disable_kq functionality in
gfx helper functions.  Add special logic for families
that require the clear state setup.

v2: use ring count as per Felix suggestion
v3: fix num_gfx_rings handling in amdgpu_gfx_graphics_queue_acquire()

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 8 ++++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 2 ++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 984e6ff6e4632..a08243dd0798e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -258,8 +258,9 @@ void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev)
 	}
 
 	/* update the number of active graphics rings */
-	adev->gfx.num_gfx_rings =
-		bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
+	if (adev->gfx.num_gfx_rings)
+		adev->gfx.num_gfx_rings =
+			bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
 }
 
 static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev,
@@ -1544,6 +1545,9 @@ static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device *dev,
 	if (adev->in_suspend && !adev->in_runpm)
 		return -EPERM;
 
+	if (adev->gfx.disable_kq)
+		return -ENOTSUPP;
+
 	ret = kstrtol(buf, 0, &value);
 
 	if (ret)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index ddf4533614bac..8fa68a4ac34f1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -483,6 +483,8 @@ struct amdgpu_gfx {
 
 	atomic_t			total_submission_cnt;
 	struct delayed_work		idle_work;
+
+	bool				disable_kq;
 };
 
 struct amdgpu_gfx_ras_reg_entry {
-- 
2.48.1

Re: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq

[Khatri, Sunil]

Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>

On 3/13/2025 8:11 PM, Alex Deucher wrote:
> Add proper checks for disable_kq functionality in
> gfx helper functions.  Add special logic for families
> that require the clear state setup.
>
> v2: use ring count as per Felix suggestion
> v3: fix num_gfx_rings handling in amdgpu_gfx_graphics_queue_acquire()
>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 8 ++++++--
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 2 ++
>   2 files changed, 8 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> index 984e6ff6e4632..a08243dd0798e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> @@ -258,8 +258,9 @@ void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev)
>   	}
>   
>   	/* update the number of active graphics rings */
> -	adev->gfx.num_gfx_rings =
> -		bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
> +	if (adev->gfx.num_gfx_rings)
> +		adev->gfx.num_gfx_rings =
> +			bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
>   }
>   
>   static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev,
> @@ -1544,6 +1545,9 @@ static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device *dev,
>   	if (adev->in_suspend && !adev->in_runpm)
>   		return -EPERM;
>   
> +	if (adev->gfx.disable_kq)
> +		return -ENOTSUPP;
> +
>   	ret = kstrtol(buf, 0, &value);
>   
>   	if (ret)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> index ddf4533614bac..8fa68a4ac34f1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> @@ -483,6 +483,8 @@ struct amdgpu_gfx {
>   
>   	atomic_t			total_submission_cnt;
>   	struct delayed_work		idle_work;
> +
> +	bool				disable_kq;
>   };
>   
>   struct amdgpu_gfx_ras_reg_entry {

RE: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq

[Liang, Prike]

[Public]

When disabling the gfx kernel queue, then the related ring function callback should be unassigned, and the clean shader callback should also not be further invoked. To avoid the clean shader resource allocated, we may need to drop the shader clean initialized at *_ sw_init() when disable gfx kernel queue.

Regards,
      Prike

> -----Original Message-----
> From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Alex
> Deucher
> Sent: Thursday, March 13, 2025 10:41 PM
> To: amd-gfx@lists.freedesktop.org
> Cc: Deucher, Alexander <Alexander.Deucher@amd.com>
> Subject: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq
>
> Add proper checks for disable_kq functionality in gfx helper functions.  Add special
> logic for families that require the clear state setup.
>
> v2: use ring count as per Felix suggestion
> v3: fix num_gfx_rings handling in amdgpu_gfx_graphics_queue_acquire()
>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 8 ++++++--
> drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 2 ++
>  2 files changed, 8 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> index 984e6ff6e4632..a08243dd0798e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> @@ -258,8 +258,9 @@ void amdgpu_gfx_graphics_queue_acquire(struct
> amdgpu_device *adev)
>       }
>
>       /* update the number of active graphics rings */
> -     adev->gfx.num_gfx_rings =
> -             bitmap_weight(adev->gfx.me.queue_bitmap,
> AMDGPU_MAX_GFX_QUEUES);
> +     if (adev->gfx.num_gfx_rings)
> +             adev->gfx.num_gfx_rings =
> +                     bitmap_weight(adev->gfx.me.queue_bitmap,
> AMDGPU_MAX_GFX_QUEUES);
>  }
>
>  static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev, @@ -1544,6
> +1545,9 @@ static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device
> *dev,
>       if (adev->in_suspend && !adev->in_runpm)
>               return -EPERM;
>
> +     if (adev->gfx.disable_kq)
> +             return -ENOTSUPP;
> +
>       ret = kstrtol(buf, 0, &value);
>
>       if (ret)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> index ddf4533614bac..8fa68a4ac34f1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> @@ -483,6 +483,8 @@ struct amdgpu_gfx {
>
>       atomic_t                        total_submission_cnt;
>       struct delayed_work             idle_work;
> +
> +     bool                            disable_kq;
>  };
>
>  struct amdgpu_gfx_ras_reg_entry {
> --
> 2.48.1

Re: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq

[Alex Deucher]

On Mon, Mar 17, 2025 at 5:07 AM Liang, Prike <Prike.Liang@amd.com> wrote:
>
> [Public]
>
> When disabling the gfx kernel queue, then the related ring function callback should be unassigned, and the clean shader callback should also not be further invoked. To avoid the clean shader resource allocated, we may need to drop the shader clean initialized at *_ sw_init() when disable gfx kernel queue.

This patch prevents the cleaner shader from being run via sysfs
already.  We still need to allocate the cleaner shader resources
though because they are used by the MES and CP firmware for user
queues.

Alex

>
> Regards,
>       Prike
>
> > -----Original Message-----
> > From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Alex
> > Deucher
> > Sent: Thursday, March 13, 2025 10:41 PM
> > To: amd-gfx@lists.freedesktop.org
> > Cc: Deucher, Alexander <Alexander.Deucher@amd.com>
> > Subject: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq
> >
> > Add proper checks for disable_kq functionality in gfx helper functions.  Add special
> > logic for families that require the clear state setup.
> >
> > v2: use ring count as per Felix suggestion
> > v3: fix num_gfx_rings handling in amdgpu_gfx_graphics_queue_acquire()
> >
> > Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> > ---
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 8 ++++++--
> > drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 2 ++
> >  2 files changed, 8 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> > index 984e6ff6e4632..a08243dd0798e 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> > @@ -258,8 +258,9 @@ void amdgpu_gfx_graphics_queue_acquire(struct
> > amdgpu_device *adev)
> >       }
> >
> >       /* update the number of active graphics rings */
> > -     adev->gfx.num_gfx_rings =
> > -             bitmap_weight(adev->gfx.me.queue_bitmap,
> > AMDGPU_MAX_GFX_QUEUES);
> > +     if (adev->gfx.num_gfx_rings)
> > +             adev->gfx.num_gfx_rings =
> > +                     bitmap_weight(adev->gfx.me.queue_bitmap,
> > AMDGPU_MAX_GFX_QUEUES);
> >  }
> >
> >  static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev, @@ -1544,6
> > +1545,9 @@ static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device
> > *dev,
> >       if (adev->in_suspend && !adev->in_runpm)
> >               return -EPERM;
> >
> > +     if (adev->gfx.disable_kq)
> > +             return -ENOTSUPP;
> > +
> >       ret = kstrtol(buf, 0, &value);
> >
> >       if (ret)
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> > index ddf4533614bac..8fa68a4ac34f1 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> > @@ -483,6 +483,8 @@ struct amdgpu_gfx {
> >
> >       atomic_t                        total_submission_cnt;
> >       struct delayed_work             idle_work;
> > +
> > +     bool                            disable_kq;
> >  };
> >
> >  struct amdgpu_gfx_ras_reg_entry {
> > --
> > 2.48.1
>

RE: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq

[Liang, Prike]

[Public]

> From: Alex Deucher <alexdeucher@gmail.com>
> Sent: Monday, March 17, 2025 9:33 PM
> To: Liang, Prike <Prike.Liang@amd.com>
> Cc: Deucher, Alexander <Alexander.Deucher@amd.com>; amd-
> gfx@lists.freedesktop.org
> Subject: Re: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq
>
> On Mon, Mar 17, 2025 at 5:07 AM Liang, Prike <Prike.Liang@amd.com> wrote:
> >
> > [Public]
> >
> > When disabling the gfx kernel queue, then the related ring function callback
> should be unassigned, and the clean shader callback should also not be further
> invoked. To avoid the clean shader resource allocated, we may need to drop the
> shader clean initialized at *_ sw_init() when disable gfx kernel queue.
>
> This patch prevents the cleaner shader from being run via sysfs already.  We still
> need to allocate the cleaner shader resources though because they are used by
> the MES and CP firmware for user queues.
>
> Alex

Yeah, I revisit the enable_cleaner_shader flag usage logic, and the driver requires it to isolate between the GFX and compute processes.

Reviewed-by: Prike Liang <Prike.Liang@amd.com>

> >
> > Regards,
> >       Prike
> >
> > > -----Original Message-----
> > > From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of
> > > Alex Deucher
> > > Sent: Thursday, March 13, 2025 10:41 PM
> > > To: amd-gfx@lists.freedesktop.org
> > > Cc: Deucher, Alexander <Alexander.Deucher@amd.com>
> > > Subject: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for
> > > disable_kq
> > >
> > > Add proper checks for disable_kq functionality in gfx helper
> > > functions.  Add special logic for families that require the clear state setup.
> > >
> > > v2: use ring count as per Felix suggestion
> > > v3: fix num_gfx_rings handling in
> > > amdgpu_gfx_graphics_queue_acquire()
> > >
> > > Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> > > ---
> > >  drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 8 ++++++--
> > > drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 2 ++
> > >  2 files changed, 8 insertions(+), 2 deletions(-)
> > >
> > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> > > b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> > > index 984e6ff6e4632..a08243dd0798e 100644
> > > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> > > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> > > @@ -258,8 +258,9 @@ void amdgpu_gfx_graphics_queue_acquire(struct
> > > amdgpu_device *adev)
> > >       }
> > >
> > >       /* update the number of active graphics rings */
> > > -     adev->gfx.num_gfx_rings =
> > > -             bitmap_weight(adev->gfx.me.queue_bitmap,
> > > AMDGPU_MAX_GFX_QUEUES);
> > > +     if (adev->gfx.num_gfx_rings)
> > > +             adev->gfx.num_gfx_rings =
> > > +                     bitmap_weight(adev->gfx.me.queue_bitmap,
> > > AMDGPU_MAX_GFX_QUEUES);
> > >  }
> > >
> > >  static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev, @@
> > > -1544,6
> > > +1545,9 @@ static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct
> > > +device
> > > *dev,
> > >       if (adev->in_suspend && !adev->in_runpm)
> > >               return -EPERM;
> > >
> > > +     if (adev->gfx.disable_kq)
> > > +             return -ENOTSUPP;
> > > +
> > >       ret = kstrtol(buf, 0, &value);
> > >
> > >       if (ret)
> > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> > > b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> > > index ddf4533614bac..8fa68a4ac34f1 100644
> > > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> > > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> > > @@ -483,6 +483,8 @@ struct amdgpu_gfx {
> > >
> > >       atomic_t                        total_submission_cnt;
> > >       struct delayed_work             idle_work;
> > > +
> > > +     bool                            disable_kq;
> > >  };
> > >
> > >  struct amdgpu_gfx_ras_reg_entry {
> > > --
> > > 2.48.1
> >