[PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Alex Deucher]

This would be set by IPs which only accept submissions
from the kernel, not userspace, such as when kernel
queues are disabled. Don't expose the rings to userspace
and reject any submissions in the CS IOCTL.

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
 3 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 5df21529b3b13..5cc18034b75df 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,
 	ring = amdgpu_job_ring(job);
 	ib = &job->ibs[job->num_ibs++];
 
+	/* submissions to kernel queus are disabled */
+	if (ring->no_user_submission)
+		return -EINVAL;
+
 	/* MM engine doesn't support user fences */
 	if (p->uf_bo && ring->funcs->no_user_fence)
 		return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_GFX:
 		type = AMD_IP_BLOCK_TYPE_GFX;
 		for (i = 0; i < adev->gfx.num_gfx_rings; i++)
-			if (adev->gfx.gfx_ring[i].sched.ready)
+			if (adev->gfx.gfx_ring[i].sched.ready &&
+			    !adev->gfx.gfx_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 32;
 		ib_size_alignment = 32;
@@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_COMPUTE:
 		type = AMD_IP_BLOCK_TYPE_GFX;
 		for (i = 0; i < adev->gfx.num_compute_rings; i++)
-			if (adev->gfx.compute_ring[i].sched.ready)
+			if (adev->gfx.compute_ring[i].sched.ready &&
+			    !adev->gfx.compute_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 32;
 		ib_size_alignment = 32;
@@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_DMA:
 		type = AMD_IP_BLOCK_TYPE_SDMA;
 		for (i = 0; i < adev->sdma.num_instances; i++)
-			if (adev->sdma.instance[i].ring.sched.ready)
+			if (adev->sdma.instance[i].ring.sched.ready &&
+			    !adev->gfx.gfx_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
@@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 			if (adev->uvd.harvest_config & (1 << i))
 				continue;
 
-			if (adev->uvd.inst[i].ring.sched.ready)
+			if (adev->uvd.inst[i].ring.sched.ready &&
+			    !adev->uvd.inst[i].ring.no_user_submission)
 				++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_VCE:
 		type = AMD_IP_BLOCK_TYPE_VCE;
 		for (i = 0; i < adev->vce.num_rings; i++)
-			if (adev->vce.ring[i].sched.ready)
+			if (adev->vce.ring[i].sched.ready &&
+			    !adev->vce.ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
@@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->uvd.num_enc_rings; j++)
-				if (adev->uvd.inst[i].ring_enc[j].sched.ready)
+				if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
+				    !adev->uvd.inst[i].ring_enc[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 			if (adev->vcn.harvest_config & (1 << i))
 				continue;
 
-			if (adev->vcn.inst[i].ring_dec.sched.ready)
+			if (adev->vcn.inst[i].ring_dec.sched.ready &&
+			    !adev->vcn.inst[i].ring_dec.no_user_submission)
 				++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
-				if (adev->vcn.inst[i].ring_enc[j].sched.ready)
+				if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
+				    !adev->vcn.inst[i].ring_enc[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
-				if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
+				if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
+				    !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 		break;
 	case AMDGPU_HW_IP_VPE:
 		type = AMD_IP_BLOCK_TYPE_VPE;
-		if (adev->vpe.ring.sched.ready)
+		if (adev->vpe.ring.sched.ready &&
+		    !adev->vpe.ring.no_user_submission)
 			++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index b4fd1e17205e9..4a97afcb38b78 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -297,6 +297,7 @@ struct amdgpu_ring {
 	struct dma_fence	*vmid_wait;
 	bool			has_compute_vm_bug;
 	bool			no_scheduler;
+	bool			no_user_submission;
 	int			hw_prio;
 	unsigned 		num_hw_submission;
 	atomic_t		*sched_score;
@@ -310,7 +311,6 @@ struct amdgpu_ring {
 	unsigned int    entry_index;
 	/* store the cached rptr to restore after reset */
 	uint64_t cached_rptr;
-
 };
 
 #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
-- 
2.48.1

Re: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Khatri, Sunil]

On 3/6/2025 2:17 AM, Alex Deucher wrote:
> This would be set by IPs which only accept submissions
> from the kernel, not userspace, such as when kernel
> queues are disabled. Don't expose the rings to userspace
> and reject any submissions in the CS IOCTL.
>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
>   3 files changed, 25 insertions(+), 11 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> index 5df21529b3b13..5cc18034b75df 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> @@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,
>   	ring = amdgpu_job_ring(job);
>   	ib = &job->ibs[job->num_ibs++];
>   
> +	/* submissions to kernel queus are disabled */
> +	if (ring->no_user_submission)
> +		return -EINVAL;
> +
Alex, dont we need to disable submission for other type of chunks, from 
function amdgpu_cs_pass2 ? chunk type of bo list and fences are 
submitted by users and we do want to handle that too.
If that's not needed then lgtm.
>   	/* MM engine doesn't support user fences */
>   	if (p->uf_bo && ring->funcs->no_user_fence)
>   		return -EINVAL;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   	case AMDGPU_HW_IP_GFX:
>   		type = AMD_IP_BLOCK_TYPE_GFX;
>   		for (i = 0; i < adev->gfx.num_gfx_rings; i++)
> -			if (adev->gfx.gfx_ring[i].sched.ready)
> +			if (adev->gfx.gfx_ring[i].sched.ready &&
> +			    !adev->gfx.gfx_ring[i].no_user_submission)
>   				++num_rings;
>   		ib_start_alignment = 32;
>   		ib_size_alignment = 32;
> @@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   	case AMDGPU_HW_IP_COMPUTE:
>   		type = AMD_IP_BLOCK_TYPE_GFX;
>   		for (i = 0; i < adev->gfx.num_compute_rings; i++)
> -			if (adev->gfx.compute_ring[i].sched.ready)
> +			if (adev->gfx.compute_ring[i].sched.ready &&
> +			    !adev->gfx.compute_ring[i].no_user_submission)
>   				++num_rings;
>   		ib_start_alignment = 32;
>   		ib_size_alignment = 32;
> @@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   	case AMDGPU_HW_IP_DMA:
>   		type = AMD_IP_BLOCK_TYPE_SDMA;
>   		for (i = 0; i < adev->sdma.num_instances; i++)
> -			if (adev->sdma.instance[i].ring.sched.ready)
> +			if (adev->sdma.instance[i].ring.sched.ready &&
> +			    !adev->gfx.gfx_ring[i].no_user_submission)
>   				++num_rings;
>   		ib_start_alignment = 256;
>   		ib_size_alignment = 4;
> @@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   			if (adev->uvd.harvest_config & (1 << i))
>   				continue;
>   
> -			if (adev->uvd.inst[i].ring.sched.ready)
> +			if (adev->uvd.inst[i].ring.sched.ready &&
> +			    !adev->uvd.inst[i].ring.no_user_submission)
>   				++num_rings;
>   		}
>   		ib_start_alignment = 256;
> @@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   	case AMDGPU_HW_IP_VCE:
>   		type = AMD_IP_BLOCK_TYPE_VCE;
>   		for (i = 0; i < adev->vce.num_rings; i++)
> -			if (adev->vce.ring[i].sched.ready)
> +			if (adev->vce.ring[i].sched.ready &&
> +			    !adev->vce.ring[i].no_user_submission)
>   				++num_rings;
>   		ib_start_alignment = 256;
>   		ib_size_alignment = 4;
> @@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   				continue;
>   
>   			for (j = 0; j < adev->uvd.num_enc_rings; j++)
> -				if (adev->uvd.inst[i].ring_enc[j].sched.ready)
> +				if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
> +				    !adev->uvd.inst[i].ring_enc[j].no_user_submission)
>   					++num_rings;
>   		}
>   		ib_start_alignment = 256;
> @@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   			if (adev->vcn.harvest_config & (1 << i))
>   				continue;
>   
> -			if (adev->vcn.inst[i].ring_dec.sched.ready)
> +			if (adev->vcn.inst[i].ring_dec.sched.ready &&
> +			    !adev->vcn.inst[i].ring_dec.no_user_submission)
>   				++num_rings;
>   		}
>   		ib_start_alignment = 256;
> @@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   				continue;
>   
>   			for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
> -				if (adev->vcn.inst[i].ring_enc[j].sched.ready)
> +				if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
> +				    !adev->vcn.inst[i].ring_enc[j].no_user_submission)
>   					++num_rings;
>   		}
>   		ib_start_alignment = 256;
> @@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   				continue;
>   
>   			for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
> -				if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
> +				if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
> +				    !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
>   					++num_rings;
>   		}
>   		ib_start_alignment = 256;
> @@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   		break;
>   	case AMDGPU_HW_IP_VPE:
>   		type = AMD_IP_BLOCK_TYPE_VPE;
> -		if (adev->vpe.ring.sched.ready)
> +		if (adev->vpe.ring.sched.ready &&
> +		    !adev->vpe.ring.no_user_submission)
>   			++num_rings;
>   		ib_start_alignment = 256;
>   		ib_size_alignment = 4;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> index b4fd1e17205e9..4a97afcb38b78 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> @@ -297,6 +297,7 @@ struct amdgpu_ring {
>   	struct dma_fence	*vmid_wait;
>   	bool			has_compute_vm_bug;
>   	bool			no_scheduler;
> +	bool			no_user_submission;
>   	int			hw_prio;
>   	unsigned 		num_hw_submission;
>   	atomic_t		*sched_score;
> @@ -310,7 +311,6 @@ struct amdgpu_ring {
>   	unsigned int    entry_index;
>   	/* store the cached rptr to restore after reset */
>   	uint64_t cached_rptr;
> -
>   };
>   
>   #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))

Re: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Alex Deucher]

On Thu, Mar 6, 2025 at 2:48 AM Khatri, Sunil <sukhatri@amd.com> wrote:
>
>
> On 3/6/2025 2:17 AM, Alex Deucher wrote:
> > This would be set by IPs which only accept submissions
> > from the kernel, not userspace, such as when kernel
> > queues are disabled. Don't expose the rings to userspace
> > and reject any submissions in the CS IOCTL.
> >
> > Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> > ---
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
> >   3 files changed, 25 insertions(+), 11 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > index 5df21529b3b13..5cc18034b75df 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > @@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,
> >       ring = amdgpu_job_ring(job);
> >       ib = &job->ibs[job->num_ibs++];
> >
> > +     /* submissions to kernel queus are disabled */
> > +     if (ring->no_user_submission)
> > +             return -EINVAL;
> > +
> Alex, dont we need to disable submission for other type of chunks, from
> function amdgpu_cs_pass2 ? chunk type of bo list and fences are
> submitted by users and we do want to handle that too.
> If that's not needed then lgtm.

That is the only chunk type which directly involves the kernel rings.

Alex


> >       /* MM engine doesn't support user fences */
> >       if (p->uf_bo && ring->funcs->no_user_fence)
> >               return -EINVAL;
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > @@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >       case AMDGPU_HW_IP_GFX:
> >               type = AMD_IP_BLOCK_TYPE_GFX;
> >               for (i = 0; i < adev->gfx.num_gfx_rings; i++)
> > -                     if (adev->gfx.gfx_ring[i].sched.ready)
> > +                     if (adev->gfx.gfx_ring[i].sched.ready &&
> > +                         !adev->gfx.gfx_ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 32;
> >               ib_size_alignment = 32;
> > @@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >       case AMDGPU_HW_IP_COMPUTE:
> >               type = AMD_IP_BLOCK_TYPE_GFX;
> >               for (i = 0; i < adev->gfx.num_compute_rings; i++)
> > -                     if (adev->gfx.compute_ring[i].sched.ready)
> > +                     if (adev->gfx.compute_ring[i].sched.ready &&
> > +                         !adev->gfx.compute_ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 32;
> >               ib_size_alignment = 32;
> > @@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >       case AMDGPU_HW_IP_DMA:
> >               type = AMD_IP_BLOCK_TYPE_SDMA;
> >               for (i = 0; i < adev->sdma.num_instances; i++)
> > -                     if (adev->sdma.instance[i].ring.sched.ready)
> > +                     if (adev->sdma.instance[i].ring.sched.ready &&
> > +                         !adev->gfx.gfx_ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 256;
> >               ib_size_alignment = 4;
> > @@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >                       if (adev->uvd.harvest_config & (1 << i))
> >                               continue;
> >
> > -                     if (adev->uvd.inst[i].ring.sched.ready)
> > +                     if (adev->uvd.inst[i].ring.sched.ready &&
> > +                         !adev->uvd.inst[i].ring.no_user_submission)
> >                               ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >       case AMDGPU_HW_IP_VCE:
> >               type = AMD_IP_BLOCK_TYPE_VCE;
> >               for (i = 0; i < adev->vce.num_rings; i++)
> > -                     if (adev->vce.ring[i].sched.ready)
> > +                     if (adev->vce.ring[i].sched.ready &&
> > +                         !adev->vce.ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 256;
> >               ib_size_alignment = 4;
> > @@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >                               continue;
> >
> >                       for (j = 0; j < adev->uvd.num_enc_rings; j++)
> > -                             if (adev->uvd.inst[i].ring_enc[j].sched.ready)
> > +                             if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
> > +                                 !adev->uvd.inst[i].ring_enc[j].no_user_submission)
> >                                       ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >                       if (adev->vcn.harvest_config & (1 << i))
> >                               continue;
> >
> > -                     if (adev->vcn.inst[i].ring_dec.sched.ready)
> > +                     if (adev->vcn.inst[i].ring_dec.sched.ready &&
> > +                         !adev->vcn.inst[i].ring_dec.no_user_submission)
> >                               ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >                               continue;
> >
> >                       for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
> > -                             if (adev->vcn.inst[i].ring_enc[j].sched.ready)
> > +                             if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
> > +                                 !adev->vcn.inst[i].ring_enc[j].no_user_submission)
> >                                       ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >                               continue;
> >
> >                       for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
> > -                             if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
> > +                             if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
> > +                                 !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
> >                                       ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >               break;
> >       case AMDGPU_HW_IP_VPE:
> >               type = AMD_IP_BLOCK_TYPE_VPE;
> > -             if (adev->vpe.ring.sched.ready)
> > +             if (adev->vpe.ring.sched.ready &&
> > +                 !adev->vpe.ring.no_user_submission)
> >                       ++num_rings;
> >               ib_start_alignment = 256;
> >               ib_size_alignment = 4;
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > index b4fd1e17205e9..4a97afcb38b78 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > @@ -297,6 +297,7 @@ struct amdgpu_ring {
> >       struct dma_fence        *vmid_wait;
> >       bool                    has_compute_vm_bug;
> >       bool                    no_scheduler;
> > +     bool                    no_user_submission;
> >       int                     hw_prio;
> >       unsigned                num_hw_submission;
> >       atomic_t                *sched_score;
> > @@ -310,7 +311,6 @@ struct amdgpu_ring {
> >       unsigned int    entry_index;
> >       /* store the cached rptr to restore after reset */
> >       uint64_t cached_rptr;
> > -
> >   };
> >
> >   #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))

[PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Alex Deucher]

This would be set by IPs which only accept submissions
from the kernel, not userspace, such as when kernel
queues are disabled. Don't expose the rings to userspace
and reject any submissions in the CS IOCTL.

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
 3 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 5df21529b3b13..5cc18034b75df 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,
 	ring = amdgpu_job_ring(job);
 	ib = &job->ibs[job->num_ibs++];
 
+	/* submissions to kernel queus are disabled */
+	if (ring->no_user_submission)
+		return -EINVAL;
+
 	/* MM engine doesn't support user fences */
 	if (p->uf_bo && ring->funcs->no_user_fence)
 		return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_GFX:
 		type = AMD_IP_BLOCK_TYPE_GFX;
 		for (i = 0; i < adev->gfx.num_gfx_rings; i++)
-			if (adev->gfx.gfx_ring[i].sched.ready)
+			if (adev->gfx.gfx_ring[i].sched.ready &&
+			    !adev->gfx.gfx_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 32;
 		ib_size_alignment = 32;
@@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_COMPUTE:
 		type = AMD_IP_BLOCK_TYPE_GFX;
 		for (i = 0; i < adev->gfx.num_compute_rings; i++)
-			if (adev->gfx.compute_ring[i].sched.ready)
+			if (adev->gfx.compute_ring[i].sched.ready &&
+			    !adev->gfx.compute_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 32;
 		ib_size_alignment = 32;
@@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_DMA:
 		type = AMD_IP_BLOCK_TYPE_SDMA;
 		for (i = 0; i < adev->sdma.num_instances; i++)
-			if (adev->sdma.instance[i].ring.sched.ready)
+			if (adev->sdma.instance[i].ring.sched.ready &&
+			    !adev->gfx.gfx_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
@@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 			if (adev->uvd.harvest_config & (1 << i))
 				continue;
 
-			if (adev->uvd.inst[i].ring.sched.ready)
+			if (adev->uvd.inst[i].ring.sched.ready &&
+			    !adev->uvd.inst[i].ring.no_user_submission)
 				++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_VCE:
 		type = AMD_IP_BLOCK_TYPE_VCE;
 		for (i = 0; i < adev->vce.num_rings; i++)
-			if (adev->vce.ring[i].sched.ready)
+			if (adev->vce.ring[i].sched.ready &&
+			    !adev->vce.ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
@@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->uvd.num_enc_rings; j++)
-				if (adev->uvd.inst[i].ring_enc[j].sched.ready)
+				if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
+				    !adev->uvd.inst[i].ring_enc[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 			if (adev->vcn.harvest_config & (1 << i))
 				continue;
 
-			if (adev->vcn.inst[i].ring_dec.sched.ready)
+			if (adev->vcn.inst[i].ring_dec.sched.ready &&
+			    !adev->vcn.inst[i].ring_dec.no_user_submission)
 				++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
-				if (adev->vcn.inst[i].ring_enc[j].sched.ready)
+				if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
+				    !adev->vcn.inst[i].ring_enc[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
-				if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
+				if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
+				    !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 		break;
 	case AMDGPU_HW_IP_VPE:
 		type = AMD_IP_BLOCK_TYPE_VPE;
-		if (adev->vpe.ring.sched.ready)
+		if (adev->vpe.ring.sched.ready &&
+		    !adev->vpe.ring.no_user_submission)
 			++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index b4fd1e17205e9..4a97afcb38b78 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -297,6 +297,7 @@ struct amdgpu_ring {
 	struct dma_fence	*vmid_wait;
 	bool			has_compute_vm_bug;
 	bool			no_scheduler;
+	bool			no_user_submission;
 	int			hw_prio;
 	unsigned 		num_hw_submission;
 	atomic_t		*sched_score;
@@ -310,7 +311,6 @@ struct amdgpu_ring {
 	unsigned int    entry_index;
 	/* store the cached rptr to restore after reset */
 	uint64_t cached_rptr;
-
 };
 
 #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
-- 
2.48.1

[PATCH V3 00/11] Add disable kernel queue support

[Alex Deucher]

To better evaluate user queues, add a module parameter
to disable kernel queues.  With this set kernel queues
are disabled and only user queues are available.  This
frees up hardware resources for use in user queues which
would otherwise be used by kernel queues and provides
a way to validate user queues without the presence
of kernel queues.

v2: use num_gfx_rings and num_compute_rings per
    Felix suggestion
v3: include num_gfx_rings fix in amdgpu_gfx.c

Alex Deucher (11):
  drm/amdgpu: add parameter to disable kernel queues
  drm/amdgpu: add ring flag for no user submissions
  drm/amdgpu/gfx: add generic handling for disable_kq
  drm/amdgpu/mes: centralize gfx_hqd mask management
  drm/amdgpu/mes: update hqd masks when disable_kq is set
  drm/amdgpu/mes: make more vmids available when disable_kq=1
  drm/amdgpu/gfx11: add support for disable_kq
  drm/amdgpu/gfx12: add support for disable_kq
  drm/amdgpu/sdma: add flag for tracking disable_kq
  drm/amdgpu/sdma6: add support for disable_kq
  drm/amdgpu/sdma7: add support for disable_kq

 drivers/gpu/drm/amd/amdgpu/amdgpu.h      |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c  |  9 +++
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c  |  8 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h  |  2 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++---
 drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c  | 26 ++++++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h |  1 +
 drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c   | 99 ++++++++++++++++--------
 drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c   | 96 ++++++++++++++---------
 drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c   |  2 +-
 drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c   |  2 +-
 drivers/gpu/drm/amd/amdgpu/mes_v11_0.c   | 16 +---
 drivers/gpu/drm/amd/amdgpu/mes_v12_0.c   | 15 +---
 drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c   |  4 +
 drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c   |  4 +
 17 files changed, 207 insertions(+), 114 deletions(-)

-- 
2.48.1

[PATCH 01/11] drm/amdgpu: add parameter to disable kernel queues

[Alex Deucher]

On chips that support user queues, setting this option
will disable kernel queues to be used to validate
user queues without kernel queues.

Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h     | 1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 9 +++++++++
 2 files changed, 10 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 87062c1adcdf7..45437a8f29d3b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -270,6 +270,7 @@ extern int amdgpu_user_partt_mode;
 extern int amdgpu_agp;
 
 extern int amdgpu_wbrf;
+extern int amdgpu_disable_kq;
 
 #define AMDGPU_VM_MAX_NUM_CTX			4096
 #define AMDGPU_SG_THRESHOLD			(256*1024*1024)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index b161daa900198..42a7619592ab9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -237,6 +237,7 @@ int amdgpu_agp = -1; /* auto */
 int amdgpu_wbrf = -1;
 int amdgpu_damage_clips = -1; /* auto */
 int amdgpu_umsch_mm_fwlog;
+int amdgpu_disable_kq = -1;
 
 DECLARE_DYNDBG_CLASSMAP(drm_debug_classes, DD_CLASS_TYPE_DISJOINT_BITS, 0,
 			"DRM_UT_CORE",
@@ -1083,6 +1084,14 @@ MODULE_PARM_DESC(wbrf,
 	"Enable Wifi RFI interference mitigation (0 = disabled, 1 = enabled, -1 = auto(default)");
 module_param_named(wbrf, amdgpu_wbrf, int, 0444);
 
+/**
+ * DOC: disable_kq (int)
+ * Disable kernel queues on systems that support user queues.
+ * (0 = kernel queues enabled, 1 = kernel queues disabled, -1 = auto (default setting))
+ */
+MODULE_PARM_DESC(disable_kq, "Disable kernel queues (-1 = auto (default), 0 = enable KQ, 1 = disable KQ)");
+module_param_named(disable_kq, amdgpu_disable_kq, int, 0444);
+
 /* These devices are not supported by amdgpu.
  * They are supported by the mach64, r128, radeon drivers
  */
-- 
2.48.1

[PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Alex Deucher]

This would be set by IPs which only accept submissions
from the kernel, not userspace, such as when kernel
queues are disabled. Don't expose the rings to userspace
and reject any submissions in the CS IOCTL.

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
 3 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 5df21529b3b13..5cc18034b75df 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,
 	ring = amdgpu_job_ring(job);
 	ib = &job->ibs[job->num_ibs++];
 
+	/* submissions to kernel queus are disabled */
+	if (ring->no_user_submission)
+		return -EINVAL;
+
 	/* MM engine doesn't support user fences */
 	if (p->uf_bo && ring->funcs->no_user_fence)
 		return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_GFX:
 		type = AMD_IP_BLOCK_TYPE_GFX;
 		for (i = 0; i < adev->gfx.num_gfx_rings; i++)
-			if (adev->gfx.gfx_ring[i].sched.ready)
+			if (adev->gfx.gfx_ring[i].sched.ready &&
+			    !adev->gfx.gfx_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 32;
 		ib_size_alignment = 32;
@@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_COMPUTE:
 		type = AMD_IP_BLOCK_TYPE_GFX;
 		for (i = 0; i < adev->gfx.num_compute_rings; i++)
-			if (adev->gfx.compute_ring[i].sched.ready)
+			if (adev->gfx.compute_ring[i].sched.ready &&
+			    !adev->gfx.compute_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 32;
 		ib_size_alignment = 32;
@@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_DMA:
 		type = AMD_IP_BLOCK_TYPE_SDMA;
 		for (i = 0; i < adev->sdma.num_instances; i++)
-			if (adev->sdma.instance[i].ring.sched.ready)
+			if (adev->sdma.instance[i].ring.sched.ready &&
+			    !adev->gfx.gfx_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
@@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 			if (adev->uvd.harvest_config & (1 << i))
 				continue;
 
-			if (adev->uvd.inst[i].ring.sched.ready)
+			if (adev->uvd.inst[i].ring.sched.ready &&
+			    !adev->uvd.inst[i].ring.no_user_submission)
 				++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_VCE:
 		type = AMD_IP_BLOCK_TYPE_VCE;
 		for (i = 0; i < adev->vce.num_rings; i++)
-			if (adev->vce.ring[i].sched.ready)
+			if (adev->vce.ring[i].sched.ready &&
+			    !adev->vce.ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
@@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->uvd.num_enc_rings; j++)
-				if (adev->uvd.inst[i].ring_enc[j].sched.ready)
+				if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
+				    !adev->uvd.inst[i].ring_enc[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 			if (adev->vcn.harvest_config & (1 << i))
 				continue;
 
-			if (adev->vcn.inst[i].ring_dec.sched.ready)
+			if (adev->vcn.inst[i].ring_dec.sched.ready &&
+			    !adev->vcn.inst[i].ring_dec.no_user_submission)
 				++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
-				if (adev->vcn.inst[i].ring_enc[j].sched.ready)
+				if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
+				    !adev->vcn.inst[i].ring_enc[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
-				if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
+				if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
+				    !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 		break;
 	case AMDGPU_HW_IP_VPE:
 		type = AMD_IP_BLOCK_TYPE_VPE;
-		if (adev->vpe.ring.sched.ready)
+		if (adev->vpe.ring.sched.ready &&
+		    !adev->vpe.ring.no_user_submission)
 			++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index b4fd1e17205e9..4a97afcb38b78 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -297,6 +297,7 @@ struct amdgpu_ring {
 	struct dma_fence	*vmid_wait;
 	bool			has_compute_vm_bug;
 	bool			no_scheduler;
+	bool			no_user_submission;
 	int			hw_prio;
 	unsigned 		num_hw_submission;
 	atomic_t		*sched_score;
@@ -310,7 +311,6 @@ struct amdgpu_ring {
 	unsigned int    entry_index;
 	/* store the cached rptr to restore after reset */
 	uint64_t cached_rptr;
-
 };
 
 #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
-- 
2.48.1

[PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq

[Alex Deucher]

Add proper checks for disable_kq functionality in
gfx helper functions.  Add special logic for families
that require the clear state setup.

v2: use ring count as per Felix suggestion
v3: fix num_gfx_rings handling in amdgpu_gfx_graphics_queue_acquire()

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 8 ++++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 2 ++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index a194bf3347cbc..2c78185a33218 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -258,8 +258,9 @@ void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev)
 	}
 
 	/* update the number of active graphics rings */
-	adev->gfx.num_gfx_rings =
-		bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
+	if (adev->gfx.num_gfx_rings)
+		adev->gfx.num_gfx_rings =
+			bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
 }
 
 static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev,
@@ -1544,6 +1545,9 @@ static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device *dev,
 	if (adev->in_suspend && !adev->in_runpm)
 		return -EPERM;
 
+	if (adev->gfx.disable_kq)
+		return -ENOTSUPP;
+
 	ret = kstrtol(buf, 0, &value);
 
 	if (ret)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index ddf4533614bac..8fa68a4ac34f1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -483,6 +483,8 @@ struct amdgpu_gfx {
 
 	atomic_t			total_submission_cnt;
 	struct delayed_work		idle_work;
+
+	bool				disable_kq;
 };
 
 struct amdgpu_gfx_ras_reg_entry {
-- 
2.48.1

[PATCH 04/11] drm/amdgpu/mes: centralize gfx_hqd mask management

[Alex Deucher]

Move it to amdgpu_mes to align with the compute and
sdma hqd masks. No functional change.

v2: rebase on new changes

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 22 ++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/mes_v11_0.c  | 16 +++-------------
 drivers/gpu/drm/amd/amdgpu/mes_v12_0.c  | 15 +++------------
 3 files changed, 28 insertions(+), 25 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index ca076306adba4..5913c5ba85ed0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -144,6 +144,28 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 	adev->mes.vmid_mask_mmhub = 0xffffff00;
 	adev->mes.vmid_mask_gfxhub = 0xffffff00;
 
+	for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++) {
+		/* use only 1st ME pipe */
+		if (i >= adev->gfx.me.num_pipe_per_me)
+			continue;
+		if (amdgpu_ip_version(adev, GC_HWIP, 0) >=
+		    IP_VERSION(12, 0, 0))
+			/*
+			 * GFX V12 has only one GFX pipe, but 8 queues in it.
+			 * GFX pipe 0 queue 0 is being used by Kernel queue.
+			 * Set GFX pipe 0 queue 1-7 for MES scheduling
+			 * mask = 1111 1110b
+			 */
+			adev->mes.gfx_hqd_mask[i] = 0xFE;
+		else
+			/*
+			 * GFX pipe 0 queue 0 is being used by Kernel queue.
+			 * Set GFX pipe 0 queue 1 for MES scheduling
+			 * mask = 10b
+			 */
+			adev->mes.gfx_hqd_mask[i] = 0x2;
+	}
+
 	for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) {
 		/* use only 1st MEC pipes */
 		if (i >= adev->gfx.mec.num_pipe_per_mec)
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
index a569d09a1a748..39b45d8b5f049 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
@@ -669,18 +669,6 @@ static int mes_v11_0_misc_op(struct amdgpu_mes *mes,
 			offsetof(union MESAPI__MISC, api_status));
 }
 
-static void mes_v11_0_set_gfx_hqd_mask(union MESAPI_SET_HW_RESOURCES *pkt)
-{
-	/*
-	 * GFX pipe 0 queue 0 is being used by Kernel queue.
-	 * Set GFX pipe 0 queue 1 for MES scheduling
-	 * mask = 10b
-	 * GFX pipe 1 can't be used for MES due to HW limitation.
-	 */
-	pkt->gfx_hqd_mask[0] = 0x2;
-	pkt->gfx_hqd_mask[1] = 0;
-}
-
 static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
 {
 	int i;
@@ -705,7 +693,9 @@ static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
 		mes_set_hw_res_pkt.compute_hqd_mask[i] =
 			mes->compute_hqd_mask[i];
 
-	mes_v11_0_set_gfx_hqd_mask(&mes_set_hw_res_pkt);
+	for (i = 0; i < MAX_GFX_PIPES; i++)
+		mes_set_hw_res_pkt.gfx_hqd_mask[i] =
+			mes->gfx_hqd_mask[i];
 
 	for (i = 0; i < MAX_SDMA_PIPES; i++)
 		mes_set_hw_res_pkt.sdma_hqd_mask[i] = mes->sdma_hqd_mask[i];
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
index 96336652d14c5..519f054bec60d 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
@@ -694,17 +694,6 @@ static int mes_v12_0_set_hw_resources_1(struct amdgpu_mes *mes, int pipe)
 			offsetof(union MESAPI_SET_HW_RESOURCES_1, api_status));
 }
 
-static void mes_v12_0_set_gfx_hqd_mask(union MESAPI_SET_HW_RESOURCES *pkt)
-{
-	/*
-	 * GFX V12 has only one GFX pipe, but 8 queues in it.
-	 * GFX pipe 0 queue 0 is being used by Kernel queue.
-	 * Set GFX pipe 0 queue 1-7 for MES scheduling
-	 * mask = 1111 1110b
-	 */
-	pkt->gfx_hqd_mask[0] = 0xFE;
-}
-
 static int mes_v12_0_set_hw_resources(struct amdgpu_mes *mes, int pipe)
 {
 	int i;
@@ -727,7 +716,9 @@ static int mes_v12_0_set_hw_resources(struct amdgpu_mes *mes, int pipe)
 			mes_set_hw_res_pkt.compute_hqd_mask[i] =
 				mes->compute_hqd_mask[i];
 
-		mes_v12_0_set_gfx_hqd_mask(&mes_set_hw_res_pkt);
+		for (i = 0; i < MAX_GFX_PIPES; i++)
+			mes_set_hw_res_pkt.gfx_hqd_mask[i] =
+				mes->gfx_hqd_mask[i];
 
 		for (i = 0; i < MAX_SDMA_PIPES; i++)
 			mes_set_hw_res_pkt.sdma_hqd_mask[i] =
-- 
2.48.1

[PATCH 05/11] drm/amdgpu/mes: update hqd masks when disable_kq is set

[Alex Deucher]

Make all resources available to user queues.

Suggested-by: Sunil Khatri <sunil.khatri@amd.com>
Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index 5913c5ba85ed0..e585e8690edf0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -156,21 +156,21 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 			 * Set GFX pipe 0 queue 1-7 for MES scheduling
 			 * mask = 1111 1110b
 			 */
-			adev->mes.gfx_hqd_mask[i] = 0xFE;
+			adev->mes.gfx_hqd_mask[i] = adev->gfx.disable_kq ? 0xFF : 0xFE;
 		else
 			/*
 			 * GFX pipe 0 queue 0 is being used by Kernel queue.
 			 * Set GFX pipe 0 queue 1 for MES scheduling
 			 * mask = 10b
 			 */
-			adev->mes.gfx_hqd_mask[i] = 0x2;
+			adev->mes.gfx_hqd_mask[i] = adev->gfx.disable_kq ? 0x3 : 0x2;
 	}
 
 	for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) {
 		/* use only 1st MEC pipes */
 		if (i >= adev->gfx.mec.num_pipe_per_mec)
 			continue;
-		adev->mes.compute_hqd_mask[i] = 0xc;
+		adev->mes.compute_hqd_mask[i] = adev->gfx.disable_kq ? 0xF : 0xC;
 	}
 
 	for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) {
-- 
2.48.1

[PATCH 06/11] drm/amdgpu/mes: make more vmids available when disable_kq=1

[Alex Deucher]

If we don't have kernel queues, the vmids can be used by
the MES for user queues.

Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 2 +-
 drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c  | 2 +-
 drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index e585e8690edf0..d7cdd2895889a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -142,7 +142,7 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 
 	adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;
 	adev->mes.vmid_mask_mmhub = 0xffffff00;
-	adev->mes.vmid_mask_gfxhub = 0xffffff00;
+	adev->mes.vmid_mask_gfxhub = adev->gfx.disable_kq ? 0xfffffffe : 0xffffff00;
 
 	for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++) {
 		/* use only 1st ME pipe */
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
index 95d894a231fcf..19a5f196829f3 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
@@ -900,7 +900,7 @@ static int gmc_v10_0_sw_init(struct amdgpu_ip_block *ip_block)
 	 * amdgpu graphics/compute will use VMIDs 1-7
 	 * amdkfd will use VMIDs 8-15
 	 */
-	adev->vm_manager.first_kfd_vmid = 8;
+	adev->vm_manager.first_kfd_vmid = adev->gfx.disable_kq ? 1 : 8;
 
 	amdgpu_vm_manager_init(adev);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
index ea7c32d8380ba..598324e736092 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
@@ -837,7 +837,7 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 	 * amdgpu graphics/compute will use VMIDs 1-7
 	 * amdkfd will use VMIDs 8-15
 	 */
-	adev->vm_manager.first_kfd_vmid = 8;
+	adev->vm_manager.first_kfd_vmid = adev->gfx.disable_kq ? 1 : 8;
 
 	amdgpu_vm_manager_init(adev);
 
-- 
2.48.1

[PATCH 07/11] drm/amdgpu/gfx11: add support for disable_kq

[Alex Deucher]

Plumb in support for disabling kernel queues in
GFX11.  We have to bring up a GFX queue briefly in
order to initialize the clear state.  After that
we can disable it.

v2: use ring counts per Felix' suggestion

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 99 +++++++++++++++++---------
 1 file changed, 65 insertions(+), 34 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
index 95eefd9a40d28..b20624f8cbbbd 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
@@ -1145,6 +1145,10 @@ static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
 
 	ring->ring_obj = NULL;
 	ring->use_doorbell = true;
+	if (adev->gfx.disable_kq) {
+		ring->no_scheduler = true;
+		ring->no_user_submission = true;
+	}
 
 	if (!ring_id)
 		ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
@@ -1577,7 +1581,7 @@ static void gfx_v11_0_alloc_ip_dump(struct amdgpu_device *adev)
 
 static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
 {
-	int i, j, k, r, ring_id = 0;
+	int i, j, k, r, ring_id;
 	int xcc_id = 0;
 	struct amdgpu_device *adev = ip_block->adev;
 
@@ -1710,37 +1714,42 @@ static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
 		return r;
 	}
 
-	/* set up the gfx ring */
-	for (i = 0; i < adev->gfx.me.num_me; i++) {
-		for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
-			for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
-				if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
-					continue;
-
-				r = gfx_v11_0_gfx_ring_init(adev, ring_id,
-							    i, k, j);
-				if (r)
-					return r;
-				ring_id++;
+	if (adev->gfx.num_gfx_rings) {
+		ring_id = 0;
+		/* set up the gfx ring */
+		for (i = 0; i < adev->gfx.me.num_me; i++) {
+			for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
+				for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
+					if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
+						continue;
+
+					r = gfx_v11_0_gfx_ring_init(adev, ring_id,
+								    i, k, j);
+					if (r)
+						return r;
+					ring_id++;
+				}
 			}
 		}
 	}
 
-	ring_id = 0;
-	/* set up the compute queues - allocate horizontally across pipes */
-	for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
-		for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
-			for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
-				if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
-								     k, j))
-					continue;
+	if (adev->gfx.num_compute_rings) {
+		ring_id = 0;
+		/* set up the compute queues - allocate horizontally across pipes */
+		for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
+			for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
+				for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
+					if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
+									     k, j))
+						continue;
 
-				r = gfx_v11_0_compute_ring_init(adev, ring_id,
-								i, k, j);
-				if (r)
-					return r;
+					r = gfx_v11_0_compute_ring_init(adev, ring_id,
+									i, k, j);
+					if (r)
+						return r;
 
-				ring_id++;
+					ring_id++;
+				}
 			}
 		}
 	}
@@ -4578,11 +4587,22 @@ static int gfx_v11_0_cp_resume(struct amdgpu_device *adev)
 			return r;
 	}
 
-	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
-		ring = &adev->gfx.gfx_ring[i];
-		r = amdgpu_ring_test_helper(ring);
-		if (r)
-			return r;
+	if (adev->gfx.disable_kq) {
+		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
+			ring = &adev->gfx.gfx_ring[i];
+			/* we don't want to set ring->ready */
+			r = amdgpu_ring_test_ring(ring);
+			if (r)
+				return r;
+		}
+		amdgpu_gfx_disable_kgq(adev, 0);
+	} else {
+		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
+			ring = &adev->gfx.gfx_ring[i];
+			r = amdgpu_ring_test_helper(ring);
+			if (r)
+				return r;
+		}
 	}
 
 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
@@ -5128,11 +5148,22 @@ static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
+	if (amdgpu_disable_kq == 1)
+		adev->gfx.disable_kq = true;
+
 	adev->gfx.funcs = &gfx_v11_0_gfx_funcs;
 
-	adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS;
-	adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
-					  AMDGPU_MAX_COMPUTE_RINGS);
+	if (adev->gfx.disable_kq) {
+		/* We need one GFX ring temporarily to set up
+		 * the clear state.
+		 */
+		adev->gfx.num_gfx_rings = 1;
+		adev->gfx.num_compute_rings = 0;
+	} else {
+		adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS;
+		adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
+						  AMDGPU_MAX_COMPUTE_RINGS);
+	}
 
 	gfx_v11_0_set_kiq_pm4_funcs(adev);
 	gfx_v11_0_set_ring_funcs(adev);
-- 
2.48.1

[PATCH 08/11] drm/amdgpu/gfx12: add support for disable_kq

[Alex Deucher]

Plumb in support for disabling kernel queues.

v2: use ring counts per Felix' suggestion

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c | 96 ++++++++++++++++----------
 1 file changed, 58 insertions(+), 38 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
index 34cf187e72d9f..23ee4651cbffb 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
@@ -1421,11 +1421,13 @@ static int gfx_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 		break;
 	}
 
-	/* recalculate compute rings to use based on hardware configuration */
-	num_compute_rings = (adev->gfx.mec.num_pipe_per_mec *
-			     adev->gfx.mec.num_queue_per_pipe) / 2;
-	adev->gfx.num_compute_rings = min(adev->gfx.num_compute_rings,
-					  num_compute_rings);
+	if (adev->gfx.num_compute_rings) {
+		/* recalculate compute rings to use based on hardware configuration */
+		num_compute_rings = (adev->gfx.mec.num_pipe_per_mec *
+				     adev->gfx.mec.num_queue_per_pipe) / 2;
+		adev->gfx.num_compute_rings = min(adev->gfx.num_compute_rings,
+						  num_compute_rings);
+	}
 
 	/* EOP Event */
 	r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
@@ -1471,37 +1473,41 @@ static int gfx_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 		return r;
 	}
 
-	/* set up the gfx ring */
-	for (i = 0; i < adev->gfx.me.num_me; i++) {
-		for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
-			for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
-				if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
-					continue;
-
-				r = gfx_v12_0_gfx_ring_init(adev, ring_id,
-							    i, k, j);
-				if (r)
-					return r;
-				ring_id++;
+	if (adev->gfx.num_gfx_rings) {
+		/* set up the gfx ring */
+		for (i = 0; i < adev->gfx.me.num_me; i++) {
+			for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
+				for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
+					if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
+						continue;
+
+					r = gfx_v12_0_gfx_ring_init(adev, ring_id,
+								    i, k, j);
+					if (r)
+						return r;
+					ring_id++;
+				}
 			}
 		}
 	}
 
-	ring_id = 0;
-	/* set up the compute queues - allocate horizontally across pipes */
-	for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
-		for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
-			for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
-				if (!amdgpu_gfx_is_mec_queue_enabled(adev,
-								0, i, k, j))
-					continue;
+	if (adev->gfx.num_compute_rings) {
+		ring_id = 0;
+		/* set up the compute queues - allocate horizontally across pipes */
+		for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
+			for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
+				for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
+					if (!amdgpu_gfx_is_mec_queue_enabled(adev,
+									     0, i, k, j))
+						continue;
 
-				r = gfx_v12_0_compute_ring_init(adev, ring_id,
-								i, k, j);
-				if (r)
-					return r;
+					r = gfx_v12_0_compute_ring_init(adev, ring_id,
+									i, k, j);
+					if (r)
+						return r;
 
-				ring_id++;
+					ring_id++;
+				}
 			}
 		}
 	}
@@ -3495,12 +3501,18 @@ static int gfx_v12_0_cp_resume(struct amdgpu_device *adev)
 	if (r)
 		return r;
 
-	if (!amdgpu_async_gfx_ring) {
-		r = gfx_v12_0_cp_gfx_resume(adev);
-		if (r)
-			return r;
+	if (adev->gfx.num_gfx_rings) {
+		if (!amdgpu_async_gfx_ring) {
+			r = gfx_v12_0_cp_gfx_resume(adev);
+			if (r)
+				return r;
+		} else {
+			r = gfx_v12_0_cp_async_gfx_ring_resume(adev);
+			if (r)
+				return r;
+		}
 	} else {
-		r = gfx_v12_0_cp_async_gfx_ring_resume(adev);
+		r = gfx_v12_0_cp_gfx_start(adev);
 		if (r)
 			return r;
 	}
@@ -3809,11 +3821,19 @@ static int gfx_v12_0_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
+	if (amdgpu_disable_kq == 1)
+		adev->gfx.disable_kq = true;
+
 	adev->gfx.funcs = &gfx_v12_0_gfx_funcs;
 
-	adev->gfx.num_gfx_rings = GFX12_NUM_GFX_RINGS;
-	adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
-					  AMDGPU_MAX_COMPUTE_RINGS);
+	if (adev->gfx.disable_kq) {
+		adev->gfx.num_gfx_rings = 0;
+		adev->gfx.num_compute_rings = 0;
+	} else {
+		adev->gfx.num_gfx_rings = GFX12_NUM_GFX_RINGS;
+		adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
+						  AMDGPU_MAX_COMPUTE_RINGS);
+	}
 
 	gfx_v12_0_set_kiq_pm4_funcs(adev);
 	gfx_v12_0_set_ring_funcs(adev);
-- 
2.48.1

[PATCH 09/11] drm/amdgpu/sdma: add flag for tracking disable_kq

[Alex Deucher]

For SDMA, we still need kernel queues for paging so
they need to be initialized, but we no not want to
accept submissions from userspace when disable_kq
is set.

Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
index 9651693200655..edc856e10337a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
@@ -129,6 +129,7 @@ struct amdgpu_sdma {
 	/* track guilty state of GFX and PAGE queues */
 	bool gfx_guilty;
 	bool page_guilty;
+	bool			no_user_submission;
 };
 
 /*
-- 
2.48.1

[PATCH 10/11] drm/amdgpu/sdma6: add support for disable_kq

[Alex Deucher]

When the parameter is set, disable user submissions
to kernel queues.

Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
index 3aa4fec4d9e4a..bcc72737f8084 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
@@ -1304,6 +1304,9 @@ static int sdma_v6_0_early_init(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	int r;
 
+	if (amdgpu_disable_kq == 1)
+		adev->sdma.no_user_submission = true;
+
 	r = amdgpu_sdma_init_microcode(adev, 0, true);
 	if (r)
 		return r;
@@ -1338,6 +1341,7 @@ static int sdma_v6_0_sw_init(struct amdgpu_ip_block *ip_block)
 		ring->ring_obj = NULL;
 		ring->use_doorbell = true;
 		ring->me = i;
+		ring->no_user_submission = adev->sdma.no_user_submission;
 
 		DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
 				ring->use_doorbell?"true":"false");
-- 
2.48.1

[PATCH 11/11] drm/amdgpu/sdma7: add support for disable_kq

[Alex Deucher]

When the parameter is set, disable user submissions
to kernel queues.

Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
index 92a79296708ae..40d45f738c0a8 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
@@ -1316,6 +1316,9 @@ static int sdma_v7_0_early_init(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	int r;
 
+	if (amdgpu_disable_kq == 1)
+		adev->sdma.no_user_submission = true;
+
 	r = amdgpu_sdma_init_microcode(adev, 0, true);
 	if (r) {
 		DRM_ERROR("Failed to init sdma firmware!\n");
@@ -1351,6 +1354,7 @@ static int sdma_v7_0_sw_init(struct amdgpu_ip_block *ip_block)
 		ring->ring_obj = NULL;
 		ring->use_doorbell = true;
 		ring->me = i;
+		ring->no_user_submission = adev->sdma.no_user_submission;
 
 		DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
 				ring->use_doorbell?"true":"false");
-- 
2.48.1

Re: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Khatri, Sunil]

Reviewed-by: Sunil Khatri<sunil.khatri@amd.com>

On 3/7/2025 8:45 PM, Alex Deucher wrote:
> This would be set by IPs which only accept submissions
> from the kernel, not userspace, such as when kernel
> queues are disabled. Don't expose the rings to userspace
> and reject any submissions in the CS IOCTL.
>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
>   3 files changed, 25 insertions(+), 11 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> index 5df21529b3b13..5cc18034b75df 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> @@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,
>   	ring = amdgpu_job_ring(job);
>   	ib = &job->ibs[job->num_ibs++];
>   
> +	/* submissions to kernel queus are disabled */
> +	if (ring->no_user_submission)
> +		return -EINVAL;
> +
>   	/* MM engine doesn't support user fences */
>   	if (p->uf_bo && ring->funcs->no_user_fence)
>   		return -EINVAL;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   	case AMDGPU_HW_IP_GFX:
>   		type = AMD_IP_BLOCK_TYPE_GFX;
>   		for (i = 0; i < adev->gfx.num_gfx_rings; i++)
> -			if (adev->gfx.gfx_ring[i].sched.ready)
> +			if (adev->gfx.gfx_ring[i].sched.ready &&
> +			    !adev->gfx.gfx_ring[i].no_user_submission)
>   				++num_rings;
>   		ib_start_alignment = 32;
>   		ib_size_alignment = 32;
> @@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   	case AMDGPU_HW_IP_COMPUTE:
>   		type = AMD_IP_BLOCK_TYPE_GFX;
>   		for (i = 0; i < adev->gfx.num_compute_rings; i++)
> -			if (adev->gfx.compute_ring[i].sched.ready)
> +			if (adev->gfx.compute_ring[i].sched.ready &&
> +			    !adev->gfx.compute_ring[i].no_user_submission)
>   				++num_rings;
>   		ib_start_alignment = 32;
>   		ib_size_alignment = 32;
> @@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   	case AMDGPU_HW_IP_DMA:
>   		type = AMD_IP_BLOCK_TYPE_SDMA;
>   		for (i = 0; i < adev->sdma.num_instances; i++)
> -			if (adev->sdma.instance[i].ring.sched.ready)
> +			if (adev->sdma.instance[i].ring.sched.ready &&
> +			    !adev->gfx.gfx_ring[i].no_user_submission)
>   				++num_rings;
>   		ib_start_alignment = 256;
>   		ib_size_alignment = 4;
> @@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   			if (adev->uvd.harvest_config & (1 << i))
>   				continue;
>   
> -			if (adev->uvd.inst[i].ring.sched.ready)
> +			if (adev->uvd.inst[i].ring.sched.ready &&
> +			    !adev->uvd.inst[i].ring.no_user_submission)
>   				++num_rings;
>   		}
>   		ib_start_alignment = 256;
> @@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   	case AMDGPU_HW_IP_VCE:
>   		type = AMD_IP_BLOCK_TYPE_VCE;
>   		for (i = 0; i < adev->vce.num_rings; i++)
> -			if (adev->vce.ring[i].sched.ready)
> +			if (adev->vce.ring[i].sched.ready &&
> +			    !adev->vce.ring[i].no_user_submission)
>   				++num_rings;
>   		ib_start_alignment = 256;
>   		ib_size_alignment = 4;
> @@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   				continue;
>   
>   			for (j = 0; j < adev->uvd.num_enc_rings; j++)
> -				if (adev->uvd.inst[i].ring_enc[j].sched.ready)
> +				if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
> +				    !adev->uvd.inst[i].ring_enc[j].no_user_submission)
>   					++num_rings;
>   		}
>   		ib_start_alignment = 256;
> @@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   			if (adev->vcn.harvest_config & (1 << i))
>   				continue;
>   
> -			if (adev->vcn.inst[i].ring_dec.sched.ready)
> +			if (adev->vcn.inst[i].ring_dec.sched.ready &&
> +			    !adev->vcn.inst[i].ring_dec.no_user_submission)
>   				++num_rings;
>   		}
>   		ib_start_alignment = 256;
> @@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   				continue;
>   
>   			for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
> -				if (adev->vcn.inst[i].ring_enc[j].sched.ready)
> +				if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
> +				    !adev->vcn.inst[i].ring_enc[j].no_user_submission)
>   					++num_rings;
>   		}
>   		ib_start_alignment = 256;
> @@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   				continue;
>   
>   			for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
> -				if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
> +				if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
> +				    !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
>   					++num_rings;
>   		}
>   		ib_start_alignment = 256;
> @@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>   		break;
>   	case AMDGPU_HW_IP_VPE:
>   		type = AMD_IP_BLOCK_TYPE_VPE;
> -		if (adev->vpe.ring.sched.ready)
> +		if (adev->vpe.ring.sched.ready &&
> +		    !adev->vpe.ring.no_user_submission)
>   			++num_rings;
>   		ib_start_alignment = 256;
>   		ib_size_alignment = 4;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> index b4fd1e17205e9..4a97afcb38b78 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> @@ -297,6 +297,7 @@ struct amdgpu_ring {
>   	struct dma_fence	*vmid_wait;
>   	bool			has_compute_vm_bug;
>   	bool			no_scheduler;
> +	bool			no_user_submission;
>   	int			hw_prio;
>   	unsigned 		num_hw_submission;
>   	atomic_t		*sched_score;
> @@ -310,7 +311,6 @@ struct amdgpu_ring {
>   	unsigned int    entry_index;
>   	/* store the cached rptr to restore after reset */
>   	uint64_t cached_rptr;
> -
>   };
>   
>   #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))

Re: [PATCH 04/11] drm/amdgpu/mes: centralize gfx_hqd mask management

[Khatri, Sunil]

Reviewed-by: Sunil Khatri<sunil.khatri@amd.com>

On 3/7/2025 8:45 PM, Alex Deucher wrote:
> Move it to amdgpu_mes to align with the compute and
> sdma hqd masks. No functional change.
>
> v2: rebase on new changes
>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 22 ++++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/mes_v11_0.c  | 16 +++-------------
>   drivers/gpu/drm/amd/amdgpu/mes_v12_0.c  | 15 +++------------
>   3 files changed, 28 insertions(+), 25 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> index ca076306adba4..5913c5ba85ed0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> @@ -144,6 +144,28 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
>   	adev->mes.vmid_mask_mmhub = 0xffffff00;
>   	adev->mes.vmid_mask_gfxhub = 0xffffff00;
>   
> +	for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++) {
> +		/* use only 1st ME pipe */
> +		if (i >= adev->gfx.me.num_pipe_per_me)
> +			continue;
> +		if (amdgpu_ip_version(adev, GC_HWIP, 0) >=
> +		    IP_VERSION(12, 0, 0))
> +			/*
> +			 * GFX V12 has only one GFX pipe, but 8 queues in it.
> +			 * GFX pipe 0 queue 0 is being used by Kernel queue.
> +			 * Set GFX pipe 0 queue 1-7 for MES scheduling
> +			 * mask = 1111 1110b
> +			 */
> +			adev->mes.gfx_hqd_mask[i] = 0xFE;
> +		else
> +			/*
> +			 * GFX pipe 0 queue 0 is being used by Kernel queue.
> +			 * Set GFX pipe 0 queue 1 for MES scheduling
> +			 * mask = 10b
> +			 */
> +			adev->mes.gfx_hqd_mask[i] = 0x2;
> +	}
> +
>   	for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) {
>   		/* use only 1st MEC pipes */
>   		if (i >= adev->gfx.mec.num_pipe_per_mec)
> diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
> index a569d09a1a748..39b45d8b5f049 100644
> --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
> @@ -669,18 +669,6 @@ static int mes_v11_0_misc_op(struct amdgpu_mes *mes,
>   			offsetof(union MESAPI__MISC, api_status));
>   }
>   
> -static void mes_v11_0_set_gfx_hqd_mask(union MESAPI_SET_HW_RESOURCES *pkt)
> -{
> -	/*
> -	 * GFX pipe 0 queue 0 is being used by Kernel queue.
> -	 * Set GFX pipe 0 queue 1 for MES scheduling
> -	 * mask = 10b
> -	 * GFX pipe 1 can't be used for MES due to HW limitation.
> -	 */
> -	pkt->gfx_hqd_mask[0] = 0x2;
> -	pkt->gfx_hqd_mask[1] = 0;
> -}
> -
>   static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
>   {
>   	int i;
> @@ -705,7 +693,9 @@ static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
>   		mes_set_hw_res_pkt.compute_hqd_mask[i] =
>   			mes->compute_hqd_mask[i];
>   
> -	mes_v11_0_set_gfx_hqd_mask(&mes_set_hw_res_pkt);
> +	for (i = 0; i < MAX_GFX_PIPES; i++)
> +		mes_set_hw_res_pkt.gfx_hqd_mask[i] =
> +			mes->gfx_hqd_mask[i];
>   
>   	for (i = 0; i < MAX_SDMA_PIPES; i++)
>   		mes_set_hw_res_pkt.sdma_hqd_mask[i] = mes->sdma_hqd_mask[i];
> diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
> index 96336652d14c5..519f054bec60d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
> @@ -694,17 +694,6 @@ static int mes_v12_0_set_hw_resources_1(struct amdgpu_mes *mes, int pipe)
>   			offsetof(union MESAPI_SET_HW_RESOURCES_1, api_status));
>   }
>   
> -static void mes_v12_0_set_gfx_hqd_mask(union MESAPI_SET_HW_RESOURCES *pkt)
> -{
> -	/*
> -	 * GFX V12 has only one GFX pipe, but 8 queues in it.
> -	 * GFX pipe 0 queue 0 is being used by Kernel queue.
> -	 * Set GFX pipe 0 queue 1-7 for MES scheduling
> -	 * mask = 1111 1110b
> -	 */
> -	pkt->gfx_hqd_mask[0] = 0xFE;
> -}
> -
>   static int mes_v12_0_set_hw_resources(struct amdgpu_mes *mes, int pipe)
>   {
>   	int i;
> @@ -727,7 +716,9 @@ static int mes_v12_0_set_hw_resources(struct amdgpu_mes *mes, int pipe)
>   			mes_set_hw_res_pkt.compute_hqd_mask[i] =
>   				mes->compute_hqd_mask[i];
>   
> -		mes_v12_0_set_gfx_hqd_mask(&mes_set_hw_res_pkt);
> +		for (i = 0; i < MAX_GFX_PIPES; i++)
> +			mes_set_hw_res_pkt.gfx_hqd_mask[i] =
> +				mes->gfx_hqd_mask[i];
>   
>   		for (i = 0; i < MAX_SDMA_PIPES; i++)
>   			mes_set_hw_res_pkt.sdma_hqd_mask[i] =

RE: [PATCH 01/11] drm/amdgpu: add parameter to disable kernel queues

[Liang, Prike]

[Public]

Reviewed-by: Prike Liang <Prike.Liang@amd.com>

Regards,
      Prike

> -----Original Message-----
> From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Alex
> Deucher
> Sent: Friday, March 7, 2025 11:16 PM
> To: amd-gfx@lists.freedesktop.org
> Cc: Deucher, Alexander <Alexander.Deucher@amd.com>; Khatri, Sunil
> <Sunil.Khatri@amd.com>
> Subject: [PATCH 01/11] drm/amdgpu: add parameter to disable kernel queues
>
> On chips that support user queues, setting this option will disable kernel queues to
> be used to validate user queues without kernel queues.
>
> Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu.h     | 1 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 9 +++++++++
>  2 files changed, 10 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index 87062c1adcdf7..45437a8f29d3b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -270,6 +270,7 @@ extern int amdgpu_user_partt_mode;  extern int
> amdgpu_agp;
>
>  extern int amdgpu_wbrf;
> +extern int amdgpu_disable_kq;
>
>  #define AMDGPU_VM_MAX_NUM_CTX                        4096
>  #define AMDGPU_SG_THRESHOLD                  (256*1024*1024)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index b161daa900198..42a7619592ab9 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -237,6 +237,7 @@ int amdgpu_agp = -1; /* auto */  int amdgpu_wbrf = -1;
> int amdgpu_damage_clips = -1; /* auto */  int amdgpu_umsch_mm_fwlog;
> +int amdgpu_disable_kq = -1;
>
>  DECLARE_DYNDBG_CLASSMAP(drm_debug_classes,
> DD_CLASS_TYPE_DISJOINT_BITS, 0,
>                       "DRM_UT_CORE",
> @@ -1083,6 +1084,14 @@ MODULE_PARM_DESC(wbrf,
>       "Enable Wifi RFI interference mitigation (0 = disabled, 1 = enabled, -1 =
> auto(default)");  module_param_named(wbrf, amdgpu_wbrf, int, 0444);
>
> +/**
> + * DOC: disable_kq (int)
> + * Disable kernel queues on systems that support user queues.
> + * (0 = kernel queues enabled, 1 = kernel queues disabled, -1 = auto
> +(default setting))  */ MODULE_PARM_DESC(disable_kq, "Disable kernel
> +queues (-1 = auto (default), 0 = enable KQ, 1 = disable KQ)");
> +module_param_named(disable_kq, amdgpu_disable_kq, int, 0444);
> +
>  /* These devices are not supported by amdgpu.
>   * They are supported by the mach64, r128, radeon drivers
>   */
> --
> 2.48.1

RE: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Liang, Prike]

[Public]

> From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Alex
> Deucher
> Sent: Friday, March 7, 2025 2:46 AM
> To: amd-gfx@lists.freedesktop.org
> Cc: Deucher, Alexander <Alexander.Deucher@amd.com>
> Subject: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions
>
> This would be set by IPs which only accept submissions from the kernel, not
> userspace, such as when kernel queues are disabled. Don't expose the rings to
> userspace and reject any submissions in the CS IOCTL.
>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
> drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
>  3 files changed, 25 insertions(+), 11 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> index 5df21529b3b13..5cc18034b75df 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> @@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser
> *p,
>       ring = amdgpu_job_ring(job);
>       ib = &job->ibs[job->num_ibs++];
>
> +     /* submissions to kernel queus are disabled */
> +     if (ring->no_user_submission)
> +             return -EINVAL;
> +

Do we need reject the kernel queue submission at the beginning placement of amdgpu_cs_ioctl()?

Thanks,
Prike
>       /* MM engine doesn't support user fences */
>       if (p->uf_bo && ring->funcs->no_user_fence)
>               return -EINVAL;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>       case AMDGPU_HW_IP_GFX:
>               type = AMD_IP_BLOCK_TYPE_GFX;
>               for (i = 0; i < adev->gfx.num_gfx_rings; i++)
> -                     if (adev->gfx.gfx_ring[i].sched.ready)
> +                     if (adev->gfx.gfx_ring[i].sched.ready &&
> +                         !adev->gfx.gfx_ring[i].no_user_submission)
>                               ++num_rings;
>               ib_start_alignment = 32;
>               ib_size_alignment = 32;
> @@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>       case AMDGPU_HW_IP_COMPUTE:
>               type = AMD_IP_BLOCK_TYPE_GFX;
>               for (i = 0; i < adev->gfx.num_compute_rings; i++)
> -                     if (adev->gfx.compute_ring[i].sched.ready)
> +                     if (adev->gfx.compute_ring[i].sched.ready &&
> +                         !adev->gfx.compute_ring[i].no_user_submission)
>                               ++num_rings;
>               ib_start_alignment = 32;
>               ib_size_alignment = 32;
> @@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>       case AMDGPU_HW_IP_DMA:
>               type = AMD_IP_BLOCK_TYPE_SDMA;
>               for (i = 0; i < adev->sdma.num_instances; i++)
> -                     if (adev->sdma.instance[i].ring.sched.ready)
> +                     if (adev->sdma.instance[i].ring.sched.ready &&
> +                         !adev->gfx.gfx_ring[i].no_user_submission)
>                               ++num_rings;
>               ib_start_alignment = 256;
>               ib_size_alignment = 4;
> @@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>                       if (adev->uvd.harvest_config & (1 << i))
>                               continue;
>
> -                     if (adev->uvd.inst[i].ring.sched.ready)
> +                     if (adev->uvd.inst[i].ring.sched.ready &&
> +                         !adev->uvd.inst[i].ring.no_user_submission)
>                               ++num_rings;
>               }
>               ib_start_alignment = 256;
> @@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>       case AMDGPU_HW_IP_VCE:
>               type = AMD_IP_BLOCK_TYPE_VCE;
>               for (i = 0; i < adev->vce.num_rings; i++)
> -                     if (adev->vce.ring[i].sched.ready)
> +                     if (adev->vce.ring[i].sched.ready &&
> +                         !adev->vce.ring[i].no_user_submission)
>                               ++num_rings;
>               ib_start_alignment = 256;
>               ib_size_alignment = 4;
> @@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>                               continue;
>
>                       for (j = 0; j < adev->uvd.num_enc_rings; j++)
> -                             if (adev->uvd.inst[i].ring_enc[j].sched.ready)
> +                             if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
> +                                 !adev->uvd.inst[i].ring_enc[j].no_user_submission)
>                                       ++num_rings;
>               }
>               ib_start_alignment = 256;
> @@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>                       if (adev->vcn.harvest_config & (1 << i))
>                               continue;
>
> -                     if (adev->vcn.inst[i].ring_dec.sched.ready)
> +                     if (adev->vcn.inst[i].ring_dec.sched.ready &&
> +                         !adev->vcn.inst[i].ring_dec.no_user_submission)
>                               ++num_rings;
>               }
>               ib_start_alignment = 256;
> @@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>                               continue;
>
>                       for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
> -                             if (adev->vcn.inst[i].ring_enc[j].sched.ready)
> +                             if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
> +                                 !adev->vcn.inst[i].ring_enc[j].no_user_submission)
>                                       ++num_rings;
>               }
>               ib_start_alignment = 256;
> @@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>                               continue;
>
>                       for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
> -                             if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
> +                             if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
> +                                 !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
>                                       ++num_rings;
>               }
>               ib_start_alignment = 256;
> @@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>               break;
>       case AMDGPU_HW_IP_VPE:
>               type = AMD_IP_BLOCK_TYPE_VPE;
> -             if (adev->vpe.ring.sched.ready)
> +             if (adev->vpe.ring.sched.ready &&
> +                 !adev->vpe.ring.no_user_submission)
>                       ++num_rings;
>               ib_start_alignment = 256;
>               ib_size_alignment = 4;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> index b4fd1e17205e9..4a97afcb38b78 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> @@ -297,6 +297,7 @@ struct amdgpu_ring {
>       struct dma_fence        *vmid_wait;
>       bool                    has_compute_vm_bug;
>       bool                    no_scheduler;
> +     bool                    no_user_submission;
>       int                     hw_prio;
>       unsigned                num_hw_submission;
>       atomic_t                *sched_score;
> @@ -310,7 +311,6 @@ struct amdgpu_ring {
>       unsigned int    entry_index;
>       /* store the cached rptr to restore after reset */
>       uint64_t cached_rptr;
> -
>  };
>
>  #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
> --
> 2.48.1

RE: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq

[Liang, Prike]

[Public]

> From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Alex
> Deucher
> Sent: Friday, March 7, 2025 11:16 PM
> To: amd-gfx@lists.freedesktop.org
> Cc: Deucher, Alexander <Alexander.Deucher@amd.com>
> Subject: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq
>
> Add proper checks for disable_kq functionality in gfx helper functions.  Add special
> logic for families that require the clear state setup.
>
> v2: use ring count as per Felix suggestion
> v3: fix num_gfx_rings handling in amdgpu_gfx_graphics_queue_acquire()
>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 8 ++++++--
> drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 2 ++
>  2 files changed, 8 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> index a194bf3347cbc..2c78185a33218 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> @@ -258,8 +258,9 @@ void amdgpu_gfx_graphics_queue_acquire(struct
> amdgpu_device *adev)
>       }
>
>       /* update the number of active graphics rings */
> -     adev->gfx.num_gfx_rings =
> -             bitmap_weight(adev->gfx.me.queue_bitmap,
> AMDGPU_MAX_GFX_QUEUES);
> +     if (adev->gfx.num_gfx_rings)
> +             adev->gfx.num_gfx_rings =
> +                     bitmap_weight(adev->gfx.me.queue_bitmap,
> AMDGPU_MAX_GFX_QUEUES);
>  }
>
>  static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev, @@ -1544,6
> +1545,9 @@ static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device
> *dev,
>       if (adev->in_suspend && !adev->in_runpm)
>               return -EPERM;
>
> +     if (adev->gfx.disable_kq)
> +             return -ENOTSUPP;
> +
Maybe here need to disable the flag enable_cleaner_shader as well?

Thanks,
Prike
>       ret = kstrtol(buf, 0, &value);
>
>       if (ret)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> index ddf4533614bac..8fa68a4ac34f1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> @@ -483,6 +483,8 @@ struct amdgpu_gfx {
>
>       atomic_t                        total_submission_cnt;
>       struct delayed_work             idle_work;
> +
> +     bool                            disable_kq;
>  };
>
>  struct amdgpu_gfx_ras_reg_entry {
> --
> 2.48.1

Re: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Alex Deucher]

On Tue, Mar 11, 2025 at 9:13 AM Liang, Prike <Prike.Liang@amd.com> wrote:
>
> [Public]
>
> > From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Alex
> > Deucher
> > Sent: Friday, March 7, 2025 2:46 AM
> > To: amd-gfx@lists.freedesktop.org
> > Cc: Deucher, Alexander <Alexander.Deucher@amd.com>
> > Subject: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions
> >
> > This would be set by IPs which only accept submissions from the kernel, not
> > userspace, such as when kernel queues are disabled. Don't expose the rings to
> > userspace and reject any submissions in the CS IOCTL.
> >
> > Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> > ---
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
> > drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
> >  3 files changed, 25 insertions(+), 11 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > index 5df21529b3b13..5cc18034b75df 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > @@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser
> > *p,
> >       ring = amdgpu_job_ring(job);
> >       ib = &job->ibs[job->num_ibs++];
> >
> > +     /* submissions to kernel queus are disabled */
> > +     if (ring->no_user_submission)
> > +             return -EINVAL;
> > +
>
> Do we need reject the kernel queue submission at the beginning placement of amdgpu_cs_ioctl()?

I think we only need to reject it if it's targeted at a kernel ring
and this is the first place we can determine that.

Alex

>
> Thanks,
> Prike
> >       /* MM engine doesn't support user fences */
> >       if (p->uf_bo && ring->funcs->no_user_fence)
> >               return -EINVAL;
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > @@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >       case AMDGPU_HW_IP_GFX:
> >               type = AMD_IP_BLOCK_TYPE_GFX;
> >               for (i = 0; i < adev->gfx.num_gfx_rings; i++)
> > -                     if (adev->gfx.gfx_ring[i].sched.ready)
> > +                     if (adev->gfx.gfx_ring[i].sched.ready &&
> > +                         !adev->gfx.gfx_ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 32;
> >               ib_size_alignment = 32;
> > @@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >       case AMDGPU_HW_IP_COMPUTE:
> >               type = AMD_IP_BLOCK_TYPE_GFX;
> >               for (i = 0; i < adev->gfx.num_compute_rings; i++)
> > -                     if (adev->gfx.compute_ring[i].sched.ready)
> > +                     if (adev->gfx.compute_ring[i].sched.ready &&
> > +                         !adev->gfx.compute_ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 32;
> >               ib_size_alignment = 32;
> > @@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >       case AMDGPU_HW_IP_DMA:
> >               type = AMD_IP_BLOCK_TYPE_SDMA;
> >               for (i = 0; i < adev->sdma.num_instances; i++)
> > -                     if (adev->sdma.instance[i].ring.sched.ready)
> > +                     if (adev->sdma.instance[i].ring.sched.ready &&
> > +                         !adev->gfx.gfx_ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 256;
> >               ib_size_alignment = 4;
> > @@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >                       if (adev->uvd.harvest_config & (1 << i))
> >                               continue;
> >
> > -                     if (adev->uvd.inst[i].ring.sched.ready)
> > +                     if (adev->uvd.inst[i].ring.sched.ready &&
> > +                         !adev->uvd.inst[i].ring.no_user_submission)
> >                               ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >       case AMDGPU_HW_IP_VCE:
> >               type = AMD_IP_BLOCK_TYPE_VCE;
> >               for (i = 0; i < adev->vce.num_rings; i++)
> > -                     if (adev->vce.ring[i].sched.ready)
> > +                     if (adev->vce.ring[i].sched.ready &&
> > +                         !adev->vce.ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 256;
> >               ib_size_alignment = 4;
> > @@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >                               continue;
> >
> >                       for (j = 0; j < adev->uvd.num_enc_rings; j++)
> > -                             if (adev->uvd.inst[i].ring_enc[j].sched.ready)
> > +                             if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
> > +                                 !adev->uvd.inst[i].ring_enc[j].no_user_submission)
> >                                       ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >                       if (adev->vcn.harvest_config & (1 << i))
> >                               continue;
> >
> > -                     if (adev->vcn.inst[i].ring_dec.sched.ready)
> > +                     if (adev->vcn.inst[i].ring_dec.sched.ready &&
> > +                         !adev->vcn.inst[i].ring_dec.no_user_submission)
> >                               ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >                               continue;
> >
> >                       for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
> > -                             if (adev->vcn.inst[i].ring_enc[j].sched.ready)
> > +                             if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
> > +                                 !adev->vcn.inst[i].ring_enc[j].no_user_submission)
> >                                       ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >                               continue;
> >
> >                       for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
> > -                             if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
> > +                             if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
> > +                                 !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
> >                                       ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >               break;
> >       case AMDGPU_HW_IP_VPE:
> >               type = AMD_IP_BLOCK_TYPE_VPE;
> > -             if (adev->vpe.ring.sched.ready)
> > +             if (adev->vpe.ring.sched.ready &&
> > +                 !adev->vpe.ring.no_user_submission)
> >                       ++num_rings;
> >               ib_start_alignment = 256;
> >               ib_size_alignment = 4;
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > index b4fd1e17205e9..4a97afcb38b78 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > @@ -297,6 +297,7 @@ struct amdgpu_ring {
> >       struct dma_fence        *vmid_wait;
> >       bool                    has_compute_vm_bug;
> >       bool                    no_scheduler;
> > +     bool                    no_user_submission;
> >       int                     hw_prio;
> >       unsigned                num_hw_submission;
> >       atomic_t                *sched_score;
> > @@ -310,7 +311,6 @@ struct amdgpu_ring {
> >       unsigned int    entry_index;
> >       /* store the cached rptr to restore after reset */
> >       uint64_t cached_rptr;
> > -
> >  };
> >
> >  #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
> > --
> > 2.48.1
>

Re: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq

[Alex Deucher]

On Tue, Mar 11, 2025 at 10:18 AM Liang, Prike <Prike.Liang@amd.com> wrote:
>
> [Public]
>
> > From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Alex
> > Deucher
> > Sent: Friday, March 7, 2025 11:16 PM
> > To: amd-gfx@lists.freedesktop.org
> > Cc: Deucher, Alexander <Alexander.Deucher@amd.com>
> > Subject: [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq
> >
> > Add proper checks for disable_kq functionality in gfx helper functions.  Add special
> > logic for families that require the clear state setup.
> >
> > v2: use ring count as per Felix suggestion
> > v3: fix num_gfx_rings handling in amdgpu_gfx_graphics_queue_acquire()
> >
> > Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> > ---
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 8 ++++++--
> > drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 2 ++
> >  2 files changed, 8 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> > index a194bf3347cbc..2c78185a33218 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> > @@ -258,8 +258,9 @@ void amdgpu_gfx_graphics_queue_acquire(struct
> > amdgpu_device *adev)
> >       }
> >
> >       /* update the number of active graphics rings */
> > -     adev->gfx.num_gfx_rings =
> > -             bitmap_weight(adev->gfx.me.queue_bitmap,
> > AMDGPU_MAX_GFX_QUEUES);
> > +     if (adev->gfx.num_gfx_rings)
> > +             adev->gfx.num_gfx_rings =
> > +                     bitmap_weight(adev->gfx.me.queue_bitmap,
> > AMDGPU_MAX_GFX_QUEUES);
> >  }
> >
> >  static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev, @@ -1544,6
> > +1545,9 @@ static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device
> > *dev,
> >       if (adev->in_suspend && !adev->in_runpm)
> >               return -EPERM;
> >
> > +     if (adev->gfx.disable_kq)
> > +             return -ENOTSUPP;
> > +
> Maybe here need to disable the flag enable_cleaner_shader as well?

We still need it, but the MES runs it when switching queues rather
than the driver in the userq case.

Alex

>
> Thanks,
> Prike
> >       ret = kstrtol(buf, 0, &value);
> >
> >       if (ret)
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> > index ddf4533614bac..8fa68a4ac34f1 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> > @@ -483,6 +483,8 @@ struct amdgpu_gfx {
> >
> >       atomic_t                        total_submission_cnt;
> >       struct delayed_work             idle_work;
> > +
> > +     bool                            disable_kq;
> >  };
> >
> >  struct amdgpu_gfx_ras_reg_entry {
> > --
> > 2.48.1
>

[PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Alex Deucher]

This would be set by IPs which only accept submissions
from the kernel, not userspace, such as when kernel
queues are disabled. Don't expose the rings to userspace
and reject any submissions in the CS IOCTL.

Reviewed-by: Sunil Khatri<sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
 3 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 5df21529b3b13..5cc18034b75df 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,
 	ring = amdgpu_job_ring(job);
 	ib = &job->ibs[job->num_ibs++];
 
+	/* submissions to kernel queus are disabled */
+	if (ring->no_user_submission)
+		return -EINVAL;
+
 	/* MM engine doesn't support user fences */
 	if (p->uf_bo && ring->funcs->no_user_fence)
 		return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_GFX:
 		type = AMD_IP_BLOCK_TYPE_GFX;
 		for (i = 0; i < adev->gfx.num_gfx_rings; i++)
-			if (adev->gfx.gfx_ring[i].sched.ready)
+			if (adev->gfx.gfx_ring[i].sched.ready &&
+			    !adev->gfx.gfx_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 32;
 		ib_size_alignment = 32;
@@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_COMPUTE:
 		type = AMD_IP_BLOCK_TYPE_GFX;
 		for (i = 0; i < adev->gfx.num_compute_rings; i++)
-			if (adev->gfx.compute_ring[i].sched.ready)
+			if (adev->gfx.compute_ring[i].sched.ready &&
+			    !adev->gfx.compute_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 32;
 		ib_size_alignment = 32;
@@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_DMA:
 		type = AMD_IP_BLOCK_TYPE_SDMA;
 		for (i = 0; i < adev->sdma.num_instances; i++)
-			if (adev->sdma.instance[i].ring.sched.ready)
+			if (adev->sdma.instance[i].ring.sched.ready &&
+			    !adev->gfx.gfx_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
@@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 			if (adev->uvd.harvest_config & (1 << i))
 				continue;
 
-			if (adev->uvd.inst[i].ring.sched.ready)
+			if (adev->uvd.inst[i].ring.sched.ready &&
+			    !adev->uvd.inst[i].ring.no_user_submission)
 				++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_VCE:
 		type = AMD_IP_BLOCK_TYPE_VCE;
 		for (i = 0; i < adev->vce.num_rings; i++)
-			if (adev->vce.ring[i].sched.ready)
+			if (adev->vce.ring[i].sched.ready &&
+			    !adev->vce.ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
@@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->uvd.num_enc_rings; j++)
-				if (adev->uvd.inst[i].ring_enc[j].sched.ready)
+				if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
+				    !adev->uvd.inst[i].ring_enc[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 			if (adev->vcn.harvest_config & (1 << i))
 				continue;
 
-			if (adev->vcn.inst[i].ring_dec.sched.ready)
+			if (adev->vcn.inst[i].ring_dec.sched.ready &&
+			    !adev->vcn.inst[i].ring_dec.no_user_submission)
 				++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
-				if (adev->vcn.inst[i].ring_enc[j].sched.ready)
+				if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
+				    !adev->vcn.inst[i].ring_enc[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
-				if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
+				if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
+				    !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 		break;
 	case AMDGPU_HW_IP_VPE:
 		type = AMD_IP_BLOCK_TYPE_VPE;
-		if (adev->vpe.ring.sched.ready)
+		if (adev->vpe.ring.sched.ready &&
+		    !adev->vpe.ring.no_user_submission)
 			++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index b4fd1e17205e9..4a97afcb38b78 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -297,6 +297,7 @@ struct amdgpu_ring {
 	struct dma_fence	*vmid_wait;
 	bool			has_compute_vm_bug;
 	bool			no_scheduler;
+	bool			no_user_submission;
 	int			hw_prio;
 	unsigned 		num_hw_submission;
 	atomic_t		*sched_score;
@@ -310,7 +311,6 @@ struct amdgpu_ring {
 	unsigned int    entry_index;
 	/* store the cached rptr to restore after reset */
 	uint64_t cached_rptr;
-
 };
 
 #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
-- 
2.48.1

[PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Alex Deucher]

This would be set by IPs which only accept submissions
from the kernel, not userspace, such as when kernel
queues are disabled. Don't expose the rings to userspace
and reject any submissions in the CS IOCTL.

Reviewed-by: Sunil Khatri<sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
 3 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 5df21529b3b13..5cc18034b75df 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,
 	ring = amdgpu_job_ring(job);
 	ib = &job->ibs[job->num_ibs++];
 
+	/* submissions to kernel queus are disabled */
+	if (ring->no_user_submission)
+		return -EINVAL;
+
 	/* MM engine doesn't support user fences */
 	if (p->uf_bo && ring->funcs->no_user_fence)
 		return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_GFX:
 		type = AMD_IP_BLOCK_TYPE_GFX;
 		for (i = 0; i < adev->gfx.num_gfx_rings; i++)
-			if (adev->gfx.gfx_ring[i].sched.ready)
+			if (adev->gfx.gfx_ring[i].sched.ready &&
+			    !adev->gfx.gfx_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 32;
 		ib_size_alignment = 32;
@@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_COMPUTE:
 		type = AMD_IP_BLOCK_TYPE_GFX;
 		for (i = 0; i < adev->gfx.num_compute_rings; i++)
-			if (adev->gfx.compute_ring[i].sched.ready)
+			if (adev->gfx.compute_ring[i].sched.ready &&
+			    !adev->gfx.compute_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 32;
 		ib_size_alignment = 32;
@@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_DMA:
 		type = AMD_IP_BLOCK_TYPE_SDMA;
 		for (i = 0; i < adev->sdma.num_instances; i++)
-			if (adev->sdma.instance[i].ring.sched.ready)
+			if (adev->sdma.instance[i].ring.sched.ready &&
+			    !adev->gfx.gfx_ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
@@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 			if (adev->uvd.harvest_config & (1 << i))
 				continue;
 
-			if (adev->uvd.inst[i].ring.sched.ready)
+			if (adev->uvd.inst[i].ring.sched.ready &&
+			    !adev->uvd.inst[i].ring.no_user_submission)
 				++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_VCE:
 		type = AMD_IP_BLOCK_TYPE_VCE;
 		for (i = 0; i < adev->vce.num_rings; i++)
-			if (adev->vce.ring[i].sched.ready)
+			if (adev->vce.ring[i].sched.ready &&
+			    !adev->vce.ring[i].no_user_submission)
 				++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
@@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->uvd.num_enc_rings; j++)
-				if (adev->uvd.inst[i].ring_enc[j].sched.ready)
+				if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
+				    !adev->uvd.inst[i].ring_enc[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 			if (adev->vcn.harvest_config & (1 << i))
 				continue;
 
-			if (adev->vcn.inst[i].ring_dec.sched.ready)
+			if (adev->vcn.inst[i].ring_dec.sched.ready &&
+			    !adev->vcn.inst[i].ring_dec.no_user_submission)
 				++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
-				if (adev->vcn.inst[i].ring_enc[j].sched.ready)
+				if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
+				    !adev->vcn.inst[i].ring_enc[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 				continue;
 
 			for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
-				if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
+				if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
+				    !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
 					++num_rings;
 		}
 		ib_start_alignment = 256;
@@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 		break;
 	case AMDGPU_HW_IP_VPE:
 		type = AMD_IP_BLOCK_TYPE_VPE;
-		if (adev->vpe.ring.sched.ready)
+		if (adev->vpe.ring.sched.ready &&
+		    !adev->vpe.ring.no_user_submission)
 			++num_rings;
 		ib_start_alignment = 256;
 		ib_size_alignment = 4;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index b4fd1e17205e9..4a97afcb38b78 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -297,6 +297,7 @@ struct amdgpu_ring {
 	struct dma_fence	*vmid_wait;
 	bool			has_compute_vm_bug;
 	bool			no_scheduler;
+	bool			no_user_submission;
 	int			hw_prio;
 	unsigned 		num_hw_submission;
 	atomic_t		*sched_score;
@@ -310,7 +311,6 @@ struct amdgpu_ring {
 	unsigned int    entry_index;
 	/* store the cached rptr to restore after reset */
 	uint64_t cached_rptr;
-
 };
 
 #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
-- 
2.48.1

Re: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Rodrigo Siqueira]

On 03/13, Alex Deucher wrote:
> This would be set by IPs which only accept submissions
> from the kernel, not userspace, such as when kernel
> queues are disabled. Don't expose the rings to userspace
> and reject any submissions in the CS IOCTL.

Just out of curiosity, is CS == Command Submission?

> 
> Reviewed-by: Sunil Khatri<sunil.khatri@amd.com>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
>  drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
>  3 files changed, 25 insertions(+), 11 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> index 5df21529b3b13..5cc18034b75df 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> @@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,
>  	ring = amdgpu_job_ring(job);
>  	ib = &job->ibs[job->num_ibs++];
>  
> +	/* submissions to kernel queus are disabled */

/queus/queues/

> +	if (ring->no_user_submission)
> +		return -EINVAL;

Since this will be set for ASICs that don't accept userspace
submissions, maybe -ENOTSUPP would be more accurate?

Thanks

> +
>  	/* MM engine doesn't support user fences */
>  	if (p->uf_bo && ring->funcs->no_user_fence)
>  		return -EINVAL;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>  	case AMDGPU_HW_IP_GFX:
>  		type = AMD_IP_BLOCK_TYPE_GFX;
>  		for (i = 0; i < adev->gfx.num_gfx_rings; i++)
> -			if (adev->gfx.gfx_ring[i].sched.ready)
> +			if (adev->gfx.gfx_ring[i].sched.ready &&
> +			    !adev->gfx.gfx_ring[i].no_user_submission)
>  				++num_rings;
>  		ib_start_alignment = 32;
>  		ib_size_alignment = 32;
> @@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>  	case AMDGPU_HW_IP_COMPUTE:
>  		type = AMD_IP_BLOCK_TYPE_GFX;
>  		for (i = 0; i < adev->gfx.num_compute_rings; i++)
> -			if (adev->gfx.compute_ring[i].sched.ready)
> +			if (adev->gfx.compute_ring[i].sched.ready &&
> +			    !adev->gfx.compute_ring[i].no_user_submission)
>  				++num_rings;
>  		ib_start_alignment = 32;
>  		ib_size_alignment = 32;
> @@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>  	case AMDGPU_HW_IP_DMA:
>  		type = AMD_IP_BLOCK_TYPE_SDMA;
>  		for (i = 0; i < adev->sdma.num_instances; i++)
> -			if (adev->sdma.instance[i].ring.sched.ready)
> +			if (adev->sdma.instance[i].ring.sched.ready &&
> +			    !adev->gfx.gfx_ring[i].no_user_submission)
>  				++num_rings;
>  		ib_start_alignment = 256;
>  		ib_size_alignment = 4;
> @@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>  			if (adev->uvd.harvest_config & (1 << i))
>  				continue;
>  
> -			if (adev->uvd.inst[i].ring.sched.ready)
> +			if (adev->uvd.inst[i].ring.sched.ready &&
> +			    !adev->uvd.inst[i].ring.no_user_submission)
>  				++num_rings;
>  		}
>  		ib_start_alignment = 256;
> @@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>  	case AMDGPU_HW_IP_VCE:
>  		type = AMD_IP_BLOCK_TYPE_VCE;
>  		for (i = 0; i < adev->vce.num_rings; i++)
> -			if (adev->vce.ring[i].sched.ready)
> +			if (adev->vce.ring[i].sched.ready &&
> +			    !adev->vce.ring[i].no_user_submission)
>  				++num_rings;
>  		ib_start_alignment = 256;
>  		ib_size_alignment = 4;
> @@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>  				continue;
>  
>  			for (j = 0; j < adev->uvd.num_enc_rings; j++)
> -				if (adev->uvd.inst[i].ring_enc[j].sched.ready)
> +				if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
> +				    !adev->uvd.inst[i].ring_enc[j].no_user_submission)
>  					++num_rings;
>  		}
>  		ib_start_alignment = 256;
> @@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>  			if (adev->vcn.harvest_config & (1 << i))
>  				continue;
>  
> -			if (adev->vcn.inst[i].ring_dec.sched.ready)
> +			if (adev->vcn.inst[i].ring_dec.sched.ready &&
> +			    !adev->vcn.inst[i].ring_dec.no_user_submission)
>  				++num_rings;
>  		}
>  		ib_start_alignment = 256;
> @@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>  				continue;
>  
>  			for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
> -				if (adev->vcn.inst[i].ring_enc[j].sched.ready)
> +				if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
> +				    !adev->vcn.inst[i].ring_enc[j].no_user_submission)
>  					++num_rings;
>  		}
>  		ib_start_alignment = 256;
> @@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>  				continue;
>  
>  			for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
> -				if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
> +				if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
> +				    !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
>  					++num_rings;
>  		}
>  		ib_start_alignment = 256;
> @@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
>  		break;
>  	case AMDGPU_HW_IP_VPE:
>  		type = AMD_IP_BLOCK_TYPE_VPE;
> -		if (adev->vpe.ring.sched.ready)
> +		if (adev->vpe.ring.sched.ready &&
> +		    !adev->vpe.ring.no_user_submission)
>  			++num_rings;
>  		ib_start_alignment = 256;
>  		ib_size_alignment = 4;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> index b4fd1e17205e9..4a97afcb38b78 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> @@ -297,6 +297,7 @@ struct amdgpu_ring {
>  	struct dma_fence	*vmid_wait;
>  	bool			has_compute_vm_bug;
>  	bool			no_scheduler;
> +	bool			no_user_submission;
>  	int			hw_prio;
>  	unsigned 		num_hw_submission;
>  	atomic_t		*sched_score;
> @@ -310,7 +311,6 @@ struct amdgpu_ring {
>  	unsigned int    entry_index;
>  	/* store the cached rptr to restore after reset */
>  	uint64_t cached_rptr;
> -
>  };
>  
>  #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
> -- 
> 2.48.1
> 

-- 
Rodrigo Siqueira

Re: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Alex Deucher]

On Thu, Mar 13, 2025 at 5:53 PM Rodrigo Siqueira <siqueira@igalia.com> wrote:
>
> On 03/13, Alex Deucher wrote:
> > This would be set by IPs which only accept submissions
> > from the kernel, not userspace, such as when kernel
> > queues are disabled. Don't expose the rings to userspace
> > and reject any submissions in the CS IOCTL.
>
> Just out of curiosity, is CS == Command Submission?

Yes.

>
> >
> > Reviewed-by: Sunil Khatri<sunil.khatri@amd.com>
> > Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> > ---
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
> >  3 files changed, 25 insertions(+), 11 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > index 5df21529b3b13..5cc18034b75df 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > @@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,
> >       ring = amdgpu_job_ring(job);
> >       ib = &job->ibs[job->num_ibs++];
> >
> > +     /* submissions to kernel queus are disabled */
>
> /queus/queues/

will fix.

>
> > +     if (ring->no_user_submission)
> > +             return -EINVAL;
>
> Since this will be set for ASICs that don't accept userspace
> submissions, maybe -ENOTSUPP would be more accurate?

I could go either way on that.  When kernel queues are disabled, CS
submissions to a kernel queue are not supported, but it's also an
invalid parameter to the IOCTL.  I don't have a strong opinion.  Maybe
-ENOTSUPP would differentiate it from just general invalid parameters.

Alex

>
> Thanks
>
> > +
> >       /* MM engine doesn't support user fences */
> >       if (p->uf_bo && ring->funcs->no_user_fence)
> >               return -EINVAL;
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > @@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >       case AMDGPU_HW_IP_GFX:
> >               type = AMD_IP_BLOCK_TYPE_GFX;
> >               for (i = 0; i < adev->gfx.num_gfx_rings; i++)
> > -                     if (adev->gfx.gfx_ring[i].sched.ready)
> > +                     if (adev->gfx.gfx_ring[i].sched.ready &&
> > +                         !adev->gfx.gfx_ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 32;
> >               ib_size_alignment = 32;
> > @@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >       case AMDGPU_HW_IP_COMPUTE:
> >               type = AMD_IP_BLOCK_TYPE_GFX;
> >               for (i = 0; i < adev->gfx.num_compute_rings; i++)
> > -                     if (adev->gfx.compute_ring[i].sched.ready)
> > +                     if (adev->gfx.compute_ring[i].sched.ready &&
> > +                         !adev->gfx.compute_ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 32;
> >               ib_size_alignment = 32;
> > @@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >       case AMDGPU_HW_IP_DMA:
> >               type = AMD_IP_BLOCK_TYPE_SDMA;
> >               for (i = 0; i < adev->sdma.num_instances; i++)
> > -                     if (adev->sdma.instance[i].ring.sched.ready)
> > +                     if (adev->sdma.instance[i].ring.sched.ready &&
> > +                         !adev->gfx.gfx_ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 256;
> >               ib_size_alignment = 4;
> > @@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >                       if (adev->uvd.harvest_config & (1 << i))
> >                               continue;
> >
> > -                     if (adev->uvd.inst[i].ring.sched.ready)
> > +                     if (adev->uvd.inst[i].ring.sched.ready &&
> > +                         !adev->uvd.inst[i].ring.no_user_submission)
> >                               ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >       case AMDGPU_HW_IP_VCE:
> >               type = AMD_IP_BLOCK_TYPE_VCE;
> >               for (i = 0; i < adev->vce.num_rings; i++)
> > -                     if (adev->vce.ring[i].sched.ready)
> > +                     if (adev->vce.ring[i].sched.ready &&
> > +                         !adev->vce.ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 256;
> >               ib_size_alignment = 4;
> > @@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >                               continue;
> >
> >                       for (j = 0; j < adev->uvd.num_enc_rings; j++)
> > -                             if (adev->uvd.inst[i].ring_enc[j].sched.ready)
> > +                             if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
> > +                                 !adev->uvd.inst[i].ring_enc[j].no_user_submission)
> >                                       ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >                       if (adev->vcn.harvest_config & (1 << i))
> >                               continue;
> >
> > -                     if (adev->vcn.inst[i].ring_dec.sched.ready)
> > +                     if (adev->vcn.inst[i].ring_dec.sched.ready &&
> > +                         !adev->vcn.inst[i].ring_dec.no_user_submission)
> >                               ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >                               continue;
> >
> >                       for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
> > -                             if (adev->vcn.inst[i].ring_enc[j].sched.ready)
> > +                             if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
> > +                                 !adev->vcn.inst[i].ring_enc[j].no_user_submission)
> >                                       ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >                               continue;
> >
> >                       for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
> > -                             if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
> > +                             if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
> > +                                 !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
> >                                       ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
> >               break;
> >       case AMDGPU_HW_IP_VPE:
> >               type = AMD_IP_BLOCK_TYPE_VPE;
> > -             if (adev->vpe.ring.sched.ready)
> > +             if (adev->vpe.ring.sched.ready &&
> > +                 !adev->vpe.ring.no_user_submission)
> >                       ++num_rings;
> >               ib_start_alignment = 256;
> >               ib_size_alignment = 4;
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > index b4fd1e17205e9..4a97afcb38b78 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > @@ -297,6 +297,7 @@ struct amdgpu_ring {
> >       struct dma_fence        *vmid_wait;
> >       bool                    has_compute_vm_bug;
> >       bool                    no_scheduler;
> > +     bool                    no_user_submission;
> >       int                     hw_prio;
> >       unsigned                num_hw_submission;
> >       atomic_t                *sched_score;
> > @@ -310,7 +311,6 @@ struct amdgpu_ring {
> >       unsigned int    entry_index;
> >       /* store the cached rptr to restore after reset */
> >       uint64_t cached_rptr;
> > -
> >  };
> >
> >  #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
> > --
> > 2.48.1
> >
>
> --
> Rodrigo Siqueira

RE: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Liang, Prike]

[-- Attachment #1: Type: text/plain, Size: 8842 bytes --]

[Public]

We might still need to export each ring's number correctly; otherwise, the Mesa driver will consider there's no available ring supported from the driver and then further assert before submitting the user queue.

If we want to keep the ring number being zero, the Mesa driver may need an attachment change to allow the command submitted to the zero-ring number if the user queue is enabled.

Hi @Olsak, Marek Do you think it's fine to have the attachment patch for the userq support? Except for such changes, maybe we also need to clean up the IB-related part.

Regards,
      Prike

> -----Original Message-----
> From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Alex
> Deucher
> Sent: Thursday, March 13, 2025 10:41 PM
> To: amd-gfx@lists.freedesktop.org
> Cc: Deucher, Alexander <Alexander.Deucher@amd.com>; Khatri, Sunil
> <Sunil.Khatri@amd.com>
> Subject: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions
>
> This would be set by IPs which only accept submissions from the kernel, not
> userspace, such as when kernel queues are disabled. Don't expose the rings to
> userspace and reject any submissions in the CS IOCTL.
>
> Reviewed-by: Sunil Khatri<sunil.khatri@amd.com>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
> drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
>  3 files changed, 25 insertions(+), 11 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> index 5df21529b3b13..5cc18034b75df 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> @@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser
> *p,
>       ring = amdgpu_job_ring(job);
>       ib = &job->ibs[job->num_ibs++];
>
> +     /* submissions to kernel queus are disabled */
> +     if (ring->no_user_submission)
> +             return -EINVAL;
> +
>       /* MM engine doesn't support user fences */
>       if (p->uf_bo && ring->funcs->no_user_fence)
>               return -EINVAL;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>       case AMDGPU_HW_IP_GFX:
>               type = AMD_IP_BLOCK_TYPE_GFX;
>               for (i = 0; i < adev->gfx.num_gfx_rings; i++)
> -                     if (adev->gfx.gfx_ring[i].sched.ready)
> +                     if (adev->gfx.gfx_ring[i].sched.ready &&
> +                         !adev->gfx.gfx_ring[i].no_user_submission)
>                               ++num_rings;
>               ib_start_alignment = 32;
>               ib_size_alignment = 32;
> @@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>       case AMDGPU_HW_IP_COMPUTE:
>               type = AMD_IP_BLOCK_TYPE_GFX;
>               for (i = 0; i < adev->gfx.num_compute_rings; i++)
> -                     if (adev->gfx.compute_ring[i].sched.ready)
> +                     if (adev->gfx.compute_ring[i].sched.ready &&
> +                         !adev->gfx.compute_ring[i].no_user_submission)
>                               ++num_rings;
>               ib_start_alignment = 32;
>               ib_size_alignment = 32;
> @@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>       case AMDGPU_HW_IP_DMA:
>               type = AMD_IP_BLOCK_TYPE_SDMA;
>               for (i = 0; i < adev->sdma.num_instances; i++)
> -                     if (adev->sdma.instance[i].ring.sched.ready)
> +                     if (adev->sdma.instance[i].ring.sched.ready &&
> +                         !adev->gfx.gfx_ring[i].no_user_submission)
>                               ++num_rings;
>               ib_start_alignment = 256;
>               ib_size_alignment = 4;
> @@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>                       if (adev->uvd.harvest_config & (1 << i))
>                               continue;
>
> -                     if (adev->uvd.inst[i].ring.sched.ready)
> +                     if (adev->uvd.inst[i].ring.sched.ready &&
> +                         !adev->uvd.inst[i].ring.no_user_submission)
>                               ++num_rings;
>               }
>               ib_start_alignment = 256;
> @@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>       case AMDGPU_HW_IP_VCE:
>               type = AMD_IP_BLOCK_TYPE_VCE;
>               for (i = 0; i < adev->vce.num_rings; i++)
> -                     if (adev->vce.ring[i].sched.ready)
> +                     if (adev->vce.ring[i].sched.ready &&
> +                         !adev->vce.ring[i].no_user_submission)
>                               ++num_rings;
>               ib_start_alignment = 256;
>               ib_size_alignment = 4;
> @@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>                               continue;
>
>                       for (j = 0; j < adev->uvd.num_enc_rings; j++)
> -                             if (adev->uvd.inst[i].ring_enc[j].sched.ready)
> +                             if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
> +                                 !adev->uvd.inst[i].ring_enc[j].no_user_submission)
>                                       ++num_rings;
>               }
>               ib_start_alignment = 256;
> @@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>                       if (adev->vcn.harvest_config & (1 << i))
>                               continue;
>
> -                     if (adev->vcn.inst[i].ring_dec.sched.ready)
> +                     if (adev->vcn.inst[i].ring_dec.sched.ready &&
> +                         !adev->vcn.inst[i].ring_dec.no_user_submission)
>                               ++num_rings;
>               }
>               ib_start_alignment = 256;
> @@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>                               continue;
>
>                       for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
> -                             if (adev->vcn.inst[i].ring_enc[j].sched.ready)
> +                             if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
> +                                 !adev->vcn.inst[i].ring_enc[j].no_user_submission)
>                                       ++num_rings;
>               }
>               ib_start_alignment = 256;
> @@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>                               continue;
>
>                       for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
> -                             if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
> +                             if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
> +                                 !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
>                                       ++num_rings;
>               }
>               ib_start_alignment = 256;
> @@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
>               break;
>       case AMDGPU_HW_IP_VPE:
>               type = AMD_IP_BLOCK_TYPE_VPE;
> -             if (adev->vpe.ring.sched.ready)
> +             if (adev->vpe.ring.sched.ready &&
> +                 !adev->vpe.ring.no_user_submission)
>                       ++num_rings;
>               ib_start_alignment = 256;
>               ib_size_alignment = 4;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> index b4fd1e17205e9..4a97afcb38b78 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> @@ -297,6 +297,7 @@ struct amdgpu_ring {
>       struct dma_fence        *vmid_wait;
>       bool                    has_compute_vm_bug;
>       bool                    no_scheduler;
> +     bool                    no_user_submission;
>       int                     hw_prio;
>       unsigned                num_hw_submission;
>       atomic_t                *sched_score;
> @@ -310,7 +311,6 @@ struct amdgpu_ring {
>       unsigned int    entry_index;
>       /* store the cached rptr to restore after reset */
>       uint64_t cached_rptr;
> -
>  };
>
>  #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
> --
> 2.48.1


[-- Attachment #2: 0001-winsys-amdgpu-allow-to-submit-userq-at-invalidate-ri.patch --]
[-- Type: application/octet-stream, Size: 1286 bytes --]

From ddf6ea248a0dfb8e5fd28bf7784b416ad7545fc3 Mon Sep 17 00:00:00 2001
From: Prike Liang <Prike.Liang@amd.com>
Date: Mon, 17 Mar 2025 14:32:37 +0800
Subject: [PATCH] winsys/amdgpu: allow to submit userq at invalidate ring_num

In the kernel driver there will disable the gfx kernel ring throughly
when user queue enabled. So the mesa driver should continue to fill
the user queue for the kernel queue disabled case.

Change-Id: I73584225af87d987182470f56cc4dbbe4c451230
Signed-off-by: Prike Liang <Prike.Liang@amd.com>
---
 src/gallium/winsys/amdgpu/drm/amdgpu_cs.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/gallium/winsys/amdgpu/drm/amdgpu_cs.cpp b/src/gallium/winsys/amdgpu/drm/amdgpu_cs.cpp
index beb16ca63beb..75d113428b9c 100644
--- a/src/gallium/winsys/amdgpu/drm/amdgpu_cs.cpp
+++ b/src/gallium/winsys/amdgpu/drm/amdgpu_cs.cpp
@@ -941,7 +941,8 @@ amdgpu_cs_create(struct radeon_cmdbuf *rcs,
 
    /* Compute the queue index by counting the IPs that have queues. */
    assert(ip_type < ARRAY_SIZE(ctx->aws->info.ip));
-   assert(ctx->aws->info.ip[ip_type].num_queues);
+   if (!acs->aws->info.use_userq)
+      assert(ctx->aws->info.ip[ip_type].num_queues);
 
    if (ip_uses_alt_fence(ip_type)) {
       acs->queue_index = INT_MAX;
-- 
2.34.1

Re: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Marek Olšák]

[-- Attachment #1: Type: text/plain, Size: 9569 bytes --]

Userspace needs a query that a queue IP type is supported.
"available_rings" is used for that right now, but if that's 0, something
else must indicate IP support.

amd_ip_info::num_queues should be non-zero even when user queues are
supported. The exact number doesn't matter with user queues.

Marek

On Mon, Mar 17, 2025 at 3:09 AM Liang, Prike <Prike.Liang@amd.com> wrote:

> [Public]
>
> We might still need to export each ring's number correctly; otherwise, the
> Mesa driver will consider there's no available ring supported from the
> driver and then further assert before submitting the user queue.
>
> If we want to keep the ring number being zero, the Mesa driver may need an
> attachment change to allow the command submitted to the zero-ring number if
> the user queue is enabled.
>
> Hi @Olsak, Marek Do you think it's fine to have the attachment patch for
> the userq support? Except for such changes, maybe we also need to clean up
> the IB-related part.
>
> Regards,
>       Prike
>
> > -----Original Message-----
> > From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Alex
> > Deucher
> > Sent: Thursday, March 13, 2025 10:41 PM
> > To: amd-gfx@lists.freedesktop.org
> > Cc: Deucher, Alexander <Alexander.Deucher@amd.com>; Khatri, Sunil
> > <Sunil.Khatri@amd.com>
> > Subject: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions
> >
> > This would be set by IPs which only accept submissions from the kernel,
> not
> > userspace, such as when kernel queues are disabled. Don't expose the
> rings to
> > userspace and reject any submissions in the CS IOCTL.
> >
> > Reviewed-by: Sunil Khatri<sunil.khatri@amd.com>
> > Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> > ---
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
> > drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
> >  3 files changed, 25 insertions(+), 11 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > index 5df21529b3b13..5cc18034b75df 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > @@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser
> > *p,
> >       ring = amdgpu_job_ring(job);
> >       ib = &job->ibs[job->num_ibs++];
> >
> > +     /* submissions to kernel queus are disabled */
> > +     if (ring->no_user_submission)
> > +             return -EINVAL;
> > +
> >       /* MM engine doesn't support user fences */
> >       if (p->uf_bo && ring->funcs->no_user_fence)
> >               return -EINVAL;
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > @@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >       case AMDGPU_HW_IP_GFX:
> >               type = AMD_IP_BLOCK_TYPE_GFX;
> >               for (i = 0; i < adev->gfx.num_gfx_rings; i++)
> > -                     if (adev->gfx.gfx_ring[i].sched.ready)
> > +                     if (adev->gfx.gfx_ring[i].sched.ready &&
> > +                         !adev->gfx.gfx_ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 32;
> >               ib_size_alignment = 32;
> > @@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >       case AMDGPU_HW_IP_COMPUTE:
> >               type = AMD_IP_BLOCK_TYPE_GFX;
> >               for (i = 0; i < adev->gfx.num_compute_rings; i++)
> > -                     if (adev->gfx.compute_ring[i].sched.ready)
> > +                     if (adev->gfx.compute_ring[i].sched.ready &&
> > +                         !adev->gfx.compute_ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 32;
> >               ib_size_alignment = 32;
> > @@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >       case AMDGPU_HW_IP_DMA:
> >               type = AMD_IP_BLOCK_TYPE_SDMA;
> >               for (i = 0; i < adev->sdma.num_instances; i++)
> > -                     if (adev->sdma.instance[i].ring.sched.ready)
> > +                     if (adev->sdma.instance[i].ring.sched.ready &&
> > +                         !adev->gfx.gfx_ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 256;
> >               ib_size_alignment = 4;
> > @@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >                       if (adev->uvd.harvest_config & (1 << i))
> >                               continue;
> >
> > -                     if (adev->uvd.inst[i].ring.sched.ready)
> > +                     if (adev->uvd.inst[i].ring.sched.ready &&
> > +                         !adev->uvd.inst[i].ring.no_user_submission)
> >                               ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >       case AMDGPU_HW_IP_VCE:
> >               type = AMD_IP_BLOCK_TYPE_VCE;
> >               for (i = 0; i < adev->vce.num_rings; i++)
> > -                     if (adev->vce.ring[i].sched.ready)
> > +                     if (adev->vce.ring[i].sched.ready &&
> > +                         !adev->vce.ring[i].no_user_submission)
> >                               ++num_rings;
> >               ib_start_alignment = 256;
> >               ib_size_alignment = 4;
> > @@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >                               continue;
> >
> >                       for (j = 0; j < adev->uvd.num_enc_rings; j++)
> > -                             if
> (adev->uvd.inst[i].ring_enc[j].sched.ready)
> > +                             if
> (adev->uvd.inst[i].ring_enc[j].sched.ready &&
> > +
>  !adev->uvd.inst[i].ring_enc[j].no_user_submission)
> >                                       ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >                       if (adev->vcn.harvest_config & (1 << i))
> >                               continue;
> >
> > -                     if (adev->vcn.inst[i].ring_dec.sched.ready)
> > +                     if (adev->vcn.inst[i].ring_dec.sched.ready &&
> > +                         !adev->vcn.inst[i].ring_dec.no_user_submission)
> >                               ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >                               continue;
> >
> >                       for (j = 0; j < adev->vcn.inst[i].num_enc_rings;
> j++)
> > -                             if
> (adev->vcn.inst[i].ring_enc[j].sched.ready)
> > +                             if
> (adev->vcn.inst[i].ring_enc[j].sched.ready &&
> > +
>  !adev->vcn.inst[i].ring_enc[j].no_user_submission)
> >                                       ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >                               continue;
> >
> >                       for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
> > -                             if
> (adev->jpeg.inst[i].ring_dec[j].sched.ready)
> > +                             if
> (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
> > +
>  !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
> >                                       ++num_rings;
> >               }
> >               ib_start_alignment = 256;
> > @@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> >               break;
> >       case AMDGPU_HW_IP_VPE:
> >               type = AMD_IP_BLOCK_TYPE_VPE;
> > -             if (adev->vpe.ring.sched.ready)
> > +             if (adev->vpe.ring.sched.ready &&
> > +                 !adev->vpe.ring.no_user_submission)
> >                       ++num_rings;
> >               ib_start_alignment = 256;
> >               ib_size_alignment = 4;
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > index b4fd1e17205e9..4a97afcb38b78 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > @@ -297,6 +297,7 @@ struct amdgpu_ring {
> >       struct dma_fence        *vmid_wait;
> >       bool                    has_compute_vm_bug;
> >       bool                    no_scheduler;
> > +     bool                    no_user_submission;
> >       int                     hw_prio;
> >       unsigned                num_hw_submission;
> >       atomic_t                *sched_score;
> > @@ -310,7 +311,6 @@ struct amdgpu_ring {
> >       unsigned int    entry_index;
> >       /* store the cached rptr to restore after reset */
> >       uint64_t cached_rptr;
> > -
> >  };
> >
> >  #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p),
> (job), (ib)))
> > --
> > 2.48.1
>
>

[-- Attachment #2: Type: text/html, Size: 12681 bytes --]

Re: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Alex Deucher]

On Mon, Mar 17, 2025 at 1:23 PM Marek Olšák <maraeo@gmail.com> wrote:
>
> Userspace needs a query that a queue IP type is supported. "available_rings" is used for that right now, but if that's 0, something else must indicate IP support.
>
> amd_ip_info::num_queues should be non-zero even when user queues are supported. The exact number doesn't matter with user queues.

How will mesa determine whether kernel queues are supported?  Can mesa
look at amd_ip_info::num_queues and if it's 0, check some new INFO
query to determine if user queues are available?  If
amd_ip_info::num_queues is always non-0, then it would be assumed that
the kernel supports kernel queues, which it may not.

Alex

>
> Marek
>
> On Mon, Mar 17, 2025 at 3:09 AM Liang, Prike <Prike.Liang@amd.com> wrote:
>>
>> [Public]
>>
>> We might still need to export each ring's number correctly; otherwise, the Mesa driver will consider there's no available ring supported from the driver and then further assert before submitting the user queue.
>>
>> If we want to keep the ring number being zero, the Mesa driver may need an attachment change to allow the command submitted to the zero-ring number if the user queue is enabled.
>>
>> Hi @Olsak, Marek Do you think it's fine to have the attachment patch for the userq support? Except for such changes, maybe we also need to clean up the IB-related part.
>>
>> Regards,
>>       Prike
>>
>> > -----Original Message-----
>> > From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Alex
>> > Deucher
>> > Sent: Thursday, March 13, 2025 10:41 PM
>> > To: amd-gfx@lists.freedesktop.org
>> > Cc: Deucher, Alexander <Alexander.Deucher@amd.com>; Khatri, Sunil
>> > <Sunil.Khatri@amd.com>
>> > Subject: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions
>> >
>> > This would be set by IPs which only accept submissions from the kernel, not
>> > userspace, such as when kernel queues are disabled. Don't expose the rings to
>> > userspace and reject any submissions in the CS IOCTL.
>> >
>> > Reviewed-by: Sunil Khatri<sunil.khatri@amd.com>
>> > Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
>> > ---
>> >  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  4 ++++
>> >  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c  | 30 ++++++++++++++++--------
>> > drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  2 +-
>> >  3 files changed, 25 insertions(+), 11 deletions(-)
>> >
>> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
>> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
>> > index 5df21529b3b13..5cc18034b75df 100644
>> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
>> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
>> > @@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser
>> > *p,
>> >       ring = amdgpu_job_ring(job);
>> >       ib = &job->ibs[job->num_ibs++];
>> >
>> > +     /* submissions to kernel queus are disabled */
>> > +     if (ring->no_user_submission)
>> > +             return -EINVAL;
>> > +
>> >       /* MM engine doesn't support user fences */
>> >       if (p->uf_bo && ring->funcs->no_user_fence)
>> >               return -EINVAL;
>> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>> > index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
>> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>> > @@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
>> > *adev,
>> >       case AMDGPU_HW_IP_GFX:
>> >               type = AMD_IP_BLOCK_TYPE_GFX;
>> >               for (i = 0; i < adev->gfx.num_gfx_rings; i++)
>> > -                     if (adev->gfx.gfx_ring[i].sched.ready)
>> > +                     if (adev->gfx.gfx_ring[i].sched.ready &&
>> > +                         !adev->gfx.gfx_ring[i].no_user_submission)
>> >                               ++num_rings;
>> >               ib_start_alignment = 32;
>> >               ib_size_alignment = 32;
>> > @@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
>> > *adev,
>> >       case AMDGPU_HW_IP_COMPUTE:
>> >               type = AMD_IP_BLOCK_TYPE_GFX;
>> >               for (i = 0; i < adev->gfx.num_compute_rings; i++)
>> > -                     if (adev->gfx.compute_ring[i].sched.ready)
>> > +                     if (adev->gfx.compute_ring[i].sched.ready &&
>> > +                         !adev->gfx.compute_ring[i].no_user_submission)
>> >                               ++num_rings;
>> >               ib_start_alignment = 32;
>> >               ib_size_alignment = 32;
>> > @@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
>> > *adev,
>> >       case AMDGPU_HW_IP_DMA:
>> >               type = AMD_IP_BLOCK_TYPE_SDMA;
>> >               for (i = 0; i < adev->sdma.num_instances; i++)
>> > -                     if (adev->sdma.instance[i].ring.sched.ready)
>> > +                     if (adev->sdma.instance[i].ring.sched.ready &&
>> > +                         !adev->gfx.gfx_ring[i].no_user_submission)
>> >                               ++num_rings;
>> >               ib_start_alignment = 256;
>> >               ib_size_alignment = 4;
>> > @@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
>> > *adev,
>> >                       if (adev->uvd.harvest_config & (1 << i))
>> >                               continue;
>> >
>> > -                     if (adev->uvd.inst[i].ring.sched.ready)
>> > +                     if (adev->uvd.inst[i].ring.sched.ready &&
>> > +                         !adev->uvd.inst[i].ring.no_user_submission)
>> >                               ++num_rings;
>> >               }
>> >               ib_start_alignment = 256;
>> > @@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
>> > *adev,
>> >       case AMDGPU_HW_IP_VCE:
>> >               type = AMD_IP_BLOCK_TYPE_VCE;
>> >               for (i = 0; i < adev->vce.num_rings; i++)
>> > -                     if (adev->vce.ring[i].sched.ready)
>> > +                     if (adev->vce.ring[i].sched.ready &&
>> > +                         !adev->vce.ring[i].no_user_submission)
>> >                               ++num_rings;
>> >               ib_start_alignment = 256;
>> >               ib_size_alignment = 4;
>> > @@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
>> > *adev,
>> >                               continue;
>> >
>> >                       for (j = 0; j < adev->uvd.num_enc_rings; j++)
>> > -                             if (adev->uvd.inst[i].ring_enc[j].sched.ready)
>> > +                             if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
>> > +                                 !adev->uvd.inst[i].ring_enc[j].no_user_submission)
>> >                                       ++num_rings;
>> >               }
>> >               ib_start_alignment = 256;
>> > @@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
>> > *adev,
>> >                       if (adev->vcn.harvest_config & (1 << i))
>> >                               continue;
>> >
>> > -                     if (adev->vcn.inst[i].ring_dec.sched.ready)
>> > +                     if (adev->vcn.inst[i].ring_dec.sched.ready &&
>> > +                         !adev->vcn.inst[i].ring_dec.no_user_submission)
>> >                               ++num_rings;
>> >               }
>> >               ib_start_alignment = 256;
>> > @@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
>> > *adev,
>> >                               continue;
>> >
>> >                       for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
>> > -                             if (adev->vcn.inst[i].ring_enc[j].sched.ready)
>> > +                             if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
>> > +                                 !adev->vcn.inst[i].ring_enc[j].no_user_submission)
>> >                                       ++num_rings;
>> >               }
>> >               ib_start_alignment = 256;
>> > @@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
>> > *adev,
>> >                               continue;
>> >
>> >                       for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
>> > -                             if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
>> > +                             if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
>> > +                                 !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
>> >                                       ++num_rings;
>> >               }
>> >               ib_start_alignment = 256;
>> > @@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
>> > *adev,
>> >               break;
>> >       case AMDGPU_HW_IP_VPE:
>> >               type = AMD_IP_BLOCK_TYPE_VPE;
>> > -             if (adev->vpe.ring.sched.ready)
>> > +             if (adev->vpe.ring.sched.ready &&
>> > +                 !adev->vpe.ring.no_user_submission)
>> >                       ++num_rings;
>> >               ib_start_alignment = 256;
>> >               ib_size_alignment = 4;
>> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
>> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
>> > index b4fd1e17205e9..4a97afcb38b78 100644
>> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
>> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
>> > @@ -297,6 +297,7 @@ struct amdgpu_ring {
>> >       struct dma_fence        *vmid_wait;
>> >       bool                    has_compute_vm_bug;
>> >       bool                    no_scheduler;
>> > +     bool                    no_user_submission;
>> >       int                     hw_prio;
>> >       unsigned                num_hw_submission;
>> >       atomic_t                *sched_score;
>> > @@ -310,7 +311,6 @@ struct amdgpu_ring {
>> >       unsigned int    entry_index;
>> >       /* store the cached rptr to restore after reset */
>> >       uint64_t cached_rptr;
>> > -
>> >  };
>> >
>> >  #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
>> > --
>> > 2.48.1
>>

Re: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions

[Marek Olšák]

[-- Attachment #1: Type: text/plain, Size: 1025 bytes --]

On Mon, Mar 17, 2025 at 2:27 PM Alex Deucher <alexdeucher@gmail.com> wrote:

> On Mon, Mar 17, 2025 at 1:23 PM Marek Olšák <maraeo@gmail.com> wrote:
> >
> > Userspace needs a query that a queue IP type is supported.
> "available_rings" is used for that right now, but if that's 0, something
> else must indicate IP support.
> >
> > amd_ip_info::num_queues should be non-zero even when user queues are
> supported. The exact number doesn't matter with user queues.
>
> How will mesa determine whether kernel queues are supported?  Can mesa
> look at amd_ip_info::num_queues and if it's 0, check some new INFO
> query to determine if user queues are available?  If
> amd_ip_info::num_queues is always non-0, then it would be assumed that
> the kernel supports kernel queues, which it may not.
>

That is to be determined. However, num_queues is also used to indicate
whether the IP is supported, so either the meaning of num_queues has to
change, or the way num_queues is set has to change.

Marek

[-- Attachment #2: Type: text/html, Size: 1471 bytes --]