* [PATCH 01/11] drm/amdgpu: add parameter to disable kernel queues
2025-03-06 18:46 [PATCH V2 00/11] Add disable kernel queue support Alex Deucher
@ 2025-03-06 18:46 ` Alex Deucher
2025-03-06 18:46 ` [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions Alex Deucher
` (9 subsequent siblings)
10 siblings, 0 replies; 20+ messages in thread
From: Alex Deucher @ 2025-03-06 18:46 UTC (permalink / raw)
To: amd-gfx; +Cc: Alex Deucher, Sunil Khatri
On chips that support user queues, setting this option
will disable kernel queues to be used to validate
user queues without kernel queues.
Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 +
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 9 +++++++++
2 files changed, 10 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 87062c1adcdf7..45437a8f29d3b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -270,6 +270,7 @@ extern int amdgpu_user_partt_mode;
extern int amdgpu_agp;
extern int amdgpu_wbrf;
+extern int amdgpu_disable_kq;
#define AMDGPU_VM_MAX_NUM_CTX 4096
#define AMDGPU_SG_THRESHOLD (256*1024*1024)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index b161daa900198..42a7619592ab9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -237,6 +237,7 @@ int amdgpu_agp = -1; /* auto */
int amdgpu_wbrf = -1;
int amdgpu_damage_clips = -1; /* auto */
int amdgpu_umsch_mm_fwlog;
+int amdgpu_disable_kq = -1;
DECLARE_DYNDBG_CLASSMAP(drm_debug_classes, DD_CLASS_TYPE_DISJOINT_BITS, 0,
"DRM_UT_CORE",
@@ -1083,6 +1084,14 @@ MODULE_PARM_DESC(wbrf,
"Enable Wifi RFI interference mitigation (0 = disabled, 1 = enabled, -1 = auto(default)");
module_param_named(wbrf, amdgpu_wbrf, int, 0444);
+/**
+ * DOC: disable_kq (int)
+ * Disable kernel queues on systems that support user queues.
+ * (0 = kernel queues enabled, 1 = kernel queues disabled, -1 = auto (default setting))
+ */
+MODULE_PARM_DESC(disable_kq, "Disable kernel queues (-1 = auto (default), 0 = enable KQ, 1 = disable KQ)");
+module_param_named(disable_kq, amdgpu_disable_kq, int, 0444);
+
/* These devices are not supported by amdgpu.
* They are supported by the mach64, r128, radeon drivers
*/
--
2.48.1
^ permalink raw reply related [flat|nested] 20+ messages in thread* [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions
2025-03-06 18:46 [PATCH V2 00/11] Add disable kernel queue support Alex Deucher
2025-03-06 18:46 ` [PATCH 01/11] drm/amdgpu: add parameter to disable kernel queues Alex Deucher
@ 2025-03-06 18:46 ` Alex Deucher
2025-03-11 13:02 ` Liang, Prike
2025-03-06 18:46 ` [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq Alex Deucher
` (8 subsequent siblings)
10 siblings, 1 reply; 20+ messages in thread
From: Alex Deucher @ 2025-03-06 18:46 UTC (permalink / raw)
To: amd-gfx; +Cc: Alex Deucher
This would be set by IPs which only accept submissions
from the kernel, not userspace, such as when kernel
queues are disabled. Don't expose the rings to userspace
and reject any submissions in the CS IOCTL.
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 4 ++++
drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 30 ++++++++++++++++--------
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 2 +-
3 files changed, 25 insertions(+), 11 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 5df21529b3b13..5cc18034b75df 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,
ring = amdgpu_job_ring(job);
ib = &job->ibs[job->num_ibs++];
+ /* submissions to kernel queus are disabled */
+ if (ring->no_user_submission)
+ return -EINVAL;
+
/* MM engine doesn't support user fences */
if (p->uf_bo && ring->funcs->no_user_fence)
return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
case AMDGPU_HW_IP_GFX:
type = AMD_IP_BLOCK_TYPE_GFX;
for (i = 0; i < adev->gfx.num_gfx_rings; i++)
- if (adev->gfx.gfx_ring[i].sched.ready)
+ if (adev->gfx.gfx_ring[i].sched.ready &&
+ !adev->gfx.gfx_ring[i].no_user_submission)
++num_rings;
ib_start_alignment = 32;
ib_size_alignment = 32;
@@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
case AMDGPU_HW_IP_COMPUTE:
type = AMD_IP_BLOCK_TYPE_GFX;
for (i = 0; i < adev->gfx.num_compute_rings; i++)
- if (adev->gfx.compute_ring[i].sched.ready)
+ if (adev->gfx.compute_ring[i].sched.ready &&
+ !adev->gfx.compute_ring[i].no_user_submission)
++num_rings;
ib_start_alignment = 32;
ib_size_alignment = 32;
@@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
case AMDGPU_HW_IP_DMA:
type = AMD_IP_BLOCK_TYPE_SDMA;
for (i = 0; i < adev->sdma.num_instances; i++)
- if (adev->sdma.instance[i].ring.sched.ready)
+ if (adev->sdma.instance[i].ring.sched.ready &&
+ !adev->gfx.gfx_ring[i].no_user_submission)
++num_rings;
ib_start_alignment = 256;
ib_size_alignment = 4;
@@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
if (adev->uvd.harvest_config & (1 << i))
continue;
- if (adev->uvd.inst[i].ring.sched.ready)
+ if (adev->uvd.inst[i].ring.sched.ready &&
+ !adev->uvd.inst[i].ring.no_user_submission)
++num_rings;
}
ib_start_alignment = 256;
@@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
case AMDGPU_HW_IP_VCE:
type = AMD_IP_BLOCK_TYPE_VCE;
for (i = 0; i < adev->vce.num_rings; i++)
- if (adev->vce.ring[i].sched.ready)
+ if (adev->vce.ring[i].sched.ready &&
+ !adev->vce.ring[i].no_user_submission)
++num_rings;
ib_start_alignment = 256;
ib_size_alignment = 4;
@@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
continue;
for (j = 0; j < adev->uvd.num_enc_rings; j++)
- if (adev->uvd.inst[i].ring_enc[j].sched.ready)
+ if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
+ !adev->uvd.inst[i].ring_enc[j].no_user_submission)
++num_rings;
}
ib_start_alignment = 256;
@@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
if (adev->vcn.harvest_config & (1 << i))
continue;
- if (adev->vcn.inst[i].ring_dec.sched.ready)
+ if (adev->vcn.inst[i].ring_dec.sched.ready &&
+ !adev->vcn.inst[i].ring_dec.no_user_submission)
++num_rings;
}
ib_start_alignment = 256;
@@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
continue;
for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
- if (adev->vcn.inst[i].ring_enc[j].sched.ready)
+ if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
+ !adev->vcn.inst[i].ring_enc[j].no_user_submission)
++num_rings;
}
ib_start_alignment = 256;
@@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
continue;
for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
- if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
+ if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
+ !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
++num_rings;
}
ib_start_alignment = 256;
@@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
break;
case AMDGPU_HW_IP_VPE:
type = AMD_IP_BLOCK_TYPE_VPE;
- if (adev->vpe.ring.sched.ready)
+ if (adev->vpe.ring.sched.ready &&
+ !adev->vpe.ring.no_user_submission)
++num_rings;
ib_start_alignment = 256;
ib_size_alignment = 4;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index b4fd1e17205e9..4a97afcb38b78 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -297,6 +297,7 @@ struct amdgpu_ring {
struct dma_fence *vmid_wait;
bool has_compute_vm_bug;
bool no_scheduler;
+ bool no_user_submission;
int hw_prio;
unsigned num_hw_submission;
atomic_t *sched_score;
@@ -310,7 +311,6 @@ struct amdgpu_ring {
unsigned int entry_index;
/* store the cached rptr to restore after reset */
uint64_t cached_rptr;
-
};
#define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
--
2.48.1
^ permalink raw reply related [flat|nested] 20+ messages in thread* RE: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions
2025-03-06 18:46 ` [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions Alex Deucher
@ 2025-03-11 13:02 ` Liang, Prike
2025-03-12 18:50 ` Alex Deucher
0 siblings, 1 reply; 20+ messages in thread
From: Liang, Prike @ 2025-03-11 13:02 UTC (permalink / raw)
To: Deucher, Alexander, amd-gfx@lists.freedesktop.org; +Cc: Deucher, Alexander
[Public]
> From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Alex
> Deucher
> Sent: Friday, March 7, 2025 2:46 AM
> To: amd-gfx@lists.freedesktop.org
> Cc: Deucher, Alexander <Alexander.Deucher@amd.com>
> Subject: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions
>
> This would be set by IPs which only accept submissions from the kernel, not
> userspace, such as when kernel queues are disabled. Don't expose the rings to
> userspace and reject any submissions in the CS IOCTL.
>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 4 ++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 30 ++++++++++++++++--------
> drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 2 +-
> 3 files changed, 25 insertions(+), 11 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> index 5df21529b3b13..5cc18034b75df 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> @@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser
> *p,
> ring = amdgpu_job_ring(job);
> ib = &job->ibs[job->num_ibs++];
>
> + /* submissions to kernel queus are disabled */
> + if (ring->no_user_submission)
> + return -EINVAL;
> +
Do we need reject the kernel queue submission at the beginning placement of amdgpu_cs_ioctl()?
Thanks,
Prike
> /* MM engine doesn't support user fences */
> if (p->uf_bo && ring->funcs->no_user_fence)
> return -EINVAL;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
> case AMDGPU_HW_IP_GFX:
> type = AMD_IP_BLOCK_TYPE_GFX;
> for (i = 0; i < adev->gfx.num_gfx_rings; i++)
> - if (adev->gfx.gfx_ring[i].sched.ready)
> + if (adev->gfx.gfx_ring[i].sched.ready &&
> + !adev->gfx.gfx_ring[i].no_user_submission)
> ++num_rings;
> ib_start_alignment = 32;
> ib_size_alignment = 32;
> @@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
> case AMDGPU_HW_IP_COMPUTE:
> type = AMD_IP_BLOCK_TYPE_GFX;
> for (i = 0; i < adev->gfx.num_compute_rings; i++)
> - if (adev->gfx.compute_ring[i].sched.ready)
> + if (adev->gfx.compute_ring[i].sched.ready &&
> + !adev->gfx.compute_ring[i].no_user_submission)
> ++num_rings;
> ib_start_alignment = 32;
> ib_size_alignment = 32;
> @@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
> case AMDGPU_HW_IP_DMA:
> type = AMD_IP_BLOCK_TYPE_SDMA;
> for (i = 0; i < adev->sdma.num_instances; i++)
> - if (adev->sdma.instance[i].ring.sched.ready)
> + if (adev->sdma.instance[i].ring.sched.ready &&
> + !adev->gfx.gfx_ring[i].no_user_submission)
> ++num_rings;
> ib_start_alignment = 256;
> ib_size_alignment = 4;
> @@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
> if (adev->uvd.harvest_config & (1 << i))
> continue;
>
> - if (adev->uvd.inst[i].ring.sched.ready)
> + if (adev->uvd.inst[i].ring.sched.ready &&
> + !adev->uvd.inst[i].ring.no_user_submission)
> ++num_rings;
> }
> ib_start_alignment = 256;
> @@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
> case AMDGPU_HW_IP_VCE:
> type = AMD_IP_BLOCK_TYPE_VCE;
> for (i = 0; i < adev->vce.num_rings; i++)
> - if (adev->vce.ring[i].sched.ready)
> + if (adev->vce.ring[i].sched.ready &&
> + !adev->vce.ring[i].no_user_submission)
> ++num_rings;
> ib_start_alignment = 256;
> ib_size_alignment = 4;
> @@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
> continue;
>
> for (j = 0; j < adev->uvd.num_enc_rings; j++)
> - if (adev->uvd.inst[i].ring_enc[j].sched.ready)
> + if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
> + !adev->uvd.inst[i].ring_enc[j].no_user_submission)
> ++num_rings;
> }
> ib_start_alignment = 256;
> @@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
> if (adev->vcn.harvest_config & (1 << i))
> continue;
>
> - if (adev->vcn.inst[i].ring_dec.sched.ready)
> + if (adev->vcn.inst[i].ring_dec.sched.ready &&
> + !adev->vcn.inst[i].ring_dec.no_user_submission)
> ++num_rings;
> }
> ib_start_alignment = 256;
> @@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
> continue;
>
> for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
> - if (adev->vcn.inst[i].ring_enc[j].sched.ready)
> + if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
> + !adev->vcn.inst[i].ring_enc[j].no_user_submission)
> ++num_rings;
> }
> ib_start_alignment = 256;
> @@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
> continue;
>
> for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
> - if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
> + if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
> + !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
> ++num_rings;
> }
> ib_start_alignment = 256;
> @@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> *adev,
> break;
> case AMDGPU_HW_IP_VPE:
> type = AMD_IP_BLOCK_TYPE_VPE;
> - if (adev->vpe.ring.sched.ready)
> + if (adev->vpe.ring.sched.ready &&
> + !adev->vpe.ring.no_user_submission)
> ++num_rings;
> ib_start_alignment = 256;
> ib_size_alignment = 4;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> index b4fd1e17205e9..4a97afcb38b78 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> @@ -297,6 +297,7 @@ struct amdgpu_ring {
> struct dma_fence *vmid_wait;
> bool has_compute_vm_bug;
> bool no_scheduler;
> + bool no_user_submission;
> int hw_prio;
> unsigned num_hw_submission;
> atomic_t *sched_score;
> @@ -310,7 +311,6 @@ struct amdgpu_ring {
> unsigned int entry_index;
> /* store the cached rptr to restore after reset */
> uint64_t cached_rptr;
> -
> };
>
> #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
> --
> 2.48.1
^ permalink raw reply [flat|nested] 20+ messages in thread* Re: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions
2025-03-11 13:02 ` Liang, Prike
@ 2025-03-12 18:50 ` Alex Deucher
0 siblings, 0 replies; 20+ messages in thread
From: Alex Deucher @ 2025-03-12 18:50 UTC (permalink / raw)
To: Liang, Prike; +Cc: Deucher, Alexander, amd-gfx@lists.freedesktop.org
On Tue, Mar 11, 2025 at 9:13 AM Liang, Prike <Prike.Liang@amd.com> wrote:
>
> [Public]
>
> > From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Alex
> > Deucher
> > Sent: Friday, March 7, 2025 2:46 AM
> > To: amd-gfx@lists.freedesktop.org
> > Cc: Deucher, Alexander <Alexander.Deucher@amd.com>
> > Subject: [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions
> >
> > This would be set by IPs which only accept submissions from the kernel, not
> > userspace, such as when kernel queues are disabled. Don't expose the rings to
> > userspace and reject any submissions in the CS IOCTL.
> >
> > Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> > ---
> > drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 4 ++++
> > drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 30 ++++++++++++++++--------
> > drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 2 +-
> > 3 files changed, 25 insertions(+), 11 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > index 5df21529b3b13..5cc18034b75df 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> > @@ -349,6 +349,10 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser
> > *p,
> > ring = amdgpu_job_ring(job);
> > ib = &job->ibs[job->num_ibs++];
> >
> > + /* submissions to kernel queus are disabled */
> > + if (ring->no_user_submission)
> > + return -EINVAL;
> > +
>
> Do we need reject the kernel queue submission at the beginning placement of amdgpu_cs_ioctl()?
I think we only need to reject it if it's targeted at a kernel ring
and this is the first place we can determine that.
Alex
>
> Thanks,
> Prike
> > /* MM engine doesn't support user fences */
> > if (p->uf_bo && ring->funcs->no_user_fence)
> > return -EINVAL;
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > index cd6eb7a3bc58a..3b7dfd56ccd0e 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > @@ -408,7 +408,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> > case AMDGPU_HW_IP_GFX:
> > type = AMD_IP_BLOCK_TYPE_GFX;
> > for (i = 0; i < adev->gfx.num_gfx_rings; i++)
> > - if (adev->gfx.gfx_ring[i].sched.ready)
> > + if (adev->gfx.gfx_ring[i].sched.ready &&
> > + !adev->gfx.gfx_ring[i].no_user_submission)
> > ++num_rings;
> > ib_start_alignment = 32;
> > ib_size_alignment = 32;
> > @@ -416,7 +417,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> > case AMDGPU_HW_IP_COMPUTE:
> > type = AMD_IP_BLOCK_TYPE_GFX;
> > for (i = 0; i < adev->gfx.num_compute_rings; i++)
> > - if (adev->gfx.compute_ring[i].sched.ready)
> > + if (adev->gfx.compute_ring[i].sched.ready &&
> > + !adev->gfx.compute_ring[i].no_user_submission)
> > ++num_rings;
> > ib_start_alignment = 32;
> > ib_size_alignment = 32;
> > @@ -424,7 +426,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> > case AMDGPU_HW_IP_DMA:
> > type = AMD_IP_BLOCK_TYPE_SDMA;
> > for (i = 0; i < adev->sdma.num_instances; i++)
> > - if (adev->sdma.instance[i].ring.sched.ready)
> > + if (adev->sdma.instance[i].ring.sched.ready &&
> > + !adev->gfx.gfx_ring[i].no_user_submission)
> > ++num_rings;
> > ib_start_alignment = 256;
> > ib_size_alignment = 4;
> > @@ -435,7 +438,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> > if (adev->uvd.harvest_config & (1 << i))
> > continue;
> >
> > - if (adev->uvd.inst[i].ring.sched.ready)
> > + if (adev->uvd.inst[i].ring.sched.ready &&
> > + !adev->uvd.inst[i].ring.no_user_submission)
> > ++num_rings;
> > }
> > ib_start_alignment = 256;
> > @@ -444,7 +448,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> > case AMDGPU_HW_IP_VCE:
> > type = AMD_IP_BLOCK_TYPE_VCE;
> > for (i = 0; i < adev->vce.num_rings; i++)
> > - if (adev->vce.ring[i].sched.ready)
> > + if (adev->vce.ring[i].sched.ready &&
> > + !adev->vce.ring[i].no_user_submission)
> > ++num_rings;
> > ib_start_alignment = 256;
> > ib_size_alignment = 4;
> > @@ -456,7 +461,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> > continue;
> >
> > for (j = 0; j < adev->uvd.num_enc_rings; j++)
> > - if (adev->uvd.inst[i].ring_enc[j].sched.ready)
> > + if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
> > + !adev->uvd.inst[i].ring_enc[j].no_user_submission)
> > ++num_rings;
> > }
> > ib_start_alignment = 256;
> > @@ -468,7 +474,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> > if (adev->vcn.harvest_config & (1 << i))
> > continue;
> >
> > - if (adev->vcn.inst[i].ring_dec.sched.ready)
> > + if (adev->vcn.inst[i].ring_dec.sched.ready &&
> > + !adev->vcn.inst[i].ring_dec.no_user_submission)
> > ++num_rings;
> > }
> > ib_start_alignment = 256;
> > @@ -481,7 +488,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> > continue;
> >
> > for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
> > - if (adev->vcn.inst[i].ring_enc[j].sched.ready)
> > + if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
> > + !adev->vcn.inst[i].ring_enc[j].no_user_submission)
> > ++num_rings;
> > }
> > ib_start_alignment = 256;
> > @@ -496,7 +504,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> > continue;
> >
> > for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
> > - if (adev->jpeg.inst[i].ring_dec[j].sched.ready)
> > + if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
> > + !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
> > ++num_rings;
> > }
> > ib_start_alignment = 256;
> > @@ -504,7 +513,8 @@ static int amdgpu_hw_ip_info(struct amdgpu_device
> > *adev,
> > break;
> > case AMDGPU_HW_IP_VPE:
> > type = AMD_IP_BLOCK_TYPE_VPE;
> > - if (adev->vpe.ring.sched.ready)
> > + if (adev->vpe.ring.sched.ready &&
> > + !adev->vpe.ring.no_user_submission)
> > ++num_rings;
> > ib_start_alignment = 256;
> > ib_size_alignment = 4;
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > index b4fd1e17205e9..4a97afcb38b78 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > @@ -297,6 +297,7 @@ struct amdgpu_ring {
> > struct dma_fence *vmid_wait;
> > bool has_compute_vm_bug;
> > bool no_scheduler;
> > + bool no_user_submission;
> > int hw_prio;
> > unsigned num_hw_submission;
> > atomic_t *sched_score;
> > @@ -310,7 +311,6 @@ struct amdgpu_ring {
> > unsigned int entry_index;
> > /* store the cached rptr to restore after reset */
> > uint64_t cached_rptr;
> > -
> > };
> >
> > #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
> > --
> > 2.48.1
>
^ permalink raw reply [flat|nested] 20+ messages in thread
* [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq
2025-03-06 18:46 [PATCH V2 00/11] Add disable kernel queue support Alex Deucher
2025-03-06 18:46 ` [PATCH 01/11] drm/amdgpu: add parameter to disable kernel queues Alex Deucher
2025-03-06 18:46 ` [PATCH 02/11] drm/amdgpu: add ring flag for no user submissions Alex Deucher
@ 2025-03-06 18:46 ` Alex Deucher
2025-03-06 18:46 ` [PATCH 04/11] drm/amdgpu/mes: centralize gfx_hqd mask management Alex Deucher
` (7 subsequent siblings)
10 siblings, 0 replies; 20+ messages in thread
From: Alex Deucher @ 2025-03-06 18:46 UTC (permalink / raw)
To: amd-gfx; +Cc: Alex Deucher
Add proper checks for disable_kq functionality in
gfx helper functions. Add special logic for families
that require the clear state setup.
v2: use ring count as per Felix suggestion
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 3 +++
drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 2 ++
2 files changed, 5 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index a194bf3347cbc..4083b4d0595a2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -1544,6 +1544,9 @@ static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device *dev,
if (adev->in_suspend && !adev->in_runpm)
return -EPERM;
+ if (adev->gfx.disable_kq)
+ return -ENOTSUPP;
+
ret = kstrtol(buf, 0, &value);
if (ret)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index ddf4533614bac..8fa68a4ac34f1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -483,6 +483,8 @@ struct amdgpu_gfx {
atomic_t total_submission_cnt;
struct delayed_work idle_work;
+
+ bool disable_kq;
};
struct amdgpu_gfx_ras_reg_entry {
--
2.48.1
^ permalink raw reply related [flat|nested] 20+ messages in thread* [PATCH 04/11] drm/amdgpu/mes: centralize gfx_hqd mask management
2025-03-06 18:46 [PATCH V2 00/11] Add disable kernel queue support Alex Deucher
` (2 preceding siblings ...)
2025-03-06 18:46 ` [PATCH 03/11] drm/amdgpu/gfx: add generic handling for disable_kq Alex Deucher
@ 2025-03-06 18:46 ` Alex Deucher
2025-03-06 18:46 ` [PATCH 05/11] drm/amdgpu/mes: update hqd masks when disable_kq is set Alex Deucher
` (6 subsequent siblings)
10 siblings, 0 replies; 20+ messages in thread
From: Alex Deucher @ 2025-03-06 18:46 UTC (permalink / raw)
To: amd-gfx; +Cc: Alex Deucher
Move it to amdgpu_mes to align with the compute and
sdma hqd masks. No functional change.
v2: rebase on new changes
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 22 ++++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 16 +++-------------
drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 15 +++------------
3 files changed, 28 insertions(+), 25 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index ca076306adba4..5913c5ba85ed0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -144,6 +144,28 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
adev->mes.vmid_mask_mmhub = 0xffffff00;
adev->mes.vmid_mask_gfxhub = 0xffffff00;
+ for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++) {
+ /* use only 1st ME pipe */
+ if (i >= adev->gfx.me.num_pipe_per_me)
+ continue;
+ if (amdgpu_ip_version(adev, GC_HWIP, 0) >=
+ IP_VERSION(12, 0, 0))
+ /*
+ * GFX V12 has only one GFX pipe, but 8 queues in it.
+ * GFX pipe 0 queue 0 is being used by Kernel queue.
+ * Set GFX pipe 0 queue 1-7 for MES scheduling
+ * mask = 1111 1110b
+ */
+ adev->mes.gfx_hqd_mask[i] = 0xFE;
+ else
+ /*
+ * GFX pipe 0 queue 0 is being used by Kernel queue.
+ * Set GFX pipe 0 queue 1 for MES scheduling
+ * mask = 10b
+ */
+ adev->mes.gfx_hqd_mask[i] = 0x2;
+ }
+
for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) {
/* use only 1st MEC pipes */
if (i >= adev->gfx.mec.num_pipe_per_mec)
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
index a569d09a1a748..39b45d8b5f049 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
@@ -669,18 +669,6 @@ static int mes_v11_0_misc_op(struct amdgpu_mes *mes,
offsetof(union MESAPI__MISC, api_status));
}
-static void mes_v11_0_set_gfx_hqd_mask(union MESAPI_SET_HW_RESOURCES *pkt)
-{
- /*
- * GFX pipe 0 queue 0 is being used by Kernel queue.
- * Set GFX pipe 0 queue 1 for MES scheduling
- * mask = 10b
- * GFX pipe 1 can't be used for MES due to HW limitation.
- */
- pkt->gfx_hqd_mask[0] = 0x2;
- pkt->gfx_hqd_mask[1] = 0;
-}
-
static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
{
int i;
@@ -705,7 +693,9 @@ static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
mes_set_hw_res_pkt.compute_hqd_mask[i] =
mes->compute_hqd_mask[i];
- mes_v11_0_set_gfx_hqd_mask(&mes_set_hw_res_pkt);
+ for (i = 0; i < MAX_GFX_PIPES; i++)
+ mes_set_hw_res_pkt.gfx_hqd_mask[i] =
+ mes->gfx_hqd_mask[i];
for (i = 0; i < MAX_SDMA_PIPES; i++)
mes_set_hw_res_pkt.sdma_hqd_mask[i] = mes->sdma_hqd_mask[i];
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
index 96336652d14c5..519f054bec60d 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
@@ -694,17 +694,6 @@ static int mes_v12_0_set_hw_resources_1(struct amdgpu_mes *mes, int pipe)
offsetof(union MESAPI_SET_HW_RESOURCES_1, api_status));
}
-static void mes_v12_0_set_gfx_hqd_mask(union MESAPI_SET_HW_RESOURCES *pkt)
-{
- /*
- * GFX V12 has only one GFX pipe, but 8 queues in it.
- * GFX pipe 0 queue 0 is being used by Kernel queue.
- * Set GFX pipe 0 queue 1-7 for MES scheduling
- * mask = 1111 1110b
- */
- pkt->gfx_hqd_mask[0] = 0xFE;
-}
-
static int mes_v12_0_set_hw_resources(struct amdgpu_mes *mes, int pipe)
{
int i;
@@ -727,7 +716,9 @@ static int mes_v12_0_set_hw_resources(struct amdgpu_mes *mes, int pipe)
mes_set_hw_res_pkt.compute_hqd_mask[i] =
mes->compute_hqd_mask[i];
- mes_v12_0_set_gfx_hqd_mask(&mes_set_hw_res_pkt);
+ for (i = 0; i < MAX_GFX_PIPES; i++)
+ mes_set_hw_res_pkt.gfx_hqd_mask[i] =
+ mes->gfx_hqd_mask[i];
for (i = 0; i < MAX_SDMA_PIPES; i++)
mes_set_hw_res_pkt.sdma_hqd_mask[i] =
--
2.48.1
^ permalink raw reply related [flat|nested] 20+ messages in thread* [PATCH 05/11] drm/amdgpu/mes: update hqd masks when disable_kq is set
2025-03-06 18:46 [PATCH V2 00/11] Add disable kernel queue support Alex Deucher
` (3 preceding siblings ...)
2025-03-06 18:46 ` [PATCH 04/11] drm/amdgpu/mes: centralize gfx_hqd mask management Alex Deucher
@ 2025-03-06 18:46 ` Alex Deucher
2025-03-06 18:46 ` [PATCH 06/11] drm/amdgpu/mes: make more vmids available when disable_kq=1 Alex Deucher
` (5 subsequent siblings)
10 siblings, 0 replies; 20+ messages in thread
From: Alex Deucher @ 2025-03-06 18:46 UTC (permalink / raw)
To: amd-gfx; +Cc: Alex Deucher, Sunil Khatri
Make all resources available to user queues.
Suggested-by: Sunil Khatri <sunil.khatri@amd.com>
Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index 5913c5ba85ed0..e585e8690edf0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -156,21 +156,21 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
* Set GFX pipe 0 queue 1-7 for MES scheduling
* mask = 1111 1110b
*/
- adev->mes.gfx_hqd_mask[i] = 0xFE;
+ adev->mes.gfx_hqd_mask[i] = adev->gfx.disable_kq ? 0xFF : 0xFE;
else
/*
* GFX pipe 0 queue 0 is being used by Kernel queue.
* Set GFX pipe 0 queue 1 for MES scheduling
* mask = 10b
*/
- adev->mes.gfx_hqd_mask[i] = 0x2;
+ adev->mes.gfx_hqd_mask[i] = adev->gfx.disable_kq ? 0x3 : 0x2;
}
for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) {
/* use only 1st MEC pipes */
if (i >= adev->gfx.mec.num_pipe_per_mec)
continue;
- adev->mes.compute_hqd_mask[i] = 0xc;
+ adev->mes.compute_hqd_mask[i] = adev->gfx.disable_kq ? 0xF : 0xC;
}
for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) {
--
2.48.1
^ permalink raw reply related [flat|nested] 20+ messages in thread* [PATCH 06/11] drm/amdgpu/mes: make more vmids available when disable_kq=1
2025-03-06 18:46 [PATCH V2 00/11] Add disable kernel queue support Alex Deucher
` (4 preceding siblings ...)
2025-03-06 18:46 ` [PATCH 05/11] drm/amdgpu/mes: update hqd masks when disable_kq is set Alex Deucher
@ 2025-03-06 18:46 ` Alex Deucher
2025-03-06 18:46 ` [PATCH 07/11] drm/amdgpu/gfx11: add support for disable_kq Alex Deucher
` (4 subsequent siblings)
10 siblings, 0 replies; 20+ messages in thread
From: Alex Deucher @ 2025-03-06 18:46 UTC (permalink / raw)
To: amd-gfx; +Cc: Alex Deucher, Sunil Khatri
If we don't have kernel queues, the vmids can be used by
the MES for user queues.
Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 2 +-
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 2 +-
drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index e585e8690edf0..d7cdd2895889a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -142,7 +142,7 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;
adev->mes.vmid_mask_mmhub = 0xffffff00;
- adev->mes.vmid_mask_gfxhub = 0xffffff00;
+ adev->mes.vmid_mask_gfxhub = adev->gfx.disable_kq ? 0xfffffffe : 0xffffff00;
for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++) {
/* use only 1st ME pipe */
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
index 95d894a231fcf..19a5f196829f3 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
@@ -900,7 +900,7 @@ static int gmc_v10_0_sw_init(struct amdgpu_ip_block *ip_block)
* amdgpu graphics/compute will use VMIDs 1-7
* amdkfd will use VMIDs 8-15
*/
- adev->vm_manager.first_kfd_vmid = 8;
+ adev->vm_manager.first_kfd_vmid = adev->gfx.disable_kq ? 1 : 8;
amdgpu_vm_manager_init(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
index ea7c32d8380ba..598324e736092 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
@@ -837,7 +837,7 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
* amdgpu graphics/compute will use VMIDs 1-7
* amdkfd will use VMIDs 8-15
*/
- adev->vm_manager.first_kfd_vmid = 8;
+ adev->vm_manager.first_kfd_vmid = adev->gfx.disable_kq ? 1 : 8;
amdgpu_vm_manager_init(adev);
--
2.48.1
^ permalink raw reply related [flat|nested] 20+ messages in thread* [PATCH 07/11] drm/amdgpu/gfx11: add support for disable_kq
2025-03-06 18:46 [PATCH V2 00/11] Add disable kernel queue support Alex Deucher
` (5 preceding siblings ...)
2025-03-06 18:46 ` [PATCH 06/11] drm/amdgpu/mes: make more vmids available when disable_kq=1 Alex Deucher
@ 2025-03-06 18:46 ` Alex Deucher
2025-03-06 18:46 ` [PATCH 08/11] drm/amdgpu/gfx12: " Alex Deucher
` (3 subsequent siblings)
10 siblings, 0 replies; 20+ messages in thread
From: Alex Deucher @ 2025-03-06 18:46 UTC (permalink / raw)
To: amd-gfx; +Cc: Alex Deucher
Plumb in support for disabling kernel queues in
GFX11. We have to bring up a GFX queue briefly in
order to initialize the clear state. After that
we can disable it.
v2: use ring counts per Felix' suggestion
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 99 +++++++++++++++++---------
1 file changed, 65 insertions(+), 34 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
index 95eefd9a40d28..b20624f8cbbbd 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
@@ -1145,6 +1145,10 @@ static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
ring->ring_obj = NULL;
ring->use_doorbell = true;
+ if (adev->gfx.disable_kq) {
+ ring->no_scheduler = true;
+ ring->no_user_submission = true;
+ }
if (!ring_id)
ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
@@ -1577,7 +1581,7 @@ static void gfx_v11_0_alloc_ip_dump(struct amdgpu_device *adev)
static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
{
- int i, j, k, r, ring_id = 0;
+ int i, j, k, r, ring_id;
int xcc_id = 0;
struct amdgpu_device *adev = ip_block->adev;
@@ -1710,37 +1714,42 @@ static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
return r;
}
- /* set up the gfx ring */
- for (i = 0; i < adev->gfx.me.num_me; i++) {
- for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
- for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
- if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
- continue;
-
- r = gfx_v11_0_gfx_ring_init(adev, ring_id,
- i, k, j);
- if (r)
- return r;
- ring_id++;
+ if (adev->gfx.num_gfx_rings) {
+ ring_id = 0;
+ /* set up the gfx ring */
+ for (i = 0; i < adev->gfx.me.num_me; i++) {
+ for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
+ for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
+ if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
+ continue;
+
+ r = gfx_v11_0_gfx_ring_init(adev, ring_id,
+ i, k, j);
+ if (r)
+ return r;
+ ring_id++;
+ }
}
}
}
- ring_id = 0;
- /* set up the compute queues - allocate horizontally across pipes */
- for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
- for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
- for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
- if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
- k, j))
- continue;
+ if (adev->gfx.num_compute_rings) {
+ ring_id = 0;
+ /* set up the compute queues - allocate horizontally across pipes */
+ for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
+ for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
+ for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
+ if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
+ k, j))
+ continue;
- r = gfx_v11_0_compute_ring_init(adev, ring_id,
- i, k, j);
- if (r)
- return r;
+ r = gfx_v11_0_compute_ring_init(adev, ring_id,
+ i, k, j);
+ if (r)
+ return r;
- ring_id++;
+ ring_id++;
+ }
}
}
}
@@ -4578,11 +4587,22 @@ static int gfx_v11_0_cp_resume(struct amdgpu_device *adev)
return r;
}
- for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
- ring = &adev->gfx.gfx_ring[i];
- r = amdgpu_ring_test_helper(ring);
- if (r)
- return r;
+ if (adev->gfx.disable_kq) {
+ for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
+ ring = &adev->gfx.gfx_ring[i];
+ /* we don't want to set ring->ready */
+ r = amdgpu_ring_test_ring(ring);
+ if (r)
+ return r;
+ }
+ amdgpu_gfx_disable_kgq(adev, 0);
+ } else {
+ for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
+ ring = &adev->gfx.gfx_ring[i];
+ r = amdgpu_ring_test_helper(ring);
+ if (r)
+ return r;
+ }
}
for (i = 0; i < adev->gfx.num_compute_rings; i++) {
@@ -5128,11 +5148,22 @@ static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block)
{
struct amdgpu_device *adev = ip_block->adev;
+ if (amdgpu_disable_kq == 1)
+ adev->gfx.disable_kq = true;
+
adev->gfx.funcs = &gfx_v11_0_gfx_funcs;
- adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS;
- adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
- AMDGPU_MAX_COMPUTE_RINGS);
+ if (adev->gfx.disable_kq) {
+ /* We need one GFX ring temporarily to set up
+ * the clear state.
+ */
+ adev->gfx.num_gfx_rings = 1;
+ adev->gfx.num_compute_rings = 0;
+ } else {
+ adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS;
+ adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
+ AMDGPU_MAX_COMPUTE_RINGS);
+ }
gfx_v11_0_set_kiq_pm4_funcs(adev);
gfx_v11_0_set_ring_funcs(adev);
--
2.48.1
^ permalink raw reply related [flat|nested] 20+ messages in thread* [PATCH 08/11] drm/amdgpu/gfx12: add support for disable_kq
2025-03-06 18:46 [PATCH V2 00/11] Add disable kernel queue support Alex Deucher
` (6 preceding siblings ...)
2025-03-06 18:46 ` [PATCH 07/11] drm/amdgpu/gfx11: add support for disable_kq Alex Deucher
@ 2025-03-06 18:46 ` Alex Deucher
2025-03-06 18:46 ` [PATCH 09/11] drm/amdgpu/sdma: add flag for tracking disable_kq Alex Deucher
` (2 subsequent siblings)
10 siblings, 0 replies; 20+ messages in thread
From: Alex Deucher @ 2025-03-06 18:46 UTC (permalink / raw)
To: amd-gfx; +Cc: Alex Deucher
Plumb in support for disabling kernel queues.
v2: use ring counts per Felix' suggestion
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c | 96 ++++++++++++++++----------
1 file changed, 58 insertions(+), 38 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
index 34cf187e72d9f..23ee4651cbffb 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
@@ -1421,11 +1421,13 @@ static int gfx_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
break;
}
- /* recalculate compute rings to use based on hardware configuration */
- num_compute_rings = (adev->gfx.mec.num_pipe_per_mec *
- adev->gfx.mec.num_queue_per_pipe) / 2;
- adev->gfx.num_compute_rings = min(adev->gfx.num_compute_rings,
- num_compute_rings);
+ if (adev->gfx.num_compute_rings) {
+ /* recalculate compute rings to use based on hardware configuration */
+ num_compute_rings = (adev->gfx.mec.num_pipe_per_mec *
+ adev->gfx.mec.num_queue_per_pipe) / 2;
+ adev->gfx.num_compute_rings = min(adev->gfx.num_compute_rings,
+ num_compute_rings);
+ }
/* EOP Event */
r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
@@ -1471,37 +1473,41 @@ static int gfx_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
return r;
}
- /* set up the gfx ring */
- for (i = 0; i < adev->gfx.me.num_me; i++) {
- for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
- for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
- if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
- continue;
-
- r = gfx_v12_0_gfx_ring_init(adev, ring_id,
- i, k, j);
- if (r)
- return r;
- ring_id++;
+ if (adev->gfx.num_gfx_rings) {
+ /* set up the gfx ring */
+ for (i = 0; i < adev->gfx.me.num_me; i++) {
+ for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
+ for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
+ if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
+ continue;
+
+ r = gfx_v12_0_gfx_ring_init(adev, ring_id,
+ i, k, j);
+ if (r)
+ return r;
+ ring_id++;
+ }
}
}
}
- ring_id = 0;
- /* set up the compute queues - allocate horizontally across pipes */
- for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
- for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
- for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
- if (!amdgpu_gfx_is_mec_queue_enabled(adev,
- 0, i, k, j))
- continue;
+ if (adev->gfx.num_compute_rings) {
+ ring_id = 0;
+ /* set up the compute queues - allocate horizontally across pipes */
+ for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
+ for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
+ for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
+ if (!amdgpu_gfx_is_mec_queue_enabled(adev,
+ 0, i, k, j))
+ continue;
- r = gfx_v12_0_compute_ring_init(adev, ring_id,
- i, k, j);
- if (r)
- return r;
+ r = gfx_v12_0_compute_ring_init(adev, ring_id,
+ i, k, j);
+ if (r)
+ return r;
- ring_id++;
+ ring_id++;
+ }
}
}
}
@@ -3495,12 +3501,18 @@ static int gfx_v12_0_cp_resume(struct amdgpu_device *adev)
if (r)
return r;
- if (!amdgpu_async_gfx_ring) {
- r = gfx_v12_0_cp_gfx_resume(adev);
- if (r)
- return r;
+ if (adev->gfx.num_gfx_rings) {
+ if (!amdgpu_async_gfx_ring) {
+ r = gfx_v12_0_cp_gfx_resume(adev);
+ if (r)
+ return r;
+ } else {
+ r = gfx_v12_0_cp_async_gfx_ring_resume(adev);
+ if (r)
+ return r;
+ }
} else {
- r = gfx_v12_0_cp_async_gfx_ring_resume(adev);
+ r = gfx_v12_0_cp_gfx_start(adev);
if (r)
return r;
}
@@ -3809,11 +3821,19 @@ static int gfx_v12_0_early_init(struct amdgpu_ip_block *ip_block)
{
struct amdgpu_device *adev = ip_block->adev;
+ if (amdgpu_disable_kq == 1)
+ adev->gfx.disable_kq = true;
+
adev->gfx.funcs = &gfx_v12_0_gfx_funcs;
- adev->gfx.num_gfx_rings = GFX12_NUM_GFX_RINGS;
- adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
- AMDGPU_MAX_COMPUTE_RINGS);
+ if (adev->gfx.disable_kq) {
+ adev->gfx.num_gfx_rings = 0;
+ adev->gfx.num_compute_rings = 0;
+ } else {
+ adev->gfx.num_gfx_rings = GFX12_NUM_GFX_RINGS;
+ adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
+ AMDGPU_MAX_COMPUTE_RINGS);
+ }
gfx_v12_0_set_kiq_pm4_funcs(adev);
gfx_v12_0_set_ring_funcs(adev);
--
2.48.1
^ permalink raw reply related [flat|nested] 20+ messages in thread* [PATCH 09/11] drm/amdgpu/sdma: add flag for tracking disable_kq
2025-03-06 18:46 [PATCH V2 00/11] Add disable kernel queue support Alex Deucher
` (7 preceding siblings ...)
2025-03-06 18:46 ` [PATCH 08/11] drm/amdgpu/gfx12: " Alex Deucher
@ 2025-03-06 18:46 ` Alex Deucher
2025-03-06 18:46 ` [PATCH 10/11] drm/amdgpu/sdma6: add support for disable_kq Alex Deucher
2025-03-06 18:46 ` [PATCH 11/11] drm/amdgpu/sdma7: " Alex Deucher
10 siblings, 0 replies; 20+ messages in thread
From: Alex Deucher @ 2025-03-06 18:46 UTC (permalink / raw)
To: amd-gfx; +Cc: Alex Deucher, Sunil Khatri
For SDMA, we still need kernel queues for paging so
they need to be initialized, but we no not want to
accept submissions from userspace when disable_kq
is set.
Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h | 1 +
1 file changed, 1 insertion(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
index 9651693200655..edc856e10337a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
@@ -129,6 +129,7 @@ struct amdgpu_sdma {
/* track guilty state of GFX and PAGE queues */
bool gfx_guilty;
bool page_guilty;
+ bool no_user_submission;
};
/*
--
2.48.1
^ permalink raw reply related [flat|nested] 20+ messages in thread* [PATCH 10/11] drm/amdgpu/sdma6: add support for disable_kq
2025-03-06 18:46 [PATCH V2 00/11] Add disable kernel queue support Alex Deucher
` (8 preceding siblings ...)
2025-03-06 18:46 ` [PATCH 09/11] drm/amdgpu/sdma: add flag for tracking disable_kq Alex Deucher
@ 2025-03-06 18:46 ` Alex Deucher
2025-03-06 18:46 ` [PATCH 11/11] drm/amdgpu/sdma7: " Alex Deucher
10 siblings, 0 replies; 20+ messages in thread
From: Alex Deucher @ 2025-03-06 18:46 UTC (permalink / raw)
To: amd-gfx; +Cc: Alex Deucher, Sunil Khatri
When the parameter is set, disable user submissions
to kernel queues.
Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
index 3aa4fec4d9e4a..bcc72737f8084 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
@@ -1304,6 +1304,9 @@ static int sdma_v6_0_early_init(struct amdgpu_ip_block *ip_block)
struct amdgpu_device *adev = ip_block->adev;
int r;
+ if (amdgpu_disable_kq == 1)
+ adev->sdma.no_user_submission = true;
+
r = amdgpu_sdma_init_microcode(adev, 0, true);
if (r)
return r;
@@ -1338,6 +1341,7 @@ static int sdma_v6_0_sw_init(struct amdgpu_ip_block *ip_block)
ring->ring_obj = NULL;
ring->use_doorbell = true;
ring->me = i;
+ ring->no_user_submission = adev->sdma.no_user_submission;
DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
ring->use_doorbell?"true":"false");
--
2.48.1
^ permalink raw reply related [flat|nested] 20+ messages in thread* [PATCH 11/11] drm/amdgpu/sdma7: add support for disable_kq
2025-03-06 18:46 [PATCH V2 00/11] Add disable kernel queue support Alex Deucher
` (9 preceding siblings ...)
2025-03-06 18:46 ` [PATCH 10/11] drm/amdgpu/sdma6: add support for disable_kq Alex Deucher
@ 2025-03-06 18:46 ` Alex Deucher
10 siblings, 0 replies; 20+ messages in thread
From: Alex Deucher @ 2025-03-06 18:46 UTC (permalink / raw)
To: amd-gfx; +Cc: Alex Deucher, Sunil Khatri
When the parameter is set, disable user submissions
to kernel queues.
Reviewed-by: Sunil Khatri <sunil.khatri@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
index 92a79296708ae..40d45f738c0a8 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
@@ -1316,6 +1316,9 @@ static int sdma_v7_0_early_init(struct amdgpu_ip_block *ip_block)
struct amdgpu_device *adev = ip_block->adev;
int r;
+ if (amdgpu_disable_kq == 1)
+ adev->sdma.no_user_submission = true;
+
r = amdgpu_sdma_init_microcode(adev, 0, true);
if (r) {
DRM_ERROR("Failed to init sdma firmware!\n");
@@ -1351,6 +1354,7 @@ static int sdma_v7_0_sw_init(struct amdgpu_ip_block *ip_block)
ring->ring_obj = NULL;
ring->use_doorbell = true;
ring->me = i;
+ ring->no_user_submission = adev->sdma.no_user_submission;
DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
ring->use_doorbell?"true":"false");
--
2.48.1
^ permalink raw reply related [flat|nested] 20+ messages in thread