* [PATCH 1/7] drm/amdgpu: Add CWSR infrastructure for user queues
2026-10-08 7:05 [PATCH 0/7] drm/amdgpu: Add CWSR and second-level trap handler support Srinivasan Shanmugam
@ 2026-10-08 7:05 ` Srinivasan Shanmugam
2026-10-12 11:47 ` Christian König
2026-10-08 7:05 ` [PATCH 2/7] drm/amdgpu: Integrate CWSR with " Srinivasan Shanmugam
` (5 subsequent siblings)
6 siblings, 1 reply; 16+ messages in thread
From: Srinivasan Shanmugam @ 2026-10-08 7:05 UTC (permalink / raw)
To: Christian König, Alex Deucher
Cc: amd-gfx, Lijo Lazar, Srinivasan Shanmugam
From: Lijo Lazar <lijo.lazar@amd.com>
Add a helper to get the number of XCCs for a partition, defaulting to
one when no partition manager is available.
Add the core CWSR handling infrastructure for user queues, including
the first-level trap handler objects and save area sizing.
Calculate the control stack, wave context and debugger memory sizes
required for CWSR, and provide validation for userspace supplied save
area parameters. Address validation is handled separately by the
caller.
Create the per-VM TMA BO using the VM root BO reservation object. This
avoids recursively acquiring reservation_ww_class_mutex while the VM
reservation is already held through drm_exec. The TMA BO is not pinned
and is covered by the VM reservation while being added to the VM.
Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
Co-developed-by: Srinivasan Shanmugam <srinivasan.shanmugam@amd.com>
Signed-off-by: Srinivasan Shanmugam <srinivasan.shanmugam@amd.com>
---
drivers/gpu/drm/amd/amdgpu/Makefile | 2 +-
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 3 +
drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c | 534 +++++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h | 81 ++++
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 19 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 13 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h | 22 +
7 files changed, 660 insertions(+), 14 deletions(-)
create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c
create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 55b5d11f1938..5086a482daf1 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -72,7 +72,7 @@ amdgpu-y += amdgpu_device.o amdgpu_reg_access.o amdgpu_doorbell_mgr.o amdgpu_kms
amdgpu_eeprom.o amdgpu_mca.o amdgpu_psp_ta.o amdgpu_lsdma.o amdgpu_lockdep.o \
amdgpu_ring_mux.o amdgpu_xcp.o amdgpu_seq64.o amdgpu_dev_coredump.o \
amdgpu_cper.o amdgpu_userq_fence.o amdgpu_eviction_fence.o amdgpu_ip.o \
- amdgpu_wb.o
+ amdgpu_wb.o amdgpu_cwsr.o
amdgpu-$(CONFIG_PROC_FS) += amdgpu_fdinfo.o
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index ce671731b700..26561a4f4db8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -332,6 +332,7 @@ struct amdgpu_reset_context;
struct amdgpu_reset_control;
struct amdgpu_coredump_info;
struct amdgpu_video_codecs;
+struct amdgpu_cwsr_isa;
enum amdgpu_cp_irq {
AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP = 0,
@@ -974,6 +975,8 @@ struct amdgpu_device {
* Must be last --ends in a flexible-array member.
*/
struct amdgpu_kfd_dev kfd;
+
+ struct amdgpu_cwsr_info *cwsr_info;
};
static inline uint32_t amdgpu_ip_version(const struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c
new file mode 100644
index 000000000000..6b6c11bb5fd7
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c
@@ -0,0 +1,534 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <drm/drm_exec.h>
+
+#include "amdgpu.h"
+#include "cwsr_trap_handler.h"
+#include "amdgpu_cwsr.h"
+
+extern int cwsr_enable;
+
+#define AMDGPU_CWSR_TBA_MAX_SIZE (2 * AMDGPU_GPU_PAGE_SIZE)
+#define AMDGPU_CWSR_TMA_MAX_SIZE (AMDGPU_GPU_PAGE_SIZE)
+#define AMDGPU_CWSR_TMA_OFFSET (AMDGPU_CWSR_TBA_MAX_SIZE)
+
+#define SGPR_SIZE_PER_CU 0x4000
+#define LDS_SIZE_PER_CU 0x10000
+#define HWREG_SIZE_PER_CU 0x1000
+#define DEBUGGER_BYTES_ALIGN 64
+#define DEBUGGER_BYTES_PER_WAVE 32
+#define SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER 40
+
+enum amdgpu_cwsr_region {
+ AMDGPU_CWSR_TBA,
+ AMDGPU_CWSR_TMA,
+};
+
+static inline uint64_t amdgpu_cwsr_tba_vaddr(struct amdgpu_device *adev)
+{
+ u64 addr = AMDGPU_VA_RESERVED_TRAP_UQ_START(adev);
+
+ addr = amdgpu_gmc_sign_extend(addr);
+
+ return addr;
+}
+
+static inline bool amdgpu_cwsr_is_supported(struct amdgpu_device *adev)
+{
+ u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
+
+ if (!cwsr_enable || adev->gfx.disable_uq ||
+ gc_ver < IP_VERSION(9, 0, 1))
+ return false;
+
+ return true;
+}
+
+uint32_t amdgpu_cwsr_size_needed(struct amdgpu_device *adev, int num_xcc)
+{
+ if (!amdgpu_cwsr_is_enabled(adev))
+ return 0;
+
+ return num_xcc *
+ (adev->cwsr_info->xcc_cwsr_sz + adev->cwsr_info->xcc_dbg_mem_sz);
+}
+
+static void amdgpu_cwsr_init_isa_details(struct amdgpu_device *adev,
+ struct amdgpu_cwsr_info *cwsr_info)
+{
+ u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
+
+ if (gc_ver < IP_VERSION(9, 0, 1)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) >
+ AMDGPU_CWSR_TBA_MAX_SIZE);
+ cwsr_info->isa_buf = cwsr_trap_gfx8_hex;
+ cwsr_info->isa_sz = sizeof(cwsr_trap_gfx8_hex);
+ } else if (gc_ver == IP_VERSION(9, 4, 1)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex) >
+ AMDGPU_CWSR_TBA_MAX_SIZE);
+ cwsr_info->isa_buf = cwsr_trap_arcturus_hex;
+ cwsr_info->isa_sz = sizeof(cwsr_trap_arcturus_hex);
+ } else if (gc_ver == IP_VERSION(9, 4, 2)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_aldebaran_hex) >
+ AMDGPU_CWSR_TBA_MAX_SIZE);
+ cwsr_info->isa_buf = cwsr_trap_aldebaran_hex;
+ cwsr_info->isa_sz = sizeof(cwsr_trap_aldebaran_hex);
+ } else if (gc_ver == IP_VERSION(9, 4, 3) ||
+ gc_ver == IP_VERSION(9, 4, 4)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_4_3_hex) >
+ AMDGPU_CWSR_TBA_MAX_SIZE);
+ cwsr_info->isa_buf = cwsr_trap_gfx9_4_3_hex;
+ cwsr_info->isa_sz = sizeof(cwsr_trap_gfx9_4_3_hex);
+ } else if (gc_ver == IP_VERSION(9, 5, 0)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_5_0_hex) > PAGE_SIZE);
+ cwsr_info->isa_buf = cwsr_trap_gfx9_5_0_hex;
+ cwsr_info->isa_sz = sizeof(cwsr_trap_gfx9_5_0_hex);
+ } else if (gc_ver < IP_VERSION(10, 1, 1)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) >
+ AMDGPU_CWSR_TBA_MAX_SIZE);
+ cwsr_info->isa_buf = cwsr_trap_gfx9_hex;
+ cwsr_info->isa_sz = sizeof(cwsr_trap_gfx9_hex);
+ } else if (gc_ver < IP_VERSION(10, 3, 0)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_nv1x_hex) >
+ AMDGPU_CWSR_TBA_MAX_SIZE);
+ cwsr_info->isa_buf = cwsr_trap_nv1x_hex;
+ cwsr_info->isa_sz = sizeof(cwsr_trap_nv1x_hex);
+ } else if (gc_ver < IP_VERSION(11, 0, 0)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex) >
+ AMDGPU_CWSR_TBA_MAX_SIZE);
+ cwsr_info->isa_buf = cwsr_trap_gfx10_hex;
+ cwsr_info->isa_sz = sizeof(cwsr_trap_gfx10_hex);
+ } else if (gc_ver < IP_VERSION(12, 0, 0)) {
+ /* The gfx11 cwsr trap handler must fit inside a single
+ page. */
+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx11_hex) > PAGE_SIZE);
+ cwsr_info->isa_buf = cwsr_trap_gfx11_hex;
+ cwsr_info->isa_sz = sizeof(cwsr_trap_gfx11_hex);
+ } else if (gc_ver < IP_VERSION(12, 1, 0)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_hex) >
+ AMDGPU_CWSR_TBA_MAX_SIZE);
+ cwsr_info->isa_buf = cwsr_trap_gfx12_hex;
+ cwsr_info->isa_sz = sizeof(cwsr_trap_gfx12_hex);
+ } else {
+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_1_0_hex) >
+ AMDGPU_CWSR_TBA_MAX_SIZE);
+ cwsr_info->isa_buf = cwsr_trap_gfx12_1_0_hex;
+ cwsr_info->isa_sz = sizeof(cwsr_trap_gfx12_1_0_hex);
+ }
+}
+
+static uint32_t amdgpu_cwsr_get_vgpr_size_per_cu(struct amdgpu_device *adev)
+{
+ u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
+ u32 vgpr_size;
+
+ switch (gc_ver) {
+ case IP_VERSION(9, 4, 1): /* GFX_VERSION_ARCTURUS */
+ case IP_VERSION(9, 4, 2): /* GFX_VERSION_ALDEBARAN */
+ case IP_VERSION(9, 4, 3): /* GFX_VERSION_AQUA_VANJARAM */
+ case IP_VERSION(9, 4, 4): /* GFX_VERSION_AQUA_VANJARAM */
+ case IP_VERSION(9, 5, 0):
+ vgpr_size = 0x80000;
+ break;
+ case IP_VERSION(11, 0, 0):
+ case IP_VERSION(11, 0, 2):
+ case IP_VERSION(11, 0, 3):
+ case IP_VERSION(12, 0, 0):
+ case IP_VERSION(12, 0, 1):
+ vgpr_size = 0x60000;
+ break;
+ default:
+ vgpr_size = 0x40000;
+ break;
+ }
+
+ return vgpr_size;
+}
+
+static uint32_t amdgpu_cwsr_get_wg_ctxt_size_per_cu(struct amdgpu_device *adev)
+{
+ u32 lds_sz_per_cu;
+
+ lds_sz_per_cu =
+ (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0)) ?
+ (adev->gfx.cu_info.lds_size << 10) :
+ LDS_SIZE_PER_CU;
+
+ return amdgpu_cwsr_get_vgpr_size_per_cu(adev) + SGPR_SIZE_PER_CU +
+ lds_sz_per_cu + HWREG_SIZE_PER_CU;
+}
+
+static uint32_t amdgpu_cwsr_ctl_stack_bytes_per_wave(struct amdgpu_device *adev)
+{
+ u32 sz;
+
+ if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 1, 0))
+ sz = 12;
+ else
+ sz = 8;
+ return sz;
+}
+
+static void amdgpu_cwsr_init_save_area_info(struct amdgpu_device *adev,
+ struct amdgpu_cwsr_info *cwsr_info)
+{
+ struct amdgpu_gfx_config *gfx_info = &adev->gfx.config;
+ u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
+ u32 ctl_stack_size, wg_data_size, dbg_mem_size;
+ u32 array_count;
+ u32 wave_num;
+ u32 cu_num;
+
+ if (gc_ver < IP_VERSION(9, 0, 1))
+ return;
+
+ array_count = gfx_info->max_shader_engines * gfx_info->max_sh_per_se;
+
+ cu_num = adev->gfx.cu_info.number / NUM_XCC(adev->gfx.xcc_mask);
+ wave_num = (gc_ver < IP_VERSION(10, 1, 0)) ? /* GFX_VERSION_NAVI10 */
+ min(cu_num * 40,
+ array_count / gfx_info->max_sh_per_se * 512) :
+ cu_num * 32;
+
+ wg_data_size = ALIGN(cu_num * amdgpu_cwsr_get_wg_ctxt_size_per_cu(adev),
+ PAGE_SIZE);
+ ctl_stack_size =
+ wave_num * amdgpu_cwsr_ctl_stack_bytes_per_wave(adev) + 8;
+ ctl_stack_size =
+ ALIGN(SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER + ctl_stack_size,
+ PAGE_SIZE);
+ dbg_mem_size =
+ ALIGN(wave_num * DEBUGGER_BYTES_PER_WAVE, DEBUGGER_BYTES_ALIGN);
+ /*
+ * HW design limits control stack size to 0x7000.
+ * This is insufficient for theoretical PM4 cases
+ * but sufficient for AQL, limited by SPI events.
+ */
+ if (IP_VERSION_MAJ(gc_ver) == 10)
+ ctl_stack_size = min(ctl_stack_size, 0x7000);
+
+ cwsr_info->xcc_ctl_stack_sz = ctl_stack_size;
+ cwsr_info->xcc_cwsr_sz = ctl_stack_size + wg_data_size;
+ cwsr_info->xcc_dbg_mem_sz = dbg_mem_size;
+}
+
+int amdgpu_cwsr_init(struct amdgpu_device *adev)
+{
+ struct amdgpu_cwsr_info *cwsr_info __free(kfree) =
+ kzalloc(sizeof(*cwsr_info), GFP_KERNEL);
+ void *ptr;
+ int r;
+
+ if (!amdgpu_cwsr_is_supported(adev))
+ return -EOPNOTSUPP;
+
+ if (!cwsr_info)
+ return -ENOMEM;
+ amdgpu_cwsr_init_isa_details(adev, cwsr_info);
+
+ if (!cwsr_info->isa_sz)
+ return -EOPNOTSUPP;
+
+ r = amdgpu_bo_create_kernel(adev, AMDGPU_CWSR_TBA_MAX_SIZE, PAGE_SIZE,
+ AMDGPU_GEM_DOMAIN_GTT, &cwsr_info->isa_bo,
+ NULL, &ptr);
+ if (r)
+ return r;
+
+ memcpy(ptr, cwsr_info->isa_buf, cwsr_info->isa_sz);
+
+ amdgpu_cwsr_init_save_area_info(adev, cwsr_info);
+ adev->cwsr_info = no_free_ptr(cwsr_info);
+
+ return 0;
+}
+
+void amdgpu_cwsr_fini(struct amdgpu_device *adev)
+{
+ if (!amdgpu_cwsr_is_enabled(adev))
+ return;
+
+ amdgpu_bo_free_kernel(&adev->cwsr_info->isa_bo, NULL, NULL);
+ kfree(adev->cwsr_info);
+ adev->cwsr_info = NULL;
+}
+
+/*
+ * amdgpu_map_cwsr_trap_handler should be called during amdgpu_vm_init
+ * it maps virtual address amdgpu_cwsr_trap_handler_vaddr() to this VM, and each
+ * compute queue can use this virtual address for wave save/restore
+ * operations to support compute preemption.
+ */
+static int amdgpu_cwsr_map_region(struct amdgpu_device *adev,
+ struct amdgpu_vm *vm,
+ struct amdgpu_cwsr_trap_obj *cwsr,
+ enum amdgpu_cwsr_region region)
+{
+ u64 cwsr_addr, va_flags, va;
+ struct amdgpu_bo_va **bo_va;
+ struct amdgpu_bo *bo;
+ u32 size;
+ int r;
+
+ if (!cwsr || !vm)
+ return -EINVAL;
+
+ cwsr_addr = amdgpu_cwsr_tba_vaddr(adev);
+
+ if (region == AMDGPU_CWSR_TBA) {
+ size = AMDGPU_CWSR_TBA_MAX_SIZE;
+ bo_va = &cwsr->tba_va;
+ bo = adev->cwsr_info->isa_bo;
+ va = cwsr_addr;
+ va_flags = (AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
+ AMDGPU_VM_PAGE_EXECUTABLE);
+ } else {
+ size = AMDGPU_CWSR_TMA_MAX_SIZE;
+ bo_va = &cwsr->tma_va;
+ bo = cwsr->tma_bo;
+ va = cwsr_addr + AMDGPU_CWSR_TMA_OFFSET;
+ va_flags = (AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE);
+ }
+
+ *bo_va = amdgpu_vm_bo_add(adev, vm, bo);
+ if (!*bo_va)
+ return -ENOMEM;
+
+ va &= AMDGPU_GMC_HOLE_MASK;
+ r = amdgpu_vm_bo_map(adev, *bo_va, va, 0, size, va_flags);
+ if (r) {
+ dev_err(adev->dev, "failed to do bo map of %s region, err=%d\n",
+ (region == AMDGPU_CWSR_TBA ? "tba" : "tma"), r);
+ amdgpu_vm_bo_del(adev, *bo_va);
+ *bo_va = NULL;
+ return r;
+ }
+
+ r = amdgpu_vm_bo_update(adev, *bo_va, false);
+ if (r) {
+ dev_err(adev->dev,
+ "failed to do page table update of %s region, err=%d\n",
+ (region == AMDGPU_CWSR_TBA ? "tba" : "tma"), r);
+ amdgpu_vm_bo_del(adev, *bo_va);
+ *bo_va = NULL;
+ return r;
+ }
+
+ if (region == AMDGPU_CWSR_TBA)
+ cwsr->tba_gpu_va_addr = va;
+ else
+ cwsr->tma_gpu_va_addr = va;
+
+ return 0;
+}
+
+static int amdgpu_cwsr_unmap_region(struct amdgpu_device *adev,
+ struct amdgpu_cwsr_trap_obj *cwsr,
+ enum amdgpu_cwsr_region region)
+{
+ struct amdgpu_bo_va **bo_va;
+ u64 va;
+ int r;
+
+ if (!cwsr)
+ return -EINVAL;
+
+ if (region == AMDGPU_CWSR_TBA) {
+ bo_va = &cwsr->tba_va;
+ va = cwsr->tba_gpu_va_addr;
+ } else {
+ bo_va = &cwsr->tma_va;
+ va = cwsr->tma_gpu_va_addr;
+ }
+
+ r = amdgpu_vm_bo_unmap(adev, *bo_va, va);
+ if (r) {
+ dev_err(adev->dev,
+ "failed to do bo_unmap on CWSR trap handler, err=%d\n",
+ r);
+ return r;
+ }
+
+ amdgpu_vm_bo_del(adev, *bo_va);
+ *bo_va = NULL;
+
+ return r;
+}
+
+int amdgpu_cwsr_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+ struct amdgpu_cwsr_trap_obj **trap_obj)
+{
+ struct amdgpu_bo_param bp;
+ struct amdgpu_cwsr_trap_obj *cwsr;
+ struct amdgpu_bo *bo;
+ struct drm_exec exec;
+ int r;
+
+ if (!amdgpu_cwsr_is_enabled(adev))
+ return -EOPNOTSUPP;
+ if (!vm || !trap_obj)
+ return -EINVAL;
+
+ cwsr = kzalloc_obj(*cwsr, GFP_KERNEL);
+ if (!cwsr)
+ return -ENOMEM;
+
+ bo = adev->cwsr_info->isa_bo;
+ drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
+ drm_exec_until_all_locked(&exec) {
+ r = amdgpu_vm_lock_pd(vm, &exec, 0);
+ if (likely(!r))
+ r = drm_exec_lock_obj(&exec, &bo->tbo.base);
+ drm_exec_retry_on_contention(&exec);
+ if (unlikely(r)) {
+ dev_err(adev->dev,
+ "failed to reserve for CWSR allocs: err=%d\n",
+ r);
+ goto err_exec;
+ }
+ }
+
+ /*
+ * Create TMA BO using the VM's already-locked resv so no new
+ * reservation_ww_class_mutex acquisition is needed, avoiding the
+ * recursive ww_mutex deadlock. Unpinned; eviction is handled via
+ * amdgpu_vm_bo_add() inside amdgpu_cwsr_map_region().
+ */
+ memset(&bp, 0, sizeof(bp));
+ bp.size = AMDGPU_CWSR_TMA_MAX_SIZE;
+ bp.byte_align = PAGE_SIZE;
+ bp.domain = AMDGPU_GEM_DOMAIN_GTT;
+ bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC |
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
+ bp.type = ttm_bo_type_kernel;
+ bp.bo_ptr_size = sizeof(struct amdgpu_bo);
+ bp.resv = vm->root.bo->tbo.base.resv;
+
+ r = amdgpu_bo_create(adev, &bp, &cwsr->tma_bo);
+ if (r)
+ goto err_exec;
+
+ /* VM resv already held, kmap directly, no separate reserve needed. */
+ r = amdgpu_bo_kmap(cwsr->tma_bo, &cwsr->tma_cpu_addr);
+ if (r)
+ goto err_tma;
+
+ memset(cwsr->tma_cpu_addr, 0, AMDGPU_CWSR_TMA_MAX_SIZE);
+
+ r = amdgpu_cwsr_map_region(adev, vm, cwsr, AMDGPU_CWSR_TMA);
+ if (r)
+ goto err_kmap;
+
+ r = amdgpu_cwsr_map_region(adev, vm, cwsr, AMDGPU_CWSR_TBA);
+ if (r) {
+ amdgpu_cwsr_unmap_region(adev, cwsr, AMDGPU_CWSR_TMA);
+ goto err_kmap;
+ }
+
+ drm_exec_fini(&exec);
+ *trap_obj = cwsr;
+ return 0;
+
+err_kmap:
+ amdgpu_bo_kunmap(cwsr->tma_bo);
+err_tma:
+ amdgpu_bo_unref(&cwsr->tma_bo);
+err_exec:
+ drm_exec_fini(&exec);
+ kfree(cwsr);
+ *trap_obj = NULL;
+ return r;
+}
+
+int amdgpu_cwsr_validate_params(struct amdgpu_device *adev,
+ struct amdgpu_cwsr_params *cwsr_params,
+ int num_xcc)
+{
+ u32 min_size, max_size;
+
+ if (!amdgpu_cwsr_is_enabled(adev))
+ return -EOPNOTSUPP;
+
+ if (!cwsr_params)
+ return -EINVAL;
+
+ min_size = amdgpu_cwsr_size_needed(adev, num_xcc);
+ max_size = 2 * min_size;
+ /*
+ * Only save area size details checked. Address validation needs to be
+ * carried out separately. User is expected to pass a cwsr buffer area
+ * with size >= ctl stack size + wg data size + debug memory size
+ */
+ if (cwsr_params->cwsr_sz < min_size ||
+ cwsr_params->cwsr_sz > max_size) {
+ dev_dbg(adev->dev,
+ "queue cwsr size 0x%x should be within min %u : max %u\n",
+ cwsr_params->cwsr_sz, min_size, max_size);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+void amdgpu_cwsr_free(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+ struct amdgpu_cwsr_trap_obj **trap_obj)
+{
+ struct amdgpu_bo *tba_bo;
+ struct amdgpu_bo *tma_bo;
+ struct drm_exec exec;
+ int r;
+
+ if (!trap_obj || !*trap_obj || !(*trap_obj)->tma_bo)
+ return;
+
+ tba_bo = adev->cwsr_info->isa_bo;
+ tma_bo = (*trap_obj)->tma_bo;
+
+ if (!tba_bo || !tma_bo)
+ return;
+
+ drm_exec_init(&exec, 0, 0);
+ drm_exec_until_all_locked(&exec) {
+ r = amdgpu_vm_lock_pd(vm, &exec, 0);
+ if (likely(!r))
+ r = drm_exec_lock_obj(&exec, &tba_bo->tbo.base);
+ drm_exec_retry_on_contention(&exec);
+ if (unlikely(r)) {
+ dev_err(adev->dev,
+ "failed to reserve CWSR BOs: err=%d\n", r);
+ goto err;
+ }
+ }
+
+ amdgpu_cwsr_unmap_region(adev, *trap_obj, AMDGPU_CWSR_TBA);
+ amdgpu_cwsr_unmap_region(adev, *trap_obj, AMDGPU_CWSR_TMA);
+
+err:
+ drm_exec_fini(&exec);
+
+ /* TMA BO not pinned, release CPU mapping and free. */
+ amdgpu_bo_kunmap((*trap_obj)->tma_bo);
+ amdgpu_bo_unref(&(*trap_obj)->tma_bo);
+ kfree(*trap_obj);
+ *trap_obj = NULL;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
new file mode 100644
index 000000000000..7f922559e297
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef AMDGPU_CWSR_H
+#define AMDGPU_CWSR_H
+
+#include <linux/types.h>
+
+struct amdgpu_bo;
+struct amdgpu_bo_va;
+struct amdgpu_device;
+struct amdgpu_vm;
+
+/**
+ * struct amdgpu_cwsr_obj - CWSR (Compute Wave Save Restore) buffer tracking
+ * @bo: Buffer object for CWSR area
+ * @bo_va: Buffer object virtual address mapping
+ */
+struct amdgpu_cwsr_trap_obj {
+ u64 tma_gpu_va_addr;
+ u64 tba_gpu_va_addr;
+
+ struct amdgpu_bo *tma_bo;
+ struct amdgpu_bo_va *tba_va;
+ struct amdgpu_bo_va *tma_va;
+ void *tma_cpu_addr;
+};
+
+struct amdgpu_cwsr_info {
+ /* cwsr isa */
+ struct amdgpu_bo *isa_bo;
+ const void *isa_buf;
+ u32 isa_sz;
+ /* cwsr size info per XCC*/
+ u32 xcc_ctl_stack_sz;
+ u32 xcc_dbg_mem_sz;
+ u32 xcc_cwsr_sz;
+};
+
+struct amdgpu_cwsr_params {
+ u64 ctx_save_area_address;
+ /* cwsr size info */
+ u32 cwsr_sz;
+};
+
+int amdgpu_cwsr_init(struct amdgpu_device *adev);
+void amdgpu_cwsr_fini(struct amdgpu_device *adev);
+
+int amdgpu_cwsr_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+ struct amdgpu_cwsr_trap_obj **cwsr_obj);
+void amdgpu_cwsr_free(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+ struct amdgpu_cwsr_trap_obj **cwsr_obj);
+static inline bool amdgpu_cwsr_is_enabled(struct amdgpu_device *adev)
+{
+ return adev->cwsr_info;
+}
+
+uint32_t amdgpu_cwsr_size_needed(struct amdgpu_device *adev, int num_xcc);
+int amdgpu_cwsr_validate_params(struct amdgpu_device *adev,
+ struct amdgpu_cwsr_params *cwsr_params,
+ int num_xcc);
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index ab6f9131ac11..b2a8a8b37006 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -764,6 +764,15 @@ MODULE_PARM_DESC(use_xgmi_p2p,
"Enable XGMI P2P interface (0 = disable; 1 = enable (default))");
module_param_named(use_xgmi_p2p, amdgpu_use_xgmi_p2p, int, 0444);
+/**
+ * DOC: cwsr_enable (int)
+ * CWSR(compute wave store and resume) allows the GPU to preempt shader execution in
+ * the middle of a compute wave. Default is 1 to enable this feature. Setting 0
+ * disables it.
+ */
+int cwsr_enable = 1;
+module_param(cwsr_enable, int, 0444);
+MODULE_PARM_DESC(cwsr_enable, "CWSR enable (0 = Off, 1 = On (Default))");
#ifdef CONFIG_HSA_AMD
/**
@@ -787,16 +796,6 @@ module_param(hws_max_conc_proc, int, 0444);
MODULE_PARM_DESC(hws_max_conc_proc,
"Max # processes HWS can execute concurrently when sched_policy=0 (0 = no concurrency, #VMIDs for KFD = Maximum(default))");
-/**
- * DOC: cwsr_enable (int)
- * CWSR(compute wave store and resume) allows the GPU to preempt shader execution in
- * the middle of a compute wave. Default is 1 to enable this feature. Setting 0
- * disables it as only in non-HWS mode.
- */
-int cwsr_enable = 1;
-module_param(cwsr_enable, int, 0444);
-MODULE_PARM_DESC(cwsr_enable, "CWSR enable (0 = Off (debugging only), 1 = On (Default))");
-
/**
* DOC: max_num_of_queues_per_device (int)
* Maximum number of queues per device. Valid setting is between 1 and 4096. Default
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 8e478d3bda1a..226caa99f6ee 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -178,10 +178,17 @@ struct amdgpu_bo_vm;
#define AMDGPU_VA_RESERVED_TRAP_SIZE (1ULL << 16)
#define AMDGPU_VA_RESERVED_TRAP_START(adev) (AMDGPU_VA_RESERVED_SEQ64_START(adev) \
- AMDGPU_VA_RESERVED_TRAP_SIZE)
+/* TBD: Ideally, existing TRAP VA should do. There is a conflict with KFD mapping that needs to be
+ * resolved. Revisit later.
+ */
+#define AMDGPU_VA_RESERVED_TRAP_UQ_SIZE (3ULL << 12)
+#define AMDGPU_VA_RESERVED_TRAP_UQ_START(adev) \
+ (AMDGPU_VA_RESERVED_TRAP_START(adev) - AMDGPU_VA_RESERVED_TRAP_UQ_SIZE)
+
#define AMDGPU_VA_RESERVED_BOTTOM (1ULL << 16)
-#define AMDGPU_VA_RESERVED_TOP (AMDGPU_VA_RESERVED_TRAP_SIZE + \
- AMDGPU_VA_RESERVED_SEQ64_SIZE + \
- AMDGPU_VA_RESERVED_CSA_SIZE)
+#define AMDGPU_VA_RESERVED_TOP \
+ (AMDGPU_VA_RESERVED_TRAP_UQ_SIZE + AMDGPU_VA_RESERVED_TRAP_SIZE + \
+ AMDGPU_VA_RESERVED_SEQ64_SIZE + AMDGPU_VA_RESERVED_CSA_SIZE)
/* See vm_update_mode */
#define AMDGPU_VM_USE_CPU_FOR_GFX (1 << 0)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
index 33157409eda0..e7c84b50ccf4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
@@ -226,4 +226,26 @@ amdgpu_get_next_xcp(struct amdgpu_xcp_mgr *xcp_mgr, int *from)
for (i = 0, xcp = amdgpu_get_next_xcp(xcp_mgr, &i); xcp; \
++i, xcp = amdgpu_get_next_xcp(xcp_mgr, &i))
+static inline int amdgpu_xcp_get_num_xcc(struct amdgpu_xcp_mgr *xcp_mgr,
+ int xcp_id)
+{
+ struct amdgpu_xcp *xcp;
+ u32 xcc_mask;
+ int i, r;
+
+ if (!xcp_mgr || xcp_id == AMDGPU_XCP_NO_PARTITION)
+ return 1;
+ for_each_xcp(xcp_mgr, xcp, i) {
+ if (xcp->id == xcp_id) {
+ r = amdgpu_xcp_get_inst_details(xcp, AMDGPU_XCP_GFX,
+ &xcc_mask);
+ if (unlikely(r))
+ return 1;
+ else
+ return hweight32(xcc_mask);
+ }
+ }
+
+ return 1;
+}
#endif
--
2.34.1
^ permalink raw reply related [flat|nested] 16+ messages in thread* Re: [PATCH 1/7] drm/amdgpu: Add CWSR infrastructure for user queues
2026-10-08 7:05 ` [PATCH 1/7] drm/amdgpu: Add CWSR infrastructure for user queues Srinivasan Shanmugam
@ 2026-10-12 11:47 ` Christian König
0 siblings, 0 replies; 16+ messages in thread
From: Christian König @ 2026-10-12 11:47 UTC (permalink / raw)
To: Srinivasan Shanmugam, Alex Deucher; +Cc: amd-gfx, Lijo Lazar
On 10/8/26 09:05, Srinivasan Shanmugam wrote:
> From: Lijo Lazar <lijo.lazar@amd.com>
>
> Add a helper to get the number of XCCs for a partition, defaulting to
> one when no partition manager is available.
>
> Add the core CWSR handling infrastructure for user queues, including
> the first-level trap handler objects and save area sizing.
>
> Calculate the control stack, wave context and debugger memory sizes
> required for CWSR, and provide validation for userspace supplied save
> area parameters. Address validation is handled separately by the
> caller.
>
> Create the per-VM TMA BO using the VM root BO reservation object. This
> avoids recursively acquiring reservation_ww_class_mutex while the VM
> reservation is already held through drm_exec. The TMA BO is not pinned
> and is covered by the VM reservation while being added to the VM.
>
> Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
> Co-developed-by: Srinivasan Shanmugam <srinivasan.shanmugam@amd.com>
> Signed-off-by: Srinivasan Shanmugam <srinivasan.shanmugam@amd.com>
I only skimmed over it and don't know some of the technical background to full judge.
So only Acked-by: Christian König <christian.koenig@amd.com> for now.
> ---
> drivers/gpu/drm/amd/amdgpu/Makefile | 2 +-
> drivers/gpu/drm/amd/amdgpu/amdgpu.h | 3 +
> drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c | 534 +++++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h | 81 ++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 19 +-
> drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 13 +-
> drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h | 22 +
> 7 files changed, 660 insertions(+), 14 deletions(-)
> create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c
> create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
> index 55b5d11f1938..5086a482daf1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> @@ -72,7 +72,7 @@ amdgpu-y += amdgpu_device.o amdgpu_reg_access.o amdgpu_doorbell_mgr.o amdgpu_kms
> amdgpu_eeprom.o amdgpu_mca.o amdgpu_psp_ta.o amdgpu_lsdma.o amdgpu_lockdep.o \
> amdgpu_ring_mux.o amdgpu_xcp.o amdgpu_seq64.o amdgpu_dev_coredump.o \
> amdgpu_cper.o amdgpu_userq_fence.o amdgpu_eviction_fence.o amdgpu_ip.o \
> - amdgpu_wb.o
> + amdgpu_wb.o amdgpu_cwsr.o
>
> amdgpu-$(CONFIG_PROC_FS) += amdgpu_fdinfo.o
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index ce671731b700..26561a4f4db8 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -332,6 +332,7 @@ struct amdgpu_reset_context;
> struct amdgpu_reset_control;
> struct amdgpu_coredump_info;
> struct amdgpu_video_codecs;
> +struct amdgpu_cwsr_isa;
>
> enum amdgpu_cp_irq {
> AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP = 0,
> @@ -974,6 +975,8 @@ struct amdgpu_device {
> * Must be last --ends in a flexible-array member.
> */
> struct amdgpu_kfd_dev kfd;
> +
> + struct amdgpu_cwsr_info *cwsr_info;
> };
>
> static inline uint32_t amdgpu_ip_version(const struct amdgpu_device *adev,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c
> new file mode 100644
> index 000000000000..6b6c11bb5fd7
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c
> @@ -0,0 +1,534 @@
> +/*
> + * Copyright 2025 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + */
> +
> +#include <drm/drm_exec.h>
> +
> +#include "amdgpu.h"
> +#include "cwsr_trap_handler.h"
> +#include "amdgpu_cwsr.h"
> +
> +extern int cwsr_enable;
> +
> +#define AMDGPU_CWSR_TBA_MAX_SIZE (2 * AMDGPU_GPU_PAGE_SIZE)
> +#define AMDGPU_CWSR_TMA_MAX_SIZE (AMDGPU_GPU_PAGE_SIZE)
> +#define AMDGPU_CWSR_TMA_OFFSET (AMDGPU_CWSR_TBA_MAX_SIZE)
> +
> +#define SGPR_SIZE_PER_CU 0x4000
> +#define LDS_SIZE_PER_CU 0x10000
> +#define HWREG_SIZE_PER_CU 0x1000
> +#define DEBUGGER_BYTES_ALIGN 64
> +#define DEBUGGER_BYTES_PER_WAVE 32
> +#define SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER 40
> +
> +enum amdgpu_cwsr_region {
> + AMDGPU_CWSR_TBA,
> + AMDGPU_CWSR_TMA,
> +};
> +
> +static inline uint64_t amdgpu_cwsr_tba_vaddr(struct amdgpu_device *adev)
> +{
> + u64 addr = AMDGPU_VA_RESERVED_TRAP_UQ_START(adev);
> +
> + addr = amdgpu_gmc_sign_extend(addr);
> +
> + return addr;
> +}
> +
> +static inline bool amdgpu_cwsr_is_supported(struct amdgpu_device *adev)
> +{
> + u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
> +
> + if (!cwsr_enable || adev->gfx.disable_uq ||
> + gc_ver < IP_VERSION(9, 0, 1))
> + return false;
> +
> + return true;
> +}
> +
> +uint32_t amdgpu_cwsr_size_needed(struct amdgpu_device *adev, int num_xcc)
> +{
> + if (!amdgpu_cwsr_is_enabled(adev))
> + return 0;
> +
> + return num_xcc *
> + (adev->cwsr_info->xcc_cwsr_sz + adev->cwsr_info->xcc_dbg_mem_sz);
> +}
> +
> +static void amdgpu_cwsr_init_isa_details(struct amdgpu_device *adev,
> + struct amdgpu_cwsr_info *cwsr_info)
> +{
> + u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
> +
> + if (gc_ver < IP_VERSION(9, 0, 1)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) >
> + AMDGPU_CWSR_TBA_MAX_SIZE);
> + cwsr_info->isa_buf = cwsr_trap_gfx8_hex;
> + cwsr_info->isa_sz = sizeof(cwsr_trap_gfx8_hex);
> + } else if (gc_ver == IP_VERSION(9, 4, 1)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex) >
> + AMDGPU_CWSR_TBA_MAX_SIZE);
> + cwsr_info->isa_buf = cwsr_trap_arcturus_hex;
> + cwsr_info->isa_sz = sizeof(cwsr_trap_arcturus_hex);
> + } else if (gc_ver == IP_VERSION(9, 4, 2)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_aldebaran_hex) >
> + AMDGPU_CWSR_TBA_MAX_SIZE);
> + cwsr_info->isa_buf = cwsr_trap_aldebaran_hex;
> + cwsr_info->isa_sz = sizeof(cwsr_trap_aldebaran_hex);
> + } else if (gc_ver == IP_VERSION(9, 4, 3) ||
> + gc_ver == IP_VERSION(9, 4, 4)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_4_3_hex) >
> + AMDGPU_CWSR_TBA_MAX_SIZE);
> + cwsr_info->isa_buf = cwsr_trap_gfx9_4_3_hex;
> + cwsr_info->isa_sz = sizeof(cwsr_trap_gfx9_4_3_hex);
> + } else if (gc_ver == IP_VERSION(9, 5, 0)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_5_0_hex) > PAGE_SIZE);
> + cwsr_info->isa_buf = cwsr_trap_gfx9_5_0_hex;
> + cwsr_info->isa_sz = sizeof(cwsr_trap_gfx9_5_0_hex);
> + } else if (gc_ver < IP_VERSION(10, 1, 1)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) >
> + AMDGPU_CWSR_TBA_MAX_SIZE);
> + cwsr_info->isa_buf = cwsr_trap_gfx9_hex;
> + cwsr_info->isa_sz = sizeof(cwsr_trap_gfx9_hex);
> + } else if (gc_ver < IP_VERSION(10, 3, 0)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_nv1x_hex) >
> + AMDGPU_CWSR_TBA_MAX_SIZE);
> + cwsr_info->isa_buf = cwsr_trap_nv1x_hex;
> + cwsr_info->isa_sz = sizeof(cwsr_trap_nv1x_hex);
> + } else if (gc_ver < IP_VERSION(11, 0, 0)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex) >
> + AMDGPU_CWSR_TBA_MAX_SIZE);
> + cwsr_info->isa_buf = cwsr_trap_gfx10_hex;
> + cwsr_info->isa_sz = sizeof(cwsr_trap_gfx10_hex);
> + } else if (gc_ver < IP_VERSION(12, 0, 0)) {
> + /* The gfx11 cwsr trap handler must fit inside a single
> + page. */
> + BUILD_BUG_ON(sizeof(cwsr_trap_gfx11_hex) > PAGE_SIZE);
> + cwsr_info->isa_buf = cwsr_trap_gfx11_hex;
> + cwsr_info->isa_sz = sizeof(cwsr_trap_gfx11_hex);
> + } else if (gc_ver < IP_VERSION(12, 1, 0)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_hex) >
> + AMDGPU_CWSR_TBA_MAX_SIZE);
> + cwsr_info->isa_buf = cwsr_trap_gfx12_hex;
> + cwsr_info->isa_sz = sizeof(cwsr_trap_gfx12_hex);
> + } else {
> + BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_1_0_hex) >
> + AMDGPU_CWSR_TBA_MAX_SIZE);
> + cwsr_info->isa_buf = cwsr_trap_gfx12_1_0_hex;
> + cwsr_info->isa_sz = sizeof(cwsr_trap_gfx12_1_0_hex);
> + }
> +}
> +
> +static uint32_t amdgpu_cwsr_get_vgpr_size_per_cu(struct amdgpu_device *adev)
> +{
> + u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
> + u32 vgpr_size;
> +
> + switch (gc_ver) {
> + case IP_VERSION(9, 4, 1): /* GFX_VERSION_ARCTURUS */
> + case IP_VERSION(9, 4, 2): /* GFX_VERSION_ALDEBARAN */
> + case IP_VERSION(9, 4, 3): /* GFX_VERSION_AQUA_VANJARAM */
> + case IP_VERSION(9, 4, 4): /* GFX_VERSION_AQUA_VANJARAM */
> + case IP_VERSION(9, 5, 0):
> + vgpr_size = 0x80000;
> + break;
> + case IP_VERSION(11, 0, 0):
> + case IP_VERSION(11, 0, 2):
> + case IP_VERSION(11, 0, 3):
> + case IP_VERSION(12, 0, 0):
> + case IP_VERSION(12, 0, 1):
> + vgpr_size = 0x60000;
> + break;
> + default:
> + vgpr_size = 0x40000;
> + break;
> + }
> +
> + return vgpr_size;
> +}
> +
> +static uint32_t amdgpu_cwsr_get_wg_ctxt_size_per_cu(struct amdgpu_device *adev)
> +{
> + u32 lds_sz_per_cu;
> +
> + lds_sz_per_cu =
> + (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0)) ?
> + (adev->gfx.cu_info.lds_size << 10) :
> + LDS_SIZE_PER_CU;
> +
> + return amdgpu_cwsr_get_vgpr_size_per_cu(adev) + SGPR_SIZE_PER_CU +
> + lds_sz_per_cu + HWREG_SIZE_PER_CU;
> +}
> +
> +static uint32_t amdgpu_cwsr_ctl_stack_bytes_per_wave(struct amdgpu_device *adev)
> +{
> + u32 sz;
> +
> + if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 1, 0))
> + sz = 12;
> + else
> + sz = 8;
> + return sz;
> +}
> +
> +static void amdgpu_cwsr_init_save_area_info(struct amdgpu_device *adev,
> + struct amdgpu_cwsr_info *cwsr_info)
> +{
> + struct amdgpu_gfx_config *gfx_info = &adev->gfx.config;
> + u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
> + u32 ctl_stack_size, wg_data_size, dbg_mem_size;
> + u32 array_count;
> + u32 wave_num;
> + u32 cu_num;
> +
> + if (gc_ver < IP_VERSION(9, 0, 1))
> + return;
> +
> + array_count = gfx_info->max_shader_engines * gfx_info->max_sh_per_se;
> +
> + cu_num = adev->gfx.cu_info.number / NUM_XCC(adev->gfx.xcc_mask);
> + wave_num = (gc_ver < IP_VERSION(10, 1, 0)) ? /* GFX_VERSION_NAVI10 */
> + min(cu_num * 40,
> + array_count / gfx_info->max_sh_per_se * 512) :
> + cu_num * 32;
> +
> + wg_data_size = ALIGN(cu_num * amdgpu_cwsr_get_wg_ctxt_size_per_cu(adev),
> + PAGE_SIZE);
> + ctl_stack_size =
> + wave_num * amdgpu_cwsr_ctl_stack_bytes_per_wave(adev) + 8;
> + ctl_stack_size =
> + ALIGN(SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER + ctl_stack_size,
> + PAGE_SIZE);
> + dbg_mem_size =
> + ALIGN(wave_num * DEBUGGER_BYTES_PER_WAVE, DEBUGGER_BYTES_ALIGN);
> + /*
> + * HW design limits control stack size to 0x7000.
> + * This is insufficient for theoretical PM4 cases
> + * but sufficient for AQL, limited by SPI events.
> + */
> + if (IP_VERSION_MAJ(gc_ver) == 10)
> + ctl_stack_size = min(ctl_stack_size, 0x7000);
> +
> + cwsr_info->xcc_ctl_stack_sz = ctl_stack_size;
> + cwsr_info->xcc_cwsr_sz = ctl_stack_size + wg_data_size;
> + cwsr_info->xcc_dbg_mem_sz = dbg_mem_size;
> +}
> +
> +int amdgpu_cwsr_init(struct amdgpu_device *adev)
> +{
> + struct amdgpu_cwsr_info *cwsr_info __free(kfree) =
> + kzalloc(sizeof(*cwsr_info), GFP_KERNEL);
> + void *ptr;
> + int r;
> +
> + if (!amdgpu_cwsr_is_supported(adev))
> + return -EOPNOTSUPP;
> +
> + if (!cwsr_info)
> + return -ENOMEM;
> + amdgpu_cwsr_init_isa_details(adev, cwsr_info);
> +
> + if (!cwsr_info->isa_sz)
> + return -EOPNOTSUPP;
> +
> + r = amdgpu_bo_create_kernel(adev, AMDGPU_CWSR_TBA_MAX_SIZE, PAGE_SIZE,
> + AMDGPU_GEM_DOMAIN_GTT, &cwsr_info->isa_bo,
> + NULL, &ptr);
> + if (r)
> + return r;
> +
> + memcpy(ptr, cwsr_info->isa_buf, cwsr_info->isa_sz);
> +
> + amdgpu_cwsr_init_save_area_info(adev, cwsr_info);
> + adev->cwsr_info = no_free_ptr(cwsr_info);
> +
> + return 0;
> +}
> +
> +void amdgpu_cwsr_fini(struct amdgpu_device *adev)
> +{
> + if (!amdgpu_cwsr_is_enabled(adev))
> + return;
> +
> + amdgpu_bo_free_kernel(&adev->cwsr_info->isa_bo, NULL, NULL);
> + kfree(adev->cwsr_info);
> + adev->cwsr_info = NULL;
> +}
> +
> +/*
> + * amdgpu_map_cwsr_trap_handler should be called during amdgpu_vm_init
> + * it maps virtual address amdgpu_cwsr_trap_handler_vaddr() to this VM, and each
> + * compute queue can use this virtual address for wave save/restore
> + * operations to support compute preemption.
> + */
> +static int amdgpu_cwsr_map_region(struct amdgpu_device *adev,
> + struct amdgpu_vm *vm,
> + struct amdgpu_cwsr_trap_obj *cwsr,
> + enum amdgpu_cwsr_region region)
> +{
> + u64 cwsr_addr, va_flags, va;
> + struct amdgpu_bo_va **bo_va;
> + struct amdgpu_bo *bo;
> + u32 size;
> + int r;
> +
> + if (!cwsr || !vm)
> + return -EINVAL;
> +
> + cwsr_addr = amdgpu_cwsr_tba_vaddr(adev);
> +
> + if (region == AMDGPU_CWSR_TBA) {
> + size = AMDGPU_CWSR_TBA_MAX_SIZE;
> + bo_va = &cwsr->tba_va;
> + bo = adev->cwsr_info->isa_bo;
> + va = cwsr_addr;
> + va_flags = (AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
> + AMDGPU_VM_PAGE_EXECUTABLE);
> + } else {
> + size = AMDGPU_CWSR_TMA_MAX_SIZE;
> + bo_va = &cwsr->tma_va;
> + bo = cwsr->tma_bo;
> + va = cwsr_addr + AMDGPU_CWSR_TMA_OFFSET;
> + va_flags = (AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE);
> + }
> +
> + *bo_va = amdgpu_vm_bo_add(adev, vm, bo);
> + if (!*bo_va)
> + return -ENOMEM;
> +
> + va &= AMDGPU_GMC_HOLE_MASK;
> + r = amdgpu_vm_bo_map(adev, *bo_va, va, 0, size, va_flags);
> + if (r) {
> + dev_err(adev->dev, "failed to do bo map of %s region, err=%d\n",
> + (region == AMDGPU_CWSR_TBA ? "tba" : "tma"), r);
> + amdgpu_vm_bo_del(adev, *bo_va);
> + *bo_va = NULL;
> + return r;
> + }
> +
> + r = amdgpu_vm_bo_update(adev, *bo_va, false);
> + if (r) {
> + dev_err(adev->dev,
> + "failed to do page table update of %s region, err=%d\n",
> + (region == AMDGPU_CWSR_TBA ? "tba" : "tma"), r);
> + amdgpu_vm_bo_del(adev, *bo_va);
> + *bo_va = NULL;
> + return r;
> + }
> +
> + if (region == AMDGPU_CWSR_TBA)
> + cwsr->tba_gpu_va_addr = va;
> + else
> + cwsr->tma_gpu_va_addr = va;
> +
> + return 0;
> +}
> +
> +static int amdgpu_cwsr_unmap_region(struct amdgpu_device *adev,
> + struct amdgpu_cwsr_trap_obj *cwsr,
> + enum amdgpu_cwsr_region region)
> +{
> + struct amdgpu_bo_va **bo_va;
> + u64 va;
> + int r;
> +
> + if (!cwsr)
> + return -EINVAL;
> +
> + if (region == AMDGPU_CWSR_TBA) {
> + bo_va = &cwsr->tba_va;
> + va = cwsr->tba_gpu_va_addr;
> + } else {
> + bo_va = &cwsr->tma_va;
> + va = cwsr->tma_gpu_va_addr;
> + }
> +
> + r = amdgpu_vm_bo_unmap(adev, *bo_va, va);
> + if (r) {
> + dev_err(adev->dev,
> + "failed to do bo_unmap on CWSR trap handler, err=%d\n",
> + r);
> + return r;
> + }
> +
> + amdgpu_vm_bo_del(adev, *bo_va);
> + *bo_va = NULL;
> +
> + return r;
> +}
> +
> +int amdgpu_cwsr_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> + struct amdgpu_cwsr_trap_obj **trap_obj)
> +{
> + struct amdgpu_bo_param bp;
> + struct amdgpu_cwsr_trap_obj *cwsr;
> + struct amdgpu_bo *bo;
> + struct drm_exec exec;
> + int r;
> +
> + if (!amdgpu_cwsr_is_enabled(adev))
> + return -EOPNOTSUPP;
> + if (!vm || !trap_obj)
> + return -EINVAL;
> +
> + cwsr = kzalloc_obj(*cwsr, GFP_KERNEL);
> + if (!cwsr)
> + return -ENOMEM;
> +
> + bo = adev->cwsr_info->isa_bo;
> + drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
> + drm_exec_until_all_locked(&exec) {
> + r = amdgpu_vm_lock_pd(vm, &exec, 0);
> + if (likely(!r))
> + r = drm_exec_lock_obj(&exec, &bo->tbo.base);
> + drm_exec_retry_on_contention(&exec);
> + if (unlikely(r)) {
> + dev_err(adev->dev,
> + "failed to reserve for CWSR allocs: err=%d\n",
> + r);
> + goto err_exec;
> + }
> + }
> +
> + /*
> + * Create TMA BO using the VM's already-locked resv so no new
> + * reservation_ww_class_mutex acquisition is needed, avoiding the
> + * recursive ww_mutex deadlock. Unpinned; eviction is handled via
> + * amdgpu_vm_bo_add() inside amdgpu_cwsr_map_region().
> + */
> + memset(&bp, 0, sizeof(bp));
> + bp.size = AMDGPU_CWSR_TMA_MAX_SIZE;
> + bp.byte_align = PAGE_SIZE;
> + bp.domain = AMDGPU_GEM_DOMAIN_GTT;
> + bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC |
> + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
> + bp.type = ttm_bo_type_kernel;
> + bp.bo_ptr_size = sizeof(struct amdgpu_bo);
> + bp.resv = vm->root.bo->tbo.base.resv;
> +
> + r = amdgpu_bo_create(adev, &bp, &cwsr->tma_bo);
> + if (r)
> + goto err_exec;
> +
> + /* VM resv already held, kmap directly, no separate reserve needed. */
> + r = amdgpu_bo_kmap(cwsr->tma_bo, &cwsr->tma_cpu_addr);
> + if (r)
> + goto err_tma;
> +
> + memset(cwsr->tma_cpu_addr, 0, AMDGPU_CWSR_TMA_MAX_SIZE);
> +
> + r = amdgpu_cwsr_map_region(adev, vm, cwsr, AMDGPU_CWSR_TMA);
> + if (r)
> + goto err_kmap;
> +
> + r = amdgpu_cwsr_map_region(adev, vm, cwsr, AMDGPU_CWSR_TBA);
> + if (r) {
> + amdgpu_cwsr_unmap_region(adev, cwsr, AMDGPU_CWSR_TMA);
> + goto err_kmap;
> + }
> +
> + drm_exec_fini(&exec);
> + *trap_obj = cwsr;
> + return 0;
> +
> +err_kmap:
> + amdgpu_bo_kunmap(cwsr->tma_bo);
> +err_tma:
> + amdgpu_bo_unref(&cwsr->tma_bo);
> +err_exec:
> + drm_exec_fini(&exec);
> + kfree(cwsr);
> + *trap_obj = NULL;
> + return r;
> +}
> +
> +int amdgpu_cwsr_validate_params(struct amdgpu_device *adev,
> + struct amdgpu_cwsr_params *cwsr_params,
> + int num_xcc)
> +{
> + u32 min_size, max_size;
> +
> + if (!amdgpu_cwsr_is_enabled(adev))
> + return -EOPNOTSUPP;
> +
> + if (!cwsr_params)
> + return -EINVAL;
> +
> + min_size = amdgpu_cwsr_size_needed(adev, num_xcc);
> + max_size = 2 * min_size;
> + /*
> + * Only save area size details checked. Address validation needs to be
> + * carried out separately. User is expected to pass a cwsr buffer area
> + * with size >= ctl stack size + wg data size + debug memory size
> + */
> + if (cwsr_params->cwsr_sz < min_size ||
> + cwsr_params->cwsr_sz > max_size) {
> + dev_dbg(adev->dev,
> + "queue cwsr size 0x%x should be within min %u : max %u\n",
> + cwsr_params->cwsr_sz, min_size, max_size);
> + return -EINVAL;
> + }
> +
> + return 0;
> +}
> +
> +void amdgpu_cwsr_free(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> + struct amdgpu_cwsr_trap_obj **trap_obj)
> +{
> + struct amdgpu_bo *tba_bo;
> + struct amdgpu_bo *tma_bo;
> + struct drm_exec exec;
> + int r;
> +
> + if (!trap_obj || !*trap_obj || !(*trap_obj)->tma_bo)
> + return;
> +
> + tba_bo = adev->cwsr_info->isa_bo;
> + tma_bo = (*trap_obj)->tma_bo;
> +
> + if (!tba_bo || !tma_bo)
> + return;
> +
> + drm_exec_init(&exec, 0, 0);
> + drm_exec_until_all_locked(&exec) {
> + r = amdgpu_vm_lock_pd(vm, &exec, 0);
> + if (likely(!r))
> + r = drm_exec_lock_obj(&exec, &tba_bo->tbo.base);
> + drm_exec_retry_on_contention(&exec);
> + if (unlikely(r)) {
> + dev_err(adev->dev,
> + "failed to reserve CWSR BOs: err=%d\n", r);
> + goto err;
> + }
> + }
> +
> + amdgpu_cwsr_unmap_region(adev, *trap_obj, AMDGPU_CWSR_TBA);
> + amdgpu_cwsr_unmap_region(adev, *trap_obj, AMDGPU_CWSR_TMA);
> +
> +err:
> + drm_exec_fini(&exec);
> +
> + /* TMA BO not pinned, release CPU mapping and free. */
> + amdgpu_bo_kunmap((*trap_obj)->tma_bo);
> + amdgpu_bo_unref(&(*trap_obj)->tma_bo);
> + kfree(*trap_obj);
> + *trap_obj = NULL;
> +}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
> new file mode 100644
> index 000000000000..7f922559e297
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
> @@ -0,0 +1,81 @@
> +/*
> + * Copyright 2025 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + */
> +
> +#ifndef AMDGPU_CWSR_H
> +#define AMDGPU_CWSR_H
> +
> +#include <linux/types.h>
> +
> +struct amdgpu_bo;
> +struct amdgpu_bo_va;
> +struct amdgpu_device;
> +struct amdgpu_vm;
> +
> +/**
> + * struct amdgpu_cwsr_obj - CWSR (Compute Wave Save Restore) buffer tracking
> + * @bo: Buffer object for CWSR area
> + * @bo_va: Buffer object virtual address mapping
> + */
> +struct amdgpu_cwsr_trap_obj {
> + u64 tma_gpu_va_addr;
> + u64 tba_gpu_va_addr;
> +
> + struct amdgpu_bo *tma_bo;
> + struct amdgpu_bo_va *tba_va;
> + struct amdgpu_bo_va *tma_va;
> + void *tma_cpu_addr;
> +};
> +
> +struct amdgpu_cwsr_info {
> + /* cwsr isa */
> + struct amdgpu_bo *isa_bo;
> + const void *isa_buf;
> + u32 isa_sz;
> + /* cwsr size info per XCC*/
> + u32 xcc_ctl_stack_sz;
> + u32 xcc_dbg_mem_sz;
> + u32 xcc_cwsr_sz;
> +};
> +
> +struct amdgpu_cwsr_params {
> + u64 ctx_save_area_address;
> + /* cwsr size info */
> + u32 cwsr_sz;
> +};
> +
> +int amdgpu_cwsr_init(struct amdgpu_device *adev);
> +void amdgpu_cwsr_fini(struct amdgpu_device *adev);
> +
> +int amdgpu_cwsr_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> + struct amdgpu_cwsr_trap_obj **cwsr_obj);
> +void amdgpu_cwsr_free(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> + struct amdgpu_cwsr_trap_obj **cwsr_obj);
> +static inline bool amdgpu_cwsr_is_enabled(struct amdgpu_device *adev)
> +{
> + return adev->cwsr_info;
> +}
> +
> +uint32_t amdgpu_cwsr_size_needed(struct amdgpu_device *adev, int num_xcc);
> +int amdgpu_cwsr_validate_params(struct amdgpu_device *adev,
> + struct amdgpu_cwsr_params *cwsr_params,
> + int num_xcc);
> +#endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index ab6f9131ac11..b2a8a8b37006 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -764,6 +764,15 @@ MODULE_PARM_DESC(use_xgmi_p2p,
> "Enable XGMI P2P interface (0 = disable; 1 = enable (default))");
> module_param_named(use_xgmi_p2p, amdgpu_use_xgmi_p2p, int, 0444);
>
> +/**
> + * DOC: cwsr_enable (int)
> + * CWSR(compute wave store and resume) allows the GPU to preempt shader execution in
> + * the middle of a compute wave. Default is 1 to enable this feature. Setting 0
> + * disables it.
> + */
> +int cwsr_enable = 1;
> +module_param(cwsr_enable, int, 0444);
> +MODULE_PARM_DESC(cwsr_enable, "CWSR enable (0 = Off, 1 = On (Default))");
>
> #ifdef CONFIG_HSA_AMD
> /**
> @@ -787,16 +796,6 @@ module_param(hws_max_conc_proc, int, 0444);
> MODULE_PARM_DESC(hws_max_conc_proc,
> "Max # processes HWS can execute concurrently when sched_policy=0 (0 = no concurrency, #VMIDs for KFD = Maximum(default))");
>
> -/**
> - * DOC: cwsr_enable (int)
> - * CWSR(compute wave store and resume) allows the GPU to preempt shader execution in
> - * the middle of a compute wave. Default is 1 to enable this feature. Setting 0
> - * disables it as only in non-HWS mode.
> - */
> -int cwsr_enable = 1;
> -module_param(cwsr_enable, int, 0444);
> -MODULE_PARM_DESC(cwsr_enable, "CWSR enable (0 = Off (debugging only), 1 = On (Default))");
> -
> /**
> * DOC: max_num_of_queues_per_device (int)
> * Maximum number of queues per device. Valid setting is between 1 and 4096. Default
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> index 8e478d3bda1a..226caa99f6ee 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> @@ -178,10 +178,17 @@ struct amdgpu_bo_vm;
> #define AMDGPU_VA_RESERVED_TRAP_SIZE (1ULL << 16)
> #define AMDGPU_VA_RESERVED_TRAP_START(adev) (AMDGPU_VA_RESERVED_SEQ64_START(adev) \
> - AMDGPU_VA_RESERVED_TRAP_SIZE)
> +/* TBD: Ideally, existing TRAP VA should do. There is a conflict with KFD mapping that needs to be
> + * resolved. Revisit later.
> + */
> +#define AMDGPU_VA_RESERVED_TRAP_UQ_SIZE (3ULL << 12)
> +#define AMDGPU_VA_RESERVED_TRAP_UQ_START(adev) \
> + (AMDGPU_VA_RESERVED_TRAP_START(adev) - AMDGPU_VA_RESERVED_TRAP_UQ_SIZE)
> +
> #define AMDGPU_VA_RESERVED_BOTTOM (1ULL << 16)
> -#define AMDGPU_VA_RESERVED_TOP (AMDGPU_VA_RESERVED_TRAP_SIZE + \
> - AMDGPU_VA_RESERVED_SEQ64_SIZE + \
> - AMDGPU_VA_RESERVED_CSA_SIZE)
> +#define AMDGPU_VA_RESERVED_TOP \
> + (AMDGPU_VA_RESERVED_TRAP_UQ_SIZE + AMDGPU_VA_RESERVED_TRAP_SIZE + \
> + AMDGPU_VA_RESERVED_SEQ64_SIZE + AMDGPU_VA_RESERVED_CSA_SIZE)
>
> /* See vm_update_mode */
> #define AMDGPU_VM_USE_CPU_FOR_GFX (1 << 0)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
> index 33157409eda0..e7c84b50ccf4 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
> @@ -226,4 +226,26 @@ amdgpu_get_next_xcp(struct amdgpu_xcp_mgr *xcp_mgr, int *from)
> for (i = 0, xcp = amdgpu_get_next_xcp(xcp_mgr, &i); xcp; \
> ++i, xcp = amdgpu_get_next_xcp(xcp_mgr, &i))
>
> +static inline int amdgpu_xcp_get_num_xcc(struct amdgpu_xcp_mgr *xcp_mgr,
> + int xcp_id)
> +{
> + struct amdgpu_xcp *xcp;
> + u32 xcc_mask;
> + int i, r;
> +
> + if (!xcp_mgr || xcp_id == AMDGPU_XCP_NO_PARTITION)
> + return 1;
> + for_each_xcp(xcp_mgr, xcp, i) {
> + if (xcp->id == xcp_id) {
> + r = amdgpu_xcp_get_inst_details(xcp, AMDGPU_XCP_GFX,
> + &xcc_mask);
> + if (unlikely(r))
> + return 1;
> + else
> + return hweight32(xcc_mask);
> + }
> + }
> +
> + return 1;
> +}
> #endif
^ permalink raw reply [flat|nested] 16+ messages in thread
* [PATCH 2/7] drm/amdgpu: Integrate CWSR with user queues
2026-10-08 7:05 [PATCH 0/7] drm/amdgpu: Add CWSR and second-level trap handler support Srinivasan Shanmugam
2026-10-08 7:05 ` [PATCH 1/7] drm/amdgpu: Add CWSR infrastructure for user queues Srinivasan Shanmugam
@ 2026-10-08 7:05 ` Srinivasan Shanmugam
2026-10-12 13:12 ` Christian König
2026-10-08 7:05 ` [PATCH 3/7] drm/amdgpu: Add CWSR userspace interfaces Srinivasan Shanmugam
` (4 subsequent siblings)
6 siblings, 1 reply; 16+ messages in thread
From: Srinivasan Shanmugam @ 2026-10-08 7:05 UTC (permalink / raw)
To: Christian König, Alex Deucher; +Cc: amd-gfx, Lijo Lazar, James Zhu
From: Lijo Lazar <lijo.lazar@amd.com>
Initialize the CWSR handler during device initialization and release
the associated resources during device teardown.
Add a per-file CWSR trap object for render-node user queues and map the
first-level CWSR trap handler into the GPUVM associated with the file.
Use the mapped TBA and TMA addresses when programming user queues.
Add user save area parameters to the MQD properties for compute queue
creation and validate the parameters before using them for MQD
initialization.
The CWSR parameter validation walks the VM mapping tree and updates the
user queue VA mapping state. Reserve the VM root BO around the
validation so the required VM reservation is held while accessing the
mapping tree.
Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
Co-developed-by: James Zhu <James.Zhu@amd.com>
Signed-off-by: James Zhu <James.Zhu@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 +
drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h | 6 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 8 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 10 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h | 8 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 25 +++++
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 8 +-
drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c | 14 +++
drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 102 +++++++++++++++++----
9 files changed, 163 insertions(+), 20 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 26561a4f4db8..bdd55ac5b06e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -333,6 +333,7 @@ struct amdgpu_reset_control;
struct amdgpu_coredump_info;
struct amdgpu_video_codecs;
struct amdgpu_cwsr_isa;
+struct amdgpu_cwsr_trap_obj;
enum amdgpu_cp_irq {
AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP = 0,
@@ -421,6 +422,7 @@ struct amdgpu_fpriv {
struct xarray bo_list_handles;
struct amdgpu_ctx_mgr ctx_mgr;
struct amdgpu_userq_mgr userq_mgr;
+ struct amdgpu_cwsr_trap_obj *cwsr_trap;
/* Eviction fence infra */
struct amdgpu_eviction_fence_mgr evf_mgr;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
index 7f922559e297..5717e1d662fc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
@@ -78,4 +78,10 @@ uint32_t amdgpu_cwsr_size_needed(struct amdgpu_device *adev, int num_xcc);
int amdgpu_cwsr_validate_params(struct amdgpu_device *adev,
struct amdgpu_cwsr_params *cwsr_params,
int num_xcc);
+static inline bool amdgpu_cwsr_has_dbg_wa(struct amdgpu_device *adev)
+{
+ u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
+
+ return gc_ver >= IP_VERSION(11, 0, 0) && gc_ver <= IP_VERSION(11, 0, 3);
+}
#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e0014611f039..5a7f111763bf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -81,6 +81,7 @@
#include "amdgpu_reset.h"
#include "amdgpu_virt.h"
#include "amdgpu_dev_coredump.h"
+#include "amdgpu_cwsr.h"
#include <linux/suspend.h>
#include <drm/task_barrier.h>
@@ -2549,6 +2550,12 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
r = amdgpu_cper_init(adev);
+ if (!r) {
+ r = amdgpu_cwsr_init(adev);
+ if (r == -EOPNOTSUPP)
+ r = 0;
+ }
+
init_failed:
return r;
@@ -2946,6 +2953,7 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
{
int i, r;
+ amdgpu_cwsr_fini(adev);
amdgpu_cper_fini(adev);
if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index f34150d20a2f..f3945b3988d8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -49,6 +49,8 @@
#include "amdgpu_userq.h"
#include "amdgpu_video_codecs.h"
#include "amdgpu_ras_mgr.h"
+#include "amdgpu_vm.h"
+#include "amdgpu_cwsr.h"
void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev)
{
@@ -1552,6 +1554,13 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
r);
amdgpu_evf_mgr_init(&fpriv->evf_mgr);
+
+ if (amdgpu_cwsr_is_enabled(adev)) {
+ r = amdgpu_cwsr_alloc(adev, &fpriv->vm, &fpriv->cwsr_trap);
+ if (r)
+ dev_dbg(adev->dev, "cwsr trap not enabled");
+ }
+
amdgpu_ctx_mgr_init(&fpriv->ctx_mgr, adev);
file_priv->driver_priv = fpriv;
@@ -1621,6 +1630,7 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
}
amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr);
+ amdgpu_cwsr_free(adev, &fpriv->vm, &fpriv->cwsr_trap);
if (pasid)
amdgpu_pasid_free_delayed(pd->tbo.base.resv, pasid);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
index 2b13a5b37539..b3df09dbcbad 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
@@ -527,6 +527,14 @@ struct amdgpu_mqd_prop {
uint32_t cu_mask_count;
uint32_t cu_flags;
bool is_user_cu_masked;
+ bool aql_queue;
+ u32 queue_percentage;
+ /* used in gfx9 and gfx12.1 */
+ u32 pm4_target_xcc;
+ /* cwsr params */
+ u64 ctx_save_area_addr;
+ u32 ctx_save_area_size;
+ u32 ctl_stack_size;
};
struct amdgpu_mqd {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index e29823ae8672..6eea0ca40556 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -35,6 +35,7 @@
#include "amdgpu_hmm.h"
#include "amdgpu_userq_fence.h"
#include "amdgpu_trace.h"
+#include "amdgpu_cwsr.h"
u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev)
{
@@ -266,6 +267,30 @@ int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
return -EINVAL;
}
+int amdgpu_userq_input_cwsr_params_validate(
+ struct amdgpu_usermode_queue *queue,
+ struct amdgpu_cwsr_params *cwsr_params)
+{
+ struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(queue->userq_mgr);
+ struct amdgpu_device *adev = queue->userq_mgr->adev;
+ u32 cwsr_size;
+ int num_xcc;
+ int r;
+
+ num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr, fpriv->xcp_id);
+ r = amdgpu_cwsr_validate_params(queue->userq_mgr->adev, cwsr_params,
+ num_xcc);
+ if (r)
+ return r;
+ cwsr_size = amdgpu_cwsr_size_needed(queue->userq_mgr->adev, num_xcc);
+ if (!cwsr_size)
+ return -EOPNOTSUPP;
+
+ return amdgpu_userq_input_va_validate(
+ adev, queue, cwsr_params->ctx_save_area_address, cwsr_size,
+ &queue->userq_vas.va.cwsr);
+}
+
static bool amdgpu_userq_buffer_va_mapped(struct amdgpu_vm *vm, u64 addr)
{
struct amdgpu_bo_va_mapping *mapping;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index 488dc21d7c81..727f60c40bbf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -41,6 +41,7 @@ enum amdgpu_userq_state {
};
struct amdgpu_mqd_prop;
+struct amdgpu_cwsr_params;
struct amdgpu_userq_obj {
void *cpu_ptr;
@@ -97,8 +98,9 @@ struct amdgpu_usermode_queue {
u64 eop;
u64 shadow;
u64 csa;
+ u64 cwsr;
} va;
- u64 va_array[6];
+ u64 va_array[7];
} userq_vas;
uint32_t gang_ctx_array_index;
@@ -200,4 +202,8 @@ int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
void amdgpu_userq_gem_va_unmap_validate(struct amdgpu_device *adev,
struct amdgpu_bo_va_mapping *mapping);
+int amdgpu_userq_input_cwsr_params_validate(
+ struct amdgpu_usermode_queue *queue,
+ struct amdgpu_cwsr_params *cwsr_params);
+
#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
index 420ce997bebc..b5326c2067db 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
@@ -3327,6 +3327,20 @@ static int gfx_v12_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
/* set CU mask */
gfx_v12_0_compute_mqd_set_cu_mask(adev, mqd, prop);
+ /* If non-zero, assume cwsr is enabled */
+ if (prop->ctx_save_area_addr) {
+ mqd->cp_hqd_persistent_state |=
+ (1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT);
+ mqd->cp_hqd_ctx_save_base_addr_lo =
+ lower_32_bits(prop->ctx_save_area_addr);
+ mqd->cp_hqd_ctx_save_base_addr_hi =
+ upper_32_bits(prop->ctx_save_area_addr);
+ mqd->cp_hqd_ctx_save_size = prop->ctx_save_area_size;
+ mqd->cp_hqd_cntl_stack_size = prop->ctl_stack_size;
+ mqd->cp_hqd_cntl_stack_offset = prop->ctl_stack_size;
+ mqd->cp_hqd_wg_state_offset = prop->ctl_stack_size;
+ }
+
return 0;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
index 1b76c10d1ac2..3b7a1479fbf7 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
@@ -27,6 +27,7 @@
#include "mes_userqueue.h"
#include "amdgpu_userq_fence.h"
#include "amdgpu_trace.h"
+#include "amdgpu_cwsr.h"
#define AMDGPU_USERQ_PROC_CTX_SZ PAGE_SIZE
#define AMDGPU_USERQ_GANG_CTX_SZ PAGE_SIZE
@@ -115,6 +116,7 @@ static int convert_to_mes_priority(int priority)
static int mes_userq_map(struct amdgpu_usermode_queue *queue)
{
struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
+ struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
struct amdgpu_device *adev = uq_mgr->adev;
struct amdgpu_userq_obj *ctx = &queue->fw_obj;
struct amdgpu_mqd_prop *userq_props = queue->userq_prop;
@@ -146,24 +148,37 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo);
queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr;
+ if (fpriv->cwsr_trap) {
+ queue_input.tba_addr = fpriv->cwsr_trap->tba_gpu_va_addr;
+ queue_input.tma_addr = fpriv->cwsr_trap->tma_gpu_va_addr;
+ queue_input.trap_en = !amdgpu_cwsr_has_dbg_wa(adev);
+ }
+
if (mes->use_rs64mem) {
if (!uq_mgr->proc_ctx_allocated) {
- r = amdgpu_mes_alloc_proc_ctx_index(mes, &uq_mgr->proc_ctx_array_index);
+ r = amdgpu_mes_alloc_proc_ctx_index(mes,
+ &uq_mgr->proc_ctx_array_index);
if (r) {
- DRM_ERROR("Failed to allocate userq process index err:%d\n", r);
+ DRM_ERROR("Failed to allocate userq process index err:%d\n",
+ r);
return r;
}
uq_mgr->proc_ctx_allocated = true;
}
- r = amdgpu_mes_alloc_gang_ctx_index(mes, &queue->gang_ctx_array_index);
+ r = amdgpu_mes_alloc_gang_ctx_index(mes,
+ &queue->gang_ctx_array_index);
if (r) {
DRM_ERROR("Failed to allocate userq gang index err:%d\n", r);
return r;
}
- queue_input.process_context_array_index = uq_mgr->proc_ctx_array_index;
- queue_input.gang_context_array_index = queue->gang_ctx_array_index;
+
+ queue_input.process_context_array_index =
+ uq_mgr->proc_ctx_array_index;
+ queue_input.gang_context_array_index =
+ queue->gang_ctx_array_index;
}
+
amdgpu_mes_lock(&adev->mes);
r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
amdgpu_mes_unlock(&adev->mes);
@@ -174,7 +189,9 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
amdgpu_mes_userq_queue_mapped(adev);
- DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index);
+ DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n",
+ userq_props->doorbell_index);
+
return 0;
}
@@ -398,6 +415,7 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) {
struct drm_amdgpu_userq_mqd_compute_gfx11 *compute_mqd;
+ struct amdgpu_cwsr_params cwsr_params;
if (mqd_user->mqd_size != sizeof(*compute_mqd)) {
DRM_ERROR("Invalid compute IP MQD size\n");
@@ -405,7 +423,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
goto free_mqd;
}
- compute_mqd = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
+ compute_mqd = memdup_user(u64_to_user_ptr(mqd_user->mqd),
+ mqd_user->mqd_size);
if (IS_ERR(compute_mqd)) {
r = PTR_ERR(compute_mqd);
goto free_mqd;
@@ -416,6 +435,7 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
kfree(compute_mqd);
goto free_mqd;
}
+
r = amdgpu_userq_input_va_validate(adev, queue,
compute_mqd->eop_va, 2048,
&queue->userq_vas.va.eop);
@@ -427,29 +447,63 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
userq_props->eop_gpu_addr = compute_mqd->eop_va;
userq_props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL;
- userq_props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM;
+ userq_props->hqd_queue_priority =
+ AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM;
userq_props->hqd_active = false;
userq_props->tmz_queue =
mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE;
+
+ if (amdgpu_cwsr_is_enabled(adev)) {
+ struct amdgpu_fpriv *fpriv =
+ uq_mgr_to_fpriv(queue->userq_mgr);
+ int num_xcc;
+
+ num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr,
+ fpriv->xcp_id);
+ cwsr_params.ctx_save_area_address =
+ userq_props->ctx_save_area_addr;
+ cwsr_params.cwsr_sz =
+ userq_props->ctx_save_area_size;
+ userq_props->ctl_stack_size =
+ adev->cwsr_info->xcc_ctl_stack_sz * num_xcc;
+
+ r = amdgpu_bo_reserve(queue->vm->root.bo, false);
+ if (r) {
+ kfree(compute_mqd);
+ goto free_mqd;
+ }
+
+ r = amdgpu_userq_input_cwsr_params_validate(
+ queue, &cwsr_params);
+ amdgpu_bo_unreserve(queue->vm->root.bo);
+ if (r) {
+ kfree(compute_mqd);
+ goto free_mqd;
+ }
+ }
+
kfree(compute_mqd);
} else if (queue->queue_type == AMDGPU_HW_IP_GFX) {
struct drm_amdgpu_userq_mqd_gfx11 *mqd_gfx_v11;
struct amdgpu_gfx_shadow_info shadow_info;
if (adev->gfx.funcs->get_gfx_shadow_info) {
- adev->gfx.funcs->get_gfx_shadow_info(adev, &shadow_info, true);
+ adev->gfx.funcs->get_gfx_shadow_info(adev,
+ &shadow_info, true);
} else {
r = -EINVAL;
goto free_mqd;
}
- if (mqd_user->mqd_size != sizeof(*mqd_gfx_v11) || !mqd_user->mqd) {
+ if (mqd_user->mqd_size != sizeof(*mqd_gfx_v11) ||
+ !mqd_user->mqd) {
DRM_ERROR("Invalid GFX MQD\n");
r = -EINVAL;
goto free_mqd;
}
- mqd_gfx_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
+ mqd_gfx_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd),
+ mqd_user->mqd_size);
if (IS_ERR(mqd_gfx_v11)) {
r = PTR_ERR(mqd_gfx_v11);
goto free_mqd;
@@ -465,7 +519,9 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
kfree(mqd_gfx_v11);
goto free_mqd;
}
- r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->shadow_va,
+
+ r = amdgpu_userq_input_va_validate(adev, queue,
+ mqd_gfx_v11->shadow_va,
shadow_info.shadow_size,
&queue->userq_vas.va.shadow);
if (r) {
@@ -474,7 +530,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
goto free_mqd;
}
- r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->csa_va,
+ r = amdgpu_userq_input_va_validate(adev, queue,
+ mqd_gfx_v11->csa_va,
shadow_info.csa_size,
&queue->userq_vas.va.csa);
amdgpu_bo_unreserve(queue->vm->root.bo);
@@ -487,13 +544,15 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
} else if (queue->queue_type == AMDGPU_HW_IP_DMA) {
struct drm_amdgpu_userq_mqd_sdma_gfx11 *mqd_sdma_v11;
- if (mqd_user->mqd_size != sizeof(*mqd_sdma_v11) || !mqd_user->mqd) {
+ if (mqd_user->mqd_size != sizeof(*mqd_sdma_v11) ||
+ !mqd_user->mqd) {
DRM_ERROR("Invalid SDMA MQD\n");
r = -EINVAL;
goto free_mqd;
}
- mqd_sdma_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
+ mqd_sdma_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd),
+ mqd_user->mqd_size);
if (IS_ERR(mqd_sdma_v11)) {
r = PTR_ERR(mqd_sdma_v11);
goto free_mqd;
@@ -504,7 +563,9 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
kfree(mqd_sdma_v11);
goto free_mqd;
}
- r = amdgpu_userq_input_va_validate(adev, queue, mqd_sdma_v11->csa_va,
+
+ r = amdgpu_userq_input_va_validate(adev, queue,
+ mqd_sdma_v11->csa_va,
32,
&queue->userq_vas.va.csa);
amdgpu_bo_unreserve(queue->vm->root.bo);
@@ -519,7 +580,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
queue->userq_prop = userq_props;
- r = mqd_hw_default->init_mqd(adev, (void *)queue->mqd.cpu_ptr, userq_props);
+ r = mqd_hw_default->init_mqd(adev, (void *)queue->mqd.cpu_ptr,
+ userq_props);
if (r) {
DRM_ERROR("Failed to initialize MQD for userqueue\n");
goto free_mqd;
@@ -528,7 +590,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
/* Create per-process MES process context BO */
r = mes_userq_create_proc_ctx_space(uq_mgr);
if (r) {
- DRM_ERROR("Failed to allocate MES process context space bo, error: %d\n", r);
+ DRM_ERROR("Failed to allocate MES process context space bo, error: %d\n",
+ r);
goto free_mqd;
}
@@ -540,7 +603,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
}
/* FW expects WPTR BOs to be mapped into GART */
- r = mes_userq_create_wptr_mapping(adev, uq_mgr, queue, userq_props->wptr_gpu_addr);
+ r = mes_userq_create_wptr_mapping(adev, uq_mgr, queue,
+ userq_props->wptr_gpu_addr);
if (r) {
DRM_ERROR("Failed to create WPTR mapping\n");
goto free_ctx;
--
2.34.1
^ permalink raw reply related [flat|nested] 16+ messages in thread* Re: [PATCH 2/7] drm/amdgpu: Integrate CWSR with user queues
2026-10-08 7:05 ` [PATCH 2/7] drm/amdgpu: Integrate CWSR with " Srinivasan Shanmugam
@ 2026-10-12 13:12 ` Christian König
0 siblings, 0 replies; 16+ messages in thread
From: Christian König @ 2026-10-12 13:12 UTC (permalink / raw)
To: Srinivasan Shanmugam, Alex Deucher; +Cc: amd-gfx, Lijo Lazar, James Zhu
On 10/8/26 09:05, Srinivasan Shanmugam wrote:
> From: Lijo Lazar <lijo.lazar@amd.com>
>
> Initialize the CWSR handler during device initialization and release
> the associated resources during device teardown.
>
> Add a per-file CWSR trap object for render-node user queues and map the
> first-level CWSR trap handler into the GPUVM associated with the file.
> Use the mapped TBA and TMA addresses when programming user queues.
>
> Add user save area parameters to the MQD properties for compute queue
> creation and validate the parameters before using them for MQD
> initialization.
>
> The CWSR parameter validation walks the VM mapping tree and updates the
> user queue VA mapping state. Reserve the VM root BO around the
> validation so the required VM reservation is held while accessing the
> mapping tree.
>
> Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
> Co-developed-by: James Zhu <James.Zhu@amd.com>
> Signed-off-by: James Zhu <James.Zhu@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 +
> drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h | 6 ++
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 8 ++
> drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 10 ++
> drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h | 8 ++
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 25 +++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 8 +-
> drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c | 14 +++
> drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 102 +++++++++++++++++----
> 9 files changed, 163 insertions(+), 20 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index 26561a4f4db8..bdd55ac5b06e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -333,6 +333,7 @@ struct amdgpu_reset_control;
> struct amdgpu_coredump_info;
> struct amdgpu_video_codecs;
> struct amdgpu_cwsr_isa;
> +struct amdgpu_cwsr_trap_obj;
>
> enum amdgpu_cp_irq {
> AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP = 0,
> @@ -421,6 +422,7 @@ struct amdgpu_fpriv {
> struct xarray bo_list_handles;
> struct amdgpu_ctx_mgr ctx_mgr;
> struct amdgpu_userq_mgr userq_mgr;
> + struct amdgpu_cwsr_trap_obj *cwsr_trap;
>
> /* Eviction fence infra */
> struct amdgpu_eviction_fence_mgr evf_mgr;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
> index 7f922559e297..5717e1d662fc 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
> @@ -78,4 +78,10 @@ uint32_t amdgpu_cwsr_size_needed(struct amdgpu_device *adev, int num_xcc);
> int amdgpu_cwsr_validate_params(struct amdgpu_device *adev,
> struct amdgpu_cwsr_params *cwsr_params,
> int num_xcc);
> +static inline bool amdgpu_cwsr_has_dbg_wa(struct amdgpu_device *adev)
> +{
> + u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
> +
> + return gc_ver >= IP_VERSION(11, 0, 0) && gc_ver <= IP_VERSION(11, 0, 3);
> +}
> #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index e0014611f039..5a7f111763bf 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -81,6 +81,7 @@
> #include "amdgpu_reset.h"
> #include "amdgpu_virt.h"
> #include "amdgpu_dev_coredump.h"
> +#include "amdgpu_cwsr.h"
>
> #include <linux/suspend.h>
> #include <drm/task_barrier.h>
> @@ -2549,6 +2550,12 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
>
> r = amdgpu_cper_init(adev);
>
> + if (!r) {
Better make that if (r) goto init_failed; to keep the style consistend.
> + r = amdgpu_cwsr_init(adev);
> + if (r == -EOPNOTSUPP)
> + r = 0;
> + }
> +
> init_failed:
>
> return r;
> @@ -2946,6 +2953,7 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
> {
> int i, r;
>
> + amdgpu_cwsr_fini(adev);
> amdgpu_cper_fini(adev);
>
> if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index f34150d20a2f..f3945b3988d8 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -49,6 +49,8 @@
> #include "amdgpu_userq.h"
> #include "amdgpu_video_codecs.h"
> #include "amdgpu_ras_mgr.h"
> +#include "amdgpu_vm.h"
> +#include "amdgpu_cwsr.h"
>
> void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev)
> {
> @@ -1552,6 +1554,13 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
> r);
>
> amdgpu_evf_mgr_init(&fpriv->evf_mgr);
> +
> + if (amdgpu_cwsr_is_enabled(adev)) {
Better move that check into amdgpu_cwsr_alloc().
Apart from those nit picks looks good on first glance,
Christian.
> + r = amdgpu_cwsr_alloc(adev, &fpriv->vm, &fpriv->cwsr_trap);
> + if (r)
> + dev_dbg(adev->dev, "cwsr trap not enabled");
> + }
> +
> amdgpu_ctx_mgr_init(&fpriv->ctx_mgr, adev);
>
> file_priv->driver_priv = fpriv;
> @@ -1621,6 +1630,7 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
> }
>
> amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr);
> + amdgpu_cwsr_free(adev, &fpriv->vm, &fpriv->cwsr_trap);
>
> if (pasid)
> amdgpu_pasid_free_delayed(pd->tbo.base.resv, pasid);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> index 2b13a5b37539..b3df09dbcbad 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> @@ -527,6 +527,14 @@ struct amdgpu_mqd_prop {
> uint32_t cu_mask_count;
> uint32_t cu_flags;
> bool is_user_cu_masked;
> + bool aql_queue;
> + u32 queue_percentage;
> + /* used in gfx9 and gfx12.1 */
> + u32 pm4_target_xcc;
> + /* cwsr params */
> + u64 ctx_save_area_addr;
> + u32 ctx_save_area_size;
> + u32 ctl_stack_size;
> };
>
> struct amdgpu_mqd {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index e29823ae8672..6eea0ca40556 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -35,6 +35,7 @@
> #include "amdgpu_hmm.h"
> #include "amdgpu_userq_fence.h"
> #include "amdgpu_trace.h"
> +#include "amdgpu_cwsr.h"
>
> u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev)
> {
> @@ -266,6 +267,30 @@ int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
> return -EINVAL;
> }
>
> +int amdgpu_userq_input_cwsr_params_validate(
> + struct amdgpu_usermode_queue *queue,
> + struct amdgpu_cwsr_params *cwsr_params)
> +{
> + struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(queue->userq_mgr);
> + struct amdgpu_device *adev = queue->userq_mgr->adev;
> + u32 cwsr_size;
> + int num_xcc;
> + int r;
> +
> + num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr, fpriv->xcp_id);
> + r = amdgpu_cwsr_validate_params(queue->userq_mgr->adev, cwsr_params,
> + num_xcc);
> + if (r)
> + return r;
> + cwsr_size = amdgpu_cwsr_size_needed(queue->userq_mgr->adev, num_xcc);
> + if (!cwsr_size)
> + return -EOPNOTSUPP;
> +
> + return amdgpu_userq_input_va_validate(
> + adev, queue, cwsr_params->ctx_save_area_address, cwsr_size,
> + &queue->userq_vas.va.cwsr);
> +}
> +
> static bool amdgpu_userq_buffer_va_mapped(struct amdgpu_vm *vm, u64 addr)
> {
> struct amdgpu_bo_va_mapping *mapping;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> index 488dc21d7c81..727f60c40bbf 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> @@ -41,6 +41,7 @@ enum amdgpu_userq_state {
> };
>
> struct amdgpu_mqd_prop;
> +struct amdgpu_cwsr_params;
>
> struct amdgpu_userq_obj {
> void *cpu_ptr;
> @@ -97,8 +98,9 @@ struct amdgpu_usermode_queue {
> u64 eop;
> u64 shadow;
> u64 csa;
> + u64 cwsr;
> } va;
> - u64 va_array[6];
> + u64 va_array[7];
> } userq_vas;
>
> uint32_t gang_ctx_array_index;
> @@ -200,4 +202,8 @@ int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
>
> void amdgpu_userq_gem_va_unmap_validate(struct amdgpu_device *adev,
> struct amdgpu_bo_va_mapping *mapping);
> +int amdgpu_userq_input_cwsr_params_validate(
> + struct amdgpu_usermode_queue *queue,
> + struct amdgpu_cwsr_params *cwsr_params);
> +
> #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
> index 420ce997bebc..b5326c2067db 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
> @@ -3327,6 +3327,20 @@ static int gfx_v12_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
> /* set CU mask */
> gfx_v12_0_compute_mqd_set_cu_mask(adev, mqd, prop);
>
> + /* If non-zero, assume cwsr is enabled */
> + if (prop->ctx_save_area_addr) {
> + mqd->cp_hqd_persistent_state |=
> + (1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT);
> + mqd->cp_hqd_ctx_save_base_addr_lo =
> + lower_32_bits(prop->ctx_save_area_addr);
> + mqd->cp_hqd_ctx_save_base_addr_hi =
> + upper_32_bits(prop->ctx_save_area_addr);
> + mqd->cp_hqd_ctx_save_size = prop->ctx_save_area_size;
> + mqd->cp_hqd_cntl_stack_size = prop->ctl_stack_size;
> + mqd->cp_hqd_cntl_stack_offset = prop->ctl_stack_size;
> + mqd->cp_hqd_wg_state_offset = prop->ctl_stack_size;
> + }
> +
> return 0;
> }
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> index 1b76c10d1ac2..3b7a1479fbf7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> +++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> @@ -27,6 +27,7 @@
> #include "mes_userqueue.h"
> #include "amdgpu_userq_fence.h"
> #include "amdgpu_trace.h"
> +#include "amdgpu_cwsr.h"
>
> #define AMDGPU_USERQ_PROC_CTX_SZ PAGE_SIZE
> #define AMDGPU_USERQ_GANG_CTX_SZ PAGE_SIZE
> @@ -115,6 +116,7 @@ static int convert_to_mes_priority(int priority)
> static int mes_userq_map(struct amdgpu_usermode_queue *queue)
> {
> struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
> + struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
> struct amdgpu_device *adev = uq_mgr->adev;
> struct amdgpu_userq_obj *ctx = &queue->fw_obj;
> struct amdgpu_mqd_prop *userq_props = queue->userq_prop;
> @@ -146,24 +148,37 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
> queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo);
> queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr;
>
> + if (fpriv->cwsr_trap) {
> + queue_input.tba_addr = fpriv->cwsr_trap->tba_gpu_va_addr;
> + queue_input.tma_addr = fpriv->cwsr_trap->tma_gpu_va_addr;
> + queue_input.trap_en = !amdgpu_cwsr_has_dbg_wa(adev);
> + }
> +
> if (mes->use_rs64mem) {
> if (!uq_mgr->proc_ctx_allocated) {
> - r = amdgpu_mes_alloc_proc_ctx_index(mes, &uq_mgr->proc_ctx_array_index);
> + r = amdgpu_mes_alloc_proc_ctx_index(mes,
> + &uq_mgr->proc_ctx_array_index);
> if (r) {
> - DRM_ERROR("Failed to allocate userq process index err:%d\n", r);
> + DRM_ERROR("Failed to allocate userq process index err:%d\n",
> + r);
> return r;
> }
> uq_mgr->proc_ctx_allocated = true;
> }
>
> - r = amdgpu_mes_alloc_gang_ctx_index(mes, &queue->gang_ctx_array_index);
> + r = amdgpu_mes_alloc_gang_ctx_index(mes,
> + &queue->gang_ctx_array_index);
> if (r) {
> DRM_ERROR("Failed to allocate userq gang index err:%d\n", r);
> return r;
> }
> - queue_input.process_context_array_index = uq_mgr->proc_ctx_array_index;
> - queue_input.gang_context_array_index = queue->gang_ctx_array_index;
> +
> + queue_input.process_context_array_index =
> + uq_mgr->proc_ctx_array_index;
> + queue_input.gang_context_array_index =
> + queue->gang_ctx_array_index;
> }
> +
> amdgpu_mes_lock(&adev->mes);
> r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
> amdgpu_mes_unlock(&adev->mes);
> @@ -174,7 +189,9 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
>
> amdgpu_mes_userq_queue_mapped(adev);
>
> - DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index);
> + DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n",
> + userq_props->doorbell_index);
> +
> return 0;
> }
>
> @@ -398,6 +415,7 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
>
> if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) {
> struct drm_amdgpu_userq_mqd_compute_gfx11 *compute_mqd;
> + struct amdgpu_cwsr_params cwsr_params;
>
> if (mqd_user->mqd_size != sizeof(*compute_mqd)) {
> DRM_ERROR("Invalid compute IP MQD size\n");
> @@ -405,7 +423,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
> goto free_mqd;
> }
>
> - compute_mqd = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
> + compute_mqd = memdup_user(u64_to_user_ptr(mqd_user->mqd),
> + mqd_user->mqd_size);
> if (IS_ERR(compute_mqd)) {
> r = PTR_ERR(compute_mqd);
> goto free_mqd;
> @@ -416,6 +435,7 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
> kfree(compute_mqd);
> goto free_mqd;
> }
> +
> r = amdgpu_userq_input_va_validate(adev, queue,
> compute_mqd->eop_va, 2048,
> &queue->userq_vas.va.eop);
> @@ -427,29 +447,63 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
>
> userq_props->eop_gpu_addr = compute_mqd->eop_va;
> userq_props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL;
> - userq_props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM;
> + userq_props->hqd_queue_priority =
> + AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM;
> userq_props->hqd_active = false;
> userq_props->tmz_queue =
> mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE;
> +
> + if (amdgpu_cwsr_is_enabled(adev)) {
> + struct amdgpu_fpriv *fpriv =
> + uq_mgr_to_fpriv(queue->userq_mgr);
> + int num_xcc;
> +
> + num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr,
> + fpriv->xcp_id);
> + cwsr_params.ctx_save_area_address =
> + userq_props->ctx_save_area_addr;
> + cwsr_params.cwsr_sz =
> + userq_props->ctx_save_area_size;
> + userq_props->ctl_stack_size =
> + adev->cwsr_info->xcc_ctl_stack_sz * num_xcc;
> +
> + r = amdgpu_bo_reserve(queue->vm->root.bo, false);
> + if (r) {
> + kfree(compute_mqd);
> + goto free_mqd;
> + }
> +
> + r = amdgpu_userq_input_cwsr_params_validate(
> + queue, &cwsr_params);
> + amdgpu_bo_unreserve(queue->vm->root.bo);
> + if (r) {
> + kfree(compute_mqd);
> + goto free_mqd;
> + }
> + }
> +
> kfree(compute_mqd);
> } else if (queue->queue_type == AMDGPU_HW_IP_GFX) {
> struct drm_amdgpu_userq_mqd_gfx11 *mqd_gfx_v11;
> struct amdgpu_gfx_shadow_info shadow_info;
>
> if (adev->gfx.funcs->get_gfx_shadow_info) {
> - adev->gfx.funcs->get_gfx_shadow_info(adev, &shadow_info, true);
> + adev->gfx.funcs->get_gfx_shadow_info(adev,
> + &shadow_info, true);
> } else {
> r = -EINVAL;
> goto free_mqd;
> }
>
> - if (mqd_user->mqd_size != sizeof(*mqd_gfx_v11) || !mqd_user->mqd) {
> + if (mqd_user->mqd_size != sizeof(*mqd_gfx_v11) ||
> + !mqd_user->mqd) {
> DRM_ERROR("Invalid GFX MQD\n");
> r = -EINVAL;
> goto free_mqd;
> }
>
> - mqd_gfx_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
> + mqd_gfx_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd),
> + mqd_user->mqd_size);
> if (IS_ERR(mqd_gfx_v11)) {
> r = PTR_ERR(mqd_gfx_v11);
> goto free_mqd;
> @@ -465,7 +519,9 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
> kfree(mqd_gfx_v11);
> goto free_mqd;
> }
> - r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->shadow_va,
> +
> + r = amdgpu_userq_input_va_validate(adev, queue,
> + mqd_gfx_v11->shadow_va,
> shadow_info.shadow_size,
> &queue->userq_vas.va.shadow);
> if (r) {
> @@ -474,7 +530,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
> goto free_mqd;
> }
>
> - r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->csa_va,
> + r = amdgpu_userq_input_va_validate(adev, queue,
> + mqd_gfx_v11->csa_va,
> shadow_info.csa_size,
> &queue->userq_vas.va.csa);
> amdgpu_bo_unreserve(queue->vm->root.bo);
> @@ -487,13 +544,15 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
> } else if (queue->queue_type == AMDGPU_HW_IP_DMA) {
> struct drm_amdgpu_userq_mqd_sdma_gfx11 *mqd_sdma_v11;
>
> - if (mqd_user->mqd_size != sizeof(*mqd_sdma_v11) || !mqd_user->mqd) {
> + if (mqd_user->mqd_size != sizeof(*mqd_sdma_v11) ||
> + !mqd_user->mqd) {
> DRM_ERROR("Invalid SDMA MQD\n");
> r = -EINVAL;
> goto free_mqd;
> }
>
> - mqd_sdma_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
> + mqd_sdma_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd),
> + mqd_user->mqd_size);
> if (IS_ERR(mqd_sdma_v11)) {
> r = PTR_ERR(mqd_sdma_v11);
> goto free_mqd;
> @@ -504,7 +563,9 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
> kfree(mqd_sdma_v11);
> goto free_mqd;
> }
> - r = amdgpu_userq_input_va_validate(adev, queue, mqd_sdma_v11->csa_va,
> +
> + r = amdgpu_userq_input_va_validate(adev, queue,
> + mqd_sdma_v11->csa_va,
> 32,
> &queue->userq_vas.va.csa);
> amdgpu_bo_unreserve(queue->vm->root.bo);
> @@ -519,7 +580,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
>
> queue->userq_prop = userq_props;
>
> - r = mqd_hw_default->init_mqd(adev, (void *)queue->mqd.cpu_ptr, userq_props);
> + r = mqd_hw_default->init_mqd(adev, (void *)queue->mqd.cpu_ptr,
> + userq_props);
> if (r) {
> DRM_ERROR("Failed to initialize MQD for userqueue\n");
> goto free_mqd;
> @@ -528,7 +590,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
> /* Create per-process MES process context BO */
> r = mes_userq_create_proc_ctx_space(uq_mgr);
> if (r) {
> - DRM_ERROR("Failed to allocate MES process context space bo, error: %d\n", r);
> + DRM_ERROR("Failed to allocate MES process context space bo, error: %d\n",
> + r);
> goto free_mqd;
> }
>
> @@ -540,7 +603,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
> }
>
> /* FW expects WPTR BOs to be mapped into GART */
> - r = mes_userq_create_wptr_mapping(adev, uq_mgr, queue, userq_props->wptr_gpu_addr);
> + r = mes_userq_create_wptr_mapping(adev, uq_mgr, queue,
> + userq_props->wptr_gpu_addr);
> if (r) {
> DRM_ERROR("Failed to create WPTR mapping\n");
> goto free_ctx;
^ permalink raw reply [flat|nested] 16+ messages in thread
* [PATCH 3/7] drm/amdgpu: Add CWSR userspace interfaces
2026-10-08 7:05 [PATCH 0/7] drm/amdgpu: Add CWSR and second-level trap handler support Srinivasan Shanmugam
2026-10-08 7:05 ` [PATCH 1/7] drm/amdgpu: Add CWSR infrastructure for user queues Srinivasan Shanmugam
2026-10-08 7:05 ` [PATCH 2/7] drm/amdgpu: Integrate CWSR with " Srinivasan Shanmugam
@ 2026-10-08 7:05 ` Srinivasan Shanmugam
2026-10-12 13:14 ` Christian König
2026-10-08 7:05 ` [PATCH 4/7] drm/amdgpu: Add second-level trap handler infrastructure Srinivasan Shanmugam
` (3 subsequent siblings)
6 siblings, 1 reply; 16+ messages in thread
From: Srinivasan Shanmugam @ 2026-10-08 7:05 UTC (permalink / raw)
To: Christian König, Alex Deucher; +Cc: amd-gfx, Lijo Lazar
From: Lijo Lazar <lijo.lazar@amd.com>
Add an ioctl to return the size information required for CWSR regions.
Add CWSR parameters to the user queue ioctl so userspace can provide
the GPU virtual address and allocated size of the save/restore buffer.
The parameters are supported for user compute queues.
Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 21 +++++++++++++++++
drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 13 +++++++----
include/uapi/drm/amdgpu_drm.h | 27 ++++++++++++++++++++++
3 files changed, 57 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index f3945b3988d8..147c0bd02731 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -1429,6 +1429,27 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
return -EINVAL;
}
}
+ case AMDGPU_INFO_CWSR: {
+ struct drm_amdgpu_info_cwsr cwsr_info;
+ int num_xcc, r;
+
+ fpriv = (struct amdgpu_fpriv *)filp->driver_priv;
+ if (!amdgpu_cwsr_is_enabled(adev) || !fpriv->cwsr_trap)
+ return -EOPNOTSUPP;
+ num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr, fpriv->xcp_id);
+ cwsr_info.ctl_stack_size =
+ adev->cwsr_info->xcc_ctl_stack_sz * num_xcc;
+ cwsr_info.dbg_mem_size =
+ adev->cwsr_info->xcc_dbg_mem_sz * num_xcc;
+ cwsr_info.min_save_area_size =
+ amdgpu_cwsr_size_needed(adev, num_xcc);
+ r = copy_to_user(out, &cwsr_info,
+ min((size_t)size, sizeof(cwsr_info))) ?
+ -EFAULT :
+ 0;
+ return r;
+ }
+
default:
DRM_DEBUG_KMS("Invalid request %d\n", info->query);
return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
index 3b7a1479fbf7..5311e1ad0201 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
@@ -461,11 +461,9 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr,
fpriv->xcp_id);
cwsr_params.ctx_save_area_address =
- userq_props->ctx_save_area_addr;
+ compute_mqd->ctx_save_area_va;
cwsr_params.cwsr_sz =
- userq_props->ctx_save_area_size;
- userq_props->ctl_stack_size =
- adev->cwsr_info->xcc_ctl_stack_sz * num_xcc;
+ compute_mqd->ctx_save_area_size;
r = amdgpu_bo_reserve(queue->vm->root.bo, false);
if (r) {
@@ -480,6 +478,13 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
kfree(compute_mqd);
goto free_mqd;
}
+
+ userq_props->ctx_save_area_addr =
+ compute_mqd->ctx_save_area_va;
+ userq_props->ctx_save_area_size =
+ compute_mqd->ctx_save_area_size;
+ userq_props->ctl_stack_size =
+ adev->cwsr_info->xcc_ctl_stack_sz * num_xcc;
}
kfree(compute_mqd);
diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
index 29009f6aabde..9317018330ec 100644
--- a/include/uapi/drm/amdgpu_drm.h
+++ b/include/uapi/drm/amdgpu_drm.h
@@ -470,6 +470,17 @@ struct drm_amdgpu_userq_mqd_compute_gfx11 {
* to get the size.
*/
__u64 eop_va;
+ /**
+ * @ctx_save_area_va: Virtual address of the GPU memory for save/restore buffer.
+ * This must be from a separate GPU object, and use AMDGPU_INFO IOCTL
+ * to get the size. This includes control stack, wave context and debugger memory.
+ */
+ __u64 ctx_save_area_va;
+ /**
+ * @ctx_save_area_size: Total size (in bytes) allocated for save/restore buffer.
+ * Use AMDGPU_INFO IOCTL to get the size.
+ */
+ __u32 ctx_save_area_size;
};
/* userq signal/wait ioctl */
@@ -1289,6 +1300,8 @@ struct drm_amdgpu_cs_chunk_cp_gfx_shadow {
#define AMDGPU_INFO_GPUVM_FAULT 0x23
/* query FW object size and alignment */
#define AMDGPU_INFO_UQ_FW_AREAS 0x24
+/* query CWSR size and alignment */
+#define AMDGPU_INFO_CWSR 0x25
#define AMDGPU_INFO_MMR_SE_INDEX_SHIFT 0
#define AMDGPU_INFO_MMR_SE_INDEX_MASK 0xff
@@ -1668,6 +1681,20 @@ struct drm_amdgpu_info_uq_metadata {
};
};
+/**
+ * struct drm_amdgpu_info_cwsr - cwsr information
+ *
+ * Gives cwsr related size details. User needs to allocate buffer based on this.
+ */
+struct drm_amdgpu_info_cwsr {
+ /* Control stack size */
+ __u32 ctl_stack_size;
+ /* Debug memory area size */
+ __u32 dbg_mem_size;
+ /* Minimum save area size required */
+ __u32 min_save_area_size;
+};
+
/*
* Supported GPU families
*/
--
2.34.1
^ permalink raw reply related [flat|nested] 16+ messages in thread* Re: [PATCH 3/7] drm/amdgpu: Add CWSR userspace interfaces
2026-10-08 7:05 ` [PATCH 3/7] drm/amdgpu: Add CWSR userspace interfaces Srinivasan Shanmugam
@ 2026-10-12 13:14 ` Christian König
0 siblings, 0 replies; 16+ messages in thread
From: Christian König @ 2026-10-12 13:14 UTC (permalink / raw)
To: Srinivasan Shanmugam, Alex Deucher; +Cc: amd-gfx, Lijo Lazar
On 10/8/26 09:05, Srinivasan Shanmugam wrote:
> From: Lijo Lazar <lijo.lazar@amd.com>
>
> Add an ioctl to return the size information required for CWSR regions.
>
> Add CWSR parameters to the user queue ioctl so userspace can provide
> the GPU virtual address and allocated size of the save/restore buffer.
> The parameters are supported for user compute queues.
>
> Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
Reviewed-by: Christian König <christian.koenig@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 21 +++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 13 +++++++----
> include/uapi/drm/amdgpu_drm.h | 27 ++++++++++++++++++++++
> 3 files changed, 57 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index f3945b3988d8..147c0bd02731 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -1429,6 +1429,27 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
> return -EINVAL;
> }
> }
> + case AMDGPU_INFO_CWSR: {
> + struct drm_amdgpu_info_cwsr cwsr_info;
> + int num_xcc, r;
> +
> + fpriv = (struct amdgpu_fpriv *)filp->driver_priv;
> + if (!amdgpu_cwsr_is_enabled(adev) || !fpriv->cwsr_trap)
> + return -EOPNOTSUPP;
> + num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr, fpriv->xcp_id);
> + cwsr_info.ctl_stack_size =
> + adev->cwsr_info->xcc_ctl_stack_sz * num_xcc;
> + cwsr_info.dbg_mem_size =
> + adev->cwsr_info->xcc_dbg_mem_sz * num_xcc;
> + cwsr_info.min_save_area_size =
> + amdgpu_cwsr_size_needed(adev, num_xcc);
> + r = copy_to_user(out, &cwsr_info,
> + min((size_t)size, sizeof(cwsr_info))) ?
> + -EFAULT :
> + 0;
> + return r;
> + }
> +
> default:
> DRM_DEBUG_KMS("Invalid request %d\n", info->query);
> return -EINVAL;
> diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> index 3b7a1479fbf7..5311e1ad0201 100644
> --- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> +++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> @@ -461,11 +461,9 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
> num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr,
> fpriv->xcp_id);
> cwsr_params.ctx_save_area_address =
> - userq_props->ctx_save_area_addr;
> + compute_mqd->ctx_save_area_va;
> cwsr_params.cwsr_sz =
> - userq_props->ctx_save_area_size;
> - userq_props->ctl_stack_size =
> - adev->cwsr_info->xcc_ctl_stack_sz * num_xcc;
> + compute_mqd->ctx_save_area_size;
>
> r = amdgpu_bo_reserve(queue->vm->root.bo, false);
> if (r) {
> @@ -480,6 +478,13 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
> kfree(compute_mqd);
> goto free_mqd;
> }
> +
> + userq_props->ctx_save_area_addr =
> + compute_mqd->ctx_save_area_va;
> + userq_props->ctx_save_area_size =
> + compute_mqd->ctx_save_area_size;
> + userq_props->ctl_stack_size =
> + adev->cwsr_info->xcc_ctl_stack_sz * num_xcc;
> }
>
> kfree(compute_mqd);
> diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
> index 29009f6aabde..9317018330ec 100644
> --- a/include/uapi/drm/amdgpu_drm.h
> +++ b/include/uapi/drm/amdgpu_drm.h
> @@ -470,6 +470,17 @@ struct drm_amdgpu_userq_mqd_compute_gfx11 {
> * to get the size.
> */
> __u64 eop_va;
> + /**
> + * @ctx_save_area_va: Virtual address of the GPU memory for save/restore buffer.
> + * This must be from a separate GPU object, and use AMDGPU_INFO IOCTL
> + * to get the size. This includes control stack, wave context and debugger memory.
> + */
> + __u64 ctx_save_area_va;
> + /**
> + * @ctx_save_area_size: Total size (in bytes) allocated for save/restore buffer.
> + * Use AMDGPU_INFO IOCTL to get the size.
> + */
> + __u32 ctx_save_area_size;
> };
>
> /* userq signal/wait ioctl */
> @@ -1289,6 +1300,8 @@ struct drm_amdgpu_cs_chunk_cp_gfx_shadow {
> #define AMDGPU_INFO_GPUVM_FAULT 0x23
> /* query FW object size and alignment */
> #define AMDGPU_INFO_UQ_FW_AREAS 0x24
> +/* query CWSR size and alignment */
> +#define AMDGPU_INFO_CWSR 0x25
>
> #define AMDGPU_INFO_MMR_SE_INDEX_SHIFT 0
> #define AMDGPU_INFO_MMR_SE_INDEX_MASK 0xff
> @@ -1668,6 +1681,20 @@ struct drm_amdgpu_info_uq_metadata {
> };
> };
>
> +/**
> + * struct drm_amdgpu_info_cwsr - cwsr information
> + *
> + * Gives cwsr related size details. User needs to allocate buffer based on this.
> + */
> +struct drm_amdgpu_info_cwsr {
> + /* Control stack size */
> + __u32 ctl_stack_size;
> + /* Debug memory area size */
> + __u32 dbg_mem_size;
> + /* Minimum save area size required */
> + __u32 min_save_area_size;
> +};
> +
> /*
> * Supported GPU families
> */
^ permalink raw reply [flat|nested] 16+ messages in thread
* [PATCH 4/7] drm/amdgpu: Add second-level trap handler infrastructure
2026-10-08 7:05 [PATCH 0/7] drm/amdgpu: Add CWSR and second-level trap handler support Srinivasan Shanmugam
` (2 preceding siblings ...)
2026-10-08 7:05 ` [PATCH 3/7] drm/amdgpu: Add CWSR userspace interfaces Srinivasan Shanmugam
@ 2026-10-08 7:05 ` Srinivasan Shanmugam
2026-10-12 14:07 ` Christian König
2026-10-08 7:05 ` [PATCH 5/7] drm/amdgpu: Make first-level trap state GPUVM-owned Srinivasan Shanmugam
` (2 subsequent siblings)
6 siblings, 1 reply; 16+ messages in thread
From: Srinivasan Shanmugam @ 2026-10-08 7:05 UTC (permalink / raw)
To: Christian König, Alex Deucher
Cc: amd-gfx, Lijo Lazar, Felix Kuehling, Lancelot Six, James Zhu,
Samuel Pitoiset, Natalie Vock, Timur Kristóf,
Pierre-Eric Pelloux-Prayer, Srinivasan Shanmugam
From: Lijo Lazar <lijo.lazar@amd.com>
Expand the CWSR infrastructure to support second-level userspace trap
handlers and rename it to reflect the broader trap handler scope.
Rename the CWSR files and common interfaces to amdgpu_trap while keeping
CWSR-specific naming for the context save/restore functionality.
Add the basic support needed for a userspace second-level trap handler.
Store the second-level TBA and TMA addresses in the first-level TMA and
provide helpers to set and clear them for a GPUVM.
The first-level TMA buffer can move in memory, so do not keep it
permanently mapped on the CPU. Map it only while accessing its contents
and unmap it afterwards.
Check that the second-level TBA and TMA address ranges are mapped in the
GPUVM before using them. Wait for GPUVM work to stop before changing the
addresses and keep the VM locked while checking and updating them.
Add support for updating the debugger trap flag in the first-level TMA.
Do not expose a standalone TRAP ioctl. The second-level trap handler is
controlled through SET_L2_TRAP and CLEAR_L2_TRAP operations on the VM
ioctl added later in the series.
Suggested-by: Christian König <christian.koenig@amd.com>
Suggested-by: Alex Deucher <alexander.deucher@amd.com>
Cc: Felix Kuehling <felix.kuehling@amd.com>
Cc: Lancelot Six <lancelot.six@amd.com>
Cc: James Zhu <james.zhu@amd.com>
Cc: Lijo Lazar <lijo.lazar@amd.com>
Cc: Samuel Pitoiset <hakzsam@gmail.com>
Cc: Natalie Vock <natalie.vock@gmx.de>
Cc: Timur Kristóf <timur.kristof@gmail.com>
Cc: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
Co-developed-by: Srinivasan Shanmugam <srinivasan.shanmugam@amd.com>
Signed-off-by: Srinivasan Shanmugam <srinivasan.shanmugam@amd.com>
---
drivers/gpu/drm/amd/amdgpu/Makefile | 2 +-
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 8 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c | 534 ---------------
drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h | 87 ---
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 6 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 1 +
drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 18 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c | 753 +++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h | 143 ++++
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 13 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 7 +-
drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 24 +-
12 files changed, 935 insertions(+), 661 deletions(-)
delete mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c
delete mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 5086a482daf1..28bd81f0f720 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -72,7 +72,7 @@ amdgpu-y += amdgpu_device.o amdgpu_reg_access.o amdgpu_doorbell_mgr.o amdgpu_kms
amdgpu_eeprom.o amdgpu_mca.o amdgpu_psp_ta.o amdgpu_lsdma.o amdgpu_lockdep.o \
amdgpu_ring_mux.o amdgpu_xcp.o amdgpu_seq64.o amdgpu_dev_coredump.o \
amdgpu_cper.o amdgpu_userq_fence.o amdgpu_eviction_fence.o amdgpu_ip.o \
- amdgpu_wb.o amdgpu_cwsr.o
+ amdgpu_wb.o amdgpu_trap.o
amdgpu-$(CONFIG_PROC_FS) += amdgpu_fdinfo.o
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index bdd55ac5b06e..e55496f5ecdd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -332,8 +332,8 @@ struct amdgpu_reset_context;
struct amdgpu_reset_control;
struct amdgpu_coredump_info;
struct amdgpu_video_codecs;
-struct amdgpu_cwsr_isa;
-struct amdgpu_cwsr_trap_obj;
+struct amdgpu_trap_isa;
+struct amdgpu_trap_obj;
enum amdgpu_cp_irq {
AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP = 0,
@@ -422,7 +422,7 @@ struct amdgpu_fpriv {
struct xarray bo_list_handles;
struct amdgpu_ctx_mgr ctx_mgr;
struct amdgpu_userq_mgr userq_mgr;
- struct amdgpu_cwsr_trap_obj *cwsr_trap;
+ struct amdgpu_trap_obj *trap_obj;
/* Eviction fence infra */
struct amdgpu_eviction_fence_mgr evf_mgr;
@@ -978,7 +978,7 @@ struct amdgpu_device {
*/
struct amdgpu_kfd_dev kfd;
- struct amdgpu_cwsr_info *cwsr_info;
+ struct amdgpu_trap_info *trap_info;
};
static inline uint32_t amdgpu_ip_version(const struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c
deleted file mode 100644
index 6b6c11bb5fd7..000000000000
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c
+++ /dev/null
@@ -1,534 +0,0 @@
-/*
- * Copyright 2025 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#include <drm/drm_exec.h>
-
-#include "amdgpu.h"
-#include "cwsr_trap_handler.h"
-#include "amdgpu_cwsr.h"
-
-extern int cwsr_enable;
-
-#define AMDGPU_CWSR_TBA_MAX_SIZE (2 * AMDGPU_GPU_PAGE_SIZE)
-#define AMDGPU_CWSR_TMA_MAX_SIZE (AMDGPU_GPU_PAGE_SIZE)
-#define AMDGPU_CWSR_TMA_OFFSET (AMDGPU_CWSR_TBA_MAX_SIZE)
-
-#define SGPR_SIZE_PER_CU 0x4000
-#define LDS_SIZE_PER_CU 0x10000
-#define HWREG_SIZE_PER_CU 0x1000
-#define DEBUGGER_BYTES_ALIGN 64
-#define DEBUGGER_BYTES_PER_WAVE 32
-#define SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER 40
-
-enum amdgpu_cwsr_region {
- AMDGPU_CWSR_TBA,
- AMDGPU_CWSR_TMA,
-};
-
-static inline uint64_t amdgpu_cwsr_tba_vaddr(struct amdgpu_device *adev)
-{
- uint64_t addr = AMDGPU_VA_RESERVED_TRAP_UQ_START(adev);
-
- addr = amdgpu_gmc_sign_extend(addr);
-
- return addr;
-}
-
-static inline bool amdgpu_cwsr_is_supported(struct amdgpu_device *adev)
-{
- uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
-
- if (!cwsr_enable || adev->gfx.disable_uq ||
- gc_ver < IP_VERSION(9, 0, 1))
- return false;
-
- return true;
-}
-
-uint32_t amdgpu_cwsr_size_needed(struct amdgpu_device *adev, int num_xcc)
-{
- if (!amdgpu_cwsr_is_enabled(adev))
- return 0;
-
- return num_xcc *
- (adev->cwsr_info->xcc_cwsr_sz + adev->cwsr_info->xcc_dbg_mem_sz);
-}
-
-static void amdgpu_cwsr_init_isa_details(struct amdgpu_device *adev,
- struct amdgpu_cwsr_info *cwsr_info)
-{
- uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
-
- if (gc_ver < IP_VERSION(9, 0, 1)) {
- BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) >
- AMDGPU_CWSR_TBA_MAX_SIZE);
- cwsr_info->isa_buf = cwsr_trap_gfx8_hex;
- cwsr_info->isa_sz = sizeof(cwsr_trap_gfx8_hex);
- } else if (gc_ver == IP_VERSION(9, 4, 1)) {
- BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex) >
- AMDGPU_CWSR_TBA_MAX_SIZE);
- cwsr_info->isa_buf = cwsr_trap_arcturus_hex;
- cwsr_info->isa_sz = sizeof(cwsr_trap_arcturus_hex);
- } else if (gc_ver == IP_VERSION(9, 4, 2)) {
- BUILD_BUG_ON(sizeof(cwsr_trap_aldebaran_hex) >
- AMDGPU_CWSR_TBA_MAX_SIZE);
- cwsr_info->isa_buf = cwsr_trap_aldebaran_hex;
- cwsr_info->isa_sz = sizeof(cwsr_trap_aldebaran_hex);
- } else if (gc_ver == IP_VERSION(9, 4, 3) ||
- gc_ver == IP_VERSION(9, 4, 4)) {
- BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_4_3_hex) >
- AMDGPU_CWSR_TBA_MAX_SIZE);
- cwsr_info->isa_buf = cwsr_trap_gfx9_4_3_hex;
- cwsr_info->isa_sz = sizeof(cwsr_trap_gfx9_4_3_hex);
- } else if (gc_ver == IP_VERSION(9, 5, 0)) {
- BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_5_0_hex) > PAGE_SIZE);
- cwsr_info->isa_buf = cwsr_trap_gfx9_5_0_hex;
- cwsr_info->isa_sz = sizeof(cwsr_trap_gfx9_5_0_hex);
- } else if (gc_ver < IP_VERSION(10, 1, 1)) {
- BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) >
- AMDGPU_CWSR_TBA_MAX_SIZE);
- cwsr_info->isa_buf = cwsr_trap_gfx9_hex;
- cwsr_info->isa_sz = sizeof(cwsr_trap_gfx9_hex);
- } else if (gc_ver < IP_VERSION(10, 3, 0)) {
- BUILD_BUG_ON(sizeof(cwsr_trap_nv1x_hex) >
- AMDGPU_CWSR_TBA_MAX_SIZE);
- cwsr_info->isa_buf = cwsr_trap_nv1x_hex;
- cwsr_info->isa_sz = sizeof(cwsr_trap_nv1x_hex);
- } else if (gc_ver < IP_VERSION(11, 0, 0)) {
- BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex) >
- AMDGPU_CWSR_TBA_MAX_SIZE);
- cwsr_info->isa_buf = cwsr_trap_gfx10_hex;
- cwsr_info->isa_sz = sizeof(cwsr_trap_gfx10_hex);
- } else if (gc_ver < IP_VERSION(12, 0, 0)) {
- /* The gfx11 cwsr trap handler must fit inside a single
- page. */
- BUILD_BUG_ON(sizeof(cwsr_trap_gfx11_hex) > PAGE_SIZE);
- cwsr_info->isa_buf = cwsr_trap_gfx11_hex;
- cwsr_info->isa_sz = sizeof(cwsr_trap_gfx11_hex);
- } else if (gc_ver < IP_VERSION(12, 1, 0)) {
- BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_hex) >
- AMDGPU_CWSR_TBA_MAX_SIZE);
- cwsr_info->isa_buf = cwsr_trap_gfx12_hex;
- cwsr_info->isa_sz = sizeof(cwsr_trap_gfx12_hex);
- } else {
- BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_1_0_hex) >
- AMDGPU_CWSR_TBA_MAX_SIZE);
- cwsr_info->isa_buf = cwsr_trap_gfx12_1_0_hex;
- cwsr_info->isa_sz = sizeof(cwsr_trap_gfx12_1_0_hex);
- }
-}
-
-static uint32_t amdgpu_cwsr_get_vgpr_size_per_cu(struct amdgpu_device *adev)
-{
- uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
- uint32_t vgpr_size;
-
- switch (gc_ver) {
- case IP_VERSION(9, 4, 1): /* GFX_VERSION_ARCTURUS */
- case IP_VERSION(9, 4, 2): /* GFX_VERSION_ALDEBARAN */
- case IP_VERSION(9, 4, 3): /* GFX_VERSION_AQUA_VANJARAM */
- case IP_VERSION(9, 4, 4): /* GFX_VERSION_AQUA_VANJARAM */
- case IP_VERSION(9, 5, 0):
- vgpr_size = 0x80000;
- break;
- case IP_VERSION(11, 0, 0):
- case IP_VERSION(11, 0, 2):
- case IP_VERSION(11, 0, 3):
- case IP_VERSION(12, 0, 0):
- case IP_VERSION(12, 0, 1):
- vgpr_size = 0x60000;
- break;
- default:
- vgpr_size = 0x40000;
- break;
- }
-
- return vgpr_size;
-}
-
-static uint32_t amdgpu_cwsr_get_wg_ctxt_size_per_cu(struct amdgpu_device *adev)
-{
- uint32_t lds_sz_per_cu;
-
- lds_sz_per_cu =
- (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0)) ?
- (adev->gfx.cu_info.lds_size << 10) :
- LDS_SIZE_PER_CU;
-
- return amdgpu_cwsr_get_vgpr_size_per_cu(adev) + SGPR_SIZE_PER_CU +
- lds_sz_per_cu + HWREG_SIZE_PER_CU;
-}
-
-static uint32_t amdgpu_cwsr_ctl_stack_bytes_per_wave(struct amdgpu_device *adev)
-{
- uint32_t sz;
-
- if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 1, 0))
- sz = 12;
- else
- sz = 8;
- return sz;
-}
-
-static void amdgpu_cwsr_init_save_area_info(struct amdgpu_device *adev,
- struct amdgpu_cwsr_info *cwsr_info)
-{
- struct amdgpu_gfx_config *gfx_info = &adev->gfx.config;
- uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
- uint32_t ctl_stack_size, wg_data_size, dbg_mem_size;
- uint32_t array_count;
- uint32_t wave_num;
- uint32_t cu_num;
-
- if (gc_ver < IP_VERSION(9, 0, 1))
- return;
-
- array_count = gfx_info->max_shader_engines * gfx_info->max_sh_per_se;
-
- cu_num = adev->gfx.cu_info.number / NUM_XCC(adev->gfx.xcc_mask);
- wave_num = (gc_ver < IP_VERSION(10, 1, 0)) ? /* GFX_VERSION_NAVI10 */
- min(cu_num * 40,
- array_count / gfx_info->max_sh_per_se * 512) :
- cu_num * 32;
-
- wg_data_size = ALIGN(cu_num * amdgpu_cwsr_get_wg_ctxt_size_per_cu(adev),
- PAGE_SIZE);
- ctl_stack_size =
- wave_num * amdgpu_cwsr_ctl_stack_bytes_per_wave(adev) + 8;
- ctl_stack_size =
- ALIGN(SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER + ctl_stack_size,
- PAGE_SIZE);
- dbg_mem_size =
- ALIGN(wave_num * DEBUGGER_BYTES_PER_WAVE, DEBUGGER_BYTES_ALIGN);
- /*
- * HW design limits control stack size to 0x7000.
- * This is insufficient for theoretical PM4 cases
- * but sufficient for AQL, limited by SPI events.
- */
- if (IP_VERSION_MAJ(gc_ver) == 10)
- ctl_stack_size = min(ctl_stack_size, 0x7000);
-
- cwsr_info->xcc_ctl_stack_sz = ctl_stack_size;
- cwsr_info->xcc_cwsr_sz = ctl_stack_size + wg_data_size;
- cwsr_info->xcc_dbg_mem_sz = dbg_mem_size;
-}
-
-int amdgpu_cwsr_init(struct amdgpu_device *adev)
-{
- struct amdgpu_cwsr_info *cwsr_info __free(kfree) =
- kzalloc(sizeof(*cwsr_info), GFP_KERNEL);
- void *ptr;
- int r;
-
- if (!amdgpu_cwsr_is_supported(adev))
- return -EOPNOTSUPP;
-
- if (!cwsr_info)
- return -ENOMEM;
- amdgpu_cwsr_init_isa_details(adev, cwsr_info);
-
- if (!cwsr_info->isa_sz)
- return -EOPNOTSUPP;
-
- r = amdgpu_bo_create_kernel(adev, AMDGPU_CWSR_TBA_MAX_SIZE, PAGE_SIZE,
- AMDGPU_GEM_DOMAIN_GTT, &cwsr_info->isa_bo,
- NULL, &ptr);
- if (r)
- return r;
-
- memcpy(ptr, cwsr_info->isa_buf, cwsr_info->isa_sz);
-
- amdgpu_cwsr_init_save_area_info(adev, cwsr_info);
- adev->cwsr_info = no_free_ptr(cwsr_info);
-
- return 0;
-}
-
-void amdgpu_cwsr_fini(struct amdgpu_device *adev)
-{
- if (!amdgpu_cwsr_is_enabled(adev))
- return;
-
- amdgpu_bo_free_kernel(&adev->cwsr_info->isa_bo, NULL, NULL);
- kfree(adev->cwsr_info);
- adev->cwsr_info = NULL;
-}
-
-/*
- * amdgpu_map_cwsr_trap_handler should be called during amdgpu_vm_init
- * it maps virtual address amdgpu_cwsr_trap_handler_vaddr() to this VM, and each
- * compute queue can use this virtual address for wave save/restore
- * operations to support compute preemption.
- */
-static int amdgpu_cwsr_map_region(struct amdgpu_device *adev,
- struct amdgpu_vm *vm,
- struct amdgpu_cwsr_trap_obj *cwsr,
- enum amdgpu_cwsr_region region)
-{
- uint64_t cwsr_addr, va_flags, va;
- struct amdgpu_bo_va **bo_va;
- struct amdgpu_bo *bo;
- uint32_t size;
- int r;
-
- if (!cwsr || !vm)
- return -EINVAL;
-
- cwsr_addr = amdgpu_cwsr_tba_vaddr(adev);
-
- if (region == AMDGPU_CWSR_TBA) {
- size = AMDGPU_CWSR_TBA_MAX_SIZE;
- bo_va = &cwsr->tba_va;
- bo = adev->cwsr_info->isa_bo;
- va = cwsr_addr;
- va_flags = (AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
- AMDGPU_VM_PAGE_EXECUTABLE);
- } else {
- size = AMDGPU_CWSR_TMA_MAX_SIZE;
- bo_va = &cwsr->tma_va;
- bo = cwsr->tma_bo;
- va = cwsr_addr + AMDGPU_CWSR_TMA_OFFSET;
- va_flags = (AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE);
- }
-
- *bo_va = amdgpu_vm_bo_add(adev, vm, bo);
- if (!*bo_va)
- return -ENOMEM;
-
- va &= AMDGPU_GMC_HOLE_MASK;
- r = amdgpu_vm_bo_map(adev, *bo_va, va, 0, size, va_flags);
- if (r) {
- dev_err(adev->dev, "failed to do bo map of %s region, err=%d\n",
- (region == AMDGPU_CWSR_TBA ? "tba" : "tma"), r);
- amdgpu_vm_bo_del(adev, *bo_va);
- *bo_va = NULL;
- return r;
- }
-
- r = amdgpu_vm_bo_update(adev, *bo_va, false);
- if (r) {
- dev_err(adev->dev,
- "failed to do page table update of %s region, err=%d\n",
- (region == AMDGPU_CWSR_TBA ? "tba" : "tma"), r);
- amdgpu_vm_bo_del(adev, *bo_va);
- *bo_va = NULL;
- return r;
- }
-
- if (region == AMDGPU_CWSR_TBA)
- cwsr->tba_gpu_va_addr = va;
- else
- cwsr->tma_gpu_va_addr = va;
-
- return 0;
-}
-
-static int amdgpu_cwsr_unmap_region(struct amdgpu_device *adev,
- struct amdgpu_cwsr_trap_obj *cwsr,
- enum amdgpu_cwsr_region region)
-{
- struct amdgpu_bo_va **bo_va;
- uint64_t va;
- int r;
-
- if (!cwsr)
- return -EINVAL;
-
- if (region == AMDGPU_CWSR_TBA) {
- bo_va = &cwsr->tba_va;
- va = cwsr->tba_gpu_va_addr;
- } else {
- bo_va = &cwsr->tma_va;
- va = cwsr->tma_gpu_va_addr;
- }
-
- r = amdgpu_vm_bo_unmap(adev, *bo_va, va);
- if (r) {
- dev_err(adev->dev,
- "failed to do bo_unmap on CWSR trap handler, err=%d\n",
- r);
- return r;
- }
-
- amdgpu_vm_bo_del(adev, *bo_va);
- *bo_va = NULL;
-
- return r;
-}
-
-int amdgpu_cwsr_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
- struct amdgpu_cwsr_trap_obj **trap_obj)
-{
- struct amdgpu_bo_param bp;
- struct amdgpu_cwsr_trap_obj *cwsr;
- struct amdgpu_bo *bo;
- struct drm_exec exec;
- int r;
-
- if (!amdgpu_cwsr_is_enabled(adev))
- return -EOPNOTSUPP;
- if (!vm || !trap_obj)
- return -EINVAL;
-
- cwsr = kzalloc(sizeof(*cwsr), GFP_KERNEL);
- if (!cwsr)
- return -ENOMEM;
-
- bo = adev->cwsr_info->isa_bo;
- drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
- drm_exec_until_all_locked(&exec) {
- r = amdgpu_vm_lock_pd(vm, &exec, 0);
- if (likely(!r))
- r = drm_exec_lock_obj(&exec, &bo->tbo.base);
- drm_exec_retry_on_contention(&exec);
- if (unlikely(r)) {
- dev_err(adev->dev,
- "failed to reserve for CWSR allocs: err=%d\n",
- r);
- goto err_exec;
- }
- }
-
- /*
- * Create TMA BO using the VM's already-locked resv so no new
- * reservation_ww_class_mutex acquisition is needed, avoiding the
- * recursive ww_mutex deadlock. Unpinned; eviction is handled via
- * amdgpu_vm_bo_add() inside amdgpu_cwsr_map_region().
- */
- memset(&bp, 0, sizeof(bp));
- bp.size = AMDGPU_CWSR_TMA_MAX_SIZE;
- bp.byte_align = PAGE_SIZE;
- bp.domain = AMDGPU_GEM_DOMAIN_GTT;
- bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC |
- AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
- bp.type = ttm_bo_type_kernel;
- bp.bo_ptr_size = sizeof(struct amdgpu_bo);
- bp.resv = vm->root.bo->tbo.base.resv;
-
- r = amdgpu_bo_create(adev, &bp, &cwsr->tma_bo);
- if (r)
- goto err_exec;
-
- /* VM resv already held, kmap directly, no separate reserve needed. */
- r = amdgpu_bo_kmap(cwsr->tma_bo, &cwsr->tma_cpu_addr);
- if (r)
- goto err_tma;
-
- memset(cwsr->tma_cpu_addr, 0, AMDGPU_CWSR_TMA_MAX_SIZE);
-
- r = amdgpu_cwsr_map_region(adev, vm, cwsr, AMDGPU_CWSR_TMA);
- if (r)
- goto err_kmap;
-
- r = amdgpu_cwsr_map_region(adev, vm, cwsr, AMDGPU_CWSR_TBA);
- if (r) {
- amdgpu_cwsr_unmap_region(adev, cwsr, AMDGPU_CWSR_TMA);
- goto err_kmap;
- }
-
- drm_exec_fini(&exec);
- *trap_obj = cwsr;
- return 0;
-
-err_kmap:
- amdgpu_bo_kunmap(cwsr->tma_bo);
-err_tma:
- amdgpu_bo_unref(&cwsr->tma_bo);
-err_exec:
- drm_exec_fini(&exec);
- kfree(cwsr);
- *trap_obj = NULL;
- return r;
-}
-
-int amdgpu_cwsr_validate_params(struct amdgpu_device *adev,
- struct amdgpu_cwsr_params *cwsr_params,
- int num_xcc)
-{
- uint32_t min_size, max_size;
-
- if (!amdgpu_cwsr_is_enabled(adev))
- return -EOPNOTSUPP;
-
- if (!cwsr_params)
- return -EINVAL;
-
- min_size = amdgpu_cwsr_size_needed(adev, num_xcc);
- max_size = 2 * min_size;
- /*
- * Only save area size details checked. Address validation needs to be
- * carried out separately. User is expected to pass a cwsr buffer area
- * with size >= ctl stack size + wg data size + debug memory size
- */
- if (cwsr_params->cwsr_sz < min_size ||
- cwsr_params->cwsr_sz > max_size) {
- dev_dbg(adev->dev,
- "queue cwsr size 0x%x should be within min %u : max %u\n",
- cwsr_params->cwsr_sz, min_size, max_size);
- return -EINVAL;
- }
-
- return 0;
-}
-
-void amdgpu_cwsr_free(struct amdgpu_device *adev, struct amdgpu_vm *vm,
- struct amdgpu_cwsr_trap_obj **trap_obj)
-{
- struct amdgpu_bo *tba_bo;
- struct amdgpu_bo *tma_bo;
- struct drm_exec exec;
- int r;
-
- if (!trap_obj || !*trap_obj || !(*trap_obj)->tma_bo)
- return;
-
- tba_bo = adev->cwsr_info->isa_bo;
- tma_bo = (*trap_obj)->tma_bo;
-
- if (!tba_bo || !tma_bo)
- return;
-
- drm_exec_init(&exec, 0, 0);
- drm_exec_until_all_locked(&exec) {
- r = amdgpu_vm_lock_pd(vm, &exec, 0);
- if (likely(!r))
- r = drm_exec_lock_obj(&exec, &tba_bo->tbo.base);
- drm_exec_retry_on_contention(&exec);
- if (unlikely(r)) {
- dev_err(adev->dev,
- "failed to reserve CWSR BOs: err=%d\n", r);
- goto err;
- }
- }
-
- amdgpu_cwsr_unmap_region(adev, *trap_obj, AMDGPU_CWSR_TBA);
- amdgpu_cwsr_unmap_region(adev, *trap_obj, AMDGPU_CWSR_TMA);
-
-err:
- drm_exec_fini(&exec);
-
- /* TMA BO not pinned, release CPU mapping and free. */
- amdgpu_bo_kunmap((*trap_obj)->tma_bo);
- amdgpu_bo_unref(&(*trap_obj)->tma_bo);
- kfree(*trap_obj);
- *trap_obj = NULL;
-}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
deleted file mode 100644
index 5717e1d662fc..000000000000
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Copyright 2025 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#ifndef AMDGPU_CWSR_H
-#define AMDGPU_CWSR_H
-
-#include <linux/types.h>
-
-struct amdgpu_bo;
-struct amdgpu_bo_va;
-struct amdgpu_device;
-struct amdgpu_vm;
-
-/**
- * struct amdgpu_cwsr_obj - CWSR (Compute Wave Save Restore) buffer tracking
- * @bo: Buffer object for CWSR area
- * @bo_va: Buffer object virtual address mapping
- */
-struct amdgpu_cwsr_trap_obj {
- uint64_t tma_gpu_va_addr;
- uint64_t tba_gpu_va_addr;
-
- struct amdgpu_bo *tma_bo;
- struct amdgpu_bo_va *tba_va;
- struct amdgpu_bo_va *tma_va;
- void *tma_cpu_addr;
-};
-
-struct amdgpu_cwsr_info {
- /* cwsr isa */
- struct amdgpu_bo *isa_bo;
- const void *isa_buf;
- uint32_t isa_sz;
- /* cwsr size info per XCC*/
- uint32_t xcc_ctl_stack_sz;
- uint32_t xcc_dbg_mem_sz;
- uint32_t xcc_cwsr_sz;
-};
-
-struct amdgpu_cwsr_params {
- uint64_t ctx_save_area_address;
- /* cwsr size info */
- uint32_t cwsr_sz;
-};
-
-int amdgpu_cwsr_init(struct amdgpu_device *adev);
-void amdgpu_cwsr_fini(struct amdgpu_device *adev);
-
-int amdgpu_cwsr_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
- struct amdgpu_cwsr_trap_obj **cwsr_obj);
-void amdgpu_cwsr_free(struct amdgpu_device *adev, struct amdgpu_vm *vm,
- struct amdgpu_cwsr_trap_obj **cwsr_obj);
-static inline bool amdgpu_cwsr_is_enabled(struct amdgpu_device *adev)
-{
- return adev->cwsr_info != NULL;
-}
-
-uint32_t amdgpu_cwsr_size_needed(struct amdgpu_device *adev, int num_xcc);
-int amdgpu_cwsr_validate_params(struct amdgpu_device *adev,
- struct amdgpu_cwsr_params *cwsr_params,
- int num_xcc);
-static inline bool amdgpu_cwsr_has_dbg_wa(struct amdgpu_device *adev)
-{
- uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
-
- return gc_ver >= IP_VERSION(11, 0, 0) && gc_ver <= IP_VERSION(11, 0, 3);
-}
-#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 5a7f111763bf..6bac724fc73f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -81,7 +81,7 @@
#include "amdgpu_reset.h"
#include "amdgpu_virt.h"
#include "amdgpu_dev_coredump.h"
-#include "amdgpu_cwsr.h"
+#include "amdgpu_trap.h"
#include <linux/suspend.h>
#include <drm/task_barrier.h>
@@ -2551,7 +2551,7 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
r = amdgpu_cper_init(adev);
if (!r) {
- r = amdgpu_cwsr_init(adev);
+ r = amdgpu_trap_init(adev);
if (r == -EOPNOTSUPP)
r = 0;
}
@@ -2953,7 +2953,7 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
{
int i, r;
- amdgpu_cwsr_fini(adev);
+ amdgpu_trap_fini(adev);
amdgpu_cper_fini(adev);
if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index b2a8a8b37006..77675c895585 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -52,6 +52,7 @@
#include "amdgpu_sched.h"
#include "amdgpu_xgmi.h"
#include "amdgpu_userq.h"
+#include "amdgpu_trap.h"
#include "amdgpu_userq_fence.h"
#include "../amdxcp/amdgpu_xcp_drv.h"
#include "amdgpu_ualink.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 147c0bd02731..6839d92b9730 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -50,7 +50,7 @@
#include "amdgpu_video_codecs.h"
#include "amdgpu_ras_mgr.h"
#include "amdgpu_vm.h"
-#include "amdgpu_cwsr.h"
+#include "amdgpu_trap.h"
void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev)
{
@@ -1434,15 +1434,15 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
int num_xcc, r;
fpriv = (struct amdgpu_fpriv *)filp->driver_priv;
- if (!amdgpu_cwsr_is_enabled(adev) || !fpriv->cwsr_trap)
+ if (!amdgpu_trap_is_enabled(adev) || !fpriv->trap_obj)
return -EOPNOTSUPP;
num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr, fpriv->xcp_id);
cwsr_info.ctl_stack_size =
- adev->cwsr_info->xcc_ctl_stack_sz * num_xcc;
+ adev->trap_info->cwsr.xcc_ctl_stack_sz * num_xcc;
cwsr_info.dbg_mem_size =
- adev->cwsr_info->xcc_dbg_mem_sz * num_xcc;
+ adev->trap_info->cwsr.xcc_dbg_mem_sz * num_xcc;
cwsr_info.min_save_area_size =
- amdgpu_cwsr_size_needed(adev, num_xcc);
+ amdgpu_trap_size_needed(adev, num_xcc);
r = copy_to_user(out, &cwsr_info,
min((size_t)size, sizeof(cwsr_info))) ?
-EFAULT :
@@ -1576,10 +1576,10 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
amdgpu_evf_mgr_init(&fpriv->evf_mgr);
- if (amdgpu_cwsr_is_enabled(adev)) {
- r = amdgpu_cwsr_alloc(adev, &fpriv->vm, &fpriv->cwsr_trap);
+ if (amdgpu_trap_is_enabled(adev)) {
+ r = amdgpu_trap_alloc(adev, &fpriv->vm, &fpriv->trap_obj);
if (r)
- dev_dbg(adev->dev, "cwsr trap not enabled");
+ dev_dbg(adev->dev, "trap handler not enabled");
}
amdgpu_ctx_mgr_init(&fpriv->ctx_mgr, adev);
@@ -1651,7 +1651,7 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
}
amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr);
- amdgpu_cwsr_free(adev, &fpriv->vm, &fpriv->cwsr_trap);
+ amdgpu_trap_free(adev, &fpriv->vm, &fpriv->trap_obj);
if (pasid)
amdgpu_pasid_free_delayed(pd->tbo.base.resv, pasid);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
new file mode 100644
index 000000000000..12e37d0467f9
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
@@ -0,0 +1,753 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include <linux/iosys-map.h>
+
+#include <drm/ttm/ttm_bo.h>
+#include <drm/drm_exec.h>
+
+#include "amdgpu.h"
+#include "cwsr_trap_handler.h"
+#include "amdgpu_trap.h"
+
+/*
+ * First-level TMA layout used by the second-level trap-handler ABI.
+ * The trap-handler ISA reads L2 TBA at offset 0 and L2 TMA at offset 8.
+ */
+#define AMDGPU_TRAP_L2_TBA_INDEX 0
+#define AMDGPU_TRAP_L2_TMA_INDEX 1
+#define AMDGPU_TRAP_DEBUG_FLAG_INDEX 2
+
+extern int cwsr_enable;
+
+#define AMDGPU_TRAP_TBA_MAX_SIZE (2 * AMDGPU_GPU_PAGE_SIZE)
+#define AMDGPU_TRAP_TMA_MAX_SIZE (AMDGPU_GPU_PAGE_SIZE)
+#define AMDGPU_TRAP_TMA_OFFSET (AMDGPU_TRAP_TBA_MAX_SIZE)
+
+#define SGPR_SIZE_PER_CU 0x4000
+#define LDS_SIZE_PER_CU 0x10000
+#define HWREG_SIZE_PER_CU 0x1000
+#define DEBUGGER_BYTES_ALIGN 64
+#define DEBUGGER_BYTES_PER_WAVE 32
+#define SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER 40
+
+enum amdgpu_trap_region {
+ AMDGPU_TRAP_TBA,
+ AMDGPU_TRAP_TMA,
+};
+
+static inline uint64_t amdgpu_trap_tba_vaddr(struct amdgpu_device *adev)
+{
+ u64 addr = AMDGPU_VA_RESERVED_TRAP_UQ_START(adev);
+
+ addr = amdgpu_gmc_sign_extend(addr);
+
+ return addr;
+}
+
+static inline bool amdgpu_trap_is_supported(struct amdgpu_device *adev)
+{
+ u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
+
+ if (!cwsr_enable || gc_ver < IP_VERSION(9, 0, 1))
+ return false;
+
+ return true;
+}
+
+uint32_t amdgpu_trap_size_needed(struct amdgpu_device *adev, int num_xcc)
+{
+ if (!amdgpu_trap_is_enabled(adev))
+ return 0;
+
+ return num_xcc *
+ (adev->trap_info->cwsr.xcc_cwsr_sz + adev->trap_info->cwsr.xcc_dbg_mem_sz);
+}
+
+static void amdgpu_trap_cwsr_init_isa_details(struct amdgpu_device *adev,
+ struct amdgpu_trap_info *trap_info)
+{
+ u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
+
+ if (gc_ver < IP_VERSION(9, 0, 1)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) >
+ AMDGPU_TRAP_TBA_MAX_SIZE);
+ trap_info->isa_buf = cwsr_trap_gfx8_hex;
+ trap_info->isa_sz = sizeof(cwsr_trap_gfx8_hex);
+ } else if (gc_ver == IP_VERSION(9, 4, 1)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex) >
+ AMDGPU_TRAP_TBA_MAX_SIZE);
+ trap_info->isa_buf = cwsr_trap_arcturus_hex;
+ trap_info->isa_sz = sizeof(cwsr_trap_arcturus_hex);
+ } else if (gc_ver == IP_VERSION(9, 4, 2)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_aldebaran_hex) >
+ AMDGPU_TRAP_TBA_MAX_SIZE);
+ trap_info->isa_buf = cwsr_trap_aldebaran_hex;
+ trap_info->isa_sz = sizeof(cwsr_trap_aldebaran_hex);
+ } else if (gc_ver == IP_VERSION(9, 4, 3) ||
+ gc_ver == IP_VERSION(9, 4, 4)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_4_3_hex) >
+ AMDGPU_TRAP_TBA_MAX_SIZE);
+ trap_info->isa_buf = cwsr_trap_gfx9_4_3_hex;
+ trap_info->isa_sz = sizeof(cwsr_trap_gfx9_4_3_hex);
+ } else if (gc_ver == IP_VERSION(9, 5, 0)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_5_0_hex) > PAGE_SIZE);
+ trap_info->isa_buf = cwsr_trap_gfx9_5_0_hex;
+ trap_info->isa_sz = sizeof(cwsr_trap_gfx9_5_0_hex);
+ } else if (gc_ver < IP_VERSION(10, 1, 1)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) >
+ AMDGPU_TRAP_TBA_MAX_SIZE);
+ trap_info->isa_buf = cwsr_trap_gfx9_hex;
+ trap_info->isa_sz = sizeof(cwsr_trap_gfx9_hex);
+ } else if (gc_ver < IP_VERSION(10, 3, 0)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_nv1x_hex) >
+ AMDGPU_TRAP_TBA_MAX_SIZE);
+ trap_info->isa_buf = cwsr_trap_nv1x_hex;
+ trap_info->isa_sz = sizeof(cwsr_trap_nv1x_hex);
+ } else if (gc_ver < IP_VERSION(11, 0, 0)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex) >
+ AMDGPU_TRAP_TBA_MAX_SIZE);
+ trap_info->isa_buf = cwsr_trap_gfx10_hex;
+ trap_info->isa_sz = sizeof(cwsr_trap_gfx10_hex);
+ } else if (gc_ver < IP_VERSION(12, 0, 0)) {
+ /* The gfx11 cwsr trap handler must fit inside a single
+ page. */
+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx11_hex) > PAGE_SIZE);
+ trap_info->isa_buf = cwsr_trap_gfx11_hex;
+ trap_info->isa_sz = sizeof(cwsr_trap_gfx11_hex);
+ } else if (gc_ver < IP_VERSION(12, 1, 0)) {
+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_hex) >
+ AMDGPU_TRAP_TBA_MAX_SIZE);
+ trap_info->isa_buf = cwsr_trap_gfx12_hex;
+ trap_info->isa_sz = sizeof(cwsr_trap_gfx12_hex);
+ } else {
+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_1_0_hex) >
+ AMDGPU_TRAP_TBA_MAX_SIZE);
+ trap_info->isa_buf = cwsr_trap_gfx12_1_0_hex;
+ trap_info->isa_sz = sizeof(cwsr_trap_gfx12_1_0_hex);
+ }
+}
+
+static uint32_t amdgpu_trap_cwsr_get_vgpr_size_per_cu(struct amdgpu_device *adev)
+{
+ u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
+ u32 vgpr_size;
+
+ switch (gc_ver) {
+ case IP_VERSION(9, 4, 1): /* GFX_VERSION_ARCTURUS */
+ case IP_VERSION(9, 4, 2): /* GFX_VERSION_ALDEBARAN */
+ case IP_VERSION(9, 4, 3): /* GFX_VERSION_AQUA_VANJARAM */
+ case IP_VERSION(9, 4, 4): /* GFX_VERSION_AQUA_VANJARAM */
+ case IP_VERSION(9, 5, 0):
+ vgpr_size = 0x80000;
+ break;
+ case IP_VERSION(11, 0, 0):
+ case IP_VERSION(11, 0, 2):
+ case IP_VERSION(11, 0, 3):
+ case IP_VERSION(12, 0, 0):
+ case IP_VERSION(12, 0, 1):
+ vgpr_size = 0x60000;
+ break;
+ default:
+ vgpr_size = 0x40000;
+ break;
+ }
+
+ return vgpr_size;
+}
+
+static uint32_t amdgpu_trap_cwsr_get_wg_ctxt_size_per_cu(struct amdgpu_device *adev)
+{
+ u32 lds_sz_per_cu;
+
+ lds_sz_per_cu =
+ (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0)) ?
+ (adev->gfx.cu_info.lds_size << 10) :
+ LDS_SIZE_PER_CU;
+
+ return amdgpu_trap_cwsr_get_vgpr_size_per_cu(adev) + SGPR_SIZE_PER_CU +
+ lds_sz_per_cu + HWREG_SIZE_PER_CU;
+}
+
+static uint32_t amdgpu_trap_cwsr_ctl_stack_bytes_per_wave(struct amdgpu_device *adev)
+{
+ u32 sz;
+
+ if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 1, 0))
+ sz = 12;
+ else
+ sz = 8;
+ return sz;
+}
+
+static void amdgpu_trap_cwsr_init_save_area_info(struct amdgpu_device *adev,
+ struct amdgpu_trap_info *trap_info)
+{
+ struct amdgpu_gfx_config *gfx_info = &adev->gfx.config;
+ u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
+ u32 ctl_stack_size, wg_data_size, dbg_mem_size;
+ u32 array_count;
+ u32 wave_num;
+ u32 cu_num;
+
+ if (gc_ver < IP_VERSION(9, 0, 1))
+ return;
+
+ array_count = gfx_info->max_shader_engines * gfx_info->max_sh_per_se;
+
+ cu_num = adev->gfx.cu_info.number / NUM_XCC(adev->gfx.xcc_mask);
+ wave_num = (gc_ver < IP_VERSION(10, 1, 0)) ? /* GFX_VERSION_NAVI10 */
+ min(cu_num * 40,
+ array_count / gfx_info->max_sh_per_se * 512) :
+ cu_num * 32;
+
+ wg_data_size = ALIGN(cu_num * amdgpu_trap_cwsr_get_wg_ctxt_size_per_cu(adev),
+ PAGE_SIZE);
+ ctl_stack_size =
+ wave_num * amdgpu_trap_cwsr_ctl_stack_bytes_per_wave(adev) + 8;
+ ctl_stack_size =
+ ALIGN(SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER + ctl_stack_size,
+ PAGE_SIZE);
+ dbg_mem_size =
+ ALIGN(wave_num * DEBUGGER_BYTES_PER_WAVE, DEBUGGER_BYTES_ALIGN);
+ /*
+ * HW design limits control stack size to 0x7000.
+ * This is insufficient for theoretical PM4 cases
+ * but sufficient for AQL, limited by SPI events.
+ */
+ if (IP_VERSION_MAJ(gc_ver) == 10)
+ ctl_stack_size = min(ctl_stack_size, 0x7000);
+
+ trap_info->cwsr.xcc_ctl_stack_sz = ctl_stack_size;
+ trap_info->cwsr.xcc_cwsr_sz = ctl_stack_size + wg_data_size;
+ trap_info->cwsr.xcc_dbg_mem_sz = dbg_mem_size;
+}
+
+int amdgpu_trap_init(struct amdgpu_device *adev)
+{
+ struct amdgpu_trap_info *trap_info __free(kfree) =
+ kzalloc(sizeof(*trap_info), GFP_KERNEL);
+ void *ptr;
+ int r;
+
+ if (!amdgpu_trap_is_supported(adev))
+ return -EOPNOTSUPP;
+
+ if (!trap_info)
+ return -ENOMEM;
+ amdgpu_trap_cwsr_init_isa_details(adev, trap_info);
+
+ if (!trap_info->isa_sz)
+ return -EOPNOTSUPP;
+
+ r = amdgpu_bo_create_kernel(adev, AMDGPU_TRAP_TBA_MAX_SIZE, PAGE_SIZE,
+ AMDGPU_GEM_DOMAIN_GTT, &trap_info->isa_bo,
+ NULL, &ptr);
+ if (r)
+ return r;
+
+ memcpy(ptr, trap_info->isa_buf, trap_info->isa_sz);
+
+ amdgpu_trap_cwsr_init_save_area_info(adev, trap_info);
+ adev->trap_info = no_free_ptr(trap_info);
+
+ return 0;
+}
+
+void amdgpu_trap_fini(struct amdgpu_device *adev)
+{
+ if (!amdgpu_trap_is_enabled(adev))
+ return;
+
+ amdgpu_bo_free_kernel(&adev->trap_info->isa_bo, NULL, NULL);
+ kfree(adev->trap_info);
+ adev->trap_info = NULL;
+}
+
+/*
+ * amdgpu_map_cwsr_trap_handler should be called during amdgpu_vm_init
+ * it maps virtual address amdgpu_trap_tba_vaddr() to this VM, and each
+ * compute queue can use this virtual address for wave save/restore
+ * operations to support compute preemption.
+ */
+static int amdgpu_trap_map_region(struct amdgpu_device *adev,
+ struct amdgpu_vm *vm,
+ struct amdgpu_trap_obj *cwsr,
+ enum amdgpu_trap_region region)
+{
+ u64 cwsr_addr, va_flags, va;
+ struct amdgpu_bo_va **bo_va;
+ struct amdgpu_bo *bo;
+ u32 size;
+ int r;
+
+ if (!cwsr || !vm)
+ return -EINVAL;
+
+ cwsr_addr = amdgpu_trap_tba_vaddr(adev);
+
+ if (region == AMDGPU_TRAP_TBA) {
+ size = AMDGPU_TRAP_TBA_MAX_SIZE;
+ bo_va = &cwsr->tba_va;
+ bo = adev->trap_info->isa_bo;
+ va = cwsr_addr;
+ va_flags = (AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
+ AMDGPU_VM_PAGE_EXECUTABLE);
+ } else {
+ size = AMDGPU_TRAP_TMA_MAX_SIZE;
+ bo_va = &cwsr->tma_va;
+ bo = cwsr->tma_bo;
+ va = cwsr_addr + AMDGPU_TRAP_TMA_OFFSET;
+ va_flags = (AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE);
+ }
+
+ *bo_va = amdgpu_vm_bo_add(adev, vm, bo);
+ if (!*bo_va)
+ return -ENOMEM;
+
+ va &= AMDGPU_GMC_HOLE_MASK;
+ r = amdgpu_vm_bo_map(adev, *bo_va, va, 0, size, va_flags);
+ if (r) {
+ dev_err(adev->dev, "failed to do bo map of %s region, err=%d\n",
+ (region == AMDGPU_TRAP_TBA ? "tba" : "tma"), r);
+ amdgpu_vm_bo_del(adev, *bo_va);
+ *bo_va = NULL;
+ return r;
+ }
+
+ r = amdgpu_vm_bo_update(adev, *bo_va, false);
+ if (r) {
+ dev_err(adev->dev,
+ "failed to do page table update of %s region, err=%d\n",
+ (region == AMDGPU_TRAP_TBA ? "tba" : "tma"), r);
+ amdgpu_vm_bo_del(adev, *bo_va);
+ *bo_va = NULL;
+ return r;
+ }
+
+ if (region == AMDGPU_TRAP_TBA)
+ cwsr->tba_gpu_va_addr = va;
+ else
+ cwsr->tma_gpu_va_addr = va;
+
+ return 0;
+}
+
+static int amdgpu_trap_unmap_region(struct amdgpu_device *adev,
+ struct amdgpu_trap_obj *cwsr,
+ enum amdgpu_trap_region region)
+{
+ struct amdgpu_bo_va **bo_va;
+ u64 va;
+ int r;
+
+ if (!cwsr)
+ return -EINVAL;
+
+ if (region == AMDGPU_TRAP_TBA) {
+ bo_va = &cwsr->tba_va;
+ va = cwsr->tba_gpu_va_addr;
+ } else {
+ bo_va = &cwsr->tma_va;
+ va = cwsr->tma_gpu_va_addr;
+ }
+
+ r = amdgpu_vm_bo_unmap(adev, *bo_va, va);
+ if (r) {
+ dev_err(adev->dev,
+ "failed to do bo_unmap on CWSR trap handler, err=%d\n",
+ r);
+ return r;
+ }
+
+ amdgpu_vm_bo_del(adev, *bo_va);
+ *bo_va = NULL;
+
+ return r;
+}
+
+int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+ struct amdgpu_trap_obj **trap_obj)
+{
+ struct amdgpu_bo_param bp;
+ struct iosys_map map;
+ struct amdgpu_trap_obj *cwsr;
+ struct amdgpu_bo *bo;
+ struct drm_exec exec;
+ int r;
+
+ if (!amdgpu_trap_is_enabled(adev))
+ return -EOPNOTSUPP;
+
+ if (!vm || !trap_obj)
+ return -EINVAL;
+
+ cwsr = kzalloc_obj(*cwsr, GFP_KERNEL);
+ if (!cwsr)
+ return -ENOMEM;
+
+ bo = adev->trap_info->isa_bo;
+
+ drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
+ drm_exec_until_all_locked(&exec) {
+ r = amdgpu_vm_lock_pd(vm, &exec, 0);
+ drm_exec_retry_on_contention(&exec);
+ if (unlikely(r))
+ goto err_exec;
+
+ r = drm_exec_lock_obj(&exec, &bo->tbo.base);
+ drm_exec_retry_on_contention(&exec);
+ if (unlikely(r))
+ goto err_exec;
+ }
+
+ /*
+ * Create the TMA BO with the VM root reservation. The reservation is
+ * already held by drm_exec, so creating the BO doesn't recursively
+ * acquire another reservation_ww_class_mutex. The TMA remains
+ * unpinned and is covered by the VM root reservation.
+ */
+ memset(&bp, 0, sizeof(bp));
+ bp.size = AMDGPU_TRAP_TMA_MAX_SIZE;
+ bp.byte_align = PAGE_SIZE;
+ bp.domain = AMDGPU_GEM_DOMAIN_GTT;
+ bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC |
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
+ bp.type = ttm_bo_type_kernel;
+ bp.bo_ptr_size = sizeof(struct amdgpu_bo);
+ bp.resv = vm->root.bo->tbo.base.resv;
+
+ r = amdgpu_bo_create(adev, &bp, &cwsr->tma_bo);
+ if (r)
+ goto err_exec;
+
+ /*
+ * The TMA BO is unpinned and movable. Map it only while accessing
+ * its contents so that no CPU mapping can become stale across BO
+ * migration. ttm_bo_vmap() and ttm_bo_vunmap() require the BO
+ * reservation, which is the already held VM root reservation.
+ */
+ r = ttm_bo_vmap(&cwsr->tma_bo->tbo, &map);
+ if (r)
+ goto err_tma;
+
+ /*
+ * Finish initialization with a DMA write barrier before allowing
+ * the GPU to observe the contents.
+ */
+ iosys_map_memset(&map, 0, 0, AMDGPU_TRAP_TMA_MAX_SIZE);
+ dma_wmb();
+
+ ttm_bo_vunmap(&cwsr->tma_bo->tbo, &map);
+
+ r = amdgpu_trap_map_region(adev, vm, cwsr, AMDGPU_TRAP_TMA);
+ if (r)
+ goto err_tma;
+
+ r = amdgpu_trap_map_region(adev, vm, cwsr, AMDGPU_TRAP_TBA);
+ if (r) {
+ amdgpu_trap_unmap_region(adev, cwsr, AMDGPU_TRAP_TMA);
+ goto err_tma;
+ }
+
+ drm_exec_fini(&exec);
+
+ *trap_obj = cwsr;
+
+ return 0;
+
+err_tma:
+ amdgpu_bo_unref(&cwsr->tma_bo);
+
+err_exec:
+ drm_exec_fini(&exec);
+ kfree(cwsr);
+ *trap_obj = NULL;
+
+ return r;
+}
+
+static int amdgpu_trap_update_l2_handler(struct amdgpu_trap_obj *trap_obj,
+ u64 tba, u64 tma)
+{
+ struct iosys_map map;
+ int r;
+
+ if (!trap_obj || !trap_obj->tma_bo)
+ return -EINVAL;
+
+ dma_resv_assert_held(trap_obj->tma_bo->tbo.base.resv);
+
+ r = ttm_bo_vmap(&trap_obj->tma_bo->tbo, &map);
+ if (r)
+ return r;
+
+ /*
+ * Publish both descriptor fields before allowing the GPU to observe
+ * the updated second-level trap handler.
+ */
+ iosys_map_wr(&map, AMDGPU_TRAP_L2_TBA_INDEX * sizeof(u64),
+ u64, tba);
+ iosys_map_wr(&map, AMDGPU_TRAP_L2_TMA_INDEX * sizeof(u64),
+ u64, tma);
+ dma_wmb();
+
+ ttm_bo_vunmap(&trap_obj->tma_bo->tbo, &map);
+
+ return 0;
+}
+
+static int amdgpu_trap_clear_l2_handler(struct amdgpu_vm *vm,
+ struct amdgpu_trap_obj *trap_obj)
+{
+ if (!trap_obj || !trap_obj->tma_bo)
+ return -EINVAL;
+
+ dma_resv_assert_held(vm->root.bo->tbo.base.resv);
+
+ return amdgpu_trap_update_l2_handler(trap_obj, 0, 0);
+}
+
+int amdgpu_trap_validate_params(struct amdgpu_device *adev,
+ struct amdgpu_trap_params *trap_params,
+ int num_xcc)
+{
+ u32 min_size, max_size;
+
+ if (!amdgpu_trap_is_enabled(adev))
+ return -EOPNOTSUPP;
+
+ if (!trap_params)
+ return -EINVAL;
+
+ min_size = amdgpu_trap_size_needed(adev, num_xcc);
+ max_size = 2 * min_size;
+ /*
+ * Only save area size details checked. Address validation needs to be
+ * carried out separately. User is expected to pass a cwsr buffer area
+ * with size >= ctl stack size + wg data size + debug memory size
+ */
+ if (trap_params->cwsr_sz < min_size ||
+ trap_params->cwsr_sz > max_size) {
+ dev_dbg(adev->dev,
+ "queue cwsr size 0x%x should be within min %u : max %u\n",
+ trap_params->cwsr_sz, min_size, max_size);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+void amdgpu_trap_free(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+ struct amdgpu_trap_obj **trap_obj)
+{
+ struct amdgpu_bo *tba_bo;
+ struct drm_exec exec;
+ int r;
+
+ if (!trap_obj || !*trap_obj)
+ return;
+
+ tba_bo = adev->trap_info->isa_bo;
+
+ drm_exec_init(&exec, 0, 0);
+ drm_exec_until_all_locked(&exec) {
+ r = amdgpu_vm_lock_pd(vm, &exec, 0);
+ drm_exec_retry_on_contention(&exec);
+ if (unlikely(r))
+ goto out_exec;
+
+ if (tba_bo) {
+ r = drm_exec_lock_obj(&exec, &tba_bo->tbo.base);
+ drm_exec_retry_on_contention(&exec);
+ if (unlikely(r))
+ goto out_exec;
+ }
+ }
+
+ if ((*trap_obj)->tba_va) {
+ amdgpu_vm_bo_del(adev, (*trap_obj)->tba_va);
+ (*trap_obj)->tba_va = NULL;
+ }
+
+ if ((*trap_obj)->tma_va) {
+ amdgpu_vm_bo_del(adev, (*trap_obj)->tma_va);
+ (*trap_obj)->tma_va = NULL;
+ }
+
+ drm_exec_fini(&exec);
+ amdgpu_bo_unref(&(*trap_obj)->tma_bo);
+ kfree(*trap_obj);
+ *trap_obj = NULL;
+
+ return;
+
+out_exec:
+ drm_exec_fini(&exec);
+}
+
+static int amdgpu_trap_validate_user_addr(struct amdgpu_device *adev,
+ struct amdgpu_vm *vm,
+ struct amdgpu_trap_usr_addr *usr_addr)
+{
+ struct amdgpu_bo_va_mapping *va_map;
+ u64 start, end, addr, last;
+
+ if (!usr_addr || !usr_addr->size)
+ return -EINVAL;
+
+ dma_resv_assert_held(vm->root.bo->tbo.base.resv);
+
+ start = usr_addr->addr & AMDGPU_GMC_HOLE_MASK;
+
+ if (check_add_overflow(start, (u64)usr_addr->size - 1, &end))
+ return -EINVAL;
+
+ addr = start >> AMDGPU_GPU_PAGE_SHIFT;
+ last = end >> AMDGPU_GPU_PAGE_SHIFT;
+
+ va_map = amdgpu_vm_bo_lookup_mapping(vm, addr);
+ if (!va_map)
+ return -EINVAL;
+
+ if (addr < va_map->start || last > va_map->last)
+ return -EINVAL;
+
+ return 0;
+}
+
+int amdgpu_trap_vm_set_l2_trap(struct amdgpu_device *adev,
+ struct amdgpu_vm *vm,
+ struct amdgpu_trap_obj *trap_obj,
+ u64 tba_va, uint32_t tba_sz,
+ u64 tma_va, uint32_t tma_sz)
+{
+ struct amdgpu_trap_usr_addr tba = {
+ .addr = tba_va,
+ .size = tba_sz,
+ };
+ struct amdgpu_trap_usr_addr tma = {
+ .addr = tma_va,
+ .size = tma_sz,
+ };
+ long timeout;
+ int r;
+
+ if (!amdgpu_trap_is_enabled(adev))
+ return -EOPNOTSUPP;
+
+ if (!vm || !trap_obj || !trap_obj->tma_bo)
+ return -EINVAL;
+
+ r = amdgpu_bo_reserve(vm->root.bo, false);
+ if (r)
+ return r;
+
+ timeout = dma_resv_wait_timeout(vm->root.bo->tbo.base.resv,
+ DMA_RESV_USAGE_BOOKKEEP,
+ true, MAX_SCHEDULE_TIMEOUT);
+ if (timeout <= 0) {
+ r = timeout < 0 ? timeout : -ETIMEDOUT;
+ goto out_unreserve;
+ }
+
+ r = amdgpu_trap_validate_user_addr(adev, vm, &tba);
+ if (r)
+ goto out_unreserve;
+
+ r = amdgpu_trap_validate_user_addr(adev, vm, &tma);
+ if (r)
+ goto out_unreserve;
+
+ r = amdgpu_trap_update_l2_handler(trap_obj, tba_va, tma_va);
+
+out_unreserve:
+ amdgpu_bo_unreserve(vm->root.bo);
+
+ return r;
+}
+
+int amdgpu_trap_vm_clear_l2_trap(struct amdgpu_device *adev,
+ struct amdgpu_vm *vm,
+ struct amdgpu_trap_obj *trap_obj)
+{
+ long timeout;
+ int r;
+
+ if (!amdgpu_trap_is_enabled(adev))
+ return -EOPNOTSUPP;
+
+ if (!vm || !trap_obj || !trap_obj->tma_bo)
+ return -EINVAL;
+
+ r = amdgpu_bo_reserve(vm->root.bo, false);
+ if (r)
+ return r;
+
+ timeout = dma_resv_wait_timeout(vm->root.bo->tbo.base.resv,
+ DMA_RESV_USAGE_BOOKKEEP,
+ true, MAX_SCHEDULE_TIMEOUT);
+ if (timeout <= 0) {
+ r = timeout < 0 ? timeout : -ETIMEDOUT;
+ goto out_unreserve;
+ }
+
+ r = amdgpu_trap_clear_l2_handler(vm, trap_obj);
+
+out_unreserve:
+ amdgpu_bo_unreserve(vm->root.bo);
+
+ return r;
+}
+
+int amdgpu_trap_set_trap_debug_flag(struct amdgpu_device *adev,
+ struct amdgpu_vm *vm,
+ struct amdgpu_trap_obj *trap_obj,
+ bool enabled)
+{
+ struct iosys_map map;
+ int r;
+
+ if (!amdgpu_trap_is_enabled(adev))
+ return -EOPNOTSUPP;
+
+ if (!vm || !trap_obj || !trap_obj->tma_bo)
+ return -EINVAL;
+
+ r = amdgpu_bo_reserve(vm->root.bo, false);
+ if (r)
+ return r;
+
+ r = ttm_bo_vmap(&trap_obj->tma_bo->tbo, &map);
+ if (r)
+ goto out_unreserve;
+
+ iosys_map_wr(&map, AMDGPU_TRAP_DEBUG_FLAG_INDEX * sizeof(u64),
+ u64, (u64)enabled);
+ dma_wmb();
+
+ ttm_bo_vunmap(&trap_obj->tma_bo->tbo, &map);
+
+out_unreserve:
+ amdgpu_bo_unreserve(vm->root.bo);
+
+ return r;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h
new file mode 100644
index 000000000000..c78c60e3d18d
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef AMDGPU_TRAP_H
+#define AMDGPU_TRAP_H
+
+#include <linux/types.h>
+
+struct amdgpu_bo;
+struct amdgpu_bo_va;
+struct amdgpu_device;
+struct amdgpu_vm;
+
+/**
+ * struct amdgpu_trap_obj - per-GPUVM CWSR trap-handler state
+ * @tma_gpu_va_addr: GPU virtual address of the first-level TMA
+ * @tba_gpu_va_addr: GPU virtual address of the first-level TBA
+ * @tma_bo: buffer object backing the first-level TMA
+ * @tba_va: VM mapping of the first-level TBA
+ * @tma_va: VM mapping of the first-level TMA
+ */
+struct amdgpu_trap_obj {
+ u64 tma_gpu_va_addr;
+ u64 tba_gpu_va_addr;
+
+ struct amdgpu_bo *tma_bo;
+ struct amdgpu_bo_va *tba_va;
+ struct amdgpu_bo_va *tma_va;
+};
+
+struct amdgpu_trap_info {
+ /* cwsr isa */
+ struct amdgpu_bo *isa_bo;
+ const void *isa_buf;
+ u32 isa_sz;
+
+ /* cwsr size info per XCC */
+ struct {
+ u32 xcc_ctl_stack_sz;
+ u32 xcc_dbg_mem_sz;
+ u32 xcc_cwsr_sz;
+ } cwsr;
+};
+
+struct amdgpu_trap_params {
+ u64 ctx_save_area_address;
+
+ /* cwsr size info */
+ u32 cwsr_sz;
+};
+
+struct amdgpu_trap_usr_addr {
+ u64 addr;
+ u32 size;
+};
+
+int amdgpu_trap_init(struct amdgpu_device *adev);
+void amdgpu_trap_fini(struct amdgpu_device *adev);
+
+int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+ struct amdgpu_trap_obj **trap_obj);
+
+void amdgpu_trap_free(struct amdgpu_device *adev,
+ struct amdgpu_vm *vm,
+ struct amdgpu_trap_obj **trap_obj);
+
+static inline bool amdgpu_trap_is_enabled(struct amdgpu_device *adev)
+{
+ return adev->trap_info;
+}
+
+uint32_t amdgpu_trap_size_needed(struct amdgpu_device *adev, int num_xcc);
+
+int amdgpu_trap_validate_params(struct amdgpu_device *adev,
+ struct amdgpu_trap_params *trap_params,
+ int num_xcc);
+
+static inline bool amdgpu_trap_has_dbg_wa(struct amdgpu_device *adev)
+{
+ u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
+
+ return gc_ver >= IP_VERSION(11, 0, 0) &&
+ gc_ver <= IP_VERSION(11, 0, 3);
+}
+
+/**
+ * amdgpu_trap_vm_set_l2_trap - install the second-level trap handler
+ * @adev: AMDGPU device
+ * @vm: GPUVM owning the first-level trap state
+ * @trap_obj: first-level trap object
+ * @tba_va: GPU virtual address of the second-level TBA
+ * @tba_sz: size of the second-level TBA
+ * @tma_va: GPU virtual address of the second-level TMA
+ * @tma_sz: size of the second-level TMA
+ *
+ * The second-level trap handler is configured per GPUVM. The descriptor is
+ * stored in the per-GPUVM first-level TMA.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int amdgpu_trap_vm_set_l2_trap(struct amdgpu_device *adev,
+ struct amdgpu_vm *vm,
+ struct amdgpu_trap_obj *trap_obj,
+ u64 tba_va, uint32_t tba_sz,
+ u64 tma_va, uint32_t tma_sz);
+
+/**
+ * amdgpu_trap_vm_clear_l2_trap - clear the second-level trap handler
+ * @adev: AMDGPU device
+ * @vm: GPUVM owning the first-level trap state
+ * @trap_obj: first-level trap object
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int amdgpu_trap_vm_clear_l2_trap(struct amdgpu_device *adev,
+ struct amdgpu_vm *vm,
+ struct amdgpu_trap_obj *trap_obj);
+
+int amdgpu_trap_set_trap_debug_flag(struct amdgpu_device *adev,
+ struct amdgpu_vm *vm,
+ struct amdgpu_trap_obj *trap_obj,
+ bool enabled);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 6eea0ca40556..b4c91e8cd37e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -35,7 +35,7 @@
#include "amdgpu_hmm.h"
#include "amdgpu_userq_fence.h"
#include "amdgpu_trace.h"
-#include "amdgpu_cwsr.h"
+#include "amdgpu_trap.h"
u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev)
{
@@ -267,9 +267,8 @@ int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
return -EINVAL;
}
-int amdgpu_userq_input_cwsr_params_validate(
- struct amdgpu_usermode_queue *queue,
- struct amdgpu_cwsr_params *cwsr_params)
+int amdgpu_userq_input_trap_params_validate(struct amdgpu_usermode_queue *queue,
+ struct amdgpu_trap_params *trap_params)
{
struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(queue->userq_mgr);
struct amdgpu_device *adev = queue->userq_mgr->adev;
@@ -278,16 +277,16 @@ int amdgpu_userq_input_cwsr_params_validate(
int r;
num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr, fpriv->xcp_id);
- r = amdgpu_cwsr_validate_params(queue->userq_mgr->adev, cwsr_params,
+ r = amdgpu_trap_validate_params(queue->userq_mgr->adev, trap_params,
num_xcc);
if (r)
return r;
- cwsr_size = amdgpu_cwsr_size_needed(queue->userq_mgr->adev, num_xcc);
+ cwsr_size = amdgpu_trap_size_needed(queue->userq_mgr->adev, num_xcc);
if (!cwsr_size)
return -EOPNOTSUPP;
return amdgpu_userq_input_va_validate(
- adev, queue, cwsr_params->ctx_save_area_address, cwsr_size,
+ adev, queue, trap_params->ctx_save_area_address, cwsr_size,
&queue->userq_vas.va.cwsr);
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index 727f60c40bbf..5eb0f3ee9b4e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -41,7 +41,7 @@ enum amdgpu_userq_state {
};
struct amdgpu_mqd_prop;
-struct amdgpu_cwsr_params;
+struct amdgpu_trap_params;
struct amdgpu_userq_obj {
void *cpu_ptr;
@@ -202,8 +202,7 @@ int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
void amdgpu_userq_gem_va_unmap_validate(struct amdgpu_device *adev,
struct amdgpu_bo_va_mapping *mapping);
-int amdgpu_userq_input_cwsr_params_validate(
- struct amdgpu_usermode_queue *queue,
- struct amdgpu_cwsr_params *cwsr_params);
+int amdgpu_userq_input_trap_params_validate(struct amdgpu_usermode_queue *queue,
+ struct amdgpu_trap_params *trap_params);
#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
index 5311e1ad0201..07b653909943 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
@@ -27,7 +27,7 @@
#include "mes_userqueue.h"
#include "amdgpu_userq_fence.h"
#include "amdgpu_trace.h"
-#include "amdgpu_cwsr.h"
+#include "amdgpu_trap.h"
#define AMDGPU_USERQ_PROC_CTX_SZ PAGE_SIZE
#define AMDGPU_USERQ_GANG_CTX_SZ PAGE_SIZE
@@ -148,10 +148,10 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo);
queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr;
- if (fpriv->cwsr_trap) {
- queue_input.tba_addr = fpriv->cwsr_trap->tba_gpu_va_addr;
- queue_input.tma_addr = fpriv->cwsr_trap->tma_gpu_va_addr;
- queue_input.trap_en = !amdgpu_cwsr_has_dbg_wa(adev);
+ if (fpriv->trap_obj) {
+ queue_input.tba_addr = fpriv->trap_obj->tba_gpu_va_addr;
+ queue_input.tma_addr = fpriv->trap_obj->tma_gpu_va_addr;
+ queue_input.trap_en = !amdgpu_trap_has_dbg_wa(adev);
}
if (mes->use_rs64mem) {
@@ -415,7 +415,7 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) {
struct drm_amdgpu_userq_mqd_compute_gfx11 *compute_mqd;
- struct amdgpu_cwsr_params cwsr_params;
+ struct amdgpu_trap_params trap_params;
if (mqd_user->mqd_size != sizeof(*compute_mqd)) {
DRM_ERROR("Invalid compute IP MQD size\n");
@@ -453,16 +453,16 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
userq_props->tmz_queue =
mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE;
- if (amdgpu_cwsr_is_enabled(adev)) {
+ if (amdgpu_trap_is_enabled(adev)) {
struct amdgpu_fpriv *fpriv =
uq_mgr_to_fpriv(queue->userq_mgr);
int num_xcc;
num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr,
fpriv->xcp_id);
- cwsr_params.ctx_save_area_address =
+ trap_params.ctx_save_area_address =
compute_mqd->ctx_save_area_va;
- cwsr_params.cwsr_sz =
+ trap_params.cwsr_sz =
compute_mqd->ctx_save_area_size;
r = amdgpu_bo_reserve(queue->vm->root.bo, false);
@@ -471,8 +471,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
goto free_mqd;
}
- r = amdgpu_userq_input_cwsr_params_validate(
- queue, &cwsr_params);
+ r = amdgpu_userq_input_trap_params_validate(
+ queue, &trap_params);
amdgpu_bo_unreserve(queue->vm->root.bo);
if (r) {
kfree(compute_mqd);
@@ -484,7 +484,7 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
userq_props->ctx_save_area_size =
compute_mqd->ctx_save_area_size;
userq_props->ctl_stack_size =
- adev->cwsr_info->xcc_ctl_stack_sz * num_xcc;
+ adev->trap_info->cwsr.xcc_ctl_stack_sz * num_xcc;
}
kfree(compute_mqd);
--
2.34.1
^ permalink raw reply related [flat|nested] 16+ messages in thread* Re: [PATCH 4/7] drm/amdgpu: Add second-level trap handler infrastructure
2026-10-08 7:05 ` [PATCH 4/7] drm/amdgpu: Add second-level trap handler infrastructure Srinivasan Shanmugam
@ 2026-10-12 14:07 ` Christian König
0 siblings, 0 replies; 16+ messages in thread
From: Christian König @ 2026-10-12 14:07 UTC (permalink / raw)
To: Srinivasan Shanmugam, Alex Deucher
Cc: amd-gfx, Lijo Lazar, Felix Kuehling, Lancelot Six, James Zhu,
Samuel Pitoiset, Natalie Vock, Timur Kristóf,
Pierre-Eric Pelloux-Prayer
On 10/8/26 09:05, Srinivasan Shanmugam wrote:
> From: Lijo Lazar <lijo.lazar@amd.com>
>
> Expand the CWSR infrastructure to support second-level userspace trap
> handlers and rename it to reflect the broader trap handler scope.
>
> Rename the CWSR files and common interfaces to amdgpu_trap while keeping
> CWSR-specific naming for the context save/restore functionality.
>
> Add the basic support needed for a userspace second-level trap handler.
> Store the second-level TBA and TMA addresses in the first-level TMA and
> provide helpers to set and clear them for a GPUVM.
>
> The first-level TMA buffer can move in memory, so do not keep it
> permanently mapped on the CPU. Map it only while accessing its contents
> and unmap it afterwards.
>
> Check that the second-level TBA and TMA address ranges are mapped in the
> GPUVM before using them. Wait for GPUVM work to stop before changing the
> addresses and keep the VM locked while checking and updating them.
>
> Add support for updating the debugger trap flag in the first-level TMA.
>
> Do not expose a standalone TRAP ioctl. The second-level trap handler is
> controlled through SET_L2_TRAP and CLEAR_L2_TRAP operations on the VM
> ioctl added later in the series.
>
> Suggested-by: Christian König <christian.koenig@amd.com>
> Suggested-by: Alex Deucher <alexander.deucher@amd.com>
> Cc: Felix Kuehling <felix.kuehling@amd.com>
> Cc: Lancelot Six <lancelot.six@amd.com>
> Cc: James Zhu <james.zhu@amd.com>
> Cc: Lijo Lazar <lijo.lazar@amd.com>
> Cc: Samuel Pitoiset <hakzsam@gmail.com>
> Cc: Natalie Vock <natalie.vock@gmx.de>
> Cc: Timur Kristóf <timur.kristof@gmail.com>
> Cc: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
> Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
> Co-developed-by: Srinivasan Shanmugam <srinivasan.shanmugam@amd.com>
> Signed-off-by: Srinivasan Shanmugam <srinivasan.shanmugam@amd.com>
Reviewed-by: Christian König <christian.koenig@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/Makefile | 2 +-
> drivers/gpu/drm/amd/amdgpu/amdgpu.h | 8 +-
> drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c | 534 ---------------
> drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h | 87 ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 6 +-
> drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 1 +
> drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 18 +-
> drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c | 753 +++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h | 143 ++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 13 +-
> drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 7 +-
> drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 24 +-
> 12 files changed, 935 insertions(+), 661 deletions(-)
> delete mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c
> delete mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
> create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
> create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
> index 5086a482daf1..28bd81f0f720 100644
> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> @@ -72,7 +72,7 @@ amdgpu-y += amdgpu_device.o amdgpu_reg_access.o amdgpu_doorbell_mgr.o amdgpu_kms
> amdgpu_eeprom.o amdgpu_mca.o amdgpu_psp_ta.o amdgpu_lsdma.o amdgpu_lockdep.o \
> amdgpu_ring_mux.o amdgpu_xcp.o amdgpu_seq64.o amdgpu_dev_coredump.o \
> amdgpu_cper.o amdgpu_userq_fence.o amdgpu_eviction_fence.o amdgpu_ip.o \
> - amdgpu_wb.o amdgpu_cwsr.o
> + amdgpu_wb.o amdgpu_trap.o
>
> amdgpu-$(CONFIG_PROC_FS) += amdgpu_fdinfo.o
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index bdd55ac5b06e..e55496f5ecdd 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -332,8 +332,8 @@ struct amdgpu_reset_context;
> struct amdgpu_reset_control;
> struct amdgpu_coredump_info;
> struct amdgpu_video_codecs;
> -struct amdgpu_cwsr_isa;
> -struct amdgpu_cwsr_trap_obj;
> +struct amdgpu_trap_isa;
> +struct amdgpu_trap_obj;
>
> enum amdgpu_cp_irq {
> AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP = 0,
> @@ -422,7 +422,7 @@ struct amdgpu_fpriv {
> struct xarray bo_list_handles;
> struct amdgpu_ctx_mgr ctx_mgr;
> struct amdgpu_userq_mgr userq_mgr;
> - struct amdgpu_cwsr_trap_obj *cwsr_trap;
> + struct amdgpu_trap_obj *trap_obj;
>
> /* Eviction fence infra */
> struct amdgpu_eviction_fence_mgr evf_mgr;
> @@ -978,7 +978,7 @@ struct amdgpu_device {
> */
> struct amdgpu_kfd_dev kfd;
>
> - struct amdgpu_cwsr_info *cwsr_info;
> + struct amdgpu_trap_info *trap_info;
> };
>
> static inline uint32_t amdgpu_ip_version(const struct amdgpu_device *adev,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c
> deleted file mode 100644
> index 6b6c11bb5fd7..000000000000
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c
> +++ /dev/null
> @@ -1,534 +0,0 @@
> -/*
> - * Copyright 2025 Advanced Micro Devices, Inc.
> - *
> - * Permission is hereby granted, free of charge, to any person obtaining a
> - * copy of this software and associated documentation files (the "Software"),
> - * to deal in the Software without restriction, including without limitation
> - * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> - * and/or sell copies of the Software, and to permit persons to whom the
> - * Software is furnished to do so, subject to the following conditions:
> - *
> - * The above copyright notice and this permission notice shall be included in
> - * all copies or substantial portions of the Software.
> - *
> - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
> - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> - * OTHER DEALINGS IN THE SOFTWARE.
> - */
> -
> -#include <drm/drm_exec.h>
> -
> -#include "amdgpu.h"
> -#include "cwsr_trap_handler.h"
> -#include "amdgpu_cwsr.h"
> -
> -extern int cwsr_enable;
> -
> -#define AMDGPU_CWSR_TBA_MAX_SIZE (2 * AMDGPU_GPU_PAGE_SIZE)
> -#define AMDGPU_CWSR_TMA_MAX_SIZE (AMDGPU_GPU_PAGE_SIZE)
> -#define AMDGPU_CWSR_TMA_OFFSET (AMDGPU_CWSR_TBA_MAX_SIZE)
> -
> -#define SGPR_SIZE_PER_CU 0x4000
> -#define LDS_SIZE_PER_CU 0x10000
> -#define HWREG_SIZE_PER_CU 0x1000
> -#define DEBUGGER_BYTES_ALIGN 64
> -#define DEBUGGER_BYTES_PER_WAVE 32
> -#define SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER 40
> -
> -enum amdgpu_cwsr_region {
> - AMDGPU_CWSR_TBA,
> - AMDGPU_CWSR_TMA,
> -};
> -
> -static inline uint64_t amdgpu_cwsr_tba_vaddr(struct amdgpu_device *adev)
> -{
> - uint64_t addr = AMDGPU_VA_RESERVED_TRAP_UQ_START(adev);
> -
> - addr = amdgpu_gmc_sign_extend(addr);
> -
> - return addr;
> -}
> -
> -static inline bool amdgpu_cwsr_is_supported(struct amdgpu_device *adev)
> -{
> - uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
> -
> - if (!cwsr_enable || adev->gfx.disable_uq ||
> - gc_ver < IP_VERSION(9, 0, 1))
> - return false;
> -
> - return true;
> -}
> -
> -uint32_t amdgpu_cwsr_size_needed(struct amdgpu_device *adev, int num_xcc)
> -{
> - if (!amdgpu_cwsr_is_enabled(adev))
> - return 0;
> -
> - return num_xcc *
> - (adev->cwsr_info->xcc_cwsr_sz + adev->cwsr_info->xcc_dbg_mem_sz);
> -}
> -
> -static void amdgpu_cwsr_init_isa_details(struct amdgpu_device *adev,
> - struct amdgpu_cwsr_info *cwsr_info)
> -{
> - uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
> -
> - if (gc_ver < IP_VERSION(9, 0, 1)) {
> - BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) >
> - AMDGPU_CWSR_TBA_MAX_SIZE);
> - cwsr_info->isa_buf = cwsr_trap_gfx8_hex;
> - cwsr_info->isa_sz = sizeof(cwsr_trap_gfx8_hex);
> - } else if (gc_ver == IP_VERSION(9, 4, 1)) {
> - BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex) >
> - AMDGPU_CWSR_TBA_MAX_SIZE);
> - cwsr_info->isa_buf = cwsr_trap_arcturus_hex;
> - cwsr_info->isa_sz = sizeof(cwsr_trap_arcturus_hex);
> - } else if (gc_ver == IP_VERSION(9, 4, 2)) {
> - BUILD_BUG_ON(sizeof(cwsr_trap_aldebaran_hex) >
> - AMDGPU_CWSR_TBA_MAX_SIZE);
> - cwsr_info->isa_buf = cwsr_trap_aldebaran_hex;
> - cwsr_info->isa_sz = sizeof(cwsr_trap_aldebaran_hex);
> - } else if (gc_ver == IP_VERSION(9, 4, 3) ||
> - gc_ver == IP_VERSION(9, 4, 4)) {
> - BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_4_3_hex) >
> - AMDGPU_CWSR_TBA_MAX_SIZE);
> - cwsr_info->isa_buf = cwsr_trap_gfx9_4_3_hex;
> - cwsr_info->isa_sz = sizeof(cwsr_trap_gfx9_4_3_hex);
> - } else if (gc_ver == IP_VERSION(9, 5, 0)) {
> - BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_5_0_hex) > PAGE_SIZE);
> - cwsr_info->isa_buf = cwsr_trap_gfx9_5_0_hex;
> - cwsr_info->isa_sz = sizeof(cwsr_trap_gfx9_5_0_hex);
> - } else if (gc_ver < IP_VERSION(10, 1, 1)) {
> - BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) >
> - AMDGPU_CWSR_TBA_MAX_SIZE);
> - cwsr_info->isa_buf = cwsr_trap_gfx9_hex;
> - cwsr_info->isa_sz = sizeof(cwsr_trap_gfx9_hex);
> - } else if (gc_ver < IP_VERSION(10, 3, 0)) {
> - BUILD_BUG_ON(sizeof(cwsr_trap_nv1x_hex) >
> - AMDGPU_CWSR_TBA_MAX_SIZE);
> - cwsr_info->isa_buf = cwsr_trap_nv1x_hex;
> - cwsr_info->isa_sz = sizeof(cwsr_trap_nv1x_hex);
> - } else if (gc_ver < IP_VERSION(11, 0, 0)) {
> - BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex) >
> - AMDGPU_CWSR_TBA_MAX_SIZE);
> - cwsr_info->isa_buf = cwsr_trap_gfx10_hex;
> - cwsr_info->isa_sz = sizeof(cwsr_trap_gfx10_hex);
> - } else if (gc_ver < IP_VERSION(12, 0, 0)) {
> - /* The gfx11 cwsr trap handler must fit inside a single
> - page. */
> - BUILD_BUG_ON(sizeof(cwsr_trap_gfx11_hex) > PAGE_SIZE);
> - cwsr_info->isa_buf = cwsr_trap_gfx11_hex;
> - cwsr_info->isa_sz = sizeof(cwsr_trap_gfx11_hex);
> - } else if (gc_ver < IP_VERSION(12, 1, 0)) {
> - BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_hex) >
> - AMDGPU_CWSR_TBA_MAX_SIZE);
> - cwsr_info->isa_buf = cwsr_trap_gfx12_hex;
> - cwsr_info->isa_sz = sizeof(cwsr_trap_gfx12_hex);
> - } else {
> - BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_1_0_hex) >
> - AMDGPU_CWSR_TBA_MAX_SIZE);
> - cwsr_info->isa_buf = cwsr_trap_gfx12_1_0_hex;
> - cwsr_info->isa_sz = sizeof(cwsr_trap_gfx12_1_0_hex);
> - }
> -}
> -
> -static uint32_t amdgpu_cwsr_get_vgpr_size_per_cu(struct amdgpu_device *adev)
> -{
> - uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
> - uint32_t vgpr_size;
> -
> - switch (gc_ver) {
> - case IP_VERSION(9, 4, 1): /* GFX_VERSION_ARCTURUS */
> - case IP_VERSION(9, 4, 2): /* GFX_VERSION_ALDEBARAN */
> - case IP_VERSION(9, 4, 3): /* GFX_VERSION_AQUA_VANJARAM */
> - case IP_VERSION(9, 4, 4): /* GFX_VERSION_AQUA_VANJARAM */
> - case IP_VERSION(9, 5, 0):
> - vgpr_size = 0x80000;
> - break;
> - case IP_VERSION(11, 0, 0):
> - case IP_VERSION(11, 0, 2):
> - case IP_VERSION(11, 0, 3):
> - case IP_VERSION(12, 0, 0):
> - case IP_VERSION(12, 0, 1):
> - vgpr_size = 0x60000;
> - break;
> - default:
> - vgpr_size = 0x40000;
> - break;
> - }
> -
> - return vgpr_size;
> -}
> -
> -static uint32_t amdgpu_cwsr_get_wg_ctxt_size_per_cu(struct amdgpu_device *adev)
> -{
> - uint32_t lds_sz_per_cu;
> -
> - lds_sz_per_cu =
> - (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0)) ?
> - (adev->gfx.cu_info.lds_size << 10) :
> - LDS_SIZE_PER_CU;
> -
> - return amdgpu_cwsr_get_vgpr_size_per_cu(adev) + SGPR_SIZE_PER_CU +
> - lds_sz_per_cu + HWREG_SIZE_PER_CU;
> -}
> -
> -static uint32_t amdgpu_cwsr_ctl_stack_bytes_per_wave(struct amdgpu_device *adev)
> -{
> - uint32_t sz;
> -
> - if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 1, 0))
> - sz = 12;
> - else
> - sz = 8;
> - return sz;
> -}
> -
> -static void amdgpu_cwsr_init_save_area_info(struct amdgpu_device *adev,
> - struct amdgpu_cwsr_info *cwsr_info)
> -{
> - struct amdgpu_gfx_config *gfx_info = &adev->gfx.config;
> - uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
> - uint32_t ctl_stack_size, wg_data_size, dbg_mem_size;
> - uint32_t array_count;
> - uint32_t wave_num;
> - uint32_t cu_num;
> -
> - if (gc_ver < IP_VERSION(9, 0, 1))
> - return;
> -
> - array_count = gfx_info->max_shader_engines * gfx_info->max_sh_per_se;
> -
> - cu_num = adev->gfx.cu_info.number / NUM_XCC(adev->gfx.xcc_mask);
> - wave_num = (gc_ver < IP_VERSION(10, 1, 0)) ? /* GFX_VERSION_NAVI10 */
> - min(cu_num * 40,
> - array_count / gfx_info->max_sh_per_se * 512) :
> - cu_num * 32;
> -
> - wg_data_size = ALIGN(cu_num * amdgpu_cwsr_get_wg_ctxt_size_per_cu(adev),
> - PAGE_SIZE);
> - ctl_stack_size =
> - wave_num * amdgpu_cwsr_ctl_stack_bytes_per_wave(adev) + 8;
> - ctl_stack_size =
> - ALIGN(SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER + ctl_stack_size,
> - PAGE_SIZE);
> - dbg_mem_size =
> - ALIGN(wave_num * DEBUGGER_BYTES_PER_WAVE, DEBUGGER_BYTES_ALIGN);
> - /*
> - * HW design limits control stack size to 0x7000.
> - * This is insufficient for theoretical PM4 cases
> - * but sufficient for AQL, limited by SPI events.
> - */
> - if (IP_VERSION_MAJ(gc_ver) == 10)
> - ctl_stack_size = min(ctl_stack_size, 0x7000);
> -
> - cwsr_info->xcc_ctl_stack_sz = ctl_stack_size;
> - cwsr_info->xcc_cwsr_sz = ctl_stack_size + wg_data_size;
> - cwsr_info->xcc_dbg_mem_sz = dbg_mem_size;
> -}
> -
> -int amdgpu_cwsr_init(struct amdgpu_device *adev)
> -{
> - struct amdgpu_cwsr_info *cwsr_info __free(kfree) =
> - kzalloc(sizeof(*cwsr_info), GFP_KERNEL);
> - void *ptr;
> - int r;
> -
> - if (!amdgpu_cwsr_is_supported(adev))
> - return -EOPNOTSUPP;
> -
> - if (!cwsr_info)
> - return -ENOMEM;
> - amdgpu_cwsr_init_isa_details(adev, cwsr_info);
> -
> - if (!cwsr_info->isa_sz)
> - return -EOPNOTSUPP;
> -
> - r = amdgpu_bo_create_kernel(adev, AMDGPU_CWSR_TBA_MAX_SIZE, PAGE_SIZE,
> - AMDGPU_GEM_DOMAIN_GTT, &cwsr_info->isa_bo,
> - NULL, &ptr);
> - if (r)
> - return r;
> -
> - memcpy(ptr, cwsr_info->isa_buf, cwsr_info->isa_sz);
> -
> - amdgpu_cwsr_init_save_area_info(adev, cwsr_info);
> - adev->cwsr_info = no_free_ptr(cwsr_info);
> -
> - return 0;
> -}
> -
> -void amdgpu_cwsr_fini(struct amdgpu_device *adev)
> -{
> - if (!amdgpu_cwsr_is_enabled(adev))
> - return;
> -
> - amdgpu_bo_free_kernel(&adev->cwsr_info->isa_bo, NULL, NULL);
> - kfree(adev->cwsr_info);
> - adev->cwsr_info = NULL;
> -}
> -
> -/*
> - * amdgpu_map_cwsr_trap_handler should be called during amdgpu_vm_init
> - * it maps virtual address amdgpu_cwsr_trap_handler_vaddr() to this VM, and each
> - * compute queue can use this virtual address for wave save/restore
> - * operations to support compute preemption.
> - */
> -static int amdgpu_cwsr_map_region(struct amdgpu_device *adev,
> - struct amdgpu_vm *vm,
> - struct amdgpu_cwsr_trap_obj *cwsr,
> - enum amdgpu_cwsr_region region)
> -{
> - uint64_t cwsr_addr, va_flags, va;
> - struct amdgpu_bo_va **bo_va;
> - struct amdgpu_bo *bo;
> - uint32_t size;
> - int r;
> -
> - if (!cwsr || !vm)
> - return -EINVAL;
> -
> - cwsr_addr = amdgpu_cwsr_tba_vaddr(adev);
> -
> - if (region == AMDGPU_CWSR_TBA) {
> - size = AMDGPU_CWSR_TBA_MAX_SIZE;
> - bo_va = &cwsr->tba_va;
> - bo = adev->cwsr_info->isa_bo;
> - va = cwsr_addr;
> - va_flags = (AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
> - AMDGPU_VM_PAGE_EXECUTABLE);
> - } else {
> - size = AMDGPU_CWSR_TMA_MAX_SIZE;
> - bo_va = &cwsr->tma_va;
> - bo = cwsr->tma_bo;
> - va = cwsr_addr + AMDGPU_CWSR_TMA_OFFSET;
> - va_flags = (AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE);
> - }
> -
> - *bo_va = amdgpu_vm_bo_add(adev, vm, bo);
> - if (!*bo_va)
> - return -ENOMEM;
> -
> - va &= AMDGPU_GMC_HOLE_MASK;
> - r = amdgpu_vm_bo_map(adev, *bo_va, va, 0, size, va_flags);
> - if (r) {
> - dev_err(adev->dev, "failed to do bo map of %s region, err=%d\n",
> - (region == AMDGPU_CWSR_TBA ? "tba" : "tma"), r);
> - amdgpu_vm_bo_del(adev, *bo_va);
> - *bo_va = NULL;
> - return r;
> - }
> -
> - r = amdgpu_vm_bo_update(adev, *bo_va, false);
> - if (r) {
> - dev_err(adev->dev,
> - "failed to do page table update of %s region, err=%d\n",
> - (region == AMDGPU_CWSR_TBA ? "tba" : "tma"), r);
> - amdgpu_vm_bo_del(adev, *bo_va);
> - *bo_va = NULL;
> - return r;
> - }
> -
> - if (region == AMDGPU_CWSR_TBA)
> - cwsr->tba_gpu_va_addr = va;
> - else
> - cwsr->tma_gpu_va_addr = va;
> -
> - return 0;
> -}
> -
> -static int amdgpu_cwsr_unmap_region(struct amdgpu_device *adev,
> - struct amdgpu_cwsr_trap_obj *cwsr,
> - enum amdgpu_cwsr_region region)
> -{
> - struct amdgpu_bo_va **bo_va;
> - uint64_t va;
> - int r;
> -
> - if (!cwsr)
> - return -EINVAL;
> -
> - if (region == AMDGPU_CWSR_TBA) {
> - bo_va = &cwsr->tba_va;
> - va = cwsr->tba_gpu_va_addr;
> - } else {
> - bo_va = &cwsr->tma_va;
> - va = cwsr->tma_gpu_va_addr;
> - }
> -
> - r = amdgpu_vm_bo_unmap(adev, *bo_va, va);
> - if (r) {
> - dev_err(adev->dev,
> - "failed to do bo_unmap on CWSR trap handler, err=%d\n",
> - r);
> - return r;
> - }
> -
> - amdgpu_vm_bo_del(adev, *bo_va);
> - *bo_va = NULL;
> -
> - return r;
> -}
> -
> -int amdgpu_cwsr_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> - struct amdgpu_cwsr_trap_obj **trap_obj)
> -{
> - struct amdgpu_bo_param bp;
> - struct amdgpu_cwsr_trap_obj *cwsr;
> - struct amdgpu_bo *bo;
> - struct drm_exec exec;
> - int r;
> -
> - if (!amdgpu_cwsr_is_enabled(adev))
> - return -EOPNOTSUPP;
> - if (!vm || !trap_obj)
> - return -EINVAL;
> -
> - cwsr = kzalloc(sizeof(*cwsr), GFP_KERNEL);
> - if (!cwsr)
> - return -ENOMEM;
> -
> - bo = adev->cwsr_info->isa_bo;
> - drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
> - drm_exec_until_all_locked(&exec) {
> - r = amdgpu_vm_lock_pd(vm, &exec, 0);
> - if (likely(!r))
> - r = drm_exec_lock_obj(&exec, &bo->tbo.base);
> - drm_exec_retry_on_contention(&exec);
> - if (unlikely(r)) {
> - dev_err(adev->dev,
> - "failed to reserve for CWSR allocs: err=%d\n",
> - r);
> - goto err_exec;
> - }
> - }
> -
> - /*
> - * Create TMA BO using the VM's already-locked resv so no new
> - * reservation_ww_class_mutex acquisition is needed, avoiding the
> - * recursive ww_mutex deadlock. Unpinned; eviction is handled via
> - * amdgpu_vm_bo_add() inside amdgpu_cwsr_map_region().
> - */
> - memset(&bp, 0, sizeof(bp));
> - bp.size = AMDGPU_CWSR_TMA_MAX_SIZE;
> - bp.byte_align = PAGE_SIZE;
> - bp.domain = AMDGPU_GEM_DOMAIN_GTT;
> - bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC |
> - AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
> - bp.type = ttm_bo_type_kernel;
> - bp.bo_ptr_size = sizeof(struct amdgpu_bo);
> - bp.resv = vm->root.bo->tbo.base.resv;
> -
> - r = amdgpu_bo_create(adev, &bp, &cwsr->tma_bo);
> - if (r)
> - goto err_exec;
> -
> - /* VM resv already held, kmap directly, no separate reserve needed. */
> - r = amdgpu_bo_kmap(cwsr->tma_bo, &cwsr->tma_cpu_addr);
> - if (r)
> - goto err_tma;
> -
> - memset(cwsr->tma_cpu_addr, 0, AMDGPU_CWSR_TMA_MAX_SIZE);
> -
> - r = amdgpu_cwsr_map_region(adev, vm, cwsr, AMDGPU_CWSR_TMA);
> - if (r)
> - goto err_kmap;
> -
> - r = amdgpu_cwsr_map_region(adev, vm, cwsr, AMDGPU_CWSR_TBA);
> - if (r) {
> - amdgpu_cwsr_unmap_region(adev, cwsr, AMDGPU_CWSR_TMA);
> - goto err_kmap;
> - }
> -
> - drm_exec_fini(&exec);
> - *trap_obj = cwsr;
> - return 0;
> -
> -err_kmap:
> - amdgpu_bo_kunmap(cwsr->tma_bo);
> -err_tma:
> - amdgpu_bo_unref(&cwsr->tma_bo);
> -err_exec:
> - drm_exec_fini(&exec);
> - kfree(cwsr);
> - *trap_obj = NULL;
> - return r;
> -}
> -
> -int amdgpu_cwsr_validate_params(struct amdgpu_device *adev,
> - struct amdgpu_cwsr_params *cwsr_params,
> - int num_xcc)
> -{
> - uint32_t min_size, max_size;
> -
> - if (!amdgpu_cwsr_is_enabled(adev))
> - return -EOPNOTSUPP;
> -
> - if (!cwsr_params)
> - return -EINVAL;
> -
> - min_size = amdgpu_cwsr_size_needed(adev, num_xcc);
> - max_size = 2 * min_size;
> - /*
> - * Only save area size details checked. Address validation needs to be
> - * carried out separately. User is expected to pass a cwsr buffer area
> - * with size >= ctl stack size + wg data size + debug memory size
> - */
> - if (cwsr_params->cwsr_sz < min_size ||
> - cwsr_params->cwsr_sz > max_size) {
> - dev_dbg(adev->dev,
> - "queue cwsr size 0x%x should be within min %u : max %u\n",
> - cwsr_params->cwsr_sz, min_size, max_size);
> - return -EINVAL;
> - }
> -
> - return 0;
> -}
> -
> -void amdgpu_cwsr_free(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> - struct amdgpu_cwsr_trap_obj **trap_obj)
> -{
> - struct amdgpu_bo *tba_bo;
> - struct amdgpu_bo *tma_bo;
> - struct drm_exec exec;
> - int r;
> -
> - if (!trap_obj || !*trap_obj || !(*trap_obj)->tma_bo)
> - return;
> -
> - tba_bo = adev->cwsr_info->isa_bo;
> - tma_bo = (*trap_obj)->tma_bo;
> -
> - if (!tba_bo || !tma_bo)
> - return;
> -
> - drm_exec_init(&exec, 0, 0);
> - drm_exec_until_all_locked(&exec) {
> - r = amdgpu_vm_lock_pd(vm, &exec, 0);
> - if (likely(!r))
> - r = drm_exec_lock_obj(&exec, &tba_bo->tbo.base);
> - drm_exec_retry_on_contention(&exec);
> - if (unlikely(r)) {
> - dev_err(adev->dev,
> - "failed to reserve CWSR BOs: err=%d\n", r);
> - goto err;
> - }
> - }
> -
> - amdgpu_cwsr_unmap_region(adev, *trap_obj, AMDGPU_CWSR_TBA);
> - amdgpu_cwsr_unmap_region(adev, *trap_obj, AMDGPU_CWSR_TMA);
> -
> -err:
> - drm_exec_fini(&exec);
> -
> - /* TMA BO not pinned, release CPU mapping and free. */
> - amdgpu_bo_kunmap((*trap_obj)->tma_bo);
> - amdgpu_bo_unref(&(*trap_obj)->tma_bo);
> - kfree(*trap_obj);
> - *trap_obj = NULL;
> -}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
> deleted file mode 100644
> index 5717e1d662fc..000000000000
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h
> +++ /dev/null
> @@ -1,87 +0,0 @@
> -/*
> - * Copyright 2025 Advanced Micro Devices, Inc.
> - *
> - * Permission is hereby granted, free of charge, to any person obtaining a
> - * copy of this software and associated documentation files (the "Software"),
> - * to deal in the Software without restriction, including without limitation
> - * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> - * and/or sell copies of the Software, and to permit persons to whom the
> - * Software is furnished to do so, subject to the following conditions:
> - *
> - * The above copyright notice and this permission notice shall be included in
> - * all copies or substantial portions of the Software.
> - *
> - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
> - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> - * OTHER DEALINGS IN THE SOFTWARE.
> - */
> -
> -#ifndef AMDGPU_CWSR_H
> -#define AMDGPU_CWSR_H
> -
> -#include <linux/types.h>
> -
> -struct amdgpu_bo;
> -struct amdgpu_bo_va;
> -struct amdgpu_device;
> -struct amdgpu_vm;
> -
> -/**
> - * struct amdgpu_cwsr_obj - CWSR (Compute Wave Save Restore) buffer tracking
> - * @bo: Buffer object for CWSR area
> - * @bo_va: Buffer object virtual address mapping
> - */
> -struct amdgpu_cwsr_trap_obj {
> - uint64_t tma_gpu_va_addr;
> - uint64_t tba_gpu_va_addr;
> -
> - struct amdgpu_bo *tma_bo;
> - struct amdgpu_bo_va *tba_va;
> - struct amdgpu_bo_va *tma_va;
> - void *tma_cpu_addr;
> -};
> -
> -struct amdgpu_cwsr_info {
> - /* cwsr isa */
> - struct amdgpu_bo *isa_bo;
> - const void *isa_buf;
> - uint32_t isa_sz;
> - /* cwsr size info per XCC*/
> - uint32_t xcc_ctl_stack_sz;
> - uint32_t xcc_dbg_mem_sz;
> - uint32_t xcc_cwsr_sz;
> -};
> -
> -struct amdgpu_cwsr_params {
> - uint64_t ctx_save_area_address;
> - /* cwsr size info */
> - uint32_t cwsr_sz;
> -};
> -
> -int amdgpu_cwsr_init(struct amdgpu_device *adev);
> -void amdgpu_cwsr_fini(struct amdgpu_device *adev);
> -
> -int amdgpu_cwsr_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> - struct amdgpu_cwsr_trap_obj **cwsr_obj);
> -void amdgpu_cwsr_free(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> - struct amdgpu_cwsr_trap_obj **cwsr_obj);
> -static inline bool amdgpu_cwsr_is_enabled(struct amdgpu_device *adev)
> -{
> - return adev->cwsr_info != NULL;
> -}
> -
> -uint32_t amdgpu_cwsr_size_needed(struct amdgpu_device *adev, int num_xcc);
> -int amdgpu_cwsr_validate_params(struct amdgpu_device *adev,
> - struct amdgpu_cwsr_params *cwsr_params,
> - int num_xcc);
> -static inline bool amdgpu_cwsr_has_dbg_wa(struct amdgpu_device *adev)
> -{
> - uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
> -
> - return gc_ver >= IP_VERSION(11, 0, 0) && gc_ver <= IP_VERSION(11, 0, 3);
> -}
> -#endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index 5a7f111763bf..6bac724fc73f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -81,7 +81,7 @@
> #include "amdgpu_reset.h"
> #include "amdgpu_virt.h"
> #include "amdgpu_dev_coredump.h"
> -#include "amdgpu_cwsr.h"
> +#include "amdgpu_trap.h"
>
> #include <linux/suspend.h>
> #include <drm/task_barrier.h>
> @@ -2551,7 +2551,7 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
> r = amdgpu_cper_init(adev);
>
> if (!r) {
> - r = amdgpu_cwsr_init(adev);
> + r = amdgpu_trap_init(adev);
> if (r == -EOPNOTSUPP)
> r = 0;
> }
> @@ -2953,7 +2953,7 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
> {
> int i, r;
>
> - amdgpu_cwsr_fini(adev);
> + amdgpu_trap_fini(adev);
> amdgpu_cper_fini(adev);
>
> if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index b2a8a8b37006..77675c895585 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -52,6 +52,7 @@
> #include "amdgpu_sched.h"
> #include "amdgpu_xgmi.h"
> #include "amdgpu_userq.h"
> +#include "amdgpu_trap.h"
> #include "amdgpu_userq_fence.h"
> #include "../amdxcp/amdgpu_xcp_drv.h"
> #include "amdgpu_ualink.h"
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index 147c0bd02731..6839d92b9730 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -50,7 +50,7 @@
> #include "amdgpu_video_codecs.h"
> #include "amdgpu_ras_mgr.h"
> #include "amdgpu_vm.h"
> -#include "amdgpu_cwsr.h"
> +#include "amdgpu_trap.h"
>
> void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev)
> {
> @@ -1434,15 +1434,15 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
> int num_xcc, r;
>
> fpriv = (struct amdgpu_fpriv *)filp->driver_priv;
> - if (!amdgpu_cwsr_is_enabled(adev) || !fpriv->cwsr_trap)
> + if (!amdgpu_trap_is_enabled(adev) || !fpriv->trap_obj)
> return -EOPNOTSUPP;
> num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr, fpriv->xcp_id);
> cwsr_info.ctl_stack_size =
> - adev->cwsr_info->xcc_ctl_stack_sz * num_xcc;
> + adev->trap_info->cwsr.xcc_ctl_stack_sz * num_xcc;
> cwsr_info.dbg_mem_size =
> - adev->cwsr_info->xcc_dbg_mem_sz * num_xcc;
> + adev->trap_info->cwsr.xcc_dbg_mem_sz * num_xcc;
> cwsr_info.min_save_area_size =
> - amdgpu_cwsr_size_needed(adev, num_xcc);
> + amdgpu_trap_size_needed(adev, num_xcc);
> r = copy_to_user(out, &cwsr_info,
> min((size_t)size, sizeof(cwsr_info))) ?
> -EFAULT :
> @@ -1576,10 +1576,10 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
>
> amdgpu_evf_mgr_init(&fpriv->evf_mgr);
>
> - if (amdgpu_cwsr_is_enabled(adev)) {
> - r = amdgpu_cwsr_alloc(adev, &fpriv->vm, &fpriv->cwsr_trap);
> + if (amdgpu_trap_is_enabled(adev)) {
> + r = amdgpu_trap_alloc(adev, &fpriv->vm, &fpriv->trap_obj);
> if (r)
> - dev_dbg(adev->dev, "cwsr trap not enabled");
> + dev_dbg(adev->dev, "trap handler not enabled");
> }
>
> amdgpu_ctx_mgr_init(&fpriv->ctx_mgr, adev);
> @@ -1651,7 +1651,7 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
> }
>
> amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr);
> - amdgpu_cwsr_free(adev, &fpriv->vm, &fpriv->cwsr_trap);
> + amdgpu_trap_free(adev, &fpriv->vm, &fpriv->trap_obj);
>
> if (pasid)
> amdgpu_pasid_free_delayed(pd->tbo.base.resv, pasid);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
> new file mode 100644
> index 000000000000..12e37d0467f9
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
> @@ -0,0 +1,753 @@
> +/*
> + * Copyright 2025 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + */
> +#include <linux/iosys-map.h>
> +
> +#include <drm/ttm/ttm_bo.h>
> +#include <drm/drm_exec.h>
> +
> +#include "amdgpu.h"
> +#include "cwsr_trap_handler.h"
> +#include "amdgpu_trap.h"
> +
> +/*
> + * First-level TMA layout used by the second-level trap-handler ABI.
> + * The trap-handler ISA reads L2 TBA at offset 0 and L2 TMA at offset 8.
> + */
> +#define AMDGPU_TRAP_L2_TBA_INDEX 0
> +#define AMDGPU_TRAP_L2_TMA_INDEX 1
> +#define AMDGPU_TRAP_DEBUG_FLAG_INDEX 2
> +
> +extern int cwsr_enable;
> +
> +#define AMDGPU_TRAP_TBA_MAX_SIZE (2 * AMDGPU_GPU_PAGE_SIZE)
> +#define AMDGPU_TRAP_TMA_MAX_SIZE (AMDGPU_GPU_PAGE_SIZE)
> +#define AMDGPU_TRAP_TMA_OFFSET (AMDGPU_TRAP_TBA_MAX_SIZE)
> +
> +#define SGPR_SIZE_PER_CU 0x4000
> +#define LDS_SIZE_PER_CU 0x10000
> +#define HWREG_SIZE_PER_CU 0x1000
> +#define DEBUGGER_BYTES_ALIGN 64
> +#define DEBUGGER_BYTES_PER_WAVE 32
> +#define SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER 40
> +
> +enum amdgpu_trap_region {
> + AMDGPU_TRAP_TBA,
> + AMDGPU_TRAP_TMA,
> +};
> +
> +static inline uint64_t amdgpu_trap_tba_vaddr(struct amdgpu_device *adev)
> +{
> + u64 addr = AMDGPU_VA_RESERVED_TRAP_UQ_START(adev);
> +
> + addr = amdgpu_gmc_sign_extend(addr);
> +
> + return addr;
> +}
> +
> +static inline bool amdgpu_trap_is_supported(struct amdgpu_device *adev)
> +{
> + u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
> +
> + if (!cwsr_enable || gc_ver < IP_VERSION(9, 0, 1))
> + return false;
> +
> + return true;
> +}
> +
> +uint32_t amdgpu_trap_size_needed(struct amdgpu_device *adev, int num_xcc)
> +{
> + if (!amdgpu_trap_is_enabled(adev))
> + return 0;
> +
> + return num_xcc *
> + (adev->trap_info->cwsr.xcc_cwsr_sz + adev->trap_info->cwsr.xcc_dbg_mem_sz);
> +}
> +
> +static void amdgpu_trap_cwsr_init_isa_details(struct amdgpu_device *adev,
> + struct amdgpu_trap_info *trap_info)
> +{
> + u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
> +
> + if (gc_ver < IP_VERSION(9, 0, 1)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) >
> + AMDGPU_TRAP_TBA_MAX_SIZE);
> + trap_info->isa_buf = cwsr_trap_gfx8_hex;
> + trap_info->isa_sz = sizeof(cwsr_trap_gfx8_hex);
> + } else if (gc_ver == IP_VERSION(9, 4, 1)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex) >
> + AMDGPU_TRAP_TBA_MAX_SIZE);
> + trap_info->isa_buf = cwsr_trap_arcturus_hex;
> + trap_info->isa_sz = sizeof(cwsr_trap_arcturus_hex);
> + } else if (gc_ver == IP_VERSION(9, 4, 2)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_aldebaran_hex) >
> + AMDGPU_TRAP_TBA_MAX_SIZE);
> + trap_info->isa_buf = cwsr_trap_aldebaran_hex;
> + trap_info->isa_sz = sizeof(cwsr_trap_aldebaran_hex);
> + } else if (gc_ver == IP_VERSION(9, 4, 3) ||
> + gc_ver == IP_VERSION(9, 4, 4)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_4_3_hex) >
> + AMDGPU_TRAP_TBA_MAX_SIZE);
> + trap_info->isa_buf = cwsr_trap_gfx9_4_3_hex;
> + trap_info->isa_sz = sizeof(cwsr_trap_gfx9_4_3_hex);
> + } else if (gc_ver == IP_VERSION(9, 5, 0)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_5_0_hex) > PAGE_SIZE);
> + trap_info->isa_buf = cwsr_trap_gfx9_5_0_hex;
> + trap_info->isa_sz = sizeof(cwsr_trap_gfx9_5_0_hex);
> + } else if (gc_ver < IP_VERSION(10, 1, 1)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) >
> + AMDGPU_TRAP_TBA_MAX_SIZE);
> + trap_info->isa_buf = cwsr_trap_gfx9_hex;
> + trap_info->isa_sz = sizeof(cwsr_trap_gfx9_hex);
> + } else if (gc_ver < IP_VERSION(10, 3, 0)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_nv1x_hex) >
> + AMDGPU_TRAP_TBA_MAX_SIZE);
> + trap_info->isa_buf = cwsr_trap_nv1x_hex;
> + trap_info->isa_sz = sizeof(cwsr_trap_nv1x_hex);
> + } else if (gc_ver < IP_VERSION(11, 0, 0)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex) >
> + AMDGPU_TRAP_TBA_MAX_SIZE);
> + trap_info->isa_buf = cwsr_trap_gfx10_hex;
> + trap_info->isa_sz = sizeof(cwsr_trap_gfx10_hex);
> + } else if (gc_ver < IP_VERSION(12, 0, 0)) {
> + /* The gfx11 cwsr trap handler must fit inside a single
> + page. */
> + BUILD_BUG_ON(sizeof(cwsr_trap_gfx11_hex) > PAGE_SIZE);
> + trap_info->isa_buf = cwsr_trap_gfx11_hex;
> + trap_info->isa_sz = sizeof(cwsr_trap_gfx11_hex);
> + } else if (gc_ver < IP_VERSION(12, 1, 0)) {
> + BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_hex) >
> + AMDGPU_TRAP_TBA_MAX_SIZE);
> + trap_info->isa_buf = cwsr_trap_gfx12_hex;
> + trap_info->isa_sz = sizeof(cwsr_trap_gfx12_hex);
> + } else {
> + BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_1_0_hex) >
> + AMDGPU_TRAP_TBA_MAX_SIZE);
> + trap_info->isa_buf = cwsr_trap_gfx12_1_0_hex;
> + trap_info->isa_sz = sizeof(cwsr_trap_gfx12_1_0_hex);
> + }
> +}
> +
> +static uint32_t amdgpu_trap_cwsr_get_vgpr_size_per_cu(struct amdgpu_device *adev)
> +{
> + u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
> + u32 vgpr_size;
> +
> + switch (gc_ver) {
> + case IP_VERSION(9, 4, 1): /* GFX_VERSION_ARCTURUS */
> + case IP_VERSION(9, 4, 2): /* GFX_VERSION_ALDEBARAN */
> + case IP_VERSION(9, 4, 3): /* GFX_VERSION_AQUA_VANJARAM */
> + case IP_VERSION(9, 4, 4): /* GFX_VERSION_AQUA_VANJARAM */
> + case IP_VERSION(9, 5, 0):
> + vgpr_size = 0x80000;
> + break;
> + case IP_VERSION(11, 0, 0):
> + case IP_VERSION(11, 0, 2):
> + case IP_VERSION(11, 0, 3):
> + case IP_VERSION(12, 0, 0):
> + case IP_VERSION(12, 0, 1):
> + vgpr_size = 0x60000;
> + break;
> + default:
> + vgpr_size = 0x40000;
> + break;
> + }
> +
> + return vgpr_size;
> +}
> +
> +static uint32_t amdgpu_trap_cwsr_get_wg_ctxt_size_per_cu(struct amdgpu_device *adev)
> +{
> + u32 lds_sz_per_cu;
> +
> + lds_sz_per_cu =
> + (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0)) ?
> + (adev->gfx.cu_info.lds_size << 10) :
> + LDS_SIZE_PER_CU;
> +
> + return amdgpu_trap_cwsr_get_vgpr_size_per_cu(adev) + SGPR_SIZE_PER_CU +
> + lds_sz_per_cu + HWREG_SIZE_PER_CU;
> +}
> +
> +static uint32_t amdgpu_trap_cwsr_ctl_stack_bytes_per_wave(struct amdgpu_device *adev)
> +{
> + u32 sz;
> +
> + if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 1, 0))
> + sz = 12;
> + else
> + sz = 8;
> + return sz;
> +}
> +
> +static void amdgpu_trap_cwsr_init_save_area_info(struct amdgpu_device *adev,
> + struct amdgpu_trap_info *trap_info)
> +{
> + struct amdgpu_gfx_config *gfx_info = &adev->gfx.config;
> + u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
> + u32 ctl_stack_size, wg_data_size, dbg_mem_size;
> + u32 array_count;
> + u32 wave_num;
> + u32 cu_num;
> +
> + if (gc_ver < IP_VERSION(9, 0, 1))
> + return;
> +
> + array_count = gfx_info->max_shader_engines * gfx_info->max_sh_per_se;
> +
> + cu_num = adev->gfx.cu_info.number / NUM_XCC(adev->gfx.xcc_mask);
> + wave_num = (gc_ver < IP_VERSION(10, 1, 0)) ? /* GFX_VERSION_NAVI10 */
> + min(cu_num * 40,
> + array_count / gfx_info->max_sh_per_se * 512) :
> + cu_num * 32;
> +
> + wg_data_size = ALIGN(cu_num * amdgpu_trap_cwsr_get_wg_ctxt_size_per_cu(adev),
> + PAGE_SIZE);
> + ctl_stack_size =
> + wave_num * amdgpu_trap_cwsr_ctl_stack_bytes_per_wave(adev) + 8;
> + ctl_stack_size =
> + ALIGN(SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER + ctl_stack_size,
> + PAGE_SIZE);
> + dbg_mem_size =
> + ALIGN(wave_num * DEBUGGER_BYTES_PER_WAVE, DEBUGGER_BYTES_ALIGN);
> + /*
> + * HW design limits control stack size to 0x7000.
> + * This is insufficient for theoretical PM4 cases
> + * but sufficient for AQL, limited by SPI events.
> + */
> + if (IP_VERSION_MAJ(gc_ver) == 10)
> + ctl_stack_size = min(ctl_stack_size, 0x7000);
> +
> + trap_info->cwsr.xcc_ctl_stack_sz = ctl_stack_size;
> + trap_info->cwsr.xcc_cwsr_sz = ctl_stack_size + wg_data_size;
> + trap_info->cwsr.xcc_dbg_mem_sz = dbg_mem_size;
> +}
> +
> +int amdgpu_trap_init(struct amdgpu_device *adev)
> +{
> + struct amdgpu_trap_info *trap_info __free(kfree) =
> + kzalloc(sizeof(*trap_info), GFP_KERNEL);
> + void *ptr;
> + int r;
> +
> + if (!amdgpu_trap_is_supported(adev))
> + return -EOPNOTSUPP;
> +
> + if (!trap_info)
> + return -ENOMEM;
> + amdgpu_trap_cwsr_init_isa_details(adev, trap_info);
> +
> + if (!trap_info->isa_sz)
> + return -EOPNOTSUPP;
> +
> + r = amdgpu_bo_create_kernel(adev, AMDGPU_TRAP_TBA_MAX_SIZE, PAGE_SIZE,
> + AMDGPU_GEM_DOMAIN_GTT, &trap_info->isa_bo,
> + NULL, &ptr);
> + if (r)
> + return r;
> +
> + memcpy(ptr, trap_info->isa_buf, trap_info->isa_sz);
> +
> + amdgpu_trap_cwsr_init_save_area_info(adev, trap_info);
> + adev->trap_info = no_free_ptr(trap_info);
> +
> + return 0;
> +}
> +
> +void amdgpu_trap_fini(struct amdgpu_device *adev)
> +{
> + if (!amdgpu_trap_is_enabled(adev))
> + return;
> +
> + amdgpu_bo_free_kernel(&adev->trap_info->isa_bo, NULL, NULL);
> + kfree(adev->trap_info);
> + adev->trap_info = NULL;
> +}
> +
> +/*
> + * amdgpu_map_cwsr_trap_handler should be called during amdgpu_vm_init
> + * it maps virtual address amdgpu_trap_tba_vaddr() to this VM, and each
> + * compute queue can use this virtual address for wave save/restore
> + * operations to support compute preemption.
> + */
> +static int amdgpu_trap_map_region(struct amdgpu_device *adev,
> + struct amdgpu_vm *vm,
> + struct amdgpu_trap_obj *cwsr,
> + enum amdgpu_trap_region region)
> +{
> + u64 cwsr_addr, va_flags, va;
> + struct amdgpu_bo_va **bo_va;
> + struct amdgpu_bo *bo;
> + u32 size;
> + int r;
> +
> + if (!cwsr || !vm)
> + return -EINVAL;
> +
> + cwsr_addr = amdgpu_trap_tba_vaddr(adev);
> +
> + if (region == AMDGPU_TRAP_TBA) {
> + size = AMDGPU_TRAP_TBA_MAX_SIZE;
> + bo_va = &cwsr->tba_va;
> + bo = adev->trap_info->isa_bo;
> + va = cwsr_addr;
> + va_flags = (AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
> + AMDGPU_VM_PAGE_EXECUTABLE);
> + } else {
> + size = AMDGPU_TRAP_TMA_MAX_SIZE;
> + bo_va = &cwsr->tma_va;
> + bo = cwsr->tma_bo;
> + va = cwsr_addr + AMDGPU_TRAP_TMA_OFFSET;
> + va_flags = (AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE);
> + }
> +
> + *bo_va = amdgpu_vm_bo_add(adev, vm, bo);
> + if (!*bo_va)
> + return -ENOMEM;
> +
> + va &= AMDGPU_GMC_HOLE_MASK;
> + r = amdgpu_vm_bo_map(adev, *bo_va, va, 0, size, va_flags);
> + if (r) {
> + dev_err(adev->dev, "failed to do bo map of %s region, err=%d\n",
> + (region == AMDGPU_TRAP_TBA ? "tba" : "tma"), r);
> + amdgpu_vm_bo_del(adev, *bo_va);
> + *bo_va = NULL;
> + return r;
> + }
> +
> + r = amdgpu_vm_bo_update(adev, *bo_va, false);
> + if (r) {
> + dev_err(adev->dev,
> + "failed to do page table update of %s region, err=%d\n",
> + (region == AMDGPU_TRAP_TBA ? "tba" : "tma"), r);
> + amdgpu_vm_bo_del(adev, *bo_va);
> + *bo_va = NULL;
> + return r;
> + }
> +
> + if (region == AMDGPU_TRAP_TBA)
> + cwsr->tba_gpu_va_addr = va;
> + else
> + cwsr->tma_gpu_va_addr = va;
> +
> + return 0;
> +}
> +
> +static int amdgpu_trap_unmap_region(struct amdgpu_device *adev,
> + struct amdgpu_trap_obj *cwsr,
> + enum amdgpu_trap_region region)
> +{
> + struct amdgpu_bo_va **bo_va;
> + u64 va;
> + int r;
> +
> + if (!cwsr)
> + return -EINVAL;
> +
> + if (region == AMDGPU_TRAP_TBA) {
> + bo_va = &cwsr->tba_va;
> + va = cwsr->tba_gpu_va_addr;
> + } else {
> + bo_va = &cwsr->tma_va;
> + va = cwsr->tma_gpu_va_addr;
> + }
> +
> + r = amdgpu_vm_bo_unmap(adev, *bo_va, va);
> + if (r) {
> + dev_err(adev->dev,
> + "failed to do bo_unmap on CWSR trap handler, err=%d\n",
> + r);
> + return r;
> + }
> +
> + amdgpu_vm_bo_del(adev, *bo_va);
> + *bo_va = NULL;
> +
> + return r;
> +}
> +
> +int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> + struct amdgpu_trap_obj **trap_obj)
> +{
> + struct amdgpu_bo_param bp;
> + struct iosys_map map;
> + struct amdgpu_trap_obj *cwsr;
> + struct amdgpu_bo *bo;
> + struct drm_exec exec;
> + int r;
> +
> + if (!amdgpu_trap_is_enabled(adev))
> + return -EOPNOTSUPP;
> +
> + if (!vm || !trap_obj)
> + return -EINVAL;
> +
> + cwsr = kzalloc_obj(*cwsr, GFP_KERNEL);
> + if (!cwsr)
> + return -ENOMEM;
> +
> + bo = adev->trap_info->isa_bo;
> +
> + drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
> + drm_exec_until_all_locked(&exec) {
> + r = amdgpu_vm_lock_pd(vm, &exec, 0);
> + drm_exec_retry_on_contention(&exec);
> + if (unlikely(r))
> + goto err_exec;
> +
> + r = drm_exec_lock_obj(&exec, &bo->tbo.base);
> + drm_exec_retry_on_contention(&exec);
> + if (unlikely(r))
> + goto err_exec;
> + }
> +
> + /*
> + * Create the TMA BO with the VM root reservation. The reservation is
> + * already held by drm_exec, so creating the BO doesn't recursively
> + * acquire another reservation_ww_class_mutex. The TMA remains
> + * unpinned and is covered by the VM root reservation.
> + */
> + memset(&bp, 0, sizeof(bp));
> + bp.size = AMDGPU_TRAP_TMA_MAX_SIZE;
> + bp.byte_align = PAGE_SIZE;
> + bp.domain = AMDGPU_GEM_DOMAIN_GTT;
> + bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC |
> + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
> + bp.type = ttm_bo_type_kernel;
> + bp.bo_ptr_size = sizeof(struct amdgpu_bo);
> + bp.resv = vm->root.bo->tbo.base.resv;
> +
> + r = amdgpu_bo_create(adev, &bp, &cwsr->tma_bo);
> + if (r)
> + goto err_exec;
> +
> + /*
> + * The TMA BO is unpinned and movable. Map it only while accessing
> + * its contents so that no CPU mapping can become stale across BO
> + * migration. ttm_bo_vmap() and ttm_bo_vunmap() require the BO
> + * reservation, which is the already held VM root reservation.
> + */
> + r = ttm_bo_vmap(&cwsr->tma_bo->tbo, &map);
> + if (r)
> + goto err_tma;
> +
> + /*
> + * Finish initialization with a DMA write barrier before allowing
> + * the GPU to observe the contents.
> + */
> + iosys_map_memset(&map, 0, 0, AMDGPU_TRAP_TMA_MAX_SIZE);
> + dma_wmb();
> +
> + ttm_bo_vunmap(&cwsr->tma_bo->tbo, &map);
> +
> + r = amdgpu_trap_map_region(adev, vm, cwsr, AMDGPU_TRAP_TMA);
> + if (r)
> + goto err_tma;
> +
> + r = amdgpu_trap_map_region(adev, vm, cwsr, AMDGPU_TRAP_TBA);
> + if (r) {
> + amdgpu_trap_unmap_region(adev, cwsr, AMDGPU_TRAP_TMA);
> + goto err_tma;
> + }
> +
> + drm_exec_fini(&exec);
> +
> + *trap_obj = cwsr;
> +
> + return 0;
> +
> +err_tma:
> + amdgpu_bo_unref(&cwsr->tma_bo);
> +
> +err_exec:
> + drm_exec_fini(&exec);
> + kfree(cwsr);
> + *trap_obj = NULL;
> +
> + return r;
> +}
> +
> +static int amdgpu_trap_update_l2_handler(struct amdgpu_trap_obj *trap_obj,
> + u64 tba, u64 tma)
> +{
> + struct iosys_map map;
> + int r;
> +
> + if (!trap_obj || !trap_obj->tma_bo)
> + return -EINVAL;
> +
> + dma_resv_assert_held(trap_obj->tma_bo->tbo.base.resv);
> +
> + r = ttm_bo_vmap(&trap_obj->tma_bo->tbo, &map);
> + if (r)
> + return r;
> +
> + /*
> + * Publish both descriptor fields before allowing the GPU to observe
> + * the updated second-level trap handler.
> + */
> + iosys_map_wr(&map, AMDGPU_TRAP_L2_TBA_INDEX * sizeof(u64),
> + u64, tba);
> + iosys_map_wr(&map, AMDGPU_TRAP_L2_TMA_INDEX * sizeof(u64),
> + u64, tma);
> + dma_wmb();
> +
> + ttm_bo_vunmap(&trap_obj->tma_bo->tbo, &map);
> +
> + return 0;
> +}
> +
> +static int amdgpu_trap_clear_l2_handler(struct amdgpu_vm *vm,
> + struct amdgpu_trap_obj *trap_obj)
> +{
> + if (!trap_obj || !trap_obj->tma_bo)
> + return -EINVAL;
> +
> + dma_resv_assert_held(vm->root.bo->tbo.base.resv);
> +
> + return amdgpu_trap_update_l2_handler(trap_obj, 0, 0);
> +}
> +
> +int amdgpu_trap_validate_params(struct amdgpu_device *adev,
> + struct amdgpu_trap_params *trap_params,
> + int num_xcc)
> +{
> + u32 min_size, max_size;
> +
> + if (!amdgpu_trap_is_enabled(adev))
> + return -EOPNOTSUPP;
> +
> + if (!trap_params)
> + return -EINVAL;
> +
> + min_size = amdgpu_trap_size_needed(adev, num_xcc);
> + max_size = 2 * min_size;
> + /*
> + * Only save area size details checked. Address validation needs to be
> + * carried out separately. User is expected to pass a cwsr buffer area
> + * with size >= ctl stack size + wg data size + debug memory size
> + */
> + if (trap_params->cwsr_sz < min_size ||
> + trap_params->cwsr_sz > max_size) {
> + dev_dbg(adev->dev,
> + "queue cwsr size 0x%x should be within min %u : max %u\n",
> + trap_params->cwsr_sz, min_size, max_size);
> + return -EINVAL;
> + }
> +
> + return 0;
> +}
> +
> +void amdgpu_trap_free(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> + struct amdgpu_trap_obj **trap_obj)
> +{
> + struct amdgpu_bo *tba_bo;
> + struct drm_exec exec;
> + int r;
> +
> + if (!trap_obj || !*trap_obj)
> + return;
> +
> + tba_bo = adev->trap_info->isa_bo;
> +
> + drm_exec_init(&exec, 0, 0);
> + drm_exec_until_all_locked(&exec) {
> + r = amdgpu_vm_lock_pd(vm, &exec, 0);
> + drm_exec_retry_on_contention(&exec);
> + if (unlikely(r))
> + goto out_exec;
> +
> + if (tba_bo) {
> + r = drm_exec_lock_obj(&exec, &tba_bo->tbo.base);
> + drm_exec_retry_on_contention(&exec);
> + if (unlikely(r))
> + goto out_exec;
> + }
> + }
> +
> + if ((*trap_obj)->tba_va) {
> + amdgpu_vm_bo_del(adev, (*trap_obj)->tba_va);
> + (*trap_obj)->tba_va = NULL;
> + }
> +
> + if ((*trap_obj)->tma_va) {
> + amdgpu_vm_bo_del(adev, (*trap_obj)->tma_va);
> + (*trap_obj)->tma_va = NULL;
> + }
> +
> + drm_exec_fini(&exec);
> + amdgpu_bo_unref(&(*trap_obj)->tma_bo);
> + kfree(*trap_obj);
> + *trap_obj = NULL;
> +
> + return;
> +
> +out_exec:
> + drm_exec_fini(&exec);
> +}
> +
> +static int amdgpu_trap_validate_user_addr(struct amdgpu_device *adev,
> + struct amdgpu_vm *vm,
> + struct amdgpu_trap_usr_addr *usr_addr)
> +{
> + struct amdgpu_bo_va_mapping *va_map;
> + u64 start, end, addr, last;
> +
> + if (!usr_addr || !usr_addr->size)
> + return -EINVAL;
> +
> + dma_resv_assert_held(vm->root.bo->tbo.base.resv);
> +
> + start = usr_addr->addr & AMDGPU_GMC_HOLE_MASK;
> +
> + if (check_add_overflow(start, (u64)usr_addr->size - 1, &end))
> + return -EINVAL;
> +
> + addr = start >> AMDGPU_GPU_PAGE_SHIFT;
> + last = end >> AMDGPU_GPU_PAGE_SHIFT;
> +
> + va_map = amdgpu_vm_bo_lookup_mapping(vm, addr);
> + if (!va_map)
> + return -EINVAL;
> +
> + if (addr < va_map->start || last > va_map->last)
> + return -EINVAL;
> +
> + return 0;
> +}
> +
> +int amdgpu_trap_vm_set_l2_trap(struct amdgpu_device *adev,
> + struct amdgpu_vm *vm,
> + struct amdgpu_trap_obj *trap_obj,
> + u64 tba_va, uint32_t tba_sz,
> + u64 tma_va, uint32_t tma_sz)
> +{
> + struct amdgpu_trap_usr_addr tba = {
> + .addr = tba_va,
> + .size = tba_sz,
> + };
> + struct amdgpu_trap_usr_addr tma = {
> + .addr = tma_va,
> + .size = tma_sz,
> + };
> + long timeout;
> + int r;
> +
> + if (!amdgpu_trap_is_enabled(adev))
> + return -EOPNOTSUPP;
> +
> + if (!vm || !trap_obj || !trap_obj->tma_bo)
> + return -EINVAL;
> +
> + r = amdgpu_bo_reserve(vm->root.bo, false);
> + if (r)
> + return r;
> +
> + timeout = dma_resv_wait_timeout(vm->root.bo->tbo.base.resv,
> + DMA_RESV_USAGE_BOOKKEEP,
> + true, MAX_SCHEDULE_TIMEOUT);
> + if (timeout <= 0) {
> + r = timeout < 0 ? timeout : -ETIMEDOUT;
> + goto out_unreserve;
> + }
> +
> + r = amdgpu_trap_validate_user_addr(adev, vm, &tba);
> + if (r)
> + goto out_unreserve;
> +
> + r = amdgpu_trap_validate_user_addr(adev, vm, &tma);
> + if (r)
> + goto out_unreserve;
> +
> + r = amdgpu_trap_update_l2_handler(trap_obj, tba_va, tma_va);
> +
> +out_unreserve:
> + amdgpu_bo_unreserve(vm->root.bo);
> +
> + return r;
> +}
> +
> +int amdgpu_trap_vm_clear_l2_trap(struct amdgpu_device *adev,
> + struct amdgpu_vm *vm,
> + struct amdgpu_trap_obj *trap_obj)
> +{
> + long timeout;
> + int r;
> +
> + if (!amdgpu_trap_is_enabled(adev))
> + return -EOPNOTSUPP;
> +
> + if (!vm || !trap_obj || !trap_obj->tma_bo)
> + return -EINVAL;
> +
> + r = amdgpu_bo_reserve(vm->root.bo, false);
> + if (r)
> + return r;
> +
> + timeout = dma_resv_wait_timeout(vm->root.bo->tbo.base.resv,
> + DMA_RESV_USAGE_BOOKKEEP,
> + true, MAX_SCHEDULE_TIMEOUT);
> + if (timeout <= 0) {
> + r = timeout < 0 ? timeout : -ETIMEDOUT;
> + goto out_unreserve;
> + }
> +
> + r = amdgpu_trap_clear_l2_handler(vm, trap_obj);
> +
> +out_unreserve:
> + amdgpu_bo_unreserve(vm->root.bo);
> +
> + return r;
> +}
> +
> +int amdgpu_trap_set_trap_debug_flag(struct amdgpu_device *adev,
> + struct amdgpu_vm *vm,
> + struct amdgpu_trap_obj *trap_obj,
> + bool enabled)
> +{
> + struct iosys_map map;
> + int r;
> +
> + if (!amdgpu_trap_is_enabled(adev))
> + return -EOPNOTSUPP;
> +
> + if (!vm || !trap_obj || !trap_obj->tma_bo)
> + return -EINVAL;
> +
> + r = amdgpu_bo_reserve(vm->root.bo, false);
> + if (r)
> + return r;
> +
> + r = ttm_bo_vmap(&trap_obj->tma_bo->tbo, &map);
> + if (r)
> + goto out_unreserve;
> +
> + iosys_map_wr(&map, AMDGPU_TRAP_DEBUG_FLAG_INDEX * sizeof(u64),
> + u64, (u64)enabled);
> + dma_wmb();
> +
> + ttm_bo_vunmap(&trap_obj->tma_bo->tbo, &map);
> +
> +out_unreserve:
> + amdgpu_bo_unreserve(vm->root.bo);
> +
> + return r;
> +}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h
> new file mode 100644
> index 000000000000..c78c60e3d18d
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h
> @@ -0,0 +1,143 @@
> +/*
> + * Copyright 2025 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + */
> +
> +#ifndef AMDGPU_TRAP_H
> +#define AMDGPU_TRAP_H
> +
> +#include <linux/types.h>
> +
> +struct amdgpu_bo;
> +struct amdgpu_bo_va;
> +struct amdgpu_device;
> +struct amdgpu_vm;
> +
> +/**
> + * struct amdgpu_trap_obj - per-GPUVM CWSR trap-handler state
> + * @tma_gpu_va_addr: GPU virtual address of the first-level TMA
> + * @tba_gpu_va_addr: GPU virtual address of the first-level TBA
> + * @tma_bo: buffer object backing the first-level TMA
> + * @tba_va: VM mapping of the first-level TBA
> + * @tma_va: VM mapping of the first-level TMA
> + */
> +struct amdgpu_trap_obj {
> + u64 tma_gpu_va_addr;
> + u64 tba_gpu_va_addr;
> +
> + struct amdgpu_bo *tma_bo;
> + struct amdgpu_bo_va *tba_va;
> + struct amdgpu_bo_va *tma_va;
> +};
> +
> +struct amdgpu_trap_info {
> + /* cwsr isa */
> + struct amdgpu_bo *isa_bo;
> + const void *isa_buf;
> + u32 isa_sz;
> +
> + /* cwsr size info per XCC */
> + struct {
> + u32 xcc_ctl_stack_sz;
> + u32 xcc_dbg_mem_sz;
> + u32 xcc_cwsr_sz;
> + } cwsr;
> +};
> +
> +struct amdgpu_trap_params {
> + u64 ctx_save_area_address;
> +
> + /* cwsr size info */
> + u32 cwsr_sz;
> +};
> +
> +struct amdgpu_trap_usr_addr {
> + u64 addr;
> + u32 size;
> +};
> +
> +int amdgpu_trap_init(struct amdgpu_device *adev);
> +void amdgpu_trap_fini(struct amdgpu_device *adev);
> +
> +int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> + struct amdgpu_trap_obj **trap_obj);
> +
> +void amdgpu_trap_free(struct amdgpu_device *adev,
> + struct amdgpu_vm *vm,
> + struct amdgpu_trap_obj **trap_obj);
> +
> +static inline bool amdgpu_trap_is_enabled(struct amdgpu_device *adev)
> +{
> + return adev->trap_info;
> +}
> +
> +uint32_t amdgpu_trap_size_needed(struct amdgpu_device *adev, int num_xcc);
> +
> +int amdgpu_trap_validate_params(struct amdgpu_device *adev,
> + struct amdgpu_trap_params *trap_params,
> + int num_xcc);
> +
> +static inline bool amdgpu_trap_has_dbg_wa(struct amdgpu_device *adev)
> +{
> + u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
> +
> + return gc_ver >= IP_VERSION(11, 0, 0) &&
> + gc_ver <= IP_VERSION(11, 0, 3);
> +}
> +
> +/**
> + * amdgpu_trap_vm_set_l2_trap - install the second-level trap handler
> + * @adev: AMDGPU device
> + * @vm: GPUVM owning the first-level trap state
> + * @trap_obj: first-level trap object
> + * @tba_va: GPU virtual address of the second-level TBA
> + * @tba_sz: size of the second-level TBA
> + * @tma_va: GPU virtual address of the second-level TMA
> + * @tma_sz: size of the second-level TMA
> + *
> + * The second-level trap handler is configured per GPUVM. The descriptor is
> + * stored in the per-GPUVM first-level TMA.
> + *
> + * Return: 0 on success or a negative error code on failure.
> + */
> +int amdgpu_trap_vm_set_l2_trap(struct amdgpu_device *adev,
> + struct amdgpu_vm *vm,
> + struct amdgpu_trap_obj *trap_obj,
> + u64 tba_va, uint32_t tba_sz,
> + u64 tma_va, uint32_t tma_sz);
> +
> +/**
> + * amdgpu_trap_vm_clear_l2_trap - clear the second-level trap handler
> + * @adev: AMDGPU device
> + * @vm: GPUVM owning the first-level trap state
> + * @trap_obj: first-level trap object
> + *
> + * Return: 0 on success or a negative error code on failure.
> + */
> +int amdgpu_trap_vm_clear_l2_trap(struct amdgpu_device *adev,
> + struct amdgpu_vm *vm,
> + struct amdgpu_trap_obj *trap_obj);
> +
> +int amdgpu_trap_set_trap_debug_flag(struct amdgpu_device *adev,
> + struct amdgpu_vm *vm,
> + struct amdgpu_trap_obj *trap_obj,
> + bool enabled);
> +
> +#endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index 6eea0ca40556..b4c91e8cd37e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -35,7 +35,7 @@
> #include "amdgpu_hmm.h"
> #include "amdgpu_userq_fence.h"
> #include "amdgpu_trace.h"
> -#include "amdgpu_cwsr.h"
> +#include "amdgpu_trap.h"
>
> u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev)
> {
> @@ -267,9 +267,8 @@ int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
> return -EINVAL;
> }
>
> -int amdgpu_userq_input_cwsr_params_validate(
> - struct amdgpu_usermode_queue *queue,
> - struct amdgpu_cwsr_params *cwsr_params)
> +int amdgpu_userq_input_trap_params_validate(struct amdgpu_usermode_queue *queue,
> + struct amdgpu_trap_params *trap_params)
> {
> struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(queue->userq_mgr);
> struct amdgpu_device *adev = queue->userq_mgr->adev;
> @@ -278,16 +277,16 @@ int amdgpu_userq_input_cwsr_params_validate(
> int r;
>
> num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr, fpriv->xcp_id);
> - r = amdgpu_cwsr_validate_params(queue->userq_mgr->adev, cwsr_params,
> + r = amdgpu_trap_validate_params(queue->userq_mgr->adev, trap_params,
> num_xcc);
> if (r)
> return r;
> - cwsr_size = amdgpu_cwsr_size_needed(queue->userq_mgr->adev, num_xcc);
> + cwsr_size = amdgpu_trap_size_needed(queue->userq_mgr->adev, num_xcc);
> if (!cwsr_size)
> return -EOPNOTSUPP;
>
> return amdgpu_userq_input_va_validate(
> - adev, queue, cwsr_params->ctx_save_area_address, cwsr_size,
> + adev, queue, trap_params->ctx_save_area_address, cwsr_size,
> &queue->userq_vas.va.cwsr);
> }
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> index 727f60c40bbf..5eb0f3ee9b4e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
> @@ -41,7 +41,7 @@ enum amdgpu_userq_state {
> };
>
> struct amdgpu_mqd_prop;
> -struct amdgpu_cwsr_params;
> +struct amdgpu_trap_params;
>
> struct amdgpu_userq_obj {
> void *cpu_ptr;
> @@ -202,8 +202,7 @@ int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
>
> void amdgpu_userq_gem_va_unmap_validate(struct amdgpu_device *adev,
> struct amdgpu_bo_va_mapping *mapping);
> -int amdgpu_userq_input_cwsr_params_validate(
> - struct amdgpu_usermode_queue *queue,
> - struct amdgpu_cwsr_params *cwsr_params);
> +int amdgpu_userq_input_trap_params_validate(struct amdgpu_usermode_queue *queue,
> + struct amdgpu_trap_params *trap_params);
>
> #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> index 5311e1ad0201..07b653909943 100644
> --- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> +++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> @@ -27,7 +27,7 @@
> #include "mes_userqueue.h"
> #include "amdgpu_userq_fence.h"
> #include "amdgpu_trace.h"
> -#include "amdgpu_cwsr.h"
> +#include "amdgpu_trap.h"
>
> #define AMDGPU_USERQ_PROC_CTX_SZ PAGE_SIZE
> #define AMDGPU_USERQ_GANG_CTX_SZ PAGE_SIZE
> @@ -148,10 +148,10 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
> queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo);
> queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr;
>
> - if (fpriv->cwsr_trap) {
> - queue_input.tba_addr = fpriv->cwsr_trap->tba_gpu_va_addr;
> - queue_input.tma_addr = fpriv->cwsr_trap->tma_gpu_va_addr;
> - queue_input.trap_en = !amdgpu_cwsr_has_dbg_wa(adev);
> + if (fpriv->trap_obj) {
> + queue_input.tba_addr = fpriv->trap_obj->tba_gpu_va_addr;
> + queue_input.tma_addr = fpriv->trap_obj->tma_gpu_va_addr;
> + queue_input.trap_en = !amdgpu_trap_has_dbg_wa(adev);
> }
>
> if (mes->use_rs64mem) {
> @@ -415,7 +415,7 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
>
> if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) {
> struct drm_amdgpu_userq_mqd_compute_gfx11 *compute_mqd;
> - struct amdgpu_cwsr_params cwsr_params;
> + struct amdgpu_trap_params trap_params;
>
> if (mqd_user->mqd_size != sizeof(*compute_mqd)) {
> DRM_ERROR("Invalid compute IP MQD size\n");
> @@ -453,16 +453,16 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
> userq_props->tmz_queue =
> mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE;
>
> - if (amdgpu_cwsr_is_enabled(adev)) {
> + if (amdgpu_trap_is_enabled(adev)) {
> struct amdgpu_fpriv *fpriv =
> uq_mgr_to_fpriv(queue->userq_mgr);
> int num_xcc;
>
> num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr,
> fpriv->xcp_id);
> - cwsr_params.ctx_save_area_address =
> + trap_params.ctx_save_area_address =
> compute_mqd->ctx_save_area_va;
> - cwsr_params.cwsr_sz =
> + trap_params.cwsr_sz =
> compute_mqd->ctx_save_area_size;
>
> r = amdgpu_bo_reserve(queue->vm->root.bo, false);
> @@ -471,8 +471,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
> goto free_mqd;
> }
>
> - r = amdgpu_userq_input_cwsr_params_validate(
> - queue, &cwsr_params);
> + r = amdgpu_userq_input_trap_params_validate(
> + queue, &trap_params);
> amdgpu_bo_unreserve(queue->vm->root.bo);
> if (r) {
> kfree(compute_mqd);
> @@ -484,7 +484,7 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
> userq_props->ctx_save_area_size =
> compute_mqd->ctx_save_area_size;
> userq_props->ctl_stack_size =
> - adev->cwsr_info->xcc_ctl_stack_sz * num_xcc;
> + adev->trap_info->cwsr.xcc_ctl_stack_sz * num_xcc;
> }
>
> kfree(compute_mqd);
^ permalink raw reply [flat|nested] 16+ messages in thread
* [PATCH 5/7] drm/amdgpu: Make first-level trap state GPUVM-owned
2026-10-08 7:05 [PATCH 0/7] drm/amdgpu: Add CWSR and second-level trap handler support Srinivasan Shanmugam
` (3 preceding siblings ...)
2026-10-08 7:05 ` [PATCH 4/7] drm/amdgpu: Add second-level trap handler infrastructure Srinivasan Shanmugam
@ 2026-10-08 7:05 ` Srinivasan Shanmugam
2026-10-12 14:32 ` Christian König
2026-10-08 7:05 ` [PATCH 6/7] drm/amdgpu: Program trap TBA/TMA for kernel-managed VMIDs Srinivasan Shanmugam
2026-10-08 7:05 ` [PATCH 7/7] drm/amdgpu/uapi: Add SET/CLEAR L2 TRAP to VM ioctl Srinivasan Shanmugam
6 siblings, 1 reply; 16+ messages in thread
From: Srinivasan Shanmugam @ 2026-10-08 7:05 UTC (permalink / raw)
To: Christian König, Alex Deucher
Cc: amd-gfx, Srinivasan Shanmugam, Felix Kuehling, Lancelot Six,
James Zhu, Lijo Lazar, Samuel Pitoiset, Natalie Vock,
Timur Kristóf, Pierre-Eric Pelloux-Prayer
The first-level trap state belongs to the GPUVM because the TBA and TMA
are mappings in that GPUVM.
User queues and kernel queues using the same GPUVM use the same
first-level trap state. There is no need to keep a separate trap object
in the file-private data.
Move the trap object into struct amdgpu_vm. Create it when the GPUVM is
initialized and free it when the GPUVM is destroyed.
Create the trap state after releasing the initial VM root reservation.
The trap allocation locks the VM internally, so doing this while the VM
root is already reserved could lock the same reservation again.
For the same reason, free the trap state before reserving the VM root
during VM cleanup.
Keep the first-level trap ISA buffer shared at the device level. The ISA
is the same for all GPUVMs on the same device, while each GPUVM keeps
its own trap mapping and TMA state.
Update the user queue path to get the first-level TBA and TMA from its
GPUVM.
Suggested-by: Christian König <christian.koenig@amd.com>
Suggested-by: Alex Deucher <alexander.deucher@amd.com>
Cc: Felix Kuehling <felix.kuehling@amd.com>
Cc: Lancelot Six <lancelot.six@amd.com>
Cc: James Zhu <james.zhu@amd.com>
Cc: Lijo Lazar <lijo.lazar@amd.com>
Cc: Samuel Pitoiset <hakzsam@gmail.com>
Cc: Natalie Vock <natalie.vock@gmx.de>
Cc: Timur Kristóf <timur.kristof@gmail.com>
Cc: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
Signed-off-by: Srinivasan Shanmugam <srinivasan.shanmugam@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 --
drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 9 +----
drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c | 41 ++--------------------
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 31 ++++++++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 4 +++
drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 7 ++--
6 files changed, 41 insertions(+), 53 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index e55496f5ecdd..acd27ff6e0e6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -333,7 +333,6 @@ struct amdgpu_reset_control;
struct amdgpu_coredump_info;
struct amdgpu_video_codecs;
struct amdgpu_trap_isa;
-struct amdgpu_trap_obj;
enum amdgpu_cp_irq {
AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP = 0,
@@ -422,7 +421,6 @@ struct amdgpu_fpriv {
struct xarray bo_list_handles;
struct amdgpu_ctx_mgr ctx_mgr;
struct amdgpu_userq_mgr userq_mgr;
- struct amdgpu_trap_obj *trap_obj;
/* Eviction fence infra */
struct amdgpu_eviction_fence_mgr evf_mgr;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 6839d92b9730..45fa77c924a3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -1434,7 +1434,7 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
int num_xcc, r;
fpriv = (struct amdgpu_fpriv *)filp->driver_priv;
- if (!amdgpu_trap_is_enabled(adev) || !fpriv->trap_obj)
+ if (!amdgpu_trap_is_enabled(adev) || !fpriv->vm.trap_obj)
return -EOPNOTSUPP;
num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr, fpriv->xcp_id);
cwsr_info.ctl_stack_size =
@@ -1576,12 +1576,6 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
amdgpu_evf_mgr_init(&fpriv->evf_mgr);
- if (amdgpu_trap_is_enabled(adev)) {
- r = amdgpu_trap_alloc(adev, &fpriv->vm, &fpriv->trap_obj);
- if (r)
- dev_dbg(adev->dev, "trap handler not enabled");
- }
-
amdgpu_ctx_mgr_init(&fpriv->ctx_mgr, adev);
file_priv->driver_priv = fpriv;
@@ -1651,7 +1645,6 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
}
amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr);
- amdgpu_trap_free(adev, &fpriv->vm, &fpriv->trap_obj);
if (pasid)
amdgpu_pasid_free_delayed(pd->tbo.base.resv, pasid);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
index 12e37d0467f9..89968b5b7ecb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
@@ -326,8 +326,6 @@ static int amdgpu_trap_map_region(struct amdgpu_device *adev,
va &= AMDGPU_GMC_HOLE_MASK;
r = amdgpu_vm_bo_map(adev, *bo_va, va, 0, size, va_flags);
if (r) {
- dev_err(adev->dev, "failed to do bo map of %s region, err=%d\n",
- (region == AMDGPU_TRAP_TBA ? "tba" : "tma"), r);
amdgpu_vm_bo_del(adev, *bo_va);
*bo_va = NULL;
return r;
@@ -335,9 +333,6 @@ static int amdgpu_trap_map_region(struct amdgpu_device *adev,
r = amdgpu_vm_bo_update(adev, *bo_va, false);
if (r) {
- dev_err(adev->dev,
- "failed to do page table update of %s region, err=%d\n",
- (region == AMDGPU_TRAP_TBA ? "tba" : "tma"), r);
amdgpu_vm_bo_del(adev, *bo_va);
*bo_va = NULL;
return r;
@@ -351,39 +346,6 @@ static int amdgpu_trap_map_region(struct amdgpu_device *adev,
return 0;
}
-static int amdgpu_trap_unmap_region(struct amdgpu_device *adev,
- struct amdgpu_trap_obj *cwsr,
- enum amdgpu_trap_region region)
-{
- struct amdgpu_bo_va **bo_va;
- uint64_t va;
- int r;
-
- if (!cwsr)
- return -EINVAL;
-
- if (region == AMDGPU_TRAP_TBA) {
- bo_va = &cwsr->tba_va;
- va = cwsr->tba_gpu_va_addr;
- } else {
- bo_va = &cwsr->tma_va;
- va = cwsr->tma_gpu_va_addr;
- }
-
- r = amdgpu_vm_bo_unmap(adev, *bo_va, va);
- if (r) {
- dev_err(adev->dev,
- "failed to do bo_unmap on CWSR trap handler, err=%d\n",
- r);
- return r;
- }
-
- amdgpu_vm_bo_del(adev, *bo_va);
- *bo_va = NULL;
-
- return r;
-}
-
int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
struct amdgpu_trap_obj **trap_obj)
{
@@ -464,7 +426,8 @@ int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
r = amdgpu_trap_map_region(adev, vm, cwsr, AMDGPU_TRAP_TBA);
if (r) {
- amdgpu_trap_unmap_region(adev, cwsr, AMDGPU_TRAP_TMA);
+ amdgpu_vm_bo_del(adev, cwsr->tma_va);
+ cwsr->tma_va = NULL;
goto err_tma;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 7d3659ef11e5..de4071db94f5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -45,6 +45,7 @@
#include "amdgpu_xgmi.h"
#include "amdgpu_dma_buf.h"
#include "amdgpu_res_cursor.h"
+#include "amdgpu_trap.h"
#include "kfd_svm.h"
/**
@@ -2606,6 +2607,7 @@ void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
int32_t xcp_id)
{
+ struct amdgpu_bo_va_mapping *mapping, *tmp;
struct amdgpu_bo *root_bo;
struct amdgpu_bo_vm *root;
int r, i;
@@ -2614,6 +2616,8 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
vm->reserved_vmid[i] = NULL;
+ vm->trap_obj = NULL;
+
amdgpu_vm_bo_status_init(&vm->kernel);
amdgpu_vm_bo_status_init(&vm->always_valid);
spin_lock_init(&vm->individual_lock);
@@ -2680,10 +2684,28 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
dev_dbg(adev->dev, "Failed to create task info for VM\n");
amdgpu_bo_unreserve(vm->root.bo);
+
+ if (amdgpu_trap_is_enabled(adev)) {
+ r = amdgpu_trap_alloc(adev, vm, &vm->trap_obj);
+ if (r)
+ goto error_free_trap;
+ }
+
amdgpu_bo_unref(&root_bo);
return 0;
+error_free_trap:
+ amdgpu_bo_reserve(root_bo, true);
+
+ list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
+ list_del(&mapping->list);
+ amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
+ }
+
+ amdgpu_vm_put_task_info(vm->task_info);
+ dma_fence_put(vm->last_update);
+
error_free_root:
amdgpu_vm_pt_free_root(adev, vm);
amdgpu_bo_unreserve(vm->root.bo);
@@ -2830,6 +2852,15 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
+ /*
+ * Free trap state before reserving root. amdgpu_trap_free()
+ * uses drm_exec to lock the page directory internally; calling
+ * it after amdgpu_bo_reserve(root) would deadlock on the same
+ * ww_mutex.
+ */
+ if (vm->trap_obj)
+ amdgpu_trap_free(adev, vm, &vm->trap_obj);
+
root = amdgpu_bo_ref(vm->root.bo);
amdgpu_bo_reserve(root, true);
dma_fence_wait(vm->last_unlocked, false);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 226caa99f6ee..2750123dd7d0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -43,6 +43,7 @@ struct amdgpu_bo_va;
struct amdgpu_job;
struct amdgpu_bo_list_entry;
struct amdgpu_bo_vm;
+struct amdgpu_trap_obj;
/*
* GPUVM handling
@@ -385,6 +386,9 @@ struct amdgpu_vm {
/* cached fault info */
struct amdgpu_vm_fault_info fault_info;
+
+ /* First-level trap state for this GPUVM; NULL if unavailable. */
+ struct amdgpu_trap_obj *trap_obj;
};
struct amdgpu_vm_manager {
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
index 07b653909943..8ffbf93ff0fa 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
@@ -116,7 +116,6 @@ static int convert_to_mes_priority(int priority)
static int mes_userq_map(struct amdgpu_usermode_queue *queue)
{
struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
- struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
struct amdgpu_device *adev = uq_mgr->adev;
struct amdgpu_userq_obj *ctx = &queue->fw_obj;
struct amdgpu_mqd_prop *userq_props = queue->userq_prop;
@@ -148,9 +147,9 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo);
queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr;
- if (fpriv->trap_obj) {
- queue_input.tba_addr = fpriv->trap_obj->tba_gpu_va_addr;
- queue_input.tma_addr = fpriv->trap_obj->tma_gpu_va_addr;
+ if (queue->vm->trap_obj) {
+ queue_input.tba_addr = queue->vm->trap_obj->tba_gpu_va_addr;
+ queue_input.tma_addr = queue->vm->trap_obj->tma_gpu_va_addr;
queue_input.trap_en = !amdgpu_trap_has_dbg_wa(adev);
}
--
2.34.1
^ permalink raw reply related [flat|nested] 16+ messages in thread* Re: [PATCH 5/7] drm/amdgpu: Make first-level trap state GPUVM-owned
2026-10-08 7:05 ` [PATCH 5/7] drm/amdgpu: Make first-level trap state GPUVM-owned Srinivasan Shanmugam
@ 2026-10-12 14:32 ` Christian König
2026-10-14 9:01 ` SRINIVASAN SHANMUGAM
0 siblings, 1 reply; 16+ messages in thread
From: Christian König @ 2026-10-12 14:32 UTC (permalink / raw)
To: Srinivasan Shanmugam, Alex Deucher
Cc: amd-gfx, Felix Kuehling, Lancelot Six, James Zhu, Lijo Lazar,
Samuel Pitoiset, Natalie Vock, Timur Kristóf,
Pierre-Eric Pelloux-Prayer
On 10/8/26 09:05, Srinivasan Shanmugam wrote:
> The first-level trap state belongs to the GPUVM because the TBA and TMA
> are mappings in that GPUVM.
Oh, that was actually not what I thought of.
The idea was to use the VM PDs root reservation object to allocate the trap handler and so simplify the handling.
Moving that into the amdgpu_vm structure was not part of this plan. It's not really a page table and so probably better suited in fpriv after all.
On the other hand we want to modify it through the VM IOCTLs and so making it part of the VM is probably not so bad.
I need to think about that a bit more when I have time.
Regards,
Christian.
>
> User queues and kernel queues using the same GPUVM use the same
> first-level trap state. There is no need to keep a separate trap object
> in the file-private data.
>
> Move the trap object into struct amdgpu_vm. Create it when the GPUVM is
> initialized and free it when the GPUVM is destroyed.
>
> Create the trap state after releasing the initial VM root reservation.
> The trap allocation locks the VM internally, so doing this while the VM
> root is already reserved could lock the same reservation again.
>
> For the same reason, free the trap state before reserving the VM root
> during VM cleanup.
>
> Keep the first-level trap ISA buffer shared at the device level. The ISA
> is the same for all GPUVMs on the same device, while each GPUVM keeps
> its own trap mapping and TMA state.
>
> Update the user queue path to get the first-level TBA and TMA from its
> GPUVM.
>
> Suggested-by: Christian König <christian.koenig@amd.com>
> Suggested-by: Alex Deucher <alexander.deucher@amd.com>
> Cc: Felix Kuehling <felix.kuehling@amd.com>
> Cc: Lancelot Six <lancelot.six@amd.com>
> Cc: James Zhu <james.zhu@amd.com>
> Cc: Lijo Lazar <lijo.lazar@amd.com>
> Cc: Samuel Pitoiset <hakzsam@gmail.com>
> Cc: Natalie Vock <natalie.vock@gmx.de>
> Cc: Timur Kristóf <timur.kristof@gmail.com>
> Cc: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
> Signed-off-by: Srinivasan Shanmugam <srinivasan.shanmugam@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 --
> drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 9 +----
> drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c | 41 ++--------------------
> drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 31 ++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 4 +++
> drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 7 ++--
> 6 files changed, 41 insertions(+), 53 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index e55496f5ecdd..acd27ff6e0e6 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -333,7 +333,6 @@ struct amdgpu_reset_control;
> struct amdgpu_coredump_info;
> struct amdgpu_video_codecs;
> struct amdgpu_trap_isa;
> -struct amdgpu_trap_obj;
>
> enum amdgpu_cp_irq {
> AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP = 0,
> @@ -422,7 +421,6 @@ struct amdgpu_fpriv {
> struct xarray bo_list_handles;
> struct amdgpu_ctx_mgr ctx_mgr;
> struct amdgpu_userq_mgr userq_mgr;
> - struct amdgpu_trap_obj *trap_obj;
>
> /* Eviction fence infra */
> struct amdgpu_eviction_fence_mgr evf_mgr;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index 6839d92b9730..45fa77c924a3 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -1434,7 +1434,7 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
> int num_xcc, r;
>
> fpriv = (struct amdgpu_fpriv *)filp->driver_priv;
> - if (!amdgpu_trap_is_enabled(adev) || !fpriv->trap_obj)
> + if (!amdgpu_trap_is_enabled(adev) || !fpriv->vm.trap_obj)
> return -EOPNOTSUPP;
> num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr, fpriv->xcp_id);
> cwsr_info.ctl_stack_size =
> @@ -1576,12 +1576,6 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
>
> amdgpu_evf_mgr_init(&fpriv->evf_mgr);
>
> - if (amdgpu_trap_is_enabled(adev)) {
> - r = amdgpu_trap_alloc(adev, &fpriv->vm, &fpriv->trap_obj);
> - if (r)
> - dev_dbg(adev->dev, "trap handler not enabled");
> - }
> -
> amdgpu_ctx_mgr_init(&fpriv->ctx_mgr, adev);
>
> file_priv->driver_priv = fpriv;
> @@ -1651,7 +1645,6 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
> }
>
> amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr);
> - amdgpu_trap_free(adev, &fpriv->vm, &fpriv->trap_obj);
>
> if (pasid)
> amdgpu_pasid_free_delayed(pd->tbo.base.resv, pasid);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
> index 12e37d0467f9..89968b5b7ecb 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
> @@ -326,8 +326,6 @@ static int amdgpu_trap_map_region(struct amdgpu_device *adev,
> va &= AMDGPU_GMC_HOLE_MASK;
> r = amdgpu_vm_bo_map(adev, *bo_va, va, 0, size, va_flags);
> if (r) {
> - dev_err(adev->dev, "failed to do bo map of %s region, err=%d\n",
> - (region == AMDGPU_TRAP_TBA ? "tba" : "tma"), r);
> amdgpu_vm_bo_del(adev, *bo_va);
> *bo_va = NULL;
> return r;
> @@ -335,9 +333,6 @@ static int amdgpu_trap_map_region(struct amdgpu_device *adev,
>
> r = amdgpu_vm_bo_update(adev, *bo_va, false);
> if (r) {
> - dev_err(adev->dev,
> - "failed to do page table update of %s region, err=%d\n",
> - (region == AMDGPU_TRAP_TBA ? "tba" : "tma"), r);
> amdgpu_vm_bo_del(adev, *bo_va);
> *bo_va = NULL;
> return r;
> @@ -351,39 +346,6 @@ static int amdgpu_trap_map_region(struct amdgpu_device *adev,
> return 0;
> }
>
> -static int amdgpu_trap_unmap_region(struct amdgpu_device *adev,
> - struct amdgpu_trap_obj *cwsr,
> - enum amdgpu_trap_region region)
> -{
> - struct amdgpu_bo_va **bo_va;
> - uint64_t va;
> - int r;
> -
> - if (!cwsr)
> - return -EINVAL;
> -
> - if (region == AMDGPU_TRAP_TBA) {
> - bo_va = &cwsr->tba_va;
> - va = cwsr->tba_gpu_va_addr;
> - } else {
> - bo_va = &cwsr->tma_va;
> - va = cwsr->tma_gpu_va_addr;
> - }
> -
> - r = amdgpu_vm_bo_unmap(adev, *bo_va, va);
> - if (r) {
> - dev_err(adev->dev,
> - "failed to do bo_unmap on CWSR trap handler, err=%d\n",
> - r);
> - return r;
> - }
> -
> - amdgpu_vm_bo_del(adev, *bo_va);
> - *bo_va = NULL;
> -
> - return r;
> -}
> -
> int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> struct amdgpu_trap_obj **trap_obj)
> {
> @@ -464,7 +426,8 @@ int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>
> r = amdgpu_trap_map_region(adev, vm, cwsr, AMDGPU_TRAP_TBA);
> if (r) {
> - amdgpu_trap_unmap_region(adev, cwsr, AMDGPU_TRAP_TMA);
> + amdgpu_vm_bo_del(adev, cwsr->tma_va);
> + cwsr->tma_va = NULL;
> goto err_tma;
> }
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> index 7d3659ef11e5..de4071db94f5 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -45,6 +45,7 @@
> #include "amdgpu_xgmi.h"
> #include "amdgpu_dma_buf.h"
> #include "amdgpu_res_cursor.h"
> +#include "amdgpu_trap.h"
> #include "kfd_svm.h"
>
> /**
> @@ -2606,6 +2607,7 @@ void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
> int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> int32_t xcp_id)
> {
> + struct amdgpu_bo_va_mapping *mapping, *tmp;
> struct amdgpu_bo *root_bo;
> struct amdgpu_bo_vm *root;
> int r, i;
> @@ -2614,6 +2616,8 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
> vm->reserved_vmid[i] = NULL;
>
> + vm->trap_obj = NULL;
> +
> amdgpu_vm_bo_status_init(&vm->kernel);
> amdgpu_vm_bo_status_init(&vm->always_valid);
> spin_lock_init(&vm->individual_lock);
> @@ -2680,10 +2684,28 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> dev_dbg(adev->dev, "Failed to create task info for VM\n");
>
> amdgpu_bo_unreserve(vm->root.bo);
> +
> + if (amdgpu_trap_is_enabled(adev)) {
> + r = amdgpu_trap_alloc(adev, vm, &vm->trap_obj);
> + if (r)
> + goto error_free_trap;
> + }
> +
> amdgpu_bo_unref(&root_bo);
>
> return 0;
>
> +error_free_trap:
> + amdgpu_bo_reserve(root_bo, true);
> +
> + list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
> + list_del(&mapping->list);
> + amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
> + }
> +
> + amdgpu_vm_put_task_info(vm->task_info);
> + dma_fence_put(vm->last_update);
> +
> error_free_root:
> amdgpu_vm_pt_free_root(adev, vm);
> amdgpu_bo_unreserve(vm->root.bo);
> @@ -2830,6 +2852,15 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
>
> amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
>
> + /*
> + * Free trap state before reserving root. amdgpu_trap_free()
> + * uses drm_exec to lock the page directory internally; calling
> + * it after amdgpu_bo_reserve(root) would deadlock on the same
> + * ww_mutex.
> + */
> + if (vm->trap_obj)
> + amdgpu_trap_free(adev, vm, &vm->trap_obj);
> +
> root = amdgpu_bo_ref(vm->root.bo);
> amdgpu_bo_reserve(root, true);
> dma_fence_wait(vm->last_unlocked, false);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> index 226caa99f6ee..2750123dd7d0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> @@ -43,6 +43,7 @@ struct amdgpu_bo_va;
> struct amdgpu_job;
> struct amdgpu_bo_list_entry;
> struct amdgpu_bo_vm;
> +struct amdgpu_trap_obj;
>
> /*
> * GPUVM handling
> @@ -385,6 +386,9 @@ struct amdgpu_vm {
>
> /* cached fault info */
> struct amdgpu_vm_fault_info fault_info;
> +
> + /* First-level trap state for this GPUVM; NULL if unavailable. */
> + struct amdgpu_trap_obj *trap_obj;
> };
>
> struct amdgpu_vm_manager {
> diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> index 07b653909943..8ffbf93ff0fa 100644
> --- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> +++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
> @@ -116,7 +116,6 @@ static int convert_to_mes_priority(int priority)
> static int mes_userq_map(struct amdgpu_usermode_queue *queue)
> {
> struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
> - struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
> struct amdgpu_device *adev = uq_mgr->adev;
> struct amdgpu_userq_obj *ctx = &queue->fw_obj;
> struct amdgpu_mqd_prop *userq_props = queue->userq_prop;
> @@ -148,9 +147,9 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
> queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo);
> queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr;
>
> - if (fpriv->trap_obj) {
> - queue_input.tba_addr = fpriv->trap_obj->tba_gpu_va_addr;
> - queue_input.tma_addr = fpriv->trap_obj->tma_gpu_va_addr;
> + if (queue->vm->trap_obj) {
> + queue_input.tba_addr = queue->vm->trap_obj->tba_gpu_va_addr;
> + queue_input.tma_addr = queue->vm->trap_obj->tma_gpu_va_addr;
> queue_input.trap_en = !amdgpu_trap_has_dbg_wa(adev);
> }
>
^ permalink raw reply [flat|nested] 16+ messages in thread* Re: [PATCH 5/7] drm/amdgpu: Make first-level trap state GPUVM-owned
2026-10-12 14:32 ` Christian König
@ 2026-10-14 9:01 ` SRINIVASAN SHANMUGAM
0 siblings, 0 replies; 16+ messages in thread
From: SRINIVASAN SHANMUGAM @ 2026-10-14 9:01 UTC (permalink / raw)
To: Christian König, Alex Deucher
Cc: amd-gfx, Felix Kuehling, Lancelot Six, James Zhu, Lijo Lazar,
Samuel Pitoiset, Natalie Vock, Timur Kristóf,
Pierre-Eric Pelloux-Prayer
[-- Attachment #1: Type: text/plain, Size: 1278 bytes --]
On 10/12/2026 8:02 PM, Christian König wrote:
> On 10/8/26 09:05, Srinivasan Shanmugam wrote:
>> The first-level trap state belongs to the GPUVM because the TBA and TMA
>> are mappings in that GPUVM.
> Oh, that was actually not what I thought of.
>
> The idea was to use the VM PDs root reservation object to allocate the trap handler and so simplify the handling.
>
> Moving that into the amdgpu_vm structure was not part of this plan. It's not really a page table and so probably better suited in fpriv after all.
>
> On the other hand we want to modify it through the VM IOCTLs and so making it part of the VM is probably not so bad.
>
> I need to think about that a bit more when I have time.
Hi Christian,
Good Morning!
Thanks for your feed backs and reviews, No rush, but when you have some
free time, Just a quick follow up on the trap object ownership discussion.
My thinking was that the trap state is tied to the GPUVM mappings
(TBA/TMA), and queues using the same GPUVM share the same trap state.
Since the control path is also moving through the VM ioctl interface,
keeping the trap object in |amdgpu_vm| felt natural to me.
However, I'm happy to follow whichever approach you think is best in the
long term.
Thanks, Srini
>
> Regards,
> Christian.
[-- Attachment #2: Type: text/html, Size: 2424 bytes --]
^ permalink raw reply [flat|nested] 16+ messages in thread
* [PATCH 6/7] drm/amdgpu: Program trap TBA/TMA for kernel-managed VMIDs
2026-10-08 7:05 [PATCH 0/7] drm/amdgpu: Add CWSR and second-level trap handler support Srinivasan Shanmugam
` (4 preceding siblings ...)
2026-10-08 7:05 ` [PATCH 5/7] drm/amdgpu: Make first-level trap state GPUVM-owned Srinivasan Shanmugam
@ 2026-10-08 7:05 ` Srinivasan Shanmugam
2026-10-12 15:05 ` Christian König
2026-10-08 7:05 ` [PATCH 7/7] drm/amdgpu/uapi: Add SET/CLEAR L2 TRAP to VM ioctl Srinivasan Shanmugam
6 siblings, 1 reply; 16+ messages in thread
From: Srinivasan Shanmugam @ 2026-10-08 7:05 UTC (permalink / raw)
To: Christian König, Alex Deucher
Cc: amd-gfx, Srinivasan Shanmugam, Lijo Lazar, Felix Kuehling,
Lancelot Six, James Zhu, Samuel Pitoiset, Natalie Vock,
Timur Kristóf, Pierre-Eric Pelloux-Prayer
User queues get their first-level TBA and TMA from MES/KIQ when the
queue is created. Kernel-managed VMIDs do not use this path, so program
their trap registers from the gfxhub setup path.
Get the reserved first-level TBA and TMA virtual addresses and program
them for each kernel-managed VMID.
The trap registers use the SRBM selector, so keep the programming under
srbm_mutex and restore the VMID selector after the registers are
updated.
Trap enablement differs across GPU generations. On GFX9, enable traps
for kernel-managed VMIDs through the debug trap configuration. On
GFX9.4.3, enable the per-VMID trap control on each XCC. On generations
where trap enable is encoded in SQ_SHADER_TBA_HI, program TRAP_EN
together with the TBA address.
Mask the TBA and TMA address fields before writing the registers. Keep
TRAP_EN disabled on the affected GFX11 ASICs to preserve the existing
trap workaround.
On multi-XCC hardware, program the requested XCCs and restore the VMID
selector on each XCC afterwards.
Keep the first-level trap VA layout inside amdgpu_trap. The gfxhub code
only needs the final TBA and TMA addresses.
Suggested-by: Christian König <christian.koenig@amd.com>
Suggested-by: Alex Deucher <alexander.deucher@amd.com>
Suggested-by: Lijo Lazar <lijo.lazar@amd.com>
Cc: Felix Kuehling <felix.kuehling@amd.com>
Cc: Lancelot Six <lancelot.six@amd.com>
Cc: James Zhu <james.zhu@amd.com>
Cc: Samuel Pitoiset <hakzsam@gmail.com>
Cc: Natalie Vock <natalie.vock@gmx.de>
Cc: Timur Kristóf <timur.kristof@gmail.com>
Cc: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
Signed-off-by: Srinivasan Shanmugam <srinivasan.shanmugam@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c | 9 +++-
drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h | 5 ++
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 29 ++++++++---
drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c | 16 +++++++
drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c | 38 +++++++++++++++
drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c | 38 +++++++++++++++
drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c | 53 +++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c | 41 ++++++++++++++++
drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c | 53 +++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c | 38 +++++++++++++++
drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c | 42 ++++++++++++++++
drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c | 46 ++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c | 46 ++++++++++++++++++
13 files changed, 446 insertions(+), 8 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
index 89968b5b7ecb..6e7dfceeee10 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
@@ -63,7 +63,14 @@ static inline uint64_t amdgpu_trap_tba_vaddr(struct amdgpu_device *adev)
return addr;
}
-static inline bool amdgpu_trap_is_supported(struct amdgpu_device *adev)
+void amdgpu_trap_get_l1_addr(struct amdgpu_device *adev,
+ u64 *tba_addr, u64 *tma_addr)
+{
+ *tba_addr = amdgpu_trap_tba_vaddr(adev);
+ *tma_addr = *tba_addr + AMDGPU_TRAP_TMA_OFFSET;
+}
+
+bool amdgpu_trap_is_supported(struct amdgpu_device *adev)
{
uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h
index c78c60e3d18d..523f65cb3d3d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h
@@ -76,6 +76,11 @@ struct amdgpu_trap_usr_addr {
int amdgpu_trap_init(struct amdgpu_device *adev);
void amdgpu_trap_fini(struct amdgpu_device *adev);
+bool amdgpu_trap_is_supported(struct amdgpu_device *adev);
+
+void amdgpu_trap_get_l1_addr(struct amdgpu_device *adev,
+ u64 *tba_addr, u64 *tma_addr);
+
int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
struct amdgpu_trap_obj **trap_obj);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 1678015ce5a9..07e249500254 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -29,6 +29,7 @@
#include "amdgpu.h"
#include "amdgpu_gfx.h"
+#include "amdgpu_trap.h"
#include "soc15.h"
#include "soc15d.h"
#include "amdgpu_atomfirmware.h"
@@ -2568,21 +2569,26 @@ static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
}
static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev,
- uint32_t first_vmid,
- uint32_t last_vmid)
+ u32 first_vmid,
+ uint32_t last_vmid)
{
uint32_t data;
- uint32_t trap_config_vmid_mask = 0;
+ u32 trap_config_vmid_mask;
int i;
- /* Calculate trap config vmid mask */
+ data = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG));
+ trap_config_vmid_mask =
+ REG_GET_FIELD(data, SPI_GDBG_TRAP_CONFIG, VMID_SEL);
+
+ /* Add the requested VMIDs to the existing trap configuration. */
for (i = first_vmid; i < last_vmid; i++)
trap_config_vmid_mask |= (1 << i);
- data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG,
- VMID_SEL, trap_config_vmid_mask);
data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG,
- TRAP_EN, 1);
+ VMID_SEL, trap_config_vmid_mask);
+ data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG,
+ TRAP_EN, 1);
+
WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data);
WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0);
@@ -4051,6 +4057,15 @@ static int gfx_v9_0_hw_init(struct amdgpu_ip_block *ip_block)
if (r)
return r;
+ if (amdgpu_trap_is_supported(adev)) {
+ if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
+ gfx_v9_4_2_debug_trap_config_init(adev, 1,
+ adev->vm_manager.first_kfd_vmid);
+ else
+ gfx_v9_0_debug_trap_config_init(adev, 1,
+ adev->vm_manager.first_kfd_vmid);
+ }
+
if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) &&
!amdgpu_sriov_vf(adev))
gfx_v9_4_2_set_power_brake_sequence(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
index 01ed1701c781..69a1e64620cd 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
@@ -25,6 +25,7 @@
#include "amdgpu.h"
#include "amdgpu_ip.h"
#include "amdgpu_gfx.h"
+#include "amdgpu_trap.h"
#include "soc15.h"
#include "soc15d.h"
#include "soc15_common.h"
@@ -1173,6 +1174,21 @@ static void gfx_v9_4_3_xcc_init_compute_vmid(struct amdgpu_device *adev,
SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
mutex_lock(&adev->srbm_mutex);
+
+ if (amdgpu_trap_is_supported(adev)) {
+ for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) {
+ soc15_grbm_select(adev, 0, 0, 0, i,
+ GET_INST(GC, xcc_id));
+
+ data = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
+ regSPI_GDBG_PER_VMID_CNTL);
+ data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL,
+ TRAP_EN, 1);
+ WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id),
+ regSPI_GDBG_PER_VMID_CNTL, data);
+ }
+ }
+
for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
soc15_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id));
/* CP and shaders */
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c
index 8be912d50601..e635d83488af 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c
@@ -22,6 +22,7 @@
*/
#include "amdgpu.h"
+#include "amdgpu_trap.h"
#include "gfxhub_v11_5_0.h"
#include "gc/gc_11_5_0_offset.h"
@@ -295,6 +296,40 @@ static void gfxhub_v11_5_0_disable_identity_aperture(struct amdgpu_device *adev)
}
+static void
+gfxhub_v11_5_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev)
+{
+ u64 tba_addr, tma_addr;
+ unsigned int vmid;
+
+ if (!amdgpu_trap_is_supported(adev) ||
+ !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
+ return;
+
+ amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
+
+ WARN_ON(!IS_ALIGNED(tba_addr, 256));
+ WARN_ON(!IS_ALIGNED(tma_addr, 256));
+
+ mutex_lock(&adev->srbm_mutex);
+ for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) {
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0);
+ WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO,
+ lower_32_bits(tba_addr >> 8));
+ WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI,
+ (upper_32_bits(tba_addr >> 8) &
+ SQ_SHADER_TBA_HI__ADDR_HI_MASK) |
+ (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT));
+ WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO,
+ lower_32_bits(tma_addr >> 8));
+ WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI,
+ upper_32_bits(tma_addr >> 8) &
+ SQ_SHADER_TMA_HI__ADDR_HI_MASK);
+ }
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
+ mutex_unlock(&adev->srbm_mutex);
+}
+
static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev)
{
struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
@@ -342,6 +377,9 @@ static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev)
}
hub->vm_cntx_cntl = tmp;
+
+ if (amdgpu_trap_is_supported(adev))
+ gfxhub_v11_5_0_program_kernel_trap_tba_tma(adev);
}
static void gfxhub_v11_5_0_program_invalidation(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c
index 4144c1c2ee13..dc7fde6962ca 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c
@@ -22,6 +22,7 @@
*/
#include "amdgpu.h"
+#include "amdgpu_trap.h"
#include "gfxhub_v12_0.h"
#include "gc/gc_12_0_0_offset.h"
@@ -300,6 +301,40 @@ static void gfxhub_v12_0_disable_identity_aperture(struct amdgpu_device *adev)
}
+static void
+gfxhub_v12_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev)
+{
+ u64 tba_addr, tma_addr;
+ unsigned int vmid;
+
+ if (!amdgpu_trap_is_supported(adev) ||
+ !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
+ return;
+
+ amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
+
+ WARN_ON(!IS_ALIGNED(tba_addr, 256));
+ WARN_ON(!IS_ALIGNED(tma_addr, 256));
+
+ mutex_lock(&adev->srbm_mutex);
+ for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) {
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0);
+ WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO,
+ lower_32_bits(tba_addr >> 8));
+ WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI,
+ (upper_32_bits(tba_addr >> 8) &
+ SQ_SHADER_TBA_HI__ADDR_HI_MASK) |
+ (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT));
+ WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO,
+ lower_32_bits(tma_addr >> 8));
+ WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI,
+ upper_32_bits(tma_addr >> 8) &
+ SQ_SHADER_TMA_HI__ADDR_HI_MASK);
+ }
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
+ mutex_unlock(&adev->srbm_mutex);
+}
+
static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev)
{
struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
@@ -347,6 +382,9 @@ static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev)
}
hub->vm_cntx_cntl = tmp;
+
+ if (amdgpu_trap_is_supported(adev))
+ gfxhub_v12_0_program_kernel_trap_tba_tma(adev);
}
static void gfxhub_v12_0_program_invalidation(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
index b13967a1c8fb..78088e72606c 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
@@ -21,6 +21,7 @@
*
*/
#include "amdgpu.h"
+#include "amdgpu_trap.h"
#include "amdgpu_xcp.h"
#include "gfxhub_v12_1.h"
@@ -392,6 +393,55 @@ static void gfxhub_v12_1_xcc_disable_identity_aperture(struct amdgpu_device *ade
}
}
+static void
+gfxhub_v12_1_program_kernel_trap_tba_tma(struct amdgpu_device *adev,
+ uint32_t xcc_mask)
+{
+ u64 tba_addr, tma_addr;
+ unsigned int vmid;
+ int xcc;
+
+ if (!amdgpu_trap_is_supported(adev) ||
+ !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
+ return;
+
+ amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
+
+ WARN_ON(!IS_ALIGNED(tba_addr, 256));
+ WARN_ON(!IS_ALIGNED(tma_addr, 256));
+
+ mutex_lock(&adev->srbm_mutex);
+ for_each_inst(xcc, xcc_mask) {
+ for (vmid = 1;
+ vmid < adev->vm_manager.first_kfd_vmid;
+ vmid++) {
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0,
+ vmid, xcc);
+ WREG32_SOC15(GC, GET_INST(GC, xcc),
+ regSQ_SHADER_TBA_LO,
+ lower_32_bits(tba_addr >> 8));
+ WREG32_SOC15(GC, GET_INST(GC, xcc),
+ regSQ_SHADER_TBA_HI,
+ (upper_32_bits(tba_addr >> 8) &
+ SQ_SHADER_TBA_HI__ADDR_HI_MASK) |
+ (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT));
+ WREG32_SOC15(GC, GET_INST(GC, xcc),
+ regSQ_SHADER_TMA_LO,
+ lower_32_bits(tma_addr >> 8));
+ WREG32_SOC15(GC, GET_INST(GC, xcc),
+ regSQ_SHADER_TMA_HI,
+ upper_32_bits(tma_addr >> 8) &
+ SQ_SHADER_TMA_HI__ADDR_HI_MASK);
+ }
+ }
+
+ /* Reset the VMID selector on every XCC instance. */
+ for_each_inst(xcc, xcc_mask)
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, xcc);
+
+ mutex_unlock(&adev->srbm_mutex);
+}
+
static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev,
uint32_t xcc_mask)
{
@@ -457,6 +507,9 @@ static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev,
hub->vm_cntx_cntl = tmp;
}
+
+ if (amdgpu_trap_is_supported(adev))
+ gfxhub_v12_1_program_kernel_trap_tba_tma(adev, xcc_mask);
}
static void gfxhub_v12_1_xcc_program_invalidation(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
index c35856180c3c..dbd993ae1d0e 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
@@ -21,6 +21,7 @@
*
*/
#include "amdgpu.h"
+#include "amdgpu_trap.h"
#include "gfxhub_v1_0.h"
#include "gfxhub_v1_1.h"
@@ -252,6 +253,43 @@ static void gfxhub_v1_0_disable_identity_aperture(struct amdgpu_device *adev)
}
+static void
+gfxhub_v1_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev)
+{
+ u64 tba_addr, tma_addr;
+ unsigned int vmid;
+
+ if (!amdgpu_trap_is_supported(adev) ||
+ !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
+ return;
+
+ amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
+
+ WARN_ON(!IS_ALIGNED(tba_addr, 256));
+ WARN_ON(!IS_ALIGNED(tma_addr, 256));
+
+ mutex_lock(&adev->srbm_mutex);
+
+ for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) {
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0);
+
+ WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO,
+ lower_32_bits(tba_addr >> 8));
+ WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI,
+ upper_32_bits(tba_addr >> 8) &
+ SQ_SHADER_TBA_HI__ADDR_HI_MASK);
+ WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO,
+ lower_32_bits(tma_addr >> 8));
+ WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI,
+ upper_32_bits(tma_addr >> 8) &
+ SQ_SHADER_TMA_HI__ADDR_HI_MASK);
+ }
+
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
+
+ mutex_unlock(&adev->srbm_mutex);
+}
+
static void gfxhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
{
struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
@@ -310,6 +348,9 @@ static void gfxhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
i * hub->ctx_addr_distance,
upper_32_bits(adev->vm_manager.max_pfn - 1));
}
+
+ if (amdgpu_trap_is_supported(adev))
+ gfxhub_v1_0_program_kernel_trap_tba_tma(adev);
}
static void gfxhub_v1_0_program_invalidation(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c
index e89813590539..76c2b6b74903 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c
@@ -22,6 +22,7 @@
*/
#include "amdgpu.h"
#include "amdgpu_ip.h"
+#include "amdgpu_trap.h"
#include "amdgpu_xcp.h"
#include "gfxhub_v1_2.h"
#include "gfxhub_v1_1.h"
@@ -328,6 +329,55 @@ gfxhub_v1_2_per_process_xnack_support(struct amdgpu_device *adev)
return !!adev->aid_mask;
}
+static void
+gfxhub_v1_2_program_kernel_trap_tba_tma(struct amdgpu_device *adev,
+ uint32_t xcc_mask)
+{
+ u64 tba_addr, tma_addr;
+ unsigned int vmid;
+ int xcc;
+
+ if (!amdgpu_trap_is_supported(adev) ||
+ !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
+ return;
+
+ amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
+
+ WARN_ON(!IS_ALIGNED(tba_addr, 256));
+ WARN_ON(!IS_ALIGNED(tma_addr, 256));
+
+ mutex_lock(&adev->srbm_mutex);
+
+ for_each_inst(xcc, xcc_mask) {
+ for (vmid = 1;
+ vmid < adev->vm_manager.first_kfd_vmid;
+ vmid++) {
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0,
+ vmid, xcc);
+
+ WREG32_SOC15(GC, GET_INST(GC, xcc),
+ regSQ_SHADER_TBA_LO,
+ lower_32_bits(tba_addr >> 8));
+ WREG32_SOC15(GC, GET_INST(GC, xcc),
+ regSQ_SHADER_TBA_HI,
+ upper_32_bits(tba_addr >> 8) &
+ SQ_SHADER_TBA_HI__ADDR_HI_MASK);
+ WREG32_SOC15(GC, GET_INST(GC, xcc),
+ regSQ_SHADER_TMA_LO,
+ lower_32_bits(tma_addr >> 8));
+ WREG32_SOC15(GC, GET_INST(GC, xcc),
+ regSQ_SHADER_TMA_HI,
+ upper_32_bits(tma_addr >> 8) &
+ SQ_SHADER_TMA_HI__ADDR_HI_MASK);
+ }
+ }
+
+ for_each_inst(xcc, xcc_mask)
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, xcc);
+
+ mutex_unlock(&adev->srbm_mutex);
+}
+
static void gfxhub_v1_2_xcc_setup_vmid_config(struct amdgpu_device *adev,
uint32_t xcc_mask)
{
@@ -398,6 +448,9 @@ static void gfxhub_v1_2_xcc_setup_vmid_config(struct amdgpu_device *adev,
upper_32_bits(adev->vm_manager.max_pfn - 1));
}
}
+
+ if (amdgpu_trap_is_supported(adev))
+ gfxhub_v1_2_program_kernel_trap_tba_tma(adev, xcc_mask);
}
static void gfxhub_v1_2_xcc_program_invalidation(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
index 04778f9f95bd..da4e1fa34d8d 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
@@ -22,6 +22,7 @@
*/
#include "amdgpu.h"
+#include "amdgpu_trap.h"
#include "gfxhub_v2_0.h"
#include "gc/gc_10_1_0_offset.h"
@@ -285,6 +286,40 @@ static void gfxhub_v2_0_disable_identity_aperture(struct amdgpu_device *adev)
}
+static void
+gfxhub_v2_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev)
+{
+ u64 tba_addr, tma_addr;
+ unsigned int vmid;
+
+ if (!amdgpu_trap_is_supported(adev) ||
+ !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
+ return;
+
+ amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
+
+ WARN_ON(!IS_ALIGNED(tba_addr, 256));
+ WARN_ON(!IS_ALIGNED(tma_addr, 256));
+
+ mutex_lock(&adev->srbm_mutex);
+ for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) {
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0);
+ WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO,
+ lower_32_bits(tba_addr >> 8));
+ WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI,
+ (upper_32_bits(tba_addr >> 8) &
+ SQ_SHADER_TBA_HI__ADDR_HI_MASK) |
+ (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT));
+ WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO,
+ lower_32_bits(tma_addr >> 8));
+ WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI,
+ upper_32_bits(tma_addr >> 8) &
+ SQ_SHADER_TMA_HI__ADDR_HI_MASK);
+ }
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
+ mutex_unlock(&adev->srbm_mutex);
+}
+
static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev)
{
struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
@@ -332,6 +367,9 @@ static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev)
}
hub->vm_cntx_cntl = tmp;
+
+ if (amdgpu_trap_is_supported(adev))
+ gfxhub_v2_0_program_kernel_trap_tba_tma(adev);
}
static void gfxhub_v2_0_program_invalidation(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c
index 6c77c17878f8..c0ff251bcbff 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c
@@ -22,6 +22,7 @@
*/
#include "amdgpu.h"
+#include "amdgpu_trap.h"
#include "gfxhub_v2_1.h"
#include "gc/gc_10_3_0_offset.h"
@@ -297,6 +298,44 @@ static void gfxhub_v2_1_disable_identity_aperture(struct amdgpu_device *adev)
}
+static void
+gfxhub_v2_1_program_kernel_trap_tba_tma(struct amdgpu_device *adev)
+{
+ u64 tba_addr, tma_addr;
+ unsigned int vmid;
+
+ if (!amdgpu_trap_is_supported(adev) ||
+ !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
+ return;
+
+ amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
+
+ WARN_ON(!IS_ALIGNED(tba_addr, 256));
+ WARN_ON(!IS_ALIGNED(tma_addr, 256));
+
+ mutex_lock(&adev->srbm_mutex);
+
+ for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) {
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0);
+
+ WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO,
+ lower_32_bits(tba_addr >> 8));
+ WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI,
+ (upper_32_bits(tba_addr >> 8) &
+ SQ_SHADER_TBA_HI__ADDR_HI_MASK) |
+ (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT));
+ WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO,
+ lower_32_bits(tma_addr >> 8));
+ WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI,
+ upper_32_bits(tma_addr >> 8) &
+ SQ_SHADER_TMA_HI__ADDR_HI_MASK);
+ }
+
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
+
+ mutex_unlock(&adev->srbm_mutex);
+}
+
static void gfxhub_v2_1_setup_vmid_config(struct amdgpu_device *adev)
{
struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
@@ -344,6 +383,9 @@ static void gfxhub_v2_1_setup_vmid_config(struct amdgpu_device *adev)
}
hub->vm_cntx_cntl = tmp;
+
+ if (amdgpu_trap_is_supported(adev))
+ gfxhub_v2_1_program_kernel_trap_tba_tma(adev);
}
static void gfxhub_v2_1_program_invalidation(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c
index e2b9fd0c8af0..1d9d4d41c42c 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c
@@ -22,6 +22,7 @@
*/
#include "amdgpu.h"
+#include "amdgpu_trap.h"
#include "gfxhub_v3_0.h"
#include "gc/gc_11_0_0_offset.h"
@@ -292,6 +293,48 @@ static void gfxhub_v3_0_disable_identity_aperture(struct amdgpu_device *adev)
}
+static void
+gfxhub_v3_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev)
+{
+ u64 tba_addr, tma_addr;
+ u32 tba_hi;
+ unsigned int vmid;
+
+ if (!amdgpu_trap_is_supported(adev) ||
+ !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
+ return;
+
+ amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
+
+ WARN_ON(!IS_ALIGNED(tba_addr, 256));
+ WARN_ON(!IS_ALIGNED(tma_addr, 256));
+
+ tba_hi = upper_32_bits(tba_addr >> 8) &
+ SQ_SHADER_TBA_HI__ADDR_HI_MASK;
+
+ if (!amdgpu_trap_has_dbg_wa(adev))
+ tba_hi |= 1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT;
+
+ mutex_lock(&adev->srbm_mutex);
+
+ for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) {
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0);
+
+ WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO,
+ lower_32_bits(tba_addr >> 8));
+ WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, tba_hi);
+ WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO,
+ lower_32_bits(tma_addr >> 8));
+ WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI,
+ upper_32_bits(tma_addr >> 8) &
+ SQ_SHADER_TMA_HI__ADDR_HI_MASK);
+ }
+
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
+
+ mutex_unlock(&adev->srbm_mutex);
+}
+
static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev)
{
struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
@@ -339,6 +382,9 @@ static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev)
}
hub->vm_cntx_cntl = tmp;
+
+ if (amdgpu_trap_is_supported(adev))
+ gfxhub_v3_0_program_kernel_trap_tba_tma(adev);
}
static void gfxhub_v3_0_program_invalidation(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c
index fa8b966b7ea1..7f2a0b21cd4f 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c
@@ -23,6 +23,7 @@
#include "amdgpu.h"
#include "gfxhub_v3_0_3.h"
+#include "amdgpu_trap.h"
#include "gc/gc_11_0_3_offset.h"
#include "gc/gc_11_0_3_sh_mask.h"
@@ -297,6 +298,48 @@ static void gfxhub_v3_0_3_disable_identity_aperture(struct amdgpu_device *adev)
}
+static void
+gfxhub_v3_0_3_program_kernel_trap_tba_tma(struct amdgpu_device *adev)
+{
+ u64 tba_addr, tma_addr;
+ u32 tba_hi;
+ unsigned int vmid;
+
+ if (!amdgpu_trap_is_supported(adev) ||
+ !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
+ return;
+
+ amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
+
+ WARN_ON(!IS_ALIGNED(tba_addr, 256));
+ WARN_ON(!IS_ALIGNED(tma_addr, 256));
+
+ tba_hi = upper_32_bits(tba_addr >> 8) &
+ SQ_SHADER_TBA_HI__ADDR_HI_MASK;
+
+ if (!amdgpu_trap_has_dbg_wa(adev))
+ tba_hi |= 1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT;
+
+ mutex_lock(&adev->srbm_mutex);
+
+ for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) {
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0);
+
+ WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO,
+ lower_32_bits(tba_addr >> 8));
+ WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, tba_hi);
+ WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO,
+ lower_32_bits(tma_addr >> 8));
+ WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI,
+ upper_32_bits(tma_addr >> 8) &
+ SQ_SHADER_TMA_HI__ADDR_HI_MASK);
+ }
+
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
+
+ mutex_unlock(&adev->srbm_mutex);
+}
+
static void gfxhub_v3_0_3_setup_vmid_config(struct amdgpu_device *adev)
{
struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
@@ -344,6 +387,9 @@ static void gfxhub_v3_0_3_setup_vmid_config(struct amdgpu_device *adev)
}
hub->vm_cntx_cntl = tmp;
+
+ if (amdgpu_trap_is_supported(adev))
+ gfxhub_v3_0_3_program_kernel_trap_tba_tma(adev);
}
static void gfxhub_v3_0_3_program_invalidation(struct amdgpu_device *adev)
--
2.34.1
^ permalink raw reply related [flat|nested] 16+ messages in thread* Re: [PATCH 6/7] drm/amdgpu: Program trap TBA/TMA for kernel-managed VMIDs
2026-10-08 7:05 ` [PATCH 6/7] drm/amdgpu: Program trap TBA/TMA for kernel-managed VMIDs Srinivasan Shanmugam
@ 2026-10-12 15:05 ` Christian König
0 siblings, 0 replies; 16+ messages in thread
From: Christian König @ 2026-10-12 15:05 UTC (permalink / raw)
To: Srinivasan Shanmugam, Alex Deucher
Cc: amd-gfx, Lijo Lazar, Felix Kuehling, Lancelot Six, James Zhu,
Samuel Pitoiset, Natalie Vock, Timur Kristóf,
Pierre-Eric Pelloux-Prayer
On 10/8/26 09:05, Srinivasan Shanmugam wrote:
> User queues get their first-level TBA and TMA from MES/KIQ when the
> queue is created. Kernel-managed VMIDs do not use this path, so program
> their trap registers from the gfxhub setup path.
>
> Get the reserved first-level TBA and TMA virtual addresses and program
> them for each kernel-managed VMID.
>
> The trap registers use the SRBM selector, so keep the programming under
> srbm_mutex and restore the VMID selector after the registers are
> updated.
>
> Trap enablement differs across GPU generations. On GFX9, enable traps
> for kernel-managed VMIDs through the debug trap configuration. On
> GFX9.4.3, enable the per-VMID trap control on each XCC. On generations
> where trap enable is encoded in SQ_SHADER_TBA_HI, program TRAP_EN
> together with the TBA address.
>
> Mask the TBA and TMA address fields before writing the registers. Keep
> TRAP_EN disabled on the affected GFX11 ASICs to preserve the existing
> trap workaround.
>
> On multi-XCC hardware, program the requested XCCs and restore the VMID
> selector on each XCC afterwards.
>
> Keep the first-level trap VA layout inside amdgpu_trap. The gfxhub code
> only needs the final TBA and TMA addresses.
>
> Suggested-by: Christian König <christian.koenig@amd.com>
> Suggested-by: Alex Deucher <alexander.deucher@amd.com>
> Suggested-by: Lijo Lazar <lijo.lazar@amd.com>
> Cc: Felix Kuehling <felix.kuehling@amd.com>
> Cc: Lancelot Six <lancelot.six@amd.com>
> Cc: James Zhu <james.zhu@amd.com>
> Cc: Samuel Pitoiset <hakzsam@gmail.com>
> Cc: Natalie Vock <natalie.vock@gmx.de>
> Cc: Timur Kristóf <timur.kristof@gmail.com>
> Cc: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
> Signed-off-by: Srinivasan Shanmugam <srinivasan.shanmugam@amd.com>
This conflicts a bit with Sunils works since he is currently replacing the VMID ranges with a VMID mask.
I suggest that Sunils work goes in first and then this here is rebased on top.
But it is mostly just rebasing and using different parameters for the function (mask instead of first/last) and then a different for loop.
Lijo or somebody else with more HW background needs to take a look as well I think.
Acked-by: Christian König <christian.koenig@amd.com>
Regards,
Christian.
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c | 9 +++-
> drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h | 5 ++
> drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 29 ++++++++---
> drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c | 16 +++++++
> drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c | 38 +++++++++++++++
> drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c | 38 +++++++++++++++
> drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c | 53 +++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c | 41 ++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c | 53 +++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c | 38 +++++++++++++++
> drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c | 42 ++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c | 46 ++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c | 46 ++++++++++++++++++
> 13 files changed, 446 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
> index 89968b5b7ecb..6e7dfceeee10 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
> @@ -63,7 +63,14 @@ static inline uint64_t amdgpu_trap_tba_vaddr(struct amdgpu_device *adev)
> return addr;
> }
>
> -static inline bool amdgpu_trap_is_supported(struct amdgpu_device *adev)
> +void amdgpu_trap_get_l1_addr(struct amdgpu_device *adev,
> + u64 *tba_addr, u64 *tma_addr)
> +{
> + *tba_addr = amdgpu_trap_tba_vaddr(adev);
> + *tma_addr = *tba_addr + AMDGPU_TRAP_TMA_OFFSET;
> +}
> +
> +bool amdgpu_trap_is_supported(struct amdgpu_device *adev)
> {
> uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h
> index c78c60e3d18d..523f65cb3d3d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h
> @@ -76,6 +76,11 @@ struct amdgpu_trap_usr_addr {
> int amdgpu_trap_init(struct amdgpu_device *adev);
> void amdgpu_trap_fini(struct amdgpu_device *adev);
>
> +bool amdgpu_trap_is_supported(struct amdgpu_device *adev);
> +
> +void amdgpu_trap_get_l1_addr(struct amdgpu_device *adev,
> + u64 *tba_addr, u64 *tma_addr);
> +
> int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> struct amdgpu_trap_obj **trap_obj);
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> index 1678015ce5a9..07e249500254 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> @@ -29,6 +29,7 @@
>
> #include "amdgpu.h"
> #include "amdgpu_gfx.h"
> +#include "amdgpu_trap.h"
> #include "soc15.h"
> #include "soc15d.h"
> #include "amdgpu_atomfirmware.h"
> @@ -2568,21 +2569,26 @@ static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
> }
>
> static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev,
> - uint32_t first_vmid,
> - uint32_t last_vmid)
> + u32 first_vmid,
> + uint32_t last_vmid)
> {
> uint32_t data;
> - uint32_t trap_config_vmid_mask = 0;
> + u32 trap_config_vmid_mask;
> int i;
>
> - /* Calculate trap config vmid mask */
> + data = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG));
> + trap_config_vmid_mask =
> + REG_GET_FIELD(data, SPI_GDBG_TRAP_CONFIG, VMID_SEL);
> +
> + /* Add the requested VMIDs to the existing trap configuration. */
> for (i = first_vmid; i < last_vmid; i++)
> trap_config_vmid_mask |= (1 << i);
>
> - data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG,
> - VMID_SEL, trap_config_vmid_mask);
> data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG,
> - TRAP_EN, 1);
> + VMID_SEL, trap_config_vmid_mask);
> + data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG,
> + TRAP_EN, 1);
> +
> WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data);
> WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0);
>
> @@ -4051,6 +4057,15 @@ static int gfx_v9_0_hw_init(struct amdgpu_ip_block *ip_block)
> if (r)
> return r;
>
> + if (amdgpu_trap_is_supported(adev)) {
> + if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
> + gfx_v9_4_2_debug_trap_config_init(adev, 1,
> + adev->vm_manager.first_kfd_vmid);
> + else
> + gfx_v9_0_debug_trap_config_init(adev, 1,
> + adev->vm_manager.first_kfd_vmid);
> + }
> +
> if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) &&
> !amdgpu_sriov_vf(adev))
> gfx_v9_4_2_set_power_brake_sequence(adev);
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
> index 01ed1701c781..69a1e64620cd 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
> @@ -25,6 +25,7 @@
> #include "amdgpu.h"
> #include "amdgpu_ip.h"
> #include "amdgpu_gfx.h"
> +#include "amdgpu_trap.h"
> #include "soc15.h"
> #include "soc15d.h"
> #include "soc15_common.h"
> @@ -1173,6 +1174,21 @@ static void gfx_v9_4_3_xcc_init_compute_vmid(struct amdgpu_device *adev,
> SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
>
> mutex_lock(&adev->srbm_mutex);
> +
> + if (amdgpu_trap_is_supported(adev)) {
> + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) {
> + soc15_grbm_select(adev, 0, 0, 0, i,
> + GET_INST(GC, xcc_id));
> +
> + data = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
> + regSPI_GDBG_PER_VMID_CNTL);
> + data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL,
> + TRAP_EN, 1);
> + WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id),
> + regSPI_GDBG_PER_VMID_CNTL, data);
> + }
> + }
> +
> for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
> soc15_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id));
> /* CP and shaders */
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c
> index 8be912d50601..e635d83488af 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c
> @@ -22,6 +22,7 @@
> */
>
> #include "amdgpu.h"
> +#include "amdgpu_trap.h"
> #include "gfxhub_v11_5_0.h"
>
> #include "gc/gc_11_5_0_offset.h"
> @@ -295,6 +296,40 @@ static void gfxhub_v11_5_0_disable_identity_aperture(struct amdgpu_device *adev)
>
> }
>
> +static void
> +gfxhub_v11_5_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev)
> +{
> + u64 tba_addr, tma_addr;
> + unsigned int vmid;
> +
> + if (!amdgpu_trap_is_supported(adev) ||
> + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
> + return;
> +
> + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
> +
> + WARN_ON(!IS_ALIGNED(tba_addr, 256));
> + WARN_ON(!IS_ALIGNED(tma_addr, 256));
> +
> + mutex_lock(&adev->srbm_mutex);
> + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) {
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0);
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO,
> + lower_32_bits(tba_addr >> 8));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI,
> + (upper_32_bits(tba_addr >> 8) &
> + SQ_SHADER_TBA_HI__ADDR_HI_MASK) |
> + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO,
> + lower_32_bits(tma_addr >> 8));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI,
> + upper_32_bits(tma_addr >> 8) &
> + SQ_SHADER_TMA_HI__ADDR_HI_MASK);
> + }
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
> + mutex_unlock(&adev->srbm_mutex);
> +}
> +
> static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev)
> {
> struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
> @@ -342,6 +377,9 @@ static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev)
> }
>
> hub->vm_cntx_cntl = tmp;
> +
> + if (amdgpu_trap_is_supported(adev))
> + gfxhub_v11_5_0_program_kernel_trap_tba_tma(adev);
> }
>
> static void gfxhub_v11_5_0_program_invalidation(struct amdgpu_device *adev)
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c
> index 4144c1c2ee13..dc7fde6962ca 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c
> @@ -22,6 +22,7 @@
> */
>
> #include "amdgpu.h"
> +#include "amdgpu_trap.h"
> #include "gfxhub_v12_0.h"
>
> #include "gc/gc_12_0_0_offset.h"
> @@ -300,6 +301,40 @@ static void gfxhub_v12_0_disable_identity_aperture(struct amdgpu_device *adev)
>
> }
>
> +static void
> +gfxhub_v12_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev)
> +{
> + u64 tba_addr, tma_addr;
> + unsigned int vmid;
> +
> + if (!amdgpu_trap_is_supported(adev) ||
> + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
> + return;
> +
> + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
> +
> + WARN_ON(!IS_ALIGNED(tba_addr, 256));
> + WARN_ON(!IS_ALIGNED(tma_addr, 256));
> +
> + mutex_lock(&adev->srbm_mutex);
> + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) {
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0);
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO,
> + lower_32_bits(tba_addr >> 8));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI,
> + (upper_32_bits(tba_addr >> 8) &
> + SQ_SHADER_TBA_HI__ADDR_HI_MASK) |
> + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO,
> + lower_32_bits(tma_addr >> 8));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI,
> + upper_32_bits(tma_addr >> 8) &
> + SQ_SHADER_TMA_HI__ADDR_HI_MASK);
> + }
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
> + mutex_unlock(&adev->srbm_mutex);
> +}
> +
> static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev)
> {
> struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
> @@ -347,6 +382,9 @@ static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev)
> }
>
> hub->vm_cntx_cntl = tmp;
> +
> + if (amdgpu_trap_is_supported(adev))
> + gfxhub_v12_0_program_kernel_trap_tba_tma(adev);
> }
>
> static void gfxhub_v12_0_program_invalidation(struct amdgpu_device *adev)
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
> index b13967a1c8fb..78088e72606c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
> @@ -21,6 +21,7 @@
> *
> */
> #include "amdgpu.h"
> +#include "amdgpu_trap.h"
> #include "amdgpu_xcp.h"
> #include "gfxhub_v12_1.h"
>
> @@ -392,6 +393,55 @@ static void gfxhub_v12_1_xcc_disable_identity_aperture(struct amdgpu_device *ade
> }
> }
>
> +static void
> +gfxhub_v12_1_program_kernel_trap_tba_tma(struct amdgpu_device *adev,
> + uint32_t xcc_mask)
> +{
> + u64 tba_addr, tma_addr;
> + unsigned int vmid;
> + int xcc;
> +
> + if (!amdgpu_trap_is_supported(adev) ||
> + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
> + return;
> +
> + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
> +
> + WARN_ON(!IS_ALIGNED(tba_addr, 256));
> + WARN_ON(!IS_ALIGNED(tma_addr, 256));
> +
> + mutex_lock(&adev->srbm_mutex);
> + for_each_inst(xcc, xcc_mask) {
> + for (vmid = 1;
> + vmid < adev->vm_manager.first_kfd_vmid;
> + vmid++) {
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0,
> + vmid, xcc);
> + WREG32_SOC15(GC, GET_INST(GC, xcc),
> + regSQ_SHADER_TBA_LO,
> + lower_32_bits(tba_addr >> 8));
> + WREG32_SOC15(GC, GET_INST(GC, xcc),
> + regSQ_SHADER_TBA_HI,
> + (upper_32_bits(tba_addr >> 8) &
> + SQ_SHADER_TBA_HI__ADDR_HI_MASK) |
> + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT));
> + WREG32_SOC15(GC, GET_INST(GC, xcc),
> + regSQ_SHADER_TMA_LO,
> + lower_32_bits(tma_addr >> 8));
> + WREG32_SOC15(GC, GET_INST(GC, xcc),
> + regSQ_SHADER_TMA_HI,
> + upper_32_bits(tma_addr >> 8) &
> + SQ_SHADER_TMA_HI__ADDR_HI_MASK);
> + }
> + }
> +
> + /* Reset the VMID selector on every XCC instance. */
> + for_each_inst(xcc, xcc_mask)
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, xcc);
> +
> + mutex_unlock(&adev->srbm_mutex);
> +}
> +
> static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev,
> uint32_t xcc_mask)
> {
> @@ -457,6 +507,9 @@ static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev,
>
> hub->vm_cntx_cntl = tmp;
> }
> +
> + if (amdgpu_trap_is_supported(adev))
> + gfxhub_v12_1_program_kernel_trap_tba_tma(adev, xcc_mask);
> }
>
> static void gfxhub_v12_1_xcc_program_invalidation(struct amdgpu_device *adev,
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
> index c35856180c3c..dbd993ae1d0e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
> @@ -21,6 +21,7 @@
> *
> */
> #include "amdgpu.h"
> +#include "amdgpu_trap.h"
> #include "gfxhub_v1_0.h"
> #include "gfxhub_v1_1.h"
>
> @@ -252,6 +253,43 @@ static void gfxhub_v1_0_disable_identity_aperture(struct amdgpu_device *adev)
>
> }
>
> +static void
> +gfxhub_v1_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev)
> +{
> + u64 tba_addr, tma_addr;
> + unsigned int vmid;
> +
> + if (!amdgpu_trap_is_supported(adev) ||
> + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
> + return;
> +
> + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
> +
> + WARN_ON(!IS_ALIGNED(tba_addr, 256));
> + WARN_ON(!IS_ALIGNED(tma_addr, 256));
> +
> + mutex_lock(&adev->srbm_mutex);
> +
> + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) {
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0);
> +
> + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO,
> + lower_32_bits(tba_addr >> 8));
> + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI,
> + upper_32_bits(tba_addr >> 8) &
> + SQ_SHADER_TBA_HI__ADDR_HI_MASK);
> + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO,
> + lower_32_bits(tma_addr >> 8));
> + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI,
> + upper_32_bits(tma_addr >> 8) &
> + SQ_SHADER_TMA_HI__ADDR_HI_MASK);
> + }
> +
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
> +
> + mutex_unlock(&adev->srbm_mutex);
> +}
> +
> static void gfxhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
> {
> struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
> @@ -310,6 +348,9 @@ static void gfxhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
> i * hub->ctx_addr_distance,
> upper_32_bits(adev->vm_manager.max_pfn - 1));
> }
> +
> + if (amdgpu_trap_is_supported(adev))
> + gfxhub_v1_0_program_kernel_trap_tba_tma(adev);
> }
>
> static void gfxhub_v1_0_program_invalidation(struct amdgpu_device *adev)
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c
> index e89813590539..76c2b6b74903 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c
> @@ -22,6 +22,7 @@
> */
> #include "amdgpu.h"
> #include "amdgpu_ip.h"
> +#include "amdgpu_trap.h"
> #include "amdgpu_xcp.h"
> #include "gfxhub_v1_2.h"
> #include "gfxhub_v1_1.h"
> @@ -328,6 +329,55 @@ gfxhub_v1_2_per_process_xnack_support(struct amdgpu_device *adev)
> return !!adev->aid_mask;
> }
>
> +static void
> +gfxhub_v1_2_program_kernel_trap_tba_tma(struct amdgpu_device *adev,
> + uint32_t xcc_mask)
> +{
> + u64 tba_addr, tma_addr;
> + unsigned int vmid;
> + int xcc;
> +
> + if (!amdgpu_trap_is_supported(adev) ||
> + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
> + return;
> +
> + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
> +
> + WARN_ON(!IS_ALIGNED(tba_addr, 256));
> + WARN_ON(!IS_ALIGNED(tma_addr, 256));
> +
> + mutex_lock(&adev->srbm_mutex);
> +
> + for_each_inst(xcc, xcc_mask) {
> + for (vmid = 1;
> + vmid < adev->vm_manager.first_kfd_vmid;
> + vmid++) {
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0,
> + vmid, xcc);
> +
> + WREG32_SOC15(GC, GET_INST(GC, xcc),
> + regSQ_SHADER_TBA_LO,
> + lower_32_bits(tba_addr >> 8));
> + WREG32_SOC15(GC, GET_INST(GC, xcc),
> + regSQ_SHADER_TBA_HI,
> + upper_32_bits(tba_addr >> 8) &
> + SQ_SHADER_TBA_HI__ADDR_HI_MASK);
> + WREG32_SOC15(GC, GET_INST(GC, xcc),
> + regSQ_SHADER_TMA_LO,
> + lower_32_bits(tma_addr >> 8));
> + WREG32_SOC15(GC, GET_INST(GC, xcc),
> + regSQ_SHADER_TMA_HI,
> + upper_32_bits(tma_addr >> 8) &
> + SQ_SHADER_TMA_HI__ADDR_HI_MASK);
> + }
> + }
> +
> + for_each_inst(xcc, xcc_mask)
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, xcc);
> +
> + mutex_unlock(&adev->srbm_mutex);
> +}
> +
> static void gfxhub_v1_2_xcc_setup_vmid_config(struct amdgpu_device *adev,
> uint32_t xcc_mask)
> {
> @@ -398,6 +448,9 @@ static void gfxhub_v1_2_xcc_setup_vmid_config(struct amdgpu_device *adev,
> upper_32_bits(adev->vm_manager.max_pfn - 1));
> }
> }
> +
> + if (amdgpu_trap_is_supported(adev))
> + gfxhub_v1_2_program_kernel_trap_tba_tma(adev, xcc_mask);
> }
>
> static void gfxhub_v1_2_xcc_program_invalidation(struct amdgpu_device *adev,
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
> index 04778f9f95bd..da4e1fa34d8d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
> @@ -22,6 +22,7 @@
> */
>
> #include "amdgpu.h"
> +#include "amdgpu_trap.h"
> #include "gfxhub_v2_0.h"
>
> #include "gc/gc_10_1_0_offset.h"
> @@ -285,6 +286,40 @@ static void gfxhub_v2_0_disable_identity_aperture(struct amdgpu_device *adev)
>
> }
>
> +static void
> +gfxhub_v2_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev)
> +{
> + u64 tba_addr, tma_addr;
> + unsigned int vmid;
> +
> + if (!amdgpu_trap_is_supported(adev) ||
> + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
> + return;
> +
> + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
> +
> + WARN_ON(!IS_ALIGNED(tba_addr, 256));
> + WARN_ON(!IS_ALIGNED(tma_addr, 256));
> +
> + mutex_lock(&adev->srbm_mutex);
> + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) {
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0);
> + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO,
> + lower_32_bits(tba_addr >> 8));
> + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI,
> + (upper_32_bits(tba_addr >> 8) &
> + SQ_SHADER_TBA_HI__ADDR_HI_MASK) |
> + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT));
> + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO,
> + lower_32_bits(tma_addr >> 8));
> + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI,
> + upper_32_bits(tma_addr >> 8) &
> + SQ_SHADER_TMA_HI__ADDR_HI_MASK);
> + }
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
> + mutex_unlock(&adev->srbm_mutex);
> +}
> +
> static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev)
> {
> struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
> @@ -332,6 +367,9 @@ static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev)
> }
>
> hub->vm_cntx_cntl = tmp;
> +
> + if (amdgpu_trap_is_supported(adev))
> + gfxhub_v2_0_program_kernel_trap_tba_tma(adev);
> }
>
> static void gfxhub_v2_0_program_invalidation(struct amdgpu_device *adev)
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c
> index 6c77c17878f8..c0ff251bcbff 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c
> @@ -22,6 +22,7 @@
> */
>
> #include "amdgpu.h"
> +#include "amdgpu_trap.h"
> #include "gfxhub_v2_1.h"
>
> #include "gc/gc_10_3_0_offset.h"
> @@ -297,6 +298,44 @@ static void gfxhub_v2_1_disable_identity_aperture(struct amdgpu_device *adev)
>
> }
>
> +static void
> +gfxhub_v2_1_program_kernel_trap_tba_tma(struct amdgpu_device *adev)
> +{
> + u64 tba_addr, tma_addr;
> + unsigned int vmid;
> +
> + if (!amdgpu_trap_is_supported(adev) ||
> + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
> + return;
> +
> + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
> +
> + WARN_ON(!IS_ALIGNED(tba_addr, 256));
> + WARN_ON(!IS_ALIGNED(tma_addr, 256));
> +
> + mutex_lock(&adev->srbm_mutex);
> +
> + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) {
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0);
> +
> + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO,
> + lower_32_bits(tba_addr >> 8));
> + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI,
> + (upper_32_bits(tba_addr >> 8) &
> + SQ_SHADER_TBA_HI__ADDR_HI_MASK) |
> + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT));
> + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO,
> + lower_32_bits(tma_addr >> 8));
> + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI,
> + upper_32_bits(tma_addr >> 8) &
> + SQ_SHADER_TMA_HI__ADDR_HI_MASK);
> + }
> +
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
> +
> + mutex_unlock(&adev->srbm_mutex);
> +}
> +
> static void gfxhub_v2_1_setup_vmid_config(struct amdgpu_device *adev)
> {
> struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
> @@ -344,6 +383,9 @@ static void gfxhub_v2_1_setup_vmid_config(struct amdgpu_device *adev)
> }
>
> hub->vm_cntx_cntl = tmp;
> +
> + if (amdgpu_trap_is_supported(adev))
> + gfxhub_v2_1_program_kernel_trap_tba_tma(adev);
> }
>
> static void gfxhub_v2_1_program_invalidation(struct amdgpu_device *adev)
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c
> index e2b9fd0c8af0..1d9d4d41c42c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c
> @@ -22,6 +22,7 @@
> */
>
> #include "amdgpu.h"
> +#include "amdgpu_trap.h"
> #include "gfxhub_v3_0.h"
>
> #include "gc/gc_11_0_0_offset.h"
> @@ -292,6 +293,48 @@ static void gfxhub_v3_0_disable_identity_aperture(struct amdgpu_device *adev)
>
> }
>
> +static void
> +gfxhub_v3_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev)
> +{
> + u64 tba_addr, tma_addr;
> + u32 tba_hi;
> + unsigned int vmid;
> +
> + if (!amdgpu_trap_is_supported(adev) ||
> + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
> + return;
> +
> + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
> +
> + WARN_ON(!IS_ALIGNED(tba_addr, 256));
> + WARN_ON(!IS_ALIGNED(tma_addr, 256));
> +
> + tba_hi = upper_32_bits(tba_addr >> 8) &
> + SQ_SHADER_TBA_HI__ADDR_HI_MASK;
> +
> + if (!amdgpu_trap_has_dbg_wa(adev))
> + tba_hi |= 1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT;
> +
> + mutex_lock(&adev->srbm_mutex);
> +
> + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) {
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0);
> +
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO,
> + lower_32_bits(tba_addr >> 8));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, tba_hi);
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO,
> + lower_32_bits(tma_addr >> 8));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI,
> + upper_32_bits(tma_addr >> 8) &
> + SQ_SHADER_TMA_HI__ADDR_HI_MASK);
> + }
> +
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
> +
> + mutex_unlock(&adev->srbm_mutex);
> +}
> +
> static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev)
> {
> struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
> @@ -339,6 +382,9 @@ static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev)
> }
>
> hub->vm_cntx_cntl = tmp;
> +
> + if (amdgpu_trap_is_supported(adev))
> + gfxhub_v3_0_program_kernel_trap_tba_tma(adev);
> }
>
> static void gfxhub_v3_0_program_invalidation(struct amdgpu_device *adev)
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c
> index fa8b966b7ea1..7f2a0b21cd4f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c
> @@ -23,6 +23,7 @@
>
> #include "amdgpu.h"
> #include "gfxhub_v3_0_3.h"
> +#include "amdgpu_trap.h"
>
> #include "gc/gc_11_0_3_offset.h"
> #include "gc/gc_11_0_3_sh_mask.h"
> @@ -297,6 +298,48 @@ static void gfxhub_v3_0_3_disable_identity_aperture(struct amdgpu_device *adev)
>
> }
>
> +static void
> +gfxhub_v3_0_3_program_kernel_trap_tba_tma(struct amdgpu_device *adev)
> +{
> + u64 tba_addr, tma_addr;
> + u32 tba_hi;
> + unsigned int vmid;
> +
> + if (!amdgpu_trap_is_supported(adev) ||
> + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
> + return;
> +
> + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr);
> +
> + WARN_ON(!IS_ALIGNED(tba_addr, 256));
> + WARN_ON(!IS_ALIGNED(tma_addr, 256));
> +
> + tba_hi = upper_32_bits(tba_addr >> 8) &
> + SQ_SHADER_TBA_HI__ADDR_HI_MASK;
> +
> + if (!amdgpu_trap_has_dbg_wa(adev))
> + tba_hi |= 1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT;
> +
> + mutex_lock(&adev->srbm_mutex);
> +
> + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) {
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0);
> +
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO,
> + lower_32_bits(tba_addr >> 8));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, tba_hi);
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO,
> + lower_32_bits(tma_addr >> 8));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI,
> + upper_32_bits(tma_addr >> 8) &
> + SQ_SHADER_TMA_HI__ADDR_HI_MASK);
> + }
> +
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
> +
> + mutex_unlock(&adev->srbm_mutex);
> +}
> +
> static void gfxhub_v3_0_3_setup_vmid_config(struct amdgpu_device *adev)
> {
> struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
> @@ -344,6 +387,9 @@ static void gfxhub_v3_0_3_setup_vmid_config(struct amdgpu_device *adev)
> }
>
> hub->vm_cntx_cntl = tmp;
> +
> + if (amdgpu_trap_is_supported(adev))
> + gfxhub_v3_0_3_program_kernel_trap_tba_tma(adev);
> }
>
> static void gfxhub_v3_0_3_program_invalidation(struct amdgpu_device *adev)
^ permalink raw reply [flat|nested] 16+ messages in thread
* [PATCH 7/7] drm/amdgpu/uapi: Add SET/CLEAR L2 TRAP to VM ioctl
2026-10-08 7:05 [PATCH 0/7] drm/amdgpu: Add CWSR and second-level trap handler support Srinivasan Shanmugam
` (5 preceding siblings ...)
2026-10-08 7:05 ` [PATCH 6/7] drm/amdgpu: Program trap TBA/TMA for kernel-managed VMIDs Srinivasan Shanmugam
@ 2026-10-08 7:05 ` Srinivasan Shanmugam
2026-10-12 15:10 ` Christian König
6 siblings, 1 reply; 16+ messages in thread
From: Srinivasan Shanmugam @ 2026-10-08 7:05 UTC (permalink / raw)
To: Christian König, Alex Deucher
Cc: amd-gfx, Srinivasan Shanmugam, Felix Kuehling, Lancelot Six,
James Zhu, Lijo Lazar, Samuel Pitoiset, Natalie Vock,
Timur Kristóf, Pierre-Eric Pelloux-Prayer
Add SET_L2_TRAP and CLEAR_L2_TRAP operations to the VM ioctl.
The second-level trap handler is set per GPUVM. All shader waves using
the same GPUVM use the same second-level trap handler, regardless of the
queue type.
SET_L2_TRAP sets the userspace TBA and TMA addresses for the GPUVM. The
kernel checks that both address ranges are mapped in the GPUVM before
using them.
CLEAR_L2_TRAP clears the second-level TBA and TMA addresses from the
first-level trap descriptor.
Userspace must keep the TBA and TMA mappings valid after SET_L2_TRAP.
The mappings can be removed after CLEAR_L2_TRAP completes successfully.
Suggested-by: Christian König <christian.koenig@amd.com>
Suggested-by: Alex Deucher <alexander.deucher@amd.com>
Cc: Felix Kuehling <felix.kuehling@amd.com>
Cc: Lancelot Six <lancelot.six@amd.com>
Cc: James Zhu <james.zhu@amd.com>
Cc: Lijo Lazar <lijo.lazar@amd.com>
Cc: Samuel Pitoiset <hakzsam@gmail.com>
Cc: Natalie Vock <natalie.vock@gmx.de>
Cc: Timur Kristóf <timur.kristof@gmail.com>
Cc: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
Signed-off-by: Srinivasan Shanmugam <srinivasan.shanmugam@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 29 ++++++++++++
include/uapi/drm/amdgpu_drm.h | 64 ++++++++++++++++++++++++++
2 files changed, 93 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index de4071db94f5..d82834004bba 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -3008,6 +3008,35 @@ int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
case AMDGPU_VM_OP_UNRESERVE_VMID:
amdgpu_vmid_free_reserved(adev, vm, AMDGPU_GFXHUB(0));
break;
+ case AMDGPU_VM_OP_SET_L2_TRAP:
+ if (args->in.l2trap._pad || args->in.l2trap._pad2)
+ return -EINVAL;
+
+ /*
+ * Install or replace the per-VM second-level trap handler.
+ * Applies to all shader waves using this GPUVM regardless of
+ * queue type.
+ */
+ if (!vm->trap_obj)
+ return -EOPNOTSUPP;
+
+ return amdgpu_trap_vm_set_l2_trap(adev, vm,
+ vm->trap_obj,
+ args->in.l2trap.tba_va,
+ args->in.l2trap.tba_sz,
+ args->in.l2trap.tma_va,
+ args->in.l2trap.tma_sz);
+ case AMDGPU_VM_OP_CLEAR_L2_TRAP:
+ if (memchr_inv(&args->in.l2trap, 0,
+ sizeof(args->in.l2trap)))
+ return -EINVAL;
+
+ /* Disable the per-VM second-level trap handler. */
+ if (!vm->trap_obj)
+ return -EOPNOTSUPP;
+
+ return amdgpu_trap_vm_clear_l2_trap(adev, vm,
+ vm->trap_obj);
default:
return -EINVAL;
}
diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
index 9317018330ec..6c86d5731ca5 100644
--- a/include/uapi/drm/amdgpu_drm.h
+++ b/include/uapi/drm/amdgpu_drm.h
@@ -614,10 +614,74 @@ struct drm_amdgpu_userq_wait {
#define AMDGPU_VM_OP_RESERVE_VMID 1
#define AMDGPU_VM_OP_UNRESERVE_VMID 2
+/**
+ * AMDGPU_VM_OP_SET_L2_TRAP - install or replace the second-level trap
+ * handler for this GPUVM.
+ *
+ * The second-level trap handler is a per-GPUVM setting. All shader waves
+ * using this GPUVM use the same handler regardless of queue type.
+ *
+ * TBA is the GPU virtual address of the second-level trap handler code.
+ * TMA is the GPU virtual address of the second-level trap handler scratch
+ * memory. Each range must be non-empty and contained within a single mapping
+ * in this GPUVM.
+ *
+ * This operation may block until outstanding GPUVM work completes.
+ *
+ * Userspace must keep the TBA and TMA mappings valid while the second-level
+ * trap handler is installed.
+ *
+ * Returns:
+ * 0 on success;
+ * -EOPNOTSUPP if the first-level CWSR handler is unavailable;
+ * -EINVAL if TBA/TMA is invalid or not mapped in this GPUVM;
+ * negative errno if waiting for GPUVM work or reserving the GPUVM fails.
+ */
+#define AMDGPU_VM_OP_SET_L2_TRAP 3
+
+/**
+ * AMDGPU_VM_OP_CLEAR_L2_TRAP - disable the second-level trap handler.
+ *
+ * This operation may block until outstanding GPUVM work completes.
+ *
+ * The l2trap input members must be zero for this operation.
+ *
+ * After this operation succeeds, userspace may remove the second-level
+ * TBA and TMA mappings.
+ *
+ * Returns:
+ * 0 on success;
+ * -EOPNOTSUPP if the first-level CWSR handler is unavailable;
+ * -EINVAL if the GPUVM trap state is invalid;
+ * negative errno if waiting for GPUVM work or reserving the GPUVM fails.
+ */
+#define AMDGPU_VM_OP_CLEAR_L2_TRAP 4
+
struct drm_amdgpu_vm_in {
/** AMDGPU_VM_OP_* */
__u32 op;
__u32 flags;
+ union {
+ struct {
+ /** GPU VA of second-level trap handler code buffer */
+ __u64 tba_va;
+ /** TBA buffer size in bytes */
+ __u32 tba_sz;
+ /** Explicit padding; tma_va starts at offset 24 */
+ __u32 _pad;
+ /** GPU VA of second-level trap handler scratch buffer */
+ __u64 tma_va;
+ /** TMA buffer size in bytes */
+ __u32 tma_sz;
+ /** Padding to fix total union size at 32 bytes */
+ __u32 _pad2;
+ } l2trap;
+ /**
+ * Reserved; keeps the union at a fixed 32-byte size for
+ * future operations.
+ */
+ __u64 reserved[4];
+ };
};
struct drm_amdgpu_vm_out {
--
2.34.1
^ permalink raw reply related [flat|nested] 16+ messages in thread* Re: [PATCH 7/7] drm/amdgpu/uapi: Add SET/CLEAR L2 TRAP to VM ioctl
2026-10-08 7:05 ` [PATCH 7/7] drm/amdgpu/uapi: Add SET/CLEAR L2 TRAP to VM ioctl Srinivasan Shanmugam
@ 2026-10-12 15:10 ` Christian König
0 siblings, 0 replies; 16+ messages in thread
From: Christian König @ 2026-10-12 15:10 UTC (permalink / raw)
To: Srinivasan Shanmugam, Alex Deucher
Cc: amd-gfx, Felix Kuehling, Lancelot Six, James Zhu, Lijo Lazar,
Samuel Pitoiset, Natalie Vock, Timur Kristóf,
Pierre-Eric Pelloux-Prayer
On 10/8/26 09:05, Srinivasan Shanmugam wrote:
> Add SET_L2_TRAP and CLEAR_L2_TRAP operations to the VM ioctl.
>
> The second-level trap handler is set per GPUVM. All shader waves using
> the same GPUVM use the same second-level trap handler, regardless of the
> queue type.
>
> SET_L2_TRAP sets the userspace TBA and TMA addresses for the GPUVM. The
> kernel checks that both address ranges are mapped in the GPUVM before
> using them.
>
> CLEAR_L2_TRAP clears the second-level TBA and TMA addresses from the
> first-level trap descriptor.
>
> Userspace must keep the TBA and TMA mappings valid after SET_L2_TRAP.
> The mappings can be removed after CLEAR_L2_TRAP completes successfully.
>
> Suggested-by: Christian König <christian.koenig@amd.com>
> Suggested-by: Alex Deucher <alexander.deucher@amd.com>
> Cc: Felix Kuehling <felix.kuehling@amd.com>
> Cc: Lancelot Six <lancelot.six@amd.com>
> Cc: James Zhu <james.zhu@amd.com>
> Cc: Lijo Lazar <lijo.lazar@amd.com>
> Cc: Samuel Pitoiset <hakzsam@gmail.com>
> Cc: Natalie Vock <natalie.vock@gmx.de>
> Cc: Timur Kristóf <timur.kristof@gmail.com>
> Cc: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
> Signed-off-by: Srinivasan Shanmugam <srinivasan.shanmugam@amd.com>
Reviewed-by: Christian König <christian.koenig@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 29 ++++++++++++
> include/uapi/drm/amdgpu_drm.h | 64 ++++++++++++++++++++++++++
> 2 files changed, 93 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> index de4071db94f5..d82834004bba 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -3008,6 +3008,35 @@ int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
> case AMDGPU_VM_OP_UNRESERVE_VMID:
> amdgpu_vmid_free_reserved(adev, vm, AMDGPU_GFXHUB(0));
> break;
> + case AMDGPU_VM_OP_SET_L2_TRAP:
> + if (args->in.l2trap._pad || args->in.l2trap._pad2)
> + return -EINVAL;
> +
> + /*
> + * Install or replace the per-VM second-level trap handler.
> + * Applies to all shader waves using this GPUVM regardless of
> + * queue type.
> + */
> + if (!vm->trap_obj)
> + return -EOPNOTSUPP;
> +
> + return amdgpu_trap_vm_set_l2_trap(adev, vm,
> + vm->trap_obj,
> + args->in.l2trap.tba_va,
> + args->in.l2trap.tba_sz,
> + args->in.l2trap.tma_va,
> + args->in.l2trap.tma_sz);
> + case AMDGPU_VM_OP_CLEAR_L2_TRAP:
> + if (memchr_inv(&args->in.l2trap, 0,
> + sizeof(args->in.l2trap)))
> + return -EINVAL;
> +
> + /* Disable the per-VM second-level trap handler. */
> + if (!vm->trap_obj)
> + return -EOPNOTSUPP;
> +
> + return amdgpu_trap_vm_clear_l2_trap(adev, vm,
> + vm->trap_obj);
> default:
> return -EINVAL;
> }
> diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
> index 9317018330ec..6c86d5731ca5 100644
> --- a/include/uapi/drm/amdgpu_drm.h
> +++ b/include/uapi/drm/amdgpu_drm.h
> @@ -614,10 +614,74 @@ struct drm_amdgpu_userq_wait {
> #define AMDGPU_VM_OP_RESERVE_VMID 1
> #define AMDGPU_VM_OP_UNRESERVE_VMID 2
>
> +/**
> + * AMDGPU_VM_OP_SET_L2_TRAP - install or replace the second-level trap
> + * handler for this GPUVM.
> + *
> + * The second-level trap handler is a per-GPUVM setting. All shader waves
> + * using this GPUVM use the same handler regardless of queue type.
> + *
> + * TBA is the GPU virtual address of the second-level trap handler code.
> + * TMA is the GPU virtual address of the second-level trap handler scratch
> + * memory. Each range must be non-empty and contained within a single mapping
> + * in this GPUVM.
> + *
> + * This operation may block until outstanding GPUVM work completes.
> + *
> + * Userspace must keep the TBA and TMA mappings valid while the second-level
> + * trap handler is installed.
> + *
> + * Returns:
> + * 0 on success;
> + * -EOPNOTSUPP if the first-level CWSR handler is unavailable;
> + * -EINVAL if TBA/TMA is invalid or not mapped in this GPUVM;
> + * negative errno if waiting for GPUVM work or reserving the GPUVM fails.
> + */
> +#define AMDGPU_VM_OP_SET_L2_TRAP 3
> +
> +/**
> + * AMDGPU_VM_OP_CLEAR_L2_TRAP - disable the second-level trap handler.
> + *
> + * This operation may block until outstanding GPUVM work completes.
> + *
> + * The l2trap input members must be zero for this operation.
> + *
> + * After this operation succeeds, userspace may remove the second-level
> + * TBA and TMA mappings.
> + *
> + * Returns:
> + * 0 on success;
> + * -EOPNOTSUPP if the first-level CWSR handler is unavailable;
> + * -EINVAL if the GPUVM trap state is invalid;
> + * negative errno if waiting for GPUVM work or reserving the GPUVM fails.
> + */
> +#define AMDGPU_VM_OP_CLEAR_L2_TRAP 4
> +
> struct drm_amdgpu_vm_in {
> /** AMDGPU_VM_OP_* */
> __u32 op;
> __u32 flags;
> + union {
> + struct {
> + /** GPU VA of second-level trap handler code buffer */
> + __u64 tba_va;
> + /** TBA buffer size in bytes */
> + __u32 tba_sz;
> + /** Explicit padding; tma_va starts at offset 24 */
> + __u32 _pad;
> + /** GPU VA of second-level trap handler scratch buffer */
> + __u64 tma_va;
> + /** TMA buffer size in bytes */
> + __u32 tma_sz;
> + /** Padding to fix total union size at 32 bytes */
> + __u32 _pad2;
> + } l2trap;
> + /**
> + * Reserved; keeps the union at a fixed 32-byte size for
> + * future operations.
> + */
> + __u64 reserved[4];
> + };
> };
>
> struct drm_amdgpu_vm_out {
^ permalink raw reply [flat|nested] 16+ messages in thread