[PATCH 01/29] drm/amdgpu: support partition drm devices

[PATCH 01/29] drm/amdgpu: support partition drm devices

[Alex Deucher]

From: James Zhu <James.Zhu@amd.com>

Support partition drm devices on GC_HWIP IP_VERSION(9, 4, 3).

This is a temporary solution and will be superceded.

Signed-off-by: Christian König <christian.koenig@amd.com>
Signed-off-by: James Zhu <James.Zhu@amd.com>
Reviewed-and-tested-by: Philip Yang<Philip.Yang@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h        |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c    | 32 ++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.h    |  2 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c    | 59 +++++++++++++++++++++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h    |  5 ++
 6 files changed, 99 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index bed6d1d09ac2..45c6522ee854 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -108,6 +108,7 @@
 #include "amdgpu_fdinfo.h"
 #include "amdgpu_mca.h"
 #include "amdgpu_ras.h"
+#include "amdgpu_xcp.h"
 
 #define MAX_GPU_INSTANCE		64
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index c2136accd523..40c5845c78df 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -6062,6 +6062,7 @@ void amdgpu_device_halt(struct amdgpu_device *adev)
 	struct pci_dev *pdev = adev->pdev;
 	struct drm_device *ddev = adev_to_drm(adev);
 
+	amdgpu_xcp_dev_unplug(adev);
 	drm_dev_unplug(ddev);
 
 	amdgpu_irq_disable_all(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 562e65ab48fa..4589cb2255a2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -2194,6 +2194,10 @@ static int amdgpu_pci_probe(struct pci_dev *pdev,
 		goto err_pci;
 	}
 
+	ret = amdgpu_xcp_dev_register(adev, ent);
+	if (ret)
+		goto err_pci;
+
 	/*
 	 * 1. don't init fbdev on hw without DCE
 	 * 2. don't init fbdev if there are no connectors
@@ -2266,6 +2270,7 @@ amdgpu_pci_remove(struct pci_dev *pdev)
 	struct drm_device *dev = pci_get_drvdata(pdev);
 	struct amdgpu_device *adev = drm_to_adev(dev);
 
+	amdgpu_xcp_dev_unplug(adev);
 	drm_dev_unplug(dev);
 
 	if (adev->pm.rpm_mode != AMDGPU_RUNPM_NONE) {
@@ -2849,6 +2854,33 @@ static const struct drm_driver amdgpu_kms_driver = {
 	.patchlevel = KMS_DRIVER_PATCHLEVEL,
 };
 
+const struct drm_driver amdgpu_partition_driver = {
+	.driver_features =
+	    DRIVER_GEM | DRIVER_RENDER | DRIVER_SYNCOBJ |
+	    DRIVER_SYNCOBJ_TIMELINE,
+	.open = amdgpu_driver_open_kms,
+	.postclose = amdgpu_driver_postclose_kms,
+	.lastclose = amdgpu_driver_lastclose_kms,
+	.ioctls = amdgpu_ioctls_kms,
+	.num_ioctls = ARRAY_SIZE(amdgpu_ioctls_kms),
+	.dumb_create = amdgpu_mode_dumb_create,
+	.dumb_map_offset = amdgpu_mode_dumb_mmap,
+	.fops = &amdgpu_driver_kms_fops,
+	.release = &amdgpu_driver_release_kms,
+
+	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
+	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
+	.gem_prime_import = amdgpu_gem_prime_import,
+	.gem_prime_mmap = drm_gem_prime_mmap,
+
+	.name = DRIVER_NAME,
+	.desc = DRIVER_DESC,
+	.date = DRIVER_DATE,
+	.major = KMS_DRIVER_MAJOR,
+	.minor = KMS_DRIVER_MINOR,
+	.patchlevel = KMS_DRIVER_PATCHLEVEL,
+};
+
 static struct pci_error_handlers amdgpu_pci_err_handler = {
 	.error_detected	= amdgpu_pci_error_detected,
 	.mmio_enabled	= amdgpu_pci_mmio_enabled,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.h
index 8178323e4bef..5bc2cb661af7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.h
@@ -42,6 +42,8 @@
 #define DRIVER_DESC		"AMD GPU"
 #define DRIVER_DATE		"20150101"
 
+extern const struct drm_driver amdgpu_partition_driver;
+
 long amdgpu_drm_ioctl(struct file *filp,
 		      unsigned int cmd, unsigned long arg);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
index bca226cc4e0b..8b28b18e4291 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
@@ -22,6 +22,9 @@
  */
 #include "amdgpu.h"
 #include "amdgpu_xcp.h"
+#include "amdgpu_drv.h"
+
+#include <drm/drm_drv.h>
 
 static int __amdgpu_xcp_run(struct amdgpu_xcp_mgr *xcp_mgr,
 			    struct amdgpu_xcp_ip *xcp_ip, int xcp_state)
@@ -217,6 +220,31 @@ int amdgpu_xcp_query_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags)
 	return mode;
 }
 
+static int amdgpu_xcp_dev_alloc(struct amdgpu_device *adev)
+{
+	struct drm_device *p_ddev;
+	struct pci_dev *pdev;
+	struct drm_device *ddev;
+	int i;
+
+	pdev = adev->pdev;
+	ddev = adev_to_drm(adev);
+
+	for (i = 0; i < MAX_XCP; i++) {
+		p_ddev = drm_dev_alloc(&amdgpu_partition_driver,
+			&pci_upstream_bridge(pdev)->dev);
+		if (IS_ERR(p_ddev))
+			return PTR_ERR(p_ddev);
+
+		/* Redirect all IOCTLs to the primary device */
+		p_ddev->render->dev = ddev;
+		p_ddev->vma_offset_manager = ddev->vma_offset_manager;
+		adev->xcp_mgr->xcp[i].ddev = p_ddev;
+	}
+
+	return 0;
+}
+
 int amdgpu_xcp_mgr_init(struct amdgpu_device *adev, int init_mode,
 			int init_num_xcps,
 			struct amdgpu_xcp_mgr_funcs *xcp_funcs)
@@ -242,7 +270,7 @@ int amdgpu_xcp_mgr_init(struct amdgpu_device *adev, int init_mode,
 
 	adev->xcp_mgr = xcp_mgr;
 
-	return 0;
+	return amdgpu_xcp_dev_alloc(adev);
 }
 
 int amdgpu_xcp_get_partition(struct amdgpu_xcp_mgr *xcp_mgr,
@@ -278,3 +306,32 @@ int amdgpu_xcp_get_inst_details(struct amdgpu_xcp *xcp,
 
 	return 0;
 }
+
+int amdgpu_xcp_dev_register(struct amdgpu_device *adev,
+			const struct pci_device_id *ent)
+{
+	int i, ret;
+
+	if (!adev->xcp_mgr)
+		return 0;
+
+	for (i = 0; i < MAX_XCP; i++) {
+		ret = drm_dev_register(adev->xcp_mgr->xcp[i].ddev, ent->driver_data);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+void amdgpu_xcp_dev_unplug(struct amdgpu_device *adev)
+{
+	int i;
+
+	if (!adev->xcp_mgr)
+		return;
+
+	for (i = 0; i < MAX_XCP; i++)
+		drm_dev_unplug(adev->xcp_mgr->xcp[i].ddev);
+}
+
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
index e1319b887bf3..dad0b98d1ae7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
@@ -70,6 +70,7 @@ struct amdgpu_xcp {
 	uint8_t id;
 	uint8_t mem_id;
 	bool valid;
+	struct drm_device *ddev;
 };
 
 struct amdgpu_xcp_mgr {
@@ -115,6 +116,10 @@ int amdgpu_xcp_get_inst_details(struct amdgpu_xcp *xcp,
 				enum AMDGPU_XCP_IP_BLOCK ip,
 				uint32_t *inst_mask);
 
+int amdgpu_xcp_dev_register(struct amdgpu_device *adev,
+				const struct pci_device_id *ent);
+void amdgpu_xcp_dev_unplug(struct amdgpu_device *adev);
+
 static inline int amdgpu_xcp_get_num_xcp(struct amdgpu_xcp_mgr *xcp_mgr)
 {
 	if (!xcp_mgr)
-- 
2.40.1

[PATCH 02/29] drm/amdgpu: find partition ID when open device

[Alex Deucher]

From: James Zhu <James.Zhu@amd.com>

Find partition ID when open device from render device minor.

Signed-off-by: Christian König <christian.koenig@amd.com>
Signed-off-by: James Zhu <James.Zhu@amd.com>
Reviewed-and-tested-by: Philip Yang<Philip.Yang@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h     |  2 ++
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c |  4 ++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c | 29 +++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h |  3 +++
 4 files changed, 38 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 45c6522ee854..4fb43baddf96 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -488,6 +488,8 @@ struct amdgpu_fpriv {
 	struct mutex		bo_list_lock;
 	struct idr		bo_list_handles;
 	struct amdgpu_ctx_mgr	ctx_mgr;
+	/** GPU partition selection */
+	uint32_t		xcp_id;
 };
 
 int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 44997c7ee89d..879718598fa4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -1223,6 +1223,10 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 		goto out_suspend;
 	}
 
+	r = amdgpu_xcp_open_device(adev, fpriv, file_priv);
+	if (r)
+		return r;
+
 	pasid = amdgpu_pasid_alloc(16);
 	if (pasid < 0) {
 		dev_warn(adev->dev, "No more PASIDs available!");
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
index 8b28b18e4291..9b627a8b1d5c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
@@ -335,3 +335,32 @@ void amdgpu_xcp_dev_unplug(struct amdgpu_device *adev)
 		drm_dev_unplug(adev->xcp_mgr->xcp[i].ddev);
 }
 
+int amdgpu_xcp_open_device(struct amdgpu_device *adev,
+			   struct amdgpu_fpriv *fpriv,
+			   struct drm_file *file_priv)
+{
+	int i;
+
+	if (!adev->xcp_mgr)
+		return 0;
+
+	fpriv->xcp_id = ~0;
+	for (i = 0; i < MAX_XCP; ++i) {
+		if (!adev->xcp_mgr->xcp[i].ddev)
+			break;
+
+		if (file_priv->minor == adev->xcp_mgr->xcp[i].ddev->render) {
+			if (adev->xcp_mgr->xcp[i].valid == FALSE) {
+				dev_err(adev->dev, "renderD%d partition %d not valid!",
+						file_priv->minor->index, i);
+				return -ENOENT;
+			}
+			dev_dbg(adev->dev, "renderD%d partition %d openned!",
+					file_priv->minor->index, i);
+			fpriv->xcp_id = i;
+			break;
+		}
+	}
+	return 0;
+}
+
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
index dad0b98d1ae7..ad60520f952c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
@@ -119,6 +119,9 @@ int amdgpu_xcp_get_inst_details(struct amdgpu_xcp *xcp,
 int amdgpu_xcp_dev_register(struct amdgpu_device *adev,
 				const struct pci_device_id *ent);
 void amdgpu_xcp_dev_unplug(struct amdgpu_device *adev);
+int amdgpu_xcp_open_device(struct amdgpu_device *adev,
+			   struct amdgpu_fpriv *fpriv,
+			   struct drm_file *file_priv);
 
 static inline int amdgpu_xcp_get_num_xcp(struct amdgpu_xcp_mgr *xcp_mgr)
 {
-- 
2.40.1

[PATCH 03/29] drm/amdgpu: add partition ID track in ring

[Alex Deucher]

From: James Zhu <James.Zhu@amd.com>

Keep track partition ID in ring.

Signed-off-by: James Zhu <James.Zhu@amd.com>
Acked-by: Lijo Lazar <lijo.lazar@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h      |  1 +
 .../drm/amd/amdgpu/aqua_vanjaram_reg_init.c   | 41 +++++++++++++++++++
 2 files changed, 42 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 5192e3577e99..baa03527bf8b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -252,6 +252,7 @@ struct amdgpu_ring {
 	uint32_t		buf_mask;
 	u32			idx;
 	u32			xcc_id;
+	u32			xcp_id;
 	u32			me;
 	u32			pipe;
 	u32			queue;
diff --git a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram_reg_init.c b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram_reg_init.c
index 97011e7e031d..c90ea34ef9ec 100644
--- a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram_reg_init.c
+++ b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram_reg_init.c
@@ -61,6 +61,47 @@ void aqua_vanjaram_doorbell_index_init(struct amdgpu_device *adev)
 	adev->doorbell_index.max_assignment = AMDGPU_DOORBELL_LAYOUT1_MAX_ASSIGNMENT << 1;
 }
 
+static void aqua_vanjaram_set_xcp_id(struct amdgpu_device *adev,
+			     uint32_t inst_idx, struct amdgpu_ring *ring)
+{
+	int xcp_id;
+	enum AMDGPU_XCP_IP_BLOCK ip_blk;
+	uint32_t inst_mask;
+
+	ring->xcp_id = ~0;
+	if (adev->xcp_mgr->mode == AMDGPU_XCP_MODE_NONE)
+		return;
+
+	inst_mask = 1 << inst_idx;
+
+	switch (ring->funcs->type) {
+	case AMDGPU_HW_IP_GFX:
+	case AMDGPU_RING_TYPE_COMPUTE:
+	case AMDGPU_RING_TYPE_KIQ:
+		ip_blk = AMDGPU_XCP_GFX;
+		break;
+	case AMDGPU_RING_TYPE_SDMA:
+		ip_blk = AMDGPU_XCP_SDMA;
+		break;
+	case AMDGPU_RING_TYPE_VCN_ENC:
+	case AMDGPU_RING_TYPE_VCN_JPEG:
+		ip_blk = AMDGPU_XCP_VCN;
+		if (adev->xcp_mgr->mode == AMDGPU_CPX_PARTITION_MODE)
+			inst_mask = 1 << (inst_idx * 2);
+		break;
+	default:
+		DRM_ERROR("Not support ring type %d!", ring->funcs->type);
+		return;
+	}
+
+	for (xcp_id = 0; xcp_id < adev->xcp_mgr->num_xcps; xcp_id++) {
+		if (adev->xcp_mgr->xcp[xcp_id].ip[ip_blk].inst_mask & inst_mask) {
+			ring->xcp_id = xcp_id;
+			break;
+		}
+	}
+}
+
 static int8_t aqua_vanjaram_logical_to_dev_inst(struct amdgpu_device *adev,
 					 enum amd_hw_ip_block_type block,
 					 int8_t inst)
-- 
2.40.1

[PATCH 04/29] drm/amdgpu: update header to support partition scheduling

[Alex Deucher]

From: James Zhu <James.Zhu@amd.com>

Update header to support partition scheduling.

Signed-off-by: James Zhu <James.Zhu@amd.com>
Acked-by: Lijo Lazar <lijo.lazar@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
index ad60520f952c..cca06d38b03d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
@@ -70,7 +70,9 @@ struct amdgpu_xcp {
 	uint8_t id;
 	uint8_t mem_id;
 	bool valid;
+	atomic_t	ref_cnt;
 	struct drm_device *ddev;
+	struct amdgpu_sched	gpu_sched[AMDGPU_HW_IP_NUM][AMDGPU_RING_PRIO_MAX];
 };
 
 struct amdgpu_xcp_mgr {
@@ -97,6 +99,10 @@ struct amdgpu_xcp_mgr_funcs {
 	int (*suspend)(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id);
 	int (*prepare_resume)(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id);
 	int (*resume)(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id);
+	int (*select_scheds)(struct amdgpu_device *adev,
+				  u32 hw_ip, u32 hw_prio, struct amdgpu_fpriv *fpriv,
+				  unsigned int *num_scheds, struct drm_gpu_scheduler ***scheds);
+	int (*update_partition_sched_list)(struct amdgpu_device *adev);
 };
 
 int amdgpu_xcp_prepare_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id);
@@ -123,6 +129,15 @@ int amdgpu_xcp_open_device(struct amdgpu_device *adev,
 			   struct amdgpu_fpriv *fpriv,
 			   struct drm_file *file_priv);
 
+#define amdgpu_xcp_select_scheds(adev, e, c, d, x, y) \
+	((adev)->xcp_mgr && (adev)->xcp_mgr->funcs && \
+	(adev)->xcp_mgr->funcs->select_scheds ? \
+	(adev)->xcp_mgr->funcs->select_scheds((adev), (e), (c), (d), (x), (y)) : -ENOENT)
+#define amdgpu_xcp_update_partition_sched_list(adev) \
+	((adev)->xcp_mgr && (adev)->xcp_mgr->funcs && \
+	(adev)->xcp_mgr->funcs->update_partition_sched_list ? \
+	(adev)->xcp_mgr->funcs->update_partition_sched_list(adev) : 0)
+
 static inline int amdgpu_xcp_get_num_xcp(struct amdgpu_xcp_mgr *xcp_mgr)
 {
 	if (!xcp_mgr)
-- 
2.40.1

[PATCH 05/29] drm/amdgpu: add partition scheduler list update

[Alex Deucher]

From: James Zhu <James.Zhu@amd.com>

Add partition scheduler list update in late init
and xcp partition mode switch.

Signed-off-by: James Zhu <James.Zhu@amd.com>
Acked-by: Lijo Lazar <lijo.lazar@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c    |  2 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c       |  2 +
 .../drm/amd/amdgpu/aqua_vanjaram_reg_init.c   | 67 ++++++++++++++++++-
 3 files changed, 70 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 40c5845c78df..321b689db601 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -2473,6 +2473,8 @@ static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
 		}
 	}
 
+	amdgpu_xcp_update_partition_sched_list(adev);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
index 9b627a8b1d5c..78fce5aab218 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
@@ -118,6 +118,7 @@ static void __amdgpu_xcp_add_block(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id,
 
 int amdgpu_xcp_init(struct amdgpu_xcp_mgr *xcp_mgr, int num_xcps, int mode)
 {
+	struct amdgpu_device *adev = xcp_mgr->adev;
 	struct amdgpu_xcp_ip ip;
 	uint8_t mem_id;
 	int i, j, ret;
@@ -153,6 +154,7 @@ int amdgpu_xcp_init(struct amdgpu_xcp_mgr *xcp_mgr, int num_xcps, int mode)
 	}
 
 	xcp_mgr->num_xcps = num_xcps;
+	amdgpu_xcp_update_partition_sched_list(adev);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram_reg_init.c b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram_reg_init.c
index c90ea34ef9ec..073ae95e6dd6 100644
--- a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram_reg_init.c
+++ b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram_reg_init.c
@@ -102,6 +102,70 @@ static void aqua_vanjaram_set_xcp_id(struct amdgpu_device *adev,
 	}
 }
 
+static void aqua_vanjaram_xcp_gpu_sched_update(
+		struct amdgpu_device *adev,
+		struct amdgpu_ring *ring,
+		unsigned int sel_xcp_id)
+{
+	unsigned int *num_gpu_sched;
+
+	num_gpu_sched = &adev->xcp_mgr->xcp[sel_xcp_id]
+			.gpu_sched[ring->funcs->type][ring->hw_prio].num_scheds;
+	adev->xcp_mgr->xcp[sel_xcp_id].gpu_sched[ring->funcs->type][ring->hw_prio]
+			.sched[(*num_gpu_sched)++] = &ring->sched;
+	DRM_DEBUG("%s :[%d] gpu_sched[%d][%d] = %d", ring->name,
+			sel_xcp_id, ring->funcs->type,
+			ring->hw_prio, *num_gpu_sched);
+}
+
+static int aqua_vanjaram_xcp_sched_list_update(
+		struct amdgpu_device *adev)
+{
+	struct amdgpu_ring *ring;
+	int i;
+
+	for (i = 0; i < MAX_XCP; i++) {
+		atomic_set(&adev->xcp_mgr->xcp[i].ref_cnt, 0);
+		memset(adev->xcp_mgr->xcp[i].gpu_sched, 0, sizeof(adev->xcp_mgr->xcp->gpu_sched));
+	}
+
+	if (adev->xcp_mgr->mode == AMDGPU_XCP_MODE_NONE)
+		return 0;
+
+	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
+		ring = adev->rings[i];
+		if (!ring || !ring->sched.ready)
+			continue;
+
+		aqua_vanjaram_xcp_gpu_sched_update(adev, ring, ring->xcp_id);
+
+		/* VCN is shared by two partitions under CPX MODE */
+		if ((ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC ||
+			ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG) &&
+			adev->xcp_mgr->mode == AMDGPU_CPX_PARTITION_MODE)
+			aqua_vanjaram_xcp_gpu_sched_update(adev, ring, ring->xcp_id + 1);
+	}
+
+	return 0;
+}
+
+static int aqua_vanjaram_update_partition_sched_list(struct amdgpu_device *adev)
+{
+	int i;
+
+	for (i = 0; i < adev->num_rings; i++) {
+		struct amdgpu_ring *ring = adev->rings[i];
+
+		if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE ||
+			ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
+			aqua_vanjaram_set_xcp_id(adev, ring->xcc_id, ring);
+		else
+			aqua_vanjaram_set_xcp_id(adev, ring->me, ring);
+	}
+
+	return aqua_vanjaram_xcp_sched_list_update(adev);
+}
+
 static int8_t aqua_vanjaram_logical_to_dev_inst(struct amdgpu_device *adev,
 					 enum amd_hw_ip_block_type block,
 					 int8_t inst)
@@ -483,7 +547,8 @@ struct amdgpu_xcp_mgr_funcs aqua_vanjaram_xcp_funcs = {
 	.switch_partition_mode = &aqua_vanjaram_switch_partition_mode,
 	.query_partition_mode = &aqua_vanjaram_query_partition_mode,
 	.get_ip_details = &aqua_vanjaram_get_xcp_ip_details,
-	.get_xcp_mem_id = &aqua_vanjaram_get_xcp_mem_id
+	.get_xcp_mem_id = &aqua_vanjaram_get_xcp_mem_id,
+	.update_partition_sched_list = &aqua_vanjaram_update_partition_sched_list
 };
 
 static int aqua_vanjaram_xcp_mgr_init(struct amdgpu_device *adev)
-- 
2.40.1

[PATCH 06/29] drm/amdgpu: keep amdgpu_ctx_mgr in ctx structure

[Alex Deucher]

From: James Zhu <James.Zhu@amd.com>

Keep amdgpu_ctx_mgr in ctx structure to track fpriv.

Signed-off-by: James Zhu <James.Zhu@amd.com>
Acked-by: Lijo Lazar <lijo.lazar@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h | 1 +
 2 files changed, 2 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index e3d047663d61..06d68a08251a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -332,6 +332,7 @@ static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority,
 	else
 		ctx->stable_pstate = current_stable_pstate;
 
+	ctx->ctx_mgr = &(fpriv->ctx_mgr);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
index 5fd79f94e2d0..85376baaa92f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
@@ -57,6 +57,7 @@ struct amdgpu_ctx {
 	unsigned long			ras_counter_ce;
 	unsigned long			ras_counter_ue;
 	uint32_t			stable_pstate;
+	struct amdgpu_ctx_mgr		*ctx_mgr;
 };
 
 struct amdgpu_ctx_mgr {
-- 
2.40.1

Re: [PATCH 06/29] drm/amdgpu: keep amdgpu_ctx_mgr in ctx structure

[Mike Lothian]

On Wed, 10 May 2023 at 22:24, Alex Deucher <alexander.deucher@amd.com> wrote:
>
> From: James Zhu <James.Zhu@amd.com>
>
> Keep amdgpu_ctx_mgr in ctx structure to track fpriv.
>
> Signed-off-by: James Zhu <James.Zhu@amd.com>
> Acked-by: Lijo Lazar <lijo.lazar@amd.com>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 1 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h | 1 +
>  2 files changed, 2 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> index e3d047663d61..06d68a08251a 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> @@ -332,6 +332,7 @@ static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority,
>         else
>                 ctx->stable_pstate = current_stable_pstate;
>
> +       ctx->ctx_mgr = &(fpriv->ctx_mgr);
>         return 0;
>  }
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> index 5fd79f94e2d0..85376baaa92f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> @@ -57,6 +57,7 @@ struct amdgpu_ctx {
>         unsigned long                   ras_counter_ce;
>         unsigned long                   ras_counter_ue;
>         uint32_t                        stable_pstate;
> +       struct amdgpu_ctx_mgr           *ctx_mgr;
>  };
>
>  struct amdgpu_ctx_mgr {
> --
> 2.40.1
>

Hi

This isn't compiling for me with clang 16

drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c:348:19: error: use of
undeclared identifier 'fpriv'
       ctx->ctx_mgr = &(fpriv->ctx_mgr);
                        ^
1 error generated.

Cheers

Mike

Re: [PATCH 06/29] drm/amdgpu: keep amdgpu_ctx_mgr in ctx structure

[Alex Deucher]

On Fri, May 19, 2023 at 8:16 AM Mike Lothian <mike@fireburn.co.uk> wrote:
>
> On Wed, 10 May 2023 at 22:24, Alex Deucher <alexander.deucher@amd.com> wrote:
> >
> > From: James Zhu <James.Zhu@amd.com>
> >
> > Keep amdgpu_ctx_mgr in ctx structure to track fpriv.
> >
> > Signed-off-by: James Zhu <James.Zhu@amd.com>
> > Acked-by: Lijo Lazar <lijo.lazar@amd.com>
> > Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> > ---
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 1 +
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h | 1 +
> >  2 files changed, 2 insertions(+)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> > index e3d047663d61..06d68a08251a 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> > @@ -332,6 +332,7 @@ static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority,
> >         else
> >                 ctx->stable_pstate = current_stable_pstate;
> >
> > +       ctx->ctx_mgr = &(fpriv->ctx_mgr);
> >         return 0;
> >  }
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> > index 5fd79f94e2d0..85376baaa92f 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> > @@ -57,6 +57,7 @@ struct amdgpu_ctx {
> >         unsigned long                   ras_counter_ce;
> >         unsigned long                   ras_counter_ue;
> >         uint32_t                        stable_pstate;
> > +       struct amdgpu_ctx_mgr           *ctx_mgr;
> >  };
> >
> >  struct amdgpu_ctx_mgr {
> > --
> > 2.40.1
> >
>
> Hi
>
> This isn't compiling for me with clang 16
>
> drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c:348:19: error: use of
> undeclared identifier 'fpriv'
>        ctx->ctx_mgr = &(fpriv->ctx_mgr);
>                         ^
> 1 error generated.

When this patch was originally written the function had fpriv defined,
but it got missed after a rebase.  Fixed up now.

Alex


>
> Cheers
>
> Mike

[PATCH 07/29] drm/amdgpu: add partition schedule for GC(9, 4, 3)

[Alex Deucher]

From: James Zhu <James.Zhu@amd.com>

Implement partition schedule for GC(9, 4, 3).

Signed-off-by: James Zhu <James.Zhu@amd.com>
Acked-by: Lijo Lazar <lijo.lazar@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 .../drm/amd/amdgpu/aqua_vanjaram_reg_init.c   | 41 +++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram_reg_init.c b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram_reg_init.c
index 073ae95e6dd6..4ca932a62ce6 100644
--- a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram_reg_init.c
+++ b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram_reg_init.c
@@ -166,6 +166,46 @@ static int aqua_vanjaram_update_partition_sched_list(struct amdgpu_device *adev)
 	return aqua_vanjaram_xcp_sched_list_update(adev);
 }
 
+int aqua_vanjaram_select_scheds(
+		struct amdgpu_device *adev,
+		u32 hw_ip,
+		u32 hw_prio,
+		struct amdgpu_fpriv *fpriv,
+		unsigned int *num_scheds,
+		struct drm_gpu_scheduler ***scheds)
+{
+	u32 sel_xcp_id;
+	int i;
+
+	if (fpriv->xcp_id == ~0) {
+		u32 least_ref_cnt = ~0;
+
+		fpriv->xcp_id = 0;
+		for (i = 0; i < adev->xcp_mgr->num_xcps; i++) {
+			u32 total_ref_cnt;
+
+			total_ref_cnt = atomic_read(&adev->xcp_mgr->xcp[i].ref_cnt);
+			if (total_ref_cnt < least_ref_cnt) {
+				fpriv->xcp_id = i;
+				least_ref_cnt = total_ref_cnt;
+			}
+		}
+	}
+	sel_xcp_id = fpriv->xcp_id;
+
+	if (adev->xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds) {
+		*num_scheds = adev->xcp_mgr->xcp[fpriv->xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds;
+		*scheds = adev->xcp_mgr->xcp[fpriv->xcp_id].gpu_sched[hw_ip][hw_prio].sched;
+		atomic_inc(&adev->xcp_mgr->xcp[sel_xcp_id].ref_cnt);
+		DRM_DEBUG("Selected partition #%d", sel_xcp_id);
+	} else {
+		DRM_ERROR("Failed to schedule partition #%d.", sel_xcp_id);
+		return -ENOENT;
+	}
+
+	return 0;
+}
+
 static int8_t aqua_vanjaram_logical_to_dev_inst(struct amdgpu_device *adev,
 					 enum amd_hw_ip_block_type block,
 					 int8_t inst)
@@ -548,6 +588,7 @@ struct amdgpu_xcp_mgr_funcs aqua_vanjaram_xcp_funcs = {
 	.query_partition_mode = &aqua_vanjaram_query_partition_mode,
 	.get_ip_details = &aqua_vanjaram_get_xcp_ip_details,
 	.get_xcp_mem_id = &aqua_vanjaram_get_xcp_mem_id,
+	.select_scheds = &aqua_vanjaram_select_scheds,
 	.update_partition_sched_list = &aqua_vanjaram_update_partition_sched_list
 };
 
-- 
2.40.1

[PATCH 08/29] drm/amdgpu: run partition schedule if it is supported

[Alex Deucher]

From: James Zhu <James.Zhu@amd.com>

Run partition schedule if it is supported during ctx init entity.

Signed-off-by: James Zhu <James.Zhu@amd.com>
Acked-by: Lijo Lazar <lijo.lazar@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index 06d68a08251a..e579bb054a58 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -222,8 +222,19 @@ static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, u32 hw_ip,
 	drm_prio = amdgpu_ctx_to_drm_sched_prio(ctx_prio);
 
 	hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM);
-	scheds = adev->gpu_sched[hw_ip][hw_prio].sched;
-	num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds;
+
+	if (!(adev)->xcp_mgr) {
+		scheds = adev->gpu_sched[hw_ip][hw_prio].sched;
+		num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds;
+	} else {
+		struct amdgpu_fpriv *fpriv;
+
+		fpriv = container_of(ctx->ctx_mgr, struct amdgpu_fpriv, ctx_mgr);
+		r = amdgpu_xcp_select_scheds(adev, hw_ip, hw_prio, fpriv,
+						&num_scheds, &scheds);
+		if (r)
+			goto cleanup_entity;
+	}
 
 	/* disable load balance if the hw engine retains context among dependent jobs */
 	if (hw_ip == AMDGPU_HW_IP_VCN_ENC ||
-- 
2.40.1

[PATCH 09/29] drm/amdgpu: update ref_cnt before ctx free

[Alex Deucher]

From: James Zhu <James.Zhu@amd.com>

Update ref_cnt before ctx free.

Signed-off-by: James Zhu <James.Zhu@amd.com>
Acked-by: Lijo Lazar <lijo.lazar@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c |  7 +++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c | 16 ++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h |  2 ++
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index e579bb054a58..3ccd709ae76a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -266,7 +266,8 @@ static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, u32 hw_ip,
 	return r;
 }
 
-static ktime_t amdgpu_ctx_fini_entity(struct amdgpu_ctx_entity *entity)
+static ktime_t amdgpu_ctx_fini_entity(struct amdgpu_device *adev,
+				  struct amdgpu_ctx_entity *entity)
 {
 	ktime_t res = ns_to_ktime(0);
 	int i;
@@ -279,6 +280,8 @@ static ktime_t amdgpu_ctx_fini_entity(struct amdgpu_ctx_entity *entity)
 		dma_fence_put(entity->fences[i]);
 	}
 
+	amdgpu_xcp_release_sched(adev, entity);
+
 	kfree(entity);
 	return res;
 }
@@ -412,7 +415,7 @@ static void amdgpu_ctx_fini(struct kref *ref)
 		for (j = 0; j < AMDGPU_MAX_ENTITY_NUM; ++j) {
 			ktime_t spend;
 
-			spend = amdgpu_ctx_fini_entity(ctx->entities[i][j]);
+			spend = amdgpu_ctx_fini_entity(adev, ctx->entities[i][j]);
 			atomic64_add(ktime_to_ns(spend), &mgr->time_spend[i]);
 		}
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
index 78fce5aab218..9b960ba0b7ac 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
@@ -366,3 +366,19 @@ int amdgpu_xcp_open_device(struct amdgpu_device *adev,
 	return 0;
 }
 
+void amdgpu_xcp_release_sched(struct amdgpu_device *adev,
+				  struct amdgpu_ctx_entity *entity)
+{
+	struct drm_gpu_scheduler *sched;
+	struct amdgpu_ring *ring;
+
+	if (!adev->xcp_mgr)
+		return;
+
+	sched = entity->entity.rq->sched;
+	if (sched->ready) {
+		ring = to_amdgpu_ring(entity->entity.rq->sched);
+		atomic_dec(&adev->xcp_mgr->xcp[ring->xcp_id].ref_cnt);
+	}
+}
+
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
index cca06d38b03d..39aca87ce204 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
@@ -128,6 +128,8 @@ void amdgpu_xcp_dev_unplug(struct amdgpu_device *adev);
 int amdgpu_xcp_open_device(struct amdgpu_device *adev,
 			   struct amdgpu_fpriv *fpriv,
 			   struct drm_file *file_priv);
+void amdgpu_xcp_release_sched(struct amdgpu_device *adev,
+			      struct amdgpu_ctx_entity *entity);
 
 #define amdgpu_xcp_select_scheds(adev, e, c, d, x, y) \
 	((adev)->xcp_mgr && (adev)->xcp_mgr->funcs && \
-- 
2.40.1

[PATCH 10/29] drm/amdgpu: Add xcp manager num_xcp_per_mem_partition

[Alex Deucher]

From: Philip Yang <Philip.Yang@amd.com>

Used by KFD to check memory limit accounting.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c | 1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
index 9b960ba0b7ac..f2981d21d4e0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
@@ -156,6 +156,7 @@ int amdgpu_xcp_init(struct amdgpu_xcp_mgr *xcp_mgr, int num_xcps, int mode)
 	xcp_mgr->num_xcps = num_xcps;
 	amdgpu_xcp_update_partition_sched_list(adev);
 
+	xcp_mgr->num_xcp_per_mem_partition = num_xcps / xcp_mgr->adev->gmc.num_mem_partitions;
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
index 39aca87ce204..68b63b970ce8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
@@ -83,6 +83,9 @@ struct amdgpu_xcp_mgr {
 	struct amdgpu_xcp xcp[MAX_XCP];
 	uint8_t num_xcps;
 	int8_t mode;
+
+	 /* Used to determine KFD memory size limits per XCP */
+	unsigned int num_xcp_per_mem_partition;
 };
 
 struct amdgpu_xcp_mgr_funcs {
-- 
2.40.1

[PATCH 11/29] drm/amdkfd: Store drm node minor number for kfd nodes

[Alex Deucher]

From: Philip Yang <Philip.Yang@amd.com>

From KFD topology, application will find kfd node with the corresponding
drm device node minor number, for example if partition drm node starts
from /dev/dri/renderD129, then KFD node 0 with store drm node minor
number 129. Application will open drm node /dev/dri/renderD129 to create
amdgpu vm for kfd node 0 with the correct vm->mem_id to indicate the
memory partition.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 6d6243b978e1..a8e25aecf839 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -1942,8 +1942,12 @@ int kfd_topology_add_device(struct kfd_node *gpu)
 		amdgpu_amdkfd_get_max_engine_clock_in_mhz(dev->gpu->adev);
 	dev->node_props.max_engine_clk_ccompute =
 		cpufreq_quick_get_max(0) / 1000;
-	dev->node_props.drm_render_minor =
-		gpu->kfd->shared_resources.drm_render_minor;
+
+	if (gpu->xcp)
+		dev->node_props.drm_render_minor = gpu->xcp->ddev->render->index;
+	else
+		dev->node_props.drm_render_minor =
+				gpu->kfd->shared_resources.drm_render_minor;
 
 	dev->node_props.hive_id = gpu->kfd->hive_id;
 	dev->node_props.num_sdma_engines = kfd_get_num_sdma_engines(gpu);
-- 
2.40.1

[PATCH 12/29] drm/amdgpu: Add memory partition id to amdgpu_vm

[Alex Deucher]

From: Philip Yang <Philip.Yang@amd.com>

If xcp_mgr is initialized, add mem_id to amdgpu_vm structure to store
memory partition number when creating amdgpu_vm for the xcp. The xcp
number is decided when opening the render device, for example
/dev/dri/renderD129 is xcp_id 0, /dev/dri/rederD130 is xcp_id 1.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 8 ++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h  | 3 +++
 drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c | 3 +++
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 879718598fa4..815098be4c2f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -1223,10 +1223,6 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 		goto out_suspend;
 	}
 
-	r = amdgpu_xcp_open_device(adev, fpriv, file_priv);
-	if (r)
-		return r;
-
 	pasid = amdgpu_pasid_alloc(16);
 	if (pasid < 0) {
 		dev_warn(adev->dev, "No more PASIDs available!");
@@ -1237,6 +1233,10 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 	if (r)
 		goto error_pasid;
 
+	r = amdgpu_xcp_open_device(adev, fpriv, file_priv);
+	if (r)
+		goto error_vm;
+
 	r = amdgpu_vm_set_pasid(adev, &fpriv->vm, pasid);
 	if (r)
 		goto error_vm;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 2fdec4114627..d551fca1780e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -332,6 +332,9 @@ struct amdgpu_vm {
 	struct ttm_lru_bulk_move lru_bulk_move;
 	/* Flag to indicate if VM is used for compute */
 	bool			is_compute_context;
+
+	/* Memory partition number, -1 means any partition */
+	int8_t			mem_id;
 };
 
 struct amdgpu_vm_manager {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
index f2981d21d4e0..610c32c4f5af 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
@@ -364,6 +364,9 @@ int amdgpu_xcp_open_device(struct amdgpu_device *adev,
 			break;
 		}
 	}
+
+	fpriv->vm.mem_id = fpriv->xcp_id == ~0 ? -1 :
+				adev->xcp_mgr->xcp[fpriv->xcp_id].mem_id;
 	return 0;
 }
 
-- 
2.40.1

[PATCH 13/29] drm/amdkfd: Show KFD node memory partition info

[Alex Deucher]

From: Philip Yang <Philip.Yang@amd.com>

Show KFD node memory partition id and size, add helper function
KFD_XCP_MEMORY_SIZE to get kfd node memory size, will be used
later to support memory accounting per partition.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 5 +++++
 drivers/gpu/drm/amd/amdkfd/kfd_device.c    | 7 ++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index e4e1dbba060a..324cb566ca2f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -330,6 +330,11 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
 		uint64_t size, u32 alloc_flag);
 
+#define KFD_XCP_MEMORY_SIZE(n) ((n)->adev->gmc.num_mem_partitions ?\
+		(n)->adev->gmc.mem_partitions[(n)->xcp->mem_id].size /\
+		(n)->adev->xcp_mgr->num_xcp_per_mem_partition :\
+		(n)->adev->gmc.real_vram_size)
+
 #if IS_ENABLED(CONFIG_HSA_AMD)
 void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
 void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index b5497d2ee984..db5b53fcdf11 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -724,7 +724,6 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 
 	kfd_cwsr_init(kfd);
 
-	/* TODO: Needs to be updated for memory partitioning */
 	svm_migrate_init(kfd->adev);
 
 	amdgpu_amdkfd_get_local_mem_info(kfd->adev, &kfd->local_mem_info);
@@ -754,6 +753,12 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 				(1U << NUM_XCC(kfd->adev->gfx.xcc_mask)) - 1;
 		}
 
+		if (node->xcp) {
+			dev_info(kfd_device, "KFD node %d partition %d size %lldM\n",
+				node->node_id, node->xcp->mem_id,
+				KFD_XCP_MEMORY_SIZE(node) >> 20);
+		}
+
 		if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 3) &&
 		    partition_mode == AMDGPU_CPX_PARTITION_MODE &&
 		    kfd->num_nodes != 1) {
-- 
2.40.1

[PATCH 14/29] drm/amdgpu: Add memory partition mem_id to amdgpu_bo

[Alex Deucher]

From: Philip Yang <Philip.Yang@amd.com>

Add mem_id_plus1 parameter to amdgpu_gem_object_create and pass it to
amdgpu_bo_create. For dGPU mode allocation, mem_id is used by VRAM
manager to get the memory partition fpfn, lpfn from xcp manager. For APU
native mode allocation, mem_id is used to get NUMA node id from xcp
manager, then pass to TTM as numa pool id to alloc memory from the
specific NUMA node. mem_id -1 means for entire VRAM or any NUMA nodes.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 4 ++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c      | 2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c          | 9 +++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h          | 3 +--
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.c       | 3 +++
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.h       | 5 +++++
 6 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 12149b317b88..6d0c25e34af1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -289,7 +289,7 @@ create_dmamap_sg_bo(struct amdgpu_device *adev,
 
 	ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, 1,
 			AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE | flags,
-			ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
+			ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj, 0);
 
 	amdgpu_bo_unreserve(mem->bo);
 
@@ -1720,7 +1720,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 			va, (*mem)->aql_queue ? size << 1 : size, domain_string(alloc_domain));
 
 	ret = amdgpu_gem_object_create(adev, aligned_size, 1, alloc_domain, alloc_flags,
-				       bo_type, NULL, &gobj);
+				       bo_type, NULL, &gobj, 0);
 	if (ret) {
 		pr_debug("Failed to create BO on domain %s. ret %d\n",
 			 domain_string(alloc_domain), ret);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index e97b1eef2c9d..8b162f05d1fd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -335,7 +335,7 @@ amdgpu_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf)
 
 	ret = amdgpu_gem_object_create(adev, dma_buf->size, PAGE_SIZE,
 				       AMDGPU_GEM_DOMAIN_CPU, flags,
-				       ttm_bo_type_sg, resv, &gobj);
+				       ttm_bo_type_sg, resv, &gobj, 0);
 	if (ret)
 		goto error;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 6936cd63df42..01029b495f5a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -97,7 +97,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 			     int alignment, u32 initial_domain,
 			     u64 flags, enum ttm_bo_type type,
 			     struct dma_resv *resv,
-			     struct drm_gem_object **obj)
+			     struct drm_gem_object **obj, int8_t mem_id_plus1)
 {
 	struct amdgpu_bo *bo;
 	struct amdgpu_bo_user *ubo;
@@ -115,6 +115,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 	bp.flags = flags;
 	bp.domain = initial_domain;
 	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
+	bp.mem_id_plus1 = mem_id_plus1;
 
 	r = amdgpu_bo_create_user(adev, &bp, &ubo);
 	if (r)
@@ -335,7 +336,7 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
 retry:
 	r = amdgpu_gem_object_create(adev, size, args->in.alignment,
 				     initial_domain,
-				     flags, ttm_bo_type_device, resv, &gobj);
+				     flags, ttm_bo_type_device, resv, &gobj, 0);
 	if (r && r != -ERESTARTSYS) {
 		if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
 			flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
@@ -404,7 +405,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
 
 	/* create a gem object to contain this object in */
 	r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU,
-				     0, ttm_bo_type_device, NULL, &gobj);
+				     0, ttm_bo_type_device, NULL, &gobj, 0);
 	if (r)
 		return r;
 
@@ -930,7 +931,7 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
 	domain = amdgpu_bo_get_preferred_domain(adev,
 				amdgpu_display_supported_domains(adev, flags));
 	r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags,
-				     ttm_bo_type_device, NULL, &gobj);
+				     ttm_bo_type_device, NULL, &gobj, 0);
 	if (r)
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h
index 637bf51dbf06..646c4fcc8e40 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h
@@ -43,8 +43,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 			     int alignment, u32 initial_domain,
 			     u64 flags, enum ttm_bo_type type,
 			     struct dma_resv *resv,
-			     struct drm_gem_object **obj);
-
+			     struct drm_gem_object **obj, int8_t mem_id_plus1);
 int amdgpu_mode_dumb_create(struct drm_file *file_priv,
 			    struct drm_device *dev,
 			    struct drm_mode_create_dumb *args);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index c6214db42bda..155b62971a33 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -574,6 +574,9 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
 
 	bo->flags = bp->flags;
 
+	/* bo->mem_id -1 means any partition */
+	bo->mem_id = bp->mem_id_plus1 - 1;
+
 	if (!amdgpu_bo_support_uswc(bo->flags))
 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 8fdfa739a4f2..521a432348a0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -56,6 +56,8 @@ struct amdgpu_bo_param {
 	bool				no_wait_gpu;
 	struct dma_resv			*resv;
 	void				(*destroy)(struct ttm_buffer_object *bo);
+	/* memory partition number plus 1, 0 means any partition */
+	int8_t				mem_id_plus1;
 };
 
 /* bo virtual addresses in a vm */
@@ -108,6 +110,9 @@ struct amdgpu_bo {
 	struct mmu_interval_notifier	notifier;
 #endif
 	struct kgd_mem                  *kfd_bo;
+
+	/* memory partition number, -1 means any partition */
+	int8_t				mem_id;
 };
 
 struct amdgpu_bo_user {
-- 
2.40.1

[PATCH 15/29] drm/amdkfd: Alloc memory of GPU support memory partition

[Alex Deucher]

From: Philip Yang <Philip.Yang@amd.com>

For dGPU mode VRAM allocation, create amdgpu_bo from amdgpu_vm->mem_id,
to alloc from the correct memory range.

For APU mode VRAM allocation, set alloc domain to GTT, and set
bp->mem_id_plus1 from amdgpu_vm->mem_id + 1 to create amdgpu_bo, to
allocate system memory from correct NUMA node.

For GTT allocation, use mem_id -1 to allocate system memory from any
NUMA nodes.

Remove amdgpu_ttm_tt_set_mem_pool, to avoid the confusion that memory
maybe allocated from different mem_id.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c  | 24 ++++++-------------
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c       | 20 +---------------
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h       |  1 -
 3 files changed, 8 insertions(+), 37 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 6d0c25e34af1..71b22d61dd27 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1640,9 +1640,9 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 	struct drm_gem_object *gobj = NULL;
 	u32 domain, alloc_domain;
 	uint64_t aligned_size;
+	int8_t mem_id = -1;
 	u64 alloc_flags;
 	int ret;
-	int mem_id = 0; /* Fixme : to be changed when mem_id support patch lands, until then NPS1, SPX only */
 
 	/*
 	 * Check on which domain to allocate BO
@@ -1652,13 +1652,14 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 
 		if (adev->gmc.is_app_apu) {
 			domain = AMDGPU_GEM_DOMAIN_GTT;
-			alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
+			alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
 			alloc_flags = 0;
 		} else {
 			alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
 			alloc_flags |= (flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) ?
 			AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED : 0;
 		}
+		mem_id = avm->mem_id;
 	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
 		domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
 		alloc_flags = 0;
@@ -1716,11 +1717,12 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 		goto err_reserve_limit;
 	}
 
-	pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s\n",
-			va, (*mem)->aql_queue ? size << 1 : size, domain_string(alloc_domain));
+	pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s mem_id %d\n",
+		 va, (*mem)->aql_queue ? size << 1 : size,
+		 domain_string(alloc_domain), mem_id);
 
 	ret = amdgpu_gem_object_create(adev, aligned_size, 1, alloc_domain, alloc_flags,
-				       bo_type, NULL, &gobj, 0);
+				       bo_type, NULL, &gobj, mem_id + 1);
 	if (ret) {
 		pr_debug("Failed to create BO on domain %s. ret %d\n",
 			 domain_string(alloc_domain), ret);
@@ -1746,17 +1748,6 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 	(*mem)->mapped_to_gpu_memory = 0;
 	(*mem)->process_info = avm->process_info;
 
-	if (adev->gmc.is_app_apu &&
-	    ((*mem)->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)) {
-		bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
-		bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
-		ret = amdgpu_ttm_tt_set_mem_pool(&bo->tbo, mem_id);
-		if (ret) {
-			pr_debug("failed to set ttm mem pool %d\n", ret);
-			goto err_set_mem_partition;
-		}
-	}
-
 	add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr);
 
 	if (user_addr) {
@@ -1783,7 +1774,6 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 allocate_init_user_pages_failed:
 err_pin_bo:
 	remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
-err_set_mem_partition:
 	drm_vma_node_revoke(&gobj->vma_node, drm_priv);
 err_node_allow:
 	/* Don't unreserve system mem limit twice */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 254927c596ba..395edca3b7f9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1064,7 +1064,7 @@ static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
 		return NULL;
 	}
 	gtt->gobj = &bo->base;
-	gtt->pool_id = NUMA_NO_NODE;
+	gtt->pool_id = abo->mem_id;
 
 	if (abo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
 		caching = ttm_write_combined;
@@ -1159,24 +1159,6 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_device *bdev,
 	return ttm_pool_free(pool, ttm);
 }
 
-/**
- * amdgpu_ttm_tt_set_mem_pool - Set the TTM memory pool for the TTM BO
- * @tbo: The ttm_buffer_object that backs the VRAM bo
- * @mem_id: to select the initialized ttm pool corresponding to the memory partition
- */
-int amdgpu_ttm_tt_set_mem_pool(struct ttm_buffer_object *tbo, int mem_id)
-{
-	struct ttm_tt *ttm = tbo->ttm;
-	struct amdgpu_ttm_tt *gtt;
-
-	if (!ttm && !ttm_tt_is_populated(ttm))
-		return -EINVAL;
-
-	gtt = ttm_to_amdgpu_ttm_tt(ttm);
-	gtt->pool_id = mem_id;
-	return 0;
-}
-
 /**
  * amdgpu_ttm_tt_get_userptr - Return the userptr GTT ttm_tt for the current
  * task
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index fe32de1bf4d5..8ef048a0a33e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -192,7 +192,6 @@ bool amdgpu_ttm_tt_has_userptr(struct ttm_tt *ttm);
 struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm);
 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
 				  unsigned long end, unsigned long *userptr);
-int amdgpu_ttm_tt_set_mem_pool(struct ttm_buffer_object *tbo, int mem_id);
 bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
 				       int *last_invalidated);
 bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm);
-- 
2.40.1

[PATCH 16/29] drm/amdkfd: SVM range allocation support memory partition

[Alex Deucher]

From: Philip Yang <Philip.Yang@amd.com>

Pass kfd node->xcp->mem_id to amdgpu bo create parameter mem_id_plus1 to
allocate new svm_bo on the specified memory partition.

This is only for dGPU mode as we don't migrate with APU mode.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index c5675c7e3b9e..f6a886d9e902 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -554,16 +554,20 @@ svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
 	bp.flags |= AMDGPU_GEM_CREATE_DISCARDABLE;
 	bp.type = ttm_bo_type_device;
 	bp.resv = NULL;
+	if (node->xcp)
+		bp.mem_id_plus1 = node->xcp->mem_id + 1;
 
-	/* TODO: Allocate memory from the right memory partition. We can sort
-	 * out the details later, once basic memory partitioning is working
-	 */
 	r = amdgpu_bo_create_user(node->adev, &bp, &ubo);
 	if (r) {
 		pr_debug("failed %d to create bo\n", r);
 		goto create_bo_failed;
 	}
 	bo = &ubo->bo;
+
+	pr_debug("alloc bo at offset 0x%lx size 0x%lx on partition %d\n",
+		 bo->tbo.resource->start << PAGE_SHIFT, bp.size,
+		 bp.mem_id_plus1 - 1);
+
 	r = amdgpu_bo_reserve(bo, true);
 	if (r) {
 		pr_debug("failed %d to reserve bo\n", r);
-- 
2.40.1

[PATCH 17/29] drm/amdgpu: dGPU mode placement support memory partition

[Alex Deucher]

From: Philip Yang <Philip.Yang@amd.com>

dGPU mode uses VRAM manager to validate bo, amdgpu bo placement use the
mem_id  to get the allocation range first, last page frame number
from xcp manager, pass to drm buddy allocator as the allowed range.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 155b62971a33..cfa14b56c419 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -132,13 +132,18 @@ void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
 	if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
 		unsigned visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
 
-		places[c].fpfn = 0;
-		places[c].lpfn = 0;
+		if (adev->gmc.mem_partitions && abo->mem_id >= 0) {
+			places[c].fpfn = adev->gmc.mem_partitions[abo->mem_id].range.fpfn;
+			places[c].lpfn = adev->gmc.mem_partitions[abo->mem_id].range.lpfn;
+		} else {
+			places[c].fpfn = 0;
+			places[c].lpfn = 0;
+		}
 		places[c].mem_type = TTM_PL_VRAM;
 		places[c].flags = 0;
 
 		if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
-			places[c].lpfn = visible_pfn;
+			places[c].lpfn = min_not_zero(places[c].lpfn, visible_pfn);
 		else if (adev->gmc.real_vram_size != adev->gmc.visible_vram_size)
 			places[c].flags |= TTM_PL_FLAG_TOPDOWN;
 
-- 
2.40.1

[PATCH 18/29] drm/amdkfd: Update MTYPE for far memory partition

[Alex Deucher]

From: Philip Yang <Philip.Yang@amd.com>

Use MTYPE RW/MTYPE_CC for mapping system memory or VRAM to KFD node
within the same memory partition, use MTYPE_NC for mapping on KFD node
from the far memory partition of the same socket or from another socket
on same XGMI hive.

On NPS4 or 4P system, MTYPE will be overridden per page depending on
the memory NUMA node id and vm->mem_id.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 15 +++++++--------
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c  |  9 +++++----
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 7dfe6a8ca91a..ee5d4d67b423 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -1191,7 +1191,7 @@ static void gmc_v9_0_get_coherence_flags(struct amdgpu_device *adev,
 	bool is_vram = bo->tbo.resource->mem_type == TTM_PL_VRAM;
 	bool coherent = bo->flags & AMDGPU_GEM_CREATE_COHERENT;
 	bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED;
-	/* TODO: memory partitions struct amdgpu_vm *vm = mapping->bo_va->base.vm;*/
+	struct amdgpu_vm *vm = mapping->bo_va->base.vm;
 	unsigned int mtype_local, mtype;
 	bool snoop = false;
 	bool is_local;
@@ -1252,8 +1252,8 @@ static void gmc_v9_0_get_coherence_flags(struct amdgpu_device *adev,
 		}
 		is_local = (!is_vram && (adev->flags & AMD_IS_APU) &&
 			    num_possible_nodes() <= 1) ||
-			   (is_vram && adev == bo_adev /* TODO: memory partitions &&
-			    bo->mem_id == vm->mem_id*/);
+			   (is_vram && adev == bo_adev &&
+			    bo->mem_id == vm->mem_id);
 		snoop = true;
 		if (uncached) {
 			mtype = MTYPE_UC;
@@ -1340,13 +1340,12 @@ static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev,
 		return;
 	}
 
-	/* TODO: memory partitions. mem_id is hard-coded to 0 for now.
-	 * FIXME: Only supported on native mode for now. For carve-out, the
+	/* FIXME: Only supported on native mode for now. For carve-out, the
 	 * NUMA affinity of the GPU/VM needs to come from the PCI info because
 	 * memory partitions are not associated with different NUMA nodes.
 	 */
-	if (adev->gmc.is_app_apu) {
-		local_node = adev->gmc.mem_partitions[/*vm->mem_id*/0].numa.node;
+	if (adev->gmc.is_app_apu && vm->mem_id >= 0) {
+		local_node = adev->gmc.mem_partitions[vm->mem_id].numa.node;
 	} else {
 		dev_dbg(adev->dev, "Only native mode APU is supported.\n");
 		return;
@@ -1361,7 +1360,7 @@ static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev,
 	}
 	nid = pfn_to_nid(addr >> PAGE_SHIFT);
 	dev_dbg(adev->dev, "vm->mem_id=%d, local_node=%d, nid=%d\n",
-		/*vm->mem_id*/0, local_node, nid);
+		vm->mem_id, local_node, nid);
 	if (nid == local_node) {
 		uint64_t old_flags = *flags;
 		unsigned int mtype_local = MTYPE_RW;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index f6a886d9e902..8b5453fd304a 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -1202,8 +1202,8 @@ svm_range_get_pte_flags(struct kfd_node *node,
 			mapping_flags |= AMDGPU_VM_MTYPE_UC;
 		} else if (domain == SVM_RANGE_VRAM_DOMAIN) {
 			/* local HBM region close to partition */
-			if (bo_node->adev == node->adev /* TODO: memory partitions &&
-			    bo_node->mem_id == node->mem_id*/)
+			if (bo_node->adev == node->adev &&
+			    (!bo_node->xcp || !node->xcp || bo_node->xcp->mem_id == node->xcp->mem_id))
 				mapping_flags |= mtype_local;
 			/* local HBM region far from partition or remote XGMI GPU */
 			else if (svm_nodes_in_same_hive(bo_node, node))
@@ -1357,8 +1357,9 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
 			 (last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0,
 			 pte_flags);
 
-		/* TODO: we still need to determine the vm_manager.vram_base_offset based on
-		 * the memory partition.
+		/* For dGPU mode, we use same vm_manager to allocate VRAM for
+		 * different memory partition based on fpfn/lpfn, we should use
+		 * same vm_manager.vram_base_offset regardless memory partition.
 		 */
 		r = amdgpu_vm_update_range(adev, vm, false, false, flush_tlb, NULL,
 					   last_start, prange->start + i,
-- 
2.40.1

[PATCH 19/29] drm/amdgpu: Alloc page table on correct memory partition

[Alex Deucher]

From: Philip Yang <Philip.Yang@amd.com>

Alloc kernel mode page table bo uses the amdgpu_vm->mem_id + 1 as bp
mem_id_plus1 parameter. For APU mode, select the correct TTM pool to
alloc page from the corresponding memory partition, this will be the
closest NUMA node. For dGPU mode, select the correct address range for
vram manager.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index 60b1da93b06d..62fc7e8d326e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -534,6 +534,8 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 
 	bp.type = ttm_bo_type_kernel;
 	bp.no_wait_gpu = immediate;
+	bp.mem_id_plus1 = vm->mem_id + 1;
+
 	if (vm->root.bo)
 		bp.resv = vm->root.bo->tbo.base.resv;
 
@@ -558,6 +560,7 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	bp.type = ttm_bo_type_kernel;
 	bp.resv = bo->tbo.base.resv;
 	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
+	bp.mem_id_plus1 = vm->mem_id + 1;
 
 	r = amdgpu_bo_create(adev, &bp, &(*vmbo)->shadow);
 
-- 
2.40.1

[PATCH 20/29] drm/amdgpu: dGPU mode set VRAM range lpfn as exclusive

[Alex Deucher]

From: Philip Yang <Philip.Yang@amd.com>

TTM place lpfn is exclusive used as end (start + size) in drm and buddy
allocator, adev->gmc memory partition range lpfn is inclusive (start +
size - 1), should plus 1 to set TTM place lpfn.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index cfa14b56c419..3002d431ce3d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -134,7 +134,11 @@ void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
 
 		if (adev->gmc.mem_partitions && abo->mem_id >= 0) {
 			places[c].fpfn = adev->gmc.mem_partitions[abo->mem_id].range.fpfn;
-			places[c].lpfn = adev->gmc.mem_partitions[abo->mem_id].range.lpfn;
+			/*
+			 * memory partition range lpfn is inclusive start + size - 1
+			 * TTM place lpfn is exclusive start + size
+			 */
+			places[c].lpfn = adev->gmc.mem_partitions[abo->mem_id].range.lpfn + 1;
 		} else {
 			places[c].fpfn = 0;
 			places[c].lpfn = 0;
-- 
2.40.1

[PATCH 21/29] drm/amdkfd: Store xcp partition id to amdgpu bo

[Alex Deucher]

From: Philip Yang <Philip.Yang@amd.com>

For memory accounting per compute partition and export drm amdgpu bo and
then import to KFD, we need the xcp id to account the memory usage or
find the KFD node of the original amdgpu bo to create the KFD bo on the
correct adev KFD node.

Set xcp_id_plus1 of amdgpu_bo_param to create bo and store xcp_id to
amddgpu bo. Add helper macro to get the mem_id from adev and xcp_id.

v2: squash in fix ("drm/amdgpu: Fix BO creation failure on GFX 9.4.3 dGPU")

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h       |  4 ++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 11 ++++++-----
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c          |  4 ++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h          |  2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.c       | 15 ++++++++++-----
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.h       | 12 ++++++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c          |  6 +++++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c        |  5 +++--
 drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c            |  2 +-
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c             |  4 ++--
 10 files changed, 42 insertions(+), 23 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 324cb566ca2f..05c54776951b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -330,6 +330,10 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
 		uint64_t size, u32 alloc_flag);
 
+#define KFD_XCP_MEM_ID(adev, xcp_id) \
+		((adev)->xcp_mgr && (xcp_id) >= 0 ?\
+		(adev)->xcp_mgr->xcp[(xcp_id)].mem_id : -1)
+
 #define KFD_XCP_MEMORY_SIZE(n) ((n)->adev->gmc.num_mem_partitions ?\
 		(n)->adev->gmc.mem_partitions[(n)->xcp->mem_id].size /\
 		(n)->adev->xcp_mgr->num_xcp_per_mem_partition :\
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 71b22d61dd27..cf8f80e4ef56 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1633,6 +1633,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 		uint64_t *offset, uint32_t flags, bool criu_resume)
 {
 	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
+	struct amdgpu_fpriv *fpriv = container_of(avm, struct amdgpu_fpriv, vm);
 	enum ttm_bo_type bo_type = ttm_bo_type_device;
 	struct sg_table *sg = NULL;
 	uint64_t user_addr = 0;
@@ -1640,7 +1641,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 	struct drm_gem_object *gobj = NULL;
 	u32 domain, alloc_domain;
 	uint64_t aligned_size;
-	int8_t mem_id = -1;
+	int8_t xcp_id = -1;
 	u64 alloc_flags;
 	int ret;
 
@@ -1659,7 +1660,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 			alloc_flags |= (flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) ?
 			AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED : 0;
 		}
-		mem_id = avm->mem_id;
+		xcp_id = fpriv->xcp_id == ~0 ? 0 : fpriv->xcp_id;
 	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
 		domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
 		alloc_flags = 0;
@@ -1717,12 +1718,12 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 		goto err_reserve_limit;
 	}
 
-	pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s mem_id %d\n",
+	pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s xcp_id %d\n",
 		 va, (*mem)->aql_queue ? size << 1 : size,
-		 domain_string(alloc_domain), mem_id);
+		 domain_string(alloc_domain), xcp_id);
 
 	ret = amdgpu_gem_object_create(adev, aligned_size, 1, alloc_domain, alloc_flags,
-				       bo_type, NULL, &gobj, mem_id + 1);
+				       bo_type, NULL, &gobj, xcp_id + 1);
 	if (ret) {
 		pr_debug("Failed to create BO on domain %s. ret %d\n",
 			 domain_string(alloc_domain), ret);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 01029b495f5a..b02d106d5a0c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -97,7 +97,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 			     int alignment, u32 initial_domain,
 			     u64 flags, enum ttm_bo_type type,
 			     struct dma_resv *resv,
-			     struct drm_gem_object **obj, int8_t mem_id_plus1)
+			     struct drm_gem_object **obj, int8_t xcp_id_plus1)
 {
 	struct amdgpu_bo *bo;
 	struct amdgpu_bo_user *ubo;
@@ -115,7 +115,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 	bp.flags = flags;
 	bp.domain = initial_domain;
 	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
-	bp.mem_id_plus1 = mem_id_plus1;
+	bp.xcp_id_plus1 = xcp_id_plus1;
 
 	r = amdgpu_bo_create_user(adev, &bp, &ubo);
 	if (r)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h
index 646c4fcc8e40..f30264782ba2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h
@@ -43,7 +43,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 			     int alignment, u32 initial_domain,
 			     u64 flags, enum ttm_bo_type type,
 			     struct dma_resv *resv,
-			     struct drm_gem_object **obj, int8_t mem_id_plus1);
+			     struct drm_gem_object **obj, int8_t xcp_id_plus1);
 int amdgpu_mode_dumb_create(struct drm_file *file_priv,
 			    struct drm_device *dev,
 			    struct drm_mode_create_dumb *args);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 3002d431ce3d..ac25314064da 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -131,14 +131,15 @@ void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
 
 	if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
 		unsigned visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
+		int8_t mem_id = KFD_XCP_MEM_ID(adev, abo->xcp_id);
 
-		if (adev->gmc.mem_partitions && abo->mem_id >= 0) {
-			places[c].fpfn = adev->gmc.mem_partitions[abo->mem_id].range.fpfn;
+		if (adev->gmc.mem_partitions && mem_id >= 0) {
+			places[c].fpfn = adev->gmc.mem_partitions[mem_id].range.fpfn;
 			/*
 			 * memory partition range lpfn is inclusive start + size - 1
 			 * TTM place lpfn is exclusive start + size
 			 */
-			places[c].lpfn = adev->gmc.mem_partitions[abo->mem_id].range.lpfn + 1;
+			places[c].lpfn = adev->gmc.mem_partitions[mem_id].range.lpfn + 1;
 		} else {
 			places[c].fpfn = 0;
 			places[c].lpfn = 0;
@@ -583,8 +584,12 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
 
 	bo->flags = bp->flags;
 
-	/* bo->mem_id -1 means any partition */
-	bo->mem_id = bp->mem_id_plus1 - 1;
+	if (adev->gmc.mem_partitions)
+		/* For GPUs with spatial partitioning, bo->xcp_id=-1 means any partition */
+		bo->xcp_id = bp->xcp_id_plus1 - 1;
+	else
+		/* For GPUs without spatial partitioning */
+		bo->xcp_id = 0;
 
 	if (!amdgpu_bo_support_uswc(bo->flags))
 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 521a432348a0..5d3440d719e4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -56,8 +56,8 @@ struct amdgpu_bo_param {
 	bool				no_wait_gpu;
 	struct dma_resv			*resv;
 	void				(*destroy)(struct ttm_buffer_object *bo);
-	/* memory partition number plus 1, 0 means any partition */
-	int8_t				mem_id_plus1;
+	/* xcp partition number plus 1, 0 means any partition */
+	int8_t				xcp_id_plus1;
 };
 
 /* bo virtual addresses in a vm */
@@ -111,8 +111,12 @@ struct amdgpu_bo {
 #endif
 	struct kgd_mem                  *kfd_bo;
 
-	/* memory partition number, -1 means any partition */
-	int8_t				mem_id;
+	/*
+	 * For GPUs with spatial partitioning, xcp partition number, -1 means
+	 * any partition. For other ASICs without spatial partition, always 0
+	 * for memory accounting.
+	 */
+	int8_t				xcp_id;
 };
 
 struct amdgpu_bo_user {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 395edca3b7f9..ad664ef640ff 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1055,6 +1055,7 @@ static void amdgpu_ttm_backend_destroy(struct ttm_device *bdev,
 static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
 					   uint32_t page_flags)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
 	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
 	struct amdgpu_ttm_tt *gtt;
 	enum ttm_caching caching;
@@ -1064,7 +1065,10 @@ static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
 		return NULL;
 	}
 	gtt->gobj = &bo->base;
-	gtt->pool_id = abo->mem_id;
+	if (adev->gmc.mem_partitions && abo->xcp_id >= 0)
+		gtt->pool_id = KFD_XCP_MEM_ID(adev, abo->xcp_id);
+	else
+		gtt->pool_id = abo->xcp_id;
 
 	if (abo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
 		caching = ttm_write_combined;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index 62fc7e8d326e..cc3b1b596e56 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -502,6 +502,7 @@ int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			int level, bool immediate, struct amdgpu_bo_vm **vmbo)
 {
+	struct amdgpu_fpriv *fpriv = container_of(vm, struct amdgpu_fpriv, vm);
 	struct amdgpu_bo_param bp;
 	struct amdgpu_bo *bo;
 	struct dma_resv *resv;
@@ -534,7 +535,7 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 
 	bp.type = ttm_bo_type_kernel;
 	bp.no_wait_gpu = immediate;
-	bp.mem_id_plus1 = vm->mem_id + 1;
+	bp.xcp_id_plus1 = fpriv->xcp_id == ~0 ? 0 : fpriv->xcp_id + 1;
 
 	if (vm->root.bo)
 		bp.resv = vm->root.bo->tbo.base.resv;
@@ -560,7 +561,7 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	bp.type = ttm_bo_type_kernel;
 	bp.resv = bo->tbo.base.resv;
 	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
-	bp.mem_id_plus1 = vm->mem_id + 1;
+	bp.xcp_id_plus1 = fpriv->xcp_id == ~0 ? 0 : fpriv->xcp_id + 1;
 
 	r = amdgpu_bo_create(adev, &bp, &(*vmbo)->shadow);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index ee5d4d67b423..b860ba503ddd 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -1253,7 +1253,7 @@ static void gmc_v9_0_get_coherence_flags(struct amdgpu_device *adev,
 		is_local = (!is_vram && (adev->flags & AMD_IS_APU) &&
 			    num_possible_nodes() <= 1) ||
 			   (is_vram && adev == bo_adev &&
-			    bo->mem_id == vm->mem_id);
+			    KFD_XCP_MEM_ID(adev, bo->xcp_id) == vm->mem_id);
 		snoop = true;
 		if (uncached) {
 			mtype = MTYPE_UC;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 8b5453fd304a..872daaef0258 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -555,7 +555,7 @@ svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
 	bp.type = ttm_bo_type_device;
 	bp.resv = NULL;
 	if (node->xcp)
-		bp.mem_id_plus1 = node->xcp->mem_id + 1;
+		bp.xcp_id_plus1 = node->xcp->id + 1;
 
 	r = amdgpu_bo_create_user(node->adev, &bp, &ubo);
 	if (r) {
@@ -566,7 +566,7 @@ svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
 
 	pr_debug("alloc bo at offset 0x%lx size 0x%lx on partition %d\n",
 		 bo->tbo.resource->start << PAGE_SHIFT, bp.size,
-		 bp.mem_id_plus1 - 1);
+		 bp.xcp_id_plus1 - 1);
 
 	r = amdgpu_bo_reserve(bo, true);
 	if (r) {
-- 
2.40.1

[PATCH 22/29] drm/amdgpu: KFD graphics interop support compute partition

[Alex Deucher]

From: Philip Yang <Philip.Yang@amd.com>

kfd_ioctl_get_dmabuf use the amdgpu bo xcp_id to get the gpu_id of the
KFD node from the exported dmabuf_adev, and then create kfd bo on the
correct adev and KFD node when importing the amdgpu bo to KFD.

Remove function kfd_device_by_adev, it is not needed as it is the same
result as dmabuf_adev->kfd.dev->nodes[0]->id.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c |  4 +++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h |  2 +-
 drivers/gpu/drm/amd/amdkfd/kfd_chardev.c   | 14 ++++++--------
 drivers/gpu/drm/amd/amdkfd/kfd_priv.h      |  1 -
 drivers/gpu/drm/amd/amdkfd/kfd_topology.c  | 18 ------------------
 5 files changed, 10 insertions(+), 29 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index bbbfe9ec4adf..00edb13d2124 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -498,7 +498,7 @@ int amdgpu_amdkfd_get_dmabuf_info(struct amdgpu_device *adev, int dma_buf_fd,
 				  struct amdgpu_device **dmabuf_adev,
 				  uint64_t *bo_size, void *metadata_buffer,
 				  size_t buffer_size, uint32_t *metadata_size,
-				  uint32_t *flags)
+				  uint32_t *flags, int8_t *xcp_id)
 {
 	struct dma_buf *dma_buf;
 	struct drm_gem_object *obj;
@@ -542,6 +542,8 @@ int amdgpu_amdkfd_get_dmabuf_info(struct amdgpu_device *adev, int dma_buf_fd,
 		if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
 			*flags |= KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC;
 	}
+	if (xcp_id)
+		*xcp_id = bo->xcp_id;
 
 out_put:
 	dma_buf_put(dma_buf);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 05c54776951b..4e6221bccffe 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -241,7 +241,7 @@ int amdgpu_amdkfd_get_dmabuf_info(struct amdgpu_device *adev, int dma_buf_fd,
 				  struct amdgpu_device **dmabuf_adev,
 				  uint64_t *bo_size, void *metadata_buffer,
 				  size_t buffer_size, uint32_t *metadata_size,
-				  uint32_t *flags);
+				  uint32_t *flags, int8_t *xcp_id);
 uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct amdgpu_device *dst,
 					  struct amdgpu_device *src);
 int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct amdgpu_device *dst,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 8c86d69938ea..344b238d6771 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -1499,6 +1499,7 @@ static int kfd_ioctl_get_dmabuf_info(struct file *filep,
 	struct amdgpu_device *dmabuf_adev;
 	void *metadata_buffer = NULL;
 	uint32_t flags;
+	int8_t xcp_id;
 	unsigned int i;
 	int r;
 
@@ -1519,17 +1520,14 @@ static int kfd_ioctl_get_dmabuf_info(struct file *filep,
 	r = amdgpu_amdkfd_get_dmabuf_info(dev->adev, args->dmabuf_fd,
 					  &dmabuf_adev, &args->size,
 					  metadata_buffer, args->metadata_size,
-					  &args->metadata_size, &flags);
+					  &args->metadata_size, &flags, &xcp_id);
 	if (r)
 		goto exit;
 
-	/* Reverse-lookup gpu_id from kgd pointer */
-	dev = kfd_device_by_adev(dmabuf_adev);
-	if (!dev) {
-		r = -EINVAL;
-		goto exit;
-	}
-	args->gpu_id = dev->id;
+	if (xcp_id >= 0)
+		args->gpu_id = dmabuf_adev->kfd.dev->nodes[xcp_id]->id;
+	else
+		args->gpu_id = dmabuf_adev->kfd.dev->nodes[0]->id;
 	args->flags = flags;
 
 	/* Copy metadata buffer to user mode */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 214d950f948e..44f4d5509db6 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -1068,7 +1068,6 @@ struct kfd_topology_device *kfd_topology_device_by_proximity_domain_no_lock(
 struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id);
 struct kfd_node *kfd_device_by_id(uint32_t gpu_id);
 struct kfd_node *kfd_device_by_pci_dev(const struct pci_dev *pdev);
-struct kfd_node *kfd_device_by_adev(const struct amdgpu_device *adev);
 static inline bool kfd_irq_is_from_node(struct kfd_node *node, uint32_t node_id,
 					uint32_t vmid)
 {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index a8e25aecf839..dbb6159344b3 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -125,24 +125,6 @@ struct kfd_node *kfd_device_by_pci_dev(const struct pci_dev *pdev)
 	return device;
 }
 
-struct kfd_node *kfd_device_by_adev(const struct amdgpu_device *adev)
-{
-	struct kfd_topology_device *top_dev;
-	struct kfd_node *device = NULL;
-
-	down_read(&topology_lock);
-
-	list_for_each_entry(top_dev, &topology_device_list, list)
-		if (top_dev->gpu && top_dev->gpu->adev == adev) {
-			device = top_dev->gpu;
-			break;
-		}
-
-	up_read(&topology_lock);
-
-	return device;
-}
-
 /* Called with write topology_lock acquired */
 static void kfd_release_topology_device(struct kfd_topology_device *dev)
 {
-- 
2.40.1

[PATCH 23/29] drm/amdgpu: use xcp partition ID for amdgpu_gem

[Alex Deucher]

From: James Zhu <James.Zhu@amd.com>

Find xcp_id from amdgpu_fpriv, use it for amdgpu_gem_object_create.

Signed-off-by: James Zhu <James.Zhu@amd.com>
Acked-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index b02d106d5a0c..aad860667ab5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -336,7 +336,7 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
 retry:
 	r = amdgpu_gem_object_create(adev, size, args->in.alignment,
 				     initial_domain,
-				     flags, ttm_bo_type_device, resv, &gobj, 0);
+				     flags, ttm_bo_type_device, resv, &gobj, fpriv->xcp_id + 1);
 	if (r && r != -ERESTARTSYS) {
 		if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
 			flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
@@ -379,6 +379,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
 	struct ttm_operation_ctx ctx = { true, false };
 	struct amdgpu_device *adev = drm_to_adev(dev);
 	struct drm_amdgpu_gem_userptr *args = data;
+	struct amdgpu_fpriv *fpriv = filp->driver_priv;
 	struct drm_gem_object *gobj;
 	struct hmm_range *range;
 	struct amdgpu_bo *bo;
@@ -405,7 +406,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
 
 	/* create a gem object to contain this object in */
 	r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU,
-				     0, ttm_bo_type_device, NULL, &gobj, 0);
+				     0, ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);
 	if (r)
 		return r;
 
@@ -908,6 +909,7 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
 			    struct drm_mode_create_dumb *args)
 {
 	struct amdgpu_device *adev = drm_to_adev(dev);
+	struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
 	struct drm_gem_object *gobj;
 	uint32_t handle;
 	u64 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
@@ -931,7 +933,7 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
 	domain = amdgpu_bo_get_preferred_domain(adev,
 				amdgpu_display_supported_domains(adev, flags));
 	r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags,
-				     ttm_bo_type_device, NULL, &gobj, 0);
+				     ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);
 	if (r)
 		return -ENOMEM;
 
-- 
2.40.1

[PATCH 24/29] drm/amdkfd: Move local_mem_info to kfd_node

[Alex Deucher]

From: Mukul Joshi <mukul.joshi@amd.com>

We need to track memory usage on a per partition basis. To do
that, store the local memory information in KFD node instead
of kfd device.

v2: squash in fix ("amdkfd: Use mem_id to access mem_partition info")

Signed-off-by: Mukul Joshi <mukul.joshi@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 17 +++++++++++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 12 +++++++-----
 drivers/gpu/drm/amd/amdkfd/kfd_chardev.c   |  7 ++++---
 drivers/gpu/drm/amd/amdkfd/kfd_crat.c      |  2 +-
 drivers/gpu/drm/amd/amdkfd/kfd_device.c    |  7 +++++--
 drivers/gpu/drm/amd/amdkfd/kfd_priv.h      |  3 ++-
 drivers/gpu/drm/amd/amdkfd/kfd_topology.c  |  7 ++++---
 7 files changed, 36 insertions(+), 19 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 00edb13d2124..85df73f2c85e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -428,14 +428,23 @@ uint32_t amdgpu_amdkfd_get_fw_version(struct amdgpu_device *adev,
 }
 
 void amdgpu_amdkfd_get_local_mem_info(struct amdgpu_device *adev,
-				      struct kfd_local_mem_info *mem_info)
+				      struct kfd_local_mem_info *mem_info,
+				      uint8_t xcp_id)
 {
 	memset(mem_info, 0, sizeof(*mem_info));
 
-	mem_info->local_mem_size_public = adev->gmc.visible_vram_size;
-	mem_info->local_mem_size_private = adev->gmc.real_vram_size -
+	if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) {
+		if (adev->gmc.real_vram_size == adev->gmc.visible_vram_size)
+			mem_info->local_mem_size_public =
+					KFD_XCP_MEMORY_SIZE(adev, xcp_id);
+		else
+			mem_info->local_mem_size_private =
+					KFD_XCP_MEMORY_SIZE(adev, xcp_id);
+	} else {
+		mem_info->local_mem_size_public = adev->gmc.visible_vram_size;
+		mem_info->local_mem_size_private = adev->gmc.real_vram_size -
 						adev->gmc.visible_vram_size;
-
+	}
 	mem_info->vram_width = adev->gmc.vram_width;
 
 	pr_debug("Address base: %pap public 0x%llx private 0x%llx\n",
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 4e6221bccffe..4bf6f5659568 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -231,7 +231,8 @@ int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem);
 uint32_t amdgpu_amdkfd_get_fw_version(struct amdgpu_device *adev,
 				      enum kgd_engine_type type);
 void amdgpu_amdkfd_get_local_mem_info(struct amdgpu_device *adev,
-				      struct kfd_local_mem_info *mem_info);
+				      struct kfd_local_mem_info *mem_info,
+				      uint8_t xcp_id);
 uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct amdgpu_device *adev);
 
 uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct amdgpu_device *adev);
@@ -334,10 +335,11 @@ void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
 		((adev)->xcp_mgr && (xcp_id) >= 0 ?\
 		(adev)->xcp_mgr->xcp[(xcp_id)].mem_id : -1)
 
-#define KFD_XCP_MEMORY_SIZE(n) ((n)->adev->gmc.num_mem_partitions ?\
-		(n)->adev->gmc.mem_partitions[(n)->xcp->mem_id].size /\
-		(n)->adev->xcp_mgr->num_xcp_per_mem_partition :\
-		(n)->adev->gmc.real_vram_size)
+#define KFD_XCP_MEMORY_SIZE(adev, xcp_id)\
+		((adev)->gmc.num_mem_partitions && (xcp_id) >= 0 ?\
+		(adev)->gmc.mem_partitions[KFD_XCP_MEM_ID((adev), (xcp_id))].size /\
+		(adev)->xcp_mgr->num_xcp_per_mem_partition :\
+		(adev)->gmc.real_vram_size)
 
 #if IS_ENABLED(CONFIG_HSA_AMD)
 void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 344b238d6771..089e1d498670 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -1023,11 +1023,12 @@ bool kfd_dev_is_large_bar(struct kfd_node *dev)
 	if (dev->kfd->use_iommu_v2)
 		return false;
 
-	if (dev->kfd->local_mem_info.local_mem_size_private == 0 &&
-	    dev->kfd->local_mem_info.local_mem_size_public > 0)
+	if (dev->local_mem_info.local_mem_size_private == 0 &&
+	    dev->local_mem_info.local_mem_size_public > 0)
 		return true;
 
-	if (dev->kfd->local_mem_info.local_mem_size_public == 0 && dev->kfd->adev->gmc.is_app_apu) {
+	if (dev->local_mem_info.local_mem_size_public == 0 &&
+	    dev->kfd->adev->gmc.is_app_apu) {
 		pr_debug("APP APU, Consider like a large bar system\n");
 		return true;
 	}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
index 1aaf933f9f48..950af6820153 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
@@ -2191,7 +2191,7 @@ static int kfd_create_vcrat_image_gpu(void *pcrat_image,
 	 * report the total FB size (public+private) as a single
 	 * private heap.
 	 */
-	local_mem_info = kdev->kfd->local_mem_info;
+	local_mem_info = kdev->local_mem_info;
 	sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
 			sub_type_hdr->length);
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index db5b53fcdf11..d41da964d2f5 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -726,7 +726,6 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 
 	svm_migrate_init(kfd->adev);
 
-	amdgpu_amdkfd_get_local_mem_info(kfd->adev, &kfd->local_mem_info);
 
 	dev_info(kfd_device, "Total number of KFD nodes to be created: %d\n",
 				kfd->num_nodes);
@@ -756,7 +755,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 		if (node->xcp) {
 			dev_info(kfd_device, "KFD node %d partition %d size %lldM\n",
 				node->node_id, node->xcp->mem_id,
-				KFD_XCP_MEMORY_SIZE(node) >> 20);
+				KFD_XCP_MEMORY_SIZE(node->adev, node->node_id) >> 20);
 		}
 
 		if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 3) &&
@@ -783,6 +782,10 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 		}
 		node->max_proc_per_quantum = max_proc_per_quantum;
 		atomic_set(&node->sram_ecc_flag, 0);
+
+		amdgpu_amdkfd_get_local_mem_info(kfd->adev,
+					&node->local_mem_info, node->xcp->id);
+
 		/* Initialize the KFD node */
 		if (kfd_init_node(node)) {
 			dev_err(kfd_device, "Error initializing KFD node\n");
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 44f4d5509db6..3bd222e8f6c3 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -313,6 +313,8 @@ struct kfd_node {
 
 	unsigned int compute_vmid_bitmap;
 
+	struct kfd_local_mem_info local_mem_info;
+
 	struct kfd_dev *kfd;
 };
 
@@ -335,7 +337,6 @@ struct kfd_dev {
 					   */
 
 	struct kgd2kfd_shared_resources shared_resources;
-	struct kfd_local_mem_info local_mem_info;
 
 	const struct kfd2kgd_calls *kfd2kgd;
 	struct mutex doorbell_mutex;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index dbb6159344b3..e0bacf017a40 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -1152,8 +1152,8 @@ static uint32_t kfd_generate_gpu_id(struct kfd_node *gpu)
 	if (!gpu)
 		return 0;
 
-	local_mem_size = gpu->kfd->local_mem_info.local_mem_size_private +
-			gpu->kfd->local_mem_info.local_mem_size_public;
+	local_mem_size = gpu->local_mem_info.local_mem_size_private +
+			gpu->local_mem_info.local_mem_size_public;
 	buf[0] = gpu->adev->pdev->devfn;
 	buf[1] = gpu->adev->pdev->subsystem_vendor |
 		(gpu->adev->pdev->subsystem_device << 16);
@@ -1234,7 +1234,8 @@ static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev)
 	 * for APUs - If CRAT from ACPI reports more than one bank, then
 	 *	all the banks will report the same mem_clk_max information
 	 */
-	amdgpu_amdkfd_get_local_mem_info(dev->gpu->adev, &local_mem_info);
+	amdgpu_amdkfd_get_local_mem_info(dev->gpu->adev, &local_mem_info,
+					 dev->gpu->xcp->id);
 
 	list_for_each_entry(mem, &dev->mem_props, list)
 		mem->mem_clk_max = local_mem_info.mem_clk_max;
-- 
2.40.1

[PATCH 25/29] drm/amdkfd: Fix memory reporting on GFX 9.4.3

[Alex Deucher]

From: Mukul Joshi <mukul.joshi@amd.com>

This patch fixes memory reporting on the GFX 9.4.3 APU and dGPU
by reporting available memory on a per partition basis. If its an
APU, available and used memory calculations take into account
system and TTM memory.

v2: squash in fix ("drm/amdkfd: Fix array out of bound warning")
    squash in fix ("drm/amdgpu: Update memory reporting for GFX9.4.3")

Signed-off-by: Mukul Joshi <mukul.joshi@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h    | 12 +--
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c  | 81 ++++++++++++++-----
 drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h       |  5 ++
 drivers/gpu/drm/amd/amdkfd/kfd_chardev.c      |  3 +-
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c          | 14 ++--
 5 files changed, 84 insertions(+), 31 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 4bf6f5659568..948d362adabb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -35,6 +35,7 @@
 #include <drm/ttm/ttm_execbuf_util.h>
 #include "amdgpu_sync.h"
 #include "amdgpu_vm.h"
+#include "amdgpu_xcp.h"
 
 extern uint64_t amdgpu_amdkfd_total_mem_size;
 
@@ -98,8 +99,8 @@ struct amdgpu_amdkfd_fence {
 
 struct amdgpu_kfd_dev {
 	struct kfd_dev *dev;
-	int64_t vram_used;
-	uint64_t vram_used_aligned;
+	int64_t vram_used[MAX_XCP];
+	uint64_t vram_used_aligned[MAX_XCP];
 	bool init_complete;
 	struct work_struct reset_work;
 
@@ -287,7 +288,8 @@ int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
 void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev,
 					void *drm_priv);
 uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv);
-size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev);
+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev,
+					uint8_t xcp_id);
 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 		struct amdgpu_device *adev, uint64_t va, uint64_t size,
 		void *drm_priv, struct kgd_mem **mem,
@@ -327,9 +329,9 @@ void amdgpu_amdkfd_block_mmu_notifications(void *p);
 int amdgpu_amdkfd_criu_resume(void *p);
 bool amdgpu_amdkfd_ras_query_utcl2_poison_status(struct amdgpu_device *adev);
 int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
-		uint64_t size, u32 alloc_flag);
+		uint64_t size, u32 alloc_flag, int8_t xcp_id);
 void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
-		uint64_t size, u32 alloc_flag);
+		uint64_t size, u32 alloc_flag, int8_t xcp_id);
 
 #define KFD_XCP_MEM_ID(adev, xcp_id) \
 		((adev)->xcp_mgr && (xcp_id) >= 0 ?\
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index cf8f80e4ef56..fa4057da0d7f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -156,12 +156,13 @@ void amdgpu_amdkfd_reserve_system_mem(uint64_t size)
  * Return: returns -ENOMEM in case of error, ZERO otherwise
  */
 int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
-		uint64_t size, u32 alloc_flag)
+		uint64_t size, u32 alloc_flag, int8_t xcp_id)
 {
 	uint64_t reserved_for_pt =
 		ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
 	size_t system_mem_needed, ttm_mem_needed, vram_needed;
 	int ret = 0;
+	uint64_t vram_size = 0;
 
 	system_mem_needed = 0;
 	ttm_mem_needed = 0;
@@ -176,6 +177,17 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 		 * 2M BO chunk.
 		 */
 		vram_needed = size;
+		/*
+		 * For GFX 9.4.3, get the VRAM size from XCP structs
+		 */
+		if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id))
+			return -EINVAL;
+
+		vram_size = KFD_XCP_MEMORY_SIZE(adev, xcp_id);
+		if (adev->gmc.is_app_apu) {
+			system_mem_needed = size;
+			ttm_mem_needed = size;
+		}
 	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
 		system_mem_needed = size;
 	} else if (!(alloc_flag &
@@ -195,8 +207,8 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 	     kfd_mem_limit.max_system_mem_limit && !no_system_mem_limit) ||
 	    (kfd_mem_limit.ttm_mem_used + ttm_mem_needed >
 	     kfd_mem_limit.max_ttm_mem_limit) ||
-	    (adev && adev->kfd.vram_used + vram_needed >
-	     adev->gmc.real_vram_size - reserved_for_pt)) {
+	    (adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] + vram_needed >
+	     vram_size - reserved_for_pt)) {
 		ret = -ENOMEM;
 		goto release;
 	}
@@ -206,9 +218,11 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 	 */
 	WARN_ONCE(vram_needed && !adev,
 		  "adev reference can't be null when vram is used");
-	if (adev) {
-		adev->kfd.vram_used += vram_needed;
-		adev->kfd.vram_used_aligned += ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN);
+	if (adev && xcp_id >= 0) {
+		adev->kfd.vram_used[xcp_id] += vram_needed;
+		adev->kfd.vram_used_aligned[xcp_id] += adev->gmc.is_app_apu ?
+				vram_needed :
+				ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN);
 	}
 	kfd_mem_limit.system_mem_used += system_mem_needed;
 	kfd_mem_limit.ttm_mem_used += ttm_mem_needed;
@@ -219,7 +233,7 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 }
 
 void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
-		uint64_t size, u32 alloc_flag)
+		uint64_t size, u32 alloc_flag, int8_t xcp_id)
 {
 	spin_lock(&kfd_mem_limit.mem_limit_lock);
 
@@ -229,9 +243,19 @@ void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
 	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
 		WARN_ONCE(!adev,
 			  "adev reference can't be null when alloc mem flags vram is set");
+		if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id))
+			goto release;
+
 		if (adev) {
-			adev->kfd.vram_used -= size;
-			adev->kfd.vram_used_aligned -= ALIGN(size, VRAM_AVAILABLITY_ALIGN);
+			adev->kfd.vram_used[xcp_id] -= size;
+			if (adev->gmc.is_app_apu) {
+				adev->kfd.vram_used_aligned[xcp_id] -= size;
+				kfd_mem_limit.system_mem_used -= size;
+				kfd_mem_limit.ttm_mem_used -= size;
+			} else {
+				adev->kfd.vram_used_aligned[xcp_id] -=
+					ALIGN(size, VRAM_AVAILABLITY_ALIGN);
+			}
 		}
 	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
 		kfd_mem_limit.system_mem_used -= size;
@@ -241,8 +265,8 @@ void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
 		pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);
 		goto release;
 	}
-	WARN_ONCE(adev && adev->kfd.vram_used < 0,
-		  "KFD VRAM memory accounting unbalanced");
+	WARN_ONCE(adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] < 0,
+		  "KFD VRAM memory accounting unbalanced for xcp: %d", xcp_id);
 	WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0,
 		  "KFD TTM memory accounting unbalanced");
 	WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
@@ -258,7 +282,8 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
 	u32 alloc_flags = bo->kfd_bo->alloc_flags;
 	u64 size = amdgpu_bo_size(bo);
 
-	amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flags);
+	amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flags,
+					  bo->xcp_id);
 
 	kfree(bo->kfd_bo);
 }
@@ -1608,23 +1633,42 @@ int amdgpu_amdkfd_criu_resume(void *p)
 	return ret;
 }
 
-size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev)
+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev,
+					  uint8_t xcp_id)
 {
 	uint64_t reserved_for_pt =
 		ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
 	ssize_t available;
+	uint64_t vram_available, system_mem_available, ttm_mem_available;
 
 	spin_lock(&kfd_mem_limit.mem_limit_lock);
-	available = adev->gmc.real_vram_size
-		- adev->kfd.vram_used_aligned
+	vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id)
+		- adev->kfd.vram_used_aligned[xcp_id]
 		- atomic64_read(&adev->vram_pin_size)
 		- reserved_for_pt;
+
+	if (adev->gmc.is_app_apu) {
+		system_mem_available = no_system_mem_limit ?
+					kfd_mem_limit.max_system_mem_limit :
+					kfd_mem_limit.max_system_mem_limit -
+					kfd_mem_limit.system_mem_used;
+
+		ttm_mem_available = kfd_mem_limit.max_ttm_mem_limit -
+				kfd_mem_limit.ttm_mem_used;
+
+		available = min3(system_mem_available, ttm_mem_available,
+				 vram_available);
+		available = ALIGN_DOWN(available, PAGE_SIZE);
+	} else {
+		available = ALIGN_DOWN(vram_available, VRAM_AVAILABLITY_ALIGN);
+	}
+
 	spin_unlock(&kfd_mem_limit.mem_limit_lock);
 
 	if (available < 0)
 		available = 0;
 
-	return ALIGN_DOWN(available, VRAM_AVAILABLITY_ALIGN);
+	return available;
 }
 
 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
@@ -1712,7 +1756,8 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 
 	amdgpu_sync_create(&(*mem)->sync);
 
-	ret = amdgpu_amdkfd_reserve_mem_limit(adev, aligned_size, flags);
+	ret = amdgpu_amdkfd_reserve_mem_limit(adev, aligned_size, flags,
+					      xcp_id);
 	if (ret) {
 		pr_debug("Insufficient memory\n");
 		goto err_reserve_limit;
@@ -1780,7 +1825,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 	/* Don't unreserve system mem limit twice */
 	goto err_reserve_limit;
 err_bo_create:
-	amdgpu_amdkfd_unreserve_mem_limit(adev, aligned_size, flags);
+	amdgpu_amdkfd_unreserve_mem_limit(adev, aligned_size, flags, xcp_id);
 err_reserve_limit:
 	mutex_destroy(&(*mem)->lock);
 	if (gobj)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
index 68b63b970ce8..9c5912b9d8bd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
@@ -24,8 +24,11 @@
 #ifndef AMDGPU_XCP_H
 #define AMDGPU_XCP_H
 
+#include <linux/pci.h>
 #include <linux/xarray.h>
 
+#include "amdgpu_ctx.h"
+
 #define MAX_XCP 8
 
 #define AMDGPU_XCP_MODE_NONE -1
@@ -34,6 +37,8 @@
 #define AMDGPU_XCP_FL_NONE 0
 #define AMDGPU_XCP_FL_LOCKED (1 << 0)
 
+struct amdgpu_fpriv;
+
 enum AMDGPU_XCP_IP_BLOCK {
 	AMDGPU_XCP_GFXHUB,
 	AMDGPU_XCP_GFX,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 089e1d498670..88fe1f31739d 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -1044,7 +1044,8 @@ static int kfd_ioctl_get_available_memory(struct file *filep,
 
 	if (!pdd)
 		return -EINVAL;
-	args->available = amdgpu_amdkfd_get_available_memory(pdd->dev->adev);
+	args->available = amdgpu_amdkfd_get_available_memory(pdd->dev->adev,
+							pdd->dev->node_id);
 	kfd_unlock_pdd(pdd);
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 872daaef0258..2dbbdad3f392 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -279,7 +279,7 @@ static void svm_range_free(struct svm_range *prange, bool update_mem_usage)
 	if (update_mem_usage && !p->xnack_enabled) {
 		pr_debug("unreserve prange 0x%p size: 0x%llx\n", prange, size);
 		amdgpu_amdkfd_unreserve_mem_limit(NULL, size,
-					KFD_IOC_ALLOC_MEM_FLAGS_USERPTR);
+					KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, 0);
 	}
 	mutex_destroy(&prange->lock);
 	mutex_destroy(&prange->migrate_mutex);
@@ -312,7 +312,7 @@ svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start,
 	p = container_of(svms, struct kfd_process, svms);
 	if (!p->xnack_enabled && update_mem_usage &&
 	    amdgpu_amdkfd_reserve_mem_limit(NULL, size << PAGE_SHIFT,
-					    KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)) {
+				    KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, 0)) {
 		pr_info("SVM mapping failed, exceeds resident system memory limit\n");
 		kfree(prange);
 		return NULL;
@@ -3036,10 +3036,10 @@ svm_range_switch_xnack_reserve_mem(struct kfd_process *p, bool xnack_enabled)
 			size = (pchild->last - pchild->start + 1) << PAGE_SHIFT;
 			if (xnack_enabled) {
 				amdgpu_amdkfd_unreserve_mem_limit(NULL, size,
-						KFD_IOC_ALLOC_MEM_FLAGS_USERPTR);
+					KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, 0);
 			} else {
 				r = amdgpu_amdkfd_reserve_mem_limit(NULL, size,
-						KFD_IOC_ALLOC_MEM_FLAGS_USERPTR);
+					KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, 0);
 				if (r)
 					goto out_unlock;
 				reserved_size += size;
@@ -3049,10 +3049,10 @@ svm_range_switch_xnack_reserve_mem(struct kfd_process *p, bool xnack_enabled)
 		size = (prange->last - prange->start + 1) << PAGE_SHIFT;
 		if (xnack_enabled) {
 			amdgpu_amdkfd_unreserve_mem_limit(NULL, size,
-						KFD_IOC_ALLOC_MEM_FLAGS_USERPTR);
+					KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, 0);
 		} else {
 			r = amdgpu_amdkfd_reserve_mem_limit(NULL, size,
-						KFD_IOC_ALLOC_MEM_FLAGS_USERPTR);
+					KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, 0);
 			if (r)
 				goto out_unlock;
 			reserved_size += size;
@@ -3065,7 +3065,7 @@ svm_range_switch_xnack_reserve_mem(struct kfd_process *p, bool xnack_enabled)
 
 	if (r)
 		amdgpu_amdkfd_unreserve_mem_limit(NULL, reserved_size,
-						KFD_IOC_ALLOC_MEM_FLAGS_USERPTR);
+					KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, 0);
 	else
 		/* Change xnack mode must be inside svms lock, to avoid race with
 		 * svm_range_deferred_list_work unreserve memory in parallel.
-- 
2.40.1

[PATCH 26/29] drm/amdkfd: APU mode set max svm range pages

[Alex Deucher]

From: Philip Yang <Philip.Yang@amd.com>

svm_migrate_init set the max svm range pages based on the KFD nodes
partition size. APU mode don't init pgmap because there is no migration.

kgd2kfd_device_init calls svm_migrate_init after KFD nodes allocation
and initialization.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdkfd/kfd_device.c  |  5 ++---
 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c |  7 +++++--
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c     | 15 ++++++++++-----
 3 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index d41da964d2f5..882ff86bba08 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -724,9 +724,6 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 
 	kfd_cwsr_init(kfd);
 
-	svm_migrate_init(kfd->adev);
-
-
 	dev_info(kfd_device, "Total number of KFD nodes to be created: %d\n",
 				kfd->num_nodes);
 
@@ -794,6 +791,8 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 		kfd->nodes[i] = node;
 	}
 
+	svm_migrate_init(kfd->adev);
+
 	if (kfd_resume_iommu(kfd))
 		goto kfd_resume_iommu_error;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 199d32c7c289..2512bf681112 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -1000,6 +1000,11 @@ int svm_migrate_init(struct amdgpu_device *adev)
 	if (!KFD_IS_SOC15(kfddev->dev))
 		return -EINVAL;
 
+	svm_range_set_max_pages(adev);
+
+	if (adev->gmc.is_app_apu)
+		return 0;
+
 	pgmap = &kfddev->pgmap;
 	memset(pgmap, 0, sizeof(*pgmap));
 
@@ -1042,8 +1047,6 @@ int svm_migrate_init(struct amdgpu_device *adev)
 
 	amdgpu_amdkfd_reserve_system_mem(SVM_HMM_PAGE_STRUCT_SIZE(size));
 
-	svm_range_set_max_pages(adev);
-
 	pr_info("HMM registered %ldMB device memory\n", size >> 20);
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 2dbbdad3f392..41dacc015983 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -1937,14 +1937,19 @@ void svm_range_set_max_pages(struct amdgpu_device *adev)
 {
 	uint64_t max_pages;
 	uint64_t pages, _pages;
+	uint64_t min_pages = 0;
+	int i;
+
+	for (i = 0; i < adev->kfd.dev->num_nodes; i++) {
+		pages = KFD_XCP_MEMORY_SIZE(adev, adev->kfd.dev->nodes[i]->xcp->id) >> 17;
+		pages = clamp(pages, 1ULL << 9, 1ULL << 18);
+		pages = rounddown_pow_of_two(pages);
+		min_pages = min_not_zero(min_pages, pages);
+	}
 
-	/* 1/32 VRAM size in pages */
-	pages = adev->gmc.real_vram_size >> 17;
-	pages = clamp(pages, 1ULL << 9, 1ULL << 18);
-	pages = rounddown_pow_of_two(pages);
 	do {
 		max_pages = READ_ONCE(max_svm_range_pages);
-		_pages = min_not_zero(max_pages, pages);
+		_pages = min_not_zero(max_pages, min_pages);
 	} while (cmpxchg(&max_svm_range_pages, max_pages, _pages) != max_pages);
 }
 
-- 
2.40.1

[PATCH 27/29] drm/amdgpu: route ioctls on primary node of XCPs to primary device

[Alex Deucher]

From: Shiwu Zhang <shiwu.zhang@amd.com>

During XCP init, unlike the primary device, there is no amdgpu_device
attached to each XCP's drm_device

In case that user trying to open/close the primary node of XCP drm_device
this rerouting is to solve the NULL pointer issue causing by referring
to any member of the amdgpu_device

 BUG: unable to handle page fault for address: 0000000000020c80
 #PF: supervisor write access in kernel mode
 #PF: error_code(0x0002) - not-present page
 Oops: 0002 [#1] PREEMPT SMP NOPTI
 Call Trace:
  <TASK>
  lock_timer_base+0x6b/0x90
  try_to_del_timer_sync+0x2b/0x80
  del_timer_sync+0x29/0x40
  flush_delayed_work+0x1c/0x50
  amdgpu_driver_open_kms+0x2c/0x280 [amdgpu]
  drm_file_alloc+0x1b3/0x260 [drm]
  drm_open+0xaa/0x280 [drm]
  drm_stub_open+0xa2/0x120 [drm]
  chrdev_open+0xa6/0x1c0

Signed-off-by: Shiwu Zhang <shiwu.zhang@amd.com>
Reviewed-by: Le Ma <le.ma@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
index 610c32c4f5af..daeb6bcc9245 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
@@ -241,6 +241,7 @@ static int amdgpu_xcp_dev_alloc(struct amdgpu_device *adev)
 
 		/* Redirect all IOCTLs to the primary device */
 		p_ddev->render->dev = ddev;
+		p_ddev->primary->dev = ddev;
 		p_ddev->vma_offset_manager = ddev->vma_offset_manager;
 		adev->xcp_mgr->xcp[i].ddev = p_ddev;
 	}
-- 
2.40.1

[PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Alex Deucher]

From: Philip Yang <Philip.Yang@amd.com>

Rename smv_migrate_init to a better name kgd2kfd_init_zone_device
because it setup zone devive pgmap for page migration and keep it in
kfd_migrate.c to access static functions svm_migrate_pgmap_ops. Call it
only once in amdgpu_device_ip_init after adev ip blocks are initialized,
but before amdgpu_amdkfd_device_init initialize kfd nodes which enable
SVM support based on pgmap.

svm_range_set_max_pages is called by kgd2kfd_device_init everytime after
switching compute partition mode.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 11 +++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  4 +++-
 drivers/gpu/drm/amd/amdkfd/kfd_device.c    |  3 ++-
 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c   |  8 +++-----
 drivers/gpu/drm/amd/amdkfd/kfd_migrate.h   |  9 ---------
 drivers/gpu/drm/amd/amdkfd/kfd_svm.h       |  4 ++++
 6 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 948d362adabb..48d12dbff968 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -372,6 +372,17 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
 {
 }
 #endif
+
+#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
+int kgd2kfd_init_zone_device(struct amdgpu_device *adev);
+#else
+static inline
+int kgd2kfd_init_zone_device(struct amdgpu_device *adev)
+{
+	return 0;
+}
+#endif
+
 /* KGD2KFD callbacks */
 int kgd2kfd_quiesce_mm(struct mm_struct *mm, uint32_t trigger);
 int kgd2kfd_resume_mm(struct mm_struct *mm);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 321b689db601..9c1a8ace6c31 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -2632,8 +2632,10 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
 		goto init_failed;
 
 	/* Don't init kfd if whole hive need to be reset during init */
-	if (!adev->gmc.xgmi.pending_reset)
+	if (!adev->gmc.xgmi.pending_reset) {
+		kgd2kfd_init_zone_device(adev);
 		amdgpu_amdkfd_device_init(adev);
+	}
 
 	amdgpu_fru_get_product_info(adev);
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 882ff86bba08..bf32e547182c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -32,6 +32,7 @@
 #include "kfd_iommu.h"
 #include "amdgpu_amdkfd.h"
 #include "kfd_smi_events.h"
+#include "kfd_svm.h"
 #include "kfd_migrate.h"
 #include "amdgpu.h"
 #include "amdgpu_xcp.h"
@@ -791,7 +792,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 		kfd->nodes[i] = node;
 	}
 
-	svm_migrate_init(kfd->adev);
+	svm_range_set_max_pages(kfd->adev);
 
 	if (kfd_resume_iommu(kfd))
 		goto kfd_resume_iommu_error;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 2512bf681112..35cf6558cf1b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -988,7 +988,7 @@ static const struct dev_pagemap_ops svm_migrate_pgmap_ops = {
 /* Each VRAM page uses sizeof(struct page) on system memory */
 #define SVM_HMM_PAGE_STRUCT_SIZE(size) ((size)/PAGE_SIZE * sizeof(struct page))
 
-int svm_migrate_init(struct amdgpu_device *adev)
+int kgd2kfd_init_zone_device(struct amdgpu_device *adev)
 {
 	struct amdgpu_kfd_dev *kfddev = &adev->kfd;
 	struct dev_pagemap *pgmap;
@@ -996,12 +996,10 @@ int svm_migrate_init(struct amdgpu_device *adev)
 	unsigned long size;
 	void *r;
 
-	/* Page migration works on Vega10 or newer */
-	if (!KFD_IS_SOC15(kfddev->dev))
+	/* Page migration works on gfx9 or newer */
+	if (adev->ip_versions[GC_HWIP][0] < IP_VERSION(9, 0, 1))
 		return -EINVAL;
 
-	svm_range_set_max_pages(adev);
-
 	if (adev->gmc.is_app_apu)
 		return 0;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h
index a5d7e6d22264..487f26368164 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h
@@ -47,15 +47,6 @@ int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm,
 unsigned long
 svm_migrate_addr_to_pfn(struct amdgpu_device *adev, unsigned long addr);
 
-int svm_migrate_init(struct amdgpu_device *adev);
-
-#else
-
-static inline int svm_migrate_init(struct amdgpu_device *adev)
-{
-	return 0;
-}
-
 #endif /* IS_ENABLED(CONFIG_HSA_AMD_SVM) */
 
 #endif /* KFD_MIGRATE_H_ */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
index 021def496f5a..762679835e31 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
@@ -265,6 +265,10 @@ static inline int kfd_criu_resume_svm(struct kfd_process *p)
 	return 0;
 }
 
+static inline void svm_range_set_max_pages(struct amdgpu_device *adev)
+{
+}
+
 #define KFD_IS_SVM_API_SUPPORTED(dev) false
 
 #endif /* IS_ENABLED(CONFIG_HSA_AMD_SVM) */
-- 
2.40.1

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Michel Dänzer]

On 5/10/23 23:23, Alex Deucher wrote:
> From: Philip Yang <Philip.Yang@amd.com>
> 
> Rename smv_migrate_init to a better name kgd2kfd_init_zone_device
> because it setup zone devive pgmap for page migration and keep it in
> kfd_migrate.c to access static functions svm_migrate_pgmap_ops. Call it
> only once in amdgpu_device_ip_init after adev ip blocks are initialized,
> but before amdgpu_amdkfd_device_init initialize kfd nodes which enable
> SVM support based on pgmap.
> 
> svm_range_set_max_pages is called by kgd2kfd_device_init everytime after
> switching compute partition mode.
> 
> Signed-off-by: Philip Yang <Philip.Yang@amd.com>
> Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

I bisected a regression to this commit, which broke HW acceleration on this ThinkPad E595 with Picasso APU.

The IB test fails for the compute rings, see dmesg below.

Reverting this commit on top of the DRM changes merged for 6.5 fixes the issue.


[drm] amdgpu kernel modesetting enabled.
amdgpu: Topology: Add APU node [0x0:0x0]
[drm] initializing kernel modesetting (RAVEN 0x1002:0x15D8 0x17AA:0x5124 0xC1).
[drm] register mmio base: 0xD0500000
[drm] register mmio size: 524288
[drm] MCBP is enabled
[drm] add ip block number 0 <soc15_common>
[drm] add ip block number 1 <gmc_v9_0>
[drm] add ip block number 2 <vega10_ih>
[drm] add ip block number 3 <psp>
[drm] add ip block number 4 <powerplay>
[drm] add ip block number 5 <dm>
[drm] add ip block number 6 <gfx_v9_0>
[drm] add ip block number 7 <sdma_v4_0>
[drm] add ip block number 8 <vcn_v1_0>
[...]
[drm] BIOS signature incorrect 0 0
amdgpu 0000:05:00.0: amdgpu: Fetched VBIOS from ROM BAR
amdgpu: ATOM BIOS: 113-PICASSO-114
[drm] VCN decode is enabled in VM mode
[drm] VCN encode is enabled in VM mode
[drm] JPEG decode is enabled in VM mode
Console: switching to colour dummy device 80x25
amdgpu 0000:05:00.0: vgaarb: deactivate vga console
amdgpu 0000:05:00.0: amdgpu: Trusted Memory Zone (TMZ) feature enabled
stackdepot: allocating hash table of 1048576 entries via kvcalloc
[drm] vm size is 262144 GB, 4 levels, block size is 9-bit, fragment size is 9-bit
amdgpu 0000:05:00.0: amdgpu: VRAM: 2048M 0x000000F400000000 - 0x000000F47FFFFFFF (2048M used)
amdgpu 0000:05:00.0: amdgpu: GART: 1024M 0x0000000000000000 - 0x000000003FFFFFFF
amdgpu 0000:05:00.0: amdgpu: AGP: 267419648M 0x000000F800000000 - 0x0000FFFFFFFFFFFF
[drm] Detected VRAM RAM=2048M, BAR=2048M
[drm] RAM width 64bits DDR4
[drm] amdgpu: 2048M of VRAM memory ready
[drm] amdgpu: 6926M of GTT memory ready.
[drm] GART: num cpu pages 262144, num gpu pages 262144
[drm] PCIE GART of 1024M enabled.
[drm] PTB located at 0x000000F400A00000
amdgpu: hwmgr_sw_init smu backed is smu10_smu
[drm] Found VCN firmware Version ENC: 1.13 DEC: 2 VEP: 0 Revision: 4
amdgpu 0000:05:00.0: amdgpu: Will use PSP to load VCN firmware
[drm] reserve 0x400000 from 0xf47fc00000 for PSP TMR
amdgpu 0000:05:00.0: amdgpu: RAS: optional ras ta ucode is not available
amdgpu 0000:05:00.0: amdgpu: RAP: optional rap ta ucode is not available
[...]
[drm] DM_PPLIB: values for F clock
[drm] DM_PPLIB:         400000 in kHz, 2749 in mV
[drm] DM_PPLIB:         933000 in kHz, 3224 in mV
[drm] DM_PPLIB:         1067000 in kHz, 3924 in mV
[drm] DM_PPLIB:         1200000 in kHz, 4074 in mV
[drm] DM_PPLIB: values for DCF clock
[drm] DM_PPLIB:         300000 in kHz, 2749 in mV
[drm] DM_PPLIB:         600000 in kHz, 3224 in mV
[drm] DM_PPLIB:         626000 in kHz, 3924 in mV
[drm] DM_PPLIB:         654000 in kHz, 4074 in mV
[drm] Display Core initialized with v3.2.236! DCN 1.0
[...]
[drm] DM_MST: Differing MST start on aconnector: 000000008d5d4db0 [id: 94]
[drm] kiq ring mec 2 pipe 1 q 0
[drm] VCN decode and encode initialized successfully(under SPG Mode).
amdgpu: HMM registered 2048MB device memory
kfd kfd: amdgpu: Allocated 3969056 bytes on gart
kfd kfd: amdgpu: Total number of KFD nodes to be created: 1
amdgpu: Topology: Add APU node [0x15d8:0x1002]
amdgpu 0000:05:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0000 address=0x169801800 flags=0x0070]
amdgpu 0000:05:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0000 address=0x13957d380 flags=0x0070]
kfd kfd: amdgpu: added device 1002:15d8
amdgpu 0000:05:00.0: amdgpu: SE 1, SH per SE 1, CU per SH 11, active_cu_number 10
amdgpu 0000:05:00.0: amdgpu: ring gfx uses VM inv eng 0 on hub 0
amdgpu 0000:05:00.0: amdgpu: ring gfx_low uses VM inv eng 1 on hub 0
amdgpu 0000:05:00.0: amdgpu: ring gfx_high uses VM inv eng 4 on hub 0
amdgpu 0000:05:00.0: amdgpu: ring comp_1.0.0 uses VM inv eng 5 on hub 0
amdgpu 0000:05:00.0: amdgpu: ring comp_1.1.0 uses VM inv eng 6 on hub 0
amdgpu 0000:05:00.0: amdgpu: ring comp_1.2.0 uses VM inv eng 7 on hub 0
amdgpu 0000:05:00.0: amdgpu: ring comp_1.3.0 uses VM inv eng 8 on hub 0
amdgpu 0000:05:00.0: amdgpu: ring comp_1.0.1 uses VM inv eng 9 on hub 0
amdgpu 0000:05:00.0: amdgpu: ring comp_1.1.1 uses VM inv eng 10 on hub 0
amdgpu 0000:05:00.0: amdgpu: ring comp_1.2.1 uses VM inv eng 11 on hub 0
amdgpu 0000:05:00.0: amdgpu: ring comp_1.3.1 uses VM inv eng 12 on hub 0
amdgpu 0000:05:00.0: amdgpu: ring kiq_0.2.1.0 uses VM inv eng 13 on hub 0
amdgpu 0000:05:00.0: amdgpu: ring sdma0 uses VM inv eng 0 on hub 8
amdgpu 0000:05:00.0: amdgpu: ring vcn_dec uses VM inv eng 1 on hub 8
amdgpu 0000:05:00.0: amdgpu: ring vcn_enc0 uses VM inv eng 4 on hub 8
amdgpu 0000:05:00.0: amdgpu: ring vcn_enc1 uses VM inv eng 5 on hub 8
amdgpu 0000:05:00.0: amdgpu: ring jpeg_dec uses VM inv eng 6 on hub 8
[...]
[drm] Initialized amdgpu 3.54.0 20150101 for 0000:05:00.0 on minor 0
[...]
amdgpu 0000:05:00.0: [drm:amdgpu_ib_ring_tests [amdgpu]] *ERROR* IB test failed on comp_1.0.0 (-110).
amdgpu 0000:05:00.0: [drm:amdgpu_ib_ring_tests [amdgpu]] *ERROR* IB test failed on comp_1.1.0 (-110).
amdgpu 0000:05:00.0: [drm:amdgpu_ib_ring_tests [amdgpu]] *ERROR* IB test failed on comp_1.2.0 (-110).
amdgpu 0000:05:00.0: [drm:amdgpu_ib_ring_tests [amdgpu]] *ERROR* IB test failed on comp_1.3.0 (-110).
amdgpu 0000:05:00.0: [drm:amdgpu_ib_ring_tests [amdgpu]] *ERROR* IB test failed on comp_1.0.1 (-110).
amdgpu 0000:05:00.0: [drm:amdgpu_ib_ring_tests [amdgpu]] *ERROR* IB test failed on comp_1.1.1 (-110).
amdgpu 0000:05:00.0: [drm:amdgpu_ib_ring_tests [amdgpu]] *ERROR* IB test failed on comp_1.2.1 (-110).
amdgpu 0000:05:00.0: [drm:amdgpu_ib_ring_tests [amdgpu]] *ERROR* IB test failed on comp_1.3.1 (-110).
[drm:process_one_work] *ERROR* ib ring test failed (-110).
[drm] Downstream port present 1, type 0
fbcon: amdgpudrmfb (fb0) is primary device
Console: switching to colour frame buffer device 192x60
amdgpu 0000:05:00.0: [drm] fb0: amdgpudrmfb frame buffer device


-- 
Earthling Michel Dänzer            |                  https://redhat.com
Libre software enthusiast          |         Mesa and Xwayland developer

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Linux regression tracking #adding (Thorsten Leemhuis)]

[TLDR: I'm adding this report to the list of tracked Linux kernel
regressions; the text you find below is based on a few templates
paragraphs you might have encountered already in similar form.
See link in footer if these mails annoy you.]

On 17.07.23 15:09, Michel Dänzer wrote:
> On 5/10/23 23:23, Alex Deucher wrote:
>> From: Philip Yang <Philip.Yang@amd.com>
>>
>> Rename smv_migrate_init to a better name kgd2kfd_init_zone_device
>> because it setup zone devive pgmap for page migration and keep it in
>> kfd_migrate.c to access static functions svm_migrate_pgmap_ops. Call it
>> only once in amdgpu_device_ip_init after adev ip blocks are initialized,
>> but before amdgpu_amdkfd_device_init initialize kfd nodes which enable
>> SVM support based on pgmap.
>>
>> svm_range_set_max_pages is called by kgd2kfd_device_init everytime after
>> switching compute partition mode.
>>
>> Signed-off-by: Philip Yang <Philip.Yang@amd.com>
>> Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
>> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> 
> I bisected a regression to this commit, which broke HW acceleration on this ThinkPad E595 with Picasso APU.
> 
> The IB test fails for the compute rings, see dmesg below.
> 
> Reverting this commit on top of the DRM changes merged for 6.5 fixes the issue.

Thanks for the report. To be sure the issue doesn't fall through the
cracks unnoticed, I'm adding it to regzbot, the Linux kernel regression
tracking bot:

#regzbot ^introduced 84b4dd3f84d
#regzbot title drm: amdgpu: HW acceleration broke on ThinkPad E595
#regzbot ignore-activity

This isn't a regression? This issue or a fix for it are already
discussed somewhere else? It was fixed already? You want to clarify when
the regression started to happen? Or point out I got the title or
something else totally wrong? Then just reply and tell me -- ideally
while also telling regzbot about it, as explained by the page listed in
the footer of this mail.

Developers: When fixing the issue, remember to add 'Link:' tags pointing
to the report (the parent of this mail). See page linked in footer for
details.

Ciao, Thorsten (wearing his 'the Linux kernel's regression tracker' hat)
--
Everything you wanna know about Linux kernel regression tracking:
https://linux-regtracking.leemhuis.info/about/#tldr
That page also explains what to do if mails like this annoy you.

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Linux regression tracking #update (Thorsten Leemhuis)]

[TLDR: This mail in primarily relevant for Linux kernel regression
tracking. See link in footer if these mails annoy you.]

On 19.07.23 18:17, Linux regression tracking #adding (Thorsten Leemhuis)
wrote:
> On 17.07.23 15:09, Michel Dänzer wrote:
>> On 5/10/23 23:23, Alex Deucher wrote:
>>> From: Philip Yang <Philip.Yang@amd.com>
>>>
>>> Rename smv_migrate_init to a better name kgd2kfd_init_zone_device
>>> because it setup zone devive pgmap for page migration and keep it in
>>> kfd_migrate.c to access static functions svm_migrate_pgmap_ops. Call it
>>> only once in amdgpu_device_ip_init after adev ip blocks are initialized,
>>> but before amdgpu_amdkfd_device_init initialize kfd nodes which enable
>>> SVM support based on pgmap.
>>>
>>> svm_range_set_max_pages is called by kgd2kfd_device_init everytime after
>>> switching compute partition mode.
>>>
>>> Signed-off-by: Philip Yang <Philip.Yang@amd.com>
>>> Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
>>> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
>>
>> I bisected a regression to this commit, which broke HW acceleration on this ThinkPad E595 with Picasso APU.

#regzbot fix: drm/amdkfd: disable IOMMUv2 support for Raven
#regzbot ignore-activity

Ciao, Thorsten (wearing his 'the Linux kernel's regression tracker' hat)
--
Everything you wanna know about Linux kernel regression tracking:
https://linux-regtracking.leemhuis.info/about/#tldr
That page also explains what to do if mails like this annoy you.

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Michel Dänzer]

On 8/11/23 11:02, Linux regression tracking #update (Thorsten Leemhuis) wrote:
> [TLDR: This mail in primarily relevant for Linux kernel regression
> tracking. See link in footer if these mails annoy you.]
> 
> On 19.07.23 18:17, Linux regression tracking #adding (Thorsten Leemhuis)
> wrote:
>> On 17.07.23 15:09, Michel Dänzer wrote:
>>> On 5/10/23 23:23, Alex Deucher wrote:
>>>> From: Philip Yang <Philip.Yang@amd.com>
>>>>
>>>> Rename smv_migrate_init to a better name kgd2kfd_init_zone_device
>>>> because it setup zone devive pgmap for page migration and keep it in
>>>> kfd_migrate.c to access static functions svm_migrate_pgmap_ops. Call it
>>>> only once in amdgpu_device_ip_init after adev ip blocks are initialized,
>>>> but before amdgpu_amdkfd_device_init initialize kfd nodes which enable
>>>> SVM support based on pgmap.
>>>>
>>>> svm_range_set_max_pages is called by kgd2kfd_device_init everytime after
>>>> switching compute partition mode.
>>>>
>>>> Signed-off-by: Philip Yang <Philip.Yang@amd.com>
>>>> Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
>>>> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
>>>
>>> I bisected a regression to this commit, which broke HW acceleration on this ThinkPad E595 with Picasso APU.
> 
> #regzbot fix: drm/amdkfd: disable IOMMUv2 support for Raven
> #regzbot ignore-activity
> 
> Ciao, Thorsten (wearing his 'the Linux kernel's regression tracker' hat)

Danke Thorsten!


-- 
Earthling Michel Dänzer            |                  https://redhat.com
Libre software enthusiast          |         Mesa and Xwayland developer

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Michel Dänzer]

On 7/17/23 15:09, Michel Dänzer wrote:
> On 5/10/23 23:23, Alex Deucher wrote:
>> From: Philip Yang <Philip.Yang@amd.com>
>>
>> Rename smv_migrate_init to a better name kgd2kfd_init_zone_device
>> because it setup zone devive pgmap for page migration and keep it in
>> kfd_migrate.c to access static functions svm_migrate_pgmap_ops. Call it
>> only once in amdgpu_device_ip_init after adev ip blocks are initialized,
>> but before amdgpu_amdkfd_device_init initialize kfd nodes which enable
>> SVM support based on pgmap.
>>
>> svm_range_set_max_pages is called by kgd2kfd_device_init everytime after
>> switching compute partition mode.
>>
>> Signed-off-by: Philip Yang <Philip.Yang@amd.com>
>> Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
>> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> 
> I bisected a regression to this commit, which broke HW acceleration on this ThinkPad E595 with Picasso APU.

Actually, it doesn't seem to break HW acceleration completely. GDM eventually comes up with HW acceleration, it takes a long time (~30s or so) to start up though.

Later, the same messages as described in https://gitlab.freedesktop.org/drm/amd/-/issues/2659 appear.

Reverting this commit fixes all of the above symptoms.


I reproduced all of the above symptoms with amd-staging-drm-next commit 75515acf4b60 ("i2c: nvidia-gpu: Add ACPI property to align with device-tree") as well.


For full disclosure, I use these kernel command line arguments:

 fbcon=font:10x18 drm_kms_helper.drm_fbdev_overalloc=112 amdgpu.noretry=1 amdgpu.mcbp=1


-- 
Earthling Michel Dänzer            |                  https://redhat.com
Libre software enthusiast          |         Mesa and Xwayland developer

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Philip Yang]

[-- Attachment #1: Type: text/html, Size: 3382 bytes --]

[-- Attachment #2: dmesg_v6.4.log --]
[-- Type: text/x-log, Size: 68264 bytes --]

[    0.000000] Linux version 6.4.0-rc7-kfd-yangp (yangp@Philip-Dev) (gcc (Ubuntu 8.4.0-1ubuntu1~18.04) 8.4.0, GNU ld (GNU Binutils for Ubuntu) 2.30) #1739 SMP PREEMPT_DYNAMIC Thu Jul 20 13:08:42 EDT 2023
[    0.000000] Command line: BOOT_IMAGE=/boot/vmlinuz-6.4.0-rc7-kfd-yangp root=UUID=2abd16aa-0792-4adc-9147-dda46bfcc9fa ro quiet splash fbcon=font:10x18 drm_kms_helper.drm_fbdev_overalloc=112 amdgpu.noretry=1 amdgpu.mcbp=1 vt.handoff=7
[    0.000000] KERNEL supported cpus:
[    0.000000]   Intel GenuineIntel
[    0.000000]   AMD AuthenticAMD
[    0.000000]   Hygon HygonGenuine
[    0.000000]   Centaur CentaurHauls
[    0.000000]   zhaoxin   Shanghai  
[    0.000000] x86/fpu: Supporting XSAVE feature 0x001: 'x87 floating point registers'
[    0.000000] x86/fpu: Supporting XSAVE feature 0x002: 'SSE registers'
[    0.000000] x86/fpu: Supporting XSAVE feature 0x004: 'AVX registers'
[    0.000000] x86/fpu: xstate_offset[2]:  576, xstate_sizes[2]:  256
[    0.000000] x86/fpu: Enabled xstate features 0x7, context size is 832 bytes, using 'compacted' format.
[    0.000000] signal: max sigframe size: 1776
[    0.000000] BIOS-provided physical RAM map:
[    0.000000] BIOS-e820: [mem 0x0000000000000000-0x000000000009f7ff] usable
[    0.000000] BIOS-e820: [mem 0x000000000009f800-0x000000000009ffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000000e0000-0x00000000000fffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000000100000-0x0000000009afffff] usable
[    0.000000] BIOS-e820: [mem 0x0000000009b00000-0x0000000009dfffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000009e00000-0x0000000009efffff] usable
[    0.000000] BIOS-e820: [mem 0x0000000009f00000-0x0000000009f0afff] ACPI NVS
[    0.000000] BIOS-e820: [mem 0x0000000009f0b000-0x000000002b57efff] usable
[    0.000000] BIOS-e820: [mem 0x000000002b57f000-0x000000002d77efff] reserved
[    0.000000] BIOS-e820: [mem 0x000000002d77f000-0x000000002f77efff] ACPI NVS
[    0.000000] BIOS-e820: [mem 0x000000002f77f000-0x000000002f7fefff] ACPI data
[    0.000000] BIOS-e820: [mem 0x000000002f7ff000-0x000000002f7fffff] usable
[    0.000000] BIOS-e820: [mem 0x000000002f800000-0x00000000afffffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000f0000000-0x00000000f7ffffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fdc00000-0x00000000fec01fff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fec10000-0x00000000fec10fff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fec20000-0x00000000fec20fff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fed80000-0x00000000fed81fff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fedc0000-0x00000000feddffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fee00000-0x00000000fee00fff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000ff800000-0x00000000fff2ffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000100000000-0x000000024f33ffff] usable
[    0.000000] NX (Execute Disable) protection: active
[    0.000000] SMBIOS 3.0 present.
[    0.000000] DMI: AMD Mandolin-PCO/Mandolin-PCO, BIOS TMC1003CD 04/03/2019
[    0.000000] tsc: Fast TSC calibration using PIT
[    0.000000] tsc: Detected 2096.007 MHz processor
[    0.008376] e820: update [mem 0x00000000-0x00000fff] usable ==> reserved
[    0.008380] e820: remove [mem 0x000a0000-0x000fffff] usable
[    0.008386] last_pfn = 0x24f340 max_arch_pfn = 0x400000000
[    0.008394] x86/PAT: Configuration [0-7]: WB  WC  UC- UC  WB  WP  UC- WT  
[    0.008739] last_pfn = 0x2f800 max_arch_pfn = 0x400000000
[    0.013054] Using GB pages for direct mapping
[    0.013641] RAMDISK: [mem 0x2a5fb000-0x2b57dfff]
[    0.013647] ACPI: Early table checksum verification disabled
[    0.013652] ACPI: RSDP 0x00000000000FE020 000024 (v02 AMD   )
[    0.013656] ACPI: XSDT 0x000000002F7D9188 0000FC (v01 AMD    Mandolin 00000001      01000013)
[    0.013662] ACPI: FACP 0x000000002F7F3000 00010C (v05 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013668] ACPI: DSDT 0x000000002F7E2000 0081BA (v01 AMD    Mandolin 00040000 ACPI 00040000)
[    0.013671] ACPI: FACS 0x000000002F764000 000040
[    0.013674] ACPI: SSDT 0x000000002F7F8000 005419 (v02 AMD    Mandolin 00000002 ACPI 00040000)
[    0.013677] ACPI: UEFI 0x000000002F7F7000 000236 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013680] ACPI: UEFI 0x000000002F7F6000 000042 (v01 AMD             00000000 ACPI 00040000)
[    0.013683] ACPI: ASF! 0x000000002F7F5000 0000A5 (v32 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013686] ACPI: BOOT 0x000000002F7F4000 000028 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013689] ACPI: HPET 0x000000002F7F2000 000038 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013692] ACPI: APIC 0x000000002F7F1000 000138 (v03 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013695] ACPI: MCFG 0x000000002F7F0000 00003C (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013698] ACPI: SLIC 0x000000002F7EF000 000176 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013701] ACPI: SPCR 0x000000002F7EE000 000050 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013704] ACPI: WSMT 0x000000002F7EB000 000028 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013706] ACPI: SSDT 0x000000002F7E0000 00046D (v01 AMD    Mandolin 00001000 ACPI 00040000)
[    0.013709] ACPI: TPM2 0x000000002F7DF000 000034 (v03 AMD    Mandolin 00000002 ACPI 00040000)
[    0.013712] ACPI: IVRS 0x000000002F7DE000 00013E (v02 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013715] ACPI: SSDT 0x000000002F7DC000 00119C (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013718] ACPI: CRAT 0x000000002F7DB000 000810 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013721] ACPI: CDIT 0x000000002F7DA000 000029 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013724] ACPI: SSDT 0x000000002F7D8000 0001D4 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013727] ACPI: SSDT 0x000000002F7D7000 0000B9 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013729] ACPI: SSDT 0x000000002F7D6000 000C33 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013732] ACPI: SSDT 0x000000002F7D4000 0010AC (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013735] ACPI: SSDT 0x000000002F7D3000 0002AA (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013738] ACPI: SSDT 0x000000002F7D1000 001C69 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013741] ACPI: FPDT 0x000000002F7D0000 000044 (v01 AMD    Mandolin 00000002 ACPI 00040000)
[    0.013744] ACPI: SSDT 0x000000002F7EC000 000F82 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013747] ACPI: SSDT 0x000000002F7CE000 001CBD (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013749] ACPI: Reserving FACP table memory at [mem 0x2f7f3000-0x2f7f310b]
[    0.013751] ACPI: Reserving DSDT table memory at [mem 0x2f7e2000-0x2f7ea1b9]
[    0.013752] ACPI: Reserving FACS table memory at [mem 0x2f764000-0x2f76403f]
[    0.013753] ACPI: Reserving SSDT table memory at [mem 0x2f7f8000-0x2f7fd418]
[    0.013754] ACPI: Reserving UEFI table memory at [mem 0x2f7f7000-0x2f7f7235]
[    0.013755] ACPI: Reserving UEFI table memory at [mem 0x2f7f6000-0x2f7f6041]
[    0.013756] ACPI: Reserving ASF! table memory at [mem 0x2f7f5000-0x2f7f50a4]
[    0.013757] ACPI: Reserving BOOT table memory at [mem 0x2f7f4000-0x2f7f4027]
[    0.013758] ACPI: Reserving HPET table memory at [mem 0x2f7f2000-0x2f7f2037]
[    0.013759] ACPI: Reserving APIC table memory at [mem 0x2f7f1000-0x2f7f1137]
[    0.013760] ACPI: Reserving MCFG table memory at [mem 0x2f7f0000-0x2f7f003b]
[    0.013761] ACPI: Reserving SLIC table memory at [mem 0x2f7ef000-0x2f7ef175]
[    0.013762] ACPI: Reserving SPCR table memory at [mem 0x2f7ee000-0x2f7ee04f]
[    0.013762] ACPI: Reserving WSMT table memory at [mem 0x2f7eb000-0x2f7eb027]
[    0.013763] ACPI: Reserving SSDT table memory at [mem 0x2f7e0000-0x2f7e046c]
[    0.013764] ACPI: Reserving TPM2 table memory at [mem 0x2f7df000-0x2f7df033]
[    0.013765] ACPI: Reserving IVRS table memory at [mem 0x2f7de000-0x2f7de13d]
[    0.013766] ACPI: Reserving SSDT table memory at [mem 0x2f7dc000-0x2f7dd19b]
[    0.013767] ACPI: Reserving CRAT table memory at [mem 0x2f7db000-0x2f7db80f]
[    0.013768] ACPI: Reserving CDIT table memory at [mem 0x2f7da000-0x2f7da028]
[    0.013769] ACPI: Reserving SSDT table memory at [mem 0x2f7d8000-0x2f7d81d3]
[    0.013770] ACPI: Reserving SSDT table memory at [mem 0x2f7d7000-0x2f7d70b8]
[    0.013770] ACPI: Reserving SSDT table memory at [mem 0x2f7d6000-0x2f7d6c32]
[    0.013771] ACPI: Reserving SSDT table memory at [mem 0x2f7d4000-0x2f7d50ab]
[    0.013772] ACPI: Reserving SSDT table memory at [mem 0x2f7d3000-0x2f7d32a9]
[    0.013773] ACPI: Reserving SSDT table memory at [mem 0x2f7d1000-0x2f7d2c68]
[    0.013774] ACPI: Reserving FPDT table memory at [mem 0x2f7d0000-0x2f7d0043]
[    0.013775] ACPI: Reserving SSDT table memory at [mem 0x2f7ec000-0x2f7ecf81]
[    0.013776] ACPI: Reserving SSDT table memory at [mem 0x2f7ce000-0x2f7cfcbc]
[    0.013893] No NUMA configuration found
[    0.013895] Faking a node at [mem 0x0000000000000000-0x000000024f33ffff]
[    0.013898] NODE_DATA(0) allocated [mem 0x24f33a000-0x24f33ffff]
[    0.013921] Zone ranges:
[    0.013922]   DMA      [mem 0x0000000000001000-0x0000000000ffffff]
[    0.013924]   DMA32    [mem 0x0000000001000000-0x00000000ffffffff]
[    0.013925]   Normal   [mem 0x0000000100000000-0x000000024f33ffff]
[    0.013927]   Device   empty
[    0.013928] Movable zone start for each node
[    0.013929] Early memory node ranges
[    0.013929]   node   0: [mem 0x0000000000001000-0x000000000009efff]
[    0.013931]   node   0: [mem 0x0000000000100000-0x0000000009afffff]
[    0.013932]   node   0: [mem 0x0000000009e00000-0x0000000009efffff]
[    0.013933]   node   0: [mem 0x0000000009f0b000-0x000000002b57efff]
[    0.013934]   node   0: [mem 0x000000002f7ff000-0x000000002f7fffff]
[    0.013935]   node   0: [mem 0x0000000100000000-0x000000024f33ffff]
[    0.013936] Initmem setup node 0 [mem 0x0000000000001000-0x000000024f33ffff]
[    0.013941] On node 0, zone DMA: 1 pages in unavailable ranges
[    0.013962] On node 0, zone DMA: 97 pages in unavailable ranges
[    0.014148] On node 0, zone DMA32: 768 pages in unavailable ranges
[    0.015519] On node 0, zone DMA32: 11 pages in unavailable ranges
[    0.015713] On node 0, zone DMA32: 17024 pages in unavailable ranges
[    0.030738] On node 0, zone Normal: 2048 pages in unavailable ranges
[    0.030780] On node 0, zone Normal: 3264 pages in unavailable ranges
[    0.030931] ACPI: PM-Timer IO Port: 0x408
[    0.030939] ACPI: LAPIC_NMI (acpi_id[0x00] high edge lint[0x1])
[    0.030940] ACPI: LAPIC_NMI (acpi_id[0x01] high edge lint[0x1])
[    0.030942] ACPI: LAPIC_NMI (acpi_id[0x02] high edge lint[0x1])
[    0.030943] ACPI: LAPIC_NMI (acpi_id[0x03] high edge lint[0x1])
[    0.030943] ACPI: LAPIC_NMI (acpi_id[0x04] high edge lint[0x1])
[    0.030944] ACPI: LAPIC_NMI (acpi_id[0x05] high edge lint[0x1])
[    0.030945] ACPI: LAPIC_NMI (acpi_id[0x06] high edge lint[0x1])
[    0.030946] ACPI: LAPIC_NMI (acpi_id[0x07] high edge lint[0x1])
[    0.030947] ACPI: LAPIC_NMI (acpi_id[0x08] high edge lint[0x1])
[    0.030948] ACPI: LAPIC_NMI (acpi_id[0x09] high edge lint[0x1])
[    0.030949] ACPI: LAPIC_NMI (acpi_id[0x0a] high edge lint[0x1])
[    0.030949] ACPI: LAPIC_NMI (acpi_id[0x0b] high edge lint[0x1])
[    0.030950] ACPI: LAPIC_NMI (acpi_id[0x0c] high edge lint[0x1])
[    0.030951] ACPI: LAPIC_NMI (acpi_id[0x0d] high edge lint[0x1])
[    0.030952] ACPI: LAPIC_NMI (acpi_id[0x0e] high edge lint[0x1])
[    0.030953] ACPI: LAPIC_NMI (acpi_id[0x0f] high edge lint[0x1])
[    0.030982] IOAPIC[0]: apic_id 33, version 33, address 0xfec00000, GSI 0-23
[    0.030996] IOAPIC[1]: apic_id 34, version 33, address 0xfec01000, GSI 24-55
[    0.030998] ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
[    0.031000] ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 low level)
[    0.031004] ACPI: Using ACPI (MADT) for SMP configuration information
[    0.031006] ACPI: HPET id: 0x10228210 base: 0xfed00000
[    0.031013] ACPI: SPCR: SPCR table version 1
[    0.031015] ACPI: SPCR: console: uart,io,0x3f8,115200
[    0.031017] smpboot: Allowing 16 CPUs, 8 hotplug CPUs
[    0.031046] PM: hibernation: Registered nosave memory: [mem 0x00000000-0x00000fff]
[    0.031049] PM: hibernation: Registered nosave memory: [mem 0x0009f000-0x0009ffff]
[    0.031050] PM: hibernation: Registered nosave memory: [mem 0x000a0000-0x000dffff]
[    0.031050] PM: hibernation: Registered nosave memory: [mem 0x000e0000-0x000fffff]
[    0.031052] PM: hibernation: Registered nosave memory: [mem 0x09b00000-0x09dfffff]
[    0.031054] PM: hibernation: Registered nosave memory: [mem 0x09f00000-0x09f0afff]
[    0.031056] PM: hibernation: Registered nosave memory: [mem 0x2b57f000-0x2d77efff]
[    0.031056] PM: hibernation: Registered nosave memory: [mem 0x2d77f000-0x2f77efff]
[    0.031057] PM: hibernation: Registered nosave memory: [mem 0x2f77f000-0x2f7fefff]
[    0.031059] PM: hibernation: Registered nosave memory: [mem 0x2f800000-0xafffffff]
[    0.031060] PM: hibernation: Registered nosave memory: [mem 0xb0000000-0xefffffff]
[    0.031061] PM: hibernation: Registered nosave memory: [mem 0xf0000000-0xf7ffffff]
[    0.031061] PM: hibernation: Registered nosave memory: [mem 0xf8000000-0xfdbfffff]
[    0.031062] PM: hibernation: Registered nosave memory: [mem 0xfdc00000-0xfec01fff]
[    0.031063] PM: hibernation: Registered nosave memory: [mem 0xfec02000-0xfec0ffff]
[    0.031064] PM: hibernation: Registered nosave memory: [mem 0xfec10000-0xfec10fff]
[    0.031065] PM: hibernation: Registered nosave memory: [mem 0xfec11000-0xfec1ffff]
[    0.031065] PM: hibernation: Registered nosave memory: [mem 0xfec20000-0xfec20fff]
[    0.031066] PM: hibernation: Registered nosave memory: [mem 0xfec21000-0xfed7ffff]
[    0.031067] PM: hibernation: Registered nosave memory: [mem 0xfed80000-0xfed81fff]
[    0.031068] PM: hibernation: Registered nosave memory: [mem 0xfed82000-0xfedbffff]
[    0.031069] PM: hibernation: Registered nosave memory: [mem 0xfedc0000-0xfeddffff]
[    0.031069] PM: hibernation: Registered nosave memory: [mem 0xfede0000-0xfedfffff]
[    0.031070] PM: hibernation: Registered nosave memory: [mem 0xfee00000-0xfee00fff]
[    0.031071] PM: hibernation: Registered nosave memory: [mem 0xfee01000-0xff7fffff]
[    0.031072] PM: hibernation: Registered nosave memory: [mem 0xff800000-0xfff2ffff]
[    0.031073] PM: hibernation: Registered nosave memory: [mem 0xfff30000-0xffffffff]
[    0.031075] [mem 0xb0000000-0xefffffff] available for PCI devices
[    0.031076] Booting paravirtualized kernel on bare hardware
[    0.031080] clocksource: refined-jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645519600211568 ns
[    0.036447] setup_percpu: NR_CPUS:256 nr_cpumask_bits:16 nr_cpu_ids:16 nr_node_ids:1
[    0.037283] percpu: Embedded 59 pages/cpu s201504 r8192 d31968 u262144
[    0.037294] pcpu-alloc: s201504 r8192 d31968 u262144 alloc=1*2097152
[    0.037297] pcpu-alloc: [0] 00 01 02 03 04 05 06 07 [0] 08 09 10 11 12 13 14 15 
[    0.037332] Kernel command line: BOOT_IMAGE=/boot/vmlinuz-6.4.0-rc7-kfd-yangp root=UUID=2abd16aa-0792-4adc-9147-dda46bfcc9fa ro quiet splash fbcon=font:10x18 drm_kms_helper.drm_fbdev_overalloc=112 amdgpu.noretry=1 amdgpu.mcbp=1 vt.handoff=7
[    0.037475] Unknown kernel command line parameters "splash BOOT_IMAGE=/boot/vmlinuz-6.4.0-rc7-kfd-yangp", will be passed to user space.
[    0.037757] random: crng init done
[    0.039201] Dentry cache hash table entries: 1048576 (order: 11, 8388608 bytes, linear)
[    0.039913] Inode-cache hash table entries: 524288 (order: 10, 4194304 bytes, linear)
[    0.040058] Fallback order for Node 0: 0 
[    0.040065] Built 1 zonelists, mobility grouping on.  Total pages: 1525278
[    0.040066] Policy zone: Normal
[    0.040075] mem auto-init: stack:off, heap alloc:off, heap free:off
[    0.040133] software IO TLB: area num 16.
[    0.074479] Memory: 5947656K/6198604K available (18432K kernel code, 3715K rwdata, 6084K rodata, 1872K init, 17684K bss, 250688K reserved, 0K cma-reserved)
[    0.074836] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=16, Nodes=1
[    0.074839] kmemleak: Kernel memory leak detector disabled
[    0.074912] ftrace: allocating 53130 entries in 208 pages
[    0.083200] ftrace: allocated 208 pages with 3 groups
[    0.084662] Dynamic Preempt: voluntary
[    0.084754] Running RCU self tests
[    0.084755] Running RCU synchronous self tests
[    0.084766] rcu: Preemptible hierarchical RCU implementation.
[    0.084767] rcu: 	RCU lockdep checking is enabled.
[    0.084768] rcu: 	RCU restricting CPUs from NR_CPUS=256 to nr_cpu_ids=16.
[    0.084769] 	Trampoline variant of Tasks RCU enabled.
[    0.084770] 	Rude variant of Tasks RCU enabled.
[    0.084771] 	Tracing variant of Tasks RCU enabled.
[    0.084771] rcu: RCU calculated value of scheduler-enlistment delay is 25 jiffies.
[    0.084772] rcu: Adjusting geometry for rcu_fanout_leaf=16, nr_cpu_ids=16
[    0.084798] Running RCU synchronous self tests
[    0.088089] NR_IRQS: 16640, nr_irqs: 1096, preallocated irqs: 16
[    0.088319] rcu: srcu_init: Setting srcu_struct sizes based on contention.
[    0.088511] spurious 8259A interrupt: IRQ7.
[    0.088555] Console: colour dummy device 80x25
[    0.088571] printk: console [tty0] enabled
[    0.088601] Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar
[    0.088603] ... MAX_LOCKDEP_SUBCLASSES:  8
[    0.088604] ... MAX_LOCK_DEPTH:          48
[    0.088605] ... MAX_LOCKDEP_KEYS:        8192
[    0.088606] ... CLASSHASH_SIZE:          4096
[    0.088607] ... MAX_LOCKDEP_ENTRIES:     32768
[    0.088608] ... MAX_LOCKDEP_CHAINS:      65536
[    0.088608] ... CHAINHASH_SIZE:          32768
[    0.088609]  memory used by lock dependency info: 6365 kB
[    0.088610]  memory used for stack traces: 4224 kB
[    0.088611]  per task-struct memory footprint: 1920 bytes
[    0.088649] ACPI: Core revision 20230331
[    0.088930] clocksource: hpet: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 133484873504 ns
[    0.088977] APIC: Switch to symmetric I/O mode setup
[    0.088979] Switched APIC routing to physical flat.
[    0.090284] ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1
[    0.108976] clocksource: tsc-early: mask: 0xffffffffffffffff max_cycles: 0x1e36749a135, max_idle_ns: 440795280167 ns
[    0.109000] Calibrating delay loop (skipped), value calculated using timer frequency.. 4192.01 BogoMIPS (lpj=8384028)
[    0.109004] pid_max: default: 32768 minimum: 301
[    0.109041] LSM: initializing lsm=capability,selinux,integrity
[    0.109051] SELinux:  Initializing.
[    0.109157] Mount-cache hash table entries: 16384 (order: 5, 131072 bytes, linear)
[    0.109182] Mountpoint-cache hash table entries: 16384 (order: 5, 131072 bytes, linear)
[    0.110089] LVT offset 1 assigned for vector 0xf9
[    0.110163] LVT offset 2 assigned for vector 0xf4
[    0.110185] process: using mwait in idle threads
[    0.110192] Last level iTLB entries: 4KB 1024, 2MB 1024, 4MB 512
[    0.110194] Last level dTLB entries: 4KB 1536, 2MB 1536, 4MB 768, 1GB 0
[    0.110200] Spectre V1 : Mitigation: usercopy/swapgs barriers and __user pointer sanitization
[    0.110203] Spectre V2 : Kernel not compiled with retpoline; no mitigation available!
[    0.110205] Spectre V2 : Vulnerable
[    0.110225] Spectre V2 : Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch
[    0.110227] Spectre V2 : Enabling Speculation Barrier for firmware calls
[    0.110228] RETBleed: Mitigation: IBPB
[    0.110230] Spectre V2 : mitigation: Enabling conditional Indirect Branch Prediction Barrier
[    0.110232] Speculative Store Bypass: Mitigation: Speculative Store Bypass disabled via prctl
[    0.113253] Freeing SMP alternatives memory: 48K
[    0.114800] Running RCU synchronous self tests
[    0.114803] Running RCU synchronous self tests
[    0.222987] smpboot: CPU0: AMD Ryzen 5 3500U with Radeon Vega Mobile Gfx (family: 0x17, model: 0x18, stepping: 0x1)
[    0.223357] cblist_init_generic: Setting adjustable number of callback queues.
[    0.223377] cblist_init_generic: Setting shift to 4 and lim to 1.
[    0.223424] cblist_init_generic: Setting shift to 4 and lim to 1.
[    0.223469] cblist_init_generic: Setting shift to 4 and lim to 1.
[    0.223500] Running RCU-tasks wait API self tests
[    0.325171] Performance Events: Fam17h+ core perfctr, AMD PMU driver.
[    0.325237] ... version:                0
[    0.325239] ... bit width:              48
[    0.325241] ... generic registers:      6
[    0.325242] ... value mask:             0000ffffffffffff
[    0.325245] ... max period:             00007fffffffffff
[    0.325247] ... fixed-purpose events:   0
[    0.325248] ... event mask:             000000000000003f
[    0.325491] rcu: Hierarchical SRCU implementation.
[    0.325494] rcu: 	Max phase no-delay instances is 1000.
[    0.326177] NMI watchdog: Enabled. Permanently consumes one hw-PMU counter.
[    0.326562] smp: Bringing up secondary CPUs ...
[    0.326904] x86: Booting SMP configuration:
[    0.326910] .... node  #0, CPUs:        #1  #2  #3  #4  #5  #6
[    0.343028] Callback from call_rcu_tasks_trace() invoked.
[    0.343028]   #7
[    0.345057] smp: Brought up 1 node, 8 CPUs
[    0.345057] smpboot: Max logical packages: 2
[    0.345057] smpboot: Total of 8 processors activated (33536.11 BogoMIPS)
[    0.345893] devtmpfs: initialized
[    0.345893] x86/mm: Memory block size: 128MB
[    0.349660] ACPI: PM: Registering ACPI NVS region [mem 0x09f00000-0x09f0afff] (45056 bytes)
[    0.349660] ACPI: PM: Registering ACPI NVS region [mem 0x2d77f000-0x2f77efff] (33554432 bytes)
[    0.350929] Running RCU synchronous self tests
[    0.350940] Running RCU synchronous self tests
[    0.350981] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns
[    0.350987] futex hash table entries: 4096 (order: 7, 524288 bytes, linear)
[    0.351143] pinctrl core: initialized pinctrl subsystem
[    0.351298] PM: RTC time: 19:02:53, date: 2023-07-20
[    0.351514] NET: Registered PF_NETLINK/PF_ROUTE protocol family
[    0.351840] audit: initializing netlink subsys (disabled)
[    0.351856] audit: type=2000 audit(1689879772.260:1): state=initialized audit_enabled=0 res=1
[    0.351856] thermal_sys: Registered thermal governor 'step_wise'
[    0.351856] thermal_sys: Registered thermal governor 'user_space'
[    0.351856] cpuidle: using governor ladder
[    0.351856] cpuidle: using governor menu
[    0.353068] Simple Boot Flag at 0x44 set to 0x1
[    0.353240] PCI: MMCONFIG for domain 0000 [bus 00-7f] at [mem 0xf0000000-0xf7ffffff] (base 0xf0000000)
[    0.353247] PCI: MMCONFIG at [mem 0xf0000000-0xf7ffffff] reserved as E820 entry
[    0.353260] PCI: Using configuration type 1 for base access
[    0.353722] mtrr: your CPUs had inconsistent variable MTRR settings
[    0.353724] mtrr: probably your BIOS does not setup all CPUs.
[    0.353725] mtrr: corrected configuration.
[    0.353815] kprobes: kprobe jump-optimization is enabled. All kprobes are optimized if possible.
[    0.353827] HugeTLB: registered 1.00 GiB page size, pre-allocated 0 pages
[    0.353827] HugeTLB: 16380 KiB vmemmap can be freed for a 1.00 GiB page
[    0.353827] HugeTLB: registered 2.00 MiB page size, pre-allocated 0 pages
[    0.353827] HugeTLB: 28 KiB vmemmap can be freed for a 2.00 MiB page
[    0.353827] ACPI: Added _OSI(Module Device)
[    0.353827] ACPI: Added _OSI(Processor Device)
[    0.353827] ACPI: Added _OSI(3.0 _SCP Extensions)
[    0.353827] ACPI: Added _OSI(Processor Aggregator Device)
[    0.373301] ACPI: 12 ACPI AML tables successfully acquired and loaded
[    0.376870] ACPI: [Firmware Bug]: BIOS _OSI(Linux) query ignored
[    0.479645] Callback from call_rcu_tasks_rude() invoked.
[    0.486568] ACPI: EC: EC started
[    0.486575] ACPI: EC: interrupt blocked
[    0.487566] ACPI: EC: EC_CMD/EC_SC=0x666, EC_DATA=0x662
[    0.487571] ACPI: \_SB_.PCI0.LPC0.EC0_: Boot DSDT EC used to handle transactions
[    0.487575] ACPI: Interpreter enabled
[    0.487639] ACPI: PM: (supports S0 S3 S4 S5)
[    0.487648] ACPI: Using IOAPIC for interrupt routing
[    0.488020] PCI: Using host bridge windows from ACPI; if necessary, use "pci=nocrs" and report a bug
[    0.488023] PCI: Using E820 reservations for host bridge windows
[    0.489245] ACPI: Enabled 4 GPEs in block 00 to 1F
[    0.507694] ACPI: \_SB_.PRWL: New power resource
[    0.508646] ACPI: PCI Root Bridge [PCI0] (domain 0000 [bus 00-ff])
[    0.508653] acpi PNP0A08:00: _OSC: OS supports [ExtendedConfig ASPM ClockPM Segments MSI HPX-Type3]
[    0.508777] acpi PNP0A08:00: _OSC: platform does not support [LTR]
[    0.509033] acpi PNP0A08:00: _OSC: OS now controls [PCIeHotplug PME AER PCIeCapability]
[    0.509061] acpi PNP0A08:00: [Firmware Info]: MMCONFIG for domain 0000 [bus 00-7f] only partially covers this bridge
[    0.509206] acpi PNP0A08:00: ignoring host bridge window [mem 0x000cc000-0x000cffff window] (conflicts with Video ROM [mem 0x000c0000-0x000cd3ff])
[    0.509549] PCI host bridge to bus 0000:00
[    0.509552] pci_bus 0000:00: root bus resource [io  0x0000-0x0cf7 window]
[    0.509554] pci_bus 0000:00: root bus resource [io  0x0d00-0xffff window]
[    0.509557] pci_bus 0000:00: root bus resource [mem 0x000a0000-0x000bffff window]
[    0.509559] pci_bus 0000:00: root bus resource [mem 0x000c0000-0x000cbfff window]
[    0.509561] pci_bus 0000:00: root bus resource [mem 0x000d0000-0x000effff window]
[    0.509562] pci_bus 0000:00: root bus resource [mem 0xb0000000-0xefffffff window]
[    0.509565] pci_bus 0000:00: root bus resource [mem 0xf8000000-0xfeafffff window]
[    0.509566] pci_bus 0000:00: root bus resource [mem 0xfed45000-0xfed814ff window]
[    0.509568] pci_bus 0000:00: root bus resource [mem 0xfed81900-0xfed81fff window]
[    0.509570] pci_bus 0000:00: root bus resource [mem 0xfedc0000-0xfedc0fff window]
[    0.509572] pci_bus 0000:00: root bus resource [mem 0xfedc6000-0xfedc6fff window]
[    0.509574] pci_bus 0000:00: root bus resource [bus 00-ff]
[    0.509635] pci 0000:00:00.0: [1022:15d0] type 00 class 0x060000
[    0.509880] pci 0000:00:00.2: [1022:15d1] type 00 class 0x080600
[    0.510111] pci 0000:00:01.0: [1022:1452] type 00 class 0x060000
[    0.510310] pci 0000:00:01.2: [1022:15d3] type 01 class 0x060400
[    0.510589] pci 0000:00:01.2: PME# supported from D0 D3hot D3cold
[    0.511108] pci 0000:00:01.3: [1022:15d3] type 01 class 0x060400
[    0.511215] pci 0000:00:01.3: enabling Extended Tags
[    0.511387] pci 0000:00:01.3: PME# supported from D0 D3hot D3cold
[    0.511820] pci 0000:00:01.7: [1022:15d3] type 01 class 0x060400
[    0.512097] pci 0000:00:01.7: PME# supported from D0 D3hot D3cold
[    0.512503] pci 0000:00:08.0: [1022:1452] type 00 class 0x060000
[    0.512684] pci 0000:00:08.1: [1022:15db] type 01 class 0x060400
[    0.512722] pci 0000:00:08.1: enabling Extended Tags
[    0.512775] pci 0000:00:08.1: PME# supported from D0 D3hot D3cold
[    0.513084] pci 0000:00:14.0: [1022:790b] type 00 class 0x0c0500
[    0.513317] pci 0000:00:14.3: [1022:790e] type 00 class 0x060100
[    0.513581] pci 0000:00:18.0: [1022:15e8] type 00 class 0x060000
[    0.513663] pci 0000:00:18.1: [1022:15e9] type 00 class 0x060000
[    0.513747] pci 0000:00:18.2: [1022:15ea] type 00 class 0x060000
[    0.513828] pci 0000:00:18.3: [1022:15eb] type 00 class 0x060000
[    0.513911] pci 0000:00:18.4: [1022:15ec] type 00 class 0x060000
[    0.513994] pci 0000:00:18.5: [1022:15ed] type 00 class 0x060000
[    0.514075] pci 0000:00:18.6: [1022:15ee] type 00 class 0x060000
[    0.514158] pci 0000:00:18.7: [1022:15ef] type 00 class 0x060000
[    0.514366] pci 0000:00:01.2: PCI bridge to [bus 01]
[    0.514541] pci 0000:02:00.0: [14e4:1687] type 00 class 0x020000
[    0.514567] pci 0000:02:00.0: reg 0x10: [mem 0xc0240000-0xc024ffff 64bit pref]
[    0.514585] pci 0000:02:00.0: reg 0x18: [mem 0xc0230000-0xc023ffff 64bit pref]
[    0.514603] pci 0000:02:00.0: reg 0x20: [mem 0xc0220000-0xc022ffff 64bit pref]
[    0.514732] pci 0000:02:00.0: PME# supported from D0 D3hot D3cold
[    0.515119] pci 0000:02:00.1: [14e4:1640] type 00 class 0x080501
[    0.515145] pci 0000:02:00.1: reg 0x10: [mem 0xc0210000-0xc021ffff 64bit pref]
[    0.515162] pci 0000:02:00.1: reg 0x18: [mem 0xc0200000-0xc020ffff 64bit pref]
[    0.515280] pci 0000:02:00.1: PME# supported from D0 D3hot D3cold
[    0.515654] pci 0000:00:01.3: PCI bridge to [bus 02]
[    0.515680] pci 0000:00:01.3:   bridge window [mem 0xc0200000-0xc02fffff 64bit pref]
[    0.515936] pci 0000:03:00.0: [144d:a80a] type 00 class 0x010802
[    0.515967] pci 0000:03:00.0: reg 0x10: [mem 0xc0800000-0xc0803fff 64bit]
[    0.516276] pci 0000:03:00.0: 4.000 Gb/s available PCIe bandwidth, limited by 2.5 GT/s PCIe x2 link at 0000:00:01.7 (capable of 63.012 Gb/s with 16.0 GT/s PCIe x4 link)
[    0.516673] pci 0000:00:01.7: PCI bridge to [bus 03]
[    0.516680] pci 0000:00:01.7:   bridge window [mem 0xc0800000-0xc08fffff]
[    0.516814] pci 0000:04:00.0: [1002:15d8] type 00 class 0x030000
[    0.516834] pci 0000:04:00.0: reg 0x10: [mem 0xb0000000-0xbfffffff 64bit pref]
[    0.516847] pci 0000:04:00.0: reg 0x18: [mem 0xc0000000-0xc01fffff 64bit pref]
[    0.516856] pci 0000:04:00.0: reg 0x20: [io  0x1000-0x10ff]
[    0.516865] pci 0000:04:00.0: reg 0x24: [mem 0xc0700000-0xc077ffff]
[    0.516879] pci 0000:04:00.0: enabling Extended Tags
[    0.516906] pci 0000:04:00.0: Video device with shadowed ROM at [mem 0x000c0000-0x000dffff]
[    0.516974] pci 0000:04:00.0: PME# supported from D1 D2 D3hot D3cold
[    0.517250] pci 0000:04:00.1: [1002:15de] type 00 class 0x040300
[    0.517266] pci 0000:04:00.1: reg 0x10: [mem 0xc07c8000-0xc07cbfff]
[    0.517315] pci 0000:04:00.1: enabling Extended Tags
[    0.517370] pci 0000:04:00.1: PME# supported from D1 D2 D3hot D3cold
[    0.517558] pci 0000:04:00.2: [1022:15df] type 00 class 0x108000
[    0.517587] pci 0000:04:00.2: reg 0x18: [mem 0xc0600000-0xc06fffff]
[    0.517609] pci 0000:04:00.2: reg 0x24: [mem 0xc07ce000-0xc07cffff]
[    0.517623] pci 0000:04:00.2: enabling Extended Tags
[    0.517845] pci 0000:04:00.3: [1022:15e0] type 00 class 0x0c0330
[    0.517864] pci 0000:04:00.3: reg 0x10: [mem 0xc0500000-0xc05fffff 64bit]
[    0.517906] pci 0000:04:00.3: enabling Extended Tags
[    0.517963] pci 0000:04:00.3: PME# supported from D0 D3hot D3cold
[    0.518175] pci 0000:04:00.4: [1022:15e1] type 00 class 0x0c0330
[    0.518194] pci 0000:04:00.4: reg 0x10: [mem 0xc0400000-0xc04fffff 64bit]
[    0.518236] pci 0000:04:00.4: enabling Extended Tags
[    0.518293] pci 0000:04:00.4: PME# supported from D0 D3hot D3cold
[    0.518484] pci 0000:04:00.5: [1022:15e2] type 00 class 0x048000
[    0.518499] pci 0000:04:00.5: reg 0x10: [mem 0xc0780000-0xc07bffff]
[    0.518547] pci 0000:04:00.5: enabling Extended Tags
[    0.518601] pci 0000:04:00.5: PME# supported from D0 D3hot D3cold
[    0.518783] pci 0000:04:00.6: [1022:15e3] type 00 class 0x040300
[    0.518798] pci 0000:04:00.6: reg 0x10: [mem 0xc07c0000-0xc07c7fff]
[    0.518847] pci 0000:04:00.6: enabling Extended Tags
[    0.518902] pci 0000:04:00.6: PME# supported from D0 D3hot D3cold
[    0.519084] pci 0000:04:00.7: [1022:15e6] type 00 class 0x000000
[    0.519113] pci 0000:04:00.7: reg 0x18: [mem 0xc0300000-0xc03fffff]
[    0.519136] pci 0000:04:00.7: reg 0x24: [mem 0xc07cc000-0xc07cdfff]
[    0.519150] pci 0000:04:00.7: enabling Extended Tags
[    0.519445] pci 0000:00:08.1: PCI bridge to [bus 04]
[    0.519452] pci 0000:00:08.1:   bridge window [io  0x1000-0x1fff]
[    0.519456] pci 0000:00:08.1:   bridge window [mem 0xc0300000-0xc07fffff]
[    0.519462] pci 0000:00:08.1:   bridge window [mem 0xb0000000-0xc01fffff 64bit pref]
[    0.520998] ACPI: PCI: Interrupt link LNKA configured for IRQ 0
[    0.521004] ACPI: PCI: Interrupt link LNKA disabled
[    0.521121] ACPI: PCI: Interrupt link LNKB configured for IRQ 0
[    0.521123] ACPI: PCI: Interrupt link LNKB disabled
[    0.521215] ACPI: PCI: Interrupt link LNKC configured for IRQ 0
[    0.521217] ACPI: PCI: Interrupt link LNKC disabled
[    0.521326] ACPI: PCI: Interrupt link LNKD configured for IRQ 0
[    0.521328] ACPI: PCI: Interrupt link LNKD disabled
[    0.521431] ACPI: PCI: Interrupt link LNKE configured for IRQ 0
[    0.521432] ACPI: PCI: Interrupt link LNKE disabled
[    0.521517] ACPI: PCI: Interrupt link LNKF configured for IRQ 0
[    0.521519] ACPI: PCI: Interrupt link LNKF disabled
[    0.521604] ACPI: PCI: Interrupt link LNKG configured for IRQ 0
[    0.521606] ACPI: PCI: Interrupt link LNKG disabled
[    0.521690] ACPI: PCI: Interrupt link LNKH configured for IRQ 0
[    0.521692] ACPI: PCI: Interrupt link LNKH disabled
[    0.533027] Callback from call_rcu_tasks() invoked.
[    0.531235] ACPI: EC: interrupt unblocked
[    0.531239] ACPI: EC: event unblocked
[    0.531253] ACPI: EC: EC_CMD/EC_SC=0x666, EC_DATA=0x662
[    0.531254] ACPI: EC: GPE=0x3
[    0.531257] ACPI: \_SB_.PCI0.LPC0.EC0_: Boot DSDT EC initialization complete
[    0.531259] ACPI: \_SB_.PCI0.LPC0.EC0_: EC: Used to handle transactions and events
[    0.533130] iommu: Default domain type: Translated 
[    0.533130] iommu: DMA domain TLB invalidation policy: lazy mode 
[    0.533356] SCSI subsystem initialized
[    0.533374] libata version 3.00 loaded.
[    0.533374] ACPI: bus type USB registered
[    0.533374] usbcore: registered new interface driver usbfs
[    0.533374] usbcore: registered new interface driver hub
[    0.533374] usbcore: registered new device driver usb
[    0.533374] pps_core: LinuxPPS API ver. 1 registered
[    0.533374] pps_core: Software ver. 5.3.6 - Copyright 2005-2007 Rodolfo Giometti <giometti@linux.it>
[    0.533374] PTP clock support registered
[    0.533410] NetLabel: Initializing
[    0.533412] NetLabel:  domain hash size = 128
[    0.533414] NetLabel:  protocols = UNLABELED CIPSOv4 CALIPSO
[    0.533446] NetLabel:  unlabeled traffic allowed by default
[    0.533538] PCI: Using ACPI for IRQ routing
[    0.538012] PCI: pci_cache_line_size set to 64 bytes
[    0.538411] resource: Expanded resource Reserved due to conflict with PCI Bus 0000:00
[    0.538419] resource: Expanded resource Reserved due to conflict with PCI Bus 0000:00
[    0.538421] e820: reserve RAM buffer [mem 0x0009f800-0x0009ffff]
[    0.538424] e820: reserve RAM buffer [mem 0x09b00000-0x0bffffff]
[    0.538426] e820: reserve RAM buffer [mem 0x09f00000-0x0bffffff]
[    0.538428] e820: reserve RAM buffer [mem 0x2b57f000-0x2bffffff]
[    0.538429] e820: reserve RAM buffer [mem 0x2f800000-0x2fffffff]
[    0.538431] e820: reserve RAM buffer [mem 0x24f340000-0x24fffffff]
[    0.541013] pci 0000:04:00.0: vgaarb: setting as boot VGA device
[    0.541017] pci 0000:04:00.0: vgaarb: bridge control possible
[    0.541019] pci 0000:04:00.0: vgaarb: VGA device added: decodes=io+mem,owns=io+mem,locks=none
[    0.541024] vgaarb: loaded
[    0.541249] clocksource: Switched to clocksource tsc-early
[    0.541879] VFS: Disk quotas dquot_6.6.0
[    0.541895] VFS: Dquot-cache hash table entries: 512 (order 0, 4096 bytes)
[    0.541961] FS-Cache: Loaded
[    0.542077] pnp: PnP ACPI init
[    0.542315] system 00:00: [mem 0xfec00000-0xfec01fff] could not be reserved
[    0.542320] system 00:00: [mem 0xfee00000-0xfee00fff] has been reserved
[    0.542322] system 00:00: [mem 0xfde00000-0xfdefffff] has been reserved
[    0.543296] system 00:03: [io  0x0400-0x04cf] has been reserved
[    0.543300] system 00:03: [io  0x04d0-0x04d1] has been reserved
[    0.543302] system 00:03: [io  0x04d6] has been reserved
[    0.543305] system 00:03: [io  0x0c00-0x0c01] has been reserved
[    0.543307] system 00:03: [io  0x0c14] has been reserved
[    0.543309] system 00:03: [io  0x0c50-0x0c52] has been reserved
[    0.543312] system 00:03: [io  0x0c6c] has been reserved
[    0.543314] system 00:03: [io  0x0c6f] has been reserved
[    0.543316] system 00:03: [io  0x0cd0-0x0cdb] has been reserved
[    0.543426] system 00:04: [mem 0x000e0000-0x000fffff] could not be reserved
[    0.543430] system 00:04: [mem 0xff800000-0xffffffff] could not be reserved
[    0.544283] pnp: PnP ACPI: found 5 devices
[    0.552346] clocksource: acpi_pm: mask: 0xffffff max_cycles: 0xffffff, max_idle_ns: 2085701024 ns
[    0.552442] NET: Registered PF_INET protocol family
[    0.552671] IP idents hash table entries: 131072 (order: 8, 1048576 bytes, linear)
[    0.554272] tcp_listen_portaddr_hash hash table entries: 4096 (order: 6, 294912 bytes, linear)
[    0.554353] Table-perturb hash table entries: 65536 (order: 6, 262144 bytes, linear)
[    0.554359] TCP established hash table entries: 65536 (order: 7, 524288 bytes, linear)
[    0.554513] TCP bind hash table entries: 65536 (order: 11, 9437184 bytes, vmalloc hugepage)
[    0.557016] TCP: Hash tables configured (established 65536 bind 65536)
[    0.557301] UDP hash table entries: 4096 (order: 7, 655360 bytes, linear)
[    0.557486] UDP-Lite hash table entries: 4096 (order: 7, 655360 bytes, linear)
[    0.557793] NET: Registered PF_UNIX/PF_LOCAL protocol family
[    0.558074] RPC: Registered named UNIX socket transport module.
[    0.558080] RPC: Registered udp transport module.
[    0.558081] RPC: Registered tcp transport module.
[    0.558082] RPC: Registered tcp NFSv4.1 backchannel transport module.
[    0.558098] pci 0000:00:01.2: bridge window [io  0x1000-0x0fff] to [bus 01] add_size 1000
[    0.558103] pci 0000:00:01.2: bridge window [mem 0x00100000-0x000fffff 64bit pref] to [bus 01] add_size 200000 add_align 100000
[    0.558106] pci 0000:00:01.2: bridge window [mem 0x00100000-0x000fffff] to [bus 01] add_size 200000 add_align 100000
[    0.558119] pci 0000:00:01.2: BAR 14: assigned [mem 0xc0900000-0xc0afffff]
[    0.558124] pci 0000:00:01.2: BAR 15: assigned [mem 0xc0b00000-0xc0cfffff 64bit pref]
[    0.558127] pci 0000:00:01.2: BAR 13: assigned [io  0x2000-0x2fff]
[    0.558131] pci 0000:00:01.2: PCI bridge to [bus 01]
[    0.558136] pci 0000:00:01.2:   bridge window [io  0x2000-0x2fff]
[    0.558146] pci 0000:00:01.2:   bridge window [mem 0xc0900000-0xc0afffff]
[    0.558153] pci 0000:00:01.2:   bridge window [mem 0xc0b00000-0xc0cfffff 64bit pref]
[    0.558164] pci 0000:00:01.3: PCI bridge to [bus 02]
[    0.558178] pci 0000:00:01.3:   bridge window [mem 0xc0200000-0xc02fffff 64bit pref]
[    0.558189] pci 0000:00:01.7: PCI bridge to [bus 03]
[    0.558198] pci 0000:00:01.7:   bridge window [mem 0xc0800000-0xc08fffff]
[    0.558218] pci 0000:00:08.1: PCI bridge to [bus 04]
[    0.558225] pci 0000:00:08.1:   bridge window [io  0x1000-0x1fff]
[    0.558230] pci 0000:00:08.1:   bridge window [mem 0xc0300000-0xc07fffff]
[    0.558234] pci 0000:00:08.1:   bridge window [mem 0xb0000000-0xc01fffff 64bit pref]
[    0.558242] pci_bus 0000:00: resource 4 [io  0x0000-0x0cf7 window]
[    0.558244] pci_bus 0000:00: resource 5 [io  0x0d00-0xffff window]
[    0.558246] pci_bus 0000:00: resource 6 [mem 0x000a0000-0x000bffff window]
[    0.558248] pci_bus 0000:00: resource 7 [mem 0x000c0000-0x000cbfff window]
[    0.558250] pci_bus 0000:00: resource 8 [mem 0x000d0000-0x000effff window]
[    0.558251] pci_bus 0000:00: resource 9 [mem 0xb0000000-0xefffffff window]
[    0.558253] pci_bus 0000:00: resource 10 [mem 0xf8000000-0xfeafffff window]
[    0.558255] pci_bus 0000:00: resource 11 [mem 0xfed45000-0xfed814ff window]
[    0.558256] pci_bus 0000:00: resource 12 [mem 0xfed81900-0xfed81fff window]
[    0.558258] pci_bus 0000:00: resource 13 [mem 0xfedc0000-0xfedc0fff window]
[    0.558259] pci_bus 0000:00: resource 14 [mem 0xfedc6000-0xfedc6fff window]
[    0.558261] pci_bus 0000:01: resource 0 [io  0x2000-0x2fff]
[    0.558263] pci_bus 0000:01: resource 1 [mem 0xc0900000-0xc0afffff]
[    0.558264] pci_bus 0000:01: resource 2 [mem 0xc0b00000-0xc0cfffff 64bit pref]
[    0.558266] pci_bus 0000:02: resource 2 [mem 0xc0200000-0xc02fffff 64bit pref]
[    0.558268] pci_bus 0000:03: resource 1 [mem 0xc0800000-0xc08fffff]
[    0.558270] pci_bus 0000:04: resource 0 [io  0x1000-0x1fff]
[    0.558272] pci_bus 0000:04: resource 1 [mem 0xc0300000-0xc07fffff]
[    0.558273] pci_bus 0000:04: resource 2 [mem 0xb0000000-0xc01fffff 64bit pref]
[    0.558738] pci 0000:04:00.1: D0 power state depends on 0000:04:00.0
[    0.558766] pci 0000:04:00.3: extending delay after power-on from D3hot to 20 msec
[    0.562726] pci 0000:04:00.4: extending delay after power-on from D3hot to 20 msec
[    0.581622] pci 0000:04:00.4: quirk_usb_early_handoff+0x0/0x6b0 took 18449 usecs
[    0.581646] PCI: CLS 64 bytes, default 64
[    0.581933] Trying to unpack rootfs image as initramfs...
[    0.582797] AMD-Vi: ivrs, add hid:PNPD0040, uid:, rdevid:152
[    0.582802] AMD-Vi: Using global IVHD EFR:0x4f77ef22294ada, EFR2:0x0
[    0.582925] pci 0000:00:00.2: AMD-Vi: IOMMU performance counters supported
[    0.583263] pci 0000:00:01.0: Adding to iommu group 0
[    0.583308] pci 0000:00:01.2: Adding to iommu group 1
[    0.583346] pci 0000:00:01.3: Adding to iommu group 2
[    0.583381] pci 0000:00:01.7: Adding to iommu group 3
[    0.583428] pci 0000:00:08.0: Adding to iommu group 4
[    0.583462] pci 0000:00:08.1: Adding to iommu group 5
[    0.583516] pci 0000:00:14.0: Adding to iommu group 6
[    0.583543] pci 0000:00:14.3: Adding to iommu group 6
[    0.583670] pci 0000:00:18.0: Adding to iommu group 7
[    0.583697] pci 0000:00:18.1: Adding to iommu group 7
[    0.583726] pci 0000:00:18.2: Adding to iommu group 7
[    0.583755] pci 0000:00:18.3: Adding to iommu group 7
[    0.583782] pci 0000:00:18.4: Adding to iommu group 7
[    0.583809] pci 0000:00:18.5: Adding to iommu group 7
[    0.583839] pci 0000:00:18.6: Adding to iommu group 7
[    0.583867] pci 0000:00:18.7: Adding to iommu group 7
[    0.583928] pci 0000:02:00.0: Adding to iommu group 8
[    0.583965] pci 0000:02:00.1: Adding to iommu group 8
[    0.584001] pci 0000:03:00.0: Adding to iommu group 9
[    0.584054] pci 0000:04:00.0: Adding to iommu group 10
[    0.584180] pci 0000:04:00.1: Adding to iommu group 11
[    0.584222] pci 0000:04:00.2: Adding to iommu group 11
[    0.584265] pci 0000:04:00.3: Adding to iommu group 11
[    0.584307] pci 0000:04:00.4: Adding to iommu group 11
[    0.584349] pci 0000:04:00.5: Adding to iommu group 11
[    0.584390] pci 0000:04:00.6: Adding to iommu group 11
[    0.584432] pci 0000:04:00.7: Adding to iommu group 11
[    0.587404] pci 0000:00:00.2: can't derive routing for PCI INT A
[    0.587413] pci 0000:00:00.2: PCI INT A: not connected
[    0.587928] pci 0000:00:00.2: AMD-Vi: Found IOMMU cap 0x40
[    0.587931] AMD-Vi: Extended features (0x4f77ef22294ada, 0x0): PPR NX GT IA GA PC GA_vAPIC
[    0.588398] PCI-DMA: Using software bounce buffering for IO (SWIOTLB)
[    0.588404] software IO TLB: mapped [mem 0x00000000265fb000-0x000000002a5fb000] (64MB)
[    0.588713] RAPL PMU: API unit is 2^-32 Joules, 1 fixed counters, 163840 ms ovfl timer
[    0.588718] RAPL PMU: hw unit of domain package 2^-16 Joules
[    0.588732] amd_uncore: 4  amd_df counters detected
[    0.588742] amd_uncore: 6  amd_l3 counters detected
[    0.588951] perf/amd_iommu: Detected AMD IOMMU #0 (2 banks, 4 counters/bank).
[    0.639941] Initialise system trusted keyrings
[    0.640122] workingset: timestamp_bits=40 max_order=21 bucket_order=0
[    0.640194] zbud: loaded
[    0.641248] NFS: Registering the id_resolver key type
[    0.641273] Key type id_resolver registered
[    0.641280] Key type id_legacy registered
[    0.641689] Key type cifs.idmap registered
[    0.641764] ntfs: driver 2.1.32 [Flags: R/W].
[    0.641955] SGI XFS with ACLs, security attributes, realtime, verbose warnings, quota, no debug enabled
[    0.646149] Freeing initrd memory: 15884K
[    0.654886] Key type asymmetric registered
[    0.654899] Asymmetric key parser 'x509' registered
[    0.654941] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 250)
[    0.655027] io scheduler mq-deadline registered
[    0.655030] io scheduler kyber registered
[    0.655537] pcieport 0000:00:01.2: PME: Signaling with IRQ 26
[    0.655685] pcieport 0000:00:01.2: AER: enabled with IRQ 26
[    0.655745] pcieport 0000:00:01.2: pciehp: Slot #0 AttnBtn- PwrCtrl- MRL- AttnInd- PwrInd- HotPlug+ Surprise- Interlock- NoCompl+ IbPresDis- LLActRep+
[    0.656281] pcieport 0000:00:01.3: PME: Signaling with IRQ 27
[    0.656402] pcieport 0000:00:01.3: AER: enabled with IRQ 27
[    0.656766] pcieport 0000:00:01.7: PME: Signaling with IRQ 28
[    0.656961] pcieport 0000:00:01.7: AER: enabled with IRQ 28
[    0.657365] pcieport 0000:00:08.1: PME: Signaling with IRQ 29
[    0.659611] ACPI: AC: AC Adapter [ACAD] (on-line)
[    0.659797] input: Power Button as /devices/LNXSYSTM:00/LNXSYBUS:00/PNP0C0C:00/input/input0
[    0.659897] ACPI: button: Power Button [PWRB]
[    0.660028] input: Lid Switch as /devices/LNXSYSTM:00/LNXSYBUS:00/PNP0C0D:00/input/input1
[    0.660093] ACPI: button: Lid Switch [LID]
[    0.660215] input: Power Button as /devices/LNXSYSTM:00/LNXPWRBN:00/input/input2
[    0.660448] ACPI: button: Power Button [PWRF]
[    0.660608] Monitor-Mwait will be used to enter C-1 state
[    0.660630] ACPI: \_PR_.C000: Found 2 idle states
[    0.660970] ACPI: \_PR_.C001: Found 2 idle states
[    0.661202] ACPI: \_PR_.C002: Found 2 idle states
[    0.661427] ACPI: \_PR_.C003: Found 2 idle states
[    0.661661] ACPI: \_PR_.C004: Found 2 idle states
[    0.661913] ACPI: \_PR_.C005: Found 2 idle states
[    0.662144] ACPI: \_PR_.C006: Found 2 idle states
[    0.662344] ACPI: \_PR_.C007: Found 2 idle states
[    0.662938] Serial: 8250/16550 driver, 4 ports, IRQ sharing enabled
[    0.663150] ACPI: battery: Slot [BATT] (battery absent)
[    0.664051] Linux agpgart interface v0.103
[    0.670813] brd: module loaded
[    0.675370] loop: module loaded
[    0.675413] Loading iSCSI transport class v2.0-870.
[    0.676625] nvme 0000:03:00.0: platform quirk: setting simple suspend
[    0.676898] nvme nvme0: pci function 0000:03:00.0
[    0.678038] tun: Universal TUN/TAP device driver, 1.6
[    0.690643] nvme nvme0: Shutdown timeout set to 10 seconds
[    0.694446] nvme nvme0: 16/0/0 default/read/poll queues
[    0.700134]  nvme0n1: p1 p2 p3
[    1.021713] tg3 0000:02:00.0 eth0: Tigon3 [partno(BCM95762) rev 5762100] (PCI Express) MAC address 00:00:1a:1c:c9:1e
[    1.021722] tg3 0000:02:00.0 eth0: attached PHY is 5762C (10/100/1000Base-T Ethernet) (WireSpeed[1], EEE[1])
[    1.021725] tg3 0000:02:00.0 eth0: RXcsums[1] LinkChgREG[0] MIirq[0] ASF[0] TSOcap[1]
[    1.021728] tg3 0000:02:00.0 eth0: dma_rwctrl[00000001] dma_mask[64-bit]
[    1.021949] e100: Intel(R) PRO/100 Network Driver
[    1.021952] e100: Copyright(c) 1999-2006 Intel Corporation
[    1.021983] e1000: Intel(R) PRO/1000 Network Driver
[    1.021985] e1000: Copyright (c) 1999-2006 Intel Corporation.
[    1.022009] e1000e: Intel(R) PRO/1000 Network Driver
[    1.022011] e1000e: Copyright(c) 1999 - 2015 Intel Corporation.
[    1.022048] igb: Intel(R) Gigabit Ethernet Network Driver
[    1.022051] igb: Copyright (c) 2007-2014 Intel Corporation.
[    1.022090] igbvf: Intel(R) Gigabit Virtual Function Network Driver
[    1.022093] igbvf: Copyright (c) 2009 - 2012 Intel Corporation.
[    1.022132] ixgbe: Intel(R) 10 Gigabit PCI Express Network Driver
[    1.022135] ixgbe: Copyright (c) 1999-2016 Intel Corporation.
[    1.022268] i40e: Intel(R) Ethernet Connection XL710 Network Driver
[    1.022274] i40e: Copyright (c) 2013 - 2019 Intel Corporation.
[    1.022408] sky2: driver version 1.30
[    1.022606] Fusion MPT base driver 3.04.20
[    1.022609] Copyright (c) 1999-2008 LSI Corporation
[    1.022620] Fusion MPT SPI Host driver 3.04.20
[    1.023089] xhci_hcd 0000:04:00.3: xHCI Host Controller
[    1.023174] xhci_hcd 0000:04:00.3: new USB bus registered, assigned bus number 1
[    1.023454] xhci_hcd 0000:04:00.3: hcc params 0x0270ffe5 hci version 0x110 quirks 0x0000000840000410
[    1.024376] xhci_hcd 0000:04:00.3: xHCI Host Controller
[    1.024385] xhci_hcd 0000:04:00.3: new USB bus registered, assigned bus number 2
[    1.024390] xhci_hcd 0000:04:00.3: Host supports USB 3.1 Enhanced SuperSpeed
[    1.025267] hub 1-0:1.0: USB hub found
[    1.025317] hub 1-0:1.0: 4 ports detected
[    1.026454] usb usb2: We don't know the algorithms for LPM for this host, disabling LPM.
[    1.026781] hub 2-0:1.0: USB hub found
[    1.026803] hub 2-0:1.0: 4 ports detected
[    1.028017] xhci_hcd 0000:04:00.4: xHCI Host Controller
[    1.028028] xhci_hcd 0000:04:00.4: new USB bus registered, assigned bus number 3
[    1.028219] xhci_hcd 0000:04:00.4: hcc params 0x0260ffe5 hci version 0x110 quirks 0x0000000840000410
[    1.028885] xhci_hcd 0000:04:00.4: xHCI Host Controller
[    1.028892] xhci_hcd 0000:04:00.4: new USB bus registered, assigned bus number 4
[    1.028897] xhci_hcd 0000:04:00.4: Host supports USB 3.1 Enhanced SuperSpeed
[    1.029265] hub 3-0:1.0: USB hub found
[    1.029285] hub 3-0:1.0: 2 ports detected
[    1.029649] usb usb4: We don't know the algorithms for LPM for this host, disabling LPM.
[    1.029936] hub 4-0:1.0: USB hub found
[    1.029955] hub 4-0:1.0: 1 port detected
[    1.030316] usbcore: registered new interface driver usb-storage
[    1.030400] i8042: PNP: PS/2 Controller [PNP0303:KBC0] at 0x60,0x64 irq 1
[    1.030405] i8042: PNP: PS/2 appears to have AUX port disabled, if this is incorrect please boot with i8042.nopnp
[    1.031587] i8042: Warning: Keylock active
[    1.031846] serio: i8042 KBD port at 0x60,0x64 irq 1
[    1.032196] mousedev: PS/2 mouse device common for all mice
[    1.032659] rtc_cmos 00:01: RTC can wake from S4
[    1.033331] rtc_cmos 00:01: registered as rtc0
[    1.033419] rtc_cmos 00:01: alarms up to one month, 114 bytes nvram, hpet irqs
[    1.033556] IR JVC protocol handler initialized
[    1.033559] IR MCE Keyboard/mouse protocol handler initialized
[    1.033561] IR NEC protocol handler initialized
[    1.033562] IR RC5(x/sz) protocol handler initialized
[    1.033564] IR RC6 protocol handler initialized
[    1.033565] IR SANYO protocol handler initialized
[    1.033567] IR Sharp protocol handler initialized
[    1.033568] IR Sony protocol handler initialized
[    1.033569] IR XMP protocol handler initialized
[    1.033571] fail to initialize ptp_kvm
[    1.033721] device-mapper: ioctl: 4.48.0-ioctl (2023-03-01) initialised: dm-devel@redhat.com
[    1.033800] hid: raw HID events driver (C) Jiri Kosina
[    1.034180] usbcore: registered new interface driver usbhid
[    1.034183] usbhid: USB HID core driver
[    1.034350] Initializing XFRM netlink socket
[    1.034362] input: AT Translated Set 2 keyboard as /devices/platform/i8042/serio0/input/input3
[    1.034424] NET: Registered PF_INET6 protocol family
[    1.035204] Segment Routing with IPv6
[    1.035216] In-situ OAM (IOAM) with IPv6
[    1.035243] sit: IPv6, IPv4 and MPLS over IPv4 tunneling driver
[    1.035587] NET: Registered PF_PACKET protocol family
[    1.035619] bridge: filtering via arp/ip/ip6tables is no longer available by default. Update your scripts to load br_netfilter if you need this.
[    1.035663] 8021q: 802.1Q VLAN Support v1.8
[    1.035691] Key type dns_resolver registered
[    1.037466] microcode: CPU0: patch_level=0x08108102
[    1.037466] microcode: CPU1: patch_level=0x08108102
[    1.037473] microcode: CPU3: patch_level=0x08108102
[    1.037478] microcode: CPU5: patch_level=0x08108102
[    1.037478] microcode: CPU4: patch_level=0x08108102
[    1.037483] microcode: CPU6: patch_level=0x08108102
[    1.037483] microcode: CPU7: patch_level=0x08108102
[    1.037514] microcode: CPU2: patch_level=0x08108102
[    1.037584] microcode: Microcode Update Driver: v2.2.
[    1.037591] IPI shorthand broadcast: enabled
[    1.047239] sched_clock: Marking stable (1045210814, 625538)->(1048899938, -3063586)
[    1.047556] registered taskstats version 1
[    1.047802] Loading compiled-in X.509 certificates
[    1.060115] Loaded X.509 cert 'Build time autogenerated kernel key: 6102cbc280931c6a134b7def5d8df54637e823b1'
[    1.068406] PM:   Magic number: 15:486:41
[    1.068430] bdi 7:6: hash matches
[    1.068682] printk: console [netcon0] enabled
[    1.068685] netconsole: network logging started
[    1.069054] acpi_cpufreq: overriding BIOS provided _PSD data
[    1.070411] clk: Disabling unused clocks
[    1.073479] Freeing unused kernel image (initmem) memory: 1872K
[    1.092585] Write protecting the kernel read-only data: 24576k
[    1.092877] Freeing unused kernel image (rodata/data gap) memory: 60K
[    1.092886] rodata_test: all tests were successful
[    1.092897] Run /init as init process
[    1.092899]   with arguments:
[    1.092901]     /init
[    1.092903]     splash
[    1.092905]   with environment:
[    1.092907]     HOME=/
[    1.092908]     TERM=linux
[    1.092910]     BOOT_IMAGE=/boot/vmlinuz-6.4.0-rc7-kfd-yangp
[    1.264703] usb 1-2: new low-speed USB device number 2 using xhci_hcd
[    1.276694] usb 3-1: new low-speed USB device number 2 using xhci_hcd
[    1.448811] input: Logitech USB-PS/2 Optical Mouse as /devices/pci0000:00/0000:00:08.1/0000:04:00.3/usb1/1-2/1-2:1.0/0003:046D:C00E.0001/input/input4
[    1.449717] hid-generic 0003:046D:C00E.0001: input,hidraw0: USB HID v1.10 Mouse [Logitech USB-PS/2 Optical Mouse] on usb-0000:04:00.3-2/input0
[    1.476959] tg3 0000:02:00.0 enp2s0f0: renamed from eth0
[    1.487720] piix4_smbus 0000:00:14.0: SMBus Host Controller at 0xb00, revision 0
[    1.487727] piix4_smbus 0000:00:14.0: Using register 0x02 for SMBus port selection
[    1.487978] piix4_smbus 0000:00:14.0: Auxiliary SMBus Host Controller at 0xb20
[    1.555517] input: Logitech USB Keyboard as /devices/pci0000:00/0000:00:08.1/0000:04:00.4/usb3/3-1/3-1:1.0/0003:046D:C31C.0002/input/input5
[    1.596803] tsc: Refined TSC clocksource calibration: 2096.053 MHz
[    1.596832] clocksource: tsc: mask: 0xffffffffffffffff max_cycles: 0x1e369fd61e0, max_idle_ns: 440795252296 ns
[    1.596928] clocksource: Switched to clocksource tsc
[    1.613555] hid-generic 0003:046D:C31C.0002: input,hidraw1: USB HID v1.10 Keyboard [Logitech USB Keyboard] on usb-0000:04:00.4-1/input0
[    1.623604] input: Logitech USB Keyboard Consumer Control as /devices/pci0000:00/0000:00:08.1/0000:04:00.4/usb3/3-1/3-1:1.1/0003:046D:C31C.0003/input/input6
[    1.681122] input: Logitech USB Keyboard System Control as /devices/pci0000:00/0000:00:08.1/0000:04:00.4/usb3/3-1/3-1:1.1/0003:046D:C31C.0003/input/input7
[    1.681549] hid-generic 0003:046D:C31C.0003: input,hidraw2: USB HID v1.10 Device [Logitech USB Keyboard] on usb-0000:04:00.4-1/input1
[    1.819625] EXT4-fs (nvme0n1p3): mounted filesystem 2abd16aa-0792-4adc-9147-dda46bfcc9fa ro with ordered data mode. Quota mode: none.
[    2.000398] systemd[1]: systemd 249.11-0ubuntu3.9 running in system mode (+PAM +AUDIT +SELINUX +APPARMOR +IMA +SMACK +SECCOMP +GCRYPT +GNUTLS +OPENSSL +ACL +BLKID +CURL +ELFUTILS +FIDO2 +IDN2 -IDN +IPTC +KMOD +LIBCRYPTSETUP +LIBFDISK +PCRE2 -PWQUALITY -P11KIT -QRENCODE +BZIP2 +LZ4 +XZ +ZLIB +ZSTD -XKBCOMMON +UTMP +SYSVINIT default-hierarchy=unified)
[    2.001094] systemd[1]: Detected architecture x86-64.
[    2.001610] systemd[1]: Hostname set to <atitest-Mandolin-PCO>.
[    2.043918] memfd_create() without MFD_EXEC nor MFD_NOEXEC_SEAL, pid=1 'systemd'
[    2.090416] block nvme0n1: the capability attribute has been deprecated.
[    2.308817] systemd[1]: Queued start job for default target Graphical Interface.
[    2.343511] systemd[1]: Created slice Slice /system/modprobe.
[    2.344999] systemd[1]: Created slice Slice /system/systemd-fsck.
[    2.345960] systemd[1]: Created slice User and Session Slice.
[    2.346377] systemd[1]: Started Forward Password Requests to Wall Directory Watch.
[    2.347278] systemd[1]: Set up automount Arbitrary Executable File Formats File System Automount Point.
[    2.347426] systemd[1]: Reached target User and Group Name Lookups.
[    2.347509] systemd[1]: Reached target Remote File Systems.
[    2.347561] systemd[1]: Reached target Slice Units.
[    2.347615] systemd[1]: Reached target Mounting snaps.
[    2.347679] systemd[1]: Reached target Local Verity Protected Volumes.
[    2.348678] systemd[1]: Listening on Syslog Socket.
[    2.349158] systemd[1]: Listening on fsck to fsckd communication Socket.
[    2.349492] systemd[1]: Listening on initctl Compatibility Named Pipe.
[    2.350457] systemd[1]: Listening on Journal Audit Socket.
[    2.350973] systemd[1]: Listening on Journal Socket (/dev/log).
[    2.351629] systemd[1]: Listening on Journal Socket.
[    2.352376] systemd[1]: Listening on udev Control Socket.
[    2.352899] systemd[1]: Listening on udev Kernel Socket.
[    2.355817] systemd[1]: Mounting Huge Pages File System...
[    2.358909] systemd[1]: Mounting POSIX Message Queue File System...
[    2.362120] systemd[1]: Mounting Kernel Debug File System...
[    2.365217] systemd[1]: Mounting Kernel Trace File System...
[    2.369758] systemd[1]: Starting Journal Service...
[    2.372723] systemd[1]: Starting Set the console keyboard layout...
[    2.376040] systemd[1]: Starting Create List of Static Device Nodes...
[    2.379227] systemd[1]: Starting Load Kernel Module chromeos_pstore...
[    2.382804] systemd[1]: Starting Load Kernel Module configfs...
[    2.388294] systemd[1]: Starting Load Kernel Module drm...
[    2.393539] systemd[1]: Starting Load Kernel Module efi_pstore...
[    2.397801] systemd[1]: Starting Load Kernel Module fuse...
[    2.403508] systemd[1]: Starting Load Kernel Module pstore_blk...
[    2.408462] systemd[1]: Starting Load Kernel Module pstore_zone...
[    2.412996] systemd[1]: Starting Load Kernel Module ramoops...
[    2.413491] systemd[1]: Condition check resulted in File System Check on Root Device being skipped.
[    2.418116] systemd[1]: Starting Load Kernel Modules...
[    2.421738] systemd[1]: Starting Remount Root and Kernel File Systems...
[    2.424051] ACPI: bus type drm_connector registered
[    2.424701] fuse: init (API version 7.38)
[    2.425892] systemd[1]: Starting Coldplug All udev Devices...
[    2.432835] systemd[1]: Mounted Huge Pages File System.
[    2.433224] systemd[1]: Mounted POSIX Message Queue File System.
[    2.433547] systemd[1]: Mounted Kernel Debug File System.
[    2.433862] systemd[1]: Mounted Kernel Trace File System.
[    2.435370] systemd[1]: Finished Create List of Static Device Nodes.
[    2.436785] systemd[1]: modprobe@chromeos_pstore.service: Deactivated successfully.
[    2.437804] systemd[1]: Finished Load Kernel Module chromeos_pstore.
[    2.438846] systemd[1]: modprobe@configfs.service: Deactivated successfully.
[    2.439903] systemd[1]: Finished Load Kernel Module configfs.
[    2.440967] systemd[1]: modprobe@drm.service: Deactivated successfully.
[    2.442148] systemd[1]: Finished Load Kernel Module drm.
[    2.443182] systemd[1]: modprobe@efi_pstore.service: Deactivated successfully.
[    2.444197] systemd[1]: Finished Load Kernel Module efi_pstore.
[    2.445284] systemd[1]: modprobe@fuse.service: Deactivated successfully.
[    2.446337] systemd[1]: Finished Load Kernel Module fuse.
[    2.447362] systemd[1]: modprobe@pstore_blk.service: Deactivated successfully.
[    2.448373] systemd[1]: Finished Load Kernel Module pstore_blk.
[    2.449563] systemd[1]: modprobe@pstore_zone.service: Deactivated successfully.
[    2.450571] systemd[1]: Finished Load Kernel Module pstore_zone.
[    2.451537] systemd[1]: modprobe@ramoops.service: Deactivated successfully.
[    2.452523] systemd[1]: Finished Load Kernel Module ramoops.
[    2.453296] EXT4-fs (nvme0n1p3): re-mounted 2abd16aa-0792-4adc-9147-dda46bfcc9fa r/w. Quota mode: none.
[    2.453875] systemd[1]: Finished Load Kernel Modules.
[    2.457437] systemd[1]: Finished Remount Root and Kernel File Systems.
[    2.460601] systemd[1]: Activating swap /swapfile...
[    2.464265] systemd[1]: Mounting FUSE Control File System...
[    2.470795] systemd[1]: Mounting Kernel Configuration File System...
[    2.471297] systemd[1]: Condition check resulted in Platform Persistent Storage Archival being skipped.
[    2.473954] Adding 2097148k swap on /swapfile.  Priority:-2 extents:16 across:832430080k SSFS
[    2.475453] systemd[1]: Starting Load/Save Random Seed...
[    2.479025] systemd[1]: Starting Apply Kernel Variables...
[    2.483878] systemd[1]: Starting Create System Users...
[    2.486406] systemd[1]: Activated swap /swapfile.
[    2.488751] systemd[1]: Mounted FUSE Control File System.
[    2.489331] systemd[1]: Mounted Kernel Configuration File System.
[    2.492489] systemd[1]: Reached target Swaps.
[    2.510505] systemd[1]: Finished Apply Kernel Variables.
[    2.516695] systemd[1]: Finished Set the console keyboard layout.
[    2.521461] systemd[1]: Finished Load/Save Random Seed.
[    2.521990] systemd[1]: Condition check resulted in First Boot Complete being skipped.
[    2.527370] systemd[1]: Finished Create System Users.
[    2.581910] systemd[1]: Starting Create Static Device Nodes in /dev...
[    2.582793] systemd[1]: Started Journal Service.
[    2.609958] systemd-journald[1017]: Received client request to flush runtime journal.
[    2.622060] loop0: detected capacity change from 0 to 8
[    2.625852] loop1: detected capacity change from 0 to 129600
[    2.629700] loop0: detected capacity change from 0 to 492776
[    2.635907] loop0: detected capacity change from 0 to 709280
[    2.640038] loop1: detected capacity change from 0 to 187776
[    2.646138] loop0: detected capacity change from 0 to 94064
[    2.649907] loop1: detected capacity change from 0 to 102072
[    2.655718] loop0: detected capacity change from 0 to 608
[    3.084482] ACPI: video: Video Device [VGA1] (multi-head: yes  rom: no  post: no)
[    3.086964] input: Video Bus as /devices/LNXSYSTM:00/LNXSYBUS:00/PNP0A08:00/device:0d/LNXVIDEO:01/input/input8
[    3.240131] AMD-Vi: AMD IOMMUv2 loaded and initialized
[    4.010868] loop0: detected capacity change from 0 to 8
[    4.662105] [drm] amdgpu kernel modesetting enabled.
[    4.680651] amdgpu: Topology: Add APU node [0x0:0x0]
[    4.681755] [drm] initializing kernel modesetting (RAVEN 0x1002:0x15D8 0x1002:0x0123 0xC2).
[    4.682441] [drm] register mmio base: 0xC0700000
[    4.682446] [drm] register mmio size: 524288
[    4.683316] [drm] add ip block number 0 <soc15_common>
[    4.683323] [drm] add ip block number 1 <gmc_v9_0>
[    4.683328] [drm] add ip block number 2 <vega10_ih>
[    4.683330] [drm] add ip block number 3 <psp>
[    4.683334] [drm] add ip block number 4 <powerplay>
[    4.683337] [drm] add ip block number 5 <dm>
[    4.683339] [drm] add ip block number 6 <gfx_v9_0>
[    4.683342] [drm] add ip block number 7 <sdma_v4_0>
[    4.683344] [drm] add ip block number 8 <vcn_v1_0>
[    4.728217] [drm] BIOS signature incorrect 0 0
[    4.728329] resource: resource sanity check: requesting [mem 0x00000000000c0000-0x00000000000dffff], which spans more than PCI Bus 0000:00 [mem 0x000c0000-0x000cbfff window]
[    4.728348] caller pci_map_rom+0x6b/0x1c0 mapping multiple BARs
[    4.731020] amdgpu 0000:04:00.0: amdgpu: Fetched VBIOS from ROM BAR
[    4.731030] amdgpu: ATOM BIOS: 113-PICASSO-114
[    4.745959] [drm] VCN decode is enabled in VM mode
[    4.745965] [drm] VCN encode is enabled in VM mode
[    4.745967] [drm] JPEG decode is enabled in VM mode
[    4.751343] [drm] MCBP is enabled
[    4.751438] amdgpu 0000:04:00.0: vgaarb: deactivate vga console
[    4.751446] amdgpu 0000:04:00.0: amdgpu: Trusted Memory Zone (TMZ) feature enabled
[    4.751740] stackdepot: allocating hash table of 524288 entries via kvcalloc
[    4.753461] [drm] vm size is 262144 GB, 4 levels, block size is 9-bit, fragment size is 9-bit
[    4.753483] amdgpu 0000:04:00.0: amdgpu: VRAM: 2048M 0x000000F400000000 - 0x000000F47FFFFFFF (2048M used)
[    4.753488] amdgpu 0000:04:00.0: amdgpu: GART: 1024M 0x0000000000000000 - 0x000000003FFFFFFF
[    4.753492] amdgpu 0000:04:00.0: amdgpu: AGP: 267419648M 0x000000F800000000 - 0x0000FFFFFFFFFFFF
[    4.753510] [drm] Detected VRAM RAM=2048M, BAR=2048M
[    4.753513] [drm] RAM width 64bits DDR4
[    4.754706] [drm] amdgpu: 2048M of VRAM memory ready
[    4.754715] [drm] amdgpu: 2912M of GTT memory ready.
[    4.755394] [drm] GART: num cpu pages 262144, num gpu pages 262144
[    4.755936] [drm] PCIE GART of 1024M enabled.
[    4.755944] [drm] PTB located at 0x000000F400A00000
[    4.756621] amdgpu: hwmgr_sw_init smu backed is smu10_smu
[    4.759442] [drm] Found VCN firmware Version ENC: 1.13 DEC: 2 VEP: 0 Revision: 3
[    4.759477] amdgpu 0000:04:00.0: amdgpu: Will use PSP to load VCN firmware
[    4.782530] [drm] reserve 0x400000 from 0xf47fc00000 for PSP TMR
[    4.895029] amdgpu 0000:04:00.0: amdgpu: RAS: optional ras ta ucode is not available
[    4.910079] amdgpu 0000:04:00.0: amdgpu: RAP: optional rap ta ucode is not available
[    4.917112] [drm] DM_PPLIB: values for F clock
[    4.917119] [drm] DM_PPLIB:	 400000 in kHz, 2874 in mV
[    4.917125] [drm] DM_PPLIB:	 933000 in kHz, 3224 in mV
[    4.917130] [drm] DM_PPLIB:	 1067000 in kHz, 3924 in mV
[    4.917132] [drm] DM_PPLIB:	 1200000 in kHz, 4074 in mV
[    4.917135] [drm] DM_PPLIB: values for DCF clock
[    4.917138] [drm] DM_PPLIB:	 300000 in kHz, 2874 in mV
[    4.917140] [drm] DM_PPLIB:	 600000 in kHz, 3224 in mV
[    4.917142] [drm] DM_PPLIB:	 626000 in kHz, 3924 in mV
[    4.917144] [drm] DM_PPLIB:	 654000 in kHz, 4074 in mV
[    4.917625] [drm] Display Core v3.2.243 initialized on DCN 1.0
[    4.965758] [drm] kiq ring mec 2 pipe 1 q 0
[    4.979300] [drm] VCN decode and encode initialized successfully(under SPG Mode).
[    5.021343] amdgpu: HMM registered 2048MB device memory
[    5.023303] kfd kfd: amdgpu: Allocated 3969056 bytes on gart
[    5.023352] kfd kfd: amdgpu: Total number of KFD nodes to be created: 1
[    5.023763] amdgpu: Topology: Add APU node [0x15d8:0x1002]
[    5.023971] kfd kfd: amdgpu: added device 1002:15d8
[    5.024011] amdgpu 0000:04:00.0: amdgpu: SE 1, SH per SE 1, CU per SH 11, active_cu_number 8
[    5.024355] amdgpu 0000:04:00.0: amdgpu: ring gfx uses VM inv eng 0 on hub 0
[    5.024359] amdgpu 0000:04:00.0: amdgpu: ring gfx_low uses VM inv eng 1 on hub 0
[    5.024362] amdgpu 0000:04:00.0: amdgpu: ring gfx_high uses VM inv eng 4 on hub 0
[    5.024368] amdgpu 0000:04:00.0: amdgpu: ring comp_1.0.0 uses VM inv eng 5 on hub 0
[    5.024372] amdgpu 0000:04:00.0: amdgpu: ring comp_1.1.0 uses VM inv eng 6 on hub 0
[    5.024379] amdgpu 0000:04:00.0: amdgpu: ring comp_1.2.0 uses VM inv eng 7 on hub 0
[    5.024381] amdgpu 0000:04:00.0: amdgpu: ring comp_1.3.0 uses VM inv eng 8 on hub 0
[    5.024383] amdgpu 0000:04:00.0: amdgpu: ring comp_1.0.1 uses VM inv eng 9 on hub 0
[    5.024386] amdgpu 0000:04:00.0: amdgpu: ring comp_1.1.1 uses VM inv eng 10 on hub 0
[    5.024391] amdgpu 0000:04:00.0: amdgpu: ring comp_1.2.1 uses VM inv eng 11 on hub 0
[    5.024395] amdgpu 0000:04:00.0: amdgpu: ring comp_1.3.1 uses VM inv eng 12 on hub 0
[    5.024398] amdgpu 0000:04:00.0: amdgpu: ring kiq_0.2.1.0 uses VM inv eng 13 on hub 0
[    5.024400] amdgpu 0000:04:00.0: amdgpu: ring sdma0 uses VM inv eng 0 on hub 8
[    5.024403] amdgpu 0000:04:00.0: amdgpu: ring vcn_dec uses VM inv eng 1 on hub 8
[    5.024405] amdgpu 0000:04:00.0: amdgpu: ring vcn_enc0 uses VM inv eng 4 on hub 8
[    5.024408] amdgpu 0000:04:00.0: amdgpu: ring vcn_enc1 uses VM inv eng 5 on hub 8
[    5.024409] amdgpu 0000:04:00.0: amdgpu: ring jpeg_dec uses VM inv eng 6 on hub 8
[    5.039987] [drm] Initialized amdgpu 3.54.0 20150101 for 0000:04:00.0 on minor 0
[    5.064819] fbcon: amdgpudrmfb (fb0) is primary device
[    5.160545] Console: switching to colour frame buffer device 200x75
[    5.178402] amdgpu 0000:04:00.0: [drm] fb0: amdgpudrmfb frame buffer device
[    7.012407] tg3 0000:02:00.0 enp2s0f0: Link is up at 100 Mbps, full duplex
[    7.012455] tg3 0000:02:00.0 enp2s0f0: Flow control is on for TX and on for RX
[    7.012461] tg3 0000:02:00.0 enp2s0f0: EEE is disabled
[    7.012722] IPv6: ADDRCONF(NETDEV_CHANGE): enp2s0f0: link becomes ready
[    8.967595] rfkill: input handler disabled

[-- Attachment #3: dmesg_v6.2.8.log --]
[-- Type: text/x-log, Size: 69859 bytes --]

[    0.000000] Linux version 6.2.8-kfd-yangp (yangp@Philip-Dev) (gcc (Ubuntu 8.4.0-1ubuntu1~18.04) 8.4.0, GNU ld (GNU Binutils for Ubuntu) 2.30) #1740 SMP PREEMPT_DYNAMIC Thu Jul 20 16:06:14 EDT 2023
[    0.000000] Command line: BOOT_IMAGE=/boot/vmlinuz-6.2.8-kfd-yangp root=UUID=2abd16aa-0792-4adc-9147-dda46bfcc9fa ro quiet splash fbcon=font:10x18 drm_kms_helper.drm_fbdev_overalloc=112 amdgpu.noretry=1 amdgpu.mcbp=1 vt.handoff=7
[    0.000000] KERNEL supported cpus:
[    0.000000]   Intel GenuineIntel
[    0.000000]   AMD AuthenticAMD
[    0.000000]   Hygon HygonGenuine
[    0.000000]   Centaur CentaurHauls
[    0.000000]   zhaoxin   Shanghai  
[    0.000000] x86/fpu: Supporting XSAVE feature 0x001: 'x87 floating point registers'
[    0.000000] x86/fpu: Supporting XSAVE feature 0x002: 'SSE registers'
[    0.000000] x86/fpu: Supporting XSAVE feature 0x004: 'AVX registers'
[    0.000000] x86/fpu: xstate_offset[2]:  576, xstate_sizes[2]:  256
[    0.000000] x86/fpu: Enabled xstate features 0x7, context size is 832 bytes, using 'compacted' format.
[    0.000000] signal: max sigframe size: 1776
[    0.000000] BIOS-provided physical RAM map:
[    0.000000] BIOS-e820: [mem 0x0000000000000000-0x000000000009f7ff] usable
[    0.000000] BIOS-e820: [mem 0x000000000009f800-0x000000000009ffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000000e0000-0x00000000000fffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000000100000-0x0000000009afffff] usable
[    0.000000] BIOS-e820: [mem 0x0000000009b00000-0x0000000009dfffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000009e00000-0x0000000009efffff] usable
[    0.000000] BIOS-e820: [mem 0x0000000009f00000-0x0000000009f0afff] ACPI NVS
[    0.000000] BIOS-e820: [mem 0x0000000009f0b000-0x000000002b57efff] usable
[    0.000000] BIOS-e820: [mem 0x000000002b57f000-0x000000002d77efff] reserved
[    0.000000] BIOS-e820: [mem 0x000000002d77f000-0x000000002f77efff] ACPI NVS
[    0.000000] BIOS-e820: [mem 0x000000002f77f000-0x000000002f7fefff] ACPI data
[    0.000000] BIOS-e820: [mem 0x000000002f7ff000-0x000000002f7fffff] usable
[    0.000000] BIOS-e820: [mem 0x000000002f800000-0x00000000afffffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000f0000000-0x00000000f7ffffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fdc00000-0x00000000fec01fff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fec10000-0x00000000fec10fff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fec20000-0x00000000fec20fff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fed80000-0x00000000fed81fff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fedc0000-0x00000000feddffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fee00000-0x00000000fee00fff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000ff800000-0x00000000fff2ffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000100000000-0x000000024f33ffff] usable
[    0.000000] NX (Execute Disable) protection: active
[    0.000000] SMBIOS 3.0 present.
[    0.000000] DMI: AMD Mandolin-PCO/Mandolin-PCO, BIOS TMC1003CD 04/03/2019
[    0.000000] tsc: Fast TSC calibration using PIT
[    0.000000] tsc: Detected 2096.007 MHz processor
[    0.008372] e820: update [mem 0x00000000-0x00000fff] usable ==> reserved
[    0.008376] e820: remove [mem 0x000a0000-0x000fffff] usable
[    0.008382] last_pfn = 0x24f340 max_arch_pfn = 0x400000000
[    0.008391] x86/PAT: Configuration [0-7]: WB  WC  UC- UC  WB  WP  UC- WT  
[    0.008743] last_pfn = 0x2f800 max_arch_pfn = 0x400000000
[    0.012987] Using GB pages for direct mapping
[    0.013324] RAMDISK: [mem 0x2a5fb000-0x2b57dfff]
[    0.013330] ACPI: Early table checksum verification disabled
[    0.013335] ACPI: RSDP 0x00000000000FE020 000024 (v02 AMD   )
[    0.013340] ACPI: XSDT 0x000000002F7D9188 0000FC (v01 AMD    Mandolin 00000001      01000013)
[    0.013347] ACPI: FACP 0x000000002F7F3000 00010C (v05 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013352] ACPI: DSDT 0x000000002F7E2000 0081BA (v01 AMD    Mandolin 00040000 ACPI 00040000)
[    0.013356] ACPI: FACS 0x000000002F764000 000040
[    0.013359] ACPI: SSDT 0x000000002F7F8000 005419 (v02 AMD    Mandolin 00000002 ACPI 00040000)
[    0.013362] ACPI: UEFI 0x000000002F7F7000 000236 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013366] ACPI: UEFI 0x000000002F7F6000 000042 (v01 AMD             00000000 ACPI 00040000)
[    0.013369] ACPI: ASF! 0x000000002F7F5000 0000A5 (v32 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013372] ACPI: BOOT 0x000000002F7F4000 000028 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013375] ACPI: HPET 0x000000002F7F2000 000038 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013379] ACPI: APIC 0x000000002F7F1000 000138 (v03 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013382] ACPI: MCFG 0x000000002F7F0000 00003C (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013385] ACPI: SLIC 0x000000002F7EF000 000176 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013388] ACPI: SPCR 0x000000002F7EE000 000050 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013391] ACPI: WSMT 0x000000002F7EB000 000028 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013394] ACPI: SSDT 0x000000002F7E0000 00046D (v01 AMD    Mandolin 00001000 ACPI 00040000)
[    0.013397] ACPI: TPM2 0x000000002F7DF000 000034 (v03 AMD    Mandolin 00000002 ACPI 00040000)
[    0.013400] ACPI: IVRS 0x000000002F7DE000 00013E (v02 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013403] ACPI: SSDT 0x000000002F7DC000 00119C (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013407] ACPI: CRAT 0x000000002F7DB000 000810 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013410] ACPI: CDIT 0x000000002F7DA000 000029 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013413] ACPI: SSDT 0x000000002F7D8000 0001D4 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013416] ACPI: SSDT 0x000000002F7D7000 0000B9 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013419] ACPI: SSDT 0x000000002F7D6000 000C33 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013422] ACPI: SSDT 0x000000002F7D4000 0010AC (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013425] ACPI: SSDT 0x000000002F7D3000 0002AA (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013428] ACPI: SSDT 0x000000002F7D1000 001C69 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013431] ACPI: FPDT 0x000000002F7D0000 000044 (v01 AMD    Mandolin 00000002 ACPI 00040000)
[    0.013434] ACPI: SSDT 0x000000002F7EC000 000F82 (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013437] ACPI: SSDT 0x000000002F7CE000 001CBD (v01 AMD    Mandolin 00000001 ACPI 00040000)
[    0.013440] ACPI: Reserving FACP table memory at [mem 0x2f7f3000-0x2f7f310b]
[    0.013442] ACPI: Reserving DSDT table memory at [mem 0x2f7e2000-0x2f7ea1b9]
[    0.013443] ACPI: Reserving FACS table memory at [mem 0x2f764000-0x2f76403f]
[    0.013444] ACPI: Reserving SSDT table memory at [mem 0x2f7f8000-0x2f7fd418]
[    0.013445] ACPI: Reserving UEFI table memory at [mem 0x2f7f7000-0x2f7f7235]
[    0.013446] ACPI: Reserving UEFI table memory at [mem 0x2f7f6000-0x2f7f6041]
[    0.013447] ACPI: Reserving ASF! table memory at [mem 0x2f7f5000-0x2f7f50a4]
[    0.013449] ACPI: Reserving BOOT table memory at [mem 0x2f7f4000-0x2f7f4027]
[    0.013450] ACPI: Reserving HPET table memory at [mem 0x2f7f2000-0x2f7f2037]
[    0.013451] ACPI: Reserving APIC table memory at [mem 0x2f7f1000-0x2f7f1137]
[    0.013452] ACPI: Reserving MCFG table memory at [mem 0x2f7f0000-0x2f7f003b]
[    0.013453] ACPI: Reserving SLIC table memory at [mem 0x2f7ef000-0x2f7ef175]
[    0.013454] ACPI: Reserving SPCR table memory at [mem 0x2f7ee000-0x2f7ee04f]
[    0.013455] ACPI: Reserving WSMT table memory at [mem 0x2f7eb000-0x2f7eb027]
[    0.013456] ACPI: Reserving SSDT table memory at [mem 0x2f7e0000-0x2f7e046c]
[    0.013457] ACPI: Reserving TPM2 table memory at [mem 0x2f7df000-0x2f7df033]
[    0.013458] ACPI: Reserving IVRS table memory at [mem 0x2f7de000-0x2f7de13d]
[    0.013459] ACPI: Reserving SSDT table memory at [mem 0x2f7dc000-0x2f7dd19b]
[    0.013460] ACPI: Reserving CRAT table memory at [mem 0x2f7db000-0x2f7db80f]
[    0.013461] ACPI: Reserving CDIT table memory at [mem 0x2f7da000-0x2f7da028]
[    0.013462] ACPI: Reserving SSDT table memory at [mem 0x2f7d8000-0x2f7d81d3]
[    0.013463] ACPI: Reserving SSDT table memory at [mem 0x2f7d7000-0x2f7d70b8]
[    0.013464] ACPI: Reserving SSDT table memory at [mem 0x2f7d6000-0x2f7d6c32]
[    0.013465] ACPI: Reserving SSDT table memory at [mem 0x2f7d4000-0x2f7d50ab]
[    0.013466] ACPI: Reserving SSDT table memory at [mem 0x2f7d3000-0x2f7d32a9]
[    0.013467] ACPI: Reserving SSDT table memory at [mem 0x2f7d1000-0x2f7d2c68]
[    0.013468] ACPI: Reserving FPDT table memory at [mem 0x2f7d0000-0x2f7d0043]
[    0.013469] ACPI: Reserving SSDT table memory at [mem 0x2f7ec000-0x2f7ecf81]
[    0.013471] ACPI: Reserving SSDT table memory at [mem 0x2f7ce000-0x2f7cfcbc]
[    0.013587] No NUMA configuration found
[    0.013588] Faking a node at [mem 0x0000000000000000-0x000000024f33ffff]
[    0.013591] NODE_DATA(0) allocated [mem 0x24f33a000-0x24f33ffff]
[    0.013616] Zone ranges:
[    0.013617]   DMA      [mem 0x0000000000001000-0x0000000000ffffff]
[    0.013619]   DMA32    [mem 0x0000000001000000-0x00000000ffffffff]
[    0.013620]   Normal   [mem 0x0000000100000000-0x000000024f33ffff]
[    0.013622]   Device   empty
[    0.013624] Movable zone start for each node
[    0.013625] Early memory node ranges
[    0.013625]   node   0: [mem 0x0000000000001000-0x000000000009efff]
[    0.013627]   node   0: [mem 0x0000000000100000-0x0000000009afffff]
[    0.013628]   node   0: [mem 0x0000000009e00000-0x0000000009efffff]
[    0.013629]   node   0: [mem 0x0000000009f0b000-0x000000002b57efff]
[    0.013630]   node   0: [mem 0x000000002f7ff000-0x000000002f7fffff]
[    0.013631]   node   0: [mem 0x0000000100000000-0x000000024f33ffff]
[    0.013633] Initmem setup node 0 [mem 0x0000000000001000-0x000000024f33ffff]
[    0.013637] On node 0, zone DMA: 1 pages in unavailable ranges
[    0.013659] On node 0, zone DMA: 97 pages in unavailable ranges
[    0.013853] On node 0, zone DMA32: 768 pages in unavailable ranges
[    0.015240] On node 0, zone DMA32: 11 pages in unavailable ranges
[    0.015445] On node 0, zone DMA32: 17024 pages in unavailable ranges
[    0.030741] On node 0, zone Normal: 2048 pages in unavailable ranges
[    0.030786] On node 0, zone Normal: 3264 pages in unavailable ranges
[    0.030938] ACPI: PM-Timer IO Port: 0x408
[    0.030945] ACPI: LAPIC_NMI (acpi_id[0x00] high edge lint[0x1])
[    0.030947] ACPI: LAPIC_NMI (acpi_id[0x01] high edge lint[0x1])
[    0.030948] ACPI: LAPIC_NMI (acpi_id[0x02] high edge lint[0x1])
[    0.030949] ACPI: LAPIC_NMI (acpi_id[0x03] high edge lint[0x1])
[    0.030950] ACPI: LAPIC_NMI (acpi_id[0x04] high edge lint[0x1])
[    0.030951] ACPI: LAPIC_NMI (acpi_id[0x05] high edge lint[0x1])
[    0.030952] ACPI: LAPIC_NMI (acpi_id[0x06] high edge lint[0x1])
[    0.030953] ACPI: LAPIC_NMI (acpi_id[0x07] high edge lint[0x1])
[    0.030954] ACPI: LAPIC_NMI (acpi_id[0x08] high edge lint[0x1])
[    0.030955] ACPI: LAPIC_NMI (acpi_id[0x09] high edge lint[0x1])
[    0.030956] ACPI: LAPIC_NMI (acpi_id[0x0a] high edge lint[0x1])
[    0.030956] ACPI: LAPIC_NMI (acpi_id[0x0b] high edge lint[0x1])
[    0.030957] ACPI: LAPIC_NMI (acpi_id[0x0c] high edge lint[0x1])
[    0.030958] ACPI: LAPIC_NMI (acpi_id[0x0d] high edge lint[0x1])
[    0.030959] ACPI: LAPIC_NMI (acpi_id[0x0e] high edge lint[0x1])
[    0.030960] ACPI: LAPIC_NMI (acpi_id[0x0f] high edge lint[0x1])
[    0.030992] IOAPIC[0]: apic_id 33, version 33, address 0xfec00000, GSI 0-23
[    0.031006] IOAPIC[1]: apic_id 34, version 33, address 0xfec01000, GSI 24-55
[    0.031008] ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
[    0.031010] ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 low level)
[    0.031015] ACPI: Using ACPI (MADT) for SMP configuration information
[    0.031016] ACPI: HPET id: 0x10228210 base: 0xfed00000
[    0.031023] ACPI: SPCR: SPCR table version 1
[    0.031025] ACPI: SPCR: console: uart,io,0x3f8,115200
[    0.031028] smpboot: Allowing 8 CPUs, 0 hotplug CPUs
[    0.031059] PM: hibernation: Registered nosave memory: [mem 0x00000000-0x00000fff]
[    0.031062] PM: hibernation: Registered nosave memory: [mem 0x0009f000-0x0009ffff]
[    0.031063] PM: hibernation: Registered nosave memory: [mem 0x000a0000-0x000dffff]
[    0.031064] PM: hibernation: Registered nosave memory: [mem 0x000e0000-0x000fffff]
[    0.031066] PM: hibernation: Registered nosave memory: [mem 0x09b00000-0x09dfffff]
[    0.031068] PM: hibernation: Registered nosave memory: [mem 0x09f00000-0x09f0afff]
[    0.031070] PM: hibernation: Registered nosave memory: [mem 0x2b57f000-0x2d77efff]
[    0.031071] PM: hibernation: Registered nosave memory: [mem 0x2d77f000-0x2f77efff]
[    0.031072] PM: hibernation: Registered nosave memory: [mem 0x2f77f000-0x2f7fefff]
[    0.031074] PM: hibernation: Registered nosave memory: [mem 0x2f800000-0xafffffff]
[    0.031075] PM: hibernation: Registered nosave memory: [mem 0xb0000000-0xefffffff]
[    0.031075] PM: hibernation: Registered nosave memory: [mem 0xf0000000-0xf7ffffff]
[    0.031076] PM: hibernation: Registered nosave memory: [mem 0xf8000000-0xfdbfffff]
[    0.031077] PM: hibernation: Registered nosave memory: [mem 0xfdc00000-0xfec01fff]
[    0.031078] PM: hibernation: Registered nosave memory: [mem 0xfec02000-0xfec0ffff]
[    0.031079] PM: hibernation: Registered nosave memory: [mem 0xfec10000-0xfec10fff]
[    0.031080] PM: hibernation: Registered nosave memory: [mem 0xfec11000-0xfec1ffff]
[    0.031081] PM: hibernation: Registered nosave memory: [mem 0xfec20000-0xfec20fff]
[    0.031082] PM: hibernation: Registered nosave memory: [mem 0xfec21000-0xfed7ffff]
[    0.031083] PM: hibernation: Registered nosave memory: [mem 0xfed80000-0xfed81fff]
[    0.031084] PM: hibernation: Registered nosave memory: [mem 0xfed82000-0xfedbffff]
[    0.031084] PM: hibernation: Registered nosave memory: [mem 0xfedc0000-0xfeddffff]
[    0.031085] PM: hibernation: Registered nosave memory: [mem 0xfede0000-0xfedfffff]
[    0.031086] PM: hibernation: Registered nosave memory: [mem 0xfee00000-0xfee00fff]
[    0.031087] PM: hibernation: Registered nosave memory: [mem 0xfee01000-0xff7fffff]
[    0.031088] PM: hibernation: Registered nosave memory: [mem 0xff800000-0xfff2ffff]
[    0.031089] PM: hibernation: Registered nosave memory: [mem 0xfff30000-0xffffffff]
[    0.031092] [mem 0xb0000000-0xefffffff] available for PCI devices
[    0.031093] Booting paravirtualized kernel on bare hardware
[    0.031097] clocksource: refined-jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645519600211568 ns
[    0.036426] setup_percpu: NR_CPUS:256 nr_cpumask_bits:8 nr_cpu_ids:8 nr_node_ids:1
[    0.036879] percpu: Embedded 58 pages/cpu s196640 r8192 d32736 u262144
[    0.036886] pcpu-alloc: s196640 r8192 d32736 u262144 alloc=1*2097152
[    0.036889] pcpu-alloc: [0] 0 1 2 3 4 5 6 7 
[    0.036927] Fallback order for Node 0: 0 
[    0.036934] Built 1 zonelists, mobility grouping on.  Total pages: 1525278
[    0.036935] Policy zone: Normal
[    0.036942] Kernel command line: BOOT_IMAGE=/boot/vmlinuz-6.2.8-kfd-yangp root=UUID=2abd16aa-0792-4adc-9147-dda46bfcc9fa ro quiet splash fbcon=font:10x18 drm_kms_helper.drm_fbdev_overalloc=112 amdgpu.noretry=1 amdgpu.mcbp=1 vt.handoff=7
[    0.037076] Unknown kernel command line parameters "splash BOOT_IMAGE=/boot/vmlinuz-6.2.8-kfd-yangp", will be passed to user space.
[    0.037348] random: crng init done
[    0.038624] Dentry cache hash table entries: 1048576 (order: 11, 8388608 bytes, linear)
[    0.039332] Inode-cache hash table entries: 524288 (order: 10, 4194304 bytes, linear)
[    0.039418] mem auto-init: stack:off, heap alloc:off, heap free:off
[    0.039477] software IO TLB: area num 8.
[    0.075345] Memory: 5949644K/6198604K available (18432K kernel code, 3699K rwdata, 6000K rodata, 1836K init, 17776K bss, 248700K reserved, 0K cma-reserved)
[    0.075583] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=8, Nodes=1
[    0.075586] kmemleak: Kernel memory leak detector disabled
[    0.075659] ftrace: allocating 52585 entries in 206 pages
[    0.084181] ftrace: allocated 206 pages with 5 groups
[    0.085692] Dynamic Preempt: voluntary
[    0.085768] Running RCU self tests
[    0.085778] rcu: Preemptible hierarchical RCU implementation.
[    0.085779] rcu: 	RCU lockdep checking is enabled.
[    0.085780] rcu: 	RCU restricting CPUs from NR_CPUS=256 to nr_cpu_ids=8.
[    0.085782] 	Trampoline variant of Tasks RCU enabled.
[    0.085782] 	Rude variant of Tasks RCU enabled.
[    0.085783] 	Tracing variant of Tasks RCU enabled.
[    0.085784] rcu: RCU calculated value of scheduler-enlistment delay is 25 jiffies.
[    0.085785] rcu: Adjusting geometry for rcu_fanout_leaf=16, nr_cpu_ids=8
[    0.089155] NR_IRQS: 16640, nr_irqs: 1032, preallocated irqs: 16
[    0.089382] rcu: srcu_init: Setting srcu_struct sizes based on contention.
[    0.089567] spurious 8259A interrupt: IRQ7.
[    0.089607] Console: colour dummy device 80x25
[    0.089623] printk: console [tty0] enabled
[    0.089656] Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar
[    0.089658] ... MAX_LOCKDEP_SUBCLASSES:  8
[    0.089659] ... MAX_LOCK_DEPTH:          48
[    0.089661] ... MAX_LOCKDEP_KEYS:        8192
[    0.089662] ... CLASSHASH_SIZE:          4096
[    0.089663] ... MAX_LOCKDEP_ENTRIES:     32768
[    0.089664] ... MAX_LOCKDEP_CHAINS:      65536
[    0.089665] ... CHAINHASH_SIZE:          32768
[    0.089666]  memory used by lock dependency info: 6365 kB
[    0.089667]  memory used for stack traces: 4224 kB
[    0.089668]  per task-struct memory footprint: 1920 bytes
[    0.089696] ACPI: Core revision 20221020
[    0.090015] clocksource: hpet: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 133484873504 ns
[    0.090062] APIC: Switch to symmetric I/O mode setup
[    0.091341] ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1
[    0.110061] clocksource: tsc-early: mask: 0xffffffffffffffff max_cycles: 0x1e36749a135, max_idle_ns: 440795280167 ns
[    0.110084] Calibrating delay loop (skipped), value calculated using timer frequency.. 4192.01 BogoMIPS (lpj=8384028)
[    0.110088] pid_max: default: 32768 minimum: 301
[    0.110120] LSM: initializing lsm=capability,integrity,selinux
[    0.110131] SELinux:  Initializing.
[    0.110133] SELinux: CONFIG_SECURITY_SELINUX_CHECKREQPROT_VALUE is non-zero.  This is deprecated and will be rejected in a future kernel release.
[    0.110161] SELinux: https://github.com/SELinuxProject/selinux-kernel/wiki/DEPRECATE-checkreqprot
[    0.110274] Mount-cache hash table entries: 16384 (order: 5, 131072 bytes, linear)
[    0.110301] Mountpoint-cache hash table entries: 16384 (order: 5, 131072 bytes, linear)
[    0.111174] LVT offset 1 assigned for vector 0xf9
[    0.111256] LVT offset 2 assigned for vector 0xf4
[    0.111278] process: using mwait in idle threads
[    0.111280] Last level iTLB entries: 4KB 1024, 2MB 1024, 4MB 512
[    0.111282] Last level dTLB entries: 4KB 1536, 2MB 1536, 4MB 768, 1GB 0
[    0.111289] Spectre V1 : Mitigation: usercopy/swapgs barriers and __user pointer sanitization
[    0.111292] Spectre V2 : Kernel not compiled with retpoline; no mitigation available!
[    0.111294] Spectre V2 : Vulnerable
[    0.111304] Spectre V2 : Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch
[    0.111306] Spectre V2 : Enabling Speculation Barrier for firmware calls
[    0.111307] RETBleed: Mitigation: IBPB
[    0.111310] Spectre V2 : mitigation: Enabling conditional Indirect Branch Prediction Barrier
[    0.111313] Speculative Store Bypass: Mitigation: Speculative Store Bypass disabled via prctl
[    0.114794] Freeing SMP alternatives memory: 48K
[    0.224613] smpboot: CPU0: AMD Ryzen 5 3500U with Radeon Vega Mobile Gfx (family: 0x17, model: 0x18, stepping: 0x1)
[    0.224989] cblist_init_generic: Setting adjustable number of callback queues.
[    0.225005] cblist_init_generic: Setting shift to 3 and lim to 1.
[    0.225050] cblist_init_generic: Setting shift to 3 and lim to 1.
[    0.225108] cblist_init_generic: Setting shift to 3 and lim to 1.
[    0.225148] Running RCU-tasks wait API self tests
[    0.326283] Performance Events: Fam17h+ core perfctr, AMD PMU driver.
[    0.326360] ... version:                0
[    0.326363] ... bit width:              48
[    0.326365] ... generic registers:      6
[    0.326367] ... value mask:             0000ffffffffffff
[    0.326370] ... max period:             00007fffffffffff
[    0.326372] ... fixed-purpose events:   0
[    0.326374] ... event mask:             000000000000003f
[    0.326669] rcu: Hierarchical SRCU implementation.
[    0.326671] rcu: 	Max phase no-delay instances is 1000.
[    0.327412] NMI watchdog: Enabled. Permanently consumes one hw-PMU counter.
[    0.327669] smp: Bringing up secondary CPUs ...
[    0.328106] x86: Booting SMP configuration:
[    0.328113] .... node  #0, CPUs:      #1 #2 #3 #4 #5 #6
[    0.344471] Callback from call_rcu_tasks_trace() invoked.
[    0.344471]  #7
[    0.346146] smp: Brought up 1 node, 8 CPUs
[    0.346146] smpboot: Max logical packages: 1
[    0.346146] smpboot: Total of 8 processors activated (33536.11 BogoMIPS)
[    0.346973] devtmpfs: initialized
[    0.346973] x86/mm: Memory block size: 128MB
[    0.351606] ACPI: PM: Registering ACPI NVS region [mem 0x09f00000-0x09f0afff] (45056 bytes)
[    0.351606] ACPI: PM: Registering ACPI NVS region [mem 0x2d77f000-0x2f77efff] (33554432 bytes)
[    0.354174] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns
[    0.354183] futex hash table entries: 2048 (order: 6, 262144 bytes, linear)
[    0.354305] pinctrl core: initialized pinctrl subsystem
[    0.354470] PM: RTC time: 20:10:40, date: 2023-07-20
[    0.354698] NET: Registered PF_NETLINK/PF_ROUTE protocol family
[    0.355042] audit: initializing netlink subsys (disabled)
[    0.355057] audit: type=2000 audit(1689883840.264:1): state=initialized audit_enabled=0 res=1
[    0.355057] thermal_sys: Registered thermal governor 'step_wise'
[    0.355057] thermal_sys: Registered thermal governor 'user_space'
[    0.355057] cpuidle: using governor ladder
[    0.355057] cpuidle: using governor menu
[    0.355057] Simple Boot Flag at 0x44 set to 0x1
[    0.355057] PCI: MMCONFIG for domain 0000 [bus 00-7f] at [mem 0xf0000000-0xf7ffffff] (base 0xf0000000)
[    0.355057] PCI: MMCONFIG at [mem 0xf0000000-0xf7ffffff] reserved as E820 entry
[    0.355057] PCI: Using configuration type 1 for base access
[    0.355057] mtrr: your CPUs had inconsistent variable MTRR settings
[    0.355057] mtrr: probably your BIOS does not setup all CPUs.
[    0.355057] mtrr: corrected configuration.
[    0.355057] kprobes: kprobe jump-optimization is enabled. All kprobes are optimized if possible.
[    0.355057] HugeTLB: registered 1.00 GiB page size, pre-allocated 0 pages
[    0.355057] HugeTLB: 16380 KiB vmemmap can be freed for a 1.00 GiB page
[    0.355057] HugeTLB: registered 2.00 MiB page size, pre-allocated 0 pages
[    0.355057] HugeTLB: 28 KiB vmemmap can be freed for a 2.00 MiB page
[    0.355057] ACPI: Added _OSI(Module Device)
[    0.355057] ACPI: Added _OSI(Processor Device)
[    0.355057] ACPI: Added _OSI(3.0 _SCP Extensions)
[    0.355057] ACPI: Added _OSI(Processor Aggregator Device)
[    0.383121] ACPI: 12 ACPI AML tables successfully acquired and loaded
[    0.389843] ACPI: [Firmware Bug]: BIOS _OSI(Linux) query ignored
[    0.395344] ACPI: EC: EC started
[    0.395350] ACPI: EC: interrupt blocked
[    0.396793] ACPI: EC: EC_CMD/EC_SC=0x666, EC_DATA=0x662
[    0.396799] ACPI: \_SB_.PCI0.LPC0.EC0_: Boot DSDT EC used to handle transactions
[    0.396803] ACPI: Interpreter enabled
[    0.396875] ACPI: PM: (supports S0 S3 S4 S5)
[    0.396885] ACPI: Using IOAPIC for interrupt routing
[    0.397316] PCI: Using host bridge windows from ACPI; if necessary, use "pci=nocrs" and report a bug
[    0.397319] PCI: Using E820 reservations for host bridge windows
[    0.398893] ACPI: Enabled 4 GPEs in block 00 to 1F
[    0.422073] ACPI: \_SB_.PRWL: New power resource
[    0.423239] ACPI: PCI Root Bridge [PCI0] (domain 0000 [bus 00-ff])
[    0.423247] acpi PNP0A08:00: _OSC: OS supports [ExtendedConfig ASPM ClockPM Segments MSI HPX-Type3]
[    0.423396] acpi PNP0A08:00: _OSC: platform does not support [LTR]
[    0.423668] acpi PNP0A08:00: _OSC: OS now controls [PCIeHotplug PME AER PCIeCapability]
[    0.423700] acpi PNP0A08:00: [Firmware Info]: MMCONFIG for domain 0000 [bus 00-7f] only partially covers this bridge
[    0.423867] acpi PNP0A08:00: ignoring host bridge window [mem 0x000cc000-0x000cffff window] (conflicts with Video ROM [mem 0x000c0000-0x000cd3ff])
[    0.424283] PCI host bridge to bus 0000:00
[    0.424286] pci_bus 0000:00: root bus resource [io  0x0000-0x0cf7 window]
[    0.424289] pci_bus 0000:00: root bus resource [io  0x0d00-0xffff window]
[    0.424291] pci_bus 0000:00: root bus resource [mem 0x000a0000-0x000bffff window]
[    0.424294] pci_bus 0000:00: root bus resource [mem 0x000c0000-0x000cbfff window]
[    0.424296] pci_bus 0000:00: root bus resource [mem 0x000d0000-0x000effff window]
[    0.424298] pci_bus 0000:00: root bus resource [mem 0xb0000000-0xefffffff window]
[    0.424301] pci_bus 0000:00: root bus resource [mem 0xf8000000-0xfeafffff window]
[    0.424303] pci_bus 0000:00: root bus resource [mem 0xfed45000-0xfed814ff window]
[    0.424305] pci_bus 0000:00: root bus resource [mem 0xfed81900-0xfed81fff window]
[    0.424308] pci_bus 0000:00: root bus resource [mem 0xfedc0000-0xfedc0fff window]
[    0.424310] pci_bus 0000:00: root bus resource [mem 0xfedc6000-0xfedc6fff window]
[    0.424312] pci_bus 0000:00: root bus resource [bus 00-ff]
[    0.424382] pci 0000:00:00.0: [1022:15d0] type 00 class 0x060000
[    0.424635] pci 0000:00:00.2: [1022:15d1] type 00 class 0x080600
[    0.426268] pci 0000:00:01.0: [1022:1452] type 00 class 0x060000
[    0.426485] pci 0000:00:01.2: [1022:15d3] type 01 class 0x060400
[    0.426769] pci 0000:00:01.2: PME# supported from D0 D3hot D3cold
[    0.427338] pci 0000:00:01.3: [1022:15d3] type 01 class 0x060400
[    0.427450] pci 0000:00:01.3: enabling Extended Tags
[    0.427624] pci 0000:00:01.3: PME# supported from D0 D3hot D3cold
[    0.428096] pci 0000:00:01.7: [1022:15d3] type 01 class 0x060400
[    0.428373] pci 0000:00:01.7: PME# supported from D0 D3hot D3cold
[    0.428813] pci 0000:00:08.0: [1022:1452] type 00 class 0x060000
[    0.429013] pci 0000:00:08.1: [1022:15db] type 01 class 0x060400
[    0.429056] pci 0000:00:08.1: enabling Extended Tags
[    0.430129] pci 0000:00:08.1: PME# supported from D0 D3hot D3cold
[    0.430362] pci 0000:00:14.0: [1022:790b] type 00 class 0x0c0500
[    0.430362] pci 0000:00:14.3: [1022:790e] type 00 class 0x060100
[    0.430362] pci 0000:00:18.0: [1022:15e8] type 00 class 0x060000
[    0.430362] pci 0000:00:18.1: [1022:15e9] type 00 class 0x060000
[    0.430362] pci 0000:00:18.2: [1022:15ea] type 00 class 0x060000
[    0.430460] pci 0000:00:18.3: [1022:15eb] type 00 class 0x060000
[    0.430561] pci 0000:00:18.4: [1022:15ec] type 00 class 0x060000
[    0.430661] pci 0000:00:18.5: [1022:15ed] type 00 class 0x060000
[    0.430762] pci 0000:00:18.6: [1022:15ee] type 00 class 0x060000
[    0.430862] pci 0000:00:18.7: [1022:15ef] type 00 class 0x060000
[    0.431109] pci 0000:00:01.2: PCI bridge to [bus 01]
[    0.431295] pci 0000:02:00.0: [14e4:1687] type 00 class 0x020000
[    0.431322] pci 0000:02:00.0: reg 0x10: [mem 0xc0240000-0xc024ffff 64bit pref]
[    0.431341] pci 0000:02:00.0: reg 0x18: [mem 0xc0230000-0xc023ffff 64bit pref]
[    0.431359] pci 0000:02:00.0: reg 0x20: [mem 0xc0220000-0xc022ffff 64bit pref]
[    0.431494] pci 0000:02:00.0: PME# supported from D0 D3hot D3cold
[    0.431919] pci 0000:02:00.1: [14e4:1640] type 00 class 0x080501
[    0.431946] pci 0000:02:00.1: reg 0x10: [mem 0xc0210000-0xc021ffff 64bit pref]
[    0.431964] pci 0000:02:00.1: reg 0x18: [mem 0xc0200000-0xc020ffff 64bit pref]
[    0.432089] pci 0000:02:00.1: PME# supported from D0 D3hot D3cold
[    0.432488] pci 0000:00:01.3: PCI bridge to [bus 02]
[    0.432516] pci 0000:00:01.3:   bridge window [mem 0xc0200000-0xc02fffff 64bit pref]
[    0.432783] pci 0000:03:00.0: [144d:a80a] type 00 class 0x010802
[    0.432816] pci 0000:03:00.0: reg 0x10: [mem 0xc0800000-0xc0803fff 64bit]
[    0.433135] pci 0000:03:00.0: 4.000 Gb/s available PCIe bandwidth, limited by 2.5 GT/s PCIe x2 link at 0000:00:01.7 (capable of 63.012 Gb/s with 16.0 GT/s PCIe x4 link)
[    0.433565] pci 0000:00:01.7: PCI bridge to [bus 03]
[    0.433573] pci 0000:00:01.7:   bridge window [mem 0xc0800000-0xc08fffff]
[    0.433726] pci 0000:04:00.0: [1002:15d8] type 00 class 0x030000
[    0.433746] pci 0000:04:00.0: reg 0x10: [mem 0xb0000000-0xbfffffff 64bit pref]
[    0.433760] pci 0000:04:00.0: reg 0x18: [mem 0xc0000000-0xc01fffff 64bit pref]
[    0.433770] pci 0000:04:00.0: reg 0x20: [io  0x1000-0x10ff]
[    0.433779] pci 0000:04:00.0: reg 0x24: [mem 0xc0700000-0xc077ffff]
[    0.433795] pci 0000:04:00.0: enabling Extended Tags
[    0.433823] pci 0000:04:00.0: Video device with shadowed ROM at [mem 0x000c0000-0x000dffff]
[    0.433895] pci 0000:04:00.0: PME# supported from D1 D2 D3hot D3cold
[    0.434188] pci 0000:04:00.1: [1002:15de] type 00 class 0x040300
[    0.434205] pci 0000:04:00.1: reg 0x10: [mem 0xc07c8000-0xc07cbfff]
[    0.434256] pci 0000:04:00.1: enabling Extended Tags
[    0.434315] pci 0000:04:00.1: PME# supported from D1 D2 D3hot D3cold
[    0.434541] pci 0000:04:00.2: [1022:15df] type 00 class 0x108000
[    0.434572] pci 0000:04:00.2: reg 0x18: [mem 0xc0600000-0xc06fffff]
[    0.434596] pci 0000:04:00.2: reg 0x24: [mem 0xc07ce000-0xc07cffff]
[    0.434611] pci 0000:04:00.2: enabling Extended Tags
[    0.434871] pci 0000:04:00.3: [1022:15e0] type 00 class 0x0c0330
[    0.434891] pci 0000:04:00.3: reg 0x10: [mem 0xc0500000-0xc05fffff 64bit]
[    0.434935] pci 0000:04:00.3: enabling Extended Tags
[    0.434997] pci 0000:04:00.3: PME# supported from D0 D3hot D3cold
[    0.435243] pci 0000:04:00.4: [1022:15e1] type 00 class 0x0c0330
[    0.435263] pci 0000:04:00.4: reg 0x10: [mem 0xc0400000-0xc04fffff 64bit]
[    0.435308] pci 0000:04:00.4: enabling Extended Tags
[    0.435369] pci 0000:04:00.4: PME# supported from D0 D3hot D3cold
[    0.435596] pci 0000:04:00.5: [1022:15e2] type 00 class 0x048000
[    0.435612] pci 0000:04:00.5: reg 0x10: [mem 0xc0780000-0xc07bffff]
[    0.435664] pci 0000:04:00.5: enabling Extended Tags
[    0.435722] pci 0000:04:00.5: PME# supported from D0 D3hot D3cold
[    0.435933] pci 0000:04:00.6: [1022:15e3] type 00 class 0x040300
[    0.435950] pci 0000:04:00.6: reg 0x10: [mem 0xc07c0000-0xc07c7fff]
[    0.436001] pci 0000:04:00.6: enabling Extended Tags
[    0.436059] pci 0000:04:00.6: PME# supported from D0 D3hot D3cold
[    0.436271] pci 0000:04:00.7: [1022:15e6] type 00 class 0x000000
[    0.436303] pci 0000:04:00.7: reg 0x18: [mem 0xc0300000-0xc03fffff]
[    0.436335] pci 0000:04:00.7: reg 0x24: [mem 0xc07cc000-0xc07cdfff]
[    0.436359] pci 0000:04:00.7: enabling Extended Tags
[    0.436710] pci 0000:00:08.1: PCI bridge to [bus 04]
[    0.436717] pci 0000:00:08.1:   bridge window [io  0x1000-0x1fff]
[    0.436722] pci 0000:00:08.1:   bridge window [mem 0xc0300000-0xc07fffff]
[    0.436728] pci 0000:00:08.1:   bridge window [mem 0xb0000000-0xc01fffff 64bit pref]
[    0.438570] ACPI: PCI: Interrupt link LNKA configured for IRQ 0
[    0.438576] ACPI: PCI: Interrupt link LNKA disabled
[    0.438712] ACPI: PCI: Interrupt link LNKB configured for IRQ 0
[    0.438714] ACPI: PCI: Interrupt link LNKB disabled
[    0.438829] ACPI: PCI: Interrupt link LNKC configured for IRQ 0
[    0.438831] ACPI: PCI: Interrupt link LNKC disabled
[    0.438967] ACPI: PCI: Interrupt link LNKD configured for IRQ 0
[    0.438969] ACPI: PCI: Interrupt link LNKD disabled
[    0.439097] ACPI: PCI: Interrupt link LNKE configured for IRQ 0
[    0.439099] ACPI: PCI: Interrupt link LNKE disabled
[    0.439197] ACPI: PCI: Interrupt link LNKF configured for IRQ 0
[    0.439199] ACPI: PCI: Interrupt link LNKF disabled
[    0.439296] ACPI: PCI: Interrupt link LNKG configured for IRQ 0
[    0.439298] ACPI: PCI: Interrupt link LNKG disabled
[    0.439396] ACPI: PCI: Interrupt link LNKH configured for IRQ 0
[    0.439398] ACPI: PCI: Interrupt link LNKH disabled
[    0.450049] ACPI: EC: interrupt unblocked
[    0.450052] ACPI: EC: event unblocked
[    0.450066] ACPI: EC: EC_CMD/EC_SC=0x666, EC_DATA=0x662
[    0.450111] ACPI: EC: GPE=0x3
[    0.450111] ACPI: \_SB_.PCI0.LPC0.EC0_: Boot DSDT EC initialization complete
[    0.450111] ACPI: \_SB_.PCI0.LPC0.EC0_: EC: Used to handle transactions and events
[    0.450329] iommu: Default domain type: Translated 
[    0.450329] iommu: DMA domain TLB invalidation policy: lazy mode 
[    0.450487] SCSI subsystem initialized
[    0.450506] libata version 3.00 loaded.
[    0.450506] ACPI: bus type USB registered
[    0.450506] usbcore: registered new interface driver usbfs
[    0.450506] usbcore: registered new interface driver hub
[    0.450506] usbcore: registered new device driver usb
[    0.450506] pps_core: LinuxPPS API ver. 1 registered
[    0.450506] pps_core: Software ver. 5.3.6 - Copyright 2005-2007 Rodolfo Giometti <giometti@linux.it>
[    0.450506] PTP clock support registered
[    0.450598] NetLabel: Initializing
[    0.450601] NetLabel:  domain hash size = 128
[    0.450603] NetLabel:  protocols = UNLABELED CIPSOv4 CALIPSO
[    0.450645] NetLabel:  unlabeled traffic allowed by default
[    0.454257] PCI: Using ACPI for IRQ routing
[    0.457414] PCI: pci_cache_line_size set to 64 bytes
[    0.457816] resource: Expanded resource Reserved due to conflict with PCI Bus 0000:00
[    0.457823] resource: Expanded resource Reserved due to conflict with PCI Bus 0000:00
[    0.457826] e820: reserve RAM buffer [mem 0x0009f800-0x0009ffff]
[    0.457829] e820: reserve RAM buffer [mem 0x09b00000-0x0bffffff]
[    0.457831] e820: reserve RAM buffer [mem 0x09f00000-0x0bffffff]
[    0.457833] e820: reserve RAM buffer [mem 0x2b57f000-0x2bffffff]
[    0.457835] e820: reserve RAM buffer [mem 0x2f800000-0x2fffffff]
[    0.457837] e820: reserve RAM buffer [mem 0x24f340000-0x24fffffff]
[    0.458098] pci 0000:04:00.0: vgaarb: setting as boot VGA device
[    0.458103] pci 0000:04:00.0: vgaarb: bridge control possible
[    0.458105] pci 0000:04:00.0: vgaarb: VGA device added: decodes=io+mem,owns=io+mem,locks=none
[    0.458110] vgaarb: loaded
[    0.507176] clocksource: Switched to clocksource tsc-early
[    0.508162] Callback from call_rcu_tasks_rude() invoked.
[    0.532106] Callback from call_rcu_tasks() invoked.
[    0.594904] VFS: Disk quotas dquot_6.6.0
[    0.594934] VFS: Dquot-cache hash table entries: 512 (order 0, 4096 bytes)
[    0.595044] FS-Cache: Loaded
[    0.595256] pnp: PnP ACPI init
[    0.595704] system 00:00: [mem 0xfec00000-0xfec01fff] could not be reserved
[    0.595712] system 00:00: [mem 0xfee00000-0xfee00fff] has been reserved
[    0.595717] system 00:00: [mem 0xfde00000-0xfdefffff] has been reserved
[    0.597442] system 00:03: [io  0x0400-0x04cf] has been reserved
[    0.597448] system 00:03: [io  0x04d0-0x04d1] has been reserved
[    0.597453] system 00:03: [io  0x04d6] has been reserved
[    0.597457] system 00:03: [io  0x0c00-0x0c01] has been reserved
[    0.597461] system 00:03: [io  0x0c14] has been reserved
[    0.597465] system 00:03: [io  0x0c50-0x0c52] has been reserved
[    0.597469] system 00:03: [io  0x0c6c] has been reserved
[    0.597473] system 00:03: [io  0x0c6f] has been reserved
[    0.597477] system 00:03: [io  0x0cd0-0x0cdb] has been reserved
[    0.597662] system 00:04: [mem 0x000e0000-0x000fffff] could not be reserved
[    0.597668] system 00:04: [mem 0xff800000-0xffffffff] could not be reserved
[    0.599479] pnp: PnP ACPI: found 5 devices
[    0.609464] clocksource: acpi_pm: mask: 0xffffff max_cycles: 0xffffff, max_idle_ns: 2085701024 ns
[    0.609735] NET: Registered PF_INET protocol family
[    0.609957] IP idents hash table entries: 131072 (order: 8, 1048576 bytes, linear)
[    0.611807] tcp_listen_portaddr_hash hash table entries: 4096 (order: 6, 294912 bytes, linear)
[    0.611892] Table-perturb hash table entries: 65536 (order: 6, 262144 bytes, linear)
[    0.611899] TCP established hash table entries: 65536 (order: 7, 524288 bytes, linear)
[    0.612032] TCP bind hash table entries: 65536 (order: 11, 9437184 bytes, vmalloc hugepage)
[    0.614620] TCP: Hash tables configured (established 65536 bind 65536)
[    0.614896] UDP hash table entries: 4096 (order: 7, 655360 bytes, linear)
[    0.615094] UDP-Lite hash table entries: 4096 (order: 7, 655360 bytes, linear)
[    0.615395] NET: Registered PF_UNIX/PF_LOCAL protocol family
[    0.615664] RPC: Registered named UNIX socket transport module.
[    0.615669] RPC: Registered udp transport module.
[    0.615671] RPC: Registered tcp transport module.
[    0.615673] RPC: Registered tcp NFSv4.1 backchannel transport module.
[    0.615689] pci 0000:00:01.2: bridge window [io  0x1000-0x0fff] to [bus 01] add_size 1000
[    0.615694] pci 0000:00:01.2: bridge window [mem 0x00100000-0x000fffff 64bit pref] to [bus 01] add_size 200000 add_align 100000
[    0.615698] pci 0000:00:01.2: bridge window [mem 0x00100000-0x000fffff] to [bus 01] add_size 200000 add_align 100000
[    0.615711] pci 0000:00:01.2: BAR 14: assigned [mem 0xc0900000-0xc0afffff]
[    0.615717] pci 0000:00:01.2: BAR 15: assigned [mem 0xc0b00000-0xc0cfffff 64bit pref]
[    0.615721] pci 0000:00:01.2: BAR 13: assigned [io  0x2000-0x2fff]
[    0.615725] pci 0000:00:01.2: PCI bridge to [bus 01]
[    0.615730] pci 0000:00:01.2:   bridge window [io  0x2000-0x2fff]
[    0.615740] pci 0000:00:01.2:   bridge window [mem 0xc0900000-0xc0afffff]
[    0.615747] pci 0000:00:01.2:   bridge window [mem 0xc0b00000-0xc0cfffff 64bit pref]
[    0.615760] pci 0000:00:01.3: PCI bridge to [bus 02]
[    0.615775] pci 0000:00:01.3:   bridge window [mem 0xc0200000-0xc02fffff 64bit pref]
[    0.615787] pci 0000:00:01.7: PCI bridge to [bus 03]
[    0.615797] pci 0000:00:01.7:   bridge window [mem 0xc0800000-0xc08fffff]
[    0.615818] pci 0000:00:08.1: PCI bridge to [bus 04]
[    0.615826] pci 0000:00:08.1:   bridge window [io  0x1000-0x1fff]
[    0.615832] pci 0000:00:08.1:   bridge window [mem 0xc0300000-0xc07fffff]
[    0.615836] pci 0000:00:08.1:   bridge window [mem 0xb0000000-0xc01fffff 64bit pref]
[    0.615845] pci_bus 0000:00: resource 4 [io  0x0000-0x0cf7 window]
[    0.615847] pci_bus 0000:00: resource 5 [io  0x0d00-0xffff window]
[    0.615849] pci_bus 0000:00: resource 6 [mem 0x000a0000-0x000bffff window]
[    0.615851] pci_bus 0000:00: resource 7 [mem 0x000c0000-0x000cbfff window]
[    0.615853] pci_bus 0000:00: resource 8 [mem 0x000d0000-0x000effff window]
[    0.615855] pci_bus 0000:00: resource 9 [mem 0xb0000000-0xefffffff window]
[    0.615857] pci_bus 0000:00: resource 10 [mem 0xf8000000-0xfeafffff window]
[    0.615860] pci_bus 0000:00: resource 11 [mem 0xfed45000-0xfed814ff window]
[    0.615862] pci_bus 0000:00: resource 12 [mem 0xfed81900-0xfed81fff window]
[    0.615864] pci_bus 0000:00: resource 13 [mem 0xfedc0000-0xfedc0fff window]
[    0.615866] pci_bus 0000:00: resource 14 [mem 0xfedc6000-0xfedc6fff window]
[    0.615868] pci_bus 0000:01: resource 0 [io  0x2000-0x2fff]
[    0.615870] pci_bus 0000:01: resource 1 [mem 0xc0900000-0xc0afffff]
[    0.615872] pci_bus 0000:01: resource 2 [mem 0xc0b00000-0xc0cfffff 64bit pref]
[    0.615874] pci_bus 0000:02: resource 2 [mem 0xc0200000-0xc02fffff 64bit pref]
[    0.615877] pci_bus 0000:03: resource 1 [mem 0xc0800000-0xc08fffff]
[    0.615879] pci_bus 0000:04: resource 0 [io  0x1000-0x1fff]
[    0.615881] pci_bus 0000:04: resource 1 [mem 0xc0300000-0xc07fffff]
[    0.615883] pci_bus 0000:04: resource 2 [mem 0xb0000000-0xc01fffff 64bit pref]
[    0.616376] pci 0000:04:00.1: D0 power state depends on 0000:04:00.0
[    0.616408] pci 0000:04:00.3: extending delay after power-on from D3hot to 20 msec
[    0.623498] pci 0000:04:00.4: extending delay after power-on from D3hot to 20 msec
[    0.642007] pci 0000:04:00.4: quirk_usb_early_handoff+0x0/0x6b0 took 18071 usecs
[    0.642031] PCI: CLS 64 bytes, default 64
[    0.642344] Trying to unpack rootfs image as initramfs...
[    0.643161] AMD-Vi: ivrs, add hid:PNPD0040, uid:, rdevid:152
[    0.643165] AMD-Vi: Using global IVHD EFR:0x4f77ef22294ada, EFR2:0x0
[    0.643287] pci 0000:00:00.2: AMD-Vi: IOMMU performance counters supported
[    0.643568] pci 0000:00:01.0: Adding to iommu group 0
[    0.643611] pci 0000:00:01.2: Adding to iommu group 1
[    0.643654] pci 0000:00:01.3: Adding to iommu group 2
[    0.643696] pci 0000:00:01.7: Adding to iommu group 3
[    0.643742] pci 0000:00:08.0: Adding to iommu group 4
[    0.643777] pci 0000:00:08.1: Adding to iommu group 5
[    0.643833] pci 0000:00:14.0: Adding to iommu group 6
[    0.643861] pci 0000:00:14.3: Adding to iommu group 6
[    0.643980] pci 0000:00:18.0: Adding to iommu group 7
[    0.644009] pci 0000:00:18.1: Adding to iommu group 7
[    0.644038] pci 0000:00:18.2: Adding to iommu group 7
[    0.644067] pci 0000:00:18.3: Adding to iommu group 7
[    0.644095] pci 0000:00:18.4: Adding to iommu group 7
[    0.644123] pci 0000:00:18.5: Adding to iommu group 7
[    0.644151] pci 0000:00:18.6: Adding to iommu group 7
[    0.644179] pci 0000:00:18.7: Adding to iommu group 7
[    0.644239] pci 0000:02:00.0: Adding to iommu group 8
[    0.644275] pci 0000:02:00.1: Adding to iommu group 8
[    0.644311] pci 0000:03:00.0: Adding to iommu group 9
[    0.644371] pci 0000:04:00.0: Adding to iommu group 10
[    0.644490] pci 0000:04:00.1: Adding to iommu group 11
[    0.644534] pci 0000:04:00.2: Adding to iommu group 11
[    0.644577] pci 0000:04:00.3: Adding to iommu group 11
[    0.644620] pci 0000:04:00.4: Adding to iommu group 11
[    0.644664] pci 0000:04:00.5: Adding to iommu group 11
[    0.644705] pci 0000:04:00.6: Adding to iommu group 11
[    0.644748] pci 0000:04:00.7: Adding to iommu group 11
[    0.647286] pci 0000:00:00.2: can't derive routing for PCI INT A
[    0.647295] pci 0000:00:00.2: PCI INT A: not connected
[    0.647813] pci 0000:00:00.2: AMD-Vi: Found IOMMU cap 0x40
[    0.647817] AMD-Vi: Extended features (0x4f77ef22294ada, 0x0): PPR NX GT IA GA PC GA_vAPIC
[    0.648251] PCI-DMA: Using software bounce buffering for IO (SWIOTLB)
[    0.648254] software IO TLB: mapped [mem 0x00000000265fb000-0x000000002a5fb000] (64MB)
[    0.648488] RAPL PMU: API unit is 2^-32 Joules, 1 fixed counters, 163840 ms ovfl timer
[    0.648492] RAPL PMU: hw unit of domain package 2^-16 Joules
[    0.648503] amd_uncore: 4  amd_df counters detected
[    0.648510] amd_uncore: 6  amd_l3 counters detected
[    0.648688] perf/amd_iommu: Detected AMD IOMMU #0 (2 banks, 4 counters/bank).
[    0.650930] Initialise system trusted keyrings
[    0.651107] workingset: timestamp_bits=40 max_order=21 bucket_order=0
[    0.651163] zbud: loaded
[    0.652031] NFS: Registering the id_resolver key type
[    0.652049] Key type id_resolver registered
[    0.652055] Key type id_legacy registered
[    0.652348] Key type cifs.idmap registered
[    0.652391] ntfs: driver 2.1.32 [Flags: R/W].
[    0.652479] SGI XFS with ACLs, security attributes, realtime, verbose warnings, quota, no debug enabled
[    0.662642] Key type asymmetric registered
[    0.662667] Asymmetric key parser 'x509' registered
[    0.662711] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 250)
[    0.662845] io scheduler mq-deadline registered
[    0.662848] io scheduler kyber registered
[    0.663433] pcieport 0000:00:01.2: PME: Signaling with IRQ 26
[    0.663691] pcieport 0000:00:01.2: AER: enabled with IRQ 26
[    0.663868] pcieport 0000:00:01.2: pciehp: Slot #0 AttnBtn- PwrCtrl- MRL- AttnInd- PwrInd- HotPlug+ Surprise- Interlock- NoCompl+ IbPresDis- LLActRep+
[    0.664785] pcieport 0000:00:01.3: PME: Signaling with IRQ 27
[    0.665016] pcieport 0000:00:01.3: AER: enabled with IRQ 27
[    0.665558] pcieport 0000:00:01.7: PME: Signaling with IRQ 28
[    0.665909] pcieport 0000:00:01.7: AER: enabled with IRQ 28
[    0.666467] pcieport 0000:00:08.1: PME: Signaling with IRQ 29
[    0.669199] ACPI: AC: AC Adapter [ACAD] (on-line)
[    0.669493] input: Power Button as /devices/LNXSYSTM:00/LNXSYBUS:00/PNP0C0C:00/input/input0
[    0.669662] ACPI: button: Power Button [PWRB]
[    0.669869] input: Lid Switch as /devices/LNXSYSTM:00/LNXSYBUS:00/PNP0C0D:00/input/input1
[    0.669986] ACPI: button: Lid Switch [LID]
[    0.670176] input: Power Button as /devices/LNXSYSTM:00/LNXPWRBN:00/input/input2
[    0.670445] ACPI: button: Power Button [PWRF]
[    0.670628] Monitor-Mwait will be used to enter C-1 state
[    0.670652] ACPI: \_PR_.C000: Found 2 idle states
[    0.671217] ACPI: \_PR_.C001: Found 2 idle states
[    0.671525] ACPI: \_PR_.C002: Found 2 idle states
[    0.671820] ACPI: \_PR_.C003: Found 2 idle states
[    0.672091] ACPI: \_PR_.C004: Found 2 idle states
[    0.672366] ACPI: \_PR_.C005: Found 2 idle states
[    0.672646] ACPI: \_PR_.C006: Found 2 idle states
[    0.672989] ACPI: \_PR_.C007: Found 2 idle states
[    0.673849] Serial: 8250/16550 driver, 4 ports, IRQ sharing enabled
[    0.674201] ACPI: battery: Slot [BATT] (battery absent)
[    0.675973] Linux agpgart interface v0.103
[    0.687038] brd: module loaded
[    0.691680] loop: module loaded
[    0.691753] Loading iSCSI transport class v2.0-870.
[    0.693014] nvme nvme0: pci function 0000:03:00.0
[    0.695132] tun: Universal TUN/TAP device driver, 1.6
[    0.710492] nvme nvme0: Shutdown timeout set to 10 seconds
[    0.710661] Freeing initrd memory: 15884K
[    0.715884] nvme nvme0: 8/0/0 default/read/poll queues
[    0.720509]  nvme0n1: p1 p2 p3
[    1.044408] tg3 0000:02:00.0 eth0: Tigon3 [partno(BCM95762) rev 5762100] (PCI Express) MAC address 00:00:1a:1c:c9:1e
[    1.044416] tg3 0000:02:00.0 eth0: attached PHY is 5762C (10/100/1000Base-T Ethernet) (WireSpeed[1], EEE[1])
[    1.044420] tg3 0000:02:00.0 eth0: RXcsums[1] LinkChgREG[0] MIirq[0] ASF[0] TSOcap[1]
[    1.044423] tg3 0000:02:00.0 eth0: dma_rwctrl[00000001] dma_mask[64-bit]
[    1.044655] e100: Intel(R) PRO/100 Network Driver
[    1.044659] e100: Copyright(c) 1999-2006 Intel Corporation
[    1.044692] e1000: Intel(R) PRO/1000 Network Driver
[    1.044694] e1000: Copyright (c) 1999-2006 Intel Corporation.
[    1.044726] e1000e: Intel(R) PRO/1000 Network Driver
[    1.044728] e1000e: Copyright(c) 1999 - 2015 Intel Corporation.
[    1.044782] igb: Intel(R) Gigabit Ethernet Network Driver
[    1.044786] igb: Copyright (c) 2007-2014 Intel Corporation.
[    1.044833] igbvf: Intel(R) Gigabit Virtual Function Network Driver
[    1.044836] igbvf: Copyright (c) 2009 - 2012 Intel Corporation.
[    1.044880] ixgbe: Intel(R) 10 Gigabit PCI Express Network Driver
[    1.044883] ixgbe: Copyright (c) 1999-2016 Intel Corporation.
[    1.045014] i40e: Intel(R) Ethernet Connection XL710 Network Driver
[    1.045018] i40e: Copyright (c) 2013 - 2019 Intel Corporation.
[    1.045131] ixgb: Intel(R) PRO/10GbE Network Driver
[    1.045134] ixgb: Copyright (c) 1999-2008 Intel Corporation.
[    1.045180] sky2: driver version 1.30
[    1.045458] Fusion MPT base driver 3.04.20
[    1.045462] Copyright (c) 1999-2008 LSI Corporation
[    1.045474] Fusion MPT SPI Host driver 3.04.20
[    1.045778] usbcore: registered new interface driver usb-storage
[    1.045885] i8042: PNP: PS/2 Controller [PNP0303:KBC0] at 0x60,0x64 irq 1
[    1.045890] i8042: PNP: PS/2 appears to have AUX port disabled, if this is incorrect please boot with i8042.nopnp
[    1.046188] xhci_hcd 0000:04:00.4: xHCI Host Controller
[    1.046189] xhci_hcd 0000:04:00.3: xHCI Host Controller
[    1.046284] xhci_hcd 0000:04:00.4: new USB bus registered, assigned bus number 1
[    1.046284] xhci_hcd 0000:04:00.3: new USB bus registered, assigned bus number 2
[    1.046286] i8042: Warning: Keylock active
[    1.046608] serio: i8042 KBD port at 0x60,0x64 irq 1
[    1.046625] xhci_hcd 0000:04:00.4: hcc params 0x0260ffe5 hci version 0x110 quirks 0x0000000840000410
[    1.046649] xhci_hcd 0000:04:00.3: hcc params 0x0270ffe5 hci version 0x110 quirks 0x0000000840000410
[    1.047219] mousedev: PS/2 mouse device common for all mice
[    1.048109] xhci_hcd 0000:04:00.4: xHCI Host Controller
[    1.048117] xhci_hcd 0000:04:00.3: xHCI Host Controller
[    1.048130] xhci_hcd 0000:04:00.3: new USB bus registered, assigned bus number 4
[    1.048134] xhci_hcd 0000:04:00.4: new USB bus registered, assigned bus number 3
[    1.048137] xhci_hcd 0000:04:00.3: Host supports USB 3.1 Enhanced SuperSpeed
[    1.048142] xhci_hcd 0000:04:00.4: Host supports USB 3.1 Enhanced SuperSpeed
[    1.048232] rtc_cmos 00:01: RTC can wake from S4
[    1.049042] input: AT Translated Set 2 keyboard as /devices/platform/i8042/serio0/input/input3
[    1.049698] hub 2-0:1.0: USB hub found
[    1.049806] hub 2-0:1.0: 4 ports detected
[    1.050227] rtc_cmos 00:01: registered as rtc0
[    1.050350] rtc_cmos 00:01: alarms up to one month, 114 bytes nvram, hpet irqs
[    1.050561] IR JVC protocol handler initialized
[    1.050565] IR MCE Keyboard/mouse protocol handler initialized
[    1.050567] IR NEC protocol handler initialized
[    1.050569] IR RC5(x/sz) protocol handler initialized
[    1.050571] IR RC6 protocol handler initialized
[    1.050573] IR SANYO protocol handler initialized
[    1.050575] IR Sharp protocol handler initialized
[    1.050577] IR Sony protocol handler initialized
[    1.050579] IR XMP protocol handler initialized
[    1.050581] fail to initialize ptp_kvm
[    1.050750] device-mapper: ioctl: 4.47.0-ioctl (2022-07-28) initialised: dm-devel@redhat.com
[    1.050814] hid: raw HID events driver (C) Jiri Kosina
[    1.051317] usbcore: registered new interface driver usbhid
[    1.051321] usbhid: USB HID core driver
[    1.051462] Initializing XFRM netlink socket
[    1.051516] NET: Registered PF_INET6 protocol family
[    1.052266] hub 1-0:1.0: USB hub found
[    1.052292] Segment Routing with IPv6
[    1.052301] hub 1-0:1.0: 2 ports detected
[    1.052308] In-situ OAM (IOAM) with IPv6
[    1.052341] sit: IPv6, IPv4 and MPLS over IPv4 tunneling driver
[    1.052906] NET: Registered PF_PACKET protocol family
[    1.052954] bridge: filtering via arp/ip/ip6tables is no longer available by default. Update your scripts to load br_netfilter if you need this.
[    1.052989] 8021q: 802.1Q VLAN Support v1.8
[    1.053028] Key type dns_resolver registered
[    1.053035] usb usb4: We don't know the algorithms for LPM for this host, disabling LPM.
[    1.053859] hub 4-0:1.0: USB hub found
[    1.053895] hub 4-0:1.0: 4 ports detected
[    1.055411] microcode: CPU0: patch_level=0x08108102
[    1.055418] microcode: CPU1: patch_level=0x08108102
[    1.055424] microcode: CPU2: patch_level=0x08108102
[    1.055425] microcode: CPU3: patch_level=0x08108102
[    1.055432] microcode: CPU4: patch_level=0x08108102
[    1.055437] microcode: CPU6: patch_level=0x08108102
[    1.055439] microcode: CPU7: patch_level=0x08108102
[    1.055474] microcode: CPU5: patch_level=0x08108102
[    1.055520] microcode: Microcode Update Driver: v2.2.
[    1.055527] IPI shorthand broadcast: enabled
[    1.055627] usb usb3: We don't know the algorithms for LPM for this host, disabling LPM.
[    1.056439] hub 3-0:1.0: USB hub found
[    1.056567] hub 3-0:1.0: 1 port detected
[    1.066156] sched_clock: Marking stable (1064106533, 647810)->(1069399296, -4644953)
[    1.067397] registered taskstats version 1
[    1.067608] Loading compiled-in X.509 certificates
[    1.081245] Loaded X.509 cert 'Build time autogenerated kernel key: 1165ea454bb660ea6efeaa24bf6e209f2a61846a'
[    1.081404] zswap: loaded using pool lzo/zbud
[    1.088644] PM:   Magic number: 15:248:195
[    1.088917] printk: console [netcon0] enabled
[    1.088920] netconsole: network logging started
[    1.089252] acpi_cpufreq: overriding BIOS provided _PSD data
[    1.093531] Freeing unused kernel image (initmem) memory: 1836K
[    1.114170] Write protecting the kernel read-only data: 24576k
[    1.114521] Freeing unused kernel image (rodata/data gap) memory: 144K
[    1.114530] rodata_test: all tests were successful
[    1.114541] Run /init as init process
[    1.114543]   with arguments:
[    1.114546]     /init
[    1.114548]     splash
[    1.114550]   with environment:
[    1.114552]     HOME=/
[    1.114554]     TERM=linux
[    1.114555]     BOOT_IMAGE=/boot/vmlinuz-6.2.8-kfd-yangp
[    1.286103] usb 2-2: new low-speed USB device number 2 using xhci_hcd
[    1.286107] usb 1-1: new low-speed USB device number 2 using xhci_hcd
[    1.480153] input: Logitech USB-PS/2 Optical Mouse as /devices/pci0000:00/0000:00:08.1/0000:04:00.3/usb2/2-2/2-2:1.0/0003:046D:C00E.0001/input/input4
[    1.480856] hid-generic 0003:046D:C00E.0001: input,hidraw0: USB HID v1.10 Mouse [Logitech USB-PS/2 Optical Mouse] on usb-0000:04:00.3-2/input0
[    1.540022] tg3 0000:02:00.0 enp2s0f0: renamed from eth0
[    1.542611] input: Logitech USB Keyboard as /devices/pci0000:00/0000:00:08.1/0000:04:00.4/usb1/1-1/1-1:1.0/0003:046D:C31C.0002/input/input5
[    1.547462] piix4_smbus 0000:00:14.0: SMBus Host Controller at 0xb00, revision 0
[    1.547470] piix4_smbus 0000:00:14.0: Using register 0x02 for SMBus port selection
[    1.547766] piix4_smbus 0000:00:14.0: Auxiliary SMBus Host Controller at 0xb20
[    1.603410] hid-generic 0003:046D:C31C.0002: input,hidraw1: USB HID v1.10 Keyboard [Logitech USB Keyboard] on usb-0000:04:00.4-1/input0
[    1.613943] input: Logitech USB Keyboard Consumer Control as /devices/pci0000:00/0000:00:08.1/0000:04:00.4/usb1/1-1/1-1:1.1/0003:046D:C31C.0003/input/input6
[    1.654180] tsc: Refined TSC clocksource calibration: 2097.897 MHz
[    1.654198] clocksource: tsc: mask: 0xffffffffffffffff max_cycles: 0x1e3d6dee22a, max_idle_ns: 440795256804 ns
[    1.654294] clocksource: Switched to clocksource tsc
[    1.670424] input: Logitech USB Keyboard System Control as /devices/pci0000:00/0000:00:08.1/0000:04:00.4/usb1/1-1/1-1:1.1/0003:046D:C31C.0003/input/input7
[    1.670627] hid-generic 0003:046D:C31C.0003: input,hidraw2: USB HID v1.10 Device [Logitech USB Keyboard] on usb-0000:04:00.4-1/input1
[    1.821605] EXT4-fs (nvme0n1p3): mounted filesystem 2abd16aa-0792-4adc-9147-dda46bfcc9fa with ordered data mode. Quota mode: none.
[    2.004255] systemd[1]: systemd 249.11-0ubuntu3.9 running in system mode (+PAM +AUDIT +SELINUX +APPARMOR +IMA +SMACK +SECCOMP +GCRYPT +GNUTLS +OPENSSL +ACL +BLKID +CURL +ELFUTILS +FIDO2 +IDN2 -IDN +IPTC +KMOD +LIBCRYPTSETUP +LIBFDISK +PCRE2 -PWQUALITY -P11KIT -QRENCODE +BZIP2 +LZ4 +XZ +ZLIB +ZSTD -XKBCOMMON +UTMP +SYSVINIT default-hierarchy=unified)
[    2.005001] systemd[1]: Detected architecture x86-64.
[    2.005528] systemd[1]: Hostname set to <atitest-Mandolin-PCO>.
[    2.340604] systemd[1]: Queued start job for default target Graphical Interface.
[    2.365378] systemd[1]: Created slice Slice /system/modprobe.
[    2.366875] systemd[1]: Created slice Slice /system/systemd-fsck.
[    2.367927] systemd[1]: Created slice User and Session Slice.
[    2.368384] systemd[1]: Started Forward Password Requests to Wall Directory Watch.
[    2.369380] systemd[1]: Set up automount Arbitrary Executable File Formats File System Automount Point.
[    2.369522] systemd[1]: Reached target User and Group Name Lookups.
[    2.369605] systemd[1]: Reached target Remote File Systems.
[    2.369655] systemd[1]: Reached target Slice Units.
[    2.369710] systemd[1]: Reached target Mounting snaps.
[    2.369777] systemd[1]: Reached target Local Verity Protected Volumes.
[    2.370962] systemd[1]: Listening on Syslog Socket.
[    2.371486] systemd[1]: Listening on fsck to fsckd communication Socket.
[    2.371820] systemd[1]: Listening on initctl Compatibility Named Pipe.
[    2.372869] systemd[1]: Listening on Journal Audit Socket.
[    2.373387] systemd[1]: Listening on Journal Socket (/dev/log).
[    2.374047] systemd[1]: Listening on Journal Socket.
[    2.374815] systemd[1]: Listening on udev Control Socket.
[    2.375307] systemd[1]: Listening on udev Kernel Socket.
[    2.378360] systemd[1]: Mounting Huge Pages File System...
[    2.381552] systemd[1]: Mounting POSIX Message Queue File System...
[    2.385045] systemd[1]: Mounting Kernel Debug File System...
[    2.388412] systemd[1]: Mounting Kernel Trace File System...
[    2.394380] systemd[1]: Starting Journal Service...
[    2.397851] systemd[1]: Starting Set the console keyboard layout...
[    2.401606] systemd[1]: Starting Create List of Static Device Nodes...
[    2.405621] systemd[1]: Starting Load Kernel Module chromeos_pstore...
[    2.410830] systemd[1]: Starting Load Kernel Module configfs...
[    2.417100] systemd[1]: Starting Load Kernel Module drm...
[    2.421153] systemd[1]: Starting Load Kernel Module efi_pstore...
[    2.426564] systemd[1]: Starting Load Kernel Module fuse...
[    2.432787] systemd[1]: Starting Load Kernel Module pstore_blk...
[    2.436806] systemd[1]: Starting Load Kernel Module pstore_zone...
[    2.442244] systemd[1]: Starting Load Kernel Module ramoops...
[    2.442915] systemd[1]: Condition check resulted in File System Check on Root Device being skipped.
[    2.449213] systemd[1]: Starting Load Kernel Modules...
[    2.452854] systemd[1]: Starting Remount Root and Kernel File Systems...
[    2.457575] systemd[1]: Starting Coldplug All udev Devices...
[    2.459370] ACPI: bus type drm_connector registered
[    2.459624] fuse: init (API version 7.38)
[    2.468721] systemd[1]: Mounted Huge Pages File System.
[    2.469342] systemd[1]: Mounted POSIX Message Queue File System.
[    2.469856] systemd[1]: Mounted Kernel Debug File System.
[    2.470418] systemd[1]: Mounted Kernel Trace File System.
[    2.472648] systemd[1]: Finished Create List of Static Device Nodes.
[    2.474205] systemd[1]: modprobe@chromeos_pstore.service: Deactivated successfully.
[    2.475481] systemd[1]: Finished Load Kernel Module chromeos_pstore.
[    2.476704] systemd[1]: modprobe@configfs.service: Deactivated successfully.
[    2.477896] systemd[1]: Finished Load Kernel Module configfs.
[    2.479099] systemd[1]: modprobe@drm.service: Deactivated successfully.
[    2.480272] systemd[1]: Finished Load Kernel Module drm.
[    2.481465] systemd[1]: modprobe@efi_pstore.service: Deactivated successfully.
[    2.482679] systemd[1]: Finished Load Kernel Module efi_pstore.
[    2.483830] systemd[1]: modprobe@fuse.service: Deactivated successfully.
[    2.485024] systemd[1]: Finished Load Kernel Module fuse.
[    2.486264] systemd[1]: modprobe@pstore_blk.service: Deactivated successfully.
[    2.487455] systemd[1]: Finished Load Kernel Module pstore_blk.
[    2.488597] systemd[1]: modprobe@pstore_zone.service: Deactivated successfully.
[    2.489770] systemd[1]: Finished Load Kernel Module pstore_zone.
[    2.491097] systemd[1]: modprobe@ramoops.service: Deactivated successfully.
[    2.492219] systemd[1]: Finished Load Kernel Module ramoops.
[    2.496233] systemd[1]: Mounting FUSE Control File System...
[    2.501240] EXT4-fs (nvme0n1p3): re-mounted 2abd16aa-0792-4adc-9147-dda46bfcc9fa. Quota mode: none.
[    2.501301] systemd[1]: Mounting Kernel Configuration File System...
[    2.510881] systemd[1]: Finished Load Kernel Modules.
[    2.513560] systemd[1]: Finished Remount Root and Kernel File Systems.
[    2.514042] systemd[1]: Mounted FUSE Control File System.
[    2.514537] systemd[1]: Mounted Kernel Configuration File System.
[    2.517580] systemd[1]: Activating swap /swapfile...
[    2.519059] systemd[1]: Condition check resulted in Platform Persistent Storage Archival being skipped.
[    2.523078] systemd[1]: Starting Load/Save Random Seed...
[    2.528785] systemd[1]: Starting Apply Kernel Variables...
[    2.534039] Adding 2097148k swap on /swapfile.  Priority:-2 extents:16 across:832430080k SSFS
[    2.534274] systemd[1]: Starting Create System Users...
[    2.534770] systemd[1]: Activated swap /swapfile.
[    2.536061] systemd[1]: Reached target Swaps.
[    2.549211] systemd[1]: Finished Set the console keyboard layout.
[    2.563425] systemd[1]: Finished Load/Save Random Seed.
[    2.564230] systemd[1]: Condition check resulted in First Boot Complete being skipped.
[    2.570519] systemd[1]: Finished Apply Kernel Variables.
[    2.580457] systemd[1]: Finished Create System Users.
[    2.583773] systemd[1]: Starting Create Static Device Nodes in /dev...
[    2.613640] systemd[1]: Finished Create Static Device Nodes in /dev.
[    2.614092] clocksource: timekeeping watchdog on CPU4: Marking clocksource 'tsc' as unstable because the skew is too large:
[    2.614116] clocksource:                       'hpet' wd_nsec: 476410130 wd_now: 228731d wd_last: 1c05d4f mask: ffffffff
[    2.614120] clocksource:                       'tsc' cs_nsec: 475993429 cs_now: 7ff6d2a88 cs_last: 7c3e7f71c mask: ffffffffffffffff
[    2.614125] clocksource:                       'tsc' is current clocksource.
[    2.614294] tsc: Marking TSC unstable due to clocksource watchdog
[    2.614346] TSC found unstable after boot, most likely due to broken BIOS. Use 'tsc=unstable'.
[    2.614352] sched_clock: Marking unstable (2613697772, 647825)<-(2618990688, -4644953)
[    2.614674] clocksource: Checking clocksource tsc synchronization from CPU 3 to CPUs 0-2,4-6.
[    2.614686] systemd[1]: Reached target Preparation for Local File Systems.
[    2.614763] clocksource: Switched to clocksource hpet
[    2.619933] systemd[1]: Mounting Mount unit for bare, revision 5...
[    2.625075] systemd[1]: Mounting Mount unit for core20, revision 1822...
[    2.629387] loop0: detected capacity change from 0 to 8
[    2.630706] systemd[1]: Mounting Mount unit for firefox, revision 2356...
[    2.635059] loop1: detected capacity change from 0 to 129600
[    2.636380] systemd[1]: Mounting Mount unit for gnome-3-38-2004, revision 119...
[    2.641045] loop0: detected capacity change from 0 to 492776
[    2.642880] systemd[1]: Mounting Mount unit for gtk-common-themes, revision 1535...
[    2.646454] loop1: detected capacity change from 0 to 709280
[    2.648120] systemd[1]: Mounting Mount unit for snap-store, revision 638...
[    2.652683] loop0: detected capacity change from 0 to 187776
[    2.653758] systemd[1]: Mounting Mount unit for snapd, revision 18357...
[    2.657790] loop1: detected capacity change from 0 to 94064
[    2.659390] systemd[1]: Mounting Mount unit for snapd-desktop-integration, revision 49...
[    2.663919] loop0: detected capacity change from 0 to 102072
[    2.668035] systemd[1]: Mounting Mount unit for firefox, revision 2356 via mount-control...
[    2.668832] loop1: detected capacity change from 0 to 608
[    2.673806] systemd[1]: Starting Rule-based Manager for Device Events and Files...
[    2.677966] systemd[1]: Started Journal Service.
[    2.779955] systemd-journald[1007]: Received client request to flush runtime journal.
[    3.184235] ACPI: video: Video Device [VGA1] (multi-head: yes  rom: no  post: no)
[    3.185471] input: Video Bus as /devices/LNXSYSTM:00/LNXSYBUS:00/PNP0A08:00/device:0d/LNXVIDEO:01/input/input8
[    3.394129] AMD-Vi: AMD IOMMUv2 loaded and initialized
[    4.189448] loop0: detected capacity change from 0 to 8
[    4.849275] [drm] amdgpu kernel modesetting enabled.
[    4.871384] amdgpu: Topology: Add APU node [0x0:0x0]
[    4.872440] [drm] initializing kernel modesetting (RAVEN 0x1002:0x15D8 0x1002:0x0123 0xC2).
[    4.875543] [drm] register mmio base: 0xC0700000
[    4.875554] [drm] register mmio size: 524288
[    4.876103] [drm] add ip block number 0 <soc15_common>
[    4.876109] [drm] add ip block number 1 <gmc_v9_0>
[    4.876112] [drm] add ip block number 2 <vega10_ih>
[    4.876115] [drm] add ip block number 3 <psp>
[    4.876118] [drm] add ip block number 4 <powerplay>
[    4.876121] [drm] add ip block number 5 <dm>
[    4.876124] [drm] add ip block number 6 <gfx_v9_0>
[    4.876126] [drm] add ip block number 7 <sdma_v4_0>
[    4.876129] [drm] add ip block number 8 <vcn_v1_0>
[    4.917024] [drm] BIOS signature incorrect 0 0
[    4.917046] resource: resource sanity check: requesting [mem 0x00000000000c0000-0x00000000000dffff], which spans more than PCI Bus 0000:00 [mem 0x000c0000-0x000cbfff window]
[    4.917068] caller pci_map_rom+0x6b/0x1c0 mapping multiple BARs
[    4.919353] amdgpu 0000:04:00.0: amdgpu: Fetched VBIOS from ROM BAR
[    4.919366] amdgpu: ATOM BIOS: 113-PICASSO-114
[    4.930060] [drm] VCN decode is enabled in VM mode
[    4.930109] [drm] VCN encode is enabled in VM mode
[    4.930113] [drm] JPEG decode is enabled in VM mode
[    4.932018] [drm] MCBP is enabled
[    4.932105] amdgpu 0000:04:00.0: vgaarb: deactivate vga console
[    4.932118] amdgpu 0000:04:00.0: amdgpu: Trusted Memory Zone (TMZ) feature enabled
[    4.932180] Stack Depot allocating hash table of 524288 entries with kvcalloc
[    4.933561] [drm] vm size is 262144 GB, 4 levels, block size is 9-bit, fragment size is 9-bit
[    4.933584] amdgpu 0000:04:00.0: amdgpu: VRAM: 2048M 0x000000F400000000 - 0x000000F47FFFFFFF (2048M used)
[    4.933591] amdgpu 0000:04:00.0: amdgpu: GART: 1024M 0x0000000000000000 - 0x000000003FFFFFFF
[    4.933595] amdgpu 0000:04:00.0: amdgpu: AGP: 267419648M 0x000000F800000000 - 0x0000FFFFFFFFFFFF
[    4.933614] [drm] Detected VRAM RAM=2048M, BAR=2048M
[    4.933618] [drm] RAM width 64bits DDR4
[    4.935429] [drm] amdgpu: 2048M of VRAM memory ready
[    4.935439] [drm] amdgpu: 2913M of GTT memory ready.
[    4.935531] [drm] GART: num cpu pages 262144, num gpu pages 262144
[    4.936068] [drm] PCIE GART of 1024M enabled.
[    4.936073] [drm] PTB located at 0x000000F400A00000
[    4.936612] amdgpu: hwmgr_sw_init smu backed is smu10_smu
[    4.939290] [drm] Found VCN firmware Version ENC: 1.13 DEC: 2 VEP: 0 Revision: 4
[    4.939336] amdgpu 0000:04:00.0: amdgpu: Will use PSP to load VCN firmware
[    4.961672] [drm] reserve 0x400000 from 0xf47fc00000 for PSP TMR
[    5.061779] amdgpu 0000:04:00.0: amdgpu: RAS: optional ras ta ucode is not available
[    5.076590] amdgpu 0000:04:00.0: amdgpu: RAP: optional rap ta ucode is not available
[    5.084284] [drm] DM_PPLIB: values for F clock
[    5.084296] [drm] DM_PPLIB:	 400000 in kHz, 2874 in mV
[    5.084303] [drm] DM_PPLIB:	 933000 in kHz, 3224 in mV
[    5.084308] [drm] DM_PPLIB:	 1067000 in kHz, 3924 in mV
[    5.084311] [drm] DM_PPLIB:	 1200000 in kHz, 4074 in mV
[    5.084317] [drm] DM_PPLIB: values for DCF clock
[    5.084323] [drm] DM_PPLIB:	 300000 in kHz, 2874 in mV
[    5.084326] [drm] DM_PPLIB:	 600000 in kHz, 3224 in mV
[    5.084329] [drm] DM_PPLIB:	 626000 in kHz, 3924 in mV
[    5.084331] [drm] DM_PPLIB:	 654000 in kHz, 4074 in mV
[    5.085518] [drm] Display Core v3.2.243 initialized on DCN 1.0
[    5.148322] [drm] kiq ring mec 2 pipe 1 q 0
[    5.160887] [drm] VCN decode and encode initialized successfully(under SPG Mode).
[    5.206647] memmap_init_zone_device initialised 524288 pages in 8ms
[    5.206668] amdgpu: HMM registered 2048MB device memory
[    5.208831] kfd kfd: amdgpu: Allocated 3969056 bytes on gart
[    5.208877] kfd kfd: amdgpu: Total number of KFD nodes to be created: 1
[    5.209249] amdgpu: Topology: Add APU node [0x15d8:0x1002]
[    5.209474] kfd kfd: amdgpu: added device 1002:15d8
[    5.209518] amdgpu 0000:04:00.0: amdgpu: SE 1, SH per SE 1, CU per SH 11, active_cu_number 8
[    5.209953] amdgpu 0000:04:00.0: amdgpu: ring gfx uses VM inv eng 0 on hub 0
[    5.209959] amdgpu 0000:04:00.0: amdgpu: ring gfx_low uses VM inv eng 1 on hub 0
[    5.209966] amdgpu 0000:04:00.0: amdgpu: ring gfx_high uses VM inv eng 4 on hub 0
[    5.209970] amdgpu 0000:04:00.0: amdgpu: ring comp_1.0.0 uses VM inv eng 5 on hub 0
[    5.209974] amdgpu 0000:04:00.0: amdgpu: ring comp_1.1.0 uses VM inv eng 6 on hub 0
[    5.209978] amdgpu 0000:04:00.0: amdgpu: ring comp_1.2.0 uses VM inv eng 7 on hub 0
[    5.209981] amdgpu 0000:04:00.0: amdgpu: ring comp_1.3.0 uses VM inv eng 8 on hub 0
[    5.209984] amdgpu 0000:04:00.0: amdgpu: ring comp_1.0.1 uses VM inv eng 9 on hub 0
[    5.209987] amdgpu 0000:04:00.0: amdgpu: ring comp_1.1.1 uses VM inv eng 10 on hub 0
[    5.209991] amdgpu 0000:04:00.0: amdgpu: ring comp_1.2.1 uses VM inv eng 11 on hub 0
[    5.209994] amdgpu 0000:04:00.0: amdgpu: ring comp_1.3.1 uses VM inv eng 12 on hub 0
[    5.209997] amdgpu 0000:04:00.0: amdgpu: ring kiq_0.2.1.0 uses VM inv eng 13 on hub 0
[    5.210000] amdgpu 0000:04:00.0: amdgpu: ring sdma0 uses VM inv eng 0 on hub 8
[    5.210004] amdgpu 0000:04:00.0: amdgpu: ring vcn_dec uses VM inv eng 1 on hub 8
[    5.210007] amdgpu 0000:04:00.0: amdgpu: ring vcn_enc0 uses VM inv eng 4 on hub 8
[    5.210010] amdgpu 0000:04:00.0: amdgpu: ring vcn_enc1 uses VM inv eng 5 on hub 8
[    5.210013] amdgpu 0000:04:00.0: amdgpu: ring jpeg_dec uses VM inv eng 6 on hub 8
[    5.222749] [drm] Initialized amdgpu 3.54.0 20150101 for 0000:04:00.0 on minor 0
[    5.235808] fbcon: amdgpudrmfb (fb0) is primary device
[    5.336625] Console: switching to colour frame buffer device 200x75
[    5.354226] amdgpu 0000:04:00.0: [drm] fb0: amdgpudrmfb frame buffer device
[    7.224013] tg3 0000:02:00.0 enp2s0f0: Link is up at 100 Mbps, full duplex
[    7.224028] tg3 0000:02:00.0 enp2s0f0: Flow control is on for TX and on for RX
[    7.224032] tg3 0000:02:00.0 enp2s0f0: EEE is disabled
[    7.224429] IPv6: ADDRCONF(NETDEV_CHANGE): enp2s0f0: link becomes ready
[    9.811957] rfkill: input handler disabled

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Michel Dänzer]

[-- Attachment #1: Type: text/plain, Size: 3138 bytes --]

On 7/20/23 22:48, Philip Yang wrote:
> On 2023-07-20 06:46, Michel Dänzer wrote:
>> On 7/17/23 15:09, Michel Dänzer wrote:
>>> On 5/10/23 23:23, Alex Deucher wrote:
>>>> From: Philip Yang <Philip.Yang@amd.com>
>>>>
>>>> Rename smv_migrate_init to a better name kgd2kfd_init_zone_device
>>>> because it setup zone devive pgmap for page migration and keep it in
>>>> kfd_migrate.c to access static functions svm_migrate_pgmap_ops. Call it
>>>> only once in amdgpu_device_ip_init after adev ip blocks are initialized,
>>>> but before amdgpu_amdkfd_device_init initialize kfd nodes which enable
>>>> SVM support based on pgmap.
>>>>
>>>> svm_range_set_max_pages is called by kgd2kfd_device_init everytime after
>>>> switching compute partition mode.
>>>>
>>>> Signed-off-by: Philip Yang <Philip.Yang@amd.com>
>>>> Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
>>>> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
>>> I bisected a regression to this commit, which broke HW acceleration on this ThinkPad E595 with Picasso APU.
>> Actually, it doesn't seem to break HW acceleration completely. GDM eventually comes up with HW acceleration, it takes a long time (~30s or so) to start up though.
>>
>> Later, the same messages as described in https://gitlab.freedesktop.org/drm/amd/-/issues/2659 appear.
>>
>> Reverting this commit fixes all of the above symptoms.
>>
>>
>> I reproduced all of the above symptoms with amd-staging-drm-next commit 75515acf4b60 ("i2c: nvidia-gpu: Add ACPI property to align with device-tree") as well.
>>
>>
>> For full disclosure, I use these kernel command line arguments:
>>
>>  fbcon=font:10x18 drm_kms_helper.drm_fbdev_overalloc=112 amdgpu.noretry=1 amdgpu.mcbp=1
> 
> Thanks for the issue report and full disclosure, but I am not able to reproduce this issue, with both drm-next branch and amd-staging-drm-next branch tip on gitlab. The test system has same device id, running Ubuntu 22.04, latest linux-firmware-20230625.tar.gz, and same BIOS version.

FWIW, your system has PCI revision ID 0xC2, while mine has 0xC1.

Also, I'm currently using linux-firmware 20230515. AFAICT there are no relevant changes in 20230625, but I'm attaching the contents of /sys/kernel/debug/dri/0/amdgpu_firmware_info just in case.


> I attached full dmesg log, could you help check if there is other difference, maybe kernel config, gcc version... it is hard to guess what could cause the basic driver gfx ring IB test timeout.

I suspect the IOMMU page faults logged in my dmesg might be relevant:

 amdgpu: Topology: Add APU node [0x15d8:0x1002]
 amdgpu 0000:05:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0000 address=0x122201800 flags=0x0070]
 amdgpu 0000:05:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0000 address=0x1125fe380 flags=0x0070]
 kfd kfd: amdgpu: added device 1002:15d8

There are no such page faults with the commit reverted.

Other than that and the IB test failure messages, our dmesg outputs are mostly identical indeed.


-- 
Earthling Michel Dänzer            |                  https://redhat.com
Libre software enthusiast          |         Mesa and Xwayland developer

[-- Attachment #2: amdgpu_firmware_info.txt --]
[-- Type: text/plain, Size: 1810 bytes --]

VCE feature version: 0, firmware version: 0x00000000
UVD feature version: 0, firmware version: 0x00000000
MC feature version: 0, firmware version: 0x00000000
ME feature version: 53, firmware version: 0x000000a6
PFP feature version: 53, firmware version: 0x000000c2
CE feature version: 53, firmware version: 0x00000050
RLC feature version: 1, firmware version: 0x0000006f
RLC SRLC feature version: 1, firmware version: 0x00000001
RLC SRLG feature version: 1, firmware version: 0x00000001
RLC SRLS feature version: 1, firmware version: 0x00000001
RLCP feature version: 0, firmware version: 0x00000000
RLCV feature version: 0, firmware version: 0x00000000
MEC feature version: 53, firmware version: 0x000001d3
MEC2 feature version: 53, firmware version: 0x000001d3
IMU feature version: 0, firmware version: 0x00000000
SOS feature version: 0, firmware version: 0x00000000
ASD feature version: 0, firmware version: 0x21000090
TA XGMI feature version: 0x00000000, firmware version: 0x00000000
TA RAS feature version: 0x00000000, firmware version: 0x00000000
TA HDCP feature version: 0x00000000, firmware version: 0x1700002e
TA DTM feature version: 0x00000000, firmware version: 0x12000012
TA RAP feature version: 0x00000000, firmware version: 0x00000000
TA SECUREDISPLAY feature version: 0x00000000, firmware version: 0x27000005
SMC feature version: 0, program: 0, firmware version: 0x00041e2a (4.30.42)
SDMA0 feature version: 41, firmware version: 0x000000a9
VCN feature version: 0, firmware version: 0x0210d004
DMCU feature version: 0, firmware version: 0x00000001
DMCUB feature version: 0, firmware version: 0x00000000
TOC feature version: 0, firmware version: 0x00000000
MES_KIQ feature version: 0, firmware version: 0x00000000
MES feature version: 0, firmware version: 0x00000000
VBIOS version: 113-PICASSO-114

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Michel Dänzer]

On 7/21/23 10:55, Michel Dänzer wrote:
> On 7/20/23 22:48, Philip Yang wrote:
>> On 2023-07-20 06:46, Michel Dänzer wrote:
>>> On 7/17/23 15:09, Michel Dänzer wrote:
>>>> On 5/10/23 23:23, Alex Deucher wrote:
>>>>> From: Philip Yang <Philip.Yang@amd.com>
>>>>>
>>>>> Rename smv_migrate_init to a better name kgd2kfd_init_zone_device
>>>>> because it setup zone devive pgmap for page migration and keep it in
>>>>> kfd_migrate.c to access static functions svm_migrate_pgmap_ops. Call it
>>>>> only once in amdgpu_device_ip_init after adev ip blocks are initialized,
>>>>> but before amdgpu_amdkfd_device_init initialize kfd nodes which enable
>>>>> SVM support based on pgmap.
>>>>>
>>>>> svm_range_set_max_pages is called by kgd2kfd_device_init everytime after
>>>>> switching compute partition mode.
>>>>>
>>>>> Signed-off-by: Philip Yang <Philip.Yang@amd.com>
>>>>> Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
>>>>> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
>>>> I bisected a regression to this commit, which broke HW acceleration on this ThinkPad E595 with Picasso APU.
>>> Actually, it doesn't seem to break HW acceleration completely. GDM eventually comes up with HW acceleration, it takes a long time (~30s or so) to start up though.
>>>
>>> Later, the same messages as described in https://gitlab.freedesktop.org/drm/amd/-/issues/2659 appear.
>>>
>>> Reverting this commit fixes all of the above symptoms.
>>>
>>>
>>> I reproduced all of the above symptoms with amd-staging-drm-next commit 75515acf4b60 ("i2c: nvidia-gpu: Add ACPI property to align with device-tree") as well.
>>>
>>>
>>> For full disclosure, I use these kernel command line arguments:
>>>
>>>  fbcon=font:10x18 drm_kms_helper.drm_fbdev_overalloc=112 amdgpu.noretry=1 amdgpu.mcbp=1
>>
>> Thanks for the issue report and full disclosure, but I am not able to reproduce this issue, with both drm-next branch and amd-staging-drm-next branch tip on gitlab. The test system has same device id, running Ubuntu 22.04, latest linux-firmware-20230625.tar.gz, and same BIOS version.
> 
> FWIW, your system has PCI revision ID 0xC2, while mine has 0xC1.
> 
> Also, I'm currently using linux-firmware 20230515. AFAICT there are no relevant changes in 20230625, but I'm attaching the contents of /sys/kernel/debug/dri/0/amdgpu_firmware_info just in case.
> 
> 
>> I attached full dmesg log, could you help check if there is other difference, maybe kernel config, gcc version... it is hard to guess what could cause the basic driver gfx ring IB test timeout.
> 
> I suspect the IOMMU page faults logged in my dmesg might be relevant:
> 
>  amdgpu: Topology: Add APU node [0x15d8:0x1002]
>  amdgpu 0000:05:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0000 address=0x122201800 flags=0x0070]
>  amdgpu 0000:05:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0000 address=0x1125fe380 flags=0x0070]
>  kfd kfd: amdgpu: added device 1002:15d8

Maybe I should mention my IOMMU related kernel build configuration:

CONFIG_IRQ_MSI_IOMMU=y
CONFIG_GART_IOMMU=y
CONFIG_VFIO_IOMMU_TYPE1=m
# CONFIG_VFIO_NOIOMMU is not set
CONFIG_IOMMU_IOVA=y
CONFIG_IOMMU_API=y
CONFIG_IOMMU_SUPPORT=y
CONFIG_IOMMU_IO_PGTABLE=y
# CONFIG_IOMMU_DEFAULT_DMA_STRICT is not set
CONFIG_IOMMU_DEFAULT_DMA_LAZY=y
# CONFIG_IOMMU_DEFAULT_PASSTHROUGH is not set
CONFIG_IOMMU_DMA=y
CONFIG_IOMMU_SVA=y
CONFIG_AMD_IOMMU=y
CONFIG_AMD_IOMMU_V2=y
# CONFIG_IOMMUFD is not set
CONFIG_IOMMU_HELPER=y
# CONFIG_IOMMU_DEBUG is not set


> There are no such page faults with the commit reverted.
> 
> Other than that and the IB test failure messages, our dmesg outputs are mostly identical indeed.
> 
> 

-- 
Earthling Michel Dänzer            |                  https://redhat.com
Libre software enthusiast          |         Mesa and Xwayland developer

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Philip Yang]

[-- Attachment #1: Type: text/html, Size: 5234 bytes --]

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Philip Yang]

[-- Attachment #1: Type: text/html, Size: 5859 bytes --]

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Michel Dänzer]

On 7/24/23 22:04, Philip Yang wrote:
> Hi Michel,
> 
> Please check if this patch "drm/amdkfd: start_cpsch don't map queues" can fix the driver loading ring test failed issue on your system, I am still not able to repro the issue.

I'm planning to test it this week, thanks for the heads up.


-- 
Earthling Michel Dänzer            |                  https://redhat.com
Libre software enthusiast          |         Mesa and Xwayland developer

RE: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Zhang, Jesse(Jie)]

[-- Attachment #1: Type: text/plain, Size: 4642 bytes --]

[AMD Official Use Only - General]

Hi Philip and Michel,

The following issue can reproduce on my side.
https://gitlab.freedesktop.org/drm/amd/-/issues/2659

when load gpu driver , disable iommu  can workaround this issue.
sudo modprobe amdgpu ignore_crat=1

Thanks
Jesse

From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Philip Yang
Sent: Tuesday, July 25, 2023 4:04 AM
To: Michel Dänzer <michel@daenzer.net>; Yang, Philip <Philip.Yang@amd.com>
Cc: Deucher, Alexander <Alexander.Deucher@amd.com>; Kuehling, Felix <Felix.Kuehling@amd.com>; amd-gfx@lists.freedesktop.org
Subject: Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch


Hi Michel,

Please check if this patch "drm/amdkfd: start_cpsch don't map queues" can fix the driver loading ring test failed issue on your system, I am still not able to repro the issue.

Regards,

Philip
On 2023-07-21 09:30, Philip Yang wrote:


On 2023-07-21 04:55, Michel Dänzer wrote:

On 7/20/23 22:48, Philip Yang wrote:

On 2023-07-20 06:46, Michel Dänzer wrote:

On 7/17/23 15:09, Michel Dänzer wrote:

On 5/10/23 23:23, Alex Deucher wrote:

From: Philip Yang <Philip.Yang@amd.com><mailto:Philip.Yang@amd.com>



Rename smv_migrate_init to a better name kgd2kfd_init_zone_device

because it setup zone devive pgmap for page migration and keep it in

kfd_migrate.c to access static functions svm_migrate_pgmap_ops. Call it

only once in amdgpu_device_ip_init after adev ip blocks are initialized,

but before amdgpu_amdkfd_device_init initialize kfd nodes which enable

SVM support based on pgmap.



svm_range_set_max_pages is called by kgd2kfd_device_init everytime after

switching compute partition mode.



Signed-off-by: Philip Yang <Philip.Yang@amd.com><mailto:Philip.Yang@amd.com>

Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com><mailto:Felix.Kuehling@amd.com>

Signed-off-by: Alex Deucher <alexander.deucher@amd.com><mailto:alexander.deucher@amd.com>

I bisected a regression to this commit, which broke HW acceleration on this ThinkPad E595 with Picasso APU.

Actually, it doesn't seem to break HW acceleration completely. GDM eventually comes up with HW acceleration, it takes a long time (~30s or so) to start up though.



Later, the same messages as described in https://gitlab.freedesktop.org/drm/amd/-/issues/2659 appear.



Reverting this commit fixes all of the above symptoms.





I reproduced all of the above symptoms with amd-staging-drm-next commit 75515acf4b60 ("i2c: nvidia-gpu: Add ACPI property to align with device-tree") as well.





For full disclosure, I use these kernel command line arguments:



 fbcon=font:10x18 drm_kms_helper.drm_fbdev_overalloc=112 amdgpu.noretry=1 amdgpu.mcbp=1

Thanks for the issue report and full disclosure, but I am not able to reproduce this issue, with both drm-next branch and amd-staging-drm-next branch tip on gitlab. The test system has same device id, running Ubuntu 22.04, latest linux-firmware-20230625.tar.gz, and same BIOS version.

FWIW, your system has PCI revision ID 0xC2, while mine has 0xC1.



Also, I'm currently using linux-firmware 20230515. AFAICT there are no relevant changes in 20230625, but I'm attaching the contents of /sys/kernel/debug/dri/0/amdgpu_firmware_info just in case.





I attached full dmesg log, could you help check if there is other difference, maybe kernel config, gcc version... it is hard to guess what could cause the basic driver gfx ring IB test timeout.

I suspect the IOMMU page faults logged in my dmesg might be relevant:



 amdgpu: Topology: Add APU node [0x15d8:0x1002]

 amdgpu 0000:05:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0000 address=0x122201800 flags=0x0070]

 amdgpu 0000:05:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0000 address=0x1125fe380 flags=0x0070]

 kfd kfd: amdgpu: added device 1002:15d8



There are no such page faults with the commit reverted.



Other than that and the IB test failure messages, our dmesg outputs are mostly identical indeed.

Yes, I don't have IO_PAGE_FAULT message on my system, thanks for the finding, I will continue investigating the root cause.

You are right, the error message could cause gfx ring IB test timeout failure, this patch does change the order of driver memory allocation. IOMMU is in translation mode on Ubuntu config.

To help confirm if this is caused by IOMMU, please add this to kernel boot option to set IOMMU to passthrough mode, check if this can workaround the issue

iommu=pt

Regards,

Philip

[-- Attachment #2: Type: text/html, Size: 9664 bytes --]

RE: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Zhang, Jesse(Jie)]

[-- Attachment #1: Type: text/plain, Size: 5513 bytes --]

[AMD Official Use Only - General]

Hi Michel,

Could you try the patch again ?  That work for me.
https://patchwork.freedesktop.org/patch/549605/

Thanks
Jesse

From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Zhang, Jesse(Jie)
Sent: Thursday, July 27, 2023 2:11 PM
To: Yang, Philip <Philip.Yang@amd.com>; Michel Dänzer <michel@daenzer.net>; Yang, Philip <Philip.Yang@amd.com>
Cc: Deucher, Alexander <Alexander.Deucher@amd.com>; Kuehling, Felix <Felix.Kuehling@amd.com>; amd-gfx@lists.freedesktop.org
Subject: RE: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch


[AMD Official Use Only - General]


[AMD Official Use Only - General]

Hi Philip and Michel,

The following issue can reproduce on my side.
https://gitlab.freedesktop.org/drm/amd/-/issues/2659

when load gpu driver , disable iommu  can workaround this issue.
sudo modprobe amdgpu ignore_crat=1

Thanks
Jesse

From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org<mailto:amd-gfx-bounces@lists.freedesktop.org>> On Behalf Of Philip Yang
Sent: Tuesday, July 25, 2023 4:04 AM
To: Michel Dänzer <michel@daenzer.net<mailto:michel@daenzer.net>>; Yang, Philip <Philip.Yang@amd.com<mailto:Philip.Yang@amd.com>>
Cc: Deucher, Alexander <Alexander.Deucher@amd.com<mailto:Alexander.Deucher@amd.com>>; Kuehling, Felix <Felix.Kuehling@amd.com<mailto:Felix.Kuehling@amd.com>>; amd-gfx@lists.freedesktop.org<mailto:amd-gfx@lists.freedesktop.org>
Subject: Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch


Hi Michel,

Please check if this patch "drm/amdkfd: start_cpsch don't map queues" can fix the driver loading ring test failed issue on your system, I am still not able to repro the issue.

Regards,

Philip
On 2023-07-21 09:30, Philip Yang wrote:


On 2023-07-21 04:55, Michel Dänzer wrote:

On 7/20/23 22:48, Philip Yang wrote:

On 2023-07-20 06:46, Michel Dänzer wrote:

On 7/17/23 15:09, Michel Dänzer wrote:

On 5/10/23 23:23, Alex Deucher wrote:

From: Philip Yang <Philip.Yang@amd.com><mailto:Philip.Yang@amd.com>



Rename smv_migrate_init to a better name kgd2kfd_init_zone_device

because it setup zone devive pgmap for page migration and keep it in

kfd_migrate.c to access static functions svm_migrate_pgmap_ops. Call it

only once in amdgpu_device_ip_init after adev ip blocks are initialized,

but before amdgpu_amdkfd_device_init initialize kfd nodes which enable

SVM support based on pgmap.



svm_range_set_max_pages is called by kgd2kfd_device_init everytime after

switching compute partition mode.



Signed-off-by: Philip Yang <Philip.Yang@amd.com><mailto:Philip.Yang@amd.com>

Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com><mailto:Felix.Kuehling@amd.com>

Signed-off-by: Alex Deucher <alexander.deucher@amd.com><mailto:alexander.deucher@amd.com>

I bisected a regression to this commit, which broke HW acceleration on this ThinkPad E595 with Picasso APU.

Actually, it doesn't seem to break HW acceleration completely. GDM eventually comes up with HW acceleration, it takes a long time (~30s or so) to start up though.



Later, the same messages as described in https://gitlab.freedesktop.org/drm/amd/-/issues/2659 appear.



Reverting this commit fixes all of the above symptoms.





I reproduced all of the above symptoms with amd-staging-drm-next commit 75515acf4b60 ("i2c: nvidia-gpu: Add ACPI property to align with device-tree") as well.





For full disclosure, I use these kernel command line arguments:



 fbcon=font:10x18 drm_kms_helper.drm_fbdev_overalloc=112 amdgpu.noretry=1 amdgpu.mcbp=1

Thanks for the issue report and full disclosure, but I am not able to reproduce this issue, with both drm-next branch and amd-staging-drm-next branch tip on gitlab. The test system has same device id, running Ubuntu 22.04, latest linux-firmware-20230625.tar.gz, and same BIOS version.

FWIW, your system has PCI revision ID 0xC2, while mine has 0xC1.



Also, I'm currently using linux-firmware 20230515. AFAICT there are no relevant changes in 20230625, but I'm attaching the contents of /sys/kernel/debug/dri/0/amdgpu_firmware_info just in case.





I attached full dmesg log, could you help check if there is other difference, maybe kernel config, gcc version... it is hard to guess what could cause the basic driver gfx ring IB test timeout.

I suspect the IOMMU page faults logged in my dmesg might be relevant:



 amdgpu: Topology: Add APU node [0x15d8:0x1002]

 amdgpu 0000:05:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0000 address=0x122201800 flags=0x0070]

 amdgpu 0000:05:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0000 address=0x1125fe380 flags=0x0070]

 kfd kfd: amdgpu: added device 1002:15d8



There are no such page faults with the commit reverted.



Other than that and the IB test failure messages, our dmesg outputs are mostly identical indeed.

Yes, I don't have IO_PAGE_FAULT message on my system, thanks for the finding, I will continue investigating the root cause.

You are right, the error message could cause gfx ring IB test timeout failure, this patch does change the order of driver memory allocation. IOMMU is in translation mode on Ubuntu config.

To help confirm if this is caused by IOMMU, please add this to kernel boot option to set IOMMU to passthrough mode, check if this can workaround the issue

iommu=pt

Regards,

Philip

[-- Attachment #2: Type: text/html, Size: 11635 bytes --]

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Michel Dänzer]

On 7/28/23 03:38, Zhang, Jesse(Jie) wrote:
> 
> Could you try the patch again ?  That work for me.
> 
> https://patchwork.freedesktop.org/patch/549605/ <https://patchwork.freedesktop.org/patch/549605/>

This patch fixes the symptoms described in https://gitlab.freedesktop.org/drm/amd/-/issues/2659 for me as well.

However, it does not fix the IOMMU page faults[0] or the IB test failures on the compute rings. Should I try amdgpu.ignore_crat=1 for these symptoms as well?


[0] Not even together with Philip's patch; maybe that was just luck when I tested the latter before.


-- 
Earthling Michel Dänzer            |                  https://redhat.com
Libre software enthusiast          |         Mesa and Xwayland developer

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Michel Dänzer]

On 7/28/23 11:30, Michel Dänzer wrote:
> On 7/28/23 03:38, Zhang, Jesse(Jie) wrote:
>>
>> Could you try the patch again ?  That work for me.
>>
>> https://patchwork.freedesktop.org/patch/549605/ <https://patchwork.freedesktop.org/patch/549605/>
> 
> This patch fixes the symptoms described in https://gitlab.freedesktop.org/drm/amd/-/issues/2659 for me as well.
> 
> However, it does not fix the IOMMU page faults[0] or the IB test failures on the compute rings. Should I try amdgpu.ignore_crat=1 for these symptoms as well?

I tried ignore_crat=1, it avoids the remaining symptoms as well.


-- 
Earthling Michel Dänzer            |                  https://redhat.com
Libre software enthusiast          |         Mesa and Xwayland developer

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Alex Deucher]

On Fri, Jul 28, 2023 at 10:25 AM Michel Dänzer <michel@daenzer.net> wrote:
>
> On 7/28/23 11:30, Michel Dänzer wrote:
> > On 7/28/23 03:38, Zhang, Jesse(Jie) wrote:
> >>
> >> Could you try the patch again ?  That work for me.
> >>
> >> https://patchwork.freedesktop.org/patch/549605/ <https://patchwork.freedesktop.org/patch/549605/>
> >
> > This patch fixes the symptoms described in https://gitlab.freedesktop.org/drm/amd/-/issues/2659 for me as well.
> >
> > However, it does not fix the IOMMU page faults[0] or the IB test failures on the compute rings. Should I try amdgpu.ignore_crat=1 for these symptoms as well?
>
> I tried ignore_crat=1, it avoids the remaining symptoms as well.

The first 3 patches of this set may also fix it:
https://patchwork.freedesktop.org/series/121538/

Alex

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Michel Dänzer]

On 7/28/23 18:43, Alex Deucher wrote:
> On Fri, Jul 28, 2023 at 10:25 AM Michel Dänzer <michel@daenzer.net> wrote:
>> On 7/28/23 11:30, Michel Dänzer wrote:
>>> On 7/28/23 03:38, Zhang, Jesse(Jie) wrote:
>>>>
>>>> Could you try the patch again ?  That work for me.
>>>>
>>>> https://patchwork.freedesktop.org/patch/549605/ <https://patchwork.freedesktop.org/patch/549605/>
>>>
>>> This patch fixes the symptoms described in https://gitlab.freedesktop.org/drm/amd/-/issues/2659 for me as well.
>>>
>>> However, it does not fix the IOMMU page faults[0] or the IB test failures on the compute rings. Should I try amdgpu.ignore_crat=1 for these symptoms as well?
>>
>> I tried ignore_crat=1, it avoids the remaining symptoms as well.
> 
> The first 3 patches of this set may also fix it:
> https://patchwork.freedesktop.org/series/121538/

They do fix all symptoms I'm seeing.

Is this a feasible solution for the final 6.5 release?


-- 
Earthling Michel Dänzer            |                  https://redhat.com
Libre software enthusiast          |         Mesa and Xwayland developer

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Alex Deucher]

On Fri, Jul 28, 2023 at 1:19 PM Michel Dänzer <michel@daenzer.net> wrote:
>
> On 7/28/23 18:43, Alex Deucher wrote:
> > On Fri, Jul 28, 2023 at 10:25 AM Michel Dänzer <michel@daenzer.net> wrote:
> >> On 7/28/23 11:30, Michel Dänzer wrote:
> >>> On 7/28/23 03:38, Zhang, Jesse(Jie) wrote:
> >>>>
> >>>> Could you try the patch again ?  That work for me.
> >>>>
> >>>> https://patchwork.freedesktop.org/patch/549605/ <https://patchwork.freedesktop.org/patch/549605/>
> >>>
> >>> This patch fixes the symptoms described in https://gitlab.freedesktop.org/drm/amd/-/issues/2659 for me as well.
> >>>
> >>> However, it does not fix the IOMMU page faults[0] or the IB test failures on the compute rings. Should I try amdgpu.ignore_crat=1 for these symptoms as well?
> >>
> >> I tried ignore_crat=1, it avoids the remaining symptoms as well.
> >
> > The first 3 patches of this set may also fix it:
> > https://patchwork.freedesktop.org/series/121538/
>
> They do fix all symptoms I'm seeing.
>
> Is this a feasible solution for the final 6.5 release?

I think the first3 are.  The 4th will have to wait for 6.6.

Alex

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Michel Dänzer]

On 7/28/23 19:20, Alex Deucher wrote:
> On Fri, Jul 28, 2023 at 1:19 PM Michel Dänzer <michel@daenzer.net> wrote:
>> On 7/28/23 18:43, Alex Deucher wrote:
>>> On Fri, Jul 28, 2023 at 10:25 AM Michel Dänzer <michel@daenzer.net> wrote:
>>>> On 7/28/23 11:30, Michel Dänzer wrote:
>>>>> On 7/28/23 03:38, Zhang, Jesse(Jie) wrote:
>>>>>>
>>>>>> Could you try the patch again ?  That work for me.
>>>>>>
>>>>>> https://patchwork.freedesktop.org/patch/549605/ <https://patchwork.freedesktop.org/patch/549605/>
>>>>>
>>>>> This patch fixes the symptoms described in https://gitlab.freedesktop.org/drm/amd/-/issues/2659 for me as well.
>>>>>
>>>>> However, it does not fix the IOMMU page faults[0] or the IB test failures on the compute rings. Should I try amdgpu.ignore_crat=1 for these symptoms as well?
>>>>
>>>> I tried ignore_crat=1, it avoids the remaining symptoms as well.
>>>
>>> The first 3 patches of this set may also fix it:
>>> https://patchwork.freedesktop.org/series/121538/
>>
>> They do fix all symptoms I'm seeing.
>>
>> Is this a feasible solution for the final 6.5 release?
> 
> I think the first3 are.

Even with Felix's feedback on patch 2?

If so, will you merge these for 6.5 final?

Or should we revert 84b4dd3f84de ("drm/amdkfd: Refactor migrate init to support partition switch") for that?


-- 
Earthling Michel Dänzer            |                  https://redhat.com
Libre software enthusiast          |         Mesa and Xwayland developer

Re: [PATCH 28/29] drm/amdkfd: Refactor migrate init to support partition switch

[Alex Deucher]

On Mon, Aug 7, 2023 at 12:04 PM Michel Dänzer <michel@daenzer.net> wrote:
>
> On 7/28/23 19:20, Alex Deucher wrote:
> > On Fri, Jul 28, 2023 at 1:19 PM Michel Dänzer <michel@daenzer.net> wrote:
> >> On 7/28/23 18:43, Alex Deucher wrote:
> >>> On Fri, Jul 28, 2023 at 10:25 AM Michel Dänzer <michel@daenzer.net> wrote:
> >>>> On 7/28/23 11:30, Michel Dänzer wrote:
> >>>>> On 7/28/23 03:38, Zhang, Jesse(Jie) wrote:
> >>>>>>
> >>>>>> Could you try the patch again ?  That work for me.
> >>>>>>
> >>>>>> https://patchwork.freedesktop.org/patch/549605/ <https://patchwork.freedesktop.org/patch/549605/>
> >>>>>
> >>>>> This patch fixes the symptoms described in https://gitlab.freedesktop.org/drm/amd/-/issues/2659 for me as well.
> >>>>>
> >>>>> However, it does not fix the IOMMU page faults[0] or the IB test failures on the compute rings. Should I try amdgpu.ignore_crat=1 for these symptoms as well?
> >>>>
> >>>> I tried ignore_crat=1, it avoids the remaining symptoms as well.
> >>>
> >>> The first 3 patches of this set may also fix it:
> >>> https://patchwork.freedesktop.org/series/121538/
> >>
> >> They do fix all symptoms I'm seeing.
> >>
> >> Is this a feasible solution for the final 6.5 release?
> >
> > I think the first3 are.
>
> Even with Felix's feedback on patch 2?
>
> If so, will you merge these for 6.5 final?

Yes, just sent out a new patch set.  The changes for Felix' feedback
was pretty trivial.  Once they are reviewed, I'll land them for 6.5.
Hopefully this week.

Alex

>
> Or should we revert 84b4dd3f84de ("drm/amdkfd: Refactor migrate init to support partition switch") for that?
>
>
> --
> Earthling Michel Dänzer            |                  https://redhat.com
> Libre software enthusiast          |         Mesa and Xwayland developer
>

[PATCH 29/29] drm/amdgpu: Correct get_xcp_mem_id calculation

[Alex Deucher]

From: Philip Yang <Philip.Yang@amd.com>

Current calculation only works for NPS4/QPX mode, correct it for
NPS4/CPX mode.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Lijo Lazar <lijo.lazar@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/aqua_vanjaram_reg_init.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram_reg_init.c b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram_reg_init.c
index 4ca932a62ce6..93e9f947a85d 100644
--- a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram_reg_init.c
+++ b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram_reg_init.c
@@ -518,10 +518,9 @@ static int aqua_vanjaram_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr,
 static int __aqua_vanjaram_get_xcp_mem_id(struct amdgpu_device *adev,
 					  int xcc_id, uint8_t *mem_id)
 {
-	/* TODO: Check if any validation is required based on current
-	 * memory/spatial modes
-	 */
+	/* memory/spatial modes validation check is already done */
 	*mem_id = xcc_id / adev->gfx.num_xcc_per_xcp;
+	*mem_id /= adev->xcp_mgr->num_xcp_per_mem_partition;
 
 	return 0;
 }
-- 
2.40.1