[PATCH 0/7] Add support for dynamic NPS switch

[PATCH 0/7] Add support for dynamic NPS switch

[Lijo Lazar]

This series adds supports for dynamic NPS switch on GC v9.4.3/9.4.4 SOC
variants. 

In order to do dynamic NPS switch a sysfs interface is provided to request a new
NPS mode. If the device is part of a hive, all hive devices are required to be
in the same NPS mode. Hence a hive device request is saved in a hive variable.
For individual device, it's saved in a gmc block variable.

In order to do a NPS mode switch, the workflow is -

1) User places a requests through sysfs node.
2) User unloads the driver
3) During unload, driver checks for any pending NPS switch request. If any
request is pending, it places the request to PSP FW. 
4) For a hive, request is placed in one-go for all devices in the hive. If
one of the requests fails, a request is placed again to revert to current NPS
mode on the successful devices.
5) User reloads the driver.
6) On reload, driver checks if NPS switch is pending and initiates a mode-1
reset.
7) During resume after a reset, NPS ranges are read again from discovery table.
8) Driver detects the new NPS mode and makes a compatible compute partition mode
switch if required.

Lijo Lazar (7):
  drm/amdgpu: Add option to refresh NPS data
  drm/amdgpu: Add PSP interface for NPS switch
  drm/amdgpu: Add gmc interface to request NPS mode
  drm/amdgpu: Add sysfs interfaces for NPS mode
  drm/amdgpu: Place NPS mode request on unload
  drm/amdgpu: Check gmc requirement for reset on init
  drm/amdgpu: Add NPS switch support for GC 9.4.3

 drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c |  68 +++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c       |   1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c       | 190 ++++++++++++++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h       |  19 ++
 drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h      |   1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c       |  25 +++
 drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h       |   1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c      |  39 ++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h      |   5 +
 drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c         |  44 ++++
 drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c        |  12 ++
 drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h       |  14 +-
 drivers/gpu/drm/amd/amdgpu/soc15.c            |   2 +
 14 files changed, 387 insertions(+), 36 deletions(-)

-- 
2.25.1

[PATCH 1/7] drm/amdgpu: Add option to refresh NPS data

[Lijo Lazar]

In certain use cases, NPS data needs to be refreshed again from
discovery table. Add API parameter to refresh NPS data from discovery
table.

Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c | 68 +++++++++++++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h |  2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c       |  2 +-
 3 files changed, 55 insertions(+), 17 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
index 4bd61c169ca8..9f9a1867da72 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
@@ -1723,37 +1723,75 @@ union nps_info {
 	struct nps_info_v1_0 v1;
 };
 
+static int amdgpu_discovery_refresh_nps_info(struct amdgpu_device *adev,
+					     union nps_info *nps_data)
+{
+	uint64_t vram_size, pos, offset;
+	struct nps_info_header *nhdr;
+	struct binary_header bhdr;
+	uint16_t checksum;
+
+	vram_size = (uint64_t)RREG32(mmRCC_CONFIG_MEMSIZE) << 20;
+	pos = vram_size - DISCOVERY_TMR_OFFSET;
+	amdgpu_device_vram_access(adev, pos, &bhdr, sizeof(bhdr), false);
+
+	offset = le16_to_cpu(bhdr.table_list[NPS_INFO].offset);
+	checksum = le16_to_cpu(bhdr.table_list[NPS_INFO].checksum);
+
+	amdgpu_device_vram_access(adev, (pos + offset), nps_data,
+				  sizeof(*nps_data), false);
+
+	nhdr = (struct nps_info_header *)(nps_data);
+	if (!amdgpu_discovery_verify_checksum((uint8_t *)nps_data,
+					      le32_to_cpu(nhdr->size_bytes),
+					      checksum)) {
+		dev_err(adev->dev, "nps data refresh, checksum mismatch\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 int amdgpu_discovery_get_nps_info(struct amdgpu_device *adev,
 				  uint32_t *nps_type,
 				  struct amdgpu_gmc_memrange **ranges,
-				  int *range_cnt)
+				  int *range_cnt, bool refresh)
 {
 	struct amdgpu_gmc_memrange *mem_ranges;
 	struct binary_header *bhdr;
 	union nps_info *nps_info;
+	union nps_info nps_data;
 	u16 offset;
-	int i;
+	int i, r;
 
 	if (!nps_type || !range_cnt || !ranges)
 		return -EINVAL;
 
-	if (!adev->mman.discovery_bin) {
-		dev_err(adev->dev,
-			"fetch mem range failed, ip discovery uninitialized\n");
-		return -EINVAL;
-	}
+	if (refresh) {
+		r = amdgpu_discovery_refresh_nps_info(adev, &nps_data);
+		if (r)
+			return r;
+		nps_info = &nps_data;
+	} else {
+		if (!adev->mman.discovery_bin) {
+			dev_err(adev->dev,
+				"fetch mem range failed, ip discovery uninitialized\n");
+			return -EINVAL;
+		}
 
-	bhdr = (struct binary_header *)adev->mman.discovery_bin;
-	offset = le16_to_cpu(bhdr->table_list[NPS_INFO].offset);
+		bhdr = (struct binary_header *)adev->mman.discovery_bin;
+		offset = le16_to_cpu(bhdr->table_list[NPS_INFO].offset);
 
-	if (!offset)
-		return -ENOENT;
+		if (!offset)
+			return -ENOENT;
 
-	/* If verification fails, return as if NPS table doesn't exist */
-	if (amdgpu_discovery_verify_npsinfo(adev, bhdr))
-		return -ENOENT;
+		/* If verification fails, return as if NPS table doesn't exist */
+		if (amdgpu_discovery_verify_npsinfo(adev, bhdr))
+			return -ENOENT;
 
-	nps_info = (union nps_info *)(adev->mman.discovery_bin + offset);
+		nps_info =
+			(union nps_info *)(adev->mman.discovery_bin + offset);
+	}
 
 	switch (le16_to_cpu(nps_info->v1.header.version_major)) {
 	case 1:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
index f5d36525ec3e..b44d56465c5b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
@@ -33,6 +33,6 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev);
 int amdgpu_discovery_get_nps_info(struct amdgpu_device *adev,
 				  uint32_t *nps_type,
 				  struct amdgpu_gmc_memrange **ranges,
-				  int *range_cnt);
+				  int *range_cnt, bool refresh);
 
 #endif /* __AMDGPU_DISCOVERY__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 17a19d49d30a..4f088a5368d8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -1184,7 +1184,7 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
 		return -EINVAL;
 
 	ret = amdgpu_discovery_get_nps_info(adev, &nps_type, &ranges,
-					    &range_cnt);
+					    &range_cnt, false);
 
 	if (ret)
 		return ret;
-- 
2.25.1

Re: [PATCH 1/7] drm/amdgpu: Add option to refresh NPS data

[Bhardwaj, Rajneesh]

Reviewed-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>

On 9/24/2024 1:56 AM, Lijo Lazar wrote:
> In certain use cases, NPS data needs to be refreshed again from
> discovery table. Add API parameter to refresh NPS data from discovery
> table.
>
> Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c | 68 +++++++++++++++----
>   drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h |  2 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c       |  2 +-
>   3 files changed, 55 insertions(+), 17 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
> index 4bd61c169ca8..9f9a1867da72 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
> @@ -1723,37 +1723,75 @@ union nps_info {
>   	struct nps_info_v1_0 v1;
>   };
>   
> +static int amdgpu_discovery_refresh_nps_info(struct amdgpu_device *adev,
> +					     union nps_info *nps_data)
> +{
> +	uint64_t vram_size, pos, offset;
> +	struct nps_info_header *nhdr;
> +	struct binary_header bhdr;
> +	uint16_t checksum;
> +
> +	vram_size = (uint64_t)RREG32(mmRCC_CONFIG_MEMSIZE) << 20;
> +	pos = vram_size - DISCOVERY_TMR_OFFSET;
> +	amdgpu_device_vram_access(adev, pos, &bhdr, sizeof(bhdr), false);
> +
> +	offset = le16_to_cpu(bhdr.table_list[NPS_INFO].offset);
> +	checksum = le16_to_cpu(bhdr.table_list[NPS_INFO].checksum);
> +
> +	amdgpu_device_vram_access(adev, (pos + offset), nps_data,
> +				  sizeof(*nps_data), false);
> +
> +	nhdr = (struct nps_info_header *)(nps_data);
> +	if (!amdgpu_discovery_verify_checksum((uint8_t *)nps_data,
> +					      le32_to_cpu(nhdr->size_bytes),
> +					      checksum)) {
> +		dev_err(adev->dev, "nps data refresh, checksum mismatch\n");
> +		return -EINVAL;
> +	}
> +
> +	return 0;
> +}
> +
>   int amdgpu_discovery_get_nps_info(struct amdgpu_device *adev,
>   				  uint32_t *nps_type,
>   				  struct amdgpu_gmc_memrange **ranges,
> -				  int *range_cnt)
> +				  int *range_cnt, bool refresh)
>   {
>   	struct amdgpu_gmc_memrange *mem_ranges;
>   	struct binary_header *bhdr;
>   	union nps_info *nps_info;
> +	union nps_info nps_data;
>   	u16 offset;
> -	int i;
> +	int i, r;
>   
>   	if (!nps_type || !range_cnt || !ranges)
>   		return -EINVAL;
>   
> -	if (!adev->mman.discovery_bin) {
> -		dev_err(adev->dev,
> -			"fetch mem range failed, ip discovery uninitialized\n");
> -		return -EINVAL;
> -	}
> +	if (refresh) {
> +		r = amdgpu_discovery_refresh_nps_info(adev, &nps_data);
> +		if (r)
> +			return r;
> +		nps_info = &nps_data;
> +	} else {
> +		if (!adev->mman.discovery_bin) {
> +			dev_err(adev->dev,
> +				"fetch mem range failed, ip discovery uninitialized\n");
> +			return -EINVAL;
> +		}
>   
> -	bhdr = (struct binary_header *)adev->mman.discovery_bin;
> -	offset = le16_to_cpu(bhdr->table_list[NPS_INFO].offset);
> +		bhdr = (struct binary_header *)adev->mman.discovery_bin;
> +		offset = le16_to_cpu(bhdr->table_list[NPS_INFO].offset);
>   
> -	if (!offset)
> -		return -ENOENT;
> +		if (!offset)
> +			return -ENOENT;
>   
> -	/* If verification fails, return as if NPS table doesn't exist */
> -	if (amdgpu_discovery_verify_npsinfo(adev, bhdr))
> -		return -ENOENT;
> +		/* If verification fails, return as if NPS table doesn't exist */
> +		if (amdgpu_discovery_verify_npsinfo(adev, bhdr))
> +			return -ENOENT;
>   
> -	nps_info = (union nps_info *)(adev->mman.discovery_bin + offset);
> +		nps_info =
> +			(union nps_info *)(adev->mman.discovery_bin + offset);
> +	}
>   
>   	switch (le16_to_cpu(nps_info->v1.header.version_major)) {
>   	case 1:
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
> index f5d36525ec3e..b44d56465c5b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
> @@ -33,6 +33,6 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev);
>   int amdgpu_discovery_get_nps_info(struct amdgpu_device *adev,
>   				  uint32_t *nps_type,
>   				  struct amdgpu_gmc_memrange **ranges,
> -				  int *range_cnt);
> +				  int *range_cnt, bool refresh);
>   
>   #endif /* __AMDGPU_DISCOVERY__ */
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> index 17a19d49d30a..4f088a5368d8 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> @@ -1184,7 +1184,7 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
>   		return -EINVAL;
>   
>   	ret = amdgpu_discovery_get_nps_info(adev, &nps_type, &ranges,
> -					    &range_cnt);
> +					    &range_cnt, false);
>   
>   	if (ret)
>   		return ret;

[PATCH 2/7] drm/amdgpu: Add PSP interface for NPS switch

[Lijo Lazar]

Implement PSP ring command interface for memory partitioning on the fly
on the supported asics.

Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c | 25 +++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h |  1 +
 drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h | 14 +++++++++++---
 3 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 944dad9ad29f..04be0fabb4f5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -1045,6 +1045,31 @@ static int psp_rl_load(struct amdgpu_device *adev)
 	return ret;
 }
 
+int psp_memory_partition(struct psp_context *psp, int mode)
+{
+	struct psp_gfx_cmd_resp *cmd;
+	int ret;
+
+	if (amdgpu_sriov_vf(psp->adev))
+		return 0;
+
+	cmd = acquire_psp_cmd_buf(psp);
+
+	cmd->cmd_id = GFX_CMD_ID_FB_NPS_MODE;
+	cmd->cmd.cmd_memory_part.mode = mode;
+
+	dev_info(psp->adev->dev,
+		 "Requesting %d memory partition change through PSP", mode);
+	ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
+	if (ret)
+		dev_err(psp->adev->dev,
+			"PSP request failed to change to NPS%d mode\n", mode);
+
+	release_psp_cmd_buf(psp);
+
+	return ret;
+}
+
 int psp_spatial_partition(struct psp_context *psp, int mode)
 {
 	struct psp_gfx_cmd_resp *cmd;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
index 76fa18ffc045..567cb1f924ca 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
@@ -553,6 +553,7 @@ int psp_load_fw_list(struct psp_context *psp,
 void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size);
 
 int psp_spatial_partition(struct psp_context *psp, int mode);
+int psp_memory_partition(struct psp_context *psp, int mode);
 
 int is_psp_fw_valid(struct psp_bin_desc bin);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
index 604301371e4f..f4a91b126c73 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
+++ b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
@@ -103,8 +103,10 @@ enum psp_gfx_cmd_id
     GFX_CMD_ID_AUTOLOAD_RLC       = 0x00000021,   /* Indicates all graphics fw loaded, start RLC autoload */
     GFX_CMD_ID_BOOT_CFG           = 0x00000022,   /* Boot Config */
     GFX_CMD_ID_SRIOV_SPATIAL_PART = 0x00000027,   /* Configure spatial partitioning mode */
-	/*IDs of performance monitoring/profiling*/
-	GFX_CMD_ID_CONFIG_SQ_PERFMON = 0x00000046,   /* Config CGTT_SQ_CLK_CTRL */
+    /*IDs of performance monitoring/profiling*/
+    GFX_CMD_ID_CONFIG_SQ_PERFMON  = 0x00000046,   /* Config CGTT_SQ_CLK_CTRL */
+    /* Dynamic memory partitioninig (NPS mode change)*/
+    GFX_CMD_ID_FB_NPS_MODE         = 0x00000048,  /* Configure memory partitioning mode */
 };
 
 /* PSP boot config sub-commands */
@@ -362,6 +364,11 @@ struct psp_gfx_cmd_config_sq_perfmon {
 	uint8_t         reserved[5];
 };
 
+struct psp_gfx_cmd_fb_memory_part {
+	uint32_t mode; /* requested NPS mode */
+	uint32_t resvd;
+};
+
 /* All GFX ring buffer commands. */
 union psp_gfx_commands
 {
@@ -376,7 +383,8 @@ union psp_gfx_commands
     struct psp_gfx_cmd_load_toc         cmd_load_toc;
     struct psp_gfx_cmd_boot_cfg         boot_cfg;
     struct psp_gfx_cmd_sriov_spatial_part cmd_spatial_part;
-	struct psp_gfx_cmd_config_sq_perfmon config_sq_perfmon;
+    struct psp_gfx_cmd_config_sq_perfmon config_sq_perfmon;
+    struct psp_gfx_cmd_fb_memory_part cmd_memory_part;
 };
 
 struct psp_gfx_uresp_reserved
-- 
2.25.1

RE: [PATCH 2/7] drm/amdgpu: Add PSP interface for NPS switch

[Xu, Feifei]

[AMD Official Use Only - AMD Internal Distribution Only]

Reviewed-by: Feifei Xu <Feifei.Xu@amd.com>

-----Original Message-----
From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Lijo Lazar
Sent: Tuesday, September 24, 2024 1:57 PM
To: amd-gfx@lists.freedesktop.org
Cc: Zhang, Hawking <Hawking.Zhang@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Koenig, Christian <Christian.Koenig@amd.com>; Bhardwaj, Rajneesh <Rajneesh.Bhardwaj@amd.com>; Errabolu, Ramesh <Ramesh.Errabolu@amd.com>
Subject: [PATCH 2/7] drm/amdgpu: Add PSP interface for NPS switch

Implement PSP ring command interface for memory partitioning on the fly on the supported asics.

Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c | 25 +++++++++++++++++++++++++  drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h |  1 +  drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h | 14 +++++++++++---
 3 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 944dad9ad29f..04be0fabb4f5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -1045,6 +1045,31 @@ static int psp_rl_load(struct amdgpu_device *adev)
        return ret;
 }

+int psp_memory_partition(struct psp_context *psp, int mode) {
+       struct psp_gfx_cmd_resp *cmd;
+       int ret;
+
+       if (amdgpu_sriov_vf(psp->adev))
+               return 0;
+
+       cmd = acquire_psp_cmd_buf(psp);
+
+       cmd->cmd_id = GFX_CMD_ID_FB_NPS_MODE;
+       cmd->cmd.cmd_memory_part.mode = mode;
+
+       dev_info(psp->adev->dev,
+                "Requesting %d memory partition change through PSP", mode);
+       ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
+       if (ret)
+               dev_err(psp->adev->dev,
+                       "PSP request failed to change to NPS%d mode\n", mode);
+
+       release_psp_cmd_buf(psp);
+
+       return ret;
+}
+
 int psp_spatial_partition(struct psp_context *psp, int mode)  {
        struct psp_gfx_cmd_resp *cmd;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
index 76fa18ffc045..567cb1f924ca 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
@@ -553,6 +553,7 @@ int psp_load_fw_list(struct psp_context *psp,  void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size);

 int psp_spatial_partition(struct psp_context *psp, int mode);
+int psp_memory_partition(struct psp_context *psp, int mode);

 int is_psp_fw_valid(struct psp_bin_desc bin);

diff --git a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
index 604301371e4f..f4a91b126c73 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
+++ b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
@@ -103,8 +103,10 @@ enum psp_gfx_cmd_id
     GFX_CMD_ID_AUTOLOAD_RLC       = 0x00000021,   /* Indicates all graphics fw loaded, start RLC autoload */
     GFX_CMD_ID_BOOT_CFG           = 0x00000022,   /* Boot Config */
     GFX_CMD_ID_SRIOV_SPATIAL_PART = 0x00000027,   /* Configure spatial partitioning mode */
-       /*IDs of performance monitoring/profiling*/
-       GFX_CMD_ID_CONFIG_SQ_PERFMON = 0x00000046,   /* Config CGTT_SQ_CLK_CTRL */
+    /*IDs of performance monitoring/profiling*/
+    GFX_CMD_ID_CONFIG_SQ_PERFMON  = 0x00000046,   /* Config CGTT_SQ_CLK_CTRL */
+    /* Dynamic memory partitioninig (NPS mode change)*/
+    GFX_CMD_ID_FB_NPS_MODE         = 0x00000048,  /* Configure memory partitioning mode */
 };

 /* PSP boot config sub-commands */
@@ -362,6 +364,11 @@ struct psp_gfx_cmd_config_sq_perfmon {
        uint8_t         reserved[5];
 };

+struct psp_gfx_cmd_fb_memory_part {
+       uint32_t mode; /* requested NPS mode */
+       uint32_t resvd;
+};
+
 /* All GFX ring buffer commands. */
 union psp_gfx_commands
 {
@@ -376,7 +383,8 @@ union psp_gfx_commands
     struct psp_gfx_cmd_load_toc         cmd_load_toc;
     struct psp_gfx_cmd_boot_cfg         boot_cfg;
     struct psp_gfx_cmd_sriov_spatial_part cmd_spatial_part;
-       struct psp_gfx_cmd_config_sq_perfmon config_sq_perfmon;
+    struct psp_gfx_cmd_config_sq_perfmon config_sq_perfmon;
+    struct psp_gfx_cmd_fb_memory_part cmd_memory_part;
 };

 struct psp_gfx_uresp_reserved
--
2.25.1

[PATCH 3/7] drm/amdgpu: Add gmc interface to request NPS mode

[Lijo Lazar]

Add a common interface in GMC to request NPS mode through PSP. Also add
a variable in hive and gmc control to track the last requested mode.

Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c  | 16 ++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h  |  6 ++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h |  1 +
 4 files changed, 24 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 4f088a5368d8..758fda4e628f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -1247,3 +1247,19 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
 
 	return ret;
 }
+
+int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev,
+					int nps_mode)
+{
+	/* Not supported on VF devices and APUs */
+	if (amdgpu_sriov_vf(adev) || (adev->flags & AMD_IS_APU))
+		return -EOPNOTSUPP;
+
+	if (!adev->psp.funcs) {
+		dev_err(adev->dev,
+			"PSP interface not available for nps mode change request");
+		return -EINVAL;
+	}
+
+	return psp_memory_partition(&adev->psp, nps_mode);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index 33b2adffd58b..f5be5112b742 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -161,6 +161,9 @@ struct amdgpu_gmc_funcs {
 
 	enum amdgpu_memory_partition (*query_mem_partition_mode)(
 		struct amdgpu_device *adev);
+	/* Request NPS mode */
+	int (*request_mem_partition_mode)(struct amdgpu_device *adev,
+					  int nps_mode);
 };
 
 struct amdgpu_xgmi_ras {
@@ -304,6 +307,7 @@ struct amdgpu_gmc {
 	struct amdgpu_mem_partition_info *mem_partitions;
 	uint8_t num_mem_partitions;
 	const struct amdgpu_gmc_funcs	*gmc_funcs;
+	enum amdgpu_memory_partition	requested_nps_mode;
 
 	struct amdgpu_xgmi xgmi;
 	struct amdgpu_irq_src	ecc_irq;
@@ -455,4 +459,6 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
 				 struct amdgpu_mem_partition_info *mem_ranges,
 				 int exp_ranges);
 
+int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev,
+					int nps_mode);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index b17e63c98a99..5d721ccb9dfd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
@@ -667,6 +667,7 @@ struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev)
 	task_barrier_init(&hive->tb);
 	hive->pstate = AMDGPU_XGMI_PSTATE_UNKNOWN;
 	hive->hi_req_gpu = NULL;
+	atomic_set(&hive->requested_nps_mode, UNKNOWN_MEMORY_PARTITION_MODE);
 
 	/*
 	 * hive pstate on boot is high in vega20 so we have to go to low
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
index d652727ca565..67abadb4f298 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
@@ -46,6 +46,7 @@ struct amdgpu_hive_info {
 	atomic_t ras_recovery;
 	struct ras_event_manager event_mgr;
 	struct work_struct reset_on_init_work;
+	atomic_t requested_nps_mode;
 };
 
 struct amdgpu_pcs_ras_field {
-- 
2.25.1

RE: [PATCH 3/7] drm/amdgpu: Add gmc interface to request NPS mode

[Xu, Feifei]

[AMD Official Use Only - AMD Internal Distribution Only]

Reviewed-by: Feifei Xu <Feifei.Xu@amd.com>

-----Original Message-----
From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Lijo Lazar
Sent: Tuesday, September 24, 2024 1:57 PM
To: amd-gfx@lists.freedesktop.org
Cc: Zhang, Hawking <Hawking.Zhang@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Koenig, Christian <Christian.Koenig@amd.com>; Bhardwaj, Rajneesh <Rajneesh.Bhardwaj@amd.com>; Errabolu, Ramesh <Ramesh.Errabolu@amd.com>
Subject: [PATCH 3/7] drm/amdgpu: Add gmc interface to request NPS mode

Add a common interface in GMC to request NPS mode through PSP. Also add a variable in hive and gmc control to track the last requested mode.

Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c  | 16 ++++++++++++++++  drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h  |  6 ++++++  drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c |  1 +  drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h |  1 +
 4 files changed, 24 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 4f088a5368d8..758fda4e628f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -1247,3 +1247,19 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,

        return ret;
 }
+
+int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev,
+                                       int nps_mode)
+{
+       /* Not supported on VF devices and APUs */
+       if (amdgpu_sriov_vf(adev) || (adev->flags & AMD_IS_APU))
+               return -EOPNOTSUPP;
+
+       if (!adev->psp.funcs) {
+               dev_err(adev->dev,
+                       "PSP interface not available for nps mode change request");
+               return -EINVAL;
+       }
+
+       return psp_memory_partition(&adev->psp, nps_mode); }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index 33b2adffd58b..f5be5112b742 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -161,6 +161,9 @@ struct amdgpu_gmc_funcs {

        enum amdgpu_memory_partition (*query_mem_partition_mode)(
                struct amdgpu_device *adev);
+       /* Request NPS mode */
+       int (*request_mem_partition_mode)(struct amdgpu_device *adev,
+                                         int nps_mode);
 };

 struct amdgpu_xgmi_ras {
@@ -304,6 +307,7 @@ struct amdgpu_gmc {
        struct amdgpu_mem_partition_info *mem_partitions;
        uint8_t num_mem_partitions;
        const struct amdgpu_gmc_funcs   *gmc_funcs;
+       enum amdgpu_memory_partition    requested_nps_mode;

        struct amdgpu_xgmi xgmi;
        struct amdgpu_irq_src   ecc_irq;
@@ -455,4 +459,6 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
                                 struct amdgpu_mem_partition_info *mem_ranges,
                                 int exp_ranges);

+int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev,
+                                       int nps_mode);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index b17e63c98a99..5d721ccb9dfd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
@@ -667,6 +667,7 @@ struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev)
        task_barrier_init(&hive->tb);
        hive->pstate = AMDGPU_XGMI_PSTATE_UNKNOWN;
        hive->hi_req_gpu = NULL;
+       atomic_set(&hive->requested_nps_mode, UNKNOWN_MEMORY_PARTITION_MODE);

        /*
         * hive pstate on boot is high in vega20 so we have to go to low diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
index d652727ca565..67abadb4f298 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
@@ -46,6 +46,7 @@ struct amdgpu_hive_info {
        atomic_t ras_recovery;
        struct ras_event_manager event_mgr;
        struct work_struct reset_on_init_work;
+       atomic_t requested_nps_mode;
 };

 struct amdgpu_pcs_ras_field {
--
2.25.1

[PATCH 4/7] drm/amdgpu: Add sysfs interfaces for NPS mode

[Lijo Lazar]

Add a sysfs interface to see available NPS modes to switch to -

	cat /sys/bus/pci/devices/../available_memory_paritition

Make the current_memory_partition sysfs node read/write for requesting a
new NPS mode. The request is only cached and at a later point a driver
unload/reload is required to switch to the new NPS mode.

Ex:
	echo NPS1 > /sys/bus/pci/devices/../current_memory_paritition
	echo NPS4 > /sys/bus/pci/devices/../current_memory_paritition

The above interfaces will be available only if the SOC supports more than
one NPS mode.

Also modify the current memory partition sysfs logic to be more
generic.

Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 114 ++++++++++++++++++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |   6 ++
 2 files changed, 104 insertions(+), 16 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 758fda4e628f..24a1f931d9ed 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -1130,6 +1130,79 @@ int amdgpu_gmc_vram_checking(struct amdgpu_device *adev)
 	return ret;
 }
 
+static const char *nps_desc[] = {
+	[AMDGPU_NPS1_PARTITION_MODE] = "NPS1",
+	[AMDGPU_NPS2_PARTITION_MODE] = "NPS2",
+	[AMDGPU_NPS3_PARTITION_MODE] = "NPS3",
+	[AMDGPU_NPS4_PARTITION_MODE] = "NPS4",
+	[AMDGPU_NPS6_PARTITION_MODE] = "NPS6",
+	[AMDGPU_NPS8_PARTITION_MODE] = "NPS8",
+};
+
+static ssize_t available_memory_partition_show(struct device *dev,
+					       struct device_attribute *addr,
+					       char *buf)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+	int size = 0, mode;
+	char *sep = "";
+
+	for_each_inst(mode, adev->gmc.supported_nps_modes) {
+		size += sysfs_emit_at(buf, size, "%s%s", sep, nps_desc[mode]);
+		sep = ", ";
+	}
+	size += sysfs_emit_at(buf, size, "\n");
+
+	return size;
+}
+
+static ssize_t current_memory_partition_store(struct device *dev,
+					      struct device_attribute *attr,
+					      const char *buf, size_t count)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+	enum amdgpu_memory_partition mode;
+	struct amdgpu_hive_info *hive;
+	int i;
+
+	mode = UNKNOWN_MEMORY_PARTITION_MODE;
+	for_each_inst(i, adev->gmc.supported_nps_modes) {
+		if (!strncasecmp(nps_desc[i], buf, strlen(nps_desc[i]))) {
+			mode = i;
+			break;
+		}
+	}
+
+	if (mode == UNKNOWN_MEMORY_PARTITION_MODE)
+		return -EINVAL;
+
+	if (mode == adev->gmc.gmc_funcs->query_mem_partition_mode(adev)) {
+		dev_info(
+			adev->dev,
+			"requested NPS mode is same as current NPS mode, skipping\n");
+		return count;
+	}
+
+	/* If device is part of hive, all devices in the hive should request the
+	 * same mode. Hence store the requested mode in hive.
+	 */
+	hive = amdgpu_get_xgmi_hive(adev);
+	if (hive) {
+		atomic_set(&hive->requested_nps_mode, mode);
+		amdgpu_put_xgmi_hive(hive);
+	} else {
+		adev->gmc.requested_nps_mode = mode;
+	}
+
+	dev_info(
+		adev->dev,
+		"NPS mode change requested, please remove and reload the driver\n");
+
+	return count;
+}
+
 static ssize_t current_memory_partition_show(
 	struct device *dev, struct device_attribute *addr, char *buf)
 {
@@ -1138,38 +1211,47 @@ static ssize_t current_memory_partition_show(
 	enum amdgpu_memory_partition mode;
 
 	mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
-	switch (mode) {
-	case AMDGPU_NPS1_PARTITION_MODE:
-		return sysfs_emit(buf, "NPS1\n");
-	case AMDGPU_NPS2_PARTITION_MODE:
-		return sysfs_emit(buf, "NPS2\n");
-	case AMDGPU_NPS3_PARTITION_MODE:
-		return sysfs_emit(buf, "NPS3\n");
-	case AMDGPU_NPS4_PARTITION_MODE:
-		return sysfs_emit(buf, "NPS4\n");
-	case AMDGPU_NPS6_PARTITION_MODE:
-		return sysfs_emit(buf, "NPS6\n");
-	case AMDGPU_NPS8_PARTITION_MODE:
-		return sysfs_emit(buf, "NPS8\n");
-	default:
+	if ((mode > ARRAY_SIZE(nps_desc)) ||
+	    (BIT(mode) & AMDGPU_ALL_NPS_MASK) != BIT(mode))
 		return sysfs_emit(buf, "UNKNOWN\n");
-	}
+
+	return sysfs_emit(buf, "%s\n", nps_desc[mode]);
 }
 
-static DEVICE_ATTR_RO(current_memory_partition);
+static DEVICE_ATTR_RW(current_memory_partition);
+static DEVICE_ATTR_RO(available_memory_partition);
 
 int amdgpu_gmc_sysfs_init(struct amdgpu_device *adev)
 {
+	bool nps_switch_support;
+	int r = 0;
+
 	if (!adev->gmc.gmc_funcs->query_mem_partition_mode)
 		return 0;
 
+	nps_switch_support = (hweight32(adev->gmc.supported_nps_modes &
+					AMDGPU_ALL_NPS_MASK) > 1);
+	if (!nps_switch_support)
+		dev_attr_current_memory_partition.attr.mode &=
+			~(S_IWUSR | S_IWGRP | S_IWOTH);
+	else
+		r = device_create_file(adev->dev,
+				       &dev_attr_available_memory_partition);
+
+	if (r)
+		return r;
+
 	return device_create_file(adev->dev,
 				  &dev_attr_current_memory_partition);
 }
 
 void amdgpu_gmc_sysfs_fini(struct amdgpu_device *adev)
 {
+	if (!adev->gmc.gmc_funcs->query_mem_partition_mode)
+		return;
+
 	device_remove_file(adev->dev, &dev_attr_current_memory_partition);
+	device_remove_file(adev->dev, &dev_attr_available_memory_partition);
 }
 
 int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index f5be5112b742..1a50639a003a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -73,6 +73,11 @@ enum amdgpu_memory_partition {
 	AMDGPU_NPS8_PARTITION_MODE = 8,
 };
 
+#define AMDGPU_ALL_NPS_MASK                                                  \
+	(BIT(AMDGPU_NPS1_PARTITION_MODE) | BIT(AMDGPU_NPS2_PARTITION_MODE) | \
+	 BIT(AMDGPU_NPS3_PARTITION_MODE) | BIT(AMDGPU_NPS4_PARTITION_MODE) | \
+	 BIT(AMDGPU_NPS6_PARTITION_MODE) | BIT(AMDGPU_NPS8_PARTITION_MODE))
+
 /*
  * GMC page fault information
  */
@@ -308,6 +313,7 @@ struct amdgpu_gmc {
 	uint8_t num_mem_partitions;
 	const struct amdgpu_gmc_funcs	*gmc_funcs;
 	enum amdgpu_memory_partition	requested_nps_mode;
+	uint32_t supported_nps_modes;
 
 	struct amdgpu_xgmi xgmi;
 	struct amdgpu_irq_src	ecc_irq;
-- 
2.25.1

Re: [PATCH 4/7] drm/amdgpu: Add sysfs interfaces for NPS mode

[Bhardwaj, Rajneesh]

Reviewed-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>

On 9/24/2024 1:56 AM, Lijo Lazar wrote:
> Add a sysfs interface to see available NPS modes to switch to -
>
> 	cat /sys/bus/pci/devices/../available_memory_paritition
>
> Make the current_memory_partition sysfs node read/write for requesting a
> new NPS mode. The request is only cached and at a later point a driver
> unload/reload is required to switch to the new NPS mode.
>
> Ex:
> 	echo NPS1 > /sys/bus/pci/devices/../current_memory_paritition
> 	echo NPS4 > /sys/bus/pci/devices/../current_memory_paritition
>
> The above interfaces will be available only if the SOC supports more than
> one NPS mode.
>
> Also modify the current memory partition sysfs logic to be more
> generic.
>
> Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 114 ++++++++++++++++++++----
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |   6 ++
>   2 files changed, 104 insertions(+), 16 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> index 758fda4e628f..24a1f931d9ed 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> @@ -1130,6 +1130,79 @@ int amdgpu_gmc_vram_checking(struct amdgpu_device *adev)
>   	return ret;
>   }
>   
> +static const char *nps_desc[] = {
> +	[AMDGPU_NPS1_PARTITION_MODE] = "NPS1",
> +	[AMDGPU_NPS2_PARTITION_MODE] = "NPS2",
> +	[AMDGPU_NPS3_PARTITION_MODE] = "NPS3",
> +	[AMDGPU_NPS4_PARTITION_MODE] = "NPS4",
> +	[AMDGPU_NPS6_PARTITION_MODE] = "NPS6",
> +	[AMDGPU_NPS8_PARTITION_MODE] = "NPS8",
> +};
> +
> +static ssize_t available_memory_partition_show(struct device *dev,
> +					       struct device_attribute *addr,
> +					       char *buf)
> +{
> +	struct drm_device *ddev = dev_get_drvdata(dev);
> +	struct amdgpu_device *adev = drm_to_adev(ddev);
> +	int size = 0, mode;
> +	char *sep = "";
> +
> +	for_each_inst(mode, adev->gmc.supported_nps_modes) {
> +		size += sysfs_emit_at(buf, size, "%s%s", sep, nps_desc[mode]);
> +		sep = ", ";
> +	}
> +	size += sysfs_emit_at(buf, size, "\n");
> +
> +	return size;
> +}
> +
> +static ssize_t current_memory_partition_store(struct device *dev,
> +					      struct device_attribute *attr,
> +					      const char *buf, size_t count)
> +{
> +	struct drm_device *ddev = dev_get_drvdata(dev);
> +	struct amdgpu_device *adev = drm_to_adev(ddev);
> +	enum amdgpu_memory_partition mode;
> +	struct amdgpu_hive_info *hive;
> +	int i;
> +
> +	mode = UNKNOWN_MEMORY_PARTITION_MODE;
> +	for_each_inst(i, adev->gmc.supported_nps_modes) {
> +		if (!strncasecmp(nps_desc[i], buf, strlen(nps_desc[i]))) {
> +			mode = i;
> +			break;
> +		}
> +	}
> +
> +	if (mode == UNKNOWN_MEMORY_PARTITION_MODE)
> +		return -EINVAL;
> +
> +	if (mode == adev->gmc.gmc_funcs->query_mem_partition_mode(adev)) {
> +		dev_info(
> +			adev->dev,
> +			"requested NPS mode is same as current NPS mode, skipping\n");
> +		return count;
> +	}
> +
> +	/* If device is part of hive, all devices in the hive should request the
> +	 * same mode. Hence store the requested mode in hive.
> +	 */
> +	hive = amdgpu_get_xgmi_hive(adev);
> +	if (hive) {
> +		atomic_set(&hive->requested_nps_mode, mode);
> +		amdgpu_put_xgmi_hive(hive);
> +	} else {
> +		adev->gmc.requested_nps_mode = mode;
> +	}
> +
> +	dev_info(
> +		adev->dev,
> +		"NPS mode change requested, please remove and reload the driver\n");
> +
> +	return count;
> +}
> +
>   static ssize_t current_memory_partition_show(
>   	struct device *dev, struct device_attribute *addr, char *buf)
>   {
> @@ -1138,38 +1211,47 @@ static ssize_t current_memory_partition_show(
>   	enum amdgpu_memory_partition mode;
>   
>   	mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
> -	switch (mode) {
> -	case AMDGPU_NPS1_PARTITION_MODE:
> -		return sysfs_emit(buf, "NPS1\n");
> -	case AMDGPU_NPS2_PARTITION_MODE:
> -		return sysfs_emit(buf, "NPS2\n");
> -	case AMDGPU_NPS3_PARTITION_MODE:
> -		return sysfs_emit(buf, "NPS3\n");
> -	case AMDGPU_NPS4_PARTITION_MODE:
> -		return sysfs_emit(buf, "NPS4\n");
> -	case AMDGPU_NPS6_PARTITION_MODE:
> -		return sysfs_emit(buf, "NPS6\n");
> -	case AMDGPU_NPS8_PARTITION_MODE:
> -		return sysfs_emit(buf, "NPS8\n");
> -	default:
> +	if ((mode > ARRAY_SIZE(nps_desc)) ||
> +	    (BIT(mode) & AMDGPU_ALL_NPS_MASK) != BIT(mode))
>   		return sysfs_emit(buf, "UNKNOWN\n");
> -	}
> +
> +	return sysfs_emit(buf, "%s\n", nps_desc[mode]);
>   }
>   
> -static DEVICE_ATTR_RO(current_memory_partition);
> +static DEVICE_ATTR_RW(current_memory_partition);
> +static DEVICE_ATTR_RO(available_memory_partition);
>   
>   int amdgpu_gmc_sysfs_init(struct amdgpu_device *adev)
>   {
> +	bool nps_switch_support;
> +	int r = 0;
> +
>   	if (!adev->gmc.gmc_funcs->query_mem_partition_mode)
>   		return 0;
>   
> +	nps_switch_support = (hweight32(adev->gmc.supported_nps_modes &
> +					AMDGPU_ALL_NPS_MASK) > 1);
> +	if (!nps_switch_support)
> +		dev_attr_current_memory_partition.attr.mode &=
> +			~(S_IWUSR | S_IWGRP | S_IWOTH);
> +	else
> +		r = device_create_file(adev->dev,
> +				       &dev_attr_available_memory_partition);
> +
> +	if (r)
> +		return r;
> +
>   	return device_create_file(adev->dev,
>   				  &dev_attr_current_memory_partition);
>   }
>   
>   void amdgpu_gmc_sysfs_fini(struct amdgpu_device *adev)
>   {
> +	if (!adev->gmc.gmc_funcs->query_mem_partition_mode)
> +		return;
> +
>   	device_remove_file(adev->dev, &dev_attr_current_memory_partition);
> +	device_remove_file(adev->dev, &dev_attr_available_memory_partition);
>   }
>   
>   int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> index f5be5112b742..1a50639a003a 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> @@ -73,6 +73,11 @@ enum amdgpu_memory_partition {
>   	AMDGPU_NPS8_PARTITION_MODE = 8,
>   };
>   
> +#define AMDGPU_ALL_NPS_MASK                                                  \
> +	(BIT(AMDGPU_NPS1_PARTITION_MODE) | BIT(AMDGPU_NPS2_PARTITION_MODE) | \
> +	 BIT(AMDGPU_NPS3_PARTITION_MODE) | BIT(AMDGPU_NPS4_PARTITION_MODE) | \
> +	 BIT(AMDGPU_NPS6_PARTITION_MODE) | BIT(AMDGPU_NPS8_PARTITION_MODE))
> +
>   /*
>    * GMC page fault information
>    */
> @@ -308,6 +313,7 @@ struct amdgpu_gmc {
>   	uint8_t num_mem_partitions;
>   	const struct amdgpu_gmc_funcs	*gmc_funcs;
>   	enum amdgpu_memory_partition	requested_nps_mode;
> +	uint32_t supported_nps_modes;
>   
>   	struct amdgpu_xgmi xgmi;
>   	struct amdgpu_irq_src	ecc_irq;

[PATCH 5/7] drm/amdgpu: Place NPS mode request on unload

[Lijo Lazar]

If a user has requested NPS mode switch, place the request through PSP
during unload of the driver. For devices which are part of a hive, all
requests are placed together. If one of them fails, revert back to the
current NPS mode.

Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c  |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c  | 47 ++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h  |  2 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 38 +++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h |  4 ++
 5 files changed, 92 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 95331294509c..d16cdcdb2114 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -2428,6 +2428,7 @@ amdgpu_pci_remove(struct pci_dev *pdev)
 	struct amdgpu_device *adev = drm_to_adev(dev);
 
 	amdgpu_xcp_dev_unplug(adev);
+	amdgpu_gmc_prepare_nps_mode_change(adev);
 	drm_dev_unplug(dev);
 
 	if (adev->pm.rpm_mode != AMDGPU_RUNPM_NONE) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 24a1f931d9ed..21f1e65c9dc9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -1345,3 +1345,50 @@ int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev,
 
 	return psp_memory_partition(&adev->psp, nps_mode);
 }
+
+static inline bool amdgpu_gmc_need_nps_switch_req(struct amdgpu_device *adev,
+						  int req_nps_mode,
+						  int cur_nps_mode)
+{
+	return (((BIT(req_nps_mode) & adev->gmc.supported_nps_modes) ==
+			BIT(req_nps_mode)) &&
+		req_nps_mode != cur_nps_mode);
+}
+
+void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev)
+{
+	int req_nps_mode, cur_nps_mode, r;
+	struct amdgpu_hive_info *hive;
+
+	if (amdgpu_sriov_vf(adev) || !adev->gmc.supported_nps_modes ||
+	    !adev->gmc.gmc_funcs->request_mem_partition_mode)
+		return;
+
+	cur_nps_mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
+	hive = amdgpu_get_xgmi_hive(adev);
+	if (hive) {
+		req_nps_mode = atomic_read(&hive->requested_nps_mode);
+		if (!amdgpu_gmc_need_nps_switch_req(adev, req_nps_mode,
+						    cur_nps_mode)) {
+			amdgpu_put_xgmi_hive(hive);
+			return;
+		}
+		r = amdgpu_xgmi_request_nps_change(adev, hive, req_nps_mode);
+		amdgpu_put_xgmi_hive(hive);
+		goto out;
+	}
+
+	req_nps_mode = adev->gmc.requested_nps_mode;
+	if (!amdgpu_gmc_need_nps_switch_req(adev, req_nps_mode, cur_nps_mode))
+		return;
+
+	/* even if this fails, we should let driver unload w/o blocking */
+	r = adev->gmc.gmc_funcs->request_mem_partition_mode(adev, req_nps_mode);
+out:
+	if (r)
+		dev_err(adev->dev, "NPS mode change request failed\n");
+	else
+		dev_info(
+			adev->dev,
+			"NPS mode change request done, reload driver to complete the change\n");
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index 1a50639a003a..b13d6adb5efd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -467,4 +467,6 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
 
 int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev,
 					int nps_mode);
+void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index 5d721ccb9dfd..db2c1b11b813 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
@@ -1564,3 +1564,41 @@ int amdgpu_xgmi_reset_on_init(struct amdgpu_device *adev)
 
 	return 0;
 }
+
+int amdgpu_xgmi_request_nps_change(struct amdgpu_device *adev,
+				   struct amdgpu_hive_info *hive,
+				   int req_nps_mode)
+{
+	struct amdgpu_device *tmp_adev;
+	int cur_nps_mode, r;
+
+	/* This is expected to be called only during unload of driver. The
+	 * request needs to be placed only once for all devices in the hive. If
+	 * one of them fail, revert the request for previous successful devices.
+	 * After placing the request, make hive mode as UNKNOWN so that other
+	 * devices don't request anymore.
+	 */
+	mutex_lock(&hive->hive_lock);
+	list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
+		r = adev->gmc.gmc_funcs->request_mem_partition_mode(
+			tmp_adev, req_nps_mode);
+		if (r)
+			goto err;
+	}
+	/* Set to UNKNOWN so that other devices don't request anymore */
+	atomic_set(&hive->requested_nps_mode, UNKNOWN_MEMORY_PARTITION_MODE);
+
+	mutex_unlock(&hive->hive_lock);
+
+	return 0;
+err:
+	/* Request back current mode if one of the requests failed */
+	cur_nps_mode = adev->gmc.gmc_funcs->query_mem_partition_mode(tmp_adev);
+	list_for_each_entry_continue_reverse(tmp_adev, &hive->device_list,
+					     gmc.xgmi.head)
+		adev->gmc.gmc_funcs->request_mem_partition_mode(tmp_adev,
+								cur_nps_mode);
+	mutex_lock(&hive->hive_lock);
+
+	return r;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
index 67abadb4f298..41d5f97fc77a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
@@ -79,4 +79,8 @@ static inline bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
 int amdgpu_xgmi_ras_sw_init(struct amdgpu_device *adev);
 int amdgpu_xgmi_reset_on_init(struct amdgpu_device *adev);
 
+int amdgpu_xgmi_request_nps_change(struct amdgpu_device *adev,
+				   struct amdgpu_hive_info *hive,
+				   int req_nps_mode);
+
 #endif
-- 
2.25.1

RE: [PATCH 5/7] drm/amdgpu: Place NPS mode request on unload

[Xu, Feifei]

[AMD Official Use Only - AMD Internal Distribution Only]

Reviewed-by: Feifei Xu <Feifei.Xu@amd.com>

-----Original Message-----
From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Lijo Lazar
Sent: Tuesday, September 24, 2024 1:57 PM
To: amd-gfx@lists.freedesktop.org
Cc: Zhang, Hawking <Hawking.Zhang@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Koenig, Christian <Christian.Koenig@amd.com>; Bhardwaj, Rajneesh <Rajneesh.Bhardwaj@amd.com>; Errabolu, Ramesh <Ramesh.Errabolu@amd.com>
Subject: [PATCH 5/7] drm/amdgpu: Place NPS mode request on unload

If a user has requested NPS mode switch, place the request through PSP during unload of the driver. For devices which are part of a hive, all requests are placed together. If one of them fails, revert back to the current NPS mode.

Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c  |  1 +  drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c  | 47 ++++++++++++++++++++++++  drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h  |  2 +  drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 38 +++++++++++++++++++  drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h |  4 ++
 5 files changed, 92 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 95331294509c..d16cdcdb2114 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -2428,6 +2428,7 @@ amdgpu_pci_remove(struct pci_dev *pdev)
        struct amdgpu_device *adev = drm_to_adev(dev);

        amdgpu_xcp_dev_unplug(adev);
+       amdgpu_gmc_prepare_nps_mode_change(adev);
        drm_dev_unplug(dev);

        if (adev->pm.rpm_mode != AMDGPU_RUNPM_NONE) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 24a1f931d9ed..21f1e65c9dc9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -1345,3 +1345,50 @@ int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev,

        return psp_memory_partition(&adev->psp, nps_mode);  }
+
+static inline bool amdgpu_gmc_need_nps_switch_req(struct amdgpu_device *adev,
+                                                 int req_nps_mode,
+                                                 int cur_nps_mode)
+{
+       return (((BIT(req_nps_mode) & adev->gmc.supported_nps_modes) ==
+                       BIT(req_nps_mode)) &&
+               req_nps_mode != cur_nps_mode);
+}
+
+void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev) {
+       int req_nps_mode, cur_nps_mode, r;
+       struct amdgpu_hive_info *hive;
+
+       if (amdgpu_sriov_vf(adev) || !adev->gmc.supported_nps_modes ||
+           !adev->gmc.gmc_funcs->request_mem_partition_mode)
+               return;
+
+       cur_nps_mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
+       hive = amdgpu_get_xgmi_hive(adev);
+       if (hive) {
+               req_nps_mode = atomic_read(&hive->requested_nps_mode);
+               if (!amdgpu_gmc_need_nps_switch_req(adev, req_nps_mode,
+                                                   cur_nps_mode)) {
+                       amdgpu_put_xgmi_hive(hive);
+                       return;
+               }
+               r = amdgpu_xgmi_request_nps_change(adev, hive, req_nps_mode);
+               amdgpu_put_xgmi_hive(hive);
+               goto out;
+       }
+
+       req_nps_mode = adev->gmc.requested_nps_mode;
+       if (!amdgpu_gmc_need_nps_switch_req(adev, req_nps_mode, cur_nps_mode))
+               return;
+
+       /* even if this fails, we should let driver unload w/o blocking */
+       r = adev->gmc.gmc_funcs->request_mem_partition_mode(adev,
+req_nps_mode);
+out:
+       if (r)
+               dev_err(adev->dev, "NPS mode change request failed\n");
+       else
+               dev_info(
+                       adev->dev,
+                       "NPS mode change request done, reload driver to complete the
+change\n"); }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index 1a50639a003a..b13d6adb5efd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -467,4 +467,6 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,

 int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev,
                                        int nps_mode);
+void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index 5d721ccb9dfd..db2c1b11b813 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
@@ -1564,3 +1564,41 @@ int amdgpu_xgmi_reset_on_init(struct amdgpu_device *adev)

        return 0;
 }
+
+int amdgpu_xgmi_request_nps_change(struct amdgpu_device *adev,
+                                  struct amdgpu_hive_info *hive,
+                                  int req_nps_mode)
+{
+       struct amdgpu_device *tmp_adev;
+       int cur_nps_mode, r;
+
+       /* This is expected to be called only during unload of driver. The
+        * request needs to be placed only once for all devices in the hive. If
+        * one of them fail, revert the request for previous successful devices.
+        * After placing the request, make hive mode as UNKNOWN so that other
+        * devices don't request anymore.
+        */
+       mutex_lock(&hive->hive_lock);
+       list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
+               r = adev->gmc.gmc_funcs->request_mem_partition_mode(
+                       tmp_adev, req_nps_mode);
+               if (r)
+                       goto err;
+       }
+       /* Set to UNKNOWN so that other devices don't request anymore */
+       atomic_set(&hive->requested_nps_mode, UNKNOWN_MEMORY_PARTITION_MODE);
+
+       mutex_unlock(&hive->hive_lock);
+
+       return 0;
+err:
+       /* Request back current mode if one of the requests failed */
+       cur_nps_mode = adev->gmc.gmc_funcs->query_mem_partition_mode(tmp_adev);
+       list_for_each_entry_continue_reverse(tmp_adev, &hive->device_list,
+                                            gmc.xgmi.head)
+               adev->gmc.gmc_funcs->request_mem_partition_mode(tmp_adev,
+                                                               cur_nps_mode);
+       mutex_lock(&hive->hive_lock);
+
+       return r;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
index 67abadb4f298..41d5f97fc77a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
@@ -79,4 +79,8 @@ static inline bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,  int amdgpu_xgmi_ras_sw_init(struct amdgpu_device *adev);  int amdgpu_xgmi_reset_on_init(struct amdgpu_device *adev);

+int amdgpu_xgmi_request_nps_change(struct amdgpu_device *adev,
+                                  struct amdgpu_hive_info *hive,
+                                  int req_nps_mode);
+
 #endif
--
2.25.1

[PATCH 6/7] drm/amdgpu: Check gmc requirement for reset on init

[Lijo Lazar]

Add a callback to check if there is any condition detected by GMC block
for reset on init. One case is if a pending NPS change request is
detected. If reset is done because of NPS switch, refresh NPS info from
discovery table.

Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 13 ++++++++++++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  5 +++++
 drivers/gpu/drm/amd/amdgpu/soc15.c      |  2 ++
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 21f1e65c9dc9..011fe3a847d0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -1261,12 +1261,15 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
 	struct amdgpu_gmc_memrange *ranges;
 	int range_cnt, ret, i, j;
 	uint32_t nps_type;
+	bool refresh;
 
 	if (!mem_ranges)
 		return -EINVAL;
 
+	refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) &&
+		  (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS);
 	ret = amdgpu_discovery_get_nps_info(adev, &nps_type, &ranges,
-					    &range_cnt, false);
+					    &range_cnt, refresh);
 
 	if (ret)
 		return ret;
@@ -1392,3 +1395,11 @@ void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev)
 			adev->dev,
 			"NPS mode change request done, reload driver to complete the change\n");
 }
+
+bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev)
+{
+	if (adev->gmc.gmc_funcs->need_reset_on_init)
+		return adev->gmc.gmc_funcs->need_reset_on_init(adev);
+
+	return false;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index b13d6adb5efd..d4cd247fe574 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -78,6 +78,8 @@ enum amdgpu_memory_partition {
 	 BIT(AMDGPU_NPS3_PARTITION_MODE) | BIT(AMDGPU_NPS4_PARTITION_MODE) | \
 	 BIT(AMDGPU_NPS6_PARTITION_MODE) | BIT(AMDGPU_NPS8_PARTITION_MODE))
 
+#define AMDGPU_GMC_INIT_RESET_NPS  BIT(0)
+
 /*
  * GMC page fault information
  */
@@ -169,6 +171,7 @@ struct amdgpu_gmc_funcs {
 	/* Request NPS mode */
 	int (*request_mem_partition_mode)(struct amdgpu_device *adev,
 					  int nps_mode);
+	bool (*need_reset_on_init)(struct amdgpu_device *adev);
 };
 
 struct amdgpu_xgmi_ras {
@@ -314,6 +317,7 @@ struct amdgpu_gmc {
 	const struct amdgpu_gmc_funcs	*gmc_funcs;
 	enum amdgpu_memory_partition	requested_nps_mode;
 	uint32_t supported_nps_modes;
+	uint32_t reset_flags;
 
 	struct amdgpu_xgmi xgmi;
 	struct amdgpu_irq_src	ecc_irq;
@@ -468,5 +472,6 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
 int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev,
 					int nps_mode);
 void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev);
+bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 619933f252aa..97ca4931a7ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -833,6 +833,8 @@ static bool soc15_need_reset_on_init(struct amdgpu_device *adev)
 
 	if (amdgpu_psp_tos_reload_needed(adev))
 		return true;
+	if (amdgpu_gmc_need_reset_on_init(adev))
+		return true;
 	/* Just return false for soc15 GPUs.  Reset does not seem to
 	 * be necessary.
 	 */
-- 
2.25.1

Re: [PATCH 6/7] drm/amdgpu: Check gmc requirement for reset on init

[Bhardwaj, Rajneesh]

Reviewed-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>

On 9/24/2024 1:56 AM, Lijo Lazar wrote:
> Add a callback to check if there is any condition detected by GMC block
> for reset on init. One case is if a pending NPS change request is
> detected. If reset is done because of NPS switch, refresh NPS info from
> discovery table.
>
> Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 13 ++++++++++++-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  5 +++++
>   drivers/gpu/drm/amd/amdgpu/soc15.c      |  2 ++
>   3 files changed, 19 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> index 21f1e65c9dc9..011fe3a847d0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> @@ -1261,12 +1261,15 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
>   	struct amdgpu_gmc_memrange *ranges;
>   	int range_cnt, ret, i, j;
>   	uint32_t nps_type;
> +	bool refresh;
>   
>   	if (!mem_ranges)
>   		return -EINVAL;
>   
> +	refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) &&
> +		  (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS);
>   	ret = amdgpu_discovery_get_nps_info(adev, &nps_type, &ranges,
> -					    &range_cnt, false);
> +					    &range_cnt, refresh);
>   
>   	if (ret)
>   		return ret;
> @@ -1392,3 +1395,11 @@ void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev)
>   			adev->dev,
>   			"NPS mode change request done, reload driver to complete the change\n");
>   }
> +
> +bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev)
> +{
> +	if (adev->gmc.gmc_funcs->need_reset_on_init)
> +		return adev->gmc.gmc_funcs->need_reset_on_init(adev);
> +
> +	return false;
> +}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> index b13d6adb5efd..d4cd247fe574 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> @@ -78,6 +78,8 @@ enum amdgpu_memory_partition {
>   	 BIT(AMDGPU_NPS3_PARTITION_MODE) | BIT(AMDGPU_NPS4_PARTITION_MODE) | \
>   	 BIT(AMDGPU_NPS6_PARTITION_MODE) | BIT(AMDGPU_NPS8_PARTITION_MODE))
>   
> +#define AMDGPU_GMC_INIT_RESET_NPS  BIT(0)
> +
>   /*
>    * GMC page fault information
>    */
> @@ -169,6 +171,7 @@ struct amdgpu_gmc_funcs {
>   	/* Request NPS mode */
>   	int (*request_mem_partition_mode)(struct amdgpu_device *adev,
>   					  int nps_mode);
> +	bool (*need_reset_on_init)(struct amdgpu_device *adev);
>   };
>   
>   struct amdgpu_xgmi_ras {
> @@ -314,6 +317,7 @@ struct amdgpu_gmc {
>   	const struct amdgpu_gmc_funcs	*gmc_funcs;
>   	enum amdgpu_memory_partition	requested_nps_mode;
>   	uint32_t supported_nps_modes;
> +	uint32_t reset_flags;
>   
>   	struct amdgpu_xgmi xgmi;
>   	struct amdgpu_irq_src	ecc_irq;
> @@ -468,5 +472,6 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
>   int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev,
>   					int nps_mode);
>   void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev);
> +bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev);
>   
>   #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
> index 619933f252aa..97ca4931a7ef 100644
> --- a/drivers/gpu/drm/amd/amdgpu/soc15.c
> +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
> @@ -833,6 +833,8 @@ static bool soc15_need_reset_on_init(struct amdgpu_device *adev)
>   
>   	if (amdgpu_psp_tos_reload_needed(adev))
>   		return true;
> +	if (amdgpu_gmc_need_reset_on_init(adev))
> +		return true;
>   	/* Just return false for soc15 GPUs.  Reset does not seem to
>   	 * be necessary.
>   	 */

RE: [PATCH 6/7] drm/amdgpu: Check gmc requirement for reset on init

[Xu, Feifei]

[AMD Official Use Only - AMD Internal Distribution Only]

>>+     refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) &&
>>+               (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS);

Is there a corner case that reloading with a different version tos and refreshing nps change co-exist?

Thanks,
Feifei

-----Original Message-----
From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Lijo Lazar
Sent: Tuesday, September 24, 2024 1:57 PM
To: amd-gfx@lists.freedesktop.org
Cc: Zhang, Hawking <Hawking.Zhang@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Koenig, Christian <Christian.Koenig@amd.com>; Bhardwaj, Rajneesh <Rajneesh.Bhardwaj@amd.com>; Errabolu, Ramesh <Ramesh.Errabolu@amd.com>
Subject: [PATCH 6/7] drm/amdgpu: Check gmc requirement for reset on init

Add a callback to check if there is any condition detected by GMC block for reset on init. One case is if a pending NPS change request is detected. If reset is done because of NPS switch, refresh NPS info from discovery table.

Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 13 ++++++++++++-  drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  5 +++++
 drivers/gpu/drm/amd/amdgpu/soc15.c      |  2 ++
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 21f1e65c9dc9..011fe3a847d0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -1261,12 +1261,15 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
        struct amdgpu_gmc_memrange *ranges;
        int range_cnt, ret, i, j;
        uint32_t nps_type;
+       bool refresh;

        if (!mem_ranges)
                return -EINVAL;

+       refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) &&
+                 (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS);
        ret = amdgpu_discovery_get_nps_info(adev, &nps_type, &ranges,
-                                           &range_cnt, false);
+                                           &range_cnt, refresh);

        if (ret)
                return ret;
@@ -1392,3 +1395,11 @@ void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev)
                        adev->dev,
                        "NPS mode change request done, reload driver to complete the change\n");  }
+
+bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev) {
+       if (adev->gmc.gmc_funcs->need_reset_on_init)
+               return adev->gmc.gmc_funcs->need_reset_on_init(adev);
+
+       return false;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index b13d6adb5efd..d4cd247fe574 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -78,6 +78,8 @@ enum amdgpu_memory_partition {
         BIT(AMDGPU_NPS3_PARTITION_MODE) | BIT(AMDGPU_NPS4_PARTITION_MODE) | \
         BIT(AMDGPU_NPS6_PARTITION_MODE) | BIT(AMDGPU_NPS8_PARTITION_MODE))

+#define AMDGPU_GMC_INIT_RESET_NPS  BIT(0)
+
 /*
  * GMC page fault information
  */
@@ -169,6 +171,7 @@ struct amdgpu_gmc_funcs {
        /* Request NPS mode */
        int (*request_mem_partition_mode)(struct amdgpu_device *adev,
                                          int nps_mode);
+       bool (*need_reset_on_init)(struct amdgpu_device *adev);
 };

 struct amdgpu_xgmi_ras {
@@ -314,6 +317,7 @@ struct amdgpu_gmc {
        const struct amdgpu_gmc_funcs   *gmc_funcs;
        enum amdgpu_memory_partition    requested_nps_mode;
        uint32_t supported_nps_modes;
+       uint32_t reset_flags;

        struct amdgpu_xgmi xgmi;
        struct amdgpu_irq_src   ecc_irq;
@@ -468,5 +472,6 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,  int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev,
                                        int nps_mode);
 void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev);
+bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev);

 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 619933f252aa..97ca4931a7ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -833,6 +833,8 @@ static bool soc15_need_reset_on_init(struct amdgpu_device *adev)

        if (amdgpu_psp_tos_reload_needed(adev))
                return true;
+       if (amdgpu_gmc_need_reset_on_init(adev))
+               return true;
        /* Just return false for soc15 GPUs.  Reset does not seem to
         * be necessary.
         */
--
2.25.1

Re: [PATCH 6/7] drm/amdgpu: Check gmc requirement for reset on init

[Lazar, Lijo]

On 9/26/2024 2:31 PM, Xu, Feifei wrote:
> [AMD Official Use Only - AMD Internal Distribution Only]
> 
>>> +     refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) &&
>>> +               (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS);
> 
> Is there a corner case that reloading with a different version tos and refreshing nps change co-exist?
> 

I guess you are referring to the below corner case
	1) Place NPS request
	2) Unload Driver
	3) Reinstall driver with a different TOS (possible but quite unlikely)
	4) Driver reload
	5) Driver checks TOS version first and goes for a reset
	6) reset_flag of GMC is not set, hence it doesn't refresh the NPS range.


I think changing the order in soc15_need_reset_on_init() to check for
NPS request before TOS version check will solve this.

Thanks,
Lijo

> Thanks,
> Feifei
> 
> -----Original Message-----
> From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Lijo Lazar
> Sent: Tuesday, September 24, 2024 1:57 PM
> To: amd-gfx@lists.freedesktop.org
> Cc: Zhang, Hawking <Hawking.Zhang@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Koenig, Christian <Christian.Koenig@amd.com>; Bhardwaj, Rajneesh <Rajneesh.Bhardwaj@amd.com>; Errabolu, Ramesh <Ramesh.Errabolu@amd.com>
> Subject: [PATCH 6/7] drm/amdgpu: Check gmc requirement for reset on init
> 
> Add a callback to check if there is any condition detected by GMC block for reset on init. One case is if a pending NPS change request is detected. If reset is done because of NPS switch, refresh NPS info from discovery table.
> 
> Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 13 ++++++++++++-  drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  5 +++++
>  drivers/gpu/drm/amd/amdgpu/soc15.c      |  2 ++
>  3 files changed, 19 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> index 21f1e65c9dc9..011fe3a847d0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> @@ -1261,12 +1261,15 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
>         struct amdgpu_gmc_memrange *ranges;
>         int range_cnt, ret, i, j;
>         uint32_t nps_type;
> +       bool refresh;
> 
>         if (!mem_ranges)
>                 return -EINVAL;
> 
> +       refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) &&
> +                 (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS);
>         ret = amdgpu_discovery_get_nps_info(adev, &nps_type, &ranges,
> -                                           &range_cnt, false);
> +                                           &range_cnt, refresh);
> 
>         if (ret)
>                 return ret;
> @@ -1392,3 +1395,11 @@ void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev)
>                         adev->dev,
>                         "NPS mode change request done, reload driver to complete the change\n");  }
> +
> +bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev) {
> +       if (adev->gmc.gmc_funcs->need_reset_on_init)
> +               return adev->gmc.gmc_funcs->need_reset_on_init(adev);
> +
> +       return false;
> +}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> index b13d6adb5efd..d4cd247fe574 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> @@ -78,6 +78,8 @@ enum amdgpu_memory_partition {
>          BIT(AMDGPU_NPS3_PARTITION_MODE) | BIT(AMDGPU_NPS4_PARTITION_MODE) | \
>          BIT(AMDGPU_NPS6_PARTITION_MODE) | BIT(AMDGPU_NPS8_PARTITION_MODE))
> 
> +#define AMDGPU_GMC_INIT_RESET_NPS  BIT(0)
> +
>  /*
>   * GMC page fault information
>   */
> @@ -169,6 +171,7 @@ struct amdgpu_gmc_funcs {
>         /* Request NPS mode */
>         int (*request_mem_partition_mode)(struct amdgpu_device *adev,
>                                           int nps_mode);
> +       bool (*need_reset_on_init)(struct amdgpu_device *adev);
>  };
> 
>  struct amdgpu_xgmi_ras {
> @@ -314,6 +317,7 @@ struct amdgpu_gmc {
>         const struct amdgpu_gmc_funcs   *gmc_funcs;
>         enum amdgpu_memory_partition    requested_nps_mode;
>         uint32_t supported_nps_modes;
> +       uint32_t reset_flags;
> 
>         struct amdgpu_xgmi xgmi;
>         struct amdgpu_irq_src   ecc_irq;
> @@ -468,5 +472,6 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,  int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev,
>                                         int nps_mode);
>  void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev);
> +bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev);
> 
>  #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
> index 619933f252aa..97ca4931a7ef 100644
> --- a/drivers/gpu/drm/amd/amdgpu/soc15.c
> +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
> @@ -833,6 +833,8 @@ static bool soc15_need_reset_on_init(struct amdgpu_device *adev)
> 
>         if (amdgpu_psp_tos_reload_needed(adev))
>                 return true;
> +       if (amdgpu_gmc_need_reset_on_init(adev))
> +               return true;
>         /* Just return false for soc15 GPUs.  Reset does not seem to
>          * be necessary.
>          */
> --
> 2.25.1
> 

Re: [PATCH 6/7] drm/amdgpu: Check gmc requirement for reset on init

[Xu, Feifei]

[-- Attachment #1: Type: text/plain, Size: 6164 bytes --]

 >>I guess you are referring to the below corner case

>>  1) Place NPS request

>>  2) Unload Driver

>>  3) Reinstall driver with a different TOS (possible but quite unlikely)

>> 4) Driver reload

>> 5) Driver checks TOS version first and goes for a reset

>> 6) reset_flag of GMC is not set, hence it doesn't refresh the NPS range.

 >>I think changing the order in soc15_need_reset_on_init() to check for 
NPS request before TOS version check will solve this.

Yes, I was thinking of  reset_flag and tOS reloading 
(adev->init_lvl->level set to AMDGPU_INIT_LEVEL_MINIMAL_XGMI) changing 
at the same time. And NPS refresh will be ignored. Though might be 
likely in debugging or regression isolation cases which changing driver 
packaged with different TOS.  And yes making NPS refresh checking before 
TOS version checking will solve this.

And if we do not return ahead when checking NPS request before tOS 
version change in soc15_need_reset_on_init(), we can drop 
(adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) check in below 
refresh checking:

+     refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) &&
+               (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS);

refresh = (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS);

Thanks
Feifei

On 9/26/2024 5:01 PM, Xu, Feifei wrote:
>>> +     refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) &&
>>> +               (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS);
> Is there a corner case that reloading with a different version tos and refreshing nps change co-exist?
>
> Thanks,
> Feifei
>
> -----Original Message-----
> From: amd-gfx<amd-gfx-bounces@lists.freedesktop.org>  On Behalf Of Lijo Lazar
> Sent: Tuesday, September 24, 2024 1:57 PM
> To:amd-gfx@lists.freedesktop.org
> Cc: Zhang, Hawking<Hawking.Zhang@amd.com>; Deucher, Alexander<Alexander.Deucher@amd.com>; Koenig, Christian<Christian.Koenig@amd.com>; Bhardwaj, Rajneesh<Rajneesh.Bhardwaj@amd.com>; Errabolu, Ramesh<Ramesh.Errabolu@amd.com>
> Subject: [PATCH 6/7] drm/amdgpu: Check gmc requirement for reset on init
>
> Add a callback to check if there is any condition detected by GMC block for reset on init. One case is if a pending NPS change request is detected. If reset is done because of NPS switch, refresh NPS info from discovery table.
>
> Signed-off-by: Lijo Lazar<lijo.lazar@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 13 ++++++++++++-  drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  5 +++++
>   drivers/gpu/drm/amd/amdgpu/soc15.c      |  2 ++
>   3 files changed, 19 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> index 21f1e65c9dc9..011fe3a847d0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> @@ -1261,12 +1261,15 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
>          struct amdgpu_gmc_memrange *ranges;
>          int range_cnt, ret, i, j;
>          uint32_t nps_type;
> +       bool refresh;
>
>          if (!mem_ranges)
>                  return -EINVAL;
>
> +       refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) &&
> +                 (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS);
>          ret = amdgpu_discovery_get_nps_info(adev, &nps_type, &ranges,
> -                                           &range_cnt, false);
> +                                           &range_cnt, refresh);
>
>          if (ret)
>                  return ret;
> @@ -1392,3 +1395,11 @@ void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev)
>                          adev->dev,
>                          "NPS mode change request done, reload driver to complete the change\n");  }
> +
> +bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev) {
> +       if (adev->gmc.gmc_funcs->need_reset_on_init)
> +               return adev->gmc.gmc_funcs->need_reset_on_init(adev);
> +
> +       return false;
> +}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> index b13d6adb5efd..d4cd247fe574 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> @@ -78,6 +78,8 @@ enum amdgpu_memory_partition {
>           BIT(AMDGPU_NPS3_PARTITION_MODE) | BIT(AMDGPU_NPS4_PARTITION_MODE) | \
>           BIT(AMDGPU_NPS6_PARTITION_MODE) | BIT(AMDGPU_NPS8_PARTITION_MODE))
>
> +#define AMDGPU_GMC_INIT_RESET_NPS  BIT(0)
> +
>   /*
>    * GMC page fault information
>    */
> @@ -169,6 +171,7 @@ struct amdgpu_gmc_funcs {
>          /* Request NPS mode */
>          int (*request_mem_partition_mode)(struct amdgpu_device *adev,
>                                            int nps_mode);
> +       bool (*need_reset_on_init)(struct amdgpu_device *adev);
>   };
>
>   struct amdgpu_xgmi_ras {
> @@ -314,6 +317,7 @@ struct amdgpu_gmc {
>          const struct amdgpu_gmc_funcs   *gmc_funcs;
>          enum amdgpu_memory_partition    requested_nps_mode;
>          uint32_t supported_nps_modes;
> +       uint32_t reset_flags;
>
>          struct amdgpu_xgmi xgmi;
>          struct amdgpu_irq_src   ecc_irq;
> @@ -468,5 +472,6 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,  int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev,
>                                          int nps_mode);
>   void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev);
> +bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev);
>
>   #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
> index 619933f252aa..97ca4931a7ef 100644
> --- a/drivers/gpu/drm/amd/amdgpu/soc15.c
> +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
> @@ -833,6 +833,8 @@ static bool soc15_need_reset_on_init(struct amdgpu_device *adev)
>
>          if (amdgpu_psp_tos_reload_needed(adev))
>                  return true;
> +       if (amdgpu_gmc_need_reset_on_init(adev))
> +               return true;
>          /* Just return false for soc15 GPUs.  Reset does not seem to
>           * be necessary.
>           */
> --
> 2.25.1
>

[-- Attachment #2: Type: text/html, Size: 44377 bytes --]

Re: [PATCH 6/7] drm/amdgpu: Check gmc requirement for reset on init

[Lazar, Lijo]

On 9/27/2024 9:26 AM, Xu, Feifei wrote:
>>>I guess you are referring to the below corner case
> 
>>>            1) Place NPS request
> 
>>>         2) Unload Driver
> 
>>>            3) Reinstall driver with a different TOS (possible but quite
> unlikely)
> 
>>>             4) Driver reload
> 
>>>             5) Driver checks TOS version first and goes for a reset
> 
>>>             6) reset_flag of GMC is not set, hence it doesn't refresh
> the NPS range.
> 
>  
> 
>  
> 
>>>I think changing the order in soc15_need_reset_on_init() to check for
> NPS request before TOS version check will solve this.
> 
> Yes, I was thinking of  reset_flag and tOS reloading
> (adev->init_lvl->level set to AMDGPU_INIT_LEVEL_MINIMAL_XGMI) changing
> at the same time. And NPS refresh will be ignored. Though might be
> likely in debugging or regression isolation cases which changing driver
> packaged with different TOS.  And yes making NPS refresh checking before
> TOS version checking will solve this.
> 
> And if we do not return ahead when checking NPS request before tOS
> version change in soc15_need_reset_on_init(), we can drop 
> (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) check in below
> refresh checking:
> 

This check is used when NPS request change is identified. During swinit
part it will be at MINIMAL_XGMI level, but at that point there is no
need to refresh this information as reset is pending. It is refreshed
after a reset when init level returns to default.

Thanks,
Lijo

> +     refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) &&
> +               (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS);
> 
> refresh = (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS);
> 
> Thanks
> Feifei
> 
> On 9/26/2024 5:01 PM, Xu, Feifei wrote:
>>>> +     refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) &&
>>>> +               (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS);
>> Is there a corner case that reloading with a different version tos and refreshing nps change co-exist?
>>
>> Thanks,
>> Feifei
>>
>> -----Original Message-----
>> From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Lijo Lazar
>> Sent: Tuesday, September 24, 2024 1:57 PM
>> To: amd-gfx@lists.freedesktop.org
>> Cc: Zhang, Hawking <Hawking.Zhang@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Koenig, Christian <Christian.Koenig@amd.com>; Bhardwaj, Rajneesh <Rajneesh.Bhardwaj@amd.com>; Errabolu, Ramesh <Ramesh.Errabolu@amd.com>
>> Subject: [PATCH 6/7] drm/amdgpu: Check gmc requirement for reset on init
>>
>> Add a callback to check if there is any condition detected by GMC block for reset on init. One case is if a pending NPS change request is detected. If reset is done because of NPS switch, refresh NPS info from discovery table.
>>
>> Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
>> ---
>>  drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 13 ++++++++++++-  drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  5 +++++
>>  drivers/gpu/drm/amd/amdgpu/soc15.c      |  2 ++
>>  3 files changed, 19 insertions(+), 1 deletion(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
>> index 21f1e65c9dc9..011fe3a847d0 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
>> @@ -1261,12 +1261,15 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
>>         struct amdgpu_gmc_memrange *ranges;
>>         int range_cnt, ret, i, j;
>>         uint32_t nps_type;
>> +       bool refresh;
>>
>>         if (!mem_ranges)
>>                 return -EINVAL;
>>
>> +       refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) &&
>> +                 (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS);
>>         ret = amdgpu_discovery_get_nps_info(adev, &nps_type, &ranges,
>> -                                           &range_cnt, false);
>> +                                           &range_cnt, refresh);
>>
>>         if (ret)
>>                 return ret;
>> @@ -1392,3 +1395,11 @@ void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev)
>>                         adev->dev,
>>                         "NPS mode change request done, reload driver to complete the change\n");  }
>> +
>> +bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev) {
>> +       if (adev->gmc.gmc_funcs->need_reset_on_init)
>> +               return adev->gmc.gmc_funcs->need_reset_on_init(adev);
>> +
>> +       return false;
>> +}
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
>> index b13d6adb5efd..d4cd247fe574 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
>> @@ -78,6 +78,8 @@ enum amdgpu_memory_partition {
>>          BIT(AMDGPU_NPS3_PARTITION_MODE) | BIT(AMDGPU_NPS4_PARTITION_MODE) | \
>>          BIT(AMDGPU_NPS6_PARTITION_MODE) | BIT(AMDGPU_NPS8_PARTITION_MODE))
>>
>> +#define AMDGPU_GMC_INIT_RESET_NPS  BIT(0)
>> +
>>  /*
>>   * GMC page fault information
>>   */
>> @@ -169,6 +171,7 @@ struct amdgpu_gmc_funcs {
>>         /* Request NPS mode */
>>         int (*request_mem_partition_mode)(struct amdgpu_device *adev,
>>                                           int nps_mode);
>> +       bool (*need_reset_on_init)(struct amdgpu_device *adev);
>>  };
>>
>>  struct amdgpu_xgmi_ras {
>> @@ -314,6 +317,7 @@ struct amdgpu_gmc {
>>         const struct amdgpu_gmc_funcs   *gmc_funcs;
>>         enum amdgpu_memory_partition    requested_nps_mode;
>>         uint32_t supported_nps_modes;
>> +       uint32_t reset_flags;
>>
>>         struct amdgpu_xgmi xgmi;
>>         struct amdgpu_irq_src   ecc_irq;
>> @@ -468,5 +472,6 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,  int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev,
>>                                         int nps_mode);
>>  void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev);
>> +bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev);
>>
>>  #endif
>> diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
>> index 619933f252aa..97ca4931a7ef 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/soc15.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
>> @@ -833,6 +833,8 @@ static bool soc15_need_reset_on_init(struct amdgpu_device *adev)
>>
>>         if (amdgpu_psp_tos_reload_needed(adev))
>>                 return true;
>> +       if (amdgpu_gmc_need_reset_on_init(adev))
>> +               return true;
>>         /* Just return false for soc15 GPUs.  Reset does not seem to
>>          * be necessary.
>>          */
>> --
>> 2.25.1
>>

[PATCH 7/7] drm/amdgpu: Add NPS switch support for GC 9.4.3

[Lijo Lazar]

Add dynamic NPS switch support for GC 9.4.3 variants. Only GC v9.4.3 and
GC v9.4.4 currently support this. NPS switch is only supported if an SOC
supports multiple NPS modes.

Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h |  1 +
 drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    | 44 ++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c   | 12 +++++++
 3 files changed, 57 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h
index f61d117b0caf..79c2f807b9fe 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h
@@ -101,6 +101,7 @@ struct amdgpu_nbio_funcs {
 	int (*get_compute_partition_mode)(struct amdgpu_device *adev);
 	u32 (*get_memory_partition_mode)(struct amdgpu_device *adev,
 					 u32 *supp_modes);
+	bool (*is_nps_switch_requested)(struct amdgpu_device *adev);
 	u64 (*get_pcie_replay_count)(struct amdgpu_device *adev);
 	void (*set_reg_remap)(struct amdgpu_device *adev);
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index cafcb24449df..6a95402985ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -1395,6 +1395,17 @@ gmc_v9_0_query_memory_partition(struct amdgpu_device *adev)
 	return gmc_v9_0_get_memory_partition(adev, NULL);
 }
 
+static bool gmc_v9_0_need_reset_on_init(struct amdgpu_device *adev)
+{
+	if (adev->nbio.funcs &&
+	    adev->nbio.funcs->is_nps_switch_requested(adev)) {
+		adev->gmc.reset_flags |= AMDGPU_GMC_INIT_RESET_NPS;
+		return true;
+	}
+
+	return false;
+}
+
 static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
 	.flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
 	.flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid,
@@ -1406,6 +1417,8 @@ static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
 	.override_vm_pte_flags = gmc_v9_0_override_vm_pte_flags,
 	.get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size,
 	.query_mem_partition_mode = &gmc_v9_0_query_memory_partition,
+	.request_mem_partition_mode = &amdgpu_gmc_request_memory_partition,
+	.need_reset_on_init = &gmc_v9_0_need_reset_on_init,
 };
 
 static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
@@ -1545,6 +1558,28 @@ static void gmc_v9_0_set_xgmi_ras_funcs(struct amdgpu_device *adev)
 		adev->gmc.xgmi.ras = &xgmi_ras;
 }
 
+static void gmc_v9_0_init_nps_details(struct amdgpu_device *adev)
+{
+	adev->gmc.supported_nps_modes = 0;
+
+	if (amdgpu_sriov_vf(adev) || (adev->flags & AMD_IS_APU))
+		return;
+
+	/*TODO: Check PSP version also which supports NPS switch. Otherwise keep
+	 * supported modes as 0.
+	 */
+	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
+	case IP_VERSION(9, 4, 3):
+	case IP_VERSION(9, 4, 4):
+		adev->gmc.supported_nps_modes =
+			BIT(AMDGPU_NPS1_PARTITION_MODE) |
+			BIT(AMDGPU_NPS4_PARTITION_MODE);
+		break;
+	default:
+		break;
+	}
+}
+
 static int gmc_v9_0_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -2165,6 +2200,7 @@ static int gmc_v9_0_sw_init(void *handle)
 	if (r)
 		return r;
 
+	gmc_v9_0_init_nps_details(adev);
 	/*
 	 * number of VMs
 	 * VMID 0 is reserved for System
@@ -2440,6 +2476,14 @@ static int gmc_v9_0_resume(void *handle)
 	int r;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	/* If a reset is done for NPS mode switch, read the memory range
+	 * information again.
+	 */
+	if (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS) {
+		gmc_v9_0_init_sw_mem_ranges(adev, adev->gmc.mem_partitions);
+		adev->gmc.reset_flags &= ~AMDGPU_GMC_INIT_RESET_NPS;
+	}
+
 	r = gmc_v9_0_hw_init(adev);
 	if (r)
 		return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c
index d1bd79bbae53..8a0a63ac88d2 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c
@@ -401,6 +401,17 @@ static int nbio_v7_9_get_compute_partition_mode(struct amdgpu_device *adev)
 	return px;
 }
 
+static bool nbio_v7_9_is_nps_switch_requested(struct amdgpu_device *adev)
+{
+	u32 tmp;
+
+	tmp = RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_MEM_STATUS);
+	tmp = REG_GET_FIELD(tmp, BIF_BX_PF0_PARTITION_MEM_STATUS,
+			    CHANGE_STATUE);
+
+	/* 0x8 - NPS switch requested */
+	return (tmp == 0x8);
+}
 static u32 nbio_v7_9_get_memory_partition_mode(struct amdgpu_device *adev,
 					       u32 *supp_modes)
 {
@@ -508,6 +519,7 @@ const struct amdgpu_nbio_funcs nbio_v7_9_funcs = {
 	.remap_hdp_registers = nbio_v7_9_remap_hdp_registers,
 	.get_compute_partition_mode = nbio_v7_9_get_compute_partition_mode,
 	.get_memory_partition_mode = nbio_v7_9_get_memory_partition_mode,
+	.is_nps_switch_requested = nbio_v7_9_is_nps_switch_requested,
 	.init_registers = nbio_v7_9_init_registers,
 	.get_pcie_replay_count = nbio_v7_9_get_pcie_replay_count,
 	.set_reg_remap = nbio_v7_9_set_reg_remap,
-- 
2.25.1

RE: [PATCH 7/7] drm/amdgpu: Add NPS switch support for GC 9.4.3

[Xu, Feifei]

[AMD Official Use Only - AMD Internal Distribution Only]

Reviewed-by: Feifei Xu <Feifei.Xu@amd.com>

-----Original Message-----
From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Lijo Lazar
Sent: Tuesday, September 24, 2024 1:57 PM
To: amd-gfx@lists.freedesktop.org
Cc: Zhang, Hawking <Hawking.Zhang@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Koenig, Christian <Christian.Koenig@amd.com>; Bhardwaj, Rajneesh <Rajneesh.Bhardwaj@amd.com>; Errabolu, Ramesh <Ramesh.Errabolu@amd.com>
Subject: [PATCH 7/7] drm/amdgpu: Add NPS switch support for GC 9.4.3

Add dynamic NPS switch support for GC 9.4.3 variants. Only GC v9.4.3 and GC v9.4.4 currently support this. NPS switch is only supported if an SOC supports multiple NPS modes.

Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h |  1 +
 drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    | 44 ++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c   | 12 +++++++
 3 files changed, 57 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h
index f61d117b0caf..79c2f807b9fe 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h
@@ -101,6 +101,7 @@ struct amdgpu_nbio_funcs {
        int (*get_compute_partition_mode)(struct amdgpu_device *adev);
        u32 (*get_memory_partition_mode)(struct amdgpu_device *adev,
                                         u32 *supp_modes);
+       bool (*is_nps_switch_requested)(struct amdgpu_device *adev);
        u64 (*get_pcie_replay_count)(struct amdgpu_device *adev);
        void (*set_reg_remap)(struct amdgpu_device *adev);  }; diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index cafcb24449df..6a95402985ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -1395,6 +1395,17 @@ gmc_v9_0_query_memory_partition(struct amdgpu_device *adev)
        return gmc_v9_0_get_memory_partition(adev, NULL);  }

+static bool gmc_v9_0_need_reset_on_init(struct amdgpu_device *adev) {
+       if (adev->nbio.funcs &&
+           adev->nbio.funcs->is_nps_switch_requested(adev)) {
+               adev->gmc.reset_flags |= AMDGPU_GMC_INIT_RESET_NPS;
+               return true;
+       }
+
+       return false;
+}
+
 static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
        .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
        .flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid, @@ -1406,6 +1417,8 @@ static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
        .override_vm_pte_flags = gmc_v9_0_override_vm_pte_flags,
        .get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size,
        .query_mem_partition_mode = &gmc_v9_0_query_memory_partition,
+       .request_mem_partition_mode = &amdgpu_gmc_request_memory_partition,
+       .need_reset_on_init = &gmc_v9_0_need_reset_on_init,
 };

 static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) @@ -1545,6 +1558,28 @@ static void gmc_v9_0_set_xgmi_ras_funcs(struct amdgpu_device *adev)
                adev->gmc.xgmi.ras = &xgmi_ras;
 }

+static void gmc_v9_0_init_nps_details(struct amdgpu_device *adev) {
+       adev->gmc.supported_nps_modes = 0;
+
+       if (amdgpu_sriov_vf(adev) || (adev->flags & AMD_IS_APU))
+               return;
+
+       /*TODO: Check PSP version also which supports NPS switch. Otherwise keep
+        * supported modes as 0.
+        */
+       switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
+       case IP_VERSION(9, 4, 3):
+       case IP_VERSION(9, 4, 4):
+               adev->gmc.supported_nps_modes =
+                       BIT(AMDGPU_NPS1_PARTITION_MODE) |
+                       BIT(AMDGPU_NPS4_PARTITION_MODE);
+               break;
+       default:
+               break;
+       }
+}
+
 static int gmc_v9_0_early_init(void *handle)  {
        struct amdgpu_device *adev = (struct amdgpu_device *)handle; @@ -2165,6 +2200,7 @@ static int gmc_v9_0_sw_init(void *handle)
        if (r)
                return r;

+       gmc_v9_0_init_nps_details(adev);
        /*
         * number of VMs
         * VMID 0 is reserved for System
@@ -2440,6 +2476,14 @@ static int gmc_v9_0_resume(void *handle)
        int r;
        struct amdgpu_device *adev = (struct amdgpu_device *)handle;

+       /* If a reset is done for NPS mode switch, read the memory range
+        * information again.
+        */
+       if (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS) {
+               gmc_v9_0_init_sw_mem_ranges(adev, adev->gmc.mem_partitions);
+               adev->gmc.reset_flags &= ~AMDGPU_GMC_INIT_RESET_NPS;
+       }
+
        r = gmc_v9_0_hw_init(adev);
        if (r)
                return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c
index d1bd79bbae53..8a0a63ac88d2 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c
@@ -401,6 +401,17 @@ static int nbio_v7_9_get_compute_partition_mode(struct amdgpu_device *adev)
        return px;
 }

+static bool nbio_v7_9_is_nps_switch_requested(struct amdgpu_device
+*adev) {
+       u32 tmp;
+
+       tmp = RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_MEM_STATUS);
+       tmp = REG_GET_FIELD(tmp, BIF_BX_PF0_PARTITION_MEM_STATUS,
+                           CHANGE_STATUE);
+
+       /* 0x8 - NPS switch requested */
+       return (tmp == 0x8);
+}
 static u32 nbio_v7_9_get_memory_partition_mode(struct amdgpu_device *adev,
                                               u32 *supp_modes)
 {
@@ -508,6 +519,7 @@ const struct amdgpu_nbio_funcs nbio_v7_9_funcs = {
        .remap_hdp_registers = nbio_v7_9_remap_hdp_registers,
        .get_compute_partition_mode = nbio_v7_9_get_compute_partition_mode,
        .get_memory_partition_mode = nbio_v7_9_get_memory_partition_mode,
+       .is_nps_switch_requested = nbio_v7_9_is_nps_switch_requested,
        .init_registers = nbio_v7_9_init_registers,
        .get_pcie_replay_count = nbio_v7_9_get_pcie_replay_count,
        .set_reg_remap = nbio_v7_9_set_reg_remap,
--
2.25.1

Re: [PATCH 0/7] Add support for dynamic NPS switch

[Lazar, Lijo]

<Ping>

Thanks,
Lijo

On 9/24/2024 11:26 AM, Lijo Lazar wrote:
> This series adds supports for dynamic NPS switch on GC v9.4.3/9.4.4 SOC
> variants. 
> 
> In order to do dynamic NPS switch a sysfs interface is provided to request a new
> NPS mode. If the device is part of a hive, all hive devices are required to be
> in the same NPS mode. Hence a hive device request is saved in a hive variable.
> For individual device, it's saved in a gmc block variable.
> 
> In order to do a NPS mode switch, the workflow is -
> 
> 1) User places a requests through sysfs node.
> 2) User unloads the driver
> 3) During unload, driver checks for any pending NPS switch request. If any
> request is pending, it places the request to PSP FW. 
> 4) For a hive, request is placed in one-go for all devices in the hive. If
> one of the requests fails, a request is placed again to revert to current NPS
> mode on the successful devices.
> 5) User reloads the driver.
> 6) On reload, driver checks if NPS switch is pending and initiates a mode-1
> reset.
> 7) During resume after a reset, NPS ranges are read again from discovery table.
> 8) Driver detects the new NPS mode and makes a compatible compute partition mode
> switch if required.
> 
> Lijo Lazar (7):
>   drm/amdgpu: Add option to refresh NPS data
>   drm/amdgpu: Add PSP interface for NPS switch
>   drm/amdgpu: Add gmc interface to request NPS mode
>   drm/amdgpu: Add sysfs interfaces for NPS mode
>   drm/amdgpu: Place NPS mode request on unload
>   drm/amdgpu: Check gmc requirement for reset on init
>   drm/amdgpu: Add NPS switch support for GC 9.4.3
> 
>  drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c |  68 +++++--
>  drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h |   2 +-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c       |   1 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c       | 190 ++++++++++++++++--
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h       |  19 ++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h      |   1 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c       |  25 +++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h       |   1 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c      |  39 ++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h      |   5 +
>  drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c         |  44 ++++
>  drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c        |  12 ++
>  drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h       |  14 +-
>  drivers/gpu/drm/amd/amdgpu/soc15.c            |   2 +
>  14 files changed, 387 insertions(+), 36 deletions(-)
>