Re: [PATCH 4/7] drm/amdgpu: Add sysfs interfaces for NPS mode

["Bhardwaj, Rajneesh" <rajneesh.bhardwaj@amd.com>]

Reviewed-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>

On 9/24/2024 1:56 AM, Lijo Lazar wrote:
> Add a sysfs interface to see available NPS modes to switch to -
>
> 	cat /sys/bus/pci/devices/../available_memory_paritition
>
> Make the current_memory_partition sysfs node read/write for requesting a
> new NPS mode. The request is only cached and at a later point a driver
> unload/reload is required to switch to the new NPS mode.
>
> Ex:
> 	echo NPS1 > /sys/bus/pci/devices/../current_memory_paritition
> 	echo NPS4 > /sys/bus/pci/devices/../current_memory_paritition
>
> The above interfaces will be available only if the SOC supports more than
> one NPS mode.
>
> Also modify the current memory partition sysfs logic to be more
> generic.
>
> Signed-off-by: Lijo Lazar <lijo.lazar@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 114 ++++++++++++++++++++----
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |   6 ++
>   2 files changed, 104 insertions(+), 16 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> index 758fda4e628f..24a1f931d9ed 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> @@ -1130,6 +1130,79 @@ int amdgpu_gmc_vram_checking(struct amdgpu_device *adev)
>   	return ret;
>   }
>   
> +static const char *nps_desc[] = {
> +	[AMDGPU_NPS1_PARTITION_MODE] = "NPS1",
> +	[AMDGPU_NPS2_PARTITION_MODE] = "NPS2",
> +	[AMDGPU_NPS3_PARTITION_MODE] = "NPS3",
> +	[AMDGPU_NPS4_PARTITION_MODE] = "NPS4",
> +	[AMDGPU_NPS6_PARTITION_MODE] = "NPS6",
> +	[AMDGPU_NPS8_PARTITION_MODE] = "NPS8",
> +};
> +
> +static ssize_t available_memory_partition_show(struct device *dev,
> +					       struct device_attribute *addr,
> +					       char *buf)
> +{
> +	struct drm_device *ddev = dev_get_drvdata(dev);
> +	struct amdgpu_device *adev = drm_to_adev(ddev);
> +	int size = 0, mode;
> +	char *sep = "";
> +
> +	for_each_inst(mode, adev->gmc.supported_nps_modes) {
> +		size += sysfs_emit_at(buf, size, "%s%s", sep, nps_desc[mode]);
> +		sep = ", ";
> +	}
> +	size += sysfs_emit_at(buf, size, "\n");
> +
> +	return size;
> +}
> +
> +static ssize_t current_memory_partition_store(struct device *dev,
> +					      struct device_attribute *attr,
> +					      const char *buf, size_t count)
> +{
> +	struct drm_device *ddev = dev_get_drvdata(dev);
> +	struct amdgpu_device *adev = drm_to_adev(ddev);
> +	enum amdgpu_memory_partition mode;
> +	struct amdgpu_hive_info *hive;
> +	int i;
> +
> +	mode = UNKNOWN_MEMORY_PARTITION_MODE;
> +	for_each_inst(i, adev->gmc.supported_nps_modes) {
> +		if (!strncasecmp(nps_desc[i], buf, strlen(nps_desc[i]))) {
> +			mode = i;
> +			break;
> +		}
> +	}
> +
> +	if (mode == UNKNOWN_MEMORY_PARTITION_MODE)
> +		return -EINVAL;
> +
> +	if (mode == adev->gmc.gmc_funcs->query_mem_partition_mode(adev)) {
> +		dev_info(
> +			adev->dev,
> +			"requested NPS mode is same as current NPS mode, skipping\n");
> +		return count;
> +	}
> +
> +	/* If device is part of hive, all devices in the hive should request the
> +	 * same mode. Hence store the requested mode in hive.
> +	 */
> +	hive = amdgpu_get_xgmi_hive(adev);
> +	if (hive) {
> +		atomic_set(&hive->requested_nps_mode, mode);
> +		amdgpu_put_xgmi_hive(hive);
> +	} else {
> +		adev->gmc.requested_nps_mode = mode;
> +	}
> +
> +	dev_info(
> +		adev->dev,
> +		"NPS mode change requested, please remove and reload the driver\n");
> +
> +	return count;
> +}
> +
>   static ssize_t current_memory_partition_show(
>   	struct device *dev, struct device_attribute *addr, char *buf)
>   {
> @@ -1138,38 +1211,47 @@ static ssize_t current_memory_partition_show(
>   	enum amdgpu_memory_partition mode;
>   
>   	mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
> -	switch (mode) {
> -	case AMDGPU_NPS1_PARTITION_MODE:
> -		return sysfs_emit(buf, "NPS1\n");
> -	case AMDGPU_NPS2_PARTITION_MODE:
> -		return sysfs_emit(buf, "NPS2\n");
> -	case AMDGPU_NPS3_PARTITION_MODE:
> -		return sysfs_emit(buf, "NPS3\n");
> -	case AMDGPU_NPS4_PARTITION_MODE:
> -		return sysfs_emit(buf, "NPS4\n");
> -	case AMDGPU_NPS6_PARTITION_MODE:
> -		return sysfs_emit(buf, "NPS6\n");
> -	case AMDGPU_NPS8_PARTITION_MODE:
> -		return sysfs_emit(buf, "NPS8\n");
> -	default:
> +	if ((mode > ARRAY_SIZE(nps_desc)) ||
> +	    (BIT(mode) & AMDGPU_ALL_NPS_MASK) != BIT(mode))
>   		return sysfs_emit(buf, "UNKNOWN\n");
> -	}
> +
> +	return sysfs_emit(buf, "%s\n", nps_desc[mode]);
>   }
>   
> -static DEVICE_ATTR_RO(current_memory_partition);
> +static DEVICE_ATTR_RW(current_memory_partition);
> +static DEVICE_ATTR_RO(available_memory_partition);
>   
>   int amdgpu_gmc_sysfs_init(struct amdgpu_device *adev)
>   {
> +	bool nps_switch_support;
> +	int r = 0;
> +
>   	if (!adev->gmc.gmc_funcs->query_mem_partition_mode)
>   		return 0;
>   
> +	nps_switch_support = (hweight32(adev->gmc.supported_nps_modes &
> +					AMDGPU_ALL_NPS_MASK) > 1);
> +	if (!nps_switch_support)
> +		dev_attr_current_memory_partition.attr.mode &=
> +			~(S_IWUSR | S_IWGRP | S_IWOTH);
> +	else
> +		r = device_create_file(adev->dev,
> +				       &dev_attr_available_memory_partition);
> +
> +	if (r)
> +		return r;
> +
>   	return device_create_file(adev->dev,
>   				  &dev_attr_current_memory_partition);
>   }
>   
>   void amdgpu_gmc_sysfs_fini(struct amdgpu_device *adev)
>   {
> +	if (!adev->gmc.gmc_funcs->query_mem_partition_mode)
> +		return;
> +
>   	device_remove_file(adev->dev, &dev_attr_current_memory_partition);
> +	device_remove_file(adev->dev, &dev_attr_available_memory_partition);
>   }
>   
>   int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> index f5be5112b742..1a50639a003a 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> @@ -73,6 +73,11 @@ enum amdgpu_memory_partition {
>   	AMDGPU_NPS8_PARTITION_MODE = 8,
>   };
>   
> +#define AMDGPU_ALL_NPS_MASK                                                  \
> +	(BIT(AMDGPU_NPS1_PARTITION_MODE) | BIT(AMDGPU_NPS2_PARTITION_MODE) | \
> +	 BIT(AMDGPU_NPS3_PARTITION_MODE) | BIT(AMDGPU_NPS4_PARTITION_MODE) | \
> +	 BIT(AMDGPU_NPS6_PARTITION_MODE) | BIT(AMDGPU_NPS8_PARTITION_MODE))
> +
>   /*
>    * GMC page fault information
>    */
> @@ -308,6 +313,7 @@ struct amdgpu_gmc {
>   	uint8_t num_mem_partitions;
>   	const struct amdgpu_gmc_funcs	*gmc_funcs;
>   	enum amdgpu_memory_partition	requested_nps_mode;
> +	uint32_t supported_nps_modes;
>   
>   	struct amdgpu_xgmi xgmi;
>   	struct amdgpu_irq_src	ecc_irq;