Re: [RFC PATCH v4 2/7] mm/demotion: Expose per node memory tier to sysfs

[Jonathan Cameron <Jonathan.Cameron@Huawei.com>]

On Fri, 27 May 2022 17:55:23 +0530
"Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com> wrote:

> From: Jagdish Gediya <jvgediya@linux.ibm.com>
> 
> Add support to read/write the memory tierindex for a NUMA node.
> 
> /sys/devices/system/node/nodeN/memtier
> 
> where N = node id
> 
> When read, It list the memory tier that the node belongs to.
> 
> When written, the kernel moves the node into the specified
> memory tier, the tier assignment of all other nodes are not
> affected.
> 
> If the memory tier does not exist, writing to the above file
> create the tier and assign the NUMA node to that tier.
creates

There was some discussion in v2 of Wei Xu's RFC that what matter
for creation is the rank, not the tier number. 

My suggestion is move to an explicit creation file such as
memtier/create_tier_from_rank
to which writing the rank gives results in a new tier
with the next device ID and requested rank.

> 
> mutex memory_tier_lock is introduced to protect memory tier
> related chanegs as it can happen from sysfs as well on hot
> plug events.
> 
> Signed-off-by: Jagdish Gediya <jvgediya@linux.ibm.com>
> Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
> ---
>  drivers/base/node.c     |  35 ++++++++++++++
>  include/linux/migrate.h |   4 +-
>  mm/migrate.c            | 103 ++++++++++++++++++++++++++++++++++++++++
>  3 files changed, 141 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/base/node.c b/drivers/base/node.c
> index ec8bb24a5a22..cf4a58446d8c 100644
> --- a/drivers/base/node.c
> +++ b/drivers/base/node.c
> @@ -20,6 +20,7 @@
>  #include <linux/pm_runtime.h>
>  #include <linux/swap.h>
>  #include <linux/slab.h>
> +#include <linux/migrate.h>
>  
>  static struct bus_type node_subsys = {
>  	.name = "node",
> @@ -560,11 +561,45 @@ static ssize_t node_read_distance(struct device *dev,
>  }
>  static DEVICE_ATTR(distance, 0444, node_read_distance, NULL);
>  
> +#ifdef CONFIG_TIERED_MEMORY
> +static ssize_t memtier_show(struct device *dev,
> +			    struct device_attribute *attr,
> +			    char *buf)
> +{
> +	int node = dev->id;
> +
> +	return sysfs_emit(buf, "%d\n", node_get_memory_tier(node));
> +}
> +
> +static ssize_t memtier_store(struct device *dev,
> +			     struct device_attribute *attr,
> +			     const char *buf, size_t count)
> +{
> +	unsigned long tier;
> +	int node = dev->id;
> +
> +	int ret = kstrtoul(buf, 10, &tier);
> +	if (ret)
> +		return ret;
> +
> +	ret = node_reset_memory_tier(node, tier);

I don't follow why reset here rather than set.

> +	if (ret)
> +		return ret;
> +
> +	return count;
> +}
> +
> +static DEVICE_ATTR_RW(memtier);
> +#endif
> +
>  static struct attribute *node_dev_attrs[] = {
>  	&dev_attr_meminfo.attr,
>  	&dev_attr_numastat.attr,
>  	&dev_attr_distance.attr,
>  	&dev_attr_vmstat.attr,
> +#ifdef CONFIG_TIERED_MEMORY
> +	&dev_attr_memtier.attr,
> +#endif
>  	NULL
>  };
>  
> diff --git a/include/linux/migrate.h b/include/linux/migrate.h
> index 0ec653623565..d37d1d5dee82 100644
> --- a/include/linux/migrate.h
> +++ b/include/linux/migrate.h
> @@ -177,13 +177,15 @@ enum memory_tier_type {
>  };
>  
>  int next_demotion_node(int node);
> -

tidy that up to reduce noise on next version.

>  extern void migrate_on_reclaim_init(void);
>  #ifdef CONFIG_HOTPLUG_CPU
>  extern void set_migration_target_nodes(void);
>  #else
>  static inline void set_migration_target_nodes(void) {}
>  #endif
> +int node_get_memory_tier(int node);
> +int node_set_memory_tier(int node, int tier);
> +int node_reset_memory_tier(int node, int tier);
>  #else
>  #define numa_demotion_enabled	false
>  static inline int next_demotion_node(int node)
> diff --git a/mm/migrate.c b/mm/migrate.c
> index f28ee93fb017..304559ba3372 100644
> --- a/mm/migrate.c
> +++ b/mm/migrate.c
> @@ -2132,6 +2132,7 @@ static struct bus_type memory_tier_subsys = {
>  	.dev_name = "memtier",
>  };
>  
> +DEFINE_MUTEX(memory_tier_lock);
>  static struct memory_tier *memory_tiers[MAX_MEMORY_TIERS];
>  
>  static ssize_t nodelist_show(struct device *dev,
> @@ -2225,6 +2226,108 @@ static const struct attribute_group *memory_tier_attr_groups[] = {
>  	NULL,
>  };
>  
> +static int __node_get_memory_tier(int node)
> +{
> +	int tier;
> +
> +	for (tier = 0; tier < MAX_MEMORY_TIERS; tier++) {
> +		if (memory_tiers[tier] && node_isset(node, memory_tiers[tier]->nodelist))
> +			return tier;
> +	}
> +
> +	return -1;
> +}
> +
> +int node_get_memory_tier(int node)
> +{
> +	int tier;
> +
> +	/*
> +	 * Make sure memory tier is not unregistered
> +	 * while it is being read.
> +	 */
> +	mutex_lock(&memory_tier_lock);
> +
> +	tier = __node_get_memory_tier(node);
> +
> +	mutex_unlock(&memory_tier_lock);
> +
> +	return tier;
> +}
> +
> +int __node_set_memory_tier(int node, int tier)
> +{
> +	int ret = 0;
> +	/*
> +	 * As register_memory_tier() for new tier can fail,
> +	 * try it before modifying existing tier. register
> +	 * tier makes tier visible in sysfs.
> +	 */
> +	if (!memory_tiers[tier]) {
> +		ret = register_memory_tier(tier);
> +		if (ret) {
> +			goto out;

no brackets around goto out;

It's also pointless, just return ret directly here



> +		}
> +	}
> +
> +	node_set(node, memory_tiers[tier]->nodelist);
> +
> +out:
> +	return ret;
> +}
> +
> +int node_reset_memory_tier(int node, int tier)
> +{
> +	int current_tier, ret = 0;
> +
> +	mutex_lock(&memory_tier_lock);
> +
> +	current_tier = __node_get_memory_tier(node);
> +	if (current_tier == tier)
> +		goto out;
> +
> +	if (current_tier != -1 )
> +		node_clear(node, memory_tiers[current_tier]->nodelist);
> +
> +	ret = __node_set_memory_tier(node, tier);
> +
> +	if (!ret) {
> +		if (nodes_empty(memory_tiers[current_tier]->nodelist))
> +			unregister_memory_tier(current_tier);
flip logic so the error path is out of line.
	if (ret) {
		/* reset..
		ret = ... 
		goto out;
	}	
and have the 'good path' here with less indent.


	if (nodes_empty(memory...

will result in more 'idiomatic' code that is easier for reviewers to
read.  It's Friday afternoon. Don't make me think :)

> +	} else {

> +		/* reset it back to older tier */
> +		ret = __node_set_memory_tier(node, current_tier);
> +	}
> +out:
> +	mutex_unlock(&memory_tier_lock);
> +
> +	return ret;
> +}
> +
> +int node_set_memory_tier(int node, int tier)
> +{
Currently seems to be unused code. Fine if it is used in 
a later patch, but call it out in the patch description.
 
> +	int current_tier, ret = 0;
> +
> +	if (tier >= MAX_MEMORY_TIERS)
> +		return -EINVAL;
> +
> +	mutex_lock(&memory_tier_lock);
> +	current_tier = __node_get_memory_tier(node);
> +	/*
> +	 * if node is already part of the tier proceed with the
> +	 * current tier value, because we might want to establish
> +	 * new migration paths now. The node might be added to a tier
> +	 * before it was made part of N_MEMORY, hence estabilish_migration_targets
> +	 * will have skipped this node.
> +	 */
> +	if (current_tier != -1)
> +		tier = current_tier;
> +	ret = __node_set_memory_tier(node, tier);
> +	mutex_unlock(&memory_tier_lock);
> +
> +	return ret;
> +}
> +
>  /*
>   * node_demotion[] example:
>   *