Re: [PATCH 2/2] Customize sched domain via cpuset (v2)

[Peter Zijlstra <peterz@infradead.org>]

On Fri, 2008-04-04 at 18:11 +0900, Hidetoshi Seto wrote:
> The implementation has some updates...
> 
> >  - Add 2 new cpuset files:
> >      sched_wake_idle_far
> >      sched_balance_newidle_far
>     -> Merged into 1 file, having levels:
>          sched_relax_domain_level
> 
> >  - Modify partition_sched_domains() and build_sched_domains()
> >    to take flags parameter passed from cpuset.
>     -> Changed to "attributes" rather than "flags."
> 
> >  - Fill newidle_idx for node domains which currently unused but
> >    might be required for sched_balance_newidle_far.
> 
>    + We can change the "default" level by boot option 'relax_domain_level='.
> 
> Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>

This seems like a sufficiently flexible interface. Paul, have you got
any outstanding objections?

Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>

> ---
>  include/asm-ia64/topology.h |    2 -
>  include/asm-sh/topology.h   |    2 -
>  include/asm-x86/topology.h  |    2 -
>  include/linux/sched.h       |   23 +++++++++++-
>  kernel/cpuset.c             |   61 ++++++++++++++++++++++++++++++++
>  kernel/sched.c              |   82 +++++++++++++++++++++++++++++++++++++++++---
>  kernel/sched_fair.c         |    4 +-
>  7 files changed, 165 insertions(+), 11 deletions(-)
> 
> Index: GIT-torvalds/include/linux/sched.h
> ===================================================================
> --- GIT-torvalds.orig/include/linux/sched.h
> +++ GIT-torvalds/include/linux/sched.h
> @@ -704,6 +704,7 @@ enum cpu_idle_type {
>  #define SD_POWERSAVINGS_BALANCE	256	/* Balance for power savings */
>  #define SD_SHARE_PKG_RESOURCES	512	/* Domain members share cpu pkg resources */
>  #define SD_SERIALIZE		1024	/* Only a single load balancing instance */
> +#define SD_WAKE_IDLE_FAR	2048	/* Gain latency sacrificing cache hit */
> 
>  #define BALANCE_FOR_MC_POWER	\
>  	(sched_smt_power_savings ? SD_POWERSAVINGS_BALANCE : 0)
> @@ -733,6 +734,24 @@ struct sched_group {
>  	u32 reciprocal_cpu_power;
>  };
> 
> +enum sched_domain_level {
> +	SD_LV_NONE = 0,
> +	SD_LV_SIBLING,
> +	SD_LV_MC,
> +	SD_LV_CPU,
> +	SD_LV_NODE,
> +	SD_LV_ALLNODES,
> +	SD_LV_MAX
> +};
> +
> +struct sched_domain_attr {
> +	int relax_domain_level;
> +};
> +
> +#define SD_ATTR_INIT	(struct sched_domain_attr) {	\
> +	.relax_domain_level = -1,			\
> +}
> +
>  struct sched_domain {
>  	/* These fields must be setup */
>  	struct sched_domain *parent;	/* top domain must be null terminated */
> @@ -750,6 +769,7 @@ struct sched_domain {
>  	unsigned int wake_idx;
>  	unsigned int forkexec_idx;
>  	int flags;			/* See SD_* */
> +	enum sched_domain_level level;
> 
>  	/* Runtime fields. */
>  	unsigned long last_balance;	/* init to jiffies. units in jiffies */
> @@ -789,7 +809,8 @@ struct sched_domain {
>  #endif
>  };
> 
> -extern void partition_sched_domains(int ndoms_new, cpumask_t *doms_new);
> +extern void partition_sched_domains(int ndoms_new, cpumask_t *doms_new,
> +				    struct sched_domain_attr *dattr_new);
>  extern int arch_reinit_sched_domains(void);
> 
>  #endif	/* CONFIG_SMP */
> Index: GIT-torvalds/kernel/sched.c
> ===================================================================
> --- GIT-torvalds.orig/kernel/sched.c
> +++ GIT-torvalds/kernel/sched.c
> @@ -6582,11 +6582,42 @@ static void init_sched_groups_power(int
>  	} while (group != child->groups);
>  }
> 
> +static int default_relax_domain_level = -1;
> +
> +static int __init setup_relax_domain_level(char *str)
> +{
> +	default_relax_domain_level = simple_strtoul(str, NULL, 0);
> +	return 1;
> +}
> +__setup("relax_domain_level=", setup_relax_domain_level);
> +
> +static void set_domain_attribute(struct sched_domain *sd,
> +				 struct sched_domain_attr *attr)
> +{
> +	int request;
> +
> +	if (!attr || attr->relax_domain_level < 0) {
> +		if (default_relax_domain_level < 0)
> +			return;
> +		else
> +			request = default_relax_domain_level;
> +	} else
> +		request = attr->relax_domain_level;
> +	if (request < sd->level) {
> +		/* turn off idle balance on this domain */
> +		sd->flags &= ~(SD_WAKE_IDLE|SD_BALANCE_NEWIDLE);
> +	} else {
> +		/* turn on idle balance on this domain */
> +		sd->flags |= (SD_WAKE_IDLE_FAR|SD_BALANCE_NEWIDLE);
> +	}
> +}
> +
>  /*
>   * Build sched domains for a given set of cpus and attach the sched domains
>   * to the individual cpus
>   */
> -static int build_sched_domains(const cpumask_t *cpu_map)
> +static int __build_sched_domains(const cpumask_t *cpu_map,
> +				 struct sched_domain_attr *attr)
>  {
>  	int i;
>  	struct root_domain *rd;
> @@ -6626,7 +6657,9 @@ static int build_sched_domains(const cpu
>  				SD_NODES_PER_DOMAIN*cpus_weight(nodemask)) {
>  			sd = &per_cpu(allnodes_domains, i);
>  			*sd = SD_ALLNODES_INIT;
> +			sd->level = SD_LV_ALLNODES;
>  			sd->span = *cpu_map;
> +			set_domain_attribute(sd, attr);
>  			cpu_to_allnodes_group(i, cpu_map, &sd->groups);
>  			p = sd;
>  			sd_allnodes = 1;
> @@ -6635,7 +6668,9 @@ static int build_sched_domains(const cpu
> 
>  		sd = &per_cpu(node_domains, i);
>  		*sd = SD_NODE_INIT;
> +		sd->level = SD_LV_NODE;
>  		sd->span = sched_domain_node_span(cpu_to_node(i));
> +		set_domain_attribute(sd, attr);
>  		sd->parent = p;
>  		if (p)
>  			p->child = sd;
> @@ -6645,7 +6680,9 @@ static int build_sched_domains(const cpu
>  		p = sd;
>  		sd = &per_cpu(phys_domains, i);
>  		*sd = SD_CPU_INIT;
> +		sd->level = SD_LV_CPU;
>  		sd->span = nodemask;
> +		set_domain_attribute(sd, attr);
>  		sd->parent = p;
>  		if (p)
>  			p->child = sd;
> @@ -6655,8 +6692,10 @@ static int build_sched_domains(const cpu
>  		p = sd;
>  		sd = &per_cpu(core_domains, i);
>  		*sd = SD_MC_INIT;
> +		sd->level = SD_LV_MC;
>  		sd->span = cpu_coregroup_map(i);
>  		cpus_and(sd->span, sd->span, *cpu_map);
> +		set_domain_attribute(sd, attr);
>  		sd->parent = p;
>  		p->child = sd;
>  		cpu_to_core_group(i, cpu_map, &sd->groups);
> @@ -6666,8 +6705,10 @@ static int build_sched_domains(const cpu
>  		p = sd;
>  		sd = &per_cpu(cpu_domains, i);
>  		*sd = SD_SIBLING_INIT;
> +		sd->level = SD_LV_SIBLING;
>  		sd->span = per_cpu(cpu_sibling_map, i);
>  		cpus_and(sd->span, sd->span, *cpu_map);
> +		set_domain_attribute(sd, attr);
>  		sd->parent = p;
>  		p->child = sd;
>  		cpu_to_cpu_group(i, cpu_map, &sd->groups);
> @@ -6840,8 +6881,15 @@ error:
>  #endif
>  }
> 
> +static int build_sched_domains(const cpumask_t *cpu_map)
> +{
> +	return __build_sched_domains(cpu_map, NULL);
> +}
> +
>  static cpumask_t *doms_cur;	/* current sched domains */
>  static int ndoms_cur;		/* number of sched domains in 'doms_cur' */
> +static struct sched_domain_attr *dattr_cur;	/* attribues of custom domains
> +						   in 'doms_cur' */
> 
>  /*
>   * Special case: If a kmalloc of a doms_cur partition (array of
> @@ -6868,6 +6916,7 @@ static int arch_init_sched_domains(const
>  	doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL);
>  	if (!doms_cur)
>  		doms_cur = &fallback_doms;
> +	dattr_cur = NULL;
>  	cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map);
>  	err = build_sched_domains(doms_cur);
>  	register_sched_domain_sysctl();
> @@ -6896,6 +6945,22 @@ static void detach_destroy_domains(const
>  	arch_destroy_sched_domains(cpu_map);
>  }
> 
> +/* handle null as "default" */
> +static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
> +			struct sched_domain_attr *new, int idx_new)
> +{
> +	struct sched_domain_attr tmp;
> +
> +	/* fast path */
> +	if (!new && !cur)
> +		return 1;
> +
> +	tmp = SD_ATTR_INIT;
> +	return !memcmp(cur ? (cur + idx_cur) : &tmp,
> +			new ? (new + idx_new) : &tmp,
> +			sizeof(struct sched_domain_attr));
> +}
> +
>  /*
>   * Partition sched domains as specified by the 'ndoms_new'
>   * cpumasks in the array doms_new[] of cpumasks. This compares
> @@ -6917,7 +6982,8 @@ static void detach_destroy_domains(const
>   *
>   * Call with hotplug lock held
>   */
> -void partition_sched_domains(int ndoms_new, cpumask_t *doms_new)
> +void partition_sched_domains(int ndoms_new, cpumask_t *doms_new,
> +			     struct sched_domain_attr *dattr_new)
>  {
>  	int i, j;
> 
> @@ -6929,13 +6995,15 @@ void partition_sched_domains(int ndoms_n
>  	if (doms_new == NULL) {
>  		ndoms_new = 1;
>  		doms_new = &fallback_doms;
> +		dattr_new = NULL;
>  		cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map);
>  	}
> 
>  	/* Destroy deleted domains */
>  	for (i = 0; i < ndoms_cur; i++) {
>  		for (j = 0; j < ndoms_new; j++) {
> -			if (cpus_equal(doms_cur[i], doms_new[j]))
> +			if (cpus_equal(doms_cur[i], doms_new[j])
> +			    && dattrs_equal(dattr_cur, i, dattr_new, j))
>  				goto match1;
>  		}
>  		/* no match - a current sched domain not in new doms_new[] */
> @@ -6947,11 +7015,13 @@ match1:
>  	/* Build new domains */
>  	for (i = 0; i < ndoms_new; i++) {
>  		for (j = 0; j < ndoms_cur; j++) {
> -			if (cpus_equal(doms_new[i], doms_cur[j]))
> +			if (cpus_equal(doms_new[i], doms_cur[j])
> +			    && dattrs_equal(dattr_new, i, dattr_cur, j))
>  				goto match2;
>  		}
>  		/* no match - add a new doms_new */
> -		build_sched_domains(doms_new + i);
> +		__build_sched_domains(doms_new + i,
> +					dattr_new ? dattr_new + i : NULL);
>  match2:
>  		;
>  	}
> @@ -6959,7 +7029,9 @@ match2:
>  	/* Remember the new sched domains */
>  	if (doms_cur != &fallback_doms)
>  		kfree(doms_cur);
> +	kfree(dattr_cur);	/* kfree(NULL) is safe */
>  	doms_cur = doms_new;
> +	dattr_cur = dattr_new;
>  	ndoms_cur = ndoms_new;
> 
>  	register_sched_domain_sysctl();
> Index: GIT-torvalds/kernel/sched_fair.c
> ===================================================================
> --- GIT-torvalds.orig/kernel/sched_fair.c
> +++ GIT-torvalds/kernel/sched_fair.c
> @@ -957,7 +957,9 @@ static int wake_idle(int cpu, struct tas
>  		return cpu;
> 
>  	for_each_domain(cpu, sd) {
> -		if (sd->flags & SD_WAKE_IDLE) {
> +		if ((sd->flags & SD_WAKE_IDLE)
> +		    || ((sd->flags & SD_WAKE_IDLE_FAR)
> +			&& !task_hot(p, task_rq(p)->clock, sd))) {
>  			cpus_and(tmp, sd->span, p->cpus_allowed);
>  			for_each_cpu_mask(i, tmp) {
>  				if (idle_cpu(i)) {
> Index: GIT-torvalds/kernel/cpuset.c
> ===================================================================
> --- GIT-torvalds.orig/kernel/cpuset.c
> +++ GIT-torvalds/kernel/cpuset.c
> @@ -98,6 +98,9 @@ struct cpuset {
>  	/* partition number for rebuild_sched_domains() */
>  	int pn;
> 
> +	/* for custom sched domain */
> +	int relax_domain_level;
> +
>  	/* used for walking a cpuset heirarchy */
>  	struct list_head stack_list;
>  };
> @@ -478,6 +481,16 @@ static int cpusets_overlap(struct cpuset
>  	return cpus_intersects(a->cpus_allowed, b->cpus_allowed);
>  }
> 
> +static void
> +update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c)
> +{
> +	if (!dattr)
> +		return;
> +	if (dattr->relax_domain_level < c->relax_domain_level)
> +		dattr->relax_domain_level = c->relax_domain_level;
> +	return;
> +}
> +
>  /*
>   * rebuild_sched_domains()
>   *
> @@ -553,12 +566,14 @@ static void rebuild_sched_domains(void)
>  	int csn;		/* how many cpuset ptrs in csa so far */
>  	int i, j, k;		/* indices for partition finding loops */
>  	cpumask_t *doms;	/* resulting partition; i.e. sched domains */
> +	struct sched_domain_attr *dattr;  /* attributes for custom domains */
>  	int ndoms;		/* number of sched domains in result */
>  	int nslot;		/* next empty doms[] cpumask_t slot */
> 
>  	q = NULL;
>  	csa = NULL;
>  	doms = NULL;
> +	dattr = NULL;
> 
>  	/* Special case for the 99% of systems with one, full, sched domain */
>  	if (is_sched_load_balance(&top_cpuset)) {
> @@ -566,6 +581,11 @@ static void rebuild_sched_domains(void)
>  		doms = kmalloc(sizeof(cpumask_t), GFP_KERNEL);
>  		if (!doms)
>  			goto rebuild;
> +		dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL);
> +		if (dattr) {
> +			*dattr = SD_ATTR_INIT;
> +			update_domain_attr(dattr, &top_cpuset);
> +		}
>  		*doms = top_cpuset.cpus_allowed;
>  		goto rebuild;
>  	}
> @@ -622,6 +642,7 @@ restart:
>  	doms = kmalloc(ndoms * sizeof(cpumask_t), GFP_KERNEL);
>  	if (!doms)
>  		goto rebuild;
> +	dattr = kmalloc(ndoms * sizeof(struct sched_domain_attr), GFP_KERNEL);
> 
>  	for (nslot = 0, i = 0; i < csn; i++) {
>  		struct cpuset *a = csa[i];
> @@ -644,12 +665,15 @@ restart:
>  			}
> 
>  			cpus_clear(*dp);
> +			if (dattr)
> +				*(dattr + nslot) = SD_ATTR_INIT;
>  			for (j = i; j < csn; j++) {
>  				struct cpuset *b = csa[j];
> 
>  				if (apn == b->pn) {
>  					cpus_or(*dp, *dp, b->cpus_allowed);
>  					b->pn = -1;
> +					update_domain_attr(dattr, b);
>  				}
>  			}
>  			nslot++;
> @@ -660,7 +684,7 @@ restart:
>  rebuild:
>  	/* Have scheduler rebuild sched domains */
>  	get_online_cpus();
> -	partition_sched_domains(ndoms, doms);
> +	partition_sched_domains(ndoms, doms, dattr);
>  	put_online_cpus();
> 
>  done:
> @@ -668,6 +692,7 @@ done:
>  		kfifo_free(q);
>  	kfree(csa);
>  	/* Don't kfree(doms) -- partition_sched_domains() does that. */
> +	/* Don't kfree(dattr) -- partition_sched_domains() does that. */
>  }
> 
>  static inline int started_after_time(struct task_struct *t1,
> @@ -1011,6 +1036,21 @@ static int update_memory_pressure_enable
>  	return 0;
>  }
> 
> +static int update_relax_domain_level(struct cpuset *cs, char *buf)
> +{
> +	int val = simple_strtol(buf, NULL, 10);
> +
> +	if (val < 0)
> +		val = -1;
> +
> +	if (val != cs->relax_domain_level) {
> +		cs->relax_domain_level = val;
> +		rebuild_sched_domains();
> +	}
> +
> +	return 0;
> +}
> +
>  /*
>   * update_flag - read a 0 or a 1 in a file and update associated flag
>   * bit:	the bit to update (CS_CPU_EXCLUSIVE, CS_MEM_EXCLUSIVE,
> @@ -1202,6 +1242,7 @@ typedef enum {
>  	FILE_CPU_EXCLUSIVE,
>  	FILE_MEM_EXCLUSIVE,
>  	FILE_SCHED_LOAD_BALANCE,
> +	FILE_SCHED_RELAX_DOMAIN_LEVEL,
>  	FILE_MEMORY_PRESSURE_ENABLED,
>  	FILE_MEMORY_PRESSURE,
>  	FILE_SPREAD_PAGE,
> @@ -1256,6 +1297,9 @@ static ssize_t cpuset_common_file_write(
>  	case FILE_SCHED_LOAD_BALANCE:
>  		retval = update_flag(CS_SCHED_LOAD_BALANCE, cs, buffer);
>  		break;
> +	case FILE_SCHED_RELAX_DOMAIN_LEVEL:
> +		retval = update_relax_domain_level(cs, buffer);
> +		break;
>  	case FILE_MEMORY_MIGRATE:
>  		retval = update_flag(CS_MEMORY_MIGRATE, cs, buffer);
>  		break;
> @@ -1354,6 +1398,9 @@ static ssize_t cpuset_common_file_read(s
>  	case FILE_SCHED_LOAD_BALANCE:
>  		*s++ = is_sched_load_balance(cs) ? '1' : '0';
>  		break;
> +	case FILE_SCHED_RELAX_DOMAIN_LEVEL:
> +		s += sprintf(s, "%d", cs->relax_domain_level);
> +		break;
>  	case FILE_MEMORY_MIGRATE:
>  		*s++ = is_memory_migrate(cs) ? '1' : '0';
>  		break;
> @@ -1424,6 +1471,13 @@ static struct cftype cft_sched_load_bala
>  	.private = FILE_SCHED_LOAD_BALANCE,
>  };
> 
> +static struct cftype cft_sched_relax_domain_level = {
> +	.name = "sched_relax_domain_level",
> +	.read = cpuset_common_file_read,
> +	.write = cpuset_common_file_write,
> +	.private = FILE_SCHED_RELAX_DOMAIN_LEVEL,
> +};
> +
>  static struct cftype cft_memory_migrate = {
>  	.name = "memory_migrate",
>  	.read = cpuset_common_file_read,
> @@ -1475,6 +1529,9 @@ static int cpuset_populate(struct cgroup
>  		return err;
>  	if ((err = cgroup_add_file(cont, ss, &cft_sched_load_balance)) < 0)
>  		return err;
> +	if ((err = cgroup_add_file(cont, ss,
> +					&cft_sched_relax_domain_level)) < 0)
> +		return err;
>  	if ((err = cgroup_add_file(cont, ss, &cft_memory_pressure)) < 0)
>  		return err;
>  	if ((err = cgroup_add_file(cont, ss, &cft_spread_page)) < 0)
> @@ -1559,6 +1616,7 @@ static struct cgroup_subsys_state *cpuse
>  	cs->mems_allowed = NODE_MASK_NONE;
>  	cs->mems_generation = cpuset_mems_generation++;
>  	fmeter_init(&cs->fmeter);
> +	cs->relax_domain_level = -1;
> 
>  	cs->parent = parent;
>  	number_of_cpusets++;
> @@ -1631,6 +1689,7 @@ int __init cpuset_init(void)
>  	fmeter_init(&top_cpuset.fmeter);
>  	top_cpuset.mems_generation = cpuset_mems_generation++;
>  	set_bit(CS_SCHED_LOAD_BALANCE, &top_cpuset.flags);
> +	top_cpuset.relax_domain_level = -1;
> 
>  	err = register_filesystem(&cpuset_fs_type);
>  	if (err < 0)
> Index: GIT-torvalds/include/asm-ia64/topology.h
> ===================================================================
> --- GIT-torvalds.orig/include/asm-ia64/topology.h
> +++ GIT-torvalds/include/asm-ia64/topology.h
> @@ -93,7 +93,7 @@ void build_cpu_to_node_map(void);
>  	.cache_nice_tries	= 2,			\
>  	.busy_idx		= 3,			\
>  	.idle_idx		= 2,			\
> -	.newidle_idx		= 0, /* unused */	\
> +	.newidle_idx		= 2,			\
>  	.wake_idx		= 1,			\
>  	.forkexec_idx		= 1,			\
>  	.flags			= SD_LOAD_BALANCE	\
> Index: GIT-torvalds/include/asm-sh/topology.h
> ===================================================================
> --- GIT-torvalds.orig/include/asm-sh/topology.h
> +++ GIT-torvalds/include/asm-sh/topology.h
> @@ -16,7 +16,7 @@
>  	.cache_nice_tries	= 2,			\
>  	.busy_idx		= 3,			\
>  	.idle_idx		= 2,			\
> -	.newidle_idx		= 0,			\
> +	.newidle_idx		= 2,			\
>  	.wake_idx		= 1,			\
>  	.forkexec_idx		= 1,			\
>  	.flags			= SD_LOAD_BALANCE	\
> Index: GIT-torvalds/include/asm-x86/topology.h
> ===================================================================
> --- GIT-torvalds.orig/include/asm-x86/topology.h
> +++ GIT-torvalds/include/asm-x86/topology.h
> @@ -129,7 +129,7 @@ extern unsigned long node_remap_size[];
> 
>  # define SD_CACHE_NICE_TRIES	2
>  # define SD_IDLE_IDX		2
> -# define SD_NEWIDLE_IDX		0
> +# define SD_NEWIDLE_IDX		2
>  # define SD_FORKEXEC_IDX	1
> 
>  #endif
>