[RFC PATCH] sched: Add cpu based entries to debugfs

[RFC PATCH] sched: Add cpu based entries to debugfs

[David Ahern]

Currently sched_debug can be added to the kernel commandline parameters
to dump domain information during boot. This method is not practical with
a large number of CPUs.

This patch adds per-cpu entries to debugfs under a sched directory.
Reading the per-cpu file shows the domain information in a human-readable
format:

$ cat /sys/kernel/debug/sched/cpu0
domain 0 / SMT:
    flags: 0x2af:  load-balance new-idle exec fork affine cpu-capacity share-pkg-resources
    span: 0-7
    groups:
        0 (cpu_capacity = 147)
        1 (cpu_capacity = 147)
        2 (cpu_capacity = 147)
        3 (cpu_capacity = 147)
        4 (cpu_capacity = 147)
        5 (cpu_capacity = 147)
        6 (cpu_capacity = 147)
        7 (cpu_capacity = 147)

domain 2 / DIE:
    flags: 0x102f:  load-balance new-idle exec fork affine prefer-sibling
    span: 0-127
    groups:
        0-7 (cpu_capacity = 1176)
        8-15 (cpu_capacity = 1176)
        16-23 (cpu_capacity = 1176)
        24-31 (cpu_capacity = 1176)
        32-39 (cpu_capacity = 1176)
        40-47 (cpu_capacity = 1176)
        48-55 (cpu_capacity = 1176)
        56-63 (cpu_capacity = 1176)
        64-71 (cpu_capacity = 1176)
        72-79 (cpu_capacity = 1176)
        80-87 (cpu_capacity = 1176)
        88-95 (cpu_capacity = 1176)
        96-103 (cpu_capacity = 1176)
        104-111 (cpu_capacity = 1176)
        112-119 (cpu_capacity = 1176)
        120-127 (cpu_capacity = 1176)

domain 3 / NUMA:
    flags: 0x642f:  load-balance new-idle exec fork affine serialize overlap numa
    span: 0-1023
    groups:
        0-127 (cpu_capacity = 18816)
        128-255 (cpu_capacity = 18816)
        256-383 (cpu_capacity = 18816)
        384-511 (cpu_capacity = 18816)
        512-639 (cpu_capacity = 18816)
        640-767 (cpu_capacity = 18816)
        768-895 (cpu_capacity = 18816)
        896-1023 (cpu_capacity = 18816)

Before spending too much time formalizing this I wanted to see if you guys
would entertain the idea of making this info available via debugfs. It does
move the existing sched_features file to sched/features -- not sure how 
acceptable it is to move files in debugfs.

TO-DO: handle hotplug

Signed-off-by: David Ahern <david.ahern@oracle.com>
---
 kernel/sched/core.c | 167 +++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 164 insertions(+), 3 deletions(-)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 62671f53202a..b4d8d0c8260e 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -268,12 +268,173 @@ static const struct file_operations sched_feat_fops = {
 	.release	= single_release,
 };
 
+static const char * const sd_flag_names[] = {
+	"load-balance",
+	"new-idle",
+	"exec",
+	"fork",
+	"wake",
+	"affine",
+	"",
+	"cpu-capacity",
+	"power-domain",
+	"share-pkg-resources",
+	"serialize",
+	"asym-packing",
+	"prefer-sibling",
+	"overlap",
+	"numa",
+	"",
+};
+static void sched_cpu_domain_show(struct seq_file *m, struct sched_domain *sd,
+				  int cpu)
+{
+	struct cpumask groupmask;
+	struct sched_group *group = sd->groups;
+	int i;
+
+	cpumask_clear(&groupmask);
+
+	seq_printf(m, "domain %d / %s:\n", sd->level, sd->name);
+	seq_printf(m, "    flags: 0x%x: ", sd->flags);
+
+	for (i = 0; i < ARRAY_SIZE(sd_flag_names); ++i) {
+		if (sd->flags & (1 << i))
+			seq_printf(m, " %s", sd_flag_names[i]);
+	}
+	seq_puts(m, "\n");
+
+	if (!(sd->flags & SD_LOAD_BALANCE) && sd->parent)
+		seq_puts(m, "           ERROR: !SD_LOAD_BALANCE domain has parent\n");
+
+	seq_printf(m, "    span: %*pbl\n",
+		   cpumask_pr_args(sched_domain_span(sd)));
+
+	if (!cpumask_test_cpu(cpu, sched_domain_span(sd)))
+		seq_printf(m, "    ERROR: domain->span does not contain CPU%d\n", cpu);
+
+	if (!cpumask_test_cpu(cpu, sched_group_cpus(group)))
+		seq_printf(m, "    ERROR: domain->groups does not contain CPU%d\n", cpu);
+
+	seq_puts(m, "    groups:\n");
+	do {
+		if (!group) {
+			seq_puts(m, "            ERROR: group is NULL\n");
+			break;
+		}
+
+		/*
+		 * Even though we initialize ->capacity to something semi-sane,
+		 * we leave capacity_orig unset. This allows us to detect if
+		 * domain iteration is still funny without causing /0 traps.
+		 */
+		if (!group->sgc->capacity_orig) {
+			seq_puts(m, "        ERROR: domain->cpu_capacity not set\n");
+			break;
+		}
+
+		if (!cpumask_weight(sched_group_cpus(group))) {
+			seq_puts(m, "        ERROR: empty group\n");
+			break;
+		}
+
+		if (!(sd->flags & SD_OVERLAP) &&
+		    cpumask_intersects(&groupmask, sched_group_cpus(group))) {
+			seq_puts(m, "        ERROR: repeated CPUs\n");
+			break;
+		}
+
+		cpumask_or(&groupmask, &groupmask, sched_group_cpus(group));
+
+		seq_printf(m, "        %*pbl",
+			   cpumask_pr_args(sched_group_cpus(group)));
+
+		if (group->sgc->capacity != SCHED_CAPACITY_SCALE) {
+			seq_printf(m, " (cpu_capacity = %d)",
+				   group->sgc->capacity);
+		}
+		seq_puts(m, "\n");
+
+		group = group->next;
+	} while (group != sd->groups);
+
+	if (!cpumask_equal(sched_domain_span(sd), &groupmask))
+		seq_puts(m, "    ERROR: groups don't span domain->span\n");
+
+	if (sd->parent &&
+	    !cpumask_subset(&groupmask, sched_domain_span(sd->parent))) {
+		seq_puts(m, "    ERROR: parent span is not a superset of domain->span\n");
+	}
+}
+
+static int sched_cpu_show(struct seq_file *m, void *unused)
+{
+	struct sched_domain *sd;
+	int cpu = (int) ((long) m->private);
+
+	if (cpu < 0 || cpu > CONFIG_NR_CPUS) {
+		seq_printf(m, "invalid CPU, %d\n", cpu);
+		return 0;
+	}
+
+	for_each_domain(cpu, sd) {
+		sched_cpu_domain_show(m, sd, cpu);
+		seq_puts(m, "\n");
+	}
+
+	return 0;
+}
+
+static int sched_cpu_open(struct inode *inode, struct file *filp)
+{
+	return single_open(filp, sched_cpu_show, inode->i_private);
+}
+static const struct file_operations sched_cpu_fops = {
+	.open		= sched_cpu_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+static struct dentry *d_sched_debug;
+static struct dentry *d_sched_cpu[CONFIG_NR_CPUS];
+
+static int sched_debugfs_add_cpu(int cpu)
+{
+	char buf[32];
+	long lcpu = cpu;
+
+	snprintf(buf, sizeof(buf), "cpu%d", cpu);
+	d_sched_cpu[cpu] = debugfs_create_file(buf, 0444, d_sched_debug,
+						(void *) lcpu, &sched_cpu_fops);
+
+	if (d_sched_cpu[cpu] == NULL)
+		pr_warn("Failed to create debugfs entry for cpu %d\n", cpu);
+
+	return 0;
+}
+
 static __init int sched_init_debug(void)
 {
-	debugfs_create_file("sched_features", 0644, NULL, NULL,
+	int cpu;
+	int rc = 0;
+
+	d_sched_debug = debugfs_create_dir("sched", NULL);
+	if (!d_sched_debug) {
+		pr_warn("Could not create debugfs 'sched' entry\n");
+		return 0;
+	}
+
+	debugfs_create_file("features", 0644, d_sched_debug, NULL,
 			&sched_feat_fops);
 
-	return 0;
+	for_each_online_cpu(cpu) {
+		rc = sched_debugfs_add_cpu(cpu);
+		if (rc)
+			goto out;
+	}
+
+out:
+	return rc;
 }
 late_initcall(sched_init_debug);
 #endif /* CONFIG_SCHED_DEBUG */
@@ -6689,7 +6850,7 @@ struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
 
 		if (!cpumask_subset(sched_domain_span(child),
 				    sched_domain_span(sd))) {
-			pr_err("BUG: arch topology borken\n");
+			pr_err("BUG: arch topology broken\n");
 #ifdef CONFIG_SCHED_DEBUG
 			pr_err("     the %s domain not a subset of the %s domain\n",
 					child->name, sd->name);
-- 
2.3.0

Re: [RFC PATCH] sched: Add cpu based entries to debugfs

[Mike Galbraith]

On Sun, 2015-03-29 at 22:13 -0400, David Ahern wrote:

> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
> index 62671f53202a..b4d8d0c8260e 100644
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -268,12 +268,173 @@ static const struct file_operations sched_feat_fops = {
>  	.release	= single_release,
>  };
>  
> +static const char * const sd_flag_names[] = {
> +	"load-balance",
> +	"new-idle",
> +	"exec",
> +	"fork",
> +	"wake",
> +	"affine",
> +	"",
> +	"cpu-capacity",
> +	"power-domain",
> +	"share-pkg-resources",
> +	"serialize",
> +	"asym-packing",
> +	"prefer-sibling",
> +	"overlap",
> +	"numa",
> +	"",
> +};

That's wrong with the names readers will want to grep for?

	-Mike

Re: [RFC PATCH] sched: Add cpu based entries to debugfs

[David Ahern]

On 3/29/15 8:32 PM, Mike Galbraith wrote:
> On Sun, 2015-03-29 at 22:13 -0400, David Ahern wrote:
>
>> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
>> index 62671f53202a..b4d8d0c8260e 100644
>> --- a/kernel/sched/core.c
>> +++ b/kernel/sched/core.c
>> @@ -268,12 +268,173 @@ static const struct file_operations sched_feat_fops = {
>>   	.release	= single_release,
>>   };
>>
>> +static const char * const sd_flag_names[] = {
>> +	"load-balance",
>> +	"new-idle",
>> +	"exec",
>> +	"fork",
>> +	"wake",
>> +	"affine",
>> +	"",
>> +	"cpu-capacity",
>> +	"power-domain",
>> +	"share-pkg-resources",
>> +	"serialize",
>> +	"asym-packing",
>> +	"prefer-sibling",
>> +	"overlap",
>> +	"numa",
>> +	"",
>> +};
>
> That's wrong with the names readers will want to grep for?

I just found lower-case easier on the eyes. I could change this list to 
correspond to the SD_XXXXX defines. Certainly something that more 
programmatically correlates the macros (bit positions) and the names 
would be better.

David

Re: [RFC PATCH] sched: Add cpu based entries to debugfs

[Mike Galbraith]

Oh, you can add /proc/sys/kernel/sched_domain diddling to the hotplug
todo... which will either create read /sys, poke modify buttons over
yonder in /proc situation, or have to duplicate that.

	-Mike

Re: [RFC PATCH] sched: Add cpu based entries to debugfs

[Peter Zijlstra]

On Sun, Mar 29, 2015 at 10:13:33PM -0400, David Ahern wrote:
> Before spending too much time formalizing this I wanted to see if you guys
> would entertain the idea of making this info available via debugfs. It does
> move the existing sched_features file to sched/features -- not sure how 
> acceptable it is to move files in debugfs.

There also already is a /proc/sys/kernel/sched_domain/ sysctl thing that
has much of the stuff in.

I'm not opposed to a debugfs thing, but only if we can take the sysctl
thing out.

Re: [RFC PATCH] sched: Add cpu based entries to debugfs

[Mike Galbraith]

On Mon, 2015-03-30 at 09:44 +0200, Peter Zijlstra wrote:
> On Sun, Mar 29, 2015 at 10:13:33PM -0400, David Ahern wrote:
> > Before spending too much time formalizing this I wanted to see if you guys
> > would entertain the idea of making this info available via debugfs. It does
> > move the existing sched_features file to sched/features -- not sure how 
> > acceptable it is to move files in debugfs.
> 
> There also already is a /proc/sys/kernel/sched_domain/ sysctl thing that
> has much of the stuff in.
> 
> I'm not opposed to a debugfs thing, but only if we can take the sysctl
> thing out.

Ew, I can see the regression reports now.  That stuff has long roots.

	-Mike

Re: [RFC PATCH] sched: Add cpu based entries to debugfs

[Peter Zijlstra]

On Mon, Mar 30, 2015 at 10:26:03AM +0200, Mike Galbraith wrote:
> On Mon, 2015-03-30 at 09:44 +0200, Peter Zijlstra wrote:
> > On Sun, Mar 29, 2015 at 10:13:33PM -0400, David Ahern wrote:
> > > Before spending too much time formalizing this I wanted to see if you guys
> > > would entertain the idea of making this info available via debugfs. It does
> > > move the existing sched_features file to sched/features -- not sure how 
> > > acceptable it is to move files in debugfs.
> > 
> > There also already is a /proc/sys/kernel/sched_domain/ sysctl thing that
> > has much of the stuff in.
> > 
> > I'm not opposed to a debugfs thing, but only if we can take the sysctl
> > thing out.
> 
> Ew, I can see the regression reports now.  That stuff has long roots.

I didn't say we could take it out, I only said it would maybe be nice to
be able to :-) But yah. It'll cause some pain if we do :/

Re: [RFC PATCH] sched: Add cpu based entries to debugfs

[Mike Galbraith]

On Mon, 2015-03-30 at 10:28 +0200, Peter Zijlstra wrote:
> On Mon, Mar 30, 2015 at 10:26:03AM +0200, Mike Galbraith wrote:
> > On Mon, 2015-03-30 at 09:44 +0200, Peter Zijlstra wrote:
> > > On Sun, Mar 29, 2015 at 10:13:33PM -0400, David Ahern wrote:
> > > > Before spending too much time formalizing this I wanted to see if you guys
> > > > would entertain the idea of making this info available via debugfs. It does
> > > > move the existing sched_features file to sched/features -- not sure how 
> > > > acceptable it is to move files in debugfs.
> > > 
> > > There also already is a /proc/sys/kernel/sched_domain/ sysctl thing that
> > > has much of the stuff in.
> > > 
> > > I'm not opposed to a debugfs thing, but only if we can take the sysctl
> > > thing out.
> > 
> > Ew, I can see the regression reports now.  That stuff has long roots.
> 
> I didn't say we could take it out, I only said it would maybe be nice to
> be able to :-) But yah. It'll cause some pain if we do :/

I think it would be a good thing if we can get away with it, but I also
think you could safely bet your life that somebody will squeak.

	-Mike

Re: [RFC PATCH] sched: Add cpu based entries to debugfs

[Peter Zijlstra]

On Mon, Mar 30, 2015 at 10:43:14AM +0200, Mike Galbraith wrote:
> I think it would be a good thing if we can get away with it, but I also
> think you could safely bet your life that somebody will squeak.

The thing I worry most about is that squeaking only happening 5 years
later :/

Re: [RFC PATCH] sched: Add cpu based entries to debugfs

[David Ahern]

On 3/30/15 1:44 AM, Peter Zijlstra wrote:
> On Sun, Mar 29, 2015 at 10:13:33PM -0400, David Ahern wrote:
>> Before spending too much time formalizing this I wanted to see if you guys
>> would entertain the idea of making this info available via debugfs. It does
>> move the existing sched_features file to sched/features -- not sure how
>> acceptable it is to move files in debugfs.
>
> There also already is a /proc/sys/kernel/sched_domain/ sysctl thing that
> has much of the stuff in.
>
> I'm not opposed to a debugfs thing, but only if we can take the sysctl
> thing out.
>

Related files I aware of; perhaps there are others:

- /sys/devices/system/cpu entries. e.g., for cpu topology (physical 
package id, core id, sibling cores and threads)

- debufs file for sched_features

- /proc/sys/kernel/sched_domain for tweaking scheduling parameters

I could not find anything that shows the sched_domain information. The 
proposal here was to create read-only debugfs files to dump the same 
info (but with better readability for large systems) that comes with the 
sched_debug parameter. I chose debugfs because a) sched_features is 
there and b) it is debug information about the scheduler.

I could just as easily out it under /proc/sys/kernel/sched_domain or 
create a sched subdir under /sys/devices/system/cpu/cpu$n if one of 
those is preferred.

David

Re: [RFC PATCH] sched: Add cpu based entries to debugfs

[David Ahern]

On 3/29/15 9:08 PM, Mike Galbraith wrote:
>
> Oh, you can add /proc/sys/kernel/sched_domain diddling to the hotplug
> todo... which will either create read /sys, poke modify buttons over
> yonder in /proc situation, or have to duplicate that.

-EPARSE.

you mean the /proc/sys/kernel/sched_domain/ files do not handle hotplug?

Can I get --verbose on what you mean above?

David

Re: [RFC PATCH] sched: Add cpu based entries to debugfs

[Mike Galbraith]

On Mon, 2015-03-30 at 07:03 -0600, David Ahern wrote:
> On 3/29/15 9:08 PM, Mike Galbraith wrote:
> >
> > Oh, you can add /proc/sys/kernel/sched_domain diddling to the hotplug
> > todo... which will either create read /sys, poke modify buttons over
> > yonder in /proc situation, or have to duplicate that.
> 
> -EPARSE.
> 
> you mean the /proc/sys/kernel/sched_domain/ files do not handle hotplug?
> 
> Can I get --verbose on what you mean above?

I meant hotplug isn't the only thing that alters what you display, and
the display is disjointed from the knobs, user acquires pretty display
in /sys, but the knobs to twiddle what he's looking at are elsewhere.

	-Mike

Re: [RFC PATCH] sched: Add cpu based entries to debugfs

[Ingo Molnar]

* Peter Zijlstra <peterz@infradead.org> wrote:

> On Mon, Mar 30, 2015 at 10:43:14AM +0200, Mike Galbraith wrote:
>
> > I think it would be a good thing if we can get away with it, but I 
> > also think you could safely bet your life that somebody will 
> > squeak.
> 
> The thing I worry most about is that squeaking only happening 5 
> years later :/

So lets start by keeping the sysctl thing with the very 
scheduler-internal names, but all zeroes and no effect of any change - 
i.e. a dead API in all but appearance. I don't think there's any 
legitimate use of those, beyond debugging, as we could change the 
internal implementation anymore and moot many of those flags.

So lets trigger the squeaking that way. If any complaint comes in 
beyond 1-2 kernel releases then I don't think it's a regression, it 
turns into a feature request ...

Thanks,

        Ingo

Re: [RFC PATCH] sched: Add cpu based entries to debugfs

[David Ahern]

On 3/31/15 3:13 AM, Ingo Molnar wrote:
>
> * Peter Zijlstra <peterz@infradead.org> wrote:
>
>> On Mon, Mar 30, 2015 at 10:43:14AM +0200, Mike Galbraith wrote:
>>
>>> I think it would be a good thing if we can get away with it, but I
>>> also think you could safely bet your life that somebody will
>>> squeak.
>>
>> The thing I worry most about is that squeaking only happening 5
>> years later :/
>
> So lets start by keeping the sysctl thing with the very
> scheduler-internal names, but all zeroes and no effect of any change -
> i.e. a dead API in all but appearance. I don't think there's any
> legitimate use of those, beyond debugging, as we could change the
> internal implementation anymore and moot many of those flags.
>
> So lets trigger the squeaking that way. If any complaint comes in
> beyond 1-2 kernel releases then I don't think it's a regression, it
> turns into a feature request ...

Sorry to be so dense but I am not clear on what is acceptable and not. 
As mentioned in a previous response, these are the scheduler related 
files I am aware of:

- debufs file for sched_features (/sys/kernel/debug/sched_features)

- /proc/sys/kernel/sched_domain for tweaking scheduling parameters

- /proc/sched_debug - various internal stats

- /sys/devices/system/cpu entries. e.g., for cpu topology (physical 
package id, core id, sibling cores and threads)

None of them show the sched_domain information which is the subject of 
this patch.

Can someone clarify on the duplication concerns and such?

Thanks,
David

Re: [RFC PATCH] sched: Add cpu based entries to debugfs

[Mike Galbraith]

On Sun, 2015-04-05 at 20:10 -0600, David Ahern wrote:
> 
> Can someone clarify on the duplication concerns and such?

My concern was eg you display flags in /sys, but to tweak flags etc, 
you must visit /proc.  Seems to me the missing info should either be 
added to /proc (some under DEBUG?), or the lot should move to /sys.

I don't care all that much, making span etc visible anywhere is an win 
over boot flag + big box (esp. + serial console).

        -Mike