[PATCH v3 0/5] trace-cmd: Improve multi-instance support

[PATCH v3 0/5] trace-cmd: Improve multi-instance support

[Howard Cochran]

NOTE: The previous two versions of this patch set were distributed off-list.

v3 changes:
    * Improved clarification of options in documentation of
      trace-cmd reset and added examples.

v2 changes:
    * Rebased this patch series on Steve's simplification of
      first_instance logic.

Improve the completeness and consistency of support for multiple
buffer instances in the extract, snapshot, stat, start, stop,
reset, and restart sub-commands.

For the commands that lacked it, added support for -B and -t options,
consistent with the existing behavior of stop, reset, and restart.
That is, including a -B instance-name option will cause these commands to
exclude the top buffer instance unless -t is also provided.

Exception: trace-cmd start retains its existing behavior regarding the
top instance, consistent with trace-cmd record.

For all of these sub-commands except start, add a new -a option, which
causes it to operate on all existing instances.  As with -B, -a will
exclude the top instance unless -t is also given.

Updated documentation accordingly.


Howard Cochran (5):
  trace-cmd extract: Add -B and -t for multiple instances
  trace-cmd extract: Add -a option for all instances
  trace-cmd stop, reset, restart: Add -a for all instances
  trace-cmd snapshot: Fully support multiple instances
  trace-cmd stat: Support -a for all buffers. Update docs.

 Documentation/trace-cmd-extract.1.txt  |  16 ++++
 Documentation/trace-cmd-reset.1.txt    |  57 ++++++++++--
 Documentation/trace-cmd-snapshot.1.txt |  16 +++-
 Documentation/trace-cmd-stat.1.txt     |  19 ++++
 Documentation/trace-cmd-stop.1.txt     |  13 ++-
 trace-cmd.c                            |   2 +-
 trace-local.h                          |   6 +-
 trace-record.c                         | 160 ++++++++++++++++++++++++++++-----
 trace-snapshot.c                       |  90 ++++++++++---------
 trace-stat.c                           |  15 +++-
 trace-usage.c                          |  42 ++++++---
 11 files changed, 341 insertions(+), 95 deletions(-)

-- 
1.9.1

[PATCH v3 1/5] trace-cmd extract: Add -B and -t for multiple instances

[Howard Cochran]

Add -B option to trace-cmd extract so that it can extract listed
instances. As with start, stop, and reset, the top level instance is not
included when any -B are given. However, -t makes it also include the
top instance. As with other commands, if neither option is given, only
operate on the top instance.

Signed-off-by: Howard Cochran <hcochran@kernelspring.com>
---
 trace-record.c | 20 +++++++++++++-------
 trace-usage.c  |  6 ++++--
 2 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/trace-record.c b/trace-record.c
index 6e0ab3b..8340416 100644
--- a/trace-record.c
+++ b/trace-record.c
@@ -523,17 +523,20 @@ static int create_recorder(struct buffer_instance *instance, int cpu,
 
 static void flush_threads(void)
 {
+	struct buffer_instance *instance;
 	long ret;
 	int i;
 
 	if (!cpu_count)
 		return;
 
-	for (i = 0; i < cpu_count; i++) {
-		/* Extract doesn't support sub buffers yet */
-		ret = create_recorder(&top_instance, i, TRACE_TYPE_EXTRACT, NULL);
-		if (ret < 0)
-			die("error reading ring buffer");
+	for_all_instances(instance) {
+		for (i = 0; i < cpu_count; i++) {
+			/* Extract doesn't support sub buffers yet */
+			ret = create_recorder(instance, i, TRACE_TYPE_EXTRACT, NULL);
+			if (ret < 0)
+				die("error reading ring buffer");
+		}
 	}
 }
 
@@ -3873,7 +3876,7 @@ void trace_record (int argc, char **argv)
 		};
 
 		if (extract)
-			opts = "+haf:Fp:co:O:sr:g:l:n:P:N:tb:ksiT";
+			opts = "+haf:Fp:co:O:sr:g:l:n:P:N:tb:B:ksiT";
 		else
 			opts = "+hae:f:Fp:cC:dDGo:O:s:r:vg:l:n:P:N:tb:R:B:ksSiTm:M:H:";
 		c = getopt_long (argc-1, argv+1, opts, long_options, &option_index);
@@ -4061,7 +4064,10 @@ void trace_record (int argc, char **argv)
 			instance->cpumask = optarg;
 			break;
 		case 't':
-			use_tcp = 1;
+			if (extract)
+				topt = 1; /* Extract top instance also */
+			else
+				use_tcp = 1;
 			break;
 		case 'b':
 			instance->buffer_size = atoi(optarg);
diff --git a/trace-usage.c b/trace-usage.c
index 520b14b..a708a85 100644
--- a/trace-usage.c
+++ b/trace-usage.c
@@ -62,9 +62,11 @@ static struct usage_help usage_help[] = {
 	{
 		"extract",
 		"extract a trace from the kernel",
-		" %s extract [-p plugin][-O option][-o file][-s]\n"
-		"          Uses same options as record, but only reads an existing trace.\n"
+		" %s extract [-p plugin][-O option][-o file][-B buf][-s][-t]\n"
+		"          Uses similar options as record, but only reads an existing trace.\n"
 		"          -s : extract the snapshot instead of the main buffer\n"
+		"          -B : extract a given buffer (more than one may be specified)\n"
+		"          -t : include the top level buffer (useful with -B)\n"
 	},
 	{
 		"stop",
-- 
1.9.1

[PATCH v3 2/5] trace-cmd extract: Add -a option for all instances

[Howard Cochran]

This option will extract all instances that currently exist in the
system, including the top instance. This differs from the meaning of -a
for record and stream (enable all events), which would have no purpose
for extract. Such difference in meaning already exists for -s, so this
seemed reasonable.

Had to fix a bug in create_instance(), in which it ignored its parameter.

Signed-off-by: Howard Cochran <hcochran@kernelspring.com>
---
 Documentation/trace-cmd-extract.1.txt | 16 +++++++
 trace-local.h                         |  2 +-
 trace-record.c                        | 81 +++++++++++++++++++++++++++++++++--
 trace-usage.c                         |  5 ++-
 4 files changed, 98 insertions(+), 6 deletions(-)

diff --git a/Documentation/trace-cmd-extract.1.txt b/Documentation/trace-cmd-extract.1.txt
index ea98c20..73eda4e 100644
--- a/Documentation/trace-cmd-extract.1.txt
+++ b/Documentation/trace-cmd-extract.1.txt
@@ -52,6 +52,22 @@ OPTIONS
     does cause the extract routine to disable all tracing. That is,
     the end of the extract will perform something similar to trace-cmd-reset(1).
 
+*-B* 'buffer-name'::
+    If the kernel supports multiple buffers, this will extract the trace for
+    only the given buffer. It does not affect any other buffer. This may be
+    used multiple times to specify different buffers. When this option is
+    used, the top level instance will not be extracted unless *-t* is given.
+
+*-a*::
+    Extract all existing buffer instances. When this option is used, the
+    top level instance will not be extracted unless *-t* is given.
+
+*-t*::
+    Extracts the top level instance buffer. Without the *-B* or *-a* option
+    this is the same as the default. But if *-B* or *-a* is used, this is
+    required if the top level instance buffer should also be extracted.
+
+
 SEE ALSO
 --------
 trace-cmd(1), trace-cmd-record(1), trace-cmd-report(1), trace-cmd-start(1),
diff --git a/trace-local.h b/trace-local.h
index ab2bd34..fb2ce71 100644
--- a/trace-local.h
+++ b/trace-local.h
@@ -176,7 +176,7 @@ extern struct buffer_instance *first_instance;
 #define for_all_instances(i) for (i = first_instance; i; \
 				  i = i == &top_instance ? buffer_instances : (i)->next)
 
-struct buffer_instance *create_instance(char *name);
+struct buffer_instance *create_instance(const char *name);
 void add_instance(struct buffer_instance *instance);
 char *get_instance_file(struct buffer_instance *instance, const char *file);
 
diff --git a/trace-record.c b/trace-record.c
index 8340416..4a78fff 100644
--- a/trace-record.c
+++ b/trace-record.c
@@ -294,18 +294,91 @@ void add_instance(struct buffer_instance *instance)
  * Returns a newly allocated instance. Note that @name will not be
  * copied, and the instance buffer will point to the string itself.
  */
-struct buffer_instance *create_instance(char *name)
+struct buffer_instance *create_instance(const char *name)
 {
 	struct buffer_instance *instance;
 
 	instance = malloc_or_die(sizeof(*instance));
 	memset(instance, 0, sizeof(*instance));
-	instance->name = optarg;
+	instance->name = name;
 
 	return instance;
 }
 
 /**
+ * add_all_instances - Add all pre-existing instances to the internal list
+ * @tracing_dir: The top-level tracing directory
+ *
+ * Returns whether the operation succeeded
+ */
+int add_all_instances(const char *tracing_dir)
+{
+	struct dirent *dent;
+	char *instances_dir;
+	struct stat st;
+	DIR *dir;
+	int ret;
+
+	if (!tracing_dir)
+		return -1;
+
+	instances_dir = append_file(tracing_dir, "instances");
+	if (!instances_dir)
+		return -1;
+
+	ret = stat(instances_dir, &st);
+	if (ret < 0 || !S_ISDIR(st.st_mode)) {
+		ret = -1;
+		goto out_free;
+	}
+
+	dir = opendir(instances_dir);
+	if (!dir) {
+		ret = -1;
+		goto out_free;
+	}
+
+	while ((dent = readdir(dir))) {
+		const char *name = strdup(dent->d_name);
+		char *instance_path;
+		struct buffer_instance *instance;
+
+		if (strcmp(name, ".") == 0 ||
+		    strcmp(name, "..") == 0)
+			continue;
+
+		instance_path = append_file(instances_dir, name);
+		ret = stat(instance_path, &st);
+		if (ret < 0 || !S_ISDIR(st.st_mode)) {
+			free(instance_path);
+			continue;
+		}
+		free(instance_path);
+
+		instance = create_instance(name);
+		add_instance(instance);
+	}
+
+	closedir(dir);
+	ret = 0;
+
+ out_free:
+	free(instances_dir);
+	return ret;
+}
+
+void add_all_instances(void)
+{
+	char *tracing_dir = tracecmd_find_tracing_dir();
+	if (!tracing_dir)
+		die("malloc");
+
+	__add_all_instances(tracing_dir);
+
+	tracecmd_put_tracing_file(tracing_dir);
+}
+
+/**
  * tracecmd_stat_cpu - show the buffer stats of a particular CPU
  * @s: the trace_seq to record the data in.
  * @cpu: the CPU to stat
@@ -3887,7 +3960,9 @@ void trace_record (int argc, char **argv)
 			usage(argv);
 			break;
 		case 'a':
-			if (!extract) {
+			if (extract) {
+				add_all_instances();
+			} else {
 				record_all = 1;
 				record_all_events();
 			}
diff --git a/trace-usage.c b/trace-usage.c
index a708a85..39de386 100644
--- a/trace-usage.c
+++ b/trace-usage.c
@@ -62,11 +62,12 @@ static struct usage_help usage_help[] = {
 	{
 		"extract",
 		"extract a trace from the kernel",
-		" %s extract [-p plugin][-O option][-o file][-B buf][-s][-t]\n"
+		" %s extract [-p plugin][-O option][-o file][-B buf][-s][-a][-t]\n"
 		"          Uses similar options as record, but only reads an existing trace.\n"
 		"          -s : extract the snapshot instead of the main buffer\n"
 		"          -B : extract a given buffer (more than one may be specified)\n"
-		"          -t : include the top level buffer (useful with -B)\n"
+		"          -a : extract all buffers (except top one)\n"
+		"          -t : extract the top level buffer (useful with -B and -a)\n"
 	},
 	{
 		"stop",
-- 
1.9.1

[PATCH v3 3/5] trace-cmd stop, reset, restart: Add -a for all instances

[Howard Cochran]

As for extract, -a means all instances except the top one.
Use -t to get the top one too.

Signed-off-by: Howard Cochran <hcochran@kernelspring.com>
---
 Documentation/trace-cmd-reset.1.txt | 57 +++++++++++++++++++++++++++++++++----
 Documentation/trace-cmd-stop.1.txt  | 13 ++++++---
 trace-record.c                      | 57 ++++++++++++++++++++++++++++++-------
 trace-usage.c                       | 21 ++++++++------
 4 files changed, 119 insertions(+), 29 deletions(-)

diff --git a/Documentation/trace-cmd-reset.1.txt b/Documentation/trace-cmd-reset.1.txt
index 06a60af..bff4530 100644
--- a/Documentation/trace-cmd-reset.1.txt
+++ b/Documentation/trace-cmd-reset.1.txt
@@ -22,6 +22,9 @@ the data in the ring buffer, and the options that were used are all lost.
 
 OPTIONS
 -------
+Please note that the order that options are specified on the command line is
+significant. See EXAMPLES.
+
 *-b* 'buffer_size'::
     When the kernel boots, the Ftrace ring buffer is of a minimal size (3
     pages per CPU). The first time the tracer is used, the ring buffer size
@@ -32,6 +35,17 @@ OPTIONS
 
     trace-cmd reset -b 1
 
+    The buffer instance affected is the one (or ones) specified by the most
+    recently preceding *-B*, *-t*, or *-a* option:
+
+    When used after *-B*, resizes the buffer instance that precedes it on
+    the command line.
+
+    When used after *-a*, resizes all buffer instances except the top one.
+
+    When used after *-t* or before any *-B* or *-a*, resizes the top
+    instance.
+
 *-B* 'buffer-name'::
     If the kernel supports multiple buffers, this will reset the trace for
     only the given buffer. It does not affect any other buffer. This may be
@@ -39,15 +53,46 @@ OPTIONS
     will not be reset if this option is given (unless the *-t* option is
     also supplied).
 
+*-a*::
+    Reset the trace for all existing buffer instances. When this option
+    is used, the top level instance will not be reset unless *-t* is given.
+
 *-d*::
-    Delete the previously specified buffer instance that was specified by
-    the *-B* option. It is invalid to use with *-t* as the top level instance
-    can not be deleted.
+    This option deletes the instance buffer(s) specified by the most recently
+    preceding *-B* or *-a* option. Because the top-level instance buffer
+    cannot be deleted, it is invalid to use this immediatly following *-t* or
+    prior to any *-B* or *-a* option on the command line.
 
 *-t*::
-    Resets the top level instance buffer. Without the *-B* option this is the
-    same as the default. But if *-B* is used, this is required if the top
-    level instance buffer should also be reset.
+    Resets the top level instance buffer. Without the *-B* or *-a* option
+    this is the same as the default. But if *-B* or *-a* is used, this is
+    required if the top level instance buffer should also be reset.
+
+EXAMPLES
+--------
+
+Reset tracing for instance-one and set its per-cpu buffer size to 4096kb.
+Also deletes instance-two. The top level instance and any other instances
+remain unaffected:
+
+    trace-cmd reset -B instance-one -b 4096 -B instance-two -d
+
+Delete all instance buffers. Top level instance remains unaffected:
+
+    trace-cmd reset -a -d
+
+Delete all instance buffers and also reset the top instance:
+
+    trace-cmd reset -t -a -d
+
+Invalid. This command implies an attempt to delete the top instance:
+
+    trace-cmd reset -a -t -d
+
+Reset the top instance and set its per-cpu buffer size to 1024kb. If any
+instance buffers exist, they will be unaffected:
+
+    trace-cmd reset -b 1024
 
 
 SEE ALSO
diff --git a/Documentation/trace-cmd-stop.1.txt b/Documentation/trace-cmd-stop.1.txt
index c3f8b33..9d4093e 100644
--- a/Documentation/trace-cmd-stop.1.txt
+++ b/Documentation/trace-cmd-stop.1.txt
@@ -30,12 +30,17 @@ OPTIONS
 *-B* 'buffer-name'::
     If the kernel supports multiple buffers, this will stop the trace for
     only the given buffer. It does not affect any other buffer. This may be
-    used multiple times to specify different buffers.
+    used multiple times to specify different buffers. When this option is
+    used, the top level instance will not be stopped unless *-t* is given.
+
+*-a*::
+    Stop the trace for all existing buffer instances. When this option
+    is used, the top level instance will not be stopped unless *-t* is given.
 
 *-t*::
-    Stops the top level instance buffer. Without the *-B* option this is the
-    same as the default. But if *-B* is used, this is required if the top
-    level instance buffer should also be stopped.
+    Stops the top level instance buffer. Without the *-B* or *-a* option this
+    is the same as the default. But if *-B* or *-a* is used, this is
+    required if the top level instance buffer should also be stopped.
 
 SEE ALSO
 --------
diff --git a/trace-record.c b/trace-record.c
index 4a78fff..7c6b7e9 100644
--- a/trace-record.c
+++ b/trace-record.c
@@ -311,7 +311,7 @@ struct buffer_instance *create_instance(const char *name)
  *
  * Returns whether the operation succeeded
  */
-int add_all_instances(const char *tracing_dir)
+int __add_all_instances(const char *tracing_dir)
 {
 	struct dirent *dent;
 	char *instances_dir;
@@ -3844,7 +3844,7 @@ void trace_record (int argc, char **argv)
 		for (;;) {
 			int c;
 
-			c = getopt(argc-1, argv+1, "tB:");
+			c = getopt(argc-1, argv+1, "hatB:");
 			if (c == -1)
 				break;
 			switch (c) {
@@ -3855,6 +3855,9 @@ void trace_record (int argc, char **argv)
 				instance = create_instance(optarg);
 				add_instance(instance);
 				break;
+			case 'a':
+				add_all_instances();
+				break;
 			case 't':
 				/* Force to use top instance */
 				topt = 1;
@@ -3872,7 +3875,7 @@ void trace_record (int argc, char **argv)
 		for (;;) {
 			int c;
 
-			c = getopt(argc-1, argv+1, "tB:");
+			c = getopt(argc-1, argv+1, "hatB:");
 			if (c == -1)
 				break;
 			switch (c) {
@@ -3883,6 +3886,9 @@ void trace_record (int argc, char **argv)
 				instance = create_instance(optarg);
 				add_instance(instance);
 				break;
+			case 'a':
+				add_all_instances();
+				break;
 			case 't':
 				/* Force to use top instance */
 				topt = 1;
@@ -3897,16 +3903,33 @@ void trace_record (int argc, char **argv)
 		enable_tracing();
 		exit(0);
 	} else if (strcmp(argv[1], "reset") == 0) {
+		/* if last arg is -a, then -b and -d apply to all instances */
+		int last_specified_all = 0;
+		struct buffer_instance *inst; /* iterator */
+
+		while ((c = getopt(argc-1, argv+1, "hab:B:td")) >= 0) {
 
-		while ((c = getopt(argc-1, argv+1, "b:B:td")) >= 0) {
 			switch (c) {
+			case 'h':
+				usage(argv);
+				break;
 			case 'b':
-				instance->buffer_size = atoi(optarg);
+			{
+				int size = atoi(optarg);
 				/* Min buffer size is 1 */
-				if (strcmp(optarg, "0") == 0)
-					instance->buffer_size = 1;
+				if (size <= 1)
+					size = 1;
+				if (last_specified_all) {
+					for_each_instance(inst) {
+						inst->buffer_size = size;
+					}
+				} else {
+					instance->buffer_size = size;
+				}
 				break;
+			}
 			case 'B':
+				last_specified_all = 0;
 				instance = create_instance(optarg);
 				add_instance(instance);
 				/* -d will remove keep */
@@ -3914,13 +3937,27 @@ void trace_record (int argc, char **argv)
 				break;
 			case 't':
 				/* Force to use top instance */
+				last_specified_all = 0;
 				topt = 1;
 				instance = &top_instance;
 				break;
+			case 'a':
+				last_specified_all = 1;
+				add_all_instances();
+				for_each_instance(instance) {
+					instance->keep = 1;
+				}
+				break;
 			case 'd':
-				if (is_top_instance(instance))
-					die("Can not delete top level buffer");
-				instance->keep = 0;
+				if (last_specified_all) {
+					for_each_instance(inst) {
+						inst->keep = 0;
+					}
+				} else {
+					if (is_top_instance(instance))
+						die("Can not delete top level buffer");
+					instance->keep = 0;
+				}
 				break;
 			}
 		}
diff --git a/trace-usage.c b/trace-usage.c
index 39de386..c56661b 100644
--- a/trace-usage.c
+++ b/trace-usage.c
@@ -72,20 +72,22 @@ static struct usage_help usage_help[] = {
 	{
 		"stop",
 		"stop the kernel from recording trace data",
-		" %s stop [-B buf [-B buf]..] [-t]\n"
+		" %s stop [-B buf [-B buf]..] [-a] [-t]\n"
 		"          Stops the tracer from recording more data.\n"
 		"          Used in conjunction with start\n"
-		"          -B stop a given buffer (more than one may be specified)\n "
-		"          -t stop the top level buffer (needed if -B is specified)\n"
+		"          -B stop a given buffer (more than one may be specified)\n"
+		"          -a stop all buffers (except top one)\n"
+		"          -t stop the top level buffer (useful with -B or -a)\n"
 	},
 	{
 		"restart",
 		"restart the kernel trace data recording",
-		" %s restart [-B buf [-B buf]..] [-t]\n"
+		" %s restart [-B buf [-B buf]..] [-a] [-t]\n"
 		"          Restarts recording after a trace-cmd stop.\n"
 		"          Used in conjunction with stop\n"
-		"          -B restart a given buffer (more than one may be specified)\n "
-		"          -t restart the top level buffer (needed if -B is specified)\n"
+		"          -B restart a given buffer (more than one may be specified)\n"
+		"          -a restart all buffers (except top one)\n"
+		"          -t restart the top level buffer (useful with -B or -a)\n"
 	},
 	{
 		"show",
@@ -114,13 +116,14 @@ static struct usage_help usage_help[] = {
 	{
 		"reset",
 		"disable all kernel tracing and clear the trace buffers",
-		" %s reset [-b size][-B buf][-d][-t]\n"
+		" %s reset [-b size][-B buf][-a][-d][-t]\n"
 		"          Disables the tracer (may reset trace file)\n"
 		"          Used in conjunction with start\n"
 		"          -b change the kernel buffer size (in kilobytes per CPU)\n"
-		"          -B reset the given buffer instance (top instance ignored)\n"
 		"          -d delete the previous specified instance\n"
-		"          -t still reset the top instance if -B option is given\n"
+		"          -B reset the given buffer instance (may specify multiple -B)\n"
+		"          -a reset all instances (except top one)\n"
+		"          -t reset the top level instance (useful with -B or -a)\n"
 	},
 	{
 		"report",
-- 
1.9.1

[PATCH v3 4/5] trace-cmd snapshot: Fully support multiple instances

[Howard Cochran]

Add support for multiple -B instances, the same as
stop, reset, restart, and extract sub-commands.
-a operates on all instances except the top one
-t also includes the top instance

Signed-off-by: Howard Cochran <hcochran@kernelspring.com>
---
 Documentation/trace-cmd-snapshot.1.txt | 16 +++++-
 trace-cmd.c                            |  2 +-
 trace-local.h                          |  4 ++
 trace-record.c                         |  4 +-
 trace-snapshot.c                       | 90 ++++++++++++++++++----------------
 trace-usage.c                          |  8 ++-
 6 files changed, 75 insertions(+), 49 deletions(-)

diff --git a/Documentation/trace-cmd-snapshot.1.txt b/Documentation/trace-cmd-snapshot.1.txt
index 98b8924..767ab67 100644
--- a/Documentation/trace-cmd-snapshot.1.txt
+++ b/Documentation/trace-cmd-snapshot.1.txt
@@ -41,8 +41,20 @@ OPTIONS
 
 *-B* 'buf'::
     If a buffer instance was created, then the *-B* option will operate on
-    the snapshot within the buffer.
-    
+    the snapshot within the buffer. This may be used multiple times to
+    specify different buffers. When this option is used, the top level
+    instance will be ignored unless *-t* is given.
+
+*-a*::
+    Operate on the snapshot for all existing buffer instances.
+    When this option is used, the top level instance will be ignored
+    unless *-t* is given.
+
+*-t*::
+    Operate on the snapshot within the top level instance buffer. Without
+    the *-B* or *-a* option this is the same as the default.  But if *-B*
+    or *-a* is used, this is required if the top level instance buffer
+    should also be affected.
 
 SEE ALSO
 --------
diff --git a/trace-cmd.c b/trace-cmd.c
index 4c5b564..647a2b4 100644
--- a/trace-cmd.c
+++ b/trace-cmd.c
@@ -72,7 +72,7 @@ void *malloc_or_die(unsigned int size)
 	return data;
 }
 
-static void dump_file_content(const char *path)
+void dump_file_content(const char *path)
 {
 	char buf[BUFSIZ];
 	ssize_t n;
diff --git a/trace-local.h b/trace-local.h
index fb2ce71..982fe6f 100644
--- a/trace-local.h
+++ b/trace-local.h
@@ -47,7 +47,9 @@ struct pid_record_data {
 	struct pevent_record	*record;
 };
 
+void dump_file_content(const char *path);
 void show_file(const char *name);
+int write_file(const char *file, const char *str, const char *type);
 
 struct tracecmd_input *read_trace_header(const char *file);
 int read_trace_files(void);
@@ -179,6 +181,8 @@ extern struct buffer_instance *first_instance;
 struct buffer_instance *create_instance(const char *name);
 void add_instance(struct buffer_instance *instance);
 char *get_instance_file(struct buffer_instance *instance, const char *file);
+void add_all_instances(void);
+void update_first_instance(int topt);
 
 void show_instance_file(struct buffer_instance *instance, const char *name);
 int count_cpus(void);
diff --git a/trace-record.c b/trace-record.c
index 7c6b7e9..ae0ebbf 100644
--- a/trace-record.c
+++ b/trace-record.c
@@ -1466,7 +1466,7 @@ static void reset_events(void)
 		reset_events_instance(instance);
 }
 
-static int write_file(const char *file, const char *str, const char *type)
+int write_file(const char *file, const char *str, const char *type)
 {
 	char buf[BUFSIZ];
 	int fd;
@@ -3778,7 +3778,7 @@ static void add_hook(struct buffer_instance *instance, const char *arg)
 	}
 }
 
-static void update_first_instance(int topt)
+void update_first_instance(int topt)
 {
 	if (topt || !buffer_instances)
 		first_instance = &top_instance;
diff --git a/trace-snapshot.c b/trace-snapshot.c
index 9d3608c..22f88df 100644
--- a/trace-snapshot.c
+++ b/trace-snapshot.c
@@ -29,27 +29,8 @@
 
 #include "trace-local.h"
 
-static void write_file(const char *name, char *val)
-{
-	char *path;
-	int fd;
-	ssize_t n;
-
-	path = tracecmd_get_tracing_file(name);
-	fd = open(path, O_WRONLY);
-	if (fd < 0)
-		die("writing %s", path);
-
-	n = write(fd, val, strlen(val));
-	if (n < 0)
-		die("failed to write '%d' to %s\n", path);
-	tracecmd_put_tracing_file(path);
-	close(fd);
-}
-
 void trace_snapshot (int argc, char **argv)
 {
-	const char *buffer = NULL;
 	const char *file = "snapshot";
 	struct stat st;
 	char *name;
@@ -58,8 +39,10 @@ void trace_snapshot (int argc, char **argv)
 	int reset_snap = 0;
 	int free_snap = 0;
 	int cpu = -1;
+	int topt = 0;
 	int ret;
 	int c;
+	struct buffer_instance *instance;
 
 	if (argc < 2)
 		usage(argv);
@@ -67,7 +50,7 @@ void trace_snapshot (int argc, char **argv)
 	if (strcmp(argv[1], "snapshot") != 0)
 		usage(argv);
 
-	while ((c = getopt(argc-1, argv+1, "srfB:c:")) >= 0) {
+	while ((c = getopt(argc-1, argv+1, "srfB:c:ta")) >= 0) {
 		switch (c) {
 		case 'h':
 			usage(argv);
@@ -86,9 +69,15 @@ void trace_snapshot (int argc, char **argv)
 			reset_snap = 1;
 			break;
 		case 'B':
-			if (buffer)
-				die("Can only do one buffer at a time");
-			buffer = optarg;
+			add_instance(create_instance(optarg));
+			break;
+		case 'a':
+			add_all_instances();
+			break;
+		case 't':
+			/* Include top instance (with -B or -a) */
+			topt = 1;
+			first_instance = &top_instance;
 			break;
 		case 'c':
 			if (cpu >= 0)
@@ -99,29 +88,46 @@ void trace_snapshot (int argc, char **argv)
 			usage(argv);
 		}
 	}
+	update_first_instance(topt);
 
-	if (cpu >= 0) {
-		snprintf(cpu_path, 128, "per_cpu/cpu%d/%s", cpu, file);
-		file = cpu_path;
-	}
+	for_all_instances(instance) {
+		name = get_instance_file(instance, "trace");
+		ret = stat(name, &st);
+		tracecmd_put_tracing_file(name);
+		if (ret < 0) {
+			printf("Ignoring non-existant instance: %s\n",
+			       instance->name);
+			continue;
+		}
 
-	name = tracecmd_get_tracing_file(file);
-	ret = stat(name, &st);
-	if (ret < 0)
-		die("Snapshot feature is not supported by this kernel");
-	tracecmd_put_tracing_file(name);
+		if (cpu >= 0) {
+			snprintf(cpu_path, 128, "per_cpu/cpu%d/%s", cpu, file);
+			file = cpu_path;
+		}
 
-	if (!reset_snap && !take_snap && !free_snap) {
-		show_file(file);
-		exit(0);
-	}
+		name = get_instance_file(instance, file);
+		ret = stat(name, &st);
+		if (ret < 0)
+			die("Snapshot feature is not supported by this kernel");
 
-	if (reset_snap)
-		write_file(file, "2");
+		if (!reset_snap && !take_snap && !free_snap) {
+			if (instance != &top_instance) {
+				printf("\nSnapshot for instance \"%s\"\n"
+				       "==================================\n",
+				       instance->name);
+			}
 
-	if (free_snap)
-		write_file(file, "0");
+			dump_file_content(name);
+		}
 
-	if (take_snap)
-		write_file(file, "1");
+		if (reset_snap)
+			write_file(name, "2", "snapshot");
+
+		if (free_snap)
+			write_file(name, "0", "snapshot");
+
+		if (take_snap)
+			write_file(name, "1", "snapshot");
+		tracecmd_put_tracing_file(name);
+	}
 }
diff --git a/trace-usage.c b/trace-usage.c
index c56661b..56c68b4 100644
--- a/trace-usage.c
+++ b/trace-usage.c
@@ -244,13 +244,17 @@ static struct usage_help usage_help[] = {
 	{
 		"snapshot",
 		"take snapshot of running trace",
-		" %s snapshot [-s][-r][-f][-B buf][-c cpu]\n"
+		" %s snapshot [-s][-r][-f][-c cpu][-B buf][-a][-t]\n"
 		"          -s take a snapshot of the trace buffer\n"
 		"          -r reset current snapshot\n"
 		"          -f free the snapshot buffer\n"
 		"            without the above three options, display snapshot\n"
 		"          -c operate on the snapshot buffer for the given CPU\n"
-		"          -B operate on the snapshot buffer for a tracing buffer instance.\n"
+		"          -B operate on the snapshot for a given instance\n"
+		"             (maybe be specified multiple times)\n"
+		"          -a operate on the snapshot for all instances\n"
+		"          -t also operate on the snapshot for top instance\n"
+		"             (useful with -B or -a)\n"
 	},
 	{
 		"stack",
-- 
1.9.1

[PATCH v3 5/5] trace-cmd stat: Support -a for all buffers. Update docs.

[Howard Cochran]

As for stop, restart, reset, extract, -a means show status of
all buffer instances except the top one. Use -t to get the top
one too.

Also added documentation for the existing -B and -t options.

Signed-off-by: Howard Cochran <hcochran@kernelspring.com>
---
 Documentation/trace-cmd-stat.1.txt | 19 +++++++++++++++++++
 trace-stat.c                       | 15 +++++++++++----
 trace-usage.c                      |  6 +++++-
 3 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/Documentation/trace-cmd-stat.1.txt b/Documentation/trace-cmd-stat.1.txt
index 7e4263e..62c2e18 100644
--- a/Documentation/trace-cmd-stat.1.txt
+++ b/Documentation/trace-cmd-stat.1.txt
@@ -41,6 +41,25 @@ system. The status that it shows is:
 
 *Uprobes:* Shows any uprobes that are defined for tracing.
 
+OPTIONS
+-------
+*-B* 'buffer-name'::
+    If the kernel supports multiple buffers, this will display the status of
+    of the given buffer instance. This may be used multiple times to
+    specify different buffers. When this option is used, the status of the
+    top level instance will not be shown unless *-t* is given.
+
+*-a*::
+    Display the status of all existing buffer instances. When this option
+    is used, the status of the top level instance will not be displayed
+    unless *-t* is given.
+
+*-t*::
+    Display the status of the top level instance buffer. Without the *-B*
+    or *-a* option, this is the same as the default.  But if *-B* or *-a*
+    is used, this is required if the top level instance buffer should also
+    be displayed.
+
 SEE ALSO
 --------
 trace-cmd(1), trace-cmd-record(1), trace-cmd-report(1), trace-cmd-start(1),
diff --git a/trace-stat.c b/trace-stat.c
index 6c38719..9bd773e 100644
--- a/trace-stat.c
+++ b/trace-stat.c
@@ -865,6 +865,12 @@ static void report_traceon(struct buffer_instance *instance)
 
 static void stat_instance(struct buffer_instance *instance)
 {
+	if (instance->name) {
+		/* Lack of trailing newline is intentional */
+		printf("\nInstance: %s\n====================",
+			instance->name);
+	}
+
 	report_plugin(instance);
 	report_events(instance);
 	report_event_filters(instance);
@@ -887,7 +893,7 @@ void trace_stat (int argc, char **argv)
 	int c;
 
 	for (;;) {
-		c = getopt(argc-1, argv+1, "tB:");
+		c = getopt(argc-1, argv+1, "hatB:");
 		if (c == -1)
 			break;
 		switch (c) {
@@ -897,9 +903,9 @@ void trace_stat (int argc, char **argv)
 		case 'B':
 			instance = create_instance(optarg);
 			add_instance(instance);
-			/* top instance requires direct access */
-			if (!topt && is_top_instance(first_instance))
-				first_instance = instance;
+			break;
+		case 'a':
+			add_all_instances();
 			break;
 		case 't':
 			/* Force to use top instance */
@@ -911,6 +917,7 @@ void trace_stat (int argc, char **argv)
 			usage(argv);
 		}
 	}
+	update_first_instance(topt);
 
 	for_all_instances(instance) {
 		stat_instance(instance);
diff --git a/trace-usage.c b/trace-usage.c
index 56c68b4..6e85fdf 100644
--- a/trace-usage.c
+++ b/trace-usage.c
@@ -182,7 +182,11 @@ static struct usage_help usage_help[] = {
 	{
 		"stat",
 		"show the status of the running tracing (ftrace) system",
-		" %s stat"
+		" %s stat [-B buf][-a][-t]\n"
+		"          Shows the status of the tracing system"
+		"          -B status for a given buffer (more than one may be specified)\n"
+		"          -a status for all buffers (except top one)\n"
+		"          -t status the top level buffer (useful with -B or -a)\n"
 	},
 	{
 		"split",
-- 
1.9.1