[PATCH 0/3] perf latency command

[PATCH 0/3] perf latency command

[Clark Williams]

[-- Attachment #1: Type: text/plain, Size: 1816 bytes --]

Ingo,

What follows is a set of three patches against tip/master that
implement a 'latency' sub-command for perf. You can also get it
by fetching the perf-latency-master branch from:

git://git.kernel.org/pub/scm/linux/kernel/git/clrkwllms/rt-linux.git

The 'perf latency' command is an interface for managing the 
hwlat_detector kernel module via its debufs interface. The kernel 
module polls the TSC looking for gaps and when a gap exceeding a
specified threshold is detected, a timestamp and the gap value (in
microseconds) is printed to stdout.

The command line arguments are a sample interval (--window), sample
duration within that window (--width), a duration for the length of
time the test should run (--duration) and a threshold (--threshold)
above which is considered a hardware latency. 

Clark Williams (3):
  debugfs utility routines for perf
  modify perf routines to use new debugfs routines
  perf latency builtin command

 tools/perf/Documentation/perf-latency.txt |   64 +++++
 tools/perf/Documentation/perf.txt         |    2 +-
 tools/perf/Makefile                       |    3 +
 tools/perf/builtin-latency.c              |  383
 +++++++++++++++++++++++++++++
 tools/perf/builtin.h                      |    2 +-
 tools/perf/command-list.txt               |    1 +
 tools/perf/perf.c                         |   45 +---
 tools/perf/util/debugfs.c                 |  241 ++++++++++++++++++
 tools/perf/util/debugfs.h                 |   22 ++
 tools/perf/util/parse-events.c            |   17 +- 10 files changed,
 727 insertions(+), 53 deletions(-) create mode 100644
 tools/perf/Documentation/perf-latency.txt create mode 100644
 tools/perf/builtin-latency.c create mode 100644
 tools/perf/util/debugfs.c create mode 100644 tools/perf/util/debugfs.h


[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 198 bytes --]

[PATCH 1/3] debugfs utility routines for perf

[Clark Williams]

[-- Attachment #1: Type: text/plain, Size: 6581 bytes --]


Add routines to locate the debugfs mount point and to manage the
mounting and unmountin of the debugfs.

Signed-off-by: Clark Williams <williams@redhat.com>
---
 tools/perf/util/debugfs.c |  241 +++++++++++++++++++++++++++++++++++++++++++++
 tools/perf/util/debugfs.h |   22 ++++
 2 files changed, 263 insertions(+), 0 deletions(-)
 create mode 100644 tools/perf/util/debugfs.c
 create mode 100644 tools/perf/util/debugfs.h

diff --git a/tools/perf/util/debugfs.c b/tools/perf/util/debugfs.c
new file mode 100644
index 0000000..34f7983
--- /dev/null
+++ b/tools/perf/util/debugfs.c
@@ -0,0 +1,241 @@
+#include "util.h"
+#include "debugfs.h"
+#include "cache.h"
+
+static int debugfs_premounted = 0;
+static char debugfs_mountpoint[MAX_PATH+1];
+
+static const char *debugfs_known_mountpoints[] = {
+	"/sys/kernel/debug/",
+	"/debug/",
+	0,
+};
+
+/* use this to force a umount */
+void debugfs_force_cleanup(void)
+{
+	debugfs_find_mountpoint();
+	debugfs_premounted = 0;
+	debugfs_umount();
+}
+
+/* construct a full path to a debugfs element */
+int debugfs_make_path(const char *element, char *buffer, int size)
+{
+	int len;
+
+	if (strlen(debugfs_mountpoint) == 0) {
+		buffer[0] = '\0';
+		return -1;
+	}
+	
+	len = strlen(debugfs_mountpoint) + strlen(element) + 1;
+	if (len >= size) 
+		return len+1;
+
+	snprintf(buffer, size-1, "%s/%s", debugfs_mountpoint, element);
+	return 0;
+}
+
+/* find the path to the mounted debugfs */
+const char *
+debugfs_find_mountpoint(void)
+{
+	static int debugfs_found = 0;
+	char type[100];
+	FILE *fp;
+	const char **ptr;
+
+	if (debugfs_found)
+		return (const char *) debugfs_mountpoint;
+
+	ptr = debugfs_known_mountpoints;
+	while(*ptr) {
+		if (debugfs_valid_mountpoint(*ptr) == 0) {
+			debugfs_found = 1;
+			strcpy(debugfs_mountpoint, *ptr);
+			return debugfs_mountpoint;
+		}
+		ptr++;
+	}
+
+	/* give up and parse /proc/mounts */
+	if ((fp = fopen("/proc/mounts","r")) == NULL)
+		die("Can't open /proc/mounts for read");
+
+	while (fscanf(fp, "%*s %"
+		      STR(MAX_PATH)
+		      "s %99s %*s %*d %*d\n",
+		      debugfs_mountpoint, type) == 2) {
+		if (strcmp(type, "debugfs") == 0)
+			break;
+	}
+	fclose(fp);
+
+	if (strcmp(type, "debugfs") != 0)
+		return NULL;
+
+	debugfs_found = 1;
+
+	return debugfs_mountpoint;
+}
+
+/* verify that a mountpoint is actually a debugfs instance */
+
+int 
+debugfs_valid_mountpoint(const char *debugfs)
+{
+	struct statfs st_fs;
+
+	if (statfs(debugfs, &st_fs) < 0)
+		return -ENOENT;
+	else if (st_fs.f_type != (long) DEBUGFS_MAGIC)
+		return -ENOENT;
+	return 0;
+}
+
+
+int 
+debugfs_valid_entry(const char *path)
+{
+	struct stat st;
+
+	if (stat(path, &st))
+		return -errno;
+	return 0;
+}
+
+/* mount the debugfs somewhere */
+
+int
+debugfs_mount(const char *mountpoint)
+{
+	char mountcmd[128];
+
+	/* see if it's already mounted */
+	if (debugfs_find_mountpoint()) {
+		debugfs_premounted = 1;
+		return 0;
+	}
+
+	/* if not mounted and no argument */
+	if (mountpoint == NULL) {
+		/* see if environment variable set */
+		mountpoint = getenv(PERF_DEBUGFS_ENVIRONMENT);
+		/* if no environment variable, use default */
+		if (mountpoint == NULL)
+			mountpoint = "/sys/kernel/debug";
+	}
+
+	/* save the mountpoint */
+	strncpy(debugfs_mountpoint, mountpoint, sizeof(debugfs_mountpoint));
+
+	/* mount it */
+	snprintf(mountcmd, sizeof(mountcmd), 
+		 "/bin/mount -t debugfs debugfs %s", mountpoint);
+	return system(mountcmd);
+}
+
+/* umount the debugfs */
+
+int
+debugfs_umount(void)
+{
+	int ret;
+	char umountcmd[128];
+
+	/* if it was already mounted, leave it */
+	if (debugfs_premounted)
+		return 0;
+
+	/* make sure it's a valid mount point */
+	if ((ret = debugfs_valid_mountpoint(debugfs_mountpoint)))
+		return ret;
+
+	snprintf(umountcmd, sizeof(umountcmd),
+		 "/bin/umount %s", debugfs_mountpoint);
+	return system(umountcmd);
+}
+
+
+int
+debugfs_write(const char *entry, const char *value)
+{
+	int ret, count;
+	int fd;
+	char path[MAX_PATH+1];
+
+	/* construct the path */
+	snprintf(path, sizeof(path), "%s/%s", debugfs_mountpoint, entry);
+	
+	/* verify that it exists */
+	if ((ret = debugfs_valid_entry(path)))
+		return ret;
+
+	/* get how many chars we're going to write */
+	count = strlen(value);
+
+	/* open the debugfs entry */
+	if ((fd = open(path, O_RDWR)) < 0)
+		return -errno;
+
+	while (count > 0) {
+		/* write it */
+		ret = write(fd, value, count);
+		if (ret <= 0) {
+			if (ret == EAGAIN)
+				continue;
+			close(fd);
+			return -errno;
+		}
+		count -= ret;
+	}
+
+	/* close it */
+	close(fd);
+
+	/* return success */
+	return 0;
+}
+
+/* 
+ * read a debugfs entry
+ * returns the number of chars read or a negative errno
+ */
+
+int
+debugfs_read(const char *entry, char *buffer, size_t size)
+{
+	int ret;
+	int fd;
+	char path[MAX_PATH+1];
+	
+	/* construct the path */
+	snprintf(path, sizeof(path), "%s/%s", debugfs_mountpoint, entry);
+	
+	/* verify that it exists */
+	if ((ret = debugfs_valid_entry(path)))
+		return ret;
+
+	/* open the debugfs entry */
+	if ((fd = open(path, O_RDONLY)) < 0)
+		return -errno;
+
+	do {
+		/* read it */
+		ret = read(fd, buffer, size);
+		if (ret == 0) {
+			close(fd);
+			return EOF;
+		}
+	} while (ret < 0 && errno == EAGAIN);
+
+	/* close it */
+	close(fd);
+
+	/* make *sure* there's a null character at the end */
+	buffer[ret] = '\0';
+
+	/* return the number of chars read */
+	return ret;
+}
diff --git a/tools/perf/util/debugfs.h b/tools/perf/util/debugfs.h
new file mode 100644
index 0000000..df14c2a
--- /dev/null
+++ b/tools/perf/util/debugfs.h
@@ -0,0 +1,22 @@
+#ifndef __DEBUGFS_H__
+#define __DEBUGFS_H__
+#include <sys/mount.h>
+#ifndef MAX_PATH
+#define MAX_PATH 256
+#endif
+
+#ifndef STR
+#define _STR(x) #x
+#define STR(x) _STR(x)
+#endif
+
+extern const char * debugfs_find_mountpoint(void);
+extern int debugfs_valid_mountpoint(const char *debugfs);
+extern int debugfs_valid_entry(const char *path);
+extern int debugfs_mount(const char *mountpoint);
+extern int debugfs_umount(void);
+extern int debugfs_write(const char *entry, const char *value);
+extern int debugfs_read(const char *entry, char *buffer, size_t size);
+extern void debugfs_force_cleanup(void);
+extern int debugfs_make_path(const char *element, char *buffer, int size);
+#endif
-- 
1.6.2.5



[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 198 bytes --]

[tip:perf/core] perf tools: Add debugfs utility routines for perf

[tip-bot for Clark Williams]

Commit-ID:  afe61f677866ffc484e69c4ecca2d316d564d78b
Gitweb:     http://git.kernel.org/tip/afe61f677866ffc484e69c4ecca2d316d564d78b
Author:     Clark Williams <williams@redhat.com>
AuthorDate: Sun, 8 Nov 2009 09:01:37 -0600
Committer:  Ingo Molnar <mingo@elte.hu>
CommitDate: Sun, 8 Nov 2009 18:01:34 +0100

perf tools: Add debugfs utility routines for perf

Add routines to locate the debugfs mount point and to manage the
mounting and unmounting of the debugfs.

Signed-off-by: Clark Williams <williams@redhat.com>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
LKML-Reference: <20091101155621.2b3503ee@torg>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
---
 tools/perf/Makefile       |    2 +
 tools/perf/util/debugfs.c |  241 +++++++++++++++++++++++++++++++++++++++++++++
 tools/perf/util/debugfs.h |   25 +++++
 3 files changed, 268 insertions(+), 0 deletions(-)

diff --git a/tools/perf/Makefile b/tools/perf/Makefile
index 542b29e..b9509b1 100644
--- a/tools/perf/Makefile
+++ b/tools/perf/Makefile
@@ -353,6 +353,7 @@ LIB_H += util/include/asm/swab.h
 LIB_H += util/include/asm/system.h
 LIB_H += util/include/asm/uaccess.h
 LIB_H += perf.h
+LIB_H += util/debugfs.h
 LIB_H += util/event.h
 LIB_H += util/types.h
 LIB_H += util/levenshtein.h
@@ -378,6 +379,7 @@ LIB_OBJS += util/abspath.o
 LIB_OBJS += util/alias.o
 LIB_OBJS += util/config.o
 LIB_OBJS += util/ctype.o
+LIB_OBJS += util/debugfs.o
 LIB_OBJS += util/environment.o
 LIB_OBJS += util/event.o
 LIB_OBJS += util/exec_cmd.o
diff --git a/tools/perf/util/debugfs.c b/tools/perf/util/debugfs.c
new file mode 100644
index 0000000..06b73ee
--- /dev/null
+++ b/tools/perf/util/debugfs.c
@@ -0,0 +1,241 @@
+#include "util.h"
+#include "debugfs.h"
+#include "cache.h"
+
+static int debugfs_premounted;
+static char debugfs_mountpoint[MAX_PATH+1];
+
+static const char *debugfs_known_mountpoints[] = {
+	"/sys/kernel/debug/",
+	"/debug/",
+	0,
+};
+
+/* use this to force a umount */
+void debugfs_force_cleanup(void)
+{
+	debugfs_find_mountpoint();
+	debugfs_premounted = 0;
+	debugfs_umount();
+}
+
+/* construct a full path to a debugfs element */
+int debugfs_make_path(const char *element, char *buffer, int size)
+{
+	int len;
+
+	if (strlen(debugfs_mountpoint) == 0) {
+		buffer[0] = '\0';
+		return -1;
+	}
+
+	len = strlen(debugfs_mountpoint) + strlen(element) + 1;
+	if (len >= size)
+		return len+1;
+
+	snprintf(buffer, size-1, "%s/%s", debugfs_mountpoint, element);
+	return 0;
+}
+
+static int debugfs_found;
+
+/* find the path to the mounted debugfs */
+const char *debugfs_find_mountpoint(void)
+{
+	const char **ptr;
+	char type[100];
+	FILE *fp;
+
+	if (debugfs_found)
+		return (const char *) debugfs_mountpoint;
+
+	ptr = debugfs_known_mountpoints;
+	while (*ptr) {
+		if (debugfs_valid_mountpoint(*ptr) == 0) {
+			debugfs_found = 1;
+			strcpy(debugfs_mountpoint, *ptr);
+			return debugfs_mountpoint;
+		}
+		ptr++;
+	}
+
+	/* give up and parse /proc/mounts */
+	fp = fopen("/proc/mounts", "r");
+	if (fp == NULL)
+		die("Can't open /proc/mounts for read");
+
+	while (fscanf(fp, "%*s %"
+		      STR(MAX_PATH)
+		      "s %99s %*s %*d %*d\n",
+		      debugfs_mountpoint, type) == 2) {
+		if (strcmp(type, "debugfs") == 0)
+			break;
+	}
+	fclose(fp);
+
+	if (strcmp(type, "debugfs") != 0)
+		return NULL;
+
+	debugfs_found = 1;
+
+	return debugfs_mountpoint;
+}
+
+/* verify that a mountpoint is actually a debugfs instance */
+
+int debugfs_valid_mountpoint(const char *debugfs)
+{
+	struct statfs st_fs;
+
+	if (statfs(debugfs, &st_fs) < 0)
+		return -ENOENT;
+	else if (st_fs.f_type != (long) DEBUGFS_MAGIC)
+		return -ENOENT;
+
+	return 0;
+}
+
+
+int debugfs_valid_entry(const char *path)
+{
+	struct stat st;
+
+	if (stat(path, &st))
+		return -errno;
+
+	return 0;
+}
+
+/* mount the debugfs somewhere */
+
+int debugfs_mount(const char *mountpoint)
+{
+	char mountcmd[128];
+
+	/* see if it's already mounted */
+	if (debugfs_find_mountpoint()) {
+		debugfs_premounted = 1;
+		return 0;
+	}
+
+	/* if not mounted and no argument */
+	if (mountpoint == NULL) {
+		/* see if environment variable set */
+		mountpoint = getenv(PERF_DEBUGFS_ENVIRONMENT);
+		/* if no environment variable, use default */
+		if (mountpoint == NULL)
+			mountpoint = "/sys/kernel/debug";
+	}
+
+	/* save the mountpoint */
+	strncpy(debugfs_mountpoint, mountpoint, sizeof(debugfs_mountpoint));
+
+	/* mount it */
+	snprintf(mountcmd, sizeof(mountcmd),
+		 "/bin/mount -t debugfs debugfs %s", mountpoint);
+	return system(mountcmd);
+}
+
+/* umount the debugfs */
+
+int debugfs_umount(void)
+{
+	char umountcmd[128];
+	int ret;
+
+	/* if it was already mounted, leave it */
+	if (debugfs_premounted)
+		return 0;
+
+	/* make sure it's a valid mount point */
+	ret = debugfs_valid_mountpoint(debugfs_mountpoint);
+	if (ret)
+		return ret;
+
+	snprintf(umountcmd, sizeof(umountcmd),
+		 "/bin/umount %s", debugfs_mountpoint);
+	return system(umountcmd);
+}
+
+int debugfs_write(const char *entry, const char *value)
+{
+	char path[MAX_PATH+1];
+	int ret, count;
+	int fd;
+
+	/* construct the path */
+	snprintf(path, sizeof(path), "%s/%s", debugfs_mountpoint, entry);
+
+	/* verify that it exists */
+	ret = debugfs_valid_entry(path);
+	if (ret)
+		return ret;
+
+	/* get how many chars we're going to write */
+	count = strlen(value);
+
+	/* open the debugfs entry */
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -errno;
+
+	while (count > 0) {
+		/* write it */
+		ret = write(fd, value, count);
+		if (ret <= 0) {
+			if (ret == EAGAIN)
+				continue;
+			close(fd);
+			return -errno;
+		}
+		count -= ret;
+	}
+
+	/* close it */
+	close(fd);
+
+	/* return success */
+	return 0;
+}
+
+/*
+ * read a debugfs entry
+ * returns the number of chars read or a negative errno
+ */
+int debugfs_read(const char *entry, char *buffer, size_t size)
+{
+	char path[MAX_PATH+1];
+	int ret;
+	int fd;
+
+	/* construct the path */
+	snprintf(path, sizeof(path), "%s/%s", debugfs_mountpoint, entry);
+
+	/* verify that it exists */
+	ret = debugfs_valid_entry(path);
+	if (ret)
+		return ret;
+
+	/* open the debugfs entry */
+	fd = open(path, O_RDONLY);
+	if (fd < 0)
+		return -errno;
+
+	do {
+		/* read it */
+		ret = read(fd, buffer, size);
+		if (ret == 0) {
+			close(fd);
+			return EOF;
+		}
+	} while (ret < 0 && errno == EAGAIN);
+
+	/* close it */
+	close(fd);
+
+	/* make *sure* there's a null character at the end */
+	buffer[ret] = '\0';
+
+	/* return the number of chars read */
+	return ret;
+}
diff --git a/tools/perf/util/debugfs.h b/tools/perf/util/debugfs.h
new file mode 100644
index 0000000..3cd14f9
--- /dev/null
+++ b/tools/perf/util/debugfs.h
@@ -0,0 +1,25 @@
+#ifndef __DEBUGFS_H__
+#define __DEBUGFS_H__
+
+#include <sys/mount.h>
+
+#ifndef MAX_PATH
+# define MAX_PATH 256
+#endif
+
+#ifndef STR
+# define _STR(x) #x
+# define STR(x) _STR(x)
+#endif
+
+extern const char *debugfs_find_mountpoint(void);
+extern int debugfs_valid_mountpoint(const char *debugfs);
+extern int debugfs_valid_entry(const char *path);
+extern int debugfs_mount(const char *mountpoint);
+extern int debugfs_umount(void);
+extern int debugfs_write(const char *entry, const char *value);
+extern int debugfs_read(const char *entry, char *buffer, size_t size);
+extern void debugfs_force_cleanup(void);
+extern int debugfs_make_path(const char *element, char *buffer, int size);
+
+#endif /* __DEBUGFS_H__ */

[PATCH 2/3] modify perf routines to use new debugfs routines

[Clark Williams]

[-- Attachment #1: Type: text/plain, Size: 3897 bytes --]


modified perf.c get_debugfs_mntpnt() to use the util/debugfs.c
debugfs_find_mountpoint()
modified util/parse-events.c to use debugfs_valid_mountpoint().

Signed-off-by: Clark Williams <williams@redhat.com>
---
 tools/perf/perf.c              |   45
++++++--------------------------------- tools/perf/util/parse-events.c
|   17 +++----------- 2 files changed, 11 insertions(+), 51 deletions(-)

diff --git a/tools/perf/perf.c b/tools/perf/perf.c
index 9cafe54..940cbf6 100644
--- a/tools/perf/perf.c
+++ b/tools/perf/perf.c
@@ -14,6 +14,7 @@
 #include "util/run-command.h"
 #include "util/parse-events.h"
 #include "util/string.h"
+#include "util/debugfs.h"
 
 const char perf_usage_string[] =
 	"perf [--version] [--help] COMMAND [ARGS]";
@@ -296,6 +297,7 @@ static void handle_internal_command(int argc, const
char **argv) { "trace", cmd_trace, 0 },
 		{ "sched", cmd_sched, 0 },
 		{ "probe", cmd_probe, 0 },
+		{ "latency", cmd_latency, 0},
 	};
 	unsigned int i;
 	static const char ext[] = STRIP_EXTENSION;
@@ -383,45 +385,12 @@ static int run_argv(int *argcp, const char
***argv) /* mini /proc/mounts parser: searching for "^blah /mount/point
debugfs" */ static void get_debugfs_mntpt(void)
 {
-	FILE *file;
-	char fs_type[100];
-	char debugfs[MAXPATHLEN];
+	const char *path = debugfs_find_mountpoint();
 
-	/*
-	 * try the standard location
-	 */
-	if (valid_debugfs_mount("/sys/kernel/debug/") == 0) {
-		strcpy(debugfs_mntpt, "/sys/kernel/debug/");
-		return;
-	}
-
-	/*
-	 * try the sane location
-	 */
-	if (valid_debugfs_mount("/debug/") == 0) {
-		strcpy(debugfs_mntpt, "/debug/");
-		return;
-	}
-
-	/*
-	 * give up and parse /proc/mounts
-	 */
-	file = fopen("/proc/mounts", "r");
-	if (file == NULL)
-		return;
-
-	while (fscanf(file, "%*s %"
-		      STR(MAXPATHLEN)
-		      "s %99s %*s %*d %*d\n",
-		      debugfs, fs_type) == 2) {
-		if (strcmp(fs_type, "debugfs") == 0)
-			break;
-	}
-	fclose(file);
-	if (strcmp(fs_type, "debugfs") == 0) {
-		strncpy(debugfs_mntpt, debugfs, MAXPATHLEN);
-		debugfs_mntpt[MAXPATHLEN - 1] = '\0';
-	}
+	if (path)
+		strncpy(debugfs_mntpt, path, sizeof(debugfs_mntpt));
+	else
+		debugfs_mntpt[0] = '\0';
 }
 
 int main(int argc, const char **argv)
diff --git a/tools/perf/util/parse-events.c
b/tools/perf/util/parse-events.c index 31baa5a..097938a 100644
--- a/tools/perf/util/parse-events.c
+++ b/tools/perf/util/parse-events.c
@@ -7,6 +7,7 @@
 #include "string.h"
 #include "cache.h"
 #include "header.h"
+#include "debugfs.h"
 
 int				nr_counters;
 
@@ -149,16 +150,6 @@ static int tp_event_has_id(struct dirent *sys_dir,
struct dirent *evt_dir) 
 #define MAX_EVENT_LENGTH 512
 
-int valid_debugfs_mount(const char *debugfs)
-{
-	struct statfs st_fs;
-
-	if (statfs(debugfs, &st_fs) < 0)
-		return -ENOENT;
-	else if (st_fs.f_type != (long) DEBUGFS_MAGIC)
-		return -ENOENT;
-	return 0;
-}
 
 struct tracepoint_path *tracepoint_id_to_path(u64 config)
 {
@@ -171,7 +162,7 @@ struct tracepoint_path *tracepoint_id_to_path(u64
config) char evt_path[MAXPATHLEN];
 	char dir_path[MAXPATHLEN];
 
-	if (valid_debugfs_mount(debugfs_path))
+	if (debugfs_valid_mountpoint(debugfs_path))
 		return NULL;
 
 	sys_dir = opendir(debugfs_path);
@@ -510,7 +501,7 @@ static enum event_result
parse_tracepoint_event(const char **strp, char
sys_name[MAX_EVENT_LENGTH]; unsigned int sys_length, evt_length;
 
-	if (valid_debugfs_mount(debugfs_path))
+	if (debugfs_valid_mountpoint(debugfs_path))
 		return 0;
 
 	evt_name = strchr(*strp, ':');
@@ -788,7 +779,7 @@ static void print_tracepoint_events(void)
 	char evt_path[MAXPATHLEN];
 	char dir_path[MAXPATHLEN];
 
-	if (valid_debugfs_mount(debugfs_path))
+	if (debugfs_valid_mountpoint(debugfs_path))
 		return;
 
 	sys_dir = opendir(debugfs_path);
-- 
1.6.2.5


[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 198 bytes --]

[tip:perf/core] perf tools: Modify perf routines to use new debugfs routines

[tip-bot for Clark Williams]

Commit-ID:  549104f22b3cd4761145eb5fba6ee4d59822da61
Gitweb:     http://git.kernel.org/tip/549104f22b3cd4761145eb5fba6ee4d59822da61
Author:     Clark Williams <williams@redhat.com>
AuthorDate: Sun, 8 Nov 2009 09:03:07 -0600
Committer:  Ingo Molnar <mingo@elte.hu>
CommitDate: Sun, 8 Nov 2009 18:01:35 +0100

perf tools: Modify perf routines to use new debugfs routines

modify perf.c get_debugfs_mntpnt() to use the util/debugfs.c
debugfs_find_mountpoint()

modify util/parse-events.c to use debugfs_valid_mountpoint().

Signed-off-by: Clark Williams <williams@redhat.com>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
LKML-Reference: <20091101155720.624cc87e@torg>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
---
 tools/perf/perf.c              |   44 +++++----------------------------------
 tools/perf/util/parse-events.c |   17 +++-----------
 2 files changed, 10 insertions(+), 51 deletions(-)

diff --git a/tools/perf/perf.c b/tools/perf/perf.c
index 624e62d..601f403 100644
--- a/tools/perf/perf.c
+++ b/tools/perf/perf.c
@@ -14,6 +14,7 @@
 #include "util/run-command.h"
 #include "util/parse-events.h"
 #include "util/string.h"
+#include "util/debugfs.h"
 
 const char perf_usage_string[] =
 	"perf [--version] [--help] COMMAND [ARGS]";
@@ -382,45 +383,12 @@ static int run_argv(int *argcp, const char ***argv)
 /* mini /proc/mounts parser: searching for "^blah /mount/point debugfs" */
 static void get_debugfs_mntpt(void)
 {
-	FILE *file;
-	char fs_type[100];
-	char debugfs[MAXPATHLEN];
+	const char *path = debugfs_find_mountpoint();
 
-	/*
-	 * try the standard location
-	 */
-	if (valid_debugfs_mount("/sys/kernel/debug/") == 0) {
-		strcpy(debugfs_mntpt, "/sys/kernel/debug/");
-		return;
-	}
-
-	/*
-	 * try the sane location
-	 */
-	if (valid_debugfs_mount("/debug/") == 0) {
-		strcpy(debugfs_mntpt, "/debug/");
-		return;
-	}
-
-	/*
-	 * give up and parse /proc/mounts
-	 */
-	file = fopen("/proc/mounts", "r");
-	if (file == NULL)
-		return;
-
-	while (fscanf(file, "%*s %"
-		      STR(MAXPATHLEN)
-		      "s %99s %*s %*d %*d\n",
-		      debugfs, fs_type) == 2) {
-		if (strcmp(fs_type, "debugfs") == 0)
-			break;
-	}
-	fclose(file);
-	if (strcmp(fs_type, "debugfs") == 0) {
-		strncpy(debugfs_mntpt, debugfs, MAXPATHLEN);
-		debugfs_mntpt[MAXPATHLEN - 1] = '\0';
-	}
+	if (path)
+		strncpy(debugfs_mntpt, path, sizeof(debugfs_mntpt));
+	else
+		debugfs_mntpt[0] = '\0';
 }
 
 int main(int argc, const char **argv)
diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c
index 31baa5a..097938a 100644
--- a/tools/perf/util/parse-events.c
+++ b/tools/perf/util/parse-events.c
@@ -7,6 +7,7 @@
 #include "string.h"
 #include "cache.h"
 #include "header.h"
+#include "debugfs.h"
 
 int				nr_counters;
 
@@ -149,16 +150,6 @@ static int tp_event_has_id(struct dirent *sys_dir, struct dirent *evt_dir)
 
 #define MAX_EVENT_LENGTH 512
 
-int valid_debugfs_mount(const char *debugfs)
-{
-	struct statfs st_fs;
-
-	if (statfs(debugfs, &st_fs) < 0)
-		return -ENOENT;
-	else if (st_fs.f_type != (long) DEBUGFS_MAGIC)
-		return -ENOENT;
-	return 0;
-}
 
 struct tracepoint_path *tracepoint_id_to_path(u64 config)
 {
@@ -171,7 +162,7 @@ struct tracepoint_path *tracepoint_id_to_path(u64 config)
 	char evt_path[MAXPATHLEN];
 	char dir_path[MAXPATHLEN];
 
-	if (valid_debugfs_mount(debugfs_path))
+	if (debugfs_valid_mountpoint(debugfs_path))
 		return NULL;
 
 	sys_dir = opendir(debugfs_path);
@@ -510,7 +501,7 @@ static enum event_result parse_tracepoint_event(const char **strp,
 	char sys_name[MAX_EVENT_LENGTH];
 	unsigned int sys_length, evt_length;
 
-	if (valid_debugfs_mount(debugfs_path))
+	if (debugfs_valid_mountpoint(debugfs_path))
 		return 0;
 
 	evt_name = strchr(*strp, ':');
@@ -788,7 +779,7 @@ static void print_tracepoint_events(void)
 	char evt_path[MAXPATHLEN];
 	char dir_path[MAXPATHLEN];
 
-	if (valid_debugfs_mount(debugfs_path))
+	if (debugfs_valid_mountpoint(debugfs_path))
 		return;
 
 	sys_dir = opendir(debugfs_path);

[PATCH 3/3] perf latency builtin command

[Clark Williams]

[-- Attachment #1: Type: text/plain, Size: 15367 bytes --]


This is the first cut at a 'perf latency' command to manage the
hwlat_detector kernel module, used to detect hardware induced
latencies (e.g. SMIs).

Signed-off-by: Clark Williams <williams@redhat.com>
---
 tools/perf/Documentation/perf-latency.txt |   64 +++++
 tools/perf/Documentation/perf.txt         |    2 +-
 tools/perf/Makefile                       |    3 +
 tools/perf/builtin-latency.c              |  383
+++++++++++++++++++++++++++++ tools/perf/builtin.h
|    2 +- tools/perf/command-list.txt               |    1 +
 6 files changed, 453 insertions(+), 2 deletions(-)
 create mode 100644 tools/perf/Documentation/perf-latency.txt
 create mode 100644 tools/perf/builtin-latency.c

diff --git a/tools/perf/Documentation/perf-latency.txt
b/tools/perf/Documentation/perf-latency.txt new file mode 100644
index 0000000..f615d08
--- /dev/null
+++ b/tools/perf/Documentation/perf-latency.txt
@@ -0,0 +1,64 @@
+perf-latency(1)
+===============
+
+NAME
+----
+perf-latency - check for hardware latencies
+
+SYNOPSIS
+--------
+[verse]
+'per latency' [OPTIONS]
+
+DESCRIPTION
+-----------
+This command manages the hwlat_detector kernel module, which is used
+to test the system for hardware induced latencies. The command runs
+for a specified amount of time (default: 60 seconds) and samples the
+system Time Stamp Counter (TSC) register, looking for gaps which
+exceed a threshold. If a gap exceeding the threshold is seen, a
+timestamp and the gap (in microseconds) is printed to the standard
+output.
+
+OPTIONS
+-------
+--duration=<n>{s,m,h,d,w}::
+	The length of time the test should run. (default: 60 seconds)
+
+--window=<n>{us,ms,s,m}::
+	The sample period for the test. (default 1 second)
+
+--width==<n>{us,ms,s,m}::
+	The test time within the sample window. (default 500
+	milliseconds)
+
+--threshold==<n>{us,ms,s}::
+	Threshold above which is considered a latency. (default
50 microseconds) +
+--cleanup::
+	Force unload of hwlat_detector module and umount of debugfs.
+
+--debug::
+	Turn on lots of debugging prints
+
+
+Time values are specified as a decimal integer value with an optional
+unit suffix
+
+        us - microseconds
+	ms - milliseconds
+	s - seconds
+	m - minutes
+	h - hours
+	d - days
+	w - weeks
+
+EXAMPLES
+--------
+
+$ perf latency --duration=1h --width=750ms --threshold=10us
+
+This invocation runs the latency detector for 1 hour, using the
+default window size of 1 second, a width of 750 milliseconds and a
+threshold of 10 microseconds. Any gap larger than 10 microseconds
+detected during sampling will be printed to the screen.
diff --git a/tools/perf/Documentation/perf.txt
b/tools/perf/Documentation/perf.txt index 69c8325..358856e 100644
--- a/tools/perf/Documentation/perf.txt
+++ b/tools/perf/Documentation/perf.txt
@@ -21,4 +21,4 @@ SEE ALSO
 --------
 linkperf:perf-stat[1], linkperf:perf-top[1],
 linkperf:perf-record[1], linkperf:perf-report[1],
-linkperf:perf-list[1]
+linkperf:perf-list[1], linkperf:perf-latency[1]
diff --git a/tools/perf/Makefile b/tools/perf/Makefile
index 800783d..68f210a 100644
--- a/tools/perf/Makefile
+++ b/tools/perf/Makefile
@@ -374,6 +374,7 @@ LIB_H += util/sort.h
 LIB_H += util/hist.h
 LIB_H += util/thread.h
 LIB_H += util/data_map.h
+LIB_H += util/debugfs.h
 
 LIB_OBJS += util/abspath.o
 LIB_OBJS += util/alias.o
@@ -415,6 +416,7 @@ LIB_OBJS += util/svghelper.o
 LIB_OBJS += util/sort.o
 LIB_OBJS += util/hist.o
 LIB_OBJS += util/data_map.o
+LIB_OBJS += util/debugfs.o
 
 BUILTIN_OBJS += builtin-annotate.o
 BUILTIN_OBJS += builtin-help.o
@@ -427,6 +429,7 @@ BUILTIN_OBJS += builtin-timechart.o
 BUILTIN_OBJS += builtin-top.o
 BUILTIN_OBJS += builtin-trace.o
 BUILTIN_OBJS += builtin-probe.o
+BUILTIN_OBJS += builtin-latency.o
 
 PERFLIBS = $(LIB_FILE)
 
diff --git a/tools/perf/builtin-latency.c b/tools/perf/builtin-latency.c
new file mode 100644
index 0000000..7a33f71
--- /dev/null
+++ b/tools/perf/builtin-latency.c
@@ -0,0 +1,383 @@
+#include "builtin.h"
+#include "perf.h"
+
+#include "util/util.h"
+#include "util/parse-options.h"
+#include "util/debugfs.h"
+
+static struct latency_params {
+	long duration;
+	long threshold;
+	long window;
+	long width;
+	int cleanup;
+	int debugging;
+} parameters = {
+	.duration = 0,
+	.threshold = -1,
+	.window = -1,
+	.width = -1,
+	.cleanup = 0,
+	.debugging = 0,
+};
+
+/* strings for argument processing */
+#define DEFAULT_DURATION  "60s"  /* 60 seconds */
+
+static char *duration_str  = (char*)DEFAULT_DURATION;
+static char *threshold_str = (char*)"-1";
+static char *window_str    = (char*)"-1";
+static char *width_str     = (char*)"-1";
+
+#define SAMPLE_POLL_INTERVAL  1  /* poll for sample every second */
+#define ENABLE_RETRIES 100       /* how many times to retry
enable/disable */ +
+static int hwlat_preloaded;
+
+static void debug(const char *fmt, ...)
+{
+	if (parameters.debugging) {
+		va_list ap;
+		fputs("PERF DEBUG: ", stderr);
+		va_start(ap, fmt);
+		vfprintf(stderr, fmt, ap);
+		va_end(ap);
+	}
+}
+
+static int check_module_loaded(const char *modname)
+{
+	char line[256], module[64];
+	FILE *fp = fopen("/proc/modules", "r");
+	int len;
+
+	if (fp == NULL)
+		return -errno;
+	len = strlen(modname);
+	while (fgets(line, sizeof(line) - 1, fp)) {
+		if (sscanf(line, "%s", module) == EOF)
+			continue;
+		if (strncmp(module, modname, len) == 0) {
+			fclose(fp);
+			return 0;
+		}
+	}
+	fclose(fp);
+	return -1;
+}
+
+static int
+hwlat_load_module(void)
+{
+	if (hwlat_preloaded)
+		return 0;
+	debug("loading hwlat_detector module\n");
+	if (check_module_loaded("hwlat_detector") == 0) {
+		debug("hwlat_detector already loaded\n");
+		hwlat_preloaded = 1;
+		return 0;
+	}
+	return system("/sbin/modprobe hwlat_detector");
+}
+
+static int
+hwlat_unload_module(void)
+{
+	if (hwlat_preloaded)
+		return 0;
+	if (check_module_loaded("hwlat_detector"))
+		return 0;
+	return system("/sbin/modprobe -r hwlat_detector");
+}
+
+static int
+hwlat_get_value(const char *entry)
+{
+	char hwlat_entry[128];
+	char buffer[256];
+	int nread, val;
+
+	snprintf(hwlat_entry, sizeof(hwlat_entry),
"hwlat_detector/%s", entry);
+	nread = debugfs_read(hwlat_entry, buffer, sizeof(buffer));
+	if (nread < 0)
+		return nread;
+	val = strtol(buffer, NULL, 10);
+	debug("read %d from %s\n", val, hwlat_entry);
+	return val;
+}	
+
+static int
+hwlat_put_value(const char *entry, int value)
+{
+	char hwlat_entry[128];
+	char buffer[256];
+
+	snprintf(hwlat_entry, sizeof(hwlat_entry),
"hwlat_detector/%s", entry);
+	snprintf(buffer, sizeof(buffer), "%d", value);
+	debug("writing %s to %s\n", buffer, hwlat_entry);
+	return debugfs_write(hwlat_entry, buffer);
+}
+
+/* 
+ * read the sample file and if we get something return it
+ * the file is opened non-blocking so handle EAGAIN
+ */
+static int
+hwlat_read_sample(int fd, char *buffer, size_t size)
+{
+	int ret;
+	ret = read(fd, buffer, size);
+	if (ret >= 0)
+		buffer[ret] = '\0';
+	/* don't error if there's nothing to read */
+	if (ret < 0 && errno == EAGAIN)
+		return 0;
+	return ret;
+}
+
+static int
+hwlat_enable(void)
+{
+	int retries = ENABLE_RETRIES;
+	int val;
+	do {
+		hwlat_put_value("enable", 1);
+		
+	} while ((val = hwlat_get_value("enable")) != 1 && --retries >
0);
+	if (val != 1)
+		return -1;
+	return 0;
+}
+
+static int
+hwlat_disable(void)
+{
+	int val;
+	int retries = ENABLE_RETRIES;
+
+	do {
+		hwlat_put_value("enable", 0);
+	} while ((val = hwlat_get_value("enable")) != 0 && --retries >
0);
+	if (val != 0)
+		return -1;
+	return 0;
+}
+
+/* perf latency options */
+
+static const char * const latency_usage[] = {
+	"perf latency [<options>]",
+	NULL
+};
+
+static const struct option options[] = {
+	OPT_STRING('d', "duration", &duration_str, "n{s,m,h,d,w}",
+		    "total time to test for hardware latencies:
<N>{smhd}"),
+	OPT_STRING('t', "threshold", &threshold_str, "n{ms,s,m,h}",
+		    "microsecond value above which is considered a
hardware latency"),
+	OPT_STRING('w', "window", &window_str, "n{ms,s,m,h}",
+		   "sample window duration"),
+	OPT_STRING('W', "width", &width_str, "n{ms,s,m,h}",
+		   "actual measurment time period (must be <=
window)"),
+	OPT_BOOLEAN('c', "cleanup", &parameters.cleanup, 
+		    "force unload of module and umount of debugfs"),
+	OPT_BOOLEAN('D', "debug", &parameters.debugging, "turn on
debugging prints"),
+	OPT_END()
+};
+
+static int
+ts_less_than(const struct timespec t1, const struct timespec t2)
+{
+	long long diff;
+	diff = NSEC_PER_SEC * (long long)((int) t1.tv_sec - (int)
t2.tv_sec);
+	diff += ((int) t1.tv_nsec - (int) t2.tv_nsec);
+	/*debug("diff: %ld\n", diff);*/
+	if (diff < 0)
+		return 1;
+	return 0;
+}
+
+static long
+str_to_us(const char *str)
+{
+	char *ptr;
+	int val = strtol(str, &ptr, 10);
+	if (ptr) {
+		/* check for milliseconds, seconds, minutes and hours
*/
+		/* ignore anything else */
+		if (strcmp(ptr, "ms") == 0)
+			val *= 1000;
+		else if (strcmp(ptr, "s") == 0)
+			val *= 1000000;
+		else if (strcmp(ptr, "m") == 0)
+			val *= 60000000;   /* 1000000 * 60 */
+		else if (strcmp(ptr, "h") == 0)
+			val *= 3600000000; /* 1000000 * 60 * 60 */
+	}
+	return val;
+}
+
+static long
+str_to_sec(const char *str)
+{
+	char *ptr;
+	int val = strtol(str, &ptr, 10);
+
+	if (ptr) {
+		/* check for minutes, hours, days, and weeks */
+		if (strcmp(ptr, "m") == 0)
+			val *= 60;
+		else if (strcmp(ptr, "h") == 0)
+			val *= 360; /* 60 * 60 */
+		else if (strcmp(ptr, "d") == 0)
+			val *= 86400; /* 60 * 60 * 24 */
+		else if (strcmp(ptr, "w") == 0)
+			val *= 604800; /* 60 * 60 * 24 * 7 */
+	}
+	return val;
+}
+
+
+/* main function for hwlat_detector operation */
+static void
+detect (struct latency_params *params)
+{
+	int ret;
+	int fd;
+	char buffer[128];
+	char sample_path[256];
+	struct timespec start, stop, now;
+
+	/* set the parameters */
+	debug("setting window to %d\n", params->window);
+	if ((ret = hwlat_put_value("window", params->window)))
+		die("error writing hwlat_detector 'window' parameter
(%ld): %s",
+		    params->window, strerror(errno));
+
+	debug("setting width to %d\n", params->width);
+	if ((ret = hwlat_put_value("width", params->width)))
+		die("error writing hwlat_detector 'width' parameter
(%ld): %s",
+		    params->width, strerror(errno));
+
+	debug("setting threshold to %d\n", params->threshold);
+	if ((ret = hwlat_put_value("threshold", params->threshold)))
+		die("error writing hwlat_detector 'threshold'
parameter (%ld): %s",
+		    params->threshold, strerror(errno));
+
+	/* open the sample entry */
+	if (debugfs_make_path("hwlat_detector/sample", sample_path,
sizeof(sample_path)))
+		die ("can't get path to sample element: %s\n",
strerror(errno)); +
+	if ((fd = open(sample_path, O_RDONLY|O_NDELAY)) == -1)
+		die("can't open %s: %s\n", sample_path,
strerror(errno)); +
+	/* get our start time */
+	if ((ret = clock_gettime(CLOCK_MONOTONIC, &start)))
+		die ("error getting start time: %s\n", strerror(-ret));
+
+	/* setup stop time */
+	stop = start;
+	stop.tv_sec += params->duration;
+
+	/* start the kmod sampling */
+	if (hwlat_enable())
+		die ("can't enable the hwlat_detector!\n");
+
+	/* read samples until duration is done */
+	if ((ret = clock_gettime(CLOCK_MONOTONIC, &now)))
+		die ("error getting initial time: %s\n",
strerror(-errno)); +
+	while(ts_less_than(now, stop)) {
+		struct timespec ts;
+		debug("checking for sample\n");
+		if ((ret = hwlat_read_sample(fd, buffer,
sizeof(buffer))) < 0)
+			die("error reading sample: %s (%d)",
strerror(errno), errno);
+		/* for now, just print it */
+		if (ret) {
+			buffer[ret-1] = '\0'; /* nuke the newline */
+			puts(buffer);
+		}
+		ts.tv_sec = SAMPLE_POLL_INTERVAL;
+		ts.tv_nsec = 0;
+		clock_nanosleep(CLOCK_MONOTONIC, 0, &ts, NULL);
+		if ((ret = clock_gettime(CLOCK_MONOTONIC, &now)))
+			die("error getting current time: %s\n",
strerror(-errno));
+	}
+
+	/* stop the kmod sampling */
+	if (hwlat_disable())
+		die ("can't stop the hwlat_detector!\n");
+
+	/* close the sample entry */
+	close(fd);
+}
+
+static void
+sig_cleanup(int signo __used)
+{
+	hwlat_disable();
+	hwlat_unload_module();
+	debugfs_umount();
+}
+
+int
+cmd_latency(int argc, const char **argv, const char *prefix __used)
+{
+	argc = parse_options(argc, argv, options, latency_usage, 0);
+	if (argc)
+		usage_with_options(latency_usage, options);
+
+	if (parameters.cleanup) {
+		printf("cleaning up\n");
+		debugfs_force_cleanup();
+		hwlat_unload_module();
+		return 0;
+	}		
+
+	/* setup parameters */
+	parameters.duration  = str_to_sec(duration_str);
+	parameters.threshold = str_to_us(threshold_str);
+	parameters.window    = str_to_us(window_str);
+	parameters.width     = str_to_us(width_str);
+
+	/* mount the debugfs */
+	if (debugfs_mount(NULL))
+		die ("error mounting debugfs: %s\n", strerror(errno));
+
+	/* load the hwlat_detector module */
+	if (hwlat_load_module())
+		die("error loading hwlat_detector kernel module:
%s\n",strerror(errno)); +
+	/* get defaults if not specified on cmdline */
+	if (parameters.window == -1)
+		parameters.window = hwlat_get_value("window");
+	if (parameters.width == -1)
+		parameters.width = hwlat_get_value("width");
+	if (parameters.threshold == -1)
+		parameters.threshold = hwlat_get_value("threshold");
+
+	/* sanity check */
+	if (parameters.width > parameters.window)
+		die ("invalid values for window/width (%ld/%ld)
(window must be larger)\n",
+		     parameters.window, parameters.width);
+
+	debug("window:    %dus\n", parameters.window);
+	debug("width:     %dus\n", parameters.width);
+	debug("threshold: %dus\n", parameters.threshold);
+	debug("duration:  %ds\n", parameters.duration);
+	debug("cleanup:   %s\n", parameters.cleanup ? "true" :
"false");
+	debug("debug:     %s\n", parameters.debugging ? "true" :
"false"); +
+	signal(SIGINT,  sig_cleanup);
+	signal(SIGABRT, sig_cleanup);
+
+	/* do detection */
+	detect(&parameters);
+	
+	/* clean up */
+	hwlat_unload_module();
+	debugfs_umount();
+
+	return 0;
+}
+
diff --git a/tools/perf/builtin.h b/tools/perf/builtin.h
index ad5f0f4..6d6c2f1 100644
--- a/tools/perf/builtin.h
+++ b/tools/perf/builtin.h
@@ -26,5 +26,5 @@ extern int cmd_top(int argc, const char **argv, const
char *prefix); extern int cmd_trace(int argc, const char **argv, const
char *prefix); extern int cmd_version(int argc, const char **argv,
const char *prefix); extern int cmd_probe(int argc, const char **argv,
const char *prefix); -
+extern int cmd_latency(int argc, const char **argv, const char
*prefix); #endif
diff --git a/tools/perf/command-list.txt b/tools/perf/command-list.txt
index 6475db4..2063e37 100644
--- a/tools/perf/command-list.txt
+++ b/tools/perf/command-list.txt
@@ -12,3 +12,4 @@ perf-timechart			mainporcelain
common perf-top			mainporcelain common
 perf-trace			mainporcelain common
 perf-probe			mainporcelain common
+perf-latency			mainporcelain common
-- 
1.6.2.5


[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 198 bytes --]

Re: [PATCH 3/3] perf latency builtin command

[Ingo Molnar]

Clark, John,

* Clark Williams <williams@redhat.com> wrote:

> 
> This is the first cut at a 'perf latency' command to manage the
> hwlat_detector kernel module, used to detect hardware induced
> latencies (e.g. SMIs).
> 
> Signed-off-by: Clark Williams <williams@redhat.com>
> ---
>  tools/perf/Documentation/perf-latency.txt |   64 +++++
>  tools/perf/Documentation/perf.txt         |    2 +-
>  tools/perf/Makefile                       |    3 +
>  tools/perf/builtin-latency.c              |  383
> +++++++++++++++++++++++++++++ tools/perf/builtin.h
> |    2 +- tools/perf/command-list.txt               |    1 +
>  6 files changed, 453 insertions(+), 2 deletions(-)
>  create mode 100644 tools/perf/Documentation/perf-latency.txt
>  create mode 100644 tools/perf/builtin-latency.c
> 
> diff --git a/tools/perf/Documentation/perf-latency.txt
> b/tools/perf/Documentation/perf-latency.txt new file mode 100644
> index 0000000..f615d08
> --- /dev/null
> +++ b/tools/perf/Documentation/perf-latency.txt
> @@ -0,0 +1,64 @@
> +perf-latency(1)
> +===============
> +
> +NAME
> +----
> +perf-latency - check for hardware latencies
> +
> +SYNOPSIS
> +--------
> +[verse]
> +'per latency' [OPTIONS]
> +
> +DESCRIPTION
> +-----------
> +This command manages the hwlat_detector kernel module, which is used
> +to test the system for hardware induced latencies. The command runs
> +for a specified amount of time (default: 60 seconds) and samples the
> +system Time Stamp Counter (TSC) register, looking for gaps which
> +exceed a threshold. If a gap exceeding the threshold is seen, a
> +timestamp and the gap (in microseconds) is printed to the standard
> +output.
> +
> +OPTIONS
> +-------
> +--duration=<n>{s,m,h,d,w}::
> +	The length of time the test should run. (default: 60 seconds)
> +
> +--window=<n>{us,ms,s,m}::
> +	The sample period for the test. (default 1 second)
> +
> +--width==<n>{us,ms,s,m}::
> +	The test time within the sample window. (default 500
> +	milliseconds)
> +
> +--threshold==<n>{us,ms,s}::
> +	Threshold above which is considered a latency. (default
> 50 microseconds) +
> +--cleanup::
> +	Force unload of hwlat_detector module and umount of debugfs.

I'm wondering whether we could do something perf event based that makes 
'perf latency' self-sufficient and eliminates the debugfs interface.

( We could still merge the first two patches in their current form as 
  they are clear improvements in terms of debugfs access within perf - 
  so no work is lost and progress is possible. )

Basically hwlat_detector is using stop_machine_run() plus a tight rdtsc 
based loop to sample what is happening in the system. Much of 
hwlat_detector.c deals with getting that information (and parameters) 
back and forth between user space and kernel space.

Couldnt we move that functionality a bit closer to perf by creating 
special events in a tight loop that generate a stream of perf events, 
and let the rest of perf events take over the details, and do the 
analysis in the user-space builtin-latency.c code?
 
Also, do we need stop_machine_run() - couldnt we do the measurement on a 
specific CPU with irqs (and NMIs) disabled [but other CPUs still 
running]?

This would all still be possible in the .33 timeframe i suspect, as what 
we need is really just a special event (via TRACE_EVENT() perhaps), and 
a way to trigger it via a 'run this many times' parameter. (i.e. event 
injection - we want to have that kind of support in perf events anyway)

This would simplify and standardize hw-latency detection, without losing 
any utility - and we wouldnt have to go via some special debugfs 
interface to access the hwlat_detect module.

Thoughts?

	Ingo

Re: [PATCH 3/3] perf latency builtin command

[Clark Williams]

[-- Attachment #1: Type: text/plain, Size: 2673 bytes --]

On Tue, 3 Nov 2009 20:28:39 +0100
Ingo Molnar <mingo@elte.hu> wrote:

> 
> Clark, John,
> 
> I'm wondering whether we could do something perf event based that makes 
> 'perf latency' self-sufficient and eliminates the debugfs interface.
> 
> ( We could still merge the first two patches in their current form as 
>   they are clear improvements in terms of debugfs access within perf - 
>   so no work is lost and progress is possible. )

Yeah, I figured that the first two patches were improvements. I may
poke around a little more to see if we can factor out some more
duplicate routines. 

> 
> Basically hwlat_detector is using stop_machine_run() plus a tight rdtsc 
> based loop to sample what is happening in the system. Much of 
> hwlat_detector.c deals with getting that information (and parameters) 
> back and forth between user space and kernel space.
> 
> Couldnt we move that functionality a bit closer to perf by creating 
> special events in a tight loop that generate a stream of perf events, 
> and let the rest of perf events take over the details, and do the 
> analysis in the user-space builtin-latency.c code?
>  
> Also, do we need stop_machine_run() - couldnt we do the measurement on a 
> specific CPU with irqs (and NMIs) disabled [but other CPUs still 
> running]?
> 

So what would the source of the event's be and how confident would we
be that they're accurate? Jon used stop_machine() so that *nothing*
under the control of Linux is going to happen during the test; no
C-state changes, no interrupts, nada. The intent is that if there's a
gap seen in the TSC values, it's because something happened that's out
of our control.

> This would all still be possible in the .33 timeframe i suspect, as what 
> we need is really just a special event (via TRACE_EVENT() perhaps), and 
> a way to trigger it via a 'run this many times' parameter. (i.e. event 
> injection - we want to have that kind of support in perf events anyway)
> 

Hmmm, seems like what you're saying is that we'd poll a free running
perf counter (or some equivalent, still learning about the guts of perf
event system), detect a gap at the low level and just send an event
with that info up to user-space? That would work...

What counter(s) would we use for detecting a gap in time?

> This would simplify and standardize hw-latency detection, without losing 
> any utility - and we wouldnt have to go via some special debugfs 
> interface to access the hwlat_detect module.
> 
> Thoughts?

As long as we feel confident that we can detect temporal gaps with a
performance counter, I'd be ok with it. 

Clark



[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 198 bytes --]

Re: [PATCH 3/3] perf latency builtin command

[Ingo Molnar]

* Clark Williams <williams@redhat.com> wrote:

> > Basically hwlat_detector is using stop_machine_run() plus a tight 
> > rdtsc based loop to sample what is happening in the system. Much of 
> > hwlat_detector.c deals with getting that information (and 
> > parameters) back and forth between user space and kernel space.
> > 
> > Couldnt we move that functionality a bit closer to perf by creating 
> > special events in a tight loop that generate a stream of perf 
> > events, and let the rest of perf events take over the details, and 
> > do the analysis in the user-space builtin-latency.c code?
> >  
> > Also, do we need stop_machine_run() - couldnt we do the measurement 
> > on a specific CPU with irqs (and NMIs) disabled [but other CPUs 
> > still running]?
> 
> So what would the source of the event's be and how confident would we 
> be that they're accurate? Jon used stop_machine() so that *nothing* 
> under the control of Linux is going to happen during the test; no 
> C-state changes, no interrupts, nada. The intent is that if there's a 
> gap seen in the TSC values, it's because something happened that's out 
> of our control.

Yes - that goal is sensible. It could be achieved by running a long 
sampling period on all online CPUs, using SCHED_FIFO:99 priority, right?

What we need to enable this is a way to start a measurement period, 
which outputs its result(s) to a perf event channel.

A first hack could be an ioctl extension in 
kernel/perf_event.c:perf_ioctl(), PERF_EVENT_IOC_INJECT or so.

PERF_EVENT_IOC_INJECT would inject an artificial trace event into the 
kernel, with its parameters defined by user-space.

Initially (for a prototype) you could hardcode it to be purely hwlat 
specific - i.e. the parameters would directly turn into 'disable irqs, 
run a tight TSC loop, report results'.

See commit 8968f9d how to define a new event via TRACE_EVENT(). Once a 
hwlat tracepoint is defined, it could be triggered via:

  trace_hwlat_result(loops, max_delay, min_delay, sum_delay)

I.e. it would look like this (in the prototype), in 
kernel/perf_event.c:perf_ioctl():

	case PERF_EVENT_IOC_INJECT: {

		unigned long hwlat_loops = arg;

		local_irq_disable();
		t0 = get_cycles();
		for (i = 0; i < hwlat_loops; i++) {
			t1 = get_cycles();
			hwlat_delay = t1 - t0;
			hwlat_delay_max = max(hwlat_delay_max, hwlat_delay);
			...
		}

		local_irq_enable();

		trace_hwlat_report(hwlat_loops, hwlat_delay_max);
	}

Note how simple the structure is - one event, one callback that does the 
hwlat measurement loop - and a line to report the resuls. All the rest 
(buffering, enumeration, post-processing, etc.) can be done via using 
'perf latency' and existing perf facilities.

Note at the advantages: we'd have hwlat _and_ the tooling in the kernel 
as one package in essence. Makes for an easy pull request: it fits 
nicely into the existing scheme of things and anyone can run 'perf 
latency' to see what it is good for.

Can you see any fundamental problems with such an approach?

	Ingo