Re: [PATCH 4/4] perf tools: add JVMTI agent library

[Adrian Hunter <adrian.hunter@intel.com>]

On 11/02/15 01:42, Stephane Eranian wrote:
> This is a standalone JVMTI library to help  profile Java jitted
> code with perf record/perf report. The library is not installed
> or compiled automatically by perf Makefile. It is not used
> directly by perf. It is arch agnostic and has been tested on
> X86 and ARM. It needs to be used with a Java runtime, such
> as OpenJDK, as follows:
> 
> $ java -agentpath:libjvmti.so .......
> 
> When used this way, java will generate a jitdump binary file in
> $HOME/.debug/java/jit/java-jit-*
> 
> This binary dump file contains information to help symbolize and
> annotate jitted code.
> 
> The next step is to inject the jitdump information into the
> perf.data file:
> $ perf inject -j $HOME/.debug/java/jit/java-jit-XXXX/jit-ZZZ.dump \
>               -i perf.data -o perf.data.jitted
> 
> This injects the MMAP records to cover the jitted code and also generates
> one ELF image for each jitted function. The ELF images are created in the
> same subdir as the jitdump file. The MMAP records point there too.
> 
> Then to visualize the function or asm profile, simply use the regular
> perf commands:
> $ perf report -i perf.data.jitted
> or
> $ perf annotate -i perf.data.jitted
> 
> JVMTI agent code adapted from OProfile's opagent code.
> 
> Signed-off-by: Stephane Eranian <eranian@google.com>
> ---
>  tools/perf/jvmti/Makefile      |  70 +++++++++
>  tools/perf/jvmti/jvmti_agent.c | 349 +++++++++++++++++++++++++++++++++++++++++
>  tools/perf/jvmti/jvmti_agent.h |  23 +++
>  tools/perf/jvmti/libjvmti.c    | 149 ++++++++++++++++++
>  4 files changed, 591 insertions(+)
>  create mode 100644 tools/perf/jvmti/Makefile
>  create mode 100644 tools/perf/jvmti/jvmti_agent.c
>  create mode 100644 tools/perf/jvmti/jvmti_agent.h
>  create mode 100644 tools/perf/jvmti/libjvmti.c
> 
> diff --git a/tools/perf/jvmti/Makefile b/tools/perf/jvmti/Makefile
> new file mode 100644
> index 0000000..9eda64b
> --- /dev/null
> +++ b/tools/perf/jvmti/Makefile
> @@ -0,0 +1,70 @@
> +ARCH=$(shell uname -m)
> +
> +ifeq ($(ARCH), x86_64)
> +JARCH=amd64
> +endif
> +ifeq ($(ARCH), armv7l)
> +JARCH=armhf
> +endif
> +ifeq ($(ARCH), armv6l)
> +JARCH=armhf
> +endif
> +ifeq ($(ARCH), ppc64)
> +JARCH=powerpc
> +endif
> +ifeq ($(ARCH), ppc64le)
> +JARCH=powerpc
> +endif
> +
> +DESTDIR=/usr/local
> +
> +VERSION=1
> +REVISION=0
> +AGE=0
> +
> +LN=ln -sf
> +RM=rm
> +
> +SJVMTI=libjvmti.so.$(VERSION).$(REVISION).$(AGE)
> +VJVMTI=libjvmti.so.$(VERSION)
> +SLDFLAGS=-shared -Wl,-soname -Wl,$(VLIBPFM)
> +SOLIBEXT=so
> +
> +JDIR=$(shell /usr/sbin/update-java-alternatives -l | head -1 | cut -d ' ' -f 3)
> +# -lrt required in 32-bit mode for clock_gettime()
> +LIBS=-lelf -lrt
> +INCDIR=-I $(JDIR)/include -I $(JDIR)/include/linux
> +
> +TARGETS=$(SJVMTI)
> +
> +SRCS=libjvmti.c jvmti_agent.c
> +OBJS=$(SRCS:.c=.o)
> +SOBJS=$(OBJS:.o=.lo)
> +OPT=-O2 -g -Werror -Wall
> +
> +CFLAGS=$(INCDIR) $(OPT)
> +
> +all: $(TARGETS)
> +
> +.c.o:
> +	$(CC) $(CFLAGS) -c $*.c
> +.c.lo:
> +	$(CC) -fPIC -DPIC $(CFLAGS) -c $*.c -o $*.lo
> +
> +$(OBJS) $(SOBJS): Makefile jvmti_agent.h ../util/jitdump.h
> +
> +$(SJVMTI):  $(SOBJS)
> +	$(CC) $(CFLAGS) $(SLDFLAGS)  -o $@ $(SOBJS) $(LIBS)
> +	$(LN) $@ libjvmti.$(SOLIBEXT)
> +
> +clean:
> +	$(RM) -f *.o *.so.* *.so *.lo
> +
> +install:
> +	-mkdir -p $(DESTDIR)/lib
> +	install -m 755 $(SJVMTI) $(DESTDIR)/lib/
> +	(cd $(DESTDIR)/lib; $(LN) $(SJVMTI) $(VJVMTI))
> +	(cd $(DESTDIR)/lib; $(LN) $(SJVMTI) libjvmti.$(SOLIBEXT))
> +	ldconfig
> +
> +.SUFFIXES: .c .S .o .lo
> diff --git a/tools/perf/jvmti/jvmti_agent.c b/tools/perf/jvmti/jvmti_agent.c
> new file mode 100644
> index 0000000..d2d5215
> --- /dev/null
> +++ b/tools/perf/jvmti/jvmti_agent.c
> @@ -0,0 +1,349 @@
> +/*
> + * jvmti_agent.c: JVMTI agent interface
> + *
> + * Adapted from the Oprofile code in opagent.c:
> + * This library is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * This library is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with this library; if not, write to the Free Software
> + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
> + *
> + * Copyright 2007 OProfile authors
> + * Jens Wilke
> + * Daniel Hansel
> + * Copyright IBM Corporation 2007
> + */
> +#include <sys/types.h>
> +#include <sys/stat.h> /* for mkdir() */
> +#include <stdio.h>
> +#include <errno.h>
> +#include <string.h>
> +#include <stdlib.h>
> +#include <stdint.h>
> +#include <limits.h>
> +#include <fcntl.h>
> +#include <unistd.h>
> +#include <time.h>
> +#include <syscall.h> /* for gettid() */
> +#include <err.h>
> +
> +#include "jvmti_agent.h"
> +#include "../util/jitdump.h"
> +
> +#define JIT_LANG "java"
> +
> +static char jit_path[PATH_MAX];
> +
> +/*
> + * padding buffer
> + */
> +static const char pad_bytes[7];
> +
> +/*
> + * perf_events event fd
> + */
> +static int perf_fd;
> +
> +static inline pid_t gettid(void)
> +{
> +	return (pid_t)syscall(__NR_gettid);
> +}
> +
> +static int get_e_machine(struct jitheader *hdr)
> +{
> +	ssize_t sret;
> +	char id[16];
> +	int fd, ret = -1;
> +	int m = -1;
> +	struct {
> +		uint16_t e_type;
> +		uint16_t e_machine;
> +	} info;
> +
> +	fd = open("/proc/self/exe", O_RDONLY);
> +	if (fd == -1)
> +		return -1;
> +
> +	sret = read(fd, id, sizeof(id));
> +	if (sret != sizeof(id))
> +		goto error;
> +
> +	/* check ELF signature */
> +	if (id[0] != 0x7f || id[1] != 'E' || id[2] != 'L' || id[3] != 'F')
> +		goto error;
> +
> +	sret = read(fd, &info, sizeof(info));
> +	if (sret != sizeof(info))
> +		goto error;
> +
> +	m = info.e_machine;
> +	if (m < 0)
> +		m = 0; /* ELF EM_NONE */
> +
> +	hdr->elf_mach = m;
> +	ret = 0;
> +error:
> +	close(fd);
> +	return ret;
> +}
> +
> +#define CLOCK_DEVICE "/dev/trace_clock"
> +#define CLOCKFD 3
> +#define FD_TO_CLOCKID(fd)	((~(clockid_t) (fd) << 3) | CLOCKFD)
> +#define CLOCKID_TO_FD(id)	((~(int) (id) >> 3) & ~CLOCKFD)
> +
> +#define NSEC_PER_SEC	1000000000
> +
> +#ifndef CLOCK_INVALID
> +#define CLOCK_INVALID -1
> +#endif
> +
> +static inline clockid_t get_clockid(int fd)
> +{
> +	return FD_TO_CLOCKID(fd);
> +}
> +
> +static int
> +perf_open_timestamp(void)
> +{
> +	int fd, id;
> +
> +	fd = open(CLOCK_DEVICE, O_RDONLY);
> +	if (fd == -1) {
> +		if (errno == ENOENT)
> +			warnx("jvmti: %s not present, check your kernel for trace_clock module", CLOCK_DEVICE);
> +		if (errno == EPERM)
> +			warnx("jvmti: %s has wrong permissions, suggesting chmod 644 %s", CLOCK_DEVICE, CLOCK_DEVICE);
> +	}
> +
> +	id = get_clockid(fd);
> +	if (CLOCK_INVALID == id)
> +		return CLOCK_INVALID;
> +
> +	return get_clockid(fd);
> +}
> +
> +static inline void
> +perf_close_timestamp(int id)
> +{
> +        close(CLOCKID_TO_FD(id));
> +}
> +
> +
> +static inline uint64_t
> +timespec_to_ns(const struct timespec *ts)
> +{
> +        return ((uint64_t) ts->tv_sec * NSEC_PER_SEC) + ts->tv_nsec;
> +}
> +
> +static inline uint64_t
> +perf_get_timestamp(int id)
> +{
> +	struct timespec ts;
> +
> +	clock_gettime(id, &ts);
> +	return timespec_to_ns(&ts);
> +}
> +
> +static int
> +debug_cache_init(void)
> +{
> +	char str[32];
> +	char *base, *p;
> +	struct tm tm;
> +	time_t t;
> +	int ret;
> +
> +	time(&t);
> +	localtime_r(&t, &tm);
> +
> +	base = getenv("JITDUMPDIR");
> +	if (!base)
> +		base = getenv("HOME");
> +	if (!base)
> +		base = ".";
> +
> +	strftime(str, sizeof(str), JIT_LANG"-jit-%Y%m%d", &tm);
> +
> +	snprintf(jit_path, PATH_MAX - 1, "%s/.debug/", base);
> +
> +	ret = mkdir(jit_path, 0755);
> +	if (ret == -1) {
> +		if (errno != EEXIST) {
> +			warn("jvmti: cannot create jit cache dir %s", jit_path);
> +			return -1;
> +		}
> +	}
> +
> +	snprintf(jit_path, PATH_MAX - 1, "%s/.debug/jit", base);
> +	ret = mkdir(jit_path, 0755);
> +	if (ret == -1) {
> +		if (errno != EEXIST) {
> +			warn("cannot create jit cache dir %s", jit_path);
> +			return -1;
> +		}
> +	}
> +
> +	snprintf(jit_path, PATH_MAX - 1, "%s/.debug/jit/%s.XXXXXXXX", base, str);
> +
> +	p = mkdtemp(jit_path);
> +	if (p != jit_path) {
> +		warn("cannot create jit cache dir %s", jit_path);
> +		return -1;
> +	}
> +
> +	return 0;
> +}
> +
> +void *jvmti_open(void)
> +{
> +	int pad_cnt;
> +	char dump_path[PATH_MAX];
> +	struct jitheader header;
> +	FILE *fp;
> +
> +	perf_fd = perf_open_timestamp();
> +	if (perf_fd == -1)
> +		warnx("jvmti: kernel does not support /dev/trace_clock or permissions are wrong on that device");
> +
> +	memset(&header, 0, sizeof(header));
> +
> +	debug_cache_init();
> +
> +	snprintf(dump_path, PATH_MAX, "%s/jit-%i.dump", jit_path, getpid());
> +
> +	fp = fopen(dump_path, "w");
> +	if (!fp) {
> +		warn("jvmti: cannot create %s", dump_path);
> +		goto error;
> +	}
> +
> +	warnx("jvmti: jitdump in %s", dump_path);
> +
> +	if (get_e_machine(&header)) {
> +		warn("get_e_machine failed\n");
> +		goto error;
> +	}
> +
> +	header.magic      = JITHEADER_MAGIC;
> +	header.version    = JITHEADER_VERSION;
> +	header.total_size = sizeof(header);
> +	header.pid        = getpid();
> +
> +	/* calculate amount of padding '\0' */
> +	pad_cnt = PADDING_8ALIGNED(header.total_size);
> +	header.total_size += pad_cnt;
> +
> +	header.timestamp = perf_get_timestamp(perf_fd);
> +
> +	if (!fwrite(&header, sizeof(header), 1, fp)) {
> +		warn("jvmti: cannot write dumpfile header");
> +		goto error;
> +	}
> +
> +	/* write padding '\0' if necessary */
> +	if (pad_cnt && !fwrite(pad_bytes, pad_cnt, 1, fp)) {
> +		warn("jvmti: cannot write dumpfile header padding");
> +		goto error;
> +	}
> +
> +	return fp;
> +error:
> +	fclose(fp);
> +	perf_close_timestamp(perf_fd);
> +	return NULL;
> +}
> +
> +int
> +jvmti_close(void *agent)
> +{
> +	struct jr_code_close rec;
> +	FILE *fp = agent;
> +
> +	if (!fp) {
> +		warnx("jvmti: incalid fd in close_agent");
> +		return -1;
> +	}
> +
> +	rec.p.id = JIT_CODE_CLOSE;
> +	rec.p.total_size = sizeof(rec);
> +
> +	rec.p.timestamp = perf_get_timestamp(perf_fd);
> +
> +	if (!fwrite(&rec, sizeof(rec), 1, fp))
> +		return -1;
> +
> +	fclose(fp);
> +
> +	perf_close_timestamp(perf_fd);
> +
> +	fp = NULL;
> +
> +	return 0;
> +}
> +
> +int jvmti_write_code(void *agent, char const *sym,
> +	uint64_t vma, void const *code, unsigned int const size)
> +{
> +	static int code_generation = 1;
> +	struct jr_code_load rec;
> +	size_t sym_len;
> +	size_t padding_count;
> +	FILE *fp = agent;
> +	int ret = -1;
> +
> +	/* don't care about 0 length function, no samples */
> +	if (size == 0)
> +		return 0;
> +
> +	if (!fp) {
> +		warnx("jvmti: invalid fd in write_native_code");
> +		return -1;
> +	}
> +
> +	sym_len = strlen(sym) + 1;
> +
> +	rec.p.id           = JIT_CODE_LOAD;
> +	rec.p.total_size   = sizeof(rec) + sym_len;
> +	padding_count      = PADDING_8ALIGNED(rec.p.total_size);
> +	rec.p. total_size += padding_count;
> +	rec.p.timestamp    = perf_get_timestamp(perf_fd);

Do you know whether the JVM is guaranteed not to start executing the
generated code before the return of compiled_method_load_cb(), otherwise the
timestamp will be too late?

> +
> +	rec.code_size  = size;
> +	rec.vma        = vma;
> +	rec.code_addr  = vma;
> +	rec.pid	       = getpid();
> +	rec.tid	       = gettid();
> +	rec.code_index = code_generation++;
> +
> +	if (code)
> +		rec.p.total_size += size;
> +
> +	/*
> +	 * If JVM is multi-threaded, nultiple concurrent calls to agent
> +	 * may be possible, so protect file writes
> +	 */
> +	flockfile(fp);
> +
> +	ret = fwrite_unlocked(&rec, sizeof(rec), 1, fp);
> +	fwrite_unlocked(sym, sym_len, 1, fp);
> +	if (code)
> +		fwrite_unlocked(code, size, 1, fp);
> +
> +	if (padding_count)
> +		fwrite_unlocked(pad_bytes, padding_count, 1, fp);
> +
> +	funlockfile(fp);
> +
> +	ret = 0;
> +
> +	return ret;
> +}
> diff --git a/tools/perf/jvmti/jvmti_agent.h b/tools/perf/jvmti/jvmti_agent.h
> new file mode 100644
> index 0000000..54e5c5e
> --- /dev/null
> +++ b/tools/perf/jvmti/jvmti_agent.h
> @@ -0,0 +1,23 @@
> +#ifndef __JVMTI_AGENT_H__
> +#define __JVMTI_AGENT_H__
> +
> +#include <sys/types.h>
> +#include <stdint.h>
> +
> +#define __unused __attribute__((unused))
> +
> +#if defined(__cplusplus)
> +extern "C" {
> +#endif
> +
> +void *jvmti_open(void);
> +int   jvmti_close(void *agent);
> +int   jvmti_write_code(void *agent, char const *symbol_name,
> +		       uint64_t vma, void const *code,
> +		       const unsigned int code_size);
> +
> +#if defined(__cplusplus)
> +}
> +
> +#endif
> +#endif /* __JVMTI_H__ */
> diff --git a/tools/perf/jvmti/libjvmti.c b/tools/perf/jvmti/libjvmti.c
> new file mode 100644
> index 0000000..8b8d782
> --- /dev/null
> +++ b/tools/perf/jvmti/libjvmti.c
> @@ -0,0 +1,149 @@
> +#include <sys/types.h>
> +#include <stdio.h>
> +#include <string.h>
> +#include <stdlib.h>
> +#include <err.h>
> +#include <jvmti.h>
> +
> +#include "jvmti_agent.h"
> +
> +void *jvmti_agent;
> +
> +static void JNICALL
> +compiled_method_load_cb(jvmtiEnv *jvmti,
> +			jmethodID method,
> +			jint code_size,
> +			void const *code_addr,
> +			jint map_length,
> +			jvmtiAddrLocationMap const *map,
> +			void const *compile_info __unused)
> +{
> +	jclass decl_class;
> +	char *class_sign = NULL;
> +	char *func_name = NULL;
> +	char *func_sign = NULL;
> +	jvmtiError ret;
> +	size_t len;
> +
> +	ret = (*jvmti)->GetMethodDeclaringClass(jvmti, method,
> +						&decl_class);
> +	if (ret != JVMTI_ERROR_NONE) {
> +		warnx("jvmti: getmethoddeclaringclass failed");
> +		return;
> +	}
> +
> +	ret = (*jvmti)->GetClassSignature(jvmti, decl_class,
> +					  &class_sign, NULL);
> +	if (ret != JVMTI_ERROR_NONE) {
> +		warnx("jvmti: getclassignature failed");
> +		goto error;
> +	}
> +
> +	ret = (*jvmti)->GetMethodName(jvmti, method, &func_name,
> +				      &func_sign, NULL);
> +	if (ret != JVMTI_ERROR_NONE) {
> +		warnx("jvmti: failed getmethodname");
> +		goto error;
> +	}
> +
> +	len = strlen(func_name) + strlen(class_sign) + strlen(func_sign) + 2;
> +
> +	{
> +		char str[len];
> +		uint64_t addr = (uint64_t)(unsigned long)code_addr;
> +		snprintf(str, len, "%s%s%s", class_sign, func_name, func_sign);
> +		ret = jvmti_write_code(jvmti_agent, str, addr, code_addr, code_size);
> +		if (ret)
> +			warnx("jvmti: write_code() failed");
> +	}
> +error:
> +	(*jvmti)->Deallocate(jvmti, (unsigned char *)func_name);
> +	(*jvmti)->Deallocate(jvmti, (unsigned char *)func_sign);
> +	(*jvmti)->Deallocate(jvmti, (unsigned char *)class_sign);
> +}
> +
> +static void JNICALL
> +code_generated_cb(jvmtiEnv *jvmti,
> +		  char const *name,
> +		  void const *code_addr,
> +		  jint code_size)
> +{
> +	uint64_t addr = (uint64_t)(unsigned long)code_addr;
> +	int ret;
> +
> +	ret = jvmti_write_code(jvmti_agent, name, addr, code_addr, code_size);
> +	if (ret)
> +		warnx("jvmti: write_code() failed for code_generated");
> +}
> +
> +JNIEXPORT jint JNICALL
> +Agent_OnLoad(JavaVM *jvm, char *options, void *reserved __unused)
> +{
> +	jvmtiEventCallbacks cb;
> +	jvmtiCapabilities caps1;
> +	jvmtiEnv *jvmti = NULL;
> +	jint ret;
> +
> +	jvmti_agent = jvmti_open();
> +	if (!jvmti_agent) {
> +		warnx("jvmti: open_agent failed");
> +		return -1;
> +	}
> +
> +	/*
> +	 * Request a JVMTI interface version 1 environment
> +	 */
> +	ret = (*jvm)->GetEnv(jvm, (void *)&jvmti, JVMTI_VERSION_1);
> +	if (ret != JNI_OK) {
> +		warnx("jvmti: jvmti version 1 not supported");
> +		return -1;
> +	}
> +
> +	/*
> +	 * acquire method_load capability, we require it
> +	 */
> +	memset(&caps1, 0, sizeof(caps1));
> +	caps1.can_generate_compiled_method_load_events = 1;
> +
> +	ret = (*jvmti)->AddCapabilities(jvmti, &caps1);
> +	if (ret != JVMTI_ERROR_NONE) {
> +		warnx("jvmti: acquire compiled_method capability failed");
> +		return -1;
> +	}
> +
> +	memset(&cb, 0, sizeof(cb));
> +
> +	cb.CompiledMethodLoad   = compiled_method_load_cb;
> +	cb.DynamicCodeGenerated = code_generated_cb;
> +
> +	ret = (*jvmti)->SetEventCallbacks(jvmti, &cb, sizeof(cb));
> +	if (ret != JVMTI_ERROR_NONE) {
> +		warnx("jvmti: cannot set event callbacks");
> +		return -1;
> +	}
> +
> +	ret = (*jvmti)->SetEventNotificationMode(jvmti, JVMTI_ENABLE,
> +			JVMTI_EVENT_COMPILED_METHOD_LOAD, NULL);
> +	if (ret != JVMTI_ERROR_NONE) {
> +		warnx("jvmti: setnotification failed for method_load");
> +		return -1;
> +	}
> +
> +	ret = (*jvmti)->SetEventNotificationMode(jvmti, JVMTI_ENABLE,
> +			JVMTI_EVENT_DYNAMIC_CODE_GENERATED, NULL);
> +	if (ret != JVMTI_ERROR_NONE) {
> +		warnx("jvmti: setnotification failed on code_generated");
> +		return -1;
> +	}
> +	return 0;
> +}
> +
> +JNIEXPORT void JNICALL
> +Agent_OnUnload(JavaVM *jvm __unused)
> +{
> +	int ret;
> +
> +	ret = jvmti_close(jvmti_agent);
> +	if (ret)
> +		errx(1, "Error: op_close_agent()");
> +}
>