Re: [PATCH] Provide additional sample information to Python scripts

[Joseph Schuchart <joseph.schuchart@tu-dresden.de>]

[-- Attachment #1.1: Type: text/plain, Size: 1364 bytes --]

On 07.03.2014 15:18, Arnaldo Carvalho de Melo wrote:
> Can you please resend, against the perf/core branch in
> git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux.git, and as an
> attachement or making sure that the patch is not mangled?

Arnaldo,

Please find attached our changes. I am sending you two patches since I
came across possible memory leaks while working on the original patch.
The first patch (perf_python_retval_decref.patch) adds calls to
Py_DECREF for the references returned by PyObject_CallObject().

The second patch (perf_python_add_sample.patch) contains our changes to
hand down the sample information for generic events. In addition, the
call-chain of the samples are constructed into a list and passed on. In
the case of generic events, this is just another entry in the dictionary
that is passed to the script as sole argument. For tracepoint events,
this adds another argument and hence changes the scripting interface.
Please feel free to remove these lines in python_process_tracepoint() if
you think that this is problematic.

I hope it is no problem that I am sending you two patches in one mail.
The patches are based on the git repository you pointed out, last
updated on April 2nd (commit 675b44bdf5f2a245f4479c5a8c40abf591007f36).

Please let me know if you have any questions.

Thanks,
Joseph

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: perf_python_add_sample.patch --]
[-- Type: text/x-patch; name="perf_python_add_sample.patch", Size: 6624 bytes --]

Perf: Provide sample information and call-chain to Python script

Provide additional sample information on generic events to Python
scripts, including pid, tid, and cpu for which the event was recorded.
Additionally, provide the call-chain recorded at each event with
resolved symbols (for both generic and tracepoint events). 
At the moment, the pointer to the sample struct is passed to scripts, 
which seems to be of little use. The patch puts this information in 
dictionaries for easy access by Python scripts.

Signed-off-by: Joseph Schuchart <joseph.schuchart@tu-dresden.de>
Acked-by: Thomas Ilsche <thomas.ilsche@tu-dresden.de>

diff --git a/tools/perf/util/scripting-engines/trace-event-python.c b/tools/perf/util/scripting-engines/trace-event-python.c
index a7c8932..255d451 100644
--- a/tools/perf/util/scripting-engines/trace-event-python.c
+++ b/tools/perf/util/scripting-engines/trace-event-python.c
@@ -32,6 +32,7 @@
 #include "../event.h"
 #include "../thread.h"
 #include "../trace-event.h"
+#include "../machine.h"
 
 PyMODINIT_FUNC initperf_trace_context(void);
 
@@ -234,12 +235,90 @@ static inline struct event_format *find_cache_event(struct perf_evsel *evsel)
 	return event;
 }
 
+
+static PyObject *python_process_callchain(struct perf_sample *sample,
+					 struct perf_evsel *evsel,
+					 struct addr_location *al)
+{
+	PyObject *pylist;
+
+	pylist = PyList_New(0);
+	if (!pylist)
+		Py_FatalError("couldn't create Python list");
+
+	if (!symbol_conf.use_callchain || !sample->callchain)
+		goto exit;
+
+	if (machine__resolve_callchain(al->machine, evsel, al->thread,
+					   sample, NULL, NULL,
+					   PERF_MAX_STACK_DEPTH) != 0) {
+		pr_err("Failed to resolve callchain. Skipping\n");
+		goto exit;
+	}
+	callchain_cursor_commit(&callchain_cursor);
+
+
+	while (1) {
+		PyObject *pyelem;
+		struct callchain_cursor_node *node;
+		node = callchain_cursor_current(&callchain_cursor);
+		if (!node)
+			break;
+
+		pyelem = PyDict_New();
+		if (!pyelem)
+			Py_FatalError("couldn't create Python dictionary");
+
+
+		pydict_set_item_string_decref(pyelem, "ip",
+				PyLong_FromUnsignedLongLong(node->ip));
+
+		if (node->sym) {
+			PyObject *pysym  = PyDict_New();
+			if (!pysym)
+				Py_FatalError("couldn't create Python dictionary");
+			pydict_set_item_string_decref(pysym, "start",
+					PyLong_FromUnsignedLongLong(node->sym->start));
+			pydict_set_item_string_decref(pysym, "end",
+					PyLong_FromUnsignedLongLong(node->sym->end));
+			pydict_set_item_string_decref(pysym, "binding",
+					PyInt_FromLong(node->sym->binding));
+			pydict_set_item_string_decref(pysym, "name",
+					PyString_FromStringAndSize(node->sym->name,
+							node->sym->namelen));
+			pydict_set_item_string_decref(pyelem, "sym", pysym);
+		}
+
+		if (node->map) {
+			struct map *map = node->map;
+			const char *dsoname = "[unknown]";
+			if (map && map->dso && (map->dso->name || map->dso->long_name)) {
+				if (symbol_conf.show_kernel_path && map->dso->long_name)
+					dsoname = map->dso->long_name;
+				else if (map->dso->name)
+					dsoname = map->dso->name;
+			}
+			pydict_set_item_string_decref(pyelem, "dso",
+					PyString_FromString(dsoname));
+		}
+
+		callchain_cursor_advance(&callchain_cursor);
+		PyList_Append(pylist, pyelem);
+		Py_DECREF(pyelem);
+	}
+
+exit:
+	return pylist;
+}
+
+
 static void python_process_tracepoint(struct perf_sample *sample,
 				      struct perf_evsel *evsel,
 				      struct thread *thread,
 				      struct addr_location *al)
 {
-	PyObject *handler, *retval, *context, *t, *obj, *dict = NULL;
+	PyObject *handler, *retval, *context, *t, *obj, *callchain;
+	PyObject *dict = NULL;
 	static char handler_name[256];
 	struct format_field *field;
 	unsigned long long val;
@@ -327,6 +406,14 @@ static void python_process_tracepoint(struct perf_sample *sample,
 			pydict_set_item_string_decref(dict, field->name, obj);
 
 	}
+
+	/* ip unwinding */
+	callchain = python_process_callchain(sample, evsel, al);
+	if (handler)
+		PyTuple_SetItem(t, n++, callchain);
+	else
+		pydict_set_item_string_decref(dict, "callchain", callchain);
+
 	if (!handler)
 		PyTuple_SetItem(t, n++, dict);
 
@@ -360,7 +447,7 @@ static void python_process_general_event(struct perf_sample *sample,
 					 struct thread *thread,
 					 struct addr_location *al)
 {
-	PyObject *handler, *retval, *t, *dict;
+	PyObject *handler, *retval, *t, *dict, *dict_sample, *callchain;
 	static char handler_name[64];
 	unsigned n = 0;
 
@@ -376,6 +463,10 @@ static void python_process_general_event(struct perf_sample *sample,
 	if (!dict)
 		Py_FatalError("couldn't create Python dictionary");
 
+	dict_sample = PyDict_New();
+	if (!dict_sample)
+		Py_FatalError("couldn't create Python dictionary");
+
 	snprintf(handler_name, sizeof(handler_name), "%s", "process_event");
 
 	handler = PyDict_GetItemString(main_dict, handler_name);
@@ -385,15 +476,30 @@ static void python_process_general_event(struct perf_sample *sample,
 	pydict_set_item_string_decref(dict, "ev_name", PyString_FromString(perf_evsel__name(evsel)));
 	pydict_set_item_string_decref(dict, "attr", PyString_FromStringAndSize(
 			(const char *)&evsel->attr, sizeof(evsel->attr)));
-	pydict_set_item_string_decref(dict, "sample", PyString_FromStringAndSize(
-			(const char *)sample, sizeof(*sample)));
+
+	pydict_set_item_string_decref(dict_sample, "pid",
+			PyInt_FromLong(sample->pid));
+	pydict_set_item_string_decref(dict_sample, "tid",
+			PyInt_FromLong(sample->tid));
+	pydict_set_item_string_decref(dict_sample, "cpu",
+			PyInt_FromLong(sample->cpu));
+	pydict_set_item_string_decref(dict_sample, "time",
+			PyLong_FromUnsignedLongLong(sample->time));
+	pydict_set_item_string_decref(dict_sample, "period",
+			PyLong_FromUnsignedLongLong(sample->period));
+	pydict_set_item_string_decref(dict, "sample", dict_sample);
+
+	/* ip unwinding */
+	callchain = python_process_callchain(sample, evsel, al);
+	pydict_set_item_string_decref(dict, "callchain", callchain);
+
 	pydict_set_item_string_decref(dict, "raw_buf", PyString_FromStringAndSize(
 			(const char *)sample->raw_data, sample->raw_size));
 	pydict_set_item_string_decref(dict, "comm",
 			PyString_FromString(thread__comm_str(thread)));
 	if (al->map) {
 		pydict_set_item_string_decref(dict, "dso",
-			PyString_FromString(al->map->dso->name));
+				PyString_FromString(al->map->dso->name));
 	}
 	if (al->sym) {
 		pydict_set_item_string_decref(dict, "symbol",

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.3: perf_python_retval_decref.patch --]
[-- Type: text/x-patch; name="perf_python_retval_decref.patch", Size: 2224 bytes --]

Perf: Fix possible memory leaks in Python interface

The function PyObject_CallObject() returns a new PyObject reference 
on which Py_DECREF has to be called to avoid memory leaks.
This patch adds these calls where necessary.

Signed-off-by: Joseph Schuchart <joseph.schuchart@tu-dresden.de>

diff --git a/tools/perf/util/scripting-engines/trace-event-python.c b/tools/perf/util/scripting-engines/trace-event-python.c
index cd9774d..ee17f64 100644
--- a/tools/perf/util/scripting-engines/trace-event-python.c
+++ b/tools/perf/util/scripting-engines/trace-event-python.c
@@ -97,6 +97,8 @@ static void define_value(enum print_arg_type field_type,
 		retval = PyObject_CallObject(handler, t);
 		if (retval == NULL)
 			handler_call_die(handler_name);
+		else
+			Py_DECREF(retval);
 	}
 
 	Py_DECREF(t);
@@ -143,6 +145,8 @@ static void define_field(enum print_arg_type field_type,
 		retval = PyObject_CallObject(handler, t);
 		if (retval == NULL)
 			handler_call_die(handler_name);
+		else
+			Py_DECREF(retval);
 	}
 
 	Py_DECREF(t);
@@ -333,6 +337,8 @@ static void python_process_tracepoint(struct perf_sample *sample,
 		retval = PyObject_CallObject(handler, t);
 		if (retval == NULL)
 			handler_call_die(handler_name);
+		else
+			Py_DECREF(retval);
 	} else {
 		handler = PyDict_GetItemString(main_dict, "trace_unhandled");
 		if (handler && PyCallable_Check(handler)) {
@@ -340,6 +346,8 @@ static void python_process_tracepoint(struct perf_sample *sample,
 			retval = PyObject_CallObject(handler, t);
 			if (retval == NULL)
 				handler_call_die("trace_unhandled");
+			else
+				Py_DECREF(retval);
 		}
 		Py_DECREF(dict);
 	}
@@ -399,6 +407,8 @@ static void python_process_general_event(struct perf_sample *sample,
 	retval = PyObject_CallObject(handler, t);
 	if (retval == NULL)
 		handler_call_die(handler_name);
+	else
+		Py_DECREF(retval);
 exit:
 	Py_DECREF(dict);
 	Py_DECREF(t);
@@ -444,8 +454,8 @@ static int run_start_sub(void)
 	retval = PyObject_CallObject(handler, NULL);
 	if (retval == NULL)
 		handler_call_die("trace_begin");
-
-	Py_DECREF(retval);
+	else
+		Py_DECREF(retval);
 	return err;
 error:
 	Py_XDECREF(main_dict);

[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4943 bytes --]