Re: [PATCH 1/8] perf db-export: Add calls parent_id

Re: [PATCH 1/8] perf db-export: Add calls parent_id

[Arnaldo Carvalho de Melo]

Em Thu, Feb 28, 2019 at 03:00:24PM +0200, Adrian Hunter escreveu:
> The call_path can be used to find the parent symbol for a call but not the
> exact parent call. To do that add parent_id to the call_return export. This
> enables the creation of a call tree from the exported data.

Thanks, applied.

- Arnaldo
 
> Signed-off-by: Adrian Hunter <adrian.hunter@intel.com>
> ---
>  tools/perf/util/db-export.c                      | 15 ++++++++++-----
>  tools/perf/util/db-export.h                      |  3 ++-
>  .../util/scripting-engines/trace-event-python.c  |  8 +++++---
>  tools/perf/util/thread-stack.c                   | 16 ++++++++++++++--
>  tools/perf/util/thread-stack.h                   |  6 ++++--
>  5 files changed, 35 insertions(+), 13 deletions(-)
> 
> diff --git a/tools/perf/util/db-export.c b/tools/perf/util/db-export.c
> index de9b4769d06c..d7315a00c731 100644
> --- a/tools/perf/util/db-export.c
> +++ b/tools/perf/util/db-export.c
> @@ -510,18 +510,23 @@ int db_export__call_path(struct db_export *dbe, struct call_path *cp)
>  	return 0;
>  }
>  
> -int db_export__call_return(struct db_export *dbe, struct call_return *cr)
> +int db_export__call_return(struct db_export *dbe, struct call_return *cr,
> +			   u64 *parent_db_id)
>  {
>  	int err;
>  
> -	if (cr->db_id)
> -		return 0;
> -
>  	err = db_export__call_path(dbe, cr->cp);
>  	if (err)
>  		return err;
>  
> -	cr->db_id = ++dbe->call_return_last_db_id;
> +	if (!cr->db_id)
> +		cr->db_id = ++dbe->call_return_last_db_id;
> +
> +	if (parent_db_id) {
> +		if (!*parent_db_id)
> +			*parent_db_id = ++dbe->call_return_last_db_id;
> +		cr->parent_db_id = *parent_db_id;
> +	}
>  
>  	if (dbe->export_call_return)
>  		return dbe->export_call_return(dbe, cr);
> diff --git a/tools/perf/util/db-export.h b/tools/perf/util/db-export.h
> index 67bc6b8ad2d6..4e2424c89df9 100644
> --- a/tools/perf/util/db-export.h
> +++ b/tools/perf/util/db-export.h
> @@ -104,6 +104,7 @@ int db_export__sample(struct db_export *dbe, union perf_event *event,
>  int db_export__branch_types(struct db_export *dbe);
>  
>  int db_export__call_path(struct db_export *dbe, struct call_path *cp);
> -int db_export__call_return(struct db_export *dbe, struct call_return *cr);
> +int db_export__call_return(struct db_export *dbe, struct call_return *cr,
> +			   u64 *parent_db_id);
>  
>  #endif
> diff --git a/tools/perf/util/scripting-engines/trace-event-python.c b/tools/perf/util/scripting-engines/trace-event-python.c
> index 0e17db41b49b..09604c6508f0 100644
> --- a/tools/perf/util/scripting-engines/trace-event-python.c
> +++ b/tools/perf/util/scripting-engines/trace-event-python.c
> @@ -1173,7 +1173,7 @@ static int python_export_call_return(struct db_export *dbe,
>  	u64 comm_db_id = cr->comm ? cr->comm->db_id : 0;
>  	PyObject *t;
>  
> -	t = tuple_new(11);
> +	t = tuple_new(12);
>  
>  	tuple_set_u64(t, 0, cr->db_id);
>  	tuple_set_u64(t, 1, cr->thread->db_id);
> @@ -1186,6 +1186,7 @@ static int python_export_call_return(struct db_export *dbe,
>  	tuple_set_u64(t, 8, cr->return_ref);
>  	tuple_set_u64(t, 9, cr->cp->parent->db_id);
>  	tuple_set_s32(t, 10, cr->flags);
> +	tuple_set_u64(t, 11, cr->parent_db_id);
>  
>  	call_object(tables->call_return_handler, t, "call_return_table");
>  
> @@ -1194,11 +1195,12 @@ static int python_export_call_return(struct db_export *dbe,
>  	return 0;
>  }
>  
> -static int python_process_call_return(struct call_return *cr, void *data)
> +static int python_process_call_return(struct call_return *cr, u64 *parent_db_id,
> +				      void *data)
>  {
>  	struct db_export *dbe = data;
>  
> -	return db_export__call_return(dbe, cr);
> +	return db_export__call_return(dbe, cr, parent_db_id);
>  }
>  
>  static void python_process_general_event(struct perf_sample *sample,
> diff --git a/tools/perf/util/thread-stack.c b/tools/perf/util/thread-stack.c
> index a8b45168513c..41942c2aaa18 100644
> --- a/tools/perf/util/thread-stack.c
> +++ b/tools/perf/util/thread-stack.c
> @@ -49,6 +49,7 @@ enum retpoline_state_t {
>   * @timestamp: timestamp (if known)
>   * @ref: external reference (e.g. db_id of sample)
>   * @branch_count: the branch count when the entry was created
> + * @db_id: id used for db-export
>   * @cp: call path
>   * @no_call: a 'call' was not seen
>   * @trace_end: a 'call' but trace ended
> @@ -59,6 +60,7 @@ struct thread_stack_entry {
>  	u64 timestamp;
>  	u64 ref;
>  	u64 branch_count;
> +	u64 db_id;
>  	struct call_path *cp;
>  	bool no_call;
>  	bool trace_end;
> @@ -280,12 +282,14 @@ static int thread_stack__call_return(struct thread *thread,
>  		.comm = ts->comm,
>  		.db_id = 0,
>  	};
> +	u64 *parent_db_id;
>  
>  	tse = &ts->stack[idx];
>  	cr.cp = tse->cp;
>  	cr.call_time = tse->timestamp;
>  	cr.return_time = timestamp;
>  	cr.branch_count = ts->branch_count - tse->branch_count;
> +	cr.db_id = tse->db_id;
>  	cr.call_ref = tse->ref;
>  	cr.return_ref = ref;
>  	if (tse->no_call)
> @@ -295,7 +299,14 @@ static int thread_stack__call_return(struct thread *thread,
>  	if (tse->non_call)
>  		cr.flags |= CALL_RETURN_NON_CALL;
>  
> -	return crp->process(&cr, crp->data);
> +	/*
> +	 * The parent db_id must be assigned before exporting the child. Note
> +	 * it is not possible to export the parent first because its information
> +	 * is not yet complete because its 'return' has not yet been processed.
> +	 */
> +	parent_db_id = idx ? &(tse - 1)->db_id : NULL;
> +
> +	return crp->process(&cr, parent_db_id, crp->data);
>  }
>  
>  static int __thread_stack__flush(struct thread *thread, struct thread_stack *ts)
> @@ -484,7 +495,7 @@ void thread_stack__sample(struct thread *thread, int cpu,
>  }
>  
>  struct call_return_processor *
> -call_return_processor__new(int (*process)(struct call_return *cr, void *data),
> +call_return_processor__new(int (*process)(struct call_return *cr, u64 *parent_db_id, void *data),
>  			   void *data)
>  {
>  	struct call_return_processor *crp;
> @@ -537,6 +548,7 @@ static int thread_stack__push_cp(struct thread_stack *ts, u64 ret_addr,
>  	tse->no_call = no_call;
>  	tse->trace_end = trace_end;
>  	tse->non_call = false;
> +	tse->db_id = 0;
>  
>  	return 0;
>  }
> diff --git a/tools/perf/util/thread-stack.h b/tools/perf/util/thread-stack.h
> index b7c04e19ad41..9c45f947f5a9 100644
> --- a/tools/perf/util/thread-stack.h
> +++ b/tools/perf/util/thread-stack.h
> @@ -55,6 +55,7 @@ enum {
>   * @call_ref: external reference to 'call' sample (e.g. db_id)
>   * @return_ref:  external reference to 'return' sample (e.g. db_id)
>   * @db_id: id used for db-export
> + * @parent_db_id: id of parent call used for db-export
>   * @flags: Call/Return flags
>   */
>  struct call_return {
> @@ -67,6 +68,7 @@ struct call_return {
>  	u64 call_ref;
>  	u64 return_ref;
>  	u64 db_id;
> +	u64 parent_db_id;
>  	u32 flags;
>  };
>  
> @@ -79,7 +81,7 @@ struct call_return {
>   */
>  struct call_return_processor {
>  	struct call_path_root *cpr;
> -	int (*process)(struct call_return *cr, void *data);
> +	int (*process)(struct call_return *cr, u64 *parent_db_id, void *data);
>  	void *data;
>  };
>  
> @@ -93,7 +95,7 @@ void thread_stack__free(struct thread *thread);
>  size_t thread_stack__depth(struct thread *thread, int cpu);
>  
>  struct call_return_processor *
> -call_return_processor__new(int (*process)(struct call_return *cr, void *data),
> +call_return_processor__new(int (*process)(struct call_return *cr, u64 *parent_db_id, void *data),
>  			   void *data);
>  void call_return_processor__free(struct call_return_processor *crp);
>  int thread_stack__process(struct thread *thread, struct comm *comm,
> -- 
> 2.17.1

-- 

- Arnaldo

Re: [PATCH 2/8] perf scripts python: export-to-sqlite.py: Export calls parent_id

[Arnaldo Carvalho de Melo]

Em Thu, Feb 28, 2019 at 03:00:25PM +0200, Adrian Hunter escreveu:
> Export to the 'calls' table the newly created 'parent_id'.
> 
> Signed-off-by: Adrian Hunter <adrian.hunter@intel.com>

Thanks, applied.

- Arnaldo

> ---
>  tools/perf/scripts/python/export-to-sqlite.py | 12 ++++++++----
>  1 file changed, 8 insertions(+), 4 deletions(-)
> 
> diff --git a/tools/perf/scripts/python/export-to-sqlite.py b/tools/perf/scripts/python/export-to-sqlite.py
> index ed237f2ed03f..eb63e6c7107f 100644
> --- a/tools/perf/scripts/python/export-to-sqlite.py
> +++ b/tools/perf/scripts/python/export-to-sqlite.py
> @@ -222,7 +222,8 @@ if perf_db_export_calls:
>  		'call_id	bigint,'
>  		'return_id	bigint,'
>  		'parent_call_path_id	bigint,'
> -		'flags		integer)')
> +		'flags		integer,'
> +		'parent_id	bigint)')
>  
>  # printf was added to sqlite in version 3.8.3
>  sqlite_has_printf = False
> @@ -321,7 +322,8 @@ if perf_db_export_calls:
>  			'call_id,'
>  			'return_id,'
>  			'CASE WHEN flags=0 THEN \'\' WHEN flags=1 THEN \'no call\' WHEN flags=2 THEN \'no return\' WHEN flags=3 THEN \'no call/return\' WHEN flags=6 THEN \'jump\' ELSE flags END AS flags,'
> -			'parent_call_path_id'
> +			'parent_call_path_id,'
> +			'parent_id'
>  		' FROM calls INNER JOIN call_paths ON call_paths.id = call_path_id')
>  
>  do_query(query, 'CREATE VIEW samples_view AS '
> @@ -373,7 +375,7 @@ if perf_db_export_calls or perf_db_export_callchains:
>  	call_path_query.prepare("INSERT INTO call_paths VALUES (?, ?, ?, ?)")
>  if perf_db_export_calls:
>  	call_query = QSqlQuery(db)
> -	call_query.prepare("INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
> +	call_query.prepare("INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
>  
>  def trace_begin():
>  	print datetime.datetime.today(), "Writing records..."
> @@ -388,6 +390,7 @@ def trace_begin():
>  	sample_table(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
>  	if perf_db_export_calls or perf_db_export_callchains:
>  		call_path_table(0, 0, 0, 0)
> +		call_return_table(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
>  
>  unhandled_count = 0
>  
> @@ -397,6 +400,7 @@ def trace_end():
>  	print datetime.datetime.today(), "Adding indexes"
>  	if perf_db_export_calls:
>  		do_query(query, 'CREATE INDEX pcpid_idx ON calls (parent_call_path_id)')
> +		do_query(query, 'CREATE INDEX pid_idx ON calls (parent_id)')
>  
>  	if (unhandled_count):
>  		print datetime.datetime.today(), "Warning: ", unhandled_count, " unhandled events"
> @@ -452,4 +456,4 @@ def call_path_table(*x):
>  	bind_exec(call_path_query, 4, x)
>  
>  def call_return_table(*x):
> -	bind_exec(call_query, 11, x)
> +	bind_exec(call_query, 12, x)
> -- 
> 2.17.1

-- 

- Arnaldo

Re: [PATCH 4/8] perf scripts python: export-to-postgresql.py: Export calls parent_id

[Arnaldo Carvalho de Melo]

Em Thu, Feb 28, 2019 at 03:00:27PM +0200, Adrian Hunter escreveu:
> Export to the 'calls' table the newly created 'parent_id' and create an
> index for it.

Thanks, applied.

- Arnaldo
 
> Signed-off-by: Adrian Hunter <adrian.hunter@intel.com>
> ---
>  tools/perf/scripts/python/export-to-postgresql.py | 14 +++++++++-----
>  1 file changed, 9 insertions(+), 5 deletions(-)
> 
> diff --git a/tools/perf/scripts/python/export-to-postgresql.py b/tools/perf/scripts/python/export-to-postgresql.py
> index 6358522a69f6..390a351d15ea 100644
> --- a/tools/perf/scripts/python/export-to-postgresql.py
> +++ b/tools/perf/scripts/python/export-to-postgresql.py
> @@ -394,7 +394,8 @@ if perf_db_export_calls:
>  		'call_id	bigint,'
>  		'return_id	bigint,'
>  		'parent_call_path_id	bigint,'
> -		'flags		integer)')
> +		'flags		integer,'
> +		'parent_id	bigint)')
>  
>  do_query(query, 'CREATE VIEW machines_view AS '
>  	'SELECT '
> @@ -479,7 +480,8 @@ if perf_db_export_calls:
>  			'call_id,'
>  			'return_id,'
>  			'CASE WHEN flags=0 THEN \'\' WHEN flags=1 THEN \'no call\' WHEN flags=2 THEN \'no return\' WHEN flags=3 THEN \'no call/return\' WHEN flags=6 THEN \'jump\' ELSE CAST ( flags AS VARCHAR(6) ) END AS flags,'
> -			'parent_call_path_id'
> +			'parent_call_path_id,'
> +			'calls.parent_id'
>  		' FROM calls INNER JOIN call_paths ON call_paths.id = call_path_id')
>  
>  do_query(query, 'CREATE VIEW samples_view AS '
> @@ -575,6 +577,7 @@ def trace_begin():
>  	sample_table(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
>  	if perf_db_export_calls or perf_db_export_callchains:
>  		call_path_table(0, 0, 0, 0)
> +		call_return_table(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
>  
>  unhandled_count = 0
>  
> @@ -657,6 +660,7 @@ def trace_end():
>  					'ADD CONSTRAINT returnfk    FOREIGN KEY (return_id)    REFERENCES samples    (id),'
>  					'ADD CONSTRAINT parent_call_pathfk FOREIGN KEY (parent_call_path_id) REFERENCES call_paths (id)')
>  		do_query(query, 'CREATE INDEX pcpid_idx ON calls (parent_call_path_id)')
> +		do_query(query, 'CREATE INDEX pid_idx ON calls (parent_id)')
>  
>  	if (unhandled_count):
>  		print datetime.datetime.today(), "Warning: ", unhandled_count, " unhandled events"
> @@ -728,7 +732,7 @@ def call_path_table(cp_id, parent_id, symbol_id, ip, *x):
>  	value = struct.pack(fmt, 4, 8, cp_id, 8, parent_id, 8, symbol_id, 8, ip)
>  	call_path_file.write(value)
>  
> -def call_return_table(cr_id, thread_id, comm_id, call_path_id, call_time, return_time, branch_count, call_id, return_id, parent_call_path_id, flags, *x):
> -	fmt = "!hiqiqiqiqiqiqiqiqiqiqii"
> -	value = struct.pack(fmt, 11, 8, cr_id, 8, thread_id, 8, comm_id, 8, call_path_id, 8, call_time, 8, return_time, 8, branch_count, 8, call_id, 8, return_id, 8, parent_call_path_id, 4, flags)
> +def call_return_table(cr_id, thread_id, comm_id, call_path_id, call_time, return_time, branch_count, call_id, return_id, parent_call_path_id, flags, parent_id, *x):
> +	fmt = "!hiqiqiqiqiqiqiqiqiqiqiiiq"
> +	value = struct.pack(fmt, 12, 8, cr_id, 8, thread_id, 8, comm_id, 8, call_path_id, 8, call_time, 8, return_time, 8, branch_count, 8, call_id, 8, return_id, 8, parent_call_path_id, 4, flags, 8, parent_id)
>  	call_file.write(value)
> -- 
> 2.17.1

-- 

- Arnaldo

Re: [PATCH 3/8] perf scripts python: export-to-postgresql.py: Fix invalid input syntax for integer error

[Arnaldo Carvalho de Melo]

Em Thu, Feb 28, 2019 at 03:00:26PM +0200, Adrian Hunter escreveu:
> Fix SQL query error "invalid input syntax for integer":
> 
>   Traceback (most recent call last):
>     File "tools/perf/scripts/python/export-to-postgresql.py", line 465, in <module>
>       do_query(query, 'CREATE VIEW calls_view AS '
>     File "tools/perf/scripts/python/export-to-postgresql.py", line 274, in do_query
>       raise Exception("Query failed: " + q.lastError().text())
>   Exception: Query failed: ERROR:  invalid input syntax for integer: ""
>   LINE 1: ...ch_count,call_id,return_id,CASE WHEN flags=0 THEN '' WHEN fl...
>                                                                ^
>   (22P02) QPSQL: Unable to create query
>   Error running python script tools/perf/scripts/python/export-to-postgresql.py

Thanks, applied.

- Arnaldo
 
> Signed-off-by: Adrian Hunter <adrian.hunter@intel.com>
> Fixes: f08046cb3082 ("perf thread-stack: Represent jmps to the start of a different symbol")
> ---
>  tools/perf/scripts/python/export-to-postgresql.py | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/tools/perf/scripts/python/export-to-postgresql.py b/tools/perf/scripts/python/export-to-postgresql.py
> index 30130213da7e..6358522a69f6 100644
> --- a/tools/perf/scripts/python/export-to-postgresql.py
> +++ b/tools/perf/scripts/python/export-to-postgresql.py
> @@ -478,7 +478,7 @@ if perf_db_export_calls:
>  			'branch_count,'
>  			'call_id,'
>  			'return_id,'
> -			'CASE WHEN flags=0 THEN \'\' WHEN flags=1 THEN \'no call\' WHEN flags=2 THEN \'no return\' WHEN flags=3 THEN \'no call/return\' WHEN flags=6 THEN \'jump\' ELSE flags END AS flags,'
> +			'CASE WHEN flags=0 THEN \'\' WHEN flags=1 THEN \'no call\' WHEN flags=2 THEN \'no return\' WHEN flags=3 THEN \'no call/return\' WHEN flags=6 THEN \'jump\' ELSE CAST ( flags AS VARCHAR(6) ) END AS flags,'
>  			'parent_call_path_id'
>  		' FROM calls INNER JOIN call_paths ON call_paths.id = call_path_id')
>  
> -- 
> 2.17.1

-- 

- Arnaldo

Re: [PATCH 5/8] perf scripts python: exported-sql-viewer.py: Factor out TreeWindowBase

[Arnaldo Carvalho de Melo]

Em Thu, Feb 28, 2019 at 03:00:28PM +0200, Adrian Hunter escreveu:
> Factor out a base class TreeWindowBase from CallGraphWindow, so that
> TreeWindowBase can be reused.

Thanks, applied.

- Arnaldo
 
> Signed-off-by: Adrian Hunter <adrian.hunter@intel.com>
> ---
>  .../scripts/python/exported-sql-viewer.py     | 50 ++++++++++++-------
>  1 file changed, 31 insertions(+), 19 deletions(-)
> 
> diff --git a/tools/perf/scripts/python/exported-sql-viewer.py b/tools/perf/scripts/python/exported-sql-viewer.py
> index 09ce73b07d35..df854f0a69f0 100755
> --- a/tools/perf/scripts/python/exported-sql-viewer.py
> +++ b/tools/perf/scripts/python/exported-sql-viewer.py
> @@ -693,28 +693,16 @@ class VBox():
>  	def Widget(self):
>  		return self.vbox
>  
> -# Context-sensitive call graph window
> -
> -class CallGraphWindow(QMdiSubWindow):
> -
> -	def __init__(self, glb, parent=None):
> -		super(CallGraphWindow, self).__init__(parent)
> +# Tree window base
>  
> -		self.model = LookupCreateModel("Context-Sensitive Call Graph", lambda x=glb: CallGraphModel(x))
> +class TreeWindowBase(QMdiSubWindow):
>  
> -		self.view = QTreeView()
> -		self.view.setModel(self.model)
> -
> -		for c, w in ((0, 250), (1, 100), (2, 60), (3, 70), (4, 70), (5, 100)):
> -			self.view.setColumnWidth(c, w)
> -
> -		self.find_bar = FindBar(self, self)
> -
> -		self.vbox = VBox(self.view, self.find_bar.Widget())
> -
> -		self.setWidget(self.vbox.Widget())
> +	def __init__(self, parent=None):
> +		super(TreeWindowBase, self).__init__(parent)
>  
> -		AddSubWindow(glb.mainwindow.mdi_area, self, "Context-Sensitive Call Graph")
> +		self.model = None
> +		self.view = None
> +		self.find_bar = None
>  
>  	def DisplayFound(self, ids):
>  		if not len(ids):
> @@ -747,6 +735,30 @@ class CallGraphWindow(QMdiSubWindow):
>  		if not found:
>  			self.find_bar.NotFound()
>  
> +
> +# Context-sensitive call graph window
> +
> +class CallGraphWindow(TreeWindowBase):
> +
> +	def __init__(self, glb, parent=None):
> +		super(CallGraphWindow, self).__init__(parent)
> +
> +		self.model = LookupCreateModel("Context-Sensitive Call Graph", lambda x=glb: CallGraphModel(x))
> +
> +		self.view = QTreeView()
> +		self.view.setModel(self.model)
> +
> +		for c, w in ((0, 250), (1, 100), (2, 60), (3, 70), (4, 70), (5, 100)):
> +			self.view.setColumnWidth(c, w)
> +
> +		self.find_bar = FindBar(self, self)
> +
> +		self.vbox = VBox(self.view, self.find_bar.Widget())
> +
> +		self.setWidget(self.vbox.Widget())
> +
> +		AddSubWindow(glb.mainwindow.mdi_area, self, "Context-Sensitive Call Graph")
> +
>  # Child data item  finder
>  
>  class ChildDataItemFinder():
> -- 
> 2.17.1

-- 

- Arnaldo

Re: [PATCH 6/8] perf scripts python: exported-sql-viewer.py: Improve TreeModel abstraction

[Arnaldo Carvalho de Melo]

Em Thu, Feb 28, 2019 at 03:00:29PM +0200, Adrian Hunter escreveu:
> Instead of passing the tree root, get it from a method that can be
> implemented in any derived class.

Thanks, applied.

- Arnaldo
 
> Signed-off-by: Adrian Hunter <adrian.hunter@intel.com>
> ---
>  .../perf/scripts/python/exported-sql-viewer.py  | 17 +++++++++++------
>  1 file changed, 11 insertions(+), 6 deletions(-)
> 
> diff --git a/tools/perf/scripts/python/exported-sql-viewer.py b/tools/perf/scripts/python/exported-sql-viewer.py
> index df854f0a69f0..b2a22525549d 100755
> --- a/tools/perf/scripts/python/exported-sql-viewer.py
> +++ b/tools/perf/scripts/python/exported-sql-viewer.py
> @@ -167,9 +167,10 @@ class Thread(QThread):
>  
>  class TreeModel(QAbstractItemModel):
>  
> -	def __init__(self, root, parent=None):
> +	def __init__(self, glb, parent=None):
>  		super(TreeModel, self).__init__(parent)
> -		self.root = root
> +		self.glb = glb
> +		self.root = self.GetRoot()
>  		self.last_row_read = 0
>  
>  	def Item(self, parent):
> @@ -562,8 +563,10 @@ class CallGraphRootItem(CallGraphLevelItemBase):
>  class CallGraphModel(TreeModel):
>  
>  	def __init__(self, glb, parent=None):
> -		super(CallGraphModel, self).__init__(CallGraphRootItem(glb), parent)
> -		self.glb = glb
> +		super(CallGraphModel, self).__init__(glb, parent)
> +
> +	def GetRoot(self):
> +		return CallGraphRootItem(self.glb)
>  
>  	def columnCount(self, parent=None):
>  		return 7
> @@ -1339,8 +1342,7 @@ class BranchModel(TreeModel):
>  	progress = Signal(object)
>  
>  	def __init__(self, glb, event_id, where_clause, parent=None):
> -		super(BranchModel, self).__init__(BranchRootItem(), parent)
> -		self.glb = glb
> +		super(BranchModel, self).__init__(glb, parent)
>  		self.event_id = event_id
>  		self.more = True
>  		self.populated = 0
> @@ -1364,6 +1366,9 @@ class BranchModel(TreeModel):
>  		self.fetcher.done.connect(self.Update)
>  		self.fetcher.Fetch(glb_chunk_sz)
>  
> +	def GetRoot(self):
> +		return BranchRootItem()
> +
>  	def columnCount(self, parent=None):
>  		return 8
>  
> -- 
> 2.17.1

-- 

- Arnaldo

Re: [PATCH 7/8] perf scripts python: exported-sql-viewer.py: Factor out CallGraphModelBase

[Arnaldo Carvalho de Melo]

Em Thu, Feb 28, 2019 at 03:00:30PM +0200, Adrian Hunter escreveu:
> Factor out a base class CallGraphModelBase from CallGraphModel, so that
> CallGraphModelBase can be reused.

Thanks, applied.

- Arnaldo
 
> Signed-off-by: Adrian Hunter <adrian.hunter@intel.com>
> ---
>  .../scripts/python/exported-sql-viewer.py     | 100 ++++++++++--------
>  1 file changed, 55 insertions(+), 45 deletions(-)
> 
> diff --git a/tools/perf/scripts/python/exported-sql-viewer.py b/tools/perf/scripts/python/exported-sql-viewer.py
> index b2a22525549d..c4a2134d85f5 100755
> --- a/tools/perf/scripts/python/exported-sql-viewer.py
> +++ b/tools/perf/scripts/python/exported-sql-viewer.py
> @@ -558,26 +558,12 @@ class CallGraphRootItem(CallGraphLevelItemBase):
>  			self.child_items.append(child_item)
>  			self.child_count += 1
>  
> -# Context-sensitive call graph data model
> +# Context-sensitive call graph data model base
>  
> -class CallGraphModel(TreeModel):
> +class CallGraphModelBase(TreeModel):
>  
>  	def __init__(self, glb, parent=None):
> -		super(CallGraphModel, self).__init__(glb, parent)
> -
> -	def GetRoot(self):
> -		return CallGraphRootItem(self.glb)
> -
> -	def columnCount(self, parent=None):
> -		return 7
> -
> -	def columnHeader(self, column):
> -		headers = ["Call Path", "Object", "Count ", "Time (ns) ", "Time (%) ", "Branch Count ", "Branch Count (%) "]
> -		return headers[column]
> -
> -	def columnAlignment(self, column):
> -		alignment = [ Qt.AlignLeft, Qt.AlignLeft, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight ]
> -		return alignment[column]
> +		super(CallGraphModelBase, self).__init__(glb, parent)
>  
>  	def FindSelect(self, value, pattern, query):
>  		if pattern:
> @@ -597,34 +583,7 @@ class CallGraphModel(TreeModel):
>  				match = " GLOB '" + str(value) + "'"
>  		else:
>  			match = " = '" + str(value) + "'"
> -		QueryExec(query, "SELECT call_path_id, comm_id, thread_id"
> -						" FROM calls"
> -						" INNER JOIN call_paths ON calls.call_path_id = call_paths.id"
> -						" INNER JOIN symbols ON call_paths.symbol_id = symbols.id"
> -						" WHERE symbols.name" + match +
> -						" GROUP BY comm_id, thread_id, call_path_id"
> -						" ORDER BY comm_id, thread_id, call_path_id")
> -
> -	def FindPath(self, query):
> -		# Turn the query result into a list of ids that the tree view can walk
> -		# to open the tree at the right place.
> -		ids = []
> -		parent_id = query.value(0)
> -		while parent_id:
> -			ids.insert(0, parent_id)
> -			q2 = QSqlQuery(self.glb.db)
> -			QueryExec(q2, "SELECT parent_id"
> -					" FROM call_paths"
> -					" WHERE id = " + str(parent_id))
> -			if not q2.next():
> -				break
> -			parent_id = q2.value(0)
> -		# The call path root is not used
> -		if ids[0] == 1:
> -			del ids[0]
> -		ids.insert(0, query.value(2))
> -		ids.insert(0, query.value(1))
> -		return ids
> +		self.DoFindSelect(query, match)
>  
>  	def Found(self, query, found):
>  		if found:
> @@ -678,6 +637,57 @@ class CallGraphModel(TreeModel):
>  	def FindDone(self, thread, callback, ids):
>  		callback(ids)
>  
> +# Context-sensitive call graph data model
> +
> +class CallGraphModel(CallGraphModelBase):
> +
> +	def __init__(self, glb, parent=None):
> +		super(CallGraphModel, self).__init__(glb, parent)
> +
> +	def GetRoot(self):
> +		return CallGraphRootItem(self.glb)
> +
> +	def columnCount(self, parent=None):
> +		return 7
> +
> +	def columnHeader(self, column):
> +		headers = ["Call Path", "Object", "Count ", "Time (ns) ", "Time (%) ", "Branch Count ", "Branch Count (%) "]
> +		return headers[column]
> +
> +	def columnAlignment(self, column):
> +		alignment = [ Qt.AlignLeft, Qt.AlignLeft, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight ]
> +		return alignment[column]
> +
> +	def DoFindSelect(self, query, match):
> +		QueryExec(query, "SELECT call_path_id, comm_id, thread_id"
> +						" FROM calls"
> +						" INNER JOIN call_paths ON calls.call_path_id = call_paths.id"
> +						" INNER JOIN symbols ON call_paths.symbol_id = symbols.id"
> +						" WHERE symbols.name" + match +
> +						" GROUP BY comm_id, thread_id, call_path_id"
> +						" ORDER BY comm_id, thread_id, call_path_id")
> +
> +	def FindPath(self, query):
> +		# Turn the query result into a list of ids that the tree view can walk
> +		# to open the tree at the right place.
> +		ids = []
> +		parent_id = query.value(0)
> +		while parent_id:
> +			ids.insert(0, parent_id)
> +			q2 = QSqlQuery(self.glb.db)
> +			QueryExec(q2, "SELECT parent_id"
> +					" FROM call_paths"
> +					" WHERE id = " + str(parent_id))
> +			if not q2.next():
> +				break
> +			parent_id = q2.value(0)
> +		# The call path root is not used
> +		if ids[0] == 1:
> +			del ids[0]
> +		ids.insert(0, query.value(2))
> +		ids.insert(0, query.value(1))
> +		return ids
> +
>  # Vertical widget layout
>  
>  class VBox():
> -- 
> 2.17.1

-- 

- Arnaldo

Re: [PATCH 8/8] perf scripts python: exported-sql-viewer.py: Add call tree

[Arnaldo Carvalho de Melo]

Em Thu, Feb 28, 2019 at 03:00:31PM +0200, Adrian Hunter escreveu:
> Add a new report to display a call tree. The Call Tree report is very
> similar to the Context-Sensitive Call Graph, but the data is not
> aggregated. Also the 'Count' column, which would be always 1, is replaced
> by the 'Call Time'.

Thanks, applied and added committer testing notes:


    Committer testing:
    
      $ cat simple-retpoline.c
      /*
    
        https://lkml.kernel.org/r/20190109091835.5570-6-adrian.hunter@intel.com
    
      $ gcc -ggdb3 -Wall -Wextra -O2 -o simple-retpoline simple-retpoline.c
      $ objdump -d simple-retpoline
      */
    
      __attribute__((noinline)) int bar(void)
      {
              return -1;
      }
    
      int foo(void)
      {
              return bar() + 1;
      }
    
      __attribute__((indirect_branch("thunk"))) int main()
      {
              int (*volatile fn)(void) = foo;
    
              fn();
              return fn();
      }
      $
      $ perf record -o simple-retpoline.perf.data -e intel_pt/cyc/u ./simple-retpoline
      $ perf script -i simple-retpoline.perf.data --itrace=be -s ~acme/libexec/perf-core/scripts/python/export-to-sqlite.py simple-retpoline.db branches calls
      $ python ~acme/libexec/perf-core/scripts/python/exported-sql-viewer.py simple-retpoline.db
    
    And in the GUI select:
    
        "Reports"
          "Call Tree"
    
        Call Path                 | Object          | Call Time (ns) | Time (ns) | Time (%) | Branch Count | Brach Count (%) |
        > simple-retpolin
          > PID:TID
            > _start                ld-2.28.so       2193855505777      156267      100.0       10602          100.0
                unknown             unknown          2193855506010        2276        1.5           1            0.0
              > _dl_start           ld-2.28.so       2193855508286      137047       87.7       10088           95.2
              > _dl_init            ld-2.28.so       2193855645444        9142        5.9         326            3.1
              > _start              simple-retpoline 2193855654587        7457        4.8         182            1.7
                > __libc_start_main <SNIP>
                  <SNIP>
                  > main            simple-retpoline 2193855657493          32        0.5          12            6.7
                    > foo           simple-retpoline 2193855657493          14       43.8           5           41.7
                  <SNIP>
 
> Signed-off-by: Adrian Hunter <adrian.hunter@intel.com>
> ---
>  .../scripts/python/exported-sql-viewer.py     | 195 +++++++++++++++++-
>  1 file changed, 186 insertions(+), 9 deletions(-)
> 
> diff --git a/tools/perf/scripts/python/exported-sql-viewer.py b/tools/perf/scripts/python/exported-sql-viewer.py
> index c4a2134d85f5..afec9479ca7f 100755
> --- a/tools/perf/scripts/python/exported-sql-viewer.py
> +++ b/tools/perf/scripts/python/exported-sql-viewer.py
> @@ -688,6 +688,150 @@ class CallGraphModel(CallGraphModelBase):
>  		ids.insert(0, query.value(1))
>  		return ids
>  
> +# Call tree data model level 2+ item base
> +
> +class CallTreeLevelTwoPlusItemBase(CallGraphLevelItemBase):
> +
> +	def __init__(self, glb, row, comm_id, thread_id, calls_id, time, branch_count, parent_item):
> +		super(CallTreeLevelTwoPlusItemBase, self).__init__(glb, row, parent_item)
> +		self.comm_id = comm_id
> +		self.thread_id = thread_id
> +		self.calls_id = calls_id
> +		self.branch_count = branch_count
> +		self.time = time
> +
> +	def Select(self):
> +		self.query_done = True;
> +		if self.calls_id == 0:
> +			comm_thread = " AND comm_id = " + str(self.comm_id) + " AND thread_id = " + str(self.thread_id)
> +		else:
> +			comm_thread = ""
> +		query = QSqlQuery(self.glb.db)
> +		QueryExec(query, "SELECT calls.id, name, short_name, call_time, return_time - call_time, branch_count"
> +					" FROM calls"
> +					" INNER JOIN call_paths ON calls.call_path_id = call_paths.id"
> +					" INNER JOIN symbols ON call_paths.symbol_id = symbols.id"
> +					" INNER JOIN dsos ON symbols.dso_id = dsos.id"
> +					" WHERE calls.parent_id = " + str(self.calls_id) + comm_thread +
> +					" ORDER BY call_time, calls.id")
> +		while query.next():
> +			child_item = CallTreeLevelThreeItem(self.glb, self.child_count, self.comm_id, self.thread_id, query.value(0), query.value(1), query.value(2), query.value(3), int(query.value(4)), int(query.value(5)), self)
> +			self.child_items.append(child_item)
> +			self.child_count += 1
> +
> +# Call tree data model level three item
> +
> +class CallTreeLevelThreeItem(CallTreeLevelTwoPlusItemBase):
> +
> +	def __init__(self, glb, row, comm_id, thread_id, calls_id, name, dso, count, time, branch_count, parent_item):
> +		super(CallTreeLevelThreeItem, self).__init__(glb, row, comm_id, thread_id, calls_id, time, branch_count, parent_item)
> +		dso = dsoname(dso)
> +		self.data = [ name, dso, str(count), str(time), PercentToOneDP(time, parent_item.time), str(branch_count), PercentToOneDP(branch_count, parent_item.branch_count) ]
> +		self.dbid = calls_id
> +
> +# Call tree data model level two item
> +
> +class CallTreeLevelTwoItem(CallTreeLevelTwoPlusItemBase):
> +
> +	def __init__(self, glb, row, comm_id, thread_id, pid, tid, parent_item):
> +		super(CallTreeLevelTwoItem, self).__init__(glb, row, comm_id, thread_id, 0, 0, 0, parent_item)
> +		self.data = [str(pid) + ":" + str(tid), "", "", "", "", "", ""]
> +		self.dbid = thread_id
> +
> +	def Select(self):
> +		super(CallTreeLevelTwoItem, self).Select()
> +		for child_item in self.child_items:
> +			self.time += child_item.time
> +			self.branch_count += child_item.branch_count
> +		for child_item in self.child_items:
> +			child_item.data[4] = PercentToOneDP(child_item.time, self.time)
> +			child_item.data[6] = PercentToOneDP(child_item.branch_count, self.branch_count)
> +
> +# Call tree data model level one item
> +
> +class CallTreeLevelOneItem(CallGraphLevelItemBase):
> +
> +	def __init__(self, glb, row, comm_id, comm, parent_item):
> +		super(CallTreeLevelOneItem, self).__init__(glb, row, parent_item)
> +		self.data = [comm, "", "", "", "", "", ""]
> +		self.dbid = comm_id
> +
> +	def Select(self):
> +		self.query_done = True;
> +		query = QSqlQuery(self.glb.db)
> +		QueryExec(query, "SELECT thread_id, pid, tid"
> +					" FROM comm_threads"
> +					" INNER JOIN threads ON thread_id = threads.id"
> +					" WHERE comm_id = " + str(self.dbid))
> +		while query.next():
> +			child_item = CallTreeLevelTwoItem(self.glb, self.child_count, self.dbid, query.value(0), query.value(1), query.value(2), self)
> +			self.child_items.append(child_item)
> +			self.child_count += 1
> +
> +# Call tree data model root item
> +
> +class CallTreeRootItem(CallGraphLevelItemBase):
> +
> +	def __init__(self, glb):
> +		super(CallTreeRootItem, self).__init__(glb, 0, None)
> +		self.dbid = 0
> +		self.query_done = True;
> +		query = QSqlQuery(glb.db)
> +		QueryExec(query, "SELECT id, comm FROM comms")
> +		while query.next():
> +			if not query.value(0):
> +				continue
> +			child_item = CallTreeLevelOneItem(glb, self.child_count, query.value(0), query.value(1), self)
> +			self.child_items.append(child_item)
> +			self.child_count += 1
> +
> +# Call Tree data model
> +
> +class CallTreeModel(CallGraphModelBase):
> +
> +	def __init__(self, glb, parent=None):
> +		super(CallTreeModel, self).__init__(glb, parent)
> +
> +	def GetRoot(self):
> +		return CallTreeRootItem(self.glb)
> +
> +	def columnCount(self, parent=None):
> +		return 7
> +
> +	def columnHeader(self, column):
> +		headers = ["Call Path", "Object", "Call Time", "Time (ns) ", "Time (%) ", "Branch Count ", "Branch Count (%) "]
> +		return headers[column]
> +
> +	def columnAlignment(self, column):
> +		alignment = [ Qt.AlignLeft, Qt.AlignLeft, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight ]
> +		return alignment[column]
> +
> +	def DoFindSelect(self, query, match):
> +		QueryExec(query, "SELECT calls.id, comm_id, thread_id"
> +						" FROM calls"
> +						" INNER JOIN call_paths ON calls.call_path_id = call_paths.id"
> +						" INNER JOIN symbols ON call_paths.symbol_id = symbols.id"
> +						" WHERE symbols.name" + match +
> +						" ORDER BY comm_id, thread_id, call_time, calls.id")
> +
> +	def FindPath(self, query):
> +		# Turn the query result into a list of ids that the tree view can walk
> +		# to open the tree at the right place.
> +		ids = []
> +		parent_id = query.value(0)
> +		while parent_id:
> +			ids.insert(0, parent_id)
> +			q2 = QSqlQuery(self.glb.db)
> +			QueryExec(q2, "SELECT parent_id"
> +					" FROM calls"
> +					" WHERE id = " + str(parent_id))
> +			if not q2.next():
> +				break
> +			parent_id = q2.value(0)
> +		ids.insert(0, query.value(2))
> +		ids.insert(0, query.value(1))
> +		return ids
> +
>  # Vertical widget layout
>  
>  class VBox():
> @@ -772,6 +916,29 @@ class CallGraphWindow(TreeWindowBase):
>  
>  		AddSubWindow(glb.mainwindow.mdi_area, self, "Context-Sensitive Call Graph")
>  
> +# Call tree window
> +
> +class CallTreeWindow(TreeWindowBase):
> +
> +	def __init__(self, glb, parent=None):
> +		super(CallTreeWindow, self).__init__(parent)
> +
> +		self.model = LookupCreateModel("Call Tree", lambda x=glb: CallTreeModel(x))
> +
> +		self.view = QTreeView()
> +		self.view.setModel(self.model)
> +
> +		for c, w in ((0, 230), (1, 100), (2, 100), (3, 70), (4, 70), (5, 100)):
> +			self.view.setColumnWidth(c, w)
> +
> +		self.find_bar = FindBar(self, self)
> +
> +		self.vbox = VBox(self.view, self.find_bar.Widget())
> +
> +		self.setWidget(self.vbox.Widget())
> +
> +		AddSubWindow(glb.mainwindow.mdi_area, self, "Call Tree")
> +
>  # Child data item  finder
>  
>  class ChildDataItemFinder():
> @@ -1890,10 +2057,10 @@ def GetEventList(db):
>  
>  # Is a table selectable
>  
> -def IsSelectable(db, table):
> +def IsSelectable(db, table, sql = ""):
>  	query = QSqlQuery(db)
>  	try:
> -		QueryExec(query, "SELECT * FROM " + table + " LIMIT 1")
> +		QueryExec(query, "SELECT * FROM " + table + " " + sql + " LIMIT 1")
>  	except:
>  		return False
>  	return True
> @@ -2302,9 +2469,10 @@ p.c2 {
>  </style>
>  <p class=c1><a href=#reports>1. Reports</a></p>
>  <p class=c2><a href=#callgraph>1.1 Context-Sensitive Call Graph</a></p>
> -<p class=c2><a href=#allbranches>1.2 All branches</a></p>
> -<p class=c2><a href=#selectedbranches>1.3 Selected branches</a></p>
> -<p class=c2><a href=#topcallsbyelapsedtime>1.4 Top calls by elapsed time</a></p>
> +<p class=c2><a href=#calltree>1.2 Call Tree</a></p>
> +<p class=c2><a href=#allbranches>1.3 All branches</a></p>
> +<p class=c2><a href=#selectedbranches>1.4 Selected branches</a></p>
> +<p class=c2><a href=#topcallsbyelapsedtime>1.5 Top calls by elapsed time</a></p>
>  <p class=c1><a href=#tables>2. Tables</a></p>
>  <h1 id=reports>1. Reports</h1>
>  <h2 id=callgraph>1.1 Context-Sensitive Call Graph</h2>
> @@ -2340,7 +2508,10 @@ v- ls
>  <h3>Find</h3>
>  Ctrl-F displays a Find bar which finds function names by either an exact match or a pattern match.
>  The pattern matching symbols are ? for any character and * for zero or more characters.
> -<h2 id=allbranches>1.2 All branches</h2>
> +<h2 id=calltree>1.2 Call Tree</h2>
> +The Call Tree report is very similar to the Context-Sensitive Call Graph, but the data is not aggregated.
> +Also the 'Count' column, which would be always 1, is replaced by the 'Call Time'.
> +<h2 id=allbranches>1.3 All branches</h2>
>  The All branches report displays all branches in chronological order.
>  Not all data is fetched immediately. More records can be fetched using the Fetch bar provided.
>  <h3>Disassembly</h3>
> @@ -2366,10 +2537,10 @@ sudo ldconfig
>  Ctrl-F displays a Find bar which finds substrings by either an exact match or a regular expression match.
>  Refer to Python documentation for the regular expression syntax.
>  All columns are searched, but only currently fetched rows are searched.
> -<h2 id=selectedbranches>1.3 Selected branches</h2>
> +<h2 id=selectedbranches>1.4 Selected branches</h2>
>  This is the same as the <a href=#allbranches>All branches</a> report but with the data reduced
>  by various selection criteria. A dialog box displays available criteria which are AND'ed together.
> -<h3>1.3.1 Time ranges</h3>
> +<h3>1.4.1 Time ranges</h3>
>  The time ranges hint text shows the total time range. Relative time ranges can also be entered in
>  ms, us or ns. Also, negative values are relative to the end of trace.  Examples:
>  <pre>
> @@ -2380,7 +2551,7 @@ ms, us or ns. Also, negative values are relative to the end of trace.  Examples:
>  	-10ms-			The last 10ms
>  </pre>
>  N.B. Due to the granularity of timestamps, there could be no branches in any given time range.
> -<h2 id=topcallsbyelapsedtime>1.4 Top calls by elapsed time</h2>
> +<h2 id=topcallsbyelapsedtime>1.5 Top calls by elapsed time</h2>
>  The Top calls by elapsed time report displays calls in descending order of time elapsed between when the function was called and when it returned.
>  The data is reduced by various selection criteria. A dialog box displays available criteria which are AND'ed together.
>  If not all data is fetched, a Fetch bar is provided. Ctrl-F displays a Find bar.
> @@ -2516,6 +2687,9 @@ class MainWindow(QMainWindow):
>  		if IsSelectable(glb.db, "calls"):
>  			reports_menu.addAction(CreateAction("Context-Sensitive Call &Graph", "Create a new window containing a context-sensitive call graph", self.NewCallGraph, self))
>  
> +		if IsSelectable(glb.db, "calls", "WHERE parent_id >= 0"):
> +			reports_menu.addAction(CreateAction("Call &Tree", "Create a new window containing a call tree", self.NewCallTree, self))
> +
>  		self.EventMenu(GetEventList(glb.db), reports_menu)
>  
>  		if IsSelectable(glb.db, "calls"):
> @@ -2576,6 +2750,9 @@ class MainWindow(QMainWindow):
>  	def NewCallGraph(self):
>  		CallGraphWindow(self.glb, self)
>  
> +	def NewCallTree(self):
> +		CallTreeWindow(self.glb, self)
> +
>  	def NewTopCalls(self):
>  		dialog = TopCallsDialog(self.glb, self)
>  		ret = dialog.exec_()
> -- 
> 2.17.1

-- 

- Arnaldo