[PATCH v12 19/19] fgraph: Skip push operation if no retfunc is registered

["Masami Hiramatsu (Google)" <mhiramat@kernel.org>]

From: Masami Hiramatsu (Google) <mhiramat@kernel.org>

Skip push operation only when there is no fgraph_ops which sets retfunc.

This is for optimizing performance of fprobe on fgraph. Since the major
use case of fprobe is putting a probe on function entry and another
probe on exit. Since these probes are independent, if user only uses
fprobe on function entry, we don't need to push a frame information on
shadow stack.

Here is the performance improvement results;

Without this:
kprobe-multi   :    6.265 ± 0.033M/s
kretprobe-multi:    4.758 ± 0.009M/s

With this:
kprobe-multi   :    6.377 ± 0.054M/s	+1.79%
kretprobe-multi:    4.815 ± 0.007M/s	+1.20%

Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
---
 include/linux/ftrace.h |    1 +
 kernel/trace/fgraph.c  |   33 +++++++++++++++++++++++++--------
 kernel/trace/fprobe.c  |   25 ++++++++++++++++++++++++-
 3 files changed, 50 insertions(+), 9 deletions(-)

diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index fabf1a0979d4..d08e5e6e725f 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -1220,6 +1220,7 @@ unsigned long *fgraph_get_task_var(struct fgraph_ops *gops);
 #define FTRACE_RETFUNC_DEPTH 50
 #define FTRACE_RETSTACK_ALLOC_SIZE 32
 
+void ftrace_graph_update_flags(void);
 extern int register_ftrace_graph(struct fgraph_ops *ops);
 extern void unregister_ftrace_graph(struct fgraph_ops *ops);
 
diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c
index cf3ae59a436e..3a23d4e5738c 100644
--- a/kernel/trace/fgraph.c
+++ b/kernel/trace/fgraph.c
@@ -175,6 +175,7 @@ int ftrace_graph_active;
 static struct fgraph_ops *fgraph_array[FGRAPH_ARRAY_SIZE];
 static unsigned long fgraph_array_bitmask;
 static bool fgraph_skip_timestamp;
+static bool fgraph_skip_all;
 
 /* LRU index table for fgraph_array */
 static int fgraph_lru_table[FGRAPH_ARRAY_SIZE];
@@ -349,6 +350,9 @@ void *fgraph_reserve_data(int idx, int size_bytes)
 	int curr_ret_stack = current->curr_ret_stack;
 	int data_size;
 
+	if (unlikely(fgraph_skip_all))
+		return NULL;
+
 	if (size_bytes > FGRAPH_MAX_DATA_SIZE)
 		return NULL;
 
@@ -632,9 +636,11 @@ int function_graph_enter_regs(unsigned long ret, unsigned long func,
 	trace.func = func;
 	trace.depth = ++current->curr_ret_depth;
 
-	offset = ftrace_push_return_trace(ret, func, frame_pointer, retp, 0);
-	if (offset < 0)
-		goto out;
+	if (likely(!fgraph_skip_all)) {
+		offset = ftrace_push_return_trace(ret, func, frame_pointer, retp, 0);
+		if (offset < 0)
+			goto out;
+	}
 
 #ifdef CONFIG_HAVE_STATIC_CALL
 	if (static_branch_likely(&fgraph_do_direct)) {
@@ -665,6 +671,8 @@ int function_graph_enter_regs(unsigned long ret, unsigned long func,
 				current->curr_ret_stack = save_curr_ret_stack;
 		}
 	}
+	if (unlikely(fgraph_skip_all))
+		goto out;
 
 	if (!bitmap)
 		goto out_ret;
@@ -1254,6 +1262,7 @@ static void ftrace_graph_disable_direct(bool disable_branch)
 
 static void update_fgraph_skip_timestamp(void)
 {
+	bool skip_all = true, skip_ts = true;
 	int i;
 
 	for (i = 0; i < FGRAPH_ARRAY_SIZE; i++) {
@@ -1262,12 +1271,20 @@ static void update_fgraph_skip_timestamp(void)
 		if (gops == &fgraph_stub)
 			continue;
 
-		if (!gops->skip_timestamp) {
-			fgraph_skip_timestamp = false;
-			return;
-		}
+		if (!gops->skip_timestamp)
+			skip_ts = false;
+		if (gops->retfunc)
+			skip_all = false;
 	}
-	fgraph_skip_timestamp = true;
+	fgraph_skip_timestamp = skip_ts;
+	fgraph_skip_all = skip_all;
+}
+
+void ftrace_graph_update_flags(void)
+{
+	mutex_lock(&ftrace_lock);
+	update_fgraph_skip_timestamp();
+	mutex_unlock(&ftrace_lock);
 }
 
 int register_ftrace_graph(struct fgraph_ops *gops)
diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c
index b108d26d7ee5..188a38ac3153 100644
--- a/kernel/trace/fprobe.c
+++ b/kernel/trace/fprobe.c
@@ -42,6 +42,9 @@ static struct hlist_head fprobe_table[FPROBE_TABLE_SIZE];
 static struct hlist_head fprobe_ip_table[FPROBE_IP_TABLE_SIZE];
 static DEFINE_MUTEX(fprobe_mutex);
 
+/* Count the number of fprobe which has the exit_handler. */
+static int fprobe_nr_exit_handlers;
+
 /*
  * Find first fprobe in the hlist. It will be iterated twice in the entry
  * probe, once for correcting the total required size, the second time is
@@ -344,11 +347,18 @@ NOKPROBE_SYMBOL(fprobe_return);
 
 static struct fgraph_ops fprobe_graph_ops = {
 	.entryfunc	= fprobe_entry,
-	.retfunc	= fprobe_return,
+	/* retfunc is set only if any fprobe.exit_handler is set. */
 	.skip_timestamp = true,
 };
 static int fprobe_graph_active;
 
+static void fprobe_graph_switch_retfunc(bool enable)
+{
+	fprobe_graph_ops.retfunc = enable ? fprobe_return : NULL;
+	if (fprobe_graph_active)
+		ftrace_graph_update_flags();
+}
+
 /* Add @addrs to the ftrace filter and register fgraph if needed. */
 static int fprobe_graph_add_ips(unsigned long *addrs, int num)
 {
@@ -480,6 +490,8 @@ static int fprobe_init(struct fprobe *fp, unsigned long *addrs, int num)
 	size = ALIGN(fp->entry_data_size, sizeof(long));
 	if (size > MAX_FPROBE_DATA_SIZE)
 		return -E2BIG;
+	if (!fp->exit_handler && size)
+		return -EINVAL;
 	fp->entry_data_size = size;
 
 	hlist_array = kzalloc(struct_size(hlist_array, array, num), GFP_KERNEL);
@@ -564,6 +576,11 @@ int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num)
 
 	mutex_lock(&fprobe_mutex);
 
+	if (fp->exit_handler) {
+		fprobe_nr_exit_handlers++;
+		if (fprobe_nr_exit_handlers == 1)
+			fprobe_graph_switch_retfunc(true);
+	}
 	hlist_array = fp->hlist_array;
 	ret = fprobe_graph_add_ips(addrs, num);
 	if (!ret) {
@@ -653,6 +670,12 @@ int unregister_fprobe(struct fprobe *fp)
 	}
 	del_fprobe_hash(fp);
 
+	if (fp->exit_handler) {
+		fprobe_nr_exit_handlers--;
+		if (!fprobe_nr_exit_handlers)
+			fprobe_graph_switch_retfunc(false);
+	}
+
 	if (count)
 		fprobe_graph_remove_ips(addrs, count);