[PATCH] x86/tracing: introduce enter/exit tracepoint pairs for page faults

[Junxuan Liao <ljx@cs.wisc.edu>]

Rename page_fault_{user,kernel} to page_fault_{user,kernel}_enter, and
add the exit counterparts. This might be useful for measuring page fault
handling latencies.

The exit tracepoints use a separate static key.

Signed-off-by: Junxuan Liao <ljx@cs.wisc.edu>
Link: https://lore.kernel.org/9e2ac1e3-d07d-4f17-970e-6b7a5248a5bb@cs.wisc.edu
---
 arch/x86/include/asm/trace/common.h     | 18 ++++++++++----
 arch/x86/include/asm/trace/exceptions.h | 32 ++++++++++++++++++++-----
 arch/x86/kernel/tracepoint.c            | 22 +++++++++++++----
 arch/x86/mm/fault.c                     | 21 +++++++++++++---
 4 files changed, 75 insertions(+), 18 deletions(-)

diff --git a/arch/x86/include/asm/trace/common.h b/arch/x86/include/asm/trace/common.h
index f0f9bcdb74d9..4b289be75173 100644
--- a/arch/x86/include/asm/trace/common.h
+++ b/arch/x86/include/asm/trace/common.h
@@ -2,11 +2,21 @@
 #define _ASM_TRACE_COMMON_H
 
 #ifdef CONFIG_TRACING
-DECLARE_STATIC_KEY_FALSE(trace_pagefault_key);
-#define trace_pagefault_enabled()			\
-	static_branch_unlikely(&trace_pagefault_key)
+DECLARE_STATIC_KEY_FALSE(trace_pagefault_enter_key);
+DECLARE_STATIC_KEY_FALSE(trace_pagefault_exit_key);
+#define trace_pagefault_enter_enabled() \
+	static_branch_unlikely(&trace_pagefault_enter_key)
+#define trace_pagefault_exit_enabled() \
+	static_branch_unlikely(&trace_pagefault_exit_key)
 #else
-static inline bool trace_pagefault_enabled(void) { return false; }
+static inline bool trace_pagefault_enter_enabled(void)
+{
+	return false;
+}
+static inline bool trace_pagefault_exit_enabled(void)
+{
+	return false;
+}
 #endif
 
 #endif
diff --git a/arch/x86/include/asm/trace/exceptions.h b/arch/x86/include/asm/trace/exceptions.h
index 6b1e87194809..76729fcdb249 100644
--- a/arch/x86/include/asm/trace/exceptions.h
+++ b/arch/x86/include/asm/trace/exceptions.h
@@ -8,8 +8,10 @@
 #include <linux/tracepoint.h>
 #include <asm/trace/common.h>
 
-extern int trace_pagefault_reg(void);
-extern void trace_pagefault_unreg(void);
+extern int trace_pagefault_enter_reg(void);
+extern void trace_pagefault_enter_unreg(void);
+extern int trace_pagefault_exit_reg(void);
+extern void trace_pagefault_exit_unreg(void);
 
 DECLARE_EVENT_CLASS(x86_exceptions,
 
@@ -34,15 +36,33 @@ DECLARE_EVENT_CLASS(x86_exceptions,
 		  (void *)__entry->address, (void *)__entry->ip,
 		  __entry->error_code) );
 
-#define DEFINE_PAGE_FAULT_EVENT(name)				\
+#define DEFINE_PAGE_FAULT_EVENT(name, reg, unreg)		\
 DEFINE_EVENT_FN(x86_exceptions, name,				\
 	TP_PROTO(unsigned long address,	struct pt_regs *regs,	\
 		 unsigned long error_code),			\
 	TP_ARGS(address, regs, error_code),			\
-	trace_pagefault_reg, trace_pagefault_unreg);
+	reg, unreg)
 
-DEFINE_PAGE_FAULT_EVENT(page_fault_user);
-DEFINE_PAGE_FAULT_EVENT(page_fault_kernel);
+DEFINE_PAGE_FAULT_EVENT(
+	page_fault_user_enter,
+	trace_pagefault_enter_reg,
+	trace_pagefault_enter_unreg
+);
+DEFINE_PAGE_FAULT_EVENT(
+	page_fault_user_exit,
+	trace_pagefault_exit_reg,
+	trace_pagefault_exit_unreg
+);
+DEFINE_PAGE_FAULT_EVENT(
+	page_fault_kernel_enter,
+	trace_pagefault_enter_reg,
+	trace_pagefault_enter_unreg
+);
+DEFINE_PAGE_FAULT_EVENT(
+	page_fault_kernel_exit,
+	trace_pagefault_exit_reg,
+	trace_pagefault_exit_unreg
+);
 
 #undef TRACE_INCLUDE_PATH
 #undef TRACE_INCLUDE_FILE
diff --git a/arch/x86/kernel/tracepoint.c b/arch/x86/kernel/tracepoint.c
index 03ae1caaa878..108285046226 100644
--- a/arch/x86/kernel/tracepoint.c
+++ b/arch/x86/kernel/tracepoint.c
@@ -7,15 +7,27 @@
 
 #include <asm/trace/exceptions.h>
 
-DEFINE_STATIC_KEY_FALSE(trace_pagefault_key);
+DEFINE_STATIC_KEY_FALSE(trace_pagefault_enter_key);
+DEFINE_STATIC_KEY_FALSE(trace_pagefault_exit_key);
 
-int trace_pagefault_reg(void)
+int trace_pagefault_enter_reg(void)
 {
-	static_branch_inc(&trace_pagefault_key);
+	static_branch_inc(&trace_pagefault_enter_key);
 	return 0;
 }
 
-void trace_pagefault_unreg(void)
+void trace_pagefault_enter_unreg(void)
 {
-	static_branch_dec(&trace_pagefault_key);
+	static_branch_dec(&trace_pagefault_enter_key);
+}
+
+int trace_pagefault_exit_reg(void)
+{
+	static_branch_inc(&trace_pagefault_exit_key);
+	return 0;
+}
+
+void trace_pagefault_exit_unreg(void)
+{
+	static_branch_dec(&trace_pagefault_exit_key);
 }
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 296d294142c8..cd933edf1e19 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -1455,13 +1455,26 @@ static __always_inline void
 trace_page_fault_entries(struct pt_regs *regs, unsigned long error_code,
 			 unsigned long address)
 {
-	if (!trace_pagefault_enabled())
+	if (!trace_pagefault_enter_enabled())
 		return;
 
 	if (user_mode(regs))
-		trace_page_fault_user(address, regs, error_code);
+		trace_page_fault_user_enter(address, regs, error_code);
 	else
-		trace_page_fault_kernel(address, regs, error_code);
+		trace_page_fault_kernel_enter(address, regs, error_code);
+}
+
+static __always_inline void
+trace_page_fault_exits(struct pt_regs *regs, unsigned long error_code,
+			 unsigned long address)
+{
+	if (!trace_pagefault_exit_enabled())
+		return;
+
+	if (user_mode(regs))
+		trace_page_fault_user_exit(address, regs, error_code);
+	else
+		trace_page_fault_kernel_exit(address, regs, error_code);
 }
 
 static __always_inline void
@@ -1487,6 +1500,8 @@ handle_page_fault(struct pt_regs *regs, unsigned long error_code,
 		 */
 		local_irq_disable();
 	}
+
+	trace_page_fault_exits(regs, error_code, address);
 }
 
 DEFINE_IDTENTRY_RAW_ERRORCODE(exc_page_fault)
-- 
2.48.1