From: Peter Zijlstra <peterz@infradead.org>
To: tglx@linutronix.de, bp@alien8.de, mingo@kernel.org,
rostedt@goodmis.org, luto@kernel.org,
torvalds@linux-foundation.org
Cc: hpa@zytor.com, dave.hansen@linux.intel.com, jgross@suse.com,
linux-kernel@vger.kernel.org, zhe.he@windriver.com,
joel@joelfernandes.org, devel@etsukata.com, peterz@infradead.org
Subject: [RFC][PATCH v2 7/7] x86/entry/64: Pull bits into C
Date: Thu, 04 Jul 2019 21:56:02 +0200 [thread overview]
Message-ID: <20190704200050.648944551@infradead.org> (raw)
In-Reply-To: 20190704195555.580363209@infradead.org
TODO: 32bit, Xen
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
arch/x86/entry/entry_64.S | 13 +++-------
arch/x86/include/asm/idtentry.h | 49 ++++++++++++++++++++++++++++++++++++++++
arch/x86/include/asm/traps.h | 1
arch/x86/kernel/cpu/mce/core.c | 2 -
arch/x86/kernel/kvm.c | 4 ---
arch/x86/kernel/traps.c | 30 ++++++++----------------
arch/x86/mm/fault.c | 4 ---
7 files changed, 68 insertions(+), 35 deletions(-)
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -841,15 +841,6 @@ apicinterrupt IRQ_WORK_VECTOR irq_work
GET_CR2_INTO(%rdx); /* can clobber %rax */
.endif
- TRACE_IRQS_OFF
-
- .if \paranoid == 0
- testb $3, CS(%rsp)
- jz .Lfrom_kernel_no_context_tracking_\@
- CALL_enter_from_user_mode
-.Lfrom_kernel_no_context_tracking_\@:
- .endif
-
movq %rsp, %rdi /* pt_regs pointer */
.if \has_error_code
@@ -863,7 +854,11 @@ apicinterrupt IRQ_WORK_VECTOR irq_work
subq $\ist_offset, CPU_TSS_IST(\shift_ist)
.endif
+ .if \paranoid
+ call \do_sym\()_paranoid
+ .else
call \do_sym
+ .endif
.if \shift_ist != -1
addq $\ist_offset, CPU_TSS_IST(\shift_ist)
--- /dev/null
+++ b/arch/x86/include/asm/idtentry.h
@@ -0,0 +1,49 @@
+#ifndef __ASM_IDTENTRY_H
+#define __ASM_IDTENTRY_H
+
+/* shamelessly stolen from linux/syscalls.h; XXX share */
+
+#define __IDT_MAP0(m,...)
+#define __IDT_MAP1(m,t,a,...) m(t,a)
+#define __IDT_MAP2(m,t,a,...) m(t,a), __IDT_MAP1(m,__VA_ARGS__)
+#define __IDT_MAP3(m,t,a,...) m(t,a), __IDT_MAP2(m,__VA_ARGS__)
+
+#define __IDT_MAP(n,...) __IDT_MAP##n(__VA_ARGS__)
+
+#define __IDT_DECL(t, a) t a
+#define __IDT_ARGS(t, a) a
+#define __IDT_TEST(t, a) (void)BUILD_BUG_ON_ZERO(sizeof(t) != sizeof(long))
+
+#ifdef CONFIG_CONTEXT_TRACKING
+#define CALL_enter_from_user_mode(_regs) \
+ if (static_branch_unlikely(&context_tracking_enabled) && user_mode(_regs)) \
+ enter_from_user_mode()
+#else
+#define CALL_enter_from_user_mode(_regs)
+#endif
+
+#define IDTENTRYx(n, name, ...) \
+ static notrace void __idt_##name(__IDT_MAP(n, __IDT_DECL, __VA_ARGS__)); \
+ NOKPROBE_SYMBOL(__idt_##name); \
+ dotraplinkage notrace void name(__IDT_MAP(n, __IDT_DECL, __VA_ARGS__)) \
+ { \
+ __IDT_MAP(n, __IDT_TEST, __VA_ARGS__); \
+ trace_hardirqs_off(); \
+ CALL_enter_from_user_mode(regs); \
+ __idt_##name(__IDT_MAP(n, __IDT_ARGS, __VA_ARGS__)); \
+ } \
+ NOKPROBE_SYMBOL(name); \
+ dotraplinkage notrace void name##_paranoid(__IDT_MAP(n, __IDT_DECL, __VA_ARGS__)) \
+ { \
+ __IDT_MAP(n, __IDT_TEST, __VA_ARGS__); \
+ trace_hardirqs_off(); \
+ __idt_##name(__IDT_MAP(n, __IDT_ARGS, __VA_ARGS__)); \
+ } \
+ NOKPROBE_SYMBOL(name##_paranoid); \
+ static notrace void __idt_##name(__IDT_MAP(n, __IDT_DECL, __VA_ARGS__))
+
+#define IDTENTRY1(name,...) IDTENTRYx(1, name, __VA_ARGS__)
+#define IDTENTRY2(name,...) IDTENTRYx(2, name, __VA_ARGS__)
+#define IDTENTRY3(name,...) IDTENTRYx(3, name, __VA_ARGS__)
+
+#endif /* __ASM_IDTENTRY_H */
--- a/arch/x86/include/asm/traps.h
+++ b/arch/x86/include/asm/traps.h
@@ -7,6 +7,7 @@
#include <asm/debugreg.h>
#include <asm/siginfo.h> /* TRAP_TRACE, ... */
+#include <asm/idtentry.h>
#define dotraplinkage __visible
--- a/arch/x86/kernel/cpu/mce/core.c
+++ b/arch/x86/kernel/cpu/mce/core.c
@@ -1842,7 +1842,7 @@ static void unexpected_machine_check(str
void (*machine_check_vector)(struct pt_regs *, long error_code) =
unexpected_machine_check;
-dotraplinkage void do_mce(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_mce, struct pt_regs *, regs, long, error_code)
{
machine_check_vector(regs, error_code);
}
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -241,8 +241,7 @@ u32 kvm_read_and_reset_pf_reason(void)
EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason);
NOKPROBE_SYMBOL(kvm_read_and_reset_pf_reason);
-dotraplinkage void
-do_async_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address)
+IDTENTRY3(do_async_page_fault, struct pt_regs *, regs, unsigned long, error_code, unsigned long, address)
{
enum ctx_state prev_state;
@@ -263,7 +262,6 @@ do_async_page_fault(struct pt_regs *regs
break;
}
}
-NOKPROBE_SYMBOL(do_async_page_fault);
static void __init paravirt_ops_setup(void)
{
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -281,7 +281,7 @@ static void do_error_trap(struct pt_regs
#define IP ((void __user *)uprobe_get_trap_addr(regs))
#define DO_ERROR(trapnr, signr, sicode, addr, str, name) \
-dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
+IDTENTRY2(do_##name, struct pt_regs *, regs, long, error_code) \
{ \
do_error_trap(regs, error_code, str, trapnr, signr, sicode, addr); \
}
@@ -313,7 +313,7 @@ __visible void __noreturn handle_stack_o
#ifdef CONFIG_X86_64
/* Runs on IST stack */
-dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsigned long cr2)
+IDTENTRY3(do_double_fault, struct pt_regs *, regs, long, error_code, unsigned long, cr2)
{
static const char str[] = "double fault";
struct task_struct *tsk = current;
@@ -428,7 +428,7 @@ dotraplinkage void do_double_fault(struc
}
#endif
-dotraplinkage void do_bounds(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_bounds, struct pt_regs *, regs, long, error_code)
{
const struct mpx_bndcsr *bndcsr;
@@ -514,8 +514,7 @@ dotraplinkage void do_bounds(struct pt_r
do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, error_code, 0, NULL);
}
-dotraplinkage void
-do_general_protection(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_general_protection, struct pt_regs *, regs, long, error_code)
{
const char *desc = "general protection fault";
struct task_struct *tsk;
@@ -564,9 +563,8 @@ do_general_protection(struct pt_regs *re
force_sig(SIGSEGV, tsk);
}
-NOKPROBE_SYMBOL(do_general_protection);
-dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_int3, struct pt_regs *, regs, long, error_code)
{
#ifdef CONFIG_DYNAMIC_FTRACE
/*
@@ -611,7 +609,6 @@ dotraplinkage void notrace do_int3(struc
exit:
ist_exit(regs);
}
-NOKPROBE_SYMBOL(do_int3);
#ifdef CONFIG_X86_64
/*
@@ -706,7 +703,7 @@ static bool is_sysenter_singlestep(struc
*
* May run on IST stack.
*/
-dotraplinkage void do_debug(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_debug, struct pt_regs *, regs, long, error_code)
{
struct task_struct *tsk = current;
int user_icebp = 0;
@@ -808,7 +805,6 @@ dotraplinkage void do_debug(struct pt_re
exit:
ist_exit(regs);
}
-NOKPROBE_SYMBOL(do_debug);
/*
* Note that we play around with the 'TS' bit in an attempt to get
@@ -855,27 +851,24 @@ static void math_error(struct pt_regs *r
(void __user *)uprobe_get_trap_addr(regs), task);
}
-dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_coprocessor_error, struct pt_regs *, regs, long, error_code)
{
RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
math_error(regs, error_code, X86_TRAP_MF);
}
-dotraplinkage void
-do_simd_coprocessor_error(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_simd_coprocessor_error, struct pt_regs *, regs, long, error_code)
{
RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
math_error(regs, error_code, X86_TRAP_XF);
}
-dotraplinkage void
-do_spurious_interrupt_bug(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_spurious_interrupt_bug, struct pt_regs *, regs, long, error_code)
{
cond_local_irq_enable(regs);
}
-dotraplinkage void
-do_device_not_available(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_device_not_available, struct pt_regs *, regs, long, error_code)
{
unsigned long cr0 = read_cr0();
@@ -906,10 +899,9 @@ do_device_not_available(struct pt_regs *
die("unexpected #NM exception", regs, error_code);
}
}
-NOKPROBE_SYMBOL(do_device_not_available);
#ifdef CONFIG_X86_32
-dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_iret_error, struct pt_regs *, regs, long, error_code)
{
RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
local_irq_enable();
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -1542,8 +1542,7 @@ trace_page_fault_entries(struct pt_regs
trace_page_fault_kernel(address, regs, error_code);
}
-dotraplinkage void
-do_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address)
+IDTENTRY3(do_page_fault, struct pt_regs *, regs, unsigned long, error_code, unsigned long, address)
{
enum ctx_state prev_state;
@@ -1552,4 +1551,3 @@ do_page_fault(struct pt_regs *regs, unsi
__do_page_fault(regs, error_code, address);
exception_exit(prev_state);
}
-NOKPROBE_SYMBOL(do_page_fault);
prev parent reply other threads:[~2019-07-04 20:03 UTC|newest]
Thread overview: 42+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-07-04 19:55 [PATCH v2 0/7] Tracing vs CR2 (and cleanups) Peter Zijlstra
2019-07-04 19:55 ` [PATCH v2 1/7] x86/paravirt: Make read_cr2() CALLEE_SAVE Peter Zijlstra
2019-07-04 21:49 ` Andy Lutomirski
2019-07-10 19:53 ` Steven Rostedt
2019-07-04 19:55 ` [PATCH v2 2/7] x86/entry/32: Simplify common_exception Peter Zijlstra
2019-07-04 21:51 ` Andy Lutomirski
2019-07-10 20:11 ` Steven Rostedt
2019-07-10 20:14 ` Peter Zijlstra
2019-07-04 19:55 ` [PATCH v2 3/7] x86/entry/64: Simplify idtentry a little Peter Zijlstra
2019-07-04 21:54 ` Andy Lutomirski
2019-07-10 20:23 ` Steven Rostedt
2019-07-04 19:55 ` [PATCH v2 4/7] x86/entry/64: Update comments and sanity tests for create_gap Peter Zijlstra
2019-07-04 21:55 ` Andy Lutomirski
2019-07-10 20:24 ` Steven Rostedt
2019-07-04 19:56 ` [PATCH v2 5/7] x86/mm, tracing: Fix CR2 corruption Peter Zijlstra
2019-07-05 2:18 ` Linus Torvalds
2019-07-05 3:16 ` Andy Lutomirski
2019-07-05 3:27 ` Linus Torvalds
2019-07-05 13:49 ` Peter Zijlstra
2019-07-06 21:41 ` Linus Torvalds
2019-07-06 22:27 ` Steven Rostedt
2019-07-06 22:41 ` Linus Torvalds
2019-07-07 0:08 ` Linus Torvalds
2019-07-07 0:36 ` Andy Lutomirski
2019-07-06 23:50 ` Andy Lutomirski
2019-07-07 3:44 ` Eiichi Tsukata
2019-07-06 11:07 ` Eiichi Tsukata
2019-07-08 7:48 ` Peter Zijlstra
2019-07-08 8:58 ` Eiichi Tsukata
2019-07-08 9:42 ` Eiichi Tsukata
2019-07-09 5:17 ` Eiichi Tsukata
2019-07-07 15:10 ` Andy Lutomirski
2019-07-07 15:11 ` Andy Lutomirski
2019-07-07 18:17 ` Linus Torvalds
2019-07-10 20:27 ` Steven Rostedt
2019-07-11 6:45 ` Greg Kroah-Hartman
2019-07-11 12:12 ` Sasha Levin
2019-07-11 12:21 ` Peter Zijlstra
2019-07-04 19:56 ` [PATCH v2 6/7] x86/entry/64: Remove TRACE_IRQS_*_DEBUG Peter Zijlstra
2019-07-11 3:24 ` Steven Rostedt
2019-07-11 8:05 ` Peter Zijlstra
2019-07-04 19:56 ` Peter Zijlstra [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190704200050.648944551@infradead.org \
--to=peterz@infradead.org \
--cc=bp@alien8.de \
--cc=dave.hansen@linux.intel.com \
--cc=devel@etsukata.com \
--cc=hpa@zytor.com \
--cc=jgross@suse.com \
--cc=joel@joelfernandes.org \
--cc=linux-kernel@vger.kernel.org \
--cc=luto@kernel.org \
--cc=mingo@kernel.org \
--cc=rostedt@goodmis.org \
--cc=tglx@linutronix.de \
--cc=torvalds@linux-foundation.org \
--cc=zhe.he@windriver.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.