[patch 5/5] x86/idt: Consolidate idt functionality

[Thomas Gleixner <tglx@linutronix.de>]

 - Move load_current_idt() out of line and replace the hideous comment
   with a lockdep assert.

 - Move debug IDT cruft into the IDT code and simplify it.  This makes
   debug_idt_ctr static and removes one function call.

 - Move the idt descriptor table definition to spare the extra #ifdef
   sections and make it static.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/include/asm/desc.h  |   45 -------------------------
 arch/x86/kernel/cpu/common.c |   17 ---------
 arch/x86/kernel/idt.c        |   75 ++++++++++++++++++++++++++++++-------------
 3 files changed, 55 insertions(+), 82 deletions(-)

--- a/arch/x86/include/asm/desc.h
+++ b/arch/x86/include/asm/desc.h
@@ -40,11 +40,6 @@ static inline void fill_ldt(struct desc_
 	desc->l			= 0;
 }
 
-extern struct desc_ptr idt_descr;
-extern gate_desc idt_table[];
-extern const struct desc_ptr debug_idt_descr;
-extern gate_desc debug_idt_table[];
-
 struct gdt_page {
 	struct desc_struct gdt[GDT_ENTRIES];
 } __attribute__((aligned(PAGE_SIZE)));
@@ -390,45 +385,7 @@ void alloc_intr_gate(unsigned int n, con
 
 extern unsigned long system_vectors[];
 
-#ifdef CONFIG_X86_64
-DECLARE_PER_CPU(u32, debug_idt_ctr);
-static __always_inline bool is_debug_idt_enabled(void)
-{
-	if (this_cpu_read(debug_idt_ctr))
-		return true;
-
-	return false;
-}
-
-static __always_inline void load_debug_idt(void)
-{
-	load_idt((const struct desc_ptr *)&debug_idt_descr);
-}
-#else
-static inline bool is_debug_idt_enabled(void)
-{
-	return false;
-}
-
-static inline void load_debug_idt(void)
-{
-}
-#endif
-
-/*
- * The load_current_idt() must be called with interrupts disabled
- * to avoid races. That way the IDT will always be set back to the expected
- * descriptor. It's also called when a CPU is being initialized, and
- * that doesn't need to disable interrupts, as nothing should be
- * bothering the CPU then.
- */
-static __always_inline void load_current_idt(void)
-{
-	if (is_debug_idt_enabled())
-		load_debug_idt();
-	else
-		load_idt((const struct desc_ptr *)&idt_descr);
-}
+void load_current_idt(void);
 
 extern void idt_setup_early_handler(void);
 extern void idt_setup_early_traps(void);
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1689,23 +1689,6 @@ void syscall_init(void)
 	       X86_EFLAGS_IOPL|X86_EFLAGS_AC|X86_EFLAGS_NT);
 }
 
-DEFINE_PER_CPU(int, debug_stack_usage);
-DEFINE_PER_CPU(u32, debug_idt_ctr);
-
-noinstr void debug_stack_set_zero(void)
-{
-	this_cpu_inc(debug_idt_ctr);
-	load_current_idt();
-}
-
-noinstr void debug_stack_reset(void)
-{
-	if (WARN_ON(!this_cpu_read(debug_idt_ctr)))
-		return;
-	if (this_cpu_dec_return(debug_idt_ctr) == 0)
-		load_current_idt();
-}
-
 #else	/* CONFIG_X86_64 */
 
 DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
--- a/arch/x86/kernel/idt.c
+++ b/arch/x86/kernel/idt.c
@@ -157,6 +157,14 @@ static const __initconst struct idt_data
 #endif
 };
 
+/* Must be page-aligned because the real IDT is used in the cpu entry area */
+static gate_desc idt_table[IDT_ENTRIES] __page_aligned_bss;
+
+static struct desc_ptr idt_descr __ro_after_init = {
+	.size		= IDT_TABLE_SIZE - 1,
+	.address	= (unsigned long) idt_table,
+};
+
 #ifdef CONFIG_X86_64
 /*
  * Early traps running on the DEFAULT_STACK because the other interrupt
@@ -173,20 +181,19 @@ static const __initconst struct idt_data
 static const __initconst struct idt_data dbg_idts[] = {
 	INTG(X86_TRAP_DB,		asm_exc_debug),
 };
-#endif
 
-/* Must be page-aligned because the real IDT is used in a fixmap. */
-gate_desc idt_table[IDT_ENTRIES] __page_aligned_bss;
+/* No need to be aligned, but done to keep all IDTs defined the same way. */
+static gate_desc debug_idt_table[IDT_ENTRIES] __page_aligned_bss;
 
-struct desc_ptr idt_descr __ro_after_init = {
-	.size		= IDT_TABLE_SIZE - 1,
-	.address	= (unsigned long) idt_table,
+/*
+ * Override for the debug_idt. Same as the default, but with interrupt
+ * stack set to DEFAULT_STACK (0). Required for NMI trap handling.
+ */
+static const struct desc_ptr debug_idt_descr = {
+	.size		= IDT_DEBUG_TABLE_SIZE - 1,
+	.address	= (unsigned long) debug_idt_table,
 };
 
-#ifdef CONFIG_X86_64
-/* No need to be aligned, but done to keep all IDTs defined the same way. */
-gate_desc debug_idt_table[IDT_ENTRIES] __page_aligned_bss;
-
 /*
  * The exceptions which use Interrupt stacks. They are setup after
  * cpu_init() when the TSS has been initialized.
@@ -200,15 +207,41 @@ static const __initconst struct idt_data
 #endif
 };
 
-/*
- * Override for the debug_idt. Same as the default, but with interrupt
- * stack set to DEFAULT_STACK (0). Required for NMI trap handling.
- */
-const struct desc_ptr debug_idt_descr = {
-	.size		= IDT_DEBUG_TABLE_SIZE - 1,
-	.address	= (unsigned long) debug_idt_table,
-};
-#endif
+DEFINE_PER_CPU(int, debug_stack_usage);
+static DEFINE_PER_CPU(u32, debug_idt_ctr);
+
+noinstr void load_current_idt(void)
+{
+	lockdep_assert_irqs_disabled();
+
+	if (this_cpu_read(debug_idt_ctr))
+		load_idt(&debug_idt_descr);
+	else
+		load_idt(&idt_descr);
+}
+
+noinstr void debug_stack_set_zero(void)
+{
+	this_cpu_inc(debug_idt_ctr);
+	load_idt(&debug_idt_descr);
+}
+
+noinstr void debug_stack_reset(void)
+{
+	if (WARN_ON(!this_cpu_read(debug_idt_ctr)))
+		return;
+	if (this_cpu_dec_return(debug_idt_ctr) == 0)
+		load_idt(&idt_descr);
+}
+
+#else /* X86_64 */
+
+noinstr void load_current_idt(void)
+{
+	load_idt(&idt_descr);
+}
+
+#endif /* !X86_64 */
 
 static inline void idt_init_desc(gate_desc *gate, const struct idt_data *d)
 {
@@ -264,7 +297,7 @@ void __init idt_setup_early_traps(void)
 {
 	idt_setup_from_table(idt_table, early_idts, ARRAY_SIZE(early_idts),
 			     true);
-	load_idt(&idt_descr);
+	load_current_idt();
 }
 
 /**
@@ -374,7 +407,7 @@ void __init idt_setup_early_handler(void
 	for ( ; i < NR_VECTORS; i++)
 		set_intr_gate(i, early_ignore_irq);
 #endif
-	load_idt(&idt_descr);
+	load_current_idt();
 }
 
 /**