[PATCH 18/39] NLKD/x86-64 - INT1/INT3 handling changes

["Jan Beulich" <JBeulich@novell.com>]

[-- Attachment #1: Type: text/plain, Size: 543 bytes --]

This
- switches the INT3 handler to run on an IST stack (to cope with
  breakpoints set by a kernel debugger on places where the kernel's
  %gs base hasn't been set up, yet); the IST stack used is shared with
  the INT1 handler's
- allows nesting of INT1/INT3 handlers so that one can, with a kernel
  debugger, debug (at least) the user-mode portions of the INT1/INT3
  handling; the nesting isn't actively enabled here since a kernel-
  debugger-free kernel doesn't need it

From: Jan Beulich <jbeulich@novell.com>

(actual patch attached)


[-- Attachment #2: linux-2.6.14-nlkd-x86_64-debug-stack.patch --]
[-- Type: application/octet-stream, Size: 8981 bytes --]

This
- switches the INT3 handler to run on an IST stack (to cope with
  breakpoints set by a kernel debugger on places where the kernel's
  %gs base hasn't been set up, yet); the IST stack used is shared with
  the INT1 handler's
- allows nesting of INT1/INT3 handlers so that one can, with a kernel
  debugger, debug (at least) the user-mode portions of the INT1/INT3
  handling; the nesting isn't actively enabled here since a kernel-
  debugger-free kernel doesn't need it

From: Jan Beulich <jbeulich@novell.com>

Index: 2.6.14-nlkd/arch/x86_64/kernel/asm-offsets.c
===================================================================
--- 2.6.14-nlkd.orig/arch/x86_64/kernel/asm-offsets.c	2005-11-09 11:18:26.000000000 +0100
+++ 2.6.14-nlkd/arch/x86_64/kernel/asm-offsets.c	2005-11-04 17:14:08.000000000 +0100
@@ -64,5 +64,10 @@ int main(void)
 	DEFINE(pbe_address, offsetof(struct pbe, address));
 	DEFINE(pbe_orig_address, offsetof(struct pbe, orig_address));
 	DEFINE(pbe_next, offsetof(struct pbe, next));
+	BLANK();
+	DEFINE(EXCEPTION_STACK_SIZE, EXCEPTION_STKSZ);
+#if DEBUG_STKSZ > EXCEPTION_STKSZ
+	DEFINE(DEBUG_IST, DEBUG_STACK);
+#endif
 	return 0;
 }
Index: 2.6.14-nlkd/arch/x86_64/kernel/entry.S
===================================================================
--- 2.6.14-nlkd.orig/arch/x86_64/kernel/entry.S	2005-11-09 11:18:26.000000000 +0100
+++ 2.6.14-nlkd/arch/x86_64/kernel/entry.S	2005-11-09 10:45:26.000000000 +0100
@@ -685,7 +685,10 @@ END(spurious_interrupt)
 
 	/* error code is on the stack already */
 	/* handle NMI like exceptions that can happen everywhere */
-	.macro paranoidentry sym
+#ifndef DEBUG_IST
+# define DEBUG_IST 0
+#endif
+	.macro paranoidentry sym, ist=0
 	SAVE_ALL
 	cld
 	movl $1,%ebx
@@ -695,10 +698,20 @@ END(spurious_interrupt)
 	js    1f
 	swapgs
 	xorl  %ebx,%ebx
-1:	movq %rsp,%rdi
+1:
+	.if \ist
+	movq	%gs:pda_data_offset, %rbp
+	.endif
+	movq %rsp,%rdi
 	movq ORIG_RAX(%rsp),%rsi
 	movq $-1,ORIG_RAX(%rsp)
+	.if \ist
+	subq	$EXCEPTION_STACK_SIZE, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
+	.endif
 	call \sym
+	.if \ist
+	addq	$EXCEPTION_STACK_SIZE, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
+	.endif
 	cli
 	.endm
 	
@@ -924,7 +937,7 @@ KPROBE_ENTRY(debug)
  	INTR_FRAME
 	pushq $0
 	CFI_ADJUST_CFA_OFFSET 8		
-	paranoidentry do_debug
+	paranoidentry do_debug, DEBUG_IST
 	jmp paranoid_exit
 	CFI_ENDPROC
 END(debug)
@@ -981,7 +994,12 @@ paranoid_schedule:
 END(nmi)
 
 KPROBE_ENTRY(int3)
-	zeroentry do_int3	
+	INTR_FRAME
+	pushq $0
+	CFI_ADJUST_CFA_OFFSET 8
+	paranoidentry do_int3, DEBUG_IST
+	jmp paranoid_exit
+	CFI_ENDPROC
 END(int3)
 	.previous .text
 
Index: 2.6.14-nlkd/arch/x86_64/kernel/setup64.c
===================================================================
--- 2.6.14-nlkd.orig/arch/x86_64/kernel/setup64.c	2005-11-09 11:18:26.000000000 +0100
+++ 2.6.14-nlkd/arch/x86_64/kernel/setup64.c	2005-11-04 16:19:33.000000000 +0100
@@ -146,7 +146,7 @@ void pda_init(int cpu)
 	pda->irqstackptr += IRQSTACKSIZE-64;
 } 
 
-char boot_exception_stacks[N_EXCEPTION_STACKS * EXCEPTION_STKSZ] 
+char boot_exception_stacks[(N_EXCEPTION_STACKS - 2) * EXCEPTION_STKSZ + DEBUG_STKSZ]
 __attribute__((section(".bss.page_aligned")));
 
 /* May not be marked __init: used by software suspend */
@@ -243,13 +243,27 @@ void __cpuinit cpu_init (void)
 	 */
 	for (v = 0; v < N_EXCEPTION_STACKS; v++) {
 		if (cpu) {
-			estacks = (char *)__get_free_pages(GFP_ATOMIC, 
-						   EXCEPTION_STACK_ORDER);
+			static const unsigned int order[N_EXCEPTION_STACKS] = {
+				[0 ... N_EXCEPTION_STACKS - 1] = EXCEPTION_STACK_ORDER,
+				[DEBUG_STACK - 1] = DEBUG_STACK_ORDER
+			};
+
+			estacks = (char *)__get_free_pages(GFP_ATOMIC, order[v]);
 			if (!estacks)
 				panic("Cannot allocate exception stack %ld %d\n",
 				      v, cpu); 
 		}
-		estacks += EXCEPTION_STKSZ;
+		switch (v + 1) {
+#if DEBUG_STKSZ > EXCEPTION_STKSZ
+		case DEBUG_STACK:
+			cpu_pda[cpu].debugstack = (unsigned long)estacks;
+			estacks += DEBUG_STKSZ;
+			break;
+#endif
+		default:
+			estacks += EXCEPTION_STKSZ;
+			break;
+		}
 		t->ist[v] = (unsigned long)estacks;
 	}
 
Index: 2.6.14-nlkd/arch/x86_64/kernel/traps.c
===================================================================
--- 2.6.14-nlkd.orig/arch/x86_64/kernel/traps.c	2005-11-09 11:18:26.000000000 +0100
+++ 2.6.14-nlkd/arch/x86_64/kernel/traps.c	2005-11-09 11:19:36.000000000 +0100
@@ -121,19 +121,31 @@ int printk_address(unsigned long address
 static unsigned long *in_exception_stack(unsigned cpu, unsigned long stack,
 					unsigned *usedp, const char **idp)
 {
-	static const char ids[N_EXCEPTION_STACKS][8] = {
+	static char ids[][8] = {
 		[DEBUG_STACK - 1] = "#DB",
 		[NMI_STACK - 1] = "NMI",
 		[DOUBLEFAULT_STACK - 1] = "#DF",
 		[STACKFAULT_STACK - 1] = "#SS",
 		[MCE_STACK - 1] = "#MC",
+#if DEBUG_STKSZ > EXCEPTION_STKSZ
+		[N_EXCEPTION_STACKS ... N_EXCEPTION_STACKS + DEBUG_STKSZ / EXCEPTION_STKSZ - 2] = "#DB[?]"
+#endif
 	};
 	unsigned k;
 
 	for (k = 0; k < N_EXCEPTION_STACKS; k++) {
 		unsigned long end;
 
-		end = per_cpu(init_tss, cpu).ist[k];
+		switch (k + 1) {
+#if DEBUG_STKSZ > EXCEPTION_STKSZ
+		case DEBUG_STACK:
+			end = cpu_pda[cpu].debugstack + DEBUG_STKSZ;
+			break;
+#endif
+		default:
+			end = per_cpu(init_tss, cpu).ist[k];
+			break;
+		}
 		if (stack >= end)
 			continue;
 		if (stack >= end - EXCEPTION_STKSZ) {
@@ -143,6 +155,22 @@ static unsigned long *in_exception_stack
 			*idp = ids[k];
 			return (unsigned long *)end;
 		}
+#if DEBUG_STKSZ > EXCEPTION_STKSZ
+		if (k == DEBUG_STACK - 1 && stack >= end - DEBUG_STKSZ) {
+			unsigned j = N_EXCEPTION_STACKS - 1;
+
+			do {
+				++j;
+				end -= EXCEPTION_STKSZ;
+				ids[j][4] = '1' + (j - N_EXCEPTION_STACKS);
+			} while (stack < end - EXCEPTION_STKSZ);
+			if (*usedp & (1U << j))
+				break;
+			*usedp |= 1U << j;
+			*idp = ids[j];
+			return (unsigned long *)end;
+		}
+#endif
 	}
 	return NULL;
 }
@@ -652,6 +680,7 @@ asmlinkage void default_do_nmi(struct pt
 		io_check_error(reason, regs);
 }
 
+/* runs on IST stack. */
 asmlinkage void __kprobes do_int3(struct pt_regs * regs, long error_code)
 {
 	if (notify_die(DIE_INT3, "int3", regs, error_code, 3, SIGTRAP) == NOTIFY_STOP) {
@@ -942,7 +971,7 @@ void __init trap_init(void)
 	set_intr_gate(0,&divide_error);
 	set_intr_gate_ist(1,&debug,DEBUG_STACK);
 	set_intr_gate_ist(2,&nmi,NMI_STACK);
-	set_system_gate(3,&int3);
+	set_system_gate_ist(3,&int3,DEBUG_STACK); /* int3 can be called from all */
 	set_system_gate(4,&overflow);	/* int4 can be called from all */
 	set_intr_gate(5,&bounds);
 	set_intr_gate(6,&invalid_op);
Index: 2.6.14-nlkd/include/asm-x86_64/desc.h
===================================================================
--- 2.6.14-nlkd.orig/include/asm-x86_64/desc.h	2005-11-09 11:18:27.000000000 +0100
+++ 2.6.14-nlkd/include/asm-x86_64/desc.h	2005-11-04 16:19:34.000000000 +0100
@@ -111,6 +111,11 @@ static inline void set_system_gate(int n
 	_set_gate(&idt_table[nr], GATE_INTERRUPT, (unsigned long) func, 3, 0); 
 } 
 
+static inline void set_system_gate_ist(int nr, void *func, unsigned ist)
+{
+	_set_gate(&idt_table[nr], GATE_INTERRUPT, (unsigned long) func, 3, ist);
+}
+
 static inline void set_tssldt_descriptor(void *ptr, unsigned long tss, unsigned type, 
 					 unsigned size) 
 { 
Index: 2.6.14-nlkd/include/asm-x86_64/pda.h
===================================================================
--- 2.6.14-nlkd.orig/include/asm-x86_64/pda.h	2005-11-09 11:18:27.000000000 +0100
+++ 2.6.14-nlkd/include/asm-x86_64/pda.h	2005-11-07 10:33:42.000000000 +0100
@@ -12,6 +12,10 @@ struct x8664_pda {
 	unsigned long data_offset;	/* Per cpu data offset from linker address */
 	unsigned long kernelstack;  /* top of kernel stack for current */ 
 	unsigned long oldrsp; 	    /* user rsp for system call */
+#ifdef CONFIG_NLKD /* Better would be DEBUG_STKSZ > EXCEPTION_STKSZ, but
+                      this file can't include processor.h. */
+	unsigned long debugstack;   /* #DB/#BP stack. */
+#endif
         int irqcount;		    /* Irq nesting counter. Starts with -1 */  	
 	int cpunumber;		    /* Logical CPU number */
 	char *irqstackptr;	/* top of irqstack */
Index: 2.6.14-nlkd/include/asm-x86_64/processor.h
===================================================================
--- 2.6.14-nlkd.orig/include/asm-x86_64/processor.h	2005-11-09 11:18:27.000000000 +0100
+++ 2.6.14-nlkd/include/asm-x86_64/processor.h	2005-11-07 10:36:06.000000000 +0100
@@ -271,8 +271,10 @@ struct thread_struct {
 #define DEBUG_STACK 4 
 #define MCE_STACK 5
 #define N_EXCEPTION_STACKS 5  /* hw limit: 7 */
-#define EXCEPTION_STKSZ (PAGE_SIZE << EXCEPTION_STACK_ORDER)
 #define EXCEPTION_STACK_ORDER 0 
+#define EXCEPTION_STKSZ (PAGE_SIZE << EXCEPTION_STACK_ORDER)
+#define DEBUG_STACK_ORDER EXCEPTION_STACK_ORDER
+#define DEBUG_STKSZ (PAGE_SIZE << DEBUG_STACK_ORDER)
 
 #define start_thread(regs,new_rip,new_rsp) do { \
 	asm volatile("movl %0,%%fs; movl %0,%%es; movl %0,%%ds": :"r" (0));	 \