Make MCA stack look like a normal kernel stack

Make MCA stack look like a normal kernel stack

[Keith Owens]

I have a long term aim to make the MCA and INIT stacks look like normal
process stacks, so we can get decent backtraces for both MCA and INIT.
This includes tracing problems in the MCA/INIT C code, even if we take
an MCA in the INIT handler :).  Eventually we should be able to take a
crash dump from MCA or INIT, with a decent backtrace for the dump
analysis to work with.

This patch makes the MCA stack the same size as a kernel stack, with
the same basic layout.  That includes using the MCA stack for RBS
instead of the existing disjoint rbstore.  The patch is being thrown
open for review, before I spend too much time on this project.

There is space reserved for pt_regs on the MCA stack, it is not being
filled in yet.  pt_regs is required to tell the unwinder that it needs
to switch bsp to chain from MCA to INIT to normal stack; only pt_regs
contains the old bsp values.

The MCA RSE stack frame needs more work before it can be used for
unwinding.  For the moment I have kept the existing layout.

Index: linux/include/asm-ia64/mca_asm.h
=================================--- linux.orig/include/asm-ia64/mca_asm.h	2005-02-09 12:30:05.000000000 +1100
+++ linux/include/asm-ia64/mca_asm.h	2005-02-10 12:09:37.000000000 +1100
@@ -231,6 +231,7 @@
 #define  rse_bspstore_offset	(rse_ifs_offset+0x08)
 #define  rse_rnat_offset	(rse_bspstore_offset+0x08)
 #define  rse_ndirty_offset	(rse_rnat_offset+0x08)
+#define  rse_stack_size		(rse_ndirty_offset+0x08)
 
 /*
  * rse_switch_context
@@ -309,4 +310,32 @@
 	srlz.i;;								\
 	rfi;;
 
+/* MCA stack in struct ia64_mca_cpu looks like a normal kernel stack, except
+ * that the rse_stack and proc_state_dump are stored at the top of the MCA
+ * stack.  All entries are 16 byte aligned.
+ *
+ *      +---------------------------+
+ *      |       RSE stack frame     |
+ *      +---------------------------+
+ *      |       proc_state_dump     |
+ *      +---------------------------+
+ *      |           pt_regs         |
+ *      +---------------------------+
+ *      |    16 byte scratch area   |
+ *      +---------------------------+ <-------- SP at start of C MCA handler
+ *      |           .....           |
+ *      +---------------------------+
+ *      |    RBS for MCA handler    |
+ *      +---------------------------+
+ *      | struct task, not used yet |
+ *      +---------------------------+ <-------- Bottom of MCA stack
+ */
+#define ALIGN16(x)			((x)&~15)
+#define MCA_STACKFRAME_OFFSET		ALIGN16(IA64_MCA_CPU_MCA_STACK_OFFSET+KERNEL_STACK_SIZE-rse_stack_size)
+#define MCA_PROC_STATE_DUMP_SIZE	(512*8)
+#define MCA_PROC_STATE_DUMP_OFFSET	ALIGN16(MCA_STACKFRAME_OFFSET-MCA_PROC_STATE_DUMP_SIZE)
+#define MCA_PT_REGS_OFFSET		ALIGN16(MCA_PROC_STATE_DUMP_OFFSET-IA64_PT_REGS_SIZE)
+#define MCA_SP_OFFSET			ALIGN16(MCA_PT_REGS_OFFSET-16)
+#define MCA_RBSTORE_OFFSET      	(IA64_MCA_CPU_MCA_STACK_OFFSET+IA64_RBS_OFFSET)
+
 #endif /* _ASM_IA64_MCA_ASM_H */
Index: linux/include/asm-ia64/mca.h
=================================--- linux.orig/include/asm-ia64/mca.h	2005-02-09 12:30:05.000000000 +1100
+++ linux/include/asm-ia64/mca.h	2005-02-10 12:27:53.000000000 +1100
@@ -11,8 +11,6 @@
 #ifndef _ASM_IA64_MCA_H
 #define _ASM_IA64_MCA_H
 
-#define IA64_MCA_STACK_SIZE	8192
-
 #if !defined(__ASSEMBLY__)
 
 #include <linux/interrupt.h>
@@ -107,10 +105,7 @@ typedef struct ia64_mca_os_to_sal_state_
 /* Per-CPU MCA state that is too big for normal per-CPU variables.  */
 
 struct ia64_mca_cpu {
-	u64 stack[IA64_MCA_STACK_SIZE/8];	/* MCA memory-stack */
-	u64 proc_state_dump[512];
-	u64 stackframe[32];
-	u64 rbstore[IA64_MCA_STACK_SIZE/8];	/* MCA reg.-backing store */
+	u64 mca_stack[KERNEL_STACK_SIZE/8];
 	u64 init_stack[KERNEL_STACK_SIZE/8];
 } __attribute__ ((aligned(16)));
 
Index: linux/arch/ia64/kernel/mca_asm.S
=================================--- linux.orig/arch/ia64/kernel/mca_asm.S	2005-02-09 12:30:07.000000000 +1100
+++ linux/arch/ia64/kernel/mca_asm.S	2005-02-10 12:26:36.000000000 +1100
@@ -316,16 +316,14 @@ done_tlb_purge_and_reload:
 	// Setup new stack frame for OS_MCA handling
 	GET_IA64_MCA_DATA(r2)
 	;;
-	add r3 = IA64_MCA_CPU_STACKFRAME_OFFSET, r2
-	add r2 = IA64_MCA_CPU_RBSTORE_OFFSET, r2
+	add r3 = MCA_STACKFRAME_OFFSET, r2
+	add r2 = MCA_RBSTORE_OFFSET, r2
 	;;
 	rse_switch_context(r6,r3,r2);;	// RSC management in this new context
 
 	GET_IA64_MCA_DATA(r2)
 	;;
-	add r2 = IA64_MCA_CPU_STACK_OFFSET+IA64_MCA_STACK_SIZE-16, r2
-	;;
-	mov r12=r2		// establish new stack-pointer
+	add r12 = MCA_SP_OFFSET, r2 // establish new stack-pointer
 
         // Enter virtual mode from physical mode
 	VIRTUAL_MODE_ENTER(r2, r3, ia64_os_mca_virtual_begin, r4)
@@ -343,7 +341,7 @@ ia64_os_mca_virtual_end:
 	// restore the original stack frame here
 	GET_IA64_MCA_DATA(r2)
 	;;
-	add r2 = IA64_MCA_CPU_STACKFRAME_OFFSET, r2
+	add r2 = MCA_STACKFRAME_OFFSET, r2
 	;;
 	movl    r4=IA64_PSR_MC
 	;;
@@ -387,7 +385,7 @@ ia64_os_mca_proc_state_dump:
 //  to virtual addressing mode.
 	GET_IA64_MCA_DATA(r2)
 	;;
-	add r2 = IA64_MCA_CPU_PROC_STATE_DUMP_OFFSET, r2
+	add r2 = MCA_PROC_STATE_DUMP_OFFSET, r2
 	;;
 // save ar.NaT
 	mov		r5=ar.unat                  // ar.unat
@@ -620,7 +618,7 @@ ia64_os_mca_proc_state_restore:
 // Restore bank1 GR16-31
 	GET_IA64_MCA_DATA(r2)
 	;;
-	add r2 = IA64_MCA_CPU_PROC_STATE_DUMP_OFFSET, r2
+	add r2 = MCA_PROC_STATE_DUMP_OFFSET, r2
 
 restore_GRs:                                    // restore bank-1 GRs 16-31
 	bsw.1;;
Index: linux/arch/ia64/kernel/asm-offsets.c
=================================--- linux.orig/arch/ia64/kernel/asm-offsets.c	2005-02-09 12:30:05.000000000 +1100
+++ linux/arch/ia64/kernel/asm-offsets.c	2005-02-10 11:26:25.000000000 +1100
@@ -211,14 +211,8 @@ void foo(void)
 #endif
 
 	BLANK();
-	DEFINE(IA64_MCA_CPU_PROC_STATE_DUMP_OFFSET,
-	       offsetof (struct ia64_mca_cpu, proc_state_dump));
-	DEFINE(IA64_MCA_CPU_STACK_OFFSET,
-	       offsetof (struct ia64_mca_cpu, stack));
-	DEFINE(IA64_MCA_CPU_STACKFRAME_OFFSET,
-	       offsetof (struct ia64_mca_cpu, stackframe));
-	DEFINE(IA64_MCA_CPU_RBSTORE_OFFSET,
-	       offsetof (struct ia64_mca_cpu, rbstore));
+	DEFINE(IA64_MCA_CPU_MCA_STACK_OFFSET,
+	       offsetof (struct ia64_mca_cpu, mca_stack));
 	DEFINE(IA64_MCA_CPU_INIT_STACK_OFFSET,
 	       offsetof (struct ia64_mca_cpu, init_stack));
 	BLANK();

Re: Make MCA stack look like a normal kernel stack

[David Mosberger]

>>>>> On Thu, 10 Feb 2005 12:32:53 +1100, Keith Owens <kaos@sgi.com> said:

  Keith> I have a long term aim to make the MCA and INIT stacks look
  Keith> like normal process stacks

That would indeed simplify life for the unwinder a bit, though it's
not strictly needed.  Alternatively, it might be possible to define
the proc_state_dump area as yet another special frame.  I would prefer
if this can be avoided, though.

  Keith> * MCA stack in struct ia64_mca_cpu looks like a normal kernel stack,
  Keith> * except that the rse_stack and proc_state_dump are stored at the
  Keith> * top of the MCA stack.  All entries are 16 byte aligned.
  Keith> *
  Keith> *      +---------------------------+
  Keith> *      |       RSE stack frame     |
  Keith> *      +---------------------------+
  Keith> *      |       proc_state_dump     |
  Keith> *      +---------------------------+
  Keith> *      |           pt_regs         |
  Keith> *      +---------------------------+
  Keith> *      |    16 byte scratch area   |
  Keith> *      +---------------------------+ <-- SP at start of C MCA handler
  Keith> *      |           .....           |
  Keith> *      +---------------------------+
  Keith> *      |    RBS for MCA handler    |
  Keith> *      +---------------------------+
  Keith> *      | struct task, not used yet |
  Keith> *      +---------------------------+ <-- Bottom of MCA stack

Hmmh, I don't understand why "RSE stack frame" is at the
highest-address-end of the stack.  Why not make it part of the RBS, as
is normally done?

Also, I'd hope that proc_state_dump isn't needed as part of the
task-structure: we should be able to store part of its contents in
pt_regs and the rest should be loaded into the preserved registers
before calling the first C routine.  No?

	--david

Re: Make MCA stack look like a normal kernel stack

[Keith Owens]

On Wed, 9 Feb 2005 18:26:37 -0800, 
David Mosberger <davidm@napali.hpl.hp.com> wrote:
>>>>>> On Thu, 10 Feb 2005 12:32:53 +1100, Keith Owens <kaos@sgi.com> said:
>
>  Keith> I have a long term aim to make the MCA and INIT stacks look
>  Keith> like normal process stacks
>
>That would indeed simplify life for the unwinder a bit, though it's
>not strictly needed.  Alternatively, it might be possible to define
>the proc_state_dump area as yet another special frame.  I would prefer
>if this can be avoided, though.

<aol>Me too</aol>.  The problem is not just the kernel unwinder, it is
ia64 unwind code in general, including gdb, lcrash and other dump
diagnosis tools.  All those unwinders would need to be updated to
define proc_state_dump.  No thanks.

>  Keith> * MCA stack in struct ia64_mca_cpu looks like a normal kernel stack,
>  Keith> * except that the rse_stack and proc_state_dump are stored at the
>  Keith> * top of the MCA stack.  All entries are 16 byte aligned.
>  Keith> *
>  Keith> *      +---------------------------+
>  Keith> *      |       RSE stack frame     |
>  Keith> *      +---------------------------+
>  Keith> *      |       proc_state_dump     |
>  Keith> *      +---------------------------+
>  Keith> *      |           pt_regs         |
>  Keith> *      +---------------------------+
>  Keith> *      |    16 byte scratch area   |
>  Keith> *      +---------------------------+ <-- SP at start of C MCA handler
>  Keith> *      |           .....           |
>  Keith> *      +---------------------------+
>  Keith> *      |    RBS for MCA handler    |
>  Keith> *      +---------------------------+
>  Keith> *      | struct task, not used yet |
>  Keith> *      +---------------------------+ <-- Bottom of MCA stack
>
>Hmmh, I don't understand why "RSE stack frame" is at the
>highest-address-end of the stack.  Why not make it part of the RBS, as
>is normally done?

As I said in my mail, RSE stack frame needs more work.  In the current
kernel it is a separate area with its own format.  I just collapsed
that area into the MCA stack, preserving the format.  Eventually it
will be part of the RBS, once I work out what format the unwinder
requires.

The problem is that some of the dirty registers will be in the original
stack, some will be in the MCA stack.  It may be easier to copy the
dirty registers from the MCA stack to the previous stack in the
MCA/INIT handlers, rather than handling it in the unwinder.  WIP.

>Also, I'd hope that proc_state_dump isn't needed as part of the
>task-structure: we should be able to store part of its contents in
>pt_regs

I agree.  This is a first cut at consolidating the existing data areas,
with minimal code changes.  As I start testing the unwind support,
pt_regs will replace most, if not all, of proc_state_dump.  That
requires much more intrusive code changes.

>and the rest should be loaded into the preserved registers
>before calling the first C routine.  No?

The preserved registers at the time of MCA/INIT still need to be saved
so they can be restored on return to SAL.  Save the preserved
registers, set the values that the kernel needs, call C code, restore
original preserved registers, return to SAL.  Hmm, I just described
struct switch_stack, didn't I?

Re: Make MCA stack look like a normal kernel stack

[David Mosberger]

>>>>> On Thu, 10 Feb 2005 13:47:47 +1100, Keith Owens <kaos@sgi.com> said:

  Keith> The problem is that some of the dirty registers will be in
  Keith> the original stack, some will be in the MCA stack.

That's OK, libunwind already handles that.  The only thing that's
needed to make this work is that ar.bsp/ar.bspstore/ar.rnat are saved
and that those saves properly be described by the unwind-info.  The
rest will be automatic.  There is one problem though: in pt_regs, we
save the loadrs value instead of ar.bsp.  That can't be described at
the moment (the pt_regs code was written long before I knew about
unwind-info...).  However, since pt_regs is special anyhow, we can
just handle that in the unwinder, analogous to what we do for
signal-handlers (check_rbs_switch() in libunwind's src/ia64/Gstep.c).

  Keith> The preserved registers at the time of MCA/INIT still need to
  Keith> be saved so they can be restored on return to SAL.  Save the
  Keith> preserved registers, set the values that the kernel needs,
  Keith> call C code, restore original preserved registers, return to
  Keith> SAL.  Hmm, I just described struct switch_stack, didn't I?

True.  If you really return, then there is no need to even use switch_stack.
You'd just return and that magically restores the preserved registers.

	--david