[PATCH 0/7] Patches for kdump vs. INIT

[PATCH 0/7] Patches for kdump vs. INIT

[Hidetoshi Seto]

Hi Tony-san, and kdump folks,

I got some trouble on kdump on IPF with INIT, and my investigation
proves there are some bugs and races in startup of kdump.
Here are fixes based on .30, for issues I found.

Since it must be serious problem for (likely big) IPF servers if we
could fail to retrieve crashdump via kdump, so I believe these patches
should be applied asap.


Thanks,
H.Seto

===

Hidetoshi Seto (7):
      ia64, kdump: Mask MCA/INIT on freezing cpus
      ia64, kexec: Make INIT safe while kdump/kexec
      ia64, kexec: Unregister MCA handler before kexec
      ia64, kdump: Don't offline APs
      ia64, kdump: Mask INIT first in panic-kdump path
      ia64, kdump: Try INIT regardless of kdump_on_init
      ia64, kdump: Short path to freeze CPUs

 arch/ia64/kernel/crash.c         |   85 ++++++++++++++++++++++++++------------
 arch/ia64/kernel/machine_kexec.c |   17 ++++++++
 arch/ia64/kernel/mca.c           |   15 ++++++-
 arch/ia64/kernel/mca_asm.S       |   47 +++++++++++++++++++++
 4 files changed, 136 insertions(+), 28 deletions(-)

[PATCH 1/7] ia64, kdump: Mask MCA/INIT on freezing cpus

[Hidetoshi Seto]

The problem is that the (badly) frozen cpus can be thawed by MCA/INIT.

The kdump_cpu_freeze() is called on cpus except one that initiates
panic and/or kdump, to stop/offline the cpu (on ia64, it means we pass
control of cpus to SAL, or put them in spin-loop).  Note that CPU0(BP)
always go to spin-loop, so if panic was happened on an AP, there are
2cpus (= the AP and BP) which not back to SAL.

On the spinning cpus, interrupts are disabled (rsm psr.i), but MCA/INIT
are still interruptible because psr.mc for mask them is not set unless
kdump_cpu_freeze is not invoked from MCA/INIT context.

Therefore, assume that a panic was happened on an AP, kdump was invoked,
new INIT handlers for kdump kernel was registered and then an INIT is
asserted.  From the viewpoint of SAL, there are 2 online cpus, so INIT
will be delivered to both of them.  It likely means that not only the AP
(= a cpu executing kdump) enters INIT handler which is newly registered,
but also BP (= another cpu spinning in panicked kernel) enters the same
INIT handler.  Of course setting of registers in BP are still old (for
panicked kernel), so what happen with running handler with wrong setting
will be extremely unexpected.  I believe this is not desirable behavior.

How to Reproduce:

  Start kdump on one of APs (e.g. cpu1)
    # taskset 0x2 echo c > /proc/sysrq-trigger
  Then assert INIT after kdump kernel booted

Sample of result:

  I got following console log by asserting INIT after prompt "root:/>".
  It seems two monarchs appeared by one INIT, and one panicked at last:

    :
    [  0 %]dropping to initramfs shell
    exiting this shell will reboot your system
    root:/> Entered OS INIT handler. PSP=fff301a0 cpu=0 monarch=0
    ia64_init_handler: Promoting cpu 0 to monarch.
    Delaying for 5 seconds...
    All OS INIT slaves have reached rendezvous
    Processes interrupted by INIT - 0 (cpu 0 task 0xa000000100af0000)
    :
    <<snip>>
    :
    Entered OS INIT handler. PSP=fff301a0 cpu=0 monarch=1
    Delaying for 5 seconds...
    mlogbuf_finish: printing switched to urgent mode, MCA/INIT might be dodgy or fail.
    OS INIT slave did not rendezvous on cpu 1 2 3
    INIT swapper 0[0]: bugcheck! 0 [1]
    :
    <<snip>>
    :
    Kernel panic - not syncing: Attempted to kill the idle task!

To avoid this problem, This patch inserts ia64_set_psr_mc() before the
deadloop to mask MCA/INIT on cpus going to be frozen.  I confirmed that
weird log like above are disappeared after applying this patch.

Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Cc: Vivek Goyal <vgoyal@redhat.com>
Cc: Haren Myneni <hbabu@us.ibm.com>
Cc: kexec@lists.infradead.org
---
 arch/ia64/kernel/crash.c   |    6 ++++++
 arch/ia64/kernel/mca_asm.S |   27 +++++++++++++++++++++++++++
 2 files changed, 33 insertions(+), 0 deletions(-)

diff --git a/arch/ia64/kernel/crash.c b/arch/ia64/kernel/crash.c
index f065093..48b69fd 100644
--- a/arch/ia64/kernel/crash.c
+++ b/arch/ia64/kernel/crash.c
@@ -95,6 +95,8 @@ kdump_wait_cpu_freeze(void)
 }
 #endif
 
+extern void ia64_set_psr_mc(void);
+
 void
 machine_crash_shutdown(struct pt_regs *pt)
 {
@@ -129,10 +131,14 @@ void
 kdump_cpu_freeze(struct unw_frame_info *info, void *arg)
 {
 	int cpuid;
+
 	local_irq_disable();
 	cpuid = smp_processor_id();
 	crash_save_this_cpu();
 	current->thread.ksp = (__u64)info->sw - 16;
+
+	ia64_set_psr_mc();	/* mask MCA/INIT and stop reentrance */
+
 	atomic_inc(&kdump_cpu_frozen);
 	kdump_status[cpuid] = 1;
 	mb();
diff --git a/arch/ia64/kernel/mca_asm.S b/arch/ia64/kernel/mca_asm.S
index a06d465..c6ee089 100644
--- a/arch/ia64/kernel/mca_asm.S
+++ b/arch/ia64/kernel/mca_asm.S
@@ -1073,3 +1073,30 @@ GLOBAL_ENTRY(ia64_get_rnat)
 	mov ar.rsc=3
 	br.ret.sptk.many rp
 END(ia64_get_rnat)
+
+
+// ia64_set_psr_mc(void)
+//
+// Set psr.mc bit to mask MCA/INIT.
+GLOBAL_ENTRY(ia64_set_psr_mc)
+	rsm psr.i | psr.ic		// disable interrupts
+	;;
+	srlz.d
+	;;
+	mov r14 = psr			// get psr{36:35,31:0}
+	movl r15 = .return
+	;;
+	dep r14 = -1, r14, PSR_MC, 1	// set psr.mc
+	;;
+	dep r14 = -1, r14, PSR_IC, 1	// set psr.ic
+	;;
+	dep r14 = -1, r14, PSR_BN, 1	// keep bank1 in use
+	;;
+	mov cr.ipsr = r14
+	mov cr.ifs = r0
+	mov cr.iip = r15
+	;;
+	rfi
+.return:
+	br.ret.sptk.many rp
+END(ia64_set_psr_mc)
-- 
1.6.0

Re: [PATCH 1/7] ia64, kdump: Mask MCA/INIT on freezing cpus

[Robin Holt]

> To avoid this problem, This patch inserts ia64_set_psr_mc() before the
> deadloop to mask MCA/INIT on cpus going to be frozen.  I confirmed that
> weird log like above are disappeared after applying this patch.

Please do not do this.  Turning off MCA/INIT is just a horrible idea.
When your code has a bug, the INIT of the cpu is the only tool we have
to find out what it is doing short of putting special hardware onto the
machine and trying to track it down.

Without thinking about it, I have a gut feeling there must be some way
to at least allow the MCA/INIT to make it through PROM and be delivered
to the OS.  From there the OS should be able to sort out a way to handle
kdump and MCAs received during a kdump.

Thanks,
Robin

Re: [PATCH 1/7] ia64, kdump: Mask MCA/INIT on freezing cpus

[Hidetoshi Seto]

Robin Holt wrote:
>> To avoid this problem, This patch inserts ia64_set_psr_mc() before the
>> deadloop to mask MCA/INIT on cpus going to be frozen.  I confirmed that
>> weird log like above are disappeared after applying this patch.
> 
> Please do not do this.  Turning off MCA/INIT is just a horrible idea.
> When your code has a bug, the INIT of the cpu is the only tool we have
> to find out what it is doing short of putting special hardware onto the
> machine and trying to track it down.

This patch never mask MCA/INIT while the system is running normally.
The first place I inserted the masking is just after panic, and just after
INIT is asserted.  This patch doesn't prevent you from taking kdump or
stack trace on your machine.

Maybe I could not catch what you pointed.
One of the problems I'm targeting here is that there is no way to allow
INIT while kernel transition.  What are you expecting with INIT if it is
asserted on the beginning of the 2nd kernel?

And note that this patch 1 of 7 is necessary to run the INIT handler of
the 2nd kernel, which might be registered by the 2nd kernel.

> Without thinking about it, I have a gut feeling there must be some way
> to at least allow the MCA/INIT to make it through PROM and be delivered
> to the OS.  From there the OS should be able to sort out a way to handle
> kdump and MCAs received during a kdump.

Do you mean that the 2nd kernel should be able to handle MCA/INIT from its
boot up?  I guess the word PROM is nearly equal to PAL/SAL firmware, if so
then I don't think there are good generic interface/procedure could be
useful here.  Do you have any concrete idea?


Thanks,
H.Seto

Re: [PATCH 1/7] ia64, kdump: Mask MCA/INIT on freezing cpus

[Robin Holt]

On Tue, Jun 23, 2009 at 09:33:46AM +0900, Hidetoshi Seto wrote:
> Robin Holt wrote:
...
> Do you mean that the 2nd kernel should be able to handle MCA/INIT from its
> boot up?  I guess the word PROM is nearly equal to PAL/SAL firmware, if so
> then I don't think there are good generic interface/procedure could be
> useful here.  Do you have any concrete idea?

No concrete ideas.  Just a really uneasy feeling whenever the INIT
is disabled.

On SGI's ia64 servers, when the INIT is first received, our PROM (PAL/SAL)
records the processors state.  This record remains for the next reset
or for salinfo to transfer to disk.  We can dump all the records in a
format somewhat similar to the output in /var/log/salinfo/decoded.

On more occasions than I could even begin to describe, those records
have helped us determine what the processor was doing despite it even
ignoring interrupts.  This includes everything except when MCA/INIT
handling is disabled or PMIs are being processed.

If you could give this some consideration, I would appreciate it.
I really don't have the time to dedicate to thinking all the way through
and therefore will not stand in the way of any patch you propose.
If I can be of help in any way, please let me know.

Thanks,
Robin

Re: [PATCH 1/7] ia64, kdump: Mask MCA/INIT on freezing cpus

[Hidetoshi Seto]

Robin Holt wrote:
> On Tue, Jun 23, 2009 at 09:33:46AM +0900, Hidetoshi Seto wrote:
>> Robin Holt wrote:
> ...
>> Do you mean that the 2nd kernel should be able to handle MCA/INIT from its
>> boot up?  I guess the word PROM is nearly equal to PAL/SAL firmware, if so
>> then I don't think there are good generic interface/procedure could be
>> useful here.  Do you have any concrete idea?
> 
> No concrete ideas.  Just a really uneasy feeling whenever the INIT
> is disabled.

Don't worry, don't be afraid.
Again, my patches don't disable INIT until kdump is invoked.
(And if kdump is invoked via INIT, it have already masked at the begging
 of INIT handlers.)

> If you could give this some consideration, I would appreciate it.
> I really don't have the time to dedicate to thinking all the way through
> and therefore will not stand in the way of any patch you propose.
> If I can be of help in any way, please let me know.

I know you guys are busy with this and that...

It would be the best if you could test my patches soon on SGI servers,
but I would appreciate it too if you could let me know if you have any
trouble on the patches later.


Thanks,
H.Seto

Re: [PATCH 1/7] ia64, kdump: Mask MCA/INIT on freezing cpus

[Robin Holt]

On Tue, Jun 23, 2009 at 05:07:14PM +0900, Hidetoshi Seto wrote:
> Robin Holt wrote:
> > On Tue, Jun 23, 2009 at 09:33:46AM +0900, Hidetoshi Seto wrote:
> >> Robin Holt wrote:
> > ...
> >> Do you mean that the 2nd kernel should be able to handle MCA/INIT from its
> >> boot up?  I guess the word PROM is nearly equal to PAL/SAL firmware, if so
> >> then I don't think there are good generic interface/procedure could be
> >> useful here.  Do you have any concrete idea?
> > 
> > No concrete ideas.  Just a really uneasy feeling whenever the INIT
> > is disabled.
> 
> Don't worry, don't be afraid.
> Again, my patches don't disable INIT until kdump is invoked.
> (And if kdump is invoked via INIT, it have already masked at the begging
>  of INIT handlers.)

The concern is that any time we prevent SAL from receiving control during
an MCA/INIT, we reduce the maintainability of the machine.  Having them
masked at any time results in the NMI/INIT not recording the PROM record
which we use to diagnose where the hang is.

In other patches, you implemented a do-nothing handler.  Could that
be used?

Alternatively, when the machine is first booted, the handler is defined
by SAL as a SAL routine.  Could you record that during kernel boot and
then just set the handler back to the SAL provided one prior to starting
the kexec kernel boot?  At that point, the machine is more like the
first boot.  Now that I think about this, this alternative seems fairly
attractive.

Thanks,
Robin

Re: [PATCH 1/7] ia64, kdump: Mask MCA/INIT on freezing cpus

[Hidetoshi Seto]

Robin Holt wrote:
> The concern is that any time we prevent SAL from receiving control during
> an MCA/INIT, we reduce the maintainability of the machine.  Having them
> masked at any time results in the NMI/INIT not recording the PROM record
> which we use to diagnose where the hang is.

Think about servers which have no such PROM record features... Please?

The original problem here, which I wrote these patches for, is that the
INIT can block retrieving crashdump via kdump.  The crashdump is the only
record which we can use to diagnose where the hang is, if the PROM record
like SGI servers have is not supported.
(I guess the even the PROM record is supported, the crashdump is better,
 more important resource for the trouble shooting.)

My patches will mask MCA/INIT on all CPUs once kdump is invoked (via
panic or INIT), and soon unmask one of them who is going to jump in 2nd
kernel (=kdump kernel) after registering a do-nothing handler.

If there was a pending INIT, it will be received on the cpu as soon as
it is unmasked.  Then the PROM will make a record on it, pass the control
to OS_INIT which does nothing, and return to interrupted context to
continue processing the kdump.

What time point are you concerning?


> In other patches, you implemented a do-nothing handler.  Could that
> be used?

... How?  Maybe I could not catch your point.

It would be useful, but it is only available from the beginning of 2nd
kernel (to be exact, from the end of 1st kernel), until new INIT handlers
for 2nd kernel is registered.


> Alternatively, when the machine is first booted, the handler is defined
> by SAL as a SAL routine.  Could you record that during kernel boot and
> then just set the handler back to the SAL provided one prior to starting
> the kexec kernel boot?  At that point, the machine is more like the
> first boot.  Now that I think about this, this alternative seems fairly
> attractive.

I think it is definitely wrong thing if SAL provides the initial handler
as OS_INIT which can be removed/replaced by OS.

Since INIT event processes PAL_INIT -> SAL_INIT -> OS_INIT(if available),
SAL should keep the entry point of its initial handler and should use it
from SAL_INIT when OS_INIT is not registered.  Ditto to OS_MCA.


Thanks,
H.Seto

Re: [PATCH 1/7] ia64, kdump: Mask MCA/INIT on freezing cpus

[Robin Holt]

Let's just leave it at you have an opinion of how things should be done
and I don't agree with that position.  If there are errors occurring
in hardware, disabling the MCA handler will do nothing but make the
kdump crash stall forever as the processor tries to consume bad data.
It also removes the ability to find out why things are broken in the event
that there are any errors in the kexec kernel which prevent the boot.
You have exceeded the amount of time I have to argue against your patches.

Good Luck,
Robin

On Thu, Jun 25, 2009 at 11:15:59AM +0900, Hidetoshi Seto wrote:
> Robin Holt wrote:
> > The concern is that any time we prevent SAL from receiving control during
> > an MCA/INIT, we reduce the maintainability of the machine.  Having them
> > masked at any time results in the NMI/INIT not recording the PROM record
> > which we use to diagnose where the hang is.
> 
> Think about servers which have no such PROM record features... Please?
> 
> The original problem here, which I wrote these patches for, is that the
> INIT can block retrieving crashdump via kdump.  The crashdump is the only
> record which we can use to diagnose where the hang is, if the PROM record
> like SGI servers have is not supported.
> (I guess the even the PROM record is supported, the crashdump is better,
>  more important resource for the trouble shooting.)
> 
> My patches will mask MCA/INIT on all CPUs once kdump is invoked (via
> panic or INIT), and soon unmask one of them who is going to jump in 2nd
> kernel (=kdump kernel) after registering a do-nothing handler.
> 
> If there was a pending INIT, it will be received on the cpu as soon as
> it is unmasked.  Then the PROM will make a record on it, pass the control
> to OS_INIT which does nothing, and return to interrupted context to
> continue processing the kdump.
> 
> What time point are you concerning?
> 
> 
> > In other patches, you implemented a do-nothing handler.  Could that
> > be used?
> 
> ... How?  Maybe I could not catch your point.
> 
> It would be useful, but it is only available from the beginning of 2nd
> kernel (to be exact, from the end of 1st kernel), until new INIT handlers
> for 2nd kernel is registered.
> 
> 
> > Alternatively, when the machine is first booted, the handler is defined
> > by SAL as a SAL routine.  Could you record that during kernel boot and
> > then just set the handler back to the SAL provided one prior to starting
> > the kexec kernel boot?  At that point, the machine is more like the
> > first boot.  Now that I think about this, this alternative seems fairly
> > attractive.
> 
> I think it is definitely wrong thing if SAL provides the initial handler
> as OS_INIT which can be removed/replaced by OS.
> 
> Since INIT event processes PAL_INIT -> SAL_INIT -> OS_INIT(if available),
> SAL should keep the entry point of its initial handler and should use it
> from SAL_INIT when OS_INIT is not registered.  Ditto to OS_MCA.
> 
> 
> Thanks,
> H.Seto

[PATCH 2/7] ia64, kexec: Make INIT safe while kdump/kexec

[Hidetoshi Seto]

In panic situation, we may receive INIT while kernel transition,
i.e. from preparation of kdump to bootstrap of kdump kernel.
Since we initialize registers on leave from current kernel, no
longer monarch/slave handlers of current kernel in virtual mode are
called safely.  (In fact system goes hang as far as I confirmed)

How to Reproduce:

  Start kdump
    # echo c > /proc/sysrq-trigger
  Then assert INIT before kdump kernel boots

We can unregister these init handlers from SAL before jumping into
new kernel, however then the INIT will fallback to default behavior,
result in warmboot by SAL (according to the SAL specification) and
we cannot retrieve the crashdump.

Therefore this patch introduces a NOP init handler and register it
to SAL before leave from current kernel, to start kdump safely by
preventing INITs from entering virtual mode and resulting in warmboot.

On the other hand, in case of kexec that not for kdump, it also
has same problem with INIT while kernel transition.
This patch handles this case differently, because for kexec
unregistering handlers will be preferred than registering NOP
handler, since the situation "no handlers registered" is usual
state for kernel's entry.

Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Cc: Vivek Goyal <vgoyal@redhat.com>
Cc: Haren Myneni <hbabu@us.ibm.com>
Cc: kexec@lists.infradead.org
---
 arch/ia64/kernel/machine_kexec.c |   14 ++++++++++++++
 arch/ia64/kernel/mca_asm.S       |   20 ++++++++++++++++++++
 2 files changed, 34 insertions(+), 0 deletions(-)

diff --git a/arch/ia64/kernel/machine_kexec.c b/arch/ia64/kernel/machine_kexec.c
index 0823de1..d35fc39 100644
--- a/arch/ia64/kernel/machine_kexec.c
+++ b/arch/ia64/kernel/machine_kexec.c
@@ -24,6 +24,8 @@
 #include <asm/delay.h>
 #include <asm/meminit.h>
 #include <asm/processor.h>
+#include <asm/sal.h>
+#include <asm/mca.h>
 
 typedef NORET_TYPE void (*relocate_new_kernel_t)(
 					unsigned long indirection_page,
@@ -47,6 +49,8 @@ struct resource boot_param_res = {
         .flags = IORESOURCE_BUSY | IORESOURCE_MEM
 };
 
+/* In mca_asm.S */
+extern void ia64_os_init_on_kdump(void);
 
 /*
  * Do what every setup is needed on image and the
@@ -85,11 +89,21 @@ static void ia64_machine_kexec(struct unw_frame_info *info, void *arg)
 	void *pal_addr = efi_get_pal_addr();
 	unsigned long code_addr = (unsigned long)page_address(image->control_code_page);
 	int ii;
+	u64 fp, gp;
+	ia64_fptr_t *init_handler = (ia64_fptr_t *)ia64_os_init_on_kdump;
 
 	BUG_ON(!image);
 	if (image->type == KEXEC_TYPE_CRASH) {
 		crash_save_this_cpu();
 		current->thread.ksp = (__u64)info->sw - 16;
+
+		/* Register noop init handler */
+		fp = ia64_tpa(init_handler->fp);
+		gp = ia64_tpa(ia64_getreg(_IA64_REG_GP));
+		ia64_sal_set_vectors(SAL_VECTOR_OS_INIT, fp, gp, 0, fp, gp, 0);
+	} else {
+		/* Unregister init handlers of current kernel */
+		ia64_sal_set_vectors(SAL_VECTOR_OS_INIT, 0, 0, 0, 0, 0, 0);
 	}
 
 	/* Interrupts aren't acceptable while we reboot */
diff --git a/arch/ia64/kernel/mca_asm.S b/arch/ia64/kernel/mca_asm.S
index c6ee089..0594c54 100644
--- a/arch/ia64/kernel/mca_asm.S
+++ b/arch/ia64/kernel/mca_asm.S
@@ -40,6 +40,7 @@
 
 	.global ia64_do_tlb_purge
 	.global ia64_os_mca_dispatch
+	.global ia64_os_init_on_kdump
 	.global ia64_os_init_dispatch_monarch
 	.global ia64_os_init_dispatch_slave
 
@@ -299,6 +300,25 @@ END(ia64_os_mca_virtual_begin)
 //StartMain////////////////////////////////////////////////////////////////////
 
 //
+// NOP init handler for kdump.  In panic situation, we may receive INIT
+// while kernel transition.  Since we initialize registers on leave from
+// current kernel, no longer monarch/slave handlers of current kernel in
+// virtual mode are called safely.
+// We can unregister these init handlers from SAL, however then the INIT
+// will result in warmboot by SAL and we cannot retrieve the crashdump.
+// Therefore register this NOP function to SAL, to prevent entering virtual
+// mode and resulting warmboot by SAL.
+//
+ia64_os_init_on_kdump:
+	mov		r8=r0		// IA64_INIT_RESUME
+	mov             r9=r10		// SAL_GP
+	mov		r22=r17		// *minstate
+	;;
+	mov		r10=r0		// return to same context
+	mov		b0=r12		// SAL_CHECK return address
+	br		b0
+
+//
 // SAL to OS entry point for INIT on all processors.  This has been defined for
 // registration purposes with SAL as a part of ia64_mca_init.  Monarch and
 // slave INIT have identical processing, except for the value of the
-- 
1.6.0

[PATCH 3/7] ia64, kexec: Unregister MCA handler before kexec

[Hidetoshi Seto]

We cannot handle machine check errors during kernel transition.
To avoid entering MCA handler from early stage of new kernel, unregister
the entry point from SAL before leave from current kernel.
Then SAL will make all MCAs to warmboot safely.

Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Cc: Vivek Goyal <vgoyal@redhat.com>
Cc: Haren Myneni <hbabu@us.ibm.com>
Cc: kexec@lists.infradead.org
---
 arch/ia64/kernel/machine_kexec.c |    3 +++
 1 files changed, 3 insertions(+), 0 deletions(-)

diff --git a/arch/ia64/kernel/machine_kexec.c b/arch/ia64/kernel/machine_kexec.c
index d35fc39..eb4f1cc 100644
--- a/arch/ia64/kernel/machine_kexec.c
+++ b/arch/ia64/kernel/machine_kexec.c
@@ -106,6 +106,9 @@ static void ia64_machine_kexec(struct unw_frame_info *info, void *arg)
 		ia64_sal_set_vectors(SAL_VECTOR_OS_INIT, 0, 0, 0, 0, 0, 0);
 	}
 
+	/* Unregister mca handler - No more recovery on current kernel */
+	ia64_sal_set_vectors(SAL_VECTOR_OS_MCA, 0, 0, 0, 0, 0, 0);
+
 	/* Interrupts aren't acceptable while we reboot */
 	local_irq_disable();
 
-- 
1.6.0

[PATCH 4/7] ia64, kdump: Don't offline APs

[Hidetoshi Seto]

INIT on AP going to be offline have a problem.

Since psr.mc is cleared when bits in psr are set to SAL_PSR_BITS_TO_SET
in ia64_jump_to_sal(), so there is a small window that the cpu can receive
INIT even if the cpu enter there via INIT handler.  In this window we do
restore of registers for SAL, so INIT asserted here will not work properly.

It is hard to remove this window by masking INIT (i.e. setting psr.mc)
because we have to unmask it later in OS, because we have to use branch
instruction (br.ret, not rfi) to return SAL, due to OS_BOOT_RENDEZ to SAL
return convention.

I suppose this window will not be a real problem on cpu offline if we can
educate people not to push INIT button during hotplug operation.  However
only exception is a race in kdump and INIT.  Now kdump returns APs to SAL
before processing dump, but the kernel might receive INIT at that point in
time.  Such INIT might be asserted by kdump itself if an AP doesn't react
IPI soon and kdump decided to use INIT to stop the AP.

Such panic+INIT or INIT+INIT cases should be rare, but it will be happy
if we can retrieve crashdump even in such cases.  So it will be better
to stop returning APs to SAL by kdump.

I confirmed that the kdump sometime hangs by concurrent INITs (another
INIT after an INIT), and it doesn't hang after applying this patch.

Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Cc: Vivek Goyal <vgoyal@redhat.com>
Cc: Haren Myneni <hbabu@us.ibm.com>
Cc: kexec@lists.infradead.org
---
 arch/ia64/kernel/crash.c |    4 ----
 1 files changed, 0 insertions(+), 4 deletions(-)

diff --git a/arch/ia64/kernel/crash.c b/arch/ia64/kernel/crash.c
index 48b69fd..eacedfc 100644
--- a/arch/ia64/kernel/crash.c
+++ b/arch/ia64/kernel/crash.c
@@ -142,10 +142,6 @@ kdump_cpu_freeze(struct unw_frame_info *info, void *arg)
 	atomic_inc(&kdump_cpu_frozen);
 	kdump_status[cpuid] = 1;
 	mb();
-#ifdef CONFIG_HOTPLUG_CPU
-	if (cpuid != 0)
-		ia64_jump_to_sal(&sal_boot_rendez_state[cpuid]);
-#endif
 	for (;;)
 		cpu_relax();
 }
-- 
1.6.0

[PATCH 5/7] ia64, kdump: Mask INIT first in panic-kdump path

[Hidetoshi Seto]

This is a fix for races on Panic and INIT.

Asserting INIT might not invoke 2nd kdump if the system is going to start
1st kdump via panic and some of cpus are already playing dead with INIT
masked.  It can be assumed that an internal agent decides to panic the
unstable system while another external agent decides to send an INIT to
the system at same time.

So mask INIT first in panic path to take the initiative on kdump.
All INITs later should be used only for freezing all other cpus.

Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Cc: Vivek Goyal <vgoyal@redhat.com>
Cc: Haren Myneni <hbabu@us.ibm.com>
Cc: kexec@lists.infradead.org
---
 arch/ia64/kernel/crash.c |   47 ++++++++++++++++++++++++++++++++++++++++-----
 1 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/arch/ia64/kernel/crash.c b/arch/ia64/kernel/crash.c
index eacedfc..1440445 100644
--- a/arch/ia64/kernel/crash.c
+++ b/arch/ia64/kernel/crash.c
@@ -23,6 +23,7 @@
 int kdump_status[NR_CPUS];
 static atomic_t kdump_cpu_frozen;
 atomic_t kdump_in_progress;
+static int kdump_freeze_monarch;
 static int kdump_on_init = 1;
 static int kdump_on_fatal_mca = 1;
 
@@ -110,6 +111,33 @@ machine_crash_shutdown(struct pt_regs *pt)
 	 */
 	kexec_disable_iosapic();
 #ifdef CONFIG_SMP
+	/*
+	 * If kdump_on_init is set and an INIT is assered here, kdump will
+	 * be started again via INIT monarch.
+	 */
+	local_irq_disable();
+	ia64_set_psr_mc();	/* mask MCA/INIT */
+	if (atomic_inc_return(&kdump_in_progress) != 1)
+		unw_init_running(kdump_cpu_freeze, NULL);		
+
+	/*
+	 * Now this cpu is ready for kdump.
+	 * Stop all others by IPI or INIT.  They could receive INIT from
+	 * outside and might be INIT monarch, but only thing they have to
+	 * do is falling into kdump_cpu_freeze().
+	 *
+	 * If an INIT is asserted here:
+	 * - All receivers might be slaves, since some of cpus could already
+	 *   be frozen and INIT might be masked on monarch.  In this case,
+	 *   all slaves will park in while (monarch_cpu == -1) loop before
+	 *   DIE_INIT_SLAVE_ENTER that for waiting monarch enters.
+	 *	=> TBD: freeze all slaves
+	 * - One might be a monarch, but INIT rendezvous will fail since
+	 *   at least this cpu already have INIT masked so it never join
+	 *   to the rendezvous.  In this case, all slaves and monarch will
+	 *   be frozen after timeout of the INIT rendezvous.
+	 *	=> TBD: freeze them without waiting timeout
+	 */
 	kdump_smp_send_stop();
 	/* not all cpu response to IPI, send INIT to freeze them */
 	if (kdump_wait_cpu_freeze() && kdump_on_init) 	{
@@ -179,13 +207,18 @@ kdump_init_notifier(struct notifier_block *self, unsigned long val, void *data)
 	switch (val) {
 	case DIE_INIT_MONARCH_PROCESS:
 		if (kdump_on_init) {
-			atomic_set(&kdump_in_progress, 1);
+			if (atomic_inc_return(&kdump_in_progress) != 1)
+				kdump_freeze_monarch = 1;
 			*(nd->monarch_cpu) = -1;
 		}
 		break;
 	case DIE_INIT_MONARCH_LEAVE:
-		if (kdump_on_init)
-			machine_kdump_on_init();
+		if (kdump_on_init) {
+			if (kdump_freeze_monarch)
+				unw_init_running(kdump_cpu_freeze, NULL);
+			else
+				machine_kdump_on_init();
+		}
 		break;
 	case DIE_INIT_SLAVE_LEAVE:
 		if (atomic_read(&kdump_in_progress))
@@ -198,9 +231,11 @@ kdump_init_notifier(struct notifier_block *self, unsigned long val, void *data)
 	case DIE_MCA_MONARCH_LEAVE:
 		/* *(nd->data) indicate if MCA is recoverable */
 		if (kdump_on_fatal_mca && !(*(nd->data))) {
-			atomic_set(&kdump_in_progress, 1);
-			*(nd->monarch_cpu) = -1;
-			machine_kdump_on_init();
+			if (atomic_inc_return(&kdump_in_progress) == 1) {
+				*(nd->monarch_cpu) = -1;
+				machine_kdump_on_init();
+			}
+			/* We got fatal MCA while kdump!? No way!! */
 		}
 		break;
 	}
-- 
1.6.0

[PATCH 6/7] ia64, kdump: Try INIT regardless of kdump_on_init

[Hidetoshi Seto]

CPUs should be frozen if possible, otherwise it might hinder kdump.
So if there are CPUs not respond to IPI, try INIT to stop them.

Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Cc: Vivek Goyal <vgoyal@redhat.com>
Cc: Haren Myneni <hbabu@us.ibm.com>
Cc: kexec@lists.infradead.org
---
 arch/ia64/kernel/crash.c |   43 +++++++++++++++++++++----------------------
 1 files changed, 21 insertions(+), 22 deletions(-)

diff --git a/arch/ia64/kernel/crash.c b/arch/ia64/kernel/crash.c
index 1440445..f43eccf 100644
--- a/arch/ia64/kernel/crash.c
+++ b/arch/ia64/kernel/crash.c
@@ -140,8 +140,10 @@ machine_crash_shutdown(struct pt_regs *pt)
 	 */
 	kdump_smp_send_stop();
 	/* not all cpu response to IPI, send INIT to freeze them */
-	if (kdump_wait_cpu_freeze() && kdump_on_init) 	{
+	if (kdump_wait_cpu_freeze()) 	{
 		kdump_smp_send_init();
+		/* wait again, don't go ahead if possible */
+		kdump_wait_cpu_freeze();
 	}
 #endif
 }
@@ -180,6 +182,19 @@ kdump_init_notifier(struct notifier_block *self, unsigned long val, void *data)
 	struct ia64_mca_notify_die *nd;
 	struct die_args *args = data;
 
+	if (atomic_read(&kdump_in_progress)) {
+		switch (val) {
+		case DIE_INIT_MONARCH_LEAVE:
+			if (!kdump_freeze_monarch)
+				break;
+			/* fall through */
+		case DIE_INIT_SLAVE_LEAVE:
+		case DIE_MCA_RENDZVOUS_LEAVE:
+			unw_init_running(kdump_cpu_freeze, NULL);
+			break;
+		}
+	}
+
 	if (!kdump_on_init && !kdump_on_fatal_mca)
 		return NOTIFY_DONE;
 
@@ -192,41 +207,25 @@ kdump_init_notifier(struct notifier_block *self, unsigned long val, void *data)
 	}
 
 	if (val != DIE_INIT_MONARCH_LEAVE &&
-	    val != DIE_INIT_SLAVE_LEAVE &&
 	    val != DIE_INIT_MONARCH_PROCESS &&
-	    val != DIE_MCA_RENDZVOUS_LEAVE &&
 	    val != DIE_MCA_MONARCH_LEAVE)
 		return NOTIFY_DONE;
 
 	nd = (struct ia64_mca_notify_die *)args->err;
-	/* Reason code 1 means machine check rendezvous*/
-	if ((val == DIE_INIT_MONARCH_LEAVE || val == DIE_INIT_SLAVE_LEAVE
-	    || val == DIE_INIT_MONARCH_PROCESS) && nd->sos->rv_rc == 1)
-		return NOTIFY_DONE;
 
 	switch (val) {
 	case DIE_INIT_MONARCH_PROCESS:
-		if (kdump_on_init) {
+		/* Reason code 1 means machine check rendezvous*/
+		if (kdump_on_init && (nd->sos->rv_rc != 1)) {
 			if (atomic_inc_return(&kdump_in_progress) != 1)
 				kdump_freeze_monarch = 1;
 			*(nd->monarch_cpu) = -1;
 		}
 		break;
 	case DIE_INIT_MONARCH_LEAVE:
-		if (kdump_on_init) {
-			if (kdump_freeze_monarch)
-				unw_init_running(kdump_cpu_freeze, NULL);
-			else
-				machine_kdump_on_init();
-		}
-		break;
-	case DIE_INIT_SLAVE_LEAVE:
-		if (atomic_read(&kdump_in_progress))
-			unw_init_running(kdump_cpu_freeze, NULL);
-		break;
-	case DIE_MCA_RENDZVOUS_LEAVE:
-		if (atomic_read(&kdump_in_progress))
-			unw_init_running(kdump_cpu_freeze, NULL);
+		/* Reason code 1 means machine check rendezvous*/
+		if (kdump_on_init && (nd->sos->rv_rc != 1))
+			machine_kdump_on_init();
 		break;
 	case DIE_MCA_MONARCH_LEAVE:
 		/* *(nd->data) indicate if MCA is recoverable */
-- 
1.6.0

[PATCH 7/7] ia64, kdump: Short path to freeze CPUs

[Hidetoshi Seto]

Setting monarch_cpu = -1 to let slaves frozen might not work, because
there might be slaves being late, not entered the rendezvous yet.
Such slaves might be caught in while (monarch_cpu == -1) loop.

Use kdump_in_progress instead of monarch_cpus to break INIT rendezvous
and let all slaves enter DIE_INIT_SLAVE_LEAVE smoothly.

And monarch no longer need to manage rendezvous if once kdump_in_progress
is set, catch the monarch in DIE_INIT_MONARCH_ENTER then.

Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Cc: Vivek Goyal <vgoyal@redhat.com>
Cc: Haren Myneni <hbabu@us.ibm.com>
Cc: kexec@lists.infradead.org
---
 arch/ia64/kernel/crash.c |   15 ++++++---------
 arch/ia64/kernel/mca.c   |   15 +++++++++++++--
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/arch/ia64/kernel/crash.c b/arch/ia64/kernel/crash.c
index f43eccf..cf833af 100644
--- a/arch/ia64/kernel/crash.c
+++ b/arch/ia64/kernel/crash.c
@@ -129,14 +129,13 @@ machine_crash_shutdown(struct pt_regs *pt)
 	 * If an INIT is asserted here:
 	 * - All receivers might be slaves, since some of cpus could already
 	 *   be frozen and INIT might be masked on monarch.  In this case,
-	 *   all slaves will park in while (monarch_cpu == -1) loop before
-	 *   DIE_INIT_SLAVE_ENTER that for waiting monarch enters.
-	 *	=> TBD: freeze all slaves
+	 *   all slaves will be frozen soon since kdump_in_progress will let
+	 *   them into DIE_INIT_SLAVE_LEAVE.
 	 * - One might be a monarch, but INIT rendezvous will fail since
 	 *   at least this cpu already have INIT masked so it never join
 	 *   to the rendezvous.  In this case, all slaves and monarch will
-	 *   be frozen after timeout of the INIT rendezvous.
-	 *	=> TBD: freeze them without waiting timeout
+	 *   be frozen soon with no wait since the INIT rendezvous is skipped
+	 *   by kdump_in_progress.
 	 */
 	kdump_smp_send_stop();
 	/* not all cpu response to IPI, send INIT to freeze them */
@@ -189,6 +188,7 @@ kdump_init_notifier(struct notifier_block *self, unsigned long val, void *data)
 				break;
 			/* fall through */
 		case DIE_INIT_SLAVE_LEAVE:
+		case DIE_INIT_MONARCH_ENTER:
 		case DIE_MCA_RENDZVOUS_LEAVE:
 			unw_init_running(kdump_cpu_freeze, NULL);
 			break;
@@ -219,7 +219,6 @@ kdump_init_notifier(struct notifier_block *self, unsigned long val, void *data)
 		if (kdump_on_init && (nd->sos->rv_rc != 1)) {
 			if (atomic_inc_return(&kdump_in_progress) != 1)
 				kdump_freeze_monarch = 1;
-			*(nd->monarch_cpu) = -1;
 		}
 		break;
 	case DIE_INIT_MONARCH_LEAVE:
@@ -230,10 +229,8 @@ kdump_init_notifier(struct notifier_block *self, unsigned long val, void *data)
 	case DIE_MCA_MONARCH_LEAVE:
 		/* *(nd->data) indicate if MCA is recoverable */
 		if (kdump_on_fatal_mca && !(*(nd->data))) {
-			if (atomic_inc_return(&kdump_in_progress) == 1) {
-				*(nd->monarch_cpu) = -1;
+			if (atomic_inc_return(&kdump_in_progress) == 1)
 				machine_kdump_on_init();
-			}
 			/* We got fatal MCA while kdump!? No way!! */
 		}
 		break;
diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c
index 8f33a88..7a34c6c 100644
--- a/arch/ia64/kernel/mca.c
+++ b/arch/ia64/kernel/mca.c
@@ -1682,14 +1682,25 @@ ia64_init_handler(struct pt_regs *regs, struct switch_stack *sw,
 
 	if (!sos->monarch) {
 		ia64_mc_info.imi_rendez_checkin[cpu] = IA64_MCA_RENDEZ_CHECKIN_INIT;
+
+#ifdef CONFIG_KEXEC
+		while (monarch_cpu == -1 && !atomic_read(&kdump_in_progress))
+			udelay(1000);
+#else
 		while (monarch_cpu == -1)
-		       cpu_relax();	/* spin until monarch enters */
+			cpu_relax();	/* spin until monarch enters */
+#endif
 
 		NOTIFY_INIT(DIE_INIT_SLAVE_ENTER, regs, (long)&nd, 1);
 		NOTIFY_INIT(DIE_INIT_SLAVE_PROCESS, regs, (long)&nd, 1);
 
+#ifdef CONFIG_KEXEC
+		while (monarch_cpu != -1 && !atomic_read(&kdump_in_progress))
+			udelay(1000);
+#else
 		while (monarch_cpu != -1)
-		       cpu_relax();	/* spin until monarch leaves */
+			cpu_relax();	/* spin until monarch leaves */
+#endif
 
 		NOTIFY_INIT(DIE_INIT_SLAVE_LEAVE, regs, (long)&nd, 1);
 
-- 
1.6.0

Re: [PATCH 0/7] Patches for kdump vs. INIT

[Jay Lan]

Kdump with INIT on IPF worked on SGI's IA64 servers when i left SGI last 
December.
Is this problem something new to you after .27?

SGI folks should test this patchset to ensure no surprise is introduced to 
their servers.

Cheers,
jay

----- Original Message ----- 
From: "Hidetoshi Seto" <seto.hidetoshi@jp.fujitsu.com>
To: <linux-ia64@vger.kernel.org>; <linux-kernel@vger.kernel.org>
Cc: "Haren Myneni" <hbabu@us.ibm.com>; <kexec@lists.infradead.org>; "Vivek 
Goyal" <vgoyal@redhat.com>
Sent: Wednesday, June 17, 2009 11:44 PM
Subject: [PATCH 0/7] Patches for kdump vs. INIT


> Hi Tony-san, and kdump folks,
>
> I got some trouble on kdump on IPF with INIT, and my investigation
> proves there are some bugs and races in startup of kdump.
> Here are fixes based on .30, for issues I found.
>
> Since it must be serious problem for (likely big) IPF servers if we
> could fail to retrieve crashdump via kdump, so I believe these patches
> should be applied asap.
>
>
> Thanks,
> H.Seto
>
> ===
>
> Hidetoshi Seto (7):
>      ia64, kdump: Mask MCA/INIT on freezing cpus
>      ia64, kexec: Make INIT safe while kdump/kexec
>      ia64, kexec: Unregister MCA handler before kexec
>      ia64, kdump: Don't offline APs
>      ia64, kdump: Mask INIT first in panic-kdump path
>      ia64, kdump: Try INIT regardless of kdump_on_init
>      ia64, kdump: Short path to freeze CPUs
>
> arch/ia64/kernel/crash.c         |   85 
> ++++++++++++++++++++++++++------------
> arch/ia64/kernel/machine_kexec.c |   17 ++++++++
> arch/ia64/kernel/mca.c           |   15 ++++++-
> arch/ia64/kernel/mca_asm.S       |   47 +++++++++++++++++++++
> 4 files changed, 136 insertions(+), 28 deletions(-)
>
>
>
> _______________________________________________
> kexec mailing list
> kexec@lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/kexec
> 

Re: [PATCH 0/7] Patches for kdump vs. INIT

[Hidetoshi Seto]

Jay Lan wrote:
> Kdump with INIT on IPF worked on SGI's IA64 servers when i left SGI last
> December.
> Is this problem something new to you after .27?

No.
This is a potential bug in design, in case of INIT+INIT and panic+INIT. 

> SGI folks should test this patchset to ensure no surprise is introduced
> to their servers.

Any tests and comments would be appreciated.


Thanks,
H.Seto

[PATCH v2 0/7] Patches for kdump vs. INIT

[Hidetoshi Seto]

Hi all,

Here is the updated version:
 - Better descriptions with justifiable reasons
 - Add comments to point where the mask is removed
 - Move extern to header file
 - Minor fix for style, typo
 - No changes in logic

Thanks,
H.Seto

===

Hidetoshi Seto (7):
      ia64, kdump: Mask MCA/INIT on frozen cpus
      ia64, kexec: Make INIT safe while transition to kdump/kexec kernel
      ia64, kexec: Unregister MCA handler before kexec
      ia64, kdump: Don't return APs to SAL from kdump
      ia64, kdump: Mask INIT first in panic-kdump path
      ia64, kdump: Try INIT regardless of kdump_on_init
      ia64, kdump: Short path to freeze CPUs

 arch/ia64/include/asm/mca.h        |    2 +
 arch/ia64/kernel/crash.c           |   83 ++++++++++++++++++++++++-----------
 arch/ia64/kernel/head.S            |    2 +-
 arch/ia64/kernel/machine_kexec.c   |   15 ++++++
 arch/ia64/kernel/mca.c             |   15 ++++++-
 arch/ia64/kernel/mca_asm.S         |   47 ++++++++++++++++++++
 arch/ia64/kernel/relocate_kernel.S |    2 +-
 7 files changed, 136 insertions(+), 30 deletions(-)

Hidetoshi Seto wrote:
> Hi Tony-san, and kdump folks,
> 
> I got some trouble on kdump on IPF with INIT, and my investigation
> proves there are some bugs and races in startup of kdump.
> Here are fixes based on .30, for issues I found.
> 
> Since it must be serious problem for (likely big) IPF servers if we
> could fail to retrieve crashdump via kdump, so I believe these patches
> should be applied asap.
> 
> 
> Thanks,
> H.Seto
> 
> ===
> 
> Hidetoshi Seto (7):
>       ia64, kdump: Mask MCA/INIT on freezing cpus
>       ia64, kexec: Make INIT safe while kdump/kexec
>       ia64, kexec: Unregister MCA handler before kexec
>       ia64, kdump: Don't offline APs
>       ia64, kdump: Mask INIT first in panic-kdump path
>       ia64, kdump: Try INIT regardless of kdump_on_init
>       ia64, kdump: Short path to freeze CPUs
> 
>  arch/ia64/kernel/crash.c         |   85 ++++++++++++++++++++++++++------------
>  arch/ia64/kernel/machine_kexec.c |   17 ++++++++
>  arch/ia64/kernel/mca.c           |   15 ++++++-
>  arch/ia64/kernel/mca_asm.S       |   47 +++++++++++++++++++++
>  4 files changed, 136 insertions(+), 28 deletions(-)

[PATCH v2 1/7] ia64, kdump: Mask MCA/INIT on frozen cpus

[Hidetoshi Seto]

Summary:

  INIT asserted on kdump kernel invokes INIT handler not only on a
  cpu that running on the kdump kernel, but also BSP of the panicked
  kernel, because the (badly) frozen BSP can be thawed by INIT.

Description:

  The kdump_cpu_freeze() is called on cpus except one that initiates
  panic and/or kdump, to stop/offline the cpu (on ia64, it means we
  pass control of cpus to SAL, or put them in spinloop).  Note that
  CPU0(BSP) always go to spinloop, so if panic was happened on an AP,
  there are at least 2cpus (= the AP and BSP) which not back to SAL.

  On the spinning cpus, interrupts are disabled (rsm psr.i), but INIT
  is still interruptible because psr.mc for mask them is not set unless
  kdump_cpu_freeze() is not called from MCA/INIT context.

  Therefore, assume that a panic was happened on an AP, kdump was
  invoked, new INIT handlers for kdump kernel was registered and then
  an INIT is asserted.  From the viewpoint of SAL, there are 2 online
  cpus, so INIT will be delivered to both of them.  It likely means
  that not only the AP (= a cpu executing kdump) enters INIT handler
  which is newly registered, but also BSP (= another cpu spinning in
  panicked kernel) enters the same INIT handler.  Of course setting of
  registers in BSP are still old (for panicked kernel), so what happen
  with running handler with wrong setting will be extremely unexpected.
  I believe this is not desirable behavior.

How to Reproduce:

  Start kdump on one of APs (e.g. cpu1)
    # taskset 0x2 echo c > /proc/sysrq-trigger
  Then assert INIT after kdump kernel is booted, after new INIT handler
  for kdump kernel is registered.

Expected results:

  An INIT handler is invoked only on the AP.

Actual results:

  An INIT handler is invoked on the AP and BSP.

Sample of results:

  I got following console log by asserting INIT after prompt "root:/>".
  It seems that two monarchs appeared by one INIT, and one panicked at
  last.  And it also seems that the panicked one supposed there were
  4 online cpus and no one did rendezvous:

    :
    [  0 %]dropping to initramfs shell
    exiting this shell will reboot your system
    root:/> Entered OS INIT handler. PSP=fff301a0 cpu=0 monarch=0
    ia64_init_handler: Promoting cpu 0 to monarch.
    Delaying for 5 seconds...
    All OS INIT slaves have reached rendezvous
    Processes interrupted by INIT - 0 (cpu 0 task 0xa000000100af0000)
    :
    <<snip>>
    :
    Entered OS INIT handler. PSP=fff301a0 cpu=0 monarch=1
    Delaying for 5 seconds...
    mlogbuf_finish: printing switched to urgent mode, MCA/INIT might be dodgy or fail.
    OS INIT slave did not rendezvous on cpu 1 2 3
    INIT swapper 0[0]: bugcheck! 0 [1]
    :
    <<snip>>
    :
    Kernel panic - not syncing: Attempted to kill the idle task!

Proposed fix:

  To avoid this problem, this patch inserts ia64_set_psr_mc() to mask
  INIT on cpus going to be frozen.  This masking have no effect if the
  kdump_cpu_freeze() is called from INIT handler when kdump_on_init == 1,
  because psr.mc is already turned on to 1 before entering OS_INIT.
  I confirmed that weird log like above are disappeared after applying
  this patch.

Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Cc: Vivek Goyal <vgoyal@redhat.com>
Cc: Haren Myneni <hbabu@us.ibm.com>
Cc: kexec@lists.infradead.org
---
 arch/ia64/include/asm/mca.h |    1 +
 arch/ia64/kernel/crash.c    |    4 ++++
 arch/ia64/kernel/head.S     |    2 +-
 arch/ia64/kernel/mca_asm.S  |   27 +++++++++++++++++++++++++++
 4 files changed, 33 insertions(+), 1 deletions(-)

diff --git a/arch/ia64/include/asm/mca.h b/arch/ia64/include/asm/mca.h
index 44a0b53..cb0952f 100644
--- a/arch/ia64/include/asm/mca.h
+++ b/arch/ia64/include/asm/mca.h
@@ -151,6 +151,7 @@ extern void ia64_mca_cmc_vector_setup(void);
 extern int  ia64_reg_MCA_extension(int (*fn)(void *, struct ia64_sal_os_state *));
 extern void ia64_unreg_MCA_extension(void);
 extern unsigned long ia64_get_rnat(unsigned long *);
+extern void ia64_set_psr_mc(void);
 extern void ia64_mca_printk(const char * fmt, ...)
 	 __attribute__ ((format (printf, 1, 2)));
 
diff --git a/arch/ia64/kernel/crash.c b/arch/ia64/kernel/crash.c
index f065093..3f3a579 100644
--- a/arch/ia64/kernel/crash.c
+++ b/arch/ia64/kernel/crash.c
@@ -129,10 +129,14 @@ void
 kdump_cpu_freeze(struct unw_frame_info *info, void *arg)
 {
 	int cpuid;
+
 	local_irq_disable();
 	cpuid = smp_processor_id();
 	crash_save_this_cpu();
 	current->thread.ksp = (__u64)info->sw - 16;
+
+	ia64_set_psr_mc();	/* mask MCA/INIT and stop reentrance */
+
 	atomic_inc(&kdump_cpu_frozen);
 	kdump_status[cpuid] = 1;
 	mb();
diff --git a/arch/ia64/kernel/head.S b/arch/ia64/kernel/head.S
index 23f846d..e1f97ac 100644
--- a/arch/ia64/kernel/head.S
+++ b/arch/ia64/kernel/head.S
@@ -1242,7 +1242,7 @@ GLOBAL_ENTRY(ia64_jump_to_sal)
 	movl r16=SAL_PSR_BITS_TO_SET;;
 	mov cr.ipsr=r16
 	mov cr.ifs=r0;;
-	rfi;;
+	rfi;;			// note: this unmask MCA/INIT (psr.mc)
 1:
 	/*
 	 * Invalidate all TLB data/inst
diff --git a/arch/ia64/kernel/mca_asm.S b/arch/ia64/kernel/mca_asm.S
index a06d465..8d2eabe 100644
--- a/arch/ia64/kernel/mca_asm.S
+++ b/arch/ia64/kernel/mca_asm.S
@@ -1073,3 +1073,30 @@ GLOBAL_ENTRY(ia64_get_rnat)
 	mov ar.rsc=3
 	br.ret.sptk.many rp
 END(ia64_get_rnat)
+
+
+// void ia64_set_psr_mc(void)
+//
+// Set psr.mc bit to mask MCA/INIT.
+GLOBAL_ENTRY(ia64_set_psr_mc)
+	rsm psr.i | psr.ic		// disable interrupts
+	;;
+	srlz.d
+	;;
+	mov r14 = psr			// get psr{36:35,31:0}
+	movl r15 = 1f
+	;;
+	dep r14 = -1, r14, PSR_MC, 1	// set psr.mc
+	;;
+	dep r14 = -1, r14, PSR_IC, 1	// set psr.ic
+	;;
+	dep r14 = -1, r14, PSR_BN, 1	// keep bank1 in use
+	;;
+	mov cr.ipsr = r14
+	mov cr.ifs = r0
+	mov cr.iip = r15
+	;;
+	rfi
+1:
+	br.ret.sptk.many rp
+END(ia64_set_psr_mc)
-- 
1.6.0

[PATCH v2 2/7] ia64, kexec: Make INIT safe while transition to kdump/kexec kernel

[Hidetoshi Seto]

Summary:

  Asserting INIT on the beginning of kdump/kexec kernel will result
  in unexpected behavior because INIT handler for previous kernel is
  invoked on new kernel.

Description:

  In panic situation, we can receive INIT while kernel transition,
  i.e. from beginning of panic to bootstrap of kdump kernel.
  Since we initialize registers on leave from current kernel, no
  longer monarch/slave handlers of current kernel in virtual mode are
  called safely.  (In fact system goes hang as far as I confirmed)

How to Reproduce:

  Start kdump
    # echo c > /proc/sysrq-trigger
  Then assert INIT while kdump kernel is booting, before new INIT
  handler for kdump kernel is registered.

Expected(Desirable) result:

  kdump kernel boots without any problem, crashdump retrieved

Actual result:

  INIT handler for previous kernel is invoked on kdump kernel
  => panic, hang etc. (unexpected)

Proposed fix:

  We can unregister these init handlers from SAL before jumping into
  new kernel, however then the INIT will fallback to default behavior,
  result in warmboot by SAL (according to the SAL specification) and
  we cannot retrieve the crashdump.

  Therefore this patch introduces a NOP init handler and register it
  to SAL before leave from current kernel, to start kdump safely by
  preventing INITs from entering virtual mode and resulting in warmboot.

  On the other hand, in case of kexec that not for kdump, it also
  has same problem with INIT while kernel transition.
  This patch handles this case differently, because for kexec
  unregistering handlers will be preferred than registering NOP
  handler, since the situation "no handlers registered" is usual
  state for kernel's entry.

Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Cc: Vivek Goyal <vgoyal@redhat.com>
Cc: Haren Myneni <hbabu@us.ibm.com>
Cc: kexec@lists.infradead.org
---
 arch/ia64/include/asm/mca.h      |    1 +
 arch/ia64/kernel/machine_kexec.c |   12 ++++++++++++
 arch/ia64/kernel/mca_asm.S       |   20 ++++++++++++++++++++
 3 files changed, 33 insertions(+), 0 deletions(-)

diff --git a/arch/ia64/include/asm/mca.h b/arch/ia64/include/asm/mca.h
index cb0952f..c171cdf 100644
--- a/arch/ia64/include/asm/mca.h
+++ b/arch/ia64/include/asm/mca.h
@@ -145,6 +145,7 @@ extern void ia64_mca_ucmc_handler(struct pt_regs *, struct ia64_sal_os_state *);
 extern void ia64_init_handler(struct pt_regs *,
 			      struct switch_stack *,
 			      struct ia64_sal_os_state *);
+extern void ia64_os_init_on_kdump(void);
 extern void ia64_monarch_init_handler(void);
 extern void ia64_slave_init_handler(void);
 extern void ia64_mca_cmc_vector_setup(void);
diff --git a/arch/ia64/kernel/machine_kexec.c b/arch/ia64/kernel/machine_kexec.c
index 0823de1..571d663 100644
--- a/arch/ia64/kernel/machine_kexec.c
+++ b/arch/ia64/kernel/machine_kexec.c
@@ -24,6 +24,8 @@
 #include <asm/delay.h>
 #include <asm/meminit.h>
 #include <asm/processor.h>
+#include <asm/sal.h>
+#include <asm/mca.h>
 
 typedef NORET_TYPE void (*relocate_new_kernel_t)(
 					unsigned long indirection_page,
@@ -85,11 +87,21 @@ static void ia64_machine_kexec(struct unw_frame_info *info, void *arg)
 	void *pal_addr = efi_get_pal_addr();
 	unsigned long code_addr = (unsigned long)page_address(image->control_code_page);
 	int ii;
+	u64 fp, gp;
+	ia64_fptr_t *init_handler = (ia64_fptr_t *)ia64_os_init_on_kdump;
 
 	BUG_ON(!image);
 	if (image->type == KEXEC_TYPE_CRASH) {
 		crash_save_this_cpu();
 		current->thread.ksp = (__u64)info->sw - 16;
+
+		/* Register noop init handler */
+		fp = ia64_tpa(init_handler->fp);
+		gp = ia64_tpa(ia64_getreg(_IA64_REG_GP));
+		ia64_sal_set_vectors(SAL_VECTOR_OS_INIT, fp, gp, 0, fp, gp, 0);
+	} else {
+		/* Unregister init handlers of current kernel */
+		ia64_sal_set_vectors(SAL_VECTOR_OS_INIT, 0, 0, 0, 0, 0, 0);
 	}
 
 	/* Interrupts aren't acceptable while we reboot */
diff --git a/arch/ia64/kernel/mca_asm.S b/arch/ia64/kernel/mca_asm.S
index 8d2eabe..7461d25 100644
--- a/arch/ia64/kernel/mca_asm.S
+++ b/arch/ia64/kernel/mca_asm.S
@@ -40,6 +40,7 @@
 
 	.global ia64_do_tlb_purge
 	.global ia64_os_mca_dispatch
+	.global ia64_os_init_on_kdump
 	.global ia64_os_init_dispatch_monarch
 	.global ia64_os_init_dispatch_slave
 
@@ -299,6 +300,25 @@ END(ia64_os_mca_virtual_begin)
 //StartMain////////////////////////////////////////////////////////////////////
 
 //
+// NOP init handler for kdump.  In panic situation, we may receive INIT
+// while kernel transition.  Since we initialize registers on leave from
+// current kernel, no longer monarch/slave handlers of current kernel in
+// virtual mode are called safely.
+// We can unregister these init handlers from SAL, however then the INIT
+// will result in warmboot by SAL and we cannot retrieve the crashdump.
+// Therefore register this NOP function to SAL, to prevent entering virtual
+// mode and resulting warmboot by SAL.
+//
+ia64_os_init_on_kdump:
+	mov		r8=r0		// IA64_INIT_RESUME
+	mov             r9=r10		// SAL_GP
+	mov		r22=r17		// *minstate
+	;;
+	mov		r10=r0		// return to same context
+	mov		b0=r12		// SAL_CHECK return address
+	br		b0
+
+//
 // SAL to OS entry point for INIT on all processors.  This has been defined for
 // registration purposes with SAL as a part of ia64_mca_init.  Monarch and
 // slave INIT have identical processing, except for the value of the
-- 
1.6.0

[PATCH v2 3/7] ia64, kexec: Unregister MCA handler before kexec

[Hidetoshi Seto]

Summary:

  MCA on the beginning of kdump/kexec kernel will result in unexpected
  behavior because MCA handler for previous kernel is invoked on the
  kdump kernel.

Description:

  Once a cpu is passed to new kernel, all resources in previous kernel
  should not be used from the cpu.  Even the resources for MCA handler
  are no exception.  So we cannot handle MCAs and its machine check
  errors during kernel transition, until new handler for new kernel is
  registered with new resources ready for handling the MCA.

How to reproduce:

  Assert MCA while kdump kernel is booting, before new MCA handler for
  kdump kernel is registered.

Expected(Desirable) results:

  No recovery, cancel kdump and reboot the system.

Actual results:

  MCA handler for previous kernel is invoked on the kdump kernel.
  => panic, hang etc. (unexpected)

Proposed fix:

  To avoid entering MCA handler from early stage of new kernel,
  unregister the entry point from SAL before leave from current
  kernel.  Then SAL will make all MCAs to warmboot safely, without
  invoking OS_MCA.

Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Cc: Vivek Goyal <vgoyal@redhat.com>
Cc: Haren Myneni <hbabu@us.ibm.com>
Cc: kexec@lists.infradead.org
---
 arch/ia64/kernel/machine_kexec.c |    3 +++
 1 files changed, 3 insertions(+), 0 deletions(-)

diff --git a/arch/ia64/kernel/machine_kexec.c b/arch/ia64/kernel/machine_kexec.c
index 571d663..3d3aeef 100644
--- a/arch/ia64/kernel/machine_kexec.c
+++ b/arch/ia64/kernel/machine_kexec.c
@@ -104,6 +104,9 @@ static void ia64_machine_kexec(struct unw_frame_info *info, void *arg)
 		ia64_sal_set_vectors(SAL_VECTOR_OS_INIT, 0, 0, 0, 0, 0, 0);
 	}
 
+	/* Unregister mca handler - No more recovery on current kernel */
+	ia64_sal_set_vectors(SAL_VECTOR_OS_MCA, 0, 0, 0, 0, 0, 0);
+
 	/* Interrupts aren't acceptable while we reboot */
 	local_irq_disable();
 
-- 
1.6.0

[PATCH v2 4/7] ia64, kdump: Don't return APs to SAL from kdump

[Hidetoshi Seto]

Summary:

  Asserting INIT on cpu going to be offline will result in unexpected
  behavior.  It will be a real problem in kdump cases where INIT might
  be asserted to unstable APs going to be offline by returning to SAL.

Description:

  Since psr.mc is cleared when bits in psr are set to SAL_PSR_BITS_TO_SET
  in ia64_jump_to_sal(), there is a small window (~few msecs) that the
  cpu can receive INIT even if the cpu enter there via INIT handler.
  In this window we do restore of registers for SAL, so INIT asserted
  here will not work properly.

  It is hard to remove this window by masking INIT (i.e. setting psr.mc)
  because we have to unmask it later in OS, because we have to use branch
  instruction (br.ret, not rfi) to return SAL, due to OS_BOOT_RENDEZ to
  SAL return convention.

  I suppose this window will not be a real problem on cpu offline if we
  can educate people not to push INIT button during hotplug operation.
  However, only exception is a race in kdump and INIT.  Now kdump returns
  APs to SAL before processing dump, but the kernel might receive INIT at
  that point in time.  Such INIT might be asserted by kdump itself if an
  AP doesn't react IPI soon and kdump decided to use INIT to stop the AP.
  Or it might be asserted by operator or an external agent to start dump
  on the unstable system.

  Such panic+INIT or INIT+INIT cases should be rare, but it will be happy
  if we can retrieve crashdump even in such cases.

How to reproduce:

  panic+INIT or INIT+INIT, with kdump configured

Expected results:

  crashdump is retrieved anyway

Actual results:

  panic, hang etc. (unexpected)

Proposed fix

  To avoid the window on the way to SAL, this patch stops returning APs
  to SAL in case of kdump.  In other words, this patch makes APs spin
  in OS instead of spinning in SAL.

  (* Note: What impact would be there?  If a cpu is spinning in SAL,
   the cpu is in BOOT_RENDEZ loop, as same as offlined cpu.
   In theory if an INIT is asserted there, cpus in the BOOT_RENDEZ loop
   should not invoke OS_INIT on it.  So in either way, no matter where
   the cpu is spinning actually in, once cpu starts spin and act as
   "frozen," INIT on the cpu have no effects.
   From another point of view, all debug information on the cpu should
   have stored to memory before the cpu start to be frozen.  So no more
   action on the cpu is required.)

  I confirmed that the kdump sometime hangs by concurrent INITs (another
  INIT after an INIT), and it doesn't hang after applying this patch.

Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Cc: Vivek Goyal <vgoyal@redhat.com>
Cc: Haren Myneni <hbabu@us.ibm.com>
Cc: kexec@lists.infradead.org
---
 arch/ia64/kernel/crash.c |    4 ----
 1 files changed, 0 insertions(+), 4 deletions(-)

diff --git a/arch/ia64/kernel/crash.c b/arch/ia64/kernel/crash.c
index 3f3a579..b2a8b3d 100644
--- a/arch/ia64/kernel/crash.c
+++ b/arch/ia64/kernel/crash.c
@@ -140,10 +140,6 @@ kdump_cpu_freeze(struct unw_frame_info *info, void *arg)
 	atomic_inc(&kdump_cpu_frozen);
 	kdump_status[cpuid] = 1;
 	mb();
-#ifdef CONFIG_HOTPLUG_CPU
-	if (cpuid != 0)
-		ia64_jump_to_sal(&sal_boot_rendez_state[cpuid]);
-#endif
 	for (;;)
 		cpu_relax();
 }
-- 
1.6.0

[PATCH v2 5/7] ia64, kdump: Mask INIT first in panic-kdump path

[Hidetoshi Seto]

Summary:

  Asserting INIT might block kdump if the system is already going to
  start kdump via panic.

Description:

  INIT can interrupt anywhere in panic path, so it can interrupt in
  middle of kdump kicked by panic.  Therefore there is a race if kdump
  is kicked concurrently, via Panic and via INIT.

  INIT could fail to invoke kdump if the system is already going to
  start kdump via panic.  It could not restart kdump from INIT handler
  if some of cpus are already playing dead with INIT masked.  It also
  means that INIT could block kdump's progress if no monarch is entered
  in the INIT rendezvous.

  Panic+INIT is a rare, but possible situation since it can be assumed
  that the kernel or an internal agent decides to panic the unstable
  system while another external agent decides to send an INIT to the
  system at same time.

How to reproduce:

  Assert INIT just after panic, before all other cpus have frozen

Expected results:

  continue kdump invoked by panic, or restart kdump from INIT

Actual results:

  might be hang, crashdump not retrieved

Proposed Fix:

  This patch masks INIT first in panic path to take the initiative on
  kdump, and reuse atomic value kdump_in_progress to make sure there is
  only one initiator of kdump.  All INITs asserted later should be used
  only for freezing all other cpus.

  This mask will be removed soon by rfi in relocate_kernel.S, before jump
  into kdump kernel, after all cpus are frozen and no-op INIT handler is
  registered.  So if INIT was in the interval while it is masked, it will
  pend on the system and will received just after the rfi, and handled by
  the no-op handler.

  If there was a MCA event while psr.mc is 1, in theory the event will
  pend on the system and will received just after the rfi same as above.
  MCA handler is unregistered here at the time, so received MCA will not
  reach to OS_MCA and will result in warmboot by SAL.

  Note that codes in this masked interval are relatively simpler than
  that in MCA/INIT handler which also executed with the mask.  So it can
  be said that probability of error in this interval is supposed not so
  higher than that in MCA/INIT handler.

Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Cc: Vivek Goyal <vgoyal@redhat.com>
Cc: Haren Myneni <hbabu@us.ibm.com>
Cc: kexec@lists.infradead.org
---
 arch/ia64/kernel/crash.c           |   47 +++++++++++++++++++++++++++++++----
 arch/ia64/kernel/relocate_kernel.S |    2 +-
 2 files changed, 42 insertions(+), 7 deletions(-)

diff --git a/arch/ia64/kernel/crash.c b/arch/ia64/kernel/crash.c
index b2a8b3d..9c851b7 100644
--- a/arch/ia64/kernel/crash.c
+++ b/arch/ia64/kernel/crash.c
@@ -23,6 +23,7 @@
 int kdump_status[NR_CPUS];
 static atomic_t kdump_cpu_frozen;
 atomic_t kdump_in_progress;
+static int kdump_freeze_monarch;
 static int kdump_on_init = 1;
 static int kdump_on_fatal_mca = 1;
 
@@ -108,6 +109,33 @@ machine_crash_shutdown(struct pt_regs *pt)
 	 */
 	kexec_disable_iosapic();
 #ifdef CONFIG_SMP
+	/*
+	 * If kdump_on_init is set and an INIT is asserted here, kdump will
+	 * be started again via INIT monarch.
+	 */
+	local_irq_disable();
+	ia64_set_psr_mc();	/* mask MCA/INIT */
+	if (atomic_inc_return(&kdump_in_progress) != 1)
+		unw_init_running(kdump_cpu_freeze, NULL);
+
+	/*
+	 * Now this cpu is ready for kdump.
+	 * Stop all others by IPI or INIT.  They could receive INIT from
+	 * outside and might be INIT monarch, but only thing they have to
+	 * do is falling into kdump_cpu_freeze().
+	 *
+	 * If an INIT is asserted here:
+	 * - All receivers might be slaves, since some of cpus could already
+	 *   be frozen and INIT might be masked on monarch.  In this case,
+	 *   all slaves will park in while (monarch_cpu == -1) loop before
+	 *   DIE_INIT_SLAVE_ENTER that for waiting monarch enters.
+	 *	=> TBD: freeze all slaves
+	 * - One might be a monarch, but INIT rendezvous will fail since
+	 *   at least this cpu already have INIT masked so it never join
+	 *   to the rendezvous.  In this case, all slaves and monarch will
+	 *   be frozen after timeout of the INIT rendezvous.
+	 *	=> TBD: freeze them without waiting timeout
+	 */
 	kdump_smp_send_stop();
 	/* not all cpu response to IPI, send INIT to freeze them */
 	if (kdump_wait_cpu_freeze() && kdump_on_init) 	{
@@ -177,13 +205,18 @@ kdump_init_notifier(struct notifier_block *self, unsigned long val, void *data)
 	switch (val) {
 	case DIE_INIT_MONARCH_PROCESS:
 		if (kdump_on_init) {
-			atomic_set(&kdump_in_progress, 1);
+			if (atomic_inc_return(&kdump_in_progress) != 1)
+				kdump_freeze_monarch = 1;
 			*(nd->monarch_cpu) = -1;
 		}
 		break;
 	case DIE_INIT_MONARCH_LEAVE:
-		if (kdump_on_init)
-			machine_kdump_on_init();
+		if (kdump_on_init) {
+			if (kdump_freeze_monarch)
+				unw_init_running(kdump_cpu_freeze, NULL);
+			else
+				machine_kdump_on_init();
+		}
 		break;
 	case DIE_INIT_SLAVE_LEAVE:
 		if (atomic_read(&kdump_in_progress))
@@ -196,9 +229,11 @@ kdump_init_notifier(struct notifier_block *self, unsigned long val, void *data)
 	case DIE_MCA_MONARCH_LEAVE:
 		/* *(nd->data) indicate if MCA is recoverable */
 		if (kdump_on_fatal_mca && !(*(nd->data))) {
-			atomic_set(&kdump_in_progress, 1);
-			*(nd->monarch_cpu) = -1;
-			machine_kdump_on_init();
+			if (atomic_inc_return(&kdump_in_progress) == 1) {
+				*(nd->monarch_cpu) = -1;
+				machine_kdump_on_init();
+			}
+			/* We got fatal MCA while kdump!? No way!! */
 		}
 		break;
 	}
diff --git a/arch/ia64/kernel/relocate_kernel.S b/arch/ia64/kernel/relocate_kernel.S
index 903babd..32f6fc1 100644
--- a/arch/ia64/kernel/relocate_kernel.S
+++ b/arch/ia64/kernel/relocate_kernel.S
@@ -52,7 +52,7 @@ GLOBAL_ENTRY(relocate_new_kernel)
 	srlz.i
 	;;
 	mov ar.rnat=r18
-	rfi
+	rfi				// note: this unmask MCA/INIT (psr.mc)
 	;;
 1:
 	//physical mode code begin
-- 
1.6.0

[PATCH v2 6/7] ia64, kdump: Try INIT regardless of kdump_on_init

[Hidetoshi Seto]

CPUs should be frozen if possible, otherwise it might hinder kdump.
So if there are CPUs not respond to IPI, try INIT to stop them.

Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Cc: Vivek Goyal <vgoyal@redhat.com>
Cc: Haren Myneni <hbabu@us.ibm.com>
Cc: kexec@lists.infradead.org
---
 arch/ia64/kernel/crash.c |   43 +++++++++++++++++++++----------------------
 1 files changed, 21 insertions(+), 22 deletions(-)

diff --git a/arch/ia64/kernel/crash.c b/arch/ia64/kernel/crash.c
index 9c851b7..0995fdc 100644
--- a/arch/ia64/kernel/crash.c
+++ b/arch/ia64/kernel/crash.c
@@ -138,8 +138,10 @@ machine_crash_shutdown(struct pt_regs *pt)
 	 */
 	kdump_smp_send_stop();
 	/* not all cpu response to IPI, send INIT to freeze them */
-	if (kdump_wait_cpu_freeze() && kdump_on_init) 	{
+	if (kdump_wait_cpu_freeze()) {
 		kdump_smp_send_init();
+		/* wait again, don't go ahead if possible */
+		kdump_wait_cpu_freeze();
 	}
 #endif
 }
@@ -178,6 +180,19 @@ kdump_init_notifier(struct notifier_block *self, unsigned long val, void *data)
 	struct ia64_mca_notify_die *nd;
 	struct die_args *args = data;
 
+	if (atomic_read(&kdump_in_progress)) {
+		switch (val) {
+		case DIE_INIT_MONARCH_LEAVE:
+			if (!kdump_freeze_monarch)
+				break;
+			/* fall through */
+		case DIE_INIT_SLAVE_LEAVE:
+		case DIE_MCA_RENDZVOUS_LEAVE:
+			unw_init_running(kdump_cpu_freeze, NULL);
+			break;
+		}
+	}
+
 	if (!kdump_on_init && !kdump_on_fatal_mca)
 		return NOTIFY_DONE;
 
@@ -190,41 +205,25 @@ kdump_init_notifier(struct notifier_block *self, unsigned long val, void *data)
 	}
 
 	if (val != DIE_INIT_MONARCH_LEAVE &&
-	    val != DIE_INIT_SLAVE_LEAVE &&
 	    val != DIE_INIT_MONARCH_PROCESS &&
-	    val != DIE_MCA_RENDZVOUS_LEAVE &&
 	    val != DIE_MCA_MONARCH_LEAVE)
 		return NOTIFY_DONE;
 
 	nd = (struct ia64_mca_notify_die *)args->err;
-	/* Reason code 1 means machine check rendezvous*/
-	if ((val == DIE_INIT_MONARCH_LEAVE || val == DIE_INIT_SLAVE_LEAVE
-	    || val == DIE_INIT_MONARCH_PROCESS) && nd->sos->rv_rc == 1)
-		return NOTIFY_DONE;
 
 	switch (val) {
 	case DIE_INIT_MONARCH_PROCESS:
-		if (kdump_on_init) {
+		/* Reason code 1 means machine check rendezvous*/
+		if (kdump_on_init && (nd->sos->rv_rc != 1)) {
 			if (atomic_inc_return(&kdump_in_progress) != 1)
 				kdump_freeze_monarch = 1;
 			*(nd->monarch_cpu) = -1;
 		}
 		break;
 	case DIE_INIT_MONARCH_LEAVE:
-		if (kdump_on_init) {
-			if (kdump_freeze_monarch)
-				unw_init_running(kdump_cpu_freeze, NULL);
-			else
-				machine_kdump_on_init();
-		}
-		break;
-	case DIE_INIT_SLAVE_LEAVE:
-		if (atomic_read(&kdump_in_progress))
-			unw_init_running(kdump_cpu_freeze, NULL);
-		break;
-	case DIE_MCA_RENDZVOUS_LEAVE:
-		if (atomic_read(&kdump_in_progress))
-			unw_init_running(kdump_cpu_freeze, NULL);
+		/* Reason code 1 means machine check rendezvous*/
+		if (kdump_on_init && (nd->sos->rv_rc != 1))
+			machine_kdump_on_init();
 		break;
 	case DIE_MCA_MONARCH_LEAVE:
 		/* *(nd->data) indicate if MCA is recoverable */
-- 
1.6.0

[PATCH v2 7/7] ia64, kdump: Short path to freeze CPUs

[Hidetoshi Seto]

Setting monarch_cpu = -1 to let slaves frozen might not work, because
there might be slaves being late, not entered the rendezvous yet.
Such slaves might be caught in while (monarch_cpu == -1) loop.

Use kdump_in_progress instead of monarch_cpus to break INIT rendezvous
and let all slaves enter DIE_INIT_SLAVE_LEAVE smoothly.

And monarch no longer need to manage rendezvous if once kdump_in_progress
is set, catch the monarch in DIE_INIT_MONARCH_ENTER then.

Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Cc: Vivek Goyal <vgoyal@redhat.com>
Cc: Haren Myneni <hbabu@us.ibm.com>
Cc: kexec@lists.infradead.org
---
 arch/ia64/kernel/crash.c |   15 ++++++---------
 arch/ia64/kernel/mca.c   |   15 +++++++++++++--
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/arch/ia64/kernel/crash.c b/arch/ia64/kernel/crash.c
index 0995fdc..6631a9d 100644
--- a/arch/ia64/kernel/crash.c
+++ b/arch/ia64/kernel/crash.c
@@ -127,14 +127,13 @@ machine_crash_shutdown(struct pt_regs *pt)
 	 * If an INIT is asserted here:
 	 * - All receivers might be slaves, since some of cpus could already
 	 *   be frozen and INIT might be masked on monarch.  In this case,
-	 *   all slaves will park in while (monarch_cpu == -1) loop before
-	 *   DIE_INIT_SLAVE_ENTER that for waiting monarch enters.
-	 *	=> TBD: freeze all slaves
+	 *   all slaves will be frozen soon since kdump_in_progress will let
+	 *   them into DIE_INIT_SLAVE_LEAVE.
 	 * - One might be a monarch, but INIT rendezvous will fail since
 	 *   at least this cpu already have INIT masked so it never join
 	 *   to the rendezvous.  In this case, all slaves and monarch will
-	 *   be frozen after timeout of the INIT rendezvous.
-	 *	=> TBD: freeze them without waiting timeout
+	 *   be frozen soon with no wait since the INIT rendezvous is skipped
+	 *   by kdump_in_progress.
 	 */
 	kdump_smp_send_stop();
 	/* not all cpu response to IPI, send INIT to freeze them */
@@ -187,6 +186,7 @@ kdump_init_notifier(struct notifier_block *self, unsigned long val, void *data)
 				break;
 			/* fall through */
 		case DIE_INIT_SLAVE_LEAVE:
+		case DIE_INIT_MONARCH_ENTER:
 		case DIE_MCA_RENDZVOUS_LEAVE:
 			unw_init_running(kdump_cpu_freeze, NULL);
 			break;
@@ -217,7 +217,6 @@ kdump_init_notifier(struct notifier_block *self, unsigned long val, void *data)
 		if (kdump_on_init && (nd->sos->rv_rc != 1)) {
 			if (atomic_inc_return(&kdump_in_progress) != 1)
 				kdump_freeze_monarch = 1;
-			*(nd->monarch_cpu) = -1;
 		}
 		break;
 	case DIE_INIT_MONARCH_LEAVE:
@@ -228,10 +227,8 @@ kdump_init_notifier(struct notifier_block *self, unsigned long val, void *data)
 	case DIE_MCA_MONARCH_LEAVE:
 		/* *(nd->data) indicate if MCA is recoverable */
 		if (kdump_on_fatal_mca && !(*(nd->data))) {
-			if (atomic_inc_return(&kdump_in_progress) == 1) {
-				*(nd->monarch_cpu) = -1;
+			if (atomic_inc_return(&kdump_in_progress) == 1)
 				machine_kdump_on_init();
-			}
 			/* We got fatal MCA while kdump!? No way!! */
 		}
 		break;
diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c
index 7b30d21..d2877a7 100644
--- a/arch/ia64/kernel/mca.c
+++ b/arch/ia64/kernel/mca.c
@@ -1682,14 +1682,25 @@ ia64_init_handler(struct pt_regs *regs, struct switch_stack *sw,
 
 	if (!sos->monarch) {
 		ia64_mc_info.imi_rendez_checkin[cpu] = IA64_MCA_RENDEZ_CHECKIN_INIT;
+
+#ifdef CONFIG_KEXEC
+		while (monarch_cpu == -1 && !atomic_read(&kdump_in_progress))
+			udelay(1000);
+#else
 		while (monarch_cpu == -1)
-		       cpu_relax();	/* spin until monarch enters */
+			cpu_relax();	/* spin until monarch enters */
+#endif
 
 		NOTIFY_INIT(DIE_INIT_SLAVE_ENTER, regs, (long)&nd, 1);
 		NOTIFY_INIT(DIE_INIT_SLAVE_PROCESS, regs, (long)&nd, 1);
 
+#ifdef CONFIG_KEXEC
+		while (monarch_cpu != -1 && !atomic_read(&kdump_in_progress))
+			udelay(1000);
+#else
 		while (monarch_cpu != -1)
-		       cpu_relax();	/* spin until monarch leaves */
+			cpu_relax();	/* spin until monarch leaves */
+#endif
 
 		NOTIFY_INIT(DIE_INIT_SLAVE_LEAVE, regs, (long)&nd, 1);
 
-- 
1.6.0