[PATCH] sysrq: add a show-stacktrace-on-all-cpus command

[PATCH] sysrq: add a show-stacktrace-on-all-cpus command

[Avi Kivity]

If a cpu is spinning in the kernel but still responding to interrupts,
pressing sysrq-y will show you where it's spinning.

Signed-off-by: Avi Kivity <avi@qumranet.com>

diff --git a/drivers/char/sysrq.c b/drivers/char/sysrq.c
index 39cc318..1dda709 100644
--- a/drivers/char/sysrq.c
+++ b/drivers/char/sysrq.c
@@ -45,6 +45,8 @@ int __read_mostly __sysrq_enabled = 1;
 
 static int __read_mostly sysrq_always_enabled;
 
+static spinlock_t show_stack_lock = SPIN_LOCK_UNLOCKED;
+
 int sysrq_on(void)
 {
 	return __sysrq_enabled || sysrq_always_enabled;
@@ -309,6 +311,26 @@ static struct sysrq_key_op sysrq_unrt_op = {
 	.enable_mask	= SYSRQ_ENABLE_RTNICE,
 };
 
+static void show_cpu_stack(void *garbage)
+{
+	spin_lock(&show_stack_lock);
+	printk("CPU%d stacktrace:\n", raw_smp_processor_id());
+	dump_stack();
+	sysrq_handle_showregs(0, NULL);
+	spin_unlock(&show_stack_lock);
+}
+
+static void sysrq_show_stacks(int key, struct tty_struct *tty)
+{
+	on_each_cpu(show_cpu_stack, NULL, 0, 1);
+}
+
+static struct sysrq_key_op sysrq_show_stacks_op = {
+	.handler	= sysrq_show_stacks,
+	.help_msg	= "stacktraces-on-all-cpus(Y)",
+	.action_msg	= "Stack traces on all cpus",
+};
+
 /* Key Operations table and lock */
 static DEFINE_SPINLOCK(sysrq_key_table_lock);
 
@@ -356,7 +378,7 @@ static struct sysrq_key_op *sysrq_key_table[36] = {
 	&sysrq_showstate_blocked_op,	/* w */
 	/* x: May be registered on ppc/powerpc for xmon */
 	NULL,				/* x */
-	NULL,				/* y */
+	&sysrq_show_stacks_op,		/* y */
 	NULL				/* z */
 };
 

Re: [PATCH] sysrq: add a show-stacktrace-on-all-cpus command

[Andrew Morton]

On Sun, 29 Jul 2007 13:29:26 +0300
Avi Kivity <avi@qumranet.com> wrote:

> If a cpu is spinning in the kernel but still responding to interrupts,
> pressing sysrq-y will show you where it's spinning.
> 
> Signed-off-by: Avi Kivity <avi@qumranet.com>
> 
> diff --git a/drivers/char/sysrq.c b/drivers/char/sysrq.c
> index 39cc318..1dda709 100644
> --- a/drivers/char/sysrq.c
> +++ b/drivers/char/sysrq.c
> @@ -45,6 +45,8 @@ int __read_mostly __sysrq_enabled = 1;
>  
>  static int __read_mostly sysrq_always_enabled;
>  
> +static spinlock_t show_stack_lock = SPIN_LOCK_UNLOCKED;

Use DEFINE_SPINLOCK to avoid confusing lockdep.

>  int sysrq_on(void)
>  {
>  	return __sysrq_enabled || sysrq_always_enabled;
> @@ -309,6 +311,26 @@ static struct sysrq_key_op sysrq_unrt_op = {
>  	.enable_mask	= SYSRQ_ENABLE_RTNICE,
>  };
>  
> +static void show_cpu_stack(void *garbage)
> +{
> +	spin_lock(&show_stack_lock);
> +	printk("CPU%d stacktrace:\n", raw_smp_processor_id());
> +	dump_stack();
> +	sysrq_handle_showregs(0, NULL);
> +	spin_unlock(&show_stack_lock);
> +}
> +
> +static void sysrq_show_stacks(int key, struct tty_struct *tty)
> +{
> +	on_each_cpu(show_cpu_stack, NULL, 0, 1);
> +}
> +
> +static struct sysrq_key_op sysrq_show_stacks_op = {
> +	.handler	= sysrq_show_stacks,
> +	.help_msg	= "stacktraces-on-all-cpus(Y)",
> +	.action_msg	= "Stack traces on all cpus",
> +};
> +
>  /* Key Operations table and lock */
>  static DEFINE_SPINLOCK(sysrq_key_table_lock);
>  
> @@ -356,7 +378,7 @@ static struct sysrq_key_op *sysrq_key_table[36] = {
>  	&sysrq_showstate_blocked_op,	/* w */
>  	/* x: May be registered on ppc/powerpc for xmon */
>  	NULL,				/* x */
> -	NULL,				/* y */
> +	&sysrq_show_stacks_op,		/* y */
>  	NULL				/* z */
>  };
>  

but, but..  sysrq handlers called from hard IRQ.  Are we sure that none of
the drivers which call into the sysrq code do so with hard IRQs disabled?

Because if we call on_each_cpu() with hard IRQs disabled, the various
implementations will emit loud warnings due to the deadlockability.

Re: [PATCH] sysrq: add a show-stacktrace-on-all-cpus command

[Avi Kivity]

Andrew Morton wrote:
> On Sun, 29 Jul 2007 13:29:26 +0300
> Avi Kivity <avi@qumranet.com> wrote:
>
>   
>> If a cpu is spinning in the kernel but still responding to interrupts,
>> pressing sysrq-y will show you where it's spinning.
>>
>> Signed-off-by: Avi Kivity <avi@qumranet.com>
>>
>> diff --git a/drivers/char/sysrq.c b/drivers/char/sysrq.c
>> index 39cc318..1dda709 100644
>> --- a/drivers/char/sysrq.c
>> +++ b/drivers/char/sysrq.c
>> @@ -45,6 +45,8 @@ int __read_mostly __sysrq_enabled = 1;
>>  
>>  static int __read_mostly sysrq_always_enabled;
>>  
>> +static spinlock_t show_stack_lock = SPIN_LOCK_UNLOCKED;
>>     
>
> Use DEFINE_SPINLOCK to avoid confusing lockdep.
>
>   

Okay.

>>  int sysrq_on(void)
>>  {
>>  	return __sysrq_enabled || sysrq_always_enabled;
>> @@ -309,6 +311,26 @@ static struct sysrq_key_op sysrq_unrt_op = {
>>  	.enable_mask	= SYSRQ_ENABLE_RTNICE,
>>  };
>>  
>> +static void show_cpu_stack(void *garbage)
>> +{
>> +	spin_lock(&show_stack_lock);
>> +	printk("CPU%d stacktrace:\n", raw_smp_processor_id());
>> +	dump_stack();
>> +	sysrq_handle_showregs(0, NULL);
>> +	spin_unlock(&show_stack_lock);
>> +}
>> +
>> +static void sysrq_show_stacks(int key, struct tty_struct *tty)
>> +{
>> +	on_each_cpu(show_cpu_stack, NULL, 0, 1);
>> +}
>> +
>> +static struct sysrq_key_op sysrq_show_stacks_op = {
>> +	.handler	= sysrq_show_stacks,
>> +	.help_msg	= "stacktraces-on-all-cpus(Y)",
>> +	.action_msg	= "Stack traces on all cpus",
>> +};
>> +
>>  /* Key Operations table and lock */
>>  static DEFINE_SPINLOCK(sysrq_key_table_lock);
>>  
>> @@ -356,7 +378,7 @@ static struct sysrq_key_op *sysrq_key_table[36] = {
>>  	&sysrq_showstate_blocked_op,	/* w */
>>  	/* x: May be registered on ppc/powerpc for xmon */
>>  	NULL,				/* x */
>> -	NULL,				/* y */
>> +	&sysrq_show_stacks_op,		/* y */
>>  	NULL				/* z */
>>  };
>>  
>>     
>
> but, but..  sysrq handlers called from hard IRQ.  Are we sure that none of
> the drivers which call into the sysrq code do so with hard IRQs disabled?
>
> Because if we call on_each_cpu() with hard IRQs disabled, the various
> implementations will emit loud warnings due to the deadlockability.
>
>   

Ah, I tested this with /proc/sysrq-trigger.

Maybe we should temporarily enable irqs here; if you press this key,
your system is already on the way out anyway.  But perhaps that would
entrap curious admins checking out what the keys do.

-- 
Do not meddle in the internals of kernels, for they are subtle and quick to panic.