3.18.7-rt2: NOHZ_FULL warning at boot

3.18.7-rt2: NOHZ_FULL warning at boot

[Luis Claudio R. Goncalves]

Hello,

I have been seeing the warning below at boot time on 3.18.7-rt2. Anybody
else seeing it?

[    1.537207] ------------[ cut here ]------------
[    1.537215] WARNING: CPU: 1 PID: 20 at /home/lclaudio/SANDBOX/kernel-rt/kernel/time/tick-sched.c:167 can_stop_full_tick+0x1ee/0x2b0()
[    1.537219] Modules linked in: efivarfs
[    1.537224] CPU: 1 PID: 20 Comm: ksoftirqd/1 Not tainted 3.18.7-rt2+ #2
[    1.537225] Hardware name: Hewlett-Packard p7-1512/2ADA, BIOS 8.15 02/05/2013
[    1.537227]  0000000000000000 000000005b4bc999 ffff880191503bb8 ffffffff8175a165
[    1.537228]  0000000000000000 0000000000000000 ffff880191503bf8 ffffffff8107add1
[    1.537229]  ffff880191503be8 ffff880198fce280 ffff880198e00000 0000000000000000
[    1.537229] Call Trace:
[    1.537233]  [<ffffffff8175a165>] dump_stack+0x4f/0x7c
[    1.537235]  [<ffffffff8107add1>] warn_slowpath_common+0x81/0xc0
[    1.537237]  [<ffffffff8107af0a>] warn_slowpath_null+0x1a/0x20
[    1.537238]  [<ffffffff811074fe>] can_stop_full_tick+0x1ee/0x2b0
[    1.537240]  [<ffffffff81107656>] __tick_nohz_full_check+0x76/0xc0
[    1.537241]  [<ffffffff811076ae>] nohz_full_kick_work_func+0xe/0x10
[    1.537243]  [<ffffffff8116ff7c>] irq_work_run_list+0x3c/0x60
[    1.537244]  [<ffffffff8117037e>] irq_work_tick+0x3e/0x90
[    1.537246]  [<ffffffff810f25c4>] run_timer_softirq+0x44/0x3b0
[    1.537248]  [<ffffffff81760855>] ? _raw_spin_unlock+0x35/0x60
[    1.537250]  [<ffffffff810b0b89>] ? vtime_account_system+0x99/0xb0
[    1.537252]  [<ffffffff810b0d9b>] ? vtime_common_account_irq_enter+0x4b/0x80
[    1.537254]  [<ffffffff8107fbd4>] do_current_softirqs.isra.11+0x274/0x550
[    1.537255]  [<ffffffff8107ff86>] run_ksoftirqd+0x26/0x50
[    1.537257]  [<ffffffff810a1a0d>] smpboot_thread_fn+0x18d/0x2d0
[    1.537259]  [<ffffffff810a1880>] ? SyS_setgroups+0x150/0x150
[    1.537260]  [<ffffffff8109d68d>] kthread+0xfd/0x120
[    1.537262]  [<ffffffff8175e177>] ? wait_for_completion+0xe7/0x120
[    1.537264]  [<ffffffff8109d590>] ? kthread_create_on_node+0x260/0x260
[    1.537266]  [<ffffffff817618ac>] ret_from_fork+0x7c/0xb0
[    1.537267]  [<ffffffff8109d590>] ? kthread_create_on_node+0x260/0x260
[    1.537268] ---[ end trace 0000000000000002 ]---


>From kernel/time/tick-sched.c line 167:

static bool can_stop_full_tick(void)
{
        WARN_ON_ONCE(!irqs_disabled());

The config in use, NO_HZ related is:

	CONFIG_NO_HZ_COMMON=y
	# CONFIG_NO_HZ_IDLE is not set
		CONFIG_NO_HZ_FULL=y
	CONFIG_NO_HZ_FULL_ALL=y
	CONFIG_NO_HZ_FULL_SYSIDLE=y
	CONFIG_NO_HZ_FULL_SYSIDLE_SMALL=8
	CONFIG_NO_HZ=y
	CONFIG_HZ_1000=y
	CONFIG_HZ=1000

Cheers,
Luis
-- 
[ Luis Claudio R. Goncalves                    Bass - Gospel - RT ]
[ Fingerprint: 4FDD B8C4 3C59 34BD 8BE9  2696 7203 D980 A448 C8F8 ]

Re: 3.18.7-rt2: NOHZ_FULL warning at boot

[Sebastian Andrzej Siewior]

* Luis Claudio R. Goncalves | 2015-02-26 17:05:47 [-0300]:

>[    1.537215] WARNING: CPU: 1 PID: 20 at /home/lclaudio/SANDBOX/kernel-rt/kernel/time/tick-sched.c:167 can_stop_full_tick+0x1ee/0x2b0()

I haven't run full-dynticks for some time. I might have broken that, I
take a look at that.

Sebastian

Re: 3.18.7-rt2: NOHZ_FULL warning at boot

[Sebastian Andrzej Siewior]

* Luis Claudio R. Goncalves | 2015-02-26 17:05:47 [-0300]:

>I have been seeing the warning below at boot time on 3.18.7-rt2. Anybody
>else seeing it?

This should fix it.

diff --git a/kernel/irq_work.c b/kernel/irq_work.c
index 19c363a..0c64912 100644
--- a/kernel/irq_work.c
+++ b/kernel/irq_work.c
@@ -71,6 +71,8 @@ void __weak arch_irq_work_raise(void)
  */
 bool irq_work_queue_on(struct irq_work *work, int cpu)
 {
+	bool raise_irqwork;
+
 	/* All work should have been flushed before going offline */
 	WARN_ON_ONCE(cpu_is_offline(cpu));
 
@@ -81,7 +83,19 @@ bool irq_work_queue_on(struct irq_work *work, int cpu)
 	if (!irq_work_claim(work))
 		return false;
 
-	if (llist_add(&work->llnode, &per_cpu(raised_list, cpu)))
+#ifdef CONFIG_PREEMPT_RT_FULL
+	if (work->flags & IRQ_WORK_HARD_IRQ)
+		raise_irqwork = llist_add(&work->llnode,
+					  &per_cpu(hirq_work_list, cpu));
+	else
+		raise_irqwork = llist_add(&work->llnode,
+					  &per_cpu(lazy_list, cpu));
+#else
+		raise_irqwork = llist_add(&work->llnode,
+					  &per_cpu(raised_list, cpu));
+#endif
+
+	if (raise_irqwork)
 		arch_send_call_function_single_ipi(cpu);
 
 	return true;
@@ -101,19 +115,14 @@ bool irq_work_queue(struct irq_work *work)
 
 #ifdef CONFIG_PREEMPT_RT_FULL
 	if (work->flags & IRQ_WORK_HARD_IRQ) {
-		if (llist_add(&work->llnode, this_cpu_ptr(&hirq_work_list))) {
-			if (work->flags & IRQ_WORK_LAZY) {
-				if (tick_nohz_tick_stopped())
-					arch_irq_work_raise();
-			} else {
-				arch_irq_work_raise();
-			}
-		}
-	/* If the work is "lazy", handle it from next tick if any */
-	} else if (work->flags & IRQ_WORK_LAZY) {
+		if (llist_add(&work->llnode, this_cpu_ptr(&hirq_work_list)))
+			arch_irq_work_raise();
+	} else {
+		if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)))
+			arch_irq_work_raise();
+	}
 #else
 	if (work->flags & IRQ_WORK_LAZY) {
-#endif
 		if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) &&
 		    tick_nohz_tick_stopped())
 			arch_irq_work_raise();
@@ -121,6 +130,7 @@ bool irq_work_queue(struct irq_work *work)
 		if (llist_add(&work->llnode, this_cpu_ptr(&raised_list)))
 			arch_irq_work_raise();
 	}
+#endif
 
 	preempt_enable();
 
@@ -137,7 +147,8 @@ bool irq_work_needs_cpu(void)
 
 	if (llist_empty(raised))
 		if (llist_empty(lazy))
-			return false;
+			if (llist_empty(this_cpu_ptr(&hirq_work_list)))
+				return false;
 
 	/* All work should have been flushed before going offline */
 	WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));

Sebastian

Re: 3.18.7-rt2: NOHZ_FULL warning at boot

[Carsten Emde]

On 03/06/2015 07:13 PM, Sebastian Andrzej Siewior wrote:
> * Luis Claudio R. Goncalves | 2015-02-26 17:05:47 [-0300]:
>
>> I have been seeing the warning below at boot time on 3.18.7-rt2. Anybody
>> else seeing it?
>
> This should fix it.
>
> diff --git a/kernel/irq_work.c b/kernel/irq_work.c
> index 19c363a..0c64912 100644
> --- a/kernel/irq_work.c
> +++ b/kernel/irq_work.c
> @@ -71,6 +71,8 @@ void __weak arch_irq_work_raise(void)
>    */
>   bool irq_work_queue_on(struct irq_work *work, int cpu)
>   {
> +	bool raise_irqwork;
> +
>   	/* All work should have been flushed before going offline */
>   	WARN_ON_ONCE(cpu_is_offline(cpu));
>
> @@ -81,7 +83,19 @@ bool irq_work_queue_on(struct irq_work *work, int cpu)
>   	if (!irq_work_claim(work))
>   		return false;
>
> -	if (llist_add(&work->llnode, &per_cpu(raised_list, cpu)))
> +#ifdef CONFIG_PREEMPT_RT_FULL
> +	if (work->flags & IRQ_WORK_HARD_IRQ)
> +		raise_irqwork = llist_add(&work->llnode,
> +					  &per_cpu(hirq_work_list, cpu));
> +	else
> +		raise_irqwork = llist_add(&work->llnode,
> +					  &per_cpu(lazy_list, cpu));
> +#else
> +		raise_irqwork = llist_add(&work->llnode,
> +					  &per_cpu(raised_list, cpu));
> +#endif
> +
> +	if (raise_irqwork)
>   		arch_send_call_function_single_ipi(cpu);
>
>   	return true;
> @@ -101,19 +115,14 @@ bool irq_work_queue(struct irq_work *work)
>
>   #ifdef CONFIG_PREEMPT_RT_FULL
>   	if (work->flags & IRQ_WORK_HARD_IRQ) {
> -		if (llist_add(&work->llnode, this_cpu_ptr(&hirq_work_list))) {
> -			if (work->flags & IRQ_WORK_LAZY) {
> -				if (tick_nohz_tick_stopped())
> -					arch_irq_work_raise();
> -			} else {
> -				arch_irq_work_raise();
> -			}
> -		}
> -	/* If the work is "lazy", handle it from next tick if any */
> -	} else if (work->flags & IRQ_WORK_LAZY) {
> +		if (llist_add(&work->llnode, this_cpu_ptr(&hirq_work_list)))
> +			arch_irq_work_raise();
> +	} else {
> +		if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)))
> +			arch_irq_work_raise();
> +	}
>   #else
>   	if (work->flags & IRQ_WORK_LAZY) {
> -#endif
>   		if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) &&
>   		    tick_nohz_tick_stopped())
>   			arch_irq_work_raise();
> @@ -121,6 +130,7 @@ bool irq_work_queue(struct irq_work *work)
>   		if (llist_add(&work->llnode, this_cpu_ptr(&raised_list)))
>   			arch_irq_work_raise();
>   	}
> +#endif
>
>   	preempt_enable();
>
> @@ -137,7 +147,8 @@ bool irq_work_needs_cpu(void)
>
>   	if (llist_empty(raised))
>   		if (llist_empty(lazy))
> -			return false;
> +			if (llist_empty(this_cpu_ptr(&hirq_work_list)))
> +				return false;
>
>   	/* All work should have been flushed before going offline */
>   	WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));

I can confirm that the warning is no longer issued when this patch is 
applied.

Thanks,
	-Carsten.