[PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[James Chapman]

Fix locking issues in the pppol2tp driver which can cause a kernel
crash on SMP boxes when hundreds of L2TP sessions are created/deleted
simultaneously (ISP environment). The driver was violating read_lock()
and write_lock() scheduling rules so we now consistently use the _irq
variants of the lock functions.

Signed-off-by: James Chapman <jchapman@katalix.com>

--
This patch has been verified by the ISP that discovered the problem.

If the patch is accepted, it should be pushed to the stable 2.6.23 and
2.6.24 trees.

Index: linux-2.6.24/drivers/net/pppol2tp.c
===================================================================
--- linux-2.6.24.orig/drivers/net/pppol2tp.c
+++ linux-2.6.24/drivers/net/pppol2tp.c
@@ -301,15 +301,16 @@ pppol2tp_session_find(struct pppol2tp_tu
 		pppol2tp_session_id_hash(tunnel, session_id);
 	struct pppol2tp_session *session;
 	struct hlist_node *walk;
+	unsigned long flags;
 
-	read_lock(&tunnel->hlist_lock);
+	read_lock_irqsave(&tunnel->hlist_lock, flags);
 	hlist_for_each_entry(session, walk, session_list, hlist) {
 		if (session->tunnel_addr.s_session == session_id) {
-			read_unlock(&tunnel->hlist_lock);
+			read_unlock_irqrestore(&tunnel->hlist_lock, flags);
 			return session;
 		}
 	}
-	read_unlock(&tunnel->hlist_lock);
+	read_unlock_irqrestore(&tunnel->hlist_lock, flags);
 
 	return NULL;
 }
@@ -319,15 +320,16 @@ pppol2tp_session_find(struct pppol2tp_tu
 static struct pppol2tp_tunnel *pppol2tp_tunnel_find(u16 tunnel_id)
 {
 	struct pppol2tp_tunnel *tunnel = NULL;
+	unsigned long flags;
 
-	read_lock(&pppol2tp_tunnel_list_lock);
+	read_lock_irqsave(&pppol2tp_tunnel_list_lock, flags);
 	list_for_each_entry(tunnel, &pppol2tp_tunnel_list, list) {
 		if (tunnel->stats.tunnel_id == tunnel_id) {
-			read_unlock(&pppol2tp_tunnel_list_lock);
+			read_unlock_irqrestore(&pppol2tp_tunnel_list_lock, flags);
 			return tunnel;
 		}
 	}
-	read_unlock(&pppol2tp_tunnel_list_lock);
+	read_unlock_irqrestore(&pppol2tp_tunnel_list_lock, flags);
 
 	return NULL;
 }
@@ -1099,6 +1101,7 @@ static void pppol2tp_tunnel_closeall(str
 	struct hlist_node *tmp;
 	struct pppol2tp_session *session;
 	struct sock *sk;
+	unsigned long flags;
 
 	if (tunnel == NULL)
 		BUG();
@@ -1106,7 +1109,7 @@ static void pppol2tp_tunnel_closeall(str
 	PRINTK(tunnel->debug, PPPOL2TP_MSG_CONTROL, KERN_INFO,
 	       "%s: closing all sessions...\n", tunnel->name);
 
-	write_lock(&tunnel->hlist_lock);
+	write_lock_irqsave(&tunnel->hlist_lock, flags);
 	for (hash = 0; hash < PPPOL2TP_HASH_SIZE; hash++) {
 again:
 		hlist_for_each_safe(walk, tmp, &tunnel->session_hlist[hash]) {
@@ -1126,7 +1129,7 @@ again:
 			 * disappear as we're jumping between locks.
 			 */
 			sock_hold(sk);
-			write_unlock(&tunnel->hlist_lock);
+			write_unlock_irqrestore(&tunnel->hlist_lock, flags);
 			lock_sock(sk);
 
 			if (sk->sk_state & (PPPOX_CONNECTED | PPPOX_BOUND)) {
@@ -1148,11 +1151,11 @@ again:
 			 * list so we are guaranteed to make forward
 			 * progress.
 			 */
-			write_lock(&tunnel->hlist_lock);
+			write_lock_irqsave(&tunnel->hlist_lock, flags);
 			goto again;
 		}
 	}
-	write_unlock(&tunnel->hlist_lock);
+	write_unlock_irqrestore(&tunnel->hlist_lock, flags);
 }
 
 /* Really kill the tunnel.
@@ -1160,10 +1163,12 @@ again:
  */
 static void pppol2tp_tunnel_free(struct pppol2tp_tunnel *tunnel)
 {
+	unsigned long flags;
+
 	/* Remove from socket list */
-	write_lock(&pppol2tp_tunnel_list_lock);
+	write_lock_irqsave(&pppol2tp_tunnel_list_lock, flags);
 	list_del_init(&tunnel->list);
-	write_unlock(&pppol2tp_tunnel_list_lock);
+	write_unlock_irqrestore(&pppol2tp_tunnel_list_lock, flags);
 
 	atomic_dec(&pppol2tp_tunnel_count);
 	kfree(tunnel);
@@ -1212,6 +1217,7 @@ end:
 static void pppol2tp_session_destruct(struct sock *sk)
 {
 	struct pppol2tp_session *session = NULL;
+	unsigned long flags;
 
 	if (sk->sk_user_data != NULL) {
 		struct pppol2tp_tunnel *tunnel;
@@ -1239,9 +1245,9 @@ static void pppol2tp_session_destruct(st
 				/* Delete the session socket from the
 				 * hash
 				 */
-				write_lock(&tunnel->hlist_lock);
+				write_lock_irqsave(&tunnel->hlist_lock, flags);
 				hlist_del_init(&session->hlist);
-				write_unlock(&tunnel->hlist_lock);
+				write_unlock_irqrestore(&tunnel->hlist_lock, flags);
 
 				atomic_dec(&pppol2tp_session_count);
 			}
@@ -1312,6 +1318,7 @@ static struct sock *pppol2tp_prepare_tun
 	struct sock *sk;
 	struct pppol2tp_tunnel *tunnel;
 	struct sock *ret = NULL;
+	unsigned long flags;
 
 	/* Get the tunnel UDP socket from the fd, which was opened by
 	 * the userspace L2TP daemon.
@@ -1386,9 +1393,9 @@ static struct sock *pppol2tp_prepare_tun
 
 	/* Add tunnel to our list */
 	INIT_LIST_HEAD(&tunnel->list);
-	write_lock(&pppol2tp_tunnel_list_lock);
+	write_lock_irqsave(&pppol2tp_tunnel_list_lock, flags);
 	list_add(&tunnel->list, &pppol2tp_tunnel_list);
-	write_unlock(&pppol2tp_tunnel_list_lock);
+	write_unlock_irqrestore(&pppol2tp_tunnel_list_lock, flags);
 	atomic_inc(&pppol2tp_tunnel_count);
 
 	/* Bump the reference count. The tunnel context is deleted
@@ -1462,6 +1469,7 @@ static int pppol2tp_connect(struct socke
 	struct pppol2tp_tunnel *tunnel;
 	struct dst_entry *dst;
 	int error = 0;
+	unsigned long irqflags;
 
 	lock_sock(sk);
 
@@ -1593,11 +1601,11 @@ static int pppol2tp_connect(struct socke
 	sk->sk_user_data = session;
 
 	/* Add session to the tunnel's hash list */
-	write_lock(&tunnel->hlist_lock);
+	write_lock_irqsave(&tunnel->hlist_lock, irqflags);
 	hlist_add_head(&session->hlist,
 		       pppol2tp_session_id_hash(tunnel,
 						session->tunnel_addr.s_session));
-	write_unlock(&tunnel->hlist_lock);
+	write_unlock_irqrestore(&tunnel->hlist_lock, irqflags);
 
 	atomic_inc(&pppol2tp_session_count);
 
@@ -2198,8 +2206,9 @@ static struct pppol2tp_session *next_ses
 	int found = 0;
 	int next = 0;
 	int i;
+	unsigned long flags;
 
-	read_lock(&tunnel->hlist_lock);
+	read_lock_irqsave(&tunnel->hlist_lock, flags);
 	for (i = 0; i < PPPOL2TP_HASH_SIZE; i++) {
 		hlist_for_each_entry(session, walk, &tunnel->session_hlist[i], hlist) {
 			if (curr == NULL) {
@@ -2217,7 +2226,7 @@ static struct pppol2tp_session *next_ses
 		}
 	}
 out:
-	read_unlock(&tunnel->hlist_lock);
+	read_unlock_irqrestore(&tunnel->hlist_lock, flags);
 	if (!found)
 		session = NULL;
 
@@ -2227,14 +2236,15 @@ out:
 static struct pppol2tp_tunnel *next_tunnel(struct pppol2tp_tunnel *curr)
 {
 	struct pppol2tp_tunnel *tunnel = NULL;
+	unsigned long flags;
 
-	read_lock(&pppol2tp_tunnel_list_lock);
+	read_lock_irqsave(&pppol2tp_tunnel_list_lock, flags);
 	if (list_is_last(&curr->list, &pppol2tp_tunnel_list)) {
 		goto out;
 	}
 	tunnel = list_entry(curr->list.next, struct pppol2tp_tunnel, list);
 out:
-	read_unlock(&pppol2tp_tunnel_list_lock);
+	read_unlock_irqrestore(&pppol2tp_tunnel_list_lock, flags);
 
 	return tunnel;
 }

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

James Chapman wrote, On 02/11/2008 10:22 AM:

> Fix locking issues in the pppol2tp driver which can cause a kernel
> crash on SMP boxes when hundreds of L2TP sessions are created/deleted
> simultaneously (ISP environment). The driver was violating read_lock()
> and write_lock() scheduling rules so we now consistently use the _irq
> variants of the lock functions.
... 

Hi,

Could you explain what exactly scheduling rules do you mean here,
and why disabling interrupts is the best solution for this?

Thanks,
Jarek P.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[James Chapman]

Jarek Poplawski wrote:
> James Chapman wrote, On 02/11/2008 10:22 AM:
> 
>> Fix locking issues in the pppol2tp driver which can cause a kernel
>> crash on SMP boxes when hundreds of L2TP sessions are created/deleted
>> simultaneously (ISP environment). The driver was violating read_lock()
>> and write_lock() scheduling rules so we now consistently use the _irq
>> variants of the lock functions.
> ... 
> 
> Hi,
> 
> Could you explain what exactly scheduling rules do you mean here,
> and why disabling interrupts is the best solution for this?

Below is example output from lockdep. The oops is reproducible when 
creating/deleting lots of sessions while passing data. The lock is being 
acquired for read and write in softirq contexts.

Is there a better way to fix this?

=================================
[ INFO: inconsistent lock state ]
2.6.24-core2 #1
---------------------------------
inconsistent {in-softirq-R} -> {softirq-on-W} usage.
openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes:
   (&tunnel->hlist_lock){---?}, at: [<f8eea157>]
pppol2tp_connect+0x517/0x6d0 [pppol2tp]
{in-softirq-R} state was registered at:
    [<c014edaf>] __lock_acquire+0x6bf/0x10a0
    [<c03ee75b>] fn_hash_lookup+0x1b/0xe0
    [<c014f804>] lock_acquire+0x74/0xa0
    [<f8ee859f>] pppol2tp_session_find+0x1f/0x80 [pppol2tp]
    [<c040427a>] _read_lock+0x2a/0x40
    [<f8ee859f>] pppol2tp_session_find+0x1f/0x80 [pppol2tp]
    [<f8ee859f>] pppol2tp_session_find+0x1f/0x80 [pppol2tp]
    [<f8ee8dc8>] pppol2tp_recv_core+0xd8/0x960 [pppol2tp]
    [<f8d3f72a>] ipt_do_table+0x23a/0x500 [ip_tables]
    [<f8ee967e>] pppol2tp_udp_encap_recv+0x2e/0x70 [pppol2tp]
    [<c0403fb4>] _read_unlock+0x14/0x20
    [<c03dd696>] udp_queue_rcv_skb+0x106/0x2a0
    [<c03ddc5a>] __udp4_lib_rcv+0x42a/0x7e0
    [<f8d57090>] ipt_hook+0x0/0x20 [iptable_filter]
    [<c03bc2da>] ip_local_deliver_finish+0xca/0x1c0
    [<c03bc23e>] ip_local_deliver_finish+0x2e/0x1c0
    [<c03bbfaf>] ip_rcv_finish+0xff/0x360
    [<c03bc6dc>] ip_rcv+0x20c/0x2a0
    [<c03bbeb0>] ip_rcv_finish+0x0/0x360
    [<c039ad87>] netif_receive_skb+0x317/0x4b0
    [<c039ab70>] netif_receive_skb+0x100/0x4b0
    [<f8d9627a>] e1000_clean_rx_irq_ps+0x28a/0x560 [e1000]
    [<f8d95ff0>] e1000_clean_rx_irq_ps+0x0/0x560 [e1000]
    [<f8d9384d>] e1000_clean+0x5d/0x290 [e1000]
    [<c039d580>] net_rx_action+0x1a0/0x2a0
    [<c039d43f>] net_rx_action+0x5f/0x2a0
    [<c0131e72>] __do_softirq+0x92/0x120
    [<c0131f78>] do_softirq+0x78/0x80
    [<c010b15a>] do_IRQ+0x4a/0xa0
    [<c0108dcc>] common_interrupt+0x24/0x34
    [<c0108dd6>] common_interrupt+0x2e/0x34
    [<c01062d6>] mwait_idle_with_hints+0x46/0x60
    [<c0106550>] mwait_idle+0x0/0x20
    [<c0106694>] cpu_idle+0x74/0xe0
    [<c0536a9a>] start_kernel+0x30a/0x3a0
    [<c0536150>] unknown_bootoption+0x0/0x1f0
    [<ffffffff>] 0xffffffff
irq event stamp: 275
hardirqs last  enabled at (275): [<c0132317>] local_bh_enable_ip+0xa7/0x120
hardirqs last disabled at (273): [<c01322a6>] local_bh_enable_ip+0x36/0x120
softirqs last  enabled at (274): [<f8eab8bc>]
ppp_register_channel+0xdc/0xf0 [ppp_generic]
softirqs last disabled at (272): [<c040410b>] _spin_lock_bh+0xb/0x40


-- 
James Chapman
Katalix Systems Ltd
http://www.katalix.com
Catalysts for your Embedded Linux software development

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote:
...
> Below is example output from lockdep. The oops is reproducible when  
> creating/deleting lots of sessions while passing data. The lock is being  
> acquired for read and write in softirq contexts.
>
> Is there a better way to fix this?
>
> =================================
> [ INFO: inconsistent lock state ]
> 2.6.24-core2 #1
> ---------------------------------
> inconsistent {in-softirq-R} -> {softirq-on-W} usage.
> openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes:
>   (&tunnel->hlist_lock){---?}, at: [<f8eea157>]
> pppol2tp_connect+0x517/0x6d0 [pppol2tp]
> {in-softirq-R} state was registered at:

IMHO, according to this, disabling bh should be enough. And if it's
like in this report: only read_lock is taken from softirqs, then this
should be necessary to change only all write_locks to write_lock_bh
(of course unless somewhere bhs are disabled already). Unless I miss
something?!

Cheers,
Jarek P.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Mon, Feb 11, 2008 at 11:49:24PM +0100, Jarek Poplawski wrote:
> On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote:
> ...
> > Below is example output from lockdep. The oops is reproducible when  
> > creating/deleting lots of sessions while passing data. The lock is being  
> > acquired for read and write in softirq contexts.

...Hmmm... And according to this, changing read_locks should be
necessary too.

Jarek P.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[James Chapman]

Jarek Poplawski wrote:
> On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote:
> ...
>> Below is example output from lockdep. The oops is reproducible when  
>> creating/deleting lots of sessions while passing data. The lock is being  
>> acquired for read and write in softirq contexts.
>>
>> Is there a better way to fix this?
>>
>> =================================
>> [ INFO: inconsistent lock state ]
>> 2.6.24-core2 #1
>> ---------------------------------
>> inconsistent {in-softirq-R} -> {softirq-on-W} usage.
>> openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes:
>>   (&tunnel->hlist_lock){---?}, at: [<f8eea157>]
>> pppol2tp_connect+0x517/0x6d0 [pppol2tp]
>> {in-softirq-R} state was registered at:
> 
> IMHO, according to this, disabling bh should be enough. And if it's
> like in this report: only read_lock is taken from softirqs, then this
> should be necessary to change only all write_locks to write_lock_bh
> (of course unless somewhere bhs are disabled already). Unless I miss
> something?!

I thought so too. I tried _bh locks first and the problem still 
occurred. Maybe I'll try it again in case I messed something up.

-- 
James Chapman
Katalix Systems Ltd
http://www.katalix.com
Catalysts for your Embedded Linux software development

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[James Chapman]

Jarek Poplawski wrote:
> On Mon, Feb 11, 2008 at 11:49:24PM +0100, Jarek Poplawski wrote:
>> On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote:
>> ...
>>> Below is example output from lockdep. The oops is reproducible when  
>>> creating/deleting lots of sessions while passing data. The lock is being  
>>> acquired for read and write in softirq contexts.
> 
> ...Hmmm... And according to this, changing read_locks should be
> necessary too.

The patch changes both read and write locks.

-- 
James Chapman
Katalix Systems Ltd
http://www.katalix.com
Catalysts for your Embedded Linux software development

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[David Miller]

From: James Chapman <jchapman@katalix.com>
Date: Mon, 11 Feb 2008 23:41:18 +0000

> Jarek Poplawski wrote:
> > On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote:
> > ...
> >> Below is example output from lockdep. The oops is reproducible when  
> >> creating/deleting lots of sessions while passing data. The lock is being  
> >> acquired for read and write in softirq contexts.
> >>
> >> Is there a better way to fix this?
> >>
> >> =================================
> >> [ INFO: inconsistent lock state ]
> >> 2.6.24-core2 #1
> >> ---------------------------------
> >> inconsistent {in-softirq-R} -> {softirq-on-W} usage.
> >> openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes:
> >>   (&tunnel->hlist_lock){---?}, at: [<f8eea157>]
> >> pppol2tp_connect+0x517/0x6d0 [pppol2tp]
> >> {in-softirq-R} state was registered at:
> > 
> > IMHO, according to this, disabling bh should be enough. And if it's
> > like in this report: only read_lock is taken from softirqs, then this
> > should be necessary to change only all write_locks to write_lock_bh
> > (of course unless somewhere bhs are disabled already). Unless I miss
> > something?!
> 
> I thought so too. I tried _bh locks first and the problem still 
> occurred. Maybe I'll try it again in case I messed something up.

I agree with Jarek here, I look at all the code paths that take
->hlist_lock and all of them are in user context or software
interrupts.

Please get a lockdep trace with the change to _bh intead of
hw interrupt protection so we can find out what that doesn't
work.

Thanks!

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Mon, Feb 11, 2008 at 11:41:18PM +0000, James Chapman wrote:
> Jarek Poplawski wrote:
>> On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote:
>> ...
>>> Below is example output from lockdep. The oops is reproducible when   
>>> creating/deleting lots of sessions while passing data. The lock is 
>>> being  acquired for read and write in softirq contexts.
>>>
>>> Is there a better way to fix this?
>>>
>>> =================================
>>> [ INFO: inconsistent lock state ]
>>> 2.6.24-core2 #1
>>> ---------------------------------
>>> inconsistent {in-softirq-R} -> {softirq-on-W} usage.
>>> openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes:
>>>   (&tunnel->hlist_lock){---?}, at: [<f8eea157>]
>>> pppol2tp_connect+0x517/0x6d0 [pppol2tp]
>>> {in-softirq-R} state was registered at:
>>
>> IMHO, according to this, disabling bh should be enough. And if it's
>> like in this report: only read_lock is taken from softirqs, then this
>> should be necessary to change only all write_locks to write_lock_bh
>> (of course unless somewhere bhs are disabled already). Unless I miss
>> something?!
>
> I thought so too. I tried _bh locks first and the problem still  
> occurred. Maybe I'll try it again in case I messed something up.

If there are any new lockdep warnings I'd be interested to have
a look. (BTW, with irqs disabling such ISP would probably get
considerable drop in performance?)

Regards,
Jarek P.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Mon, Feb 11, 2008 at 11:42:12PM +0000, James Chapman wrote:
> Jarek Poplawski wrote:
>> On Mon, Feb 11, 2008 at 11:49:24PM +0100, Jarek Poplawski wrote:
>>> On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote:
>>> ...
>>>> Below is example output from lockdep. The oops is reproducible when 
>>>>  creating/deleting lots of sessions while passing data. The lock is 
>>>> being  acquired for read and write in softirq contexts.
>>
>> ...Hmmm... And according to this, changing read_locks should be
>> necessary too.
>
> The patch changes both read and write locks.

Right! This was only "errata" to my earlier comment where I considered
only lockdep info and forgot about yours...

Sorry,
Jarek P.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[James Chapman]

David Miller wrote:
> From: James Chapman <jchapman@katalix.com>
> Date: Mon, 11 Feb 2008 23:41:18 +0000
> 
>> Jarek Poplawski wrote:
>>> On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote:
>>> ...
>>>> Below is example output from lockdep. The oops is reproducible when  
>>>> creating/deleting lots of sessions while passing data. The lock is being  
>>>> acquired for read and write in softirq contexts.
>>>>
>>>> Is there a better way to fix this?
>>>>
>>>> =================================
>>>> [ INFO: inconsistent lock state ]
>>>> 2.6.24-core2 #1
>>>> ---------------------------------
>>>> inconsistent {in-softirq-R} -> {softirq-on-W} usage.
>>>> openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes:
>>>>   (&tunnel->hlist_lock){---?}, at: [<f8eea157>]
>>>> pppol2tp_connect+0x517/0x6d0 [pppol2tp]
>>>> {in-softirq-R} state was registered at:
>>> IMHO, according to this, disabling bh should be enough. And if it's
>>> like in this report: only read_lock is taken from softirqs, then this
>>> should be necessary to change only all write_locks to write_lock_bh
>>> (of course unless somewhere bhs are disabled already). Unless I miss
>>> something?!
>> I thought so too. I tried _bh locks first and the problem still 
>> occurred. Maybe I'll try it again in case I messed something up.
> 
> I agree with Jarek here, I look at all the code paths that take
> ->hlist_lock and all of them are in user context or software
> interrupts.
> 
> Please get a lockdep trace with the change to _bh intead of
> hw interrupt protection so we can find out what that doesn't
> work.
> 
> Thanks!

Here is a trace from when we had _bh locks.

Feb  5 16:26:32  ======================================================
Feb  5 16:26:32  [ INFO: soft-safe -> soft-unsafe lock order detected ]
Feb  5 16:26:32  2.6.24-core2 #1
Feb  5 16:26:32  ------------------------------------------------------
Feb  5 16:26:32  pppd/3224 [HC0[0]:SC0[2]:HE1:SE0] is trying to acquire:
Feb  5 16:26:32  (&sk->sk_dst_lock){----}, at: [<f8efacac>] 
pppol2tp_xmit+0x23c/0x460 [pppol2tp]
Feb  5 16:26:32
Feb  5 16:26:32  and this task is already holding:
Feb  5 16:26:32  (&pch->downl){-...}, at: [<f8eb828e>] 
ppp_push+0x44e/0x620 [ppp_generic]
Feb  5 16:26:32  which would create a new lock dependency:
Feb  5 16:26:32  (&pch->downl){-...} -> (&sk->sk_dst_lock){----}
Feb  5 16:26:32
Feb  5 16:26:32  but this new dependency connects a soft-irq-safe lock:
Feb  5 16:26:32  (&pch->upl){-.-+}
Feb  5 16:26:32  ... which became soft-irq-safe at:
Feb  5 16:26:32  [<c014d87f>] check_usage_backwards+0x1f/0x50
Feb  5 16:26:32  [<c014c479>] save_trace+0x39/0xa0
Feb  5 16:26:32  [<c014edaf>] __lock_acquire+0x6bf/0x10a0
Feb  5 16:26:32  [<c014ee8e>] __lock_acquire+0x79e/0x10a0
Feb  5 16:26:32  [<c014ee8e>] __lock_acquire+0x79e/0x10a0
Feb  5 16:26:32  [<c014f804>] lock_acquire+0x74/0xa0
Feb  5 16:26:32  [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic]
Feb  5 16:26:32  [<c040425f>] _read_lock_bh+0x2f/0x40
Feb  5 16:26:32  [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic]
Feb  5 16:26:32  [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic]
Feb  5 16:26:32  [<f8ef915a>] pppol2tp_recv_core+0x46a/0x960 [pppol2tp]
Feb  5 16:26:32  [<f8ef967e>] pppol2tp_udp_encap_recv+0x2e/0x70 [pppol2tp]
Feb  5 16:26:32  [<c0403fd4>] _read_unlock+0x14/0x20
Feb  5 16:26:32  [<c03dd6b6>] udp_queue_rcv_skb+0x106/0x2a0
Feb  5 16:26:32  [<c03ddc7a>] __udp4_lib_rcv+0x42a/0x7e0
Feb  5 16:26:32  [<f8d5c090>] ipt_hook+0x0/0x20 [iptable_filter]
Feb  5 16:26:32  [<c03bc2fa>] ip_local_deliver_finish+0xca/0x1c0
Feb  5 16:26:32  [<c03bc25e>] ip_local_deliver_finish+0x2e/0x1c0
Feb  5 16:26:32  [<c03bbfcf>] ip_rcv_finish+0xff/0x360
Feb  5 16:26:32  [<c03bc6fc>] ip_rcv+0x20c/0x2a0
Feb  5 16:26:32  [<c03bbed0>] ip_rcv_finish+0x0/0x360
Feb  5 16:26:32  [<c039ad87>] netif_receive_skb+0x317/0x4b0
Feb  5 16:26:32  [<c039ab70>] netif_receive_skb+0x100/0x4b0
Feb  5 16:26:32  [<f8d7e27a>] e1000_clean_rx_irq_ps+0x28a/0x560 [e1000]
Feb  5 16:26:32  [<f8d7dff0>] e1000_clean_rx_irq_ps+0x0/0x560 [e1000]
Feb  5 16:26:32  [<f8d7b84d>] e1000_clean+0x5d/0x290 [e1000]
Feb  5 16:26:32  [<c039d580>] net_rx_action+0x1a0/0x2a0
Feb  5 16:26:32  [<c039d43f>] net_rx_action+0x5f/0x2a0
Feb  5 16:26:32  [<c0131e72>] __do_softirq+0x92/0x120
Feb  5 16:26:32  [<c0131f78>] do_softirq+0x78/0x80
Feb  5 16:26:32  [<c010b15a>] do_IRQ+0x4a/0xa0
Feb  5 16:26:32  [<c0127af0>] finish_task_switch+0x0/0xc0
Feb  5 16:26:32  [<c0108dcc>] common_interrupt+0x24/0x34
Feb  5 16:26:32  [<c0108dd6>] common_interrupt+0x2e/0x34
Feb  5 16:26:32  [<c01062d6>] mwait_idle_with_hints+0x46/0x60
Feb  5 16:26:32  [<c0106550>] mwait_idle+0x0/0x20
Feb  5 16:26:32  [<c0106694>] cpu_idle+0x74/0xe0
Feb  5 16:26:32  [<c0536a9a>] start_kernel+0x30a/0x3a0
Feb  5 16:26:32  [<c0536150>] unknown_bootoption+0x0/0x1f0
Feb  5 16:26:32  [<ffffffff>] 0xffffffff
Feb  5 16:26:32
Feb  5 16:26:32  to a soft-irq-unsafe lock:
Feb  5 16:26:32  (&sk->sk_dst_lock){----}
Feb  5 16:26:32  ... which became soft-irq-unsafe at:
Feb  5 16:26:32  ...  [<c014e02e>] mark_held_locks+0x5e/0x80
Feb  5 16:26:32  [<c014ed92>] __lock_acquire+0x6a2/0x10a0
Feb  5 16:26:32  [<c010f5b0>] save_stack_trace+0x20/0x40
Feb  5 16:26:32  [<c014c524>] add_lock_to_list+0x44/0xb0
Feb  5 16:26:32  [<c03dea29>] __udp_lib_get_port+0x19/0x200
Feb  5 16:26:32  [<c014f735>] __lock_acquire+0x1045/0x10a0
Feb  5 16:26:32  [<c014f804>] lock_acquire+0x74/0xa0
Feb  5 16:26:32  [<c03db0a3>] ip4_datagram_connect+0x53/0x380
Feb  5 16:26:32  [<c040418a>] _write_lock+0x2a/0x40
Feb  5 16:26:32  [<c03db0a3>] ip4_datagram_connect+0x53/0x380
Feb  5 16:26:32  [<c03db0a3>] ip4_datagram_connect+0x53/0x380
Feb  5 16:26:32  [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
Feb  5 16:26:32  [<c0132317>] local_bh_enable_ip+0xa7/0x120
Feb  5 16:26:32  [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
Feb  5 16:26:32  [<c040414f>] _spin_lock_bh+0x2f/0x40
Feb  5 16:26:32  [<c03e4d55>] inet_dgram_connect+0x35/0x80
Feb  5 16:26:32  [<c038ec52>] sys_connect+0x82/0xd0
Feb  5 16:26:32  [<c01455df>] down_read_trylock+0x4f/0x60
Feb  5 16:26:32  [<c011fe9c>] do_page_fault+0xfc/0x940
Feb  5 16:26:32  [<c0404024>] _spin_unlock+0x14/0x20
Feb  5 16:26:32  [<c03905f8>] sys_socketcall+0x98/0x280
Feb  5 16:26:32  [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
Feb  5 16:26:32  [<c02a86ba>] copy_to_user+0x3a/0x70
Feb  5 16:26:32  [<c0108417>] restore_nocheck+0x12/0x15
Feb  5 16:26:32  [<c01083aa>] syscall_call+0x7/0xb
Feb  5 16:26:32  [<ffffffff>] 0xffffffff
Feb  5 16:26:32
Feb  5 16:26:32  other info that might help us debug this:
Feb  5 16:26:32
Feb  5 16:26:32  2 locks held by pppd/3224:
Feb  5 16:26:32  #0:  (&ppp->wlock){-...}, at: [<f8eb8478>] 
ppp_xmit_process+0x18/0x630 [ppp_generic]
Feb  5 16:26:32  #1:  (&pch->downl){-...}, at: [<f8eb828e>] 
ppp_push+0x44e/0x620 [ppp_generic]
Feb  5 16:26:32
Feb  5 16:26:32  the soft-irq-safe lock's dependencies:
Feb  5 16:26:32  -> (&pch->upl){-.-+} ops: 0 {
Feb  5 16:26:32  initial-use  at:
Feb  5 16:26:32  [<c014e80d>] __lock_acquire+0x11d/0x10a0
Feb  5 16:26:32  [<f8efacf3>] pppol2tp_xmit+0x283/0x460 [pppol2tp]
Feb  5 16:26:32  [<c014f804>] lock_acquire+0x74/0xa0
Feb  5 16:26:32  [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic]
Feb  5 16:26:32  [<c040425f>] _read_lock_bh+0x2f/0x40
Feb  5 16:26:32  [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic]
Feb  5 16:26:32  [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic]
Feb  5 16:26:32  [<f8eb9ffd>] ppp_write+0xdd/0x110 [ppp_generic]
Feb  5 16:26:32  [<c0181f86>] vfs_write+0xa6/0x140
Feb  5 16:26:32  [<c0108417>] restore_nocheck+0x12/0x15
Feb  5 16:26:32  [<f8eb9f20>] ppp_write+0x0/0x110 [ppp_generic]
Feb  5 16:26:32  [<c0182651>] sys_write+0x41/0x70
Feb  5 16:26:32  [<c01083aa>] syscall_call+0x7/0xb
Feb  5 16:26:32  [<ffffffff>] 0xffffffff
Feb  5 16:26:32  hardirq-on-W at:
Feb  5 16:26:32  [<c014ee8e>] __lock_acquire+0x79e/0x10a0
Feb  5 16:26:32  [<c014c524>] add_lock_to_list+0x44/0xb0
Feb  5 16:26:32  [<c014ed68>] __lock_acquire+0x678/0x10a0
Feb  5 16:26:32  [<c014e02e>] mark_held_locks+0x5e/0x80
Feb  5 16:26:32  [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
Feb  5 16:26:32  [<c014f804>] lock_acquire+0x74/0xa0
Feb  5 16:26:32  [<f8eb7387>] ppp_ioctl+0x647/0xf40 [ppp_generic]
Feb  5 16:26:32  [<c04041cf>] _write_lock_bh+0x2f/0x40
Feb  5 16:26:32  [<f8eb7387>] ppp_ioctl+0x647/0xf40 [ppp_generic]
Feb  5 16:26:32  [<f8eb7387>] ppp_ioctl+0x647/0xf40 [ppp_generic]
Feb  5 16:26:32  [<c0404665>] _spin_unlock_irqrestore+0x55/0x70
Feb  5 16:26:32  [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
Feb  5 16:26:32  [<c0403e4e>] __down+0xde/0xed
Feb  5 16:26:32  [<c0124220>] default_wake_function+0x0/0x10
Feb  5 16:26:32  [<c0403bdb>] __down_failed+0x7/0xc
Feb  5 16:26:32  [<c018d8f8>] do_ioctl+0x78/0x90
Feb  5 16:26:32  [<c018d96c>] vfs_ioctl+0x5c/0x290
Feb  5 16:26:32  [<c018dbdd>] sys_ioctl+0x3d/0x70
Feb  5 16:26:32  [<c01083aa>] syscall_call+0x7/0xb
Feb  5 16:26:32  [<ffffffff>] 0xffffffff
Feb  5 16:26:32  in-softirq-R at:
Feb  5 16:26:32  [<c014d87f>] check_usage_backwards+0x1f/0x50
Feb  5 16:26:32  [<c014c479>] save_trace+0x39/0xa0
Feb  5 16:26:32  [<c014edaf>] __lock_acquire+0x6bf/0x10a0
Feb  5 16:26:32  [<c014ee8e>] __lock_acquire+0x79e/0x10a0
Feb  5 16:26:32  [<c014ee8e>] __lock_acquire+0x79e/0x10a0
Feb  5 16:26:32  [<c014f804>] lock_acquire+0x74/0xa0
Feb  5 16:26:32  [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic]
Feb  5 16:26:32  [<c040425f>] _read_lock_bh+0x2f/0x40
Feb  5 16:26:32  [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic]
Feb  5 16:26:32  [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic]
Feb  5 16:26:32  [<f8ef915a>] pppol2tp_recv_core+0x46a/0x960 [pppol2tp]
Feb  5 16:26:32  [<f8ef967e>] pppol2tp_udp_encap_recv+0x2e/0x70 [pppol2tp]
Feb  5 16:26:32  [<c0403fd4>] _read_unlock+0x14/0x20
Feb  5 16:26:32  [<c03dd6b6>] udp_queue_rcv_skb+0x106/0x2a0
Feb  5 16:26:32  [<c03ddc7a>] __udp4_lib_rcv+0x42a/0x7e0
Feb  5 16:26:32  [<f8d5c090>] ipt_hook+0x0/0x20 [iptable_filter]
Feb  5 16:26:32  [<c03bc2fa>] ip_local_deliver_finish+0xca/0x1c0
Feb  5 16:26:32  [<c03bc25e>] ip_local_deliver_finish+0x2e/0x1c0
Feb  5 16:26:32  [<c03bbfcf>] ip_rcv_finish+0xff/0x360
Feb  5 16:26:32  [<c03bc6fc>] ip_rcv+0x20c/0x2a0
Feb  5 16:26:32  [<c03bbed0>] ip_rcv_finish+0x0/0x360
Feb  5 16:26:32  [<c039ad87>] netif_receive_skb+0x317/0x4b0
Feb  5 16:26:32  [<c039ab70>] netif_receive_skb+0x100/0x4b0
Feb  5 16:26:32  [<f8d7e27a>] e1000_clean_rx_irq_ps+0x28a/0x560 [e1000]
Feb  5 16:26:32  [<f8d7dff0>] e1000_clean_rx_irq_ps+0x0/0x560 [e1000]
Feb  5 16:26:32  [<f8d7b84d>] e1000_clean+0x5d/0x290 [e1000]
Feb  5 16:26:32  [<c039d580>] net_rx_action+0x1a0/0x2a0
Feb  5 16:26:32  [<c039d43f>] net_rx_action+0x5f/0x2a0
Feb  5 16:26:32  [<c0131e72>] __do_softirq+0x92/0x120
Feb  5 16:26:32  [<c0131f78>] do_softirq+0x78/0x80
Feb  5 16:26:32  [<c010b15a>] do_IRQ+0x4a/0xa0
Feb  5 16:26:32  [<c0127af0>] finish_task_switch+0x0/0xc0
Feb  5 16:26:32  [<c0108dcc>] common_interrupt+0x24/0x34
Feb  5 16:26:32  [<c0108dd6>] common_interrupt+0x2e/0x34
Feb  5 16:26:32  [<c01062d6>] mwait_idle_with_hints+0x46/0x60
Feb  5 16:26:32  [<c0106550>] mwait_idle+0x0/0x20
Feb  5 16:26:32  [<c0106694>] cpu_idle+0x74/0xe0
Feb  5 16:26:32  [<c0536a9a>] start_kernel+0x30a/0x3a0
Feb  5 16:26:32  [<c0536150>] unknown_bootoption+0x0/0x1f0
Feb  5 16:26:32  [<ffffffff>] 0xffffffff
Feb  5 16:26:32  hardirq-on-R at:
Feb  5 16:26:32  [<c014da27>] mark_lock+0x77/0x620
Feb  5 16:26:32  [<c014ec2f>] __lock_acquire+0x53f/0x10a0
Feb  5 16:26:32  [<f8efacf3>] pppol2tp_xmit+0x283/0x460 [pppol2tp]
Feb  5 16:26:32  [<c014f804>] lock_acquire+0x74/0xa0
Feb  5 16:26:32  [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic]
Feb  5 16:26:32  [<c040425f>] _read_lock_bh+0x2f/0x40
Feb  5 16:26:32  [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic]
Feb  5 16:26:32  [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic]
Feb  5 16:26:32  [<f8eb9ffd>] ppp_write+0xdd/0x110 [ppp_generic]
Feb  5 16:26:32  [<c0181f86>] vfs_write+0xa6/0x140
Feb  5 16:26:32  [<c0108417>] restore_nocheck+0x12/0x15
Feb  5 16:26:32  [<f8eb9f20>] ppp_write+0x0/0x110 [ppp_generic]
Feb  5 16:26:32  [<c0182651>] sys_write+0x41/0x70
Feb  5 16:26:32  [<c01083aa>] syscall_call+0x7/0xb
Feb  5 16:26:32  [<ffffffff>] 0xffffffff
Feb  5 16:26:32  }
Feb  5 16:26:32  ... key      at: [<f8ebdf20>] 
__key.30415+0x0/0xffffc2e7 [ppp_generic]
Feb  5 16:26:32  -> (&ppp->wlock){-...} ops: 0 {
Feb  5 16:26:32  initial-use  at:
Feb  5 16:26:32  [<c014e80d>] __lock_acquire+0x11d/0x10a0
Feb  5 16:26:32  [<c014f804>] lock_acquire+0x74/0xa0
Feb  5 16:26:32  [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic]
Feb  5 16:26:32  [<c040414f>] _spin_lock_bh+0x2f/0x40
Feb  5 16:26:32  [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic]
Feb  5 16:26:32  [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic]
Feb  5 16:26:32  [<c0404665>] _spin_unlock_irqrestore+0x55/0x70
Feb  5 16:26:32  [<c0403e4e>] __down+0xde/0xed
Feb  5 16:26:32  [<c0124220>] default_wake_function+0x0/0x10
Feb  5 16:26:32  [<c0403bdb>] __down_failed+0x7/0xc
Feb  5 16:26:32  [<c018d8f8>] do_ioctl+0x78/0x90
Feb  5 16:26:32  [<c018d96c>] vfs_ioctl+0x5c/0x290
Feb  5 16:26:32  [<c018dbdd>] sys_ioctl+0x3d/0x70
Feb  5 16:26:32  [<c01083aa>] syscall_call+0x7/0xb
Feb  5 16:26:32  [<ffffffff>] 0xffffffff
Feb  5 16:26:32  hardirq-on-W at:
Feb  5 16:26:32  [<c014da27>] mark_lock+0x77/0x620
Feb  5 16:26:32  [<c014ed68>] __lock_acquire+0x678/0x10a0
Feb  5 16:26:32  [<c014f804>] lock_acquire+0x74/0xa0
Feb  5 16:26:32  [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic]
Feb  5 16:26:32  [<c040414f>] _spin_lock_bh+0x2f/0x40
Feb  5 16:26:32  [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic]
Feb  5 16:26:32  [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic]
Feb  5 16:26:32  [<c0404665>] _spin_unlock_irqrestore+0x55/0x70
Feb  5 16:26:32  [<c0403e4e>] __down+0xde/0xed
Feb  5 16:26:32  [<c0124220>] default_wake_function+0x0/0x10
Feb  5 16:26:32  [<c0403bdb>] __down_failed+0x7/0xc
Feb  5 16:26:32  [<c018d8f8>] do_ioctl+0x78/0x90
Feb  5 16:26:32  [<c018d96c>] vfs_ioctl+0x5c/0x290
Feb  5 16:26:32  [<c018dbdd>] sys_ioctl+0x3d/0x70
Feb  5 16:26:32  [<c01083aa>] syscall_call+0x7/0xb
Feb  5 16:26:32  [<ffffffff>] 0xffffffff
Feb  5 16:26:32  }
Feb  5 16:26:32  ... key      at: [<f8ebdf08>] 
__key.30659+0x0/0xffffc2ff [ppp_generic]
Feb  5 16:26:32  -> (&ppp->rlock){-...} ops: 0 {
Feb  5 16:26:32  initial-use  at:
Feb  5 16:26:32  [<c014e80d>] __lock_acquire+0x11d/0x10a0
Feb  5 16:26:32  [<c014f804>] lock_acquire+0x74/0xa0
Feb  5 16:26:32  [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic]
Feb  5 16:26:32  [<c040414f>] _spin_lock_bh+0x2f/0x40
Feb  5 16:26:32  [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic]
Feb  5 16:26:32  [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic]
Feb  5 16:26:32  [<c0404665>] _spin_unlock_irqrestore+0x55/0x70
Feb  5 16:26:32  [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
Feb  5 16:26:32  [<c0403e4e>] __down+0xde/0xed
Feb  5 16:26:32  [<c0124220>] default_wake_function+0x0/0x10
Feb  5 16:26:32  [<c0403bdb>] __down_failed+0x7/0xc
Feb  5 16:26:32  [<c018d8f8>] do_ioctl+0x78/0x90
Feb  5 16:26:32  [<c018d96c>] vfs_ioctl+0x5c/0x290
Feb  5 16:26:32  [<c018dbdd>] sys_ioctl+0x3d/0x70
Feb  5 16:26:32  [<c01083aa>] syscall_call+0x7/0xb
Feb  5 16:26:32  [<ffffffff>] 0xffffffff
Feb  5 16:26:32  hardirq-on-W at:
Feb  5 16:26:32  [<c014da27>] mark_lock+0x77/0x620
Feb  5 16:26:32  [<c014ed68>] __lock_acquire+0x678/0x10a0
Feb  5 16:26:32  [<c014f804>] lock_acquire+0x74/0xa0
Feb  5 16:26:32  [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic]
Feb  5 16:26:32  [<c040414f>] _spin_lock_bh+0x2f/0x40
Feb  5 16:26:32  [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic]
Feb  5 16:26:32  [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic]
Feb  5 16:26:32  [<c0404665>] _spin_unlock_irqrestore+0x55/0x70
Feb  5 16:26:32  [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
Feb  5 16:26:32  [<c0403e4e>] __down+0xde/0xed
Feb  5 16:26:32  [<c0124220>] default_wake_function+0x0/0x10
Feb  5 16:26:32  [<c0403bdb>] __down_failed+0x7/0xc
Feb  5 16:26:32  [<c018d8f8>] do_ioctl+0x78/0x90
Feb  5 16:26:32  [<c018d96c>] vfs_ioctl+0x5c/0x290
Feb  5 16:26:32  [<c018dbdd>] sys_ioctl+0x3d/0x70
Feb  5 16:26:32  [<c01083aa>] syscall_call+0x7/0xb
Feb  5 16:26:32  [<ffffffff>] 0xffffffff
Feb  5 16:26:32  }
Feb  5 16:26:32  ... key      at: [<f8ebdf10>] 
__key.30658+0x0/0xffffc2f7 [ppp_generic]
Feb  5 16:26:32  ... acquired at:
Feb  5 16:26:32  [<c014f68f>] __lock_acquire+0xf9f/0x10a0
Feb  5 16:26:32  [<f8eb76a5>] ppp_ioctl+0x965/0xf40 [ppp_generic]
Feb  5 16:26:32  [<c014f804>] lock_acquire+0x74/0xa0
Feb  5 16:26:32  [<f8eb76a5>] ppp_ioctl+0x965/0xf40 [ppp_generic]
Feb  5 16:26:32  [<c040414f>] _spin_lock_bh+0x2f/0x40
Feb  5 16:26:32  [<f8eb76a5>] ppp_ioctl+0x965/0xf40 [ppp_generic]
Feb  5 16:26:32  [<f8eb76a5>] ppp_ioctl+0x965/0xf40 [ppp_generic]
Feb  5 16:26:32  [<c0404665>] _spin_unlock_irqrestore+0x55/0x70
Feb  5 16:26:32  [<c0403e4e>] __down+0xde/0xed
Feb  5 16:26:32  [<c0124220>] default_wake_function+0x0/0x10
Feb  5 16:26:32  [<c0403bdb>] __down_failed+0x7/0xc
Feb  5 16:26:32  [<c018d8f8>] do_ioctl+0x78/0x90
Feb  5 16:26:32  [<c018d96c>] vfs_ioctl+0x5c/0x290
Feb  5 16:26:32  [<c018dbdd>] sys_ioctl+0x3d/0x70
Feb  5 16:26:32  [<c01083aa>] syscall_call+0x7/0xb
Feb  5 16:26:32  [<ffffffff>] 0xffffffff
Feb  5 16:26:32
Feb  5 16:26:32  -> (&list->lock#4){.+..} ops: 0 {
Feb  5 16:26:32  initial-use  at:
Feb  5 16:26:32  [<c014e80d>] __lock_acquire+0x11d/0x10a0
Feb  5 16:26:32  [<c0193ba2>] dput+0xa2/0x130
Feb  5 16:26:32  [<c014ee8e>] __lock_acquire+0x79e/0x10a0
Feb  5 16:26:32  [<c014f804>] lock_acquire+0x74/0xa0
Feb  5 16:26:32  [<c0394b9c>] skb_queue_tail+0x1c/0x50
Feb  5 16:26:32  [<c040450c>] _spin_lock_irqsave+0x4c/0x70
Feb  5 16:26:32  [<c0394b9c>] skb_queue_tail+0x1c/0x50
Feb  5 16:26:32  [<c0394b9c>] skb_queue_tail+0x1c/0x50
Feb  5 16:26:32  [<f8eb9fd5>] ppp_write+0xb5/0x110 [ppp_generic]
Feb  5 16:26:32  [<c0181f86>] vfs_write+0xa6/0x140
Feb  5 16:26:32  [<c0108417>] restore_nocheck+0x12/0x15
Feb  5 16:26:32  [<f8eb9f20>] ppp_write+0x0/0x110 [ppp_generic]
Feb  5 16:26:32  [<c0182651>] sys_write+0x41/0x70
Feb  5 16:26:32  [<c01083aa>] syscall_call+0x7/0xb
Feb  5 16:26:32  [<ffffffff>] 0xffffffff
Feb  5 16:26:32  in-softirq-W at:
Feb  5 16:26:32  [<c014c479>] save_trace+0x39/0xa0
Feb  5 16:26:32  [<c014db32>] mark_lock+0x182/0x620
Feb  5 16:26:32  [<c014ed36>] __lock_acquire+0x646/0x10a0
Feb  5 16:26:32  [<c014ef26>] __lock_acquire+0x836/0x10a0
Feb  5 16:26:32  [<c014ee8e>] __lock_acquire+0x79e/0x10a0
Feb  5 16:26:32  [<c014f804>] lock_acquire+0x74/0xa0
Feb  5 16:26:32  [<c0394b9c>] skb_queue_tail+0x1c/0x50
Feb  5 16:26:32  [<c040450c>] _spin_lock_irqsave+0x4c/0x70
Feb  5 16:26:32  [<c0394b9c>] skb_queue_tail+0x1c/0x50
Feb  5 16:26:32  [<c0394b9c>] skb_queue_tail+0x1c/0x50
Feb  5 16:26:32  [<f8eba0a8>] ppp_input+0x78/0x140 [ppp_generic]
Feb  5 16:26:32  [<f8ef915a>] pppol2tp_recv_core+0x46a/0x960 [pppol2tp]
Feb  5 16:26:32  [<f8ef967e>] pppol2tp_udp_encap_recv+0x2e/0x70 [pppol2tp]
Feb  5 16:26:32  [<c0403fd4>] _read_unlock+0x14/0x20
Feb  5 16:26:32  [<c03dd6b6>] udp_queue_rcv_skb+0x106/0x2a0
Feb  5 16:26:32  [<c03ddc7a>] __udp4_lib_rcv+0x42a/0x7e0
Feb  5 16:26:32  [<f8d5c090>] ipt_hook+0x0/0x20 [iptable_filter]
Feb  5 16:26:32  [<c03bc2fa>] ip_local_deliver_finish+0xca/0x1c0
Feb  5 16:26:32  [<c03bc25e>] ip_local_deliver_finish+0x2e/0x1c0
Feb  5 16:26:32  [<c03bbfcf>] ip_rcv_finish+0xff/0x360
Feb  5 16:26:32  [<c03bc6fc>] ip_rcv+0x20c/0x2a0
Feb  5 16:26:32  [<c03bbed0>] ip_rcv_finish+0x0/0x360
Feb  5 16:26:32  [<c039ad87>] netif_receive_skb+0x317/0x4b0
Feb  5 16:26:32  [<c039ab70>] netif_receive_skb+0x100/0x4b0
Feb  5 16:26:32  [<f8d7e27a>] e1000_clean_rx_irq_ps+0x28a/0x560 [e1000]
Feb  5 16:26:32  [<f8d7dff0>] e1000_clean_rx_irq_ps+0x0/0x560 [e1000]
Feb  5 16:26:32  [<f8d7b84d>] e1000_clean+0x5d/0x290 [e1000]
Feb  5 16:26:32  [<c039d580>] net_rx_action+0x1a0/0x2a0
Feb  5 16:26:32  [<c039d43f>] net_rx_action+0x5f/0x2a0
Feb  5 16:26:32  [<c0131e72>] __do_softirq+0x92/0x120
Feb  5 16:26:32  [<c0131f78>] do_softirq+0x78/0x80
Feb  5 16:26:32  [<c010b15a>] do_IRQ+0x4a/0xa0
Feb  5 16:26:32  [<c0127af0>] finish_task_switch+0x0/0xc0
Feb  5 16:26:32  [<c0108dcc>] common_interrupt+0x24/0x34
Feb  5 16:26:32  [<c0108dd6>] common_interrupt+0x2e/0x34
Feb  5 16:26:32  [<c01062d6>] mwait_idle_with_hints+0x46/0x60
Feb  5 16:26:32  [<c0106550>] mwait_idle+0x0/0x20
Feb  5 16:26:32  [<c0106694>] cpu_idle+0x74/0xe0
Feb  5 16:26:32  [<c0536a9a>] start_kernel+0x30a/0x3a0

-- 
James Chapman
Katalix Systems Ltd
http://www.katalix.com
Catalysts for your Embedded Linux software development

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Tue, Feb 12, 2008 at 10:58:21AM +0000, James Chapman wrote:
...
> Here is a trace from when we had _bh locks.

Very nice...

...But since it's quite long, and if you don't know all these paths
this could take some time, maybe one question: so if lockdep got these
locks right (sometimes it can be wrong when the same structures are
nested), then it seems some problem is with this place below. This
lock is taken for writing with softirqs enabled here, and IMHO it
would be interesting to test if changing this is enough for lockdep.
It seems this is in ip4_datagram_connect() during sk_dst_reset() or
sk_dst_set(). So maybe you could try with local_bh_disable/enable()
around them (or maybe some better idea)? Anyway, I'll try to learn
this more in the meantime.

Jarek P.

> Feb  5 16:26:32  to a soft-irq-unsafe lock:
> Feb  5 16:26:32  (&sk->sk_dst_lock){----}
> Feb  5 16:26:32  ... which became soft-irq-unsafe at:
> Feb  5 16:26:32  ...  [<c014e02e>] mark_held_locks+0x5e/0x80
> Feb  5 16:26:32  [<c014ed92>] __lock_acquire+0x6a2/0x10a0
> Feb  5 16:26:32  [<c010f5b0>] save_stack_trace+0x20/0x40
> Feb  5 16:26:32  [<c014c524>] add_lock_to_list+0x44/0xb0
> Feb  5 16:26:32  [<c03dea29>] __udp_lib_get_port+0x19/0x200
> Feb  5 16:26:32  [<c014f735>] __lock_acquire+0x1045/0x10a0
> Feb  5 16:26:32  [<c014f804>] lock_acquire+0x74/0xa0
> Feb  5 16:26:32  [<c03db0a3>] ip4_datagram_connect+0x53/0x380
> Feb  5 16:26:32  [<c040418a>] _write_lock+0x2a/0x40
> Feb  5 16:26:32  [<c03db0a3>] ip4_datagram_connect+0x53/0x380
> Feb  5 16:26:32  [<c03db0a3>] ip4_datagram_connect+0x53/0x380
> Feb  5 16:26:32  [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
> Feb  5 16:26:32  [<c0132317>] local_bh_enable_ip+0xa7/0x120
> Feb  5 16:26:32  [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
> Feb  5 16:26:32  [<c040414f>] _spin_lock_bh+0x2f/0x40
> Feb  5 16:26:32  [<c03e4d55>] inet_dgram_connect+0x35/0x80
> Feb  5 16:26:32  [<c038ec52>] sys_connect+0x82/0xd0
> Feb  5 16:26:32  [<c01455df>] down_read_trylock+0x4f/0x60
> Feb  5 16:26:32  [<c011fe9c>] do_page_fault+0xfc/0x940
> Feb  5 16:26:32  [<c0404024>] _spin_unlock+0x14/0x20
> Feb  5 16:26:32  [<c03905f8>] sys_socketcall+0x98/0x280
> Feb  5 16:26:32  [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
> Feb  5 16:26:32  [<c02a86ba>] copy_to_user+0x3a/0x70
> Feb  5 16:26:32  [<c0108417>] restore_nocheck+0x12/0x15
> Feb  5 16:26:32  [<c01083aa>] syscall_call+0x7/0xb
> Feb  5 16:26:32  [<ffffffff>] 0xffffffff

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[David Miller]

From: James Chapman <jchapman@katalix.com>
Date: Tue, 12 Feb 2008 10:58:21 +0000

> Here is a trace from when we had _bh locks.

The problem is that the pppol2tp code calls sk_dst_get() in software
interrupt context and that is not allowed.

sk_dst_get() grabs sk->sk_dst_lock without any BH enabling or
disabling.

It can do that because the usage is to make all the lock
taking calls in user context, and in the packet processing
paths use __sk_dst_get().

Probably what the pppol2tp code should do is use __sk_dst_check()
instead of sk_dst_get().  You then have to be able to handle
NULL returns, just like UDP sendmsg() does, which means you'll
need to cook up a routing lookup if __sk_dst_check() gives you
NULL because the route became obsolete.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Tue, Feb 12, 2008 at 10:00:03PM -0800, David Miller wrote:
> From: James Chapman <jchapman@katalix.com>
> Date: Tue, 12 Feb 2008 10:58:21 +0000
> 
> > Here is a trace from when we had _bh locks.
> 
> The problem is that the pppol2tp code calls sk_dst_get() in software
> interrupt context and that is not allowed.

Actually, this lockdep report probably warns of something else:
sk_dst_lock which is seen in process context with softirqs enabled
implicitly endangers some other (ppp_generic) locks, which depend
on ppp_generic pch->upl read_lock, which is taken in softirq context.

I can't see on this report any hardirqs dependancies, so it seems even
blocking hard interrupts, just like in this patch, shouldn't solve
this problem: if BHs were really disabled in exactly the same places,
then it seems this warning should trigger in both variants.

I studied long ago some similar problem with pppoe, and it looked like
ppp_generic needed some fix, but now I've to recall or re-learn almost
all and it needs more time... Anyway, there is a possibility, this
warning isn't so dangerous.

> sk_dst_get() grabs sk->sk_dst_lock without any BH enabling or
> disabling.
> 
> It can do that because the usage is to make all the lock
> taking calls in user context, and in the packet processing
> paths use __sk_dst_get().

BTW, does it mean that ip4_datagram_connect() can be used only in user
context?

> Probably what the pppol2tp code should do is use __sk_dst_check()
> instead of sk_dst_get().  You then have to be able to handle
> NULL returns, just like UDP sendmsg() does, which means you'll
> need to cook up a routing lookup if __sk_dst_check() gives you
> NULL because the route became obsolete.

I think that without changing ppp_generic or changing ip_queue_xmit()
to something else in pppol2tp this would be really hard to avoid such
a warning (and maybe not very necessary).

Regards,
Jarek P.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

Hi,

It seems, this nice report is still uncomplete: could you check if
there could have been something more yet?

Thanks,
Jarek P.

On Tue, Feb 12, 2008 at 10:58:21AM +0000, James Chapman wrote:
...
> Here is a trace from when we had _bh locks.
>
> Feb  5 16:26:32  ======================================================
> Feb  5 16:26:32  [ INFO: soft-safe -> soft-unsafe lock order detected ]
> Feb  5 16:26:32  2.6.24-core2 #1
> Feb  5 16:26:32  ------------------------------------------------------
> Feb  5 16:26:32  pppd/3224 [HC0[0]:SC0[2]:HE1:SE0] is trying to acquire:
...
> Feb  5 16:26:32  [<f8d7b84d>] e1000_clean+0x5d/0x290 [e1000]
> Feb  5 16:26:32  [<c039d580>] net_rx_action+0x1a0/0x2a0
> Feb  5 16:26:32  [<c039d43f>] net_rx_action+0x5f/0x2a0
> Feb  5 16:26:32  [<c0131e72>] __do_softirq+0x92/0x120
> Feb  5 16:26:32  [<c0131f78>] do_softirq+0x78/0x80
> Feb  5 16:26:32  [<c010b15a>] do_IRQ+0x4a/0xa0
> Feb  5 16:26:32  [<c0127af0>] finish_task_switch+0x0/0xc0
> Feb  5 16:26:32  [<c0108dcc>] common_interrupt+0x24/0x34
> Feb  5 16:26:32  [<c0108dd6>] common_interrupt+0x2e/0x34
> Feb  5 16:26:32  [<c01062d6>] mwait_idle_with_hints+0x46/0x60
> Feb  5 16:26:32  [<c0106550>] mwait_idle+0x0/0x20
> Feb  5 16:26:32  [<c0106694>] cpu_idle+0x74/0xe0
> Feb  5 16:26:32  [<c0536a9a>] start_kernel+0x30a/0x3a0
>
> -- 
> James Chapman
> Katalix Systems Ltd
> http://www.katalix.com
> Catalysts for your Embedded Linux software development
>

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[James Chapman]

Jarek Poplawski wrote:
> Hi,
> 
> It seems, this nice report is still uncomplete: could you check if
> there could have been something more yet?

Unfortunately the ISP's syslog stops. But I've been able to borrow two Quad Xeon boxes and have reproduced the problem.

Here's a new version of the patch. The patch avoids disabling irqs and fixes the sk_dst_get() usage that DaveM mentioned. But even with this patch, lockdep still complains if hundreds of ppp sessions are inserted into a tunnel as rapidly as possible (lockdep trace is below). I can stop these errors by wrapping the call to ppp_input() in pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is a better fix?

btw, I'm not sending this from my usual mail client so the patch will probably be mangled; I'm sending it only for feedback / discussion at this stage.

Index: linux-2.6.24.2/drivers/net/pppol2tp.c
===================================================================
--- linux-2.6.24.2.orig/drivers/net/pppol2tp.c
+++ linux-2.6.24.2/drivers/net/pppol2tp.c
@@ -302,14 +302,14 @@ pppol2tp_session_find(struct pppol2tp_tu
 	struct pppol2tp_session *session;
 	struct hlist_node *walk;
 
-	read_lock(&tunnel->hlist_lock);
+	read_lock_bh(&tunnel->hlist_lock);
 	hlist_for_each_entry(session, walk, session_list, hlist) {
 		if (session->tunnel_addr.s_session == session_id) {
-			read_unlock(&tunnel->hlist_lock);
+			read_unlock_bh(&tunnel->hlist_lock);
 			return session;
 		}
 	}
-	read_unlock(&tunnel->hlist_lock);
+	read_unlock_bh(&tunnel->hlist_lock);
 
 	return NULL;
 }
@@ -320,14 +320,14 @@ static struct pppol2tp_tunnel *pppol2tp_
 {
 	struct pppol2tp_tunnel *tunnel = NULL;
 
-	read_lock(&pppol2tp_tunnel_list_lock);
+	read_lock_bh(&pppol2tp_tunnel_list_lock);
 	list_for_each_entry(tunnel, &pppol2tp_tunnel_list, list) {
 		if (tunnel->stats.tunnel_id == tunnel_id) {
-			read_unlock(&pppol2tp_tunnel_list_lock);
+			read_unlock_bh(&pppol2tp_tunnel_list_lock);
 			return tunnel;
 		}
 	}
-	read_unlock(&pppol2tp_tunnel_list_lock);
+	read_unlock_bh(&pppol2tp_tunnel_list_lock);
 
 	return NULL;
 }
@@ -344,7 +344,7 @@ static void pppol2tp_recv_queue_skb(stru
 	struct sk_buff *skbp;
 	u16 ns = PPPOL2TP_SKB_CB(skb)->ns;
 
-	spin_lock(&session->reorder_q.lock);
+	spin_lock_bh(&session->reorder_q.lock);
 	skb_queue_walk(&session->reorder_q, skbp) {
 		if (PPPOL2TP_SKB_CB(skbp)->ns > ns) {
 			__skb_insert(skb, skbp->prev, skbp, &session->reorder_q);
@@ -360,7 +360,7 @@ static void pppol2tp_recv_queue_skb(stru
 	__skb_queue_tail(&session->reorder_q, skb);
 
 out:
-	spin_unlock(&session->reorder_q.lock);
+	spin_unlock_bh(&session->reorder_q.lock);
 }
 
 /* Dequeue a single skb.
@@ -442,7 +442,7 @@ static void pppol2tp_recv_dequeue(struct
 	 * expect to send up next, dequeue it and any other
 	 * in-sequence packets behind it.
 	 */
-	spin_lock(&session->reorder_q.lock);
+	spin_lock_bh(&session->reorder_q.lock);
 	skb_queue_walk_safe(&session->reorder_q, skb, tmp) {
 		if (time_after(jiffies, PPPOL2TP_SKB_CB(skb)->expires)) {
 			session->stats.rx_seq_discards++;
@@ -469,13 +469,13 @@ static void pppol2tp_recv_dequeue(struct
 				goto out;
 			}
 		}
-		spin_unlock(&session->reorder_q.lock);
+		spin_unlock_bh(&session->reorder_q.lock);
 		pppol2tp_recv_dequeue_skb(session, skb);
-		spin_lock(&session->reorder_q.lock);
+		spin_lock_bh(&session->reorder_q.lock);
 	}
 
 out:
-	spin_unlock(&session->reorder_q.lock);
+	spin_unlock_bh(&session->reorder_q.lock);
 }
 
 /* Internal receive frame. Do the real work of receiving an L2TP data frame
@@ -964,6 +964,7 @@ static int pppol2tp_xmit(struct ppp_chan
 	static const u8 ppph[2] = { 0xff, 0x03 };
 	struct sock *sk = (struct sock *) chan->private;
 	struct sock *sk_tun;
+	struct rtable *rt;
 	int hdr_len;
 	struct pppol2tp_session *session;
 	struct pppol2tp_tunnel *tunnel;
@@ -1057,8 +1058,32 @@ static int pppol2tp_xmit(struct ppp_chan
 	nf_reset(skb);
 
 	/* Get routing info from the tunnel socket */
-	dst_release(skb->dst);
-	skb->dst = sk_dst_get(sk_tun);
+	rt = (struct rtable *)__sk_dst_check(sk_tun, 0);
+	if (rt == NULL) {
+		struct flowi fl = { .oif = sk_tun->sk_bound_dev_if,
+				    .nl_u = { .ip4_u =
+					      { .daddr = inet->daddr,
+						.saddr = inet->saddr,
+						.tos = inet->tos } },
+				    .proto = sk_tun->sk_protocol,
+				    .uli_u = { .ports =
+					       { .sport = inet->sport,
+						 .dport = inet->dport } } };
+		security_sk_classify_flow(sk_tun, &fl);
+		rc = ip_route_output_flow(&rt, &fl, sk_tun, 1);
+		if (rc) {
+			if (rc == -ENETUNREACH)
+				IP_INC_STATS_BH(IPSTATS_MIB_OUTNOROUTES);
+			goto abort;
+		}
+
+		sk_dst_set(sk_tun, dst_clone(&rt->u.dst));
+	}
+
+	write_lock_bh(&sk_tun->sk_dst_lock);
+	skb->dst = __sk_dst_get(sk_tun);
+	dst_hold(skb->dst);
+	write_unlock_bh(&sk_tun->sk_dst_lock);
 	skb_orphan(skb);
 	skb->sk = sk_tun;
 
@@ -1106,7 +1131,7 @@ static void pppol2tp_tunnel_closeall(str
 	PRINTK(tunnel->debug, PPPOL2TP_MSG_CONTROL, KERN_INFO,
 	       "%s: closing all sessions...\n", tunnel->name);
 
-	write_lock(&tunnel->hlist_lock);
+	write_lock_bh(&tunnel->hlist_lock);
 	for (hash = 0; hash < PPPOL2TP_HASH_SIZE; hash++) {
 again:
 		hlist_for_each_safe(walk, tmp, &tunnel->session_hlist[hash]) {
@@ -1126,7 +1151,7 @@ again:
 			 * disappear as we're jumping between locks.
 			 */
 			sock_hold(sk);
-			write_unlock(&tunnel->hlist_lock);
+			write_unlock_bh(&tunnel->hlist_lock);
 			lock_sock(sk);
 
 			if (sk->sk_state & (PPPOX_CONNECTED | PPPOX_BOUND)) {
@@ -1148,11 +1173,11 @@ again:
 			 * list so we are guaranteed to make forward
 			 * progress.
 			 */
-			write_lock(&tunnel->hlist_lock);
+			write_lock_bh(&tunnel->hlist_lock);
 			goto again;
 		}
 	}
-	write_unlock(&tunnel->hlist_lock);
+	write_unlock_bh(&tunnel->hlist_lock);
 }
 
 /* Really kill the tunnel.
@@ -1161,9 +1186,9 @@ again:
 static void pppol2tp_tunnel_free(struct pppol2tp_tunnel *tunnel)
 {
 	/* Remove from socket list */
-	write_lock(&pppol2tp_tunnel_list_lock);
+	write_lock_bh(&pppol2tp_tunnel_list_lock);
 	list_del_init(&tunnel->list);
-	write_unlock(&pppol2tp_tunnel_list_lock);
+	write_unlock_bh(&pppol2tp_tunnel_list_lock);
 
 	atomic_dec(&pppol2tp_tunnel_count);
 	kfree(tunnel);
@@ -1239,9 +1264,9 @@ static void pppol2tp_session_destruct(st
 				/* Delete the session socket from the
 				 * hash
 				 */
-				write_lock(&tunnel->hlist_lock);
+				write_lock_bh(&tunnel->hlist_lock);
 				hlist_del_init(&session->hlist);
-				write_unlock(&tunnel->hlist_lock);
+				write_unlock_bh(&tunnel->hlist_lock);
 
 				atomic_dec(&pppol2tp_session_count);
 			}
@@ -1386,9 +1411,9 @@ static struct sock *pppol2tp_prepare_tun
 
 	/* Add tunnel to our list */
 	INIT_LIST_HEAD(&tunnel->list);
-	write_lock(&pppol2tp_tunnel_list_lock);
+	write_lock_bh(&pppol2tp_tunnel_list_lock);
 	list_add(&tunnel->list, &pppol2tp_tunnel_list);
-	write_unlock(&pppol2tp_tunnel_list_lock);
+	write_unlock_bh(&pppol2tp_tunnel_list_lock);
 	atomic_inc(&pppol2tp_tunnel_count);
 
 	/* Bump the reference count. The tunnel context is deleted
@@ -1546,14 +1571,15 @@ static int pppol2tp_connect(struct socke
 	session->mtu = session->mru = 1500 - PPPOL2TP_HEADER_OVERHEAD;
 
 	/* If PMTU discovery was enabled, use the MTU that was discovered */
-	dst = sk_dst_get(sk);
+	read_lock_bh(&tunnel_sock->sk_dst_lock);
+	dst = __sk_dst_get(sk);
 	if (dst != NULL) {
-		u32 pmtu = dst_mtu(__sk_dst_get(sk));
+		u32 pmtu = dst_mtu(dst);
 		if (pmtu != 0)
 			session->mtu = session->mru = pmtu -
 				PPPOL2TP_HEADER_OVERHEAD;
-		dst_release(dst);
 	}
+	read_unlock_bh(&tunnel_sock->sk_dst_lock);
 
 	/* Special case: if source & dest session_id == 0x0000, this socket is
 	 * being created to manage the tunnel. Don't add the session to the
@@ -1593,11 +1619,11 @@ static int pppol2tp_connect(struct socke
 	sk->sk_user_data = session;
 
 	/* Add session to the tunnel's hash list */
-	write_lock(&tunnel->hlist_lock);
+	write_lock_bh(&tunnel->hlist_lock);
 	hlist_add_head(&session->hlist,
 		       pppol2tp_session_id_hash(tunnel,
 						session->tunnel_addr.s_session));
-	write_unlock(&tunnel->hlist_lock);
+	write_unlock_bh(&tunnel->hlist_lock);
 
 	atomic_inc(&pppol2tp_session_count);
 
@@ -2199,7 +2225,7 @@ static struct pppol2tp_session *next_ses
 	int next = 0;
 	int i;
 
-	read_lock(&tunnel->hlist_lock);
+	read_lock_bh(&tunnel->hlist_lock);
 	for (i = 0; i < PPPOL2TP_HASH_SIZE; i++) {
 		hlist_for_each_entry(session, walk, &tunnel->session_hlist[i], hlist) {
 			if (curr == NULL) {
@@ -2217,7 +2243,7 @@ static struct pppol2tp_session *next_ses
 		}
 	}
 out:
-	read_unlock(&tunnel->hlist_lock);
+	read_unlock_bh(&tunnel->hlist_lock);
 	if (!found)
 		session = NULL;
 
@@ -2228,13 +2254,13 @@ static struct pppol2tp_tunnel *next_tunn
 {
 	struct pppol2tp_tunnel *tunnel = NULL;
 
-	read_lock(&pppol2tp_tunnel_list_lock);
+	read_lock_bh(&pppol2tp_tunnel_list_lock);
 	if (list_is_last(&curr->list, &pppol2tp_tunnel_list)) {
 		goto out;
 	}
 	tunnel = list_entry(curr->list.next, struct pppol2tp_tunnel, list);
 out:
-	read_unlock(&pppol2tp_tunnel_list_lock);
+	read_unlock_bh(&pppol2tp_tunnel_list_lock);
 
 	return tunnel;
 }

......................................................
Here's the lockdep trace (long).

Feb 18 20:56:18 localhost kernel: 
Feb 18 20:56:18 localhost kernel: ======================================================
Feb 18 20:56:18 localhost kernel: [ INFO: soft-safe -> soft-unsafe lock order detected ]
Feb 18 20:56:18 localhost kernel: 2.6.24.2 #1
Feb 18 20:56:18 localhost kernel: ------------------------------------------------------
Feb 18 20:56:18 localhost kernel: pppd/3241 [HC0[0]:SC0[2]:HE1:SE0] is trying to acquire:
Feb 18 20:56:18 localhost kernel:  (&pch->downl){-...}, at: [<f8c566db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 20:56:18 localhost kernel: 
Feb 18 20:56:18 localhost kernel: and this task is already holding:
Feb 18 20:56:18 localhost kernel:  (&ppp->wlock){-...}, at: [<f8c56f10>] ppp_xmit_process+0x15/0x5a1 [ppp_generic]
Feb 18 20:56:18 localhost kernel: which would create a new lock dependency:
Feb 18 20:56:18 localhost kernel:  (&ppp->wlock){-...} -> (&pch->downl){-...}
Feb 18 20:56:18 localhost kernel: 
Feb 18 20:56:18 localhost kernel: but this new dependency connects a soft-irq-safe lock:
Feb 18 20:56:18 localhost kernel:  (&pch->upl){-.-+}
Feb 18 20:56:18 localhost kernel: ... which became soft-irq-safe at:
Feb 18 20:56:18 localhost kernel:   [<c0448013>] check_usage_backwards+0x19/0x41
Feb 18 20:56:18 localhost kernel:   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:18 localhost kernel:   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:18 localhost kernel:   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:18 localhost kernel:   [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 20:56:18 localhost kernel:   [<c061b373>] _read_lock_bh+0x2e/0x39
Feb 18 20:56:18 localhost kernel:   [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 20:56:18 localhost kernel:   [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 20:56:18 localhost kernel:   [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 20:56:18 localhost kernel:   [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 20:56:18 localhost kernel:   [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 20:56:18 localhost kernel:   [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 20:56:18 localhost kernel:   [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 20:56:18 localhost kernel:   [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 20:56:18 localhost kernel:   [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 20:56:18 localhost kernel:   [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 20:56:18 localhost kernel:   [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:18 localhost kernel:   [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 20:56:18 localhost kernel:   [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 20:56:18 localhost kernel:   [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:18 localhost kernel:   [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 20:56:18 localhost kernel:   [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 20:56:18 localhost kernel:   [<f8984f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 20:56:19 localhost kernel:   [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 20:56:19 localhost kernel:   [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 20:56:19 localhost kernel:   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:19 localhost kernel:   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:19 localhost kernel:   [<ffffffff>] 0xffffffff
Feb 18 20:56:19 localhost kernel: 
Feb 18 20:56:19 localhost kernel: to a soft-irq-unsafe lock:
Feb 18 20:56:19 localhost kernel:  (&sk->sk_dst_lock){----}
Feb 18 20:56:19 localhost kernel: ... which became soft-irq-unsafe at:
Feb 18 20:56:19 localhost kernel: ...  [<c0449046>] __lock_acquire+0x48b/0xbf1
Feb 18 20:56:19 localhost kernel:   [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 20:56:19 localhost kernel:   [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 20:56:19 localhost kernel:   [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 20:56:19 localhost kernel:   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:19 localhost kernel:   [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:56:19 localhost kernel:   [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 20:56:19 localhost kernel:   [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:56:19 localhost kernel:   [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:56:19 localhost kernel:   [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 20:56:19 localhost kernel:   [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 20:56:19 localhost kernel:   [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 20:56:19 localhost kernel:   [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 20:56:19 localhost kernel:   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:19 localhost kernel:   [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 20:56:19 localhost kernel:   [<ffffffff>] 0xffffffff
Feb 18 20:56:19 localhost kernel: 
Feb 18 20:56:19 localhost kernel: other info that might help us debug this:
Feb 18 20:56:19 localhost kernel: 
Feb 18 20:56:19 localhost kernel: 1 lock held by pppd/3241:
Feb 18 20:56:19 localhost kernel:  #0:  (&ppp->wlock){-...}, at: [<f8c56f10>] ppp_xmit_process+0x15/0x5a1 [ppp_generic]
Feb 18 20:56:19 localhost kernel: 
Feb 18 20:56:19 localhost kernel: the soft-irq-safe lock's dependencies:
Feb 18 20:56:19 localhost kernel: -> (&pch->upl){-.-+} ops: 13 {
Feb 18 20:56:19 localhost kernel:    initial-use  at:
Feb 18 20:56:19 localhost kernel:                         [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:56:19 localhost kernel:                         [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:19 localhost kernel:                         [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:19 localhost kernel:                         [<f8c57b7c>] ppp_ioctl+0x4df/0xc06 [ppp_generic]
Feb 18 20:56:19 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:19 localhost kernel:                         [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 20:56:19 localhost kernel:                         [<c061b300>] _write_lock_bh+0x2e/0x39
Feb 18 20:56:20 localhost kernel:                         [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 20:56:20 localhost kernel:                         [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 20:56:20 localhost kernel:                         [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:20 localhost kernel:                         [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:20 localhost kernel:                         [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:20 localhost kernel:                         [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:20 localhost kernel:                         [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:56:20 localhost kernel:                         [<c061ae17>] __down_failed+0x7/0xc
Feb 18 20:56:20 localhost kernel:                         [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 20:56:20 localhost kernel:                         [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 20:56:20 localhost kernel:                         [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 20:56:20 localhost kernel:                         [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:56:20 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 20:56:20 localhost kernel:    hardirq-on-W at:
Feb 18 20:56:20 localhost kernel:                         [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:20 localhost kernel:                         [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:56:20 localhost kernel:                         [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 20:56:20 localhost kernel:                         [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:20 localhost kernel:                         [<f8c57b7c>] ppp_ioctl+0x4df/0xc06 [ppp_generic]
Feb 18 20:56:20 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:20 localhost kernel:                         [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 20:56:20 localhost kernel:                         [<c061b300>] _write_lock_bh+0x2e/0x39
Feb 18 20:56:20 localhost kernel:                         [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 20:56:20 localhost kernel:                         [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 20:56:20 localhost kernel:                         [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:20 localhost kernel:                         [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:20 localhost kernel:                         [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:20 localhost kernel:                         [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:20 localhost kernel:                         [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:56:20 localhost kernel:                         [<c061ae17>] __down_failed+0x7/0xc
Feb 18 20:56:20 localhost kernel:                         [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 20:56:20 localhost kernel:                         [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 20:56:20 localhost kernel:                         [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 20:56:20 localhost kernel:                         [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:56:20 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 20:56:20 localhost kernel:    in-softirq-R at:
Feb 18 20:56:20 localhost kernel:                         [<c0448013>] check_usage_backwards+0x19/0x41
Feb 18 20:56:20 localhost kernel:                         [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:20 localhost kernel:                         [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:20 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:20 localhost kernel:                         [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 20:56:21 localhost kernel:                         [<c061b373>] _read_lock_bh+0x2e/0x39
Feb 18 20:56:21 localhost kernel:                         [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 20:56:21 localhost kernel:                         [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 20:56:21 localhost kernel:                         [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 20:56:21 localhost kernel:                         [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 20:56:21 localhost kernel:                         [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 20:56:21 localhost kernel:                         [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 20:56:21 localhost kernel:                         [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 20:56:21 localhost kernel:                         [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 20:56:21 localhost kernel:                         [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 20:56:21 localhost kernel:                         [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 20:56:21 localhost kernel:                         [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:21 localhost kernel:                         [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 20:56:21 localhost kernel:                         [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 20:56:21 localhost kernel:                         [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:21 localhost kernel:                         [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 20:56:21 localhost kernel:                         [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 20:56:21 localhost kernel:                         [<f8984f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 20:56:21 localhost kernel:                         [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 20:56:21 localhost kernel:                         [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 20:56:21 localhost kernel:                         [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:21 localhost kernel:                         [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:21 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 20:56:21 localhost kernel:    hardirq-on-R at:
Feb 18 20:56:21 localhost kernel:                         [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 20:56:21 localhost kernel:                         [<c0449001>] __lock_acquire+0x446/0xbf1
Feb 18 20:56:21 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:21 localhost kernel:                         [<f8c5834b>] ppp_channel_push+0x71/0x90 [ppp_generic]
Feb 18 20:56:21 localhost kernel:                         [<c061b373>] _read_lock_bh+0x2e/0x39
Feb 18 20:56:21 localhost kernel:                         [<f8c5834b>] ppp_channel_push+0x71/0x90 [ppp_generic]
Feb 18 20:56:21 localhost kernel:                         [<f8c5834b>] ppp_channel_push+0x71/0x90 [ppp_generic]
Feb 18 20:56:21 localhost kernel:                         [<f8c593bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 20:56:21 localhost kernel:                         [<f8c592ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 20:56:21 localhost kernel:                         [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 20:56:21 localhost kernel:                         [<c047ecfd>] sys_write+0x41/0x67
Feb 18 20:56:21 localhost kernel:                         [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:56:21 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 20:56:21 localhost kernel:  }
Feb 18 20:56:21 localhost kernel:  ... key      at: [<f8c5dd20>] __key.29741+0x0/0xffffb822 [ppp_generic]
Feb 18 20:56:21 localhost kernel:  -> (&ppp->wlock){-...} ops: 11 {
Feb 18 20:56:22 localhost kernel:     initial-use  at:
Feb 18 20:56:22 localhost kernel:                           [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:22 localhost kernel:                           [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 20:56:22 localhost kernel:                           [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:22 localhost kernel:                           [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:56:22 localhost kernel:                           [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 20:56:22 localhost kernel:                           [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:56:22 localhost kernel:                           [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:56:22 localhost kernel:                           [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:22 localhost kernel:                           [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:22 localhost kernel:                           [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:22 localhost kernel:                           [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:22 localhost kernel:                           [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:56:22 localhost kernel:                           [<c061ae17>] __down_failed+0x7/0xc
Feb 18 20:56:22 localhost kernel:                           [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 20:56:22 localhost kernel:                           [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 20:56:22 localhost kernel:                           [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 20:56:22 localhost kernel:                           [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:56:22 localhost kernel:                           [<ffffffff>] 0xffffffff
Feb 18 20:56:22 localhost kernel:     hardirq-on-W at:
Feb 18 20:56:22 localhost kernel:                           [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 20:56:22 localhost kernel:                           [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 20:56:22 localhost kernel:                           [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:22 localhost kernel:                           [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:56:22 localhost kernel:                           [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 20:56:22 localhost kernel:                           [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:56:22 localhost kernel:                           [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:56:22 localhost kernel:                           [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:22 localhost kernel:                           [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:22 localhost kernel:                           [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:22 localhost kernel:                           [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:22 localhost kernel:                           [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:56:22 localhost kernel:                           [<c061ae17>] __down_failed+0x7/0xc
Feb 18 20:56:22 localhost kernel:                           [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 20:56:22 localhost kernel:                           [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 20:56:22 localhost kernel:                           [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 20:56:22 localhost kernel:                           [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:56:22 localhost kernel:                           [<ffffffff>] 0xffffffff
Feb 18 20:56:22 localhost kernel:   }
Feb 18 20:56:22 localhost kernel:   ... key      at: [<f8c5dd08>] __key.29980+0x0/0xffffb83a [ppp_generic]
Feb 18 20:56:22 localhost kernel:   -> (&ppp->rlock){-+..} ops: 5 {
Feb 18 20:56:22 localhost kernel:      initial-use  at:
Feb 18 20:56:22 localhost kernel:                             [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:23 localhost kernel:                             [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:56:23 localhost kernel:                             [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:23 localhost kernel:                             [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:56:23 localhost kernel:                             [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 20:56:23 localhost kernel:                             [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:56:23 localhost kernel:                             [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:56:23 localhost kernel:                             [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:23 localhost kernel:                             [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:23 localhost kernel:                             [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:23 localhost kernel:                             [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:23 localhost kernel:                             [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:56:23 localhost kernel:                             [<c061ae17>] __down_failed+0x7/0xc
Feb 18 20:56:23 localhost kernel:                             [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 20:56:23 localhost kernel:                             [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 20:56:23 localhost kernel:                             [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 20:56:23 localhost kernel:                             [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:56:23 localhost kernel:                             [<ffffffff>] 0xffffffff
Feb 18 20:56:23 localhost kernel:      in-softirq-W at:
Feb 18 20:56:23 localhost kernel:                             [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:56:23 localhost kernel:                             [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:23 localhost kernel:                             [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:23 localhost kernel:                             [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:23 localhost kernel:                             [<f8c59473>] ppp_input+0xab/0xef [ppp_generic]
Feb 18 20:56:23 localhost kernel:                             [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 20:56:23 localhost kernel:                             [<f8c59473>] ppp_input+0xab/0xef [ppp_generic]
Feb 18 20:56:23 localhost kernel:                             [<f8c59473>] ppp_input+0xab/0xef [ppp_generic]
Feb 18 20:56:23 localhost kernel:                             [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 20:56:23 localhost kernel:                             [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 20:56:23 localhost kernel:                             [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 20:56:23 localhost kernel:                             [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 20:56:23 localhost kernel:                             [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 20:56:23 localhost kernel:                             [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 20:56:23 localhost kernel:                             [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 20:56:23 localhost kernel:                             [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 20:56:23 localhost kernel:                             [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:23 localhost kernel:                             [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 20:56:23 localhost kernel:                             [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 20:56:23 localhost kernel:                             [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:23 localhost kernel:                             [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 20:56:23 localhost kernel:                             [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 20:56:23 localhost kernel:                             [<f8984f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 20:56:24 localhost kernel:                             [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 20:56:24 localhost kernel:                             [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 20:56:24 localhost kernel:                             [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:24 localhost kernel:                             [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:24 localhost kernel:                             [<ffffffff>] 0xffffffff
Feb 18 20:56:24 localhost kernel:      hardirq-on-W at:
Feb 18 20:56:24 localhost kernel:                             [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 20:56:24 localhost kernel:                             [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:56:24 localhost kernel:                             [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:24 localhost kernel:                             [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:56:24 localhost kernel:                             [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 20:56:24 localhost kernel:                             [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:56:24 localhost kernel:                             [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:56:24 localhost kernel:                             [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:24 localhost kernel:                             [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:24 localhost kernel:                             [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:24 localhost kernel:                             [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:24 localhost kernel:                             [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:56:24 localhost kernel:                             [<c061ae17>] __down_failed+0x7/0xc
Feb 18 20:56:24 localhost kernel:                             [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 20:56:24 localhost kernel:                             [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 20:56:24 localhost kernel:                             [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 20:56:24 localhost kernel:                             [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:56:24 localhost kernel:                             [<ffffffff>] 0xffffffff
Feb 18 20:56:24 localhost kernel:    }
Feb 18 20:56:24 localhost kernel:    ... key      at: [<f8c5dd10>] __key.29979+0x0/0xffffb832 [ppp_generic]
Feb 18 20:56:24 localhost kernel:    -> (&list->lock#7){.+..} ops: 31 {
Feb 18 20:56:24 localhost kernel:       initial-use  at:
Feb 18 20:56:24 localhost kernel:                               [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:24 localhost kernel:                               [<c046b874>] __do_fault+0x31b/0x35d
Feb 18 20:56:24 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:24 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:24 localhost kernel:                               [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:24 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:24 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:24 localhost kernel:                               [<f8c5939f>] ppp_write+0xb2/0xdb [ppp_generic]
Feb 18 20:56:24 localhost kernel:                               [<f8c592ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 20:56:24 localhost kernel:                               [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 20:56:24 localhost kernel:                               [<c047ecfd>] sys_write+0x41/0x67
Feb 18 20:56:24 localhost kernel:                               [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:56:24 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 20:56:24 localhost kernel:       in-softirq-W at:
Feb 18 20:56:24 localhost kernel:                               [<c0446ab0>] save_trace+0x37/0x8b
Feb 18 20:56:25 localhost kernel:                               [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:25 localhost kernel:                               [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:25 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:25 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:25 localhost kernel:                               [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:25 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:25 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:25 localhost kernel:                               [<f8c59433>] ppp_input+0x6b/0xef [ppp_generic]
Feb 18 20:56:25 localhost kernel:                               [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 20:56:25 localhost kernel:                               [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 20:56:25 localhost kernel:                               [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 20:56:25 localhost kernel:                               [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 20:56:25 localhost kernel:                               [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 20:56:25 localhost kernel:                               [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 20:56:25 localhost kernel:                               [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 20:56:25 localhost kernel:                               [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 20:56:25 localhost kernel:                               [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:25 localhost kernel:                               [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 20:56:25 localhost kernel:                               [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 20:56:25 localhost kernel:                               [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:25 localhost kernel:                               [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 20:56:25 localhost kernel:                               [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 20:56:25 localhost kernel:                               [<f8984f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 20:56:25 localhost kernel:                               [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 20:56:25 localhost kernel:                               [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 20:56:25 localhost kernel:                               [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:25 localhost kernel:                               [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:25 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 20:56:25 localhost kernel:     }
Feb 18 20:56:25 localhost kernel:     ... key      at: [<f8c5dd18>] __key.21159+0x0/0xffffb82a [ppp_generic]
Feb 18 20:56:25 localhost kernel:    ... acquired at:
Feb 18 20:56:25 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:56:25 localhost kernel:    [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:25 localhost kernel:    [<c0446ab0>] save_trace+0x37/0x8b
Feb 18 20:56:25 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:25 localhost kernel:    [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:25 localhost kernel:    [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:25 localhost kernel:    [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:25 localhost kernel:    [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:25 localhost kernel:    [<f8c589f2>] ppp_receive_nonmp_frame+0x4e6/0x70a [ppp_generic]
Feb 18 20:56:25 localhost kernel:    [<f8c59473>] ppp_input+0xab/0xef [ppp_generic]
Feb 18 20:56:26 localhost kernel:    [<f8c59487>] ppp_input+0xbf/0xef [ppp_generic]
Feb 18 20:56:26 localhost kernel:    [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 20:56:26 localhost kernel:    [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 20:56:26 localhost kernel:    [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 20:56:26 localhost kernel:    [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 20:56:26 localhost kernel:    [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 20:56:26 localhost kernel:    [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 20:56:26 localhost kernel:    [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 20:56:26 localhost kernel:    [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 20:56:26 localhost kernel:    [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:26 localhost kernel:    [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 20:56:26 localhost kernel:    [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 20:56:26 localhost kernel:    [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:26 localhost kernel:    [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 20:56:26 localhost kernel:    [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 20:56:26 localhost kernel:    [<f8984f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 20:56:26 localhost kernel:    [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 20:56:26 localhost kernel:    [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 20:56:26 localhost kernel:    [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:26 localhost kernel:    [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:26 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:56:26 localhost kernel: 
Feb 18 20:56:26 localhost kernel:    -> (&q->lock){++..} ops: 468037 {
Feb 18 20:56:26 localhost kernel:       initial-use  at:
Feb 18 20:56:26 localhost kernel:                               [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:56:26 localhost kernel:                               [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:26 localhost kernel:                               [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:56:26 localhost kernel:                               [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 20:56:26 localhost kernel:                               [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:26 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:26 localhost kernel:                               [<c0619856>] wait_for_common+0x2a/0xe0
Feb 18 20:56:26 localhost kernel:                               [<c061b4a0>] _spin_lock_irq+0x2f/0x3a
Feb 18 20:56:26 localhost kernel:                               [<c0619856>] wait_for_common+0x2a/0xe0
Feb 18 20:56:26 localhost kernel:                               [<c0619856>] wait_for_common+0x2a/0xe0
Feb 18 20:56:26 localhost kernel:                               [<c043d4b2>] kthread_create+0x70/0xa4
Feb 18 20:56:26 localhost kernel:                               [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:26 localhost kernel:                               [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:56:26 localhost kernel:                               [<c04265bc>] migration_call+0x49/0x364
Feb 18 20:56:26 localhost kernel:                               [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:26 localhost kernel:                               [<c075e01b>] migration_init+0x19/0x40
Feb 18 20:56:26 localhost kernel:                               [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:27 localhost kernel:                               [<c074e39e>] kernel_init+0x55/0x2af
Feb 18 20:56:27 localhost kernel:                               [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:56:27 localhost kernel:                               [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:27 localhost kernel:                               [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:27 localhost kernel:                               [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:27 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 20:56:27 localhost kernel:       in-hardirq-W at:
Feb 18 20:56:27 localhost kernel:                               [<c04480df>] check_usage_forwards+0x19/0x41
Feb 18 20:56:27 localhost kernel:                               [<c0446ab0>] save_trace+0x37/0x8b
Feb 18 20:56:27 localhost kernel:                               [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:56:27 localhost kernel:                               [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:27 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:27 localhost kernel:                               [<c04235da>] complete+0x12/0x44
Feb 18 20:56:27 localhost kernel:                               [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:27 localhost kernel:                               [<c04235da>] complete+0x12/0x44
Feb 18 20:56:27 localhost kernel:                               [<c04235da>] complete+0x12/0x44
Feb 18 20:56:27 localhost kernel:                               [<c058446e>] i8042_aux_test_irq+0x44/0x5c
Feb 18 20:56:27 localhost kernel:                               [<c045bcde>] handle_IRQ_event+0x13/0x3d
Feb 18 20:56:27 localhost kernel:                               [<c045cd38>] handle_edge_irq+0xc2/0xff
Feb 18 20:56:27 localhost kernel:                               [<c045cc76>] handle_edge_irq+0x0/0xff
Feb 18 20:56:27 localhost kernel:                               [<c04071b4>] do_IRQ+0xac/0xd4
Feb 18 20:56:27 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 20:56:27 localhost kernel:       in-softirq-W at:
Feb 18 20:56:27 localhost kernel:                               [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:27 localhost kernel:                               [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:27 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:27 localhost kernel:                               [<c04235da>] complete+0x12/0x44
Feb 18 20:56:27 localhost kernel:                               [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:27 localhost kernel:                               [<c04235da>] complete+0x12/0x44
Feb 18 20:56:27 localhost kernel:                               [<c04235da>] complete+0x12/0x44
Feb 18 20:56:27 localhost kernel:                               [<c0430b61>] tasklet_action+0x23/0xa4
Feb 18 20:56:27 localhost kernel:                               [<c043b647>] __rcu_process_callbacks+0xfc/0x16e
Feb 18 20:56:27 localhost kernel:                               [<c043b6d1>] rcu_process_callbacks+0x18/0x30
Feb 18 20:56:27 localhost kernel:                               [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:56:27 localhost kernel:                               [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:27 localhost kernel:                               [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:27 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 20:56:27 localhost kernel:     }
Feb 18 20:56:27 localhost kernel:     ... key      at: [<c0804610>] __key.17392+0x0/0x8
Feb 18 20:56:27 localhost kernel:     -> (&rq->rq_lock_key){++..} ops: 138094 {
Feb 18 20:56:28 localhost kernel:        initial-use  at:
Feb 18 20:56:28 localhost kernel:                                 [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:28 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:28 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:28 localhost kernel:                                 [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:28 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:28 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:28 localhost kernel:                                 [<c075e3c9>] sched_init+0x27d/0x292
Feb 18 20:56:28 localhost kernel:                                 [<c074e756>] start_kernel+0x15e/0x327
Feb 18 20:56:28 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 20:56:28 localhost kernel:        in-hardirq-W at:
Feb 18 20:56:28 localhost kernel:                                 [<c04480a2>] find_usage_forwards+0x67/0x8b
Feb 18 20:56:28 localhost kernel:                                 [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:56:28 localhost kernel:                                 [<c0442f77>] clocksource_get_next+0xa/0x3f
Feb 18 20:56:28 localhost kernel:                                 [<c0442fa6>] clocksource_get_next+0x39/0x3f
Feb 18 20:56:28 localhost kernel:                                 [<c044204f>] update_wall_time+0x58d/0x71c
Feb 18 20:56:28 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:28 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:28 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:28 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:28 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:28 localhost kernel:                                 [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:56:28 localhost kernel:                                 [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:56:28 localhost kernel:                                 [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:56:28 localhost kernel:                                 [<c0407b80>] timer_interrupt+0x44/0x4a
Feb 18 20:56:28 localhost kernel:                                 [<c045bcde>] handle_IRQ_event+0x13/0x3d
Feb 18 20:56:28 localhost kernel:                                 [<c045d17e>] handle_level_irq+0x78/0xbe
Feb 18 20:56:28 localhost kernel:                                 [<c045d106>] handle_level_irq+0x0/0xbe
Feb 18 20:56:28 localhost kernel:                                 [<c04071b4>] do_IRQ+0xac/0xd4
Feb 18 20:56:28 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 20:56:28 localhost kernel:        in-softirq-W at:
Feb 18 20:56:28 localhost kernel:                                 [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 20:56:28 localhost kernel:                                 [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 20:56:28 localhost kernel:                                 [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:28 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:28 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:28 localhost kernel:                                 [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:56:28 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:28 localhost kernel:                                 [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:56:28 localhost kernel:                                 [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:56:28 localhost kernel:                                 [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 20:56:29 localhost kernel:                                 [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 20:56:29 localhost kernel:                                 [<c0430afa>] __do_softirq+0xc9/0xde
Feb 18 20:56:29 localhost kernel:                                 [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:29 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 20:56:29 localhost kernel:      }
Feb 18 20:56:29 localhost kernel:      ... key      at: [<c2a22488>] 0xc2a22488
Feb 18 20:56:29 localhost kernel:      -> (&rq->rq_lock_key#2){++..} ops: 382046 {
Feb 18 20:56:29 localhost kernel:         initial-use  at:
Feb 18 20:56:29 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:29 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:29 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:29 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:29 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:29 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:29 localhost kernel:                                   [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:56:29 localhost kernel:                                   [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:56:29 localhost kernel:                                   [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:56:29 localhost kernel:                                   [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:56:29 localhost kernel:                                   [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:56:29 localhost kernel:                                   [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:56:29 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:29 localhost kernel:                                   [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:29 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:29 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:29 localhost kernel:                                   [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:56:29 localhost kernel:                                   [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:56:29 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:29 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:29 localhost kernel:                                   [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:56:29 localhost kernel:                                   [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:56:29 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:29 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:29 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:29 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:56:29 localhost kernel:         in-hardirq-W at:
Feb 18 20:56:29 localhost kernel:                                   [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:56:29 localhost kernel:                                   [<c040a9d6>] save_stack_trace+0x20/0x3a
Feb 18 20:56:29 localhost kernel:                                   [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:56:29 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:29 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:29 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:30 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:30 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:30 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:30 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:30 localhost kernel:                                   [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:56:30 localhost kernel:                                   [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:56:30 localhost kernel:                                   [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:56:30 localhost kernel:                                   [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:56:30 localhost kernel:                                   [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:56:30 localhost kernel:                                   [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:56:30 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:30 localhost kernel:                                   [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:56:30 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:30 localhost kernel:                                   [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:56:30 localhost kernel:                                   [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:56:30 localhost kernel:                                   [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:56:30 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:56:30 localhost kernel:         in-softirq-W at:
Feb 18 20:56:30 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:30 localhost kernel:                                   [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 20:56:30 localhost kernel:                                   [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 20:56:30 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:30 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:30 localhost kernel:                                   [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:56:30 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:30 localhost kernel:                                   [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:56:30 localhost kernel:                                   [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:56:30 localhost kernel:                                   [<c0433efb>] process_timeout+0x0/0x5
Feb 18 20:56:30 localhost kernel:                                   [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 20:56:30 localhost kernel:                                   [<c0433d59>] run_timer_softirq+0x113/0x188
Feb 18 20:56:30 localhost kernel:                                   [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:56:30 localhost kernel:                                   [<c0433efb>] process_timeout+0x0/0x5
Feb 18 20:56:30 localhost kernel:                                   [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 20:56:30 localhost kernel:                                   [<c0433efb>] process_timeout+0x0/0x5
Feb 18 20:56:30 localhost kernel:                                   [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 20:56:30 localhost kernel:                                   [<c0433efb>] process_timeout+0x0/0x5
Feb 18 20:56:30 localhost kernel:                                   [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 20:56:30 localhost kernel:                                   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:30 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:30 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:56:30 localhost kernel:       }
Feb 18 20:56:31 localhost kernel:       ... key      at: [<c2a2b488>] 0xc2a2b488
Feb 18 20:56:31 localhost kernel:       -> (&rq->rq_lock_key#3){++..} ops: 250064 {
Feb 18 20:56:31 localhost kernel:          initial-use  at:
Feb 18 20:56:31 localhost kernel:                                     [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:31 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:31 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:31 localhost kernel:                                     [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:31 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:31 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:31 localhost kernel:                                     [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:56:31 localhost kernel:                                     [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:56:31 localhost kernel:                                     [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:56:31 localhost kernel:                                     [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:56:31 localhost kernel:                                     [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:56:31 localhost kernel:                                     [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:56:31 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:31 localhost kernel:                                     [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:31 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:31 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:31 localhost kernel:                                     [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:56:31 localhost kernel:                                     [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:56:31 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:31 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:31 localhost kernel:                                     [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:56:31 localhost kernel:                                     [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:56:31 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:31 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:31 localhost kernel:                                     [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:31 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 20:56:31 localhost kernel:          in-hardirq-W at:
Feb 18 20:56:31 localhost kernel:                                     [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:56:31 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:31 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:31 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:31 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:31 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:31 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:31 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:31 localhost kernel:                                     [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:56:31 localhost kernel:                                     [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:56:31 localhost kernel:                                     [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:56:32 localhost kernel:                                     [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:56:32 localhost kernel:                                     [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:56:32 localhost kernel:                                     [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:56:32 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:32 localhost kernel:                                     [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:56:32 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:32 localhost kernel:                                     [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:56:32 localhost kernel:                                     [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:56:32 localhost kernel:                                     [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:56:32 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 20:56:32 localhost kernel:          in-softirq-W at:
Feb 18 20:56:32 localhost kernel:                                     [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:56:32 localhost kernel:                                     [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:32 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:32 localhost kernel:                                     [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:56:32 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:32 localhost kernel:                                     [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:56:32 localhost kernel:                                     [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:56:32 localhost kernel:                                     [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:56:32 localhost kernel:                                     [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:56:32 localhost kernel:                                     [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:56:32 localhost kernel:                                     [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:32 localhost kernel:                                     [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:32 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 20:56:32 localhost kernel:        }
Feb 18 20:56:32 localhost kernel:        ... key      at: [<c2a34488>] 0xc2a34488
Feb 18 20:56:32 localhost kernel:        -> (&rq->rq_lock_key#4){++..} ops: 233519 {
Feb 18 20:56:32 localhost kernel:           initial-use  at:
Feb 18 20:56:32 localhost kernel:                                       [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:32 localhost kernel:                                       [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:32 localhost kernel:                                       [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:32 localhost kernel:                                       [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:32 localhost kernel:                                       [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:32 localhost kernel:                                       [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:32 localhost kernel:                                       [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:56:32 localhost kernel:                                       [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:56:32 localhost kernel:                                       [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:56:32 localhost kernel:                                       [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:56:32 localhost kernel:                                       [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:56:32 localhost kernel:                                       [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:56:32 localhost kernel:                                       [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:32 localhost kernel:                                       [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:33 localhost kernel:                                       [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:33 localhost kernel:                                       [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:33 localhost kernel:                                       [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:56:33 localhost kernel:                                       [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:56:33 localhost kernel:                                       [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:33 localhost kernel:                                       [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:33 localhost kernel:                                       [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:56:33 localhost kernel:                                       [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:56:33 localhost kernel:                                       [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:33 localhost kernel:                                       [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:33 localhost kernel:                                       [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:33 localhost kernel:                                       [<ffffffff>] 0xffffffff
Feb 18 20:56:33 localhost kernel:           in-hardirq-W at:
Feb 18 20:56:33 localhost kernel:                                       [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:56:33 localhost kernel:                                       [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:33 localhost kernel:                                       [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:33 localhost kernel:                                       [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:33 localhost kernel:                                       [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:33 localhost kernel:                                       [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:33 localhost kernel:                                       [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:33 localhost kernel:                                       [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:33 localhost kernel:                                       [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:56:33 localhost kernel:                                       [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:56:33 localhost kernel:                                       [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:56:33 localhost kernel:                                       [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:56:33 localhost kernel:                                       [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:56:33 localhost kernel:                                       [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:56:33 localhost kernel:                                       [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:33 localhost kernel:                                       [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:56:33 localhost kernel:                                       [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:33 localhost kernel:                                       [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:56:33 localhost kernel:                                       [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:56:33 localhost kernel:                                       [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:56:33 localhost kernel:                                       [<ffffffff>] 0xffffffff
Feb 18 20:56:33 localhost kernel:           in-softirq-W at:
Feb 18 20:56:33 localhost kernel:                                       [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:33 localhost kernel:                                       [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:33 localhost kernel:                                       [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:33 localhost kernel:                                       [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:33 localhost kernel:                                       [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:33 localhost kernel:                                       [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:33 localhost kernel:                                       [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:56:34 localhost kernel:                                       [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:56:34 localhost kernel:                                       [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:56:34 localhost kernel:                                       [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:34 localhost kernel:                                       [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:34 localhost kernel:                                       [<ffffffff>] 0xffffffff
Feb 18 20:56:34 localhost kernel:         }
Feb 18 20:56:34 localhost kernel:         ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 20:56:34 localhost kernel:        ... acquired at:
Feb 18 20:56:34 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:56:34 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:34 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:34 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:34 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:34 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:34 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:34 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:56:34 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:34 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:56:34 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:34 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:56:34 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:56:34 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:34 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:56:34 localhost kernel: 
Feb 18 20:56:34 localhost kernel:       ... acquired at:
Feb 18 20:56:34 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:56:34 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:34 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:34 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:34 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:34 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:34 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:34 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:56:34 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:34 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:56:34 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:34 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:56:34 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:56:34 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:34 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:56:35 localhost kernel: 
Feb 18 20:56:35 localhost kernel:       -> (&rq->rq_lock_key#4){++..} ops: 233519 {
Feb 18 20:56:35 localhost kernel:          initial-use  at:
Feb 18 20:56:35 localhost kernel:                                     [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:35 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:35 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:35 localhost kernel:                                     [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:35 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:35 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:35 localhost kernel:                                     [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:56:35 localhost kernel:                                     [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:56:35 localhost kernel:                                     [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:56:35 localhost kernel:                                     [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:56:35 localhost kernel:                                     [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:56:35 localhost kernel:                                     [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:56:35 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:35 localhost kernel:                                     [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:35 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:35 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:35 localhost kernel:                                     [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:56:35 localhost kernel:                                     [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:56:35 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:35 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:35 localhost kernel:                                     [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:56:35 localhost kernel:                                     [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:56:35 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:35 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:35 localhost kernel:                                     [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:35 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 20:56:35 localhost kernel:          in-hardirq-W at:
Feb 18 20:56:35 localhost kernel:                                     [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:56:35 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:35 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:35 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:35 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:35 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:35 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:35 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:35 localhost kernel:                                     [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:56:35 localhost kernel:                                     [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:56:35 localhost kernel:                                     [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:56:36 localhost kernel:                                     [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:56:36 localhost kernel:                                     [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:56:36 localhost kernel:                                     [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:56:36 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:36 localhost kernel:                                     [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:56:36 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:36 localhost kernel:                                     [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:56:36 localhost kernel:                                     [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:56:36 localhost kernel:                                     [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:56:36 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 20:56:36 localhost kernel:          in-softirq-W at:
Feb 18 20:56:36 localhost kernel:                                     [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:36 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:36 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:36 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:36 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:36 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:36 localhost kernel:                                     [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:56:36 localhost kernel:                                     [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:56:36 localhost kernel:                                     [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:56:36 localhost kernel:                                     [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:36 localhost kernel:                                     [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:36 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 20:56:36 localhost kernel:        }
Feb 18 20:56:36 localhost kernel:        ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 20:56:36 localhost kernel:       ... acquired at:
Feb 18 20:56:36 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:56:36 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:36 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:36 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:36 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:36 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:36 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:36 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:56:36 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:36 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:56:36 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:36 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:56:36 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:56:36 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:36 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:56:37 localhost kernel: 
Feb 18 20:56:37 localhost kernel:      ... acquired at:
Feb 18 20:56:37 localhost kernel:    [<c0446b6a>] add_lock_to_list+0x66/0x89
Feb 18 20:56:37 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:56:37 localhost kernel:    [<c0619307>] schedule+0x22e/0x6ec
Feb 18 20:56:37 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:37 localhost kernel:    [<c0619307>] schedule+0x22e/0x6ec
Feb 18 20:56:37 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:37 localhost kernel:    [<c0619307>] schedule+0x22e/0x6ec
Feb 18 20:56:37 localhost kernel:    [<c0619307>] schedule+0x22e/0x6ec
Feb 18 20:56:37 localhost kernel:    [<c0619a92>] schedule_timeout+0x13/0x8d
Feb 18 20:56:37 localhost kernel:    [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 20:56:37 localhost kernel:    [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:56:37 localhost kernel:    [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:37 localhost kernel:    [<c06198d2>] wait_for_common+0xa6/0xe0
Feb 18 20:56:37 localhost kernel:    [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:56:37 localhost kernel:    [<c0439c5c>] call_usermodehelper_exec+0x7a/0xbe
Feb 18 20:56:37 localhost kernel:    [<c04ecbd6>] kobject_uevent_env+0x348/0x37c
Feb 18 20:56:37 localhost kernel:    [<c055f4c0>] device_add+0x318/0x54c
Feb 18 20:56:37 localhost kernel:    [<c055fa67>] device_create+0x7a/0x9a
Feb 18 20:56:37 localhost kernel:    [<c053c13e>] tty_register_device+0xb5/0xbd
Feb 18 20:56:37 localhost kernel:    [<c04efab1>] vsnprintf+0x440/0x47c
Feb 18 20:56:37 localhost kernel:    [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 20:56:37 localhost kernel:    [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:37 localhost kernel:    [<c042c9c6>] register_console+0x4b/0x1fe
Feb 18 20:56:37 localhost kernel:    [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:37 localhost kernel:    [<c055aff0>] serial8250_pm+0x0/0xb2
Feb 18 20:56:37 localhost kernel:    [<c0558795>] uart_change_pm+0x23/0x2b
Feb 18 20:56:37 localhost kernel:    [<c055998a>] uart_add_one_port+0x29d/0x2f6
Feb 18 20:56:37 localhost kernel:    [<c05623c9>] platform_device_add+0xee/0x11c
Feb 18 20:56:37 localhost kernel:    [<c055ed94>] device_initialize+0x7b/0x93
Feb 18 20:56:37 localhost kernel:    [<c05624ee>] platform_device_alloc+0x40/0x51
Feb 18 20:56:37 localhost kernel:    [<c076a545>] serial8250_init+0xca/0x110
Feb 18 20:56:37 localhost kernel:    [<c074e491>] kernel_init+0x148/0x2af
Feb 18 20:56:37 localhost kernel:    [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:37 localhost kernel:    [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:37 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:37 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:56:37 localhost kernel: 
Feb 18 20:56:37 localhost kernel:      -> (&rq->rq_lock_key#4){++..} ops: 233519 {
Feb 18 20:56:37 localhost kernel:         initial-use  at:
Feb 18 20:56:38 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:38 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:38 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:38 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:38 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:38 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:38 localhost kernel:                                   [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:56:38 localhost kernel:                                   [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:56:38 localhost kernel:                                   [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:56:38 localhost kernel:                                   [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:56:38 localhost kernel:                                   [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:56:38 localhost kernel:                                   [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:56:38 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:38 localhost kernel:                                   [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:38 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:38 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:38 localhost kernel:                                   [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:56:38 localhost kernel:                                   [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:56:38 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:38 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:38 localhost kernel:                                   [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:56:38 localhost kernel:                                   [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:56:38 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:38 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:38 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:38 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:56:38 localhost kernel:         in-hardirq-W at:
Feb 18 20:56:38 localhost kernel:                                   [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:56:38 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:38 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:38 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:38 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:38 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:38 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:38 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:38 localhost kernel:                                   [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:56:38 localhost kernel:                                   [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:56:38 localhost kernel:                                   [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:56:38 localhost kernel:                                   [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:56:38 localhost kernel:                                   [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:56:38 localhost kernel:                                   [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:56:39 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:39 localhost kernel:                                   [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:56:39 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:39 localhost kernel:                                   [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:56:39 localhost kernel:                                   [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:56:39 localhost kernel:                                   [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:56:39 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:56:39 localhost kernel:         in-softirq-W at:
Feb 18 20:56:39 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:39 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:39 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:39 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:39 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:39 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:39 localhost kernel:                                   [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:56:39 localhost kernel:                                   [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:56:39 localhost kernel:                                   [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:56:39 localhost kernel:                                   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:39 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:39 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:56:39 localhost kernel:       }
Feb 18 20:56:39 localhost kernel:       ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 20:56:39 localhost kernel:      ... acquired at:
Feb 18 20:56:39 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:56:39 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:39 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:39 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:39 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:39 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:39 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:39 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:56:39 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:39 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:56:39 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:39 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:56:39 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:56:39 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:39 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:56:39 localhost kernel: 
Feb 18 20:56:39 localhost kernel:      -> (&rq->rq_lock_key#3){++..} ops: 250064 {
Feb 18 20:56:39 localhost kernel:         initial-use  at:
Feb 18 20:56:40 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:40 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:40 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:40 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:40 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:40 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:40 localhost kernel:                                   [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:56:40 localhost kernel:                                   [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:56:40 localhost kernel:                                   [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:56:40 localhost kernel:                                   [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:56:40 localhost kernel:                                   [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:56:40 localhost kernel:                                   [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:56:40 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:40 localhost kernel:                                   [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:40 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:40 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:40 localhost kernel:                                   [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:56:40 localhost kernel:                                   [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:56:40 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:40 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:40 localhost kernel:                                   [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:56:40 localhost kernel:                                   [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:56:40 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:40 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:40 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:40 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:56:40 localhost kernel:         in-hardirq-W at:
Feb 18 20:56:40 localhost kernel:                                   [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:56:40 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:40 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:40 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:40 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:40 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:40 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:40 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:40 localhost kernel:                                   [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:56:40 localhost kernel:                                   [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:56:40 localhost kernel:                                   [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:56:40 localhost kernel:                                   [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:56:40 localhost kernel:                                   [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:56:40 localhost kernel:                                   [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:56:41 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:41 localhost kernel:                                   [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:56:41 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:41 localhost kernel:                                   [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:56:41 localhost kernel:                                   [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:56:41 localhost kernel:                                   [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:56:41 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:56:41 localhost kernel:         in-softirq-W at:
Feb 18 20:56:41 localhost kernel:                                   [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:56:41 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:41 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:41 localhost kernel:                                   [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:56:41 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:41 localhost kernel:                                   [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:56:41 localhost kernel:                                   [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:56:41 localhost kernel:                                   [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:56:41 localhost kernel:                                   [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:56:41 localhost kernel:                                   [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:56:41 localhost kernel:                                   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:41 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:41 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:56:41 localhost kernel:       }
Feb 18 20:56:41 localhost kernel:       ... key      at: [<c2a34488>] 0xc2a34488
Feb 18 20:56:41 localhost kernel:       -> (&rq->rq_lock_key#4){++..} ops: 233519 {
Feb 18 20:56:41 localhost kernel:          initial-use  at:
Feb 18 20:56:41 localhost kernel:                                     [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:41 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:41 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:41 localhost kernel:                                     [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:41 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:41 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:41 localhost kernel:                                     [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:56:41 localhost kernel:                                     [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:56:41 localhost kernel:                                     [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:56:41 localhost kernel:                                     [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:56:41 localhost kernel:                                     [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:56:41 localhost kernel:                                     [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:56:41 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:42 localhost kernel:                                     [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:43 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:48 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:53 localhost kernel:                                     [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:56:58 localhost kernel:                                     [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:03 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:09 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:13 localhost kernel:                                     [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:57:18 localhost kernel:                                     [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:57:21 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:25 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:26 localhost kernel:                                     [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:27 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 20:57:27 localhost kernel:          in-hardirq-W at:
Feb 18 20:57:27 localhost kernel:                                     [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:57:27 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:27 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:27 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:27 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:27 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:27 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:27 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:27 localhost kernel:                                     [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:57:27 localhost kernel:                                     [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:57:27 localhost kernel:                                     [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:57:27 localhost kernel:                                     [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:57:27 localhost kernel:                                     [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:57:27 localhost kernel:                                     [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:57:27 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:27 localhost kernel:                                     [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:57:27 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:27 localhost kernel:                                     [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:57:27 localhost kernel:                                     [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:57:27 localhost kernel:                                     [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:57:27 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 20:57:27 localhost kernel:          in-softirq-W at:
Feb 18 20:57:27 localhost kernel:                                     [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:27 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:27 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:27 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:27 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel:                                     [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:57:28 localhost kernel:                                     [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:57:28 localhost kernel:                                     [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:57:28 localhost kernel:                                     [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:28 localhost kernel:                                     [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:28 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 20:57:28 localhost kernel:        }
Feb 18 20:57:28 localhost kernel:        ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 20:57:28 localhost kernel:       ... acquired at:
Feb 18 20:57:28 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:28 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:28 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:28 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:57:28 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:28 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:57:28 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:28 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:57:28 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:28 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:28 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:57:28 localhost kernel: 
Feb 18 20:57:28 localhost kernel:      ... acquired at:
Feb 18 20:57:28 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:28 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:28 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:28 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:57:28 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:28 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:57:28 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:28 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:57:28 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:28 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:29 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:57:29 localhost kernel: 
Feb 18 20:57:29 localhost kernel:     ... acquired at:
Feb 18 20:57:29 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:29 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:29 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:29 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:29 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:29 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:29 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:29 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:29 localhost kernel:    [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 20:57:29 localhost kernel:    [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 20:57:29 localhost kernel:    [<c04235fe>] complete+0x36/0x44
Feb 18 20:57:29 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:29 localhost kernel:    [<c043d502>] kthread+0x1c/0x5e
Feb 18 20:57:29 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:29 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:29 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:57:29 localhost kernel: 
Feb 18 20:57:29 localhost kernel:     -> (&rq->rq_lock_key#2){++..} ops: 382046 {
Feb 18 20:57:29 localhost kernel:        initial-use  at:
Feb 18 20:57:29 localhost kernel:                                 [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:29 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:29 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:29 localhost kernel:                                 [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:29 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:29 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:29 localhost kernel:                                 [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:57:29 localhost kernel:                                 [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:57:29 localhost kernel:                                 [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:57:29 localhost kernel:                                 [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:57:29 localhost kernel:                                 [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:57:29 localhost kernel:                                 [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:57:29 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:29 localhost kernel:                                 [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:29 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:29 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:29 localhost kernel:                                 [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:57:29 localhost kernel:                                 [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:29 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:29 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:30 localhost kernel:                                 [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:57:30 localhost kernel:                                 [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:57:30 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:30 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:30 localhost kernel:                                 [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:30 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 20:57:30 localhost kernel:        in-hardirq-W at:
Feb 18 20:57:30 localhost kernel:                                 [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:57:30 localhost kernel:                                 [<c040a9d6>] save_stack_trace+0x20/0x3a
Feb 18 20:57:30 localhost kernel:                                 [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:57:30 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:30 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:30 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:30 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:30 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:30 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:30 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:30 localhost kernel:                                 [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:57:30 localhost kernel:                                 [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:57:30 localhost kernel:                                 [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:57:30 localhost kernel:                                 [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:57:30 localhost kernel:                                 [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:57:30 localhost kernel:                                 [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:57:30 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:30 localhost kernel:                                 [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:57:30 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:30 localhost kernel:                                 [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:57:30 localhost kernel:                                 [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:57:30 localhost kernel:                                 [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:57:30 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 20:57:30 localhost kernel:        in-softirq-W at:
Feb 18 20:57:30 localhost kernel:                                 [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:30 localhost kernel:                                 [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 20:57:30 localhost kernel:                                 [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 20:57:30 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:30 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:30 localhost kernel:                                 [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:30 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:30 localhost kernel:                                 [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:30 localhost kernel:                                 [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:30 localhost kernel:                                 [<c0433efb>] process_timeout+0x0/0x5
Feb 18 20:57:31 localhost kernel:                                 [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 20:57:31 localhost kernel:                                 [<c0433d59>] run_timer_softirq+0x113/0x188
Feb 18 20:57:31 localhost kernel:                                 [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:31 localhost kernel:                                 [<c0433efb>] process_timeout+0x0/0x5
Feb 18 20:57:31 localhost kernel:                                 [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 20:57:31 localhost kernel:                                 [<c0433efb>] process_timeout+0x0/0x5
Feb 18 20:57:31 localhost kernel:                                 [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 20:57:31 localhost kernel:                                 [<c0433efb>] process_timeout+0x0/0x5
Feb 18 20:57:31 localhost kernel:                                 [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 20:57:31 localhost kernel:                                 [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:31 localhost kernel:                                 [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:31 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 20:57:31 localhost kernel:      }
Feb 18 20:57:31 localhost kernel:      ... key      at: [<c2a2b488>] 0xc2a2b488
Feb 18 20:57:31 localhost kernel:      -> (&rq->rq_lock_key#3){++..} ops: 250064 {
Feb 18 20:57:31 localhost kernel:         initial-use  at:
Feb 18 20:57:31 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:31 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:31 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:31 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:31 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:31 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:31 localhost kernel:                                   [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:57:31 localhost kernel:                                   [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:57:31 localhost kernel:                                   [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:57:31 localhost kernel:                                   [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:57:31 localhost kernel:                                   [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:57:31 localhost kernel:                                   [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:57:31 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:31 localhost kernel:                                   [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:31 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:31 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:31 localhost kernel:                                   [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:57:31 localhost kernel:                                   [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:31 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:31 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:31 localhost kernel:                                   [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:57:31 localhost kernel:                                   [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:57:31 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:31 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:31 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:32 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:57:32 localhost kernel:         in-hardirq-W at:
Feb 18 20:57:32 localhost kernel:                                   [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:57:32 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:32 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:32 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:32 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:32 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:32 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:32 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:32 localhost kernel:                                   [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:57:32 localhost kernel:                                   [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:57:32 localhost kernel:                                   [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:57:32 localhost kernel:                                   [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:57:32 localhost kernel:                                   [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:57:32 localhost kernel:                                   [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:57:32 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:32 localhost kernel:                                   [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:57:32 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:32 localhost kernel:                                   [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:57:32 localhost kernel:                                   [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:57:32 localhost kernel:                                   [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:57:32 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:57:32 localhost kernel:         in-softirq-W at:
Feb 18 20:57:32 localhost kernel:                                   [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:57:32 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:32 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:32 localhost kernel:                                   [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:57:32 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:32 localhost kernel:                                   [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:57:32 localhost kernel:                                   [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:57:32 localhost kernel:                                   [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:57:32 localhost kernel:                                   [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:57:32 localhost kernel:                                   [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:57:32 localhost kernel:                                   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:32 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:32 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:57:32 localhost kernel:       }
Feb 18 20:57:32 localhost kernel:       ... key      at: [<c2a34488>] 0xc2a34488
Feb 18 20:57:32 localhost kernel:       -> (&rq->rq_lock_key#4){++..} ops: 233519 {
Feb 18 20:57:32 localhost kernel:          initial-use  at:
Feb 18 20:57:33 localhost kernel:                                     [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:33 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:33 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:33 localhost kernel:                                     [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:33 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:33 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:33 localhost kernel:                                     [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:57:33 localhost kernel:                                     [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:57:33 localhost kernel:                                     [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:57:33 localhost kernel:                                     [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:57:33 localhost kernel:                                     [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:57:33 localhost kernel:                                     [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:57:33 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:33 localhost kernel:                                     [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:33 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:33 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:33 localhost kernel:                                     [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:57:33 localhost kernel:                                     [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:33 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:33 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:33 localhost kernel:                                     [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:57:33 localhost kernel:                                     [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:57:33 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:33 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:33 localhost kernel:                                     [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:33 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 20:57:33 localhost kernel:          in-hardirq-W at:
Feb 18 20:57:33 localhost kernel:                                     [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:57:33 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:33 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:33 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:33 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:33 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:33 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:33 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:33 localhost kernel:                                     [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:57:33 localhost kernel:                                     [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:57:33 localhost kernel:                                     [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:57:33 localhost kernel:                                     [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:57:33 localhost kernel:                                     [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:57:33 localhost kernel:                                     [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:57:33 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:33 localhost kernel:                                     [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:57:34 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:34 localhost kernel:                                     [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:57:34 localhost kernel:                                     [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:57:34 localhost kernel:                                     [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:57:34 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 20:57:34 localhost kernel:          in-softirq-W at:
Feb 18 20:57:34 localhost kernel:                                     [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:34 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:34 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:34 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:34 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:34 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:34 localhost kernel:                                     [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:57:34 localhost kernel:                                     [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:57:34 localhost kernel:                                     [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:57:34 localhost kernel:                                     [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:34 localhost kernel:                                     [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:34 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 20:57:34 localhost kernel:        }
Feb 18 20:57:34 localhost kernel:        ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 20:57:34 localhost kernel:       ... acquired at:
Feb 18 20:57:34 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:34 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:34 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:34 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:34 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:34 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:34 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:34 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:57:34 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:34 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:57:34 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:34 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:57:34 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:34 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:34 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:57:34 localhost kernel: 
Feb 18 20:57:34 localhost kernel:      ... acquired at:
Feb 18 20:57:34 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:34 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:34 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:34 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:35 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:35 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:35 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:35 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:57:35 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:35 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:57:35 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:35 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:57:35 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:35 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:35 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:57:35 localhost kernel: 
Feb 18 20:57:35 localhost kernel:      -> (&rq->rq_lock_key#4){++..} ops: 233519 {
Feb 18 20:57:35 localhost kernel:         initial-use  at:
Feb 18 20:57:35 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:35 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:35 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:35 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:35 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:35 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:35 localhost kernel:                                   [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:57:35 localhost kernel:                                   [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:57:35 localhost kernel:                                   [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:57:35 localhost kernel:                                   [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:57:35 localhost kernel:                                   [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:57:35 localhost kernel:                                   [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:57:35 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:35 localhost kernel:                                   [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:35 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:35 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:35 localhost kernel:                                   [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:57:35 localhost kernel:                                   [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:35 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:35 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:35 localhost kernel:                                   [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:57:35 localhost kernel:                                   [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:57:35 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:35 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:35 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:35 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:57:35 localhost kernel:         in-hardirq-W at:
Feb 18 20:57:35 localhost kernel:                                   [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:57:36 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:36 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:36 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:36 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:36 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:36 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:36 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:36 localhost kernel:                                   [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:57:36 localhost kernel:                                   [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:57:36 localhost kernel:                                   [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:57:36 localhost kernel:                                   [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:57:36 localhost kernel:                                   [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:57:36 localhost kernel:                                   [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:57:36 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:36 localhost kernel:                                   [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:57:36 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:36 localhost kernel:                                   [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:57:36 localhost kernel:                                   [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:57:36 localhost kernel:                                   [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:57:36 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:57:36 localhost kernel:         in-softirq-W at:
Feb 18 20:57:36 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:36 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:36 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:36 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:36 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:36 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:36 localhost kernel:                                   [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:57:36 localhost kernel:                                   [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:57:36 localhost kernel:                                   [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:57:36 localhost kernel:                                   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:36 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:36 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:57:36 localhost kernel:       }
Feb 18 20:57:36 localhost kernel:       ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 20:57:36 localhost kernel:      ... acquired at:
Feb 18 20:57:36 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:36 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:36 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:36 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:37 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:37 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:37 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:37 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:57:37 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:37 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:57:37 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:37 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:57:37 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:37 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:37 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:57:37 localhost kernel: 
Feb 18 20:57:37 localhost kernel:     ... acquired at:
Feb 18 20:57:37 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:37 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:37 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:37 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:37 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:37 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:37 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:37 localhost kernel:    [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 20:57:37 localhost kernel:    [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 20:57:37 localhost kernel:    [<c04235fe>] complete+0x36/0x44
Feb 18 20:57:37 localhost kernel:    [<c043aad8>] worker_thread+0x0/0xc4
Feb 18 20:57:37 localhost kernel:    [<c043d502>] kthread+0x1c/0x5e
Feb 18 20:57:37 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:37 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:37 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:57:37 localhost kernel: 
Feb 18 20:57:37 localhost kernel:     -> (&rq->rq_lock_key#3){++..} ops: 250064 {
Feb 18 20:57:37 localhost kernel:        initial-use  at:
Feb 18 20:57:37 localhost kernel:                                 [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:37 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:37 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:37 localhost kernel:                                 [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:37 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:37 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:37 localhost kernel:                                 [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:57:37 localhost kernel:                                 [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:57:37 localhost kernel:                                 [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:57:38 localhost kernel:                                 [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:57:38 localhost kernel:                                 [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:57:38 localhost kernel:                                 [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:57:38 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:38 localhost kernel:                                 [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:38 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:38 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:38 localhost kernel:                                 [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:57:38 localhost kernel:                                 [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:38 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:38 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:38 localhost kernel:                                 [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:57:38 localhost kernel:                                 [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:57:38 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:38 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:38 localhost kernel:                                 [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:38 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 20:57:38 localhost kernel:        in-hardirq-W at:
Feb 18 20:57:38 localhost kernel:                                 [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:57:38 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:38 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:38 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:38 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:38 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:38 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:38 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:38 localhost kernel:                                 [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:57:38 localhost kernel:                                 [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:57:38 localhost kernel:                                 [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:57:38 localhost kernel:                                 [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:57:38 localhost kernel:                                 [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:57:38 localhost kernel:                                 [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:57:38 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:38 localhost kernel:                                 [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:57:38 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:38 localhost kernel:                                 [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:57:38 localhost kernel:                                 [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:57:38 localhost kernel:                                 [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:57:38 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 20:57:38 localhost kernel:        in-softirq-W at:
Feb 18 20:57:38 localhost kernel:                                 [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:57:38 localhost kernel:                                 [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:39 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:39 localhost kernel:                                 [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:57:39 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:39 localhost kernel:                                 [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:57:39 localhost kernel:                                 [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:57:39 localhost kernel:                                 [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:57:39 localhost kernel:                                 [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:57:39 localhost kernel:                                 [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:57:39 localhost kernel:                                 [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:39 localhost kernel:                                 [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:39 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 20:57:39 localhost kernel:      }
Feb 18 20:57:39 localhost kernel:      ... key      at: [<c2a34488>] 0xc2a34488
Feb 18 20:57:39 localhost kernel:      -> (&rq->rq_lock_key#4){++..} ops: 233519 {
Feb 18 20:57:39 localhost kernel:         initial-use  at:
Feb 18 20:57:39 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:39 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:39 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:39 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:39 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:39 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:39 localhost kernel:                                   [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:57:39 localhost kernel:                                   [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:57:39 localhost kernel:                                   [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:57:39 localhost kernel:                                   [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:57:39 localhost kernel:                                   [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:57:39 localhost kernel:                                   [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:57:39 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:39 localhost kernel:                                   [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:39 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:39 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:39 localhost kernel:                                   [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:57:39 localhost kernel:                                   [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:39 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:39 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:39 localhost kernel:                                   [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:57:39 localhost kernel:                                   [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:57:39 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:39 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:39 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:39 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:57:40 localhost kernel:         in-hardirq-W at:
Feb 18 20:57:40 localhost kernel:                                   [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:57:40 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:40 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:40 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:40 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:40 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:40 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:40 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:40 localhost kernel:                                   [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:57:40 localhost kernel:                                   [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:57:40 localhost kernel:                                   [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:57:40 localhost kernel:                                   [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:57:40 localhost kernel:                                   [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:57:40 localhost kernel:                                   [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:57:40 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:40 localhost kernel:                                   [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:57:40 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:40 localhost kernel:                                   [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:57:40 localhost kernel:                                   [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:57:40 localhost kernel:                                   [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:57:40 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:57:40 localhost kernel:         in-softirq-W at:
Feb 18 20:57:40 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:40 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:40 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:40 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:40 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:40 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:40 localhost kernel:                                   [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:57:40 localhost kernel:                                   [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:57:40 localhost kernel:                                   [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:57:40 localhost kernel:                                   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:40 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:40 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 20:57:40 localhost kernel:       }
Feb 18 20:57:40 localhost kernel:       ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 20:57:40 localhost kernel:      ... acquired at:
Feb 18 20:57:40 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:40 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:40 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:40 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:41 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:41 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:41 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:41 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:57:41 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:41 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:57:41 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:41 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:57:41 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:41 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:41 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:57:41 localhost kernel: 
Feb 18 20:57:41 localhost kernel:     ... acquired at:
Feb 18 20:57:41 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:41 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:41 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:41 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:41 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:41 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:41 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:41 localhost kernel:    [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 20:57:41 localhost kernel:    [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 20:57:41 localhost kernel:    [<c04235fe>] complete+0x36/0x44
Feb 18 20:57:41 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:41 localhost kernel:    [<c0428996>] migration_thread+0x186/0x1d2
Feb 18 20:57:41 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:41 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:57:41 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:41 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:41 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:57:41 localhost kernel: 
Feb 18 20:57:41 localhost kernel:     -> (&rq->rq_lock_key#4){++..} ops: 233519 {
Feb 18 20:57:41 localhost kernel:        initial-use  at:
Feb 18 20:57:41 localhost kernel:                                 [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:41 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:41 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:41 localhost kernel:                                 [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:41 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:41 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:41 localhost kernel:                                 [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:57:41 localhost kernel:                                 [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:57:42 localhost kernel:                                 [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:57:42 localhost kernel:                                 [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:57:42 localhost kernel:                                 [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:57:42 localhost kernel:                                 [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:57:42 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:42 localhost kernel:                                 [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:42 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:42 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:42 localhost kernel:                                 [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:57:42 localhost kernel:                                 [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:42 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:42 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:42 localhost kernel:                                 [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:57:42 localhost kernel:                                 [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:57:42 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:42 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:42 localhost kernel:                                 [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:42 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 20:57:42 localhost kernel:        in-hardirq-W at:
Feb 18 20:57:42 localhost kernel:                                 [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:57:42 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:42 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:42 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:42 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:42 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:42 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:42 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:42 localhost kernel:                                 [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:57:42 localhost kernel:                                 [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:57:42 localhost kernel:                                 [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:57:42 localhost kernel:                                 [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:57:42 localhost kernel:                                 [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:57:42 localhost kernel:                                 [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:57:42 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:42 localhost kernel:                                 [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:57:42 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:42 localhost kernel:                                 [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:57:42 localhost kernel:                                 [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:57:42 localhost kernel:                                 [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:57:42 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 20:57:42 localhost kernel:        in-softirq-W at:
Feb 18 20:57:43 localhost kernel:                                 [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:43 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:43 localhost kernel:                                 [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:43 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:43 localhost kernel:                                 [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:43 localhost kernel:                                 [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:43 localhost kernel:                                 [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:57:43 localhost kernel:                                 [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:57:43 localhost kernel:                                 [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:57:43 localhost kernel:                                 [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:43 localhost kernel:                                 [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:43 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 20:57:43 localhost kernel:      }
Feb 18 20:57:43 localhost kernel:      ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 20:57:43 localhost kernel:     ... acquired at:
Feb 18 20:57:43 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:43 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:43 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:43 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:43 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:43 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:43 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:43 localhost kernel:    [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 20:57:43 localhost kernel:    [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 20:57:43 localhost kernel:    [<c04235fe>] complete+0x36/0x44
Feb 18 20:57:43 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:43 localhost kernel:    [<c0428996>] migration_thread+0x186/0x1d2
Feb 18 20:57:43 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:43 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:57:43 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:43 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:43 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:57:43 localhost kernel: 
Feb 18 20:57:43 localhost kernel:    ... acquired at:
Feb 18 20:57:43 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:43 localhost kernel:    [<c042366f>] __wake_up+0x18/0x42
Feb 18 20:57:43 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:43 localhost kernel:    [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:57:43 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:43 localhost kernel:    [<c042366f>] __wake_up+0x18/0x42
Feb 18 20:57:43 localhost kernel:    [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:44 localhost kernel:    [<c042366f>] __wake_up+0x18/0x42
Feb 18 20:57:44 localhost kernel:    [<c042366f>] __wake_up+0x18/0x42
Feb 18 20:57:44 localhost kernel:    [<f8c58a2b>] ppp_receive_nonmp_frame+0x51f/0x70a [ppp_generic]
Feb 18 20:57:44 localhost kernel:    [<f8c59487>] ppp_input+0xbf/0xef [ppp_generic]
Feb 18 20:57:44 localhost kernel:    [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 20:57:44 localhost kernel:    [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 20:57:44 localhost kernel:    [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 20:57:44 localhost kernel:    [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 20:57:44 localhost kernel:    [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 20:57:44 localhost kernel:    [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 20:57:44 localhost kernel:    [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 20:57:44 localhost kernel:    [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 20:57:44 localhost kernel:    [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:57:44 localhost kernel:    [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 20:57:44 localhost kernel:    [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 20:57:44 localhost kernel:    [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:57:44 localhost kernel:    [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 20:57:44 localhost kernel:    [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 20:57:44 localhost kernel:    [<f8984f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 20:57:44 localhost kernel:    [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 20:57:44 localhost kernel:    [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 20:57:44 localhost kernel:    [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:44 localhost kernel:    [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:44 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:57:44 localhost kernel: 
Feb 18 20:57:44 localhost kernel:   ... acquired at:
Feb 18 20:57:44 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:44 localhost kernel:    [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:57:44 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:44 localhost kernel:    [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:57:44 localhost kernel:    [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 20:57:44 localhost kernel:    [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:57:44 localhost kernel:    [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:57:44 localhost kernel:    [<c061b008>] __down+0x82/0xb8
Feb 18 20:57:44 localhost kernel:    [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:44 localhost kernel:    [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:44 localhost kernel:    [<c061b008>] __down+0x82/0xb8
Feb 18 20:57:44 localhost kernel:    [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:57:44 localhost kernel:    [<c061ae17>] __down_failed+0x7/0xc
Feb 18 20:57:44 localhost kernel:    [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 20:57:44 localhost kernel:    [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 20:57:45 localhost kernel:    [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 20:57:45 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:57:45 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:57:45 localhost kernel: 
Feb 18 20:57:45 localhost kernel:   -> (&list->lock#7){.+..} ops: 31 {
Feb 18 20:57:45 localhost kernel:      initial-use  at:
Feb 18 20:57:45 localhost kernel:                             [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:45 localhost kernel:                             [<c046b874>] __do_fault+0x31b/0x35d
Feb 18 20:57:45 localhost kernel:                             [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:45 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:57:45 localhost kernel:                             [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:45 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:57:45 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:57:45 localhost kernel:                             [<f8c5939f>] ppp_write+0xb2/0xdb [ppp_generic]
Feb 18 20:57:45 localhost kernel:                             [<f8c592ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 20:57:45 localhost kernel:                             [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 20:57:45 localhost kernel:                             [<c047ecfd>] sys_write+0x41/0x67
Feb 18 20:57:45 localhost kernel:                             [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:57:45 localhost kernel:                             [<ffffffff>] 0xffffffff
Feb 18 20:57:45 localhost kernel:      in-softirq-W at:
Feb 18 20:57:45 localhost kernel:                             [<c0446ab0>] save_trace+0x37/0x8b
Feb 18 20:57:45 localhost kernel:                             [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:45 localhost kernel:                             [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:45 localhost kernel:                             [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:45 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:57:45 localhost kernel:                             [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:45 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:57:45 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:57:45 localhost kernel:                             [<f8c59433>] ppp_input+0x6b/0xef [ppp_generic]
Feb 18 20:57:45 localhost kernel:                             [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 20:57:45 localhost kernel:                             [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 20:57:45 localhost kernel:                             [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 20:57:45 localhost kernel:                             [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 20:57:45 localhost kernel:                             [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 20:57:45 localhost kernel:                             [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 20:57:45 localhost kernel:                             [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 20:57:45 localhost kernel:                             [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 20:57:45 localhost kernel:                             [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:57:45 localhost kernel:                             [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 20:57:45 localhost kernel:                             [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 20:57:45 localhost kernel:                             [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:57:45 localhost kernel:                             [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 20:57:45 localhost kernel:                             [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 20:57:46 localhost kernel:                             [<f8984f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 20:57:46 localhost kernel:                             [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 20:57:46 localhost kernel:                             [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 20:57:46 localhost kernel:                             [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:46 localhost kernel:                             [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:46 localhost kernel:                             [<ffffffff>] 0xffffffff
Feb 18 20:57:46 localhost kernel:    }
Feb 18 20:57:46 localhost kernel:    ... key      at: [<f8c5dd18>] __key.21159+0x0/0xffffb82a [ppp_generic]
Feb 18 20:57:46 localhost kernel:   ... acquired at:
Feb 18 20:57:46 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:46 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 20:57:46 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:46 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:46 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 20:57:46 localhost kernel:    [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:46 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 20:57:46 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 20:57:46 localhost kernel:    [<f8c57424>] ppp_xmit_process+0x529/0x5a1 [ppp_generic]
Feb 18 20:57:46 localhost kernel:    [<f8c5834b>] ppp_channel_push+0x71/0x90 [ppp_generic]
Feb 18 20:57:46 localhost kernel:    [<c061b373>] _read_lock_bh+0x2e/0x39
Feb 18 20:57:46 localhost kernel:    [<f8c5835a>] ppp_channel_push+0x80/0x90 [ppp_generic]
Feb 18 20:57:46 localhost kernel:    [<f8c593bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 20:57:46 localhost kernel:    [<f8c592ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 20:57:46 localhost kernel:    [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 20:57:46 localhost kernel:    [<c047ecfd>] sys_write+0x41/0x67
Feb 18 20:57:46 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:57:46 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:57:46 localhost kernel: 
Feb 18 20:57:46 localhost kernel:  ... acquired at:
Feb 18 20:57:46 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:46 localhost kernel:    [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:57:46 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:46 localhost kernel:    [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:57:46 localhost kernel:    [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 20:57:46 localhost kernel:    [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:57:46 localhost kernel:    [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:57:46 localhost kernel:    [<c061b008>] __down+0x82/0xb8
Feb 18 20:57:46 localhost kernel:    [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:46 localhost kernel:    [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:46 localhost kernel:    [<c061b008>] __down+0x82/0xb8
Feb 18 20:57:46 localhost kernel:    [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:57:47 localhost kernel:    [<c061ae17>] __down_failed+0x7/0xc
Feb 18 20:57:47 localhost kernel:    [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 20:57:47 localhost kernel:    [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 20:57:47 localhost kernel:    [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 20:57:47 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:57:47 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 20:57:47 localhost kernel: 
Feb 18 20:57:47 localhost kernel: 
Feb 18 20:57:47 localhost kernel: the soft-irq-unsafe lock's dependencies:
Feb 18 20:57:47 localhost kernel: -> (&sk->sk_dst_lock){----} ops: 284 {
Feb 18 20:57:47 localhost kernel:    initial-use  at:
Feb 18 20:57:47 localhost kernel:                         [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:47 localhost kernel:                         [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 20:57:47 localhost kernel:                         [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 20:57:47 localhost kernel:                         [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 20:57:47 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:47 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:47 localhost kernel:                         [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 20:57:47 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:47 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:47 localhost kernel:                         [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 20:57:47 localhost kernel:                         [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 20:57:47 localhost kernel:                         [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 20:57:47 localhost kernel:                         [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 20:57:47 localhost kernel:                         [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:47 localhost kernel:                         [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 20:57:47 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 20:57:47 localhost kernel:    softirq-on-W at:
Feb 18 20:57:47 localhost kernel:                         [<c0449046>] __lock_acquire+0x48b/0xbf1
Feb 18 20:57:47 localhost kernel:                         [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 20:57:47 localhost kernel:                         [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 20:57:47 localhost kernel:                         [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 20:57:47 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:47 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:47 localhost kernel:                         [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 20:57:47 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:47 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:47 localhost kernel:                         [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 20:57:47 localhost kernel:                         [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 20:57:47 localhost kernel:                         [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 20:57:47 localhost kernel:                         [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 20:57:47 localhost kernel:                         [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:48 localhost kernel:                         [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 20:57:48 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 20:57:48 localhost kernel:    hardirq-on-W at:
Feb 18 20:57:48 localhost kernel:                         [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 20:57:48 localhost kernel:                         [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 20:57:48 localhost kernel:                         [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 20:57:48 localhost kernel:                         [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 20:57:48 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:48 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:48 localhost kernel:                         [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 20:57:48 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:48 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:48 localhost kernel:                         [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 20:57:48 localhost kernel:                         [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 20:57:48 localhost kernel:                         [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 20:57:48 localhost kernel:                         [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 20:57:48 localhost kernel:                         [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:48 localhost kernel:                         [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 20:57:48 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 20:57:48 localhost kernel:    softirq-on-R at:
Feb 18 20:57:48 localhost kernel:                         [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:48 localhost kernel:                         [<c0449046>] __lock_acquire+0x48b/0xbf1
Feb 18 20:57:48 localhost kernel:                         [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:48 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:48 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 20:57:48 localhost kernel:                         [<c061b3a7>] _read_lock+0x29/0x34
Feb 18 20:57:48 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 20:57:48 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 20:57:48 localhost kernel:                         [<f8a6a9b1>] ipv6_chk_addr+0x8a/0x96 [ipv6]
Feb 18 20:57:48 localhost kernel:                         [<f8a649b0>] ip6_sk_dst_lookup+0x2d/0x17a [ipv6]
Feb 18 20:57:48 localhost kernel:                         [<c0489d61>] __pollwait+0x0/0xac
Feb 18 20:57:48 localhost kernel:                         [<f8a75e6d>] udpv6_sendmsg+0x4c1/0x8a1 [ipv6]
Feb 18 20:57:48 localhost kernel:                         [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:57:48 localhost last message repeated 2 times
Feb 18 20:57:48 localhost kernel:                         [<c05f5fc7>] inet_sendmsg+0x3b/0x45
Feb 18 20:57:48 localhost kernel:                         [<c05ab8cd>] sock_sendmsg+0xc9/0xe4
Feb 18 20:57:48 localhost kernel:                         [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 20:57:48 localhost kernel:                         [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:48 localhost kernel:                         [<c04f05f4>] copy_from_user+0x32/0x5e
Feb 18 20:57:49 localhost kernel:                         [<c04f05f4>] copy_from_user+0x32/0x5e
Feb 18 20:57:49 localhost kernel:                         [<c05aba7a>] sys_sendmsg+0x192/0x1f7
Feb 18 20:57:49 localhost kernel:                         [<c048f764>] file_update_time+0x22/0x6a
Feb 18 20:57:49 localhost kernel:                         [<c0483d90>] pipe_write+0x392/0x3ef
Feb 18 20:57:49 localhost kernel:                         [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:49 localhost kernel:                         [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 20:57:49 localhost kernel:                         [<c05acd2b>] sys_socketcall+0x240/0x261
Feb 18 20:57:49 localhost kernel:                         [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:57:49 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 20:57:49 localhost kernel:    hardirq-on-R at:
Feb 18 20:57:49 localhost kernel:                         [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:49 localhost kernel:                         [<c0449001>] __lock_acquire+0x446/0xbf1
Feb 18 20:57:49 localhost kernel:                         [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:49 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:49 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 20:57:49 localhost kernel:                         [<c061b3a7>] _read_lock+0x29/0x34
Feb 18 20:57:49 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 20:57:49 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 20:57:49 localhost kernel:                         [<f8a6a9b1>] ipv6_chk_addr+0x8a/0x96 [ipv6]
Feb 18 20:57:49 localhost kernel:                         [<f8a649b0>] ip6_sk_dst_lookup+0x2d/0x17a [ipv6]
Feb 18 20:57:49 localhost kernel:                         [<c0489d61>] __pollwait+0x0/0xac
Feb 18 20:57:49 localhost kernel:                         [<f8a75e6d>] udpv6_sendmsg+0x4c1/0x8a1 [ipv6]
Feb 18 20:57:49 localhost kernel:                         [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:57:49 localhost last message repeated 2 times
Feb 18 20:57:49 localhost kernel:                         [<c05f5fc7>] inet_sendmsg+0x3b/0x45
Feb 18 20:57:49 localhost kernel:                         [<c05ab8cd>] sock_sendmsg+0xc9/0xe4
Feb 18 20:57:49 localhost kernel:                         [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 20:57:49 localhost kernel:                         [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:49 localhost kernel:                         [<c04f05f4>] copy_from_user+0x32/0x5e
Feb 18 20:57:49 localhost kernel:                         [<c04f05f4>] copy_from_user+0x32/0x5e
Feb 18 20:57:49 localhost kernel:                         [<c05aba7a>] sys_sendmsg+0x192/0x1f7
Feb 18 20:57:49 localhost kernel:                         [<c048f764>] file_update_time+0x22/0x6a
Feb 18 20:57:49 localhost kernel:                         [<c0483d90>] pipe_write+0x392/0x3ef
Feb 18 20:57:49 localhost kernel:                         [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:49 localhost kernel:                         [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 20:57:49 localhost kernel:                         [<c05acd2b>] sys_socketcall+0x240/0x261
Feb 18 20:57:49 localhost kernel:                         [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:57:49 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 20:57:49 localhost kernel:  }
Feb 18 20:57:49 localhost kernel:  ... key      at: [<c0a3dc28>] __key.35996+0x0/0x8
Feb 18 20:57:49 localhost kernel: 
Feb 18 20:57:49 localhost kernel: stack backtrace:
Feb 18 20:57:50 localhost kernel: Pid: 3241, comm: pppd Not tainted 2.6.24.2 #1
Feb 18 20:57:50 localhost kernel:  [<c0448b1e>] check_usage+0x24e/0x258
Feb 18 20:57:50 localhost kernel:  [<c0449555>] __lock_acquire+0x99a/0xbf1
Feb 18 20:57:50 localhost kernel:  [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:50 localhost kernel:  [<f8c566db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 20:57:50 localhost kernel:  [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 20:57:50 localhost kernel:  [<f8c566db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 20:57:50 localhost kernel:  [<f8c566db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 20:57:50 localhost kernel:  [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:50 localhost kernel:  [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 20:57:50 localhost kernel:  [<f8c573f1>] ppp_xmit_process+0x4f6/0x5a1 [ppp_generic]
Feb 18 20:57:50 localhost kernel:  [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:50 localhost kernel:  [<f8c593b4>] ppp_write+0xc7/0xdb [ppp_generic]
Feb 18 20:57:50 localhost kernel:  [<f8c592ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 20:57:50 localhost kernel:  [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 20:57:50 localhost kernel:  [<c047ecfd>] sys_write+0x41/0x67
Feb 18 20:57:50 localhost kernel:  [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:57:50 localhost kernel:  =======================

> 
> Thanks,
> Jarek P.
> 
> On Tue, Feb 12, 2008 at 10:58:21AM +0000, James Chapman wrote:
> ...
>> Here is a trace from when we had _bh locks.
>>
>> Feb  5 16:26:32  ======================================================
>> Feb  5 16:26:32  [ INFO: soft-safe -> soft-unsafe lock order detected ]
>> Feb  5 16:26:32  2.6.24-core2 #1
>> Feb  5 16:26:32  ------------------------------------------------------
>> Feb  5 16:26:32  pppd/3224 [HC0[0]:SC0[2]:HE1:SE0] is trying to acquire:
> ...
>> Feb  5 16:26:32  [<f8d7b84d>] e1000_clean+0x5d/0x290 [e1000]
>> Feb  5 16:26:32  [<c039d580>] net_rx_action+0x1a0/0x2a0
>> Feb  5 16:26:32  [<c039d43f>] net_rx_action+0x5f/0x2a0
>> Feb  5 16:26:32  [<c0131e72>] __do_softirq+0x92/0x120
>> Feb  5 16:26:32  [<c0131f78>] do_softirq+0x78/0x80
>> Feb  5 16:26:32  [<c010b15a>] do_IRQ+0x4a/0xa0
>> Feb  5 16:26:32  [<c0127af0>] finish_task_switch+0x0/0xc0
>> Feb  5 16:26:32  [<c0108dcc>] common_interrupt+0x24/0x34
>> Feb  5 16:26:32  [<c0108dd6>] common_interrupt+0x2e/0x34
>> Feb  5 16:26:32  [<c01062d6>] mwait_idle_with_hints+0x46/0x60
>> Feb  5 16:26:32  [<c0106550>] mwait_idle+0x0/0x20
>> Feb  5 16:26:32  [<c0106694>] cpu_idle+0x74/0xe0
>> Feb  5 16:26:32  [<c0536a9a>] start_kernel+0x30a/0x3a0

-- 
James Chapman
Katalix Systems Ltd
http://www.katalix.com
Catalysts for your Embedded Linux software development

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Mon, Feb 18, 2008 at 10:09:24PM +0000, James Chapman wrote:
> Jarek Poplawski wrote:
>> Hi,
>>
>> It seems, this nice report is still uncomplete: could you check if
>> there could have been something more yet?
>
> Unfortunately the ISP's syslog stops. But I've been able to borrow two
> Quad Xeon boxes and have reproduced the problem.
>
> Here's a new version of the patch. The patch avoids disabling irqs and
> fixes the sk_dst_get() usage that DaveM mentioned. But even with this
> patch, lockdep still complains if hundreds of ppp sessions are inserted
> into a tunnel as rapidly as possible (lockdep trace is below). I can
> stop these errors by wrapping the call to ppp_input() in
> pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is a
> better fix?

Hmm... This is a really long report and quite a bit different from
the previous one. I need some time for this. BTW: you sent before a
lockdep report with hlist_lock problem. I think this could be fixed
in some independent patch to make this all more readable. Are all
the other changes in this current patch only because of this or
previous lockdep report or for some other reasons (or reports) yet?

Regards,
Jarek P.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[David Miller]

From: James Chapman <jchapman@katalix.com>
Date: Mon, 18 Feb 2008 22:09:24 +0000

> Here's a new version of the patch. The patch avoids disabling irqs
> and fixes the sk_dst_get() usage that DaveM mentioned. But even with
> this patch, lockdep still complains if hundreds of ppp sessions are
> inserted into a tunnel as rapidly as possible (lockdep trace is
> below). I can stop these errors by wrapping the call to ppp_input()
> in pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is
> a better fix?

Firstly, let's fix one thing at a time.  Leave the sk_dst_get()
thing alone until we can prove that it's part of the lockdep
traces.

Next, I can't see why ppp_input() needs to be invoked with
interrupts disabled.  There are many other things that invoke
that in software interrupt context, such as pppoe.

Please provide the lockdep traces without the ppp_input() IRQ
disabling so this can be properly analyzed.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[James Chapman]

David Miller wrote:
> From: James Chapman <jchapman@katalix.com>
> Date: Mon, 18 Feb 2008 22:09:24 +0000
> 
>> Here's a new version of the patch. The patch avoids disabling irqs
>> and fixes the sk_dst_get() usage that DaveM mentioned. But even with
>> this patch, lockdep still complains if hundreds of ppp sessions are
>> inserted into a tunnel as rapidly as possible (lockdep trace is
>> below). I can stop these errors by wrapping the call to ppp_input()
>> in pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is
>> a better fix?
> 
> Firstly, let's fix one thing at a time.  Leave the sk_dst_get()
> thing alone until we can prove that it's part of the lockdep
> traces.

In reproducing the problem, I obtained several lockdep traces that 
implicated sk_dst_get(). I changed the code to use __sk_dst_check() as 
you suggested and they went away. At that point, I was hopeful the 
locking issues were fixed. But after several minutes of 
creating/deleting hundreds of ppp sessions, lockdep dumped another 
error. It is that error that I posted yesterday.

> Next, I can't see why ppp_input() needs to be invoked with
> interrupts disabled.  There are many other things that invoke
> that in software interrupt context, such as pppoe.

I agree. I'm seeking advice on what the underlying cause is of this new 
trace.

> Please provide the lockdep traces without the ppp_input() IRQ
> disabling so this can be properly analyzed.

The trace _was_ without ppp_input IRQ disabling. The trace doesn't occur 
if I disable IRQs around the ppp_input() call. The patch I sent showed 
the changes I made before running the tests that created the new lockdep 
trace. I'm sorry this wasn't clear.


-- 
James Chapman
Katalix Systems Ltd
http://www.katalix.com
Catalysts for your Embedded Linux software development

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[James Chapman]

Jarek Poplawski wrote:
> On Mon, Feb 18, 2008 at 10:09:24PM +0000, James Chapman wrote:
>> Jarek Poplawski wrote:
>>> Hi,
>>>
>>> It seems, this nice report is still uncomplete: could you check if
>>> there could have been something more yet?
>> Unfortunately the ISP's syslog stops. But I've been able to borrow two
>> Quad Xeon boxes and have reproduced the problem.
>>
>> Here's a new version of the patch. The patch avoids disabling irqs and
>> fixes the sk_dst_get() usage that DaveM mentioned. But even with this
>> patch, lockdep still complains if hundreds of ppp sessions are inserted
>> into a tunnel as rapidly as possible (lockdep trace is below). I can
>> stop these errors by wrapping the call to ppp_input() in
>> pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is a
>> better fix?
> 
> Hmm... This is a really long report and quite a bit different from
> the previous one. I need some time for this. BTW: you sent before a
> lockdep report with hlist_lock problem. I think this could be fixed
> in some independent patch to make this all more readable. Are all
> the other changes in this current patch only because of this or
> previous lockdep report or for some other reasons (or reports) yet?

As I mentioned in my reply to davem, modifying the pppol2tp driver as 
described in the patch I sent made the original lockdep problems go away.


-- 
James Chapman
Katalix Systems Ltd
http://www.katalix.com
Catalysts for your Embedded Linux software development

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Tue, Feb 19, 2008 at 09:03:12AM +0000, James Chapman wrote:
> David Miller wrote:
>> From: James Chapman <jchapman@katalix.com>
>> Date: Mon, 18 Feb 2008 22:09:24 +0000
>>
>>> Here's a new version of the patch. The patch avoids disabling irqs
>>> and fixes the sk_dst_get() usage that DaveM mentioned. But even with
>>> this patch, lockdep still complains if hundreds of ppp sessions are
>>> inserted into a tunnel as rapidly as possible (lockdep trace is
>>> below). I can stop these errors by wrapping the call to ppp_input()
>>> in pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is
>>> a better fix?
>>
>> Firstly, let's fix one thing at a time.  Leave the sk_dst_get()
>> thing alone until we can prove that it's part of the lockdep
>> traces.
>
> In reproducing the problem, I obtained several lockdep traces that  
> implicated sk_dst_get().

As a matter of fact I missed just that kind information on previous
lockdep report, so if you could send them too this should be still
helpful.

...
> I agree. I'm seeking advice on what the underlying cause is of this new  
> trace.

IMHO, just like I wrote earlier, the main problem is in ppp_generic(),
especially ppp_connect_channel(), where main tx & rx locks are used.
I didn't know enough about this sk_dst_lock traces yet. I hope I could
help with this, but after these changes I need some time to figure
this out again.

Jarek P.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Tue, Feb 19, 2008 at 10:30:47AM +0000, Jarek Poplawski wrote:
...
> IMHO, just like I wrote earlier, the main problem is in ppp_generic(),

...or maybe ppp_generic.c? Whatever...

Jarek P.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[James Chapman]

[-- Attachment #1: Type: text/plain, Size: 1402 bytes --]

Jarek Poplawski wrote:
> On Tue, Feb 19, 2008 at 09:03:12AM +0000, James Chapman wrote:
>> David Miller wrote:
>>> From: James Chapman <jchapman@katalix.com>
>>> Date: Mon, 18 Feb 2008 22:09:24 +0000
>>>
>>>> Here's a new version of the patch. The patch avoids disabling irqs
>>>> and fixes the sk_dst_get() usage that DaveM mentioned. But even with
>>>> this patch, lockdep still complains if hundreds of ppp sessions are
>>>> inserted into a tunnel as rapidly as possible (lockdep trace is
>>>> below). I can stop these errors by wrapping the call to ppp_input()
>>>> in pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is
>>>> a better fix?
>>> Firstly, let's fix one thing at a time.  Leave the sk_dst_get()
>>> thing alone until we can prove that it's part of the lockdep
>>> traces.
>> In reproducing the problem, I obtained several lockdep traces that  
>> implicated sk_dst_get().
> 
> As a matter of fact I missed just that kind information on previous
> lockdep report, so if you could send them too this should be still
> helpful.

Here you go.

Just to be clear, the initial lockdep report was with non-bh locks. 
Yesterday's report was with _bh locks and the sk_dst_get() changes. This 
(older) report is with _bh locks but without the sk_dst_get() changes.

-- 
James Chapman
Katalix Systems Ltd
http://www.katalix.com
Catalysts for your Embedded Linux software development


[-- Attachment #2: lockdep-sk_dst.log --]
[-- Type: text/plain, Size: 205001 bytes --]

Feb 18 17:25:42 localhost kernel: ======================================================
Feb 18 17:25:42 localhost kernel: [ INFO: soft-safe -> soft-unsafe lock order detected ]
Feb 18 17:25:42 localhost kernel: 2.6.24.2 #1
Feb 18 17:25:42 localhost kernel: ------------------------------------------------------
Feb 18 17:25:42 localhost kernel: pppd/13863 [HC0[0]:SC0[1]:HE1:SE0] is trying to acquire:
Feb 18 17:25:42 localhost kernel:  (&sk->sk_dst_lock){----}, at: [<f8bbf73e>] pppol2tp_xmit+0x3a5/0x4a2 [pppol2tp]
Feb 18 17:25:42 localhost kernel: 
Feb 18 17:25:42 localhost kernel: and this task is already holding:
Feb 18 17:25:42 localhost kernel:  (&pch->downl){-...}, at: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:25:42 localhost kernel: which would create a new lock dependency:
Feb 18 17:25:42 localhost pppd[13863]: Connection terminated.
Feb 18 17:25:42 localhost kernel:  (&pch->downl){-...} -> (&sk->sk_dst_lock){----}
Feb 18 17:25:42 localhost pppd[13863]: Exit.
Feb 18 17:25:42 localhost kernel: 
Feb 18 17:25:42 localhost kernel: but this new dependency connects a soft-irq-safe lock:
Feb 18 17:25:42 localhost kernel:  (&pch->upl){-.-+}
Feb 18 17:25:42 localhost kernel: ... which became soft-irq-safe at:
Feb 18 17:25:43 localhost kernel:   [<c0448013>] check_usage_backwards+0x19/0x41
Feb 18 17:25:43 localhost kernel:   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:43 localhost kernel:   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:43 localhost kernel:   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:43 localhost kernel:   [<f8bb840d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 17:25:43 localhost kernel:   [<c061b373>] _read_lock_bh+0x2e/0x39
Feb 18 17:25:43 localhost kernel:   [<f8bb840d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 17:25:43 localhost kernel:   [<f8bb840d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 17:25:43 localhost kernel:   [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:25:43 localhost kernel:   [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:25:43 localhost kernel:   [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:25:43 localhost kernel:   [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:25:43 localhost kernel:   [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:25:43 localhost kernel:   [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:25:43 localhost kernel:   [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:25:43 localhost kernel:   [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:25:43 localhost kernel:   [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:25:43 localhost kernel:   [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:25:43 localhost kernel:   [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:25:43 localhost kernel:   [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:25:43 localhost kernel:   [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:25:43 localhost kernel:   [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:25:43 localhost kernel:   [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:25:43 localhost kernel:   [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:25:43 localhost kernel:   [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:25:43 localhost kernel:   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:43 localhost kernel:   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:43 localhost kernel:   [<ffffffff>] 0xffffffff
Feb 18 17:25:43 localhost kernel: 
Feb 18 17:25:43 localhost kernel: to a soft-irq-unsafe lock:
Feb 18 17:25:43 localhost kernel:  (&sk->sk_dst_lock){----}
Feb 18 17:25:43 localhost kernel: ... which became soft-irq-unsafe at:
Feb 18 17:25:43 localhost kernel: ...  [<c0449046>] __lock_acquire+0x48b/0xbf1
Feb 18 17:25:43 localhost kernel:   [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:25:43 localhost kernel:   [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 17:25:43 localhost kernel:   [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 17:25:43 localhost kernel:   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:43 localhost kernel:   [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:25:43 localhost kernel:   [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 17:25:43 localhost kernel:   [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:25:43 localhost kernel:   [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:25:43 localhost kernel:   [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 17:25:44 localhost kernel:   [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 17:25:44 localhost kernel:   [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 17:25:44 localhost kernel:   [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 17:25:44 localhost kernel:   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:44 localhost kernel:   [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:25:44 localhost kernel:   [<ffffffff>] 0xffffffff
Feb 18 17:25:44 localhost kernel: 
Feb 18 17:25:44 localhost kernel: other info that might help us debug this:
Feb 18 17:25:44 localhost kernel: 
Feb 18 17:25:44 localhost kernel: 1 lock held by pppd/13863:
Feb 18 17:25:44 localhost kernel:  #0:  (&pch->downl){-...}, at: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:25:44 localhost kernel: 
Feb 18 17:25:44 localhost kernel: the soft-irq-safe lock's dependencies:
Feb 18 17:25:44 localhost kernel: -> (&pch->upl){-.-+} ops: 13 {
Feb 18 17:25:44 localhost kernel:    initial-use  at:
Feb 18 17:25:44 localhost kernel:                         [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:44 localhost kernel:                         [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:44 localhost kernel:                         [<f8bb6b7c>] ppp_ioctl+0x4df/0xc06 [ppp_generic]
Feb 18 17:25:44 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:44 localhost kernel:                         [<f8bb6ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 17:25:44 localhost kernel:                         [<c061b300>] _write_lock_bh+0x2e/0x39
Feb 18 17:25:44 localhost kernel:                         [<f8bb6ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 17:25:44 localhost kernel:                         [<f8bb6ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 17:25:44 localhost kernel:                         [<c061b008>] __down+0x82/0xb8
Feb 18 17:25:44 localhost kernel:                         [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:44 localhost kernel:                         [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:44 localhost kernel:                         [<c061b008>] __down+0x82/0xb8
Feb 18 17:25:44 localhost kernel:                         [<c0425474>] default_wake_function+0x0/0x8
Feb 18 17:25:44 localhost kernel:                         [<c061ae17>] __down_failed+0x7/0xc
Feb 18 17:25:44 localhost kernel:                         [<c0488854>] do_ioctl+0x4ebalance_domains+0x28/0x99
Feb 18 17:25:44 localhost kernel:                                       [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:44 localhost kernel:                                       [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:44 localhost kernel:                                       [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:44 localhost kernel:                                       [<ffffffff>] 0xffffffff
Feb 18 17:25:44 localhost kernel:         }
Feb 18 17:25:44 localhost kernel:         ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:25:44 localhost kernel:        ... acquired at:
Feb 18 17:25:44 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:44 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:44 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:45 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:45 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:45 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:45 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:45 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:45 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:45 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:45 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:45 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:25:45 localhost kernel: 
Feb 18 17:25:45 localhost kernel:       ... acquired at:
Feb 18 17:25:45 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:45 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:45 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:45 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:45 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:45 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:45 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:45 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:45 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:45 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:45 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:25:45 localhost kernel: 
Feb 18 17:25:45 localhost kernel:       -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:25:45 localhost kernel:          initial-use  at:
Feb 18 17:25:45 localhost kernel:                                     [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:45 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:45 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:45 localhost kernel:                                     [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:45 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:45 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:45 localhost kernel:                                     [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:45 localhost kernel:                                     [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:45 localhost kernel:                                     [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:46 localhost kernel:                                     [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:46 localhost kernel:                                     [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:46 localhost kernel:                                     [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:46 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:46 localhost kernel:                                     [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:46 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:46 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:46 localhost kernel:                                     [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:46 localhost kernel:                                     [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:46 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:46 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:46 localhost kernel:                                     [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:46 localhost kernel:                                     [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:46 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:46 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:46 localhost kernel:                                     [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:46 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:46 localhost kernel:          in-hardirq-W at:
Feb 18 17:25:46 localhost kernel:                                     [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:46 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:46 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:46 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:46 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:46 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:46 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:46 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:46 localhost kernel:                                     [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:46 localhost kernel:                                     [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:46 localhost kernel:                                     [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:46 localhost kernel:                                     [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:46 localhost kernel:                                     [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:46 localhost kernel:                                     [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:46 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:46 localhost kernel:                                     [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:46 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:46 localhost kernel:                                     [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:46 localhost kernel:                                     [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:46 localhost kernel:                                     [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:46 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:46 localhost kernel:          in-softirq-W at:
Feb 18 17:25:46 localhost kernel:                                     [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:25:46 localhost kernel:                                     [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:47 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:47 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:47 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:                                     [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:47 localhost kernel:                                     [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:47 localhost kernel:                                     [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:47 localhost kernel:                                     [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:47 localhost kernel:                                     [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:47 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:47 localhost kernel:        }
Feb 18 17:25:47 localhost kernel:        ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:25:47 localhost kernel:       ... acquired at:
Feb 18 17:25:47 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:47 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:47 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:47 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:47 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:47 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:47 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:47 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:47 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:47 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:47 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:25:47 localhost kernel: 
Feb 18 17:25:47 localhost kernel:      ... acquired at:
Feb 18 17:25:47 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:47 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:47 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:47 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:    [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:47 localhost kernel:    [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:48 localhost kernel:    [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:48 localhost kernel:    [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:48 localhost kernel:    [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:48 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:25:48 localhost kernel: 
Feb 18 17:25:48 localhost kernel:      -> (&rq->rq_lock_key#3){++..} ops: 1928539 {
Feb 18 17:25:48 localhost kernel:         initial-use  at:
Feb 18 17:25:48 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:48 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:48 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:48 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:48 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:48 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:48 localhost kernel:                                   [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:48 localhost kernel:                                   [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:48 localhost kernel:                                   [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:48 localhost kernel:                                   [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:48 localhost kernel:                                   [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:48 localhost kernel:                                   [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:48 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:48 localhost kernel:                                   [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:48 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:48 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:48 localhost kernel:                                   [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:48 localhost kernel:                                   [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:48 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:48 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:48 localhost kernel:                                   [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:48 localhost kernel:                                   [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:48 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:48 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:48 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:48 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:48 localhost kernel:         in-hardirq-W at:
Feb 18 17:25:48 localhost kernel:                                   [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:48 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:48 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:48 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:48 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:48 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:48 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:48 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:48 localhost kernel:                                   [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:49 localhost kernel:                                   [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:49 localhost kernel:                                   [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:49 localhost kernel:                                   [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:49 localhost kernel:                                   [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:49 localhost kernel:                                   [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:49 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:49 localhost kernel:                                   [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:49 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:49 localhost kernel:                                   [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:49 localhost kernel:                                   [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:49 localhost kernel:                                   [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:49 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:49 localhost kernel:         in-softirq-W at:
Feb 18 17:25:49 localhost kernel:                                   [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:49 localhost kernel:                                   [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:25:49 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:49 localhost kernel:                                   [<c04422f1>] getnstimeofday+0x30/0xbc
Feb 18 17:25:49 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:49 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:49 localhost kernel:                                   [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:49 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:49 localhost kernel:                                   [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:49 localhost kernel:                                   [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:49 localhost kernel:                                   [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:25:49 localhost kernel:                                   [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:25:49 localhost kernel:                                   [<c0430afa>] __do_softirq+0xc9/0xde
Feb 18 17:25:49 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:49 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:49 localhost kernel:       }
Feb 18 17:25:49 localhost kernel:       ... key      at: [<c2a34488>] 0xc2a34488
Feb 18 17:25:49 localhost kernel:       -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:25:49 localhost kernel:          initial-use  at:
Feb 18 17:25:49 localhost kernel:                                     [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:49 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:49 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:49 localhost kernel:                                     [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:49 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:49 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:49 localhost kernel:                                     [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:50 localhost kernel:                                     [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:50 localhost kernel:                                     [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:50 localhost kernel:                                     [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:50 localhost kernel:                                     [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:50 localhost kernel:                                     [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:50 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:50 localhost kernel:                                     [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:50 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:50 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:50 localhost kernel:                                     [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:50 localhost kernel:                                     [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:50 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:50 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:50 localhost kernel:                                     [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:50 localhost kernel:                                     [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:50 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:50 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:50 localhost kernel:                                     [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:50 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:50 localhost kernel:          in-hardirq-W at:
Feb 18 17:25:50 localhost kernel:                                     [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:50 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:50 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:50 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:50 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:50 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:50 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:50 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:50 localhost kernel:                                     [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:50 localhost kernel:                                     [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:50 localhost kernel:                                     [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:50 localhost kernel:                                     [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:50 localhost kernel:                                     [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:50 localhost kernel:                                     [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:50 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:50 localhost kernel:                                     [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:50 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:50 localhost kernel:                                     [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:50 localhost kernel:                                     [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:50 localhost kernel:                                     [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:50 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:51 localhost kernel:          in-softirq-W at:
Feb 18 17:25:51 localhost kernel:                                     [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:25:51 localhost kernel:                                     [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:51 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:51 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:51 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:                                     [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:51 localhost kernel:                                     [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:51 localhost kernel:                                     [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:51 localhost kernel:                                     [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:51 localhost kernel:                                     [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:51 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:51 localhost kernel:        }
Feb 18 17:25:51 localhost kernel:        ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:25:51 localhost kernel:       ... acquired at:
Feb 18 17:25:51 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:51 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:51 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:51 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:51 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:51 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:51 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:51 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:51 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:51 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:51 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:25:51 localhost kernel: 
Feb 18 17:25:51 localhost kernel:      ... acquired at:
Feb 18 17:25:51 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:51 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:51 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:51 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:    [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:52 localhost kernel:    [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:52 localhost kernel:    [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:52 localhost kernel:    [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:52 localhost kernel:    [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:52 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:25:52 localhost kernel: 
Feb 18 17:25:52 localhost kernel:      -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:25:52 localhost kernel:         initial-use  at:
Feb 18 17:25:52 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:52 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:52 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:52 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:52 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:52 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:52 localhost kernel:                                   [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:52 localhost kernel:                                   [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:52 localhost kernel:                                   [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:52 localhost kernel:                                   [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:52 localhost kernel:                                   [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:52 localhost kernel:                                   [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:52 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:52 localhost kernel:                                   [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:52 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:52 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:52 localhost kernel:                                   [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:52 localhost kernel:                                   [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:52 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:52 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:52 localhost kernel:                                   [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:52 localhost kernel:                                   [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:52 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:52 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:52 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:52 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:52 localhost kernel:         in-hardirq-W at:
Feb 18 17:25:52 localhost kernel:                                   [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:52 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:52 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:52 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:53 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:53 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:53 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:53 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:53 localhost kernel:                                   [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:53 localhost kernel:                                   [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:53 localhost kernel:                                   [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:53 localhost kernel:                                   [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:53 localhost kernel:                                   [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:53 localhost kernel:                                   [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:53 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:53 localhost kernel:                                   [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:53 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:53 localhost kernel:                                   [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:53 localhost kernel:                                   [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:53 localhost kernel:                                   [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:53 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:53 localhost kernel:         in-softirq-W at:
Feb 18 17:25:53 localhost kernel:                                   [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:25:53 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:53 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:53 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:53 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel:                                   [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:53 localhost kernel:                                   [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:53 localhost kernel:                                   [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:53 localhost kernel:                                   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:53 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:53 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:53 localhost kernel:       }
Feb 18 17:25:53 localhost kernel:       ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:25:53 localhost kernel:      ... acquired at:
Feb 18 17:25:53 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:53 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:53 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:53 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:54 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:54 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:54 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:54 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:54 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:54 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:54 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:25:54 localhost kernel: 
Feb 18 17:25:54 localhost kernel:     ... acquired at:
Feb 18 17:25:54 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:54 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:54 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:54 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:54 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:54 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:54 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:54 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:54 localhost kernel:    [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:25:54 localhost kernel:    [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 17:25:54 localhost kernel:    [<c04235fe>] complete+0x36/0x44
Feb 18 17:25:54 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:54 localhost kernel:    [<c043d502>] kthread+0x1c/0x5e
Feb 18 17:25:54 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:54 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:54 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:25:54 localhost kernel: 
Feb 18 17:25:54 localhost kernel:     -> (&rq->rq_lock_key#2){++..} ops: 1235688 {
Feb 18 17:25:54 localhost kernel:        initial-use  at:
Feb 18 17:25:54 localhost kernel:                                 [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:54 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:54 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:54 localhost kernel:                                 [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:54 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:54 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:54 localhost kernel:                                 [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:54 localhost kernel:                                 [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:54 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:54 localhost kernel:                                 [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:54 localhost kernel:                                 [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:54 localhost kernel:                                 [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:55 localhost kernel:                                 [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:55 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:55 localhost kernel:                                 [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:55 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:55 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:55 localhost kernel:                                 [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:55 localhost kernel:                                 [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:55 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:55 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:55 localhost kernel:                                 [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:55 localhost kernel:                                 [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:55 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:55 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:55 localhost kernel:                                 [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:55 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:25:55 localhost kernel:        in-hardirq-W at:
Feb 18 17:25:55 localhost kernel:                                 [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:25:55 localhost kernel:                                 [<c040a9d6>] save_stack_trace+0x20/0x3a
Feb 18 17:25:55 localhost kernel:                                 [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:55 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:55 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:55 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:55 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:55 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:55 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:55 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:55 localhost kernel:                                 [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:55 localhost kernel:                                 [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:55 localhost kernel:                                 [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:55 localhost kernel:                                 [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:55 localhost kernel:                                 [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:55 localhost kernel:                                 [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:55 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:55 localhost kernel:                                 [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:55 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:55 localhost kernel:                                 [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:55 localhost kernel:                                 [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:55 localhost kernel:                                 [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:55 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:25:55 localhost kernel:        in-softirq-W at:
Feb 18 17:25:55 localhost kernel:                                 [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:56 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:56 localhost kernel:                                 [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:56 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:56 localhost kernel:                                 [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:56 localhost kernel:                                 [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:56 localhost kernel:                                 [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:56 localhost kernel:                                 [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:56 localhost kernel:                                 [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:56 localhost kernel:                                 [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:56 localhost kernel:                                 [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:56 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:25:56 localhost kernel:      }
Feb 18 17:25:56 localhost kernel:      ... key      at: [<c2a2b488>] 0xc2a2b488
Feb 18 17:25:56 localhost kernel:      -> (&rq->rq_lock_key#3){++..} ops: 1928539 {
Feb 18 17:25:56 localhost kernel:         initial-use  at:
Feb 18 17:25:56 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:56 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:56 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:56 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:56 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:56 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:56 localhost kernel:                                   [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:56 localhost kernel:                                   [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:56 localhost kernel:                                   [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:56 localhost kernel:                                   [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:56 localhost kernel:                                   [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:56 localhost kernel:                                   [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:56 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:56 localhost kernel:                                   [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:56 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:56 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:56 localhost kernel:                                   [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:56 localhost kernel:                                   [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:56 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:56 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:56 localhost kernel:                                   [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:56 localhost kernel:                                   [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:56 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:56 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:56 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:56 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:56 localhost kernel:         in-hardirq-W at:
Feb 18 17:25:57 localhost kernel:                                   [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:57 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:57 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:57 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:57 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:57 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:57 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:57 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:57 localhost kernel:                                   [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:57 localhost kernel:                                   [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:57 localhost kernel:                                   [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:57 localhost kernel:                                   [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:57 localhost kernel:                                   [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:57 localhost kernel:                                   [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:57 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:57 localhost kernel:                                   [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:57 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:57 localhost kernel:                                   [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:57 localhost kernel:                                   [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:57 localhost kernel:                                   [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:57 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:57 localhost kernel:         in-softirq-W at:
Feb 18 17:25:57 localhost kernel:                                   [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:57 localhost kernel:                                   [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:25:57 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:57 localhost kernel:                                   [<c04422f1>] getnstimeofday+0x30/0xbc
Feb 18 17:25:57 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:57 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:57 localhost kernel:                                   [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:57 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:57 localhost kernel:                                   [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:57 localhost kernel:                                   [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:57 localhost kernel:                                   [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:25:57 localhost kernel:                                   [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:25:57 localhost kernel:                                   [<c0430afa>] __do_softirq+0xc9/0xde
Feb 18 17:25:57 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:57 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:57 localhost kernel:       }
Feb 18 17:25:57 localhost kernel:       ... key      at: [<c2a34488>] 0xc2a34488
Feb 18 17:25:57 localhost kernel:       -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:25:58 localhost kernel:          initial-use  at:
Feb 18 17:25:58 localhost kernel:                                     [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:58 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:58 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:58 localhost kernel:                                     [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:58 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:58 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:58 localhost kernel:                                     [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:58 localhost kernel:                                     [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:58 localhost kernel:                                     [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:58 localhost kernel:                                     [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:58 localhost kernel:                                     [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:58 localhost kernel:                                     [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:58 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:58 localhost kernel:                                     [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:58 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:58 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:58 localhost kernel:                                     [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:58 localhost kernel:                                     [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:58 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:58 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:58 localhost kernel:                                     [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:58 localhost kernel:                                     [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:58 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:58 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:58 localhost kernel:                                     [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:58 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:58 localhost kernel:          in-hardirq-W at:
Feb 18 17:25:58 localhost kernel:                                     [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:58 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:58 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:58 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:58 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:58 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:58 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:58 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:58 localhost kernel:                                     [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:58 localhost kernel:                                     [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:58 localhost kernel:                                     [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:58 localhost kernel:                                     [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:58 localhost kernel:                                     [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:59 localhost kernel:                                     [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:59 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:59 localhost kernel:                                     [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:59 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:59 localhost kernel:                                     [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:59 localhost kernel:                                     [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:59 localhost kernel:                                     [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:59 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:59 localhost kernel:          in-softirq-W at:
Feb 18 17:25:59 localhost kernel:                                     [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:25:59 localhost kernel:                                     [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:59 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:59 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:59 localhost openl2tpd: Loading plugin ppp_unix.so, version V1.0
Feb 18 17:25:59 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost openl2tpd[14004]: Start, trace_flags=00000000
Feb 18 17:25:59 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost openl2tpd[14004]: OpenL2TP V1.0, (c) Copyright 2004,2005,2006,2007,2008 Katalix Systems Ltd.
Feb 18 17:25:59 localhost kernel:                                     [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:59 localhost kernel:                                     [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:59 localhost kernel:                                     [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:59 localhost kernel:                                     [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:59 localhost kernel:                                     [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:59 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:59 localhost kernel:        }
Feb 18 17:25:59 localhost kernel:        ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:25:59 localhost kernel:       ... acquired at:
Feb 18 17:25:59 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:59 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:59 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:59 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:59 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:59 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:59 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:59 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:59 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:00 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:00 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:00 localhost kernel: 
Feb 18 17:26:00 localhost kernel:      ... acquired at:
Feb 18 17:26:00 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:00 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:00 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:00 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:00 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:00 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:00 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:00 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:26:00 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:00 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:26:00 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:00 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:00 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:00 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:00 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:00 localhost kernel: 
Feb 18 17:26:00 localhost kernel:      -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:26:00 localhost kernel:         initial-use  at:
Feb 18 17:26:00 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:00 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:00 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:00 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:00 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:00 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:00 localhost kernel:                                   [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:26:00 localhost kernel:                                   [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:26:00 localhost kernel:                                   [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:26:00 localhost kernel:                                   [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:26:00 localhost kernel:                                   [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:26:00 localhost kernel:                                   [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:26:00 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:00 localhost kernel:                                   [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:00 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:00 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:00 localhost kernel:                                   [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:26:00 localhost kernel:                                   [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:26:01 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:01 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:01 localhost kernel:                                   [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:26:01 localhost kernel:                                   [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:26:01 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:01 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:01 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:01 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:01 localhost kernel:         in-hardirq-W at:
Feb 18 17:26:01 localhost kernel:                                   [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:26:01 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:01 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:01 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:01 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:01 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:01 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:01 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:01 localhost kernel:                                   [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:26:01 localhost kernel:                                   [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:26:01 localhost kernel:                                   [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:26:01 localhost kernel:                                   [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:26:01 localhost kernel:                                   [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:26:01 localhost kernel:                                   [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:26:01 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:01 localhost kernel:                                   [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:26:01 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:01 localhost kernel:                                   [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:26:01 localhost kernel:                                   [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:26:01 localhost kernel:                                   [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:26:01 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:01 localhost kernel:         in-softirq-W at:
Feb 18 17:26:01 localhost kernel:                                   [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:01 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:01 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:01 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:01 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:01 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:01 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:01 localhost kernel:                                   [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:26:01 localhost kernel:                                   [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:26:01 localhost kernel:                                   [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:01 localhost kernel:                                   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:02 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:02 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:02 localhost kernel:       }
Feb 18 17:26:02 localhost kernel:       ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:26:02 localhost kernel:      ... acquired at:
Feb 18 17:26:02 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:02 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:02 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:02 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:02 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:02 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:02 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:02 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:26:02 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:02 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:26:02 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:02 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:02 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:02 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:02 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:02 localhost kernel: 
Feb 18 17:26:02 localhost kernel:     ... acquired at:
Feb 18 17:26:02 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:02 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:02 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:02 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:02 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:02 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:02 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:02 localhost kernel:    [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:26:02 localhost kernel:    [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 17:26:02 localhost kernel:    [<c04235fe>] complete+0x36/0x44
Feb 18 17:26:02 localhost kernel:    [<c043aad8>] worker_thread+0x0/0xc4
Feb 18 17:26:02 localhost kernel:    [<c043d502>] kthread+0x1c/0x5e
Feb 18 17:26:02 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:02 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:02 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:02 localhost kernel: 
Feb 18 17:26:02 localhost kernel:     -> (&rq->rq_lock_key#3){++..} ops: 1928539 {
Feb 18 17:26:02 localhost kernel:        initial-use  at:
Feb 18 17:26:02 localhost kernel:                                 [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:03 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:03 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:03 localhost kernel:                                 [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:03 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:03 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:03 localhost kernel:                                 [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:26:03 localhost kernel:                                 [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:26:03 localhost kernel:                                 [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:26:03 localhost kernel:                                 [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:26:03 localhost kernel:                                 [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:26:03 localhost kernel:                                 [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:26:03 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:03 localhost kernel:                                 [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:03 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:03 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:03 localhost kernel:                                 [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:26:03 localhost kernel:                                 [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:26:03 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:03 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:03 localhost kernel:                                 [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:26:03 localhost kernel:                                 [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:26:03 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:03 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:03 localhost kernel:                                 [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:03 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:03 localhost kernel:        in-hardirq-W at:
Feb 18 17:26:03 localhost kernel:                                 [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:26:03 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:03 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:03 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:03 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:03 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:03 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:03 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:03 localhost kernel:                                 [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:26:03 localhost kernel:                                 [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:26:03 localhost kernel:                                 [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:26:03 localhost kernel:                                 [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:26:03 localhost kernel:                                 [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:26:03 localhost kernel:                                 [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:26:03 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:04 localhost kernel:                                 [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:26:04 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:04 localhost kernel:                                 [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:26:04 localhost kernel:                                 [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:26:04 localhost kernel:                                 [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:26:04 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:04 localhost kernel:        in-softirq-W at:
Feb 18 17:26:04 localhost kernel:                                 [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:26:04 localhost kernel:                                 [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:26:04 localhost kernel:                                 [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:04 localhost kernel:                                 [<c04422f1>] getnstimeofday+0x30/0xbc
Feb 18 17:26:04 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:04 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:04 localhost kernel:                                 [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:04 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:04 localhost kernel:                                 [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:04 localhost kernel:                                 [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:04 localhost kernel:                                 [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:26:04 localhost kernel:                                 [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:04 localhost kernel:                                 [<c0430afa>] __do_softirq+0xc9/0xde
Feb 18 17:26:04 localhost kernel:                                 [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:04 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:04 localhost kernel:      }
Feb 18 17:26:04 localhost kernel:      ... key      at: [<c2a34488>] 0xc2a34488
Feb 18 17:26:04 localhost kernel:      -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:26:04 localhost kernel:         initial-use  at:
Feb 18 17:26:04 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:04 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:04 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:04 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:04 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:04 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:04 localhost kernel:                                   [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:26:04 localhost kernel:                                   [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:26:04 localhost kernel:                                   [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:26:04 localhost kernel:                                   [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:26:04 localhost kernel:                                   [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:26:04 localhost kernel:                                   [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:26:04 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:04 localhost kernel:                                   [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:04 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:05 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:05 localhost kernel:                                   [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:26:05 localhost kernel:                                   [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:26:05 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:05 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:05 localhost kernel:                                   [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:26:05 localhost kernel:                                   [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:26:05 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:05 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:05 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:05 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:05 localhost kernel:         in-hardirq-W at:
Feb 18 17:26:05 localhost kernel:                                   [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:26:05 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:05 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:05 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:05 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:05 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:05 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:05 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:05 localhost kernel:                                   [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:26:05 localhost kernel:                                   [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:26:05 localhost kernel:                                   [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:26:05 localhost kernel:                                   [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:26:05 localhost kernel:                                   [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:26:05 localhost kernel:                                   [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:26:05 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:05 localhost kernel:                                   [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:26:05 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:05 localhost kernel:                                   [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:26:05 localhost kernel:                                   [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:26:05 localhost kernel:                                   [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:26:05 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:05 localhost kernel:         in-softirq-W at:
Feb 18 17:26:05 localhost kernel:                                   [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:05 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:05 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:05 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:05 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:05 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:05 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:05 localhost kernel:                                   [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:26:06 localhost kernel:                                   [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:26:06 localhost kernel:                                   [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:06 localhost kernel:                                   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:06 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:06 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:06 localhost kernel:       }
Feb 18 17:26:06 localhost kernel:       ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:26:06 localhost kernel:      ... acquired at:
Feb 18 17:26:06 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:06 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:06 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:06 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:06 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:06 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:06 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:06 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:26:06 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:06 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:26:06 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:06 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:06 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:06 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:06 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:06 localhost kernel: 
Feb 18 17:26:06 localhost kernel:     ... acquired at:
Feb 18 17:26:06 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:06 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:06 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:06 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:06 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:06 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:06 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:06 localhost kernel:    [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:26:06 localhost kernel:    [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 17:26:06 localhost kernel:    [<c04235fe>] complete+0x36/0x44
Feb 18 17:26:06 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:06 localhost kernel:    [<c0428996>] migration_thread+0x186/0x1d2
Feb 18 17:26:06 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:06 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:06 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:07 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:07 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:07 localhost kernel: 
Feb 18 17:26:07 localhost kernel:     -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:26:07 localhost kernel:        initial-use  at:
Feb 18 17:26:07 localhost kernel:                                 [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:07 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:07 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:07 localhost kernel:                                 [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:07 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:07 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:07 localhost kernel:                                 [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:26:07 localhost kernel:                                 [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:26:07 localhost kernel:                                 [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:26:07 localhost kernel:                                 [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:26:07 localhost kernel:                                 [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:26:07 localhost kernel:                                 [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:26:07 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:07 localhost kernel:                                 [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:07 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:07 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:07 localhost kernel:                                 [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:26:07 localhost kernel:                                 [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:26:07 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:07 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:07 localhost kernel:                                 [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:26:07 localhost kernel:                                 [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:26:07 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:07 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:07 localhost kernel:                                 [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:07 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:07 localhost kernel:        in-hardirq-W at:
Feb 18 17:26:07 localhost kernel:                                 [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:26:07 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:07 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:07 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:07 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:07 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:07 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:07 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:07 localhost kernel:                                 [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:26:08 localhost kernel:                                 [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:26:08 localhost kernel:                                 [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:26:08 localhost kernel:                                 [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:26:08 localhost kernel:                                 [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:26:08 localhost kernel:                                 [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:26:08 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:08 localhost kernel:                                 [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:26:08 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:08 localhost kernel:                                 [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:26:08 localhost kernel:                                 [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:26:08 localhost kernel:                                 [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:26:08 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:08 localhost kernel:        in-softirq-W at:
Feb 18 17:26:08 localhost kernel:                                 [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:08 localhost kernel:                                 [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:08 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:08 localhost kernel:                                 [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:08 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:08 localhost kernel:                                 [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:08 localhost kernel:                                 [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:08 localhost kernel:                                 [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:26:08 localhost kernel:                                 [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:26:08 localhost kernel:                                 [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:08 localhost kernel:                                 [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:08 localhost kernel:                                 [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:08 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:08 localhost kernel:      }
Feb 18 17:26:08 localhost kernel:      ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:26:08 localhost kernel:     ... acquired at:
Feb 18 17:26:08 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:08 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:08 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:08 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:08 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:08 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:08 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:08 localhost kernel:    [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:26:08 localhost kernel:    [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 17:26:08 localhost kernel:    [<c04235fe>] complete+0x36/0x44
Feb 18 17:26:08 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:09 localhost kernel:    [<c0428996>] migration_thread+0x186/0x1d2
Feb 18 17:26:09 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:09 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:09 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:09 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:09 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:09 localhost kernel: 
Feb 18 17:26:09 localhost kernel:    ... acquired at:
Feb 18 17:26:09 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:09 localhost kernel:    [<c042366f>] __wake_up+0x18/0x42
Feb 18 17:26:09 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:09 localhost kernel:    [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:09 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:09 localhost kernel:    [<c042366f>] __wake_up+0x18/0x42
Feb 18 17:26:09 localhost kernel:    [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:09 localhost kernel:    [<c042366f>] __wake_up+0x18/0x42
Feb 18 17:26:09 localhost kernel:    [<c042366f>] __wake_up+0x18/0x42
Feb 18 17:26:09 localhost kernel:    [<f8bb7a2b>] ppp_receive_nonmp_frame+0x51f/0x70a [ppp_generic]
Feb 18 17:26:09 localhost kernel:    [<f8bb8487>] ppp_input+0xbf/0xef [ppp_generic]
Feb 18 17:26:09 localhost kernel:    [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:26:09 localhost kernel:    [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:26:09 localhost kernel:    [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:09 localhost kernel:    [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:26:09 localhost kernel:    [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:26:09 localhost kernel:    [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:26:09 localhost kernel:    [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:26:09 localhost kernel:    [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:26:09 localhost kernel:    [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:09 localhost kernel:    [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:26:09 localhost kernel:    [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:09 localhost kernel:    [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:09 localhost kernel:    [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:09 localhost kernel:    [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:09 localhost kernel:    [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:09 localhost kernel:    [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:09 localhost kernel:    [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:09 localhost kernel:    [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:09 localhost kernel:    [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:09 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:09 localhost kernel: 
Feb 18 17:26:09 localhost kernel:   ... acquired at:
Feb 18 17:26:09 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:10 localhost kernel:    [<f8bb6bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 17:26:10 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:10 localhost kernel:    [<f8bb6bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 17:26:10 localhost kernel:    [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:10 localhost kernel:    [<f8bb6bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 17:26:10 localhost kernel:    [<f8bb6bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 17:26:10 localhost kernel:    [<c061b008>] __down+0x82/0xb8
Feb 18 17:26:10 localhost kernel:    [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:10 localhost kernel:    [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:10 localhost kernel:    [<c061b008>] __down+0x82/0xb8
Feb 18 17:26:10 localhost kernel:    [<c0425474>] default_wake_function+0x0/0x8
Feb 18 17:26:10 localhost kernel:    [<c061ae17>] __down_failed+0x7/0xc
Feb 18 17:26:10 localhost kernel:    [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 17:26:10 localhost kernel:    [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 17:26:10 localhost kernel:    [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 17:26:10 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:10 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:10 localhost kernel: 
Feb 18 17:26:10 localhost kernel:   -> (&list->lock#8){.+..} ops: 46 {
Feb 18 17:26:10 localhost kernel:      initial-use  at:
Feb 18 17:26:10 localhost kernel:                             [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:10 localhost kernel:                             [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:10 localhost kernel:                             [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:10 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:10 localhost kernel:                             [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:10 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:10 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:10 localhost kernel:                             [<f8bb8433>] ppp_input+0x6b/0xef [ppp_generic]
Feb 18 17:26:10 localhost kernel:                             [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:26:10 localhost kernel:                             [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:26:10 localhost kernel:                             [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:10 localhost kernel:                             [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:26:10 localhost kernel:                             [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:26:10 localhost kernel:                             [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:26:10 localhost kernel:                             [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:26:10 localhost kernel:                             [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:26:10 localhost kernel:                             [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:10 localhost kernel:                             [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:26:10 localhost kernel:                             [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:10 localhost kernel:                             [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:10 localhost kernel:                             [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:11 localhost kernel:                             [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:11 localhost kernel:                             [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:11 localhost kernel:                             [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:11 localhost kernel:                             [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:11 localhost kernel:                             [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:11 localhost kernel:                             [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:11 localhost kernel:                             [<ffffffff>] 0xffffffff
Feb 18 17:26:11 localhost kernel:      in-softirq-W at:
Feb 18 17:26:11 localhost kernel:                             [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:11 localhost kernel:                             [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:11 localhost kernel:                             [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:11 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:11 localhost kernel:                             [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:11 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:11 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:11 localhost kernel:                             [<f8bb8433>] ppp_input+0x6b/0xef [ppp_generic]
Feb 18 17:26:11 localhost kernel:                             [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:26:11 localhost kernel:                             [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:26:11 localhost kernel:                             [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:11 localhost kernel:                             [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:26:11 localhost kernel:                             [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:26:11 localhost kernel:                             [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:26:11 localhost kernel:                             [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:26:11 localhost kernel:                             [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:26:11 localhost kernel:                             [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:11 localhost kernel:                             [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:26:11 localhost kernel:                             [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:11 localhost kernel:                             [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:11 localhost kernel:                             [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:11 localhost kernel:                             [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:11 localhost kernel:                             [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:11 localhost kernel:                             [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:11 localhost kernel:                             [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:11 localhost kernel:                             [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:11 localhost kernel:                             [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:11 localhost kernel:                             [<ffffffff>] 0xffffffff
Feb 18 17:26:11 localhost kernel:    }
Feb 18 17:26:11 localhost kernel:    ... key      at: [<f8bbcd18>] __key.21159+0x0/0xffffb82a [ppp_generic]
Feb 18 17:26:11 localhost kernel:   ... acquired at:
Feb 18 17:26:11 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:11 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:12 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:12 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:12 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:12 localhost kernel:    [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:12 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:12 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:12 localhost kernel:    [<f8bb6424>] ppp_xmit_process+0x529/0x5a1 [ppp_generic]
Feb 18 17:26:12 localhost kernel:    [<f8bb734b>] ppp_channel_push+0x71/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel:    [<c061b373>] _read_lock_bh+0x2e/0x39
Feb 18 17:26:12 localhost kernel:    [<f8bb735a>] ppp_channel_push+0x80/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel:    [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:12 localhost kernel:    [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:12 localhost kernel:    [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:12 localhost kernel:    [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:12 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:12 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:12 localhost kernel: 
Feb 18 17:26:12 localhost kernel:   -> (&pch->downl){-...} ops: 8 {
Feb 18 17:26:12 localhost kernel:      initial-use  at:
Feb 18 17:26:12 localhost kernel:                             [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:12 localhost kernel:                             [<c046b874>] __do_fault+0x31b/0x35d
Feb 18 17:26:12 localhost kernel:                             [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:12 localhost kernel:                             [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel:                             [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:12 localhost kernel:                             [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel:                             [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel:                             [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:12 localhost kernel:                             [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:12 localhost kernel:                             [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:12 localhost kernel:                             [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:12 localhost kernel:                             [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:12 localhost kernel:                             [<ffffffff>] 0xffffffff
Feb 18 17:26:12 localhost kernel:      hardirq-on-W at:
Feb 18 17:26:12 localhost kernel:                             [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:12 localhost kernel:                             [<c046b874>] __do_fault+0x31b/0x35d
Feb 18 17:26:12 localhost kernel:                             [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:12 localhost kernel:                             [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel:                             [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:13 localhost kernel:                             [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:13 localhost kernel:                             [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:13 localhost kernel:                             [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:13 localhost kernel:                             [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:13 localhost kernel:                             [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:13 localhost kernel:                             [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:13 localhost kernel:                             [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:13 localhost kernel:                             [<ffffffff>] 0xffffffff
Feb 18 17:26:13 localhost kernel:    }
Feb 18 17:26:13 localhost kernel:    ... key      at: [<f8bbcd28>] __key.29740+0x0/0xffffb81a [ppp_generic]
Feb 18 17:26:13 localhost kernel:    -> (&list->lock#8){.+..} ops: 46 {
Feb 18 17:26:13 localhost kernel:       initial-use  at:
Feb 18 17:26:13 localhost kernel:                               [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:13 localhost kernel:                               [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:13 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:13 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:13 localhost kernel:                               [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:13 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:13 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:13 localhost kernel:                               [<f8bb8433>] ppp_input+0x6b/0xef [ppp_generic]
Feb 18 17:26:13 localhost kernel:                               [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:26:13 localhost kernel:                               [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:26:13 localhost kernel:                               [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:13 localhost kernel:                               [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:26:13 localhost kernel:                               [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:26:13 localhost kernel:                               [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:26:13 localhost kernel:                               [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:26:13 localhost kernel:                               [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:26:13 localhost kernel:                               [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:13 localhost kernel:                               [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:26:13 localhost kernel:                               [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:13 localhost kernel:                               [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:13 localhost kernel:                               [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:13 localhost kernel:                               [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:13 localhost kernel:                               [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:13 localhost kernel:                               [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:13 localhost kernel:                               [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:13 localhost kernel:                               [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:13 localhost kernel:                               [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:13 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:13 localhost kernel:       in-softirq-W at:
Feb 18 17:26:14 localhost kernel:                               [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:14 localhost kernel:                               [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:14 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:14 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:14 localhost kernel:                               [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:14 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:14 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:14 localhost kernel:                               [<f8bb8433>] ppp_input+0x6b/0xef [ppp_generic]
Feb 18 17:26:14 localhost kernel:                               [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:26:14 localhost kernel:                               [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:26:14 localhost kernel:                               [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:14 localhost kernel:                               [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:26:14 localhost kernel:                               [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:26:14 localhost kernel:                               [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:26:14 localhost kernel:                               [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:26:14 localhost kernel:                               [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:26:14 localhost kernel:                               [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:14 localhost kernel:                               [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:26:14 localhost kernel:                               [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:14 localhost kernel:                               [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:14 localhost kernel:                               [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:14 localhost kernel:                               [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:14 localhost kernel:                               [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:14 localhost kernel:                               [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:14 localhost kernel:                               [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:14 localhost kernel:                               [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:14 localhost kernel:                               [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:14 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:14 localhost kernel:     }
Feb 18 17:26:14 localhost kernel:     ... key      at: [<f8bbcd18>] __key.21159+0x0/0xffffb82a [ppp_generic]
Feb 18 17:26:14 localhost kernel:    ... acquired at:
Feb 18 17:26:14 localhost kernel:    [<c0446b6a>] add_lock_to_list+0x66/0x89
Feb 18 17:26:14 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:14 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:14 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:14 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:14 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:14 localhost kernel:    [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:14 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:14 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:14 localhost kernel:    [<f8bb72ff>] ppp_channel_push+0x25/0x90 [ppp_generic]
Feb 18 17:26:14 localhost kernel:    [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:14 localhost kernel:    [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:15 localhost kernel:    [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:15 localhost kernel:    [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:15 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:15 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:15 localhost kernel: 
Feb 18 17:26:15 localhost kernel:    -> (inet_peer_idlock){-+..} ops: 1947 {
Feb 18 17:26:15 localhost kernel:       initial-use  at:
Feb 18 17:26:15 localhost kernel:                               [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:15 localhost kernel:                               [<c05d0e17>] rt_bind_peer+0x1b/0x50
Feb 18 17:26:15 localhost kernel:                               [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:15 localhost kernel:                               [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:15 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:15 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel:                               [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:15 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel:                               [<c05d4ea2>] ip_push_pending_frames+0x206/0x33b
Feb 18 17:26:15 localhost kernel:                               [<c05f368d>] icmp_send+0x310/0x386
Feb 18 17:26:15 localhost kernel:                               [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:15 localhost kernel:                               [<c05bcd7a>] neigh_timer_handler+0x14/0x276
Feb 18 17:26:15 localhost kernel:                               [<c05ce07c>] ipv4_link_failure+0x1c/0x44
Feb 18 17:26:15 localhost kernel:                               [<c05f1375>] arp_error_report+0x1c/0x24
Feb 18 17:26:15 localhost kernel:                               [<c05bcf16>] neigh_timer_handler+0x1b0/0x276
Feb 18 17:26:15 localhost kernel:                               [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:15 localhost kernel:                               [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 17:26:15 localhost kernel:                               [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:15 localhost kernel:                               [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:15 localhost kernel:                               [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:15 localhost kernel:                               [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:15 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:15 localhost kernel:       in-softirq-W at:
Feb 18 17:26:15 localhost kernel:                               [<c0446ab0>] save_trace+0x37/0x8b
Feb 18 17:26:15 localhost kernel:                               [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:15 localhost kernel:                               [<c05d0e17>] rt_bind_peer+0x1b/0x50
Feb 18 17:26:15 localhost kernel:                               [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:15 localhost kernel:                               [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:15 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:15 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel:                               [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:15 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel:                               [<c05d4ea2>] ip_push_pending_frames+0x206/0x33b
Feb 18 17:26:16 localhost kernel:                               [<c05f368d>] icmp_send+0x310/0x386
Feb 18 17:26:16 localhost kernel:                               [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:16 localhost kernel:                               [<c05bcd7a>] neigh_timer_handler+0x14/0x276
Feb 18 17:26:16 localhost kernel:                               [<c05ce07c>] ipv4_link_failure+0x1c/0x44
Feb 18 17:26:16 localhost kernel:                               [<c05f1375>] arp_error_report+0x1c/0x24
Feb 18 17:26:16 localhost kernel:                               [<c05bcf16>] neigh_timer_handler+0x1b0/0x276
Feb 18 17:26:16 localhost kernel:                               [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:16 localhost kernel:                               [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 17:26:16 localhost kernel:                               [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:16 localhost kernel:                               [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:16 localhost kernel:                               [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:16 localhost kernel:                               [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:16 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:16 localhost kernel:       hardirq-on-W at:
Feb 18 17:26:16 localhost kernel:                               [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:16 localhost kernel:                               [<c05d0e17>] rt_bind_peer+0x1b/0x50
Feb 18 17:26:16 localhost kernel:                               [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:16 localhost kernel:                               [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:16 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:16 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:16 localhost kernel:                               [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:16 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:16 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:16 localhost kernel:                               [<c05d4ea2>] ip_push_pending_frames+0x206/0x33b
Feb 18 17:26:16 localhost kernel:                               [<c05f368d>] icmp_send+0x310/0x386
Feb 18 17:26:16 localhost kernel:                               [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:16 localhost kernel:                               [<c05bcd7a>] neigh_timer_handler+0x14/0x276
Feb 18 17:26:16 localhost kernel:                               [<c05ce07c>] ipv4_link_failure+0x1c/0x44
Feb 18 17:26:16 localhost kernel:                               [<c05f1375>] arp_error_report+0x1c/0x24
Feb 18 17:26:16 localhost kernel:                               [<c05bcf16>] neigh_timer_handler+0x1b0/0x276
Feb 18 17:26:16 localhost kernel:                               [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:16 localhost kernel:                               [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 17:26:16 localhost kernel:                               [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:16 localhost kernel:                               [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:16 localhost kernel:                               [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:16 localhost kernel:                               [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:16 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:16 localhost kernel:     }
Feb 18 17:26:16 localhost kernel:     ... key      at: [<c07380f0>] inet_peer_idlock+0x10/0x1c
Feb 18 17:26:16 localhost kernel:    ... acquired at:
Feb 18 17:26:16 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:16 localhost kernel:    [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:17 localhost kernel:    [<c048d217>] __d_lookup+0x0/0x12a
Feb 18 17:26:17 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:17 localhost kernel:    [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:17 localhost kernel:    [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:17 localhost kernel:    [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:17 localhost kernel:    [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:17 localhost kernel:    [<c043badc>] __kernel_text_address+0x18/0x23
Feb 18 17:26:17 localhost kernel:    [<c05d6ec6>] ip_queue_xmit+0x27b/0x34d
Feb 18 17:26:17 localhost kernel:    [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:17 localhost kernel:    [<c047a1bd>] kfree+0x9f/0xa6
Feb 18 17:26:17 localhost kernel:    [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:17 localhost kernel:    [<c05b0c83>] pskb_expand_head+0xde/0x144
Feb 18 17:26:17 localhost kernel:    [<f8bbf7bf>] pppol2tp_xmit+0x426/0x4a2 [pppol2tp]
Feb 18 17:26:17 localhost kernel:    [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:17 localhost kernel:    [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:17 localhost kernel:    [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:17 localhost kernel:    [<f8bb730e>] ppp_channel_push+0x34/0x90 [ppp_generic]
Feb 18 17:26:17 localhost kernel:    [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:17 localhost kernel:    [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:17 localhost kernel:    [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:17 localhost kernel:    [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:17 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:17 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:17 localhost kernel: 
Feb 18 17:26:17 localhost kernel:    -> (&dev->queue_lock#2){-+..} ops: 47607 {
Feb 18 17:26:17 localhost kernel:       initial-use  at:
Feb 18 17:26:17 localhost kernel:                               [<c047a1bd>] kfree+0x9f/0xa6
Feb 18 17:26:17 localhost kernel:                               [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:17 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:17 localhost kernel:                               [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:17 localhost kernel:                               [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:17 localhost kernel:                               [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:17 localhost kernel:                               [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:17 localhost kernel:                               [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:17 localhost kernel:                               [<c05b680a>] register_netdevice+0x29f/0x2f9
Feb 18 17:26:17 localhost kernel:                               [<c05b6893>] register_netdev+0x2f/0x3b
Feb 18 17:26:17 localhost kernel:                               [<c076aded>] loopback_net_init+0x36/0x68
Feb 18 17:26:17 localhost kernel:                               [<c05b3fa0>] register_pernet_operations+0x10/0x11
Feb 18 17:26:17 localhost kernel:                               [<c05b3ffc>] register_pernet_device+0x1c/0x48
Feb 18 17:26:17 localhost kernel:                               [<c074e491>] kernel_init+0x148/0x2af
Feb 18 17:26:17 localhost kernel:                               [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:18 localhost kernel:                               [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:18 localhost kernel:                               [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:18 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:18 localhost kernel:       in-softirq-W at:
Feb 18 17:26:18 localhost kernel:                               [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:18 localhost kernel:                               [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:18 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:18 localhost kernel:                               [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:18 localhost kernel:                               [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:18 localhost kernel:                               [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:18 localhost kernel:                               [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:18 localhost kernel:                               [<c05b7e60>] dev_queue_xmit+0x125/0x2e9
Feb 18 17:26:18 localhost kernel:                               [<c05c341e>] eth_header+0x0/0xb6
Feb 18 17:26:18 localhost kernel:                               [<f8f3a487>] mld_sendpack+0x242/0x355 [ipv6]
Feb 18 17:26:18 localhost kernel:                               [<f8f3b453>] mld_ifc_timer_expire+0x143/0x208 [ipv6]
Feb 18 17:26:18 localhost kernel:                               [<f8f3b4e5>] mld_ifc_timer_expire+0x1d5/0x208 [ipv6]
Feb 18 17:26:18 localhost kernel:                               [<f8f3b4f0>] mld_ifc_timer_expire+0x1e0/0x208 [ipv6]
Feb 18 17:26:18 localhost kernel:                               [<f8f3b310>] mld_ifc_timer_expire+0x0/0x208 [ipv6]
Feb 18 17:26:18 localhost kernel:                               [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 17:26:18 localhost kernel:                               [<f8f3b310>] mld_ifc_timer_expire+0x0/0x208 [ipv6]
Feb 18 17:26:18 localhost kernel:                               [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:18 localhost kernel:                               [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:18 localhost kernel:                               [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:18 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:18 localhost kernel:       hardirq-on-W at:
Feb 18 17:26:18 localhost kernel:                               [<c047a1bd>] kfree+0x9f/0xa6
Feb 18 17:26:18 localhost kernel:                               [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:18 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:18 localhost kernel:                               [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:18 localhost kernel:                               [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:18 localhost kernel:                               [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:18 localhost kernel:                               [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:18 localhost kernel:                               [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:18 localhost kernel:                               [<c05b680a>] register_netdevice+0x29f/0x2f9
Feb 18 17:26:18 localhost kernel:                               [<c05b6893>] register_netdev+0x2f/0x3b
Feb 18 17:26:18 localhost kernel:                               [<c076aded>] loopback_net_init+0x36/0x68
Feb 18 17:26:18 localhost kernel:                               [<c05b3fa0>] register_pernet_operations+0x10/0x11
Feb 18 17:26:18 localhost kernel:                               [<c05b3ffc>] register_pernet_device+0x1c/0x48
Feb 18 17:26:18 localhost kernel:                               [<c074e491>] kernel_init+0x148/0x2af
Feb 18 17:26:18 localhost kernel:                               [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:18 localhost kernel:                               [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:18 localhost kernel:                               [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:19 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:19 localhost kernel:     }
Feb 18 17:26:19 localhost kernel:     ... key      at: [<c0a3e1c4>] __key.31614+0x0/0x8
Feb 18 17:26:19 localhost kernel:     -> (&dev->ingress_lock){-...} ops: 208 {
Feb 18 17:26:19 localhost kernel:        initial-use  at:
Feb 18 17:26:19 localhost kernel:                                 [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:19 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:19 localhost kernel:                                 [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:19 localhost kernel:                                 [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel:                                 [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel:                                 [<c05b680a>] register_netdevice+0x29f/0x2f9
Feb 18 17:26:19 localhost kernel:                                 [<c05b6893>] register_netdev+0x2f/0x3b
Feb 18 17:26:19 localhost kernel:                                 [<c076aded>] loopback_net_init+0x36/0x68
Feb 18 17:26:19 localhost kernel:                                 [<c05b3fa0>] register_pernet_operations+0x10/0x11
Feb 18 17:26:19 localhost kernel:                                 [<c05b3ffc>] register_pernet_device+0x1c/0x48
Feb 18 17:26:19 localhost kernel:                                 [<c074e491>] kernel_init+0x148/0x2af
Feb 18 17:26:19 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:19 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:19 localhost kernel:                                 [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:19 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:19 localhost kernel:        hardirq-on-W at:
Feb 18 17:26:19 localhost kernel:                                 [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:19 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:19 localhost kernel:                                 [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:19 localhost kernel:                                 [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel:                                 [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel:                                 [<c05b680a>] register_netdevice+0x29f/0x2f9
Feb 18 17:26:19 localhost kernel:                                 [<c05b6893>] register_netdev+0x2f/0x3b
Feb 18 17:26:19 localhost kernel:                                 [<c076aded>] loopback_net_init+0x36/0x68
Feb 18 17:26:19 localhost kernel:                                 [<c05b3fa0>] register_pernet_operations+0x10/0x11
Feb 18 17:26:19 localhost kernel:                                 [<c05b3ffc>] register_pernet_device+0x1c/0x48
Feb 18 17:26:19 localhost kernel:                                 [<c074e491>] kernel_init+0x148/0x2af
Feb 18 17:26:19 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:19 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:19 localhost kernel:                                 [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:19 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:19 localhost kernel:      }
Feb 18 17:26:19 localhost kernel:      ... key      at: [<c0a3e1b4>] __key.31616+0x0/0x8
Feb 18 17:26:19 localhost kernel:      -> (&list->lock#6){....} ops: 675 {
Feb 18 17:26:19 localhost kernel:         initial-use  at:
Feb 18 17:26:19 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:19 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:20 localhost kernel:                                   [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:20 localhost kernel:                                   [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel:                                   [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel:                                   [<c05b0afd>] skb_queue_purge+0x11/0x17
Feb 18 17:26:20 localhost kernel:                                   [<c05c4eca>] pfifo_fast_reset+0x14/0x2f
Feb 18 17:26:20 localhost kernel:                                   [<c05c4712>] qdisc_reset+0x10/0x11
Feb 18 17:26:20 localhost kernel:                                   [<c05c4b83>] dev_deactivate+0x2a/0xfd
Feb 18 17:26:20 localhost kernel:                                   [<c05bf1f4>] __linkwatch_run_queue+0x143/0x17a
Feb 18 17:26:20 localhost kernel:                                   [<c05bf248>] linkwatch_event+0x1d/0x22
Feb 18 17:26:20 localhost kernel:                                   [<c043a0f3>] run_workqueue+0xdb/0x1ae
Feb 18 17:26:20 localhost kernel:                                   [<c043a09f>] run_workqueue+0x87/0x1ae
Feb 18 17:26:20 localhost kernel:                                   [<c05bf22b>] linkwatch_event+0x0/0x22
Feb 18 17:26:20 localhost kernel:                                   [<c043aad8>] worker_thread+0x0/0xc4
Feb 18 17:26:20 localhost kernel:                                   [<c043ab92>] worker_thread+0xba/0xc4
Feb 18 17:26:20 localhost kernel:                                   [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:20 localhost kernel:                                   [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:20 localhost kernel:                                   [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:20 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:20 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:20 localhost kernel:       }
Feb 18 17:26:20 localhost kernel:       ... key      at: [<c0a3e7d4>] __key.21167+0x0/0x8
Feb 18 17:26:20 localhost kernel:      ... acquired at:
Feb 18 17:26:20 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:20 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:20 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:20 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel:    [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:20 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel:    [<c05b0afd>] skb_queue_purge+0x11/0x17
Feb 18 17:26:20 localhost kernel:    [<c05c4eca>] pfifo_fast_reset+0x14/0x2f
Feb 18 17:26:20 localhost kernel:    [<c05c48f7>] qdisc_destroy+0x36/0x69
Feb 18 17:26:20 localhost kernel:    [<c05c4954>] dev_shutdown+0x2a/0x7b
Feb 18 17:26:20 localhost kernel:    [<c05b6210>] rollback_registered+0xb8/0x11e
Feb 18 17:26:20 localhost kernel:    [<c05b627e>] unregister_netdevice+0x8/0x35
Feb 18 17:26:20 localhost kernel:    [<c05b62ba>] unregister_netdev+0xf/0x15
Feb 18 17:26:20 localhost kernel:    [<f8bb523f>] ppp_shutdown_interface+0x58/0xad [ppp_generic]
Feb 18 17:26:20 localhost kernel:    [<f8bb5484>] ppp_release+0x29/0x56 [ppp_generic]
Feb 18 17:26:20 localhost kernel:    [<c047f08a>] __fput+0xba/0x172
Feb 18 17:26:20 localhost kernel:    [<c047c7df>] filp_close+0x51/0x58
Feb 18 17:26:21 localhost kernel:    [<c047d9c4>] sys_close+0x70/0xab
Feb 18 17:26:21 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:21 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:21 localhost kernel: 
Feb 18 17:26:21 localhost kernel:     ... acquired at:
Feb 18 17:26:21 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:21 localhost kernel:    [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:21 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:21 localhost kernel:    [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:21 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:21 localhost kernel:    [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:21 localhost kernel:    [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:21 localhost kernel:    [<c05b680a>] register_netdevice+0x29f/0x2f9
Feb 18 17:26:21 localhost kernel:    [<c05b6893>] register_netdev+0x2f/0x3b
Feb 18 17:26:21 localhost kernel:    [<c076aded>] loopback_net_init+0x36/0x68
Feb 18 17:26:21 localhost kernel:    [<c05b3fa0>] register_pernet_operations+0x10/0x11
Feb 18 17:26:21 localhost kernel:    [<c05b3ffc>] register_pernet_device+0x1c/0x48
Feb 18 17:26:21 localhost kernel:    [<c074e491>] kernel_init+0x148/0x2af
Feb 18 17:26:21 localhost kernel:    [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:21 localhost kernel:    [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:21 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:21 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:21 localhost kernel: 
Feb 18 17:26:21 localhost kernel:     -> (&list->lock#6){....} ops: 675 {
Feb 18 17:26:21 localhost kernel:        initial-use  at:
Feb 18 17:26:21 localhost kernel:                                 [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:21 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:21 localhost kernel:                                 [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:21 localhost kernel:                                 [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:21 localhost kernel:                                 [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:21 localhost kernel:                                 [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:21 localhost kernel:                                 [<c05b0afd>] skb_queue_purge+0x11/0x17
Feb 18 17:26:21 localhost kernel:                                 [<c05c4eca>] pfifo_fast_reset+0x14/0x2f
Feb 18 17:26:21 localhost kernel:                                 [<c05c4712>] qdisc_reset+0x10/0x11
Feb 18 17:26:21 localhost kernel:                                 [<c05c4b83>] dev_deactivate+0x2a/0xfd
Feb 18 17:26:21 localhost kernel:                                 [<c05bf1f4>] __linkwatch_run_queue+0x143/0x17a
Feb 18 17:26:21 localhost kernel:                                 [<c05bf248>] linkwatch_event+0x1d/0x22
Feb 18 17:26:21 localhost kernel:                                 [<c043a0f3>] run_workqueue+0xdb/0x1ae
Feb 18 17:26:21 localhost kernel:                                 [<c043a09f>] run_workqueue+0x87/0x1ae
Feb 18 17:26:21 localhost kernel:                                 [<c05bf22b>] linkwatch_event+0x0/0x22
Feb 18 17:26:21 localhost kernel:                                 [<c043aad8>] worker_thread+0x0/0xc4
Feb 18 17:26:21 localhost kernel:                                 [<c043ab92>] worker_thread+0xba/0xc4
Feb 18 17:26:22 localhost kernel:                                 [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:22 localhost kernel:                                 [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:22 localhost kernel:                                 [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:22 localhost kernel:                                 [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:22 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:22 localhost kernel:      }
Feb 18 17:26:22 localhost kernel:      ... key      at: [<c0a3e7d4>] __key.21167+0x0/0x8
Feb 18 17:26:22 localhost kernel:     ... acquired at:
Feb 18 17:26:22 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:22 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:22 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:22 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:22 localhost kernel:    [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:22 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:22 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:22 localhost kernel:    [<c05b0afd>] skb_queue_purge+0x11/0x17
Feb 18 17:26:22 localhost kernel:    [<c05c4eca>] pfifo_fast_reset+0x14/0x2f
Feb 18 17:26:22 localhost kernel:    [<c05c4712>] qdisc_reset+0x10/0x11
Feb 18 17:26:22 localhost kernel:    [<c05c4b83>] dev_deactivate+0x2a/0xfd
Feb 18 17:26:22 localhost kernel:    [<c05bf1f4>] __linkwatch_run_queue+0x143/0x17a
Feb 18 17:26:22 localhost kernel:    [<c05bf248>] linkwatch_event+0x1d/0x22
Feb 18 17:26:22 localhost kernel:    [<c043a0f3>] run_workqueue+0xdb/0x1ae
Feb 18 17:26:22 localhost kernel:    [<c043a09f>] run_workqueue+0x87/0x1ae
Feb 18 17:26:22 localhost kernel:    [<c05bf22b>] linkwatch_event+0x0/0x22
Feb 18 17:26:22 localhost kernel:    [<c043aad8>] worker_thread+0x0/0xc4
Feb 18 17:26:22 localhost kernel:    [<c043ab92>] worker_thread+0xba/0xc4
Feb 18 17:26:22 localhost kernel:    [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:22 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:22 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:22 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:22 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:22 localhost kernel: 
Feb 18 17:26:22 localhost kernel:    ... acquired at:
Feb 18 17:26:22 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:22 localhost kernel:    [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:22 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:22 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:22 localhost kernel:    [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:22 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:22 localhost kernel:    [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:22 localhost kernel:    [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:22 localhost kernel:    [<c05b7e60>] dev_queue_xmit+0x125/0x2e9
Feb 18 17:26:23 localhost kernel:    [<c05d62e7>] ip_finish_output+0x1e6/0x21e
Feb 18 17:26:23 localhost kernel:    [<c05d6f58>] ip_queue_xmit+0x30d/0x34d
Feb 18 17:26:23 localhost kernel:    [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:23 localhost kernel:    [<c047a1bd>] kfree+0x9f/0xa6
Feb 18 17:26:23 localhost kernel:    [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:23 localhost kernel:    [<c05b0c83>] pskb_expand_head+0xde/0x144
Feb 18 17:26:23 localhost kernel:    [<f8bbf7bf>] pppol2tp_xmit+0x426/0x4a2 [pppol2tp]
Feb 18 17:26:23 localhost kernel:    [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:23 localhost kernel:    [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:23 localhost kernel:    [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:23 localhost kernel:    [<f8bb730e>] ppp_channel_push+0x34/0x90 [ppp_generic]
Feb 18 17:26:23 localhost kernel:    [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:23 localhost kernel:    [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:23 localhost kernel:    [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:23 localhost kernel:    [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:23 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:23 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:23 localhost kernel: 
Feb 18 17:26:23 localhost kernel:    -> (&rt_hash_locks[i]){-+..} ops: 24708841 {
Feb 18 17:26:23 localhost kernel:       initial-use  at:
Feb 18 17:26:23 localhost kernel:                               [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:23 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:23 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:23 localhost kernel:                               [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:23 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:23 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:23 localhost kernel:                               [<c05f90b6>] ip_mc_inc_group+0x184/0x1c1
Feb 18 17:26:23 localhost kernel:                               [<c05f9134>] ip_mc_up+0x41/0x59
Feb 18 17:26:23 localhost kernel:                               [<c05f4f9f>] inetdev_event+0x257/0x46e
Feb 18 17:26:23 localhost kernel:                               [<c05c258a>] fib_rules_event+0x120/0x12a
Feb 18 17:26:23 localhost kernel:                               [<c05c24b4>] fib_rules_event+0x4a/0x12a
Feb 18 17:26:23 localhost kernel:                               [<c061d06c>] notifier_call_chain+0x2a/0x47
Feb 18 17:26:23 localhost kernel:                               [<c0441084>] raw_notifier_call_chain+0x17/0x1a
Feb 18 17:26:23 localhost kernel:                               [<c05b73cf>] dev_open+0x71/0x77
Feb 18 17:26:23 localhost kernel:                               [<c05b5ffa>] dev_change_flags+0x9d/0x149
Feb 18 17:26:23 localhost kernel:                               [<c05b49b5>] __dev_get_by_name+0x67/0x72
Feb 18 17:26:23 localhost kernel:                               [<c05f54f7>] devinet_ioctl+0x22e/0x539
Feb 18 17:26:23 localhost kernel:                               [<c05b7188>] dev_ioctl+0x46f/0x5e0
Feb 18 17:26:23 localhost kernel:                               [<c05ab078>] sock_ioctl+0x1bb/0x1e0
Feb 18 17:26:23 localhost kernel:                               [<c05aaebd>] sock_ioctl+0x0/0x1e0
Feb 18 17:26:23 localhost kernel:                               [<c0488827>] do_ioctl+0x1f/0x62
Feb 18 17:26:23 localhost kernel:                               [<c061ccb2>] do_page_fault+0x234/0x5c4
Feb 18 17:26:23 localhost kernel:                               [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 17:26:24 localhost kernel:                               [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 17:26:24 localhost kernel:                               [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:24 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:24 localhost kernel:       in-softirq-W at:
Feb 18 17:26:24 localhost kernel:                               [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:24 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:24 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel:                               [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:24 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel:                               [<c05cee70>] rt_run_flush+0x0/0x8b
Feb 18 17:26:24 localhost kernel:                               [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 17:26:24 localhost kernel:                               [<c05cee70>] rt_run_flush+0x0/0x8b
Feb 18 17:26:24 localhost kernel:                               [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:24 localhost kernel:                               [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:24 localhost kernel:                               [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:24 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:24 localhost kernel:       hardirq-on-W at:
Feb 18 17:26:24 localhost kernel:                               [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:24 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:24 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel:                               [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:24 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel:                               [<c05f90b6>] ip_mc_inc_group+0x184/0x1c1
Feb 18 17:26:24 localhost kernel:                               [<c05f9134>] ip_mc_up+0x41/0x59
Feb 18 17:26:24 localhost kernel:                               [<c05f4f9f>] inetdev_event+0x257/0x46e
Feb 18 17:26:24 localhost kernel:                               [<c05c258a>] fib_rules_event+0x120/0x12a
Feb 18 17:26:24 localhost kernel:                               [<c05c24b4>] fib_rules_event+0x4a/0x12a
Feb 18 17:26:24 localhost kernel:                               [<c061d06c>] notifier_call_chain+0x2a/0x47
Feb 18 17:26:24 localhost kernel:                               [<c0441084>] raw_notifier_call_chain+0x17/0x1a
Feb 18 17:26:24 localhost kernel:                               [<c05b73cf>] dev_open+0x71/0x77
Feb 18 17:26:24 localhost kernel:                               [<c05b5ffa>] dev_change_flags+0x9d/0x149
Feb 18 17:26:24 localhost kernel:                               [<c05b49b5>] __dev_get_by_name+0x67/0x72
Feb 18 17:26:24 localhost kernel:                               [<c05f54f7>] devinet_ioctl+0x22e/0x539
Feb 18 17:26:24 localhost kernel:                               [<c05b7188>] dev_ioctl+0x46f/0x5e0
Feb 18 17:26:24 localhost kernel:                               [<c05ab078>] sock_ioctl+0x1bb/0x1e0
Feb 18 17:26:24 localhost kernel:                               [<c05aaebd>] sock_ioctl+0x0/0x1e0
Feb 18 17:26:24 localhost kernel:                               [<c0488827>] do_ioctl+0x1f/0x62
Feb 18 17:26:24 localhost kernel:                               [<c061ccb2>] do_page_fault+0x234/0x5c4
Feb 18 17:26:24 localhost kernel:                               [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 17:26:24 localhost kernel:                               [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 17:26:24 localhost kernel:                               [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:25 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:25 localhost kernel:     }
Feb 18 17:26:25 localhost kernel:     ... key      at: [<c0a3ea90>] __key.39046+0x0/0x8
Feb 18 17:26:25 localhost kernel:     -> (&parent->list_lock){++..} ops: 252663 {
Feb 18 17:26:25 localhost kernel:        initial-use  at:
Feb 18 17:26:25 localhost kernel:                                 [<c040a9d6>] save_stack_trace+0x20/0x3a
Feb 18 17:26:25 localhost kernel:                                 [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:25 localhost kernel:                                 [<c043badc>] __kernel_text_address+0x18/0x23
Feb 18 17:26:25 localhost kernel:                                 [<c0448013>] check_usage_backwards+0x19/0x41
Feb 18 17:26:25 localhost kernel:                                 [<c0446ab0>] save_trace+0x37/0x8b
Feb 18 17:26:25 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:25 localhost kernel:                                 [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:25 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:25 localhost kernel:                                 [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:25 localhost kernel:                                 [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:25 localhost kernel:                                 [<c047b89c>] kmem_cache_create+0x62/0x38e
Feb 18 17:26:25 localhost kernel:                                 [<c047a790>] kmem_cache_alloc+0x53/0x9f
Feb 18 17:26:25 localhost kernel:                                 [<c047b99d>] kmem_cache_create+0x163/0x38e
Feb 18 17:26:25 localhost kernel:                                 [<c0479da2>] cache_estimate+0x6c/0x89
Feb 18 17:26:25 localhost kernel:                                 [<c0762241>] kmem_cache_init+0x182/0x3f6
Feb 18 17:26:25 localhost kernel:                                 [<c074e88b>] start_kernel+0x293/0x327
Feb 18 17:26:25 localhost kernel:                                 [<c074e0e0>] unknown_bootoption+0x0/0x195
Feb 18 17:26:25 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:25 localhost kernel:        in-hardirq-W at:
Feb 18 17:26:25 localhost kernel:                                 [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:26:25 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:25 localhost kernel:                                 [<c047b7ec>] do_drain+0x1e/0x49
Feb 18 17:26:25 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:25 localhost kernel:                                 [<c047b7ec>] do_drain+0x1e/0x49
Feb 18 17:26:25 localhost kernel:                                 [<c047b7ec>] do_drain+0x1e/0x49
Feb 18 17:26:25 localhost kernel:                                 [<c047b7ce>] do_drain+0x0/0x49
Feb 18 17:26:25 localhost kernel:                                 [<c0418ed5>] smp_call_function_interrupt+0x37/0x62
Feb 18 17:26:25 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:25 localhost kernel:                                 [<c04059b3>] call_function_interrupt+0x33/0x38
Feb 18 17:26:25 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:25 localhost kernel:                                 [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:26:25 localhost kernel:                                 [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:26:25 localhost kernel:                                 [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:26:25 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:25 localhost kernel:        in-softirq-W at:
Feb 18 17:26:25 localhost kernel:                                 [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:26:25 localhost kernel:                                 [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:26 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:26 localhost kernel:                                 [<c04488b6>] debug_check_no_locks_freed+0x105/0x11f
Feb 18 17:26:26 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:26 localhost kernel:                                 [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:26 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:26 localhost kernel:                                 [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:26 localhost kernel:                                 [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:26 localhost kernel:                                 [<c047a1fa>] kmem_cache_free+0x36/0x5e
Feb 18 17:26:26 localhost kernel:                                 [<c043b647>] __rcu_process_callbacks+0xfc/0x16e
Feb 18 17:26:26 localhost kernel:                                 [<c043b6d1>] rcu_process_callbacks+0x18/0x30
Feb 18 17:26:26 localhost kernel:                                 [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:26 localhost kernel:                                 [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:26 localhost kernel:                                 [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:26 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:26 localhost kernel:      }
Feb 18 17:26:26 localhost kernel:      ... key      at: [<c0a2bbe8>] __key.19871+0x0/0x8
Feb 18 17:26:26 localhost kernel:      -> (&zone->lock){.+..} ops: 63132 {
Feb 18 17:26:26 localhost kernel:         initial-use  at:
Feb 18 17:26:26 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:26 localhost kernel:                                   [<c04ef19a>] number+0x147/0x215
Feb 18 17:26:26 localhost kernel:                                   [<c04efab1>] vsnprintf+0x440/0x47c
Feb 18 17:26:26 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:26 localhost kernel:                                   [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:26 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:26 localhost kernel:                                   [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:26 localhost kernel:                                   [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:26 localhost kernel:                                   [<c04645c0>] free_hot_cold_page+0x133/0x175
Feb 18 17:26:26 localhost kernel:                                   [<c07606b8>] free_all_bootmem_core+0xdd/0x158
Feb 18 17:26:26 localhost kernel:                                   [<c075d82e>] mem_init+0x8c/0x383
Feb 18 17:26:26 localhost kernel:                                   [<c042c8c7>] printk+0x1b/0x1f
Feb 18 17:26:26 localhost kernel:                                   [<c0761a7b>] alloc_large_system_hash+0x1f7/0x222
Feb 18 17:26:26 localhost kernel:                                   [<c04f2f65>] __spin_lock_init+0x29/0x49
Feb 18 17:26:26 localhost kernel:                                   [<c076287c>] inode_init_early+0x49/0x72
Feb 18 17:26:26 localhost kernel:                                   [<c074e886>] start_kernel+0x28e/0x327
Feb 18 17:26:26 localhost kernel:                                   [<c074e0e0>] unknown_bootoption+0x0/0x195
Feb 18 17:26:26 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:26 localhost kernel:         in-softirq-W at:
Feb 18 17:26:26 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:26 localhost kernel:                                   [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:26:26 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:26 localhost kernel:                                   [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:27 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:27 localhost kernel:                                   [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:27 localhost kernel:                                   [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:27 localhost kernel:                                   [<c04645c0>] free_hot_cold_page+0x133/0x175
Feb 18 17:26:27 localhost kernel:                                   [<c042a030>] free_task+0x18/0x25
Feb 18 17:26:27 localhost kernel:                                   [<c043b647>] __rcu_process_callbacks+0xfc/0x16e
Feb 18 17:26:27 localhost kernel:                                   [<c043b6d1>] rcu_process_callbacks+0x18/0x30
Feb 18 17:26:27 localhost kernel:                                   [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:27 localhost kernel:                                   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:27 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:27 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:27 localhost kernel:       }
Feb 18 17:26:27 localhost kernel:       ... key      at: [<c0a21fb8>] __key.25233+0x0/0x8
Feb 18 17:26:27 localhost kernel:      ... acquired at:
Feb 18 17:26:27 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:27 localhost kernel:    [<c046472e>] __free_pages_ok+0x101/0x2a8
Feb 18 17:26:27 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:27 localhost kernel:    [<c046472e>] __free_pages_ok+0x101/0x2a8
Feb 18 17:26:27 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:27 localhost kernel:    [<c046472e>] __free_pages_ok+0x101/0x2a8
Feb 18 17:26:27 localhost kernel:    [<c046472e>] __free_pages_ok+0x101/0x2a8
Feb 18 17:26:27 localhost kernel:    [<c0479f5b>] kmem_freepages+0x4e/0xa5
Feb 18 17:26:27 localhost kernel:    [<c047a2a2>] slab_destroy+0x2e/0x49
Feb 18 17:26:27 localhost kernel:    [<c047a370>] free_block+0xb3/0xec
Feb 18 17:26:27 localhost kernel:    [<c047a433>] drain_array+0x8a/0xb5
Feb 18 17:26:27 localhost kernel:    [<c047b4ec>] cache_reap+0x7e/0xf4
Feb 18 17:26:27 localhost kernel:    [<c043a0f3>] run_workqueue+0xdb/0x1ae
Feb 18 17:26:27 localhost kernel:    [<c043a09f>] run_workqueue+0x87/0x1ae
Feb 18 17:26:27 localhost kernel:    [<c047b46e>] cache_reap+0x0/0xf4
Feb 18 17:26:27 localhost kernel:    [<c043aad8>] worker_thread+0x0/0xc4
Feb 18 17:26:27 localhost kernel:    [<c043ab92>] worker_thread+0xba/0xc4
Feb 18 17:26:27 localhost kernel:    [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:27 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:27 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:27 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:27 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:27 localhost kernel: 
Feb 18 17:26:27 localhost kernel:      -> (&on_slab_l3_key){.+..} ops: 28386 {
Feb 18 17:26:27 localhost kernel:         initial-use  at:
Feb 18 17:26:27 localhost kernel:                                   [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:27 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:27 localhost kernel:                                   [<c040a9d6>] save_stack_trace+0x20/0x3a
Feb 18 17:26:27 localhost kernel:                                   [<c0464b77>] get_page_from_freelist+0x24b/0x37a
Feb 18 17:26:28 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:28 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:28 localhost kernel:                                   [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:28 localhost kernel:                                   [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel:                                   [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel:                                   [<c046a328>] mod_zone_page_state+0x35/0x3c
Feb 18 17:26:28 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:28 localhost kernel:                                   [<c047a790>] kmem_cache_alloc+0x53/0x9f
Feb 18 17:26:28 localhost kernel:                                   [<c047ab07>] cache_alloc_refill+0x32b/0x493
Feb 18 17:26:28 localhost kernel:                                   [<c047a790>] kmem_cache_alloc+0x53/0x9f
Feb 18 17:26:28 localhost kernel:                                   [<c075ee99>] pidmap_init+0x15/0x137
Feb 18 17:26:28 localhost kernel:                                   [<c074e8a0>] start_kernel+0x2a8/0x327
Feb 18 17:26:28 localhost kernel:                                   [<c074e0e0>] unknown_bootoption+0x0/0x195
Feb 18 17:26:28 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:28 localhost kernel:         in-softirq-W at:
Feb 18 17:26:28 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:28 localhost kernel:                                   [<c05fdd2b>] fn_hash_lookup+0xaa/0xb4
Feb 18 17:26:28 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:28 localhost kernel:                                   [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:28 localhost kernel:                                   [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel:                                   [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel:                                   [<c047acf5>] __kmalloc+0x86/0xd2
Feb 18 17:26:28 localhost kernel:                                   [<c05bb6b9>] neigh_create+0x2e4/0x41a
Feb 18 17:26:28 localhost kernel:                                   [<c05baadb>] neigh_lookup+0x94/0x9d
Feb 18 17:26:28 localhost kernel:                                   [<c05bb82b>] neigh_event_ns+0x3c/0x76
Feb 18 17:26:28 localhost kernel:                                   [<c05f2116>] arp_process+0x218/0x60c
Feb 18 17:26:28 localhost kernel:                                   [<c05f1efe>] arp_process+0x0/0x60c
Feb 18 17:26:28 localhost kernel:                                   [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:28 localhost kernel:                                   [<c05f250a>] arp_rcv+0x0/0x105
Feb 18 17:26:28 localhost kernel:                                   [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:28 localhost kernel:                                   [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:28 localhost kernel:                                   [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:28 localhost kernel:                                   [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:28 localhost kernel:                                   [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:28 localhost kernel:                                   [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:28 localhost kernel:                                   [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:28 localhost kernel:                                   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:28 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:28 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:28 localhost kernel:       }
Feb 18 17:26:29 localhost kernel:       ... key      at: [<c0a2bc0c>] on_slab_l3_key+0x0/0x8
Feb 18 17:26:29 localhost kernel:       -> (&zone->lock){.+..} ops: 63132 {
Feb 18 17:26:29 localhost kernel:          initial-use  at:
Feb 18 17:26:29 localhost kernel:                                     [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:29 localhost kernel:                                     [<c04ef19a>] number+0x147/0x215
Feb 18 17:26:29 localhost kernel:                                     [<c04efab1>] vsnprintf+0x440/0x47c
Feb 18 17:26:29 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:29 localhost kernel:                                     [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:29 localhost kernel:                                     [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel:                                     [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel:                                     [<c04645c0>] free_hot_cold_page+0x133/0x175
Feb 18 17:26:29 localhost kernel:                                     [<c07606b8>] free_all_bootmem_core+0xdd/0x158
Feb 18 17:26:29 localhost kernel:                                     [<c075d82e>] mem_init+0x8c/0x383
Feb 18 17:26:29 localhost kernel:                                     [<c042c8c7>] printk+0x1b/0x1f
Feb 18 17:26:29 localhost kernel:                                     [<c0761a7b>] alloc_large_system_hash+0x1f7/0x222
Feb 18 17:26:29 localhost kernel:                                     [<c04f2f65>] __spin_lock_init+0x29/0x49
Feb 18 17:26:29 localhost kernel:                                     [<c076287c>] inode_init_early+0x49/0x72
Feb 18 17:26:29 localhost kernel:                                     [<c074e886>] start_kernel+0x28e/0x327
Feb 18 17:26:29 localhost kernel:                                     [<c074e0e0>] unknown_bootoption+0x0/0x195
Feb 18 17:26:29 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:26:29 localhost kernel:          in-softirq-W at:
Feb 18 17:26:29 localhost kernel:                                     [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:29 localhost kernel:                                     [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:26:29 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:29 localhost kernel:                                     [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:29 localhost kernel:                                     [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel:                                     [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel:                                     [<c04645c0>] free_hot_cold_page+0x133/0x175
Feb 18 17:26:29 localhost kernel:                                     [<c042a030>] free_task+0x18/0x25
Feb 18 17:26:29 localhost kernel:                                     [<c043b647>] __rcu_process_callbacks+0xfc/0x16e
Feb 18 17:26:29 localhost kernel:                                     [<c043b6d1>] rcu_process_callbacks+0x18/0x30
Feb 18 17:26:29 localhost kernel:                                     [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:29 localhost kernel:                                     [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:29 localhost kernel:                                     [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:29 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:26:29 localhost kernel:        }
Feb 18 17:26:29 localhost kernel:        ... key      at: [<c0a21fb8>] __key.25233+0x0/0x8
Feb 18 17:26:29 localhost kernel:       ... acquired at:
Feb 18 17:26:29 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:29 localhost kernel:    [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:29 localhost kernel:    [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:30 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:30 localhost kernel:    [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:30 localhost kernel:    [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:30 localhost kernel:    [<c04645c0>] free_hot_cold_page+0x133/0x175
Feb 18 17:26:30 localhost kernel:    [<c047a2a2>] slab_destroy+0x2e/0x49
Feb 18 17:26:30 localhost kernel:    [<c047a370>] free_block+0xb3/0xec
Feb 18 17:26:30 localhost kernel:    [<c047a0f2>] cache_flusharray+0x74/0xa0
Feb 18 17:26:30 localhost kernel:    [<c047a19c>] kfree+0x7e/0xa6
Feb 18 17:26:30 localhost kernel:    [<c05f594c>] snmp_mib_free+0x1a/0x29
Feb 18 17:26:30 localhost kernel:    [<f8f283ac>] in6_dev_finish_destroy+0x8e/0xb9 [ipv6]
Feb 18 17:26:30 localhost kernel:    [<f8f28dd9>] addrconf_ifdown+0x2df/0x2eb [ipv6]
Feb 18 17:26:30 localhost kernel:    [<f8f2b0f5>] addrconf_notify+0x738/0x7c2 [ipv6]
Feb 18 17:26:30 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:30 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:30 localhost kernel:    [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:30 localhost kernel:    [<c0614a16>] packet_notifier+0x149/0x151
Feb 18 17:26:30 localhost kernel:    [<c06148f3>] packet_notifier+0x26/0x151
Feb 18 17:26:30 localhost kernel:    [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:30 localhost kernel:    [<c0614a16>] packet_notifier+0x149/0x151
Feb 18 17:26:30 localhost kernel:    [<c061d06c>] notifier_call_chain+0x2a/0x47
Feb 18 17:26:30 localhost kernel:    [<c0441084>] raw_notifier_call_chain+0x17/0x1a
Feb 18 17:26:30 localhost kernel:    [<c05b621c>] rollback_registered+0xc4/0x11e
Feb 18 17:26:30 localhost kernel:    [<c05b627e>] unregister_netdevice+0x8/0x35
Feb 18 17:26:30 localhost kernel:    [<c05b62ba>] unregister_netdev+0xf/0x15
Feb 18 17:26:30 localhost kernel:    [<f8bb523f>] ppp_shutdown_interface+0x58/0xad [ppp_generic]
Feb 18 17:26:30 localhost kernel:    [<f8bb5484>] ppp_release+0x29/0x56 [ppp_generic]
Feb 18 17:26:30 localhost kernel:    [<c047f08a>] __fput+0xba/0x172
Feb 18 17:26:30 localhost kernel:    [<c047c7df>] filp_close+0x51/0x58
Feb 18 17:26:30 localhost kernel:    [<c047d9c4>] sys_close+0x70/0xab
Feb 18 17:26:30 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:30 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:30 localhost kernel: 
Feb 18 17:26:30 localhost kernel:      ... acquired at:
Feb 18 17:26:30 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:30 localhost kernel:    [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:30 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:30 localhost kernel:    [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:30 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:30 localhost kernel:    [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:30 localhost kernel:    [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:30 localhost kernel:    [<c047a1fa>] kmem_cache_free+0x36/0x5e
Feb 18 17:26:30 localhost kernel:    [<c047a370>] free_block+0xb3/0xec
Feb 18 17:26:30 localhost kernel:    [<c047a0f2>] cache_flusharray+0x74/0xa0
Feb 18 17:26:31 localhost kernel:    [<c047a19c>] kfree+0x7e/0xa6
Feb 18 17:26:31 localhost kernel:    [<f8f28d8e>] addrconf_ifdown+0x294/0x2eb [ipv6]
Feb 18 17:26:31 localhost kernel:    [<f8f2b0f5>] addrconf_notify+0x738/0x7c2 [ipv6]
Feb 18 17:26:31 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:31 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:31 localhost kernel:    [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:31 localhost kernel:    [<c0614a16>] packet_notifier+0x149/0x151
Feb 18 17:26:31 localhost kernel:    [<c06148f3>] packet_notifier+0x26/0x151
Feb 18 17:26:31 localhost kernel:    [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:31 localhost kernel:    [<c0614a16>] packet_notifier+0x149/0x151
Feb 18 17:26:31 localhost kernel:    [<c061d06c>] notifier_call_chain+0x2a/0x47
Feb 18 17:26:31 localhost kernel:    [<c0441084>] raw_notifier_call_chain+0x17/0x1a
Feb 18 17:26:31 localhost kernel:    [<c05b621c>] rollback_registered+0xc4/0x11e
Feb 18 17:26:31 localhost kernel:    [<c05b627e>] unregister_netdevice+0x8/0x35
Feb 18 17:26:31 localhost kernel:    [<c05b62ba>] unregister_netdev+0xf/0x15
Feb 18 17:26:31 localhost kernel:    [<f8bb523f>] ppp_shutdown_interface+0x58/0xad [ppp_generic]
Feb 18 17:26:31 localhost kernel:    [<f8bb5484>] ppp_release+0x29/0x56 [ppp_generic]
Feb 18 17:26:31 localhost kernel:    [<c047f08a>] __fput+0xba/0x172
Feb 18 17:26:31 localhost kernel:    [<c047c7df>] filp_close+0x51/0x58
Feb 18 17:26:31 localhost kernel:    [<c047d9c4>] sys_close+0x70/0xab
Feb 18 17:26:31 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:31 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:31 localhost kernel: 
Feb 18 17:26:31 localhost kernel:     ... acquired at:
Feb 18 17:26:31 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:31 localhost kernel:    [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:31 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:31 localhost kernel:    [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:31 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:31 localhost kernel:    [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:31 localhost kernel:    [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:31 localhost kernel:    [<c047a790>] kmem_cache_alloc+0x53/0x9f
Feb 18 17:26:31 localhost kernel:    [<c05bb516>] neigh_create+0x141/0x41a
Feb 18 17:26:31 localhost kernel:    [<c05baadb>] neigh_lookup+0x94/0x9d
Feb 18 17:26:31 localhost kernel:    [<c05f144f>] arp_bind_neighbour+0x62/0x74
Feb 18 17:26:31 localhost kernel:    [<c05cf248>] rt_intern_hash+0x236/0x322
Feb 18 17:26:31 localhost kernel:    [<c05cfab6>] __ip_route_output_key+0x782/0x847
Feb 18 17:26:31 localhost kernel:    [<c05cf948>] __ip_route_output_key+0x614/0x847
Feb 18 17:26:31 localhost kernel:    [<c05eace7>] tcp_v4_connect+0xef/0x5a6
Feb 18 17:26:31 localhost kernel:    [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:31 localhost kernel:    [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 17:26:32 localhost kernel:    [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:32 localhost kernel:    [<c05f6944>] inet_stream_connect+0x7f/0x208
Feb 18 17:26:32 localhost kernel:    [<c04f05f4>] copy_from_user+0x32/0x5e
Feb 18 17:26:32 localhost kernel:    [<c05ac371>] sys_connect+0x7b/0xa6
Feb 18 17:26:32 localhost kernel:    [<c05aea5a>] sock_setsockopt+0x492/0x4aa
Feb 18 17:26:32 localhost kernel:    [<c05aea5a>] sock_setsockopt+0x492/0x4aa
Feb 18 17:26:32 localhost kernel:    [<c05db937>] tcp_setsockopt+0x321/0x339
Feb 18 17:26:32 localhost kernel:    [<c05ab618>] sock_map_fd+0x41/0x4a
Feb 18 17:26:32 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:32 localhost kernel:    [<c05ab47f>] sys_setsockopt+0x62/0x9c
Feb 18 17:26:32 localhost kernel:    [<c05acb97>] sys_socketcall+0xac/0x261
Feb 18 17:26:32 localhost kernel:    [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:32 localhost kernel:    [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:32 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:32 localhost kernel: 
Feb 18 17:26:32 localhost kernel:    ... acquired at:
Feb 18 17:26:32 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:32 localhost kernel:    [<c05cf06f>] rt_intern_hash+0x5d/0x322
Feb 18 17:26:32 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:32 localhost kernel:    [<c05cf06f>] rt_intern_hash+0x5d/0x322
Feb 18 17:26:32 localhost kernel:    [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:32 localhost kernel:    [<c05cf06f>] rt_intern_hash+0x5d/0x322
Feb 18 17:26:32 localhost kernel:    [<c05cf06f>] rt_intern_hash+0x5d/0x322
Feb 18 17:26:32 localhost kernel:    [<c05cf000>] rt_set_nexthop+0x105/0x117
Feb 18 17:26:32 localhost kernel:    [<c05cfab6>] __ip_route_output_key+0x782/0x847
Feb 18 17:26:32 localhost kernel:    [<c05cf948>] __ip_route_output_key+0x614/0x847
Feb 18 17:26:32 localhost kernel:    [<c0464b77>] get_page_from_freelist+0x24b/0x37a
Feb 18 17:26:32 localhost kernel:    [<c05cfb8f>] ip_route_output_flow+0x14/0x1ef
Feb 18 17:26:32 localhost kernel:    [<c05b0c83>] pskb_expand_head+0xde/0x144
Feb 18 17:26:32 localhost kernel:    [<f8bbf6fe>] pppol2tp_xmit+0x365/0x4a2 [pppol2tp]
Feb 18 17:26:32 localhost kernel:    [<f8bb730e>] ppp_channel_push+0x34/0x90 [ppp_generic]
Feb 18 17:26:32 localhost kernel:    [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:32 localhost kernel:    [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:32 localhost kernel:    [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:32 localhost kernel:    [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:32 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:32 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:32 localhost kernel: 
Feb 18 17:26:32 localhost kernel:   ... acquired at:
Feb 18 17:26:32 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:32 localhost kernel:    [<f8bb56db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 17:26:32 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:32 localhost kernel:    [<f8bb56db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:33 localhost kernel:    [<f8bb56db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<f8bb56db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:33 localhost kernel:    [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:33 localhost kernel:    [<f8bb63f1>] ppp_xmit_process+0x4f6/0x5a1 [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:33 localhost kernel:    [<f8bb83b4>] ppp_write+0xc7/0xdb [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:33 localhost kernel:    [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:33 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:33 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:33 localhost kernel: 
Feb 18 17:26:33 localhost kernel:  ... acquired at:
Feb 18 17:26:33 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:33 localhost kernel:    [<f8bb6bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:33 localhost kernel:    [<f8bb6bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:33 localhost kernel:    [<f8bb6bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<f8bb6bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<c061b008>] __down+0x82/0xb8
Feb 18 17:26:33 localhost kernel:    [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:33 localhost kernel:    [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:33 localhost kernel:    [<c061b008>] __down+0x82/0xb8
Feb 18 17:26:33 localhost kernel:    [<c0425474>] default_wake_function+0x0/0x8
Feb 18 17:26:33 localhost kernel:    [<c061ae17>] __down_failed+0x7/0xc
Feb 18 17:26:33 localhost kernel:    [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 17:26:33 localhost kernel:    [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 17:26:33 localhost kernel:    [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 17:26:33 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:33 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:33 localhost kernel: 
Feb 18 17:26:33 localhost kernel: 
Feb 18 17:26:33 localhost kernel: the soft-irq-unsafe lock's dependencies:
Feb 18 17:26:33 localhost kernel: -> (&sk->sk_dst_lock){----} ops: 3988 {
Feb 18 17:26:33 localhost kernel:    initial-use  at:
Feb 18 17:26:33 localhost kernel:                         [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:33 localhost kernel:                         [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:33 localhost kernel:                         [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 17:26:33 localhost kernel:                         [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 17:26:33 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 17:26:34 localhost kernel:                         [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 17:26:34 localhost kernel:                         [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 17:26:34 localhost kernel:                         [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 17:26:34 localhost kernel:                         [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:34 localhost kernel:                         [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:34 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 17:26:34 localhost kernel:    softirq-on-W at:
Feb 18 17:26:34 localhost kernel:                         [<c0449046>] __lock_acquire+0x48b/0xbf1
Feb 18 17:26:34 localhost kernel:                         [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:34 localhost kernel:                         [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 17:26:34 localhost kernel:                         [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 17:26:34 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 17:26:34 localhost kernel:                         [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 17:26:34 localhost kernel:                         [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 17:26:34 localhost kernel:                         [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 17:26:34 localhost kernel:                         [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:34 localhost kernel:                         [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:34 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 17:26:34 localhost kernel:    hardirq-on-W at:
Feb 18 17:26:34 localhost kernel:                         [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:34 localhost kernel:                         [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:34 localhost kernel:                         [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 17:26:34 localhost kernel:                         [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 17:26:34 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 17:26:34 localhost kernel:                         [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 17:26:34 localhost kernel:                         [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 17:26:34 localhost kernel:                         [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 17:26:34 localhost kernel:                         [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:35 localhost kernel:                         [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:35 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 17:26:35 localhost kernel:    softirq-on-R at:
Feb 18 17:26:35 localhost kernel:                         [<c0449046>] __lock_acquire+0x48b/0xbf1
Feb 18 17:26:35 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:35 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel:                         [<c061b3a7>] _read_lock+0x29/0x34
Feb 18 17:26:35 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel:                         [<c05baadb>] neigh_lookup+0x94/0x9d
Feb 18 17:26:35 localhost kernel:                         [<c05f0609>] udp_sendmsg+0x240/0x59d
Feb 18 17:26:35 localhost kernel:                         [<c05cf322>] rt_intern_hash+0x310/0x322
Feb 18 17:26:35 localhost kernel:                         [<c05f5fc7>] inet_sendmsg+0x3b/0x45
Feb 18 17:26:35 localhost kernel:                         [<c05ab8cd>] sock_sendmsg+0xc9/0xe4
Feb 18 17:26:35 localhost kernel:                         [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:35 localhost kernel:                         [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:35 localhost kernel:                         [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:35 localhost kernel:                         [<c061b528>] _read_unlock_irq+0x20/0x23
Feb 18 17:26:35 localhost kernel:                         [<c061b528>] _read_unlock_irq+0x20/0x23
Feb 18 17:26:35 localhost kernel:                         [<c05ac29b>] sys_sendto+0x11b/0x13b
Feb 18 17:26:35 localhost kernel:                         [<c046b794>] __do_fault+0x23b/0x35d
Feb 18 17:26:35 localhost kernel:                         [<c046b874>] __do_fault+0x31b/0x35d
Feb 18 17:26:35 localhost kernel:                         [<c046d4ff>] handle_mm_fault+0x2a2/0x5f7
Feb 18 17:26:35 localhost kernel:                         [<c05ac2f2>] sys_send+0x37/0x3b
Feb 18 17:26:35 localhost kernel:                         [<c05acc35>] sys_socketcall+0x14a/0x261
Feb 18 17:26:35 localhost kernel:                         [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:35 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 17:26:35 localhost kernel:    hardirq-on-R at:
Feb 18 17:26:35 localhost kernel:                         [<c048926a>] do_sys_poll+0x260/0x2e0
Feb 18 17:26:35 localhost kernel:                         [<c0449001>] __lock_acquire+0x446/0xbf1
Feb 18 17:26:35 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:35 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel:                         [<c061b3a7>] _read_lock+0x29/0x34
Feb 18 17:26:35 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel:                         [<c05baadb>] neigh_lookup+0x94/0x9d
Feb 18 17:26:35 localhost kernel:                         [<c05f0609>] udp_sendmsg+0x240/0x59d
Feb 18 17:26:35 localhost kernel:                         [<c05cf322>] rt_intern_hash+0x310/0x322
Feb 18 17:26:35 localhost kernel:                         [<c05f5fc7>] inet_sendmsg+0x3b/0x45
Feb 18 17:26:35 localhost kernel:                         [<c05ab8cd>] sock_sendmsg+0xc9/0xe4
Feb 18 17:26:35 localhost kernel:                         [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:35 localhost kernel:                         [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:35 localhost kernel:                         [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:36 localhost kernel:                         [<c061b528>] _read_unlock_irq+0x20/0x23
Feb 18 17:26:36 localhost kernel:                         [<c061b528>] _read_unlock_irq+0x20/0x23
Feb 18 17:26:36 localhost kernel:                         [<c05ac29b>] sys_sendto+0x11b/0x13b
Feb 18 17:26:36 localhost kernel:                         [<c046b794>] __do_fault+0x23b/0x35d
Feb 18 17:26:36 localhost kernel:                         [<c046b874>] __do_fault+0x31b/0x35d
Feb 18 17:26:36 localhost kernel:                         [<c046d4ff>] handle_mm_fault+0x2a2/0x5f7
Feb 18 17:26:36 localhost kernel:                         [<c05ac2f2>] sys_send+0x37/0x3b
Feb 18 17:26:36 localhost kernel:                         [<c05acc35>] sys_socketcall+0x14a/0x261
Feb 18 17:26:36 localhost kernel:                         [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:36 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 17:26:36 localhost kernel:  }
Feb 18 17:26:36 localhost kernel:  ... key      at: [<c0a3dc28>] __key.35996+0x0/0x8
Feb 18 17:26:36 localhost kernel: 
Feb 18 17:26:36 localhost kernel: stack backtrace:
Feb 18 17:26:36 localhost kernel: Pid: 13863, comm: pppd Not tainted 2.6.24.2 #1
Feb 18 17:26:36 localhost kernel:  [<c0448b1e>] check_usage+0x24e/0x258
Feb 18 17:26:36 localhost kernel:  [<c0449555>] __lock_acquire+0x99a/0xbf1
Feb 18 17:26:36 localhost kernel:  [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:36 localhost kernel:  [<f8bbf73e>] pppol2tp_xmit+0x3a5/0x4a2 [pppol2tp]
Feb 18 17:26:36 localhost kernel:  [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 17:26:36 localhost kernel:  [<f8bbf73e>] pppol2tp_xmit+0x3a5/0x4a2 [pppol2tp]
Feb 18 17:26:36 localhost kernel:  [<f8bbf73e>] pppol2tp_xmit+0x3a5/0x4a2 [pppol2tp]
Feb 18 17:26:36 localhost kernel:  [<f8bb730e>] ppp_channel_push+0x34/0x90 [ppp_generic]
Feb 18 17:26:36 localhost kernel:  [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:36 localhost kernel:  [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:36 localhost kernel:  [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:36 localhost kernel:  [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:36 localhost kernel:  [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:36 localhost kernel:  =======================

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Mon, Feb 18, 2008 at 10:09:24PM +0000, James Chapman wrote:
...
> Unfortunately the ISP's syslog stops. But I've been able to borrow
> two Quad Xeon boxes and have reproduced the problem.
>
> Here's a new version of the patch. The patch avoids disabling irqs
> and fixes the sk_dst_get() usage that DaveM mentioned. But even with
> this patch, lockdep still complains if hundreds of ppp sessions are
> inserted into a tunnel as rapidly as possible (lockdep trace is below).
> I can stop these errors by wrapping the call to ppp_input() in
> pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is a
> better fix?

I send here my proposal: it's intended for testing and to check one of
possible solutions here. IMHO your lockdep reports show there is no
use to change anything around sk_dst_lock: it would need the global
change of this lock to fix this problem. So the fix should be done
around pch->upl lock and this means changing ppp_generic.

In the patch below I've used trylock in places which seem to allow
for skipping some things (while config is changed only) or simply
don't need this lock because there is no ppp struct. This could be
modified to add some waiting loop if necessary. Another option is to
change the write side of this lock: it looks like more vulnerable if
something missed because there are more locks involved, but probably
should be enough to solve this problem too.

I think pppol2tp need to be first checked only with hlist_lock bh
patch, unless there were some lockdep reports on these other locks
too. (BTW, I added ppp maintainer to CC - I hope we get Paul's opinion
on this.)

Regards,
Jarek P.

(testing patch #1)
---

 drivers/net/ppp_generic.c |   33 +++++++++++++++++++++++----------
 1 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/drivers/net/ppp_generic.c b/drivers/net/ppp_generic.c
index 4dc5b4b..5cbc534 100644
--- a/drivers/net/ppp_generic.c
+++ b/drivers/net/ppp_generic.c
@@ -1473,7 +1473,7 @@ void
 ppp_input(struct ppp_channel *chan, struct sk_buff *skb)
 {
 	struct channel *pch = chan->ppp;
-	int proto;
+	int proto, locked;
 
 	if (!pch || skb->len == 0) {
 		kfree_skb(skb);
@@ -1481,8 +1481,13 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb)
 	}
 
 	proto = PPP_PROTO(skb);
-	read_lock_bh(&pch->upl);
-	if (!pch->ppp || proto >= 0xc000 || proto == PPP_CCPFRAG) {
+	/*
+	 * We use trylock to avoid dependency between soft-irq-safe upl lock
+	 * and soft-irq-unsafe sk_dst_lock.
+	 */
+	local_bh_disable();
+	locked = read_trylock(&pch->upl);
+	if (!locked || !pch->ppp || proto >= 0xc000 || proto == PPP_CCPFRAG) {
 		/* put it on the channel queue */
 		skb_queue_tail(&pch->file.rq, skb);
 		/* drop old frames if queue too long */
@@ -1493,7 +1498,10 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb)
 	} else {
 		ppp_do_recv(pch->ppp, skb, pch);
 	}
-	read_unlock_bh(&pch->upl);
+
+	if (locked)
+		read_unlock(&pch->upl);
+	local_bh_enable();
 }
 
 /* Put a 0-length skb in the receive queue as an error indication */
@@ -1506,16 +1514,18 @@ ppp_input_error(struct ppp_channel *chan, int code)
 	if (!pch)
 		return;
 
-	read_lock_bh(&pch->upl);
-	if (pch->ppp) {
+	/* a trylock comment in ppp_input() */
+	local_bh_disable();
+	if (read_trylock(&pch->upl) && pch->ppp) {
 		skb = alloc_skb(0, GFP_ATOMIC);
 		if (skb) {
 			skb->len = 0;		/* probably unnecessary */
 			skb->cb[0] = code;
 			ppp_do_recv(pch->ppp, skb, pch);
 		}
+		read_unlock(&pch->upl);
 	}
-	read_unlock_bh(&pch->upl);
+	local_bh_enable();
 }
 
 /*
@@ -2044,10 +2054,13 @@ int ppp_unit_number(struct ppp_channel *chan)
 	int unit = -1;
 
 	if (pch) {
-		read_lock_bh(&pch->upl);
-		if (pch->ppp)
+		/* a trylock comment in ppp_input() */
+		local_bh_disable();
+		if (read_trylock(&pch->upl) && pch->ppp) {
 			unit = pch->ppp->file.index;
-		read_unlock_bh(&pch->upl);
+			read_unlock(&pch->upl);
+		}
+		local_bh_enable();
 	}
 	return unit;
 }

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Wed, Feb 20, 2008 at 12:06:40AM +0100, Jarek Poplawski wrote:
...
> (testing patch #1)

SORRY!!! ----> take 2 (unlocking fixed)

---

 drivers/net/ppp_generic.c |   39 +++++++++++++++++++++++++++++----------
 1 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/drivers/net/ppp_generic.c b/drivers/net/ppp_generic.c
index 4dc5b4b..c4e3808 100644
--- a/drivers/net/ppp_generic.c
+++ b/drivers/net/ppp_generic.c
@@ -1473,7 +1473,7 @@ void
 ppp_input(struct ppp_channel *chan, struct sk_buff *skb)
 {
 	struct channel *pch = chan->ppp;
-	int proto;
+	int proto, locked;
 
 	if (!pch || skb->len == 0) {
 		kfree_skb(skb);
@@ -1481,8 +1481,13 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb)
 	}
 
 	proto = PPP_PROTO(skb);
-	read_lock_bh(&pch->upl);
-	if (!pch->ppp || proto >= 0xc000 || proto == PPP_CCPFRAG) {
+	/*
+	 * We use trylock to avoid dependency between soft-irq-safe upl lock
+	 * and soft-irq-unsafe sk_dst_lock.
+	 */
+	local_bh_disable();
+	locked = read_trylock(&pch->upl);
+	if (!locked || !pch->ppp || proto >= 0xc000 || proto == PPP_CCPFRAG) {
 		/* put it on the channel queue */
 		skb_queue_tail(&pch->file.rq, skb);
 		/* drop old frames if queue too long */
@@ -1493,7 +1498,10 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb)
 	} else {
 		ppp_do_recv(pch->ppp, skb, pch);
 	}
-	read_unlock_bh(&pch->upl);
+
+	if (locked)
+		read_unlock(&pch->upl);
+	local_bh_enable();
 }
 
 /* Put a 0-length skb in the receive queue as an error indication */
@@ -1502,12 +1510,14 @@ ppp_input_error(struct ppp_channel *chan, int code)
 {
 	struct channel *pch = chan->ppp;
 	struct sk_buff *skb;
+	int locked;
 
 	if (!pch)
 		return;
 
-	read_lock_bh(&pch->upl);
-	if (pch->ppp) {
+	/* a trylock comment in ppp_input() */
+	local_bh_disable();
+	if ((locked = read_trylock(&pch->upl)) && pch->ppp) {
 		skb = alloc_skb(0, GFP_ATOMIC);
 		if (skb) {
 			skb->len = 0;		/* probably unnecessary */
@@ -1515,7 +1525,10 @@ ppp_input_error(struct ppp_channel *chan, int code)
 			ppp_do_recv(pch->ppp, skb, pch);
 		}
 	}
-	read_unlock_bh(&pch->upl);
+
+	if (locked)
+		read_unlock(&pch->upl);
+	local_bh_enable();
 }
 
 /*
@@ -2044,10 +2057,16 @@ int ppp_unit_number(struct ppp_channel *chan)
 	int unit = -1;
 
 	if (pch) {
-		read_lock_bh(&pch->upl);
-		if (pch->ppp)
+		int locked;
+
+		/* a trylock comment in ppp_input() */
+		local_bh_disable();
+		if ((locked = read_trylock(&pch->upl)) && pch->ppp)
 			unit = pch->ppp->file.index;
-		read_unlock_bh(&pch->upl);
+
+		if (locked)
+			read_unlock(&pch->upl);
+		local_bh_enable();
 	}
 	return unit;
 }

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[James Chapman]

Jarek Poplawski wrote:
> On Mon, Feb 18, 2008 at 10:09:24PM +0000, James Chapman wrote:
> ...
>> Unfortunately the ISP's syslog stops. But I've been able to borrow
>> two Quad Xeon boxes and have reproduced the problem.
>>
>> Here's a new version of the patch. The patch avoids disabling irqs
>> and fixes the sk_dst_get() usage that DaveM mentioned. But even with
>> this patch, lockdep still complains if hundreds of ppp sessions are
>> inserted into a tunnel as rapidly as possible (lockdep trace is below).
>> I can stop these errors by wrapping the call to ppp_input() in
>> pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is a
>> better fix?
> 
> I send here my proposal: it's intended for testing and to check one of
> possible solutions here. IMHO your lockdep reports show there is no
> use to change anything around sk_dst_lock: it would need the global
> change of this lock to fix this problem. So the fix should be done
> around pch->upl lock and this means changing ppp_generic.

Hmm, I need to study the lockdep report again. It seems I'm misreading 
the lockdep output. :(

> In the patch below I've used trylock in places which seem to allow
> for skipping some things (while config is changed only) or simply
> don't need this lock because there is no ppp struct. This could be
> modified to add some waiting loop if necessary. Another option is to
> change the write side of this lock: it looks like more vulnerable if
> something missed because there are more locks involved, but probably
> should be enough to solve this problem too.
> 
> I think pppol2tp need to be first checked only with hlist_lock bh
> patch, unless there were some lockdep reports on these other locks
> too. (BTW, I added ppp maintainer to CC - I hope we get Paul's opinion
> on this.)

I tried your ppp_generic patch with only the hlist_lock bh patch in 
pppol2tp and it seems to fix the ppp create/delete issue. However, when 
I added much more traffic into the test (flood pings over ppp interfaces 
while repeatedly creating/deleting the L2TP (PPP) sessions) I get a soft 
lockup detected in pppol2tp_xmit() after anything between 1 minute and 
an hour. :( I'm investigating that now.

Thanks for your help!

> (testing patch #1)
> ---
> 
>  drivers/net/ppp_generic.c |   33 +++++++++++++++++++++++----------
>  1 files changed, 23 insertions(+), 10 deletions(-)
> 
> diff --git a/drivers/net/ppp_generic.c b/drivers/net/ppp_generic.c
> index 4dc5b4b..5cbc534 100644
> --- a/drivers/net/ppp_generic.c
> +++ b/drivers/net/ppp_generic.c
> @@ -1473,7 +1473,7 @@ void
>  ppp_input(struct ppp_channel *chan, struct sk_buff *skb)
>  {
>  	struct channel *pch = chan->ppp;
> -	int proto;
> +	int proto, locked;
>  
>  	if (!pch || skb->len == 0) {
>  		kfree_skb(skb);
> @@ -1481,8 +1481,13 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb)
>  	}
>  
>  	proto = PPP_PROTO(skb);
> -	read_lock_bh(&pch->upl);
> -	if (!pch->ppp || proto >= 0xc000 || proto == PPP_CCPFRAG) {
> +	/*
> +	 * We use trylock to avoid dependency between soft-irq-safe upl lock
> +	 * and soft-irq-unsafe sk_dst_lock.
> +	 */
> +	local_bh_disable();
> +	locked = read_trylock(&pch->upl);
> +	if (!locked || !pch->ppp || proto >= 0xc000 || proto == PPP_CCPFRAG) {
>  		/* put it on the channel queue */
>  		skb_queue_tail(&pch->file.rq, skb);
>  		/* drop old frames if queue too long */
> @@ -1493,7 +1498,10 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb)
>  	} else {
>  		ppp_do_recv(pch->ppp, skb, pch);
>  	}
> -	read_unlock_bh(&pch->upl);
> +
> +	if (locked)
> +		read_unlock(&pch->upl);
> +	local_bh_enable();
>  }
>  
>  /* Put a 0-length skb in the receive queue as an error indication */
> @@ -1506,16 +1514,18 @@ ppp_input_error(struct ppp_channel *chan, int code)
>  	if (!pch)
>  		return;
>  
> -	read_lock_bh(&pch->upl);
> -	if (pch->ppp) {
> +	/* a trylock comment in ppp_input() */
> +	local_bh_disable();
> +	if (read_trylock(&pch->upl) && pch->ppp) {
>  		skb = alloc_skb(0, GFP_ATOMIC);
>  		if (skb) {
>  			skb->len = 0;		/* probably unnecessary */
>  			skb->cb[0] = code;
>  			ppp_do_recv(pch->ppp, skb, pch);
>  		}
> +		read_unlock(&pch->upl);
>  	}
> -	read_unlock_bh(&pch->upl);
> +	local_bh_enable();
>  }
>  
>  /*
> @@ -2044,10 +2054,13 @@ int ppp_unit_number(struct ppp_channel *chan)
>  	int unit = -1;
>  
>  	if (pch) {
> -		read_lock_bh(&pch->upl);
> -		if (pch->ppp)
> +		/* a trylock comment in ppp_input() */
> +		local_bh_disable();
> +		if (read_trylock(&pch->upl) && pch->ppp) {
>  			unit = pch->ppp->file.index;
> -		read_unlock_bh(&pch->upl);
> +			read_unlock(&pch->upl);
> +		}
> +		local_bh_enable();
>  	}
>  	return unit;
>  }
> --


-- 
James Chapman
Katalix Systems Ltd
http://www.katalix.com
Catalysts for your Embedded Linux software development

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Wed, Feb 20, 2008 at 04:02:52PM +0000, James Chapman wrote:
...
> I tried your ppp_generic patch with only the hlist_lock bh patch in  
> pppol2tp and it seems to fix the ppp create/delete issue. However, when  
> I added much more traffic into the test (flood pings over ppp interfaces  
> while repeatedly creating/deleting the L2TP (PPP) sessions) I get a soft  
> lockup detected in pppol2tp_xmit() after anything between 1 minute and  
> an hour. :( I'm investigating that now.
>
> Thanks for your help!

Not at all!

>
>> (testing patch #1)

But I hope you tested with the fixed (take 2) version of this patch...

Since it's quite experimental (testing) this patch could be wrong
as it is, but I hope it should show the proper way to solve this
problem. Probably you did some of these, but here are a few of my
suggestions for testing this:

1) try my patch with your full bh locking changing patch;
2) add while loops to these trylocks on failure, with e.g.  __delay(1);
   this should work like full locks again, but there should be no (this
   kind of) lockdep reports;
3) I send here another testing patch with this second way to do this:
   on the write side, but it's even more "experimental" and only a
   proof of concept (should be applied on vanilla ppp_generic).

Regards,
Jarek P.

(testing patch #2)

---

 drivers/net/ppp_generic.c |   20 +++++++++++++-------
 1 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/drivers/net/ppp_generic.c b/drivers/net/ppp_generic.c
index 4dc5b4b..70bd255 100644
--- a/drivers/net/ppp_generic.c
+++ b/drivers/net/ppp_generic.c
@@ -2606,11 +2606,16 @@ ppp_connect_channel(struct channel *pch, int unit)
 	ppp = ppp_find_unit(unit);
 	if (!ppp)
 		goto out;
-	write_lock_bh(&pch->upl);
+
 	ret = -EINVAL;
-	if (pch->ppp)
-		goto outl;
+	read_lock_bh(&pch->upl);
+	if (pch->ppp) {
+		read_unlock_bh(&pch->upl);
+		goto out;
+	}
+	read_unlock_bh(&pch->upl);
 
+	atomic_inc(&ppp->file.refcnt);
 	ppp_lock(ppp);
 	if (pch->file.hdrlen > ppp->file.hdrlen)
 		ppp->file.hdrlen = pch->file.hdrlen;
@@ -2619,13 +2624,14 @@ ppp_connect_channel(struct channel *pch, int unit)
 		ppp->dev->hard_header_len = hdrlen;
 	list_add_tail(&pch->clist, &ppp->channels);
 	++ppp->n_channels;
-	pch->ppp = ppp;
-	atomic_inc(&ppp->file.refcnt);
 	ppp_unlock(ppp);
-	ret = 0;
 
- outl:
+	/* avoid lock dependency with ppp_locks */
+	write_lock_bh(&pch->upl);
+	BUG_ON(pch->ppp);
+	pch->ppp = ppp;
 	write_unlock_bh(&pch->upl);
+	ret = 0;
  out:
 	mutex_unlock(&all_ppp_mutex);
 	return ret;

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[James Chapman]

[-- Attachment #1: Type: text/plain, Size: 1152 bytes --]

Jarek Poplawski wrote:

>>> (testing patch #1)
> 
> But I hope you tested with the fixed (take 2) version of this patch...

Yes I did. :)

But I just got another lockdep error (attached).

> Since it's quite experimental (testing) this patch could be wrong
> as it is, but I hope it should show the proper way to solve this
> problem. Probably you did some of these, but here are a few of my
> suggestions for testing this:
> 
> 1) try my patch with your full bh locking changing patch;
> 2) add while loops to these trylocks on failure, with e.g.  __delay(1);
>    this should work like full locks again, but there should be no (this
>    kind of) lockdep reports;

Hmm, isn't this just bypassing the lockdep checks?

> 3) I send here another testing patch with this second way to do this:
>    on the write side, but it's even more "experimental" and only a
>    proof of concept (should be applied on vanilla ppp_generic).

I'll look over it. I think I need to take a step back and look at what's 
happening in more detail though.

-- 
James Chapman
Katalix Systems Ltd
http://www.katalix.com
Catalysts for your Embedded Linux software development


[-- Attachment #2: lockdep.log --]
[-- Type: text/plain, Size: 8600 bytes --]

Feb 20 22:11:41 localhost kernel: =================================
Feb 20 22:11:43 localhost kernel: [ INFO: inconsistent lock state ]
Feb 20 22:11:44 localhost kernel: 2.6.24.2 #1
Feb 20 22:11:44 localhost kernel: ---------------------------------
Feb 20 22:11:44 localhost kernel: inconsistent {softirq-on-W} -> {in-softirq-R} usage.
Feb 20 22:11:44 localhost kernel: pppd/3744 [HC0[0]:SC1[5]:HE1:SE0] takes:
Feb 20 22:11:44 localhost kernel:  (&sk->sk_dst_lock){---?}, at: [<f8bac857>] pppol2tp_xmit+0x320/0x3d9 [pppol2tp]
Feb 20 22:11:45 localhost kernel: {softirq-on-W} state was registered at:
Feb 20 22:11:45 localhost kernel:   [<c0449046>] __lock_acquire+0x48b/0xbf1
Feb 20 22:11:45 localhost kernel:   [<c0448594>] mark_held_locks+0x39/0x53
Feb 20 22:11:45 localhost kernel:   [<c043083a>] local_bh_enable+0x10e/0x115
Feb 20 22:11:45 localhost kernel:   [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 20 22:11:45 localhost kernel:   [<c044981c>] lock_acquire+0x70/0x8a
Feb 20 22:11:45 localhost kernel:   [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 20 22:11:45 localhost kernel:   [<c061b2c7>] _write_lock+0x29/0x34
Feb 20 22:11:45 localhost kernel:   [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 20 22:11:45 localhost kernel:   [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 20 22:11:45 localhost kernel:   [<c05f66da>] inet_listen+0x3b/0x5e
Feb 20 22:11:45 localhost kernel:   [<c05ab68f>] sys_listen+0x43/0x5f
Feb 20 22:11:45 localhost kernel:   [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 20 22:11:45 localhost kernel:   [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 20 22:11:45 localhost kernel:   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 20 22:11:45 localhost kernel:   [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 20 22:11:45 localhost kernel:   [<ffffffff>] 0xffffffff
Feb 20 22:11:45 localhost kernel: irq event stamp: 41702
Feb 20 22:11:45 localhost kernel: hardirqs last  enabled at (41702): [<c047a1bd>] kfree+0x9f/0xa6
Feb 20 22:11:45 localhost kernel: hardirqs last disabled at (41701): [<c047a135>] kfree+0x17/0xa6
Feb 20 22:11:45 localhost kernel: softirqs last  enabled at (41630): [<c05ceed4>] rt_run_flush+0x64/0x8b
Feb 20 22:11:45 localhost kernel: softirqs last disabled at (41631): [<c040709f>] do_softirq+0x5e/0xc7
Feb 20 22:11:45 localhost kernel: 
Feb 20 22:11:45 localhost kernel: other info that might help us debug this:
Feb 20 22:11:45 localhost kernel: 10 locks held by pppd/3744:
Feb 20 22:11:45 localhost kernel:  #0:  (rtnl_mutex){--..}, at: [<c05f53bd>] devinet_ioctl+0xf4/0x539
Feb 20 22:11:45 localhost kernel:  #1:  ((inetaddr_chain).rwsem){..--}, at: [<c04411c9>] __blocking_notifier_call_chain+0x22/0x51
Feb 20 22:11:45 localhost kernel:  #2:  (rcu_read_lock){..--}, at: [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 20 22:11:45 localhost kernel:  #3:  (rcu_read_lock){..--}, at: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 20 22:11:45 localhost kernel:  #4:  (rcu_read_lock){..--}, at: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 20 22:11:45 localhost kernel:  #5:  (slock-AF_INET){-+..}, at: [<c05f2d21>] icmp_reply+0x52/0x1a4
Feb 20 22:11:45 localhost kernel:  #6:  (rcu_read_lock){..--}, at: [<c05b7e60>] dev_queue_xmit+0x125/0x2e9
Feb 20 22:11:45 localhost kernel:  #7:  (_xmit_PPP){-+..}, at: [<c05c4cb1>] __qdisc_run+0x5b/0x156
Feb 20 22:11:45 localhost kernel:  #8:  (&ppp->wlock){-+..}, at: [<f8b9ef10>] ppp_xmit_process+0x15/0x5a1 [ppp_generic]
Feb 20 22:11:45 localhost kernel:  #9:  (&pch->downl){-+..}, at: [<f8b9e6db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 20 22:11:45 localhost kernel: 
Feb 20 22:11:45 localhost kernel: stack backtrace:
Feb 20 22:11:45 localhost kernel: Pid: 3744, comm: pppd Not tainted 2.6.24.2 #1
Feb 20 22:11:45 localhost kernel:  [<c0447933>] print_usage_bug+0x139/0x143
Feb 20 22:11:45 localhost kernel:  [<c04482d2>] mark_lock+0x1cb/0x454
Feb 20 22:11:45 localhost kernel:  [<c04480a2>] find_usage_forwards+0x67/0x8b
Feb 20 22:11:45 localhost kernel:  [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 20 22:11:46 localhost kernel:  [<c0448b8e>] check_noncircular+0x66/0x93
Feb 20 22:11:46 localhost kernel:  [<c0448594>] mark_held_locks+0x39/0x53
Feb 20 22:11:46 localhost kernel:  [<c047a1bd>] kfree+0x9f/0xa6
Feb 20 22:11:46 localhost kernel:  [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 20 22:11:46 localhost kernel:  [<c044981c>] lock_acquire+0x70/0x8a
Feb 20 22:11:46 localhost kernel:  [<f8bac857>] pppol2tp_xmit+0x320/0x3d9 [pppol2tp]
Feb 20 22:11:46 localhost kernel:  [<c061b3a7>] _read_lock+0x29/0x34
Feb 20 22:11:46 localhost kernel:  [<f8bac857>] pppol2tp_xmit+0x320/0x3d9 [pppol2tp]
Feb 20 22:11:46 localhost kernel:  [<f8bac857>] pppol2tp_xmit+0x320/0x3d9 [pppol2tp]
Feb 20 22:11:46 localhost kernel:  [<f8b9e6db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 20 22:11:46 localhost kernel:  [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 20 22:11:46 localhost kernel:  [<f8b9e6ee>] ppp_push+0x76/0x50d [ppp_generic]
Feb 20 22:11:46 localhost kernel:  [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 20 22:11:46 localhost kernel:  [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 20 22:11:46 localhost kernel:  [<f8b9f3f1>] ppp_xmit_process+0x4f6/0x5a1 [ppp_generic]
Feb 20 22:11:46 localhost kernel:  [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 20 22:11:46 localhost kernel:  [<f8b9f5d2>] ppp_start_xmit+0x136/0x167 [ppp_generic]
Feb 20 22:11:46 localhost kernel:  [<c05b5d29>] dev_hard_start_xmit+0x245/0x29f
Feb 20 22:11:46 localhost kernel:  [<c061b25a>] _spin_lock+0x29/0x34
Feb 20 22:11:46 localhost kernel:  [<c05c4cc0>] __qdisc_run+0x6a/0x156
Feb 20 22:11:46 localhost kernel:  [<c05b7ee0>] dev_queue_xmit+0x1a5/0x2e9
Feb 20 22:11:46 localhost kernel:  [<c05b7e60>] dev_queue_xmit+0x125/0x2e9
Feb 20 22:11:46 localhost kernel:  [<c05d62f5>] ip_finish_output+0x1f4/0x21e
Feb 20 22:11:46 localhost kernel:  [<c05d4f78>] ip_push_pending_frames+0x2dc/0x33b
Feb 20 22:11:46 localhost kernel:  [<c05f2de5>] icmp_reply+0x116/0x1a4
Feb 20 22:11:46 localhost kernel:  [<c05f32b6>] icmp_echo+0x49/0x4d
Feb 20 22:11:46 localhost kernel:  [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 20 22:11:46 localhost kernel:  [<c05f2f67>] icmp_rcv+0xf4/0x116
Feb 20 22:11:46 localhost kernel:  [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 20 22:11:46 localhost kernel:  [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 20 22:11:46 localhost kernel:  [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 20 22:11:46 localhost kernel:  [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 20 22:11:46 localhost kernel:  [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 20 22:11:46 localhost kernel:  [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 20 22:11:46 localhost kernel:  [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 20 22:11:46 localhost kernel:  [<c05b7c53>] process_backlog+0x6c/0xb9
Feb 20 22:11:46 localhost kernel:  [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 20 22:11:46 localhost kernel:  [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 20 22:11:46 localhost kernel:  [<c0430a9a>] __do_softirq+0x69/0xde
Feb 20 22:11:46 localhost kernel:  [<c040709f>] do_softirq+0x5e/0xc7
Feb 20 22:11:47 localhost kernel:  [<c05ceed4>] rt_run_flush+0x64/0x8b
Feb 20 22:11:47 localhost kernel:  [<c05ceed4>] rt_run_flush+0x64/0x8b
Feb 20 22:11:47 localhost kernel:  [<c04308ee>] local_bh_enable_ip+0xad/0xd5
Feb 20 22:11:47 localhost kernel:  [<c05ceed4>] rt_run_flush+0x64/0x8b
Feb 20 22:11:47 localhost kernel:  [<c05fa7e7>] fib_disable_ip+0x1e/0x26
Feb 20 22:11:47 localhost kernel:  [<c05faf99>] fib_inetaddr_event+0x19c/0x1b0
Feb 20 22:11:47 localhost kernel:  [<c061d06c>] notifier_call_chain+0x2a/0x47
Feb 20 22:11:47 localhost kernel:  [<c04411e5>] __blocking_notifier_call_chain+0x3e/0x51
Feb 20 22:11:47 localhost kernel:  [<c044120f>] blocking_notifier_call_chain+0x17/0x1a
Feb 20 22:11:47 localhost kernel:  [<c05f450b>] __inet_del_ifa+0x12f/0x1f0
Feb 20 22:11:47 localhost kernel:  [<c05f45e3>] inet_del_ifa+0x17/0x1a
Feb 20 22:11:47 localhost kernel:  [<c05f5716>] devinet_ioctl+0x44d/0x539
Feb 20 22:11:47 localhost kernel:  [<c05b7188>] dev_ioctl+0x46f/0x5e0
Feb 20 22:11:47 localhost kernel:  [<c05ab078>] sock_ioctl+0x1bb/0x1e0
Feb 20 22:11:47 localhost kernel:  [<c05aaebd>] sock_ioctl+0x0/0x1e0
Feb 20 22:11:47 localhost kernel:  [<c0488827>] do_ioctl+0x1f/0x62
Feb 20 22:11:47 localhost kernel:  [<c0458df0>] audit_syscall_entry+0x10d/0x137
Feb 20 22:11:47 localhost kernel:  [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 20 22:11:47 localhost kernel:  [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 20 22:11:47 localhost kernel:  [<c0404efa>] syscall_call+0x7/0xb
Feb 20 22:11:47 localhost kernel:  =======================

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Wed, Feb 20, 2008 at 10:37:57PM +0000, James Chapman wrote:
> Jarek Poplawski wrote:
>
>>>> (testing patch #1)
>>
>> But I hope you tested with the fixed (take 2) version of this patch...
>
> Yes I did. :)
>
> But I just got another lockdep error (attached).
>
>> Since it's quite experimental (testing) this patch could be wrong
>> as it is, but I hope it should show the proper way to solve this
>> problem. Probably you did some of these, but here are a few of my
>> suggestions for testing this:
>>
>> 1) try my patch with your full bh locking changing patch;
>> 2) add while loops to these trylocks on failure, with e.g.  __delay(1);
>>    this should work like full locks again, but there should be no (this
>>    kind of) lockdep reports;
>
> Hmm, isn't this just bypassing the lockdep checks?

Yes! But it's only for debugging: to find if this change in locking
is to be blamed for these new lockups. It should effectively work just
like without this patch, but without this lockdep warning. So, if
after such change lockups still happen, then it would seem you didn't
test this enough before. Otherwise the new patch is to blame and needs
reworking.

>
>> 3) I send here another testing patch with this second way to do this:
>>    on the write side, but it's even more "experimental" and only a
>>    proof of concept (should be applied on vanilla ppp_generic).
>
> I'll look over it. I think I need to take a step back and look at what's  
> happening in more detail though.

This is something completely new and changes all the picture: the xmit
path wasn't expected (at least by me) to be called in softirq context
at all, and there were no traces of this on previous reports. But,
since lockdep always stops after the first warning, there could be
even more surprises like this in the future. I'll check this report.

Regards,
Jarek P.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[James Chapman]

Jarek Poplawski wrote:
> On Wed, Feb 20, 2008 at 10:37:57PM +0000, James Chapman wrote:
>> Jarek Poplawski wrote:
>>
>>>>> (testing patch #1)
>>> But I hope you tested with the fixed (take 2) version of this patch...
>> Yes I did. :)
>>
>> But I just got another lockdep error (attached).
>>
>>> Since it's quite experimental (testing) this patch could be wrong
>>> as it is, but I hope it should show the proper way to solve this
>>> problem. Probably you did some of these, but here are a few of my
>>> suggestions for testing this:
>>>
>>> 1) try my patch with your full bh locking changing patch;
>>> 2) add while loops to these trylocks on failure, with e.g.  __delay(1);
>>>    this should work like full locks again, but there should be no (this
>>>    kind of) lockdep reports;
>> Hmm, isn't this just bypassing the lockdep checks?
> 
> Yes! But it's only for debugging: to find if this change in locking
> is to be blamed for these new lockups. It should effectively work just
> like without this patch, but without this lockdep warning. So, if
> after such change lockups still happen, then it would seem you didn't
> test this enough before. Otherwise the new patch is to blame and needs
> reworking.

The lockups still happen, but I think they are now due to a different 
problem, as you say.

Some background on this issue might be useful to help get feedback from 
others on the list. This issue was first reported by an ISP who found 
random lockups if an L2TP tunnel carrying hundreds/thousands of L2TP 
sessions went down due to a network outage and then recovered itself. On 
recovery, all of the tunnel's sessions (PPP) are created rapidly. 
Sometimes the tunnel would recover just fine, but other times not. The 
ISP put some effort into reproducing the problem and found that 
repeatedly creating/deleting a tunnel with lots of L2TP sessions would 
cause the failure after a random time between a few minutes and several 
hours. The original lockdep trace came from the ISP. I initially 
couldn't reproduce the problem but I borrowed two equivalent quad-core 
systems and can now reproduce it. Subsequent lockdep traces have been 
from my testing.

The _bh locking fixes in pppol2tp combined with your ppp_generic change 
solved that problem. So I then added data traffic into the mix (since 
this will happen in a real network) and found that lockups still happen. 
But the lockdep trace in this case is different, as you noted.

Does PPPoE stress the PPP setup code as much as this scenario? I guess 
in theory it could if lots of PPPoE clients connected at the same time, 
but there is no aggregate tunnel like there is with L2TP to cause all 
sessions to connect simultaneously. Perhaps PPTP also suffers from these 
issues? Perhaps not because it tends to be used only in VPN setups where 
there is only 1 session per tunnel.

>>> 3) I send here another testing patch with this second way to do this:
>>>    on the write side, but it's even more "experimental" and only a
>>>    proof of concept (should be applied on vanilla ppp_generic).
>> I'll look over it. I think I need to take a step back and look at what's  
>> happening in more detail though.
> 
> This is something completely new and changes all the picture: the xmit
> path wasn't expected (at least by me) to be called in softirq context
> at all, and there were no traces of this on previous reports. But,
> since lockdep always stops after the first warning, there could be
> even more surprises like this in the future. I'll check this report.

Doesn't the TX softirq do transmits if they've been queued up?

-- 
James Chapman
Katalix Systems Ltd
http://www.katalix.com
Catalysts for your Embedded Linux software development

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Thu, Feb 21, 2008 at 09:53:56AM +0000, James Chapman wrote:
> Jarek Poplawski wrote:
...
> The _bh locking fixes in pppol2tp combined with your ppp_generic change  
> solved that problem. So I then added data traffic into the mix (since  
> this will happen in a real network) and found that lockups still happen.  
> But the lockdep trace in this case is different, as you noted.

I'm not sure what do you mean by "solved that problem": lack of lockups
or lack of this kind of lockdep reports. This lockdep report shows a
real danger in this case, probably very little probable, unless a lot
of tries. So if you think just this kind of lockup happend there, this
is would be nice. But there could be something less nice too: we are
"fighting" with lockdep in one place but the lockup happens somewhere
else for some totally different reason...

> Does PPPoE stress the PPP setup code as much as this scenario? I guess  
> in theory it could if lots of PPPoE clients connected at the same time,  
> but there is no aggregate tunnel like there is with L2TP to cause all  
> sessions to connect simultaneously. Perhaps PPTP also suffers from these  
> issues? Perhaps not because it tends to be used only in VPN setups where  
> there is only 1 session per tunnel.

I don't know this code enough, but it seems it should be easier to
maintain or debug if there are similar solutions where possible.

>>>> 3) I send here another testing patch with this second way to do this:
>>>>    on the write side, but it's even more "experimental" and only a
>>>>    proof of concept (should be applied on vanilla ppp_generic).
>>> I'll look over it. I think I need to take a step back and look at 
>>> what's  happening in more detail though.
>>
>> This is something completely new and changes all the picture: the xmit
>> path wasn't expected (at least by me) to be called in softirq context
>> at all, and there were no traces of this on previous reports. But,
>> since lockdep always stops after the first warning, there could be
>> even more surprises like this in the future. I'll check this report.
>
> Doesn't the TX softirq do transmits if they've been queued up?

I've probably too much looked at these reports, and should've expected
this could happen. Probably queueing could be separated, but since
there could be no queue at all and it's done like this, then this
current proof of concept seems to be dead end, and we have to go back
to fixing sk_dst_lock handling and my patches could be dumped...

So if I don't miss something again (and I need more time for this new
report) you should try to fix the problem not reported originally by
lockdep, but forseen by David(!): we need to avoid any path like:
ppp_generic -> pppol2tp -> something, which could take sk_dst_lock
while holding any ppp_generic writing lock:  they all are softirq
"safe" (i.e. endangered). David gave some example, but I'm not sure
you did your patch like this (sk_dst_set()). Probably ip_queue_xmit()
can't work with this too.

Another, probably simpler way would be to move almost all pppol2tp_xmit
code to a workqueue: this should let to break most of dependencies with
ppp_generic locks, but I don't know how much it would affect other
things (e.g. performance). So you should really rethink these things.

Jarek P.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

Jarek Poplawski wrote, On 02/21/2008 01:08 PM:
...

> Another, probably simpler way would be to move almost all pppol2tp_xmit
...

Actually, the simplest off all seems to be now this old idea to maybe make
sk_dst_lock globally softirq immune. At least I think it's worth of testing,
to check for these other possible lockdep warnings. It should only need to
change all write_ and read_lock(&sk->sk_dst_lock) in very few places:
include/net/sock.h, include/net/ip6_route.h, and net/ipv6/ipv6_sockglue.c.
This could be tested together with you full _bh locking patch (maybe except
these other changes in pppol2tp_xmit).

Jarek P.

[PATCH][NET] sock.c: sk_dst_lock lockdep keys and names per af_family

[Jarek Poplawski]

On Thu, Feb 21, 2008 at 09:53:56AM +0000, James Chapman wrote:
...
> The lockups still happen, but I think they are now due to a different  
> problem, as you say.
...

I hope, this patch should help to remove some possibly false lockdep
warnings during this current testing.

Regards,
Jarek P.

------------------->

Subject: [NET] sock.c: sk_dst_lock lockdep keys and names per af_family

Initialize sk_dst_lock lockdep keys and names per af_family.

Additionally some reorder is done in lockdep related code to keep
it in one place and to use static key tables only when needed.


Signed-off-by: Jarek Poplawski <jarkao2@gmail.com>

[not tested]
---

 net/core/sock.c |   79 +++++++++++++++++++++++++++++++++++++++----------------
 1 files changed, 56 insertions(+), 23 deletions(-)

diff --git a/net/core/sock.c b/net/core/sock.c
index 433715f..18c33d2 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -131,14 +131,17 @@
 #include <net/tcp.h>
 #endif
 
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+
 /*
- * Each address family might have different locking rules, so we have
- * one slock key per address family:
+ * Each address family might have different locking rules, so we have one:
+ * sk_lock, slock, sk_callback_lock and sk_dst_lock key per address family:
  */
 static struct lock_class_key af_family_keys[AF_MAX];
 static struct lock_class_key af_family_slock_keys[AF_MAX];
+static struct lock_class_key af_family_callback_keys[AF_MAX];
+static struct lock_class_key af_family_dst_keys[AF_MAX];
 
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
 /*
  * Make lock validator output more readable. (we pre-construct these
  * strings build-time, so that runtime initialization of socket
@@ -186,13 +189,52 @@ static const char *af_family_clock_key_strings[AF_MAX+1] = {
   "clock-AF_TIPC"  , "clock-AF_BLUETOOTH", "clock-AF_IUCV"     ,
   "clock-AF_RXRPC" , "clock-AF_MAX"
 };
-#endif
+static const char *af_family_dst_key_strings[AF_MAX+1] = {
+  "sk_dst-AF_UNSPEC", "sk_dst-AF_UNIX"     , "sk_dst-AF_INET"     ,
+  "sk_dst-AF_AX25"  , "sk_dst-AF_IPX"      , "sk_dst-AF_APPLETALK",
+  "sk_dst-AF_NETROM", "sk_dst-AF_BRIDGE"   , "sk_dst-AF_ATMPVC"   ,
+  "sk_dst-AF_X25"   , "sk_dst-AF_INET6"    , "sk_dst-AF_ROSE"     ,
+  "sk_dst-AF_DECnet", "sk_dst-AF_NETBEUI"  , "sk_dst-AF_SECURITY" ,
+  "sk_dst-AF_KEY"   , "sk_dst-AF_NETLINK"  , "sk_dst-AF_PACKET"   ,
+  "sk_dst-AF_ASH"   , "sk_dst-AF_ECONET"   , "sk_dst-AF_ATMSVC"   ,
+  "sk_dst-21"       , "sk_dst-AF_SNA"      , "sk_dst-AF_IRDA"     ,
+  "sk_dst-AF_PPPOX" , "sk_dst-AF_WANPIPE"  , "sk_dst-AF_LLC"      ,
+  "sk_dst-27"       , "sk_dst-28"          , "sk_dst-29"          ,
+  "sk_dst-AF_TIPC"  , "sk_dst-AF_BLUETOOTH", "sk_dst-AF_IUCV"     ,
+  "sk_dst-AF_RXRPC" , "sk_dst-AF_MAX"
+};
 
-/*
- * sk_callback_lock locking rules are per-address-family,
- * so split the lock classes by using a per-AF key:
- */
-static struct lock_class_key af_callback_keys[AF_MAX];
+
+static inline void sock_lock_init(struct sock *sk)
+{
+	sock_lock_init_class_and_name(sk,
+			af_family_slock_key_strings[sk->sk_family],
+			af_family_slock_keys + sk->sk_family,
+			af_family_key_strings[sk->sk_family],
+			af_family_keys + sk->sk_family);
+}
+
+#define lockdep_set_sk_callback_lock(lock, family) \
+		lockdep_set_class_and_name(lock, \
+					af_family_callback_keys + (family), \
+					af_family_clock_key_strings[(family)])
+
+#define lockdep_set_sk_dst_lock(lock, family) \
+		lockdep_set_class_and_name(lock, \
+					af_family_dst_keys + (family), \
+					af_family_dst_key_strings[(family)])
+
+#else
+
+static inline void sock_lock_init(struct sock *sk)
+{
+	sock_lock_init_class_and_name(sk, 0, 0, 0, 0);
+}
+
+#define lockdep_set_sk_callback_lock(lock, family)	do {} while (0)
+#define lockdep_set_sk_dst_lock(lock, family)		do {} while (0)
+
+#endif	/* CONFIG_DEBUG_LOCK_ALLOC */
 
 /* Take into consideration the size of the struct sk_buff overhead in the
  * determination of these values, since that is non-constant across
@@ -866,14 +908,6 @@ lenout:
  *
  * (We also register the sk_lock with the lock validator.)
  */
-static inline void sock_lock_init(struct sock *sk)
-{
-	sock_lock_init_class_and_name(sk,
-			af_family_slock_key_strings[sk->sk_family],
-			af_family_slock_keys + sk->sk_family,
-			af_family_key_strings[sk->sk_family],
-			af_family_keys + sk->sk_family);
-}
 
 static void sock_copy(struct sock *nsk, const struct sock *osk)
 {
@@ -1014,10 +1048,10 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority)
 #endif
 
 		rwlock_init(&newsk->sk_dst_lock);
+		lockdep_set_sk_dst_lock(&newsk->sk_dst_lock, newsk->sk_family);
 		rwlock_init(&newsk->sk_callback_lock);
-		lockdep_set_class_and_name(&newsk->sk_callback_lock,
-				af_callback_keys + newsk->sk_family,
-				af_family_clock_key_strings[newsk->sk_family]);
+		lockdep_set_sk_callback_lock(&newsk->sk_callback_lock,
+					     newsk->sk_family);
 
 		newsk->sk_dst_cache	= NULL;
 		newsk->sk_wmem_queued	= 0;
@@ -1703,10 +1737,9 @@ void sock_init_data(struct socket *sock, struct sock *sk)
 		sk->sk_sleep	=	NULL;
 
 	rwlock_init(&sk->sk_dst_lock);
+	lockdep_set_sk_dst_lock(&sk->sk_dst_lock, sk->sk_family);
 	rwlock_init(&sk->sk_callback_lock);
-	lockdep_set_class_and_name(&sk->sk_callback_lock,
-			af_callback_keys + sk->sk_family,
-			af_family_clock_key_strings[sk->sk_family]);
+	lockdep_set_sk_callback_lock(&sk->sk_callback_lock, sk->sk_family);
 
 	sk->sk_state_change	=	sock_def_wakeup;
 	sk->sk_data_ready	=	sock_def_readable;

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[James Chapman]

[-- Attachment #1: Type: text/plain, Size: 960 bytes --]

Jarek Poplawski wrote:
> Jarek Poplawski wrote, On 02/21/2008 01:08 PM:
> ...
> 
>> Another, probably simpler way would be to move almost all pppol2tp_xmit
> ...
> 
> Actually, the simplest off all seems to be now this old idea to maybe make
> sk_dst_lock globally softirq immune. At least I think it's worth of testing,
> to check for these other possible lockdep warnings. It should only need to
> change all write_ and read_lock(&sk->sk_dst_lock) in very few places:
> include/net/sock.h, include/net/ip6_route.h, and net/ipv6/ipv6_sockglue.c.
> This could be tested together with you full _bh locking patch (maybe except
> these other changes in pppol2tp_xmit).

I did this and all lockdep errors have now gone. Tests ran all weekend. 
See attached patch.

Is this an acceptable solution? If so, I'll prepare and send official 
patches.


-- 
James Chapman
Katalix Systems Ltd
http://www.katalix.com
Catalysts for your Embedded Linux software development


[-- Attachment #2: sk_dst_lock-softirq-safe.patch --]
[-- Type: text/x-diff, Size: 2648 bytes --]

Index: linux-2.6.24.2/include/net/ip6_route.h
===================================================================
--- linux-2.6.24.2.orig/include/net/ip6_route.h
+++ linux-2.6.24.2/include/net/ip6_route.h
@@ -150,9 +150,9 @@ static inline void __ip6_dst_store(struc
 static inline void ip6_dst_store(struct sock *sk, struct dst_entry *dst,
 				 struct in6_addr *daddr, struct in6_addr *saddr)
 {
-	write_lock(&sk->sk_dst_lock);
+	write_lock_bh(&sk->sk_dst_lock);
 	__ip6_dst_store(sk, dst, daddr, saddr);
-	write_unlock(&sk->sk_dst_lock);
+	write_unlock_bh(&sk->sk_dst_lock);
 }
 
 static inline int ipv6_unicast_destination(struct sk_buff *skb)
Index: linux-2.6.24.2/include/net/sock.h
===================================================================
--- linux-2.6.24.2.orig/include/net/sock.h
+++ linux-2.6.24.2/include/net/sock.h
@@ -1058,11 +1058,11 @@ sk_dst_get(struct sock *sk)
 {
 	struct dst_entry *dst;
 
-	read_lock(&sk->sk_dst_lock);
+	read_lock_bh(&sk->sk_dst_lock);
 	dst = sk->sk_dst_cache;
 	if (dst)
 		dst_hold(dst);
-	read_unlock(&sk->sk_dst_lock);
+	read_unlock_bh(&sk->sk_dst_lock);
 	return dst;
 }
 
@@ -1079,9 +1079,9 @@ __sk_dst_set(struct sock *sk, struct dst
 static inline void
 sk_dst_set(struct sock *sk, struct dst_entry *dst)
 {
-	write_lock(&sk->sk_dst_lock);
+	write_lock_bh(&sk->sk_dst_lock);
 	__sk_dst_set(sk, dst);
-	write_unlock(&sk->sk_dst_lock);
+	write_unlock_bh(&sk->sk_dst_lock);
 }
 
 static inline void
@@ -1097,9 +1097,9 @@ __sk_dst_reset(struct sock *sk)
 static inline void
 sk_dst_reset(struct sock *sk)
 {
-	write_lock(&sk->sk_dst_lock);
+	write_lock_bh(&sk->sk_dst_lock);
 	__sk_dst_reset(sk);
-	write_unlock(&sk->sk_dst_lock);
+	write_unlock_bh(&sk->sk_dst_lock);
 }
 
 extern struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie);
Index: linux-2.6.24.2/net/ipv6/ipv6_sockglue.c
===================================================================
--- linux-2.6.24.2.orig/net/ipv6/ipv6_sockglue.c
+++ linux-2.6.24.2/net/ipv6/ipv6_sockglue.c
@@ -440,9 +440,9 @@ static int do_ipv6_setsockopt(struct soc
 			opt = xchg(&np->opt, opt);
 			sk_dst_reset(sk);
 		} else {
-			write_lock(&sk->sk_dst_lock);
+			write_lock_bh(&sk->sk_dst_lock);
 			opt = xchg(&np->opt, opt);
-			write_unlock(&sk->sk_dst_lock);
+			write_unlock_bh(&sk->sk_dst_lock);
 			sk_dst_reset(sk);
 		}
 sticky_done:
@@ -504,9 +504,9 @@ update:
 			opt = xchg(&np->opt, opt);
 			sk_dst_reset(sk);
 		} else {
-			write_lock(&sk->sk_dst_lock);
+			write_lock_bh(&sk->sk_dst_lock);
 			opt = xchg(&np->opt, opt);
-			write_unlock(&sk->sk_dst_lock);
+			write_unlock_bh(&sk->sk_dst_lock);
 			sk_dst_reset(sk);
 		}
 

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Mon, Feb 25, 2008 at 12:19:50PM +0000, James Chapman wrote:
> Jarek Poplawski wrote:
>> Jarek Poplawski wrote, On 02/21/2008 01:08 PM:
>> ...
>>
>>> Another, probably simpler way would be to move almost all pppol2tp_xmit
>> ...
>>
>> Actually, the simplest off all seems to be now this old idea to maybe make
>> sk_dst_lock globally softirq immune. At least I think it's worth of testing,
>> to check for these other possible lockdep warnings. It should only need to
>> change all write_ and read_lock(&sk->sk_dst_lock) in very few places:
>> include/net/sock.h, include/net/ip6_route.h, and net/ipv6/ipv6_sockglue.c.
>> This could be tested together with you full _bh locking patch (maybe except
>> these other changes in pppol2tp_xmit).
>
> I did this and all lockdep errors have now gone. Tests ran all weekend.  
> See attached patch.
>
> Is this an acceptable solution? If so, I'll prepare and send official  
> patches.

IMHO this should be acceptable because I can't see any reason for
changing properly working code if there is so simple and not costly
solution. But maybe David or somebody else finds some cons? Since
this patch isn't very big I think you could try to send this
officially and we will simply see...

Regards,
Jarek P.


>
>
> -- 
> James Chapman
> Katalix Systems Ltd
> http://www.katalix.com
> Catalysts for your Embedded Linux software development
>

> Index: linux-2.6.24.2/include/net/ip6_route.h
> ===================================================================
> --- linux-2.6.24.2.orig/include/net/ip6_route.h
> +++ linux-2.6.24.2/include/net/ip6_route.h
> @@ -150,9 +150,9 @@ static inline void __ip6_dst_store(struc
>  static inline void ip6_dst_store(struct sock *sk, struct dst_entry *dst,
>  				 struct in6_addr *daddr, struct in6_addr *saddr)
>  {
> -	write_lock(&sk->sk_dst_lock);
> +	write_lock_bh(&sk->sk_dst_lock);
>  	__ip6_dst_store(sk, dst, daddr, saddr);
> -	write_unlock(&sk->sk_dst_lock);
> +	write_unlock_bh(&sk->sk_dst_lock);
>  }
>  
>  static inline int ipv6_unicast_destination(struct sk_buff *skb)
> Index: linux-2.6.24.2/include/net/sock.h
> ===================================================================
> --- linux-2.6.24.2.orig/include/net/sock.h
> +++ linux-2.6.24.2/include/net/sock.h
> @@ -1058,11 +1058,11 @@ sk_dst_get(struct sock *sk)
>  {
>  	struct dst_entry *dst;
>  
> -	read_lock(&sk->sk_dst_lock);
> +	read_lock_bh(&sk->sk_dst_lock);
>  	dst = sk->sk_dst_cache;
>  	if (dst)
>  		dst_hold(dst);
> -	read_unlock(&sk->sk_dst_lock);
> +	read_unlock_bh(&sk->sk_dst_lock);
>  	return dst;
>  }
>  
> @@ -1079,9 +1079,9 @@ __sk_dst_set(struct sock *sk, struct dst
>  static inline void
>  sk_dst_set(struct sock *sk, struct dst_entry *dst)
>  {
> -	write_lock(&sk->sk_dst_lock);
> +	write_lock_bh(&sk->sk_dst_lock);
>  	__sk_dst_set(sk, dst);
> -	write_unlock(&sk->sk_dst_lock);
> +	write_unlock_bh(&sk->sk_dst_lock);
>  }
>  
>  static inline void
> @@ -1097,9 +1097,9 @@ __sk_dst_reset(struct sock *sk)
>  static inline void
>  sk_dst_reset(struct sock *sk)
>  {
> -	write_lock(&sk->sk_dst_lock);
> +	write_lock_bh(&sk->sk_dst_lock);
>  	__sk_dst_reset(sk);
> -	write_unlock(&sk->sk_dst_lock);
> +	write_unlock_bh(&sk->sk_dst_lock);
>  }
>  
>  extern struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie);
> Index: linux-2.6.24.2/net/ipv6/ipv6_sockglue.c
> ===================================================================
> --- linux-2.6.24.2.orig/net/ipv6/ipv6_sockglue.c
> +++ linux-2.6.24.2/net/ipv6/ipv6_sockglue.c
> @@ -440,9 +440,9 @@ static int do_ipv6_setsockopt(struct soc
>  			opt = xchg(&np->opt, opt);
>  			sk_dst_reset(sk);
>  		} else {
> -			write_lock(&sk->sk_dst_lock);
> +			write_lock_bh(&sk->sk_dst_lock);
>  			opt = xchg(&np->opt, opt);
> -			write_unlock(&sk->sk_dst_lock);
> +			write_unlock_bh(&sk->sk_dst_lock);
>  			sk_dst_reset(sk);
>  		}
>  sticky_done:
> @@ -504,9 +504,9 @@ update:
>  			opt = xchg(&np->opt, opt);
>  			sk_dst_reset(sk);
>  		} else {
> -			write_lock(&sk->sk_dst_lock);
> +			write_lock_bh(&sk->sk_dst_lock);
>  			opt = xchg(&np->opt, opt);
> -			write_unlock(&sk->sk_dst_lock);
> +			write_unlock_bh(&sk->sk_dst_lock);
>  			sk_dst_reset(sk);
>  		}
>  

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Mon, Feb 25, 2008 at 01:05:08PM +0000, Jarek Poplawski wrote:
...
> On Mon, Feb 25, 2008 at 12:19:50PM +0000, James Chapman wrote:
> > Is this an acceptable solution? If so, I'll prepare and send official  
> > patches.
> 
> IMHO this should be acceptable because I can't see any reason for
> changing properly working code if there is so simple and not costly
> solution. But maybe David or somebody else finds some cons? [...]

Hmm... Wait a minute! But on the other hand David has written about
his cons here, and it looks reasonable: this place would be fixed,
but some others can start reports like this. Maybe, it's better to
analyze yet if it's really so hard to eliminate taking this lock
on the xmit path?

Jarek P.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Mon, Feb 25, 2008 at 01:39:48PM +0000, Jarek Poplawski wrote:
...
> Maybe, it's better to
> analyze yet if it's really so hard to eliminate taking this lock
> on the xmit path?

BTW, I'm not sure if it helps, but this matters only for the sockets
which could be used (and locked) outside of pppol2tp code (so before
pppol2tp code is called).

Jarek P.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

Jarek Poplawski wrote, On 02/25/2008 02:39 PM:
...
> Hmm... Wait a minute! But on the other hand David has written about
> his cons here, and it looks reasonable: this place would be fixed,
> but some others can start reports like this. Maybe, it's better to
> analyze yet if it's really so hard to eliminate taking this lock
> on the xmit path?

James, I wonder if you could try to test this patch below?
ip_queue_xmit() seems to do proper things with __sk_dst_check(), and
if some other functions don't behave similarly lockdep should tell.
I think, you could test it with your "locks to _bh" patch (without
pppol2tp_xmit() part), and maybe with my sock.c lockdep patch (it
should help lockdep to see these locks a bit more distinctly).

Jarek P.

PS: Since ppp_generic isn't endangered for now I removed Paul from CC.

---

diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c
index e0b072d..b94659a 100644
--- a/drivers/net/pppol2tp.c
+++ b/drivers/net/pppol2tp.c
@@ -1058,7 +1058,7 @@ static int pppol2tp_xmit(struct ppp_channel *chan, struct sk_buff *skb)
 
 	/* Get routing info from the tunnel socket */
 	dst_release(skb->dst);
-	skb->dst = sk_dst_get(sk_tun);
+	skb->dst = NULL;
 	skb_orphan(skb);
 	skb->sk = sk_tun;
 

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[James Chapman]

Jarek Poplawski wrote:
> Jarek Poplawski wrote, On 02/25/2008 02:39 PM:
> ...
>> Hmm... Wait a minute! But on the other hand David has written about
>> his cons here, and it looks reasonable: this place would be fixed,
>> but some others can start reports like this. Maybe, it's better to
>> analyze yet if it's really so hard to eliminate taking this lock
>> on the xmit path?
> 
> James, I wonder if you could try to test this patch below?
> ip_queue_xmit() seems to do proper things with __sk_dst_check(), and
> if some other functions don't behave similarly lockdep should tell.
> I think, you could test it with your "locks to _bh" patch (without
> pppol2tp_xmit() part), and maybe with my sock.c lockdep patch (it
> should help lockdep to see these locks a bit more distinctly).

I found the same thing and was running a variant of your patch last 
night; rather than set skb->dst to NULL though, I use __sk_dst_get() and 
let ip_queue_xmit() do the route lookup if it returns NULL. But this has 
the same symptoms as the code I tried a few days ago - no lockdep errors 
but a system lockup after up to several hours. Nothing is logged in the 
syslog.

Luckily, I'm in the lab where my two borrowed servers are today so I 
have access to their consoles. Hopefully I'll be able to find out why 
there are hanging. Btw, they don't hang if I disable irqs around the 
ppp_input() call.

Will update you later.

/james

> PS: Since ppp_generic isn't endangered for now I removed Paul from CC.
> 
> ---
> 
> diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c
> index e0b072d..b94659a 100644
> --- a/drivers/net/pppol2tp.c
> +++ b/drivers/net/pppol2tp.c
> @@ -1058,7 +1058,7 @@ static int pppol2tp_xmit(struct ppp_channel *chan, struct sk_buff *skb)
>  
>  	/* Get routing info from the tunnel socket */
>  	dst_release(skb->dst);
> -	skb->dst = sk_dst_get(sk_tun);
> +	skb->dst = NULL;
>  	skb_orphan(skb);
>  	skb->sk = sk_tun;
>  
> --
> To unsubscribe from this list: send the line "unsubscribe netdev" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 


-- 
James Chapman
Katalix Systems Ltd
http://www.katalix.com
Catalysts for your Embedded Linux software development

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Tue, Feb 26, 2008 at 12:14:26PM +0000, James Chapman wrote:
> Jarek Poplawski wrote:
>> Jarek Poplawski wrote, On 02/25/2008 02:39 PM:
>> ...
>>> Hmm... Wait a minute! But on the other hand David has written about
>>> his cons here, and it looks reasonable: this place would be fixed,
>>> but some others can start reports like this. Maybe, it's better to
>>> analyze yet if it's really so hard to eliminate taking this lock
>>> on the xmit path?
>>
>> James, I wonder if you could try to test this patch below?
>> ip_queue_xmit() seems to do proper things with __sk_dst_check(), and
>> if some other functions don't behave similarly lockdep should tell.
>> I think, you could test it with your "locks to _bh" patch (without
>> pppol2tp_xmit() part), and maybe with my sock.c lockdep patch (it
>> should help lockdep to see these locks a bit more distinctly).
>
> I found the same thing and was running a variant of your patch last  
> night; rather than set skb->dst to NULL though, I use __sk_dst_get() and  
> let ip_queue_xmit() do the route lookup if it returns NULL. But this has  
> the same symptoms as the code I tried a few days ago - no lockdep errors  
> but a system lockup after up to several hours. Nothing is logged in the  
> syslog.

I guess you are going to try this together with this sk_dst_lock with
bh patch too. If it's possible I'd suggest to try this skb->dst = NULL
as well (__sk_dst_get instead of __sk_dst_check seems to be too racy).

> Luckily, I'm in the lab where my two borrowed servers are today so I  
> have access to their consoles. Hopefully I'll be able to find out why  
> there are hanging. Btw, they don't hang if I disable irqs around the  
> ppp_input() call.

...and disabling bh instead isn't enough, BTW?

> Will update you later.

Thanks,
Jarek P.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Tue, Feb 26, 2008 at 01:03:34PM +0000, Jarek Poplawski wrote:
> On Tue, Feb 26, 2008 at 12:14:26PM +0000, James Chapman wrote:
...
> > there are hanging. Btw, they don't hang if I disable irqs around the  
> > ppp_input() call.
> 
> ...and disabling bh instead isn't enough, BTW?

I guess not: they are mostly disabled by ppp_input() itself...

So, it looks like a network card could mess here?

Jarek P.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

James Chapman wrote, On 02/26/2008 01:14 PM:
...
> Luckily, I'm in the lab where my two borrowed servers are today so I 
> have access to their consoles. Hopefully I'll be able to find out why 
> there are hanging. Btw, they don't hang if I disable irqs around the 
> ppp_input() call.

Maybe you've found the same, or there is some other reason yet, but
IMHO this locking break around ppp_input() is wrong. Probably there
is needed more advanced solution, but this should fix the problem if
it really exists (isn't there possible a race e.g. between receive
from socket and from network card?).

Jarek P.
---

 drivers/net/pppol2tp.c |   16 ++++++++--------
 1 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c
index e0b072d..7c6fcb9 100644
--- a/drivers/net/pppol2tp.c
+++ b/drivers/net/pppol2tp.c
@@ -363,18 +363,17 @@ out:
 	spin_unlock(&session->reorder_q.lock);
 }
 
-/* Dequeue a single skb.
+/* Requeue a single skb.
  */
-static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct sk_buff *skb)
+static void pppol2tp_recv_requeue_skb(struct pppol2tp_session *session, struct sk_buff *skb)
 {
 	struct pppol2tp_tunnel *tunnel = session->tunnel;
 	int length = PPPOL2TP_SKB_CB(skb)->length;
 	struct sock *session_sock = NULL;
 
-	/* We're about to requeue the skb, so unlink it and return resources
+	/* We're about to requeue the skb, so return resources
 	 * to its current owner (a socket receive buffer).
 	 */
-	skb_unlink(skb, &session->reorder_q);
 	skb_orphan(skb);
 
 	tunnel->stats.rx_packets++;
@@ -436,14 +435,14 @@ static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct s
 static void pppol2tp_recv_dequeue(struct pppol2tp_session *session)
 {
 	struct sk_buff *skb;
-	struct sk_buff *tmp;
 
 	/* If the pkt at the head of the queue has the nr that we
 	 * expect to send up next, dequeue it and any other
 	 * in-sequence packets behind it.
 	 */
+again:
 	spin_lock(&session->reorder_q.lock);
-	skb_queue_walk_safe(&session->reorder_q, skb, tmp) {
+	skb_queue_walk(&session->reorder_q, skb) {
 		if (time_after(jiffies, PPPOL2TP_SKB_CB(skb)->expires)) {
 			session->stats.rx_seq_discards++;
 			session->stats.rx_errors++;
@@ -469,9 +468,10 @@ static void pppol2tp_recv_dequeue(struct pppol2tp_session *session)
 				goto out;
 			}
 		}
+		__skb_unlink(skb, &session->reorder_q);
 		spin_unlock(&session->reorder_q.lock);
-		pppol2tp_recv_dequeue_skb(session, skb);
-		spin_lock(&session->reorder_q.lock);
+		pppol2tp_recv_requeue_skb(session, skb);
+		goto again;
 	}
 
 out:

[PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue()

[Jarek Poplawski]

Hi,

Here is some tiny BTW fix (probably not connected with these lockups).

Regards,
Jarek P.

-------------->

Subject: [PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue()

Every skb removed from session->reorder_q needs sock_put().


Signed-off-by: Jarek Poplawski <jarkao2@gmail.com>

---

 drivers/net/pppol2tp.c |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c
index e0b072d..dcd4991 100644
--- a/drivers/net/pppol2tp.c
+++ b/drivers/net/pppol2tp.c
@@ -455,6 +455,7 @@ static void pppol2tp_recv_dequeue(struct pppol2tp_session *session)
 			       skb_queue_len(&session->reorder_q));
 			__skb_unlink(skb, &session->reorder_q);
 			kfree_skb(skb);
+			sock_put(session->sock);
 			continue;
 		}
 

[PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall()

[Jarek Poplawski]

And here is one more similar BTW fix.

Jarek P.

-------------->

Subject: [PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall()

Every skb removed from session->reorder_q needs sock_put().


Signed-off-by: Jarek Poplawski <jarkao2@gmail.com>

---

 drivers/net/pppol2tp.c |    7 ++++++-
 1 files changed, 6 insertions(+), 1 deletions(-)

diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c
index e0b072d..ee9e2a7 100644
--- a/drivers/net/pppol2tp.c
+++ b/drivers/net/pppol2tp.c
@@ -1110,6 +1110,8 @@ static void pppol2tp_tunnel_closeall(struct pppol2tp_tunnel *tunnel)
 	for (hash = 0; hash < PPPOL2TP_HASH_SIZE; hash++) {
 again:
 		hlist_for_each_safe(walk, tmp, &tunnel->session_hlist[hash]) {
+			struct sk_buff *skb;
+
 			session = hlist_entry(walk, struct pppol2tp_session, hlist);
 
 			sk = session->sock;
@@ -1138,7 +1140,10 @@ again:
 			/* Purge any queued data */
 			skb_queue_purge(&sk->sk_receive_queue);
 			skb_queue_purge(&sk->sk_write_queue);
-			skb_queue_purge(&session->reorder_q);
+			while ((skb = skb_dequeue(&session->reorder_q))) {
+				kfree_skb(skb);
+				sock_put(sk);
+			}
 
 			release_sock(sk);
 			sock_put(sk);

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[James Chapman]

Jarek Poplawski wrote:
> James Chapman wrote, On 02/26/2008 01:14 PM:
> ...
>> Luckily, I'm in the lab where my two borrowed servers are today so I 
>> have access to their consoles. Hopefully I'll be able to find out why 
>> there are hanging. Btw, they don't hang if I disable irqs around the 
>> ppp_input() call.
> 
> Maybe you've found the same, or there is some other reason yet, but
> IMHO this locking break around ppp_input() is wrong. Probably there
> is needed more advanced solution, but this should fix the problem if
> it really exists (isn't there possible a race e.g. between receive
> from socket and from network card?).

I tried your patch but the result is the same.

> 
> Jarek P.
> ---
> 
>  drivers/net/pppol2tp.c |   16 ++++++++--------
>  1 files changed, 8 insertions(+), 8 deletions(-)
> 
> diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c
> index e0b072d..7c6fcb9 100644
> --- a/drivers/net/pppol2tp.c
> +++ b/drivers/net/pppol2tp.c
> @@ -363,18 +363,17 @@ out:
>  	spin_unlock(&session->reorder_q.lock);
>  }
>  
> -/* Dequeue a single skb.
> +/* Requeue a single skb.
>   */
> -static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct sk_buff *skb)
> +static void pppol2tp_recv_requeue_skb(struct pppol2tp_session *session, struct sk_buff *skb)
>  {
>  	struct pppol2tp_tunnel *tunnel = session->tunnel;
>  	int length = PPPOL2TP_SKB_CB(skb)->length;
>  	struct sock *session_sock = NULL;
>  
> -	/* We're about to requeue the skb, so unlink it and return resources
> +	/* We're about to requeue the skb, so return resources
>  	 * to its current owner (a socket receive buffer).
>  	 */
> -	skb_unlink(skb, &session->reorder_q);
>  	skb_orphan(skb);
>  
>  	tunnel->stats.rx_packets++;
> @@ -436,14 +435,14 @@ static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct s
>  static void pppol2tp_recv_dequeue(struct pppol2tp_session *session)
>  {
>  	struct sk_buff *skb;
> -	struct sk_buff *tmp;
>  
>  	/* If the pkt at the head of the queue has the nr that we
>  	 * expect to send up next, dequeue it and any other
>  	 * in-sequence packets behind it.
>  	 */
> +again:
>  	spin_lock(&session->reorder_q.lock);
> -	skb_queue_walk_safe(&session->reorder_q, skb, tmp) {
> +	skb_queue_walk(&session->reorder_q, skb) {

I think this needs the _safe() call because the list may be modified in 
the loop body.

>  		if (time_after(jiffies, PPPOL2TP_SKB_CB(skb)->expires)) {
>  			session->stats.rx_seq_discards++;
>  			session->stats.rx_errors++;
> @@ -469,9 +468,10 @@ static void pppol2tp_recv_dequeue(struct pppol2tp_session *session)
>  				goto out;
>  			}
>  		}
> +		__skb_unlink(skb, &session->reorder_q);
>  		spin_unlock(&session->reorder_q.lock);
> -		pppol2tp_recv_dequeue_skb(session, skb);
> -		spin_lock(&session->reorder_q.lock);
> +		pppol2tp_recv_requeue_skb(session, skb);
> +		goto again;

I'd prefer to take the spinlock again after the recv_dequeue_skb() call 
to avoid the goto.


-- 
James Chapman
Katalix Systems Ltd
http://www.katalix.com
Catalysts for your Embedded Linux software development

Re: [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue()

[James Chapman]

Jarek Poplawski wrote:

> Subject: [PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue()
> 
> Every skb removed from session->reorder_q needs sock_put().
> 
> 
> Signed-off-by: Jarek Poplawski <jarkao2@gmail.com>
Acked-by: James Chapman <jchapman@katalix.com>

> 
> ---
> 
>  drivers/net/pppol2tp.c |    1 +
>  1 files changed, 1 insertions(+), 0 deletions(-)
> 
> diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c
> index e0b072d..dcd4991 100644
> --- a/drivers/net/pppol2tp.c
> +++ b/drivers/net/pppol2tp.c
> @@ -455,6 +455,7 @@ static void pppol2tp_recv_dequeue(struct pppol2tp_session *session)
>  			       skb_queue_len(&session->reorder_q));
>  			__skb_unlink(skb, &session->reorder_q);
>  			kfree_skb(skb);
> +			sock_put(session->sock);
>  			continue;
>  		}
>  


-- 
James Chapman
Katalix Systems Ltd
http://www.katalix.com
Catalysts for your Embedded Linux software development

Re: [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall()

[James Chapman]

Jarek Poplawski wrote:
> 
> Subject: [PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall()
> 
> Every skb removed from session->reorder_q needs sock_put().
> 
> 
> Signed-off-by: Jarek Poplawski <jarkao2@gmail.com>

Acked-by: James Chapman <jchapman@katalix.com>

>  drivers/net/pppol2tp.c |    7 ++++++-
>  1 files changed, 6 insertions(+), 1 deletions(-)
> 
> diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c
> index e0b072d..ee9e2a7 100644
> --- a/drivers/net/pppol2tp.c
> +++ b/drivers/net/pppol2tp.c
> @@ -1110,6 +1110,8 @@ static void pppol2tp_tunnel_closeall(struct pppol2tp_tunnel *tunnel)
>  	for (hash = 0; hash < PPPOL2TP_HASH_SIZE; hash++) {
>  again:
>  		hlist_for_each_safe(walk, tmp, &tunnel->session_hlist[hash]) {
> +			struct sk_buff *skb;
> +
>  			session = hlist_entry(walk, struct pppol2tp_session, hlist);
>  
>  			sk = session->sock;
> @@ -1138,7 +1140,10 @@ again:
>  			/* Purge any queued data */
>  			skb_queue_purge(&sk->sk_receive_queue);
>  			skb_queue_purge(&sk->sk_write_queue);
> -			skb_queue_purge(&session->reorder_q);
> +			while ((skb = skb_dequeue(&session->reorder_q))) {
> +				kfree_skb(skb);
> +				sock_put(sk);
> +			}
>  
>  			release_sock(sk);
>  			sock_put(sk);


-- 
James Chapman
Katalix Systems Ltd
http://www.katalix.com
Catalysts for your Embedded Linux software development

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Sun, Mar 02, 2008 at 08:29:58PM +0000, James Chapman wrote:
> Jarek Poplawski wrote:
>> James Chapman wrote, On 02/26/2008 01:14 PM:
...
>
> I tried your patch but the result is the same.
>
>>
>> Jarek P.
>> ---
>>
>>  drivers/net/pppol2tp.c |   16 ++++++++--------
>>  1 files changed, 8 insertions(+), 8 deletions(-)
>>
>> diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c
>> index e0b072d..7c6fcb9 100644
>> --- a/drivers/net/pppol2tp.c
>> +++ b/drivers/net/pppol2tp.c
>> @@ -363,18 +363,17 @@ out:
>>  	spin_unlock(&session->reorder_q.lock);
>>  }
>>  -/* Dequeue a single skb.
>> +/* Requeue a single skb.
>>   */
>> -static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct sk_buff *skb)
>> +static void pppol2tp_recv_requeue_skb(struct pppol2tp_session *session, struct sk_buff *skb)
>>  {
>>  	struct pppol2tp_tunnel *tunnel = session->tunnel;
>>  	int length = PPPOL2TP_SKB_CB(skb)->length;
>>  	struct sock *session_sock = NULL;
>>  -	/* We're about to requeue the skb, so unlink it and return resources
>> +	/* We're about to requeue the skb, so return resources
>>  	 * to its current owner (a socket receive buffer).
>>  	 */
>> -	skb_unlink(skb, &session->reorder_q);
>>  	skb_orphan(skb);
>>   	tunnel->stats.rx_packets++;
>> @@ -436,14 +435,14 @@ static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct s
>>  static void pppol2tp_recv_dequeue(struct pppol2tp_session *session)
>>  {
>>  	struct sk_buff *skb;
>> -	struct sk_buff *tmp;
>>   	/* If the pkt at the head of the queue has the nr that we
>>  	 * expect to send up next, dequeue it and any other
>>  	 * in-sequence packets behind it.
>>  	 */
>> +again:
>>  	spin_lock(&session->reorder_q.lock);
>> -	skb_queue_walk_safe(&session->reorder_q, skb, tmp) {
>> +	skb_queue_walk(&session->reorder_q, skb) {
>
> I think this needs the _safe() call because the list may be modified in  
> the loop body.

And you are very right.

>
>>  		if (time_after(jiffies, PPPOL2TP_SKB_CB(skb)->expires)) {
>>  			session->stats.rx_seq_discards++;
>>  			session->stats.rx_errors++;
>> @@ -469,9 +468,10 @@ static void pppol2tp_recv_dequeue(struct pppol2tp_session *session)
>>  				goto out;
>>  			}
>>  		}
>> +		__skb_unlink(skb, &session->reorder_q);
>>  		spin_unlock(&session->reorder_q.lock);
>> -		pppol2tp_recv_dequeue_skb(session, skb);
>> -		spin_lock(&session->reorder_q.lock);
>> +		pppol2tp_recv_requeue_skb(session, skb);
>> +		goto again;
>
> I'd prefer to take the spinlock again after the recv_dequeue_skb() call  
> to avoid the goto.

IMHO we have to do something like goto here because during the unlock
there could be everything changed (including tmp) in this list, but
maybe I miss something.

Anyway, this place seems to need more locking, e.g.: session->nr++ or
some other session accesses in pppol2tp_recv_dequeue_skb() are done
without locking. Maybe it would be interesting to try (in a test only)
with calling this with reorder_q.lock held (so without unlocking
inside this pppol2tp_recv_dequeue's loop at all: at least to see
possible lockdep report. I can't see how this irq disabling could
help for anything but some recursion during those misterious lockups.

BTW, since this bug is so hard to trigger, and it can take some time
yet, I think it would be better to merge to David's 2.6.26 tree at
least some parts of your "bh" patch and maybe other changes, which
don't seem to make it worse at least, to try new things on some common
base?

Regards,
Jarek P.

Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver

[Jarek Poplawski]

On Mon, Mar 03, 2008 at 08:22:04AM +0000, Jarek Poplawski wrote:
...
> Maybe it would be interesting to try (in a test only)
> with calling this with reorder_q.lock held (so without unlocking
> inside this pppol2tp_recv_dequeue's loop at all: at least to see
> possible lockdep report.

...But we can't forget to change this skb_unlink() in
pppol2tp_recv_dequeue_skb() then...

Jarek P.

Re: [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue()

[David Miller]

From: James Chapman <jchapman@katalix.com>
Date: Sun, 02 Mar 2008 20:31:17 +0000

> Jarek Poplawski wrote:
> 
> > Subject: [PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue()
> > 
> > Every skb removed from session->reorder_q needs sock_put().
> > 
> > 
> > Signed-off-by: Jarek Poplawski <jarkao2@gmail.com>
> Acked-by: James Chapman <jchapman@katalix.com>

Patch applied.

Re: [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall()

[David Miller]

From: James Chapman <jchapman@katalix.com>
Date: Sun, 02 Mar 2008 20:32:41 +0000

> Jarek Poplawski wrote:
> > 
> > Subject: [PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall()
> > 
> > Every skb removed from session->reorder_q needs sock_put().
> > 
> > 
> > Signed-off-by: Jarek Poplawski <jarkao2@gmail.com>
> 
> Acked-by: James Chapman <jchapman@katalix.com>

Also applied, thanks everyone.