* [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver
@ 2008-02-11 9:22 James Chapman
2008-02-11 18:57 ` Jarek Poplawski
0 siblings, 1 reply; 51+ messages in thread
From: James Chapman @ 2008-02-11 9:22 UTC (permalink / raw)
To: netdev
Fix locking issues in the pppol2tp driver which can cause a kernel
crash on SMP boxes when hundreds of L2TP sessions are created/deleted
simultaneously (ISP environment). The driver was violating read_lock()
and write_lock() scheduling rules so we now consistently use the _irq
variants of the lock functions.
Signed-off-by: James Chapman <jchapman@katalix.com>
--
This patch has been verified by the ISP that discovered the problem.
If the patch is accepted, it should be pushed to the stable 2.6.23 and
2.6.24 trees.
Index: linux-2.6.24/drivers/net/pppol2tp.c
===================================================================
--- linux-2.6.24.orig/drivers/net/pppol2tp.c
+++ linux-2.6.24/drivers/net/pppol2tp.c
@@ -301,15 +301,16 @@ pppol2tp_session_find(struct pppol2tp_tu
pppol2tp_session_id_hash(tunnel, session_id);
struct pppol2tp_session *session;
struct hlist_node *walk;
+ unsigned long flags;
- read_lock(&tunnel->hlist_lock);
+ read_lock_irqsave(&tunnel->hlist_lock, flags);
hlist_for_each_entry(session, walk, session_list, hlist) {
if (session->tunnel_addr.s_session == session_id) {
- read_unlock(&tunnel->hlist_lock);
+ read_unlock_irqrestore(&tunnel->hlist_lock, flags);
return session;
}
}
- read_unlock(&tunnel->hlist_lock);
+ read_unlock_irqrestore(&tunnel->hlist_lock, flags);
return NULL;
}
@@ -319,15 +320,16 @@ pppol2tp_session_find(struct pppol2tp_tu
static struct pppol2tp_tunnel *pppol2tp_tunnel_find(u16 tunnel_id)
{
struct pppol2tp_tunnel *tunnel = NULL;
+ unsigned long flags;
- read_lock(&pppol2tp_tunnel_list_lock);
+ read_lock_irqsave(&pppol2tp_tunnel_list_lock, flags);
list_for_each_entry(tunnel, &pppol2tp_tunnel_list, list) {
if (tunnel->stats.tunnel_id == tunnel_id) {
- read_unlock(&pppol2tp_tunnel_list_lock);
+ read_unlock_irqrestore(&pppol2tp_tunnel_list_lock, flags);
return tunnel;
}
}
- read_unlock(&pppol2tp_tunnel_list_lock);
+ read_unlock_irqrestore(&pppol2tp_tunnel_list_lock, flags);
return NULL;
}
@@ -1099,6 +1101,7 @@ static void pppol2tp_tunnel_closeall(str
struct hlist_node *tmp;
struct pppol2tp_session *session;
struct sock *sk;
+ unsigned long flags;
if (tunnel == NULL)
BUG();
@@ -1106,7 +1109,7 @@ static void pppol2tp_tunnel_closeall(str
PRINTK(tunnel->debug, PPPOL2TP_MSG_CONTROL, KERN_INFO,
"%s: closing all sessions...\n", tunnel->name);
- write_lock(&tunnel->hlist_lock);
+ write_lock_irqsave(&tunnel->hlist_lock, flags);
for (hash = 0; hash < PPPOL2TP_HASH_SIZE; hash++) {
again:
hlist_for_each_safe(walk, tmp, &tunnel->session_hlist[hash]) {
@@ -1126,7 +1129,7 @@ again:
* disappear as we're jumping between locks.
*/
sock_hold(sk);
- write_unlock(&tunnel->hlist_lock);
+ write_unlock_irqrestore(&tunnel->hlist_lock, flags);
lock_sock(sk);
if (sk->sk_state & (PPPOX_CONNECTED | PPPOX_BOUND)) {
@@ -1148,11 +1151,11 @@ again:
* list so we are guaranteed to make forward
* progress.
*/
- write_lock(&tunnel->hlist_lock);
+ write_lock_irqsave(&tunnel->hlist_lock, flags);
goto again;
}
}
- write_unlock(&tunnel->hlist_lock);
+ write_unlock_irqrestore(&tunnel->hlist_lock, flags);
}
/* Really kill the tunnel.
@@ -1160,10 +1163,12 @@ again:
*/
static void pppol2tp_tunnel_free(struct pppol2tp_tunnel *tunnel)
{
+ unsigned long flags;
+
/* Remove from socket list */
- write_lock(&pppol2tp_tunnel_list_lock);
+ write_lock_irqsave(&pppol2tp_tunnel_list_lock, flags);
list_del_init(&tunnel->list);
- write_unlock(&pppol2tp_tunnel_list_lock);
+ write_unlock_irqrestore(&pppol2tp_tunnel_list_lock, flags);
atomic_dec(&pppol2tp_tunnel_count);
kfree(tunnel);
@@ -1212,6 +1217,7 @@ end:
static void pppol2tp_session_destruct(struct sock *sk)
{
struct pppol2tp_session *session = NULL;
+ unsigned long flags;
if (sk->sk_user_data != NULL) {
struct pppol2tp_tunnel *tunnel;
@@ -1239,9 +1245,9 @@ static void pppol2tp_session_destruct(st
/* Delete the session socket from the
* hash
*/
- write_lock(&tunnel->hlist_lock);
+ write_lock_irqsave(&tunnel->hlist_lock, flags);
hlist_del_init(&session->hlist);
- write_unlock(&tunnel->hlist_lock);
+ write_unlock_irqrestore(&tunnel->hlist_lock, flags);
atomic_dec(&pppol2tp_session_count);
}
@@ -1312,6 +1318,7 @@ static struct sock *pppol2tp_prepare_tun
struct sock *sk;
struct pppol2tp_tunnel *tunnel;
struct sock *ret = NULL;
+ unsigned long flags;
/* Get the tunnel UDP socket from the fd, which was opened by
* the userspace L2TP daemon.
@@ -1386,9 +1393,9 @@ static struct sock *pppol2tp_prepare_tun
/* Add tunnel to our list */
INIT_LIST_HEAD(&tunnel->list);
- write_lock(&pppol2tp_tunnel_list_lock);
+ write_lock_irqsave(&pppol2tp_tunnel_list_lock, flags);
list_add(&tunnel->list, &pppol2tp_tunnel_list);
- write_unlock(&pppol2tp_tunnel_list_lock);
+ write_unlock_irqrestore(&pppol2tp_tunnel_list_lock, flags);
atomic_inc(&pppol2tp_tunnel_count);
/* Bump the reference count. The tunnel context is deleted
@@ -1462,6 +1469,7 @@ static int pppol2tp_connect(struct socke
struct pppol2tp_tunnel *tunnel;
struct dst_entry *dst;
int error = 0;
+ unsigned long irqflags;
lock_sock(sk);
@@ -1593,11 +1601,11 @@ static int pppol2tp_connect(struct socke
sk->sk_user_data = session;
/* Add session to the tunnel's hash list */
- write_lock(&tunnel->hlist_lock);
+ write_lock_irqsave(&tunnel->hlist_lock, irqflags);
hlist_add_head(&session->hlist,
pppol2tp_session_id_hash(tunnel,
session->tunnel_addr.s_session));
- write_unlock(&tunnel->hlist_lock);
+ write_unlock_irqrestore(&tunnel->hlist_lock, irqflags);
atomic_inc(&pppol2tp_session_count);
@@ -2198,8 +2206,9 @@ static struct pppol2tp_session *next_ses
int found = 0;
int next = 0;
int i;
+ unsigned long flags;
- read_lock(&tunnel->hlist_lock);
+ read_lock_irqsave(&tunnel->hlist_lock, flags);
for (i = 0; i < PPPOL2TP_HASH_SIZE; i++) {
hlist_for_each_entry(session, walk, &tunnel->session_hlist[i], hlist) {
if (curr == NULL) {
@@ -2217,7 +2226,7 @@ static struct pppol2tp_session *next_ses
}
}
out:
- read_unlock(&tunnel->hlist_lock);
+ read_unlock_irqrestore(&tunnel->hlist_lock, flags);
if (!found)
session = NULL;
@@ -2227,14 +2236,15 @@ out:
static struct pppol2tp_tunnel *next_tunnel(struct pppol2tp_tunnel *curr)
{
struct pppol2tp_tunnel *tunnel = NULL;
+ unsigned long flags;
- read_lock(&pppol2tp_tunnel_list_lock);
+ read_lock_irqsave(&pppol2tp_tunnel_list_lock, flags);
if (list_is_last(&curr->list, &pppol2tp_tunnel_list)) {
goto out;
}
tunnel = list_entry(curr->list.next, struct pppol2tp_tunnel, list);
out:
- read_unlock(&pppol2tp_tunnel_list_lock);
+ read_unlock_irqrestore(&pppol2tp_tunnel_list_lock, flags);
return tunnel;
}
^ permalink raw reply [flat|nested] 51+ messages in thread* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-11 9:22 [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver James Chapman @ 2008-02-11 18:57 ` Jarek Poplawski 2008-02-11 22:19 ` James Chapman 0 siblings, 1 reply; 51+ messages in thread From: Jarek Poplawski @ 2008-02-11 18:57 UTC (permalink / raw) To: James Chapman; +Cc: netdev James Chapman wrote, On 02/11/2008 10:22 AM: > Fix locking issues in the pppol2tp driver which can cause a kernel > crash on SMP boxes when hundreds of L2TP sessions are created/deleted > simultaneously (ISP environment). The driver was violating read_lock() > and write_lock() scheduling rules so we now consistently use the _irq > variants of the lock functions. ... Hi, Could you explain what exactly scheduling rules do you mean here, and why disabling interrupts is the best solution for this? Thanks, Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-11 18:57 ` Jarek Poplawski @ 2008-02-11 22:19 ` James Chapman 2008-02-11 22:49 ` Jarek Poplawski 0 siblings, 1 reply; 51+ messages in thread From: James Chapman @ 2008-02-11 22:19 UTC (permalink / raw) To: Jarek Poplawski; +Cc: netdev Jarek Poplawski wrote: > James Chapman wrote, On 02/11/2008 10:22 AM: > >> Fix locking issues in the pppol2tp driver which can cause a kernel >> crash on SMP boxes when hundreds of L2TP sessions are created/deleted >> simultaneously (ISP environment). The driver was violating read_lock() >> and write_lock() scheduling rules so we now consistently use the _irq >> variants of the lock functions. > ... > > Hi, > > Could you explain what exactly scheduling rules do you mean here, > and why disabling interrupts is the best solution for this? Below is example output from lockdep. The oops is reproducible when creating/deleting lots of sessions while passing data. The lock is being acquired for read and write in softirq contexts. Is there a better way to fix this? ================================= [ INFO: inconsistent lock state ] 2.6.24-core2 #1 --------------------------------- inconsistent {in-softirq-R} -> {softirq-on-W} usage. openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes: (&tunnel->hlist_lock){---?}, at: [<f8eea157>] pppol2tp_connect+0x517/0x6d0 [pppol2tp] {in-softirq-R} state was registered at: [<c014edaf>] __lock_acquire+0x6bf/0x10a0 [<c03ee75b>] fn_hash_lookup+0x1b/0xe0 [<c014f804>] lock_acquire+0x74/0xa0 [<f8ee859f>] pppol2tp_session_find+0x1f/0x80 [pppol2tp] [<c040427a>] _read_lock+0x2a/0x40 [<f8ee859f>] pppol2tp_session_find+0x1f/0x80 [pppol2tp] [<f8ee859f>] pppol2tp_session_find+0x1f/0x80 [pppol2tp] [<f8ee8dc8>] pppol2tp_recv_core+0xd8/0x960 [pppol2tp] [<f8d3f72a>] ipt_do_table+0x23a/0x500 [ip_tables] [<f8ee967e>] pppol2tp_udp_encap_recv+0x2e/0x70 [pppol2tp] [<c0403fb4>] _read_unlock+0x14/0x20 [<c03dd696>] udp_queue_rcv_skb+0x106/0x2a0 [<c03ddc5a>] __udp4_lib_rcv+0x42a/0x7e0 [<f8d57090>] ipt_hook+0x0/0x20 [iptable_filter] [<c03bc2da>] ip_local_deliver_finish+0xca/0x1c0 [<c03bc23e>] ip_local_deliver_finish+0x2e/0x1c0 [<c03bbfaf>] ip_rcv_finish+0xff/0x360 [<c03bc6dc>] ip_rcv+0x20c/0x2a0 [<c03bbeb0>] ip_rcv_finish+0x0/0x360 [<c039ad87>] netif_receive_skb+0x317/0x4b0 [<c039ab70>] netif_receive_skb+0x100/0x4b0 [<f8d9627a>] e1000_clean_rx_irq_ps+0x28a/0x560 [e1000] [<f8d95ff0>] e1000_clean_rx_irq_ps+0x0/0x560 [e1000] [<f8d9384d>] e1000_clean+0x5d/0x290 [e1000] [<c039d580>] net_rx_action+0x1a0/0x2a0 [<c039d43f>] net_rx_action+0x5f/0x2a0 [<c0131e72>] __do_softirq+0x92/0x120 [<c0131f78>] do_softirq+0x78/0x80 [<c010b15a>] do_IRQ+0x4a/0xa0 [<c0108dcc>] common_interrupt+0x24/0x34 [<c0108dd6>] common_interrupt+0x2e/0x34 [<c01062d6>] mwait_idle_with_hints+0x46/0x60 [<c0106550>] mwait_idle+0x0/0x20 [<c0106694>] cpu_idle+0x74/0xe0 [<c0536a9a>] start_kernel+0x30a/0x3a0 [<c0536150>] unknown_bootoption+0x0/0x1f0 [<ffffffff>] 0xffffffff irq event stamp: 275 hardirqs last enabled at (275): [<c0132317>] local_bh_enable_ip+0xa7/0x120 hardirqs last disabled at (273): [<c01322a6>] local_bh_enable_ip+0x36/0x120 softirqs last enabled at (274): [<f8eab8bc>] ppp_register_channel+0xdc/0xf0 [ppp_generic] softirqs last disabled at (272): [<c040410b>] _spin_lock_bh+0xb/0x40 -- James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-11 22:19 ` James Chapman @ 2008-02-11 22:49 ` Jarek Poplawski 2008-02-11 22:55 ` Jarek Poplawski 2008-02-11 23:41 ` James Chapman 0 siblings, 2 replies; 51+ messages in thread From: Jarek Poplawski @ 2008-02-11 22:49 UTC (permalink / raw) To: James Chapman; +Cc: netdev On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote: ... > Below is example output from lockdep. The oops is reproducible when > creating/deleting lots of sessions while passing data. The lock is being > acquired for read and write in softirq contexts. > > Is there a better way to fix this? > > ================================= > [ INFO: inconsistent lock state ] > 2.6.24-core2 #1 > --------------------------------- > inconsistent {in-softirq-R} -> {softirq-on-W} usage. > openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes: > (&tunnel->hlist_lock){---?}, at: [<f8eea157>] > pppol2tp_connect+0x517/0x6d0 [pppol2tp] > {in-softirq-R} state was registered at: IMHO, according to this, disabling bh should be enough. And if it's like in this report: only read_lock is taken from softirqs, then this should be necessary to change only all write_locks to write_lock_bh (of course unless somewhere bhs are disabled already). Unless I miss something?! Cheers, Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-11 22:49 ` Jarek Poplawski @ 2008-02-11 22:55 ` Jarek Poplawski 2008-02-11 23:42 ` James Chapman 2008-02-11 23:41 ` James Chapman 1 sibling, 1 reply; 51+ messages in thread From: Jarek Poplawski @ 2008-02-11 22:55 UTC (permalink / raw) To: James Chapman; +Cc: netdev On Mon, Feb 11, 2008 at 11:49:24PM +0100, Jarek Poplawski wrote: > On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote: > ... > > Below is example output from lockdep. The oops is reproducible when > > creating/deleting lots of sessions while passing data. The lock is being > > acquired for read and write in softirq contexts. ...Hmmm... And according to this, changing read_locks should be necessary too. Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-11 22:55 ` Jarek Poplawski @ 2008-02-11 23:42 ` James Chapman 2008-02-12 10:42 ` Jarek Poplawski 0 siblings, 1 reply; 51+ messages in thread From: James Chapman @ 2008-02-11 23:42 UTC (permalink / raw) To: Jarek Poplawski; +Cc: netdev Jarek Poplawski wrote: > On Mon, Feb 11, 2008 at 11:49:24PM +0100, Jarek Poplawski wrote: >> On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote: >> ... >>> Below is example output from lockdep. The oops is reproducible when >>> creating/deleting lots of sessions while passing data. The lock is being >>> acquired for read and write in softirq contexts. > > ...Hmmm... And according to this, changing read_locks should be > necessary too. The patch changes both read and write locks. -- James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-11 23:42 ` James Chapman @ 2008-02-12 10:42 ` Jarek Poplawski 0 siblings, 0 replies; 51+ messages in thread From: Jarek Poplawski @ 2008-02-12 10:42 UTC (permalink / raw) To: James Chapman; +Cc: netdev On Mon, Feb 11, 2008 at 11:42:12PM +0000, James Chapman wrote: > Jarek Poplawski wrote: >> On Mon, Feb 11, 2008 at 11:49:24PM +0100, Jarek Poplawski wrote: >>> On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote: >>> ... >>>> Below is example output from lockdep. The oops is reproducible when >>>> creating/deleting lots of sessions while passing data. The lock is >>>> being acquired for read and write in softirq contexts. >> >> ...Hmmm... And according to this, changing read_locks should be >> necessary too. > > The patch changes both read and write locks. Right! This was only "errata" to my earlier comment where I considered only lockdep info and forgot about yours... Sorry, Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-11 22:49 ` Jarek Poplawski 2008-02-11 22:55 ` Jarek Poplawski @ 2008-02-11 23:41 ` James Chapman 2008-02-12 5:30 ` David Miller 2008-02-12 7:19 ` [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver Jarek Poplawski 1 sibling, 2 replies; 51+ messages in thread From: James Chapman @ 2008-02-11 23:41 UTC (permalink / raw) To: Jarek Poplawski; +Cc: netdev Jarek Poplawski wrote: > On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote: > ... >> Below is example output from lockdep. The oops is reproducible when >> creating/deleting lots of sessions while passing data. The lock is being >> acquired for read and write in softirq contexts. >> >> Is there a better way to fix this? >> >> ================================= >> [ INFO: inconsistent lock state ] >> 2.6.24-core2 #1 >> --------------------------------- >> inconsistent {in-softirq-R} -> {softirq-on-W} usage. >> openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes: >> (&tunnel->hlist_lock){---?}, at: [<f8eea157>] >> pppol2tp_connect+0x517/0x6d0 [pppol2tp] >> {in-softirq-R} state was registered at: > > IMHO, according to this, disabling bh should be enough. And if it's > like in this report: only read_lock is taken from softirqs, then this > should be necessary to change only all write_locks to write_lock_bh > (of course unless somewhere bhs are disabled already). Unless I miss > something?! I thought so too. I tried _bh locks first and the problem still occurred. Maybe I'll try it again in case I messed something up. -- James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-11 23:41 ` James Chapman @ 2008-02-12 5:30 ` David Miller 2008-02-12 10:58 ` James Chapman 2008-02-12 7:19 ` [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver Jarek Poplawski 1 sibling, 1 reply; 51+ messages in thread From: David Miller @ 2008-02-12 5:30 UTC (permalink / raw) To: jchapman; +Cc: jarkao2, netdev From: James Chapman <jchapman@katalix.com> Date: Mon, 11 Feb 2008 23:41:18 +0000 > Jarek Poplawski wrote: > > On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote: > > ... > >> Below is example output from lockdep. The oops is reproducible when > >> creating/deleting lots of sessions while passing data. The lock is being > >> acquired for read and write in softirq contexts. > >> > >> Is there a better way to fix this? > >> > >> ================================= > >> [ INFO: inconsistent lock state ] > >> 2.6.24-core2 #1 > >> --------------------------------- > >> inconsistent {in-softirq-R} -> {softirq-on-W} usage. > >> openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes: > >> (&tunnel->hlist_lock){---?}, at: [<f8eea157>] > >> pppol2tp_connect+0x517/0x6d0 [pppol2tp] > >> {in-softirq-R} state was registered at: > > > > IMHO, according to this, disabling bh should be enough. And if it's > > like in this report: only read_lock is taken from softirqs, then this > > should be necessary to change only all write_locks to write_lock_bh > > (of course unless somewhere bhs are disabled already). Unless I miss > > something?! > > I thought so too. I tried _bh locks first and the problem still > occurred. Maybe I'll try it again in case I messed something up. I agree with Jarek here, I look at all the code paths that take ->hlist_lock and all of them are in user context or software interrupts. Please get a lockdep trace with the change to _bh intead of hw interrupt protection so we can find out what that doesn't work. Thanks! ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-12 5:30 ` David Miller @ 2008-02-12 10:58 ` James Chapman 2008-02-12 13:24 ` Jarek Poplawski ` (2 more replies) 0 siblings, 3 replies; 51+ messages in thread From: James Chapman @ 2008-02-12 10:58 UTC (permalink / raw) To: David Miller, jarkao2; +Cc: netdev David Miller wrote: > From: James Chapman <jchapman@katalix.com> > Date: Mon, 11 Feb 2008 23:41:18 +0000 > >> Jarek Poplawski wrote: >>> On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote: >>> ... >>>> Below is example output from lockdep. The oops is reproducible when >>>> creating/deleting lots of sessions while passing data. The lock is being >>>> acquired for read and write in softirq contexts. >>>> >>>> Is there a better way to fix this? >>>> >>>> ================================= >>>> [ INFO: inconsistent lock state ] >>>> 2.6.24-core2 #1 >>>> --------------------------------- >>>> inconsistent {in-softirq-R} -> {softirq-on-W} usage. >>>> openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes: >>>> (&tunnel->hlist_lock){---?}, at: [<f8eea157>] >>>> pppol2tp_connect+0x517/0x6d0 [pppol2tp] >>>> {in-softirq-R} state was registered at: >>> IMHO, according to this, disabling bh should be enough. And if it's >>> like in this report: only read_lock is taken from softirqs, then this >>> should be necessary to change only all write_locks to write_lock_bh >>> (of course unless somewhere bhs are disabled already). Unless I miss >>> something?! >> I thought so too. I tried _bh locks first and the problem still >> occurred. Maybe I'll try it again in case I messed something up. > > I agree with Jarek here, I look at all the code paths that take > ->hlist_lock and all of them are in user context or software > interrupts. > > Please get a lockdep trace with the change to _bh intead of > hw interrupt protection so we can find out what that doesn't > work. > > Thanks! Here is a trace from when we had _bh locks. Feb 5 16:26:32 ====================================================== Feb 5 16:26:32 [ INFO: soft-safe -> soft-unsafe lock order detected ] Feb 5 16:26:32 2.6.24-core2 #1 Feb 5 16:26:32 ------------------------------------------------------ Feb 5 16:26:32 pppd/3224 [HC0[0]:SC0[2]:HE1:SE0] is trying to acquire: Feb 5 16:26:32 (&sk->sk_dst_lock){----}, at: [<f8efacac>] pppol2tp_xmit+0x23c/0x460 [pppol2tp] Feb 5 16:26:32 Feb 5 16:26:32 and this task is already holding: Feb 5 16:26:32 (&pch->downl){-...}, at: [<f8eb828e>] ppp_push+0x44e/0x620 [ppp_generic] Feb 5 16:26:32 which would create a new lock dependency: Feb 5 16:26:32 (&pch->downl){-...} -> (&sk->sk_dst_lock){----} Feb 5 16:26:32 Feb 5 16:26:32 but this new dependency connects a soft-irq-safe lock: Feb 5 16:26:32 (&pch->upl){-.-+} Feb 5 16:26:32 ... which became soft-irq-safe at: Feb 5 16:26:32 [<c014d87f>] check_usage_backwards+0x1f/0x50 Feb 5 16:26:32 [<c014c479>] save_trace+0x39/0xa0 Feb 5 16:26:32 [<c014edaf>] __lock_acquire+0x6bf/0x10a0 Feb 5 16:26:32 [<c014ee8e>] __lock_acquire+0x79e/0x10a0 Feb 5 16:26:32 [<c014ee8e>] __lock_acquire+0x79e/0x10a0 Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0 Feb 5 16:26:32 [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic] Feb 5 16:26:32 [<c040425f>] _read_lock_bh+0x2f/0x40 Feb 5 16:26:32 [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic] Feb 5 16:26:32 [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic] Feb 5 16:26:32 [<f8ef915a>] pppol2tp_recv_core+0x46a/0x960 [pppol2tp] Feb 5 16:26:32 [<f8ef967e>] pppol2tp_udp_encap_recv+0x2e/0x70 [pppol2tp] Feb 5 16:26:32 [<c0403fd4>] _read_unlock+0x14/0x20 Feb 5 16:26:32 [<c03dd6b6>] udp_queue_rcv_skb+0x106/0x2a0 Feb 5 16:26:32 [<c03ddc7a>] __udp4_lib_rcv+0x42a/0x7e0 Feb 5 16:26:32 [<f8d5c090>] ipt_hook+0x0/0x20 [iptable_filter] Feb 5 16:26:32 [<c03bc2fa>] ip_local_deliver_finish+0xca/0x1c0 Feb 5 16:26:32 [<c03bc25e>] ip_local_deliver_finish+0x2e/0x1c0 Feb 5 16:26:32 [<c03bbfcf>] ip_rcv_finish+0xff/0x360 Feb 5 16:26:32 [<c03bc6fc>] ip_rcv+0x20c/0x2a0 Feb 5 16:26:32 [<c03bbed0>] ip_rcv_finish+0x0/0x360 Feb 5 16:26:32 [<c039ad87>] netif_receive_skb+0x317/0x4b0 Feb 5 16:26:32 [<c039ab70>] netif_receive_skb+0x100/0x4b0 Feb 5 16:26:32 [<f8d7e27a>] e1000_clean_rx_irq_ps+0x28a/0x560 [e1000] Feb 5 16:26:32 [<f8d7dff0>] e1000_clean_rx_irq_ps+0x0/0x560 [e1000] Feb 5 16:26:32 [<f8d7b84d>] e1000_clean+0x5d/0x290 [e1000] Feb 5 16:26:32 [<c039d580>] net_rx_action+0x1a0/0x2a0 Feb 5 16:26:32 [<c039d43f>] net_rx_action+0x5f/0x2a0 Feb 5 16:26:32 [<c0131e72>] __do_softirq+0x92/0x120 Feb 5 16:26:32 [<c0131f78>] do_softirq+0x78/0x80 Feb 5 16:26:32 [<c010b15a>] do_IRQ+0x4a/0xa0 Feb 5 16:26:32 [<c0127af0>] finish_task_switch+0x0/0xc0 Feb 5 16:26:32 [<c0108dcc>] common_interrupt+0x24/0x34 Feb 5 16:26:32 [<c0108dd6>] common_interrupt+0x2e/0x34 Feb 5 16:26:32 [<c01062d6>] mwait_idle_with_hints+0x46/0x60 Feb 5 16:26:32 [<c0106550>] mwait_idle+0x0/0x20 Feb 5 16:26:32 [<c0106694>] cpu_idle+0x74/0xe0 Feb 5 16:26:32 [<c0536a9a>] start_kernel+0x30a/0x3a0 Feb 5 16:26:32 [<c0536150>] unknown_bootoption+0x0/0x1f0 Feb 5 16:26:32 [<ffffffff>] 0xffffffff Feb 5 16:26:32 Feb 5 16:26:32 to a soft-irq-unsafe lock: Feb 5 16:26:32 (&sk->sk_dst_lock){----} Feb 5 16:26:32 ... which became soft-irq-unsafe at: Feb 5 16:26:32 ... [<c014e02e>] mark_held_locks+0x5e/0x80 Feb 5 16:26:32 [<c014ed92>] __lock_acquire+0x6a2/0x10a0 Feb 5 16:26:32 [<c010f5b0>] save_stack_trace+0x20/0x40 Feb 5 16:26:32 [<c014c524>] add_lock_to_list+0x44/0xb0 Feb 5 16:26:32 [<c03dea29>] __udp_lib_get_port+0x19/0x200 Feb 5 16:26:32 [<c014f735>] __lock_acquire+0x1045/0x10a0 Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0 Feb 5 16:26:32 [<c03db0a3>] ip4_datagram_connect+0x53/0x380 Feb 5 16:26:32 [<c040418a>] _write_lock+0x2a/0x40 Feb 5 16:26:32 [<c03db0a3>] ip4_datagram_connect+0x53/0x380 Feb 5 16:26:32 [<c03db0a3>] ip4_datagram_connect+0x53/0x380 Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170 Feb 5 16:26:32 [<c0132317>] local_bh_enable_ip+0xa7/0x120 Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170 Feb 5 16:26:32 [<c040414f>] _spin_lock_bh+0x2f/0x40 Feb 5 16:26:32 [<c03e4d55>] inet_dgram_connect+0x35/0x80 Feb 5 16:26:32 [<c038ec52>] sys_connect+0x82/0xd0 Feb 5 16:26:32 [<c01455df>] down_read_trylock+0x4f/0x60 Feb 5 16:26:32 [<c011fe9c>] do_page_fault+0xfc/0x940 Feb 5 16:26:32 [<c0404024>] _spin_unlock+0x14/0x20 Feb 5 16:26:32 [<c03905f8>] sys_socketcall+0x98/0x280 Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170 Feb 5 16:26:32 [<c02a86ba>] copy_to_user+0x3a/0x70 Feb 5 16:26:32 [<c0108417>] restore_nocheck+0x12/0x15 Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb Feb 5 16:26:32 [<ffffffff>] 0xffffffff Feb 5 16:26:32 Feb 5 16:26:32 other info that might help us debug this: Feb 5 16:26:32 Feb 5 16:26:32 2 locks held by pppd/3224: Feb 5 16:26:32 #0: (&ppp->wlock){-...}, at: [<f8eb8478>] ppp_xmit_process+0x18/0x630 [ppp_generic] Feb 5 16:26:32 #1: (&pch->downl){-...}, at: [<f8eb828e>] ppp_push+0x44e/0x620 [ppp_generic] Feb 5 16:26:32 Feb 5 16:26:32 the soft-irq-safe lock's dependencies: Feb 5 16:26:32 -> (&pch->upl){-.-+} ops: 0 { Feb 5 16:26:32 initial-use at: Feb 5 16:26:32 [<c014e80d>] __lock_acquire+0x11d/0x10a0 Feb 5 16:26:32 [<f8efacf3>] pppol2tp_xmit+0x283/0x460 [pppol2tp] Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0 Feb 5 16:26:32 [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic] Feb 5 16:26:32 [<c040425f>] _read_lock_bh+0x2f/0x40 Feb 5 16:26:32 [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic] Feb 5 16:26:32 [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic] Feb 5 16:26:32 [<f8eb9ffd>] ppp_write+0xdd/0x110 [ppp_generic] Feb 5 16:26:32 [<c0181f86>] vfs_write+0xa6/0x140 Feb 5 16:26:32 [<c0108417>] restore_nocheck+0x12/0x15 Feb 5 16:26:32 [<f8eb9f20>] ppp_write+0x0/0x110 [ppp_generic] Feb 5 16:26:32 [<c0182651>] sys_write+0x41/0x70 Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb Feb 5 16:26:32 [<ffffffff>] 0xffffffff Feb 5 16:26:32 hardirq-on-W at: Feb 5 16:26:32 [<c014ee8e>] __lock_acquire+0x79e/0x10a0 Feb 5 16:26:32 [<c014c524>] add_lock_to_list+0x44/0xb0 Feb 5 16:26:32 [<c014ed68>] __lock_acquire+0x678/0x10a0 Feb 5 16:26:32 [<c014e02e>] mark_held_locks+0x5e/0x80 Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170 Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0 Feb 5 16:26:32 [<f8eb7387>] ppp_ioctl+0x647/0xf40 [ppp_generic] Feb 5 16:26:32 [<c04041cf>] _write_lock_bh+0x2f/0x40 Feb 5 16:26:32 [<f8eb7387>] ppp_ioctl+0x647/0xf40 [ppp_generic] Feb 5 16:26:32 [<f8eb7387>] ppp_ioctl+0x647/0xf40 [ppp_generic] Feb 5 16:26:32 [<c0404665>] _spin_unlock_irqrestore+0x55/0x70 Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170 Feb 5 16:26:32 [<c0403e4e>] __down+0xde/0xed Feb 5 16:26:32 [<c0124220>] default_wake_function+0x0/0x10 Feb 5 16:26:32 [<c0403bdb>] __down_failed+0x7/0xc Feb 5 16:26:32 [<c018d8f8>] do_ioctl+0x78/0x90 Feb 5 16:26:32 [<c018d96c>] vfs_ioctl+0x5c/0x290 Feb 5 16:26:32 [<c018dbdd>] sys_ioctl+0x3d/0x70 Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb Feb 5 16:26:32 [<ffffffff>] 0xffffffff Feb 5 16:26:32 in-softirq-R at: Feb 5 16:26:32 [<c014d87f>] check_usage_backwards+0x1f/0x50 Feb 5 16:26:32 [<c014c479>] save_trace+0x39/0xa0 Feb 5 16:26:32 [<c014edaf>] __lock_acquire+0x6bf/0x10a0 Feb 5 16:26:32 [<c014ee8e>] __lock_acquire+0x79e/0x10a0 Feb 5 16:26:32 [<c014ee8e>] __lock_acquire+0x79e/0x10a0 Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0 Feb 5 16:26:32 [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic] Feb 5 16:26:32 [<c040425f>] _read_lock_bh+0x2f/0x40 Feb 5 16:26:32 [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic] Feb 5 16:26:32 [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic] Feb 5 16:26:32 [<f8ef915a>] pppol2tp_recv_core+0x46a/0x960 [pppol2tp] Feb 5 16:26:32 [<f8ef967e>] pppol2tp_udp_encap_recv+0x2e/0x70 [pppol2tp] Feb 5 16:26:32 [<c0403fd4>] _read_unlock+0x14/0x20 Feb 5 16:26:32 [<c03dd6b6>] udp_queue_rcv_skb+0x106/0x2a0 Feb 5 16:26:32 [<c03ddc7a>] __udp4_lib_rcv+0x42a/0x7e0 Feb 5 16:26:32 [<f8d5c090>] ipt_hook+0x0/0x20 [iptable_filter] Feb 5 16:26:32 [<c03bc2fa>] ip_local_deliver_finish+0xca/0x1c0 Feb 5 16:26:32 [<c03bc25e>] ip_local_deliver_finish+0x2e/0x1c0 Feb 5 16:26:32 [<c03bbfcf>] ip_rcv_finish+0xff/0x360 Feb 5 16:26:32 [<c03bc6fc>] ip_rcv+0x20c/0x2a0 Feb 5 16:26:32 [<c03bbed0>] ip_rcv_finish+0x0/0x360 Feb 5 16:26:32 [<c039ad87>] netif_receive_skb+0x317/0x4b0 Feb 5 16:26:32 [<c039ab70>] netif_receive_skb+0x100/0x4b0 Feb 5 16:26:32 [<f8d7e27a>] e1000_clean_rx_irq_ps+0x28a/0x560 [e1000] Feb 5 16:26:32 [<f8d7dff0>] e1000_clean_rx_irq_ps+0x0/0x560 [e1000] Feb 5 16:26:32 [<f8d7b84d>] e1000_clean+0x5d/0x290 [e1000] Feb 5 16:26:32 [<c039d580>] net_rx_action+0x1a0/0x2a0 Feb 5 16:26:32 [<c039d43f>] net_rx_action+0x5f/0x2a0 Feb 5 16:26:32 [<c0131e72>] __do_softirq+0x92/0x120 Feb 5 16:26:32 [<c0131f78>] do_softirq+0x78/0x80 Feb 5 16:26:32 [<c010b15a>] do_IRQ+0x4a/0xa0 Feb 5 16:26:32 [<c0127af0>] finish_task_switch+0x0/0xc0 Feb 5 16:26:32 [<c0108dcc>] common_interrupt+0x24/0x34 Feb 5 16:26:32 [<c0108dd6>] common_interrupt+0x2e/0x34 Feb 5 16:26:32 [<c01062d6>] mwait_idle_with_hints+0x46/0x60 Feb 5 16:26:32 [<c0106550>] mwait_idle+0x0/0x20 Feb 5 16:26:32 [<c0106694>] cpu_idle+0x74/0xe0 Feb 5 16:26:32 [<c0536a9a>] start_kernel+0x30a/0x3a0 Feb 5 16:26:32 [<c0536150>] unknown_bootoption+0x0/0x1f0 Feb 5 16:26:32 [<ffffffff>] 0xffffffff Feb 5 16:26:32 hardirq-on-R at: Feb 5 16:26:32 [<c014da27>] mark_lock+0x77/0x620 Feb 5 16:26:32 [<c014ec2f>] __lock_acquire+0x53f/0x10a0 Feb 5 16:26:32 [<f8efacf3>] pppol2tp_xmit+0x283/0x460 [pppol2tp] Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0 Feb 5 16:26:32 [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic] Feb 5 16:26:32 [<c040425f>] _read_lock_bh+0x2f/0x40 Feb 5 16:26:32 [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic] Feb 5 16:26:32 [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic] Feb 5 16:26:32 [<f8eb9ffd>] ppp_write+0xdd/0x110 [ppp_generic] Feb 5 16:26:32 [<c0181f86>] vfs_write+0xa6/0x140 Feb 5 16:26:32 [<c0108417>] restore_nocheck+0x12/0x15 Feb 5 16:26:32 [<f8eb9f20>] ppp_write+0x0/0x110 [ppp_generic] Feb 5 16:26:32 [<c0182651>] sys_write+0x41/0x70 Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb Feb 5 16:26:32 [<ffffffff>] 0xffffffff Feb 5 16:26:32 } Feb 5 16:26:32 ... key at: [<f8ebdf20>] __key.30415+0x0/0xffffc2e7 [ppp_generic] Feb 5 16:26:32 -> (&ppp->wlock){-...} ops: 0 { Feb 5 16:26:32 initial-use at: Feb 5 16:26:32 [<c014e80d>] __lock_acquire+0x11d/0x10a0 Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0 Feb 5 16:26:32 [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic] Feb 5 16:26:32 [<c040414f>] _spin_lock_bh+0x2f/0x40 Feb 5 16:26:32 [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic] Feb 5 16:26:32 [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic] Feb 5 16:26:32 [<c0404665>] _spin_unlock_irqrestore+0x55/0x70 Feb 5 16:26:32 [<c0403e4e>] __down+0xde/0xed Feb 5 16:26:32 [<c0124220>] default_wake_function+0x0/0x10 Feb 5 16:26:32 [<c0403bdb>] __down_failed+0x7/0xc Feb 5 16:26:32 [<c018d8f8>] do_ioctl+0x78/0x90 Feb 5 16:26:32 [<c018d96c>] vfs_ioctl+0x5c/0x290 Feb 5 16:26:32 [<c018dbdd>] sys_ioctl+0x3d/0x70 Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb Feb 5 16:26:32 [<ffffffff>] 0xffffffff Feb 5 16:26:32 hardirq-on-W at: Feb 5 16:26:32 [<c014da27>] mark_lock+0x77/0x620 Feb 5 16:26:32 [<c014ed68>] __lock_acquire+0x678/0x10a0 Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0 Feb 5 16:26:32 [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic] Feb 5 16:26:32 [<c040414f>] _spin_lock_bh+0x2f/0x40 Feb 5 16:26:32 [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic] Feb 5 16:26:32 [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic] Feb 5 16:26:32 [<c0404665>] _spin_unlock_irqrestore+0x55/0x70 Feb 5 16:26:32 [<c0403e4e>] __down+0xde/0xed Feb 5 16:26:32 [<c0124220>] default_wake_function+0x0/0x10 Feb 5 16:26:32 [<c0403bdb>] __down_failed+0x7/0xc Feb 5 16:26:32 [<c018d8f8>] do_ioctl+0x78/0x90 Feb 5 16:26:32 [<c018d96c>] vfs_ioctl+0x5c/0x290 Feb 5 16:26:32 [<c018dbdd>] sys_ioctl+0x3d/0x70 Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb Feb 5 16:26:32 [<ffffffff>] 0xffffffff Feb 5 16:26:32 } Feb 5 16:26:32 ... key at: [<f8ebdf08>] __key.30659+0x0/0xffffc2ff [ppp_generic] Feb 5 16:26:32 -> (&ppp->rlock){-...} ops: 0 { Feb 5 16:26:32 initial-use at: Feb 5 16:26:32 [<c014e80d>] __lock_acquire+0x11d/0x10a0 Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0 Feb 5 16:26:32 [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic] Feb 5 16:26:32 [<c040414f>] _spin_lock_bh+0x2f/0x40 Feb 5 16:26:32 [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic] Feb 5 16:26:32 [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic] Feb 5 16:26:32 [<c0404665>] _spin_unlock_irqrestore+0x55/0x70 Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170 Feb 5 16:26:32 [<c0403e4e>] __down+0xde/0xed Feb 5 16:26:32 [<c0124220>] default_wake_function+0x0/0x10 Feb 5 16:26:32 [<c0403bdb>] __down_failed+0x7/0xc Feb 5 16:26:32 [<c018d8f8>] do_ioctl+0x78/0x90 Feb 5 16:26:32 [<c018d96c>] vfs_ioctl+0x5c/0x290 Feb 5 16:26:32 [<c018dbdd>] sys_ioctl+0x3d/0x70 Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb Feb 5 16:26:32 [<ffffffff>] 0xffffffff Feb 5 16:26:32 hardirq-on-W at: Feb 5 16:26:32 [<c014da27>] mark_lock+0x77/0x620 Feb 5 16:26:32 [<c014ed68>] __lock_acquire+0x678/0x10a0 Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0 Feb 5 16:26:32 [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic] Feb 5 16:26:32 [<c040414f>] _spin_lock_bh+0x2f/0x40 Feb 5 16:26:32 [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic] Feb 5 16:26:32 [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic] Feb 5 16:26:32 [<c0404665>] _spin_unlock_irqrestore+0x55/0x70 Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170 Feb 5 16:26:32 [<c0403e4e>] __down+0xde/0xed Feb 5 16:26:32 [<c0124220>] default_wake_function+0x0/0x10 Feb 5 16:26:32 [<c0403bdb>] __down_failed+0x7/0xc Feb 5 16:26:32 [<c018d8f8>] do_ioctl+0x78/0x90 Feb 5 16:26:32 [<c018d96c>] vfs_ioctl+0x5c/0x290 Feb 5 16:26:32 [<c018dbdd>] sys_ioctl+0x3d/0x70 Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb Feb 5 16:26:32 [<ffffffff>] 0xffffffff Feb 5 16:26:32 } Feb 5 16:26:32 ... key at: [<f8ebdf10>] __key.30658+0x0/0xffffc2f7 [ppp_generic] Feb 5 16:26:32 ... acquired at: Feb 5 16:26:32 [<c014f68f>] __lock_acquire+0xf9f/0x10a0 Feb 5 16:26:32 [<f8eb76a5>] ppp_ioctl+0x965/0xf40 [ppp_generic] Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0 Feb 5 16:26:32 [<f8eb76a5>] ppp_ioctl+0x965/0xf40 [ppp_generic] Feb 5 16:26:32 [<c040414f>] _spin_lock_bh+0x2f/0x40 Feb 5 16:26:32 [<f8eb76a5>] ppp_ioctl+0x965/0xf40 [ppp_generic] Feb 5 16:26:32 [<f8eb76a5>] ppp_ioctl+0x965/0xf40 [ppp_generic] Feb 5 16:26:32 [<c0404665>] _spin_unlock_irqrestore+0x55/0x70 Feb 5 16:26:32 [<c0403e4e>] __down+0xde/0xed Feb 5 16:26:32 [<c0124220>] default_wake_function+0x0/0x10 Feb 5 16:26:32 [<c0403bdb>] __down_failed+0x7/0xc Feb 5 16:26:32 [<c018d8f8>] do_ioctl+0x78/0x90 Feb 5 16:26:32 [<c018d96c>] vfs_ioctl+0x5c/0x290 Feb 5 16:26:32 [<c018dbdd>] sys_ioctl+0x3d/0x70 Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb Feb 5 16:26:32 [<ffffffff>] 0xffffffff Feb 5 16:26:32 Feb 5 16:26:32 -> (&list->lock#4){.+..} ops: 0 { Feb 5 16:26:32 initial-use at: Feb 5 16:26:32 [<c014e80d>] __lock_acquire+0x11d/0x10a0 Feb 5 16:26:32 [<c0193ba2>] dput+0xa2/0x130 Feb 5 16:26:32 [<c014ee8e>] __lock_acquire+0x79e/0x10a0 Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0 Feb 5 16:26:32 [<c0394b9c>] skb_queue_tail+0x1c/0x50 Feb 5 16:26:32 [<c040450c>] _spin_lock_irqsave+0x4c/0x70 Feb 5 16:26:32 [<c0394b9c>] skb_queue_tail+0x1c/0x50 Feb 5 16:26:32 [<c0394b9c>] skb_queue_tail+0x1c/0x50 Feb 5 16:26:32 [<f8eb9fd5>] ppp_write+0xb5/0x110 [ppp_generic] Feb 5 16:26:32 [<c0181f86>] vfs_write+0xa6/0x140 Feb 5 16:26:32 [<c0108417>] restore_nocheck+0x12/0x15 Feb 5 16:26:32 [<f8eb9f20>] ppp_write+0x0/0x110 [ppp_generic] Feb 5 16:26:32 [<c0182651>] sys_write+0x41/0x70 Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb Feb 5 16:26:32 [<ffffffff>] 0xffffffff Feb 5 16:26:32 in-softirq-W at: Feb 5 16:26:32 [<c014c479>] save_trace+0x39/0xa0 Feb 5 16:26:32 [<c014db32>] mark_lock+0x182/0x620 Feb 5 16:26:32 [<c014ed36>] __lock_acquire+0x646/0x10a0 Feb 5 16:26:32 [<c014ef26>] __lock_acquire+0x836/0x10a0 Feb 5 16:26:32 [<c014ee8e>] __lock_acquire+0x79e/0x10a0 Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0 Feb 5 16:26:32 [<c0394b9c>] skb_queue_tail+0x1c/0x50 Feb 5 16:26:32 [<c040450c>] _spin_lock_irqsave+0x4c/0x70 Feb 5 16:26:32 [<c0394b9c>] skb_queue_tail+0x1c/0x50 Feb 5 16:26:32 [<c0394b9c>] skb_queue_tail+0x1c/0x50 Feb 5 16:26:32 [<f8eba0a8>] ppp_input+0x78/0x140 [ppp_generic] Feb 5 16:26:32 [<f8ef915a>] pppol2tp_recv_core+0x46a/0x960 [pppol2tp] Feb 5 16:26:32 [<f8ef967e>] pppol2tp_udp_encap_recv+0x2e/0x70 [pppol2tp] Feb 5 16:26:32 [<c0403fd4>] _read_unlock+0x14/0x20 Feb 5 16:26:32 [<c03dd6b6>] udp_queue_rcv_skb+0x106/0x2a0 Feb 5 16:26:32 [<c03ddc7a>] __udp4_lib_rcv+0x42a/0x7e0 Feb 5 16:26:32 [<f8d5c090>] ipt_hook+0x0/0x20 [iptable_filter] Feb 5 16:26:32 [<c03bc2fa>] ip_local_deliver_finish+0xca/0x1c0 Feb 5 16:26:32 [<c03bc25e>] ip_local_deliver_finish+0x2e/0x1c0 Feb 5 16:26:32 [<c03bbfcf>] ip_rcv_finish+0xff/0x360 Feb 5 16:26:32 [<c03bc6fc>] ip_rcv+0x20c/0x2a0 Feb 5 16:26:32 [<c03bbed0>] ip_rcv_finish+0x0/0x360 Feb 5 16:26:32 [<c039ad87>] netif_receive_skb+0x317/0x4b0 Feb 5 16:26:32 [<c039ab70>] netif_receive_skb+0x100/0x4b0 Feb 5 16:26:32 [<f8d7e27a>] e1000_clean_rx_irq_ps+0x28a/0x560 [e1000] Feb 5 16:26:32 [<f8d7dff0>] e1000_clean_rx_irq_ps+0x0/0x560 [e1000] Feb 5 16:26:32 [<f8d7b84d>] e1000_clean+0x5d/0x290 [e1000] Feb 5 16:26:32 [<c039d580>] net_rx_action+0x1a0/0x2a0 Feb 5 16:26:32 [<c039d43f>] net_rx_action+0x5f/0x2a0 Feb 5 16:26:32 [<c0131e72>] __do_softirq+0x92/0x120 Feb 5 16:26:32 [<c0131f78>] do_softirq+0x78/0x80 Feb 5 16:26:32 [<c010b15a>] do_IRQ+0x4a/0xa0 Feb 5 16:26:32 [<c0127af0>] finish_task_switch+0x0/0xc0 Feb 5 16:26:32 [<c0108dcc>] common_interrupt+0x24/0x34 Feb 5 16:26:32 [<c0108dd6>] common_interrupt+0x2e/0x34 Feb 5 16:26:32 [<c01062d6>] mwait_idle_with_hints+0x46/0x60 Feb 5 16:26:32 [<c0106550>] mwait_idle+0x0/0x20 Feb 5 16:26:32 [<c0106694>] cpu_idle+0x74/0xe0 Feb 5 16:26:32 [<c0536a9a>] start_kernel+0x30a/0x3a0 -- James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-12 10:58 ` James Chapman @ 2008-02-12 13:24 ` Jarek Poplawski 2008-02-13 6:00 ` David Miller 2008-02-14 13:00 ` Jarek Poplawski 2 siblings, 0 replies; 51+ messages in thread From: Jarek Poplawski @ 2008-02-12 13:24 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, netdev On Tue, Feb 12, 2008 at 10:58:21AM +0000, James Chapman wrote: ... > Here is a trace from when we had _bh locks. Very nice... ...But since it's quite long, and if you don't know all these paths this could take some time, maybe one question: so if lockdep got these locks right (sometimes it can be wrong when the same structures are nested), then it seems some problem is with this place below. This lock is taken for writing with softirqs enabled here, and IMHO it would be interesting to test if changing this is enough for lockdep. It seems this is in ip4_datagram_connect() during sk_dst_reset() or sk_dst_set(). So maybe you could try with local_bh_disable/enable() around them (or maybe some better idea)? Anyway, I'll try to learn this more in the meantime. Jarek P. > Feb 5 16:26:32 to a soft-irq-unsafe lock: > Feb 5 16:26:32 (&sk->sk_dst_lock){----} > Feb 5 16:26:32 ... which became soft-irq-unsafe at: > Feb 5 16:26:32 ... [<c014e02e>] mark_held_locks+0x5e/0x80 > Feb 5 16:26:32 [<c014ed92>] __lock_acquire+0x6a2/0x10a0 > Feb 5 16:26:32 [<c010f5b0>] save_stack_trace+0x20/0x40 > Feb 5 16:26:32 [<c014c524>] add_lock_to_list+0x44/0xb0 > Feb 5 16:26:32 [<c03dea29>] __udp_lib_get_port+0x19/0x200 > Feb 5 16:26:32 [<c014f735>] __lock_acquire+0x1045/0x10a0 > Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0 > Feb 5 16:26:32 [<c03db0a3>] ip4_datagram_connect+0x53/0x380 > Feb 5 16:26:32 [<c040418a>] _write_lock+0x2a/0x40 > Feb 5 16:26:32 [<c03db0a3>] ip4_datagram_connect+0x53/0x380 > Feb 5 16:26:32 [<c03db0a3>] ip4_datagram_connect+0x53/0x380 > Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170 > Feb 5 16:26:32 [<c0132317>] local_bh_enable_ip+0xa7/0x120 > Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170 > Feb 5 16:26:32 [<c040414f>] _spin_lock_bh+0x2f/0x40 > Feb 5 16:26:32 [<c03e4d55>] inet_dgram_connect+0x35/0x80 > Feb 5 16:26:32 [<c038ec52>] sys_connect+0x82/0xd0 > Feb 5 16:26:32 [<c01455df>] down_read_trylock+0x4f/0x60 > Feb 5 16:26:32 [<c011fe9c>] do_page_fault+0xfc/0x940 > Feb 5 16:26:32 [<c0404024>] _spin_unlock+0x14/0x20 > Feb 5 16:26:32 [<c03905f8>] sys_socketcall+0x98/0x280 > Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170 > Feb 5 16:26:32 [<c02a86ba>] copy_to_user+0x3a/0x70 > Feb 5 16:26:32 [<c0108417>] restore_nocheck+0x12/0x15 > Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb > Feb 5 16:26:32 [<ffffffff>] 0xffffffff ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-12 10:58 ` James Chapman 2008-02-12 13:24 ` Jarek Poplawski @ 2008-02-13 6:00 ` David Miller 2008-02-13 7:29 ` Jarek Poplawski 2008-02-14 13:00 ` Jarek Poplawski 2 siblings, 1 reply; 51+ messages in thread From: David Miller @ 2008-02-13 6:00 UTC (permalink / raw) To: jchapman; +Cc: jarkao2, netdev From: James Chapman <jchapman@katalix.com> Date: Tue, 12 Feb 2008 10:58:21 +0000 > Here is a trace from when we had _bh locks. The problem is that the pppol2tp code calls sk_dst_get() in software interrupt context and that is not allowed. sk_dst_get() grabs sk->sk_dst_lock without any BH enabling or disabling. It can do that because the usage is to make all the lock taking calls in user context, and in the packet processing paths use __sk_dst_get(). Probably what the pppol2tp code should do is use __sk_dst_check() instead of sk_dst_get(). You then have to be able to handle NULL returns, just like UDP sendmsg() does, which means you'll need to cook up a routing lookup if __sk_dst_check() gives you NULL because the route became obsolete. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-13 6:00 ` David Miller @ 2008-02-13 7:29 ` Jarek Poplawski 0 siblings, 0 replies; 51+ messages in thread From: Jarek Poplawski @ 2008-02-13 7:29 UTC (permalink / raw) To: David Miller; +Cc: jchapman, netdev On Tue, Feb 12, 2008 at 10:00:03PM -0800, David Miller wrote: > From: James Chapman <jchapman@katalix.com> > Date: Tue, 12 Feb 2008 10:58:21 +0000 > > > Here is a trace from when we had _bh locks. > > The problem is that the pppol2tp code calls sk_dst_get() in software > interrupt context and that is not allowed. Actually, this lockdep report probably warns of something else: sk_dst_lock which is seen in process context with softirqs enabled implicitly endangers some other (ppp_generic) locks, which depend on ppp_generic pch->upl read_lock, which is taken in softirq context. I can't see on this report any hardirqs dependancies, so it seems even blocking hard interrupts, just like in this patch, shouldn't solve this problem: if BHs were really disabled in exactly the same places, then it seems this warning should trigger in both variants. I studied long ago some similar problem with pppoe, and it looked like ppp_generic needed some fix, but now I've to recall or re-learn almost all and it needs more time... Anyway, there is a possibility, this warning isn't so dangerous. > sk_dst_get() grabs sk->sk_dst_lock without any BH enabling or > disabling. > > It can do that because the usage is to make all the lock > taking calls in user context, and in the packet processing > paths use __sk_dst_get(). BTW, does it mean that ip4_datagram_connect() can be used only in user context? > Probably what the pppol2tp code should do is use __sk_dst_check() > instead of sk_dst_get(). You then have to be able to handle > NULL returns, just like UDP sendmsg() does, which means you'll > need to cook up a routing lookup if __sk_dst_check() gives you > NULL because the route became obsolete. I think that without changing ppp_generic or changing ip_queue_xmit() to something else in pppol2tp this would be really hard to avoid such a warning (and maybe not very necessary). Regards, Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-12 10:58 ` James Chapman 2008-02-12 13:24 ` Jarek Poplawski 2008-02-13 6:00 ` David Miller @ 2008-02-14 13:00 ` Jarek Poplawski 2008-02-18 22:09 ` James Chapman 2 siblings, 1 reply; 51+ messages in thread From: Jarek Poplawski @ 2008-02-14 13:00 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, netdev Hi, It seems, this nice report is still uncomplete: could you check if there could have been something more yet? Thanks, Jarek P. On Tue, Feb 12, 2008 at 10:58:21AM +0000, James Chapman wrote: ... > Here is a trace from when we had _bh locks. > > Feb 5 16:26:32 ====================================================== > Feb 5 16:26:32 [ INFO: soft-safe -> soft-unsafe lock order detected ] > Feb 5 16:26:32 2.6.24-core2 #1 > Feb 5 16:26:32 ------------------------------------------------------ > Feb 5 16:26:32 pppd/3224 [HC0[0]:SC0[2]:HE1:SE0] is trying to acquire: ... > Feb 5 16:26:32 [<f8d7b84d>] e1000_clean+0x5d/0x290 [e1000] > Feb 5 16:26:32 [<c039d580>] net_rx_action+0x1a0/0x2a0 > Feb 5 16:26:32 [<c039d43f>] net_rx_action+0x5f/0x2a0 > Feb 5 16:26:32 [<c0131e72>] __do_softirq+0x92/0x120 > Feb 5 16:26:32 [<c0131f78>] do_softirq+0x78/0x80 > Feb 5 16:26:32 [<c010b15a>] do_IRQ+0x4a/0xa0 > Feb 5 16:26:32 [<c0127af0>] finish_task_switch+0x0/0xc0 > Feb 5 16:26:32 [<c0108dcc>] common_interrupt+0x24/0x34 > Feb 5 16:26:32 [<c0108dd6>] common_interrupt+0x2e/0x34 > Feb 5 16:26:32 [<c01062d6>] mwait_idle_with_hints+0x46/0x60 > Feb 5 16:26:32 [<c0106550>] mwait_idle+0x0/0x20 > Feb 5 16:26:32 [<c0106694>] cpu_idle+0x74/0xe0 > Feb 5 16:26:32 [<c0536a9a>] start_kernel+0x30a/0x3a0 > > -- > James Chapman > Katalix Systems Ltd > http://www.katalix.com > Catalysts for your Embedded Linux software development > ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-14 13:00 ` Jarek Poplawski @ 2008-02-18 22:09 ` James Chapman 2008-02-18 23:01 ` Jarek Poplawski ` (2 more replies) 0 siblings, 3 replies; 51+ messages in thread From: James Chapman @ 2008-02-18 22:09 UTC (permalink / raw) To: Jarek Poplawski, David Miller; +Cc: netdev Jarek Poplawski wrote: > Hi, > > It seems, this nice report is still uncomplete: could you check if > there could have been something more yet? Unfortunately the ISP's syslog stops. But I've been able to borrow two Quad Xeon boxes and have reproduced the problem. Here's a new version of the patch. The patch avoids disabling irqs and fixes the sk_dst_get() usage that DaveM mentioned. But even with this patch, lockdep still complains if hundreds of ppp sessions are inserted into a tunnel as rapidly as possible (lockdep trace is below). I can stop these errors by wrapping the call to ppp_input() in pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is a better fix? btw, I'm not sending this from my usual mail client so the patch will probably be mangled; I'm sending it only for feedback / discussion at this stage. Index: linux-2.6.24.2/drivers/net/pppol2tp.c =================================================================== --- linux-2.6.24.2.orig/drivers/net/pppol2tp.c +++ linux-2.6.24.2/drivers/net/pppol2tp.c @@ -302,14 +302,14 @@ pppol2tp_session_find(struct pppol2tp_tu struct pppol2tp_session *session; struct hlist_node *walk; - read_lock(&tunnel->hlist_lock); + read_lock_bh(&tunnel->hlist_lock); hlist_for_each_entry(session, walk, session_list, hlist) { if (session->tunnel_addr.s_session == session_id) { - read_unlock(&tunnel->hlist_lock); + read_unlock_bh(&tunnel->hlist_lock); return session; } } - read_unlock(&tunnel->hlist_lock); + read_unlock_bh(&tunnel->hlist_lock); return NULL; } @@ -320,14 +320,14 @@ static struct pppol2tp_tunnel *pppol2tp_ { struct pppol2tp_tunnel *tunnel = NULL; - read_lock(&pppol2tp_tunnel_list_lock); + read_lock_bh(&pppol2tp_tunnel_list_lock); list_for_each_entry(tunnel, &pppol2tp_tunnel_list, list) { if (tunnel->stats.tunnel_id == tunnel_id) { - read_unlock(&pppol2tp_tunnel_list_lock); + read_unlock_bh(&pppol2tp_tunnel_list_lock); return tunnel; } } - read_unlock(&pppol2tp_tunnel_list_lock); + read_unlock_bh(&pppol2tp_tunnel_list_lock); return NULL; } @@ -344,7 +344,7 @@ static void pppol2tp_recv_queue_skb(stru struct sk_buff *skbp; u16 ns = PPPOL2TP_SKB_CB(skb)->ns; - spin_lock(&session->reorder_q.lock); + spin_lock_bh(&session->reorder_q.lock); skb_queue_walk(&session->reorder_q, skbp) { if (PPPOL2TP_SKB_CB(skbp)->ns > ns) { __skb_insert(skb, skbp->prev, skbp, &session->reorder_q); @@ -360,7 +360,7 @@ static void pppol2tp_recv_queue_skb(stru __skb_queue_tail(&session->reorder_q, skb); out: - spin_unlock(&session->reorder_q.lock); + spin_unlock_bh(&session->reorder_q.lock); } /* Dequeue a single skb. @@ -442,7 +442,7 @@ static void pppol2tp_recv_dequeue(struct * expect to send up next, dequeue it and any other * in-sequence packets behind it. */ - spin_lock(&session->reorder_q.lock); + spin_lock_bh(&session->reorder_q.lock); skb_queue_walk_safe(&session->reorder_q, skb, tmp) { if (time_after(jiffies, PPPOL2TP_SKB_CB(skb)->expires)) { session->stats.rx_seq_discards++; @@ -469,13 +469,13 @@ static void pppol2tp_recv_dequeue(struct goto out; } } - spin_unlock(&session->reorder_q.lock); + spin_unlock_bh(&session->reorder_q.lock); pppol2tp_recv_dequeue_skb(session, skb); - spin_lock(&session->reorder_q.lock); + spin_lock_bh(&session->reorder_q.lock); } out: - spin_unlock(&session->reorder_q.lock); + spin_unlock_bh(&session->reorder_q.lock); } /* Internal receive frame. Do the real work of receiving an L2TP data frame @@ -964,6 +964,7 @@ static int pppol2tp_xmit(struct ppp_chan static const u8 ppph[2] = { 0xff, 0x03 }; struct sock *sk = (struct sock *) chan->private; struct sock *sk_tun; + struct rtable *rt; int hdr_len; struct pppol2tp_session *session; struct pppol2tp_tunnel *tunnel; @@ -1057,8 +1058,32 @@ static int pppol2tp_xmit(struct ppp_chan nf_reset(skb); /* Get routing info from the tunnel socket */ - dst_release(skb->dst); - skb->dst = sk_dst_get(sk_tun); + rt = (struct rtable *)__sk_dst_check(sk_tun, 0); + if (rt == NULL) { + struct flowi fl = { .oif = sk_tun->sk_bound_dev_if, + .nl_u = { .ip4_u = + { .daddr = inet->daddr, + .saddr = inet->saddr, + .tos = inet->tos } }, + .proto = sk_tun->sk_protocol, + .uli_u = { .ports = + { .sport = inet->sport, + .dport = inet->dport } } }; + security_sk_classify_flow(sk_tun, &fl); + rc = ip_route_output_flow(&rt, &fl, sk_tun, 1); + if (rc) { + if (rc == -ENETUNREACH) + IP_INC_STATS_BH(IPSTATS_MIB_OUTNOROUTES); + goto abort; + } + + sk_dst_set(sk_tun, dst_clone(&rt->u.dst)); + } + + write_lock_bh(&sk_tun->sk_dst_lock); + skb->dst = __sk_dst_get(sk_tun); + dst_hold(skb->dst); + write_unlock_bh(&sk_tun->sk_dst_lock); skb_orphan(skb); skb->sk = sk_tun; @@ -1106,7 +1131,7 @@ static void pppol2tp_tunnel_closeall(str PRINTK(tunnel->debug, PPPOL2TP_MSG_CONTROL, KERN_INFO, "%s: closing all sessions...\n", tunnel->name); - write_lock(&tunnel->hlist_lock); + write_lock_bh(&tunnel->hlist_lock); for (hash = 0; hash < PPPOL2TP_HASH_SIZE; hash++) { again: hlist_for_each_safe(walk, tmp, &tunnel->session_hlist[hash]) { @@ -1126,7 +1151,7 @@ again: * disappear as we're jumping between locks. */ sock_hold(sk); - write_unlock(&tunnel->hlist_lock); + write_unlock_bh(&tunnel->hlist_lock); lock_sock(sk); if (sk->sk_state & (PPPOX_CONNECTED | PPPOX_BOUND)) { @@ -1148,11 +1173,11 @@ again: * list so we are guaranteed to make forward * progress. */ - write_lock(&tunnel->hlist_lock); + write_lock_bh(&tunnel->hlist_lock); goto again; } } - write_unlock(&tunnel->hlist_lock); + write_unlock_bh(&tunnel->hlist_lock); } /* Really kill the tunnel. @@ -1161,9 +1186,9 @@ again: static void pppol2tp_tunnel_free(struct pppol2tp_tunnel *tunnel) { /* Remove from socket list */ - write_lock(&pppol2tp_tunnel_list_lock); + write_lock_bh(&pppol2tp_tunnel_list_lock); list_del_init(&tunnel->list); - write_unlock(&pppol2tp_tunnel_list_lock); + write_unlock_bh(&pppol2tp_tunnel_list_lock); atomic_dec(&pppol2tp_tunnel_count); kfree(tunnel); @@ -1239,9 +1264,9 @@ static void pppol2tp_session_destruct(st /* Delete the session socket from the * hash */ - write_lock(&tunnel->hlist_lock); + write_lock_bh(&tunnel->hlist_lock); hlist_del_init(&session->hlist); - write_unlock(&tunnel->hlist_lock); + write_unlock_bh(&tunnel->hlist_lock); atomic_dec(&pppol2tp_session_count); } @@ -1386,9 +1411,9 @@ static struct sock *pppol2tp_prepare_tun /* Add tunnel to our list */ INIT_LIST_HEAD(&tunnel->list); - write_lock(&pppol2tp_tunnel_list_lock); + write_lock_bh(&pppol2tp_tunnel_list_lock); list_add(&tunnel->list, &pppol2tp_tunnel_list); - write_unlock(&pppol2tp_tunnel_list_lock); + write_unlock_bh(&pppol2tp_tunnel_list_lock); atomic_inc(&pppol2tp_tunnel_count); /* Bump the reference count. The tunnel context is deleted @@ -1546,14 +1571,15 @@ static int pppol2tp_connect(struct socke session->mtu = session->mru = 1500 - PPPOL2TP_HEADER_OVERHEAD; /* If PMTU discovery was enabled, use the MTU that was discovered */ - dst = sk_dst_get(sk); + read_lock_bh(&tunnel_sock->sk_dst_lock); + dst = __sk_dst_get(sk); if (dst != NULL) { - u32 pmtu = dst_mtu(__sk_dst_get(sk)); + u32 pmtu = dst_mtu(dst); if (pmtu != 0) session->mtu = session->mru = pmtu - PPPOL2TP_HEADER_OVERHEAD; - dst_release(dst); } + read_unlock_bh(&tunnel_sock->sk_dst_lock); /* Special case: if source & dest session_id == 0x0000, this socket is * being created to manage the tunnel. Don't add the session to the @@ -1593,11 +1619,11 @@ static int pppol2tp_connect(struct socke sk->sk_user_data = session; /* Add session to the tunnel's hash list */ - write_lock(&tunnel->hlist_lock); + write_lock_bh(&tunnel->hlist_lock); hlist_add_head(&session->hlist, pppol2tp_session_id_hash(tunnel, session->tunnel_addr.s_session)); - write_unlock(&tunnel->hlist_lock); + write_unlock_bh(&tunnel->hlist_lock); atomic_inc(&pppol2tp_session_count); @@ -2199,7 +2225,7 @@ static struct pppol2tp_session *next_ses int next = 0; int i; - read_lock(&tunnel->hlist_lock); + read_lock_bh(&tunnel->hlist_lock); for (i = 0; i < PPPOL2TP_HASH_SIZE; i++) { hlist_for_each_entry(session, walk, &tunnel->session_hlist[i], hlist) { if (curr == NULL) { @@ -2217,7 +2243,7 @@ static struct pppol2tp_session *next_ses } } out: - read_unlock(&tunnel->hlist_lock); + read_unlock_bh(&tunnel->hlist_lock); if (!found) session = NULL; @@ -2228,13 +2254,13 @@ static struct pppol2tp_tunnel *next_tunn { struct pppol2tp_tunnel *tunnel = NULL; - read_lock(&pppol2tp_tunnel_list_lock); + read_lock_bh(&pppol2tp_tunnel_list_lock); if (list_is_last(&curr->list, &pppol2tp_tunnel_list)) { goto out; } tunnel = list_entry(curr->list.next, struct pppol2tp_tunnel, list); out: - read_unlock(&pppol2tp_tunnel_list_lock); + read_unlock_bh(&pppol2tp_tunnel_list_lock); return tunnel; } ...................................................... Here's the lockdep trace (long). Feb 18 20:56:18 localhost kernel: Feb 18 20:56:18 localhost kernel: ====================================================== Feb 18 20:56:18 localhost kernel: [ INFO: soft-safe -> soft-unsafe lock order detected ] Feb 18 20:56:18 localhost kernel: 2.6.24.2 #1 Feb 18 20:56:18 localhost kernel: ------------------------------------------------------ Feb 18 20:56:18 localhost kernel: pppd/3241 [HC0[0]:SC0[2]:HE1:SE0] is trying to acquire: Feb 18 20:56:18 localhost kernel: (&pch->downl){-...}, at: [<f8c566db>] ppp_push+0x63/0x50d [ppp_generic] Feb 18 20:56:18 localhost kernel: Feb 18 20:56:18 localhost kernel: and this task is already holding: Feb 18 20:56:18 localhost kernel: (&ppp->wlock){-...}, at: [<f8c56f10>] ppp_xmit_process+0x15/0x5a1 [ppp_generic] Feb 18 20:56:18 localhost kernel: which would create a new lock dependency: Feb 18 20:56:18 localhost kernel: (&ppp->wlock){-...} -> (&pch->downl){-...} Feb 18 20:56:18 localhost kernel: Feb 18 20:56:18 localhost kernel: but this new dependency connects a soft-irq-safe lock: Feb 18 20:56:18 localhost kernel: (&pch->upl){-.-+} Feb 18 20:56:18 localhost kernel: ... which became soft-irq-safe at: Feb 18 20:56:18 localhost kernel: [<c0448013>] check_usage_backwards+0x19/0x41 Feb 18 20:56:18 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:56:18 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:18 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:18 localhost kernel: [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic] Feb 18 20:56:18 localhost kernel: [<c061b373>] _read_lock_bh+0x2e/0x39 Feb 18 20:56:18 localhost kernel: [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic] Feb 18 20:56:18 localhost kernel: [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic] Feb 18 20:56:18 localhost kernel: [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp] Feb 18 20:56:18 localhost kernel: [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp] Feb 18 20:56:18 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c Feb 18 20:56:18 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259 Feb 18 20:56:18 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758 Feb 18 20:56:18 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8 Feb 18 20:56:18 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8 Feb 18 20:56:18 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338 Feb 18 20:56:18 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 20:56:18 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237 Feb 18 20:56:18 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4 Feb 18 20:56:18 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 20:56:18 localhost kernel: [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000] Feb 18 20:56:18 localhost kernel: [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000] Feb 18 20:56:18 localhost kernel: [<f8984f13>] e1000_clean+0x63/0x203 [e1000] Feb 18 20:56:19 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3 Feb 18 20:56:19 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3 Feb 18 20:56:19 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:56:19 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:56:19 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:19 localhost kernel: Feb 18 20:56:19 localhost kernel: to a soft-irq-unsafe lock: Feb 18 20:56:19 localhost kernel: (&sk->sk_dst_lock){----} Feb 18 20:56:19 localhost kernel: ... which became soft-irq-unsafe at: Feb 18 20:56:19 localhost kernel: ... [<c0449046>] __lock_acquire+0x48b/0xbf1 Feb 18 20:56:19 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 20:56:19 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115 Feb 18 20:56:19 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb Feb 18 20:56:19 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:19 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 20:56:19 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34 Feb 18 20:56:19 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 20:56:19 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 20:56:19 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e Feb 18 20:56:19 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f Feb 18 20:56:19 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261 Feb 18 20:56:19 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5 Feb 18 20:56:19 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:19 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5 Feb 18 20:56:19 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:19 localhost kernel: Feb 18 20:56:19 localhost kernel: other info that might help us debug this: Feb 18 20:56:19 localhost kernel: Feb 18 20:56:19 localhost kernel: 1 lock held by pppd/3241: Feb 18 20:56:19 localhost kernel: #0: (&ppp->wlock){-...}, at: [<f8c56f10>] ppp_xmit_process+0x15/0x5a1 [ppp_generic] Feb 18 20:56:19 localhost kernel: Feb 18 20:56:19 localhost kernel: the soft-irq-safe lock's dependencies: Feb 18 20:56:19 localhost kernel: -> (&pch->upl){-.-+} ops: 13 { Feb 18 20:56:19 localhost kernel: initial-use at: Feb 18 20:56:19 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 20:56:19 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:56:19 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:19 localhost kernel: [<f8c57b7c>] ppp_ioctl+0x4df/0xc06 [ppp_generic] Feb 18 20:56:19 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:19 localhost kernel: [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic] Feb 18 20:56:19 localhost kernel: [<c061b300>] _write_lock_bh+0x2e/0x39 Feb 18 20:56:20 localhost kernel: [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic] Feb 18 20:56:20 localhost kernel: [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic] Feb 18 20:56:20 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 20:56:20 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:56:20 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:20 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 20:56:20 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8 Feb 18 20:56:20 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc Feb 18 20:56:20 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62 Feb 18 20:56:20 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249 Feb 18 20:56:20 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d Feb 18 20:56:20 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 20:56:20 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:20 localhost kernel: hardirq-on-W at: Feb 18 20:56:20 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:20 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 20:56:20 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1 Feb 18 20:56:20 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:20 localhost kernel: [<f8c57b7c>] ppp_ioctl+0x4df/0xc06 [ppp_generic] Feb 18 20:56:20 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:20 localhost kernel: [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic] Feb 18 20:56:20 localhost kernel: [<c061b300>] _write_lock_bh+0x2e/0x39 Feb 18 20:56:20 localhost kernel: [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic] Feb 18 20:56:20 localhost kernel: [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic] Feb 18 20:56:20 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 20:56:20 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:56:20 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:20 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 20:56:20 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8 Feb 18 20:56:20 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc Feb 18 20:56:20 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62 Feb 18 20:56:20 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249 Feb 18 20:56:20 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d Feb 18 20:56:20 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 20:56:20 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:20 localhost kernel: in-softirq-R at: Feb 18 20:56:20 localhost kernel: [<c0448013>] check_usage_backwards+0x19/0x41 Feb 18 20:56:20 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:56:20 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:20 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:20 localhost kernel: [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic] Feb 18 20:56:21 localhost kernel: [<c061b373>] _read_lock_bh+0x2e/0x39 Feb 18 20:56:21 localhost kernel: [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic] Feb 18 20:56:21 localhost kernel: [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic] Feb 18 20:56:21 localhost kernel: [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp] Feb 18 20:56:21 localhost kernel: [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp] Feb 18 20:56:21 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c Feb 18 20:56:21 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259 Feb 18 20:56:21 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758 Feb 18 20:56:21 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8 Feb 18 20:56:21 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8 Feb 18 20:56:21 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338 Feb 18 20:56:21 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 20:56:21 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237 Feb 18 20:56:21 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4 Feb 18 20:56:21 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 20:56:21 localhost kernel: [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000] Feb 18 20:56:21 localhost kernel: [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000] Feb 18 20:56:21 localhost kernel: [<f8984f13>] e1000_clean+0x63/0x203 [e1000] Feb 18 20:56:21 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3 Feb 18 20:56:21 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3 Feb 18 20:56:21 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:56:21 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:56:21 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:21 localhost kernel: hardirq-on-R at: Feb 18 20:56:21 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 20:56:21 localhost kernel: [<c0449001>] __lock_acquire+0x446/0xbf1 Feb 18 20:56:21 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:21 localhost kernel: [<f8c5834b>] ppp_channel_push+0x71/0x90 [ppp_generic] Feb 18 20:56:21 localhost kernel: [<c061b373>] _read_lock_bh+0x2e/0x39 Feb 18 20:56:21 localhost kernel: [<f8c5834b>] ppp_channel_push+0x71/0x90 [ppp_generic] Feb 18 20:56:21 localhost kernel: [<f8c5834b>] ppp_channel_push+0x71/0x90 [ppp_generic] Feb 18 20:56:21 localhost kernel: [<f8c593bd>] ppp_write+0xd0/0xdb [ppp_generic] Feb 18 20:56:21 localhost kernel: [<f8c592ed>] ppp_write+0x0/0xdb [ppp_generic] Feb 18 20:56:21 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d Feb 18 20:56:21 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67 Feb 18 20:56:21 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 20:56:21 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:21 localhost kernel: } Feb 18 20:56:21 localhost kernel: ... key at: [<f8c5dd20>] __key.29741+0x0/0xffffb822 [ppp_generic] Feb 18 20:56:21 localhost kernel: -> (&ppp->wlock){-...} ops: 11 { Feb 18 20:56:22 localhost kernel: initial-use at: Feb 18 20:56:22 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:56:22 localhost kernel: [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic] Feb 18 20:56:22 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:22 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic] Feb 18 20:56:22 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 20:56:22 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic] Feb 18 20:56:22 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic] Feb 18 20:56:22 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 20:56:22 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:56:22 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:22 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 20:56:22 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8 Feb 18 20:56:22 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc Feb 18 20:56:22 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62 Feb 18 20:56:22 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249 Feb 18 20:56:22 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d Feb 18 20:56:22 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 20:56:22 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:22 localhost kernel: hardirq-on-W at: Feb 18 20:56:22 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1 Feb 18 20:56:22 localhost kernel: [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic] Feb 18 20:56:22 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:22 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic] Feb 18 20:56:22 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 20:56:22 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic] Feb 18 20:56:22 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic] Feb 18 20:56:22 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 20:56:22 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:56:22 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:22 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 20:56:22 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8 Feb 18 20:56:22 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc Feb 18 20:56:22 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62 Feb 18 20:56:22 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249 Feb 18 20:56:22 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d Feb 18 20:56:22 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 20:56:22 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:22 localhost kernel: } Feb 18 20:56:22 localhost kernel: ... key at: [<f8c5dd08>] __key.29980+0x0/0xffffb83a [ppp_generic] Feb 18 20:56:22 localhost kernel: -> (&ppp->rlock){-+..} ops: 5 { Feb 18 20:56:22 localhost kernel: initial-use at: Feb 18 20:56:22 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:56:23 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic] Feb 18 20:56:23 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:23 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic] Feb 18 20:56:23 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 20:56:23 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic] Feb 18 20:56:23 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic] Feb 18 20:56:23 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 20:56:23 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:56:23 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:23 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 20:56:23 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8 Feb 18 20:56:23 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc Feb 18 20:56:23 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62 Feb 18 20:56:23 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249 Feb 18 20:56:23 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d Feb 18 20:56:23 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 20:56:23 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:23 localhost kernel: in-softirq-W at: Feb 18 20:56:23 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 20:56:23 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:56:23 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:23 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:23 localhost kernel: [<f8c59473>] ppp_input+0xab/0xef [ppp_generic] Feb 18 20:56:23 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 20:56:23 localhost kernel: [<f8c59473>] ppp_input+0xab/0xef [ppp_generic] Feb 18 20:56:23 localhost kernel: [<f8c59473>] ppp_input+0xab/0xef [ppp_generic] Feb 18 20:56:23 localhost kernel: [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp] Feb 18 20:56:23 localhost kernel: [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp] Feb 18 20:56:23 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c Feb 18 20:56:23 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259 Feb 18 20:56:23 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758 Feb 18 20:56:23 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8 Feb 18 20:56:23 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8 Feb 18 20:56:23 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338 Feb 18 20:56:23 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 20:56:23 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237 Feb 18 20:56:23 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4 Feb 18 20:56:23 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 20:56:23 localhost kernel: [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000] Feb 18 20:56:23 localhost kernel: [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000] Feb 18 20:56:23 localhost kernel: [<f8984f13>] e1000_clean+0x63/0x203 [e1000] Feb 18 20:56:24 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3 Feb 18 20:56:24 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3 Feb 18 20:56:24 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:56:24 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:56:24 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:24 localhost kernel: hardirq-on-W at: Feb 18 20:56:24 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1 Feb 18 20:56:24 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic] Feb 18 20:56:24 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:24 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic] Feb 18 20:56:24 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 20:56:24 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic] Feb 18 20:56:24 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic] Feb 18 20:56:24 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 20:56:24 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:56:24 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:24 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 20:56:24 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8 Feb 18 20:56:24 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc Feb 18 20:56:24 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62 Feb 18 20:56:24 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249 Feb 18 20:56:24 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d Feb 18 20:56:24 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 20:56:24 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:24 localhost kernel: } Feb 18 20:56:24 localhost kernel: ... key at: [<f8c5dd10>] __key.29979+0x0/0xffffb832 [ppp_generic] Feb 18 20:56:24 localhost kernel: -> (&list->lock#7){.+..} ops: 31 { Feb 18 20:56:24 localhost kernel: initial-use at: Feb 18 20:56:24 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:56:24 localhost kernel: [<c046b874>] __do_fault+0x31b/0x35d Feb 18 20:56:24 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:24 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 20:56:24 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:56:24 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 20:56:24 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 20:56:24 localhost kernel: [<f8c5939f>] ppp_write+0xb2/0xdb [ppp_generic] Feb 18 20:56:24 localhost kernel: [<f8c592ed>] ppp_write+0x0/0xdb [ppp_generic] Feb 18 20:56:24 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d Feb 18 20:56:24 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67 Feb 18 20:56:24 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 20:56:24 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:24 localhost kernel: in-softirq-W at: Feb 18 20:56:24 localhost kernel: [<c0446ab0>] save_trace+0x37/0x8b Feb 18 20:56:25 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:56:25 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:25 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:25 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 20:56:25 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:56:25 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 20:56:25 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 20:56:25 localhost kernel: [<f8c59433>] ppp_input+0x6b/0xef [ppp_generic] Feb 18 20:56:25 localhost kernel: [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp] Feb 18 20:56:25 localhost kernel: [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp] Feb 18 20:56:25 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c Feb 18 20:56:25 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259 Feb 18 20:56:25 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758 Feb 18 20:56:25 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8 Feb 18 20:56:25 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8 Feb 18 20:56:25 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338 Feb 18 20:56:25 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 20:56:25 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237 Feb 18 20:56:25 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4 Feb 18 20:56:25 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 20:56:25 localhost kernel: [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000] Feb 18 20:56:25 localhost kernel: [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000] Feb 18 20:56:25 localhost kernel: [<f8984f13>] e1000_clean+0x63/0x203 [e1000] Feb 18 20:56:25 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3 Feb 18 20:56:25 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3 Feb 18 20:56:25 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:56:25 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:56:25 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:25 localhost kernel: } Feb 18 20:56:25 localhost kernel: ... key at: [<f8c5dd18>] __key.21159+0x0/0xffffb82a [ppp_generic] Feb 18 20:56:25 localhost kernel: ... acquired at: Feb 18 20:56:25 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:56:25 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 20:56:25 localhost kernel: [<c0446ab0>] save_trace+0x37/0x8b Feb 18 20:56:25 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:25 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 20:56:25 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:56:25 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 20:56:25 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 20:56:25 localhost kernel: [<f8c589f2>] ppp_receive_nonmp_frame+0x4e6/0x70a [ppp_generic] Feb 18 20:56:25 localhost kernel: [<f8c59473>] ppp_input+0xab/0xef [ppp_generic] Feb 18 20:56:26 localhost kernel: [<f8c59487>] ppp_input+0xbf/0xef [ppp_generic] Feb 18 20:56:26 localhost kernel: [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp] Feb 18 20:56:26 localhost kernel: [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp] Feb 18 20:56:26 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c Feb 18 20:56:26 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259 Feb 18 20:56:26 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758 Feb 18 20:56:26 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8 Feb 18 20:56:26 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8 Feb 18 20:56:26 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338 Feb 18 20:56:26 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 20:56:26 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237 Feb 18 20:56:26 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4 Feb 18 20:56:26 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 20:56:26 localhost kernel: [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000] Feb 18 20:56:26 localhost kernel: [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000] Feb 18 20:56:26 localhost kernel: [<f8984f13>] e1000_clean+0x63/0x203 [e1000] Feb 18 20:56:26 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3 Feb 18 20:56:26 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3 Feb 18 20:56:26 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:56:26 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:56:26 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:26 localhost kernel: Feb 18 20:56:26 localhost kernel: -> (&q->lock){++..} ops: 468037 { Feb 18 20:56:26 localhost kernel: initial-use at: Feb 18 20:56:26 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 20:56:26 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:56:26 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 20:56:26 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 20:56:26 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:56:26 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:26 localhost kernel: [<c0619856>] wait_for_common+0x2a/0xe0 Feb 18 20:56:26 localhost kernel: [<c061b4a0>] _spin_lock_irq+0x2f/0x3a Feb 18 20:56:26 localhost kernel: [<c0619856>] wait_for_common+0x2a/0xe0 Feb 18 20:56:26 localhost kernel: [<c0619856>] wait_for_common+0x2a/0xe0 Feb 18 20:56:26 localhost kernel: [<c043d4b2>] kthread_create+0x70/0xa4 Feb 18 20:56:26 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:56:26 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 20:56:26 localhost kernel: [<c04265bc>] migration_call+0x49/0x364 Feb 18 20:56:26 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:56:26 localhost kernel: [<c075e01b>] migration_init+0x19/0x40 Feb 18 20:56:26 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:27 localhost kernel: [<c074e39e>] kernel_init+0x55/0x2af Feb 18 20:56:27 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 20:56:27 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:27 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:27 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:56:27 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:27 localhost kernel: in-hardirq-W at: Feb 18 20:56:27 localhost kernel: [<c04480df>] check_usage_forwards+0x19/0x41 Feb 18 20:56:27 localhost kernel: [<c0446ab0>] save_trace+0x37/0x8b Feb 18 20:56:27 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 20:56:27 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:27 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:27 localhost kernel: [<c04235da>] complete+0x12/0x44 Feb 18 20:56:27 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:56:27 localhost kernel: [<c04235da>] complete+0x12/0x44 Feb 18 20:56:27 localhost kernel: [<c04235da>] complete+0x12/0x44 Feb 18 20:56:27 localhost kernel: [<c058446e>] i8042_aux_test_irq+0x44/0x5c Feb 18 20:56:27 localhost kernel: [<c045bcde>] handle_IRQ_event+0x13/0x3d Feb 18 20:56:27 localhost kernel: [<c045cd38>] handle_edge_irq+0xc2/0xff Feb 18 20:56:27 localhost kernel: [<c045cc76>] handle_edge_irq+0x0/0xff Feb 18 20:56:27 localhost kernel: [<c04071b4>] do_IRQ+0xac/0xd4 Feb 18 20:56:27 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:27 localhost kernel: in-softirq-W at: Feb 18 20:56:27 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:56:27 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:27 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:27 localhost kernel: [<c04235da>] complete+0x12/0x44 Feb 18 20:56:27 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:56:27 localhost kernel: [<c04235da>] complete+0x12/0x44 Feb 18 20:56:27 localhost kernel: [<c04235da>] complete+0x12/0x44 Feb 18 20:56:27 localhost kernel: [<c0430b61>] tasklet_action+0x23/0xa4 Feb 18 20:56:27 localhost kernel: [<c043b647>] __rcu_process_callbacks+0xfc/0x16e Feb 18 20:56:27 localhost kernel: [<c043b6d1>] rcu_process_callbacks+0x18/0x30 Feb 18 20:56:27 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 20:56:27 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:56:27 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:56:27 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:27 localhost kernel: } Feb 18 20:56:27 localhost kernel: ... key at: [<c0804610>] __key.17392+0x0/0x8 Feb 18 20:56:27 localhost kernel: -> (&rq->rq_lock_key){++..} ops: 138094 { Feb 18 20:56:28 localhost kernel: initial-use at: Feb 18 20:56:28 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:56:28 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:28 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:28 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:56:28 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:28 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:28 localhost kernel: [<c075e3c9>] sched_init+0x27d/0x292 Feb 18 20:56:28 localhost kernel: [<c074e756>] start_kernel+0x15e/0x327 Feb 18 20:56:28 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:28 localhost kernel: in-hardirq-W at: Feb 18 20:56:28 localhost kernel: [<c04480a2>] find_usage_forwards+0x67/0x8b Feb 18 20:56:28 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 20:56:28 localhost kernel: [<c0442f77>] clocksource_get_next+0xa/0x3f Feb 18 20:56:28 localhost kernel: [<c0442fa6>] clocksource_get_next+0x39/0x3f Feb 18 20:56:28 localhost kernel: [<c044204f>] update_wall_time+0x58d/0x71c Feb 18 20:56:28 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:28 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:28 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:28 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:28 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:28 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 20:56:28 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 20:56:28 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 20:56:28 localhost kernel: [<c0407b80>] timer_interrupt+0x44/0x4a Feb 18 20:56:28 localhost kernel: [<c045bcde>] handle_IRQ_event+0x13/0x3d Feb 18 20:56:28 localhost kernel: [<c045d17e>] handle_level_irq+0x78/0xbe Feb 18 20:56:28 localhost kernel: [<c045d106>] handle_level_irq+0x0/0xbe Feb 18 20:56:28 localhost kernel: [<c04071b4>] do_IRQ+0xac/0xd4 Feb 18 20:56:28 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:28 localhost kernel: in-softirq-W at: Feb 18 20:56:28 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21 Feb 18 20:56:28 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3 Feb 18 20:56:28 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:56:28 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:28 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:28 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:56:28 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:28 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:56:28 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:56:28 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5 Feb 18 20:56:29 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 20:56:29 localhost kernel: [<c0430afa>] __do_softirq+0xc9/0xde Feb 18 20:56:29 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:56:29 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:29 localhost kernel: } Feb 18 20:56:29 localhost kernel: ... key at: [<c2a22488>] 0xc2a22488 Feb 18 20:56:29 localhost kernel: -> (&rq->rq_lock_key#2){++..} ops: 382046 { Feb 18 20:56:29 localhost kernel: initial-use at: Feb 18 20:56:29 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:56:29 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:29 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:29 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:56:29 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:29 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:29 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 20:56:29 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 20:56:29 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f Feb 18 20:56:29 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 20:56:29 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 20:56:29 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 20:56:29 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:56:29 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:56:29 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:29 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:56:29 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 20:56:29 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 20:56:29 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:29 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:29 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 20:56:29 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 20:56:29 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:29 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:29 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:56:29 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:29 localhost kernel: in-hardirq-W at: Feb 18 20:56:29 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 20:56:29 localhost kernel: [<c040a9d6>] save_stack_trace+0x20/0x3a Feb 18 20:56:29 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 20:56:29 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:29 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:29 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:30 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:30 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:30 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:30 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:30 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 20:56:30 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 20:56:30 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 20:56:30 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 20:56:30 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 20:56:30 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 20:56:30 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:56:30 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 20:56:30 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:56:30 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 20:56:30 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 20:56:30 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 20:56:30 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:30 localhost kernel: in-softirq-W at: Feb 18 20:56:30 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:56:30 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21 Feb 18 20:56:30 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3 Feb 18 20:56:30 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:30 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:30 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:56:30 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:30 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:56:30 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:56:30 localhost kernel: [<c0433efb>] process_timeout+0x0/0x5 Feb 18 20:56:30 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5 Feb 18 20:56:30 localhost kernel: [<c0433d59>] run_timer_softirq+0x113/0x188 Feb 18 20:56:30 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 20:56:30 localhost kernel: [<c0433efb>] process_timeout+0x0/0x5 Feb 18 20:56:30 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 20:56:30 localhost kernel: [<c0433efb>] process_timeout+0x0/0x5 Feb 18 20:56:30 localhost kernel: [<c0433d67>] run_timer_softirq+0x121/0x188 Feb 18 20:56:30 localhost kernel: [<c0433efb>] process_timeout+0x0/0x5 Feb 18 20:56:30 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 20:56:30 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:56:30 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:56:30 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:30 localhost kernel: } Feb 18 20:56:31 localhost kernel: ... key at: [<c2a2b488>] 0xc2a2b488 Feb 18 20:56:31 localhost kernel: -> (&rq->rq_lock_key#3){++..} ops: 250064 { Feb 18 20:56:31 localhost kernel: initial-use at: Feb 18 20:56:31 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:56:31 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:31 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:31 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:56:31 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:31 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:31 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 20:56:31 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 20:56:31 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f Feb 18 20:56:31 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 20:56:31 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 20:56:31 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 20:56:31 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:56:31 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:56:31 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:31 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:56:31 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 20:56:31 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 20:56:31 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:31 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:31 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 20:56:31 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 20:56:31 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:31 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:31 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:56:31 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:31 localhost kernel: in-hardirq-W at: Feb 18 20:56:31 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 20:56:31 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:31 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:31 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:31 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:31 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:31 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:31 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:31 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 20:56:31 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 20:56:31 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 20:56:32 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 20:56:32 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 20:56:32 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 20:56:32 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:56:32 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 20:56:32 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:56:32 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 20:56:32 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 20:56:32 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 20:56:32 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:32 localhost kernel: in-softirq-W at: Feb 18 20:56:32 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 20:56:32 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:56:32 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:32 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40 Feb 18 20:56:32 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:32 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40 Feb 18 20:56:32 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40 Feb 18 20:56:32 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 20:56:32 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 20:56:32 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 20:56:32 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:56:32 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:56:32 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:32 localhost kernel: } Feb 18 20:56:32 localhost kernel: ... key at: [<c2a34488>] 0xc2a34488 Feb 18 20:56:32 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 233519 { Feb 18 20:56:32 localhost kernel: initial-use at: Feb 18 20:56:32 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:56:32 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:32 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:32 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:56:32 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:32 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:32 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 20:56:32 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 20:56:32 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f Feb 18 20:56:32 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 20:56:32 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 20:56:32 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 20:56:32 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:56:32 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:56:33 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:33 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:56:33 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 20:56:33 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 20:56:33 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:33 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:33 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 20:56:33 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 20:56:33 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:33 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:33 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:56:33 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:33 localhost kernel: in-hardirq-W at: Feb 18 20:56:33 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 20:56:33 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:33 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:33 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:33 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:33 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:33 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:33 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:33 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 20:56:33 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 20:56:33 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 20:56:33 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 20:56:33 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 20:56:33 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 20:56:33 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:56:33 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 20:56:33 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:56:33 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 20:56:33 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 20:56:33 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 20:56:33 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:33 localhost kernel: in-softirq-W at: Feb 18 20:56:33 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:56:33 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:33 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:33 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:33 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:33 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:33 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 20:56:34 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 20:56:34 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 20:56:34 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:56:34 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:56:34 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:34 localhost kernel: } Feb 18 20:56:34 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488 Feb 18 20:56:34 localhost kernel: ... acquired at: Feb 18 20:56:34 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:56:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:34 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:34 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:34 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 20:56:34 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:56:34 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 20:56:34 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:56:34 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 20:56:34 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 20:56:34 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:56:34 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:34 localhost kernel: Feb 18 20:56:34 localhost kernel: ... acquired at: Feb 18 20:56:34 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:56:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:34 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:34 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:34 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 20:56:34 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:56:34 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 20:56:34 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:56:34 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 20:56:34 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 20:56:34 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:56:34 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:35 localhost kernel: Feb 18 20:56:35 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 233519 { Feb 18 20:56:35 localhost kernel: initial-use at: Feb 18 20:56:35 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:56:35 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:35 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:35 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:56:35 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:35 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:35 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 20:56:35 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 20:56:35 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f Feb 18 20:56:35 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 20:56:35 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 20:56:35 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 20:56:35 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:56:35 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:56:35 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:35 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:56:35 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 20:56:35 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 20:56:35 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:35 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:35 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 20:56:35 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 20:56:35 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:35 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:35 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:56:35 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:35 localhost kernel: in-hardirq-W at: Feb 18 20:56:35 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 20:56:35 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:35 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:35 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:35 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:35 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:35 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:35 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:35 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 20:56:35 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 20:56:35 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 20:56:36 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 20:56:36 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 20:56:36 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 20:56:36 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:56:36 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 20:56:36 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:56:36 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 20:56:36 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 20:56:36 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 20:56:36 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:36 localhost kernel: in-softirq-W at: Feb 18 20:56:36 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:56:36 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:36 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:36 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 20:56:36 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 20:56:36 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 20:56:36 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:56:36 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:56:36 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:36 localhost kernel: } Feb 18 20:56:36 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488 Feb 18 20:56:36 localhost kernel: ... acquired at: Feb 18 20:56:36 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:56:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:36 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:36 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:36 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 20:56:36 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:56:36 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 20:56:36 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:56:36 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 20:56:36 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 20:56:36 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:56:36 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:37 localhost kernel: Feb 18 20:56:37 localhost kernel: ... acquired at: Feb 18 20:56:37 localhost kernel: [<c0446b6a>] add_lock_to_list+0x66/0x89 Feb 18 20:56:37 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:56:37 localhost kernel: [<c0619307>] schedule+0x22e/0x6ec Feb 18 20:56:37 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:37 localhost kernel: [<c0619307>] schedule+0x22e/0x6ec Feb 18 20:56:37 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:37 localhost kernel: [<c0619307>] schedule+0x22e/0x6ec Feb 18 20:56:37 localhost kernel: [<c0619307>] schedule+0x22e/0x6ec Feb 18 20:56:37 localhost kernel: [<c0619a92>] schedule_timeout+0x13/0x8d Feb 18 20:56:37 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 20:56:37 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 20:56:37 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:37 localhost kernel: [<c06198d2>] wait_for_common+0xa6/0xe0 Feb 18 20:56:37 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8 Feb 18 20:56:37 localhost kernel: [<c0439c5c>] call_usermodehelper_exec+0x7a/0xbe Feb 18 20:56:37 localhost kernel: [<c04ecbd6>] kobject_uevent_env+0x348/0x37c Feb 18 20:56:37 localhost kernel: [<c055f4c0>] device_add+0x318/0x54c Feb 18 20:56:37 localhost kernel: [<c055fa67>] device_create+0x7a/0x9a Feb 18 20:56:37 localhost kernel: [<c053c13e>] tty_register_device+0xb5/0xbd Feb 18 20:56:37 localhost kernel: [<c04efab1>] vsnprintf+0x440/0x47c Feb 18 20:56:37 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 20:56:37 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:56:37 localhost kernel: [<c042c9c6>] register_console+0x4b/0x1fe Feb 18 20:56:37 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:37 localhost kernel: [<c055aff0>] serial8250_pm+0x0/0xb2 Feb 18 20:56:37 localhost kernel: [<c0558795>] uart_change_pm+0x23/0x2b Feb 18 20:56:37 localhost kernel: [<c055998a>] uart_add_one_port+0x29d/0x2f6 Feb 18 20:56:37 localhost kernel: [<c05623c9>] platform_device_add+0xee/0x11c Feb 18 20:56:37 localhost kernel: [<c055ed94>] device_initialize+0x7b/0x93 Feb 18 20:56:37 localhost kernel: [<c05624ee>] platform_device_alloc+0x40/0x51 Feb 18 20:56:37 localhost kernel: [<c076a545>] serial8250_init+0xca/0x110 Feb 18 20:56:37 localhost kernel: [<c074e491>] kernel_init+0x148/0x2af Feb 18 20:56:37 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:37 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:37 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:56:37 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:37 localhost kernel: Feb 18 20:56:37 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 233519 { Feb 18 20:56:37 localhost kernel: initial-use at: Feb 18 20:56:38 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:56:38 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:38 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:38 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:56:38 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:38 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:38 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 20:56:38 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 20:56:38 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f Feb 18 20:56:38 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 20:56:38 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 20:56:38 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 20:56:38 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:56:38 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:56:38 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:38 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:56:38 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 20:56:38 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 20:56:38 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:38 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:38 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 20:56:38 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 20:56:38 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:38 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:38 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:56:38 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:38 localhost kernel: in-hardirq-W at: Feb 18 20:56:38 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 20:56:38 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:38 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:38 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:38 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:38 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:38 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:38 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:38 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 20:56:38 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 20:56:38 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 20:56:38 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 20:56:38 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 20:56:38 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 20:56:39 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:56:39 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 20:56:39 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:56:39 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 20:56:39 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 20:56:39 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 20:56:39 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:39 localhost kernel: in-softirq-W at: Feb 18 20:56:39 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:56:39 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:39 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:39 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:39 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:39 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:39 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 20:56:39 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 20:56:39 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 20:56:39 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:56:39 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:56:39 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:39 localhost kernel: } Feb 18 20:56:39 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488 Feb 18 20:56:39 localhost kernel: ... acquired at: Feb 18 20:56:39 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:56:39 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:39 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:39 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:39 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:39 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:39 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:56:39 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 20:56:39 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:56:39 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 20:56:39 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:56:39 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 20:56:39 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 20:56:39 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:56:39 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:39 localhost kernel: Feb 18 20:56:39 localhost kernel: -> (&rq->rq_lock_key#3){++..} ops: 250064 { Feb 18 20:56:39 localhost kernel: initial-use at: Feb 18 20:56:40 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:56:40 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:40 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:40 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:56:40 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:40 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:40 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 20:56:40 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 20:56:40 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f Feb 18 20:56:40 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 20:56:40 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 20:56:40 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 20:56:40 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:56:40 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:56:40 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:40 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:56:40 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 20:56:40 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 20:56:40 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:40 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:40 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 20:56:40 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 20:56:40 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:40 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:56:40 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:56:40 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:40 localhost kernel: in-hardirq-W at: Feb 18 20:56:40 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 20:56:40 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:40 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:56:40 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:40 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:40 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:40 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:40 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:56:40 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 20:56:40 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 20:56:40 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 20:56:40 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 20:56:40 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 20:56:40 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 20:56:41 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:56:41 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 20:56:41 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:56:41 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 20:56:41 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 20:56:41 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 20:56:41 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:41 localhost kernel: in-softirq-W at: Feb 18 20:56:41 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 20:56:41 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:56:41 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:41 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40 Feb 18 20:56:41 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:56:41 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40 Feb 18 20:56:41 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40 Feb 18 20:56:41 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 20:56:41 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 20:56:41 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 20:56:41 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:56:41 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:56:41 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:56:41 localhost kernel: } Feb 18 20:56:41 localhost kernel: ... key at: [<c2a34488>] 0xc2a34488 Feb 18 20:56:41 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 233519 { Feb 18 20:56:41 localhost kernel: initial-use at: Feb 18 20:56:41 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:56:41 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:56:41 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:41 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:56:41 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:41 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:56:41 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 20:56:41 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 20:56:41 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f Feb 18 20:56:41 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 20:56:41 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 20:56:41 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 20:56:41 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:56:42 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:56:43 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:56:48 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:56:53 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 20:56:58 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 20:57:03 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:09 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:13 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 20:57:18 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 20:57:21 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:25 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:26 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:27 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:27 localhost kernel: in-hardirq-W at: Feb 18 20:57:27 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 20:57:27 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:27 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:27 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:27 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:27 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:27 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:27 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:27 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 20:57:27 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 20:57:27 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 20:57:27 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 20:57:27 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 20:57:27 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 20:57:27 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:57:27 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 20:57:27 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:57:27 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 20:57:27 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 20:57:27 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 20:57:27 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:27 localhost kernel: in-softirq-W at: Feb 18 20:57:27 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:57:27 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:27 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:27 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:27 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:28 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 20:57:28 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 20:57:28 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 20:57:28 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:57:28 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:57:28 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:28 localhost kernel: } Feb 18 20:57:28 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488 Feb 18 20:57:28 localhost kernel: ... acquired at: Feb 18 20:57:28 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:28 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:28 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:28 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 20:57:28 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:57:28 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 20:57:28 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:57:28 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 20:57:28 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 20:57:28 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:28 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:28 localhost kernel: Feb 18 20:57:28 localhost kernel: ... acquired at: Feb 18 20:57:28 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:28 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:28 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:28 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 20:57:28 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:57:28 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 20:57:28 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:57:28 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 20:57:28 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 20:57:28 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:29 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:29 localhost kernel: Feb 18 20:57:29 localhost kernel: ... acquired at: Feb 18 20:57:29 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:57:29 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:29 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:29 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:29 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:29 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:29 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:29 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:29 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5 Feb 18 20:57:29 localhost kernel: [<c0421394>] __wake_up_common+0x32/0x5c Feb 18 20:57:29 localhost kernel: [<c04235fe>] complete+0x36/0x44 Feb 18 20:57:29 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:57:29 localhost kernel: [<c043d502>] kthread+0x1c/0x5e Feb 18 20:57:29 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 20:57:29 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:29 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:29 localhost kernel: Feb 18 20:57:29 localhost kernel: -> (&rq->rq_lock_key#2){++..} ops: 382046 { Feb 18 20:57:29 localhost kernel: initial-use at: Feb 18 20:57:29 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:57:29 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:29 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:29 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:57:29 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:29 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:29 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 20:57:29 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 20:57:29 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f Feb 18 20:57:29 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 20:57:29 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 20:57:29 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 20:57:29 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:57:29 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:57:29 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:29 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:57:29 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 20:57:29 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 20:57:29 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:29 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:30 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 20:57:30 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 20:57:30 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:30 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:30 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:30 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:30 localhost kernel: in-hardirq-W at: Feb 18 20:57:30 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 20:57:30 localhost kernel: [<c040a9d6>] save_stack_trace+0x20/0x3a Feb 18 20:57:30 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 20:57:30 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:30 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:30 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:30 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:30 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:30 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:30 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:30 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 20:57:30 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 20:57:30 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 20:57:30 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 20:57:30 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 20:57:30 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 20:57:30 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:57:30 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 20:57:30 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:57:30 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 20:57:30 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 20:57:30 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 20:57:30 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:30 localhost kernel: in-softirq-W at: Feb 18 20:57:30 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:57:30 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21 Feb 18 20:57:30 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3 Feb 18 20:57:30 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:30 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:30 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:30 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:30 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:30 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:30 localhost kernel: [<c0433efb>] process_timeout+0x0/0x5 Feb 18 20:57:31 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5 Feb 18 20:57:31 localhost kernel: [<c0433d59>] run_timer_softirq+0x113/0x188 Feb 18 20:57:31 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 20:57:31 localhost kernel: [<c0433efb>] process_timeout+0x0/0x5 Feb 18 20:57:31 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 20:57:31 localhost kernel: [<c0433efb>] process_timeout+0x0/0x5 Feb 18 20:57:31 localhost kernel: [<c0433d67>] run_timer_softirq+0x121/0x188 Feb 18 20:57:31 localhost kernel: [<c0433efb>] process_timeout+0x0/0x5 Feb 18 20:57:31 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 20:57:31 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:57:31 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:57:31 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:31 localhost kernel: } Feb 18 20:57:31 localhost kernel: ... key at: [<c2a2b488>] 0xc2a2b488 Feb 18 20:57:31 localhost kernel: -> (&rq->rq_lock_key#3){++..} ops: 250064 { Feb 18 20:57:31 localhost kernel: initial-use at: Feb 18 20:57:31 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:57:31 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:31 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:31 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:57:31 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:31 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:31 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 20:57:31 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 20:57:31 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f Feb 18 20:57:31 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 20:57:31 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 20:57:31 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 20:57:31 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:57:31 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:57:31 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:31 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:57:31 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 20:57:31 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 20:57:31 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:31 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:31 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 20:57:31 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 20:57:31 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:31 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:31 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:32 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:32 localhost kernel: in-hardirq-W at: Feb 18 20:57:32 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 20:57:32 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:32 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:32 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:32 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:32 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:32 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:32 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:32 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 20:57:32 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 20:57:32 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 20:57:32 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 20:57:32 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 20:57:32 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 20:57:32 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:57:32 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 20:57:32 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:57:32 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 20:57:32 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 20:57:32 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 20:57:32 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:32 localhost kernel: in-softirq-W at: Feb 18 20:57:32 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 20:57:32 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:57:32 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:32 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40 Feb 18 20:57:32 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:32 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40 Feb 18 20:57:32 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40 Feb 18 20:57:32 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 20:57:32 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 20:57:32 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 20:57:32 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:57:32 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:57:32 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:32 localhost kernel: } Feb 18 20:57:32 localhost kernel: ... key at: [<c2a34488>] 0xc2a34488 Feb 18 20:57:32 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 233519 { Feb 18 20:57:32 localhost kernel: initial-use at: Feb 18 20:57:33 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:57:33 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:33 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:33 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:57:33 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:33 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:33 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 20:57:33 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 20:57:33 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f Feb 18 20:57:33 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 20:57:33 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 20:57:33 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 20:57:33 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:57:33 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:57:33 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:33 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:57:33 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 20:57:33 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 20:57:33 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:33 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:33 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 20:57:33 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 20:57:33 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:33 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:33 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:33 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:33 localhost kernel: in-hardirq-W at: Feb 18 20:57:33 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 20:57:33 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:33 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:33 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:33 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:33 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:33 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:33 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:33 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 20:57:33 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 20:57:33 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 20:57:33 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 20:57:33 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 20:57:33 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 20:57:33 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:57:33 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 20:57:34 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:57:34 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 20:57:34 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 20:57:34 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 20:57:34 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:34 localhost kernel: in-softirq-W at: Feb 18 20:57:34 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:57:34 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:34 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:34 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 20:57:34 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 20:57:34 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 20:57:34 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:57:34 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:57:34 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:34 localhost kernel: } Feb 18 20:57:34 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488 Feb 18 20:57:34 localhost kernel: ... acquired at: Feb 18 20:57:34 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:34 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:34 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:34 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 20:57:34 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:57:34 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 20:57:34 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:57:34 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 20:57:34 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 20:57:34 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:34 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:34 localhost kernel: Feb 18 20:57:34 localhost kernel: ... acquired at: Feb 18 20:57:34 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:34 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:35 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:35 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:35 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:35 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 20:57:35 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:57:35 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 20:57:35 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:57:35 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 20:57:35 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 20:57:35 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:35 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:35 localhost kernel: Feb 18 20:57:35 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 233519 { Feb 18 20:57:35 localhost kernel: initial-use at: Feb 18 20:57:35 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:57:35 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:35 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:35 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:57:35 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:35 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:35 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 20:57:35 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 20:57:35 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f Feb 18 20:57:35 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 20:57:35 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 20:57:35 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 20:57:35 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:57:35 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:57:35 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:35 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:57:35 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 20:57:35 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 20:57:35 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:35 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:35 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 20:57:35 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 20:57:35 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:35 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:35 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:35 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:35 localhost kernel: in-hardirq-W at: Feb 18 20:57:35 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 20:57:36 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:36 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:36 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:36 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:36 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:36 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:36 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:36 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 20:57:36 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 20:57:36 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 20:57:36 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 20:57:36 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 20:57:36 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 20:57:36 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:57:36 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 20:57:36 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:57:36 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 20:57:36 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 20:57:36 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 20:57:36 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:36 localhost kernel: in-softirq-W at: Feb 18 20:57:36 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:57:36 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:36 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:36 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 20:57:36 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 20:57:36 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 20:57:36 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:57:36 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:57:36 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:36 localhost kernel: } Feb 18 20:57:36 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488 Feb 18 20:57:36 localhost kernel: ... acquired at: Feb 18 20:57:36 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:57:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:36 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:37 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:37 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:37 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:37 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 20:57:37 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:57:37 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 20:57:37 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:57:37 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 20:57:37 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 20:57:37 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:37 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:37 localhost kernel: Feb 18 20:57:37 localhost kernel: ... acquired at: Feb 18 20:57:37 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:57:37 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:37 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:37 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:37 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:37 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:37 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:37 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5 Feb 18 20:57:37 localhost kernel: [<c0421394>] __wake_up_common+0x32/0x5c Feb 18 20:57:37 localhost kernel: [<c04235fe>] complete+0x36/0x44 Feb 18 20:57:37 localhost kernel: [<c043aad8>] worker_thread+0x0/0xc4 Feb 18 20:57:37 localhost kernel: [<c043d502>] kthread+0x1c/0x5e Feb 18 20:57:37 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 20:57:37 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:37 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:37 localhost kernel: Feb 18 20:57:37 localhost kernel: -> (&rq->rq_lock_key#3){++..} ops: 250064 { Feb 18 20:57:37 localhost kernel: initial-use at: Feb 18 20:57:37 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:57:37 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:37 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:37 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:57:37 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:37 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:37 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 20:57:37 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 20:57:37 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f Feb 18 20:57:38 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 20:57:38 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 20:57:38 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 20:57:38 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:57:38 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:57:38 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:38 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:57:38 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 20:57:38 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 20:57:38 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:38 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:38 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 20:57:38 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 20:57:38 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:38 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:38 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:38 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:38 localhost kernel: in-hardirq-W at: Feb 18 20:57:38 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 20:57:38 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:38 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:38 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:38 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:38 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:38 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:38 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:38 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 20:57:38 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 20:57:38 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 20:57:38 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 20:57:38 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 20:57:38 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 20:57:38 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:57:38 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 20:57:38 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:57:38 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 20:57:38 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 20:57:38 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 20:57:38 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:38 localhost kernel: in-softirq-W at: Feb 18 20:57:38 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 20:57:38 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:57:39 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:39 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40 Feb 18 20:57:39 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:39 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40 Feb 18 20:57:39 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40 Feb 18 20:57:39 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 20:57:39 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 20:57:39 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 20:57:39 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:57:39 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:57:39 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:39 localhost kernel: } Feb 18 20:57:39 localhost kernel: ... key at: [<c2a34488>] 0xc2a34488 Feb 18 20:57:39 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 233519 { Feb 18 20:57:39 localhost kernel: initial-use at: Feb 18 20:57:39 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:57:39 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:39 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:39 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:57:39 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:39 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:39 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 20:57:39 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 20:57:39 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f Feb 18 20:57:39 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 20:57:39 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 20:57:39 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 20:57:39 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:57:39 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:57:39 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:39 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:57:39 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 20:57:39 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 20:57:39 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:39 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:39 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 20:57:39 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 20:57:39 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:39 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:39 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:39 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:40 localhost kernel: in-hardirq-W at: Feb 18 20:57:40 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 20:57:40 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:40 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:40 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:40 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:40 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:40 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:40 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:40 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 20:57:40 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 20:57:40 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 20:57:40 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 20:57:40 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 20:57:40 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 20:57:40 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:57:40 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 20:57:40 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:57:40 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 20:57:40 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 20:57:40 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 20:57:40 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:40 localhost kernel: in-softirq-W at: Feb 18 20:57:40 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:57:40 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:40 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:40 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:40 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:40 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:40 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 20:57:40 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 20:57:40 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 20:57:40 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:57:40 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:57:40 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:40 localhost kernel: } Feb 18 20:57:40 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488 Feb 18 20:57:40 localhost kernel: ... acquired at: Feb 18 20:57:40 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:57:40 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:40 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:40 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:41 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:41 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:41 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:41 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 20:57:41 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:57:41 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 20:57:41 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:57:41 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 20:57:41 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 20:57:41 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:41 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:41 localhost kernel: Feb 18 20:57:41 localhost kernel: ... acquired at: Feb 18 20:57:41 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:57:41 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:41 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:41 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:41 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:41 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:41 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:41 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5 Feb 18 20:57:41 localhost kernel: [<c0421394>] __wake_up_common+0x32/0x5c Feb 18 20:57:41 localhost kernel: [<c04235fe>] complete+0x36/0x44 Feb 18 20:57:41 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:57:41 localhost kernel: [<c0428996>] migration_thread+0x186/0x1d2 Feb 18 20:57:41 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:57:41 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 20:57:41 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 20:57:41 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:41 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:41 localhost kernel: Feb 18 20:57:41 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 233519 { Feb 18 20:57:41 localhost kernel: initial-use at: Feb 18 20:57:41 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:57:41 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:41 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:41 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:57:41 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:41 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 20:57:41 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 20:57:41 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 20:57:42 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f Feb 18 20:57:42 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 20:57:42 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 20:57:42 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 20:57:42 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:57:42 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:57:42 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:42 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 20:57:42 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 20:57:42 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 20:57:42 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:42 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:42 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 20:57:42 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 20:57:42 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:42 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 20:57:42 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:42 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:42 localhost kernel: in-hardirq-W at: Feb 18 20:57:42 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 20:57:42 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:42 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:42 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:42 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:42 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:42 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:42 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 20:57:42 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 20:57:42 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 20:57:42 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 20:57:42 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 20:57:42 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 20:57:42 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 20:57:42 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:57:42 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 20:57:42 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 20:57:42 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 20:57:42 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 20:57:42 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 20:57:42 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:42 localhost kernel: in-softirq-W at: Feb 18 20:57:43 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:57:43 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:43 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:43 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:43 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:43 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 20:57:43 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 20:57:43 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 20:57:43 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 20:57:43 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:57:43 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:57:43 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:43 localhost kernel: } Feb 18 20:57:43 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488 Feb 18 20:57:43 localhost kernel: ... acquired at: Feb 18 20:57:43 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:57:43 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:43 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:43 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:43 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 20:57:43 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:43 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 20:57:43 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5 Feb 18 20:57:43 localhost kernel: [<c0421394>] __wake_up_common+0x32/0x5c Feb 18 20:57:43 localhost kernel: [<c04235fe>] complete+0x36/0x44 Feb 18 20:57:43 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:57:43 localhost kernel: [<c0428996>] migration_thread+0x186/0x1d2 Feb 18 20:57:43 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 20:57:43 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 20:57:43 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 20:57:43 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 20:57:43 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:43 localhost kernel: Feb 18 20:57:43 localhost kernel: ... acquired at: Feb 18 20:57:43 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:57:43 localhost kernel: [<c042366f>] __wake_up+0x18/0x42 Feb 18 20:57:43 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:43 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 20:57:43 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:43 localhost kernel: [<c042366f>] __wake_up+0x18/0x42 Feb 18 20:57:43 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:57:44 localhost kernel: [<c042366f>] __wake_up+0x18/0x42 Feb 18 20:57:44 localhost kernel: [<c042366f>] __wake_up+0x18/0x42 Feb 18 20:57:44 localhost kernel: [<f8c58a2b>] ppp_receive_nonmp_frame+0x51f/0x70a [ppp_generic] Feb 18 20:57:44 localhost kernel: [<f8c59487>] ppp_input+0xbf/0xef [ppp_generic] Feb 18 20:57:44 localhost kernel: [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp] Feb 18 20:57:44 localhost kernel: [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp] Feb 18 20:57:44 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c Feb 18 20:57:44 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259 Feb 18 20:57:44 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758 Feb 18 20:57:44 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8 Feb 18 20:57:44 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8 Feb 18 20:57:44 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338 Feb 18 20:57:44 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 20:57:44 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237 Feb 18 20:57:44 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4 Feb 18 20:57:44 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 20:57:44 localhost kernel: [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000] Feb 18 20:57:44 localhost kernel: [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000] Feb 18 20:57:44 localhost kernel: [<f8984f13>] e1000_clean+0x63/0x203 [e1000] Feb 18 20:57:44 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3 Feb 18 20:57:44 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3 Feb 18 20:57:44 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:57:44 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:57:44 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:44 localhost kernel: Feb 18 20:57:44 localhost kernel: ... acquired at: Feb 18 20:57:44 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:57:44 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic] Feb 18 20:57:44 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:44 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic] Feb 18 20:57:44 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 20:57:44 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic] Feb 18 20:57:44 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic] Feb 18 20:57:44 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 20:57:44 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:57:44 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:44 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 20:57:44 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8 Feb 18 20:57:44 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc Feb 18 20:57:44 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62 Feb 18 20:57:44 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249 Feb 18 20:57:45 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d Feb 18 20:57:45 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 20:57:45 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:45 localhost kernel: Feb 18 20:57:45 localhost kernel: -> (&list->lock#7){.+..} ops: 31 { Feb 18 20:57:45 localhost kernel: initial-use at: Feb 18 20:57:45 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:57:45 localhost kernel: [<c046b874>] __do_fault+0x31b/0x35d Feb 18 20:57:45 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:45 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 20:57:45 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:57:45 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 20:57:45 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 20:57:45 localhost kernel: [<f8c5939f>] ppp_write+0xb2/0xdb [ppp_generic] Feb 18 20:57:45 localhost kernel: [<f8c592ed>] ppp_write+0x0/0xdb [ppp_generic] Feb 18 20:57:45 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d Feb 18 20:57:45 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67 Feb 18 20:57:45 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 20:57:45 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:45 localhost kernel: in-softirq-W at: Feb 18 20:57:45 localhost kernel: [<c0446ab0>] save_trace+0x37/0x8b Feb 18 20:57:45 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 20:57:45 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:45 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:45 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 20:57:45 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:57:45 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 20:57:45 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 20:57:45 localhost kernel: [<f8c59433>] ppp_input+0x6b/0xef [ppp_generic] Feb 18 20:57:45 localhost kernel: [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp] Feb 18 20:57:45 localhost kernel: [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp] Feb 18 20:57:45 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c Feb 18 20:57:45 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259 Feb 18 20:57:45 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758 Feb 18 20:57:45 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8 Feb 18 20:57:45 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8 Feb 18 20:57:45 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338 Feb 18 20:57:45 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 20:57:45 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237 Feb 18 20:57:45 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4 Feb 18 20:57:45 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 20:57:45 localhost kernel: [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000] Feb 18 20:57:45 localhost kernel: [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000] Feb 18 20:57:46 localhost kernel: [<f8984f13>] e1000_clean+0x63/0x203 [e1000] Feb 18 20:57:46 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3 Feb 18 20:57:46 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3 Feb 18 20:57:46 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 20:57:46 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 20:57:46 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:46 localhost kernel: } Feb 18 20:57:46 localhost kernel: ... key at: [<f8c5dd18>] __key.21159+0x0/0xffffb82a [ppp_generic] Feb 18 20:57:46 localhost kernel: ... acquired at: Feb 18 20:57:46 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:57:46 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 20:57:46 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:46 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:46 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 20:57:46 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 20:57:46 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 20:57:46 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 20:57:46 localhost kernel: [<f8c57424>] ppp_xmit_process+0x529/0x5a1 [ppp_generic] Feb 18 20:57:46 localhost kernel: [<f8c5834b>] ppp_channel_push+0x71/0x90 [ppp_generic] Feb 18 20:57:46 localhost kernel: [<c061b373>] _read_lock_bh+0x2e/0x39 Feb 18 20:57:46 localhost kernel: [<f8c5835a>] ppp_channel_push+0x80/0x90 [ppp_generic] Feb 18 20:57:46 localhost kernel: [<f8c593bd>] ppp_write+0xd0/0xdb [ppp_generic] Feb 18 20:57:46 localhost kernel: [<f8c592ed>] ppp_write+0x0/0xdb [ppp_generic] Feb 18 20:57:46 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d Feb 18 20:57:46 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67 Feb 18 20:57:46 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 20:57:46 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:46 localhost kernel: Feb 18 20:57:46 localhost kernel: ... acquired at: Feb 18 20:57:46 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 20:57:46 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic] Feb 18 20:57:46 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:46 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic] Feb 18 20:57:46 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 20:57:46 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic] Feb 18 20:57:46 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic] Feb 18 20:57:46 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 20:57:46 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:57:46 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:46 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 20:57:46 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8 Feb 18 20:57:47 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc Feb 18 20:57:47 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62 Feb 18 20:57:47 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249 Feb 18 20:57:47 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d Feb 18 20:57:47 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 20:57:47 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:47 localhost kernel: Feb 18 20:57:47 localhost kernel: Feb 18 20:57:47 localhost kernel: the soft-irq-unsafe lock's dependencies: Feb 18 20:57:47 localhost kernel: -> (&sk->sk_dst_lock){----} ops: 284 { Feb 18 20:57:47 localhost kernel: initial-use at: Feb 18 20:57:47 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 20:57:47 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 20:57:47 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115 Feb 18 20:57:47 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb Feb 18 20:57:47 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:47 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 20:57:47 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34 Feb 18 20:57:47 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 20:57:47 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 20:57:47 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e Feb 18 20:57:47 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f Feb 18 20:57:47 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261 Feb 18 20:57:47 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5 Feb 18 20:57:47 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:47 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5 Feb 18 20:57:47 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:47 localhost kernel: softirq-on-W at: Feb 18 20:57:47 localhost kernel: [<c0449046>] __lock_acquire+0x48b/0xbf1 Feb 18 20:57:47 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 20:57:47 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115 Feb 18 20:57:47 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb Feb 18 20:57:47 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:47 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 20:57:47 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34 Feb 18 20:57:47 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 20:57:47 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 20:57:47 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e Feb 18 20:57:47 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f Feb 18 20:57:47 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261 Feb 18 20:57:47 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5 Feb 18 20:57:47 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:48 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5 Feb 18 20:57:48 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:48 localhost kernel: hardirq-on-W at: Feb 18 20:57:48 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1 Feb 18 20:57:48 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 20:57:48 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115 Feb 18 20:57:48 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb Feb 18 20:57:48 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:48 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 20:57:48 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34 Feb 18 20:57:48 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 20:57:48 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 20:57:48 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e Feb 18 20:57:48 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f Feb 18 20:57:48 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261 Feb 18 20:57:48 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5 Feb 18 20:57:48 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:48 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5 Feb 18 20:57:48 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:48 localhost kernel: softirq-on-R at: Feb 18 20:57:48 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:48 localhost kernel: [<c0449046>] __lock_acquire+0x48b/0xbf1 Feb 18 20:57:48 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:48 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:48 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105 Feb 18 20:57:48 localhost kernel: [<c061b3a7>] _read_lock+0x29/0x34 Feb 18 20:57:48 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105 Feb 18 20:57:48 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105 Feb 18 20:57:48 localhost kernel: [<f8a6a9b1>] ipv6_chk_addr+0x8a/0x96 [ipv6] Feb 18 20:57:48 localhost kernel: [<f8a649b0>] ip6_sk_dst_lookup+0x2d/0x17a [ipv6] Feb 18 20:57:48 localhost kernel: [<c0489d61>] __pollwait+0x0/0xac Feb 18 20:57:48 localhost kernel: [<f8a75e6d>] udpv6_sendmsg+0x4c1/0x8a1 [ipv6] Feb 18 20:57:48 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8 Feb 18 20:57:48 localhost last message repeated 2 times Feb 18 20:57:48 localhost kernel: [<c05f5fc7>] inet_sendmsg+0x3b/0x45 Feb 18 20:57:48 localhost kernel: [<c05ab8cd>] sock_sendmsg+0xc9/0xe4 Feb 18 20:57:48 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35 Feb 18 20:57:48 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:48 localhost kernel: [<c04f05f4>] copy_from_user+0x32/0x5e Feb 18 20:57:49 localhost kernel: [<c04f05f4>] copy_from_user+0x32/0x5e Feb 18 20:57:49 localhost kernel: [<c05aba7a>] sys_sendmsg+0x192/0x1f7 Feb 18 20:57:49 localhost kernel: [<c048f764>] file_update_time+0x22/0x6a Feb 18 20:57:49 localhost kernel: [<c0483d90>] pipe_write+0x392/0x3ef Feb 18 20:57:49 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:49 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35 Feb 18 20:57:49 localhost kernel: [<c05acd2b>] sys_socketcall+0x240/0x261 Feb 18 20:57:49 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 20:57:49 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:49 localhost kernel: hardirq-on-R at: Feb 18 20:57:49 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:49 localhost kernel: [<c0449001>] __lock_acquire+0x446/0xbf1 Feb 18 20:57:49 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:49 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:49 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105 Feb 18 20:57:49 localhost kernel: [<c061b3a7>] _read_lock+0x29/0x34 Feb 18 20:57:49 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105 Feb 18 20:57:49 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105 Feb 18 20:57:49 localhost kernel: [<f8a6a9b1>] ipv6_chk_addr+0x8a/0x96 [ipv6] Feb 18 20:57:49 localhost kernel: [<f8a649b0>] ip6_sk_dst_lookup+0x2d/0x17a [ipv6] Feb 18 20:57:49 localhost kernel: [<c0489d61>] __pollwait+0x0/0xac Feb 18 20:57:49 localhost kernel: [<f8a75e6d>] udpv6_sendmsg+0x4c1/0x8a1 [ipv6] Feb 18 20:57:49 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8 Feb 18 20:57:49 localhost last message repeated 2 times Feb 18 20:57:49 localhost kernel: [<c05f5fc7>] inet_sendmsg+0x3b/0x45 Feb 18 20:57:49 localhost kernel: [<c05ab8cd>] sock_sendmsg+0xc9/0xe4 Feb 18 20:57:49 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35 Feb 18 20:57:49 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:49 localhost kernel: [<c04f05f4>] copy_from_user+0x32/0x5e Feb 18 20:57:49 localhost kernel: [<c04f05f4>] copy_from_user+0x32/0x5e Feb 18 20:57:49 localhost kernel: [<c05aba7a>] sys_sendmsg+0x192/0x1f7 Feb 18 20:57:49 localhost kernel: [<c048f764>] file_update_time+0x22/0x6a Feb 18 20:57:49 localhost kernel: [<c0483d90>] pipe_write+0x392/0x3ef Feb 18 20:57:49 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 20:57:49 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35 Feb 18 20:57:49 localhost kernel: [<c05acd2b>] sys_socketcall+0x240/0x261 Feb 18 20:57:49 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 20:57:49 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 20:57:49 localhost kernel: } Feb 18 20:57:49 localhost kernel: ... key at: [<c0a3dc28>] __key.35996+0x0/0x8 Feb 18 20:57:49 localhost kernel: Feb 18 20:57:49 localhost kernel: stack backtrace: Feb 18 20:57:50 localhost kernel: Pid: 3241, comm: pppd Not tainted 2.6.24.2 #1 Feb 18 20:57:50 localhost kernel: [<c0448b1e>] check_usage+0x24e/0x258 Feb 18 20:57:50 localhost kernel: [<c0449555>] __lock_acquire+0x99a/0xbf1 Feb 18 20:57:50 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 20:57:50 localhost kernel: [<f8c566db>] ppp_push+0x63/0x50d [ppp_generic] Feb 18 20:57:50 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 20:57:50 localhost kernel: [<f8c566db>] ppp_push+0x63/0x50d [ppp_generic] Feb 18 20:57:50 localhost kernel: [<f8c566db>] ppp_push+0x63/0x50d [ppp_generic] Feb 18 20:57:50 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 20:57:50 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 20:57:50 localhost kernel: [<f8c573f1>] ppp_xmit_process+0x4f6/0x5a1 [ppp_generic] Feb 18 20:57:50 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 20:57:50 localhost kernel: [<f8c593b4>] ppp_write+0xc7/0xdb [ppp_generic] Feb 18 20:57:50 localhost kernel: [<f8c592ed>] ppp_write+0x0/0xdb [ppp_generic] Feb 18 20:57:50 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d Feb 18 20:57:50 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67 Feb 18 20:57:50 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 20:57:50 localhost kernel: ======================= > > Thanks, > Jarek P. > > On Tue, Feb 12, 2008 at 10:58:21AM +0000, James Chapman wrote: > ... >> Here is a trace from when we had _bh locks. >> >> Feb 5 16:26:32 ====================================================== >> Feb 5 16:26:32 [ INFO: soft-safe -> soft-unsafe lock order detected ] >> Feb 5 16:26:32 2.6.24-core2 #1 >> Feb 5 16:26:32 ------------------------------------------------------ >> Feb 5 16:26:32 pppd/3224 [HC0[0]:SC0[2]:HE1:SE0] is trying to acquire: > ... >> Feb 5 16:26:32 [<f8d7b84d>] e1000_clean+0x5d/0x290 [e1000] >> Feb 5 16:26:32 [<c039d580>] net_rx_action+0x1a0/0x2a0 >> Feb 5 16:26:32 [<c039d43f>] net_rx_action+0x5f/0x2a0 >> Feb 5 16:26:32 [<c0131e72>] __do_softirq+0x92/0x120 >> Feb 5 16:26:32 [<c0131f78>] do_softirq+0x78/0x80 >> Feb 5 16:26:32 [<c010b15a>] do_IRQ+0x4a/0xa0 >> Feb 5 16:26:32 [<c0127af0>] finish_task_switch+0x0/0xc0 >> Feb 5 16:26:32 [<c0108dcc>] common_interrupt+0x24/0x34 >> Feb 5 16:26:32 [<c0108dd6>] common_interrupt+0x2e/0x34 >> Feb 5 16:26:32 [<c01062d6>] mwait_idle_with_hints+0x46/0x60 >> Feb 5 16:26:32 [<c0106550>] mwait_idle+0x0/0x20 >> Feb 5 16:26:32 [<c0106694>] cpu_idle+0x74/0xe0 >> Feb 5 16:26:32 [<c0536a9a>] start_kernel+0x30a/0x3a0 -- James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-18 22:09 ` James Chapman @ 2008-02-18 23:01 ` Jarek Poplawski 2008-02-19 9:09 ` James Chapman 2008-02-19 4:29 ` David Miller 2008-02-19 23:06 ` Jarek Poplawski 2 siblings, 1 reply; 51+ messages in thread From: Jarek Poplawski @ 2008-02-18 23:01 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, netdev On Mon, Feb 18, 2008 at 10:09:24PM +0000, James Chapman wrote: > Jarek Poplawski wrote: >> Hi, >> >> It seems, this nice report is still uncomplete: could you check if >> there could have been something more yet? > > Unfortunately the ISP's syslog stops. But I've been able to borrow two > Quad Xeon boxes and have reproduced the problem. > > Here's a new version of the patch. The patch avoids disabling irqs and > fixes the sk_dst_get() usage that DaveM mentioned. But even with this > patch, lockdep still complains if hundreds of ppp sessions are inserted > into a tunnel as rapidly as possible (lockdep trace is below). I can > stop these errors by wrapping the call to ppp_input() in > pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is a > better fix? Hmm... This is a really long report and quite a bit different from the previous one. I need some time for this. BTW: you sent before a lockdep report with hlist_lock problem. I think this could be fixed in some independent patch to make this all more readable. Are all the other changes in this current patch only because of this or previous lockdep report or for some other reasons (or reports) yet? Regards, Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-18 23:01 ` Jarek Poplawski @ 2008-02-19 9:09 ` James Chapman 0 siblings, 0 replies; 51+ messages in thread From: James Chapman @ 2008-02-19 9:09 UTC (permalink / raw) To: Jarek Poplawski; +Cc: David Miller, netdev Jarek Poplawski wrote: > On Mon, Feb 18, 2008 at 10:09:24PM +0000, James Chapman wrote: >> Jarek Poplawski wrote: >>> Hi, >>> >>> It seems, this nice report is still uncomplete: could you check if >>> there could have been something more yet? >> Unfortunately the ISP's syslog stops. But I've been able to borrow two >> Quad Xeon boxes and have reproduced the problem. >> >> Here's a new version of the patch. The patch avoids disabling irqs and >> fixes the sk_dst_get() usage that DaveM mentioned. But even with this >> patch, lockdep still complains if hundreds of ppp sessions are inserted >> into a tunnel as rapidly as possible (lockdep trace is below). I can >> stop these errors by wrapping the call to ppp_input() in >> pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is a >> better fix? > > Hmm... This is a really long report and quite a bit different from > the previous one. I need some time for this. BTW: you sent before a > lockdep report with hlist_lock problem. I think this could be fixed > in some independent patch to make this all more readable. Are all > the other changes in this current patch only because of this or > previous lockdep report or for some other reasons (or reports) yet? As I mentioned in my reply to davem, modifying the pppol2tp driver as described in the patch I sent made the original lockdep problems go away. -- James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-18 22:09 ` James Chapman 2008-02-18 23:01 ` Jarek Poplawski @ 2008-02-19 4:29 ` David Miller 2008-02-19 9:03 ` James Chapman 2008-02-19 23:06 ` Jarek Poplawski 2 siblings, 1 reply; 51+ messages in thread From: David Miller @ 2008-02-19 4:29 UTC (permalink / raw) To: jchapman; +Cc: jarkao2, netdev From: James Chapman <jchapman@katalix.com> Date: Mon, 18 Feb 2008 22:09:24 +0000 > Here's a new version of the patch. The patch avoids disabling irqs > and fixes the sk_dst_get() usage that DaveM mentioned. But even with > this patch, lockdep still complains if hundreds of ppp sessions are > inserted into a tunnel as rapidly as possible (lockdep trace is > below). I can stop these errors by wrapping the call to ppp_input() > in pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is > a better fix? Firstly, let's fix one thing at a time. Leave the sk_dst_get() thing alone until we can prove that it's part of the lockdep traces. Next, I can't see why ppp_input() needs to be invoked with interrupts disabled. There are many other things that invoke that in software interrupt context, such as pppoe. Please provide the lockdep traces without the ppp_input() IRQ disabling so this can be properly analyzed. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-19 4:29 ` David Miller @ 2008-02-19 9:03 ` James Chapman 2008-02-19 10:30 ` Jarek Poplawski 0 siblings, 1 reply; 51+ messages in thread From: James Chapman @ 2008-02-19 9:03 UTC (permalink / raw) To: David Miller; +Cc: jarkao2, netdev David Miller wrote: > From: James Chapman <jchapman@katalix.com> > Date: Mon, 18 Feb 2008 22:09:24 +0000 > >> Here's a new version of the patch. The patch avoids disabling irqs >> and fixes the sk_dst_get() usage that DaveM mentioned. But even with >> this patch, lockdep still complains if hundreds of ppp sessions are >> inserted into a tunnel as rapidly as possible (lockdep trace is >> below). I can stop these errors by wrapping the call to ppp_input() >> in pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is >> a better fix? > > Firstly, let's fix one thing at a time. Leave the sk_dst_get() > thing alone until we can prove that it's part of the lockdep > traces. In reproducing the problem, I obtained several lockdep traces that implicated sk_dst_get(). I changed the code to use __sk_dst_check() as you suggested and they went away. At that point, I was hopeful the locking issues were fixed. But after several minutes of creating/deleting hundreds of ppp sessions, lockdep dumped another error. It is that error that I posted yesterday. > Next, I can't see why ppp_input() needs to be invoked with > interrupts disabled. There are many other things that invoke > that in software interrupt context, such as pppoe. I agree. I'm seeking advice on what the underlying cause is of this new trace. > Please provide the lockdep traces without the ppp_input() IRQ > disabling so this can be properly analyzed. The trace _was_ without ppp_input IRQ disabling. The trace doesn't occur if I disable IRQs around the ppp_input() call. The patch I sent showed the changes I made before running the tests that created the new lockdep trace. I'm sorry this wasn't clear. -- James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-19 9:03 ` James Chapman @ 2008-02-19 10:30 ` Jarek Poplawski 2008-02-19 10:36 ` Jarek Poplawski 2008-02-19 14:37 ` James Chapman 0 siblings, 2 replies; 51+ messages in thread From: Jarek Poplawski @ 2008-02-19 10:30 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, netdev On Tue, Feb 19, 2008 at 09:03:12AM +0000, James Chapman wrote: > David Miller wrote: >> From: James Chapman <jchapman@katalix.com> >> Date: Mon, 18 Feb 2008 22:09:24 +0000 >> >>> Here's a new version of the patch. The patch avoids disabling irqs >>> and fixes the sk_dst_get() usage that DaveM mentioned. But even with >>> this patch, lockdep still complains if hundreds of ppp sessions are >>> inserted into a tunnel as rapidly as possible (lockdep trace is >>> below). I can stop these errors by wrapping the call to ppp_input() >>> in pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is >>> a better fix? >> >> Firstly, let's fix one thing at a time. Leave the sk_dst_get() >> thing alone until we can prove that it's part of the lockdep >> traces. > > In reproducing the problem, I obtained several lockdep traces that > implicated sk_dst_get(). As a matter of fact I missed just that kind information on previous lockdep report, so if you could send them too this should be still helpful. ... > I agree. I'm seeking advice on what the underlying cause is of this new > trace. IMHO, just like I wrote earlier, the main problem is in ppp_generic(), especially ppp_connect_channel(), where main tx & rx locks are used. I didn't know enough about this sk_dst_lock traces yet. I hope I could help with this, but after these changes I need some time to figure this out again. Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-19 10:30 ` Jarek Poplawski @ 2008-02-19 10:36 ` Jarek Poplawski 2008-02-19 14:37 ` James Chapman 1 sibling, 0 replies; 51+ messages in thread From: Jarek Poplawski @ 2008-02-19 10:36 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, netdev On Tue, Feb 19, 2008 at 10:30:47AM +0000, Jarek Poplawski wrote: ... > IMHO, just like I wrote earlier, the main problem is in ppp_generic(), ...or maybe ppp_generic.c? Whatever... Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-19 10:30 ` Jarek Poplawski 2008-02-19 10:36 ` Jarek Poplawski @ 2008-02-19 14:37 ` James Chapman 1 sibling, 0 replies; 51+ messages in thread From: James Chapman @ 2008-02-19 14:37 UTC (permalink / raw) To: Jarek Poplawski; +Cc: David Miller, netdev [-- Attachment #1: Type: text/plain, Size: 1402 bytes --] Jarek Poplawski wrote: > On Tue, Feb 19, 2008 at 09:03:12AM +0000, James Chapman wrote: >> David Miller wrote: >>> From: James Chapman <jchapman@katalix.com> >>> Date: Mon, 18 Feb 2008 22:09:24 +0000 >>> >>>> Here's a new version of the patch. The patch avoids disabling irqs >>>> and fixes the sk_dst_get() usage that DaveM mentioned. But even with >>>> this patch, lockdep still complains if hundreds of ppp sessions are >>>> inserted into a tunnel as rapidly as possible (lockdep trace is >>>> below). I can stop these errors by wrapping the call to ppp_input() >>>> in pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is >>>> a better fix? >>> Firstly, let's fix one thing at a time. Leave the sk_dst_get() >>> thing alone until we can prove that it's part of the lockdep >>> traces. >> In reproducing the problem, I obtained several lockdep traces that >> implicated sk_dst_get(). > > As a matter of fact I missed just that kind information on previous > lockdep report, so if you could send them too this should be still > helpful. Here you go. Just to be clear, the initial lockdep report was with non-bh locks. Yesterday's report was with _bh locks and the sk_dst_get() changes. This (older) report is with _bh locks but without the sk_dst_get() changes. -- James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development [-- Attachment #2: lockdep-sk_dst.log --] [-- Type: text/plain, Size: 205001 bytes --] Feb 18 17:25:42 localhost kernel: ====================================================== Feb 18 17:25:42 localhost kernel: [ INFO: soft-safe -> soft-unsafe lock order detected ] Feb 18 17:25:42 localhost kernel: 2.6.24.2 #1 Feb 18 17:25:42 localhost kernel: ------------------------------------------------------ Feb 18 17:25:42 localhost kernel: pppd/13863 [HC0[0]:SC0[1]:HE1:SE0] is trying to acquire: Feb 18 17:25:42 localhost kernel: (&sk->sk_dst_lock){----}, at: [<f8bbf73e>] pppol2tp_xmit+0x3a5/0x4a2 [pppol2tp] Feb 18 17:25:42 localhost kernel: Feb 18 17:25:42 localhost kernel: and this task is already holding: Feb 18 17:25:42 localhost kernel: (&pch->downl){-...}, at: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic] Feb 18 17:25:42 localhost kernel: which would create a new lock dependency: Feb 18 17:25:42 localhost pppd[13863]: Connection terminated. Feb 18 17:25:42 localhost kernel: (&pch->downl){-...} -> (&sk->sk_dst_lock){----} Feb 18 17:25:42 localhost pppd[13863]: Exit. Feb 18 17:25:42 localhost kernel: Feb 18 17:25:42 localhost kernel: but this new dependency connects a soft-irq-safe lock: Feb 18 17:25:42 localhost kernel: (&pch->upl){-.-+} Feb 18 17:25:42 localhost kernel: ... which became soft-irq-safe at: Feb 18 17:25:43 localhost kernel: [<c0448013>] check_usage_backwards+0x19/0x41 Feb 18 17:25:43 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:25:43 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:43 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:43 localhost kernel: [<f8bb840d>] ppp_input+0x45/0xef [ppp_generic] Feb 18 17:25:43 localhost kernel: [<c061b373>] _read_lock_bh+0x2e/0x39 Feb 18 17:25:43 localhost kernel: [<f8bb840d>] ppp_input+0x45/0xef [ppp_generic] Feb 18 17:25:43 localhost kernel: [<f8bb840d>] ppp_input+0x45/0xef [ppp_generic] Feb 18 17:25:43 localhost kernel: [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp] Feb 18 17:25:43 localhost kernel: [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp] Feb 18 17:25:43 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c Feb 18 17:25:43 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259 Feb 18 17:25:43 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758 Feb 18 17:25:43 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8 Feb 18 17:25:43 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8 Feb 18 17:25:43 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338 Feb 18 17:25:43 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 17:25:43 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237 Feb 18 17:25:43 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4 Feb 18 17:25:43 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 17:25:43 localhost kernel: [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000] Feb 18 17:25:43 localhost kernel: [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000] Feb 18 17:25:43 localhost kernel: [<f8960f13>] e1000_clean+0x63/0x203 [e1000] Feb 18 17:25:43 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3 Feb 18 17:25:43 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3 Feb 18 17:25:43 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:25:43 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:25:43 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:43 localhost kernel: Feb 18 17:25:43 localhost kernel: to a soft-irq-unsafe lock: Feb 18 17:25:43 localhost kernel: (&sk->sk_dst_lock){----} Feb 18 17:25:43 localhost kernel: ... which became soft-irq-unsafe at: Feb 18 17:25:43 localhost kernel: ... [<c0449046>] __lock_acquire+0x48b/0xbf1 Feb 18 17:25:43 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 17:25:43 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115 Feb 18 17:25:43 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb Feb 18 17:25:43 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:43 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 17:25:43 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34 Feb 18 17:25:43 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 17:25:43 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 17:25:43 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e Feb 18 17:25:44 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f Feb 18 17:25:44 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261 Feb 18 17:25:44 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5 Feb 18 17:25:44 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:25:44 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5 Feb 18 17:25:44 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:44 localhost kernel: Feb 18 17:25:44 localhost kernel: other info that might help us debug this: Feb 18 17:25:44 localhost kernel: Feb 18 17:25:44 localhost kernel: 1 lock held by pppd/13863: Feb 18 17:25:44 localhost kernel: #0: (&pch->downl){-...}, at: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic] Feb 18 17:25:44 localhost kernel: Feb 18 17:25:44 localhost kernel: the soft-irq-safe lock's dependencies: Feb 18 17:25:44 localhost kernel: -> (&pch->upl){-.-+} ops: 13 { Feb 18 17:25:44 localhost kernel: initial-use at: Feb 18 17:25:44 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:25:44 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:25:44 localhost kernel: [<f8bb6b7c>] ppp_ioctl+0x4df/0xc06 [ppp_generic] Feb 18 17:25:44 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:44 localhost kernel: [<f8bb6ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic] Feb 18 17:25:44 localhost kernel: [<c061b300>] _write_lock_bh+0x2e/0x39 Feb 18 17:25:44 localhost kernel: [<f8bb6ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic] Feb 18 17:25:44 localhost kernel: [<f8bb6ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic] Feb 18 17:25:44 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 17:25:44 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:25:44 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:25:44 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 17:25:44 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8 Feb 18 17:25:44 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc Feb 18 17:25:44 localhost kernel: [<c0488854>] do_ioctl+0x4ebalance_domains+0x28/0x99 Feb 18 17:25:44 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 17:25:44 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:25:44 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:25:44 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:44 localhost kernel: } Feb 18 17:25:44 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488 Feb 18 17:25:44 localhost kernel: ... acquired at: Feb 18 17:25:44 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:25:44 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:44 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:45 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:45 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:45 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:45 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:45 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 17:25:45 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:25:45 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 17:25:45 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:25:45 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 17:25:45 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 17:25:45 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:25:45 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:45 localhost kernel: Feb 18 17:25:45 localhost kernel: ... acquired at: Feb 18 17:25:45 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:25:45 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:45 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:45 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:45 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:45 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:45 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:45 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 17:25:45 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:25:45 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 17:25:45 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:25:45 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 17:25:45 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 17:25:45 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:25:45 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:45 localhost kernel: Feb 18 17:25:45 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 1787266 { Feb 18 17:25:45 localhost kernel: initial-use at: Feb 18 17:25:45 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:25:45 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:45 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:45 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:25:45 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:45 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:45 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 17:25:45 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 17:25:45 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21 Feb 18 17:25:46 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 17:25:46 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 17:25:46 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 17:25:46 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:25:46 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:25:46 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:25:46 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:25:46 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 17:25:46 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 17:25:46 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:25:46 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:46 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 17:25:46 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 17:25:46 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:46 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:46 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:25:46 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:46 localhost kernel: in-hardirq-W at: Feb 18 17:25:46 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 17:25:46 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:46 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:46 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:46 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:46 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:46 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:46 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:46 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 17:25:46 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 17:25:46 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 17:25:46 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 17:25:46 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 17:25:46 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 17:25:46 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:25:46 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 17:25:46 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:25:46 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 17:25:46 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 17:25:46 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 17:25:46 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:46 localhost kernel: in-softirq-W at: Feb 18 17:25:46 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 17:25:46 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:25:47 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:47 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:47 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 17:25:47 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 17:25:47 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 17:25:47 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:25:47 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:25:47 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:47 localhost kernel: } Feb 18 17:25:47 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488 Feb 18 17:25:47 localhost kernel: ... acquired at: Feb 18 17:25:47 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:47 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:47 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:47 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 17:25:47 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:25:47 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 17:25:47 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:25:47 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 17:25:47 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 17:25:47 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:25:47 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:47 localhost kernel: Feb 18 17:25:47 localhost kernel: ... acquired at: Feb 18 17:25:47 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:47 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:47 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:47 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 17:25:47 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 17:25:48 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 17:25:48 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:25:48 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:25:48 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:48 localhost kernel: Feb 18 17:25:48 localhost kernel: -> (&rq->rq_lock_key#3){++..} ops: 1928539 { Feb 18 17:25:48 localhost kernel: initial-use at: Feb 18 17:25:48 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:25:48 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:48 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:48 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:25:48 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:48 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:48 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 17:25:48 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 17:25:48 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21 Feb 18 17:25:48 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 17:25:48 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 17:25:48 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 17:25:48 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:25:48 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:25:48 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:25:48 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:25:48 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 17:25:48 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 17:25:48 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:25:48 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:48 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 17:25:48 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 17:25:48 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:48 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:48 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:25:48 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:48 localhost kernel: in-hardirq-W at: Feb 18 17:25:48 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 17:25:48 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:48 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:48 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:48 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:48 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:48 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:48 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:48 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 17:25:49 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 17:25:49 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 17:25:49 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 17:25:49 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 17:25:49 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 17:25:49 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:25:49 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 17:25:49 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:25:49 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 17:25:49 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 17:25:49 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 17:25:49 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:49 localhost kernel: in-softirq-W at: Feb 18 17:25:49 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21 Feb 18 17:25:49 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3 Feb 18 17:25:49 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:25:49 localhost kernel: [<c04422f1>] getnstimeofday+0x30/0xbc Feb 18 17:25:49 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:49 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:49 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:25:49 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:49 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:25:49 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:25:49 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5 Feb 18 17:25:49 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 17:25:49 localhost kernel: [<c0430afa>] __do_softirq+0xc9/0xde Feb 18 17:25:49 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:25:49 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:49 localhost kernel: } Feb 18 17:25:49 localhost kernel: ... key at: [<c2a34488>] 0xc2a34488 Feb 18 17:25:49 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 1787266 { Feb 18 17:25:49 localhost kernel: initial-use at: Feb 18 17:25:49 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:25:49 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:49 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:49 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:25:49 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:49 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:49 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 17:25:50 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 17:25:50 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21 Feb 18 17:25:50 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 17:25:50 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 17:25:50 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 17:25:50 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:25:50 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:25:50 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:25:50 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:25:50 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 17:25:50 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 17:25:50 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:25:50 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:50 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 17:25:50 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 17:25:50 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:50 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:50 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:25:50 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:50 localhost kernel: in-hardirq-W at: Feb 18 17:25:50 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 17:25:50 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:50 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:50 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:50 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:50 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:50 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:50 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:50 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 17:25:50 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 17:25:50 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 17:25:50 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 17:25:50 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 17:25:50 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 17:25:50 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:25:50 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 17:25:50 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:25:50 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 17:25:50 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 17:25:50 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 17:25:50 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:51 localhost kernel: in-softirq-W at: Feb 18 17:25:51 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 17:25:51 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:25:51 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:51 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:51 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 17:25:51 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 17:25:51 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 17:25:51 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:25:51 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:25:51 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:51 localhost kernel: } Feb 18 17:25:51 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488 Feb 18 17:25:51 localhost kernel: ... acquired at: Feb 18 17:25:51 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:51 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:51 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:51 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 17:25:51 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:25:51 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 17:25:51 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:25:51 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 17:25:51 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 17:25:51 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:25:51 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:51 localhost kernel: Feb 18 17:25:51 localhost kernel: ... acquired at: Feb 18 17:25:51 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:51 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:51 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:51 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 17:25:52 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 17:25:52 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 17:25:52 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:25:52 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:25:52 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:52 localhost kernel: Feb 18 17:25:52 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 1787266 { Feb 18 17:25:52 localhost kernel: initial-use at: Feb 18 17:25:52 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:25:52 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:52 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:52 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:25:52 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:52 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:52 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 17:25:52 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 17:25:52 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21 Feb 18 17:25:52 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 17:25:52 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 17:25:52 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 17:25:52 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:25:52 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:25:52 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:25:52 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:25:52 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 17:25:52 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 17:25:52 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:25:52 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:52 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 17:25:52 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 17:25:52 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:52 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:52 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:25:52 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:52 localhost kernel: in-hardirq-W at: Feb 18 17:25:52 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 17:25:52 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:52 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:52 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:53 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:53 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:53 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:53 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:53 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 17:25:53 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 17:25:53 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 17:25:53 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 17:25:53 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 17:25:53 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 17:25:53 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:25:53 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 17:25:53 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:25:53 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 17:25:53 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 17:25:53 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 17:25:53 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:53 localhost kernel: in-softirq-W at: Feb 18 17:25:53 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 17:25:53 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:25:53 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:53 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:53 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:53 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:53 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:53 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 17:25:53 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 17:25:53 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 17:25:53 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:25:53 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:25:53 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:53 localhost kernel: } Feb 18 17:25:53 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488 Feb 18 17:25:53 localhost kernel: ... acquired at: Feb 18 17:25:53 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:25:53 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:53 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:53 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:53 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:53 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:53 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:53 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 17:25:54 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:25:54 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 17:25:54 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:25:54 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 17:25:54 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 17:25:54 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:25:54 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:54 localhost kernel: Feb 18 17:25:54 localhost kernel: ... acquired at: Feb 18 17:25:54 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:25:54 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:25:54 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:54 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:54 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:25:54 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:54 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:25:54 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:25:54 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5 Feb 18 17:25:54 localhost kernel: [<c0421394>] __wake_up_common+0x32/0x5c Feb 18 17:25:54 localhost kernel: [<c04235fe>] complete+0x36/0x44 Feb 18 17:25:54 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:25:54 localhost kernel: [<c043d502>] kthread+0x1c/0x5e Feb 18 17:25:54 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 17:25:54 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:25:54 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:54 localhost kernel: Feb 18 17:25:54 localhost kernel: -> (&rq->rq_lock_key#2){++..} ops: 1235688 { Feb 18 17:25:54 localhost kernel: initial-use at: Feb 18 17:25:54 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:25:54 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:54 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:54 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:25:54 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:54 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:54 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 17:25:54 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 17:25:54 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:54 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21 Feb 18 17:25:54 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 17:25:54 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 17:25:55 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 17:25:55 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:25:55 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:25:55 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:25:55 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:25:55 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 17:25:55 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 17:25:55 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:25:55 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:55 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 17:25:55 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 17:25:55 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:55 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:55 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:25:55 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:55 localhost kernel: in-hardirq-W at: Feb 18 17:25:55 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 17:25:55 localhost kernel: [<c040a9d6>] save_stack_trace+0x20/0x3a Feb 18 17:25:55 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 17:25:55 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:55 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:55 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:55 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:55 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:55 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:55 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:55 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 17:25:55 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 17:25:55 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 17:25:55 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 17:25:55 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 17:25:55 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 17:25:55 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:25:55 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 17:25:55 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:25:55 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 17:25:55 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 17:25:55 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 17:25:55 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:55 localhost kernel: in-softirq-W at: Feb 18 17:25:55 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:25:56 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:56 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:56 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:56 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:56 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:56 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 17:25:56 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 17:25:56 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 17:25:56 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:25:56 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:25:56 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:56 localhost kernel: } Feb 18 17:25:56 localhost kernel: ... key at: [<c2a2b488>] 0xc2a2b488 Feb 18 17:25:56 localhost kernel: -> (&rq->rq_lock_key#3){++..} ops: 1928539 { Feb 18 17:25:56 localhost kernel: initial-use at: Feb 18 17:25:56 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:25:56 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:56 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:56 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:25:56 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:56 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:56 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 17:25:56 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 17:25:56 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21 Feb 18 17:25:56 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 17:25:56 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 17:25:56 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 17:25:56 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:25:56 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:25:56 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:25:56 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:25:56 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 17:25:56 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 17:25:56 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:25:56 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:56 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 17:25:56 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 17:25:56 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:56 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:56 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:25:56 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:56 localhost kernel: in-hardirq-W at: Feb 18 17:25:57 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 17:25:57 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:57 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:57 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:57 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:57 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:57 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:57 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:57 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 17:25:57 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 17:25:57 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 17:25:57 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 17:25:57 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 17:25:57 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 17:25:57 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:25:57 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 17:25:57 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:25:57 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 17:25:57 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 17:25:57 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 17:25:57 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:57 localhost kernel: in-softirq-W at: Feb 18 17:25:57 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21 Feb 18 17:25:57 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3 Feb 18 17:25:57 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:25:57 localhost kernel: [<c04422f1>] getnstimeofday+0x30/0xbc Feb 18 17:25:57 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:57 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:57 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:25:57 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:57 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:25:57 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:25:57 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5 Feb 18 17:25:57 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 17:25:57 localhost kernel: [<c0430afa>] __do_softirq+0xc9/0xde Feb 18 17:25:57 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:25:57 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:57 localhost kernel: } Feb 18 17:25:57 localhost kernel: ... key at: [<c2a34488>] 0xc2a34488 Feb 18 17:25:57 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 1787266 { Feb 18 17:25:58 localhost kernel: initial-use at: Feb 18 17:25:58 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:25:58 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:58 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:58 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:25:58 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:58 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:25:58 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 17:25:58 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 17:25:58 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21 Feb 18 17:25:58 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 17:25:58 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 17:25:58 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 17:25:58 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:25:58 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:25:58 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:25:58 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:25:58 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 17:25:58 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 17:25:58 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:25:58 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:58 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 17:25:58 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 17:25:58 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:58 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:25:58 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:25:58 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:58 localhost kernel: in-hardirq-W at: Feb 18 17:25:58 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 17:25:58 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:58 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:25:58 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:58 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:58 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:58 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:58 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:25:58 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 17:25:58 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 17:25:58 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 17:25:58 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 17:25:58 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 17:25:59 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 17:25:59 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:25:59 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 17:25:59 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:25:59 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 17:25:59 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 17:25:59 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 17:25:59 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:59 localhost kernel: in-softirq-W at: Feb 18 17:25:59 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 17:25:59 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:25:59 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:59 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:59 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:59 localhost openl2tpd: Loading plugin ppp_unix.so, version V1.0 Feb 18 17:25:59 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:59 localhost openl2tpd[14004]: Start, trace_flags=00000000 Feb 18 17:25:59 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:59 localhost openl2tpd[14004]: OpenL2TP V1.0, (c) Copyright 2004,2005,2006,2007,2008 Katalix Systems Ltd. Feb 18 17:25:59 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 17:25:59 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 17:25:59 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 17:25:59 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:25:59 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:25:59 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:25:59 localhost kernel: } Feb 18 17:25:59 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488 Feb 18 17:25:59 localhost kernel: ... acquired at: Feb 18 17:25:59 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:25:59 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:59 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:25:59 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:59 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:25:59 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:59 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:25:59 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 17:25:59 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:25:59 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 17:25:59 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:25:59 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 17:25:59 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 17:26:00 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:00 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:00 localhost kernel: Feb 18 17:26:00 localhost kernel: ... acquired at: Feb 18 17:26:00 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:00 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:00 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:00 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:00 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:00 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:00 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:00 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 17:26:00 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:26:00 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 17:26:00 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:26:00 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 17:26:00 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 17:26:00 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:00 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:00 localhost kernel: Feb 18 17:26:00 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 1787266 { Feb 18 17:26:00 localhost kernel: initial-use at: Feb 18 17:26:00 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:00 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:00 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:26:00 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:26:00 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:26:00 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:26:00 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 17:26:00 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 17:26:00 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21 Feb 18 17:26:00 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 17:26:00 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 17:26:00 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 17:26:00 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:26:00 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:26:00 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:00 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:26:00 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 17:26:00 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 17:26:01 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:01 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:01 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 17:26:01 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 17:26:01 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:01 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:01 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:01 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:01 localhost kernel: in-hardirq-W at: Feb 18 17:26:01 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 17:26:01 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:01 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:01 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:01 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:26:01 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:01 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:26:01 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:26:01 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 17:26:01 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 17:26:01 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 17:26:01 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 17:26:01 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 17:26:01 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 17:26:01 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:26:01 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 17:26:01 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:26:01 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 17:26:01 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 17:26:01 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 17:26:01 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:01 localhost kernel: in-softirq-W at: Feb 18 17:26:01 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 17:26:01 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:26:01 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:01 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:01 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:01 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:01 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:01 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 17:26:01 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 17:26:01 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 17:26:01 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:26:02 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:02 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:02 localhost kernel: } Feb 18 17:26:02 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488 Feb 18 17:26:02 localhost kernel: ... acquired at: Feb 18 17:26:02 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:02 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:02 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:02 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:02 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:02 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:02 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:02 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 17:26:02 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:26:02 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 17:26:02 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:26:02 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 17:26:02 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 17:26:02 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:02 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:02 localhost kernel: Feb 18 17:26:02 localhost kernel: ... acquired at: Feb 18 17:26:02 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:02 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:26:02 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:02 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:26:02 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:02 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:26:02 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:26:02 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5 Feb 18 17:26:02 localhost kernel: [<c0421394>] __wake_up_common+0x32/0x5c Feb 18 17:26:02 localhost kernel: [<c04235fe>] complete+0x36/0x44 Feb 18 17:26:02 localhost kernel: [<c043aad8>] worker_thread+0x0/0xc4 Feb 18 17:26:02 localhost kernel: [<c043d502>] kthread+0x1c/0x5e Feb 18 17:26:02 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 17:26:02 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:02 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:02 localhost kernel: Feb 18 17:26:02 localhost kernel: -> (&rq->rq_lock_key#3){++..} ops: 1928539 { Feb 18 17:26:02 localhost kernel: initial-use at: Feb 18 17:26:02 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:03 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:03 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:26:03 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:26:03 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:26:03 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:26:03 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 17:26:03 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 17:26:03 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21 Feb 18 17:26:03 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 17:26:03 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 17:26:03 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 17:26:03 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:26:03 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:26:03 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:03 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:26:03 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 17:26:03 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 17:26:03 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:03 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:03 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 17:26:03 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 17:26:03 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:03 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:03 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:03 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:03 localhost kernel: in-hardirq-W at: Feb 18 17:26:03 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 17:26:03 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:03 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:03 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:03 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:26:03 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:03 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:26:03 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:26:03 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 17:26:03 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 17:26:03 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 17:26:03 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 17:26:03 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 17:26:03 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 17:26:03 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:26:04 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 17:26:04 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:26:04 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 17:26:04 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 17:26:04 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 17:26:04 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:04 localhost kernel: in-softirq-W at: Feb 18 17:26:04 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21 Feb 18 17:26:04 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3 Feb 18 17:26:04 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:26:04 localhost kernel: [<c04422f1>] getnstimeofday+0x30/0xbc Feb 18 17:26:04 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:04 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:04 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:26:04 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:04 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:26:04 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:26:04 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5 Feb 18 17:26:04 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 17:26:04 localhost kernel: [<c0430afa>] __do_softirq+0xc9/0xde Feb 18 17:26:04 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:04 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:04 localhost kernel: } Feb 18 17:26:04 localhost kernel: ... key at: [<c2a34488>] 0xc2a34488 Feb 18 17:26:04 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 1787266 { Feb 18 17:26:04 localhost kernel: initial-use at: Feb 18 17:26:04 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:04 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:04 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:26:04 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:26:04 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:26:04 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:26:04 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 17:26:04 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 17:26:04 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21 Feb 18 17:26:04 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 17:26:04 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 17:26:04 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 17:26:04 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:26:04 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:26:04 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:05 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:26:05 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 17:26:05 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 17:26:05 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:05 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:05 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 17:26:05 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 17:26:05 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:05 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:05 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:05 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:05 localhost kernel: in-hardirq-W at: Feb 18 17:26:05 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 17:26:05 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:05 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:05 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:05 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:26:05 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:05 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:26:05 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:26:05 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 17:26:05 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 17:26:05 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 17:26:05 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 17:26:05 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 17:26:05 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 17:26:05 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:26:05 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 17:26:05 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:26:05 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 17:26:05 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 17:26:05 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 17:26:05 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:05 localhost kernel: in-softirq-W at: Feb 18 17:26:05 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 17:26:05 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:26:05 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:05 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:05 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:05 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:05 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:05 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 17:26:06 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 17:26:06 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 17:26:06 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:26:06 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:06 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:06 localhost kernel: } Feb 18 17:26:06 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488 Feb 18 17:26:06 localhost kernel: ... acquired at: Feb 18 17:26:06 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:06 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:06 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:06 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:06 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:06 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:06 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:06 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0 Feb 18 17:26:06 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:26:06 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2 Feb 18 17:26:06 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:26:06 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 17:26:06 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 17:26:06 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:06 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:06 localhost kernel: Feb 18 17:26:06 localhost kernel: ... acquired at: Feb 18 17:26:06 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:06 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:26:06 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:06 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:26:06 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:06 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:26:06 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:26:06 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5 Feb 18 17:26:06 localhost kernel: [<c0421394>] __wake_up_common+0x32/0x5c Feb 18 17:26:06 localhost kernel: [<c04235fe>] complete+0x36/0x44 Feb 18 17:26:06 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:26:06 localhost kernel: [<c0428996>] migration_thread+0x186/0x1d2 Feb 18 17:26:06 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:26:06 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 17:26:06 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 17:26:07 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:07 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:07 localhost kernel: Feb 18 17:26:07 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 1787266 { Feb 18 17:26:07 localhost kernel: initial-use at: Feb 18 17:26:07 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:07 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:07 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:26:07 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:26:07 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:26:07 localhost kernel: [<c0423559>] init_idle+0x77/0xa1 Feb 18 17:26:07 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d Feb 18 17:26:07 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9 Feb 18 17:26:07 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21 Feb 18 17:26:07 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f Feb 18 17:26:07 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137 Feb 18 17:26:07 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3 Feb 18 17:26:07 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:26:07 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:26:07 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:07 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d Feb 18 17:26:07 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb Feb 18 17:26:07 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23 Feb 18 17:26:07 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:07 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:07 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af Feb 18 17:26:07 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15 Feb 18 17:26:07 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:07 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:07 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:07 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:07 localhost kernel: in-hardirq-W at: Feb 18 17:26:07 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 17:26:07 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:07 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:07 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:07 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:26:07 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:07 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:26:07 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d Feb 18 17:26:07 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44 Feb 18 17:26:08 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d Feb 18 17:26:08 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c Feb 18 17:26:08 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1 Feb 18 17:26:08 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80 Feb 18 17:26:08 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38 Feb 18 17:26:08 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:26:08 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38 Feb 18 17:26:08 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:26:08 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 17:26:08 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 17:26:08 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 17:26:08 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:08 localhost kernel: in-softirq-W at: Feb 18 17:26:08 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 17:26:08 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:26:08 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:08 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:08 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:08 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:08 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40 Feb 18 17:26:08 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393 Feb 18 17:26:08 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99 Feb 18 17:26:08 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 17:26:08 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:26:08 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:08 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:08 localhost kernel: } Feb 18 17:26:08 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488 Feb 18 17:26:08 localhost kernel: ... acquired at: Feb 18 17:26:08 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:08 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:26:08 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:08 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:26:08 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:08 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:26:08 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50 Feb 18 17:26:08 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5 Feb 18 17:26:08 localhost kernel: [<c0421394>] __wake_up_common+0x32/0x5c Feb 18 17:26:08 localhost kernel: [<c04235fe>] complete+0x36/0x44 Feb 18 17:26:08 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:26:09 localhost kernel: [<c0428996>] migration_thread+0x186/0x1d2 Feb 18 17:26:09 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2 Feb 18 17:26:09 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 17:26:09 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 17:26:09 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:09 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:09 localhost kernel: Feb 18 17:26:09 localhost kernel: ... acquired at: Feb 18 17:26:09 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:09 localhost kernel: [<c042366f>] __wake_up+0x18/0x42 Feb 18 17:26:09 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:09 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 17:26:09 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:09 localhost kernel: [<c042366f>] __wake_up+0x18/0x42 Feb 18 17:26:09 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:26:09 localhost kernel: [<c042366f>] __wake_up+0x18/0x42 Feb 18 17:26:09 localhost kernel: [<c042366f>] __wake_up+0x18/0x42 Feb 18 17:26:09 localhost kernel: [<f8bb7a2b>] ppp_receive_nonmp_frame+0x51f/0x70a [ppp_generic] Feb 18 17:26:09 localhost kernel: [<f8bb8487>] ppp_input+0xbf/0xef [ppp_generic] Feb 18 17:26:09 localhost kernel: [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp] Feb 18 17:26:09 localhost kernel: [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp] Feb 18 17:26:09 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c Feb 18 17:26:09 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259 Feb 18 17:26:09 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758 Feb 18 17:26:09 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8 Feb 18 17:26:09 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8 Feb 18 17:26:09 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338 Feb 18 17:26:09 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 17:26:09 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237 Feb 18 17:26:09 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4 Feb 18 17:26:09 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 17:26:09 localhost kernel: [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000] Feb 18 17:26:09 localhost kernel: [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000] Feb 18 17:26:09 localhost kernel: [<f8960f13>] e1000_clean+0x63/0x203 [e1000] Feb 18 17:26:09 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3 Feb 18 17:26:09 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3 Feb 18 17:26:09 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:26:09 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:09 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:09 localhost kernel: Feb 18 17:26:09 localhost kernel: ... acquired at: Feb 18 17:26:09 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:10 localhost kernel: [<f8bb6bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic] Feb 18 17:26:10 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:10 localhost kernel: [<f8bb6bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic] Feb 18 17:26:10 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 17:26:10 localhost kernel: [<f8bb6bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic] Feb 18 17:26:10 localhost kernel: [<f8bb6bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic] Feb 18 17:26:10 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 17:26:10 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:26:10 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:10 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 17:26:10 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8 Feb 18 17:26:10 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc Feb 18 17:26:10 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62 Feb 18 17:26:10 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249 Feb 18 17:26:10 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d Feb 18 17:26:10 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 17:26:10 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:10 localhost kernel: Feb 18 17:26:10 localhost kernel: -> (&list->lock#8){.+..} ops: 46 { Feb 18 17:26:10 localhost kernel: initial-use at: Feb 18 17:26:10 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:10 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:10 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:10 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 17:26:10 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:26:10 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 17:26:10 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 17:26:10 localhost kernel: [<f8bb8433>] ppp_input+0x6b/0xef [ppp_generic] Feb 18 17:26:10 localhost kernel: [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp] Feb 18 17:26:10 localhost kernel: [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp] Feb 18 17:26:10 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c Feb 18 17:26:10 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259 Feb 18 17:26:10 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758 Feb 18 17:26:10 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8 Feb 18 17:26:10 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8 Feb 18 17:26:10 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338 Feb 18 17:26:10 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 17:26:10 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237 Feb 18 17:26:10 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4 Feb 18 17:26:10 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 17:26:10 localhost kernel: [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000] Feb 18 17:26:11 localhost kernel: [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000] Feb 18 17:26:11 localhost kernel: [<f8960f13>] e1000_clean+0x63/0x203 [e1000] Feb 18 17:26:11 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3 Feb 18 17:26:11 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3 Feb 18 17:26:11 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:26:11 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:11 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:11 localhost kernel: in-softirq-W at: Feb 18 17:26:11 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:26:11 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:11 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:11 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 17:26:11 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:26:11 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 17:26:11 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 17:26:11 localhost kernel: [<f8bb8433>] ppp_input+0x6b/0xef [ppp_generic] Feb 18 17:26:11 localhost kernel: [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp] Feb 18 17:26:11 localhost kernel: [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp] Feb 18 17:26:11 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c Feb 18 17:26:11 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259 Feb 18 17:26:11 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758 Feb 18 17:26:11 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8 Feb 18 17:26:11 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8 Feb 18 17:26:11 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338 Feb 18 17:26:11 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 17:26:11 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237 Feb 18 17:26:11 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4 Feb 18 17:26:11 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 17:26:11 localhost kernel: [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000] Feb 18 17:26:11 localhost kernel: [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000] Feb 18 17:26:11 localhost kernel: [<f8960f13>] e1000_clean+0x63/0x203 [e1000] Feb 18 17:26:11 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3 Feb 18 17:26:11 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3 Feb 18 17:26:11 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:26:11 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:11 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:11 localhost kernel: } Feb 18 17:26:11 localhost kernel: ... key at: [<f8bbcd18>] __key.21159+0x0/0xffffb82a [ppp_generic] Feb 18 17:26:11 localhost kernel: ... acquired at: Feb 18 17:26:11 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:11 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:12 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:12 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:12 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:12 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:26:12 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:12 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:12 localhost kernel: [<f8bb6424>] ppp_xmit_process+0x529/0x5a1 [ppp_generic] Feb 18 17:26:12 localhost kernel: [<f8bb734b>] ppp_channel_push+0x71/0x90 [ppp_generic] Feb 18 17:26:12 localhost kernel: [<c061b373>] _read_lock_bh+0x2e/0x39 Feb 18 17:26:12 localhost kernel: [<f8bb735a>] ppp_channel_push+0x80/0x90 [ppp_generic] Feb 18 17:26:12 localhost kernel: [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic] Feb 18 17:26:12 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic] Feb 18 17:26:12 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d Feb 18 17:26:12 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67 Feb 18 17:26:12 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 17:26:12 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:12 localhost kernel: Feb 18 17:26:12 localhost kernel: -> (&pch->downl){-...} ops: 8 { Feb 18 17:26:12 localhost kernel: initial-use at: Feb 18 17:26:12 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:12 localhost kernel: [<c046b874>] __do_fault+0x31b/0x35d Feb 18 17:26:12 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:12 localhost kernel: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic] Feb 18 17:26:12 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 17:26:12 localhost kernel: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic] Feb 18 17:26:12 localhost kernel: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic] Feb 18 17:26:12 localhost kernel: [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic] Feb 18 17:26:12 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic] Feb 18 17:26:12 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d Feb 18 17:26:12 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67 Feb 18 17:26:12 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 17:26:12 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:12 localhost kernel: hardirq-on-W at: Feb 18 17:26:12 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1 Feb 18 17:26:12 localhost kernel: [<c046b874>] __do_fault+0x31b/0x35d Feb 18 17:26:12 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:12 localhost kernel: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic] Feb 18 17:26:12 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 17:26:13 localhost kernel: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic] Feb 18 17:26:13 localhost kernel: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic] Feb 18 17:26:13 localhost kernel: [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic] Feb 18 17:26:13 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic] Feb 18 17:26:13 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d Feb 18 17:26:13 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67 Feb 18 17:26:13 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 17:26:13 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:13 localhost kernel: } Feb 18 17:26:13 localhost kernel: ... key at: [<f8bbcd28>] __key.29740+0x0/0xffffb81a [ppp_generic] Feb 18 17:26:13 localhost kernel: -> (&list->lock#8){.+..} ops: 46 { Feb 18 17:26:13 localhost kernel: initial-use at: Feb 18 17:26:13 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:13 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:13 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:13 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 17:26:13 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:26:13 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 17:26:13 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 17:26:13 localhost kernel: [<f8bb8433>] ppp_input+0x6b/0xef [ppp_generic] Feb 18 17:26:13 localhost kernel: [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp] Feb 18 17:26:13 localhost kernel: [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp] Feb 18 17:26:13 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c Feb 18 17:26:13 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259 Feb 18 17:26:13 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758 Feb 18 17:26:13 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8 Feb 18 17:26:13 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8 Feb 18 17:26:13 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338 Feb 18 17:26:13 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 17:26:13 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237 Feb 18 17:26:13 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4 Feb 18 17:26:13 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 17:26:13 localhost kernel: [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000] Feb 18 17:26:13 localhost kernel: [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000] Feb 18 17:26:13 localhost kernel: [<f8960f13>] e1000_clean+0x63/0x203 [e1000] Feb 18 17:26:13 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3 Feb 18 17:26:13 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3 Feb 18 17:26:13 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:26:13 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:13 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:13 localhost kernel: in-softirq-W at: Feb 18 17:26:14 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:26:14 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:14 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:14 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 17:26:14 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:26:14 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 17:26:14 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d Feb 18 17:26:14 localhost kernel: [<f8bb8433>] ppp_input+0x6b/0xef [ppp_generic] Feb 18 17:26:14 localhost kernel: [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp] Feb 18 17:26:14 localhost kernel: [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp] Feb 18 17:26:14 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c Feb 18 17:26:14 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259 Feb 18 17:26:14 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758 Feb 18 17:26:14 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8 Feb 18 17:26:14 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8 Feb 18 17:26:14 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338 Feb 18 17:26:14 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 17:26:14 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237 Feb 18 17:26:14 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4 Feb 18 17:26:14 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 17:26:14 localhost kernel: [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000] Feb 18 17:26:14 localhost kernel: [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000] Feb 18 17:26:14 localhost kernel: [<f8960f13>] e1000_clean+0x63/0x203 [e1000] Feb 18 17:26:14 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3 Feb 18 17:26:14 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3 Feb 18 17:26:14 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:26:14 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:14 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:14 localhost kernel: } Feb 18 17:26:14 localhost kernel: ... key at: [<f8bbcd18>] __key.21159+0x0/0xffffb82a [ppp_generic] Feb 18 17:26:14 localhost kernel: ... acquired at: Feb 18 17:26:14 localhost kernel: [<c0446b6a>] add_lock_to_list+0x66/0x89 Feb 18 17:26:14 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:14 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:14 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:14 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:14 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:14 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:26:14 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:14 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:14 localhost kernel: [<f8bb72ff>] ppp_channel_push+0x25/0x90 [ppp_generic] Feb 18 17:26:14 localhost kernel: [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic] Feb 18 17:26:14 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic] Feb 18 17:26:15 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d Feb 18 17:26:15 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67 Feb 18 17:26:15 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 17:26:15 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:15 localhost kernel: Feb 18 17:26:15 localhost kernel: -> (inet_peer_idlock){-+..} ops: 1947 { Feb 18 17:26:15 localhost kernel: initial-use at: Feb 18 17:26:15 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 17:26:15 localhost kernel: [<c05d0e17>] rt_bind_peer+0x1b/0x50 Feb 18 17:26:15 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:15 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 17:26:15 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:15 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa Feb 18 17:26:15 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 17:26:15 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa Feb 18 17:26:15 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa Feb 18 17:26:15 localhost kernel: [<c05d4ea2>] ip_push_pending_frames+0x206/0x33b Feb 18 17:26:15 localhost kernel: [<c05f368d>] icmp_send+0x310/0x386 Feb 18 17:26:15 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:15 localhost kernel: [<c05bcd7a>] neigh_timer_handler+0x14/0x276 Feb 18 17:26:15 localhost kernel: [<c05ce07c>] ipv4_link_failure+0x1c/0x44 Feb 18 17:26:15 localhost kernel: [<c05f1375>] arp_error_report+0x1c/0x24 Feb 18 17:26:15 localhost kernel: [<c05bcf16>] neigh_timer_handler+0x1b0/0x276 Feb 18 17:26:15 localhost kernel: [<c05bcd66>] neigh_timer_handler+0x0/0x276 Feb 18 17:26:15 localhost kernel: [<c0433d67>] run_timer_softirq+0x121/0x188 Feb 18 17:26:15 localhost kernel: [<c05bcd66>] neigh_timer_handler+0x0/0x276 Feb 18 17:26:15 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 17:26:15 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:26:15 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:15 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:15 localhost kernel: in-softirq-W at: Feb 18 17:26:15 localhost kernel: [<c0446ab0>] save_trace+0x37/0x8b Feb 18 17:26:15 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 17:26:15 localhost kernel: [<c05d0e17>] rt_bind_peer+0x1b/0x50 Feb 18 17:26:15 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:26:15 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 17:26:15 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:15 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa Feb 18 17:26:15 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 17:26:15 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa Feb 18 17:26:15 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa Feb 18 17:26:15 localhost kernel: [<c05d4ea2>] ip_push_pending_frames+0x206/0x33b Feb 18 17:26:16 localhost kernel: [<c05f368d>] icmp_send+0x310/0x386 Feb 18 17:26:16 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:16 localhost kernel: [<c05bcd7a>] neigh_timer_handler+0x14/0x276 Feb 18 17:26:16 localhost kernel: [<c05ce07c>] ipv4_link_failure+0x1c/0x44 Feb 18 17:26:16 localhost kernel: [<c05f1375>] arp_error_report+0x1c/0x24 Feb 18 17:26:16 localhost kernel: [<c05bcf16>] neigh_timer_handler+0x1b0/0x276 Feb 18 17:26:16 localhost kernel: [<c05bcd66>] neigh_timer_handler+0x0/0x276 Feb 18 17:26:16 localhost kernel: [<c0433d67>] run_timer_softirq+0x121/0x188 Feb 18 17:26:16 localhost kernel: [<c05bcd66>] neigh_timer_handler+0x0/0x276 Feb 18 17:26:16 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 17:26:16 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:26:16 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:16 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:16 localhost kernel: hardirq-on-W at: Feb 18 17:26:16 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 17:26:16 localhost kernel: [<c05d0e17>] rt_bind_peer+0x1b/0x50 Feb 18 17:26:16 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1 Feb 18 17:26:16 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 17:26:16 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:16 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa Feb 18 17:26:16 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 17:26:16 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa Feb 18 17:26:16 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa Feb 18 17:26:16 localhost kernel: [<c05d4ea2>] ip_push_pending_frames+0x206/0x33b Feb 18 17:26:16 localhost kernel: [<c05f368d>] icmp_send+0x310/0x386 Feb 18 17:26:16 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:16 localhost kernel: [<c05bcd7a>] neigh_timer_handler+0x14/0x276 Feb 18 17:26:16 localhost kernel: [<c05ce07c>] ipv4_link_failure+0x1c/0x44 Feb 18 17:26:16 localhost kernel: [<c05f1375>] arp_error_report+0x1c/0x24 Feb 18 17:26:16 localhost kernel: [<c05bcf16>] neigh_timer_handler+0x1b0/0x276 Feb 18 17:26:16 localhost kernel: [<c05bcd66>] neigh_timer_handler+0x0/0x276 Feb 18 17:26:16 localhost kernel: [<c0433d67>] run_timer_softirq+0x121/0x188 Feb 18 17:26:16 localhost kernel: [<c05bcd66>] neigh_timer_handler+0x0/0x276 Feb 18 17:26:16 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 17:26:16 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:26:16 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:16 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:16 localhost kernel: } Feb 18 17:26:16 localhost kernel: ... key at: [<c07380f0>] inet_peer_idlock+0x10/0x1c Feb 18 17:26:16 localhost kernel: ... acquired at: Feb 18 17:26:16 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:16 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa Feb 18 17:26:17 localhost kernel: [<c048d217>] __d_lookup+0x0/0x12a Feb 18 17:26:17 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:17 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa Feb 18 17:26:17 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 17:26:17 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa Feb 18 17:26:17 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa Feb 18 17:26:17 localhost kernel: [<c043badc>] __kernel_text_address+0x18/0x23 Feb 18 17:26:17 localhost kernel: [<c05d6ec6>] ip_queue_xmit+0x27b/0x34d Feb 18 17:26:17 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 17:26:17 localhost kernel: [<c047a1bd>] kfree+0x9f/0xa6 Feb 18 17:26:17 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 17:26:17 localhost kernel: [<c05b0c83>] pskb_expand_head+0xde/0x144 Feb 18 17:26:17 localhost kernel: [<f8bbf7bf>] pppol2tp_xmit+0x426/0x4a2 [pppol2tp] Feb 18 17:26:17 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 17:26:17 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:26:17 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 17:26:17 localhost kernel: [<f8bb730e>] ppp_channel_push+0x34/0x90 [ppp_generic] Feb 18 17:26:17 localhost kernel: [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic] Feb 18 17:26:17 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic] Feb 18 17:26:17 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d Feb 18 17:26:17 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67 Feb 18 17:26:17 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 17:26:17 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:17 localhost kernel: Feb 18 17:26:17 localhost kernel: -> (&dev->queue_lock#2){-+..} ops: 47607 { Feb 18 17:26:17 localhost kernel: initial-use at: Feb 18 17:26:17 localhost kernel: [<c047a1bd>] kfree+0x9f/0xa6 Feb 18 17:26:17 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:17 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:17 localhost kernel: [<c05c47ad>] qdisc_lock_tree+0x14/0x1c Feb 18 17:26:17 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 17:26:17 localhost kernel: [<c05c47ad>] qdisc_lock_tree+0x14/0x1c Feb 18 17:26:17 localhost kernel: [<c05c47ad>] qdisc_lock_tree+0x14/0x1c Feb 18 17:26:17 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f Feb 18 17:26:17 localhost kernel: [<c05b680a>] register_netdevice+0x29f/0x2f9 Feb 18 17:26:17 localhost kernel: [<c05b6893>] register_netdev+0x2f/0x3b Feb 18 17:26:17 localhost kernel: [<c076aded>] loopback_net_init+0x36/0x68 Feb 18 17:26:17 localhost kernel: [<c05b3fa0>] register_pernet_operations+0x10/0x11 Feb 18 17:26:17 localhost kernel: [<c05b3ffc>] register_pernet_device+0x1c/0x48 Feb 18 17:26:17 localhost kernel: [<c074e491>] kernel_init+0x148/0x2af Feb 18 17:26:17 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:18 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:18 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:18 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:18 localhost kernel: in-softirq-W at: Feb 18 17:26:18 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:26:18 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:18 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:18 localhost kernel: [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9 Feb 18 17:26:18 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:18 localhost kernel: [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9 Feb 18 17:26:18 localhost kernel: [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9 Feb 18 17:26:18 localhost kernel: [<c05b7e60>] dev_queue_xmit+0x125/0x2e9 Feb 18 17:26:18 localhost kernel: [<c05c341e>] eth_header+0x0/0xb6 Feb 18 17:26:18 localhost kernel: [<f8f3a487>] mld_sendpack+0x242/0x355 [ipv6] Feb 18 17:26:18 localhost kernel: [<f8f3b453>] mld_ifc_timer_expire+0x143/0x208 [ipv6] Feb 18 17:26:18 localhost kernel: [<f8f3b4e5>] mld_ifc_timer_expire+0x1d5/0x208 [ipv6] Feb 18 17:26:18 localhost kernel: [<f8f3b4f0>] mld_ifc_timer_expire+0x1e0/0x208 [ipv6] Feb 18 17:26:18 localhost kernel: [<f8f3b310>] mld_ifc_timer_expire+0x0/0x208 [ipv6] Feb 18 17:26:18 localhost kernel: [<c0433d67>] run_timer_softirq+0x121/0x188 Feb 18 17:26:18 localhost kernel: [<f8f3b310>] mld_ifc_timer_expire+0x0/0x208 [ipv6] Feb 18 17:26:18 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 17:26:18 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:26:18 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:18 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:18 localhost kernel: hardirq-on-W at: Feb 18 17:26:18 localhost kernel: [<c047a1bd>] kfree+0x9f/0xa6 Feb 18 17:26:18 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1 Feb 18 17:26:18 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:18 localhost kernel: [<c05c47ad>] qdisc_lock_tree+0x14/0x1c Feb 18 17:26:18 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 17:26:18 localhost kernel: [<c05c47ad>] qdisc_lock_tree+0x14/0x1c Feb 18 17:26:18 localhost kernel: [<c05c47ad>] qdisc_lock_tree+0x14/0x1c Feb 18 17:26:18 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f Feb 18 17:26:18 localhost kernel: [<c05b680a>] register_netdevice+0x29f/0x2f9 Feb 18 17:26:18 localhost kernel: [<c05b6893>] register_netdev+0x2f/0x3b Feb 18 17:26:18 localhost kernel: [<c076aded>] loopback_net_init+0x36/0x68 Feb 18 17:26:18 localhost kernel: [<c05b3fa0>] register_pernet_operations+0x10/0x11 Feb 18 17:26:18 localhost kernel: [<c05b3ffc>] register_pernet_device+0x1c/0x48 Feb 18 17:26:18 localhost kernel: [<c074e491>] kernel_init+0x148/0x2af Feb 18 17:26:18 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:18 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:18 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:19 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:19 localhost kernel: } Feb 18 17:26:19 localhost kernel: ... key at: [<c0a3e1c4>] __key.31614+0x0/0x8 Feb 18 17:26:19 localhost kernel: -> (&dev->ingress_lock){-...} ops: 208 { Feb 18 17:26:19 localhost kernel: initial-use at: Feb 18 17:26:19 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:19 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:19 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f Feb 18 17:26:19 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:19 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f Feb 18 17:26:19 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f Feb 18 17:26:19 localhost kernel: [<c05b680a>] register_netdevice+0x29f/0x2f9 Feb 18 17:26:19 localhost kernel: [<c05b6893>] register_netdev+0x2f/0x3b Feb 18 17:26:19 localhost kernel: [<c076aded>] loopback_net_init+0x36/0x68 Feb 18 17:26:19 localhost kernel: [<c05b3fa0>] register_pernet_operations+0x10/0x11 Feb 18 17:26:19 localhost kernel: [<c05b3ffc>] register_pernet_device+0x1c/0x48 Feb 18 17:26:19 localhost kernel: [<c074e491>] kernel_init+0x148/0x2af Feb 18 17:26:19 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:19 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:19 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:19 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:19 localhost kernel: hardirq-on-W at: Feb 18 17:26:19 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1 Feb 18 17:26:19 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:19 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f Feb 18 17:26:19 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:19 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f Feb 18 17:26:19 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f Feb 18 17:26:19 localhost kernel: [<c05b680a>] register_netdevice+0x29f/0x2f9 Feb 18 17:26:19 localhost kernel: [<c05b6893>] register_netdev+0x2f/0x3b Feb 18 17:26:19 localhost kernel: [<c076aded>] loopback_net_init+0x36/0x68 Feb 18 17:26:19 localhost kernel: [<c05b3fa0>] register_pernet_operations+0x10/0x11 Feb 18 17:26:19 localhost kernel: [<c05b3ffc>] register_pernet_device+0x1c/0x48 Feb 18 17:26:19 localhost kernel: [<c074e491>] kernel_init+0x148/0x2af Feb 18 17:26:19 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:19 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:19 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:19 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:19 localhost kernel: } Feb 18 17:26:19 localhost kernel: ... key at: [<c0a3e1b4>] __key.31616+0x0/0x8 Feb 18 17:26:19 localhost kernel: -> (&list->lock#6){....} ops: 675 { Feb 18 17:26:19 localhost kernel: initial-use at: Feb 18 17:26:19 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:19 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:20 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:20 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:26:20 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:20 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:20 localhost kernel: [<c05b0afd>] skb_queue_purge+0x11/0x17 Feb 18 17:26:20 localhost kernel: [<c05c4eca>] pfifo_fast_reset+0x14/0x2f Feb 18 17:26:20 localhost kernel: [<c05c4712>] qdisc_reset+0x10/0x11 Feb 18 17:26:20 localhost kernel: [<c05c4b83>] dev_deactivate+0x2a/0xfd Feb 18 17:26:20 localhost kernel: [<c05bf1f4>] __linkwatch_run_queue+0x143/0x17a Feb 18 17:26:20 localhost kernel: [<c05bf248>] linkwatch_event+0x1d/0x22 Feb 18 17:26:20 localhost kernel: [<c043a0f3>] run_workqueue+0xdb/0x1ae Feb 18 17:26:20 localhost kernel: [<c043a09f>] run_workqueue+0x87/0x1ae Feb 18 17:26:20 localhost kernel: [<c05bf22b>] linkwatch_event+0x0/0x22 Feb 18 17:26:20 localhost kernel: [<c043aad8>] worker_thread+0x0/0xc4 Feb 18 17:26:20 localhost kernel: [<c043ab92>] worker_thread+0xba/0xc4 Feb 18 17:26:20 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35 Feb 18 17:26:20 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 17:26:20 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 17:26:20 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:20 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:20 localhost kernel: } Feb 18 17:26:20 localhost kernel: ... key at: [<c0a3e7d4>] __key.21167+0x0/0x8 Feb 18 17:26:20 localhost kernel: ... acquired at: Feb 18 17:26:20 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:20 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:20 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:20 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:20 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:20 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:26:20 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:20 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:20 localhost kernel: [<c05b0afd>] skb_queue_purge+0x11/0x17 Feb 18 17:26:20 localhost kernel: [<c05c4eca>] pfifo_fast_reset+0x14/0x2f Feb 18 17:26:20 localhost kernel: [<c05c48f7>] qdisc_destroy+0x36/0x69 Feb 18 17:26:20 localhost kernel: [<c05c4954>] dev_shutdown+0x2a/0x7b Feb 18 17:26:20 localhost kernel: [<c05b6210>] rollback_registered+0xb8/0x11e Feb 18 17:26:20 localhost kernel: [<c05b627e>] unregister_netdevice+0x8/0x35 Feb 18 17:26:20 localhost kernel: [<c05b62ba>] unregister_netdev+0xf/0x15 Feb 18 17:26:20 localhost kernel: [<f8bb523f>] ppp_shutdown_interface+0x58/0xad [ppp_generic] Feb 18 17:26:20 localhost kernel: [<f8bb5484>] ppp_release+0x29/0x56 [ppp_generic] Feb 18 17:26:20 localhost kernel: [<c047f08a>] __fput+0xba/0x172 Feb 18 17:26:20 localhost kernel: [<c047c7df>] filp_close+0x51/0x58 Feb 18 17:26:21 localhost kernel: [<c047d9c4>] sys_close+0x70/0xab Feb 18 17:26:21 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 17:26:21 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:21 localhost kernel: Feb 18 17:26:21 localhost kernel: ... acquired at: Feb 18 17:26:21 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:21 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f Feb 18 17:26:21 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:21 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f Feb 18 17:26:21 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:21 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f Feb 18 17:26:21 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f Feb 18 17:26:21 localhost kernel: [<c05b680a>] register_netdevice+0x29f/0x2f9 Feb 18 17:26:21 localhost kernel: [<c05b6893>] register_netdev+0x2f/0x3b Feb 18 17:26:21 localhost kernel: [<c076aded>] loopback_net_init+0x36/0x68 Feb 18 17:26:21 localhost kernel: [<c05b3fa0>] register_pernet_operations+0x10/0x11 Feb 18 17:26:21 localhost kernel: [<c05b3ffc>] register_pernet_device+0x1c/0x48 Feb 18 17:26:21 localhost kernel: [<c074e491>] kernel_init+0x148/0x2af Feb 18 17:26:21 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:21 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af Feb 18 17:26:21 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:21 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:21 localhost kernel: Feb 18 17:26:21 localhost kernel: -> (&list->lock#6){....} ops: 675 { Feb 18 17:26:21 localhost kernel: initial-use at: Feb 18 17:26:21 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:21 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:21 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:21 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:26:21 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:21 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:21 localhost kernel: [<c05b0afd>] skb_queue_purge+0x11/0x17 Feb 18 17:26:21 localhost kernel: [<c05c4eca>] pfifo_fast_reset+0x14/0x2f Feb 18 17:26:21 localhost kernel: [<c05c4712>] qdisc_reset+0x10/0x11 Feb 18 17:26:21 localhost kernel: [<c05c4b83>] dev_deactivate+0x2a/0xfd Feb 18 17:26:21 localhost kernel: [<c05bf1f4>] __linkwatch_run_queue+0x143/0x17a Feb 18 17:26:21 localhost kernel: [<c05bf248>] linkwatch_event+0x1d/0x22 Feb 18 17:26:21 localhost kernel: [<c043a0f3>] run_workqueue+0xdb/0x1ae Feb 18 17:26:21 localhost kernel: [<c043a09f>] run_workqueue+0x87/0x1ae Feb 18 17:26:21 localhost kernel: [<c05bf22b>] linkwatch_event+0x0/0x22 Feb 18 17:26:21 localhost kernel: [<c043aad8>] worker_thread+0x0/0xc4 Feb 18 17:26:21 localhost kernel: [<c043ab92>] worker_thread+0xba/0xc4 Feb 18 17:26:22 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35 Feb 18 17:26:22 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 17:26:22 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 17:26:22 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:22 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:22 localhost kernel: } Feb 18 17:26:22 localhost kernel: ... key at: [<c0a3e7d4>] __key.21167+0x0/0x8 Feb 18 17:26:22 localhost kernel: ... acquired at: Feb 18 17:26:22 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:22 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:22 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:22 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:22 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41 Feb 18 17:26:22 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:22 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f Feb 18 17:26:22 localhost kernel: [<c05b0afd>] skb_queue_purge+0x11/0x17 Feb 18 17:26:22 localhost kernel: [<c05c4eca>] pfifo_fast_reset+0x14/0x2f Feb 18 17:26:22 localhost kernel: [<c05c4712>] qdisc_reset+0x10/0x11 Feb 18 17:26:22 localhost kernel: [<c05c4b83>] dev_deactivate+0x2a/0xfd Feb 18 17:26:22 localhost kernel: [<c05bf1f4>] __linkwatch_run_queue+0x143/0x17a Feb 18 17:26:22 localhost kernel: [<c05bf248>] linkwatch_event+0x1d/0x22 Feb 18 17:26:22 localhost kernel: [<c043a0f3>] run_workqueue+0xdb/0x1ae Feb 18 17:26:22 localhost kernel: [<c043a09f>] run_workqueue+0x87/0x1ae Feb 18 17:26:22 localhost kernel: [<c05bf22b>] linkwatch_event+0x0/0x22 Feb 18 17:26:22 localhost kernel: [<c043aad8>] worker_thread+0x0/0xc4 Feb 18 17:26:22 localhost kernel: [<c043ab92>] worker_thread+0xba/0xc4 Feb 18 17:26:22 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35 Feb 18 17:26:22 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 17:26:22 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 17:26:22 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:22 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:22 localhost kernel: Feb 18 17:26:22 localhost kernel: ... acquired at: Feb 18 17:26:22 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:22 localhost kernel: [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9 Feb 18 17:26:22 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:22 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:22 localhost kernel: [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9 Feb 18 17:26:22 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:22 localhost kernel: [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9 Feb 18 17:26:22 localhost kernel: [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9 Feb 18 17:26:22 localhost kernel: [<c05b7e60>] dev_queue_xmit+0x125/0x2e9 Feb 18 17:26:23 localhost kernel: [<c05d62e7>] ip_finish_output+0x1e6/0x21e Feb 18 17:26:23 localhost kernel: [<c05d6f58>] ip_queue_xmit+0x30d/0x34d Feb 18 17:26:23 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 17:26:23 localhost kernel: [<c047a1bd>] kfree+0x9f/0xa6 Feb 18 17:26:23 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 17:26:23 localhost kernel: [<c05b0c83>] pskb_expand_head+0xde/0x144 Feb 18 17:26:23 localhost kernel: [<f8bbf7bf>] pppol2tp_xmit+0x426/0x4a2 [pppol2tp] Feb 18 17:26:23 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 17:26:23 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:26:23 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 17:26:23 localhost kernel: [<f8bb730e>] ppp_channel_push+0x34/0x90 [ppp_generic] Feb 18 17:26:23 localhost kernel: [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic] Feb 18 17:26:23 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic] Feb 18 17:26:23 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d Feb 18 17:26:23 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67 Feb 18 17:26:23 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 17:26:23 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:23 localhost kernel: Feb 18 17:26:23 localhost kernel: -> (&rt_hash_locks[i]){-+..} ops: 24708841 { Feb 18 17:26:23 localhost kernel: initial-use at: Feb 18 17:26:23 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:23 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:23 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b Feb 18 17:26:23 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 17:26:23 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b Feb 18 17:26:23 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b Feb 18 17:26:23 localhost kernel: [<c05f90b6>] ip_mc_inc_group+0x184/0x1c1 Feb 18 17:26:23 localhost kernel: [<c05f9134>] ip_mc_up+0x41/0x59 Feb 18 17:26:23 localhost kernel: [<c05f4f9f>] inetdev_event+0x257/0x46e Feb 18 17:26:23 localhost kernel: [<c05c258a>] fib_rules_event+0x120/0x12a Feb 18 17:26:23 localhost kernel: [<c05c24b4>] fib_rules_event+0x4a/0x12a Feb 18 17:26:23 localhost kernel: [<c061d06c>] notifier_call_chain+0x2a/0x47 Feb 18 17:26:23 localhost kernel: [<c0441084>] raw_notifier_call_chain+0x17/0x1a Feb 18 17:26:23 localhost kernel: [<c05b73cf>] dev_open+0x71/0x77 Feb 18 17:26:23 localhost kernel: [<c05b5ffa>] dev_change_flags+0x9d/0x149 Feb 18 17:26:23 localhost kernel: [<c05b49b5>] __dev_get_by_name+0x67/0x72 Feb 18 17:26:23 localhost kernel: [<c05f54f7>] devinet_ioctl+0x22e/0x539 Feb 18 17:26:23 localhost kernel: [<c05b7188>] dev_ioctl+0x46f/0x5e0 Feb 18 17:26:23 localhost kernel: [<c05ab078>] sock_ioctl+0x1bb/0x1e0 Feb 18 17:26:23 localhost kernel: [<c05aaebd>] sock_ioctl+0x0/0x1e0 Feb 18 17:26:23 localhost kernel: [<c0488827>] do_ioctl+0x1f/0x62 Feb 18 17:26:23 localhost kernel: [<c061ccb2>] do_page_fault+0x234/0x5c4 Feb 18 17:26:23 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249 Feb 18 17:26:24 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d Feb 18 17:26:24 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5 Feb 18 17:26:24 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:24 localhost kernel: in-softirq-W at: Feb 18 17:26:24 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:26:24 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:24 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b Feb 18 17:26:24 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 17:26:24 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b Feb 18 17:26:24 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b Feb 18 17:26:24 localhost kernel: [<c05cee70>] rt_run_flush+0x0/0x8b Feb 18 17:26:24 localhost kernel: [<c0433d67>] run_timer_softirq+0x121/0x188 Feb 18 17:26:24 localhost kernel: [<c05cee70>] rt_run_flush+0x0/0x8b Feb 18 17:26:24 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 17:26:24 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:26:24 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:24 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:24 localhost kernel: hardirq-on-W at: Feb 18 17:26:24 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1 Feb 18 17:26:24 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:24 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b Feb 18 17:26:24 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 17:26:24 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b Feb 18 17:26:24 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b Feb 18 17:26:24 localhost kernel: [<c05f90b6>] ip_mc_inc_group+0x184/0x1c1 Feb 18 17:26:24 localhost kernel: [<c05f9134>] ip_mc_up+0x41/0x59 Feb 18 17:26:24 localhost kernel: [<c05f4f9f>] inetdev_event+0x257/0x46e Feb 18 17:26:24 localhost kernel: [<c05c258a>] fib_rules_event+0x120/0x12a Feb 18 17:26:24 localhost kernel: [<c05c24b4>] fib_rules_event+0x4a/0x12a Feb 18 17:26:24 localhost kernel: [<c061d06c>] notifier_call_chain+0x2a/0x47 Feb 18 17:26:24 localhost kernel: [<c0441084>] raw_notifier_call_chain+0x17/0x1a Feb 18 17:26:24 localhost kernel: [<c05b73cf>] dev_open+0x71/0x77 Feb 18 17:26:24 localhost kernel: [<c05b5ffa>] dev_change_flags+0x9d/0x149 Feb 18 17:26:24 localhost kernel: [<c05b49b5>] __dev_get_by_name+0x67/0x72 Feb 18 17:26:24 localhost kernel: [<c05f54f7>] devinet_ioctl+0x22e/0x539 Feb 18 17:26:24 localhost kernel: [<c05b7188>] dev_ioctl+0x46f/0x5e0 Feb 18 17:26:24 localhost kernel: [<c05ab078>] sock_ioctl+0x1bb/0x1e0 Feb 18 17:26:24 localhost kernel: [<c05aaebd>] sock_ioctl+0x0/0x1e0 Feb 18 17:26:24 localhost kernel: [<c0488827>] do_ioctl+0x1f/0x62 Feb 18 17:26:24 localhost kernel: [<c061ccb2>] do_page_fault+0x234/0x5c4 Feb 18 17:26:24 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249 Feb 18 17:26:24 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d Feb 18 17:26:24 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5 Feb 18 17:26:25 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:25 localhost kernel: } Feb 18 17:26:25 localhost kernel: ... key at: [<c0a3ea90>] __key.39046+0x0/0x8 Feb 18 17:26:25 localhost kernel: -> (&parent->list_lock){++..} ops: 252663 { Feb 18 17:26:25 localhost kernel: initial-use at: Feb 18 17:26:25 localhost kernel: [<c040a9d6>] save_stack_trace+0x20/0x3a Feb 18 17:26:25 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:25 localhost kernel: [<c043badc>] __kernel_text_address+0x18/0x23 Feb 18 17:26:25 localhost kernel: [<c0448013>] check_usage_backwards+0x19/0x41 Feb 18 17:26:25 localhost kernel: [<c0446ab0>] save_trace+0x37/0x8b Feb 18 17:26:25 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:25 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493 Feb 18 17:26:25 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:25 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493 Feb 18 17:26:25 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493 Feb 18 17:26:25 localhost kernel: [<c047b89c>] kmem_cache_create+0x62/0x38e Feb 18 17:26:25 localhost kernel: [<c047a790>] kmem_cache_alloc+0x53/0x9f Feb 18 17:26:25 localhost kernel: [<c047b99d>] kmem_cache_create+0x163/0x38e Feb 18 17:26:25 localhost kernel: [<c0479da2>] cache_estimate+0x6c/0x89 Feb 18 17:26:25 localhost kernel: [<c0762241>] kmem_cache_init+0x182/0x3f6 Feb 18 17:26:25 localhost kernel: [<c074e88b>] start_kernel+0x293/0x327 Feb 18 17:26:25 localhost kernel: [<c074e0e0>] unknown_bootoption+0x0/0x195 Feb 18 17:26:25 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:25 localhost kernel: in-hardirq-W at: Feb 18 17:26:25 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1 Feb 18 17:26:25 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:25 localhost kernel: [<c047b7ec>] do_drain+0x1e/0x49 Feb 18 17:26:25 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:25 localhost kernel: [<c047b7ec>] do_drain+0x1e/0x49 Feb 18 17:26:25 localhost kernel: [<c047b7ec>] do_drain+0x1e/0x49 Feb 18 17:26:25 localhost kernel: [<c047b7ce>] do_drain+0x0/0x49 Feb 18 17:26:25 localhost kernel: [<c0418ed5>] smp_call_function_interrupt+0x37/0x62 Feb 18 17:26:25 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:26:25 localhost kernel: [<c04059b3>] call_function_interrupt+0x33/0x38 Feb 18 17:26:25 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54 Feb 18 17:26:25 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126 Feb 18 17:26:25 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54 Feb 18 17:26:25 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0 Feb 18 17:26:25 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:25 localhost kernel: in-softirq-W at: Feb 18 17:26:25 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3 Feb 18 17:26:25 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:26:26 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:26 localhost kernel: [<c04488b6>] debug_check_no_locks_freed+0x105/0x11f Feb 18 17:26:26 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:26 localhost kernel: [<c047a0a0>] cache_flusharray+0x22/0xa0 Feb 18 17:26:26 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:26 localhost kernel: [<c047a0a0>] cache_flusharray+0x22/0xa0 Feb 18 17:26:26 localhost kernel: [<c047a0a0>] cache_flusharray+0x22/0xa0 Feb 18 17:26:26 localhost kernel: [<c047a1fa>] kmem_cache_free+0x36/0x5e Feb 18 17:26:26 localhost kernel: [<c043b647>] __rcu_process_callbacks+0xfc/0x16e Feb 18 17:26:26 localhost kernel: [<c043b6d1>] rcu_process_callbacks+0x18/0x30 Feb 18 17:26:26 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 17:26:26 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:26:26 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:26 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:26 localhost kernel: } Feb 18 17:26:26 localhost kernel: ... key at: [<c0a2bbe8>] __key.19871+0x0/0x8 Feb 18 17:26:26 localhost kernel: -> (&zone->lock){.+..} ops: 63132 { Feb 18 17:26:26 localhost kernel: initial-use at: Feb 18 17:26:26 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:26 localhost kernel: [<c04ef19a>] number+0x147/0x215 Feb 18 17:26:26 localhost kernel: [<c04efab1>] vsnprintf+0x440/0x47c Feb 18 17:26:26 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:26 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6 Feb 18 17:26:26 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:26 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6 Feb 18 17:26:26 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6 Feb 18 17:26:26 localhost kernel: [<c04645c0>] free_hot_cold_page+0x133/0x175 Feb 18 17:26:26 localhost kernel: [<c07606b8>] free_all_bootmem_core+0xdd/0x158 Feb 18 17:26:26 localhost kernel: [<c075d82e>] mem_init+0x8c/0x383 Feb 18 17:26:26 localhost kernel: [<c042c8c7>] printk+0x1b/0x1f Feb 18 17:26:26 localhost kernel: [<c0761a7b>] alloc_large_system_hash+0x1f7/0x222 Feb 18 17:26:26 localhost kernel: [<c04f2f65>] __spin_lock_init+0x29/0x49 Feb 18 17:26:26 localhost kernel: [<c076287c>] inode_init_early+0x49/0x72 Feb 18 17:26:26 localhost kernel: [<c074e886>] start_kernel+0x28e/0x327 Feb 18 17:26:26 localhost kernel: [<c074e0e0>] unknown_bootoption+0x0/0x195 Feb 18 17:26:26 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:26 localhost kernel: in-softirq-W at: Feb 18 17:26:26 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:26:26 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3 Feb 18 17:26:26 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:26 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6 Feb 18 17:26:27 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:27 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6 Feb 18 17:26:27 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6 Feb 18 17:26:27 localhost kernel: [<c04645c0>] free_hot_cold_page+0x133/0x175 Feb 18 17:26:27 localhost kernel: [<c042a030>] free_task+0x18/0x25 Feb 18 17:26:27 localhost kernel: [<c043b647>] __rcu_process_callbacks+0xfc/0x16e Feb 18 17:26:27 localhost kernel: [<c043b6d1>] rcu_process_callbacks+0x18/0x30 Feb 18 17:26:27 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 17:26:27 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:26:27 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:27 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:27 localhost kernel: } Feb 18 17:26:27 localhost kernel: ... key at: [<c0a21fb8>] __key.25233+0x0/0x8 Feb 18 17:26:27 localhost kernel: ... acquired at: Feb 18 17:26:27 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:27 localhost kernel: [<c046472e>] __free_pages_ok+0x101/0x2a8 Feb 18 17:26:27 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:27 localhost kernel: [<c046472e>] __free_pages_ok+0x101/0x2a8 Feb 18 17:26:27 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:27 localhost kernel: [<c046472e>] __free_pages_ok+0x101/0x2a8 Feb 18 17:26:27 localhost kernel: [<c046472e>] __free_pages_ok+0x101/0x2a8 Feb 18 17:26:27 localhost kernel: [<c0479f5b>] kmem_freepages+0x4e/0xa5 Feb 18 17:26:27 localhost kernel: [<c047a2a2>] slab_destroy+0x2e/0x49 Feb 18 17:26:27 localhost kernel: [<c047a370>] free_block+0xb3/0xec Feb 18 17:26:27 localhost kernel: [<c047a433>] drain_array+0x8a/0xb5 Feb 18 17:26:27 localhost kernel: [<c047b4ec>] cache_reap+0x7e/0xf4 Feb 18 17:26:27 localhost kernel: [<c043a0f3>] run_workqueue+0xdb/0x1ae Feb 18 17:26:27 localhost kernel: [<c043a09f>] run_workqueue+0x87/0x1ae Feb 18 17:26:27 localhost kernel: [<c047b46e>] cache_reap+0x0/0xf4 Feb 18 17:26:27 localhost kernel: [<c043aad8>] worker_thread+0x0/0xc4 Feb 18 17:26:27 localhost kernel: [<c043ab92>] worker_thread+0xba/0xc4 Feb 18 17:26:27 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35 Feb 18 17:26:27 localhost kernel: [<c043d51e>] kthread+0x38/0x5e Feb 18 17:26:27 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e Feb 18 17:26:27 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10 Feb 18 17:26:27 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:27 localhost kernel: Feb 18 17:26:27 localhost kernel: -> (&on_slab_l3_key){.+..} ops: 28386 { Feb 18 17:26:27 localhost kernel: initial-use at: Feb 18 17:26:27 localhost kernel: [<c044816c>] mark_lock+0x65/0x454 Feb 18 17:26:27 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:27 localhost kernel: [<c040a9d6>] save_stack_trace+0x20/0x3a Feb 18 17:26:27 localhost kernel: [<c0464b77>] get_page_from_freelist+0x24b/0x37a Feb 18 17:26:28 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:28 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:28 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493 Feb 18 17:26:28 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:28 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493 Feb 18 17:26:28 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493 Feb 18 17:26:28 localhost kernel: [<c046a328>] mod_zone_page_state+0x35/0x3c Feb 18 17:26:28 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:28 localhost kernel: [<c047a790>] kmem_cache_alloc+0x53/0x9f Feb 18 17:26:28 localhost kernel: [<c047ab07>] cache_alloc_refill+0x32b/0x493 Feb 18 17:26:28 localhost kernel: [<c047a790>] kmem_cache_alloc+0x53/0x9f Feb 18 17:26:28 localhost kernel: [<c075ee99>] pidmap_init+0x15/0x137 Feb 18 17:26:28 localhost kernel: [<c074e8a0>] start_kernel+0x2a8/0x327 Feb 18 17:26:28 localhost kernel: [<c074e0e0>] unknown_bootoption+0x0/0x195 Feb 18 17:26:28 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:28 localhost kernel: in-softirq-W at: Feb 18 17:26:28 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:26:28 localhost kernel: [<c05fdd2b>] fn_hash_lookup+0xaa/0xb4 Feb 18 17:26:28 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:28 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493 Feb 18 17:26:28 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:28 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493 Feb 18 17:26:28 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493 Feb 18 17:26:28 localhost kernel: [<c047acf5>] __kmalloc+0x86/0xd2 Feb 18 17:26:28 localhost kernel: [<c05bb6b9>] neigh_create+0x2e4/0x41a Feb 18 17:26:28 localhost kernel: [<c05baadb>] neigh_lookup+0x94/0x9d Feb 18 17:26:28 localhost kernel: [<c05bb82b>] neigh_event_ns+0x3c/0x76 Feb 18 17:26:28 localhost kernel: [<c05f2116>] arp_process+0x218/0x60c Feb 18 17:26:28 localhost kernel: [<c05f1efe>] arp_process+0x0/0x60c Feb 18 17:26:28 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 17:26:28 localhost kernel: [<c05f250a>] arp_rcv+0x0/0x105 Feb 18 17:26:28 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4 Feb 18 17:26:28 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 18 17:26:28 localhost kernel: [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000] Feb 18 17:26:28 localhost kernel: [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000] Feb 18 17:26:28 localhost kernel: [<f8960f13>] e1000_clean+0x63/0x203 [e1000] Feb 18 17:26:28 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3 Feb 18 17:26:28 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3 Feb 18 17:26:28 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:26:28 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:28 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:28 localhost kernel: } Feb 18 17:26:29 localhost kernel: ... key at: [<c0a2bc0c>] on_slab_l3_key+0x0/0x8 Feb 18 17:26:29 localhost kernel: -> (&zone->lock){.+..} ops: 63132 { Feb 18 17:26:29 localhost kernel: initial-use at: Feb 18 17:26:29 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:29 localhost kernel: [<c04ef19a>] number+0x147/0x215 Feb 18 17:26:29 localhost kernel: [<c04efab1>] vsnprintf+0x440/0x47c Feb 18 17:26:29 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:29 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6 Feb 18 17:26:29 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:29 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6 Feb 18 17:26:29 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6 Feb 18 17:26:29 localhost kernel: [<c04645c0>] free_hot_cold_page+0x133/0x175 Feb 18 17:26:29 localhost kernel: [<c07606b8>] free_all_bootmem_core+0xdd/0x158 Feb 18 17:26:29 localhost kernel: [<c075d82e>] mem_init+0x8c/0x383 Feb 18 17:26:29 localhost kernel: [<c042c8c7>] printk+0x1b/0x1f Feb 18 17:26:29 localhost kernel: [<c0761a7b>] alloc_large_system_hash+0x1f7/0x222 Feb 18 17:26:29 localhost kernel: [<c04f2f65>] __spin_lock_init+0x29/0x49 Feb 18 17:26:29 localhost kernel: [<c076287c>] inode_init_early+0x49/0x72 Feb 18 17:26:29 localhost kernel: [<c074e886>] start_kernel+0x28e/0x327 Feb 18 17:26:29 localhost kernel: [<c074e0e0>] unknown_bootoption+0x0/0x195 Feb 18 17:26:29 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:29 localhost kernel: in-softirq-W at: Feb 18 17:26:29 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 18 17:26:29 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3 Feb 18 17:26:29 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:29 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6 Feb 18 17:26:29 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:29 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6 Feb 18 17:26:29 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6 Feb 18 17:26:29 localhost kernel: [<c04645c0>] free_hot_cold_page+0x133/0x175 Feb 18 17:26:29 localhost kernel: [<c042a030>] free_task+0x18/0x25 Feb 18 17:26:29 localhost kernel: [<c043b647>] __rcu_process_callbacks+0xfc/0x16e Feb 18 17:26:29 localhost kernel: [<c043b6d1>] rcu_process_callbacks+0x18/0x30 Feb 18 17:26:29 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4 Feb 18 17:26:29 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 18 17:26:29 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 18 17:26:29 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:29 localhost kernel: } Feb 18 17:26:29 localhost kernel: ... key at: [<c0a21fb8>] __key.25233+0x0/0x8 Feb 18 17:26:29 localhost kernel: ... acquired at: Feb 18 17:26:29 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:29 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6 Feb 18 17:26:29 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:29 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6 Feb 18 17:26:30 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:30 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6 Feb 18 17:26:30 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6 Feb 18 17:26:30 localhost kernel: [<c04645c0>] free_hot_cold_page+0x133/0x175 Feb 18 17:26:30 localhost kernel: [<c047a2a2>] slab_destroy+0x2e/0x49 Feb 18 17:26:30 localhost kernel: [<c047a370>] free_block+0xb3/0xec Feb 18 17:26:30 localhost kernel: [<c047a0f2>] cache_flusharray+0x74/0xa0 Feb 18 17:26:30 localhost kernel: [<c047a19c>] kfree+0x7e/0xa6 Feb 18 17:26:30 localhost kernel: [<c05f594c>] snmp_mib_free+0x1a/0x29 Feb 18 17:26:30 localhost kernel: [<f8f283ac>] in6_dev_finish_destroy+0x8e/0xb9 [ipv6] Feb 18 17:26:30 localhost kernel: [<f8f28dd9>] addrconf_ifdown+0x2df/0x2eb [ipv6] Feb 18 17:26:30 localhost kernel: [<f8f2b0f5>] addrconf_notify+0x738/0x7c2 [ipv6] Feb 18 17:26:30 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:30 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:30 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:26:30 localhost kernel: [<c0614a16>] packet_notifier+0x149/0x151 Feb 18 17:26:30 localhost kernel: [<c06148f3>] packet_notifier+0x26/0x151 Feb 18 17:26:30 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c Feb 18 17:26:30 localhost kernel: [<c0614a16>] packet_notifier+0x149/0x151 Feb 18 17:26:30 localhost kernel: [<c061d06c>] notifier_call_chain+0x2a/0x47 Feb 18 17:26:30 localhost kernel: [<c0441084>] raw_notifier_call_chain+0x17/0x1a Feb 18 17:26:30 localhost kernel: [<c05b621c>] rollback_registered+0xc4/0x11e Feb 18 17:26:30 localhost kernel: [<c05b627e>] unregister_netdevice+0x8/0x35 Feb 18 17:26:30 localhost kernel: [<c05b62ba>] unregister_netdev+0xf/0x15 Feb 18 17:26:30 localhost kernel: [<f8bb523f>] ppp_shutdown_interface+0x58/0xad [ppp_generic] Feb 18 17:26:30 localhost kernel: [<f8bb5484>] ppp_release+0x29/0x56 [ppp_generic] Feb 18 17:26:30 localhost kernel: [<c047f08a>] __fput+0xba/0x172 Feb 18 17:26:30 localhost kernel: [<c047c7df>] filp_close+0x51/0x58 Feb 18 17:26:30 localhost kernel: [<c047d9c4>] sys_close+0x70/0xab Feb 18 17:26:30 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 17:26:30 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:30 localhost kernel: Feb 18 17:26:30 localhost kernel: ... acquired at: Feb 18 17:26:30 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:30 localhost kernel: [<c047a0a0>] cache_flusharray+0x22/0xa0 Feb 18 17:26:30 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:30 localhost kernel: [<c047a0a0>] cache_flusharray+0x22/0xa0 Feb 18 17:26:30 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:30 localhost kernel: [<c047a0a0>] cache_flusharray+0x22/0xa0 Feb 18 17:26:30 localhost kernel: [<c047a0a0>] cache_flusharray+0x22/0xa0 Feb 18 17:26:30 localhost kernel: [<c047a1fa>] kmem_cache_free+0x36/0x5e Feb 18 17:26:30 localhost kernel: [<c047a370>] free_block+0xb3/0xec Feb 18 17:26:30 localhost kernel: [<c047a0f2>] cache_flusharray+0x74/0xa0 Feb 18 17:26:31 localhost kernel: [<c047a19c>] kfree+0x7e/0xa6 Feb 18 17:26:31 localhost kernel: [<f8f28d8e>] addrconf_ifdown+0x294/0x2eb [ipv6] Feb 18 17:26:31 localhost kernel: [<f8f2b0f5>] addrconf_notify+0x738/0x7c2 [ipv6] Feb 18 17:26:31 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:31 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:31 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:26:31 localhost kernel: [<c0614a16>] packet_notifier+0x149/0x151 Feb 18 17:26:31 localhost kernel: [<c06148f3>] packet_notifier+0x26/0x151 Feb 18 17:26:31 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c Feb 18 17:26:31 localhost kernel: [<c0614a16>] packet_notifier+0x149/0x151 Feb 18 17:26:31 localhost kernel: [<c061d06c>] notifier_call_chain+0x2a/0x47 Feb 18 17:26:31 localhost kernel: [<c0441084>] raw_notifier_call_chain+0x17/0x1a Feb 18 17:26:31 localhost kernel: [<c05b621c>] rollback_registered+0xc4/0x11e Feb 18 17:26:31 localhost kernel: [<c05b627e>] unregister_netdevice+0x8/0x35 Feb 18 17:26:31 localhost kernel: [<c05b62ba>] unregister_netdev+0xf/0x15 Feb 18 17:26:31 localhost kernel: [<f8bb523f>] ppp_shutdown_interface+0x58/0xad [ppp_generic] Feb 18 17:26:31 localhost kernel: [<f8bb5484>] ppp_release+0x29/0x56 [ppp_generic] Feb 18 17:26:31 localhost kernel: [<c047f08a>] __fput+0xba/0x172 Feb 18 17:26:31 localhost kernel: [<c047c7df>] filp_close+0x51/0x58 Feb 18 17:26:31 localhost kernel: [<c047d9c4>] sys_close+0x70/0xab Feb 18 17:26:31 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 17:26:31 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:31 localhost kernel: Feb 18 17:26:31 localhost kernel: ... acquired at: Feb 18 17:26:31 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:31 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493 Feb 18 17:26:31 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:31 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493 Feb 18 17:26:31 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:31 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493 Feb 18 17:26:31 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493 Feb 18 17:26:31 localhost kernel: [<c047a790>] kmem_cache_alloc+0x53/0x9f Feb 18 17:26:31 localhost kernel: [<c05bb516>] neigh_create+0x141/0x41a Feb 18 17:26:31 localhost kernel: [<c05baadb>] neigh_lookup+0x94/0x9d Feb 18 17:26:31 localhost kernel: [<c05f144f>] arp_bind_neighbour+0x62/0x74 Feb 18 17:26:31 localhost kernel: [<c05cf248>] rt_intern_hash+0x236/0x322 Feb 18 17:26:31 localhost kernel: [<c05cfab6>] __ip_route_output_key+0x782/0x847 Feb 18 17:26:31 localhost kernel: [<c05cf948>] __ip_route_output_key+0x614/0x847 Feb 18 17:26:31 localhost kernel: [<c05eace7>] tcp_v4_connect+0xef/0x5a6 Feb 18 17:26:31 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 17:26:31 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115 Feb 18 17:26:32 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:32 localhost kernel: [<c05f6944>] inet_stream_connect+0x7f/0x208 Feb 18 17:26:32 localhost kernel: [<c04f05f4>] copy_from_user+0x32/0x5e Feb 18 17:26:32 localhost kernel: [<c05ac371>] sys_connect+0x7b/0xa6 Feb 18 17:26:32 localhost kernel: [<c05aea5a>] sock_setsockopt+0x492/0x4aa Feb 18 17:26:32 localhost kernel: [<c05aea5a>] sock_setsockopt+0x492/0x4aa Feb 18 17:26:32 localhost kernel: [<c05db937>] tcp_setsockopt+0x321/0x339 Feb 18 17:26:32 localhost kernel: [<c05ab618>] sock_map_fd+0x41/0x4a Feb 18 17:26:32 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 18 17:26:32 localhost kernel: [<c05ab47f>] sys_setsockopt+0x62/0x9c Feb 18 17:26:32 localhost kernel: [<c05acb97>] sys_socketcall+0xac/0x261 Feb 18 17:26:32 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:32 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5 Feb 18 17:26:32 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:32 localhost kernel: Feb 18 17:26:32 localhost kernel: ... acquired at: Feb 18 17:26:32 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:32 localhost kernel: [<c05cf06f>] rt_intern_hash+0x5d/0x322 Feb 18 17:26:32 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:32 localhost kernel: [<c05cf06f>] rt_intern_hash+0x5d/0x322 Feb 18 17:26:32 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 17:26:32 localhost kernel: [<c05cf06f>] rt_intern_hash+0x5d/0x322 Feb 18 17:26:32 localhost kernel: [<c05cf06f>] rt_intern_hash+0x5d/0x322 Feb 18 17:26:32 localhost kernel: [<c05cf000>] rt_set_nexthop+0x105/0x117 Feb 18 17:26:32 localhost kernel: [<c05cfab6>] __ip_route_output_key+0x782/0x847 Feb 18 17:26:32 localhost kernel: [<c05cf948>] __ip_route_output_key+0x614/0x847 Feb 18 17:26:32 localhost kernel: [<c0464b77>] get_page_from_freelist+0x24b/0x37a Feb 18 17:26:32 localhost kernel: [<c05cfb8f>] ip_route_output_flow+0x14/0x1ef Feb 18 17:26:32 localhost kernel: [<c05b0c83>] pskb_expand_head+0xde/0x144 Feb 18 17:26:32 localhost kernel: [<f8bbf6fe>] pppol2tp_xmit+0x365/0x4a2 [pppol2tp] Feb 18 17:26:32 localhost kernel: [<f8bb730e>] ppp_channel_push+0x34/0x90 [ppp_generic] Feb 18 17:26:32 localhost kernel: [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic] Feb 18 17:26:32 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic] Feb 18 17:26:32 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d Feb 18 17:26:32 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67 Feb 18 17:26:32 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 17:26:32 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:32 localhost kernel: Feb 18 17:26:32 localhost kernel: ... acquired at: Feb 18 17:26:32 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:32 localhost kernel: [<f8bb56db>] ppp_push+0x63/0x50d [ppp_generic] Feb 18 17:26:32 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:32 localhost kernel: [<f8bb56db>] ppp_push+0x63/0x50d [ppp_generic] Feb 18 17:26:33 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 17:26:33 localhost kernel: [<f8bb56db>] ppp_push+0x63/0x50d [ppp_generic] Feb 18 17:26:33 localhost kernel: [<f8bb56db>] ppp_push+0x63/0x50d [ppp_generic] Feb 18 17:26:33 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:26:33 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 18 17:26:33 localhost kernel: [<f8bb63f1>] ppp_xmit_process+0x4f6/0x5a1 [ppp_generic] Feb 18 17:26:33 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:33 localhost kernel: [<f8bb83b4>] ppp_write+0xc7/0xdb [ppp_generic] Feb 18 17:26:33 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic] Feb 18 17:26:33 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d Feb 18 17:26:33 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67 Feb 18 17:26:33 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 17:26:33 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:33 localhost kernel: Feb 18 17:26:33 localhost kernel: ... acquired at: Feb 18 17:26:33 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1 Feb 18 17:26:33 localhost kernel: [<f8bb6bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic] Feb 18 17:26:33 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:33 localhost kernel: [<f8bb6bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic] Feb 18 17:26:33 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 18 17:26:33 localhost kernel: [<f8bb6bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic] Feb 18 17:26:33 localhost kernel: [<f8bb6bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic] Feb 18 17:26:33 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 17:26:33 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 18 17:26:33 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:33 localhost kernel: [<c061b008>] __down+0x82/0xb8 Feb 18 17:26:33 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8 Feb 18 17:26:33 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc Feb 18 17:26:33 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62 Feb 18 17:26:33 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249 Feb 18 17:26:33 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d Feb 18 17:26:33 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 17:26:33 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:33 localhost kernel: Feb 18 17:26:33 localhost kernel: Feb 18 17:26:33 localhost kernel: the soft-irq-unsafe lock's dependencies: Feb 18 17:26:33 localhost kernel: -> (&sk->sk_dst_lock){----} ops: 3988 { Feb 18 17:26:33 localhost kernel: initial-use at: Feb 18 17:26:33 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1 Feb 18 17:26:33 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 17:26:33 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115 Feb 18 17:26:33 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb Feb 18 17:26:33 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 17:26:34 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34 Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 17:26:34 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e Feb 18 17:26:34 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f Feb 18 17:26:34 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261 Feb 18 17:26:34 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5 Feb 18 17:26:34 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:34 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5 Feb 18 17:26:34 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:34 localhost kernel: softirq-on-W at: Feb 18 17:26:34 localhost kernel: [<c0449046>] __lock_acquire+0x48b/0xbf1 Feb 18 17:26:34 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 17:26:34 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115 Feb 18 17:26:34 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb Feb 18 17:26:34 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 17:26:34 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34 Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 17:26:34 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e Feb 18 17:26:34 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f Feb 18 17:26:34 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261 Feb 18 17:26:34 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5 Feb 18 17:26:34 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:34 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5 Feb 18 17:26:34 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:34 localhost kernel: hardirq-on-W at: Feb 18 17:26:34 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1 Feb 18 17:26:34 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 17:26:34 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115 Feb 18 17:26:34 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb Feb 18 17:26:34 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 17:26:34 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34 Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 18 17:26:34 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e Feb 18 17:26:34 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f Feb 18 17:26:34 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261 Feb 18 17:26:34 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5 Feb 18 17:26:34 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 18 17:26:35 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5 Feb 18 17:26:35 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:35 localhost kernel: softirq-on-R at: Feb 18 17:26:35 localhost kernel: [<c0449046>] __lock_acquire+0x48b/0xbf1 Feb 18 17:26:35 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:35 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105 Feb 18 17:26:35 localhost kernel: [<c061b3a7>] _read_lock+0x29/0x34 Feb 18 17:26:35 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105 Feb 18 17:26:35 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105 Feb 18 17:26:35 localhost kernel: [<c05baadb>] neigh_lookup+0x94/0x9d Feb 18 17:26:35 localhost kernel: [<c05f0609>] udp_sendmsg+0x240/0x59d Feb 18 17:26:35 localhost kernel: [<c05cf322>] rt_intern_hash+0x310/0x322 Feb 18 17:26:35 localhost kernel: [<c05f5fc7>] inet_sendmsg+0x3b/0x45 Feb 18 17:26:35 localhost kernel: [<c05ab8cd>] sock_sendmsg+0xc9/0xe4 Feb 18 17:26:35 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35 Feb 18 17:26:35 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:35 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 17:26:35 localhost kernel: [<c061b528>] _read_unlock_irq+0x20/0x23 Feb 18 17:26:35 localhost kernel: [<c061b528>] _read_unlock_irq+0x20/0x23 Feb 18 17:26:35 localhost kernel: [<c05ac29b>] sys_sendto+0x11b/0x13b Feb 18 17:26:35 localhost kernel: [<c046b794>] __do_fault+0x23b/0x35d Feb 18 17:26:35 localhost kernel: [<c046b874>] __do_fault+0x31b/0x35d Feb 18 17:26:35 localhost kernel: [<c046d4ff>] handle_mm_fault+0x2a2/0x5f7 Feb 18 17:26:35 localhost kernel: [<c05ac2f2>] sys_send+0x37/0x3b Feb 18 17:26:35 localhost kernel: [<c05acc35>] sys_socketcall+0x14a/0x261 Feb 18 17:26:35 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 17:26:35 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:35 localhost kernel: hardirq-on-R at: Feb 18 17:26:35 localhost kernel: [<c048926a>] do_sys_poll+0x260/0x2e0 Feb 18 17:26:35 localhost kernel: [<c0449001>] __lock_acquire+0x446/0xbf1 Feb 18 17:26:35 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:35 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105 Feb 18 17:26:35 localhost kernel: [<c061b3a7>] _read_lock+0x29/0x34 Feb 18 17:26:35 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105 Feb 18 17:26:35 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105 Feb 18 17:26:35 localhost kernel: [<c05baadb>] neigh_lookup+0x94/0x9d Feb 18 17:26:35 localhost kernel: [<c05f0609>] udp_sendmsg+0x240/0x59d Feb 18 17:26:35 localhost kernel: [<c05cf322>] rt_intern_hash+0x310/0x322 Feb 18 17:26:35 localhost kernel: [<c05f5fc7>] inet_sendmsg+0x3b/0x45 Feb 18 17:26:35 localhost kernel: [<c05ab8cd>] sock_sendmsg+0xc9/0xe4 Feb 18 17:26:35 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35 Feb 18 17:26:35 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1 Feb 18 17:26:35 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 18 17:26:36 localhost kernel: [<c061b528>] _read_unlock_irq+0x20/0x23 Feb 18 17:26:36 localhost kernel: [<c061b528>] _read_unlock_irq+0x20/0x23 Feb 18 17:26:36 localhost kernel: [<c05ac29b>] sys_sendto+0x11b/0x13b Feb 18 17:26:36 localhost kernel: [<c046b794>] __do_fault+0x23b/0x35d Feb 18 17:26:36 localhost kernel: [<c046b874>] __do_fault+0x31b/0x35d Feb 18 17:26:36 localhost kernel: [<c046d4ff>] handle_mm_fault+0x2a2/0x5f7 Feb 18 17:26:36 localhost kernel: [<c05ac2f2>] sys_send+0x37/0x3b Feb 18 17:26:36 localhost kernel: [<c05acc35>] sys_socketcall+0x14a/0x261 Feb 18 17:26:36 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 17:26:36 localhost kernel: [<ffffffff>] 0xffffffff Feb 18 17:26:36 localhost kernel: } Feb 18 17:26:36 localhost kernel: ... key at: [<c0a3dc28>] __key.35996+0x0/0x8 Feb 18 17:26:36 localhost kernel: Feb 18 17:26:36 localhost kernel: stack backtrace: Feb 18 17:26:36 localhost kernel: Pid: 13863, comm: pppd Not tainted 2.6.24.2 #1 Feb 18 17:26:36 localhost kernel: [<c0448b1e>] check_usage+0x24e/0x258 Feb 18 17:26:36 localhost kernel: [<c0449555>] __lock_acquire+0x99a/0xbf1 Feb 18 17:26:36 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 18 17:26:36 localhost kernel: [<f8bbf73e>] pppol2tp_xmit+0x3a5/0x4a2 [pppol2tp] Feb 18 17:26:36 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34 Feb 18 17:26:36 localhost kernel: [<f8bbf73e>] pppol2tp_xmit+0x3a5/0x4a2 [pppol2tp] Feb 18 17:26:36 localhost kernel: [<f8bbf73e>] pppol2tp_xmit+0x3a5/0x4a2 [pppol2tp] Feb 18 17:26:36 localhost kernel: [<f8bb730e>] ppp_channel_push+0x34/0x90 [ppp_generic] Feb 18 17:26:36 localhost kernel: [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic] Feb 18 17:26:36 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic] Feb 18 17:26:36 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d Feb 18 17:26:36 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67 Feb 18 17:26:36 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 18 17:26:36 localhost kernel: ======================= ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-18 22:09 ` James Chapman 2008-02-18 23:01 ` Jarek Poplawski 2008-02-19 4:29 ` David Miller @ 2008-02-19 23:06 ` Jarek Poplawski 2008-02-19 23:28 ` Jarek Poplawski 2008-02-20 16:02 ` James Chapman 2 siblings, 2 replies; 51+ messages in thread From: Jarek Poplawski @ 2008-02-19 23:06 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, Paul Mackerras, netdev On Mon, Feb 18, 2008 at 10:09:24PM +0000, James Chapman wrote: ... > Unfortunately the ISP's syslog stops. But I've been able to borrow > two Quad Xeon boxes and have reproduced the problem. > > Here's a new version of the patch. The patch avoids disabling irqs > and fixes the sk_dst_get() usage that DaveM mentioned. But even with > this patch, lockdep still complains if hundreds of ppp sessions are > inserted into a tunnel as rapidly as possible (lockdep trace is below). > I can stop these errors by wrapping the call to ppp_input() in > pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is a > better fix? I send here my proposal: it's intended for testing and to check one of possible solutions here. IMHO your lockdep reports show there is no use to change anything around sk_dst_lock: it would need the global change of this lock to fix this problem. So the fix should be done around pch->upl lock and this means changing ppp_generic. In the patch below I've used trylock in places which seem to allow for skipping some things (while config is changed only) or simply don't need this lock because there is no ppp struct. This could be modified to add some waiting loop if necessary. Another option is to change the write side of this lock: it looks like more vulnerable if something missed because there are more locks involved, but probably should be enough to solve this problem too. I think pppol2tp need to be first checked only with hlist_lock bh patch, unless there were some lockdep reports on these other locks too. (BTW, I added ppp maintainer to CC - I hope we get Paul's opinion on this.) Regards, Jarek P. (testing patch #1) --- drivers/net/ppp_generic.c | 33 +++++++++++++++++++++++---------- 1 files changed, 23 insertions(+), 10 deletions(-) diff --git a/drivers/net/ppp_generic.c b/drivers/net/ppp_generic.c index 4dc5b4b..5cbc534 100644 --- a/drivers/net/ppp_generic.c +++ b/drivers/net/ppp_generic.c @@ -1473,7 +1473,7 @@ void ppp_input(struct ppp_channel *chan, struct sk_buff *skb) { struct channel *pch = chan->ppp; - int proto; + int proto, locked; if (!pch || skb->len == 0) { kfree_skb(skb); @@ -1481,8 +1481,13 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb) } proto = PPP_PROTO(skb); - read_lock_bh(&pch->upl); - if (!pch->ppp || proto >= 0xc000 || proto == PPP_CCPFRAG) { + /* + * We use trylock to avoid dependency between soft-irq-safe upl lock + * and soft-irq-unsafe sk_dst_lock. + */ + local_bh_disable(); + locked = read_trylock(&pch->upl); + if (!locked || !pch->ppp || proto >= 0xc000 || proto == PPP_CCPFRAG) { /* put it on the channel queue */ skb_queue_tail(&pch->file.rq, skb); /* drop old frames if queue too long */ @@ -1493,7 +1498,10 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb) } else { ppp_do_recv(pch->ppp, skb, pch); } - read_unlock_bh(&pch->upl); + + if (locked) + read_unlock(&pch->upl); + local_bh_enable(); } /* Put a 0-length skb in the receive queue as an error indication */ @@ -1506,16 +1514,18 @@ ppp_input_error(struct ppp_channel *chan, int code) if (!pch) return; - read_lock_bh(&pch->upl); - if (pch->ppp) { + /* a trylock comment in ppp_input() */ + local_bh_disable(); + if (read_trylock(&pch->upl) && pch->ppp) { skb = alloc_skb(0, GFP_ATOMIC); if (skb) { skb->len = 0; /* probably unnecessary */ skb->cb[0] = code; ppp_do_recv(pch->ppp, skb, pch); } + read_unlock(&pch->upl); } - read_unlock_bh(&pch->upl); + local_bh_enable(); } /* @@ -2044,10 +2054,13 @@ int ppp_unit_number(struct ppp_channel *chan) int unit = -1; if (pch) { - read_lock_bh(&pch->upl); - if (pch->ppp) + /* a trylock comment in ppp_input() */ + local_bh_disable(); + if (read_trylock(&pch->upl) && pch->ppp) { unit = pch->ppp->file.index; - read_unlock_bh(&pch->upl); + read_unlock(&pch->upl); + } + local_bh_enable(); } return unit; } ^ permalink raw reply related [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-19 23:06 ` Jarek Poplawski @ 2008-02-19 23:28 ` Jarek Poplawski 2008-02-20 16:02 ` James Chapman 1 sibling, 0 replies; 51+ messages in thread From: Jarek Poplawski @ 2008-02-19 23:28 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, Paul Mackerras, netdev On Wed, Feb 20, 2008 at 12:06:40AM +0100, Jarek Poplawski wrote: ... > (testing patch #1) SORRY!!! ----> take 2 (unlocking fixed) --- drivers/net/ppp_generic.c | 39 +++++++++++++++++++++++++++++---------- 1 files changed, 29 insertions(+), 10 deletions(-) diff --git a/drivers/net/ppp_generic.c b/drivers/net/ppp_generic.c index 4dc5b4b..c4e3808 100644 --- a/drivers/net/ppp_generic.c +++ b/drivers/net/ppp_generic.c @@ -1473,7 +1473,7 @@ void ppp_input(struct ppp_channel *chan, struct sk_buff *skb) { struct channel *pch = chan->ppp; - int proto; + int proto, locked; if (!pch || skb->len == 0) { kfree_skb(skb); @@ -1481,8 +1481,13 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb) } proto = PPP_PROTO(skb); - read_lock_bh(&pch->upl); - if (!pch->ppp || proto >= 0xc000 || proto == PPP_CCPFRAG) { + /* + * We use trylock to avoid dependency between soft-irq-safe upl lock + * and soft-irq-unsafe sk_dst_lock. + */ + local_bh_disable(); + locked = read_trylock(&pch->upl); + if (!locked || !pch->ppp || proto >= 0xc000 || proto == PPP_CCPFRAG) { /* put it on the channel queue */ skb_queue_tail(&pch->file.rq, skb); /* drop old frames if queue too long */ @@ -1493,7 +1498,10 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb) } else { ppp_do_recv(pch->ppp, skb, pch); } - read_unlock_bh(&pch->upl); + + if (locked) + read_unlock(&pch->upl); + local_bh_enable(); } /* Put a 0-length skb in the receive queue as an error indication */ @@ -1502,12 +1510,14 @@ ppp_input_error(struct ppp_channel *chan, int code) { struct channel *pch = chan->ppp; struct sk_buff *skb; + int locked; if (!pch) return; - read_lock_bh(&pch->upl); - if (pch->ppp) { + /* a trylock comment in ppp_input() */ + local_bh_disable(); + if ((locked = read_trylock(&pch->upl)) && pch->ppp) { skb = alloc_skb(0, GFP_ATOMIC); if (skb) { skb->len = 0; /* probably unnecessary */ @@ -1515,7 +1525,10 @@ ppp_input_error(struct ppp_channel *chan, int code) ppp_do_recv(pch->ppp, skb, pch); } } - read_unlock_bh(&pch->upl); + + if (locked) + read_unlock(&pch->upl); + local_bh_enable(); } /* @@ -2044,10 +2057,16 @@ int ppp_unit_number(struct ppp_channel *chan) int unit = -1; if (pch) { - read_lock_bh(&pch->upl); - if (pch->ppp) + int locked; + + /* a trylock comment in ppp_input() */ + local_bh_disable(); + if ((locked = read_trylock(&pch->upl)) && pch->ppp) unit = pch->ppp->file.index; - read_unlock_bh(&pch->upl); + + if (locked) + read_unlock(&pch->upl); + local_bh_enable(); } return unit; } ^ permalink raw reply related [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-19 23:06 ` Jarek Poplawski 2008-02-19 23:28 ` Jarek Poplawski @ 2008-02-20 16:02 ` James Chapman 2008-02-20 18:38 ` Jarek Poplawski 1 sibling, 1 reply; 51+ messages in thread From: James Chapman @ 2008-02-20 16:02 UTC (permalink / raw) To: Jarek Poplawski; +Cc: David Miller, Paul Mackerras, netdev Jarek Poplawski wrote: > On Mon, Feb 18, 2008 at 10:09:24PM +0000, James Chapman wrote: > ... >> Unfortunately the ISP's syslog stops. But I've been able to borrow >> two Quad Xeon boxes and have reproduced the problem. >> >> Here's a new version of the patch. The patch avoids disabling irqs >> and fixes the sk_dst_get() usage that DaveM mentioned. But even with >> this patch, lockdep still complains if hundreds of ppp sessions are >> inserted into a tunnel as rapidly as possible (lockdep trace is below). >> I can stop these errors by wrapping the call to ppp_input() in >> pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is a >> better fix? > > I send here my proposal: it's intended for testing and to check one of > possible solutions here. IMHO your lockdep reports show there is no > use to change anything around sk_dst_lock: it would need the global > change of this lock to fix this problem. So the fix should be done > around pch->upl lock and this means changing ppp_generic. Hmm, I need to study the lockdep report again. It seems I'm misreading the lockdep output. :( > In the patch below I've used trylock in places which seem to allow > for skipping some things (while config is changed only) or simply > don't need this lock because there is no ppp struct. This could be > modified to add some waiting loop if necessary. Another option is to > change the write side of this lock: it looks like more vulnerable if > something missed because there are more locks involved, but probably > should be enough to solve this problem too. > > I think pppol2tp need to be first checked only with hlist_lock bh > patch, unless there were some lockdep reports on these other locks > too. (BTW, I added ppp maintainer to CC - I hope we get Paul's opinion > on this.) I tried your ppp_generic patch with only the hlist_lock bh patch in pppol2tp and it seems to fix the ppp create/delete issue. However, when I added much more traffic into the test (flood pings over ppp interfaces while repeatedly creating/deleting the L2TP (PPP) sessions) I get a soft lockup detected in pppol2tp_xmit() after anything between 1 minute and an hour. :( I'm investigating that now. Thanks for your help! > (testing patch #1) > --- > > drivers/net/ppp_generic.c | 33 +++++++++++++++++++++++---------- > 1 files changed, 23 insertions(+), 10 deletions(-) > > diff --git a/drivers/net/ppp_generic.c b/drivers/net/ppp_generic.c > index 4dc5b4b..5cbc534 100644 > --- a/drivers/net/ppp_generic.c > +++ b/drivers/net/ppp_generic.c > @@ -1473,7 +1473,7 @@ void > ppp_input(struct ppp_channel *chan, struct sk_buff *skb) > { > struct channel *pch = chan->ppp; > - int proto; > + int proto, locked; > > if (!pch || skb->len == 0) { > kfree_skb(skb); > @@ -1481,8 +1481,13 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb) > } > > proto = PPP_PROTO(skb); > - read_lock_bh(&pch->upl); > - if (!pch->ppp || proto >= 0xc000 || proto == PPP_CCPFRAG) { > + /* > + * We use trylock to avoid dependency between soft-irq-safe upl lock > + * and soft-irq-unsafe sk_dst_lock. > + */ > + local_bh_disable(); > + locked = read_trylock(&pch->upl); > + if (!locked || !pch->ppp || proto >= 0xc000 || proto == PPP_CCPFRAG) { > /* put it on the channel queue */ > skb_queue_tail(&pch->file.rq, skb); > /* drop old frames if queue too long */ > @@ -1493,7 +1498,10 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb) > } else { > ppp_do_recv(pch->ppp, skb, pch); > } > - read_unlock_bh(&pch->upl); > + > + if (locked) > + read_unlock(&pch->upl); > + local_bh_enable(); > } > > /* Put a 0-length skb in the receive queue as an error indication */ > @@ -1506,16 +1514,18 @@ ppp_input_error(struct ppp_channel *chan, int code) > if (!pch) > return; > > - read_lock_bh(&pch->upl); > - if (pch->ppp) { > + /* a trylock comment in ppp_input() */ > + local_bh_disable(); > + if (read_trylock(&pch->upl) && pch->ppp) { > skb = alloc_skb(0, GFP_ATOMIC); > if (skb) { > skb->len = 0; /* probably unnecessary */ > skb->cb[0] = code; > ppp_do_recv(pch->ppp, skb, pch); > } > + read_unlock(&pch->upl); > } > - read_unlock_bh(&pch->upl); > + local_bh_enable(); > } > > /* > @@ -2044,10 +2054,13 @@ int ppp_unit_number(struct ppp_channel *chan) > int unit = -1; > > if (pch) { > - read_lock_bh(&pch->upl); > - if (pch->ppp) > + /* a trylock comment in ppp_input() */ > + local_bh_disable(); > + if (read_trylock(&pch->upl) && pch->ppp) { > unit = pch->ppp->file.index; > - read_unlock_bh(&pch->upl); > + read_unlock(&pch->upl); > + } > + local_bh_enable(); > } > return unit; > } > -- -- James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-20 16:02 ` James Chapman @ 2008-02-20 18:38 ` Jarek Poplawski 2008-02-20 22:37 ` James Chapman 0 siblings, 1 reply; 51+ messages in thread From: Jarek Poplawski @ 2008-02-20 18:38 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, Paul Mackerras, netdev On Wed, Feb 20, 2008 at 04:02:52PM +0000, James Chapman wrote: ... > I tried your ppp_generic patch with only the hlist_lock bh patch in > pppol2tp and it seems to fix the ppp create/delete issue. However, when > I added much more traffic into the test (flood pings over ppp interfaces > while repeatedly creating/deleting the L2TP (PPP) sessions) I get a soft > lockup detected in pppol2tp_xmit() after anything between 1 minute and > an hour. :( I'm investigating that now. > > Thanks for your help! Not at all! > >> (testing patch #1) But I hope you tested with the fixed (take 2) version of this patch... Since it's quite experimental (testing) this patch could be wrong as it is, but I hope it should show the proper way to solve this problem. Probably you did some of these, but here are a few of my suggestions for testing this: 1) try my patch with your full bh locking changing patch; 2) add while loops to these trylocks on failure, with e.g. __delay(1); this should work like full locks again, but there should be no (this kind of) lockdep reports; 3) I send here another testing patch with this second way to do this: on the write side, but it's even more "experimental" and only a proof of concept (should be applied on vanilla ppp_generic). Regards, Jarek P. (testing patch #2) --- drivers/net/ppp_generic.c | 20 +++++++++++++------- 1 files changed, 13 insertions(+), 7 deletions(-) diff --git a/drivers/net/ppp_generic.c b/drivers/net/ppp_generic.c index 4dc5b4b..70bd255 100644 --- a/drivers/net/ppp_generic.c +++ b/drivers/net/ppp_generic.c @@ -2606,11 +2606,16 @@ ppp_connect_channel(struct channel *pch, int unit) ppp = ppp_find_unit(unit); if (!ppp) goto out; - write_lock_bh(&pch->upl); + ret = -EINVAL; - if (pch->ppp) - goto outl; + read_lock_bh(&pch->upl); + if (pch->ppp) { + read_unlock_bh(&pch->upl); + goto out; + } + read_unlock_bh(&pch->upl); + atomic_inc(&ppp->file.refcnt); ppp_lock(ppp); if (pch->file.hdrlen > ppp->file.hdrlen) ppp->file.hdrlen = pch->file.hdrlen; @@ -2619,13 +2624,14 @@ ppp_connect_channel(struct channel *pch, int unit) ppp->dev->hard_header_len = hdrlen; list_add_tail(&pch->clist, &ppp->channels); ++ppp->n_channels; - pch->ppp = ppp; - atomic_inc(&ppp->file.refcnt); ppp_unlock(ppp); - ret = 0; - outl: + /* avoid lock dependency with ppp_locks */ + write_lock_bh(&pch->upl); + BUG_ON(pch->ppp); + pch->ppp = ppp; write_unlock_bh(&pch->upl); + ret = 0; out: mutex_unlock(&all_ppp_mutex); return ret; ^ permalink raw reply related [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-20 18:38 ` Jarek Poplawski @ 2008-02-20 22:37 ` James Chapman 2008-02-21 8:59 ` Jarek Poplawski 0 siblings, 1 reply; 51+ messages in thread From: James Chapman @ 2008-02-20 22:37 UTC (permalink / raw) To: Jarek Poplawski; +Cc: David Miller, Paul Mackerras, netdev [-- Attachment #1: Type: text/plain, Size: 1152 bytes --] Jarek Poplawski wrote: >>> (testing patch #1) > > But I hope you tested with the fixed (take 2) version of this patch... Yes I did. :) But I just got another lockdep error (attached). > Since it's quite experimental (testing) this patch could be wrong > as it is, but I hope it should show the proper way to solve this > problem. Probably you did some of these, but here are a few of my > suggestions for testing this: > > 1) try my patch with your full bh locking changing patch; > 2) add while loops to these trylocks on failure, with e.g. __delay(1); > this should work like full locks again, but there should be no (this > kind of) lockdep reports; Hmm, isn't this just bypassing the lockdep checks? > 3) I send here another testing patch with this second way to do this: > on the write side, but it's even more "experimental" and only a > proof of concept (should be applied on vanilla ppp_generic). I'll look over it. I think I need to take a step back and look at what's happening in more detail though. -- James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development [-- Attachment #2: lockdep.log --] [-- Type: text/plain, Size: 8600 bytes --] Feb 20 22:11:41 localhost kernel: ================================= Feb 20 22:11:43 localhost kernel: [ INFO: inconsistent lock state ] Feb 20 22:11:44 localhost kernel: 2.6.24.2 #1 Feb 20 22:11:44 localhost kernel: --------------------------------- Feb 20 22:11:44 localhost kernel: inconsistent {softirq-on-W} -> {in-softirq-R} usage. Feb 20 22:11:44 localhost kernel: pppd/3744 [HC0[0]:SC1[5]:HE1:SE0] takes: Feb 20 22:11:44 localhost kernel: (&sk->sk_dst_lock){---?}, at: [<f8bac857>] pppol2tp_xmit+0x320/0x3d9 [pppol2tp] Feb 20 22:11:45 localhost kernel: {softirq-on-W} state was registered at: Feb 20 22:11:45 localhost kernel: [<c0449046>] __lock_acquire+0x48b/0xbf1 Feb 20 22:11:45 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 20 22:11:45 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115 Feb 20 22:11:45 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb Feb 20 22:11:45 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 20 22:11:45 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 20 22:11:45 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34 Feb 20 22:11:45 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 20 22:11:45 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed Feb 20 22:11:45 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e Feb 20 22:11:45 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f Feb 20 22:11:45 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261 Feb 20 22:11:45 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5 Feb 20 22:11:45 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c Feb 20 22:11:45 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5 Feb 20 22:11:45 localhost kernel: [<ffffffff>] 0xffffffff Feb 20 22:11:45 localhost kernel: irq event stamp: 41702 Feb 20 22:11:45 localhost kernel: hardirqs last enabled at (41702): [<c047a1bd>] kfree+0x9f/0xa6 Feb 20 22:11:45 localhost kernel: hardirqs last disabled at (41701): [<c047a135>] kfree+0x17/0xa6 Feb 20 22:11:45 localhost kernel: softirqs last enabled at (41630): [<c05ceed4>] rt_run_flush+0x64/0x8b Feb 20 22:11:45 localhost kernel: softirqs last disabled at (41631): [<c040709f>] do_softirq+0x5e/0xc7 Feb 20 22:11:45 localhost kernel: Feb 20 22:11:45 localhost kernel: other info that might help us debug this: Feb 20 22:11:45 localhost kernel: 10 locks held by pppd/3744: Feb 20 22:11:45 localhost kernel: #0: (rtnl_mutex){--..}, at: [<c05f53bd>] devinet_ioctl+0xf4/0x539 Feb 20 22:11:45 localhost kernel: #1: ((inetaddr_chain).rwsem){..--}, at: [<c04411c9>] __blocking_notifier_call_chain+0x22/0x51 Feb 20 22:11:45 localhost kernel: #2: (rcu_read_lock){..--}, at: [<c05b7782>] net_rx_action+0x4e/0x1b3 Feb 20 22:11:45 localhost kernel: #3: (rcu_read_lock){..--}, at: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 20 22:11:45 localhost kernel: #4: (rcu_read_lock){..--}, at: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8 Feb 20 22:11:45 localhost kernel: #5: (slock-AF_INET){-+..}, at: [<c05f2d21>] icmp_reply+0x52/0x1a4 Feb 20 22:11:45 localhost kernel: #6: (rcu_read_lock){..--}, at: [<c05b7e60>] dev_queue_xmit+0x125/0x2e9 Feb 20 22:11:45 localhost kernel: #7: (_xmit_PPP){-+..}, at: [<c05c4cb1>] __qdisc_run+0x5b/0x156 Feb 20 22:11:45 localhost kernel: #8: (&ppp->wlock){-+..}, at: [<f8b9ef10>] ppp_xmit_process+0x15/0x5a1 [ppp_generic] Feb 20 22:11:45 localhost kernel: #9: (&pch->downl){-+..}, at: [<f8b9e6db>] ppp_push+0x63/0x50d [ppp_generic] Feb 20 22:11:45 localhost kernel: Feb 20 22:11:45 localhost kernel: stack backtrace: Feb 20 22:11:45 localhost kernel: Pid: 3744, comm: pppd Not tainted 2.6.24.2 #1 Feb 20 22:11:45 localhost kernel: [<c0447933>] print_usage_bug+0x139/0x143 Feb 20 22:11:45 localhost kernel: [<c04482d2>] mark_lock+0x1cb/0x454 Feb 20 22:11:45 localhost kernel: [<c04480a2>] find_usage_forwards+0x67/0x8b Feb 20 22:11:45 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1 Feb 20 22:11:46 localhost kernel: [<c0448b8e>] check_noncircular+0x66/0x93 Feb 20 22:11:46 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53 Feb 20 22:11:46 localhost kernel: [<c047a1bd>] kfree+0x9f/0xa6 Feb 20 22:11:46 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 20 22:11:46 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a Feb 20 22:11:46 localhost kernel: [<f8bac857>] pppol2tp_xmit+0x320/0x3d9 [pppol2tp] Feb 20 22:11:46 localhost kernel: [<c061b3a7>] _read_lock+0x29/0x34 Feb 20 22:11:46 localhost kernel: [<f8bac857>] pppol2tp_xmit+0x320/0x3d9 [pppol2tp] Feb 20 22:11:46 localhost kernel: [<f8bac857>] pppol2tp_xmit+0x320/0x3d9 [pppol2tp] Feb 20 22:11:46 localhost kernel: [<f8b9e6db>] ppp_push+0x63/0x50d [ppp_generic] Feb 20 22:11:46 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39 Feb 20 22:11:46 localhost kernel: [<f8b9e6ee>] ppp_push+0x76/0x50d [ppp_generic] Feb 20 22:11:46 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39 Feb 20 22:11:46 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 20 22:11:46 localhost kernel: [<f8b9f3f1>] ppp_xmit_process+0x4f6/0x5a1 [ppp_generic] Feb 20 22:11:46 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c Feb 20 22:11:46 localhost kernel: [<f8b9f5d2>] ppp_start_xmit+0x136/0x167 [ppp_generic] Feb 20 22:11:46 localhost kernel: [<c05b5d29>] dev_hard_start_xmit+0x245/0x29f Feb 20 22:11:46 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34 Feb 20 22:11:46 localhost kernel: [<c05c4cc0>] __qdisc_run+0x6a/0x156 Feb 20 22:11:46 localhost kernel: [<c05b7ee0>] dev_queue_xmit+0x1a5/0x2e9 Feb 20 22:11:46 localhost kernel: [<c05b7e60>] dev_queue_xmit+0x125/0x2e9 Feb 20 22:11:46 localhost kernel: [<c05d62f5>] ip_finish_output+0x1f4/0x21e Feb 20 22:11:46 localhost kernel: [<c05d4f78>] ip_push_pending_frames+0x2dc/0x33b Feb 20 22:11:46 localhost kernel: [<c05f2de5>] icmp_reply+0x116/0x1a4 Feb 20 22:11:46 localhost kernel: [<c05f32b6>] icmp_echo+0x49/0x4d Feb 20 22:11:46 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8 Feb 20 22:11:46 localhost kernel: [<c05f2f67>] icmp_rcv+0xf4/0x116 Feb 20 22:11:46 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8 Feb 20 22:11:46 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8 Feb 20 22:11:46 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338 Feb 20 22:11:46 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 20 22:11:46 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237 Feb 20 22:11:46 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4 Feb 20 22:11:46 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4 Feb 20 22:11:46 localhost kernel: [<c05b7c53>] process_backlog+0x6c/0xb9 Feb 20 22:11:46 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3 Feb 20 22:11:46 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3 Feb 20 22:11:46 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde Feb 20 22:11:46 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7 Feb 20 22:11:47 localhost kernel: [<c05ceed4>] rt_run_flush+0x64/0x8b Feb 20 22:11:47 localhost kernel: [<c05ceed4>] rt_run_flush+0x64/0x8b Feb 20 22:11:47 localhost kernel: [<c04308ee>] local_bh_enable_ip+0xad/0xd5 Feb 20 22:11:47 localhost kernel: [<c05ceed4>] rt_run_flush+0x64/0x8b Feb 20 22:11:47 localhost kernel: [<c05fa7e7>] fib_disable_ip+0x1e/0x26 Feb 20 22:11:47 localhost kernel: [<c05faf99>] fib_inetaddr_event+0x19c/0x1b0 Feb 20 22:11:47 localhost kernel: [<c061d06c>] notifier_call_chain+0x2a/0x47 Feb 20 22:11:47 localhost kernel: [<c04411e5>] __blocking_notifier_call_chain+0x3e/0x51 Feb 20 22:11:47 localhost kernel: [<c044120f>] blocking_notifier_call_chain+0x17/0x1a Feb 20 22:11:47 localhost kernel: [<c05f450b>] __inet_del_ifa+0x12f/0x1f0 Feb 20 22:11:47 localhost kernel: [<c05f45e3>] inet_del_ifa+0x17/0x1a Feb 20 22:11:47 localhost kernel: [<c05f5716>] devinet_ioctl+0x44d/0x539 Feb 20 22:11:47 localhost kernel: [<c05b7188>] dev_ioctl+0x46f/0x5e0 Feb 20 22:11:47 localhost kernel: [<c05ab078>] sock_ioctl+0x1bb/0x1e0 Feb 20 22:11:47 localhost kernel: [<c05aaebd>] sock_ioctl+0x0/0x1e0 Feb 20 22:11:47 localhost kernel: [<c0488827>] do_ioctl+0x1f/0x62 Feb 20 22:11:47 localhost kernel: [<c0458df0>] audit_syscall_entry+0x10d/0x137 Feb 20 22:11:47 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249 Feb 20 22:11:47 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d Feb 20 22:11:47 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb Feb 20 22:11:47 localhost kernel: ======================= ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-20 22:37 ` James Chapman @ 2008-02-21 8:59 ` Jarek Poplawski 2008-02-21 9:53 ` James Chapman 0 siblings, 1 reply; 51+ messages in thread From: Jarek Poplawski @ 2008-02-21 8:59 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, Paul Mackerras, netdev On Wed, Feb 20, 2008 at 10:37:57PM +0000, James Chapman wrote: > Jarek Poplawski wrote: > >>>> (testing patch #1) >> >> But I hope you tested with the fixed (take 2) version of this patch... > > Yes I did. :) > > But I just got another lockdep error (attached). > >> Since it's quite experimental (testing) this patch could be wrong >> as it is, but I hope it should show the proper way to solve this >> problem. Probably you did some of these, but here are a few of my >> suggestions for testing this: >> >> 1) try my patch with your full bh locking changing patch; >> 2) add while loops to these trylocks on failure, with e.g. __delay(1); >> this should work like full locks again, but there should be no (this >> kind of) lockdep reports; > > Hmm, isn't this just bypassing the lockdep checks? Yes! But it's only for debugging: to find if this change in locking is to be blamed for these new lockups. It should effectively work just like without this patch, but without this lockdep warning. So, if after such change lockups still happen, then it would seem you didn't test this enough before. Otherwise the new patch is to blame and needs reworking. > >> 3) I send here another testing patch with this second way to do this: >> on the write side, but it's even more "experimental" and only a >> proof of concept (should be applied on vanilla ppp_generic). > > I'll look over it. I think I need to take a step back and look at what's > happening in more detail though. This is something completely new and changes all the picture: the xmit path wasn't expected (at least by me) to be called in softirq context at all, and there were no traces of this on previous reports. But, since lockdep always stops after the first warning, there could be even more surprises like this in the future. I'll check this report. Regards, Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-21 8:59 ` Jarek Poplawski @ 2008-02-21 9:53 ` James Chapman 2008-02-21 12:08 ` Jarek Poplawski 2008-02-22 14:16 ` [PATCH][NET] sock.c: sk_dst_lock lockdep keys and names per af_family Jarek Poplawski 0 siblings, 2 replies; 51+ messages in thread From: James Chapman @ 2008-02-21 9:53 UTC (permalink / raw) To: Jarek Poplawski; +Cc: David Miller, Paul Mackerras, netdev Jarek Poplawski wrote: > On Wed, Feb 20, 2008 at 10:37:57PM +0000, James Chapman wrote: >> Jarek Poplawski wrote: >> >>>>> (testing patch #1) >>> But I hope you tested with the fixed (take 2) version of this patch... >> Yes I did. :) >> >> But I just got another lockdep error (attached). >> >>> Since it's quite experimental (testing) this patch could be wrong >>> as it is, but I hope it should show the proper way to solve this >>> problem. Probably you did some of these, but here are a few of my >>> suggestions for testing this: >>> >>> 1) try my patch with your full bh locking changing patch; >>> 2) add while loops to these trylocks on failure, with e.g. __delay(1); >>> this should work like full locks again, but there should be no (this >>> kind of) lockdep reports; >> Hmm, isn't this just bypassing the lockdep checks? > > Yes! But it's only for debugging: to find if this change in locking > is to be blamed for these new lockups. It should effectively work just > like without this patch, but without this lockdep warning. So, if > after such change lockups still happen, then it would seem you didn't > test this enough before. Otherwise the new patch is to blame and needs > reworking. The lockups still happen, but I think they are now due to a different problem, as you say. Some background on this issue might be useful to help get feedback from others on the list. This issue was first reported by an ISP who found random lockups if an L2TP tunnel carrying hundreds/thousands of L2TP sessions went down due to a network outage and then recovered itself. On recovery, all of the tunnel's sessions (PPP) are created rapidly. Sometimes the tunnel would recover just fine, but other times not. The ISP put some effort into reproducing the problem and found that repeatedly creating/deleting a tunnel with lots of L2TP sessions would cause the failure after a random time between a few minutes and several hours. The original lockdep trace came from the ISP. I initially couldn't reproduce the problem but I borrowed two equivalent quad-core systems and can now reproduce it. Subsequent lockdep traces have been from my testing. The _bh locking fixes in pppol2tp combined with your ppp_generic change solved that problem. So I then added data traffic into the mix (since this will happen in a real network) and found that lockups still happen. But the lockdep trace in this case is different, as you noted. Does PPPoE stress the PPP setup code as much as this scenario? I guess in theory it could if lots of PPPoE clients connected at the same time, but there is no aggregate tunnel like there is with L2TP to cause all sessions to connect simultaneously. Perhaps PPTP also suffers from these issues? Perhaps not because it tends to be used only in VPN setups where there is only 1 session per tunnel. >>> 3) I send here another testing patch with this second way to do this: >>> on the write side, but it's even more "experimental" and only a >>> proof of concept (should be applied on vanilla ppp_generic). >> I'll look over it. I think I need to take a step back and look at what's >> happening in more detail though. > > This is something completely new and changes all the picture: the xmit > path wasn't expected (at least by me) to be called in softirq context > at all, and there were no traces of this on previous reports. But, > since lockdep always stops after the first warning, there could be > even more surprises like this in the future. I'll check this report. Doesn't the TX softirq do transmits if they've been queued up? -- James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-21 9:53 ` James Chapman @ 2008-02-21 12:08 ` Jarek Poplawski 2008-02-21 17:09 ` Jarek Poplawski 2008-02-22 14:16 ` [PATCH][NET] sock.c: sk_dst_lock lockdep keys and names per af_family Jarek Poplawski 1 sibling, 1 reply; 51+ messages in thread From: Jarek Poplawski @ 2008-02-21 12:08 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, Paul Mackerras, netdev On Thu, Feb 21, 2008 at 09:53:56AM +0000, James Chapman wrote: > Jarek Poplawski wrote: ... > The _bh locking fixes in pppol2tp combined with your ppp_generic change > solved that problem. So I then added data traffic into the mix (since > this will happen in a real network) and found that lockups still happen. > But the lockdep trace in this case is different, as you noted. I'm not sure what do you mean by "solved that problem": lack of lockups or lack of this kind of lockdep reports. This lockdep report shows a real danger in this case, probably very little probable, unless a lot of tries. So if you think just this kind of lockup happend there, this is would be nice. But there could be something less nice too: we are "fighting" with lockdep in one place but the lockup happens somewhere else for some totally different reason... > Does PPPoE stress the PPP setup code as much as this scenario? I guess > in theory it could if lots of PPPoE clients connected at the same time, > but there is no aggregate tunnel like there is with L2TP to cause all > sessions to connect simultaneously. Perhaps PPTP also suffers from these > issues? Perhaps not because it tends to be used only in VPN setups where > there is only 1 session per tunnel. I don't know this code enough, but it seems it should be easier to maintain or debug if there are similar solutions where possible. >>>> 3) I send here another testing patch with this second way to do this: >>>> on the write side, but it's even more "experimental" and only a >>>> proof of concept (should be applied on vanilla ppp_generic). >>> I'll look over it. I think I need to take a step back and look at >>> what's happening in more detail though. >> >> This is something completely new and changes all the picture: the xmit >> path wasn't expected (at least by me) to be called in softirq context >> at all, and there were no traces of this on previous reports. But, >> since lockdep always stops after the first warning, there could be >> even more surprises like this in the future. I'll check this report. > > Doesn't the TX softirq do transmits if they've been queued up? I've probably too much looked at these reports, and should've expected this could happen. Probably queueing could be separated, but since there could be no queue at all and it's done like this, then this current proof of concept seems to be dead end, and we have to go back to fixing sk_dst_lock handling and my patches could be dumped... So if I don't miss something again (and I need more time for this new report) you should try to fix the problem not reported originally by lockdep, but forseen by David(!): we need to avoid any path like: ppp_generic -> pppol2tp -> something, which could take sk_dst_lock while holding any ppp_generic writing lock: they all are softirq "safe" (i.e. endangered). David gave some example, but I'm not sure you did your patch like this (sk_dst_set()). Probably ip_queue_xmit() can't work with this too. Another, probably simpler way would be to move almost all pppol2tp_xmit code to a workqueue: this should let to break most of dependencies with ppp_generic locks, but I don't know how much it would affect other things (e.g. performance). So you should really rethink these things. Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-21 12:08 ` Jarek Poplawski @ 2008-02-21 17:09 ` Jarek Poplawski 2008-02-25 12:19 ` James Chapman 0 siblings, 1 reply; 51+ messages in thread From: Jarek Poplawski @ 2008-02-21 17:09 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, Paul Mackerras, netdev Jarek Poplawski wrote, On 02/21/2008 01:08 PM: ... > Another, probably simpler way would be to move almost all pppol2tp_xmit ... Actually, the simplest off all seems to be now this old idea to maybe make sk_dst_lock globally softirq immune. At least I think it's worth of testing, to check for these other possible lockdep warnings. It should only need to change all write_ and read_lock(&sk->sk_dst_lock) in very few places: include/net/sock.h, include/net/ip6_route.h, and net/ipv6/ipv6_sockglue.c. This could be tested together with you full _bh locking patch (maybe except these other changes in pppol2tp_xmit). Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-21 17:09 ` Jarek Poplawski @ 2008-02-25 12:19 ` James Chapman 2008-02-25 13:05 ` Jarek Poplawski 0 siblings, 1 reply; 51+ messages in thread From: James Chapman @ 2008-02-25 12:19 UTC (permalink / raw) To: Jarek Poplawski; +Cc: David Miller, Paul Mackerras, netdev [-- Attachment #1: Type: text/plain, Size: 960 bytes --] Jarek Poplawski wrote: > Jarek Poplawski wrote, On 02/21/2008 01:08 PM: > ... > >> Another, probably simpler way would be to move almost all pppol2tp_xmit > ... > > Actually, the simplest off all seems to be now this old idea to maybe make > sk_dst_lock globally softirq immune. At least I think it's worth of testing, > to check for these other possible lockdep warnings. It should only need to > change all write_ and read_lock(&sk->sk_dst_lock) in very few places: > include/net/sock.h, include/net/ip6_route.h, and net/ipv6/ipv6_sockglue.c. > This could be tested together with you full _bh locking patch (maybe except > these other changes in pppol2tp_xmit). I did this and all lockdep errors have now gone. Tests ran all weekend. See attached patch. Is this an acceptable solution? If so, I'll prepare and send official patches. -- James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development [-- Attachment #2: sk_dst_lock-softirq-safe.patch --] [-- Type: text/x-diff, Size: 2648 bytes --] Index: linux-2.6.24.2/include/net/ip6_route.h =================================================================== --- linux-2.6.24.2.orig/include/net/ip6_route.h +++ linux-2.6.24.2/include/net/ip6_route.h @@ -150,9 +150,9 @@ static inline void __ip6_dst_store(struc static inline void ip6_dst_store(struct sock *sk, struct dst_entry *dst, struct in6_addr *daddr, struct in6_addr *saddr) { - write_lock(&sk->sk_dst_lock); + write_lock_bh(&sk->sk_dst_lock); __ip6_dst_store(sk, dst, daddr, saddr); - write_unlock(&sk->sk_dst_lock); + write_unlock_bh(&sk->sk_dst_lock); } static inline int ipv6_unicast_destination(struct sk_buff *skb) Index: linux-2.6.24.2/include/net/sock.h =================================================================== --- linux-2.6.24.2.orig/include/net/sock.h +++ linux-2.6.24.2/include/net/sock.h @@ -1058,11 +1058,11 @@ sk_dst_get(struct sock *sk) { struct dst_entry *dst; - read_lock(&sk->sk_dst_lock); + read_lock_bh(&sk->sk_dst_lock); dst = sk->sk_dst_cache; if (dst) dst_hold(dst); - read_unlock(&sk->sk_dst_lock); + read_unlock_bh(&sk->sk_dst_lock); return dst; } @@ -1079,9 +1079,9 @@ __sk_dst_set(struct sock *sk, struct dst static inline void sk_dst_set(struct sock *sk, struct dst_entry *dst) { - write_lock(&sk->sk_dst_lock); + write_lock_bh(&sk->sk_dst_lock); __sk_dst_set(sk, dst); - write_unlock(&sk->sk_dst_lock); + write_unlock_bh(&sk->sk_dst_lock); } static inline void @@ -1097,9 +1097,9 @@ __sk_dst_reset(struct sock *sk) static inline void sk_dst_reset(struct sock *sk) { - write_lock(&sk->sk_dst_lock); + write_lock_bh(&sk->sk_dst_lock); __sk_dst_reset(sk); - write_unlock(&sk->sk_dst_lock); + write_unlock_bh(&sk->sk_dst_lock); } extern struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie); Index: linux-2.6.24.2/net/ipv6/ipv6_sockglue.c =================================================================== --- linux-2.6.24.2.orig/net/ipv6/ipv6_sockglue.c +++ linux-2.6.24.2/net/ipv6/ipv6_sockglue.c @@ -440,9 +440,9 @@ static int do_ipv6_setsockopt(struct soc opt = xchg(&np->opt, opt); sk_dst_reset(sk); } else { - write_lock(&sk->sk_dst_lock); + write_lock_bh(&sk->sk_dst_lock); opt = xchg(&np->opt, opt); - write_unlock(&sk->sk_dst_lock); + write_unlock_bh(&sk->sk_dst_lock); sk_dst_reset(sk); } sticky_done: @@ -504,9 +504,9 @@ update: opt = xchg(&np->opt, opt); sk_dst_reset(sk); } else { - write_lock(&sk->sk_dst_lock); + write_lock_bh(&sk->sk_dst_lock); opt = xchg(&np->opt, opt); - write_unlock(&sk->sk_dst_lock); + write_unlock_bh(&sk->sk_dst_lock); sk_dst_reset(sk); } ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-25 12:19 ` James Chapman @ 2008-02-25 13:05 ` Jarek Poplawski 2008-02-25 13:39 ` Jarek Poplawski 0 siblings, 1 reply; 51+ messages in thread From: Jarek Poplawski @ 2008-02-25 13:05 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, Paul Mackerras, netdev On Mon, Feb 25, 2008 at 12:19:50PM +0000, James Chapman wrote: > Jarek Poplawski wrote: >> Jarek Poplawski wrote, On 02/21/2008 01:08 PM: >> ... >> >>> Another, probably simpler way would be to move almost all pppol2tp_xmit >> ... >> >> Actually, the simplest off all seems to be now this old idea to maybe make >> sk_dst_lock globally softirq immune. At least I think it's worth of testing, >> to check for these other possible lockdep warnings. It should only need to >> change all write_ and read_lock(&sk->sk_dst_lock) in very few places: >> include/net/sock.h, include/net/ip6_route.h, and net/ipv6/ipv6_sockglue.c. >> This could be tested together with you full _bh locking patch (maybe except >> these other changes in pppol2tp_xmit). > > I did this and all lockdep errors have now gone. Tests ran all weekend. > See attached patch. > > Is this an acceptable solution? If so, I'll prepare and send official > patches. IMHO this should be acceptable because I can't see any reason for changing properly working code if there is so simple and not costly solution. But maybe David or somebody else finds some cons? Since this patch isn't very big I think you could try to send this officially and we will simply see... Regards, Jarek P. > > > -- > James Chapman > Katalix Systems Ltd > http://www.katalix.com > Catalysts for your Embedded Linux software development > > Index: linux-2.6.24.2/include/net/ip6_route.h > =================================================================== > --- linux-2.6.24.2.orig/include/net/ip6_route.h > +++ linux-2.6.24.2/include/net/ip6_route.h > @@ -150,9 +150,9 @@ static inline void __ip6_dst_store(struc > static inline void ip6_dst_store(struct sock *sk, struct dst_entry *dst, > struct in6_addr *daddr, struct in6_addr *saddr) > { > - write_lock(&sk->sk_dst_lock); > + write_lock_bh(&sk->sk_dst_lock); > __ip6_dst_store(sk, dst, daddr, saddr); > - write_unlock(&sk->sk_dst_lock); > + write_unlock_bh(&sk->sk_dst_lock); > } > > static inline int ipv6_unicast_destination(struct sk_buff *skb) > Index: linux-2.6.24.2/include/net/sock.h > =================================================================== > --- linux-2.6.24.2.orig/include/net/sock.h > +++ linux-2.6.24.2/include/net/sock.h > @@ -1058,11 +1058,11 @@ sk_dst_get(struct sock *sk) > { > struct dst_entry *dst; > > - read_lock(&sk->sk_dst_lock); > + read_lock_bh(&sk->sk_dst_lock); > dst = sk->sk_dst_cache; > if (dst) > dst_hold(dst); > - read_unlock(&sk->sk_dst_lock); > + read_unlock_bh(&sk->sk_dst_lock); > return dst; > } > > @@ -1079,9 +1079,9 @@ __sk_dst_set(struct sock *sk, struct dst > static inline void > sk_dst_set(struct sock *sk, struct dst_entry *dst) > { > - write_lock(&sk->sk_dst_lock); > + write_lock_bh(&sk->sk_dst_lock); > __sk_dst_set(sk, dst); > - write_unlock(&sk->sk_dst_lock); > + write_unlock_bh(&sk->sk_dst_lock); > } > > static inline void > @@ -1097,9 +1097,9 @@ __sk_dst_reset(struct sock *sk) > static inline void > sk_dst_reset(struct sock *sk) > { > - write_lock(&sk->sk_dst_lock); > + write_lock_bh(&sk->sk_dst_lock); > __sk_dst_reset(sk); > - write_unlock(&sk->sk_dst_lock); > + write_unlock_bh(&sk->sk_dst_lock); > } > > extern struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie); > Index: linux-2.6.24.2/net/ipv6/ipv6_sockglue.c > =================================================================== > --- linux-2.6.24.2.orig/net/ipv6/ipv6_sockglue.c > +++ linux-2.6.24.2/net/ipv6/ipv6_sockglue.c > @@ -440,9 +440,9 @@ static int do_ipv6_setsockopt(struct soc > opt = xchg(&np->opt, opt); > sk_dst_reset(sk); > } else { > - write_lock(&sk->sk_dst_lock); > + write_lock_bh(&sk->sk_dst_lock); > opt = xchg(&np->opt, opt); > - write_unlock(&sk->sk_dst_lock); > + write_unlock_bh(&sk->sk_dst_lock); > sk_dst_reset(sk); > } > sticky_done: > @@ -504,9 +504,9 @@ update: > opt = xchg(&np->opt, opt); > sk_dst_reset(sk); > } else { > - write_lock(&sk->sk_dst_lock); > + write_lock_bh(&sk->sk_dst_lock); > opt = xchg(&np->opt, opt); > - write_unlock(&sk->sk_dst_lock); > + write_unlock_bh(&sk->sk_dst_lock); > sk_dst_reset(sk); > } > ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-25 13:05 ` Jarek Poplawski @ 2008-02-25 13:39 ` Jarek Poplawski 2008-02-25 14:02 ` Jarek Poplawski 2008-02-25 21:58 ` Jarek Poplawski 0 siblings, 2 replies; 51+ messages in thread From: Jarek Poplawski @ 2008-02-25 13:39 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, Paul Mackerras, netdev On Mon, Feb 25, 2008 at 01:05:08PM +0000, Jarek Poplawski wrote: ... > On Mon, Feb 25, 2008 at 12:19:50PM +0000, James Chapman wrote: > > Is this an acceptable solution? If so, I'll prepare and send official > > patches. > > IMHO this should be acceptable because I can't see any reason for > changing properly working code if there is so simple and not costly > solution. But maybe David or somebody else finds some cons? [...] Hmm... Wait a minute! But on the other hand David has written about his cons here, and it looks reasonable: this place would be fixed, but some others can start reports like this. Maybe, it's better to analyze yet if it's really so hard to eliminate taking this lock on the xmit path? Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-25 13:39 ` Jarek Poplawski @ 2008-02-25 14:02 ` Jarek Poplawski 2008-02-25 21:58 ` Jarek Poplawski 1 sibling, 0 replies; 51+ messages in thread From: Jarek Poplawski @ 2008-02-25 14:02 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, Paul Mackerras, netdev On Mon, Feb 25, 2008 at 01:39:48PM +0000, Jarek Poplawski wrote: ... > Maybe, it's better to > analyze yet if it's really so hard to eliminate taking this lock > on the xmit path? BTW, I'm not sure if it helps, but this matters only for the sockets which could be used (and locked) outside of pppol2tp code (so before pppol2tp code is called). Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-25 13:39 ` Jarek Poplawski 2008-02-25 14:02 ` Jarek Poplawski @ 2008-02-25 21:58 ` Jarek Poplawski 2008-02-26 12:14 ` James Chapman 1 sibling, 1 reply; 51+ messages in thread From: Jarek Poplawski @ 2008-02-25 21:58 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, netdev Jarek Poplawski wrote, On 02/25/2008 02:39 PM: ... > Hmm... Wait a minute! But on the other hand David has written about > his cons here, and it looks reasonable: this place would be fixed, > but some others can start reports like this. Maybe, it's better to > analyze yet if it's really so hard to eliminate taking this lock > on the xmit path? James, I wonder if you could try to test this patch below? ip_queue_xmit() seems to do proper things with __sk_dst_check(), and if some other functions don't behave similarly lockdep should tell. I think, you could test it with your "locks to _bh" patch (without pppol2tp_xmit() part), and maybe with my sock.c lockdep patch (it should help lockdep to see these locks a bit more distinctly). Jarek P. PS: Since ppp_generic isn't endangered for now I removed Paul from CC. --- diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c index e0b072d..b94659a 100644 --- a/drivers/net/pppol2tp.c +++ b/drivers/net/pppol2tp.c @@ -1058,7 +1058,7 @@ static int pppol2tp_xmit(struct ppp_channel *chan, struct sk_buff *skb) /* Get routing info from the tunnel socket */ dst_release(skb->dst); - skb->dst = sk_dst_get(sk_tun); + skb->dst = NULL; skb_orphan(skb); skb->sk = sk_tun; ^ permalink raw reply related [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-25 21:58 ` Jarek Poplawski @ 2008-02-26 12:14 ` James Chapman 2008-02-26 13:03 ` Jarek Poplawski ` (3 more replies) 0 siblings, 4 replies; 51+ messages in thread From: James Chapman @ 2008-02-26 12:14 UTC (permalink / raw) To: Jarek Poplawski; +Cc: David Miller, netdev Jarek Poplawski wrote: > Jarek Poplawski wrote, On 02/25/2008 02:39 PM: > ... >> Hmm... Wait a minute! But on the other hand David has written about >> his cons here, and it looks reasonable: this place would be fixed, >> but some others can start reports like this. Maybe, it's better to >> analyze yet if it's really so hard to eliminate taking this lock >> on the xmit path? > > James, I wonder if you could try to test this patch below? > ip_queue_xmit() seems to do proper things with __sk_dst_check(), and > if some other functions don't behave similarly lockdep should tell. > I think, you could test it with your "locks to _bh" patch (without > pppol2tp_xmit() part), and maybe with my sock.c lockdep patch (it > should help lockdep to see these locks a bit more distinctly). I found the same thing and was running a variant of your patch last night; rather than set skb->dst to NULL though, I use __sk_dst_get() and let ip_queue_xmit() do the route lookup if it returns NULL. But this has the same symptoms as the code I tried a few days ago - no lockdep errors but a system lockup after up to several hours. Nothing is logged in the syslog. Luckily, I'm in the lab where my two borrowed servers are today so I have access to their consoles. Hopefully I'll be able to find out why there are hanging. Btw, they don't hang if I disable irqs around the ppp_input() call. Will update you later. /james > PS: Since ppp_generic isn't endangered for now I removed Paul from CC. > > --- > > diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c > index e0b072d..b94659a 100644 > --- a/drivers/net/pppol2tp.c > +++ b/drivers/net/pppol2tp.c > @@ -1058,7 +1058,7 @@ static int pppol2tp_xmit(struct ppp_channel *chan, struct sk_buff *skb) > > /* Get routing info from the tunnel socket */ > dst_release(skb->dst); > - skb->dst = sk_dst_get(sk_tun); > + skb->dst = NULL; > skb_orphan(skb); > skb->sk = sk_tun; > > -- > To unsubscribe from this list: send the line "unsubscribe netdev" in > the body of a message to majordomo@vger.kernel.org > More majordomo info at http://vger.kernel.org/majordomo-info.html > -- James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-26 12:14 ` James Chapman @ 2008-02-26 13:03 ` Jarek Poplawski 2008-02-26 13:18 ` Jarek Poplawski 2008-02-26 20:00 ` Jarek Poplawski ` (2 subsequent siblings) 3 siblings, 1 reply; 51+ messages in thread From: Jarek Poplawski @ 2008-02-26 13:03 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, netdev On Tue, Feb 26, 2008 at 12:14:26PM +0000, James Chapman wrote: > Jarek Poplawski wrote: >> Jarek Poplawski wrote, On 02/25/2008 02:39 PM: >> ... >>> Hmm... Wait a minute! But on the other hand David has written about >>> his cons here, and it looks reasonable: this place would be fixed, >>> but some others can start reports like this. Maybe, it's better to >>> analyze yet if it's really so hard to eliminate taking this lock >>> on the xmit path? >> >> James, I wonder if you could try to test this patch below? >> ip_queue_xmit() seems to do proper things with __sk_dst_check(), and >> if some other functions don't behave similarly lockdep should tell. >> I think, you could test it with your "locks to _bh" patch (without >> pppol2tp_xmit() part), and maybe with my sock.c lockdep patch (it >> should help lockdep to see these locks a bit more distinctly). > > I found the same thing and was running a variant of your patch last > night; rather than set skb->dst to NULL though, I use __sk_dst_get() and > let ip_queue_xmit() do the route lookup if it returns NULL. But this has > the same symptoms as the code I tried a few days ago - no lockdep errors > but a system lockup after up to several hours. Nothing is logged in the > syslog. I guess you are going to try this together with this sk_dst_lock with bh patch too. If it's possible I'd suggest to try this skb->dst = NULL as well (__sk_dst_get instead of __sk_dst_check seems to be too racy). > Luckily, I'm in the lab where my two borrowed servers are today so I > have access to their consoles. Hopefully I'll be able to find out why > there are hanging. Btw, they don't hang if I disable irqs around the > ppp_input() call. ...and disabling bh instead isn't enough, BTW? > Will update you later. Thanks, Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-26 13:03 ` Jarek Poplawski @ 2008-02-26 13:18 ` Jarek Poplawski 0 siblings, 0 replies; 51+ messages in thread From: Jarek Poplawski @ 2008-02-26 13:18 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, netdev On Tue, Feb 26, 2008 at 01:03:34PM +0000, Jarek Poplawski wrote: > On Tue, Feb 26, 2008 at 12:14:26PM +0000, James Chapman wrote: ... > > there are hanging. Btw, they don't hang if I disable irqs around the > > ppp_input() call. > > ...and disabling bh instead isn't enough, BTW? I guess not: they are mostly disabled by ppp_input() itself... So, it looks like a network card could mess here? Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-26 12:14 ` James Chapman 2008-02-26 13:03 ` Jarek Poplawski @ 2008-02-26 20:00 ` Jarek Poplawski 2008-03-02 20:29 ` James Chapman 2008-02-27 10:54 ` [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue() Jarek Poplawski 2008-02-27 11:48 ` [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall() Jarek Poplawski 3 siblings, 1 reply; 51+ messages in thread From: Jarek Poplawski @ 2008-02-26 20:00 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, netdev James Chapman wrote, On 02/26/2008 01:14 PM: ... > Luckily, I'm in the lab where my two borrowed servers are today so I > have access to their consoles. Hopefully I'll be able to find out why > there are hanging. Btw, they don't hang if I disable irqs around the > ppp_input() call. Maybe you've found the same, or there is some other reason yet, but IMHO this locking break around ppp_input() is wrong. Probably there is needed more advanced solution, but this should fix the problem if it really exists (isn't there possible a race e.g. between receive from socket and from network card?). Jarek P. --- drivers/net/pppol2tp.c | 16 ++++++++-------- 1 files changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c index e0b072d..7c6fcb9 100644 --- a/drivers/net/pppol2tp.c +++ b/drivers/net/pppol2tp.c @@ -363,18 +363,17 @@ out: spin_unlock(&session->reorder_q.lock); } -/* Dequeue a single skb. +/* Requeue a single skb. */ -static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct sk_buff *skb) +static void pppol2tp_recv_requeue_skb(struct pppol2tp_session *session, struct sk_buff *skb) { struct pppol2tp_tunnel *tunnel = session->tunnel; int length = PPPOL2TP_SKB_CB(skb)->length; struct sock *session_sock = NULL; - /* We're about to requeue the skb, so unlink it and return resources + /* We're about to requeue the skb, so return resources * to its current owner (a socket receive buffer). */ - skb_unlink(skb, &session->reorder_q); skb_orphan(skb); tunnel->stats.rx_packets++; @@ -436,14 +435,14 @@ static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct s static void pppol2tp_recv_dequeue(struct pppol2tp_session *session) { struct sk_buff *skb; - struct sk_buff *tmp; /* If the pkt at the head of the queue has the nr that we * expect to send up next, dequeue it and any other * in-sequence packets behind it. */ +again: spin_lock(&session->reorder_q.lock); - skb_queue_walk_safe(&session->reorder_q, skb, tmp) { + skb_queue_walk(&session->reorder_q, skb) { if (time_after(jiffies, PPPOL2TP_SKB_CB(skb)->expires)) { session->stats.rx_seq_discards++; session->stats.rx_errors++; @@ -469,9 +468,10 @@ static void pppol2tp_recv_dequeue(struct pppol2tp_session *session) goto out; } } + __skb_unlink(skb, &session->reorder_q); spin_unlock(&session->reorder_q.lock); - pppol2tp_recv_dequeue_skb(session, skb); - spin_lock(&session->reorder_q.lock); + pppol2tp_recv_requeue_skb(session, skb); + goto again; } out: ^ permalink raw reply related [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-26 20:00 ` Jarek Poplawski @ 2008-03-02 20:29 ` James Chapman 2008-03-03 8:22 ` Jarek Poplawski 0 siblings, 1 reply; 51+ messages in thread From: James Chapman @ 2008-03-02 20:29 UTC (permalink / raw) To: Jarek Poplawski; +Cc: David Miller, netdev Jarek Poplawski wrote: > James Chapman wrote, On 02/26/2008 01:14 PM: > ... >> Luckily, I'm in the lab where my two borrowed servers are today so I >> have access to their consoles. Hopefully I'll be able to find out why >> there are hanging. Btw, they don't hang if I disable irqs around the >> ppp_input() call. > > Maybe you've found the same, or there is some other reason yet, but > IMHO this locking break around ppp_input() is wrong. Probably there > is needed more advanced solution, but this should fix the problem if > it really exists (isn't there possible a race e.g. between receive > from socket and from network card?). I tried your patch but the result is the same. > > Jarek P. > --- > > drivers/net/pppol2tp.c | 16 ++++++++-------- > 1 files changed, 8 insertions(+), 8 deletions(-) > > diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c > index e0b072d..7c6fcb9 100644 > --- a/drivers/net/pppol2tp.c > +++ b/drivers/net/pppol2tp.c > @@ -363,18 +363,17 @@ out: > spin_unlock(&session->reorder_q.lock); > } > > -/* Dequeue a single skb. > +/* Requeue a single skb. > */ > -static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct sk_buff *skb) > +static void pppol2tp_recv_requeue_skb(struct pppol2tp_session *session, struct sk_buff *skb) > { > struct pppol2tp_tunnel *tunnel = session->tunnel; > int length = PPPOL2TP_SKB_CB(skb)->length; > struct sock *session_sock = NULL; > > - /* We're about to requeue the skb, so unlink it and return resources > + /* We're about to requeue the skb, so return resources > * to its current owner (a socket receive buffer). > */ > - skb_unlink(skb, &session->reorder_q); > skb_orphan(skb); > > tunnel->stats.rx_packets++; > @@ -436,14 +435,14 @@ static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct s > static void pppol2tp_recv_dequeue(struct pppol2tp_session *session) > { > struct sk_buff *skb; > - struct sk_buff *tmp; > > /* If the pkt at the head of the queue has the nr that we > * expect to send up next, dequeue it and any other > * in-sequence packets behind it. > */ > +again: > spin_lock(&session->reorder_q.lock); > - skb_queue_walk_safe(&session->reorder_q, skb, tmp) { > + skb_queue_walk(&session->reorder_q, skb) { I think this needs the _safe() call because the list may be modified in the loop body. > if (time_after(jiffies, PPPOL2TP_SKB_CB(skb)->expires)) { > session->stats.rx_seq_discards++; > session->stats.rx_errors++; > @@ -469,9 +468,10 @@ static void pppol2tp_recv_dequeue(struct pppol2tp_session *session) > goto out; > } > } > + __skb_unlink(skb, &session->reorder_q); > spin_unlock(&session->reorder_q.lock); > - pppol2tp_recv_dequeue_skb(session, skb); > - spin_lock(&session->reorder_q.lock); > + pppol2tp_recv_requeue_skb(session, skb); > + goto again; I'd prefer to take the spinlock again after the recv_dequeue_skb() call to avoid the goto. -- James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-03-02 20:29 ` James Chapman @ 2008-03-03 8:22 ` Jarek Poplawski 2008-03-03 9:35 ` Jarek Poplawski 0 siblings, 1 reply; 51+ messages in thread From: Jarek Poplawski @ 2008-03-03 8:22 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, netdev On Sun, Mar 02, 2008 at 08:29:58PM +0000, James Chapman wrote: > Jarek Poplawski wrote: >> James Chapman wrote, On 02/26/2008 01:14 PM: ... > > I tried your patch but the result is the same. > >> >> Jarek P. >> --- >> >> drivers/net/pppol2tp.c | 16 ++++++++-------- >> 1 files changed, 8 insertions(+), 8 deletions(-) >> >> diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c >> index e0b072d..7c6fcb9 100644 >> --- a/drivers/net/pppol2tp.c >> +++ b/drivers/net/pppol2tp.c >> @@ -363,18 +363,17 @@ out: >> spin_unlock(&session->reorder_q.lock); >> } >> -/* Dequeue a single skb. >> +/* Requeue a single skb. >> */ >> -static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct sk_buff *skb) >> +static void pppol2tp_recv_requeue_skb(struct pppol2tp_session *session, struct sk_buff *skb) >> { >> struct pppol2tp_tunnel *tunnel = session->tunnel; >> int length = PPPOL2TP_SKB_CB(skb)->length; >> struct sock *session_sock = NULL; >> - /* We're about to requeue the skb, so unlink it and return resources >> + /* We're about to requeue the skb, so return resources >> * to its current owner (a socket receive buffer). >> */ >> - skb_unlink(skb, &session->reorder_q); >> skb_orphan(skb); >> tunnel->stats.rx_packets++; >> @@ -436,14 +435,14 @@ static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct s >> static void pppol2tp_recv_dequeue(struct pppol2tp_session *session) >> { >> struct sk_buff *skb; >> - struct sk_buff *tmp; >> /* If the pkt at the head of the queue has the nr that we >> * expect to send up next, dequeue it and any other >> * in-sequence packets behind it. >> */ >> +again: >> spin_lock(&session->reorder_q.lock); >> - skb_queue_walk_safe(&session->reorder_q, skb, tmp) { >> + skb_queue_walk(&session->reorder_q, skb) { > > I think this needs the _safe() call because the list may be modified in > the loop body. And you are very right. > >> if (time_after(jiffies, PPPOL2TP_SKB_CB(skb)->expires)) { >> session->stats.rx_seq_discards++; >> session->stats.rx_errors++; >> @@ -469,9 +468,10 @@ static void pppol2tp_recv_dequeue(struct pppol2tp_session *session) >> goto out; >> } >> } >> + __skb_unlink(skb, &session->reorder_q); >> spin_unlock(&session->reorder_q.lock); >> - pppol2tp_recv_dequeue_skb(session, skb); >> - spin_lock(&session->reorder_q.lock); >> + pppol2tp_recv_requeue_skb(session, skb); >> + goto again; > > I'd prefer to take the spinlock again after the recv_dequeue_skb() call > to avoid the goto. IMHO we have to do something like goto here because during the unlock there could be everything changed (including tmp) in this list, but maybe I miss something. Anyway, this place seems to need more locking, e.g.: session->nr++ or some other session accesses in pppol2tp_recv_dequeue_skb() are done without locking. Maybe it would be interesting to try (in a test only) with calling this with reorder_q.lock held (so without unlocking inside this pppol2tp_recv_dequeue's loop at all: at least to see possible lockdep report. I can't see how this irq disabling could help for anything but some recursion during those misterious lockups. BTW, since this bug is so hard to trigger, and it can take some time yet, I think it would be better to merge to David's 2.6.26 tree at least some parts of your "bh" patch and maybe other changes, which don't seem to make it worse at least, to try new things on some common base? Regards, Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-03-03 8:22 ` Jarek Poplawski @ 2008-03-03 9:35 ` Jarek Poplawski 0 siblings, 0 replies; 51+ messages in thread From: Jarek Poplawski @ 2008-03-03 9:35 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, netdev On Mon, Mar 03, 2008 at 08:22:04AM +0000, Jarek Poplawski wrote: ... > Maybe it would be interesting to try (in a test only) > with calling this with reorder_q.lock held (so without unlocking > inside this pppol2tp_recv_dequeue's loop at all: at least to see > possible lockdep report. ...But we can't forget to change this skb_unlink() in pppol2tp_recv_dequeue_skb() then... Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
* [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue() 2008-02-26 12:14 ` James Chapman 2008-02-26 13:03 ` Jarek Poplawski 2008-02-26 20:00 ` Jarek Poplawski @ 2008-02-27 10:54 ` Jarek Poplawski 2008-03-02 20:31 ` James Chapman 2008-02-27 11:48 ` [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall() Jarek Poplawski 3 siblings, 1 reply; 51+ messages in thread From: Jarek Poplawski @ 2008-02-27 10:54 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, netdev Hi, Here is some tiny BTW fix (probably not connected with these lockups). Regards, Jarek P. --------------> Subject: [PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue() Every skb removed from session->reorder_q needs sock_put(). Signed-off-by: Jarek Poplawski <jarkao2@gmail.com> --- drivers/net/pppol2tp.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c index e0b072d..dcd4991 100644 --- a/drivers/net/pppol2tp.c +++ b/drivers/net/pppol2tp.c @@ -455,6 +455,7 @@ static void pppol2tp_recv_dequeue(struct pppol2tp_session *session) skb_queue_len(&session->reorder_q)); __skb_unlink(skb, &session->reorder_q); kfree_skb(skb); + sock_put(session->sock); continue; } ^ permalink raw reply related [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue() 2008-02-27 10:54 ` [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue() Jarek Poplawski @ 2008-03-02 20:31 ` James Chapman 2008-03-04 4:49 ` David Miller 0 siblings, 1 reply; 51+ messages in thread From: James Chapman @ 2008-03-02 20:31 UTC (permalink / raw) To: Jarek Poplawski; +Cc: David Miller, netdev Jarek Poplawski wrote: > Subject: [PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue() > > Every skb removed from session->reorder_q needs sock_put(). > > > Signed-off-by: Jarek Poplawski <jarkao2@gmail.com> Acked-by: James Chapman <jchapman@katalix.com> > > --- > > drivers/net/pppol2tp.c | 1 + > 1 files changed, 1 insertions(+), 0 deletions(-) > > diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c > index e0b072d..dcd4991 100644 > --- a/drivers/net/pppol2tp.c > +++ b/drivers/net/pppol2tp.c > @@ -455,6 +455,7 @@ static void pppol2tp_recv_dequeue(struct pppol2tp_session *session) > skb_queue_len(&session->reorder_q)); > __skb_unlink(skb, &session->reorder_q); > kfree_skb(skb); > + sock_put(session->sock); > continue; > } > -- James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue() 2008-03-02 20:31 ` James Chapman @ 2008-03-04 4:49 ` David Miller 0 siblings, 0 replies; 51+ messages in thread From: David Miller @ 2008-03-04 4:49 UTC (permalink / raw) To: jchapman; +Cc: jarkao2, netdev From: James Chapman <jchapman@katalix.com> Date: Sun, 02 Mar 2008 20:31:17 +0000 > Jarek Poplawski wrote: > > > Subject: [PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue() > > > > Every skb removed from session->reorder_q needs sock_put(). > > > > > > Signed-off-by: Jarek Poplawski <jarkao2@gmail.com> > Acked-by: James Chapman <jchapman@katalix.com> Patch applied. ^ permalink raw reply [flat|nested] 51+ messages in thread
* [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall() 2008-02-26 12:14 ` James Chapman ` (2 preceding siblings ...) 2008-02-27 10:54 ` [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue() Jarek Poplawski @ 2008-02-27 11:48 ` Jarek Poplawski 2008-03-02 20:32 ` James Chapman 3 siblings, 1 reply; 51+ messages in thread From: Jarek Poplawski @ 2008-02-27 11:48 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, netdev And here is one more similar BTW fix. Jarek P. --------------> Subject: [PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall() Every skb removed from session->reorder_q needs sock_put(). Signed-off-by: Jarek Poplawski <jarkao2@gmail.com> --- drivers/net/pppol2tp.c | 7 ++++++- 1 files changed, 6 insertions(+), 1 deletions(-) diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c index e0b072d..ee9e2a7 100644 --- a/drivers/net/pppol2tp.c +++ b/drivers/net/pppol2tp.c @@ -1110,6 +1110,8 @@ static void pppol2tp_tunnel_closeall(struct pppol2tp_tunnel *tunnel) for (hash = 0; hash < PPPOL2TP_HASH_SIZE; hash++) { again: hlist_for_each_safe(walk, tmp, &tunnel->session_hlist[hash]) { + struct sk_buff *skb; + session = hlist_entry(walk, struct pppol2tp_session, hlist); sk = session->sock; @@ -1138,7 +1140,10 @@ again: /* Purge any queued data */ skb_queue_purge(&sk->sk_receive_queue); skb_queue_purge(&sk->sk_write_queue); - skb_queue_purge(&session->reorder_q); + while ((skb = skb_dequeue(&session->reorder_q))) { + kfree_skb(skb); + sock_put(sk); + } release_sock(sk); sock_put(sk); ^ permalink raw reply related [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall() 2008-02-27 11:48 ` [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall() Jarek Poplawski @ 2008-03-02 20:32 ` James Chapman 2008-03-04 4:49 ` David Miller 0 siblings, 1 reply; 51+ messages in thread From: James Chapman @ 2008-03-02 20:32 UTC (permalink / raw) To: Jarek Poplawski; +Cc: David Miller, netdev Jarek Poplawski wrote: > > Subject: [PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall() > > Every skb removed from session->reorder_q needs sock_put(). > > > Signed-off-by: Jarek Poplawski <jarkao2@gmail.com> Acked-by: James Chapman <jchapman@katalix.com> > drivers/net/pppol2tp.c | 7 ++++++- > 1 files changed, 6 insertions(+), 1 deletions(-) > > diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c > index e0b072d..ee9e2a7 100644 > --- a/drivers/net/pppol2tp.c > +++ b/drivers/net/pppol2tp.c > @@ -1110,6 +1110,8 @@ static void pppol2tp_tunnel_closeall(struct pppol2tp_tunnel *tunnel) > for (hash = 0; hash < PPPOL2TP_HASH_SIZE; hash++) { > again: > hlist_for_each_safe(walk, tmp, &tunnel->session_hlist[hash]) { > + struct sk_buff *skb; > + > session = hlist_entry(walk, struct pppol2tp_session, hlist); > > sk = session->sock; > @@ -1138,7 +1140,10 @@ again: > /* Purge any queued data */ > skb_queue_purge(&sk->sk_receive_queue); > skb_queue_purge(&sk->sk_write_queue); > - skb_queue_purge(&session->reorder_q); > + while ((skb = skb_dequeue(&session->reorder_q))) { > + kfree_skb(skb); > + sock_put(sk); > + } > > release_sock(sk); > sock_put(sk); -- James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development ^ permalink raw reply [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall() 2008-03-02 20:32 ` James Chapman @ 2008-03-04 4:49 ` David Miller 0 siblings, 0 replies; 51+ messages in thread From: David Miller @ 2008-03-04 4:49 UTC (permalink / raw) To: jchapman; +Cc: jarkao2, netdev From: James Chapman <jchapman@katalix.com> Date: Sun, 02 Mar 2008 20:32:41 +0000 > Jarek Poplawski wrote: > > > > Subject: [PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall() > > > > Every skb removed from session->reorder_q needs sock_put(). > > > > > > Signed-off-by: Jarek Poplawski <jarkao2@gmail.com> > > Acked-by: James Chapman <jchapman@katalix.com> Also applied, thanks everyone. ^ permalink raw reply [flat|nested] 51+ messages in thread
* [PATCH][NET] sock.c: sk_dst_lock lockdep keys and names per af_family 2008-02-21 9:53 ` James Chapman 2008-02-21 12:08 ` Jarek Poplawski @ 2008-02-22 14:16 ` Jarek Poplawski 1 sibling, 0 replies; 51+ messages in thread From: Jarek Poplawski @ 2008-02-22 14:16 UTC (permalink / raw) To: James Chapman; +Cc: David Miller, Paul Mackerras, netdev On Thu, Feb 21, 2008 at 09:53:56AM +0000, James Chapman wrote: ... > The lockups still happen, but I think they are now due to a different > problem, as you say. ... I hope, this patch should help to remove some possibly false lockdep warnings during this current testing. Regards, Jarek P. -------------------> Subject: [NET] sock.c: sk_dst_lock lockdep keys and names per af_family Initialize sk_dst_lock lockdep keys and names per af_family. Additionally some reorder is done in lockdep related code to keep it in one place and to use static key tables only when needed. Signed-off-by: Jarek Poplawski <jarkao2@gmail.com> [not tested] --- net/core/sock.c | 79 +++++++++++++++++++++++++++++++++++++++---------------- 1 files changed, 56 insertions(+), 23 deletions(-) diff --git a/net/core/sock.c b/net/core/sock.c index 433715f..18c33d2 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -131,14 +131,17 @@ #include <net/tcp.h> #endif +#ifdef CONFIG_DEBUG_LOCK_ALLOC + /* - * Each address family might have different locking rules, so we have - * one slock key per address family: + * Each address family might have different locking rules, so we have one: + * sk_lock, slock, sk_callback_lock and sk_dst_lock key per address family: */ static struct lock_class_key af_family_keys[AF_MAX]; static struct lock_class_key af_family_slock_keys[AF_MAX]; +static struct lock_class_key af_family_callback_keys[AF_MAX]; +static struct lock_class_key af_family_dst_keys[AF_MAX]; -#ifdef CONFIG_DEBUG_LOCK_ALLOC /* * Make lock validator output more readable. (we pre-construct these * strings build-time, so that runtime initialization of socket @@ -186,13 +189,52 @@ static const char *af_family_clock_key_strings[AF_MAX+1] = { "clock-AF_TIPC" , "clock-AF_BLUETOOTH", "clock-AF_IUCV" , "clock-AF_RXRPC" , "clock-AF_MAX" }; -#endif +static const char *af_family_dst_key_strings[AF_MAX+1] = { + "sk_dst-AF_UNSPEC", "sk_dst-AF_UNIX" , "sk_dst-AF_INET" , + "sk_dst-AF_AX25" , "sk_dst-AF_IPX" , "sk_dst-AF_APPLETALK", + "sk_dst-AF_NETROM", "sk_dst-AF_BRIDGE" , "sk_dst-AF_ATMPVC" , + "sk_dst-AF_X25" , "sk_dst-AF_INET6" , "sk_dst-AF_ROSE" , + "sk_dst-AF_DECnet", "sk_dst-AF_NETBEUI" , "sk_dst-AF_SECURITY" , + "sk_dst-AF_KEY" , "sk_dst-AF_NETLINK" , "sk_dst-AF_PACKET" , + "sk_dst-AF_ASH" , "sk_dst-AF_ECONET" , "sk_dst-AF_ATMSVC" , + "sk_dst-21" , "sk_dst-AF_SNA" , "sk_dst-AF_IRDA" , + "sk_dst-AF_PPPOX" , "sk_dst-AF_WANPIPE" , "sk_dst-AF_LLC" , + "sk_dst-27" , "sk_dst-28" , "sk_dst-29" , + "sk_dst-AF_TIPC" , "sk_dst-AF_BLUETOOTH", "sk_dst-AF_IUCV" , + "sk_dst-AF_RXRPC" , "sk_dst-AF_MAX" +}; -/* - * sk_callback_lock locking rules are per-address-family, - * so split the lock classes by using a per-AF key: - */ -static struct lock_class_key af_callback_keys[AF_MAX]; + +static inline void sock_lock_init(struct sock *sk) +{ + sock_lock_init_class_and_name(sk, + af_family_slock_key_strings[sk->sk_family], + af_family_slock_keys + sk->sk_family, + af_family_key_strings[sk->sk_family], + af_family_keys + sk->sk_family); +} + +#define lockdep_set_sk_callback_lock(lock, family) \ + lockdep_set_class_and_name(lock, \ + af_family_callback_keys + (family), \ + af_family_clock_key_strings[(family)]) + +#define lockdep_set_sk_dst_lock(lock, family) \ + lockdep_set_class_and_name(lock, \ + af_family_dst_keys + (family), \ + af_family_dst_key_strings[(family)]) + +#else + +static inline void sock_lock_init(struct sock *sk) +{ + sock_lock_init_class_and_name(sk, 0, 0, 0, 0); +} + +#define lockdep_set_sk_callback_lock(lock, family) do {} while (0) +#define lockdep_set_sk_dst_lock(lock, family) do {} while (0) + +#endif /* CONFIG_DEBUG_LOCK_ALLOC */ /* Take into consideration the size of the struct sk_buff overhead in the * determination of these values, since that is non-constant across @@ -866,14 +908,6 @@ lenout: * * (We also register the sk_lock with the lock validator.) */ -static inline void sock_lock_init(struct sock *sk) -{ - sock_lock_init_class_and_name(sk, - af_family_slock_key_strings[sk->sk_family], - af_family_slock_keys + sk->sk_family, - af_family_key_strings[sk->sk_family], - af_family_keys + sk->sk_family); -} static void sock_copy(struct sock *nsk, const struct sock *osk) { @@ -1014,10 +1048,10 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority) #endif rwlock_init(&newsk->sk_dst_lock); + lockdep_set_sk_dst_lock(&newsk->sk_dst_lock, newsk->sk_family); rwlock_init(&newsk->sk_callback_lock); - lockdep_set_class_and_name(&newsk->sk_callback_lock, - af_callback_keys + newsk->sk_family, - af_family_clock_key_strings[newsk->sk_family]); + lockdep_set_sk_callback_lock(&newsk->sk_callback_lock, + newsk->sk_family); newsk->sk_dst_cache = NULL; newsk->sk_wmem_queued = 0; @@ -1703,10 +1737,9 @@ void sock_init_data(struct socket *sock, struct sock *sk) sk->sk_sleep = NULL; rwlock_init(&sk->sk_dst_lock); + lockdep_set_sk_dst_lock(&sk->sk_dst_lock, sk->sk_family); rwlock_init(&sk->sk_callback_lock); - lockdep_set_class_and_name(&sk->sk_callback_lock, - af_callback_keys + sk->sk_family, - af_family_clock_key_strings[sk->sk_family]); + lockdep_set_sk_callback_lock(&sk->sk_callback_lock, sk->sk_family); sk->sk_state_change = sock_def_wakeup; sk->sk_data_ready = sock_def_readable; ^ permalink raw reply related [flat|nested] 51+ messages in thread
* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver 2008-02-11 23:41 ` James Chapman 2008-02-12 5:30 ` David Miller @ 2008-02-12 7:19 ` Jarek Poplawski 1 sibling, 0 replies; 51+ messages in thread From: Jarek Poplawski @ 2008-02-12 7:19 UTC (permalink / raw) To: James Chapman; +Cc: netdev On Mon, Feb 11, 2008 at 11:41:18PM +0000, James Chapman wrote: > Jarek Poplawski wrote: >> On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote: >> ... >>> Below is example output from lockdep. The oops is reproducible when >>> creating/deleting lots of sessions while passing data. The lock is >>> being acquired for read and write in softirq contexts. >>> >>> Is there a better way to fix this? >>> >>> ================================= >>> [ INFO: inconsistent lock state ] >>> 2.6.24-core2 #1 >>> --------------------------------- >>> inconsistent {in-softirq-R} -> {softirq-on-W} usage. >>> openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes: >>> (&tunnel->hlist_lock){---?}, at: [<f8eea157>] >>> pppol2tp_connect+0x517/0x6d0 [pppol2tp] >>> {in-softirq-R} state was registered at: >> >> IMHO, according to this, disabling bh should be enough. And if it's >> like in this report: only read_lock is taken from softirqs, then this >> should be necessary to change only all write_locks to write_lock_bh >> (of course unless somewhere bhs are disabled already). Unless I miss >> something?! > > I thought so too. I tried _bh locks first and the problem still > occurred. Maybe I'll try it again in case I messed something up. If there are any new lockdep warnings I'd be interested to have a look. (BTW, with irqs disabling such ISP would probably get considerable drop in performance?) Regards, Jarek P. ^ permalink raw reply [flat|nested] 51+ messages in thread
end of thread, other threads:[~2008-03-04 4:49 UTC | newest] Thread overview: 51+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2008-02-11 9:22 [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver James Chapman 2008-02-11 18:57 ` Jarek Poplawski 2008-02-11 22:19 ` James Chapman 2008-02-11 22:49 ` Jarek Poplawski 2008-02-11 22:55 ` Jarek Poplawski 2008-02-11 23:42 ` James Chapman 2008-02-12 10:42 ` Jarek Poplawski 2008-02-11 23:41 ` James Chapman 2008-02-12 5:30 ` David Miller 2008-02-12 10:58 ` James Chapman 2008-02-12 13:24 ` Jarek Poplawski 2008-02-13 6:00 ` David Miller 2008-02-13 7:29 ` Jarek Poplawski 2008-02-14 13:00 ` Jarek Poplawski 2008-02-18 22:09 ` James Chapman 2008-02-18 23:01 ` Jarek Poplawski 2008-02-19 9:09 ` James Chapman 2008-02-19 4:29 ` David Miller 2008-02-19 9:03 ` James Chapman 2008-02-19 10:30 ` Jarek Poplawski 2008-02-19 10:36 ` Jarek Poplawski 2008-02-19 14:37 ` James Chapman 2008-02-19 23:06 ` Jarek Poplawski 2008-02-19 23:28 ` Jarek Poplawski 2008-02-20 16:02 ` James Chapman 2008-02-20 18:38 ` Jarek Poplawski 2008-02-20 22:37 ` James Chapman 2008-02-21 8:59 ` Jarek Poplawski 2008-02-21 9:53 ` James Chapman 2008-02-21 12:08 ` Jarek Poplawski 2008-02-21 17:09 ` Jarek Poplawski 2008-02-25 12:19 ` James Chapman 2008-02-25 13:05 ` Jarek Poplawski 2008-02-25 13:39 ` Jarek Poplawski 2008-02-25 14:02 ` Jarek Poplawski 2008-02-25 21:58 ` Jarek Poplawski 2008-02-26 12:14 ` James Chapman 2008-02-26 13:03 ` Jarek Poplawski 2008-02-26 13:18 ` Jarek Poplawski 2008-02-26 20:00 ` Jarek Poplawski 2008-03-02 20:29 ` James Chapman 2008-03-03 8:22 ` Jarek Poplawski 2008-03-03 9:35 ` Jarek Poplawski 2008-02-27 10:54 ` [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue() Jarek Poplawski 2008-03-02 20:31 ` James Chapman 2008-03-04 4:49 ` David Miller 2008-02-27 11:48 ` [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall() Jarek Poplawski 2008-03-02 20:32 ` James Chapman 2008-03-04 4:49 ` David Miller 2008-02-22 14:16 ` [PATCH][NET] sock.c: sk_dst_lock lockdep keys and names per af_family Jarek Poplawski 2008-02-12 7:19 ` [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver Jarek Poplawski
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for NNTP newsgroup(s).