Netdev List
 help / color / mirror / Atom feed
* Re: [PATCH 3/17 net-2.6.26] [NETFILTER]: Consolidate masq_inet_event and masq_device_event.
From: Patrick McHardy @ 2008-02-19 14:10 UTC (permalink / raw)
  To: Denis V. Lunev; +Cc: davem, netdev, containers, devel
In-Reply-To: <1203406297-32725-3-git-send-email-den@openvz.org>

Denis V. Lunev wrote:
> They do exactly the same job.
> 
> Signed-off-by: Denis V. Lunev <den@openvz.org>
> ---
>  net/ipv4/netfilter/ipt_MASQUERADE.c |   14 ++------------
>  1 files changed, 2 insertions(+), 12 deletions(-)

Looks fine.

^ permalink raw reply

* Re: [ANNOUNCE] ESFQ --> SFQ patches for Linux 2.6.24
From: Patrick McHardy @ 2008-02-19 14:18 UTC (permalink / raw)
  To: David Miller; +Cc: brockn, linux-net, Corey Hickey, Linux Netdev List
In-Reply-To: <20080209.221332.215185649.davem@davemloft.net>

[-- Attachment #1: Type: text/plain, Size: 825 bytes --]

David Miller wrote:
> From: "Brock Noland" <brockn@gmail.com>
> Date: Sat, 9 Feb 2008 20:30:58 -0600
> 
>> Is this going to be merged anytime soon?
> 
> If it gets submitted to the proper mailing list, it might.
> 'linux-net' is for user questions, it is not where the networking
> developers hang out, 'netdev' is.
> 
> And you have to post patches for review, not URL's point to
> the patches.  It has to be int he email, in an applyable form
> so people can review the thing properly.


Since SFQ is not exactly simple and I needed something like this
myself, I followed Paul's suggestion and added a new scheduler
(DRR) for this with more flexible limits.

I'll rediff against net-2.6.26 within the next days and send
a final version for review (anyone interested is welcome to
already review this version of course :).


[-- Attachment #2: x --]
[-- Type: text/plain, Size: 13517 bytes --]

commit 13d0cc64d0f7fed945c357cf4ca43330c8f95ad2
Author: Patrick McHardy <kaber@trash.net>
Date:   Mon Feb 18 22:21:55 2008 +0100

    [NET_SCHED]: Add DRR scheduler
    
    Signed-off-by: Patrick McHardy <kaber@trash.net>

diff --git a/include/linux/pkt_sched.h b/include/linux/pkt_sched.h
index dbb7ac3..2fca9c4 100644
--- a/include/linux/pkt_sched.h
+++ b/include/linux/pkt_sched.h
@@ -482,4 +482,20 @@ struct tc_netem_corrupt
 
 #define NETEM_DIST_SCALE	8192
 
+/* DRR */
+
+enum
+{
+	TCA_DRR_UNSPEC,
+	TCA_DRR_QUANTUM,
+	__TCA_DRR_MAX
+};
+
+#define TCA_DRR_MAX	(__TCA_DRR_MAX - 1)
+
+struct tc_drr_stats
+{
+	s32	deficit;
+};
+
 #endif
diff --git a/net/sched/Kconfig b/net/sched/Kconfig
index 82adfe6..7e1ab99 100644
--- a/net/sched/Kconfig
+++ b/net/sched/Kconfig
@@ -196,6 +196,9 @@ config NET_SCH_NETEM
 
 	  If unsure, say N.
 
+config NET_SCH_DRR
+	tristate "DRR scheduler"
+
 config NET_SCH_INGRESS
 	tristate "Ingress Qdisc"
 	depends on NET_CLS_ACT
diff --git a/net/sched/Makefile b/net/sched/Makefile
index 1d2b0f7..b055f74 100644
--- a/net/sched/Makefile
+++ b/net/sched/Makefile
@@ -28,6 +28,7 @@ obj-$(CONFIG_NET_SCH_TEQL)	+= sch_teql.o
 obj-$(CONFIG_NET_SCH_PRIO)	+= sch_prio.o
 obj-$(CONFIG_NET_SCH_ATM)	+= sch_atm.o
 obj-$(CONFIG_NET_SCH_NETEM)	+= sch_netem.o
+obj-$(CONFIG_NET_SCH_DRR)	+= sch_drr.o
 obj-$(CONFIG_NET_CLS_U32)	+= cls_u32.o
 obj-$(CONFIG_NET_CLS_ROUTE4)	+= cls_route.o
 obj-$(CONFIG_NET_CLS_FW)	+= cls_fw.o
diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c
new file mode 100644
index 0000000..aa241b5
--- /dev/null
+++ b/net/sched/sch_drr.c
@@ -0,0 +1,534 @@
+/*
+ * net/sched/sch_drr.c         Deficit Round Robin scheduler
+ *
+ * Copyright (c) 2008 Patrick McHardy <kaber@trash.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/netdevice.h>
+#include <linux/pkt_sched.h>
+#include <net/sch_generic.h>
+#include <net/pkt_sched.h>
+#include <net/pkt_cls.h>
+
+struct drr_class {
+	struct hlist_node		hlist;
+	u32				classid;
+	unsigned int			refcnt;
+
+	struct gnet_stats_basic		bstats;
+	struct gnet_stats_queue		qstats;
+	struct gnet_stats_rate_est	rate_est;
+	struct list_head		alist;
+	struct Qdisc *			qdisc;
+
+	u32				quantum;
+	s32				deficit;
+};
+
+#define DRR_HSIZE	16
+
+struct drr_sched {
+	struct list_head		active;
+	struct tcf_proto *		filter_list;
+	unsigned int			filter_cnt;
+	struct hlist_head		clhash[DRR_HSIZE];
+	struct sk_buff *		requeue;
+};
+
+static unsigned int drr_hash(u32 h)
+{
+	h ^= h >> 8;
+	h ^= h >> 4;
+
+	return h & (DRR_HSIZE - 1);
+}
+
+static struct drr_class *drr_find_class(struct Qdisc *sch, u32 classid)
+{
+	struct drr_sched *q = qdisc_priv(sch);
+	struct drr_class *cl;
+	struct hlist_node *n;
+
+	hlist_for_each_entry(cl, n, &q->clhash[drr_hash(classid)], hlist) {
+		if (cl->classid == classid)
+			return cl;
+	}
+	return NULL;
+}
+
+static void drr_purge_queue(struct drr_class *cl)
+{
+	unsigned int len = cl->qdisc->q.qlen;
+
+	qdisc_reset(cl->qdisc);
+	qdisc_tree_decrease_qlen(cl->qdisc, len);
+}
+
+static const struct nla_policy drr_policy[TCA_DRR_MAX + 1] = {
+	[TCA_DRR_QUANTUM]	= { .type = NLA_U32 },
+};
+
+static int drr_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
+			    struct nlattr **tca, unsigned long *arg)
+{
+	struct drr_sched *q = qdisc_priv(sch);
+	struct drr_class *cl = (struct drr_class *)*arg;
+	struct nlattr *tb[TCA_DRR_MAX + 1];
+	u32 quantum;
+	int err;
+
+	err = nla_parse_nested(tb, TCA_DRR_MAX, tca[TCA_OPTIONS], drr_policy);
+	if (err < 0)
+		return err;
+
+	if (tb[TCA_DRR_QUANTUM]) {
+		quantum = nla_get_u32(tb[TCA_DRR_QUANTUM]);
+		if (quantum == 0)
+			return -EINVAL;
+	} else
+		quantum = psched_mtu(sch->dev);
+
+	if (cl != NULL) {
+		sch_tree_lock(sch);
+		if (tb[TCA_DRR_QUANTUM])
+			cl->quantum = quantum;
+		sch_tree_unlock(sch);
+
+		if (tca[TCA_RATE])
+			gen_replace_estimator(&cl->bstats, &cl->rate_est,
+					      &sch->dev->queue_lock,
+					      tca[TCA_RATE]);
+		return 0;
+	}
+
+	cl = kzalloc(sizeof(struct drr_class), GFP_KERNEL);
+	if (cl == NULL)
+		return -ENOBUFS;
+
+	cl->refcnt	= 1;
+	cl->classid	= classid;
+	cl->quantum	= quantum;
+	cl->deficit	= quantum;
+	cl->qdisc	= qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops,
+					    classid);
+	if (cl->qdisc == NULL)
+		cl->qdisc = &noop_qdisc;
+
+	if (tca[TCA_RATE])
+		gen_replace_estimator(&cl->bstats, &cl->rate_est,
+				      &sch->dev->queue_lock, tca[TCA_RATE]);
+
+	sch_tree_lock(sch);
+	hlist_add_head(&cl->hlist, &q->clhash[drr_hash(classid)]);
+	sch_tree_unlock(sch);
+
+	*arg = (unsigned long)cl;
+	return 0;
+}
+
+static void drr_destroy_class(struct Qdisc *sch, struct drr_class *cl)
+{
+	gen_kill_estimator(&cl->bstats, &cl->rate_est);
+	qdisc_destroy(cl->qdisc);
+	kfree(cl);
+}
+
+static int drr_delete_class(struct Qdisc *sch, unsigned long arg)
+{
+	struct drr_class *cl = (struct drr_class *)arg;
+
+	sch_tree_lock(sch);
+
+	drr_purge_queue(cl);
+	hlist_del(&cl->hlist);
+
+	if (--cl->refcnt == 0)
+		drr_destroy_class(sch, cl);
+
+	sch_tree_unlock(sch);
+	return 0;
+}
+
+static unsigned long drr_get_class(struct Qdisc *sch, u32 classid)
+{
+	struct drr_class *cl = drr_find_class(sch, classid);
+
+	if (cl != NULL)
+		cl->refcnt++;
+
+	return (unsigned long)cl;
+}
+
+static void drr_put_class(struct Qdisc *sch, unsigned long arg)
+{
+	struct drr_class *cl = (struct drr_class *)arg;
+
+	if (--cl->refcnt == 0)
+		drr_destroy_class(sch, cl);
+}
+
+static struct tcf_proto **drr_tcf_chain(struct Qdisc *sch, unsigned long cl)
+{
+	struct drr_sched *q = qdisc_priv(sch);
+
+	if (cl)
+		return NULL;
+
+	return &q->filter_list;
+}
+
+static unsigned long drr_bind_tcf(struct Qdisc *sch, unsigned long parent,
+				  u32 classid)
+{
+	struct drr_sched *q = qdisc_priv(sch);
+	struct drr_class *cl = drr_find_class(sch, classid);
+
+	if (cl != NULL)
+		q->filter_cnt++;
+
+	return (unsigned long)cl;
+}
+
+static void drr_unbind_tcf(struct Qdisc *sch, unsigned long arg)
+{
+	struct drr_sched *q = qdisc_priv(sch);
+
+	q->filter_cnt--;
+}
+
+static int drr_graft_class(struct Qdisc *sch, unsigned long arg,
+			   struct Qdisc *new, struct Qdisc **old)
+{
+	struct drr_class *cl = (struct drr_class *)arg;
+
+	if (new == NULL) {
+		new = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops,
+					cl->classid);
+		if (new == NULL)
+			new = &noop_qdisc;
+	}
+
+	sch_tree_lock(sch);
+	drr_purge_queue(cl);
+	*old = xchg(&cl->qdisc, new);
+	sch_tree_unlock(sch);
+	return 0;
+}
+
+static struct Qdisc *drr_class_leaf(struct Qdisc *sch, unsigned long arg)
+{
+	struct drr_class *cl = (struct drr_class *)arg;
+
+	return cl->qdisc;
+}
+
+static void drr_qlen_notify(struct Qdisc *csh, unsigned long arg)
+{
+	struct drr_class *cl = (struct drr_class *)arg;
+
+	if (cl->qdisc->q.qlen == 0)
+		list_del(&cl->alist);
+}
+
+static int drr_dump_class(struct Qdisc *sch, unsigned long arg,
+			  struct sk_buff *skb, struct tcmsg *tcm)
+{
+	struct drr_class *cl = (struct drr_class *)arg;
+	struct nlattr *nest;
+
+	tcm->tcm_parent	= TC_H_ROOT;
+	tcm->tcm_handle	= cl->classid;
+	tcm->tcm_info	= cl->qdisc->handle;
+
+	nest = nla_nest_start(skb, TCA_OPTIONS);
+	if (nest == NULL)
+		goto nla_put_failure;
+
+	NLA_PUT_U32(skb, TCA_DRR_QUANTUM, cl->quantum);
+
+	nla_nest_end(skb, nest);
+	return skb->len;
+
+nla_put_failure:
+	nla_nest_cancel(skb, nest);
+	return -1;
+}
+
+static int drr_dump_class_stats(struct Qdisc *sch, unsigned long arg,
+				struct gnet_dump *d)
+{
+	struct drr_class *cl = (struct drr_class *)arg;
+	struct tc_drr_stats xstats = {
+		.deficit	= cl->deficit,
+	};
+
+	if (gnet_stats_copy_basic(d, &cl->bstats) < 0 ||
+	    gnet_stats_copy_rate_est(d, &cl->rate_est) < 0 ||
+	    gnet_stats_copy_queue(d, &cl->qdisc->qstats) < 0)
+		return -1;
+
+	return gnet_stats_copy_app(d, &xstats, sizeof(xstats));
+}
+
+static void drr_walk(struct Qdisc *sch, struct qdisc_walker *arg)
+{
+	struct drr_sched *q = qdisc_priv(sch);
+	struct drr_class *cl;
+	struct hlist_node *n;
+	unsigned int i;
+
+	if (arg->stop)
+		return;
+
+	for (i = 0; i < DRR_HSIZE; i++) {
+		hlist_for_each_entry(cl, n, &q->clhash[i], hlist) {
+			if (arg->count < arg->skip) {
+				arg->count++;
+				continue;
+			}
+			if (arg->fn(sch, (unsigned long)cl, arg) < 0) {
+				arg->stop = 1;
+				return;
+			}
+			arg->count++;
+		}
+	}
+}
+
+static struct drr_class *drr_classify(struct sk_buff *skb, struct Qdisc *sch,
+				      int *qerr)
+{
+	struct drr_sched *q = qdisc_priv(sch);
+	struct drr_class *cl;
+	struct tcf_result res;
+	int result;
+
+	if (TC_H_MAJ(skb->priority ^ sch->handle) == 0) {
+		cl = drr_find_class(sch, skb->priority);
+		if (cl != NULL)
+			return cl;
+	}
+
+	*qerr = NET_XMIT_BYPASS;
+	result = tc_classify(skb, q->filter_list, &res);
+	if (result >= 0) {
+#ifdef CONFIG_NET_CLS_ACT
+		switch (result) {
+		case TC_ACT_QUEUED:
+		case TC_ACT_STOLEN:
+			*qerr = NET_XMIT_SUCCESS;
+		case TC_ACT_SHOT:
+			return NULL;
+		}
+#endif
+		cl = (struct drr_class *)res.class;
+		if (cl == NULL)
+			cl = drr_find_class(sch, res.classid);
+		return cl;
+	}
+	return NULL;
+}
+
+static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+{
+	struct drr_sched *q = qdisc_priv(sch);
+	struct drr_class *cl;
+	unsigned int len;
+	int err;
+
+	cl = drr_classify(skb, sch, &err);
+	if (cl == NULL) {
+		if (err == NET_XMIT_BYPASS)
+			sch->qstats.drops++;
+		kfree_skb(skb);
+		return err;
+	}
+
+	len = skb->len;
+	err = cl->qdisc->enqueue(skb, cl->qdisc);
+	if (unlikely(err != NET_XMIT_SUCCESS)) {
+		cl->qstats.drops++;
+		sch->qstats.drops++;
+		return err;
+	}
+
+	if (cl->qdisc->q.qlen == 1)
+		list_add_tail(&cl->alist, &q->active);
+
+	cl->bstats.packets++;
+	cl->bstats.bytes += len;
+	sch->bstats.packets++;
+	sch->bstats.bytes++;
+
+	sch->q.qlen++;
+	return err;
+}
+
+static struct sk_buff *drr_dequeue(struct Qdisc *sch)
+{
+	struct drr_sched *q = qdisc_priv(sch);
+	struct drr_class *cl, *next;
+	struct sk_buff *skb;
+
+	if (q->requeue != NULL) {
+		skb = q->requeue;
+		q->requeue = NULL;
+		goto out;
+	}
+
+	list_for_each_entry_safe(cl, next, &q->active, alist) {
+		if (cl->deficit <= 0) {
+			WARN_ON(!cl->qdisc->q.qlen);
+			list_move_tail(&cl->alist, &q->active);
+			cl->deficit += cl->quantum;
+			continue;
+		}
+
+		skb = cl->qdisc->dequeue(cl->qdisc);
+		if (skb == NULL)
+			continue;
+
+		cl->deficit -= skb->len;
+		if (cl->deficit <= 0) {
+			if (cl->qdisc->q.qlen)
+				list_move_tail(&cl->alist, &q->active);
+			cl->deficit += cl->quantum;
+		}
+		if (!cl->qdisc->q.qlen)
+			list_del(&cl->alist);
+out:
+		sch->q.qlen--;
+		return skb;
+	}
+
+	return NULL;
+}
+
+static int drr_requeue(struct sk_buff *skb, struct Qdisc *sch)
+{
+	struct drr_sched *q = qdisc_priv(sch);
+
+	q->requeue = skb;
+	sch->q.qlen++;
+
+	return NET_XMIT_SUCCESS;
+}
+
+static unsigned int drr_drop(struct Qdisc *sch)
+{
+	struct drr_sched *q = qdisc_priv(sch);
+	struct drr_class *cl;
+	unsigned int len;
+
+	list_for_each_entry(cl, &q->active, alist) {
+		if (cl->qdisc->ops->drop) {
+			len = cl->qdisc->ops->drop(cl->qdisc);
+			if (len > 0) {
+				if (cl->qdisc->q.qlen == 0)
+					list_del(&cl->alist);
+				return len;
+			}
+		}
+	}
+	return 0;
+}
+
+static int drr_init_qdisc(struct Qdisc *sch, struct nlattr *opt)
+{
+	struct drr_sched *q = qdisc_priv(sch);
+	unsigned int i;
+
+	for (i = 0; i < DRR_HSIZE; i++)
+		INIT_HLIST_HEAD(&q->clhash[i]);
+	INIT_LIST_HEAD(&q->active);
+	return 0;
+}
+
+static void drr_reset_qdisc(struct Qdisc *sch)
+{
+	struct drr_sched *q = qdisc_priv(sch);
+	struct drr_class *cl;
+	struct hlist_node *n;
+	unsigned int i;
+
+	if (q->requeue) {
+		kfree_skb(q->requeue);
+		q->requeue = NULL;
+	}
+	for (i = 0; i < DRR_HSIZE; i++) {
+		hlist_for_each_entry(cl, n, &q->clhash[i], hlist) {
+			if (cl->qdisc->q.qlen)
+				list_del(&cl->alist);
+			qdisc_reset(cl->qdisc);
+			cl->deficit = cl->quantum;
+		}
+	}
+	sch->q.qlen = 0;
+}
+
+static void drr_destroy_qdisc(struct Qdisc *sch)
+{
+	struct drr_sched *q = qdisc_priv(sch);
+	struct drr_class *cl;
+	struct hlist_node *n, *next;
+	unsigned int i;
+
+	tcf_destroy_chain(q->filter_list);
+
+	for (i = 0; i < DRR_HSIZE; i++) {
+		hlist_for_each_entry_safe(cl, n, next, &q->clhash[i], hlist)
+			drr_destroy_class(sch, cl);
+	}
+}
+
+static const struct Qdisc_class_ops drr_class_ops = {
+	.change		= drr_change_class,
+	.delete		= drr_delete_class,
+	.get		= drr_get_class,
+	.put		= drr_put_class,
+	.tcf_chain	= drr_tcf_chain,
+	.bind_tcf	= drr_bind_tcf,
+	.unbind_tcf	= drr_unbind_tcf,
+	.graft		= drr_graft_class,
+	.leaf		= drr_class_leaf,
+	.qlen_notify	= drr_qlen_notify,
+	.dump		= drr_dump_class,
+	.dump_stats	= drr_dump_class_stats,
+	.walk		= drr_walk,
+};
+
+static struct Qdisc_ops drr_qdisc_ops __read_mostly = {
+	.cl_ops		= &drr_class_ops,
+	.id		= "drr",
+	.priv_size	= sizeof(struct drr_sched),
+	.enqueue	= drr_enqueue,
+	.dequeue	= drr_dequeue,
+	.requeue	= drr_requeue,
+	.drop		= drr_drop,
+	.init		= drr_init_qdisc,
+	.reset		= drr_reset_qdisc,
+	.destroy	= drr_destroy_qdisc,
+	.owner		= THIS_MODULE,
+};
+
+static int __init drr_init(void)
+{
+	return register_qdisc(&drr_qdisc_ops);
+}
+
+static void __exit drr_exit(void)
+{
+	unregister_qdisc(&drr_qdisc_ops);
+}
+
+module_init(drr_init);
+module_exit(drr_exit);
+MODULE_LICENSE("GPL");

^ permalink raw reply related

* [PATCH] [NETLABEL] Minor cleanup: remove unused method definition
From: Rami Rosen @ 2008-02-19 14:25 UTC (permalink / raw)
  To: David Miller, netdev

[-- Attachment #1: Type: text/plain, Size: 218 bytes --]

Hi,

This patch removes definition of netlbl_cfg_cipsov4_del() method in
netlabel/netlabel_kapi.c and in include/net/netlabel.h as it is not used.


Regards,
Rami Rosen


Signed-off-by: Rami Rosen <ramirose@gmail.com>

[-- Attachment #2: patch.txt --]
[-- Type: text/plain, Size: 1608 bytes --]

diff --git a/include/net/netlabel.h b/include/net/netlabel.h
index 0ca67d7..911d8c6 100644
--- a/include/net/netlabel.h
+++ b/include/net/netlabel.h
@@ -350,7 +350,6 @@ int netlbl_cfg_cipsov4_add(struct cipso_v4_doi *doi_def,
 int netlbl_cfg_cipsov4_add_map(struct cipso_v4_doi *doi_def,
 			       const char *domain,
 			       struct netlbl_audit *audit_info);
-int netlbl_cfg_cipsov4_del(u32 doi, struct netlbl_audit *audit_info);
 
 /*
  * LSM security attribute operations
@@ -408,11 +407,6 @@ static inline int netlbl_cfg_cipsov4_add_map(struct cipso_v4_doi *doi_def,
 {
 	return -ENOSYS;
 }
-static inline int netlbl_cfg_cipsov4_del(u32 doi,
-					 struct netlbl_audit *audit_info)
-{
-	return -ENOSYS;
-}
 static inline int netlbl_secattr_catmap_walk(
 	                              struct netlbl_lsm_secattr_catmap *catmap,
 				      u32 offset)
diff --git a/net/netlabel/netlabel_kapi.c b/net/netlabel/netlabel_kapi.c
index 39793a1..02b268d 100644
--- a/net/netlabel/netlabel_kapi.c
+++ b/net/netlabel/netlabel_kapi.c
@@ -203,21 +203,6 @@ cfg_cipsov4_add_map_failure:
 	return ret_val;
 }
 
-/**
- * netlbl_cfg_cipsov4_del - Removean existing CIPSOv4 DOI definition
- * @doi: the CIPSO DOI value
- * @audit_info: NetLabel audit information
- *
- * Description:
- * Removes an existing CIPSOv4 DOI definition from the NetLabel subsystem.
- * Returns zero on success, negative values on failure.
- *
- */
-int netlbl_cfg_cipsov4_del(u32 doi, struct netlbl_audit *audit_info)
-{
-	return cipso_v4_doi_remove(doi, audit_info, netlbl_cipsov4_doi_free);
-}
-
 /*
  * Security Attribute Functions
  */

^ permalink raw reply related

* Re: [NETFILTER]: Introduce nf_inet_address
From: David Woodhouse @ 2008-02-19 14:30 UTC (permalink / raw)
  To: Patrick McHardy; +Cc: netdev, Jan Engelhardt, David S. Miller, varekova
In-Reply-To: <47BAE13D.4060206@trash.net>

On Tue, 2008-02-19 at 15:01 +0100, Patrick McHardy wrote:
> David Woodhouse wrote:
> >> +union nf_inet_addr {
> >> +	u_int32_t	all[4];
> >> +	__be32		ip;
> >> +	__be32		ip6[4];
> >> +};
> >> +
> >>  #ifdef __KERNEL__
> >>  #ifdef CONFIG_NETFILTER
> > 
> > This breaks the busybox build:
> > 
> > CC      ipsvd/tcpudp.o
> > In file included from /usr/include/linux/netfilter_ipv4.h:8,
> >                  from ipsvd/tcpudp.c:33:
> > /usr/include/linux/netfilter.h:40: error: expected specifier-qualifier-list before 'u_int32_t'
> > 
> > What is this 'u_int32_t' nonsense anyway?
> > 
> > If a user-visible header is likely to be included by libc directly from
> > a 'standard' header, it may not require <stdint.h>. Therefore it should
> > use the system-specific types such as '__u32'.
> 
> Right, I queued this patch to fix it.

That does the trick -- but are we using u_int32_t elsewhere in
user-visible headers? Does it work there? How?

> > A later commit adds struct in_addr and struct in6_addr to this union
> > too, which breaks busybox even harder.
> 
> Thats odd, the iptables headers have always used struct in_addr and
> struct in6_addr in struct ipt_ip/struct ip6t_ip6, which are also
> used by userspace. What is "ipsvd/tcpudp.c"? I couldn't find it in
> the Debian busybox source.

It's in busybox 1.9.1. Just including <netinet/in.h> seems to be
sufficient to make it happy again. I wonder if netfilter.h should
include that for itself?

-- 
dwmw2


^ permalink raw reply

* Re: [PATCH][PPPOL2TP]: Fix SMP oops in pppol2tp driver
From: James Chapman @ 2008-02-19 14:37 UTC (permalink / raw)
  To: Jarek Poplawski; +Cc: David Miller, netdev
In-Reply-To: <20080219103047.GA3898@ff.dom.local>

[-- Attachment #1: Type: text/plain, Size: 1402 bytes --]

Jarek Poplawski wrote:
> On Tue, Feb 19, 2008 at 09:03:12AM +0000, James Chapman wrote:
>> David Miller wrote:
>>> From: James Chapman <jchapman@katalix.com>
>>> Date: Mon, 18 Feb 2008 22:09:24 +0000
>>>
>>>> Here's a new version of the patch. The patch avoids disabling irqs
>>>> and fixes the sk_dst_get() usage that DaveM mentioned. But even with
>>>> this patch, lockdep still complains if hundreds of ppp sessions are
>>>> inserted into a tunnel as rapidly as possible (lockdep trace is
>>>> below). I can stop these errors by wrapping the call to ppp_input()
>>>> in pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is
>>>> a better fix?
>>> Firstly, let's fix one thing at a time.  Leave the sk_dst_get()
>>> thing alone until we can prove that it's part of the lockdep
>>> traces.
>> In reproducing the problem, I obtained several lockdep traces that  
>> implicated sk_dst_get().
> 
> As a matter of fact I missed just that kind information on previous
> lockdep report, so if you could send them too this should be still
> helpful.

Here you go.

Just to be clear, the initial lockdep report was with non-bh locks. 
Yesterday's report was with _bh locks and the sk_dst_get() changes. This 
(older) report is with _bh locks but without the sk_dst_get() changes.

-- 
James Chapman
Katalix Systems Ltd
http://www.katalix.com
Catalysts for your Embedded Linux software development


[-- Attachment #2: lockdep-sk_dst.log --]
[-- Type: text/plain, Size: 205001 bytes --]

Feb 18 17:25:42 localhost kernel: ======================================================
Feb 18 17:25:42 localhost kernel: [ INFO: soft-safe -> soft-unsafe lock order detected ]
Feb 18 17:25:42 localhost kernel: 2.6.24.2 #1
Feb 18 17:25:42 localhost kernel: ------------------------------------------------------
Feb 18 17:25:42 localhost kernel: pppd/13863 [HC0[0]:SC0[1]:HE1:SE0] is trying to acquire:
Feb 18 17:25:42 localhost kernel:  (&sk->sk_dst_lock){----}, at: [<f8bbf73e>] pppol2tp_xmit+0x3a5/0x4a2 [pppol2tp]
Feb 18 17:25:42 localhost kernel: 
Feb 18 17:25:42 localhost kernel: and this task is already holding:
Feb 18 17:25:42 localhost kernel:  (&pch->downl){-...}, at: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:25:42 localhost kernel: which would create a new lock dependency:
Feb 18 17:25:42 localhost pppd[13863]: Connection terminated.
Feb 18 17:25:42 localhost kernel:  (&pch->downl){-...} -> (&sk->sk_dst_lock){----}
Feb 18 17:25:42 localhost pppd[13863]: Exit.
Feb 18 17:25:42 localhost kernel: 
Feb 18 17:25:42 localhost kernel: but this new dependency connects a soft-irq-safe lock:
Feb 18 17:25:42 localhost kernel:  (&pch->upl){-.-+}
Feb 18 17:25:42 localhost kernel: ... which became soft-irq-safe at:
Feb 18 17:25:43 localhost kernel:   [<c0448013>] check_usage_backwards+0x19/0x41
Feb 18 17:25:43 localhost kernel:   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:43 localhost kernel:   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:43 localhost kernel:   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:43 localhost kernel:   [<f8bb840d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 17:25:43 localhost kernel:   [<c061b373>] _read_lock_bh+0x2e/0x39
Feb 18 17:25:43 localhost kernel:   [<f8bb840d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 17:25:43 localhost kernel:   [<f8bb840d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 17:25:43 localhost kernel:   [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:25:43 localhost kernel:   [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:25:43 localhost kernel:   [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:25:43 localhost kernel:   [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:25:43 localhost kernel:   [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:25:43 localhost kernel:   [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:25:43 localhost kernel:   [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:25:43 localhost kernel:   [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:25:43 localhost kernel:   [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:25:43 localhost kernel:   [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:25:43 localhost kernel:   [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:25:43 localhost kernel:   [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:25:43 localhost kernel:   [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:25:43 localhost kernel:   [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:25:43 localhost kernel:   [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:25:43 localhost kernel:   [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:25:43 localhost kernel:   [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:25:43 localhost kernel:   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:43 localhost kernel:   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:43 localhost kernel:   [<ffffffff>] 0xffffffff
Feb 18 17:25:43 localhost kernel: 
Feb 18 17:25:43 localhost kernel: to a soft-irq-unsafe lock:
Feb 18 17:25:43 localhost kernel:  (&sk->sk_dst_lock){----}
Feb 18 17:25:43 localhost kernel: ... which became soft-irq-unsafe at:
Feb 18 17:25:43 localhost kernel: ...  [<c0449046>] __lock_acquire+0x48b/0xbf1
Feb 18 17:25:43 localhost kernel:   [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:25:43 localhost kernel:   [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 17:25:43 localhost kernel:   [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 17:25:43 localhost kernel:   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:43 localhost kernel:   [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:25:43 localhost kernel:   [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 17:25:43 localhost kernel:   [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:25:43 localhost kernel:   [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:25:43 localhost kernel:   [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 17:25:44 localhost kernel:   [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 17:25:44 localhost kernel:   [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 17:25:44 localhost kernel:   [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 17:25:44 localhost kernel:   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:44 localhost kernel:   [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:25:44 localhost kernel:   [<ffffffff>] 0xffffffff
Feb 18 17:25:44 localhost kernel: 
Feb 18 17:25:44 localhost kernel: other info that might help us debug this:
Feb 18 17:25:44 localhost kernel: 
Feb 18 17:25:44 localhost kernel: 1 lock held by pppd/13863:
Feb 18 17:25:44 localhost kernel:  #0:  (&pch->downl){-...}, at: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:25:44 localhost kernel: 
Feb 18 17:25:44 localhost kernel: the soft-irq-safe lock's dependencies:
Feb 18 17:25:44 localhost kernel: -> (&pch->upl){-.-+} ops: 13 {
Feb 18 17:25:44 localhost kernel:    initial-use  at:
Feb 18 17:25:44 localhost kernel:                         [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:44 localhost kernel:                         [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:44 localhost kernel:                         [<f8bb6b7c>] ppp_ioctl+0x4df/0xc06 [ppp_generic]
Feb 18 17:25:44 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:44 localhost kernel:                         [<f8bb6ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 17:25:44 localhost kernel:                         [<c061b300>] _write_lock_bh+0x2e/0x39
Feb 18 17:25:44 localhost kernel:                         [<f8bb6ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 17:25:44 localhost kernel:                         [<f8bb6ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 17:25:44 localhost kernel:                         [<c061b008>] __down+0x82/0xb8
Feb 18 17:25:44 localhost kernel:                         [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:44 localhost kernel:                         [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:44 localhost kernel:                         [<c061b008>] __down+0x82/0xb8
Feb 18 17:25:44 localhost kernel:                         [<c0425474>] default_wake_function+0x0/0x8
Feb 18 17:25:44 localhost kernel:                         [<c061ae17>] __down_failed+0x7/0xc
Feb 18 17:25:44 localhost kernel:                         [<c0488854>] do_ioctl+0x4ebalance_domains+0x28/0x99
Feb 18 17:25:44 localhost kernel:                                       [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:44 localhost kernel:                                       [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:44 localhost kernel:                                       [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:44 localhost kernel:                                       [<ffffffff>] 0xffffffff
Feb 18 17:25:44 localhost kernel:         }
Feb 18 17:25:44 localhost kernel:         ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:25:44 localhost kernel:        ... acquired at:
Feb 18 17:25:44 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:44 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:44 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:45 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:45 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:45 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:45 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:45 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:45 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:45 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:45 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:45 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:25:45 localhost kernel: 
Feb 18 17:25:45 localhost kernel:       ... acquired at:
Feb 18 17:25:45 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:45 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:45 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:45 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:45 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:45 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:45 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:45 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:45 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:45 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:45 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:25:45 localhost kernel: 
Feb 18 17:25:45 localhost kernel:       -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:25:45 localhost kernel:          initial-use  at:
Feb 18 17:25:45 localhost kernel:                                     [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:45 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:45 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:45 localhost kernel:                                     [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:45 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:45 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:45 localhost kernel:                                     [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:45 localhost kernel:                                     [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:45 localhost kernel:                                     [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:46 localhost kernel:                                     [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:46 localhost kernel:                                     [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:46 localhost kernel:                                     [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:46 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:46 localhost kernel:                                     [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:46 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:46 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:46 localhost kernel:                                     [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:46 localhost kernel:                                     [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:46 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:46 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:46 localhost kernel:                                     [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:46 localhost kernel:                                     [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:46 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:46 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:46 localhost kernel:                                     [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:46 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:46 localhost kernel:          in-hardirq-W at:
Feb 18 17:25:46 localhost kernel:                                     [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:46 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:46 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:46 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:46 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:46 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:46 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:46 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:46 localhost kernel:                                     [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:46 localhost kernel:                                     [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:46 localhost kernel:                                     [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:46 localhost kernel:                                     [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:46 localhost kernel:                                     [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:46 localhost kernel:                                     [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:46 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:46 localhost kernel:                                     [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:46 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:46 localhost kernel:                                     [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:46 localhost kernel:                                     [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:46 localhost kernel:                                     [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:46 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:46 localhost kernel:          in-softirq-W at:
Feb 18 17:25:46 localhost kernel:                                     [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:25:46 localhost kernel:                                     [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:47 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:47 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:47 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:                                     [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:47 localhost kernel:                                     [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:47 localhost kernel:                                     [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:47 localhost kernel:                                     [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:47 localhost kernel:                                     [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:47 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:47 localhost kernel:        }
Feb 18 17:25:47 localhost kernel:        ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:25:47 localhost kernel:       ... acquired at:
Feb 18 17:25:47 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:47 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:47 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:47 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:47 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:47 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:47 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:47 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:47 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:47 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:47 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:25:47 localhost kernel: 
Feb 18 17:25:47 localhost kernel:      ... acquired at:
Feb 18 17:25:47 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:47 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:47 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:47 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel:    [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:47 localhost kernel:    [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:48 localhost kernel:    [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:48 localhost kernel:    [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:48 localhost kernel:    [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:48 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:25:48 localhost kernel: 
Feb 18 17:25:48 localhost kernel:      -> (&rq->rq_lock_key#3){++..} ops: 1928539 {
Feb 18 17:25:48 localhost kernel:         initial-use  at:
Feb 18 17:25:48 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:48 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:48 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:48 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:48 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:48 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:48 localhost kernel:                                   [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:48 localhost kernel:                                   [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:48 localhost kernel:                                   [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:48 localhost kernel:                                   [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:48 localhost kernel:                                   [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:48 localhost kernel:                                   [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:48 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:48 localhost kernel:                                   [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:48 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:48 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:48 localhost kernel:                                   [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:48 localhost kernel:                                   [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:48 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:48 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:48 localhost kernel:                                   [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:48 localhost kernel:                                   [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:48 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:48 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:48 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:48 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:48 localhost kernel:         in-hardirq-W at:
Feb 18 17:25:48 localhost kernel:                                   [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:48 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:48 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:48 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:48 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:48 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:48 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:48 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:48 localhost kernel:                                   [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:49 localhost kernel:                                   [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:49 localhost kernel:                                   [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:49 localhost kernel:                                   [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:49 localhost kernel:                                   [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:49 localhost kernel:                                   [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:49 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:49 localhost kernel:                                   [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:49 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:49 localhost kernel:                                   [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:49 localhost kernel:                                   [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:49 localhost kernel:                                   [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:49 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:49 localhost kernel:         in-softirq-W at:
Feb 18 17:25:49 localhost kernel:                                   [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:49 localhost kernel:                                   [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:25:49 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:49 localhost kernel:                                   [<c04422f1>] getnstimeofday+0x30/0xbc
Feb 18 17:25:49 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:49 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:49 localhost kernel:                                   [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:49 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:49 localhost kernel:                                   [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:49 localhost kernel:                                   [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:49 localhost kernel:                                   [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:25:49 localhost kernel:                                   [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:25:49 localhost kernel:                                   [<c0430afa>] __do_softirq+0xc9/0xde
Feb 18 17:25:49 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:49 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:49 localhost kernel:       }
Feb 18 17:25:49 localhost kernel:       ... key      at: [<c2a34488>] 0xc2a34488
Feb 18 17:25:49 localhost kernel:       -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:25:49 localhost kernel:          initial-use  at:
Feb 18 17:25:49 localhost kernel:                                     [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:49 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:49 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:49 localhost kernel:                                     [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:49 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:49 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:49 localhost kernel:                                     [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:50 localhost kernel:                                     [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:50 localhost kernel:                                     [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:50 localhost kernel:                                     [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:50 localhost kernel:                                     [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:50 localhost kernel:                                     [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:50 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:50 localhost kernel:                                     [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:50 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:50 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:50 localhost kernel:                                     [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:50 localhost kernel:                                     [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:50 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:50 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:50 localhost kernel:                                     [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:50 localhost kernel:                                     [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:50 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:50 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:50 localhost kernel:                                     [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:50 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:50 localhost kernel:          in-hardirq-W at:
Feb 18 17:25:50 localhost kernel:                                     [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:50 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:50 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:50 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:50 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:50 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:50 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:50 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:50 localhost kernel:                                     [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:50 localhost kernel:                                     [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:50 localhost kernel:                                     [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:50 localhost kernel:                                     [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:50 localhost kernel:                                     [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:50 localhost kernel:                                     [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:50 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:50 localhost kernel:                                     [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:50 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:50 localhost kernel:                                     [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:50 localhost kernel:                                     [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:50 localhost kernel:                                     [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:50 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:51 localhost kernel:          in-softirq-W at:
Feb 18 17:25:51 localhost kernel:                                     [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:25:51 localhost kernel:                                     [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:51 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:51 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:51 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:                                     [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:51 localhost kernel:                                     [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:51 localhost kernel:                                     [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:51 localhost kernel:                                     [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:51 localhost kernel:                                     [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:51 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:51 localhost kernel:        }
Feb 18 17:25:51 localhost kernel:        ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:25:51 localhost kernel:       ... acquired at:
Feb 18 17:25:51 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:51 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:51 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:51 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:51 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:51 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:51 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:51 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:51 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:51 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:51 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:25:51 localhost kernel: 
Feb 18 17:25:51 localhost kernel:      ... acquired at:
Feb 18 17:25:51 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:51 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:51 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:51 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel:    [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:52 localhost kernel:    [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:52 localhost kernel:    [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:52 localhost kernel:    [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:52 localhost kernel:    [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:52 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:25:52 localhost kernel: 
Feb 18 17:25:52 localhost kernel:      -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:25:52 localhost kernel:         initial-use  at:
Feb 18 17:25:52 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:52 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:52 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:52 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:52 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:52 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:52 localhost kernel:                                   [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:52 localhost kernel:                                   [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:52 localhost kernel:                                   [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:52 localhost kernel:                                   [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:52 localhost kernel:                                   [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:52 localhost kernel:                                   [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:52 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:52 localhost kernel:                                   [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:52 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:52 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:52 localhost kernel:                                   [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:52 localhost kernel:                                   [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:52 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:52 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:52 localhost kernel:                                   [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:52 localhost kernel:                                   [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:52 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:52 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:52 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:52 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:52 localhost kernel:         in-hardirq-W at:
Feb 18 17:25:52 localhost kernel:                                   [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:52 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:52 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:52 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:53 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:53 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:53 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:53 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:53 localhost kernel:                                   [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:53 localhost kernel:                                   [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:53 localhost kernel:                                   [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:53 localhost kernel:                                   [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:53 localhost kernel:                                   [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:53 localhost kernel:                                   [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:53 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:53 localhost kernel:                                   [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:53 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:53 localhost kernel:                                   [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:53 localhost kernel:                                   [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:53 localhost kernel:                                   [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:53 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:53 localhost kernel:         in-softirq-W at:
Feb 18 17:25:53 localhost kernel:                                   [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:25:53 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:53 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:53 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:53 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel:                                   [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:53 localhost kernel:                                   [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:53 localhost kernel:                                   [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:53 localhost kernel:                                   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:53 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:53 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:53 localhost kernel:       }
Feb 18 17:25:53 localhost kernel:       ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:25:53 localhost kernel:      ... acquired at:
Feb 18 17:25:53 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:53 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:53 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:53 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:54 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:54 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:54 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:54 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:54 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:54 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:54 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:25:54 localhost kernel: 
Feb 18 17:25:54 localhost kernel:     ... acquired at:
Feb 18 17:25:54 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:54 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:54 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:54 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:54 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:54 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:54 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:54 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:54 localhost kernel:    [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:25:54 localhost kernel:    [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 17:25:54 localhost kernel:    [<c04235fe>] complete+0x36/0x44
Feb 18 17:25:54 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:54 localhost kernel:    [<c043d502>] kthread+0x1c/0x5e
Feb 18 17:25:54 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:54 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:54 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:25:54 localhost kernel: 
Feb 18 17:25:54 localhost kernel:     -> (&rq->rq_lock_key#2){++..} ops: 1235688 {
Feb 18 17:25:54 localhost kernel:        initial-use  at:
Feb 18 17:25:54 localhost kernel:                                 [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:54 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:54 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:54 localhost kernel:                                 [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:54 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:54 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:54 localhost kernel:                                 [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:54 localhost kernel:                                 [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:54 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:54 localhost kernel:                                 [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:54 localhost kernel:                                 [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:54 localhost kernel:                                 [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:55 localhost kernel:                                 [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:55 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:55 localhost kernel:                                 [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:55 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:55 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:55 localhost kernel:                                 [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:55 localhost kernel:                                 [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:55 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:55 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:55 localhost kernel:                                 [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:55 localhost kernel:                                 [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:55 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:55 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:55 localhost kernel:                                 [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:55 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:25:55 localhost kernel:        in-hardirq-W at:
Feb 18 17:25:55 localhost kernel:                                 [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:25:55 localhost kernel:                                 [<c040a9d6>] save_stack_trace+0x20/0x3a
Feb 18 17:25:55 localhost kernel:                                 [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:55 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:55 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:55 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:55 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:55 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:55 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:55 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:55 localhost kernel:                                 [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:55 localhost kernel:                                 [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:55 localhost kernel:                                 [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:55 localhost kernel:                                 [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:55 localhost kernel:                                 [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:55 localhost kernel:                                 [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:55 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:55 localhost kernel:                                 [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:55 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:55 localhost kernel:                                 [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:55 localhost kernel:                                 [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:55 localhost kernel:                                 [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:55 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:25:55 localhost kernel:        in-softirq-W at:
Feb 18 17:25:55 localhost kernel:                                 [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:56 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:56 localhost kernel:                                 [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:56 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:56 localhost kernel:                                 [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:56 localhost kernel:                                 [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:56 localhost kernel:                                 [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:56 localhost kernel:                                 [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:56 localhost kernel:                                 [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:56 localhost kernel:                                 [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:56 localhost kernel:                                 [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:56 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:25:56 localhost kernel:      }
Feb 18 17:25:56 localhost kernel:      ... key      at: [<c2a2b488>] 0xc2a2b488
Feb 18 17:25:56 localhost kernel:      -> (&rq->rq_lock_key#3){++..} ops: 1928539 {
Feb 18 17:25:56 localhost kernel:         initial-use  at:
Feb 18 17:25:56 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:56 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:56 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:56 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:56 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:56 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:56 localhost kernel:                                   [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:56 localhost kernel:                                   [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:56 localhost kernel:                                   [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:56 localhost kernel:                                   [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:56 localhost kernel:                                   [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:56 localhost kernel:                                   [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:56 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:56 localhost kernel:                                   [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:56 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:56 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:56 localhost kernel:                                   [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:56 localhost kernel:                                   [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:56 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:56 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:56 localhost kernel:                                   [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:56 localhost kernel:                                   [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:56 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:56 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:56 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:56 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:56 localhost kernel:         in-hardirq-W at:
Feb 18 17:25:57 localhost kernel:                                   [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:57 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:57 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:57 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:57 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:57 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:57 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:57 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:57 localhost kernel:                                   [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:57 localhost kernel:                                   [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:57 localhost kernel:                                   [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:57 localhost kernel:                                   [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:57 localhost kernel:                                   [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:57 localhost kernel:                                   [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:57 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:57 localhost kernel:                                   [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:57 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:57 localhost kernel:                                   [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:57 localhost kernel:                                   [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:57 localhost kernel:                                   [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:57 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:57 localhost kernel:         in-softirq-W at:
Feb 18 17:25:57 localhost kernel:                                   [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:57 localhost kernel:                                   [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:25:57 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:57 localhost kernel:                                   [<c04422f1>] getnstimeofday+0x30/0xbc
Feb 18 17:25:57 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:57 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:57 localhost kernel:                                   [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:57 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:57 localhost kernel:                                   [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:57 localhost kernel:                                   [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:57 localhost kernel:                                   [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:25:57 localhost kernel:                                   [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:25:57 localhost kernel:                                   [<c0430afa>] __do_softirq+0xc9/0xde
Feb 18 17:25:57 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:57 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:25:57 localhost kernel:       }
Feb 18 17:25:57 localhost kernel:       ... key      at: [<c2a34488>] 0xc2a34488
Feb 18 17:25:57 localhost kernel:       -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:25:58 localhost kernel:          initial-use  at:
Feb 18 17:25:58 localhost kernel:                                     [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:58 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:58 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:58 localhost kernel:                                     [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:58 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:58 localhost kernel:                                     [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:58 localhost kernel:                                     [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:58 localhost kernel:                                     [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:58 localhost kernel:                                     [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:58 localhost kernel:                                     [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:58 localhost kernel:                                     [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:58 localhost kernel:                                     [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:58 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:58 localhost kernel:                                     [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:58 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:58 localhost kernel:                                     [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:58 localhost kernel:                                     [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:58 localhost kernel:                                     [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:58 localhost kernel:                                     [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:58 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:58 localhost kernel:                                     [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:58 localhost kernel:                                     [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:58 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:58 localhost kernel:                                     [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:58 localhost kernel:                                     [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:58 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:58 localhost kernel:          in-hardirq-W at:
Feb 18 17:25:58 localhost kernel:                                     [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:58 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:58 localhost kernel:                                     [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:58 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:58 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:58 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:58 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:58 localhost kernel:                                     [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:58 localhost kernel:                                     [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:58 localhost kernel:                                     [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:58 localhost kernel:                                     [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:58 localhost kernel:                                     [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:58 localhost kernel:                                     [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:59 localhost kernel:                                     [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:59 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:59 localhost kernel:                                     [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:59 localhost kernel:                                     [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:59 localhost kernel:                                     [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:59 localhost kernel:                                     [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:59 localhost kernel:                                     [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:59 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:59 localhost kernel:          in-softirq-W at:
Feb 18 17:25:59 localhost kernel:                                     [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:25:59 localhost kernel:                                     [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:59 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:59 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:59 localhost openl2tpd: Loading plugin ppp_unix.so, version V1.0
Feb 18 17:25:59 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost openl2tpd[14004]: Start, trace_flags=00000000
Feb 18 17:25:59 localhost kernel:                                     [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost openl2tpd[14004]: OpenL2TP V1.0, (c) Copyright 2004,2005,2006,2007,2008 Katalix Systems Ltd.
Feb 18 17:25:59 localhost kernel:                                     [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:59 localhost kernel:                                     [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:59 localhost kernel:                                     [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:59 localhost kernel:                                     [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:59 localhost kernel:                                     [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:59 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:25:59 localhost kernel:        }
Feb 18 17:25:59 localhost kernel:        ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:25:59 localhost kernel:       ... acquired at:
Feb 18 17:25:59 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:59 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:59 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:59 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:59 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:59 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:59 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:59 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:59 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:00 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:00 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:00 localhost kernel: 
Feb 18 17:26:00 localhost kernel:      ... acquired at:
Feb 18 17:26:00 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:00 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:00 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:00 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:00 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:00 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:00 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:00 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:26:00 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:00 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:26:00 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:00 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:00 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:00 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:00 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:00 localhost kernel: 
Feb 18 17:26:00 localhost kernel:      -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:26:00 localhost kernel:         initial-use  at:
Feb 18 17:26:00 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:00 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:00 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:00 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:00 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:00 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:00 localhost kernel:                                   [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:26:00 localhost kernel:                                   [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:26:00 localhost kernel:                                   [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:26:00 localhost kernel:                                   [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:26:00 localhost kernel:                                   [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:26:00 localhost kernel:                                   [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:26:00 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:00 localhost kernel:                                   [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:00 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:00 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:00 localhost kernel:                                   [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:26:00 localhost kernel:                                   [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:26:01 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:01 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:01 localhost kernel:                                   [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:26:01 localhost kernel:                                   [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:26:01 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:01 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:01 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:01 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:01 localhost kernel:         in-hardirq-W at:
Feb 18 17:26:01 localhost kernel:                                   [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:26:01 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:01 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:01 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:01 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:01 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:01 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:01 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:01 localhost kernel:                                   [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:26:01 localhost kernel:                                   [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:26:01 localhost kernel:                                   [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:26:01 localhost kernel:                                   [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:26:01 localhost kernel:                                   [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:26:01 localhost kernel:                                   [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:26:01 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:01 localhost kernel:                                   [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:26:01 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:01 localhost kernel:                                   [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:26:01 localhost kernel:                                   [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:26:01 localhost kernel:                                   [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:26:01 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:01 localhost kernel:         in-softirq-W at:
Feb 18 17:26:01 localhost kernel:                                   [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:01 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:01 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:01 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:01 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:01 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:01 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:01 localhost kernel:                                   [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:26:01 localhost kernel:                                   [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:26:01 localhost kernel:                                   [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:01 localhost kernel:                                   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:02 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:02 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:02 localhost kernel:       }
Feb 18 17:26:02 localhost kernel:       ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:26:02 localhost kernel:      ... acquired at:
Feb 18 17:26:02 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:02 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:02 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:02 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:02 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:02 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:02 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:02 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:26:02 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:02 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:26:02 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:02 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:02 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:02 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:02 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:02 localhost kernel: 
Feb 18 17:26:02 localhost kernel:     ... acquired at:
Feb 18 17:26:02 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:02 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:02 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:02 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:02 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:02 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:02 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:02 localhost kernel:    [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:26:02 localhost kernel:    [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 17:26:02 localhost kernel:    [<c04235fe>] complete+0x36/0x44
Feb 18 17:26:02 localhost kernel:    [<c043aad8>] worker_thread+0x0/0xc4
Feb 18 17:26:02 localhost kernel:    [<c043d502>] kthread+0x1c/0x5e
Feb 18 17:26:02 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:02 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:02 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:02 localhost kernel: 
Feb 18 17:26:02 localhost kernel:     -> (&rq->rq_lock_key#3){++..} ops: 1928539 {
Feb 18 17:26:02 localhost kernel:        initial-use  at:
Feb 18 17:26:02 localhost kernel:                                 [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:03 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:03 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:03 localhost kernel:                                 [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:03 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:03 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:03 localhost kernel:                                 [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:26:03 localhost kernel:                                 [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:26:03 localhost kernel:                                 [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:26:03 localhost kernel:                                 [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:26:03 localhost kernel:                                 [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:26:03 localhost kernel:                                 [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:26:03 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:03 localhost kernel:                                 [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:03 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:03 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:03 localhost kernel:                                 [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:26:03 localhost kernel:                                 [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:26:03 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:03 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:03 localhost kernel:                                 [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:26:03 localhost kernel:                                 [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:26:03 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:03 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:03 localhost kernel:                                 [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:03 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:03 localhost kernel:        in-hardirq-W at:
Feb 18 17:26:03 localhost kernel:                                 [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:26:03 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:03 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:03 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:03 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:03 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:03 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:03 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:03 localhost kernel:                                 [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:26:03 localhost kernel:                                 [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:26:03 localhost kernel:                                 [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:26:03 localhost kernel:                                 [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:26:03 localhost kernel:                                 [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:26:03 localhost kernel:                                 [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:26:03 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:04 localhost kernel:                                 [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:26:04 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:04 localhost kernel:                                 [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:26:04 localhost kernel:                                 [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:26:04 localhost kernel:                                 [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:26:04 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:04 localhost kernel:        in-softirq-W at:
Feb 18 17:26:04 localhost kernel:                                 [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:26:04 localhost kernel:                                 [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:26:04 localhost kernel:                                 [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:04 localhost kernel:                                 [<c04422f1>] getnstimeofday+0x30/0xbc
Feb 18 17:26:04 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:04 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:04 localhost kernel:                                 [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:04 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:04 localhost kernel:                                 [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:04 localhost kernel:                                 [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:04 localhost kernel:                                 [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:26:04 localhost kernel:                                 [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:04 localhost kernel:                                 [<c0430afa>] __do_softirq+0xc9/0xde
Feb 18 17:26:04 localhost kernel:                                 [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:04 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:04 localhost kernel:      }
Feb 18 17:26:04 localhost kernel:      ... key      at: [<c2a34488>] 0xc2a34488
Feb 18 17:26:04 localhost kernel:      -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:26:04 localhost kernel:         initial-use  at:
Feb 18 17:26:04 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:04 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:04 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:04 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:04 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:04 localhost kernel:                                   [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:04 localhost kernel:                                   [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:26:04 localhost kernel:                                   [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:26:04 localhost kernel:                                   [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:26:04 localhost kernel:                                   [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:26:04 localhost kernel:                                   [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:26:04 localhost kernel:                                   [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:26:04 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:04 localhost kernel:                                   [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:04 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:05 localhost kernel:                                   [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:05 localhost kernel:                                   [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:26:05 localhost kernel:                                   [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:26:05 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:05 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:05 localhost kernel:                                   [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:26:05 localhost kernel:                                   [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:26:05 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:05 localhost kernel:                                   [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:05 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:05 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:05 localhost kernel:         in-hardirq-W at:
Feb 18 17:26:05 localhost kernel:                                   [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:26:05 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:05 localhost kernel:                                   [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:05 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:05 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:05 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:05 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:05 localhost kernel:                                   [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:05 localhost kernel:                                   [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:26:05 localhost kernel:                                   [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:26:05 localhost kernel:                                   [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:26:05 localhost kernel:                                   [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:26:05 localhost kernel:                                   [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:26:05 localhost kernel:                                   [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:26:05 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:05 localhost kernel:                                   [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:26:05 localhost kernel:                                   [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:05 localhost kernel:                                   [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:26:05 localhost kernel:                                   [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:26:05 localhost kernel:                                   [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:26:05 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:05 localhost kernel:         in-softirq-W at:
Feb 18 17:26:05 localhost kernel:                                   [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:05 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:05 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:05 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:05 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:05 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:05 localhost kernel:                                   [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:05 localhost kernel:                                   [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:26:06 localhost kernel:                                   [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:26:06 localhost kernel:                                   [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:06 localhost kernel:                                   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:06 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:06 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:06 localhost kernel:       }
Feb 18 17:26:06 localhost kernel:       ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:26:06 localhost kernel:      ... acquired at:
Feb 18 17:26:06 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:06 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:06 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:06 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:06 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:06 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:06 localhost kernel:    [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:06 localhost kernel:    [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:26:06 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:06 localhost kernel:    [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:26:06 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:06 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:06 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:06 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:06 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:06 localhost kernel: 
Feb 18 17:26:06 localhost kernel:     ... acquired at:
Feb 18 17:26:06 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:06 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:06 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:06 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:06 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:06 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:06 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:06 localhost kernel:    [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:26:06 localhost kernel:    [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 17:26:06 localhost kernel:    [<c04235fe>] complete+0x36/0x44
Feb 18 17:26:06 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:06 localhost kernel:    [<c0428996>] migration_thread+0x186/0x1d2
Feb 18 17:26:06 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:06 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:06 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:07 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:07 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:07 localhost kernel: 
Feb 18 17:26:07 localhost kernel:     -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:26:07 localhost kernel:        initial-use  at:
Feb 18 17:26:07 localhost kernel:                                 [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:07 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:07 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:07 localhost kernel:                                 [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:07 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:07 localhost kernel:                                 [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:07 localhost kernel:                                 [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:26:07 localhost kernel:                                 [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:26:07 localhost kernel:                                 [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:26:07 localhost kernel:                                 [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:26:07 localhost kernel:                                 [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:26:07 localhost kernel:                                 [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:26:07 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:07 localhost kernel:                                 [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:07 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:07 localhost kernel:                                 [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:07 localhost kernel:                                 [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:26:07 localhost kernel:                                 [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:26:07 localhost kernel:                                 [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:07 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:07 localhost kernel:                                 [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:26:07 localhost kernel:                                 [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:26:07 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:07 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:07 localhost kernel:                                 [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:07 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:07 localhost kernel:        in-hardirq-W at:
Feb 18 17:26:07 localhost kernel:                                 [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:26:07 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:07 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:07 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:07 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:07 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:07 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:07 localhost kernel:                                 [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:07 localhost kernel:                                 [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:26:08 localhost kernel:                                 [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:26:08 localhost kernel:                                 [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:26:08 localhost kernel:                                 [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:26:08 localhost kernel:                                 [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:26:08 localhost kernel:                                 [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:26:08 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:08 localhost kernel:                                 [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:26:08 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:08 localhost kernel:                                 [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:26:08 localhost kernel:                                 [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:26:08 localhost kernel:                                 [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:26:08 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:08 localhost kernel:        in-softirq-W at:
Feb 18 17:26:08 localhost kernel:                                 [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:08 localhost kernel:                                 [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:08 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:08 localhost kernel:                                 [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:08 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:08 localhost kernel:                                 [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:08 localhost kernel:                                 [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:08 localhost kernel:                                 [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:26:08 localhost kernel:                                 [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:26:08 localhost kernel:                                 [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:08 localhost kernel:                                 [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:08 localhost kernel:                                 [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:08 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:08 localhost kernel:      }
Feb 18 17:26:08 localhost kernel:      ... key      at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:26:08 localhost kernel:     ... acquired at:
Feb 18 17:26:08 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:08 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:08 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:08 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:08 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:08 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:08 localhost kernel:    [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:08 localhost kernel:    [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:26:08 localhost kernel:    [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 17:26:08 localhost kernel:    [<c04235fe>] complete+0x36/0x44
Feb 18 17:26:08 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:09 localhost kernel:    [<c0428996>] migration_thread+0x186/0x1d2
Feb 18 17:26:09 localhost kernel:    [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:09 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:09 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:09 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:09 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:09 localhost kernel: 
Feb 18 17:26:09 localhost kernel:    ... acquired at:
Feb 18 17:26:09 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:09 localhost kernel:    [<c042366f>] __wake_up+0x18/0x42
Feb 18 17:26:09 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:09 localhost kernel:    [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:09 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:09 localhost kernel:    [<c042366f>] __wake_up+0x18/0x42
Feb 18 17:26:09 localhost kernel:    [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:09 localhost kernel:    [<c042366f>] __wake_up+0x18/0x42
Feb 18 17:26:09 localhost kernel:    [<c042366f>] __wake_up+0x18/0x42
Feb 18 17:26:09 localhost kernel:    [<f8bb7a2b>] ppp_receive_nonmp_frame+0x51f/0x70a [ppp_generic]
Feb 18 17:26:09 localhost kernel:    [<f8bb8487>] ppp_input+0xbf/0xef [ppp_generic]
Feb 18 17:26:09 localhost kernel:    [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:26:09 localhost kernel:    [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:26:09 localhost kernel:    [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:09 localhost kernel:    [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:26:09 localhost kernel:    [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:26:09 localhost kernel:    [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:26:09 localhost kernel:    [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:26:09 localhost kernel:    [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:26:09 localhost kernel:    [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:09 localhost kernel:    [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:26:09 localhost kernel:    [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:09 localhost kernel:    [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:09 localhost kernel:    [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:09 localhost kernel:    [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:09 localhost kernel:    [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:09 localhost kernel:    [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:09 localhost kernel:    [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:09 localhost kernel:    [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:09 localhost kernel:    [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:09 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:09 localhost kernel: 
Feb 18 17:26:09 localhost kernel:   ... acquired at:
Feb 18 17:26:09 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:10 localhost kernel:    [<f8bb6bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 17:26:10 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:10 localhost kernel:    [<f8bb6bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 17:26:10 localhost kernel:    [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:10 localhost kernel:    [<f8bb6bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 17:26:10 localhost kernel:    [<f8bb6bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 17:26:10 localhost kernel:    [<c061b008>] __down+0x82/0xb8
Feb 18 17:26:10 localhost kernel:    [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:10 localhost kernel:    [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:10 localhost kernel:    [<c061b008>] __down+0x82/0xb8
Feb 18 17:26:10 localhost kernel:    [<c0425474>] default_wake_function+0x0/0x8
Feb 18 17:26:10 localhost kernel:    [<c061ae17>] __down_failed+0x7/0xc
Feb 18 17:26:10 localhost kernel:    [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 17:26:10 localhost kernel:    [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 17:26:10 localhost kernel:    [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 17:26:10 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:10 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:10 localhost kernel: 
Feb 18 17:26:10 localhost kernel:   -> (&list->lock#8){.+..} ops: 46 {
Feb 18 17:26:10 localhost kernel:      initial-use  at:
Feb 18 17:26:10 localhost kernel:                             [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:10 localhost kernel:                             [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:10 localhost kernel:                             [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:10 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:10 localhost kernel:                             [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:10 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:10 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:10 localhost kernel:                             [<f8bb8433>] ppp_input+0x6b/0xef [ppp_generic]
Feb 18 17:26:10 localhost kernel:                             [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:26:10 localhost kernel:                             [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:26:10 localhost kernel:                             [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:10 localhost kernel:                             [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:26:10 localhost kernel:                             [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:26:10 localhost kernel:                             [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:26:10 localhost kernel:                             [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:26:10 localhost kernel:                             [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:26:10 localhost kernel:                             [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:10 localhost kernel:                             [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:26:10 localhost kernel:                             [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:10 localhost kernel:                             [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:10 localhost kernel:                             [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:11 localhost kernel:                             [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:11 localhost kernel:                             [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:11 localhost kernel:                             [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:11 localhost kernel:                             [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:11 localhost kernel:                             [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:11 localhost kernel:                             [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:11 localhost kernel:                             [<ffffffff>] 0xffffffff
Feb 18 17:26:11 localhost kernel:      in-softirq-W at:
Feb 18 17:26:11 localhost kernel:                             [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:11 localhost kernel:                             [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:11 localhost kernel:                             [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:11 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:11 localhost kernel:                             [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:11 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:11 localhost kernel:                             [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:11 localhost kernel:                             [<f8bb8433>] ppp_input+0x6b/0xef [ppp_generic]
Feb 18 17:26:11 localhost kernel:                             [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:26:11 localhost kernel:                             [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:26:11 localhost kernel:                             [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:11 localhost kernel:                             [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:26:11 localhost kernel:                             [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:26:11 localhost kernel:                             [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:26:11 localhost kernel:                             [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:26:11 localhost kernel:                             [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:26:11 localhost kernel:                             [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:11 localhost kernel:                             [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:26:11 localhost kernel:                             [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:11 localhost kernel:                             [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:11 localhost kernel:                             [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:11 localhost kernel:                             [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:11 localhost kernel:                             [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:11 localhost kernel:                             [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:11 localhost kernel:                             [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:11 localhost kernel:                             [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:11 localhost kernel:                             [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:11 localhost kernel:                             [<ffffffff>] 0xffffffff
Feb 18 17:26:11 localhost kernel:    }
Feb 18 17:26:11 localhost kernel:    ... key      at: [<f8bbcd18>] __key.21159+0x0/0xffffb82a [ppp_generic]
Feb 18 17:26:11 localhost kernel:   ... acquired at:
Feb 18 17:26:11 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:11 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:12 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:12 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:12 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:12 localhost kernel:    [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:12 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:12 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:12 localhost kernel:    [<f8bb6424>] ppp_xmit_process+0x529/0x5a1 [ppp_generic]
Feb 18 17:26:12 localhost kernel:    [<f8bb734b>] ppp_channel_push+0x71/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel:    [<c061b373>] _read_lock_bh+0x2e/0x39
Feb 18 17:26:12 localhost kernel:    [<f8bb735a>] ppp_channel_push+0x80/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel:    [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:12 localhost kernel:    [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:12 localhost kernel:    [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:12 localhost kernel:    [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:12 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:12 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:12 localhost kernel: 
Feb 18 17:26:12 localhost kernel:   -> (&pch->downl){-...} ops: 8 {
Feb 18 17:26:12 localhost kernel:      initial-use  at:
Feb 18 17:26:12 localhost kernel:                             [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:12 localhost kernel:                             [<c046b874>] __do_fault+0x31b/0x35d
Feb 18 17:26:12 localhost kernel:                             [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:12 localhost kernel:                             [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel:                             [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:12 localhost kernel:                             [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel:                             [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel:                             [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:12 localhost kernel:                             [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:12 localhost kernel:                             [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:12 localhost kernel:                             [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:12 localhost kernel:                             [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:12 localhost kernel:                             [<ffffffff>] 0xffffffff
Feb 18 17:26:12 localhost kernel:      hardirq-on-W at:
Feb 18 17:26:12 localhost kernel:                             [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:12 localhost kernel:                             [<c046b874>] __do_fault+0x31b/0x35d
Feb 18 17:26:12 localhost kernel:                             [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:12 localhost kernel:                             [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel:                             [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:13 localhost kernel:                             [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:13 localhost kernel:                             [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:13 localhost kernel:                             [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:13 localhost kernel:                             [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:13 localhost kernel:                             [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:13 localhost kernel:                             [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:13 localhost kernel:                             [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:13 localhost kernel:                             [<ffffffff>] 0xffffffff
Feb 18 17:26:13 localhost kernel:    }
Feb 18 17:26:13 localhost kernel:    ... key      at: [<f8bbcd28>] __key.29740+0x0/0xffffb81a [ppp_generic]
Feb 18 17:26:13 localhost kernel:    -> (&list->lock#8){.+..} ops: 46 {
Feb 18 17:26:13 localhost kernel:       initial-use  at:
Feb 18 17:26:13 localhost kernel:                               [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:13 localhost kernel:                               [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:13 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:13 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:13 localhost kernel:                               [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:13 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:13 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:13 localhost kernel:                               [<f8bb8433>] ppp_input+0x6b/0xef [ppp_generic]
Feb 18 17:26:13 localhost kernel:                               [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:26:13 localhost kernel:                               [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:26:13 localhost kernel:                               [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:13 localhost kernel:                               [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:26:13 localhost kernel:                               [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:26:13 localhost kernel:                               [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:26:13 localhost kernel:                               [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:26:13 localhost kernel:                               [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:26:13 localhost kernel:                               [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:13 localhost kernel:                               [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:26:13 localhost kernel:                               [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:13 localhost kernel:                               [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:13 localhost kernel:                               [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:13 localhost kernel:                               [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:13 localhost kernel:                               [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:13 localhost kernel:                               [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:13 localhost kernel:                               [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:13 localhost kernel:                               [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:13 localhost kernel:                               [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:13 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:13 localhost kernel:       in-softirq-W at:
Feb 18 17:26:14 localhost kernel:                               [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:14 localhost kernel:                               [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:14 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:14 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:14 localhost kernel:                               [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:14 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:14 localhost kernel:                               [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:14 localhost kernel:                               [<f8bb8433>] ppp_input+0x6b/0xef [ppp_generic]
Feb 18 17:26:14 localhost kernel:                               [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:26:14 localhost kernel:                               [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:26:14 localhost kernel:                               [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:14 localhost kernel:                               [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:26:14 localhost kernel:                               [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:26:14 localhost kernel:                               [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:26:14 localhost kernel:                               [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:26:14 localhost kernel:                               [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:26:14 localhost kernel:                               [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:14 localhost kernel:                               [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:26:14 localhost kernel:                               [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:14 localhost kernel:                               [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:14 localhost kernel:                               [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:14 localhost kernel:                               [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:14 localhost kernel:                               [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:14 localhost kernel:                               [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:14 localhost kernel:                               [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:14 localhost kernel:                               [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:14 localhost kernel:                               [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:14 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:14 localhost kernel:     }
Feb 18 17:26:14 localhost kernel:     ... key      at: [<f8bbcd18>] __key.21159+0x0/0xffffb82a [ppp_generic]
Feb 18 17:26:14 localhost kernel:    ... acquired at:
Feb 18 17:26:14 localhost kernel:    [<c0446b6a>] add_lock_to_list+0x66/0x89
Feb 18 17:26:14 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:14 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:14 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:14 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:14 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:14 localhost kernel:    [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:14 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:14 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:14 localhost kernel:    [<f8bb72ff>] ppp_channel_push+0x25/0x90 [ppp_generic]
Feb 18 17:26:14 localhost kernel:    [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:14 localhost kernel:    [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:15 localhost kernel:    [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:15 localhost kernel:    [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:15 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:15 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:15 localhost kernel: 
Feb 18 17:26:15 localhost kernel:    -> (inet_peer_idlock){-+..} ops: 1947 {
Feb 18 17:26:15 localhost kernel:       initial-use  at:
Feb 18 17:26:15 localhost kernel:                               [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:15 localhost kernel:                               [<c05d0e17>] rt_bind_peer+0x1b/0x50
Feb 18 17:26:15 localhost kernel:                               [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:15 localhost kernel:                               [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:15 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:15 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel:                               [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:15 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel:                               [<c05d4ea2>] ip_push_pending_frames+0x206/0x33b
Feb 18 17:26:15 localhost kernel:                               [<c05f368d>] icmp_send+0x310/0x386
Feb 18 17:26:15 localhost kernel:                               [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:15 localhost kernel:                               [<c05bcd7a>] neigh_timer_handler+0x14/0x276
Feb 18 17:26:15 localhost kernel:                               [<c05ce07c>] ipv4_link_failure+0x1c/0x44
Feb 18 17:26:15 localhost kernel:                               [<c05f1375>] arp_error_report+0x1c/0x24
Feb 18 17:26:15 localhost kernel:                               [<c05bcf16>] neigh_timer_handler+0x1b0/0x276
Feb 18 17:26:15 localhost kernel:                               [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:15 localhost kernel:                               [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 17:26:15 localhost kernel:                               [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:15 localhost kernel:                               [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:15 localhost kernel:                               [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:15 localhost kernel:                               [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:15 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:15 localhost kernel:       in-softirq-W at:
Feb 18 17:26:15 localhost kernel:                               [<c0446ab0>] save_trace+0x37/0x8b
Feb 18 17:26:15 localhost kernel:                               [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:15 localhost kernel:                               [<c05d0e17>] rt_bind_peer+0x1b/0x50
Feb 18 17:26:15 localhost kernel:                               [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:15 localhost kernel:                               [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:15 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:15 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel:                               [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:15 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel:                               [<c05d4ea2>] ip_push_pending_frames+0x206/0x33b
Feb 18 17:26:16 localhost kernel:                               [<c05f368d>] icmp_send+0x310/0x386
Feb 18 17:26:16 localhost kernel:                               [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:16 localhost kernel:                               [<c05bcd7a>] neigh_timer_handler+0x14/0x276
Feb 18 17:26:16 localhost kernel:                               [<c05ce07c>] ipv4_link_failure+0x1c/0x44
Feb 18 17:26:16 localhost kernel:                               [<c05f1375>] arp_error_report+0x1c/0x24
Feb 18 17:26:16 localhost kernel:                               [<c05bcf16>] neigh_timer_handler+0x1b0/0x276
Feb 18 17:26:16 localhost kernel:                               [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:16 localhost kernel:                               [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 17:26:16 localhost kernel:                               [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:16 localhost kernel:                               [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:16 localhost kernel:                               [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:16 localhost kernel:                               [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:16 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:16 localhost kernel:       hardirq-on-W at:
Feb 18 17:26:16 localhost kernel:                               [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:16 localhost kernel:                               [<c05d0e17>] rt_bind_peer+0x1b/0x50
Feb 18 17:26:16 localhost kernel:                               [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:16 localhost kernel:                               [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:16 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:16 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:16 localhost kernel:                               [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:16 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:16 localhost kernel:                               [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:16 localhost kernel:                               [<c05d4ea2>] ip_push_pending_frames+0x206/0x33b
Feb 18 17:26:16 localhost kernel:                               [<c05f368d>] icmp_send+0x310/0x386
Feb 18 17:26:16 localhost kernel:                               [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:16 localhost kernel:                               [<c05bcd7a>] neigh_timer_handler+0x14/0x276
Feb 18 17:26:16 localhost kernel:                               [<c05ce07c>] ipv4_link_failure+0x1c/0x44
Feb 18 17:26:16 localhost kernel:                               [<c05f1375>] arp_error_report+0x1c/0x24
Feb 18 17:26:16 localhost kernel:                               [<c05bcf16>] neigh_timer_handler+0x1b0/0x276
Feb 18 17:26:16 localhost kernel:                               [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:16 localhost kernel:                               [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 17:26:16 localhost kernel:                               [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:16 localhost kernel:                               [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:16 localhost kernel:                               [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:16 localhost kernel:                               [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:16 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:16 localhost kernel:     }
Feb 18 17:26:16 localhost kernel:     ... key      at: [<c07380f0>] inet_peer_idlock+0x10/0x1c
Feb 18 17:26:16 localhost kernel:    ... acquired at:
Feb 18 17:26:16 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:16 localhost kernel:    [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:17 localhost kernel:    [<c048d217>] __d_lookup+0x0/0x12a
Feb 18 17:26:17 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:17 localhost kernel:    [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:17 localhost kernel:    [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:17 localhost kernel:    [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:17 localhost kernel:    [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:17 localhost kernel:    [<c043badc>] __kernel_text_address+0x18/0x23
Feb 18 17:26:17 localhost kernel:    [<c05d6ec6>] ip_queue_xmit+0x27b/0x34d
Feb 18 17:26:17 localhost kernel:    [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:17 localhost kernel:    [<c047a1bd>] kfree+0x9f/0xa6
Feb 18 17:26:17 localhost kernel:    [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:17 localhost kernel:    [<c05b0c83>] pskb_expand_head+0xde/0x144
Feb 18 17:26:17 localhost kernel:    [<f8bbf7bf>] pppol2tp_xmit+0x426/0x4a2 [pppol2tp]
Feb 18 17:26:17 localhost kernel:    [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:17 localhost kernel:    [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:17 localhost kernel:    [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:17 localhost kernel:    [<f8bb730e>] ppp_channel_push+0x34/0x90 [ppp_generic]
Feb 18 17:26:17 localhost kernel:    [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:17 localhost kernel:    [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:17 localhost kernel:    [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:17 localhost kernel:    [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:17 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:17 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:17 localhost kernel: 
Feb 18 17:26:17 localhost kernel:    -> (&dev->queue_lock#2){-+..} ops: 47607 {
Feb 18 17:26:17 localhost kernel:       initial-use  at:
Feb 18 17:26:17 localhost kernel:                               [<c047a1bd>] kfree+0x9f/0xa6
Feb 18 17:26:17 localhost kernel:                               [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:17 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:17 localhost kernel:                               [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:17 localhost kernel:                               [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:17 localhost kernel:                               [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:17 localhost kernel:                               [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:17 localhost kernel:                               [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:17 localhost kernel:                               [<c05b680a>] register_netdevice+0x29f/0x2f9
Feb 18 17:26:17 localhost kernel:                               [<c05b6893>] register_netdev+0x2f/0x3b
Feb 18 17:26:17 localhost kernel:                               [<c076aded>] loopback_net_init+0x36/0x68
Feb 18 17:26:17 localhost kernel:                               [<c05b3fa0>] register_pernet_operations+0x10/0x11
Feb 18 17:26:17 localhost kernel:                               [<c05b3ffc>] register_pernet_device+0x1c/0x48
Feb 18 17:26:17 localhost kernel:                               [<c074e491>] kernel_init+0x148/0x2af
Feb 18 17:26:17 localhost kernel:                               [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:18 localhost kernel:                               [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:18 localhost kernel:                               [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:18 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:18 localhost kernel:       in-softirq-W at:
Feb 18 17:26:18 localhost kernel:                               [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:18 localhost kernel:                               [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:18 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:18 localhost kernel:                               [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:18 localhost kernel:                               [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:18 localhost kernel:                               [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:18 localhost kernel:                               [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:18 localhost kernel:                               [<c05b7e60>] dev_queue_xmit+0x125/0x2e9
Feb 18 17:26:18 localhost kernel:                               [<c05c341e>] eth_header+0x0/0xb6
Feb 18 17:26:18 localhost kernel:                               [<f8f3a487>] mld_sendpack+0x242/0x355 [ipv6]
Feb 18 17:26:18 localhost kernel:                               [<f8f3b453>] mld_ifc_timer_expire+0x143/0x208 [ipv6]
Feb 18 17:26:18 localhost kernel:                               [<f8f3b4e5>] mld_ifc_timer_expire+0x1d5/0x208 [ipv6]
Feb 18 17:26:18 localhost kernel:                               [<f8f3b4f0>] mld_ifc_timer_expire+0x1e0/0x208 [ipv6]
Feb 18 17:26:18 localhost kernel:                               [<f8f3b310>] mld_ifc_timer_expire+0x0/0x208 [ipv6]
Feb 18 17:26:18 localhost kernel:                               [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 17:26:18 localhost kernel:                               [<f8f3b310>] mld_ifc_timer_expire+0x0/0x208 [ipv6]
Feb 18 17:26:18 localhost kernel:                               [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:18 localhost kernel:                               [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:18 localhost kernel:                               [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:18 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:18 localhost kernel:       hardirq-on-W at:
Feb 18 17:26:18 localhost kernel:                               [<c047a1bd>] kfree+0x9f/0xa6
Feb 18 17:26:18 localhost kernel:                               [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:18 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:18 localhost kernel:                               [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:18 localhost kernel:                               [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:18 localhost kernel:                               [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:18 localhost kernel:                               [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:18 localhost kernel:                               [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:18 localhost kernel:                               [<c05b680a>] register_netdevice+0x29f/0x2f9
Feb 18 17:26:18 localhost kernel:                               [<c05b6893>] register_netdev+0x2f/0x3b
Feb 18 17:26:18 localhost kernel:                               [<c076aded>] loopback_net_init+0x36/0x68
Feb 18 17:26:18 localhost kernel:                               [<c05b3fa0>] register_pernet_operations+0x10/0x11
Feb 18 17:26:18 localhost kernel:                               [<c05b3ffc>] register_pernet_device+0x1c/0x48
Feb 18 17:26:18 localhost kernel:                               [<c074e491>] kernel_init+0x148/0x2af
Feb 18 17:26:18 localhost kernel:                               [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:18 localhost kernel:                               [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:18 localhost kernel:                               [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:19 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:19 localhost kernel:     }
Feb 18 17:26:19 localhost kernel:     ... key      at: [<c0a3e1c4>] __key.31614+0x0/0x8
Feb 18 17:26:19 localhost kernel:     -> (&dev->ingress_lock){-...} ops: 208 {
Feb 18 17:26:19 localhost kernel:        initial-use  at:
Feb 18 17:26:19 localhost kernel:                                 [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:19 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:19 localhost kernel:                                 [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:19 localhost kernel:                                 [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel:                                 [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel:                                 [<c05b680a>] register_netdevice+0x29f/0x2f9
Feb 18 17:26:19 localhost kernel:                                 [<c05b6893>] register_netdev+0x2f/0x3b
Feb 18 17:26:19 localhost kernel:                                 [<c076aded>] loopback_net_init+0x36/0x68
Feb 18 17:26:19 localhost kernel:                                 [<c05b3fa0>] register_pernet_operations+0x10/0x11
Feb 18 17:26:19 localhost kernel:                                 [<c05b3ffc>] register_pernet_device+0x1c/0x48
Feb 18 17:26:19 localhost kernel:                                 [<c074e491>] kernel_init+0x148/0x2af
Feb 18 17:26:19 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:19 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:19 localhost kernel:                                 [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:19 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:19 localhost kernel:        hardirq-on-W at:
Feb 18 17:26:19 localhost kernel:                                 [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:19 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:19 localhost kernel:                                 [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:19 localhost kernel:                                 [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel:                                 [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel:                                 [<c05b680a>] register_netdevice+0x29f/0x2f9
Feb 18 17:26:19 localhost kernel:                                 [<c05b6893>] register_netdev+0x2f/0x3b
Feb 18 17:26:19 localhost kernel:                                 [<c076aded>] loopback_net_init+0x36/0x68
Feb 18 17:26:19 localhost kernel:                                 [<c05b3fa0>] register_pernet_operations+0x10/0x11
Feb 18 17:26:19 localhost kernel:                                 [<c05b3ffc>] register_pernet_device+0x1c/0x48
Feb 18 17:26:19 localhost kernel:                                 [<c074e491>] kernel_init+0x148/0x2af
Feb 18 17:26:19 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:19 localhost kernel:                                 [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:19 localhost kernel:                                 [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:19 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:19 localhost kernel:      }
Feb 18 17:26:19 localhost kernel:      ... key      at: [<c0a3e1b4>] __key.31616+0x0/0x8
Feb 18 17:26:19 localhost kernel:      -> (&list->lock#6){....} ops: 675 {
Feb 18 17:26:19 localhost kernel:         initial-use  at:
Feb 18 17:26:19 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:19 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:20 localhost kernel:                                   [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel:                                   [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:20 localhost kernel:                                   [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel:                                   [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel:                                   [<c05b0afd>] skb_queue_purge+0x11/0x17
Feb 18 17:26:20 localhost kernel:                                   [<c05c4eca>] pfifo_fast_reset+0x14/0x2f
Feb 18 17:26:20 localhost kernel:                                   [<c05c4712>] qdisc_reset+0x10/0x11
Feb 18 17:26:20 localhost kernel:                                   [<c05c4b83>] dev_deactivate+0x2a/0xfd
Feb 18 17:26:20 localhost kernel:                                   [<c05bf1f4>] __linkwatch_run_queue+0x143/0x17a
Feb 18 17:26:20 localhost kernel:                                   [<c05bf248>] linkwatch_event+0x1d/0x22
Feb 18 17:26:20 localhost kernel:                                   [<c043a0f3>] run_workqueue+0xdb/0x1ae
Feb 18 17:26:20 localhost kernel:                                   [<c043a09f>] run_workqueue+0x87/0x1ae
Feb 18 17:26:20 localhost kernel:                                   [<c05bf22b>] linkwatch_event+0x0/0x22
Feb 18 17:26:20 localhost kernel:                                   [<c043aad8>] worker_thread+0x0/0xc4
Feb 18 17:26:20 localhost kernel:                                   [<c043ab92>] worker_thread+0xba/0xc4
Feb 18 17:26:20 localhost kernel:                                   [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:20 localhost kernel:                                   [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:20 localhost kernel:                                   [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:20 localhost kernel:                                   [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:20 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:20 localhost kernel:       }
Feb 18 17:26:20 localhost kernel:       ... key      at: [<c0a3e7d4>] __key.21167+0x0/0x8
Feb 18 17:26:20 localhost kernel:      ... acquired at:
Feb 18 17:26:20 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:20 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:20 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:20 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel:    [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:20 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel:    [<c05b0afd>] skb_queue_purge+0x11/0x17
Feb 18 17:26:20 localhost kernel:    [<c05c4eca>] pfifo_fast_reset+0x14/0x2f
Feb 18 17:26:20 localhost kernel:    [<c05c48f7>] qdisc_destroy+0x36/0x69
Feb 18 17:26:20 localhost kernel:    [<c05c4954>] dev_shutdown+0x2a/0x7b
Feb 18 17:26:20 localhost kernel:    [<c05b6210>] rollback_registered+0xb8/0x11e
Feb 18 17:26:20 localhost kernel:    [<c05b627e>] unregister_netdevice+0x8/0x35
Feb 18 17:26:20 localhost kernel:    [<c05b62ba>] unregister_netdev+0xf/0x15
Feb 18 17:26:20 localhost kernel:    [<f8bb523f>] ppp_shutdown_interface+0x58/0xad [ppp_generic]
Feb 18 17:26:20 localhost kernel:    [<f8bb5484>] ppp_release+0x29/0x56 [ppp_generic]
Feb 18 17:26:20 localhost kernel:    [<c047f08a>] __fput+0xba/0x172
Feb 18 17:26:20 localhost kernel:    [<c047c7df>] filp_close+0x51/0x58
Feb 18 17:26:21 localhost kernel:    [<c047d9c4>] sys_close+0x70/0xab
Feb 18 17:26:21 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:21 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:21 localhost kernel: 
Feb 18 17:26:21 localhost kernel:     ... acquired at:
Feb 18 17:26:21 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:21 localhost kernel:    [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:21 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:21 localhost kernel:    [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:21 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:21 localhost kernel:    [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:21 localhost kernel:    [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:21 localhost kernel:    [<c05b680a>] register_netdevice+0x29f/0x2f9
Feb 18 17:26:21 localhost kernel:    [<c05b6893>] register_netdev+0x2f/0x3b
Feb 18 17:26:21 localhost kernel:    [<c076aded>] loopback_net_init+0x36/0x68
Feb 18 17:26:21 localhost kernel:    [<c05b3fa0>] register_pernet_operations+0x10/0x11
Feb 18 17:26:21 localhost kernel:    [<c05b3ffc>] register_pernet_device+0x1c/0x48
Feb 18 17:26:21 localhost kernel:    [<c074e491>] kernel_init+0x148/0x2af
Feb 18 17:26:21 localhost kernel:    [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:21 localhost kernel:    [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:21 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:21 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:21 localhost kernel: 
Feb 18 17:26:21 localhost kernel:     -> (&list->lock#6){....} ops: 675 {
Feb 18 17:26:21 localhost kernel:        initial-use  at:
Feb 18 17:26:21 localhost kernel:                                 [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:21 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:21 localhost kernel:                                 [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:21 localhost kernel:                                 [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:21 localhost kernel:                                 [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:21 localhost kernel:                                 [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:21 localhost kernel:                                 [<c05b0afd>] skb_queue_purge+0x11/0x17
Feb 18 17:26:21 localhost kernel:                                 [<c05c4eca>] pfifo_fast_reset+0x14/0x2f
Feb 18 17:26:21 localhost kernel:                                 [<c05c4712>] qdisc_reset+0x10/0x11
Feb 18 17:26:21 localhost kernel:                                 [<c05c4b83>] dev_deactivate+0x2a/0xfd
Feb 18 17:26:21 localhost kernel:                                 [<c05bf1f4>] __linkwatch_run_queue+0x143/0x17a
Feb 18 17:26:21 localhost kernel:                                 [<c05bf248>] linkwatch_event+0x1d/0x22
Feb 18 17:26:21 localhost kernel:                                 [<c043a0f3>] run_workqueue+0xdb/0x1ae
Feb 18 17:26:21 localhost kernel:                                 [<c043a09f>] run_workqueue+0x87/0x1ae
Feb 18 17:26:21 localhost kernel:                                 [<c05bf22b>] linkwatch_event+0x0/0x22
Feb 18 17:26:21 localhost kernel:                                 [<c043aad8>] worker_thread+0x0/0xc4
Feb 18 17:26:21 localhost kernel:                                 [<c043ab92>] worker_thread+0xba/0xc4
Feb 18 17:26:22 localhost kernel:                                 [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:22 localhost kernel:                                 [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:22 localhost kernel:                                 [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:22 localhost kernel:                                 [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:22 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:22 localhost kernel:      }
Feb 18 17:26:22 localhost kernel:      ... key      at: [<c0a3e7d4>] __key.21167+0x0/0x8
Feb 18 17:26:22 localhost kernel:     ... acquired at:
Feb 18 17:26:22 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:22 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:22 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:22 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:22 localhost kernel:    [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:22 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:22 localhost kernel:    [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:22 localhost kernel:    [<c05b0afd>] skb_queue_purge+0x11/0x17
Feb 18 17:26:22 localhost kernel:    [<c05c4eca>] pfifo_fast_reset+0x14/0x2f
Feb 18 17:26:22 localhost kernel:    [<c05c4712>] qdisc_reset+0x10/0x11
Feb 18 17:26:22 localhost kernel:    [<c05c4b83>] dev_deactivate+0x2a/0xfd
Feb 18 17:26:22 localhost kernel:    [<c05bf1f4>] __linkwatch_run_queue+0x143/0x17a
Feb 18 17:26:22 localhost kernel:    [<c05bf248>] linkwatch_event+0x1d/0x22
Feb 18 17:26:22 localhost kernel:    [<c043a0f3>] run_workqueue+0xdb/0x1ae
Feb 18 17:26:22 localhost kernel:    [<c043a09f>] run_workqueue+0x87/0x1ae
Feb 18 17:26:22 localhost kernel:    [<c05bf22b>] linkwatch_event+0x0/0x22
Feb 18 17:26:22 localhost kernel:    [<c043aad8>] worker_thread+0x0/0xc4
Feb 18 17:26:22 localhost kernel:    [<c043ab92>] worker_thread+0xba/0xc4
Feb 18 17:26:22 localhost kernel:    [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:22 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:22 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:22 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:22 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:22 localhost kernel: 
Feb 18 17:26:22 localhost kernel:    ... acquired at:
Feb 18 17:26:22 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:22 localhost kernel:    [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:22 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:22 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:22 localhost kernel:    [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:22 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:22 localhost kernel:    [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:22 localhost kernel:    [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:22 localhost kernel:    [<c05b7e60>] dev_queue_xmit+0x125/0x2e9
Feb 18 17:26:23 localhost kernel:    [<c05d62e7>] ip_finish_output+0x1e6/0x21e
Feb 18 17:26:23 localhost kernel:    [<c05d6f58>] ip_queue_xmit+0x30d/0x34d
Feb 18 17:26:23 localhost kernel:    [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:23 localhost kernel:    [<c047a1bd>] kfree+0x9f/0xa6
Feb 18 17:26:23 localhost kernel:    [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:23 localhost kernel:    [<c05b0c83>] pskb_expand_head+0xde/0x144
Feb 18 17:26:23 localhost kernel:    [<f8bbf7bf>] pppol2tp_xmit+0x426/0x4a2 [pppol2tp]
Feb 18 17:26:23 localhost kernel:    [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:23 localhost kernel:    [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:23 localhost kernel:    [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:23 localhost kernel:    [<f8bb730e>] ppp_channel_push+0x34/0x90 [ppp_generic]
Feb 18 17:26:23 localhost kernel:    [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:23 localhost kernel:    [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:23 localhost kernel:    [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:23 localhost kernel:    [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:23 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:23 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:23 localhost kernel: 
Feb 18 17:26:23 localhost kernel:    -> (&rt_hash_locks[i]){-+..} ops: 24708841 {
Feb 18 17:26:23 localhost kernel:       initial-use  at:
Feb 18 17:26:23 localhost kernel:                               [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:23 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:23 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:23 localhost kernel:                               [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:23 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:23 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:23 localhost kernel:                               [<c05f90b6>] ip_mc_inc_group+0x184/0x1c1
Feb 18 17:26:23 localhost kernel:                               [<c05f9134>] ip_mc_up+0x41/0x59
Feb 18 17:26:23 localhost kernel:                               [<c05f4f9f>] inetdev_event+0x257/0x46e
Feb 18 17:26:23 localhost kernel:                               [<c05c258a>] fib_rules_event+0x120/0x12a
Feb 18 17:26:23 localhost kernel:                               [<c05c24b4>] fib_rules_event+0x4a/0x12a
Feb 18 17:26:23 localhost kernel:                               [<c061d06c>] notifier_call_chain+0x2a/0x47
Feb 18 17:26:23 localhost kernel:                               [<c0441084>] raw_notifier_call_chain+0x17/0x1a
Feb 18 17:26:23 localhost kernel:                               [<c05b73cf>] dev_open+0x71/0x77
Feb 18 17:26:23 localhost kernel:                               [<c05b5ffa>] dev_change_flags+0x9d/0x149
Feb 18 17:26:23 localhost kernel:                               [<c05b49b5>] __dev_get_by_name+0x67/0x72
Feb 18 17:26:23 localhost kernel:                               [<c05f54f7>] devinet_ioctl+0x22e/0x539
Feb 18 17:26:23 localhost kernel:                               [<c05b7188>] dev_ioctl+0x46f/0x5e0
Feb 18 17:26:23 localhost kernel:                               [<c05ab078>] sock_ioctl+0x1bb/0x1e0
Feb 18 17:26:23 localhost kernel:                               [<c05aaebd>] sock_ioctl+0x0/0x1e0
Feb 18 17:26:23 localhost kernel:                               [<c0488827>] do_ioctl+0x1f/0x62
Feb 18 17:26:23 localhost kernel:                               [<c061ccb2>] do_page_fault+0x234/0x5c4
Feb 18 17:26:23 localhost kernel:                               [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 17:26:24 localhost kernel:                               [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 17:26:24 localhost kernel:                               [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:24 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:24 localhost kernel:       in-softirq-W at:
Feb 18 17:26:24 localhost kernel:                               [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:24 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:24 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel:                               [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:24 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel:                               [<c05cee70>] rt_run_flush+0x0/0x8b
Feb 18 17:26:24 localhost kernel:                               [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 17:26:24 localhost kernel:                               [<c05cee70>] rt_run_flush+0x0/0x8b
Feb 18 17:26:24 localhost kernel:                               [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:24 localhost kernel:                               [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:24 localhost kernel:                               [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:24 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:24 localhost kernel:       hardirq-on-W at:
Feb 18 17:26:24 localhost kernel:                               [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:24 localhost kernel:                               [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:24 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel:                               [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:24 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel:                               [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel:                               [<c05f90b6>] ip_mc_inc_group+0x184/0x1c1
Feb 18 17:26:24 localhost kernel:                               [<c05f9134>] ip_mc_up+0x41/0x59
Feb 18 17:26:24 localhost kernel:                               [<c05f4f9f>] inetdev_event+0x257/0x46e
Feb 18 17:26:24 localhost kernel:                               [<c05c258a>] fib_rules_event+0x120/0x12a
Feb 18 17:26:24 localhost kernel:                               [<c05c24b4>] fib_rules_event+0x4a/0x12a
Feb 18 17:26:24 localhost kernel:                               [<c061d06c>] notifier_call_chain+0x2a/0x47
Feb 18 17:26:24 localhost kernel:                               [<c0441084>] raw_notifier_call_chain+0x17/0x1a
Feb 18 17:26:24 localhost kernel:                               [<c05b73cf>] dev_open+0x71/0x77
Feb 18 17:26:24 localhost kernel:                               [<c05b5ffa>] dev_change_flags+0x9d/0x149
Feb 18 17:26:24 localhost kernel:                               [<c05b49b5>] __dev_get_by_name+0x67/0x72
Feb 18 17:26:24 localhost kernel:                               [<c05f54f7>] devinet_ioctl+0x22e/0x539
Feb 18 17:26:24 localhost kernel:                               [<c05b7188>] dev_ioctl+0x46f/0x5e0
Feb 18 17:26:24 localhost kernel:                               [<c05ab078>] sock_ioctl+0x1bb/0x1e0
Feb 18 17:26:24 localhost kernel:                               [<c05aaebd>] sock_ioctl+0x0/0x1e0
Feb 18 17:26:24 localhost kernel:                               [<c0488827>] do_ioctl+0x1f/0x62
Feb 18 17:26:24 localhost kernel:                               [<c061ccb2>] do_page_fault+0x234/0x5c4
Feb 18 17:26:24 localhost kernel:                               [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 17:26:24 localhost kernel:                               [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 17:26:24 localhost kernel:                               [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:25 localhost kernel:                               [<ffffffff>] 0xffffffff
Feb 18 17:26:25 localhost kernel:     }
Feb 18 17:26:25 localhost kernel:     ... key      at: [<c0a3ea90>] __key.39046+0x0/0x8
Feb 18 17:26:25 localhost kernel:     -> (&parent->list_lock){++..} ops: 252663 {
Feb 18 17:26:25 localhost kernel:        initial-use  at:
Feb 18 17:26:25 localhost kernel:                                 [<c040a9d6>] save_stack_trace+0x20/0x3a
Feb 18 17:26:25 localhost kernel:                                 [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:25 localhost kernel:                                 [<c043badc>] __kernel_text_address+0x18/0x23
Feb 18 17:26:25 localhost kernel:                                 [<c0448013>] check_usage_backwards+0x19/0x41
Feb 18 17:26:25 localhost kernel:                                 [<c0446ab0>] save_trace+0x37/0x8b
Feb 18 17:26:25 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:25 localhost kernel:                                 [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:25 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:25 localhost kernel:                                 [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:25 localhost kernel:                                 [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:25 localhost kernel:                                 [<c047b89c>] kmem_cache_create+0x62/0x38e
Feb 18 17:26:25 localhost kernel:                                 [<c047a790>] kmem_cache_alloc+0x53/0x9f
Feb 18 17:26:25 localhost kernel:                                 [<c047b99d>] kmem_cache_create+0x163/0x38e
Feb 18 17:26:25 localhost kernel:                                 [<c0479da2>] cache_estimate+0x6c/0x89
Feb 18 17:26:25 localhost kernel:                                 [<c0762241>] kmem_cache_init+0x182/0x3f6
Feb 18 17:26:25 localhost kernel:                                 [<c074e88b>] start_kernel+0x293/0x327
Feb 18 17:26:25 localhost kernel:                                 [<c074e0e0>] unknown_bootoption+0x0/0x195
Feb 18 17:26:25 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:25 localhost kernel:        in-hardirq-W at:
Feb 18 17:26:25 localhost kernel:                                 [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:26:25 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:25 localhost kernel:                                 [<c047b7ec>] do_drain+0x1e/0x49
Feb 18 17:26:25 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:25 localhost kernel:                                 [<c047b7ec>] do_drain+0x1e/0x49
Feb 18 17:26:25 localhost kernel:                                 [<c047b7ec>] do_drain+0x1e/0x49
Feb 18 17:26:25 localhost kernel:                                 [<c047b7ce>] do_drain+0x0/0x49
Feb 18 17:26:25 localhost kernel:                                 [<c0418ed5>] smp_call_function_interrupt+0x37/0x62
Feb 18 17:26:25 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:25 localhost kernel:                                 [<c04059b3>] call_function_interrupt+0x33/0x38
Feb 18 17:26:25 localhost kernel:                                 [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:25 localhost kernel:                                 [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:26:25 localhost kernel:                                 [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:26:25 localhost kernel:                                 [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:26:25 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:25 localhost kernel:        in-softirq-W at:
Feb 18 17:26:25 localhost kernel:                                 [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:26:25 localhost kernel:                                 [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:26 localhost kernel:                                 [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:26 localhost kernel:                                 [<c04488b6>] debug_check_no_locks_freed+0x105/0x11f
Feb 18 17:26:26 localhost kernel:                                 [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:26 localhost kernel:                                 [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:26 localhost kernel:                                 [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:26 localhost kernel:                                 [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:26 localhost kernel:                                 [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:26 localhost kernel:                                 [<c047a1fa>] kmem_cache_free+0x36/0x5e
Feb 18 17:26:26 localhost kernel:                                 [<c043b647>] __rcu_process_callbacks+0xfc/0x16e
Feb 18 17:26:26 localhost kernel:                                 [<c043b6d1>] rcu_process_callbacks+0x18/0x30
Feb 18 17:26:26 localhost kernel:                                 [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:26 localhost kernel:                                 [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:26 localhost kernel:                                 [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:26 localhost kernel:                                 [<ffffffff>] 0xffffffff
Feb 18 17:26:26 localhost kernel:      }
Feb 18 17:26:26 localhost kernel:      ... key      at: [<c0a2bbe8>] __key.19871+0x0/0x8
Feb 18 17:26:26 localhost kernel:      -> (&zone->lock){.+..} ops: 63132 {
Feb 18 17:26:26 localhost kernel:         initial-use  at:
Feb 18 17:26:26 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:26 localhost kernel:                                   [<c04ef19a>] number+0x147/0x215
Feb 18 17:26:26 localhost kernel:                                   [<c04efab1>] vsnprintf+0x440/0x47c
Feb 18 17:26:26 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:26 localhost kernel:                                   [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:26 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:26 localhost kernel:                                   [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:26 localhost kernel:                                   [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:26 localhost kernel:                                   [<c04645c0>] free_hot_cold_page+0x133/0x175
Feb 18 17:26:26 localhost kernel:                                   [<c07606b8>] free_all_bootmem_core+0xdd/0x158
Feb 18 17:26:26 localhost kernel:                                   [<c075d82e>] mem_init+0x8c/0x383
Feb 18 17:26:26 localhost kernel:                                   [<c042c8c7>] printk+0x1b/0x1f
Feb 18 17:26:26 localhost kernel:                                   [<c0761a7b>] alloc_large_system_hash+0x1f7/0x222
Feb 18 17:26:26 localhost kernel:                                   [<c04f2f65>] __spin_lock_init+0x29/0x49
Feb 18 17:26:26 localhost kernel:                                   [<c076287c>] inode_init_early+0x49/0x72
Feb 18 17:26:26 localhost kernel:                                   [<c074e886>] start_kernel+0x28e/0x327
Feb 18 17:26:26 localhost kernel:                                   [<c074e0e0>] unknown_bootoption+0x0/0x195
Feb 18 17:26:26 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:26 localhost kernel:         in-softirq-W at:
Feb 18 17:26:26 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:26 localhost kernel:                                   [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:26:26 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:26 localhost kernel:                                   [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:27 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:27 localhost kernel:                                   [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:27 localhost kernel:                                   [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:27 localhost kernel:                                   [<c04645c0>] free_hot_cold_page+0x133/0x175
Feb 18 17:26:27 localhost kernel:                                   [<c042a030>] free_task+0x18/0x25
Feb 18 17:26:27 localhost kernel:                                   [<c043b647>] __rcu_process_callbacks+0xfc/0x16e
Feb 18 17:26:27 localhost kernel:                                   [<c043b6d1>] rcu_process_callbacks+0x18/0x30
Feb 18 17:26:27 localhost kernel:                                   [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:27 localhost kernel:                                   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:27 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:27 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:27 localhost kernel:       }
Feb 18 17:26:27 localhost kernel:       ... key      at: [<c0a21fb8>] __key.25233+0x0/0x8
Feb 18 17:26:27 localhost kernel:      ... acquired at:
Feb 18 17:26:27 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:27 localhost kernel:    [<c046472e>] __free_pages_ok+0x101/0x2a8
Feb 18 17:26:27 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:27 localhost kernel:    [<c046472e>] __free_pages_ok+0x101/0x2a8
Feb 18 17:26:27 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:27 localhost kernel:    [<c046472e>] __free_pages_ok+0x101/0x2a8
Feb 18 17:26:27 localhost kernel:    [<c046472e>] __free_pages_ok+0x101/0x2a8
Feb 18 17:26:27 localhost kernel:    [<c0479f5b>] kmem_freepages+0x4e/0xa5
Feb 18 17:26:27 localhost kernel:    [<c047a2a2>] slab_destroy+0x2e/0x49
Feb 18 17:26:27 localhost kernel:    [<c047a370>] free_block+0xb3/0xec
Feb 18 17:26:27 localhost kernel:    [<c047a433>] drain_array+0x8a/0xb5
Feb 18 17:26:27 localhost kernel:    [<c047b4ec>] cache_reap+0x7e/0xf4
Feb 18 17:26:27 localhost kernel:    [<c043a0f3>] run_workqueue+0xdb/0x1ae
Feb 18 17:26:27 localhost kernel:    [<c043a09f>] run_workqueue+0x87/0x1ae
Feb 18 17:26:27 localhost kernel:    [<c047b46e>] cache_reap+0x0/0xf4
Feb 18 17:26:27 localhost kernel:    [<c043aad8>] worker_thread+0x0/0xc4
Feb 18 17:26:27 localhost kernel:    [<c043ab92>] worker_thread+0xba/0xc4
Feb 18 17:26:27 localhost kernel:    [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:27 localhost kernel:    [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:27 localhost kernel:    [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:27 localhost kernel:    [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:27 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:27 localhost kernel: 
Feb 18 17:26:27 localhost kernel:      -> (&on_slab_l3_key){.+..} ops: 28386 {
Feb 18 17:26:27 localhost kernel:         initial-use  at:
Feb 18 17:26:27 localhost kernel:                                   [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:27 localhost kernel:                                   [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:27 localhost kernel:                                   [<c040a9d6>] save_stack_trace+0x20/0x3a
Feb 18 17:26:27 localhost kernel:                                   [<c0464b77>] get_page_from_freelist+0x24b/0x37a
Feb 18 17:26:28 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:28 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:28 localhost kernel:                                   [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:28 localhost kernel:                                   [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel:                                   [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel:                                   [<c046a328>] mod_zone_page_state+0x35/0x3c
Feb 18 17:26:28 localhost kernel:                                   [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:28 localhost kernel:                                   [<c047a790>] kmem_cache_alloc+0x53/0x9f
Feb 18 17:26:28 localhost kernel:                                   [<c047ab07>] cache_alloc_refill+0x32b/0x493
Feb 18 17:26:28 localhost kernel:                                   [<c047a790>] kmem_cache_alloc+0x53/0x9f
Feb 18 17:26:28 localhost kernel:                                   [<c075ee99>] pidmap_init+0x15/0x137
Feb 18 17:26:28 localhost kernel:                                   [<c074e8a0>] start_kernel+0x2a8/0x327
Feb 18 17:26:28 localhost kernel:                                   [<c074e0e0>] unknown_bootoption+0x0/0x195
Feb 18 17:26:28 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:28 localhost kernel:         in-softirq-W at:
Feb 18 17:26:28 localhost kernel:                                   [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:28 localhost kernel:                                   [<c05fdd2b>] fn_hash_lookup+0xaa/0xb4
Feb 18 17:26:28 localhost kernel:                                   [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:28 localhost kernel:                                   [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel:                                   [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:28 localhost kernel:                                   [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel:                                   [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel:                                   [<c047acf5>] __kmalloc+0x86/0xd2
Feb 18 17:26:28 localhost kernel:                                   [<c05bb6b9>] neigh_create+0x2e4/0x41a
Feb 18 17:26:28 localhost kernel:                                   [<c05baadb>] neigh_lookup+0x94/0x9d
Feb 18 17:26:28 localhost kernel:                                   [<c05bb82b>] neigh_event_ns+0x3c/0x76
Feb 18 17:26:28 localhost kernel:                                   [<c05f2116>] arp_process+0x218/0x60c
Feb 18 17:26:28 localhost kernel:                                   [<c05f1efe>] arp_process+0x0/0x60c
Feb 18 17:26:28 localhost kernel:                                   [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:28 localhost kernel:                                   [<c05f250a>] arp_rcv+0x0/0x105
Feb 18 17:26:28 localhost kernel:                                   [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:28 localhost kernel:                                   [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:28 localhost kernel:                                   [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:28 localhost kernel:                                   [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:28 localhost kernel:                                   [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:28 localhost kernel:                                   [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:28 localhost kernel:                                   [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:28 localhost kernel:                                   [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:28 localhost kernel:                                   [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:28 localhost kernel:                                   [<ffffffff>] 0xffffffff
Feb 18 17:26:28 localhost kernel:       }
Feb 18 17:26:29 localhost kernel:       ... key      at: [<c0a2bc0c>] on_slab_l3_key+0x0/0x8
Feb 18 17:26:29 localhost kernel:       -> (&zone->lock){.+..} ops: 63132 {
Feb 18 17:26:29 localhost kernel:          initial-use  at:
Feb 18 17:26:29 localhost kernel:                                     [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:29 localhost kernel:                                     [<c04ef19a>] number+0x147/0x215
Feb 18 17:26:29 localhost kernel:                                     [<c04efab1>] vsnprintf+0x440/0x47c
Feb 18 17:26:29 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:29 localhost kernel:                                     [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:29 localhost kernel:                                     [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel:                                     [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel:                                     [<c04645c0>] free_hot_cold_page+0x133/0x175
Feb 18 17:26:29 localhost kernel:                                     [<c07606b8>] free_all_bootmem_core+0xdd/0x158
Feb 18 17:26:29 localhost kernel:                                     [<c075d82e>] mem_init+0x8c/0x383
Feb 18 17:26:29 localhost kernel:                                     [<c042c8c7>] printk+0x1b/0x1f
Feb 18 17:26:29 localhost kernel:                                     [<c0761a7b>] alloc_large_system_hash+0x1f7/0x222
Feb 18 17:26:29 localhost kernel:                                     [<c04f2f65>] __spin_lock_init+0x29/0x49
Feb 18 17:26:29 localhost kernel:                                     [<c076287c>] inode_init_early+0x49/0x72
Feb 18 17:26:29 localhost kernel:                                     [<c074e886>] start_kernel+0x28e/0x327
Feb 18 17:26:29 localhost kernel:                                     [<c074e0e0>] unknown_bootoption+0x0/0x195
Feb 18 17:26:29 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:26:29 localhost kernel:          in-softirq-W at:
Feb 18 17:26:29 localhost kernel:                                     [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:29 localhost kernel:                                     [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:26:29 localhost kernel:                                     [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:29 localhost kernel:                                     [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel:                                     [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:29 localhost kernel:                                     [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel:                                     [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel:                                     [<c04645c0>] free_hot_cold_page+0x133/0x175
Feb 18 17:26:29 localhost kernel:                                     [<c042a030>] free_task+0x18/0x25
Feb 18 17:26:29 localhost kernel:                                     [<c043b647>] __rcu_process_callbacks+0xfc/0x16e
Feb 18 17:26:29 localhost kernel:                                     [<c043b6d1>] rcu_process_callbacks+0x18/0x30
Feb 18 17:26:29 localhost kernel:                                     [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:29 localhost kernel:                                     [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:29 localhost kernel:                                     [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:29 localhost kernel:                                     [<ffffffff>] 0xffffffff
Feb 18 17:26:29 localhost kernel:        }
Feb 18 17:26:29 localhost kernel:        ... key      at: [<c0a21fb8>] __key.25233+0x0/0x8
Feb 18 17:26:29 localhost kernel:       ... acquired at:
Feb 18 17:26:29 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:29 localhost kernel:    [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:29 localhost kernel:    [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:30 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:30 localhost kernel:    [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:30 localhost kernel:    [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:30 localhost kernel:    [<c04645c0>] free_hot_cold_page+0x133/0x175
Feb 18 17:26:30 localhost kernel:    [<c047a2a2>] slab_destroy+0x2e/0x49
Feb 18 17:26:30 localhost kernel:    [<c047a370>] free_block+0xb3/0xec
Feb 18 17:26:30 localhost kernel:    [<c047a0f2>] cache_flusharray+0x74/0xa0
Feb 18 17:26:30 localhost kernel:    [<c047a19c>] kfree+0x7e/0xa6
Feb 18 17:26:30 localhost kernel:    [<c05f594c>] snmp_mib_free+0x1a/0x29
Feb 18 17:26:30 localhost kernel:    [<f8f283ac>] in6_dev_finish_destroy+0x8e/0xb9 [ipv6]
Feb 18 17:26:30 localhost kernel:    [<f8f28dd9>] addrconf_ifdown+0x2df/0x2eb [ipv6]
Feb 18 17:26:30 localhost kernel:    [<f8f2b0f5>] addrconf_notify+0x738/0x7c2 [ipv6]
Feb 18 17:26:30 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:30 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:30 localhost kernel:    [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:30 localhost kernel:    [<c0614a16>] packet_notifier+0x149/0x151
Feb 18 17:26:30 localhost kernel:    [<c06148f3>] packet_notifier+0x26/0x151
Feb 18 17:26:30 localhost kernel:    [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:30 localhost kernel:    [<c0614a16>] packet_notifier+0x149/0x151
Feb 18 17:26:30 localhost kernel:    [<c061d06c>] notifier_call_chain+0x2a/0x47
Feb 18 17:26:30 localhost kernel:    [<c0441084>] raw_notifier_call_chain+0x17/0x1a
Feb 18 17:26:30 localhost kernel:    [<c05b621c>] rollback_registered+0xc4/0x11e
Feb 18 17:26:30 localhost kernel:    [<c05b627e>] unregister_netdevice+0x8/0x35
Feb 18 17:26:30 localhost kernel:    [<c05b62ba>] unregister_netdev+0xf/0x15
Feb 18 17:26:30 localhost kernel:    [<f8bb523f>] ppp_shutdown_interface+0x58/0xad [ppp_generic]
Feb 18 17:26:30 localhost kernel:    [<f8bb5484>] ppp_release+0x29/0x56 [ppp_generic]
Feb 18 17:26:30 localhost kernel:    [<c047f08a>] __fput+0xba/0x172
Feb 18 17:26:30 localhost kernel:    [<c047c7df>] filp_close+0x51/0x58
Feb 18 17:26:30 localhost kernel:    [<c047d9c4>] sys_close+0x70/0xab
Feb 18 17:26:30 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:30 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:30 localhost kernel: 
Feb 18 17:26:30 localhost kernel:      ... acquired at:
Feb 18 17:26:30 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:30 localhost kernel:    [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:30 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:30 localhost kernel:    [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:30 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:30 localhost kernel:    [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:30 localhost kernel:    [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:30 localhost kernel:    [<c047a1fa>] kmem_cache_free+0x36/0x5e
Feb 18 17:26:30 localhost kernel:    [<c047a370>] free_block+0xb3/0xec
Feb 18 17:26:30 localhost kernel:    [<c047a0f2>] cache_flusharray+0x74/0xa0
Feb 18 17:26:31 localhost kernel:    [<c047a19c>] kfree+0x7e/0xa6
Feb 18 17:26:31 localhost kernel:    [<f8f28d8e>] addrconf_ifdown+0x294/0x2eb [ipv6]
Feb 18 17:26:31 localhost kernel:    [<f8f2b0f5>] addrconf_notify+0x738/0x7c2 [ipv6]
Feb 18 17:26:31 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:31 localhost kernel:    [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:31 localhost kernel:    [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:31 localhost kernel:    [<c0614a16>] packet_notifier+0x149/0x151
Feb 18 17:26:31 localhost kernel:    [<c06148f3>] packet_notifier+0x26/0x151
Feb 18 17:26:31 localhost kernel:    [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:31 localhost kernel:    [<c0614a16>] packet_notifier+0x149/0x151
Feb 18 17:26:31 localhost kernel:    [<c061d06c>] notifier_call_chain+0x2a/0x47
Feb 18 17:26:31 localhost kernel:    [<c0441084>] raw_notifier_call_chain+0x17/0x1a
Feb 18 17:26:31 localhost kernel:    [<c05b621c>] rollback_registered+0xc4/0x11e
Feb 18 17:26:31 localhost kernel:    [<c05b627e>] unregister_netdevice+0x8/0x35
Feb 18 17:26:31 localhost kernel:    [<c05b62ba>] unregister_netdev+0xf/0x15
Feb 18 17:26:31 localhost kernel:    [<f8bb523f>] ppp_shutdown_interface+0x58/0xad [ppp_generic]
Feb 18 17:26:31 localhost kernel:    [<f8bb5484>] ppp_release+0x29/0x56 [ppp_generic]
Feb 18 17:26:31 localhost kernel:    [<c047f08a>] __fput+0xba/0x172
Feb 18 17:26:31 localhost kernel:    [<c047c7df>] filp_close+0x51/0x58
Feb 18 17:26:31 localhost kernel:    [<c047d9c4>] sys_close+0x70/0xab
Feb 18 17:26:31 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:31 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:31 localhost kernel: 
Feb 18 17:26:31 localhost kernel:     ... acquired at:
Feb 18 17:26:31 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:31 localhost kernel:    [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:31 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:31 localhost kernel:    [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:31 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:31 localhost kernel:    [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:31 localhost kernel:    [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:31 localhost kernel:    [<c047a790>] kmem_cache_alloc+0x53/0x9f
Feb 18 17:26:31 localhost kernel:    [<c05bb516>] neigh_create+0x141/0x41a
Feb 18 17:26:31 localhost kernel:    [<c05baadb>] neigh_lookup+0x94/0x9d
Feb 18 17:26:31 localhost kernel:    [<c05f144f>] arp_bind_neighbour+0x62/0x74
Feb 18 17:26:31 localhost kernel:    [<c05cf248>] rt_intern_hash+0x236/0x322
Feb 18 17:26:31 localhost kernel:    [<c05cfab6>] __ip_route_output_key+0x782/0x847
Feb 18 17:26:31 localhost kernel:    [<c05cf948>] __ip_route_output_key+0x614/0x847
Feb 18 17:26:31 localhost kernel:    [<c05eace7>] tcp_v4_connect+0xef/0x5a6
Feb 18 17:26:31 localhost kernel:    [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:31 localhost kernel:    [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 17:26:32 localhost kernel:    [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:32 localhost kernel:    [<c05f6944>] inet_stream_connect+0x7f/0x208
Feb 18 17:26:32 localhost kernel:    [<c04f05f4>] copy_from_user+0x32/0x5e
Feb 18 17:26:32 localhost kernel:    [<c05ac371>] sys_connect+0x7b/0xa6
Feb 18 17:26:32 localhost kernel:    [<c05aea5a>] sock_setsockopt+0x492/0x4aa
Feb 18 17:26:32 localhost kernel:    [<c05aea5a>] sock_setsockopt+0x492/0x4aa
Feb 18 17:26:32 localhost kernel:    [<c05db937>] tcp_setsockopt+0x321/0x339
Feb 18 17:26:32 localhost kernel:    [<c05ab618>] sock_map_fd+0x41/0x4a
Feb 18 17:26:32 localhost kernel:    [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:32 localhost kernel:    [<c05ab47f>] sys_setsockopt+0x62/0x9c
Feb 18 17:26:32 localhost kernel:    [<c05acb97>] sys_socketcall+0xac/0x261
Feb 18 17:26:32 localhost kernel:    [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:32 localhost kernel:    [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:32 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:32 localhost kernel: 
Feb 18 17:26:32 localhost kernel:    ... acquired at:
Feb 18 17:26:32 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:32 localhost kernel:    [<c05cf06f>] rt_intern_hash+0x5d/0x322
Feb 18 17:26:32 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:32 localhost kernel:    [<c05cf06f>] rt_intern_hash+0x5d/0x322
Feb 18 17:26:32 localhost kernel:    [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:32 localhost kernel:    [<c05cf06f>] rt_intern_hash+0x5d/0x322
Feb 18 17:26:32 localhost kernel:    [<c05cf06f>] rt_intern_hash+0x5d/0x322
Feb 18 17:26:32 localhost kernel:    [<c05cf000>] rt_set_nexthop+0x105/0x117
Feb 18 17:26:32 localhost kernel:    [<c05cfab6>] __ip_route_output_key+0x782/0x847
Feb 18 17:26:32 localhost kernel:    [<c05cf948>] __ip_route_output_key+0x614/0x847
Feb 18 17:26:32 localhost kernel:    [<c0464b77>] get_page_from_freelist+0x24b/0x37a
Feb 18 17:26:32 localhost kernel:    [<c05cfb8f>] ip_route_output_flow+0x14/0x1ef
Feb 18 17:26:32 localhost kernel:    [<c05b0c83>] pskb_expand_head+0xde/0x144
Feb 18 17:26:32 localhost kernel:    [<f8bbf6fe>] pppol2tp_xmit+0x365/0x4a2 [pppol2tp]
Feb 18 17:26:32 localhost kernel:    [<f8bb730e>] ppp_channel_push+0x34/0x90 [ppp_generic]
Feb 18 17:26:32 localhost kernel:    [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:32 localhost kernel:    [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:32 localhost kernel:    [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:32 localhost kernel:    [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:32 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:32 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:32 localhost kernel: 
Feb 18 17:26:32 localhost kernel:   ... acquired at:
Feb 18 17:26:32 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:32 localhost kernel:    [<f8bb56db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 17:26:32 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:32 localhost kernel:    [<f8bb56db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:33 localhost kernel:    [<f8bb56db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<f8bb56db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:33 localhost kernel:    [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:33 localhost kernel:    [<f8bb63f1>] ppp_xmit_process+0x4f6/0x5a1 [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:33 localhost kernel:    [<f8bb83b4>] ppp_write+0xc7/0xdb [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:33 localhost kernel:    [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:33 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:33 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:33 localhost kernel: 
Feb 18 17:26:33 localhost kernel:  ... acquired at:
Feb 18 17:26:33 localhost kernel:    [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:33 localhost kernel:    [<f8bb6bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:33 localhost kernel:    [<f8bb6bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:33 localhost kernel:    [<f8bb6bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<f8bb6bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 17:26:33 localhost kernel:    [<c061b008>] __down+0x82/0xb8
Feb 18 17:26:33 localhost kernel:    [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:33 localhost kernel:    [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:33 localhost kernel:    [<c061b008>] __down+0x82/0xb8
Feb 18 17:26:33 localhost kernel:    [<c0425474>] default_wake_function+0x0/0x8
Feb 18 17:26:33 localhost kernel:    [<c061ae17>] __down_failed+0x7/0xc
Feb 18 17:26:33 localhost kernel:    [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 17:26:33 localhost kernel:    [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 17:26:33 localhost kernel:    [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 17:26:33 localhost kernel:    [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:33 localhost kernel:    [<ffffffff>] 0xffffffff
Feb 18 17:26:33 localhost kernel: 
Feb 18 17:26:33 localhost kernel: 
Feb 18 17:26:33 localhost kernel: the soft-irq-unsafe lock's dependencies:
Feb 18 17:26:33 localhost kernel: -> (&sk->sk_dst_lock){----} ops: 3988 {
Feb 18 17:26:33 localhost kernel:    initial-use  at:
Feb 18 17:26:33 localhost kernel:                         [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:33 localhost kernel:                         [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:33 localhost kernel:                         [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 17:26:33 localhost kernel:                         [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 17:26:33 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 17:26:34 localhost kernel:                         [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 17:26:34 localhost kernel:                         [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 17:26:34 localhost kernel:                         [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 17:26:34 localhost kernel:                         [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:34 localhost kernel:                         [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:34 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 17:26:34 localhost kernel:    softirq-on-W at:
Feb 18 17:26:34 localhost kernel:                         [<c0449046>] __lock_acquire+0x48b/0xbf1
Feb 18 17:26:34 localhost kernel:                         [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:34 localhost kernel:                         [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 17:26:34 localhost kernel:                         [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 17:26:34 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 17:26:34 localhost kernel:                         [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 17:26:34 localhost kernel:                         [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 17:26:34 localhost kernel:                         [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 17:26:34 localhost kernel:                         [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:34 localhost kernel:                         [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:34 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 17:26:34 localhost kernel:    hardirq-on-W at:
Feb 18 17:26:34 localhost kernel:                         [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:34 localhost kernel:                         [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:34 localhost kernel:                         [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 17:26:34 localhost kernel:                         [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 17:26:34 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel:                         [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 17:26:34 localhost kernel:                         [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 17:26:34 localhost kernel:                         [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 17:26:34 localhost kernel:                         [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 17:26:34 localhost kernel:                         [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:35 localhost kernel:                         [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:35 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 17:26:35 localhost kernel:    softirq-on-R at:
Feb 18 17:26:35 localhost kernel:                         [<c0449046>] __lock_acquire+0x48b/0xbf1
Feb 18 17:26:35 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:35 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel:                         [<c061b3a7>] _read_lock+0x29/0x34
Feb 18 17:26:35 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel:                         [<c05baadb>] neigh_lookup+0x94/0x9d
Feb 18 17:26:35 localhost kernel:                         [<c05f0609>] udp_sendmsg+0x240/0x59d
Feb 18 17:26:35 localhost kernel:                         [<c05cf322>] rt_intern_hash+0x310/0x322
Feb 18 17:26:35 localhost kernel:                         [<c05f5fc7>] inet_sendmsg+0x3b/0x45
Feb 18 17:26:35 localhost kernel:                         [<c05ab8cd>] sock_sendmsg+0xc9/0xe4
Feb 18 17:26:35 localhost kernel:                         [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:35 localhost kernel:                         [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:35 localhost kernel:                         [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:35 localhost kernel:                         [<c061b528>] _read_unlock_irq+0x20/0x23
Feb 18 17:26:35 localhost kernel:                         [<c061b528>] _read_unlock_irq+0x20/0x23
Feb 18 17:26:35 localhost kernel:                         [<c05ac29b>] sys_sendto+0x11b/0x13b
Feb 18 17:26:35 localhost kernel:                         [<c046b794>] __do_fault+0x23b/0x35d
Feb 18 17:26:35 localhost kernel:                         [<c046b874>] __do_fault+0x31b/0x35d
Feb 18 17:26:35 localhost kernel:                         [<c046d4ff>] handle_mm_fault+0x2a2/0x5f7
Feb 18 17:26:35 localhost kernel:                         [<c05ac2f2>] sys_send+0x37/0x3b
Feb 18 17:26:35 localhost kernel:                         [<c05acc35>] sys_socketcall+0x14a/0x261
Feb 18 17:26:35 localhost kernel:                         [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:35 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 17:26:35 localhost kernel:    hardirq-on-R at:
Feb 18 17:26:35 localhost kernel:                         [<c048926a>] do_sys_poll+0x260/0x2e0
Feb 18 17:26:35 localhost kernel:                         [<c0449001>] __lock_acquire+0x446/0xbf1
Feb 18 17:26:35 localhost kernel:                         [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:35 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel:                         [<c061b3a7>] _read_lock+0x29/0x34
Feb 18 17:26:35 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel:                         [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel:                         [<c05baadb>] neigh_lookup+0x94/0x9d
Feb 18 17:26:35 localhost kernel:                         [<c05f0609>] udp_sendmsg+0x240/0x59d
Feb 18 17:26:35 localhost kernel:                         [<c05cf322>] rt_intern_hash+0x310/0x322
Feb 18 17:26:35 localhost kernel:                         [<c05f5fc7>] inet_sendmsg+0x3b/0x45
Feb 18 17:26:35 localhost kernel:                         [<c05ab8cd>] sock_sendmsg+0xc9/0xe4
Feb 18 17:26:35 localhost kernel:                         [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:35 localhost kernel:                         [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:35 localhost kernel:                         [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:36 localhost kernel:                         [<c061b528>] _read_unlock_irq+0x20/0x23
Feb 18 17:26:36 localhost kernel:                         [<c061b528>] _read_unlock_irq+0x20/0x23
Feb 18 17:26:36 localhost kernel:                         [<c05ac29b>] sys_sendto+0x11b/0x13b
Feb 18 17:26:36 localhost kernel:                         [<c046b794>] __do_fault+0x23b/0x35d
Feb 18 17:26:36 localhost kernel:                         [<c046b874>] __do_fault+0x31b/0x35d
Feb 18 17:26:36 localhost kernel:                         [<c046d4ff>] handle_mm_fault+0x2a2/0x5f7
Feb 18 17:26:36 localhost kernel:                         [<c05ac2f2>] sys_send+0x37/0x3b
Feb 18 17:26:36 localhost kernel:                         [<c05acc35>] sys_socketcall+0x14a/0x261
Feb 18 17:26:36 localhost kernel:                         [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:36 localhost kernel:                         [<ffffffff>] 0xffffffff
Feb 18 17:26:36 localhost kernel:  }
Feb 18 17:26:36 localhost kernel:  ... key      at: [<c0a3dc28>] __key.35996+0x0/0x8
Feb 18 17:26:36 localhost kernel: 
Feb 18 17:26:36 localhost kernel: stack backtrace:
Feb 18 17:26:36 localhost kernel: Pid: 13863, comm: pppd Not tainted 2.6.24.2 #1
Feb 18 17:26:36 localhost kernel:  [<c0448b1e>] check_usage+0x24e/0x258
Feb 18 17:26:36 localhost kernel:  [<c0449555>] __lock_acquire+0x99a/0xbf1
Feb 18 17:26:36 localhost kernel:  [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:36 localhost kernel:  [<f8bbf73e>] pppol2tp_xmit+0x3a5/0x4a2 [pppol2tp]
Feb 18 17:26:36 localhost kernel:  [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 17:26:36 localhost kernel:  [<f8bbf73e>] pppol2tp_xmit+0x3a5/0x4a2 [pppol2tp]
Feb 18 17:26:36 localhost kernel:  [<f8bbf73e>] pppol2tp_xmit+0x3a5/0x4a2 [pppol2tp]
Feb 18 17:26:36 localhost kernel:  [<f8bb730e>] ppp_channel_push+0x34/0x90 [ppp_generic]
Feb 18 17:26:36 localhost kernel:  [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:36 localhost kernel:  [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:36 localhost kernel:  [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:36 localhost kernel:  [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:36 localhost kernel:  [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:36 localhost kernel:  =======================

^ permalink raw reply

* Re: [NETFILTER]: Introduce nf_inet_address
From: Patrick McHardy @ 2008-02-19 14:45 UTC (permalink / raw)
  To: David Woodhouse; +Cc: netdev, Jan Engelhardt, David S. Miller, varekova
In-Reply-To: <1203431447.3223.33.camel@shinybook.infradead.org>

David Woodhouse wrote:
> On Tue, 2008-02-19 at 15:01 +0100, Patrick McHardy wrote:
>> David Woodhouse wrote:
>>>> +union nf_inet_addr {
>>>> +	u_int32_t	all[4];
>>>> +	__be32		ip;
>>>> +	__be32		ip6[4];
>>>> +};
>>>> +
>>>>  #ifdef __KERNEL__
>>>>  #ifdef CONFIG_NETFILTER
>>> This breaks the busybox build:
>>>
>>> CC      ipsvd/tcpudp.o
>>> In file included from /usr/include/linux/netfilter_ipv4.h:8,
>>>                  from ipsvd/tcpudp.c:33:
>>> /usr/include/linux/netfilter.h:40: error: expected specifier-qualifier-list before 'u_int32_t'
>>>
>>> What is this 'u_int32_t' nonsense anyway?
>>>
>>> If a user-visible header is likely to be included by libc directly from
>>> a 'standard' header, it may not require <stdint.h>. Therefore it should
>>> use the system-specific types such as '__u32'.
>> Right, I queued this patch to fix it.
> 
> That does the trick -- but are we using u_int32_t elsewhere in
> user-visible headers? Does it work there? How?


Its used in nearly every ip_tables header file. Those are most likely
not included by glibc and iptables includes sys/types.h. Besides
iptables I'm only aware of a perl module that uses these files,
which is probably also including sys/types.h.

>>> A later commit adds struct in_addr and struct in6_addr to this union
>>> too, which breaks busybox even harder.
>> Thats odd, the iptables headers have always used struct in_addr and
>> struct in6_addr in struct ipt_ip/struct ip6t_ip6, which are also
>> used by userspace. What is "ipsvd/tcpudp.c"? I couldn't find it in
>> the Debian busybox source.
> 
> It's in busybox 1.9.1. Just including <netinet/in.h> seems to be
> sufficient to make it happy again. I wonder if netfilter.h should
> include that for itself?


That would break iptables compilation, which already includes
linux/in.h in some files. I guess the best fix for now is to
include netinet/in.h in busybox and long-term clean this up
properly.



^ permalink raw reply

* AW: AW: AW: Problem receiving multicast with kernel.2.6.24, REAL solution
From: Reither Robert @ 2008-02-19 14:47 UTC (permalink / raw)
  To: Patrick McHardy; +Cc: netdev

Visit AVD on prolight+sound in Frankfurt from 12.-15. March 2008 - Hall 8.0, Stand G16
________________________________________________________________________



Hurray, your patch does the trick for me !!!

Now join/leave und receiving of multicasts work like it should !!

May i kiss you Patrick, this saves my week ;-)

Hope we see a minor kernel release with this patch very soon ...

Greetings
Robert

-----Ursprüngliche Nachricht-----
Von: Patrick McHardy [mailto:kaber@trash.net]
Gesendet: Dienstag, 19. Februar 2008 15:03
An: Reither Robert
Cc: David Stevens; netdev@vger.kernel.org
Betreff: Re: AW: AW: Problem receiving multicast with kernel.2.6.24 #3

> 
> OK, found one solution (but i think, this should not the normal way)
> 
> MC joining first the VLAN device (eth0.3) and than the physical (eth0) does the trick, 100% success ...
> 
> Doing it the opposite way, gives the same problem as before ...
> 
> So somehow joining the VLAN interface disrupts the join to the physical one ...
> 
> Ideas ?

Does this patch help?

^ permalink raw reply

* Re: AW: AW: AW: Problem receiving multicast with kernel.2.6.24, REAL solution
From: Patrick McHardy @ 2008-02-19 14:56 UTC (permalink / raw)
  To: Reither Robert; +Cc: netdev
In-Reply-To: <BA41E8A07B73A041948C2E3A4EF6239D062F1B@av-digital-exch.avdigital.at>

Reither Robert wrote:
> Visit AVD on prolight+sound in Frankfurt from 12.-15. March 2008 - Hall 8.0, Stand G16
> ________________________________________________________________________
> 
> 
> 
> Hurray, your patch does the trick for me !!!
> 
> Now join/leave und receiving of multicasts work like it should !!
> 
> May i kiss you Patrick, this saves my week ;-)

The credit goes to Jorge Boncompte, I only broke it :|

> Hope we see a minor kernel release with this patch very soon ...

Yes, I'll send it to -stable as soon as it hits upstream.


^ permalink raw reply

* [NET]: Messed multicast lists after dev_mc_sync/unsync
From: Patrick McHardy @ 2008-02-19 14:56 UTC (permalink / raw)
  To: David S. Miller; +Cc: Linux Netdev List

[-- Attachment #1: Type: text/plain, Size: 0 bytes --]



[-- Attachment #2: x --]
[-- Type: text/plain, Size: 1058 bytes --]

commit 6548b91f39381b2c5f02f99c14734546354bff89
Author: Jorge Boncompte [DTI2] <jorge@dti2.net>
Date:   Mon Feb 18 16:02:01 2008 +0100

    [NET]: Messed multicast lists after dev_mc_sync/unsync
    
    Commit a0a400d79e3dd7843e7e81baa3ef2957bdc292d0 from you
    introduced a new field "da_synced" to struct dev_addr_list that is
    not properly initialized to 0. So when any of the current users (8021q,
    macvlan, mac80211) calls dev_mc_sync/unsync they mess the address
    list for both devices.
    
    The attached patch fixed it for me and avoid future problems.
    
    Signed-off-by: Jorge Boncompte [DTI2] <jorge@dti2.net>
    Signed-off-by: Patrick McHardy <kaber@trash.net>

diff --git a/net/core/dev.c b/net/core/dev.c
index 6cfc123..9516105 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -2900,7 +2900,7 @@ int __dev_addr_add(struct dev_addr_list **list, int *count,
 		}
 	}
 
-	da = kmalloc(sizeof(*da), GFP_ATOMIC);
+	da = kzalloc(sizeof(*da), GFP_ATOMIC);
 	if (da == NULL)
 		return -ENOMEM;
 	memcpy(da->da_addr, addr, alen);

^ permalink raw reply related

* Re: [PATCH 6/17 net-2.6.26] [NETNS]: Default arp parameters lookup.
From: Daniel Lezcano @ 2008-02-19 15:16 UTC (permalink / raw)
  To: Daniel Lezcano; +Cc: Denis V. Lunev, davem, netdev, containers, devel
In-Reply-To: <47BAADEB.3080005@fr.ibm.com>

Daniel Lezcano wrote:
> Denis V. Lunev wrote:
>> On Tue, 2008-02-19 at 10:51 +0100, Daniel Lezcano wrote:
>>> Denis V. Lunev wrote:
>>>> On Tue, 2008-02-19 at 10:14 +0100, Daniel Lezcano wrote:
>>>>> Denis V. Lunev wrote:
>>>>>> Default ARP parameters should be findable regardless of the context.
>>>>>> Required to make inetdev_event working.
>>>>>>
>>>>>> Signed-off-by: Denis V. Lunev <den@openvz.org>
>>>>>> ---
>>>>>>  net/core/neighbour.c |    4 +---
>>>>>>  1 files changed, 1 insertions(+), 3 deletions(-)
>>>>>>
>>>>>> diff --git a/net/core/neighbour.c b/net/core/neighbour.c
>>>>>> index c895ad4..45ed620 100644
>>>>>> --- a/net/core/neighbour.c
>>>>>> +++ b/net/core/neighbour.c
>>>>>> @@ -1275,9 +1275,7 @@ static inline struct neigh_parms 
>>>>>> *lookup_neigh_params(struct neigh_table *tbl,
>>>>>>      struct neigh_parms *p;
>>>>>>
>>>>>>      for (p = &tbl->parms; p; p = p->next) {
>>>>>> -        if (p->net != net)
>>>>>> -            continue;
>>>>>> -        if ((p->dev && p->dev->ifindex == ifindex) ||
>>>>>> +        if ((p->dev && p->dev->ifindex == ifindex && p->net == 
>>>>>> net) ||
>>>>>>              (!p->dev && !ifindex))
>>>>>>              return p;
>>>>>>      }
>>>>> If the values are:
>>>>>     p->dev == NULL
>>>>>     ifindex == 0
>>>>>     p->net != net
>>>>>
>>>>> The parms should not be taken into account and the looping must 
>>>>> continue. But with this modification it is not the case, if we 
>>>>> specify parms ifindex == 0, the first parms with the dev field set 
>>>>> to NULL will be taken belonging or not to the right net.
>>>> They should be taken. In the other case inetdev_event will fail for 
>>>> sure
>>>> in the middle. You could check.
>>>>
>>>> These are ARP defaults and I do not see a problem for now to get them.
>>> Because there is a parms default per namespace. So several instances 
>>> of them per nd table. That was the initial approach with Eric's 
>>> patchset.
>>>
>>
>> These changes are not in mainstream and I do not want to touch ARP as
>> this is not a simple thing. In reality ARP will be needed only when
>> we'll have a real device inside a namespace.
>>
>> Right now I prefer to have minimal set of working changes to finish IP
>> and upper layers.
> 
> core/neighbour.c is a common part between several protocols, especially 
> ipv4 and ipv6. If you modify this function just to fit your need in the 
> arp that will block me for ipv6 until you make parms default per 
> namespace. So please, find another way to do that, perhaps just add a 
> helper function.
> 
> I suggest you do parms default per namespace first, it is quite small 
> and easy :)
> 
> Just let me the time to send the copy-parms-default function.

Ok, so after a long discussion with Denis about this patch, I will 
change the ipv6 code to share the neigh->parms. It is not a problem.
Having the behavior of the neighbour subsystem per namespace is not a 
must-have.

Acked-by: Daniel Lezcano <dlezcano@fr.ibm.com>

^ permalink raw reply

* Re: [PATCH] [NETLABEL] Minor cleanup: remove unused method definition
From: Paul Moore @ 2008-02-19 16:08 UTC (permalink / raw)
  To: Rami Rosen; +Cc: David Miller, netdev, casey
In-Reply-To: <eb3ff54b0802190625v5315aac9m7ffd580b30bce8cd@mail.gmail.com>

On Tuesday 19 February 2008 9:25:31 am Rami Rosen wrote:
> Hi,
>
> This patch removes definition of netlbl_cfg_cipsov4_del() method in
> netlabel/netlabel_kapi.c and in include/net/netlabel.h as it is not
> used.
>
>
> Regards,
> Rami Rosen
>
>
> Signed-off-by: Rami Rosen <ramirose@gmail.com>

This was added for use by Smack (and any other LSMs which want to 
configure NetLabel directly) and since this is an area that is 
undergoing a lot of churn at this point I'd prefer if this function was 
left in place for the time being.

At a later date if this function is still unused, I'll gladly ack it's 
removal or do so myself.

-- 
paul moore
linux security @ hp

^ permalink raw reply

* Re: TG3 network data corruption regression 2.6.24/2.6.23.4
From: Tony Battersby @ 2008-02-19 16:16 UTC (permalink / raw)
  To: Michael Chan; +Cc: David Miller, herbert, netdev, gregkh, linux-kernel
In-Reply-To: <1203383046.13495.87.camel@dell>

Michael Chan wrote:
> On Mon, 2008-02-18 at 16:35 -0800, David Miller wrote:
>
>   
>> One consequence of Herbert's change is that the chip will see a
>> different datastream.  The initial skb->data linear area will be
>> smaller, and the transition to the fragmented area of pages will be
>> quicker.
>>
>>     
>
> I see.  Perhaps when we get to the end of the data-stream, there is a
> tiny frag that the chip cannot handle.  That's the only thing I can
> think of.
>
> Please try this patch to see if the problem goes away.  This will
> disable SG on 5701 so we always get linear SKBs.
>
> diff --git a/drivers/net/tg3.c b/drivers/net/tg3.c
> index db606b6..bb37e76 100644
> --- a/drivers/net/tg3.c
> +++ b/drivers/net/tg3.c
> @@ -12717,6 +12717,9 @@ static int __devinit tg3_init_one(struct pci_dev *pdev,
>  	} else
>  		tp->tg3_flags &= ~TG3_FLAG_RX_CHECKSUMS;
>  
> +	if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)
> +		dev->features &= ~(NETIF_F_IP_CSUM | NETIF_F_SG);
> +
>  	/* flow control autonegotiation is default behavior */
>  	tp->tg3_flags |= TG3_FLAG_PAUSE_AUTONEG;
>  	tp->link_config.flowctrl = TG3_FLOW_CTRL_TX | TG3_FLOW_CTRL_RX;
>
>
>
>   
This patch does appear to fix the data corruption (tested with
2.6.24.2).  However, it results in performance problems with the iSCSI
application that I am trying to run on this machine.

The test program that I described in the previous message still gets
good performance in both directions.  "iperf -r" gets good performance
in both directions (940 Mbits/s or 117 MB/s).  However, my target-mode
iSCSI application (which obviously generates rx/tx traffic patterns more
complicated than the synthetic tests) gets very poor performance in one
direction but good performance in the other direction.  iSCSI
performance drops to 6 - 15 MB/s when the 3Com NIC is doing heavy rx
with light tx, but remains at a decent 115 MB/s when the 3Com NIC is
doing heavy tx with light rx.  When I revert Herbert's patch instead of
applying the patch above, I get 115 MB/s in both cases.  (With a stock
unpatched kernel, the test fails almost immediately because the iSCSI
control PDUs are corrupted, causing the TCP connection to be dropped.)

The SysKonnect NIC that does not exhibit this problem has a chip that
says "BCM5411KQM" "TT0128 P2Q" and "56975E".

Tony


^ permalink raw reply

* [net-2.6][DRIVER][VETH] fix dev refcount race
From: Daniel Lezcano @ 2008-02-19 16:18 UTC (permalink / raw)
  To: David Miller; +Cc: Pavel Emelianov, Linux Netdev List

[-- Attachment #1: Type: text/plain, Size: 1 bytes --]



[-- Attachment #2: veth-delete-fix.patch --]
[-- Type: text/x-patch, Size: 2830 bytes --]

Subject: veth fix dev refcount race
From: Daniel Lezcano <dlezcano@fr.ibm.com>

When deleting the veth driver, veth_close calls netif_carrier_off
for the two extremities of the network device. netif_carrier_off on
the peer device will fire an event and hold a reference on the peer
device. Just after, the peer is unregistered taking the rtnl_lock while
the linkwatch_event is scheduled. If __linkwatch_run_queue does not
occurs before the unregistering, unregister_netdevice will wait for
the dev refcount to reach zero holding the rtnl_lock and linkwatch_event
will wait for the rtnl_lock and hold the dev refcount.

Signed-off-by: Daniel Lezcano <dlezcano@fr.ibm.com>
---
 drivers/net/veth.c |   53 ++++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 40 insertions(+), 13 deletions(-)

Index: net-2.6/drivers/net/veth.c
===================================================================
--- net-2.6.orig/drivers/net/veth.c
+++ net-2.6/drivers/net/veth.c
@@ -244,18 +244,6 @@ static int veth_open(struct net_device *
 	return 0;
 }
 
-static int veth_close(struct net_device *dev)
-{
-	struct veth_priv *priv;
-
-	if (netif_carrier_ok(dev)) {
-		priv = netdev_priv(dev);
-		netif_carrier_off(dev);
-		netif_carrier_off(priv->peer);
-	}
-	return 0;
-}
-
 static int veth_dev_init(struct net_device *dev)
 {
 	struct veth_net_stats *stats;
@@ -286,13 +274,50 @@ static void veth_setup(struct net_device
 	dev->hard_start_xmit = veth_xmit;
 	dev->get_stats = veth_get_stats;
 	dev->open = veth_open;
-	dev->stop = veth_close;
 	dev->ethtool_ops = &veth_ethtool_ops;
 	dev->features |= NETIF_F_LLTX;
 	dev->init = veth_dev_init;
 	dev->destructor = veth_dev_free;
 }
 
+static void veth_change_state(struct net_device *dev)
+{
+	struct net_device *peer;
+	struct veth_priv *priv;
+
+	priv = netdev_priv(dev);
+	peer = priv->peer;
+
+	if (netif_carrier_ok(peer)) {
+		if (!netif_carrier_ok(dev))
+			netif_carrier_on(dev);
+	} else {
+		if (netif_carrier_ok(dev))
+			netif_carrier_off(dev);
+	}
+}
+
+static int veth_device_event(struct notifier_block *unused,
+			     unsigned long event, void *ptr)
+{
+	struct net_device *dev = ptr;
+
+	if (dev->open != veth_open)
+		goto out;
+
+	switch (event) {
+	case NETDEV_CHANGE:
+		veth_change_state(dev);
+		break;
+	}
+out:
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block veth_notifier_block __read_mostly = {
+	.notifier_call	= veth_device_event,
+};
+
 /*
  * netlink interface
  */
@@ -454,12 +479,14 @@ static struct rtnl_link_ops veth_link_op
 
 static __init int veth_init(void)
 {
+	register_netdevice_notifier(&veth_notifier_block);
 	return rtnl_link_register(&veth_link_ops);
 }
 
 static __exit void veth_exit(void)
 {
 	rtnl_link_unregister(&veth_link_ops);
+	unregister_netdevice_notifier(&veth_notifier_block);
 }
 
 module_init(veth_init);

^ permalink raw reply

* Re: [net-2.6][DRIVER][VETH] fix dev refcount race
From: Pavel Emelyanov @ 2008-02-19 16:30 UTC (permalink / raw)
  To: Daniel Lezcano; +Cc: David Miller, Linux Netdev List
In-Reply-To: <47BB0138.1000104@fr.ibm.com>

Daniel Lezcano wrote:
> Subject: veth fix dev refcount race
> From: Daniel Lezcano <dlezcano@fr.ibm.com>
> 
> When deleting the veth driver, veth_close calls netif_carrier_off
> for the two extremities of the network device. netif_carrier_off on
> the peer device will fire an event and hold a reference on the peer
> device. Just after, the peer is unregistered taking the rtnl_lock while
> the linkwatch_event is scheduled. If __linkwatch_run_queue does not
> occurs before the unregistering, unregister_netdevice will wait for
> the dev refcount to reach zero holding the rtnl_lock and linkwatch_event

Brr... AFAIS the unregister process waits for refcount without
the rtnl_lock. The run-todo was invented for this. No?

> will wait for the rtnl_lock and hold the dev refcount.
> 
> Signed-off-by: Daniel Lezcano <dlezcano@fr.ibm.com>
> ---
>  drivers/net/veth.c |   53 ++++++++++++++++++++++++++++++++++++++++-------------
>  1 file changed, 40 insertions(+), 13 deletions(-)

^ permalink raw reply

* [PATCH][SCTP]: Pick up an orphaned sctp_sockets_allocated counter.
From: Pavel Emelyanov @ 2008-02-19 16:36 UTC (permalink / raw)
  To: David Miller, Vlad Yasevich; +Cc: Linux Netdev List, lksctp-developers

This counter is currently write-only.

Drawing an analogy with the similar tcp counter, I think
that this one should be pointed by the sockets_allocated
members of sctp_prot and sctpv6_prot.

Or should it be instead removed at all?

Signed-off-by: Pavel Emelyanov <xemul@openvz.org>

---

diff --git a/net/sctp/socket.c b/net/sctp/socket.c
index d47d578..44797ad 100644
--- a/net/sctp/socket.c
+++ b/net/sctp/socket.c
@@ -6488,6 +6488,7 @@ struct proto sctp_prot = {
 	.memory_pressure = &sctp_memory_pressure,
 	.enter_memory_pressure = sctp_enter_memory_pressure,
 	.memory_allocated = &sctp_memory_allocated,
+	.sockets_allocated = &sctp_sockets_allocated,
 	REF_PROTO_INUSE(sctp)
 };
 
@@ -6521,6 +6522,7 @@ struct proto sctpv6_prot = {
 	.memory_pressure = &sctp_memory_pressure,
 	.enter_memory_pressure = sctp_enter_memory_pressure,
 	.memory_allocated = &sctp_memory_allocated,
+	.sockets_allocated = &sctp_sockets_allocated,
 	REF_PROTO_INUSE(sctpv6)
 };
 #endif /* defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) */

^ permalink raw reply related

* Re: [Lksctp-developers] [PATCH][SCTP]: Pick up an orphaned sctp_sockets_allocated counter.
From: Vlad Yasevich @ 2008-02-19 16:43 UTC (permalink / raw)
  To: Pavel Emelyanov; +Cc: David Miller, Linux Netdev List, lksctp-developers
In-Reply-To: <47BB05AA.6090101@openvz.org>

Pavel Emelyanov wrote:
> This counter is currently write-only.
> 
> Drawing an analogy with the similar tcp counter, I think
> that this one should be pointed by the sockets_allocated
> members of sctp_prot and sctpv6_prot.
> 
> Or should it be instead removed at all?
> 
> Signed-off-by: Pavel Emelyanov <xemul@openvz.org>

Ack.  Looks like it got missed.  It should be added.

-vlad

> 
> ---
> 
> diff --git a/net/sctp/socket.c b/net/sctp/socket.c
> index d47d578..44797ad 100644
> --- a/net/sctp/socket.c
> +++ b/net/sctp/socket.c
> @@ -6488,6 +6488,7 @@ struct proto sctp_prot = {
>  	.memory_pressure = &sctp_memory_pressure,
>  	.enter_memory_pressure = sctp_enter_memory_pressure,
>  	.memory_allocated = &sctp_memory_allocated,
> +	.sockets_allocated = &sctp_sockets_allocated,
>  	REF_PROTO_INUSE(sctp)
>  };
>  
> @@ -6521,6 +6522,7 @@ struct proto sctpv6_prot = {
>  	.memory_pressure = &sctp_memory_pressure,
>  	.enter_memory_pressure = sctp_enter_memory_pressure,
>  	.memory_allocated = &sctp_memory_allocated,
> +	.sockets_allocated = &sctp_sockets_allocated,
>  	REF_PROTO_INUSE(sctpv6)
>  };
>  #endif /* defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) */
> 
> -------------------------------------------------------------------------
> This SF.net email is sponsored by: Microsoft
> Defy all challenges. Microsoft(R) Visual Studio 2008.
> http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/
> _______________________________________________
> Lksctp-developers mailing list
> Lksctp-developers@lists.sourceforge.net
> https://lists.sourceforge.net/lists/listinfo/lksctp-developers
> 


^ permalink raw reply

* Re: e1000: Detected Tx Unit Hang
From: Kok, Auke @ 2008-02-19 16:47 UTC (permalink / raw)
  To: Bernd Schubert; +Cc: netdev
In-Reply-To: <200802160126.43069.bernd-schubert@gmx.de>

Bernd Schubert wrote:
> On Saturday 16 February 2008, Kok, Auke wrote:
>> Bernd Schubert wrote:
>>> Hello,
>>>
>>> I can't login to one of our servers and just got this in an ipmi sol
>>> session:
>>>
>>> [18169.209181] e1000: eth0: e1000_clean_tx_irq: Detected Tx Unit Hang
>>> [18169.209183]   Tx Queue             <0>
>>> [18169.209184]   TDH                  <e3>
>>> [18169.209185]   TDT                  <e3>
>>> [18169.209186]   next_to_use          <e3>
>>> [18169.209187]   next_to_clean        <bd>
>>> [18169.209188] buffer_info[next_to_clean]
>>> [18169.209189]   time_stamp           <10043e4d2>
>>> [18169.209190]   next_to_watch        <be>
>>> [18169.209191]   jiffies              <10043e6f6>
>>> [18169.209192]   next_to_watch.status <1>
>>> [18169.256978] e1000: eth2: e1000_clean_tx_irq: Detected Tx Unit Hang
>>> [18169.256979]   Tx Queue             <0>
>>> [18169.256980]   TDH                  <de>
>>> [18169.256982]   TDT                  <de>
>>> [18169.256983]   next_to_use          <de>
>>> [18169.256984]   next_to_clean        <bc>
>>> [18169.256985] buffer_info[next_to_clean]
>>> [18169.256986]   time_stamp           <10043e511>
>>> [18169.256987]   next_to_watch        <bd>
>>> [18169.256988]   jiffies              <10043e701>
>>> [18169.256989]   next_to_watch.status <1>
>>>
>>> This is with 2.6.22.18. Is there any chance to recover the system? For
>>> some reasons I would prefer not to reboot now.
>> if that's all you have then it was false alarm. there should be a 'netdev
>> timeout - link reset' following those messages. can you send some more
>> context on those messages?
> 
> All I presently know is that there are 20 servers and login doesn't work any 
> more - sysrq+t does show me it hangs in fuse, which is accessing the 
> underlying nfs (we are using unionfs-fuse). While I checked the sysrq-t 
> output suddenly these e1000 messages appeared.
> Thinking a bit about it, it either could be 2.6.22.18 has an e1000 bug, which 
> 2.6.22.X didn't have (X=16, I think, but I'm not sure) or someone  
> mis-configured the switch/network environment today. 
> Hmm, now that I think about the last part, there already had been other 
> networking problems today, which were supposed to be fixed several hours ago. 
> Seems they didn't fix it properly.
> 
>> in real tx hang cases, the hardware is reset within 2 seconds, and
>> everything continues as normal.
> 
> Thanks, this gives me hope I don't need to reboot the serves (reboot would 
> mean I would need to start 60 md-raid rebuilds...).

my first thought after I read this e-mail is that the tx-hang message is just a
symptom of your system not responding or being spinlocked all the time. These TX
hang issues normally completely do not interfere with normal system operation and
unless you have continuous TX resets you would be able to logon perfectly fine.

I think you might have hit another kernel bug here... perhaps even unionfs/fuse
related and that certainly looks plausible from your problem description.

looking at the changelog for 2.6.22.16->2.6.22.18 I can't see anything relevant
(see
http://git.kernel.org/?p=linux/kernel/git/stable/linux-2.6.22.y.git;a=shortlog),
but there are definately no e1000 driver changes in that range anyway.

I don't suppose you can do a git-bisect? that would certainly help. I don't think
we can rule out anything just yet here.

At least try to revert some of your systems to the previous kernel version and see
if the problem goes away...

Auke

^ permalink raw reply

* Re: [2.6.25-rc2] e100: Trying to free already-free IRQ 11 during suspend ...
From: Kok, Auke @ 2008-02-19 17:18 UTC (permalink / raw)
  To: Andrew Morton; +Cc: e1000-devel, netdev, Andrey Borzenkov, linux-kernel
In-Reply-To: <20080218050113.1b4b0897.akpm@linux-foundation.org>

Andrew Morton wrote:
> On Sun, 17 Feb 2008 15:36:50 +0300 Andrey Borzenkov <arvidjaar@mail.ru> wrote:
> 
>> ... and possibly reboot/poweroff (it flows by too fast to be legible).
>>
>> [ 8803.850634] ACPI: Preparing to enter system sleep state S3
>> [ 8803.853141] Suspending console(s)
>> [ 8805.287505] serial 00:09: disabled
>> [ 8805.291564] Trying to free already-free IRQ 11
>> [ 8805.291579] Pid: 6920, comm: pm-suspend Not tainted 2.6.25-rc2-1avb #2
>> [ 8805.291628]  [<c0152127>] free_irq+0xb7/0x130
>> [ 8805.291675]  [<c024bd80>] e100_suspend+0xc0/0x100
>> [ 8805.291724]  [<c01eaa36>] pci_device_suspend+0x26/0x70
>> [ 8805.291747]  [<c0243674>] suspend_device+0x94/0xd0
>> [ 8805.291763]  [<c02439a3>] device_suspend+0x153/0x240
>> [ 8805.291784]  [<c014314f>] suspend_devices_and_enter+0x4f/0xf0
>> [ 8805.291808]  [<c0143a5f>] ? freeze_processes+0x3f/0x80
>> [ 8805.291825]  [<c01432fa>] enter_state+0xaa/0x140
>> [ 8805.291840]  [<c014341f>] state_store+0x8f/0xd0
>> [ 8805.291852]  [<c0143390>] ? state_store+0x0/0xd0
>> [ 8805.291866]  [<c01d3404>] kobj_attr_store+0x24/0x30
>> [ 8805.291901]  [<c01b547b>] sysfs_write_file+0xbb/0x110
>> [ 8805.291936]  [<c0177d79>] vfs_write+0x99/0x130
>> [ 8805.291963]  [<c01b53c0>] ? sysfs_write_file+0x0/0x110
>> [ 8805.291979]  [<c01782fd>] sys_write+0x3d/0x70
>> [ 8805.291998]  [<c010409a>] sysenter_past_esp+0x5f/0xa5
>> [ 8805.292038]  =======================
>> [ 8805.347640] ACPI: PCI interrupt for device 0000:00:06.0 disabled
>> [ 8805.361128] ACPI: PCI interrupt for device 0000:00:02.0 disabled
>> [ 8805.376670]  hwsleep-0322 [00] enter_sleep_state     : Entering sleep state [S3]
>> [ 8805.376670] Back to C!
>>
>> Interface is unused normally (only for netconsole sometimes). dmesg and config
>> attached.
> 
> Does reverting this:
> 
> commit 8543da6672b0994921f014f2250e27ae81645580
> Author: Auke Kok <auke-jan.h.kok@intel.com>
> Date:   Wed Dec 12 16:30:42 2007 -0800
> 
>     e100: free IRQ to remove warningwhenrebooting
>     
> with this patch:
> 
> --- a/drivers/net/e100.c~revert-1
> +++ a/drivers/net/e100.c
> @@ -2804,9 +2804,8 @@ static int e100_suspend(struct pci_dev *
>  		pci_enable_wake(pdev, PCI_D3cold, 0);
>  	}
>  
> -	free_irq(pdev->irq, netdev);
> -
>  	pci_disable_device(pdev);
> +	free_irq(pdev->irq, netdev);
>  	pci_set_power_state(pdev, PCI_D3hot);
>  
>  	return 0;
> @@ -2848,8 +2847,6 @@ static void e100_shutdown(struct pci_dev
>  		pci_enable_wake(pdev, PCI_D3cold, 0);
>  	}
>  
> -	free_irq(pdev->irq, netdev);
> -
>  	pci_disable_device(pdev);
>  	pci_set_power_state(pdev, PCI_D3hot);
>  }
> _
> 
> fix it?
> 
>> Hmm ... after resume device has disappeared at all ...
>>
>> {pts/1}% cat /proc/interrupts
>>            CPU0
>>   0:    1290492    XT-PIC-XT        timer
>>   1:       6675    XT-PIC-XT        i8042
>>   2:          0    XT-PIC-XT        cascade
>>   3:          2    XT-PIC-XT
>>   4:          2    XT-PIC-XT
>>   5:          3    XT-PIC-XT
>>   7:          4    XT-PIC-XT        irda0
>>   8:          0    XT-PIC-XT        rtc0
>>   9:        583    XT-PIC-XT        acpi
>>  10:          2    XT-PIC-XT
>>  11:      31483    XT-PIC-XT        yenta, yenta, yenta, ohci_hcd:usb1, ALI 5451, pcmcia0.0
>>  12:      28070    XT-PIC-XT        i8042
>>  14:      21705    XT-PIC-XT        ide0
>>  15:      82123    XT-PIC-XT        ide1
>> NMI:          0   Non-maskable interrupts
>> TRM:          0   Thermal event interrupts
>> SPU:          0   Spurious interrupts
>> ERR:          0
> 
> I hope that's not a separate bug...

I'll take a look at this as well. thanks for reporting.

Auke

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft
Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

^ permalink raw reply

* [PATCH] [2.6.25-rc2] igb: Fix panic with NICs with 1000BASE-SX PHY
From: Andy Gospodarek @ 2008-02-19 17:46 UTC (permalink / raw)
  To: auke-jan.h.kok, jeff, davem; +Cc: netdev

From: Bill Hayes <bill.hayes@hp.com>

Auke, Jeff and David,

This patch eliminates a kernel panic with the igb driver in 2.6.25-rc2 when 
running on a Intel 82575 Ethernet controller with a 1000BASE-SX PHY.  The 
panic does not happen with the 1000BASE-T PHY, only with a SX connection.  

Signed-off-by: Bill Hayes <bill.hayes@hp.com>
Signed-off-by: Andy Gospodarek <andy@greyhouse.net>

 drivers/net/igb/igb_main.c |    6 ++++--
 1 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/drivers/net/igb/igb_main.c b/drivers/net/igb/igb_main.c
index d4eb8e2..666c8f0 100644
--- a/drivers/net/igb/igb_main.c
+++ b/drivers/net/igb/igb_main.c
@@ -843,7 +843,8 @@ void igb_reset(struct igb_adapter *adapter)
 	wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
 
 	igb_reset_adaptive(&adapter->hw);
-	adapter->hw.phy.ops.get_phy_info(&adapter->hw);
+	if (adapter->hw.phy.ops.get_phy_info)
+		adapter->hw.phy.ops.get_phy_info(&adapter->hw);
 	igb_release_manageability(adapter);
 }
 
@@ -2083,7 +2084,8 @@ static void igb_set_multi(struct net_device *netdev)
 static void igb_update_phy_info(unsigned long data)
 {
 	struct igb_adapter *adapter = (struct igb_adapter *) data;
-	adapter->hw.phy.ops.get_phy_info(&adapter->hw);
+	if (adapter->hw.phy.ops.get_phy_info)
+		adapter->hw.phy.ops.get_phy_info(&adapter->hw);
 }
 
 /**




^ permalink raw reply related

* Re: [2.6.25-rc2, 2.6.24-rc8] page allocation failure...
From: Kok, Auke @ 2008-02-19 17:37 UTC (permalink / raw)
  To: Andrew Morton; +Cc: Daniel J Blueman, Linux Kernel, netdev
In-Reply-To: <20080218045849.59311851.akpm@linux-foundation.org>

Andrew Morton wrote:
> On Sun, 17 Feb 2008 13:20:59 +0000 "Daniel J Blueman" <daniel.blueman@gmail.com> wrote:
> 
>> I'm still hitting this with e1000e on 2.6.25-rc2, 10 times again.

are you sure? I don't think that's the case and you're seeing e1000 dumps here...

>> It's clearly non-fatal, but then do we expect it to occur?
>>
>> Daniel
>>
>> --- [dmesg]
>>
>> [ 1250.822786] swapper: page allocation failure. order:3, mode:0x4020
>> [ 1250.822786] Pid: 0, comm: swapper Not tainted 2.6.25-rc2-119 #2
>> [ 1250.822786]
>> [ 1250.822786] Call Trace:
>> [ 1250.822786]  <IRQ>  [<ffffffff8025fe9e>] __alloc_pages+0x34e/0x3a0
>> [ 1250.822786]  [<ffffffff8048c6df>] ? __netdev_alloc_skb+0x1f/0x40
>> [ 1250.822786]  [<ffffffff8027acc2>] __slab_alloc+0x102/0x3d0
>> [ 1250.822786]  [<ffffffff8048c6df>] ? __netdev_alloc_skb+0x1f/0x40
>> [ 1250.822786]  [<ffffffff8027b8cb>] __kmalloc_track_caller+0x7b/0xc0
>> [ 1250.822786]  [<ffffffff8048b74f>] __alloc_skb+0x6f/0x160
>> [ 1250.822786]  [<ffffffff8048c6df>] __netdev_alloc_skb+0x1f/0x40
>> [ 1250.822786]  [<ffffffff8042652d>] e1000_alloc_rx_buffers+0x1ed/0x260
>> [ 1250.822786]  [<ffffffff80426b5a>] e1000_clean_rx_irq+0x22a/0x330
>> [ 1250.822786]  [<ffffffff80422981>] e1000_clean+0x1e1/0x540
>> [ 1250.822786]  [<ffffffff8024b7a5>] ? tick_program_event+0x45/0x70
>> [ 1250.822786]  [<ffffffff804930ba>] net_rx_action+0x9a/0x150
>> [ 1250.822786]  [<ffffffff802336b4>] __do_softirq+0x74/0xf0
>> [ 1250.822786]  [<ffffffff8020c5fc>] call_softirq+0x1c/0x30
>> [ 1250.822786]  [<ffffffff8020eaad>] do_softirq+0x3d/0x80
>> [ 1250.822786]  [<ffffffff80233635>] irq_exit+0x85/0x90
>> [ 1250.822786]  [<ffffffff8020eba5>] do_IRQ+0x85/0x100
>> [ 1250.822786]  [<ffffffff8020a5b0>] ? mwait_idle+0x0/0x50
>> [ 1250.822786]  [<ffffffff8020b981>] ret_from_intr+0x0/0xa
>> [ 1250.822786]  <EOI>  [<ffffffff8020a5f5>] ? mwait_idle+0x45/0x50
>> [ 1250.822786]  [<ffffffff80209a92>] ? enter_idle+0x22/0x30
>> [ 1250.822786]  [<ffffffff8020a534>] ? cpu_idle+0x74/0xa0
>> [ 1250.822786]  [<ffffffff80527825>] ? rest_init+0x55/0x60
> 
> They're regularly reported with e1000 too - I don't think aything really
> changed.
> 
> e1000 has this crazy problem where because of a cascade of follies (mainly
> borked hardware) it has to do a 32kb allocation for a 9kb(?) packet.  It
> would be sad if that was carried over into e1000e?

can't be, I personally removed that code.

for MTU > 1500 e1000e uses a plain normal sized SKB. for anything bigger e1000e
uses pages.

so I don't see how this bug could still be showing up for e1000e at all. The large
skb receive code is all gone (literally, removed).

*please* rmmod e1000; modprobe e1000e and show the dumps again so we know for sure
that we're not looking at e1000 dumps.

short fix: increase ring size for e1000 with `modprobe e1000 RxDescriptors=4096`
(or use ethtool) and `echo -n 8192 > /proc/sys/vm/min_free_kbytes` or something
like that.

what nic hardware is this on? lspci?

Auke

^ permalink raw reply

* Re: [PATCH 1/3] Support arbitrary initial TCP timestamps
From: Glenn Griffin @ 2008-02-19 18:02 UTC (permalink / raw)
  To: Glenn Griffin; +Cc: David Miller, netdev, linux-kernel
In-Reply-To: <20080218164723.GA13941@fin>

> > Adding yet another member to the already bloated tcp_sock structure to
> > implement this is too high a cost.
> 
> Yes, I was worried that would be deemed too high of a cost, but it was
> the most efficient way I could think to accomplish what I wanted.
> 
> > I would instead prefer that there be some global random number
> > calculated when the first TCP socket is created, and use that as a
> > global offset.  You can even recompute it every few hours if you
> > like.
> 
> That would work fine if my mine purpose was to randomize the tcp
> timestamp to mitigate the leak in information regarding uptime, but
> despite the brief description, that's only a side effect of what I
> intended to do.  What I wanted was a way to be able to choose an initial
> tcp timestamp for a particular connection that was not tied directly to
> jiffies.
> 
> The two patches following this show my intended use case.  I intend to
> enhance syncookie support to allow it to support advanced tcp options
> (sack and window scaling).  Normally syncookies encode the bare minimum
> state of a connection in the ISN they choose, but the 32bit ISN isn't
> enough to encode advanced tcp options so you are left with a working but
> crippled tcp stack during a synflood attack.  If in addition to choosing
> an ISN we are able to choose an initial tcp timestamp, we are then able
> to encode an additional 32 bits of information that can contain more of
> the advanced tcp options.

Perhaps I should clarify.  I don't see a way to implement the additional
syncookie enhancements without storing an offset in some type of
per-socket structure.  Given that, is it still deemed too high of a cost?
Is enhancing syncookies not deemed important enough to warrant the
additional 4 bytes?  What if there was an additional config variable to
support arbitrary/random tcp timestamps, and then syncookies only used
the additional options when that setting was chosen?  Is there some
possiblity I'm missing that could get this feature in a form suitable
for inclusion?  Thanks.

--Glenn


^ permalink raw reply

* [PATCH] igb: Fix panic with NICs with 1000BASE-SX PHY
From: Auke Kok @ 2008-02-19 18:24 UTC (permalink / raw)
  To: jeff; +Cc: netdev, e1000-devel, bill.hayes, andy

From: Bill Hayes <bill.hayes@hp.com>

This patch eliminates a kernel panic with the igb driver in 2.6.25-rc2 when 
running on a Intel 82575 Ethernet controller with a 1000BASE-SX PHY.  The 
panic does not happen with the 1000BASE-T PHY, only with a SX connection.  

Signed-off-by: Bill Hayes <bill.hayes@hp.com>
Signed-off-by: Andy Gospodarek <andy@greyhouse.net>
Signed-off-by: Auke Kok <auke-jan.h.kok@intel.com>
---

 drivers/net/igb/igb_main.c |    6 ++++--
 1 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/drivers/net/igb/igb_main.c b/drivers/net/igb/igb_main.c
index 3480cc7..6a1f230 100644
--- a/drivers/net/igb/igb_main.c
+++ b/drivers/net/igb/igb_main.c
@@ -816,7 +816,8 @@ void igb_reset(struct igb_adapter *adapter)
 	wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
 
 	igb_reset_adaptive(&adapter->hw);
-	adapter->hw.phy.ops.get_phy_info(&adapter->hw);
+	if (adapter->hw.phy.ops.get_phy_info)
+		adapter->hw.phy.ops.get_phy_info(&adapter->hw);
 }
 
 /**
@@ -2052,7 +2053,8 @@ static void igb_set_multi(struct net_device *netdev)
 static void igb_update_phy_info(unsigned long data)
 {
 	struct igb_adapter *adapter = (struct igb_adapter *) data;
-	adapter->hw.phy.ops.get_phy_info(&adapter->hw);
+	if (adapter->hw.phy.ops.get_phy_info)
+		adapter->hw.phy.ops.get_phy_info(&adapter->hw);
 }
 
 /**


^ permalink raw reply related

* Re: TG3 network data corruption regression 2.6.24/2.6.23.4
From: Michael Chan @ 2008-02-19 19:11 UTC (permalink / raw)
  To: Tony Battersby; +Cc: David Miller, herbert, netdev, gregkh, linux-kernel
In-Reply-To: <47BB00EC.3010607@cybernetics.com>

On Tue, 2008-02-19 at 11:16 -0500, Tony Battersby wrote:
> iSCSI
> performance drops to 6 - 15 MB/s when the 3Com NIC is doing heavy rx
> with light tx,

That's strange.  The patch should only affect TX performance slightly
since we are just turning off SG for TX.  Please take an ethereal trace
to see what's happening and compare with a good trace.

> but remains at a decent 115 MB/s when the 3Com NIC is
> doing heavy tx with light rx.  When I revert Herbert's patch instead
> of
> applying the patch above, I get 115 MB/s in both cases.  (With a stock
> unpatched kernel, the test fails almost immediately because the iSCSI
> control PDUs are corrupted, causing the TCP connection to be dropped.)
> 
> The SysKonnect NIC that does not exhibit this problem has a chip that
> says "BCM5411KQM" "TT0128 P2Q" and "56975E".
> 

I think this is the 5700, but please send me the tg3 output that
identifies the chip and the revision.  Something like this:

eth2: Tigon3 [partno(BCM95705) rev 3003 PHY(5705)] (PCI:66MHz:32-bit) 10/100/1000Base-T Ethernet 00:10:18:04:57:0d
eth2: RXcsums[1] LinkChgREG[0] MIirq[0] ASF[0] WireSpeed[0] TSOcap[1]






^ permalink raw reply

* Re: TG3 network data corruption regression 2.6.24/2.6.23.4
From: Tony Battersby @ 2008-02-19 19:26 UTC (permalink / raw)
  To: Michael Chan; +Cc: David Miller, herbert, netdev, gregkh, linux-kernel
In-Reply-To: <1203448265.13495.95.camel@dell>

Michael Chan wrote:
>> The SysKonnect NIC that does not exhibit this problem has a chip that
>> says "BCM5411KQM" "TT0128 P2Q" and "56975E".
> I think this is the 5700, but please send me the tg3 output that
> identifies the chip and the revision.  Something like this:
>
> eth2: Tigon3 [partno(BCM95705) rev 3003 PHY(5705)] (PCI:66MHz:32-bit) 10/100/1000Base-T Ethernet 00:10:18:04:57:0d
> eth2: RXcsums[1] LinkChgREG[0] MIirq[0] ASF[0] WireSpeed[0] TSOcap[1]
>   
Here is the dmesg output for the SysKonnect NIC:

eth0: Tigon3 [partno(SK-9D21) rev 7104 PHY(5411)] (PCI:66MHz:64-bit)
10/100/1000Base-T Ethernet 00:00:5a:9d:0c:4a
eth0: RXcsums[1] LinkChgREG[1] MIirq[1] ASF[0] WireSpeed[0] TSOcap[0]
eth0: dma_rwctrl[76ff000f] dma_mask[64-bit]

Tony


^ permalink raw reply

* [RESEND/patch 2.6.25-rc2-git] net/enc28j60: low power mode
From: David Brownell @ 2008-02-19 20:54 UTC (permalink / raw)
  To: netdev; +Cc: Claudio Lanconelli
In-Reply-To: <47AAE7CD.9050801@eptar.com>

Keep enc28j60 chips in low-power mode when they're not in use.
At typically 120 mA, these chips run hot even when idle; this
low power mode cuts that power usage by a factor of around 100.

This version provides a generic routine to poll a register until
its masked value equals some value ... e.g. bit set or cleared.
It's basically what the previous wait_phy_ready() did, but this
version is generalized to support the handshaking needed to
enter and exit low power mode.

Signed-off-by: David Brownell <dbrownell@users.sourceforge.net>
Signed-off-by: Claudio Lanconelli <lanconelli.claudio@eptar.com>
---
 drivers/net/enc28j60.c |   82 ++++++++++++++++++++++++++++++++++---------------
 1 files changed, 58 insertions(+), 24 deletions(-)

--- a/drivers/net/enc28j60.c
+++ b/drivers/net/enc28j60.c
@@ -400,26 +400,31 @@ enc28j60_packet_write(struct enc28j60_ne
 	mutex_unlock(&priv->lock);
 }
 
-/*
- * Wait until the PHY operation is complete.
- */
-static int wait_phy_ready(struct enc28j60_net *priv)
+static unsigned long msec20_to_jiffies;
+
+static int poll_ready(struct enc28j60_net *priv, u8 reg, u8 mask, u8 val)
 {
-	unsigned long timeout = jiffies + 20 * HZ / 1000;
-	int ret = 1;
+	unsigned long timeout = jiffies + msec20_to_jiffies;
 
 	/* 20 msec timeout read */
-	while (nolock_regb_read(priv, MISTAT) & MISTAT_BUSY) {
+	while ((nolock_regb_read(priv, reg) & mask) != val) {
 		if (time_after(jiffies, timeout)) {
 			if (netif_msg_drv(priv))
-				printk(KERN_DEBUG DRV_NAME
-					": PHY ready timeout!\n");
-			ret = 0;
-			break;
+				dev_dbg(&priv->spi->dev,
+					"reg %02x ready timeout!\n", reg);
+			return -ETIMEDOUT;
 		}
 		cpu_relax();
 	}
-	return ret;
+	return 0;
+}
+
+/*
+ * Wait until the PHY operation is complete.
+ */
+static int wait_phy_ready(struct enc28j60_net *priv)
+{
+	return poll_ready(priv, MISTAT, MISTAT_BUSY, 0) ? 0 : 1;
 }
 
 /*
@@ -594,6 +599,32 @@ static void nolock_txfifo_init(struct en
 	nolock_regw_write(priv, ETXNDL, end);
 }
 
+/*
+ * Low power mode shrinks power consumption about 100x, so we'd like
+ * the chip to be in that mode whenever it's inactive.  (However, we
+ * can't stay in lowpower mode during suspend with WOL active.)
+ */
+static void enc28j60_lowpower(struct enc28j60_net *priv, bool is_low)
+{
+	if (netif_msg_drv(priv))
+		dev_dbg(&priv->spi->dev, "%s power...\n",
+				is_low ? "low" : "high");
+
+	mutex_lock(&priv->lock);
+	if (is_low) {
+		nolock_reg_bfclr(priv, ECON1, ECON1_RXEN);
+		poll_ready(priv, ESTAT, ESTAT_RXBUSY, 0);
+		poll_ready(priv, ECON1, ECON1_TXRTS, 0);
+		/* ECON2_VRPS was set during initialization */
+		nolock_reg_bfset(priv, ECON2, ECON2_PWRSV);
+	} else {
+		nolock_reg_bfclr(priv, ECON2, ECON2_PWRSV);
+		poll_ready(priv, ESTAT, ESTAT_CLKRDY, ESTAT_CLKRDY);
+		/* caller sets ECON1_RXEN */
+	}
+	mutex_unlock(&priv->lock);
+}
+
 static int enc28j60_hw_init(struct enc28j60_net *priv)
 {
 	u8 reg;
@@ -612,8 +643,8 @@ static int enc28j60_hw_init(struct enc28
 	priv->tx_retry_count = 0;
 	priv->max_pk_counter = 0;
 	priv->rxfilter = RXFILTER_NORMAL;
-	/* enable address auto increment */
-	nolock_regb_write(priv, ECON2, ECON2_AUTOINC);
+	/* enable address auto increment and voltage regulator powersave */
+	nolock_regb_write(priv, ECON2, ECON2_AUTOINC | ECON2_VRPS);
 
 	nolock_rxfifo_init(priv, RXSTART_INIT, RXEND_INIT);
 	nolock_txfifo_init(priv, TXSTART_INIT, TXEND_INIT);
@@ -690,7 +721,7 @@ static int enc28j60_hw_init(struct enc28
 
 static void enc28j60_hw_enable(struct enc28j60_net *priv)
 {
-	/* enable interrutps */
+	/* enable interrupts */
 	if (netif_msg_hw(priv))
 		printk(KERN_DEBUG DRV_NAME ": %s() enabling interrupts.\n",
 			__FUNCTION__);
@@ -726,15 +757,12 @@ enc28j60_setlink(struct net_device *ndev
 	int ret = 0;
 
 	if (!priv->hw_enable) {
-		if (autoneg == AUTONEG_DISABLE && speed == SPEED_10) {
+		/* link is in low power mode now; duplex setting
+		 * will take effect on next enc28j60_hw_init().
+		 */
+		if (autoneg == AUTONEG_DISABLE && speed == SPEED_10)
 			priv->full_duplex = (duplex == DUPLEX_FULL);
-			if (!enc28j60_hw_init(priv)) {
-				if (netif_msg_drv(priv))
-					dev_err(&ndev->dev,
-						"hw_reset() failed\n");
-				ret = -EINVAL;
-			}
-		} else {
+		else {
 			if (netif_msg_link(priv))
 				dev_warn(&ndev->dev,
 					"unsupported link setting\n");
@@ -1307,7 +1335,8 @@ static int enc28j60_net_open(struct net_
 		}
 		return -EADDRNOTAVAIL;
 	}
-	/* Reset the hardware here */
+	/* Reset the hardware here (and take it out of low power mode) */
+	enc28j60_lowpower(priv, false);
 	enc28j60_hw_disable(priv);
 	if (!enc28j60_hw_init(priv)) {
 		if (netif_msg_ifup(priv))
@@ -1337,6 +1366,7 @@ static int enc28j60_net_close(struct net
 		printk(KERN_DEBUG DRV_NAME ": %s() enter\n", __FUNCTION__);
 
 	enc28j60_hw_disable(priv);
+	enc28j60_lowpower(priv, true);
 	netif_stop_queue(dev);
 
 	return 0;
@@ -1537,6 +1567,8 @@ static int __devinit enc28j60_probe(stru
 	dev->watchdog_timeo = TX_TIMEOUT;
 	SET_ETHTOOL_OPS(dev, &enc28j60_ethtool_ops);
 
+	enc28j60_lowpower(priv, true);
+
 	ret = register_netdev(dev);
 	if (ret) {
 		if (netif_msg_probe(priv))
@@ -1582,6 +1614,8 @@ static struct spi_driver enc28j60_driver
 
 static int __init enc28j60_init(void)
 {
+	msec20_to_jiffies = msecs_to_jiffies(20);
+
 	return spi_register_driver(&enc28j60_driver);
 }
 

^ permalink raw reply


This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox