2.6.19-rt14 e1000 shutdown problem

2.6.19-rt14 e1000 shutdown problem

[Tim Chen]

Ingo,

While trying out the 2.6.19.1-rt14 kernel with a 
x86_64 system with Clovertown processor, it hung when it 
was shutting down e1000 ethernet interface running the command:

/sbin/ip link set dev eth0 down

Same problem was also seen for 2.6.19.1-rt15.  The default 
kernel from your RPM was used. Sysreq was still working and 
call trace of the ip command using Sysreq-prtscr-t is listed below:

ip   D [ffff810139201040] ffff81013fe47840    0  3526 3433
(NOTLB)
 ffff81012ce75ba8 0000000000000086 0000000000000007 0000000000000297
 00000000000c4994 ffff810139201278 ffff810139201040 ffff81013fe47800
 0000005681bed269 0000000400000080 ffff81013fe47800 00000004802df143
Call Trace:
 [<ffffffff80265da0>] schedule+0xd8/0xfc
 [<ffffffff802a0bab>] flush_cpu_workqueue+0x72/0xa7
 [<ffffffff802a0e13>] flush_workqueue+0x59/0x81
 [<ffffffff802a0f79>] schedule_on_each_cpu+0xe6/0xfd
 [<ffffffff802e33ca>] filevec_add_drain_all+0x12/0x14
 [<ffffffff8030bb38>] remove_proc_entry+0xaf/0x25a
 [<ffffffff802c720d>] unregister_handler_proc+0x43/0x48
 [<ffffffff802c56c4>] free_irq+0xdd/0x117
 [<ffffffff88133101>] :e1000:e1000_free_irq+0x22/0x3b
 [<ffffffff88135c88>] :e1000:e1000_close+0x4d/0xb9
 [<ffffffff80429735>] dev_close+0x5b/0x7c
 [<ffffffff80429ab9>] dev_change_flags+0x64/0x139
 [<ffffffff8046048b>] devinet_ioctl+0x252/0x5e1
 [<ffffffff80460ac1>] inet_ioctl+0x71/0x8f
 [<ffffffff80422042>] sock_ioctl+0x1d7/0x1ff
 [<ffffffff802433b8>] do_ioctl+0x27/0x74
 [<ffffffff80231339>] vfs_ioctl+0x263/0x280
 [<ffffffff8024dfb7>] sys_ioctl+0x5f/0x82
 [<ffffffff8026042c>] tracesys+0x151/0x1c5
 [<0000003055ac6267>]   

Tim                      

Re: 2.6.19-rt14 e1000 shutdown problem

[Ingo Molnar]

* Tim Chen <tim.c.chen@linux.intel.com> wrote:

> Ingo,
> 
> While trying out the 2.6.19.1-rt14 kernel with a x86_64 system with 
> Clovertown processor, it hung when it was shutting down e1000 ethernet 
> interface running the command:
> 
> /sbin/ip link set dev eth0 down

does the patch below solve it for you?

	Ingo

Index: linux/fs/file_table.c
===================================================================
--- linux.orig/fs/file_table.c
+++ linux/fs/file_table.c
@@ -333,6 +333,22 @@ static void __filevec_add(struct filevec
 	filevec_reinit(fvec);
 }
 
+/*
+ * Flush files per-CPU workqueue:
+ */
+static struct workqueue_struct *flush_files_workqueue;
+
+int __init flush_files_init(void)
+{
+        flush_files_workqueue = create_workqueue("flush_filesd");
+        if (!flush_files_workqueue)
+                panic("Failed to create flush_filesd\n");
+
+	return 0;
+}
+
+__initcall(flush_files_init);
+
 static void filevec_add_drain(void)
 {
 	int cpu;
@@ -349,7 +365,8 @@ static void filevec_add_drain_per_cpu(st
 
 int filevec_add_drain_all(void)
 {
-	return schedule_on_each_cpu(filevec_add_drain_per_cpu);
+	return schedule_on_each_cpu_wq(flush_files_workqueue,
+				       filevec_add_drain_per_cpu);
 }
 EXPORT_SYMBOL_GPL(filevec_add_drain_all);
 
Index: linux/include/linux/workqueue.h
===================================================================
--- linux.orig/include/linux/workqueue.h
+++ linux/include/linux/workqueue.h
@@ -181,6 +181,7 @@ extern int FASTCALL(schedule_delayed_wor
 
 extern int schedule_delayed_work_on(int cpu, struct delayed_work *work, unsigned long delay);
 extern int schedule_on_each_cpu(work_func_t func);
+extern int schedule_on_each_cpu_wq(struct workqueue_struct *wq, work_func_t func);
 extern void flush_scheduled_work(void);
 extern int current_is_keventd(void);
 extern int keventd_up(void);
Index: linux/kernel/workqueue.c
===================================================================
--- linux.orig/kernel/workqueue.c
+++ linux/kernel/workqueue.c
@@ -700,6 +700,44 @@ int schedule_on_each_cpu(work_func_t fun
 	return 0;
 }
 
+/**
+ * schedule_on_each_cpu_wq - call a function on each online CPU on a per-CPU wq
+ * @func: the function to call
+ *
+ * Returns zero on success.
+ * Returns -ve errno on failure.
+ *
+ * Appears to be racy against CPU hotplug.
+ *
+ * schedule_on_each_cpu() is very slow.
+ */
+int schedule_on_each_cpu_wq(struct workqueue_struct *wq, work_func_t func)
+{
+	int cpu;
+	struct work_struct *works;
+
+	if (is_single_threaded(wq)) {
+		WARN_ON(1);
+		return -EINVAL;
+	}
+	works = alloc_percpu(struct work_struct);
+	if (!works)
+		return -ENOMEM;
+
+	for_each_online_cpu(cpu) {
+		struct work_struct *work = per_cpu_ptr(works, cpu);
+
+		INIT_WORK(work, func);
+		set_bit(WORK_STRUCT_PENDING, work_data_bits(work));
+		__queue_work(per_cpu_ptr(wq->cpu_wq, cpu), work);
+	}
+	flush_workqueue(wq);
+	free_percpu(works);
+
+	return 0;
+}
+
+
 void flush_scheduled_work(void)
 {
 	flush_workqueue(keventd_wq);

Re: 2.6.19-rt14 e1000 shutdown problem

[Tim Chen]

On Thu, 2006-12-21 at 21:13 +0100, Ingo Molnar wrote:
> * Tim Chen <tim.c.chen@linux.intel.com> wrote:
> 
> > Ingo,
> > 
> > While trying out the 2.6.19.1-rt14 kernel with a x86_64 system with 
> > Clovertown processor, it hung when it was shutting down e1000 ethernet 
> > interface running the command:
> > 
> > /sbin/ip link set dev eth0 down
> 
> does the patch below solve it for you?
> 
> 	Ingo
> 

Yes, the patch took care of the problem.  Thanks.

Tim