[PATCH 1/2]: softirq: Add support for triggering softirq work on softirqs.

[PATCH 1/2]: softirq: Add support for triggering softirq work on softirqs.

[David Miller]

softirq: Add support for triggering softirq work on softirqs.

This is basically a genericization of Jens Axboe's block layer
remote softirq changes.

Signed-off-by: David S. Miller <davem@davemloft.net>
---
 include/linux/interrupt.h |    6 ++-
 kernel/softirq.c          |  103 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 108 insertions(+), 1 deletions(-)

diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 58ff4e7..806b38f 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -11,6 +11,8 @@
 #include <linux/hardirq.h>
 #include <linux/sched.h>
 #include <linux/irqflags.h>
+#include <linux/smp.h>
+#include <linux/percpu.h>
 #include <asm/atomic.h>
 #include <asm/ptrace.h>
 #include <asm/system.h>
@@ -270,7 +272,9 @@ extern void softirq_init(void);
 #define __raise_softirq_irqoff(nr) do { or_softirq_pending(1UL << (nr)); } while (0)
 extern void raise_softirq_irqoff(unsigned int nr);
 extern void raise_softirq(unsigned int nr);
-
+DECLARE_PER_CPU(struct list_head, softirq_work_list[32]);
+extern void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq);
+extern void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq);
 
 /* Tasklets --- multithreaded analogue of BHs.
 
diff --git a/kernel/softirq.c b/kernel/softirq.c
index c506f26..01e733f 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -6,6 +6,8 @@
  *	Distribute under GPLv2.
  *
  *	Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
+ *
+ *	Remote softirq infrastructure is by Jens Axboe.
  */
 
 #include <linux/module.h>
@@ -463,17 +465,118 @@ void tasklet_kill(struct tasklet_struct *t)
 
 EXPORT_SYMBOL(tasklet_kill);
 
+DEFINE_PER_CPU(struct list_head, softirq_work_list[32]);
+
+static void __local_trigger(struct call_single_data *cp, int softirq)
+{
+	struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
+
+	list_add_tail(&cp->list, head);
+	if (head->next == &cp->list)
+		raise_softirq_irqoff(softirq);
+}
+
+#if defined(CONFIG_SMP) && defined(CONFIG_USE_GENERIC_SMP_HELPERS)
+static void remote_softirq_receive(void *data)
+{
+	struct call_single_data *cp = data;
+	unsigned long flags;
+	int softirq;
+
+	softirq = cp->flags >> 16;
+
+	local_irq_save(flags);
+	__local_trigger(cp, softirq);
+	local_irq_restore(flags);
+}
+
+static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
+{
+	if (cpu_online(cpu)) {
+		cp->func = remote_softirq_receive;
+		cp->info = cp;
+		cp->flags = softirq << 16;
+		__smp_call_function_single(cpu, cp);
+		return 0;
+	}
+	return 1;
+}
+#else /* CONFIG_SMP && CONFIG_USE_GENERIC_SMP_HELPERS */
+static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
+{
+	return 1;
+}
+#endif
+
+void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
+{
+	if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
+		__local_trigger(cp, softirq);
+}
+EXPORT_SYMBOL(__send_remote_softirq);
+
+void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
+{
+	unsigned long flags;
+	int this_cpu;
+
+	local_irq_save(flags);
+	this_cpu = smp_processor_id();
+	__send_remote_softirq(cp, cpu, this_cpu, softirq);
+	local_irq_restore(flags);
+}
+EXPORT_SYMBOL(send_remote_softirq);
+
+static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
+					       unsigned long action, void *hcpu)
+{
+	/*
+	 * If a CPU goes away, splice its entries to the current CPU
+	 * and trigger a run of the softirq
+	 */
+	if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
+		int cpu = (unsigned long) hcpu;
+		int i;
+
+		local_irq_disable();
+		for (i = 0; i < 32; i++) {
+			struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
+			struct list_head *local_head;
+
+			if (list_empty(head))
+				continue;
+
+			local_head = &__get_cpu_var(softirq_work_list[i]);
+			list_splice_init(head, local_head);
+			raise_softirq_irqoff(i);
+		}
+		local_irq_enable();
+	}
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
+	.notifier_call	= remote_softirq_cpu_notify,
+};
+
 void __init softirq_init(void)
 {
 	int cpu;
 
 	for_each_possible_cpu(cpu) {
+		int i;
+
 		per_cpu(tasklet_vec, cpu).tail =
 			&per_cpu(tasklet_vec, cpu).head;
 		per_cpu(tasklet_hi_vec, cpu).tail =
 			&per_cpu(tasklet_hi_vec, cpu).head;
+		for (i = 0; i < 32; i++)
+			INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 	}
 
+	register_hotcpu_notifier(&remote_softirq_cpu_notifier);
+
 	open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 	open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 }
-- 
1.5.6.5

Re: [PATCH 1/2]: softirq: Add support for triggering softirq work on softirqs.

[Jens Axboe]

On Wed, Sep 17 2008, David Miller wrote:
> 
> softirq: Add support for triggering softirq work on softirqs.
> 
> This is basically a genericization of Jens Axboe's block layer
> remote softirq changes.

Looks good Dave! One small comment - instead of using 32, how about just
adding a NR_SOFTIRQ at the bottom of the softirq enum and use that for
the softirq_work_list sizing?

Apart from that, you can add my Signed-off-by.

-- 
Jens Axboe

Re: [PATCH 1/2]: softirq: Add support for triggering softirq work on softirqs.

[David Miller]

From: Jens Axboe <jens.axboe@oracle.com>
Date: Thu, 18 Sep 2008 06:35:42 +0200

> On Wed, Sep 17 2008, David Miller wrote:
> > 
> > softirq: Add support for triggering softirq work on softirqs.
> > 
> > This is basically a genericization of Jens Axboe's block layer
> > remote softirq changes.
> 
> Looks good Dave! One small comment - instead of using 32, how about just
> adding a NR_SOFTIRQ at the bottom of the softirq enum and use that for
> the softirq_work_list sizing?

Good idea.

> Apart from that, you can add my Signed-off-by.

Great, thanks for looking at this.

Re: [PATCH 1/2]: softirq: Add support for triggering softirq work on softirqs.

[Ilpo Järvinen]

On Wed, 17 Sep 2008, David Miller wrote:

> From: Jens Axboe <jens.axboe@oracle.com>
> Date: Thu, 18 Sep 2008 06:35:42 +0200
> 
> > On Wed, Sep 17 2008, David Miller wrote:
> > > 
> > > softirq: Add support for triggering softirq work on softirqs.
> > > 
> > > This is basically a genericization of Jens Axboe's block layer
> > > remote softirq changes.
> > 
> > Looks good Dave! One small comment - instead of using 32, how about just
> > adding a NR_SOFTIRQ at the bottom of the softirq enum and use that for
> > the softirq_work_list sizing?
> 
> Good idea.

I suppose you could add lkml too as Andrew might not be happy if this 
kind of thing gets done behind his back on netdev/net-next only 
(considering some past experience)? :-)

-- 
 i.

Re: [PATCH 1/2]: softirq: Add support for triggering softirq work on softirqs.

[David Miller]

From: "Ilpo Järvinen" <ilpo.jarvinen@helsinki.fi>
Date: Thu, 18 Sep 2008 10:40:06 +0300 (EEST)

> I suppose you could add lkml too as Andrew might not be happy if this 
> kind of thing gets done behind his back on netdev/net-next only 
> (considering some past experience)? :-)

Yep, good idea, I'll do that when I next report this stuff.

Re: [PATCH 1/2]: softirq: Add support for triggering softirq work on softirqs.

[Stephen Hemminger]

I wonder if the added overhead of local_irq_disable/local_irq_enable is
killing any performance gain. Certainly it adds more latency to a sensitive code path.

Is there a good lock-less solution?

Re: [PATCH 1/2]: softirq: Add support for triggering softirq work on softirqs.

[David Miller]

From: Stephen Hemminger <stephen.hemminger@vyatta.com>
Date: Fri, 19 Sep 2008 09:57:30 -0700 (PDT)

> I wonder if the added overhead of local_irq_disable/local_irq_enable is
> killing any performance gain. Certainly it adds more latency to a sensitive code path.
> 
> Is there a good lock-less solution?

The cross-cpu IPI is 1,000 times more expensive than the local IRQ
enable/disable (on x86 it's a bunch of APIC programming), so if you're
looking for cost analysis start there.