From: "Bradley D. LaRonde" <brad@ltc.com>
To: <linux-mips@oss.sgi.com>
Subject: Fw: Mips scalibility problems & softirq.c improvments
Date: Mon, 13 May 2002 10:28:10 -0400 [thread overview]
Message-ID: <006c01c1fa8a$68ad1910$4c00a8c0@prefect> (raw)
[-- Attachment #1: Type: text/plain, Size: 5624 bytes --]
----- Original Message -----
From: "D.J. Barrow" <barrow_dj@yahoo.com>
To: "Kernel Mailing List" <linux-kernel@vger.kernel.org>; "Netfilter"
<netfilter-devel@lists.samba.org>
Sent: Monday, May 13, 2002 6:12 AM
Subject: Mips scalibility problems & softirq.c improvments
> Hi,
> While testing the SMP performance of iptables with a lot of rules on a
mips based cpu,
> I found that the SMP performance was 40% lower on 2 cpus than 1 cpu.
>
> There is a number of reasons for this the primary being that the rules
were bigger
> than the shared L2 cache, little enough can be done about this.
>
> The second is that interrupts are on every mips port I bothered checking
> are only delivered on cpu 0 ( this is really pathetic ).
>
>
> See the code that prints /proc/interrupts in arch/mips/kernel/irq.c
> int get_irq_list(char *buf)
> {
> struct irqaction * action;
> char *p = buf;
> int i;
>
> p += sprintf(p, " ");
> for (i=0; i < 1 /*smp_num_cpus*/; i++)
>
> Need I say more.....
>
> As softirqs are usually bound to the same
> cpu that start the softirqs softirqs performs really really badly,
> also the fact that the softirq.c code checks in_interrupt on
> entry means that it frequently does a quick exit.
>
>
> I also will be providing a patch I developed to the developers of a mips
based
> system on chip which distributes the irqs over all cpus using 2 polices
> even interrupts to cpu 0 odd interrupts to cpu 1 or leaving the interrupts
> enter in all cpus & only call do_IRQ on the cpu with the lowest
local_irq_count
> & local_bh_count this should cause softirqs to perform will on this
> system anyway.
>
>
> I've provided a small patch to irq.c which fixes /proc/interrupts in
2.4.18 mips32
> hopefully somebody will be kind enough to fix up the 64 bit &
> the latest stuff in mips64 & the latest oss.sgi.com cvs trees.
>
> --- linux.orig/arch/mips/kernel/irq.c Sun Sep 9 18:43:01 2001
> +++ linux/arch/mips/kernel/irq.c Mon May 13 10:34:15 2002
> @@ -71,13 +71,13 @@
>
> int get_irq_list(char *buf)
> {
> + int i, j;
> struct irqaction * action;
> char *p = buf;
> - int i;
>
> p += sprintf(p, " ");
> - for (i=0; i < 1 /*smp_num_cpus*/; i++)
> - p += sprintf(p, "CPU%d ", i);
> + for (j=0; j<smp_num_cpus; j++)
> + p += sprintf(p, "CPU%d ",j);
> *p++ = '\n';
>
> for (i = 0 ; i < NR_IRQS ; i++) {
> @@ -85,7 +85,13 @@
> if (!action)
> continue;
> p += sprintf(p, "%3d: ",i);
> +#ifndef CONFIG_SMP
> p += sprintf(p, "%10u ", kstat_irqs(i));
> +#else
> + for (j = 0; j < smp_num_cpus; j++)
> + p += sprintf(p, "%10u ",
> + kstat.irqs[cpu_logical_map(j)][i]);
> +#endif
> p += sprintf(p, " %14s", irq_desc[i].handler->typename);
> p += sprintf(p, " %s", action->name);
>
> @@ -93,7 +99,7 @@
> p += sprintf(p, ", %s", action->name);
> *p++ = '\n';
> }
> - p += sprintf(p, "ERR: %10lu\n", irq_err_count);
> + p += sprintf(p, "ERR: %10u\n", atomic_read(&irq_err_count));
> return p - buf;
> }
>
>
>
> I also provide a small patch for softirq.c which makes sure the
> the softirqs stay running if in cpu_idle & no reschedule is pending.
> This improves softirq.c performance a small bit as it usually exits
> after calling each softirq once rather than staying in the loop
> if it has nothing better to do.
>
> --- linux.old/kernel/softirq.c Tue Jan 15 04:13:43 2002
> +++ linux.new/kernel/softirq.c Thu May 9 12:36:46 2002
> @@ -95,7 +95,8 @@
> local_irq_disable();
>
> pending = softirq_pending(cpu);
> - if (pending & mask) {
> + if ((pending && current==idle_task(cpu) &&
!current->need_resched )
> + || (pending & mask) ) {
> mask &= ~pending;
> goto restart;
> }
> diff -u -r linux.old/include/linux/sched.h linux.new/include/linux/sched.h
> --- linux.old/include/linux/sched.h Thu May 9 18:08:42 2002
> +++ linux.new/include/linux/sched.h Thu May 9 10:30:34 2002
> @@ -936,6 +936,19 @@
> return res;
> }
>
> +#ifdef CONFIG_SMP
> +
> +#define idle_task(cpu) (init_tasks[cpu_number_map(cpu)])
> +#define can_schedule(p,cpu) \
> + ((p)->cpus_runnable & (p)->cpus_allowed & (1 << cpu))
> +
> +#else
> +
> +#define idle_task(cpu) (&init_task)
> +#define can_schedule(p,cpu) (1)
> +
> +#endif
> +
> #endif /* __KERNEL__ */
>
> #endif
>
> diff -u -r linux.old/kernel/sched.c linux.new/kernel/sched.c
> --- linux.old/kernel/sched.c Wed May 1 10:40:26 2002
> +++ linux.new/kernel/sched.c Thu May 9 10:30:26 2002
> @@ -112,18 +112,7 @@
> struct kernel_stat kstat;
> extern struct task_struct *child_reaper;
>
> -#ifdef CONFIG_SMP
>
> -#define idle_task(cpu) (init_tasks[cpu_number_map(cpu)])
> -#define can_schedule(p,cpu) \
> - ((p)->cpus_runnable & (p)->cpus_allowed & (1 << cpu))
> -
> -#else
> -
> -#define idle_task(cpu) (&init_task)
> -#define can_schedule(p,cpu) (1)
> -
> -#endif
>
> void scheduling_functions_start_here(void) { }
>
>
> Also find the patches sent as attachments.
>
>
> =====
> D.J. Barrow Linux kernel developer
> eMail: dj_barrow@ariasoft.ie
> Home: +353-22-47196.
> Work: +353-91-758353
>
> __________________________________________________
> Do You Yahoo!?
> LAUNCH - Your Yahoo! Music Experience
> http://launch.yahoo.com
[-- Attachment #2: softirq_fix.diff --]
[-- Type: application/octet-stream, Size: 1536 bytes --]
--- linux.old/kernel/softirq.c Tue Jan 15 04:13:43 2002
+++ linux.new/kernel/softirq.c Thu May 9 12:36:46 2002
@@ -95,7 +95,8 @@
local_irq_disable();
pending = softirq_pending(cpu);
- if (pending & mask) {
+ if ((pending && current==idle_task(cpu) && !current->need_resched )
+ || (pending & mask) ) {
mask &= ~pending;
goto restart;
}
diff -u -r linux.old/include/linux/sched.h linux.new/include/linux/sched.h
--- linux.old/include/linux/sched.h Thu May 9 18:08:42 2002
+++ linux.new/include/linux/sched.h Thu May 9 10:30:34 2002
@@ -936,6 +936,19 @@
return res;
}
+#ifdef CONFIG_SMP
+
+#define idle_task(cpu) (init_tasks[cpu_number_map(cpu)])
+#define can_schedule(p,cpu) \
+ ((p)->cpus_runnable & (p)->cpus_allowed & (1 << cpu))
+
+#else
+
+#define idle_task(cpu) (&init_task)
+#define can_schedule(p,cpu) (1)
+
+#endif
+
#endif /* __KERNEL__ */
#endif
diff -u -r linux.old/kernel/sched.c linux.new/kernel/sched.c
--- linux.old/kernel/sched.c Wed May 1 10:40:26 2002
+++ linux.new/kernel/sched.c Thu May 9 10:30:26 2002
@@ -112,18 +112,7 @@
struct kernel_stat kstat;
extern struct task_struct *child_reaper;
-#ifdef CONFIG_SMP
-#define idle_task(cpu) (init_tasks[cpu_number_map(cpu)])
-#define can_schedule(p,cpu) \
- ((p)->cpus_runnable & (p)->cpus_allowed & (1 << cpu))
-
-#else
-
-#define idle_task(cpu) (&init_task)
-#define can_schedule(p,cpu) (1)
-
-#endif
void scheduling_functions_start_here(void) { }
[-- Attachment #3: mips32_irq.c_fix.diff --]
[-- Type: application/octet-stream, Size: 1113 bytes --]
--- linux.orig/arch/mips/kernel/irq.c Sun Sep 9 18:43:01 2001
+++ linux/arch/mips/kernel/irq.c Mon May 13 10:34:15 2002
@@ -71,13 +71,13 @@
int get_irq_list(char *buf)
{
+ int i, j;
struct irqaction * action;
char *p = buf;
- int i;
p += sprintf(p, " ");
- for (i=0; i < 1 /*smp_num_cpus*/; i++)
- p += sprintf(p, "CPU%d ", i);
+ for (j=0; j<smp_num_cpus; j++)
+ p += sprintf(p, "CPU%d ",j);
*p++ = '\n';
for (i = 0 ; i < NR_IRQS ; i++) {
@@ -85,7 +85,13 @@
if (!action)
continue;
p += sprintf(p, "%3d: ",i);
+#ifndef CONFIG_SMP
p += sprintf(p, "%10u ", kstat_irqs(i));
+#else
+ for (j = 0; j < smp_num_cpus; j++)
+ p += sprintf(p, "%10u ",
+ kstat.irqs[cpu_logical_map(j)][i]);
+#endif
p += sprintf(p, " %14s", irq_desc[i].handler->typename);
p += sprintf(p, " %s", action->name);
@@ -93,7 +99,7 @@
p += sprintf(p, ", %s", action->name);
*p++ = '\n';
}
- p += sprintf(p, "ERR: %10lu\n", irq_err_count);
+ p += sprintf(p, "ERR: %10u\n", atomic_read(&irq_err_count));
return p - buf;
}
WARNING: multiple messages have this Message-ID (diff)
From: "Bradley D. LaRonde" <brad@ltc.com>
To: linux-mips@oss.sgi.com
Subject: Fw: Mips scalibility problems & softirq.c improvments
Date: Mon, 13 May 2002 10:28:10 -0400 [thread overview]
Message-ID: <006c01c1fa8a$68ad1910$4c00a8c0@prefect> (raw)
Message-ID: <20020513142810.lG2M7l7i4ixeHxPZxSAVNFai5fr4luEjZYY9SA7xADE@z> (raw)
[-- Attachment #1: Type: text/plain, Size: 5624 bytes --]
----- Original Message -----
From: "D.J. Barrow" <barrow_dj@yahoo.com>
To: "Kernel Mailing List" <linux-kernel@vger.kernel.org>; "Netfilter"
<netfilter-devel@lists.samba.org>
Sent: Monday, May 13, 2002 6:12 AM
Subject: Mips scalibility problems & softirq.c improvments
> Hi,
> While testing the SMP performance of iptables with a lot of rules on a
mips based cpu,
> I found that the SMP performance was 40% lower on 2 cpus than 1 cpu.
>
> There is a number of reasons for this the primary being that the rules
were bigger
> than the shared L2 cache, little enough can be done about this.
>
> The second is that interrupts are on every mips port I bothered checking
> are only delivered on cpu 0 ( this is really pathetic ).
>
>
> See the code that prints /proc/interrupts in arch/mips/kernel/irq.c
> int get_irq_list(char *buf)
> {
> struct irqaction * action;
> char *p = buf;
> int i;
>
> p += sprintf(p, " ");
> for (i=0; i < 1 /*smp_num_cpus*/; i++)
>
> Need I say more.....
>
> As softirqs are usually bound to the same
> cpu that start the softirqs softirqs performs really really badly,
> also the fact that the softirq.c code checks in_interrupt on
> entry means that it frequently does a quick exit.
>
>
> I also will be providing a patch I developed to the developers of a mips
based
> system on chip which distributes the irqs over all cpus using 2 polices
> even interrupts to cpu 0 odd interrupts to cpu 1 or leaving the interrupts
> enter in all cpus & only call do_IRQ on the cpu with the lowest
local_irq_count
> & local_bh_count this should cause softirqs to perform will on this
> system anyway.
>
>
> I've provided a small patch to irq.c which fixes /proc/interrupts in
2.4.18 mips32
> hopefully somebody will be kind enough to fix up the 64 bit &
> the latest stuff in mips64 & the latest oss.sgi.com cvs trees.
>
> --- linux.orig/arch/mips/kernel/irq.c Sun Sep 9 18:43:01 2001
> +++ linux/arch/mips/kernel/irq.c Mon May 13 10:34:15 2002
> @@ -71,13 +71,13 @@
>
> int get_irq_list(char *buf)
> {
> + int i, j;
> struct irqaction * action;
> char *p = buf;
> - int i;
>
> p += sprintf(p, " ");
> - for (i=0; i < 1 /*smp_num_cpus*/; i++)
> - p += sprintf(p, "CPU%d ", i);
> + for (j=0; j<smp_num_cpus; j++)
> + p += sprintf(p, "CPU%d ",j);
> *p++ = '\n';
>
> for (i = 0 ; i < NR_IRQS ; i++) {
> @@ -85,7 +85,13 @@
> if (!action)
> continue;
> p += sprintf(p, "%3d: ",i);
> +#ifndef CONFIG_SMP
> p += sprintf(p, "%10u ", kstat_irqs(i));
> +#else
> + for (j = 0; j < smp_num_cpus; j++)
> + p += sprintf(p, "%10u ",
> + kstat.irqs[cpu_logical_map(j)][i]);
> +#endif
> p += sprintf(p, " %14s", irq_desc[i].handler->typename);
> p += sprintf(p, " %s", action->name);
>
> @@ -93,7 +99,7 @@
> p += sprintf(p, ", %s", action->name);
> *p++ = '\n';
> }
> - p += sprintf(p, "ERR: %10lu\n", irq_err_count);
> + p += sprintf(p, "ERR: %10u\n", atomic_read(&irq_err_count));
> return p - buf;
> }
>
>
>
> I also provide a small patch for softirq.c which makes sure the
> the softirqs stay running if in cpu_idle & no reschedule is pending.
> This improves softirq.c performance a small bit as it usually exits
> after calling each softirq once rather than staying in the loop
> if it has nothing better to do.
>
> --- linux.old/kernel/softirq.c Tue Jan 15 04:13:43 2002
> +++ linux.new/kernel/softirq.c Thu May 9 12:36:46 2002
> @@ -95,7 +95,8 @@
> local_irq_disable();
>
> pending = softirq_pending(cpu);
> - if (pending & mask) {
> + if ((pending && current==idle_task(cpu) &&
!current->need_resched )
> + || (pending & mask) ) {
> mask &= ~pending;
> goto restart;
> }
> diff -u -r linux.old/include/linux/sched.h linux.new/include/linux/sched.h
> --- linux.old/include/linux/sched.h Thu May 9 18:08:42 2002
> +++ linux.new/include/linux/sched.h Thu May 9 10:30:34 2002
> @@ -936,6 +936,19 @@
> return res;
> }
>
> +#ifdef CONFIG_SMP
> +
> +#define idle_task(cpu) (init_tasks[cpu_number_map(cpu)])
> +#define can_schedule(p,cpu) \
> + ((p)->cpus_runnable & (p)->cpus_allowed & (1 << cpu))
> +
> +#else
> +
> +#define idle_task(cpu) (&init_task)
> +#define can_schedule(p,cpu) (1)
> +
> +#endif
> +
> #endif /* __KERNEL__ */
>
> #endif
>
> diff -u -r linux.old/kernel/sched.c linux.new/kernel/sched.c
> --- linux.old/kernel/sched.c Wed May 1 10:40:26 2002
> +++ linux.new/kernel/sched.c Thu May 9 10:30:26 2002
> @@ -112,18 +112,7 @@
> struct kernel_stat kstat;
> extern struct task_struct *child_reaper;
>
> -#ifdef CONFIG_SMP
>
> -#define idle_task(cpu) (init_tasks[cpu_number_map(cpu)])
> -#define can_schedule(p,cpu) \
> - ((p)->cpus_runnable & (p)->cpus_allowed & (1 << cpu))
> -
> -#else
> -
> -#define idle_task(cpu) (&init_task)
> -#define can_schedule(p,cpu) (1)
> -
> -#endif
>
> void scheduling_functions_start_here(void) { }
>
>
> Also find the patches sent as attachments.
>
>
> =====
> D.J. Barrow Linux kernel developer
> eMail: dj_barrow@ariasoft.ie
> Home: +353-22-47196.
> Work: +353-91-758353
>
> __________________________________________________
> Do You Yahoo!?
> LAUNCH - Your Yahoo! Music Experience
> http://launch.yahoo.com
[-- Attachment #2: softirq_fix.diff --]
[-- Type: application/octet-stream, Size: 1536 bytes --]
--- linux.old/kernel/softirq.c Tue Jan 15 04:13:43 2002
+++ linux.new/kernel/softirq.c Thu May 9 12:36:46 2002
@@ -95,7 +95,8 @@
local_irq_disable();
pending = softirq_pending(cpu);
- if (pending & mask) {
+ if ((pending && current==idle_task(cpu) && !current->need_resched )
+ || (pending & mask) ) {
mask &= ~pending;
goto restart;
}
diff -u -r linux.old/include/linux/sched.h linux.new/include/linux/sched.h
--- linux.old/include/linux/sched.h Thu May 9 18:08:42 2002
+++ linux.new/include/linux/sched.h Thu May 9 10:30:34 2002
@@ -936,6 +936,19 @@
return res;
}
+#ifdef CONFIG_SMP
+
+#define idle_task(cpu) (init_tasks[cpu_number_map(cpu)])
+#define can_schedule(p,cpu) \
+ ((p)->cpus_runnable & (p)->cpus_allowed & (1 << cpu))
+
+#else
+
+#define idle_task(cpu) (&init_task)
+#define can_schedule(p,cpu) (1)
+
+#endif
+
#endif /* __KERNEL__ */
#endif
diff -u -r linux.old/kernel/sched.c linux.new/kernel/sched.c
--- linux.old/kernel/sched.c Wed May 1 10:40:26 2002
+++ linux.new/kernel/sched.c Thu May 9 10:30:26 2002
@@ -112,18 +112,7 @@
struct kernel_stat kstat;
extern struct task_struct *child_reaper;
-#ifdef CONFIG_SMP
-#define idle_task(cpu) (init_tasks[cpu_number_map(cpu)])
-#define can_schedule(p,cpu) \
- ((p)->cpus_runnable & (p)->cpus_allowed & (1 << cpu))
-
-#else
-
-#define idle_task(cpu) (&init_task)
-#define can_schedule(p,cpu) (1)
-
-#endif
void scheduling_functions_start_here(void) { }
[-- Attachment #3: mips32_irq.c_fix.diff --]
[-- Type: application/octet-stream, Size: 1113 bytes --]
--- linux.orig/arch/mips/kernel/irq.c Sun Sep 9 18:43:01 2001
+++ linux/arch/mips/kernel/irq.c Mon May 13 10:34:15 2002
@@ -71,13 +71,13 @@
int get_irq_list(char *buf)
{
+ int i, j;
struct irqaction * action;
char *p = buf;
- int i;
p += sprintf(p, " ");
- for (i=0; i < 1 /*smp_num_cpus*/; i++)
- p += sprintf(p, "CPU%d ", i);
+ for (j=0; j<smp_num_cpus; j++)
+ p += sprintf(p, "CPU%d ",j);
*p++ = '\n';
for (i = 0 ; i < NR_IRQS ; i++) {
@@ -85,7 +85,13 @@
if (!action)
continue;
p += sprintf(p, "%3d: ",i);
+#ifndef CONFIG_SMP
p += sprintf(p, "%10u ", kstat_irqs(i));
+#else
+ for (j = 0; j < smp_num_cpus; j++)
+ p += sprintf(p, "%10u ",
+ kstat.irqs[cpu_logical_map(j)][i]);
+#endif
p += sprintf(p, " %14s", irq_desc[i].handler->typename);
p += sprintf(p, " %s", action->name);
@@ -93,7 +99,7 @@
p += sprintf(p, ", %s", action->name);
*p++ = '\n';
}
- p += sprintf(p, "ERR: %10lu\n", irq_err_count);
+ p += sprintf(p, "ERR: %10u\n", atomic_read(&irq_err_count));
return p - buf;
}
next reply other threads:[~2002-05-13 18:20 UTC|newest]
Thread overview: 2+ messages / expand[flat|nested] mbox.gz Atom feed top
2002-05-13 14:28 Bradley D. LaRonde [this message]
2002-05-13 14:28 ` Fw: Mips scalibility problems & softirq.c improvments Bradley D. LaRonde
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to='006c01c1fa8a$68ad1910$4c00a8c0@prefect' \
--to=brad@ltc.com \
--cc=linux-mips@oss.sgi.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox