[PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Chris Wedgwood]

Right now CONFIG_4KSTACKS implies IRQ-stacks.  Some people though
really need 8K stacks and it would be nice to have IRQ-stacks for them
too.

This splits the option in two with the intention of removing the
IRQ-stacks option completely.

 arch/i386/Kconfig.debug        |   13 ++++++++++---
 arch/i386/defconfig            |    1 -
 arch/i386/kernel/irq.c         |   14 +++++++-------
 include/asm-i386/irq.h         |    6 +++---
 include/asm-i386/module.h      |    6 +++---
 include/asm-i386/thread_info.h |    6 +++---
 6 files changed, 26 insertions(+), 20 deletions(-)

Signed-off-by: Chris Wedgwood <cw@f00f.org>



diff -Nru a/arch/i386/Kconfig.debug b/arch/i386/Kconfig.debug
--- a/arch/i386/Kconfig.debug	2004-09-09 16:06:04 -07:00
+++ b/arch/i386/Kconfig.debug	2004-09-09 16:06:04 -07:00
@@ -46,14 +46,21 @@
 	  This results in a large slowdown, but helps to find certain types
 	  of memory corruptions.
 
-config 4KSTACKS
+config I386_4KSTACKS
 	bool "Use 4Kb for kernel stacks instead of 8Kb"
+	default n
 	help
 	  If you say Y here the kernel will use a 4Kb stacksize for the
 	  kernel stack attached to each process/thread. This facilitates
 	  running more threads on a system and also reduces the pressure
-	  on the VM subsystem for higher order allocations. This option
-	  will also use IRQ stacks to compensate for the reduced stackspace.
+	  on the VM subsystem for higher order allocations.
+
+config I386_IRQSTACKS
+	bool "Allocate separate IRQ stacks"
+	default y
+	help
+	  If you say Y here the kernel will allocate and use separate
+	  stacks for interrupts.
 
 config SCHEDSTATS
 	bool "Collect scheduler statistics"
diff -Nru a/arch/i386/defconfig b/arch/i386/defconfig
--- a/arch/i386/defconfig	2004-09-09 16:06:04 -07:00
+++ b/arch/i386/defconfig	2004-09-09 16:06:04 -07:00
@@ -1221,7 +1221,6 @@
 CONFIG_EARLY_PRINTK=y
 CONFIG_DEBUG_SPINLOCK_SLEEP=y
 # CONFIG_FRAME_POINTER is not set
-CONFIG_4KSTACKS=y
 CONFIG_X86_FIND_SMP_CONFIG=y
 CONFIG_X86_MPPARSE=y
 
diff -Nru a/arch/i386/kernel/irq.c b/arch/i386/kernel/irq.c
--- a/arch/i386/kernel/irq.c	2004-09-09 16:06:04 -07:00
+++ b/arch/i386/kernel/irq.c	2004-09-09 16:06:04 -07:00
@@ -76,10 +76,10 @@
 /*
  * per-CPU IRQ handling stacks
  */
-#ifdef CONFIG_4KSTACKS
+#ifdef CONFIG_I386_IRQSTACKS
 union irq_ctx *hardirq_ctx[NR_CPUS];
 union irq_ctx *softirq_ctx[NR_CPUS];
-#endif
+#endif /* CONFIG_I386_IRQSTACKS */
 
 /*
  * Special irq handlers.
@@ -489,7 +489,7 @@
 	 * useful for irq hardware that does not mask cleanly in an
 	 * SMP environment.
 	 */
-#ifdef CONFIG_4KSTACKS
+#ifdef CONFIG_I386_IRQSTACKS
 
 	for (;;) {
 		irqreturn_t action_ret;
@@ -542,7 +542,7 @@
 		desc->status &= ~IRQ_PENDING;
 	}
 
-#else
+#else /* not using CONFIG_I386_IRQSTACKS */
 
 	for (;;) {
 		irqreturn_t action_ret;
@@ -558,7 +558,7 @@
 			break;
 		desc->status &= ~IRQ_PENDING;
 	}
-#endif
+#endif /* CONFIG_I386_IRQSTACKS */
 	desc->status &= ~IRQ_INPROGRESS;
 
 out:
@@ -1077,7 +1077,7 @@
 }
 
 
-#ifdef CONFIG_4KSTACKS
+#ifdef CONFIG_I386_IRQSTACKS
 /*
  * These should really be __section__(".bss.page_aligned") as well, but
  * gcc's 3.0 and earlier don't handle that correctly.
@@ -1155,4 +1155,4 @@
 }
 
 EXPORT_SYMBOL(do_softirq);
-#endif
+#endif /* CONFIG_I386_IRQSTACKS */
diff -Nru a/include/asm-i386/irq.h b/include/asm-i386/irq.h
--- a/include/asm-i386/irq.h	2004-09-09 16:06:04 -07:00
+++ b/include/asm-i386/irq.h	2004-09-09 16:06:04 -07:00
@@ -31,7 +31,7 @@
 #define ARCH_HAS_NMI_WATCHDOG		/* See include/linux/nmi.h */
 #endif
 
-#ifdef CONFIG_4KSTACKS
+#ifdef CONFIG_I386_IRQSTACKS
 /*
  * per-CPU IRQ handling contexts (thread information and stack)
  */
@@ -46,9 +46,9 @@
 extern void irq_ctx_init(int cpu);
 
 #define __ARCH_HAS_DO_SOFTIRQ
-#else
+#else /* not using CONFIG_I386_IRQSTACKS */
 #define irq_ctx_init(cpu) do { ; } while (0)
-#endif
+#endif /* CONFIG_I386_IRQSTACKS */
 
 struct irqaction;
 struct pt_regs;
diff -Nru a/include/asm-i386/module.h b/include/asm-i386/module.h
--- a/include/asm-i386/module.h	2004-09-09 16:06:04 -07:00
+++ b/include/asm-i386/module.h	2004-09-09 16:06:04 -07:00
@@ -60,11 +60,11 @@
 #define MODULE_REGPARM ""
 #endif
 
-#ifdef CONFIG_4KSTACKS
+#ifdef CONFIG_I386_4KSTACKS
 #define MODULE_STACKSIZE "4KSTACKS "
-#else
+#else /* not using CONFIG_I386_4KSTACKS */
 #define MODULE_STACKSIZE ""
-#endif
+#endif /* CONFIG_I386_4KSTACKS */
 
 #define MODULE_ARCH_VERMAGIC MODULE_PROC_FAMILY MODULE_REGPARM MODULE_STACKSIZE
 
diff -Nru a/include/asm-i386/thread_info.h b/include/asm-i386/thread_info.h
--- a/include/asm-i386/thread_info.h	2004-09-09 16:06:04 -07:00
+++ b/include/asm-i386/thread_info.h	2004-09-09 16:06:04 -07:00
@@ -52,11 +52,11 @@
 #endif
 
 #define PREEMPT_ACTIVE		0x4000000
-#ifdef CONFIG_4KSTACKS
+#ifdef CONFIG_I386_4KSTACKS
 #define THREAD_SIZE            (4096)
-#else
+#else /* not using CONFIG_I386_4KSTACKS */
 #define THREAD_SIZE		(8192)
-#endif
+#endif /* CONFIG_I386_4KSTACKS */
 
 #define STACK_WARN             (THREAD_SIZE/8)
 /*

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Brian Gerst]

Chris Wedgwood wrote:
> Right now CONFIG_4KSTACKS implies IRQ-stacks.  Some people though
> really need 8K stacks and it would be nice to have IRQ-stacks for them
> too.
> 
> This splits the option in two with the intention of removing the
> IRQ-stacks option completely.
> 
>  arch/i386/Kconfig.debug        |   13 ++++++++++---
>  arch/i386/defconfig            |    1 -
>  arch/i386/kernel/irq.c         |   14 +++++++-------
>  include/asm-i386/irq.h         |    6 +++---
>  include/asm-i386/module.h      |    6 +++---
>  include/asm-i386/thread_info.h |    6 +++---
>  6 files changed, 26 insertions(+), 20 deletions(-)
> 
> Signed-off-by: Chris Wedgwood <cw@f00f.org>
> 
> 
> 
> diff -Nru a/arch/i386/Kconfig.debug b/arch/i386/Kconfig.debug
> --- a/arch/i386/Kconfig.debug	2004-09-09 16:06:04 -07:00
> +++ b/arch/i386/Kconfig.debug	2004-09-09 16:06:04 -07:00
> @@ -46,14 +46,21 @@
>  	  This results in a large slowdown, but helps to find certain types
>  	  of memory corruptions.
>  
> -config 4KSTACKS
> +config I386_4KSTACKS
>  	bool "Use 4Kb for kernel stacks instead of 8Kb"

What's the point of changing 4KSTACKS to I386_4KSTACKS?  Best to just 
leave this alone as external code is likely to check it.

--
				Brian Gerst

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Chris Wedgwood]

On Thu, Sep 09, 2004 at 07:55:03PM -0400, Brian Gerst wrote:

> What's the point of changing 4KSTACKS to I386_4KSTACKS?

Two (no very good reasons).  (1) I wanted to make it clear it was an
i386 specific thing (2) it made grepping for old vs new changes more
obvious

> Best to just leave this alone as external code is likely to check
> it.

i personally prefer the change but am not against dropping it if
people object

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Arjan van de Ven]

[-- Attachment #1: Type: text/plain, Size: 470 bytes --]

On Fri, 2004-09-10 at 01:25, Chris Wedgwood wrote:
> Right now CONFIG_4KSTACKS implies IRQ-stacks.  Some people though
> really need 8K stacks and it would be nice to have IRQ-stacks for them
> too.

Well I always assumed the future plan was to remove 8k stacks entirely;
4k+irqstacks and 8k basically have near comparable stack space, with
this patch you create an option that has more but that is/should be
deprecated. I'm not convinced that's a good idea.


[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Chris Wedgwood]

On Fri, Sep 10, 2004 at 08:40:28AM +0200, Arjan van de Ven wrote:

> Well I always assumed the future plan was to remove 8k stacks
> entirely;

you know as well as i do that 4K stacks don't work for some people
right now and it will probably be months if not longer before this
changes

> 4k+irqstacks and 8k basically have near comparable stack space, with
> this patch you create an option that has more but that is/should be
> deprecated.

maybe, but *who* says this is depricated?

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Arjan van de Ven]

[-- Attachment #1: Type: text/plain, Size: 690 bytes --]

On Thu, Sep 09, 2004 at 11:45:19PM -0700, Chris Wedgwood wrote:
> 
> > 4k+irqstacks and 8k basically have near comparable stack space, with
> > this patch you create an option that has more but that is/should be
> > deprecated.
> 
> maybe, but *who* says this is depricated?

I just did ;)

the roadmap was

8K stacks  ->  dual 4k/8k option -> 4k stacks

this roadmap comes down to basically keeping the available stack space equal
over time (because yes if you see overflows with 4k you WILL see them with
8k too just it takes some fun firewall rules and a network interrupt at just
the right time). 

Your proposal adds a temporary "bulb" that suddenly "allows" code to bloat
into it...


[-- Attachment #2: Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Chris Wedgwood]

On Fri, Sep 10, 2004 at 08:52:13AM +0200, Arjan van de Ven wrote:

> I just did ;)

riiight

> the roadmap was
> 
> 8K stacks  ->  dual 4k/8k option -> 4k stacks

URL?


also, why 4K and not 8K or 2K?  because it's the page size?  why not
4K to then on amd64 or ppc64?

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Arjan van de Ven]

[-- Attachment #1: Type: text/plain, Size: 336 bytes --]

On Fri, Sep 10, 2004 at 12:15:30AM -0700, Chris Wedgwood wrote:
> > 
> > 8K stacks  ->  dual 4k/8k option -> 4k stacks
> 
> URL?
> 
> 
> also, why 4K and not 8K or 2K?  

it used to be 8K unified for user context and softirq context and hardirq
context. Basically that got "split up" into 4k for user and 4k each for the
irq contexts.


[-- Attachment #2: Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Chris Wedgwood]

On Fri, Sep 10, 2004 at 09:21:21AM +0200, Arjan van de Ven wrote:

> it used to be 8K unified for user context and softirq context and
> hardirq context. Basically that got "split up" into 4k for user and
> 4k each for the irq contexts.

you didn't answer my second question and that's not really an answer
of why 4K is the 'right' value

what makes 4K suitable for i386 when x86-64 uses 8K and ppc64 uses
16K?  the argument used to be about memory fragmentation but that not
solved for the latter

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Arjan van de Ven]

[-- Attachment #1: Type: text/plain, Size: 667 bytes --]

On Fri, Sep 10, 2004 at 12:23:28AM -0700, Chris Wedgwood wrote:
> On Fri, Sep 10, 2004 at 09:21:21AM +0200, Arjan van de Ven wrote:
> 
> > it used to be 8K unified for user context and softirq context and
> > hardirq context. Basically that got "split up" into 4k for user and
> > 4k each for the irq contexts.
> 
> you didn't answer my second question and that's not really an answer
> of why 4K is the 'right' value

because in 2.4 user context already only had 4Kb.  That makes it a basically
unchanged value.
(yes 2.4 had a 8k total but aobut 1.6k went to the task struct, and
somewhere about 2 and 3 Kb are used by irqs leaving a bit under 4Kb for user
context)

[-- Attachment #2: Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Alan Cox]

On Gwe, 2004-09-10 at 08:15, Chris Wedgwood wrote:
> also, why 4K and not 8K or 2K?  because it's the page size?  why not
> 4K to then on amd64 or ppc64?

It would be really really nice to get AMD64 down to 4K stacks. The
bigger sizes of stack frames may make that harder for current big 
stack users. The pain/gain ratio isn't so sweet as with x86 down to
4K.

Also the x86 one really didnt change anything. AMD64 has IRQ stacks
while x86 did not. That meant in practical terms anything using > 4K
was eventually going to go bang 

Alan

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Alan Cox]

On Gwe, 2004-09-10 at 07:40, Arjan van de Ven wrote:
> Well I always assumed the future plan was to remove 8k stacks entirely;
> 4k+irqstacks and 8k basically have near comparable stack space, with
> this patch you create an option that has more but that is/should be
> deprecated. I'm not convinced that's a good idea.

Its probably appropriate to drop gcc 2.x support at that point too since
it's the major cause of remaining problems

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Russell King]

On Fri, Sep 10, 2004 at 10:14:11AM +0100, Alan Cox wrote:
> On Gwe, 2004-09-10 at 07:40, Arjan van de Ven wrote:
> > Well I always assumed the future plan was to remove 8k stacks entirely;
> > 4k+irqstacks and 8k basically have near comparable stack space, with
> > this patch you create an option that has more but that is/should be
> > deprecated. I'm not convinced that's a good idea.
> 
> Its probably appropriate to drop gcc 2.x support at that point too since
> it's the major cause of remaining problems

I have no problem with that - gcc 3.3 on ARM has proven itself (to me
at least) be a stable compiler so I'm not using anything older than
that.

-- 
Russell King
 Linux kernel    2.6 ARM Linux   - http://www.arm.linux.org.uk/
 maintainer of:  2.6 PCMCIA      - http://pcmcia.arm.linux.org.uk/
                 2.6 Serial core

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Martin J. Bligh]

--Alan Cox <alan@lxorguk.ukuu.org.uk> wrote (on Friday, September 10, 2004 10:14:11 +0100):

> On Gwe, 2004-09-10 at 07:40, Arjan van de Ven wrote:
>> Well I always assumed the future plan was to remove 8k stacks entirely;
>> 4k+irqstacks and 8k basically have near comparable stack space, with
>> this patch you create an option that has more but that is/should be
>> deprecated. I'm not convinced that's a good idea.
> 
> Its probably appropriate to drop gcc 2.x support at that point too since
> it's the major cause of remaining problems

What problems does it cause? 2.95.4 still seems to work fine for me.

I agree about killing anything but 4K stacks though - having the single
page is very compelling - not only can we allocate it easier, but we can
also use cache-hot pages from the hot list.

M.

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Chris Wedgwood]

On Fri, Sep 10, 2004 at 07:30:52AM -0700, Martin J. Bligh wrote:

> What problems does it cause? 2.95.4 still seems to work fine for me.

Older (and even current) gcc's spill badly in some circumstances and
also fail to reuse / better-use spill/stack space.  It's been
suggested to me that gcc from CVS as of a week ago is much better (i
tried buidling it and didn't get far, I'll have to try again)

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Hugh Dickins]

On Fri, 10 Sep 2004, Martin J. Bligh wrote:
> 
> I agree about killing anything but 4K stacks though - having the single
> page is very compelling - not only can we allocate it easier, but we can
> also use cache-hot pages from the hot list.

I think we all agree that's a promising future, and a good discipline.
But I'm not the only one to doubt we're there yet.

Chris's patch seems eminently sensible to me.  Why should having separate
interrupt stack depend on whether you're configured for 4K or 8K stacks?

Wasn't Andrea worried, a couple of months back, about nested interrupts
overflowing the 4K interrupt stack?  He was trying to work out how to
have an 8K interrupt stack even with the 4K task stack, proposed thread
info at both top and bottom of stack; but his "current" still looked to
me like it'd be significantly more costly than the present one.

I'm all for Chris's patch.

Hugh

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Alan Cox]

On Gwe, 2004-09-10 at 16:07, Hugh Dickins wrote:
> Chris's patch seems eminently sensible to me.  Why should having separate
> interrupt stack depend on whether you're configured for 4K or 8K stacks?

You only have 4K safe to use in all current configurations. Its a case
of simply fixing the sloppy code (and or pushing up compiler versions 
where the compiler is the offender). 

> Wasn't Andrea worried, a couple of months back, about nested interrupts
> overflowing the 4K interrupt stack? 

We've seen no evidence of this and assuming apps could use 4K safely the
interrupt "stack" was about 2.5K before. Limiting it either by size or
by depth is not a big problem at all.

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Arjan van de Ven]

[-- Attachment #1: Type: text/plain, Size: 962 bytes --]

On Fri, Sep 10, 2004 at 04:07:21PM +0100, Hugh Dickins wrote:
> On Fri, 10 Sep 2004, Martin J. Bligh wrote:
> > 
> > I agree about killing anything but 4K stacks though - having the single
> > page is very compelling - not only can we allocate it easier, but we can
> > also use cache-hot pages from the hot list.
> 
> I think we all agree that's a promising future, and a good discipline.
> But I'm not the only one to doubt we're there yet.
> 

> Chris's patch seems eminently sensible to me.  Why should having separate
> interrupt stack depend on whether you're configured for 4K or 8K stacks?

because it gives people a reason to do sloppy coding.

> Wasn't Andrea worried, a couple of months back, about nested interrupts
> overflowing the 4K interrupt stack?  

I don't think so; interrupts seem to behave quite ok in this regard.
What we should consider regardless is disable the nesting of irqs for
performance reasons but that's an independent matter


[-- Attachment #2: Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Andrea Arcangeli]

On Fri, Sep 10, 2004 at 05:15:38PM +0200, Arjan van de Ven wrote:
> because it gives people a reason to do sloppy coding.

it's not about bad drivers, it's about the number of nested interrupts
not being limited by software right now.

> What we should consider regardless is disable the nesting of irqs for
> performance reasons but that's an independent matter

disabling nesting completely sounds a bit too aggressive, but limiting
the nesting is probably a good idea.

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Alan Cox]

On Gwe, 2004-09-10 at 16:28, Andrea Arcangeli wrote:
> > What we should consider regardless is disable the nesting of irqs for
> > performance reasons but that's an independent matter
> 
> disabling nesting completely sounds a bit too aggressive, but limiting
> the nesting is probably a good idea.

Thats a trivial change because you can poke the overflow IRQs into
IRQ_PENDING and then do a cleanup loop if it was hit

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Arjan van de Ven]

[-- Attachment #1: Type: text/plain, Size: 765 bytes --]


On Fri, Sep 10, 2004 at 05:28:52PM +0200, Andrea Arcangeli wrote:
> On Fri, Sep 10, 2004 at 05:15:38PM +0200, Arjan van de Ven wrote:
> > because it gives people a reason to do sloppy coding.
> 
> it's not about bad drivers, it's about the number of nested interrupts
> not being limited by software right now.
> 
> > What we should consider regardless is disable the nesting of irqs for
> > performance reasons but that's an independent matter
> 
> disabling nesting completely sounds a bit too aggressive, but limiting
> the nesting is probably a good idea.

disabling is actually not a bad idea; hard irq handlers run for a very short
time, but when they nest you effectively have like a semi context switch in
the middle of the work so performance suffers...


[-- Attachment #2: Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Andrea Arcangeli]

On Fri, Sep 10, 2004 at 05:34:21PM +0200, Arjan van de Ven wrote:
> disabling is actually not a bad idea; hard irq handlers run for a very short

you mean hard irq handlers "should run" for a very short time. There can
be slow hardware that needs a long time, and fast hardware that needs a
short time, and in turn it makes perfect sense to allow nesting to give
low latency to the "fast" onces, like it has always happened so far (not
only in linux AFIK). Disabling nesting completely sounds a very bad
idea to me, when "limiting nesting" can be achieved easily as confirmed
by Alan too.

> time, but when they nest you effectively have like a semi context switch in
> the middle of the work so performance suffers...

It's not nearly a semi context switch, it's like the timer irq that runs
on top of userspace, not nearly the cost of a context switch (no time
wasted calling the scheduler or the softirqs either for obvious reasons).

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Martin J. Bligh]

--Andrea Arcangeli <andrea@novell.com> wrote (on Sunday, September 12, 2004 16:17:01 +0200):

> On Fri, Sep 10, 2004 at 05:34:21PM +0200, Arjan van de Ven wrote:
>> disabling is actually not a bad idea; hard irq handlers run for a very short
> 
> you mean hard irq handlers "should run" for a very short time. There can
> be slow hardware that needs a long time, and fast hardware that needs a
> short time, and in turn it makes perfect sense to allow nesting to give
> low latency to the "fast" onces, like it has always happened so far (not
> only in linux AFIK). Disabling nesting completely sounds a very bad
> idea to me, when "limiting nesting" can be achieved easily as confirmed
> by Alan too.

IIRC, what we did in PTX was have 16 SPL levels, each interrupt was assigned
a prio, and higher prio interrupts could interrupt lower prio ones (but not
the same prio or higher). There's some support for that in the APIC, I think,
something like the high nybble is prio, and the low nybble is just an index.

M.

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Zwane Mwaikambo]

yOn Sun, 12 Sep 2004, Martin J. Bligh wrote:

> --Andrea Arcangeli <andrea@novell.com> wrote (on Sunday, September 12, 2004 16:17:01 +0200):
> 
> > On Fri, Sep 10, 2004 at 05:34:21PM +0200, Arjan van de Ven wrote:
> >> disabling is actually not a bad idea; hard irq handlers run for a very short
> > 
> > you mean hard irq handlers "should run" for a very short time. There can
> > be slow hardware that needs a long time, and fast hardware that needs a
> > short time, and in turn it makes perfect sense to allow nesting to give
> > low latency to the "fast" onces, like it has always happened so far (not
> > only in linux AFIK). Disabling nesting completely sounds a very bad
> > idea to me, when "limiting nesting" can be achieved easily as confirmed
> > by Alan too.
> 
> IIRC, what we did in PTX was have 16 SPL levels, each interrupt was assigned
> a prio, and higher prio interrupts could interrupt lower prio ones (but not
> the same prio or higher). There's some support for that in the APIC, I think,
> something like the high nybble is prio, and the low nybble is just an index.

Currently we do use priorities on i386/APIC, albeit unintentionally by 
assigning higher IRQs higher vectors resulting in a higher priority.
However interrupt priorities on non deterministic general purpose 
operating systems seems pointless for the vast majority of the devices 
plugged into boxes these days. Not to mention possible starvation issues 
from high frequency long running interrupts.

	Zwane

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Martin J. Bligh]

--Zwane Mwaikambo <zwane@linuxpower.ca> wrote (on Sunday, September 12, 2004 11:45:19 -0400):

> yOn Sun, 12 Sep 2004, Martin J. Bligh wrote:
> 
>> --Andrea Arcangeli <andrea@novell.com> wrote (on Sunday, September 12, 2004 16:17:01 +0200):
>> 
>> > On Fri, Sep 10, 2004 at 05:34:21PM +0200, Arjan van de Ven wrote:
>> >> disabling is actually not a bad idea; hard irq handlers run for a very short
>> > 
>> > you mean hard irq handlers "should run" for a very short time. There can
>> > be slow hardware that needs a long time, and fast hardware that needs a
>> > short time, and in turn it makes perfect sense to allow nesting to give
>> > low latency to the "fast" onces, like it has always happened so far (not
>> > only in linux AFIK). Disabling nesting completely sounds a very bad
>> > idea to me, when "limiting nesting" can be achieved easily as confirmed
>> > by Alan too.
>> 
>> IIRC, what we did in PTX was have 16 SPL levels, each interrupt was assigned
>> a prio, and higher prio interrupts could interrupt lower prio ones (but not
>> the same prio or higher). There's some support for that in the APIC, I think,
>> something like the high nybble is prio, and the low nybble is just an index.
> 
> Currently we do use priorities on i386/APIC, albeit unintentionally by 
> assigning higher IRQs higher vectors resulting in a higher priority.
> However interrupt priorities on non deterministic general purpose 
> operating systems seems pointless for the vast majority of the devices 
> plugged into boxes these days. Not to mention possible starvation issues 
> from high frequency long running interrupts.

Well, the idea is that long running interrupts get low prios ;-)

But yes, we're fairly non-deterministic, apart from the timer interrupt, etc
which are at one end of the scale, though I forget which ;-)

M.

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Lee Revell]

On Sun, 2004-09-12 at 11:45, Zwane Mwaikambo wrote:
> yOn Sun, 12 Sep 2004, Martin J. Bligh wrote:
> 
> > --Andrea Arcangeli <andrea@novell.com> wrote (on Sunday, September 12, 2004 16:17:01 +0200):
> > 
> > > On Fri, Sep 10, 2004 at 05:34:21PM +0200, Arjan van de Ven wrote:
> > >> disabling is actually not a bad idea; hard irq handlers run for a very short
> > > 
> > > you mean hard irq handlers "should run" for a very short time. There can
> > > be slow hardware that needs a long time, and fast hardware that needs a
> > > short time, and in turn it makes perfect sense to allow nesting to give
> > > low latency to the "fast" onces, like it has always happened so far (not
> > > only in linux AFIK). Disabling nesting completely sounds a very bad
> > > idea to me, when "limiting nesting" can be achieved easily as confirmed
> > > by Alan too.
> > 
> > IIRC, what we did in PTX was have 16 SPL levels, each interrupt was assigned
> > a prio, and higher prio interrupts could interrupt lower prio ones (but not
> > the same prio or higher). There's some support for that in the APIC, I think,
> > something like the high nybble is prio, and the low nybble is just an index.
> 
> Currently we do use priorities on i386/APIC, albeit unintentionally by 
> assigning higher IRQs higher vectors resulting in a higher priority.

Are these really "priorities" in real life?  I am not being facetious,
this is actually a common myth among users, that you will get better
performance by putting the device you care about on a "high priority"
irq or tweaking the priorities in your local APIC.  My impression was
that this is pointless because it only determines which interrupt the
CPU sees first if they fire at _exactly_ the same time.  Since we allow
interrupt to nest this does not matter in practice, right?

Lee 

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Zwane Mwaikambo]

On Wed, 20 Oct 2004, Lee Revell wrote:

> Are these really "priorities" in real life?  I am not being facetious,
> this is actually a common myth among users, that you will get better
> performance by putting the device you care about on a "high priority"
> irq or tweaking the priorities in your local APIC.  My impression was
> that this is pointless because it only determines which interrupt the
> CPU sees first if they fire at _exactly_ the same time.  Since we allow
> interrupt to nest this does not matter in practice, right?

Yep, it only affects which vector gets dispatched first in the multiple 
vectors queued scenario which doesn't really matter with nesting as you 
noted.

	Zwane

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Lee Revell]

On Fri, 2004-09-10 at 11:34, Arjan van de Ven wrote:
> On Fri, Sep 10, 2004 at 05:28:52PM +0200, Andrea Arcangeli wrote:
> > On Fri, Sep 10, 2004 at 05:15:38PM +0200, Arjan van de Ven wrote: 
> > > What we should consider regardless is disable the nesting of irqs for
> > > performance reasons but that's an independent matter
> > 
> > disabling nesting completely sounds a bit too aggressive, but limiting
> > the nesting is probably a good idea.
> 
> disabling is actually not a bad idea; hard irq handlers run for a very short
> time

The glaring exception is the IDE io completion, which can run for 2000+
usec even with a modern chipset and drive.  Here's a 600 usec trace:

http://krustophenia.net/testresults.php?dataset=2.6.8-rc4-bk3-O7#/var/www/2.6.8-rc4-bk3-O7/ide_irq_latency_trace.txt

The timer, RTC, and soundcard interrupts (among others) will not like
being delayed this long.  Ingo mentioned that this was not always done
in hardirq context; presumaby the I/O completion was done in a softirq
like SCSI.  What was the motivation for moving such a long code path
into the hard irq handler?

Lee

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Chris Wedgwood]

On Sun, Sep 12, 2004 at 03:18:07PM -0400, Lee Revell wrote:

> The glaring exception is the IDE io completion, which can run for
> 2000+ usec even with a modern chipset and drive.

does un-masking irqs help?

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Ingo Molnar]

* Chris Wedgwood <cw@f00f.org> wrote:

> > The glaring exception is the IDE io completion, which can run for
> > 2000+ usec even with a modern chipset and drive.
> 
> does un-masking irqs help?

no. A 2 msec nonpreemptable delay is a 2 msec delay, irqs on or off
alike.

but it's not a big problem with IRQ threading, there most hardirqs are
preemptable.

	Ingo

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Chris Wedgwood]

On Sun, Sep 12, 2004 at 09:35:42PM +0200, Ingo Molnar wrote:

> no. A 2 msec nonpreemptable delay is a 2 msec delay, irqs on or off
> alike.

maybe im retarded, 2ms seems like a long time --- is this strictly
necessary on modern chipsets?  surely we can fix the code to not do
this for most people without hosing their data?

> but it's not a big problem with IRQ threading, there most hardirqs
> are preemptable.

most?  what determines which and when?

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Ingo Molnar]

* Chris Wedgwood <cw@f00f.org> wrote:

> > but it's not a big problem with IRQ threading, there most hardirqs
> > are preemptable.
> 
> most?  what determines which and when?

it depends on how it uses spinlocks. E.g. most of the IDE irq handler
can run preemptible. Check out the voluntary-preempt patches (and VP
discussions on lkml).

	Ingo

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Andrea Arcangeli]

On Sun, Sep 12, 2004 at 09:35:42PM +0200, Ingo Molnar wrote:
> no. A 2 msec nonpreemptable delay is a 2 msec delay, irqs on or off
> alike.

irq-latency is an order of magnitude higher prio, than the scheduler
latency. above you only talk about scheduler latency.

with irq on and nested irqs enabled, the irq latency will have a chance
to be less than 2 msec.

take audio playback for example, with an huge buffering between
userspace and kernel and kernel submitting the next I/O via irqs, the
scheduler/userspace latency almost doesn't matter, only the
_irq_latency_ matters. 

hence I don't think not allowing nested irqs at all is a good idea and
it's a nice feature to support them.

ideally all irq handlers should be quick, but if some is not quick,
really it must enable irqs and allow other lowlatency interrupts to be
nested on top of it (like audio/video etc..).

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Lee Revell]

On Sun, 2004-09-12 at 16:33, Andrea Arcangeli wrote:
> On Sun, Sep 12, 2004 at 09:35:42PM +0200, Ingo Molnar wrote:
> > no. A 2 msec nonpreemptable delay is a 2 msec delay, irqs on or off
> > alike.
> 
> irq-latency is an order of magnitude higher prio, than the scheduler
> latency. above you only talk about scheduler latency.
> 

But in this case the hardirq handler can run for 2ms, which caused a
scheduler latency problem, because nothing could run but other IRQs. 
The IRQ threading in Ingo's patches solves the problem, and seems to me
to be the correct solution.

> hence I don't think not allowing nested irqs at all is a good idea and
> it's a nice feature to support them.

Agreed.  I was just pointing out that in addition to being a bad idea it
wouldn't work unless the IDE i/o completion issue is addressed.

The issue IIRC was the potential for stack overflow with 4K stacks if we
get deeply nested IRQs.  I believe the VP patches allow you to see the
IRQ nesting in the logs, you could probably grep the logs for these
incidents.

Seems like you could test the stack overflow theory by setting up a
stream of interrupts from the sound card and RTC to coincide with the
timer interrupt, then stressing the disks and network.

Lee

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Andrea Arcangeli]

On Sun, Sep 12, 2004 at 05:36:41PM -0400, Lee Revell wrote:
> But in this case the hardirq handler can run for 2ms, which caused a
> scheduler latency problem, because nothing could run but other IRQs. 
> The IRQ threading in Ingo's patches solves the problem, and seems to me
> to be the correct solution.

the irq threading must have a cost, doesn't it? I doubt you want to
offload irqs to a kernel thread on a server, *that* would be slow (irq
nesting is zerocost compared to scheduling a kernel thread).

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Lee Revell]

On Sun, 2004-09-12 at 18:07, Andrea Arcangeli wrote:
> On Sun, Sep 12, 2004 at 05:36:41PM -0400, Lee Revell wrote:
> > But in this case the hardirq handler can run for 2ms, which caused a
> > scheduler latency problem, because nothing could run but other IRQs. 
> > The IRQ threading in Ingo's patches solves the problem, and seems to me
> > to be the correct solution.
> 
> the irq threading must have a cost, doesn't it? I doubt you want to
> offload irqs to a kernel thread on a server, *that* would be slow (irq
> nesting is zerocost compared to scheduling a kernel thread).

Yes, on a server you would probably disable threading for the disk and
network IRQs (the VP patch lets you set this via /proc).  This feature
effectively gives you IPLs on Linux, albeit only two of them.  For
example to prevent heavy traffic on one network interface from impacting
the latency of the other, you could enable threading for the first and
disable it for the second.

I am still unsure why the IDE i/o completion is the one place that
breaks the assumption that hardirq handlers execute quickly.

Lee

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> Yes, on a server you would probably disable threading for the disk and
> network IRQs (the VP patch lets you set this via /proc).  This feature
> effectively gives you IPLs on Linux, albeit only two of them. [...]

nono, this has no relation to IPLs. IPLs are a pretty crude hack to
implement exclusion on a very (and too) broad level. IRQ threading is a
way to serialize hardirq contexts into a process context and to make
them schedulable and preemptable. It basically 'flattens out' all the
hardirq nesting (and parallelism) that may happen on a default kernel
and together with softirq 'flattening' it creates a deterministic
execution environment.

it is not intended for servers, due to the overhead of redirection. It's
for realtime workloads and for latency-sensitive audio desktop
workloads. For servers and normal desktops the current IRQ and softirq
model is pretty OK.

	Ingo

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Lee Revell]

On Mon, 2004-09-13 at 02:16, Ingo Molnar wrote:
> For servers and normal desktops the current IRQ and softirq
> model is pretty OK.
> 

Agreed.  I was just pointing out that it's not safe to assume that all
hardirq handlers execute quickly.  This doesn't seem to be a problem for
normal workloads.

Lee

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Bill Davidsen]

Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> 
>>Yes, on a server you would probably disable threading for the disk and
>>network IRQs (the VP patch lets you set this via /proc).  This feature
>>effectively gives you IPLs on Linux, albeit only two of them. [...]
> 
> 
> nono, this has no relation to IPLs. IPLs are a pretty crude hack to
> implement exclusion on a very (and too) broad level. IRQ threading is a
> way to serialize hardirq contexts into a process context and to make
> them schedulable and preemptable. It basically 'flattens out' all the
> hardirq nesting (and parallelism) that may happen on a default kernel
> and together with softirq 'flattening' it creates a deterministic
> execution environment.
> 
> it is not intended for servers, due to the overhead of redirection. It's
> for realtime workloads and for latency-sensitive audio desktop
> workloads. For servers and normal desktops the current IRQ and softirq
> model is pretty OK.

Okay, I'll be the one to ask... what overload of the IPL acronym are you 
using here? I asked google and several jargon files, and they all say 
that IPL (initial program load) is IBMspeak for cold boot. Somehow I 
don't think that's what you mean here.

-- 
    -bill davidsen (davidsen@tmr.com)
"The secret to procrastination is to put things off until the
  last possible moment - but no longer"  -me

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Andrea Arcangeli]

On Wed, Sep 15, 2004 at 03:43:54PM -0400, Bill Davidsen wrote:
> Okay, I'll be the one to ask... what overload of the IPL acronym are you 

in this context I parsed it as interrupt priority level.

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Ingo Molnar]

* Bill Davidsen <davidsen@tmr.com> wrote:

> Okay, I'll be the one to ask... what overload of the IPL acronym are
> you using here? I asked google and several jargon files, and they all
> say that IPL (initial program load) is IBMspeak for cold boot. Somehow
> I don't think that's what you mean here.

i understood it as Interrupt Privilege Levels. The notion of having some
sort of scalar 'limit' - all interrupts with a higher priority than that
will execute, all interrupts with lower priority will block.  This is a
fundamentally dodgy concept because in reality interrupt sources are
independent entities so the natural description for of them is a bitmask
(or an array, or whatever), not a level.

	Ingo

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Lee Revell]

On Wed, 2004-09-15 at 16:00, Ingo Molnar wrote:
> * Bill Davidsen <davidsen@tmr.com> wrote:
> 
> > Okay, I'll be the one to ask... what overload of the IPL acronym are
> > you using here? I asked google and several jargon files, and they all
> > say that IPL (initial program load) is IBMspeak for cold boot. Somehow
> > I don't think that's what you mean here.
> 
> i understood it as Interrupt Privilege Levels. The notion of having some
> sort of scalar 'limit' - all interrupts with a higher priority than that
> will execute, all interrupts with lower priority will block.  This is a
> fundamentally dodgy concept because in reality interrupt sources are
> independent entities so the natural description for of them is a bitmask
> (or an array, or whatever), not a level.
> 

Yeah, I was talking about interrupt priority levels.  I have to admit my
only exposure to them is a book on Solaris, they are intended as a
mechanism to deal with priority inversions.  Sounds like a neat trick on
paper but they don't seem to be applicable here.

Lee

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Timothy Miller]

Ingo Molnar wrote:

> 
> it is not intended for servers, due to the overhead of redirection. It's
> for realtime workloads and for latency-sensitive audio desktop
> workloads. For servers and normal desktops the current IRQ and softirq
> model is pretty OK.
> 


I commented previously about the preemption allowing the kernel to keep 
more resources busy.  So if you have 6 disk controllers and 4 NICs, the 
preemption can allow the kernel to keep more of them busier at the same 
time, while without preemption, one resource might get starved while 
non-preemptable code is servicing another.

This makes the overhead very much worth it.

Are there test results which demonstrate that this theory doesn't apply 
practically to real-world server loads?

I would expect a very busy server to be helped by this in a noticable 
way, but then I am often short-sighted.  :)

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Jens Axboe]

On Sun, Sep 12 2004, Lee Revell wrote:
> I am still unsure why the IDE i/o completion is the one place that
> breaks the assumption that hardirq handlers execute quickly.

I'm not sure why you think so. You also mention IDE as being the
'glaring exception' and it's definitely wrong. Most drivers run the
completion from hardirq context, SCSI is the exception since it does it
defers the completion to softirq context.

-- 
Jens Axboe

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Andrea Arcangeli]

On Mon, Sep 13, 2004 at 09:32:59AM +0200, Jens Axboe wrote:
> completion from hardirq context, SCSI is the exception since it does it
> defers the completion to softirq context.

which btw doesn't really make much difference at all to run it in irq
context with irq enabled since it'll run from irq context anyways and
you'll still depend on nested hardirqs to avoid huge latencies from
softirq handlers (the most difference between softirq and hardirq with
irq enabled happens if you can use spin_lock_bh instead of spin_lock_irq
in the critical sections to protect against other cpus).

softirq runs inside an hardirq in all common cases, so a softirq is
still an hardirq and to allow other hardirq to run you need nested
interrupts in the hardware (which again explains why it's a bad idea to
forbid nesting by design, and really I don't buy the slowdown argument,
enter/exit kernel would happen anyways, it's just the pipeline will be
stalled once and the cache may be trashed a bit, but irqs have not an
huge memory footprint anyways).

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Arjan van de Ven]

[-- Attachment #1: Type: text/plain, Size: 903 bytes --]

On Mon, Sep 13, 2004 at 12:31:58PM +0200, Andrea Arcangeli wrote:
> On Mon, Sep 13, 2004 at 09:32:59AM +0200, Jens Axboe wrote:
> > completion from hardirq context, SCSI is the exception since it does it
> > defers the completion to softirq context.
> 
> which btw doesn't really make much difference at all to run it in irq
> context with irq enabled since it'll run from irq context anyways and
> you'll still depend on nested hardirqs to avoid huge latencies from
> softirq handlers (the most difference between softirq and hardirq with
> irq enabled happens if you can use spin_lock_bh instead of spin_lock_irq
> in the critical sections to protect against other cpus).

and there is improved batching under really high (irq) load, esp when you
deal with stupid hw that can't give an irq for multiple packets or IO
requests at the same time (qlogic used to be like that until the driver got
fixed)


[-- Attachment #2: Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Jens Axboe]

On Mon, Sep 13 2004, Andrea Arcangeli wrote:
> On Mon, Sep 13, 2004 at 09:32:59AM +0200, Jens Axboe wrote:
> > completion from hardirq context, SCSI is the exception since it does it
> > defers the completion to softirq context.
> 
> which btw doesn't really make much difference at all to run it in irq
> context with irq enabled since it'll run from irq context anyways and
> you'll still depend on nested hardirqs to avoid huge latencies from
> softirq handlers (the most difference between softirq and hardirq with
> irq enabled happens if you can use spin_lock_bh instead of spin_lock_irq
> in the critical sections to protect against other cpus).
> 
> softirq runs inside an hardirq in all common cases, so a softirq is
> still an hardirq and to allow other hardirq to run you need nested
> interrupts in the hardware (which again explains why it's a bad idea to
> forbid nesting by design, and really I don't buy the slowdown argument,
> enter/exit kernel would happen anyways, it's just the pipeline will be
> stalled once and the cache may be trashed a bit, but irqs have not an
> huge memory footprint anyways).

It also does the completion lockless from the softirq context, which I
guess means it's preemptable with Ingo's patches.

The main point of the mail was that SCSI is the exception, not IDE. This
is important to note if Lee thinks that the latency problems with io
completion is IDE only and have been fixed.

-- 
Jens Axboe

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Timothy Miller]

Lee Revell wrote:
> On Sun, 2004-09-12 at 18:07, Andrea Arcangeli wrote:
> 
>>On Sun, Sep 12, 2004 at 05:36:41PM -0400, Lee Revell wrote:
>>
>>>But in this case the hardirq handler can run for 2ms, which caused a
>>>scheduler latency problem, because nothing could run but other IRQs. 
>>>The IRQ threading in Ingo's patches solves the problem, and seems to me
>>>to be the correct solution.
>>
>>the irq threading must have a cost, doesn't it? I doubt you want to
>>offload irqs to a kernel thread on a server, *that* would be slow (irq
>>nesting is zerocost compared to scheduling a kernel thread).
> 
> 
> Yes, on a server you would probably disable threading for the disk and
> network IRQs (the VP patch lets you set this via /proc).  This feature
> effectively gives you IPLs on Linux, albeit only two of them.  For
> example to prevent heavy traffic on one network interface from impacting
> the latency of the other, you could enable threading for the first and
> disable it for the second.
> 
> I am still unsure why the IDE i/o completion is the one place that
> breaks the assumption that hardirq handlers execute quickly.


On the one hand, all this preemption introduces overhead which increases 
latency and reduces throughput for any one individual activity.

On the other hand, the preemption allows the CPU to attend to more 
different resources at the same time, keeping more resources busy. 
Under heavy load, this can increase overall system throughput significantly.

So, both approaches can increase throughput in different ways.  Where is 
the balance between the two which yields the most satisfactory result?

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Alan Cox]

On Sul, 2004-09-12 at 22:36, Lee Revell wrote:
> > hence I don't think not allowing nested irqs at all is a good idea and
> > it's a nice feature to support them.
> 
> Agreed.  I was just pointing out that in addition to being a bad idea it
> wouldn't work unless the IDE i/o completion issue is addressed.

Limited nesting is fine and we already have the framework to make it
easy to limit nesting nicely if we want to. Doesn't alter the fact the
IDE one wants looking at once the more serious IDE work is done.

Alan

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Bill Davidsen]

Lee Revell wrote:
> On Fri, 2004-09-10 at 11:34, Arjan van de Ven wrote:
> 
>>On Fri, Sep 10, 2004 at 05:28:52PM +0200, Andrea Arcangeli wrote:
>>
>>>On Fri, Sep 10, 2004 at 05:15:38PM +0200, Arjan van de Ven wrote: 
>>>
>>>>What we should consider regardless is disable the nesting of irqs for
>>>>performance reasons but that's an independent matter
>>>
>>>disabling nesting completely sounds a bit too aggressive, but limiting
>>>the nesting is probably a good idea.
>>
>>disabling is actually not a bad idea; hard irq handlers run for a very short
>>time
> 
> 
> The glaring exception is the IDE io completion, which can run for 2000+
> usec even with a modern chipset and drive.  Here's a 600 usec trace:
> 
> http://krustophenia.net/testresults.php?dataset=2.6.8-rc4-bk3-O7#/var/www/2.6.8-rc4-bk3-O7/ide_irq_latency_trace.txt
> 
> The timer, RTC, and soundcard interrupts (among others) will not like
> being delayed this long.  Ingo mentioned that this was not always done
> in hardirq context; presumaby the I/O completion was done in a softirq
> like SCSI.  What was the motivation for moving such a long code path
> into the hard irq handler?

Certainly if you run ppp the serial port won't like being ignored that 
long, and if you pull down data on a parallel port that really won't 
like it. The soundcard is probably only a problem if you're recording 
input, in spite of some posts here about skipping, the world doesn't end 
if you get a skip, although 2ms shouldn't cause that anyway.

-- 
bill davidsen <davidsen@tmr.com>
   CTO TMR Associates, Inc
   Doing interesting things with small computers since 1979

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Lee Revell]

On Mon, 2004-09-13 at 16:26, Bill Davidsen wrote:
> Certainly if you run ppp the serial port won't like being ignored that 
> long, and if you pull down data on a parallel port that really won't 
> like it. The soundcard is probably only a problem if you're recording 
> input, in spite of some posts here about skipping, the world doesn't end 
> if you get a skip, although 2ms shouldn't cause that anyway.

The world also doesn't end if your web server returns a "Server too
busy" error either.

Sorry, wrong answer.

Lee

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Timothy Miller]

Arjan van de Ven wrote:
> On Fri, Sep 10, 2004 at 05:28:52PM +0200, Andrea Arcangeli wrote:
> 
>>On Fri, Sep 10, 2004 at 05:15:38PM +0200, Arjan van de Ven wrote:
>>
>>>because it gives people a reason to do sloppy coding.
>>
>>it's not about bad drivers, it's about the number of nested interrupts
>>not being limited by software right now.
>>
>>
>>>What we should consider regardless is disable the nesting of irqs for
>>>performance reasons but that's an independent matter
>>
>>disabling nesting completely sounds a bit too aggressive, but limiting
>>the nesting is probably a good idea.
> 
> 
> disabling is actually not a bad idea; hard irq handlers run for a very short
> time, but when they nest you effectively have like a semi context switch in
> the middle of the work so performance suffers...
> 


The rules about how long a hard irq would be allowed to run would have 
to be draconian.

... not that that would be a problem.

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Arjan van de Ven]

On Wed, Oct 20, 2004 at 11:46:32AM -0400, Timothy Miller wrote:
> The rules about how long a hard irq would be allowed to run would have 
> to be draconian.

they already are.

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Lee Revell]

On Wed, 2004-10-20 at 11:35, Arjan van de Ven wrote:
> On Wed, Oct 20, 2004 at 11:46:32AM -0400, Timothy Miller wrote:
> > The rules about how long a hard irq would be allowed to run would have 
> > to be draconian.
> 
> they already are.

The IDE I/O completion in hardirq context means that one can run for
almost 3ms.  Apparently at OLS it was decided that the target for
desktop responsiveness was 1ms.  So this is a real problem.

What exactly do you mean by "draconian"?

Lee 

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Andrea Arcangeli]

On Wed, Oct 20, 2004 at 12:39:06PM -0400, Lee Revell wrote:
> The IDE I/O completion in hardirq context means that one can run for
> almost 3ms.  Apparently at OLS it was decided that the target for
> desktop responsiveness was 1ms.  So this is a real problem.

comparing netsted irqs to a context switch is a red herring.

nested irqs statistically do good, with long irq handlers vs short irq
handlers. The irq handlers that really know to be extremely quick aren't
forced to re-enable irqs, so they can enforce no-nesting by themself.
Enforcing it globally sounds bad to me.

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Lee Revell]

On Wed, 2004-10-20 at 12:50, Andrea Arcangeli wrote:
> On Wed, Oct 20, 2004 at 12:39:06PM -0400, Lee Revell wrote:
> > The IDE I/O completion in hardirq context means that one can run for
> > almost 3ms.  Apparently at OLS it was decided that the target for
> > desktop responsiveness was 1ms.  So this is a real problem.
> 
> comparing netsted irqs to a context switch is a red herring.

This was not my point, I agree that the two have nothing to do with each
other.  But if a hardirq handler runs for 3ms then no user code can run
for 3ms.  Therefore this is a problem if our goal for desktop response
is 1ms.

Lee

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Andrea Arcangeli]

On Wed, Oct 20, 2004 at 12:55:17PM -0400, Lee Revell wrote:
> This was not my point, I agree that the two have nothing to do with each
> other.  But if a hardirq handler runs for 3ms then no user code can run
> for 3ms.  Therefore this is a problem if our goal for desktop response
> is 1ms.

I sure agree it's a problem, but not always userspace code needs to run
for the user not to notice. With ring buffers in the kernel for playback
all you need is a nested irq for the user not to notice skips.

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Lee Revell]

On Wed, 2004-10-20 at 13:08, Andrea Arcangeli wrote:
> On Wed, Oct 20, 2004 at 12:55:17PM -0400, Lee Revell wrote:
> > This was not my point, I agree that the two have nothing to do with each
> > other.  But if a hardirq handler runs for 3ms then no user code can run
> > for 3ms.  Therefore this is a problem if our goal for desktop response
> > is 1ms.
> 
> I sure agree it's a problem, but not always userspace code needs to run
> for the user not to notice. With ring buffers in the kernel for playback
> all you need is a nested irq for the user not to notice skips.
> 

I was thinking of X, not audio.  This might be a problem for AV
playback.  Maybe that would depend on if DRI was in use.  Anyway I use a
low latency kernel because i need sub-ms response, I was just pointing
out that the current behavior might not be compatible with the 1ms
target.

However with the patch in -mm that enables setting max_sectors_kb, the
1ms goal can be achived by lowering this to 256 or so.  If I were
shipping a desktop distro I would lower this by default.  My tests show
the slowdown is barely measurable if at all.

Lee

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Andrea Arcangeli]

On Wed, Oct 20, 2004 at 01:15:16PM -0400, Lee Revell wrote:
> I was thinking of X, not audio.  This might be a problem for AV

sure I got it, maybe I wasn't clear but I wasn't in disagrement with
you, I've never said IDE is right taking 3msec, I just taken the example
you made, to add some more comment from a different point of view. Right
now IDE takes 3 msec, we can turn it down to 1msec (or better move it to
bh handler) but still nesting can be beneficial. It's up to the lowlevel
driver to re-enable irqs if it's doing something that takes a long time.
If we want to do something global that would be a nesting-max-limit
(which is a sort of requirement with 4k stack IMHO), not a no-nesting,
since no-nesting can be already enforced by every single irq handler,
and clearly many are re-enabling irqs like IDE since they take a long
time. Every time an irq handlers re-enable irqs means it's _asking_ for
nesting to happen.

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Timothy Miller]

Lee Revell wrote:
> On Wed, 2004-10-20 at 11:35, Arjan van de Ven wrote:
> 
>>On Wed, Oct 20, 2004 at 11:46:32AM -0400, Timothy Miller wrote:
>>
>>>The rules about how long a hard irq would be allowed to run would have 
>>>to be draconian.
>>
>>they already are.
> 
> 
> The IDE I/O completion in hardirq context means that one can run for
> almost 3ms.  Apparently at OLS it was decided that the target for
> desktop responsiveness was 1ms.  So this is a real problem.

What is happening that takes 3ms, and why can't it be broken up?

Are you talking here about PIO?  Is there a limited amount of time you 
have to do the PIO before the data goes away?  It may be acceptable to 
just live with PIO being slow, since high-performance systems will all 
be using DMA anyhow.

> 
> What exactly do you mean by "draconian"?

In this case, I mean extremely harsh and restrictive.  Usually, it means 
"excessively harsh", but in this case, I mean that in a good way.  :)

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Lee Revell]

On Wed, 2004-10-20 at 13:58, Timothy Miller wrote:
> Lee Revell wrote:
> > On Wed, 2004-10-20 at 11:35, Arjan van de Ven wrote:
> > 
> >>On Wed, Oct 20, 2004 at 11:46:32AM -0400, Timothy Miller wrote:
> >>
> >>>The rules about how long a hard irq would be allowed to run would have 
> >>>to be draconian.
> >>
> >>they already are.
> > 
> > 
> > The IDE I/O completion in hardirq context means that one can run for
> > almost 3ms.  Apparently at OLS it was decided that the target for
> > desktop responsiveness was 1ms.  So this is a real problem.
> 
> What is happening that takes 3ms, and why can't it be broken up?
> 

http://lkml.org/lkml/2004/7/24/26

> Are you talking here about PIO?  Is there a limited amount of time you 
> have to do the PIO before the data goes away?  It may be acceptable to 
> just live with PIO being slow, since high-performance systems will all 
> be using DMA anyhow.
> 

Nope, DMA.

> > 
> > What exactly do you mean by "draconian"?
> 
> In this case, I mean extremely harsh and restrictive.  Usually, it means 
> "excessively harsh", but in this case, I mean that in a good way.  :)
> 

Heh, I know what the word means, I was asking what the specific rules
are that could be considered as such.

Lee

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Timothy Miller]

Lee Revell wrote:

>>>
>>>The IDE I/O completion in hardirq context means that one can run for
>>>almost 3ms.  Apparently at OLS it was decided that the target for
>>>desktop responsiveness was 1ms.  So this is a real problem.
>>
>>What is happening that takes 3ms, and why can't it be broken up?
>>
> 
> 
> http://lkml.org/lkml/2004/7/24/26

Ah, I see.  Lots of cache misses.



> 
>>>What exactly do you mean by "draconian"?
>>
>>In this case, I mean extremely harsh and restrictive.  Usually, it means 
>>"excessively harsh", but in this case, I mean that in a good way.  :)
>>
> 
> 
> Heh, I know what the word means, I was asking what the specific rules
> are that could be considered as such.
> 
>

I believe using the word 'draconian' was a poor choice on my part. 
Maybe "strictly bounded with no exceptions" would be a better way of 
expressing it.

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Lee Revell]

On Wed, 2004-10-20 at 14:25, Timothy Miller wrote:
> I believe using the word 'draconian' was a poor choice on my part. 
> Maybe "strictly bounded with no exceptions" would be a better way of 
> expressing it.
> 

What I was asking is what rules you are referring to.  Do you mean
obvious ones like "no sleeping in ISRs", or guidelines like "ISRs must
execute quickly" where "quickly" is defined by some consensus?

Anyway my point is that if you mean the latter then the rules are not
quite draconian.  Fortunately this seems to be the only exception, and
the problem and the fix seem to be well understood.

I should add that unless LBA48 is in use the max request size is 128KB
which effectively hides the issue.

Lee

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Alan Cox]

On Mer, 2004-10-20 at 17:39, Lee Revell wrote:
> The IDE I/O completion in hardirq context means that one can run for
> almost 3ms.  Apparently at OLS it was decided that the target for
> desktop responsiveness was 1ms.  So this is a real problem.
> 
> What exactly do you mean by "draconian"?

It means "fix the ide layer", patches welcome. 

Doing 4K IRQ stacks is actually not the problem - nor really is nesting.
If that worries you then get_free_page() the irq structure and use the
rest for the stack for _that_ IRQ handler. You won't need that much
memory.

Alan

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Lee Revell]

On Wed, 2004-10-20 at 14:51, Alan Cox wrote:
> On Mer, 2004-10-20 at 17:39, Lee Revell wrote:
> > The IDE I/O completion in hardirq context means that one can run for
> > almost 3ms.  Apparently at OLS it was decided that the target for
> > desktop responsiveness was 1ms.  So this is a real problem.
> > 
> > What exactly do you mean by "draconian"?
> 
> It means "fix the ide layer", patches welcome. 
> 

In addition to the IDE layer how about this (from the link in my
previous mail):

On Sat, 2004-07-24 at 02:43, Ingo Molnar wrote: 
> 
> Another thing would be to create a compound structure for bio and
> [typical sizes of] bio->bi_io_vec and free them as one entity, this
> would get rid of one of the cachemisses. (there cannot be a 3-way
> compound structure that includes the bh too because the bh is freed
> later on by ext3.)
> 

Sounds like a big win for any user of the bio layer, if it's as
straightforward as Ingo says...

Lee

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Martin J. Bligh]

--Hugh Dickins <hugh@veritas.com> wrote (on Friday, September 10, 2004 16:07:21 +0100):

> On Fri, 10 Sep 2004, Martin J. Bligh wrote:
>> 
>> I agree about killing anything but 4K stacks though - having the single
>> page is very compelling - not only can we allocate it easier, but we can
>> also use cache-hot pages from the hot list.
> 
> I think we all agree that's a promising future, and a good discipline.
> But I'm not the only one to doubt we're there yet.
> 
> Chris's patch seems eminently sensible to me.  Why should having separate
> interrupt stack depend on whether you're configured for 4K or 8K stacks?
> 
> Wasn't Andrea worried, a couple of months back, about nested interrupts
> overflowing the 4K interrupt stack?  He was trying to work out how to
> have an 8K interrupt stack even with the 4K task stack, proposed thread
> info at both top and bottom of stack; but his "current" still looked to
> me like it'd be significantly more costly than the present one.
> 
> I'm all for Chris's patch.

I have no problem with 8K interrupt stacks - they're static, and per CPU,
so I doubt anyone cares ... 

But 8K task stacks + interrupt stacks seems to just encourage bloat to me.
And if you agree that we're going to 4K, I don't really see the point - 
if people are really hitting problems (I don't recall any actual reports)
then I'd prefer to see them fixed properly by poking the fat bloater with
a big pin.

M.

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Timothy Miller]

Hugh Dickins wrote:

> 
> Wasn't Andrea worried, a couple of months back, about nested interrupts
> overflowing the 4K interrupt stack?  He was trying to work out how to
> have an 8K interrupt stack even with the 4K task stack, proposed thread
> info at both top and bottom of stack; but his "current" still looked to
> me like it'd be significantly more costly than the present one.

Yeah, one of the driving forces behind going to 4K task stacks is that 
you significantly reduce the amount of kernel memory reserved for user 
processes.  But then, there came the requirement to keep around a 
handful of irq stacks which doesn't hurt.

Given the finite and small number of IRQ stacks required, I see no 
reason not to make them 8K, other than for elegance sake.  You're not 
really wasting much memory by giving IRQ's 8k stacks.

Yes, I know, it does feel icky, though.  :)

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Arjan van de Ven]

> 
> Given the finite and small number of IRQ stacks required, I see no 
> reason not to make them 8K, other than for elegance sake.  You're not 
> really wasting much memory by giving IRQ's 8k stacks.

really sucky though; look at how current is implemented.

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Alexander E. Patrakov]

Martin J. Bligh wrote:
> --Alan Cox <alan@lxorguk.ukuu.org.uk> wrote (on Friday, September 10, 2004 10:14:11 +0100):
>>Its probably appropriate to drop gcc 2.x support at that point too since
>>it's the major cause of remaining problems
> 
> What problems does it cause? 2.95.4 still seems to work fine for me.

The latest gcc2 on the ftp.gnu.org site is gcc 2.95.3. There is 
officially no such thing as "gcc 2.95.4". Probably you are talking about 
a patched version of some gcc2 cvs snapshot - that's what distros 
provide. Please specify exactly what gcc version you are talking about.

And there _is_ problem with gcc-2.95.3-compiled kernel: latest cvs glibc 
testsuite segfaults in nptl tests. There are no failures with the kernel 
identically configured, but compiled with gcc 3.3.4 or 3.4.1. So gcc 
2.95.3 as supplied by gnu.org miscompiles the kernel (futexes?). Either 
fix the kernel or drop gcc2 support.

-- 
Alexander E. Patrakov

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Nathan Bryant]

Alexander E. Patrakov wrote:
> Martin J. Bligh wrote:
> 
>> --Alan Cox <alan@lxorguk.ukuu.org.uk> wrote (on Friday, September 10, 
>> 2004 10:14:11 +0100):
>>
>>> Its probably appropriate to drop gcc 2.x support at that point too since
>>> it's the major cause of remaining problems
>>
>>
>> What problems does it cause? 2.95.4 still seems to work fine for me.
> 
> 
> The latest gcc2 on the ftp.gnu.org site is gcc 2.95.3. There is 
> officially no such thing as "gcc 2.95.4". Probably you are talking about 
> a patched version of some gcc2 cvs snapshot - that's what distros 
> provide. Please specify exactly what gcc version you are talking about.

2.95.4, if I remember correctly, contained fixes that went onto the gcc 
2.95 branch after 2.95.3 was released. Some of the fixes were for 
Linux-2.2/2.4 and glibc2.2 compatibility. This compiler was distributed 
by Debian, I think.

> 
> And there _is_ problem with gcc-2.95.3-compiled kernel: latest cvs glibc 
> testsuite segfaults in nptl tests. There are no failures with the kernel 
> identically configured, but compiled with gcc 3.3.4 or 3.4.1. So gcc 
> 2.95.3 as supplied by gnu.org miscompiles the kernel (futexes?). Either 
> fix the kernel or drop gcc2 support.
> 

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Martin J. Bligh]

>>>> Its probably appropriate to drop gcc 2.x support at that point too since
>>>> it's the major cause of remaining problems
>>> 
>>> 
>>> What problems does it cause? 2.95.4 still seems to work fine for me.
>> 
>> 
>> The latest gcc2 on the ftp.gnu.org site is gcc 2.95.3. There is 
>> officially no such thing as "gcc 2.95.4". Probably you are talking about 
>> a patched version of some gcc2 cvs snapshot - that's what distros 
>> provide. Please specify exactly what gcc version you are talking about.
> 
> 2.95.4, if I remember correctly, contained fixes that went onto the gcc 2.95 branch after 2.95.3 was released. Some of the fixes were for Linux-2.2/2.4 and glibc2.2 compatibility. This compiler was distributed by Debian, I think.
> 
>> 
>> And there _is_ problem with gcc-2.95.3-compiled kernel: latest cvs glibc 
>> testsuite segfaults in nptl tests. There are no failures with the kernel 
>> identically configured, but compiled with gcc 3.3.4 or 3.4.1. So gcc 
>> 2.95.3 as supplied by gnu.org miscompiles the kernel (futexes?). Either 
>> fix the kernel or drop gcc2 support.

If 2.95.3 is broken, then drop that. Before dropping gcc2 support entirely,
you should prove all versions are broken.

And yes, the 2.95.4 I've been using happily for about 4 years is Debian.
Not only does it compile about twice as quickly, it produces better code
than anything else I've tested (though 3.x is finally getting close).

M.

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Bill Davidsen]

Martin J. Bligh wrote:
> --Alan Cox <alan@lxorguk.ukuu.org.uk> wrote (on Friday, September 10, 2004 10:14:11 +0100):
> 
> 
>>On Gwe, 2004-09-10 at 07:40, Arjan van de Ven wrote:
>>
>>>Well I always assumed the future plan was to remove 8k stacks entirely;
>>>4k+irqstacks and 8k basically have near comparable stack space, with
>>>this patch you create an option that has more but that is/should be
>>>deprecated. I'm not convinced that's a good idea.
>>
>>Its probably appropriate to drop gcc 2.x support at that point too since
>>it's the major cause of remaining problems
> 
> 
> What problems does it cause? 2.95.4 still seems to work fine for me.

The RH7.3 remnant 2.96 seems to work for me on my expendable test box, 
and I don't really have space for an upgrade. I haven't seen any 
problems, other than old systems being dog slow. And gcc3 is even slower 
it seems, although my machines running that have enough CPU to pretty 
much overpower the bloat.
> 
> I agree about killing anything but 4K stacks though - having the single
> page is very compelling - not only can we allocate it easier, but we can
> also use cache-hot pages from the hot list.

I have no problems with making 4k the default, but I'd really like the 
option of going back to 8k when I see problems, just to eliminate that 
as a possible cause of hangs or other instances of evil.

Is everyone claiming that everything in the kernel is 4k safe now? Or is 
"stable" total fiction? The 8k code doesn't take up that much space, it 
is well tested, and if you make 4k the default most people will try it 
with 4k anyway.

-- 
    -bill davidsen (davidsen@tmr.com)
"The secret to procrastination is to put things off until the
  last possible moment - but no longer"  -me

Re: [PATCH 1/3] Separate IRQ-stacks from 4K-stacks option

[Alan Cox]

On Gwe, 2004-09-10 at 00:25, Chris Wedgwood wrote:
> Right now CONFIG_4KSTACKS implies IRQ-stacks.  Some people though
> really need 8K stacks and it would be nice to have IRQ-stacks for them
> too.

This is a lot of added code and complexity that does nothing. In
8K stack mode without IRQ stacks you already can only safely use 4K.
So any code that is broken in 4K stack mode is broken in the current
8K stack mode although it'll fail less often since the failure will
depend upon random IRQ/other timings.

8K + IRQ stacks is just making the stacks bigger (which is expensive)
and stressing the vm more. Fix the broken code instead, or just stop
supporting gcc 2.9x which will fix most of it for you