[PATCH 0/3] ARM 4Kstacks: introduction

[PATCH 0/3] ARM 4Kstacks: introduction

[Tim Bird]

I'm about to submit a set of patches (really pretty small)
to add 4K stack support to ARM (defaulted to 'N').

This has been kicking around in my Sony tree for a few years,
and it's about time to mainline it.  The first patch is
the actual 4KSTACKS patch.  See subsequent patches
for tools to help with stack reduction to avoid the problems
that apparently led to the removal of this feature on x86.

This set includes:
  0001-Add-support-for-4K-kernel-stacks-to-ARM.patch
  0002-Add-static-kernel-function-stack-size-analyzer-for-ARM.patch
  0003-Decrease-poll-and-select-stack-usage-when-using-4K-stacks.patch

=============================
Tim Bird
Architecture Group Chair, CE Workgroup of the Linux Foundation
Senior Staff Engineer, Sony Network Entertainment
=============================

[PATCH 1/3] ARM 4Kstacks: Add support for 4K kernel stacks to ARM

[Tim Bird]

This saves 4K per kernel thread and user-space process.
For systems with lots of threads, this can save
100s of K of memory.

Signed-off-by: Tim Bird <tim.bird@am.sony.com>
---
 arch/arm/Kconfig                   |   13 +++++++++++++
 arch/arm/include/asm/thread_info.h |    7 ++++++-
 arch/arm/kernel/entry-header.S     |    4 ++--
 arch/arm/mm/proc-macros.S          |    4 ++--
 4 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 3146ed3..321ee11 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -1555,6 +1555,19 @@ config OABI_COMPAT
 	  UNPREDICTABLE (in fact it can be predicted that it won't work
 	  at all). If in doubt say Y.

+config 4KSTACKS
+	bool "Use 4K kernel stacks instead of 8K"
+	default n
+	help
+	  This option will use less memory for each kernel stack in the
+	  system, which can save potentially hundreds of kilobytes of memory.
+	  It also helps reduce fragmentation in kernel memory.  However,
+	  it can be dangerous since not all kernel code paths are thrifty
+	  in their use of the stack.  Unless you have carefully monitored your
+	  kernel stack usage and verified that your programs and kernel
+	  usage patterns will not overflow 4K, you should leave this set
+	  to N.
+
 config ARCH_HAS_HOLES_MEMORYMODEL
 	bool

diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h
index 7b5cc8d..3021d06 100644
--- a/arch/arm/include/asm/thread_info.h
+++ b/arch/arm/include/asm/thread_info.h
@@ -15,8 +15,13 @@
 #include <linux/compiler.h>
 #include <asm/fpstate.h>

+#ifndef CONFIG_4KSTACKS
 #define THREAD_SIZE_ORDER	1
-#define THREAD_SIZE		8192
+#else
+#define THREAD_SIZE_ORDER	0
+#endif
+
+#define THREAD_SIZE		(4096 << THREAD_SIZE_ORDER)
 #define THREAD_START_SP		(THREAD_SIZE - 8)

 #ifndef __ASSEMBLY__
diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S
index 9a8531e..c818505 100644
--- a/arch/arm/kernel/entry-header.S
+++ b/arch/arm/kernel/entry-header.S
@@ -109,8 +109,8 @@
 	.endm

 	.macro	get_thread_info, rd
-	mov	\rd, sp, lsr #13
-	mov	\rd, \rd, lsl #13
+	mov	\rd, sp, lsr #12 + THREAD_SIZE_ORDER
+	mov	\rd, \rd, lsl #12 + THREAD_SIZE_ORDER
 	.endm

 	@
diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S
index 307a4de..27c0907 100644
--- a/arch/arm/mm/proc-macros.S
+++ b/arch/arm/mm/proc-macros.S
@@ -30,8 +30,8 @@
  * act_mm - get current->active_mm
  */
 	.macro	act_mm, rd
-	bic	\rd, sp, #8128
-	bic	\rd, \rd, #63
+	mov	\rd, sp, lsr #12 + THREAD_SIZE_ORDER
+	mov	\rd, \rd, lsl #12 + THREAD_SIZE_ORDER
 	ldr	\rd, [\rd, #TI_TASK]
 	ldr	\rd, [\rd, #TSK_ACTIVE_MM]
 	.endm
-- 
1.6.6

[PATCH 2/3] ARM: Add static kernel function stack size analyzer, for ARM

[Tim Bird]

This tool can be used to analyze the function stack size for ARM kernels,
statically, based on a disassembly of vmlinux (or an individual .o file

Signed-off-by: Tim Bird <tim.bird@am.sony.com>
---
 scripts/stack_size |  289 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 289 insertions(+), 0 deletions(-)
 create mode 100755 scripts/stack_size

diff --git a/scripts/stack_size b/scripts/stack_size
new file mode 100755
index 0000000..af9fdac
--- /dev/null
+++ b/scripts/stack_size
@@ -0,0 +1,289 @@
+#!/usr/bin/python
+# stack_size - a program to parse ARM assembly files and produce
+# a listing of stack sizes for the functions encountered.
+#
+# To use: compile your kernel as normal, then run this program on vmlinux:
+# $ scripts/stac_size vmlinux
+#
+# If you have cross-compiled, then your CROSS_COMPILE environment variable
+# should be set to the cross prefix for your toolchain
+#
+# Author: Tim Bird <tim dot bird ~(at)~ am dot sony dot com>
+# Copyright 2011 Sony Network Entertainment, Inc
+#
+# GPL version 2.0 applies.
+#
+
+import os, sys
+version = "1.0"
+
+debug = 0
+
+def dprint(msg):
+	global debug
+	if debug:
+		print msg
+
+def usage():
+	print """Usage: %s [options] <filename>
+
+Show stack size for functions in a program or object file.
+Generate and parse the ARM assembly for the indicated filename,
+and print out the size of the stack for each function.
+
+If you used a cross-compiler, please set the cross-compiler prefix
+in the environment variable 'CROSS_COMPILE', before using this tool.
+
+Options:
+ -h       show this usage help
+ -t <n>   show the top n functions with the largest stacks
+          (default is to show top 10 functions with largest stacks)
+ -f       show full function list
+ -g       show histogram of stack depths
+ -V or --version    show version information and exit
+
+Notes: This leaves the assembly file in the same directory as the source
+filename, called '<filename>.S'  This program currently only supports
+ARM EABI.
+
+""" % os.path.basename(sys.argv[0])
+	sys.exit(0)
+
+class function_class():
+	def __init__(self, name, addr):
+		self.name = name
+		self.addr = addr
+		self.depth = 0
+
+	def __repr__(self):
+		return "function: %s with depth %d" % (self.name, self.depth)
+
+def open_assembly(filename):
+	base = filename
+	if base.endswith(".o"):
+		base = base[:-2]
+
+	assembly_filename = "%s.S" % base
+
+	try:
+		fd = open(assembly_filename)
+	except:
+		print "Could not find %s - trying to generate from %s\n" % \
+			(assembly_filename, filename)
+		print "This might take a few minutes..."
+		try:
+			cross = os.environ["CROSS_COMPILE"]
+		except:
+			cross = ""
+		cmd = "%sobjdump -d %s >%s" % \
+			(cross, filename, assembly_filename)
+		print "Using command: '%s'" % cmd
+		os.system(cmd)
+
+	try:
+		fd = open(assembly_filename)
+	except:
+		print "Could not find %s, and conversion didn't work\n" % \
+			(assembly_filename)
+		sys.exit(1)
+
+	return fd;
+
+def do_show_histogram(functions):
+	depth_count = {}
+
+	max_depth_bucket = 0
+	for funcname in functions.keys():
+		depth = functions[funcname].depth
+		depth_bucket = depth/10
+		if depth_bucket > max_depth_bucket:
+			max_depth_bucket = depth_bucket
+
+		try:
+			depth_count[depth_bucket] += 1
+		except:
+			depth_count[depth_bucket] = 1
+
+	print "=========== HISTOGRAM =============="
+	for i in range(0,max_depth_bucket+1):
+		try:
+			count = depth_count[i]
+		except:
+			count = 0
+
+		# use numbers for bars that are too long
+		if count<72:
+			bar = count*"*"
+		else:
+			bar = (72-8)*"*" + "(%d)*" % count
+		print "%3d: %s" % (i*10,bar)
+
+def cmp_depth(f1, f2):
+	return cmp(f1.depth, f2.depth)
+
+def main():
+	global debug
+
+	full_list = 0
+	show_histogram = 0
+	debug = 0
+	top_count = 10
+
+	if "-h" in sys.argv:
+		usage()
+
+	if "-f" in sys.argv:
+		full_list = 1
+		sys.argv.remove("-f")
+
+	if "-g" in sys.argv:
+		show_histogram = 1
+		sys.argv.remove("-g")
+
+	if "-t" in sys.argv:
+		top_count = sys.argv[sys.argv.index("-t")+1]
+		sys.argv.remove(top_count)
+		sys.argv.remove("-t")
+		top_count = int(top_count)
+
+	if "--debug" in sys.argv:
+		debug = 1
+		sys.argv.remove("--debug")
+
+	if "--version" in sys.argv or "-V" in sys.argv:
+		print "%s version %s" % (os.path.basename(sys.argv[0]),version)
+		sys.exit(0)
+
+	try:
+		filename = sys.argv[1]
+	except:
+		print "Error: missing filename to scan"
+		usage()
+
+	fd = open_assembly(filename)
+
+	functions = {}
+	func = None
+
+	max_cur_depth = 0
+	depth = 0
+	func_line_count = 0
+	PROGRAM_ENTRY_MAXLINES = 4
+
+	for line in fd.xreadlines():
+		# parse lines
+
+		# find function starts
+		try:
+			if line[8:10]==" <":
+				func_line = line
+				(func_addr_s,rest) = func_line.split(" ",1)
+				funcname = rest[1:-3]
+				#print line,
+				func_addr = int(func_addr_s, 16)
+				#print "func_addr=%x, func=%s" % (func_addr, func)
+				# record depth of last function
+				if func:
+					func.depth = max_cur_depth
+					dprint("stack depth: %d" % max_cur_depth)
+
+				# start new function
+				dprint("function: %s" % funcname)
+				func = function_class(funcname, func_addr)
+				functions[funcname] = func
+				depth = 0
+				max_cur_depth = 0
+				func_line_count = 0
+
+		except:
+			pass
+
+		func_line_count += 1
+
+		# calculate stack depth
+		# pattern is: initial register push (with "push {...}
+		# reservation of local stack space
+		if line.find("push\t") != -1 and \
+		    func_line_count<PROGRAM_ENTRY_MAXLINES:
+			# parse push
+			(before, args) = line.split("push",1)
+			dprint("push args=%s" % args)
+			regs = line.split(",")
+			depth += 4 * len(regs)
+			if depth>max_cur_depth:
+				max_cur_depth = depth
+
+			dprint("depth=%d" % depth)
+
+		if line.find("sub\tsp, sp, #") != -1 and \
+		    func_line_count<PROGRAM_ENTRY_MAXLINES:
+			# parse sub
+			(before, args) = line.split("sub",1)
+			if debug:
+				print "sub args=%s" % args
+			(sp1,sp2,imm) = line.split(",",2)
+			# next line splits: ' #84\t ; 0x54\n'
+			# into ['#84',';','0x54']
+			# take [0]th element, and strip leading '#'
+			#print "imm='%s'" % imm
+			depth += int(imm.split()[0][1:])
+			if depth>max_cur_depth:
+				max_cur_depth = depth
+
+			dprint("depth=%d" % depth)
+
+		# disable check for pops (used for debugging)
+		if None and line.find("pop\t") != -1:
+			# parse pop
+			(before, args) = line.split("pop",1)
+			dprint("pop args=%s" % args)
+			regs = line.split(",")
+			depth -= 4 * len(regs)
+			if depth<0:
+				print "Parser Error: function: %s, depth=%d" % \
+					(funcname, depth)
+			dprint("depth=%d" % depth)
+
+	# record depth of last function
+	if func:
+		func.depth = max_cur_depth
+		dprint("max stack depth: %d" % max_cur_depth)
+
+	fd.close()
+
+	if not functions:
+		print "No functions found. Done."
+		return
+
+	# calculate results
+	max_depth = 0
+	maxfunc = func
+	for func in functions.values():
+		if func.depth > max_depth:
+			max_depth = func.depth
+			maxfunc = func
+
+	print "============ RESULTS ==============="
+	print "number of functions     = %d" % len(functions)
+	print "max function stack depth= %d" % maxfunc.depth
+	print "function with max depth = %s\n" % maxfunc.name
+
+	if full_list or top_count:
+		funclist = functions.values()
+		funclist.sort(cmp_depth)
+		print "%-32s %s" % ("Function Name        ","Stack Depth")
+		print "%-32s %s" % ("=====================","===========")
+
+		if full_list:
+			top_count = len(funclist)
+
+		start = len(funclist)-top_count
+		for i in range(start, start+top_count):
+			func = funclist[i]
+			print "%-32s %4d" % (func.name, func.depth)
+
+	if show_histogram:
+		do_show_histogram(functions)
+
+if __name__=="__main__":
+	main()
-- 
1.6.6

[PATCH 3/3] ARM 4Kstacks: Decrease poll and select stack usage, when using 4K stacks

[Tim Bird]

Reduce the max allowed size for struct poll_wqueue to avoid
using too much stack space in do_select() and do_sys_poll(),
when the kernel is using 4K stacks.

Signed-off-by: Tim Bird <tim.bird@am.sony.com>
---
 include/linux/poll.h |    4 ++++
 1 files changed, 4 insertions(+), 0 deletions(-)

diff --git a/include/linux/poll.h b/include/linux/poll.h
index cf40010..b7153a3 100644
--- a/include/linux/poll.h
+++ b/include/linux/poll.h
@@ -16,7 +16,11 @@
 extern struct ctl_table epoll_table[]; /* for sysctl */
 /* ~832 bytes of stack space used max in sys_select/sys_poll before allocating
    additional memory. */
+#ifndef CONFIG_4KSTACKS
 #define MAX_STACK_ALLOC 832
+#else
+#define MAX_STACK_ALLOC 448
+#endif
 #define FRONTEND_STACK_ALLOC	256
 #define SELECT_STACK_ALLOC	FRONTEND_STACK_ALLOC
 #define POLL_STACK_ALLOC	FRONTEND_STACK_ALLOC
-- 
1.6.6

[PATCH 0/3] ARM 4Kstacks: introduction

[Joe Perches]

On Tue, 2011-10-18 at 16:27 -0700, Tim Bird wrote:
> I'm about to submit a set of patches (really pretty small)
> to add 4K stack support to ARM (defaulted to 'N').

When 4k stacks went away, I thought it safe enough
to submit vsnprintf recursion using %pV.

I believe the current vsnprintf max recursion is 3.
Each recursion uses at least 300 bytes of stack.

That might be some additional issue for 4k stacks.

[PATCH 0/3] ARM 4Kstacks: introduction

[Tim Bird]

On 10/18/2011 05:14 PM, Joe Perches wrote:
> On Tue, 2011-10-18 at 16:27 -0700, Tim Bird wrote:
>> I'm about to submit a set of patches (really pretty small)
>> to add 4K stack support to ARM (defaulted to 'N').
> 
> When 4k stacks went away, I thought it safe enough
> to submit vsnprintf recursion using %pV.
> 
> I believe the current vsnprintf max recursion is 3.
> Each recursion uses at least 300 bytes of stack.
> 
> That might be some additional issue for 4k stacks.

Interesting.  Thanks very much for the heads up.
I don't know how prevalent %pV is, but I'll take a
look tomorrow and see if I think this would be an
issue.  One option would be to turn off recursion
in the 4K stacks case (but I don't know how badly
this would mangle the code).

I'm not sure I'm willing to advocate eliminating
all occurrences of high stack usage in the kernel.
It seems like this would generate a lot of friction
for getting this mainlined.  As long as I can document
the trouble that people might get in when they
turn this on, I think that would be very good.

Even inside Sony, usage of 4K stacks is limited
to some very special cases, where memory is exceedingly
tight (we have one system with 4M of RAM).  And we
don't mind lopping off features or coding around
problem areas to support our special case.

Thanks again for the notice.
 -- Tim


=============================
Tim Bird
Architecture Group Chair, CE Workgroup of the Linux Foundation
Senior Staff Engineer, Sony Network Entertainment
=============================

[PATCH 0/3] ARM 4Kstacks: introduction

[Joe Perches]

On Tue, 2011-10-18 at 17:26 -0700, Tim Bird wrote:
> On 10/18/2011 05:14 PM, Joe Perches wrote:
> > On Tue, 2011-10-18 at 16:27 -0700, Tim Bird wrote:
> >> I'm about to submit a set of patches (really pretty small)
> >> to add 4K stack support to ARM (defaulted to 'N').
> > When 4k stacks went away, I thought it safe enough
> > to submit vsnprintf recursion using %pV.
> Even inside Sony, usage of 4K stacks is limited
> to some very special cases, where memory is exceedingly
> tight (we have one system with 4M of RAM).  And we
> don't mind lopping off features or coding around
> problem areas to support our special case.

I believe CONFIG_PRINTK=n assures no %pV problem.

[PATCH 0/3] ARM 4Kstacks: introduction

[Dave Chinner]

On Tue, Oct 18, 2011 at 04:27:45PM -0700, Tim Bird wrote:
> I'm about to submit a set of patches (really pretty small)
> to add 4K stack support to ARM (defaulted to 'N').
> 
> This has been kicking around in my Sony tree for a few years,
> and it's about time to mainline it.  The first patch is
> the actual 4KSTACKS patch.  See subsequent patches
> for tools to help with stack reduction to avoid the problems
> that apparently led to the removal of this feature on x86.

Not this again. There's a good reason 4k stacks went away: it's
simply not enough space for the deep 60+ function call stacks we see
with even trivial storage stack configurations.

The stack usage on 32 bit ARM and x86 is going to be similar, so
you're going to be fighting a losing battle - there is no stack
space tha tcan be trimmed from most paths. To make matter worse,
there's been stuff done to the storage stack that significantly
increases stack usage since 4k stacks went away (e.g. the on-stack
block plugging changes).

And FWIW, XFS is widely used on ARM based NAS devices, so this isn't
a theoretical problem I'm making up here...

Cheers,

Dave.
-- 
Dave Chinner
david at fromorbit.com

[PATCH 0/3] ARM 4Kstacks: introduction

[Dave Chinner]

On Tue, Oct 18, 2011 at 05:26:44PM -0700, Tim Bird wrote:
> On 10/18/2011 05:14 PM, Joe Perches wrote:
> > On Tue, 2011-10-18 at 16:27 -0700, Tim Bird wrote:
> >> I'm about to submit a set of patches (really pretty small)
> >> to add 4K stack support to ARM (defaulted to 'N').
> > 
> > When 4k stacks went away, I thought it safe enough
> > to submit vsnprintf recursion using %pV.
> > 
> > I believe the current vsnprintf max recursion is 3.
> > Each recursion uses at least 300 bytes of stack.
> > 
> > That might be some additional issue for 4k stacks.
> 
> Interesting.  Thanks very much for the heads up.
> I don't know how prevalent %pV is, but I'll take a
> look tomorrow and see if I think this would be an
> issue.  One option would be to turn off recursion
> in the 4K stacks case (but I don't know how badly
> this would mangle the code).

Badly.

For example, every single message emitted by XFS uses %pV to add
consistent message prefixes to the messages. See __xfs_printk().

And a quick grep shows this same technique is used in ext3, ext4,
NILFS, FAT and other filesystems, as well as netdev_printk,
dev_printk, the libata code and more. IOWs, you'd basically break
messages from most subsystems and drivers in the kernel...

> I'm not sure I'm willing to advocate eliminating
> all occurrences of high stack usage in the kernel.
> It seems like this would generate a lot of friction
> for getting this mainlined.  As long as I can document
> the trouble that people might get in when they
> turn this on, I think that would be very good.

"You can't use block based storage because the storage stack doesn't
fit in 4k."

> Even inside Sony, usage of 4K stacks is limited
> to some very special cases, where memory is exceedingly
> tight (we have one system with 4M of RAM).  And we
> don't mind lopping off features or coding around
> problem areas to support our special case.

Sounds to me like it's less overhead simply to maintain the patches
for the systems that need it than it is to put it in mainline,
document all the stuff that doesn't work and then, of course, have
to triage all the problems people report because they haven't read
the documentation.... :/

Cheers,

Dave.
-- 
Dave Chinner
david at fromorbit.com

[PATCH 0/3] ARM 4Kstacks: introduction

[Andi Kleen]

> "You can't use block based storage because the storage stack doesn't
> fit in 4k."

Likely far more than that.

Perhaps a better way would be to simply put the stack into a virtual
mapping and allocate more as needed.

I agree it's a bad idea in mainline.

-Andi

[PATCH 0/3] ARM 4Kstacks: introduction

[Russell King - ARM Linux]

On Tue, Oct 18, 2011 at 04:27:45PM -0700, Tim Bird wrote:
> I'm about to submit a set of patches (really pretty small)
> to add 4K stack support to ARM (defaulted to 'N').
> 
> This has been kicking around in my Sony tree for a few years,
> and it's about time to mainline it.  The first patch is
> the actual 4KSTACKS patch.  See subsequent patches
> for tools to help with stack reduction to avoid the problems
> that apparently led to the removal of this feature on x86.

This isn't a good idea - if the feature has been abandoned on x86,
people aren't going to be soo concerned about the stack usage of
each function.  They're going to accept that there's more stack
space available, and we'll see code paths that start expecting that.

If we switch to 4K stacks, the problems of stack usage become ours.
Do we have the capacity to be able to address these issues as they
crop up?

[PATCH 2/3] ARM: Add static kernel function stack size analyzer, for ARM

[Arnd Bergmann]

On Tuesday 18 October 2011 16:31:50 Tim Bird wrote:
> 
> This tool can be used to analyze the function stack size for ARM kernels,
> statically, based on a disassembly of vmlinux (or an individual .o file
> 
> Signed-off-by: Tim Bird <tim.bird@am.sony.com>

Hi Tim,

I thought we now had a gcc option for this. gcc documentation points to -Wstack-usage=XXX.
This was first introduced on s390 in architecture specific gcc code, but should now work
everywhere.

	Arnd

[PATCH 0/3] ARM 4Kstacks: introduction

[Arnd Bergmann]

On Tuesday 18 October 2011 17:26:44 Tim Bird wrote:
> Even inside Sony, usage of 4K stacks is limited
> to some very special cases, where memory is exceedingly
> tight (we have one system with 4M of RAM).  And we
> don't mind lopping off features or coding around
> problem areas to support our special case.

I would imagine that in those cases, you can gain more by reducing the
number of threads in the system. What is the highest number of
concurrent threads that you expect in a limited use case with no
networking or block devices?

	Arnd

[PATCH 0/3] ARM 4Kstacks: introduction

[Tim Bird]

On 10/19/2011 12:35 AM, Russell King - ARM Linux wrote:
> On Tue, Oct 18, 2011 at 04:27:45PM -0700, Tim Bird wrote:
>> I'm about to submit a set of patches (really pretty small)
>> to add 4K stack support to ARM (defaulted to 'N').
>>
>> This has been kicking around in my Sony tree for a few years,
>> and it's about time to mainline it.  The first patch is
>> the actual 4KSTACKS patch.  See subsequent patches
>> for tools to help with stack reduction to avoid the problems
>> that apparently led to the removal of this feature on x86.
> 
> This isn't a good idea - if the feature has been abandoned on x86,
> people aren't going to be soo concerned about the stack usage of
> each function.  They're going to accept that there's more stack
> space available, and we'll see code paths that start expecting that.

Well, the difference in size is not an order of magnitude.  Current
stacks are only twice as big as what this feature optionally allows.
So people really shouldn't go crazy with their stack usage.

Granted we're already seeing a bit of stack bloat, but IMHO this should
be nipped in the bud.  If the stack grows a lot more, it will likely
need to shift to 16K, which would REALLY be costly on machines
with lots of threads and low memory.

I don't want to give up on a small-footprint Linux so easily,
and the discipline for keeping stacks small is probably good
for code hygiene anyway.

> If we switch to 4K stacks, the problems of stack usage become ours.
> Do we have the capacity to be able to address these issues as they
> crop up?

Since it's optional, it would become the problem of the people
who actually use the option.  If you think tracking down stack
abusers is going to be a pain, just don't do it.

If I do it, and get pushback, then I'll make the case to individual
function authors, and we'll see what happens from there.  As it is,
this patch has been useful for me, and continues to be useful
even in the current kernel version, with no other changes
required in mainline.

Having said this, I've written a tool to rather easily identify
the functions which use the most stack, and so far the routines
I've looked at have had rather simple solutions.  It almost always
boils down to a single big data item on the stack.

I can understand if people don't want to take patches which
unconditionally limit their stack usage - especially if there's
a performance impact. But I don't see much rationale for not
limiting their stack usage conditionally (that is, do something
slightly different on #ifdef CONFIG_4KSTACKS).  If people
object to even that, then I'll maintain those patches out of
tree, if they're really needed.

Once again, our products work even without such patches now,
based on the configs we're using and the user-space we're running.
Note that this is defaulted to 'N', so no one should be negatively
affected unless they opt in.

So, I understand your point.  I would, however, like to have
this patch applied, and then deal with any problems as they
come up, rather than preemptively kill the feature on speculative
problems.
 -- Tim

=============================
Tim Bird
Architecture Group Chair, CE Workgroup of the Linux Foundation
Senior Staff Engineer, Sony Network Entertainment
=============================

[PATCH 0/3] ARM 4Kstacks: introduction

[Tim Bird]

On 10/18/2011 05:31 PM, Joe Perches wrote:
> On Tue, 2011-10-18 at 17:26 -0700, Tim Bird wrote:
>> On 10/18/2011 05:14 PM, Joe Perches wrote:
>>> On Tue, 2011-10-18 at 16:27 -0700, Tim Bird wrote:
>>>> I'm about to submit a set of patches (really pretty small)
>>>> to add 4K stack support to ARM (defaulted to 'N').
>>> When 4k stacks went away, I thought it safe enough
>>> to submit vsnprintf recursion using %pV.
>> Even inside Sony, usage of 4K stacks is limited
>> to some very special cases, where memory is exceedingly
>> tight (we have one system with 4M of RAM).  And we
>> don't mind lopping off features or coding around
>> problem areas to support our special case.
> 
> I believe CONFIG_PRINTK=n assures no %pV problem.

Indeed. Hehe. :-)  We might already be doing that in
the places we're using this.

Actually, this recursion is interesting.
My stack_size tool to detects stack usage for one instance
of a function.  But it would be easy to add a special
case to do a multiplier for functions known to potentially
recurse.  I'll have to look at that and maybe add something.

Relatedly, it might be interesting to augment the static
analysis with runtime information from the stack tracer.
 -- Tim

=============================
Tim Bird
Architecture Group Chair, CE Workgroup of the Linux Foundation
Senior Staff Engineer, Sony Network Entertainment
=============================

[PATCH 0/3] ARM 4Kstacks: introduction

[Måns Rullgård]

Tim Bird <tim.bird@am.sony.com> writes:

> On 10/19/2011 12:35 AM, Russell King - ARM Linux wrote:
>> On Tue, Oct 18, 2011 at 04:27:45PM -0700, Tim Bird wrote:
>>> I'm about to submit a set of patches (really pretty small)
>>> to add 4K stack support to ARM (defaulted to 'N').
>>>
>>> This has been kicking around in my Sony tree for a few years,
>>> and it's about time to mainline it.  The first patch is
>>> the actual 4KSTACKS patch.  See subsequent patches
>>> for tools to help with stack reduction to avoid the problems
>>> that apparently led to the removal of this feature on x86.
>> 
>> This isn't a good idea - if the feature has been abandoned on x86,
>> people aren't going to be soo concerned about the stack usage of
>> each function.  They're going to accept that there's more stack
>> space available, and we'll see code paths that start expecting that.
>
> Well, the difference in size is not an order of magnitude.  Current
> stacks are only twice as big as what this feature optionally allows.
> So people really shouldn't go crazy with their stack usage.
>
> Granted we're already seeing a bit of stack bloat, but IMHO this should
> be nipped in the bud.  If the stack grows a lot more, it will likely
> need to shift to 16K, which would REALLY be costly on machines
> with lots of threads and low memory.

In binary, 8k is an order of magnitude greater than 4k.  Having suffered
the consequences of 4k stacks on x86, I have to side with those
reluctant to include this.

> I don't want to give up on a small-footprint Linux so easily,
> and the discipline for keeping stacks small is probably good
> for code hygiene anyway.

I see your point of view, but an optional setting available only on an
architecture not used by the majority is probably not going to have much
of an impact on code hygiene.

4k stacks were tried, and the conclusion was that too much of the kernel
simply cannot work within so small a space.  Maintaining a list of safe
(or unsafe) combinations is probably not feasible either.

>> If we switch to 4K stacks, the problems of stack usage become ours.
>> Do we have the capacity to be able to address these issues as they
>> crop up?
>
> Since it's optional, it would become the problem of the people
> who actually use the option.  If you think tracking down stack
> abusers is going to be a pain, just don't do it.

It will still be one more thing to sanity check in every bug report.
The end result will be more crashes going ignored due to lacking
information, and I don't think that's a good thing.

> If I do it, and get pushback, then I'll make the case to individual
> function authors, and we'll see what happens from there.  As it is,
> this patch has been useful for me, and continues to be useful
> even in the current kernel version, with no other changes
> required in mainline.
>
> Having said this, I've written a tool to rather easily identify
> the functions which use the most stack, and so far the routines
> I've looked at have had rather simple solutions.  It almost always
> boils down to a single big data item on the stack.

Chasing heavy stack consumers is of course always a good thing, whatever
the stack size.

> I can understand if people don't want to take patches which
> unconditionally limit their stack usage - especially if there's
> a performance impact. But I don't see much rationale for not
> limiting their stack usage conditionally (that is, do something
> slightly different on #ifdef CONFIG_4KSTACKS).  If people
> object to even that, then I'll maintain those patches out of
> tree, if they're really needed.

Having such a conditional increases the maintenance burden for that
function.  Changes will be made without testing the 4k alternative,
which sooner or later will break, possibly in subtle ways.

> Once again, our products work even without such patches now,
> based on the configs we're using and the user-space we're running.
> Note that this is defaulted to 'N', so no one should be negatively
> affected unless they opt in.
>
> So, I understand your point.  I would, however, like to have
> this patch applied, and then deal with any problems as they
> come up, rather than preemptively kill the feature on speculative
> problems.

An option that will probably, let's be realistic, only be used by you
doesn't really belong in mainline.  You are essentially asking everybody
else to carry the burden of supporting something only you seem to need.

-- 
M?ns Rullg?rd
mans at mansr.com

[PATCH 0/3] ARM 4Kstacks: introduction

[Tim Bird]

On 10/19/2011 05:13 PM, ? wrote:
> Tim Bird <tim.bird@am.sony.com> writes:
>> Well, the difference in size is not an order of magnitude.  Current
>> stacks are only twice as big as what this feature optionally allows.
>> So people really shouldn't go crazy with their stack usage.
>>
>> Granted we're already seeing a bit of stack bloat, but IMHO this should
>> be nipped in the bud.  If the stack grows a lot more, it will likely
>> need to shift to 16K, which would REALLY be costly on machines
>> with lots of threads and low memory.
> 
> In binary, 8k is an order of magnitude greater than 4k.
Touche' ;-)

> Having suffered
> the consequences of 4k stacks on x86, I have to side with those
> reluctant to include this.

Can you please provide more detail on what consequences
you suffered?  I think it would help the discussion.

If you had some examples where the burden of supporting 4K stacks
was onerous, that would be useful.  I know that defaulting it to
'Y' introduced lots of subtle bugs that desktop users had issues
with.  The situation is completely different in deeply embedded,
because our user-space is completely constrained and controlled.

Also, there are a now a plethora of stack depth monitoring, guarding,
and reporting tools already in mainline, some of which weren't
there during the painful era.

>> I don't want to give up on a small-footprint Linux so easily,
>> and the discipline for keeping stacks small is probably good
>> for code hygiene anyway.
> 
> I see your point of view, but an optional setting available only on an
> architecture not used by the majority is probably not going to have much
> of an impact on code hygiene.
> 
> 4k stacks were tried, and the conclusion was that too much of the kernel
> simply cannot work within so small a space.  Maintaining a list of safe
> (or unsafe) combinations is probably not feasible either.

That's not what was in the message of the commit that removed them
on x86.  Instead it was conjecture about high numbers of threads
being only applicable to x86_64.

Are you basing the conclusion above on conjecture or measurement?  In
my configuration, the worst code paths are using approximately 2K of
stack most of the time, and peak out at about 2.5K.

>>> If we switch to 4K stacks, the problems of stack usage become ours.
>>> Do we have the capacity to be able to address these issues as they
>>> crop up?
>>
>> Since it's optional, it would become the problem of the people
>> who actually use the option.  If you think tracking down stack
>> abusers is going to be a pain, just don't do it.
> 
> It will still be one more thing to sanity check in every bug report.
> The end result will be more crashes going ignored due to lacking
> information, and I don't think that's a good thing.

I don't think mainline is overwhelmed with bug reports from
embedded developers.  Most of the time, embedded devs are so far behind
in kernel version that they can't reasonably (and don't) report many
bugs found. (I know that this is not helping my case, but I'm trying to be
honest here.)

If 4K stacks were being turned on for the general population of Linux
users, then I would agree with you.  But this will likely only be
turned on by a few people, in controlled situations. If you think
tainting the kernel would help with the bug reporting, I'd be
OK with that.  We could introduce a "crazy embedded config - you're
on your own" taint flag.

> 
>> If I do it, and get pushback, then I'll make the case to individual
>> function authors, and we'll see what happens from there.  As it is,
>> this patch has been useful for me, and continues to be useful
>> even in the current kernel version, with no other changes
>> required in mainline.
>>
>> Having said this, I've written a tool to rather easily identify
>> the functions which use the most stack, and so far the routines
>> I've looked at have had rather simple solutions.  It almost always
>> boils down to a single big data item on the stack.
> 
> Chasing heavy stack consumers is of course always a good thing, whatever
> the stack size.
> 
>> I can understand if people don't want to take patches which
>> unconditionally limit their stack usage - especially if there's
>> a performance impact. But I don't see much rationale for not
>> limiting their stack usage conditionally (that is, do something
>> slightly different on #ifdef CONFIG_4KSTACKS).  If people
>> object to even that, then I'll maintain those patches out of
>> tree, if they're really needed.
> 
> Having such a conditional increases the maintenance burden for that
> function.  Changes will be made without testing the 4k alternative,
> which sooner or later will break, possibly in subtle ways.

Which will affect no one but those who opt in.  I just don't
think the maintenance burden will be that big.  (At least,
no one need be bothered by this, unless contacted by someone
who cares.)

>> Once again, our products work even without such patches now,
>> based on the configs we're using and the user-space we're running.
>> Note that this is defaulted to 'N', so no one should be negatively
>> affected unless they opt in.
>>
>> So, I understand your point.  I would, however, like to have
>> this patch applied, and then deal with any problems as they
>> come up, rather than preemptively kill the feature on speculative
>> problems.
> 
> An option that will probably, let's be realistic, only be used by you
> doesn't really belong in mainline.  You are essentially asking everybody
> else to carry the burden of supporting something only you seem to need.

I'm not so sure that I'm the only one interested in this.
I'll ask around and see if I can find others.  I'll have
a good chance at ELC Europe next week.

But in general this attitude is really depressing.  I can
remember when support for niche use cases was considered good.
I understand the tradeoff if the maintenance burden is high,
but I'm not convinced it is.  I don't expect anyone to
proactively guard against or remedy stack depth issues.  But if
they'd be open to accepting patches from those who care, that'd
be nice.

If you're saying that Linux is only for use on general purpose
platforms, and deeply embedded doesn't matter, I think that's sad.
 -- Tim

=============================
Tim Bird
Architecture Group Chair, CE Workgroup of the Linux Foundation
Senior Staff Engineer, Sony Network Entertainment
=============================

[PATCH 0/3] ARM 4Kstacks: introduction

[Måns Rullgård]

Tim Bird <tim.bird@am.sony.com> writes:

> On 10/19/2011 05:13 PM, ? wrote:
>> Tim Bird <tim.bird@am.sony.com> writes:
>>> Well, the difference in size is not an order of magnitude.  Current
>>> stacks are only twice as big as what this feature optionally allows.
>>> So people really shouldn't go crazy with their stack usage.
>>>
>>> Granted we're already seeing a bit of stack bloat, but IMHO this should
>>> be nipped in the bud.  If the stack grows a lot more, it will likely
>>> need to shift to 16K, which would REALLY be costly on machines
>>> with lots of threads and low memory.
>> 
>> In binary, 8k is an order of magnitude greater than 4k.
> Touche' ;-)
>
>> Having suffered
>> the consequences of 4k stacks on x86, I have to side with those
>> reluctant to include this.
>
> Can you please provide more detail on what consequences
> you suffered?  I think it would help the discussion.

Random crashes which I eventually figured out were caused by
xfs + device-mapper + md blowing the 4k stack in certain situations.
At the time, 4k stacks were enabled default with the option hidden away
in some obscure place.

> If you had some examples where the burden of supporting 4K stacks
> was onerous, that would be useful.  I know that defaulting it to
> 'Y' introduced lots of subtle bugs that desktop users had issues
> with.  The situation is completely different in deeply embedded,
> because our user-space is completely constrained and controlled.
>
> Also, there are a now a plethora of stack depth monitoring, guarding,
> and reporting tools already in mainline, some of which weren't
> there during the painful era.

The tools have certainly got better since then, but I they are still not
able to guard against every possible failure.

>>> I don't want to give up on a small-footprint Linux so easily,
>>> and the discipline for keeping stacks small is probably good
>>> for code hygiene anyway.
>> 
>> I see your point of view, but an optional setting available only on an
>> architecture not used by the majority is probably not going to have much
>> of an impact on code hygiene.
>> 
>> 4k stacks were tried, and the conclusion was that too much of the kernel
>> simply cannot work within so small a space.  Maintaining a list of safe
>> (or unsafe) combinations is probably not feasible either.
>
> That's not what was in the message of the commit that removed them
> on x86.  Instead it was conjecture about high numbers of threads
> being only applicable to x86_64.
>
> Are you basing the conclusion above on conjecture or measurement?  In
> my configuration, the worst code paths are using approximately 2K of
> stack most of the time, and peak out at about 2.5K.

Someone else in this thread mentioned several combinations known to
require more than 4k.  Layered block devices was one of them.

>>>> If we switch to 4K stacks, the problems of stack usage become ours.
>>>> Do we have the capacity to be able to address these issues as they
>>>> crop up?
>>>
>>> Since it's optional, it would become the problem of the people
>>> who actually use the option.  If you think tracking down stack
>>> abusers is going to be a pain, just don't do it.
>> 
>> It will still be one more thing to sanity check in every bug report.
>> The end result will be more crashes going ignored due to lacking
>> information, and I don't think that's a good thing.
>
> I don't think mainline is overwhelmed with bug reports from
> embedded developers. 

Mainline might not be, but various embedded forums are.  Embedded just
ain't what it used to be.

> Most of the time, embedded devs are so far behind in kernel version
> that they can't reasonably (and don't) report many bugs found.

What makes you say that?  Consumer devices available from shops are a
few versions behind (my Android phone runs 2.6.35), but devices under
development use quite recent kernels.  A few years ago, being stuck with
something hopelessly outdated was the norm, but things have thankfully
improved since then.

> (I know that this is not helping my case, but I'm trying to be honest
> here.)
>
> If 4K stacks were being turned on for the general population of Linux
> users, then I would agree with you.  But this will likely only be
> turned on by a few people, in controlled situations.

The trouble is figuring out which configurations are safe.

> If you think tainting the kernel would help with the bug reporting,
> I'd be OK with that.  We could introduce a "crazy embedded config -
> you're on your own" taint flag.

Taint flags are but a means of automatically ignoring bug reports from
anyone running such a configuration.  They do not make the bugs go away.

>>> I can understand if people don't want to take patches which
>>> unconditionally limit their stack usage - especially if there's
>>> a performance impact. But I don't see much rationale for not
>>> limiting their stack usage conditionally (that is, do something
>>> slightly different on #ifdef CONFIG_4KSTACKS).  If people
>>> object to even that, then I'll maintain those patches out of
>>> tree, if they're really needed.
>> 
>> Having such a conditional increases the maintenance burden for that
>> function.  Changes will be made without testing the 4k alternative,
>> which sooner or later will break, possibly in subtle ways.
>
> Which will affect no one but those who opt in.  I just don't
> think the maintenance burden will be that big.  (At least,
> no one need be bothered by this, unless contacted by someone
> who cares.)

I would certainly not want to be responsible for any code with such
conditionals in it.

>>> Once again, our products work even without such patches now,
>>> based on the configs we're using and the user-space we're running.
>>> Note that this is defaulted to 'N', so no one should be negatively
>>> affected unless they opt in.
>>>
>>> So, I understand your point.  I would, however, like to have
>>> this patch applied, and then deal with any problems as they
>>> come up, rather than preemptively kill the feature on speculative
>>> problems.
>> 
>> An option that will probably, let's be realistic, only be used by you
>> doesn't really belong in mainline.  You are essentially asking everybody
>> else to carry the burden of supporting something only you seem to need.
>
> I'm not so sure that I'm the only one interested in this.
> I'll ask around and see if I can find others.  I'll have
> a good chance at ELC Europe next week.

I'll be there as well.

> But in general this attitude is really depressing.  I can
> remember when support for niche use cases was considered good.

It's always a tradeoff of utility versus overall impact.  The more
obscure a feature is, the more confined it will have to be.  A driver
for some obscure piece of hardware can usually be accommodated without
touching any other code.  The 4k stack feature, however, has the
potential to require changes or impose constraints pretty much
everywhere in the entire kernel.

> I understand the tradeoff if the maintenance burden is high,
> but I'm not convinced it is.

The removal of the 4k stack option on x86 suggests otherwise.

> I don't expect anyone to proactively guard against or remedy stack
> depth issues.  But if they'd be open to accepting patches from those
> who care, that'd be nice.

Of course gratuitous stack bloat should be fixed whenever possible.  I
would never suggest otherwise.  Expecting support for 4k stacks through
an alternate (presumably reduced-functionality) path is however pushing
your luck.  Many maintainers will not even have the hardware necessary
to test this configuration, and not being able to test code I maintain
is not a situation I would like to be in (I am not a maintainer of
anything, so perhaps this carries little weight).

> If you're saying that Linux is only for use on general purpose
> platforms, and deeply embedded doesn't matter, I think that's sad.

I am saying the line has to be drawn somewhere, and it seems that 4k
stacks are pushing things a bit too far.  I would probably support (in
principle, if not with code) an effort to make 4k stacks work
everywhere (possibly with a few well-defined exceptions).  Were it not
for the previous failed attempt, I might even believe it could succeed.

-- 
M?ns Rullg?rd
mans at mansr.com

[PATCH 3/3] ARM 4Kstacks: Decrease poll and select stack usage, when using 4K stacks

[Arnd Bergmann]

On Wednesday 19 October 2011, Tim Bird wrote:
> Reduce the max allowed size for struct poll_wqueue to avoid
> using too much stack space in do_select() and do_sys_poll(),
> when the kernel is using 4K stacks.
> 
> Signed-off-by: Tim Bird <tim.bird@am.sony.com>

Can you explain why this is necessary with 4K stacks? I would
expect the poll() function never to be deeply nested, so 1K
used in there doesn't sound too problematic.

	Arnd

[PATCH 0/3] ARM 4Kstacks: introduction

[Dave Chinner]

On Wed, Oct 19, 2011 at 04:36:59PM -0700, Tim Bird wrote:
> On 10/19/2011 12:35 AM, Russell King - ARM Linux wrote:
> > On Tue, Oct 18, 2011 at 04:27:45PM -0700, Tim Bird wrote:
> >> I'm about to submit a set of patches (really pretty small)
> >> to add 4K stack support to ARM (defaulted to 'N').
> >>
> >> This has been kicking around in my Sony tree for a few years,
> >> and it's about time to mainline it.  The first patch is
> >> the actual 4KSTACKS patch.  See subsequent patches
> >> for tools to help with stack reduction to avoid the problems
> >> that apparently led to the removal of this feature on x86.
> > 
> > This isn't a good idea - if the feature has been abandoned on x86,
> > people aren't going to be soo concerned about the stack usage of
> > each function.  They're going to accept that there's more stack
> > space available, and we'll see code paths that start expecting that.
> 
> Well, the difference in size is not an order of magnitude.  Current
> stacks are only twice as big as what this feature optionally allows.
> So people really shouldn't go crazy with their stack usage.
> 
> Granted we're already seeing a bit of stack bloat, but IMHO this should
> be nipped in the bud.  If the stack grows a lot more, it will likely
> need to shift to 16K, which would REALLY be costly on machines
> with lots of threads and low memory.

FWIW, I was seriously considering bringing up 16k stacks on x86-64
as a topic for the kernel summit. We've been getting reports of the
8k stack being blown on different filesystems for the past year or
so on relatively trivial storage configs....

Cheers,

Dave.
-- 
Dave Chinner
david at fromorbit.com

[PATCH 2/3] ARM: Add static kernel function stack size analyzer, for ARM

[Andi Kleen]

> 
> I thought we now had a gcc option for this. gcc documentation points to -Wstack-usage=XXX.
> This was first introduced on s390 in architecture specific gcc code, but should now work
> everywhere.

CONFIG_FRAME_WARN=...

Has been there forever for any architecture.

-Andi

[PATCH 0/3] ARM 4Kstacks: introduction

[Ming Lei]

On Wed, Oct 19, 2011 at 6:51 PM, Arnd Bergmann <arnd@arndb.de> wrote:
> On Tuesday 18 October 2011 17:26:44 Tim Bird wrote:
>> Even inside Sony, usage of 4K stacks is limited
>> to some very special cases, where memory is exceedingly
>> tight (we have one system with 4M of RAM). ?And we
>> don't mind lopping off features or coding around
>> problem areas to support our special case.
>
> I would imagine that in those cases, you can gain more by reducing the
> number of threads in the system. What is the highest number of
> concurrent threads that you expect in a limited use case with no
> networking or block devices?

If system run for some time, sometimes it may be difficult for
memory allocator to allocate 2 continuous page frames even  there are
many spare page frames in system because of
fragment issue, so the patch does make sense. Anyway,
it provides one option for user to apply 4k stack to avoid
such kind of process creation failure.

thanks,
-- 
Ming Lei

[PATCH 0/3] ARM 4Kstacks: introduction

[Måns Rullgård]

Ming Lei <tom.leiming@gmail.com> writes:

> On Wed, Oct 19, 2011 at 6:51 PM, Arnd Bergmann <arnd@arndb.de> wrote:
>> On Tuesday 18 October 2011 17:26:44 Tim Bird wrote:
>>> Even inside Sony, usage of 4K stacks is limited
>>> to some very special cases, where memory is exceedingly
>>> tight (we have one system with 4M of RAM). ?And we
>>> don't mind lopping off features or coding around
>>> problem areas to support our special case.
>>
>> I would imagine that in those cases, you can gain more by reducing the
>> number of threads in the system. What is the highest number of
>> concurrent threads that you expect in a limited use case with no
>> networking or block devices?
>
> If system run for some time, sometimes it may be difficult for
> memory allocator to allocate 2 continuous page frames even  there are
> many spare page frames in system because of
> fragment issue, so the patch does make sense. Anyway,
> it provides one option for user to apply 4k stack to avoid
> such kind of process creation failure.

Can't page migration be used in these situations?

-- 
M?ns Rullg?rd
mans at mansr.com

[PATCH 0/3] ARM 4Kstacks: introduction

[Russell King - ARM Linux]

On Sat, Oct 22, 2011 at 04:50:15PM +0800, Ming Lei wrote:
> On Wed, Oct 19, 2011 at 6:51 PM, Arnd Bergmann <arnd@arndb.de> wrote:
> > On Tuesday 18 October 2011 17:26:44 Tim Bird wrote:
> >> Even inside Sony, usage of 4K stacks is limited
> >> to some very special cases, where memory is exceedingly
> >> tight (we have one system with 4M of RAM). ?And we
> >> don't mind lopping off features or coding around
> >> problem areas to support our special case.
> >
> > I would imagine that in those cases, you can gain more by reducing the
> > number of threads in the system. What is the highest number of
> > concurrent threads that you expect in a limited use case with no
> > networking or block devices?
> 
> If system run for some time, sometimes it may be difficult for
> memory allocator to allocate 2 continuous page frames even  there are
> many spare page frames in system because of
> fragment issue, so the patch does make sense.

If memory fragmentation is an issue for this, it probably means that we
need to switch to a software page size of 8K (or maybe 16K) rather than
stick with the hardware 4K size.  That would be a much more reliable
solution, especially as the L1 page table is 16K (if you're suffering
from memory fragmentation, the first thing which'd get you is the L1
page table allocation, not the kernel stack allocation.)

> Anyway, it provides one option for user to apply 4k stack to avoid
> such kind of process creation failure.

I refer you to the comments made by people who've tried running with 4K
stacks on x86, and their _vast_ experience of doing this.  If they say
that it causes stack overflows, then it's a problem.

The possibility of a kernel stack overflow is not something that should
be taken lightly - an overflow of the kernel stack means that the thread
data will be corrupted, so destroying the ability for debug information
to be emitted.

[PATCH 0/3] ARM 4Kstacks: introduction

[Andi Kleen]

>> If system run for some time, sometimes it may be difficult for
>> memory allocator to allocate 2 continuous page frames even  there are
>> many spare page frames in system because of
>> fragment issue, so the patch does make sense. Anyway,
>> it provides one option for user to apply 4k stack to avoid
>> such kind of process creation failure.
>
> Can't page migration be used in these situations?

The kernel already uses it of course, assuming the pages are movable.
So if you are really worried about this, just define a movable zone.

-Andi

[PATCH 0/3] ARM 4Kstacks: introduction

[Bernd Petrovitsch]

On Mit, 2011-10-19 at 12:51 +0200, Arnd Bergmann wrote:
> On Tuesday 18 October 2011 17:26:44 Tim Bird wrote:
> > Even inside Sony, usage of 4K stacks is limited
> > to some very special cases, where memory is exceedingly
> > tight (we have one system with 4M of RAM).  And we
> > don't mind lopping off features or coding around
> > problem areas to support our special case.

Yes, for low memory situations (as in up to 8M) it is usually necessary
anyways to write every "./configure" line yourself for packages to make
sure everything possible is disabled unless it is absolutely necessary.
And a minimal custom kernel config is also unavoidable.

And then you don't have any fancy filesystems (like ext2 or some FAT
variant), loopback devices, stacking block devices or other "recursive"
parts - just a flash file system (and not necessarily that if you run
the system from the initrd which is loaded and decompressed by the boot
loader).

So all the points with "stack overflow with stacked $WHATEVER" can't
happen anyways. And in this situation one controls the user-space too,
so there won't be any obscure "root can do" issues anyways.

[ and there are more potential savings in the kernel - and user-space -
for these devices/systems. ]

> I would imagine that in those cases, you can gain more by reducing the
> number of threads in the system. What is the highest number of

If you are very limited on memory, you do this anyways: minimizing the
number of processes and threads - or better just do not use threads at
all because it also saves space.

	Bernd
-- 
Bernd Petrovitsch                  Email : bernd at petrovitsch.priv.at
                     LUGA : http://www.luga.at
 

[PATCH 0/3] ARM 4Kstacks: introduction

[Tim Bird]

On 10/22/2011 6:36 AM, Russell King - ARM Linux wrote:
> On Sat, Oct 22, 2011 at 04:50:15PM +0800, Ming Lei wrote:
>> On Wed, Oct 19, 2011 at 6:51 PM, Arnd Bergmann<arnd@arndb.de>  wrote:
>>> On Tuesday 18 October 2011 17:26:44 Tim Bird wrote:
>>>> Even inside Sony, usage of 4K stacks is limited
>>>> to some very special cases, where memory is exceedingly
>>>> tight (we have one system with 4M of RAM).  And we
>>>> don't mind lopping off features or coding around
>>>> problem areas to support our special case.
>>> I would imagine that in those cases, you can gain more by reducing the
>>> number of threads in the system. What is the highest number of
>>> concurrent threads that you expect in a limited use case with no
>>> networking or block devices?
We have about 50 hard real-time threads, that are part of a
software stack for digital cameras that was ported over from
micro-itron.  It has taken a _LONG_ time (on the order of a
few years) to tune the Linux system using RT-preempt to run
these threads as is.  It would be very painful to re-architect this
part of the system.

Note that these threads don't have any of the issues that
people have raised about filesystem stack depth or printk
recursion, since they avoid a whole range of Linux syscalls
to avoid real-time issues (and stack size issues).

I'm looking at possibly implementing a mixed stack
size system, but I don't know if that will work, or whether
it would be acceptable upstream.

>>> If system run for some time, sometimes it may be difficult for
>>> memory allocator to allocate 2 continuous page frames even  there are
>>> many spare page frames in system because of
>>> fragment issue, so the patch does make sense.
> If memory fragmentation is an issue for this, it probably means that we
> need to switch to a software page size of 8K (or maybe 16K) rather than
> stick with the hardware 4K size.  That would be a much more reliable
> solution, especially as the L1 page table is 16K (if you're suffering
> from memory fragmentation, the first thing which'd get you is the L1
> page table allocation, not the kernel stack allocation.)
>
>> Anyway, it provides one option for user to apply 4k stack to avoid
>> such kind of process creation failure.
> I refer you to the comments made by people who've tried running with 4K
> stacks on x86, and their _vast_ experience of doing this.  If they say
> that it causes stack overflows, then it's a problem.
>
I really don't think anyone on x86 has any experience whatsoever
with anything like this.  This is on a digital camera, with a flash
filesystem, with a 10M memory budget, with just about every
in and out-of-tree patch from Linux-tiny applied and configured.
Many of the things that bloat up the kernel just aren't there at
all.

> The possibility of a kernel stack overflow is not something that should
> be taken lightly
Agreed.

This kind of development is done with extensive in-house testing,
and an absolutely fixed users space. This may sound like an
isolated case, but I know Sony is not alone and that lots of
embedded products are developed like this.  I'm pretty sure
others would benefit from this patch.

We've already shipped tens of thousands of cameras
with this, with no problems, so it's certainly possible to get it
right.

Whether to include this comes down to a question of whether
the ability of someone to get it wrong should preclude
allowing the *option* into the kernel.
   -- Tim

[PATCH 0/3] ARM 4Kstacks: introduction

[Russell King - ARM Linux]

On Sun, Oct 23, 2011 at 12:25:10PM -0700, Tim Bird wrote:
> We have about 50 hard real-time threads, that are part of a
> software stack for digital cameras that was ported over from
> micro-itron.  It has taken a _LONG_ time (on the order of a
> few years) to tune the Linux system using RT-preempt to run
> these threads as is.  It would be very painful to re-architect this
> part of the system.
>
> Note that these threads don't have any of the issues that
> people have raised about filesystem stack depth or printk
> recursion, since they avoid a whole range of Linux syscalls
> to avoid real-time issues (and stack size issues).
>
> I'm looking at possibly implementing a mixed stack
> size system, but I don't know if that will work, or whether
> it would be acceptable upstream.

I can't see how it would work, because the extremes of the kernel stack
are rather fixed.  At the top of the stack, we have the kernel stack
growing downwards.  At the bottom of the stack, we have the thread_info
structure.

If we have different stack sizes depending on the thread which is running,
you have a catch-22 situation - you need some per-thread data to find
out how large the kernel stack is, and that can't be stored in the
thread_info struct because to find that you need to know how large the
kernel stack is.

>> If memory fragmentation is an issue for this, it probably means that we
>> need to switch to a software page size of 8K (or maybe 16K) rather than
>> stick with the hardware 4K size.  That would be a much more reliable
>> solution, especially as the L1 page table is 16K (if you're suffering
>> from memory fragmentation, the first thing which'd get you is the L1
>> page table allocation, not the kernel stack allocation.)

So, the reason you don't have a problem with this is because you're
running 50 threads of the _same_ process, which are all sharing the
same L1 page table.  That information has a direct bearing on this
discussion.

>> The possibility of a kernel stack overflow is not something that should
>> be taken lightly
> Agreed.
>
> This kind of development is done with extensive in-house testing,
> and an absolutely fixed users space. This may sound like an
> isolated case, but I know Sony is not alone and that lots of
> embedded products are developed like this.  I'm pretty sure
> others would benefit from this patch.
>
> We've already shipped tens of thousands of cameras
> with this, with no problems, so it's certainly possible to get it
> right.
>
> Whether to include this comes down to a question of whether
> the ability of someone to get it wrong should preclude
> allowing the *option* into the kernel.

Given this comment, I assume you've already measured how much kernel
stack gets used, and would be able to provide those figures?

The kernel can give you this if you enable CONFIG_DEBUG_STACK_USAGE.  The
kernel will then report whenever a thread exits having used more kernel
stack than any previous exiting thread.

[PATCH 0/3] ARM 4Kstacks: introduction

[Ming Lei]

Hi,

On Sat, Oct 22, 2011 at 9:36 PM, Russell King - ARM Linux
<linux@arm.linux.org.uk> wrote:
> On Sat, Oct 22, 2011 at 04:50:15PM +0800, Ming Lei wrote:
>> On Wed, Oct 19, 2011 at 6:51 PM, Arnd Bergmann <arnd@arndb.de> wrote:
>> > On Tuesday 18 October 2011 17:26:44 Tim Bird wrote:
>> >> Even inside Sony, usage of 4K stacks is limited
>> >> to some very special cases, where memory is exceedingly
>> >> tight (we have one system with 4M of RAM). ?And we
>> >> don't mind lopping off features or coding around
>> >> problem areas to support our special case.
>> >
>> > I would imagine that in those cases, you can gain more by reducing the
>> > number of threads in the system. What is the highest number of
>> > concurrent threads that you expect in a limited use case with no
>> > networking or block devices?
>>
>> If system run for some time, sometimes it may be difficult for
>> memory allocator to allocate 2 continuous page frames even ?there are
>> many spare page frames in system because of
>> fragment issue, so the patch does make sense.
>
> If memory fragmentation is an issue for this, it probably means that we
> need to switch to a software page size of 8K (or maybe 16K) rather than
> stick with the hardware 4K size. ?That would be a much more reliable
> solution, especially as the L1 page table is 16K (if you're suffering
> from memory fragmentation, the first thing which'd get you is the L1
> page table allocation, not the kernel stack allocation.)

If large page size is taken, memory utilization will be decreased a lot, so
may cause some low memory system not workable at all.

>> Anyway, it provides one option for user to apply 4k stack to avoid
>> such kind of process creation failure.
>
> I refer you to the comments made by people who've tried running with 4K
> stacks on x86, and their _vast_ experience of doing this. ?If they say
> that it causes stack overflows, then it's a problem.

Stack overflow depends on code running path. For some low memory
and simple application, the code path is not deep as x86, so providing
a option of enabling 4K stack still makes sense for this kind of case or
application.


thanks,
-- 
Ming Lei