Re: [RFC 0/4] Transparent on-demand struct page initialization embedded in the buddy allocator

Yinghai Lu
2013-07-19T23:51:50Z

[-- Attachment #1: Type: text/plain, Size: 1102 bytes --]

On Wed, Jul 17, 2013 at 2:30 AM, Robin Holt <holt@sgi.com> wrote:
> On Wed, Jul 17, 2013 at 01:17:44PM +0800, Sam Ben wrote:
>> >With this patch, we did boot a 16TiB machine. Without the patches,
>> >the v3.10 kernel with the same configuration took 407 seconds for
>> >free_all_bootmem. With the patches and operating on 2MiB pages instead
>> >of 1GiB, it took 26 seconds so performance was improved. I have no feel
>> >for how the 1GiB chunk size will perform.
>>
>> How to test how much time spend on free_all_bootmem?
>
> We had put a pr_emerg at the beginning and end of free_all_bootmem and
> then used a modified version of script which record the time in uSecs
> at the beginning of each line of output.

used two patches, found 3TiB system will take 100s before slub is ready.

about three portions:
1. sparse vmemap buf allocation, it is with bootmem wrapper, so clear those
struct page area take about 30s.
2. memmap_init_zone: take about 25s
3. mem_init/free_all_bootmem about 30s.

so still wonder why 16TiB will need hours.

also your patches looks like only address 2 and 3.

Yinghai

[-- Attachment #2: printk_time_tsc_0.patch --]
[-- Type: application/octet-stream, Size: 2624 bytes --]

Subject: [PATCH] printk_time: prepare stub for using other than cpu_clock

-v2: refresh to v3.10

Signed-off-by: Yinghai Lu <yhlu.kernel@gmail.com>

---
include/linux/printk.h | 3 +++
init/main.c | 1 +
kernel/printk.c | 17 +++++++++++++++--
3 files changed, 19 insertions(+), 2 deletions(-)

Index: linux-2.6/init/main.c
===================================================================
--- linux-2.6.orig/init/main.c
+++ linux-2.6/init/main.c
@@ -601,6 +601,7 @@ asmlinkage void __init start_kernel(void
late_time_init();
sched_clock_init();
calibrate_delay();
+ set_printk_time_clock(local_clock);
pidmap_init();
anon_vma_init();
#ifdef CONFIG_X86
Index: linux-2.6/kernel/printk.c
===================================================================
--- linux-2.6.orig/kernel/printk.c
+++ linux-2.6/kernel/printk.c
@@ -217,6 +217,19 @@ struct log {
static DEFINE_RAW_SPINLOCK(logbuf_lock);

#ifdef CONFIG_PRINTK
+
+static u64 default_printk_time_clock(void)
+{
+ return 0;
+}
+
+static u64 (*printk_time_clock)(void) = default_printk_time_clock;
+
+void set_printk_time_clock(u64 (*fn)(void))
+{
+ printk_time_clock = fn;
+}
+
DECLARE_WAIT_QUEUE_HEAD(log_wait);
/* the next printk record to read by syslog(READ) or /proc/kmsg */
static u64 syslog_seq;
@@ -354,7 +367,7 @@ static void log_store(int facility, int
if (ts_nsec > 0)
msg->ts_nsec = ts_nsec;
else
- msg->ts_nsec = local_clock();
+ msg->ts_nsec = printk_time_clock();
memset(log_dict(msg) + dict_len, 0, pad_len);
msg->len = sizeof(struct log) + text_len + dict_len + pad_len;

@@ -1457,7 +1470,7 @@ static bool cont_add(int facility, int l
cont.facility = facility;
cont.level = level;
cont.owner = current;
- cont.ts_nsec = local_clock();
+ cont.ts_nsec = printk_time_clock();
cont.flags = 0;
cont.cons = 0;
cont.flushed = false;
Index: linux-2.6/include/linux/printk.h
===================================================================
--- linux-2.6.orig/include/linux/printk.h
+++ linux-2.6/include/linux/printk.h
@@ -107,6 +107,7 @@ void early_printk(const char *s, ...) {
#endif

#ifdef CONFIG_PRINTK
+extern void set_printk_time_clock(u64 (*fn)(void));
asmlinkage __printf(5, 0)
int vprintk_emit(int facility, int level,
const char *dict, size_t dictlen,
@@ -150,6 +151,8 @@ void dump_stack_set_arch_desc(const char
void dump_stack_print_info(const char *log_lvl);
void show_regs_print_info(const char *log_lvl);
#else
+static inline void set_printk_time_clock(u64 (*fn)(void)) { }
+
static inline __printf(1, 0)
int vprintk(const char *s, va_list args)
{

[-- Attachment #3: printk_time_tsc_1.patch --]
[-- Type: application/octet-stream, Size: 1201 bytes --]

Subject: [PATCH] x86: printk_time to use tsc before cpu_clock is ready

so can get tsc value on printk

need to apply after
[PATCH] printk_time: prepare stub for using other than cpu_clock

-v2: refresh on v3.10

Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
arch/x86/kernel/cpu/common.c | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)

Index: linux-2.6/arch/x86/kernel/cpu/common.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/cpu/common.c
+++ linux-2.6/arch/x86/kernel/cpu/common.c
@@ -737,6 +737,25 @@ static void __init early_identify_cpu(st
setup_force_cpu_cap(X86_FEATURE_ALWAYS);
}

+static u64 __initdata tsc_clock_base;
+
+static __init u64 tsc_clock_offset(void)
+{
+ unsigned long long t;
+
+ rdtscll(t);
+
+ return t - tsc_clock_base;
+}
+
+static __init void early_printk_time_init(void)
+{
+ if (cpu_has_tsc) {
+ rdtscll(tsc_clock_base);
+ set_printk_time_clock(tsc_clock_offset);
+ }
+}
+
void __init early_cpu_init(void)
{
const struct cpu_dev *const *cdev;
@@ -761,6 +780,8 @@ void __init early_cpu_init(void)
}

early_identify_cpu(&boot_cpu_data);
+
+ early_printk_time_init();
}

/*