* [RFC 1/5] x86_64: Fix early references to cpumask_of_cpu
2008-07-01 12:16 [RFC 0/5] percpu: Optimize percpu accesses Mike Travis
@ 2008-07-01 12:16 ` Mike Travis
2008-07-01 12:16 ` [RFC 2/5] x86_64: Fold pda into per cpu area Mike Travis
` (4 subsequent siblings)
5 siblings, 0 replies; 12+ messages in thread
From: Mike Travis @ 2008-07-01 12:16 UTC (permalink / raw)
To: Ingo Molnar, Jeremy Fitzhardinge, Eric W. Biederman
Cc: Christoph Lameter, H. Peter Anvin, Andrew Morton, linux-kernel
[-- Attachment #1: cleanup_percpu --]
[-- Type: text/plain, Size: 1795 bytes --]
* Initialize the cpumask_of_cpu_map to contain a cpumask for cpu 0
in the initdata section. This allows references before the real
cpumask_of_cpu_map is setup avoiding possible null pointer deref
panics.
Based on linux-2.6.tip/master
Signed-off-by: Mike Travis <travis@sgi.com>
---
arch/x86/kernel/setup_percpu.c | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
--- linux-2.6.tip.orig/arch/x86/kernel/setup_percpu.c
+++ linux-2.6.tip/arch/x86/kernel/setup_percpu.c
@@ -51,7 +51,7 @@ static void __init setup_node_to_cpumask
static inline void setup_node_to_cpumask_map(void) { }
#endif
-#if defined(CONFIG_HAVE_SETUP_PER_CPU_AREA) && defined(CONFIG_SMP)
+#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
/*
* Copy data used in early init routines from the initial arrays to the
* per cpu data areas. These arrays then become expendable and the
@@ -81,16 +81,25 @@ static void __init setup_per_cpu_maps(vo
}
#ifdef CONFIG_HAVE_CPUMASK_OF_CPU_MAP
-cpumask_t *cpumask_of_cpu_map __read_mostly;
+
+/* Configure an initial cpumask_of_cpu(0) for early users */
+static cpumask_t initial_cpumask_of_cpu_map __initdata = (cpumask_t) { {
+ [BITS_TO_LONGS(NR_CPUS)-1] = 1
+} };
+cpumask_t *cpumask_of_cpu_map __read_mostly =
+ (cpumask_t *)&initial_cpumask_of_cpu_map;
EXPORT_SYMBOL(cpumask_of_cpu_map);
-/* requires nr_cpu_ids to be initialized */
+/* Requires nr_cpu_ids to be initialized */
static void __init setup_cpumask_of_cpu(void)
{
int i;
/* alloc_bootmem zeroes memory */
cpumask_of_cpu_map = alloc_bootmem_low(sizeof(cpumask_t) * nr_cpu_ids);
+#ifdef CONFIG_DEBUG_PER_CPU_MAPS
+ printk(KERN_DEBUG "cpumask_of_cpu_map %p\n", cpumask_of_cpu_map);
+#endif
for (i = 0; i < nr_cpu_ids; i++)
cpu_set(i, cpumask_of_cpu_map[i]);
}
--
^ permalink raw reply [flat|nested] 12+ messages in thread* [RFC 2/5] x86_64: Fold pda into per cpu area
2008-07-01 12:16 [RFC 0/5] percpu: Optimize percpu accesses Mike Travis
2008-07-01 12:16 ` [RFC 1/5] x86_64: Fix early references to cpumask_of_cpu Mike Travis
@ 2008-07-01 12:16 ` Mike Travis
2008-07-01 12:16 ` [RFC 3/5] x86_64: Reference zero-based percpu variables offset from gs Mike Travis
` (3 subsequent siblings)
5 siblings, 0 replies; 12+ messages in thread
From: Mike Travis @ 2008-07-01 12:16 UTC (permalink / raw)
To: Ingo Molnar, Jeremy Fitzhardinge, Eric W. Biederman
Cc: Christoph Lameter, H. Peter Anvin, Andrew Morton, linux-kernel
[-- Attachment #1: zero_based_fold --]
[-- Type: text/plain, Size: 14961 bytes --]
* Declare the pda as a per cpu variable.
* Make the x86_64 per cpu area start at zero.
* Relocate the initial pda and per_cpu(gdt_page) in head_64.S for the
boot cpu (0). For secondary cpus, do_boot_cpu() sets up the correct
initial pda and gdt_page pointer.
* Initialize per_cpu_offset to point to static pda in the per_cpu area
(@ __per_cpu_load).
* After allocation of the per cpu area for the boot cpu (0), reload the
gdt page pointer.
Based on linux-2.6.tip/master
Signed-off-by: Christoph Lameter <cl@linux-foundation.org>
Signed-off-by: Mike Travis <travis@sgi.com>
---
arch/x86/Kconfig | 3 +
arch/x86/kernel/acpi/sleep.c | 9 +++
arch/x86/kernel/cpu/common_64.c | 4 -
arch/x86/kernel/head64.c | 24 +--------
arch/x86/kernel/head_64.S | 45 +++++++++++++++--
arch/x86/kernel/setup_percpu.c | 100 ++++++++++++++++-----------------------
arch/x86/kernel/smpboot.c | 52 --------------------
arch/x86/kernel/vmlinux_64.lds.S | 1
include/asm-x86/desc.h | 5 +
include/asm-x86/pda.h | 3 -
include/asm-x86/percpu.h | 13 -----
include/asm-x86/trampoline.h | 1
12 files changed, 112 insertions(+), 148 deletions(-)
--- linux-2.6.tip.orig/arch/x86/Kconfig
+++ linux-2.6.tip/arch/x86/Kconfig
@@ -129,6 +129,9 @@ config HAVE_SETUP_PER_CPU_AREA
config HAVE_CPUMASK_OF_CPU_MAP
def_bool X86_64_SMP
+config HAVE_ZERO_BASED_PER_CPU
+ def_bool X86_64_SMP
+
config ARCH_HIBERNATION_POSSIBLE
def_bool y
depends on !SMP || !X86_VOYAGER
--- linux-2.6.tip.orig/arch/x86/kernel/acpi/sleep.c
+++ linux-2.6.tip/arch/x86/kernel/acpi/sleep.c
@@ -89,6 +89,15 @@ int acpi_save_state_mem(void)
#ifdef CONFIG_SMP
stack_start.sp = temp_stack + 4096;
#endif
+ /*
+ * FIXME: with zero-based percpu variables, the pda and gdt_page
+ * addresses must be offset by the base of this cpu's percpu area.
+ * Where/how should we do this?
+ *
+ * for secondary cpu startup in smpboot.c:do_boot_cpu() this is done:
+ * early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
+ * initial_pda = (unsigned long)get_cpu_pda(cpu);
+ */
initial_code = (unsigned long)wakeup_long64;
saved_magic = 0x123456789abcdef0;
#endif /* CONFIG_64BIT */
--- linux-2.6.tip.orig/arch/x86/kernel/cpu/common_64.c
+++ linux-2.6.tip/arch/x86/kernel/cpu/common_64.c
@@ -423,8 +423,8 @@ __setup("clearcpuid=", setup_disablecpui
cpumask_t cpu_initialized __cpuinitdata = CPU_MASK_NONE;
-struct x8664_pda **_cpu_pda __read_mostly;
-EXPORT_SYMBOL(_cpu_pda);
+DEFINE_PER_CPU_FIRST(struct x8664_pda, pda);
+EXPORT_PER_CPU_SYMBOL(pda);
struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table };
--- linux-2.6.tip.orig/arch/x86/kernel/head64.c
+++ linux-2.6.tip/arch/x86/kernel/head64.c
@@ -25,20 +25,6 @@
#include <asm/e820.h>
#include <asm/bios_ebda.h>
-/* boot cpu pda */
-static struct x8664_pda _boot_cpu_pda __read_mostly;
-
-#ifdef CONFIG_SMP
-/*
- * We install an empty cpu_pda pointer table to indicate to early users
- * (numa_set_node) that the cpu_pda pointer table for cpus other than
- * the boot cpu is not yet setup.
- */
-static struct x8664_pda *__cpu_pda[NR_CPUS] __initdata;
-#else
-static struct x8664_pda *__cpu_pda[NR_CPUS] __read_mostly;
-#endif
-
static void __init zap_identity_mappings(void)
{
pgd_t *pgd = pgd_offset_k(0UL);
@@ -91,6 +77,10 @@ void __init x86_64_start_kernel(char * r
/* Cleanup the over mapped high alias */
cleanup_highmap();
+ /* Initialize boot cpu_pda data */
+ /* (See head_64.S for earlier pda/gdt initialization) */
+ pda_init(0);
+
for (i = 0; i < NUM_EXCEPTION_VECTORS; i++) {
#ifdef CONFIG_EARLY_PRINTK
set_intr_gate(i, &early_idt_handlers[i]);
@@ -102,12 +92,6 @@ void __init x86_64_start_kernel(char * r
early_printk("Kernel alive\n");
- _cpu_pda = __cpu_pda;
- cpu_pda(0) = &_boot_cpu_pda;
- pda_init(0);
-
- early_printk("Kernel really alive\n");
-
copy_bootdata(__va(real_mode_data));
reserve_early(__pa_symbol(&_text), __pa_symbol(&_end), "TEXT DATA BSS");
--- linux-2.6.tip.orig/arch/x86/kernel/head_64.S
+++ linux-2.6.tip/arch/x86/kernel/head_64.S
@@ -12,6 +12,7 @@
#include <linux/linkage.h>
#include <linux/threads.h>
#include <linux/init.h>
+#include <asm/asm-offsets.h>
#include <asm/desc.h>
#include <asm/segment.h>
#include <asm/pgtable.h>
@@ -203,7 +204,27 @@ ENTRY(secondary_startup_64)
* addresses where we're currently running on. We have to do that here
* because in 32bit we couldn't load a 64bit linear address.
*/
- lgdt early_gdt_descr(%rip)
+
+#ifdef CONFIG_SMP
+ /*
+ * For zero-based percpu variables, the base (__per_cpu_load) must
+ * be added to the offset of per_cpu__gdt_page. This is only needed
+ * for the boot cpu but we can't do this prior to secondary_startup_64.
+ * So we use a NULL gdt adrs to indicate that we are starting up the
+ * boot cpu and not the secondary cpus. do_boot_cpu() will fixup
+ * the gdt adrs for those cpus.
+ */
+#define PER_CPU_GDT_PAGE 0
+ movq early_gdt_descr_base(%rip), %rax
+ testq %rax, %rax
+ jnz 1f
+ movq $__per_cpu_load, %rax
+ addq $per_cpu__gdt_page, %rax
+ movq %rax, early_gdt_descr_base(%rip)
+#else
+#define PER_CPU_GDT_PAGE per_cpu__gdt_page
+#endif
+1: lgdt early_gdt_descr(%rip)
/* set up data segments. actually 0 would do too */
movl $__KERNEL_DS,%eax
@@ -220,14 +241,21 @@ ENTRY(secondary_startup_64)
movl %eax,%gs
/*
- * Setup up a dummy PDA. this is just for some early bootup code
- * that does in_interrupt()
+ * Setup up the real PDA.
+ *
+ * For SMP, the boot cpu (0) uses the static pda which is the first
+ * element in the percpu area (@__per_cpu_load). This pda is moved
+ * to the real percpu area once that is allocated. Secondary cpus
+ * will use the initial_pda value setup in do_boot_cpu().
*/
movl $MSR_GS_BASE,%ecx
- movq $empty_zero_page,%rax
+ movq initial_pda(%rip), %rax
movq %rax,%rdx
shrq $32,%rdx
wrmsr
+#ifdef CONFIG_SMP
+ movq %rax, %gs:pda_data_offset
+#endif
/* esi is pointer to real mode structure with interesting info.
pass it to C */
@@ -250,6 +278,12 @@ ENTRY(secondary_startup_64)
.align 8
ENTRY(initial_code)
.quad x86_64_start_kernel
+ ENTRY(initial_pda)
+#ifdef CONFIG_SMP
+ .quad __per_cpu_load # Overwritten for secondary CPUs
+#else
+ .quad per_cpu__pda
+#endif
__FINITDATA
ENTRY(stack_start)
@@ -394,7 +428,8 @@ NEXT_PAGE(level2_spare_pgt)
.globl early_gdt_descr
early_gdt_descr:
.word GDT_ENTRIES*8-1
- .quad per_cpu__gdt_page
+early_gdt_descr_base:
+ .quad PER_CPU_GDT_PAGE # Overwritten for secondary CPUs
ENTRY(phys_base)
/* This must match the first entry in level2_kernel_pgt */
--- linux-2.6.tip.orig/arch/x86/kernel/setup_percpu.c
+++ linux-2.6.tip/arch/x86/kernel/setup_percpu.c
@@ -14,6 +14,7 @@
#include <asm/mpspec.h>
#include <asm/apicdef.h>
#include <asm/highmem.h>
+#include <asm/desc.h>
#ifdef CONFIG_X86_LOCAL_APIC
unsigned int num_processors;
@@ -107,64 +108,19 @@ static void __init setup_cpumask_of_cpu(
static inline void setup_cpumask_of_cpu(void) { }
#endif
-#ifdef CONFIG_X86_32
-/*
- * Great future not-so-futuristic plan: make i386 and x86_64 do it
- * the same way
- */
+#ifndef CONFIG_HAVE_ZERO_BASED_PER_CPU
unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
EXPORT_SYMBOL(__per_cpu_offset);
-static inline void setup_cpu_pda_map(void) { }
-
-#elif !defined(CONFIG_SMP)
-static inline void setup_cpu_pda_map(void) { }
-
-#else /* CONFIG_SMP && CONFIG_X86_64 */
-
-/*
- * Allocate cpu_pda pointer table and array via alloc_bootmem.
- */
-static void __init setup_cpu_pda_map(void)
-{
- char *pda;
- struct x8664_pda **new_cpu_pda;
- unsigned long size;
- int cpu;
-
- size = roundup(sizeof(struct x8664_pda), cache_line_size());
-
- /* allocate cpu_pda array and pointer table */
- {
- unsigned long tsize = nr_cpu_ids * sizeof(void *);
- unsigned long asize = size * (nr_cpu_ids - 1);
-
- tsize = roundup(tsize, cache_line_size());
- new_cpu_pda = alloc_bootmem(tsize + asize);
- pda = (char *)new_cpu_pda + tsize;
- }
-
- /* initialize pointer table to static pda's */
- for_each_possible_cpu(cpu) {
- if (cpu == 0) {
- /* leave boot cpu pda in place */
- new_cpu_pda[0] = cpu_pda(0);
- continue;
- }
- new_cpu_pda[cpu] = (struct x8664_pda *)pda;
- new_cpu_pda[cpu]->in_bootmem = 1;
- pda += size;
- }
+#else
- /* point to new pointer table */
- _cpu_pda = new_cpu_pda;
-}
+/* Initialize percpu offset for boot cpu (0) to static percpu area */
+unsigned long __per_cpu_offset[NR_CPUS] __read_mostly = {
+ [0] = (unsigned long)__per_cpu_load
+};
+EXPORT_SYMBOL(__per_cpu_offset);
#endif
-/*
- * Great future plan:
- * Declare PDA itself and support (irqstack,tss,pgd) as per cpu data.
- * Always point %gs to its beginning
- */
+/* Allocate and initialize the per cpu areas which include the PDA's */
void __init setup_per_cpu_areas(void)
{
ssize_t size = PERCPU_ENOUGH_ROOM;
@@ -181,9 +137,6 @@ void __init setup_per_cpu_areas(void)
nr_cpu_ids = num_processors;
#endif
- /* Setup cpu_pda map */
- setup_cpu_pda_map();
-
/* Copy section for each CPU (we discard the original) */
size = PERCPU_ENOUGH_ROOM;
printk(KERN_INFO "PERCPU: Allocating %zd bytes of per cpu data\n",
@@ -203,9 +156,42 @@ void __init setup_per_cpu_areas(void)
else
ptr = alloc_bootmem_pages_node(NODE_DATA(node), size);
#endif
+ /* Initialize each cpu's per_cpu area and save pointer */
+ memcpy(ptr, __per_cpu_load, __per_cpu_size);
per_cpu_offset(cpu) = ptr - __per_cpu_start;
- memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
+#ifdef CONFIG_DEBUG_PER_CPU_MAPS
+ printk(KERN_DEBUG "PERCPU: cpu %d %p\n", cpu, ptr);
+#endif
+
+#ifdef CONFIG_X86_64
+ /*
+ * Note the boot cpu has been using the static per_cpu load
+ * area for it's pda. We need to zero out the pda's for the
+ * other cpu's that are coming online.
+ *
+ * Additionally, the gdt page must be reloaded as we moved
+ * it from the static percpu area to the newly allocated area.
+ */
+ {
+ /* We rely on the fact that pda is the first element */
+ struct x8664_pda *pda = (struct x8664_pda *)ptr;
+
+ if (cpu) {
+ memset(pda, 0, sizeof(*pda));
+ pda->data_offset = (unsigned long)ptr;
+ } else {
+ struct desc_ptr gdt_descr = early_gdt_descr;
+
+ pda->data_offset = (unsigned long)ptr;
+ gdt_descr.address =
+ (unsigned long)get_cpu_gdt_table(0);
+ native_load_gdt(&gdt_descr);
+ pda_init(0);
+ }
+
+ }
+#endif
}
printk(KERN_DEBUG "NR_CPUS: %d, nr_cpu_ids: %d, nr_node_ids %d\n",
--- linux-2.6.tip.orig/arch/x86/kernel/smpboot.c
+++ linux-2.6.tip/arch/x86/kernel/smpboot.c
@@ -762,45 +762,6 @@ static void __cpuinit do_fork_idle(struc
complete(&c_idle->done);
}
-#ifdef CONFIG_X86_64
-/*
- * Allocate node local memory for the AP pda.
- *
- * Must be called after the _cpu_pda pointer table is initialized.
- */
-static int __cpuinit get_local_pda(int cpu)
-{
- struct x8664_pda *oldpda, *newpda;
- unsigned long size = sizeof(struct x8664_pda);
- int node = cpu_to_node(cpu);
-
- if (cpu_pda(cpu) && !cpu_pda(cpu)->in_bootmem)
- return 0;
-
- oldpda = cpu_pda(cpu);
- newpda = kmalloc_node(size, GFP_ATOMIC, node);
- if (!newpda) {
- printk(KERN_ERR "Could not allocate node local PDA "
- "for CPU %d on node %d\n", cpu, node);
-
- if (oldpda)
- return 0; /* have a usable pda */
- else
- return -1;
- }
-
- if (oldpda) {
- memcpy(newpda, oldpda, size);
- if (!after_bootmem)
- free_bootmem((unsigned long)oldpda, size);
- }
-
- newpda->in_bootmem = 0;
- cpu_pda(cpu) = newpda;
- return 0;
-}
-#endif /* CONFIG_X86_64 */
-
static int __cpuinit do_boot_cpu(int apicid, int cpu)
/*
* NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
@@ -818,16 +779,6 @@ static int __cpuinit do_boot_cpu(int api
};
INIT_WORK(&c_idle.work, do_fork_idle);
-#ifdef CONFIG_X86_64
- /* Allocate node local memory for AP pdas */
- if (cpu > 0) {
- boot_error = get_local_pda(cpu);
- if (boot_error)
- goto restore_state;
- /* if can't get pda memory, can't start cpu */
- }
-#endif
-
alternatives_smp_switch(1);
c_idle.idle = get_idle_for_cpu(cpu);
@@ -865,6 +816,7 @@ do_rest:
#else
cpu_pda(cpu)->pcurrent = c_idle.idle;
clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
+ initial_pda = (unsigned long)get_cpu_pda(cpu);
#endif
early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
initial_code = (unsigned long)start_secondary;
@@ -940,8 +892,6 @@ do_rest:
}
}
-restore_state:
-
if (boot_error) {
/* Try to put things back the way they were before ... */
numa_remove_cpu(cpu); /* was set by numa_add_cpu */
--- linux-2.6.tip.orig/arch/x86/kernel/vmlinux_64.lds.S
+++ linux-2.6.tip/arch/x86/kernel/vmlinux_64.lds.S
@@ -16,6 +16,7 @@ jiffies_64 = jiffies;
_proxy_pda = 1;
PHDRS {
text PT_LOAD FLAGS(5); /* R_E */
+ percpu PT_LOAD FLAGS(7); /* RWE */
data PT_LOAD FLAGS(7); /* RWE */
user PT_LOAD FLAGS(7); /* RWE */
data.init PT_LOAD FLAGS(7); /* RWE */
--- linux-2.6.tip.orig/include/asm-x86/desc.h
+++ linux-2.6.tip/include/asm-x86/desc.h
@@ -41,6 +41,11 @@ static inline struct desc_struct *get_cp
#ifdef CONFIG_X86_64
+static inline struct x8664_pda *get_cpu_pda(unsigned int cpu)
+{
+ return &per_cpu(pda, cpu);
+}
+
static inline void pack_gate(gate_desc *gate, unsigned type, unsigned long func,
unsigned dpl, unsigned ist, unsigned seg)
{
--- linux-2.6.tip.orig/include/asm-x86/pda.h
+++ linux-2.6.tip/include/asm-x86/pda.h
@@ -37,10 +37,9 @@ struct x8664_pda {
unsigned irq_spurious_count;
} ____cacheline_aligned_in_smp;
-extern struct x8664_pda **_cpu_pda;
extern void pda_init(int);
-#define cpu_pda(i) (_cpu_pda[i])
+#define cpu_pda(cpu) (&per_cpu(pda, cpu))
/*
* There is no fast way to get the base address of the PDA, all the accesses
--- linux-2.6.tip.orig/include/asm-x86/percpu.h
+++ linux-2.6.tip/include/asm-x86/percpu.h
@@ -3,20 +3,11 @@
#ifdef CONFIG_X86_64
#include <linux/compiler.h>
-
-/* Same as asm-generic/percpu.h, except that we store the per cpu offset
- in the PDA. Longer term the PDA and every per cpu variable
- should be just put into a single section and referenced directly
- from %gs */
-
-#ifdef CONFIG_SMP
#include <asm/pda.h>
-#define __per_cpu_offset(cpu) (cpu_pda(cpu)->data_offset)
+/* Same as asm-generic/percpu.h */
+#ifdef CONFIG_SMP
#define __my_cpu_offset read_pda(data_offset)
-
-#define per_cpu_offset(x) (__per_cpu_offset(x))
-
#endif
#include <asm-generic/percpu.h>
--- linux-2.6.tip.orig/include/asm-x86/trampoline.h
+++ linux-2.6.tip/include/asm-x86/trampoline.h
@@ -12,6 +12,7 @@ extern unsigned char *trampoline_base;
extern unsigned long init_rsp;
extern unsigned long initial_code;
+extern unsigned long initial_pda;
#define TRAMPOLINE_BASE 0x6000
extern unsigned long setup_trampoline(void);
--
^ permalink raw reply [flat|nested] 12+ messages in thread* [RFC 3/5] x86_64: Reference zero-based percpu variables offset from gs
2008-07-01 12:16 [RFC 0/5] percpu: Optimize percpu accesses Mike Travis
2008-07-01 12:16 ` [RFC 1/5] x86_64: Fix early references to cpumask_of_cpu Mike Travis
2008-07-01 12:16 ` [RFC 2/5] x86_64: Fold pda into per cpu area Mike Travis
@ 2008-07-01 12:16 ` Mike Travis
2008-07-01 12:16 ` [RFC 4/5] x86_64: Replace cpu_pda ops with percpu ops Mike Travis
` (2 subsequent siblings)
5 siblings, 0 replies; 12+ messages in thread
From: Mike Travis @ 2008-07-01 12:16 UTC (permalink / raw)
To: Ingo Molnar, Jeremy Fitzhardinge, Eric W. Biederman
Cc: Christoph Lameter, H. Peter Anvin, Andrew Morton, linux-kernel
[-- Attachment #1: zero_based_use_gs --]
[-- Type: text/plain, Size: 2261 bytes --]
* Since %gs is pointing to the pda, it will then also point to the per cpu
variables and can be accessed thusly:
%gs:[&per_cpu_xxxx - __per_cpu_start]
... and since __per_cpu_start == 0 then:
%gs:&per_cpu_var(xxx)
becomes the optimized effective address.
Signed-off-by: Christoph Lameter <cl@linux-foundation.org>
Signed-off-by: Mike Travis <travis@sgi.com>
---
include/asm-x86/percpu.h | 36 +++++++++++++-----------------------
1 file changed, 13 insertions(+), 23 deletions(-)
--- linux-2.6.tip.orig/include/asm-x86/percpu.h
+++ linux-2.6.tip/include/asm-x86/percpu.h
@@ -5,15 +5,19 @@
#include <linux/compiler.h>
#include <asm/pda.h>
-/* Same as asm-generic/percpu.h */
+/* Same as asm-generic/percpu.h, except we use %gs as a segment offset. */
#ifdef CONFIG_SMP
#define __my_cpu_offset read_pda(data_offset)
+#define __percpu_seg "%%gs:"
+#else
+#define __percpu_seg ""
#endif
+
#include <asm-generic/percpu.h>
DECLARE_PER_CPU(struct x8664_pda, pda);
-#else /* CONFIG_X86_64 */
+#else /* !CONFIG_X86_64 */
#ifdef __ASSEMBLY__
@@ -42,36 +46,23 @@ DECLARE_PER_CPU(struct x8664_pda, pda);
#else /* ...!ASSEMBLY */
-/*
- * PER_CPU finds an address of a per-cpu variable.
- *
- * Args:
- * var - variable name
- * cpu - 32bit register containing the current CPU number
- *
- * The resulting address is stored in the "cpu" argument.
- *
- * Example:
- * PER_CPU(cpu_gdt_descr, %ebx)
- */
#ifdef CONFIG_SMP
-
#define __my_cpu_offset x86_read_percpu(this_cpu_off)
-
-/* fs segment starts at (positive) offset == __per_cpu_offset[cpu] */
#define __percpu_seg "%%fs:"
-
-#else /* !SMP */
-
+#else
#define __percpu_seg ""
-
-#endif /* SMP */
+#endif
#include <asm-generic/percpu.h>
/* We can use this directly for local CPU (faster). */
DECLARE_PER_CPU(unsigned long, this_cpu_off);
+#endif /* __ASSEMBLY__ */
+#endif /* !CONFIG_X86_64 */
+
+#ifndef __ASSEMBLY__
+
/* For arch-specific code, we can use direct single-insn ops (they
* don't give an lvalue though). */
extern void __bad_percpu_size(void);
@@ -206,7 +197,6 @@ do { \
percpu_cmpxchg_op(per_cpu_var(var), old, new)
#endif /* !__ASSEMBLY__ */
-#endif /* !CONFIG_X86_64 */
#ifdef CONFIG_SMP
--
^ permalink raw reply [flat|nested] 12+ messages in thread* [RFC 4/5] x86_64: Replace cpu_pda ops with percpu ops
2008-07-01 12:16 [RFC 0/5] percpu: Optimize percpu accesses Mike Travis
` (2 preceding siblings ...)
2008-07-01 12:16 ` [RFC 3/5] x86_64: Reference zero-based percpu variables offset from gs Mike Travis
@ 2008-07-01 12:16 ` Mike Travis
2008-07-01 12:16 ` [RFC 5/5] x86_64: Replace xxx_pda() operations with x86_xxx_percpu() Mike Travis
2008-07-01 14:54 ` [RFC 0/5] percpu: Optimize percpu accesses Ingo Molnar
5 siblings, 0 replies; 12+ messages in thread
From: Mike Travis @ 2008-07-01 12:16 UTC (permalink / raw)
To: Ingo Molnar, Jeremy Fitzhardinge, Eric W. Biederman
Cc: Christoph Lameter, H. Peter Anvin, Andrew Morton, linux-kernel
[-- Attachment #1: zero_based_replace_cpu_pda --]
[-- Type: text/plain, Size: 6553 bytes --]
* Replace cpu_pda(i) references with percpu(pda, i).
Based on linux-2.6.tip/master
Signed-off-by: Christoph Lameter <cl@linux-foundation.org>
Signed-off-by: Mike Travis <travis@sgi.com>
---
arch/x86/kernel/cpu/common_64.c | 2 +-
arch/x86/kernel/irq_64.c | 36 ++++++++++++++++++++----------------
arch/x86/kernel/setup_percpu.c | 22 ++++++++++++++--------
arch/x86/kernel/smpboot.c | 2 +-
arch/x86/kernel/traps_64.c | 11 +++++++----
5 files changed, 43 insertions(+), 30 deletions(-)
--- linux-2.6.tip.orig/arch/x86/kernel/cpu/common_64.c
+++ linux-2.6.tip/arch/x86/kernel/cpu/common_64.c
@@ -477,7 +477,7 @@ __setup("noexec32=", nonx32_setup);
void pda_init(int cpu)
{
- struct x8664_pda *pda = cpu_pda(cpu);
+ struct x8664_pda *pda = &per_cpu(pda, cpu);
/* Setup up data that may be needed in __get_free_pages early */
asm volatile("movl %0,%%fs ; movl %0,%%gs" :: "r" (0));
--- linux-2.6.tip.orig/arch/x86/kernel/irq_64.c
+++ linux-2.6.tip/arch/x86/kernel/irq_64.c
@@ -115,39 +115,43 @@ skip:
} else if (i == NR_IRQS) {
seq_printf(p, "NMI: ");
for_each_online_cpu(j)
- seq_printf(p, "%10u ", cpu_pda(j)->__nmi_count);
+ seq_printf(p, "%10u ", per_cpu(pda.__nmi_count, j));
seq_printf(p, " Non-maskable interrupts\n");
seq_printf(p, "LOC: ");
for_each_online_cpu(j)
- seq_printf(p, "%10u ", cpu_pda(j)->apic_timer_irqs);
+ seq_printf(p, "%10u ", per_cpu(pda.apic_timer_irqs, j));
seq_printf(p, " Local timer interrupts\n");
#ifdef CONFIG_SMP
seq_printf(p, "RES: ");
for_each_online_cpu(j)
- seq_printf(p, "%10u ", cpu_pda(j)->irq_resched_count);
+ seq_printf(p, "%10u ",
+ per_cpu(pda.irq_resched_count, j));
seq_printf(p, " Rescheduling interrupts\n");
seq_printf(p, "CAL: ");
for_each_online_cpu(j)
- seq_printf(p, "%10u ", cpu_pda(j)->irq_call_count);
+ seq_printf(p, "%10u ", per_cpu(pda.irq_call_count, j));
seq_printf(p, " function call interrupts\n");
seq_printf(p, "TLB: ");
for_each_online_cpu(j)
- seq_printf(p, "%10u ", cpu_pda(j)->irq_tlb_count);
+ seq_printf(p, "%10u ", per_cpu(pda.irq_tlb_count, j));
seq_printf(p, " TLB shootdowns\n");
#endif
#ifdef CONFIG_X86_MCE
seq_printf(p, "TRM: ");
for_each_online_cpu(j)
- seq_printf(p, "%10u ", cpu_pda(j)->irq_thermal_count);
+ seq_printf(p, "%10u ",
+ per_cpu(pda.irq_thermal_count, j));
seq_printf(p, " Thermal event interrupts\n");
seq_printf(p, "THR: ");
for_each_online_cpu(j)
- seq_printf(p, "%10u ", cpu_pda(j)->irq_threshold_count);
+ seq_printf(p, "%10u ",
+ per_cpu(pda.irq_threshold_count, j));
seq_printf(p, " Threshold APIC interrupts\n");
#endif
seq_printf(p, "SPU: ");
for_each_online_cpu(j)
- seq_printf(p, "%10u ", cpu_pda(j)->irq_spurious_count);
+ seq_printf(p, "%10u ",
+ per_cpu(pda.irq_spurious_count, j));
seq_printf(p, " Spurious interrupts\n");
seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count));
}
@@ -159,19 +163,19 @@ skip:
*/
u64 arch_irq_stat_cpu(unsigned int cpu)
{
- u64 sum = cpu_pda(cpu)->__nmi_count;
+ u64 sum = per_cpu(pda.__nmi_count, cpu);
- sum += cpu_pda(cpu)->apic_timer_irqs;
+ sum += per_cpu(pda.apic_timer_irqs, cpu);
#ifdef CONFIG_SMP
- sum += cpu_pda(cpu)->irq_resched_count;
- sum += cpu_pda(cpu)->irq_call_count;
- sum += cpu_pda(cpu)->irq_tlb_count;
+ sum += per_cpu(pda.irq_resched_count, cpu);
+ sum += per_cpu(pda.irq_call_count, cpu);
+ sum += per_cpu(pda.irq_tlb_count, cpu);
#endif
#ifdef CONFIG_X86_MCE
- sum += cpu_pda(cpu)->irq_thermal_count;
- sum += cpu_pda(cpu)->irq_threshold_count;
+ sum += per_cpu(pda.irq_thermal_count, cpu);
+ sum += per_cpu(pda.irq_threshold_count, cpu);
#endif
- sum += cpu_pda(cpu)->irq_spurious_count;
+ sum += per_cpu(pda.irq_spurious_count, cpu);
return sum;
}
--- linux-2.6.tip.orig/arch/x86/kernel/setup_percpu.c
+++ linux-2.6.tip/arch/x86/kernel/setup_percpu.c
@@ -243,17 +243,23 @@ void __cpuinit numa_set_node(int cpu, in
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
- if (cpu_pda(cpu) && node != NUMA_NO_NODE)
- cpu_pda(cpu)->nodenumber = node;
-
- if (cpu_to_node_map)
+ /* early setting, no percpu area yet */
+ if (cpu_to_node_map) {
cpu_to_node_map[cpu] = node;
+ return;
+ }
- else if (per_cpu_offset(cpu))
- per_cpu(x86_cpu_to_node_map, cpu) = node;
+#ifdef CONFIG_DEBUG_PER_CPU_MAPS
+ if (cpu >= nr_cpu_ids || !per_cpu_offset(cpu)) {
+ printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu);
+ dump_stack();
+ return;
+ }
+#endif
+ per_cpu(x86_cpu_to_node_map, cpu) = node;
- else
- Dprintk(KERN_INFO "Setting node for non-present cpu %d\n", cpu);
+ if (node != NUMA_NO_NODE)
+ per_cpu(pda.nodenumber, cpu) = node;
}
void __cpuinit numa_clear_node(int cpu)
--- linux-2.6.tip.orig/arch/x86/kernel/smpboot.c
+++ linux-2.6.tip/arch/x86/kernel/smpboot.c
@@ -814,7 +814,7 @@ do_rest:
/* Stack for startup_32 can be just as for start_secondary onwards */
irq_ctx_init(cpu);
#else
- cpu_pda(cpu)->pcurrent = c_idle.idle;
+ per_cpu(pda.pcurrent, cpu) = c_idle.idle;
clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
initial_pda = (unsigned long)get_cpu_pda(cpu);
#endif
--- linux-2.6.tip.orig/arch/x86/kernel/traps_64.c
+++ linux-2.6.tip/arch/x86/kernel/traps_64.c
@@ -265,7 +265,8 @@ void dump_trace(struct task_struct *tsk,
const struct stacktrace_ops *ops, void *data)
{
const unsigned cpu = get_cpu();
- unsigned long *irqstack_end = (unsigned long*)cpu_pda(cpu)->irqstackptr;
+ unsigned long *irqstack_end =
+ (unsigned long *)per_cpu(pda.irqstackptr, cpu);
unsigned used = 0;
struct thread_info *tinfo;
@@ -399,8 +400,10 @@ _show_stack(struct task_struct *tsk, str
unsigned long *stack;
int i;
const int cpu = smp_processor_id();
- unsigned long *irqstack_end = (unsigned long *) (cpu_pda(cpu)->irqstackptr);
- unsigned long *irqstack = (unsigned long *) (cpu_pda(cpu)->irqstackptr - IRQSTACKSIZE);
+ unsigned long *irqstack_end =
+ (unsigned long *)per_cpu(pda.irqstackptr, cpu);
+ unsigned long *irqstack =
+ (unsigned long *)(per_cpu(pda.irqstackptr, cpu) - IRQSTACKSIZE);
// debugging aid: "show_stack(NULL, NULL);" prints the
// back trace for this cpu.
@@ -464,7 +467,7 @@ void show_registers(struct pt_regs *regs
int i;
unsigned long sp;
const int cpu = smp_processor_id();
- struct task_struct *cur = cpu_pda(cpu)->pcurrent;
+ struct task_struct *cur = __get_cpu_var(pda.pcurrent);
u8 *ip;
unsigned int code_prologue = code_bytes * 43 / 64;
unsigned int code_len = code_bytes;
--
^ permalink raw reply [flat|nested] 12+ messages in thread* [RFC 5/5] x86_64: Replace xxx_pda() operations with x86_xxx_percpu().
2008-07-01 12:16 [RFC 0/5] percpu: Optimize percpu accesses Mike Travis
` (3 preceding siblings ...)
2008-07-01 12:16 ` [RFC 4/5] x86_64: Replace cpu_pda ops with percpu ops Mike Travis
@ 2008-07-01 12:16 ` Mike Travis
2008-07-01 14:54 ` [RFC 0/5] percpu: Optimize percpu accesses Ingo Molnar
5 siblings, 0 replies; 12+ messages in thread
From: Mike Travis @ 2008-07-01 12:16 UTC (permalink / raw)
To: Ingo Molnar, Jeremy Fitzhardinge, Eric W. Biederman
Cc: Christoph Lameter, H. Peter Anvin, Andrew Morton, linux-kernel
[-- Attachment #1: zero_based_replace_pda_operations --]
[-- Type: text/plain, Size: 15155 bytes --]
* It is now possible to use percpu operations for pda access
since the pda is in the percpu area. Drop the pda operations.
Thus:
read_pda --> x86_read_percpu
write_pda --> x86_write_percpu
add_pda (+1) --> x86_inc_percpu
or_pda --> x86_or_percpu
* Remove unused field (in_bootmem) from the pda.
* One pda op (test_and_clear_bit_pda) cannot be easily removed
but since the pda is the first element in the per cpu area,
then it can be left in place as is.
Based on linux-2.6.tip/master
Signed-off-by: Christoph Lameter <cl@linux-foundation.org>
Signed-off-by: Mike Travis <travis@sgi.com>
---
arch/x86/kernel/apic_64.c | 4 -
arch/x86/kernel/cpu/mcheck/mce_amd_64.c | 2
arch/x86/kernel/cpu/mcheck/mce_intel_64.c | 2
arch/x86/kernel/nmi.c | 3 -
arch/x86/kernel/process_64.c | 12 ++--
arch/x86/kernel/smp.c | 4 -
arch/x86/kernel/time_64.c | 2
arch/x86/kernel/tlb_64.c | 12 ++--
arch/x86/kernel/traps_64.c | 2
arch/x86/kernel/x8664_ksyms_64.c | 2
arch/x86/xen/smp.c | 2
include/asm-x86/current.h | 3 -
include/asm-x86/hardirq_64.h | 6 +-
include/asm-x86/mmu_context_64.h | 12 ++--
include/asm-x86/pda.h | 77 ++----------------------------
include/asm-x86/percpu.h | 2
include/asm-x86/smp.h | 2
include/asm-x86/stackprotector.h | 2
include/asm-x86/thread_info.h | 3 -
include/asm-x86/topology.h | 2
20 files changed, 47 insertions(+), 109 deletions(-)
--- linux-2.6.tip.orig/arch/x86/kernel/apic_64.c
+++ linux-2.6.tip/arch/x86/kernel/apic_64.c
@@ -457,7 +457,7 @@ static void local_apic_timer_interrupt(v
/*
* the NMI deadlock-detector uses this.
*/
- add_pda(apic_timer_irqs, 1);
+ x86_inc_percpu(pda.apic_timer_irqs);
evt->event_handler(evt);
}
@@ -965,7 +965,7 @@ asmlinkage void smp_spurious_interrupt(v
if (v & (1 << (SPURIOUS_APIC_VECTOR & 0x1f)))
ack_APIC_irq();
- add_pda(irq_spurious_count, 1);
+ x86_inc_percpu(pda.irq_spurious_count);
irq_exit();
}
--- linux-2.6.tip.orig/arch/x86/kernel/cpu/mcheck/mce_amd_64.c
+++ linux-2.6.tip/arch/x86/kernel/cpu/mcheck/mce_amd_64.c
@@ -237,7 +237,7 @@ asmlinkage void mce_threshold_interrupt(
}
}
out:
- add_pda(irq_threshold_count, 1);
+ x86_inc_percpu(pda.irq_threshold_count);
irq_exit();
}
--- linux-2.6.tip.orig/arch/x86/kernel/cpu/mcheck/mce_intel_64.c
+++ linux-2.6.tip/arch/x86/kernel/cpu/mcheck/mce_intel_64.c
@@ -26,7 +26,7 @@ asmlinkage void smp_thermal_interrupt(vo
if (therm_throt_process(msr_val & 1))
mce_log_therm_throt_event(smp_processor_id(), msr_val);
- add_pda(irq_thermal_count, 1);
+ x86_inc_percpu(pda.irq_thermal_count);
irq_exit();
}
--- linux-2.6.tip.orig/arch/x86/kernel/nmi.c
+++ linux-2.6.tip/arch/x86/kernel/nmi.c
@@ -82,7 +82,8 @@ static inline int mce_in_progress(void)
static inline unsigned int get_timer_irqs(int cpu)
{
#ifdef CONFIG_X86_64
- return read_pda(apic_timer_irqs) + read_pda(irq0_irqs);
+ return x86_read_percpu(pda.apic_timer_irqs) +
+ x86_read_percpu(pda.irq0_irqs);
#else
return per_cpu(irq_stat, cpu).apic_timer_irqs +
per_cpu(irq_stat, cpu).irq0_irqs;
--- linux-2.6.tip.orig/arch/x86/kernel/process_64.c
+++ linux-2.6.tip/arch/x86/kernel/process_64.c
@@ -66,7 +66,7 @@ void idle_notifier_register(struct notif
void enter_idle(void)
{
- write_pda(isidle, 1);
+ x86_write_percpu(pda.isidle, 1);
atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
}
@@ -410,7 +410,7 @@ start_thread(struct pt_regs *regs, unsig
load_gs_index(0);
regs->ip = new_ip;
regs->sp = new_sp;
- write_pda(oldrsp, new_sp);
+ x86_write_percpu(pda.oldrsp, new_sp);
regs->cs = __USER_CS;
regs->ss = __USER_DS;
regs->flags = 0x200;
@@ -646,11 +646,11 @@ __switch_to(struct task_struct *prev_p,
/*
* Switch the PDA and FPU contexts.
*/
- prev->usersp = read_pda(oldrsp);
- write_pda(oldrsp, next->usersp);
- write_pda(pcurrent, next_p);
+ prev->usersp = x86_read_percpu(pda.oldrsp);
+ x86_write_percpu(pda.oldrsp, next->usersp);
+ x86_write_percpu(pda.pcurrent, next_p);
- write_pda(kernelstack,
+ x86_write_percpu(pda.kernelstack,
(unsigned long)task_stack_page(next_p) + THREAD_SIZE - PDA_STACKOFFSET);
#ifdef CONFIG_CC_STACKPROTECTOR
/*
--- linux-2.6.tip.orig/arch/x86/kernel/smp.c
+++ linux-2.6.tip/arch/x86/kernel/smp.c
@@ -295,7 +295,7 @@ void smp_reschedule_interrupt(struct pt_
#ifdef CONFIG_X86_32
__get_cpu_var(irq_stat).irq_resched_count++;
#else
- add_pda(irq_resched_count, 1);
+ x86_inc_percpu(pda.irq_resched_count);
#endif
}
@@ -320,7 +320,7 @@ void smp_call_function_interrupt(struct
#ifdef CONFIG_X86_32
__get_cpu_var(irq_stat).irq_call_count++;
#else
- add_pda(irq_call_count, 1);
+ x86_inc_percpu(pda.irq_call_count);
#endif
irq_exit();
--- linux-2.6.tip.orig/arch/x86/kernel/time_64.c
+++ linux-2.6.tip/arch/x86/kernel/time_64.c
@@ -46,7 +46,7 @@ EXPORT_SYMBOL(profile_pc);
static irqreturn_t timer_event_interrupt(int irq, void *dev_id)
{
- add_pda(irq0_irqs, 1);
+ x86_inc_percpu(pda.irq0_irqs);
global_clock_event->event_handler(global_clock_event);
--- linux-2.6.tip.orig/arch/x86/kernel/tlb_64.c
+++ linux-2.6.tip/arch/x86/kernel/tlb_64.c
@@ -62,9 +62,9 @@ static DEFINE_PER_CPU(union smp_flush_st
*/
void leave_mm(int cpu)
{
- if (read_pda(mmu_state) == TLBSTATE_OK)
+ if (x86_read_percpu(pda.mmu_state) == TLBSTATE_OK)
BUG();
- cpu_clear(cpu, read_pda(active_mm)->cpu_vm_mask);
+ cpu_clear(cpu, x86_read_percpu(pda.active_mm)->cpu_vm_mask);
load_cr3(swapper_pg_dir);
}
EXPORT_SYMBOL_GPL(leave_mm);
@@ -142,8 +142,8 @@ asmlinkage void smp_invalidate_interrupt
* BUG();
*/
- if (f->flush_mm == read_pda(active_mm)) {
- if (read_pda(mmu_state) == TLBSTATE_OK) {
+ if (f->flush_mm == x86_read_percpu(pda.active_mm)) {
+ if (x86_read_percpu(pda.mmu_state) == TLBSTATE_OK) {
if (f->flush_va == TLB_FLUSH_ALL)
local_flush_tlb();
else
@@ -154,7 +154,7 @@ asmlinkage void smp_invalidate_interrupt
out:
ack_APIC_irq();
cpu_clear(cpu, f->flush_cpumask);
- add_pda(irq_tlb_count, 1);
+ x86_inc_percpu(pda.irq_tlb_count);
}
void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
@@ -269,7 +269,7 @@ static void do_flush_tlb_all(void *info)
unsigned long cpu = smp_processor_id();
__flush_tlb_all();
- if (read_pda(mmu_state) == TLBSTATE_LAZY)
+ if (x86_read_percpu(pda.mmu_state) == TLBSTATE_LAZY)
leave_mm(cpu);
}
--- linux-2.6.tip.orig/arch/x86/kernel/traps_64.c
+++ linux-2.6.tip/arch/x86/kernel/traps_64.c
@@ -878,7 +878,7 @@ asmlinkage notrace __kprobes void
do_nmi(struct pt_regs *regs, long error_code)
{
nmi_enter();
- add_pda(__nmi_count, 1);
+ x86_inc_percpu(pda.__nmi_count);
if (!ignore_nmis)
default_do_nmi(regs);
nmi_exit();
--- linux-2.6.tip.orig/arch/x86/kernel/x8664_ksyms_64.c
+++ linux-2.6.tip/arch/x86/kernel/x8664_ksyms_64.c
@@ -58,5 +58,3 @@ EXPORT_SYMBOL(__memcpy);
EXPORT_SYMBOL(empty_zero_page);
EXPORT_SYMBOL(init_level4_pgt);
EXPORT_SYMBOL(load_gs_index);
-
-EXPORT_SYMBOL(_proxy_pda);
--- linux-2.6.tip.orig/arch/x86/xen/smp.c
+++ linux-2.6.tip/arch/x86/xen/smp.c
@@ -68,7 +68,7 @@ static irqreturn_t xen_reschedule_interr
#ifdef CONFIG_X86_32
__get_cpu_var(irq_stat).irq_resched_count++;
#else
- add_pda(irq_resched_count, 1);
+ x86_inc_percpu(pda.irq_resched_count);
#endif
return IRQ_HANDLED;
--- linux-2.6.tip.orig/include/asm-x86/current.h
+++ linux-2.6.tip/include/asm-x86/current.h
@@ -17,12 +17,13 @@ static __always_inline struct task_struc
#ifndef __ASSEMBLY__
#include <asm/pda.h>
+#include <asm/percpu.h>
struct task_struct;
static __always_inline struct task_struct *get_current(void)
{
- return read_pda(pcurrent);
+ return x86_read_percpu(pda.pcurrent);
}
#else /* __ASSEMBLY__ */
--- linux-2.6.tip.orig/include/asm-x86/hardirq_64.h
+++ linux-2.6.tip/include/asm-x86/hardirq_64.h
@@ -11,12 +11,12 @@
#define __ARCH_IRQ_STAT 1
-#define local_softirq_pending() read_pda(__softirq_pending)
+#define local_softirq_pending() x86_read_percpu(pda.__softirq_pending)
#define __ARCH_SET_SOFTIRQ_PENDING 1
-#define set_softirq_pending(x) write_pda(__softirq_pending, (x))
-#define or_softirq_pending(x) or_pda(__softirq_pending, (x))
+#define set_softirq_pending(x) x86_write_percpu(pda.__softirq_pending, (x))
+#define or_softirq_pending(x) x86_or_percpu(pda.__softirq_pending, (x))
extern void ack_bad_irq(unsigned int irq);
--- linux-2.6.tip.orig/include/asm-x86/mmu_context_64.h
+++ linux-2.6.tip/include/asm-x86/mmu_context_64.h
@@ -20,8 +20,8 @@ void destroy_context(struct mm_struct *m
static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
{
#ifdef CONFIG_SMP
- if (read_pda(mmu_state) == TLBSTATE_OK)
- write_pda(mmu_state, TLBSTATE_LAZY);
+ if (x86_read_percpu(pda.mmu_state) == TLBSTATE_OK)
+ x86_write_percpu(pda.mmu_state, TLBSTATE_LAZY);
#endif
}
@@ -33,8 +33,8 @@ static inline void switch_mm(struct mm_s
/* stop flush ipis for the previous mm */
cpu_clear(cpu, prev->cpu_vm_mask);
#ifdef CONFIG_SMP
- write_pda(mmu_state, TLBSTATE_OK);
- write_pda(active_mm, next);
+ x86_write_percpu(pda.mmu_state, TLBSTATE_OK);
+ x86_write_percpu(pda.active_mm, next);
#endif
cpu_set(cpu, next->cpu_vm_mask);
load_cr3(next->pgd);
@@ -44,8 +44,8 @@ static inline void switch_mm(struct mm_s
}
#ifdef CONFIG_SMP
else {
- write_pda(mmu_state, TLBSTATE_OK);
- if (read_pda(active_mm) != next)
+ x86_write_percpu(pda.mmu_state, TLBSTATE_OK);
+ if (x86_read_percpu(pda.active_mm) != next)
BUG();
if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
/* We were in lazy tlb mode and leave_mm disabled
--- linux-2.6.tip.orig/include/asm-x86/pda.h
+++ linux-2.6.tip/include/asm-x86/pda.h
@@ -21,7 +21,7 @@ struct x8664_pda {
offset 40!!! */
char *irqstackptr;
short nodenumber; /* number of current node (32k max) */
- short in_bootmem; /* pda lives in bootmem */
+ short unused1; /* unused */
unsigned int __softirq_pending;
unsigned int __nmi_count; /* number of NMI on this CPUs */
short mmu_state;
@@ -42,12 +42,6 @@ extern void pda_init(int);
#define cpu_pda(cpu) (&per_cpu(pda, cpu))
/*
- * There is no fast way to get the base address of the PDA, all the accesses
- * have to mention %fs/%gs. So it needs to be done this Torvaldian way.
- */
-extern void __bad_pda_field(void) __attribute__((noreturn));
-
-/*
* proxy_pda doesn't actually exist, but tell gcc it is accessed for
* all PDA accesses so it gets read/write dependencies right.
*/
@@ -55,69 +49,11 @@ extern struct x8664_pda _proxy_pda;
#define pda_offset(field) offsetof(struct x8664_pda, field)
-#define pda_to_op(op, field, val) \
-do { \
- typedef typeof(_proxy_pda.field) T__; \
- if (0) { T__ tmp__; tmp__ = (val); } /* type checking */ \
- switch (sizeof(_proxy_pda.field)) { \
- case 2: \
- asm(op "w %1,%%gs:%c2" : \
- "+m" (_proxy_pda.field) : \
- "ri" ((T__)val), \
- "i"(pda_offset(field))); \
- break; \
- case 4: \
- asm(op "l %1,%%gs:%c2" : \
- "+m" (_proxy_pda.field) : \
- "ri" ((T__)val), \
- "i" (pda_offset(field))); \
- break; \
- case 8: \
- asm(op "q %1,%%gs:%c2": \
- "+m" (_proxy_pda.field) : \
- "ri" ((T__)val), \
- "i"(pda_offset(field))); \
- break; \
- default: \
- __bad_pda_field(); \
- } \
-} while (0)
-
-#define pda_from_op(op, field) \
-({ \
- typeof(_proxy_pda.field) ret__; \
- switch (sizeof(_proxy_pda.field)) { \
- case 2: \
- asm(op "w %%gs:%c1,%0" : \
- "=r" (ret__) : \
- "i" (pda_offset(field)), \
- "m" (_proxy_pda.field)); \
- break; \
- case 4: \
- asm(op "l %%gs:%c1,%0": \
- "=r" (ret__): \
- "i" (pda_offset(field)), \
- "m" (_proxy_pda.field)); \
- break; \
- case 8: \
- asm(op "q %%gs:%c1,%0": \
- "=r" (ret__) : \
- "i" (pda_offset(field)), \
- "m" (_proxy_pda.field)); \
- break; \
- default: \
- __bad_pda_field(); \
- } \
- ret__; \
-})
-
-#define read_pda(field) pda_from_op("mov", field)
-#define write_pda(field, val) pda_to_op("mov", field, val)
-#define add_pda(field, val) pda_to_op("add", field, val)
-#define sub_pda(field, val) pda_to_op("sub", field, val)
-#define or_pda(field, val) pda_to_op("or", field, val)
-
-/* This is not atomic against other CPUs -- CPU preemption needs to be off */
+/*
+ * This is not atomic against other CPUs -- CPU preemption needs to be off
+ * NOTE: This relies on the fact that the cpu_pda is the *first* field in
+ * the per cpu area. Move it and you'll need to change this.
+ */
#define test_and_clear_bit_pda(bit, field) \
({ \
int old__; \
@@ -127,6 +63,7 @@ do { \
old__; \
})
+
#endif
#define PDA_STACKOFFSET (5*8)
--- linux-2.6.tip.orig/include/asm-x86/percpu.h
+++ linux-2.6.tip/include/asm-x86/percpu.h
@@ -7,7 +7,7 @@
/* Same as asm-generic/percpu.h, except we use %gs as a segment offset. */
#ifdef CONFIG_SMP
-#define __my_cpu_offset read_pda(data_offset)
+#define __my_cpu_offset (x86_read_percpu(pda.data_offset))
#define __percpu_seg "%%gs:"
#else
#define __percpu_seg ""
--- linux-2.6.tip.orig/include/asm-x86/smp.h
+++ linux-2.6.tip/include/asm-x86/smp.h
@@ -134,7 +134,7 @@ DECLARE_PER_CPU(int, cpu_number);
extern int safe_smp_processor_id(void);
#elif defined(CONFIG_X86_64_SMP)
-#define raw_smp_processor_id() read_pda(cpunumber)
+#define raw_smp_processor_id() x86_read_percpu(pda.cpunumber)
#define stack_smp_processor_id() \
({ \
--- linux-2.6.tip.orig/include/asm-x86/stackprotector.h
+++ linux-2.6.tip/include/asm-x86/stackprotector.h
@@ -32,7 +32,7 @@ static __always_inline void boot_init_st
canary += tsc + (tsc << 32UL);
current->stack_canary = canary;
- write_pda(stack_canary, canary);
+ x86_write_percpu(pda.stack_canary, canary);
}
#endif
--- linux-2.6.tip.orig/include/asm-x86/thread_info.h
+++ linux-2.6.tip/include/asm-x86/thread_info.h
@@ -200,7 +200,8 @@ static inline struct thread_info *curren
static inline struct thread_info *current_thread_info(void)
{
struct thread_info *ti;
- ti = (void *)(read_pda(kernelstack) + PDA_STACKOFFSET - THREAD_SIZE);
+ ti = (void *)(x86_read_percpu(pda.kernelstack) +
+ PDA_STACKOFFSET - THREAD_SIZE);
return ti;
}
--- linux-2.6.tip.orig/include/asm-x86/topology.h
+++ linux-2.6.tip/include/asm-x86/topology.h
@@ -72,7 +72,7 @@ extern cpumask_t *node_to_cpumask_map;
DECLARE_EARLY_PER_CPU(int, x86_cpu_to_node_map);
/* Returns the number of the current Node. */
-#define numa_node_id() read_pda(nodenumber)
+#define numa_node_id() x86_read_percpu(pda.nodenumber)
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
extern int cpu_to_node(int cpu);
--
^ permalink raw reply [flat|nested] 12+ messages in thread* Re: [RFC 0/5] percpu: Optimize percpu accesses
2008-07-01 12:16 [RFC 0/5] percpu: Optimize percpu accesses Mike Travis
` (4 preceding siblings ...)
2008-07-01 12:16 ` [RFC 5/5] x86_64: Replace xxx_pda() operations with x86_xxx_percpu() Mike Travis
@ 2008-07-01 14:54 ` Ingo Molnar
2008-07-01 15:03 ` Ingo Molnar
5 siblings, 1 reply; 12+ messages in thread
From: Ingo Molnar @ 2008-07-01 14:54 UTC (permalink / raw)
To: Mike Travis
Cc: Jeremy Fitzhardinge, Eric W. Biederman, Christoph Lameter,
H. Peter Anvin, Andrew Morton, linux-kernel
* Mike Travis <travis@sgi.com> wrote:
> This patchset provides the following:
>
> * Cleanup: Fix early references to cpumask_of_cpu(0)
> * Generic: Percpu infrastructure to rebase the per cpu area to zero
> * x86_64: Fold pda into per cpu area
> * x86_64: Rebase per cpu variables to zero
thanks Mike - i've started testing it in -tip. One small merge fallout
fix is below. (it's for the new generic-ipi topic that was not in
tip/master yet when you merged percpu-zerobased to it)
Ingo
---------------->
commit d1d4ed23aab090298b9d4da0fe445b96afcdd309
Author: Ingo Molnar <mingo@elte.hu>
Date: Tue Jul 1 16:44:45 2008 +0200
x86: percpu-zerobased fallout fixlet
add_pda(x,1) => x86_inc_percpu(pda.x)
Signed-off-by: Ingo Molnar <mingo@elte.hu>
diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c
index 361b7a4..1ddf629 100644
--- a/arch/x86/kernel/smp.c
+++ b/arch/x86/kernel/smp.c
@@ -206,7 +206,7 @@ void smp_call_function_single_interrupt(struct pt_regs *regs)
#ifdef CONFIG_X86_32
__get_cpu_var(irq_stat).irq_call_count++;
#else
- add_pda(irq_call_count, 1);
+ x86_inc_percpu(pda.irq_call_count);
#endif
irq_exit();
}
^ permalink raw reply related [flat|nested] 12+ messages in thread* Re: [RFC 0/5] percpu: Optimize percpu accesses
2008-07-01 14:54 ` [RFC 0/5] percpu: Optimize percpu accesses Ingo Molnar
@ 2008-07-01 15:03 ` Ingo Molnar
2008-07-01 15:07 ` Mike Travis
2008-07-01 18:47 ` Mike Travis
0 siblings, 2 replies; 12+ messages in thread
From: Ingo Molnar @ 2008-07-01 15:03 UTC (permalink / raw)
To: Mike Travis
Cc: Jeremy Fitzhardinge, Eric W. Biederman, Christoph Lameter,
H. Peter Anvin, Andrew Morton, linux-kernel
* Ingo Molnar <mingo@elte.hu> wrote:
> > This patchset provides the following:
> >
> > * Cleanup: Fix early references to cpumask_of_cpu(0)
> > * Generic: Percpu infrastructure to rebase the per cpu area to zero
> > * x86_64: Fold pda into per cpu area
> > * x86_64: Rebase per cpu variables to zero
>
> thanks Mike - i've started testing it in -tip. One small merge fallout
> fix is below. (it's for the new generic-ipi topic that was not in
> tip/master yet when you merged percpu-zerobased to it)
ok, -tip testing found an early boot crash caused by your patchset, on
64-bit x86:
[ 0.396000] calling net_ns_init+0x0/0x143
[ 0.400000] net_namespace: 944 bytes
[ 0.403578] BUG: unable to handle kernel NULL pointer dereference at 0000000000000000
[ 0.404000] IP: [<ffffffff80e05b0e>] net_ns_init+0x9f/0x143
[ 0.404000] PGD 0
[ 0.404000] Oops: 0000 [1] SMP
[ 0.404000] CPU 0
[...]
full bootlog and config can be found at:
http://redhat.com/~mingo/misc/crashlog-Tue_Jul__1_16_48_45_CEST_2008.bad
http://redhat.com/~mingo/misc/config-Tue_Jul__1_16_48_45_CEST_2008.bad
(another 64-bit testbox crashed as well, so this should be readily
reproducible.)
i've pushed this tree out to tip/tmp.core/percpu-zerobased.Jul__1_16_48
topic branch, that is the 2.6.26-rc8-tip-00250-g90874b0 kernel you can
see in the crashlog.
Ingo
^ permalink raw reply [flat|nested] 12+ messages in thread
* Re: [RFC 0/5] percpu: Optimize percpu accesses
2008-07-01 15:03 ` Ingo Molnar
@ 2008-07-01 15:07 ` Mike Travis
2008-07-02 14:04 ` Mike Travis
2008-07-01 18:47 ` Mike Travis
1 sibling, 1 reply; 12+ messages in thread
From: Mike Travis @ 2008-07-01 15:07 UTC (permalink / raw)
To: Ingo Molnar
Cc: Jeremy Fitzhardinge, Eric W. Biederman, Christoph Lameter,
H. Peter Anvin, Andrew Morton, linux-kernel
Ingo Molnar wrote:
> * Ingo Molnar <mingo@elte.hu> wrote:
>
>>> This patchset provides the following:
>>>
>>> * Cleanup: Fix early references to cpumask_of_cpu(0)
>>> * Generic: Percpu infrastructure to rebase the per cpu area to zero
>>> * x86_64: Fold pda into per cpu area
>>> * x86_64: Rebase per cpu variables to zero
>> thanks Mike - i've started testing it in -tip. One small merge fallout
>> fix is below. (it's for the new generic-ipi topic that was not in
>> tip/master yet when you merged percpu-zerobased to it)
>
> ok, -tip testing found an early boot crash caused by your patchset, on
> 64-bit x86:
>
> [ 0.396000] calling net_ns_init+0x0/0x143
> [ 0.400000] net_namespace: 944 bytes
> [ 0.403578] BUG: unable to handle kernel NULL pointer dereference at 0000000000000000
> [ 0.404000] IP: [<ffffffff80e05b0e>] net_ns_init+0x9f/0x143
> [ 0.404000] PGD 0
> [ 0.404000] Oops: 0000 [1] SMP
> [ 0.404000] CPU 0
> [...]
>
> full bootlog and config can be found at:
>
> http://redhat.com/~mingo/misc/crashlog-Tue_Jul__1_16_48_45_CEST_2008.bad
> http://redhat.com/~mingo/misc/config-Tue_Jul__1_16_48_45_CEST_2008.bad
>
> (another 64-bit testbox crashed as well, so this should be readily
> reproducible.)
>
> i've pushed this tree out to tip/tmp.core/percpu-zerobased.Jul__1_16_48
> topic branch, that is the 2.6.26-rc8-tip-00250-g90874b0 kernel you can
> see in the crashlog.
>
> Ingo
Wow, that was quick!
At least this one gets to someplace debuggable. Some others that don't
even get to the first "kernel alive" messages are really annoying. ;-)
Thanks,
Mike
^ permalink raw reply [flat|nested] 12+ messages in thread
* Re: [RFC 0/5] percpu: Optimize percpu accesses
2008-07-01 15:07 ` Mike Travis
@ 2008-07-02 14:04 ` Mike Travis
2008-07-18 6:52 ` Ingo Molnar
0 siblings, 1 reply; 12+ messages in thread
From: Mike Travis @ 2008-07-02 14:04 UTC (permalink / raw)
To: Ingo Molnar
Cc: Jeremy Fitzhardinge, Eric W. Biederman, Christoph Lameter,
H. Peter Anvin, Andrew Morton, linux-kernel
Mike Travis wrote:
> Ingo Molnar wrote:
>> * Ingo Molnar <mingo@elte.hu> wrote:
>>
>>>> This patchset provides the following:
>>>>
>>>> * Cleanup: Fix early references to cpumask_of_cpu(0)
>>>> * Generic: Percpu infrastructure to rebase the per cpu area to zero
>>>> * x86_64: Fold pda into per cpu area
>>>> * x86_64: Rebase per cpu variables to zero
>>> thanks Mike - i've started testing it in -tip. One small merge fallout
>>> fix is below. (it's for the new generic-ipi topic that was not in
>>> tip/master yet when you merged percpu-zerobased to it)
>> ok, -tip testing found an early boot crash caused by your patchset, on
>> 64-bit x86:
Hi Ingo,
Which gcc version are you using?
Thanks,
Mike
>>
>> [ 0.396000] calling net_ns_init+0x0/0x143
>> [ 0.400000] net_namespace: 944 bytes
>> [ 0.403578] BUG: unable to handle kernel NULL pointer dereference at 0000000000000000
>> [ 0.404000] IP: [<ffffffff80e05b0e>] net_ns_init+0x9f/0x143
>> [ 0.404000] PGD 0
>> [ 0.404000] Oops: 0000 [1] SMP
>> [ 0.404000] CPU 0
>> [...]
>>
>> full bootlog and config can be found at:
>>
>> http://redhat.com/~mingo/misc/crashlog-Tue_Jul__1_16_48_45_CEST_2008.bad
>> http://redhat.com/~mingo/misc/config-Tue_Jul__1_16_48_45_CEST_2008.bad
>>
>> (another 64-bit testbox crashed as well, so this should be readily
>> reproducible.)
>>
>> i've pushed this tree out to tip/tmp.core/percpu-zerobased.Jul__1_16_48
>> topic branch, that is the 2.6.26-rc8-tip-00250-g90874b0 kernel you can
>> see in the crashlog.
>>
>> Ingo
>
> Wow, that was quick!
>
> At least this one gets to someplace debuggable. Some others that don't
> even get to the first "kernel alive" messages are really annoying. ;-)
>
> Thanks,
> Mike
^ permalink raw reply [flat|nested] 12+ messages in thread
* Re: [RFC 0/5] percpu: Optimize percpu accesses
2008-07-02 14:04 ` Mike Travis
@ 2008-07-18 6:52 ` Ingo Molnar
0 siblings, 0 replies; 12+ messages in thread
From: Ingo Molnar @ 2008-07-18 6:52 UTC (permalink / raw)
To: Mike Travis
Cc: Jeremy Fitzhardinge, Eric W. Biederman, Christoph Lameter,
H. Peter Anvin, Andrew Morton, linux-kernel
* Mike Travis <travis@sgi.com> wrote:
> Mike Travis wrote:
> > Ingo Molnar wrote:
> >> * Ingo Molnar <mingo@elte.hu> wrote:
> >>
> >>>> This patchset provides the following:
> >>>>
> >>>> * Cleanup: Fix early references to cpumask_of_cpu(0)
> >>>> * Generic: Percpu infrastructure to rebase the per cpu area to zero
> >>>> * x86_64: Fold pda into per cpu area
> >>>> * x86_64: Rebase per cpu variables to zero
> >>> thanks Mike - i've started testing it in -tip. One small merge fallout
> >>> fix is below. (it's for the new generic-ipi topic that was not in
> >>> tip/master yet when you merged percpu-zerobased to it)
> >> ok, -tip testing found an early boot crash caused by your patchset, on
> >> 64-bit x86:
>
> Hi Ingo,
>
> Which gcc version are you using?
vanilla gcc-4.2.3 (unpatched).
Ingo
^ permalink raw reply [flat|nested] 12+ messages in thread
* Re: [RFC 0/5] percpu: Optimize percpu accesses
2008-07-01 15:03 ` Ingo Molnar
2008-07-01 15:07 ` Mike Travis
@ 2008-07-01 18:47 ` Mike Travis
1 sibling, 0 replies; 12+ messages in thread
From: Mike Travis @ 2008-07-01 18:47 UTC (permalink / raw)
To: Ingo Molnar
Cc: Jeremy Fitzhardinge, Eric W. Biederman, Christoph Lameter,
H. Peter Anvin, Andrew Morton, linux-kernel, Hugh Dickins,
Jack Steiner
Ingo Molnar wrote:
>
> full bootlog and config can be found at:
>
> http://redhat.com/~mingo/misc/crashlog-Tue_Jul__1_16_48_45_CEST_2008.bad
> http://redhat.com/~mingo/misc/config-Tue_Jul__1_16_48_45_CEST_2008.bad
>
> (another 64-bit testbox crashed as well, so this should be readily
> reproducible.)
>
> i've pushed this tree out to tip/tmp.core/percpu-zerobased.Jul__1_16_48
> topic branch, that is the 2.6.26-rc8-tip-00250-g90874b0 kernel you can
> see in the crashlog.
>
> Ingo
Ok, two problems here. First, it's getting a stack overflow (proved by
increasing THREAD_ORDER to 4 [probably 2 would work as well.])
Second, the following change caused the panic below. I'll dissect the
object code to see if I can spot the problem. It does sounds a bit like
the previous discussion in this thread:
"Re: v2.6.26-rc7: BUG: unable to handle kernel NULL pointer dereference"
About the stack overflow, the largest users are listed last (this is after
applying the "sched: Reduce stack size in isolated_cpu_setup()" patch which
I just sent.) My gcc (4.2.0) says this for "stackprotector".
#warning You have selected the CONFIG_CC_STACKPROTECTOR option,
but the gcc used does not support this.
... is there a known version where this _is_ supported? Or any other
advice on tracking this down?
[I did take your advice and broke out every change in the last patch of the
patchset into individual patches. Was fairly easy in this case, and pinpointed
the failing change very quickly.]
Thanks!
Mike
---
include/asm-x86/thread_info.h | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--- linux-2.6.tip.orig/include/asm-x86/thread_info.h 2008-07-01 10:41:33.000000000 -0700
+++ linux-2.6.tip/include/asm-x86/thread_info.h 2008-07-01 10:49:15.172370813 -0700
@@ -200,7 +200,8 @@ static inline struct thread_info *curren
static inline struct thread_info *current_thread_info(void)
{
struct thread_info *ti;
- ti = (void *)(read_pda(kernelstack) + PDA_STACKOFFSET - THREAD_SIZE);
+ ti = (void *)(x86_read_percpu(pda.kernelstack) +
+ PDA_STACKOFFSET - THREAD_SIZE);
return ti;
}
ccessful
[ 0.188000] CPU0: Intel(R) Xeon(R) CPU E5345 @ 2.33GHz stepping 07
[ 0.199685] Using local APIC timer interrupts.
[ 0.208000] APIC timer calibration result 20781829
[ 0.212000] Detected 20.781 MHz APIC timer.
[ 0.216000] BUG: unable to handle kernel NULL pointer dereference at 0000000000000000
[ 0.216000] IP: [<0000000000000000>]
[ 0.216000] PGD 0
[ 0.216000] Oops: 0010 [1] SMP
[ 0.216000] CPU 0
[ 0.216000] Pid: 1, comm: swapper Not tainted 2.6.26-rc8-tip-ingo-bad-0701-00208-g79a4d68-dirty #17
[ 0.216000] RIP: 0010:[<0000000000000000>] [<0000000000000000>]
[ 0.216000] RSP: 0000:ffff81022ed1fe18 EFLAGS: 00010282
[ 0.216000] RAX: 0000000000000001 RBX: ffff81022ed1fe84 RCX: ffffffff80ca6750
[ 0.216000] RDX: 0000000000000001 RSI: 0000000000000003 RDI: ffffffff80ca6750
[ 0.216000] RBP: ffff81022ed1fe50 R08: 0000000000000000 R09: ffff8100010ff710
[ 0.216000] R10: ffff8100010ed740 R11: 0000000000000000 R12: 00000000fffffffd
[ 0.216000] R13: ffffffff80ca7790 R14: 0000000000000001 R15: 0000000000000003
[ 0.216000] FS: 0000000000000000(0000) GS:ffff8100010eb000(0000) knlGS:0000000000000000
[ 0.216000] CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b
[ 0.216000] CR2: 0000000000000000 CR3: 0000000000201000 CR4: 00000000000006e0
[ 0.216000] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 0.216000] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 0.216000] Process swapper (pid: 1, threadinfo ffff81022ed10000, task ffff81022ec930e0)
[ 0.216000] Stack: ffffffff8024b800 0000000000000000 0000000000000001 0000000000000001
[ 0.216000] 00000000fffffff0 ffffffff80e263a0 0000000000092fd0 ffff81022ed1fe60
[ 0.216000] ffffffff8024b880 ffff81022ed1fea0 ffffffff808e8526 0000000000000008
[ 0.216000] Call Trace:
[ 0.216000] [<ffffffff8024b800>] ? notifier_call_chain+0x38/0x60
[ 0.216000] [<ffffffff8024b880>] __raw_notifier_call_chain+0xe/0x10
[ 0.216000] [<ffffffff808e8526>] cpu_up+0xa8/0x138
[ 0.216000] [<ffffffff80ded993>] kernel_init+0xcf/0x316
[ 0.216000] [<ffffffff8020d458>] child_rip+0xa/0x12
[ 0.216000] [<ffffffff8020c8f5>] ? restore_args+0x0/0x30
[ 0.216000] [<ffffffff80ded8c4>] ? kernel_init+0x0/0x316
[ 0.216000] [<ffffffff8020d44e>] ? child_rip+0x0/0x12
[ 0.216000]
[ 0.216000]
[ 0.216000] Code: Bad RIP value.
[ 0.216000] RIP [<0000000000000000>]
[ 0.216000] RSP <ffff81022ed1fe18>
[ 0.216000] CR2: 0000000000000000
Stack usages >= 1000 bytes:
2184 __build_sched_domains
2056 kthreadd
1576 tick_notify
1576 setup_IO_APIC_irq
1576 move_task_off_dead_cpu
1560 arch_setup_ht_irq
1560 __assign_irq_vector
1544 tick_handle_oneshot_broadcast
1352 zc0301_ioctl_v4l2
1336 i2o_cfg_compat_ioctl
1192 sn9c102_ioctl_v4l2
1152 e1000_check_options
1128 setup_IO_APIC
1096 _cpu_down
1080 sched_balance_self
1080 do_ida_request
1064 native_smp_call_function_mask
1048 setup_timer_IRQ0_pin
1048 setup_ioapic_dest
1048 set_ioapic_affinity_irq
1048 set_ht_irq_affinity
1048 sched_rt_period_timer
1048 native_machine_crash_shutdown
1032 tick_do_periodic_broadcast
1032 sched_setaffinity
1032 native_flush_tlb_others
1032 local_cpus_show
1032 local_cpulist_show
1032 irq_select_affinity
1032 irq_complete_move
1032 irq_affinity_write_proc
1032 ioapic_retrigger_irq
1032 flush_tlb_mm
1032 flush_tlb_current_task
1032 fixup_irqs
1032 do_cciss_request
1032 create_irq
1024 uv_vector_allocation_domain
1024 uv_send_IPI_allbutself
1024 smp_call_function_single
1024 smp_call_function
1024 physflat_send_IPI_allbutself
1024 pci_bus_show_cpuaffinity
1024 move_masked_irq
1024 flush_tlb_page
1024 flat_send_IPI_allbutself
1000 security_load_policy
^ permalink raw reply [flat|nested] 12+ messages in thread