From: David Howells Subject: [PATCH] Move DECLARE_/DEFINE_PER_CPU*() to before the #inclusion of Move DECLARE_/DEFINE_PER_CPU*() to before the #inclusion of in linux/percpu.h. This puts the DECLARE and DEFINE macros together. The declarations have to be before the #inclusion of because some asm/percpu.h's declare such variables - something that is probably done in the wrong place. To make this work, this patch also has to rearrange and split a number of header files to eliminate circular references. Signed-off-by: David Howells --- arch/alpha/include/asm/percpu.h | 6 - arch/x86/include/asm/alternative.h | 4 + arch/x86/include/asm/apic-defs.h | 14 +++ arch/x86/include/asm/apic.h | 5 - arch/x86/include/asm/current.h | 3 - arch/x86/include/asm/irq_regs.h | 2 arch/x86/include/asm/mm_types.h | 12 ++ arch/x86/include/asm/percpu-defs.h | 67 +++++++++++++ arch/x86/include/asm/percpu.h | 66 ------------ arch/x86/include/asm/prefetch.h | 49 +++++++++ arch/x86/include/asm/processor.h | 49 --------- arch/x86/include/asm/smp.h | 12 +- arch/x86/include/asm/stackprotector.h | 2 arch/x86/include/asm/thread_info-defs.h | 165 +++++++++++++++++++++++++++++++ arch/x86/include/asm/thread_info.h | 166 ++----------------------------- arch/x86/kernel/acpi/cstate.c | 1 drivers/net/loopback.c | 1 fs/nfs/iostat.h | 1 include/asm-generic/percpu-defs.h | 33 ++++++ include/asm-generic/percpu.h | 46 --------- include/linux/completion-defs.h | 30 ++++++ include/linux/completion.h | 18 --- include/linux/fs.h | 2 include/linux/genhd.h | 1 include/linux/gfp.h | 10 +- include/linux/mmzone.h | 6 + include/linux/percpu-alloc.h | 42 ++++++++ include/linux/percpu-defs.h | 79 +++++++++++++++ include/linux/percpu.h | 52 +--------- include/linux/preempt.h | 2 include/linux/prefetch.h | 3 - include/linux/rcupdate.h | 2 include/linux/seqlock-defs.h | 37 +++++++ include/linux/seqlock.h | 20 ++-- include/linux/spinlock.h | 1 include/linux/thread_info-defs.h | 54 ++++++++++ include/linux/thread_info.h | 44 -------- include/linux/wait-defs.h | 25 +++++ include/linux/wait.h | 21 ---- kernel/module.c | 1 kernel/srcu.c | 2 kernel/stop_machine.c | 1 kernel/workqueue.c | 1 lib/percpu_counter.c | 1 net/core/neighbour.c | 1 net/ipv4/af_inet.c | 1 46 files changed, 684 insertions(+), 477 deletions(-) create mode 100644 arch/x86/include/asm/apic-defs.h create mode 100644 arch/x86/include/asm/mm_types.h create mode 100644 arch/x86/include/asm/percpu-defs.h create mode 100644 arch/x86/include/asm/prefetch.h create mode 100644 arch/x86/include/asm/thread_info-defs.h create mode 100644 include/asm-generic/percpu-defs.h create mode 100644 include/linux/completion-defs.h create mode 100644 include/linux/percpu-alloc.h create mode 100644 include/linux/percpu-defs.h create mode 100644 include/linux/seqlock-defs.h create mode 100644 include/linux/thread_info-defs.h create mode 100644 include/linux/wait-defs.h diff --git a/arch/alpha/include/asm/percpu.h b/arch/alpha/include/asm/percpu.h index e9e0bb5..896c578 100644 --- a/arch/alpha/include/asm/percpu.h +++ b/arch/alpha/include/asm/percpu.h @@ -3,12 +3,6 @@ #include #include -/* - * Determine the real variable name from the name visible in the - * kernel sources. - */ -#define per_cpu_var(var) per_cpu__##var - #ifdef CONFIG_SMP /* diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h index f6aa18e..83a97dd 100644 --- a/arch/x86/include/asm/alternative.h +++ b/arch/x86/include/asm/alternative.h @@ -1,6 +1,8 @@ #ifndef _ASM_X86_ALTERNATIVE_H #define _ASM_X86_ALTERNATIVE_H +#ifndef __ASSEMBLER__ + #include #include #include @@ -180,4 +182,6 @@ extern void add_nops(void *insns, unsigned int len); extern void *text_poke(void *addr, const void *opcode, size_t len); extern void *text_poke_early(void *addr, const void *opcode, size_t len); +#endif /* !__ASSEMBLER__ */ + #endif /* _ASM_X86_ALTERNATIVE_H */ diff --git a/arch/x86/include/asm/apic-defs.h b/arch/x86/include/asm/apic-defs.h new file mode 100644 index 0000000..3f0261d --- /dev/null +++ b/arch/x86/include/asm/apic-defs.h @@ -0,0 +1,14 @@ +#ifndef _ASM_X86_APIC_DEFS_H +#define _ASM_X86_APIC_DEFS_H + +#include + +#ifdef CONFIG_X86_64 +DECLARE_PER_CPU(int, x2apic_extra_bits); +#endif + +#ifdef CONFIG_X86_LOCAL_APIC +DECLARE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid); +#endif + +#endif /* _ASM_X86_APIC_DEFS_H */ diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 42f2f83..e40c194 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -4,7 +4,9 @@ #include #include #include +#include +#include #include #include #include @@ -432,7 +434,6 @@ extern int default_acpi_madt_oem_check(char *, char *); extern void apic_send_IPI_self(int vector); extern struct apic apic_x2apic_uv_x; -DECLARE_PER_CPU(int, x2apic_extra_bits); extern int default_cpu_present_to_apicid(int mps_cpu); extern int default_check_phys_apicid_present(int boot_cpu_physical_apicid); @@ -463,8 +464,6 @@ static inline const struct cpumask *default_target_cpus(void) #endif } -DECLARE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid); - static inline unsigned int read_apic_id(void) { diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h index c68c361..11b7123 100644 --- a/arch/x86/include/asm/current.h +++ b/arch/x86/include/asm/current.h @@ -2,9 +2,10 @@ #define _ASM_X86_CURRENT_H #include -#include #ifndef __ASSEMBLY__ +#include + struct task_struct; DECLARE_PER_CPU(struct task_struct *, current_task); diff --git a/arch/x86/include/asm/irq_regs.h b/arch/x86/include/asm/irq_regs.h index 7784322..2c5fee5 100644 --- a/arch/x86/include/asm/irq_regs.h +++ b/arch/x86/include/asm/irq_regs.h @@ -7,7 +7,7 @@ #ifndef _ASM_X86_IRQ_REGS_H #define _ASM_X86_IRQ_REGS_H -#include +#include #define ARCH_HAS_OWN_IRQ_REGS diff --git a/arch/x86/include/asm/mm_types.h b/arch/x86/include/asm/mm_types.h new file mode 100644 index 0000000..09f9742 --- /dev/null +++ b/arch/x86/include/asm/mm_types.h @@ -0,0 +1,12 @@ +#ifndef _ASM_X86_MM_TYPES_H +#define _ASM_X86_MM_TYPES_H + +#ifndef __ASSEMBLY__ + +typedef struct { + unsigned long seg; +} mm_segment_t; + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_X86_MM_TYPES_H */ diff --git a/arch/x86/include/asm/percpu-defs.h b/arch/x86/include/asm/percpu-defs.h new file mode 100644 index 0000000..801217d --- /dev/null +++ b/arch/x86/include/asm/percpu-defs.h @@ -0,0 +1,67 @@ +#ifndef _ASM_X86_PERCPU_DEFS_H +#define _ASM_X86_PERCPU_DEFS_H + +#include + +#ifdef CONFIG_X86_64_SMP +#define init_per_cpu_var(var) init_per_cpu__##var +#else +#define init_per_cpu_var(var) per_cpu_var(var) +#endif + +/* + * Initialized pointers to per-cpu variables needed for the boot + * processor need to use these macros to get the proper address + * offset from __per_cpu_load on SMP. + * + * There also must be an entry in vmlinux_64.lds.S + */ +#define DECLARE_INIT_PER_CPU(var) \ + extern typeof(per_cpu_var(var)) init_per_cpu_var(var) + +#ifdef CONFIG_SMP + +/* + * Define the "EARLY_PER_CPU" macros. These are used for some per_cpu + * variables that are initialized and accessed before there are per_cpu + * areas allocated. + */ + +#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ + DEFINE_PER_CPU(_type, _name) = _initvalue; \ + __typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \ + { [0 ... NR_CPUS-1] = _initvalue }; \ + __typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map + +#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ + EXPORT_PER_CPU_SYMBOL(_name) + +#define DECLARE_EARLY_PER_CPU(_type, _name) \ + DECLARE_PER_CPU(_type, _name); \ + extern __typeof__(_type) *_name##_early_ptr; \ + extern __typeof__(_type) _name##_early_map[] + +#define early_per_cpu_ptr(_name) (_name##_early_ptr) +#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx]) +#define early_per_cpu(_name, _cpu) \ + *(early_per_cpu_ptr(_name) ? \ + &early_per_cpu_ptr(_name)[_cpu] : \ + &per_cpu(_name, _cpu)) + +#else /* !CONFIG_SMP */ +#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ + DEFINE_PER_CPU(_type, _name) = _initvalue + +#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ + EXPORT_PER_CPU_SYMBOL(_name) + +#define DECLARE_EARLY_PER_CPU(_type, _name) \ + DECLARE_PER_CPU(_type, _name) + +#define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu) +#define early_per_cpu_ptr(_name) NULL +/* no early_per_cpu_map() */ + +#endif /* !CONFIG_SMP */ + +#endif /* _ASM_X86_PERCPU_DEFS_H */ diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index aee103b..60b461f 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -9,8 +9,6 @@ #define __percpu_mov_op movl #endif -#ifdef __ASSEMBLY__ - /* * PER_CPU finds an address of a per-cpu variable. * @@ -40,7 +38,8 @@ #define INIT_PER_CPU_VAR(var) per_cpu__##var #endif -#else /* ...!ASSEMBLY */ + +#ifndef __ASSEMBLY__ #include @@ -51,22 +50,6 @@ #define __percpu_arg(x) "%" #x #endif -/* - * Initialized pointers to per-cpu variables needed for the boot - * processor need to use these macros to get the proper address - * offset from __per_cpu_load on SMP. - * - * There also must be an entry in vmlinux_64.lds.S - */ -#define DECLARE_INIT_PER_CPU(var) \ - extern typeof(per_cpu_var(var)) init_per_cpu_var(var) - -#ifdef CONFIG_X86_64_SMP -#define init_per_cpu_var(var) init_per_cpu__##var -#else -#define init_per_cpu_var(var) per_cpu_var(var) -#endif - /* For arch-specific code, we can use direct single-insn ops (they * don't give an lvalue though). */ extern void __bad_percpu_size(void); @@ -157,49 +140,4 @@ DECLARE_PER_CPU(unsigned long, this_cpu_off); #endif /* !__ASSEMBLY__ */ -#ifdef CONFIG_SMP - -/* - * Define the "EARLY_PER_CPU" macros. These are used for some per_cpu - * variables that are initialized and accessed before there are per_cpu - * areas allocated. - */ - -#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ - DEFINE_PER_CPU(_type, _name) = _initvalue; \ - __typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \ - { [0 ... NR_CPUS-1] = _initvalue }; \ - __typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map - -#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ - EXPORT_PER_CPU_SYMBOL(_name) - -#define DECLARE_EARLY_PER_CPU(_type, _name) \ - DECLARE_PER_CPU(_type, _name); \ - extern __typeof__(_type) *_name##_early_ptr; \ - extern __typeof__(_type) _name##_early_map[] - -#define early_per_cpu_ptr(_name) (_name##_early_ptr) -#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx]) -#define early_per_cpu(_name, _cpu) \ - *(early_per_cpu_ptr(_name) ? \ - &early_per_cpu_ptr(_name)[_cpu] : \ - &per_cpu(_name, _cpu)) - -#else /* !CONFIG_SMP */ -#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ - DEFINE_PER_CPU(_type, _name) = _initvalue - -#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ - EXPORT_PER_CPU_SYMBOL(_name) - -#define DECLARE_EARLY_PER_CPU(_type, _name) \ - DECLARE_PER_CPU(_type, _name) - -#define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu) -#define early_per_cpu_ptr(_name) NULL -/* no early_per_cpu_map() */ - -#endif /* !CONFIG_SMP */ - #endif /* _ASM_X86_PERCPU_H */ diff --git a/arch/x86/include/asm/prefetch.h b/arch/x86/include/asm/prefetch.h new file mode 100644 index 0000000..ae17dd6 --- /dev/null +++ b/arch/x86/include/asm/prefetch.h @@ -0,0 +1,49 @@ +#ifndef _ASM_X86_PREFETCH_H +#define _ASM_X86_PREFETCH_H + +#include + +#define ARCH_HAS_PREFETCHW +#define ARCH_HAS_SPINLOCK_PREFETCH + +#ifdef CONFIG_X86_32 +# define BASE_PREFETCH ASM_NOP4 +# define ARCH_HAS_PREFETCH +#else +# define BASE_PREFETCH "prefetcht0 (%1)" +#endif + +/* + * Prefetch instructions for Pentium III (+) and AMD Athlon (+) + * + * It's not worth to care about 3dnow prefetches for the K6 + * because they are microcoded there and very slow. + */ +static inline void prefetch(const void *x) +{ + alternative_input(BASE_PREFETCH, + "prefetchnta (%1)", + X86_FEATURE_XMM, + "r" (x)); +} + +/* + * 3dnow prefetch to get an exclusive cache line. + * Useful for spinlocks to avoid one state transition in the + * cache coherency protocol: + */ +static inline void prefetchw(const void *x) +{ + alternative_input(BASE_PREFETCH, + "prefetchw (%1)", + X86_FEATURE_3DNOW, + "r" (x)); +} + +static inline void spin_lock_prefetch(const void *x) +{ + prefetchw(x); +} + + +#endif /* _ASM_X86_PREFETCH_H */ diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index c2cceae..15ba82c 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -7,21 +7,21 @@ struct task_struct; struct mm_struct; +#include #include #include #include #include #include -#include #include #include #include #include -#include #include #include #include #include +#include #include #include @@ -615,10 +615,6 @@ static inline void clear_in_cr4(unsigned long mask) write_cr4(cr4); } -typedef struct { - unsigned long seg; -} mm_segment_t; - /* * create a kernel thread without removing it from tasklists @@ -818,47 +814,6 @@ extern int bootloader_type; extern char ignore_fpu_irq; #define HAVE_ARCH_PICK_MMAP_LAYOUT 1 -#define ARCH_HAS_PREFETCHW -#define ARCH_HAS_SPINLOCK_PREFETCH - -#ifdef CONFIG_X86_32 -# define BASE_PREFETCH ASM_NOP4 -# define ARCH_HAS_PREFETCH -#else -# define BASE_PREFETCH "prefetcht0 (%1)" -#endif - -/* - * Prefetch instructions for Pentium III (+) and AMD Athlon (+) - * - * It's not worth to care about 3dnow prefetches for the K6 - * because they are microcoded there and very slow. - */ -static inline void prefetch(const void *x) -{ - alternative_input(BASE_PREFETCH, - "prefetchnta (%1)", - X86_FEATURE_XMM, - "r" (x)); -} - -/* - * 3dnow prefetch to get an exclusive cache line. - * Useful for spinlocks to avoid one state transition in the - * cache coherency protocol: - */ -static inline void prefetchw(const void *x) -{ - alternative_input(BASE_PREFETCH, - "prefetchw (%1)", - X86_FEATURE_3DNOW, - "r" (x)); -} - -static inline void spin_lock_prefetch(const void *x) -{ - prefetchw(x); -} #ifdef CONFIG_X86_32 /* diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h index 19e0d88..9776d86 100644 --- a/arch/x86/include/asm/smp.h +++ b/arch/x86/include/asm/smp.h @@ -3,14 +3,14 @@ #ifndef __ASSEMBLY__ #include #include -#include +#include /* * We need the APIC definitions automatically as part of 'smp.h' */ #ifdef CONFIG_X86_LOCAL_APIC # include -# include +# include # ifdef CONFIG_X86_IO_APIC # include # endif @@ -26,6 +26,11 @@ DECLARE_PER_CPU(cpumask_var_t, cpu_core_map); DECLARE_PER_CPU(u16, cpu_llc_id); DECLARE_PER_CPU(int, cpu_number); +DECLARE_EARLY_PER_CPU(u16, x86_cpu_to_apicid); +DECLARE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid); + +#include + static inline struct cpumask *cpu_sibling_mask(int cpu) { return per_cpu(cpu_sibling_map, cpu); @@ -36,9 +41,6 @@ static inline struct cpumask *cpu_core_mask(int cpu) return per_cpu(cpu_core_map, cpu); } -DECLARE_EARLY_PER_CPU(u16, x86_cpu_to_apicid); -DECLARE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid); - /* Static state in head.S used to set up a CPU */ extern struct { void *sp; diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h index c2d742c..8aad2c0 100644 --- a/arch/x86/include/asm/stackprotector.h +++ b/arch/x86/include/asm/stackprotector.h @@ -37,7 +37,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/x86/include/asm/thread_info-defs.h b/arch/x86/include/asm/thread_info-defs.h new file mode 100644 index 0000000..57c662a --- /dev/null +++ b/arch/x86/include/asm/thread_info-defs.h @@ -0,0 +1,165 @@ +/* thread_info.h: low-level thread information + * + * Copyright (C) 2002 David Howells (dhowells@redhat.com) + * - Incorporating suggestions made by Linus Torvalds and Dave Miller + */ + +#ifndef _ASM_X86_THREAD_INFO_DEFS_H +#define _ASM_X86_THREAD_INFO_DEFS_H + +#include +#include +#include + +/* + * low level task data that entry.S needs immediate access to + * - this struct should fit entirely inside of one cache line + * - this struct shares the supervisor stack pages + */ +#ifndef __ASSEMBLY__ +struct task_struct; +struct exec_domain; + +struct thread_info { + struct task_struct *task; /* main task structure */ + struct exec_domain *exec_domain; /* execution domain */ + __u32 flags; /* low level flags */ + __u32 status; /* thread synchronous flags */ + __u32 cpu; /* current CPU */ + int preempt_count; /* 0 => preemptable, + <0 => BUG */ + mm_segment_t addr_limit; + struct restart_block restart_block; + void __user *sysenter_return; +#ifdef CONFIG_X86_32 + unsigned long previous_esp; /* ESP of the previous stack in + case of nested (IRQ) stacks + */ + __u8 supervisor_stack[0]; +#endif + int uaccess_err; +}; + +#define INIT_THREAD_INFO(tsk) \ +{ \ + .task = &tsk, \ + .exec_domain = &default_exec_domain, \ + .flags = 0, \ + .cpu = 0, \ + .preempt_count = 1, \ + .addr_limit = KERNEL_DS, \ + .restart_block = { \ + .fn = do_no_restart_syscall, \ + }, \ +} + +#define init_thread_info (init_thread_union.thread_info) +#define init_stack (init_thread_union.stack) + +#else /* !__ASSEMBLY__ */ + +#include + +#endif /* !__ASSEMBLY__ */ + +/* + * thread information flags + * - these are process state flags that various assembly files + * may need to access + * - pending work-to-be-done flags are in LSW + * - other flags in MSW + * Warning: layout of LSW is hardcoded in entry.S + */ +#define TIF_SYSCALL_TRACE 0 /* syscall trace active */ +#define TIF_NOTIFY_RESUME 1 /* callback before returning to user */ +#define TIF_SIGPENDING 2 /* signal pending */ +#define TIF_NEED_RESCHED 3 /* rescheduling necessary */ +#define TIF_SINGLESTEP 4 /* reenable singlestep on user return*/ +#define TIF_IRET 5 /* force IRET */ +#define TIF_SYSCALL_EMU 6 /* syscall emulation active */ +#define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ +#define TIF_SECCOMP 8 /* secure computing */ +#define TIF_MCE_NOTIFY 10 /* notify userspace of an MCE */ +#define TIF_NOTSC 16 /* TSC is not accessible in userland */ +#define TIF_IA32 17 /* 32bit process */ +#define TIF_FORK 18 /* ret_from_fork */ +#define TIF_ABI_PENDING 19 +#define TIF_MEMDIE 20 +#define TIF_DEBUG 21 /* uses debug registers */ +#define TIF_IO_BITMAP 22 /* uses I/O bitmap */ +#define TIF_FREEZE 23 /* is freezing for suspend */ +#define TIF_FORCED_TF 24 /* true if TF in eflags artificially */ +#define TIF_DEBUGCTLMSR 25 /* uses thread_struct.debugctlmsr */ +#define TIF_DS_AREA_MSR 26 /* uses thread_struct.ds_area_msr */ +#define TIF_SYSCALL_FTRACE 27 /* for ftrace syscall instrumentation */ + +#define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) +#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) +#define _TIF_SIGPENDING (1 << TIF_SIGPENDING) +#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) +#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) +#define _TIF_IRET (1 << TIF_IRET) +#define _TIF_SYSCALL_EMU (1 << TIF_SYSCALL_EMU) +#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) +#define _TIF_SECCOMP (1 << TIF_SECCOMP) +#define _TIF_MCE_NOTIFY (1 << TIF_MCE_NOTIFY) +#define _TIF_NOTSC (1 << TIF_NOTSC) +#define _TIF_IA32 (1 << TIF_IA32) +#define _TIF_FORK (1 << TIF_FORK) +#define _TIF_ABI_PENDING (1 << TIF_ABI_PENDING) +#define _TIF_DEBUG (1 << TIF_DEBUG) +#define _TIF_IO_BITMAP (1 << TIF_IO_BITMAP) +#define _TIF_FREEZE (1 << TIF_FREEZE) +#define _TIF_FORCED_TF (1 << TIF_FORCED_TF) +#define _TIF_DEBUGCTLMSR (1 << TIF_DEBUGCTLMSR) +#define _TIF_DS_AREA_MSR (1 << TIF_DS_AREA_MSR) +#define _TIF_SYSCALL_FTRACE (1 << TIF_SYSCALL_FTRACE) + +/* work to do in syscall_trace_enter() */ +#define _TIF_WORK_SYSCALL_ENTRY \ + (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_EMU | _TIF_SYSCALL_FTRACE | \ + _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | _TIF_SINGLESTEP) + +/* work to do in syscall_trace_leave() */ +#define _TIF_WORK_SYSCALL_EXIT \ + (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SINGLESTEP | \ + _TIF_SYSCALL_FTRACE) + +/* work to do on interrupt/exception return */ +#define _TIF_WORK_MASK \ + (0x0000FFFF & \ + ~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT| \ + _TIF_SINGLESTEP|_TIF_SECCOMP|_TIF_SYSCALL_EMU)) + +/* work to do on any return to user space */ +#define _TIF_ALLWORK_MASK ((0x0000FFFF & ~_TIF_SECCOMP) | _TIF_SYSCALL_FTRACE) + +/* Only used for 64 bit */ +#define _TIF_DO_NOTIFY_MASK \ + (_TIF_SIGPENDING|_TIF_MCE_NOTIFY|_TIF_NOTIFY_RESUME) + +/* flags to check in __switch_to() */ +#define _TIF_WORK_CTXSW \ + (_TIF_IO_BITMAP|_TIF_DEBUGCTLMSR|_TIF_DS_AREA_MSR|_TIF_NOTSC) + +#define _TIF_WORK_CTXSW_PREV _TIF_WORK_CTXSW +#define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW|_TIF_DEBUG) + +#define PREEMPT_ACTIVE 0x10000000 + +/* + * Thread-synchronous status. + * + * This is different from the flags in that nobody else + * ever touches our thread-synchronous status, so we don't + * have to worry about atomic accesses. + */ +#define TS_USEDFPU 0x0001 /* FPU was used by this task + this quantum (SMP) */ +#define TS_COMPAT 0x0002 /* 32bit syscall active (64BIT)*/ +#define TS_POLLING 0x0004 /* true if in idle loop + and not sleeping */ +#define TS_RESTORE_SIGMASK 0x0008 /* restore signal mask in do_signal() */ +#define TS_XSAVE 0x0010 /* Use xsave/xrstor */ + +#endif /* _ASM_X86_THREAD_INFO_DEFS_H */ diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h index 8820a73..c4716b1 100644 --- a/arch/x86/include/asm/thread_info.h +++ b/arch/x86/include/asm/thread_info.h @@ -8,147 +8,13 @@ #define _ASM_X86_THREAD_INFO_H #include -#include -#include +#include -/* - * low level task data that entry.S needs immediate access to - * - this struct should fit entirely inside of one cache line - * - this struct shares the supervisor stack pages - */ -#ifndef __ASSEMBLY__ -struct task_struct; -struct exec_domain; -#include +#ifndef __ASSEMBLER__ #include #include - -struct thread_info { - struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ - __u32 flags; /* low level flags */ - __u32 status; /* thread synchronous flags */ - __u32 cpu; /* current CPU */ - int preempt_count; /* 0 => preemptable, - <0 => BUG */ - mm_segment_t addr_limit; - struct restart_block restart_block; - void __user *sysenter_return; -#ifdef CONFIG_X86_32 - unsigned long previous_esp; /* ESP of the previous stack in - case of nested (IRQ) stacks - */ - __u8 supervisor_stack[0]; +#include #endif - int uaccess_err; -}; - -#define INIT_THREAD_INFO(tsk) \ -{ \ - .task = &tsk, \ - .exec_domain = &default_exec_domain, \ - .flags = 0, \ - .cpu = 0, \ - .preempt_count = 1, \ - .addr_limit = KERNEL_DS, \ - .restart_block = { \ - .fn = do_no_restart_syscall, \ - }, \ -} - -#define init_thread_info (init_thread_union.thread_info) -#define init_stack (init_thread_union.stack) - -#else /* !__ASSEMBLY__ */ - -#include - -#endif - -/* - * thread information flags - * - these are process state flags that various assembly files - * may need to access - * - pending work-to-be-done flags are in LSW - * - other flags in MSW - * Warning: layout of LSW is hardcoded in entry.S - */ -#define TIF_SYSCALL_TRACE 0 /* syscall trace active */ -#define TIF_NOTIFY_RESUME 1 /* callback before returning to user */ -#define TIF_SIGPENDING 2 /* signal pending */ -#define TIF_NEED_RESCHED 3 /* rescheduling necessary */ -#define TIF_SINGLESTEP 4 /* reenable singlestep on user return*/ -#define TIF_IRET 5 /* force IRET */ -#define TIF_SYSCALL_EMU 6 /* syscall emulation active */ -#define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ -#define TIF_SECCOMP 8 /* secure computing */ -#define TIF_MCE_NOTIFY 10 /* notify userspace of an MCE */ -#define TIF_NOTSC 16 /* TSC is not accessible in userland */ -#define TIF_IA32 17 /* 32bit process */ -#define TIF_FORK 18 /* ret_from_fork */ -#define TIF_ABI_PENDING 19 -#define TIF_MEMDIE 20 -#define TIF_DEBUG 21 /* uses debug registers */ -#define TIF_IO_BITMAP 22 /* uses I/O bitmap */ -#define TIF_FREEZE 23 /* is freezing for suspend */ -#define TIF_FORCED_TF 24 /* true if TF in eflags artificially */ -#define TIF_DEBUGCTLMSR 25 /* uses thread_struct.debugctlmsr */ -#define TIF_DS_AREA_MSR 26 /* uses thread_struct.ds_area_msr */ -#define TIF_SYSCALL_FTRACE 27 /* for ftrace syscall instrumentation */ - -#define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) -#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) -#define _TIF_SIGPENDING (1 << TIF_SIGPENDING) -#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) -#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) -#define _TIF_IRET (1 << TIF_IRET) -#define _TIF_SYSCALL_EMU (1 << TIF_SYSCALL_EMU) -#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) -#define _TIF_SECCOMP (1 << TIF_SECCOMP) -#define _TIF_MCE_NOTIFY (1 << TIF_MCE_NOTIFY) -#define _TIF_NOTSC (1 << TIF_NOTSC) -#define _TIF_IA32 (1 << TIF_IA32) -#define _TIF_FORK (1 << TIF_FORK) -#define _TIF_ABI_PENDING (1 << TIF_ABI_PENDING) -#define _TIF_DEBUG (1 << TIF_DEBUG) -#define _TIF_IO_BITMAP (1 << TIF_IO_BITMAP) -#define _TIF_FREEZE (1 << TIF_FREEZE) -#define _TIF_FORCED_TF (1 << TIF_FORCED_TF) -#define _TIF_DEBUGCTLMSR (1 << TIF_DEBUGCTLMSR) -#define _TIF_DS_AREA_MSR (1 << TIF_DS_AREA_MSR) -#define _TIF_SYSCALL_FTRACE (1 << TIF_SYSCALL_FTRACE) - -/* work to do in syscall_trace_enter() */ -#define _TIF_WORK_SYSCALL_ENTRY \ - (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_EMU | _TIF_SYSCALL_FTRACE | \ - _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | _TIF_SINGLESTEP) - -/* work to do in syscall_trace_leave() */ -#define _TIF_WORK_SYSCALL_EXIT \ - (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SINGLESTEP | \ - _TIF_SYSCALL_FTRACE) - -/* work to do on interrupt/exception return */ -#define _TIF_WORK_MASK \ - (0x0000FFFF & \ - ~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT| \ - _TIF_SINGLESTEP|_TIF_SECCOMP|_TIF_SYSCALL_EMU)) - -/* work to do on any return to user space */ -#define _TIF_ALLWORK_MASK ((0x0000FFFF & ~_TIF_SECCOMP) | _TIF_SYSCALL_FTRACE) - -/* Only used for 64 bit */ -#define _TIF_DO_NOTIFY_MASK \ - (_TIF_SIGPENDING|_TIF_MCE_NOTIFY|_TIF_NOTIFY_RESUME) - -/* flags to check in __switch_to() */ -#define _TIF_WORK_CTXSW \ - (_TIF_IO_BITMAP|_TIF_DEBUGCTLMSR|_TIF_DS_AREA_MSR|_TIF_NOTSC) - -#define _TIF_WORK_CTXSW_PREV _TIF_WORK_CTXSW -#define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW|_TIF_DEBUG) - -#define PREEMPT_ACTIVE 0x10000000 /* thread information allocation */ #ifdef CONFIG_DEBUG_STACK_USAGE @@ -165,6 +31,7 @@ struct thread_info { #ifdef CONFIG_X86_32 #define STACK_WARN (THREAD_SIZE/8) + /* * macros/functions for gaining access to the thread information structure * @@ -172,7 +39,6 @@ struct thread_info { */ #ifndef __ASSEMBLY__ - /* how to get the current stack pointer from C */ register unsigned long current_stack_pointer asm("esp") __used; @@ -194,17 +60,19 @@ static inline struct thread_info *current_thread_info(void) #define GET_THREAD_INFO_WITH_ESP(reg) \ andl $-THREAD_SIZE, reg -#endif +#endif /* !__ASSEMBLY__ */ #else /* X86_32 */ -#include +#include #define KERNEL_STACK_OFFSET (5*8) /* * macros/functions for gaining access to the thread information structure * preempt_count needs to be 1 initially, until the scheduler is functional. */ +#include + #ifndef __ASSEMBLY__ DECLARE_PER_CPU(unsigned long, kernel_stack); @@ -223,28 +91,17 @@ static inline struct thread_info *current_thread_info(void) movq PER_CPU_VAR(kernel_stack),reg ; \ subq $(THREAD_SIZE-KERNEL_STACK_OFFSET),reg -#endif +#endif /* !__ASSEMBLY__ */ #endif /* !X86_32 */ /* - * Thread-synchronous status. - * - * This is different from the flags in that nobody else - * ever touches our thread-synchronous status, so we don't - * have to worry about atomic accesses. + * Thread-synchronous status accessors */ -#define TS_USEDFPU 0x0001 /* FPU was used by this task - this quantum (SMP) */ -#define TS_COMPAT 0x0002 /* 32bit syscall active (64BIT)*/ -#define TS_POLLING 0x0004 /* true if in idle loop - and not sleeping */ -#define TS_RESTORE_SIGMASK 0x0008 /* restore signal mask in do_signal() */ -#define TS_XSAVE 0x0010 /* Use xsave/xrstor */ +#ifndef __ASSEMBLY__ #define tsk_is_polling(t) (task_thread_info(t)->status & TS_POLLING) -#ifndef __ASSEMBLY__ #define HAVE_SET_RESTORE_SIGMASK 1 static inline void set_restore_sigmask(void) { @@ -260,4 +117,5 @@ extern void free_thread_info(struct thread_info *ti); extern int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src); #define arch_task_cache_init arch_task_cache_init #endif + #endif /* _ASM_X86_THREAD_INFO_H */ diff --git a/arch/x86/kernel/acpi/cstate.c b/arch/x86/kernel/acpi/cstate.c index bbbe4bb..73781d8 100644 --- a/arch/x86/kernel/acpi/cstate.c +++ b/arch/x86/kernel/acpi/cstate.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include diff --git a/drivers/net/loopback.c b/drivers/net/loopback.c index b7d438a..f212bad 100644 --- a/drivers/net/loopback.c +++ b/drivers/net/loopback.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include diff --git a/fs/nfs/iostat.h b/fs/nfs/iostat.h index a2ab252..304f668 100644 --- a/fs/nfs/iostat.h +++ b/fs/nfs/iostat.h @@ -11,6 +11,7 @@ #define _NFS_IOSTAT #include +#include #include #include diff --git a/include/asm-generic/percpu-defs.h b/include/asm-generic/percpu-defs.h new file mode 100644 index 0000000..6ed3f57 --- /dev/null +++ b/include/asm-generic/percpu-defs.h @@ -0,0 +1,33 @@ +#ifndef _ASM_GENERIC_PERCPU_DEFS_H +#define _ASM_GENERIC_PERCPU_DEFS_H + + +#ifndef PER_CPU_BASE_SECTION +#ifdef CONFIG_SMP +#define PER_CPU_BASE_SECTION ".data.percpu" +#else +#define PER_CPU_BASE_SECTION ".data" +#endif +#endif + +#ifdef CONFIG_SMP + +#ifdef MODULE +#define PER_CPU_SHARED_ALIGNED_SECTION "" +#else +#define PER_CPU_SHARED_ALIGNED_SECTION ".shared_aligned" +#endif +#define PER_CPU_FIRST_SECTION ".first" + +#else + +#define PER_CPU_SHARED_ALIGNED_SECTION "" +#define PER_CPU_FIRST_SECTION "" + +#endif + +#ifndef PER_CPU_ATTRIBUTES +#define PER_CPU_ATTRIBUTES +#endif + +#endif /* _ASM_GENERIC_PERCPU_DEFS_H */ diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h index af47b9e..8dfca11 100644 --- a/include/asm-generic/percpu.h +++ b/include/asm-generic/percpu.h @@ -73,50 +73,4 @@ extern void setup_per_cpu_areas(void); #endif /* SMP */ -#ifndef PER_CPU_BASE_SECTION -#ifdef CONFIG_SMP -#define PER_CPU_BASE_SECTION ".data.percpu" -#else -#define PER_CPU_BASE_SECTION ".data" -#endif -#endif - -#ifdef CONFIG_SMP - -#ifdef MODULE -#define PER_CPU_SHARED_ALIGNED_SECTION "" -#else -#define PER_CPU_SHARED_ALIGNED_SECTION ".shared_aligned" -#endif -#define PER_CPU_FIRST_SECTION ".first" - -#else - -#define PER_CPU_SHARED_ALIGNED_SECTION "" -#define PER_CPU_FIRST_SECTION "" - -#endif - -#ifndef PER_CPU_ATTRIBUTES -#define PER_CPU_ATTRIBUTES -#endif - -#define DECLARE_PER_CPU_SECTION(type, name, section) \ - extern \ - __attribute__((__section__(PER_CPU_BASE_SECTION section))) \ - PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name - -#define DECLARE_PER_CPU(type, name) \ - DECLARE_PER_CPU_SECTION(type, name, "") - -#define DECLARE_PER_CPU_SHARED_ALIGNED(type, name) \ - DECLARE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \ - ____cacheline_aligned_in_smp - -#define DECLARE_PER_CPU_PAGE_ALIGNED(type, name) \ - DECLARE_PER_CPU_SECTION(type, name, ".page_aligned") - -#define DECLARE_PER_CPU_FIRST(type, name) \ - DECLARE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION) - #endif /* _ASM_GENERIC_PERCPU_H_ */ diff --git a/include/linux/completion-defs.h b/include/linux/completion-defs.h new file mode 100644 index 0000000..8c10130 --- /dev/null +++ b/include/linux/completion-defs.h @@ -0,0 +1,30 @@ +#ifndef __LINUX_COMPLETION_DEFS_H +#define __LINUX_COMPLETION_DEFS_H + +/* + * (C) Copyright 2001 Linus Torvalds + * + * Atomic wait-for-completion handler data structures. + * See kernel/sched.c for details. + */ + +#include + +/** + * struct completion - structure used to maintain state for a "completion" + * + * This is the opaque structure used to maintain the state for a "completion". + * Completions currently use a FIFO to queue threads that have to wait for + * the "completion" event. + * + * See also: complete(), wait_for_completion() (and friends _timeout, + * _interruptible, _interruptible_timeout, and _killable), init_completion(), + * and macros DECLARE_COMPLETION(), DECLARE_COMPLETION_ONSTACK(), and + * INIT_COMPLETION(). + */ +struct completion { + unsigned int done; + wait_queue_head_t wait; +}; + +#endif /* __LINUX_COMPLETION_DEFS_H */ diff --git a/include/linux/completion.h b/include/linux/completion.h index 4a6b604..a3833cd 100644 --- a/include/linux/completion.h +++ b/include/linux/completion.h @@ -8,25 +8,9 @@ * See kernel/sched.c for details. */ +#include #include -/** - * struct completion - structure used to maintain state for a "completion" - * - * This is the opaque structure used to maintain the state for a "completion". - * Completions currently use a FIFO to queue threads that have to wait for - * the "completion" event. - * - * See also: complete(), wait_for_completion() (and friends _timeout, - * _interruptible, _interruptible_timeout, and _killable), init_completion(), - * and macros DECLARE_COMPLETION(), DECLARE_COMPLETION_ONSTACK(), and - * INIT_COMPLETION(). - */ -struct completion { - unsigned int done; - wait_queue_head_t wait; -}; - #define COMPLETION_INITIALIZER(work) \ { 0, __WAIT_QUEUE_HEAD_INITIALIZER((work).wait) } diff --git a/include/linux/fs.h b/include/linux/fs.h index 562d285..4b8e0d6 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -297,7 +297,7 @@ struct inodes_stat_t { #ifdef __KERNEL__ #include -#include +#include #include #include #include diff --git a/include/linux/genhd.h b/include/linux/genhd.h index 634c530..41dc97c 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -12,6 +12,7 @@ #include #include #include +#include #ifdef CONFIG_BLOCK diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 0bbc15f..91d78a6 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -1,10 +1,7 @@ #ifndef __LINUX_GFP_H #define __LINUX_GFP_H -#include -#include -#include -#include +#include struct vm_area_struct; @@ -99,6 +96,11 @@ struct vm_area_struct; /* 4GB DMA on some platforms */ #define GFP_DMA32 __GFP_DMA32 +#include +#include +#include +#include + /* Convert GFP flags to their corresponding migrate type */ static inline int allocflags_to_migratetype(gfp_t gfp_flags) { diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 186ec6a..86d23d2 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -4,15 +4,15 @@ #ifndef __ASSEMBLY__ #ifndef __GENERATING_BOUNDS_H -#include +#include #include -#include +#include #include #include #include #include #include -#include +#include #include #include #include diff --git a/include/linux/percpu-alloc.h b/include/linux/percpu-alloc.h new file mode 100644 index 0000000..c75f8ff --- /dev/null +++ b/include/linux/percpu-alloc.h @@ -0,0 +1,42 @@ +#ifndef __LINUX_PERCPU_ALLOC_H +#define __LINUX_PERCPU_ALLOC_H + +#include +#include /* For kmalloc() */ + +#ifdef CONFIG_SMP + +#ifdef CONFIG_HAVE_DYNAMIC_PER_CPU_AREA + +extern void *__alloc_reserved_percpu(size_t size, size_t align); + +#endif /* CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */ + +extern void *__alloc_percpu(size_t size, size_t align); +extern void free_percpu(void *__pdata); + +#else /* CONFIG_SMP */ + +static inline void *__alloc_percpu(size_t size, size_t align) +{ + /* + * Can't easily make larger alignment work with kmalloc. WARN + * on it. Larger alignment should only be used for module + * percpu sections on SMP for which this path isn't used. + */ + WARN_ON_ONCE(align > SMP_CACHE_BYTES); + return kzalloc(size, GFP_KERNEL); +} + +static inline void free_percpu(void *p) +{ + kfree(p); +} + +#endif /* CONFIG_SMP */ + +#define alloc_percpu(type) (type *)__alloc_percpu(sizeof(type), \ + __alignof__(type)) + + +#endif /* __LINUX_PERCPU_ALLOC_H */ diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h new file mode 100644 index 0000000..b99eab5 --- /dev/null +++ b/include/linux/percpu-defs.h @@ -0,0 +1,79 @@ +#ifndef _LINUX_PERCPU_DEFS_H +#define _LINUX_PERCPU_DEFS_H + +#include + +/* + * Determine the real variable name from the name visible in the + * kernel sources. + */ +#define per_cpu_var(var) per_cpu__##var + +/* + * Base implementations of per-CPU variable declarations and definitions, where + * the section in which the variable is to be placed is provided by the + * 'section' argument. This may be used to affect the parameters governing the + * variable's storage. + * + * NOTE! The sections for the DECLARE and for the DEFINE must match, lest + * linkage errors occur due the compiler generating the wrong code to access + * that section. + */ +#define DECLARE_PER_CPU_SECTION(type, name, section) \ + extern \ + __attribute__((__section__(PER_CPU_BASE_SECTION section))) \ + PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name + +#define DEFINE_PER_CPU_SECTION(type, name, section) \ + __attribute__((__section__(PER_CPU_BASE_SECTION section))) \ + PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name + +/* + * Variant on the per-CPU variable declaration/definition theme used for + * ordinary per-CPU variables. + */ +#define DECLARE_PER_CPU(type, name) \ + DECLARE_PER_CPU_SECTION(type, name, "") + +#define DEFINE_PER_CPU(type, name) \ + DEFINE_PER_CPU_SECTION(type, name, "") + +/* + * Declaration/definition used for per-CPU variables that must come first in + * the set of variables. + */ +#define DECLARE_PER_CPU_FIRST(type, name) \ + DECLARE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION) + +#define DEFINE_PER_CPU_FIRST(type, name) \ + DEFINE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION) + +/* + * Declaration/definition used for per-CPU variables that must be cacheline + * aligned under SMP conditions and shared (?). + */ +#define DECLARE_PER_CPU_SHARED_ALIGNED(type, name) \ + DECLARE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \ + ____cacheline_aligned_in_smp + +#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ + DEFINE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \ + ____cacheline_aligned_in_smp + +/* + * Declaration/definition used for per-CPU variables that must be page aligned. + */ +#define DECLARE_PER_CPU_PAGE_ALIGNED(type, name) \ + DECLARE_PER_CPU_SECTION(type, name, ".page_aligned") + +#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \ + DEFINE_PER_CPU_SECTION(type, name, ".page_aligned") + +/* + * Intermodule exports for per-CPU variables. + */ +#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var) +#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var) + + +#endif /* _LINUX_PERCPU_DEFS_H */ diff --git a/include/linux/percpu.h b/include/linux/percpu.h index f052d81..4cb47c8 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -1,33 +1,16 @@ #ifndef __LINUX_PERCPU_H #define __LINUX_PERCPU_H +#include +#ifndef __ASSEMBLER__ #include -#include /* For kmalloc() */ -#include #include #include - +#include +#endif #include -#define DEFINE_PER_CPU_SECTION(type, name, section) \ - __attribute__((__section__(PER_CPU_BASE_SECTION section))) \ - PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name - -#define DEFINE_PER_CPU(type, name) \ - DEFINE_PER_CPU_SECTION(type, name, "") - -#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ - DEFINE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \ - ____cacheline_aligned_in_smp - -#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \ - DEFINE_PER_CPU_SECTION(type, name, ".page_aligned") - -#define DEFINE_PER_CPU_FIRST(type, name) \ - DEFINE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION) - -#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var) -#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var) +#ifndef __ASSEMBLER__ /* enough to cover all DEFINE_PER_CPUs in modules */ #ifdef CONFIG_MODULES @@ -98,8 +81,6 @@ extern ssize_t __init pcpu_embed_first_chunk( */ #define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))) -extern void *__alloc_reserved_percpu(size_t size, size_t align); - #else /* CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */ struct percpu_data { @@ -116,34 +97,12 @@ struct percpu_data { #endif /* CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */ -extern void *__alloc_percpu(size_t size, size_t align); -extern void free_percpu(void *__pdata); - #else /* CONFIG_SMP */ #define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); }) -static inline void *__alloc_percpu(size_t size, size_t align) -{ - /* - * Can't easily make larger alignment work with kmalloc. WARN - * on it. Larger alignment should only be used for module - * percpu sections on SMP for which this path isn't used. - */ - WARN_ON_ONCE(align > SMP_CACHE_BYTES); - return kzalloc(size, GFP_KERNEL); -} - -static inline void free_percpu(void *p) -{ - kfree(p); -} - #endif /* CONFIG_SMP */ -#define alloc_percpu(type) (type *)__alloc_percpu(sizeof(type), \ - __alignof__(type)) - /* * Optional methods for optimized non-lvalue per-cpu variable access. * @@ -196,4 +155,5 @@ do { \ # define percpu_xor(var, val) __percpu_generic_to_op(var, (val), ^=) #endif +#endif /* !__ASSEMBLER__ */ #endif /* __LINUX_PERCPU_H */ diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 72b1a10..a3c0b06 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -6,7 +6,7 @@ * preempt_count (used for kernel preemption, interrupt count, etc.) */ -#include +#include #include #include diff --git a/include/linux/prefetch.h b/include/linux/prefetch.h index af7c36a..cfe920a 100644 --- a/include/linux/prefetch.h +++ b/include/linux/prefetch.h @@ -11,8 +11,7 @@ #define _LINUX_PREFETCH_H #include -#include -#include +#include /* prefetch(x) attempts to pre-emptively get the memory pointed to diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 15fbb3c..f1a6c0e 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -39,7 +39,7 @@ #include #include #include -#include +#include /** * struct rcu_head - callback structure for use with RCU diff --git a/include/linux/seqlock-defs.h b/include/linux/seqlock-defs.h new file mode 100644 index 0000000..a06a460 --- /dev/null +++ b/include/linux/seqlock-defs.h @@ -0,0 +1,37 @@ +/* + * Reader/writer consistent mechanism without starving writers. This type of + * lock for data where the reader wants a consistent set of information + * and is willing to retry if the information changes. Readers never + * block but they may have to retry if a writer is in + * progress. Writers do not wait for readers. + * + * This is not as cache friendly as brlock. Also, this will not work + * for data that contains pointers, because any writer could + * invalidate a pointer that a reader was following. + * + * Expected reader usage: + * do { + * seq = read_seqbegin(&foo); + * ... + * } while (read_seqretry(&foo, seq)); + * + * + * On non-SMP the spin locks disappear but the writer still needs + * to increment the sequence variables because an interrupt routine could + * change the state of the data. + * + * Based on x86_64 vsyscall gettimeofday + * by Keith Owens and Andrea Arcangeli + */ + +#ifndef __LINUX_SEQLOCK_DEFS_H +#define __LINUX_SEQLOCK_DEFS_H + +#include + +typedef struct { + unsigned sequence; + spinlock_t lock; +} seqlock_t; + +#endif /* __LINUX_SEQLOCK_DEFS_H */ diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 632205c..a19b500 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -1,11 +1,9 @@ -#ifndef __LINUX_SEQLOCK_H -#define __LINUX_SEQLOCK_H /* * Reader/writer consistent mechanism without starving writers. This type of * lock for data where the reader wants a consistent set of information * and is willing to retry if the information changes. Readers never * block but they may have to retry if a writer is in - * progress. Writers do not wait for readers. + * progress. Writers do not wait for readers. * * This is not as cache friendly as brlock. Also, this will not work * for data that contains pointers, because any writer could @@ -22,17 +20,16 @@ * to increment the sequence variables because an interrupt routine could * change the state of the data. * - * Based on x86_64 vsyscall gettimeofday + * Based on x86_64 vsyscall gettimeofday * by Keith Owens and Andrea Arcangeli */ -#include -#include +#ifndef __LINUX_SEQLOCK_H +#define __LINUX_SEQLOCK_H -typedef struct { - unsigned sequence; - spinlock_t lock; -} seqlock_t; +#include +#include +#include /* * These macros triggered gcc-3.x compile-time problems. We think these are @@ -53,7 +50,8 @@ typedef struct { #define DEFINE_SEQLOCK(x) \ seqlock_t x = __SEQLOCK_UNLOCKED(x) -/* Lock out other writers and update the count. +/* + * Lock out other writers and update the count. * Acts like a normal spin_lock/unlock. * Don't need preempt_disable() because that is in the spin_lock already. */ diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 252b245..137dcff 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -50,7 +50,6 @@ #include #include #include -#include #include #include #include diff --git a/include/linux/thread_info-defs.h b/include/linux/thread_info-defs.h new file mode 100644 index 0000000..fe1257e --- /dev/null +++ b/include/linux/thread_info-defs.h @@ -0,0 +1,54 @@ +/* Common low-level thread information definitions + * + * Copyright (C) 2002 David Howells (dhowells@redhat.com) + * - Incorporating suggestions made by Linus Torvalds + */ + +#ifndef _LINUX_THREAD_INFO_DEFS_H +#define _LINUX_THREAD_INFO_DEFS_H + +#include + +struct timespec; +struct compat_timespec; + +/* + * System call restart block. + */ +struct restart_block { + long (*fn)(struct restart_block *); + union { + struct { + unsigned long arg0, arg1, arg2, arg3; + }; + /* For futex_wait */ + struct { + u32 *uaddr; + u32 val; + u32 flags; + u32 bitset; + u64 time; + } futex; + /* For nanosleep */ + struct { + clockid_t index; + struct timespec __user *rmtp; +#ifdef CONFIG_COMPAT + struct compat_timespec __user *compat_rmtp; +#endif + u64 expires; + } nanosleep; + /* For poll */ + struct { + struct pollfd __user *ufds; + int nfds; + int has_timeout; + unsigned long tv_sec; + unsigned long tv_nsec; + } poll; + }; +}; + +#include + +#endif /* _LINUX_THREAD_INFO_DEFS_H */ diff --git a/include/linux/thread_info.h b/include/linux/thread_info.h index e6b820f..e9cc8da 100644 --- a/include/linux/thread_info.h +++ b/include/linux/thread_info.h @@ -7,52 +7,12 @@ #ifndef _LINUX_THREAD_INFO_H #define _LINUX_THREAD_INFO_H -#include - -struct timespec; -struct compat_timespec; - -/* - * System call restart block. - */ -struct restart_block { - long (*fn)(struct restart_block *); - union { - struct { - unsigned long arg0, arg1, arg2, arg3; - }; - /* For futex_wait */ - struct { - u32 *uaddr; - u32 val; - u32 flags; - u32 bitset; - u64 time; - } futex; - /* For nanosleep */ - struct { - clockid_t index; - struct timespec __user *rmtp; -#ifdef CONFIG_COMPAT - struct compat_timespec __user *compat_rmtp; -#endif - u64 expires; - } nanosleep; - /* For poll */ - struct { - struct pollfd __user *ufds; - int nfds; - int has_timeout; - unsigned long tv_sec; - unsigned long tv_nsec; - } poll; - }; -}; +#include extern long do_no_restart_syscall(struct restart_block *parm); -#include #include +#include #ifdef __KERNEL__ diff --git a/include/linux/wait-defs.h b/include/linux/wait-defs.h new file mode 100644 index 0000000..4d0f312 --- /dev/null +++ b/include/linux/wait-defs.h @@ -0,0 +1,25 @@ +#ifndef _LINUX_WAIT_DEFS_H +#define _LINUX_WAIT_DEFS_H + +#include +#include + +struct __wait_queue_head { + spinlock_t lock; + struct list_head task_list; +}; +typedef struct __wait_queue_head wait_queue_head_t; + +typedef struct __wait_queue wait_queue_t; + +typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int sync, void *key); + +struct __wait_queue { + unsigned int flags; +#define WQ_FLAG_EXCLUSIVE 0x01 + void *private; + wait_queue_func_t func; + struct list_head task_list; +}; + +#endif /* _LINUX_WAIT_DEFS_H */ diff --git a/include/linux/wait.h b/include/linux/wait.h index 5d631c1..8848962 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -21,22 +21,10 @@ #include #include -#include +#include #include #include -typedef struct __wait_queue wait_queue_t; -typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int sync, void *key); -int default_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key); - -struct __wait_queue { - unsigned int flags; -#define WQ_FLAG_EXCLUSIVE 0x01 - void *private; - wait_queue_func_t func; - struct list_head task_list; -}; - struct wait_bit_key { void *flags; int bit_nr; @@ -47,17 +35,12 @@ struct wait_bit_queue { wait_queue_t wait; }; -struct __wait_queue_head { - spinlock_t lock; - struct list_head task_list; -}; -typedef struct __wait_queue_head wait_queue_head_t; - struct task_struct; /* * Macros for declaration and initialisaton of the datatypes */ +extern int default_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key); #define __WAITQUEUE_INITIALIZER(name, tsk) { \ .private = tsk, \ diff --git a/kernel/module.c b/kernel/module.c index e797812..f5c238c 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -52,6 +52,7 @@ #include #include #include +#include #if 0 #define DEBUGP printk diff --git a/kernel/srcu.c b/kernel/srcu.c index b0aeeaf..8b50f56 100644 --- a/kernel/srcu.c +++ b/kernel/srcu.c @@ -26,7 +26,7 @@ #include #include -#include +#include #include #include #include diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c index 912823e..067cdf3 100644 --- a/kernel/stop_machine.c +++ b/kernel/stop_machine.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include diff --git a/kernel/workqueue.c b/kernel/workqueue.c index f71fb2a..d607306 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -33,6 +33,7 @@ #include #include #include +#include #include /* diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c index aeaa6d7..e72afc7 100644 --- a/lib/percpu_counter.c +++ b/lib/percpu_counter.c @@ -3,6 +3,7 @@ */ #include +#include #include #include #include diff --git a/net/core/neighbour.c b/net/core/neighbour.c index a1cbce7..d45b376 100644 --- a/net/core/neighbour.c +++ b/net/core/neighbour.c @@ -35,6 +35,7 @@ #include #include #include +#include #define NEIGH_DEBUG 1 diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index 7f03373..16572e0 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -86,6 +86,7 @@ #include #include #include +#include #include #include