--- include/linux/ipipe.h | 17 +++----- include/linux/smp.h | 8 --- kernel/ipipe/Kconfig.debug | 11 ++++- kernel/ipipe/core.c | 16 ++++--- kernel/ipipe/tracer.c | 91 +++++++++++++++++++++++---------------------- lib/smp_processor_id.c | 17 +++++++- 6 files changed, 90 insertions(+), 70 deletions(-) Index: linux-2.6.20/include/linux/ipipe.h =================================================================== --- linux-2.6.20.orig/include/linux/ipipe.h +++ linux-2.6.20/include/linux/ipipe.h @@ -96,16 +96,17 @@ #define IPIPE_NR_CPUS NR_CPUS #define ipipe_declare_cpuid int cpuid #define ipipe_load_cpuid() do { \ - cpuid = ipipe_processor_id(); \ + cpuid = smp_processor_id(); \ } while(0) #define ipipe_lock_cpu(flags) do { \ local_irq_save_hw(flags); \ - cpuid = ipipe_processor_id(); \ + cpuid = smp_processor_id(); \ } while(0) #define ipipe_unlock_cpu(flags) local_irq_restore_hw(flags) #define ipipe_get_cpu(flags) ipipe_lock_cpu(flags) #define ipipe_put_cpu(flags) ipipe_unlock_cpu(flags) -#define ipipe_current_domain per_cpu(ipipe_percpu_domain, ipipe_processor_id()) +#define ipipe_current_domain \ + per_cpu(ipipe_percpu_domain, raw_smp_processor_id()) #else /* !CONFIG_SMP */ @@ -304,7 +305,7 @@ void __ipipe_remove_domain_proc(struct i void __ipipe_flush_printk(unsigned irq, void *cookie); -void fastcall __ipipe_walk_pipeline(struct list_head *pos, int cpuid); +void __ipipe_walk_pipeline(struct list_head *pos); int fastcall __ipipe_schedule_irq(unsigned irq, struct list_head *head); @@ -402,9 +403,8 @@ do { \ #define ipipe_trap_notify(ex, regs) \ ({ \ - ipipe_declare_cpuid; \ + int cpuid = raw_smp_processor_id(); \ int ret = 0; \ - ipipe_load_cpuid(); \ if ((test_bit(IPIPE_NOSTACK_FLAG, &ipipe_current_domain->cpudata[cpuid].status) || \ ((current)->flags & PF_EVNOTIFY)) && \ __ipipe_event_monitored_p(ex)) \ @@ -529,7 +529,7 @@ static inline void ipipe_restore_pipelin * the truth value (if this is wrong, the failed optimization will * be caught in __ipipe_restore_pipeline_head() if * CONFIG_DEBUG_KERNEL is set). */ - if ((x ^ test_bit(IPIPE_STALL_FLAG, &head->cpudata[ipipe_processor_id()].status)) & 1) + if ((x ^ test_bit(IPIPE_STALL_FLAG, &head->cpudata[smp_processor_id()].status)) & 1) __ipipe_restore_pipeline_head(head,x); } @@ -635,8 +635,6 @@ int ipipe_disable_ondemand_mappings(stru local_irq_disable_hw(); \ } while(0) -#define smp_processor_id_hw() ipipe_processor_id() - #define ipipe_irq_lock(irq) \ do { \ ipipe_declare_cpuid; \ @@ -670,7 +668,6 @@ int ipipe_disable_ondemand_mappings(stru #define local_irq_disable_hw_cond() do { } while(0) #define local_irq_save_hw_cond(flags) do { (void)(flags); } while(0) #define local_irq_restore_hw_cond(flags) do { } while(0) -#define smp_processor_id_hw() smp_processor_id() #define ipipe_irq_lock(irq) do { } while(0) #define ipipe_irq_unlock(irq) do { } while(0) Index: linux-2.6.20/kernel/ipipe/core.c =================================================================== --- linux-2.6.20.orig/kernel/ipipe/core.c +++ linux-2.6.20/kernel/ipipe/core.c @@ -272,7 +272,7 @@ void fastcall ipipe_unstall_pipeline_fro else pos = __ipipe_pipeline.next; - __ipipe_walk_pipeline(pos, cpuid); + __ipipe_walk_pipeline(pos); if (__ipipe_pipeline_head_p(ipd)) local_irq_enable_hw(); @@ -316,7 +316,7 @@ void ipipe_unstall_pipeline_head(void) if (likely(head == per_cpu(ipipe_percpu_domain, cpuid))) __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); else - __ipipe_walk_pipeline(&head->p_link, cpuid); + __ipipe_walk_pipeline(&head->p_link); } local_irq_enable_hw(); @@ -352,7 +352,7 @@ void fastcall __ipipe_restore_pipeline_h if (likely(head == per_cpu(ipipe_percpu_domain, cpuid))) __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); else - __ipipe_walk_pipeline(&head->p_link, cpuid); + __ipipe_walk_pipeline(&head->p_link); } local_irq_enable_hw(); } @@ -361,9 +361,13 @@ void fastcall __ipipe_restore_pipeline_h /* __ipipe_walk_pipeline(): Plays interrupts pending in the log. Must be called with local hw interrupts disabled. */ -void fastcall __ipipe_walk_pipeline(struct list_head *pos, int cpuid) +void __ipipe_walk_pipeline(struct list_head *pos) { - struct ipipe_domain *this_domain = per_cpu(ipipe_percpu_domain, cpuid); + struct ipipe_domain *this_domain; + ipipe_declare_cpuid; + + ipipe_load_cpuid(); + this_domain = per_cpu(ipipe_percpu_domain, cpuid); while (pos != &__ipipe_pipeline) { struct ipipe_domain *next_domain = @@ -810,7 +814,7 @@ void fastcall __ipipe_sync_stage(unsigne __ipipe_run_isr(ipd, irq, cpuid); #ifdef CONFIG_SMP { - int _cpuid = ipipe_processor_id(); + int _cpuid = smp_processor_id(); if (_cpuid != cpuid) { /* Handle CPU migration. */ /* Index: linux-2.6.20/kernel/ipipe/tracer.c =================================================================== --- linux-2.6.20.orig/kernel/ipipe/tracer.c +++ linux-2.6.20/kernel/ipipe/tracer.c @@ -144,19 +144,20 @@ static void __ipipe_print_symname(struct static notrace void -__ipipe_store_domain_states(struct ipipe_trace_point *point, int cpu_id) +__ipipe_store_domain_states(struct ipipe_trace_point *point) { struct list_head *pos; int i = 0; + int cpuid = raw_smp_processor_id(); list_for_each_prev(pos, &__ipipe_pipeline) { struct ipipe_domain *ipd = list_entry(pos, struct ipipe_domain, p_link); - if (test_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpu_id].status)) + if (test_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status)) point->flags |= 1 << (i + IPIPE_TFLG_DOMSTATE_SHIFT); - if (ipd == per_cpu(ipipe_percpu_domain, cpu_id)) + if (ipd == per_cpu(ipipe_percpu_domain, cpuid)) point->flags |= i << IPIPE_TFLG_CURRDOM_SHIFT; if (++i > IPIPE_TFLG_DOMSTATE_BITS) @@ -164,17 +165,18 @@ __ipipe_store_domain_states(struct ipipe } } -static notrace int __ipipe_get_free_trace_path(int old, int cpu_id) +static notrace int __ipipe_get_free_trace_path(int old) { int new_active = old; struct ipipe_trace_path *tp; + int cpuid = raw_smp_processor_id(); do { if (++new_active == IPIPE_TRACE_PATHS) new_active = 0; - tp = &trace_paths[cpu_id][new_active]; - } while ((new_active == max_path[cpu_id]) || - (new_active == frozen_path[cpu_id]) || + tp = &trace_paths[cpuid][new_active]; + } while ((new_active == max_path[cpuid]) || + (new_active == frozen_path[cpuid]) || tp->dump_lock); return new_active; @@ -198,30 +200,31 @@ __ipipe_migrate_pre_trace(struct ipipe_t } static notrace struct ipipe_trace_path * -__ipipe_trace_end(int cpu_id, struct ipipe_trace_path *tp, int pos) +__ipipe_trace_end(struct ipipe_trace_path *tp, int pos) { struct ipipe_trace_path *old_tp = tp; - long active = active_path[cpu_id]; unsigned long long length; + int cpuid = raw_smp_processor_id(); + long active = active_path[cpuid]; /* do we have a new worst case? */ length = tp->point[tp->end].timestamp - tp->point[tp->begin].timestamp; - if (length > (trace_paths[cpu_id][max_path[cpu_id]]).length) { + if (length > (trace_paths[cpuid][max_path[cpuid]]).length) { /* we need protection here against other cpus trying to start a proc dump */ spin_lock(&global_path_lock); /* active path holds new worst case */ tp->length = length; - max_path[cpu_id] = active; + max_path[cpuid] = active; /* find next unused trace path */ - active = __ipipe_get_free_trace_path(active, cpu_id); + active = __ipipe_get_free_trace_path(active); spin_unlock(&global_path_lock); - tp = &trace_paths[cpu_id][active]; + tp = &trace_paths[cpuid][active]; /* migrate last entries for pre-tracing */ __ipipe_migrate_pre_trace(tp, old_tp, pos); @@ -231,10 +234,11 @@ __ipipe_trace_end(int cpu_id, struct ipi } static notrace struct ipipe_trace_path * -__ipipe_trace_freeze(int cpu_id, struct ipipe_trace_path *tp, int pos) +__ipipe_trace_freeze(struct ipipe_trace_path *tp, int pos) { struct ipipe_trace_path *old_tp = tp; - long active = active_path[cpu_id]; + int cpuid = raw_smp_processor_id(); + long active = active_path[cpuid]; int i; /* frozen paths have no core (begin=end) */ @@ -244,21 +248,21 @@ __ipipe_trace_freeze(int cpu_id, struct * to set their frozen path or to start a proc dump */ spin_lock(&global_path_lock); - frozen_path[cpu_id] = active; + frozen_path[cpuid] = active; /* find next unused trace path */ - active = __ipipe_get_free_trace_path(active, cpu_id); + active = __ipipe_get_free_trace_path(active); /* check if this is the first frozen path */ for_each_online_cpu(i) { - if ((i != cpu_id) && + if ((i != cpuid) && (trace_paths[i][frozen_path[i]].end >= 0)) tp->end = -1; } spin_unlock(&global_path_lock); - tp = &trace_paths[cpu_id][active]; + tp = &trace_paths[cpuid][active]; /* migrate last entries for pre-tracing */ __ipipe_migrate_pre_trace(tp, old_tp, pos); @@ -274,13 +278,13 @@ __ipipe_trace(enum ipipe_trace_type type int pos, next_pos, begin; struct ipipe_trace_point *point; unsigned long flags; - int cpu_id; + int cpuid; local_irq_save_hw_notrace(flags); - cpu_id = ipipe_processor_id(); + cpuid = raw_smp_processor_id(); restart: - tp = old_tp = &trace_paths[cpu_id][active_path[cpu_id]]; + tp = old_tp = &trace_paths[cpuid][active_path[cpuid]]; /* here starts a race window with NMIs - catched below */ @@ -307,7 +311,7 @@ __ipipe_trace(enum ipipe_trace_type type /* check active_path again - some nasty NMI may have switched * it meanwhile */ - if (unlikely(tp != &trace_paths[cpu_id][active_path[cpu_id]])) { + if (unlikely(tp != &trace_paths[cpuid][active_path[cpuid]])) { /* release lock on wrong path and restart */ tp->flags &= ~IPIPE_TFLG_NMI_LOCK; @@ -328,7 +332,7 @@ __ipipe_trace(enum ipipe_trace_type type point->v = v; ipipe_read_tsc(point->timestamp); - __ipipe_store_domain_states(point, cpu_id); + __ipipe_store_domain_states(point); /* forward to next point buffer */ next_pos = WRAP_POINT_NO(pos+1); @@ -346,7 +350,7 @@ __ipipe_trace(enum ipipe_trace_type type /* freeze only if the slot is free and we are not already freezing */ if (unlikely(type == IPIPE_TRACE_FREEZE) && - (trace_paths[cpu_id][frozen_path[cpu_id]].begin < 0) && + (trace_paths[cpuid][frozen_path[cpuid]].begin < 0) && !(tp->flags & IPIPE_TFLG_FREEZING)) { tp->post_trace = post_trace + 1; tp->flags |= IPIPE_TFLG_FREEZING; @@ -369,9 +373,9 @@ __ipipe_trace(enum ipipe_trace_type type enforce_end: if (tp->flags & IPIPE_TFLG_FREEZING) - tp = __ipipe_trace_freeze(cpu_id, tp, pos); + tp = __ipipe_trace_freeze(tp, pos); else - tp = __ipipe_trace_end(cpu_id, tp, pos); + tp = __ipipe_trace_end(tp, pos); /* reset the active path, maybe already start a new one */ tp->begin = (type == IPIPE_TRACE_BEGIN) ? @@ -381,7 +385,7 @@ __ipipe_trace(enum ipipe_trace_type type tp->flags = 0; /* update active_path not earlier to avoid races with NMIs */ - active_path[cpu_id] = tp - trace_paths[cpu_id]; + active_path[cpuid] = tp - trace_paths[cpuid]; } /* we still have old_tp and point, @@ -405,14 +409,14 @@ __ipipe_trace(enum ipipe_trace_type type static unsigned long __ipipe_global_path_lock(void) { unsigned long flags; - int cpu_id; struct ipipe_trace_path *tp; + int cpuid; spin_lock_irqsave(&global_path_lock, flags); - cpu_id = ipipe_processor_id(); + cpuid = raw_smp_processor_id(); restart: - tp = &trace_paths[cpu_id][active_path[cpu_id]]; + tp = &trace_paths[cpuid][active_path[cpuid]]; /* here is small race window with NMIs - catched below */ @@ -423,7 +427,7 @@ static unsigned long __ipipe_global_path /* check active_path again - some nasty NMI may have switched * it meanwhile */ - if (tp != &trace_paths[cpu_id][active_path[cpu_id]]) { + if (tp != &trace_paths[cpuid][active_path[cpuid]]) { /* release lock on wrong path and restart */ tp->flags &= ~IPIPE_TFLG_NMI_LOCK; @@ -437,14 +441,13 @@ static unsigned long __ipipe_global_path static void __ipipe_global_path_unlock(unsigned long flags) { - int cpu_id; struct ipipe_trace_path *tp; + int cpuid = raw_smp_processor_id(); /* release spinlock first - it's not involved in the NMI issue */ spin_unlock(&global_path_lock); - cpu_id = ipipe_processor_id(); - tp = &trace_paths[cpu_id][active_path[cpu_id]]; + tp = &trace_paths[cpuid][active_path[cpuid]]; tp->flags &= ~IPIPE_TFLG_NMI_LOCK; @@ -505,15 +508,15 @@ EXPORT_SYMBOL(ipipe_trace_pid); int ipipe_trace_max_reset(void) { - int cpu_id; unsigned long flags; struct ipipe_trace_path *path; + int cpuid; int ret = 0; flags = __ipipe_global_path_lock(); - for_each_online_cpu(cpu_id) { - path = &trace_paths[cpu_id][max_path[cpu_id]]; + for_each_online_cpu(cpuid) { + path = &trace_paths[cpuid][max_path[cpuid]]; if (path->dump_lock) { ret = -EBUSY; @@ -534,15 +537,15 @@ EXPORT_SYMBOL(ipipe_trace_max_reset); int ipipe_trace_frozen_reset(void) { - int cpu_id; unsigned long flags; struct ipipe_trace_path *path; + int cpuid; int ret = 0; flags = __ipipe_global_path_lock(); - for_each_online_cpu(cpu_id) { - path = &trace_paths[cpu_id][frozen_path[cpu_id]]; + for_each_online_cpu(cpuid) { + path = &trace_paths[cpuid][frozen_path[cpuid]]; if (path->dump_lock) { ret = -EBUSY; @@ -564,7 +567,7 @@ EXPORT_SYMBOL(ipipe_trace_frozen_reset); void ipipe_trace_panic_freeze(void) { unsigned long flags; - int cpu_id; + int cpuid; if (!ipipe_trace_enable) return; @@ -572,9 +575,9 @@ void ipipe_trace_panic_freeze(void) ipipe_trace_enable = 0; local_irq_save_hw_notrace(flags); - cpu_id = ipipe_processor_id(); + cpuid = raw_smp_processor_id(); - panic_path = &trace_paths[cpu_id][active_path[cpu_id]]; + panic_path = &trace_paths[cpuid][active_path[cpuid]]; local_irq_restore_hw(flags); } Index: linux-2.6.20/include/linux/smp.h =================================================================== --- linux-2.6.20.orig/include/linux/smp.h +++ linux-2.6.20/include/linux/smp.h @@ -109,8 +109,6 @@ static inline int smp_call_function_sing #endif /* !SMP */ -#include - /* * smp_processor_id(): get the current CPU ID. * @@ -130,11 +128,7 @@ static inline int smp_call_function_sing extern unsigned int debug_smp_processor_id(void); # define smp_processor_id() debug_smp_processor_id() #else -# define smp_processor_id() \ - ({ \ - ipipe_check_context(ipipe_root_domain); \ - raw_smp_processor_id(); \ - }) +# define smp_processor_id() raw_smp_processor_id() #endif #define get_cpu() ({ preempt_disable(); smp_processor_id(); }) Index: linux-2.6.20/lib/smp_processor_id.c =================================================================== --- linux-2.6.20.orig/lib/smp_processor_id.c +++ linux-2.6.20/lib/smp_processor_id.c @@ -13,7 +13,22 @@ unsigned int debug_smp_processor_id(void int this_cpu = raw_smp_processor_id(); cpumask_t this_mask; - ipipe_check_context(ipipe_root_domain); +#ifdef CONFIG_IPIPE + if (irqs_disabled_hw()) + goto out; + + if (unlikely(!ipipe_root_domain_p && !ipipe_test_pipeline())) { + ipipe_trace_panic_freeze(); + printk(KERN_ERR "BUG: using smp_processor_id() in non-atomic " + "context. I-pipe domain: %s\n", + ipipe_current_domain->name); + print_symbol("caller is %s\n", + (long)__builtin_return_address(0)); + dump_stack(); + ipipe_trace_panic_dump(); + goto out; + } +#endif /* CONFIG_IPIPE */ if (likely(preempt_count)) goto out; Index: linux-2.6.20/kernel/ipipe/Kconfig.debug =================================================================== --- linux-2.6.20.orig/kernel/ipipe/Kconfig.debug +++ linux-2.6.20/kernel/ipipe/Kconfig.debug @@ -2,9 +2,15 @@ config IPIPE_DEBUG bool "I-pipe debugging" depends on IPIPE +if IPIPE_DEBUG + +comment "HINT: Switch on DEBUG_PREEMPT in order to detect broken" + depends on !DEBUG_PREEMPT +comment "usages of smp_processor_id() also over non-root domains." + depends on !DEBUG_PREEMPT + config IPIPE_DEBUG_CONTEXT bool "Check for illicit cross-domain calls" - depends on IPIPE_DEBUG default y ---help--- Enable this feature to arm checkpoints in the kernel that @@ -14,7 +20,6 @@ config IPIPE_DEBUG_CONTEXT config IPIPE_TRACE bool "Latency tracing" - depends on IPIPE_DEBUG select FRAME_POINTER select KALLSYMS select PROC_FS @@ -78,3 +83,5 @@ config IPIPE_TRACE_ENABLE_VALUE int default 0 if !IPIPE_TRACE_ENABLE default 1 if IPIPE_TRACE_ENABLE + +endif