* [Xenomai-help] [PATCH 1/6] Refactor xnlock debugging
[not found] <47C51A34.203@domain.hid>
@ 2008-03-02 10:38 ` Jan Kiszka
2008-03-02 10:38 ` [Xenomai-help] [PATCH 2/6] Whitespace fixes for asm-generic/system.h Jan Kiszka
` (10 subsequent siblings)
11 siblings, 0 replies; 12+ messages in thread
From: Jan Kiszka @ 2008-03-02 10:38 UTC (permalink / raw)
To: xenomai-help
[-- Attachment #1.1: Type: text/plain, Size: 290 bytes --]
Concentrate all xnlock debugging related code in static inline wrappers
that can easily be switched on and off without visual harm for the lock
services themselves. This does not yet change the debugging logic, it
just saves a few byte text by avoiding redundant processor-id accesses.
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: refactor-xnlock-debugging.patch --]
[-- Type: text/x-patch; name="refactor-xnlock-debugging.patch", Size: 11043 bytes --]
---
include/asm-generic/system.h | 296 ++++++++++++++++++++++---------------------
1 file changed, 154 insertions(+), 142 deletions(-)
Index: b/include/asm-generic/system.h
===================================================================
--- a/include/asm-generic/system.h
+++ b/include/asm-generic/system.h
@@ -47,6 +47,10 @@
#define CONFIG_XENO_OPT_DEBUG_NUCLEUS 0
#endif
+#ifdef __cplusplus
+extern "C" {
+#endif
+
/* Time base export */
#define xnarch_declare_tbase(base) do { } while(0)
@@ -81,7 +85,13 @@ typedef unsigned long spl_t;
#define spltest() rthal_local_irq_test()
#define splget(x) rthal_local_irq_flags(x)
-#if defined(CONFIG_SMP) && defined(CONFIG_XENO_OPT_DEBUG)
+static inline unsigned xnarch_current_cpu(void)
+{
+ return rthal_processor_id();
+}
+
+#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+
typedef struct {
unsigned long long spin_time;
@@ -114,12 +124,96 @@ typedef struct {
0LL, \
}
-#else /* !(CONFIG_SMP && CONFIG_XENO_OPT_DEBUG) */
+#define XNLOCK_DBG_CONTEXT , __FILE__, __LINE__, __FUNCTION__
+#define XNLOCK_DBG_CONTEXT_ARGS \
+ , const char *file, int line, const char *function
+#define XNLOCK_DBG_PASS_CONTEXT , file, line, function
+#define XNLOCK_DBG_MAX_SPINS 10000000
+
+static inline void
+xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit)
+{
+ *start = rthal_rdtsc();
+ *spin_limit = XNLOCK_DBG_MAX_SPINS;
+}
+
+static inline void
+xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit,
+ const char *file, int line, const char *function)
+{
+ if (--*spin_limit == 0) {
+ rthal_emergency_console();
+ printk(KERN_ERR "Xenomai: stuck on nucleus lock %p\n"
+ " waiter = %s:%u (%s(), CPU #%d)\n"
+ " owner = %s:%u (%s(), CPU #%d)\n",
+ lock, file, line, function, cpu,
+ lock->file, lock->line, lock->function, lock->cpu);
+ show_stack(NULL, NULL);
+ }
+}
+
+static inline void
+xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start,
+ const char *file, int line, const char *function)
+{
+ lock->lock_date = *start;
+ lock->spin_time = rthal_rdtsc() - *start;
+ lock->file = file;
+ lock->function = function;
+ lock->line = line;
+ lock->cpu = cpu;
+}
+
+static inline void xnlock_dbg_release(xnlock_t *lock)
+{
+ extern xnlockinfo_t xnlock_stats[];
+ unsigned long long lock_time = rthal_rdtsc() - lock->lock_date;
+ xnlockinfo_t *stats = &xnlock_stats[xnarch_current_cpu()];
+
+ lock->cpu = -lock->cpu; /* File that we released it. */
+
+ if (lock_time > stats->lock_time) {
+ stats->lock_time = lock_time;
+ stats->spin_time = lock->spin_time;
+ stats->file = lock->file;
+ stats->function = lock->function;
+ stats->line = lock->line;
+ }
+}
+
+static inline void xnlock_dbg_invalid_release(xnlock_t *lock)
+{
+ rthal_emergency_console();
+ printk(KERN_ERR "Xenomai: unlocking unlocked nucleus lock %p\n"
+ " owner = %s:%u (%s(), CPU #%d)\n",
+ lock, lock->file, lock->line, lock->function, lock->cpu);
+ show_stack(NULL,NULL);
+}
+
+#else /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
typedef struct { atomic_t owner; } xnlock_t;
#define XNARCH_LOCK_UNLOCKED (xnlock_t) { { ~0 } }
-#endif /* !(CONFIG_SMP && CONFIG_XENO_OPT_DEBUG) */
+
+#define XNLOCK_DBG_CONTEXT
+#define XNLOCK_DBG_CONTEXT_ARGS
+#define XNLOCK_DBG_PASS_CONTEXT
+
+static inline void
+xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit)
+{ }
+
+static inline void
+xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit) { }
+
+static inline void
+xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start) { }
+
+static inline void xnlock_dbg_release(xnlock_t *lock) { }
+static inline void xnlock_dbg_invalid_release(xnlock_t *lock) { }
+
+#endif /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
#define XNARCH_NR_CPUS RTHAL_NR_CPUS
@@ -167,10 +261,6 @@ typedef struct xnarch_heapcb {
} xnarch_heapcb_t;
-#ifdef __cplusplus
-extern "C" {
-#endif
-
unsigned long long xnarch_get_host_time(void);
long long xnarch_tsc_to_ns(long long ts);
@@ -189,11 +279,6 @@ static inline unsigned long long xnarch_
return RTHAL_CPU_FREQ;
}
-static inline unsigned xnarch_current_cpu(void)
-{
- return rthal_processor_id();
-}
-
#define xnarch_halt(emsg) \
do { \
rthal_emergency_console(); \
@@ -213,17 +298,11 @@ static inline int xnarch_setimask (int i
#ifdef CONFIG_SMP
-#if XENO_DEBUG(NUCLEUS)
-#define xnlock_get(lock) \
- __xnlock_get(lock, __FILE__, __LINE__,__FUNCTION__)
+#define xnlock_get(lock) __xnlock_get(lock XNLOCK_DBG_CONTEXT)
#define xnlock_get_irqsave(lock,x) \
- ((x) = __xnlock_get_irqsave(lock, __FILE__, __LINE__,__FUNCTION__))
-#else /* !XENO_DEBUG(NUCLEUS) */
-#define xnlock_get(lock) __xnlock_get(lock)
-#define xnlock_get_irqsave(lock,x) ((x) = __xnlock_get_irqsave(lock))
-#endif /* !XENO_DEBUG(NUCLEUS) */
-#define xnlock_clear_irqoff(lock) xnlock_put_irqrestore(lock,1)
-#define xnlock_clear_irqon(lock) xnlock_put_irqrestore(lock,0)
+ ((x) = __xnlock_get_irqsave(lock XNLOCK_DBG_CONTEXT))
+#define xnlock_clear_irqoff(lock) xnlock_put_irqrestore(lock, 1)
+#define xnlock_clear_irqon(lock) xnlock_put_irqrestore(lock, 0)
static inline void xnlock_init (xnlock_t *lock)
{
@@ -235,132 +314,65 @@ static inline void xnlock_init (xnlock_t
#define DEFINE_XNLOCK(lock) xnlock_t lock = XNARCH_LOCK_UNLOCKED
#define DEFINE_PRIVATE_XNLOCK(lock) static DEFINE_XNLOCK(lock)
-#if XENO_DEBUG(NUCLEUS)
+static inline int __xnlock_get(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
+{
+ unsigned long long start;
+ unsigned int spin_limit;
+ int cpu = xnarch_current_cpu();
+
+ if (atomic_read(&lock->owner) == cpu)
+ return 1;
+
+ xnlock_dbg_prepare_acquire(&start, &spin_limit);
+
+ while (atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0)
+ do {
+ cpu_relax();
+ xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */
+ XNLOCK_DBG_PASS_CONTEXT);
+ } while(atomic_read(&lock->owner) != ~0);
+
+ xnlock_dbg_acquired(lock, cpu, &start /*, */ XNLOCK_DBG_PASS_CONTEXT);
+
+ return 0;
+}
+
+static inline void xnlock_put(xnlock_t *lock)
+{
+ if (likely(atomic_read(&lock->owner) == xnarch_current_cpu())) {
+ xnlock_dbg_release(lock);
+
+ /*
+ * Make sure all data written inside the lock is visible to
+ * other CPUs before we release the lock.
+ */
+ xnarch_memory_barrier();
+
+ atomic_set(&lock->owner, ~0);
+ } else
+ xnlock_dbg_invalid_release(lock);
+}
+
+static inline spl_t
+__xnlock_get_irqsave(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
+{
+ unsigned long flags;
+
+ rthal_local_irq_save(flags);
-#define XNARCH_DEBUG_SPIN_LIMIT 3000000
+ if (__xnlock_get(lock /*, */ XNLOCK_DBG_PASS_CONTEXT))
+ flags |= 2; /* Recursive acquisition */
-static inline int __xnlock_get (xnlock_t *lock,
- const char *file,
- unsigned line,
- const char *function)
-{
- unsigned spin_count = 0;
-#else /* !XENO_DEBUG(NUCLEUS) */
-static inline int __xnlock_get (xnlock_t *lock)
-{
-#endif /* !XENO_DEBUG(NUCLEUS) */
- int recursing;
-
- recursing = (atomic_read(&lock->owner) == rthal_processor_id());
- if (!recursing) {
-#if XENO_DEBUG(NUCLEUS)
- unsigned long long lock_date = rthal_rdtsc();
-#endif /* XENO_DEBUG(NUCLEUS) */
- while(atomic_cmpxchg(&lock->owner, ~0, rthal_processor_id()) != ~0)
- do {
- cpu_relax();
-
-#if XENO_DEBUG(NUCLEUS)
- if (++spin_count == XNARCH_DEBUG_SPIN_LIMIT) {
- rthal_emergency_console();
- printk(KERN_ERR
- "Xenomai: stuck on nucleus lock %p\n"
- " waiter = %s:%u (%s(), CPU #%d)\n"
- " owner = %s:%u (%s(), CPU #%d)\n",
- lock,file,line,function,rthal_processor_id(),
- lock->file,lock->line,lock->function,lock->cpu);
- show_stack(NULL,NULL);
- for (;;)
- cpu_relax();
- }
-#endif /* XENO_DEBUG(NUCLEUS) */
- } while(atomic_read(&lock->owner) != ~0);
-
-#if XENO_DEBUG(NUCLEUS)
- lock->spin_time = rthal_rdtsc() - lock_date;
- lock->lock_date = lock_date;
- lock->file = file;
- lock->function = function;
- lock->line = line;
- lock->cpu = rthal_processor_id();
-#endif /* XENO_DEBUG(NUCLEUS) */
- }
-
- return recursing;
-}
-
-static inline void xnlock_put (xnlock_t *lock)
-{
- if (likely(atomic_read(&lock->owner) == rthal_processor_id())) {
-
-#if XENO_DEBUG(NUCLEUS)
- extern xnlockinfo_t xnlock_stats[];
-
- unsigned long long lock_time = rthal_rdtsc() - lock->lock_date;
- int cpu = rthal_processor_id();
-
- if (lock_time > xnlock_stats[cpu].lock_time) {
- xnlock_stats[cpu].lock_time = lock_time;
- xnlock_stats[cpu].spin_time = lock->spin_time;
- xnlock_stats[cpu].file = lock->file;
- xnlock_stats[cpu].function = lock->function;
- xnlock_stats[cpu].line = lock->line;
- }
-#endif /* XENO_DEBUG(NUCLEUS) */
- /*
- * Make sure all data written inside the lock is visible to
- * other CPUs before we release the lock.
- */
- xnarch_memory_barrier();
-
- atomic_set(&lock->owner, ~0);
- }
-#if XENO_DEBUG(NUCLEUS)
- else {
- rthal_emergency_console();
- printk(KERN_ERR
- "Xenomai: unlocking unlocked nucleus lock %p\n"
- " owner = %s:%u (%s(), CPU #%d)\n",
- lock,lock->file,lock->line,lock->function,lock->cpu);
- show_stack(NULL,NULL);
- for (;;)
- cpu_relax();
- }
-#endif /* XENO_DEBUG(NUCLEUS) */
-}
-
-#if XENO_DEBUG(NUCLEUS)
-
-static inline spl_t __xnlock_get_irqsave (xnlock_t *lock,
- const char *file,
- unsigned line,
- const char *function)
-{
-#else /* !XENO_DEBUG(NUCLEUS) */
-static inline spl_t __xnlock_get_irqsave (xnlock_t *lock)
-{
-#endif /* !XENO_DEBUG(NUCLEUS) */
- unsigned long flags;
-
- rthal_local_irq_save(flags);
-
-#if XENO_DEBUG(NUCLEUS)
- if (__xnlock_get(lock, file, line, function))
- flags |= 2;
-#else /* !XENO_DEBUG(NUCLEUS) */
- if (__xnlock_get(lock))
- flags |= 2;
-#endif /* !XENO_DEBUG(NUCLEUS) */
-
- return flags;
+ return flags;
}
-static inline void xnlock_put_irqrestore (xnlock_t *lock, spl_t flags)
+static inline void xnlock_put_irqrestore(xnlock_t *lock, spl_t flags)
{
- if (!(flags & 2))
- xnlock_put(lock);
+ /* Only release the lock if we didn't take it recursively. */
+ if (!(flags & 2))
+ xnlock_put(lock);
- rthal_local_irq_restore(flags & 1);
+ rthal_local_irq_restore(flags & 1);
}
static inline int xnarch_send_ipi (xnarch_cpumask_t cpumask)
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 254 bytes --]
^ permalink raw reply [flat|nested] 12+ messages in thread* [Xenomai-help] [PATCH 2/6] Whitespace fixes for asm-generic/system.h
[not found] <47C51A34.203@domain.hid>
2008-03-02 10:38 ` [Xenomai-help] [PATCH 1/6] Refactor xnlock debugging Jan Kiszka
@ 2008-03-02 10:38 ` Jan Kiszka
2008-03-02 10:38 ` [Xenomai-help] [PATCH 3/6] Consolidate xnlock_put debugging Jan Kiszka
` (9 subsequent siblings)
11 siblings, 0 replies; 12+ messages in thread
From: Jan Kiszka @ 2008-03-02 10:38 UTC (permalink / raw)
To: xenomai-help
[-- Attachment #1.1: Type: text/plain, Size: 24 bytes --]
No functional changes.
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: system-h-whitespace-cleanups.patch --]
[-- Type: text/x-patch; name="system-h-whitespace-cleanups.patch", Size: 9800 bytes --]
---
include/asm-generic/system.h | 145 +++++++++++++++++++++----------------------
1 file changed, 73 insertions(+), 72 deletions(-)
Index: b/include/asm-generic/system.h
===================================================================
--- a/include/asm-generic/system.h
+++ b/include/asm-generic/system.h
@@ -56,13 +56,13 @@ extern "C" {
/* Tracer interface */
#define xnarch_trace_max_begin(v) rthal_trace_max_begin(v)
-#define xnarch_trace_max_end(v) rthal_trace_max_end(v)
+#define xnarch_trace_max_end(v) rthal_trace_max_end(v)
#define xnarch_trace_max_reset() rthal_trace_max_reset()
#define xnarch_trace_user_start() rthal_trace_user_start()
#define xnarch_trace_user_stop(v) rthal_trace_user_stop(v)
-#define xnarch_trace_user_freeze(v, once) rthal_trace_user_freeze(v, once)
+#define xnarch_trace_user_freeze(v, once) rthal_trace_user_freeze(v, once)
#define xnarch_trace_special(id, v) rthal_trace_special(id, v)
-#define xnarch_trace_special_u64(id, v) rthal_trace_special_u64(id, v)
+#define xnarch_trace_special_u64(id, v) rthal_trace_special_u64(id, v)
#define xnarch_trace_pid(pid, prio) rthal_trace_pid(pid, prio)
#define xnarch_trace_panic_freeze() rthal_trace_panic_freeze()
#define xnarch_trace_panic_dump() rthal_trace_panic_dump()
@@ -94,23 +94,23 @@ static inline unsigned xnarch_current_cp
typedef struct {
- unsigned long long spin_time;
- unsigned long long lock_time;
- const char *file;
- const char *function;
- unsigned line;
+ unsigned long long spin_time;
+ unsigned long long lock_time;
+ const char *file;
+ const char *function;
+ unsigned line;
} xnlockinfo_t;
typedef struct {
- atomic_t owner;
- const char *file;
- const char *function;
- unsigned line;
- int cpu;
- unsigned long long spin_time;
- unsigned long long lock_date;
+ atomic_t owner;
+ const char *file;
+ const char *function;
+ unsigned line;
+ int cpu;
+ unsigned long long spin_time;
+ unsigned long long lock_date;
} xnlock_t;
@@ -194,7 +194,7 @@ static inline void xnlock_dbg_invalid_re
typedef struct { atomic_t owner; } xnlock_t;
-#define XNARCH_LOCK_UNLOCKED (xnlock_t) { { ~0 } }
+#define XNARCH_LOCK_UNLOCKED (xnlock_t) { { ~0 } }
#define XNLOCK_DBG_CONTEXT
#define XNLOCK_DBG_CONTEXT_ARGS
@@ -215,49 +215,48 @@ static inline void xnlock_dbg_invalid_re
#endif /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
-#define XNARCH_NR_CPUS RTHAL_NR_CPUS
+#define XNARCH_NR_CPUS RTHAL_NR_CPUS
-#define XNARCH_NR_IRQS RTHAL_NR_IRQS
-#define XNARCH_TIMER_IRQ RTHAL_TIMER_IRQ
-#define XNARCH_TIMER_DEVICE RTHAL_TIMER_DEVICE
-#define XNARCH_CLOCK_DEVICE RTHAL_CLOCK_DEVICE
+#define XNARCH_NR_IRQS RTHAL_NR_IRQS
+#define XNARCH_TIMER_IRQ RTHAL_TIMER_IRQ
+#define XNARCH_TIMER_DEVICE RTHAL_TIMER_DEVICE
+#define XNARCH_CLOCK_DEVICE RTHAL_CLOCK_DEVICE
-#define XNARCH_ROOT_STACKSZ 0 /* Only a placeholder -- no stack */
+#define XNARCH_ROOT_STACKSZ 0 /* Only a placeholder -- no stack */
#define XNARCH_PROMPT "Xenomai: "
-#define xnarch_loginfo(fmt,args...) printk(KERN_INFO XNARCH_PROMPT fmt , ##args)
-#define xnarch_logwarn(fmt,args...) printk(KERN_WARNING XNARCH_PROMPT fmt , ##args)
-#define xnarch_logerr(fmt,args...) printk(KERN_ERR XNARCH_PROMPT fmt , ##args)
-#define xnarch_printf(fmt,args...) printk(KERN_INFO XNARCH_PROMPT fmt , ##args)
+#define xnarch_loginfo(fmt, args...) printk(KERN_INFO XNARCH_PROMPT fmt, ##args)
+#define xnarch_logwarn(fmt, args...) printk(KERN_WARNING XNARCH_PROMPT fmt, ##args)
+#define xnarch_logerr(fmt, args...) printk(KERN_ERR XNARCH_PROMPT fmt, ##args)
+#define xnarch_printf(fmt, args...) printk(KERN_INFO XNARCH_PROMPT fmt, ##args)
typedef cpumask_t xnarch_cpumask_t;
#ifdef CONFIG_SMP
-#define xnarch_cpu_online_map cpu_online_map
+#define xnarch_cpu_online_map cpu_online_map
#else
-#define xnarch_cpu_online_map cpumask_of_cpu(0)
+#define xnarch_cpu_online_map cpumask_of_cpu(0)
#endif
-#define xnarch_num_online_cpus() num_online_cpus()
-#define xnarch_cpu_set(cpu, mask) cpu_set(cpu, (mask))
-#define xnarch_cpu_clear(cpu, mask) cpu_clear(cpu, (mask))
-#define xnarch_cpus_clear(mask) cpus_clear(mask)
-#define xnarch_cpu_isset(cpu, mask) cpu_isset(cpu, (mask))
-#define xnarch_cpus_and(dst, src1, src2) cpus_and((dst), (src1), (src2))
-#define xnarch_cpus_equal(mask1, mask2) cpus_equal((mask1), (mask2))
-#define xnarch_cpus_empty(mask) cpus_empty(mask)
-#define xnarch_cpumask_of_cpu(cpu) cpumask_of_cpu(cpu)
-#define xnarch_cpu_test_and_set(cpu,mask) cpu_test_and_set(cpu, (mask))
-
-#define xnarch_first_cpu(mask) first_cpu(mask)
-#define XNARCH_CPU_MASK_ALL CPU_MASK_ALL
+#define xnarch_num_online_cpus() num_online_cpus()
+#define xnarch_cpu_set(cpu, mask) cpu_set(cpu, (mask))
+#define xnarch_cpu_clear(cpu, mask) cpu_clear(cpu, (mask))
+#define xnarch_cpus_clear(mask) cpus_clear(mask)
+#define xnarch_cpu_isset(cpu, mask) cpu_isset(cpu, (mask))
+#define xnarch_cpus_and(dst, src1, src2) cpus_and((dst), (src1), (src2))
+#define xnarch_cpus_equal(mask1, mask2) cpus_equal((mask1), (mask2))
+#define xnarch_cpus_empty(mask) cpus_empty(mask)
+#define xnarch_cpumask_of_cpu(cpu) cpumask_of_cpu(cpu)
+#define xnarch_cpu_test_and_set(cpu, mask) cpu_test_and_set(cpu, (mask))
+#define xnarch_first_cpu(mask) first_cpu(mask)
+#define XNARCH_CPU_MASK_ALL CPU_MASK_ALL
typedef struct xnarch_heapcb {
- atomic_t numaps; /* # of active user-space mappings. */
+ atomic_t numaps; /* # of active user-space mappings. */
- int kmflags; /* Kernel memory flags (0 if vmalloc()). */
+ int kmflags; /* Kernel memory flags (0 if vmalloc()). */
- void *heapbase; /* Shared heap memory base. */
+ void *heapbase; /* Shared heap memory base. */
} xnarch_heapcb_t;
@@ -267,7 +266,7 @@ long long xnarch_tsc_to_ns(long long ts)
static inline long long xnarch_tsc_to_ns_rounded(long long ts)
{
- return (xnarch_llimd(ts, 1000000000, RTHAL_CPU_FREQ/2) + 1) / 2;
+ return (xnarch_llimd(ts, 1000000000, RTHAL_CPU_FREQ/2) + 1) / 2;
}
long long xnarch_ns_to_tsc(long long ns);
@@ -276,24 +275,26 @@ unsigned long long xnarch_get_cpu_time(v
static inline unsigned long long xnarch_get_cpu_freq(void)
{
- return RTHAL_CPU_FREQ;
+ return RTHAL_CPU_FREQ;
}
-#define xnarch_halt(emsg) \
-do { \
- rthal_emergency_console(); \
- xnarch_logerr("fatal: %s\n",emsg); \
- show_stack(NULL,NULL); \
- xnarch_trace_panic_dump(); \
- for (;;) cpu_relax(); \
-} while(0)
+#define xnarch_halt(emsg) \
+ do { \
+ rthal_emergency_console(); \
+ xnarch_logerr("fatal: %s\n", emsg); \
+ show_stack(NULL,NULL); \
+ xnarch_trace_panic_dump(); \
+ for (;;) \
+ cpu_relax(); \
+ } while(0)
static inline int xnarch_setimask (int imask)
{
- spl_t s;
- splhigh(s);
- splexit(!!imask);
- return !!s;
+ spl_t s;
+
+ splhigh(s);
+ splexit(!!imask);
+ return !!s;
}
#ifdef CONFIG_SMP
@@ -306,7 +307,7 @@ static inline int xnarch_setimask (int i
static inline void xnlock_init (xnlock_t *lock)
{
- *lock = XNARCH_LOCK_UNLOCKED;
+ *lock = XNARCH_LOCK_UNLOCKED;
}
#define DECLARE_XNLOCK(lock) xnlock_t lock
@@ -375,9 +376,9 @@ static inline void xnlock_put_irqrestore
rthal_local_irq_restore(flags & 1);
}
-static inline int xnarch_send_ipi (xnarch_cpumask_t cpumask)
+static inline int xnarch_send_ipi(xnarch_cpumask_t cpumask)
{
- return rthal_send_ipi(RTHAL_SERVICE_IPI0, cpumask);
+ return rthal_send_ipi(RTHAL_SERVICE_IPI0, cpumask);
}
static inline int xnlock_is_owner(xnlock_t *lock)
@@ -387,14 +388,14 @@ static inline int xnlock_is_owner(xnlock
#else /* !CONFIG_SMP */
-#define xnlock_init(lock) do { } while(0)
-#define xnlock_get(lock) do { } while(0)
-#define xnlock_put(lock) do { } while(0)
-#define xnlock_get_irqsave(lock,x) rthal_local_irq_save(x)
-#define xnlock_put_irqrestore(lock,x) rthal_local_irq_restore(x)
-#define xnlock_clear_irqoff(lock) rthal_local_irq_disable()
-#define xnlock_clear_irqon(lock) rthal_local_irq_enable()
-#define xnlock_is_owner(lock) 1
+#define xnlock_init(lock) do { } while(0)
+#define xnlock_get(lock) do { } while(0)
+#define xnlock_put(lock) do { } while(0)
+#define xnlock_get_irqsave(lock,x) rthal_local_irq_save(x)
+#define xnlock_put_irqrestore(lock,x) rthal_local_irq_restore(x)
+#define xnlock_clear_irqoff(lock) rthal_local_irq_disable()
+#define xnlock_clear_irqon(lock) rthal_local_irq_enable()
+#define xnlock_is_owner(lock) 1
#define DECLARE_XNLOCK(lock)
#define DECLARE_EXTERN_XNLOCK(lock)
@@ -403,7 +404,7 @@ static inline int xnlock_is_owner(xnlock
static inline int xnarch_send_ipi (xnarch_cpumask_t cpumask)
{
- return 0;
+ return 0;
}
#endif /* !CONFIG_SMP */
@@ -418,7 +419,7 @@ static inline int xnarch_remap_vm_page(s
unsigned long from,
unsigned long to)
{
- return wrap_remap_vm_page(vma,from,to);
+ return wrap_remap_vm_page(vma, from, to);
}
static inline int xnarch_remap_io_page_range(struct vm_area_struct *vma,
@@ -427,7 +428,7 @@ static inline int xnarch_remap_io_page_r
unsigned long size,
pgprot_t prot)
{
- return wrap_remap_io_page_range(vma,from,to,size,prot);
+ return wrap_remap_io_page_range(vma, from, to, size, prot);
}
#ifndef xnarch_hisyscall_entry
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 254 bytes --]
^ permalink raw reply [flat|nested] 12+ messages in thread* [Xenomai-help] [PATCH 3/6] Consolidate xnlock_put debugging
[not found] <47C51A34.203@domain.hid>
2008-03-02 10:38 ` [Xenomai-help] [PATCH 1/6] Refactor xnlock debugging Jan Kiszka
2008-03-02 10:38 ` [Xenomai-help] [PATCH 2/6] Whitespace fixes for asm-generic/system.h Jan Kiszka
@ 2008-03-02 10:38 ` Jan Kiszka
2008-03-02 10:38 ` [Xenomai-help] [PATCH 4/6] Move spinning code out-of-line Jan Kiszka
` (8 subsequent siblings)
11 siblings, 0 replies; 12+ messages in thread
From: Jan Kiszka @ 2008-03-02 10:38 UTC (permalink / raw)
To: xenomai-help
[-- Attachment #1.1: Type: text/plain, Size: 424 bytes --]
The owner check on xnlock release looks light-weight in first sight, but
it isn't. It requires a processor-id lookup + it adds a conditional to
the otherwise unconditional release path. Moreover, this code is inlined
into every xnlock_put user, thus increases the text size of hot paths.
This patch therefore moves the check under the same switch
(XENO_OPT_DEBUG_NUCLEUS) as the rest of xnlock debugging already is.
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: consolidate-xnlock_put-debugging.patch --]
[-- Type: text/x-patch; name="consolidate-xnlock_put-debugging.patch", Size: 2866 bytes --]
---
include/asm-generic/system.h | 52 +++++++++++++++++++++++--------------------
1 file changed, 28 insertions(+), 24 deletions(-)
Index: b/include/asm-generic/system.h
===================================================================
--- a/include/asm-generic/system.h
+++ b/include/asm-generic/system.h
@@ -164,11 +164,23 @@ xnlock_dbg_acquired(xnlock_t *lock, int
lock->cpu = cpu;
}
-static inline void xnlock_dbg_release(xnlock_t *lock)
+static inline int xnlock_dbg_release(xnlock_t *lock)
{
extern xnlockinfo_t xnlock_stats[];
unsigned long long lock_time = rthal_rdtsc() - lock->lock_date;
- xnlockinfo_t *stats = &xnlock_stats[xnarch_current_cpu()];
+ int cpu = xnarch_current_cpu();
+ xnlockinfo_t *stats = &xnlock_stats[cpu];
+
+ if (unlikely(atomic_read(&lock->owner) != cpu)) {
+ rthal_emergency_console();
+ printk(KERN_ERR "Xenomai: unlocking unlocked nucleus lock %p"
+ " on CPU #%d\n"
+ " owner = %s:%u (%s(), CPU #%d)\n",
+ lock, cpu, lock->file, lock->line, lock->function,
+ lock->cpu);
+ show_stack(NULL,NULL);
+ return 1;
+ }
lock->cpu = -lock->cpu; /* File that we released it. */
@@ -179,15 +191,7 @@ static inline void xnlock_dbg_release(xn
stats->function = lock->function;
stats->line = lock->line;
}
-}
-
-static inline void xnlock_dbg_invalid_release(xnlock_t *lock)
-{
- rthal_emergency_console();
- printk(KERN_ERR "Xenomai: unlocking unlocked nucleus lock %p\n"
- " owner = %s:%u (%s(), CPU #%d)\n",
- lock, lock->file, lock->line, lock->function, lock->cpu);
- show_stack(NULL,NULL);
+ return 0;
}
#else /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
@@ -210,8 +214,10 @@ xnlock_dbg_spinning(xnlock_t *lock, int
static inline void
xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start) { }
-static inline void xnlock_dbg_release(xnlock_t *lock) { }
-static inline void xnlock_dbg_invalid_release(xnlock_t *lock) { }
+static inline int xnlock_dbg_release(xnlock_t *lock)
+{
+ return 0;
+}
#endif /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
@@ -340,18 +346,16 @@ static inline int __xnlock_get(xnlock_t
static inline void xnlock_put(xnlock_t *lock)
{
- if (likely(atomic_read(&lock->owner) == xnarch_current_cpu())) {
- xnlock_dbg_release(lock);
+ if (xnlock_dbg_release(lock))
+ return;
+
+ /*
+ * Make sure all data written inside the lock is visible to
+ * other CPUs before we release the lock.
+ */
+ xnarch_memory_barrier();
- /*
- * Make sure all data written inside the lock is visible to
- * other CPUs before we release the lock.
- */
- xnarch_memory_barrier();
-
- atomic_set(&lock->owner, ~0);
- } else
- xnlock_dbg_invalid_release(lock);
+ atomic_set(&lock->owner, ~0);
}
static inline spl_t
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 254 bytes --]
^ permalink raw reply [flat|nested] 12+ messages in thread* [Xenomai-help] [PATCH 4/6] Move spinning code out-of-line
[not found] <47C51A34.203@domain.hid>
` (2 preceding siblings ...)
2008-03-02 10:38 ` [Xenomai-help] [PATCH 3/6] Consolidate xnlock_put debugging Jan Kiszka
@ 2008-03-02 10:38 ` Jan Kiszka
2008-03-02 10:38 ` [Xenomai-help] [PATCH 5/6] Introduce optional recursive ticket xnlock Jan Kiszka
` (7 subsequent siblings)
11 siblings, 0 replies; 12+ messages in thread
From: Jan Kiszka @ 2008-03-02 10:38 UTC (permalink / raw)
To: xenomai-help
[-- Attachment #1.1: Type: text/plain, Size: 130 bytes --]
To save text thus and thus avoid cache misses in the worst-case path,
this patch moves the slow path of xnlock_get out-of-line.
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: introduce-xnlock_spin.patch --]
[-- Type: text/x-patch; name="introduce-xnlock_spin.patch", Size: 3499 bytes --]
---
include/asm-generic/bits/pod.h | 18 ++++++++++++++++++
include/asm-generic/system.h | 29 ++++++++++++++---------------
2 files changed, 32 insertions(+), 15 deletions(-)
Index: b/include/asm-generic/bits/pod.h
===================================================================
--- a/include/asm-generic/bits/pod.h
+++ b/include/asm-generic/bits/pod.h
@@ -295,4 +295,22 @@ unsigned long long xnarch_get_cpu_time(v
EXPORT_SYMBOL(xnarch_get_cpu_time);
+#ifdef CONFIG_SMP
+void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
+{
+ unsigned int spin_limit;
+ int cpu = xnarch_current_cpu();
+
+ xnlock_dbg_prepare_spin(&spin_limit);
+
+ while (atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0)
+ do {
+ cpu_relax();
+ xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */
+ XNLOCK_DBG_PASS_CONTEXT);
+ } while(atomic_read(&lock->owner) != ~0);
+}
+EXPORT_SYMBOL(__xnlock_spin);
+#endif /* CONFIG_SMP */
+
#endif /* !_XENO_ASM_GENERIC_BITS_POD_H */
Index: b/include/asm-generic/system.h
===================================================================
--- a/include/asm-generic/system.h
+++ b/include/asm-generic/system.h
@@ -130,10 +130,13 @@ typedef struct {
#define XNLOCK_DBG_PASS_CONTEXT , file, line, function
#define XNLOCK_DBG_MAX_SPINS 10000000
-static inline void
-xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit)
+static inline void xnlock_dbg_prepare_acquire(unsigned long long *start)
{
*start = rthal_rdtsc();
+}
+
+static inline void xnlock_dbg_prepare_spin(unsigned *spin_limit)
+{
*spin_limit = XNLOCK_DBG_MAX_SPINS;
}
@@ -204,15 +207,14 @@ typedef struct { atomic_t owner; } xnloc
#define XNLOCK_DBG_CONTEXT_ARGS
#define XNLOCK_DBG_PASS_CONTEXT
-static inline void
-xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit)
-{ }
+static inline void xnlock_dbg_prepare_acquire(unsigned long long *start) { }
+static inline void xnlock_dbg_prepare_spin(unsigned *spin_limit) { }
static inline void
-xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit) { }
+xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit) { }
static inline void
-xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start) { }
+xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start) { }
static inline int xnlock_dbg_release(xnlock_t *lock)
{
@@ -321,23 +323,20 @@ static inline void xnlock_init (xnlock_t
#define DEFINE_XNLOCK(lock) xnlock_t lock = XNARCH_LOCK_UNLOCKED
#define DEFINE_PRIVATE_XNLOCK(lock) static DEFINE_XNLOCK(lock)
+void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS);
+
static inline int __xnlock_get(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
{
unsigned long long start;
- unsigned int spin_limit;
int cpu = xnarch_current_cpu();
if (atomic_read(&lock->owner) == cpu)
return 1;
- xnlock_dbg_prepare_acquire(&start, &spin_limit);
+ xnlock_dbg_prepare_acquire(&start);
- while (atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0)
- do {
- cpu_relax();
- xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */
- XNLOCK_DBG_PASS_CONTEXT);
- } while(atomic_read(&lock->owner) != ~0);
+ if (unlikely(atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0))
+ __xnlock_spin(lock /*, */ XNLOCK_DBG_PASS_CONTEXT);
xnlock_dbg_acquired(lock, cpu, &start /*, */ XNLOCK_DBG_PASS_CONTEXT);
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 254 bytes --]
^ permalink raw reply [flat|nested] 12+ messages in thread* [Xenomai-help] [PATCH 5/6] Introduce optional recursive ticket xnlock
[not found] <47C51A34.203@domain.hid>
` (3 preceding siblings ...)
2008-03-02 10:38 ` [Xenomai-help] [PATCH 4/6] Move spinning code out-of-line Jan Kiszka
@ 2008-03-02 10:38 ` Jan Kiszka
2008-03-02 10:38 ` [Xenomai-help] [PATCH 6/6] Allow xnlock debugging on single-CPU systems Jan Kiszka
` (6 subsequent siblings)
11 siblings, 0 replies; 12+ messages in thread
From: Jan Kiszka @ 2008-03-02 10:38 UTC (permalink / raw)
To: xenomai-help
[-- Attachment #1.1: Type: text/plain, Size: 976 bytes --]
The origin of this series: When Nick Piggin posted his first suggestion
for ticket spinlocks on LKML, I immediately liked the idea. For details
check LWN [1], in a nutshell: This algorithm enforces strict FIFO order
for the admission to contended spinlocks, thus it improves the
determinism on SMP systems with more than 2 CPUs.
Meanwhile, ticket spinlocks are mainline (2.6.25). But that version has
to drawbacks for us: it doesn't support nesting like xnlock does, and it
is x86-only so far.
So I designed a version for Xenomai which is both nestable and
arch-independent. It is certainly not as optimal as mainline's version,
but our code path stresses the locking code differently anyway, and no
one denies us to introduce per-arch optimized variants later on (if this
lock implementation gains broader relevance).
Survived heavy testing on a 4-way box (15, 20, 25, and 30 us latency
instances across the cores).
[1] http://lwn.net/Articles/267331
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: ticket-xnlock-v2.patch --]
[-- Type: text/x-patch; name="ticket-xnlock-v2.patch", Size: 7144 bytes --]
---
include/asm-generic/bits/pod.h | 4 -
include/asm-generic/system.h | 122 +++++++++++++++++++++++++++++++++++++----
ksrc/nucleus/Kconfig | 12 ++++
3 files changed, 126 insertions(+), 12 deletions(-)
Index: b/include/asm-generic/system.h
===================================================================
--- a/include/asm-generic/system.h
+++ b/include/asm-generic/system.h
@@ -104,7 +104,12 @@ typedef struct {
typedef struct {
+#ifdef CONFIG_XENO_OPT_TICKET_LOCK
+ atomic_t fifo;
+ int owner;
+#else
atomic_t owner;
+#endif
const char *file;
const char *function;
unsigned line;
@@ -115,7 +120,7 @@ typedef struct {
} xnlock_t;
#define XNARCH_LOCK_UNLOCKED (xnlock_t) { \
- { ~0 }, \
+ XNLOCK_INIT_VALUE, \
NULL, \
NULL, \
0, \
@@ -167,6 +172,8 @@ xnlock_dbg_acquired(xnlock_t *lock, int
lock->cpu = cpu;
}
+static inline int xnlock_owner(xnlock_t *lock);
+
static inline int xnlock_dbg_release(xnlock_t *lock)
{
extern xnlockinfo_t xnlock_stats[];
@@ -174,7 +181,7 @@ static inline int xnlock_dbg_release(xnl
int cpu = xnarch_current_cpu();
xnlockinfo_t *stats = &xnlock_stats[cpu];
- if (unlikely(atomic_read(&lock->owner) != cpu)) {
+ if (unlikely(xnlock_owner(lock) != cpu)) {
rthal_emergency_console();
printk(KERN_ERR "Xenomai: unlocking unlocked nucleus lock %p"
" on CPU #%d\n"
@@ -199,9 +206,18 @@ static inline int xnlock_dbg_release(xnl
#else /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
+#ifdef CONFIG_XENO_OPT_TICKET_LOCK
+typedef struct {
+
+ atomic_t fifo;
+ int owner;
+
+} xnlock_t;
+#else
typedef struct { atomic_t owner; } xnlock_t;
+#endif
-#define XNARCH_LOCK_UNLOCKED (xnlock_t) { { ~0 } }
+#define XNARCH_LOCK_UNLOCKED (xnlock_t) { XNLOCK_INIT_VALUE }
#define XNLOCK_DBG_CONTEXT
#define XNLOCK_DBG_CONTEXT_ARGS
@@ -313,11 +329,6 @@ static inline int xnarch_setimask (int i
#define xnlock_clear_irqoff(lock) xnlock_put_irqrestore(lock, 1)
#define xnlock_clear_irqon(lock) xnlock_put_irqrestore(lock, 0)
-static inline void xnlock_init (xnlock_t *lock)
-{
- *lock = XNARCH_LOCK_UNLOCKED;
-}
-
#define DECLARE_XNLOCK(lock) xnlock_t lock
#define DECLARE_EXTERN_XNLOCK(lock) extern xnlock_t lock
#define DEFINE_XNLOCK(lock) xnlock_t lock = XNARCH_LOCK_UNLOCKED
@@ -325,12 +336,97 @@ static inline void xnlock_init (xnlock_t
void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS);
+#ifdef CONFIG_XENO_OPT_TICKET_LOCK
+/* Note: We assume atomic_t is at least 32-bit wide. */
+#define XNLOCK_HEAD_MASK 0x000000FF
+#define XNLOCK_HEAD_INC 0x00000001
+#define XNLOCK_TAIL_MASK 0xFF000000
+#define XNLOCK_TAIL_SHIFT 24
+#define XNLOCK_TAIL_INC 0x01000000
+#define XNLOCK_TICKET_MASK (XNLOCK_HEAD_MASK | XNLOCK_TAIL_MASK)
+#define XNLOCK_INIT_VALUE { 0x00000001 }, ~0
+
+static inline int xnlock_owner(xnlock_t *lock)
+{
+ return lock->owner;
+}
+
+static inline int __xnlock_get(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
+{
+ unsigned long long start;
+ int cpu = xnarch_current_cpu();
+ u32 fifo;
+ u8 ticket;
+
+ if (xnlock_owner(lock) == cpu)
+ return 1;
+
+ xnlock_dbg_prepare_acquire(&start);
+
+ fifo = atomic_add_return(XNLOCK_TAIL_INC, &lock->fifo);
+ ticket = fifo >> XNLOCK_TAIL_SHIFT;
+
+ /* First check if we are already the owner (head == tail). */
+ if (unlikely((u8)fifo != ticket)) {
+ unsigned int spin_limit;
+
+ xnlock_dbg_prepare_spin(&spin_limit);
+
+ /*
+ * Spin until head equals our ticket. In this case,
+ * moving out-of-line does not pay off.
+ */
+ do {
+ cpu_relax();
+ xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */
+ XNLOCK_DBG_PASS_CONTEXT);
+ } while ((u8)atomic_read(&lock->fifo) != ticket);
+ }
+ lock->owner = cpu;
+
+ xnlock_dbg_acquired(lock, cpu, &start /*, */ XNLOCK_DBG_PASS_CONTEXT);
+
+ return 0;
+}
+
+static inline void xnlock_put(xnlock_t *lock)
+{
+ u32 fifo = atomic_read(&lock->fifo);
+
+ if (xnlock_dbg_release(lock))
+ return;
+
+ lock->owner = ~0;
+
+ /*
+ * Make sure all data written inside the lock is visible to
+ * other CPUs before we release the lock.
+ */
+ xnarch_memory_barrier();
+
+ /*
+ * Carefully calculate the addend so that we confine the
+ * XNLOCK_HEAD_INC just to the first byte of fifo.
+ */
+ atomic_add(((fifo + XNLOCK_HEAD_INC) & XNLOCK_TICKET_MASK) - fifo,
+ &lock->fifo);
+}
+
+#else /* ! CONFIG_XENO_OPT_TICKET_LOCK */
+
+#define XNLOCK_INIT_VALUE { ~0 }
+
+static inline int xnlock_owner(xnlock_t *lock)
+{
+ return atomic_read(&lock->owner);
+}
+
static inline int __xnlock_get(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
{
unsigned long long start;
int cpu = xnarch_current_cpu();
- if (atomic_read(&lock->owner) == cpu)
+ if (xnlock_owner(lock) == cpu)
return 1;
xnlock_dbg_prepare_acquire(&start);
@@ -356,6 +452,12 @@ static inline void xnlock_put(xnlock_t *
atomic_set(&lock->owner, ~0);
}
+#endif /* ! CONFIG_XENO_OPT_TICKET_LOCK */
+
+static inline void xnlock_init(xnlock_t *lock)
+{
+ *lock = XNARCH_LOCK_UNLOCKED;
+}
static inline spl_t
__xnlock_get_irqsave(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
@@ -386,7 +488,7 @@ static inline int xnarch_send_ipi(xnarch
static inline int xnlock_is_owner(xnlock_t *lock)
{
- return atomic_read(&lock->owner) == xnarch_current_cpu();
+ return xnlock_owner(lock) == xnarch_current_cpu();
}
#else /* !CONFIG_SMP */
Index: b/ksrc/nucleus/Kconfig
===================================================================
--- a/ksrc/nucleus/Kconfig
+++ b/ksrc/nucleus/Kconfig
@@ -340,6 +340,18 @@ config XENO_OPT_TIMER_WHEEL_STEP
Set the duration in ns of a timer wheel step. At each step,
the timer wheel use the next hash bucket.
+config XENO_OPT_TICKET_LOCK
+ bool "FIFO ticket spinlocks"
+ depends on SMP
+ help
+
+ Uses a ticket spinlock algorithm which enforces strict FIFO
+ admission order on contended locks, improving the determinism
+ of lock acquisitions on large SMP systems. The ticket spinlock
+ has a higher overhead than the unordered algorithm and should
+ therefore only be used if there are more than two CPUs, thus
+ more than one waiter in the worst case.
+
endmenu
endif
Index: b/include/asm-generic/bits/pod.h
===================================================================
--- a/include/asm-generic/bits/pod.h
+++ b/include/asm-generic/bits/pod.h
@@ -295,7 +295,7 @@ unsigned long long xnarch_get_cpu_time(v
EXPORT_SYMBOL(xnarch_get_cpu_time);
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) && !defined(CONFIG_XENO_OPT_TICKET_LOCK)
void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
{
unsigned int spin_limit;
@@ -311,6 +311,6 @@ void __xnlock_spin(xnlock_t *lock /*, */
} while(atomic_read(&lock->owner) != ~0);
}
EXPORT_SYMBOL(__xnlock_spin);
-#endif /* CONFIG_SMP */
+#endif /* CONFIG_SMP && !CONFIG_XENO_OPT_TICKET_LOCK */
#endif /* !_XENO_ASM_GENERIC_BITS_POD_H */
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 254 bytes --]
^ permalink raw reply [flat|nested] 12+ messages in thread* [Xenomai-help] [PATCH 6/6] Allow xnlock debugging on single-CPU systems
[not found] <47C51A34.203@domain.hid>
` (4 preceding siblings ...)
2008-03-02 10:38 ` [Xenomai-help] [PATCH 5/6] Introduce optional recursive ticket xnlock Jan Kiszka
@ 2008-03-02 10:38 ` Jan Kiszka
2008-03-02 11:02 ` [Xenomai-core] [PATCH 1/6] Refactor xnlock debugging Jan Kiszka
` (5 subsequent siblings)
11 siblings, 0 replies; 12+ messages in thread
From: Jan Kiszka @ 2008-03-02 10:38 UTC (permalink / raw)
To: xenomai-help
[-- Attachment #1.1: Type: text/plain, Size: 105 bytes --]
After all the refactoring, arming xnlock_dbg services also for
!CONFIG_SMP setups was straightforward.
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: xnlock-debugging-for-up.patch --]
[-- Type: text/x-patch; name="xnlock-debugging-for-up.patch", Size: 7137 bytes --]
---
include/asm-generic/bits/pod.h | 4 ++--
include/asm-generic/system.h | 31 +++++++++++++++----------------
ksrc/nucleus/intr.c | 4 ++--
ksrc/nucleus/module.c | 12 ++++++------
ksrc/nucleus/pod.c | 4 ++--
ksrc/skins/posix/signal.c | 2 +-
6 files changed, 28 insertions(+), 29 deletions(-)
Index: b/include/asm-generic/bits/pod.h
===================================================================
--- a/include/asm-generic/bits/pod.h
+++ b/include/asm-generic/bits/pod.h
@@ -295,7 +295,7 @@ unsigned long long xnarch_get_cpu_time(v
EXPORT_SYMBOL(xnarch_get_cpu_time);
-#if defined(CONFIG_SMP) && !defined(CONFIG_XENO_OPT_TICKET_LOCK)
+#if (defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)) && !defined(CONFIG_XENO_OPT_TICKET_LOCK)
void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
{
unsigned int spin_limit;
@@ -311,6 +311,6 @@ void __xnlock_spin(xnlock_t *lock /*, */
} while(atomic_read(&lock->owner) != ~0);
}
EXPORT_SYMBOL(__xnlock_spin);
-#endif /* CONFIG_SMP && !CONFIG_XENO_OPT_TICKET_LOCK */
+#endif /* (CONFIG_SMP || XENO_DEBUG(NUCLEUS)) && !CONFIG_XENO_OPT_TICKET_LOCK */
#endif /* !_XENO_ASM_GENERIC_BITS_POD_H */
Index: b/include/asm-generic/system.h
===================================================================
--- a/include/asm-generic/system.h
+++ b/include/asm-generic/system.h
@@ -76,11 +76,11 @@ extern "C" {
typedef unsigned long spl_t;
#define splhigh(x) rthal_local_irq_save(x)
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
#define splexit(x) rthal_local_irq_restore((x) & 1)
-#else /* !CONFIG_SMP */
+#else /* !CONFIG_SMP || XENO_DEBUG(NUCLEUS) */
#define splexit(x) rthal_local_irq_restore(x)
-#endif /* !CONFIG_SMP */
+#endif /* !CONFIG_SMP || XENO_DEBUG(NUCLEUS) */
#define splnone() rthal_local_irq_enable()
#define spltest() rthal_local_irq_test()
#define splget(x) rthal_local_irq_flags(x)
@@ -90,7 +90,7 @@ static inline unsigned xnarch_current_cp
return rthal_processor_id();
}
-#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+#if XENO_DEBUG(NUCLEUS)
typedef struct {
@@ -204,7 +204,7 @@ static inline int xnlock_dbg_release(xnl
return 0;
}
-#else /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
+#else /* !XENO_DEBUG(NUCLEUS) */
#ifdef CONFIG_XENO_OPT_TICKET_LOCK
typedef struct {
@@ -237,7 +237,7 @@ static inline int xnlock_dbg_release(xnl
return 0;
}
-#endif /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
+#endif /* !XENO_DEBUG(NUCLEUS) */
#define XNARCH_NR_CPUS RTHAL_NR_CPUS
@@ -321,7 +321,7 @@ static inline int xnarch_setimask (int i
return !!s;
}
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
#define xnlock_get(lock) __xnlock_get(lock XNLOCK_DBG_CONTEXT)
#define xnlock_get_irqsave(lock,x) \
@@ -481,17 +481,12 @@ static inline void xnlock_put_irqrestore
rthal_local_irq_restore(flags & 1);
}
-static inline int xnarch_send_ipi(xnarch_cpumask_t cpumask)
-{
- return rthal_send_ipi(RTHAL_SERVICE_IPI0, cpumask);
-}
-
static inline int xnlock_is_owner(xnlock_t *lock)
{
return xnlock_owner(lock) == xnarch_current_cpu();
}
-#else /* !CONFIG_SMP */
+#else /* !(CONFIG_SMP || XENO_DEBUG(NUCLEUS)) */
#define xnlock_init(lock) do { } while(0)
#define xnlock_get(lock) do { } while(0)
@@ -507,12 +502,16 @@ static inline int xnlock_is_owner(xnlock
#define DEFINE_XNLOCK(lock)
#define DEFINE_PRIVATE_XNLOCK(lock)
-static inline int xnarch_send_ipi (xnarch_cpumask_t cpumask)
+#endif /* !(CONFIG_SMP || XENO_DEBUG(NUCLEUS)) */
+
+static inline int xnarch_send_ipi(xnarch_cpumask_t cpumask)
{
+#ifdef CONFIG_SMP
+ return rthal_send_ipi(RTHAL_SERVICE_IPI0, cpumask);
+#else /* !CONFIG_SMP */
return 0;
-}
-
#endif /* !CONFIG_SMP */
+}
#define xnlock_sync_irq(lock, x) \
do { \
Index: b/ksrc/nucleus/intr.c
===================================================================
--- a/ksrc/nucleus/intr.c
+++ b/ksrc/nucleus/intr.c
@@ -387,7 +387,7 @@ static inline int xnintr_irq_detach(xnin
#else /* !CONFIG_XENO_OPT_SHIRQ */
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
typedef struct xnintr_irq {
DECLARE_XNLOCK(lock);
@@ -395,7 +395,7 @@ typedef struct xnintr_irq {
} ____cacheline_aligned_in_smp xnintr_irq_t;
static xnintr_irq_t xnirqs[XNARCH_NR_IRQS];
-#endif /* CONFIG_SMP */
+#endif /* CONFIG_SMP || XENO_DEBUG(NUCLEUS) */
static inline xnintr_t *xnintr_shirq_first(unsigned irq)
{
Index: b/ksrc/nucleus/module.c
===================================================================
--- a/ksrc/nucleus/module.c
+++ b/ksrc/nucleus/module.c
@@ -664,7 +664,7 @@ void xnpod_discard_tbase_proc(xntbase_t
#endif /* CONFIG_XENO_OPT_STATS */
-#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+#if XENO_DEBUG(NUCLEUS)
xnlockinfo_t xnlock_stats[RTHAL_NR_CPUS];
@@ -712,7 +712,7 @@ static int lock_read_proc(char *page,
EXPORT_SYMBOL(xnlock_stats);
-#endif /* CONFIG_SMP && XENO_DEBUG(NUCLEUS) */
+#endif /* XENO_DEBUG(NUCLEUS) */
static int latency_read_proc(char *page,
char **start,
@@ -1033,9 +1033,9 @@ void xnpod_init_proc(void)
create_proc_entry("timerstat", S_IFDIR, rthal_proc_root);
#endif /* CONFIG_XENO_OPT_STATS */
-#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+#if XENO_DEBUG(NUCLEUS)
add_proc_leaf("lock", &lock_read_proc, NULL, NULL, rthal_proc_root);
-#endif /* CONFIG_SMP && XENO_DEBUG(NUCLEUS) */
+#endif /* XENO_DEBUG(NUCLEUS) */
add_proc_leaf("latency",
&latency_read_proc,
@@ -1087,9 +1087,9 @@ void xnpod_delete_proc(void)
remove_proc_entry("timerstat", rthal_proc_root);
remove_proc_entry("stat", rthal_proc_root);
#endif /* CONFIG_XENO_OPT_STATS */
-#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+#if XENO_DEBUG(NUCLEUS)
remove_proc_entry("lock", rthal_proc_root);
-#endif /* CONFIG_SMP && XENO_DEBUG(NUCLEUS) */
+#endif /* XENO_DEBUG(NUCLEUS) */
}
#ifdef CONFIG_XENO_OPT_PERVASIVE
Index: b/ksrc/nucleus/pod.c
===================================================================
--- a/ksrc/nucleus/pod.c
+++ b/ksrc/nucleus/pod.c
@@ -3472,7 +3472,7 @@ EXPORT_SYMBOL(xnpod_welcome_thread);
EXPORT_SYMBOL(nkpod_struct);
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
EXPORT_SYMBOL(nklock);
-#endif /* CONFIG_SMP */
+#endif /* CONFIG_SMP || XENO_DEBUG(NUCLEUS) */
EXPORT_SYMBOL(nklatency);
Index: b/ksrc/skins/posix/signal.c
===================================================================
--- a/ksrc/skins/posix/signal.c
+++ b/ksrc/skins/posix/signal.c
@@ -64,7 +64,7 @@ typedef void siginfo_handler_t(int, sigi
#define SIGRTMAX 64
static struct sigaction actions[SIGRTMAX];
static pse51_siginfo_t pse51_infos_pool[PSE51_SIGQUEUE_MAX];
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
static xnlock_t pse51_infos_lock = XNARCH_LOCK_UNLOCKED;
#endif
static xnpqueue_t pse51_infos_free_list;
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 254 bytes --]
^ permalink raw reply [flat|nested] 12+ messages in thread* [Xenomai-core] [PATCH 1/6] Refactor xnlock debugging
[not found] <47C51A34.203@domain.hid>
` (5 preceding siblings ...)
2008-03-02 10:38 ` [Xenomai-help] [PATCH 6/6] Allow xnlock debugging on single-CPU systems Jan Kiszka
@ 2008-03-02 11:02 ` Jan Kiszka
2008-03-02 11:02 ` [Xenomai-core] [PATCH 2/6] Whitespace fixes for asm-generic/system.h Jan Kiszka
` (4 subsequent siblings)
11 siblings, 0 replies; 12+ messages in thread
From: Jan Kiszka @ 2008-03-02 11:02 UTC (permalink / raw)
To: xenomai-core
[-- Attachment #1.1: Type: text/plain, Size: 292 bytes --]
Concentrate all xnlock debugging related code in static inline wrappers
that can easily be switched on and off without visual harm for the lock
services themselves. This does not yet change the debugging logic, it
just saves a few byte text by avoiding redundant processor-id accesses.
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: refactor-xnlock-debugging.patch --]
[-- Type: text/x-patch; name="refactor-xnlock-debugging.patch", Size: 11045 bytes --]
---
include/asm-generic/system.h | 296 ++++++++++++++++++++++---------------------
1 file changed, 154 insertions(+), 142 deletions(-)
Index: b/include/asm-generic/system.h
===================================================================
--- a/include/asm-generic/system.h
+++ b/include/asm-generic/system.h
@@ -47,6 +47,10 @@
#define CONFIG_XENO_OPT_DEBUG_NUCLEUS 0
#endif
+#ifdef __cplusplus
+extern "C" {
+#endif
+
/* Time base export */
#define xnarch_declare_tbase(base) do { } while(0)
@@ -81,7 +85,13 @@ typedef unsigned long spl_t;
#define spltest() rthal_local_irq_test()
#define splget(x) rthal_local_irq_flags(x)
-#if defined(CONFIG_SMP) && defined(CONFIG_XENO_OPT_DEBUG)
+static inline unsigned xnarch_current_cpu(void)
+{
+ return rthal_processor_id();
+}
+
+#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+
typedef struct {
unsigned long long spin_time;
@@ -114,12 +124,96 @@ typedef struct {
0LL, \
}
-#else /* !(CONFIG_SMP && CONFIG_XENO_OPT_DEBUG) */
+#define XNLOCK_DBG_CONTEXT , __FILE__, __LINE__, __FUNCTION__
+#define XNLOCK_DBG_CONTEXT_ARGS \
+ , const char *file, int line, const char *function
+#define XNLOCK_DBG_PASS_CONTEXT , file, line, function
+#define XNLOCK_DBG_MAX_SPINS 10000000
+
+static inline void
+xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit)
+{
+ *start = rthal_rdtsc();
+ *spin_limit = XNLOCK_DBG_MAX_SPINS;
+}
+
+static inline void
+xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit,
+ const char *file, int line, const char *function)
+{
+ if (--*spin_limit == 0) {
+ rthal_emergency_console();
+ printk(KERN_ERR "Xenomai: stuck on nucleus lock %p\n"
+ " waiter = %s:%u (%s(), CPU #%d)\n"
+ " owner = %s:%u (%s(), CPU #%d)\n",
+ lock, file, line, function, cpu,
+ lock->file, lock->line, lock->function, lock->cpu);
+ show_stack(NULL, NULL);
+ }
+}
+
+static inline void
+xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start,
+ const char *file, int line, const char *function)
+{
+ lock->lock_date = *start;
+ lock->spin_time = rthal_rdtsc() - *start;
+ lock->file = file;
+ lock->function = function;
+ lock->line = line;
+ lock->cpu = cpu;
+}
+
+static inline void xnlock_dbg_release(xnlock_t *lock)
+{
+ extern xnlockinfo_t xnlock_stats[];
+ unsigned long long lock_time = rthal_rdtsc() - lock->lock_date;
+ xnlockinfo_t *stats = &xnlock_stats[xnarch_current_cpu()];
+
+ lock->cpu = -lock->cpu; /* File that we released it. */
+
+ if (lock_time > stats->lock_time) {
+ stats->lock_time = lock_time;
+ stats->spin_time = lock->spin_time;
+ stats->file = lock->file;
+ stats->function = lock->function;
+ stats->line = lock->line;
+ }
+}
+
+static inline void xnlock_dbg_invalid_release(xnlock_t *lock)
+{
+ rthal_emergency_console();
+ printk(KERN_ERR "Xenomai: unlocking unlocked nucleus lock %p\n"
+ " owner = %s:%u (%s(), CPU #%d)\n",
+ lock, lock->file, lock->line, lock->function, lock->cpu);
+ show_stack(NULL,NULL);
+}
+
+#else /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
typedef struct { atomic_t owner; } xnlock_t;
#define XNARCH_LOCK_UNLOCKED (xnlock_t) { { ~0 } }
-#endif /* !(CONFIG_SMP && CONFIG_XENO_OPT_DEBUG) */
+
+#define XNLOCK_DBG_CONTEXT
+#define XNLOCK_DBG_CONTEXT_ARGS
+#define XNLOCK_DBG_PASS_CONTEXT
+
+static inline void
+xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit)
+{ }
+
+static inline void
+xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit) { }
+
+static inline void
+xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start) { }
+
+static inline void xnlock_dbg_release(xnlock_t *lock) { }
+static inline void xnlock_dbg_invalid_release(xnlock_t *lock) { }
+
+#endif /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
#define XNARCH_NR_CPUS RTHAL_NR_CPUS
@@ -167,10 +261,6 @@ typedef struct xnarch_heapcb {
} xnarch_heapcb_t;
-#ifdef __cplusplus
-extern "C" {
-#endif
-
unsigned long long xnarch_get_host_time(void);
long long xnarch_tsc_to_ns(long long ts);
@@ -189,11 +279,6 @@ static inline unsigned long long xnarch_
return RTHAL_CPU_FREQ;
}
-static inline unsigned xnarch_current_cpu(void)
-{
- return rthal_processor_id();
-}
-
#define xnarch_halt(emsg) \
do { \
rthal_emergency_console(); \
@@ -213,17 +298,11 @@ static inline int xnarch_setimask (int i
#ifdef CONFIG_SMP
-#if XENO_DEBUG(NUCLEUS)
-#define xnlock_get(lock) \
- __xnlock_get(lock, __FILE__, __LINE__,__FUNCTION__)
+#define xnlock_get(lock) __xnlock_get(lock XNLOCK_DBG_CONTEXT)
#define xnlock_get_irqsave(lock,x) \
- ((x) = __xnlock_get_irqsave(lock, __FILE__, __LINE__,__FUNCTION__))
-#else /* !XENO_DEBUG(NUCLEUS) */
-#define xnlock_get(lock) __xnlock_get(lock)
-#define xnlock_get_irqsave(lock,x) ((x) = __xnlock_get_irqsave(lock))
-#endif /* !XENO_DEBUG(NUCLEUS) */
-#define xnlock_clear_irqoff(lock) xnlock_put_irqrestore(lock,1)
-#define xnlock_clear_irqon(lock) xnlock_put_irqrestore(lock,0)
+ ((x) = __xnlock_get_irqsave(lock XNLOCK_DBG_CONTEXT))
+#define xnlock_clear_irqoff(lock) xnlock_put_irqrestore(lock, 1)
+#define xnlock_clear_irqon(lock) xnlock_put_irqrestore(lock, 0)
static inline void xnlock_init (xnlock_t *lock)
{
@@ -235,132 +314,65 @@ static inline void xnlock_init (xnlock_t
#define DEFINE_XNLOCK(lock) xnlock_t lock = XNARCH_LOCK_UNLOCKED
#define DEFINE_PRIVATE_XNLOCK(lock) static DEFINE_XNLOCK(lock)
-#if XENO_DEBUG(NUCLEUS)
+static inline int __xnlock_get(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
+{
+ unsigned long long start;
+ unsigned int spin_limit;
+ int cpu = xnarch_current_cpu();
+
+ if (atomic_read(&lock->owner) == cpu)
+ return 1;
+
+ xnlock_dbg_prepare_acquire(&start, &spin_limit);
+
+ while (atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0)
+ do {
+ cpu_relax();
+ xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */
+ XNLOCK_DBG_PASS_CONTEXT);
+ } while(atomic_read(&lock->owner) != ~0);
+
+ xnlock_dbg_acquired(lock, cpu, &start /*, */ XNLOCK_DBG_PASS_CONTEXT);
+
+ return 0;
+}
+
+static inline void xnlock_put(xnlock_t *lock)
+{
+ if (likely(atomic_read(&lock->owner) == xnarch_current_cpu())) {
+ xnlock_dbg_release(lock);
+
+ /*
+ * Make sure all data written inside the lock is visible to
+ * other CPUs before we release the lock.
+ */
+ xnarch_memory_barrier();
+
+ atomic_set(&lock->owner, ~0);
+ } else
+ xnlock_dbg_invalid_release(lock);
+}
+
+static inline spl_t
+__xnlock_get_irqsave(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
+{
+ unsigned long flags;
+
+ rthal_local_irq_save(flags);
-#define XNARCH_DEBUG_SPIN_LIMIT 3000000
+ if (__xnlock_get(lock /*, */ XNLOCK_DBG_PASS_CONTEXT))
+ flags |= 2; /* Recursive acquisition */
-static inline int __xnlock_get (xnlock_t *lock,
- const char *file,
- unsigned line,
- const char *function)
-{
- unsigned spin_count = 0;
-#else /* !XENO_DEBUG(NUCLEUS) */
-static inline int __xnlock_get (xnlock_t *lock)
-{
-#endif /* !XENO_DEBUG(NUCLEUS) */
- int recursing;
-
- recursing = (atomic_read(&lock->owner) == rthal_processor_id());
- if (!recursing) {
-#if XENO_DEBUG(NUCLEUS)
- unsigned long long lock_date = rthal_rdtsc();
-#endif /* XENO_DEBUG(NUCLEUS) */
- while(atomic_cmpxchg(&lock->owner, ~0, rthal_processor_id()) != ~0)
- do {
- cpu_relax();
-
-#if XENO_DEBUG(NUCLEUS)
- if (++spin_count == XNARCH_DEBUG_SPIN_LIMIT) {
- rthal_emergency_console();
- printk(KERN_ERR
- "Xenomai: stuck on nucleus lock %p\n"
- " waiter = %s:%u (%s(), CPU #%d)\n"
- " owner = %s:%u (%s(), CPU #%d)\n",
- lock,file,line,function,rthal_processor_id(),
- lock->file,lock->line,lock->function,lock->cpu);
- show_stack(NULL,NULL);
- for (;;)
- cpu_relax();
- }
-#endif /* XENO_DEBUG(NUCLEUS) */
- } while(atomic_read(&lock->owner) != ~0);
-
-#if XENO_DEBUG(NUCLEUS)
- lock->spin_time = rthal_rdtsc() - lock_date;
- lock->lock_date = lock_date;
- lock->file = file;
- lock->function = function;
- lock->line = line;
- lock->cpu = rthal_processor_id();
-#endif /* XENO_DEBUG(NUCLEUS) */
- }
-
- return recursing;
-}
-
-static inline void xnlock_put (xnlock_t *lock)
-{
- if (likely(atomic_read(&lock->owner) == rthal_processor_id())) {
-
-#if XENO_DEBUG(NUCLEUS)
- extern xnlockinfo_t xnlock_stats[];
-
- unsigned long long lock_time = rthal_rdtsc() - lock->lock_date;
- int cpu = rthal_processor_id();
-
- if (lock_time > xnlock_stats[cpu].lock_time) {
- xnlock_stats[cpu].lock_time = lock_time;
- xnlock_stats[cpu].spin_time = lock->spin_time;
- xnlock_stats[cpu].file = lock->file;
- xnlock_stats[cpu].function = lock->function;
- xnlock_stats[cpu].line = lock->line;
- }
-#endif /* XENO_DEBUG(NUCLEUS) */
- /*
- * Make sure all data written inside the lock is visible to
- * other CPUs before we release the lock.
- */
- xnarch_memory_barrier();
-
- atomic_set(&lock->owner, ~0);
- }
-#if XENO_DEBUG(NUCLEUS)
- else {
- rthal_emergency_console();
- printk(KERN_ERR
- "Xenomai: unlocking unlocked nucleus lock %p\n"
- " owner = %s:%u (%s(), CPU #%d)\n",
- lock,lock->file,lock->line,lock->function,lock->cpu);
- show_stack(NULL,NULL);
- for (;;)
- cpu_relax();
- }
-#endif /* XENO_DEBUG(NUCLEUS) */
-}
-
-#if XENO_DEBUG(NUCLEUS)
-
-static inline spl_t __xnlock_get_irqsave (xnlock_t *lock,
- const char *file,
- unsigned line,
- const char *function)
-{
-#else /* !XENO_DEBUG(NUCLEUS) */
-static inline spl_t __xnlock_get_irqsave (xnlock_t *lock)
-{
-#endif /* !XENO_DEBUG(NUCLEUS) */
- unsigned long flags;
-
- rthal_local_irq_save(flags);
-
-#if XENO_DEBUG(NUCLEUS)
- if (__xnlock_get(lock, file, line, function))
- flags |= 2;
-#else /* !XENO_DEBUG(NUCLEUS) */
- if (__xnlock_get(lock))
- flags |= 2;
-#endif /* !XENO_DEBUG(NUCLEUS) */
-
- return flags;
+ return flags;
}
-static inline void xnlock_put_irqrestore (xnlock_t *lock, spl_t flags)
+static inline void xnlock_put_irqrestore(xnlock_t *lock, spl_t flags)
{
- if (!(flags & 2))
- xnlock_put(lock);
+ /* Only release the lock if we didn't take it recursively. */
+ if (!(flags & 2))
+ xnlock_put(lock);
- rthal_local_irq_restore(flags & 1);
+ rthal_local_irq_restore(flags & 1);
}
static inline int xnarch_send_ipi (xnarch_cpumask_t cpumask)
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 254 bytes --]
^ permalink raw reply [flat|nested] 12+ messages in thread* [Xenomai-core] [PATCH 2/6] Whitespace fixes for asm-generic/system.h
[not found] <47C51A34.203@domain.hid>
` (6 preceding siblings ...)
2008-03-02 11:02 ` [Xenomai-core] [PATCH 1/6] Refactor xnlock debugging Jan Kiszka
@ 2008-03-02 11:02 ` Jan Kiszka
2008-03-02 11:02 ` [Xenomai-core] [PATCH 3/6] Consolidate xnlock_put debugging Jan Kiszka
` (3 subsequent siblings)
11 siblings, 0 replies; 12+ messages in thread
From: Jan Kiszka @ 2008-03-02 11:02 UTC (permalink / raw)
To: Xenomai-core
[-- Attachment #1.1: Type: text/plain, Size: 26 bytes --]
No functional changes.
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: system-h-whitespace-cleanups.patch --]
[-- Type: text/x-patch; name="system-h-whitespace-cleanups.patch", Size: 9802 bytes --]
---
include/asm-generic/system.h | 145 +++++++++++++++++++++----------------------
1 file changed, 73 insertions(+), 72 deletions(-)
Index: b/include/asm-generic/system.h
===================================================================
--- a/include/asm-generic/system.h
+++ b/include/asm-generic/system.h
@@ -56,13 +56,13 @@ extern "C" {
/* Tracer interface */
#define xnarch_trace_max_begin(v) rthal_trace_max_begin(v)
-#define xnarch_trace_max_end(v) rthal_trace_max_end(v)
+#define xnarch_trace_max_end(v) rthal_trace_max_end(v)
#define xnarch_trace_max_reset() rthal_trace_max_reset()
#define xnarch_trace_user_start() rthal_trace_user_start()
#define xnarch_trace_user_stop(v) rthal_trace_user_stop(v)
-#define xnarch_trace_user_freeze(v, once) rthal_trace_user_freeze(v, once)
+#define xnarch_trace_user_freeze(v, once) rthal_trace_user_freeze(v, once)
#define xnarch_trace_special(id, v) rthal_trace_special(id, v)
-#define xnarch_trace_special_u64(id, v) rthal_trace_special_u64(id, v)
+#define xnarch_trace_special_u64(id, v) rthal_trace_special_u64(id, v)
#define xnarch_trace_pid(pid, prio) rthal_trace_pid(pid, prio)
#define xnarch_trace_panic_freeze() rthal_trace_panic_freeze()
#define xnarch_trace_panic_dump() rthal_trace_panic_dump()
@@ -94,23 +94,23 @@ static inline unsigned xnarch_current_cp
typedef struct {
- unsigned long long spin_time;
- unsigned long long lock_time;
- const char *file;
- const char *function;
- unsigned line;
+ unsigned long long spin_time;
+ unsigned long long lock_time;
+ const char *file;
+ const char *function;
+ unsigned line;
} xnlockinfo_t;
typedef struct {
- atomic_t owner;
- const char *file;
- const char *function;
- unsigned line;
- int cpu;
- unsigned long long spin_time;
- unsigned long long lock_date;
+ atomic_t owner;
+ const char *file;
+ const char *function;
+ unsigned line;
+ int cpu;
+ unsigned long long spin_time;
+ unsigned long long lock_date;
} xnlock_t;
@@ -194,7 +194,7 @@ static inline void xnlock_dbg_invalid_re
typedef struct { atomic_t owner; } xnlock_t;
-#define XNARCH_LOCK_UNLOCKED (xnlock_t) { { ~0 } }
+#define XNARCH_LOCK_UNLOCKED (xnlock_t) { { ~0 } }
#define XNLOCK_DBG_CONTEXT
#define XNLOCK_DBG_CONTEXT_ARGS
@@ -215,49 +215,48 @@ static inline void xnlock_dbg_invalid_re
#endif /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
-#define XNARCH_NR_CPUS RTHAL_NR_CPUS
+#define XNARCH_NR_CPUS RTHAL_NR_CPUS
-#define XNARCH_NR_IRQS RTHAL_NR_IRQS
-#define XNARCH_TIMER_IRQ RTHAL_TIMER_IRQ
-#define XNARCH_TIMER_DEVICE RTHAL_TIMER_DEVICE
-#define XNARCH_CLOCK_DEVICE RTHAL_CLOCK_DEVICE
+#define XNARCH_NR_IRQS RTHAL_NR_IRQS
+#define XNARCH_TIMER_IRQ RTHAL_TIMER_IRQ
+#define XNARCH_TIMER_DEVICE RTHAL_TIMER_DEVICE
+#define XNARCH_CLOCK_DEVICE RTHAL_CLOCK_DEVICE
-#define XNARCH_ROOT_STACKSZ 0 /* Only a placeholder -- no stack */
+#define XNARCH_ROOT_STACKSZ 0 /* Only a placeholder -- no stack */
#define XNARCH_PROMPT "Xenomai: "
-#define xnarch_loginfo(fmt,args...) printk(KERN_INFO XNARCH_PROMPT fmt , ##args)
-#define xnarch_logwarn(fmt,args...) printk(KERN_WARNING XNARCH_PROMPT fmt , ##args)
-#define xnarch_logerr(fmt,args...) printk(KERN_ERR XNARCH_PROMPT fmt , ##args)
-#define xnarch_printf(fmt,args...) printk(KERN_INFO XNARCH_PROMPT fmt , ##args)
+#define xnarch_loginfo(fmt, args...) printk(KERN_INFO XNARCH_PROMPT fmt, ##args)
+#define xnarch_logwarn(fmt, args...) printk(KERN_WARNING XNARCH_PROMPT fmt, ##args)
+#define xnarch_logerr(fmt, args...) printk(KERN_ERR XNARCH_PROMPT fmt, ##args)
+#define xnarch_printf(fmt, args...) printk(KERN_INFO XNARCH_PROMPT fmt, ##args)
typedef cpumask_t xnarch_cpumask_t;
#ifdef CONFIG_SMP
-#define xnarch_cpu_online_map cpu_online_map
+#define xnarch_cpu_online_map cpu_online_map
#else
-#define xnarch_cpu_online_map cpumask_of_cpu(0)
+#define xnarch_cpu_online_map cpumask_of_cpu(0)
#endif
-#define xnarch_num_online_cpus() num_online_cpus()
-#define xnarch_cpu_set(cpu, mask) cpu_set(cpu, (mask))
-#define xnarch_cpu_clear(cpu, mask) cpu_clear(cpu, (mask))
-#define xnarch_cpus_clear(mask) cpus_clear(mask)
-#define xnarch_cpu_isset(cpu, mask) cpu_isset(cpu, (mask))
-#define xnarch_cpus_and(dst, src1, src2) cpus_and((dst), (src1), (src2))
-#define xnarch_cpus_equal(mask1, mask2) cpus_equal((mask1), (mask2))
-#define xnarch_cpus_empty(mask) cpus_empty(mask)
-#define xnarch_cpumask_of_cpu(cpu) cpumask_of_cpu(cpu)
-#define xnarch_cpu_test_and_set(cpu,mask) cpu_test_and_set(cpu, (mask))
-
-#define xnarch_first_cpu(mask) first_cpu(mask)
-#define XNARCH_CPU_MASK_ALL CPU_MASK_ALL
+#define xnarch_num_online_cpus() num_online_cpus()
+#define xnarch_cpu_set(cpu, mask) cpu_set(cpu, (mask))
+#define xnarch_cpu_clear(cpu, mask) cpu_clear(cpu, (mask))
+#define xnarch_cpus_clear(mask) cpus_clear(mask)
+#define xnarch_cpu_isset(cpu, mask) cpu_isset(cpu, (mask))
+#define xnarch_cpus_and(dst, src1, src2) cpus_and((dst), (src1), (src2))
+#define xnarch_cpus_equal(mask1, mask2) cpus_equal((mask1), (mask2))
+#define xnarch_cpus_empty(mask) cpus_empty(mask)
+#define xnarch_cpumask_of_cpu(cpu) cpumask_of_cpu(cpu)
+#define xnarch_cpu_test_and_set(cpu, mask) cpu_test_and_set(cpu, (mask))
+#define xnarch_first_cpu(mask) first_cpu(mask)
+#define XNARCH_CPU_MASK_ALL CPU_MASK_ALL
typedef struct xnarch_heapcb {
- atomic_t numaps; /* # of active user-space mappings. */
+ atomic_t numaps; /* # of active user-space mappings. */
- int kmflags; /* Kernel memory flags (0 if vmalloc()). */
+ int kmflags; /* Kernel memory flags (0 if vmalloc()). */
- void *heapbase; /* Shared heap memory base. */
+ void *heapbase; /* Shared heap memory base. */
} xnarch_heapcb_t;
@@ -267,7 +266,7 @@ long long xnarch_tsc_to_ns(long long ts)
static inline long long xnarch_tsc_to_ns_rounded(long long ts)
{
- return (xnarch_llimd(ts, 1000000000, RTHAL_CPU_FREQ/2) + 1) / 2;
+ return (xnarch_llimd(ts, 1000000000, RTHAL_CPU_FREQ/2) + 1) / 2;
}
long long xnarch_ns_to_tsc(long long ns);
@@ -276,24 +275,26 @@ unsigned long long xnarch_get_cpu_time(v
static inline unsigned long long xnarch_get_cpu_freq(void)
{
- return RTHAL_CPU_FREQ;
+ return RTHAL_CPU_FREQ;
}
-#define xnarch_halt(emsg) \
-do { \
- rthal_emergency_console(); \
- xnarch_logerr("fatal: %s\n",emsg); \
- show_stack(NULL,NULL); \
- xnarch_trace_panic_dump(); \
- for (;;) cpu_relax(); \
-} while(0)
+#define xnarch_halt(emsg) \
+ do { \
+ rthal_emergency_console(); \
+ xnarch_logerr("fatal: %s\n", emsg); \
+ show_stack(NULL,NULL); \
+ xnarch_trace_panic_dump(); \
+ for (;;) \
+ cpu_relax(); \
+ } while(0)
static inline int xnarch_setimask (int imask)
{
- spl_t s;
- splhigh(s);
- splexit(!!imask);
- return !!s;
+ spl_t s;
+
+ splhigh(s);
+ splexit(!!imask);
+ return !!s;
}
#ifdef CONFIG_SMP
@@ -306,7 +307,7 @@ static inline int xnarch_setimask (int i
static inline void xnlock_init (xnlock_t *lock)
{
- *lock = XNARCH_LOCK_UNLOCKED;
+ *lock = XNARCH_LOCK_UNLOCKED;
}
#define DECLARE_XNLOCK(lock) xnlock_t lock
@@ -375,9 +376,9 @@ static inline void xnlock_put_irqrestore
rthal_local_irq_restore(flags & 1);
}
-static inline int xnarch_send_ipi (xnarch_cpumask_t cpumask)
+static inline int xnarch_send_ipi(xnarch_cpumask_t cpumask)
{
- return rthal_send_ipi(RTHAL_SERVICE_IPI0, cpumask);
+ return rthal_send_ipi(RTHAL_SERVICE_IPI0, cpumask);
}
static inline int xnlock_is_owner(xnlock_t *lock)
@@ -387,14 +388,14 @@ static inline int xnlock_is_owner(xnlock
#else /* !CONFIG_SMP */
-#define xnlock_init(lock) do { } while(0)
-#define xnlock_get(lock) do { } while(0)
-#define xnlock_put(lock) do { } while(0)
-#define xnlock_get_irqsave(lock,x) rthal_local_irq_save(x)
-#define xnlock_put_irqrestore(lock,x) rthal_local_irq_restore(x)
-#define xnlock_clear_irqoff(lock) rthal_local_irq_disable()
-#define xnlock_clear_irqon(lock) rthal_local_irq_enable()
-#define xnlock_is_owner(lock) 1
+#define xnlock_init(lock) do { } while(0)
+#define xnlock_get(lock) do { } while(0)
+#define xnlock_put(lock) do { } while(0)
+#define xnlock_get_irqsave(lock,x) rthal_local_irq_save(x)
+#define xnlock_put_irqrestore(lock,x) rthal_local_irq_restore(x)
+#define xnlock_clear_irqoff(lock) rthal_local_irq_disable()
+#define xnlock_clear_irqon(lock) rthal_local_irq_enable()
+#define xnlock_is_owner(lock) 1
#define DECLARE_XNLOCK(lock)
#define DECLARE_EXTERN_XNLOCK(lock)
@@ -403,7 +404,7 @@ static inline int xnlock_is_owner(xnlock
static inline int xnarch_send_ipi (xnarch_cpumask_t cpumask)
{
- return 0;
+ return 0;
}
#endif /* !CONFIG_SMP */
@@ -418,7 +419,7 @@ static inline int xnarch_remap_vm_page(s
unsigned long from,
unsigned long to)
{
- return wrap_remap_vm_page(vma,from,to);
+ return wrap_remap_vm_page(vma, from, to);
}
static inline int xnarch_remap_io_page_range(struct vm_area_struct *vma,
@@ -427,7 +428,7 @@ static inline int xnarch_remap_io_page_r
unsigned long size,
pgprot_t prot)
{
- return wrap_remap_io_page_range(vma,from,to,size,prot);
+ return wrap_remap_io_page_range(vma, from, to, size, prot);
}
#ifndef xnarch_hisyscall_entry
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 254 bytes --]
^ permalink raw reply [flat|nested] 12+ messages in thread* [Xenomai-core] [PATCH 3/6] Consolidate xnlock_put debugging
[not found] <47C51A34.203@domain.hid>
` (7 preceding siblings ...)
2008-03-02 11:02 ` [Xenomai-core] [PATCH 2/6] Whitespace fixes for asm-generic/system.h Jan Kiszka
@ 2008-03-02 11:02 ` Jan Kiszka
2008-03-02 11:02 ` [Xenomai-core] [PATCH 4/6] Move spinning code out-of-line Jan Kiszka
` (2 subsequent siblings)
11 siblings, 0 replies; 12+ messages in thread
From: Jan Kiszka @ 2008-03-02 11:02 UTC (permalink / raw)
To: Xenomai-core
[-- Attachment #1.1: Type: text/plain, Size: 426 bytes --]
The owner check on xnlock release looks light-weight in first sight, but
it isn't. It requires a processor-id lookup + it adds a conditional to
the otherwise unconditional release path. Moreover, this code is inlined
into every xnlock_put user, thus increases the text size of hot paths.
This patch therefore moves the check under the same switch
(XENO_OPT_DEBUG_NUCLEUS) as the rest of xnlock debugging already is.
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: consolidate-xnlock_put-debugging.patch --]
[-- Type: text/x-patch; name="consolidate-xnlock_put-debugging.patch", Size: 2868 bytes --]
---
include/asm-generic/system.h | 52 +++++++++++++++++++++++--------------------
1 file changed, 28 insertions(+), 24 deletions(-)
Index: b/include/asm-generic/system.h
===================================================================
--- a/include/asm-generic/system.h
+++ b/include/asm-generic/system.h
@@ -164,11 +164,23 @@ xnlock_dbg_acquired(xnlock_t *lock, int
lock->cpu = cpu;
}
-static inline void xnlock_dbg_release(xnlock_t *lock)
+static inline int xnlock_dbg_release(xnlock_t *lock)
{
extern xnlockinfo_t xnlock_stats[];
unsigned long long lock_time = rthal_rdtsc() - lock->lock_date;
- xnlockinfo_t *stats = &xnlock_stats[xnarch_current_cpu()];
+ int cpu = xnarch_current_cpu();
+ xnlockinfo_t *stats = &xnlock_stats[cpu];
+
+ if (unlikely(atomic_read(&lock->owner) != cpu)) {
+ rthal_emergency_console();
+ printk(KERN_ERR "Xenomai: unlocking unlocked nucleus lock %p"
+ " on CPU #%d\n"
+ " owner = %s:%u (%s(), CPU #%d)\n",
+ lock, cpu, lock->file, lock->line, lock->function,
+ lock->cpu);
+ show_stack(NULL,NULL);
+ return 1;
+ }
lock->cpu = -lock->cpu; /* File that we released it. */
@@ -179,15 +191,7 @@ static inline void xnlock_dbg_release(xn
stats->function = lock->function;
stats->line = lock->line;
}
-}
-
-static inline void xnlock_dbg_invalid_release(xnlock_t *lock)
-{
- rthal_emergency_console();
- printk(KERN_ERR "Xenomai: unlocking unlocked nucleus lock %p\n"
- " owner = %s:%u (%s(), CPU #%d)\n",
- lock, lock->file, lock->line, lock->function, lock->cpu);
- show_stack(NULL,NULL);
+ return 0;
}
#else /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
@@ -210,8 +214,10 @@ xnlock_dbg_spinning(xnlock_t *lock, int
static inline void
xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start) { }
-static inline void xnlock_dbg_release(xnlock_t *lock) { }
-static inline void xnlock_dbg_invalid_release(xnlock_t *lock) { }
+static inline int xnlock_dbg_release(xnlock_t *lock)
+{
+ return 0;
+}
#endif /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
@@ -340,18 +346,16 @@ static inline int __xnlock_get(xnlock_t
static inline void xnlock_put(xnlock_t *lock)
{
- if (likely(atomic_read(&lock->owner) == xnarch_current_cpu())) {
- xnlock_dbg_release(lock);
+ if (xnlock_dbg_release(lock))
+ return;
+
+ /*
+ * Make sure all data written inside the lock is visible to
+ * other CPUs before we release the lock.
+ */
+ xnarch_memory_barrier();
- /*
- * Make sure all data written inside the lock is visible to
- * other CPUs before we release the lock.
- */
- xnarch_memory_barrier();
-
- atomic_set(&lock->owner, ~0);
- } else
- xnlock_dbg_invalid_release(lock);
+ atomic_set(&lock->owner, ~0);
}
static inline spl_t
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 254 bytes --]
^ permalink raw reply [flat|nested] 12+ messages in thread* [Xenomai-core] [PATCH 4/6] Move spinning code out-of-line
[not found] <47C51A34.203@domain.hid>
` (8 preceding siblings ...)
2008-03-02 11:02 ` [Xenomai-core] [PATCH 3/6] Consolidate xnlock_put debugging Jan Kiszka
@ 2008-03-02 11:02 ` Jan Kiszka
2008-03-02 11:02 ` [Xenomai-core] [PATCH 5/6] Introduce optional recursive ticket xnlock Jan Kiszka
2008-03-02 11:02 ` [Xenomai-core] [PATCH 6/6] Allow xnlock debugging on single-CPU systems Jan Kiszka
11 siblings, 0 replies; 12+ messages in thread
From: Jan Kiszka @ 2008-03-02 11:02 UTC (permalink / raw)
To: Xenomai-core
[-- Attachment #1.1: Type: text/plain, Size: 132 bytes --]
To save text thus and thus avoid cache misses in the worst-case path,
this patch moves the slow path of xnlock_get out-of-line.
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: introduce-xnlock_spin.patch --]
[-- Type: text/x-patch; name="introduce-xnlock_spin.patch", Size: 3501 bytes --]
---
include/asm-generic/bits/pod.h | 18 ++++++++++++++++++
include/asm-generic/system.h | 29 ++++++++++++++---------------
2 files changed, 32 insertions(+), 15 deletions(-)
Index: b/include/asm-generic/bits/pod.h
===================================================================
--- a/include/asm-generic/bits/pod.h
+++ b/include/asm-generic/bits/pod.h
@@ -295,4 +295,22 @@ unsigned long long xnarch_get_cpu_time(v
EXPORT_SYMBOL(xnarch_get_cpu_time);
+#ifdef CONFIG_SMP
+void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
+{
+ unsigned int spin_limit;
+ int cpu = xnarch_current_cpu();
+
+ xnlock_dbg_prepare_spin(&spin_limit);
+
+ while (atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0)
+ do {
+ cpu_relax();
+ xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */
+ XNLOCK_DBG_PASS_CONTEXT);
+ } while(atomic_read(&lock->owner) != ~0);
+}
+EXPORT_SYMBOL(__xnlock_spin);
+#endif /* CONFIG_SMP */
+
#endif /* !_XENO_ASM_GENERIC_BITS_POD_H */
Index: b/include/asm-generic/system.h
===================================================================
--- a/include/asm-generic/system.h
+++ b/include/asm-generic/system.h
@@ -130,10 +130,13 @@ typedef struct {
#define XNLOCK_DBG_PASS_CONTEXT , file, line, function
#define XNLOCK_DBG_MAX_SPINS 10000000
-static inline void
-xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit)
+static inline void xnlock_dbg_prepare_acquire(unsigned long long *start)
{
*start = rthal_rdtsc();
+}
+
+static inline void xnlock_dbg_prepare_spin(unsigned *spin_limit)
+{
*spin_limit = XNLOCK_DBG_MAX_SPINS;
}
@@ -204,15 +207,14 @@ typedef struct { atomic_t owner; } xnloc
#define XNLOCK_DBG_CONTEXT_ARGS
#define XNLOCK_DBG_PASS_CONTEXT
-static inline void
-xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit)
-{ }
+static inline void xnlock_dbg_prepare_acquire(unsigned long long *start) { }
+static inline void xnlock_dbg_prepare_spin(unsigned *spin_limit) { }
static inline void
-xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit) { }
+xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit) { }
static inline void
-xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start) { }
+xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start) { }
static inline int xnlock_dbg_release(xnlock_t *lock)
{
@@ -321,23 +323,20 @@ static inline void xnlock_init (xnlock_t
#define DEFINE_XNLOCK(lock) xnlock_t lock = XNARCH_LOCK_UNLOCKED
#define DEFINE_PRIVATE_XNLOCK(lock) static DEFINE_XNLOCK(lock)
+void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS);
+
static inline int __xnlock_get(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
{
unsigned long long start;
- unsigned int spin_limit;
int cpu = xnarch_current_cpu();
if (atomic_read(&lock->owner) == cpu)
return 1;
- xnlock_dbg_prepare_acquire(&start, &spin_limit);
+ xnlock_dbg_prepare_acquire(&start);
- while (atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0)
- do {
- cpu_relax();
- xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */
- XNLOCK_DBG_PASS_CONTEXT);
- } while(atomic_read(&lock->owner) != ~0);
+ if (unlikely(atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0))
+ __xnlock_spin(lock /*, */ XNLOCK_DBG_PASS_CONTEXT);
xnlock_dbg_acquired(lock, cpu, &start /*, */ XNLOCK_DBG_PASS_CONTEXT);
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 254 bytes --]
^ permalink raw reply [flat|nested] 12+ messages in thread* [Xenomai-core] [PATCH 5/6] Introduce optional recursive ticket xnlock
[not found] <47C51A34.203@domain.hid>
` (9 preceding siblings ...)
2008-03-02 11:02 ` [Xenomai-core] [PATCH 4/6] Move spinning code out-of-line Jan Kiszka
@ 2008-03-02 11:02 ` Jan Kiszka
2008-03-02 11:02 ` [Xenomai-core] [PATCH 6/6] Allow xnlock debugging on single-CPU systems Jan Kiszka
11 siblings, 0 replies; 12+ messages in thread
From: Jan Kiszka @ 2008-03-02 11:02 UTC (permalink / raw)
To: Xenomai-core
[-- Attachment #1.1: Type: text/plain, Size: 978 bytes --]
The origin of this series: When Nick Piggin posted his first suggestion
for ticket spinlocks on LKML, I immediately liked the idea. For details
check LWN [1], in a nutshell: This algorithm enforces strict FIFO order
for the admission to contended spinlocks, thus it improves the
determinism on SMP systems with more than 2 CPUs.
Meanwhile, ticket spinlocks are mainline (2.6.25). But that version has
to drawbacks for us: it doesn't support nesting like xnlock does, and it
is x86-only so far.
So I designed a version for Xenomai which is both nestable and
arch-independent. It is certainly not as optimal as mainline's version,
but our code path stresses the locking code differently anyway, and no
one denies us to introduce per-arch optimized variants later on (if this
lock implementation gains broader relevance).
Survived heavy testing on a 4-way box (15, 20, 25, and 30 us latency
instances across the cores).
[1] http://lwn.net/Articles/267331
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: ticket-xnlock-v2.patch --]
[-- Type: text/x-patch; name="ticket-xnlock-v2.patch", Size: 7146 bytes --]
---
include/asm-generic/bits/pod.h | 4 -
include/asm-generic/system.h | 122 +++++++++++++++++++++++++++++++++++++----
ksrc/nucleus/Kconfig | 12 ++++
3 files changed, 126 insertions(+), 12 deletions(-)
Index: b/include/asm-generic/system.h
===================================================================
--- a/include/asm-generic/system.h
+++ b/include/asm-generic/system.h
@@ -104,7 +104,12 @@ typedef struct {
typedef struct {
+#ifdef CONFIG_XENO_OPT_TICKET_LOCK
+ atomic_t fifo;
+ int owner;
+#else
atomic_t owner;
+#endif
const char *file;
const char *function;
unsigned line;
@@ -115,7 +120,7 @@ typedef struct {
} xnlock_t;
#define XNARCH_LOCK_UNLOCKED (xnlock_t) { \
- { ~0 }, \
+ XNLOCK_INIT_VALUE, \
NULL, \
NULL, \
0, \
@@ -167,6 +172,8 @@ xnlock_dbg_acquired(xnlock_t *lock, int
lock->cpu = cpu;
}
+static inline int xnlock_owner(xnlock_t *lock);
+
static inline int xnlock_dbg_release(xnlock_t *lock)
{
extern xnlockinfo_t xnlock_stats[];
@@ -174,7 +181,7 @@ static inline int xnlock_dbg_release(xnl
int cpu = xnarch_current_cpu();
xnlockinfo_t *stats = &xnlock_stats[cpu];
- if (unlikely(atomic_read(&lock->owner) != cpu)) {
+ if (unlikely(xnlock_owner(lock) != cpu)) {
rthal_emergency_console();
printk(KERN_ERR "Xenomai: unlocking unlocked nucleus lock %p"
" on CPU #%d\n"
@@ -199,9 +206,18 @@ static inline int xnlock_dbg_release(xnl
#else /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
+#ifdef CONFIG_XENO_OPT_TICKET_LOCK
+typedef struct {
+
+ atomic_t fifo;
+ int owner;
+
+} xnlock_t;
+#else
typedef struct { atomic_t owner; } xnlock_t;
+#endif
-#define XNARCH_LOCK_UNLOCKED (xnlock_t) { { ~0 } }
+#define XNARCH_LOCK_UNLOCKED (xnlock_t) { XNLOCK_INIT_VALUE }
#define XNLOCK_DBG_CONTEXT
#define XNLOCK_DBG_CONTEXT_ARGS
@@ -313,11 +329,6 @@ static inline int xnarch_setimask (int i
#define xnlock_clear_irqoff(lock) xnlock_put_irqrestore(lock, 1)
#define xnlock_clear_irqon(lock) xnlock_put_irqrestore(lock, 0)
-static inline void xnlock_init (xnlock_t *lock)
-{
- *lock = XNARCH_LOCK_UNLOCKED;
-}
-
#define DECLARE_XNLOCK(lock) xnlock_t lock
#define DECLARE_EXTERN_XNLOCK(lock) extern xnlock_t lock
#define DEFINE_XNLOCK(lock) xnlock_t lock = XNARCH_LOCK_UNLOCKED
@@ -325,12 +336,97 @@ static inline void xnlock_init (xnlock_t
void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS);
+#ifdef CONFIG_XENO_OPT_TICKET_LOCK
+/* Note: We assume atomic_t is at least 32-bit wide. */
+#define XNLOCK_HEAD_MASK 0x000000FF
+#define XNLOCK_HEAD_INC 0x00000001
+#define XNLOCK_TAIL_MASK 0xFF000000
+#define XNLOCK_TAIL_SHIFT 24
+#define XNLOCK_TAIL_INC 0x01000000
+#define XNLOCK_TICKET_MASK (XNLOCK_HEAD_MASK | XNLOCK_TAIL_MASK)
+#define XNLOCK_INIT_VALUE { 0x00000001 }, ~0
+
+static inline int xnlock_owner(xnlock_t *lock)
+{
+ return lock->owner;
+}
+
+static inline int __xnlock_get(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
+{
+ unsigned long long start;
+ int cpu = xnarch_current_cpu();
+ u32 fifo;
+ u8 ticket;
+
+ if (xnlock_owner(lock) == cpu)
+ return 1;
+
+ xnlock_dbg_prepare_acquire(&start);
+
+ fifo = atomic_add_return(XNLOCK_TAIL_INC, &lock->fifo);
+ ticket = fifo >> XNLOCK_TAIL_SHIFT;
+
+ /* First check if we are already the owner (head == tail). */
+ if (unlikely((u8)fifo != ticket)) {
+ unsigned int spin_limit;
+
+ xnlock_dbg_prepare_spin(&spin_limit);
+
+ /*
+ * Spin until head equals our ticket. In this case,
+ * moving out-of-line does not pay off.
+ */
+ do {
+ cpu_relax();
+ xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */
+ XNLOCK_DBG_PASS_CONTEXT);
+ } while ((u8)atomic_read(&lock->fifo) != ticket);
+ }
+ lock->owner = cpu;
+
+ xnlock_dbg_acquired(lock, cpu, &start /*, */ XNLOCK_DBG_PASS_CONTEXT);
+
+ return 0;
+}
+
+static inline void xnlock_put(xnlock_t *lock)
+{
+ u32 fifo = atomic_read(&lock->fifo);
+
+ if (xnlock_dbg_release(lock))
+ return;
+
+ lock->owner = ~0;
+
+ /*
+ * Make sure all data written inside the lock is visible to
+ * other CPUs before we release the lock.
+ */
+ xnarch_memory_barrier();
+
+ /*
+ * Carefully calculate the addend so that we confine the
+ * XNLOCK_HEAD_INC just to the first byte of fifo.
+ */
+ atomic_add(((fifo + XNLOCK_HEAD_INC) & XNLOCK_TICKET_MASK) - fifo,
+ &lock->fifo);
+}
+
+#else /* ! CONFIG_XENO_OPT_TICKET_LOCK */
+
+#define XNLOCK_INIT_VALUE { ~0 }
+
+static inline int xnlock_owner(xnlock_t *lock)
+{
+ return atomic_read(&lock->owner);
+}
+
static inline int __xnlock_get(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
{
unsigned long long start;
int cpu = xnarch_current_cpu();
- if (atomic_read(&lock->owner) == cpu)
+ if (xnlock_owner(lock) == cpu)
return 1;
xnlock_dbg_prepare_acquire(&start);
@@ -356,6 +452,12 @@ static inline void xnlock_put(xnlock_t *
atomic_set(&lock->owner, ~0);
}
+#endif /* ! CONFIG_XENO_OPT_TICKET_LOCK */
+
+static inline void xnlock_init(xnlock_t *lock)
+{
+ *lock = XNARCH_LOCK_UNLOCKED;
+}
static inline spl_t
__xnlock_get_irqsave(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
@@ -386,7 +488,7 @@ static inline int xnarch_send_ipi(xnarch
static inline int xnlock_is_owner(xnlock_t *lock)
{
- return atomic_read(&lock->owner) == xnarch_current_cpu();
+ return xnlock_owner(lock) == xnarch_current_cpu();
}
#else /* !CONFIG_SMP */
Index: b/ksrc/nucleus/Kconfig
===================================================================
--- a/ksrc/nucleus/Kconfig
+++ b/ksrc/nucleus/Kconfig
@@ -340,6 +340,18 @@ config XENO_OPT_TIMER_WHEEL_STEP
Set the duration in ns of a timer wheel step. At each step,
the timer wheel use the next hash bucket.
+config XENO_OPT_TICKET_LOCK
+ bool "FIFO ticket spinlocks"
+ depends on SMP
+ help
+
+ Uses a ticket spinlock algorithm which enforces strict FIFO
+ admission order on contended locks, improving the determinism
+ of lock acquisitions on large SMP systems. The ticket spinlock
+ has a higher overhead than the unordered algorithm and should
+ therefore only be used if there are more than two CPUs, thus
+ more than one waiter in the worst case.
+
endmenu
endif
Index: b/include/asm-generic/bits/pod.h
===================================================================
--- a/include/asm-generic/bits/pod.h
+++ b/include/asm-generic/bits/pod.h
@@ -295,7 +295,7 @@ unsigned long long xnarch_get_cpu_time(v
EXPORT_SYMBOL(xnarch_get_cpu_time);
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) && !defined(CONFIG_XENO_OPT_TICKET_LOCK)
void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
{
unsigned int spin_limit;
@@ -311,6 +311,6 @@ void __xnlock_spin(xnlock_t *lock /*, */
} while(atomic_read(&lock->owner) != ~0);
}
EXPORT_SYMBOL(__xnlock_spin);
-#endif /* CONFIG_SMP */
+#endif /* CONFIG_SMP && !CONFIG_XENO_OPT_TICKET_LOCK */
#endif /* !_XENO_ASM_GENERIC_BITS_POD_H */
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 254 bytes --]
^ permalink raw reply [flat|nested] 12+ messages in thread* [Xenomai-core] [PATCH 6/6] Allow xnlock debugging on single-CPU systems
[not found] <47C51A34.203@domain.hid>
` (10 preceding siblings ...)
2008-03-02 11:02 ` [Xenomai-core] [PATCH 5/6] Introduce optional recursive ticket xnlock Jan Kiszka
@ 2008-03-02 11:02 ` Jan Kiszka
11 siblings, 0 replies; 12+ messages in thread
From: Jan Kiszka @ 2008-03-02 11:02 UTC (permalink / raw)
To: Xenomai-core
[-- Attachment #1.1: Type: text/plain, Size: 107 bytes --]
After all the refactoring, arming xnlock_dbg services also for
!CONFIG_SMP setups was straightforward.
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: xnlock-debugging-for-up.patch --]
[-- Type: text/x-patch; name="xnlock-debugging-for-up.patch", Size: 7139 bytes --]
---
include/asm-generic/bits/pod.h | 4 ++--
include/asm-generic/system.h | 31 +++++++++++++++----------------
ksrc/nucleus/intr.c | 4 ++--
ksrc/nucleus/module.c | 12 ++++++------
ksrc/nucleus/pod.c | 4 ++--
ksrc/skins/posix/signal.c | 2 +-
6 files changed, 28 insertions(+), 29 deletions(-)
Index: b/include/asm-generic/bits/pod.h
===================================================================
--- a/include/asm-generic/bits/pod.h
+++ b/include/asm-generic/bits/pod.h
@@ -295,7 +295,7 @@ unsigned long long xnarch_get_cpu_time(v
EXPORT_SYMBOL(xnarch_get_cpu_time);
-#if defined(CONFIG_SMP) && !defined(CONFIG_XENO_OPT_TICKET_LOCK)
+#if (defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)) && !defined(CONFIG_XENO_OPT_TICKET_LOCK)
void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
{
unsigned int spin_limit;
@@ -311,6 +311,6 @@ void __xnlock_spin(xnlock_t *lock /*, */
} while(atomic_read(&lock->owner) != ~0);
}
EXPORT_SYMBOL(__xnlock_spin);
-#endif /* CONFIG_SMP && !CONFIG_XENO_OPT_TICKET_LOCK */
+#endif /* (CONFIG_SMP || XENO_DEBUG(NUCLEUS)) && !CONFIG_XENO_OPT_TICKET_LOCK */
#endif /* !_XENO_ASM_GENERIC_BITS_POD_H */
Index: b/include/asm-generic/system.h
===================================================================
--- a/include/asm-generic/system.h
+++ b/include/asm-generic/system.h
@@ -76,11 +76,11 @@ extern "C" {
typedef unsigned long spl_t;
#define splhigh(x) rthal_local_irq_save(x)
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
#define splexit(x) rthal_local_irq_restore((x) & 1)
-#else /* !CONFIG_SMP */
+#else /* !CONFIG_SMP || XENO_DEBUG(NUCLEUS) */
#define splexit(x) rthal_local_irq_restore(x)
-#endif /* !CONFIG_SMP */
+#endif /* !CONFIG_SMP || XENO_DEBUG(NUCLEUS) */
#define splnone() rthal_local_irq_enable()
#define spltest() rthal_local_irq_test()
#define splget(x) rthal_local_irq_flags(x)
@@ -90,7 +90,7 @@ static inline unsigned xnarch_current_cp
return rthal_processor_id();
}
-#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+#if XENO_DEBUG(NUCLEUS)
typedef struct {
@@ -204,7 +204,7 @@ static inline int xnlock_dbg_release(xnl
return 0;
}
-#else /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
+#else /* !XENO_DEBUG(NUCLEUS) */
#ifdef CONFIG_XENO_OPT_TICKET_LOCK
typedef struct {
@@ -237,7 +237,7 @@ static inline int xnlock_dbg_release(xnl
return 0;
}
-#endif /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
+#endif /* !XENO_DEBUG(NUCLEUS) */
#define XNARCH_NR_CPUS RTHAL_NR_CPUS
@@ -321,7 +321,7 @@ static inline int xnarch_setimask (int i
return !!s;
}
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
#define xnlock_get(lock) __xnlock_get(lock XNLOCK_DBG_CONTEXT)
#define xnlock_get_irqsave(lock,x) \
@@ -481,17 +481,12 @@ static inline void xnlock_put_irqrestore
rthal_local_irq_restore(flags & 1);
}
-static inline int xnarch_send_ipi(xnarch_cpumask_t cpumask)
-{
- return rthal_send_ipi(RTHAL_SERVICE_IPI0, cpumask);
-}
-
static inline int xnlock_is_owner(xnlock_t *lock)
{
return xnlock_owner(lock) == xnarch_current_cpu();
}
-#else /* !CONFIG_SMP */
+#else /* !(CONFIG_SMP || XENO_DEBUG(NUCLEUS)) */
#define xnlock_init(lock) do { } while(0)
#define xnlock_get(lock) do { } while(0)
@@ -507,12 +502,16 @@ static inline int xnlock_is_owner(xnlock
#define DEFINE_XNLOCK(lock)
#define DEFINE_PRIVATE_XNLOCK(lock)
-static inline int xnarch_send_ipi (xnarch_cpumask_t cpumask)
+#endif /* !(CONFIG_SMP || XENO_DEBUG(NUCLEUS)) */
+
+static inline int xnarch_send_ipi(xnarch_cpumask_t cpumask)
{
+#ifdef CONFIG_SMP
+ return rthal_send_ipi(RTHAL_SERVICE_IPI0, cpumask);
+#else /* !CONFIG_SMP */
return 0;
-}
-
#endif /* !CONFIG_SMP */
+}
#define xnlock_sync_irq(lock, x) \
do { \
Index: b/ksrc/nucleus/intr.c
===================================================================
--- a/ksrc/nucleus/intr.c
+++ b/ksrc/nucleus/intr.c
@@ -387,7 +387,7 @@ static inline int xnintr_irq_detach(xnin
#else /* !CONFIG_XENO_OPT_SHIRQ */
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
typedef struct xnintr_irq {
DECLARE_XNLOCK(lock);
@@ -395,7 +395,7 @@ typedef struct xnintr_irq {
} ____cacheline_aligned_in_smp xnintr_irq_t;
static xnintr_irq_t xnirqs[XNARCH_NR_IRQS];
-#endif /* CONFIG_SMP */
+#endif /* CONFIG_SMP || XENO_DEBUG(NUCLEUS) */
static inline xnintr_t *xnintr_shirq_first(unsigned irq)
{
Index: b/ksrc/nucleus/module.c
===================================================================
--- a/ksrc/nucleus/module.c
+++ b/ksrc/nucleus/module.c
@@ -664,7 +664,7 @@ void xnpod_discard_tbase_proc(xntbase_t
#endif /* CONFIG_XENO_OPT_STATS */
-#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+#if XENO_DEBUG(NUCLEUS)
xnlockinfo_t xnlock_stats[RTHAL_NR_CPUS];
@@ -712,7 +712,7 @@ static int lock_read_proc(char *page,
EXPORT_SYMBOL(xnlock_stats);
-#endif /* CONFIG_SMP && XENO_DEBUG(NUCLEUS) */
+#endif /* XENO_DEBUG(NUCLEUS) */
static int latency_read_proc(char *page,
char **start,
@@ -1033,9 +1033,9 @@ void xnpod_init_proc(void)
create_proc_entry("timerstat", S_IFDIR, rthal_proc_root);
#endif /* CONFIG_XENO_OPT_STATS */
-#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+#if XENO_DEBUG(NUCLEUS)
add_proc_leaf("lock", &lock_read_proc, NULL, NULL, rthal_proc_root);
-#endif /* CONFIG_SMP && XENO_DEBUG(NUCLEUS) */
+#endif /* XENO_DEBUG(NUCLEUS) */
add_proc_leaf("latency",
&latency_read_proc,
@@ -1087,9 +1087,9 @@ void xnpod_delete_proc(void)
remove_proc_entry("timerstat", rthal_proc_root);
remove_proc_entry("stat", rthal_proc_root);
#endif /* CONFIG_XENO_OPT_STATS */
-#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+#if XENO_DEBUG(NUCLEUS)
remove_proc_entry("lock", rthal_proc_root);
-#endif /* CONFIG_SMP && XENO_DEBUG(NUCLEUS) */
+#endif /* XENO_DEBUG(NUCLEUS) */
}
#ifdef CONFIG_XENO_OPT_PERVASIVE
Index: b/ksrc/nucleus/pod.c
===================================================================
--- a/ksrc/nucleus/pod.c
+++ b/ksrc/nucleus/pod.c
@@ -3472,7 +3472,7 @@ EXPORT_SYMBOL(xnpod_welcome_thread);
EXPORT_SYMBOL(nkpod_struct);
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
EXPORT_SYMBOL(nklock);
-#endif /* CONFIG_SMP */
+#endif /* CONFIG_SMP || XENO_DEBUG(NUCLEUS) */
EXPORT_SYMBOL(nklatency);
Index: b/ksrc/skins/posix/signal.c
===================================================================
--- a/ksrc/skins/posix/signal.c
+++ b/ksrc/skins/posix/signal.c
@@ -64,7 +64,7 @@ typedef void siginfo_handler_t(int, sigi
#define SIGRTMAX 64
static struct sigaction actions[SIGRTMAX];
static pse51_siginfo_t pse51_infos_pool[PSE51_SIGQUEUE_MAX];
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
static xnlock_t pse51_infos_lock = XNARCH_LOCK_UNLOCKED;
#endif
static xnpqueue_t pse51_infos_free_list;
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 254 bytes --]
^ permalink raw reply [flat|nested] 12+ messages in thread